Proton/media-converter/src/audioconvbin.rs

/*
 * Copyright (c) 2020 Valve Corporation
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without modification,
 * are permitted provided that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright notice, this
 * list of conditions and the following disclaimer.
 *
 * 2. Redistributions in binary form must reproduce the above copyright notice,
 * this list of conditions and the following disclaimer in the documentation and/or
 * other materials provided with the distribution.
 *
 * 3. Neither the name of the copyright holder nor the names of its contributors
 * may be used to endorse or promote products derived from this software without
 * specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

use glib;
use glib::subclass;
use glib::subclass::prelude::*;

use gst;
use gst::prelude::*;
use gst::subclass::prelude::*;
use gst::EventView;

use once_cell::sync::Lazy;

/* Opus is a great fit for our usecase except for one problem: it only supports a few samplerates.
 * Notably it doesn't support 44100 Hz, which is a very frequently used samplerate. This bin
 * provides a capssetter element which will override the rate we get from Opus with the rate the
 * application requested. Similarly, on the transcoder side, we just encode the audio as if it were
 * at 48 kHz, even if it is actually at 44.1 kHz.
 *
 * The downside to this is a small decrease in audio quality. If Opus is most responsive between 20
 * Hz and 20 kHz, then when 44.1 audio is converted to 48, we'll gain noise between 18-20 Hz
 * (although WMA probably already filtered that out) and start to lose audio above 18,375 kHz. This
 * is at the very edge of human hearing, so we're unlikely to lose any noticeable quality.
 *
 * Resampling is an option, but has some problems. It's significantly more complicated, and more
 * CPU-intensive.  Also, XAudio2 buffers can be started and ended at arbitrary points, so if we
 * start moving audio data from one buffer into another due to resampling, it may result in audible
 * artifacts. I think just encoding at the wrong rate is the best compromise. If the application
 * actually cared about audio quality, they probably would not have used WMA in the first place.
 */

static CAT: Lazy<gst::DebugCategory> = Lazy::new(|| {
    gst::DebugCategory::new(
        "protonaudioconverterbin",
        gst::DebugColorFlags::empty(),
        Some("Proton audio converter bin"))
});

struct AudioConvBin {
    audioconv: gst::Element,
    opusdec: gst::Element,
    capssetter: gst::Element,
    srcpad: gst::GhostPad,
    sinkpad: gst::GhostPad,
}

impl ObjectSubclass for AudioConvBin {
    const NAME: &'static str = "ProtonAudioConverterBin";

    type ParentType = gst::Bin;
    type Instance = gst::subclass::ElementInstanceStruct<Self>;
    type Class = subclass::simple::ClassStruct<Self>;

    glib_object_subclass!();

    fn with_class(klass: &subclass::simple::ClassStruct<Self>) -> Self {

        let templ = klass.get_pad_template("src").unwrap();
        let srcpad = gst::GhostPad::builder_with_template(&templ, Some("src")).build();

        let templ = klass.get_pad_template("sink").unwrap();
        let sinkpad = gst::GhostPad::builder_with_template(&templ, Some("sink"))
            .event_function(|pad, parent, event| {
                AudioConvBin::catch_panic_pad_function(
                    parent,
                    || false,
                    |audioconvbin, element| audioconvbin.sink_event(pad, element, event)
                )
            }).build();

        let audioconv = gst::ElementFactory::make("protonaudioconverter", None).unwrap();
        let opusdec = gst::ElementFactory::make("opusdec", None).unwrap();
        let capssetter = gst::ElementFactory::make("capssetter", None).unwrap();

        AudioConvBin {
            audioconv,
            opusdec,
            capssetter,
            srcpad,
            sinkpad,
        }
    }

    fn class_init(klass: &mut subclass::simple::ClassStruct<Self>) {
        klass.set_metadata("Proton audio converter with rate fixup",
                           "Codec/Parser",
                           "Converts audio for Proton, fixing up samplerates",
                           "Andrew Eikum <aeikum@codeweavers.com>");

        let mut caps = gst::Caps::new_empty();
        {
            let caps = caps.get_mut().unwrap();
            caps.append(gst::Caps::builder("audio/x-wma").build());
        }
        let sink_pad_template = gst::PadTemplate::new(
            "sink",
            gst::PadDirection::Sink,
            gst::PadPresence::Always,
            &caps).unwrap();
        klass.add_pad_template(sink_pad_template);

        let caps = gst::Caps::builder("audio/x-raw").build();
        let src_pad_template = gst::PadTemplate::new(
            "src",
            gst::PadDirection::Src,
            gst::PadPresence::Always,
            &caps).unwrap();
        klass.add_pad_template(src_pad_template);
    }
}

impl ObjectImpl for AudioConvBin {
    glib_object_impl!();

    fn constructed(&self, obj: &glib::Object) {
        self.parent_constructed(obj);

        let bin = obj.downcast_ref::<gst::Bin>().unwrap();

        bin.add(&self.audioconv).unwrap();
        bin.add(&self.opusdec).unwrap();
        bin.add(&self.capssetter).unwrap();

        self.audioconv.link(&self.opusdec).unwrap();
        self.opusdec.link(&self.capssetter).unwrap();

        self.sinkpad
            .set_target(Some(&self.audioconv.get_static_pad("sink").unwrap()))
            .unwrap();
        self.srcpad
            .set_target(Some(&self.capssetter.get_static_pad("src").unwrap()))
            .unwrap();

        bin.add_pad(&self.sinkpad).unwrap();
        bin.add_pad(&self.srcpad).unwrap();
    }
}

impl BinImpl for AudioConvBin { }

impl ElementImpl for AudioConvBin { }

impl AudioConvBin {
    fn sink_event(
        &self,
        pad: &gst::GhostPad,
        element: &gst::Element,
        event: gst::Event
    ) -> bool {
        match event.view() {
            EventView::Caps(event_caps) => {
                /* set up capssetter with this rate */
                if let Some(s) = event_caps.get_caps().get_structure(0) {

                    if let Ok(override_rate) = s.get_some::<i32>("rate") {

                        let mut rate_caps = gst::Caps::new_empty();
                        {
                            let rate_caps = rate_caps.get_mut().unwrap();
                            let s = gst::Structure::builder("audio/x-raw")
                                .field("rate", &override_rate)
                                .build();
                            rate_caps.append_structure(s);
                        }
                        self.capssetter.set_property("caps",
                                                     &rate_caps).unwrap();
                    }else{
                        gst_warning!(CAT, "event has no rate");
                    }
                } else {
                    gst_warning!(CAT, "event has no structure");
                }

                /* forward on to the real pad */
                self.audioconv.get_static_pad("sink").unwrap()
                    .send_event(event)
            },
            _ => pad.event_default(Some(element), event)
        }
    }
}

pub fn register(plugin: &gst::Plugin) -> Result<(), glib::BoolError> {
    gst::Element::register(
        Some(plugin),
        "protonaudioconverterbin",
        gst::Rank::Marginal + 1,
        AudioConvBin::get_type()
    )
}