Working linearized Submixes

include/boo/audiodev/IAudioVoice.hpp

@@ -3,6 +3,7 @@
 
 #include <stddef.h>
 #include <stdint.h>
+#include <string.h>
 
 namespace boo
 {
@@ -91,6 +92,23 @@ struct IAudioVoiceCallback
     /** boo calls this on behalf of the audio platform to request more audio
      *  frames from the client */
     virtual size_t supplyAudio(IAudioVoice& voice, size_t frames, int16_t* data)=0;
+
+    /** after resampling, boo calls this for each submix that this voice targets;
+     *  client performs volume processing and bus-routing this way */
+    virtual void routeAudio(size_t frames, double dt, int busId, int16_t* in, int16_t* out)
+    {
+        memmove(out, in, frames * 2);
+    }
+
+    virtual void routeAudio(size_t frames, double dt, int busId, int32_t* in, int32_t* out)
+    {
+        memmove(out, in, frames * 4);
+    }
+
+    virtual void routeAudio(size_t frames, double dt, int busId, float* in, float* out)
+    {
+        memmove(out, in, frames * 4);
+    }
 };
 
 }

include/boo/audiodev/IAudioVoiceEngine.hpp

@@ -33,7 +33,7 @@ struct IAudioVoiceEngine
                                                                 bool dynamicPitch=false)=0;
 
     /** Client calls this to allocate a Submix for gathering audio together for effects processing */
-    virtual std::unique_ptr<IAudioSubmix> allocateNewSubmix(bool mainOut, IAudioSubmixCallback* cb=nullptr)=0;
+    virtual std::unique_ptr<IAudioSubmix> allocateNewSubmix(bool mainOut, IAudioSubmixCallback* cb, int busId)=0;
 
     /** Client may optionally register a 200-virtual-updates each second callback for the stream */
     virtual void register5MsCallback(std::function<void(double dt)>&& callback)=0;

lib/audiodev/AudioSubmix.cpp

@@ -7,17 +7,8 @@
 namespace boo
 {
 
-typedef union
+AudioSubmix::AudioSubmix(BaseAudioVoiceEngine& root, IAudioSubmixCallback* cb, int busId, bool mainOut)
-{
+: m_root(root), m_busId(busId), m_cb(cb), m_mainOut(mainOut)
-    float v[4];
-#if __SSE__
-    __m128 q;
-    __m64 d[2];
-#endif
-} TVectorUnion;
-
-AudioSubmix::AudioSubmix(BaseAudioVoiceEngine& root, IAudioSubmixCallback* cb, bool mainOut)
-: m_root(root), m_cb(cb), m_mainOut(mainOut)
 {
     if (mainOut)
         setSendLevel(&m_root.m_mainSubmix, 1.f, false);
@@ -38,11 +29,13 @@ bool AudioSubmix::_mergeC3(std::list<AudioSubmix*>& output,
 {
     for (auto outerIt = lists.begin() ; outerIt != lists.cend() ; ++outerIt)
     {
+        if (outerIt->empty())
+            continue;
         AudioSubmix* smx = outerIt->front();
         bool found = false;
         for (auto innerIt = lists.begin() ; innerIt != lists.cend() ; ++innerIt)
         {
-            if (outerIt == innerIt)
+            if (innerIt->empty() || outerIt == innerIt)
                 continue;
             if (smx == innerIt->front())
             {
@@ -135,12 +128,23 @@ float* AudioSubmix::_getMergeBufFlt(size_t frames)
 
 size_t AudioSubmix::_pumpAndMix16(size_t frames)
 {
-    size_t chanCount = m_root.m_mixInfo.m_channelMap.m_channelCount;
+    ChannelMap& chMap = m_root.m_mixInfo.m_channelMap;
+    size_t chanCount = chMap.m_channelCount;
 
     if (m_redirect16)
+    {
+        if (m_cb && m_cb->canApplyEffect())
+            m_cb->applyEffect(m_redirect16, frames, chMap, m_root.m_mixInfo.m_sampleRate);
         m_redirect16 += chanCount * frames;
+    }
     else
     {
+        size_t sampleCount = frames * chanCount;
+        if (m_scratch16.size() < sampleCount)
+            m_scratch16.resize(sampleCount);
+        if (m_cb && m_cb->canApplyEffect())
+            m_cb->applyEffect(m_scratch16.data(), frames, chMap, m_root.m_mixInfo.m_sampleRate);
+
         size_t curSlewFrame = m_slewFrames;
         for (auto& smx : m_sendGains)
         {
@@ -184,12 +188,23 @@ size_t AudioSubmix::_pumpAndMix16(size_t frames)
 
 size_t AudioSubmix::_pumpAndMix32(size_t frames)
 {
-    size_t chanCount = m_root.m_mixInfo.m_channelMap.m_channelCount;
+    ChannelMap& chMap = m_root.m_mixInfo.m_channelMap;
+    size_t chanCount = chMap.m_channelCount;
 
     if (m_redirect32)
+    {
+        if (m_cb && m_cb->canApplyEffect())
+            m_cb->applyEffect(m_redirect32, frames, chMap, m_root.m_mixInfo.m_sampleRate);
         m_redirect32 += chanCount * frames;
+    }
     else
     {
+        size_t sampleCount = frames * chanCount;
+        if (m_scratch32.size() < sampleCount)
+            m_scratch32.resize(sampleCount);
+        if (m_cb && m_cb->canApplyEffect())
+            m_cb->applyEffect(m_scratch32.data(), frames, chMap, m_root.m_mixInfo.m_sampleRate);
+
         size_t curSlewFrame = m_slewFrames;
         for (auto& smx : m_sendGains)
         {
@@ -233,12 +248,23 @@ size_t AudioSubmix::_pumpAndMix32(size_t frames)
 
 size_t AudioSubmix::_pumpAndMixFlt(size_t frames)
 {
-    size_t chanCount = m_root.m_mixInfo.m_channelMap.m_channelCount;
+    ChannelMap& chMap = m_root.m_mixInfo.m_channelMap;
+    size_t chanCount = chMap.m_channelCount;
 
     if (m_redirectFlt)
+    {
+        if (m_cb && m_cb->canApplyEffect())
+            m_cb->applyEffect(m_redirectFlt, frames, chMap, m_root.m_mixInfo.m_sampleRate);
         m_redirectFlt += chanCount * frames;
+    }
     else
     {
+        size_t sampleCount = frames * chanCount;
+        if (m_scratchFlt.size() < sampleCount)
+            m_scratchFlt.resize(sampleCount);
+        if (m_cb && m_cb->canApplyEffect())
+            m_cb->applyEffect(m_scratchFlt.data(), frames, chMap, m_root.m_mixInfo.m_sampleRate);
+
         size_t curSlewFrame = m_slewFrames;
         for (auto& smx : m_sendGains)
         {

lib/audiodev/AudioSubmix.hpp

@@ -33,6 +33,7 @@ class AudioSubmix : public IAudioSubmix
 
     /* Mixer-engine relationships */
     BaseAudioVoiceEngine& m_root;
+    int m_busId;
     std::list<AudioSubmix*>::iterator m_parentIt;
     bool m_mainOut;
     bool m_bound = false;
@@ -87,7 +88,7 @@ class AudioSubmix : public IAudioSubmix
 
 public:
     ~AudioSubmix();
-    AudioSubmix(BaseAudioVoiceEngine& root, IAudioSubmixCallback* cb, bool mainOut);
+    AudioSubmix(BaseAudioVoiceEngine& root, IAudioSubmixCallback* cb, int busId, bool mainOut);
 
     void resetSendLevels();
     void setSendLevel(IAudioSubmix* submix, float level, bool slew);

lib/audiodev/AudioVoice.cpp

@@ -122,20 +122,26 @@ size_t AudioVoiceMono::SRCCallback(AudioVoiceMono* ctx, int16_t** data, size_t f
 
 size_t AudioVoiceMono::pumpAndMix16(size_t frames)
 {
-    std::vector<int16_t>& scratch16 = m_root.m_scratch16;
+    std::vector<int16_t>& scratch16Pre = m_root.m_scratch16Pre;
-    if (scratch16.size() < frames)
+    if (scratch16Pre.size() < frames)
-        scratch16.resize(frames);
+        scratch16Pre.resize(frames);
 
-    m_cb->preSupplyAudio(*this, frames / m_sampleRateOut);
+    std::vector<int16_t>& scratch16Post = m_root.m_scratch16Post;
+    if (scratch16Post.size() < frames)
+        scratch16Post.resize(frames);
+
+    double dt = frames / m_sampleRateOut;
+    m_cb->preSupplyAudio(*this, dt);
     _midUpdate();
-    size_t oDone = soxr_output(m_src, scratch16.data(), frames);
+    size_t oDone = soxr_output(m_src, scratch16Pre.data(), frames);
 
     if (oDone)
     {
         for (auto& mtx : m_sendMatrices)
         {
             AudioSubmix& smx = *reinterpret_cast<AudioSubmix*>(mtx.first);
-            mtx.second.mixMonoSampleData(m_root.m_mixInfo, scratch16.data(), smx._getMergeBuf16(oDone), oDone);
+            m_cb->routeAudio(oDone, dt, smx.m_busId, scratch16Pre.data(), scratch16Post.data());
+            mtx.second.mixMonoSampleData(m_root.m_mixInfo, scratch16Post.data(), smx._getMergeBuf16(oDone), oDone);
         }
     }
 
@@ -144,20 +150,26 @@ size_t AudioVoiceMono::pumpAndMix16(size_t frames)
 
 size_t AudioVoiceMono::pumpAndMix32(size_t frames)
 {
-    std::vector<int32_t>& scratch32 = m_root.m_scratch32;
+    std::vector<int32_t>& scratch32Pre = m_root.m_scratch32Pre;
-    if (scratch32.size() < frames)
+    if (scratch32Pre.size() < frames)
-        scratch32.resize(frames);
+        scratch32Pre.resize(frames);
 
-    m_cb->preSupplyAudio(*this, frames / m_sampleRateOut);
+    std::vector<int32_t>& scratch32Post = m_root.m_scratch32Post;
+    if (scratch32Post.size() < frames)
+        scratch32Post.resize(frames);
+
+    double dt = frames / m_sampleRateOut;
+    m_cb->preSupplyAudio(*this, dt);
     _midUpdate();
-    size_t oDone = soxr_output(m_src, scratch32.data(), frames);
+    size_t oDone = soxr_output(m_src, scratch32Pre.data(), frames);
 
     if (oDone)
     {
         for (auto& mtx : m_sendMatrices)
         {
             AudioSubmix& smx = *reinterpret_cast<AudioSubmix*>(mtx.first);
-            mtx.second.mixMonoSampleData(m_root.m_mixInfo, scratch32.data(), smx._getMergeBuf32(oDone), oDone);
+            m_cb->routeAudio(oDone, dt, smx.m_busId, scratch32Pre.data(), scratch32Post.data());
+            mtx.second.mixMonoSampleData(m_root.m_mixInfo, scratch32Post.data(), smx._getMergeBuf32(oDone), oDone);
         }
     }
 
@@ -166,20 +178,26 @@ size_t AudioVoiceMono::pumpAndMix32(size_t frames)
 
 size_t AudioVoiceMono::pumpAndMixFlt(size_t frames)
 {
-    std::vector<float>& scratchFlt = m_root.m_scratchFlt;
+    std::vector<float>& scratchFltPre = m_root.m_scratchFltPre;
-    if (scratchFlt.size() < frames)
+    if (scratchFltPre.size() < frames)
-        scratchFlt.resize(frames + 2);
+        scratchFltPre.resize(frames + 2);
 
-    m_cb->preSupplyAudio(*this, frames / m_sampleRateOut);
+    std::vector<float>& scratchFltPost = m_root.m_scratchFltPost;
+    if (scratchFltPost.size() < frames)
+        scratchFltPost.resize(frames + 2);
+
+    double dt = frames / m_sampleRateOut;
+    m_cb->preSupplyAudio(*this, dt);
     _midUpdate();
-    size_t oDone = soxr_output(m_src, scratchFlt.data(), frames);
+    size_t oDone = soxr_output(m_src, scratchFltPre.data(), frames);
 
     if (oDone)
     {
         for (auto& mtx : m_sendMatrices)
         {
             AudioSubmix& smx = *reinterpret_cast<AudioSubmix*>(mtx.first);
-            mtx.second.mixMonoSampleData(m_root.m_mixInfo, scratchFlt.data(), smx._getMergeBufFlt(oDone), oDone);
+            m_cb->routeAudio(oDone, dt, smx.m_busId, scratchFltPre.data(), scratchFltPost.data());
+            mtx.second.mixMonoSampleData(m_root.m_mixInfo, scratchFltPost.data(), smx._getMergeBufFlt(oDone), oDone);
         }
     }
 
@@ -278,21 +296,28 @@ size_t AudioVoiceStereo::SRCCallback(AudioVoiceStereo* ctx, int16_t** data, size
 
 size_t AudioVoiceStereo::pumpAndMix16(size_t frames)
 {
-    std::vector<int16_t>& scratch16 = m_root.m_scratch16;
     size_t samples = frames * 2;
-    if (scratch16.size() < samples)
-        scratch16.resize(samples);
 
-    m_cb->preSupplyAudio(*this, frames / m_sampleRateOut);
+    std::vector<int16_t>& scratch16Pre = m_root.m_scratch16Pre;
+    if (scratch16Pre.size() < samples)
+        scratch16Pre.resize(samples);
+
+    std::vector<int16_t>& scratch16Post = m_root.m_scratch16Post;
+    if (scratch16Post.size() < samples)
+        scratch16Post.resize(samples);
+
+    double dt = frames / m_sampleRateOut;
+    m_cb->preSupplyAudio(*this, dt);
     _midUpdate();
-    size_t oDone = soxr_output(m_src, scratch16.data(), frames);
+    size_t oDone = soxr_output(m_src, scratch16Pre.data(), frames);
 
     if (oDone)
     {
         for (auto& mtx : m_sendMatrices)
         {
             AudioSubmix& smx = *reinterpret_cast<AudioSubmix*>(mtx.first);
-            mtx.second.mixStereoSampleData(m_root.m_mixInfo, scratch16.data(), smx._getMergeBuf16(oDone), oDone);
+            m_cb->routeAudio(oDone, dt, smx.m_busId, scratch16Pre.data(), scratch16Post.data());
+            mtx.second.mixStereoSampleData(m_root.m_mixInfo, scratch16Post.data(), smx._getMergeBuf16(oDone), oDone);
         }
     }
 
@@ -301,21 +326,28 @@ size_t AudioVoiceStereo::pumpAndMix16(size_t frames)
 
 size_t AudioVoiceStereo::pumpAndMix32(size_t frames)
 {
-    std::vector<int32_t>& scratch32 = m_root.m_scratch32;
     size_t samples = frames * 2;
-    if (scratch32.size() < samples)
-        scratch32.resize(samples);
 
-    m_cb->preSupplyAudio(*this, frames / m_sampleRateOut);
+    std::vector<int32_t>& scratch32Pre = m_root.m_scratch32Pre;
+    if (scratch32Pre.size() < samples)
+        scratch32Pre.resize(samples);
+
+    std::vector<int32_t>& scratch32Post = m_root.m_scratch32Post;
+    if (scratch32Post.size() < samples)
+        scratch32Post.resize(samples);
+
+    double dt = frames / m_sampleRateOut;
+    m_cb->preSupplyAudio(*this, dt);
     _midUpdate();
-    size_t oDone = soxr_output(m_src, scratch32.data(), frames);
+    size_t oDone = soxr_output(m_src, scratch32Pre.data(), frames);
 
     if (oDone)
     {
         for (auto& mtx : m_sendMatrices)
         {
             AudioSubmix& smx = *reinterpret_cast<AudioSubmix*>(mtx.first);
-            mtx.second.mixStereoSampleData(m_root.m_mixInfo, scratch32.data(), smx._getMergeBuf32(oDone), oDone);
+            m_cb->routeAudio(oDone, dt, smx.m_busId, scratch32Pre.data(), scratch32Post.data());
+            mtx.second.mixStereoSampleData(m_root.m_mixInfo, scratch32Post.data(), smx._getMergeBuf32(oDone), oDone);
         }
     }
 
@@ -324,21 +356,28 @@ size_t AudioVoiceStereo::pumpAndMix32(size_t frames)
 
 size_t AudioVoiceStereo::pumpAndMixFlt(size_t frames)
 {
-    std::vector<float>& scratchFlt = m_root.m_scratchFlt;
     size_t samples = frames * 2;
-    if (scratchFlt.size() < samples)
-        scratchFlt.resize(samples + 4);
 
-    m_cb->preSupplyAudio(*this, frames / m_sampleRateOut);
+    std::vector<float>& scratchFltPre = m_root.m_scratchFltPre;
+    if (scratchFltPre.size() < samples)
+        scratchFltPre.resize(samples + 4);
+
+    std::vector<float>& scratchFltPost = m_root.m_scratchFltPost;
+    if (scratchFltPost.size() < samples)
+        scratchFltPost.resize(samples + 4);
+
+    double dt = frames / m_sampleRateOut;
+    m_cb->preSupplyAudio(*this, dt);
     _midUpdate();
-    size_t oDone = soxr_output(m_src, scratchFlt.data(), frames);
+    size_t oDone = soxr_output(m_src, scratchFltPre.data(), frames);
 
     if (oDone)
     {
         for (auto& mtx : m_sendMatrices)
         {
             AudioSubmix& smx = *reinterpret_cast<AudioSubmix*>(mtx.first);
-            mtx.second.mixStereoSampleData(m_root.m_mixInfo, scratchFlt.data(), smx._getMergeBufFlt(oDone), oDone);
+            m_cb->routeAudio(oDone, dt, smx.m_busId, scratchFltPre.data(), scratchFltPost.data());
+            mtx.second.mixStereoSampleData(m_root.m_mixInfo, scratchFltPost.data(), smx._getMergeBufFlt(oDone), oDone);
         }
     }
 

lib/audiodev/AudioVoiceEngine.cpp

@@ -180,9 +180,9 @@ BaseAudioVoiceEngine::allocateNewStereoVoice(double sampleRate,
 }
 
 std::unique_ptr<IAudioSubmix>
-BaseAudioVoiceEngine::allocateNewSubmix(bool mainOut, IAudioSubmixCallback* cb)
+BaseAudioVoiceEngine::allocateNewSubmix(bool mainOut, IAudioSubmixCallback* cb, int busId)
 {
-    std::unique_ptr<IAudioSubmix> ret = std::make_unique<AudioSubmix>(*this, cb, mainOut);
+    std::unique_ptr<IAudioSubmix> ret = std::make_unique<AudioSubmix>(*this, cb, busId, mainOut);
     AudioSubmix* retIntern = static_cast<AudioSubmix*>(ret.get());
     retIntern->bindSubmix(m_activeSubmixes.insert(m_activeSubmixes.end(), retIntern));
     return ret;

lib/audiodev/AudioVoiceEngine.hpp

@@ -36,9 +36,12 @@ protected:
 
     /* Shared scratch buffers for accumulating audio data for resampling */
     std::vector<int16_t> m_scratchIn;
-    std::vector<int16_t> m_scratch16;
+    std::vector<int16_t> m_scratch16Pre;
-    std::vector<int32_t> m_scratch32;
+    std::vector<int32_t> m_scratch32Pre;
-    std::vector<float> m_scratchFlt;
+    std::vector<float> m_scratchFltPre;
+    std::vector<int16_t> m_scratch16Post;
+    std::vector<int32_t> m_scratch32Post;
+    std::vector<float> m_scratchFltPost;
 
     AudioSubmix m_mainSubmix;
     std::list<AudioSubmix*> m_linearizedSubmixes;
@@ -52,7 +55,7 @@ protected:
     void _unbindFrom(std::list<AudioSubmix*>::iterator it);
 
 public:
-    BaseAudioVoiceEngine() : m_mainSubmix(*this, nullptr, false) {}
+    BaseAudioVoiceEngine() : m_mainSubmix(*this, nullptr, -1, false) {}
     ~BaseAudioVoiceEngine();
     std::unique_ptr<IAudioVoice> allocateNewMonoVoice(double sampleRate,
                                                       IAudioVoiceCallback* cb,
@@ -62,7 +65,7 @@ public:
                                                         IAudioVoiceCallback* cb,
                                                         bool dynamicPitch=false);
 
-    std::unique_ptr<IAudioSubmix> allocateNewSubmix(bool mainOut, IAudioSubmixCallback* cb);
+    std::unique_ptr<IAudioSubmix> allocateNewSubmix(bool mainOut, IAudioSubmixCallback* cb, int busId);
 
     void register5MsCallback(std::function<void(double dt)>&& callback);