Allow audio AQS buffering to occur during frame idle

include/boo/IWindow.hpp

@@ -13,6 +13,7 @@ namespace boo
 {
 struct IGraphicsCommandQueue;
 struct IGraphicsDataFactory;
+struct IAudioVoiceEngine;
 
 enum class EMouseButton
 {
@@ -294,7 +295,7 @@ public:
     virtual bool clipboardCopy(EClipboardType type, const uint8_t* data, size_t sz)=0;
     virtual std::unique_ptr<uint8_t[]> clipboardPaste(EClipboardType type, size_t& sz)=0;
 
-    virtual void waitForRetrace()=0;
+    virtual void waitForRetrace(IAudioVoiceEngine* voxEngine=nullptr)=0;
     
     virtual uintptr_t getPlatformHandle() const=0;
     virtual void _incomingEvent(void* event) {(void)event;}

include/boo/audiodev/IAudioVoiceEngine.hpp

@@ -9,6 +9,19 @@
 
 namespace boo
 {
+struct IAudioVoiceEngine;
+
+/** Time-sensitive event callback for synchronizing the client with rendered audio waveform */
+struct IAudioVoiceEngineCallback
+{
+    /** All mixing occurs in virtual 5ms intervals;
+     *  this is called at the start of each interval for all mixable entities */
+    virtual void on5MsInterval(IAudioVoiceEngine& engine, double dt) {}
+
+    /** When a pumping cycle is complete this is called to allow the client to
+     *  perform periodic cleanup tasks */
+    virtual void onPumpCycleComplete(IAudioVoiceEngine& engine) {}
+};
 
 /** Mixing and sample-rate-conversion system. Allocates voices and mixes them
  *  before sending the final samples to an OS-supplied audio-queue */
@@ -35,11 +48,8 @@ struct IAudioVoiceEngine
     /** Client calls this to allocate a Submix for gathering audio together for effects processing */
     virtual std::unique_ptr<IAudioSubmix> allocateNewSubmix(bool mainOut, IAudioSubmixCallback* cb, int busId)=0;
 
-    /** Client may optionally register a 200-virtual-updates each second callback for the stream */
+    /** Client can register for key callback events from the mixing engine this way */
-    virtual void register5MsCallback(std::function<void(double dt)>&& callback)=0;
+    virtual void setCallbackInterface(IAudioVoiceEngineCallback* cb)=0;
-
-    /** Unregister callback for stable cleanup */
-    virtual void unregister5MsCallback()=0;
 
     /** Client may use this to determine current speaker-setup */
     virtual AudioChannelSet getAvailableSet()=0;
@@ -76,6 +86,12 @@ struct IAudioVoiceEngine
 
     /** Get canonical count of frames for each 5ms output block */
     virtual size_t get5MsFrames() const=0;
+
+    /** IWindow::waitForRetrace() enter - for platforms that spend v-sync waits synchronously pumping audio */
+    virtual void _pumpAndMixVoicesRetrace() { pumpAndMixVoices(); }
+
+    /** IWindow::waitForRetrace() break - signal retrace event to break out of pumping cycles */
+    virtual void _retraceBreak() {}
 };
 
 /** Construct host platform's voice engine */

lib/audiodev/AQS.cpp

@@ -47,7 +47,8 @@ struct AQSAudioVoiceEngine : BaseAudioVoiceEngine
     std::mutex m_engineMutex;
     std::condition_variable m_engineEnterCv;
     std::condition_variable m_engineLeaveCv;
-    bool m_cbWaiting = false;
+    bool m_inRetrace = false;
+    bool m_inCb = false;
     bool m_cbRunning = true;
 
     static void Callback(AQSAudioVoiceEngine* engine, AudioQueueRef inAQ, AudioQueueBufferRef inBuffer)
@@ -56,25 +57,30 @@ struct AQSAudioVoiceEngine : BaseAudioVoiceEngine
             return;
 
         std::unique_lock<std::mutex> lk(engine->m_engineMutex);
-        engine->m_cbWaiting = true;
+        engine->m_inCb = true;
-        if (engine->m_engineEnterCv.wait_for(lk,
+        if (!engine->m_inRetrace)
-        std::chrono::nanoseconds(engine->m_mixInfo.m_periodFrames * 750000000 /
-                                 size_t(engine->m_mixInfo.m_sampleRate))) == std::cv_status::timeout)
         {
-            inBuffer->mAudioDataByteSize = engine->m_frameBytes;
+            if (engine->m_engineEnterCv.wait_for(lk,
-            memset(inBuffer->mAudioData, 0, engine->m_frameBytes);
+            std::chrono::nanoseconds(engine->m_mixInfo.m_periodFrames * 1000000000 /
-            AudioQueueEnqueueBuffer(inAQ, inBuffer, 0, nullptr);
+                                     size_t(engine->m_mixInfo.m_sampleRate))) == std::cv_status::timeout ||
-            engine->m_cbWaiting = false;
+                                     !engine->m_inRetrace)
-            engine->m_engineLeaveCv.notify_one();
+            {
-            return;
+                inBuffer->mAudioDataByteSize = engine->m_frameBytes;
+                memset(inBuffer->mAudioData, 0, engine->m_frameBytes);
+                AudioQueueEnqueueBuffer(inAQ, inBuffer, 0, nullptr);
+                engine->m_engineLeaveCv.notify_one();
+                engine->m_inCb = false;
+                return;
+            }
         }
-        engine->m_cbWaiting = false;
 
-        engine->_pumpAndMixVoices(engine->m_mixInfo.m_periodFrames, reinterpret_cast<float*>(inBuffer->mAudioData));
+        engine->_pumpAndMixVoices(engine->m_mixInfo.m_periodFrames,
+                                  reinterpret_cast<float*>(inBuffer->mAudioData));
         inBuffer->mAudioDataByteSize = engine->m_frameBytes;
         AudioQueueEnqueueBuffer(inAQ, inBuffer, 0, nullptr);
 
         engine->m_engineLeaveCv.notify_one();
+        engine->m_inCb = false;
     }
 
     static void DummyCallback(AQSAudioVoiceEngine* engine, AudioQueueRef inAQ, AudioQueueBufferRef inBuffer) {}
@@ -698,7 +704,7 @@ struct AQSAudioVoiceEngine : BaseAudioVoiceEngine
         while (chMapOut.m_channelCount < chCount)
             chMapOut.m_channels[chMapOut.m_channelCount++] = AudioChannel::Unknown;
 
-        m_mixInfo.m_periodFrames = 2400 * size_t(actualSampleRate) / 48000;
+        m_mixInfo.m_periodFrames = 1200 * size_t(actualSampleRate) / 48000;
         for (int i=0 ; i<3 ; ++i)
             if (AudioQueueAllocateBuffer(m_queue, m_mixInfo.m_periodFrames * chCount * 4, &m_buffers[i]))
             {
@@ -726,22 +732,50 @@ struct AQSAudioVoiceEngine : BaseAudioVoiceEngine
     ~AQSAudioVoiceEngine()
     {
         m_cbRunning = false;
-        if (m_cbWaiting)
+        if (m_inCb)
             m_engineEnterCv.notify_one();
         AudioQueueDispose(m_queue, true);
         if (m_midiClient)
             MIDIClientDispose(m_midiClient);
     }
 
+    /* This is temperamental for AudioQueueServices
+     * (which has unpredictable buffering windows).
+     * _pumpAndMixVoicesRetrace() is highly recommended. */
     void pumpAndMixVoices()
     {
         std::unique_lock<std::mutex> lk(m_engineMutex);
-        if (m_cbWaiting)
+        if (m_inCb)
         {
+            /* Wake up callback */
             m_engineEnterCv.notify_one();
+            /* Wait for callback completion */
             m_engineLeaveCv.wait(lk);
         }
     }
+
+    void _pumpAndMixVoicesRetrace()
+    {
+        std::unique_lock<std::mutex> lk(m_engineMutex);
+        m_inRetrace = true;
+        while (m_inRetrace)
+        {
+            if (m_inCb) /* Wake up callback */
+                m_engineEnterCv.notify_one();
+            /* Wait for callback completion */
+            m_engineLeaveCv.wait(lk);
+        }
+    }
+
+    void _retraceBreak()
+    {
+        std::unique_lock<std::mutex> lk(m_engineMutex);
+        m_inRetrace = false;
+        if (m_inCb) /* Break out of callback */
+            m_engineEnterCv.notify_one();
+        else /* Break out of client */
+            m_engineLeaveCv.notify_one();
+    }
 };
 
 std::unique_ptr<IAudioVoiceEngine> NewAudioVoiceEngine()

lib/audiodev/AudioVoiceEngine.cpp

@@ -30,14 +30,14 @@ void BaseAudioVoiceEngine::_pumpAndMixVoices(size_t frames, int16_t* dataOut)
         if (remFrames < m_5msFrames)
         {
             thisFrames = remFrames;
-            if (m_5msCallback)
+            if (m_engineCallback)
-                m_5msCallback(thisFrames / double(m_5msFrames) * 5.0 / 1000.0);
+                m_engineCallback->on5MsInterval(*this, thisFrames / double(m_5msFrames) * 5.0 / 1000.0);
         }
         else
         {
             thisFrames = m_5msFrames;
-            if (m_5msCallback)
+            if (m_engineCallback)
-                m_5msCallback(5.0 / 1000.0);
+                m_engineCallback->on5MsInterval(*this, 5.0 / 1000.0);
         }
 
         for (auto it = m_linearizedSubmixes.rbegin() ; it != m_linearizedSubmixes.rend() ; ++it)
@@ -57,6 +57,9 @@ void BaseAudioVoiceEngine::_pumpAndMixVoices(size_t frames, int16_t* dataOut)
         remFrames -= thisFrames;
         dataOut += sampleCount;
     }
+
+    if (m_engineCallback)
+        m_engineCallback->onPumpCycleComplete(*this);
 }
 
 void BaseAudioVoiceEngine::_pumpAndMixVoices(size_t frames, int32_t* dataOut)
@@ -77,14 +80,14 @@ void BaseAudioVoiceEngine::_pumpAndMixVoices(size_t frames, int32_t* dataOut)
         if (remFrames < m_5msFrames)
         {
             thisFrames = remFrames;
-            if (m_5msCallback)
+            if (m_engineCallback)
-                m_5msCallback(thisFrames / double(m_5msFrames) * 5.0 / 1000.0);
+                m_engineCallback->on5MsInterval(*this, thisFrames / double(m_5msFrames) * 5.0 / 1000.0);
         }
         else
         {
             thisFrames = m_5msFrames;
-            if (m_5msCallback)
+            if (m_engineCallback)
-                m_5msCallback(5.0 / 1000.0);
+                m_engineCallback->on5MsInterval(*this, 5.0 / 1000.0);
         }
 
         for (auto it = m_linearizedSubmixes.rbegin() ; it != m_linearizedSubmixes.rend() ; ++it)
@@ -104,6 +107,9 @@ void BaseAudioVoiceEngine::_pumpAndMixVoices(size_t frames, int32_t* dataOut)
         remFrames -= thisFrames;
         dataOut += sampleCount;
     }
+
+    if (m_engineCallback)
+        m_engineCallback->onPumpCycleComplete(*this);
 }
 
 void BaseAudioVoiceEngine::_pumpAndMixVoices(size_t frames, float* dataOut)
@@ -124,14 +130,14 @@ void BaseAudioVoiceEngine::_pumpAndMixVoices(size_t frames, float* dataOut)
         if (remFrames < m_5msFrames)
         {
             thisFrames = remFrames;
-            if (m_5msCallback)
+            if (m_engineCallback)
-                m_5msCallback(thisFrames / double(m_5msFrames) * 5.0 / 1000.0);
+                m_engineCallback->on5MsInterval(*this, thisFrames / double(m_5msFrames) * 5.0 / 1000.0);
         }
         else
         {
             thisFrames = m_5msFrames;
-            if (m_5msCallback)
+            if (m_engineCallback)
-                m_5msCallback(5.0 / 1000.0);
+                m_engineCallback->on5MsInterval(*this, 5.0 / 1000.0);
         }
 
         for (auto it = m_linearizedSubmixes.rbegin() ; it != m_linearizedSubmixes.rend() ; ++it)
@@ -151,6 +157,9 @@ void BaseAudioVoiceEngine::_pumpAndMixVoices(size_t frames, float* dataOut)
         remFrames -= thisFrames;
         dataOut += sampleCount;
     }
+
+    if (m_engineCallback)
+        m_engineCallback->onPumpCycleComplete(*this);
 }
 
 void BaseAudioVoiceEngine::_unbindFrom(std::list<AudioVoice*>::iterator it)
@@ -197,14 +206,9 @@ BaseAudioVoiceEngine::allocateNewSubmix(bool mainOut, IAudioSubmixCallback* cb,
     return ret;
 }
 
-void BaseAudioVoiceEngine::register5MsCallback(std::function<void(double dt)>&& callback)
+void BaseAudioVoiceEngine::setCallbackInterface(IAudioVoiceEngineCallback* cb)
 {
-    m_5msCallback = std::move(callback);
+    m_engineCallback = cb;
-}
-
-void BaseAudioVoiceEngine::unregister5MsCallback()
-{
-    m_5msCallback = {};
 }
 
 void BaseAudioVoiceEngine::setVolume(float vol)

lib/audiodev/AudioVoiceEngine.hpp

@@ -33,7 +33,7 @@ protected:
     std::list<AudioVoice*> m_activeVoices;
     std::list<AudioSubmix*> m_activeSubmixes;
     size_t m_5msFrames = 0;
-    std::function<void(double dt)> m_5msCallback;
+    IAudioVoiceEngineCallback* m_engineCallback = nullptr;
 
     /* Shared scratch buffers for accumulating audio data for resampling */
     std::vector<int16_t> m_scratchIn;
@@ -68,8 +68,7 @@ public:
 
     std::unique_ptr<IAudioSubmix> allocateNewSubmix(bool mainOut, IAudioSubmixCallback* cb, int busId);
 
-    void register5MsCallback(std::function<void(double dt)>&& callback);
+    void setCallbackInterface(IAudioVoiceEngineCallback* cb);
-    void unregister5MsCallback();
 
     void setVolume(float vol);
     const AudioVoiceEngineMixInfo& mixInfo() const;

lib/mac/WindowCocoa.mm

@@ -7,6 +7,7 @@
 #include "boo/IApplication.hpp"
 #include "boo/IWindow.hpp"
 #include "boo/IGraphicsContext.hpp"
+#include "boo/audiodev/IAudioVoiceEngine.hpp"
 
 #include <LogVisor/LogVisor.hpp>
 
@@ -112,6 +113,7 @@ protected:
 
     std::mutex m_dlmt;
     std::condition_variable m_dlcv;
+    IAudioVoiceEngine* m_voxEngine = nullptr;
 
     static CVReturn DLCallback(CVDisplayLinkRef displayLink,
                                const CVTimeStamp * inNow,
@@ -120,7 +122,11 @@ protected:
                                CVOptionFlags * flagsOut,
                                GraphicsContextCocoa* ctx)
     {
-        ctx->m_dlcv.notify_one();
+        std::unique_lock<std::mutex> lk(ctx->m_dlmt);
+        if (ctx->m_voxEngine)
+            ctx->m_voxEngine->_retraceBreak();
+        else
+            ctx->m_dlcv.notify_one();
         return kCVReturnSuccess;
     }
 
@@ -135,10 +141,21 @@ public:
     }
 
     IWindowCallback* m_callback = nullptr;
-    void waitForRetrace()
+    void waitForRetrace(IAudioVoiceEngine* voxEngine)
     {
         std::unique_lock<std::mutex> lk(m_dlmt);
-        m_dlcv.wait(lk);
+        if (voxEngine)
+        {
+            m_voxEngine = voxEngine;
+            lk.unlock();
+            voxEngine->_pumpAndMixVoicesRetrace();
+            lk.lock();
+            m_voxEngine = nullptr;
+        }
+        else
+        {
+            m_dlcv.wait(lk);
+        }
     }
     virtual BooCocoaResponder* responder() const=0;
 };
@@ -1500,9 +1517,9 @@ public:
         });
     }
 
-    void waitForRetrace()
+    void waitForRetrace(IAudioVoiceEngine* voxEngine)
     {
-        static_cast<GraphicsContextCocoa*>(m_gfxCtx)->waitForRetrace();
+        static_cast<GraphicsContextCocoa*>(m_gfxCtx)->waitForRetrace(voxEngine);
     }
 
     uintptr_t getPlatformHandle() const

lib/win/WindowWin32.cpp

@@ -9,6 +9,7 @@
 #include "boo/graphicsdev/GL.hpp"
 #include "boo/graphicsdev/glew.h"
 #include "boo/graphicsdev/wglew.h"
+#include "boo/audiodev/IAudioVoiceEngine.hpp"
 
 #if BOO_HAS_VULKAN
 #include "boo/graphicsdev/Vulkan.hpp"
@@ -1215,8 +1216,10 @@ public:
         return std::unique_ptr<uint8_t[]>();
     }
 
-    void waitForRetrace()
+    void waitForRetrace(IAudioVoiceEngine* engine)
     {
+        if (engine)
+            engine->pumpAndMixVoices();
         m_gfxCtx->m_output->WaitForVBlank();
     }
 

lib/x11/WindowWayland.cpp

@@ -1,5 +1,6 @@
 #include "boo/IWindow.hpp"
 #include "boo/IGraphicsContext.hpp"
+#include "boo/audiodev/IAudioVoiceEngine.hpp"
 
 #include <X11/Xlib.h>
 #undef None
@@ -199,8 +200,10 @@ struct WindowWayland : IWindow
         return std::unique_ptr<uint8_t[]>();
     }
 
-    void waitForRetrace()
+    void waitForRetrace(IAudioVoiceEngine* engine)
     {
+        if (engine)
+            engine->pumpAndMixVoices();
     }
 
     uintptr_t getPlatformHandle() const

lib/x11/WindowXlib.cpp

@@ -2,6 +2,7 @@
 #include "boo/IGraphicsContext.hpp"
 #include "boo/IApplication.hpp"
 #include "boo/graphicsdev/GL.hpp"
+#include "boo/audiodev/IAudioVoiceEngine.hpp"
 
 #if BOO_HAS_VULKAN
 #include "boo/graphicsdev/Vulkan.hpp"
@@ -1418,8 +1419,10 @@ public:
         XSendEvent(m_xDisp, se->requestor, False, 0, &reply);
     }
 
-    void waitForRetrace()
+    void waitForRetrace(IAudioVoiceEngine* engine)
     {
+        if (engine)
+            engine->pumpAndMixVoices();
         std::unique_lock<std::mutex> lk(m_gfxCtx->m_vsyncmt);
         m_gfxCtx->m_vsynccv.wait(lk);
     }