Add support for MemorySanitizer instrumentation

lib/Common.hpp

@@ -74,4 +74,28 @@ static inline uint32_t flp2(uint32_t x) {
   return x - (x >> 1);
 }
 
+/* When instrumenting with MemorySanitizer, external libraries
+ * (particularly the OpenGL implementation) will report tons of false
+ * positives. The BOO_MSAN_NO_INTERCEPT macro declares a RAII object
+ * to temporarily suspend memory tracking so external calls can be made.
+ */
+#if defined(__has_feature)
+#if __has_feature(memory_sanitizer)
+#define BOO_MSAN 1
+#include <sanitizer/msan_interface.h>
+struct InterceptorScope {
+  InterceptorScope() { __msan_scoped_disable_interceptor_checks(); }
+  ~InterceptorScope() { __msan_scoped_enable_interceptor_checks(); }
+};
+#define BOO_MSAN_NO_INTERCEPT InterceptorScope _no_intercept;
+#define BOO_MSAN_UNPOISON(data, length) __msan_unpoison(data, length)
+#endif
+#endif
+#ifndef BOO_MSAN_NO_INTERCEPT
+#define BOO_MSAN_NO_INTERCEPT
+#endif
+#ifndef BOO_MSAN_UNPOISON
+#define BOO_MSAN_UNPOISON(data, length)
+#endif
+
 } // namespace boo

lib/audiodev/PulseAudio.cpp

@@ -336,7 +336,7 @@ struct PulseAudioVoiceEngine : LinuxMidi {
       return;
     }
 
-    void* data;
+    void* data = nullptr;
     size_t periodSz = m_mixInfo.m_periodFrames * frameSz;
     size_t nbytes = writablePeriods * periodSz;
     if (pa_stream_begin_write(m_stream, &data, &nbytes)) {

lib/graphicsdev/GL.cpp

@@ -80,7 +80,7 @@ class GLDataFactoryImpl : public GLDataFactory, public GraphicsDataFactoryHead {
 
     if (GLEW_ARB_tessellation_shader) {
       m_hasTessellation = true;
-      GLint maxPVerts;
+      GLint maxPVerts = 0;
       glGetIntegerv(GL_MAX_PATCH_VERTICES, &maxPVerts);
       m_maxPatchSize = uint32_t(maxPVerts);
     }
@@ -215,6 +215,7 @@ public:
 
 ObjToken<IGraphicsBufferS> GLDataFactory::Context::newStaticBuffer(BufferUse use, const void* data, size_t stride,
                                                                    size_t count) {
+  BOO_MSAN_NO_INTERCEPT
   return {new GLGraphicsBufferS(m_data, use, data, stride * count)};
 }
 
@@ -604,6 +605,7 @@ ObjToken<ITextureS> GLDataFactory::Context::newStaticTexture(size_t width, size_
                                                              TextureFormat fmt, TextureClampMode clampMode,
                                                              const void* data, size_t sz) {
   GLDataFactoryImpl& factory = static_cast<GLDataFactoryImpl&>(m_parent);
+  BOO_MSAN_NO_INTERCEPT
   return {new GLTextureS(m_data, width, height, mips, fmt, clampMode, factory.m_glCtx->m_anisotropy, data, sz)};
 }
 
@@ -612,6 +614,7 @@ ObjToken<ITextureSA> GLDataFactory::Context::newStaticArrayTexture(size_t width,
                                                                    TextureClampMode clampMode, const void* data,
                                                                    size_t sz) {
   GLDataFactoryImpl& factory = static_cast<GLDataFactoryImpl&>(m_parent);
+  BOO_MSAN_NO_INTERCEPT
   return {
       new GLTextureSA(m_data, width, height, layers, mips, fmt, clampMode, factory.m_glCtx->m_anisotropy, data, sz)};
 }
@@ -690,7 +693,7 @@ class GLShaderStage : public GraphicsDataNode<IShaderStage> {
 
     glShaderSource(m_shad, 1, &source, nullptr);
     glCompileShader(m_shad);
-    GLint status;
+    GLint status = GL_FALSE;
     glGetShaderiv(m_shad, GL_COMPILE_STATUS, &status);
     if (status != GL_TRUE) {
       GLint logLen;
@@ -816,7 +819,7 @@ public:
       if (m_evaluation)
         glDetachShader(m_prog, m_evaluation.cast<GLShaderStage>()->getShader());
 
-      GLint status;
+      GLint status = GL_FALSE;
       glGetProgramiv(m_prog, GL_LINK_STATUS, &status);
       if (status != GL_TRUE) {
         GLint logLen;
@@ -910,6 +913,7 @@ ObjToken<IShaderStage> GLDataFactory::Context::newShaderStage(const uint8_t* dat
       Log.report(logvisor::Fatal, "Device does not support tessellation shaders");
   }
 
+  BOO_MSAN_NO_INTERCEPT
   return {new GLShaderStage(m_data, (char*)data, stage)};
 }
 
@@ -927,6 +931,7 @@ ObjToken<IShaderPipeline> GLDataFactory::Context::newShaderPipeline(
                  int(additionalInfo.patchSize));
   }
 
+  BOO_MSAN_NO_INTERCEPT
   return {new GLShaderPipeline(m_data, vertex, fragment, geometry, control, evaluation, vtxFmt, additionalInfo)};
 }
 
@@ -1028,6 +1033,7 @@ void GLDataFactoryImpl::commitTransaction(const FactoryCommitFunc& trans __BooTr
 }
 
 ObjToken<IGraphicsBufferD> GLDataFactoryImpl::newPoolBuffer(BufferUse use, size_t stride, size_t count __BooTraceArgs) {
+  BOO_MSAN_NO_INTERCEPT
   ObjToken<BaseGraphicsPool> pool(new BaseGraphicsPool(*this __BooTraceArgsUse));
   return {new GLGraphicsBufferD<BaseGraphicsPool>(pool, use, stride * count)};
 }
@@ -1197,6 +1203,7 @@ struct GLCommandQueue : IGraphicsCommandQueue {
   }
 
   static void RenderingWorker(GLCommandQueue* self) {
+    BOO_MSAN_NO_INTERCEPT
 #if _WIN32
     std::string thrName = WCSTMBS(APP->getFriendlyName().data()) + " Render";
 #else
@@ -1212,12 +1219,12 @@ struct GLCommandQueue : IGraphicsCommandQueue {
       self->m_parent->postInit();
       glClearColor(0.f, 0.f, 0.f, 0.f);
       if (GLEW_EXT_texture_filter_anisotropic) {
-        GLint maxAniso;
+        GLint maxAniso = 0;
         glGetIntegerv(GL_MAX_TEXTURE_MAX_ANISOTROPY_EXT, &maxAniso);
         self->m_glCtx->m_anisotropy = std::min(uint32_t(maxAniso), self->m_glCtx->m_anisotropy);
         glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAX_ANISOTROPY_EXT, maxAniso);
       }
-      GLint maxSamples;
+      GLint maxSamples = 0;
       glGetIntegerv(GL_MAX_SAMPLES, &maxSamples);
       self->m_glCtx->m_sampleCount =
           flp2(std::min(uint32_t(maxSamples), std::max(uint32_t(1), self->m_glCtx->m_sampleCount)) - 1);
@@ -1570,6 +1577,7 @@ struct GLCommandQueue : IGraphicsCommandQueue {
   }
 
   void execute() {
+    BOO_MSAN_NO_INTERCEPT
     std::unique_lock<std::mutex> lk(m_mt);
     m_completeBuf = m_fillBuf;
     for (int i = 0; i < 3; ++i) {
@@ -1613,11 +1621,13 @@ struct GLCommandQueue : IGraphicsCommandQueue {
 };
 
 ObjToken<IGraphicsBufferD> GLDataFactory::Context::newDynamicBuffer(BufferUse use, size_t stride, size_t count) {
+  BOO_MSAN_NO_INTERCEPT
   return {new GLGraphicsBufferD<BaseGraphicsData>(m_data, use, stride * count)};
 }
 
 ObjToken<ITextureD> GLDataFactory::Context::newDynamicTexture(size_t width, size_t height, TextureFormat fmt,
                                                               TextureClampMode clampMode) {
+  BOO_MSAN_NO_INTERCEPT
   return {new GLTextureD(m_data, width, height, fmt, clampMode)};
 }
 
@@ -1685,6 +1695,7 @@ ObjToken<ITextureR> GLDataFactory::Context::newRenderTexture(size_t width, size_
                                                              size_t colorBindingCount, size_t depthBindingCount) {
   GLDataFactoryImpl& factory = static_cast<GLDataFactoryImpl&>(m_parent);
   GLCommandQueue* q = static_cast<GLCommandQueue*>(factory.m_parent->getCommandQueue());
+  BOO_MSAN_NO_INTERCEPT
   ObjToken<ITextureR> retval(new GLTextureR(m_data, q, width, height, factory.m_glCtx->m_sampleCount,
                                             factory.m_glCtx->m_deepColor ? GL_RGBA16 : GL_RGBA8, clampMode,
                                             colorBindingCount, depthBindingCount));
@@ -1700,6 +1711,7 @@ ObjToken<IShaderDataBinding> GLDataFactory::Context::newShaderDataBinding(
     const bool* depthBind, size_t baseVert, size_t baseInst) {
   GLDataFactoryImpl& factory = static_cast<GLDataFactoryImpl&>(m_parent);
   GLCommandQueue* q = static_cast<GLCommandQueue*>(factory.m_parent->getCommandQueue());
+  BOO_MSAN_NO_INTERCEPT
   ObjToken<GLShaderDataBinding> ret = {new GLShaderDataBinding(m_data, pipeline, vbo, instVbo, ibo, ubufCount, ubufs,
                                                                ubufOffs, ubufSizes, texCount, texs, texBindIdx,
                                                                depthBind, baseVert, baseInst, q)};

lib/graphicsdev/Vulkan.cpp

@@ -147,11 +147,6 @@ static inline void ThrowIfFailed(VkResult res) {
     Log.report(logvisor::Fatal, "%d\n", res);
 }
 
-static inline void ThrowIfFalse(bool res) {
-  if (!res)
-    Log.report(logvisor::Fatal, "operation failed\n", res);
-}
-
 static VKAPI_ATTR VkBool32 VKAPI_CALL dbgFunc(VkDebugReportFlagsEXT msgFlags, VkDebugReportObjectTypeEXT objType,
                                               uint64_t srcObject, size_t location, int32_t msgCode,
                                               const char* pLayerPrefix, const char* pMsg, void* pUserData) {
@@ -1297,7 +1292,7 @@ class VulkanTextureS : public GraphicsDataNode<ITextureS> {
   , m_height(height)
   , m_mips(mips)
   , m_clampMode(clampMode) {
-    VkFormat pfmt;
+    VkFormat pfmt = VK_FORMAT_UNDEFINED;
     switch (fmt) {
     case TextureFormat::RGBA8:
       pfmt = VK_FORMAT_R8G8B8A8_UNORM;
@@ -1466,7 +1461,7 @@ class VulkanTextureSA : public GraphicsDataNode<ITextureSA> {
   , m_layers(layers)
   , m_mips(mips)
   , m_clampMode(clampMode) {
-    VkFormat pfmt;
+    VkFormat pfmt = VK_FORMAT_UNDEFINED;
     switch (fmt) {
     case TextureFormat::RGBA8:
       pfmt = VK_FORMAT_R8G8B8A8_UNORM;
@@ -1619,7 +1614,7 @@ class VulkanTextureD : public GraphicsDataNode<ITextureD> {
   VulkanTextureD(const boo::ObjToken<BaseGraphicsData>& parent, VulkanCommandQueue* q, size_t width, size_t height,
                  TextureFormat fmt, TextureClampMode clampMode)
   : GraphicsDataNode<ITextureD>(parent), m_width(width), m_height(height), m_fmt(fmt), m_clampMode(clampMode), m_q(q) {
-    VkFormat pfmt;
+    VkFormat pfmt = VK_FORMAT_UNDEFINED;
     switch (fmt) {
     case TextureFormat::RGBA8:
       pfmt = VK_FORMAT_R8G8B8A8_UNORM;
@@ -1731,6 +1726,7 @@ public:
 class VulkanTextureR : public GraphicsDataNode<ITextureR> {
   friend class VulkanDataFactory;
   friend struct VulkanCommandQueue;
+  VulkanCommandQueue* m_q;
   size_t m_width = 0;
   size_t m_height = 0;
   VkSampleCountFlags m_samplesColor, m_samplesDepth;
@@ -1856,7 +1852,6 @@ class VulkanTextureR : public GraphicsDataNode<ITextureR> {
     m_passBeginInfo.pClearValues = nullptr;
   }
 
-  VulkanCommandQueue* m_q;
   VulkanTextureR(const boo::ObjToken<BaseGraphicsData>& parent, VulkanCommandQueue* q, size_t width, size_t height,
                  TextureClampMode clampMode, size_t colorBindCount, size_t depthBindCount);
 

lib/x11/ApplicationXlib.hpp

@@ -4,6 +4,7 @@
 
 #include "boo/IApplication.hpp"
 #include "boo/graphicsdev/GL.hpp"
+#include "../Common.hpp"
 
 #include <X11/Xlib.h>
 #include <X11/XKBlib.h>
@@ -148,7 +149,9 @@ class ApplicationXlib final : public IApplication {
 
   /* DBus single-instance */
   bool m_singleInstance;
+#ifndef BOO_MSAN
   DBusConnection* m_dbus = nullptr;
+#endif
 
   /* All windows */
   std::unordered_map<Window, std::weak_ptr<IWindow>> m_windows;
@@ -239,6 +242,7 @@ public:
 #endif
       Log.report(logvisor::Info, "using OpenGL renderer");
 
+#ifndef BOO_MSAN
     /* DBus single instance registration */
     bool isFirst;
     m_dbus = RegisterDBus(uniqueName.data(), isFirst);
@@ -272,6 +276,7 @@ public:
         dbus_connection_flush(m_dbus);
       }
     }
+#endif
 
     if (!XInitThreads()) {
       Log.report(logvisor::Fatal, "X doesn't support multithreading");
@@ -353,7 +358,11 @@ public:
 
     /* Get file descriptors of xcb and dbus interfaces */
     m_x11Fd = ConnectionNumber(m_xDisp);
+#ifndef BOO_MSAN
     dbus_connection_get_unix_fd(m_dbus, &m_dbusFd);
+#else
+    m_dbusFd = 0;
+#endif
     m_maxFd = MAX(m_x11Fd, m_dbusFd);
 
     XFlush(m_xDisp);
@@ -433,7 +442,7 @@ public:
 
         XLockDisplay(m_xDisp);
         while (XPending(m_xDisp)) {
-          XEvent event;
+          XEvent event = {};
           XNextEvent(m_xDisp, &event);
           if (XFilterEvent(&event, None))
             continue;
@@ -455,6 +464,7 @@ public:
           break;
       }
 
+#ifndef BOO_MSAN
       if (FD_ISSET(m_dbusFd, &fds)) {
         DBusMessage* msg;
         dbus_connection_read_write(m_dbus, 0);
@@ -476,6 +486,7 @@ public:
           dbus_message_unref(msg);
         }
       }
+#endif
     }
 
     m_callback.appQuitting(this);

lib/x11/WindowXlib.cpp

@@ -4,6 +4,7 @@
 #include "boo/graphicsdev/GL.hpp"
 #include "boo/audiodev/IAudioVoiceEngine.hpp"
 #include "boo/graphicsdev/glew.h"
+#include "../Common.hpp"
 
 #if BOO_HAS_VULKAN
 #include "boo/graphicsdev/Vulkan.hpp"
@@ -715,6 +716,7 @@ public:
   WindowXlib(std::string_view title, Display* display, void* xcbConn, int defaultScreen, XIM xIM,
              XIMStyle bestInputStyle, XFontSet fontset, GLXContext lastCtx, void* vulkanHandle, GLContext* glCtx)
   : m_xDisp(display), m_callback(nullptr), m_bestStyle(bestInputStyle) {
+    BOO_MSAN_NO_INTERCEPT
     if (!S_ATOMS)
       S_ATOMS = new XlibAtoms(display);
 
@@ -753,6 +755,7 @@ public:
       int x, y, w, h;
       int nmonitors = 0;
       XRRMonitorInfo* mInfo = XRRGetMonitors(m_xDisp, screen->root, true, &nmonitors);
+      BOO_MSAN_UNPOISON(mInfo, sizeof(XRRMonitorInfo) * nmonitors);
       if (nmonitors) {
         genFrameDefault(mInfo, x, y, w, h);
         m_pixelFactor = mInfo->width / (float)mInfo->mwidth / REF_DPMM;
@@ -940,16 +943,19 @@ public:
   }
 
   double getWindowRefreshRate() const {
+    BOO_MSAN_NO_INTERCEPT
     double ret = 60.0;
     int nmonitors;
     Screen* screen = DefaultScreenOfDisplay(m_xDisp);
     XRRMonitorInfo* mInfo = XRRGetMonitors(m_xDisp, screen->root, true, &nmonitors);
+    BOO_MSAN_UNPOISON(mInfo, sizeof(XRRMonitorInfo) * nmonitors);
     if (nmonitors) {
       XRRScreenResources* res = XRRGetScreenResourcesCurrent(m_xDisp, screen->root);
       XRROutputInfo* oinfo = XRRGetOutputInfo(m_xDisp, res, *mInfo->outputs);
       XRRCrtcInfo* ci = XRRGetCrtcInfo(m_xDisp, res, oinfo->crtc);
       for (int i = 0; i < res->nmode; ++i) {
         const XRRModeInfo& mode = res->modes[i];
+        BOO_MSAN_UNPOISON(&mode, sizeof(XRRModeInfo));
         if (mode.id == ci->mode) {
           ret = calculateRefreshRate(mode);
           break;
@@ -964,9 +970,11 @@ public:
   }
 
   void setWindowFrameDefault() {
+    BOO_MSAN_NO_INTERCEPT
     int x, y, w, h, nmonitors;
     Screen* screen = DefaultScreenOfDisplay(m_xDisp);
     XRRMonitorInfo* mInfo = XRRGetMonitors(m_xDisp, screen->root, true, &nmonitors);
+    BOO_MSAN_UNPOISON(mInfo, sizeof(XRRMonitorInfo) * nmonitors);
     if (nmonitors)
       genFrameDefault(mInfo, x, y, w, h);
     else
@@ -979,7 +987,8 @@ public:
   }
 
   void getWindowFrame(float& xOut, float& yOut, float& wOut, float& hOut) const {
-    XWindowAttributes attrs;
+    BOO_MSAN_NO_INTERCEPT
+    XWindowAttributes attrs = {};
     XLockDisplay(m_xDisp);
     XGetWindowAttributes(m_xDisp, m_windowId, &attrs);
     XUnlockDisplay(m_xDisp);
@@ -990,7 +999,8 @@ public:
   }
 
   void getWindowFrame(int& xOut, int& yOut, int& wOut, int& hOut) const {
-    XWindowAttributes attrs;
+    BOO_MSAN_NO_INTERCEPT
+    XWindowAttributes attrs = {};
     XLockDisplay(m_xDisp);
     XGetWindowAttributes(m_xDisp, m_windowId, &attrs);
     XUnlockDisplay(m_xDisp);
@@ -1001,6 +1011,7 @@ public:
   }
 
   void setWindowFrame(float x, float y, float w, float h) {
+    BOO_MSAN_NO_INTERCEPT
     XWindowChanges values = {(int)x, (int)y, (int)w, (int)h};
     XLockDisplay(m_xDisp);
     XConfigureWindow(m_xDisp, m_windowId, CWX | CWY | CWWidth | CWHeight, &values);
@@ -1008,6 +1019,7 @@ public:
   }
 
   void setWindowFrame(int x, int y, int w, int h) {
+    BOO_MSAN_NO_INTERCEPT
     XWindowChanges values = {x, y, w, h};
     XLockDisplay(m_xDisp);
     XConfigureWindow(m_xDisp, m_windowId, CWX | CWY | CWWidth | CWHeight, &values);
@@ -1262,7 +1274,8 @@ public:
   }
 
   int waitForRetrace() {
-    struct timespec tp;
+    BOO_MSAN_NO_INTERCEPT
+    struct timespec tp = {};
     clock_gettime(CLOCK_REALTIME, &tp);
     if (!m_lastWaitTime.tv_sec) {
       /* Initialize reference point */
@@ -1416,9 +1429,9 @@ public:
       return true;
     }
     case Expose: {
-      Window nw;
+      Window nw = 0;
-      XWindowAttributes wxa;
+      XWindowAttributes wxa = {};
-      int x, y;
+      int x = 0, y = 0;
       XTranslateCoordinates(m_xDisp, m_windowId, DefaultRootWindow(m_xDisp), event->xexpose.x, event->xexpose.y, &x, &y,
                             &nw);
       XGetWindowAttributes(m_xDisp, m_windowId, &wxa);
@@ -1441,9 +1454,9 @@ public:
       return false;
     }
     case ConfigureNotify: {
-      Window nw;
+      Window nw = 0;
-      XWindowAttributes wxa;
+      XWindowAttributes wxa = {};
-      int x, y;
+      int x = 0, y = 0;
       XTranslateCoordinates(m_xDisp, m_windowId, DefaultRootWindow(m_xDisp), event->xconfigure.x, event->xconfigure.y,
                             &x, &y, &nw);
       XGetWindowAttributes(m_xDisp, m_windowId, &wxa);