diff --git a/CMakeLists.txt b/CMakeLists.txt
index 6700b015c..8028edaab 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -127,7 +127,7 @@ if(MSVC)
     set(CMAKE_STATIC_LINKER_FLAGS_RELEASE "/LTCG")
     set(CMAKE_STATIC_LINKER_FLAGS_RELWITHDEBINFO "/LTCG")
     set(CMAKE_EXE_LINKER_FLAGS_RELEASE "/RELEASE /LTCG /OPT:REF /OPT:ICF /INCREMENTAL:NO")
-    set(CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO "/DEBUG /RELEASE /LTCG /OPT:REF /OPT:ICF /INCREMENTAL:NO /DEBUGTYPE:cv,fixup")
+    set(CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO "/DEBUG /RELEASE /LTCG /OPT:REF /OPT:ICF /INCREMENTAL:NO /DEBUGTYPE:cv,fixup /Oy-")
   endif()
 
 else()
@@ -161,7 +161,7 @@ else()
     add_compile_options(-fno-plt)
   endif()
   check_cxx_compiler_flag(-fno-asynchronous-unwind-tables HAS_NO_ASYNC_UNWIND_TABLES)
-  if (HAS_NO_ASYNC_UNWIND_TABLES)
+  if (HAS_NO_ASYNC_UNWIND_TABLES AND ${CMAKE_BUILD_TYPE} STREQUAL Release)
     # Binary size reduction
     add_compile_options(-fno-asynchronous-unwind-tables)
   endif()
@@ -197,9 +197,9 @@ endif()
 
 if("${CMAKE_SYSTEM_NAME}" STREQUAL "Linux")
   if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")
-    if(${CMAKE_BUILD_TYPE} STREQUAL Debug)
+    if(${CMAKE_BUILD_TYPE} STREQUAL Debug OR ${CMAKE_BUILD_TYPE} STREQUAL RelWithDebInfo)
       # This is required to summarize std::string
-      add_compile_options(-fno-limit-debug-info)
+      add_compile_options(-fno-limit-debug-info -fno-omit-frame-pointer)
     endif()
     option(USE_LD_LLD "Link with LLD" ON)
   elseif("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU")