Merge pull request #32 from libAthena/utf8-windows

windows fixes and proper UTF-8 conversion

atdna/CMakeLists.txt

@@ -2,6 +2,12 @@
 # ATDNA Build #
 ###############
 
+# Force this binary to Release flags (to make MSVC happy)
+if(MSVC)
+set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_RELEASE}")
+set(CMAKE_EXE_LINKER_FLAGS_DEBUG "${CMAKE_EXE_LINKER_FLAGS_RELEASE}")
+endif()
+
 # Find dependencies
 include(FindLLVM.cmake)
 if(NOT LLVM_FOUND)

extern/yaml/CMakeLists.txt

@@ -6,9 +6,8 @@ set(YAML_VERSION_STRING "${YAML_VERSION_MAJOR}.${YAML_VERSION_MINOR}.${YAML_VERS
 file(GLOB SRC src/*.c)
 
 include_directories(include win32)
-add_definitions(-DYAML_DECLARE_STATIC)
 add_library(AthenaLibYaml STATIC ${SRC} include/yaml.h)
 
 set(YAML_INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/include CACHE PATH "YAML include path" FORCE)
-install(DIRECTORY include/ DESTINATION include/Athena COMPONENT yaml)
+#install(DIRECTORY include/ DESTINATION include/Athena COMPONENT yaml)
 install(TARGETS AthenaLibYaml DESTINATION lib COMPONENT yaml)

extern/yaml/include/yaml.h

@@ -26,7 +26,7 @@ extern "C" {
 
 /** The public API declaration. */
 
-#ifdef _WIN32
+#if 0 /* Disabled for Athena */
 #   if defined(YAML_DECLARE_STATIC)
 #       define  YAML_DECLARE(type)  type
 #   elif defined(YAML_DECLARE_EXPORT)

include/Athena/DNAYaml.hpp

@@ -7,6 +7,13 @@
  * Any changes to the types or namespacing must be reflected in 'atdna/main.cpp'
  */
 
+#if _WIN32
+#ifndef WIN32_LEAN_AND_MEAN
+#define WIN32_LEAN_AND_MEAN 1
+#endif
+#include <windows.h>
+#endif
+
 #include <string.h>
 #include <yaml.h>
 #include "DNA.hpp"
@@ -374,29 +381,44 @@ inline std::unique_ptr<YAMLNode> ValToNode(const char* val)
 template <>
 inline std::wstring NodeToVal(const YAMLNode* node)
 {
+#if _WIN32
+    int len = MultiByteToWideChar(CP_UTF8, 0, node->m_scalarString.c_str(), node->m_scalarString.size(), nullptr, 0);
+    std::wstring retval(len, L'\0');
+    MultiByteToWideChar(CP_UTF8, 0, node->m_scalarString.c_str(), node->m_scalarString.size(), &retval[0], len);
+    return retval;
+#else
     std::wstring retval;
     retval.reserve(node->m_scalarString.length());
     const char* buf = node->m_scalarString.c_str();
+    std::mbstate_t state = {};
     while (*buf)
     {
         wchar_t wc;
-        buf += std::mbtowc(&wc, buf, MB_CUR_MAX);
+        buf += std::mbrtowc(&wc, buf, MB_LEN_MAX, &state);
         retval += wc;
     }
     return retval;
+#endif
 }
 
 template <>
 inline std::unique_ptr<YAMLNode> ValToNode(const std::wstring& val)
 {
     YAMLNode* ret = new YAMLNode(YAML_SCALAR_NODE);
+#if _WIN32
+    int len = WideCharToMultiByte(CP_UTF8, 0, val.c_str(), val.size(), nullptr, 0, nullptr, nullptr);
+    ret->m_scalarString.assign(len, '\0');
+    WideCharToMultiByte(CP_UTF8, 0, val.c_str(), val.size(), &ret->m_scalarString[0], len, nullptr, nullptr);
+#else
     ret->m_scalarString.reserve(val.length());
+    std::mbstate_t state = {};
     for (wchar_t ch : val)
     {
-        char mb[4];
+        char mb[MB_LEN_MAX];
-        int c = std::wctomb(mb, ch);
+        int c = std::wcrtomb(mb, ch, &state);
         ret->m_scalarString.append(mb, c);
     }
+#endif
     return std::unique_ptr<YAMLNode>(ret);
 }
 

include/Athena/Global.hpp

@@ -64,10 +64,10 @@ namespace error
 {
 enum Level
 {
-    MESSAGE,
+    LevelMessage,
-    WARNING,
+    LevelWarning,
-    ERROR,
+    LevelError,
-    FATAL
+    LevelFatal
 };
 }
 enum SeekOrigin
@@ -125,7 +125,7 @@ std::ostream& operator<<(std::ostream& os, const Athena::Endian& endian);
 #define atDebug(fmt, ...) \
     do { atEXCEPTION_HANDLER __handler = atGetExceptionHandler(); \
     if (__handler) \
-        __handler(Athena::error::MESSAGE, __FILE__, AT_PRETTY_FUNCTION, __LINE__, fmt, ##__VA_ARGS__); \
+        __handler(Athena::error::LevelMessage, __FILE__, AT_PRETTY_FUNCTION, __LINE__, fmt, ##__VA_ARGS__); \
 } while(0)
 #else
 #define atDebug(fmt, ...)
@@ -134,25 +134,25 @@ std::ostream& operator<<(std::ostream& os, const Athena::Endian& endian);
 #define atMessage(fmt, ...) \
     do { atEXCEPTION_HANDLER __handler = atGetExceptionHandler(); \
     if (__handler) \
-        __handler(Athena::error::MESSAGE, __FILE__, AT_PRETTY_FUNCTION, __LINE__, fmt, ##__VA_ARGS__); \
+        __handler(Athena::error::LevelMessage, __FILE__, AT_PRETTY_FUNCTION, __LINE__, fmt, ##__VA_ARGS__); \
 } while(0)
 
 #define atWarning(fmt, ...) \
     do { atEXCEPTION_HANDLER __handler = atGetExceptionHandler(); \
     if (__handler) \
-        __handler(Athena::error::WARNING, __FILE__, AT_PRETTY_FUNCTION, __LINE__, fmt, ##__VA_ARGS__); \
+        __handler(Athena::error::LevelWarning, __FILE__, AT_PRETTY_FUNCTION, __LINE__, fmt, ##__VA_ARGS__); \
 } while(0)
 
 #define atError(fmt, ...) \
     do { atEXCEPTION_HANDLER __handler = atGetExceptionHandler(); \
     if (__handler) \
-        __handler(Athena::error::ERROR, __FILE__, AT_PRETTY_FUNCTION, __LINE__, fmt, ##__VA_ARGS__); \
+        __handler(Athena::error::LevelError, __FILE__, AT_PRETTY_FUNCTION, __LINE__, fmt, ##__VA_ARGS__); \
 } while(0)
 
 #define atFatal(fmt, ...) \
     do { atEXCEPTION_HANDLER __handler = atGetExceptionHandler(); \
     if (__handler) \
-        __handler(Athena::error::FATAL, __FILE__, AT_PRETTY_FUNCTION, __LINE__, fmt, ##__VA_ARGS__); \
+        __handler(Athena::error::LevelFatal, __FILE__, AT_PRETTY_FUNCTION, __LINE__, fmt, ##__VA_ARGS__); \
 } while(0)
 #elif defined(__GNUC__)
 
@@ -160,7 +160,7 @@ std::ostream& operator<<(std::ostream& os, const Athena::Endian& endian);
 #define atDebug(fmt...) \
     do { atEXCEPTION_HANDLER __handler = atGetExceptionHandler(); \
     if (__handler) \
-        __handler(Athena::error::MESSAGE, __FILE__, AT_PRETTY_FUNCTION, __LINE__, fmt); \
+        __handler(Athena::error::LevelMessage, __FILE__, AT_PRETTY_FUNCTION, __LINE__, fmt); \
 } while(0)
 #else // _MSC_VER
 #define atDebug(fmt, ...)
@@ -169,25 +169,25 @@ std::ostream& operator<<(std::ostream& os, const Athena::Endian& endian);
 #define atMessage(fmt...) \
     do { atEXCEPTION_HANDLER __handler = atGetExceptionHandler(); \
     if (__handler) \
-        __handler(Athena::error::MESSAGE, __FILE__, AT_PRETTY_FUNCTION, __LINE__, fmt); \
+        __handler(Athena::error::LevelMessage, __FILE__, AT_PRETTY_FUNCTION, __LINE__, fmt); \
 } while(0)
 
 #define atWarning(fmt...) \
     do { atEXCEPTION_HANDLER __handler = atGetExceptionHandler(); \
     if (__handler) \
-        __handler(Athena::error::WARNING, __FILE__, AT_PRETTY_FUNCTION, __LINE__, fmt); \
+        __handler(Athena::error::LevelWarning, __FILE__, AT_PRETTY_FUNCTION, __LINE__, fmt); \
 } while(0)
 
 #define atError(fmt...) \
     do { atEXCEPTION_HANDLER __handler = atGetExceptionHandler(); \
     if (__handler) \
-        __handler(Athena::error::ERROR, __FILE__, AT_PRETTY_FUNCTION, __LINE__, fmt); \
+        __handler(Athena::error::LevelError, __FILE__, AT_PRETTY_FUNCTION, __LINE__, fmt); \
 } while(0)
 
 #define atFatal(fmt...) \
     do { atEXCEPTION_HANDLER __handler = atGetExceptionHandler(); \
     if (__handler) \
-        __handler(Athena::error::FATAL, __FILE__, AT_PRETTY_FUNCTION, __LINE__, fmt); \
+        __handler(Athena::error::LevelFatal, __FILE__, AT_PRETTY_FUNCTION, __LINE__, fmt); \
 } while(0)
 #endif // defined(__GNUC__)
 

include/Athena/IStreamReader.hpp

@@ -1,6 +1,13 @@
 #ifndef ISTREAMREADER_HPP
 #define ISTREAMREADER_HPP
 
+#if _WIN32
+#ifndef WIN32_LEAN_AND_MEAN
+#define WIN32_LEAN_AND_MEAN 1
+#endif
+#include <windows.h>
+#endif
+
 #include <memory>
 #include <functional>
 #include "IStream.hpp"
@@ -590,7 +597,8 @@ public:
      */
     inline std::string readWStringAsString(atInt32 fixedLen = -1)
     {
-        std::string retval;
+#if _WIN32
+        std::wstring wstr;
         atUint16 chr = readUint16();
 
         atInt32 i;
@@ -602,12 +610,33 @@ public:
             if (!chr)
                 break;
 
-            char mb[4];
+            wstr += chr;
-            int c = std::wctomb(mb, chr);
-            retval.append(mb, c);
             chr = readUint16();
         }
 
+        int len = WideCharToMultiByte(CP_UTF8, 0, wstr.c_str(), wstr.size(), nullptr, 0, nullptr, nullptr);
+        std::string retval(len, '\0');
+        WideCharToMultiByte(CP_UTF8, 0, wstr.c_str(), wstr.size(), &retval[0], len, nullptr, nullptr);
+#else
+        std::string retval;
+        atUint16 chr = readUint16();
+
+        atInt32 i;
+        std::mbstate_t state = {};
+        for (i=0 ;; ++i)
+        {
+            if (fixedLen >= 0 && i >= fixedLen - 1)
+                break;
+
+            if (!chr)
+                break;
+
+            char mb[MB_LEN_MAX];
+            int c = std::wcrtomb(mb, chr, &state);
+            retval.append(mb, c);
+            chr = readUint16();
+        }
+#endif
         if (fixedLen >= 0 && i < fixedLen)
             seek(fixedLen - i);
 
@@ -616,7 +645,8 @@ public:
 
     inline std::string readWStringAsStringLittle(atInt32 fixedLen = -1)
     {
-        std::string retval;
+#if _WIN32
+        std::wstring wstr;
         atUint16 chr = readUint16Little();
 
         atInt32 i;
@@ -628,12 +658,33 @@ public:
             if (!chr)
                 break;
 
-            char mb[4];
+            wstr += chr;
-            int c = std::wctomb(mb, chr);
-            retval.append(mb, c);
             chr = readUint16Little();
         }
 
+        int len = WideCharToMultiByte(CP_UTF8, 0, wstr.c_str(), wstr.size(), nullptr, 0, nullptr, nullptr);
+        std::string retval(len, '\0');
+        WideCharToMultiByte(CP_UTF8, 0, wstr.c_str(), wstr.size(), &retval[0], len, nullptr, nullptr);
+#else
+        std::string retval;
+        atUint16 chr = readUint16Little();
+
+        atInt32 i;
+        std::mbstate_t state = {};
+        for (i=0 ;; ++i)
+        {
+            if (fixedLen >= 0 && i >= fixedLen - 1)
+                break;
+
+            if (!chr)
+                break;
+
+            char mb[MB_LEN_MAX];
+            int c = std::wcrtomb(mb, chr, &state);
+            retval.append(mb, c);
+            chr = readUint16Little();
+        }
+#endif
         if (fixedLen >= 0 && i < fixedLen)
             seek(fixedLen - i);
 
@@ -642,7 +693,8 @@ public:
 
     inline std::string readWStringAsStringBig(atInt32 fixedLen = -1)
     {
-        std::string retval;
+#if _WIN32
+        std::wstring wstr;
         atUint16 chr = readUint16Big();
 
         atInt32 i;
@@ -654,12 +706,33 @@ public:
             if (!chr)
                 break;
 
-            char mb[4];
+            wstr += chr;
-            int c = std::wctomb(mb, chr);
-            retval.append(mb, c);
             chr = readUint16Big();
         }
 
+        int len = WideCharToMultiByte(CP_UTF8, 0, wstr.c_str(), wstr.size(), nullptr, 0, nullptr, nullptr);
+        std::string retval(len, '\0');
+        WideCharToMultiByte(CP_UTF8, 0, wstr.c_str(), wstr.size(), &retval[0], len, nullptr, nullptr);
+#else
+        std::string retval;
+        atUint16 chr = readUint16Big();
+
+        atInt32 i;
+        std::mbstate_t state = {};
+        for (i = 0 ;; ++i)
+        {
+            if (fixedLen >= 0 && i >= fixedLen - 1)
+                break;
+
+            if (!chr)
+                break;
+
+            char mb[MB_LEN_MAX];
+            int c = std::wcrtomb(mb, chr, &state);
+            retval.append(mb, c);
+            chr = readUint16Big();
+        }
+#endif
         if (fixedLen >= 0 && i < fixedLen)
             seek(fixedLen - i);
 

include/Athena/IStreamWriter.hpp

@@ -1,6 +1,13 @@
 #ifndef ISTREAMWRITER_HPP
 #define ISTREAMWRITER_HPP
 
+#if _WIN32
+#ifndef WIN32_LEAN_AND_MEAN
+#define WIN32_LEAN_AND_MEAN 1
+#endif
+#include <windows.h>
+#endif
+
 #include "IStream.hpp"
 
 namespace Athena
@@ -437,14 +444,42 @@ public:
     inline void writeStringAsWString(const std::string& str, atInt32 fixedLen = -1)
     {
        std::string tmpStr = "\xEF\xBB\xBF" + str;
-       const char* buf = tmpStr.c_str();
 
+#if _WIN32
+       int len = MultiByteToWideChar(CP_UTF8, 0, tmpStr.c_str(), tmpStr.size(), nullptr, 0);
+       std::wstring retval(len, L'\0');
+       MultiByteToWideChar(CP_UTF8, 0, tmpStr.c_str(), tmpStr.size(), &retval[0], len);
+       if (fixedLen < 0)
+       {
+           for (wchar_t ch : retval)
+           {
+               if (ch != 0xFEFF)
+                   writeUint16(ch);
+           }
+           writeUint16(0);
+       }
+       else
+       {
+           for (atInt32 i=0 ; i<fixedLen ; ++i)
+           {
+               wchar_t wc = retval[i];
+               if (wc == 0xFEFF)
+               {
+                   --i;
+                   continue;
+               }
+               writeUint16(wc);
+           }
+       }
+#else
+       const char* buf = tmpStr.c_str();
+       std::mbstate_t state = {};
        if (fixedLen < 0)
        {
            while (*buf)
            {
                wchar_t wc;
-               buf += std::mbtowc(&wc, buf, MB_CUR_MAX);
+               buf += std::mbrtowc(&wc, buf, MB_LEN_MAX, &state);
                if (wc != 0xFEFF)
                    writeUint16(wc);
            }
@@ -456,7 +491,7 @@ public:
            {
                wchar_t wc = 0;
                if (*buf)
-                   buf += std::mbtowc(&wc, buf, MB_CUR_MAX);
+                   buf += std::mbrtowc(&wc, buf, MB_LEN_MAX, &state);
 
                if (wc == 0xFEFF)
                {
@@ -467,19 +502,48 @@ public:
                writeUint16(wc);
            }
        }
+#endif
     }
 
     inline void writeStringAsWStringLittle(const std::string& str, atInt32 fixedLen = -1)
     {
         std::string tmpStr = "\xEF\xBB\xBF" + str;
-        const char* buf = tmpStr.c_str();
 
+#if _WIN32
+        int len = MultiByteToWideChar(CP_UTF8, 0, tmpStr.c_str(), tmpStr.size(), nullptr, 0);
+        std::wstring retval(len, L'\0');
+        MultiByteToWideChar(CP_UTF8, 0, tmpStr.c_str(), tmpStr.size(), &retval[0], len);
+        if (fixedLen < 0)
+        {
+            for (wchar_t ch : retval)
+            {
+                if (ch != 0xFEFF)
+                    writeUint16(ch);
+            }
+            writeUint16Little(0);
+        }
+        else
+        {
+            for (atInt32 i = 0; i<fixedLen; ++i)
+            {
+                wchar_t wc = retval[i];
+                if (wc == 0xFEFF)
+                {
+                    --i;
+                    continue;
+                }
+                writeUint16Little(wc);
+            }
+        }
+#else
+        const char* buf = tmpStr.c_str();
+        std::mbstate_t state = {};
         if (fixedLen < 0)
         {
             while (*buf)
             {
                 wchar_t wc;
-                buf += std::mbtowc(&wc, buf, MB_CUR_MAX);
+                buf += std::mbrtowc(&wc, buf, MB_LEN_MAX, &state);
                 if (wc != 0xFEFF)
                     writeUint16Little(wc);
             }
@@ -491,7 +555,7 @@ public:
             {
                 wchar_t wc = 0;
                 if (*buf)
-                    buf += std::mbtowc(&wc, buf, MB_CUR_MAX);
+                    buf += std::mbrtowc(&wc, buf, MB_LEN_MAX, &state);
 
                 if (wc == 0xFEFF)
                 {
@@ -502,19 +566,48 @@ public:
                 writeUint16Little(wc);
             }
         }
+#endif
     }
 
     inline void writeStringAsWStringBig(const std::string& str, atInt32 fixedLen = -1)
     {
         std::string tmpStr = "\xEF\xBB\xBF" + str;
-        const char* buf = tmpStr.c_str();
 
+#if _WIN32
+        int len = MultiByteToWideChar(CP_UTF8, 0, tmpStr.c_str(), tmpStr.size(), nullptr, 0);
+        std::wstring retval(len, L'\0');
+        MultiByteToWideChar(CP_UTF8, 0, tmpStr.c_str(), tmpStr.size(), &retval[0], len);
+        if (fixedLen < 0)
+        {
+            for (wchar_t ch : retval)
+            {
+                if (ch != 0xFEFF)
+                    writeUint16(ch);
+            }
+            writeUint16Big(0);
+        }
+        else
+        {
+            for (atInt32 i = 0; i<fixedLen; ++i)
+            {
+                wchar_t wc = retval[i];
+                if (wc == 0xFEFF)
+                {
+                    --i;
+                    continue;
+                }
+                writeUint16Big(wc);
+            }
+        }
+#else
+        const char* buf = tmpStr.c_str();
+        std::mbstate_t state = {};
         if (fixedLen < 0)
         {
             while (*buf)
             {
                 wchar_t wc;
-                buf += std::mbtowc(&wc, buf, MB_CUR_MAX);
+                buf += std::mbrtowc(&wc, buf, MB_LEN_MAX, &state);
                 if (wc != 0xFEFF)
                     writeUint16Big(wc);
             }
@@ -526,7 +619,7 @@ public:
             {
                 wchar_t wc = 0;
                 if (*buf)
-                    buf += std::mbtowc(&wc, buf, MB_CUR_MAX);
+                    buf += std::mbrtowc(&wc, buf, MB_LEN_MAX, &state);
 
                 if (wc == 0xFEFF)
                 {
@@ -537,6 +630,7 @@ public:
                 writeUint16Big(wc);
             }
         }
+#endif
     }
 
     /*! \brief Writes an string to the buffer and advances the buffer.

src/Athena/DNAYaml.cpp

@@ -175,7 +175,7 @@ static inline bool EmitKeyScalar(yaml_emitter_t* doc, const char* val)
     if (!yaml_scalar_event_initialize(&event, nullptr, nullptr, (yaml_char_t*)val,
                                       strlen(val), true, true, YAML_PLAIN_SCALAR_STYLE))
         return false;
-    return yaml_emitter_emit(doc, &event);
+    return yaml_emitter_emit(doc, &event) != 0;
 }
 
 static inline yaml_scalar_style_t ScalarStyle(const YAMLNode& node)

src/Athena/Global.cpp

@@ -46,13 +46,13 @@ static void __defaultExceptionHandler(const Athena::error::Level& level, const c
     std::string levelStr;
     switch(level)
     {
-        case Athena::error::WARNING:
+        case Athena::error::LevelWarning:
             levelStr = "[WARNING] ";
             break;
-        case Athena::error::ERROR:
+        case Athena::error::LevelError:
             levelStr = "[ERROR  ] ";
             break;
-        case Athena::error::FATAL:
+        case Athena::error::LevelFatal:
             levelStr = "[FATAL  ] ";
             break;
         default: break;