libAthena
/
athena
mirror of https://github.com/libAthena/athena.git
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #32 from libAthena/utf8-windows 

windows fixes and proper UTF-8 conversion
This commit is contained in:
Jack Andersen 2015-08-30 19:32:32 -10:00
parent 1de67f30b8 0de254e846
commit 9c44d5f4de
10 changed files with 2211 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
atdna/CMakeLists.txt
View File

@ -2,6 +2,12 @@
# ATDNA Build #
###############
# Force this binary to Release flags (to make MSVC happy)
if(MSVC)
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_RELEASE}")
set(CMAKE_EXE_LINKER_FLAGS_DEBUG "${CMAKE_EXE_LINKER_FLAGS_RELEASE}")
endif()
# Find dependencies
include(FindLLVM.cmake)
if(NOT LLVM_FOUND)

3
extern/yaml/CMakeLists.txt vendored
View File

@ -6,9 +6,8 @@ set(YAML_VERSION_STRING "${YAML_VERSION_MAJOR}.${YAML_VERSION_MINOR}.${YAML_VERS
file(GLOB SRC src/*.c)
include_directories(include win32)
add_definitions(-DYAML_DECLARE_STATIC)
add_library(AthenaLibYaml STATIC ${SRC} include/yaml.h)
set(YAML_INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/include CACHE PATH "YAML include path" FORCE)
install(DIRECTORY include/ DESTINATION include/Athena COMPONENT yaml)
#install(DIRECTORY include/ DESTINATION include/Athena COMPONENT yaml)
install(TARGETS AthenaLibYaml DESTINATION lib COMPONENT yaml)

2
extern/yaml/include/yaml.h vendored
View File

@ -26,7 +26,7 @@ extern "C" {
/** The public API declaration. */
#ifdef _WIN32
#if 0 /* Disabled for Athena */
# if defined(YAML_DECLARE_STATIC)
# define YAML_DECLARE(type) type
# elif defined(YAML_DECLARE_EXPORT)

28
include/Athena/DNAYaml.hpp
View File

@ -7,6 +7,13 @@
* Any changes to the types or namespacing must be reflected in 'atdna/main.cpp'
*/
#if _WIN32
#ifndef WIN32_LEAN_AND_MEAN
#define WIN32_LEAN_AND_MEAN 1
#endif
#include <windows.h>
#endif
#include <string.h>
#include <yaml.h>
#include "DNA.hpp"
@ -374,29 +381,44 @@ inline std::unique_ptr<YAMLNode> ValToNode(const char* val)
template <>
inline std::wstring NodeToVal(const YAMLNode* node)
{
#if _WIN32
int len = MultiByteToWideChar(CP_UTF8, 0, node->m_scalarString.c_str(), node->m_scalarString.size(), nullptr, 0);
std::wstring retval(len, L'\0');
MultiByteToWideChar(CP_UTF8, 0, node->m_scalarString.c_str(), node->m_scalarString.size(), &retval[0], len);
return retval;
#else
std::wstring retval;
retval.reserve(node->m_scalarString.length());
const char* buf = node->m_scalarString.c_str();
std::mbstate_t state = {};
while (*buf)
{
wchar_t wc;
buf += std::mbtowc(&wc, buf, MB_CUR_MAX);
buf += std::mbrtowc(&wc, buf, MB_LEN_MAX, &state);
retval += wc;
}
return retval;
#endif
}
template <>
inline std::unique_ptr<YAMLNode> ValToNode(const std::wstring& val)
{
YAMLNode* ret = new YAMLNode(YAML_SCALAR_NODE);
#if _WIN32
int len = WideCharToMultiByte(CP_UTF8, 0, val.c_str(), val.size(), nullptr, 0, nullptr, nullptr);
ret->m_scalarString.assign(len, '\0');
WideCharToMultiByte(CP_UTF8, 0, val.c_str(), val.size(), &ret->m_scalarString[0], len, nullptr, nullptr);
#else
ret->m_scalarString.reserve(val.length());
std::mbstate_t state = {};
for (wchar_t ch : val)
{
char mb[4];
int c = std::wctomb(mb, ch);
char mb[MB_LEN_MAX];
int c = std::wcrtomb(mb, ch, &state);
ret->m_scalarString.append(mb, c);
}
#endif
return std::unique_ptr<YAMLNode>(ret);
}

28
include/Athena/Global.hpp
View File

@ -64,10 +64,10 @@ namespace error
{
enum Level
{
MESSAGE,
WARNING,
ERROR,
FATAL
LevelMessage,
LevelWarning,
LevelError,
LevelFatal
};
}
enum SeekOrigin
@ -125,7 +125,7 @@ std::ostream& operator<<(std::ostream& os, const Athena::Endian& endian);
#define atDebug(fmt, ...) \
do { atEXCEPTION_HANDLER __handler = atGetExceptionHandler(); \
if (__handler) \
__handler(Athena::error::MESSAGE, __FILE__, AT_PRETTY_FUNCTION, __LINE__, fmt, ##__VA_ARGS__); \
__handler(Athena::error::LevelMessage, __FILE__, AT_PRETTY_FUNCTION, __LINE__, fmt, ##__VA_ARGS__); \
} while(0)
#else
#define atDebug(fmt, ...)
@ -134,25 +134,25 @@ std::ostream& operator<<(std::ostream& os, const Athena::Endian& endian);
#define atMessage(fmt, ...) \
do { atEXCEPTION_HANDLER __handler = atGetExceptionHandler(); \
if (__handler) \
__handler(Athena::error::MESSAGE, __FILE__, AT_PRETTY_FUNCTION, __LINE__, fmt, ##__VA_ARGS__); \
__handler(Athena::error::LevelMessage, __FILE__, AT_PRETTY_FUNCTION, __LINE__, fmt, ##__VA_ARGS__); \
} while(0)
#define atWarning(fmt, ...) \
do { atEXCEPTION_HANDLER __handler = atGetExceptionHandler(); \
if (__handler) \
__handler(Athena::error::WARNING, __FILE__, AT_PRETTY_FUNCTION, __LINE__, fmt, ##__VA_ARGS__); \
__handler(Athena::error::LevelWarning, __FILE__, AT_PRETTY_FUNCTION, __LINE__, fmt, ##__VA_ARGS__); \
} while(0)
#define atError(fmt, ...) \
do { atEXCEPTION_HANDLER __handler = atGetExceptionHandler(); \
if (__handler) \
__handler(Athena::error::ERROR, __FILE__, AT_PRETTY_FUNCTION, __LINE__, fmt, ##__VA_ARGS__); \
__handler(Athena::error::LevelError, __FILE__, AT_PRETTY_FUNCTION, __LINE__, fmt, ##__VA_ARGS__); \
} while(0)
#define atFatal(fmt, ...) \
do { atEXCEPTION_HANDLER __handler = atGetExceptionHandler(); \
if (__handler) \
__handler(Athena::error::FATAL, __FILE__, AT_PRETTY_FUNCTION, __LINE__, fmt, ##__VA_ARGS__); \
__handler(Athena::error::LevelFatal, __FILE__, AT_PRETTY_FUNCTION, __LINE__, fmt, ##__VA_ARGS__); \
} while(0)
#elif defined(__GNUC__)
@ -160,7 +160,7 @@ std::ostream& operator<<(std::ostream& os, const Athena::Endian& endian);
#define atDebug(fmt...) \
do { atEXCEPTION_HANDLER __handler = atGetExceptionHandler(); \
if (__handler) \
__handler(Athena::error::MESSAGE, __FILE__, AT_PRETTY_FUNCTION, __LINE__, fmt); \
__handler(Athena::error::LevelMessage, __FILE__, AT_PRETTY_FUNCTION, __LINE__, fmt); \
} while(0)
#else // _MSC_VER
#define atDebug(fmt, ...)
@ -169,25 +169,25 @@ std::ostream& operator<<(std::ostream& os, const Athena::Endian& endian);
#define atMessage(fmt...) \
do { atEXCEPTION_HANDLER __handler = atGetExceptionHandler(); \
if (__handler) \
__handler(Athena::error::MESSAGE, __FILE__, AT_PRETTY_FUNCTION, __LINE__, fmt); \
__handler(Athena::error::LevelMessage, __FILE__, AT_PRETTY_FUNCTION, __LINE__, fmt); \
} while(0)
#define atWarning(fmt...) \
do { atEXCEPTION_HANDLER __handler = atGetExceptionHandler(); \
if (__handler) \
__handler(Athena::error::WARNING, __FILE__, AT_PRETTY_FUNCTION, __LINE__, fmt); \
__handler(Athena::error::LevelWarning, __FILE__, AT_PRETTY_FUNCTION, __LINE__, fmt); \
} while(0)
#define atError(fmt...) \
do { atEXCEPTION_HANDLER __handler = atGetExceptionHandler(); \
if (__handler) \
__handler(Athena::error::ERROR, __FILE__, AT_PRETTY_FUNCTION, __LINE__, fmt); \
__handler(Athena::error::LevelError, __FILE__, AT_PRETTY_FUNCTION, __LINE__, fmt); \
} while(0)
#define atFatal(fmt...) \
do { atEXCEPTION_HANDLER __handler = atGetExceptionHandler(); \
if (__handler) \
__handler(Athena::error::FATAL, __FILE__, AT_PRETTY_FUNCTION, __LINE__, fmt); \
__handler(Athena::error::LevelFatal, __FILE__, AT_PRETTY_FUNCTION, __LINE__, fmt); \
} while(0)
#endif // defined(__GNUC__)

99
include/Athena/IStreamReader.hpp
View File

@ -1,6 +1,13 @@
#ifndef ISTREAMREADER_HPP
#define ISTREAMREADER_HPP
#if _WIN32
#ifndef WIN32_LEAN_AND_MEAN
#define WIN32_LEAN_AND_MEAN 1
#endif
#include <windows.h>
#endif
#include <memory>
#include <functional>
#include "IStream.hpp"
@ -590,11 +597,12 @@ public:
*/
inline std::string readWStringAsString(atInt32 fixedLen = -1)
{
std::string retval;
#if _WIN32
std::wstring wstr;
atUint16 chr = readUint16();
atInt32 i;
for (i = 0 ;; ++i)
for (i=0 ;; ++i)
{
if (fixedLen >= 0 && i >= fixedLen - 1)
break;
@ -602,12 +610,33 @@ public:
if (!chr)
break;
char mb[4];
int c = std::wctomb(mb, chr);
retval.append(mb, c);
wstr += chr;
chr = readUint16();
}
int len = WideCharToMultiByte(CP_UTF8, 0, wstr.c_str(), wstr.size(), nullptr, 0, nullptr, nullptr);
std::string retval(len, '\0');
WideCharToMultiByte(CP_UTF8, 0, wstr.c_str(), wstr.size(), &retval[0], len, nullptr, nullptr);
#else
std::string retval;
atUint16 chr = readUint16();
atInt32 i;
std::mbstate_t state = {};
for (i=0 ;; ++i)
{
if (fixedLen >= 0 && i >= fixedLen - 1)
break;
if (!chr)
break;
char mb[MB_LEN_MAX];
int c = std::wcrtomb(mb, chr, &state);
retval.append(mb, c);
chr = readUint16();
}
#endif
if (fixedLen >= 0 && i < fixedLen)
seek(fixedLen - i);
@ -616,11 +645,12 @@ public:
inline std::string readWStringAsStringLittle(atInt32 fixedLen = -1)
{
std::string retval;
#if _WIN32
std::wstring wstr;
atUint16 chr = readUint16Little();
atInt32 i;
for (i = 0 ;; ++i)
for (i=0 ;; ++i)
{
if (fixedLen >= 0 && i >= fixedLen - 1)
break;
@ -628,12 +658,33 @@ public:
if (!chr)
break;
char mb[4];
int c = std::wctomb(mb, chr);
retval.append(mb, c);
wstr += chr;
chr = readUint16Little();
}
int len = WideCharToMultiByte(CP_UTF8, 0, wstr.c_str(), wstr.size(), nullptr, 0, nullptr, nullptr);
std::string retval(len, '\0');
WideCharToMultiByte(CP_UTF8, 0, wstr.c_str(), wstr.size(), &retval[0], len, nullptr, nullptr);
#else
std::string retval;
atUint16 chr = readUint16Little();
atInt32 i;
std::mbstate_t state = {};
for (i=0 ;; ++i)
{
if (fixedLen >= 0 && i >= fixedLen - 1)
break;
if (!chr)
break;
char mb[MB_LEN_MAX];
int c = std::wcrtomb(mb, chr, &state);
retval.append(mb, c);
chr = readUint16Little();
}
#endif
if (fixedLen >= 0 && i < fixedLen)
seek(fixedLen - i);
@ -642,10 +693,32 @@ public:
inline std::string readWStringAsStringBig(atInt32 fixedLen = -1)
{
#if _WIN32
std::wstring wstr;
atUint16 chr = readUint16Big();
atInt32 i;
for (i=0 ;; ++i)
{
if (fixedLen >= 0 && i >= fixedLen - 1)
break;
if (!chr)
break;
wstr += chr;
chr = readUint16Big();
}
int len = WideCharToMultiByte(CP_UTF8, 0, wstr.c_str(), wstr.size(), nullptr, 0, nullptr, nullptr);
std::string retval(len, '\0');
WideCharToMultiByte(CP_UTF8, 0, wstr.c_str(), wstr.size(), &retval[0], len, nullptr, nullptr);
#else
std::string retval;
atUint16 chr = readUint16Big();
atInt32 i;
std::mbstate_t state = {};
for (i = 0 ;; ++i)
{
if (fixedLen >= 0 && i >= fixedLen - 1)
@ -654,12 +727,12 @@ public:
if (!chr)
break;
char mb[4];
int c = std::wctomb(mb, chr);
char mb[MB_LEN_MAX];
int c = std::wcrtomb(mb, chr, &state);
retval.append(mb, c);
chr = readUint16Big();
}
#endif
if (fixedLen >= 0 && i < fixedLen)
seek(fixedLen - i);

112
include/Athena/IStreamWriter.hpp
View File

@ -1,6 +1,13 @@
#ifndef ISTREAMWRITER_HPP
#define ISTREAMWRITER_HPP
#if _WIN32
#ifndef WIN32_LEAN_AND_MEAN
#define WIN32_LEAN_AND_MEAN 1
#endif
#include <windows.h>
#endif
#include "IStream.hpp"
namespace Athena
@ -437,14 +444,42 @@ public:
inline void writeStringAsWString(const std::string& str, atInt32 fixedLen = -1)
{
std::string tmpStr = "\xEF\xBB\xBF" + str;
const char* buf = tmpStr.c_str();
#if _WIN32
int len = MultiByteToWideChar(CP_UTF8, 0, tmpStr.c_str(), tmpStr.size(), nullptr, 0);
std::wstring retval(len, L'\0');
MultiByteToWideChar(CP_UTF8, 0, tmpStr.c_str(), tmpStr.size(), &retval[0], len);
if (fixedLen < 0)
{
for (wchar_t ch : retval)
{
if (ch != 0xFEFF)
writeUint16(ch);
}
writeUint16(0);
}
else
{
for (atInt32 i=0 ; i<fixedLen ; ++i)
{
wchar_t wc = retval[i];
if (wc == 0xFEFF)
{
--i;
continue;
}
writeUint16(wc);
}
}
#else
const char* buf = tmpStr.c_str();
std::mbstate_t state = {};
if (fixedLen < 0)
{
while (*buf)
{
wchar_t wc;
buf += std::mbtowc(&wc, buf, MB_CUR_MAX);
buf += std::mbrtowc(&wc, buf, MB_LEN_MAX, &state);
if (wc != 0xFEFF)
writeUint16(wc);
}
@ -456,7 +491,7 @@ public:
{
wchar_t wc = 0;
if (*buf)
buf += std::mbtowc(&wc, buf, MB_CUR_MAX);
buf += std::mbrtowc(&wc, buf, MB_LEN_MAX, &state);
if (wc == 0xFEFF)
{
@ -467,19 +502,48 @@ public:
writeUint16(wc);
}
}
#endif
}
inline void writeStringAsWStringLittle(const std::string& str, atInt32 fixedLen = -1)
{
std::string tmpStr = "\xEF\xBB\xBF" + str;
const char* buf = tmpStr.c_str();
#if _WIN32
int len = MultiByteToWideChar(CP_UTF8, 0, tmpStr.c_str(), tmpStr.size(), nullptr, 0);
std::wstring retval(len, L'\0');
MultiByteToWideChar(CP_UTF8, 0, tmpStr.c_str(), tmpStr.size(), &retval[0], len);
if (fixedLen < 0)
{
for (wchar_t ch : retval)
{
if (ch != 0xFEFF)
writeUint16(ch);
}
writeUint16Little(0);
}
else
{
for (atInt32 i = 0; i<fixedLen; ++i)
{
wchar_t wc = retval[i];
if (wc == 0xFEFF)
{
--i;
continue;
}
writeUint16Little(wc);
}
}
#else
const char* buf = tmpStr.c_str();
std::mbstate_t state = {};
if (fixedLen < 0)
{
while (*buf)
{
wchar_t wc;
buf += std::mbtowc(&wc, buf, MB_CUR_MAX);
buf += std::mbrtowc(&wc, buf, MB_LEN_MAX, &state);
if (wc != 0xFEFF)
writeUint16Little(wc);
}
@ -491,7 +555,7 @@ public:
{
wchar_t wc = 0;
if (*buf)
buf += std::mbtowc(&wc, buf, MB_CUR_MAX);
buf += std::mbrtowc(&wc, buf, MB_LEN_MAX, &state);
if (wc == 0xFEFF)
{
@ -502,19 +566,48 @@ public:
writeUint16Little(wc);
}
}
#endif
}
inline void writeStringAsWStringBig(const std::string& str, atInt32 fixedLen = -1)
{
std::string tmpStr = "\xEF\xBB\xBF" + str;
const char* buf = tmpStr.c_str();
#if _WIN32
int len = MultiByteToWideChar(CP_UTF8, 0, tmpStr.c_str(), tmpStr.size(), nullptr, 0);
std::wstring retval(len, L'\0');
MultiByteToWideChar(CP_UTF8, 0, tmpStr.c_str(), tmpStr.size(), &retval[0], len);
if (fixedLen < 0)
{
for (wchar_t ch : retval)
{
if (ch != 0xFEFF)
writeUint16(ch);
}
writeUint16Big(0);
}
else
{
for (atInt32 i = 0; i<fixedLen; ++i)
{
wchar_t wc = retval[i];
if (wc == 0xFEFF)
{
--i;
continue;
}
writeUint16Big(wc);
}
}
#else
const char* buf = tmpStr.c_str();
std::mbstate_t state = {};
if (fixedLen < 0)
{
while (*buf)
{
wchar_t wc;
buf += std::mbtowc(&wc, buf, MB_CUR_MAX);
buf += std::mbrtowc(&wc, buf, MB_LEN_MAX, &state);
if (wc != 0xFEFF)
writeUint16Big(wc);
}
@ -526,7 +619,7 @@ public:
{
wchar_t wc = 0;
if (*buf)
buf += std::mbtowc(&wc, buf, MB_CUR_MAX);
buf += std::mbrtowc(&wc, buf, MB_LEN_MAX, &state);
if (wc == 0xFEFF)
{
@ -537,6 +630,7 @@ public:
writeUint16Big(wc);
}
}
#endif
}
/*! \brief Writes an string to the buffer and advances the buffer.

1971
include/yaml.h Normal file
View File

File diff suppressed because it is too large Load Diff

2
src/Athena/DNAYaml.cpp
View File

@ -175,7 +175,7 @@ static inline bool EmitKeyScalar(yaml_emitter_t* doc, const char* val)
if (!yaml_scalar_event_initialize(&event, nullptr, nullptr, (yaml_char_t*)val,
strlen(val), true, true, YAML_PLAIN_SCALAR_STYLE))
return false;
return yaml_emitter_emit(doc, &event);
return yaml_emitter_emit(doc, &event) != 0;
}
static inline yaml_scalar_style_t ScalarStyle(const YAMLNode& node)

6
src/Athena/Global.cpp
View File

@ -46,13 +46,13 @@ static void __defaultExceptionHandler(const Athena::error::Level& level, const c
std::string levelStr;
switch(level)
{
case Athena::error::WARNING:
case Athena::error::LevelWarning:
levelStr = "[WARNING] ";
break;
case Athena::error::ERROR:
case Athena::error::LevelError:
levelStr = "[ERROR ] ";
break;
case Athena::error::FATAL:
case Athena::error::LevelFatal:
levelStr = "[FATAL ] ";
break;
default: break;