From 461893d7a1f2881ccd28fda6732563d51a284e5a Mon Sep 17 00:00:00 2001 From: Jack Andersen Date: Sat, 23 May 2015 18:51:16 -1000 Subject: [PATCH] database refinements; blender shell --- hecl/blender/CBlenderConnection.cpp | 198 +++++++++++++++++++++++ hecl/blender/CBlenderConnection.hpp | 22 ++- hecl/blender/addon/hmdl/__init__.py | 3 +- hecl/blender/addon/hmdl/hmdl_shader.py | 14 +- hecl/blender/addon/hmdl/hmdl_skin.py | 5 +- hecl/blender/blendershell.py | 17 +- hecl/driver/main.cpp | 5 + hecl/include/HECLDatabase.hpp | 111 ++----------- hecl/lib/database/CLooseDatabase.hpp | 82 ---------- hecl/lib/database/CMemoryDatabase.hpp | 60 ------- hecl/lib/database/CPackedDatabase.hpp | 59 ------- hecl/lib/database/CProject.cpp | 21 +-- hecl/lib/database/CRuntime.cpp | 1 + hecl/lib/database/CSQLite.hpp | 208 ------------------------- hecl/lib/database/CSQLiteCooked.hpp | 82 ++++++++++ hecl/lib/database/CSQLiteMain.hpp | 134 ++++++++++++++++ hecl/lib/database/HECLDatabase.cpp | 28 ---- hecl/lib/database/database.pri | 9 +- 18 files changed, 488 insertions(+), 571 deletions(-) delete mode 100644 hecl/lib/database/CLooseDatabase.hpp delete mode 100644 hecl/lib/database/CMemoryDatabase.hpp delete mode 100644 hecl/lib/database/CPackedDatabase.hpp delete mode 100644 hecl/lib/database/CSQLite.hpp create mode 100644 hecl/lib/database/CSQLiteCooked.hpp create mode 100644 hecl/lib/database/CSQLiteMain.hpp delete mode 100644 hecl/lib/database/HECLDatabase.cpp diff --git a/hecl/blender/CBlenderConnection.cpp b/hecl/blender/CBlenderConnection.cpp index e69de29bb..d077f139c 100644 --- a/hecl/blender/CBlenderConnection.cpp +++ b/hecl/blender/CBlenderConnection.cpp @@ -0,0 +1,198 @@ +#if _WIN32 +#else +#include +#include +#include +#include +#include +#include +#endif + +#include "CBlenderConnection.hpp" + +#ifdef __APPLE__ +#define DEFAULT_BLENDER_BIN "/Applications/Blender.app/Contents/MacOS/blender" +#elif _WIN32 +#define DEFAULT_BLENDER_BIN "%ProgramFiles%\\Blender Foundation\\Blender\\blender.exe" +#else +#define DEFAULT_BLENDER_BIN "blender" +#endif + +#define TEMP_SHELLSCRIPT "/home/jacko/hecl/blender/blendershell.py" + +size_t CBlenderConnection::readLine(char* buf, size_t bufSz) +{ + size_t readBytes = 0; + while (true) + { + if (readBytes >= bufSz) + throw std::length_error("Pipe buffer overrun"); + ssize_t ret = read(m_readpipe[0], buf, 1); + if (ret < 0) + goto err; + else if (ret == 1) + { + if (*buf == '\n') + { + *buf = '\0'; + return readBytes; + } + ++readBytes; + ++buf; + } + else + { + *buf = '\0'; + return readBytes; + } + } +err: + throw std::error_code(errno, std::system_category()); + return 0; +} + +size_t CBlenderConnection::writeLine(const char* buf) +{ + ssize_t ret, nlerr; + ret = write(m_writepipe[1], buf, strlen(buf)); + if (ret < 0) + goto err; + nlerr = write(m_writepipe[1], "\n", 1); + if (nlerr < 0) + goto err; + return (size_t)ret; +err: + throw std::error_code(errno, std::system_category()); +} + +size_t CBlenderConnection::readBuf(char* buf, size_t len) +{ + ssize_t ret = read(m_readpipe[0], buf, len); + if (ret < 0) + throw std::error_code(errno, std::system_category()); + return ret; +} + +size_t CBlenderConnection::writeBuf(const char* buf, size_t len) +{ + ssize_t ret = write(m_writepipe[1], buf, len); + if (ret < 0) + throw std::error_code(errno, std::system_category()); + return ret; +} + +void CBlenderConnection::closePipe() +{ + close(m_readpipe[0]); + close(m_writepipe[1]); +} + +CBlenderConnection::CBlenderConnection(bool silenceBlender) +{ + /* Construct communication pipes */ + pipe(m_readpipe); + pipe(m_writepipe); + + /* User-specified blender path */ + char* blenderBin = getenv("BLENDER_BIN"); + + /* Child process of blender */ + pid_t pid = fork(); + if (!pid) + { + close(m_writepipe[1]); + close(m_readpipe[0]); + + if (silenceBlender) + { + close(STDOUT_FILENO); + close(STDERR_FILENO); + } + + char errbuf[256]; + char readfds[32]; + snprintf(readfds, 32, "%d", m_writepipe[0]); + char writefds[32]; + snprintf(writefds, 32, "%d", m_readpipe[1]); + + /* User-specified blender first */ + if (blenderBin) + { + execlp(blenderBin, blenderBin, "--background", "-P", TEMP_SHELLSCRIPT, + "--", readfds, writefds, NULL); + if (errno != ENOENT) + { + snprintf(errbuf, 256, "NOLAUNCH %s\n", strerror(errno)); + write(m_writepipe[1], errbuf, strlen(errbuf)); + exit(1); + } + } + + /* Default blender next */ + execlp(DEFAULT_BLENDER_BIN, DEFAULT_BLENDER_BIN, "--background", "-P", TEMP_SHELLSCRIPT, + "--", readfds, writefds, NULL); + if (errno != ENOENT) + { + snprintf(errbuf, 256, "NOLAUNCH %s\n", strerror(errno)); + write(m_writepipe[1], errbuf, strlen(errbuf)); + exit(1); + } + + /* Unable to find blender */ + write(m_writepipe[1], "NOBLENDER\n", 10); + exit(1); + + } + close(m_writepipe[0]); + close(m_readpipe[1]); + m_blenderProc = pid; + + /* Handle first response */ + char lineBuf[256]; + readLine(lineBuf, sizeof(lineBuf)); + if (!strcmp(lineBuf, "NOLAUNCH")) + { + closePipe(); + throw std::runtime_error("Unable to launch blender"); + } + else if (!strcmp(lineBuf, "NOBLENDER")) + { + closePipe(); + if (blenderBin) + throw std::runtime_error("Unable to find blender at '" + std::string(blenderBin) + "' or '" + + std::string(DEFAULT_BLENDER_BIN) + "'"); + else + throw std::runtime_error("Unable to find blender at '" + + std::string(DEFAULT_BLENDER_BIN) + "'"); + } + else if (!strcmp(lineBuf, "NOADDON")) + { + closePipe(); + throw std::runtime_error("HECL addon not installed within blender"); + } + else if (strcmp(lineBuf, "READY")) + { + closePipe(); + throw std::runtime_error("read '" + std::string(lineBuf) + "' from blender; expected 'READY'"); + } + writeLine("ACK"); + + writeLine("HELLOBLENDER!!"); + readLine(lineBuf, sizeof(lineBuf)); + printf("%s\n", lineBuf); + quitBlender(); + +} + +CBlenderConnection::~CBlenderConnection() +{ + closePipe(); +} + +void CBlenderConnection::quitBlender() +{ + writeLine("QUIT"); + char lineBuf[256]; + readLine(lineBuf, sizeof(lineBuf)); + printf("%s\n", lineBuf); +} diff --git a/hecl/blender/CBlenderConnection.hpp b/hecl/blender/CBlenderConnection.hpp index f8a96654c..32af66020 100644 --- a/hecl/blender/CBlenderConnection.hpp +++ b/hecl/blender/CBlenderConnection.hpp @@ -1,14 +1,34 @@ #ifndef CBLENDERCONNECTION_HPP #define CBLENDERCONNECTION_HPP +#if _WIN32 +#define _WIN32_LEAN_AND_MEAN 1 +#include +#else #include +#endif class CBlenderConnection { +#if _WIN32 + HANDLE m_blenderProc; + HANDLE m_readpipe; + HANDLE m_writepipe; +#else pid_t m_blenderProc; + int m_readpipe[2]; + int m_writepipe[2]; +#endif + size_t readLine(char* buf, size_t bufSz); + size_t writeLine(const char* buf); + size_t readBuf(char* buf, size_t len); + size_t writeBuf(const char* buf, size_t len); + void closePipe(); public: - CBlenderConnection(); + CBlenderConnection(bool silenceBlender=false); ~CBlenderConnection(); + + void quitBlender(); }; #endif // CBLENDERCONNECTION_HPP diff --git a/hecl/blender/addon/hmdl/__init__.py b/hecl/blender/addon/hmdl/__init__.py index 34101356f..ce8ab5ce5 100644 --- a/hecl/blender/addon/hmdl/__init__.py +++ b/hecl/blender/addon/hmdl/__init__.py @@ -97,7 +97,7 @@ def generate_skeleton_info(armature, endian_char='<'): def cook(writefd, platform_type, endian_char): mesh_obj = bpy.data.objects[bpy.context.scene.hecl_mesh_obj] if mesh_obj.type != 'MESH': - raise RuntimeError("{0} is not a mesh".format(mesh_obj.name)) + raise RuntimeError("%s is not a mesh" % mesh_obj.name) # Partial meshes part_meshes = set() @@ -162,7 +162,6 @@ def cook(writefd, platform_type, endian_char): for mat in bpy.data.materials: if mat.name.endswith('_%u_%u' % (grp_idx, mat_idx)): hecl_str = hmdl_shader.shader(mat, mesh_obj, bpy.data.filepath) - else: mat = mesh_obj.data.materials[mat_idx] diff --git a/hecl/blender/addon/hmdl/hmdl_shader.py b/hecl/blender/addon/hmdl/hmdl_shader.py index 0a208dca4..0b2b28c7b 100644 --- a/hecl/blender/addon/hmdl/hmdl_shader.py +++ b/hecl/blender/addon/hmdl/hmdl_shader.py @@ -42,8 +42,11 @@ def recursive_color_trace(mat_obj, mesh_obj, blend_path, node, socket=None): elif node.type == 'TEXTURE': + if not node.texture or not hasattr(node.texture, 'name'): + raise RuntimeError("HMDL texture nodes must specify a texture object") + if not node.inputs['Vector'].is_linked: - raise RuntimeError("HMDL texture nodes must have a 'Geometry', 'Group' UV modifier node linked") + raise RuntimeError("HMDL texture nodes must have a 'Geometry' or 'Group' UV modifier node linked") # Determine matrix generator type matrix_str = None @@ -89,6 +92,8 @@ def recursive_color_trace(mat_obj, mesh_obj, blend_path, node, socket=None): if soc_from.name == 'UV': uv_name = soc_from.node.uv_layer uv_idx = mesh_obj.data.uv_layers.find(uv_name) + if uv_idx == -1: + raise RuntimeError('UV Layer "%s" doesn\'t exist' % uv_name) uvsource_str = 'hecl_TexCoord[%d]' % uv_idx elif soc_from.name == 'Normal': @@ -163,8 +168,11 @@ def recursive_alpha_trace(mat_obj, mesh_obj, blend_path, node, socket=None): elif node.type == 'TEXTURE': + if not node.texture or not hasattr(node.texture, 'name'): + raise RuntimeError("HMDL texture nodes must specify a texture object") + if not node.inputs['Vector'].is_linked: - raise RuntimeError("HMDL texture nodes must have a 'Geometry', 'Group' UV modifier node linked") + raise RuntimeError("HMDL texture nodes must have a 'Geometry' or 'Group' UV modifier node linked") # Determine matrix generator type matrix_str = None @@ -210,6 +218,8 @@ def recursive_alpha_trace(mat_obj, mesh_obj, blend_path, node, socket=None): if soc_from.name == 'UV': uv_name = soc_from.node.uv_layer uv_idx = mesh_obj.data.uv_layers.find(uv_name) + if uv_idx == -1: + raise RuntimeError('UV Layer "%s" doesn\'t exist' % uv_name) uvsource_str = 'hecl_TexCoord[%d]' % uv_idx elif soc_from.name == 'Normal': diff --git a/hecl/blender/addon/hmdl/hmdl_skin.py b/hecl/blender/addon/hmdl/hmdl_skin.py index d66fbabf9..ccdb3d0c3 100644 --- a/hecl/blender/addon/hmdl/hmdl_skin.py +++ b/hecl/blender/addon/hmdl/hmdl_skin.py @@ -3,7 +3,7 @@ HMDL Export Blender Addon By Jack Andersen This file defines the `hmdl_skin` class to iteratively construct -a Skinning Info Section for `PAR1` HMDL files. Used by draw-format +a Skinning Info Section for HMDL files. Used by draw-format generators to select an optimal skin entry for a draw primitive, or have a new one established. ''' @@ -96,6 +96,3 @@ class hmdl_skin: return info_bytes - - - diff --git a/hecl/blender/blendershell.py b/hecl/blender/blendershell.py index fced84a53..aa0e84f64 100644 --- a/hecl/blender/blendershell.py +++ b/hecl/blender/blendershell.py @@ -1,9 +1,13 @@ import bpy, sys, os # Extract pipe file descriptors from arguments +print(sys.argv) +if '--' not in sys.argv: + bpy.ops.wm.quit_blender() args = sys.argv[sys.argv.index('--')+1:] readfd = int(args[0]) writefd = int(args[1]) +print('READ', readfd, 'WRITE', writefd) def readpipeline(): retval = bytearray() @@ -20,6 +24,12 @@ def quitblender(): writepipeline(b'QUITTING') bpy.ops.wm.quit_blender() +# Check that HECL addon is installed/enabled +if 'hecl' not in bpy.context.user_preferences.addons: + if 'FINISHED' not in bpy.ops.wm.addon_enable(module='hecl'): + writepipeline(b'NOADDON') + bpy.ops.wm.quit_blender() + # Intro handshake writepipeline(b'READY') ackbytes = readpipeline() @@ -34,9 +44,12 @@ while True: quitblender() elif cmdline[0] == b'OPEN': - bpy.ops.wm.open_mainfile(filepath=cmdline[1].encode()) + bpy.ops.wm.open_mainfile(filepath=cmdline[1].decode()) writepipeline(b'SUCCESS') elif cmdline[0] == b'TYPE': - objname = cmdline[1].encode() + objname = cmdline[1].decode() + + else: + writepipeline(b'RESP ' + cmdline[0]) diff --git a/hecl/driver/main.cpp b/hecl/driver/main.cpp index b53d31ccd..128a183dc 100644 --- a/hecl/driver/main.cpp +++ b/hecl/driver/main.cpp @@ -49,8 +49,13 @@ static void whiddleArgs(std::string& args, const std::regex& regex) args = remArgs; } +#include "../blender/CBlenderConnection.hpp" + int main(int argc, const char** argv) { + CBlenderConnection bconn(false); + return 0; + /* Basic usage check */ if (argc == 1) { diff --git a/hecl/include/HECLDatabase.hpp b/hecl/include/HECLDatabase.hpp index e6d674cc7..baf6483b7 100644 --- a/hecl/include/HECLDatabase.hpp +++ b/hecl/include/HECLDatabase.hpp @@ -13,7 +13,6 @@ namespace HECLDatabase { -class IDatabase; class IProject; /** @@ -49,11 +48,6 @@ public: */ virtual const std::string& path() const=0; - /** - * @brief Retrieve the database this object is stored within - * @return database object - */ - virtual IDatabase* parentDatabase() const=0; }; /** @@ -86,74 +80,6 @@ public: virtual std::vector::const_iterator end() const=0; }; -/** - * @brief Root database interface - */ -class IDatabase -{ -public: - virtual ~IDatabase() {} - - /** - * @brief Database backend type - */ - enum Type - { - T_UNKNOWN, - T_MEMORY, /**< In-memory database; ideal for gathering small groups of frequently-accessed objects */ - T_LOOSE, /**< Loose database; ideal for read/write database construction or platforms with good filesystems */ - T_PACKED /**< Packed database; ideal for read-only archived data */ - }; - virtual Type getType() const=0; - - /** - * @brief Database access type - */ - enum Access - { - A_INVALID, - A_READONLY, /**< Read-only access; packed databases always use this mode */ - A_READWRITE /**< Read/write access; used for building fresh databases */ - }; - virtual Access getAccess() const=0; - - /** - * @brief Lookup object by database primary-key - * @param id Primary-key of object - * @return Data object - */ - virtual const IDataObject* lookupObject(size_t id) const=0; - - /** - * @brief Lookup object by name - * @param name Name of object - * @return Data object - */ - virtual const IDataObject* lookupObject(const std::string& name) const=0; - - /** - * @brief Write a full copy of the database to another type/path - * @param type Type of new database - * @param path Target path of new database - * @return True on success - */ - virtual bool writeDatabase(IDatabase::Type type, const std::string& path) const=0; - -}; - -/** - * @brief Creates a new (empty) database - * @param type Type of new database - * @param access Requested level of access - * @return New database object - * - * Generally, the preferred method for working with HECL databases is via the - * IProject interface. NewProject() will automatically construct the necessary - * internal database objects. - */ -IDatabase* NewDatabase(IDatabase::Type type, IDatabase::Access access, const std::string& path); - - /** * @brief Base object to subclass for integrating with key project operations * @@ -272,24 +198,6 @@ public: C_HEAVY }; - /** - * @brief Access internal database interface for working files - * @return main working database object - * - * It's generally recommended for HECL frontends to avoid modifying - * databases via this returned object. - */ - virtual IDatabase* mainDatabase() const=0; - - /** - * @brief Access internal database interface for cooked objects - * @return main cooked database object - * - * It's generally recommended for HECL frontends to avoid modifying - * databases via this returned object. - */ - virtual IDatabase* cookedDatabase() const=0; - /** * @brief Register an optional callback to report log-messages using * @param logger logger-callback @@ -552,7 +460,7 @@ public: */ struct RegistryEntry { - typedef std::function TPathClaimer; + typedef std::function TPathClaimer; typedef std::function TProjectFactory; typedef std::function TRuntimeFactory; const HECL::FourCC& fcc; @@ -566,7 +474,7 @@ struct RegistryEntry }; static RegistryEntry::TPathClaimer NULL_PATH_CLAIMER = - [](const std::string&) -> bool {return false;}; + [](const std::string&, const std::string&) -> bool {return false;}; static RegistryEntry::TProjectFactory NULL_PROJECT_FACTORY = [](const HECLDatabase::CProjectObject::ConstructionInfo&) -> HECLDatabase::CProjectObject* {return nullptr;}; @@ -577,24 +485,27 @@ static RegistryEntry::TRuntimeFactory NULL_RUNTIME_FACTORY = #if !defined(HECL_STRIP_PROJECT) && !defined(HECL_STRIP_RUNTIME) #define REGISTRY_ENTRY(fourcc, projectClass, runtimeClass) {fourcc, \ -[](const std::string& path) -> bool {return projectClass::ClaimPath(path);}, \ +[](const std::string& path, const std::string& subpath) -> \ + bool {return projectClass::ClaimPath(path, subpath);}, \ [](const HECLDatabase::CProjectObject::ConstructionInfo& info) -> \ HECLDatabase::CProjectObject* {return new projectClass(info);}, \ [](const HECLDatabase::CRuntimeObject::ConstructionInfo& info) -> \ HECLDatabase::CRuntimeObject* {return new runtimeClass(info);}} #define REGISTRY_SENTINEL() \ - {HECL::FourCC(), NULL_PATH_CLAIMER, \ - NULL_PROJECT_FACTORY, NULL_RUNTIME_FACTORY} + {HECL::FourCC(), HECLDatabase::NULL_PATH_CLAIMER, \ + HECLDatabase::NULL_PROJECT_FACTORY, HECLDatabase::NULL_RUNTIME_FACTORY} #elif !defined(HECL_STRIP_PROJECT) #define REGISTRY_ENTRY(fourcc, projectClass, runtimeClass) {fourcc, \ -[](const std::string& path) -> bool {return projectClass::ClaimPath(path);}, \ +[](const std::string& path, const std::string& subpath) -> \ + bool {return projectClass::ClaimPath(path, subpath);}, \ [](const HECLDatabase::CProjectObject::ConstructionInfo& info) -> \ HECLDatabase::CProjectObject* {return new projectClass(info);}} -#define REGISTRY_SENTINEL() {HECL::FourCC(), NULL_PATH_CLAIMER, NULL_PROJECT_FACTORY} +#define REGISTRY_SENTINEL() {HECL::FourCC(), \ + HECLDatabase::NULL_PATH_CLAIMER, HECLDatabase::NULL_PROJECT_FACTORY} #elif !defined(HECL_STRIP_RUNTIME) @@ -602,7 +513,7 @@ static RegistryEntry::TRuntimeFactory NULL_RUNTIME_FACTORY = [](const HECLDatabase::CRuntimeObject::ConstructionInfo& info) -> \ HECLDatabase::CRuntimeObject* {return new runtimeClass(info);}} -#define REGISTRY_SENTINEL() {HECL::FourCC(), NULL_RUNTIME_FACTORY} +#define REGISTRY_SENTINEL() {HECL::FourCC(), HECLDatabase::NULL_RUNTIME_FACTORY} #endif diff --git a/hecl/lib/database/CLooseDatabase.hpp b/hecl/lib/database/CLooseDatabase.hpp deleted file mode 100644 index f026888b8..000000000 --- a/hecl/lib/database/CLooseDatabase.hpp +++ /dev/null @@ -1,82 +0,0 @@ -#ifndef CLOOSEDATABASE_HPP -#define CLOOSEDATABASE_HPP - -#include -#include -#include - -#include "HECLDatabase.hpp" -#include "CSQLite.hpp" - -namespace HECLDatabase -{ - -class CLooseDatabase final : public IDatabase -{ - CSQLite m_sql; - Access m_access; -public: - CLooseDatabase(const std::string& path, Access access) - : m_sql(path.c_str(), (access == A_READONLY) ? true : false), - m_access(access) - { - - } - - ~CLooseDatabase() - { - - } - - Type getType() const - { - return T_LOOSE; - } - - Access getAccess() const - { - return m_access; - } - - const IDataObject* lookupObject(size_t id) const - { - } - - const IDataObject* lookupObject(const std::string& name) const - { - } - - const IDataObject* addDataBlob(const std::string& name, const void* data, size_t length) - { - /* Hash data */ - HECL::ObjectHash hash(data, length); - - /* Compress data into file */ - FILE* fp = fopen("", "wb"); - - m_sql.insertObject(name, "DUMB", hash, length, length); - } - - const IDataObject* addDataBlob(const void* data, size_t length) - { - return addDataBlob(std::string(), data, length); - } - - bool writeDatabase(IDatabase::Type type, const std::string& path) const - { - if (type == T_PACKED) - { - size_t bufSz; - void* buf = m_sql.fillDBBuffer(bufSz); - FILE* fp = fopen(path.c_str(), "wb"); - fwrite(buf, 1, bufSz, fp); - return true; - } - return false; - } - -}; - -} - -#endif // CLOOSEDATABASE_HPP diff --git a/hecl/lib/database/CMemoryDatabase.hpp b/hecl/lib/database/CMemoryDatabase.hpp deleted file mode 100644 index 3918c540b..000000000 --- a/hecl/lib/database/CMemoryDatabase.hpp +++ /dev/null @@ -1,60 +0,0 @@ -#ifndef CMEMORYDATABASE_HPP -#define CMEMORYDATABASE_HPP - -#include "HECLDatabase.hpp" -#include "CSQLite.hpp" - -namespace HECLDatabase -{ - -class CMemoryDatabase final : public IDatabase -{ - CSQLite m_sql; - Access m_access; -public: - CMemoryDatabase(Access access) - : m_sql(":memory:", (m_access == A_READONLY) ? true : false), m_access(access) - { - - } - - ~CMemoryDatabase() - { - - } - - Type getType() const - { - return T_MEMORY; - } - - Access getAccess() const - { - return m_access; - } - - const IDataObject* lookupObject(size_t id) const - { - } - - const IDataObject* lookupObject(const std::string& name) const - { - } - - const IDataObject* addDataBlob(const std::string& name, const void* data, size_t length) - { - } - - const IDataObject* addDataBlob(const void* data, size_t length) - { - } - - bool writeDatabase(IDatabase::Type type, const std::string& path) const - { - } - -}; - -} - -#endif // CMEMORYDATABASE_HPP diff --git a/hecl/lib/database/CPackedDatabase.hpp b/hecl/lib/database/CPackedDatabase.hpp deleted file mode 100644 index 3ef2c9bb6..000000000 --- a/hecl/lib/database/CPackedDatabase.hpp +++ /dev/null @@ -1,59 +0,0 @@ -#ifndef CPACKEDDATABASE_HPP -#define CPACKEDDATABASE_HPP - -#include "HECLDatabase.hpp" -#include "CSQLite.hpp" - -namespace HECLDatabase -{ - -class CPackedDatabase final : public IDatabase -{ - CSQLite m_sql; -public: - CPackedDatabase(const std::string& path) - : m_sql(path.c_str(), true) - { - - } - - ~CPackedDatabase() - { - - } - - Type getType() const - { - return T_PACKED; - } - - Access getAccess() const - { - return A_READONLY; - } - - const IDataObject* lookupObject(size_t id) const - { - } - - const IDataObject* lookupObject(const std::string& name) const - { - } - - const IDataObject* addDataBlob(const std::string& name, const void* data, size_t length) - { - } - - const IDataObject* addDataBlob(const void* data, size_t length) - { - } - - bool writeDatabase(IDatabase::Type type, const std::string& path) const - { - } - -}; - -} - -#endif // CPACKEDDATABASE_HPP diff --git a/hecl/lib/database/CProject.cpp b/hecl/lib/database/CProject.cpp index 65b7a5ac4..338cd7e21 100644 --- a/hecl/lib/database/CProject.cpp +++ b/hecl/lib/database/CProject.cpp @@ -4,7 +4,7 @@ #include #include "HECLDatabase.hpp" -#include "CLooseDatabase.hpp" +#include "CSQLiteMain.hpp" namespace HECLDatabase { @@ -12,8 +12,7 @@ namespace HECLDatabase class CProject : public IProject { std::string m_rootPath; - IDatabase* m_mainDb; - IDatabase* m_cookedDb; + CSQLiteMain* m_db; public: CProject(const std::string& rootPath) : m_rootPath(rootPath) @@ -34,24 +33,12 @@ public: } /* Create or open databases */ - m_mainDb = new CLooseDatabase(m_rootPath + "/.hecl/main.db", IDatabase::A_READWRITE); - m_cookedDb = new CLooseDatabase(m_rootPath + "/.hecl/cooked.db", IDatabase::A_READWRITE); + m_db = new CSQLiteMain(m_rootPath + "/.hecl/main.db"); } ~CProject() { - delete m_mainDb; - delete m_cookedDb; - } - - IDatabase* mainDatabase() const - { - return m_mainDb; - } - - IDatabase* cookedDatabase() const - { - return m_cookedDb; + delete m_db; } void registerLogger(HECL::TLogger logger) diff --git a/hecl/lib/database/CRuntime.cpp b/hecl/lib/database/CRuntime.cpp index 38fe4d472..3a6731342 100644 --- a/hecl/lib/database/CRuntime.cpp +++ b/hecl/lib/database/CRuntime.cpp @@ -1,5 +1,6 @@ #include "HECLDatabase.hpp" +#include "CSQLiteMain.hpp" namespace HECLDatabase { diff --git a/hecl/lib/database/CSQLite.hpp b/hecl/lib/database/CSQLite.hpp deleted file mode 100644 index b60828ad4..000000000 --- a/hecl/lib/database/CSQLite.hpp +++ /dev/null @@ -1,208 +0,0 @@ -#ifndef CSQLITE_HPP -#define CSQLITE_HPP - -#include -#include -#include - -#include "HECLDatabase.hpp" -#include "sqlite_hecl_vfs.h" - -namespace HECLDatabase -{ - -/* Private sqlite3 backend to be used by database subclasses */ - -static const char* skMAINDBINIT = -"PRAGMA foreign_keys = ON;\n" -"CREATE TABLE IF NOT EXISTS grps(" - "grpid INTEGER PRIMARY KEY," /* Unique group identifier (used as in-game ref) */ - "path);\n" /* Directory path collecting working files for group */ -"CREATE TABLE IF NOT EXISTS objs(" - "objid INTEGER PRIMARY KEY," /* Unique object identifier (used as in-game ref) */ - "path," /* Path of working file */ - "subpath DEFAULT NULL," /* String name of sub-object within working file (i.e. blender object) */ - "cookedHash64 INTEGER DEFAULT NULL," /* Hash of last cooking pass */ - "cookedTime64 INTEGER DEFAULT NULL);\n"; /* UTC unix-time of last cooking pass */ - -static const char* skCOOKEDDBINIT = -"PRAGMA foreign_keys = ON;\n" -"CREATE TABLE IF NOT EXISTS cgrps(" - "grpid INTEGER PRIMARY KEY," /* Unique group identifier (from main DB) */ - "offset UNSIGNED INTEGER," /* Group-blob offset within package */ - "compLen UNSIGNED INTEGER," /* Compressed blob-length */ - "decompLen UNSIGNED INTEGER);\n" /* Decompressed blob-length */ -"CREATE TABLE IF NOT EXISTS cobjs(" - "objid INTEGER PRIMARY KEY," /* Unique object identifier (from main DB) */ - "type4cc UNSIGNED INTEGER," /* Type FourCC as claimed by first project class in dataspec */ - "loosegrp REFERENCES cgrps(grpid) ON DELETE SET NULL DEFAULT NULL);\n" /* single-object group of ungrouped object */ -"CREATE TABLE IF NOT EXISTS cgrplinks(" - "grpid REFERENCES cgrps(grpid) ON DELETE CASCADE," /* Group ref */ - "objid REFERENCES cobjs(objid) ON DELETE CASCADE," /* Object ref */ - "offset UNSIGNED INTEGER," /* Offset within decompressed group-blob */ - "decompLen UNSIGNED INTEGER," /* Decompressed object length */ - "UNIQUE (grpid, objid) ON CONFLICT IGNORE);\n" -"CREATE INDEX IF NOT EXISTS grpidx ON cgrplinks(grpid);\n"; - -#define PREPSTMT(stmtSrc, outVar)\ -if (sqlite3_prepare_v2(m_db, stmtSrc, 0, &outVar, NULL) != SQLITE_OK)\ -{\ - throw std::runtime_error(sqlite3_errmsg(m_db));\ - sqlite3_close(m_db);\ - return;\ -} - -class CSQLiteMain -{ - sqlite3* m_db; - - sqlite3_stmt* m_selObjs; - sqlite3_stmt* m_selGrps; - - - struct SCloseBuf - { - void* buf = NULL; - size_t sz = 0; - }; - static void _vfsClose(void* buf, size_t bufSz, SCloseBuf* ctx) - { - ctx->buf = buf; - ctx->sz = bufSz; - } - -public: - CSQLiteMain(const char* path, bool readonly) - { - /* Open database connection */ - int errCode = 0; - if ((errCode = sqlite3_open_v2(path, &m_db, readonly ? - SQLITE_OPEN_READONLY : - SQLITE_OPEN_READWRITE | - SQLITE_OPEN_CREATE, NULL)) != SQLITE_OK) - { - throw std::runtime_error(sqlite3_errstr(errCode)); - sqlite3_close(m_db); - return; - } - - /* Execute bootstrap statements */ - char* errMsg = NULL; - sqlite3_exec(m_db, skDBINIT, NULL, NULL, &errMsg); - if (errMsg) - { - throw std::runtime_error(errMsg); - sqlite3_free(errMsg); - sqlite3_close(m_db); - return; - } - - /* Precompile statements */ - PREPSTMT("SELECT rowid,name,type4cc,hash64,compLen,decompLen FROM objects", m_selObjects); - PREPSTMT("SELECT rowid FROM objects WHERE name=?1", m_selObjectByName); - PREPSTMT("SELECT DISTINCT groupId FROM deplinks", m_selDistictDepGroups); - PREPSTMT("SELECT DISTINCT objId FROM deplinks WHERE groupId=?1", m_selDepGroupObjects); - PREPSTMT("INSERT INTO objects(name,type4cc,hash64,compLen,decompLen) VALUES (?1,?2,?3,?4,?5)", m_insObject); - } - - ~CSQLiteMain() - { - sqlite3_finalize(m_selObjects); - sqlite3_finalize(m_selObjectByName); - sqlite3_finalize(m_selDistictDepGroups); - sqlite3_finalize(m_selDepGroupObjects); - sqlite3_finalize(m_insObject); - sqlite3_close(m_db); - } - - void buildMemoryIndex(const std::function // decompLen - objectAdder) - { - while (sqlite3_step(m_selObjects) == SQLITE_ROW) - { - /* <3 Move Lambdas!! */ - objectAdder(sqlite3_column_int64(m_selObjects, 0), - (const char*)sqlite3_column_text(m_selObjects, 1), - sqlite3_column_int(m_selObjects, 2), - sqlite3_column_int64(m_selObjects, 3), - sqlite3_column_int64(m_selObjects, 4), - sqlite3_column_int64(m_selObjects, 5)); - } - sqlite3_reset(m_selObjects); - } - - size_t objectIdFromName(const std::string& name) - { - sqlite3_bind_text(m_selObjectByName, 1, name.c_str(), name.length(), NULL); - size_t retval = 0; - if (sqlite3_step(m_selObjectByName) == SQLITE_ROW) - retval = sqlite3_column_int64(m_selObjectByName, 0); - sqlite3_reset(m_selObjectByName); - return retval; - } - - bool insertObject(const std::string& name, - const HECL::FourCC& type, - const HECL::ObjectHash& hash, - size_t compLen, size_t decompLen) - { - } - - void* fillDBBuffer(size_t& bufSzOut) const - { - /* Instructs vfs that a close operation is premature and buffer should be freed */ - sqlite_hecl_mem_vfs_register(NULL, NULL); - - /* Open pure-memory DB */ - sqlite3* memDb; - int errCode; - if ((errCode = sqlite3_open_v2("", &memDb, SQLITE_OPEN_READWRITE, "hecl_mem")) != SQLITE_OK) - { - throw std::runtime_error(sqlite3_errstr(errCode)); - sqlite3_close(memDb); - return NULL; - } - - /* Perform backup (row copy) */ - sqlite3_backup* backup = sqlite3_backup_init(memDb, "main", m_db, "main"); - if (!backup) - { - throw std::runtime_error(sqlite3_errmsg(memDb)); - sqlite3_close(memDb); - return NULL; - } - sqlite3_backup_step(backup, -1); - sqlite3_backup_finish(backup); - - /* Now a close operation is useful; register close callback */ - SCloseBuf closeBuf; - sqlite_hecl_mem_vfs_register((TCloseCallback)_vfsClose, &closeBuf); - sqlite3_close(memDb); - - /* This should be set by close callback */ - if (!closeBuf.buf) - { - throw std::runtime_error("close operation did not write buffer"); - return NULL; - } - - /* All good! */ - bufSzOut = closeBuf.sz; - return closeBuf.buf; - } - - static void freeDBBuffer(void* buf) - { - sqlite3_free(buf); - } - -}; - -} - -#endif // CSQLITE_HPP diff --git a/hecl/lib/database/CSQLiteCooked.hpp b/hecl/lib/database/CSQLiteCooked.hpp new file mode 100644 index 000000000..f3ca12747 --- /dev/null +++ b/hecl/lib/database/CSQLiteCooked.hpp @@ -0,0 +1,82 @@ +#ifndef CSQLITECOOKED_HPP +#define CSQLITECOOKED_HPP + +#include +#include +#include + +#include "HECLDatabase.hpp" +#include "sqlite_hecl_vfs.h" + +namespace HECLDatabase +{ + +static const char* skCOOKEDDBINIT = +"PRAGMA foreign_keys = ON;\n" +"CREATE TABLE IF NOT EXISTS cgrps(" + "grpid INTEGER PRIMARY KEY," /* Unique group identifier (from main DB) */ + "offset UNSIGNED INTEGER," /* Group-blob offset within package */ + "compLen UNSIGNED INTEGER," /* Compressed blob-length */ + "decompLen UNSIGNED INTEGER);\n" /* Decompressed blob-length */ +"CREATE TABLE IF NOT EXISTS cobjs(" + "objid INTEGER PRIMARY KEY," /* Unique object identifier (from main DB) */ + "type4cc UNSIGNED INTEGER," /* Type FourCC as claimed by first project class in dataspec */ + "loosegrp REFERENCES cgrps(grpid) ON DELETE SET NULL DEFAULT NULL);\n" /* single-object group of ungrouped object */ +"CREATE TABLE IF NOT EXISTS cgrplinks(" + "grpid REFERENCES cgrps(grpid) ON DELETE CASCADE," /* Group ref */ + "objid REFERENCES cobjs(objid) ON DELETE CASCADE," /* Object ref */ + "offset UNSIGNED INTEGER," /* Offset within decompressed group-blob */ + "decompLen UNSIGNED INTEGER," /* Decompressed object length */ + "UNIQUE (grpid, objid) ON CONFLICT IGNORE);\n" +"CREATE INDEX IF NOT EXISTS grpidx ON cgrplinks(grpid);\n"; + +#define PREPSTMT(stmtSrc, outVar)\ +if (sqlite3_prepare_v2(m_db, stmtSrc, 0, &outVar, NULL) != SQLITE_OK)\ +{\ + throw std::runtime_error(sqlite3_errmsg(m_db));\ + sqlite3_close(m_db);\ + return;\ +} + +class CSQLiteCooked +{ + sqlite3* m_db; + +public: + CSQLiteCooked(const char* path, bool readonly) + { + /* Open database connection */ + int errCode = 0; + if ((errCode = sqlite3_open_v2(path, &m_db, SQLITE_OPEN_READONLY, + "hecl_memlba")) != SQLITE_OK) + { + throw std::runtime_error(sqlite3_errstr(errCode)); + sqlite3_close(m_db); + return; + } + + /* Execute bootstrap statements */ + char* errMsg = NULL; + sqlite3_exec(m_db, skCOOKEDDBINIT, NULL, NULL, &errMsg); + if (errMsg) + { + throw std::runtime_error(errMsg); + sqlite3_free(errMsg); + sqlite3_close(m_db); + return; + } + + /* Precompile statements */ + + } + + ~CSQLiteCooked() + { + sqlite3_close(m_db); + } + +}; + +} + +#endif // CSQLITE_HPP diff --git a/hecl/lib/database/CSQLiteMain.hpp b/hecl/lib/database/CSQLiteMain.hpp new file mode 100644 index 000000000..de232bd27 --- /dev/null +++ b/hecl/lib/database/CSQLiteMain.hpp @@ -0,0 +1,134 @@ +#ifndef CSQLITEMAIN_HPP +#define CSQLITEMAIN_HPP + +#include +#include +#include + +#include "HECLDatabase.hpp" +#include "sqlite_hecl_vfs.h" + +namespace HECLDatabase +{ + +static const char* skMAINDBINIT = +"PRAGMA foreign_keys = ON;\n" +"CREATE TABLE IF NOT EXISTS grps(" + "grpid INTEGER PRIMARY KEY," /* Unique group identifier (used as in-game ref) */ + "path);\n" /* Directory path collecting working files for group */ +"CREATE TABLE IF NOT EXISTS objs(" + "objid INTEGER PRIMARY KEY," /* Unique object identifier (used as in-game ref) */ + "path," /* Path of working file */ + "subpath DEFAULT NULL," /* String name of sub-object within working file (i.e. blender object) */ + "cookedHash64 INTEGER DEFAULT NULL," /* Hash of last cooking pass */ + "cookedTime64 INTEGER DEFAULT NULL);\n"; /* UTC unix-time of last cooking pass */ + +#define PREPSTMT(stmtSrc, outVar)\ +if (sqlite3_prepare_v2(m_db, stmtSrc, 0, &outVar, NULL) != SQLITE_OK)\ +{\ + throw std::runtime_error(sqlite3_errmsg(m_db));\ + sqlite3_close(m_db);\ + return;\ +} + +class CSQLiteMain +{ + sqlite3* m_db; + + struct SCloseBuf + { + void* buf = NULL; + size_t sz = 0; + }; + static void _vfsClose(void* buf, size_t bufSz, SCloseBuf* ctx) + { + ctx->buf = buf; + ctx->sz = bufSz; + } + +public: + CSQLiteMain(const std::string& path) + { + /* Open database connection */ + int errCode = 0; + if ((errCode = sqlite3_open(path.c_str(), &m_db) != SQLITE_OK)) + { + throw std::runtime_error(sqlite3_errstr(errCode)); + sqlite3_close(m_db); + return; + } + + /* Execute bootstrap statements */ + char* errMsg = NULL; + sqlite3_exec(m_db, skMAINDBINIT, NULL, NULL, &errMsg); + if (errMsg) + { + throw std::runtime_error(errMsg); + sqlite3_free(errMsg); + sqlite3_close(m_db); + return; + } + + /* Precompile statements */ + + } + + ~CSQLiteMain() + { + sqlite3_close(m_db); + } + + + void* fillDBBuffer(size_t& bufSzOut) const + { + /* Instructs vfs that a close operation is premature and buffer should be freed */ + sqlite_hecl_mem_vfs_register(NULL, NULL); + + /* Open pure-memory DB */ + sqlite3* memDb; + int errCode; + if ((errCode = sqlite3_open_v2("", &memDb, SQLITE_OPEN_READWRITE, "hecl_mem")) != SQLITE_OK) + { + throw std::runtime_error(sqlite3_errstr(errCode)); + sqlite3_close(memDb); + return NULL; + } + + /* Perform backup (row copy) */ + sqlite3_backup* backup = sqlite3_backup_init(memDb, "main", m_db, "main"); + if (!backup) + { + throw std::runtime_error(sqlite3_errmsg(memDb)); + sqlite3_close(memDb); + return NULL; + } + sqlite3_backup_step(backup, -1); + sqlite3_backup_finish(backup); + + /* Now a close operation is useful; register close callback */ + SCloseBuf closeBuf; + sqlite_hecl_mem_vfs_register((TCloseCallback)_vfsClose, &closeBuf); + sqlite3_close(memDb); + + /* This should be set by close callback */ + if (!closeBuf.buf) + { + throw std::runtime_error("close operation did not write buffer"); + return NULL; + } + + /* All good! */ + bufSzOut = closeBuf.sz; + return closeBuf.buf; + } + + static void freeDBBuffer(void* buf) + { + sqlite3_free(buf); + } + +}; + +} + +#endif // CSQLITEMAIN_HPP diff --git a/hecl/lib/database/HECLDatabase.cpp b/hecl/lib/database/HECLDatabase.cpp deleted file mode 100644 index fa2b2de59..000000000 --- a/hecl/lib/database/HECLDatabase.cpp +++ /dev/null @@ -1,28 +0,0 @@ - - -#include "HECLDatabase.hpp" - -#include "CLooseDatabase.hpp" -#include "CPackedDatabase.hpp" -#include "CMemoryDatabase.hpp" - -namespace HECLDatabase -{ - -IDatabase* NewDatabase(IDatabase::Type type, IDatabase::Access access, const std::string& path) -{ - switch (type) - { - case IDatabase::T_LOOSE: - return new CLooseDatabase(path, access); - case IDatabase::T_PACKED: - return new CPackedDatabase(path); - case IDatabase::T_MEMORY: - return new CMemoryDatabase(access); - case IDatabase::T_UNKNOWN: - return nullptr; - } - return nullptr; -} - -} diff --git a/hecl/lib/database/database.pri b/hecl/lib/database/database.pri index 3951aa393..977f71b8d 100644 --- a/hecl/lib/database/database.pri +++ b/hecl/lib/database/database.pri @@ -1,12 +1,9 @@ HEADERS += \ - $$PWD/CPackedDatabase.hpp \ - $$PWD/CMemoryDatabase.hpp \ - $$PWD/CLooseDatabase.hpp \ - $$PWD/CSQLite.hpp \ - $$PWD/sqlite_hecl_vfs.h + $$PWD/sqlite_hecl_vfs.h \ + $$PWD/CSQLiteMain.hpp \ + $$PWD/CSQLiteCooked.hpp SOURCES += \ - $$PWD/HECLDatabase.cpp \ $$PWD/CRuntime.cpp \ $$PWD/CProject.cpp \ $$PWD/sqlite_hecl_mem_vfs.c \