AxioDL/metaforce
2
0
Fork 0
mirror of https://github.com/AxioDL/metaforce.git synced 2025-12-09 19:07:44 +00:00
Code Issues Projects Releases Wiki Activity

database refinements; blender shell

Browse Source
This commit is contained in:
Jack Andersen
2015-05-23 18:51:16 -10:00
parent 20ca4e407f
commit 461893d7a1
18 changed files with 488 additions and 571 deletions
Download Patch File Download Diff File Expand all files Collapse all files

82
hecl/lib/database/CLooseDatabase.hpp
View File

@@ -1,82 +0,0 @@
#ifndef CLOOSEDATABASE_HPP
#define CLOOSEDATABASE_HPP
#include <stdio.h>
#include <blowfish/blowfish.h>
#include <zlib/zlib.h>
#include "HECLDatabase.hpp"
#include "CSQLite.hpp"
namespace HECLDatabase
{
class CLooseDatabase final : public IDatabase
{
CSQLite m_sql;
Access m_access;
public:
CLooseDatabase(const std::string& path, Access access)
: m_sql(path.c_str(), (access == A_READONLY) ? true : false),
m_access(access)
{
}
~CLooseDatabase()
{
}
Type getType() const
{
return T_LOOSE;
}
Access getAccess() const
{
return m_access;
}
const IDataObject* lookupObject(size_t id) const
{
}
const IDataObject* lookupObject(const std::string& name) const
{
}
const IDataObject* addDataBlob(const std::string& name, const void* data, size_t length)
{
/* Hash data */
HECL::ObjectHash hash(data, length);
/* Compress data into file */
FILE* fp = fopen("", "wb");
m_sql.insertObject(name, "DUMB", hash, length, length);
}
const IDataObject* addDataBlob(const void* data, size_t length)
{
return addDataBlob(std::string(), data, length);
}
bool writeDatabase(IDatabase::Type type, const std::string& path) const
{
if (type == T_PACKED)
{
size_t bufSz;
void* buf = m_sql.fillDBBuffer(bufSz);
FILE* fp = fopen(path.c_str(), "wb");
fwrite(buf, 1, bufSz, fp);
return true;
}
return false;
}
};
}
#endif // CLOOSEDATABASE_HPP

60
hecl/lib/database/CMemoryDatabase.hpp
View File

@@ -1,60 +0,0 @@
#ifndef CMEMORYDATABASE_HPP
#define CMEMORYDATABASE_HPP
#include "HECLDatabase.hpp"
#include "CSQLite.hpp"
namespace HECLDatabase
{
class CMemoryDatabase final : public IDatabase
{
CSQLite m_sql;
Access m_access;
public:
CMemoryDatabase(Access access)
: m_sql(":memory:", (m_access == A_READONLY) ? true : false), m_access(access)
{
}
~CMemoryDatabase()
{
}
Type getType() const
{
return T_MEMORY;
}
Access getAccess() const
{
return m_access;
}
const IDataObject* lookupObject(size_t id) const
{
}
const IDataObject* lookupObject(const std::string& name) const
{
}
const IDataObject* addDataBlob(const std::string& name, const void* data, size_t length)
{
}
const IDataObject* addDataBlob(const void* data, size_t length)
{
}
bool writeDatabase(IDatabase::Type type, const std::string& path) const
{
}
};
}
#endif // CMEMORYDATABASE_HPP

59
hecl/lib/database/CPackedDatabase.hpp
View File

@@ -1,59 +0,0 @@
#ifndef CPACKEDDATABASE_HPP
#define CPACKEDDATABASE_HPP
#include "HECLDatabase.hpp"
#include "CSQLite.hpp"
namespace HECLDatabase
{
class CPackedDatabase final : public IDatabase
{
CSQLite m_sql;
public:
CPackedDatabase(const std::string& path)
: m_sql(path.c_str(), true)
{
}
~CPackedDatabase()
{
}
Type getType() const
{
return T_PACKED;
}
Access getAccess() const
{
return A_READONLY;
}
const IDataObject* lookupObject(size_t id) const
{
}
const IDataObject* lookupObject(const std::string& name) const
{
}
const IDataObject* addDataBlob(const std::string& name, const void* data, size_t length)
{
}
const IDataObject* addDataBlob(const void* data, size_t length)
{
}
bool writeDatabase(IDatabase::Type type, const std::string& path) const
{
}
};
}
#endif // CPACKEDDATABASE_HPP

21
hecl/lib/database/CProject.cpp
View File

@@ -4,7 +4,7 @@
#include <system_error>
#include "HECLDatabase.hpp"
#include "CLooseDatabase.hpp"
#include "CSQLiteMain.hpp"
namespace HECLDatabase
{
@@ -12,8 +12,7 @@ namespace HECLDatabase
class CProject : public IProject
{
std::string m_rootPath;
IDatabase* m_mainDb;
IDatabase* m_cookedDb;
CSQLiteMain* m_db;
public:
CProject(const std::string& rootPath)
: m_rootPath(rootPath)
@@ -34,24 +33,12 @@ public:
}
/* Create or open databases */
m_mainDb = new CLooseDatabase(m_rootPath + "/.hecl/main.db", IDatabase::A_READWRITE);
m_cookedDb = new CLooseDatabase(m_rootPath + "/.hecl/cooked.db", IDatabase::A_READWRITE);
m_db = new CSQLiteMain(m_rootPath + "/.hecl/main.db");
}
~CProject()
{
delete m_mainDb;
delete m_cookedDb;
}
IDatabase* mainDatabase() const
{
return m_mainDb;
}
IDatabase* cookedDatabase() const
{
return m_cookedDb;
delete m_db;
}
void registerLogger(HECL::TLogger logger)

1
hecl/lib/database/CRuntime.cpp
View File

@@ -1,5 +1,6 @@
#include "HECLDatabase.hpp"
#include "CSQLiteMain.hpp"
namespace HECLDatabase
{

208
hecl/lib/database/CSQLite.hpp
View File

@@ -1,208 +0,0 @@
#ifndef CSQLITE_HPP
#define CSQLITE_HPP
#include <sqlite3.h>
#include <stdexcept>
#include <functional>
#include "HECLDatabase.hpp"
#include "sqlite_hecl_vfs.h"
namespace HECLDatabase
{
/* Private sqlite3 backend to be used by database subclasses */
static const char* skMAINDBINIT =
"PRAGMA foreign_keys = ON;\n"
"CREATE TABLE IF NOT EXISTS grps("
"grpid INTEGER PRIMARY KEY," /* Unique group identifier (used as in-game ref) */
"path);\n" /* Directory path collecting working files for group */
"CREATE TABLE IF NOT EXISTS objs("
"objid INTEGER PRIMARY KEY," /* Unique object identifier (used as in-game ref) */
"path," /* Path of working file */
"subpath DEFAULT NULL," /* String name of sub-object within working file (i.e. blender object) */
"cookedHash64 INTEGER DEFAULT NULL," /* Hash of last cooking pass */
"cookedTime64 INTEGER DEFAULT NULL);\n"; /* UTC unix-time of last cooking pass */
static const char* skCOOKEDDBINIT =
"PRAGMA foreign_keys = ON;\n"
"CREATE TABLE IF NOT EXISTS cgrps("
"grpid INTEGER PRIMARY KEY," /* Unique group identifier (from main DB) */
"offset UNSIGNED INTEGER," /* Group-blob offset within package */
"compLen UNSIGNED INTEGER," /* Compressed blob-length */
"decompLen UNSIGNED INTEGER);\n" /* Decompressed blob-length */
"CREATE TABLE IF NOT EXISTS cobjs("
"objid INTEGER PRIMARY KEY," /* Unique object identifier (from main DB) */
"type4cc UNSIGNED INTEGER," /* Type FourCC as claimed by first project class in dataspec */
"loosegrp REFERENCES cgrps(grpid) ON DELETE SET NULL DEFAULT NULL);\n" /* single-object group of ungrouped object */
"CREATE TABLE IF NOT EXISTS cgrplinks("
"grpid REFERENCES cgrps(grpid) ON DELETE CASCADE," /* Group ref */
"objid REFERENCES cobjs(objid) ON DELETE CASCADE," /* Object ref */
"offset UNSIGNED INTEGER," /* Offset within decompressed group-blob */
"decompLen UNSIGNED INTEGER," /* Decompressed object length */
"UNIQUE (grpid, objid) ON CONFLICT IGNORE);\n"
"CREATE INDEX IF NOT EXISTS grpidx ON cgrplinks(grpid);\n";
#define PREPSTMT(stmtSrc, outVar)\
if (sqlite3_prepare_v2(m_db, stmtSrc, 0, &outVar, NULL) != SQLITE_OK)\
{\
throw std::runtime_error(sqlite3_errmsg(m_db));\
sqlite3_close(m_db);\
return;\
}
class CSQLiteMain
{
sqlite3* m_db;
sqlite3_stmt* m_selObjs;
sqlite3_stmt* m_selGrps;
struct SCloseBuf
{
void* buf = NULL;
size_t sz = 0;
};
static void _vfsClose(void* buf, size_t bufSz, SCloseBuf* ctx)
{
ctx->buf = buf;
ctx->sz = bufSz;
}
public:
CSQLiteMain(const char* path, bool readonly)
{
/* Open database connection */
int errCode = 0;
if ((errCode = sqlite3_open_v2(path, &m_db, readonly ?
SQLITE_OPEN_READONLY :
SQLITE_OPEN_READWRITE |
SQLITE_OPEN_CREATE, NULL)) != SQLITE_OK)
{
throw std::runtime_error(sqlite3_errstr(errCode));
sqlite3_close(m_db);
return;
}
/* Execute bootstrap statements */
char* errMsg = NULL;
sqlite3_exec(m_db, skDBINIT, NULL, NULL, &errMsg);
if (errMsg)
{
throw std::runtime_error(errMsg);
sqlite3_free(errMsg);
sqlite3_close(m_db);
return;
}
/* Precompile statements */
PREPSTMT("SELECT rowid,name,type4cc,hash64,compLen,decompLen FROM objects", m_selObjects);
PREPSTMT("SELECT rowid FROM objects WHERE name=?1", m_selObjectByName);
PREPSTMT("SELECT DISTINCT groupId FROM deplinks", m_selDistictDepGroups);
PREPSTMT("SELECT DISTINCT objId FROM deplinks WHERE groupId=?1", m_selDepGroupObjects);
PREPSTMT("INSERT INTO objects(name,type4cc,hash64,compLen,decompLen) VALUES (?1,?2,?3,?4,?5)", m_insObject);
}
~CSQLiteMain()
{
sqlite3_finalize(m_selObjects);
sqlite3_finalize(m_selObjectByName);
sqlite3_finalize(m_selDistictDepGroups);
sqlite3_finalize(m_selDepGroupObjects);
sqlite3_finalize(m_insObject);
sqlite3_close(m_db);
}
void buildMemoryIndex(const std::function<void(size_t&&, // id
const std::string&&, // name
uint32_t&&, // type4cc
uint64_t&&, // hash64
size_t&&, // compLen
size_t&&)> // decompLen
objectAdder)
{
while (sqlite3_step(m_selObjects) == SQLITE_ROW)
{
/* <3 Move Lambdas!! */
objectAdder(sqlite3_column_int64(m_selObjects, 0),
(const char*)sqlite3_column_text(m_selObjects, 1),
sqlite3_column_int(m_selObjects, 2),
sqlite3_column_int64(m_selObjects, 3),
sqlite3_column_int64(m_selObjects, 4),
sqlite3_column_int64(m_selObjects, 5));
}
sqlite3_reset(m_selObjects);
}
size_t objectIdFromName(const std::string& name)
{
sqlite3_bind_text(m_selObjectByName, 1, name.c_str(), name.length(), NULL);
size_t retval = 0;
if (sqlite3_step(m_selObjectByName) == SQLITE_ROW)
retval = sqlite3_column_int64(m_selObjectByName, 0);
sqlite3_reset(m_selObjectByName);
return retval;
}
bool insertObject(const std::string& name,
const HECL::FourCC& type,
const HECL::ObjectHash& hash,
size_t compLen, size_t decompLen)
{
}
void* fillDBBuffer(size_t& bufSzOut) const
{
/* Instructs vfs that a close operation is premature and buffer should be freed */
sqlite_hecl_mem_vfs_register(NULL, NULL);
/* Open pure-memory DB */
sqlite3* memDb;
int errCode;
if ((errCode = sqlite3_open_v2("", &memDb, SQLITE_OPEN_READWRITE, "hecl_mem")) != SQLITE_OK)
{
throw std::runtime_error(sqlite3_errstr(errCode));
sqlite3_close(memDb);
return NULL;
}
/* Perform backup (row copy) */
sqlite3_backup* backup = sqlite3_backup_init(memDb, "main", m_db, "main");
if (!backup)
{
throw std::runtime_error(sqlite3_errmsg(memDb));
sqlite3_close(memDb);
return NULL;
}
sqlite3_backup_step(backup, -1);
sqlite3_backup_finish(backup);
/* Now a close operation is useful; register close callback */
SCloseBuf closeBuf;
sqlite_hecl_mem_vfs_register((TCloseCallback)_vfsClose, &closeBuf);
sqlite3_close(memDb);
/* This should be set by close callback */
if (!closeBuf.buf)
{
throw std::runtime_error("close operation did not write buffer");
return NULL;
}
/* All good! */
bufSzOut = closeBuf.sz;
return closeBuf.buf;
}
static void freeDBBuffer(void* buf)
{
sqlite3_free(buf);
}
};
}
#endif // CSQLITE_HPP

82
hecl/lib/database/CSQLiteCooked.hpp Normal file
View File

@@ -0,0 +1,82 @@
#ifndef CSQLITECOOKED_HPP
#define CSQLITECOOKED_HPP
#include <sqlite3.h>
#include <stdexcept>
#include <functional>
#include "HECLDatabase.hpp"
#include "sqlite_hecl_vfs.h"
namespace HECLDatabase
{
static const char* skCOOKEDDBINIT =
"PRAGMA foreign_keys = ON;\n"
"CREATE TABLE IF NOT EXISTS cgrps("
"grpid INTEGER PRIMARY KEY," /* Unique group identifier (from main DB) */
"offset UNSIGNED INTEGER," /* Group-blob offset within package */
"compLen UNSIGNED INTEGER," /* Compressed blob-length */
"decompLen UNSIGNED INTEGER);\n" /* Decompressed blob-length */
"CREATE TABLE IF NOT EXISTS cobjs("
"objid INTEGER PRIMARY KEY," /* Unique object identifier (from main DB) */
"type4cc UNSIGNED INTEGER," /* Type FourCC as claimed by first project class in dataspec */
"loosegrp REFERENCES cgrps(grpid) ON DELETE SET NULL DEFAULT NULL);\n" /* single-object group of ungrouped object */
"CREATE TABLE IF NOT EXISTS cgrplinks("
"grpid REFERENCES cgrps(grpid) ON DELETE CASCADE," /* Group ref */
"objid REFERENCES cobjs(objid) ON DELETE CASCADE," /* Object ref */
"offset UNSIGNED INTEGER," /* Offset within decompressed group-blob */
"decompLen UNSIGNED INTEGER," /* Decompressed object length */
"UNIQUE (grpid, objid) ON CONFLICT IGNORE);\n"
"CREATE INDEX IF NOT EXISTS grpidx ON cgrplinks(grpid);\n";
#define PREPSTMT(stmtSrc, outVar)\
if (sqlite3_prepare_v2(m_db, stmtSrc, 0, &outVar, NULL) != SQLITE_OK)\
{\
throw std::runtime_error(sqlite3_errmsg(m_db));\
sqlite3_close(m_db);\
return;\
}
class CSQLiteCooked
{
sqlite3* m_db;
public:
CSQLiteCooked(const char* path, bool readonly)
{
/* Open database connection */
int errCode = 0;
if ((errCode = sqlite3_open_v2(path, &m_db, SQLITE_OPEN_READONLY,
"hecl_memlba")) != SQLITE_OK)
{
throw std::runtime_error(sqlite3_errstr(errCode));
sqlite3_close(m_db);
return;
}
/* Execute bootstrap statements */
char* errMsg = NULL;
sqlite3_exec(m_db, skCOOKEDDBINIT, NULL, NULL, &errMsg);
if (errMsg)
{
throw std::runtime_error(errMsg);
sqlite3_free(errMsg);
sqlite3_close(m_db);
return;
}
/* Precompile statements */
}
~CSQLiteCooked()
{
sqlite3_close(m_db);
}
};
}
#endif // CSQLITE_HPP

134
hecl/lib/database/CSQLiteMain.hpp Normal file
View File

@@ -0,0 +1,134 @@
#ifndef CSQLITEMAIN_HPP
#define CSQLITEMAIN_HPP
#include <sqlite3.h>
#include <stdexcept>
#include <functional>
#include "HECLDatabase.hpp"
#include "sqlite_hecl_vfs.h"
namespace HECLDatabase
{
static const char* skMAINDBINIT =
"PRAGMA foreign_keys = ON;\n"
"CREATE TABLE IF NOT EXISTS grps("
"grpid INTEGER PRIMARY KEY," /* Unique group identifier (used as in-game ref) */
"path);\n" /* Directory path collecting working files for group */
"CREATE TABLE IF NOT EXISTS objs("
"objid INTEGER PRIMARY KEY," /* Unique object identifier (used as in-game ref) */
"path," /* Path of working file */
"subpath DEFAULT NULL," /* String name of sub-object within working file (i.e. blender object) */
"cookedHash64 INTEGER DEFAULT NULL," /* Hash of last cooking pass */
"cookedTime64 INTEGER DEFAULT NULL);\n"; /* UTC unix-time of last cooking pass */
#define PREPSTMT(stmtSrc, outVar)\
if (sqlite3_prepare_v2(m_db, stmtSrc, 0, &outVar, NULL) != SQLITE_OK)\
{\
throw std::runtime_error(sqlite3_errmsg(m_db));\
sqlite3_close(m_db);\
return;\
}
class CSQLiteMain
{
sqlite3* m_db;
struct SCloseBuf
{
void* buf = NULL;
size_t sz = 0;
};
static void _vfsClose(void* buf, size_t bufSz, SCloseBuf* ctx)
{
ctx->buf = buf;
ctx->sz = bufSz;
}
public:
CSQLiteMain(const std::string& path)
{
/* Open database connection */
int errCode = 0;
if ((errCode = sqlite3_open(path.c_str(), &m_db) != SQLITE_OK))
{
throw std::runtime_error(sqlite3_errstr(errCode));
sqlite3_close(m_db);
return;
}
/* Execute bootstrap statements */
char* errMsg = NULL;
sqlite3_exec(m_db, skMAINDBINIT, NULL, NULL, &errMsg);
if (errMsg)
{
throw std::runtime_error(errMsg);
sqlite3_free(errMsg);
sqlite3_close(m_db);
return;
}
/* Precompile statements */
}
~CSQLiteMain()
{
sqlite3_close(m_db);
}
void* fillDBBuffer(size_t& bufSzOut) const
{
/* Instructs vfs that a close operation is premature and buffer should be freed */
sqlite_hecl_mem_vfs_register(NULL, NULL);
/* Open pure-memory DB */
sqlite3* memDb;
int errCode;
if ((errCode = sqlite3_open_v2("", &memDb, SQLITE_OPEN_READWRITE, "hecl_mem")) != SQLITE_OK)
{
throw std::runtime_error(sqlite3_errstr(errCode));
sqlite3_close(memDb);
return NULL;
}
/* Perform backup (row copy) */
sqlite3_backup* backup = sqlite3_backup_init(memDb, "main", m_db, "main");
if (!backup)
{
throw std::runtime_error(sqlite3_errmsg(memDb));
sqlite3_close(memDb);
return NULL;
}
sqlite3_backup_step(backup, -1);
sqlite3_backup_finish(backup);
/* Now a close operation is useful; register close callback */
SCloseBuf closeBuf;
sqlite_hecl_mem_vfs_register((TCloseCallback)_vfsClose, &closeBuf);
sqlite3_close(memDb);
/* This should be set by close callback */
if (!closeBuf.buf)
{
throw std::runtime_error("close operation did not write buffer");
return NULL;
}
/* All good! */
bufSzOut = closeBuf.sz;
return closeBuf.buf;
}
static void freeDBBuffer(void* buf)
{
sqlite3_free(buf);
}
};
}
#endif // CSQLITEMAIN_HPP

28
hecl/lib/database/HECLDatabase.cpp
View File

@@ -1,28 +0,0 @@
#include "HECLDatabase.hpp"
#include "CLooseDatabase.hpp"
#include "CPackedDatabase.hpp"
#include "CMemoryDatabase.hpp"
namespace HECLDatabase
{
IDatabase* NewDatabase(IDatabase::Type type, IDatabase::Access access, const std::string& path)
{
switch (type)
{
case IDatabase::T_LOOSE:
return new CLooseDatabase(path, access);
case IDatabase::T_PACKED:
return new CPackedDatabase(path);
case IDatabase::T_MEMORY:
return new CMemoryDatabase(access);
case IDatabase::T_UNKNOWN:
return nullptr;
}
return nullptr;
}
}

9
hecl/lib/database/database.pri
View File

@@ -1,12 +1,9 @@
HEADERS += \
$$PWD/CPackedDatabase.hpp \
$$PWD/CMemoryDatabase.hpp \
$$PWD/CLooseDatabase.hpp \
$$PWD/CSQLite.hpp \
$$PWD/sqlite_hecl_vfs.h
$$PWD/sqlite_hecl_vfs.h \
$$PWD/CSQLiteMain.hpp \
$$PWD/CSQLiteCooked.hpp
SOURCES += \
$$PWD/HECLDatabase.cpp \
$$PWD/CRuntime.cpp \
$$PWD/CProject.cpp \
$$PWD/sqlite_hecl_mem_vfs.c \