2
0
mirror of https://github.com/AxioDL/metaforce.git synced 2025-12-09 17:07:41 +00:00

Much refactoring

This commit is contained in:
Jack Andersen
2015-06-09 12:19:59 -10:00
parent 851221f861
commit 84f8c62d28
30 changed files with 825 additions and 1844 deletions

View File

@@ -1,18 +1,125 @@
#include <sys/stat.h>
#include <errno.h>
#include <stdio.h>
#include <string.h>
#include <system_error>
#include "HECLDatabase.hpp"
#include "CSQLiteMain.hpp"
namespace HECLDatabase
{
static inline bool CheckNewLineAdvance(std::string::const_iterator& it)
{
if (*it == '\n' || *it == '\0')
{
it += 1;
return true;
}
else if (*it == '\r')
{
if (*(it+1) == '\n')
{
it += 2;
return true;
}
}
return false;
}
class CProject : public IProject
{
std::string m_rootPath;
CSQLiteMain* m_db;
HECL::SystemString m_rootPath;
class ConfigFile
{
const CProject& m_project;
const HECL::SystemString& m_name;
HECL::SystemString m_filepath;
public:
ConfigFile(const CProject& project, const HECL::SystemString& name)
: m_project(project), m_name(name)
{
m_filepath = project.m_rootPath + _S("/.hecl/config/") + name;
}
std::vector<std::string> readLines()
{
FILE* fp = HECL::Fopen(m_filepath.c_str(), _S("r"));
std::string mainString;
char readBuf[1024];
size_t readSz;
while ((readSz = fread(readBuf, 1, 1024, fp)))
mainString += std::string(readBuf, readSz);
fclose(fp);
std::string::const_iterator begin = mainString.begin();
std::string::const_iterator end = mainString.begin();
std::vector<std::string> retval;
while (end != mainString.end())
{
std::string::const_iterator origEnd = end;
if (CheckNewLineAdvance(end))
{
if (begin != origEnd)
retval.push_back(std::string(begin, origEnd));
begin = end;
continue;
}
++end;
}
if (begin != end)
retval.push_back(std::string(begin, end));
return retval;
}
void addLine(const std::string& line)
{
std::vector<std::string> curLines = readLines();
FILE* fp = HECL::Fopen(m_filepath.c_str(), _S("w"));
for (std::string& line : curLines)
{
fwrite(line.data(), 1, line.length(), fp);
fwrite("\n", 1, 1, fp);
}
fwrite(line.data(), 1, line.length(), fp);
fwrite("\n", 1, 1, fp);
fclose(fp);
}
void removeLine(const std::string& refLine)
{
std::vector<std::string> curLines = readLines();
FILE* fp = HECL::Fopen(m_filepath.c_str(), _S("w"));
for (std::string& line : curLines)
{
if (line.compare(refLine))
{
fwrite(line.data(), 1, line.length(), fp);
fwrite("\n", 1, 1, fp);
}
}
fclose(fp);
}
bool checkForLine(const std::string& refLine)
{
std::vector<std::string> curLines = readLines();
for (std::string& line : curLines)
{
if (!line.compare(refLine))
return true;
}
return false;
}
};
public:
CProject(const std::string& rootPath)
: m_rootPath(rootPath)
@@ -25,20 +132,16 @@ public:
if (!S_ISDIR(myStat.st_mode))
throw std::invalid_argument("provided path must be a directory; '" + m_rootPath + "' isn't");
/* Create project directory */
if (mkdir((m_rootPath + "/.hecl").c_str(), 0755))
{
if (errno != EEXIST)
throw std::error_code(errno, std::system_category());
}
/* Create project directory structure */
HECL::MakeDir(m_rootPath + "/.hecl");
HECL::MakeDir(m_rootPath + "/.hecl/cooked");
HECL::MakeDir(m_rootPath + "/.hecl/config");
/* Create or open databases */
m_db = new CSQLiteMain(m_rootPath + "/.hecl/main.db");
}
~CProject()
{
delete m_db;
}
void registerLogger(HECL::TLogger logger)
@@ -49,11 +152,11 @@ public:
{
}
bool addPath(const std::string& path)
bool addPaths(const std::vector<HECL::ProjectPath>& paths)
{
}
bool removePath(const std::string& path, bool recursive)
bool removePaths(const std::vector<HECL::ProjectPath>& paths, bool recursive)
{
}
@@ -65,9 +168,21 @@ public:
{
}
const std::map<const std::string, const bool>& listPlatforms()
{
}
bool enablePlatforms(const std::vector<std::string>& platforms)
{
}
bool disablePlatforms(const std::vector<std::string>& platforms)
{
}
bool cookPath(const std::string& path,
std::function<void(std::string&, Cost, unsigned)> feedbackCb,
bool recursive)
std::function<void(std::string&, Cost, unsigned)> feedbackCb,
bool recursive)
{
}

View File

@@ -1,12 +0,0 @@
#include "HECLDatabase.hpp"
#include "CSQLiteMain.hpp"
namespace HECLDatabase
{
class CRuntime : public IRuntime
{
};
}

View File

@@ -1,82 +0,0 @@
#ifndef CSQLITECOOKED_HPP
#define CSQLITECOOKED_HPP
#include <sqlite3.h>
#include <stdexcept>
#include <functional>
#include "HECLDatabase.hpp"
#include "sqlite_hecl_vfs.h"
namespace HECLDatabase
{
static const char* skCOOKEDDBINIT =
"PRAGMA foreign_keys = ON;\n"
"CREATE TABLE IF NOT EXISTS cgrps("
"grpid INTEGER PRIMARY KEY," /* Unique group identifier (from main DB) */
"offset UNSIGNED INTEGER," /* Group-blob offset within package */
"compLen UNSIGNED INTEGER," /* Compressed blob-length */
"decompLen UNSIGNED INTEGER);\n" /* Decompressed blob-length */
"CREATE TABLE IF NOT EXISTS cobjs("
"objid INTEGER PRIMARY KEY," /* Unique object identifier (from main DB) */
"type4cc UNSIGNED INTEGER," /* Type FourCC as claimed by first project class in dataspec */
"loosegrp REFERENCES cgrps(grpid) ON DELETE SET NULL DEFAULT NULL);\n" /* single-object group of ungrouped object */
"CREATE TABLE IF NOT EXISTS cgrplinks("
"grpid REFERENCES cgrps(grpid) ON DELETE CASCADE," /* Group ref */
"objid REFERENCES cobjs(objid) ON DELETE CASCADE," /* Object ref */
"offset UNSIGNED INTEGER," /* Offset within decompressed group-blob */
"decompLen UNSIGNED INTEGER," /* Decompressed object length */
"UNIQUE (grpid, objid) ON CONFLICT IGNORE);\n"
"CREATE INDEX IF NOT EXISTS grpidx ON cgrplinks(grpid);\n";
#define PREPSTMT(stmtSrc, outVar)\
if (sqlite3_prepare_v2(m_db, stmtSrc, 0, &outVar, NULL) != SQLITE_OK)\
{\
throw std::runtime_error(sqlite3_errmsg(m_db));\
sqlite3_close(m_db);\
return;\
}
class CSQLiteCooked
{
sqlite3* m_db;
public:
CSQLiteCooked(const char* path, bool readonly)
{
/* Open database connection */
int errCode = 0;
if ((errCode = sqlite3_open_v2(path, &m_db, SQLITE_OPEN_READONLY,
"hecl_memlba")) != SQLITE_OK)
{
throw std::runtime_error(sqlite3_errstr(errCode));
sqlite3_close(m_db);
return;
}
/* Execute bootstrap statements */
char* errMsg = NULL;
sqlite3_exec(m_db, skCOOKEDDBINIT, NULL, NULL, &errMsg);
if (errMsg)
{
throw std::runtime_error(errMsg);
sqlite3_free(errMsg);
sqlite3_close(m_db);
return;
}
/* Precompile statements */
}
~CSQLiteCooked()
{
sqlite3_close(m_db);
}
};
}
#endif // CSQLITE_HPP

View File

@@ -1,134 +0,0 @@
#ifndef CSQLITEMAIN_HPP
#define CSQLITEMAIN_HPP
#include <sqlite3.h>
#include <stdexcept>
#include <functional>
#include "HECLDatabase.hpp"
#include "sqlite_hecl_vfs.h"
namespace HECLDatabase
{
static const char* skMAINDBINIT =
"PRAGMA foreign_keys = ON;\n"
"CREATE TABLE IF NOT EXISTS grps("
"grpid INTEGER PRIMARY KEY," /* Unique group identifier (used as in-game ref) */
"path);\n" /* Directory path collecting working files for group */
"CREATE TABLE IF NOT EXISTS objs("
"objid INTEGER PRIMARY KEY," /* Unique object identifier (used as in-game ref) */
"path," /* Path of working file */
"subpath DEFAULT NULL," /* String name of sub-object within working file (i.e. blender object) */
"cookedHash64 INTEGER DEFAULT NULL," /* Hash of last cooking pass */
"cookedTime64 INTEGER DEFAULT NULL);\n"; /* UTC unix-time of last cooking pass */
#define PREPSTMT(stmtSrc, outVar)\
if (sqlite3_prepare_v2(m_db, stmtSrc, 0, &outVar, NULL) != SQLITE_OK)\
{\
throw std::runtime_error(sqlite3_errmsg(m_db));\
sqlite3_close(m_db);\
return;\
}
class CSQLiteMain
{
sqlite3* m_db;
struct SCloseBuf
{
void* buf = NULL;
size_t sz = 0;
};
static void _vfsClose(void* buf, size_t bufSz, SCloseBuf* ctx)
{
ctx->buf = buf;
ctx->sz = bufSz;
}
public:
CSQLiteMain(const std::string& path)
{
/* Open database connection */
int errCode = 0;
if ((errCode = sqlite3_open(path.c_str(), &m_db) != SQLITE_OK))
{
throw std::runtime_error(sqlite3_errstr(errCode));
sqlite3_close(m_db);
return;
}
/* Execute bootstrap statements */
char* errMsg = NULL;
sqlite3_exec(m_db, skMAINDBINIT, NULL, NULL, &errMsg);
if (errMsg)
{
throw std::runtime_error(errMsg);
sqlite3_free(errMsg);
sqlite3_close(m_db);
return;
}
/* Precompile statements */
}
~CSQLiteMain()
{
sqlite3_close(m_db);
}
void* fillDBBuffer(size_t& bufSzOut) const
{
/* Instructs vfs that a close operation is premature and buffer should be freed */
sqlite_hecl_mem_vfs_register(NULL, NULL);
/* Open pure-memory DB */
sqlite3* memDb;
int errCode;
if ((errCode = sqlite3_open_v2("", &memDb, SQLITE_OPEN_READWRITE, "hecl_mem")) != SQLITE_OK)
{
throw std::runtime_error(sqlite3_errstr(errCode));
sqlite3_close(memDb);
return NULL;
}
/* Perform backup (row copy) */
sqlite3_backup* backup = sqlite3_backup_init(memDb, "main", m_db, "main");
if (!backup)
{
throw std::runtime_error(sqlite3_errmsg(memDb));
sqlite3_close(memDb);
return NULL;
}
sqlite3_backup_step(backup, -1);
sqlite3_backup_finish(backup);
/* Now a close operation is useful; register close callback */
SCloseBuf closeBuf;
sqlite_hecl_mem_vfs_register((TCloseCallback)_vfsClose, &closeBuf);
sqlite3_close(memDb);
/* This should be set by close callback */
if (!closeBuf.buf)
{
throw std::runtime_error("close operation did not write buffer");
return NULL;
}
/* All good! */
bufSzOut = closeBuf.sz;
return closeBuf.buf;
}
static void freeDBBuffer(void* buf)
{
sqlite3_free(buf);
}
};
}
#endif // CSQLITEMAIN_HPP

View File

@@ -1,11 +1,4 @@
HEADERS += \
$$PWD/sqlite_hecl_vfs.h \
$$PWD/CSQLiteMain.hpp \
$$PWD/CSQLiteCooked.hpp
HEADERS +=
SOURCES += \
$$PWD/CRuntime.cpp \
$$PWD/CProject.cpp \
$$PWD/sqlite_hecl_mem_vfs.c \
$$PWD/sqlite_hecl_memlba_vfs.c \
$$PWD/sqlite_hecl_memlba_make.c
$$PWD/CProject.cpp

View File

@@ -1,386 +0,0 @@
#include "sqlite_hecl_vfs.h"
#include "sqlite3.h"
#include <assert.h>
#include <string.h>
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
/*
* Modified test_onefile.c VFS for sqlite3
*
* This VFS gets registered with sqlite to access an in-memory,
* block-compressed LBA. It's designed for read-only access of
* a packed object-database.
*
* Journal and temp read/write is unsupported and will call abort
* if attempted.
*/
/*
* App-supplied callback called when file is closed (ready to access)
*/
static TCloseCallback CLOSE_CALLBACK = NULL;
static void* CLOSE_CTX = NULL;
typedef struct mem_file mem_file;
struct mem_file
{
sqlite3_file base;
int nSize;
int nAlloc;
char* zAlloc;
};
/*
** Method declarations for mem_file.
*/
static int memClose(sqlite3_file*);
static int memRead(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst);
static int memWrite(sqlite3_file*, const void*, int iAmt, sqlite3_int64 iOfst);
static int memTruncate(sqlite3_file*, sqlite3_int64 size);
static int memSync(sqlite3_file*, int flags);
static int memFileSize(sqlite3_file*, sqlite3_int64* pSize);
static int memLock(sqlite3_file*, int);
static int memUnlock(sqlite3_file*, int);
static int memCheckReservedLock(sqlite3_file*, int* pResOut);
static int memFileControl(sqlite3_file*, int op, void* pArg);
static int memSectorSize(sqlite3_file*);
static int memDeviceCharacteristics(sqlite3_file*);
/*
** Method declarations for fs_vfs.
*/
static int memOpen(sqlite3_vfs*, const char*, sqlite3_file*, int , int*);
static int memDelete(sqlite3_vfs*, const char* zName, int syncDir);
static int memAccess(sqlite3_vfs*, const char* zName, int flags, int*);
static int memFullPathname(sqlite3_vfs*, const char* zName, int nOut, char* zOut);
static void* memDlOpen(sqlite3_vfs*, const char* zFilename);
static void memDlError(sqlite3_vfs*, int nByte, char* zErrMsg);
static void (*memDlSym(sqlite3_vfs*, void*, const char* zSymbol))(void);
static void memDlClose(sqlite3_vfs*, void*);
static int memRandomness(sqlite3_vfs*, int nByte, char* zOut);
static int memSleep(sqlite3_vfs*, int microseconds);
static int memCurrentTime(sqlite3_vfs*, double*);
static sqlite3_vfs mem_vfs =
{
1, /* iVersion */
0, /* szOsFile */
0, /* mxPathname */
0, /* pNext */
"hecl_mem", /* zName */
0, /* pAppData */
memOpen, /* xOpen */
memDelete, /* xDelete */
memAccess, /* xAccess */
memFullPathname, /* xFullPathname */
memDlOpen, /* xDlOpen */
memDlError, /* xDlError */
memDlSym, /* xDlSym */
memDlClose, /* xDlClose */
memRandomness, /* xRandomness */
memSleep, /* xSleep */
memCurrentTime, /* xCurrentTime */
0 /* xCurrentTimeInt64 */
};
static sqlite3_io_methods mem_io_methods =
{
1, /* iVersion */
memClose, /* xClose */
memRead, /* xRead */
memWrite, /* xWrite */
memTruncate, /* xTruncate */
memSync, /* xSync */
memFileSize, /* xFileSize */
memLock, /* xLock */
memUnlock, /* xUnlock */
memCheckReservedLock, /* xCheckReservedLock */
memFileControl, /* xFileControl */
memSectorSize, /* xSectorSize */
memDeviceCharacteristics, /* xDeviceCharacteristics */
0, /* xShmMap */
0, /* xShmLock */
0, /* xShmBarrier */
0, /* xShmUnmap */
0,
0
};
/* Useful macros used in several places */
#define MIN(x,y) ((x)<(y)?(x):(y))
#define MAX(x,y) ((x)>(y)?(x):(y))
/*
** Close a mem-file.
*/
static int memClose(sqlite3_file* pFile)
{
mem_file* pTmp = (mem_file*)pFile;
if(CLOSE_CALLBACK)
CLOSE_CALLBACK(pTmp->zAlloc, pTmp->nSize, CLOSE_CTX);
else
sqlite3_free(pTmp->zAlloc);
return SQLITE_OK;
}
/*
** Read data from a mem-file.
*/
static int memRead(
sqlite3_file* pFile,
void* zBuf,
int iAmt,
sqlite_int64 iOfst
)
{
mem_file* pTmp = (mem_file*)pFile;
if((iAmt + iOfst) > pTmp->nSize)
return SQLITE_IOERR_SHORT_READ;
memcpy(zBuf, &pTmp->zAlloc[iOfst], iAmt);
return SQLITE_OK;
}
/*
** Write data to a mem-file.
*/
static int memWrite(
sqlite3_file* pFile,
const void* zBuf,
int iAmt,
sqlite_int64 iOfst
)
{
mem_file* pTmp = (mem_file*)pFile;
if((iAmt + iOfst) > pTmp->nAlloc)
{
int nNew = (int)(2 * (iAmt + iOfst + pTmp->nAlloc));
char* zNew = sqlite3_realloc(pTmp->zAlloc, nNew);
if(!zNew)
return SQLITE_NOMEM;
pTmp->zAlloc = zNew;
pTmp->nAlloc = nNew;
}
memcpy(&pTmp->zAlloc[iOfst], zBuf, iAmt);
pTmp->nSize = (int)MAX(pTmp->nSize, iOfst + iAmt);
return SQLITE_OK;
}
/*
** Truncate a mem-file.
*/
static int memTruncate(sqlite3_file* pFile, sqlite_int64 size)
{
mem_file* pTmp = (mem_file*)pFile;
pTmp->nSize = (int)MIN(pTmp->nSize, size);
return SQLITE_OK;
}
/*
** Sync a mem-file.
*/
static int memSync(sqlite3_file* pFile, int flags)
{
return SQLITE_OK;
}
/*
** Return the current file-size of a mem-file.
*/
static int memFileSize(sqlite3_file* pFile, sqlite_int64* pSize)
{
mem_file* pTmp = (mem_file*)pFile;
*pSize = pTmp->nSize;
return SQLITE_OK;
}
/*
** Lock a mem-file.
*/
static int memLock(sqlite3_file* pFile, int eLock)
{
return SQLITE_OK;
}
/*
** Unlock a mem-file.
*/
static int memUnlock(sqlite3_file* pFile, int eLock)
{
return SQLITE_OK;
}
/*
** Check if another file-handle holds a RESERVED lock on a mem-file.
*/
static int memCheckReservedLock(sqlite3_file* pFile, int* pResOut)
{
*pResOut = 0;
return SQLITE_OK;
}
/*
** File control method. For custom operations on a mem-file.
*/
static int memFileControl(sqlite3_file* pFile, int op, void* pArg)
{
return SQLITE_OK;
}
/*
** Return the sector-size in bytes for a mem-file.
*/
static int memSectorSize(sqlite3_file* pFile)
{
return 0;
}
/*
** Return the device characteristic flags supported by a mem-file.
*/
static int memDeviceCharacteristics(sqlite3_file* pFile)
{
return 0;
}
/*
** Open an mem file handle.
*/
static int memOpen(
sqlite3_vfs* pVfs,
const char* zName,
sqlite3_file* pFile,
int flags,
int* pOutFlags
)
{
if ((flags & SQLITE_OPEN_MAIN_DB) != SQLITE_OPEN_MAIN_DB)
{
fprintf(stderr, "the sqlite hecl mem VFS only supports main-db reading/writing\n");
return SQLITE_CANTOPEN;
}
mem_file* p2 = (mem_file*)pFile;
memset(p2, 0, sizeof(*p2));
p2->base.pMethods = &mem_io_methods;
return SQLITE_OK;
}
/*
** Delete the file located at zPath. If the dirSync argument is true,
** ensure the file-system modifications are synced to disk before
** returning.
*/
static int memDelete(sqlite3_vfs* pVfs, const char* zPath, int dirSync)
{
return SQLITE_OK;
}
/*
** Test for access permissions. Return true if the requested permission
** is available, or false otherwise.
*/
static int memAccess(
sqlite3_vfs* pVfs,
const char* zPath,
int flags,
int* pResOut
)
{
if(flags & SQLITE_ACCESS_READ | SQLITE_ACCESS_READWRITE)
return 1;
return 0;
}
/*
** Populate buffer zOut with the full canonical pathname corresponding
** to the pathname in zPath. zOut is guaranteed to point to a buffer
** of at least (FS_MAX_PATHNAME+1) bytes.
*/
static int memFullPathname(
sqlite3_vfs* pVfs, /* Pointer to vfs object */
const char* zPath, /* Possibly relative input path */
int nOut, /* Size of output buffer in bytes */
char* zOut) /* Output buffer */
{
strncpy(zOut, zPath, nOut);
return SQLITE_OK;
}
/*
** Open the dynamic library located at zPath and return a handle.
*/
static void* memDlOpen(sqlite3_vfs* pVfs, const char* zPath)
{
return NULL;
}
/*
** Populate the buffer zErrMsg (size nByte bytes) with a human readable
** utf-8 string describing the most recent error encountered associated
** with dynamic libraries.
*/
static void memDlError(sqlite3_vfs* pVfs, int nByte, char* zErrMsg)
{
}
/*
** Return a pointer to the symbol zSymbol in the dynamic library pHandle.
*/
static void (*memDlSym(sqlite3_vfs* pVfs, void* pH, const char* zSym))(void)
{
}
/*
** Close the dynamic library handle pHandle.
*/
static void memDlClose(sqlite3_vfs* pVfs, void* pHandle)
{
}
/*
** Populate the buffer pointed to by zBufOut with nByte bytes of
** random data.
*/
static int memRandomness(sqlite3_vfs* pVfs, int nByte, char* zBufOut)
{
for(int i = 0 ; i < nByte ; ++i)
zBufOut[i] = rand();
return nByte;
}
/*
** Sleep for nMicro microseconds. Return the number of microseconds
** actually slept.
*/
static int memSleep(sqlite3_vfs* pVfs, int nMicro)
{
int seconds = (nMicro + 999999) / 1000000;
sleep(seconds);
return seconds * 1000000;
}
/*
** Return the current time as a Julian Day number in *pTimeOut.
*/
static int memCurrentTime(sqlite3_vfs* pVfs, double* pTimeOut)
{
*pTimeOut = 0.0;
return 0;
}
/*
** This procedure registers the mem vfs with SQLite. If the argument is
** true, the mem vfs becomes the new default vfs. It is the only publicly
** available function in this file.
*/
int sqlite_hecl_mem_vfs_register(TCloseCallback closeCb, void* ctx)
{
CLOSE_CALLBACK = closeCb;
CLOSE_CTX = ctx;
if(mem_vfs.szOsFile) return SQLITE_OK;
mem_vfs.szOsFile = sizeof(mem_file);
return sqlite3_vfs_register(&mem_vfs, 0);
}

View File

@@ -1,108 +0,0 @@
#include "sqlite_hecl_vfs.h"
#include <zlib/zlib.h>
#include <stdint.h>
#include <stdbool.h>
/*
* Block-compression LBA generator used for storing packed sqlite3 database
*/
#define BLOCK_SIZE 0x4000
#define ROUND_UP_BLOCK(val) (((val) + (BLOCK_SIZE-1)) & ~(BLOCK_SIZE-1))
#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
static inline uint32_t makeu32(uint32_t val, bool bigEndian)
{
if (!bigEndian)
return val;
#if __GNUC__
return __builtin_bswap32(val);
#elif _WIN32
return _byteswap_ulong(val);
#else
val = (val & 0x0000FFFF) << 16 | (val & 0xFFFF0000) >> 16;
val = (val & 0x00FF00FF) << 8 | (val & 0xFF00FF00) >> 8;
return val;
#endif
}
#else
static inline uint32_t makeu32(uint32_t val, bool bigEndian)
{
if (bigEndian)
return val;
#if __GNUC__
return __builtin_bswap32(val);
#elif _WIN32
return _byteswap_ulong(val);
#else
val = (val & 0x0000FFFF) << 16 | (val & 0xFFFF0000) >> 16;
val = (val & 0x00FF00FF) << 8 | (val & 0xFF00FF00) >> 8;
return val;
#endif
}
#endif
/* Useful macros used in several places */
#define MIN(x,y) ((x)<(y)?(x):(y))
#define MAX(x,y) ((x)>(y)?(x):(y))
void sqlite_hecl_memlba_make(FILE* fout, void* bufin, size_t bufinLen, bool bigEndian)
{
unsigned i;
/* Magic */
fwrite("HLPK", 1, 4, fout);
/* Block size */
uint32_t blockSize_s = makeu32(BLOCK_SIZE, bigEndian);
fwrite(&blockSize_s, 1, 4, fout);
/* Block count */
size_t blockCount = ROUND_UP_BLOCK(bufinLen) / BLOCK_SIZE;
uint32_t blockCount_s = makeu32(blockCount, bigEndian);
fwrite(&blockCount_s, 1, 4, fout);
/* Header+TOC+DB Size */
fwrite("\0\0\0\0", 1, 4, fout);
/* Block TOC buffer and file-space */
uint32_t* blockTOC = calloc(blockCount+1, 4);
for (i=0 ; i<blockCount+1 ; ++i)
fwrite("\0\0\0\0", 1, 4, fout);
/* Block-compression context */
z_stream zstrm = {};
deflateInit(&zstrm, 7);
zstrm.next_in = bufin;
/* Compress! */
size_t curOff = 16 + (blockCount+1) * 4;
size_t remSz = bufinLen;
for (i=0 ; i<blockCount ; ++i)
{
unsigned char compBuf[BLOCK_SIZE*2];
zstrm.avail_in = MIN(remSz, BLOCK_SIZE);
zstrm.avail_out = BLOCK_SIZE*2;
zstrm.next_out = compBuf;
deflate(&zstrm, Z_FINISH);
fwrite(compBuf, 1, zstrm.total_out, fout);
blockTOC[i] = makeu32(curOff, bigEndian);
curOff += zstrm.total_out;
remSz -= zstrm.total_in;
deflateReset(&zstrm);
}
/* Write Header+TOC+DB Size */
fseek(fout, 12, SEEK_SET);
uint32_t headSz_s = makeu32(curOff, bigEndian);
fwrite(&headSz_s, 1, 4, fout);
/* Write TOC */
blockTOC[blockCount] = headSz_s;
fwrite(blockTOC, 4, blockCount+1, fout);
/* Cleanup */
deflateEnd(&zstrm);
free(blockTOC);
}

View File

@@ -1,493 +0,0 @@
#include "sqlite_hecl_vfs.h"
#include "sqlite3.h"
#include <assert.h>
#include <string.h>
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <stdint.h>
#include <zlib/zlib.h>
/*
* Modified test_onefile.c VFS for sqlite3
*
* This VFS gets registered with sqlite to access an in-memory,
* block-compressed LBA. It's designed for read-only access of
* a packed object-database.
*
* Journal and temp read/write is unsupported and will call abort
* if attempted.
*/
/*
* App-supplied pointer to head buffer (Header+TOC+DB)
*/
static void* HEAD_BUF = NULL;
#define BLOCK_SLOTS 4
typedef struct memlba_file memlba_file;
struct memlba_file
{
sqlite3_file base;
struct
{
char magic[4];
uint32_t blockSize;
uint32_t blockCount;
uint32_t headSz;
uint32_t blockTOC[];
}* headBuf;
void* cachedBlockBufs[BLOCK_SLOTS];
/* All initialized to -1 */
int cachedBlockIndices[BLOCK_SLOTS];
/* Ages start at 0, newly inserted block is 1.
* Non-0 blocks incremented on every insertion.
* If any slot is BLOCK_SLOTS upon insertion, surrounding blocks
* are incremented and that slot is reset to 1 (oldest-block caching)
*/
int cachedBlockAges[BLOCK_SLOTS];
z_stream zstrm;
};
static int newBlockSlot(memlba_file* file)
{
unsigned i;
for (i=0 ; i<BLOCK_SLOTS ; ++i)
if (file->cachedBlockAges[i] != 0)
++file->cachedBlockAges[i];
for (i=0 ; i<BLOCK_SLOTS ; ++i)
if (file->cachedBlockAges[i] == 0)
{
file->cachedBlockAges[i] = 1;
return i;
}
for (i=0 ; i<BLOCK_SLOTS ; ++i)
if (file->cachedBlockAges[i] == BLOCK_SLOTS+1)
{
file->cachedBlockAges[i] = 1;
return i;
}
/* Shouldn't happen (fallback) */
for (i=1 ; i<BLOCK_SLOTS ; ++i)
{
file->cachedBlockAges[i] = 0;
file->cachedBlockIndices[i] = -1;
}
file->cachedBlockAges[0] = 1;
return 0;
}
static void decompressBlock(memlba_file* file, unsigned blockIdx, int targetSlot)
{
if (blockIdx >= file->headBuf->blockCount)
{
fprintf(stderr, "exceeded memlba block range");
abort();
}
void* dbBlock = ((void*)file->headBuf) + file->headBuf->blockTOC[blockIdx];
file->zstrm.next_in = dbBlock;
file->zstrm.avail_in = file->headBuf->blockTOC[blockIdx+1] - file->headBuf->blockTOC[blockIdx];
file->zstrm.next_out = file->cachedBlockBufs[targetSlot];
file->zstrm.avail_out = file->headBuf->blockSize;
inflate(&file->zstrm, Z_FINISH);
inflateReset(&file->zstrm);
}
static int getBlockSlot(memlba_file* file, int blockIdx)
{
unsigned i;
for (i=0 ; i<BLOCK_SLOTS ; ++i)
if (file->cachedBlockIndices[i] != blockIdx)
return i;
int newSlot = newBlockSlot(file);
file->cachedBlockIndices[newSlot] = blockIdx;
decompressBlock(file, blockIdx, newSlot);
return newSlot;
}
/*
** Method declarations for memlba_file.
*/
static int memlbaClose(sqlite3_file*);
static int memlbaRead(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst);
static int memlbaWrite(sqlite3_file*, const void*, int iAmt, sqlite3_int64 iOfst);
static int memlbaTruncate(sqlite3_file*, sqlite3_int64 size);
static int memlbaSync(sqlite3_file*, int flags);
static int memlbaFileSize(sqlite3_file*, sqlite3_int64* pSize);
static int memlbaLock(sqlite3_file*, int);
static int memlbaUnlock(sqlite3_file*, int);
static int memlbaCheckReservedLock(sqlite3_file*, int* pResOut);
static int memlbaFileControl(sqlite3_file*, int op, void* pArg);
static int memlbaSectorSize(sqlite3_file*);
static int memlbaDeviceCharacteristics(sqlite3_file*);
/*
** Method declarations for fs_vfs.
*/
static int memlbaOpen(sqlite3_vfs*, const char*, sqlite3_file*, int , int*);
static int memlbaDelete(sqlite3_vfs*, const char* zName, int syncDir);
static int memlbaAccess(sqlite3_vfs*, const char* zName, int flags, int*);
static int memlbaFullPathname(sqlite3_vfs*, const char* zName, int nOut, char* zOut);
static void* memlbaDlOpen(sqlite3_vfs*, const char* zFilename);
static void memlbaDlError(sqlite3_vfs*, int nByte, char* zErrMsg);
static void (*memlbaDlSym(sqlite3_vfs*, void*, const char* zSymbol))(void);
static void memlbaDlClose(sqlite3_vfs*, void*);
static int memlbaRandomness(sqlite3_vfs*, int nByte, char* zOut);
static int memlbaSleep(sqlite3_vfs*, int microseconds);
static int memlbaCurrentTime(sqlite3_vfs*, double*);
static sqlite3_vfs memlba_vfs =
{
1, /* iVersion */
0, /* szOsFile */
0, /* mxPathname */
0, /* pNext */
"hecl_memlba", /* zName */
0, /* pAppData */
memlbaOpen, /* xOpen */
memlbaDelete, /* xDelete */
memlbaAccess, /* xAccess */
memlbaFullPathname, /* xFullPathname */
memlbaDlOpen, /* xDlOpen */
memlbaDlError, /* xDlError */
memlbaDlSym, /* xDlSym */
memlbaDlClose, /* xDlClose */
memlbaRandomness, /* xRandomness */
memlbaSleep, /* xSleep */
memlbaCurrentTime, /* xCurrentTime */
0, 0, 0, 0, 0
};
static sqlite3_io_methods memlba_io_methods =
{
1, /* iVersion */
memlbaClose, /* xClose */
memlbaRead, /* xRead */
memlbaWrite, /* xWrite */
memlbaTruncate, /* xTruncate */
memlbaSync, /* xSync */
memlbaFileSize, /* xFileSize */
memlbaLock, /* xLock */
memlbaUnlock, /* xUnlock */
memlbaCheckReservedLock, /* xCheckReservedLock */
memlbaFileControl, /* xFileControl */
memlbaSectorSize, /* xSectorSize */
memlbaDeviceCharacteristics, /* xDeviceCharacteristics */
0, /* xShmMap */
0, /* xShmLock */
0, /* xShmBarrier */
0, /* xShmUnmap */
0,
0
};
/* Useful macros used in several places */
#define MIN(x,y) ((x)<(y)?(x):(y))
#define MAX(x,y) ((x)>(y)?(x):(y))
/*
** Close a memlba-file.
*/
static int memlbaClose(sqlite3_file* pFile)
{
memlba_file* pTmp = (memlba_file*)pFile;
free(pTmp->headBuf);
free(pTmp->cachedBlockBufs[0]);
inflateEnd(&pTmp->zstrm);
return SQLITE_OK;
}
/*
** Read data from a memlba-file.
*/
static int memlbaRead(
sqlite3_file* pFile,
void* zBuf,
int iAmt,
sqlite_int64 iOfst
)
{
memlba_file* pTmp = (memlba_file*)pFile;
unsigned blockIdx = iOfst / pTmp->headBuf->blockSize;
unsigned firstOff = iOfst % pTmp->headBuf->blockSize;
unsigned firstRemBytes = pTmp->headBuf->blockSize - firstOff;
int slot = getBlockSlot(pTmp, blockIdx);
unsigned toRead = MIN((unsigned)iAmt, firstRemBytes);
memcpy(zBuf, pTmp->cachedBlockBufs[slot] + firstOff, toRead);
iAmt -= toRead;
zBuf += toRead;
while (iAmt > 0)
{
slot = getBlockSlot(pTmp, ++blockIdx);
toRead = MIN((unsigned)iAmt, pTmp->headBuf->blockSize);
memcpy(zBuf, pTmp->cachedBlockBufs[slot], toRead);
iAmt -= toRead;
zBuf += toRead;
}
return SQLITE_OK;
}
/*
** Write data to a memlba-file.
*/
static int memlbaWrite(
sqlite3_file* pFile,
const void* zBuf,
int iAmt,
sqlite_int64 iOfst
)
{
(void)pFile; (void)zBuf; (void)iAmt; (void)iOfst;
return SQLITE_OK;
}
/*
** Truncate a memlba-file.
*/
static int memlbaTruncate(sqlite3_file* pFile, sqlite_int64 size)
{
(void)pFile; (void)size;
return SQLITE_OK;
}
/*
** Sync a memlba-file.
*/
static int memlbaSync(sqlite3_file* pFile, int flags)
{
(void)pFile; (void)flags;
return SQLITE_OK;
}
/*
** Return the current file-size of a memlba-file.
*/
static int memlbaFileSize(sqlite3_file* pFile, sqlite_int64* pSize)
{
memlba_file* pTmp = (memlba_file*)pFile;
*pSize = pTmp->headBuf->headSz - pTmp->headBuf->blockTOC[0];
return SQLITE_OK;
}
/*
** Lock a memlba-file.
*/
static int memlbaLock(sqlite3_file* pFile, int eLock)
{
(void)pFile; (void)eLock;
return SQLITE_OK;
}
/*
** Unlock a memlba-file.
*/
static int memlbaUnlock(sqlite3_file* pFile, int eLock)
{
(void)pFile; (void)eLock;
return SQLITE_OK;
}
/*
** Check if another file-handle holds a RESERVED lock on a memlba-file.
*/
static int memlbaCheckReservedLock(sqlite3_file* pFile, int* pResOut)
{
(void)pFile;
*pResOut = 0;
return SQLITE_OK;
}
/*
** File control method. For custom operations on a memlba-file.
*/
static int memlbaFileControl(sqlite3_file* pFile, int op, void* pArg)
{
(void)pFile; (void)op; (void)pArg;
return SQLITE_OK;
}
/*
** Return the sector-size in bytes for a memlba-file.
*/
static int memlbaSectorSize(sqlite3_file* pFile)
{
(void)pFile;
return 0;
}
/*
** Return the device characteristic flags supported by a memlba-file.
*/
static int memlbaDeviceCharacteristics(sqlite3_file* pFile)
{
(void)pFile;
return 0;
}
/*
** Open an memlba file handle.
*/
static int memlbaOpen(
sqlite3_vfs* pVfs,
const char* zName,
sqlite3_file* pFile,
int flags,
int* pOutFlags
)
{
(void)pVfs; (void)zName; (void)pOutFlags;
if ((flags & SQLITE_OPEN_MAIN_DB) != SQLITE_OPEN_MAIN_DB ||
(flags & SQLITE_OPEN_READONLY) != SQLITE_OPEN_READONLY)
{
fprintf(stderr, "the sqlite hecl memlba VFS only supports main-db reading\n");
return SQLITE_CANTOPEN;
}
memlba_file* p2 = (memlba_file*)pFile;
memset(p2, 0, sizeof(*p2));
p2->base.pMethods = &memlba_io_methods;
inflateInit(&p2->zstrm);
p2->headBuf = HEAD_BUF;
unsigned i;
void* blockBufs = calloc(BLOCK_SLOTS, p2->headBuf->blockSize);
for (i=0 ; i<BLOCK_SLOTS ; ++i)
{
p2->cachedBlockBufs[i] = blockBufs + p2->headBuf->blockSize * i;
p2->cachedBlockIndices[i] = -1;
}
return SQLITE_OK;
}
/*
** Delete the file located at zPath. If the dirSync argument is true,
** ensure the file-system modifications are synced to disk before
** returning.
*/
static int memlbaDelete(sqlite3_vfs* pVfs, const char* zPath, int dirSync)
{
(void)pVfs; (void)zPath; (void)dirSync;
return SQLITE_OK;
}
/*
** Test for access permissions. Return true if the requested permission
** is available, or false otherwise.
*/
static int memlbaAccess(
sqlite3_vfs* pVfs,
const char* zPath,
int flags,
int* pResOut
)
{
(void)pVfs; (void)zPath; (void)pResOut;
if (flags & (SQLITE_ACCESS_READ | SQLITE_ACCESS_READWRITE))
return 1;
return 0;
}
/*
** Populate buffer zOut with the full canonical pathname corresponding
** to the pathname in zPath. zOut is guaranteed to point to a buffer
** of at least (FS_MAX_PATHNAME+1) bytes.
*/
static int memlbaFullPathname(
sqlite3_vfs* pVfs, /* Pointer to vfs object */
const char* zPath, /* Possibly relative input path */
int nOut, /* Size of output buffer in bytes */
char* zOut) /* Output buffer */
{
(void)pVfs;
strncpy(zOut, zPath, nOut);
return SQLITE_OK;
}
/*
** Open the dynamic library located at zPath and return a handle.
*/
static void* memlbaDlOpen(sqlite3_vfs* pVfs, const char* zPath)
{
(void)pVfs; (void)zPath;
return NULL;
}
/*
** Populate the buffer zErrMsg (size nByte bytes) with a human readable
** utf-8 string describing the most recent error encountered associated
** with dynamic libraries.
*/
static void memlbaDlError(sqlite3_vfs* pVfs, int nByte, char* zErrMsg)
{
(void)pVfs; (void)nByte; (void)zErrMsg;
}
/*
** Return a pointer to the symbol zSymbol in the dynamic library pHandle.
*/
static void (*memlbaDlSym(sqlite3_vfs* pVfs, void* pH, const char* zSym))(void)
{
(void)pVfs; (void)pH; (void)zSym;
return NULL;
}
/*
** Close the dynamic library handle pHandle.
*/
static void memlbaDlClose(sqlite3_vfs* pVfs, void* pHandle)
{
(void)pVfs; (void)pHandle;
}
/*
** Populate the buffer pointed to by zBufOut with nByte bytes of
** random data.
*/
static int memlbaRandomness(sqlite3_vfs* pVfs, int nByte, char* zBufOut)
{
(void)pVfs;
for(int i = 0 ; i < nByte ; ++i)
zBufOut[i] = rand();
return nByte;
}
/*
** Sleep for nMicro microseconds. Return the number of microseconds
** actually slept.
*/
static int memlbaSleep(sqlite3_vfs* pVfs, int nMicro)
{
(void)pVfs;
int seconds = (nMicro + 999999) / 1000000;
sleep(seconds);
return seconds * 1000000;
}
/*
** Return the current time as a Julian Day number in *pTimeOut.
*/
static int memlbaCurrentTime(sqlite3_vfs* pVfs, double* pTimeOut)
{
(void)pVfs;
*pTimeOut = 0.0;
return 0;
}
/*
** This procedure registers the memlba vfs with SQLite. If the argument is
** true, the memlba vfs becomes the new default vfs. It is the only publicly
** available function in this file.
*/
int sqlite_hecl_memlba_vfs_register(void* headBuf)
{
HEAD_BUF = headBuf;
if(memlba_vfs.szOsFile) return SQLITE_OK;
memlba_vfs.szOsFile = sizeof(memlba_file);
return sqlite3_vfs_register(&memlba_vfs, 0);
}

View File

@@ -1,21 +0,0 @@
#ifndef SQLITE_HECL_VFS
#define SQLITE_HECL_VFS
#ifdef __cplusplus
extern "C" {
#endif
#include <stdio.h>
#include <stdlib.h>
#include <stdbool.h>
typedef void(*TCloseCallback)(void* buf, size_t bufLen, void* ctx);
int sqlite_hecl_mem_vfs_register(TCloseCallback closeCb, void* ctx);
int sqlite_hecl_memlba_vfs_register(void* headBuf);
void sqlite_hecl_memlba_make(FILE* fout, void* bufin, size_t bufinLen, bool bigEndian);
#ifdef __cplusplus
}
#endif
#endif // SQLITE_HECL_VFS