initial database interface

hecl/driver/driver.pro

@@ -5,8 +5,10 @@ LIBS   -= -lQtGui -lQtCore
 unix:QMAKE_CXXFLAGS += -std=c++11 -stdlib=libc++
 unix:!macx:LIBS += -std=c++11 -stdlib=libc++ -lc++abi
 
-LIBPATH += $$OUT_PWD/../lib
+INCLUDEPATH += ../include
-LIBS += -lhecl
+
+LIBPATH += $$OUT_PWD/../lib $$OUT_PWD/../extern/sqlite3
+LIBS += -lhecl -lsqlite3
 
 SOURCES += \
     $$PWD/main.cpp

hecl/driver/main.cpp

@@ -1,7 +1,15 @@
 #include <stdio.h>
+#include <HECLDatabase.hpp>
+
+#define TESTDATA "TESTING 123"
 
 int main(int argc, char** argv)
 {
     printf("Hello World!\n");
+    HECLDatabase::IDatabase* db = HECLDatabase::NewDatabase(HECLDatabase::IDatabase::LOOSE,
+                                                            HECLDatabase::IDatabase::READWRITE,
+                                                            "test.db");
+    db->addDataBlob("TestObj", (void*)TESTDATA, sizeof(TESTDATA));
+    delete db;
     return 0;
 }

hecl/hecl.pro

@@ -16,7 +16,9 @@ HEADERS += \
     include/HECLRuntime.hpp
 
 SUBDIRS += \
+    extern/sqlite3 \
     lib \
     driver
 
+lib.depends = extern/sqlite3
 driver.depends = lib

hecl/include/HECLDatabase.hpp

@@ -1,4 +1,204 @@
 #ifndef HECLDATABASE_HPP
 #define HECLDATABASE_HPP
 
+#include <iterator>
+#include <string>
+
+namespace HECLDatabase
+{
+
+class IDatabase;
+
+/**
+ * @brief The IDataObject class
+ *
+ * This abstract base-class is a typeless object node for entities in an
+ * underlying database.
+ */
+class IDataObject
+{
+public:
+    /**
+     * @brief Data-key of object
+     * @return Primary key
+     */
+    virtual std::size_t id() const=0;
+
+    /**
+     * @brief Textual name of object
+     * @return Name
+     */
+    virtual const std::string& name() const=0;
+
+    /**
+     * @brief Data-hash of object
+     * @return Object hash truncated to system's size-type
+     */
+    virtual std::size_t hash() const=0;
+
+    /**
+     * @brief Retrieve the database this object is stored within
+     * @return database object
+     */
+    virtual IDatabase* parent() const=0;
+};
+
+/**
+ * @brief An iterable collection of objects tracked within the database
+ */
+class IDataDependencyGroup : public IDataObject
+{
+public:
+    /**
+     * @brief Count of objects in the group
+     * @return object count
+     */
+    virtual std::size_t length() const=0;
+
+    /**
+     * @brief Alias of length()
+     * @return object count
+     */
+    inline std::size_t size() const {return length();}
+
+    /**
+     * @brief Retrieve object at specified internal index within the group
+     * @param idx internal index of object to fetch (range [0,length()-1])
+     * @return object or nullptr
+     */
+    virtual const IDataObject* at(std::size_t idx) const=0;
+    inline const IDataObject* operator[](std::size_t idx) {return at(idx);}
+
+    /**
+     * @brief Simple IDataDependencyGroup iterator
+     *
+     * Use begin()/end() or C++11 ranged-for to use
+     */
+    class iterator : std::iterator<std::forward_iterator_tag, const IDataObject*>
+    {
+        friend class IDataDependencyGroup;
+        const IDataDependencyGroup& m_grp;
+        std::size_t m_idx = 0;
+        inline iterator(const IDataDependencyGroup& grp) : m_grp(grp) {}
+    public:
+        inline bool operator!=(const iterator& other)
+        {return &m_grp != &other.m_grp || m_idx != other.m_idx;}
+        inline iterator& operator++() {++m_idx; return *this;}
+    };
+    inline iterator begin() const {return iterator(*this);}
+    inline iterator end() const {auto it = iterator(*this); it.m_idx = length(); return it;}
+};
+
+/**
+ * @brief A generic database object storing raw bytes
+ */
+class IDataBlob : public IDataObject
+{
+public:
+    /**
+     * @brief Accesses the length of the object (in bytes)
+     * @return Data length
+     */
+    virtual std::size_t length() const=0;
+
+    /**
+     * @brief Alias for the length() function
+     * @return Data length
+     */
+    inline std::size_t size() const {return length();}
+
+    /**
+     * @brief Accesses the object's data
+     * @return Immutable pointer to object's data
+     *
+     * Note that the database may not have data loaded immediately loaded.
+     * This function will perform a blocking-load of the data object.
+     * Gather objects into an ILoadTransaction to pre-emptively load
+     * collections of objects.
+     */
+    virtual const void* data() const=0;
+};
+
+/**
+ * @brief Root database interface
+ */
+class IDatabase
+{
+public:
+    virtual ~IDatabase() {}
+
+    /**
+     * @brief Database backend type
+     */
+    enum Type
+    {
+        UNKNOWN,
+        MEMORY, /**< In-memory database; ideal for gathering small groups of frequently-accessed objects */
+        LOOSE, /**< Loose database; ideal for read/write database construction or platforms with good filesystems */
+        PACKED /**< Packed database; ideal for read-only archived data */
+    };
+    virtual Type getType() const=0;
+
+    /**
+     * @brief Database access type
+     */
+    enum Access
+    {
+        INVALID,
+        READONLY, /**< Read-only access; packed databases always use this mode */
+        READWRITE /**< Read/write access; used for building fresh databases */
+    };
+    virtual Access getAccess() const=0;
+
+    /**
+     * @brief Lookup object by database primary-key
+     * @param id Primary-key of object
+     * @return Data object
+     */
+    virtual const IDataObject* lookupObject(std::size_t id) const=0;
+
+    /**
+     * @brief Lookup object by name
+     * @param name Name of object
+     * @return Data object
+     */
+    virtual const IDataObject* lookupObject(const std::string& name) const=0;
+
+    /**
+     * @brief Insert named data-blob object into a database with read/write access
+     * @param name Name of object
+     * @param data Pointer to object data (will be copied)
+     * @param length Size of object data to copy
+     * @return New data object
+     */
+    virtual const IDataObject* addDataBlob(const std::string& name, const void* data, std::size_t length)=0;
+
+    /**
+     * @brief Insert Data-blob object into a database with read/write access
+     * @param data Pointer to object data (will be copied)
+     * @param length size of object data to copy
+     * @return New data object
+     */
+    virtual const IDataObject* addDataBlob(const void* data, std::size_t length)=0;
+
+    /**
+     * @brief Write a full copy of the database to another type/path
+     * @param type Type of new database
+     * @param path Target path of new database
+     * @return True on success
+     */
+    virtual bool writeDatabase(IDatabase::Type type, const std::string& path) const=0;
+
+};
+
+/**
+ * @brief Creates a new (empty) database
+ * @param type Type of new database
+ * @param access Requested level of access
+ * @return New database object
+ */
+IDatabase* NewDatabase(IDatabase::Type type, IDatabase::Access access, const std::string& path);
+
+}
+
 #endif // HECLDATABASE_HPP

hecl/lib/backend/backend.pri

@@ -17,10 +17,10 @@ SOURCES += \
     $$PWD/HLSL/CHLSLObject.cpp \
     $$PWD/HLSL/CHLSLSpec.cpp \
     $$PWD/HLSL/CHLSLVertEmitter.cpp \
-    $$PWD/VSPIR/CVSPIRFragEmitter.cpp \
+    $$PWD/SPIRV/CSPIRVFragEmitter.cpp \
-    $$PWD/VSPIR/CVSPIRObject.cpp \
+    $$PWD/SPIRV/CSPIRVObject.cpp \
-    $$PWD/VSPIR/CVSPIRSpec.cpp \
+    $$PWD/SPIRV/CSPIRVSpec.cpp \
-    $$PWD/VSPIR/CVSPIRVertEmitter.cpp \
+    $$PWD/SPIRV/CSPIRVVertEmitter.cpp \
     $$PWD/HECLBackend.cpp
 
 !isEmpty(LLVMROOT) {

hecl/lib/database/CLooseDatabase.hpp

@@ -0,0 +1,59 @@
+#ifndef HECLDATABASE_CPP
+#error This file must only be included from HECLDatabase.cpp
+#endif
+
+#include "HECLDatabase.hpp"
+#include "CSQLite.hpp"
+
+namespace HECLDatabase
+{
+
+class CLooseDatabase final : public IDatabase
+{
+    CSQLite m_sql;
+    Access m_access;
+public:
+    CLooseDatabase(const std::string& path, Access access)
+    : m_sql(path.c_str(), (m_access == READONLY) ? true : false), m_access(access)
+    {
+
+    }
+
+    ~CLooseDatabase()
+    {
+
+    }
+
+    Type getType() const
+    {
+        return LOOSE;
+    }
+
+    Access getAccess() const
+    {
+        return m_access;
+    }
+
+    const IDataObject* lookupObject(std::size_t id) const
+    {
+    }
+
+    const IDataObject* lookupObject(const std::string& name) const
+    {
+    }
+
+    const IDataObject* addDataBlob(const std::string& name, const void* data, std::size_t length)
+    {
+    }
+
+    const IDataObject* addDataBlob(const void* data, std::size_t length)
+    {
+    }
+
+    bool writeDatabase(IDatabase::Type type, const std::string& path) const
+    {
+    }
+
+};
+
+}

hecl/lib/database/CMemoryDatabase.hpp

@@ -0,0 +1,59 @@
+#ifndef HECLDATABASE_CPP
+#error This file must only be included from HECLDatabase.cpp
+#endif
+
+#include "HECLDatabase.hpp"
+#include "CSQLite.hpp"
+
+namespace HECLDatabase
+{
+
+class CMemoryDatabase final : public IDatabase
+{
+    CSQLite m_sql;
+    Access m_access;
+public:
+    CMemoryDatabase(Access access)
+    : m_sql(":memory:"), m_access(access)
+    {
+
+    }
+
+    ~CMemoryDatabase()
+    {
+
+    }
+
+    Type getType() const
+    {
+        return MEMORY;
+    }
+
+    Access getAccess() const
+    {
+        return m_access;
+    }
+
+    const IDataObject* lookupObject(std::size_t id) const
+    {
+    }
+
+    const IDataObject* lookupObject(const std::string& name) const
+    {
+    }
+
+    const IDataObject* addDataBlob(const std::string& name, const void* data, std::size_t length)
+    {
+    }
+
+    const IDataObject* addDataBlob(const void* data, std::size_t length)
+    {
+    }
+
+    bool writeDatabase(IDatabase::Type type, const std::string& path) const
+    {
+    }
+
+};
+
+}

hecl/lib/database/CPackedDatabase.hpp

@@ -0,0 +1,58 @@
+#ifndef HECLDATABASE_CPP
+#error This file must only be included from HECLDatabase.cpp
+#endif
+
+#include "HECLDatabase.hpp"
+#include "CSQLite.hpp"
+
+namespace HECLDatabase
+{
+
+class CPackedDatabase final : public IDatabase
+{
+    CSQLite m_sql;
+public:
+    CPackedDatabase(const std::string& path)
+    : m_sql(path.c_str(), (m_access == READONLY) ? true : false)
+    {
+
+    }
+
+    ~CPackedDatabase()
+    {
+
+    }
+
+    Type getType() const
+    {
+        return PACKED;
+    }
+
+    Access getAccess() const
+    {
+        return READONLY;
+    }
+
+    const IDataObject* lookupObject(std::size_t id) const
+    {
+    }
+
+    const IDataObject* lookupObject(const std::string& name) const
+    {
+    }
+
+    const IDataObject* addDataBlob(const std::string& name, const void* data, std::size_t length)
+    {
+    }
+
+    const IDataObject* addDataBlob(const void* data, std::size_t length)
+    {
+    }
+
+    bool writeDatabase(IDatabase::Type type, const std::string& path) const
+    {
+    }
+
+};
+
+}

hecl/lib/database/CSQLite.hpp

@@ -0,0 +1,119 @@
+#ifndef CSQLITE_HPP
+#define CSQLITE_HPP
+
+#include <sqlite3.h>
+#include <stdexcept>
+#include <functional>
+
+namespace HECLDatabase
+{
+
+/* Private sqlite3 backend to be used by database subclasses */
+
+static const char* skDBINIT =
+"PRAGMA foreign_keys = ON;\n"
+"CREATE TABLE IF NOT EXISTS objects(rowid INTEGER PRIMARY KEY,"
+                                   "name UNIQUE,"
+                                   "type4cc UNSIGNED INTEGER,"
+                                   "hash64 UNSIGNED INTEGER,"
+                                   "compLen UNSIGNED INTEGER,"
+                                   "decompLen UNSIGNED INTEGER);\n"
+"CREATE INDEX IF NOT EXISTS nameidx ON objects(name);\n"
+"CREATE TABLE IF NOT EXISTS deplinks(groupId, "
+                                    "objId REFERENCES objects(rowid) ON DELETE CASCADE, "
+                                    "UNIQUE (groupId, objId) ON CONFLICT IGNORE);\n"
+"CREATE INDEX IF NOT EXISTS grpidx ON deplinks(groupId);\n"
+"CREATE INDEX IF NOT EXISTS depidx ON deplinks(objId);\n";
+
+#define PREPSTMT(stmtSrc, outVar)\
+if (sqlite3_prepare_v2(m_db, stmtSrc, 0, &outVar, NULL) != SQLITE_OK)\
+{\
+    throw std::runtime_error(sqlite3_errmsg(m_db));\
+    sqlite3_close(m_db);\
+    return;\
+}
+
+class CSQLite
+{
+    sqlite3* m_db;
+    sqlite3_stmt* m_selObjects;
+    sqlite3_stmt* m_selObjectByName;
+    sqlite3_stmt* m_selDistictDepGroups;
+    sqlite3_stmt* m_selDepGroupObjects;
+
+public:
+    CSQLite(const char* path, bool readonly)
+    {
+        /* Open database connection */
+        if (sqlite3_open_v2(path, &m_db, readonly ?
+                            SQLITE_OPEN_READONLY :
+                            SQLITE_OPEN_READWRITE |
+                            SQLITE_OPEN_CREATE, NULL) != SQLITE_OK)
+        {
+            throw std::runtime_error(sqlite3_errmsg(m_db));
+            sqlite3_close(m_db);
+            return;
+        }
+
+        /* Execute bootstrap statements */
+        char* errMsg = NULL;
+        sqlite3_exec(m_db, skDBINIT, NULL, NULL, &errMsg);
+        if (errMsg)
+        {
+            throw std::runtime_error(errMsg);
+            sqlite3_free(errMsg);
+            sqlite3_close(m_db);
+            return;
+        }
+
+        /* Precompile statements */
+        PREPSTMT("SELECT rowid,name,type4cc,hash64,compLen,decompLen FROM objects", m_selObjects);
+        PREPSTMT("SELECT rowid FROM objects WHERE name=?1", m_selObjectByName);
+        PREPSTMT("SELECT DISTINCT groupId FROM deplinks", m_selDistictDepGroups);
+        PREPSTMT("SELECT DISTINCT objId FROM deplinks WHERE groupId=?1", m_selDepGroupObjects);
+    }
+
+    ~CSQLite()
+    {
+        sqlite3_finalize(m_selObjects);
+        sqlite3_finalize(m_selObjectByName);
+        sqlite3_finalize(m_selDistictDepGroups);
+        sqlite3_finalize(m_selDepGroupObjects);
+        sqlite3_close(m_db);
+    }
+
+    void buildMemoryIndex(const std::function<void(std::size_t&&,       // id
+                                                   const std::string&&, // name
+                                                   uint32_t&&,          // type4cc
+                                                   uint64_t&&,          // hash64
+                                                   std::size_t&&,       // compLen
+                                                   std::size_t&&)>&     // decompLen
+                                                   objectAdder)
+    {
+        while (sqlite3_step(m_selObjects) == SQLITE_ROW)
+        {
+            objectAdder(sqlite3_column_int64(m_selObjects, 0),
+                        (const char*)sqlite3_column_text(m_selObjects, 1),
+                        sqlite3_column_int(m_selObjects, 2),
+                        sqlite3_column_int64(m_selObjects, 3),
+                        sqlite3_column_int64(m_selObjects, 4),
+                        sqlite3_column_int64(m_selObjects, 5));
+        }
+        sqlite3_reset(m_selObjects);
+    }
+
+    std::size_t objectIdFromName(const std::string& name)
+    {
+        sqlite3_bind_text(m_selObjectByName, 1, name.c_str(), name.length(), NULL);
+        std::size_t retval = 0;
+        if (sqlite3_step(m_selObjectByName) == SQLITE_ROW)
+            retval = sqlite3_column_int64(m_selObjectByName, 0);
+        sqlite3_reset(m_selObjectByName);
+        return retval;
+    }
+
+};
+
+}
+
+#endif // CSQLITE_HPP

hecl/lib/database/HECLDatabase.cpp

@@ -0,0 +1,27 @@
+#include "HECLDatabase.hpp"
+
+#define HECLDATABASE_CPP
+#include "CLooseDatabase.hpp"
+#include "CPackedDatabase.hpp"
+#include "CMemoryDatabase.hpp"
+
+namespace HECLDatabase
+{
+
+IDatabase* NewDatabase(IDatabase::Type type, IDatabase::Access access, const std::string& path)
+{
+    switch (type)
+    {
+    case IDatabase::LOOSE:
+        return new CLooseDatabase(path, access);
+    case IDatabase::PACKED:
+        return new CPackedDatabase(path);
+    case IDatabase::MEMORY:
+        return new CMemoryDatabase(access);
+    case IDatabase::UNKNOWN:
+        return nullptr;
+    }
+    return nullptr;
+}
+
+}

hecl/lib/database/IObjectStore.hpp

@@ -1,4 +0,0 @@
-#ifndef IOBJECTSTORE_HPP
-#define IOBJECTSTORE_HPP
-
-#endif // IOBJECTSTORE_HPP

hecl/lib/database/database.pri

@@ -1,8 +1,8 @@
 HEADERS += \
-    $$PWD/IObjectStore.hpp
+    $$PWD/CPackedDatabase.hpp \
+    $$PWD/CMemoryDatabase.hpp \
+    $$PWD/CLooseDatabase.hpp \
+    $$PWD/CSQLite.hpp
 
 SOURCES += \
-    $$PWD/CDataObject.cpp \
-    $$PWD/CObjectStoreDir.cpp \
-    $$PWD/CObjectStorePacked.cpp \
     $$PWD/HECLDatabase.cpp

hecl/lib/lib.pro

@@ -5,7 +5,7 @@ CONFIG -= Qt
 QT      =
 unix:QMAKE_CXXFLAGS += -std=c++11 -stdlib=libc++
 
-INCLUDEPATH += $$PWD
+INCLUDEPATH += $$PWD ../include ../extern
 
 include (frontend/frontend.pri)
 include (backend/backend.pri)