tons of database interface work

hecl/dataspec/DUMB.hpp

@@ -0,0 +1,53 @@
+#ifndef DUMB_HPP
+#define DUMB_HPP
+
+#include "HECLDatabase.hpp"
+
+class CDUMBProject : public HECLDatabase::CProjectObject
+{
+    bool _cookObject(TDataAppender dataAppender,
+                     DataEndianness endianness, DataPlatform platform)
+    {
+        return true;
+    }
+
+    void _gatherDeps(TDepAdder depAdder)
+    {
+
+    }
+
+public:
+    CDUMBProject(const ConstructionInfo& info)
+    : CProjectObject(info)
+    {
+    }
+
+    ~CDUMBProject()
+    {
+    }
+};
+
+class CDUMBRuntime : public HECLDatabase::CRuntimeObject
+{
+
+    bool _objectFinishedLoading(const void* data, size_t len)
+    {
+        return true;
+    }
+
+    void _objectWillUnload()
+    {
+
+    }
+
+public:
+    CDUMBRuntime(const ConstructionInfo& info)
+    : CRuntimeObject(info)
+    {
+    }
+    ~CDUMBRuntime()
+    {
+    }
+};
+
+#endif // DUMB_HPP

hecl/dataspec/HMDL.hpp

@@ -0,0 +1,53 @@
+#ifndef HMDL_HPP
+#define HMDL_HPP
+
+#include "HECLDatabase.hpp"
+
+class CHMDLProject : public HECLDatabase::CProjectObject
+{
+    bool _cookObject(TDataAppender dataAppender,
+                     DataEndianness endianness, DataPlatform platform)
+    {
+        return true;
+    }
+
+    void _gatherDeps(TDepAdder depAdder)
+    {
+
+    }
+
+public:
+    CHMDLProject(const ConstructionInfo& info)
+    : CProjectObject(info)
+    {
+    }
+
+    ~CHMDLProject()
+    {
+    }
+};
+
+class CHMDLRuntime : public HECLDatabase::CRuntimeObject
+{
+
+    bool _objectFinishedLoading(const void* data, size_t len)
+    {
+        return true;
+    }
+
+    void _objectWillUnload()
+    {
+
+    }
+
+public:
+    CHMDLRuntime(const ConstructionInfo& info)
+    : CRuntimeObject(info)
+    {
+    }
+    ~CHMDLRuntime()
+    {
+    }
+};
+
+#endif // HMDL_HPP

hecl/dataspec/MATR.hpp

@@ -0,0 +1,53 @@
+#ifndef MATR_HPP
+#define MATR_HPP
+
+#include "HECLDatabase.hpp"
+
+class CMATRProject : public HECLDatabase::CProjectObject
+{
+    bool _cookObject(TDataAppender dataAppender,
+                     DataEndianness endianness, DataPlatform platform)
+    {
+        return true;
+    }
+
+    void _gatherDeps(TDepAdder depAdder)
+    {
+
+    }
+
+public:
+    CMATRProject(const ConstructionInfo& info)
+    : CProjectObject(info)
+    {
+    }
+
+    ~CMATRProject()
+    {
+    }
+};
+
+class CMATRRuntime : public HECLDatabase::CRuntimeObject
+{
+
+    bool _objectFinishedLoading(const void* data, size_t len)
+    {
+        return true;
+    }
+
+    void _objectWillUnload()
+    {
+
+    }
+
+public:
+    CMATRRuntime(const ConstructionInfo& info)
+    : CRuntimeObject(info)
+    {
+    }
+    ~CMATRRuntime()
+    {
+    }
+};
+
+#endif // MATR_HPP

hecl/dataspec/STRG.hpp

@@ -0,0 +1,53 @@
+#ifndef STRG_HPP
+#define STRG_HPP
+
+#include "HECLDatabase.hpp"
+
+class CSTRGProject : public HECLDatabase::CProjectObject
+{
+    bool _cookObject(TDataAppender dataAppender,
+                     DataEndianness endianness, DataPlatform platform)
+    {
+        return true;
+    }
+
+    void _gatherDeps(TDepAdder depAdder)
+    {
+
+    }
+
+public:
+    CSTRGProject(const ConstructionInfo& info)
+    : CProjectObject(info)
+    {
+    }
+
+    ~CSTRGProject()
+    {
+    }
+};
+
+class CSTRGRuntime : public HECLDatabase::CRuntimeObject
+{
+
+    bool _objectFinishedLoading(const void* data, size_t len)
+    {
+        return true;
+    }
+
+    void _objectWillUnload()
+    {
+
+    }
+
+public:
+    CSTRGRuntime(const ConstructionInfo& info)
+    : CRuntimeObject(info)
+    {
+    }
+    ~CSTRGRuntime()
+    {
+    }
+};
+
+#endif // STRG_HPP

hecl/dataspec/TXTR.hpp

@@ -0,0 +1,53 @@
+#ifndef TXTR_HPP
+#define TXTR_HPP
+
+#include "HECLDatabase.hpp"
+
+class CTXTRProject : public HECLDatabase::CProjectObject
+{
+    bool _cookObject(TDataAppender dataAppender,
+                     DataEndianness endianness, DataPlatform platform)
+    {
+        return true;
+    }
+
+    void _gatherDeps(TDepAdder depAdder)
+    {
+
+    }
+
+public:
+    CTXTRProject(const ConstructionInfo& info)
+    : CProjectObject(info)
+    {
+    }
+
+    ~CTXTRProject()
+    {
+    }
+};
+
+class CTXTRRuntime : public HECLDatabase::CRuntimeObject
+{
+
+    bool _objectFinishedLoading(const void* data, size_t len)
+    {
+        return true;
+    }
+
+    void _objectWillUnload()
+    {
+
+    }
+
+public:
+    CTXTRRuntime(const ConstructionInfo& info)
+    : CRuntimeObject(info)
+    {
+    }
+    ~CTXTRRuntime()
+    {
+    }
+};
+
+#endif // TXTR_HPP

hecl/dataspec/dataspec.cpp

@@ -0,0 +1,17 @@
+#include "HECLDatabase.hpp"
+
+#include "DUMB.hpp"
+#include "HMDL.hpp"
+#include "MATR.hpp"
+#include "STRG.hpp"
+#include "TXTR.hpp"
+
+const HECLDatabase::RegistryEntry DATASPEC_TYPES[]
+{
+    REGISTRY_ENTRY("DUMB", CDUMBProject, CDUMBRuntime),
+    REGISTRY_ENTRY("HMDL", CHMDLProject, CHMDLRuntime),
+    REGISTRY_ENTRY("MATR", CMATRProject, CMATRRuntime),
+    REGISTRY_ENTRY("STRG", CSTRGProject, CSTRGRuntime),
+    REGISTRY_ENTRY("TXTR", CTXTRProject, CTXTRRuntime),
+    REGISTRY_SENTINEL()
+};

hecl/dataspec/dataspec.pro

@@ -0,0 +1,23 @@
+TEMPLATE = lib
+CONFIG += staticlib
+TARGET = dataspec
+CONFIG -= Qt
+QT      =
+unix:QMAKE_CXXFLAGS += -std=c++11
+unix:QMAKE_CFLAGS += -std=c99
+unix:LIBS += -std=c++11
+clang:QMAKE_CXXFLAGS += -stdlib=libc++
+clang:LIBS += -stdlib=libc++ -lc++abi
+
+INCLUDEPATH += $$PWD ../include ../extern
+
+HEADERS += \
+    DUMB.hpp \
+    HMDL.hpp \
+    MATR.hpp \
+    STRG.hpp \
+    TXTR.hpp
+
+SOURCES += \
+    dataspec.cpp
+

hecl/driver/CToolHelp.hpp

@@ -29,7 +29,7 @@ public:
                "................/...........................................,}\n"
                "............../........................................,:`^`..}\n"
                "............./.......................................,:\"...../\n"
-               "............?.....__..................................:`....../\n"
+               "............?.....__..................................:`...../\n"
                ".........../__.(...\"~-,_...........................,:`....../\n"
                "........../(_....\"~,_....\"~,_.....................,:`...._/ \n"
                "..........{.._$;_....\"=,_.....\"-,_......,.-~-,},.~\";/....} \n"

hecl/driver/driver.pro

@@ -10,12 +10,13 @@ clang:LIBS += -stdlib=libc++ -lc++abi
 INCLUDEPATH += ../include
 
 LIBPATH += $$OUT_PWD/../lib \
+    $$OUT_PWD/../dataspec \
     $$OUT_PWD/../extern/sqlite3 \
     $$OUT_PWD/../extern/blowfish \
     $$OUT_PWD/../extern/libpng \
     $$OUT_PWD/../extern/zlib
 
-LIBS += -lhecl -lsqlite3 -lblowfish -lpng -lz
+LIBS += -lhecl -ldataspec -lsqlite3 -lblowfish -lpng -lz
 
 SOURCES += \
     $$PWD/main.cpp

hecl/driver/main.cpp

@@ -33,7 +33,7 @@ static void printHelp(const char* pname)
 static const std::regex regOPEN("-o\\s*(\\S+)", std::regex::ECMAScript|std::regex::optimize);
 static const std::regex regVERBOSE("-v(v*)", std::regex::ECMAScript|std::regex::optimize);
 static const std::regex regFORCE("-f", std::regex::ECMAScript|std::regex::optimize);
-static const std::regex regWS("\\S+", std::regex::ECMAScript|std::regex::optimize);
+static const std::regex regNOWS("\\S+", std::regex::ECMAScript|std::regex::optimize);
 
 /* Iterates string segments around matched arguments and
  * filters args string accordingly */
@@ -101,7 +101,7 @@ int main(int argc, const char** argv)
         }
 
         /* Gather remaining args */
-        for (std::sregex_token_iterator it(args.begin(), args.end(), regWS);
+        for (std::sregex_token_iterator it(args.begin(), args.end(), regNOWS);
              it != std::sregex_token_iterator() ; ++it)
         {
             const std::string& str = *it;

hecl/hecl.pro

@@ -22,6 +22,7 @@ SUBDIRS += \
     extern/libpng \
     extern/zlib \
     lib \
+    dataspec \
     driver
 
 driver.depends = extern/sqlite3
@@ -29,3 +30,4 @@ driver.depends = extern/blowfish
 driver.depends = extern/libpng
 driver.depends = extern/zlib
 driver.depends = lib
+driver.depends = dataspec

hecl/include/HECL.hpp

@@ -38,6 +38,8 @@ class FourCC
         uint32_t num;
     };
 public:
+    FourCC() /* Sentinel FourCC */
+    : num(0) {}
     FourCC(const char* name)
     : num(*(uint32_t*)name) {}
     inline bool operator==(FourCC& other) {return num == other.num;}
@@ -57,6 +59,8 @@ class ObjectHash
 public:
     ObjectHash(const void* buf, size_t len)
     : hash(Blowfish_hash(buf, len)) {}
+    ObjectHash(int64_t hashin)
+    : hash(hashin) {}
     inline bool operator==(ObjectHash& other) {return hash == other.hash;}
     inline bool operator!=(ObjectHash& other) {return hash != other.hash;}
     inline bool operator<(ObjectHash& other) {return hash < other.hash;}

hecl/include/HECLDatabase.hpp

@@ -5,6 +5,7 @@
 #include <string>
 #include <functional>
 #include <vector>
+#include <stdexcept>
 #include <stdint.h>
 
 #include "HECL.hpp"
@@ -13,9 +14,10 @@ namespace HECLDatabase
 {
 
 class IDatabase;
+class IProject;
 
 /**
- * @brief The IDataObject class
+ * @brief Generic Database Object Class
  *
  * This abstract base-class is a typeless object node for entities in an
  * underlying database.
@@ -27,31 +29,37 @@ public:
      * @brief Data-key of object
      * @return Primary key
      */
-    virtual size_t id() const=0;
+    virtual int64_t id() const=0;
 
     /**
-     * @brief Textual name of object
+     * @brief FourCC type of object
-     * @return Name
+     * @return FourCC type
      */
-    virtual const std::string& name() const=0;
+    virtual const HECL::FourCC& type() const=0;
 
     /**
      * @brief Data-hash of object
      * @return Object hash truncated to system's size-type
      */
-    virtual size_t hash() const=0;
+    virtual const HECL::ObjectHash& hash() const=0;
+
+    /**
+     * @brief Original path of object
+     * @return Name
+     */
+    virtual const std::string& path() const=0;
 
     /**
      * @brief Retrieve the database this object is stored within
      * @return database object
      */
-    virtual IDatabase* parent() const=0;
+    virtual IDatabase* parentDatabase() const=0;
 };
 
 /**
  * @brief An iterable collection of objects tracked within the database
  */
-class IDataDependencyGroup : public IDataObject
+class IDataDependencyGroup
 {
 public:
     /**
@@ -78,36 +86,6 @@ public:
     virtual std::vector<IDataObject*>::const_iterator end() const=0;
 };
 
-/**
- * @brief A generic database object storing raw bytes
- */
-class IDataBlob : public IDataObject
-{
-public:
-    /**
-     * @brief Accesses the length of the object (in bytes)
-     * @return Data length
-     */
-    virtual size_t length() const=0;
-
-    /**
-     * @brief Alias for the length() function
-     * @return Data length
-     */
-    inline size_t size() const {return length();}
-
-    /**
-     * @brief Accesses the object's data
-     * @return Immutable pointer to object's data
-     *
-     * Note that the database may not have data loaded immediately loaded.
-     * This function will perform a blocking-load of the data object.
-     * Gather objects into an ILoadTransaction to pre-emptively load
-     * collections of objects.
-     */
-    virtual const void* data() const=0;
-};
-
 /**
  * @brief Root database interface
  */
@@ -121,10 +99,10 @@ public:
      */
     enum Type
     {
-        UNKNOWN,
+        T_UNKNOWN,
-        MEMORY, /**< In-memory database; ideal for gathering small groups of frequently-accessed objects */
+        T_MEMORY, /**< In-memory database; ideal for gathering small groups of frequently-accessed objects */
-        LOOSE, /**< Loose database; ideal for read/write database construction or platforms with good filesystems */
+        T_LOOSE, /**< Loose database; ideal for read/write database construction or platforms with good filesystems */
-        PACKED /**< Packed database; ideal for read-only archived data */
+        T_PACKED /**< Packed database; ideal for read-only archived data */
     };
     virtual Type getType() const=0;
 
@@ -133,9 +111,9 @@ public:
      */
     enum Access
     {
-        INVALID,
+        A_INVALID,
-        READONLY, /**< Read-only access; packed databases always use this mode */
+        A_READONLY, /**< Read-only access; packed databases always use this mode */
-        READWRITE /**< Read/write access; used for building fresh databases */
+        A_READWRITE /**< Read/write access; used for building fresh databases */
     };
     virtual Access getAccess() const=0;
 
@@ -153,23 +131,6 @@ public:
      */
     virtual const IDataObject* lookupObject(const std::string& name) const=0;
 
-    /**
-     * @brief Insert named data-blob object into a database with read/write access
-     * @param name Name of object
-     * @param data Pointer to object data (will be copied)
-     * @param length Size of object data to copy
-     * @return New data object
-     */
-    virtual const IDataObject* addDataBlob(const std::string& name, const void* data, size_t length)=0;
-
-    /**
-     * @brief Insert Data-blob object into a database with read/write access
-     * @param data Pointer to object data (will be copied)
-     * @param length size of object data to copy
-     * @return New data object
-     */
-    virtual const IDataObject* addDataBlob(const void* data, size_t length)=0;
-
     /**
      * @brief Write a full copy of the database to another type/path
      * @param type Type of new database
@@ -185,9 +146,96 @@ public:
  * @param type Type of new database
  * @param access Requested level of access
  * @return New database object
+ *
+ * Generally, the preferred method for working with HECL databases is via the
+ * IProject interface. NewProject() will automatically construct the necessary
+ * internal database objects.
  */
 IDatabase* NewDatabase(IDatabase::Type type, IDatabase::Access access, const std::string& path);
 
+
+/**
+ * @brief Base object to subclass for integrating with key project operations
+ *
+ * All project objects are provided with IDataObject pointers to their database
+ * entries. Subclasses register themselves with a type registry so instances
+ * are automatically constructed when performing operations like cooking and packaging.
+ *
+ * DO NOT CONSTRUCT THIS OR SUBCLASSES DIRECTLY!!
+ */
+class CProjectObject
+{
+protected:
+    friend class CProject;
+
+    /**
+     * @brief Byte-order of target system
+     */
+    enum DataEndianness
+    {
+        DE_NONE,
+        DE_BIG, /**< Big-endian (PowerPC) */
+        DE_LITTLE /**< Little-endian (Intel) */
+    };
+
+    /**
+     * @brief Data-formats of target system
+     */
+    enum DataPlatform
+    {
+        DP_NONE,
+        DP_GENERIC, /**< Scanline textures and 3-way shader bundle (GLSL, HLSL, SPIR-V) */
+        DP_REVOLUTION, /**< Tiled textures and GX register buffers */
+        DP_CAFE /**< Swizzled textures and R700 shader objects */
+    };
+
+    typedef std::function<void(const void* data, size_t len)> TDataAppender;
+
+    /**
+     * @brief Optional private method implemented by CProjectObject subclasses to cook objects
+     * @param dataAppender subclass calls this function zero or more times to provide cooked-data linearly
+     * @param endianness byte-order to target
+     * @param platform data-formats to target
+     * @return true if cook succeeded
+     *
+     * This method is called during IProject::cookPath().
+     * Part of the cooking process may include embedding database-refs to dependencies.
+     * This method should store the 64-bit value provided by IDataObject::id() when doing this.
+     */
+    virtual bool _cookObject(TDataAppender dataAppender,
+                             DataEndianness endianness, DataPlatform platform)
+    {(void)dataAppender;(void)endianness;(void)platform;return true;}
+
+    typedef std::function<void(CProjectObject*)> TDepAdder;
+
+    /**
+     * @brief Optional private method implemented by CProjectObject subclasses to resolve dependencies
+     * @param depAdder subclass calls this function zero or more times to register each dependency
+     *
+     * This method is called during IProject::packagePath().
+     * Dependencies registered via this method will eventually have this method called on themselves
+     * as well. This is a non-recursive operation, no need for subclasses to implement recursion-control.
+     */
+    virtual void _gatherDeps(TDepAdder depAdder)
+    {(void)depAdder;}
+
+protected:
+    std::string m_path;
+    IDataObject* m_mainObj;
+    IDataObject* m_cookedObj;
+public:
+    virtual ~CProjectObject();
+    struct ConstructionInfo
+    {
+        IDataObject* mainObj;
+        IDataObject* cookedObj;
+        const std::string& path;
+    };
+    CProjectObject(const ConstructionInfo& info)
+    : m_path(info.path), m_mainObj(info.mainObj), m_cookedObj(info.cookedObj) {}
+};
+
+
 /**
  * @brief Main project interface
  *
@@ -200,22 +248,52 @@ class IProject
 public:
     virtual ~IProject() {}
 
+    /**
+     * @brief Internal packagePath() exception
+     *
+     * Due to the recursive nature of packagePath(), there are potential
+     * pitfalls like infinite-recursion. HECL throws this whenever there
+     * are uncooked dependencies or if the maximum dependency-recursion
+     * limit is exceeded.
+     */
+    class PackageException : public std::runtime_error {};
+
     /**
      * @brief A rough description of how 'expensive' a given cook operation is
      *
      * This is used to provide pretty colors during the cook operation
      */
-    enum LoadType
+    enum Cost
     {
-        LOAD_INFO,
+        C_NONE,
-        LOAD_LIGHT,
+        C_LIGHT,
-        LOAD_MEDIUM,
+        C_MEDIUM,
-        LOAD_HEAVY
+        C_HEAVY
     };
 
+    /**
+     * @brief Access internal database interface for working files
+     * @return main working database object
+     *
+     * It's generally recommended for HECL frontends to avoid modifying
+     * databases via this returned object.
+     */
+    virtual IDatabase* mainDatabase() const=0;
+
+    /**
+     * @brief Access internal database interface for cooked objects
+     * @return main cooked database object
+     *
+     * It's generally recommended for HECL frontends to avoid modifying
+     * databases via this returned object.
+     */
+    virtual IDatabase* cookedDatabase() const=0;
+
     /**
      * @brief Register an optional callback to report log-messages using
      * @param logger logger-callback
+     *
+     * If this method is never called, all project operations will run silently.
      */
     virtual void registerLogger(HECL::TLogger logger)=0;
 
@@ -223,48 +301,312 @@ public:
      * @brief Get the path of the project's root-directory
      * @param absolute return as absolute-path
      * @return project root path
+     *
+     * Self explanatory
      */
-    virtual std::string getProjectRootPath(bool absolute)=0;
+    virtual std::string getProjectRootPath(bool absolute=false) const=0;
+
+    /**
+     * @brief Determine if an arbitrary absolute or relative path lies within project
+     * @param subpath directory or file to validate
+     * @return true if valid
+     */
+    virtual bool validateSubPath(const std::string& subpath) const=0;
 
     /**
      * @brief Add a given file or file-pattern to the database
      * @param path file or pattern within project
      * @return true on success
+     *
+     * This method blocks while object hashing takes place
      */
     virtual bool addPath(const std::string& path)=0;
 
     /**
-     * @brief Begin cook process for specified file or directory
+     * @brief Remove a given file or file-pattern from the database
-     * @param path file or directory of intermediates to cook
+     * @param path file or pattern within project
+     * @param recursive traverse into matched subdirectories
+     * @return true on success
+     *
+     * This method will not delete actual working files from the project
+     * directory. It will delete associated cooked objects though.
+     */
+    virtual bool removePath(const std::string& path, bool recursive=false)=0;
+
+    /**
+     * @brief Register a working sub-directory as a Dependency Group
+     * @param path directory to register as Dependency Group
+     * @return true on success
+     *
+     * Dependency Groups are used at runtime to stage burst load-transactions.
+     * They may only be added to directories and will automatically claim
+     * subdirectories as well.
+     *
+     * Cooked objects in dependency groups will be packaged contiguously
+     * and automatically duplicated if shared with other dependency groups.
+     * This contiguous storage makes for optimal loading from slow block-devices
+     * like optical drives.
+     */
+    virtual bool addGroup(const std::string& path)=0;
+
+    /**
+     * @brief Unregister a working sub-directory as a dependency group
+     * @param path directory to unregister as Dependency Group
+     * @return true on success
+     */
+    virtual bool removeGroup(const std::string& path)=0;
+
+    /**
+     * @brief Begin cook process for specified directory
+     * @param path directory of intermediates to cook
      * @param feedbackCb a callback to run reporting cook-progress
      * @param recursive traverse subdirectories to cook as well
      * @return true on success
+     *
+     * Object cooking is generally an expensive process for large projects.
+     * This method blocks execution during the procedure, with periodic
+     * feedback delivered via feedbackCb.
      */
     virtual bool cookPath(const std::string& path,
-                          std::function<void(std::string&, LoadType, unsigned)> feedbackCb,
+                          std::function<void(std::string&, Cost, unsigned)> feedbackCb,
                           bool recursive=false)=0;
 
     /**
      * @brief Interrupts a cook in progress (call from SIGINT handler)
+     *
+     * Database corruption is bad! sqlite is pretty robust at avoiding data corruption,
+     * but HECL spreads its data objects through the filesystem; this ensures that
+     * those objects are cleanly finalized or discarded before stopping.
+     *
+     * Note that this method returns immediately; the resumed cookPath()
+     * call will return as quickly as possible.
      */
     virtual void interruptCook()=0;
 
     /**
-     * @brief Delete cooked objects for file or directory
+     * @brief Delete cooked objects for directory
-     * @param path file or directory of intermediates to clean
+     * @param path directory of intermediates to clean
      * @param recursive traverse subdirectories to clean as well
      * @return true on success
+     *
+     * Developers understand how useful 'clean' is. While ideally not required,
+     * it's useful for verifying that a rebuild from ground-up is doable.
      */
     virtual bool cleanPath(const std::string& path, bool recursive=false)=0;
 
+    /**
+     * @brief Package cooked objects for directory
+     * @param path directory of intermediates to package
+     * @param recursive traverse subdirectories to package as well
+     * @return true on success
+     *
+     * Once all dependent resources are cooked, this method archives specified
+     * intermediates into a packed database file located alongside the specified
+     * directory. This is a similar process to 'linking' in software development.
+     *
+     * Part of this process involves calling IDataObject::_gatherDeps() to calculate
+     * object dependencies. This makes package-assembly simple, as dependencies will
+     * automatically be added as needed. The frontend needn't be concerned about
+     * gathering leaf-objects buried in corners of the working directory.
+     */
+    virtual bool packagePath(const std::string& path, bool recursive=false)=0;
+
 };
 
 /**
  * @brief Creates a new (empty) project using specified root directory
- * @param path Path to project root-directory (may be relative)
+ * @param rootPath Path to project root-directory (may be relative)
  * @return New project object
+ *
+ * This is the preferred way to open an existing or create a new HECL project.
+ * All necessary database index files and object directories will be established
+ * within the specified directory path.
  */
-IProject* NewProject(const std::string& path);
+IProject* NewProject(const std::string& rootPath);
+
+
+/**
+ * @brief Base object to subclass for integrating with key runtime operations
+ *
+ * All runtime objects are provided with IDataObject pointers to their database
+ * entries. Subclasses register themselves with a type registry so instances
+ * are automatically constructed when performing operations like runtime-integration.
+ *
+ * DO NOT CONSTRUCT THIS OR SUBCLASSES DIRECTLY!!
+ */
+class CRuntimeObject
+{
+    unsigned m_refCount = 0;
+    bool m_loaded = false;
+protected:
+
+    /**
+     * @brief Optional subclass method called on background thread or in response to interrupt when data is ready
+     * @param data fully-loaded data buffer
+     * @param len length of buffer
+     * @return true when data is successfully integrated into the runtime
+     */
+    virtual bool _objectFinishedLoading(const void* data, size_t len)
+    {(void)data;(void)len;return true;}
+
+    /**
+     * @brief Optional subclass method called in response to reference-count dropping to 0
+     */
+    virtual void _objectWillUnload() {}
+
+protected:
+    IDataObject* m_obj;
+public:
+    struct ConstructionInfo
+    {
+        IDataObject* obj;
+    };
+    CRuntimeObject(const ConstructionInfo& info)
+    : m_obj(info.obj) {}
+
+    /**
+     * @brief Determine if object is fully loaded and constructed
+     * @return true if so
+     */
+    inline bool isLoaded() const {return m_loaded;}
+
+    /**
+     * @brief Increment object's reference count
+     * @return true if already loaded (ready to use) false if load is staged (poll with isLoaded())
+     *
+     * CRuntimeObject instances initially have an internal reference-count of 0.
+     * By calling this method from 0, an asynchronous load operation takes place.
+     * Synchronous loads are discouraged by HECL in order to avoid stalling game
+     * systems. Please poll with isLoaded() to keep things running smoothly!
+     */
+    bool incRef();
+
+    /**
+     * @brief Decrement object's reference count
+     *
+     * If the internal reference-count reaches 0, the object's unload procedure takes place
+     */
+    void decRef();
+
+};
+
+/**
+ * @brief Runtime data-management interface
+ *
+ * Interface for controlling runtime data-operations like object lookup
+ * and burst load-transactions. The runtime's implementation automatically
+ * constructs CRuntimeObject instances as needed.
+ */
+class IRuntime
+{
+public:
+    virtual ~IRuntime() {}
+
+    /**
+     * @brief Lookup singular object by database ID
+     * @param id database ID of object
+     * @return runtime object
+     */
+    virtual CRuntimeObject* lookupObjectById(size_t id);
+
+    /**
+     * @brief Iterable group view providing a load interface for Dependency Groups
+     *
+     * HECL uses a background thread or other asynchronous loading mechanism to
+     * efficiently load and construct IRuntimeObject instances.
+     *
+     * The iterator interface may be used immediately to access contained objects.
+     */
+    class IStagedGroup
+    {
+    public:
+        /**
+         * @brief Poll to see if transaction complete
+         * @return true if complete
+         */
+        virtual bool isDone() const=0;
+
+        virtual std::vector<CRuntimeObject*>::iterator begin() const=0;
+        virtual std::vector<CRuntimeObject*>::iterator end() const=0;
+    };
+
+    /**
+     * @brief Begin asynchronously loading a dependency group by id
+     * @param groupId the id of the dependency group within the database
+     * @return Staged group interface scheduled to load ASAP
+     */
+    virtual IStagedGroup* loadDependencyGroup(int64_t groupId);
+
+    /**
+     * @brief Unload a previously-loaded dependency group or cancel a load in-progress
+     * @param group Staged Group obtained via loadDependencyGroup()
+     */
+    virtual void unloadDependencyGroup(IStagedGroup* group);
+};
+
+/**
+ * @brief Statically-constructed structure registering a FourCC with project
+ * and runtime factories. This is used for constructing key operational subclasses
+ * for cooking/packaging during development and runtime-integrating during gameplay.
+ */
+struct RegistryEntry
+{
+    const HECL::FourCC& fcc;
+#ifndef HECL_STRIP_PROJECT
+    std::function<CProjectObject*(const CProjectObject::ConstructionInfo&)> projectFactory;
+#endif
+#ifndef HECL_STRIP_RUNTIME
+    std::function<CRuntimeObject*(const CRuntimeObject::ConstructionInfo&)> runtimeFactory;
+#endif
+};
+
+#if !defined(HECL_STRIP_PROJECT) && !defined(HECL_STRIP_RUNTIME)
+
+#define REGISTRY_ENTRY(fourcc, projectClass, runtimeClass) {fourcc, \
+[](const HECLDatabase::CProjectObject::ConstructionInfo& info) -> \
+    HECLDatabase::CProjectObject* {return new projectClass(info);}, \
+[](const HECLDatabase::CRuntimeObject::ConstructionInfo& info) -> \
+    HECLDatabase::CRuntimeObject* {return new runtimeClass(info);}}
+
+#define REGISTRY_SENTINEL() { HECL::FourCC(), \
+[](const HECLDatabase::CProjectObject::ConstructionInfo&) -> \
+    HECLDatabase::CProjectObject* {return nullptr;}, \
+[](const HECLDatabase::CRuntimeObject::ConstructionInfo&) -> \
+    HECLDatabase::CRuntimeObject* {return nullptr;}}
+
+#elif !defined(HECL_STRIP_PROJECT)
+
+#define REGISTRY_ENTRY(fourcc, projectClass, runtimeClass) {fourcc, \
+[](const HECLDatabase::CProjectObject::ConstructionInfo& info) -> \
+    HECLDatabase::CProjectObject* {return new projectClass(info);}}
+
+#define REGISTRY_SENTINEL() { HECL::FourCC(), \
+[](const HECLDatabase::CProjectObject::ConstructionInfo&) -> \
+    HECLDatabase::CProjectObject* {return nullptr;}}
+
+#elif !defined(HECL_STRIP_RUNTIME)
+
+#define REGISTRY_ENTRY(fourcc, projectClass, runtimeClass) {fourcc, \
+[](const HECLDatabase::CRuntimeObject::ConstructionInfo& info) -> \
+    HECLDatabase::CRuntimeObject* {return new runtimeClass(info);}}
+
+#define REGISTRY_SENTINEL() { HECL::FourCC(), \
+[](const HECLDatabase::CRuntimeObject::ConstructionInfo&) -> \
+    HECLDatabase::CRuntimeObject* {return nullptr;}}
+
+#endif
+
+/**
+ * @brief Statically-constructed table of registered types
+ *
+ * Table is defined in dataspec/dataspec.cpp of HECL's codebase.
+ * Developers are encouraged to modify/extend the default data model as
+ * required by their project.
+ *
+ * The REGISTRY_ENTRY macro is a helper for defining entries.
+ * The REGISTRY_SENTINEL must be inserted at the end of the table.
+ */
+extern const RegistryEntry DATASPEC_TYPE_REGISTRY[];
 
 }
 

hecl/lib/database/CLooseDatabase.hpp

@@ -1,6 +1,5 @@
-#ifndef HECLDATABASE_CPP
+#ifndef CLOOSEDATABASE_HPP
-#error This file must only be included from HECLDatabase.cpp
+#define CLOOSEDATABASE_HPP
-#endif
 
 #include <stdio.h>
 #include <blowfish/blowfish.h>
@@ -18,7 +17,7 @@ class CLooseDatabase final : public IDatabase
     Access m_access;    
 public:
     CLooseDatabase(const std::string& path, Access access)
-    : m_sql(path.c_str(), (access == READONLY) ? true : false),
+    : m_sql(path.c_str(), (access == A_READONLY) ? true : false),
       m_access(access)
     {
 
@@ -31,7 +30,7 @@ public:
 
     Type getType() const
     {
-        return LOOSE;
+        return T_LOOSE;
     }
 
     Access getAccess() const
@@ -65,7 +64,7 @@ public:
 
     bool writeDatabase(IDatabase::Type type, const std::string& path) const
     {
-        if (type == PACKED)
+        if (type == T_PACKED)
         {
             size_t bufSz;
             void* buf = m_sql.fillDBBuffer(bufSz);
@@ -79,3 +78,5 @@ public:
 };
 
 }
+
+#endif // CLOOSEDATABASE_HPP

hecl/lib/database/CMemoryDatabase.hpp

@@ -1,6 +1,5 @@
-#ifndef HECLDATABASE_CPP
+#ifndef CMEMORYDATABASE_HPP
-#error This file must only be included from HECLDatabase.cpp
+#define CMEMORYDATABASE_HPP
-#endif
 
 #include "HECLDatabase.hpp"
 #include "CSQLite.hpp"
@@ -14,7 +13,7 @@ class CMemoryDatabase final : public IDatabase
     Access m_access;
 public:
     CMemoryDatabase(Access access)
-    : m_sql(":memory:", (m_access == READONLY) ? true : false), m_access(access)
+    : m_sql(":memory:", (m_access == A_READONLY) ? true : false), m_access(access)
     {
 
     }
@@ -26,7 +25,7 @@ public:
 
     Type getType() const
     {
-        return MEMORY;
+        return T_MEMORY;
     }
 
     Access getAccess() const
@@ -57,3 +56,5 @@ public:
 };
 
 }
+
+#endif // CMEMORYDATABASE_HPP

hecl/lib/database/CPackedDatabase.hpp

@@ -1,6 +1,5 @@
-#ifndef HECLDATABASE_CPP
+#ifndef CPACKEDDATABASE_HPP
-#error This file must only be included from HECLDatabase.cpp
+#define CPACKEDDATABASE_HPP
-#endif
 
 #include "HECLDatabase.hpp"
 #include "CSQLite.hpp"
@@ -25,12 +24,12 @@ public:
 
     Type getType() const
     {
-        return PACKED;
+        return T_PACKED;
     }
 
     Access getAccess() const
     {
-        return READONLY;
+        return A_READONLY;
     }
 
     const IDataObject* lookupObject(size_t id) const
@@ -56,3 +55,5 @@ public:
 };
 
 }
+
+#endif // CPACKEDDATABASE_HPP

hecl/lib/database/CProject.cpp

@@ -0,0 +1,102 @@
+#include <sys/stat.h>
+#include <errno.h>
+#include <string.h>
+#include <system_error>
+
+#include "HECLDatabase.hpp"
+#include "CLooseDatabase.hpp"
+
+namespace HECLDatabase
+{
+
+class CProject : public IProject
+{
+    std::string m_rootPath;
+    IDatabase* m_mainDb;
+    IDatabase* m_cookedDb;
+public:
+    CProject(const std::string& rootPath)
+    : m_rootPath(rootPath)
+    {
+        /* Stat for existing project directory (must already exist) */
+        struct stat myStat;
+        if (stat(m_rootPath.c_str(), &myStat))
+            throw std::error_code(errno, std::system_category());
+
+        if (!S_ISDIR(myStat.st_mode))
+            throw std::invalid_argument("provided path must be a directory; '" + m_rootPath + "' isn't");
+
+        /* Create project directory */
+        if (mkdir((m_rootPath + "/.hecl").c_str(), 0755))
+        {
+            if (errno != EEXIST)
+                throw std::error_code(errno, std::system_category());
+        }
+
+        /* Create or open databases */
+        m_mainDb = new CLooseDatabase(m_rootPath + "/.hecl/main.db", IDatabase::A_READWRITE);
+        m_cookedDb = new CLooseDatabase(m_rootPath + "/.hecl/cooked.db", IDatabase::A_READWRITE);
+    }
+
+    IDatabase* mainDatabase() const
+    {
+    }
+
+    IDatabase* cookedDatabase() const
+    {
+    }
+
+    void registerLogger(HECL::TLogger logger)
+    {
+    }
+
+    std::string getProjectRootPath(bool absolute) const
+    {
+    }
+
+    bool validateSubPath(const std::string& subpath) const
+    {
+    }
+
+    bool addPath(const std::string& path)
+    {
+    }
+
+    bool removePath(const std::string& path, bool recursive)
+    {
+    }
+
+    bool addGroup(const std::string& path)
+    {
+    }
+
+    bool removeGroup(const std::string& path)
+    {
+    }
+
+    bool cookPath(const std::string& path,
+                          std::function<void(std::string&, Cost, unsigned)> feedbackCb,
+                          bool recursive)
+    {
+    }
+
+    void interruptCook()
+    {
+    }
+
+    bool cleanPath(const std::string& path, bool recursive)
+    {
+    }
+
+    bool packagePath(const std::string& path, bool recursive)
+    {
+    }
+
+};
+
+IProject* NewProject(const std::string& rootPath)
+{
+    return new CProject(rootPath);
+}
+
+}

hecl/lib/database/CRuntime.cpp

@@ -0,0 +1,11 @@
+
+#include "HECLDatabase.hpp"
+
+namespace HECLDatabase
+{
+
+class CRuntime : public IRuntime
+{
+};
+
+}

hecl/lib/database/HECLDatabase.cpp

@@ -1,6 +1,7 @@
+
+
 #include "HECLDatabase.hpp"
 
-#define HECLDATABASE_CPP
 #include "CLooseDatabase.hpp"
 #include "CPackedDatabase.hpp"
 #include "CMemoryDatabase.hpp"
@@ -12,13 +13,13 @@ IDatabase* NewDatabase(IDatabase::Type type, IDatabase::Access access, const std
 {
     switch (type)
     {
-    case IDatabase::LOOSE:
+    case IDatabase::T_LOOSE:
         return new CLooseDatabase(path, access);
-    case IDatabase::PACKED:
+    case IDatabase::T_PACKED:
         return new CPackedDatabase(path);
-    case IDatabase::MEMORY:
+    case IDatabase::T_MEMORY:
         return new CMemoryDatabase(access);
-    case IDatabase::UNKNOWN:
+    case IDatabase::T_UNKNOWN:
         return nullptr;
     }
     return nullptr;

hecl/lib/database/database.pri

@@ -7,4 +7,6 @@ HEADERS += \
 
 SOURCES += \
     $$PWD/HECLDatabase.cpp \
-    $$PWD/sqlite_hecl_mem_vfs.c
+    $$PWD/sqlite_hecl_mem_vfs.c \
+    $$PWD/CRuntime.cpp \
+    $$PWD/CProject.cpp