lots of dataspec implementation

hecl/dataspec/dataspec.cpp

@@ -1,10 +1,30 @@
 #include "HECLDatabase.hpp"
 
-const std::pair<std::string, std::string> DATA_SPECS[] =
+namespace HECL
 {
-    {"hecl-little", "Targets little-endian pc apps using the HECL runtime"},
+namespace Database
-    {"hecl-big", "Targets big-endian pc apps using the HECL runtime"},
+{
-    {"hecl-revolution", "Targets Wii apps using the HECL runtime"},
+
-    {"hecl-cafe", "Targets Wii U apps using the HECL runtime"},
+IDataSpec* NewDataSpec_little();
-    {"mp1", ""}
+IDataSpec* NewDataSpec_big();
+IDataSpec* NewDataSpec_revolution();
+IDataSpec* NewDataSpec_cafe();
+
+IDataSpec* NewDataSpec_mp1();
+IDataSpec* NewDataSpec_mp2();
+IDataSpec* NewDataSpec_mp3();
+
+const DataSpecEntry DATA_SPEC_REGISTRY[] =
+{
+    {_S("hecl-little"), _S("Targets little-endian PC apps using the HECL runtime"), NewDataSpec_little},
+    {_S("hecl-big"), _S("Targets big-endian PC apps using the HECL runtime"), NewDataSpec_big},
+    {_S("hecl-revolution"), _S("Targets Wii apps using the HECL runtime"), NewDataSpec_revolution},
+    {_S("hecl-cafe"), _S("Targets Wii U apps using the HECL runtime"), NewDataSpec_cafe},
+    {_S("mp1"), _S("Targets original Metroid Prime engine"), NewDataSpec_mp1},
+    {_S("mp2"), _S("Targets original Metroid Prime 2 engine"), NewDataSpec_mp2},
+    {_S("mp3"), _S("Targets original Metroid Prime 3 engine"), NewDataSpec_mp3},
+    {}
 };
+
+}
+}

hecl/dataspec/hecl/hecl.cpp

@@ -0,0 +1,50 @@
+#include "HECLDatabase.hpp"
+
+namespace HECL
+{
+namespace Database
+{
+
+class HECLBaseDataSpec : public IDataSpec
+{
+
+};
+
+class HECLLittleDataSpec : public HECLBaseDataSpec
+{
+};
+
+class HECLBigDataSpec : public HECLBaseDataSpec
+{
+};
+
+class HECLRevolutionDataSpec : public HECLBaseDataSpec
+{
+};
+
+class HECLCafeDataSpec : public HECLBaseDataSpec
+{
+};
+
+IDataSpec* NewDataSpec_little()
+{
+    return new HECLLittleDataSpec();
+}
+
+IDataSpec* NewDataSpec_big()
+{
+    return new HECLBigDataSpec();
+}
+
+IDataSpec* NewDataSpec_revolution()
+{
+    return new HECLRevolutionDataSpec();
+}
+
+IDataSpec* NewDataSpec_cafe()
+{
+    return new HECLCafeDataSpec();
+}
+
+}
+}

hecl/dataspec/hecl/hecl.pri

@@ -0,0 +1,2 @@
+SOURCES += \
+    $$PWD/hecl.cpp

hecl/dataspec/mp1/mp1.cpp

@@ -0,0 +1,22 @@
+#include "HECLDatabase.hpp"
+
+#include "CMDL.hpp"
+#include "MREA.hpp"
+
+namespace HECL
+{
+namespace Database
+{
+
+class MP1DataSpec : public IDataSpec
+{
+
+};
+
+IDataSpec* NewDataSpec_mp1()
+{
+    return new MP1DataSpec();
+}
+
+}
+}

hecl/dataspec/mp1/mp1.pri

@@ -1,2 +1,6 @@
 SOURCES += \
     $$PWD/mp1.cpp
+
+HEADERS += \
+    $$PWD/MREA.hpp \
+    $$PWD/CMDL.hpp

hecl/dataspec/mp2/mp2.cpp

@@ -0,0 +1,22 @@
+#include "HECLDatabase.hpp"
+
+#include "CMDL.hpp"
+#include "MREA.hpp"
+
+namespace HECL
+{
+namespace Database
+{
+
+class MP2DataSpec : public IDataSpec
+{
+
+};
+
+IDataSpec* NewDataSpec_mp2()
+{
+    return new MP2DataSpec();
+}
+
+}
+}

hecl/dataspec/mp2/mp2.pri

@@ -1,2 +1,6 @@
 SOURCES += \
     $$PWD/mp2.cpp
+
+HEADERS += \
+    $$PWD/MREA.hpp \
+    $$PWD/CMDL.hpp

hecl/dataspec/mp3/mp3.cpp

@@ -0,0 +1,22 @@
+#include "HECLDatabase.hpp"
+
+#include "CMDL.hpp"
+#include "MREA.hpp"
+
+namespace HECL
+{
+namespace Database
+{
+
+class MP3DataSpec : public IDataSpec
+{
+
+};
+
+IDataSpec* NewDataSpec_mp3()
+{
+    return new MP3DataSpec();
+}
+
+}
+}

hecl/dataspec/mp3/mp3.pri

@@ -1,2 +1,6 @@
 SOURCES += \
     $$PWD/mp3.cpp
+
+HEADERS += \
+    $$PWD/MREA.hpp \
+    $$PWD/CMDL.hpp

hecl/driver/ToolBase.hpp

@@ -15,6 +15,7 @@ struct ToolPassInfo
     HECL::SystemString cwd;
     std::vector<HECL::SystemString> args;
     HECL::SystemString output;
+    HECL::Database::Project* project = NULL;
     unsigned verbosityLevel = 0;
     bool force = false;
 };

hecl/driver/ToolSpec.hpp

@@ -3,17 +3,59 @@
 
 #include "ToolBase.hpp"
 #include <stdio.h>
+#include <map>
 
 class ToolSpec final : public ToolBase
 {
+    enum Mode
+    {
+        MLIST = 0,
+        MENABLE,
+        MDISABLE
+    } mode = MLIST;
 public:
     ToolSpec(const ToolPassInfo& info)
     : ToolBase(info)
     {
-    }
+        if (info.args.empty())
+            return;
 
-    ~ToolSpec()
+        if (!info.project)
+            throw HECL::Exception(_S("hecl spec must be ran within a project directory"));
+
+        const auto& specs = info.project->getDataSpecs();
+        HECL::SystemString firstArg = info.args[0];
+        HECL::ToLower(firstArg);
+
+        static const HECL::SystemString enable(_S("enable"));
+        static const HECL::SystemString disable(_S("disable"));
+        if (!firstArg.compare(enable))
+            mode = MENABLE;
+        else if (!firstArg.compare(disable))
+            mode = MDISABLE;
+        else
+            return;
+
+        if (info.args.size() < 2)
+            throw HECL::Exception(_S("Speclist argument required"));
+
+        for (auto it = info.args.begin()+1;
+             it != info.args.end();
+             ++it)
         {
+
+            bool found = false;
+            for (auto& spec : specs)
+            {
+                if (!spec.first.name.compare(*it))
+                {
+                    found = true;
+                    break;
+                }
+            }
+            if (!found)
+                throw HECL::Exception(_S("'") + *it + _S("' is not found in the dataspec registry"));
+        }
     }
 
     static void Help(HelpOutput& help)
@@ -46,6 +88,67 @@ public:
 
     int run()
     {
+        if (!m_info.project)
+        {
+            for (const HECL::Database::DataSpecEntry* spec = HECL::Database::DATA_SPEC_REGISTRY;
+                 spec->name.size();
+                 ++spec)
+            {
+                if (XTERM_COLOR)
+                    HECL::Printf(_S("" BOLD CYAN "%s" NORMAL "\n"), spec->name.c_str());
+                else
+                    HECL::Printf(_S("%s\n"), spec->name.c_str());
+                HECL::Printf(_S("  %s\n"), spec->desc.c_str());
+            }
+            return 0;
+        }
+
+        const auto& specs = m_info.project->getDataSpecs();
+        if (mode == MLIST)
+        {
+            for (auto& spec : specs)
+            {
+                if (XTERM_COLOR)
+                    HECL::Printf(_S("" BOLD CYAN "%s" NORMAL ""), spec.first.name.c_str());
+                else
+                    HECL::Printf(_S("%s"), spec.first.name.c_str());
+                if (spec.second)
+                {
+                    if (XTERM_COLOR)
+                        HECL::Printf(_S(" " BOLD GREEN "[ENABLED]" NORMAL ""));
+                    else
+                        HECL::Printf(_S(" [ENABLED]"));
+                }
+                HECL::Printf(_S("\n  %s\n"), spec.first.desc.c_str());
+            }
+            return 0;
+        }
+
+        std::vector<HECL::SystemString> opSpecs;
+        for (auto it = m_info.args.begin()+1;
+             it != m_info.args.end();
+             ++it)
+        {
+            HECL::SystemString itName = *it;
+            HECL::ToLower(itName);
+            for (auto& spec : specs)
+            {
+                if (!spec.first.name.compare(itName))
+                {
+                    opSpecs.push_back(spec.first.name);
+                    break;
+                }
+            }
+        }
+
+        if (opSpecs.size())
+        {
+            if (mode == MENABLE)
+                m_info.project->enableDataSpecs(opSpecs);
+            else if (mode == MDISABLE)
+                m_info.project->disableDataSpecs(opSpecs);
+        }
+
         return 0;
     }
 };

hecl/driver/driver.pro

@@ -7,7 +7,7 @@ unix:LIBS += -std=c++11
 clang:QMAKE_CXXFLAGS += -stdlib=libc++
 clang:LIBS += -stdlib=libc++ -lc++abi
 
-INCLUDEPATH += ../include
+INCLUDEPATH += ../include ../extern/Athena/include
 
 LIBPATH += $$OUT_PWD/../lib \
     $$OUT_PWD/../dataspec \

hecl/driver/main.cpp

@@ -149,34 +149,49 @@ int main(int argc, const char** argv)
             info.args.push_back(arg);
     }
 
+    /* Attempt to find hecl project */
+    HECL::ProjectRootPath* rootPath = HECL::SearchForProject(info.cwd);
+    std::unique_ptr<HECL::Database::Project> project;
+    if (rootPath)
+    {
+        try
+        {
+            project.reset(new HECL::Database::Project(*rootPath));
+            info.project = project.get();
+        }
+        catch (HECL::Exception& ex)
+        {
+            HECL::FPrintf(stderr,
+                          _S("Unable to open discovered project at '%s':\n%s\n"),
+                          rootPath->getAbsolutePath().c_str(), ex.swhat());
+            return -1;
+        }
+    }
+
     /* Construct selected tool */
     HECL::SystemString toolName(argv[1]);
-#if HECL_UCS2
+    HECL::ToLower(toolName);
-    std::transform(toolName.begin(), toolName.end(), toolName.begin(), towlower);
+    std::unique_ptr<ToolBase> tool;
-#else
-    std::transform(toolName.begin(), toolName.end(), toolName.begin(), tolower);
-#endif
-    ToolBase* tool = NULL;
     try
     {
         if (toolName == _S("init"))
-            tool = new ToolInit(info);
+            tool.reset(new ToolInit(info));
         else if (toolName == _S("spec"))
-            tool = new ToolSpec(info);
+            tool.reset(new ToolSpec(info));
         else if (toolName == _S("add"))
-            tool = new ToolAdd(info);
+            tool.reset(new ToolAdd(info));
         else if (toolName == _S("remove") || toolName == _S("rm"))
-            tool = new ToolRemove(info);
+            tool.reset(new ToolRemove(info));
         else if (toolName == _S("group"))
-            tool = new ToolGroup(info);
+            tool.reset(new ToolGroup(info));
         else if (toolName == _S("cook"))
-            tool = new ToolCook(info);
+            tool.reset(new ToolCook(info));
         else if (toolName == _S("clean"))
-            tool = new ToolClean(info);
+            tool.reset(new ToolClean(info));
         else if (toolName == _S("package") || toolName == _S("pack"))
-            tool = new ToolPackage(info);
+            tool.reset(new ToolPackage(info));
         else if (toolName == _S("help"))
-            tool = new ToolHelp(info);
+            tool.reset(new ToolHelp(info));
         else
             throw HECL::Exception(_S("unrecognized tool '") + toolName + _S("'"));
     }
@@ -185,7 +200,6 @@ int main(int argc, const char** argv)
         HECL::FPrintf(stderr,
                       _S("Unable to construct HECL tool '%s':\n%s\n"),
                       toolName.c_str(), ex.swhat());
-        delete tool;
         return -1;
     }
 
@@ -201,10 +215,8 @@ int main(int argc, const char** argv)
     catch (HECL::Exception& ex)
     {
         HECL::FPrintf(stderr, _S("Error running HECL tool '%s':\n%s\n"), toolName.c_str(), ex.swhat());
-        delete tool;
         return -1;
     }
 
-    delete tool;
     return retval;
 }

hecl/include/HECL.hpp

@@ -17,6 +17,7 @@ char* win_realpath(const char* name, char* restrict resolved);
 #include <functional>
 #include <stdexcept>
 #include <string>
+#include <algorithm>
 #include <regex>
 #include "../extern/blowfish/blowfish.h"
 
@@ -33,6 +34,10 @@ std::wstring UTF8ToWide(const std::string& src);
 #if HECL_UCS2
 typedef wchar_t SystemChar;
 typedef std::wstring SystemString;
+static inline void ToLower(SystemString& str)
+{std::transform(str.begin(), str.end(), str.begin(), towlower);}
+static inline void ToUpper(SystemString& str)
+{std::transform(str.begin(), str.end(), str.begin(), towupper);}
 class SystemUTF8View
 {
     std::string m_utf8;
@@ -55,6 +60,10 @@ public:
 #else
 typedef char SystemChar;
 typedef std::string SystemString;
+static inline void ToLower(SystemString& str)
+{std::transform(str.begin(), str.end(), str.begin(), tolower);}
+static inline void ToUpper(SystemString& str)
+{std::transform(str.begin(), str.end(), str.begin(), toupper);}
 class SystemUTF8View
 {
     const std::string& m_utf8;
@@ -232,11 +241,11 @@ public:
     : num(0) {}
     FourCC(const char* name)
     : num(*(uint32_t*)name) {}
-    inline bool operator==(FourCC& other) {return num == other.num;}
+    inline bool operator==(const FourCC& other) const {return num == other.num;}
-    inline bool operator!=(FourCC& other) {return num != other.num;}
+    inline bool operator!=(const FourCC& other) const {return num != other.num;}
-    inline bool operator==(const char* other) {return num == *(uint32_t*)other;}
+    inline bool operator==(const char* other) const {return num == *(uint32_t*)other;}
-    inline bool operator!=(const char* other) {return num != *(uint32_t*)other;}
+    inline bool operator!=(const char* other) const {return num != *(uint32_t*)other;}
-    inline std::string toString() {return std::string(fcc, 4);}
+    inline std::string toString() const {return std::string(fcc, 4);}
 };
 
 /**
@@ -270,12 +279,12 @@ public:
  */
 class Time final
 {
-    uint64_t ts;
+    time_t ts;
 public:
     Time() : ts(time(NULL)) {}
-    Time(uint64_t ti) : ts(ti) {}
+    Time(time_t ti) : ts(ti) {}
     Time(const Time& other) {ts = other.ts;}
-    inline uint64_t getTs() const {return ts;}
+    inline time_t getTs() const {return ts;}
     inline Time& operator=(const Time& other) {ts = other.ts; return *this;}
     inline bool operator==(const Time& other) const {return ts == other.ts;}
     inline bool operator!=(const Time& other) const {return ts != other.ts;}
@@ -302,7 +311,7 @@ class ProjectPath
 {
 protected:
     SystemString m_absPath;
-    const SystemChar* m_relPath = NULL;
+    SystemString m_relPath;
     size_t m_hash = 0;
 #if HECL_UCS2
     std::string m_utf8AbsPath;
@@ -322,7 +331,7 @@ public:
      * @brief Determine if ProjectPath represents project root directory
      * @return true if project root directory
      */
-    inline bool isRoot() const {return (m_relPath == NULL);}
+    inline bool isRoot() const {return m_relPath.empty();}
 
     /**
      * @brief Access fully-canonicalized absolute path
@@ -334,11 +343,12 @@ public:
      * @brief Access fully-canonicalized project-relative path
      * @return Relative pointer to within absolute-path or "." for project root-directory (use isRoot to detect)
      */
-    inline const SystemChar* getRelativePath() const
+    inline const SystemString& getRelativePath() const
     {
-        if (m_relPath)
+        if (m_relPath.size())
             return m_relPath;
-        return _S(".");
+        static const SystemString dot = _S(".");
+        return dot;
     }
 
     /**
@@ -354,7 +364,7 @@ public:
 #endif
     }
 
-    inline const char* getRelativePathUTF8() const
+    inline const std::string& getRelativePathUTF8() const
     {
 #if HECL_UCS2
         return m_utf8RelPath;
@@ -416,11 +426,15 @@ class ProjectRootPath : public ProjectPath
 {
 public:
     ProjectRootPath(const SystemString& path)
-    {
+    {_canonAbsPath(path);}
-        _canonAbsPath(path);
-    }
 };
 
+/**
+ * @brief Search from within provided directory for the project root
+ * @param path absolute or relative file path to search from
+ * @return Newly-constructed root path or NULL if not found
+ */
+ProjectRootPath* SearchForProject(const SystemString& path);
 
 
 /* Type-sensitive byte swappers */

hecl/include/HECLDatabase.hpp

@@ -12,6 +12,8 @@
 #include <stdexcept>
 #include <stdint.h>
 
+#include <Athena/IStreamReader.hpp>
+
 #include "HECL.hpp"
 
 namespace HECL
@@ -19,6 +21,68 @@ namespace HECL
 namespace Database
 {
 
+/**
+ * @brief Nodegraph class for gathering dependency-resolved objects for packaging
+ */
+class PackageDepsgraph
+{
+public:
+    struct Node
+    {
+        enum
+        {
+            NODE_DATA,
+            NODE_GROUP
+        } type;
+        SystemString path;
+        class ObjectBase* projectObj;
+        Node* sub;
+        Node* next;
+    };
+private:
+    friend class Project;
+    std::vector<Node> m_nodes;
+public:
+    const Node* getRootNode() const {return &m_nodes[0];}
+};
+
+/**
+ * @brief Subclassed by dataspec entries to manage per-game aspects of the data pipeline
+ *
+ * The DataSpec class manages interfaces for unpackaging, cooking, and packaging
+ * of data for interacting with a specific system/game-engine.
+ */
+class IDataSpec
+{
+public:
+    struct ExtractPassInfo
+    {
+        SystemString srcpath;
+        ProjectPath subpath;
+    };
+    virtual bool canExtract(const ExtractPassInfo& info) {(void)info;return false;}
+    virtual void doExtract(const ExtractPassInfo& info) {(void)info;}
+
+    struct PackagePassInfo
+    {
+        PackageDepsgraph& depsgraph;
+        ProjectPath subpath;
+    };
+    virtual bool canPackage(const PackagePassInfo& info) {(void)info;return false;}
+    virtual void doPackage(const PackagePassInfo& info) {(void)info;}
+};
+
+/**
+ * @brief IDataSpec registry entry
+ */
+struct DataSpecEntry
+{
+    SystemString name;
+    SystemString desc;
+    std::function<IDataSpec*(void)> factory;
+};
+extern const HECL::Database::DataSpecEntry DATA_SPEC_REGISTRY[];
+
 /**
  * @brief Base object to subclass for integrating with key project operations
  *
@@ -31,7 +95,7 @@ namespace Database
 class ObjectBase
 {
     friend class Project;
-    std::string m_path;
+    SystemString m_path;
 protected:
 
     /**
@@ -86,16 +150,10 @@ protected:
     {(void)depAdder;}
 
 public:
-    ObjectBase(const std::string& path)
+    ObjectBase(const SystemString& path)
     : m_path(path) {}
 
-    inline const std::string& getPath() const {return m_path;}
+    inline const SystemString& getPath() const {return m_path;}
-
-    /**
-     * @brief Overridable function to verify data at provided path
-     * @return true if ProjectObject subclass handles data at provided path/subpath
-     */
-    static bool ClaimPath(const std::string& /*path*/, const std::string& /*subpath*/) {return false;}
 
 };
 
@@ -109,7 +167,11 @@ public:
  */
 class Project
 {
+public:
+    typedef std::vector<std::pair<const DataSpecEntry&, bool>> CompiledSpecs;
+private:
     ProjectRootPath m_rootPath;
+    CompiledSpecs m_compiledSpecs;
 public:
     Project(const HECL::ProjectRootPath& rootPath);
 
@@ -133,6 +195,9 @@ public:
         void unlockAndDiscard();
         void unlockAndCommit();
     };
+    ConfigFile m_specs;
+    ConfigFile m_paths;
+    ConfigFile m_groups;
 
     /**
      * @brief Index file handle
@@ -145,6 +210,7 @@ public:
         SystemString m_filepath;
         const Project& m_project;
         size_t m_maxPathLen = 0;
+        size_t m_onlyUpdatedMaxPathLen = 0;
         FILE* m_lockedFile = NULL;
     public:
         class Entry
@@ -152,12 +218,13 @@ public:
             friend class IndexFile;
             ProjectPath m_path;
             HECL::Time m_lastModtime;
-            bool m_removed = false;
+            bool m_updated = false;
             Entry(const ProjectPath& path, const HECL::Time& lastModtime)
             : m_path(path), m_lastModtime(lastModtime) {}
             Entry(const ProjectPath& path);
         };
     private:
+        size_t m_updatedCount = 0;
         std::vector<Entry> m_entryStore;
         std::unordered_map<ProjectPath, Entry*> m_entryLookup;
     public:
@@ -165,10 +232,10 @@ public:
         const std::vector<Entry>& lockAndRead();
         const std::vector<ProjectPath*> getChangedPaths();
         void addOrUpdatePath(const ProjectPath& path);
-        void removePath(const ProjectPath& path);
         void unlockAndDiscard();
-        void unlockAndCommit();
+        void unlockAndCommit(bool onlyUpdated=false);
     };
+    IndexFile m_index;
 
     /**
      * @brief Internal packagePath() exception
@@ -257,21 +324,21 @@ public:
      * @brief Return map populated with dataspecs targetable by this project interface
      * @return Platform map with name-string keys and enable-status values
      */
-    const std::map<const std::string, const bool>& listDataSpecs();
+    inline const CompiledSpecs& getDataSpecs() {return m_compiledSpecs;}
 
     /**
      * @brief Enable persistent user preference for particular spec string(s)
      * @param specs String(s) representing unique spec(s) from listDataSpecs
      * @return true on success
      */
-    bool enableDataSpecs(const std::vector<std::string>& specs);
+    bool enableDataSpecs(const std::vector<SystemString>& specs);
 
     /**
      * @brief Disable persistent user preference for particular spec string(s)
      * @param specs String(s) representing unique spec(s) from listDataSpecs
      * @return true on success
      */
-    bool disableDataSpecs(const std::vector<std::string>& specs);
+    bool disableDataSpecs(const std::vector<SystemString>& specs);
 
     /**
      * @brief Begin cook process for specified directory
@@ -285,7 +352,7 @@ public:
      * feedback delivered via feedbackCb.
      */
     bool cookPath(const ProjectPath& path,
-                  std::function<void(std::string&, Cost, unsigned)> feedbackCb,
+                  std::function<void(SystemString&, Cost, unsigned)> feedbackCb,
                   bool recursive=false);
 
     /**
@@ -311,31 +378,6 @@ public:
      */
     bool cleanPath(const ProjectPath& path, bool recursive=false);
 
-    /**
-     * @brief Nodegraph class for gathering dependency-resolved objects for packaging
-     */
-    class PackageDepsgraph
-    {
-    public:
-        struct Node
-        {
-            enum
-            {
-                NODE_DATA,
-                NODE_GROUP
-            } type;
-            std::string path;
-            ObjectBase* projectObj;
-            Node* sub;
-            Node* next;
-        };
-    private:
-        friend class Project;
-        std::vector<Node> m_nodes;
-    public:
-        const Node* getRootNode() const {return &m_nodes[0];}
-    };
-
     /**
      * @brief Constructs a full depsgraph of the project-subpath provided
      * @param path Subpath of project to root depsgraph at
@@ -345,19 +387,6 @@ public:
 
 };
 
-
-/**
- * @brief Subclassed by dataspec entries to manage per-game aspects of the data pipeline
- *
- * The DataSpec class manages interfaces for unpackaging, cooking, and packaging
- * of data for interacting with a specific system/game-engine.
- */
-class IDataSpec
-{
-public:
-    virtual Project::PackageDepsgraph packageData();
-};
-
 }
 }
 

hecl/lib/ProjectPath.cpp

@@ -47,11 +47,10 @@ ProjectPath::ProjectPath(const ProjectRootPath& rootPath, const SystemString& pa
         /* Copies of the project root are permitted */
         return;
     }
-    m_relPath = m_absPath.c_str() + ((ProjectPath&)rootPath).m_absPath.size();
+    SystemString::iterator beginit = m_absPath.begin() + ((ProjectPath&)rootPath).m_absPath.size();
-    if (m_relPath[0] == _S('/'))
+    if (*beginit == _S('/'))
-        ++m_relPath;
+        ++beginit;
-    if (m_relPath[0] == _S('\0'))
+    m_relPath = SystemString(beginit, m_absPath.end());
-        m_relPath = NULL;
 
     std::hash<std::string> hash_fn;
     m_hash = hash_fn(std::string(m_relPath));
@@ -81,7 +80,50 @@ ProjectPath::PathType ProjectPath::getPathType() const
 
 Time ProjectPath::getModtime() const
 {
-
+    struct stat theStat;
+    time_t latestTime = 0;
+    if (std::regex_search(m_absPath, regGLOB))
+    {
+        std::vector<SystemString> globReults;
+        getGlobResults(globReults);
+        for (SystemString& path : globReults)
+        {
+            if (!HECL::Stat(path.c_str(), &theStat))
+            {
+                if (S_ISREG(theStat.st_mode) && theStat.st_mtime > latestTime)
+                    latestTime = theStat.st_mtime;
+            }
+        }
+    }
+    if (!HECL::Stat(m_absPath.c_str(), &theStat))
+    {
+        if (S_ISREG(theStat.st_mode))
+        {
+            return Time(theStat.st_mtime);
+        }
+        else if (S_ISDIR(theStat.st_mode))
+        {
+#if _WIN32
+#else
+            DIR* dir = opendir(m_absPath.c_str());
+            dirent* de;
+            while ((de = readdir(dir)))
+            {
+                if (de->d_name[0] == '.')
+                    continue;
+                if (!HECL::Stat(de->d_name, &theStat))
+                {
+                    if (S_ISREG(theStat.st_mode) && theStat.st_mtime > latestTime)
+                        latestTime = theStat.st_mtime;
+                }
+            }
+            closedir(dir);
+#endif
+            return Time(latestTime);
+        }
+    }
+    throw HECL::Exception(_S("invalid path type"));
+    return Time();
 }
 
 static void _recursiveGlob(std::vector<SystemString>& outPaths,
@@ -155,4 +197,43 @@ void ProjectPath::getGlobResults(std::vector<SystemString>& outPaths) const
         _recursiveGlob(outPaths, 1, pathCompMatches, itStr, false);
 }
 
+ProjectRootPath* SearchForProject(const SystemString& path)
+{
+    ProjectRootPath testRoot(path);
+    SystemString::const_iterator begin = testRoot.getAbsolutePath().begin();
+    SystemString::const_iterator end = testRoot.getAbsolutePath().end();
+    while (begin != end)
+    {
+        while (begin != end && *(end-1) != _S('/') && *(end-1) != _S('\\'))
+            --end;
+        if (begin == end)
+            break;
+
+        SystemString testPath(begin, end);
+        SystemString testIndexPath = testPath + _S("/.hecl/index");
+        struct stat theStat;
+        if (!HECL::Stat(testIndexPath.c_str(), &theStat))
+        {
+            if (S_ISREG(theStat.st_mode))
+            {
+                FILE* fp = HECL::Fopen(testIndexPath.c_str(), _S("rb"));
+                if (!fp)
+                    continue;
+                char magic[4];
+                size_t readSize = fread(magic, 1, 4, fp);
+                fclose(fp);
+                if (readSize != 4)
+                    continue;
+                static const HECL::FourCC hecl("HECL");
+                if (HECL::FourCC(magic) != hecl)
+                    continue;
+                return new ProjectRootPath(testPath);
+            }
+        }
+
+        --end;
+    }
+    return NULL;
+}
+
 }

hecl/lib/database/Project.cpp

@@ -176,6 +176,8 @@ const std::vector<Project::IndexFile::Entry>& Project::IndexFile::lockAndRead()
         return m_entryStore;
 
     m_lockedFile = HECL::Fopen(m_filepath.c_str(), _S("r+"), LWRITE);
+    m_maxPathLen = 0;
+    m_onlyUpdatedMaxPathLen = 0;
 
     SIndexHeader header;
     if (fread(&header, 1, sizeof(header), m_lockedFile) != sizeof(header))
@@ -208,6 +210,7 @@ const std::vector<Project::IndexFile::Entry>& Project::IndexFile::lockAndRead()
         }
     }
     delete[] pathBuf;
+    return m_entryStore;
 }
 
 const std::vector<ProjectPath*> Project::IndexFile::getChangedPaths()
@@ -217,12 +220,8 @@ const std::vector<ProjectPath*> Project::IndexFile::getChangedPaths()
 
     std::vector<ProjectPath*> retval;
     for (Project::IndexFile::Entry& ent : m_entryStore)
-    {
-        if (ent.m_removed)
-            continue;
         if (ent.m_lastModtime != ent.m_path.getModtime())
             retval.push_back(&ent.m_path);
-    }
     return retval;
 }
 
@@ -231,27 +230,20 @@ void Project::IndexFile::addOrUpdatePath(const ProjectPath& path)
     if (!m_lockedFile)
         throw HECL::Exception(_S("Project::IndexFile::lockAndRead not yet called"));
 
+    size_t pathLen = path.getRelativePath().size();
+    if (pathLen > m_onlyUpdatedMaxPathLen)
+        m_onlyUpdatedMaxPathLen = pathLen;
+
     std::unordered_map<ProjectPath, Entry*>::iterator it = m_entryLookup.find(path);
     if (it == m_entryLookup.end())
     {
         m_entryStore.push_back(Project::IndexFile::Entry(path, path.getModtime()));
         m_entryLookup[path] = &m_entryStore.back();
+        m_entryStore.back().m_updated = true;
         return;
     }
     (*it).second->m_lastModtime = path.getModtime();
-}
+    (*it).second->m_updated = true;
-
-void Project::IndexFile::removePath(const ProjectPath& path)
-{
-    if (!m_lockedFile)
-        throw HECL::Exception(_S("Project::IndexFile::lockAndRead not yet called"));
-
-    std::unordered_map<ProjectPath, Entry*>::iterator it = m_entryLookup.find(path);
-    if (it != m_entryLookup.end())
-    {
-        (*it).second->m_removed = true;
-        m_entryLookup.erase(it);
-    }
 }
 
 void Project::IndexFile::unlockAndDiscard()
@@ -265,7 +257,7 @@ void Project::IndexFile::unlockAndDiscard()
     m_lockedFile = NULL;
 }
 
-void Project::IndexFile::unlockAndCommit()
+void Project::IndexFile::unlockAndCommit(bool onlyUpdated)
 {
     if (!m_lockedFile)
         throw HECL::Exception(_S("Project::IndexFile::lockAndRead not yet called"));
@@ -279,20 +271,24 @@ void Project::IndexFile::unlockAndCommit()
 
     SIndexHeader header =
     {
-        HECL::FourCC("HECL"),
+        HECL::FourCC("HECL"), 1,
-        1, (uint32_t)m_entryStore.size(), (uint32_t)m_maxPathLen
+        (uint32_t)(onlyUpdated ? m_updatedCount : m_entryStore.size()),
+        (uint32_t)(onlyUpdated ? m_onlyUpdatedMaxPathLen : m_maxPathLen)
     };
     header.swapWithNative();
     fwrite(&header, 1, sizeof(header), m_lockedFile);
 
     for (Project::IndexFile::Entry& ent : m_entryStore)
+    {
+        if (!onlyUpdated || ent.m_updated)
         {
             uint64_t mt = ToBig(ent.m_lastModtime.getTs());
             fwrite(&mt, 1, 8, m_lockedFile);
-        size_t strLen = strlen(ent.m_path.getRelativePathUTF8());
+            size_t strLen = ent.m_path.getRelativePathUTF8().size();
             uint32_t strLenb = ToBig(strLen);
             fwrite(&strLenb, 1, 4, m_lockedFile);
-        fwrite(ent.m_path.getRelativePathUTF8(), 1, strLen, m_lockedFile);
+            fwrite(ent.m_path.getRelativePathUTF8().c_str(), 1, strLen, m_lockedFile);
+        }
     }
 
     m_entryLookup.clear();
@@ -306,7 +302,11 @@ void Project::IndexFile::unlockAndCommit()
  **********************************************/
 
 Project::Project(const ProjectRootPath& rootPath)
-: m_rootPath(rootPath)
+: m_rootPath(rootPath),
+  m_specs(*this, _S("specs")),
+  m_paths(*this, _S("paths")),
+  m_groups(*this, _S("groups")),
+  m_index(*this)
 {
     /* Stat for existing project directory (must already exist) */
     struct stat myStat;
@@ -321,8 +321,6 @@ Project::Project(const ProjectRootPath& rootPath)
     HECL::MakeDir(m_rootPath.getAbsolutePath() + _S("/.hecl"));
     HECL::MakeDir(m_rootPath.getAbsolutePath() + _S("/.hecl/cooked"));
     HECL::MakeDir(m_rootPath.getAbsolutePath() + _S("/.hecl/config"));
-
-    /* Create or open databases */
 }
 
 void Project::registerLogger(FLogger logger)
@@ -345,10 +343,6 @@ bool Project::removeGroup(const ProjectPath& path)
 {
 }
 
-const std::map<const std::string, const bool>& Project::listDataSpecs()
-{
-}
-
 bool Project::enableDataSpecs(const std::vector<std::string>& specs)
 {
 }
@@ -371,7 +365,7 @@ bool Project::cleanPath(const ProjectPath& path, bool recursive)
 {
 }
 
-Project::PackageDepsgraph Project::buildPackageDepsgraph(const ProjectPath& path)
+PackageDepsgraph Project::buildPackageDepsgraph(const ProjectPath& path)
 {
 }
 

hecl/lib/lib.pro

@@ -9,7 +9,7 @@ unix:LIBS += -std=c++11
 clang:QMAKE_CXXFLAGS += -stdlib=libc++
 clang:LIBS += -stdlib=libc++ -lc++abi
 
-INCLUDEPATH += $$PWD ../include ../extern
+INCLUDEPATH += $$PWD ../include ../extern ../extern/Athena/include
 
 include (frontend/frontend.pri)
 include (backend/backend.pri)