Initial integration on cook tool

hecl/driver/ToolBase.hpp

@@ -22,7 +22,7 @@ struct ToolPassInfo
     HECL::SystemString cwd;
     std::list<HECL::SystemString> args;
     HECL::SystemString output;
-    HECL::Database::Project* project = NULL;
+    HECL::Database::Project* project = nullptr;
     unsigned verbosityLevel = 0;
     bool force = false;
 };
@@ -210,4 +210,87 @@ public:
     }
 };
 
+void ToolPrintProgress(const HECL::SystemChar* message, const HECL::SystemChar* submessage,
+                       int lidx, float factor, int& lineIdx)
+{
+    factor = std::max(0.0f, std::min(1.0f, factor));
+    int iFactor = factor * 100.0;
+    if (XTERM_COLOR)
+        HECL::Printf(_S("" HIDE_CURSOR ""));
+
+    if (lidx > lineIdx)
+    {
+        HECL::Printf(_S("\n  "));
+        lineIdx = lidx;
+    }
+    else
+        HECL::Printf(_S("  "));
+
+    int width = HECL::ConsoleWidth();
+    int half = width / 2 - 2;
+
+    if (!message)
+        message = _S("");
+    size_t messageLen = HECL::StrLen(message);
+    if (!submessage)
+        submessage = _S("");
+    size_t submessageLen = HECL::StrLen(submessage);
+    if (half - messageLen < submessageLen-2)
+        submessageLen = 0;
+
+    if (submessageLen)
+    {
+        if (messageLen > half-submessageLen-1)
+            HECL::Printf(_S("%.*s... "), half-int(submessageLen)-4, message);
+        else
+        {
+            HECL::Printf(_S("%s"), message);
+            for (int i=half-messageLen-submessageLen-1 ; i>=0 ; --i)
+                HECL::Printf(_S(" "));
+            HECL::Printf(_S("%s "), submessage);
+        }
+    }
+    else
+    {
+        if (messageLen > half)
+            HECL::Printf(_S("%.*s... "), half-3, message);
+        else
+        {
+            HECL::Printf(_S("%s"), message);
+            for (int i=half-messageLen ; i>=0 ; --i)
+                HECL::Printf(_S(" "));
+        }
+    }
+
+    if (XTERM_COLOR)
+    {
+        size_t blocks = half - 7;
+        size_t filled = blocks * factor;
+        size_t rem = blocks - filled;
+        HECL::Printf(_S("" BOLD "%3d%% ["), iFactor);
+        for (int b=0 ; b<filled ; ++b)
+            HECL::Printf(_S("#"));
+        for (int b=0 ; b<rem ; ++b)
+            HECL::Printf(_S("-"));
+        HECL::Printf(_S("]" NORMAL ""));
+    }
+    else
+    {
+        size_t blocks = half - 7;
+        size_t filled = blocks * factor;
+        size_t rem = blocks - filled;
+        HECL::Printf(_S("%3d%% ["), iFactor);
+        for (int b=0 ; b<filled ; ++b)
+            HECL::Printf(_S("#"));
+        for (int b=0 ; b<rem ; ++b)
+            HECL::Printf(_S("-"));
+        HECL::Printf(_S("]"));
+    }
+
+    HECL::Printf(_S("\r"));
+    if (XTERM_COLOR)
+        HECL::Printf(_S("" SHOW_CURSOR ""));
+    fflush(stdout);
+}
+
 #endif // CTOOL_BASE

hecl/driver/ToolCook.hpp

@@ -6,14 +6,60 @@
 
 class ToolCook final : public ToolBase
 {
+    std::list<HECL::ProjectPath> m_selectedItems;
+    std::unique_ptr<HECL::Database::Project> m_fallbackProj;
+    HECL::Database::Project* m_useProj;
+    bool m_recursive = false;
 public:
     ToolCook(const ToolPassInfo& info)
-    : ToolBase(info)
+    : ToolBase(info), m_useProj(info.project)
     {
-    }
+        /* Scan args */
+        if (info.args.size())
+        {
+            /* See if project path is supplied via args and use that over the getcwd one */
+            for (const HECL::SystemString& arg : info.args)
+            {
+                if (arg.empty())
+                    continue;
+                if (arg.size() >= 2 && arg[0] == _S('-'))
+                {
+                    switch (arg[1])
+                    {
+                    case _S('r'):
+                        m_recursive = true;
+                        break;
+                    default: break;
+                    }
+                    continue;
+                }
+                HECL::SystemString subPath;
+                HECL::ProjectRootPath root = HECL::SearchForProject(arg, subPath);
+                if (root)
+                {
+                    if (!m_fallbackProj)
+                    {
+                        m_fallbackProj.reset(new HECL::Database::Project(root));
+                        m_useProj = m_fallbackProj.get();
+                    }
+                    else if (m_fallbackProj->getProjectRootPath() != root)
+                        LogModule.report(LogVisor::FatalError,
+                                         _S("hecl cook can only process multiple items in the same project; ")
+                                         _S("'%s' and '%s' are different projects"),
+                                         m_fallbackProj->getProjectRootPath().getAbsolutePath().c_str(),
+                                         root.getAbsolutePath().c_str());
+                    m_selectedItems.emplace_back(*m_useProj, subPath);
+                }
+            }
+        }
+        if (!m_useProj)
+            LogModule.report(LogVisor::FatalError,
+                             "hecl cook must be ran within a project directory or "
+                             "provided a path within a project");
 
-    ~ToolCook()
+        /* Default case: recursive at root */
-    {
+        if (m_selectedItems.empty())
+            m_selectedItems.push_back({HECL::ProjectPath(*m_useProj, _S("."))});
     }
 
     static void Help(HelpOutput& help)
@@ -72,8 +118,19 @@ public:
 
     HECL::SystemString toolName() const {return _S("cook");}
 
+    using ProjectDataSpec = HECL::Database::Project::ProjectDataSpec;
     int run()
     {
+        for (const HECL::ProjectPath& path : m_selectedItems)
+        {
+            int lineIdx = 0;
+            m_useProj->cookPath(path,
+            [&lineIdx](const HECL::SystemChar* message, const HECL::SystemChar* submessage,
+                       int lidx, float factor)
+            {
+                ToolPrintProgress(message, submessage, lidx, factor, lineIdx);
+            }, m_recursive);
+        }
         return 0;
     }
 };

hecl/driver/ToolExtract.hpp

@@ -22,8 +22,8 @@ class ToolExtract final : public ToolBase
         SpecExtractPass(const SpecExtractPass& other) = delete;
         SpecExtractPass(SpecExtractPass&& other) = default;
     };
-    std::vector<SpecExtractPass> m_specPasses;
+    std::list<SpecExtractPass> m_specPasses;
-    std::vector<HECL::Database::IDataSpec::ExtractReport> m_reps;
+    std::list<HECL::Database::IDataSpec::ExtractReport> m_reps;
     std::unique_ptr<HECL::Database::Project> m_fallbackProj;
     HECL::Database::Project* m_useProj;
 public:
@@ -63,7 +63,6 @@ public:
             m_useProj = info.project;
 
         m_einfo.srcpath = m_info.args.front();
-        m_einfo.extractArgs.reserve(info.args.size() - 1);
         m_einfo.force = info.force;
         std::list<HECL::SystemString>::const_iterator it=info.args.begin();
         ++it;
@@ -191,84 +190,7 @@ public:
             [&lineIdx](const HECL::SystemChar* message, const HECL::SystemChar* submessage,
                        int lidx, float factor)
             {
-                factor = std::max(0.0f, std::min(1.0f, factor));
+                ToolPrintProgress(message, submessage, lidx, factor, lineIdx);
-                int iFactor = factor * 100.0;
-                if (XTERM_COLOR)
-                    HECL::Printf(_S("" HIDE_CURSOR ""));
-
-                if (lidx > lineIdx)
-                {
-                    HECL::Printf(_S("\n  "));
-                    lineIdx = lidx;
-                }
-                else
-                    HECL::Printf(_S("  "));
-
-                int width = HECL::ConsoleWidth();
-                int half = width / 2 - 2;
-
-                if (!message)
-                    message = _S("");
-                size_t messageLen = HECL::StrLen(message);
-                if (!submessage)
-                    submessage = _S("");
-                size_t submessageLen = HECL::StrLen(submessage);
-                if (half - messageLen < submessageLen-2)
-                    submessageLen = 0;
-                
-                if (submessageLen)
-                {
-                    if (messageLen > half-submessageLen-1)
-                        HECL::Printf(_S("%.*s... "), half-int(submessageLen)-4, message);
-                    else
-                    {
-                        HECL::Printf(_S("%s"), message);
-                        for (int i=half-messageLen-submessageLen-1 ; i>=0 ; --i)
-                            HECL::Printf(_S(" "));
-                        HECL::Printf(_S("%s "), submessage);
-                    }
-                }
-                else
-                {
-                    if (messageLen > half)
-                        HECL::Printf(_S("%.*s... "), half-3, message);
-                    else
-                    {
-                        HECL::Printf(_S("%s"), message);
-                        for (int i=half-messageLen ; i>=0 ; --i)
-                            HECL::Printf(_S(" "));
-                    }
-                }
-
-                if (XTERM_COLOR)
-                {
-                    size_t blocks = half - 7;
-                    size_t filled = blocks * factor;
-                    size_t rem = blocks - filled;
-                    HECL::Printf(_S("" BOLD "%3d%% ["), iFactor);
-                    for (int b=0 ; b<filled ; ++b)
-                        HECL::Printf(_S("#"));
-                    for (int b=0 ; b<rem ; ++b)
-                        HECL::Printf(_S("-"));
-                    HECL::Printf(_S("]" NORMAL ""));
-                }
-                else
-                {
-                    size_t blocks = half - 7;
-                    size_t filled = blocks * factor;
-                    size_t rem = blocks - filled;
-                    HECL::Printf(_S("%3d%% ["), iFactor);
-                    for (int b=0 ; b<filled ; ++b)
-                        HECL::Printf(_S("#"));
-                    for (int b=0 ; b<rem ; ++b)
-                        HECL::Printf(_S("-"));
-                    HECL::Printf(_S("]"));
-                }
-
-                HECL::Printf(_S("\r"));
-                if (XTERM_COLOR)
-                    HECL::Printf(_S("" SHOW_CURSOR ""));
-                fflush(stdout);
             });
             HECL::Printf(_S("\n\n"));
         }

hecl/driver/main.cpp

@@ -217,12 +217,12 @@ int main(int argc, const char** argv)
     }
 
     /* Attempt to find hecl project */
-    std::unique_ptr<HECL::ProjectRootPath> rootPath = HECL::SearchForProject(info.cwd);
+    HECL::ProjectRootPath rootPath = HECL::SearchForProject(info.cwd);
     std::unique_ptr<HECL::Database::Project> project;
-    if (rootPath.get())
+    if (rootPath)
     {
         size_t ErrorRef = LogVisor::ErrorCount;
-        HECL::Database::Project* newProj = new HECL::Database::Project(*rootPath);
+        HECL::Database::Project* newProj = new HECL::Database::Project(rootPath);
         if (LogVisor::ErrorCount > ErrorRef)
         {
 #if WIN_PAUSE

hecl/include/HECL/Database.hpp

@@ -28,6 +28,8 @@ class Project;
 
 extern LogVisor::LogModule LogModule;
 
+typedef std::function<void(const HECL::SystemChar*, const HECL::SystemChar*, int, float)> FProgress;
+
 /**
  * @brief Nodegraph class for gathering dependency-resolved objects for packaging
  */
@@ -64,6 +66,7 @@ class IDataSpec
 {
 public:
     virtual ~IDataSpec() {}
+    using FProgress = FProgress;
 
     /**
      * @brief Extract Pass Info
@@ -74,7 +77,7 @@ public:
     struct ExtractPassInfo
     {
         SystemString srcpath;
-        std::vector<SystemString> extractArgs;
+        std::list<SystemString> extractArgs;
         bool force;
     };
 
@@ -91,29 +94,15 @@ public:
         std::vector<ExtractReport> childOpts;
     };
 
-    typedef std::function<void(const HECL::SystemChar*, const HECL::SystemChar*, int, float)> FExtractProgress;
+    virtual bool canExtract(const ExtractPassInfo& info, std::list<ExtractReport>& reps)
-
-    virtual bool canExtract(const ExtractPassInfo& info, std::vector<ExtractReport>& reps)
     {(void)info;LogModule.report(LogVisor::Error, "not implemented");return false;}
-    virtual void doExtract(const ExtractPassInfo& info, FExtractProgress progress)
+    virtual void doExtract(const ExtractPassInfo& info, FProgress progress)
     {(void)info;(void)progress;}
 
-    /**
+    virtual bool canCook(const ProjectPath& path)
-     * @brief Cook Task Info
+    {(void)path;LogModule.report(LogVisor::Error, "not implemented");return false;}
-     *
+    virtual void doCook(const ProjectPath& path, const ProjectPath& cookedPath)
-     * A cook task takes a single tracked path and generates the
+    {(void)path;(void)cookedPath;}
-     * corresponding cooked version
-     */
-    struct CookTaskInfo
-    {
-        ProjectPath path;
-        ProjectPath cookedPath;
-    };
-    virtual bool canCook(const CookTaskInfo& info,
-                         SystemString& reasonNo)
-    {(void)info;reasonNo=_S("not implemented");return false;}
-    virtual void doCook(const CookTaskInfo& info)
-    {(void)info;}
 
     /**
      * @brief Package Pass Info
@@ -243,7 +232,7 @@ public:
     ObjectBase(const SystemString& path)
     : m_path(path) {}
 
-    inline const SystemString& getPath() const {return m_path;}
+    const SystemString& getPath() const {return m_path;}
 
 };
 
@@ -266,6 +255,7 @@ public:
     };
 private:
     ProjectRootPath m_rootPath;
+    ProjectPath m_workRoot;
     ProjectPath m_dotPath;
     ProjectPath m_cookedRoot;
     std::vector<ProjectDataSpec> m_compiledSpecs;
@@ -297,16 +287,6 @@ public:
     ConfigFile m_paths;
     ConfigFile m_groups;
 
-    /**
-     * @brief Internal packagePath() exception
-     *
-     * Due to the recursive nature of packagePath(), there are potential
-     * pitfalls like infinite-recursion. HECL throws this whenever there
-     * are uncooked dependencies or if the maximum dependency-recursion
-     * limit is exceeded.
-     */
-    class PackageException : public std::runtime_error {};
-
     /**
      * @brief A rough description of how 'expensive' a given cook operation is
      *
@@ -326,7 +306,7 @@ public:
      *
      * Self explanatory
      */
-    inline const ProjectRootPath& getProjectRootPath() const {return m_rootPath;}
+    const ProjectPath& getProjectRootPath() const {return m_workRoot;}
 
     /**
      * @brief Get the path of project's cooked directory for a specific DataSpec
@@ -336,7 +316,7 @@ public:
      * The cooked path matches the directory layout of the working directory,
      * except data is
      */
-    inline const ProjectPath& getProjectCookedPath(const DataSpecEntry& spec) const
+    const ProjectPath& getProjectCookedPath(const DataSpecEntry& spec) const
     {
         for (const ProjectDataSpec& sp : m_compiledSpecs)
             if (&sp.spec == &spec)
@@ -400,7 +380,7 @@ public:
      * @brief Return map populated with dataspecs targetable by this project interface
      * @return Platform map with name-string keys and enable-status values
      */
-    inline const std::vector<ProjectDataSpec>& getDataSpecs() {return m_compiledSpecs;}
+    const std::vector<ProjectDataSpec>& getDataSpecs() {return m_compiledSpecs;}
 
     /**
      * @brief Enable persistent user preference for particular spec string(s)
@@ -427,9 +407,7 @@ public:
      * This method blocks execution during the procedure, with periodic
      * feedback delivered via feedbackCb.
      */
-    bool cookPath(const ProjectPath& path,
+    bool cookPath(const ProjectPath& path, FProgress feedbackCb, bool recursive=false);
-                  std::function<void(SystemString&, Cost, unsigned)> feedbackCb,
-                  bool recursive=false);
 
     /**
      * @brief Interrupts a cook in progress (call from SIGINT handler)

hecl/include/HECL/HECL.hpp

@@ -29,12 +29,18 @@
 #include <string>
 #include <algorithm>
 #include <regex>
+#include <list>
 #include <LogVisor/LogVisor.hpp>
-#include <Athena/DNA.hpp>
 #include "../extern/xxhash/xxhash.h"
 
 namespace HECL
 {
+namespace Database
+{
+class Project;
+class DataSpecEntry;
+}
+
 
 extern unsigned VerbosityLevel;
 extern LogVisor::LogModule LogModule;
@@ -334,9 +340,6 @@ class ProjectRootPath;
  * FourCCs are efficient, mnemonic four-char-sequences used to represent types
  * while fitting comfortably in a 32-bit word. HECL uses a four-char array
  * to remain endian-independent.
- *
- * This class also functions as a read/write Athena DNA type,
- * for easy initialization of FourCCs in DNA data records.
  */
 class FourCC
 {
@@ -425,8 +428,34 @@ public:
 class ProjectRootPath
 {
     SystemString m_projRoot;
+    Hash m_hash = 0;
 public:
-    ProjectRootPath(const SystemString& path) : m_projRoot(path) {SanitizePath(m_projRoot);}
+    /**
+     * @brief Empty constructor
+     *
+     * Used to preallocate ProjectPath for later population using assign()
+     */
+    ProjectRootPath() = default;
+
+    /**
+     * @brief Tests for non-empty project root path
+     */
+    operator bool() const {return m_projRoot.size() != 0;}
+
+    /**
+     * @brief Construct a representation of a project root path
+     * @param path valid filesystem-path (relative or absolute) to project root
+     */
+    ProjectRootPath(const SystemString& path) : m_projRoot(path)
+    {
+        SanitizePath(m_projRoot);
+        m_hash = Hash(m_projRoot);
+    }
+
+    /**
+     * @brief Access fully-canonicalized absolute path
+     * @return Absolute path reference
+     */
     const SystemString& getAbsolutePath() const {return m_projRoot;}
 
     /**
@@ -436,6 +465,14 @@ public:
      * If directory already exists, no action taken.
      */
     void makeDir() const {MakeDir(m_projRoot.c_str());}
+
+    /**
+     * @brief HECL-specific xxhash
+     * @return unique hash value
+     */
+    size_t hash() const {return m_hash.val();}
+    bool operator==(const ProjectRootPath& other) const {return m_hash == other.m_hash;}
+    bool operator!=(const ProjectRootPath& other) const {return m_hash != other.m_hash;}
 };
 
 /**
@@ -453,7 +490,7 @@ public:
  */
 class ProjectPath
 {
-    const ProjectRootPath* m_projRoot = nullptr;
+    Database::Project* m_proj = nullptr;
     SystemString m_absPath;
     SystemString m_relPath;
     Hash m_hash = 0;
@@ -475,16 +512,16 @@ public:
     operator bool() const {return m_absPath.size() != 0;}
 
     /**
-     * @brief Construct a project subpath representation within a root path
+     * @brief Construct a project subpath representation within a project's root path
-     * @param parentPath previously constructed ProjectRootPath
+     * @param project previously constructed Project to use root path of
      * @param path valid filesystem-path (relative or absolute) to subpath
      */
-    ProjectPath(const ProjectRootPath& parentPath, const SystemString& path) {assign(parentPath, path);}
+    ProjectPath(Database::Project& project, const SystemString& path) {assign(project, path);}
-    void assign(const ProjectRootPath& parentPath, const SystemString& path);
+    void assign(Database::Project& project, const SystemString& path);
 
 #if HECL_UCS2
-    ProjectPath(const ProjectRootPath& parentPath, const std::string& path) {assign(parentPath, path);}
+    ProjectPath(Database::Project& project, const std::string& path) {assign(project, path);}
-    void assign(const ProjectRootPath& parentPath, const std::string& path);
+    void assign(Database::Project& project, const std::string& path);
 #endif
 
     /**
@@ -537,6 +574,41 @@ public:
         return dot;
     }
 
+    /**
+     * @brief Obtain cooked equivalent of this ProjectPath
+     * @param spec DataSpec to get path against
+     * @return Cooked representation path
+     */
+    ProjectPath getCookedPath(const Database::DataSpecEntry& spec) const;
+
+    /**
+     * @brief Obtain path of parent entity (a directory for file paths)
+     * @return Parent Path
+     *
+     * This will not resolve outside the project root (error in that case)
+     */
+    ProjectPath getParentPath() const
+    {
+        if (m_relPath == _S("."))
+            LogModule.report(LogVisor::FatalError, "attempted to resolve parent of root project path");
+        size_t pos = m_relPath.rfind(_S('/'));
+        if (pos == SystemString::npos)
+            return ProjectPath(*m_proj, _S(""));
+        return ProjectPath(*m_proj, SystemString(m_relPath.begin(), m_relPath.begin() + pos));
+    }
+
+    /**
+     * @brief Obtain c-string of final path component (stored within relative path)
+     * @return Final component c-string
+     */
+    const SystemChar* getLastComponent() const
+    {
+        size_t pos = m_relPath.rfind(_S('/'));
+        if (pos == SystemString::npos)
+            return m_relPath.c_str() + m_relPath.size();
+        return m_relPath.c_str() + pos + 1;
+    }
+
     /**
      * @brief Access fully-canonicalized absolute path in UTF-8
      * @return Absolute path reference
@@ -586,11 +658,17 @@ public:
      */
     Time getModtime() const;
 
+    /**
+     * @brief Insert directory children into list
+     * @param outPaths list to append children to
+     */
+    void getDirChildren(std::list<ProjectPath>& outPaths) const;
+
     /**
      * @brief Insert glob matches into existing vector
      * @param outPaths Vector to add matches to (will not erase existing contents)
      */
-    void getGlobResults(std::vector<SystemString>& outPaths) const;
+    void getGlobResults(std::list<ProjectPath>& outPaths) const;
 
     /**
      * @brief Count how many directory levels deep in project path is
@@ -627,7 +705,7 @@ public:
         MakeLink(relTarget.c_str(), m_absPath.c_str());
     }
     /**
-     * @brief HECL-specific blowfish hash
+     * @brief HECL-specific xxhash
      * @return unique hash value
      */
     size_t hash() const {return m_hash.val();}
@@ -639,9 +717,17 @@ public:
 /**
  * @brief Search from within provided directory for the project root
  * @param path absolute or relative file path to search from
- * @return Newly-constructed root path or NULL if not found
+ * @return Newly-constructed root path (bool-evaluating to false if not found)
  */
-std::unique_ptr<ProjectRootPath> SearchForProject(const SystemString& path);
+ProjectRootPath SearchForProject(const SystemString& path);
+
+/**
+ * @brief Search from within provided directory for the project root
+ * @param path absolute or relative file path to search from
+ * @param subpathOut remainder of provided path assigned to this ProjectPath
+ * @return Newly-constructed root path (bool-evaluating to false if not found)
+ */
+ProjectRootPath SearchForProject(const SystemString& path, SystemString& subpathOut);
 
 #undef bswap16
 #undef bswap32

hecl/lib/Database/Project.cpp

@@ -198,7 +198,8 @@ bool Project::ConfigFile::unlockAndCommit()
 
 Project::Project(const ProjectRootPath& rootPath)
 : m_rootPath(rootPath),
-  m_dotPath(m_rootPath, _S(".hecl")),
+  m_workRoot(*this, _S("")),
+  m_dotPath(m_workRoot, _S(".hecl")),
   m_cookedRoot(m_dotPath, _S("cooked")),
   m_specs(*this, _S("specs")),
   m_paths(*this, _S("paths")),
@@ -338,11 +339,118 @@ bool Project::disableDataSpecs(const std::vector<SystemString>& specs)
     return result;
 }
 
-bool Project::cookPath(const ProjectPath& path,
+static void InsertPath(std::list<std::pair<ProjectPath, std::list<ProjectPath>>>& dirs,
-                       std::function<void(SystemString&, Cost, unsigned)> feedbackCb,
+                       ProjectPath&& path)
-                       bool recursive)
 {
-    return false;
+    ProjectPath thisDir = path.getParentPath();
+    for (std::pair<ProjectPath, std::list<ProjectPath>>& dir : dirs)
+    {
+        if (dir.first == thisDir)
+        {
+            dir.second.push_back(std::move(path));
+            return;
+        }
+    }
+    dirs.emplace_back(std::move(thisDir), std::list<ProjectPath>(std::move(path)));
+}
+
+static void VisitDirectory(std::list<std::pair<ProjectPath, std::list<ProjectPath>>>& allDirs,
+                           const ProjectPath& dir, bool recursive)
+{
+    std::list<ProjectPath> children;
+    dir.getDirChildren(children);
+    for (ProjectPath& child : children)
+    {
+        allDirs.emplace_back(dir, std::list<ProjectPath>());
+        std::list<ProjectPath>& ch = allDirs.back().second;
+        switch (child.getPathType())
+        {
+        case ProjectPath::PT_FILE:
+        {
+            ch.push_back(std::move(child));
+            break;
+        }
+        case ProjectPath::PT_DIRECTORY:
+        {
+            if (recursive)
+                VisitDirectory(allDirs, child, recursive);
+            break;
+        }
+        default: break;
+        }
+    }
+}
+
+bool Project::cookPath(const ProjectPath& path, FProgress feedbackCb, bool recursive)
+{
+    /* Construct DataSpec instances for cooking */
+    std::list<std::pair<const DataSpecEntry*, std::unique_ptr<IDataSpec>>> specInsts;
+    for (const ProjectDataSpec& spec : m_compiledSpecs)
+        if (spec.active)
+            specInsts.emplace_back(&spec.spec, std::unique_ptr<IDataSpec>(spec.spec.m_factory(*this, TOOL_COOK)));
+
+    /* Gather complete directory/file list */
+    std::list<std::pair<ProjectPath, std::list<ProjectPath>>> allDirs;
+    switch (path.getPathType())
+    {
+    case ProjectPath::PT_FILE:
+    {
+        InsertPath(allDirs, std::move(ProjectPath(path)));
+        break;
+    }
+    case ProjectPath::PT_DIRECTORY:
+    {
+        VisitDirectory(allDirs, path, recursive);
+        break;
+    }
+    case ProjectPath::PT_GLOB:
+    {
+        std::list<ProjectPath> results;
+        path.getGlobResults(results);
+        for (ProjectPath& result : results)
+        {
+            switch (result.getPathType())
+            {
+            case ProjectPath::PT_FILE:
+            {
+                InsertPath(allDirs, std::move(result));
+                break;
+            }
+            case ProjectPath::PT_DIRECTORY:
+            {
+                VisitDirectory(allDirs, path, recursive);
+                break;
+            }
+            default: break;
+            }
+        }
+    }
+    default: break;
+    }
+
+    /* Iterate and cook */
+    int lidx = 0;
+    for (const std::pair<ProjectPath, std::list<ProjectPath>>& dir : allDirs)
+    {
+        float dirSz = dir.second.size();
+        int pidx = 0;
+        for (const ProjectPath& path : dir.second)
+        {
+            feedbackCb(dir.first.getLastComponent(), path.getLastComponent(), lidx, pidx++/dirSz);
+            for (std::pair<const DataSpecEntry*, std::unique_ptr<IDataSpec>>& spec : specInsts)
+            {
+                if (spec.second->canCook(path))
+                {
+                    ProjectPath cooked = path.getCookedPath(*spec.first);
+                    if (path.getModtime() > cooked.getModtime())
+                        spec.second->doCook(path, cooked);
+                }
+            }
+        }
+        feedbackCb(dir.first.getLastComponent(), nullptr, lidx++, 1.0);
+    }
+
+    return true;
 }
 
 void Project::interruptCook()

hecl/lib/ProjectPath.cpp

@@ -1,4 +1,5 @@
 #include "HECL/HECL.hpp"
+#include "HECL/Database.hpp"
 #include <regex>
 
 namespace HECL
@@ -55,11 +56,11 @@ static SystemString canonRelPath(const SystemString& path)
     return _S(".");
 }
 
-void ProjectPath::assign(const ProjectRootPath& parentPath, const SystemString& path)
+void ProjectPath::assign(Database::Project& project, const SystemString& path)
 {
-    m_projRoot = &parentPath;
+    m_proj = &project;
     m_relPath = canonRelPath(path);
-    m_absPath = parentPath.getAbsolutePath() + _S('/') + m_relPath;
+    m_absPath = project.getProjectRootPath().getAbsolutePath() + _S('/') + m_relPath;
     SanitizePath(m_relPath);
     SanitizePath(m_absPath);
     m_hash = Hash(m_relPath);
@@ -71,12 +72,12 @@ void ProjectPath::assign(const ProjectRootPath& parentPath, const SystemString&
 }
 
 #if HECL_UCS2
-void ProjectPath::assign(const ProjectRootPath& parentPath, const std::string& path)
+void ProjectPath::assign(Database::Project& project, const std::string& path)
 {
-    m_projRoot = &parentPath;
+    m_proj = &project;
     std::wstring wpath = UTF8ToWide(path);
     m_relPath = canonRelPath(wpath);
-    m_absPath = parentPath.getAbsolutePath() + _S('/') + m_relPath;
+    m_absPath = project.getProjectRootPath().getAbsolutePath() + _S('/') + m_relPath;
     SanitizePath(m_relPath);
     SanitizePath(m_absPath);
     m_hash = Hash(m_relPath);
@@ -87,9 +88,9 @@ void ProjectPath::assign(const ProjectRootPath& parentPath, const std::string& p
 
 void ProjectPath::assign(const ProjectPath& parentPath, const SystemString& path)
 {
-    m_projRoot = parentPath.m_projRoot;
+    m_proj = parentPath.m_proj;
     m_relPath = canonRelPath(parentPath.m_relPath + _S('/') + path);
-    m_absPath = parentPath.m_projRoot->getAbsolutePath() + _S('/') + m_relPath;
+    m_absPath = m_proj->getProjectRootPath().getAbsolutePath() + _S('/') + m_relPath;
     SanitizePath(m_relPath);
     SanitizePath(m_absPath);
     m_hash = Hash(m_relPath);
@@ -106,7 +107,7 @@ void ProjectPath::assign(const ProjectPath& parentPath, const std::string& path)
     m_projRoot = parentPath.m_projRoot;
     std::wstring wpath = UTF8ToWide(path);
     m_relPath = canonRelPath(parentPath.m_relPath + _S('/') + wpath);
-    m_absPath = parentPath.m_projRoot->getAbsolutePath() + _S('/') + m_relPath;
+    m_absPath = m_proj->getProjectRootPath().getAbsolutePath() + _S('/') + m_relPath;
     SanitizePath(m_relPath);
     SanitizePath(m_absPath);
     m_hash = Hash(m_relPath);
@@ -115,6 +116,11 @@ void ProjectPath::assign(const ProjectPath& parentPath, const std::string& path)
 }
 #endif
 
+ProjectPath ProjectPath::getCookedPath(const Database::DataSpecEntry& spec) const
+{
+    return ProjectPath(m_proj->getProjectCookedPath(spec), m_relPath);
+}
+
 ProjectPath::PathType ProjectPath::getPathType() const
 {
     if (std::regex_search(m_absPath, regGLOB))
@@ -135,11 +141,11 @@ Time ProjectPath::getModtime() const
     time_t latestTime = 0;
     if (std::regex_search(m_absPath, regGLOB))
     {
-        std::vector<SystemString> globReults;
+        std::list<ProjectPath> globResults;
-        getGlobResults(globReults);
+        getGlobResults(globResults);
-        for (SystemString& path : globReults)
+        for (ProjectPath& path : globResults)
         {
-            if (!HECL::Stat(path.c_str(), &theStat))
+            if (!HECL::Stat(path.getAbsolutePath().c_str(), &theStat))
             {
                 if (S_ISREG(theStat.st_mode) && theStat.st_mtime > latestTime)
                     latestTime = theStat.st_mtime;
@@ -177,7 +183,8 @@ Time ProjectPath::getModtime() const
     return Time();
 }
 
-static void _recursiveGlob(std::vector<SystemString>& outPaths,
+static void _recursiveGlob(Database::Project& proj,
+                           std::list<ProjectPath>& outPaths,
                            size_t level,
                            const SystemRegexMatch& pathCompMatches,
                            const SystemString& itStr,
@@ -193,7 +200,7 @@ static void _recursiveGlob(std::vector<SystemString>& outPaths,
         if (needSlash)
             nextItStr += _S('/');
         nextItStr += comp;
-        _recursiveGlob(outPaths, level+1, pathCompMatches, nextItStr, true);
+        _recursiveGlob(proj, outPaths, level+1, pathCompMatches, nextItStr, true);
         return;
     }
 
@@ -224,15 +231,36 @@ static void _recursiveGlob(std::vector<SystemString>& outPaths,
                 continue;
 
             if (S_ISDIR(theStat.st_mode))
-                _recursiveGlob(outPaths, level+1, pathCompMatches, nextItStr, true);
+                _recursiveGlob(proj, outPaths, level+1, pathCompMatches, nextItStr, true);
             else if (S_ISREG(theStat.st_mode))
-                outPaths.push_back(nextItStr);
+                outPaths.emplace_back(proj, nextItStr);
         }
     }
+
+    closedir(dir);
 #endif
 }
 
-void ProjectPath::getGlobResults(std::vector<SystemString>& outPaths) const
+void ProjectPath::getDirChildren(std::list<ProjectPath>& outPaths) const
+{
+#if _WIN32
+#else
+    DIR* dir = opendir(m_absPath.c_str());
+    if (!dir)
+    {
+        LogModule.report(LogVisor::Error, "unable to open directory for traversal at '%s'", m_absPath.c_str());
+        return;
+    }
+
+    struct dirent* de;
+    while ((de = readdir(dir)))
+        outPaths.emplace_back(*this, de->d_name);
+
+    closedir(dir);
+#endif
+}
+
+void ProjectPath::getGlobResults(std::list<ProjectPath>& outPaths) const
 {
 #if _WIN32
     SystemString itStr;
@@ -248,10 +276,10 @@ void ProjectPath::getGlobResults(std::vector<SystemString>& outPaths) const
 
     SystemRegexMatch pathCompMatches;
     if (std::regex_search(m_absPath, pathCompMatches, regPATHCOMP))
-        _recursiveGlob(outPaths, 1, pathCompMatches, itStr, false);
+        _recursiveGlob(*m_proj, outPaths, 1, pathCompMatches, itStr, false);
 }
 
-std::unique_ptr<ProjectRootPath> SearchForProject(const SystemString& path)
+ProjectRootPath SearchForProject(const SystemString& path)
 {
     ProjectRootPath testRoot(path);
     SystemString::const_iterator begin = testRoot.getAbsolutePath().begin();
@@ -276,7 +304,7 @@ std::unique_ptr<ProjectRootPath> SearchForProject(const SystemString& path)
                 static const HECL::FourCC hecl("HECL");
                 if (HECL::FourCC(magic) != hecl)
                     continue;
-                return std::unique_ptr<ProjectRootPath>(new ProjectRootPath(testPath));
+                return ProjectRootPath(testPath);
             }
         }
 
@@ -285,7 +313,49 @@ std::unique_ptr<ProjectRootPath> SearchForProject(const SystemString& path)
         if (begin != end)
             --end;
     }
-    return std::unique_ptr<ProjectRootPath>();
+    return ProjectRootPath();
+}
+
+ProjectRootPath SearchForProject(const SystemString& path, SystemString& subpathOut)
+{
+    ProjectRootPath testRoot(path);
+    SystemString::const_iterator begin = testRoot.getAbsolutePath().begin();
+    SystemString::const_iterator end = testRoot.getAbsolutePath().end();
+    while (begin != end)
+    {
+        SystemString testPath(begin, end);
+        SystemString testIndexPath = testPath + _S("/.hecl/beacon");
+        Sstat theStat;
+        if (!HECL::Stat(testIndexPath.c_str(), &theStat))
+        {
+            if (S_ISREG(theStat.st_mode))
+            {
+                FILE* fp = HECL::Fopen(testIndexPath.c_str(), _S("rb"));
+                if (!fp)
+                    continue;
+                char magic[4];
+                size_t readSize = fread(magic, 1, 4, fp);
+                fclose(fp);
+                if (readSize != 4)
+                    continue;
+                static const HECL::FourCC hecl("HECL");
+                if (HECL::FourCC(magic) != hecl)
+                    continue;
+                ProjectRootPath newRootPath = ProjectRootPath(testPath);
+                SystemString::const_iterator origEnd = testRoot.getAbsolutePath().end();
+                while (end != origEnd && *end != _S('/') && *end != _S('\\'))
+                    ++end;
+                subpathOut.assign(end, origEnd);
+                return newRootPath;
+            }
+        }
+
+        while (begin != end && *(end-1) != _S('/') && *(end-1) != _S('\\'))
+            --end;
+        if (begin != end)
+            --end;
+    }
+    return ProjectRootPath();
 }
 
 }