Implement `hecl package`

hecl/blender/hecl/sact/__init__.py

@@ -374,6 +374,19 @@ def get_subtype_names(writebuf):
         writebuf(struct.pack('I', len(subtype.name)))
         writebuf(subtype.name.encode())
 
+# Access subtype's contained overlay names
+def get_subtype_overlay_names(writebuf, subtypeName):
+    sact_data = bpy.context.scene.hecl_sact_data
+    for sub_idx in range(len(sact_data.subtypes)):
+        subtype = sact_data.subtypes[sub_idx]
+        if subtype.name == subtypeName:
+            writebuf(struct.pack('I', len(subtype.overlays)))
+            for overlay in subtype.overlays:
+                writebuf(struct.pack('I', len(overlay.name)))
+                writebuf(overlay.name.encode())
+            return
+    writebuf(struct.pack('I', 0))
+
 # Access actor's contained action names
 def get_action_names(writebuf):
     sact_data = bpy.context.scene.hecl_sact_data
@@ -383,7 +396,6 @@ def get_action_names(writebuf):
         writebuf(struct.pack('I', len(action.name)))
         writebuf(action.name.encode())
 
-
 # Panel draw
 def draw(layout, context):
     SACTSubtype.draw(layout.box(), context)

hecl/blender/hecl_blendershell.py

@@ -392,6 +392,11 @@ def dataout_loop():
             writepipestr(b'OK')
             hecl.sact.get_subtype_names(writepipebuf)
 
+        elif cmdargs[0] == 'GETSUBTYPEOVERLAYNAMES':
+            subtypeName = cmdargs[1]
+            writepipestr(b'OK')
+            hecl.sact.get_subtype_overlay_names(writepipebuf, subtypeName)
+
         elif cmdargs[0] == 'GETACTIONNAMES':
             writepipestr(b'OK')
             hecl.sact.get_action_names(writepipebuf)

hecl/driver/ToolBase.hpp

@@ -9,6 +9,7 @@
 
 #ifndef _WIN32
 #include <unistd.h>
+#include <termios.h>
 #endif
 
 #include "hecl/Database.hpp"
@@ -29,23 +30,6 @@ struct ToolPassInfo
     bool yes = false;
 };
 
-class ToolBase
-{
-protected:
-    const ToolPassInfo& m_info;
-    bool m_good = false;
-public:
-    ToolBase(const ToolPassInfo& info)
-    : m_info(info)
-    {
-        hecl::VerbosityLevel = info.verbosityLevel;
-    }
-    virtual ~ToolBase() {}
-    virtual hecl::SystemString toolName() const=0;
-    virtual int run()=0;
-    inline operator bool() const {return m_good;}
-};
-
 #define RED "\033[0;31m"
 #define GREEN "\033[0;32m"
 #define YELLOW "\033[0;33m"
@@ -62,6 +46,58 @@ public:
 
 extern bool XTERM_COLOR;
 
+class ToolBase
+{
+protected:
+    const ToolPassInfo& m_info;
+    bool m_good = false;
+
+    bool continuePrompt()
+    {
+        if (!m_info.yes)
+        {
+            if (XTERM_COLOR)
+                hecl::Printf(_S("\n" BLUE BOLD "Continue?" NORMAL " (Y/n) "));
+            else
+                hecl::Printf(_S("\nContinue? (Y/n) "));
+
+            int ch;
+#ifndef _WIN32
+            struct termios tioOld, tioNew;
+            tcgetattr(0, &tioOld);
+            tioNew = tioOld;
+            tioNew.c_lflag &= ~ICANON;
+            tcsetattr(0, TCSANOW, &tioNew);
+            while ((ch = getchar()))
+#else
+            while ((ch = getch()))
+#endif
+            {
+                if (ch == 'n' || ch == 'N')
+                    return false;
+                if (ch == 'y' || ch == 'Y' || ch == '\r' || ch == '\n')
+                    break;
+            }
+#ifndef _WIN32
+            tcsetattr(0, TCSANOW, &tioOld);
+#endif
+        }
+        hecl::Printf(_S("\n"));
+        return true;
+    }
+
+public:
+    ToolBase(const ToolPassInfo& info)
+    : m_info(info)
+    {
+        hecl::VerbosityLevel = info.verbosityLevel;
+    }
+    virtual ~ToolBase() {}
+    virtual hecl::SystemString toolName() const=0;
+    virtual int run()=0;
+    inline operator bool() const {return m_good;}
+};
+
 class HelpOutput
 {
 public:

hecl/driver/ToolCook.hpp

@@ -3,6 +3,7 @@
 
 #include "ToolBase.hpp"
 #include <stdio.h>
+#include "hecl/ClientProcess.hpp"
 
 class ToolCook final : public ToolBase
 {
@@ -136,16 +137,13 @@ public:
 
     int run()
     {
+        hecl::ClientProcess cp(m_info.verbosityLevel, m_fast, m_info.force);
         for (const hecl::ProjectPath& path : m_selectedItems)
         {
             int lineIdx = 0;
-            m_useProj->cookPath(path,
+            m_useProj->cookPath(path, {}, m_recursive, m_info.force, m_fast, &cp);
-            [&lineIdx](const hecl::SystemChar* message,
-                       const hecl::SystemChar* submessage,
-                       int lidx, float factor)
-            {ToolPrintProgress(message, submessage, lidx, factor, lineIdx);},
-            m_recursive, m_info.force, m_fast);
         }
+        cp.waitUntilComplete();
         return 0;
     }
 };

hecl/driver/ToolExtract.hpp

@@ -25,7 +25,7 @@ class ToolExtract final : public ToolBase
     std::vector<SpecExtractPass> m_specPasses;
     std::vector<hecl::Database::IDataSpec::ExtractReport> m_reps;
     std::unique_ptr<hecl::Database::Project> m_fallbackProj;
-    hecl::Database::Project* m_useProj;
+    hecl::Database::Project* m_useProj = nullptr;
 public:
     ToolExtract(const ToolPassInfo& info)
     : ToolBase(info)
@@ -154,37 +154,8 @@ public:
             hecl::Printf(_S("\n"));
         }
 
-        if (!m_info.yes)
+        if (continuePrompt())
         {
-            if (XTERM_COLOR)
-                hecl::Printf(_S("\n" BLUE BOLD "Continue?" NORMAL " (Y/n) "));
-            else
-                hecl::Printf(_S("\nContinue? (Y/n) "));
-
-            int ch;
-#ifndef _WIN32
-            struct termios tioOld, tioNew;
-            tcgetattr(0, &tioOld);
-            tioNew = tioOld;
-            tioNew.c_lflag &= ~ICANON;
-            tcsetattr(0, TCSANOW, &tioNew);
-            while ((ch = getchar()))
-#else
-            while ((ch = getch()))
-#endif
-            {
-                if (ch == 'n' || ch == 'N')
-                    return 0;
-                if (ch == 'y' || ch == 'Y' || ch == '\r' || ch == '\n')
-                    break;
-            }
-#ifndef _WIN32
-            tcsetattr(0, TCSANOW, &tioOld);
-#endif
-        }
-
-        hecl::Printf(_S("\n"));
-
             for (SpecExtractPass& ds : m_specPasses)
             {
                 if (XTERM_COLOR)
@@ -200,6 +171,7 @@ public:
                                          {ToolPrintProgress(message, submessage, lidx, factor, lineIdx);});
                 hecl::Printf(_S("\n\n"));
             }
+        }
 
         return 0;
     }

hecl/driver/ToolPackage.hpp

@@ -8,12 +8,97 @@
 
 class ToolPackage final : public ToolBase
 {
+    std::vector<hecl::ProjectPath> m_selectedItems;
+    std::unique_ptr<hecl::Database::Project> m_fallbackProj;
+    hecl::Database::Project* m_useProj;
+    bool m_fast = false;
+
+    void AddSelectedItem(const hecl::ProjectPath& path)
+    {
+        for (const hecl::ProjectPath& item : m_selectedItems)
+            if (item == path)
+                return;
+        m_selectedItems.push_back(path);
+    }
+
+    void FindSelectedItems(const hecl::ProjectPath& path, bool checkGeneral)
+    {
+        size_t origSize = m_selectedItems.size();
+
+        hecl::DirectoryEnumerator dEnum(path.getAbsolutePath(),
+                                        hecl::DirectoryEnumerator::Mode::DirsThenFilesSorted, false, false, true);
+        for (const auto& ent : dEnum)
+        {
+            hecl::ProjectPath childPath(path, ent.m_name);
+            if (ent.m_isDir)
+                FindSelectedItems(childPath, checkGeneral && childPath.getPathComponents().size() <= 2);
+            else if (ent.m_name == _S("!world.blend"))
+                AddSelectedItem(childPath);
+        }
+
+        /* Directory with 2 components, not "Shared" and no nested !world.blend files == General PAK */
+        if (path.getPathType() == hecl::ProjectPath::Type::Directory && checkGeneral &&
+            origSize == m_selectedItems.size())
+        {
+            auto pathComps = path.getPathComponents();
+            if (pathComps.size() == 2 && pathComps[0] != _S("out") && pathComps[1] != _S("Shared"))
+                AddSelectedItem(path);
+        }
+    }
+
 public:
     ToolPackage(const ToolPassInfo& info)
-    : ToolBase(info)
+    : ToolBase(info), m_useProj(info.project)
     {
         if (!info.project)
             LogModule.report(logvisor::Fatal, "hecl package must be ran within a project directory");
+
+        /* Scan args */
+        if (info.args.size())
+        {
+            /* See if project path is supplied via args and use that over the getcwd one */
+            m_selectedItems.reserve(info.args.size());
+            for (const hecl::SystemString& arg : info.args)
+            {
+                if (arg.empty())
+                    continue;
+                else if (!arg.compare(_S("--fast")))
+                {
+                    m_fast = true;
+                    continue;
+                }
+                else if (arg.size() >= 2 && arg[0] == _S('-') && arg[1] == _S('-'))
+                    continue;
+
+                hecl::SystemString subPath;
+                hecl::ProjectRootPath root = hecl::SearchForProject(MakePathArgAbsolute(arg, info.cwd), subPath);
+
+                if (root)
+                {
+                    if (!m_fallbackProj)
+                    {
+                        m_fallbackProj.reset(new hecl::Database::Project(root));
+                        m_useProj = m_fallbackProj.get();
+                    }
+                    else if (m_fallbackProj->getProjectRootPath() != root)
+                        LogModule.report(logvisor::Fatal,
+                                         _S("hecl package can only process multiple items in the same project; ")
+                                         _S("'%s' and '%s' are different projects"),
+                                         m_fallbackProj->getProjectRootPath().getAbsolutePath().c_str(),
+                                         root.getAbsolutePath().c_str());
+
+                    FindSelectedItems({*m_useProj, subPath}, true);
+                }
+            }
+        }
+        if (!m_useProj)
+            LogModule.report(logvisor::Fatal,
+                             "hecl package must be ran within a project directory or "
+                             "provided a path within a project");
+
+        /* Default case: recursive at root */
+        if (m_selectedItems.empty())
+            FindSelectedItems({*m_useProj, _S("")}, true);
     }
 
     ~ToolPackage()
@@ -37,7 +122,7 @@ public:
         help.beginWrap();
         help.wrap(_S("This command initiates a packaging pass on the project database. Packaging ")
                   _S("is analogous to linking in software development. All objects necessary to ")
-                  _S("generate a complete package are gathered, grouped, and indexed within an .hlpk file.\n"));
+                  _S("generate a complete package are gathered, grouped, and indexed within a .upak file.\n"));
         help.endWrap();
 
         help.secHead(_S("OPTIONS"));
@@ -51,7 +136,7 @@ public:
         help.optionHead(_S("-o <package-out>"), _S("output package file"));
         help.beginWrap();
         help.wrap(_S("Specifies a target path to write the package. If not specified, the package ")
-                  _S("is written into <project-root>/out/<relative-input-dirs>/<input-dir>.hlpk\n"));
+                  _S("is written into <project-root>/out/<relative-input-dirs>/<input-dir>.upak\n"));
         help.endWrap();
 
         help.optionHead(_S("-a"), _S("auto cook"));
@@ -65,6 +150,25 @@ public:
 
     int run()
     {
+        if (XTERM_COLOR)
+            hecl::Printf(_S("" GREEN BOLD "ABOUT TO PACKAGE:" NORMAL "\n"));
+        else
+            hecl::Printf(_S("ABOUT TO PACKAGE:\n"));
+
+        for (auto& item : m_selectedItems)
+            hecl::Printf(_S("%s\n"), item.getRelativePath().c_str());
+
+        if (continuePrompt())
+        {
+            hecl::ClientProcess cp(m_info.verbosityLevel, m_fast, m_info.force);
+            for (const hecl::ProjectPath& path : m_selectedItems)
+            {
+                if (!m_useProj->packagePath(path, {}, m_fast, &cp))
+                    LogModule.report(logvisor::Error, _S("Unable to package %s"), path.getAbsolutePath().c_str());
+            }
+            cp.waitUntilComplete();
+        }
+
         return 0;
     }
 };

hecl/include/hecl/Blender/BlenderConnection.hpp

@@ -905,6 +905,7 @@ public:
         std::vector<std::string> getArmatureNames();
         std::vector<std::string> getSubtypeNames();
         std::vector<std::string> getActionNames();
+        std::vector<std::string> getSubtypeOverlayNames(const std::string& name);
 
         struct Matrix3f
         {

hecl/include/hecl/ClientProcess.hpp

@@ -20,6 +20,8 @@ class ClientProcess
     std::condition_variable m_initCv;
     std::condition_variable m_waitCv;
     int m_verbosity;
+    bool m_fast;
+    bool m_force;
 
 public:
     struct Transaction
@@ -84,7 +86,7 @@ private:
     static ThreadLocalPtr<ClientProcess::Worker> ThreadWorker;
 
 public:
-    ClientProcess(int verbosityLevel=1);
+    ClientProcess(int verbosityLevel=1, bool fast=false, bool force=false);
     ~ClientProcess() {shutdown();}
     std::shared_ptr<const BufferTransaction>
     addBufferTransaction(const hecl::ProjectPath& path, void* target,

hecl/include/hecl/Database.hpp

@@ -23,6 +23,7 @@
 namespace hecl
 {
 class BlenderToken;
+class ClientProcess;
 
 namespace Database
 {
@@ -116,24 +117,11 @@ public:
                         bool fast, BlenderToken& btok, FCookProgress progress)
     {(void)path;(void)cookedPath;(void)fast;(void)progress;}
 
-    /**
+    virtual bool canPackage(const hecl::ProjectPath& path)
-     * @brief Package Pass Info
+    {(void)path;return false;}
-     *
+    virtual void doPackage(const hecl::ProjectPath& path, const hecl::Database::DataSpecEntry* entry,
-     * A package pass performs last-minute queries of source resources,
+                           bool fast, hecl::BlenderToken& btok, FProgress progress, ClientProcess* cp=nullptr)
-     * gathers dependencies and packages cooked data together in the
+    {(void)path;}
-     * most efficient form for the dataspec
-     */
-    struct PackagePassInfo
-    {
-        const PackageDepsgraph& depsgraph;
-        ProjectPath subpath;
-        ProjectPath outpath;
-    };
-    virtual bool canPackage(const PackagePassInfo& info,
-                            SystemString& reasonNo)
-    {(void)info;reasonNo=_S("not implemented");return false;}
-    virtual void doPackage(const PackagePassInfo& info)
-    {(void)info;}
 
     const DataSpecEntry* getDataSpecEntry() const {return m_specEntry;}
 };
@@ -271,6 +259,8 @@ private:
     ProjectPath m_cookedRoot;
     std::vector<ProjectDataSpec> m_compiledSpecs;
     std::unordered_map<uint64_t, ProjectPath> m_bridgePathCache;
+    std::vector<std::unique_ptr<IDataSpec>> m_cookSpecs;
+    std::unique_ptr<IDataSpec> m_lastPackageSpec;
     bool m_valid = false;
 public:
     Project(const hecl::ProjectRootPath& rootPath);
@@ -414,6 +404,7 @@ public:
      * @param feedbackCb a callback to run reporting cook-progress
      * @param recursive traverse subdirectories to cook as well
      * @param fast enables faster (draft) extraction for supported data types
+     * @param cp if non-null, cook asynchronously via the ClientProcess
      * @return true on success
      *
      * Object cooking is generally an expensive process for large projects.
@@ -421,7 +412,18 @@ public:
      * feedback delivered via feedbackCb.
      */
     bool cookPath(const ProjectPath& path, FProgress feedbackCb,
-                  bool recursive=false, bool force=false, bool fast=false);
+                  bool recursive=false, bool force=false, bool fast=false,
+                  ClientProcess* cp=nullptr);
+
+    /**
+     * @brief Begin package process for specified !world.blend or directory
+     * @param path Path to !world.blend or directory
+     * @param feedbackCb a callback to run reporting cook-progress
+     * @param fast enables faster (draft) extraction for supported data types
+     * @param cp if non-null, cook asynchronously via the ClientProcess
+     */
+    bool packagePath(const ProjectPath& path, FProgress feedbackCb,
+                     bool fast=false, ClientProcess* cp=nullptr);
 
     /**
      * @brief Interrupts a cook in progress (call from SIGINT handler)

hecl/include/hecl/hecl.hpp

@@ -1163,9 +1163,8 @@ public:
     /**
      * @brief Insert glob matches into existing vector
      * @param outPaths Vector to add matches to (will not erase existing contents)
-     * @param startPath Path to start searching for matches from
      */
-    void getGlobResults(std::vector<ProjectPath>& outPaths, const SystemString& startPath = _S("")) const;
+    void getGlobResults(std::vector<ProjectPath>& outPaths) const;
 
     /**
      * @brief Count how many directory levels deep in project path is

hecl/lib/Blender/BlenderConnection.cpp

@@ -1791,6 +1791,39 @@ std::vector<std::string> BlenderConnection::DataStream::getActionNames()
     return ret;
 }
 
+std::vector<std::string> BlenderConnection::DataStream::getSubtypeOverlayNames(const std::string& name)
+{
+    if (m_parent->m_loadedType != BlendType::Actor)
+        BlenderLog.report(logvisor::Fatal, _S("%s is not an ACTOR blend"),
+                          m_parent->m_loadedBlend.getAbsolutePath().c_str());
+
+    char req[128];
+    snprintf(req, 128, "GETSUBTYPEOVERLAYNAMES %s", name.c_str());
+    m_parent->_writeStr(req);
+
+    char readBuf[256];
+    m_parent->_readStr(readBuf, 256);
+    if (strcmp(readBuf, "OK"))
+        BlenderLog.report(logvisor::Fatal, "unable to get subtype overlays of actor: %s", readBuf);
+
+    std::vector<std::string> ret;
+
+    uint32_t subCount;
+    m_parent->_readBuf(&subCount, 4);
+    ret.reserve(subCount);
+    for (uint32_t i=0 ; i<subCount ; ++i)
+    {
+        ret.emplace_back();
+        std::string& name = ret.back();
+        uint32_t bufSz;
+        m_parent->_readBuf(&bufSz, 4);
+        name.assign(bufSz, ' ');
+        m_parent->_readBuf(&name[0], bufSz);
+    }
+
+    return ret;
+}
+
 std::unordered_map<std::string, BlenderConnection::DataStream::Matrix3f>
 BlenderConnection::DataStream::getBoneMatrices(const std::string& name)
 {

hecl/lib/ClientProcess.cpp

@@ -47,13 +47,6 @@ void ClientProcess::BufferTransaction::run(BlenderToken& btok)
 void ClientProcess::CookTransaction::run(BlenderToken& btok)
 {
     m_dataSpec->setThreadProject();
-    if (m_path.getAuxInfo().empty())
-        LogModule.report(logvisor::Info, _S("Cooking %s"),
-                         m_path.getRelativePath().c_str());
-    else
-        LogModule.report(logvisor::Info, _S("Cooking %s|%s"),
-                         m_path.getRelativePath().c_str(),
-                         m_path.getAuxInfo().c_str());
     m_returnResult = m_parent.syncCook(m_path, m_dataSpec, btok);
     m_complete = true;
 }
@@ -105,8 +98,8 @@ void ClientProcess::Worker::proc()
     m_blendTok.shutdown();
 }
 
-ClientProcess::ClientProcess(int verbosityLevel)
+ClientProcess::ClientProcess(int verbosityLevel, bool fast, bool force)
-: m_verbosity(verbosityLevel)
+: m_verbosity(verbosityLevel), m_fast(fast), m_force(force)
 {
 #ifdef HECL_MULTIPROCESSOR
     const int cpuCount = GetCPUCount();
@@ -161,8 +154,21 @@ bool ClientProcess::syncCook(const hecl::ProjectPath& path, Database::IDataSpec*
         if (specEnt)
         {
             hecl::ProjectPath cooked = path.getCookedPath(*specEnt);
+            if (m_fast)
+                cooked = cooked.getWithExtension(_S(".fast"));
             cooked.makeDirChain(false);
+            if (m_force || cooked.getPathType() == ProjectPath::Type::None ||
+                path.getModtime() > cooked.getModtime())
+            {
+                if (path.getAuxInfo().empty())
+                    LogModule.report(logvisor::Info, _S("Cooking %s"),
+                                     path.getRelativePath().c_str());
+                else
+                    LogModule.report(logvisor::Info, _S("Cooking %s|%s"),
+                                     path.getRelativePath().c_str(),
+                                     path.getAuxInfo().c_str());
                 spec->doCook(path, cooked, false, btok, [](const SystemChar*) {});
+            }
             return true;
         }
     }

hecl/lib/Project.cpp

@@ -11,6 +11,7 @@
 
 #include "hecl/Database.hpp"
 #include "hecl/Blender/BlenderConnection.hpp"
+#include "hecl/ClientProcess.hpp"
 
 namespace hecl
 {
@@ -56,6 +57,7 @@ std::vector<std::string>& Project::ConfigFile::lockAndRead()
         return m_lines;
 
     m_lockedFile = hecl::Fopen(m_filepath.c_str(), _S("a+"), FileLockType::Write);
+    hecl::FSeek(m_lockedFile, 0, SEEK_SET);
 
     std::string mainString;
     char readBuf[1024];
@@ -366,6 +368,7 @@ public:
         submsg += _S(" (");
         submsg += specEnt->m_name;
         submsg += _S(')');
+        if (m_progFunc)
             m_progFunc(m_dir, submsg.c_str(), lidx, m_prog);
     }
     void reportFile(const DataSpecEntry* specEnt, const SystemChar* extra)
@@ -376,22 +379,32 @@ public:
         submsg += _S(", ");
         submsg += extra;
         submsg += _S(')');
+        if (m_progFunc)
             m_progFunc(m_dir, submsg.c_str(), lidx, m_prog);
     }
-    void reportDirComplete() {m_progFunc(m_dir, nullptr, lidx, 1.0);}
+    void reportDirComplete()
+    {
+        if (m_progFunc)
+            m_progFunc(m_dir, nullptr, lidx, 1.0);
+    }
 };
 
-using SpecInst = std::pair<const DataSpecEntry*, std::unique_ptr<IDataSpec>>;
-
 static void VisitFile(const ProjectPath& path, bool force, bool fast,
-                      std::vector<SpecInst>& specInsts,
+                      std::vector<std::unique_ptr<IDataSpec>>& specInsts,
-                      CookProgress& progress)
+                      CookProgress& progress, ClientProcess* cp)
 {
-    for (SpecInst& spec : specInsts)
+    for (auto& spec : specInsts)
     {
-        if (spec.second->canCook(path, hecl::SharedBlenderToken))
+        if (spec->canCook(path, hecl::SharedBlenderToken))
         {
-            const DataSpecEntry* override = spec.second->overrideDataSpec(path, spec.first, hecl::SharedBlenderToken);
+            if (cp)
+            {
+                cp->addCookTransaction(path, spec.get());
+            }
+            else
+            {
+                const DataSpecEntry* override = spec->overrideDataSpec(path, spec->getDataSpecEntry(),
+                                                                       hecl::SharedBlenderToken);
                 if (!override)
                     continue;
                 ProjectPath cooked = path.getCookedPath(*override);
@@ -401,7 +414,7 @@ static void VisitFile(const ProjectPath& path, bool force, bool fast,
                     path.getModtime() > cooked.getModtime())
                 {
                     progress.reportFile(override);
-                spec.second->doCook(path, cooked, fast, hecl::SharedBlenderToken,
+                    spec->doCook(path, cooked, fast, hecl::SharedBlenderToken,
                                  [&](const SystemChar* extra)
                                  {
                                      progress.reportFile(override, extra);
@@ -409,13 +422,17 @@ static void VisitFile(const ProjectPath& path, bool force, bool fast,
                 }
             }
         }
+    }
 }
 
 static void VisitDirectory(const ProjectPath& dir,
                            bool recursive, bool force, bool fast,
-                           std::vector<SpecInst>& specInsts,
+                           std::vector<std::unique_ptr<IDataSpec>>& specInsts,
-                           CookProgress& progress)
+                           CookProgress& progress, ClientProcess* cp)
 {
+    if (dir.getLastComponent()[0] == _S('.'))
+        return;
+
     std::map<SystemString, ProjectPath> children;
     dir.getDirChildren(children);
 
@@ -434,7 +451,7 @@ static void VisitDirectory(const ProjectPath& dir,
         if (child.second.getPathType() == ProjectPath::Type::File)
         {
             progress.changeFile(child.first.c_str(), progNum++/progDenom);
-            VisitFile(child.second, force, fast, specInsts, progress);
+            VisitFile(child.second, force, fast, specInsts, progress, cp);
         }
     }
     progress.reportDirComplete();
@@ -448,7 +465,7 @@ static void VisitDirectory(const ProjectPath& dir,
             {
             case ProjectPath::Type::Directory:
             {
-                VisitDirectory(child.second, recursive, force, fast, specInsts, progress);
+                VisitDirectory(child.second, recursive, force, fast, specInsts, progress, cp);
                 break;
             }
             default: break;
@@ -457,70 +474,32 @@ static void VisitDirectory(const ProjectPath& dir,
     }
 }
 
-static void VisitGlob(const ProjectPath& path,
-                      bool recursive, bool force, bool fast,
-                      std::vector<SpecInst>& specInsts,
-                      CookProgress& progress)
-{
-    std::vector<ProjectPath> children;
-    path.getGlobResults(children, path.getProject().getProjectRootPath().getAbsolutePath());
-
-    /* Pass 1: child file count */
-    int childFileCount = 0;
-    for (ProjectPath& child : children)
-        if (child.getPathType() == ProjectPath::Type::File)
-            ++childFileCount;
-
-    /* Pass 2: child files */
-    int progNum = 0;
-    float progDenom = childFileCount;
-    progress.changeDir(path.getLastComponent());
-    for (ProjectPath& child : children)
-    {
-        if (child.getPathType() == ProjectPath::Type::File)
-        {
-            progress.changeFile(child.getLastComponent(), progNum++/progDenom);
-            VisitFile(child, force, fast, specInsts, progress);
-        }
-    }
-    progress.reportDirComplete();
-
-    /* Pass 3: child directories */
-    if (recursive)
-        for (ProjectPath& child : children)
-            if (child.getPathType() == ProjectPath::Type::Directory)
-                VisitDirectory(child, recursive, force, fast, specInsts, progress);
-}
-
 bool Project::cookPath(const ProjectPath& path, FProgress progress,
-                       bool recursive, bool force, bool fast)
+                       bool recursive, bool force, bool fast, ClientProcess* cp)
 {
     /* Construct DataSpec instances for cooking */
-    std::vector<SpecInst> specInsts;
+    if (m_cookSpecs.empty())
-    specInsts.reserve(m_compiledSpecs.size());
+    {
+        m_cookSpecs.reserve(m_compiledSpecs.size());
         for (const ProjectDataSpec& spec : m_compiledSpecs)
             if (spec.active && spec.spec.m_factory)
-            specInsts.emplace_back(&spec.spec,
+                m_cookSpecs.push_back(std::unique_ptr<IDataSpec>(spec.spec.m_factory(*this, DataSpecTool::Cook)));
-            std::unique_ptr<IDataSpec>(spec.spec.m_factory(*this, DataSpecTool::Cook)));
+    }
 
     /* Iterate complete directory/file/glob list */
     CookProgress cookProg(progress);
     switch (path.getPathType())
     {
     case ProjectPath::Type::File:
+    case ProjectPath::Type::Glob:
     {
         cookProg.changeFile(path.getLastComponent(), 0.0);
-        VisitFile(path, force, fast, specInsts, cookProg);
+        VisitFile(path, force, fast, m_cookSpecs, cookProg, cp);
         break;
     }
     case ProjectPath::Type::Directory:
     {
-        VisitDirectory(path, recursive, force, fast, specInsts, cookProg);
+        VisitDirectory(path, recursive, force, fast, m_cookSpecs, cookProg, cp);
-        break;
-    }
-    case ProjectPath::Type::Glob:
-    {
-        VisitGlob(path, recursive, force, fast, specInsts, cookProg);
         break;
     }
     default: break;
@@ -529,6 +508,39 @@ bool Project::cookPath(const ProjectPath& path, FProgress progress,
     return true;
 }
 
+bool Project::packagePath(const ProjectPath& path, FProgress progress, bool fast, ClientProcess* cp)
+{
+    /* Construct DataSpec instance for packaging */
+    const DataSpecEntry* specEntry = nullptr;
+    bool foundPC = false;
+    for (const ProjectDataSpec& spec : m_compiledSpecs)
+    {
+        if (spec.active && spec.spec.m_factory)
+        {
+            if (hecl::StringUtils::EndsWith(spec.spec.m_name, _S("-PC")))
+            {
+                foundPC = true;
+                specEntry = &spec.spec;
+            }
+            else if (!foundPC)
+            {
+                specEntry = &spec.spec;
+            }
+        }
+    }
+
+    if (specEntry && (!m_lastPackageSpec || m_lastPackageSpec->getDataSpecEntry() != specEntry))
+        m_lastPackageSpec = std::unique_ptr<IDataSpec>(specEntry->m_factory(*this, DataSpecTool::Package));
+
+    if (m_lastPackageSpec->canPackage(path))
+    {
+        m_lastPackageSpec->doPackage(path, specEntry, fast, hecl::SharedBlenderToken, progress, cp);
+        return true;
+    }
+
+    return false;
+}
+
 void Project::interruptCook()
 {
 }

hecl/lib/ProjectPath.cpp

@@ -151,7 +151,7 @@ Time ProjectPath::getModtime() const
     if (m_absPath.find(_S('*')) != SystemString::npos)
     {
         std::vector<ProjectPath> globResults;
-        getGlobResults(globResults, m_proj->getProjectRootPath().getAbsolutePath());
+        getGlobResults(globResults);
         for (ProjectPath& path : globResults)
         {
             if (!hecl::Stat(path.getAbsolutePath().c_str(), &theStat))
@@ -197,13 +197,21 @@ static void _recursiveGlob(Database::Project& proj,
         return;
 
     const SystemString& comp = matches[1];
-    if (comp.find(_S('*')) != SystemString::npos)
+    if (comp.find(_S('*')) == SystemString::npos)
     {
         SystemString nextItStr = itStr;
         if (needSlash)
             nextItStr += _S('/');
         nextItStr += comp;
+
+        hecl::Sstat theStat;
+        if (Stat(nextItStr.c_str(), &theStat))
+            return;
+
+        if (S_ISDIR(theStat.st_mode))
             _recursiveGlob(proj, outPaths, matches.suffix(), nextItStr, true);
+        else
+            outPaths.emplace_back(proj, nextItStr);
         return;
     }
 
@@ -213,7 +221,7 @@ static void _recursiveGlob(Database::Project& proj,
     hecl::DirectoryEnumerator de(itStr);
     for (const hecl::DirectoryEnumerator::Entry& ent : de)
     {
-        if (std::regex_search(ent.m_name, regComp))
+        if (std::regex_match(ent.m_name, regComp))
         {
             SystemString nextItStr = itStr;
             if (needSlash)
@@ -244,24 +252,10 @@ hecl::DirectoryEnumerator ProjectPath::enumerateDir() const
     return hecl::DirectoryEnumerator(m_absPath);
 }
 
-void ProjectPath::getGlobResults(std::vector<ProjectPath>& outPaths, const SystemString& startPath) const
+void ProjectPath::getGlobResults(std::vector<ProjectPath>& outPaths) const
 {
-    SystemString itStr;
+    const SystemString& rootPath = m_proj->getProjectRootPath().getAbsolutePath();
-    if (startPath == _S(""))
+    _recursiveGlob(*m_proj, outPaths, m_relPath, rootPath, rootPath.back() != _S('/'));
-    {
-#if _WIN32
-        SystemRegexMatch letterMatch;
-        if (m_absPath.compare(0, 2, _S("//")))
-            itStr = _S("\\\\");
-        else if (std::regex_search(m_absPath, letterMatch, regDRIVELETTER))
-            if (letterMatch[1].str().size())
-                itStr = letterMatch[1];
-#else
-        itStr = _S("/");
-#endif
-    }
-
-    _recursiveGlob(*m_proj, outPaths, m_absPath, itStr, false);
 }
 
 ProjectRootPath SearchForProject(const SystemString& path)