Add parallel progress printing

hecl/driver/ToolBase.hpp

@@ -270,115 +270,4 @@ static hecl::SystemString MakePathArgAbsolute(const hecl::SystemString& arg,
 #endif
 }
 
-static bool g_HasLastProgTime = false;
-static std::chrono::steady_clock::time_point g_LastProgTime;
-
-void ToolPrintProgress(const hecl::SystemChar* message, const hecl::SystemChar* submessage,
-                       int lidx, float factor, int& lineIdx)
-{
-    if (g_HasLastProgTime)
-    {
-        std::chrono::steady_clock::time_point newPoint = std::chrono::steady_clock::now();
-        std::chrono::milliseconds::rep delta =
-                std::chrono::duration_cast<std::chrono::milliseconds>(newPoint - g_LastProgTime).count();
-        if (delta < 50)
-            return;
-        g_LastProgTime = newPoint;
-    }
-    else
-    {
-        g_HasLastProgTime = true;
-        g_LastProgTime = std::chrono::steady_clock::now();
-    }
-
-    auto lk = logvisor::LockLog();
-
-    bool blocks = factor >= 0.0;
-    factor = std::max(0.0f, std::min(1.0f, factor));
-    int iFactor = factor * 100.0;
-    if (XTERM_COLOR)
-        hecl::Printf(_S("" HIDE_CURSOR ""));
-
-    if (lidx > lineIdx)
-    {
-        hecl::Printf(_S("\n  "));
-        lineIdx = lidx;
-    }
-    else
-        hecl::Printf(_S("  "));
-
-    int width = (hecl::GuiMode ? 120 : std::max(80, hecl::ConsoleWidth()));
-    int half;
-    if (blocks)
-        half = width / 2 - 2;
-    else
-        half = width - 4;
-
-    if (!message)
-        message = _S("");
-    int messageLen = hecl::StrLen(message);
-    if (!submessage)
-        submessage = _S("");
-    int submessageLen = hecl::StrLen(submessage);
-    if (half - messageLen < submessageLen-2)
-        submessageLen = 0;
-
-    if (submessageLen)
-    {
-        if (messageLen > half-submessageLen-1)
-            hecl::Printf(_S("%.*s... %s "), half-submessageLen-4, message, submessage);
-        else
-        {
-            hecl::Printf(_S("%s"), message);
-            for (int i=half-messageLen-submessageLen-1 ; i>=0 ; --i)
-                hecl::Printf(_S(" "));
-            hecl::Printf(_S("%s "), submessage);
-        }
-    }
-    else
-    {
-        if (messageLen > half)
-            hecl::Printf(_S("%.*s... "), half-3, message);
-        else
-        {
-            hecl::Printf(_S("%s"), message);
-            for (int i=half-messageLen ; i>=0 ; --i)
-                hecl::Printf(_S(" "));
-        }
-    }
-
-    if (blocks)
-    {
-        if (XTERM_COLOR)
-        {
-            int blocks = half - 7;
-            int filled = blocks * factor;
-            int rem = blocks - filled;
-            hecl::Printf(_S("" BOLD "%3d%% ["), iFactor);
-            for (int b=0 ; b<filled ; ++b)
-                hecl::Printf(_S("#"));
-            for (int b=0 ; b<rem ; ++b)
-                hecl::Printf(_S("-"));
-            hecl::Printf(_S("]" NORMAL ""));
-        }
-        else
-        {
-            int blocks = half - 7;
-            int filled = blocks * factor;
-            int rem = blocks - filled;
-            hecl::Printf(_S("%3d%% ["), iFactor);
-            for (int b=0 ; b<filled ; ++b)
-                hecl::Printf(_S("#"));
-            for (int b=0 ; b<rem ; ++b)
-                hecl::Printf(_S("-"));
-            hecl::Printf(_S("]"));
-        }
-    }
-
-    hecl::Printf(_S("\r"));
-    if (XTERM_COLOR)
-        hecl::Printf(_S("" SHOW_CURSOR ""));
-    fflush(stdout);
-}
-
 #endif // CTOOL_BASE

hecl/driver/ToolCook.hpp

@@ -10,6 +10,7 @@ class ToolCook final : public ToolBase
     std::vector<hecl::ProjectPath> m_selectedItems;
     std::unique_ptr<hecl::Database::Project> m_fallbackProj;
     hecl::Database::Project* m_useProj;
+    const hecl::Database::DataSpecEntry* m_spec = nullptr;
     bool m_recursive = false;
     bool m_fast = false;
 public:
@@ -35,6 +36,21 @@ public:
                     m_fast = true;
                     continue;
                 }
+                else if (arg.size() >= 8 && !arg.compare(0, 7, _S("--spec=")))
+                {
+                    hecl::SystemString specName(arg.begin() + 7, arg.end());
+                    for (const hecl::Database::DataSpecEntry* spec : hecl::Database::DATA_SPEC_REGISTRY)
+                    {
+                        if (!hecl::StrCaseCmp(spec->m_name.data(), specName.c_str()))
+                        {
+                            m_spec = spec;
+                            break;
+                        }
+                    }
+                    if (!m_spec)
+                        LogModule.report(logvisor::Fatal, "unable to find data spec '%s'", specName.c_str());
+                    continue;
+                }
                 else if (arg.size() >= 2 && arg[0] == _S('-') && arg[1] == _S('-'))
                     continue;
 
@@ -80,7 +96,7 @@ public:
 
         help.secHead(_S("SYNOPSIS"));
         help.beginWrap();
-        help.wrap(_S("hecl cook [-rf] [--fast] [<pathspec>...]\n"));
+        help.wrap(_S("hecl cook [-rf] [--fast] [--spec=<spec>] [<pathspec>...]\n"));
         help.endWrap();
 
         help.secHead(_S("DESCRIPTION"));
@@ -131,18 +147,29 @@ public:
         help.beginWrap();
         help.wrap(_S("Performs draft-optimization cooking for supported data types.\n"));
         help.endWrap();
+
+        help.optionHead(_S("--spec=<spec>"), _S("data specification"));
+        help.beginWrap();
+        help.wrap(_S("Specifies a DataSpec to use when cooking. ")
+                  _S("This build of hecl supports the following values of <spec>:\n"));
+        for (const hecl::Database::DataSpecEntry* spec : hecl::Database::DATA_SPEC_REGISTRY)
+        {
+            if (!spec->m_factory)
+                continue;
+            help.wrap(_S("  "));
+            help.wrapBold(spec->m_name.data());
+            help.wrap(_S("\n"));
+        }
     }
 
     hecl::SystemString toolName() const {return _S("cook");}
 
     int run()
     {
-        hecl::ClientProcess cp(m_info.verbosityLevel, m_fast, m_info.force);
+        hecl::MultiProgressPrinter printer(true);
+        hecl::ClientProcess cp(&printer, m_info.verbosityLevel);
         for (const hecl::ProjectPath& path : m_selectedItems)
-        {
+            m_useProj->cookPath(path, printer, m_recursive, m_info.force, m_fast, m_spec, &cp);
-            int lineIdx = 0;
-            m_useProj->cookPath(path, {}, m_recursive, m_info.force, m_fast, &cp);
-        }
         cp.waitUntilComplete();
         return 0;
     }

hecl/driver/ToolExtract.hpp

@@ -10,6 +10,8 @@
 #include <termios.h>
 #endif
 
+#include "hecl/MultiProgressPrinter.hpp"
+
 class ToolExtract final : public ToolBase
 {
     hecl::Database::IDataSpec::ExtractPassInfo m_einfo;
@@ -177,12 +179,7 @@ public:
                 else
                     hecl::Printf(_S("Using DataSpec %s:\n"), ds.m_entry->m_name.data());
 
-                int lineIdx = 0;
+                ds.m_instance->doExtract(m_einfo, {true});
-                ds.m_instance->doExtract(m_einfo,
-                                         [&lineIdx](const hecl::SystemChar* message,
-                                                    const hecl::SystemChar* submessage,
-                                                    int lidx, float factor)
-                                         {ToolPrintProgress(message, submessage, lidx, factor, lineIdx);});
                 hecl::Printf(_S("\n\n"));
             }
         }

hecl/driver/ToolPackage.hpp

@@ -11,6 +11,7 @@ class ToolPackage final : public ToolBase
     std::vector<hecl::ProjectPath> m_selectedItems;
     std::unique_ptr<hecl::Database::Project> m_fallbackProj;
     hecl::Database::Project* m_useProj;
+    const hecl::Database::DataSpecEntry* m_spec = nullptr;
     bool m_fast = false;
 
     void AddSelectedItem(const hecl::ProjectPath& path)
@@ -85,6 +86,21 @@ public:
                     m_fast = true;
                     continue;
                 }
+                else if (arg.size() >= 8 && !arg.compare(0, 7, _S("--spec=")))
+                {
+                    hecl::SystemString specName(arg.begin() + 7, arg.end());
+                    for (const hecl::Database::DataSpecEntry* spec : hecl::Database::DATA_SPEC_REGISTRY)
+                    {
+                        if (!hecl::StrCaseCmp(spec->m_name.data(), specName.c_str()))
+                        {
+                            m_spec = spec;
+                            break;
+                        }
+                    }
+                    if (!m_spec)
+                        LogModule.report(logvisor::Fatal, "unable to find data spec '%s'", specName.c_str());
+                    continue;
+                }
                 else if (arg.size() >= 2 && arg[0] == _S('-') && arg[1] == _S('-'))
                     continue;
 
@@ -133,7 +149,7 @@ public:
 
         help.secHead(_S("SYNOPSIS"));
         help.beginWrap();
-        help.wrap(_S("hecl package [-a] [-o <package-out>] [<input-dir>]\n"));
+        help.wrap(_S("hecl package [--spec=<spec>] [<input-dir>]\n"));
         help.endWrap();
 
         help.secHead(_S("DESCRIPTION"));
@@ -144,23 +160,26 @@ public:
         help.endWrap();
 
         help.secHead(_S("OPTIONS"));
-        help.optionHead(_S("<dir>"), _S("input directory"));
+        help.optionHead(_S("--spec=<spec>"), _S("data specification"));
+        help.beginWrap();
+        help.wrap(_S("Specifies a DataSpec to use when cooking and generating the package. ")
+                  _S("This build of hecl supports the following values of <spec>:\n"));
+        for (const hecl::Database::DataSpecEntry* spec : hecl::Database::DATA_SPEC_REGISTRY)
+        {
+            if (!spec->m_factory)
+                continue;
+            help.wrap(_S("  "));
+            help.wrapBold(spec->m_name.data());
+            help.wrap(_S("\n"));
+        }
+        help.endWrap();
+
+        help.secHead(_S("OPTIONS"));
+        help.optionHead(_S("<input-dir>"), _S("input directory"));
         help.beginWrap();
         help.wrap(_S("Specifies a project subdirectory to root the resulting package from. ")
-                  _S("If any dependent-files fall outside this subdirectory, they will implicitly ")
+                  _S("If any dependent files fall outside this subdirectory, they will be implicitly ")
-                  _S("be gathered and packaged.\n"));
+                  _S("gathered and packaged.\n"));
-        help.endWrap();
-
-        help.optionHead(_S("-o <package-out>"), _S("output package file"));
-        help.beginWrap();
-        help.wrap(_S("Specifies a target path to write the package. If not specified, the package ")
-                  _S("is written into <project-root>/out/<relative-input-dirs>/<input-dir>.upak\n"));
-        help.endWrap();
-
-        help.optionHead(_S("-a"), _S("auto cook"));
-        help.beginWrap();
-        help.wrap(_S("Any referenced objects that haven't already been cooked are automatically cooked as ")
-                  _S("part of the packaging process. If this flag is omitted, the packaging process will abort.\n"));
         help.endWrap();
     }
 
@@ -179,10 +198,11 @@ public:
 
         if (continuePrompt())
         {
-            hecl::ClientProcess cp(m_info.verbosityLevel, m_fast, m_info.force);
+            hecl::MultiProgressPrinter printer(true);
+            hecl::ClientProcess cp(&printer, m_info.verbosityLevel);
             for (const hecl::ProjectPath& path : m_selectedItems)
             {
-                if (!m_useProj->packagePath(path, {}, m_fast, &cp))
+                if (!m_useProj->packagePath(path, printer, m_fast, m_spec, &cp))
                     LogModule.report(logvisor::Error, _S("Unable to package %s"), path.getAbsolutePath().data());
             }
             cp.waitUntilComplete();

hecl/extern/boo

@@ -1 +1 @@
-Subproject commit 4580196f6df62574cab8d7045ae05b0335c0b310
+Subproject commit 088cddfea765a47751e318e2ced31b678b1d406d

hecl/include/hecl/ClientProcess.hpp

@@ -19,9 +19,10 @@ class ClientProcess
     std::condition_variable m_cv;
     std::condition_variable m_initCv;
     std::condition_variable m_waitCv;
+    const MultiProgressPrinter* m_progPrinter;
+    int m_completedCooks = 0;
+    int m_addedCooks = 0;
     int m_verbosity;
-    bool m_fast;
-    bool m_force;
 
 public:
     struct Transaction
@@ -55,9 +56,13 @@ public:
         ProjectPath m_path;
         Database::IDataSpec* m_dataSpec;
         bool m_returnResult = false;
+        bool m_force;
+        bool m_fast;
         void run(blender::Token& btok);
-        CookTransaction(ClientProcess& parent, const ProjectPath& path, Database::IDataSpec* spec)
+        CookTransaction(ClientProcess& parent, const ProjectPath& path,
-        : Transaction(parent, Type::Cook), m_path(path), m_dataSpec(spec) {}
+                        bool force, bool fast, Database::IDataSpec* spec)
+        : Transaction(parent, Type::Cook), m_path(path), m_dataSpec(spec),
+          m_force(force), m_fast(fast) {}
     };
     struct LambdaTransaction : Transaction
     {
@@ -86,16 +91,18 @@ private:
     static ThreadLocalPtr<ClientProcess::Worker> ThreadWorker;
 
 public:
-    ClientProcess(int verbosityLevel=1, bool fast=false, bool force=false);
+    ClientProcess(const MultiProgressPrinter* progPrinter=nullptr, int verbosityLevel=1);
     ~ClientProcess() {shutdown();}
     std::shared_ptr<const BufferTransaction>
     addBufferTransaction(const hecl::ProjectPath& path, void* target,
                          size_t maxLen, size_t offset);
     std::shared_ptr<const CookTransaction>
-    addCookTransaction(const hecl::ProjectPath& path, Database::IDataSpec* spec);
+    addCookTransaction(const hecl::ProjectPath& path, bool force,
+                       bool fast, Database::IDataSpec* spec);
     std::shared_ptr<const LambdaTransaction>
     addLambdaTransaction(std::function<void(blender::Token&)>&& func);
-    bool syncCook(const hecl::ProjectPath& path, Database::IDataSpec* spec, blender::Token& btok);
+    bool syncCook(const hecl::ProjectPath& path, Database::IDataSpec* spec, blender::Token& btok,
+                  bool force, bool fast);
     void swapCompletedQueue(std::list<std::shared_ptr<Transaction>>& queue);
     void waitUntilComplete();
     void shutdown();

hecl/include/hecl/Database.hpp

@@ -30,8 +30,6 @@ class Project;
 
 extern logvisor::Module LogModule;
 
-typedef std::function<void(const hecl::SystemChar*, const hecl::SystemChar*, int, float)> FProgress;
-
 /**
  * @brief Nodegraph class for gathering dependency-resolved objects for packaging
  */
@@ -70,7 +68,6 @@ class IDataSpec
 public:
     IDataSpec(const DataSpecEntry* specEntry) : m_specEntry(specEntry) {}
     virtual ~IDataSpec() {}
-    using FProgress = Database::FProgress;
     using FCookProgress = std::function<void(const SystemChar*)>;
 
     /**
@@ -103,7 +100,7 @@ public:
 
     virtual bool canExtract(const ExtractPassInfo& info, std::vector<ExtractReport>& reps)
     {(void)info;(void)reps;LogModule.report(logvisor::Error, "not implemented");return false;}
-    virtual void doExtract(const ExtractPassInfo& info, FProgress progress)
+    virtual void doExtract(const ExtractPassInfo& info, const MultiProgressPrinter& progress)
     {(void)info;(void)progress;}
 
     virtual bool canCook(const ProjectPath& path, blender::Token& btok)
@@ -119,7 +116,8 @@ public:
     virtual bool canPackage(const ProjectPath& path)
     {(void)path;return false;}
     virtual void doPackage(const ProjectPath& path, const Database::DataSpecEntry* entry,
-                           bool fast, blender::Token& btok, FProgress progress, ClientProcess* cp=nullptr)
+                           bool fast, blender::Token& btok, const MultiProgressPrinter& progress,
+                           ClientProcess* cp=nullptr)
     {(void)path;}
 
     const DataSpecEntry* getDataSpecEntry() const {return m_specEntry;}
@@ -385,14 +383,14 @@ public:
 
     /**
      * @brief Enable persistent user preference for particular spec string(s)
-     * @param specs String(s) representing unique spec(s) from listDataSpecs
+     * @param specs String(s) representing unique spec(s) from getDataSpecs
      * @return true on success
      */
     bool enableDataSpecs(const std::vector<SystemString>& specs);
 
     /**
      * @brief Disable persistent user preference for particular spec string(s)
-     * @param specs String(s) representing unique spec(s) from listDataSpecs
+     * @param specs String(s) representing unique spec(s) from getDataSpecs
      * @return true on success
      */
     bool disableDataSpecs(const std::vector<SystemString>& specs);
@@ -403,6 +401,7 @@ public:
      * @param feedbackCb a callback to run reporting cook-progress
      * @param recursive traverse subdirectories to cook as well
      * @param fast enables faster (draft) extraction for supported data types
+     * @param spec if non-null, cook using a manually-selected dataspec
      * @param cp if non-null, cook asynchronously via the ClientProcess
      * @return true on success
      *
@@ -410,19 +409,21 @@ public:
      * This method blocks execution during the procedure, with periodic
      * feedback delivered via feedbackCb.
      */
-    bool cookPath(const ProjectPath& path, FProgress feedbackCb,
+    bool cookPath(const ProjectPath& path, const MultiProgressPrinter& feedbackCb,
                   bool recursive=false, bool force=false, bool fast=false,
-                  ClientProcess* cp=nullptr);
+                  const DataSpecEntry* spec=nullptr, ClientProcess* cp=nullptr);
 
     /**
      * @brief Begin package process for specified !world.blend or directory
      * @param path Path to !world.blend or directory
      * @param feedbackCb a callback to run reporting cook-progress
      * @param fast enables faster (draft) extraction for supported data types
+     * @param spec if non-null, cook using a manually-selected dataspec
      * @param cp if non-null, cook asynchronously via the ClientProcess
      */
-    bool packagePath(const ProjectPath& path, FProgress feedbackCb,
+    bool packagePath(const ProjectPath& path, const MultiProgressPrinter& feedbackCb,
-                     bool fast=false, ClientProcess* cp=nullptr);
+                     bool fast=false, const DataSpecEntry* spec=nullptr,
+                     ClientProcess* cp=nullptr);
 
     /**
      * @brief Interrupts a cook in progress (call from SIGINT handler)

hecl/include/hecl/MultiProgressPrinter.hpp

@@ -0,0 +1,59 @@
+#ifndef HECLMULTIPROGRESSPRINTER_HPP
+#define HECLMULTIPROGRESSPRINTER_HPP
+
+#include "hecl.hpp"
+
+namespace hecl
+{
+
+class MultiProgressPrinter
+{
+    std::thread m_logThread;
+    mutable std::mutex m_logLock;
+    bool m_newLineAfter;
+
+    struct TermInfo
+    {
+#if _WIN32
+        HANDLE console;
+#endif
+        int width;
+        bool xtermColor = false;
+    } m_termInfo;
+
+    struct ThreadStat
+    {
+        hecl::SystemString m_message, m_submessage;
+        float m_factor = 0.f;
+        bool m_active = false;
+        void print(const TermInfo& tinfo) const;
+    };
+    mutable std::vector<ThreadStat> m_threadStats;
+
+    mutable float m_mainFactor = -1.f;
+    mutable int m_indeterminateCounter = 0;
+    mutable int m_curThreadLines = 0;
+    mutable int m_curProgLines = 0;
+    mutable int m_latestThread = 0;
+    mutable bool m_running = false;
+    mutable bool m_dirty = false;
+    mutable bool m_mainIndeterminate = false;
+    uint64_t m_lastLogCounter = 0;
+    void LogProc();
+    void DoPrint();
+    void DrawIndeterminateBar();
+    void MoveCursorUp(int n);
+public:
+    MultiProgressPrinter(bool activate = false);
+    ~MultiProgressPrinter();
+    void print(const hecl::SystemChar* message, const hecl::SystemChar* submessage,
+               float factor = -1.f, int threadIdx = 0) const;
+    void setMainFactor(float factor) const;
+    void setMainIndeterminate(bool indeterminate) const;
+    void startNewLine() const;
+    void flush() const;
+};
+
+}
+
+#endif // HECLMULTIPROGRESSPRINTER_HPP

hecl/include/hecl/SystemChar.hpp

@@ -38,6 +38,9 @@ static inline void ToUpper(SystemString& str)
 #ifndef _S
 #define _S(val) L ## val
 #endif
+#ifndef FMT_CSTR_SYS
+#define FMT_CSTR_SYS "S"
+#endif
 typedef struct _stat Sstat;
 #else
 typedef char SystemChar;
@@ -51,6 +54,9 @@ static inline void ToUpper(SystemString& str)
 #ifndef _S
 #define _S(val) val
 #endif
+#ifndef FMT_CSTR_SYS
+#define FMT_CSTR_SYS "s"
+#endif
 typedef struct stat Sstat;
 #endif
 

hecl/include/hecl/hecl.hpp

@@ -514,6 +514,8 @@ static inline int ConsoleWidth()
     return retval;
 }
 
+class MultiProgressPrinter;
+
 typedef std::basic_regex<SystemChar> SystemRegex;
 typedef std::regex_token_iterator<SystemString::const_iterator> SystemRegexTokenIterator;
 typedef std::match_results<SystemString::const_iterator> SystemRegexMatch;

hecl/lib/CMakeLists.txt

@@ -30,6 +30,7 @@ set(HECL_HEADERS
     ../include/hecl/Console.hpp
     ../include/hecl/CVarCommons.hpp
     ../include/hecl/hecl.hpp
+    ../include/hecl/MultiProgressPrinter.hpp
     ../include/hecl/FourCC.hpp
     ../include/hecl/HMDLMeta.hpp
     ../include/hecl/Backend/Backend.hpp
@@ -52,6 +53,7 @@ set(HECL_HEADERS
     ../include/hecl/VertexBufferPool.hpp)
 set(COMMON_SOURCES
     hecl.cpp
+    MultiProgressPrinter.cpp
     Project.cpp
     ProjectPath.cpp
     HumanizeNumber.cpp

hecl/lib/ClientProcess.cpp

@@ -2,6 +2,7 @@
 #include "hecl/Database.hpp"
 #include "athena/FileReader.hpp"
 #include "hecl/Blender/Connection.hpp"
+#include "hecl/MultiProgressPrinter.hpp"
 
 #ifdef _WIN32
 #define WIN32_LEAN_AND_MEAN
@@ -47,7 +48,10 @@ void ClientProcess::BufferTransaction::run(blender::Token& btok)
 void ClientProcess::CookTransaction::run(blender::Token& btok)
 {
     m_dataSpec->setThreadProject();
-    m_returnResult = m_parent.syncCook(m_path, m_dataSpec, btok);
+    m_returnResult = m_parent.syncCook(m_path, m_dataSpec, btok, m_force, m_fast);
+    std::unique_lock<std::mutex> lk(m_parent.m_mutex);
+    ++m_parent.m_completedCooks;
+    m_parent.m_progPrinter->setMainFactor(m_parent.m_completedCooks / float(m_parent.m_addedCooks));
     m_complete = true;
 }
 
@@ -98,8 +102,8 @@ void ClientProcess::Worker::proc()
     m_blendTok.shutdown();
 }
 
-ClientProcess::ClientProcess(int verbosityLevel, bool fast, bool force)
+ClientProcess::ClientProcess(const MultiProgressPrinter* progPrinter, int verbosityLevel)
-: m_verbosity(verbosityLevel), m_fast(fast), m_force(force)
+: m_progPrinter(progPrinter), m_verbosity(verbosityLevel)
 {
 #if HECL_MULTIPROCESSOR
     const int cpuCount = GetCPUCount();
@@ -127,12 +131,15 @@ ClientProcess::addBufferTransaction(const ProjectPath& path, void* target,
 }
 
 std::shared_ptr<const ClientProcess::CookTransaction>
-ClientProcess::addCookTransaction(const hecl::ProjectPath& path, Database::IDataSpec* spec)
+ClientProcess::addCookTransaction(const hecl::ProjectPath& path, bool force,
+                                  bool fast, Database::IDataSpec* spec)
 {
     std::unique_lock<std::mutex> lk(m_mutex);
-    auto ret = std::make_shared<CookTransaction>(*this, path, spec);
+    auto ret = std::make_shared<CookTransaction>(*this, path, force, fast, spec);
     m_pendingQueue.emplace_back(ret);
     m_cv.notify_one();
+    ++m_addedCooks;
+    m_progPrinter->setMainFactor(m_completedCooks / float(m_addedCooks));
     return ret;
 }
 
@@ -146,7 +153,8 @@ ClientProcess::addLambdaTransaction(std::function<void(blender::Token&)>&& func)
     return ret;
 }
 
-bool ClientProcess::syncCook(const hecl::ProjectPath& path, Database::IDataSpec* spec, blender::Token& btok)
+bool ClientProcess::syncCook(const hecl::ProjectPath& path, Database::IDataSpec* spec, blender::Token& btok,
+                             bool force, bool fast)
 {
     if (spec->canCook(path, btok))
     {
@@ -154,20 +162,43 @@ bool ClientProcess::syncCook(const hecl::ProjectPath& path, Database::IDataSpec*
         if (specEnt)
         {
             hecl::ProjectPath cooked = path.getCookedPath(*specEnt);
-            if (m_fast)
+            if (fast)
                 cooked = cooked.getWithExtension(_S(".fast"));
             cooked.makeDirChain(false);
-            if (m_force || cooked.getPathType() == ProjectPath::Type::None ||
+            if (force || cooked.getPathType() == ProjectPath::Type::None ||
                 path.getModtime() > cooked.getModtime())
             {
-                if (path.getAuxInfo().empty())
+                if (m_progPrinter)
-                    LogModule.report(logvisor::Info, _S("Cooking %s"),
+                {
-                                     path.getRelativePath().data());
+                    hecl::SystemString str;
+                    if (path.getAuxInfo().empty())
+                        str = hecl::SysFormat(_S("Cooking %s"), path.getRelativePath().data());
+                    else
+                        str = hecl::SysFormat(_S("Cooking %s|%s"), path.getRelativePath().data(), path.getAuxInfo().data());
+                    m_progPrinter->print(str.c_str(), nullptr, -1.f, hecl::ClientProcess::GetThreadWorkerIdx());
+                    m_progPrinter->flush();
+                }
                 else
-                    LogModule.report(logvisor::Info, _S("Cooking %s|%s"),
+                {
-                                     path.getRelativePath().data(),
+                    if (path.getAuxInfo().empty())
-                                     path.getAuxInfo().data());
+                        LogModule.report(logvisor::Info, _S("Cooking %s"),
+                                         path.getRelativePath().data());
+                    else
+                        LogModule.report(logvisor::Info, _S("Cooking %s|%s"),
+                                         path.getRelativePath().data(),
+                                         path.getAuxInfo().data());
+                }
                 spec->doCook(path, cooked, false, btok, [](const SystemChar*) {});
+                if (m_progPrinter)
+                {
+                    hecl::SystemString str;
+                    if (path.getAuxInfo().empty())
+                        str = hecl::SysFormat(_S("Cooked  %s"), path.getRelativePath().data());
+                    else
+                        str = hecl::SysFormat(_S("Cooked  %s|%s"), path.getRelativePath().data(), path.getAuxInfo().data());
+                    m_progPrinter->print(str.c_str(), nullptr, -1.f, hecl::ClientProcess::GetThreadWorkerIdx());
+                    m_progPrinter->flush();
+                }
             }
             return true;
         }

hecl/lib/MultiProgressPrinter.cpp

@@ -0,0 +1,380 @@
+#include "hecl/MultiProgressPrinter.hpp"
+
+#define BOLD "\033[1m"
+#define NORMAL "\033[0m"
+#define PREV_LINE "\033[%dF"
+#define HIDE_CURSOR "\033[?25l"
+#define SHOW_CURSOR "\033[?25h"
+
+#if _WIN32
+#define FOREGROUND_WHITE FOREGROUND_RED|FOREGROUND_GREEN|FOREGROUND_BLUE
+#endif
+
+namespace hecl
+{
+
+void MultiProgressPrinter::ThreadStat::print(const TermInfo& tinfo) const
+{
+    bool blocks = m_factor >= 0.f;
+    float factor = std::max(0.f, std::min(1.f, m_factor));
+    int iFactor = factor * 100.f;
+
+    int half;
+    if (blocks)
+        half = (tinfo.width + 1) / 2 - 2;
+    else
+        half = tinfo.width - 4;
+    int rightHalf = tinfo.width - half - 4;
+
+    int messageLen = m_message.size();
+    int submessageLen = m_submessage.size();
+    if (half - messageLen < submessageLen-2)
+        submessageLen = 0;
+
+    if (submessageLen)
+    {
+        if (messageLen > half-submessageLen-1)
+            hecl::Printf(_S("  %.*s... %s "), half-submessageLen-4, m_message.c_str(), m_submessage.c_str());
+        else
+        {
+            hecl::Printf(_S("  %s"), m_message.c_str());
+            for (int i=half-messageLen-submessageLen-1 ; i>=0 ; --i)
+                hecl::Printf(_S(" "));
+            hecl::Printf(_S("%s "), m_submessage.c_str());
+        }
+    }
+    else
+    {
+        if (messageLen > half)
+            hecl::Printf(_S("  %.*s... "), half-3, m_message.c_str());
+        else
+        {
+            hecl::Printf(_S("  %s"), m_message.c_str());
+            for (int i=half-messageLen ; i>=0 ; --i)
+                hecl::Printf(_S(" "));
+        }
+    }
+
+    if (blocks)
+    {
+        int blocks = rightHalf - 7;
+        int filled = blocks * factor;
+        int rem = blocks - filled;
+
+        if (tinfo.xtermColor)
+        {
+            hecl::Printf(_S("" BOLD "%3d%% ["), iFactor);
+            for (int b=0 ; b<filled ; ++b)
+                hecl::Printf(_S("#"));
+            for (int b=0 ; b<rem ; ++b)
+                hecl::Printf(_S("-"));
+            hecl::Printf(_S("]" NORMAL ""));
+        }
+        else
+        {
+#if _WIN32
+            SetConsoleTextAttribute(tinfo.console, FOREGROUND_INTENSITY | FOREGROUND_WHITE);
+#endif
+            hecl::Printf(_S("%3d%% ["), iFactor);
+            for (int b=0 ; b<filled ; ++b)
+                hecl::Printf(_S("#"));
+            for (int b=0 ; b<rem ; ++b)
+                hecl::Printf(_S("-"));
+            hecl::Printf(_S("]"));
+#if _WIN32
+            SetConsoleTextAttribute(tinfo.console, FOREGROUND_WHITE);
+#endif
+        }
+    }
+}
+
+void MultiProgressPrinter::DrawIndeterminateBar()
+{
+    int half = m_termInfo.width - 2;
+    int blocks = half - 2;
+
+    ++m_indeterminateCounter;
+    if (m_indeterminateCounter <= -blocks)
+        m_indeterminateCounter = -blocks + 1;
+    else if (m_indeterminateCounter >= blocks)
+        m_indeterminateCounter = -blocks + 2;
+    int absCounter = std::abs(m_indeterminateCounter);
+
+    int pre = absCounter;
+    int rem = blocks - pre - 1;
+
+    if (m_termInfo.xtermColor)
+    {
+        hecl::Printf(_S("" BOLD " ["));
+        for (int b=0 ; b<pre ; ++b)
+            hecl::Printf(_S("-"));
+        hecl::Printf(_S("#"));
+        for (int b=0 ; b<rem ; ++b)
+            hecl::Printf(_S("-"));
+        hecl::Printf(_S("]" NORMAL ""));
+    }
+    else
+    {
+#if _WIN32
+        SetConsoleTextAttribute(m_termInfo.console, FOREGROUND_INTENSITY | FOREGROUND_WHITE);
+#endif
+        hecl::Printf(_S(" ["));
+        for (int b=0 ; b<pre ; ++b)
+            hecl::Printf(_S("-"));
+        hecl::Printf(_S("#"));
+        for (int b=0 ; b<rem ; ++b)
+            hecl::Printf(_S("-"));
+        hecl::Printf(_S("]"));
+#if _WIN32
+        SetConsoleTextAttribute(m_termInfo.console, FOREGROUND_WHITE);
+#endif
+    }
+}
+
+void MultiProgressPrinter::MoveCursorUp(int n)
+{
+    if (n)
+    {
+        if (m_termInfo.xtermColor)
+        {
+            hecl::Printf(_S("" PREV_LINE ""), n);
+        }
+#if _WIN32
+        else
+        {
+            CONSOLE_SCREEN_BUFFER_INFO consoleInfo;
+            GetConsoleScreenBufferInfo(m_termInfo.console, &consoleInfo);
+            consoleInfo.dwCursorPosition.X = 0;
+            consoleInfo.dwCursorPosition.Y -= n;
+            SetConsoleCursorPosition(m_termInfo.console, consoleInfo.dwCursorPosition);
+        }
+#endif
+    }
+    else
+    {
+        hecl::Printf(_S("\r"));
+    }
+}
+
+void MultiProgressPrinter::DoPrint()
+{
+    auto logLk = logvisor::LockLog();
+    uint64_t logCounter = logvisor::GetLogCounter();
+    if (logCounter != m_lastLogCounter)
+    {
+        m_curThreadLines = 0;
+        m_lastLogCounter = logCounter;
+    }
+#if _WIN32
+    CONSOLE_CURSOR_INFO cursorInfo;
+    GetConsoleCursorInfo(m_termInfo.console, &cursorInfo);
+    cursorInfo.bVisible = FALSE;
+    SetConsoleCursorInfo(m_termInfo.console, &cursorInfo);
+#endif
+    if (m_termInfo.xtermColor)
+        hecl::Printf(_S("" HIDE_CURSOR ""));
+
+    if (m_dirty)
+    {
+        m_termInfo.width = (hecl::GuiMode ? 120 : std::max(80, hecl::ConsoleWidth()));
+        MoveCursorUp(m_curThreadLines + m_curProgLines);
+        m_curThreadLines = m_curProgLines = 0;
+
+        if (m_newLineAfter)
+        {
+            for (const ThreadStat& stat : m_threadStats)
+            {
+                if (stat.m_active)
+                {
+                    stat.print(m_termInfo);
+                    hecl::Printf(_S("\n"));
+                    ++m_curThreadLines;
+                }
+            }
+
+            if (m_mainIndeterminate)
+            {
+                DrawIndeterminateBar();
+                hecl::Printf(_S("\n"));
+                ++m_curProgLines;
+            }
+            else if (m_mainFactor >= 0.f)
+            {
+                float factor = std::max(0.0f, std::min(1.0f, m_mainFactor));
+                int iFactor = factor * 100.0;
+                int half = m_termInfo.width - 2;
+
+                int blocks = half - 8;
+                int filled = blocks * factor;
+                int rem = blocks - filled;
+
+                if (m_termInfo.xtermColor)
+                {
+                    hecl::Printf(_S("" BOLD "  %3d%% ["), iFactor);
+                    for (int b=0 ; b<filled ; ++b)
+                        hecl::Printf(_S("#"));
+                    for (int b=0 ; b<rem ; ++b)
+                        hecl::Printf(_S("-"));
+                    hecl::Printf(_S("]" NORMAL ""));
+                }
+                else
+                {
+#if _WIN32
+                    SetConsoleTextAttribute(m_termInfo.console, FOREGROUND_INTENSITY | FOREGROUND_WHITE);
+#endif
+                    hecl::Printf(_S("  %3d%% ["), iFactor);
+                    for (int b=0 ; b<filled ; ++b)
+                        hecl::Printf(_S("#"));
+                    for (int b=0 ; b<rem ; ++b)
+                        hecl::Printf(_S("-"));
+                    hecl::Printf(_S("]"));
+#if _WIN32
+                    SetConsoleTextAttribute(m_termInfo.console, FOREGROUND_WHITE);
+#endif
+                }
+
+                hecl::Printf(_S("\n"));
+                ++m_curProgLines;
+            }
+        }
+        else
+        {
+            const ThreadStat& stat = m_threadStats[m_latestThread];
+            stat.print(m_termInfo);
+            hecl::Printf(_S("\r"));
+        }
+        m_dirty = false;
+    }
+    else if (m_mainIndeterminate)
+    {
+        m_termInfo.width = (hecl::GuiMode ? 120 : std::max(80, hecl::ConsoleWidth()));
+        MoveCursorUp(m_curProgLines);
+        m_curProgLines = 0;
+        DrawIndeterminateBar();
+        hecl::Printf(_S("\n"));
+        ++m_curProgLines;
+    }
+
+    if (m_termInfo.xtermColor)
+        hecl::Printf(_S("" SHOW_CURSOR ""));
+    fflush(stdout);
+
+#if _WIN32
+    cursorInfo.bVisible = TRUE;
+    SetConsoleCursorInfo(m_termInfo.console, &cursorInfo);
+#endif
+}
+
+void MultiProgressPrinter::LogProc()
+{
+    while (m_running)
+    {
+        std::this_thread::sleep_for(std::chrono::milliseconds(100));
+        if (!m_dirty && !m_mainIndeterminate)
+            continue;
+        std::lock_guard<std::mutex> lk(m_logLock);
+        DoPrint();
+    }
+}
+
+MultiProgressPrinter::MultiProgressPrinter(bool activate)
+{
+    if (activate)
+    {
+        /* Xterm check */
+#if _WIN32
+        m_newLineAfter = true;
+        m_termInfo.console = GetStdHandle(STD_OUTPUT_HANDLE);
+        const char* conemuANSI = getenv("ConEmuANSI");
+        if (conemuANSI && !strcmp(conemuANSI, "ON"))
+            m_termInfo.xtermColor = true;
+#else
+        m_newLineAfter = false;
+        const char* term = getenv("TERM");
+        if (term && !strncmp(term, "xterm", 5))
+        {
+            m_termInfo.xtermColor = true;
+            m_newLineAfter = true;
+        }
+#endif
+
+        m_running = true;
+        m_logThread = std::thread(std::bind(&MultiProgressPrinter::LogProc, this));
+    }
+}
+
+MultiProgressPrinter::~MultiProgressPrinter()
+{
+    m_running = false;
+    if (m_logThread.joinable())
+        m_logThread.join();
+}
+
+void MultiProgressPrinter::print(const hecl::SystemChar* message,
+                                 const hecl::SystemChar* submessage,
+                                 float factor, int threadIdx) const
+{
+    if (!m_running)
+        return;
+    std::lock_guard<std::mutex> lk(m_logLock);
+    if (threadIdx < 0)
+        threadIdx = 0;
+    if (threadIdx >= m_threadStats.size())
+        m_threadStats.resize(threadIdx + 1);
+    ThreadStat& stat = m_threadStats[threadIdx];
+    if (message)
+        stat.m_message = message;
+    else
+        stat.m_message.clear();
+    if (submessage)
+        stat.m_submessage = submessage;
+    else
+        stat.m_submessage.clear();
+    stat.m_factor = factor;
+    stat.m_active = true;
+    m_latestThread = threadIdx;
+    m_dirty = true;
+}
+
+void MultiProgressPrinter::setMainFactor(float factor) const
+{
+    if (!m_running)
+        return;
+    std::lock_guard<std::mutex> lk(m_logLock);
+    if (!m_mainIndeterminate)
+        m_dirty = true;
+    m_mainFactor = factor;
+}
+
+void MultiProgressPrinter::setMainIndeterminate(bool indeterminate) const
+{
+    if (!m_running)
+        return;
+    std::lock_guard<std::mutex> lk(m_logLock);
+    if (m_mainIndeterminate != indeterminate)
+    {
+        m_mainIndeterminate = indeterminate;
+        m_dirty = true;
+    }
+}
+
+void MultiProgressPrinter::startNewLine() const
+{
+    if (!m_running)
+        return;
+    std::lock_guard<std::mutex> lk(m_logLock);
+    const_cast<MultiProgressPrinter&>(*this).DoPrint();
+    m_threadStats.clear();
+    m_curThreadLines = 0;
+    m_mainFactor = -1.f;
+    auto logLk = logvisor::LockLog();
+    hecl::Printf(_S("\n"));
+}
+
+void MultiProgressPrinter::flush() const
+{
+    std::lock_guard<std::mutex> lk(m_logLock);
+    const_cast<MultiProgressPrinter&>(*this).DoPrint();
+}
+
+}

hecl/lib/Project.cpp

@@ -12,6 +12,7 @@
 #include "hecl/Database.hpp"
 #include "hecl/Blender/Connection.hpp"
 #include "hecl/ClientProcess.hpp"
+#include "hecl/MultiProgressPrinter.hpp"
 
 namespace hecl::Database
 {
@@ -353,14 +354,13 @@ bool Project::disableDataSpecs(const std::vector<SystemString>& specs)
 
 class CookProgress
 {
-    FProgress& m_progFunc;
+    const hecl::MultiProgressPrinter& m_progPrinter;
     const SystemChar* m_dir = nullptr;
     const SystemChar* m_file = nullptr;
-    int lidx = 0;
+    float m_prog = 0.f;
-    float m_prog = 0.0;
 public:
-    CookProgress(FProgress& progFunc) : m_progFunc(progFunc) {}
+    CookProgress(const hecl::MultiProgressPrinter& progPrinter) : m_progPrinter(progPrinter) {}
-    void changeDir(const SystemChar* dir) {m_dir = dir; ++lidx;}
+    void changeDir(const SystemChar* dir) {m_dir = dir; m_progPrinter.startNewLine();}
     void changeFile(const SystemChar* file, float prog) {m_file = file; m_prog = prog;}
     void reportFile(const DataSpecEntry* specEnt)
     {
@@ -368,8 +368,7 @@ public:
         submsg += _S(" (");
         submsg += specEnt->m_name.data();
         submsg += _S(')');
-        if (m_progFunc)
+        m_progPrinter.print(m_dir, submsg.c_str(), m_prog);
-            m_progFunc(m_dir, submsg.c_str(), lidx, m_prog);
     }
     void reportFile(const DataSpecEntry* specEnt, const SystemChar* extra)
     {
@@ -379,13 +378,11 @@ public:
         submsg += _S(", ");
         submsg += extra;
         submsg += _S(')');
-        if (m_progFunc)
+        m_progPrinter.print(m_dir, submsg.c_str(), m_prog);
-            m_progFunc(m_dir, submsg.c_str(), lidx, m_prog);
     }
     void reportDirComplete()
     {
-        if (m_progFunc)
+        m_progPrinter.print(m_dir, nullptr, 1.f);
-            m_progFunc(m_dir, nullptr, lidx, 1.0);
     }
 };
 
@@ -399,7 +396,7 @@ static void VisitFile(const ProjectPath& path, bool force, bool fast,
         {
             if (cp)
             {
-                cp->addCookTransaction(path, spec.get());
+                cp->addCookTransaction(path, force, fast, spec.get());
             }
             else
             {
@@ -474,11 +471,21 @@ static void VisitDirectory(const ProjectPath& dir,
     }
 }
 
-bool Project::cookPath(const ProjectPath& path, FProgress progress,
+bool Project::cookPath(const ProjectPath& path, const hecl::MultiProgressPrinter& progress,
-                       bool recursive, bool force, bool fast, ClientProcess* cp)
+                       bool recursive, bool force, bool fast, const DataSpecEntry* spec,
+                       ClientProcess* cp)
 {
     /* Construct DataSpec instances for cooking */
-    if (m_cookSpecs.empty())
+    if (spec)
+    {
+        if (m_cookSpecs.size() != 1 || m_cookSpecs[0]->getDataSpecEntry() != spec)
+        {
+            m_cookSpecs.clear();
+            if (spec->m_factory)
+                m_cookSpecs.push_back(std::unique_ptr<IDataSpec>(spec->m_factory(*this, DataSpecTool::Cook)));
+        }
+    }
+    else if (m_cookSpecs.empty())
     {
         m_cookSpecs.reserve(m_compiledSpecs.size());
         for (const ProjectDataSpec& spec : m_compiledSpecs)
@@ -493,7 +500,7 @@ bool Project::cookPath(const ProjectPath& path, FProgress progress,
     case ProjectPath::Type::File:
     case ProjectPath::Type::Glob:
     {
-        cookProg.changeFile(path.getLastComponent().data(), 0.0);
+        cookProg.changeFile(path.getLastComponent().data(), 0.f);
         VisitFile(path, force, fast, m_cookSpecs, cookProg, cp);
         break;
     }
@@ -508,23 +515,32 @@ bool Project::cookPath(const ProjectPath& path, FProgress progress,
     return true;
 }
 
-bool Project::packagePath(const ProjectPath& path, FProgress progress, bool fast, ClientProcess* cp)
+bool Project::packagePath(const ProjectPath& path, const hecl::MultiProgressPrinter& progress,
+                          bool fast, const DataSpecEntry* spec, ClientProcess* cp)
 {
     /* Construct DataSpec instance for packaging */
     const DataSpecEntry* specEntry = nullptr;
-    bool foundPC = false;
+    if (spec)
-    for (const ProjectDataSpec& spec : m_compiledSpecs)
     {
-        if (spec.active && spec.spec.m_factory)
+        if (spec->m_factory)
+            specEntry = spec;
+    }
+    else
+    {
+        bool foundPC = false;
+        for (const ProjectDataSpec& spec : m_compiledSpecs)
         {
-            if (hecl::StringUtils::EndsWith(spec.spec.m_name, _S("-PC")))
+            if (spec.active && spec.spec.m_factory)
             {
-                foundPC = true;
+                if (hecl::StringUtils::EndsWith(spec.spec.m_name, _S("-PC")))
-                specEntry = &spec.spec;
+                {
-            }
+                    foundPC = true;
-            else if (!foundPC)
+                    specEntry = &spec.spec;
-            {
+                }
-                specEntry = &spec.spec;
+                else if (!foundPC)
+                {
+                    specEntry = &spec.spec;
+                }
             }
         }
     }