Add multithreading to property name generator

src/Core/CMakeLists.txt

@@ -9,6 +9,7 @@ find_package(lzokay CONFIG REQUIRED)
 find_package(OpenGL REQUIRED)
 find_package(assimp CONFIG REQUIRED)
 find_package(ZLIB REQUIRED)
+find_package(Threads REQUIRED)
 
 # AssImp's cmake config is pretty awful. It doesn't include necesary libraries. Hopefully this can be fixed later.
 find_library(IIRXML_LIBRARY NAMES IrrXMLd IrrXML)
@@ -39,6 +40,7 @@ target_link_libraries(
     lzokay::lzokay
     OpenGL::GL
     assimp::assimp
+    Threads::Threads
     ${IIRXML_LIBRARY}
     ${ZLIB_LIBRARY}
 )

src/Core/IProgressNotifier.h

@@ -28,19 +28,19 @@ public:
         mTaskCount = Math::Max(mTaskCount, TaskIndex + 1);
     }
 
-    void Report(int StepIndex, int StepCount, const TString& rkStepDesc = "")
+    void Report(uint64 StepIndex, uint64 StepCount, const TString& rkStepDesc = "")
     {
         ASSERT(mTaskCount >= 1);
 
         // Make sure TaskCount and StepCount are at least 1 so we don't have divide-by-zero errors
         int TaskCount = Math::Max(mTaskCount, 1);
-        StepCount = Math::Max(StepCount, 1);
+        StepCount = Math::Max<uint64>(StepCount, 1);
 
         // Calculate percentage
-        float TaskPercent = 1.f / (float) TaskCount;
+        double TaskPercent = 1.f / (double) TaskCount;
-        float StepPercent = (StepCount >= 0 ? (float) StepIndex / (float) StepCount : 0.f);
+        double StepPercent = (StepCount >= 0 ? (double) StepIndex / (double) StepCount : 0.f);
-        float ProgressPercent = (TaskPercent * mTaskIndex) + (TaskPercent * StepPercent);
+        double ProgressPercent = (TaskPercent * mTaskIndex) + (TaskPercent * StepPercent);
-        UpdateProgress(mTaskName, rkStepDesc, ProgressPercent);
+        UpdateProgress(mTaskName, rkStepDesc, (float) ProgressPercent);
     }
 
     void Report(const TString& rkStepDesc)

src/Core/Resource/Script/Property/CPropertyNameGenerator.cpp

@@ -5,16 +5,20 @@
 #include <Common/Hash/CCRC32.h>
 
 #include <memory>
+#include <thread>
 
 /** Default constructor */
 CPropertyNameGenerator::CPropertyNameGenerator() = default;
 
 void CPropertyNameGenerator::Warmup()
 {
-    // Clear output from previous runs
+    std::unique_lock lock{mWarmupMutex};
-    ASSERT(!mWordListLoadStarted || mWordListLoadFinished);
+
+    if (mWordListLoadFinished)
+    {
+        return;
+    }
     mWordListLoadFinished = false;
-    mWordListLoadStarted = true;
     mWords.clear();
 
     // Load the word list from the file
@@ -28,10 +32,7 @@ void CPropertyNameGenerator::Warmup()
         std::fgets(WordBuffer, sizeof(WordBuffer), pListFile.get());
         WordBuffer[0] = TString::CharToUpper(WordBuffer[0]);
 
-        SWord Word;
+        mWords.emplace_back(TString(WordBuffer).Trimmed());
-        Word.Word = TString(WordBuffer).Trimmed();
-        Word.Usages = 0;
-        mWords.push_back(std::move(Word));
     }
 
     mWordListLoadFinished = true;
@@ -45,7 +46,6 @@ void CPropertyNameGenerator::Generate(const SPropertyNameGenerationParameters& r
     mGeneratedNames.clear();
     mValidTypePairMap.clear();
     mIsRunning = true;
-    mFinishedRunning = false;
 
     // Convert the type pair map.
     // Also, replace the normal type name list with whatever is in the ID pairs list we were given.
@@ -71,30 +71,56 @@ void CPropertyNameGenerator::Generate(const SPropertyNameGenerationParameters& r
     }
 
     // If we haven't loaded the word list yet, load it.
-    // If we are still loading the word list, wait until we're finished.
-    if (!mWordListLoadFinished)
-    {
-        if (mWordListLoadStarted)
-            while (!mWordListLoadFinished) {}
-        else
     Warmup();
-    }
 
     // Calculate the number of steps involved in this task.
     const int kNumWords = mWords.size();
     const int kMaxWords = rkParams.MaxWords;
-    int TestsDone = 0;
+    TotalTests = 1;
-    int TotalTests = 1;
 
     for (int i = 0; i < kMaxWords; i++)
         TotalTests *= kNumWords;
 
     pProgress->SetOneShotTask("Generating property names");
-    pProgress->Report(TestsDone, TotalTests);
+    pProgress->Report(0, TotalTests);
+
+    const uint WordsPerThread = kNumWords / rkParams.ConcurrentTasks;
+    std::vector<std::thread> Threads;
+    for (int i = 0; i < rkParams.ConcurrentTasks; ++i)
+    {
+        SPropertyNameGenerationTaskParameters Params{};
+        Params.TaskIndex = i;
+        Params.StartWord = WordsPerThread * i;
+        if (i == rkParams.ConcurrentTasks - 1)
+        {
+            // Ensure last task takes any remaining words
+            Params.EndWord = kNumWords - 1;
+        }
+        else
+        {
+            Params.EndWord = Params.StartWord + WordsPerThread;
+        }
+        Threads.emplace_back(&CPropertyNameGenerator::GenerateTask, this, rkParams, Params, pProgress);
+    }
+    for (auto& Thread : Threads)
+    {
+        Thread.join();
+    }
+
+    mIsRunning = false;
+}
+
+void CPropertyNameGenerator::GenerateTask(const SPropertyNameGenerationParameters& rkParams,
+                                          SPropertyNameGenerationTaskParameters taskParams,
+                                          IProgressNotifier* pProgress)
+{
+    const int kNumWords = mWords.size();
+    const int kMaxWords = rkParams.MaxWords;
 
     // Configure params needed to run the name generation!
     bool WriteToLog = rkParams.PrintToLog;
     bool SaveResults = true;
+    uint64 TestsDone = 0;
 
     // The prefix only needs to be hashed this one time
     CCRC32 PrefixHash;
@@ -105,12 +131,12 @@ void CPropertyNameGenerator::Generate(const SPropertyNameGenerationParameters& r
     // the same hashes over and over. Init the stack with the first word.
     struct SWordCache
     {
-        int WordIndex;
+        uint WordIndex;
         CCRC32 Hash;
     };
     std::vector<SWordCache> WordCache;
 
-    SWordCache FirstWord { -1, CCRC32() };
+    SWordCache FirstWord { taskParams.StartWord - 1, CCRC32() };
     WordCache.push_back(FirstWord);
 
     while ( true )
@@ -119,19 +145,22 @@ void CPropertyNameGenerator::Generate(const SPropertyNameGenerationParameters& r
         int RecalcIndex = WordCache.size() - 1;
         WordCache.back().WordIndex++;
 
-        while (WordCache[RecalcIndex].WordIndex >= kNumWords)
+        while (WordCache[RecalcIndex].WordIndex >= kNumWords ||
+               (RecalcIndex == 0 && WordCache[0].WordIndex >= taskParams.EndWord))
         {
-            WordCache[RecalcIndex].WordIndex = 0;
+            if (RecalcIndex == 0)
+            {
+                WordCache[0].WordIndex = taskParams.StartWord;
 
-            if (RecalcIndex > 0)
+                SWordCache NewWord { 0, CCRC32() };
-            {
+                WordCache.push_back(NewWord);
-                RecalcIndex--;
-                WordCache[RecalcIndex].WordIndex++;
             }
             else
             {
-                SWordCache NewWord { 0, CCRC32() };
+                WordCache[RecalcIndex].WordIndex = 0;
-                WordCache.push_back(NewWord);
+
+                RecalcIndex--;
+                WordCache[RecalcIndex].WordIndex++;
             }
         }
 
@@ -149,7 +178,7 @@ void CPropertyNameGenerator::Generate(const SPropertyNameGenerationParameters& r
             // For camelcase, hash the first letter of the first word as lowercase
             if (RecalcIndex == 0 && rkParams.Casing == ENameCasing::camelCase)
             {
-                const char* pkWord = *mWords[Index].Word;
+                const char* pkWord = *mWords[Index];
                 LastValidHash.Hash( TString::CharToLower( pkWord[0] ) );
                 LastValidHash.Hash( &pkWord[1] );
             }
@@ -159,7 +188,7 @@ void CPropertyNameGenerator::Generate(const SPropertyNameGenerationParameters& r
                 if (RecalcIndex > 0 && rkParams.Casing == ENameCasing::Snake_Case)
                     LastValidHash.Hash("_");
 
-                LastValidHash.Hash( *mWords[Index].Word );
+                LastValidHash.Hash( *mWords[Index] );
             }
 
             WordCache[RecalcIndex].Hash = LastValidHash;
@@ -179,6 +208,8 @@ void CPropertyNameGenerator::Generate(const SPropertyNameGenerationParameters& r
             // Check if this hash is a property ID
             if (IsValidPropertyID(PropertyID, pkTypeName, rkParams))
             {
+                std::unique_lock lock{mPropertyCheckMutex};
+
                 SGeneratedPropertyName PropertyName;
                 NPropertyMap::RetrieveXMLsWithProperty(PropertyID, pkTypeName, PropertyName.XmlList);
 
@@ -194,7 +225,7 @@ void CPropertyNameGenerator::Generate(const SPropertyNameGenerationParameters& r
                         PropertyName.Name += "_";
                     }
 
-                    PropertyName.Name += mWords[Index].Word;
+                    PropertyName.Name += mWords[Index];
                 }
 
                 if (rkParams.Casing == ENameCasing::camelCase)
@@ -244,12 +275,10 @@ void CPropertyNameGenerator::Generate(const SPropertyNameGenerationParameters& r
             if (pProgress->ShouldCancel())
                 break;
 
-            pProgress->Report(TestsDone, TotalTests);
+            auto Value = TotalTestsDone += 250;
+            pProgress->Report(Value, TotalTests);
         }
     }
-
-    mIsRunning = false;
-    mFinishedRunning = true;
 }
 
 /** Returns whether a given property ID is valid */

src/Core/Resource/Script/Property/CPropertyNameGenerator.h

@@ -4,6 +4,9 @@
 #include "Core/IProgressNotifier.h"
 #include <Common/Common.h>
 
+#include <atomic>
+#include <mutex>
+
 /** Name casing parameter */
 enum class ENameCasing
 {
@@ -22,6 +25,9 @@ struct SPropertyIdTypePair
 /** Parameters for using the name generator */
 struct SPropertyNameGenerationParameters
 {
+    /** Number of concurrent tasks to run */
+    int ConcurrentTasks;
+
     /** Maximum number of words per name; name generation will complete when all possibilities have been checked */
     int MaxWords;
 
@@ -50,6 +56,18 @@ struct SPropertyNameGenerationParameters
     bool PrintToLog;
 };
 
+struct SPropertyNameGenerationTaskParameters
+{
+    /** Task index */
+    uint TaskIndex;
+
+    /** Base word start index */
+    uint StartWord;
+
+    /** Base word end index */
+    uint EndWord;
+};
+
 /** A generated property name */
 struct SGeneratedPropertyName
 {
@@ -62,18 +80,12 @@ struct SGeneratedPropertyName
 /** Generates property names and validates them against know property IDs. */
 class CPropertyNameGenerator
 {
-    /** Whether we have started loading the word list */
-    bool mWordListLoadStarted = false;
-
     /** Whether the word list has been fully loaded */
-    bool mWordListLoadFinished = false;
+    std::atomic<bool> mWordListLoadFinished = false;
 
     /** Whether the generation process is running */
     bool mIsRunning = false;
 
-    /** Whether the generation process finished running */
-    bool mFinishedRunning = false;
-
     /** List of valid property types to check against */
     std::vector<TString> mTypeNames;
 
@@ -81,18 +93,26 @@ class CPropertyNameGenerator
     std::unordered_map<uint32, const char*> mValidTypePairMap;
 
     /** List of words */
-    struct SWord
+    std::vector<TString> mWords;
-    {
-        TString Word;
-        int Usages;
-    };
-    std::vector<SWord> mWords;
 
     /** List of output generated property names */
     std::list<SGeneratedPropertyName> mGeneratedNames;
 
-    /** List of word indices */
+    /** Warmup() mutex */
-    std::vector<int> mWordIndices;
+    std::mutex mWarmupMutex;
+
+    /** Property check mutex */
+    std::mutex mPropertyCheckMutex;
+
+    /** Total number of tests to perform */
+    uint64 TotalTests;
+
+    /** Current number of tests performed */
+    std::atomic<uint64> TotalTestsDone{0};
+
+    void GenerateTask(const SPropertyNameGenerationParameters& rkParams,
+                      SPropertyNameGenerationTaskParameters taskParams,
+                      IProgressNotifier* pProgressNotifier);
 
 public:
     /** Default constructor */

src/Editor/CGeneratePropertyNamesDialog.cpp

@@ -34,6 +34,9 @@ CGeneratePropertyNamesDialog::CGeneratePropertyNamesDialog(QWidget* pParent)
     int TreeWidth = mpUI->OutputTreeWidget->width();
     mpUI->OutputTreeWidget->setColumnWidth(0, TreeWidth * 1.5);
     mpUI->OutputTreeWidget->setHeaderHidden(false);
+    // Don't sort by default
+    mpUI->OutputTreeWidget->header()->setSortIndicator(-1, Qt::AscendingOrder);
+    mpUI->OutputTreeWidget->setSortingEnabled(true);
 
     // Allow the generator to initialize in the background while the user is getting set up
     QtConcurrent::run(&mGenerator, &CPropertyNameGenerator::Warmup);
@@ -152,6 +155,7 @@ void CGeneratePropertyNamesDialog::StartGeneration()
     }
 
     Params.MaxWords = mpUI->NumWordsSpinBox->value();
+    Params.ConcurrentTasks = mpUI->ThreadsSpinBox->value();
     Params.Prefix = TO_TSTRING(mpUI->PrefixLineEdit->text());
     Params.Suffix = TO_TSTRING(mpUI->SuffixLineEdit->text());
     Params.Casing = mpUI->CasingComboBox->currentEnum();

src/Editor/CGeneratePropertyNamesDialog.ui

@@ -7,7 +7,7 @@
     <x>0</x>
     <y>0</y>
     <width>693</width>
-    <height>604</height>
+    <height>673</height>
    </rect>
   </property>
   <property name="windowTitle">
@@ -47,35 +47,52 @@
              </widget>
             </item>
             <item row="1" column="0">
+             <widget class="QLabel" name="ThreadsLabel">
+              <property name="text">
+               <string>Threads:</string>
+              </property>
+             </widget>
+            </item>
+            <item row="2" column="0">
              <widget class="QLabel" name="PrefixLabel">
               <property name="text">
                <string>Prefix:</string>
               </property>
              </widget>
             </item>
-            <item row="1" column="1">
+            <item row="2" column="1">
              <widget class="QLineEdit" name="PrefixLineEdit"/>
             </item>
-            <item row="2" column="0">
+            <item row="3" column="0">
              <widget class="QLabel" name="SuffixLabel">
               <property name="text">
                <string>Suffix:</string>
               </property>
              </widget>
             </item>
-            <item row="2" column="1">
+            <item row="3" column="1">
              <widget class="QLineEdit" name="SuffixLineEdit"/>
             </item>
-            <item row="3" column="0">
+            <item row="4" column="0">
              <widget class="QLabel" name="label">
               <property name="text">
                <string>Casing:</string>
               </property>
              </widget>
             </item>
-            <item row="3" column="1">
+            <item row="4" column="1">
              <widget class="CNameCasingComboBox" name="CasingComboBox"/>
             </item>
+            <item row="1" column="1">
+             <widget class="QSpinBox" name="ThreadsSpinBox">
+              <property name="minimum">
+               <number>1</number>
+              </property>
+              <property name="maximum">
+               <number>256</number>
+              </property>
+             </widget>
+            </item>
            </layout>
           </item>
           <item>