AxioDL
/
PrimeWorldEditor
mirror of https://github.com/AxioDL/PrimeWorldEditor.git
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add multithreading to property name generator

This commit is contained in:
Luke Street 2021-08-06 18:52:21 -04:00
parent ceecab0151
commit 023aef478b
6 changed files with 135 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/Core/CMakeLists.txt
View File

@ -9,6 +9,7 @@ find_package(lzokay CONFIG REQUIRED)
find_package(OpenGL REQUIRED)
find_package(assimp CONFIG REQUIRED)
find_package(ZLIB REQUIRED)
find_package(Threads REQUIRED)
# AssImp's cmake config is pretty awful. It doesn't include necesary libraries. Hopefully this can be fixed later.
find_library(IIRXML_LIBRARY NAMES IrrXMLd IrrXML)
@ -39,6 +40,7 @@ target_link_libraries(
lzokay::lzokay
OpenGL::GL
assimp::assimp
Threads::Threads
${IIRXML_LIBRARY}
${ZLIB_LIBRARY}
)

12
src/Core/IProgressNotifier.h
View File

@ -28,19 +28,19 @@ public:
mTaskCount = Math::Max(mTaskCount, TaskIndex + 1);
}
void Report(int StepIndex, int StepCount, const TString& rkStepDesc = "")
void Report(uint64 StepIndex, uint64 StepCount, const TString& rkStepDesc = "")
{
ASSERT(mTaskCount >= 1);
// Make sure TaskCount and StepCount are at least 1 so we don't have divide-by-zero errors
int TaskCount = Math::Max(mTaskCount, 1);
StepCount = Math::Max(StepCount, 1);
StepCount = Math::Max<uint64>(StepCount, 1);
// Calculate percentage
float TaskPercent = 1.f / (float) TaskCount;
float StepPercent = (StepCount >= 0 ? (float) StepIndex / (float) StepCount : 0.f);
float ProgressPercent = (TaskPercent * mTaskIndex) + (TaskPercent * StepPercent);
UpdateProgress(mTaskName, rkStepDesc, ProgressPercent);
double TaskPercent = 1.f / (double) TaskCount;
double StepPercent = (StepCount >= 0 ? (double) StepIndex / (double) StepCount : 0.f);
double ProgressPercent = (TaskPercent * mTaskIndex) + (TaskPercent * StepPercent);
UpdateProgress(mTaskName, rkStepDesc, (float) ProgressPercent);
}
void Report(const TString& rkStepDesc)

99
src/Core/Resource/Script/Property/CPropertyNameGenerator.cpp
View File

@ -5,16 +5,20 @@
#include <Common/Hash/CCRC32.h>
#include <memory>
#include <thread>
/** Default constructor */
CPropertyNameGenerator::CPropertyNameGenerator() = default;
void CPropertyNameGenerator::Warmup()
{
// Clear output from previous runs
ASSERT(!mWordListLoadStarted || mWordListLoadFinished);
std::unique_lock lock{mWarmupMutex};
if (mWordListLoadFinished)
{
return;
}
mWordListLoadFinished = false;
mWordListLoadStarted = true;
mWords.clear();
// Load the word list from the file
@ -28,10 +32,7 @@ void CPropertyNameGenerator::Warmup()
std::fgets(WordBuffer, sizeof(WordBuffer), pListFile.get());
WordBuffer[0] = TString::CharToUpper(WordBuffer[0]);
SWord Word;
Word.Word = TString(WordBuffer).Trimmed();
Word.Usages = 0;
mWords.push_back(std::move(Word));
mWords.emplace_back(TString(WordBuffer).Trimmed());
}
mWordListLoadFinished = true;
@ -45,7 +46,6 @@ void CPropertyNameGenerator::Generate(const SPropertyNameGenerationParameters& r
mGeneratedNames.clear();
mValidTypePairMap.clear();
mIsRunning = true;
mFinishedRunning = false;
// Convert the type pair map.
// Also, replace the normal type name list with whatever is in the ID pairs list we were given.
@ -71,30 +71,56 @@ void CPropertyNameGenerator::Generate(const SPropertyNameGenerationParameters& r
}
// If we haven't loaded the word list yet, load it.
// If we are still loading the word list, wait until we're finished.
if (!mWordListLoadFinished)
{
if (mWordListLoadStarted)
while (!mWordListLoadFinished) {}
else
Warmup();
}
// Calculate the number of steps involved in this task.
const int kNumWords = mWords.size();
const int kMaxWords = rkParams.MaxWords;
int TestsDone = 0;
int TotalTests = 1;
TotalTests = 1;
for (int i = 0; i < kMaxWords; i++)
TotalTests *= kNumWords;
pProgress->SetOneShotTask("Generating property names");
pProgress->Report(TestsDone, TotalTests);
pProgress->Report(0, TotalTests);
const uint WordsPerThread = kNumWords / rkParams.ConcurrentTasks;
std::vector<std::thread> Threads;
for (int i = 0; i < rkParams.ConcurrentTasks; ++i)
{
SPropertyNameGenerationTaskParameters Params{};
Params.TaskIndex = i;
Params.StartWord = WordsPerThread * i;
if (i == rkParams.ConcurrentTasks - 1)
{
// Ensure last task takes any remaining words
Params.EndWord = kNumWords - 1;
}
else
{
Params.EndWord = Params.StartWord + WordsPerThread;
}
Threads.emplace_back(&CPropertyNameGenerator::GenerateTask, this, rkParams, Params, pProgress);
}
for (auto& Thread : Threads)
{
Thread.join();
}
mIsRunning = false;
}
void CPropertyNameGenerator::GenerateTask(const SPropertyNameGenerationParameters& rkParams,
SPropertyNameGenerationTaskParameters taskParams,
IProgressNotifier* pProgress)
{
const int kNumWords = mWords.size();
const int kMaxWords = rkParams.MaxWords;
// Configure params needed to run the name generation!
bool WriteToLog = rkParams.PrintToLog;
bool SaveResults = true;
uint64 TestsDone = 0;
// The prefix only needs to be hashed this one time
CCRC32 PrefixHash;
@ -105,12 +131,12 @@ void CPropertyNameGenerator::Generate(const SPropertyNameGenerationParameters& r
// the same hashes over and over. Init the stack with the first word.
struct SWordCache
{
int WordIndex;
uint WordIndex;
CCRC32 Hash;
};
std::vector<SWordCache> WordCache;
SWordCache FirstWord { -1, CCRC32() };
SWordCache FirstWord { taskParams.StartWord - 1, CCRC32() };
WordCache.push_back(FirstWord);
while ( true )
@ -119,19 +145,22 @@ void CPropertyNameGenerator::Generate(const SPropertyNameGenerationParameters& r
int RecalcIndex = WordCache.size() - 1;
WordCache.back().WordIndex++;
while (WordCache[RecalcIndex].WordIndex >= kNumWords)
while (WordCache[RecalcIndex].WordIndex >= kNumWords ||
(RecalcIndex == 0 && WordCache[0].WordIndex >= taskParams.EndWord))
{
WordCache[RecalcIndex].WordIndex = 0;
if (RecalcIndex == 0)
{
WordCache[0].WordIndex = taskParams.StartWord;
if (RecalcIndex > 0)
{
RecalcIndex--;
WordCache[RecalcIndex].WordIndex++;
SWordCache NewWord { 0, CCRC32() };
WordCache.push_back(NewWord);
}
else
{
SWordCache NewWord { 0, CCRC32() };
WordCache.push_back(NewWord);
WordCache[RecalcIndex].WordIndex = 0;
RecalcIndex--;
WordCache[RecalcIndex].WordIndex++;
}
}
@ -149,7 +178,7 @@ void CPropertyNameGenerator::Generate(const SPropertyNameGenerationParameters& r
// For camelcase, hash the first letter of the first word as lowercase
if (RecalcIndex == 0 && rkParams.Casing == ENameCasing::camelCase)
{
const char* pkWord = *mWords[Index].Word;
const char* pkWord = *mWords[Index];
LastValidHash.Hash( TString::CharToLower( pkWord[0] ) );
LastValidHash.Hash( &pkWord[1] );
}
@ -159,7 +188,7 @@ void CPropertyNameGenerator::Generate(const SPropertyNameGenerationParameters& r
if (RecalcIndex > 0 && rkParams.Casing == ENameCasing::Snake_Case)
LastValidHash.Hash("_");
LastValidHash.Hash( *mWords[Index].Word );
LastValidHash.Hash( *mWords[Index] );
}
WordCache[RecalcIndex].Hash = LastValidHash;
@ -179,6 +208,8 @@ void CPropertyNameGenerator::Generate(const SPropertyNameGenerationParameters& r
// Check if this hash is a property ID
if (IsValidPropertyID(PropertyID, pkTypeName, rkParams))
{
std::unique_lock lock{mPropertyCheckMutex};
SGeneratedPropertyName PropertyName;
NPropertyMap::RetrieveXMLsWithProperty(PropertyID, pkTypeName, PropertyName.XmlList);
@ -194,7 +225,7 @@ void CPropertyNameGenerator::Generate(const SPropertyNameGenerationParameters& r
PropertyName.Name += "_";
}
PropertyName.Name += mWords[Index].Word;
PropertyName.Name += mWords[Index];
}
if (rkParams.Casing == ENameCasing::camelCase)
@ -244,12 +275,10 @@ void CPropertyNameGenerator::Generate(const SPropertyNameGenerationParameters& r
if (pProgress->ShouldCancel())
break;
pProgress->Report(TestsDone, TotalTests);
auto Value = TotalTestsDone += 250;
pProgress->Report(Value, TotalTests);
}
}
mIsRunning = false;
mFinishedRunning = true;
}
/** Returns whether a given property ID is valid */

50
src/Core/Resource/Script/Property/CPropertyNameGenerator.h
View File

@ -4,6 +4,9 @@
#include "Core/IProgressNotifier.h"
#include <Common/Common.h>
#include <atomic>
#include <mutex>
/** Name casing parameter */
enum class ENameCasing
{
@ -22,6 +25,9 @@ struct SPropertyIdTypePair
/** Parameters for using the name generator */
struct SPropertyNameGenerationParameters
{
/** Number of concurrent tasks to run */
int ConcurrentTasks;
/** Maximum number of words per name; name generation will complete when all possibilities have been checked */
int MaxWords;
@ -50,6 +56,18 @@ struct SPropertyNameGenerationParameters
bool PrintToLog;
};
struct SPropertyNameGenerationTaskParameters
{
/** Task index */
uint TaskIndex;
/** Base word start index */
uint StartWord;
/** Base word end index */
uint EndWord;
};
/** A generated property name */
struct SGeneratedPropertyName
{
@ -62,18 +80,12 @@ struct SGeneratedPropertyName
/** Generates property names and validates them against know property IDs. */
class CPropertyNameGenerator
{
/** Whether we have started loading the word list */
bool mWordListLoadStarted = false;
/** Whether the word list has been fully loaded */
bool mWordListLoadFinished = false;
std::atomic<bool> mWordListLoadFinished = false;
/** Whether the generation process is running */
bool mIsRunning = false;
/** Whether the generation process finished running */
bool mFinishedRunning = false;
/** List of valid property types to check against */
std::vector<TString> mTypeNames;
@ -81,18 +93,26 @@ class CPropertyNameGenerator
std::unordered_map<uint32, const char*> mValidTypePairMap;
/** List of words */
struct SWord
{
TString Word;
int Usages;
};
std::vector<SWord> mWords;
std::vector<TString> mWords;
/** List of output generated property names */
std::list<SGeneratedPropertyName> mGeneratedNames;
/** List of word indices */
std::vector<int> mWordIndices;
/** Warmup() mutex */
std::mutex mWarmupMutex;
/** Property check mutex */
std::mutex mPropertyCheckMutex;
/** Total number of tests to perform */
uint64 TotalTests;
/** Current number of tests performed */
std::atomic<uint64> TotalTestsDone{0};
void GenerateTask(const SPropertyNameGenerationParameters& rkParams,
SPropertyNameGenerationTaskParameters taskParams,
IProgressNotifier* pProgressNotifier);
public:
/** Default constructor */

4
src/Editor/CGeneratePropertyNamesDialog.cpp
View File

@ -34,6 +34,9 @@ CGeneratePropertyNamesDialog::CGeneratePropertyNamesDialog(QWidget* pParent)
int TreeWidth = mpUI->OutputTreeWidget->width();
mpUI->OutputTreeWidget->setColumnWidth(0, TreeWidth * 1.5);
mpUI->OutputTreeWidget->setHeaderHidden(false);
// Don't sort by default
mpUI->OutputTreeWidget->header()->setSortIndicator(-1, Qt::AscendingOrder);
mpUI->OutputTreeWidget->setSortingEnabled(true);
// Allow the generator to initialize in the background while the user is getting set up
QtConcurrent::run(&mGenerator, &CPropertyNameGenerator::Warmup);
@ -152,6 +155,7 @@ void CGeneratePropertyNamesDialog::StartGeneration()
}
Params.MaxWords = mpUI->NumWordsSpinBox->value();
Params.ConcurrentTasks = mpUI->ThreadsSpinBox->value();
Params.Prefix = TO_TSTRING(mpUI->PrefixLineEdit->text());
Params.Suffix = TO_TSTRING(mpUI->SuffixLineEdit->text());
Params.Casing = mpUI->CasingComboBox->currentEnum();

29
src/Editor/CGeneratePropertyNamesDialog.ui
View File

@ -7,7 +7,7 @@
<x>0</x>
<y>0</y>
<width>693</width>
<height>604</height>
<height>673</height>
</rect>
</property>
<property name="windowTitle">
@ -47,35 +47,52 @@
</widget>
</item>
<item row="1" column="0">
<widget class="QLabel" name="ThreadsLabel">
<property name="text">
<string>Threads:</string>
</property>
</widget>
</item>
<item row="2" column="0">
<widget class="QLabel" name="PrefixLabel">
<property name="text">
<string>Prefix:</string>
</property>
</widget>
</item>
<item row="1" column="1">
<item row="2" column="1">
<widget class="QLineEdit" name="PrefixLineEdit"/>
</item>
<item row="2" column="0">
<item row="3" column="0">
<widget class="QLabel" name="SuffixLabel">
<property name="text">
<string>Suffix:</string>
</property>
</widget>
</item>
<item row="2" column="1">
<item row="3" column="1">
<widget class="QLineEdit" name="SuffixLineEdit"/>
</item>
<item row="3" column="0">
<item row="4" column="0">
<widget class="QLabel" name="label">
<property name="text">
<string>Casing:</string>
</property>
</widget>
</item>
<item row="3" column="1">
<item row="4" column="1">
<widget class="CNameCasingComboBox" name="CasingComboBox"/>
</item>
<item row="1" column="1">
<widget class="QSpinBox" name="ThreadsSpinBox">
<property name="minimum">
<number>1</number>
</property>
<property name="maximum">
<number>256</number>
</property>
</widget>
</item>
</layout>
</item>
<item>