String cooking support

This commit is contained in:
Aruki 2019-01-12 23:43:41 -08:00
parent a1d94cc58f
commit e9e1ccb8d6
12 changed files with 431 additions and 15 deletions

View File

@ -252,6 +252,7 @@ HEADERS += \
Tweaks/CTweakData.h \
Tweaks/CTweakLoader.h \
Tweaks/CTweakCooker.h \
Resource/Cooker/CStringCooker.h \
Resource/Scan/CScan.h \
Resource/Scan/SScanParametersMP1.h \
Resource/Scan/ELogbookCategory.h
@ -370,6 +371,7 @@ SOURCES += \
Tweaks/CTweakManager.cpp \
Tweaks/CTweakLoader.cpp \
Tweaks/CTweakCooker.cpp \
Resource/Cooker/CStringCooker.cpp \
Resource/Scan/CScan.cpp
# Codegen

View File

@ -4,6 +4,7 @@
#include "CAreaCooker.h"
#include "CModelCooker.h"
#include "CPoiToWorldCooker.h"
#include "CStringCooker.h"
#include "CWorldCooker.h"
#include "Core/Tweaks/CTweakCooker.h"
@ -25,6 +26,7 @@ public:
case EResourceType::Area: return CAreaCooker::CookMREA((CGameArea*) pRes, rOutput);
case EResourceType::Model: return CModelCooker::CookCMDL((CModel*) pRes, rOutput);
case EResourceType::StaticGeometryMap: return CPoiToWorldCooker::CookEGMC((CPoiToWorld*) pRes, rOutput);
case EResourceType::StringTable: return CStringCooker::CookSTRG((CStringTable*) pRes, rOutput);
case EResourceType::Tweaks: return CTweakCooker::CookCTWK((CTweakData*) pRes, rOutput);
case EResourceType::World: return CWorldCooker::CookMLVL((CWorld*) pRes, rOutput);

View File

@ -0,0 +1,328 @@
#include "CStringCooker.h"
#include <Common/NBasics.h>
#include <algorithm>
void CStringCooker::WritePrimeDemoSTRG(IOutputStream& STRG)
{
uint StartOffset = STRG.Tell();
uint NumStrings = mpStringTable->NumStrings();
// Start writing the file...
STRG.WriteLong(0); // Dummy file size
uint TableStart = STRG.Tell();
STRG.WriteLong(NumStrings);
// Dummy string offsets
for (uint StringIdx = 0; StringIdx < NumStrings; StringIdx++)
STRG.WriteLong(0);
// Write strings
std::vector<uint> StringOffsets(NumStrings);
for (uint StringIdx = 0; StringIdx < NumStrings; StringIdx++)
{
StringOffsets[StringIdx] = STRG.Tell() - TableStart;
STRG.Write16String( mpStringTable->GetString(ELanguage::English, StringIdx).ToUTF16() );
}
// Fill in offsets
uint FileSize = STRG.Tell() - StartOffset;
STRG.GoTo(StartOffset);
STRG.WriteLong(FileSize);
STRG.Skip(4);
for (uint StringIdx = 0; StringIdx < NumStrings; StringIdx++)
STRG.WriteLong( StringOffsets[StringIdx] );
}
void CStringCooker::WritePrimeSTRG(IOutputStream& STRG)
{
// Magic/Version
STRG.WriteLong( 0x87654321 );
STRG.WriteLong( mpStringTable->Game() >= EGame::EchoesDemo ? 1 : 0 );
STRG.WriteLong( mpStringTable->NumLanguages() );
STRG.WriteLong( mpStringTable->NumStrings() );
// Language info
uint LanguagesStart = STRG.Tell();
for (uint LanguageIdx = 0; LanguageIdx < mpStringTable->NumLanguages(); LanguageIdx++)
{
const CStringTable::SLanguageData& kLanguage = mpStringTable->mLanguages[LanguageIdx];
STRG.WriteLong( (uint) kLanguage.Language );
STRG.WriteLong( 0 ); // Dummy offset
if ( mpStringTable->Game() >= EGame::EchoesDemo )
{
STRG.WriteLong( 0 ); // Dummy size
}
}
// Name Table
if ( mpStringTable->Game() >= EGame::EchoesDemo )
{
WriteNameTable(STRG);
}
// Strings
uint StringDataStart = STRG.Tell();
std::vector<uint> LanguageOffsets( mpStringTable->NumLanguages() );
std::vector<uint> LanguageSizes( mpStringTable->NumLanguages() );
for (uint LanguageIdx = 0; LanguageIdx < mpStringTable->NumLanguages(); LanguageIdx++)
{
const CStringTable::SLanguageData& kLanguage = mpStringTable->mLanguages[LanguageIdx];
uint LanguageStart = STRG.Tell();
LanguageOffsets[LanguageIdx] = LanguageStart - StringDataStart;
if ( mpStringTable->Game() == EGame::Prime )
{
STRG.WriteLong( 0 ); // Dummy size
}
// Fill dummy string offsets
uint StringOffsetBase = STRG.Tell();
for (uint StringIdx = 0; StringIdx < mpStringTable->NumStrings(); StringIdx++)
{
STRG.WriteLong( 0 );
}
// Write strings
std::vector<uint> StringOffsets( mpStringTable->NumStrings() );
for (uint StringIdx = 0; StringIdx < mpStringTable->NumStrings(); StringIdx++)
{
StringOffsets[StringIdx] = STRG.Tell() - StringOffsetBase;
STRG.Write16String( kLanguage.Strings[StringIdx].String.ToUTF16() );
}
// Go back and fill in size/offsets
uint LanguageEnd = STRG.Tell();
LanguageSizes[LanguageIdx] = LanguageEnd - StringOffsetBase;
STRG.GoTo(LanguageStart);
if ( mpStringTable->Game() == EGame::Prime )
{
STRG.WriteLong( LanguageSizes[LanguageIdx] );
}
for (uint StringIdx = 0; StringIdx < mpStringTable->NumStrings(); StringIdx++)
{
STRG.WriteLong( StringOffsets[StringIdx] );
}
STRG.GoTo(LanguageEnd);
}
uint STRGEnd = STRG.Tell();
// Fill in missing language data
STRG.GoTo(LanguagesStart);
for (uint LanguageIdx = 0; LanguageIdx < mpStringTable->NumLanguages(); LanguageIdx++)
{
STRG.Skip(4); // Skip language ID
STRG.WriteLong( LanguageOffsets[LanguageIdx] );
if ( mpStringTable->Game() >= EGame::EchoesDemo )
{
STRG.WriteLong( LanguageSizes[LanguageIdx] );
}
}
STRG.GoTo(STRGEnd);
}
void CStringCooker::WriteCorruptionSTRG(IOutputStream& STRG)
{
// Magic/Version
STRG.WriteLong( 0x87654321 );
STRG.WriteLong( 3 );
STRG.WriteLong( mpStringTable->NumLanguages() );
STRG.WriteLong( mpStringTable->NumStrings() );
// Name Table
WriteNameTable(STRG);
// Create some structures before continuing...
// In MP3 and DKCR, if a string has not been localized, then the English text
// is reused, instead of duplicating the string data like MP1 and MP2 would have.
struct SCookedLanguageData
{
ELanguage Language;
std::vector<uint> StringOffsets;
uint TotalSize;
};
std::vector<SCookedLanguageData> CookedLanguageData( mpStringTable->NumLanguages() );
for (uint LanguageIdx = 0; LanguageIdx < mpStringTable->NumLanguages(); LanguageIdx++)
{
const CStringTable::SLanguageData& kLanguageData = mpStringTable->mLanguages[LanguageIdx];
SCookedLanguageData& CookedData = CookedLanguageData[LanguageIdx];
CookedData.Language = kLanguageData.Language;
CookedData.StringOffsets.resize( mpStringTable->NumStrings() );
CookedData.TotalSize = 0;
}
// Language IDs
for (uint LanguageIdx = 0; LanguageIdx < mpStringTable->NumLanguages(); LanguageIdx++)
{
STRG.WriteLong( (uint) CookedLanguageData[LanguageIdx].Language );
}
// Language Info
uint LanguageInfoStart = STRG.Tell();
for (uint LanguageIdx = 0; LanguageIdx < mpStringTable->NumLanguages(); LanguageIdx++)
{
// Fill language size/offsets with dummy data...
STRG.WriteLong( 0 );
for (uint StringIdx = 0; StringIdx < mpStringTable->NumStrings(); StringIdx++)
STRG.WriteLong( 0 );
}
// Some of the following code assumes that language 0 is English.
ASSERT( mpStringTable->mLanguages[0].Language == ELanguage::English );
// Strings
uint StringsStart = STRG.Tell();
for (uint StringIdx = 0; StringIdx < mpStringTable->NumStrings(); StringIdx++)
{
for (uint LanguageIdx = 0; LanguageIdx < mpStringTable->NumLanguages(); LanguageIdx++)
{
const CStringTable::SLanguageData& kLanguageData = mpStringTable->mLanguages[LanguageIdx];
const CStringTable::SStringData& kStringData = kLanguageData.Strings[StringIdx];
SCookedLanguageData& CookedData = CookedLanguageData[LanguageIdx];
// If the "localized" flag is disabled, then we will not write this string. Instead, it will
// reuse the offset for the English text.
if (LanguageIdx == 0 || kStringData.IsLocalized)
{
CookedData.StringOffsets[StringIdx] = STRG.Tell() - StringsStart;
CookedData.TotalSize += kStringData.String.Size() + 1; // +1 for terminating zero
STRG.WriteLong( kStringData.String.Size() + 1 );
STRG.WriteString( kStringData.String );
}
else
{
CookedData.StringOffsets[StringIdx] = CookedLanguageData[0].StringOffsets[StringIdx];
CookedData.TotalSize += mpStringTable->mLanguages[0].Strings[StringIdx].String.Size() + 1; // +1 for terminating zero
}
}
}
uint STRGEnd = STRG.Tell();
// Fill in missing language data
STRG.GoTo(LanguageInfoStart);
for (uint LanguageIdx = 0; LanguageIdx < mpStringTable->NumLanguages(); LanguageIdx++)
{
const SCookedLanguageData& kCookedData = CookedLanguageData[LanguageIdx];
STRG.WriteLong( kCookedData.TotalSize );
for (uint StringIdx = 0; StringIdx < mpStringTable->NumStrings(); StringIdx++)
STRG.WriteLong( kCookedData.StringOffsets[StringIdx] );
}
STRG.GoTo(STRGEnd);
}
void CStringCooker::WriteNameTable(IOutputStream& STRG)
{
// Build a list of name entries to put in the map
struct SNameEntry
{
uint Offset;
uint Index;
TString Name;
};
std::vector<SNameEntry> NameEntries;
for (uint StringIdx = 0; StringIdx < mpStringTable->NumStrings(); StringIdx++)
{
SNameEntry Entry;
Entry.Offset = 0;
Entry.Index = StringIdx;
Entry.Name = mpStringTable->StringNameByIndex(StringIdx);
if (!Entry.Name.IsEmpty())
{
NameEntries.push_back(Entry);
}
}
std::stable_sort( NameEntries.begin(), NameEntries.end(), [](const SNameEntry& kLHS, const SNameEntry& kRHS) -> bool {
return kLHS.Name < kRHS.Name;
});
// Write out name entries
uint NameTableStart = STRG.Tell();
STRG.WriteLong( NameEntries.size() );
STRG.WriteLong( 0 ); // Dummy name table size
uint NameTableOffsetsStart = STRG.Tell();
for (uint NameIdx = 0; NameIdx < NameEntries.size(); NameIdx++)
{
STRG.WriteLong( 0 ); // Dummy name offset
STRG.WriteLong( NameEntries[NameIdx].Index );
}
// Write out names
std::vector<uint> NameOffsets( NameEntries.size() );
for (uint NameIdx = 0; NameIdx < NameEntries.size(); NameIdx++)
{
NameOffsets[NameIdx] = STRG.Tell() - NameTableOffsetsStart;
STRG.WriteString( NameEntries[NameIdx].Name );
}
// Fill out sizes and offsets
uint NameTableEnd = STRG.Tell();
uint NameTableSize = NameTableEnd - NameTableOffsetsStart;
STRG.GoTo(NameTableStart);
STRG.Skip(4);
STRG.WriteLong(NameTableSize);
for (uint NameIdx = 0; NameIdx < NameEntries.size(); NameIdx++)
{
STRG.WriteLong( NameOffsets[NameIdx] );
STRG.Skip(4);
}
STRG.GoTo(NameTableEnd);
}
/** Static entry point */
bool CStringCooker::CookSTRG(CStringTable* pStringTable, IOutputStream& STRG)
{
CStringCooker Cooker(pStringTable);
switch (pStringTable->Game())
{
case EGame::PrimeDemo:
Cooker.WritePrimeDemoSTRG(STRG);
return true;
case EGame::Prime:
case EGame::EchoesDemo:
case EGame::Echoes:
case EGame::CorruptionProto:
Cooker.WritePrimeSTRG(STRG);
return true;
case EGame::Corruption:
case EGame::DKCReturns:
Cooker.WriteCorruptionSTRG(STRG);
return true;
default:
return false;
}
}

View File

@ -0,0 +1,25 @@
#ifndef CSTRINGCOOKER_H
#define CSTRINGCOOKER_H
#include "Core/Resource/TResPtr.h"
#include "Core/Resource/StringTable/CStringTable.h"
/** Cooker class for writing game-compatible STRG assets */
class CStringCooker
{
TResPtr<CStringTable> mpStringTable;
CStringCooker(CStringTable* pStringTable)
: mpStringTable(pStringTable) {}
public:
void WritePrimeDemoSTRG(IOutputStream& STRG);
void WritePrimeSTRG(IOutputStream& STRG);
void WriteCorruptionSTRG(IOutputStream& STRG);
void WriteNameTable(IOutputStream& STRG);
/** Static entry point */
static bool CookSTRG(CStringTable* pStringTable, IOutputStream& STRG);
};
#endif // CSTRINGCOOKER_H

View File

@ -1,7 +1,7 @@
#ifndef CSCANLOADER_H
#define CSCANLOADER_H
#include "Core/Resource/CScan.h"
#include "Core/Resource/Scan/CScan.h"
#include <Common/EGame.h>
class CScanLoader

View File

@ -24,7 +24,7 @@ void CStringLoader::LoadPrimeDemoSTRG(IInputStream& STRG)
for (uint StringIdx = 0; StringIdx < NumStrings; StringIdx++)
{
STRG.GoTo( TableStart + StringOffsets[StringIdx] );
Language.Strings[StringIdx] = STRG.Read16String().ToUTF8();
Language.Strings[StringIdx].String = STRG.Read16String().ToUTF8();
}
}
@ -51,6 +51,9 @@ void CStringLoader::LoadPrimeSTRG(IInputStream& STRG)
}
}
// Some of the following code assumes that language 0 is English
ASSERT( mpStringTable->mLanguages[0].Language == ELanguage::English );
// String names
if (mVersion >= EGame::EchoesDemo)
{
@ -85,7 +88,17 @@ void CStringLoader::LoadPrimeSTRG(IInputStream& STRG)
for (uint StringIdx = 0; StringIdx < NumStrings; StringIdx++)
{
STRG.GoTo( StringOffsets[StringIdx] );
Language.Strings[StringIdx] = STRG.Read16String().ToUTF8();
TString String = STRG.Read16String().ToUTF8();
// Flag the string as localized if it is different than the English
// version of the same string.
const TString& kEnglishString = (StringIdx == 0 ? String :
mpStringTable->mLanguages[0].Strings[StringIdx].String);
bool IsLocalized = (LanguageIdx == 0 || String != kEnglishString);
Language.Strings[StringIdx].String = String;
Language.Strings[StringIdx].IsLocalized = IsLocalized;
}
}
}
@ -120,6 +133,9 @@ void CStringLoader::LoadCorruptionSTRG(IInputStream& STRG)
}
}
// Some of the following code assumes that language 0 is English
ASSERT( mpStringTable->mLanguages[0].Language == ELanguage::English );
// Strings
uint StringsStart = STRG.Tell();
@ -132,7 +148,11 @@ void CStringLoader::LoadCorruptionSTRG(IInputStream& STRG)
{
STRG.GoTo( StringsStart + LanguageOffsets[LanguageIdx][StringIdx] );
STRG.Skip(4); // Skipping string size
Language.Strings[StringIdx] = STRG.ReadString();
Language.Strings[StringIdx].String = STRG.ReadString();
// Flag the string as localized if it has a different offset than the English string
Language.Strings[StringIdx].IsLocalized = (LanguageIdx == 0 ||
LanguageOffsets[LanguageIdx][StringIdx] != LanguageOffsets[0][StringIdx]);
}
}
}

View File

@ -6,7 +6,6 @@
#include "CFont.h"
#include "CPoiToWorld.h"
#include "CResource.h"
#include "CScan.h"
#include "CTexture.h"
#include "CWorld.h"
#include "Core/Resource/Animation/CAnimation.h"
@ -15,6 +14,7 @@
#include "Core/Resource/Animation/CSkin.h"
#include "Core/Resource/Area/CGameArea.h"
#include "Core/Resource/Model/CModel.h"
#include "Core/Resource/Scan/CScan.h"
#include "Core/Resource/StringTable/CStringTable.h"
#endif // RESOURCES_H

View File

@ -69,7 +69,7 @@ void CPropertyNameGenerator::Generate(const SPropertyNameGenerationParameters& r
}
// If TestIntsAsChoices is enabled, and int is in the type list, then choice must be in the type list too.
if (rkParams.TestIntsAsChoices && NBasics::VectorContains(mTypeNames, TString("int")))
if (rkParams.TestIntsAsChoices && NBasics::VectorFind(mTypeNames, TString("int")) >= 0)
{
NBasics::VectorAddUnique(mTypeNames, TString("choice"));
}

View File

@ -23,8 +23,8 @@ const ELanguage gkSupportedLanguagesMP1PAL[] =
// Supported languages in Metroid Prime 3
const ELanguage gkSupportedLanguagesMP3[] =
{
ELanguage::English, ELanguage::German, ELanguage::French,
ELanguage::Spanish, ELanguage::Italian, ELanguage::Japanese
ELanguage::English, ELanguage::Japanese, ELanguage::German,
ELanguage::French, ELanguage::Spanish, ELanguage::Italian
};
// Supported languages in DKCR
@ -57,7 +57,7 @@ TString CStringTable::GetString(ELanguage Language, uint StringIndex) const
if (LanguageIdx >= 0 && mLanguages[LanguageIdx].Strings.size() > StringIndex)
{
return mLanguages[LanguageIdx].Strings[StringIndex];
return mLanguages[LanguageIdx].Strings[StringIndex].String;
}
else
{
@ -72,7 +72,9 @@ void CStringTable::SetString(ELanguage Language, uint StringIndex, const TString
if (LanguageIdx >= 0 && mLanguages[LanguageIdx].Strings.size() > StringIndex)
{
mLanguages[LanguageIdx].Strings[StringIndex] = kNewString;
mLanguages[LanguageIdx].Strings[StringIndex].String = kNewString;
mLanguages[LanguageIdx].Strings[StringIndex].IsLocalized =
(LanguageIdx == 0 || kNewString != mLanguages[0].Strings[StringIndex].String);
}
}
@ -108,7 +110,7 @@ void CStringTable::MoveString(uint StringIndex, uint NewIndex)
for (uint LanguageIdx = 0; LanguageIdx < mLanguages.size(); LanguageIdx++)
{
SLanguageData& Language = mLanguages[LanguageIdx];
TString String = Language.Strings[StringIndex];
SStringData String = Language.Strings[StringIndex];
if (NewIndex > StringIndex)
{
@ -171,7 +173,7 @@ void CStringTable::AddString(uint AtIndex)
for (uint LanguageIdx = 0; LanguageIdx < mLanguages.size(); LanguageIdx++)
{
SLanguageData& Language = mLanguages[LanguageIdx];
Language.Strings.insert( Language.Strings.begin() + AtIndex, 1, "" );
Language.Strings.insert( Language.Strings.begin() + AtIndex, 1, SStringData() );
}
}
@ -216,7 +218,7 @@ CDependencyTree* CStringTable::BuildDependencyTree() const
for (uint StringIdx = 0; StringIdx < kLanguage.Strings.size(); StringIdx++)
{
const TString& kString = kLanguage.Strings[StringIdx];
const TString& kString = kLanguage.Strings[StringIdx].String;
for (int TagIdx = kString.IndexOf('&'); TagIdx != -1; TagIdx = kString.IndexOf('&', TagIdx + 1))
{

View File

@ -15,15 +15,32 @@ class CStringTable : public CResource
{
DECLARE_RESOURCE_TYPE(StringTable)
friend class CStringLoader;
friend class CStringCooker;
/** List of string names. Optional data, can be empty. */
std::vector<TString> mStringNames;
/** String data for a language */
struct SStringData
{
TString String;
bool IsLocalized;
SStringData()
: IsLocalized(false)
{}
void Serialize(IArchive& Arc)
{
Arc << SerialParameter("String", String)
<< SerialParameter("IsLocalized", IsLocalized, SH_Optional, true);
}
};
struct SLanguageData
{
ELanguage Language;
std::vector<TString> Strings;
std::vector<SStringData> Strings;
void Serialize(IArchive& Arc)
{

View File

@ -60,6 +60,21 @@ CStringEditor::~CStringEditor()
delete mpUI;
}
bool CStringEditor::Save()
{
if (!mpStringTable->Entry()->Save())
{
UICommon::ErrorMsg(this, "Failed to save!");
return false;
}
else
{
UndoStack().setClean();
setWindowModified(false);
return true;
}
}
bool CStringEditor::eventFilter(QObject* pWatched, QEvent* pEvent)
{
if (pEvent->type() == QEvent::FocusOut)
@ -120,6 +135,9 @@ void CStringEditor::InitUI()
connect( mpListModel, SIGNAL(rowsMoved(QModelIndex,int,int,QModelIndex,int)),
this, SLOT(OnRowsMoved(QModelIndex,int,int,QModelIndex,int)) );
connect( mpUI->ActionSave, SIGNAL(triggered(bool)), this, SLOT(Save()) );
connect( mpUI->ActionSaveAndCook, SIGNAL(triggered(bool)), this, SLOT(SaveAndRepack()) );
connect( &UndoStack(), SIGNAL(indexChanged(int)), this, SLOT(UpdateUI()) );
mpUI->ToolBar->addSeparator();
@ -224,6 +242,7 @@ void CStringEditor::UpdateUI()
pSelectionModel->blockSignals(true);
pSelectionModel->setCurrentIndex(NewStringIndex, QItemSelectionModel::ClearAndSelect);
pSelectionModel->blockSignals(false);
mpUI->StringNameListView->scrollTo(NewStringIndex);
mpUI->StringNameListView->update(OldStringIndex);
}
mpUI->StringNameListView->update(NewStringIndex);

View File

@ -43,7 +43,8 @@ public:
explicit CStringEditor(CStringTable* pStringTable, QWidget* pParent = 0);
~CStringEditor();
bool eventFilter(QObject* pWatched, QEvent* pEvent);
virtual bool Save() override;
virtual bool eventFilter(QObject* pWatched, QEvent* pEvent) override;
void InitUI();
void UpdateStatusBar();