diff --git a/resources/list/AssetListDKCR.xml b/resources/list/AssetListDKCR.xml new file mode 100644 index 00000000..5d54f87a --- /dev/null +++ b/resources/list/AssetListDKCR.xml @@ -0,0 +1,2 @@ + + \ No newline at end of file diff --git a/resources/list/AssetListMP1.xml b/resources/list/AssetListMP1.xml new file mode 100644 index 00000000..b9e79702 --- /dev/null +++ b/resources/list/AssetListMP1.xml @@ -0,0 +1,6 @@ + + + Samus\ + PhazonSuit + + \ No newline at end of file diff --git a/resources/list/AssetListMP1Demo.xml b/resources/list/AssetListMP1Demo.xml new file mode 100644 index 00000000..5d54f87a --- /dev/null +++ b/resources/list/AssetListMP1Demo.xml @@ -0,0 +1,2 @@ + + \ No newline at end of file diff --git a/resources/list/AssetListMP2.xml b/resources/list/AssetListMP2.xml new file mode 100644 index 00000000..5d54f87a --- /dev/null +++ b/resources/list/AssetListMP2.xml @@ -0,0 +1,2 @@ + + \ No newline at end of file diff --git a/resources/list/AssetListMP2Demo.xml b/resources/list/AssetListMP2Demo.xml new file mode 100644 index 00000000..5d54f87a --- /dev/null +++ b/resources/list/AssetListMP2Demo.xml @@ -0,0 +1,2 @@ + + \ No newline at end of file diff --git a/resources/list/AssetListMP3.xml b/resources/list/AssetListMP3.xml new file mode 100644 index 00000000..5d54f87a --- /dev/null +++ b/resources/list/AssetListMP3.xml @@ -0,0 +1,2 @@ + + \ No newline at end of file diff --git a/resources/list/AssetListMP3Proto.xml b/resources/list/AssetListMP3Proto.xml new file mode 100644 index 00000000..5d54f87a --- /dev/null +++ b/resources/list/AssetListMP3Proto.xml @@ -0,0 +1,2 @@ + + \ No newline at end of file diff --git a/src/Common/CUniqueID.cpp b/src/Common/CUniqueID.cpp index 14a26d7f..79fbece4 100644 --- a/src/Common/CUniqueID.cpp +++ b/src/Common/CUniqueID.cpp @@ -13,13 +13,13 @@ using IOUtil::eBigEndian; CUniqueID::CUniqueID() : mLength(eInvalidUIDLength) { - memset(mID, 0xFF, 16); + memset(mID, 0, 16); } CUniqueID::CUniqueID(u64 ID) { // This constructor is intended to be used with both 32-bit and 64-bit input values - memset(mID, 0xFF, 16); + memset(mID, 0, 16); // 64-bit - check for valid content in upper 32 bits (at least one bit set + one bit unset) if ((ID & 0xFFFFFFFF00000000) && (~ID & 0xFFFFFFFF00000000)) @@ -43,7 +43,7 @@ CUniqueID::CUniqueID(u64 ID) CUniqueID::CUniqueID(u64 ID, EUIDLength Length) { // This constructor shouldn't be used for 128-bit - memset(mID, 0xFF, 16); + memset(mID, 0, 16); // 64-bit if (Length == e64Bit || Length == e128Bit) diff --git a/src/Common/FileUtil.cpp b/src/Common/FileUtil.cpp index 3840ff18..13fce003 100644 --- a/src/Common/FileUtil.cpp +++ b/src/Common/FileUtil.cpp @@ -16,7 +16,7 @@ bool Exists(const TWideString &rkFilePath) bool IsRoot(const TWideString& rkPath) { - // todo: verify that this is actually a good way of checking for root + // todo: is this actually a good way of checking for root? TWideString AbsPath = MakeAbsolute(rkPath); TWideStringList Split = AbsPath.Split(L"\\/"); return (Split.size() <= 1); @@ -34,11 +34,23 @@ bool IsDirectory(const TWideString& rkDirPath) bool CreateDirectory(const TWideString& rkNewDir) { + if (!IsValidPath(rkNewDir, true)) + { + Log::Error("Unable to create directory because name contains illegal characters: " + rkNewDir.ToUTF8()); + return false; + } + return create_directories(*rkNewDir); } bool CopyFile(const TWideString& rkOrigPath, const TWideString& rkNewPath) { + if (!IsValidPath(rkNewPath, false)) + { + Log::Error("Unable to copy file because destination name contains illegal characters: " + rkNewPath.ToUTF8()); + return false; + } + boost::system::error_code Error; copy(*rkOrigPath, *rkNewPath, Error); return (Error == boost::system::errc::success); @@ -46,6 +58,12 @@ bool CopyFile(const TWideString& rkOrigPath, const TWideString& rkNewPath) bool CopyDirectory(const TWideString& rkOrigPath, const TWideString& rkNewPath) { + if (!IsValidPath(rkNewPath, true)) + { + Log::Error("Unable to copy directory because destination name contains illegal characters: " + rkNewPath.ToUTF8()); + return false; + } + boost::system::error_code Error; copy_directory(*rkOrigPath, *rkNewPath, Error); return (Error == boost::system::errc::success); @@ -53,6 +71,12 @@ bool CopyDirectory(const TWideString& rkOrigPath, const TWideString& rkNewPath) bool MoveFile(const TWideString& rkOldPath, const TWideString& rkNewPath) { + if (!IsValidPath(rkNewPath, false)) + { + Log::Error("Unable to move file because destination name contains illegal characters: " + rkNewPath.ToUTF8()); + return false; + } + if (CopyFile(rkOldPath, rkNewPath)) return DeleteFile(rkOldPath); else @@ -61,6 +85,12 @@ bool MoveFile(const TWideString& rkOldPath, const TWideString& rkNewPath) bool MoveDirectory(const TWideString& rkOldPath, const TWideString& rkNewPath) { + if (!IsValidPath(rkNewPath, true)) + { + Log::Error("Unable to move directory because destination name contains illegal characters: " + rkNewPath.ToUTF8()); + return false; + } + if (CopyDirectory(rkOldPath, rkNewPath)) return DeleteDirectory(rkOldPath); else @@ -201,6 +231,147 @@ TWideString MakeRelative(const TWideString& rkPath, const TWideString& rkRelativ return Out; } +static const wchar_t gskIllegalNameChars[] = { + L'<', L'>', L'\"', L'/', L'\\', L'|', L'?', L'*' +}; + +TWideString SanitizeName(TWideString Name, bool Directory, bool RootDir /*= false*/) +{ + // Windows only atm + if (Directory && (Name == L"." || Name == L"..")) + return Name; + + // Remove illegal characters from path + u32 NumIllegalChars = sizeof(gskIllegalNameChars) / sizeof(wchar_t); + + for (u32 iChr = 0; iChr < Name.Size(); iChr++) + { + wchar_t Chr = Name[iChr]; + bool Remove = false; + + if (Chr >= 0 && Chr <= 31) + Remove = true; + + // For root, allow colon only as the last character of the name + else if (Chr == L':' && (!RootDir || iChr != Name.Size() - 1)) + Remove = true; + + else + { + for (u32 iBan = 0; iBan < NumIllegalChars; iBan++) + { + if (Chr == gskIllegalNameChars[iBan]) + { + Remove = true; + break; + } + } + } + + if (Remove) + { + Name.Remove(iChr, 1); + iChr--; + } + } + + // For directories, space and dot are not allowed at the end of the path + if (Directory) + { + u32 ChopNum = 0; + + for (int iChr = (int) Name.Size() - 1; iChr >= 0; iChr--) + { + wchar_t Chr = Name[iChr]; + + if (Chr == L' ' || Chr == L'.') + ChopNum++; + else + break; + } + + if (ChopNum > 0) Name = Name.ChopBack(ChopNum); + } + + return Name; +} + +TWideString SanitizePath(TWideString Path, bool Directory) +{ + TWideStringList Components = Path.Split(L"\\/"); + u32 CompIdx = 0; + Path = L""; + + for (auto It = Components.begin(); It != Components.end(); It++) + { + TWideString Comp = *It; + bool IsDir = Directory || CompIdx < Components.size() - 1; + bool IsRoot = CompIdx == 0; + SanitizeName(Comp, IsDir, IsRoot); + + Path += Comp; + if (IsDir) Path += L'\\'; + CompIdx++; + } + + return Path; +} + +bool IsValidName(const TWideString& rkName, bool Directory, bool RootDir /*= false*/) +{ + // Windows only atm + u32 NumIllegalChars = sizeof(gskIllegalNameChars) / sizeof(wchar_t); + + if (Directory && (rkName == L"." || rkName == L"..")) + return true; + + // Check for banned characters + for (u32 iChr = 0; iChr < rkName.Size(); iChr++) + { + wchar_t Chr = rkName[iChr]; + + if (Chr >= 0 && Chr <= 31) + return false; + + // Allow colon only on last character of root + if (Chr == L':' && (!RootDir || iChr != rkName.Size() - 1)) + return false; + + for (u32 iBan = 0; iBan < NumIllegalChars; iBan++) + { + if (Chr == gskIllegalNameChars[iBan]) + return false; + } + } + + if (Directory && (rkName.Back() == L' ' || rkName.Back() == L'.')) + return false; + + return true; +} + +bool IsValidPath(const TWideString& rkPath, bool Directory) +{ + // Windows only atm + TWideStringList Components = rkPath.Split(L"\\/"); + + // Validate other components + u32 CompIdx = 0; + + for (auto It = Components.begin(); It != Components.end(); It++) + { + bool IsRoot = CompIdx == 0; + bool IsDir = Directory || CompIdx < (Components.size() - 1); + + if (!IsValidName(*It, IsDir, IsRoot)) + return false; + + CompIdx++; + } + + return true; +} + void GetDirectoryContents(TWideString DirPath, TWideStringList& rOut, bool Recursive /*= true*/, bool IncludeFiles /*= true*/, bool IncludeDirs /*= true*/) { if (IsDirectory(DirPath)) diff --git a/src/Common/FileUtil.h b/src/Common/FileUtil.h index 61a763f4..f1477cf1 100644 --- a/src/Common/FileUtil.h +++ b/src/Common/FileUtil.h @@ -24,6 +24,10 @@ u64 LastModifiedTime(const TWideString& rkFilePath); TWideString WorkingDirectory(); TWideString MakeAbsolute(TWideString Path); TWideString MakeRelative(const TWideString& rkPath, const TWideString& rkRelativeTo = WorkingDirectory()); +TWideString SanitizeName(TWideString Name, bool Directory, bool RootDir = false); +TWideString SanitizePath(TWideString Path, bool Directory); +bool IsValidName(const TWideString& rkName, bool Directory, bool RootDir = false); +bool IsValidPath(const TWideString& rkPath, bool Directory); void GetDirectoryContents(TWideString DirPath, TWideStringList& rOut, bool Recursive = true, bool IncludeFiles = true, bool IncludeDirs = true); } diff --git a/src/Common/TString.h b/src/Common/TString.h index 029ec0cf..5bab5225 100644 --- a/src/Common/TString.h +++ b/src/Common/TString.h @@ -29,8 +29,9 @@ * be encoded in UTF-16. */ -// Helper macro for creating string literals of the correct char type. Internal use only! Invalid outside of this header! +// Helper macros for creating string literals of the correct char type. Internal use only! Invalid outside of this header! #define LITERAL(Text) (typeid(CharType) == typeid(char)) ? (const CharType*) ##Text : (const CharType*) L##Text +#define CHAR_LITERAL(Text) (CharType) Text // ************ TBasicString ************ template @@ -116,14 +117,21 @@ public: return Size(); } + inline u32 IndexOf(CharType Character, u32 Offset) const + { + size_t Pos = mInternalString.find_first_of(Character, Offset); + return (Pos == _TStdString::npos ? -1 : (u32) Pos); + } + + inline u32 IndexOf(CharType Character) const + { + return IndexOf(Character, 0); + } + inline u32 IndexOf(const CharType* pkCharacters, u32 Offset) const { size_t Pos = mInternalString.find_first_of(pkCharacters, Offset); - - if (Pos == _TStdString::npos) - return -1; - else - return (u32) Pos; + return (Pos == _TStdString::npos ? -1 : (u32) Pos); } inline u32 IndexOf(const CharType* pkCharacters) const @@ -233,6 +241,20 @@ public: mInternalString.erase(Pos, Len); } + inline void Remove(const CharType* pkStr, bool CaseSensitive = false) + { + u32 InStrLen = CStringLength(pkStr); + + for (u32 Idx = IndexOfPhrase(pkStr, CaseSensitive); Idx != -1; Idx = IndexOfPhrase(pkStr, Idx, CaseSensitive)) + Remove(Idx, InStrLen); + } + + inline void Remove(CharType Chr) + { + for (u32 Idx = IndexOf(Chr); Idx != -1; Idx = IndexOf(Chr, Idx)) + Remove(Idx, 1); + } + inline void Replace(const CharType* pkStr, const CharType *pkReplacement, bool CaseSensitive = false) { u32 Offset = 0; @@ -581,6 +603,29 @@ public: return SubString(0, EndName); } + _TString GetParentDirectoryPath(_TString ParentDirName, bool CaseSensitive = true) + { + if (!CaseSensitive) ParentDirName = ParentDirName.ToUpper(); + + int IdxA = 0; + int IdxB = IndexOf(LITERAL("\\/")); + if (IdxB == -1) return _TString(); + + while (IdxB != -1) + { + _TString DirName = SubString(IdxA, IdxB - IdxA); + if (!CaseSensitive) DirName = DirName.ToUpper(); + + if (DirName == ParentDirName) + return Truncate(IdxB + 1); + + IdxA = IdxB + 1; + IdxB = IndexOf(LITERAL("\\/"), IdxA); + } + + return _TString(); + } + // Operators inline _TString& operator=(const CharType* pkText) { @@ -788,23 +833,23 @@ public: static TBasicString FromInt32(s32 Value, int Width = 0, int Base = 16) { std::basic_stringstream sstream; - sstream << std::setbase(Base) << std::setw(Width) << std::setfill('0') << Value; + sstream << std::setbase(Base) << std::setw(Width) << std::setfill(CHAR_LITERAL('0')) << Value; return sstream.str(); } static TBasicString FromInt64(s64 Value, int Width = 0, int Base = 16) { std::basic_stringstream sstream; - sstream << std::setbase(Base) << std::setw(Width) << std::setfill('0') << Value; + sstream << std::setbase(Base) << std::setw(Width) << std::setfill(CHAR_LITERAL('0')) << Value; return sstream.str(); } static TBasicString FromFloat(float Value, int MinDecimals = 1) { TString Out = std::to_string(Value); - int NumZeroes = Out.Size() - (Out.IndexOf(".") + 1); + int NumZeroes = Out.Size() - (Out.IndexOf(LITERAL(".")) + 1); - while (Out.Back() == '0' && NumZeroes > MinDecimals) + while (Out.Back() == CHAR_LITERAL('0') && NumZeroes > MinDecimals) { Out = Out.ChopBack(1); NumZeroes--; @@ -830,7 +875,7 @@ public: _TString str = sstream.str(); if (Uppercase) str = str.ToUpper(); - if (AddPrefix) str.Prepend("0x"); + if (AddPrefix) str.Prepend(LITERAL("0x")); return str; } @@ -861,16 +906,17 @@ public: static bool IsWhitespace(CharType c) { - return ( (c == '\t') || - (c == '\n') || - (c == '\v') || - (c == '\f') || - (c == '\r') || - (c == ' ') ); + return ( (c == CHAR_LITERAL('\t')) || + (c == CHAR_LITERAL('\n')) || + (c == CHAR_LITERAL('\v')) || + (c == CHAR_LITERAL('\f')) || + (c == CHAR_LITERAL('\r')) || + (c == CHAR_LITERAL(' ')) ); } }; #undef LITERAL +#undef CHAR_LITERAL // ************ TString ************ class TString : public TBasicString diff --git a/src/Core/GameProject/CGameExporter.cpp b/src/Core/GameProject/CGameExporter.cpp index 4d335412..2ffb57af 100644 --- a/src/Core/GameProject/CGameExporter.cpp +++ b/src/Core/GameProject/CGameExporter.cpp @@ -1,12 +1,16 @@ #include "CGameExporter.h" +#include "Core/Resource/CResCache.h" +#include "Core/Resource/CWorld.h" #include #include #include #include #include +#include -#define COPY_DISC_DATA 1 +#define COPY_DISC_DATA 0 #define LOAD_PAKS 1 +#define EXPORT_WORLDS 1 #define EXPORT_COOKED 1 CGameExporter::CGameExporter(const TString& rkInputDir, const TString& rkOutputDir) @@ -15,25 +19,37 @@ CGameExporter::CGameExporter(const TString& rkInputDir, const TString& rkOutputD mExportDir = FileUtil::MakeAbsolute(rkOutputDir); mpProject = new CGameProject(mExportDir); - mDiscDir = mpProject->DiscDir(); - mResDir = mpProject->ResourcesDir(); - mWorldsDir = mpProject->WorldsDir(); - mCookedDir = mpProject->CookedDir(); - mCookedResDir = mpProject->CookedResourcesDir(); - mCookedWorldsDir = mpProject->CookedWorldsDir(); + mDiscDir = mpProject->DiscDir(true); + mResDir = mpProject->ResourcesDir(true); + mWorldsDir = mpProject->WorldsDir(true); + mCookedDir = mpProject->CookedDir(false); + mCookedResDir = mpProject->CookedResourcesDir(true); + mCookedWorldsDir = mpProject->CookedWorldsDir(true); } bool CGameExporter::Export() { SCOPED_TIMER(ExportGame); + gResCache.SetGameExporter(this); FileUtil::CreateDirectory(mExportDir); FileUtil::ClearDirectory(mExportDir); + CopyDiscData(); + LoadAssetList(); LoadPaks(); + ExportWorlds(); ExportCookedResources(); + + gResCache.SetGameExporter(nullptr); return true; } +void CGameExporter::LoadResource(const CUniqueID& rkID, std::vector& rBuffer) +{ + SResourceInstance *pInst = FindResourceInstance(rkID); + if (pInst) LoadResource(*pInst, rBuffer); +} + // ************ PROTECTED ************ void CGameExporter::CopyDiscData() { @@ -93,6 +109,57 @@ void CGameExporter::CopyDiscData() ASSERT(Game() != eUnknownVersion); } +void CGameExporter::LoadAssetList() +{ + SCOPED_TIMER(LoadAssetList); + + // Determine the asset list to use + TString ListFile = "../resources/list/AssetList"; + + switch (Game()) + { + case ePrimeDemo: ListFile += "MP1Demo"; break; + case ePrime: ListFile += "MP1"; break; + case eEchoesDemo: ListFile += "MP2Demo"; break; + case eEchoes: ListFile += "MP2"; break; + case eCorruptionProto: ListFile += "MP3Proto"; break; + case eCorruption: ListFile += "MP3"; break; + case eReturns: ListFile += "DKCR"; break; + default: ASSERT(false); + } + + ListFile += ".xml"; + + // Load list + tinyxml2::XMLDocument List; + List.LoadFile(*ListFile); + + if (List.Error()) + { + Log::Error("Couldn't open asset list: " + ListFile); + return; + } + + tinyxml2::XMLElement *pRoot = List.FirstChildElement("AssetList"); + tinyxml2::XMLElement *pAsset = pRoot->FirstChildElement("Asset"); + + while (pAsset) + { + u64 ResourceID = TString(pAsset->Attribute("ID")).ToInt64(16); + + tinyxml2::XMLElement *pDir = pAsset->FirstChildElement("Dir"); + TString Dir = pDir ? pDir->GetText() : ""; + + tinyxml2::XMLElement *pName = pAsset->FirstChildElement("Name"); + TString Name = pName ? pName->GetText() : ""; + + if (!Dir.EndsWith("/") && !Dir.EndsWith("\\")) Dir.Append("\\"); + SetResourcePath(ResourceID, mResDir + Dir.ToUTF16(), Name.ToUTF16()); + + pAsset = pAsset->NextSiblingElement("Asset"); + } +} + // ************ RESOURCE LOADING ************ void CGameExporter::LoadPaks() { @@ -117,7 +184,7 @@ void CGameExporter::LoadPaks() continue; } - CPackage *pPackage = new CPackage(CharPak.GetFileName(false)); + CPackage *pPackage = new CPackage(CharPak.GetFileName(false), FileUtil::MakeRelative(PakPath.GetFileDirectory(), mExportDir)); // MP1-MP3Proto if (Game() < eCorruption) @@ -134,11 +201,11 @@ void CGameExporter::LoadPaks() for (u32 iName = 0; iName < NumNamedResources; iName++) { - Pak.Seek(0x4, SEEK_CUR); // Skip resource type + CFourCC ResType = Pak.ReadLong(); CUniqueID ResID(Pak, IDLength); u32 NameLen = Pak.ReadLong(); TString Name = Pak.ReadString(NameLen); - pPackage->AddNamedResource(Name, ResID); + pPackage->AddNamedResource(Name, ResID, ResType); } u32 NumResources = Pak.ReadLong(); @@ -153,7 +220,7 @@ void CGameExporter::LoadPaks() u64 IntegralID = ResID.ToLongLong(); if (mResourceMap.find(IntegralID) == mResourceMap.end()) - mResourceMap[IntegralID] = SResourceInstance { PakPath, ResID, ResType, ResOffset, ResSize, Compressed }; + mResourceMap[IntegralID] = SResourceInstance { PakPath, ResID, ResType, ResOffset, ResSize, Compressed, false }; } } } @@ -194,9 +261,9 @@ void CGameExporter::LoadPaks() for (u32 iName = 0; iName < NumNamedResources; iName++) { TString Name = Pak.ReadString(); - Pak.Seek(0x4, SEEK_CUR); // Skip type + CFourCC ResType = Pak.ReadLong(); CUniqueID ResID(Pak, IDLength); - pPackage->AddNamedResource(Name, ResID); + pPackage->AddNamedResource(Name, ResID, ResType); } } @@ -216,7 +283,7 @@ void CGameExporter::LoadPaks() u64 IntegralID = ResID.ToLongLong(); if (mResourceMap.find(IntegralID) == mResourceMap.end()) - mResourceMap[IntegralID] = SResourceInstance { PakPath, ResID, Type, Offset, Size, Compressed }; + mResourceMap[IntegralID] = SResourceInstance { PakPath, ResID, Type, Offset, Size, Compressed, false }; } } @@ -231,7 +298,7 @@ void CGameExporter::LoadPaks() #endif } -void CGameExporter::LoadPakResource(const SResourceInstance& rkResource, std::vector& rBuffer) +void CGameExporter::LoadResource(const SResourceInstance& rkResource, std::vector& rBuffer) { CFileInStream Pak(rkResource.PakFile.ToUTF8().ToStdString(), IOUtil::eBigEndian); @@ -327,30 +394,106 @@ void CGameExporter::LoadPakResource(const SResourceInstance& rkResource, std::ve } } +void CGameExporter::ExportWorlds() +{ +#if EXPORT_WORLDS + SCOPED_TIMER(ExportWorlds); + //CResourceDatabase *pResDB = mpProject->ResourceDatabase(); + + for (u32 iPak = 0; iPak < mpProject->NumWorldPaks(); iPak++) + { + CPackage *pPak = mpProject->WorldPakByIndex(iPak); + + // Get output path. DKCR paks are stored in a Worlds folder so we should get the path relative to that so we don't have Worlds\Worlds\. + // Other games have all paks in the game root dir so we're fine just taking the original root dir-relative directory. + TWideString PakPath = pPak->PakPath(); + TWideString WorldsDir = PakPath.GetParentDirectoryPath(L"Worlds", false); + + if (!WorldsDir.IsEmpty()) + PakPath = FileUtil::MakeRelative(PakPath, WorldsDir); + + for (u32 iRes = 0; iRes < pPak->NumNamedResources(); iRes++) + { + const SNamedResource& rkRes = pPak->NamedResourceByIndex(iRes); + + if (rkRes.Type == "MLVL" && !rkRes.Name.EndsWith("NODEPEND")) + { + TResPtr pWorld = (CWorld*) gResCache.GetResource(rkRes.ID, rkRes.Type); + + if (!pWorld) + { + Log::Error("Couldn't load world " + rkRes.Name + " from package " + pPak->PakName() + "; unable to export"); + continue; + } + + // Export world + TWideString Name = rkRes.Name.ToUTF16(); + TWideString WorldDir = mWorldsDir + PakPath + FileUtil::SanitizeName(Name, true) + L"\\"; + FileUtil::CreateDirectory(mCookedDir + WorldDir); + + SResourceInstance *pInst = FindResourceInstance(rkRes.ID); + ASSERT(pInst != nullptr); + + SetResourcePath(rkRes.ID, WorldDir, Name); + ExportResource(*pInst); + + // Export areas + for (u32 iArea = 0; iArea < pWorld->NumAreas(); iArea++) + { + // Determine area names + TWideString InternalAreaName = pWorld->AreaInternalName(iArea).ToUTF16(); + if (InternalAreaName.IsEmpty()) InternalAreaName = TWideString::FromInt32(iArea, 2, 10); + + TWideString GameAreaName; + CStringTable *pTable = pWorld->AreaName(iArea); + if (pTable) GameAreaName = pTable->String("ENGL", 0); + if (GameAreaName.IsEmpty()) GameAreaName = InternalAreaName; + + // Load area + CUniqueID AreaID = pWorld->AreaResourceID(iArea); + CGameArea *pArea = (CGameArea*) gResCache.GetResource(AreaID, "MREA"); + + if (!pArea) + { + Log::Error("Unable to export area " + GameAreaName.ToUTF8() + " from world " + rkRes.Name + "; couldn't load area"); + continue; + } + + // Export area + TWideString AreaDir = WorldDir + TWideString::FromInt32(iArea, 2, 10) + L"_" + FileUtil::SanitizeName(GameAreaName, true) + L"\\"; + FileUtil::CreateDirectory(mCookedDir + AreaDir); + + SResourceInstance *pInst = FindResourceInstance(AreaID); + ASSERT(pInst != nullptr); + + SetResourcePath(AreaID, AreaDir, InternalAreaName); + ExportResource(*pInst); + } + + gResCache.Clean(); + } + + else + { + Log::Error("Unexpected named resource type in world pak: " + rkRes.Type.ToString()); + } + } + } +#endif +} + void CGameExporter::ExportCookedResources() { #if EXPORT_COOKED CResourceDatabase *pResDB = mpProject->ResourceDatabase(); { SCOPED_TIMER(ExportCookedResources); - FileUtil::CreateDirectory(mCookedResDir); + FileUtil::CreateDirectory(mCookedDir + mResDir); for (auto It = mResourceMap.begin(); It != mResourceMap.end(); It++) { - const SResourceInstance& rkRes = It->second; - std::vector ResourceData; - LoadPakResource(rkRes, ResourceData); - - TString OutName = rkRes.ResourceID.ToString() + "." + rkRes.ResourceType.ToString(); - TString OutDir = mCookedResDir.ToUTF8() + "\\"; - TString OutPath = OutDir + OutName; - CFileOutStream Out(OutPath.ToStdString(), IOUtil::eBigEndian); - - if (Out.IsValid()) - Out.WriteBytes(ResourceData.data(), ResourceData.size()); - - // Add to resource DB - pResDB->RegisterResource(rkRes.ResourceID, FileUtil::MakeRelative(OutDir, mCookedDir), OutName, CResource::ResTypeForExtension(rkRes.ResourceType)); + SResourceInstance& rRes = It->second; + ExportResource(rRes); } } { @@ -359,3 +502,37 @@ void CGameExporter::ExportCookedResources() } #endif } + +void CGameExporter::ExportResource(SResourceInstance& rRes) +{ + if (!rRes.Exported) + { + std::vector ResourceData; + LoadResource(rRes, ResourceData); + + // Determine output path + SResourcePath *pPath = FindResourcePath(rRes.ResourceID); + TString OutName, OutDir; + + if (pPath) + { + OutName = pPath->Name.ToUTF8(); + OutDir = pPath->Dir.ToUTF8(); + } + + if (OutName.IsEmpty()) OutName = rRes.ResourceID.ToString(); + if (OutDir.IsEmpty()) OutDir = mResDir; + + // Write to file + FileUtil::CreateDirectory(mCookedDir + OutDir.ToUTF16()); + TString OutPath = mCookedDir.ToUTF8() + OutDir + OutName + "." + rRes.ResourceType.ToString(); + CFileOutStream Out(OutPath.ToStdString(), IOUtil::eBigEndian); + + if (Out.IsValid()) + Out.WriteBytes(ResourceData.data(), ResourceData.size()); + + // Add to resource DB + mpProject->ResourceDatabase()->RegisterResource(rRes.ResourceID, OutDir, OutName, CResource::ResTypeForExtension(rRes.ResourceType)); + rRes.Exported = true; + } +} diff --git a/src/Core/GameProject/CGameExporter.h b/src/Core/GameProject/CGameExporter.h index 1ac6e4a4..3c8112bd 100644 --- a/src/Core/GameProject/CGameExporter.h +++ b/src/Core/GameProject/CGameExporter.h @@ -35,18 +35,55 @@ class CGameExporter u32 PakOffset; u32 PakSize; bool Compressed; + bool Exported; }; std::map mResourceMap; + struct SResourcePath + { + TWideString Dir; + TWideString Name; + }; + std::map mResourcePaths; + public: CGameExporter(const TString& rkInputDir, const TString& rkOutputDir); bool Export(); + void LoadResource(const CUniqueID& rkID, std::vector& rBuffer); protected: void CopyDiscData(); + void LoadAssetList(); void LoadPaks(); - void LoadPakResource(const SResourceInstance& rkResource, std::vector& rBuffer); + void LoadResource(const SResourceInstance& rkResource, std::vector& rBuffer); + void ExportWorlds(); void ExportCookedResources(); + void ExportResource(SResourceInstance& rRes); + + // Convenience Functions + inline SResourceInstance* FindResourceInstance(const CUniqueID& rkID) + { + u64 IntegralID = rkID.ToLongLong(); + auto Found = mResourceMap.find(IntegralID); + return (Found == mResourceMap.end() ? nullptr : &Found->second); + } + + inline SResourcePath* FindResourcePath(const CUniqueID& rkID) + { + u64 IntegralID = rkID.ToLongLong(); + auto Found = mResourcePaths.find(IntegralID); + return (Found == mResourcePaths.end() ? nullptr : &Found->second); + } + + inline void SetResourcePath(const CUniqueID& rkID, const TWideString& rkDir, const TWideString& rkName) + { + SetResourcePath(rkID.ToLongLong(), rkDir, rkName); + } + + inline void SetResourcePath(u64 ID, const TWideString& rkDir, const TWideString& rkName) + { + mResourcePaths[ID] = SResourcePath { rkDir, rkName }; + } inline EGame Game() const { return mpProject->Game(); } inline void SetGame(EGame Game) { mpProject->SetGame(Game); } diff --git a/src/Core/GameProject/CGameProject.h b/src/Core/GameProject/CGameProject.h index c282d83f..76c14d40 100644 --- a/src/Core/GameProject/CGameProject.h +++ b/src/Core/GameProject/CGameProject.h @@ -28,18 +28,21 @@ public: void AddPackage(CPackage *pPackage, bool WorldPak); // Directory Handling - inline TWideString ProjectRoot() const { return mProjectRoot; } - inline TWideString DiscDir() const { return mProjectRoot + L"Disc\\"; } - inline TWideString ResourcesDir() const { return mProjectRoot + L"Resources\\"; } - inline TWideString WorldsDir() const { return mProjectRoot + L"Worlds\\"; } - inline TWideString CookedDir() const { return mProjectRoot + L"Cooked\\"; } - inline TWideString CookedResourcesDir() const { return CookedDir() + L"Resources\\"; } - inline TWideString CookedWorldsDir() const { return CookedDir() + L"Worlds\\"; } + inline TWideString ProjectRoot() const { return mProjectRoot; } + inline TWideString DiscDir(bool Relative) const { return Relative ? L"Disc\\" : mProjectRoot + L"Disc\\"; } + inline TWideString ResourcesDir(bool Relative) const { return Relative ? L"Resources\\" : mProjectRoot + L"Resources\\"; } + inline TWideString WorldsDir(bool Relative) const { return Relative ? L"Worlds\\" : mProjectRoot + L"Worlds\\"; } + inline TWideString CookedDir(bool Relative) const { return Relative ? L"Cooked\\" : mProjectRoot + L"Cooked\\"; } + inline TWideString CookedResourcesDir(bool Relative) const { return CookedDir(Relative) + L"Resources\\"; } + inline TWideString CookedWorldsDir(bool Relative) const { return CookedDir(Relative) + L"Worlds\\"; } // Accessors inline void SetGame(EGame Game) { mGame = Game; } inline void SetProjectName(const TString& rkName) { mProjectName = rkName; } + inline u32 NumWorldPaks() const { return mWorldPaks.size(); } + inline CPackage* WorldPakByIndex(u32 Index) const { return mWorldPaks[Index]; } + inline EGame Game() const { return mGame; } inline CResourceDatabase* ResourceDatabase() const { return mpResourceDatabase; } }; diff --git a/src/Core/GameProject/CPackage.h b/src/Core/GameProject/CPackage.h index cef6f244..6fd6f98e 100644 --- a/src/Core/GameProject/CPackage.h +++ b/src/Core/GameProject/CPackage.h @@ -9,25 +9,32 @@ struct SNamedResource { TString Name; CUniqueID ID; + CFourCC Type; }; class CPackage { TString mPakName; + TWideString mPakPath; std::vector mNamedResources; std::vector mPakResources; public: CPackage() {} - CPackage(const TString& rkName) : mPakName(rkName) {} + + CPackage(const TString& rkName, const TWideString& rkPath) + : mPakName(rkName) + , mPakPath(rkPath) + {} inline TString PakName() const { return mPakName; } + inline TWideString PakPath() const { return mPakPath; } inline u32 NumNamedResources() const { return mNamedResources.size(); } inline const SNamedResource& NamedResourceByIndex(u32 Index) const { return mNamedResources[Index]; } - inline void SetPakName(TString NewName) { mPakName = NewName; } - inline void AddNamedResource(TString Name, const CUniqueID& rkID) { mNamedResources.push_back( SNamedResource { Name, rkID } ); } - inline void AddPakResource(const CUniqueID& rkID) { mPakResources.push_back(rkID); } + inline void SetPakName(TString NewName) { mPakName = NewName; } + inline void AddNamedResource(TString Name, const CUniqueID& rkID, const CFourCC& rkType) { mNamedResources.push_back( SNamedResource { Name, rkID, rkType } ); } + inline void AddPakResource(const CUniqueID& rkID) { mPakResources.push_back(rkID); } }; #endif // CPACKAGE diff --git a/src/Core/GameProject/CResourceDatabase.cpp b/src/Core/GameProject/CResourceDatabase.cpp index 8f0d4fc1..0648f99b 100644 --- a/src/Core/GameProject/CResourceDatabase.cpp +++ b/src/Core/GameProject/CResourceDatabase.cpp @@ -28,7 +28,7 @@ TString CResourceEntry::RawAssetPath() const TString CResourceEntry::CookedAssetPath() const { TWideString Ext = GetCookedExtension(mResourceType, mpDatabase->GameProject()->Game()).ToUTF16(); - return mpDatabase->GameProject()->CookedDir() + mFileDir + mFileName + L"." + Ext; + return mpDatabase->GameProject()->CookedDir(false) + mFileDir + mFileName + L"." + Ext; } bool CResourceEntry::NeedsRecook() const diff --git a/src/Core/Resource/CResCache.cpp b/src/Core/Resource/CResCache.cpp index cfff085a..60fa696d 100644 --- a/src/Core/Resource/CResCache.cpp +++ b/src/Core/Resource/CResCache.cpp @@ -14,12 +14,14 @@ #include "Core/Resource/Factory/CWorldLoader.h" #include +#include #include #include #include #include CResCache::CResCache() + : mpGameExporter(nullptr) { } @@ -71,8 +73,33 @@ TString CResCache::GetSourcePath() CResource* CResCache::GetResource(CUniqueID ResID, CFourCC Type) { if (!ResID.IsValid()) return nullptr; - TString Source = mResDir + ResID.ToString() + "." + Type.ToString(); - return GetResource(Source); + TString StringName = ResID.ToString() + "." + Type.ToString(); + + // With Game Exporter - get data buffer from exporter + if (mpGameExporter) + { + // Check if we already have resource loaded + auto Got = mResourceCache.find(ResID.ToLongLong()); + if (Got != mResourceCache.end()) + return Got->second; + + // Otherwise load resource + std::vector DataBuffer; + mpGameExporter->LoadResource(ResID, DataBuffer); + if (DataBuffer.empty()) return nullptr; + + CMemoryInStream MemStream(DataBuffer.data(), DataBuffer.size(), IOUtil::eBigEndian); + CResource *pRes = InternalLoadResource(MemStream, ResID, Type); + pRes->mResSource = StringName; + return pRes; + } + + // Without Game Exporter - load from file + else + { + TString Source = mResDir + StringName; + return GetResource(Source); + } } CResource* CResCache::GetResource(const TString& rkResPath) @@ -98,32 +125,11 @@ CResource* CResCache::GetResource(const TString& rkResPath) mResDir = rkResPath.GetFileDirectory(); // Load resource - CResource *pRes = nullptr; CFourCC Type = rkResPath.GetFileExtension().ToUpper(); - bool SupportedFormat = true; - - if (Type == "CMDL") pRes = CModelLoader::LoadCMDL(File); - else if (Type == "TXTR") pRes = CTextureDecoder::LoadTXTR(File); - else if (Type == "ANCS") pRes = CAnimSetLoader::LoadANCS(File); - else if (Type == "CHAR") pRes = CAnimSetLoader::LoadCHAR(File); - else if (Type == "MREA") pRes = CAreaLoader::LoadMREA(File); - else if (Type == "MLVL") pRes = CWorldLoader::LoadMLVL(File); - else if (Type == "STRG") pRes = CStringLoader::LoadSTRG(File); - else if (Type == "FONT") pRes = CFontLoader::LoadFONT(File); - else if (Type == "SCAN") pRes = CScanLoader::LoadSCAN(File); - else if (Type == "DCLN") pRes = CCollisionLoader::LoadDCLN(File); - else if (Type == "EGMC") pRes = CPoiToWorldLoader::LoadEGMC(File); - else if (Type == "CINF") pRes = CSkeletonLoader::LoadCINF(File); - else if (Type == "ANIM") pRes = CAnimationLoader::LoadANIM(File); - else if (Type == "CSKR") pRes = CSkinLoader::LoadCSKR(File); - else SupportedFormat = false; - - if (!pRes) pRes = new CResource(); // Default for unsupported formats + CResource *pRes = InternalLoadResource(File, ResID, Type); + pRes->mResSource = rkResPath; // Add to cache and cleanup - pRes->mID = *rkResPath; - pRes->mResSource = rkResPath; - mResourceCache[ResID.ToLongLong()] = pRes; mResDir = OldResDir; return pRes; } @@ -168,4 +174,36 @@ void CResCache::DeleteResource(CUniqueID ResID) } } +// ************ PROTECTED ************ +CResource* CResCache::InternalLoadResource(IInputStream& rInput, const CUniqueID& rkID, CFourCC Type) +{ + // todo - need some sort of auto-registration of loaders to avoid this if-else mess + ASSERT(mResourceCache.find(rkID.ToLongLong()) == mResourceCache.end()); // this test should be done before calling this func! + CResource *pRes = nullptr; + + // Load resource + if (Type == "CMDL") pRes = CModelLoader::LoadCMDL(rInput); + else if (Type == "TXTR") pRes = CTextureDecoder::LoadTXTR(rInput); + else if (Type == "ANCS") pRes = CAnimSetLoader::LoadANCS(rInput); + else if (Type == "CHAR") pRes = CAnimSetLoader::LoadCHAR(rInput); + else if (Type == "MREA") pRes = CAreaLoader::LoadMREA(rInput); + else if (Type == "MLVL") pRes = CWorldLoader::LoadMLVL(rInput); + else if (Type == "STRG") pRes = CStringLoader::LoadSTRG(rInput); + else if (Type == "FONT") pRes = CFontLoader::LoadFONT(rInput); + else if (Type == "SCAN") pRes = CScanLoader::LoadSCAN(rInput); + else if (Type == "DCLN") pRes = CCollisionLoader::LoadDCLN(rInput); + else if (Type == "EGMC") pRes = CPoiToWorldLoader::LoadEGMC(rInput); + else if (Type == "CINF") pRes = CSkeletonLoader::LoadCINF(rInput); + else if (Type == "ANIM") pRes = CAnimationLoader::LoadANIM(rInput); + else if (Type == "CSKR") pRes = CSkinLoader::LoadCSKR(rInput); + if (!pRes) pRes = new CResource(); // Default for unsupported formats + + ASSERT(pRes->mRefCount == 0); + + // Cache and return + pRes->mID = rkID; + mResourceCache[rkID.ToLongLong()] = pRes; + return pRes; +} + CResCache gResCache; diff --git a/src/Core/Resource/CResCache.h b/src/Core/Resource/CResCache.h index 69e26ab4..9c4f8120 100644 --- a/src/Core/Resource/CResCache.h +++ b/src/Core/Resource/CResCache.h @@ -2,6 +2,7 @@ #define CRESCACHE_H #include "CResource.h" +#include "Core/GameProject/CGameExporter.h" #include #include #include @@ -10,6 +11,7 @@ class CResCache { std::unordered_map mResourceCache; TString mResDir; + CGameExporter *mpGameExporter; public: CResCache(); @@ -22,6 +24,11 @@ public: CFourCC FindResourceType(CUniqueID ResID, const TStringList& rkPossibleTypes); void CacheResource(CResource *pRes); void DeleteResource(CUniqueID ResID); + + inline void SetGameExporter(CGameExporter *pExporter) { mpGameExporter = pExporter; } + +protected: + CResource* InternalLoadResource(IInputStream& rInput, const CUniqueID& rkID, CFourCC Type); }; extern CResCache gResCache; diff --git a/src/Core/Resource/CResource.cpp b/src/Core/Resource/CResource.cpp index 0f305fd8..fa8afcfe 100644 --- a/src/Core/Resource/CResource.cpp +++ b/src/Core/Resource/CResource.cpp @@ -120,7 +120,7 @@ REGISTER_RESOURCE_TYPE(MREA, eArea, ePrimeDemo, eReturns) REGISTER_RESOURCE_TYPE(NTWK, eTweak, eEchoesDemo, eReturns) REGISTER_RESOURCE_TYPE(PAK , ePackage, ePrimeDemo, eReturns) REGISTER_RESOURCE_TYPE(PART, eParticle, ePrimeDemo, eReturns) -REGISTER_RESOURCE_TYPE(PATH, eNavMesh, ePrimeDemo, eCorruption) +REGISTER_RESOURCE_TYPE(PATH, ePathfinding, ePrimeDemo, eCorruption) REGISTER_RESOURCE_TYPE(PTLA, ePortalArea, eEchoesDemo, eCorruption) REGISTER_RESOURCE_TYPE(RULE, eRuleSet, eEchoesDemo, eReturns) REGISTER_RESOURCE_TYPE(SAND, eSourceAnimData, eCorruptionProto, eCorruption) diff --git a/src/Core/Resource/EResType.h b/src/Core/Resource/EResType.h index 9e7df1d5..86ff47d9 100644 --- a/src/Core/Resource/EResType.h +++ b/src/Core/Resource/EResType.h @@ -14,7 +14,6 @@ enum EResType eAudioMacro, eAudioGroupSet, eAudioSample, - eStreamedAudio, eAudioLookupTable, eBinaryData, eBurstFireData, @@ -25,14 +24,12 @@ enum EResType eGuiFrame, eGuiKeyFrame, eHintSystem, - eInvalidResType, eMapArea, eMapWorld, eMapUniverse, eMidi, eModel, eMusicTrack, - eNavMesh, ePackage, eParticle, eParticleCollisionResponse, @@ -43,6 +40,7 @@ enum EResType eParticleSwoosh, eParticleTransform, eParticleWeapon, + ePathfinding, ePortalArea, eResource, eRuleSet, @@ -56,6 +54,7 @@ enum EResType eStateMachine, eStateMachine2, // For distinguishing AFSM/FSM2 eStaticGeometryMap, + eStreamedAudio, eStringList, eStringTable, eTexture, @@ -63,7 +62,9 @@ enum EResType eUnknown_CAAD, eUserEvaluatorData, eVideo, - eWorld + eWorld, + + eInvalidResType = -1 }; // defined in CResource.cpp diff --git a/src/Core/Resource/Factory/CTextureDecoder.cpp b/src/Core/Resource/Factory/CTextureDecoder.cpp index 6d333fe3..c253f428 100644 --- a/src/Core/Resource/Factory/CTextureDecoder.cpp +++ b/src/Core/Resource/Factory/CTextureDecoder.cpp @@ -201,6 +201,10 @@ void CTextureDecoder::ReadDDS(IInputStream& rDDS) // ************ DECODE ************ void CTextureDecoder::PartialDecodeGXTexture(IInputStream& TXTR) { + // TODO: This function doesn't handle very small mipmaps correctly. + // The format applies padding when the size of a mipmap is less than the block size for that format. + // The decode needs to be adjusted to account for the padding and skip over it (since we don't have padding in OpenGL). + // Get image data size, create output buffer u32 ImageStart = TXTR.Tell(); TXTR.Seek(0x0, SEEK_END); @@ -233,11 +237,22 @@ void CTextureDecoder::PartialDecodeGXTexture(IInputStream& TXTR) MipH /= 4; } + // This value set to true if we hit the end of the file earlier than expected. + // This is necessary due to a mistake Retro made in their cooker for I8 textures where very small mipmaps are cut off early, resulting in an out-of-bounds memory access. + // This affects one texture that I know of - Echoes 3bb2c034.TXTR + bool BreakEarly = false; + for (u32 iMip = 0; iMip < mNumMipMaps; iMip++) { + if (MipW < BWidth) MipW = BWidth; + if (MipH < BHeight) MipH = BHeight; + for (u32 iBlockY = 0; iBlockY < MipH; iBlockY += BHeight) - for (u32 iBlockX = 0; iBlockX < MipW; iBlockX += BWidth) { - for (u32 iImgY = iBlockY; iImgY < iBlockY + BHeight; iImgY++) { + { + for (u32 iBlockX = 0; iBlockX < MipW; iBlockX += BWidth) + { + for (u32 iImgY = iBlockY; iImgY < iBlockY + BHeight; iImgY++) + { for (u32 iImgX = iBlockX; iImgX < iBlockX + BWidth; iImgX++) { u32 DstPos = ((iImgY * MipW) + iImgX) * PixelStride; @@ -256,10 +271,17 @@ void CTextureDecoder::PartialDecodeGXTexture(IInputStream& TXTR) // I4 and C4 have 4bpp images, so I'm forced to read two pixels at a time. if ((mTexelFormat == eGX_I4) || (mTexelFormat == eGX_C4)) iImgX++; + + // Check if we're at the end of the file. + if (TXTR.EoF()) BreakEarly = true; } + if (BreakEarly) break; } if (mTexelFormat == eGX_RGBA8) TXTR.Seek(0x20, SEEK_CUR); + if (BreakEarly) break; } + if (BreakEarly) break; + } u32 MipSize = (u32) (MipW * MipH * gskPixelsToBytes[mTexelFormat]); if (mTexelFormat == eGX_CMPR) MipSize *= 16; // Since we're pretending the image is 1/4 its actual size, we have to multiply the size by 16 to get the correct offset @@ -267,8 +289,8 @@ void CTextureDecoder::PartialDecodeGXTexture(IInputStream& TXTR) MipOffset += MipSize; MipW /= 2; MipH /= 2; - if (MipW < BWidth) MipW = BWidth; - if (MipH < BHeight) MipH = BHeight; + + if (BreakEarly) break; } } diff --git a/src/Core/Resource/Script/CScriptObject.h b/src/Core/Resource/Script/CScriptObject.h index 0c671993..66162747 100644 --- a/src/Core/Resource/Script/CScriptObject.h +++ b/src/Core/Resource/Script/CScriptObject.h @@ -23,7 +23,7 @@ class CScriptObject friend class CAreaLoader; CScriptTemplate *mpTemplate; - TResPtr mpArea; + CGameArea *mpArea; CScriptLayer *mpLayer; u32 mVersion;