From 9d6798b7ae23bca89a004db6d7ff3fb0a8b3a4af Mon Sep 17 00:00:00 2001 From: Aruki Date: Wed, 3 May 2017 03:07:34 -0600 Subject: [PATCH] Fixed up MP3 asset name generation, implemented a bunch of extra checks and safeguards to ensure asset names/directories are valid --- src/Common/FileUtil.cpp | 45 ++++++- src/Common/FileUtil.h | 3 +- src/Common/TString.h | 28 ++++- src/Core/GameProject/AssetNameGeneration.cpp | 81 ++++++++++--- src/Core/GameProject/CGameExporter.cpp | 30 ++--- src/Core/GameProject/CGameProject.cpp | 3 +- src/Core/GameProject/CGameProject.h | 1 - src/Core/GameProject/CResourceEntry.cpp | 4 +- src/Core/GameProject/CResourceStore.cpp | 101 ++++++++-------- src/Core/GameProject/CResourceStore.h | 7 +- src/Core/GameProject/CVirtualDirectory.cpp | 93 +++++++++++++-- src/Core/GameProject/CVirtualDirectory.h | 5 +- src/Core/Resource/Animation/CSourceAnimData.h | 12 ++ src/Core/Resource/CResTypeInfo.cpp | 8 +- src/Core/Resource/EResType.h | 2 +- src/Core/Resource/Factory/CAnimSetLoader.cpp | 110 ++++++++++-------- 16 files changed, 358 insertions(+), 175 deletions(-) diff --git a/src/Common/FileUtil.cpp b/src/Common/FileUtil.cpp index be872b24..014dc8f7 100644 --- a/src/Common/FileUtil.cpp +++ b/src/Common/FileUtil.cpp @@ -39,7 +39,7 @@ bool IsAbsolute(const TWideString& rkDirPath) bool IsRelative(const TWideString& rkDirPath) { - return !boost::filesystem::path(*rkDirPath).is_relative(); + return boost::filesystem::path(*rkDirPath).is_relative(); } bool IsEmpty(const TWideString& rkDirPath) @@ -116,7 +116,7 @@ bool MoveDirectory(const TWideString& rkOldPath, const TWideString& rkNewPath) } if (CopyDirectory(rkOldPath, rkNewPath)) - return DeleteDirectory(rkOldPath); + return DeleteDirectory(rkOldPath, false); else return false; } @@ -127,7 +127,7 @@ bool DeleteFile(const TWideString& rkFilePath) return remove(*rkFilePath) == 1; } -bool DeleteDirectory(const TWideString& rkDirPath) +bool DeleteDirectory(const TWideString& rkDirPath, bool FailIfNotEmpty) { // This is an extremely destructive function, be careful using it! if (!IsDirectory(rkDirPath)) return false; @@ -142,6 +142,11 @@ bool DeleteDirectory(const TWideString& rkDirPath) return false; } + // Check if directory is empty + if (FailIfNotEmpty && !IsEmpty(rkDirPath)) + return false; + + // Delete directory boost::system::error_code Error; remove_all(*rkDirPath, Error); return (Error == boost::system::errc::success); @@ -173,7 +178,7 @@ bool ClearDirectory(const TWideString& rkDirPath) if (IsFile(*It)) Success = DeleteFile(*It); else if (IsDirectory(*It)) - Success = DeleteDirectory(*It); + Success = DeleteDirectory(*It, false); if (!Success) Log::Error("Failed to delete filesystem object: " + TWideString(*It).ToUTF8()); @@ -259,6 +264,32 @@ TWideString MakeRelative(const TWideString& rkPath, const TWideString& rkRelativ return Out; } +TWideString SimplifyRelativePath(const TWideString& rkPath) +{ + TWideStringList PathComponents = rkPath.Split(L"/\\"); + + TWideStringList::iterator Iter = PathComponents.begin(); + TWideStringList::iterator PrevIter = Iter; + + for (auto Iter = PathComponents.begin(); Iter != PathComponents.end(); PrevIter = Iter, Iter++) + { + if (*Iter == L".." && *PrevIter != L"..") + { + PrevIter = PathComponents.erase(PrevIter); + PrevIter = PathComponents.erase(PrevIter); + Iter = PrevIter; + Iter--; + } + } + + TWideString Out; + + for (auto Iter = PathComponents.begin(); Iter != PathComponents.end(); Iter++) + Out += *Iter + L'\\'; + + return Out; +} + static const wchar_t gskIllegalNameChars[] = { L'<', L'>', L'\"', L'/', L'\\', L'|', L'?', L'*' }; @@ -355,7 +386,8 @@ TWideString SanitizePath(TWideString Path, bool Directory) bool IsValidName(const TWideString& rkName, bool Directory, bool RootDir /*= false*/) { - // Windows only atm + // Only accounting for Windows limitations right now. However, this function should + // ideally return the same output on all platforms to ensure projects are cross compatible. if (rkName.IsEmpty()) return false; @@ -391,7 +423,8 @@ bool IsValidName(const TWideString& rkName, bool Directory, bool RootDir /*= fal bool IsValidPath(const TWideString& rkPath, bool Directory) { - // Windows only atm + // Only accounting for Windows limitations right now. However, this function should + // ideally return the same output on all platforms to ensure projects are cross compatible. TWideStringList Components = rkPath.Split(L"\\/"); // Validate other components diff --git a/src/Common/FileUtil.h b/src/Common/FileUtil.h index a68d3384..c7ef5580 100644 --- a/src/Common/FileUtil.h +++ b/src/Common/FileUtil.h @@ -20,13 +20,14 @@ bool CopyDirectory(const TWideString& rkOrigPath, const TWideString& rkNewPath); bool MoveFile(const TWideString& rkOldPath, const TWideString& rkNewPath); bool MoveDirectory(const TWideString& rkOldPath, const TWideString& rkNewPath); bool DeleteFile(const TWideString& rkFilePath); -bool DeleteDirectory(const TWideString& rkDirPath); // This is an extremely destructive function, be careful using it! +bool DeleteDirectory(const TWideString& rkDirPath, bool FailIfNotEmpty); // This is an extremely destructive function, be careful using it! bool ClearDirectory(const TWideString& rkDirPath); // This is an extremely destructive function, be careful using it! u64 FileSize(const TWideString& rkFilePath); u64 LastModifiedTime(const TWideString& rkFilePath); TWideString WorkingDirectory(); TWideString MakeAbsolute(TWideString Path); TWideString MakeRelative(const TWideString& rkPath, const TWideString& rkRelativeTo = WorkingDirectory()); +TWideString SimplifyRelativePath(const TWideString& rkPath); TWideString SanitizeName(TWideString Name, bool Directory, bool RootDir = false); TWideString SanitizePath(TWideString Path, bool Directory); bool IsValidName(const TWideString& rkName, bool Directory, bool RootDir = false); diff --git a/src/Common/TString.h b/src/Common/TString.h index cc173ef5..589068a8 100644 --- a/src/Common/TString.h +++ b/src/Common/TString.h @@ -459,7 +459,7 @@ public: return mInternalString; } - _TStringList Split(const CharType* pkTokens) const + _TStringList Split(const CharType* pkTokens, bool KeepEmptyParts = false) const { _TStringList Out; u32 LastSplit = 0; @@ -475,7 +475,7 @@ public: if (mInternalString[iChr] == pkTokens[iTok]) { // Token found - split string - if (iChr > LastSplit) + if (iChr > LastSplit || KeepEmptyParts) Out.push_back(SubString(LastSplit, iChr - LastSplit)); LastSplit = iChr + 1; @@ -485,7 +485,7 @@ public: } // Add final string - if (LastSplit != Length()) + if (LastSplit != Length() || KeepEmptyParts) Out.push_back(SubString(LastSplit, Length() - LastSplit)); return Out; @@ -509,6 +509,14 @@ public: return (Size() == 0); } + bool StartsWith(CharType Chr, bool CaseSensitive = true) const + { + if (IsEmpty()) + return false; + + return CaseSensitive ? Front() == Chr : CharToUpper(Front()) == CharToUpper(Chr); + } + bool StartsWith(const _TString& rkStr, bool CaseSensitive = true) const { if (Size() < rkStr.Size()) @@ -518,6 +526,14 @@ public: return CaseSensitive ? SubStr == rkStr : SubStr.CaseInsensitiveCompare(rkStr); } + bool EndsWith(CharType Chr, bool CaseSensitive = true) const + { + if (IsEmpty()) + return false; + + return CaseSensitive ? Back() == Chr : CharToUpper(Back()) == CharToUpper(Chr); + } + bool EndsWith(const _TString& rkStr, bool CaseSensitive = true) const { if (Size() < rkStr.Size()) @@ -596,7 +612,7 @@ public: _TString GetFileDirectory() const { size_t EndPath = mInternalString.find_last_of(LITERAL("\\/")); - return SubString(0, EndPath + 1); + return EndPath == _TStdString::npos ? LITERAL("") : SubString(0, EndPath + 1); } _TString GetFileName(bool WithExtension = true) const @@ -618,13 +634,13 @@ public: _TString GetFileExtension() const { size_t EndName = mInternalString.find_last_of(LITERAL(".")); - return SubString(EndName + 1, Size() - EndName); + return EndName == _TStdString::npos ? LITERAL("") : SubString(EndName + 1, Size() - EndName); } _TString GetFilePathWithoutExtension() const { size_t EndName = mInternalString.find_last_of(LITERAL(".")); - return SubString(0, EndName); + return EndName == _TStdString::npos ? *this : SubString(0, EndName); } _TString GetParentDirectoryPath(const _TString& rkParentDirName, bool CaseSensitive = true) diff --git a/src/Core/GameProject/AssetNameGeneration.cpp b/src/Core/GameProject/AssetNameGeneration.cpp index 53a045d9..7598735d 100644 --- a/src/Core/GameProject/AssetNameGeneration.cpp +++ b/src/Core/GameProject/AssetNameGeneration.cpp @@ -183,18 +183,41 @@ void GenerateAssetNames(CGameProject *pProj) for (u32 iMat = 0; iMat < pMaterials->NumMaterials(); iMat++) { CMaterial *pMat = pMaterials->MaterialByIndex(iMat); + bool FoundLightmap = false; - if (pMat->Options().HasFlag(CMaterial::eLightmap)) + for (u32 iPass = 0; iPass < pMat->PassCount(); iPass++) { - CTexture *pLightmapTex = pMat->Pass(0)->Texture(); - CResourceEntry *pTexEntry = pLightmapTex->Entry(); - if (pTexEntry->IsCategorized()) continue; + CMaterialPass *pPass = pMat->Pass(iPass); - TWideString TexName = TWideString::Format(L"%s_lit_lightmap%d", *AreaName, LightmapNum); - ApplyGeneratedName(pTexEntry, AreaCookedDir, TexName); - pTexEntry->SetHidden(true); - LightmapNum++; + bool IsLightmap = ( (pArea->Game() <= eEchoes && pMat->Options().HasFlag(CMaterial::eLightmap) && iPass == 0) || + (pArea->Game() >= eCorruptionProto && pPass->Type() == "DIFF") ); + bool IsBloomLightmap = (pArea->Game() >= eCorruptionProto && pPass->Type() == "BLOL"); + + TWideString TexName; + + if (IsLightmap) + { + TexName = TWideString::Format(L"%s_lit_lightmap%d", *AreaName, LightmapNum); + } + else if (IsBloomLightmap) + { + TexName = TWideString::Format(L"%s_lit_lightmap_bloom%d", *AreaName, LightmapNum); + } + + if (!TexName.IsEmpty()) + { + CTexture *pLightmapTex = pPass->Texture(); + CResourceEntry *pTexEntry = pLightmapTex->Entry(); + if (pTexEntry->IsCategorized()) continue; + + ApplyGeneratedName(pTexEntry, AreaCookedDir, TexName); + pTexEntry->SetHidden(true); + FoundLightmap = true; + } } + + if (FoundLightmap) + LightmapNum++; } // Generate names from script instance names @@ -329,16 +352,24 @@ void GenerateAssetNames(CGameProject *pProj) { CMaterial *pMat = pSet->MaterialByIndex(iMat); - if (pMat->Options().HasFlag(CMaterial::eLightmap)) + for (u32 iPass = 0; iPass < pMat->PassCount(); iPass++) { - CTexture *pLightmapTex = pMat->Pass(0)->Texture(); - CResourceEntry *pTexEntry = pLightmapTex->Entry(); - if (pTexEntry->IsNamed() || pTexEntry->IsCategorized()) continue; + CMaterialPass *pPass = pMat->Pass(iPass); - TWideString TexName = TWideString::Format(L"%s_lightmap%d", *It->Name(), LightmapNum); - ApplyGeneratedName(pTexEntry, pModel->Entry()->DirectoryPath(), TexName); - pTexEntry->SetHidden(true); - LightmapNum++; + bool IsLightmap = ( (pMat->Version() <= eEchoes && pMat->Options().HasFlag(CMaterial::eLightmap) && iPass == 0) || + (pMat->Version() >= eCorruptionProto && pPass->Type() == "DIFF") ); + + if (IsLightmap) + { + CTexture *pLightmapTex = pPass->Texture(); + CResourceEntry *pTexEntry = pLightmapTex->Entry(); + if (pTexEntry->IsNamed() || pTexEntry->IsCategorized()) continue; + + TWideString TexName = TWideString::Format(L"%s_lightmap%d", *It->Name(), LightmapNum); + ApplyGeneratedName(pTexEntry, pModel->Entry()->DirectoryPath(), TexName); + pTexEntry->SetHidden(true); + LightmapNum++; + } } } } @@ -375,14 +406,27 @@ void GenerateAssetNames(CGameProject *pProj) CResourceEntry *pSample = pStore->FindEntry(SampleID); if (pSample && !pSample->IsNamed()) - ApplyGeneratedName(pSample, kSfxDir, TWideString::Format(L"%s_sample%d", *MacroName, iSamp)); + { + TWideString SampleName; + + if (pMacro->NumSamples() == 1) + SampleName = MacroName; + else + SampleName = TWideString::Format(L"%s_%d", *MacroName, iSamp); + + ApplyGeneratedName(pSample, kSfxDir, SampleName); + } } } #endif #if PROCESS_ANIM_CHAR_SETS // Generate animation format names - for (TResourceIterator It(pStore); It; ++It) + // Hacky syntax because animsets are under eAnimSet in MP1/2 and eCharacter in MP3/DKCR + CResourceIterator *pIter = (pProj->Game() <= eEchoes ? (CResourceIterator*) new TResourceIterator : (CResourceIterator*) new TResourceIterator); + CResourceIterator& It = *pIter; + + for (; It; ++It) { TWideString SetDir = It->DirectoryPath(); TWideString NewSetName; @@ -462,6 +506,7 @@ void GenerateAssetNames(CGameProject *pProj) } } } + delete pIter; #endif #if PROCESS_STRINGS diff --git a/src/Core/GameProject/CGameExporter.cpp b/src/Core/GameProject/CGameExporter.cpp index 2d41ba0a..5bcb462e 100644 --- a/src/Core/GameProject/CGameExporter.cpp +++ b/src/Core/GameProject/CGameExporter.cpp @@ -54,7 +54,6 @@ bool CGameExporter::Export(nod::DiscBase *pDisc, const TString& rkOutputDir, CAs // Create project mpProject = CGameProject::CreateProjectForExport( - this, mExportDir, mGame, mRegion, @@ -446,23 +445,13 @@ void CGameExporter::LoadResource(const SResourceInstance& rkResource, std::vecto void CGameExporter::ExportCookedResources() { - { - SCOPED_TIMER(ExportCookedResources); - FileUtil::MakeDirectory(mCookedDir); + SCOPED_TIMER(ExportCookedResources); + FileUtil::MakeDirectory(mCookedDir); - for (auto It = mResourceMap.begin(); It != mResourceMap.end(); It++) - { - SResourceInstance& rRes = It->second; - ExportResource(rRes); - } - } + for (auto It = mResourceMap.begin(); It != mResourceMap.end(); It++) { - SCOPED_TIMER(SaveResourceDatabase); -#if EXPORT_COOKED - mpStore->SaveResourceDatabase(); -#endif - bool SaveSuccess = mpProject->Save(); - ASSERT(SaveSuccess); + SResourceInstance& rRes = It->second; + ExportResource(rRes); } } @@ -512,8 +501,13 @@ void CGameExporter::ExportResourceEditorData() } } { - // All resources should have dependencies generated, so save the cache file - SCOPED_TIMER(SaveResourceCacheData); + // All resources should have dependencies generated, so save the project files + SCOPED_TIMER(SaveResourceDatabase); +#if EXPORT_COOKED + mpStore->SaveResourceDatabase(); +#endif + bool SaveSuccess = mpProject->Save(); + ASSERT(SaveSuccess); mpStore->SaveCacheFile(); } } diff --git a/src/Core/GameProject/CGameProject.cpp b/src/Core/GameProject/CGameProject.cpp index 0f9169ec..cc688daa 100644 --- a/src/Core/GameProject/CGameProject.cpp +++ b/src/Core/GameProject/CGameProject.cpp @@ -144,7 +144,6 @@ CAssetID CGameProject::FindNamedResource(const TString& rkName) const } CGameProject* CGameProject::CreateProjectForExport( - CGameExporter *pExporter, const TWideString& rkProjRootDir, EGame Game, ERegion Region, @@ -168,7 +167,7 @@ CGameProject* CGameProject::CreateProjectForExport( pProj->mProjectRoot = rkProjRootDir; pProj->mProjectRoot.Replace(L"/", L"\\"); - pProj->mpResourceStore = new CResourceStore(pProj, pExporter, L"Content\\", L"Cooked\\", Game); + pProj->mpResourceStore = new CResourceStore(pProj, L"Content\\", L"Cooked\\", Game); pProj->mpGameInfo->LoadGameInfo(Game); pProj->mLoadSuccess = true; return pProj; diff --git a/src/Core/GameProject/CGameProject.h b/src/Core/GameProject/CGameProject.h index 344c4972..852d26b1 100644 --- a/src/Core/GameProject/CGameProject.h +++ b/src/Core/GameProject/CGameProject.h @@ -70,7 +70,6 @@ public: // Static static CGameProject* CreateProjectForExport( - CGameExporter *pExporter, const TWideString& rkProjRootDir, EGame Game, ERegion Region, diff --git a/src/Core/GameProject/CResourceEntry.cpp b/src/Core/GameProject/CResourceEntry.cpp index 8165539d..0c70f648 100644 --- a/src/Core/GameProject/CResourceEntry.cpp +++ b/src/Core/GameProject/CResourceEntry.cpp @@ -356,8 +356,8 @@ bool CResourceEntry::CanMoveTo(const TWideString& rkDir, const TWideString& rkNa // Transient resources can't be moved if (IsTransient()) return false; - // Validate that the path/name are valid file paths - if (!FileUtil::IsValidPath(rkDir, true) || !FileUtil::IsValidName(rkName, false)) return false; + // Validate that the path/name are valid + if (!mpStore->IsValidResourcePath(rkDir, rkName)) return false; // We need to validate the path isn't taken already - either the directory doesn't exist, or doesn't have a resource by this name CVirtualDirectory *pDir = mpStore->GetVirtualDirectory(rkDir, false, false); diff --git a/src/Core/GameProject/CResourceStore.cpp b/src/Core/GameProject/CResourceStore.cpp index cbd879e8..96d4de91 100644 --- a/src/Core/GameProject/CResourceStore.cpp +++ b/src/Core/GameProject/CResourceStore.cpp @@ -18,7 +18,6 @@ CResourceStore *gpEditorStore = nullptr; CResourceStore::CResourceStore(const TWideString& rkDatabasePath) : mpProj(nullptr) , mGame(eUnknownGame) - , mpExporter(nullptr) , mDatabaseDirty(false) , mCacheFileDirty(false) { @@ -28,12 +27,11 @@ CResourceStore::CResourceStore(const TWideString& rkDatabasePath) } // Constructor for game exporter -CResourceStore::CResourceStore(CGameProject *pProject, CGameExporter *pExporter, const TWideString& rkRawDir, const TWideString& rkCookedDir, EGame Game) +CResourceStore::CResourceStore(CGameProject *pProject, const TWideString& rkRawDir, const TWideString& rkCookedDir, EGame Game) : mpProj(nullptr) , mGame(Game) , mRawDir(rkRawDir) , mCookedDir(rkCookedDir) - , mpExporter(pExporter) , mDatabaseDirty(false) , mCacheFileDirty(false) { @@ -45,7 +43,6 @@ CResourceStore::CResourceStore(CGameProject *pProject) : mpProj(nullptr) , mGame(eUnknownGame) , mpDatabaseRoot(nullptr) - , mpExporter(nullptr) , mDatabaseDirty(false) , mCacheFileDirty(false) { @@ -352,8 +349,8 @@ void CResourceStore::ConditionalDeleteDirectory(CVirtualDirectory *pDir) // If this directory is part of the project, then we should delete the corresponding filesystem directories if (pDir->GetRoot() == mpDatabaseRoot) { - FileUtil::DeleteDirectory(RawDir(false) + pDir->FullPath()); - FileUtil::DeleteDirectory(CookedDir(false) + pDir->FullPath()); + FileUtil::DeleteDirectory(RawDir(false) + pDir->FullPath(), true); + FileUtil::DeleteDirectory(CookedDir(false) + pDir->FullPath(), true); } CVirtualDirectory *pParent = pDir->Parent(); @@ -398,8 +395,15 @@ CResourceEntry* CResourceStore::RegisterResource(const CAssetID& rkID, EResType else { - pEntry = new CResourceEntry(this, rkID, rkDir, rkName, Type); - mResourceEntries[rkID] = pEntry; + // Validate directory + if (IsValidResourcePath(rkDir, rkName)) + { + pEntry = new CResourceEntry(this, rkID, rkDir, rkName, Type); + mResourceEntries[rkID] = pEntry; + } + + else + Log::Error("Invalid resource path, failed to register: " + rkDir.ToUTF8() + rkName.ToUTF8()); } return pEntry; @@ -434,50 +438,32 @@ CResource* CResourceStore::LoadResource(const CAssetID& rkID, const CFourCC& rkT if (Find != mLoadedResources.end()) return Find->second->Resource(); - // With Game Exporter - Get data buffer from exporter - if (mpExporter) - { - std::vector DataBuffer; - mpExporter->LoadResource(rkID, DataBuffer); - if (DataBuffer.empty()) return nullptr; + // Check for resource in store + CResourceEntry *pEntry = FindEntry(rkID); + if (pEntry) return pEntry->Load(); - CMemoryInStream MemStream(DataBuffer.data(), DataBuffer.size(), IOUtil::eBigEndian); - EResType Type = CResTypeInfo::TypeForCookedExtension(mGame, rkType)->Type(); - CResourceEntry *pEntry = RegisterTransientResource(Type, rkID); - CResource *pRes = pEntry->LoadCooked(MemStream); + // Check in transient load directory - this only works for cooked + EResType Type = CResTypeInfo::TypeForCookedExtension(mGame, rkType)->Type(); + + if (Type != eInvalidResType) + { + // Note the entry may not be able to find the resource on its own (due to not knowing what game + // it is) so we will attempt to open the file stream ourselves and pass it to the entry instead. + TString Name = rkID.ToString(); + CResourceEntry *pEntry = RegisterTransientResource(Type, mTransientLoadDir, Name.ToUTF16()); + + TString Path = mTransientLoadDir.ToUTF8() + Name + "." + rkType.ToString(); + CFileInStream File(Path.ToStdString(), IOUtil::eBigEndian); + CResource *pRes = pEntry->LoadCooked(File); + + if (!pRes) DeleteResourceEntry(pEntry); return pRes; } - // Without Game Exporter - Check store resource entries and transient load directory. else { - // Check for resource in store - CResourceEntry *pEntry = FindEntry(rkID); - if (pEntry) return pEntry->Load(); - - // Check in transient load directory - this only works for cooked - EResType Type = CResTypeInfo::TypeForCookedExtension(mGame, rkType)->Type(); - - if (Type != eInvalidResType) - { - // Note the entry may not be able to find the resource on its own (due to not knowing what game - // it is) so we will attempt to open the file stream ourselves and pass it to the entry instead. - TString Name = rkID.ToString(); - CResourceEntry *pEntry = RegisterTransientResource(Type, mTransientLoadDir, Name.ToUTF16()); - - TString Path = mTransientLoadDir.ToUTF8() + Name + "." + rkType.ToString(); - CFileInStream File(Path.ToStdString(), IOUtil::eBigEndian); - CResource *pRes = pEntry->LoadCooked(File); - - if (!pRes) DeleteResourceEntry(pEntry); - return pRes; - } - - else - { - Log::Error("Can't load requested resource with ID \"" + rkID.ToString() + "\"; can't locate resource. Note: Loading raw assets from an arbitrary directory is unsupported.");; - return nullptr; - } + Log::Error("Can't load requested resource with ID \"" + rkID.ToString() + "\"; can't locate resource. Note: Loading raw assets from an arbitrary directory is unsupported.");; + return nullptr; } } @@ -662,7 +648,7 @@ void CResourceStore::ImportNamesFromPakContentsTxt(const TString& rkTxtPath, boo u32 IDEnd = Line.IndexOf(" \t", IDStart); u32 PathStart = IDEnd + 1; - u32 PathEnd = Line.Size() - 4; + u32 PathEnd = Line.Size() - 5; TString IDStr = Line.SubString(IDStart, IDEnd - IDStart); TString Path = Line.SubString(PathStart, PathEnd - PathStart); @@ -679,8 +665,9 @@ void CResourceStore::ImportNamesFromPakContentsTxt(const TString& rkTxtPath, boo if (RepStart != -1) Path = Path.ChopFront(RepStart + 5); - // If the "x_rep" folder doesn't exist in this path for some reason, then just chop off the drive letter - else + // If the "x_rep" folder doesn't exist in this path for some reason, but this is still a path, then just chop off the drive letter. + // Otherwise, this is most likely just a standalone name, so use the full name as-is. + else if (Path[1] == ':') Path = Path.ChopFront(3); PathMap[pEntry] = Path; @@ -696,9 +683,23 @@ void CResourceStore::ImportNamesFromPakContentsTxt(const TString& rkTxtPath, boo if (UnnamedOnly && pEntry->IsNamed()) continue; TWideString Path = Iter->second.ToUTF16(); - pEntry->Move(Path.GetFileDirectory(), Path.GetFileName(false)); + TWideString Dir = Path.GetFileDirectory(); + TWideString Name = Path.GetFileName(false); + if (Dir.IsEmpty()) Dir = pEntry->DirectoryPath(); + + pEntry->Move(Dir, Name); } // Save ConditionalSaveStore(); } + +bool CResourceStore::IsValidResourcePath(const TWideString& rkPath, const TWideString& rkName) +{ + // Path must not be an absolute path and must not go outside the project structure. + // Name must not be a path. + return ( CVirtualDirectory::IsValidDirectoryPath(rkPath) && + FileUtil::IsValidName(rkName, false) && + !rkName.Contains(L'/') && + !rkName.Contains(L'\\') ); +} diff --git a/src/Core/GameProject/CResourceStore.h b/src/Core/GameProject/CResourceStore.h index 627cccbb..2b1c1ff9 100644 --- a/src/Core/GameProject/CResourceStore.h +++ b/src/Core/GameProject/CResourceStore.h @@ -35,9 +35,6 @@ class CResourceStore TWideString mCookedDir; TWideString mTransientLoadDir; - // Game exporter currently in use - lets us load from paks being exported - CGameExporter *mpExporter; - enum EDatabaseVersion { eVer_Initial, @@ -48,7 +45,7 @@ class CResourceStore public: CResourceStore(const TWideString& rkDatabasePath); - CResourceStore(CGameProject *pProject, CGameExporter *pExporter, const TWideString& rkRawDir, const TWideString& rkCookedDir, EGame Game); + CResourceStore(CGameProject *pProject, const TWideString& rkRawDir, const TWideString& rkCookedDir, EGame Game); CResourceStore(CGameProject *pProject); ~CResourceStore(); void SerializeResourceDatabase(IArchive& rArc); @@ -78,6 +75,8 @@ public: void ImportNamesFromPakContentsTxt(const TString& rkTxtPath, bool UnnamedOnly); + static bool IsValidResourcePath(const TWideString& rkPath, const TWideString& rkName); + // Accessors inline CGameProject* Project() const { return mpProj; } inline EGame Game() const { return mGame; } diff --git a/src/Core/GameProject/CVirtualDirectory.cpp b/src/Core/GameProject/CVirtualDirectory.cpp index f4825e21..cfbc31ba 100644 --- a/src/Core/GameProject/CVirtualDirectory.cpp +++ b/src/Core/GameProject/CVirtualDirectory.cpp @@ -10,11 +10,15 @@ CVirtualDirectory::CVirtualDirectory(CResourceStore *pStore) CVirtualDirectory::CVirtualDirectory(const TWideString& rkName, CResourceStore *pStore) : mpParent(nullptr), mName(rkName), mpStore(pStore) -{} +{ + ASSERT(!mName.IsEmpty() && FileUtil::IsValidName(mName, true)); +} CVirtualDirectory::CVirtualDirectory(CVirtualDirectory *pParent, const TWideString& rkName, CResourceStore *pStore) : mpParent(pParent), mName(rkName), mpStore(pStore) -{} +{ + ASSERT(!mName.IsEmpty() && FileUtil::IsValidName(mName, true)); +} CVirtualDirectory::~CVirtualDirectory() { @@ -34,7 +38,10 @@ bool CVirtualDirectory::IsEmpty() const TWideString CVirtualDirectory::FullPath() const { - return (mpParent && !mpParent->IsRoot() ? mpParent->FullPath() + mName + L'\\' : mName + L'\\'); + if (IsRoot()) + return L""; + else + return (mpParent && !mpParent->IsRoot() ? mpParent->FullPath() + mName + L'\\' : mName + L'\\'); } CVirtualDirectory* CVirtualDirectory::GetRoot() @@ -70,10 +77,12 @@ CVirtualDirectory* CVirtualDirectory::FindChildDirectory(const TWideString& rkNa if (AllowCreate) { - AddChild(rkName, nullptr); - CVirtualDirectory *pOut = FindChildDirectory(rkName, false); - ASSERT(pOut != nullptr); - return pOut; + if ( AddChild(rkName, nullptr) ) + { + CVirtualDirectory *pOut = FindChildDirectory(rkName, false); + ASSERT(pOut != nullptr); + return pOut; + } } return nullptr; @@ -111,21 +120,28 @@ CResourceEntry* CVirtualDirectory::FindChildResource(const TWideString& rkName, return nullptr; } -void CVirtualDirectory::AddChild(const TWideString &rkPath, CResourceEntry *pEntry) +bool CVirtualDirectory::AddChild(const TWideString &rkPath, CResourceEntry *pEntry) { if (rkPath.IsEmpty()) { if (pEntry) + { mResources.push_back(pEntry); + return true; + } + else + return false; } - else + else if (IsValidDirectoryPath(rkPath)) { u32 SlashIdx = rkPath.IndexOf(L"\\/"); TWideString DirName = (SlashIdx == -1 ? rkPath : rkPath.SubString(0, SlashIdx)); - CVirtualDirectory *pSubdir = nullptr; + TWideString Remaining = (SlashIdx == -1 ? L"" : rkPath.SubString(SlashIdx + 1, rkPath.Size() - SlashIdx)); // Check if this subdirectory already exists + CVirtualDirectory *pSubdir = nullptr; + for (u32 iSub = 0; iSub < mSubdirectories.size(); iSub++) { if (mSubdirectories[iSub]->Name() == DirName) @@ -137,17 +153,41 @@ void CVirtualDirectory::AddChild(const TWideString &rkPath, CResourceEntry *pEnt if (!pSubdir) { + // Create new subdirectory pSubdir = new CVirtualDirectory(this, DirName, mpStore); mSubdirectories.push_back(pSubdir); std::sort(mSubdirectories.begin(), mSubdirectories.end(), [](CVirtualDirectory *pLeft, CVirtualDirectory *pRight) -> bool { return (pLeft->Name().ToUpper() < pRight->Name().ToUpper()); }); + + // As an optimization, don't recurse here. We've already verified the full path is valid, so we don't need to do it again. + // We also know none of the remaining directories already exist because this is a new, empty directory. + TWideStringList Components = Remaining.Split(L"/\\"); + + for (auto Iter = Components.begin(); Iter != Components.end(); Iter++) + { + pSubdir = new CVirtualDirectory(pSubdir, *Iter, mpStore); + pSubdir->Parent()->mSubdirectories.push_back(pSubdir); + } + + if (pEntry) + pSubdir->mResources.push_back(pEntry); + + return true; } - TWideString Remaining = (SlashIdx == -1 ? L"" : rkPath.SubString(SlashIdx + 1, rkPath.Size() - SlashIdx)); - pSubdir->AddChild(Remaining, pEntry); + // If we have another valid child to add, return whether that operation completed successfully + else if (!Remaining.IsEmpty() || pEntry) + return pSubdir->AddChild(Remaining, pEntry); + + // Otherwise, we're done, so just return true + else + return true; } + + else + return false; } bool CVirtualDirectory::RemoveChildDirectory(CVirtualDirectory *pSubdir) @@ -180,3 +220,32 @@ bool CVirtualDirectory::RemoveChildResource(CResourceEntry *pEntry) return false; } + +// ************ STATIC ************ +bool CVirtualDirectory::IsValidDirectoryName(const TWideString& rkName) +{ + return ( rkName != L"." && + rkName != L".." && + FileUtil::IsValidName(rkName, true) ); +} + +bool CVirtualDirectory::IsValidDirectoryPath(TWideString Path) +{ + // Entirely empty path is valid - this refers to root + if (Path.IsEmpty()) + return true; + + // One trailing slash is allowed, but will cause IsValidName to fail, so we remove it here + if (Path.EndsWith(L'/') || Path.EndsWith(L'\\')) + Path = Path.ChopBack(1); + + TWideStringList Parts = Path.Split(L"/\\", true); + + for (auto Iter = Parts.begin(); Iter != Parts.end(); Iter++) + { + if (!IsValidDirectoryName(*Iter)) + return false; + } + + return true; +} diff --git a/src/Core/GameProject/CVirtualDirectory.h b/src/Core/GameProject/CVirtualDirectory.h index 8e48f541..750430b6 100644 --- a/src/Core/GameProject/CVirtualDirectory.h +++ b/src/Core/GameProject/CVirtualDirectory.h @@ -30,10 +30,13 @@ public: CVirtualDirectory* FindChildDirectory(const TWideString& rkName, bool AllowCreate); CResourceEntry* FindChildResource(const TWideString& rkPath); CResourceEntry* FindChildResource(const TWideString& rkName, EResType Type); - void AddChild(const TWideString& rkPath, CResourceEntry *pEntry); + bool AddChild(const TWideString& rkPath, CResourceEntry *pEntry); bool RemoveChildDirectory(CVirtualDirectory *pSubdir); bool RemoveChildResource(CResourceEntry *pEntry); + static bool IsValidDirectoryName(const TWideString& rkName); + static bool IsValidDirectoryPath(TWideString Path); + // Accessors inline CVirtualDirectory* Parent() const { return mpParent; } inline bool IsRoot() const { return !mpParent; } diff --git a/src/Core/Resource/Animation/CSourceAnimData.h b/src/Core/Resource/Animation/CSourceAnimData.h index a55bbc3a..1a3eab6b 100644 --- a/src/Core/Resource/Animation/CSourceAnimData.h +++ b/src/Core/Resource/Animation/CSourceAnimData.h @@ -52,6 +52,18 @@ public: return new CDependencyTree(); } + void GetUniquePrimitives(std::set& rPrimSet) const + { + for (u32 TransIdx = 0; TransIdx < mTransitions.size(); TransIdx++) + mTransitions[TransIdx].pTransition->GetUniquePrimitives(rPrimSet); + + for (u32 HalfIdx = 0; HalfIdx < mHalfTransitions.size(); HalfIdx++) + mHalfTransitions[HalfIdx].pTransition->GetUniquePrimitives(rPrimSet); + + if (mpDefaultTransition) + mpDefaultTransition->GetUniquePrimitives(rPrimSet); + } + void AddTransitionDependencies(CDependencyTree *pTree) { // Note: All CHAR animations must have been added to the tree before this function is run diff --git a/src/Core/Resource/CResTypeInfo.cpp b/src/Core/Resource/CResTypeInfo.cpp index 02a56709..3cc3eec9 100644 --- a/src/Core/Resource/CResTypeInfo.cpp +++ b/src/Core/Resource/CResTypeInfo.cpp @@ -174,6 +174,10 @@ void CResTypeInfo::CResTypeInfoFactory::InitTypes() CResTypeInfo *pType = new CResTypeInfo(eArea, "Area"); AddExtension(pType, "MREA", ePrimeDemo, eReturns); } + { + CResTypeInfo *pType = new CResTypeInfo(eAudioAmplitudeData, "Audio Amplitude Data"); + AddExtension(pType, "CAAD", eCorruption, eCorruption); + } { CResTypeInfo *pType = new CResTypeInfo(eAudioGroup, "Audio Group"); AddExtension(pType, "AGSC", ePrimeDemo, eEchoes); @@ -401,10 +405,6 @@ void CResTypeInfo::CResTypeInfoFactory::InitTypes() AddExtension(pType, "CTWK", ePrimeDemo, ePrime); pType->mCanHaveDependencies = false; } - { - CResTypeInfo *pType = new CResTypeInfo(eUnknown_CAAD, "CAAD"); - AddExtension(pType, "CAAD", eCorruption, eCorruption); - } { CResTypeInfo *pType = new CResTypeInfo(eUserEvaluatorData, "User Evaluator Data"); AddExtension(pType, "USRC", eCorruptionProto, eCorruption); diff --git a/src/Core/Resource/EResType.h b/src/Core/Resource/EResType.h index 5e61bb74..0ee52ed1 100644 --- a/src/Core/Resource/EResType.h +++ b/src/Core/Resource/EResType.h @@ -18,6 +18,7 @@ enum EResType eAreaSurfaceBounds, eAreaOctree, eAreaVisibilityTree, + eAudioAmplitudeData, eAudioGroup, eAudioMacro, eAudioSample, @@ -66,7 +67,6 @@ enum EResType eStringTable, eTexture, eTweak, - eUnknown_CAAD, eUserEvaluatorData, eVideo, eWorld, diff --git a/src/Core/Resource/Factory/CAnimSetLoader.cpp b/src/Core/Resource/Factory/CAnimSetLoader.cpp index 50bbf7c1..1e148742 100644 --- a/src/Core/Resource/Factory/CAnimSetLoader.cpp +++ b/src/Core/Resource/Factory/CAnimSetLoader.cpp @@ -92,6 +92,7 @@ CAnimSet* CAnimSetLoader::LoadCorruptionCHAR(IInputStream& rCHAR) rChar.SoundEffects.push_back(SoundID); } + ProcessPrimitives(); return pSet; } @@ -282,6 +283,14 @@ void CAnimSetLoader::ProcessPrimitives() for (u32 iTrans = 0; iTrans < pSet->mHalfTransitions.size(); iTrans++) pSet->mHalfTransitions[iTrans].pMetaTrans->GetUniquePrimitives(UniquePrimitives); + if (mGame == eCorruptionProto || mGame == eCorruption) + { + CSourceAnimData *pAnimData = (CSourceAnimData*) gpResourceStore->LoadResource( pSet->mCharacters[0].AnimDataID, "SAND" ); + + if (pAnimData) + pAnimData->GetUniquePrimitives(UniquePrimitives); + } + // Copy anim primitives into the animset for (auto Iter = UniquePrimitives.begin(); Iter != UniquePrimitives.end(); Iter++) { @@ -292,79 +301,82 @@ void CAnimSetLoader::ProcessPrimitives() pSet->mAnimPrimitives.resize(ID + 1); pSet->mAnimPrimitives[ID] = rkPrim; - ASSERT(pSet->Animation(ID)->pMetaAnim->Type() == eMAT_Play); } - // Add animations referenced by default transition - if (pSet->mpDefaultTransition) + // Add used animation indices from the animset to the character's list + if (mGame <= eEchoes) { - std::set DefaultTransPrimitives; - pSet->mpDefaultTransition->GetUniquePrimitives(DefaultTransPrimitives); + // Add animations referenced by default transition + if (pSet->mpDefaultTransition) + { + std::set DefaultTransPrimitives; + pSet->mpDefaultTransition->GetUniquePrimitives(DefaultTransPrimitives); + for (u32 iChar = 0; iChar < pSet->mCharacters.size(); iChar++) + { + SSetCharacter& rChar = pSet->mCharacters[iChar]; + + for (auto Iter = DefaultTransPrimitives.begin(); Iter != DefaultTransPrimitives.end(); Iter++) + { + const CAnimPrimitive& rkPrim = *Iter; + rChar.UsedAnimationIndices.insert(rkPrim.ID()); + } + } + } + + // Add animations referenced by used transitions for (u32 iChar = 0; iChar < pSet->mCharacters.size(); iChar++) { SSetCharacter& rChar = pSet->mCharacters[iChar]; + bool AddedNewAnims = true; - for (auto Iter = DefaultTransPrimitives.begin(); Iter != DefaultTransPrimitives.end(); Iter++) + // Loop this until we run out of new animations. This is in case any animations + // referenced by any transitions are also referenced by earlier transitions. + while (AddedNewAnims) { - const CAnimPrimitive& rkPrim = *Iter; - rChar.UsedAnimationIndices.insert(rkPrim.ID()); - } - } - } + AddedNewAnims = false; - // Add animations referenced by used transitions - for (u32 iChar = 0; iChar < pSet->mCharacters.size(); iChar++) - { - SSetCharacter& rChar = pSet->mCharacters[iChar]; - bool AddedNewAnims = true; - - // Loop this until we run out of new animations. This is in case any animations - // referenced by any transitions are also referenced by earlier transitions. - while (AddedNewAnims) - { - AddedNewAnims = false; - - for (u32 iTrans = 0; iTrans < pSet->mTransitions.size(); iTrans++) - { - STransition& rTrans = pSet->mTransitions[iTrans]; - - if ( rChar.UsedAnimationIndices.find(rTrans.AnimIdA) != rChar.UsedAnimationIndices.end() && - rChar.UsedAnimationIndices.find(rTrans.AnimIdB) != rChar.UsedAnimationIndices.end() ) + for (u32 iTrans = 0; iTrans < pSet->mTransitions.size(); iTrans++) { - std::set Primitives; - rTrans.pMetaTrans->GetUniquePrimitives(Primitives); + STransition& rTrans = pSet->mTransitions[iTrans]; - for (auto Iter = Primitives.begin(); Iter != Primitives.end(); Iter++) + if ( rChar.UsedAnimationIndices.find(rTrans.AnimIdA) != rChar.UsedAnimationIndices.end() && + rChar.UsedAnimationIndices.find(rTrans.AnimIdB) != rChar.UsedAnimationIndices.end() ) { - const CAnimPrimitive& rkPrim = *Iter; + std::set Primitives; + rTrans.pMetaTrans->GetUniquePrimitives(Primitives); - if (rChar.UsedAnimationIndices.find(rkPrim.ID()) == rChar.UsedAnimationIndices.end()) + for (auto Iter = Primitives.begin(); Iter != Primitives.end(); Iter++) { - rChar.UsedAnimationIndices.insert(rkPrim.ID()); - AddedNewAnims = true; + const CAnimPrimitive& rkPrim = *Iter; + + if (rChar.UsedAnimationIndices.find(rkPrim.ID()) == rChar.UsedAnimationIndices.end()) + { + rChar.UsedAnimationIndices.insert(rkPrim.ID()); + AddedNewAnims = true; + } } } } - } - for (u32 iHalf = 0; iHalf < pSet->mHalfTransitions.size(); iHalf++) - { - SHalfTransition& rTrans = pSet->mHalfTransitions[iHalf]; - - if (rChar.UsedAnimationIndices.find(rTrans.AnimID) != rChar.UsedAnimationIndices.end()) + for (u32 iHalf = 0; iHalf < pSet->mHalfTransitions.size(); iHalf++) { - std::set Primitives; - rTrans.pMetaTrans->GetUniquePrimitives(Primitives); + SHalfTransition& rTrans = pSet->mHalfTransitions[iHalf]; - for (auto Iter = Primitives.begin(); Iter != Primitives.end(); Iter++) + if (rChar.UsedAnimationIndices.find(rTrans.AnimID) != rChar.UsedAnimationIndices.end()) { - const CAnimPrimitive& rkPrim = *Iter; + std::set Primitives; + rTrans.pMetaTrans->GetUniquePrimitives(Primitives); - if (rChar.UsedAnimationIndices.find(rkPrim.ID()) == rChar.UsedAnimationIndices.end()) + for (auto Iter = Primitives.begin(); Iter != Primitives.end(); Iter++) { - rChar.UsedAnimationIndices.insert(rkPrim.ID()); - AddedNewAnims = true; + const CAnimPrimitive& rkPrim = *Iter; + + if (rChar.UsedAnimationIndices.find(rkPrim.ID()) == rChar.UsedAnimationIndices.end()) + { + rChar.UsedAnimationIndices.insert(rkPrim.ID()); + AddedNewAnims = true; + } } } }