AxioDL
/
PrimeWorldEditor
mirror of https://github.com/AxioDL/PrimeWorldEditor.git
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Initial implementation of world/area exports, added support for asset lists to specify a path/name for resources

This commit is contained in:
parax0 2016-06-05 01:57:31 -06:00
parent f15aca3f99
commit 8293f1d206
22 changed files with 630 additions and 99 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
resources/list/AssetListDKCR.xml Normal file
View File

@ -0,0 +1,2 @@
<AssetList>
</AssetList>

6
resources/list/AssetListMP1.xml Normal file
View File

@ -0,0 +1,6 @@
<AssetList>
<Asset ID="0x2782648E">
<Dir>Samus\</Dir>
<Name>PhazonSuit</Name>
</Asset>
</AssetList>

2
resources/list/AssetListMP1Demo.xml Normal file
View File

@ -0,0 +1,2 @@
<AssetList>
</AssetList>

2
resources/list/AssetListMP2.xml Normal file
View File

@ -0,0 +1,2 @@
<AssetList>
</AssetList>

2
resources/list/AssetListMP2Demo.xml Normal file
View File

@ -0,0 +1,2 @@
<AssetList>
</AssetList>

2
resources/list/AssetListMP3.xml Normal file
View File

@ -0,0 +1,2 @@
<AssetList>
</AssetList>

2
resources/list/AssetListMP3Proto.xml Normal file
View File

@ -0,0 +1,2 @@
<AssetList>
</AssetList>

6
src/Common/CUniqueID.cpp
View File

@ -13,13 +13,13 @@ using IOUtil::eBigEndian;
CUniqueID::CUniqueID()
: mLength(eInvalidUIDLength)
{
memset(mID, 0xFF, 16);
memset(mID, 0, 16);
}
CUniqueID::CUniqueID(u64 ID)
{
// This constructor is intended to be used with both 32-bit and 64-bit input values
memset(mID, 0xFF, 16);
memset(mID, 0, 16);
// 64-bit - check for valid content in upper 32 bits (at least one bit set + one bit unset)
if ((ID & 0xFFFFFFFF00000000) && (~ID & 0xFFFFFFFF00000000))
@ -43,7 +43,7 @@ CUniqueID::CUniqueID(u64 ID)
CUniqueID::CUniqueID(u64 ID, EUIDLength Length)
{
// This constructor shouldn't be used for 128-bit
memset(mID, 0xFF, 16);
memset(mID, 0, 16);
// 64-bit
if (Length == e64Bit || Length == e128Bit)

173
src/Common/FileUtil.cpp
View File

@ -16,7 +16,7 @@ bool Exists(const TWideString &rkFilePath)
bool IsRoot(const TWideString& rkPath)
{
// todo: verify that this is actually a good way of checking for root
// todo: is this actually a good way of checking for root?
TWideString AbsPath = MakeAbsolute(rkPath);
TWideStringList Split = AbsPath.Split(L"\\/");
return (Split.size() <= 1);
@ -34,11 +34,23 @@ bool IsDirectory(const TWideString& rkDirPath)
bool CreateDirectory(const TWideString& rkNewDir)
{
if (!IsValidPath(rkNewDir, true))
{
Log::Error("Unable to create directory because name contains illegal characters: " + rkNewDir.ToUTF8());
return false;
}
return create_directories(*rkNewDir);
}
bool CopyFile(const TWideString& rkOrigPath, const TWideString& rkNewPath)
{
if (!IsValidPath(rkNewPath, false))
{
Log::Error("Unable to copy file because destination name contains illegal characters: " + rkNewPath.ToUTF8());
return false;
}
boost::system::error_code Error;
copy(*rkOrigPath, *rkNewPath, Error);
return (Error == boost::system::errc::success);
@ -46,6 +58,12 @@ bool CopyFile(const TWideString& rkOrigPath, const TWideString& rkNewPath)
bool CopyDirectory(const TWideString& rkOrigPath, const TWideString& rkNewPath)
{
if (!IsValidPath(rkNewPath, true))
{
Log::Error("Unable to copy directory because destination name contains illegal characters: " + rkNewPath.ToUTF8());
return false;
}
boost::system::error_code Error;
copy_directory(*rkOrigPath, *rkNewPath, Error);
return (Error == boost::system::errc::success);
@ -53,6 +71,12 @@ bool CopyDirectory(const TWideString& rkOrigPath, const TWideString& rkNewPath)
bool MoveFile(const TWideString& rkOldPath, const TWideString& rkNewPath)
{
if (!IsValidPath(rkNewPath, false))
{
Log::Error("Unable to move file because destination name contains illegal characters: " + rkNewPath.ToUTF8());
return false;
}
if (CopyFile(rkOldPath, rkNewPath))
return DeleteFile(rkOldPath);
else
@ -61,6 +85,12 @@ bool MoveFile(const TWideString& rkOldPath, const TWideString& rkNewPath)
bool MoveDirectory(const TWideString& rkOldPath, const TWideString& rkNewPath)
{
if (!IsValidPath(rkNewPath, true))
{
Log::Error("Unable to move directory because destination name contains illegal characters: " + rkNewPath.ToUTF8());
return false;
}
if (CopyDirectory(rkOldPath, rkNewPath))
return DeleteDirectory(rkOldPath);
else
@ -201,6 +231,147 @@ TWideString MakeRelative(const TWideString& rkPath, const TWideString& rkRelativ
return Out;
}
static const wchar_t gskIllegalNameChars[] = {
L'<', L'>', L'\"', L'/', L'\\', L'|', L'?', L'*'
};
TWideString SanitizeName(TWideString Name, bool Directory, bool RootDir /*= false*/)
{
// Windows only atm
if (Directory && (Name == L"." || Name == L".."))
return Name;
// Remove illegal characters from path
u32 NumIllegalChars = sizeof(gskIllegalNameChars) / sizeof(wchar_t);
for (u32 iChr = 0; iChr < Name.Size(); iChr++)
{
wchar_t Chr = Name[iChr];
bool Remove = false;
if (Chr >= 0 && Chr <= 31)
Remove = true;
// For root, allow colon only as the last character of the name
else if (Chr == L':' && (!RootDir || iChr != Name.Size() - 1))
Remove = true;
else
{
for (u32 iBan = 0; iBan < NumIllegalChars; iBan++)
{
if (Chr == gskIllegalNameChars[iBan])
{
Remove = true;
break;
}
}
}
if (Remove)
{
Name.Remove(iChr, 1);
iChr--;
}
}
// For directories, space and dot are not allowed at the end of the path
if (Directory)
{
u32 ChopNum = 0;
for (int iChr = (int) Name.Size() - 1; iChr >= 0; iChr--)
{
wchar_t Chr = Name[iChr];
if (Chr == L' ' || Chr == L'.')
ChopNum++;
else
break;
}
if (ChopNum > 0) Name = Name.ChopBack(ChopNum);
}
return Name;
}
TWideString SanitizePath(TWideString Path, bool Directory)
{
TWideStringList Components = Path.Split(L"\\/");
u32 CompIdx = 0;
Path = L"";
for (auto It = Components.begin(); It != Components.end(); It++)
{
TWideString Comp = *It;
bool IsDir = Directory || CompIdx < Components.size() - 1;
bool IsRoot = CompIdx == 0;
SanitizeName(Comp, IsDir, IsRoot);
Path += Comp;
if (IsDir) Path += L'\\';
CompIdx++;
}
return Path;
}
bool IsValidName(const TWideString& rkName, bool Directory, bool RootDir /*= false*/)
{
// Windows only atm
u32 NumIllegalChars = sizeof(gskIllegalNameChars) / sizeof(wchar_t);
if (Directory && (rkName == L"." || rkName == L".."))
return true;
// Check for banned characters
for (u32 iChr = 0; iChr < rkName.Size(); iChr++)
{
wchar_t Chr = rkName[iChr];
if (Chr >= 0 && Chr <= 31)
return false;
// Allow colon only on last character of root
if (Chr == L':' && (!RootDir || iChr != rkName.Size() - 1))
return false;
for (u32 iBan = 0; iBan < NumIllegalChars; iBan++)
{
if (Chr == gskIllegalNameChars[iBan])
return false;
}
}
if (Directory && (rkName.Back() == L' ' || rkName.Back() == L'.'))
return false;
return true;
}
bool IsValidPath(const TWideString& rkPath, bool Directory)
{
// Windows only atm
TWideStringList Components = rkPath.Split(L"\\/");
// Validate other components
u32 CompIdx = 0;
for (auto It = Components.begin(); It != Components.end(); It++)
{
bool IsRoot = CompIdx == 0;
bool IsDir = Directory || CompIdx < (Components.size() - 1);
if (!IsValidName(*It, IsDir, IsRoot))
return false;
CompIdx++;
}
return true;
}
void GetDirectoryContents(TWideString DirPath, TWideStringList& rOut, bool Recursive /*= true*/, bool IncludeFiles /*= true*/, bool IncludeDirs /*= true*/)
{
if (IsDirectory(DirPath))

4
src/Common/FileUtil.h
View File

@ -24,6 +24,10 @@ u64 LastModifiedTime(const TWideString& rkFilePath);
TWideString WorkingDirectory();
TWideString MakeAbsolute(TWideString Path);
TWideString MakeRelative(const TWideString& rkPath, const TWideString& rkRelativeTo = WorkingDirectory());
TWideString SanitizeName(TWideString Name, bool Directory, bool RootDir = false);
TWideString SanitizePath(TWideString Path, bool Directory);
bool IsValidName(const TWideString& rkName, bool Directory, bool RootDir = false);
bool IsValidPath(const TWideString& rkPath, bool Directory);
void GetDirectoryContents(TWideString DirPath, TWideStringList& rOut, bool Recursive = true, bool IncludeFiles = true, bool IncludeDirs = true);
}

80
src/Common/TString.h
View File

@ -29,8 +29,9 @@
* be encoded in UTF-16.
*/
// Helper macro for creating string literals of the correct char type. Internal use only! Invalid outside of this header!
// Helper macros for creating string literals of the correct char type. Internal use only! Invalid outside of this header!
#define LITERAL(Text) (typeid(CharType) == typeid(char)) ? (const CharType*) ##Text : (const CharType*) L##Text
#define CHAR_LITERAL(Text) (CharType) Text
// ************ TBasicString ************
template<class CharType>
@ -116,14 +117,21 @@ public:
return Size();
}
inline u32 IndexOf(CharType Character, u32 Offset) const
{
size_t Pos = mInternalString.find_first_of(Character, Offset);
return (Pos == _TStdString::npos ? -1 : (u32) Pos);
}
inline u32 IndexOf(CharType Character) const
{
return IndexOf(Character, 0);
}
inline u32 IndexOf(const CharType* pkCharacters, u32 Offset) const
{
size_t Pos = mInternalString.find_first_of(pkCharacters, Offset);
if (Pos == _TStdString::npos)
return -1;
else
return (u32) Pos;
return (Pos == _TStdString::npos ? -1 : (u32) Pos);
}
inline u32 IndexOf(const CharType* pkCharacters) const
@ -233,6 +241,20 @@ public:
mInternalString.erase(Pos, Len);
}
inline void Remove(const CharType* pkStr, bool CaseSensitive = false)
{
u32 InStrLen = CStringLength(pkStr);
for (u32 Idx = IndexOfPhrase(pkStr, CaseSensitive); Idx != -1; Idx = IndexOfPhrase(pkStr, Idx, CaseSensitive))
Remove(Idx, InStrLen);
}
inline void Remove(CharType Chr)
{
for (u32 Idx = IndexOf(Chr); Idx != -1; Idx = IndexOf(Chr, Idx))
Remove(Idx, 1);
}
inline void Replace(const CharType* pkStr, const CharType *pkReplacement, bool CaseSensitive = false)
{
u32 Offset = 0;
@ -581,6 +603,29 @@ public:
return SubString(0, EndName);
}
_TString GetParentDirectoryPath(_TString ParentDirName, bool CaseSensitive = true)
{
if (!CaseSensitive) ParentDirName = ParentDirName.ToUpper();
int IdxA = 0;
int IdxB = IndexOf(LITERAL("\\/"));
if (IdxB == -1) return _TString();
while (IdxB != -1)
{
_TString DirName = SubString(IdxA, IdxB - IdxA);
if (!CaseSensitive) DirName = DirName.ToUpper();
if (DirName == ParentDirName)
return Truncate(IdxB + 1);
IdxA = IdxB + 1;
IdxB = IndexOf(LITERAL("\\/"), IdxA);
}
return _TString();
}
// Operators
inline _TString& operator=(const CharType* pkText)
{
@ -788,23 +833,23 @@ public:
static TBasicString<CharType> FromInt32(s32 Value, int Width = 0, int Base = 16)
{
std::basic_stringstream<CharType> sstream;
sstream << std::setbase(Base) << std::setw(Width) << std::setfill('0') << Value;
sstream << std::setbase(Base) << std::setw(Width) << std::setfill(CHAR_LITERAL('0')) << Value;
return sstream.str();
}
static TBasicString<CharType> FromInt64(s64 Value, int Width = 0, int Base = 16)
{
std::basic_stringstream<CharType> sstream;
sstream << std::setbase(Base) << std::setw(Width) << std::setfill('0') << Value;
sstream << std::setbase(Base) << std::setw(Width) << std::setfill(CHAR_LITERAL('0')) << Value;
return sstream.str();
}
static TBasicString<CharType> FromFloat(float Value, int MinDecimals = 1)
{
TString Out = std::to_string(Value);
int NumZeroes = Out.Size() - (Out.IndexOf(".") + 1);
int NumZeroes = Out.Size() - (Out.IndexOf(LITERAL(".")) + 1);
while (Out.Back() == '0' && NumZeroes > MinDecimals)
while (Out.Back() == CHAR_LITERAL('0') && NumZeroes > MinDecimals)
{
Out = Out.ChopBack(1);
NumZeroes--;
@ -830,7 +875,7 @@ public:
_TString str = sstream.str();
if (Uppercase) str = str.ToUpper();
if (AddPrefix) str.Prepend("0x");
if (AddPrefix) str.Prepend(LITERAL("0x"));
return str;
}
@ -861,16 +906,17 @@ public:
static bool IsWhitespace(CharType c)
{
return ( (c == '\t') ||
(c == '\n') ||
(c == '\v') ||
(c == '\f') ||
(c == '\r') ||
(c == ' ') );
return ( (c == CHAR_LITERAL('\t')) ||
(c == CHAR_LITERAL('\n')) ||
(c == CHAR_LITERAL('\v')) ||
(c == CHAR_LITERAL('\f')) ||
(c == CHAR_LITERAL('\r')) ||
(c == CHAR_LITERAL(' ')) );
}
};
#undef LITERAL
#undef CHAR_LITERAL
// ************ TString ************
class TString : public TBasicString<char>

237
src/Core/GameProject/CGameExporter.cpp
View File

@ -1,12 +1,16 @@
#include "CGameExporter.h"
#include "Core/Resource/CResCache.h"
#include "Core/Resource/CWorld.h"
#include <FileIO/FileIO.h>
#include <Common/AssertMacro.h>
#include <Common/CompressionUtil.h>
#include <Common/CScopedTimer.h>
#include <Common/FileUtil.h>
#include <tinyxml2.h>
#define COPY_DISC_DATA 1
#define COPY_DISC_DATA 0
#define LOAD_PAKS 1
#define EXPORT_WORLDS 1
#define EXPORT_COOKED 1
CGameExporter::CGameExporter(const TString& rkInputDir, const TString& rkOutputDir)
@ -15,25 +19,37 @@ CGameExporter::CGameExporter(const TString& rkInputDir, const TString& rkOutputD
mExportDir = FileUtil::MakeAbsolute(rkOutputDir);
mpProject = new CGameProject(mExportDir);
mDiscDir = mpProject->DiscDir();
mResDir = mpProject->ResourcesDir();
mWorldsDir = mpProject->WorldsDir();
mCookedDir = mpProject->CookedDir();
mCookedResDir = mpProject->CookedResourcesDir();
mCookedWorldsDir = mpProject->CookedWorldsDir();
mDiscDir = mpProject->DiscDir(true);
mResDir = mpProject->ResourcesDir(true);
mWorldsDir = mpProject->WorldsDir(true);
mCookedDir = mpProject->CookedDir(false);
mCookedResDir = mpProject->CookedResourcesDir(true);
mCookedWorldsDir = mpProject->CookedWorldsDir(true);
}
bool CGameExporter::Export()
{
SCOPED_TIMER(ExportGame);
gResCache.SetGameExporter(this);
FileUtil::CreateDirectory(mExportDir);
FileUtil::ClearDirectory(mExportDir);
CopyDiscData();
LoadAssetList();
LoadPaks();
ExportWorlds();
ExportCookedResources();
gResCache.SetGameExporter(nullptr);
return true;
}
void CGameExporter::LoadResource(const CUniqueID& rkID, std::vector<u8>& rBuffer)
{
SResourceInstance *pInst = FindResourceInstance(rkID);
if (pInst) LoadResource(*pInst, rBuffer);
}
// ************ PROTECTED ************
void CGameExporter::CopyDiscData()
{
@ -93,6 +109,57 @@ void CGameExporter::CopyDiscData()
ASSERT(Game() != eUnknownVersion);
}
void CGameExporter::LoadAssetList()
{
SCOPED_TIMER(LoadAssetList);
// Determine the asset list to use
TString ListFile = "../resources/list/AssetList";
switch (Game())
{
case ePrimeDemo: ListFile += "MP1Demo"; break;
case ePrime: ListFile += "MP1"; break;
case eEchoesDemo: ListFile += "MP2Demo"; break;
case eEchoes: ListFile += "MP2"; break;
case eCorruptionProto: ListFile += "MP3Proto"; break;
case eCorruption: ListFile += "MP3"; break;
case eReturns: ListFile += "DKCR"; break;
default: ASSERT(false);
}
ListFile += ".xml";
// Load list
tinyxml2::XMLDocument List;
List.LoadFile(*ListFile);
if (List.Error())
{
Log::Error("Couldn't open asset list: " + ListFile);
return;
}
tinyxml2::XMLElement *pRoot = List.FirstChildElement("AssetList");
tinyxml2::XMLElement *pAsset = pRoot->FirstChildElement("Asset");
while (pAsset)
{
u64 ResourceID = TString(pAsset->Attribute("ID")).ToInt64(16);
tinyxml2::XMLElement *pDir = pAsset->FirstChildElement("Dir");
TString Dir = pDir ? pDir->GetText() : "";
tinyxml2::XMLElement *pName = pAsset->FirstChildElement("Name");
TString Name = pName ? pName->GetText() : "";
if (!Dir.EndsWith("/") && !Dir.EndsWith("\\")) Dir.Append("\\");
SetResourcePath(ResourceID, mResDir + Dir.ToUTF16(), Name.ToUTF16());
pAsset = pAsset->NextSiblingElement("Asset");
}
}
// ************ RESOURCE LOADING ************
void CGameExporter::LoadPaks()
{
@ -117,7 +184,7 @@ void CGameExporter::LoadPaks()
continue;
}
CPackage *pPackage = new CPackage(CharPak.GetFileName(false));
CPackage *pPackage = new CPackage(CharPak.GetFileName(false), FileUtil::MakeRelative(PakPath.GetFileDirectory(), mExportDir));
// MP1-MP3Proto
if (Game() < eCorruption)
@ -134,11 +201,11 @@ void CGameExporter::LoadPaks()
for (u32 iName = 0; iName < NumNamedResources; iName++)
{
Pak.Seek(0x4, SEEK_CUR); // Skip resource type
CFourCC ResType = Pak.ReadLong();
CUniqueID ResID(Pak, IDLength);
u32 NameLen = Pak.ReadLong();
TString Name = Pak.ReadString(NameLen);
pPackage->AddNamedResource(Name, ResID);
pPackage->AddNamedResource(Name, ResID, ResType);
}
u32 NumResources = Pak.ReadLong();
@ -153,7 +220,7 @@ void CGameExporter::LoadPaks()
u64 IntegralID = ResID.ToLongLong();
if (mResourceMap.find(IntegralID) == mResourceMap.end())
mResourceMap[IntegralID] = SResourceInstance { PakPath, ResID, ResType, ResOffset, ResSize, Compressed };
mResourceMap[IntegralID] = SResourceInstance { PakPath, ResID, ResType, ResOffset, ResSize, Compressed, false };
}
}
}
@ -194,9 +261,9 @@ void CGameExporter::LoadPaks()
for (u32 iName = 0; iName < NumNamedResources; iName++)
{
TString Name = Pak.ReadString();
Pak.Seek(0x4, SEEK_CUR); // Skip type
CFourCC ResType = Pak.ReadLong();
CUniqueID ResID(Pak, IDLength);
pPackage->AddNamedResource(Name, ResID);
pPackage->AddNamedResource(Name, ResID, ResType);
}
}
@ -216,7 +283,7 @@ void CGameExporter::LoadPaks()
u64 IntegralID = ResID.ToLongLong();
if (mResourceMap.find(IntegralID) == mResourceMap.end())
mResourceMap[IntegralID] = SResourceInstance { PakPath, ResID, Type, Offset, Size, Compressed };
mResourceMap[IntegralID] = SResourceInstance { PakPath, ResID, Type, Offset, Size, Compressed, false };
}
}
@ -231,7 +298,7 @@ void CGameExporter::LoadPaks()
#endif
}
void CGameExporter::LoadPakResource(const SResourceInstance& rkResource, std::vector<u8>& rBuffer)
void CGameExporter::LoadResource(const SResourceInstance& rkResource, std::vector<u8>& rBuffer)
{
CFileInStream Pak(rkResource.PakFile.ToUTF8().ToStdString(), IOUtil::eBigEndian);
@ -327,30 +394,106 @@ void CGameExporter::LoadPakResource(const SResourceInstance& rkResource, std::ve
}
}
void CGameExporter::ExportWorlds()
{
#if EXPORT_WORLDS
SCOPED_TIMER(ExportWorlds);
//CResourceDatabase *pResDB = mpProject->ResourceDatabase();
for (u32 iPak = 0; iPak < mpProject->NumWorldPaks(); iPak++)
{
CPackage *pPak = mpProject->WorldPakByIndex(iPak);
// Get output path. DKCR paks are stored in a Worlds folder so we should get the path relative to that so we don't have Worlds\Worlds\.
// Other games have all paks in the game root dir so we're fine just taking the original root dir-relative directory.
TWideString PakPath = pPak->PakPath();
TWideString WorldsDir = PakPath.GetParentDirectoryPath(L"Worlds", false);
if (!WorldsDir.IsEmpty())
PakPath = FileUtil::MakeRelative(PakPath, WorldsDir);
for (u32 iRes = 0; iRes < pPak->NumNamedResources(); iRes++)
{
const SNamedResource& rkRes = pPak->NamedResourceByIndex(iRes);
if (rkRes.Type == "MLVL" && !rkRes.Name.EndsWith("NODEPEND"))
{
TResPtr<CWorld> pWorld = (CWorld*) gResCache.GetResource(rkRes.ID, rkRes.Type);
if (!pWorld)
{
Log::Error("Couldn't load world " + rkRes.Name + " from package " + pPak->PakName() + "; unable to export");
continue;
}
// Export world
TWideString Name = rkRes.Name.ToUTF16();
TWideString WorldDir = mWorldsDir + PakPath + FileUtil::SanitizeName(Name, true) + L"\\";
FileUtil::CreateDirectory(mCookedDir + WorldDir);
SResourceInstance *pInst = FindResourceInstance(rkRes.ID);
ASSERT(pInst != nullptr);
SetResourcePath(rkRes.ID, WorldDir, Name);
ExportResource(*pInst);
// Export areas
for (u32 iArea = 0; iArea < pWorld->NumAreas(); iArea++)
{
// Determine area names
TWideString InternalAreaName = pWorld->AreaInternalName(iArea).ToUTF16();
if (InternalAreaName.IsEmpty()) InternalAreaName = TWideString::FromInt32(iArea, 2, 10);
TWideString GameAreaName;
CStringTable *pTable = pWorld->AreaName(iArea);
if (pTable) GameAreaName = pTable->String("ENGL", 0);
if (GameAreaName.IsEmpty()) GameAreaName = InternalAreaName;
// Load area
CUniqueID AreaID = pWorld->AreaResourceID(iArea);
CGameArea *pArea = (CGameArea*) gResCache.GetResource(AreaID, "MREA");
if (!pArea)
{
Log::Error("Unable to export area " + GameAreaName.ToUTF8() + " from world " + rkRes.Name + "; couldn't load area");
continue;
}
// Export area
TWideString AreaDir = WorldDir + TWideString::FromInt32(iArea, 2, 10) + L"_" + FileUtil::SanitizeName(GameAreaName, true) + L"\\";
FileUtil::CreateDirectory(mCookedDir + AreaDir);
SResourceInstance *pInst = FindResourceInstance(AreaID);
ASSERT(pInst != nullptr);
SetResourcePath(AreaID, AreaDir, InternalAreaName);
ExportResource(*pInst);
}
gResCache.Clean();
}
else
{
Log::Error("Unexpected named resource type in world pak: " + rkRes.Type.ToString());
}
}
}
#endif
}
void CGameExporter::ExportCookedResources()
{
#if EXPORT_COOKED
CResourceDatabase *pResDB = mpProject->ResourceDatabase();
{
SCOPED_TIMER(ExportCookedResources);
FileUtil::CreateDirectory(mCookedResDir);
FileUtil::CreateDirectory(mCookedDir + mResDir);
for (auto It = mResourceMap.begin(); It != mResourceMap.end(); It++)
{
const SResourceInstance& rkRes = It->second;
std::vector<u8> ResourceData;
LoadPakResource(rkRes, ResourceData);
TString OutName = rkRes.ResourceID.ToString() + "." + rkRes.ResourceType.ToString();
TString OutDir = mCookedResDir.ToUTF8() + "\\";
TString OutPath = OutDir + OutName;
CFileOutStream Out(OutPath.ToStdString(), IOUtil::eBigEndian);
if (Out.IsValid())
Out.WriteBytes(ResourceData.data(), ResourceData.size());
// Add to resource DB
pResDB->RegisterResource(rkRes.ResourceID, FileUtil::MakeRelative(OutDir, mCookedDir), OutName, CResource::ResTypeForExtension(rkRes.ResourceType));
SResourceInstance& rRes = It->second;
ExportResource(rRes);
}
}
{
@ -359,3 +502,37 @@ void CGameExporter::ExportCookedResources()
}
#endif
}
void CGameExporter::ExportResource(SResourceInstance& rRes)
{
if (!rRes.Exported)
{
std::vector<u8> ResourceData;
LoadResource(rRes, ResourceData);
// Determine output path
SResourcePath *pPath = FindResourcePath(rRes.ResourceID);
TString OutName, OutDir;
if (pPath)
{
OutName = pPath->Name.ToUTF8();
OutDir = pPath->Dir.ToUTF8();
}
if (OutName.IsEmpty()) OutName = rRes.ResourceID.ToString();
if (OutDir.IsEmpty()) OutDir = mResDir;
// Write to file
FileUtil::CreateDirectory(mCookedDir + OutDir.ToUTF16());
TString OutPath = mCookedDir.ToUTF8() + OutDir + OutName + "." + rRes.ResourceType.ToString();
CFileOutStream Out(OutPath.ToStdString(), IOUtil::eBigEndian);
if (Out.IsValid())
Out.WriteBytes(ResourceData.data(), ResourceData.size());
// Add to resource DB
mpProject->ResourceDatabase()->RegisterResource(rRes.ResourceID, OutDir, OutName, CResource::ResTypeForExtension(rRes.ResourceType));
rRes.Exported = true;
}
}

39
src/Core/GameProject/CGameExporter.h
View File

@ -35,18 +35,55 @@ class CGameExporter
u32 PakOffset;
u32 PakSize;
bool Compressed;
bool Exported;
};
std::map<u64, SResourceInstance> mResourceMap;
struct SResourcePath
{
TWideString Dir;
TWideString Name;
};
std::map<u64, SResourcePath> mResourcePaths;
public:
CGameExporter(const TString& rkInputDir, const TString& rkOutputDir);
bool Export();
void LoadResource(const CUniqueID& rkID, std::vector<u8>& rBuffer);
protected:
void CopyDiscData();
void LoadAssetList();
void LoadPaks();
void LoadPakResource(const SResourceInstance& rkResource, std::vector<u8>& rBuffer);
void LoadResource(const SResourceInstance& rkResource, std::vector<u8>& rBuffer);
void ExportWorlds();
void ExportCookedResources();
void ExportResource(SResourceInstance& rRes);
// Convenience Functions
inline SResourceInstance* FindResourceInstance(const CUniqueID& rkID)
{
u64 IntegralID = rkID.ToLongLong();
auto Found = mResourceMap.find(IntegralID);
return (Found == mResourceMap.end() ? nullptr : &Found->second);
}
inline SResourcePath* FindResourcePath(const CUniqueID& rkID)
{
u64 IntegralID = rkID.ToLongLong();
auto Found = mResourcePaths.find(IntegralID);
return (Found == mResourcePaths.end() ? nullptr : &Found->second);
}
inline void SetResourcePath(const CUniqueID& rkID, const TWideString& rkDir, const TWideString& rkName)
{
SetResourcePath(rkID.ToLongLong(), rkDir, rkName);
}
inline void SetResourcePath(u64 ID, const TWideString& rkDir, const TWideString& rkName)
{
mResourcePaths[ID] = SResourcePath { rkDir, rkName };
}
inline EGame Game() const { return mpProject->Game(); }
inline void SetGame(EGame Game) { mpProject->SetGame(Game); }

15
src/Core/GameProject/CGameProject.h
View File

@ -29,17 +29,20 @@ public:
// Directory Handling
inline TWideString ProjectRoot() const { return mProjectRoot; }
inline TWideString DiscDir() const { return mProjectRoot + L"Disc\\"; }
inline TWideString ResourcesDir() const { return mProjectRoot + L"Resources\\"; }
inline TWideString WorldsDir() const { return mProjectRoot + L"Worlds\\"; }
inline TWideString CookedDir() const { return mProjectRoot + L"Cooked\\"; }
inline TWideString CookedResourcesDir() const { return CookedDir() + L"Resources\\"; }
inline TWideString CookedWorldsDir() const { return CookedDir() + L"Worlds\\"; }
inline TWideString DiscDir(bool Relative) const { return Relative ? L"Disc\\" : mProjectRoot + L"Disc\\"; }
inline TWideString ResourcesDir(bool Relative) const { return Relative ? L"Resources\\" : mProjectRoot + L"Resources\\"; }
inline TWideString WorldsDir(bool Relative) const { return Relative ? L"Worlds\\" : mProjectRoot + L"Worlds\\"; }
inline TWideString CookedDir(bool Relative) const { return Relative ? L"Cooked\\" : mProjectRoot + L"Cooked\\"; }
inline TWideString CookedResourcesDir(bool Relative) const { return CookedDir(Relative) + L"Resources\\"; }
inline TWideString CookedWorldsDir(bool Relative) const { return CookedDir(Relative) + L"Worlds\\"; }
// Accessors
inline void SetGame(EGame Game) { mGame = Game; }
inline void SetProjectName(const TString& rkName) { mProjectName = rkName; }
inline u32 NumWorldPaks() const { return mWorldPaks.size(); }
inline CPackage* WorldPakByIndex(u32 Index) const { return mWorldPaks[Index]; }
inline EGame Game() const { return mGame; }
inline CResourceDatabase* ResourceDatabase() const { return mpResourceDatabase; }
};

11
src/Core/GameProject/CPackage.h
View File

@ -9,24 +9,31 @@ struct SNamedResource
{
TString Name;
CUniqueID ID;
CFourCC Type;
};
class CPackage
{
TString mPakName;
TWideString mPakPath;
std::vector<SNamedResource> mNamedResources;
std::vector<CUniqueID> mPakResources;
public:
CPackage() {}
CPackage(const TString& rkName) : mPakName(rkName) {}
CPackage(const TString& rkName, const TWideString& rkPath)
: mPakName(rkName)
, mPakPath(rkPath)
{}
inline TString PakName() const { return mPakName; }
inline TWideString PakPath() const { return mPakPath; }
inline u32 NumNamedResources() const { return mNamedResources.size(); }
inline const SNamedResource& NamedResourceByIndex(u32 Index) const { return mNamedResources[Index]; }
inline void SetPakName(TString NewName) { mPakName = NewName; }
inline void AddNamedResource(TString Name, const CUniqueID& rkID) { mNamedResources.push_back( SNamedResource { Name, rkID } ); }
inline void AddNamedResource(TString Name, const CUniqueID& rkID, const CFourCC& rkType) { mNamedResources.push_back( SNamedResource { Name, rkID, rkType } ); }
inline void AddPakResource(const CUniqueID& rkID) { mPakResources.push_back(rkID); }
};

2
src/Core/GameProject/CResourceDatabase.cpp
View File

@ -28,7 +28,7 @@ TString CResourceEntry::RawAssetPath() const
TString CResourceEntry::CookedAssetPath() const
{
TWideString Ext = GetCookedExtension(mResourceType, mpDatabase->GameProject()->Game()).ToUTF16();
return mpDatabase->GameProject()->CookedDir() + mFileDir + mFileName + L"." + Ext;
return mpDatabase->GameProject()->CookedDir(false) + mFileDir + mFileName + L"." + Ext;
}
bool CResourceEntry::NeedsRecook() const

86
src/Core/Resource/CResCache.cpp
View File

@ -14,12 +14,14 @@
#include "Core/Resource/Factory/CWorldLoader.h"
#include <FileIO/FileIO.h>
#include <Common/AssertMacro.h>
#include <Common/FileUtil.h>
#include <Common/Log.h>
#include <Common/TString.h>
#include <iostream>
CResCache::CResCache()
: mpGameExporter(nullptr)
{
}
@ -71,9 +73,34 @@ TString CResCache::GetSourcePath()
CResource* CResCache::GetResource(CUniqueID ResID, CFourCC Type)
{
if (!ResID.IsValid()) return nullptr;
TString Source = mResDir + ResID.ToString() + "." + Type.ToString();
TString StringName = ResID.ToString() + "." + Type.ToString();
// With Game Exporter - get data buffer from exporter
if (mpGameExporter)
{
// Check if we already have resource loaded
auto Got = mResourceCache.find(ResID.ToLongLong());
if (Got != mResourceCache.end())
return Got->second;
// Otherwise load resource
std::vector<u8> DataBuffer;
mpGameExporter->LoadResource(ResID, DataBuffer);
if (DataBuffer.empty()) return nullptr;
CMemoryInStream MemStream(DataBuffer.data(), DataBuffer.size(), IOUtil::eBigEndian);
CResource *pRes = InternalLoadResource(MemStream, ResID, Type);
pRes->mResSource = StringName;
return pRes;
}
// Without Game Exporter - load from file
else
{
TString Source = mResDir + StringName;
return GetResource(Source);
}
}
CResource* CResCache::GetResource(const TString& rkResPath)
{
@ -98,32 +125,11 @@ CResource* CResCache::GetResource(const TString& rkResPath)
mResDir = rkResPath.GetFileDirectory();
// Load resource
CResource *pRes = nullptr;
CFourCC Type = rkResPath.GetFileExtension().ToUpper();
bool SupportedFormat = true;
if (Type == "CMDL") pRes = CModelLoader::LoadCMDL(File);
else if (Type == "TXTR") pRes = CTextureDecoder::LoadTXTR(File);
else if (Type == "ANCS") pRes = CAnimSetLoader::LoadANCS(File);
else if (Type == "CHAR") pRes = CAnimSetLoader::LoadCHAR(File);
else if (Type == "MREA") pRes = CAreaLoader::LoadMREA(File);
else if (Type == "MLVL") pRes = CWorldLoader::LoadMLVL(File);
else if (Type == "STRG") pRes = CStringLoader::LoadSTRG(File);
else if (Type == "FONT") pRes = CFontLoader::LoadFONT(File);
else if (Type == "SCAN") pRes = CScanLoader::LoadSCAN(File);
else if (Type == "DCLN") pRes = CCollisionLoader::LoadDCLN(File);
else if (Type == "EGMC") pRes = CPoiToWorldLoader::LoadEGMC(File);
else if (Type == "CINF") pRes = CSkeletonLoader::LoadCINF(File);
else if (Type == "ANIM") pRes = CAnimationLoader::LoadANIM(File);
else if (Type == "CSKR") pRes = CSkinLoader::LoadCSKR(File);
else SupportedFormat = false;
if (!pRes) pRes = new CResource(); // Default for unsupported formats
CResource *pRes = InternalLoadResource(File, ResID, Type);
pRes->mResSource = rkResPath;
// Add to cache and cleanup
pRes->mID = *rkResPath;
pRes->mResSource = rkResPath;
mResourceCache[ResID.ToLongLong()] = pRes;
mResDir = OldResDir;
return pRes;
}
@ -168,4 +174,36 @@ void CResCache::DeleteResource(CUniqueID ResID)
}
}
// ************ PROTECTED ************
CResource* CResCache::InternalLoadResource(IInputStream& rInput, const CUniqueID& rkID, CFourCC Type)
{
// todo - need some sort of auto-registration of loaders to avoid this if-else mess
ASSERT(mResourceCache.find(rkID.ToLongLong()) == mResourceCache.end()); // this test should be done before calling this func!
CResource *pRes = nullptr;
// Load resource
if (Type == "CMDL") pRes = CModelLoader::LoadCMDL(rInput);
else if (Type == "TXTR") pRes = CTextureDecoder::LoadTXTR(rInput);
else if (Type == "ANCS") pRes = CAnimSetLoader::LoadANCS(rInput);
else if (Type == "CHAR") pRes = CAnimSetLoader::LoadCHAR(rInput);
else if (Type == "MREA") pRes = CAreaLoader::LoadMREA(rInput);
else if (Type == "MLVL") pRes = CWorldLoader::LoadMLVL(rInput);
else if (Type == "STRG") pRes = CStringLoader::LoadSTRG(rInput);
else if (Type == "FONT") pRes = CFontLoader::LoadFONT(rInput);
else if (Type == "SCAN") pRes = CScanLoader::LoadSCAN(rInput);
else if (Type == "DCLN") pRes = CCollisionLoader::LoadDCLN(rInput);
else if (Type == "EGMC") pRes = CPoiToWorldLoader::LoadEGMC(rInput);
else if (Type == "CINF") pRes = CSkeletonLoader::LoadCINF(rInput);
else if (Type == "ANIM") pRes = CAnimationLoader::LoadANIM(rInput);
else if (Type == "CSKR") pRes = CSkinLoader::LoadCSKR(rInput);
if (!pRes) pRes = new CResource(); // Default for unsupported formats
ASSERT(pRes->mRefCount == 0);
// Cache and return
pRes->mID = rkID;
mResourceCache[rkID.ToLongLong()] = pRes;
return pRes;
}
CResCache gResCache;

7
src/Core/Resource/CResCache.h
View File

@ -2,6 +2,7 @@
#define CRESCACHE_H
#include "CResource.h"
#include "Core/GameProject/CGameExporter.h"
#include <Common/types.h>
#include <Common/TString.h>
#include <unordered_map>
@ -10,6 +11,7 @@ class CResCache
{
std::unordered_map<u64, CResource*> mResourceCache;
TString mResDir;
CGameExporter *mpGameExporter;
public:
CResCache();
@ -22,6 +24,11 @@ public:
CFourCC FindResourceType(CUniqueID ResID, const TStringList& rkPossibleTypes);
void CacheResource(CResource *pRes);
void DeleteResource(CUniqueID ResID);
inline void SetGameExporter(CGameExporter *pExporter) { mpGameExporter = pExporter; }
protected:
CResource* InternalLoadResource(IInputStream& rInput, const CUniqueID& rkID, CFourCC Type);
};
extern CResCache gResCache;

2
src/Core/Resource/CResource.cpp
View File

@ -120,7 +120,7 @@ REGISTER_RESOURCE_TYPE(MREA, eArea, ePrimeDemo, eReturns)
REGISTER_RESOURCE_TYPE(NTWK, eTweak, eEchoesDemo, eReturns)
REGISTER_RESOURCE_TYPE(PAK , ePackage, ePrimeDemo, eReturns)
REGISTER_RESOURCE_TYPE(PART, eParticle, ePrimeDemo, eReturns)
REGISTER_RESOURCE_TYPE(PATH, eNavMesh, ePrimeDemo, eCorruption)
REGISTER_RESOURCE_TYPE(PATH, ePathfinding, ePrimeDemo, eCorruption)
REGISTER_RESOURCE_TYPE(PTLA, ePortalArea, eEchoesDemo, eCorruption)
REGISTER_RESOURCE_TYPE(RULE, eRuleSet, eEchoesDemo, eReturns)
REGISTER_RESOURCE_TYPE(SAND, eSourceAnimData, eCorruptionProto, eCorruption)

9
src/Core/Resource/EResType.h
View File

@ -14,7 +14,6 @@ enum EResType
eAudioMacro,
eAudioGroupSet,
eAudioSample,
eStreamedAudio,
eAudioLookupTable,
eBinaryData,
eBurstFireData,
@ -25,14 +24,12 @@ enum EResType
eGuiFrame,
eGuiKeyFrame,
eHintSystem,
eInvalidResType,
eMapArea,
eMapWorld,
eMapUniverse,
eMidi,
eModel,
eMusicTrack,
eNavMesh,
ePackage,
eParticle,
eParticleCollisionResponse,
@ -43,6 +40,7 @@ enum EResType
eParticleSwoosh,
eParticleTransform,
eParticleWeapon,
ePathfinding,
ePortalArea,
eResource,
eRuleSet,
@ -56,6 +54,7 @@ enum EResType
eStateMachine,
eStateMachine2, // For distinguishing AFSM/FSM2
eStaticGeometryMap,
eStreamedAudio,
eStringList,
eStringTable,
eTexture,
@ -63,7 +62,9 @@ enum EResType
eUnknown_CAAD,
eUserEvaluatorData,
eVideo,
eWorld
eWorld,
eInvalidResType = -1
};
// defined in CResource.cpp

30
src/Core/Resource/Factory/CTextureDecoder.cpp
View File

@ -201,6 +201,10 @@ void CTextureDecoder::ReadDDS(IInputStream& rDDS)
// ************ DECODE ************
void CTextureDecoder::PartialDecodeGXTexture(IInputStream& TXTR)
{
// TODO: This function doesn't handle very small mipmaps correctly.
// The format applies padding when the size of a mipmap is less than the block size for that format.
// The decode needs to be adjusted to account for the padding and skip over it (since we don't have padding in OpenGL).
// Get image data size, create output buffer
u32 ImageStart = TXTR.Tell();
TXTR.Seek(0x0, SEEK_END);
@ -233,11 +237,22 @@ void CTextureDecoder::PartialDecodeGXTexture(IInputStream& TXTR)
MipH /= 4;
}
// This value set to true if we hit the end of the file earlier than expected.
// This is necessary due to a mistake Retro made in their cooker for I8 textures where very small mipmaps are cut off early, resulting in an out-of-bounds memory access.
// This affects one texture that I know of - Echoes 3bb2c034.TXTR
bool BreakEarly = false;
for (u32 iMip = 0; iMip < mNumMipMaps; iMip++)
{
if (MipW < BWidth) MipW = BWidth;
if (MipH < BHeight) MipH = BHeight;
for (u32 iBlockY = 0; iBlockY < MipH; iBlockY += BHeight)
for (u32 iBlockX = 0; iBlockX < MipW; iBlockX += BWidth) {
for (u32 iImgY = iBlockY; iImgY < iBlockY + BHeight; iImgY++) {
{
for (u32 iBlockX = 0; iBlockX < MipW; iBlockX += BWidth)
{
for (u32 iImgY = iBlockY; iImgY < iBlockY + BHeight; iImgY++)
{
for (u32 iImgX = iBlockX; iImgX < iBlockX + BWidth; iImgX++)
{
u32 DstPos = ((iImgY * MipW) + iImgX) * PixelStride;
@ -256,9 +271,16 @@ void CTextureDecoder::PartialDecodeGXTexture(IInputStream& TXTR)
// I4 and C4 have 4bpp images, so I'm forced to read two pixels at a time.
if ((mTexelFormat == eGX_I4) || (mTexelFormat == eGX_C4)) iImgX++;
// Check if we're at the end of the file.
if (TXTR.EoF()) BreakEarly = true;
}
if (BreakEarly) break;
}
if (mTexelFormat == eGX_RGBA8) TXTR.Seek(0x20, SEEK_CUR);
if (BreakEarly) break;
}
if (BreakEarly) break;
}
u32 MipSize = (u32) (MipW * MipH * gskPixelsToBytes[mTexelFormat]);
@ -267,8 +289,8 @@ void CTextureDecoder::PartialDecodeGXTexture(IInputStream& TXTR)
MipOffset += MipSize;
MipW /= 2;
MipH /= 2;
if (MipW < BWidth) MipW = BWidth;
if (MipH < BHeight) MipH = BHeight;
if (BreakEarly) break;
}
}

2
src/Core/Resource/Script/CScriptObject.h
View File

@ -23,7 +23,7 @@ class CScriptObject
friend class CAreaLoader;
CScriptTemplate *mpTemplate;
TResPtr<CGameArea> mpArea;
CGameArea *mpArea;
CScriptLayer *mpLayer;
u32 mVersion;