Completely overhauled resource loading in preparation for projects

This commit is contained in:
parax0
2016-06-29 17:18:31 -06:00
parent e53a895b29
commit 2d6dfad2d3
102 changed files with 1334 additions and 835 deletions

View File

@@ -1,5 +1,5 @@
#include "CGameExporter.h"
#include "Core/Resource/CResCache.h"
#include "Core/GameProject/CResourceStore.h"
#include "Core/Resource/CWorld.h"
#include <FileIO/FileIO.h>
#include <Common/AssertMacro.h>
@@ -30,7 +30,7 @@ CGameExporter::CGameExporter(const TString& rkInputDir, const TString& rkOutputD
bool CGameExporter::Export()
{
SCOPED_TIMER(ExportGame);
gResCache.SetGameExporter(this);
gResourceStore.SetGameExporter(this);
FileUtil::CreateDirectory(mExportDir);
FileUtil::ClearDirectory(mExportDir);
@@ -40,7 +40,7 @@ bool CGameExporter::Export()
ExportWorlds();
ExportCookedResources();
gResCache.SetGameExporter(nullptr);
gResourceStore.SetGameExporter(nullptr);
return true;
}
@@ -57,7 +57,7 @@ void CGameExporter::CopyDiscData()
SCOPED_TIMER(CopyDiscData);
// Create Disc output folder
FileUtil::CreateDirectory(mDiscDir);
FileUtil::CreateDirectory(mExportDir + mDiscDir);
#endif
// Copy data
@@ -97,7 +97,7 @@ void CGameExporter::CopyDiscData()
#if COPY_DISC_DATA
// Create directory
TWideString OutFile = mDiscDir + RelPath;
TWideString OutFile = mExportDir + mDiscDir + RelPath;
FileUtil::CreateDirectory(OutFile.GetFileDirectory());
// Copy file
@@ -184,7 +184,7 @@ void CGameExporter::LoadPaks()
continue;
}
CPackage *pPackage = new CPackage(CharPak.GetFileName(false), FileUtil::MakeRelative(PakPath.GetFileDirectory(), mExportDir));
CPackage *pPackage = new CPackage(CharPak.GetFileName(false), FileUtil::MakeRelative(PakPath.GetFileDirectory(), mGameDir));
// MP1-MP3Proto
if (Game() < eCorruption)
@@ -407,10 +407,10 @@ void CGameExporter::ExportWorlds()
// Get output path. DKCR paks are stored in a Worlds folder so we should get the path relative to that so we don't have Worlds\Worlds\.
// Other games have all paks in the game root dir so we're fine just taking the original root dir-relative directory.
TWideString PakPath = pPak->PakPath();
TWideString WorldsDir = PakPath.GetParentDirectoryPath(L"Worlds", false);
TWideString GameWorldsDir = PakPath.GetParentDirectoryPath(L"Worlds", false);
if (!WorldsDir.IsEmpty())
PakPath = FileUtil::MakeRelative(PakPath, WorldsDir);
if (!GameWorldsDir.IsEmpty())
PakPath = FileUtil::MakeRelative(PakPath, GameWorldsDir);
for (u32 iRes = 0; iRes < pPak->NumNamedResources(); iRes++)
{
@@ -418,7 +418,7 @@ void CGameExporter::ExportWorlds()
if (rkRes.Type == "MLVL" && !rkRes.Name.EndsWith("NODEPEND"))
{
TResPtr<CWorld> pWorld = (CWorld*) gResCache.GetResource(rkRes.ID, rkRes.Type);
TResPtr<CWorld> pWorld = (CWorld*) gResourceStore.LoadResource(rkRes.ID, rkRes.Type);
if (!pWorld)
{
@@ -451,7 +451,7 @@ void CGameExporter::ExportWorlds()
// Load area
CUniqueID AreaID = pWorld->AreaResourceID(iArea);
CGameArea *pArea = (CGameArea*) gResCache.GetResource(AreaID, "MREA");
CGameArea *pArea = (CGameArea*) gResourceStore.LoadResource(AreaID, "MREA");
if (!pArea)
{
@@ -470,7 +470,7 @@ void CGameExporter::ExportWorlds()
ExportResource(*pInst);
}
gResCache.Clean();
gResourceStore.DestroyUnreferencedResources();
}
else
@@ -485,7 +485,7 @@ void CGameExporter::ExportWorlds()
void CGameExporter::ExportCookedResources()
{
#if EXPORT_COOKED
CResourceDatabase *pResDB = mpProject->ResourceDatabase();
gResourceStore.CloseActiveProject();
{
SCOPED_TIMER(ExportCookedResources);
FileUtil::CreateDirectory(mCookedDir + mResDir);
@@ -498,7 +498,7 @@ void CGameExporter::ExportCookedResources()
}
{
SCOPED_TIMER(SaveResourceDatabase);
pResDB->Save(this->mExportDir.ToUTF8() + "ResourceDatabase.rdb");
gResourceStore.SaveResourceDatabase(this->mExportDir.ToUTF8() + "ResourceDatabase.rdb");
}
#endif
}
@@ -532,7 +532,7 @@ void CGameExporter::ExportResource(SResourceInstance& rRes)
Out.WriteBytes(ResourceData.data(), ResourceData.size());
// Add to resource DB
mpProject->ResourceDatabase()->RegisterResource(rRes.ResourceID, OutDir, OutName, CResource::ResTypeForExtension(rRes.ResourceType));
gResourceStore.RegisterResource(rRes.ResourceID, CResource::ResTypeForExtension(rRes.ResourceType), OutDir, OutName);
rRes.Exported = true;
}
}

View File

@@ -2,7 +2,7 @@
#define CGAMEPROJECT_H
#include "CPackage.h"
#include "CResourceDatabase.h"
#include "CResourceStore.h"
#include "Core/Resource/EGame.h"
#include <Common/CUniqueID.h>
#include <Common/TString.h>
@@ -13,7 +13,7 @@ class CGameProject
EGame mGame;
TString mProjectName;
TWideString mProjectRoot;
CResourceDatabase *mpResourceDatabase;
TWideString mResourceDBPath;
std::vector<CPackage*> mWorldPaks;
std::vector<CPackage*> mResourcePaks;
@@ -22,13 +22,14 @@ public:
: mGame(eUnknownVersion)
, mProjectName("UnnamedProject")
, mProjectRoot(rkProjRootDir)
, mpResourceDatabase(new CResourceDatabase(this))
, mResourceDBPath(L"ResourceDB.rdb")
{}
void AddPackage(CPackage *pPackage, bool WorldPak);
// Directory Handling
inline TWideString ProjectRoot() const { return mProjectRoot; }
inline TWideString ResourceDBPath(bool Relative) const { return Relative ? mResourceDBPath : mProjectRoot + mResourceDBPath; }
inline TWideString DiscDir(bool Relative) const { return Relative ? L"Disc\\" : mProjectRoot + L"Disc\\"; }
inline TWideString ResourcesDir(bool Relative) const { return Relative ? L"Resources\\" : mProjectRoot + L"Resources\\"; }
inline TWideString WorldsDir(bool Relative) const { return Relative ? L"Worlds\\" : mProjectRoot + L"Worlds\\"; }
@@ -44,7 +45,6 @@ public:
inline CPackage* WorldPakByIndex(u32 Index) const { return mWorldPaks[Index]; }
inline EGame Game() const { return mGame; }
inline CResourceDatabase* ResourceDatabase() const { return mpResourceDatabase; }
};
extern CGameProject *gpProject;

View File

@@ -1,215 +0,0 @@
#include "CResourceDatabase.h"
#include "CGameProject.h"
#include "Core/Resource/CResCache.h"
#include <Common/AssertMacro.h>
#include <Common/FileUtil.h>
#include <Common/Log.h>
#include <tinyxml2.h>
using namespace tinyxml2;
// ************ CResourceEntry ************
bool CResourceEntry::HasRawVersion() const
{
return FileUtil::Exists(RawAssetPath());
}
bool CResourceEntry::HasCookedVersion() const
{
return FileUtil::Exists(CookedAssetPath());
}
TString CResourceEntry::RawAssetPath() const
{
TWideString Ext = GetRawExtension(mResourceType, mpDatabase->GameProject()->Game()).ToUTF16();
return mpDatabase->GameProject()->ProjectRoot() + mFileDir + mFileName + L"." + Ext;
}
TString CResourceEntry::CookedAssetPath() const
{
TWideString Ext = GetCookedExtension(mResourceType, mpDatabase->GameProject()->Game()).ToUTF16();
return mpDatabase->GameProject()->CookedDir(false) + mFileDir + mFileName + L"." + Ext;
}
bool CResourceEntry::NeedsRecook() const
{
// Assets that do not have a raw version can't be recooked since they will always just be saved cooked to begin with.
// We will recook any asset where the raw version has been updated but not recooked yet. mNeedsRecook can also be
// toggled to arbitrarily flag any asset for recook.
if (!HasRawVersion()) return false;
if (!HasCookedVersion()) return true;
if (mNeedsRecook) return true;
return (FileUtil::LastModifiedTime(CookedAssetPath()) < FileUtil::LastModifiedTime(RawAssetPath()));
}
// ************ CResourceDatabase ************
CResourceDatabase::CResourceDatabase(CGameProject *pProj)
: mpProj(pProj)
{}
CResourceDatabase::~CResourceDatabase()
{
}
void CResourceDatabase::Load(const TString& rkPath)
{
XMLDocument Doc;
Doc.LoadFile(*rkPath);
if (!Doc.Error())
{
XMLElement *pRoot = Doc.FirstChildElement("ResourceDatabase");
//EVersion DatabaseVersion = (EVersion) TString(pRoot->Attribute("Version")).ToInt32(10); // Version currently unused
XMLElement *pResources = pRoot->FirstChildElement("Resources");
XMLElement *pRes = pResources->FirstChildElement("Resource");
u32 ResIndex = 0;
while (pRes)
{
XMLElement *pChild = pRes->FirstChildElement();
bool HasID = false, HasType = false, HasDir = false, HasName = false;
CUniqueID ID;
EResType Type;
TWideString FileDir;
TWideString FileName;
while (pChild)
{
TString NodeName = pChild->Name();
if (NodeName == "ID")
{
ID = CUniqueID::FromString(pChild->GetText());
HasID = true;
}
else if (NodeName == "Type")
{
Type = CResource::ResTypeForExtension(pChild->GetText());
HasType = true;
ASSERT(Type != eInvalidResType);
}
else if (NodeName == "FileDir")
{
FileDir = pChild->GetText();
HasDir = true;
}
else if (NodeName == "FileName")
{
FileName = pChild->GetText();
HasName = true;
}
pChild = pChild->NextSiblingElement();
}
if (HasID && HasType && HasDir && HasName)
RegisterResource(ID, FileDir, FileName, Type);
else
Log::Error("Error reading " + rkPath + ": Resource entry " + TString::FromInt32(ResIndex, 0, 10) + " is missing one or more components");
ResIndex++;
pRes = pRes->NextSiblingElement("Resource");
}
}
}
void CResourceDatabase::Save(const TString& rkPath) const
{
XMLDocument Doc;
XMLDeclaration *pDecl = Doc.NewDeclaration();
Doc.LinkEndChild(pDecl);
XMLElement *pRoot = Doc.NewElement("ResourceDatabase");
pRoot->SetAttribute("Version", eVer_Current);
Doc.LinkEndChild(pRoot);
XMLElement *pResources = Doc.NewElement("Resources");
pRoot->LinkEndChild(pResources);
for (auto It = mResourceMap.begin(); It != mResourceMap.end(); It++)
{
CResourceEntry *pEntry = It->second;
XMLElement *pRes = Doc.NewElement("Resource");
pResources->LinkEndChild(pRes);
XMLElement *pID = Doc.NewElement("ID");
pID->SetText(*pEntry->ID().ToString());
pRes->LinkEndChild(pID);
XMLElement *pType = Doc.NewElement("Type");
pType->SetText(*GetCookedExtension(pEntry->ResourceType(), mpProj->Game()));
pRes->LinkEndChild(pType);
XMLElement *pDir = Doc.NewElement("FileDir");
pDir->SetText(*pEntry->FileDirectory());
pRes->LinkEndChild(pDir);
XMLElement *pName = Doc.NewElement("FileName");
pName->SetText(*pEntry->FileName());
pRes->LinkEndChild(pName);
XMLElement *pRecook = Doc.NewElement("NeedsRecook");
pRecook->SetText(pEntry->NeedsRecook() ? "true" : "false");
pRes->LinkEndChild(pRecook);
}
Doc.SaveFile(*rkPath);
}
CResourceEntry* CResourceDatabase::FindResourceEntry(const CUniqueID& rkID) const
{
auto Found = mResourceMap.find(rkID);
if (Found == mResourceMap.end()) return nullptr;
else return Found->second;
}
CResource* CResourceDatabase::LoadResource(const CUniqueID& rkID) const
{
// todo: no handling for raw assets yet
CResourceEntry *pEntry = FindResourceEntry(rkID);
if (pEntry)
{
TString AssetPath = pEntry->CookedAssetPath();
if (FileUtil::Exists(AssetPath))
return gResCache.GetResource(pEntry->CookedAssetPath());
}
return nullptr;
}
bool CResourceDatabase::RegisterResource(const CUniqueID& rkID, const TWideString& rkDir, const TWideString& rkFileName, EResType Type)
{
CResourceEntry *pEntry = FindResourceEntry(rkID);
if (pEntry)
{
Log::Error("Attempted to register resource that's already tracked in the database: " + rkID.ToString() + " / " + rkDir.ToUTF8() + " / " + rkFileName.ToUTF8());
return false;
}
else
{
pEntry = new CResourceEntry(this, rkID, rkDir, rkFileName.GetFileName(false), Type);
if (!pEntry->HasCookedVersion() && !pEntry->HasRawVersion())
{
Log::Error("Attempted to register a resource that doesn't exist: " + rkID.ToString() + " | " + rkDir.ToUTF8() + " | " + rkFileName.ToUTF8());
delete pEntry;
return false;
}
else
{
mResourceMap[rkID] = pEntry;
return true;
}
}
}

View File

@@ -1,74 +0,0 @@
#ifndef CRESOURCEDATABASE_H
#define CRESOURCEDATABASE_H
#include "Core/Resource/CResource.h"
#include <Common/CUniqueID.h>
#include <Common/TString.h>
#include <Common/types.h>
#include <map>
class CGameProject;
class CResourceDatabase;
class CResourceEntry
{
CResourceDatabase *mpDatabase;
CUniqueID mID;
TWideString mFileDir;
TWideString mFileName;
EResType mResourceType;
bool mNeedsRecook;
public:
CResourceEntry(CResourceDatabase *pDatabase, const CUniqueID& rkID,
const TWideString& rkDir, const TWideString& rkFilename, EResType Type)
: mpDatabase(pDatabase)
, mID(rkID)
, mFileDir(rkDir)
, mFileName(rkFilename)
, mResourceType(Type)
, mNeedsRecook(false)
{}
bool HasRawVersion() const;
bool HasCookedVersion() const;
TString RawAssetPath() const;
TString CookedAssetPath() const;
bool NeedsRecook() const;
// Accessors
void SetDirty() { mNeedsRecook = true; }
inline CUniqueID ID() const { return mID; }
inline TString FileDirectory() const { return mFileDir; }
inline TString FileName() const { return mFileName; }
inline EResType ResourceType() const { return mResourceType; }
};
class CResourceDatabase
{
CGameProject *mpProj;
std::map<CUniqueID, CResourceEntry*> mResourceMap;
enum EVersion
{
eVer_Initial,
eVer_Max,
eVer_Current = eVer_Max - 1
};
public:
CResourceDatabase(CGameProject *pProj);
~CResourceDatabase();
void Load(const TString& rkPath);
void Save(const TString& rkPath) const;
CResourceEntry* FindResourceEntry(const CUniqueID& rkID) const;
CResource* LoadResource(const CUniqueID& rkID) const;
bool RegisterResource(const CUniqueID& rkID, const TWideString& rkDir, const TWideString& rkFileName, EResType Type);
inline CGameProject* GameProject() const { return mpProj; }
};
#endif // CRESOURCEDATABASE_H

View File

@@ -0,0 +1,151 @@
#include "CResourceEntry.h"
#include "CGameProject.h"
#include "CResourceStore.h"
#include "Core/Resource/CResource.h"
#include <Common/FileUtil.h>
#include <Common/TString.h>
// Resource Loaders
// todo: come up with a factory setup that doesn't suck
#include "Core/Resource/Factory/CAnimationLoader.h"
#include "Core/Resource/Factory/CAnimSetLoader.h"
#include "Core/Resource/Factory/CAreaLoader.h"
#include "Core/Resource/Factory/CCollisionLoader.h"
#include "Core/Resource/Factory/CFontLoader.h"
#include "Core/Resource/Factory/CMaterialLoader.h"
#include "Core/Resource/Factory/CModelLoader.h"
#include "Core/Resource/Factory/CPoiToWorldLoader.h"
#include "Core/Resource/Factory/CScanLoader.h"
#include "Core/Resource/Factory/CScriptLoader.h"
#include "Core/Resource/Factory/CSkeletonLoader.h"
#include "Core/Resource/Factory/CSkinLoader.h"
#include "Core/Resource/Factory/CStringLoader.h"
#include "Core/Resource/Factory/CTextureDecoder.h"
#include "Core/Resource/Factory/CWorldLoader.h"
CResourceEntry::CResourceEntry(CResourceStore *pStore, const CUniqueID& rkID,
const TWideString& rkDir, const TWideString& rkFilename,
EResType Type, bool Transient /*= false*/)
: mpStore(pStore)
, mpResource(nullptr)
, mID(rkID)
, mFileName(rkFilename)
, mType(Type)
, mNeedsRecook(false)
, mTransient(Transient)
{
mpDirectory = mpStore->GetVirtualDirectory(rkDir, Transient, true);
if (mpDirectory) mpDirectory->AddChild(L"", this);
mGame = ((mTransient || !mpStore->ActiveProject()) ? eUnknownVersion : mpStore->ActiveProject()->Game());
}
CResourceEntry::~CResourceEntry()
{
if (mpResource) delete mpResource;
}
bool CResourceEntry::HasRawVersion() const
{
return FileUtil::Exists(RawAssetPath());
}
bool CResourceEntry::HasCookedVersion() const
{
return FileUtil::Exists(CookedAssetPath());
}
TString CResourceEntry::RawAssetPath(bool Relative) const
{
TWideString Ext = GetResourceRawExtension(mType, mGame).ToUTF16();
TWideString Path = mpDirectory ? mpDirectory->FullPath() : L"";
TWideString Name = mFileName + L"." + Ext;
return ((mTransient || Relative) ? Path + Name : mpStore->ActiveProject()->ResourcesDir(false) + Path + Name);
}
TString CResourceEntry::CookedAssetPath(bool Relative) const
{
TWideString Ext = GetResourceCookedExtension(mType, mGame).ToUTF16();
TWideString Path = mpDirectory ? mpDirectory->FullPath() : L"";
TWideString Name = mFileName + L"." + Ext;
return ((mTransient || Relative) ? Path + Name : mpStore->ActiveProject()->CookedResourcesDir(false) + Path + Name);
}
bool CResourceEntry::NeedsRecook() const
{
// Assets that do not have a raw version can't be recooked since they will always just be saved cooked to begin with.
// We will recook any asset where the raw version has been updated but not recooked yet. mNeedsRecook can also be
// toggled to arbitrarily flag any asset for recook.
if (!HasRawVersion()) return false;
if (!HasCookedVersion()) return true;
if (mNeedsRecook) return true;
return (FileUtil::LastModifiedTime(CookedAssetPath()) < FileUtil::LastModifiedTime(RawAssetPath()));
}
void CResourceEntry::SetGame(EGame NewGame)
{
if (mGame != NewGame)
{
// todo: implement checks here. This needs work because we should trigger a recook and if the extension changes
// we should delete the old file. Also a lot of resources can't evaluate this correctly due to file version
// numbers being shared between games.
mGame = NewGame;
}
}
CResource* CResourceEntry::Load()
{
// todo: load raw
if (mpResource) return mpResource;
if (!HasCookedVersion()) return nullptr;
CFileInStream File(CookedAssetPath().ToStdString(), IOUtil::eBigEndian);
if (!File.IsValid())
{
Log::Error("Failed to open cooked resource: " + CookedAssetPath(true));
return nullptr;
}
return Load(File);
}
CResource* CResourceEntry::Load(IInputStream& rInput)
{
// Overload to allow for load from an arbitrary input stream.
if (mpResource) return mpResource;
if (!rInput.IsValid()) return nullptr;
switch (mType)
{
case eAnimation: mpResource = CAnimationLoader::LoadANIM(rInput, this); break;
case eArea: mpResource = CAreaLoader::LoadMREA(rInput, this); break;
case eDynamicCollision: mpResource = CCollisionLoader::LoadDCLN(rInput, this); break;
case eFont: mpResource = CFontLoader::LoadFONT(rInput, this); break;
case eModel: mpResource = CModelLoader::LoadCMDL(rInput, this); break;
case eScan: mpResource = CScanLoader::LoadSCAN(rInput, this); break;
case eSkeleton: mpResource = CSkeletonLoader::LoadCINF(rInput, this); break;
case eSkin: mpResource = CSkinLoader::LoadCSKR(rInput, this); break;
case eStaticGeometryMap: mpResource = CPoiToWorldLoader::LoadEGMC(rInput, this); break;
case eStringTable: mpResource = CStringLoader::LoadSTRG(rInput, this); break;
case eTexture: mpResource = CTextureDecoder::LoadTXTR(rInput, this); break;
case eWorld: mpResource = CWorldLoader::LoadMLVL(rInput, this); break;
case eAnimSet:
if (mGame <= eEchoes)
mpResource = CAnimSetLoader::LoadANCS(rInput, this);
else
mpResource = CAnimSetLoader::LoadCHAR(rInput, this);
break;
default: mpResource = new CResource(this); break;
}
return mpResource;
}
bool CResourceEntry::Unload()
{
ASSERT(mpResource != nullptr);
delete mpResource;
mpResource = nullptr;
return true;
}

View File

@@ -0,0 +1,55 @@
#ifndef CRESOURCEENTRY_H
#define CRESOURCEENTRY_H
#include "CVirtualDirectory.h"
#include "Core/Resource/EResType.h"
#include <Common/CUniqueID.h>
#include <Common/types.h>
class CResource;
class CResourceStore;
class CResourceEntry
{
CResourceStore *mpStore;
CResource *mpResource;
CUniqueID mID;
EResType mType;
EGame mGame;
CVirtualDirectory *mpDirectory;
TWideString mFileName;
bool mNeedsRecook;
bool mTransient;
public:
CResourceEntry(CResourceStore *pStore, const CUniqueID& rkID,
const TWideString& rkDir, const TWideString& rkFilename,
EResType Type, bool Transient = false);
~CResourceEntry();
bool HasRawVersion() const;
bool HasCookedVersion() const;
TString RawAssetPath(bool Relative = false) const;
TString CookedAssetPath(bool Relative = false) const;
bool NeedsRecook() const;
void SetGame(EGame NewGame);
CResource* Load();
CResource* Load(IInputStream& rInput);
bool Unload();
// Accessors
void SetDirty() { mNeedsRecook = true; }
inline bool IsLoaded() const { return mpResource != nullptr; }
inline CResource* Resource() const { return mpResource; }
inline CUniqueID ID() const { return mID; }
inline EGame Game() const { return mGame; }
inline CVirtualDirectory* Directory() const { return mpDirectory; }
inline TString FileName() const { return mFileName; }
inline EResType ResourceType() const { return mType; }
inline bool IsTransient() const { return mTransient; }
protected:
CResource* InternalLoad(IInputStream& rInput);
};
#endif // CRESOURCEENTRY_H

View File

@@ -0,0 +1,403 @@
#include "CResourceStore.h"
#include "CGameExporter.h"
#include "CGameProject.h"
#include "Core/Resource/CResource.h"
#include <Common/AssertMacro.h>
#include <Common/FileUtil.h>
#include <Common/Log.h>
#include <tinyxml2.h>
using namespace tinyxml2;
CResourceStore gResourceStore;
CResourceStore::CResourceStore()
: mpProj(nullptr)
, mpProjectRoot(nullptr)
, mpExporter(nullptr)
{}
CResourceStore::~CResourceStore()
{
}
void CResourceStore::LoadResourceDatabase(const TString& rkPath)
{
XMLDocument Doc;
Doc.LoadFile(*rkPath);
if (!Doc.Error())
{
XMLElement *pRoot = Doc.FirstChildElement("ResourceDatabase");
//EDatabaseVersion Version = (EDatabaseVersion) TString(pRoot->Attribute("Version")).ToInt32(10); // Version currently unused
XMLElement *pResources = pRoot->FirstChildElement("Resources");
XMLElement *pRes = pResources->FirstChildElement("Resource");
u32 ResIndex = 0;
while (pRes)
{
XMLElement *pChild = pRes->FirstChildElement();
bool HasID = false, HasType = false, HasDir = false, HasName = false;
CUniqueID ID;
EResType Type;
TWideString FileDir;
TWideString FileName;
while (pChild)
{
TString NodeName = pChild->Name();
if (NodeName == "ID")
{
ID = CUniqueID::FromString(pChild->GetText());
HasID = true;
}
else if (NodeName == "Type")
{
Type = CResource::ResTypeForExtension(pChild->GetText());
HasType = true;
ASSERT(Type != eInvalidResType);
}
else if (NodeName == "FileDir")
{
FileDir = pChild->GetText();
HasDir = true;
}
else if (NodeName == "FileName")
{
FileName = pChild->GetText();
HasName = true;
}
pChild = pChild->NextSiblingElement();
}
if (HasID && HasType && HasDir && HasName)
RegisterResource(ID, Type, FileDir, FileName);
else
Log::Error("Error reading " + rkPath + ": Resource entry " + TString::FromInt32(ResIndex, 0, 10) + " is missing one or more components");
ResIndex++;
pRes = pRes->NextSiblingElement("Resource");
}
}
}
void CResourceStore::SaveResourceDatabase(const TString& rkPath) const
{
XMLDocument Doc;
XMLDeclaration *pDecl = Doc.NewDeclaration();
Doc.LinkEndChild(pDecl);
XMLElement *pRoot = Doc.NewElement("ResourceDatabase");
pRoot->SetAttribute("Version", eVer_Current);
Doc.LinkEndChild(pRoot);
XMLElement *pResources = Doc.NewElement("Resources");
pRoot->LinkEndChild(pResources);
for (auto It = mResourceEntries.begin(); It != mResourceEntries.end(); It++)
{
CResourceEntry *pEntry = It->second;
if (pEntry->IsTransient()) continue;
XMLElement *pRes = Doc.NewElement("Resource");
pResources->LinkEndChild(pRes);
XMLElement *pID = Doc.NewElement("ID");
pID->SetText(*pEntry->ID().ToString());
pRes->LinkEndChild(pID);
XMLElement *pType = Doc.NewElement("Type");
pType->SetText(*GetResourceCookedExtension(pEntry->ResourceType(), pEntry->Game()));
pRes->LinkEndChild(pType);
XMLElement *pDir = Doc.NewElement("FileDir");
pDir->SetText(*pEntry->Directory()->FullPath().ToUTF8());
pRes->LinkEndChild(pDir);
XMLElement *pName = Doc.NewElement("FileName");
pName->SetText(*pEntry->FileName());
pRes->LinkEndChild(pName);
XMLElement *pRecook = Doc.NewElement("NeedsRecook");
pRecook->SetText(pEntry->NeedsRecook() ? "true" : "false");
pRes->LinkEndChild(pRecook);
}
Doc.SaveFile(*rkPath);
}
void CResourceStore::SetActiveProject(CGameProject *pProj)
{
if (mpProj == pProj) return;
CloseActiveProject();
mpProj = pProj;
if (pProj)
{
mpProjectRoot = new CVirtualDirectory();
LoadResourceDatabase(pProj->ResourceDBPath(false));
}
}
void CResourceStore::CloseActiveProject()
{
// Delete all entries from old project
for (auto It = mResourceEntries.begin(); It != mResourceEntries.end(); It++)
{
CResourceEntry *pEntry = It->second;
if (!pEntry->IsTransient())
{
delete pEntry;
It = mResourceEntries.erase(It);
}
}
delete mpProjectRoot;
mpProjectRoot = nullptr;
mpProj = nullptr;
}
CVirtualDirectory* CResourceStore::GetVirtualDirectory(const TWideString& rkPath, bool Transient, bool AllowCreate)
{
if (rkPath.IsEmpty()) return nullptr;
else if (Transient)
{
for (u32 iTrans = 0; iTrans < mTransientRoots.size(); iTrans++)
{
if (mTransientRoots[iTrans]->Name() == rkPath)
return mTransientRoots[iTrans];
}
if (AllowCreate)
{
CVirtualDirectory *pDir = new CVirtualDirectory(rkPath);
mTransientRoots.push_back(pDir);
return pDir;
}
else return nullptr;
}
else if (mpProjectRoot)
{
return mpProjectRoot->FindChildDirectory(rkPath, AllowCreate);
}
else return nullptr;
}
CResourceEntry* CResourceStore::FindEntry(const CUniqueID& rkID) const
{
if (!rkID.IsValid()) return nullptr;
auto Found = mResourceEntries.find(rkID);
if (Found == mResourceEntries.end()) return nullptr;
else return Found->second;
}
bool CResourceStore::RegisterResource(const CUniqueID& rkID, EResType Type, const TWideString& rkDir, const TWideString& rkFileName)
{
CResourceEntry *pEntry = FindEntry(rkID);
if (pEntry)
{
Log::Error("Attempted to register resource that's already tracked in the database: " + rkID.ToString() + " / " + rkDir.ToUTF8() + " / " + rkFileName.ToUTF8());
return false;
}
else
{
pEntry = new CResourceEntry(this, rkID, rkDir, rkFileName.GetFileName(false), Type);
if (!pEntry->HasCookedVersion() && !pEntry->HasRawVersion())
{
Log::Error("Attempted to register a resource that doesn't exist: " + rkID.ToString() + " | " + rkDir.ToUTF8() + " | " + rkFileName.ToUTF8());
delete pEntry;
return false;
}
else
{
mResourceEntries[rkID] = pEntry;
return true;
}
}
}
CResourceEntry* CResourceStore::CreateTransientEntry(EResType Type, const TWideString& rkDir /*= L""*/, const TWideString& rkFileName /*= L""*/)
{
CResourceEntry *pEntry = new CResourceEntry(this, CUniqueID::RandomID(), rkDir, rkFileName, Type, true);
mResourceEntries[pEntry->ID()] = pEntry;
return pEntry;
}
CResourceEntry* CResourceStore::CreateTransientEntry(EResType Type, const CUniqueID& rkID, const TWideString& rkDir /*=L ""*/, const TWideString& rkFileName /*= L""*/)
{
CResourceEntry *pEntry = new CResourceEntry(this, rkID, rkDir, rkFileName, Type, true);
mResourceEntries[rkID] = pEntry;
return pEntry;
}
CResource* CResourceStore::LoadResource(const CUniqueID& rkID, const CFourCC& rkType)
{
if (!rkID.IsValid()) return nullptr;
// Check if resource is already loaded
auto Find = mLoadedResources.find(rkID);
if (Find != mLoadedResources.end())
return Find->second->Resource();
// With Game Exporter - Get data buffer from exporter
if (mpExporter)
{
std::vector<u8> DataBuffer;
mpExporter->LoadResource(rkID, DataBuffer);
if (DataBuffer.empty()) return nullptr;
CMemoryInStream MemStream(DataBuffer.data(), DataBuffer.size(), IOUtil::eBigEndian);
EResType Type = CResource::ResTypeForExtension(rkType);
CResourceEntry *pEntry = CreateTransientEntry(Type, rkID);
CResource *pRes = pEntry->Load(MemStream);
if (pRes) mLoadedResources[rkID] = pEntry;
return pRes;
}
// Without Game Exporter - Check store resource entries and transient load directory.
else
{
// Check for resource in store
CResourceEntry *pEntry = FindEntry(rkID);
if (pEntry) return pEntry->Load();
// Check in transient load directory
EResType Type = CResource::ResTypeForExtension(rkType);
if (Type != eInvalidResType)
{
TWideString Name = rkID.ToString().ToUTF16();
CResourceEntry *pEntry = CreateTransientEntry(Type, mTransientLoadDir, Name);
CResource *pRes = pEntry->Load();
if (pRes) mLoadedResources[rkID] = pEntry;
return pRes;
}
else
{
Log::Error("Can't load requested resource with ID \"" + rkID.ToString() + "\"; can't locate resource. Note: Loading raw assets from an arbitrary directory is unsupported.");;
return nullptr;
}
}
}
CResource* CResourceStore::LoadResource(const TString& rkPath)
{
// Construct ID from string, check if resource is loaded already
TWideString Dir = FileUtil::MakeAbsolute(TWideString(rkPath.GetFileDirectory()));
TString Name = rkPath.GetFileName(false);
CUniqueID ID = (Name.IsHexString() ? Name.ToInt64() : rkPath.Hash64());
auto Find = mLoadedResources.find(ID);
if (Find != mLoadedResources.end())
return Find->second->Resource();
// Determine type
TString Extension = rkPath.GetFileExtension().ToUpper();
EResType Type = CResource::ResTypeForExtension(Extension);
if (Type == eInvalidResType)
{
Log::Error("Unable to load resource " + rkPath + "; unrecognized extension: " + Extension);
return nullptr;
}
TString OldTransientDir = mTransientLoadDir;
mTransientLoadDir = Dir;
CResourceEntry *pEntry = CreateTransientEntry(Type, ID, Dir, Name);
CResource *pRes = pEntry->Load();
if (pRes) mLoadedResources[ID] = pEntry;
mTransientLoadDir = OldTransientDir;
return pRes;
}
CFourCC CResourceStore::ResourceTypeByID(const CUniqueID& rkID, const TStringList& rkPossibleTypes) const
{
if (!rkID.IsValid()) return eInvalidResType;
if (rkPossibleTypes.size() == 1) return CFourCC(rkPossibleTypes.front());
// Check for existing entry
auto Find = mResourceEntries.find(rkID);
if (Find != mResourceEntries.end())
return GetResourceCookedExtension(Find->second->ResourceType(), Find->second->Game());
// Determine extension from filesystem - try every extension until we find the file
TString PathBase = mTransientLoadDir.ToUTF8() + rkID.ToString() + '.';
for (auto It = rkPossibleTypes.begin(); It != rkPossibleTypes.end(); It++)
{
TString NewPath = PathBase + *It;
if (FileUtil::Exists(NewPath))
return CFourCC(*It);
}
// Couldn't find one, so return unknown. Note that it'd be possible to look up the extension from the
// filesystem even if it's not one of the provided possible types, but this would be too slow.
return "UNKN";
}
void CResourceStore::DestroyUnreferencedResources()
{
// This can be updated to avoid the do-while loop when reference lookup is implemented.
u32 NumDeleted;
do
{
NumDeleted = 0;
for (auto It = mLoadedResources.begin(); It != mLoadedResources.end(); It++)
{
CResourceEntry *pEntry = It->second;
if (!pEntry->Resource()->IsReferenced())
{
bool Unloaded = pEntry->Unload();
if (Unloaded)
{
It = mLoadedResources.erase(It);
NumDeleted++;
}
}
}
} while (NumDeleted > 0);
// Destroy empty virtual directories
for (auto DirIt = mTransientRoots.begin(); DirIt != mTransientRoots.end(); DirIt++)
{
CVirtualDirectory *pRoot = *DirIt;
if (pRoot->IsEmpty())
{
delete pRoot;
DirIt = mTransientRoots.erase(DirIt);
}
}
Log::Write(TString::FromInt32(mLoadedResources.size(), 0, 10) + " resources loaded");
}
void CResourceStore::SetTransientLoadDir(const TString& rkDir)
{
mTransientLoadDir = rkDir;
mTransientLoadDir.EnsureEndsWith('\\');
Log::Write("Set resource directory: " + rkDir);
}

View File

@@ -0,0 +1,65 @@
#ifndef CRESOURCEDATABASE_H
#define CRESOURCEDATABASE_H
#include "CVirtualDirectory.h"
#include "Core/Resource/EResType.h"
#include <Common/CFourCC.h>
#include <Common/CUniqueID.h>
#include <Common/TString.h>
#include <Common/types.h>
#include <map>
#include <set>
class CGameExporter;
class CGameProject;
class CResource;
class CResourceStore
{
CGameProject *mpProj;
CVirtualDirectory *mpProjectRoot;
std::vector<CVirtualDirectory*> mTransientRoots;
std::map<CUniqueID, CResourceEntry*> mResourceEntries;
std::map<CUniqueID, CResourceEntry*> mLoadedResources;
// Directory to look for transient resources in
TWideString mTransientLoadDir;
// Game exporter currently in use - lets us load from paks being exported
CGameExporter *mpExporter;
enum EDatabaseVersion
{
eVer_Initial,
eVer_Max,
eVer_Current = eVer_Max - 1
};
public:
CResourceStore();
~CResourceStore();
void LoadResourceDatabase(const TString& rkPath);
void SaveResourceDatabase(const TString& rkPath) const;
void SetActiveProject(CGameProject *pProj);
void CloseActiveProject();
CVirtualDirectory* GetVirtualDirectory(const TWideString& rkPath, bool Transient, bool AllowCreate);
bool RegisterResource(const CUniqueID& rkID, EResType Type, const TWideString& rkDir, const TWideString& rkFileName);
CResourceEntry* FindEntry(const CUniqueID& rkID) const;
CResourceEntry* CreateTransientEntry(EResType Type, const TWideString& rkDir = L"", const TWideString& rkFileName = L"");
CResourceEntry* CreateTransientEntry(EResType Type, const CUniqueID& rkID, const TWideString& rkDir = L"", const TWideString& rkFileName = L"");
CResource* LoadResource(const CUniqueID& rkID, const CFourCC& rkType);
CResource* LoadResource(const TString& rkPath);
CFourCC ResourceTypeByID(const CUniqueID& rkID, const TStringList& rkPossibleTypes) const;
void DestroyUnreferencedResources();
void SetTransientLoadDir(const TString& rkDir);
inline CGameProject* ActiveProject() const { return mpProj; }
inline void SetGameExporter(CGameExporter *pExporter) { mpExporter = pExporter; }
};
extern CResourceStore gResourceStore;
#endif // CRESOURCEDATABASE_H

View File

@@ -0,0 +1,156 @@
#include "CVirtualDirectory.h"
#include "CResourceEntry.h"
#include "CResourceStore.h"
#include <algorithm>
CVirtualDirectory::CVirtualDirectory()
: mpParent(nullptr)
{}
CVirtualDirectory::CVirtualDirectory(const TWideString& rkName)
: mpParent(nullptr), mName(rkName)
{}
CVirtualDirectory::CVirtualDirectory(CVirtualDirectory *pParent, const TWideString& rkName)
: mpParent(pParent), mName(rkName)
{}
CVirtualDirectory::~CVirtualDirectory()
{
for (u32 iSub = 0; iSub < mSubdirectories.size(); iSub++)
delete mSubdirectories[iSub];
}
bool CVirtualDirectory::IsEmpty() const
{
if (!mResources.empty()) return false;
for (u32 iSub = 0; iSub < mSubdirectories.size(); iSub++)
if (!mSubdirectories[iSub]->IsEmpty()) return false;
return true;
}
TWideString CVirtualDirectory::FullPath() const
{
return (mpParent && !mpParent->IsRoot() ? mpParent->FullPath() + L'\\' + mName + L"\\" : mName + L"\\");
}
CVirtualDirectory* CVirtualDirectory::GetRoot()
{
return (mpParent ? mpParent->GetRoot() : this);
}
CVirtualDirectory* CVirtualDirectory::FindChildDirectory(const TWideString& rkName, bool AllowCreate)
{
u32 SlashIdx = rkName.IndexOf(L"\\/");
TWideString DirName = (SlashIdx == -1 ? rkName : rkName.SubString(0, SlashIdx));
for (u32 iSub = 0; iSub < mSubdirectories.size(); iSub++)
{
CVirtualDirectory *pChild = mSubdirectories[iSub];
if (pChild->Name() == DirName)
{
if (SlashIdx == -1)
return pChild;
else
return pChild->FindChildDirectory( rkName.SubString(SlashIdx + 1, rkName.Size() - SlashIdx), AllowCreate );
}
}
if (AllowCreate)
{
AddChild(rkName, nullptr);
CVirtualDirectory *pOut = FindChildDirectory(rkName, false);
ASSERT(pOut != nullptr);
return pOut;
}
return nullptr;
}
void CVirtualDirectory::AddChild(const TWideString &rkPath, CResourceEntry *pEntry)
{
if (rkPath.IsEmpty())
{
if (pEntry)
{
#if !PUBLIC_RELEASE
for (u32 iRes = 0; iRes < mResources.size(); iRes++)
ASSERT(mResources[iRes] != pEntry);
#endif
mResources.push_back(pEntry);
}
}
else
{
u32 SlashIdx = rkPath.IndexOf(L"\\/");
TWideString DirName = (SlashIdx == -1 ? rkPath : rkPath.SubString(0, SlashIdx));
CVirtualDirectory *pSubdir = nullptr;
// Check if this subdirectory already exists
for (u32 iSub = 0; iSub < mSubdirectories.size(); iSub++)
{
if (mSubdirectories[iSub]->Name() == DirName)
{
pSubdir = mSubdirectories[iSub];
break;
}
}
if (!pSubdir)
{
pSubdir = new CVirtualDirectory(this, DirName);
mSubdirectories.push_back(pSubdir);
std::sort(mSubdirectories.begin(), mSubdirectories.end(), [](CVirtualDirectory *pLeft, CVirtualDirectory *pRight) -> bool {
return (pLeft->Name() < pRight->Name());
});
}
TWideString Remaining = (SlashIdx == -1 ? L"" : rkPath.SubString(SlashIdx + 1, rkPath.Size() - SlashIdx));
pSubdir->AddChild(Remaining, pEntry);
}
}
bool CVirtualDirectory::RemoveChildDirectory(CVirtualDirectory *pSubdir)
{
ASSERT(pSubdir->IsEmpty());
for (auto It = mSubdirectories.begin(); It != mSubdirectories.end(); It++)
{
if (*It == pSubdir)
{
mSubdirectories.erase(It);
delete pSubdir;
if (mpParent && IsEmpty())
mpParent->RemoveChildDirectory(this);
return true;
}
}
return false;
}
bool CVirtualDirectory::RemoveChildResource(CResourceEntry *pEntry)
{
for (auto It = mResources.begin(); It != mResources.end(); It++)
{
if (*It == pEntry)
{
mResources.erase(It);
if (mpParent && IsEmpty())
mpParent->RemoveChildDirectory(this);
return true;
}
}
return false;
}

View File

@@ -0,0 +1,44 @@
#ifndef CVIRTUALDIRECTORY
#define CVIRTUALDIRECTORY
/* Virtual directory system used to look up resources by their location in the filesystem. */
#include <Common/AssertMacro.h>
#include <Common/TString.h>
#include <vector>
class CResourceEntry;
class CVirtualDirectory
{
CVirtualDirectory *mpParent;
TWideString mName;
std::vector<CVirtualDirectory*> mSubdirectories;
std::vector<CResourceEntry*> mResources;
public:
CVirtualDirectory();
CVirtualDirectory(const TWideString& rkName);
CVirtualDirectory(CVirtualDirectory *pParent, const TWideString& rkName);
~CVirtualDirectory();
bool IsEmpty() const;
TWideString FullPath() const;
CVirtualDirectory* GetRoot();
CVirtualDirectory* FindChildDirectory(const TWideString& rkName, bool AllowCreate);
void AddChild(const TWideString& rkPath, CResourceEntry *pEntry);
bool RemoveChildDirectory(CVirtualDirectory *pSubdir);
bool RemoveChildResource(CResourceEntry *pEntry);
// Accessors
inline CVirtualDirectory* Parent() const { return mpParent; }
inline bool IsRoot() const { return !mpParent; }
inline TWideString Name() const { return mName; }
inline u32 NumSubdirectories() const { return mSubdirectories.size(); }
inline CVirtualDirectory* SubdirectoryByIndex(u32 Index) { return mSubdirectories[Index]; }
inline u32 NumResources() const { return mResources.size(); }
inline CResourceEntry* ResourceByIndex(u32 Index) { return mResources[Index]; }
};
#endif // CVIRTUALDIRECTORY