Applied a bunch of fixes to get the current game exporter functionality working with the resource store system

This commit is contained in:
parax0
2016-07-04 20:28:17 -06:00
parent 2f2ec13ced
commit 24c5ad5cd7
28 changed files with 318 additions and 174 deletions

View File

@@ -14,23 +14,25 @@
#define EXPORT_COOKED 1
CGameExporter::CGameExporter(const TString& rkInputDir, const TString& rkOutputDir)
: mStore(this)
{
mGameDir = FileUtil::MakeAbsolute(rkInputDir);
mExportDir = FileUtil::MakeAbsolute(rkOutputDir);
mpProject = new CGameProject(mExportDir);
mDiscDir = mpProject->DiscDir(true);
mResDir = mpProject->ResourcesDir(true);
mWorldsDir = mpProject->WorldsDir(true);
mContentDir = mpProject->ContentDir(false);
mCookedDir = mpProject->CookedDir(false);
mCookedResDir = mpProject->CookedResourcesDir(true);
mCookedWorldsDir = mpProject->CookedWorldsDir(true);
mWorldsDirName = L"Worlds\\";
mStore.SetActiveProject(mpProject);
}
bool CGameExporter::Export()
{
SCOPED_TIMER(ExportGame);
gResourceStore.SetGameExporter(this);
CResourceStore *pOldStore = gpResourceStore;
gpResourceStore = &mStore;
FileUtil::CreateDirectory(mExportDir);
FileUtil::ClearDirectory(mExportDir);
@@ -40,7 +42,7 @@ bool CGameExporter::Export()
ExportWorlds();
ExportCookedResources();
gResourceStore.SetGameExporter(nullptr);
gpResourceStore = pOldStore;
return true;
}
@@ -154,7 +156,7 @@ void CGameExporter::LoadAssetList()
TString Name = pName ? pName->GetText() : "";
if (!Dir.EndsWith("/") && !Dir.EndsWith("\\")) Dir.Append("\\");
SetResourcePath(ResourceID, mResDir + Dir.ToUTF16(), Name.ToUTF16());
SetResourcePath(ResourceID, Dir.ToUTF16(), Name.ToUTF16());
pAsset = pAsset->NextSiblingElement("Asset");
}
@@ -398,7 +400,6 @@ void CGameExporter::ExportWorlds()
{
#if EXPORT_WORLDS
SCOPED_TIMER(ExportWorlds);
//CResourceDatabase *pResDB = mpProject->ResourceDatabase();
for (u32 iPak = 0; iPak < mpProject->NumWorldPaks(); iPak++)
{
@@ -418,7 +419,8 @@ void CGameExporter::ExportWorlds()
if (rkRes.Type == "MLVL" && !rkRes.Name.EndsWith("NODEPEND"))
{
TResPtr<CWorld> pWorld = (CWorld*) gResourceStore.LoadResource(rkRes.ID, rkRes.Type);
// Load world
CWorld *pWorld = (CWorld*) mStore.LoadResource(rkRes.ID, rkRes.Type);
if (!pWorld)
{
@@ -428,7 +430,7 @@ void CGameExporter::ExportWorlds()
// Export world
TWideString Name = rkRes.Name.ToUTF16();
TWideString WorldDir = mWorldsDir + PakPath + FileUtil::SanitizeName(Name, true) + L"\\";
TWideString WorldDir = mWorldsDirName + PakPath + FileUtil::SanitizeName(Name, true) + L"\\";
FileUtil::CreateDirectory(mCookedDir + WorldDir);
SResourceInstance *pInst = FindResourceInstance(rkRes.ID);
@@ -442,7 +444,8 @@ void CGameExporter::ExportWorlds()
{
// Determine area names
TWideString InternalAreaName = pWorld->AreaInternalName(iArea).ToUTF16();
if (InternalAreaName.IsEmpty()) InternalAreaName = TWideString::FromInt32(iArea, 2, 10);
bool HasInternalName = !InternalAreaName.IsEmpty();
if (!HasInternalName) InternalAreaName = TWideString::FromInt32(iArea, 2, 10);
TWideString GameAreaName;
CStringTable *pTable = pWorld->AreaName(iArea);
@@ -451,7 +454,7 @@ void CGameExporter::ExportWorlds()
// Load area
CUniqueID AreaID = pWorld->AreaResourceID(iArea);
CGameArea *pArea = (CGameArea*) gResourceStore.LoadResource(AreaID, "MREA");
CGameArea *pArea = (CGameArea*) mStore.LoadResource(AreaID, "MREA");
if (!pArea)
{
@@ -466,11 +469,11 @@ void CGameExporter::ExportWorlds()
SResourceInstance *pInst = FindResourceInstance(AreaID);
ASSERT(pInst != nullptr);
SetResourcePath(AreaID, AreaDir, InternalAreaName);
SetResourcePath(AreaID, AreaDir, HasInternalName ? InternalAreaName : GameAreaName);
ExportResource(*pInst);
}
gResourceStore.DestroyUnreferencedResources();
mStore.DestroyUnreferencedResources();
}
else
@@ -485,10 +488,9 @@ void CGameExporter::ExportWorlds()
void CGameExporter::ExportCookedResources()
{
#if EXPORT_COOKED
gResourceStore.CloseActiveProject();
{
SCOPED_TIMER(ExportCookedResources);
FileUtil::CreateDirectory(mCookedDir + mResDir);
FileUtil::CreateDirectory(mCookedDir);
for (auto It = mResourceMap.begin(); It != mResourceMap.end(); It++)
{
@@ -498,7 +500,7 @@ void CGameExporter::ExportCookedResources()
}
{
SCOPED_TIMER(SaveResourceDatabase);
gResourceStore.SaveResourceDatabase(this->mExportDir.ToUTF8() + "ResourceDatabase.rdb");
mStore.SaveResourceDatabase(this->mExportDir.ToUTF8() + "ResourceDatabase.rdb");
}
#endif
}
@@ -512,27 +514,29 @@ void CGameExporter::ExportResource(SResourceInstance& rRes)
// Determine output path
SResourcePath *pPath = FindResourcePath(rRes.ResourceID);
TString OutName, OutDir;
TWideString OutName, OutDir;
if (pPath)
{
OutName = pPath->Name.ToUTF8();
OutDir = pPath->Dir.ToUTF8();
OutName = pPath->Name;
OutDir = pPath->Dir;
}
if (OutName.IsEmpty()) OutName = rRes.ResourceID.ToString();
if (OutDir.IsEmpty()) OutDir = mResDir;
if (OutName.IsEmpty()) OutName = rRes.ResourceID.ToString().ToUTF16();
if (OutDir.IsEmpty()) OutDir = L"Uncategorized\\";
// Write to file
FileUtil::CreateDirectory(mCookedDir + OutDir.ToUTF16());
TString OutPath = mCookedDir.ToUTF8() + OutDir + OutName + "." + rRes.ResourceType.ToString();
CFileOutStream Out(OutPath.ToStdString(), IOUtil::eBigEndian);
// Register resource and write to file
CResourceEntry *pEntry = mStore.RegisterResource(rRes.ResourceID, CResource::ResTypeForExtension(rRes.ResourceType), OutDir, OutName);
// Cooked (todo: save raw)
TWideString OutPath = pEntry->CookedAssetPath();
FileUtil::CreateDirectory(OutPath.GetFileDirectory());
CFileOutStream Out(OutPath.ToUTF8().ToStdString(), IOUtil::eBigEndian);
if (Out.IsValid())
Out.WriteBytes(ResourceData.data(), ResourceData.size());
// Add to resource DB
gResourceStore.RegisterResource(rRes.ResourceID, CResource::ResTypeForExtension(rRes.ResourceType), OutDir, OutName);
rRes.Exported = true;
ASSERT(pEntry->HasCookedVersion());
}
}

View File

@@ -2,6 +2,7 @@
#define CGAMEEXPORTER_H
#include "CGameProject.h"
#include "CResourceStore.h"
#include <Common/CUniqueID.h>
#include <Common/Flags.h>
#include <Common/TString.h>
@@ -12,16 +13,16 @@ class CGameExporter
{
// Project
CGameProject *mpProject;
CResourceStore mStore;
// Directories
TWideString mGameDir;
TWideString mExportDir;
TWideString mDiscDir;
TWideString mResDir;
TWideString mWorldsDir;
TWideString mContentDir;
TWideString mCookedDir;
TWideString mCookedResDir;
TWideString mCookedWorldsDir;
TWideString mWorldsDirName;
// Resources
TWideStringList mWorldPaks;

View File

@@ -31,11 +31,8 @@ public:
inline TWideString ProjectRoot() const { return mProjectRoot; }
inline TWideString ResourceDBPath(bool Relative) const { return Relative ? mResourceDBPath : mProjectRoot + mResourceDBPath; }
inline TWideString DiscDir(bool Relative) const { return Relative ? L"Disc\\" : mProjectRoot + L"Disc\\"; }
inline TWideString ResourcesDir(bool Relative) const { return Relative ? L"Resources\\" : mProjectRoot + L"Resources\\"; }
inline TWideString WorldsDir(bool Relative) const { return Relative ? L"Worlds\\" : mProjectRoot + L"Worlds\\"; }
inline TWideString ContentDir(bool Relative) const { return Relative ? L"Content\\" : mProjectRoot + L"Content\\"; }
inline TWideString CookedDir(bool Relative) const { return Relative ? L"Cooked\\" : mProjectRoot + L"Cooked\\"; }
inline TWideString CookedResourcesDir(bool Relative) const { return CookedDir(Relative) + L"Resources\\"; }
inline TWideString CookedWorldsDir(bool Relative) const { return CookedDir(Relative) + L"Worlds\\"; }
// Accessors
inline void SetGame(EGame Game) { mGame = Game; }

View File

@@ -29,7 +29,7 @@ CResourceEntry::CResourceEntry(CResourceStore *pStore, const CUniqueID& rkID,
: mpStore(pStore)
, mpResource(nullptr)
, mID(rkID)
, mFileName(rkFilename)
, mName(rkFilename)
, mType(Type)
, mNeedsRecook(false)
, mTransient(Transient)
@@ -58,16 +58,16 @@ TString CResourceEntry::RawAssetPath(bool Relative) const
{
TWideString Ext = GetResourceRawExtension(mType, mGame).ToUTF16();
TWideString Path = mpDirectory ? mpDirectory->FullPath() : L"";
TWideString Name = mFileName + L"." + Ext;
return ((mTransient || Relative) ? Path + Name : mpStore->ActiveProject()->ResourcesDir(false) + Path + Name);
TWideString Name = mName + L"." + Ext;
return ((mTransient || Relative) ? Path + Name : mpStore->ActiveProject()->ContentDir(false) + Path + Name);
}
TString CResourceEntry::CookedAssetPath(bool Relative) const
{
TWideString Ext = GetResourceCookedExtension(mType, mGame).ToUTF16();
TWideString Path = mpDirectory ? mpDirectory->FullPath() : L"";
TWideString Name = mFileName + L"." + Ext;
return ((mTransient || Relative) ? Path + Name : mpStore->ActiveProject()->CookedResourcesDir(false) + Path + Name);
TWideString Name = mName + L"." + Ext;
return ((mTransient || Relative) ? Path + Name : mpStore->ActiveProject()->CookedDir(false) + Path + Name);
}
bool CResourceEntry::NeedsRecook() const
@@ -147,3 +147,68 @@ bool CResourceEntry::Unload()
mpResource = nullptr;
return true;
}
void CResourceEntry::Move(const TWideString& rkDir, const TWideString& rkName)
{
// Store old paths
TString OldCookedPath = CookedAssetPath();
TString OldRawPath = RawAssetPath();
// Set new directory and name
bool HasDirectory = mpDirectory != nullptr;
CVirtualDirectory *pNewDir = mpStore->GetVirtualDirectory(rkDir, mTransient, true);
if (pNewDir != mpDirectory)
{
if (mpDirectory)
mpDirectory->RemoveChildResource(this);
mpDirectory = pNewDir;
}
if (mName != rkName)
ASSERT(mpDirectory->FindChildResource(rkName) == nullptr);
mName = rkName;
// Move files
if (HasDirectory)
{
TString CookedPath = CookedAssetPath();
TString RawPath = RawAssetPath();
if (FileUtil::Exists(OldCookedPath) && CookedPath != OldCookedPath)
FileUtil::MoveFile(OldCookedPath, CookedPath);
if (FileUtil::Exists(OldRawPath) && RawPath != OldRawPath)
FileUtil::MoveFile(OldRawPath, RawPath);
}
}
void CResourceEntry::AddToProject(const TWideString& rkDir, const TWideString& rkName)
{
if (mTransient)
{
mTransient = false;
Move(rkDir, rkName);
}
else
{
Log::Error("AddToProject called on non-transient resource entry: " + CookedAssetPath(true));
}
}
void CResourceEntry::RemoveFromProject()
{
if (!mTransient)
{
TString Dir = CookedAssetPath().GetFileDirectory();
mTransient = true;
Move(Dir, mName);
}
else
{
Log::Error("RemoveFromProject called on transient resource entry: " + CookedAssetPath(true));
}
}

View File

@@ -17,7 +17,7 @@ class CResourceEntry
EResType mType;
EGame mGame;
CVirtualDirectory *mpDirectory;
TWideString mFileName;
TWideString mName;
bool mNeedsRecook;
bool mTransient;
@@ -35,6 +35,9 @@ public:
CResource* Load();
CResource* Load(IInputStream& rInput);
bool Unload();
void Move(const TWideString& rkDir, const TWideString& rkName);
void AddToProject(const TWideString& rkDir, const TWideString& rkName);
void RemoveFromProject();
// Accessors
void SetDirty() { mNeedsRecook = true; }
@@ -44,7 +47,7 @@ public:
inline CUniqueID ID() const { return mID; }
inline EGame Game() const { return mGame; }
inline CVirtualDirectory* Directory() const { return mpDirectory; }
inline TString FileName() const { return mFileName; }
inline TWideString Name() const { return mName; }
inline EResType ResourceType() const { return mType; }
inline bool IsTransient() const { return mTransient; }

View File

@@ -8,7 +8,7 @@
#include <tinyxml2.h>
using namespace tinyxml2;
CResourceStore gResourceStore;
CResourceStore *gpResourceStore = new CResourceStore;
CResourceStore::CResourceStore()
: mpProj(nullptr)
@@ -16,8 +16,21 @@ CResourceStore::CResourceStore()
, mpExporter(nullptr)
{}
CResourceStore::CResourceStore(CGameExporter *pExporter)
: mpProj(nullptr)
, mpProjectRoot(nullptr)
, mpExporter(pExporter)
{}
CResourceStore::~CResourceStore()
{
CloseActiveProject();
for (auto It = mResourceEntries.begin(); It != mResourceEntries.end(); It++)
delete It->second;
for (auto It = mTransientRoots.begin(); It != mTransientRoots.end(); It++)
delete *It;
}
void CResourceStore::LoadResourceDatabase(const TString& rkPath)
@@ -122,7 +135,7 @@ void CResourceStore::SaveResourceDatabase(const TString& rkPath) const
pRes->LinkEndChild(pDir);
XMLElement *pName = Doc.NewElement("FileName");
pName->SetText(*pEntry->FileName());
pName->SetText(*pEntry->Name().ToUTF8());
pRes->LinkEndChild(pName);
XMLElement *pRecook = Doc.NewElement("NeedsRecook");
@@ -143,7 +156,9 @@ void CResourceStore::SetActiveProject(CGameProject *pProj)
if (pProj)
{
mpProjectRoot = new CVirtualDirectory();
LoadResourceDatabase(pProj->ResourceDBPath(false));
if (!mpExporter)
LoadResourceDatabase(pProj->ResourceDBPath(false));
}
}
@@ -157,6 +172,7 @@ void CResourceStore::CloseActiveProject()
{
delete pEntry;
It = mResourceEntries.erase(It);
if (It == mResourceEntries.end()) break;
}
}
@@ -203,33 +219,34 @@ CResourceEntry* CResourceStore::FindEntry(const CUniqueID& rkID) const
else return Found->second;
}
bool CResourceStore::RegisterResource(const CUniqueID& rkID, EResType Type, const TWideString& rkDir, const TWideString& rkFileName)
bool CResourceStore::IsResourceRegistered(const CUniqueID& rkID) const
{
return FindEntry(rkID) == nullptr;
}
CResourceEntry* CResourceStore::RegisterResource(const CUniqueID& rkID, EResType Type, const TWideString& rkDir, const TWideString& rkFileName)
{
CResourceEntry *pEntry = FindEntry(rkID);
if (pEntry)
{
Log::Error("Attempted to register resource that's already tracked in the database: " + rkID.ToString() + " / " + rkDir.ToUTF8() + " / " + rkFileName.ToUTF8());
return false;
if (pEntry->IsTransient())
{
ASSERT(pEntry->ResourceType() == Type);
pEntry->AddToProject(rkDir, rkFileName);
}
else
Log::Error("Attempted to register resource that's already tracked in the database: " + rkID.ToString() + " / " + rkDir.ToUTF8() + " / " + rkFileName.ToUTF8());
}
else
{
pEntry = new CResourceEntry(this, rkID, rkDir, rkFileName.GetFileName(false), Type);
if (!pEntry->HasCookedVersion() && !pEntry->HasRawVersion())
{
Log::Error("Attempted to register a resource that doesn't exist: " + rkID.ToString() + " | " + rkDir.ToUTF8() + " | " + rkFileName.ToUTF8());
delete pEntry;
return false;
}
else
{
mResourceEntries[rkID] = pEntry;
return true;
}
mResourceEntries[rkID] = pEntry;
}
return pEntry;
}
CResourceEntry* CResourceStore::RegisterTransientResource(EResType Type, const TWideString& rkDir /*= L""*/, const TWideString& rkFileName /*= L""*/)
@@ -241,8 +258,17 @@ CResourceEntry* CResourceStore::RegisterTransientResource(EResType Type, const T
CResourceEntry* CResourceStore::RegisterTransientResource(EResType Type, const CUniqueID& rkID, const TWideString& rkDir /*=L ""*/, const TWideString& rkFileName /*= L""*/)
{
CResourceEntry *pEntry = new CResourceEntry(this, rkID, rkDir, rkFileName, Type, true);
mResourceEntries[rkID] = pEntry;
CResourceEntry *pEntry = FindEntry(rkID);
if (pEntry)
Log::Error("Attempted to register transient resource that already exists: " + rkID.ToString() + " / Dir: " + rkDir.ToUTF8() + " / Name: " + rkFileName.ToUTF8());
else
{
pEntry = new CResourceEntry(this, rkID, rkDir, rkFileName, Type, true);
mResourceEntries[rkID] = pEntry;
}
return pEntry;
}
@@ -266,7 +292,12 @@ CResource* CResourceStore::LoadResource(const CUniqueID& rkID, const CFourCC& rk
EResType Type = CResource::ResTypeForExtension(rkType);
CResourceEntry *pEntry = RegisterTransientResource(Type, rkID);
CResource *pRes = pEntry->Load(MemStream);
if (pRes) mLoadedResources[rkID] = pEntry;
if (pRes)
{
mLoadedResources[rkID] = pEntry;
}
return pRes;
}
@@ -292,6 +323,7 @@ CResource* CResourceStore::LoadResource(const CUniqueID& rkID, const CFourCC& rk
CResource *pRes = pEntry->Load(File);
if (pRes) mLoadedResources[rkID] = pEntry;
else DeleteResourceEntry(pEntry);
return pRes;
}
@@ -341,6 +373,8 @@ CResource* CResourceStore::LoadResource(const TString& rkPath)
CResource *pRes = pEntry->Load(File);
if (pRes) mLoadedResources[ID] = pEntry;
else DeleteResourceEntry(pEntry);
mTransientLoadDir = OldTransientDir;
return pRes;
@@ -385,16 +419,16 @@ void CResourceStore::DestroyUnreferencedResources()
{
CResourceEntry *pEntry = It->second;
if (!pEntry->Resource()->IsReferenced())
if (!pEntry->Resource()->IsReferenced() && pEntry->Unload())
{
bool Unloaded = pEntry->Unload();
It = mLoadedResources.erase(It);
NumDeleted++;
if (Unloaded)
{
It = mLoadedResources.erase(It);
NumDeleted++;
if (It == mLoadedResources.end()) break;
}
// Transient resources should have their entries cleared out when the resource is unloaded
if (pEntry->IsTransient())
DeleteResourceEntry(pEntry);
if (It == mLoadedResources.end()) break;
}
}
} while (NumDeleted > 0);
@@ -414,6 +448,28 @@ void CResourceStore::DestroyUnreferencedResources()
Log::Write(TString::FromInt32(mLoadedResources.size(), 0, 10) + " resources loaded");
}
bool CResourceStore::DeleteResourceEntry(CResourceEntry *pEntry)
{
CUniqueID ID = pEntry->ID();
if (pEntry->IsLoaded())
{
if (!pEntry->Unload())
return false;
auto It = mLoadedResources.find(ID);
ASSERT(It != mLoadedResources.end());
mLoadedResources.erase(It);
}
auto It = mResourceEntries.find(ID);
ASSERT(It != mResourceEntries.end());
mResourceEntries.erase(It);
delete pEntry;
return true;
}
void CResourceStore::SetTransientLoadDir(const TString& rkDir)
{
mTransientLoadDir = rkDir;

View File

@@ -38,6 +38,7 @@ class CResourceStore
public:
CResourceStore();
CResourceStore(CGameExporter *pExporter);
~CResourceStore();
void LoadResourceDatabase(const TString& rkPath);
void SaveResourceDatabase(const TString& rkPath) const;
@@ -45,7 +46,8 @@ public:
void CloseActiveProject();
CVirtualDirectory* GetVirtualDirectory(const TWideString& rkPath, bool Transient, bool AllowCreate);
bool RegisterResource(const CUniqueID& rkID, EResType Type, const TWideString& rkDir, const TWideString& rkFileName);
bool IsResourceRegistered(const CUniqueID& rkID) const;
CResourceEntry* RegisterResource(const CUniqueID& rkID, EResType Type, const TWideString& rkDir, const TWideString& rkFileName);
CResourceEntry* FindEntry(const CUniqueID& rkID) const;
CResourceEntry* RegisterTransientResource(EResType Type, const TWideString& rkDir = L"", const TWideString& rkFileName = L"");
CResourceEntry* RegisterTransientResource(EResType Type, const CUniqueID& rkID, const TWideString& rkDir = L"", const TWideString& rkFileName = L"");
@@ -54,12 +56,13 @@ public:
CResource* LoadResource(const TString& rkPath);
CFourCC ResourceTypeByID(const CUniqueID& rkID, const TStringList& rkPossibleTypes) const;
void DestroyUnreferencedResources();
bool DeleteResourceEntry(CResourceEntry *pEntry);
void SetTransientLoadDir(const TString& rkDir);
inline CGameProject* ActiveProject() const { return mpProj; }
inline void SetGameExporter(CGameExporter *pExporter) { mpExporter = pExporter; }
// Accessors
inline CGameProject* ActiveProject() const { return mpProj; }
};
extern CResourceStore gResourceStore;
extern CResourceStore *gpResourceStore;
#endif // CRESOURCEDATABASE_H

View File

@@ -54,8 +54,16 @@ CVirtualDirectory* CVirtualDirectory::FindChildDirectory(const TWideString& rkNa
{
if (SlashIdx == -1)
return pChild;
else
return pChild->FindChildDirectory( rkName.SubString(SlashIdx + 1, rkName.Size() - SlashIdx), AllowCreate );
{
TWideString Remaining = rkName.SubString(SlashIdx + 1, rkName.Size() - SlashIdx);
if (Remaining.IsEmpty())
return pChild;
else
return pChild->FindChildDirectory(Remaining, AllowCreate);
}
}
}
@@ -70,6 +78,17 @@ CVirtualDirectory* CVirtualDirectory::FindChildDirectory(const TWideString& rkNa
return nullptr;
}
CResourceEntry* CVirtualDirectory::FindChildResource(const TWideString& rkName) const
{
for (u32 iRes = 0; iRes < mResources.size(); iRes++)
{
if (mResources[iRes]->Name() == rkName)
return mResources[iRes];
}
return nullptr;
}
void CVirtualDirectory::AddChild(const TWideString &rkPath, CResourceEntry *pEntry)
{
if (rkPath.IsEmpty())

View File

@@ -25,6 +25,7 @@ public:
TWideString FullPath() const;
CVirtualDirectory* GetRoot();
CVirtualDirectory* FindChildDirectory(const TWideString& rkName, bool AllowCreate);
CResourceEntry* FindChildResource(const TWideString& rkName) const;
void AddChild(const TWideString& rkPath, CResourceEntry *pEntry);
bool RemoveChildDirectory(CVirtualDirectory *pSubdir);
bool RemoveChildResource(CResourceEntry *pEntry);