AxioDL/PrimeWorldEditor
2
0
Fork 0
mirror of https://github.com/AxioDL/PrimeWorldEditor.git synced 2025-12-17 08:57:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

Modified all editor file formats to use the serialization system; changed dependency caching so all resource cache data is in one file

Browse Source
This commit is contained in:
parax0
2016-08-26 19:33:33 -06:00
parent 3dc0d71403
commit 20bddd5ed7
30 changed files with 565 additions and 658 deletions
Download Patch File Download Diff File Expand all files Collapse all files

159
src/Core/GameProject/CDependencyTree.cpp
View File

@@ -3,6 +3,8 @@
#include "Core/Resource/Script/CScriptLayer.h"
#include "Core/Resource/Script/CScriptObject.h"
CDependencyNodeFactory gDependencyNodeFactory;
// ************ IDependencyNode ************
IDependencyNode::~IDependencyNode()
{
@@ -27,28 +29,10 @@ EDependencyNodeType CDependencyTree::Type() const
return eDNT_DependencyTree;
}
void CDependencyTree::Read(IInputStream& rFile, EIDLength IDLength)
void CDependencyTree::Serialize(IArchive& rArc)
{
mRootID = CAssetID(rFile, IDLength);
u32 NumDepends = rFile.ReadLong();
mChildren.reserve(NumDepends);
for (u32 iDep = 0; iDep < NumDepends; iDep++)
{
CResourceDependency *pDepend = new CResourceDependency;
pDepend->Read(rFile, IDLength);
mChildren.push_back(pDepend);
}
}
void CDependencyTree::Write(IOutputStream& rFile) const
{
mRootID.Write(rFile);
rFile.WriteLong( mChildren.size() );
for (u32 iDep = 0; iDep < mChildren.size(); iDep++)
mChildren[iDep]->Write(rFile);
rArc << SERIAL("RootID", mRootID)
<< SERIAL_ABSTRACT_CONTAINER("Children", mChildren, "Child", &gDependencyNodeFactory);
}
void CDependencyTree::AddDependency(CResource *pRes, bool AvoidDuplicates /*= true*/)
@@ -70,14 +54,9 @@ EDependencyNodeType CResourceDependency::Type() const
return eDNT_ResourceDependency;
}
void CResourceDependency::Read(IInputStream& rFile, EIDLength IDLength)
void CResourceDependency::Serialize(IArchive& rArc)
{
mID = CAssetID(rFile, IDLength);
}
void CResourceDependency::Write(IOutputStream& rFile) const
{
mID.Write(rFile);
rArc << SERIAL("ID", mID);
}
bool CResourceDependency::HasDependency(const CAssetID& rkID) const
@@ -91,16 +70,10 @@ EDependencyNodeType CPropertyDependency::Type() const
return eDNT_ScriptProperty;
}
void CPropertyDependency::Read(IInputStream& rFile, EIDLength IDLength)
void CPropertyDependency::Serialize(IArchive& rArc)
{
mIDString = rFile.ReadString();
CResourceDependency::Read(rFile, IDLength);
}
void CPropertyDependency::Write(IOutputStream& rFile) const
{
rFile.WriteString(mIDString.ToStdString());
CResourceDependency::Write(rFile);
rArc << SERIAL("PropertyID", mIDString);
CResourceDependency::Serialize(rArc);
}
// ************ CCharacterPropertyDependency ************
@@ -109,16 +82,10 @@ EDependencyNodeType CCharPropertyDependency::Type() const
return eDNT_CharacterProperty;
}
void CCharPropertyDependency::Read(IInputStream& rFile, EIDLength IDLength)
void CCharPropertyDependency::Serialize(IArchive& rArc)
{
CPropertyDependency::Read(rFile, IDLength);
mUsedChar = rFile.ReadLong();
}
void CCharPropertyDependency::Write(IOutputStream& rFile) const
{
CPropertyDependency::Write(rFile);
rFile.WriteLong(mUsedChar);
CPropertyDependency::Serialize(rArc);
rArc << SERIAL("CharIndex", mUsedChar);
}
// ************ CScriptInstanceDependency ************
@@ -127,32 +94,10 @@ EDependencyNodeType CScriptInstanceDependency::Type() const
return eDNT_ScriptInstance;
}
void CScriptInstanceDependency::Read(IInputStream& rFile, EIDLength IDLength)
void CScriptInstanceDependency::Serialize(IArchive& rArc)
{
mObjectType = rFile.ReadLong();
u32 NumProperties = rFile.ReadLong();
mChildren.reserve(NumProperties);
for (u32 iProp = 0; iProp < NumProperties; iProp++)
{
bool IsCharacter = rFile.ReadBool();
CPropertyDependency *pProp = (IsCharacter ? new CCharPropertyDependency() : new CPropertyDependency());
pProp->Read(rFile, IDLength);
mChildren.push_back(pProp);
}
}
void CScriptInstanceDependency::Write(IOutputStream& rFile) const
{
rFile.WriteLong(mObjectType);
rFile.WriteLong(mChildren.size());
for (u32 iProp = 0; iProp < mChildren.size(); iProp++)
{
CPropertyDependency *pProp = static_cast<CPropertyDependency*>(mChildren[iProp]);
rFile.WriteBool( pProp->Type() == eDNT_CharacterProperty );
pProp->Write(rFile);
}
rArc << SERIAL("ObjectType", mObjectType)
<< SERIAL_ABSTRACT_CONTAINER("Properties", mChildren, "Property", &gDependencyNodeFactory);
}
// Static
@@ -216,23 +161,10 @@ EDependencyNodeType CAnimSetDependencyTree::Type() const
return eDNT_AnimSet;
}
void CAnimSetDependencyTree::Read(IInputStream& rFile, EIDLength IDLength)
void CAnimSetDependencyTree::Serialize(IArchive& rArc)
{
CDependencyTree::Read(rFile, IDLength);
u32 NumChars = rFile.ReadLong();
mCharacterOffsets.reserve(NumChars);
for (u32 iChar = 0; iChar < NumChars; iChar++)
mCharacterOffsets.push_back( rFile.ReadLong() );
}
void CAnimSetDependencyTree::Write(IOutputStream& rFile) const
{
CDependencyTree::Write(rFile);
rFile.WriteLong(mCharacterOffsets.size());
for (u32 iChar = 0; iChar < mCharacterOffsets.size(); iChar++)
rFile.WriteLong( mCharacterOffsets[iChar] );
CDependencyTree::Serialize(rArc);
rArc << SERIAL_CONTAINER("CharacterOffsets", mCharacterOffsets, "Offset");
}
void CAnimSetDependencyTree::AddCharacter(const SSetCharacter *pkChar, const std::set<CAssetID>& rkBaseUsedSet)
@@ -282,57 +214,10 @@ EDependencyNodeType CAreaDependencyTree::Type() const
return eDNT_Area;
}
void CAreaDependencyTree::Read(IInputStream& rFile, EIDLength IDLength)
void CAreaDependencyTree::Serialize(IArchive& rArc)
{
mRootID = CAssetID(rFile, IDLength);
// Base dependency list contains non-script dependencies (world geometry textures + PATH/PTLA/EGMC)
u32 NumBaseDependencies = rFile.ReadLong();
mChildren.reserve(NumBaseDependencies);
for (u32 iDep = 0; iDep < NumBaseDependencies; iDep++)
{
CResourceDependency *pDep = new CResourceDependency;
pDep->Read(rFile, IDLength);
mChildren.push_back(pDep);
}
u32 NumScriptInstances = rFile.ReadLong();
mChildren.reserve(mChildren.size() + NumScriptInstances);
for (u32 iInst = 0; iInst < NumScriptInstances; iInst++)
{
CScriptInstanceDependency *pInst = new CScriptInstanceDependency;
pInst->Read(rFile, IDLength);
mChildren.push_back(pInst);
}
u32 NumLayers = rFile.ReadLong();
mLayerOffsets.reserve(NumLayers);
for (u32 iLyr = 0; iLyr < NumLayers; iLyr++)
mLayerOffsets.push_back( rFile.ReadLong() );
}
void CAreaDependencyTree::Write(IOutputStream& rFile) const
{
mRootID.Write(rFile);
u32 NumBaseDependencies = (mLayerOffsets.empty() ? mChildren.size() : mLayerOffsets.front());
rFile.WriteLong(NumBaseDependencies);
for (u32 iDep = 0; iDep < NumBaseDependencies; iDep++)
mChildren[iDep]->Write(rFile);
rFile.WriteLong(mChildren.size() - NumBaseDependencies);
for (u32 iDep = NumBaseDependencies; iDep < mChildren.size(); iDep++)
mChildren[iDep]->Write(rFile);
rFile.WriteLong(mLayerOffsets.size());
for (u32 iLyr = 0; iLyr < mLayerOffsets.size(); iLyr++)
rFile.WriteLong(mLayerOffsets[iLyr]);
CDependencyTree::Serialize(rArc);
rArc << SERIAL_CONTAINER("LayerOffsets", mLayerOffsets, "Offset");
}
void CAreaDependencyTree::AddScriptLayer(CScriptLayer *pLayer)

62
src/Core/GameProject/CDependencyTree.h
View File

@@ -14,13 +14,13 @@ struct SSetCharacter;
// Group of node classes forming a tree of cached resource dependencies.
enum EDependencyNodeType
{
eDNT_DependencyTree,
eDNT_ResourceDependency,
eDNT_ScriptInstance,
eDNT_ScriptProperty,
eDNT_CharacterProperty,
eDNT_AnimSet,
eDNT_Area,
eDNT_DependencyTree = FOURCC_CONSTEXPR('T', 'R', 'E', 'E'),
eDNT_ResourceDependency = FOURCC_CONSTEXPR('R', 'S', 'D', 'P'),
eDNT_ScriptInstance = FOURCC_CONSTEXPR('S', 'C', 'I', 'N'),
eDNT_ScriptProperty = FOURCC_CONSTEXPR('S', 'C', 'P', 'R'),
eDNT_CharacterProperty = FOURCC_CONSTEXPR('C', 'R', 'P', 'R'),
eDNT_AnimSet = FOURCC_CONSTEXPR('A', 'N', 'C', 'S'),
eDNT_Area = FOURCC_CONSTEXPR('A', 'R', 'E', 'A'),
};
// Base class providing an interface for a basic dependency node.
@@ -32,8 +32,7 @@ protected:
public:
virtual ~IDependencyNode();
virtual EDependencyNodeType Type() const = 0;
virtual void Read(IInputStream& rFile, EIDLength IDLength) = 0;
virtual void Write(IOutputStream& rFile) const = 0;
virtual void Serialize(IArchive& rArc) = 0;
virtual bool HasDependency(const CAssetID& rkID) const;
// Accessors
@@ -48,11 +47,11 @@ protected:
CAssetID mRootID;
public:
CDependencyTree() {}
CDependencyTree(const CAssetID& rkID) : mRootID(rkID) {}
virtual EDependencyNodeType Type() const;
virtual void Read(IInputStream& rFile, EIDLength IDLength);
virtual void Write(IOutputStream& rFile) const;
virtual void Serialize(IArchive& rArc);
void AddDependency(const CAssetID& rkID, bool AvoidDuplicates = true);
void AddDependency(CResource *pRes, bool AvoidDuplicates = true);
@@ -73,8 +72,7 @@ public:
CResourceDependency(const CAssetID& rkID) : mID(rkID) {}
virtual EDependencyNodeType Type() const;
virtual void Read(IInputStream& rFile, EIDLength IDLength);
virtual void Write(IOutputStream& rFile) const;
virtual void Serialize(IArchive& rArc);
virtual bool HasDependency(const CAssetID& rkID) const;
// Accessors
@@ -98,8 +96,7 @@ public:
{}
virtual EDependencyNodeType Type() const;
virtual void Read(IInputStream& rFile, EIDLength IDLength);
virtual void Write(IOutputStream& rFile) const;
virtual void Serialize(IArchive& rArc);
// Accessors
inline TString PropertyID() const { return mIDString; }
@@ -123,8 +120,7 @@ public:
{}
virtual EDependencyNodeType Type() const;
virtual void Read(IInputStream& rFile, EIDLength IDLength);
virtual void Write(IOutputStream& rFile) const;
virtual void Serialize(IArchive& rArc);
// Accessors
inline u32 UsedChar() const { return mUsedChar; }
@@ -138,8 +134,7 @@ protected:
public:
virtual EDependencyNodeType Type() const;
virtual void Read(IInputStream& rFile, EIDLength IDLength);
virtual void Write(IOutputStream& rFile) const;
virtual void Serialize(IArchive& rArc);
// Accessors
inline u32 ObjectType() const { return mObjectType; }
@@ -157,10 +152,10 @@ protected:
std::vector<u32> mCharacterOffsets;
public:
CAnimSetDependencyTree() : CDependencyTree() {}
CAnimSetDependencyTree(const CAssetID& rkID) : CDependencyTree(rkID) {}
virtual EDependencyNodeType Type() const;
virtual void Read(IInputStream& rFile, EIDLength IDLength);
virtual void Write(IOutputStream& rFile) const;
virtual void Serialize(IArchive& rArc);
void AddCharacter(const SSetCharacter *pkChar, const std::set<CAssetID>& rkBaseUsedSet);
void AddCharDependency(const CAssetID& rkID, std::set<CAssetID>& rUsedSet);
@@ -178,11 +173,11 @@ protected:
std::vector<u32> mLayerOffsets;
public:
CAreaDependencyTree() : CDependencyTree() {}
CAreaDependencyTree(const CAssetID& rkID) : CDependencyTree(rkID) {}
virtual EDependencyNodeType Type() const;
virtual void Read(IInputStream& rFile, EIDLength IDLength);
virtual void Write(IOutputStream& rFile) const;
virtual void Serialize(IArchive& rArc);
void AddScriptLayer(CScriptLayer *pLayer);
void GetModuleDependencies(EGame Game, std::vector<TString>& rModuleDepsOut, std::vector<u32>& rModuleLayerOffsetsOut) const;
@@ -192,5 +187,26 @@ public:
inline u32 ScriptLayerOffset(u32 LayerIdx) const { return mLayerOffsets[LayerIdx]; }
};
// Dependency node factory for serialization
class CDependencyNodeFactory
{
public:
IDependencyNode* SpawnObject(u32 NodeID)
{
switch (NodeID)
{
case eDNT_DependencyTree: return new CDependencyTree;
case eDNT_ResourceDependency: return new CResourceDependency;
case eDNT_ScriptInstance: return new CScriptInstanceDependency;
case eDNT_ScriptProperty: return new CPropertyDependency;
case eDNT_CharacterProperty: return new CCharPropertyDependency;
case eDNT_AnimSet: return new CAnimSetDependencyTree;
case eDNT_Area: return new CAreaDependencyTree;
default: return nullptr;
}
}
};
extern CDependencyNodeFactory gDependencyNodeFactory;
#endif // CDEPENDENCYTREE

14
src/Core/GameProject/CGameExporter.cpp
View File

@@ -526,7 +526,7 @@ void CGameExporter::ExportCookedResources()
{
SCOPED_TIMER(SaveResourceDatabase);
#if EXPORT_COOKED
mStore.SaveResourceDatabase(mpProject->ResourceDBPath(false).ToUTF8());
mStore.SaveResourceDatabase();
#endif
mpProject->Save();
}
@@ -537,6 +537,9 @@ void CGameExporter::ExportCookedResources()
// some resources will fail to load if their dependencies don't exist
SCOPED_TIMER(SaveRawResources);
// todo: we're wasting a ton of time loading the same resources over and over because most resources automatically
// load all their dependencies and then we just clear it out from memory even though we'll need it again later. we
// should really be doing this by dependency order instead of by ID order.
for (CResourceIterator It(&mStore); It; ++It)
{
if (!It->IsTransient())
@@ -556,11 +559,16 @@ void CGameExporter::ExportCookedResources()
}
}
// Save raw resource + cache data
It->Save();
// Save raw resource + generate dependencies
It->Save(true);
}
}
}
{
// All resources should have dependencies generated, so save the cache file
SCOPED_TIMER(SaveResourceCacheData);
mStore.SaveCacheFile();
}
}
void CGameExporter::ExportResource(SResourceInstance& rRes)

124
src/Core/GameProject/CGameProject.cpp
View File

@@ -1,8 +1,7 @@
#include "CGameProject.h"
#include "Core/Resource/Script/CMasterTemplate.h"
#include <tinyxml2.h>
#include <Common/Serialization/XML.h>
using namespace tinyxml2;
CGameProject *CGameProject::mspActiveProject = nullptr;
CGameProject::~CGameProject()
@@ -16,107 +15,56 @@ CGameProject::~CGameProject()
bool CGameProject::Load(const TWideString& rkPath)
{
TString ProjPath = rkPath.ToUTF8();
XMLDocument Doc;
Doc.LoadFile(*ProjPath);
if (Doc.Error())
{
Log::Error("Unable to open game project at " + ProjPath);
return false;
}
XMLElement *pRoot = Doc.FirstChildElement("GameProject");
//EProjectVersion Version = (EProjectVersion) TString(pRoot->Attribute("Version")).ToInt32(10);
// Verify all elements are present
XMLElement *pProjName = pRoot->FirstChildElement("Name");
XMLElement *pGame = pRoot->FirstChildElement("Game");
XMLElement *pResDB = pRoot->FirstChildElement("ResourceDB");
XMLElement *pPackages = pRoot->FirstChildElement("Packages");
if (!pProjName || !pGame || !pResDB || !pPackages)
{
TString MissingElem = pProjName ? (pGame ? (pResDB ? "Packages" : "ResourceDB") : "Game") : "Name";
Log::Error("Unable to load game project at " + ProjPath + "; " + MissingElem + " element is missing");
return false;
}
mProjectName = pProjName->GetText();
mGame = CMasterTemplate::FindGameForName( pGame->GetText() );
mResourceDBPath = pResDB->GetText();
mProjectRoot = rkPath.GetFileDirectory();
mProjectRoot.Replace(L"/", L"\\");
// Load packages
XMLElement *pPkgElem = pPackages->FirstChildElement("Package");
while (pPkgElem)
{
TString Path = pPkgElem->Attribute("Path");
if (Path.IsEmpty())
Log::Error("Failed to load package in game project " + ProjPath + "; Path attribute is missing or empty");
else
{
CPackage *pPackage = new CPackage(this, Path.GetFileName(false), TString(Path.GetFileDirectory()).ToUTF16());
pPackage->Load();
mPackages.push_back(pPackage);
}
pPkgElem = pPkgElem->NextSiblingElement("Package");
}
// All loaded!
TString ProjPath = rkPath.ToUTF8();
CXMLReader Reader(ProjPath);
Serialize(Reader);
return true;
}
void CGameProject::Save()
{
XMLDocument Doc;
TString ProjPath = ProjectPath().ToUTF8();
CXMLWriter Writer(ProjPath, "GameProject", eVer_Current, mGame);
Serialize(Writer);
}
XMLDeclaration *pDecl = Doc.NewDeclaration();
Doc.LinkEndChild(pDecl);
void CGameProject::Serialize(IArchive& rArc)
{
rArc << SERIAL("Name", mProjectName)
<< SERIAL("Game", mGame)
<< SERIAL("ResourceDB", mResourceDBPath);
XMLElement *pRoot = Doc.NewElement("GameProject");
pRoot->SetAttribute("Version", eVer_Current);
Doc.LinkEndChild(pRoot);
// Packages
std::vector<TString> PackageList;
XMLElement *pProjName = Doc.NewElement("Name");
pProjName->SetText(*mProjectName);
pRoot->LinkEndChild(pProjName);
XMLElement *pGame = Doc.NewElement("Game");
pGame->SetText(*CMasterTemplate::FindGameName(mGame));
pRoot->LinkEndChild(pGame);
XMLElement *pResDB = Doc.NewElement("ResourceDB");
pResDB->SetText(*mResourceDBPath.ToUTF8());
pRoot->LinkEndChild(pResDB);
XMLElement *pPackages = Doc.NewElement("Packages");
pRoot->LinkEndChild(pPackages);
for (u32 iPkg = 0; iPkg < mPackages.size(); iPkg++)
if (!rArc.IsReader())
{
CPackage *pPackage = mPackages[iPkg];
TWideString FullDefPath = pPackage->DefinitionPath(false);
TWideString RelDefPath = FileUtil::MakeRelative(FullDefPath.GetFileDirectory(), PackagesDir(false));
TString DefPath = TWideString(RelDefPath + FullDefPath.GetFileName()).ToUTF8();
XMLElement *pPakElem = Doc.NewElement("Package");
pPakElem->SetAttribute("Path", *DefPath);
pPackages->LinkEndChild(pPakElem);
for (u32 iPkg = 0; iPkg < mPackages.size(); iPkg++)
PackageList.push_back( mPackages[iPkg]->DefinitionPath(true).ToUTF8() );
}
// Save Project
TString ProjPath = ProjectPath().ToUTF8();
XMLError Result = Doc.SaveFile(*ProjPath);
rArc << SERIAL_CONTAINER("Packages", PackageList, "Package");
if (Result != XML_SUCCESS)
Log::Error("Failed to save game project at: " + ProjPath);
if (rArc.IsReader())
{
for (u32 iPkg = 0; iPkg < mPackages.size(); iPkg++)
delete mPackages[iPkg];
mPackages.clear();
for (u32 iPkg = 0; iPkg < PackageList.size(); iPkg++)
{
const TWideString& rkPackagePath = PackageList[iPkg];
TString PackageName = TWideString(rkPackagePath.GetFileName(false)).ToUTF8();
TWideString PackageDir = rkPackagePath.GetFileDirectory();
CPackage *pPackage = new CPackage(this, PackageName, PackageDir);
pPackage->Load();
mPackages.push_back(pPackage);
}
}
}
void CGameProject::SetActive()

2
src/Core/GameProject/CGameProject.h
View File

@@ -46,6 +46,7 @@ public:
bool Load(const TWideString& rkPath);
void Save();
void Serialize(IArchive& rArc);
void SetActive();
void GetWorldList(std::list<CAssetID>& rOut) const;
@@ -58,6 +59,7 @@ public:
inline TWideString CookedDir(bool Relative) const { return Relative ? L"Cooked\\" : mProjectRoot + L"Cooked\\"; }
inline TWideString PackagesDir(bool Relative) const { return Relative ? L"Packages\\" : mProjectRoot + L"Packages\\"; }
inline TWideString ProjectPath() const { return mProjectRoot + FileUtil::SanitizeName(mProjectName.ToUTF16(), false) + L".prj"; }
inline TWideString ResourceCachePath(bool Relative) const { return ResourceDBPath(Relative).GetFileDirectory() + L"ResourceCacheData.rcd"; }
// Accessors
inline void SetGame(EGame Game) { mGame = Game; }

103
src/Core/GameProject/CPackage.cpp
View File

@@ -6,58 +6,15 @@
#include <Common/AssertMacro.h>
#include <Common/CompressionUtil.h>
#include <Common/FileUtil.h>
#include <tinyxml2.h>
#include <Common/Serialization/XML.h>
using namespace tinyxml2;
void CPackage::Load()
{
TWideString DefPath = DefinitionPath(false);
XMLDocument Doc;
Doc.LoadFile(*DefPath.ToUTF8());
if (Doc.Error())
{
Log::Error("Couldn't open pak definition at path: " + DefPath.ToUTF8());
return;
}
XMLElement *pRoot = Doc.FirstChildElement("PackageDefinition");
//EPackageDefinitionVersion Version = (EPackageDefinitionVersion) TString(pRoot->Attribute("Version")).ToInt32(10);
XMLElement *pColElem = pRoot->FirstChildElement("ResourceCollection");
while (pColElem)
{
CResourceCollection *pCollection = AddCollection( pColElem->Attribute("Name") );
XMLElement *pResElem = pColElem->FirstChildElement("NamedResource");
while (pResElem)
{
XMLElement *pNameElem = pResElem->FirstChildElement("Name");
XMLElement *pIDElem = pResElem->FirstChildElement("ID");
XMLElement *pTypeElem = pResElem->FirstChildElement("Type");
if (!pIDElem || !pNameElem || !pTypeElem)
{
TString ElemName = (pNameElem ? (pIDElem ? "Type" : "ID") : "Name");
Log::Error("Can't add named resource from pak definition at " + DefPath.ToUTF8() + "; " + ElemName + " element missing");
}
else
{
CAssetID ID = CAssetID::FromString(pIDElem->GetText());
TString Name = pNameElem->GetText();
CFourCC Type = CFourCC(pTypeElem->GetText());
pCollection->AddResource(Name, ID, Type);
}
pResElem = pResElem->NextSiblingElement("NamedResource");
}
pColElem = pColElem->NextSiblingElement("ResourceCollection");
}
CXMLReader Reader(DefPath.ToUTF8());
Serialize(Reader);
}
void CPackage::Save()
@@ -65,49 +22,13 @@ void CPackage::Save()
TWideString DefPath = DefinitionPath(false);
FileUtil::CreateDirectory(DefPath.GetFileDirectory());
// Write XML
XMLDocument Doc;
CXMLWriter Writer(DefPath.ToUTF8(), "PackageDefinition", 0, mpProject ? mpProject->Game() : eUnknownGame);
Serialize(Writer);
}
XMLDeclaration *pDecl = Doc.NewDeclaration();
Doc.LinkEndChild(pDecl);
XMLElement *pRoot = Doc.NewElement("PackageDefinition");
pRoot->SetAttribute("Version", eVer_Current);
Doc.LinkEndChild(pRoot);
for (u32 iCol = 0; iCol < mCollections.size(); iCol++)
{
CResourceCollection *pCollection = mCollections[iCol];
XMLElement *pColElem = Doc.NewElement("ResourceCollection");
pColElem->SetAttribute("Name", *pCollection->Name());
pRoot->LinkEndChild(pColElem);
for (u32 iRes = 0; iRes < pCollection->NumResources(); iRes++)
{
const SNamedResource& rkRes = pCollection->ResourceByIndex(iRes);
XMLElement *pResElem = Doc.NewElement("NamedResource");
pColElem->LinkEndChild(pResElem);
XMLElement *pName = Doc.NewElement("Name");
pName->SetText(*rkRes.Name);
pResElem->LinkEndChild(pName);
XMLElement *pID = Doc.NewElement("ID");
pID->SetText(*rkRes.ID.ToString());
pResElem->LinkEndChild(pID);
XMLElement *pType = Doc.NewElement("Type");
pType->SetText(*rkRes.Type.ToString());
pResElem->LinkEndChild(pType);
}
}
XMLError Error = Doc.SaveFile(*DefPath.ToUTF8());
if (Error != XML_SUCCESS)
Log::Error("Failed to save pak definition at path: " + DefPath.ToUTF8());
void CPackage::Serialize(IArchive& rArc)
{
rArc << SERIAL_CONTAINER("Collections", mCollections, "ResourceCollection");
}
void CPackage::Cook()
@@ -355,12 +276,14 @@ void CPackage::CompareOriginalAssetList(const std::list<CAssetID>& rkNewList)
TWideString CPackage::DefinitionPath(bool Relative) const
{
return mpProject->PackagesDir(Relative) + mPakPath + mPakName.ToUTF16() + L".pkd";
TWideString RelPath = mPakPath + mPakName.ToUTF16() + L".pkd";
return Relative ? RelPath : mpProject->PackagesDir(false) + RelPath;
}
TWideString CPackage::CookedPackagePath(bool Relative) const
{
return mpProject->DiscDir(Relative) + mPakPath + mPakName.ToUTF16() + L".pak";
TWideString RelPath = mPakPath + mPakName.ToUTF16() + L".pak";
return Relative ? RelPath : mpProject->DiscDir(false) + RelPath;
}
CResourceCollection* CPackage::AddCollection(const TString& rkName)

12
src/Core/GameProject/CPackage.h
View File

@@ -4,6 +4,7 @@
#include <Common/CAssetID.h>
#include <Common/CFourCC.h>
#include <Common/TString.h>
#include <Common/Serialization/IArchive.h>
class CGameProject;
@@ -12,6 +13,11 @@ struct SNamedResource
TString Name;
CAssetID ID;
CFourCC Type;
void Serialize(IArchive& rArc)
{
rArc << SERIAL_AUTO(Name) << SERIAL_AUTO(ID) << SERIAL_AUTO(Type);
}
};
class CResourceCollection
@@ -23,6 +29,11 @@ public:
CResourceCollection() : mName("UNNAMED") {}
CResourceCollection(const TString& rkName) : mName(rkName) {}
void Serialize(IArchive& rArc)
{
rArc << SERIAL("Name", mName) << SERIAL_CONTAINER("Resources", mNamedResources, "Resource");
}
inline TString Name() const { return mName; }
inline u32 NumResources() const { return mNamedResources.size(); }
inline const SNamedResource& ResourceByIndex(u32 Idx) const { return mNamedResources[Idx]; }
@@ -59,6 +70,7 @@ public:
void Load();
void Save();
void Serialize(IArchive& rArc);
void Cook();
void CompareOriginalAssetList(const std::list<CAssetID>& rkNewList);

106
src/Core/GameProject/CResourceEntry.cpp
View File

@@ -34,87 +34,18 @@ CResourceEntry::~CResourceEntry()
if (mpDependencies) delete mpDependencies;
}
bool CResourceEntry::LoadCacheData()
void CResourceEntry::SerializeCacheData(IArchive& rArc)
{
ASSERT(!IsTransient());
TWideString Path = CacheDataPath(false);
CFileInStream File(Path.ToUTF8().ToStdString(), IOUtil::eLittleEndian);
u32 Flags = mFlags & eREF_SavedFlags;
rArc << SERIAL_AUTO(Flags);
if (rArc.IsReader()) mFlags = Flags & eREF_SavedFlags;
if (!File.IsValid())
{
Log::Error("Unable to load cache data " + Path.ToUTF8() + "; couldn't open file");
return false;
}
// Header
TString Magic = File.ReadString(4);
ASSERT(Magic == "CACH");
File.Seek(0x4, SEEK_CUR); // Skip Version
mFlags = File.ReadLong() & eREF_SavedFlags;
// Dependency Tree
u32 DepsTreeSize = File.ReadLong();
u32 DepsTreeStart = File.Tell();
if (mpDependencies)
{
delete mpDependencies;
mpDependencies = nullptr;
}
if (DepsTreeSize > 0)
{
if (mType == eArea) mpDependencies = new CAreaDependencyTree(mID);
else if (mType == eAnimSet) mpDependencies = new CAnimSetDependencyTree(mID);
else mpDependencies = new CDependencyTree(mID);
mpDependencies->Read(File, Game() <= eEchoes ? e32Bit : e64Bit);
ASSERT(File.Tell() - DepsTreeStart == DepsTreeSize);
}
return true;
}
bool CResourceEntry::SaveCacheData()
{
ASSERT(!IsTransient());
TWideString Path = CacheDataPath(false);
TWideString Dir = Path.GetFileDirectory();
FileUtil::CreateDirectory(Dir);
CFileOutStream File(Path.ToUTF8().ToStdString(), IOUtil::eLittleEndian);
if (!File.IsValid())
{
Log::Error("Unable to save cache data " + TString(Path.GetFileName()) + "; couldn't open file");
return false;
}
// Header
File.WriteString("CACH", 4);
File.WriteLong(0); // Reserved Space (Version)
File.WriteLong(mFlags & eREF_SavedFlags);
// Dependency Tree
if (!mpDependencies) UpdateDependencies();
u32 DepsSizeOffset = File.Tell();
File.WriteLong(0);
u32 DepsStart = File.Tell();
if (mpDependencies) mpDependencies->Write(File);
u32 DepsEnd = File.Tell();
u32 DepsSize = DepsEnd- DepsStart;
File.Seek(DepsSizeOffset, SEEK_SET);
File.WriteLong(DepsSize);
File.Seek(DepsEnd, SEEK_SET);
// Thumbnail
File.WriteLong(0); // Reserved Space (Thumbnail Size)
return true;
// Note: If the dependency tree format is changed this should be adjusted so that
// we regenerate the dependencies from scratch instead of reading the tree if the
// file version number is too low
rArc << SERIAL_ABSTRACT("Dependencies", mpDependencies, &gDependencyNodeFactory);
}
void CResourceEntry::UpdateDependencies()
@@ -125,6 +56,14 @@ void CResourceEntry::UpdateDependencies()
mpDependencies = nullptr;
}
// todo: more robust method of skipping dependency updates. For now just skip the most
// time-consuming type that can't have dependencies (textures).
if (ResourceType() == eTexture)
{
mpDependencies = new CDependencyTree(ID());
return;
}
bool WasLoaded = IsLoaded();
if (!mpResource)
@@ -225,8 +164,13 @@ void CResourceEntry::SetGame(EGame NewGame)
}
}
bool CResourceEntry::Save()
bool CResourceEntry::Save(bool SkipCacheSave /*= false*/)
{
// SkipCacheSave argument tells us not to save the resource cache file. This is generally not advised because we don't
// want the actual resource data to desync from the cache data. However, there are occasions where we save multiple
// resources at a time and in that case it's preferable to only save the cache once. If you do set SkipCacheSave to true,
// then make sure you manually update the cache file afterwards.
//
// For now, always save the resource when this function is called even if there's been no changes made to it in memory.
// In the future this might not be desired behavior 100% of the time.
// We also might want this function to trigger a cook for certain resource types eventually.
@@ -249,9 +193,11 @@ bool CResourceEntry::Save()
mpResource->Serialize(Writer);
}
// Resource has been saved, now update cache file
// Resource has been saved, now update dependencies + cache file
UpdateDependencies();
SaveCacheData();
if (!SkipCacheSave)
mpStore->SaveCacheFile();
if (ShouldCollectGarbage)
mpStore->DestroyUnreferencedResources();

5
src/Core/GameProject/CResourceEntry.h
View File

@@ -44,8 +44,7 @@ public:
EResType Type, bool Transient = false);
~CResourceEntry();
bool LoadCacheData();
bool SaveCacheData();
void SerializeCacheData(IArchive& rArc);
void UpdateDependencies();
TWideString CacheDataPath(bool Relative = false) const;
@@ -58,7 +57,7 @@ public:
u64 Size() const;
bool NeedsRecook() const;
void SetGame(EGame NewGame);
bool Save();
bool Save(bool SkipCacheSave = false);
CResource* Load();
CResource* LoadCooked(IInputStream& rInput);
bool Unload();

212
src/Core/GameProject/CResourceStore.cpp
View File

@@ -1,10 +1,13 @@
#include "CResourceStore.h"
#include "CGameExporter.h"
#include "CGameProject.h"
#include "CResourceIterator.h"
#include "Core/Resource/CResource.h"
#include <Common/AssertMacro.h>
#include <Common/FileUtil.h>
#include <Common/Log.h>
#include <Common/Serialization/Binary.h>
#include <Common/Serialization/XML.h>
#include <tinyxml2.h>
using namespace tinyxml2;
@@ -34,92 +37,151 @@ CResourceStore::~CResourceStore()
delete *It;
}
void CResourceStore::LoadResourceDatabase(const TString& rkPath)
void CResourceStore::SerializeResourceDatabase(IArchive& rArc)
{
XMLDocument Doc;
Doc.LoadFile(*rkPath);
if (!Doc.Error())
struct SDatabaseResource
{
XMLElement *pRoot = Doc.FirstChildElement("ResourceDatabase");
//EDatabaseVersion Version = (EDatabaseVersion) TString(pRoot->Attribute("Version")).ToInt32(10); // Version currently unused
CAssetID ID;
CFourCC Type;
TWideString Directory;
TWideString Name;
XMLElement *pResources = pRoot->FirstChildElement("Resources");
XMLElement *pRes = pResources->FirstChildElement("Resource");
u32 ResIndex = 0;
while (pRes)
void Serialize(IArchive& rArc)
{
XMLElement *pID = pRes->FirstChildElement("ID");
XMLElement *pType = pRes->FirstChildElement("Type");
XMLElement *pDir = pRes->FirstChildElement("FileDir");
XMLElement *pName = pRes->FirstChildElement("FileName");
rArc << SERIAL_AUTO(ID) << SERIAL_AUTO(Type) << SERIAL_AUTO(Directory) << SERIAL_AUTO(Name);
}
};
std::vector<SDatabaseResource> Resources;
if (pID && pType && pDir && pName)
{
CAssetID ID = CAssetID::FromString(pID->GetText());
EResType Type = CResource::ResTypeForExtension(pType->GetText());
TWideString FileDir = pDir->GetText();
TWideString FileName = pName->GetText();
RegisterResource(ID, Type, FileDir, FileName);
}
else
Log::Error("Error reading " + rkPath + ": Resource entry " + TString::FromInt32(ResIndex, 0, 10) + " is missing one or more required components");
// Populate resource list
if (!rArc.IsReader())
{
Resources.reserve(mResourceEntries.size());
ResIndex++;
pRes = pRes->NextSiblingElement("Resource");
for (CResourceIterator It(this); It; ++It)
{
if (!It->IsTransient())
Resources.push_back( SDatabaseResource { It->ID(), It->CookedExtension(), It->Directory()->FullPath(), It->Name() } );
}
}
// All resources registered - load cache data
for (auto It = mResourceEntries.begin(); It != mResourceEntries.end(); It++)
// Serialize
rArc << SERIAL_CONTAINER_AUTO(Resources, "Resource");
// Register resources
if (rArc.IsReader())
{
CResourceEntry *pEntry = It->second;
if (!pEntry->IsTransient())
pEntry->LoadCacheData();
for (auto Iter = Resources.begin(); Iter != Resources.end(); Iter++)
{
SDatabaseResource& rRes = *Iter;
RegisterResource(rRes.ID, CResource::ResTypeForExtension(rRes.Type), rRes.Directory, rRes.Name);
}
}
}
void CResourceStore::SaveResourceDatabase(const TString& rkPath) const
void CResourceStore::LoadResourceDatabase()
{
XMLDocument Doc;
ASSERT(mpProj);
TString Path = mpProj->ResourceDBPath(false).ToUTF8();
XMLDeclaration *pDecl = Doc.NewDeclaration();
Doc.LinkEndChild(pDecl);
CXMLReader Reader(Path);
SerializeResourceDatabase(Reader);
LoadCacheFile();
}
XMLElement *pRoot = Doc.NewElement("ResourceDatabase");
pRoot->SetAttribute("Version", eVer_Current);
Doc.LinkEndChild(pRoot);
void CResourceStore::SaveResourceDatabase()
{
ASSERT(mpProj);
TString Path = mpProj->ResourceDBPath(false).ToUTF8();
XMLElement *pResources = Doc.NewElement("Resources");
pRoot->LinkEndChild(pResources);
CXMLWriter Writer(Path, "ResourceDB", 0, mpProj ? mpProj->Game() : eUnknownGame);
SerializeResourceDatabase(Writer);
}
for (auto It = mResourceEntries.begin(); It != mResourceEntries.end(); It++)
void CResourceStore::LoadCacheFile()
{
TString CacheDataPath = mpProj->ResourceCachePath(false).ToUTF8();
CFileInStream CacheFile(CacheDataPath.ToStdString(), IOUtil::eBigEndian);
ASSERT(CacheFile.IsValid());
// Cache header
CFourCC Magic(CacheFile);
if (Magic != "CACH")
{
CResourceEntry *pEntry = It->second;
if (pEntry->IsTransient()) continue;
XMLElement *pRes = Doc.NewElement("Resource");
pResources->LinkEndChild(pRes);
XMLElement *pID = Doc.NewElement("ID");
pID->SetText(*pEntry->ID().ToString());
pRes->LinkEndChild(pID);
XMLElement *pType = Doc.NewElement("Type");
pType->SetText(*GetResourceCookedExtension(pEntry->ResourceType(), pEntry->Game()));
pRes->LinkEndChild(pType);
XMLElement *pDir = Doc.NewElement("FileDir");
pDir->SetText(*pEntry->Directory()->FullPath().ToUTF8());
pRes->LinkEndChild(pDir);
XMLElement *pName = Doc.NewElement("FileName");
pName->SetText(*pEntry->Name().ToUTF8());
pRes->LinkEndChild(pName);
Log::Error("Invalid resource cache data magic: " + Magic.ToString());
return;
}
Doc.SaveFile(*rkPath);
CSerialVersion Version(CacheFile);
u32 NumResources = CacheFile.ReadLong();
for (u32 iRes = 0; iRes < NumResources; iRes++)
{
CAssetID ID(CacheFile, Version.Game());
u32 EntryCacheSize = CacheFile.ReadLong();
u32 EntryCacheEnd = CacheFile.Tell() + EntryCacheSize;
CResourceEntry *pEntry = FindEntry(ID);
if (pEntry && !pEntry->IsTransient())
{
CBasicBinaryReader Reader(&CacheFile, Version);
if (Reader.ParamBegin("EntryCache"))
{
pEntry->SerializeCacheData(Reader);
Reader.ParamEnd();
}
}
CacheFile.Seek(EntryCacheEnd, SEEK_SET);
}
}
void CResourceStore::SaveCacheFile()
{
TString CacheDataPath = mpProj->ResourceCachePath(false).ToUTF8();
CFileOutStream CacheFile(CacheDataPath.ToStdString(), IOUtil::eBigEndian);
ASSERT(CacheFile.IsValid());
// Cache header
CFourCC("CACH").Write(CacheFile);
CSerialVersion Version(0, 0, mpProj->Game());
Version.Write(CacheFile);
u32 ResCountOffset = CacheFile.Tell();
u32 ResCount = 0;
CacheFile.WriteLong(0); // Resource count dummy - fill in when we know the real count
// Save entry cache data
// Structure: Entry Asset ID -> Entry Cache Size -> Serialized Entry Cache Data
for (CResourceIterator It(this); It; ++It)
{
if (!It->IsTransient())
{
ResCount++;
It->ID().Write(CacheFile);
u32 SizeOffset = CacheFile.Tell();
CacheFile.WriteLong(0);
CBasicBinaryWriter Writer(&CacheFile, Version.FileVersion(), Version.Game());
if (Writer.ParamBegin("EntryCache"))
{
It->SerializeCacheData(Writer);
Writer.ParamEnd();
}
u32 EntryCacheEnd = CacheFile.Tell();
CacheFile.Seek(SizeOffset, SEEK_SET);
CacheFile.WriteLong(EntryCacheEnd - SizeOffset - 4);
CacheFile.Seek(EntryCacheEnd, SEEK_SET);
}
}
CacheFile.Seek(ResCountOffset, SEEK_SET);
CacheFile.WriteLong(ResCount);
}
void CResourceStore::SetActiveProject(CGameProject *pProj)
@@ -134,7 +196,7 @@ void CResourceStore::SetActiveProject(CGameProject *pProj)
mpProjectRoot = new CVirtualDirectory();
if (!mpExporter)
LoadResourceDatabase(pProj->ResourceDBPath(false));
LoadResourceDatabase();
}
}
@@ -145,7 +207,9 @@ void CResourceStore::CloseActiveProject()
DestroyUnreferencedResources();
// Delete all entries from old project
for (auto It = mResourceEntries.begin(); It != mResourceEntries.end(); It++)
auto It = mResourceEntries.begin();
while (It != mResourceEntries.end())
{
CResourceEntry *pEntry = It->second;
@@ -161,11 +225,12 @@ void CResourceStore::CloseActiveProject()
mLoadedResources.erase(LoadIt);
}
It = mResourceEntries.erase(It);
if (It == mResourceEntries.end()) break;
delete pEntry;
It = mResourceEntries.erase(It);
}
else
It++;
}
delete mpProjectRoot;
@@ -402,8 +467,9 @@ void CResourceStore::DestroyUnreferencedResources()
do
{
NumDeleted = 0;
auto It = mLoadedResources.begin();
for (auto It = mLoadedResources.begin(); It != mLoadedResources.end(); It++)
while (It != mLoadedResources.end())
{
CResourceEntry *pEntry = It->second;
@@ -415,9 +481,9 @@ void CResourceStore::DestroyUnreferencedResources()
// Transient resources should have their entries cleared out when the resource is unloaded
if (pEntry->IsTransient())
DeleteResourceEntry(pEntry);
if (It == mLoadedResources.end()) break;
}
else It++;
}
} while (NumDeleted > 0);

7
src/Core/GameProject/CResourceStore.h
View File

@@ -42,8 +42,11 @@ public:
CResourceStore();
CResourceStore(CGameExporter *pExporter);
~CResourceStore();
void LoadResourceDatabase(const TString& rkPath);
void SaveResourceDatabase(const TString& rkPath) const;
void SerializeResourceDatabase(IArchive& rArc);
void LoadResourceDatabase();
void SaveResourceDatabase();
void LoadCacheFile();
void SaveCacheFile();
void SetActiveProject(CGameProject *pProj);
void CloseActiveProject();
CVirtualDirectory* GetVirtualDirectory(const TWideString& rkPath, bool Transient, bool AllowCreate);