From 20bddd5ed7380f7d6bf70fc592689e99e4262b1b Mon Sep 17 00:00:00 2001 From: parax0 Date: Fri, 26 Aug 2016 19:33:33 -0600 Subject: [PATCH] Modified all editor file formats to use the serialization system; changed dependency caching so all resource cache data is in one file --- src/Common/CAssetID.cpp | 2 +- src/Common/CFourCC.h | 8 - src/Common/Common.pro | 8 +- src/Common/EGame.cpp | 8 + src/Common/EGame.h | 2 + src/Common/Log.cpp | 6 +- src/Common/Serialization/Binary.h | 5 + src/Common/Serialization/CBasicBinaryReader.h | 19 +- src/Common/Serialization/CBasicBinaryWriter.h | 38 ++-- src/Common/Serialization/CBinaryReader.h | 73 +++--- src/Common/Serialization/CBinaryWriter.h | 91 ++++---- src/Common/Serialization/CSerialVersion.cpp | 34 +++ src/Common/Serialization/CSerialVersion.h | 27 +++ src/Common/Serialization/CXMLReader.h | 10 +- src/Common/Serialization/CXMLWriter.h | 13 +- src/Common/Serialization/IArchive.h | 50 +++-- src/Common/Serialization/XML.h | 3 + src/Core/GameProject/CDependencyTree.cpp | 159 ++----------- src/Core/GameProject/CDependencyTree.h | 62 +++-- src/Core/GameProject/CGameExporter.cpp | 14 +- src/Core/GameProject/CGameProject.cpp | 124 +++------- src/Core/GameProject/CGameProject.h | 2 + src/Core/GameProject/CPackage.cpp | 103 ++------- src/Core/GameProject/CPackage.h | 12 + src/Core/GameProject/CResourceEntry.cpp | 106 +++------ src/Core/GameProject/CResourceEntry.h | 5 +- src/Core/GameProject/CResourceStore.cpp | 212 ++++++++++++------ src/Core/GameProject/CResourceStore.h | 7 +- src/Core/Resource/Area/CGameArea.cpp | 8 +- src/Core/Resource/CWorld.cpp | 12 +- 30 files changed, 565 insertions(+), 658 deletions(-) create mode 100644 src/Common/Serialization/Binary.h create mode 100644 src/Common/Serialization/CSerialVersion.cpp create mode 100644 src/Common/Serialization/CSerialVersion.h create mode 100644 src/Common/Serialization/XML.h diff --git a/src/Common/CAssetID.cpp b/src/Common/CAssetID.cpp index 8042aad2..c71acbbe 100644 --- a/src/Common/CAssetID.cpp +++ b/src/Common/CAssetID.cpp @@ -68,7 +68,7 @@ TString CAssetID::ToString() const bool CAssetID::IsValid() const { - return *this != InvalidID(mLength); + return (mID != 0 && mLength != eInvalidIDLength && mID != InvalidID(mLength).mID); } // ************ STATIC ************ diff --git a/src/Common/CFourCC.h b/src/Common/CFourCC.h index 776798cd..cda56fb4 100644 --- a/src/Common/CFourCC.h +++ b/src/Common/CFourCC.h @@ -4,7 +4,6 @@ #include "AssertMacro.h" #include "types.h" #include "TString.h" -#include "Common/Serialization/IArchive.h" #include #include @@ -42,13 +41,6 @@ public: rOutput.WriteLong(Val); } - inline void Serialize(IArchive& rArc) - { - TString Str = ToString(); - rArc.SerializePrimitive(Str); - if (rArc.IsReader()) *this = CFourCC(Str); - } - inline u32 ToLong() const { return mFourCC; diff --git a/src/Common/Common.pro b/src/Common/Common.pro index a37e68ed..316feedf 100644 --- a/src/Common/Common.pro +++ b/src/Common/Common.pro @@ -85,7 +85,10 @@ HEADERS += \ Serialization/CBasicBinaryWriter.h \ Serialization/CBasicBinaryReader.h \ Serialization/CBinaryWriter.h \ - Serialization/CBinaryReader.h + Serialization/CBinaryReader.h \ + Serialization/CSerialVersion.h \ + Serialization/XML.h \ + Serialization/Binary.h # Source Files SOURCES += \ @@ -96,4 +99,5 @@ SOURCES += \ Log.cpp \ FileUtil.cpp \ CAssetID.cpp \ - EGame.cpp + EGame.cpp \ + Serialization/CSerialVersion.cpp diff --git a/src/Common/EGame.cpp b/src/Common/EGame.cpp index 656dc8a5..38322416 100644 --- a/src/Common/EGame.cpp +++ b/src/Common/EGame.cpp @@ -1,5 +1,6 @@ #include "EGame.h" #include "CFourCC.h" +#include "Common/Serialization/IArchive.h" CFourCC GetGameID(EGame Game) { @@ -27,3 +28,10 @@ EGame GetGameForID(const CFourCC& rkID) if (rkID == "DKCR") return eReturns; return eUnknownGame; } + +void Serialize(IArchive& rArc, EGame& rGame) +{ + CFourCC GameID = GetGameID(rGame); + rArc.SerializePrimitive(GameID); + if (rArc.IsReader()) rGame = GetGameForID(GameID); +} diff --git a/src/Common/EGame.h b/src/Common/EGame.h index 77a9acc4..e2ff9a21 100644 --- a/src/Common/EGame.h +++ b/src/Common/EGame.h @@ -5,6 +5,7 @@ #include "types.h" class CFourCC; +class IArchive; enum EGame { @@ -20,5 +21,6 @@ enum EGame CFourCC GetGameID(EGame Game); EGame GetGameForID(const CFourCC& rkID); +void Serialize(IArchive& rArc, EGame& rGame); #endif // EGAME_H diff --git a/src/Common/Log.cpp b/src/Common/Log.cpp index ff099eea..39643149 100644 --- a/src/Common/Log.cpp +++ b/src/Common/Log.cpp @@ -69,17 +69,19 @@ bool InitLog(const TString& rkFilename) void Write(const TString& rkMessage) { + double Time = CTimer::GlobalTime() - gAppStartTime; + if (!gInitialized) gPreInitLogs.push_back(rkMessage); else if (gpLogFile) { - double Time = CTimer::GlobalTime() - gAppStartTime; fprintf(gpLogFile, "[%08.3f] %s\n", Time, *rkMessage); fflush(gpLogFile); } - std::cout << rkMessage << "\n"; + std::cout << std::fixed << std::setprecision(3) + << "[" << Time << "] " << rkMessage << "\n"; } void Error(const TString& rkMessage) diff --git a/src/Common/Serialization/Binary.h b/src/Common/Serialization/Binary.h new file mode 100644 index 00000000..438382d8 --- /dev/null +++ b/src/Common/Serialization/Binary.h @@ -0,0 +1,5 @@ +#include "IArchive.h" +#include "CBinaryReader.h" +#include "CBinaryWriter.h" +#include "CBasicBinaryReader.h" +#include "CBasicBinaryWriter.h" diff --git a/src/Common/Serialization/CBasicBinaryReader.h b/src/Common/Serialization/CBasicBinaryReader.h index 99e15614..7de43029 100644 --- a/src/Common/Serialization/CBasicBinaryReader.h +++ b/src/Common/Serialization/CBasicBinaryReader.h @@ -2,6 +2,7 @@ #define CBASICBINARYREADER #include "IArchive.h" +#include "CSerialVersion.h" #include "Common/CFourCC.h" #include @@ -21,21 +22,17 @@ public: mpStream = new CFileInStream(rkFilename.ToStdString(), IOUtil::eBigEndian); ASSERT(mpStream->IsValid()); - mFileVersion = mpStream->ReadShort(); - mArchiveVersion = mpStream->ReadShort(); - mGame = GetGameForID( CFourCC(*mpStream) ); + CSerialVersion Version(*mpStream); + SetVersion(Version); } - CBasicBinaryReader(IInputStream *pStream) + CBasicBinaryReader(IInputStream *pStream, const CSerialVersion& rkVersion) : IArchive(true, false) , mOwnsStream(false) { ASSERT(pStream->IsValid()); mpStream = pStream; - - mFileVersion = mpStream->ReadShort(); - mArchiveVersion = mpStream->ReadShort(); - mGame = GetGameForID( CFourCC(*mpStream) ); + SetVersion(rkVersion); } ~CBasicBinaryReader() @@ -62,15 +59,13 @@ public: virtual void SerializePrimitive(float& rValue) { rValue = mpStream->ReadFloat(); } virtual void SerializePrimitive(double& rValue) { rValue = mpStream->ReadDouble(); } virtual void SerializePrimitive(TString& rValue) { rValue = mpStream->ReadSizedString(); } + virtual void SerializePrimitive(TWideString& rValue) { rValue = mpStream->ReadSizedWString(); } + virtual void SerializePrimitive(CFourCC& rValue) { rValue = CFourCC(*mpStream); } virtual void SerializePrimitive(CAssetID& rValue) { rValue = CAssetID(*mpStream, mGame); } - virtual void SerializeHexPrimitive(s8& rValue) { rValue = mpStream->ReadByte(); } virtual void SerializeHexPrimitive(u8& rValue) { rValue = mpStream->ReadByte(); } - virtual void SerializeHexPrimitive(s16& rValue) { rValue = mpStream->ReadShort(); } virtual void SerializeHexPrimitive(u16& rValue) { rValue = mpStream->ReadShort(); } - virtual void SerializeHexPrimitive(s32& rValue) { rValue = mpStream->ReadLong(); } virtual void SerializeHexPrimitive(u32& rValue) { rValue = mpStream->ReadLong(); } - virtual void SerializeHexPrimitive(s64& rValue) { rValue = mpStream->ReadLongLong(); } virtual void SerializeHexPrimitive(u64& rValue) { rValue = mpStream->ReadLongLong(); } }; diff --git a/src/Common/Serialization/CBasicBinaryWriter.h b/src/Common/Serialization/CBasicBinaryWriter.h index b8b0c90b..01fe4463 100644 --- a/src/Common/Serialization/CBasicBinaryWriter.h +++ b/src/Common/Serialization/CBasicBinaryWriter.h @@ -14,34 +14,32 @@ class CBasicBinaryWriter : public IArchive bool mOwnsStream; public: - CBasicBinaryWriter(const TString& rkFilename, u32 FileVersion, EGame Game = eUnknownGame, IOUtil::EEndianness = IOUtil::eLittleEndian) + CBasicBinaryWriter(const TString& rkFilename, u16 FileVersion, EGame Game = eUnknownGame, IOUtil::EEndianness = IOUtil::eLittleEndian) : IArchive(false, true) , mOwnsStream(true) { mpStream = new CFileOutStream(rkFilename.ToStdString(), IOUtil::eBigEndian); ASSERT(mpStream->IsValid()); - - mFileVersion = FileVersion; - mGame = Game; - - mpStream->WriteShort((u16) FileVersion); - mpStream->WriteShort((u16) skCurrentArchiveVersion); - GetGameID(Game).Write(*mpStream); + SetVersion(skCurrentArchiveVersion, FileVersion, Game); + GetVersionInfo().Write(*mpStream); } - CBasicBinaryWriter(IOutputStream *pStream, u32 FileVersion, EGame Game = eUnknownGame) + CBasicBinaryWriter(IOutputStream *pStream, u16 FileVersion, EGame Game = eUnknownGame) : IArchive(false, true) , mOwnsStream(false) { ASSERT(pStream->IsValid()); mpStream = pStream; + SetVersion(skCurrentArchiveVersion, FileVersion, Game); + } - mFileVersion = FileVersion; - mGame = Game; - - mpStream->WriteShort((u16) FileVersion); - mpStream->WriteShort((u16) skCurrentArchiveVersion); - GetGameID(Game).Write(*mpStream); + CBasicBinaryWriter(IOutputStream *pStream, const CSerialVersion& rkVersion) + : IArchive(false, true) + , mOwnsStream(false) + { + ASSERT(pStream->IsValid()); + mpStream = pStream; + SetVersion(rkVersion); } ~CBasicBinaryWriter() @@ -53,8 +51,8 @@ public: virtual bool ParamBegin(const char*) { return true; } virtual void ParamEnd() { } - virtual void SerializeContainerSize(u32& rSize) { SerializePrimitive(rSize); } - virtual void SerializeAbstractObjectType(u32& rType) { SerializePrimitive(rType); } + virtual void SerializeContainerSize(u32& rSize) { mpStream->WriteLong(rSize); } + virtual void SerializeAbstractObjectType(u32& rType) { mpStream->WriteLong(rType); } virtual void SerializePrimitive(bool& rValue) { mpStream->WriteBool(rValue); } virtual void SerializePrimitive(char& rValue) { mpStream->WriteByte(rValue); } virtual void SerializePrimitive(s8& rValue) { mpStream->WriteByte(rValue); } @@ -68,15 +66,13 @@ public: virtual void SerializePrimitive(float& rValue) { mpStream->WriteFloat(rValue); } virtual void SerializePrimitive(double& rValue) { mpStream->WriteDouble(rValue); } virtual void SerializePrimitive(TString& rValue) { mpStream->WriteSizedString(rValue.ToStdString()); } + virtual void SerializePrimitive(TWideString& rValue) { mpStream->WriteSizedWideString(rValue.ToStdString()); } + virtual void SerializePrimitive(CFourCC& rValue) { rValue.Write(*mpStream); } virtual void SerializePrimitive(CAssetID& rValue) { rValue.Write(*mpStream); } - virtual void SerializeHexPrimitive(s8& rValue) { mpStream->WriteByte(rValue); } virtual void SerializeHexPrimitive(u8& rValue) { mpStream->WriteByte(rValue); } - virtual void SerializeHexPrimitive(s16& rValue) { mpStream->WriteShort(rValue); } virtual void SerializeHexPrimitive(u16& rValue) { mpStream->WriteShort(rValue); } - virtual void SerializeHexPrimitive(s32& rValue) { mpStream->WriteLong(rValue); } virtual void SerializeHexPrimitive(u32& rValue) { mpStream->WriteLong(rValue); } - virtual void SerializeHexPrimitive(s64& rValue) { mpStream->WriteLongLong(rValue); } virtual void SerializeHexPrimitive(u64& rValue) { mpStream->WriteLongLong(rValue); } }; diff --git a/src/Common/Serialization/CBinaryReader.h b/src/Common/Serialization/CBinaryReader.h index a6c1332b..67ca17bd 100644 --- a/src/Common/Serialization/CBinaryReader.h +++ b/src/Common/Serialization/CBinaryReader.h @@ -2,6 +2,7 @@ #define CBINARYREADER #include "IArchive.h" +#include "CSerialVersion.h" #include "Common/CFourCC.h" class CBinaryReader : public IArchive @@ -11,6 +12,7 @@ class CBinaryReader : public IArchive u32 Offset; u16 Size; bool HasChildren; + bool Abstract; }; std::vector mParamStack; @@ -25,23 +27,19 @@ public: mpStream = new CFileInStream(rkFilename.ToStdString(), IOUtil::eBigEndian); ASSERT(mpStream->IsValid()); - mFileVersion = mpStream->ReadShort(); - mArchiveVersion = mpStream->ReadShort(); - mGame = GetGameForID( CFourCC(*mpStream) ); + CSerialVersion Version(*mpStream); + SetVersion(Version); mParamStack.reserve(20); } - CBinaryReader(IInputStream *pStream) + CBinaryReader(IInputStream *pStream, const CSerialVersion& rkVersion) : IArchive(true, false) , mOwnsStream(false) { ASSERT(pStream->IsValid()); mpStream = pStream; - - mFileVersion = mpStream->ReadShort(); - mArchiveVersion = mpStream->ReadShort(); - mGame = GetGameForID( CFourCC(*mpStream) ); + SetVersion(rkVersion); mParamStack.reserve(20); } @@ -70,16 +68,16 @@ public: // Does the next parameter ID match the current one? if (NextID == ParamID) { - mParamStack.push_back( SParameter { Offset, NextSize, false } ); + mParamStack.push_back( SParameter { Offset, NextSize, false, false } ); return true; } // It's not a match - return to the parent parameter's first child and check all children to find a match if (!mParamStack.empty()) { + bool ParentAbstract = mParamStack.back().Abstract; u32 ParentOffset = mParamStack.back().Offset; - mpStream->Seek(ParentOffset, SEEK_SET); - mpStream->Seek(0x6, SEEK_CUR); + mpStream->Seek(ParentOffset + (ParentAbstract ? 0xA : 0x6), SEEK_SET); u16 NumChildren = mpStream->ReadShort(); for (u32 iChild = 0; iChild < NumChildren; iChild++) @@ -91,7 +89,7 @@ public: mpStream->Seek(ChildSize, SEEK_CUR); else { - mParamStack.push_back( SParameter { mpStream->Tell() - 6, NextSize, false } ); + mParamStack.push_back( SParameter { mpStream->Tell() - 6, NextSize, false, false } ); return true; } } @@ -111,37 +109,40 @@ public: mParamStack.pop_back(); } - void SerializeContainerSize(u32& rSize) + virtual void SerializeContainerSize(u32& rSize) { // Mostly handled by ParamBegin, we just need to return the size correctly so the container can be resized rSize = (u32) mpStream->PeekShort(); } - void SerializeAbstractObjectType(u32& rType) { rType = mpStream->ReadLong(); } + virtual void SerializeAbstractObjectType(u32& rType) + { + // Mark current parameter as abstract so we can account for the object type in the filestream + rType = mpStream->ReadLong(); + mParamStack.back().Abstract = true; + } - void SerializePrimitive(bool& rValue) { rValue = mpStream->ReadBool(); } - void SerializePrimitive(char& rValue) { rValue = mpStream->ReadByte(); } - void SerializePrimitive(s8& rValue) { rValue = mpStream->ReadByte(); } - void SerializePrimitive(u8& rValue) { rValue = mpStream->ReadByte(); } - void SerializePrimitive(s16& rValue) { rValue = mpStream->ReadShort(); } - void SerializePrimitive(u16& rValue) { rValue = mpStream->ReadShort(); } - void SerializePrimitive(s32& rValue) { rValue = mpStream->ReadLong(); } - void SerializePrimitive(u32& rValue) { rValue = mpStream->ReadLong(); } - void SerializePrimitive(s64& rValue) { rValue = mpStream->ReadLongLong(); } - void SerializePrimitive(u64& rValue) { rValue = mpStream->ReadLongLong(); } - void SerializePrimitive(float& rValue) { rValue = mpStream->ReadFloat(); } - void SerializePrimitive(double& rValue) { rValue = mpStream->ReadDouble(); } - void SerializePrimitive(TString& rValue) { rValue = mpStream->ReadSizedString(); } - void SerializePrimitive(CAssetID& rValue) { rValue = CAssetID(*mpStream, Game()); } + virtual void SerializePrimitive(bool& rValue) { rValue = mpStream->ReadBool(); } + virtual void SerializePrimitive(char& rValue) { rValue = mpStream->ReadByte(); } + virtual void SerializePrimitive(s8& rValue) { rValue = mpStream->ReadByte(); } + virtual void SerializePrimitive(u8& rValue) { rValue = mpStream->ReadByte(); } + virtual void SerializePrimitive(s16& rValue) { rValue = mpStream->ReadShort(); } + virtual void SerializePrimitive(u16& rValue) { rValue = mpStream->ReadShort(); } + virtual void SerializePrimitive(s32& rValue) { rValue = mpStream->ReadLong(); } + virtual void SerializePrimitive(u32& rValue) { rValue = mpStream->ReadLong(); } + virtual void SerializePrimitive(s64& rValue) { rValue = mpStream->ReadLongLong(); } + virtual void SerializePrimitive(u64& rValue) { rValue = mpStream->ReadLongLong(); } + virtual void SerializePrimitive(float& rValue) { rValue = mpStream->ReadFloat(); } + virtual void SerializePrimitive(double& rValue) { rValue = mpStream->ReadDouble(); } + virtual void SerializePrimitive(TString& rValue) { rValue = mpStream->ReadSizedString(); } + virtual void SerializePrimitive(TWideString& rValue) { rValue = mpStream->ReadSizedWString(); } + virtual void SerializePrimitive(CFourCC& rValue) { rValue = CFourCC(*mpStream); } + virtual void SerializePrimitive(CAssetID& rValue) { rValue = CAssetID(*mpStream, Game()); } - void SerializeHexPrimitive(s8& rValue) { rValue = mpStream->ReadByte(); } - void SerializeHexPrimitive(u8& rValue) { rValue = mpStream->ReadByte(); } - void SerializeHexPrimitive(s16& rValue) { rValue = mpStream->ReadShort(); } - void SerializeHexPrimitive(u16& rValue) { rValue = mpStream->ReadShort(); } - void SerializeHexPrimitive(s32& rValue) { rValue = mpStream->ReadLong(); } - void SerializeHexPrimitive(u32& rValue) { rValue = mpStream->ReadLong(); } - void SerializeHexPrimitive(s64& rValue) { rValue = mpStream->ReadLongLong(); } - void SerializeHexPrimitive(u64& rValue) { rValue = mpStream->ReadLongLong(); } + virtual void SerializeHexPrimitive(u8& rValue) { rValue = mpStream->ReadByte(); } + virtual void SerializeHexPrimitive(u16& rValue) { rValue = mpStream->ReadShort(); } + virtual void SerializeHexPrimitive(u32& rValue) { rValue = mpStream->ReadLong(); } + virtual void SerializeHexPrimitive(u64& rValue) { rValue = mpStream->ReadLongLong(); } }; #endif // CBINARYREADER diff --git a/src/Common/Serialization/CBinaryWriter.h b/src/Common/Serialization/CBinaryWriter.h index 295eaf0e..1a23413c 100644 --- a/src/Common/Serialization/CBinaryWriter.h +++ b/src/Common/Serialization/CBinaryWriter.h @@ -10,6 +10,7 @@ class CBinaryWriter : public IArchive { u32 Offset; u16 NumSubParams; + bool Abstract; }; std::vector mParamStack; @@ -17,34 +18,33 @@ class CBinaryWriter : public IArchive bool mOwnsStream; public: - CBinaryWriter(const TString& rkFilename, u32 FileVersion, EGame Game = eUnknownGame) + CBinaryWriter(const TString& rkFilename, u16 FileVersion, EGame Game = eUnknownGame) : IArchive(false, true) , mOwnsStream(true) { mpStream = new CFileOutStream(rkFilename.ToStdString(), IOUtil::eBigEndian); ASSERT(mpStream->IsValid()); - mFileVersion = FileVersion; - mGame = Game; - - mpStream->WriteShort((u16) FileVersion); - mpStream->WriteShort((u16) skCurrentArchiveVersion); - GetGameID(Game).Write(*mpStream); + SetVersion(skCurrentArchiveVersion, FileVersion, Game); + GetVersionInfo().Write(*mpStream); } - CBinaryWriter(IOutputStream *pStream, u32 FileVersion, EGame Game = eUnknownGame) + CBinaryWriter(IOutputStream *pStream, u16 FileVersion, EGame Game = eUnknownGame) : IArchive(false, true) , mOwnsStream(false) { ASSERT(pStream->IsValid()); mpStream = pStream; + SetVersion(skCurrentArchiveVersion, FileVersion, Game); + } - mFileVersion = FileVersion; - mGame = Game; - - mpStream->WriteShort((u16) FileVersion); - mpStream->WriteShort((u16) skCurrentArchiveVersion); - GetGameID(Game).Write(*mpStream); + CBinaryWriter(IOutputStream *pStream, const CSerialVersion& rkVersion) + : IArchive(false, true) + , mOwnsStream(false) + { + ASSERT(pStream->IsValid()); + mpStream = pStream; + SetVersion(rkVersion); } ~CBinaryWriter() @@ -63,7 +63,7 @@ public: mpStream->WriteShort(-1); // Sub-param count filler } - mParamStack.push_back( SParameter { mpStream->Tell(), 0 } ); + mParamStack.push_back( SParameter { mpStream->Tell(), 0, false } ); u32 ParamID = TString(pkName).Hash32(); mpStream->WriteLong(ParamID); @@ -76,47 +76,56 @@ public: SParameter& rParam = mParamStack.back(); u32 StartOffset = rParam.Offset; u32 EndOffset = mpStream->Tell(); - u16 ParamSize = (EndOffset - StartOffset) - 6; + u16 ParamSize = (u16) (EndOffset - StartOffset) - 6; mpStream->Seek(StartOffset + 4, SEEK_SET); mpStream->WriteShort(ParamSize); - if (rParam.NumSubParams > 0) mpStream->WriteShort(rParam.NumSubParams); + + if (rParam.NumSubParams > 0) + { + if (rParam.Abstract) mpStream->Seek(4, SEEK_CUR); + mpStream->WriteShort(rParam.NumSubParams); + } + mpStream->Seek(EndOffset, SEEK_SET); mParamStack.pop_back(); } - void SerializeContainerSize(u32& rSize) + virtual void SerializeContainerSize(u32& rSize) { // Normally handled by ParamBegin and ParamEnd but we need to do something here to account for zero-sized containers if (rSize == 0) mpStream->WriteShort(0); } - void SerializeAbstractObjectType(u32& rType) { mpStream->WriteLong(rType); } + virtual void SerializeAbstractObjectType(u32& rType) + { + // Mark this parameter as abstract so we can account for the object type in the filestream + mpStream->WriteLong(rType); + mParamStack.back().Abstract = true; + } - void SerializePrimitive(bool& rValue) { mpStream->WriteBool(rValue); } - void SerializePrimitive(char& rValue) { mpStream->WriteByte(rValue); } - void SerializePrimitive(s8& rValue) { mpStream->WriteByte(rValue); } - void SerializePrimitive(u8& rValue) { mpStream->WriteByte(rValue); } - void SerializePrimitive(s16& rValue) { mpStream->WriteShort(rValue); } - void SerializePrimitive(u16& rValue) { mpStream->WriteShort(rValue); } - void SerializePrimitive(s32& rValue) { mpStream->WriteLong(rValue); } - void SerializePrimitive(u32& rValue) { mpStream->WriteLong(rValue); } - void SerializePrimitive(s64& rValue) { mpStream->WriteLongLong(rValue); } - void SerializePrimitive(u64& rValue) { mpStream->WriteLongLong(rValue); } - void SerializePrimitive(float& rValue) { mpStream->WriteFloat(rValue); } - void SerializePrimitive(double& rValue) { mpStream->WriteDouble(rValue); } - void SerializePrimitive(TString& rValue) { mpStream->WriteSizedString(rValue.ToStdString()); } - void SerializePrimitive(CAssetID& rValue) { rValue.Write(*mpStream); } + virtual void SerializePrimitive(bool& rValue) { mpStream->WriteBool(rValue); } + virtual void SerializePrimitive(char& rValue) { mpStream->WriteByte(rValue); } + virtual void SerializePrimitive(s8& rValue) { mpStream->WriteByte(rValue); } + virtual void SerializePrimitive(u8& rValue) { mpStream->WriteByte(rValue); } + virtual void SerializePrimitive(s16& rValue) { mpStream->WriteShort(rValue); } + virtual void SerializePrimitive(u16& rValue) { mpStream->WriteShort(rValue); } + virtual void SerializePrimitive(s32& rValue) { mpStream->WriteLong(rValue); } + virtual void SerializePrimitive(u32& rValue) { mpStream->WriteLong(rValue); } + virtual void SerializePrimitive(s64& rValue) { mpStream->WriteLongLong(rValue); } + virtual void SerializePrimitive(u64& rValue) { mpStream->WriteLongLong(rValue); } + virtual void SerializePrimitive(float& rValue) { mpStream->WriteFloat(rValue); } + virtual void SerializePrimitive(double& rValue) { mpStream->WriteDouble(rValue); } + virtual void SerializePrimitive(TString& rValue) { mpStream->WriteSizedString(rValue.ToStdString()); } + virtual void SerializePrimitive(TWideString& rValue) { mpStream->WriteSizedWideString(rValue.ToStdString()); } + virtual void SerializePrimitive(CFourCC& rValue) { rValue.Write(*mpStream); } + virtual void SerializePrimitive(CAssetID& rValue) { rValue.Write(*mpStream); } - void SerializeHexPrimitive(s8& rValue) { mpStream->WriteByte(rValue); } - void SerializeHexPrimitive(u8& rValue) { mpStream->WriteByte(rValue); } - void SerializeHexPrimitive(s16& rValue) { mpStream->WriteShort(rValue); } - void SerializeHexPrimitive(u16& rValue) { mpStream->WriteShort(rValue); } - void SerializeHexPrimitive(s32& rValue) { mpStream->WriteLong(rValue); } - void SerializeHexPrimitive(u32& rValue) { mpStream->WriteLong(rValue); } - void SerializeHexPrimitive(s64& rValue) { mpStream->WriteLongLong(rValue); } - void SerializeHexPrimitive(u64& rValue) { mpStream->WriteLongLong(rValue); } + virtual void SerializeHexPrimitive(u8& rValue) { mpStream->WriteByte(rValue); } + virtual void SerializeHexPrimitive(u16& rValue) { mpStream->WriteShort(rValue); } + virtual void SerializeHexPrimitive(u32& rValue) { mpStream->WriteLong(rValue); } + virtual void SerializeHexPrimitive(u64& rValue) { mpStream->WriteLongLong(rValue); } }; #endif // CBINARYWRITER diff --git a/src/Common/Serialization/CSerialVersion.cpp b/src/Common/Serialization/CSerialVersion.cpp new file mode 100644 index 00000000..2e816238 --- /dev/null +++ b/src/Common/Serialization/CSerialVersion.cpp @@ -0,0 +1,34 @@ +#include "CSerialVersion.h" +#include "Common/CFourCC.h" + +CSerialVersion::CSerialVersion() +{ +} + +CSerialVersion::CSerialVersion(u16 ArchiveVer, u16 FileVer, EGame Game) + : mArchiveVersion(ArchiveVer) + , mFileVersion(FileVer) + , mGame(Game) +{ +} + +CSerialVersion::CSerialVersion(IInputStream& rInput) +{ + Read(rInput); +} + +void CSerialVersion::Read(IInputStream& rInput) +{ + mArchiveVersion = rInput.ReadShort(); + mFileVersion = rInput.ReadShort(); + CFourCC GameID(rInput); + mGame = GetGameForID(GameID); +} + +void CSerialVersion::Write(IOutputStream& rOutput) +{ + rOutput.WriteShort(mArchiveVersion); + rOutput.WriteShort(mFileVersion); + CFourCC GameID = GetGameID(mGame); + GameID.Write(rOutput); +} diff --git a/src/Common/Serialization/CSerialVersion.h b/src/Common/Serialization/CSerialVersion.h new file mode 100644 index 00000000..2a456fa7 --- /dev/null +++ b/src/Common/Serialization/CSerialVersion.h @@ -0,0 +1,27 @@ +#ifndef CSERIALVERSION +#define CSERIALVERSION + +#include "Common/EGame.h" +#include "Common/types.h" +#include + +class CSerialVersion +{ + u16 mArchiveVersion; + u16 mFileVersion; + EGame mGame; + +public: + CSerialVersion(); + CSerialVersion(u16 ArchiveVer, u16 FileVer, EGame Game); + CSerialVersion(IInputStream& rInput); + void Read(IInputStream& rInput); + void Write(IOutputStream& rOutput); + + inline u16 ArchiveVersion() const { return mArchiveVersion; } + inline u16 FileVersion() const { return mFileVersion; } + inline EGame Game() const { return mGame; } +}; + +#endif // CSERIALVERSION + diff --git a/src/Common/Serialization/CXMLReader.h b/src/Common/Serialization/CXMLReader.h index 7dddac7d..e1f283c8 100644 --- a/src/Common/Serialization/CXMLReader.h +++ b/src/Common/Serialization/CXMLReader.h @@ -20,8 +20,8 @@ public: mpCurElem = mDoc.FirstChildElement(); ASSERT(mpCurElem != nullptr); - mFileVersion = TString( mpCurElem->Attribute("FileVer") ).ToInt32(10); - mArchiveVersion = TString( mpCurElem->Attribute("ArchiveVer") ).ToInt32(10); + mArchiveVersion = (u16) TString( mpCurElem->Attribute("ArchiveVer") ).ToInt32(10); + mFileVersion = (u16) TString( mpCurElem->Attribute("FileVer") ).ToInt32(10); const char *pkGameAttr = mpCurElem->Attribute("Game"); mGame = pkGameAttr ? GetGameForID( CFourCC(pkGameAttr) ) : eUnknownGame; } @@ -107,15 +107,13 @@ public: virtual void SerializePrimitive(float& rValue) { rValue = ReadParam().ToFloat(); } virtual void SerializePrimitive(double& rValue) { rValue = (double) ReadParam().ToFloat(); } virtual void SerializePrimitive(TString& rValue) { rValue = ReadParam(); } + virtual void SerializePrimitive(TWideString& rValue) { rValue = ReadParam().ToUTF16(); } + virtual void SerializePrimitive(CFourCC& rValue) { rValue = CFourCC( ReadParam() ); } virtual void SerializePrimitive(CAssetID& rValue) { rValue = CAssetID::FromString( ReadParam() ); } - virtual void SerializeHexPrimitive(s8& rValue) { rValue = (s8) ReadParam().ToInt32(16); } virtual void SerializeHexPrimitive(u8& rValue) { rValue = (u8) ReadParam().ToInt32(16); } - virtual void SerializeHexPrimitive(s16& rValue) { rValue = (s16) ReadParam().ToInt32(16); } virtual void SerializeHexPrimitive(u16& rValue) { rValue = (u16) ReadParam().ToInt32(16); } - virtual void SerializeHexPrimitive(s32& rValue) { rValue = (s32) ReadParam().ToInt32(16); } virtual void SerializeHexPrimitive(u32& rValue) { rValue = (u32) ReadParam().ToInt32(16); } - virtual void SerializeHexPrimitive(s64& rValue) { rValue = (s64) ReadParam().ToInt32(16); } virtual void SerializeHexPrimitive(u64& rValue) { rValue = (u64) ReadParam().ToInt32(16); } }; diff --git a/src/Common/Serialization/CXMLWriter.h b/src/Common/Serialization/CXMLWriter.h index e0f46751..4456a938 100644 --- a/src/Common/Serialization/CXMLWriter.h +++ b/src/Common/Serialization/CXMLWriter.h @@ -13,12 +13,11 @@ class CXMLWriter : public IArchive tinyxml2::XMLElement *mpCurElem; public: - CXMLWriter(const TString& rkFileName, const TString& rkRootName, u32 FileVersion, EGame Game = eUnknownGame) + CXMLWriter(const TString& rkFileName, const TString& rkRootName, u16 FileVersion, EGame Game = eUnknownGame) : IArchive(false, true) , mOutFilename(rkFileName) { - mFileVersion = FileVersion; - mGame = Game; + SetVersion(skCurrentArchiveVersion, FileVersion, Game); // Create declaration and root node tinyxml2::XMLDeclaration *pDecl = mDoc.NewDeclaration(); @@ -28,8 +27,8 @@ public: mDoc.LinkEndChild(mpCurElem); // Write version data - mpCurElem->SetAttribute("FileVer", (int) FileVersion); mpCurElem->SetAttribute("ArchiveVer", (int) skCurrentArchiveVersion); + mpCurElem->SetAttribute("FileVer", (int) FileVersion); if (Game != eUnknownGame) mpCurElem->SetAttribute("Game", *GetGameID(Game).ToString()); } @@ -82,15 +81,13 @@ public: virtual void SerializePrimitive(float& rValue) { WriteParam(*TString::FromFloat(rValue)); } virtual void SerializePrimitive(double& rValue) { WriteParam(*TString::FromFloat((float) rValue)); } virtual void SerializePrimitive(TString& rValue) { WriteParam(*rValue); } + virtual void SerializePrimitive(TWideString& rValue) { WriteParam(*rValue.ToUTF8()); } + virtual void SerializePrimitive(CFourCC& rValue) { WriteParam(*rValue.ToString()); } virtual void SerializePrimitive(CAssetID& rValue) { WriteParam(*rValue.ToString()); } - virtual void SerializeHexPrimitive(s8& rValue) { WriteParam(*TString::HexString((u8) rValue, 2)); } virtual void SerializeHexPrimitive(u8& rValue) { WriteParam(*TString::HexString(rValue, 2)); } - virtual void SerializeHexPrimitive(s16& rValue) { WriteParam(*TString::HexString((u16) rValue, 4)); } virtual void SerializeHexPrimitive(u16& rValue) { WriteParam(*TString::HexString(rValue, 4)); } - virtual void SerializeHexPrimitive(s32& rValue) { WriteParam(*TString::HexString((u32) rValue, 8)); } virtual void SerializeHexPrimitive(u32& rValue) { WriteParam(*TString::HexString(rValue, 8)); } - virtual void SerializeHexPrimitive(s64& rValue) { WriteParam(*TString::HexString((u32) rValue, 16)); } virtual void SerializeHexPrimitive(u64& rValue) { WriteParam(*TString::HexString((u32) rValue, 16)); } }; diff --git a/src/Common/Serialization/IArchive.h b/src/Common/Serialization/IArchive.h index 93cd4469..346d9b59 100644 --- a/src/Common/Serialization/IArchive.h +++ b/src/Common/Serialization/IArchive.h @@ -1,15 +1,19 @@ #ifndef IARCHIVE #define IARCHIVE +#include "CSerialVersion.h" #include "Common/AssertMacro.h" #include "Common/CAssetID.h" +#include "Common/CFourCC.h" #include "Common/EGame.h" #include "Common/TString.h" #include "Common/types.h" -/* This is a custom serialization implementation intended for saving editor-friendly game assets - * out to XML. Support for other output formats can be added by implementing new subclasses of - * IArchive. +/* This is a custom serialization implementation intended for saving game assets out to editor- + * friendly formats, such as XML. The main goals of the serialization system is to simplify the + * code for reading and writing editor files and to be able to easily update those files without + * breaking compatibility with older versions. Support for new output formats can be added by + * implementing new subclasses of IArchive. * * To use a class with the serialization system, it must have a Serialize function implemented. * There are two ways this function can be defined: @@ -22,9 +26,10 @@ * ensure that files are easily backwards-compatible if parameters are moved or added/removed. * * Polymorphism is supported. There are two requirements for a polymorphic class to work with the - * serialization system. First, the base class must contain a virtual Type() parameter that returns - * an integral value (an enum or an integer). Second, there must be a factory class with a SpawnObject - * method that takes the same Type value and returns an object of the specified class. + * serialization system. First, the base class must contain a virtual Type() function that returns + * an integral value (an enum or an integer), as well as a virtual Serialize(IArchive&) function. + * Second, there must be a factory object with a SpawnObject(u32) method that takes the same Type value + * and returns an object of the correct class. * * Containers are also supported. Containers require a different macro that allows you to specify the * name of the elements in the container. The currently-supported containers are std::vector, std::list, @@ -166,8 +171,8 @@ public: class IArchive { protected: - s32 mFileVersion; - s32 mArchiveVersion; + u16 mArchiveVersion; + u16 mFileVersion; EGame mGame; bool mIsReader; bool mIsWriter; @@ -383,23 +388,40 @@ public: virtual void SerializePrimitive(float& rValue) = 0; virtual void SerializePrimitive(double& rValue) = 0; virtual void SerializePrimitive(TString& rValue) = 0; + virtual void SerializePrimitive(TWideString& rValue) = 0; + virtual void SerializePrimitive(CFourCC& rValue) = 0; virtual void SerializePrimitive(CAssetID& rValue) = 0; - virtual void SerializeHexPrimitive(s8& rValue) = 0; virtual void SerializeHexPrimitive(u8& rValue) = 0; - virtual void SerializeHexPrimitive(s16& rValue) = 0; virtual void SerializeHexPrimitive(u16& rValue) = 0; - virtual void SerializeHexPrimitive(s32& rValue) = 0; virtual void SerializeHexPrimitive(u32& rValue) = 0; - virtual void SerializeHexPrimitive(s64& rValue) = 0; virtual void SerializeHexPrimitive(u64& rValue) = 0; // Accessors - inline u32 FileVersion() const { return mFileVersion; } - inline u32 ArchiveVersion() const { return mArchiveVersion; } + inline u16 ArchiveVersion() const { return mArchiveVersion; } + inline u16 FileVersion() const { return mFileVersion; } inline EGame Game() const { return mGame; } inline bool IsReader() const { return mIsReader; } inline bool IsWriter() const { return mIsWriter; } + + inline void SetVersion(u16 ArchiveVersion, u16 FileVersion, EGame Game) + { + mArchiveVersion = ArchiveVersion; + mFileVersion = FileVersion; + mGame = Game; + } + + inline void SetVersion(const CSerialVersion& rkVersion) + { + mArchiveVersion = rkVersion.ArchiveVersion(); + mFileVersion = rkVersion.FileVersion(); + mGame = rkVersion.Game(); + } + + inline CSerialVersion GetVersionInfo() const + { + return CSerialVersion(mArchiveVersion, mFileVersion, mGame); + } }; // Container serialize methods diff --git a/src/Common/Serialization/XML.h b/src/Common/Serialization/XML.h new file mode 100644 index 00000000..2192b9d0 --- /dev/null +++ b/src/Common/Serialization/XML.h @@ -0,0 +1,3 @@ +#include "IArchive.h" +#include "CXMLReader.h" +#include "CXMLWriter.h" diff --git a/src/Core/GameProject/CDependencyTree.cpp b/src/Core/GameProject/CDependencyTree.cpp index 882700a3..2e49b604 100644 --- a/src/Core/GameProject/CDependencyTree.cpp +++ b/src/Core/GameProject/CDependencyTree.cpp @@ -3,6 +3,8 @@ #include "Core/Resource/Script/CScriptLayer.h" #include "Core/Resource/Script/CScriptObject.h" +CDependencyNodeFactory gDependencyNodeFactory; + // ************ IDependencyNode ************ IDependencyNode::~IDependencyNode() { @@ -27,28 +29,10 @@ EDependencyNodeType CDependencyTree::Type() const return eDNT_DependencyTree; } -void CDependencyTree::Read(IInputStream& rFile, EIDLength IDLength) +void CDependencyTree::Serialize(IArchive& rArc) { - mRootID = CAssetID(rFile, IDLength); - - u32 NumDepends = rFile.ReadLong(); - mChildren.reserve(NumDepends); - - for (u32 iDep = 0; iDep < NumDepends; iDep++) - { - CResourceDependency *pDepend = new CResourceDependency; - pDepend->Read(rFile, IDLength); - mChildren.push_back(pDepend); - } -} - -void CDependencyTree::Write(IOutputStream& rFile) const -{ - mRootID.Write(rFile); - rFile.WriteLong( mChildren.size() ); - - for (u32 iDep = 0; iDep < mChildren.size(); iDep++) - mChildren[iDep]->Write(rFile); + rArc << SERIAL("RootID", mRootID) + << SERIAL_ABSTRACT_CONTAINER("Children", mChildren, "Child", &gDependencyNodeFactory); } void CDependencyTree::AddDependency(CResource *pRes, bool AvoidDuplicates /*= true*/) @@ -70,14 +54,9 @@ EDependencyNodeType CResourceDependency::Type() const return eDNT_ResourceDependency; } -void CResourceDependency::Read(IInputStream& rFile, EIDLength IDLength) +void CResourceDependency::Serialize(IArchive& rArc) { - mID = CAssetID(rFile, IDLength); -} - -void CResourceDependency::Write(IOutputStream& rFile) const -{ - mID.Write(rFile); + rArc << SERIAL("ID", mID); } bool CResourceDependency::HasDependency(const CAssetID& rkID) const @@ -91,16 +70,10 @@ EDependencyNodeType CPropertyDependency::Type() const return eDNT_ScriptProperty; } -void CPropertyDependency::Read(IInputStream& rFile, EIDLength IDLength) +void CPropertyDependency::Serialize(IArchive& rArc) { - mIDString = rFile.ReadString(); - CResourceDependency::Read(rFile, IDLength); -} - -void CPropertyDependency::Write(IOutputStream& rFile) const -{ - rFile.WriteString(mIDString.ToStdString()); - CResourceDependency::Write(rFile); + rArc << SERIAL("PropertyID", mIDString); + CResourceDependency::Serialize(rArc); } // ************ CCharacterPropertyDependency ************ @@ -109,16 +82,10 @@ EDependencyNodeType CCharPropertyDependency::Type() const return eDNT_CharacterProperty; } -void CCharPropertyDependency::Read(IInputStream& rFile, EIDLength IDLength) +void CCharPropertyDependency::Serialize(IArchive& rArc) { - CPropertyDependency::Read(rFile, IDLength); - mUsedChar = rFile.ReadLong(); -} - -void CCharPropertyDependency::Write(IOutputStream& rFile) const -{ - CPropertyDependency::Write(rFile); - rFile.WriteLong(mUsedChar); + CPropertyDependency::Serialize(rArc); + rArc << SERIAL("CharIndex", mUsedChar); } // ************ CScriptInstanceDependency ************ @@ -127,32 +94,10 @@ EDependencyNodeType CScriptInstanceDependency::Type() const return eDNT_ScriptInstance; } -void CScriptInstanceDependency::Read(IInputStream& rFile, EIDLength IDLength) +void CScriptInstanceDependency::Serialize(IArchive& rArc) { - mObjectType = rFile.ReadLong(); - u32 NumProperties = rFile.ReadLong(); - mChildren.reserve(NumProperties); - - for (u32 iProp = 0; iProp < NumProperties; iProp++) - { - bool IsCharacter = rFile.ReadBool(); - CPropertyDependency *pProp = (IsCharacter ? new CCharPropertyDependency() : new CPropertyDependency()); - pProp->Read(rFile, IDLength); - mChildren.push_back(pProp); - } -} - -void CScriptInstanceDependency::Write(IOutputStream& rFile) const -{ - rFile.WriteLong(mObjectType); - rFile.WriteLong(mChildren.size()); - - for (u32 iProp = 0; iProp < mChildren.size(); iProp++) - { - CPropertyDependency *pProp = static_cast(mChildren[iProp]); - rFile.WriteBool( pProp->Type() == eDNT_CharacterProperty ); - pProp->Write(rFile); - } + rArc << SERIAL("ObjectType", mObjectType) + << SERIAL_ABSTRACT_CONTAINER("Properties", mChildren, "Property", &gDependencyNodeFactory); } // Static @@ -216,23 +161,10 @@ EDependencyNodeType CAnimSetDependencyTree::Type() const return eDNT_AnimSet; } -void CAnimSetDependencyTree::Read(IInputStream& rFile, EIDLength IDLength) +void CAnimSetDependencyTree::Serialize(IArchive& rArc) { - CDependencyTree::Read(rFile, IDLength); - u32 NumChars = rFile.ReadLong(); - mCharacterOffsets.reserve(NumChars); - - for (u32 iChar = 0; iChar < NumChars; iChar++) - mCharacterOffsets.push_back( rFile.ReadLong() ); -} - -void CAnimSetDependencyTree::Write(IOutputStream& rFile) const -{ - CDependencyTree::Write(rFile); - rFile.WriteLong(mCharacterOffsets.size()); - - for (u32 iChar = 0; iChar < mCharacterOffsets.size(); iChar++) - rFile.WriteLong( mCharacterOffsets[iChar] ); + CDependencyTree::Serialize(rArc); + rArc << SERIAL_CONTAINER("CharacterOffsets", mCharacterOffsets, "Offset"); } void CAnimSetDependencyTree::AddCharacter(const SSetCharacter *pkChar, const std::set& rkBaseUsedSet) @@ -282,57 +214,10 @@ EDependencyNodeType CAreaDependencyTree::Type() const return eDNT_Area; } -void CAreaDependencyTree::Read(IInputStream& rFile, EIDLength IDLength) +void CAreaDependencyTree::Serialize(IArchive& rArc) { - mRootID = CAssetID(rFile, IDLength); - - // Base dependency list contains non-script dependencies (world geometry textures + PATH/PTLA/EGMC) - u32 NumBaseDependencies = rFile.ReadLong(); - mChildren.reserve(NumBaseDependencies); - - for (u32 iDep = 0; iDep < NumBaseDependencies; iDep++) - { - CResourceDependency *pDep = new CResourceDependency; - pDep->Read(rFile, IDLength); - mChildren.push_back(pDep); - } - - u32 NumScriptInstances = rFile.ReadLong(); - mChildren.reserve(mChildren.size() + NumScriptInstances); - - for (u32 iInst = 0; iInst < NumScriptInstances; iInst++) - { - CScriptInstanceDependency *pInst = new CScriptInstanceDependency; - pInst->Read(rFile, IDLength); - mChildren.push_back(pInst); - } - - u32 NumLayers = rFile.ReadLong(); - mLayerOffsets.reserve(NumLayers); - - for (u32 iLyr = 0; iLyr < NumLayers; iLyr++) - mLayerOffsets.push_back( rFile.ReadLong() ); -} - -void CAreaDependencyTree::Write(IOutputStream& rFile) const -{ - mRootID.Write(rFile); - - u32 NumBaseDependencies = (mLayerOffsets.empty() ? mChildren.size() : mLayerOffsets.front()); - rFile.WriteLong(NumBaseDependencies); - - for (u32 iDep = 0; iDep < NumBaseDependencies; iDep++) - mChildren[iDep]->Write(rFile); - - rFile.WriteLong(mChildren.size() - NumBaseDependencies); - - for (u32 iDep = NumBaseDependencies; iDep < mChildren.size(); iDep++) - mChildren[iDep]->Write(rFile); - - rFile.WriteLong(mLayerOffsets.size()); - - for (u32 iLyr = 0; iLyr < mLayerOffsets.size(); iLyr++) - rFile.WriteLong(mLayerOffsets[iLyr]); + CDependencyTree::Serialize(rArc); + rArc << SERIAL_CONTAINER("LayerOffsets", mLayerOffsets, "Offset"); } void CAreaDependencyTree::AddScriptLayer(CScriptLayer *pLayer) diff --git a/src/Core/GameProject/CDependencyTree.h b/src/Core/GameProject/CDependencyTree.h index 70a49eb6..8a61bede 100644 --- a/src/Core/GameProject/CDependencyTree.h +++ b/src/Core/GameProject/CDependencyTree.h @@ -14,13 +14,13 @@ struct SSetCharacter; // Group of node classes forming a tree of cached resource dependencies. enum EDependencyNodeType { - eDNT_DependencyTree, - eDNT_ResourceDependency, - eDNT_ScriptInstance, - eDNT_ScriptProperty, - eDNT_CharacterProperty, - eDNT_AnimSet, - eDNT_Area, + eDNT_DependencyTree = FOURCC_CONSTEXPR('T', 'R', 'E', 'E'), + eDNT_ResourceDependency = FOURCC_CONSTEXPR('R', 'S', 'D', 'P'), + eDNT_ScriptInstance = FOURCC_CONSTEXPR('S', 'C', 'I', 'N'), + eDNT_ScriptProperty = FOURCC_CONSTEXPR('S', 'C', 'P', 'R'), + eDNT_CharacterProperty = FOURCC_CONSTEXPR('C', 'R', 'P', 'R'), + eDNT_AnimSet = FOURCC_CONSTEXPR('A', 'N', 'C', 'S'), + eDNT_Area = FOURCC_CONSTEXPR('A', 'R', 'E', 'A'), }; // Base class providing an interface for a basic dependency node. @@ -32,8 +32,7 @@ protected: public: virtual ~IDependencyNode(); virtual EDependencyNodeType Type() const = 0; - virtual void Read(IInputStream& rFile, EIDLength IDLength) = 0; - virtual void Write(IOutputStream& rFile) const = 0; + virtual void Serialize(IArchive& rArc) = 0; virtual bool HasDependency(const CAssetID& rkID) const; // Accessors @@ -48,11 +47,11 @@ protected: CAssetID mRootID; public: + CDependencyTree() {} CDependencyTree(const CAssetID& rkID) : mRootID(rkID) {} virtual EDependencyNodeType Type() const; - virtual void Read(IInputStream& rFile, EIDLength IDLength); - virtual void Write(IOutputStream& rFile) const; + virtual void Serialize(IArchive& rArc); void AddDependency(const CAssetID& rkID, bool AvoidDuplicates = true); void AddDependency(CResource *pRes, bool AvoidDuplicates = true); @@ -73,8 +72,7 @@ public: CResourceDependency(const CAssetID& rkID) : mID(rkID) {} virtual EDependencyNodeType Type() const; - virtual void Read(IInputStream& rFile, EIDLength IDLength); - virtual void Write(IOutputStream& rFile) const; + virtual void Serialize(IArchive& rArc); virtual bool HasDependency(const CAssetID& rkID) const; // Accessors @@ -98,8 +96,7 @@ public: {} virtual EDependencyNodeType Type() const; - virtual void Read(IInputStream& rFile, EIDLength IDLength); - virtual void Write(IOutputStream& rFile) const; + virtual void Serialize(IArchive& rArc); // Accessors inline TString PropertyID() const { return mIDString; } @@ -123,8 +120,7 @@ public: {} virtual EDependencyNodeType Type() const; - virtual void Read(IInputStream& rFile, EIDLength IDLength); - virtual void Write(IOutputStream& rFile) const; + virtual void Serialize(IArchive& rArc); // Accessors inline u32 UsedChar() const { return mUsedChar; } @@ -138,8 +134,7 @@ protected: public: virtual EDependencyNodeType Type() const; - virtual void Read(IInputStream& rFile, EIDLength IDLength); - virtual void Write(IOutputStream& rFile) const; + virtual void Serialize(IArchive& rArc); // Accessors inline u32 ObjectType() const { return mObjectType; } @@ -157,10 +152,10 @@ protected: std::vector mCharacterOffsets; public: + CAnimSetDependencyTree() : CDependencyTree() {} CAnimSetDependencyTree(const CAssetID& rkID) : CDependencyTree(rkID) {} virtual EDependencyNodeType Type() const; - virtual void Read(IInputStream& rFile, EIDLength IDLength); - virtual void Write(IOutputStream& rFile) const; + virtual void Serialize(IArchive& rArc); void AddCharacter(const SSetCharacter *pkChar, const std::set& rkBaseUsedSet); void AddCharDependency(const CAssetID& rkID, std::set& rUsedSet); @@ -178,11 +173,11 @@ protected: std::vector mLayerOffsets; public: + CAreaDependencyTree() : CDependencyTree() {} CAreaDependencyTree(const CAssetID& rkID) : CDependencyTree(rkID) {} virtual EDependencyNodeType Type() const; - virtual void Read(IInputStream& rFile, EIDLength IDLength); - virtual void Write(IOutputStream& rFile) const; + virtual void Serialize(IArchive& rArc); void AddScriptLayer(CScriptLayer *pLayer); void GetModuleDependencies(EGame Game, std::vector& rModuleDepsOut, std::vector& rModuleLayerOffsetsOut) const; @@ -192,5 +187,26 @@ public: inline u32 ScriptLayerOffset(u32 LayerIdx) const { return mLayerOffsets[LayerIdx]; } }; +// Dependency node factory for serialization +class CDependencyNodeFactory +{ +public: + IDependencyNode* SpawnObject(u32 NodeID) + { + switch (NodeID) + { + case eDNT_DependencyTree: return new CDependencyTree; + case eDNT_ResourceDependency: return new CResourceDependency; + case eDNT_ScriptInstance: return new CScriptInstanceDependency; + case eDNT_ScriptProperty: return new CPropertyDependency; + case eDNT_CharacterProperty: return new CCharPropertyDependency; + case eDNT_AnimSet: return new CAnimSetDependencyTree; + case eDNT_Area: return new CAreaDependencyTree; + default: return nullptr; + } + } +}; +extern CDependencyNodeFactory gDependencyNodeFactory; + #endif // CDEPENDENCYTREE diff --git a/src/Core/GameProject/CGameExporter.cpp b/src/Core/GameProject/CGameExporter.cpp index dad52d3c..626d289f 100644 --- a/src/Core/GameProject/CGameExporter.cpp +++ b/src/Core/GameProject/CGameExporter.cpp @@ -526,7 +526,7 @@ void CGameExporter::ExportCookedResources() { SCOPED_TIMER(SaveResourceDatabase); #if EXPORT_COOKED - mStore.SaveResourceDatabase(mpProject->ResourceDBPath(false).ToUTF8()); + mStore.SaveResourceDatabase(); #endif mpProject->Save(); } @@ -537,6 +537,9 @@ void CGameExporter::ExportCookedResources() // some resources will fail to load if their dependencies don't exist SCOPED_TIMER(SaveRawResources); + // todo: we're wasting a ton of time loading the same resources over and over because most resources automatically + // load all their dependencies and then we just clear it out from memory even though we'll need it again later. we + // should really be doing this by dependency order instead of by ID order. for (CResourceIterator It(&mStore); It; ++It) { if (!It->IsTransient()) @@ -556,11 +559,16 @@ void CGameExporter::ExportCookedResources() } } - // Save raw resource + cache data - It->Save(); + // Save raw resource + generate dependencies + It->Save(true); } } } + { + // All resources should have dependencies generated, so save the cache file + SCOPED_TIMER(SaveResourceCacheData); + mStore.SaveCacheFile(); + } } void CGameExporter::ExportResource(SResourceInstance& rRes) diff --git a/src/Core/GameProject/CGameProject.cpp b/src/Core/GameProject/CGameProject.cpp index f02a95ca..5d640bb4 100644 --- a/src/Core/GameProject/CGameProject.cpp +++ b/src/Core/GameProject/CGameProject.cpp @@ -1,8 +1,7 @@ #include "CGameProject.h" #include "Core/Resource/Script/CMasterTemplate.h" -#include +#include -using namespace tinyxml2; CGameProject *CGameProject::mspActiveProject = nullptr; CGameProject::~CGameProject() @@ -16,107 +15,56 @@ CGameProject::~CGameProject() bool CGameProject::Load(const TWideString& rkPath) { - TString ProjPath = rkPath.ToUTF8(); - XMLDocument Doc; - Doc.LoadFile(*ProjPath); - - if (Doc.Error()) - { - Log::Error("Unable to open game project at " + ProjPath); - return false; - } - - XMLElement *pRoot = Doc.FirstChildElement("GameProject"); - //EProjectVersion Version = (EProjectVersion) TString(pRoot->Attribute("Version")).ToInt32(10); - - // Verify all elements are present - XMLElement *pProjName = pRoot->FirstChildElement("Name"); - XMLElement *pGame = pRoot->FirstChildElement("Game"); - XMLElement *pResDB = pRoot->FirstChildElement("ResourceDB"); - XMLElement *pPackages = pRoot->FirstChildElement("Packages"); - - if (!pProjName || !pGame || !pResDB || !pPackages) - { - TString MissingElem = pProjName ? (pGame ? (pResDB ? "Packages" : "ResourceDB") : "Game") : "Name"; - Log::Error("Unable to load game project at " + ProjPath + "; " + MissingElem + " element is missing"); - return false; - } - - mProjectName = pProjName->GetText(); - mGame = CMasterTemplate::FindGameForName( pGame->GetText() ); - mResourceDBPath = pResDB->GetText(); - mProjectRoot = rkPath.GetFileDirectory(); mProjectRoot.Replace(L"/", L"\\"); - // Load packages - XMLElement *pPkgElem = pPackages->FirstChildElement("Package"); - - while (pPkgElem) - { - TString Path = pPkgElem->Attribute("Path"); - - if (Path.IsEmpty()) - Log::Error("Failed to load package in game project " + ProjPath + "; Path attribute is missing or empty"); - - else - { - CPackage *pPackage = new CPackage(this, Path.GetFileName(false), TString(Path.GetFileDirectory()).ToUTF16()); - pPackage->Load(); - mPackages.push_back(pPackage); - } - - pPkgElem = pPkgElem->NextSiblingElement("Package"); - } - - // All loaded! + TString ProjPath = rkPath.ToUTF8(); + CXMLReader Reader(ProjPath); + Serialize(Reader); return true; } void CGameProject::Save() { - XMLDocument Doc; + TString ProjPath = ProjectPath().ToUTF8(); + CXMLWriter Writer(ProjPath, "GameProject", eVer_Current, mGame); + Serialize(Writer); +} - XMLDeclaration *pDecl = Doc.NewDeclaration(); - Doc.LinkEndChild(pDecl); +void CGameProject::Serialize(IArchive& rArc) +{ + rArc << SERIAL("Name", mProjectName) + << SERIAL("Game", mGame) + << SERIAL("ResourceDB", mResourceDBPath); - XMLElement *pRoot = Doc.NewElement("GameProject"); - pRoot->SetAttribute("Version", eVer_Current); - Doc.LinkEndChild(pRoot); + // Packages + std::vector PackageList; - XMLElement *pProjName = Doc.NewElement("Name"); - pProjName->SetText(*mProjectName); - pRoot->LinkEndChild(pProjName); - - XMLElement *pGame = Doc.NewElement("Game"); - pGame->SetText(*CMasterTemplate::FindGameName(mGame)); - pRoot->LinkEndChild(pGame); - - XMLElement *pResDB = Doc.NewElement("ResourceDB"); - pResDB->SetText(*mResourceDBPath.ToUTF8()); - pRoot->LinkEndChild(pResDB); - - XMLElement *pPackages = Doc.NewElement("Packages"); - pRoot->LinkEndChild(pPackages); - - for (u32 iPkg = 0; iPkg < mPackages.size(); iPkg++) + if (!rArc.IsReader()) { - CPackage *pPackage = mPackages[iPkg]; - TWideString FullDefPath = pPackage->DefinitionPath(false); - TWideString RelDefPath = FileUtil::MakeRelative(FullDefPath.GetFileDirectory(), PackagesDir(false)); - TString DefPath = TWideString(RelDefPath + FullDefPath.GetFileName()).ToUTF8(); - - XMLElement *pPakElem = Doc.NewElement("Package"); - pPakElem->SetAttribute("Path", *DefPath); - pPackages->LinkEndChild(pPakElem); + for (u32 iPkg = 0; iPkg < mPackages.size(); iPkg++) + PackageList.push_back( mPackages[iPkg]->DefinitionPath(true).ToUTF8() ); } - // Save Project - TString ProjPath = ProjectPath().ToUTF8(); - XMLError Result = Doc.SaveFile(*ProjPath); + rArc << SERIAL_CONTAINER("Packages", PackageList, "Package"); - if (Result != XML_SUCCESS) - Log::Error("Failed to save game project at: " + ProjPath); + if (rArc.IsReader()) + { + for (u32 iPkg = 0; iPkg < mPackages.size(); iPkg++) + delete mPackages[iPkg]; + mPackages.clear(); + + for (u32 iPkg = 0; iPkg < PackageList.size(); iPkg++) + { + const TWideString& rkPackagePath = PackageList[iPkg]; + TString PackageName = TWideString(rkPackagePath.GetFileName(false)).ToUTF8(); + TWideString PackageDir = rkPackagePath.GetFileDirectory(); + + CPackage *pPackage = new CPackage(this, PackageName, PackageDir); + pPackage->Load(); + mPackages.push_back(pPackage); + } + } } void CGameProject::SetActive() diff --git a/src/Core/GameProject/CGameProject.h b/src/Core/GameProject/CGameProject.h index fa57a022..18ee2f3b 100644 --- a/src/Core/GameProject/CGameProject.h +++ b/src/Core/GameProject/CGameProject.h @@ -46,6 +46,7 @@ public: bool Load(const TWideString& rkPath); void Save(); + void Serialize(IArchive& rArc); void SetActive(); void GetWorldList(std::list& rOut) const; @@ -58,6 +59,7 @@ public: inline TWideString CookedDir(bool Relative) const { return Relative ? L"Cooked\\" : mProjectRoot + L"Cooked\\"; } inline TWideString PackagesDir(bool Relative) const { return Relative ? L"Packages\\" : mProjectRoot + L"Packages\\"; } inline TWideString ProjectPath() const { return mProjectRoot + FileUtil::SanitizeName(mProjectName.ToUTF16(), false) + L".prj"; } + inline TWideString ResourceCachePath(bool Relative) const { return ResourceDBPath(Relative).GetFileDirectory() + L"ResourceCacheData.rcd"; } // Accessors inline void SetGame(EGame Game) { mGame = Game; } diff --git a/src/Core/GameProject/CPackage.cpp b/src/Core/GameProject/CPackage.cpp index d50cdc54..c222114c 100644 --- a/src/Core/GameProject/CPackage.cpp +++ b/src/Core/GameProject/CPackage.cpp @@ -6,58 +6,15 @@ #include #include #include -#include +#include using namespace tinyxml2; void CPackage::Load() { TWideString DefPath = DefinitionPath(false); - - XMLDocument Doc; - Doc.LoadFile(*DefPath.ToUTF8()); - - if (Doc.Error()) - { - Log::Error("Couldn't open pak definition at path: " + DefPath.ToUTF8()); - return; - } - - XMLElement *pRoot = Doc.FirstChildElement("PackageDefinition"); - //EPackageDefinitionVersion Version = (EPackageDefinitionVersion) TString(pRoot->Attribute("Version")).ToInt32(10); - - XMLElement *pColElem = pRoot->FirstChildElement("ResourceCollection"); - - while (pColElem) - { - CResourceCollection *pCollection = AddCollection( pColElem->Attribute("Name") ); - XMLElement *pResElem = pColElem->FirstChildElement("NamedResource"); - - while (pResElem) - { - XMLElement *pNameElem = pResElem->FirstChildElement("Name"); - XMLElement *pIDElem = pResElem->FirstChildElement("ID"); - XMLElement *pTypeElem = pResElem->FirstChildElement("Type"); - - if (!pIDElem || !pNameElem || !pTypeElem) - { - TString ElemName = (pNameElem ? (pIDElem ? "Type" : "ID") : "Name"); - Log::Error("Can't add named resource from pak definition at " + DefPath.ToUTF8() + "; " + ElemName + " element missing"); - } - - else - { - CAssetID ID = CAssetID::FromString(pIDElem->GetText()); - TString Name = pNameElem->GetText(); - CFourCC Type = CFourCC(pTypeElem->GetText()); - pCollection->AddResource(Name, ID, Type); - } - - pResElem = pResElem->NextSiblingElement("NamedResource"); - } - - pColElem = pColElem->NextSiblingElement("ResourceCollection"); - } + CXMLReader Reader(DefPath.ToUTF8()); + Serialize(Reader); } void CPackage::Save() @@ -65,49 +22,13 @@ void CPackage::Save() TWideString DefPath = DefinitionPath(false); FileUtil::CreateDirectory(DefPath.GetFileDirectory()); - // Write XML - XMLDocument Doc; + CXMLWriter Writer(DefPath.ToUTF8(), "PackageDefinition", 0, mpProject ? mpProject->Game() : eUnknownGame); + Serialize(Writer); +} - XMLDeclaration *pDecl = Doc.NewDeclaration(); - Doc.LinkEndChild(pDecl); - - XMLElement *pRoot = Doc.NewElement("PackageDefinition"); - pRoot->SetAttribute("Version", eVer_Current); - Doc.LinkEndChild(pRoot); - - for (u32 iCol = 0; iCol < mCollections.size(); iCol++) - { - CResourceCollection *pCollection = mCollections[iCol]; - - XMLElement *pColElem = Doc.NewElement("ResourceCollection"); - pColElem->SetAttribute("Name", *pCollection->Name()); - pRoot->LinkEndChild(pColElem); - - for (u32 iRes = 0; iRes < pCollection->NumResources(); iRes++) - { - const SNamedResource& rkRes = pCollection->ResourceByIndex(iRes); - - XMLElement *pResElem = Doc.NewElement("NamedResource"); - pColElem->LinkEndChild(pResElem); - - XMLElement *pName = Doc.NewElement("Name"); - pName->SetText(*rkRes.Name); - pResElem->LinkEndChild(pName); - - XMLElement *pID = Doc.NewElement("ID"); - pID->SetText(*rkRes.ID.ToString()); - pResElem->LinkEndChild(pID); - - XMLElement *pType = Doc.NewElement("Type"); - pType->SetText(*rkRes.Type.ToString()); - pResElem->LinkEndChild(pType); - } - } - - XMLError Error = Doc.SaveFile(*DefPath.ToUTF8()); - - if (Error != XML_SUCCESS) - Log::Error("Failed to save pak definition at path: " + DefPath.ToUTF8()); +void CPackage::Serialize(IArchive& rArc) +{ + rArc << SERIAL_CONTAINER("Collections", mCollections, "ResourceCollection"); } void CPackage::Cook() @@ -355,12 +276,14 @@ void CPackage::CompareOriginalAssetList(const std::list& rkNewList) TWideString CPackage::DefinitionPath(bool Relative) const { - return mpProject->PackagesDir(Relative) + mPakPath + mPakName.ToUTF16() + L".pkd"; + TWideString RelPath = mPakPath + mPakName.ToUTF16() + L".pkd"; + return Relative ? RelPath : mpProject->PackagesDir(false) + RelPath; } TWideString CPackage::CookedPackagePath(bool Relative) const { - return mpProject->DiscDir(Relative) + mPakPath + mPakName.ToUTF16() + L".pak"; + TWideString RelPath = mPakPath + mPakName.ToUTF16() + L".pak"; + return Relative ? RelPath : mpProject->DiscDir(false) + RelPath; } CResourceCollection* CPackage::AddCollection(const TString& rkName) diff --git a/src/Core/GameProject/CPackage.h b/src/Core/GameProject/CPackage.h index 2914e731..0c20318f 100644 --- a/src/Core/GameProject/CPackage.h +++ b/src/Core/GameProject/CPackage.h @@ -4,6 +4,7 @@ #include #include #include +#include class CGameProject; @@ -12,6 +13,11 @@ struct SNamedResource TString Name; CAssetID ID; CFourCC Type; + + void Serialize(IArchive& rArc) + { + rArc << SERIAL_AUTO(Name) << SERIAL_AUTO(ID) << SERIAL_AUTO(Type); + } }; class CResourceCollection @@ -23,6 +29,11 @@ public: CResourceCollection() : mName("UNNAMED") {} CResourceCollection(const TString& rkName) : mName(rkName) {} + void Serialize(IArchive& rArc) + { + rArc << SERIAL("Name", mName) << SERIAL_CONTAINER("Resources", mNamedResources, "Resource"); + } + inline TString Name() const { return mName; } inline u32 NumResources() const { return mNamedResources.size(); } inline const SNamedResource& ResourceByIndex(u32 Idx) const { return mNamedResources[Idx]; } @@ -59,6 +70,7 @@ public: void Load(); void Save(); + void Serialize(IArchive& rArc); void Cook(); void CompareOriginalAssetList(const std::list& rkNewList); diff --git a/src/Core/GameProject/CResourceEntry.cpp b/src/Core/GameProject/CResourceEntry.cpp index 989d67f3..0a6c8564 100644 --- a/src/Core/GameProject/CResourceEntry.cpp +++ b/src/Core/GameProject/CResourceEntry.cpp @@ -34,87 +34,18 @@ CResourceEntry::~CResourceEntry() if (mpDependencies) delete mpDependencies; } -bool CResourceEntry::LoadCacheData() +void CResourceEntry::SerializeCacheData(IArchive& rArc) { ASSERT(!IsTransient()); - TWideString Path = CacheDataPath(false); - CFileInStream File(Path.ToUTF8().ToStdString(), IOUtil::eLittleEndian); + u32 Flags = mFlags & eREF_SavedFlags; + rArc << SERIAL_AUTO(Flags); + if (rArc.IsReader()) mFlags = Flags & eREF_SavedFlags; - if (!File.IsValid()) - { - Log::Error("Unable to load cache data " + Path.ToUTF8() + "; couldn't open file"); - return false; - } - - // Header - TString Magic = File.ReadString(4); - ASSERT(Magic == "CACH"); - File.Seek(0x4, SEEK_CUR); // Skip Version - mFlags = File.ReadLong() & eREF_SavedFlags; - - // Dependency Tree - u32 DepsTreeSize = File.ReadLong(); - u32 DepsTreeStart = File.Tell(); - - if (mpDependencies) - { - delete mpDependencies; - mpDependencies = nullptr; - } - - if (DepsTreeSize > 0) - { - if (mType == eArea) mpDependencies = new CAreaDependencyTree(mID); - else if (mType == eAnimSet) mpDependencies = new CAnimSetDependencyTree(mID); - else mpDependencies = new CDependencyTree(mID); - - mpDependencies->Read(File, Game() <= eEchoes ? e32Bit : e64Bit); - ASSERT(File.Tell() - DepsTreeStart == DepsTreeSize); - } - - return true; -} - -bool CResourceEntry::SaveCacheData() -{ - ASSERT(!IsTransient()); - - TWideString Path = CacheDataPath(false); - TWideString Dir = Path.GetFileDirectory(); - FileUtil::CreateDirectory(Dir); - CFileOutStream File(Path.ToUTF8().ToStdString(), IOUtil::eLittleEndian); - - if (!File.IsValid()) - { - Log::Error("Unable to save cache data " + TString(Path.GetFileName()) + "; couldn't open file"); - return false; - } - - // Header - File.WriteString("CACH", 4); - File.WriteLong(0); // Reserved Space (Version) - File.WriteLong(mFlags & eREF_SavedFlags); - - // Dependency Tree - if (!mpDependencies) UpdateDependencies(); - - u32 DepsSizeOffset = File.Tell(); - File.WriteLong(0); - - u32 DepsStart = File.Tell(); - if (mpDependencies) mpDependencies->Write(File); - u32 DepsEnd = File.Tell(); - u32 DepsSize = DepsEnd- DepsStart; - - File.Seek(DepsSizeOffset, SEEK_SET); - File.WriteLong(DepsSize); - File.Seek(DepsEnd, SEEK_SET); - - // Thumbnail - File.WriteLong(0); // Reserved Space (Thumbnail Size) - - return true; + // Note: If the dependency tree format is changed this should be adjusted so that + // we regenerate the dependencies from scratch instead of reading the tree if the + // file version number is too low + rArc << SERIAL_ABSTRACT("Dependencies", mpDependencies, &gDependencyNodeFactory); } void CResourceEntry::UpdateDependencies() @@ -125,6 +56,14 @@ void CResourceEntry::UpdateDependencies() mpDependencies = nullptr; } + // todo: more robust method of skipping dependency updates. For now just skip the most + // time-consuming type that can't have dependencies (textures). + if (ResourceType() == eTexture) + { + mpDependencies = new CDependencyTree(ID()); + return; + } + bool WasLoaded = IsLoaded(); if (!mpResource) @@ -225,8 +164,13 @@ void CResourceEntry::SetGame(EGame NewGame) } } -bool CResourceEntry::Save() +bool CResourceEntry::Save(bool SkipCacheSave /*= false*/) { + // SkipCacheSave argument tells us not to save the resource cache file. This is generally not advised because we don't + // want the actual resource data to desync from the cache data. However, there are occasions where we save multiple + // resources at a time and in that case it's preferable to only save the cache once. If you do set SkipCacheSave to true, + // then make sure you manually update the cache file afterwards. + // // For now, always save the resource when this function is called even if there's been no changes made to it in memory. // In the future this might not be desired behavior 100% of the time. // We also might want this function to trigger a cook for certain resource types eventually. @@ -249,9 +193,11 @@ bool CResourceEntry::Save() mpResource->Serialize(Writer); } - // Resource has been saved, now update cache file + // Resource has been saved, now update dependencies + cache file UpdateDependencies(); - SaveCacheData(); + + if (!SkipCacheSave) + mpStore->SaveCacheFile(); if (ShouldCollectGarbage) mpStore->DestroyUnreferencedResources(); diff --git a/src/Core/GameProject/CResourceEntry.h b/src/Core/GameProject/CResourceEntry.h index 294cfab5..ccfb28b1 100644 --- a/src/Core/GameProject/CResourceEntry.h +++ b/src/Core/GameProject/CResourceEntry.h @@ -44,8 +44,7 @@ public: EResType Type, bool Transient = false); ~CResourceEntry(); - bool LoadCacheData(); - bool SaveCacheData(); + void SerializeCacheData(IArchive& rArc); void UpdateDependencies(); TWideString CacheDataPath(bool Relative = false) const; @@ -58,7 +57,7 @@ public: u64 Size() const; bool NeedsRecook() const; void SetGame(EGame NewGame); - bool Save(); + bool Save(bool SkipCacheSave = false); CResource* Load(); CResource* LoadCooked(IInputStream& rInput); bool Unload(); diff --git a/src/Core/GameProject/CResourceStore.cpp b/src/Core/GameProject/CResourceStore.cpp index 98c18c67..a36f95cb 100644 --- a/src/Core/GameProject/CResourceStore.cpp +++ b/src/Core/GameProject/CResourceStore.cpp @@ -1,10 +1,13 @@ #include "CResourceStore.h" #include "CGameExporter.h" #include "CGameProject.h" +#include "CResourceIterator.h" #include "Core/Resource/CResource.h" #include #include #include +#include +#include #include using namespace tinyxml2; @@ -34,92 +37,151 @@ CResourceStore::~CResourceStore() delete *It; } -void CResourceStore::LoadResourceDatabase(const TString& rkPath) +void CResourceStore::SerializeResourceDatabase(IArchive& rArc) { - XMLDocument Doc; - Doc.LoadFile(*rkPath); - - if (!Doc.Error()) + struct SDatabaseResource { - XMLElement *pRoot = Doc.FirstChildElement("ResourceDatabase"); - //EDatabaseVersion Version = (EDatabaseVersion) TString(pRoot->Attribute("Version")).ToInt32(10); // Version currently unused + CAssetID ID; + CFourCC Type; + TWideString Directory; + TWideString Name; - XMLElement *pResources = pRoot->FirstChildElement("Resources"); - XMLElement *pRes = pResources->FirstChildElement("Resource"); - u32 ResIndex = 0; - - while (pRes) + void Serialize(IArchive& rArc) { - XMLElement *pID = pRes->FirstChildElement("ID"); - XMLElement *pType = pRes->FirstChildElement("Type"); - XMLElement *pDir = pRes->FirstChildElement("FileDir"); - XMLElement *pName = pRes->FirstChildElement("FileName"); + rArc << SERIAL_AUTO(ID) << SERIAL_AUTO(Type) << SERIAL_AUTO(Directory) << SERIAL_AUTO(Name); + } + }; + std::vector Resources; - if (pID && pType && pDir && pName) - { - CAssetID ID = CAssetID::FromString(pID->GetText()); - EResType Type = CResource::ResTypeForExtension(pType->GetText()); - TWideString FileDir = pDir->GetText(); - TWideString FileName = pName->GetText(); - RegisterResource(ID, Type, FileDir, FileName); - } - else - Log::Error("Error reading " + rkPath + ": Resource entry " + TString::FromInt32(ResIndex, 0, 10) + " is missing one or more required components"); + // Populate resource list + if (!rArc.IsReader()) + { + Resources.reserve(mResourceEntries.size()); - ResIndex++; - pRes = pRes->NextSiblingElement("Resource"); + for (CResourceIterator It(this); It; ++It) + { + if (!It->IsTransient()) + Resources.push_back( SDatabaseResource { It->ID(), It->CookedExtension(), It->Directory()->FullPath(), It->Name() } ); } } - // All resources registered - load cache data - for (auto It = mResourceEntries.begin(); It != mResourceEntries.end(); It++) + // Serialize + rArc << SERIAL_CONTAINER_AUTO(Resources, "Resource"); + + // Register resources + if (rArc.IsReader()) { - CResourceEntry *pEntry = It->second; - if (!pEntry->IsTransient()) - pEntry->LoadCacheData(); + for (auto Iter = Resources.begin(); Iter != Resources.end(); Iter++) + { + SDatabaseResource& rRes = *Iter; + RegisterResource(rRes.ID, CResource::ResTypeForExtension(rRes.Type), rRes.Directory, rRes.Name); + } } } -void CResourceStore::SaveResourceDatabase(const TString& rkPath) const +void CResourceStore::LoadResourceDatabase() { - XMLDocument Doc; + ASSERT(mpProj); + TString Path = mpProj->ResourceDBPath(false).ToUTF8(); - XMLDeclaration *pDecl = Doc.NewDeclaration(); - Doc.LinkEndChild(pDecl); + CXMLReader Reader(Path); + SerializeResourceDatabase(Reader); + LoadCacheFile(); +} - XMLElement *pRoot = Doc.NewElement("ResourceDatabase"); - pRoot->SetAttribute("Version", eVer_Current); - Doc.LinkEndChild(pRoot); +void CResourceStore::SaveResourceDatabase() +{ + ASSERT(mpProj); + TString Path = mpProj->ResourceDBPath(false).ToUTF8(); - XMLElement *pResources = Doc.NewElement("Resources"); - pRoot->LinkEndChild(pResources); + CXMLWriter Writer(Path, "ResourceDB", 0, mpProj ? mpProj->Game() : eUnknownGame); + SerializeResourceDatabase(Writer); +} - for (auto It = mResourceEntries.begin(); It != mResourceEntries.end(); It++) +void CResourceStore::LoadCacheFile() +{ + TString CacheDataPath = mpProj->ResourceCachePath(false).ToUTF8(); + CFileInStream CacheFile(CacheDataPath.ToStdString(), IOUtil::eBigEndian); + ASSERT(CacheFile.IsValid()); + + // Cache header + CFourCC Magic(CacheFile); + + if (Magic != "CACH") { - CResourceEntry *pEntry = It->second; - if (pEntry->IsTransient()) continue; - - XMLElement *pRes = Doc.NewElement("Resource"); - pResources->LinkEndChild(pRes); - - XMLElement *pID = Doc.NewElement("ID"); - pID->SetText(*pEntry->ID().ToString()); - pRes->LinkEndChild(pID); - - XMLElement *pType = Doc.NewElement("Type"); - pType->SetText(*GetResourceCookedExtension(pEntry->ResourceType(), pEntry->Game())); - pRes->LinkEndChild(pType); - - XMLElement *pDir = Doc.NewElement("FileDir"); - pDir->SetText(*pEntry->Directory()->FullPath().ToUTF8()); - pRes->LinkEndChild(pDir); - - XMLElement *pName = Doc.NewElement("FileName"); - pName->SetText(*pEntry->Name().ToUTF8()); - pRes->LinkEndChild(pName); + Log::Error("Invalid resource cache data magic: " + Magic.ToString()); + return; } - Doc.SaveFile(*rkPath); + CSerialVersion Version(CacheFile); + u32 NumResources = CacheFile.ReadLong(); + + for (u32 iRes = 0; iRes < NumResources; iRes++) + { + CAssetID ID(CacheFile, Version.Game()); + u32 EntryCacheSize = CacheFile.ReadLong(); + u32 EntryCacheEnd = CacheFile.Tell() + EntryCacheSize; + + CResourceEntry *pEntry = FindEntry(ID); + + if (pEntry && !pEntry->IsTransient()) + { + CBasicBinaryReader Reader(&CacheFile, Version); + + if (Reader.ParamBegin("EntryCache")) + { + pEntry->SerializeCacheData(Reader); + Reader.ParamEnd(); + } + } + + CacheFile.Seek(EntryCacheEnd, SEEK_SET); + } +} + +void CResourceStore::SaveCacheFile() +{ + TString CacheDataPath = mpProj->ResourceCachePath(false).ToUTF8(); + CFileOutStream CacheFile(CacheDataPath.ToStdString(), IOUtil::eBigEndian); + ASSERT(CacheFile.IsValid()); + + // Cache header + CFourCC("CACH").Write(CacheFile); + CSerialVersion Version(0, 0, mpProj->Game()); + Version.Write(CacheFile); + + u32 ResCountOffset = CacheFile.Tell(); + u32 ResCount = 0; + CacheFile.WriteLong(0); // Resource count dummy - fill in when we know the real count + + // Save entry cache data + // Structure: Entry Asset ID -> Entry Cache Size -> Serialized Entry Cache Data + for (CResourceIterator It(this); It; ++It) + { + if (!It->IsTransient()) + { + ResCount++; + It->ID().Write(CacheFile); + u32 SizeOffset = CacheFile.Tell(); + CacheFile.WriteLong(0); + + CBasicBinaryWriter Writer(&CacheFile, Version.FileVersion(), Version.Game()); + + if (Writer.ParamBegin("EntryCache")) + { + It->SerializeCacheData(Writer); + Writer.ParamEnd(); + } + + u32 EntryCacheEnd = CacheFile.Tell(); + CacheFile.Seek(SizeOffset, SEEK_SET); + CacheFile.WriteLong(EntryCacheEnd - SizeOffset - 4); + CacheFile.Seek(EntryCacheEnd, SEEK_SET); + } + } + + CacheFile.Seek(ResCountOffset, SEEK_SET); + CacheFile.WriteLong(ResCount); } void CResourceStore::SetActiveProject(CGameProject *pProj) @@ -134,7 +196,7 @@ void CResourceStore::SetActiveProject(CGameProject *pProj) mpProjectRoot = new CVirtualDirectory(); if (!mpExporter) - LoadResourceDatabase(pProj->ResourceDBPath(false)); + LoadResourceDatabase(); } } @@ -145,7 +207,9 @@ void CResourceStore::CloseActiveProject() DestroyUnreferencedResources(); // Delete all entries from old project - for (auto It = mResourceEntries.begin(); It != mResourceEntries.end(); It++) + auto It = mResourceEntries.begin(); + + while (It != mResourceEntries.end()) { CResourceEntry *pEntry = It->second; @@ -161,11 +225,12 @@ void CResourceStore::CloseActiveProject() mLoadedResources.erase(LoadIt); } - It = mResourceEntries.erase(It); - if (It == mResourceEntries.end()) break; - delete pEntry; + It = mResourceEntries.erase(It); } + + else + It++; } delete mpProjectRoot; @@ -402,8 +467,9 @@ void CResourceStore::DestroyUnreferencedResources() do { NumDeleted = 0; + auto It = mLoadedResources.begin(); - for (auto It = mLoadedResources.begin(); It != mLoadedResources.end(); It++) + while (It != mLoadedResources.end()) { CResourceEntry *pEntry = It->second; @@ -415,9 +481,9 @@ void CResourceStore::DestroyUnreferencedResources() // Transient resources should have their entries cleared out when the resource is unloaded if (pEntry->IsTransient()) DeleteResourceEntry(pEntry); - - if (It == mLoadedResources.end()) break; } + + else It++; } } while (NumDeleted > 0); diff --git a/src/Core/GameProject/CResourceStore.h b/src/Core/GameProject/CResourceStore.h index 64e43ace..c3e62042 100644 --- a/src/Core/GameProject/CResourceStore.h +++ b/src/Core/GameProject/CResourceStore.h @@ -42,8 +42,11 @@ public: CResourceStore(); CResourceStore(CGameExporter *pExporter); ~CResourceStore(); - void LoadResourceDatabase(const TString& rkPath); - void SaveResourceDatabase(const TString& rkPath) const; + void SerializeResourceDatabase(IArchive& rArc); + void LoadResourceDatabase(); + void SaveResourceDatabase(); + void LoadCacheFile(); + void SaveCacheFile(); void SetActiveProject(CGameProject *pProj); void CloseActiveProject(); CVirtualDirectory* GetVirtualDirectory(const TWideString& rkPath, bool Transient, bool AllowCreate); diff --git a/src/Core/Resource/Area/CGameArea.cpp b/src/Core/Resource/Area/CGameArea.cpp index c28d79a9..e4fa04e6 100644 --- a/src/Core/Resource/Area/CGameArea.cpp +++ b/src/Core/Resource/Area/CGameArea.cpp @@ -43,8 +43,12 @@ CDependencyTree* CGameArea::BuildDependencyTree() const pTree->AddDependency(*Iter); pTree->AddDependency(mPathID); - pTree->AddDependency(mPortalAreaID); - pTree->AddDependency(mpPoiToWorldMap); + + if (Game() >= eEchoesDemo) + { + pTree->AddDependency(mPortalAreaID); + pTree->AddDependency(mpPoiToWorldMap); + } // Layer dependencies for (u32 iLayer = 0; iLayer < mScriptLayers.size(); iLayer++) diff --git a/src/Core/Resource/CWorld.cpp b/src/Core/Resource/CWorld.cpp index 15cfa970..4eb6c0bf 100644 --- a/src/Core/Resource/CWorld.cpp +++ b/src/Core/Resource/CWorld.cpp @@ -62,14 +62,14 @@ void CWorld::SetAreaLayerInfo(CGameArea *pArea) // ************ SERIALIZATION ************ void CWorld::Serialize(IArchive& rArc) { - rArc << SERIAL("WorldNameSTRG", mpWorldName); + rArc << SERIAL("WorldNameString", mpWorldName); if (rArc.Game() == eEchoesDemo || rArc.Game() == eEchoes) - rArc << SERIAL("DarkWorldNameSTRG", mpDarkWorldName); + rArc << SERIAL("DarkWorldNameString", mpDarkWorldName); rArc << SERIAL("WorldSaveInfo", mpSaveWorld) << SERIAL("WorldMap", mpMapWorld) - << SERIAL("DefaultSkyCMDL", mpDefaultSkybox); + << SERIAL("DefaultSkyModel", mpDefaultSkybox); if (rArc.Game() >= eEchoesDemo && rArc.Game() <= eCorruption) rArc << SERIAL("TempleKeyWorldIndex", mTempleKeyWorldIndex); @@ -94,11 +94,11 @@ void Serialize(IArchive& rArc, CWorld::SMemoryRelay& rMemRelay) void Serialize(IArchive& rArc, CWorld::SArea& rArea) { rArc << SERIAL("Name", rArea.InternalName) - << SERIAL("NameSTRG", rArea.pAreaName) + << SERIAL("NameString", rArea.pAreaName) + << SERIAL("MREA", rArea.AreaResID) + << SERIAL("ID", rArea.AreaID) << SERIAL("Transform", rArea.Transform) << SERIAL("BoundingBox", rArea.AetherBox) - << SERIAL("AreaMREA", rArea.AreaResID) - << SERIAL("AreaID", rArea.AreaID) << SERIAL("AllowPakDuplicates", rArea.AllowPakDuplicates) << SERIAL_CONTAINER("AttachedAreas", rArea.AttachedAreaIDs, "AreaIndex") << SERIAL_CONTAINER("RelModules", rArea.RelFilenames, "Module")