Modified all editor file formats to use the serialization system; changed dependency caching so all resource cache data is in one file

src/Common/CAssetID.cpp

@@ -68,7 +68,7 @@ TString CAssetID::ToString() const
 
 bool CAssetID::IsValid() const
 {
-    return *this != InvalidID(mLength);
+    return (mID != 0 && mLength != eInvalidIDLength && mID != InvalidID(mLength).mID);
 }
 
 // ************ STATIC ************

src/Common/CFourCC.h

@@ -4,7 +4,6 @@
 #include "AssertMacro.h"
 #include "types.h"
 #include "TString.h"
-#include "Common/Serialization/IArchive.h"
 #include <FileIO/IInputStream.h>
 #include <FileIO/IOutputStream.h>
 
@@ -42,13 +41,6 @@ public:
         rOutput.WriteLong(Val);
     }
 
-    inline void Serialize(IArchive& rArc)
-    {
-        TString Str = ToString();
-        rArc.SerializePrimitive(Str);
-        if (rArc.IsReader()) *this = CFourCC(Str);
-    }
-
     inline u32 ToLong() const
     {
         return mFourCC;

src/Common/Common.pro

@@ -85,7 +85,10 @@ HEADERS += \
     Serialization/CBasicBinaryWriter.h \
     Serialization/CBasicBinaryReader.h \
     Serialization/CBinaryWriter.h \
-    Serialization/CBinaryReader.h
+    Serialization/CBinaryReader.h \
+    Serialization/CSerialVersion.h \
+    Serialization/XML.h \
+    Serialization/Binary.h
 
 # Source Files
 SOURCES += \
@@ -96,4 +99,5 @@ SOURCES += \
     Log.cpp \
     FileUtil.cpp \
     CAssetID.cpp \
-    EGame.cpp
+    EGame.cpp \
+    Serialization/CSerialVersion.cpp

src/Common/EGame.cpp

@@ -1,5 +1,6 @@
 #include "EGame.h"
 #include "CFourCC.h"
+#include "Common/Serialization/IArchive.h"
 
 CFourCC GetGameID(EGame Game)
 {
@@ -27,3 +28,10 @@ EGame GetGameForID(const CFourCC& rkID)
     if (rkID == "DKCR") return eReturns;
     return eUnknownGame;
 }
+
+void Serialize(IArchive& rArc, EGame& rGame)
+{
+    CFourCC GameID = GetGameID(rGame);
+    rArc.SerializePrimitive(GameID);
+    if (rArc.IsReader()) rGame = GetGameForID(GameID);
+}

src/Common/EGame.h

@@ -5,6 +5,7 @@
 #include "types.h"
 
 class CFourCC;
+class IArchive;
 
 enum EGame
 {
@@ -20,5 +21,6 @@ enum EGame
 
 CFourCC GetGameID(EGame Game);
 EGame GetGameForID(const CFourCC& rkID);
+void Serialize(IArchive& rArc, EGame& rGame);
 
 #endif // EGAME_H

src/Common/Log.cpp

@@ -69,17 +69,19 @@ bool InitLog(const TString& rkFilename)
 
 void Write(const TString& rkMessage)
 {
+    double Time = CTimer::GlobalTime() - gAppStartTime;
+
     if (!gInitialized)
         gPreInitLogs.push_back(rkMessage);
 
     else if (gpLogFile)
     {
-        double Time = CTimer::GlobalTime() - gAppStartTime;
         fprintf(gpLogFile, "[%08.3f] %s\n", Time, *rkMessage);
         fflush(gpLogFile);
     }
 
-    std::cout << rkMessage << "\n";
+    std::cout << std::fixed << std::setprecision(3)
+              << "["  << Time << "] " << rkMessage << "\n";
 }
 
 void Error(const TString& rkMessage)

src/Common/Serialization/Binary.h

@@ -0,0 +1,5 @@
+#include "IArchive.h"
+#include "CBinaryReader.h"
+#include "CBinaryWriter.h"
+#include "CBasicBinaryReader.h"
+#include "CBasicBinaryWriter.h"

src/Common/Serialization/CBasicBinaryReader.h

@@ -2,6 +2,7 @@
 #define CBASICBINARYREADER
 
 #include "IArchive.h"
+#include "CSerialVersion.h"
 #include "Common/CFourCC.h"
 #include <FileIO/IInputStream.h>
 
@@ -21,21 +22,17 @@ public:
         mpStream = new CFileInStream(rkFilename.ToStdString(), IOUtil::eBigEndian);
         ASSERT(mpStream->IsValid());
 
-        mFileVersion = mpStream->ReadShort();
+        CSerialVersion Version(*mpStream);
-        mArchiveVersion = mpStream->ReadShort();
+        SetVersion(Version);
-        mGame = GetGameForID( CFourCC(*mpStream) );
     }
 
-    CBasicBinaryReader(IInputStream *pStream)
+    CBasicBinaryReader(IInputStream *pStream, const CSerialVersion& rkVersion)
         : IArchive(true, false)
         , mOwnsStream(false)
     {
         ASSERT(pStream->IsValid());
         mpStream = pStream;
-
+        SetVersion(rkVersion);
-        mFileVersion = mpStream->ReadShort();
-        mArchiveVersion = mpStream->ReadShort();
-        mGame = GetGameForID( CFourCC(*mpStream) );
     }
 
     ~CBasicBinaryReader()
@@ -62,15 +59,13 @@ public:
     virtual void SerializePrimitive(float& rValue)          { rValue = mpStream->ReadFloat(); }
     virtual void SerializePrimitive(double& rValue)         { rValue = mpStream->ReadDouble(); }
     virtual void SerializePrimitive(TString& rValue)        { rValue = mpStream->ReadSizedString(); }
+    virtual void SerializePrimitive(TWideString& rValue)    { rValue = mpStream->ReadSizedWString(); }
+    virtual void SerializePrimitive(CFourCC& rValue)        { rValue = CFourCC(*mpStream); }
     virtual void SerializePrimitive(CAssetID& rValue)       { rValue = CAssetID(*mpStream, mGame); }
 
-    virtual void SerializeHexPrimitive(s8& rValue)          { rValue = mpStream->ReadByte(); }
     virtual void SerializeHexPrimitive(u8& rValue)          { rValue = mpStream->ReadByte(); }
-    virtual void SerializeHexPrimitive(s16& rValue)         { rValue = mpStream->ReadShort(); }
     virtual void SerializeHexPrimitive(u16& rValue)         { rValue = mpStream->ReadShort(); }
-    virtual void SerializeHexPrimitive(s32& rValue)         { rValue = mpStream->ReadLong(); }
     virtual void SerializeHexPrimitive(u32& rValue)         { rValue = mpStream->ReadLong(); }
-    virtual void SerializeHexPrimitive(s64& rValue)         { rValue = mpStream->ReadLongLong(); }
     virtual void SerializeHexPrimitive(u64& rValue)         { rValue = mpStream->ReadLongLong(); }
 };
 

src/Common/Serialization/CBasicBinaryWriter.h

@@ -14,34 +14,32 @@ class CBasicBinaryWriter : public IArchive
     bool mOwnsStream;
 
 public:
-    CBasicBinaryWriter(const TString& rkFilename, u32 FileVersion, EGame Game = eUnknownGame, IOUtil::EEndianness = IOUtil::eLittleEndian)
+    CBasicBinaryWriter(const TString& rkFilename, u16 FileVersion, EGame Game = eUnknownGame, IOUtil::EEndianness = IOUtil::eLittleEndian)
         : IArchive(false, true)
         , mOwnsStream(true)
     {
         mpStream = new CFileOutStream(rkFilename.ToStdString(), IOUtil::eBigEndian);
         ASSERT(mpStream->IsValid());
-
+        SetVersion(skCurrentArchiveVersion, FileVersion, Game);
-        mFileVersion = FileVersion;
+        GetVersionInfo().Write(*mpStream);
-        mGame = Game;
-
-        mpStream->WriteShort((u16) FileVersion);
-        mpStream->WriteShort((u16) skCurrentArchiveVersion);
-        GetGameID(Game).Write(*mpStream);
     }
 
-    CBasicBinaryWriter(IOutputStream *pStream, u32 FileVersion, EGame Game = eUnknownGame)
+    CBasicBinaryWriter(IOutputStream *pStream, u16 FileVersion, EGame Game = eUnknownGame)
         : IArchive(false, true)
         , mOwnsStream(false)
     {
         ASSERT(pStream->IsValid());
         mpStream = pStream;
+        SetVersion(skCurrentArchiveVersion, FileVersion, Game);
+    }
 
-        mFileVersion = FileVersion;
+    CBasicBinaryWriter(IOutputStream *pStream, const CSerialVersion& rkVersion)
-        mGame = Game;
+        : IArchive(false, true)
-
+        , mOwnsStream(false)
-        mpStream->WriteShort((u16) FileVersion);
+    {
-        mpStream->WriteShort((u16) skCurrentArchiveVersion);
+        ASSERT(pStream->IsValid());
-        GetGameID(Game).Write(*mpStream);
+        mpStream = pStream;
+        SetVersion(rkVersion);
     }
 
     ~CBasicBinaryWriter()
@@ -53,8 +51,8 @@ public:
     virtual bool ParamBegin(const char*)    { return true; }
     virtual void ParamEnd()                 { }
 
-    virtual void SerializeContainerSize(u32& rSize)         { SerializePrimitive(rSize); }
+    virtual void SerializeContainerSize(u32& rSize)         { mpStream->WriteLong(rSize); }
-    virtual void SerializeAbstractObjectType(u32& rType)    { SerializePrimitive(rType); }
+    virtual void SerializeAbstractObjectType(u32& rType)    { mpStream->WriteLong(rType); }
     virtual void SerializePrimitive(bool& rValue)           { mpStream->WriteBool(rValue); }
     virtual void SerializePrimitive(char& rValue)           { mpStream->WriteByte(rValue); }
     virtual void SerializePrimitive(s8& rValue)             { mpStream->WriteByte(rValue); }
@@ -68,15 +66,13 @@ public:
     virtual void SerializePrimitive(float& rValue)          { mpStream->WriteFloat(rValue); }
     virtual void SerializePrimitive(double& rValue)         { mpStream->WriteDouble(rValue); }
     virtual void SerializePrimitive(TString& rValue)        { mpStream->WriteSizedString(rValue.ToStdString()); }
+    virtual void SerializePrimitive(TWideString& rValue)    { mpStream->WriteSizedWideString(rValue.ToStdString()); }
+    virtual void SerializePrimitive(CFourCC& rValue)        { rValue.Write(*mpStream); }
     virtual void SerializePrimitive(CAssetID& rValue)       { rValue.Write(*mpStream); }
 
-    virtual void SerializeHexPrimitive(s8& rValue)          { mpStream->WriteByte(rValue); }
     virtual void SerializeHexPrimitive(u8& rValue)          { mpStream->WriteByte(rValue); }
-    virtual void SerializeHexPrimitive(s16& rValue)         { mpStream->WriteShort(rValue); }
     virtual void SerializeHexPrimitive(u16& rValue)         { mpStream->WriteShort(rValue); }
-    virtual void SerializeHexPrimitive(s32& rValue)         { mpStream->WriteLong(rValue); }
     virtual void SerializeHexPrimitive(u32& rValue)         { mpStream->WriteLong(rValue); }
-    virtual void SerializeHexPrimitive(s64& rValue)         { mpStream->WriteLongLong(rValue); }
     virtual void SerializeHexPrimitive(u64& rValue)         { mpStream->WriteLongLong(rValue); }
 };
 

src/Common/Serialization/CBinaryReader.h

@@ -2,6 +2,7 @@
 #define CBINARYREADER
 
 #include "IArchive.h"
+#include "CSerialVersion.h"
 #include "Common/CFourCC.h"
 
 class CBinaryReader : public IArchive
@@ -11,6 +12,7 @@ class CBinaryReader : public IArchive
         u32 Offset;
         u16 Size;
         bool HasChildren;
+        bool Abstract;
     };
     std::vector<SParameter> mParamStack;
 
@@ -25,23 +27,19 @@ public:
         mpStream = new CFileInStream(rkFilename.ToStdString(), IOUtil::eBigEndian);
         ASSERT(mpStream->IsValid());
 
-        mFileVersion = mpStream->ReadShort();
+        CSerialVersion Version(*mpStream);
-        mArchiveVersion = mpStream->ReadShort();
+        SetVersion(Version);
-        mGame = GetGameForID( CFourCC(*mpStream) );
 
         mParamStack.reserve(20);
     }
 
-    CBinaryReader(IInputStream *pStream)
+    CBinaryReader(IInputStream *pStream, const CSerialVersion& rkVersion)
         : IArchive(true, false)
         , mOwnsStream(false)
     {
         ASSERT(pStream->IsValid());
         mpStream = pStream;
-
+        SetVersion(rkVersion);
-        mFileVersion = mpStream->ReadShort();
-        mArchiveVersion = mpStream->ReadShort();
-        mGame = GetGameForID( CFourCC(*mpStream) );
 
         mParamStack.reserve(20);
     }
@@ -70,16 +68,16 @@ public:
         // Does the next parameter ID match the current one?
         if (NextID == ParamID)
         {
-            mParamStack.push_back( SParameter { Offset, NextSize, false } );
+            mParamStack.push_back( SParameter { Offset, NextSize, false, false } );
             return true;
         }
 
         // It's not a match - return to the parent parameter's first child and check all children to find a match
         if (!mParamStack.empty())
         {
+            bool ParentAbstract = mParamStack.back().Abstract;
             u32 ParentOffset = mParamStack.back().Offset;
-            mpStream->Seek(ParentOffset, SEEK_SET);
+            mpStream->Seek(ParentOffset + (ParentAbstract ? 0xA : 0x6), SEEK_SET);
-            mpStream->Seek(0x6, SEEK_CUR);
             u16 NumChildren = mpStream->ReadShort();
 
             for (u32 iChild = 0; iChild < NumChildren; iChild++)
@@ -91,7 +89,7 @@ public:
                     mpStream->Seek(ChildSize, SEEK_CUR);
                 else
                 {
-                    mParamStack.push_back( SParameter { mpStream->Tell() - 6, NextSize, false } );
+                    mParamStack.push_back( SParameter { mpStream->Tell() - 6, NextSize, false, false } );
                     return true;
                 }
             }
@@ -111,37 +109,40 @@ public:
         mParamStack.pop_back();
     }
 
-    void SerializeContainerSize(u32& rSize)
+    virtual void SerializeContainerSize(u32& rSize)
     {
         // Mostly handled by ParamBegin, we just need to return the size correctly so the container can be resized
         rSize = (u32) mpStream->PeekShort();
     }
 
-    void SerializeAbstractObjectType(u32& rType)    { rType = mpStream->ReadLong(); }
+    virtual void SerializeAbstractObjectType(u32& rType)
+    {
+        // Mark current parameter as abstract so we can account for the object type in the filestream
+        rType = mpStream->ReadLong();
+        mParamStack.back().Abstract = true;
+    }
 
-    void SerializePrimitive(bool& rValue)           { rValue = mpStream->ReadBool(); }
+    virtual void SerializePrimitive(bool& rValue)           { rValue = mpStream->ReadBool(); }
-    void SerializePrimitive(char& rValue)           { rValue = mpStream->ReadByte(); }
+    virtual void SerializePrimitive(char& rValue)           { rValue = mpStream->ReadByte(); }
-    void SerializePrimitive(s8& rValue)             { rValue = mpStream->ReadByte(); }
+    virtual void SerializePrimitive(s8& rValue)             { rValue = mpStream->ReadByte(); }
-    void SerializePrimitive(u8& rValue)             { rValue = mpStream->ReadByte(); }
+    virtual void SerializePrimitive(u8& rValue)             { rValue = mpStream->ReadByte(); }
-    void SerializePrimitive(s16& rValue)            { rValue = mpStream->ReadShort(); }
+    virtual void SerializePrimitive(s16& rValue)            { rValue = mpStream->ReadShort(); }
-    void SerializePrimitive(u16& rValue)            { rValue = mpStream->ReadShort(); }
+    virtual void SerializePrimitive(u16& rValue)            { rValue = mpStream->ReadShort(); }
-    void SerializePrimitive(s32& rValue)            { rValue = mpStream->ReadLong(); }
+    virtual void SerializePrimitive(s32& rValue)            { rValue = mpStream->ReadLong(); }
-    void SerializePrimitive(u32& rValue)            { rValue = mpStream->ReadLong(); }
+    virtual void SerializePrimitive(u32& rValue)            { rValue = mpStream->ReadLong(); }
-    void SerializePrimitive(s64& rValue)            { rValue = mpStream->ReadLongLong(); }
+    virtual void SerializePrimitive(s64& rValue)            { rValue = mpStream->ReadLongLong(); }
-    void SerializePrimitive(u64& rValue)            { rValue = mpStream->ReadLongLong(); }
+    virtual void SerializePrimitive(u64& rValue)            { rValue = mpStream->ReadLongLong(); }
-    void SerializePrimitive(float& rValue)          { rValue = mpStream->ReadFloat(); }
+    virtual void SerializePrimitive(float& rValue)          { rValue = mpStream->ReadFloat(); }
-    void SerializePrimitive(double& rValue)         { rValue = mpStream->ReadDouble(); }
+    virtual void SerializePrimitive(double& rValue)         { rValue = mpStream->ReadDouble(); }
-    void SerializePrimitive(TString& rValue)        { rValue = mpStream->ReadSizedString(); }
+    virtual void SerializePrimitive(TString& rValue)        { rValue = mpStream->ReadSizedString(); }
-    void SerializePrimitive(CAssetID& rValue)       { rValue = CAssetID(*mpStream, Game()); }
+    virtual void SerializePrimitive(TWideString& rValue)    { rValue = mpStream->ReadSizedWString(); }
+    virtual void SerializePrimitive(CFourCC& rValue)        { rValue = CFourCC(*mpStream); }
+    virtual void SerializePrimitive(CAssetID& rValue)       { rValue = CAssetID(*mpStream, Game()); }
 
-    void SerializeHexPrimitive(s8& rValue)          { rValue = mpStream->ReadByte(); }
+    virtual void SerializeHexPrimitive(u8& rValue)          { rValue = mpStream->ReadByte(); }
-    void SerializeHexPrimitive(u8& rValue)          { rValue = mpStream->ReadByte(); }
+    virtual void SerializeHexPrimitive(u16& rValue)         { rValue = mpStream->ReadShort(); }
-    void SerializeHexPrimitive(s16& rValue)         { rValue = mpStream->ReadShort(); }
+    virtual void SerializeHexPrimitive(u32& rValue)         { rValue = mpStream->ReadLong(); }
-    void SerializeHexPrimitive(u16& rValue)         { rValue = mpStream->ReadShort(); }
+    virtual void SerializeHexPrimitive(u64& rValue)         { rValue = mpStream->ReadLongLong(); }
-    void SerializeHexPrimitive(s32& rValue)         { rValue = mpStream->ReadLong(); }
-    void SerializeHexPrimitive(u32& rValue)         { rValue = mpStream->ReadLong(); }
-    void SerializeHexPrimitive(s64& rValue)         { rValue = mpStream->ReadLongLong(); }
-    void SerializeHexPrimitive(u64& rValue)         { rValue = mpStream->ReadLongLong(); }
 };
 
 #endif // CBINARYREADER

src/Common/Serialization/CBinaryWriter.h

@@ -10,6 +10,7 @@ class CBinaryWriter : public IArchive
     {
         u32 Offset;
         u16 NumSubParams;
+        bool Abstract;
     };
     std::vector<SParameter> mParamStack;
 
@@ -17,34 +18,33 @@ class CBinaryWriter : public IArchive
     bool mOwnsStream;
 
 public:
-    CBinaryWriter(const TString& rkFilename, u32 FileVersion, EGame Game = eUnknownGame)
+    CBinaryWriter(const TString& rkFilename, u16 FileVersion, EGame Game = eUnknownGame)
         : IArchive(false, true)
         , mOwnsStream(true)
     {
         mpStream = new CFileOutStream(rkFilename.ToStdString(), IOUtil::eBigEndian);
         ASSERT(mpStream->IsValid());
 
-        mFileVersion = FileVersion;
+        SetVersion(skCurrentArchiveVersion, FileVersion, Game);
-        mGame = Game;
+        GetVersionInfo().Write(*mpStream);
-
-        mpStream->WriteShort((u16) FileVersion);
-        mpStream->WriteShort((u16) skCurrentArchiveVersion);
-        GetGameID(Game).Write(*mpStream);
     }
 
-    CBinaryWriter(IOutputStream *pStream, u32 FileVersion, EGame Game = eUnknownGame)
+    CBinaryWriter(IOutputStream *pStream, u16 FileVersion, EGame Game = eUnknownGame)
         : IArchive(false, true)
         , mOwnsStream(false)
     {
         ASSERT(pStream->IsValid());
         mpStream = pStream;
+        SetVersion(skCurrentArchiveVersion, FileVersion, Game);
+    }
 
-        mFileVersion = FileVersion;
+    CBinaryWriter(IOutputStream *pStream, const CSerialVersion& rkVersion)
-        mGame = Game;
+        : IArchive(false, true)
-
+        , mOwnsStream(false)
-        mpStream->WriteShort((u16) FileVersion);
+    {
-        mpStream->WriteShort((u16) skCurrentArchiveVersion);
+        ASSERT(pStream->IsValid());
-        GetGameID(Game).Write(*mpStream);
+        mpStream = pStream;
+        SetVersion(rkVersion);
     }
 
     ~CBinaryWriter()
@@ -63,7 +63,7 @@ public:
                 mpStream->WriteShort(-1); // Sub-param count filler
         }
 
-        mParamStack.push_back( SParameter { mpStream->Tell(), 0 } );
+        mParamStack.push_back( SParameter { mpStream->Tell(), 0, false } );
 
         u32 ParamID = TString(pkName).Hash32();
         mpStream->WriteLong(ParamID);
@@ -76,47 +76,56 @@ public:
         SParameter& rParam = mParamStack.back();
         u32 StartOffset = rParam.Offset;
         u32 EndOffset = mpStream->Tell();
-        u16 ParamSize = (EndOffset - StartOffset) - 6;
+        u16 ParamSize = (u16) (EndOffset - StartOffset) - 6;
 
         mpStream->Seek(StartOffset + 4, SEEK_SET);
         mpStream->WriteShort(ParamSize);
-        if (rParam.NumSubParams > 0) mpStream->WriteShort(rParam.NumSubParams);
+
+        if (rParam.NumSubParams > 0)
+        {
+            if (rParam.Abstract) mpStream->Seek(4, SEEK_CUR);
+            mpStream->WriteShort(rParam.NumSubParams);
+        }
+
         mpStream->Seek(EndOffset, SEEK_SET);
         mParamStack.pop_back();
     }
 
-    void SerializeContainerSize(u32& rSize)
+    virtual void SerializeContainerSize(u32& rSize)
     {
         // Normally handled by ParamBegin and ParamEnd but we need to do something here to account for zero-sized containers
         if (rSize == 0)
             mpStream->WriteShort(0);
     }
 
-    void SerializeAbstractObjectType(u32& rType)    { mpStream->WriteLong(rType); }
+    virtual void SerializeAbstractObjectType(u32& rType)
+    {
+        // Mark this parameter as abstract so we can account for the object type in the filestream
+        mpStream->WriteLong(rType);
+        mParamStack.back().Abstract = true;
+    }
 
-    void SerializePrimitive(bool& rValue)           { mpStream->WriteBool(rValue); }
+    virtual void SerializePrimitive(bool& rValue)           { mpStream->WriteBool(rValue); }
-    void SerializePrimitive(char& rValue)           { mpStream->WriteByte(rValue); }
+    virtual void SerializePrimitive(char& rValue)           { mpStream->WriteByte(rValue); }
-    void SerializePrimitive(s8& rValue)             { mpStream->WriteByte(rValue); }
+    virtual void SerializePrimitive(s8& rValue)             { mpStream->WriteByte(rValue); }
-    void SerializePrimitive(u8& rValue)             { mpStream->WriteByte(rValue); }
+    virtual void SerializePrimitive(u8& rValue)             { mpStream->WriteByte(rValue); }
-    void SerializePrimitive(s16& rValue)            { mpStream->WriteShort(rValue); }
+    virtual void SerializePrimitive(s16& rValue)            { mpStream->WriteShort(rValue); }
-    void SerializePrimitive(u16& rValue)            { mpStream->WriteShort(rValue); }
+    virtual void SerializePrimitive(u16& rValue)            { mpStream->WriteShort(rValue); }
-    void SerializePrimitive(s32& rValue)            { mpStream->WriteLong(rValue); }
+    virtual void SerializePrimitive(s32& rValue)            { mpStream->WriteLong(rValue); }
-    void SerializePrimitive(u32& rValue)            { mpStream->WriteLong(rValue); }
+    virtual void SerializePrimitive(u32& rValue)            { mpStream->WriteLong(rValue); }
-    void SerializePrimitive(s64& rValue)            { mpStream->WriteLongLong(rValue); }
+    virtual void SerializePrimitive(s64& rValue)            { mpStream->WriteLongLong(rValue); }
-    void SerializePrimitive(u64& rValue)            { mpStream->WriteLongLong(rValue); }
+    virtual void SerializePrimitive(u64& rValue)            { mpStream->WriteLongLong(rValue); }
-    void SerializePrimitive(float& rValue)          { mpStream->WriteFloat(rValue); }
+    virtual void SerializePrimitive(float& rValue)          { mpStream->WriteFloat(rValue); }
-    void SerializePrimitive(double& rValue)         { mpStream->WriteDouble(rValue); }
+    virtual void SerializePrimitive(double& rValue)         { mpStream->WriteDouble(rValue); }
-    void SerializePrimitive(TString& rValue)        { mpStream->WriteSizedString(rValue.ToStdString()); }
+    virtual void SerializePrimitive(TString& rValue)        { mpStream->WriteSizedString(rValue.ToStdString()); }
-    void SerializePrimitive(CAssetID& rValue)       { rValue.Write(*mpStream); }
+    virtual void SerializePrimitive(TWideString& rValue)    { mpStream->WriteSizedWideString(rValue.ToStdString()); }
+    virtual void SerializePrimitive(CFourCC& rValue)        { rValue.Write(*mpStream); }
+    virtual void SerializePrimitive(CAssetID& rValue)       { rValue.Write(*mpStream); }
 
-    void SerializeHexPrimitive(s8& rValue)          { mpStream->WriteByte(rValue); }
+    virtual void SerializeHexPrimitive(u8& rValue)          { mpStream->WriteByte(rValue); }
-    void SerializeHexPrimitive(u8& rValue)          { mpStream->WriteByte(rValue); }
+    virtual void SerializeHexPrimitive(u16& rValue)         { mpStream->WriteShort(rValue); }
-    void SerializeHexPrimitive(s16& rValue)         { mpStream->WriteShort(rValue); }
+    virtual void SerializeHexPrimitive(u32& rValue)         { mpStream->WriteLong(rValue); }
-    void SerializeHexPrimitive(u16& rValue)         { mpStream->WriteShort(rValue); }
+    virtual void SerializeHexPrimitive(u64& rValue)         { mpStream->WriteLongLong(rValue); }
-    void SerializeHexPrimitive(s32& rValue)         { mpStream->WriteLong(rValue); }
-    void SerializeHexPrimitive(u32& rValue)         { mpStream->WriteLong(rValue); }
-    void SerializeHexPrimitive(s64& rValue)         { mpStream->WriteLongLong(rValue); }
-    void SerializeHexPrimitive(u64& rValue)         { mpStream->WriteLongLong(rValue); }
 };
 
 #endif // CBINARYWRITER

src/Common/Serialization/CSerialVersion.cpp

@@ -0,0 +1,34 @@
+#include "CSerialVersion.h"
+#include "Common/CFourCC.h"
+
+CSerialVersion::CSerialVersion()
+{
+}
+
+CSerialVersion::CSerialVersion(u16 ArchiveVer, u16 FileVer, EGame Game)
+    : mArchiveVersion(ArchiveVer)
+    , mFileVersion(FileVer)
+    , mGame(Game)
+{
+}
+
+CSerialVersion::CSerialVersion(IInputStream& rInput)
+{
+    Read(rInput);
+}
+
+void CSerialVersion::Read(IInputStream& rInput)
+{
+    mArchiveVersion = rInput.ReadShort();
+    mFileVersion = rInput.ReadShort();
+    CFourCC GameID(rInput);
+    mGame = GetGameForID(GameID);
+}
+
+void CSerialVersion::Write(IOutputStream& rOutput)
+{
+    rOutput.WriteShort(mArchiveVersion);
+    rOutput.WriteShort(mFileVersion);
+    CFourCC GameID = GetGameID(mGame);
+    GameID.Write(rOutput);
+}

src/Common/Serialization/CSerialVersion.h

@@ -0,0 +1,27 @@
+#ifndef CSERIALVERSION
+#define CSERIALVERSION
+
+#include "Common/EGame.h"
+#include "Common/types.h"
+#include <FileIO/FileIO.h>
+
+class CSerialVersion
+{
+    u16 mArchiveVersion;
+    u16 mFileVersion;
+    EGame mGame;
+
+public:
+    CSerialVersion();
+    CSerialVersion(u16 ArchiveVer, u16 FileVer, EGame Game);
+    CSerialVersion(IInputStream& rInput);
+    void Read(IInputStream& rInput);
+    void Write(IOutputStream& rOutput);
+
+    inline u16 ArchiveVersion() const   { return mArchiveVersion; }
+    inline u16 FileVersion() const      { return mFileVersion; }
+    inline EGame Game() const           { return mGame; }
+};
+
+#endif // CSERIALVERSION
+

src/Common/Serialization/CXMLReader.h

@@ -20,8 +20,8 @@ public:
         mpCurElem = mDoc.FirstChildElement();
         ASSERT(mpCurElem != nullptr);
 
-        mFileVersion = TString( mpCurElem->Attribute("FileVer") ).ToInt32(10);
+        mArchiveVersion = (u16) TString( mpCurElem->Attribute("ArchiveVer") ).ToInt32(10);
-        mArchiveVersion = TString( mpCurElem->Attribute("ArchiveVer") ).ToInt32(10);
+        mFileVersion = (u16) TString( mpCurElem->Attribute("FileVer") ).ToInt32(10);
         const char *pkGameAttr = mpCurElem->Attribute("Game");
         mGame = pkGameAttr ? GetGameForID( CFourCC(pkGameAttr) ) : eUnknownGame;
     }
@@ -107,15 +107,13 @@ public:
     virtual void SerializePrimitive(float& rValue)       { rValue = ReadParam().ToFloat(); }
     virtual void SerializePrimitive(double& rValue)      { rValue = (double) ReadParam().ToFloat(); }
     virtual void SerializePrimitive(TString& rValue)     { rValue = ReadParam(); }
+    virtual void SerializePrimitive(TWideString& rValue) { rValue = ReadParam().ToUTF16(); }
+    virtual void SerializePrimitive(CFourCC& rValue)     { rValue = CFourCC( ReadParam() ); }
     virtual void SerializePrimitive(CAssetID& rValue)    { rValue = CAssetID::FromString( ReadParam() ); }
 
-    virtual void SerializeHexPrimitive(s8& rValue)       { rValue = (s8) ReadParam().ToInt32(16); }
     virtual void SerializeHexPrimitive(u8& rValue)       { rValue = (u8) ReadParam().ToInt32(16); }
-    virtual void SerializeHexPrimitive(s16& rValue)      { rValue = (s16) ReadParam().ToInt32(16); }
     virtual void SerializeHexPrimitive(u16& rValue)      { rValue = (u16) ReadParam().ToInt32(16); }
-    virtual void SerializeHexPrimitive(s32& rValue)      { rValue = (s32) ReadParam().ToInt32(16); }
     virtual void SerializeHexPrimitive(u32& rValue)      { rValue = (u32) ReadParam().ToInt32(16); }
-    virtual void SerializeHexPrimitive(s64& rValue)      { rValue = (s64) ReadParam().ToInt32(16); }
     virtual void SerializeHexPrimitive(u64& rValue)      { rValue = (u64) ReadParam().ToInt32(16); }
 };
 

src/Common/Serialization/CXMLWriter.h

@@ -13,12 +13,11 @@ class CXMLWriter : public IArchive
     tinyxml2::XMLElement *mpCurElem;
 
 public:
-    CXMLWriter(const TString& rkFileName, const TString& rkRootName, u32 FileVersion, EGame Game = eUnknownGame)
+    CXMLWriter(const TString& rkFileName, const TString& rkRootName, u16 FileVersion, EGame Game = eUnknownGame)
         : IArchive(false, true)
         , mOutFilename(rkFileName)
     {
-        mFileVersion = FileVersion;
+        SetVersion(skCurrentArchiveVersion, FileVersion, Game);
-        mGame = Game;
 
         // Create declaration and root node
         tinyxml2::XMLDeclaration *pDecl = mDoc.NewDeclaration();
@@ -28,8 +27,8 @@ public:
         mDoc.LinkEndChild(mpCurElem);
 
         // Write version data
-        mpCurElem->SetAttribute("FileVer", (int) FileVersion);
         mpCurElem->SetAttribute("ArchiveVer", (int) skCurrentArchiveVersion);
+        mpCurElem->SetAttribute("FileVer", (int) FileVersion);
         if (Game != eUnknownGame) mpCurElem->SetAttribute("Game", *GetGameID(Game).ToString());
     }
 
@@ -82,15 +81,13 @@ public:
     virtual void SerializePrimitive(float& rValue)       { WriteParam(*TString::FromFloat(rValue)); }
     virtual void SerializePrimitive(double& rValue)      { WriteParam(*TString::FromFloat((float) rValue)); }
     virtual void SerializePrimitive(TString& rValue)     { WriteParam(*rValue); }
+    virtual void SerializePrimitive(TWideString& rValue) { WriteParam(*rValue.ToUTF8()); }
+    virtual void SerializePrimitive(CFourCC& rValue)     { WriteParam(*rValue.ToString()); }
     virtual void SerializePrimitive(CAssetID& rValue)    { WriteParam(*rValue.ToString()); }
 
-    virtual void SerializeHexPrimitive(s8& rValue)       { WriteParam(*TString::HexString((u8) rValue, 2)); }
     virtual void SerializeHexPrimitive(u8& rValue)       { WriteParam(*TString::HexString(rValue, 2)); }
-    virtual void SerializeHexPrimitive(s16& rValue)      { WriteParam(*TString::HexString((u16) rValue, 4)); }
     virtual void SerializeHexPrimitive(u16& rValue)      { WriteParam(*TString::HexString(rValue, 4)); }
-    virtual void SerializeHexPrimitive(s32& rValue)      { WriteParam(*TString::HexString((u32) rValue, 8)); }
     virtual void SerializeHexPrimitive(u32& rValue)      { WriteParam(*TString::HexString(rValue, 8)); }
-    virtual void SerializeHexPrimitive(s64& rValue)      { WriteParam(*TString::HexString((u32) rValue, 16)); }
     virtual void SerializeHexPrimitive(u64& rValue)      { WriteParam(*TString::HexString((u32) rValue, 16)); }
 };
 

src/Common/Serialization/IArchive.h

@@ -1,15 +1,19 @@
 #ifndef IARCHIVE
 #define IARCHIVE
 
+#include "CSerialVersion.h"
 #include "Common/AssertMacro.h"
 #include "Common/CAssetID.h"
+#include "Common/CFourCC.h"
 #include "Common/EGame.h"
 #include "Common/TString.h"
 #include "Common/types.h"
 
-/* This is a custom serialization implementation intended for saving editor-friendly game assets
+/* This is a custom serialization implementation intended for saving game assets out to editor-
- * out to XML. Support for other output formats can be added by implementing new subclasses of
+ * friendly formats, such as XML. The main goals of the serialization system is to simplify the
- * IArchive.
+ * code for reading and writing editor files and to be able to easily update those files without
+ * breaking compatibility with older versions. Support for new output formats can be added by
+ * implementing new subclasses of IArchive.
  *
  * To use a class with the serialization system, it must have a Serialize function implemented.
  * There are two ways this function can be defined:
@@ -22,9 +26,10 @@
  * ensure that files are easily backwards-compatible if parameters are moved or added/removed.
  *
  * Polymorphism is supported. There are two requirements for a polymorphic class to work with the
- * serialization system. First, the base class must contain a virtual Type() parameter that returns
+ * serialization system. First, the base class must contain a virtual Type() function that returns
- * an integral value (an enum or an integer). Second, there must be a factory class with a SpawnObject
+ * an integral value (an enum or an integer), as well as a virtual Serialize(IArchive&) function.
- * method that takes the same Type value and returns an object of the specified class.
+ * Second, there must be a factory object with a SpawnObject(u32) method that takes the same Type value
+ * and returns an object of the correct class.
  *
  * Containers are also supported. Containers require a different macro that allows you to specify the
  * name of the elements in the container. The currently-supported containers are std::vector, std::list,
@@ -166,8 +171,8 @@ public:
 class IArchive
 {
 protected:
-    s32 mFileVersion;
+    u16 mArchiveVersion;
-    s32 mArchiveVersion;
+    u16 mFileVersion;
     EGame mGame;
     bool mIsReader;
     bool mIsWriter;
@@ -383,23 +388,40 @@ public:
     virtual void SerializePrimitive(float& rValue) = 0;
     virtual void SerializePrimitive(double& rValue) = 0;
     virtual void SerializePrimitive(TString& rValue) = 0;
+    virtual void SerializePrimitive(TWideString& rValue) = 0;
+    virtual void SerializePrimitive(CFourCC& rValue) = 0;
     virtual void SerializePrimitive(CAssetID& rValue) = 0;
 
-    virtual void SerializeHexPrimitive(s8& rValue) = 0;
     virtual void SerializeHexPrimitive(u8& rValue) = 0;
-    virtual void SerializeHexPrimitive(s16& rValue) = 0;
     virtual void SerializeHexPrimitive(u16& rValue) = 0;
-    virtual void SerializeHexPrimitive(s32& rValue) = 0;
     virtual void SerializeHexPrimitive(u32& rValue) = 0;
-    virtual void SerializeHexPrimitive(s64& rValue) = 0;
     virtual void SerializeHexPrimitive(u64& rValue) = 0;
 
     // Accessors
-    inline u32 FileVersion() const      { return mFileVersion; }
+    inline u16 ArchiveVersion() const   { return mArchiveVersion; }
-    inline u32 ArchiveVersion() const   { return mArchiveVersion; }
+    inline u16 FileVersion() const      { return mFileVersion; }
     inline EGame Game() const           { return mGame; }
     inline bool IsReader() const        { return mIsReader; }
     inline bool IsWriter() const        { return mIsWriter; }
+
+    inline void SetVersion(u16 ArchiveVersion, u16 FileVersion, EGame Game)
+    {
+        mArchiveVersion = ArchiveVersion;
+        mFileVersion = FileVersion;
+        mGame = Game;
+    }
+
+    inline void SetVersion(const CSerialVersion& rkVersion)
+    {
+        mArchiveVersion = rkVersion.ArchiveVersion();
+        mFileVersion = rkVersion.FileVersion();
+        mGame = rkVersion.Game();
+    }
+
+    inline CSerialVersion GetVersionInfo() const
+    {
+        return CSerialVersion(mArchiveVersion, mFileVersion, mGame);
+    }
 };
 
 // Container serialize methods

src/Common/Serialization/XML.h

@@ -0,0 +1,3 @@
+#include "IArchive.h"
+#include "CXMLReader.h"
+#include "CXMLWriter.h"

src/Core/GameProject/CDependencyTree.cpp

@@ -3,6 +3,8 @@
 #include "Core/Resource/Script/CScriptLayer.h"
 #include "Core/Resource/Script/CScriptObject.h"
 
+CDependencyNodeFactory gDependencyNodeFactory;
+
 // ************ IDependencyNode ************
 IDependencyNode::~IDependencyNode()
 {
@@ -27,28 +29,10 @@ EDependencyNodeType CDependencyTree::Type() const
     return eDNT_DependencyTree;
 }
 
-void CDependencyTree::Read(IInputStream& rFile, EIDLength IDLength)
+void CDependencyTree::Serialize(IArchive& rArc)
 {
-    mRootID = CAssetID(rFile, IDLength);
+    rArc << SERIAL("RootID", mRootID)
-
+         << SERIAL_ABSTRACT_CONTAINER("Children", mChildren, "Child", &gDependencyNodeFactory);
-    u32 NumDepends = rFile.ReadLong();
-    mChildren.reserve(NumDepends);
-
-    for (u32 iDep = 0; iDep < NumDepends; iDep++)
-    {
-        CResourceDependency *pDepend = new CResourceDependency;
-        pDepend->Read(rFile, IDLength);
-        mChildren.push_back(pDepend);
-    }
-}
-
-void CDependencyTree::Write(IOutputStream& rFile) const
-{
-    mRootID.Write(rFile);
-    rFile.WriteLong( mChildren.size() );
-
-    for (u32 iDep = 0; iDep < mChildren.size(); iDep++)
-        mChildren[iDep]->Write(rFile);
 }
 
 void CDependencyTree::AddDependency(CResource *pRes, bool AvoidDuplicates /*= true*/)
@@ -70,14 +54,9 @@ EDependencyNodeType CResourceDependency::Type() const
     return eDNT_ResourceDependency;
 }
 
-void CResourceDependency::Read(IInputStream& rFile, EIDLength IDLength)
+void CResourceDependency::Serialize(IArchive& rArc)
 {
-    mID = CAssetID(rFile, IDLength);
+    rArc << SERIAL("ID", mID);
-}
-
-void CResourceDependency::Write(IOutputStream& rFile) const
-{
-    mID.Write(rFile);
 }
 
 bool CResourceDependency::HasDependency(const CAssetID& rkID) const
@@ -91,16 +70,10 @@ EDependencyNodeType CPropertyDependency::Type() const
     return eDNT_ScriptProperty;
 }
 
-void CPropertyDependency::Read(IInputStream& rFile, EIDLength IDLength)
+void CPropertyDependency::Serialize(IArchive& rArc)
 {
-    mIDString = rFile.ReadString();
+    rArc << SERIAL("PropertyID", mIDString);
-    CResourceDependency::Read(rFile, IDLength);
+    CResourceDependency::Serialize(rArc);
-}
-
-void CPropertyDependency::Write(IOutputStream& rFile) const
-{
-    rFile.WriteString(mIDString.ToStdString());
-    CResourceDependency::Write(rFile);
 }
 
 // ************ CCharacterPropertyDependency ************
@@ -109,16 +82,10 @@ EDependencyNodeType CCharPropertyDependency::Type() const
     return eDNT_CharacterProperty;
 }
 
-void CCharPropertyDependency::Read(IInputStream& rFile, EIDLength IDLength)
+void CCharPropertyDependency::Serialize(IArchive& rArc)
 {
-    CPropertyDependency::Read(rFile, IDLength);
+    CPropertyDependency::Serialize(rArc);
-    mUsedChar = rFile.ReadLong();
+    rArc << SERIAL("CharIndex", mUsedChar);
-}
-
-void CCharPropertyDependency::Write(IOutputStream& rFile) const
-{
-    CPropertyDependency::Write(rFile);
-    rFile.WriteLong(mUsedChar);
 }
 
 // ************ CScriptInstanceDependency ************
@@ -127,32 +94,10 @@ EDependencyNodeType CScriptInstanceDependency::Type() const
     return eDNT_ScriptInstance;
 }
 
-void CScriptInstanceDependency::Read(IInputStream& rFile, EIDLength IDLength)
+void CScriptInstanceDependency::Serialize(IArchive& rArc)
 {
-    mObjectType = rFile.ReadLong();
+    rArc << SERIAL("ObjectType", mObjectType)
-    u32 NumProperties = rFile.ReadLong();
+         << SERIAL_ABSTRACT_CONTAINER("Properties", mChildren, "Property", &gDependencyNodeFactory);
-    mChildren.reserve(NumProperties);
-
-    for (u32 iProp = 0; iProp < NumProperties; iProp++)
-    {
-        bool IsCharacter = rFile.ReadBool();
-        CPropertyDependency *pProp = (IsCharacter ? new CCharPropertyDependency() : new CPropertyDependency());
-        pProp->Read(rFile, IDLength);
-        mChildren.push_back(pProp);
-    }
-}
-
-void CScriptInstanceDependency::Write(IOutputStream& rFile) const
-{
-    rFile.WriteLong(mObjectType);
-    rFile.WriteLong(mChildren.size());
-
-    for (u32 iProp = 0; iProp < mChildren.size(); iProp++)
-    {
-        CPropertyDependency *pProp = static_cast<CPropertyDependency*>(mChildren[iProp]);
-        rFile.WriteBool( pProp->Type() == eDNT_CharacterProperty );
-        pProp->Write(rFile);
-    }
 }
 
 // Static
@@ -216,23 +161,10 @@ EDependencyNodeType CAnimSetDependencyTree::Type() const
     return eDNT_AnimSet;
 }
 
-void CAnimSetDependencyTree::Read(IInputStream& rFile, EIDLength IDLength)
+void CAnimSetDependencyTree::Serialize(IArchive& rArc)
 {
-    CDependencyTree::Read(rFile, IDLength);
+    CDependencyTree::Serialize(rArc);
-    u32 NumChars = rFile.ReadLong();
+    rArc << SERIAL_CONTAINER("CharacterOffsets", mCharacterOffsets, "Offset");
-    mCharacterOffsets.reserve(NumChars);
-
-    for (u32 iChar = 0; iChar < NumChars; iChar++)
-        mCharacterOffsets.push_back( rFile.ReadLong() );
-}
-
-void CAnimSetDependencyTree::Write(IOutputStream& rFile) const
-{
-    CDependencyTree::Write(rFile);
-    rFile.WriteLong(mCharacterOffsets.size());
-
-    for (u32 iChar = 0; iChar < mCharacterOffsets.size(); iChar++)
-        rFile.WriteLong( mCharacterOffsets[iChar] );
 }
 
 void CAnimSetDependencyTree::AddCharacter(const SSetCharacter *pkChar, const std::set<CAssetID>& rkBaseUsedSet)
@@ -282,57 +214,10 @@ EDependencyNodeType CAreaDependencyTree::Type() const
     return eDNT_Area;
 }
 
-void CAreaDependencyTree::Read(IInputStream& rFile, EIDLength IDLength)
+void CAreaDependencyTree::Serialize(IArchive& rArc)
 {
-    mRootID = CAssetID(rFile, IDLength);
+    CDependencyTree::Serialize(rArc);
-
+    rArc << SERIAL_CONTAINER("LayerOffsets", mLayerOffsets, "Offset");
-    // Base dependency list contains non-script dependencies (world geometry textures + PATH/PTLA/EGMC)
-    u32 NumBaseDependencies = rFile.ReadLong();
-    mChildren.reserve(NumBaseDependencies);
-
-    for (u32 iDep = 0; iDep < NumBaseDependencies; iDep++)
-    {
-        CResourceDependency *pDep = new CResourceDependency;
-        pDep->Read(rFile, IDLength);
-        mChildren.push_back(pDep);
-    }
-
-    u32 NumScriptInstances = rFile.ReadLong();
-    mChildren.reserve(mChildren.size() + NumScriptInstances);
-
-    for (u32 iInst = 0; iInst < NumScriptInstances; iInst++)
-    {
-        CScriptInstanceDependency *pInst = new CScriptInstanceDependency;
-        pInst->Read(rFile, IDLength);
-        mChildren.push_back(pInst);
-    }
-
-    u32 NumLayers = rFile.ReadLong();
-    mLayerOffsets.reserve(NumLayers);
-
-    for (u32 iLyr = 0; iLyr < NumLayers; iLyr++)
-        mLayerOffsets.push_back( rFile.ReadLong() );
-}
-
-void CAreaDependencyTree::Write(IOutputStream& rFile) const
-{
-    mRootID.Write(rFile);
-
-    u32 NumBaseDependencies = (mLayerOffsets.empty() ? mChildren.size() : mLayerOffsets.front());
-    rFile.WriteLong(NumBaseDependencies);
-
-    for (u32 iDep = 0; iDep < NumBaseDependencies; iDep++)
-        mChildren[iDep]->Write(rFile);
-
-    rFile.WriteLong(mChildren.size() - NumBaseDependencies);
-
-    for (u32 iDep = NumBaseDependencies; iDep < mChildren.size(); iDep++)
-        mChildren[iDep]->Write(rFile);
-
-    rFile.WriteLong(mLayerOffsets.size());
-
-    for (u32 iLyr = 0; iLyr < mLayerOffsets.size(); iLyr++)
-        rFile.WriteLong(mLayerOffsets[iLyr]);
 }
 
 void CAreaDependencyTree::AddScriptLayer(CScriptLayer *pLayer)

src/Core/GameProject/CDependencyTree.h

@@ -14,13 +14,13 @@ struct SSetCharacter;
 // Group of node classes forming a tree of cached resource dependencies.
 enum EDependencyNodeType
 {
-    eDNT_DependencyTree,
+    eDNT_DependencyTree     = FOURCC_CONSTEXPR('T', 'R', 'E', 'E'),
-    eDNT_ResourceDependency,
+    eDNT_ResourceDependency = FOURCC_CONSTEXPR('R', 'S', 'D', 'P'),
-    eDNT_ScriptInstance,
+    eDNT_ScriptInstance     = FOURCC_CONSTEXPR('S', 'C', 'I', 'N'),
-    eDNT_ScriptProperty,
+    eDNT_ScriptProperty     = FOURCC_CONSTEXPR('S', 'C', 'P', 'R'),
-    eDNT_CharacterProperty,
+    eDNT_CharacterProperty  = FOURCC_CONSTEXPR('C', 'R', 'P', 'R'),
-    eDNT_AnimSet,
+    eDNT_AnimSet            = FOURCC_CONSTEXPR('A', 'N', 'C', 'S'),
-    eDNT_Area,
+    eDNT_Area               = FOURCC_CONSTEXPR('A', 'R', 'E', 'A'),
 };
 
 // Base class providing an interface for a basic dependency node.
@@ -32,8 +32,7 @@ protected:
 public:
     virtual ~IDependencyNode();
     virtual EDependencyNodeType Type() const = 0;
-    virtual void Read(IInputStream& rFile, EIDLength IDLength) = 0;
+    virtual void Serialize(IArchive& rArc) = 0;
-    virtual void Write(IOutputStream& rFile) const = 0;
     virtual bool HasDependency(const CAssetID& rkID) const;
 
     // Accessors
@@ -48,11 +47,11 @@ protected:
     CAssetID mRootID;
 
 public:
+    CDependencyTree() {}
     CDependencyTree(const CAssetID& rkID) : mRootID(rkID) {}
 
     virtual EDependencyNodeType Type() const;
-    virtual void Read(IInputStream& rFile, EIDLength IDLength);
+    virtual void Serialize(IArchive& rArc);
-    virtual void Write(IOutputStream& rFile) const;
 
     void AddDependency(const CAssetID& rkID, bool AvoidDuplicates = true);
     void AddDependency(CResource *pRes, bool AvoidDuplicates = true);
@@ -73,8 +72,7 @@ public:
     CResourceDependency(const CAssetID& rkID) : mID(rkID) {}
 
     virtual EDependencyNodeType Type() const;
-    virtual void Read(IInputStream& rFile, EIDLength IDLength);
+    virtual void Serialize(IArchive& rArc);
-    virtual void Write(IOutputStream& rFile) const;
     virtual bool HasDependency(const CAssetID& rkID) const;
 
     // Accessors
@@ -98,8 +96,7 @@ public:
     {}
 
     virtual EDependencyNodeType Type() const;
-    virtual void Read(IInputStream& rFile, EIDLength IDLength);
+    virtual void Serialize(IArchive& rArc);
-    virtual void Write(IOutputStream& rFile) const;
 
     // Accessors
     inline TString PropertyID() const   { return mIDString; }
@@ -123,8 +120,7 @@ public:
     {}
 
     virtual EDependencyNodeType Type() const;
-    virtual void Read(IInputStream& rFile, EIDLength IDLength);
+    virtual void Serialize(IArchive& rArc);
-    virtual void Write(IOutputStream& rFile) const;
 
     // Accessors
     inline u32 UsedChar() const                 { return mUsedChar; }
@@ -138,8 +134,7 @@ protected:
 
 public:
     virtual EDependencyNodeType Type() const;
-    virtual void Read(IInputStream& rFile, EIDLength IDLength);
+    virtual void Serialize(IArchive& rArc);
-    virtual void Write(IOutputStream& rFile) const;
 
     // Accessors
     inline u32 ObjectType() const       { return mObjectType; }
@@ -157,10 +152,10 @@ protected:
     std::vector<u32> mCharacterOffsets;
 
 public:
+    CAnimSetDependencyTree() : CDependencyTree() {}
     CAnimSetDependencyTree(const CAssetID& rkID) : CDependencyTree(rkID) {}
     virtual EDependencyNodeType Type() const;
-    virtual void Read(IInputStream& rFile, EIDLength IDLength);
+    virtual void Serialize(IArchive& rArc);
-    virtual void Write(IOutputStream& rFile) const;
 
     void AddCharacter(const SSetCharacter *pkChar, const std::set<CAssetID>& rkBaseUsedSet);
     void AddCharDependency(const CAssetID& rkID, std::set<CAssetID>& rUsedSet);
@@ -178,11 +173,11 @@ protected:
     std::vector<u32> mLayerOffsets;
 
 public:
+    CAreaDependencyTree() : CDependencyTree() {}
     CAreaDependencyTree(const CAssetID& rkID) : CDependencyTree(rkID) {}
 
     virtual EDependencyNodeType Type() const;
-    virtual void Read(IInputStream& rFile, EIDLength IDLength);
+    virtual void Serialize(IArchive& rArc);
-    virtual void Write(IOutputStream& rFile) const;
 
     void AddScriptLayer(CScriptLayer *pLayer);
     void GetModuleDependencies(EGame Game, std::vector<TString>& rModuleDepsOut, std::vector<u32>& rModuleLayerOffsetsOut) const;
@@ -192,5 +187,26 @@ public:
     inline u32 ScriptLayerOffset(u32 LayerIdx) const    { return mLayerOffsets[LayerIdx]; }
 };
 
+// Dependency node factory for serialization
+class CDependencyNodeFactory
+{
+public:
+    IDependencyNode* SpawnObject(u32 NodeID)
+    {
+        switch (NodeID)
+        {
+        case eDNT_DependencyTree:       return new CDependencyTree;
+        case eDNT_ResourceDependency:   return new CResourceDependency;
+        case eDNT_ScriptInstance:       return new CScriptInstanceDependency;
+        case eDNT_ScriptProperty:       return new CPropertyDependency;
+        case eDNT_CharacterProperty:    return new CCharPropertyDependency;
+        case eDNT_AnimSet:              return new CAnimSetDependencyTree;
+        case eDNT_Area:                 return new CAreaDependencyTree;
+        default:                        return nullptr;
+        }
+    }
+};
+extern CDependencyNodeFactory gDependencyNodeFactory;
+
 #endif // CDEPENDENCYTREE
 

src/Core/GameProject/CGameExporter.cpp

@@ -526,7 +526,7 @@ void CGameExporter::ExportCookedResources()
     {
         SCOPED_TIMER(SaveResourceDatabase);
 #if EXPORT_COOKED
-        mStore.SaveResourceDatabase(mpProject->ResourceDBPath(false).ToUTF8());
+        mStore.SaveResourceDatabase();
 #endif
         mpProject->Save();
     }
@@ -537,6 +537,9 @@ void CGameExporter::ExportCookedResources()
         // some resources will fail to load if their dependencies don't exist
         SCOPED_TIMER(SaveRawResources);
 
+        // todo: we're wasting a ton of time loading the same resources over and over because most resources automatically
+        // load all their dependencies and then we just clear it out from memory even though we'll need it again later. we
+        // should really be doing this by dependency order instead of by ID order.
         for (CResourceIterator It(&mStore); It; ++It)
         {
             if (!It->IsTransient())
@@ -556,11 +559,16 @@ void CGameExporter::ExportCookedResources()
                     }
                 }
 
-                // Save raw resource + cache data
+                // Save raw resource + generate dependencies
-                It->Save();
+                It->Save(true);
             }
         }
     }
+    {
+        // All resources should have dependencies generated, so save the cache file
+        SCOPED_TIMER(SaveResourceCacheData);
+        mStore.SaveCacheFile();
+    }
 }
 
 void CGameExporter::ExportResource(SResourceInstance& rRes)

src/Core/GameProject/CGameProject.cpp

@@ -1,8 +1,7 @@
 #include "CGameProject.h"
 #include "Core/Resource/Script/CMasterTemplate.h"
-#include <tinyxml2.h>
+#include <Common/Serialization/XML.h>
 
-using namespace tinyxml2;
 CGameProject *CGameProject::mspActiveProject = nullptr;
 
 CGameProject::~CGameProject()
@@ -16,107 +15,56 @@ CGameProject::~CGameProject()
 
 bool CGameProject::Load(const TWideString& rkPath)
 {
-    TString ProjPath = rkPath.ToUTF8();
-    XMLDocument Doc;
-    Doc.LoadFile(*ProjPath);
-
-    if (Doc.Error())
-    {
-        Log::Error("Unable to open game project at " + ProjPath);
-        return false;
-    }
-
-    XMLElement *pRoot = Doc.FirstChildElement("GameProject");
-    //EProjectVersion Version = (EProjectVersion) TString(pRoot->Attribute("Version")).ToInt32(10);
-
-    // Verify all elements are present
-    XMLElement *pProjName = pRoot->FirstChildElement("Name");
-    XMLElement *pGame = pRoot->FirstChildElement("Game");
-    XMLElement *pResDB = pRoot->FirstChildElement("ResourceDB");
-    XMLElement *pPackages = pRoot->FirstChildElement("Packages");
-
-    if (!pProjName || !pGame || !pResDB || !pPackages)
-    {
-        TString MissingElem = pProjName ? (pGame ? (pResDB ? "Packages" : "ResourceDB") : "Game") : "Name";
-        Log::Error("Unable to load game project at " + ProjPath + "; " + MissingElem + " element is missing");
-        return false;
-    }
-
-    mProjectName = pProjName->GetText();
-    mGame = CMasterTemplate::FindGameForName( pGame->GetText() );
-    mResourceDBPath = pResDB->GetText();
-
     mProjectRoot = rkPath.GetFileDirectory();
     mProjectRoot.Replace(L"/", L"\\");
 
-    // Load packages
+    TString ProjPath = rkPath.ToUTF8();
-    XMLElement *pPkgElem = pPackages->FirstChildElement("Package");
+    CXMLReader Reader(ProjPath);
-
+    Serialize(Reader);
-    while (pPkgElem)
-    {
-        TString Path = pPkgElem->Attribute("Path");
-
-        if (Path.IsEmpty())
-            Log::Error("Failed to load package in game project " + ProjPath + "; Path attribute is missing or empty");
-
-        else
-        {
-            CPackage *pPackage = new CPackage(this, Path.GetFileName(false), TString(Path.GetFileDirectory()).ToUTF16());
-            pPackage->Load();
-            mPackages.push_back(pPackage);
-        }
-
-        pPkgElem = pPkgElem->NextSiblingElement("Package");
-    }
-
-    // All loaded!
     return true;
 }
 
 void CGameProject::Save()
 {
-    XMLDocument Doc;
+    TString ProjPath = ProjectPath().ToUTF8();
+    CXMLWriter Writer(ProjPath, "GameProject", eVer_Current, mGame);
+    Serialize(Writer);
+}
 
-    XMLDeclaration *pDecl = Doc.NewDeclaration();
+void CGameProject::Serialize(IArchive& rArc)
-    Doc.LinkEndChild(pDecl);
+{
+    rArc << SERIAL("Name", mProjectName)
+         << SERIAL("Game", mGame)
+         << SERIAL("ResourceDB", mResourceDBPath);
 
-    XMLElement *pRoot = Doc.NewElement("GameProject");
+    // Packages
-    pRoot->SetAttribute("Version", eVer_Current);
+    std::vector<TString> PackageList;
-    Doc.LinkEndChild(pRoot);
 
-    XMLElement *pProjName = Doc.NewElement("Name");
+    if (!rArc.IsReader())
-    pProjName->SetText(*mProjectName);
-    pRoot->LinkEndChild(pProjName);
-
-    XMLElement *pGame = Doc.NewElement("Game");
-    pGame->SetText(*CMasterTemplate::FindGameName(mGame));
-    pRoot->LinkEndChild(pGame);
-
-    XMLElement *pResDB = Doc.NewElement("ResourceDB");
-    pResDB->SetText(*mResourceDBPath.ToUTF8());
-    pRoot->LinkEndChild(pResDB);
-
-    XMLElement *pPackages = Doc.NewElement("Packages");
-    pRoot->LinkEndChild(pPackages);
-
-    for (u32 iPkg = 0; iPkg < mPackages.size(); iPkg++)
     {
-        CPackage *pPackage = mPackages[iPkg];
+        for (u32 iPkg = 0; iPkg < mPackages.size(); iPkg++)
-        TWideString FullDefPath = pPackage->DefinitionPath(false);
+            PackageList.push_back( mPackages[iPkg]->DefinitionPath(true).ToUTF8() );
-        TWideString RelDefPath = FileUtil::MakeRelative(FullDefPath.GetFileDirectory(), PackagesDir(false));
-        TString DefPath = TWideString(RelDefPath + FullDefPath.GetFileName()).ToUTF8();
-
-        XMLElement *pPakElem = Doc.NewElement("Package");
-        pPakElem->SetAttribute("Path", *DefPath);
-        pPackages->LinkEndChild(pPakElem);
     }
 
-    // Save Project
+    rArc << SERIAL_CONTAINER("Packages", PackageList, "Package");
-    TString ProjPath = ProjectPath().ToUTF8();
-    XMLError Result = Doc.SaveFile(*ProjPath);
 
-    if (Result != XML_SUCCESS)
+    if (rArc.IsReader())
-        Log::Error("Failed to save game project at: " + ProjPath);
+    {
+        for (u32 iPkg = 0; iPkg < mPackages.size(); iPkg++)
+            delete mPackages[iPkg];
+        mPackages.clear();
+
+        for (u32 iPkg = 0; iPkg < PackageList.size(); iPkg++)
+        {
+            const TWideString& rkPackagePath = PackageList[iPkg];
+            TString PackageName = TWideString(rkPackagePath.GetFileName(false)).ToUTF8();
+            TWideString PackageDir = rkPackagePath.GetFileDirectory();
+
+            CPackage *pPackage = new CPackage(this, PackageName, PackageDir);
+            pPackage->Load();
+            mPackages.push_back(pPackage);
+        }
+    }
 }
 
 void CGameProject::SetActive()

src/Core/GameProject/CGameProject.h

@@ -46,6 +46,7 @@ public:
 
     bool Load(const TWideString& rkPath);
     void Save();
+    void Serialize(IArchive& rArc);
     void SetActive();
     void GetWorldList(std::list<CAssetID>& rOut) const;
 
@@ -58,6 +59,7 @@ public:
     inline TWideString CookedDir(bool Relative) const           { return Relative ? L"Cooked\\" : mProjectRoot + L"Cooked\\"; }
     inline TWideString PackagesDir(bool Relative) const         { return Relative ? L"Packages\\" : mProjectRoot + L"Packages\\"; }
     inline TWideString ProjectPath() const                      { return mProjectRoot + FileUtil::SanitizeName(mProjectName.ToUTF16(), false) + L".prj"; }
+    inline TWideString ResourceCachePath(bool Relative) const   { return ResourceDBPath(Relative).GetFileDirectory() + L"ResourceCacheData.rcd"; }
 
     // Accessors
     inline void SetGame(EGame Game)                     { mGame = Game; }

src/Core/GameProject/CPackage.cpp

@@ -6,58 +6,15 @@
 #include <Common/AssertMacro.h>
 #include <Common/CompressionUtil.h>
 #include <Common/FileUtil.h>
-#include <tinyxml2.h>
+#include <Common/Serialization/XML.h>
 
 using namespace tinyxml2;
 
 void CPackage::Load()
 {
     TWideString DefPath = DefinitionPath(false);
-
+    CXMLReader Reader(DefPath.ToUTF8());
-    XMLDocument Doc;
+    Serialize(Reader);
-    Doc.LoadFile(*DefPath.ToUTF8());
-
-    if (Doc.Error())
-    {
-        Log::Error("Couldn't open pak definition at path: " + DefPath.ToUTF8());
-        return;
-    }
-
-    XMLElement *pRoot = Doc.FirstChildElement("PackageDefinition");
-    //EPackageDefinitionVersion Version = (EPackageDefinitionVersion) TString(pRoot->Attribute("Version")).ToInt32(10);
-
-    XMLElement *pColElem = pRoot->FirstChildElement("ResourceCollection");
-
-    while (pColElem)
-    {
-        CResourceCollection *pCollection = AddCollection( pColElem->Attribute("Name") );
-        XMLElement *pResElem = pColElem->FirstChildElement("NamedResource");
-
-        while (pResElem)
-        {
-            XMLElement *pNameElem = pResElem->FirstChildElement("Name");
-            XMLElement *pIDElem = pResElem->FirstChildElement("ID");
-            XMLElement *pTypeElem = pResElem->FirstChildElement("Type");
-
-            if (!pIDElem || !pNameElem || !pTypeElem)
-            {
-                TString ElemName = (pNameElem ? (pIDElem ? "Type" : "ID") : "Name");
-                Log::Error("Can't add named resource from pak definition at " + DefPath.ToUTF8() + "; " + ElemName + " element missing");
-            }
-
-            else
-            {
-                CAssetID ID = CAssetID::FromString(pIDElem->GetText());
-                TString Name = pNameElem->GetText();
-                CFourCC Type = CFourCC(pTypeElem->GetText());
-                pCollection->AddResource(Name, ID, Type);
-            }
-
-            pResElem = pResElem->NextSiblingElement("NamedResource");
-        }
-
-        pColElem = pColElem->NextSiblingElement("ResourceCollection");
-    }
 }
 
 void CPackage::Save()
@@ -65,49 +22,13 @@ void CPackage::Save()
     TWideString DefPath = DefinitionPath(false);
     FileUtil::CreateDirectory(DefPath.GetFileDirectory());
 
-    // Write XML
+    CXMLWriter Writer(DefPath.ToUTF8(), "PackageDefinition", 0, mpProject ? mpProject->Game() : eUnknownGame);
-    XMLDocument Doc;
+    Serialize(Writer);
+}
 
-    XMLDeclaration *pDecl = Doc.NewDeclaration();
+void CPackage::Serialize(IArchive& rArc)
-    Doc.LinkEndChild(pDecl);
+{
-
+    rArc << SERIAL_CONTAINER("Collections", mCollections, "ResourceCollection");
-    XMLElement *pRoot = Doc.NewElement("PackageDefinition");
-    pRoot->SetAttribute("Version", eVer_Current);
-    Doc.LinkEndChild(pRoot);
-
-    for (u32 iCol = 0; iCol < mCollections.size(); iCol++)
-    {
-        CResourceCollection *pCollection = mCollections[iCol];
-
-        XMLElement *pColElem = Doc.NewElement("ResourceCollection");
-        pColElem->SetAttribute("Name", *pCollection->Name());
-        pRoot->LinkEndChild(pColElem);
-
-        for (u32 iRes = 0; iRes < pCollection->NumResources(); iRes++)
-        {
-            const SNamedResource& rkRes = pCollection->ResourceByIndex(iRes);
-
-            XMLElement *pResElem = Doc.NewElement("NamedResource");
-            pColElem->LinkEndChild(pResElem);
-
-            XMLElement *pName = Doc.NewElement("Name");
-            pName->SetText(*rkRes.Name);
-            pResElem->LinkEndChild(pName);
-
-            XMLElement *pID = Doc.NewElement("ID");
-            pID->SetText(*rkRes.ID.ToString());
-            pResElem->LinkEndChild(pID);
-
-            XMLElement *pType = Doc.NewElement("Type");
-            pType->SetText(*rkRes.Type.ToString());
-            pResElem->LinkEndChild(pType);
-        }
-    }
-
-    XMLError Error = Doc.SaveFile(*DefPath.ToUTF8());
-
-    if (Error != XML_SUCCESS)
-        Log::Error("Failed to save pak definition at path: " + DefPath.ToUTF8());
 }
 
 void CPackage::Cook()
@@ -355,12 +276,14 @@ void CPackage::CompareOriginalAssetList(const std::list<CAssetID>& rkNewList)
 
 TWideString CPackage::DefinitionPath(bool Relative) const
 {
-    return mpProject->PackagesDir(Relative) + mPakPath + mPakName.ToUTF16() + L".pkd";
+    TWideString RelPath = mPakPath + mPakName.ToUTF16() + L".pkd";
+    return Relative ? RelPath : mpProject->PackagesDir(false) + RelPath;
 }
 
 TWideString CPackage::CookedPackagePath(bool Relative) const
 {
-    return mpProject->DiscDir(Relative) + mPakPath + mPakName.ToUTF16() + L".pak";
+    TWideString RelPath = mPakPath + mPakName.ToUTF16() + L".pak";
+    return Relative ? RelPath : mpProject->DiscDir(false) + RelPath;
 }
 
 CResourceCollection* CPackage::AddCollection(const TString& rkName)

src/Core/GameProject/CPackage.h

@@ -4,6 +4,7 @@
 #include <Common/CAssetID.h>
 #include <Common/CFourCC.h>
 #include <Common/TString.h>
+#include <Common/Serialization/IArchive.h>
 
 class CGameProject;
 
@@ -12,6 +13,11 @@ struct SNamedResource
     TString Name;
     CAssetID ID;
     CFourCC Type;
+
+    void Serialize(IArchive& rArc)
+    {
+        rArc << SERIAL_AUTO(Name) << SERIAL_AUTO(ID) << SERIAL_AUTO(Type);
+    }
 };
 
 class CResourceCollection
@@ -23,6 +29,11 @@ public:
     CResourceCollection() : mName("UNNAMED") {}
     CResourceCollection(const TString& rkName) : mName(rkName) {}
 
+    void Serialize(IArchive& rArc)
+    {
+        rArc << SERIAL("Name", mName) << SERIAL_CONTAINER("Resources", mNamedResources, "Resource");
+    }
+
     inline TString Name() const                                 { return mName; }
     inline u32 NumResources() const                             { return mNamedResources.size(); }
     inline const SNamedResource& ResourceByIndex(u32 Idx) const { return mNamedResources[Idx]; }
@@ -59,6 +70,7 @@ public:
 
     void Load();
     void Save();
+    void Serialize(IArchive& rArc);
 
     void Cook();
     void CompareOriginalAssetList(const std::list<CAssetID>& rkNewList);

src/Core/GameProject/CResourceEntry.cpp

@@ -34,87 +34,18 @@ CResourceEntry::~CResourceEntry()
     if (mpDependencies) delete mpDependencies;
 }
 
-bool CResourceEntry::LoadCacheData()
+void CResourceEntry::SerializeCacheData(IArchive& rArc)
 {
     ASSERT(!IsTransient());
 
-    TWideString Path = CacheDataPath(false);
+    u32 Flags = mFlags & eREF_SavedFlags;
-    CFileInStream File(Path.ToUTF8().ToStdString(), IOUtil::eLittleEndian);
+    rArc << SERIAL_AUTO(Flags);
+    if (rArc.IsReader()) mFlags = Flags & eREF_SavedFlags;
 
-    if (!File.IsValid())
+    // Note: If the dependency tree format is changed this should be adjusted so that
-    {
+    // we regenerate the dependencies from scratch instead of reading the tree if the
-        Log::Error("Unable to load cache data " + Path.ToUTF8() + "; couldn't open file");
+    // file version number is too low
-        return false;
+    rArc << SERIAL_ABSTRACT("Dependencies", mpDependencies, &gDependencyNodeFactory);
-    }
-
-    // Header
-    TString Magic = File.ReadString(4);
-    ASSERT(Magic == "CACH");
-    File.Seek(0x4, SEEK_CUR); // Skip Version
-    mFlags = File.ReadLong() & eREF_SavedFlags;
-
-    // Dependency Tree
-    u32 DepsTreeSize = File.ReadLong();
-    u32 DepsTreeStart = File.Tell();
-
-    if (mpDependencies)
-    {
-        delete mpDependencies;
-        mpDependencies = nullptr;
-    }
-
-    if (DepsTreeSize > 0)
-    {
-        if (mType == eArea)         mpDependencies = new CAreaDependencyTree(mID);
-        else if (mType == eAnimSet) mpDependencies = new CAnimSetDependencyTree(mID);
-        else                        mpDependencies = new CDependencyTree(mID);
-
-        mpDependencies->Read(File, Game() <= eEchoes ? e32Bit : e64Bit);
-        ASSERT(File.Tell() - DepsTreeStart == DepsTreeSize);
-    }
-
-    return true;
-}
-
-bool CResourceEntry::SaveCacheData()
-{
-    ASSERT(!IsTransient());
-
-    TWideString Path = CacheDataPath(false);
-    TWideString Dir = Path.GetFileDirectory();
-    FileUtil::CreateDirectory(Dir);
-    CFileOutStream File(Path.ToUTF8().ToStdString(), IOUtil::eLittleEndian);
-
-    if (!File.IsValid())
-    {
-        Log::Error("Unable to save cache data " + TString(Path.GetFileName()) + "; couldn't open file");
-        return false;
-    }
-
-    // Header
-    File.WriteString("CACH", 4);
-    File.WriteLong(0); // Reserved Space (Version)
-    File.WriteLong(mFlags & eREF_SavedFlags);
-
-    // Dependency Tree
-    if (!mpDependencies) UpdateDependencies();
-
-    u32 DepsSizeOffset = File.Tell();
-    File.WriteLong(0);
-
-    u32 DepsStart = File.Tell();
-    if (mpDependencies) mpDependencies->Write(File);
-    u32 DepsEnd = File.Tell();
-    u32 DepsSize = DepsEnd- DepsStart;
-
-    File.Seek(DepsSizeOffset, SEEK_SET);
-    File.WriteLong(DepsSize);
-    File.Seek(DepsEnd, SEEK_SET);
-
-    // Thumbnail
-    File.WriteLong(0); // Reserved Space (Thumbnail Size)
-
-    return true;
 }
 
 void CResourceEntry::UpdateDependencies()
@@ -125,6 +56,14 @@ void CResourceEntry::UpdateDependencies()
         mpDependencies = nullptr;
     }
 
+    // todo: more robust method of skipping dependency updates. For now just skip the most
+    // time-consuming type that can't have dependencies (textures).
+    if (ResourceType() == eTexture)
+    {
+        mpDependencies = new CDependencyTree(ID());
+        return;
+    }
+
     bool WasLoaded = IsLoaded();
 
     if (!mpResource)
@@ -225,8 +164,13 @@ void CResourceEntry::SetGame(EGame NewGame)
     }
 }
 
-bool CResourceEntry::Save()
+bool CResourceEntry::Save(bool SkipCacheSave /*= false*/)
 {
+    // SkipCacheSave argument tells us not to save the resource cache file. This is generally not advised because we don't
+    // want the actual resource data to desync from the cache data. However, there are occasions where we save multiple
+    // resources at a time and in that case it's preferable to only save the cache once. If you do set SkipCacheSave to true,
+    // then make sure you manually update the cache file afterwards.
+    //
     // For now, always save the resource when this function is called even if there's been no changes made to it in memory.
     // In the future this might not be desired behavior 100% of the time.
     // We also might want this function to trigger a cook for certain resource types eventually.
@@ -249,9 +193,11 @@ bool CResourceEntry::Save()
         mpResource->Serialize(Writer);
     }
 
-    // Resource has been saved, now update cache file
+    // Resource has been saved, now update dependencies + cache file
     UpdateDependencies();
-    SaveCacheData();
+
+    if (!SkipCacheSave)
+        mpStore->SaveCacheFile();
 
     if (ShouldCollectGarbage)
         mpStore->DestroyUnreferencedResources();

src/Core/GameProject/CResourceEntry.h

@@ -44,8 +44,7 @@ public:
                    EResType Type, bool Transient = false);
     ~CResourceEntry();
 
-    bool LoadCacheData();
+    void SerializeCacheData(IArchive& rArc);
-    bool SaveCacheData();
     void UpdateDependencies();
     TWideString CacheDataPath(bool Relative = false) const;
 
@@ -58,7 +57,7 @@ public:
     u64 Size() const;
     bool NeedsRecook() const;
     void SetGame(EGame NewGame);
-    bool Save();
+    bool Save(bool SkipCacheSave = false);
     CResource* Load();
     CResource* LoadCooked(IInputStream& rInput);
     bool Unload();

src/Core/GameProject/CResourceStore.cpp

@@ -1,10 +1,13 @@
 #include "CResourceStore.h"
 #include "CGameExporter.h"
 #include "CGameProject.h"
+#include "CResourceIterator.h"
 #include "Core/Resource/CResource.h"
 #include <Common/AssertMacro.h>
 #include <Common/FileUtil.h>
 #include <Common/Log.h>
+#include <Common/Serialization/Binary.h>
+#include <Common/Serialization/XML.h>
 #include <tinyxml2.h>
 
 using namespace tinyxml2;
@@ -34,92 +37,151 @@ CResourceStore::~CResourceStore()
         delete *It;
 }
 
-void CResourceStore::LoadResourceDatabase(const TString& rkPath)
+void CResourceStore::SerializeResourceDatabase(IArchive& rArc)
 {
-    XMLDocument Doc;
+    struct SDatabaseResource
-    Doc.LoadFile(*rkPath);
-
-    if (!Doc.Error())
     {
-        XMLElement *pRoot = Doc.FirstChildElement("ResourceDatabase");
+        CAssetID ID;
-        //EDatabaseVersion Version = (EDatabaseVersion) TString(pRoot->Attribute("Version")).ToInt32(10); // Version currently unused
+        CFourCC Type;
+        TWideString Directory;
+        TWideString Name;
 
-        XMLElement *pResources = pRoot->FirstChildElement("Resources");
+        void Serialize(IArchive& rArc)
-        XMLElement *pRes = pResources->FirstChildElement("Resource");
-        u32 ResIndex = 0;
-
-        while (pRes)
         {
-            XMLElement *pID = pRes->FirstChildElement("ID");
+            rArc << SERIAL_AUTO(ID) << SERIAL_AUTO(Type) << SERIAL_AUTO(Directory) << SERIAL_AUTO(Name);
-            XMLElement *pType = pRes->FirstChildElement("Type");
+        }
-            XMLElement *pDir = pRes->FirstChildElement("FileDir");
+    };
-            XMLElement *pName = pRes->FirstChildElement("FileName");
+    std::vector<SDatabaseResource> Resources;
 
-            if (pID && pType && pDir && pName)
+    // Populate resource list
-            {
+    if (!rArc.IsReader())
-                CAssetID ID = CAssetID::FromString(pID->GetText());
+    {
-                EResType Type = CResource::ResTypeForExtension(pType->GetText());
+        Resources.reserve(mResourceEntries.size());
-                TWideString FileDir = pDir->GetText();
-                TWideString FileName = pName->GetText();
-                RegisterResource(ID, Type, FileDir, FileName);
-            }
-            else
-                Log::Error("Error reading " + rkPath + ": Resource entry " + TString::FromInt32(ResIndex, 0, 10) + " is missing one or more required components");
 
-            ResIndex++;
+        for (CResourceIterator It(this); It; ++It)
-            pRes = pRes->NextSiblingElement("Resource");
+        {
+            if (!It->IsTransient())
+                Resources.push_back( SDatabaseResource { It->ID(), It->CookedExtension(), It->Directory()->FullPath(), It->Name() } );
         }
     }
 
-    // All resources registered - load cache data
+    // Serialize
-    for (auto It = mResourceEntries.begin(); It != mResourceEntries.end(); It++)
+    rArc << SERIAL_CONTAINER_AUTO(Resources, "Resource");
+
+    // Register resources
+    if (rArc.IsReader())
     {
-        CResourceEntry *pEntry = It->second;
+        for (auto Iter = Resources.begin(); Iter != Resources.end(); Iter++)
-        if (!pEntry->IsTransient())
+        {
-            pEntry->LoadCacheData();
+            SDatabaseResource& rRes = *Iter;
+            RegisterResource(rRes.ID, CResource::ResTypeForExtension(rRes.Type), rRes.Directory, rRes.Name);
+        }
     }
 }
 
-void CResourceStore::SaveResourceDatabase(const TString& rkPath) const
+void CResourceStore::LoadResourceDatabase()
 {
-    XMLDocument Doc;
+    ASSERT(mpProj);
+    TString Path = mpProj->ResourceDBPath(false).ToUTF8();
 
-    XMLDeclaration *pDecl = Doc.NewDeclaration();
+    CXMLReader Reader(Path);
-    Doc.LinkEndChild(pDecl);
+    SerializeResourceDatabase(Reader);
+    LoadCacheFile();
+}
 
-    XMLElement *pRoot = Doc.NewElement("ResourceDatabase");
+void CResourceStore::SaveResourceDatabase()
-    pRoot->SetAttribute("Version", eVer_Current);
+{
-    Doc.LinkEndChild(pRoot);
+    ASSERT(mpProj);
+    TString Path = mpProj->ResourceDBPath(false).ToUTF8();
 
-    XMLElement *pResources = Doc.NewElement("Resources");
+    CXMLWriter Writer(Path, "ResourceDB", 0, mpProj ? mpProj->Game() : eUnknownGame);
-    pRoot->LinkEndChild(pResources);
+    SerializeResourceDatabase(Writer);
+}
 
-    for (auto It = mResourceEntries.begin(); It != mResourceEntries.end(); It++)
+void CResourceStore::LoadCacheFile()
+{
+    TString CacheDataPath = mpProj->ResourceCachePath(false).ToUTF8();
+    CFileInStream CacheFile(CacheDataPath.ToStdString(), IOUtil::eBigEndian);
+    ASSERT(CacheFile.IsValid());
+
+    // Cache header
+    CFourCC Magic(CacheFile);
+
+    if (Magic != "CACH")
     {
-        CResourceEntry *pEntry = It->second;
+        Log::Error("Invalid resource cache data magic: " + Magic.ToString());
-        if (pEntry->IsTransient()) continue;
+        return;
-
-        XMLElement *pRes = Doc.NewElement("Resource");
-        pResources->LinkEndChild(pRes);
-
-        XMLElement *pID = Doc.NewElement("ID");
-        pID->SetText(*pEntry->ID().ToString());
-        pRes->LinkEndChild(pID);
-
-        XMLElement *pType = Doc.NewElement("Type");
-        pType->SetText(*GetResourceCookedExtension(pEntry->ResourceType(), pEntry->Game()));
-        pRes->LinkEndChild(pType);
-
-        XMLElement *pDir = Doc.NewElement("FileDir");
-        pDir->SetText(*pEntry->Directory()->FullPath().ToUTF8());
-        pRes->LinkEndChild(pDir);
-
-        XMLElement *pName = Doc.NewElement("FileName");
-        pName->SetText(*pEntry->Name().ToUTF8());
-        pRes->LinkEndChild(pName);
     }
 
-    Doc.SaveFile(*rkPath);
+    CSerialVersion Version(CacheFile);
+    u32 NumResources = CacheFile.ReadLong();
+
+    for (u32 iRes = 0; iRes < NumResources; iRes++)
+    {
+        CAssetID ID(CacheFile, Version.Game());
+        u32 EntryCacheSize = CacheFile.ReadLong();
+        u32 EntryCacheEnd = CacheFile.Tell() + EntryCacheSize;
+
+        CResourceEntry *pEntry = FindEntry(ID);
+
+        if (pEntry && !pEntry->IsTransient())
+        {
+            CBasicBinaryReader Reader(&CacheFile, Version);
+
+            if (Reader.ParamBegin("EntryCache"))
+            {
+                pEntry->SerializeCacheData(Reader);
+                Reader.ParamEnd();
+            }
+        }
+
+        CacheFile.Seek(EntryCacheEnd, SEEK_SET);
+    }
+}
+
+void CResourceStore::SaveCacheFile()
+{
+    TString CacheDataPath = mpProj->ResourceCachePath(false).ToUTF8();
+    CFileOutStream CacheFile(CacheDataPath.ToStdString(), IOUtil::eBigEndian);
+    ASSERT(CacheFile.IsValid());
+
+    // Cache header
+    CFourCC("CACH").Write(CacheFile);
+    CSerialVersion Version(0, 0, mpProj->Game());
+    Version.Write(CacheFile);
+
+    u32 ResCountOffset = CacheFile.Tell();
+    u32 ResCount = 0;
+    CacheFile.WriteLong(0); // Resource count dummy - fill in when we know the real count
+
+    // Save entry cache data
+    // Structure: Entry Asset ID -> Entry Cache Size -> Serialized Entry Cache Data
+    for (CResourceIterator It(this); It; ++It)
+    {
+        if (!It->IsTransient())
+        {
+            ResCount++;
+            It->ID().Write(CacheFile);
+            u32 SizeOffset = CacheFile.Tell();
+            CacheFile.WriteLong(0);
+
+            CBasicBinaryWriter Writer(&CacheFile, Version.FileVersion(), Version.Game());
+
+            if (Writer.ParamBegin("EntryCache"))
+            {
+                It->SerializeCacheData(Writer);
+                Writer.ParamEnd();
+            }
+
+            u32 EntryCacheEnd = CacheFile.Tell();
+            CacheFile.Seek(SizeOffset, SEEK_SET);
+            CacheFile.WriteLong(EntryCacheEnd - SizeOffset - 4);
+            CacheFile.Seek(EntryCacheEnd, SEEK_SET);
+        }
+    }
+
+    CacheFile.Seek(ResCountOffset, SEEK_SET);
+    CacheFile.WriteLong(ResCount);
 }
 
 void CResourceStore::SetActiveProject(CGameProject *pProj)
@@ -134,7 +196,7 @@ void CResourceStore::SetActiveProject(CGameProject *pProj)
         mpProjectRoot = new CVirtualDirectory();
 
         if (!mpExporter)
-            LoadResourceDatabase(pProj->ResourceDBPath(false));
+            LoadResourceDatabase();
     }
 }
 
@@ -145,7 +207,9 @@ void CResourceStore::CloseActiveProject()
     DestroyUnreferencedResources();
 
     // Delete all entries from old project
-    for (auto It = mResourceEntries.begin(); It != mResourceEntries.end(); It++)
+    auto It = mResourceEntries.begin();
+
+    while (It != mResourceEntries.end())
     {
         CResourceEntry *pEntry = It->second;
 
@@ -161,11 +225,12 @@ void CResourceStore::CloseActiveProject()
                 mLoadedResources.erase(LoadIt);
             }
 
-            It = mResourceEntries.erase(It);
-            if (It == mResourceEntries.end()) break;
-
             delete pEntry;
+            It = mResourceEntries.erase(It);
         }
+
+        else
+            It++;
     }
 
     delete mpProjectRoot;
@@ -402,8 +467,9 @@ void CResourceStore::DestroyUnreferencedResources()
     do
     {
         NumDeleted = 0;
+        auto It = mLoadedResources.begin();
 
-        for (auto It = mLoadedResources.begin(); It != mLoadedResources.end(); It++)
+        while (It != mLoadedResources.end())
         {
             CResourceEntry *pEntry = It->second;
 
@@ -415,9 +481,9 @@ void CResourceStore::DestroyUnreferencedResources()
                 // Transient resources should have their entries cleared out when the resource is unloaded
                 if (pEntry->IsTransient())
                     DeleteResourceEntry(pEntry);
-
-                if (It == mLoadedResources.end()) break;
             }
+
+            else It++;
         }
     } while (NumDeleted > 0);
 

src/Core/GameProject/CResourceStore.h

@@ -42,8 +42,11 @@ public:
     CResourceStore();
     CResourceStore(CGameExporter *pExporter);
     ~CResourceStore();
-    void LoadResourceDatabase(const TString& rkPath);
+    void SerializeResourceDatabase(IArchive& rArc);
-    void SaveResourceDatabase(const TString& rkPath) const;
+    void LoadResourceDatabase();
+    void SaveResourceDatabase();
+    void LoadCacheFile();
+    void SaveCacheFile();
     void SetActiveProject(CGameProject *pProj);
     void CloseActiveProject();
     CVirtualDirectory* GetVirtualDirectory(const TWideString& rkPath, bool Transient, bool AllowCreate);

src/Core/Resource/Area/CGameArea.cpp

@@ -43,8 +43,12 @@ CDependencyTree* CGameArea::BuildDependencyTree() const
         pTree->AddDependency(*Iter);
 
     pTree->AddDependency(mPathID);
-    pTree->AddDependency(mPortalAreaID);
+
-    pTree->AddDependency(mpPoiToWorldMap);
+    if (Game() >= eEchoesDemo)
+    {
+        pTree->AddDependency(mPortalAreaID);
+        pTree->AddDependency(mpPoiToWorldMap);
+    }
     
     // Layer dependencies
     for (u32 iLayer = 0; iLayer < mScriptLayers.size(); iLayer++)

src/Core/Resource/CWorld.cpp

@@ -62,14 +62,14 @@ void CWorld::SetAreaLayerInfo(CGameArea *pArea)
 // ************ SERIALIZATION ************
 void CWorld::Serialize(IArchive& rArc)
 {
-    rArc << SERIAL("WorldNameSTRG", mpWorldName);
+    rArc << SERIAL("WorldNameString", mpWorldName);
 
     if (rArc.Game() == eEchoesDemo || rArc.Game() == eEchoes)
-        rArc << SERIAL("DarkWorldNameSTRG", mpDarkWorldName);
+        rArc << SERIAL("DarkWorldNameString", mpDarkWorldName);
 
     rArc << SERIAL("WorldSaveInfo", mpSaveWorld)
          << SERIAL("WorldMap", mpMapWorld)
-         << SERIAL("DefaultSkyCMDL", mpDefaultSkybox);
+         << SERIAL("DefaultSkyModel", mpDefaultSkybox);
 
     if (rArc.Game() >= eEchoesDemo && rArc.Game() <= eCorruption)
         rArc << SERIAL("TempleKeyWorldIndex", mTempleKeyWorldIndex);
@@ -94,11 +94,11 @@ void Serialize(IArchive& rArc, CWorld::SMemoryRelay& rMemRelay)
 void Serialize(IArchive& rArc, CWorld::SArea& rArea)
 {
     rArc << SERIAL("Name", rArea.InternalName)
-         << SERIAL("NameSTRG", rArea.pAreaName)
+         << SERIAL("NameString", rArea.pAreaName)
+         << SERIAL("MREA", rArea.AreaResID)
+         << SERIAL("ID", rArea.AreaID)
          << SERIAL("Transform", rArea.Transform)
          << SERIAL("BoundingBox", rArea.AetherBox)
-         << SERIAL("AreaMREA", rArea.AreaResID)
-         << SERIAL("AreaID", rArea.AreaID)
          << SERIAL("AllowPakDuplicates", rArea.AllowPakDuplicates)
          << SERIAL_CONTAINER("AttachedAreas", rArea.AttachedAreaIDs, "AreaIndex")
          << SERIAL_CONTAINER("RelModules", rArea.RelFilenames, "Module")