AxioDL
/
PrimeWorldEditor
mirror of https://github.com/AxioDL/PrimeWorldEditor.git
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Lots of changes to how resource database/entry data is serialized; resource database file is now binary and merged with the cache data file. Binary reader/writer now use 32-bit sizes.

This commit is contained in:
Aruki 2017-07-04 19:02:56 -06:00
parent 4652e125e5
commit 3fa1279d29
17 changed files with 329 additions and 831 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
resources/ResourceCacheData.bin
View File

Binary file not shown.

505
resources/ResourceDatabase.xml
View File

@ -1,505 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<ResourceDB ArchiveVer="0" FileVer="0" Game="MPRM">
<RawDir></RawDir>
<CookedDir></CookedDir>
<Resources Size="83">
<Resource>
<ID>08F992A8</ID>
<Type>TXTR</Type>
<Directory>script/common/</Directory>
<Name>RandomRelay</Name>
</Resource>
<Resource>
<ID>0A11104B</ID>
<Type>CMDL</Type>
<Directory>editor/</Directory>
<Name>TranslateX</Name>
</Resource>
<Resource>
<ID>0E04ED00</ID>
<Type>TXTR</Type>
<Directory></Directory>
<Name>LightAmbientMask</Name>
</Resource>
<Resource>
<ID>0EDB1C7A</ID>
<Type>CMDL</Type>
<Directory>script/dkcr/</Directory>
<Name>Waypoint</Name>
</Resource>
<Resource>
<ID>0FB28A24</ID>
<Type>TXTR</Type>
<Directory></Directory>
<Name>LightAmbient</Name>
</Resource>
<Resource>
<ID>0FD29E3B</ID>
<Type>CMDL</Type>
<Directory>script/common/</Directory>
<Name>Waypoint</Name>
</Resource>
<Resource>
<ID>115C21DD</ID>
<Type>TXTR</Type>
<Directory>script/common/</Directory>
<Name>CameraFilterKeyframe</Name>
</Resource>
<Resource>
<ID>1C010ADC</ID>
<Type>CMDL</Type>
<Directory>editor/</Directory>
<Name>TranslatePolyXY</Name>
</Resource>
<Resource>
<ID>1FFC7E43</ID>
<Type>CMDL</Type>
<Directory>editor/</Directory>
<Name>RotateXYZ</Name>
</Resource>
<Resource>
<ID>2A14558D</ID>
<Type>TXTR</Type>
<Directory>script/common/</Directory>
<Name>HUDMemo</Name>
</Resource>
<Resource>
<ID>2B75E193</ID>
<Type>TXTR</Type>
<Directory>script/common/</Directory>
<Name>VisorFlare</Name>
</Resource>
<Resource>
<ID>31485556</ID>
<Type>TXTR</Type>
<Directory>script/mp2/</Directory>
<Name>AdvancedCounter</Name>
</Resource>
<Resource>
<ID>31992094</ID>
<Type>CMDL</Type>
<Directory>editor/</Directory>
<Name>TranslateY</Name>
</Resource>
<Resource>
<ID>32432719</ID>
<Type>CMDL</Type>
<Directory>script/common/</Directory>
<Name>SpiderBallWaypoint</Name>
</Resource>
<Resource>
<ID>32C9F9D1</ID>
<Type>TXTR</Type>
<Directory>script/common/</Directory>
<Name>StreamedAudio</Name>
</Resource>
<Resource>
<ID>3440526F</ID>
<Type>CMDL</Type>
<Directory>editor/</Directory>
<Name>TranslateLinesXY</Name>
</Resource>
<Resource>
<ID>3A947189</ID>
<Type>CMDL</Type>
<Directory>script/common/</Directory>
<Name>AIWaypoint</Name>
</Resource>
<Resource>
<ID>3C093300</ID>
<Type>CMDL</Type>
<Directory>editor/</Directory>
<Name>ScaleXYZ</Name>
</Resource>
<Resource>
<ID>40969072</ID>
<Type>TXTR</Type>
<Directory>script/common/</Directory>
<Name>AreaAttributes</Name>
</Resource>
<Resource>
<ID>43891B25</ID>
<Type>CMDL</Type>
<Directory>editor/</Directory>
<Name>TranslatePolyXZ</Name>
</Resource>
<Resource>
<ID>43E9B0A9</ID>
<Type>CMDL</Type>
<Directory>editor/</Directory>
<Name>ScaleX</Name>
</Resource>
<Resource>
<ID>4443279C</ID>
<Type>TXTR</Type>
<Directory>script/common/</Directory>
<Name>CameraBlurKeyframe</Name>
</Resource>
<Resource>
<ID>4A2D76D1</ID>
<Type>TXTR</Type>
<Directory>script/common/</Directory>
<Name>DamageableTrigger</Name>
</Resource>
<Resource>
<ID>4C3D3E7F</ID>
<Type>TXTR</Type>
<Directory>script/common/</Directory>
<Name>GrapplePoint</Name>
</Resource>
<Resource>
<ID>4E133EF2</ID>
<Type>TXTR</Type>
<Directory>script/common/</Directory>
<Name>SpiderBallAttractionSurface</Name>
</Resource>
<Resource>
<ID>57F8715F</ID>
<Type>TXTR</Type>
<Directory>script/common/</Directory>
<Name>Dock</Name>
</Resource>
<Resource>
<ID>592130DD</ID>
<Type>CMDL</Type>
<Directory>editor/</Directory>
<Name>TranslateZ</Name>
</Resource>
<Resource>
<ID>5BC862B8</ID>
<Type>CMDL</Type>
<Directory>editor/</Directory>
<Name>TranslateLinesXZ</Name>
</Resource>
<Resource>
<ID>5D08CCF6</ID>
<Type>TXTR</Type>
<Directory>script/mp3/</Directory>
<Name>WeaponGenerator</Name>
</Resource>
<Resource>
<ID>5F3B141B</ID>
<Type>CMDL</Type>
<Directory></Directory>
<Name>VolumeBox</Name>
</Resource>
<Resource>
<ID>61B511E5</ID>
<Type>TXTR</Type>
<Directory></Directory>
<Name>Checkerboard</Name>
</Resource>
<Resource>
<ID>6649251D</ID>
<Type>CMDL</Type>
<Directory>script/common/</Directory>
<Name>Camera</Name>
</Resource>
<Resource>
<ID>6A17B38F</ID>
<Type>TXTR</Type>
<Directory>script/common/</Directory>
<Name>Sound</Name>
</Resource>
<Resource>
<ID>6B1FABDD</ID>
<Type>CMDL</Type>
<Directory></Directory>
<Name>Cube</Name>
</Resource>
<Resource>
<ID>6B71C0F2</ID>
<Type>CMDL</Type>
<Directory>editor/</Directory>
<Name>ScaleY</Name>
</Resource>
<Resource>
<ID>6C6CE7FE</ID>
<Type>TXTR</Type>
<Directory></Directory>
<Name>LightSpotMask</Name>
</Resource>
<Resource>
<ID>6D5BC167</ID>
<Type>TXTR</Type>
<Directory>script/common/</Directory>
<Name>Effect</Name>
</Resource>
<Resource>
<ID>6DD88D5D</ID>
<Type>CMDL</Type>
<Directory></Directory>
<Name>RotationArrow</Name>
</Resource>
<Resource>
<ID>6E57F7E0</ID>
<Type>CMDL</Type>
<Directory></Directory>
<Name>VolumeCylinder</Name>
</Resource>
<Resource>
<ID>72978DCF</ID>
<Type>TXTR</Type>
<Directory>script/common/</Directory>
<Name>Relay</Name>
</Resource>
<Resource>
<ID>791A7BFD</ID>
<Type>CMDL</Type>
<Directory></Directory>
<Name>SphereDoubleSided</Name>
</Resource>
<Resource>
<ID>7F494724</ID>
<Type>CMDL</Type>
<Directory>editor/</Directory>
<Name>RotateX</Name>
</Resource>
<Resource>
<ID>81326A53</ID>
<Type>CMDL</Type>
<Directory>editor/</Directory>
<Name>ScaleLinesYZ</Name>
</Resource>
<Resource>
<ID>825CDFA8</ID>
<Type>CMDL</Type>
<Directory>editor/</Directory>
<Name>ScalePolyYZ</Name>
</Resource>
<Resource>
<ID>826C9800</ID>
<Type>TXTR</Type>
<Directory>script/common/</Directory>
<Name>ColorModulate</Name>
</Resource>
<Resource>
<ID>85CE16D7</ID>
<Type>TXTR</Type>
<Directory>script/common/</Directory>
<Name>StreamedMovie</Name>
</Resource>
<Resource>
<ID>875A6FD7</ID>
<Type>TXTR</Type>
<Directory>script/common/</Directory>
<Name>PointOfInterest</Name>
</Resource>
<Resource>
<ID>9039791A</ID>
<Type>TXTR</Type>
<Directory></Directory>
<Name>LightDirectional</Name>
</Resource>
<Resource>
<ID>91437414</ID>
<Type>CMDL</Type>
<Directory>script/common/</Directory>
<Name>CameraWaypoint</Name>
</Resource>
<Resource>
<ID>92F9D13B</ID>
<Type>CMDL</Type>
<Directory>editor/</Directory>
<Name>ScaleZ</Name>
</Resource>
<Resource>
<ID>95261BB8</ID>
<Type>CMDL</Type>
<Directory></Directory>
<Name>WireSphere</Name>
</Resource>
<Resource>
<ID>968C405E</ID>
<Type>TXTR</Type>
<Directory>script/common/</Directory>
<Name>SoundModifier</Name>
</Resource>
<Resource>
<ID>9FF04AA1</ID>
<Type>TXTR</Type>
<Directory>script/common/</Directory>
<Name>Generator</Name>
</Resource>
<Resource>
<ID>A6D1576D</ID>
<Type>CMDL</Type>
<Directory>editor/</Directory>
<Name>RotateY</Name>
</Resource>
<Resource>
<ID>A82A3F02</ID>
<Type>TXTR</Type>
<Directory></Directory>
<Name>LightSpot</Name>
</Resource>
<Resource>
<ID>AC18950A</ID>
<Type>TXTR</Type>
<Directory>script/common/</Directory>
<Name>RadialDamage</Name>
</Resource>
<Resource>
<ID>B0D52FF7</ID>
<Type>TXTR</Type>
<Directory></Directory>
<Name>LightCustomMask</Name>
</Resource>
<Resource>
<ID>B0E09096</ID>
<Type>TXTR</Type>
<Directory></Directory>
<Name>LightDirectionalMask</Name>
</Resource>
<Resource>
<ID>B23F1022</ID>
<Type>TXTR</Type>
<Directory>script/common/</Directory>
<Name>VisorGoo</Name>
</Resource>
<Resource>
<ID>B2DBCAED</ID>
<Type>TXTR</Type>
<Directory>script/common/</Directory>
<Name>PickupGenerator</Name>
</Resource>
<Resource>
<ID>B3050E38</ID>
<Type>TXTR</Type>
<Directory>script/common/</Directory>
<Name>SequenceTimer</Name>
</Resource>
<Resource>
<ID>B97DB26B</ID>
<Type>TXTR</Type>
<Directory>script/common/</Directory>
<Name>Timer</Name>
</Resource>
<Resource>
<ID>BA7EA4F6</ID>
<Type>TXTR</Type>
<Directory>script/common/</Directory>
<Name>Trigger</Name>
</Resource>
<Resource>
<ID>C0FBED3D</ID>
<Type>CMDL</Type>
<Directory>editor/</Directory>
<Name>ScaleLinesXY</Name>
</Resource>
<Resource>
<ID>C2266292</ID>
<Type>CMDL</Type>
<Directory>editor/</Directory>
<Name>ScalePolyXY</Name>
</Resource>
<Resource>
<ID>C4E83425</ID>
<Type>TXTR</Type>
<Directory>script/common/</Directory>
<Name>DistanceFog</Name>
</Resource>
<Resource>
<ID>C7BEFE29</ID>
<Type>CMDL</Type>
<Directory></Directory>
<Name>VolumeSphere</Name>
</Resource>
<Resource>
<ID>CB904B3D</ID>
<Type>TXTR</Type>
<Directory>script/common/</Directory>
<Name>SpecialFunction</Name>
</Resource>
<Resource>
<ID>CE5967B6</ID>
<Type>CMDL</Type>
<Directory>editor/</Directory>
<Name>RotateZ</Name>
</Resource>
<Resource>
<ID>D2F07DAF</ID>
<Type>TXTR</Type>
<Directory>script/common/</Directory>
<Name>ConditionalRelay</Name>
</Resource>
<Resource>
<ID>D7B6A50D</ID>
<Type>CMDL</Type>
<Directory></Directory>
<Name>Sphere</Name>
</Resource>
<Resource>
<ID>DA185196</ID>
<Type>TXTR</Type>
<Directory></Directory>
<Name>VolumeCheckerboard</Name>
</Resource>
<Resource>
<ID>DC3787F2</ID>
<Type>CMDL</Type>
<Directory>editor/</Directory>
<Name>TranslatePolyYZ</Name>
</Resource>
<Resource>
<ID>E1A0D860</ID>
<Type>TXTR</Type>
<Directory>script/common/</Directory>
<Name>StreamedAudioModifier</Name>
</Resource>
<Resource>
<ID>E883FD86</ID>
<Type>CMDL</Type>
<Directory>editor/</Directory>
<Name>ScaleLinesXZ</Name>
</Resource>
<Resource>
<ID>E9AE72DB</ID>
<Type>CMDL</Type>
<Directory>editor/</Directory>
<Name>ScalePolyXZ</Name>
</Resource>
<Resource>
<ID>EE340FC4</ID>
<Type>TXTR</Type>
<Directory>script/common/</Directory>
<Name>MemoryRelay</Name>
</Resource>
<Resource>
<ID>F476CF85</ID>
<Type>CMDL</Type>
<Directory>editor/</Directory>
<Name>TranslateLinesYZ</Name>
</Resource>
<Resource>
<ID>F76B369A</ID>
<Type>TXTR</Type>
<Directory>script/common/</Directory>
<Name>Counter</Name>
</Resource>
<Resource>
<ID>FB50DA78</ID>
<Type>CMDL</Type>
<Directory>editor/</Directory>
<Name>RotateClipOutline</Name>
</Resource>
<Resource>
<ID>FC8FA98B</ID>
<Type>TXTR</Type>
<Directory></Directory>
<Name>LightCustom</Name>
</Resource>
<Resource>
<ID>FD78FBC8</ID>
<Type>TXTR</Type>
<Directory>script/mp1/</Directory>
<Name>NewCameraShaker</Name>
</Resource>
<Resource>
<ID>FFAC2525</ID>
<Type>TXTR</Type>
<Directory>script/common/</Directory>
<Name>CameraShaker</Name>
</Resource>
</Resources>
</ResourceDB>

2
src/Common/Flags.h
View File

@ -7,7 +7,7 @@
template<typename FlagEnum> template<typename FlagEnum>
class TFlags class TFlags
{ {
int mValue; u32 mValue;
public: public:
TFlags() : mValue(0) {} TFlags() : mValue(0) {}

11
src/Common/Serialization/CBasicBinaryReader.h
View File

@ -12,22 +12,27 @@
class CBasicBinaryReader : public IArchive class CBasicBinaryReader : public IArchive
{ {
IInputStream *mpStream; IInputStream *mpStream;
bool mMagicValid;
bool mOwnsStream; bool mOwnsStream;
public: public:
CBasicBinaryReader(const TString& rkFilename, IOUtil::EEndianness = IOUtil::eLittleEndian) CBasicBinaryReader(const TString& rkFilename, u32 Magic)
: IArchive(true, false) : IArchive(true, false)
, mOwnsStream(true) , mOwnsStream(true)
{ {
mpStream = new CFileInStream(rkFilename, IOUtil::eBigEndian); mpStream = new CFileInStream(rkFilename, IOUtil::eBigEndian);
ASSERT(mpStream->IsValid());
if (mpStream->IsValid())
{
mMagicValid = (mpStream->ReadLong() == Magic);
CSerialVersion Version(*mpStream); CSerialVersion Version(*mpStream);
SetVersion(Version); SetVersion(Version);
} }
}
CBasicBinaryReader(IInputStream *pStream, const CSerialVersion& rkVersion) CBasicBinaryReader(IInputStream *pStream, const CSerialVersion& rkVersion)
: IArchive(true, false) : IArchive(true, false)
, mMagicValid(true)
, mOwnsStream(false) , mOwnsStream(false)
{ {
ASSERT(pStream->IsValid()); ASSERT(pStream->IsValid());
@ -40,6 +45,8 @@ public:
if (mOwnsStream) delete mpStream; if (mOwnsStream) delete mpStream;
} }
inline bool IsValid() const { return mpStream->IsValid(); }
// Interface // Interface
virtual bool ParamBegin(const char*) { return true; } virtual bool ParamBegin(const char*) { return true; }
virtual void ParamEnd() { } virtual void ParamEnd() { }

22
src/Common/Serialization/CBasicBinaryWriter.h
View File

@ -11,20 +11,26 @@
class CBasicBinaryWriter : public IArchive class CBasicBinaryWriter : public IArchive
{ {
IOutputStream *mpStream; IOutputStream *mpStream;
u32 mMagic;
bool mOwnsStream; bool mOwnsStream;
public: public:
CBasicBinaryWriter(const TString& rkFilename, u16 FileVersion, EGame Game = eUnknownGame, IOUtil::EEndianness = IOUtil::eLittleEndian) CBasicBinaryWriter(const TString& rkFilename, u32 Magic, u16 FileVersion, EGame Game)
: IArchive(false, true) : IArchive(false, true)
, mMagic(Magic)
, mOwnsStream(true) , mOwnsStream(true)
{ {
mpStream = new CFileOutStream(rkFilename, IOUtil::eBigEndian); mpStream = new CFileOutStream(rkFilename, IOUtil::eBigEndian);
ASSERT(mpStream->IsValid());
if (mpStream->IsValid())
{
mpStream->WriteLong(0); // Magic is written after the rest of the file is successfully saved
SetVersion(skCurrentArchiveVersion, FileVersion, Game); SetVersion(skCurrentArchiveVersion, FileVersion, Game);
GetVersionInfo().Write(*mpStream); GetVersionInfo().Write(*mpStream);
} }
}
CBasicBinaryWriter(IOutputStream *pStream, u16 FileVersion, EGame Game = eUnknownGame) CBasicBinaryWriter(IOutputStream *pStream, u16 FileVersion, EGame Game)
: IArchive(false, true) : IArchive(false, true)
, mOwnsStream(false) , mOwnsStream(false)
{ {
@ -44,8 +50,16 @@ public:
~CBasicBinaryWriter() ~CBasicBinaryWriter()
{ {
if (mOwnsStream) delete mpStream; // Write magic and delete stream
if (mOwnsStream)
{
mpStream->GoTo(0);
mpStream->WriteLong(mMagic);
delete mpStream;
} }
}
inline bool IsValid() const { return mpStream->IsValid(); }
// Interface // Interface
virtual bool ParamBegin(const char*) { return true; } virtual bool ParamBegin(const char*) { return true; }

50
src/Common/Serialization/CBinaryReader.h
View File

@ -10,35 +10,40 @@ class CBinaryReader : public IArchive
struct SParameter struct SParameter
{ {
u32 Offset; u32 Offset;
u16 Size; u32 Size;
u16 NumChildren; u32 NumChildren;
u16 ChildIndex; u32 ChildIndex;
bool Abstract; bool Abstract;
}; };
std::vector<SParameter> mParamStack; std::vector<SParameter> mParamStack;
IInputStream *mpStream; IInputStream *mpStream;
bool mMagicValid;
bool mOwnsStream; bool mOwnsStream;
public: public:
CBinaryReader(const TString& rkFilename) CBinaryReader(const TString& rkFilename, u32 Magic)
: IArchive(true, false) : IArchive(true, false)
, mOwnsStream(true) , mOwnsStream(true)
{ {
mpStream = new CFileInStream(rkFilename, IOUtil::eBigEndian); mpStream = new CFileInStream(rkFilename, IOUtil::eBigEndian);
ASSERT(mpStream->IsValid());
if (mpStream->IsValid())
{
mMagicValid = (mpStream->ReadLong() == Magic);
CSerialVersion Version(*mpStream); CSerialVersion Version(*mpStream);
SetVersion(Version); SetVersion(Version);
}
InitParamStack(); InitParamStack();
} }
CBinaryReader(IInputStream *pStream, const CSerialVersion& rkVersion) CBinaryReader(IInputStream *pStream, const CSerialVersion& rkVersion)
: IArchive(true, false) : IArchive(true, false)
, mMagicValid(true)
, mOwnsStream(false) , mOwnsStream(false)
{ {
ASSERT(pStream->IsValid()); ASSERT(pStream && pStream->IsValid());
mpStream = pStream; mpStream = pStream;
SetVersion(rkVersion); SetVersion(rkVersion);
@ -50,25 +55,32 @@ public:
if (mOwnsStream) delete mpStream; if (mOwnsStream) delete mpStream;
} }
inline bool IsValid() const { return mpStream->IsValid() && mMagicValid; }
private: private:
void InitParamStack() void InitParamStack()
{ {
mpStream->Skip(4); // Skip root ID (which is always -1) mpStream->Skip(4); // Skip root ID (which is always -1)
u16 Size = mpStream->ReadShort(); u32 Size = ReadSize();
u32 Offset = mpStream->Tell(); u32 Offset = mpStream->Tell();
u16 NumChildren = mpStream->ReadShort(); u32 NumChildren = ReadSize();
mParamStack.push_back( SParameter { Offset, Size, NumChildren, 0, false } ); mParamStack.push_back( SParameter { Offset, Size, NumChildren, 0, false } );
mParamStack.reserve(20); mParamStack.reserve(20);
} }
public: public:
// Interface // Interface
u32 ReadSize()
{
return (mArchiveVersion < eArVer_32BitBinarySize ? (u32) mpStream->ReadShort() : mpStream->ReadLong());
}
virtual bool ParamBegin(const char *pkName) virtual bool ParamBegin(const char *pkName)
{ {
// If this is the parent parameter's first child, then read the child count // If this is the parent parameter's first child, then read the child count
if (mParamStack.back().NumChildren == 0xFFFF) if (mParamStack.back().NumChildren == 0xFFFFFFFF)
{ {
mParamStack.back().NumChildren = mpStream->ReadShort(); mParamStack.back().NumChildren = ReadSize();
} }
// Save current offset // Save current offset
@ -79,12 +91,12 @@ public:
if (mParamStack.back().ChildIndex < mParamStack.back().NumChildren) if (mParamStack.back().ChildIndex < mParamStack.back().NumChildren)
{ {
u32 NextID = mpStream->ReadLong(); u32 NextID = mpStream->ReadLong();
u16 NextSize = mpStream->ReadShort(); u32 NextSize = ReadSize();
// Does the next parameter ID match the current one? // Does the next parameter ID match the current one?
if (NextID == ParamID) if (NextID == ParamID)
{ {
mParamStack.push_back( SParameter { Offset, NextSize, 0xFFFF, 0, false } ); mParamStack.push_back( SParameter { mpStream->Tell(), NextSize, 0xFFFFFFFF, 0, false } );
return true; return true;
} }
} }
@ -94,20 +106,20 @@ public:
{ {
bool ParentAbstract = mParamStack.back().Abstract; bool ParentAbstract = mParamStack.back().Abstract;
u32 ParentOffset = mParamStack.back().Offset; u32 ParentOffset = mParamStack.back().Offset;
u16 NumChildren = mParamStack.back().NumChildren; u32 NumChildren = mParamStack.back().NumChildren;
mpStream->GoTo(ParentOffset + (ParentAbstract ? 0xC : 0x8)); mpStream->GoTo(ParentOffset + (ParentAbstract ? 4 : 0));
for (u32 ChildIdx = 0; ChildIdx < NumChildren; ChildIdx++) for (u32 ChildIdx = 0; ChildIdx < NumChildren; ChildIdx++)
{ {
u32 ChildID = mpStream->ReadLong(); u32 ChildID = mpStream->ReadLong();
u16 ChildSize = mpStream->ReadShort(); u32 ChildSize = ReadSize();
if (ChildID != ParamID) if (ChildID != ParamID)
mpStream->Skip(ChildSize); mpStream->Skip(ChildSize);
else else
{ {
mParamStack.back().ChildIndex = (u16) ChildIdx; mParamStack.back().ChildIndex = ChildIdx;
mParamStack.push_back( SParameter { mpStream->Tell() - 6, ChildSize, 0xFFFF, 0, false } ); mParamStack.push_back( SParameter { mpStream->Tell(), ChildSize, 0xFFFFFFFF, 0, false } );
return true; return true;
} }
} }
@ -122,7 +134,7 @@ public:
{ {
// Make sure we're at the end of the parameter // Make sure we're at the end of the parameter
SParameter& rParam = mParamStack.back(); SParameter& rParam = mParamStack.back();
u32 EndOffset = rParam.Offset + rParam.Size + 6; u32 EndOffset = rParam.Offset + rParam.Size;
mpStream->GoTo(EndOffset); mpStream->GoTo(EndOffset);
mParamStack.pop_back(); mParamStack.pop_back();
@ -134,7 +146,7 @@ public:
virtual void SerializeContainerSize(u32& rSize, const TString& /*rkElemName*/) virtual void SerializeContainerSize(u32& rSize, const TString& /*rkElemName*/)
{ {
// Mostly handled by ParamBegin, we just need to return the size correctly so the container can be resized // Mostly handled by ParamBegin, we just need to return the size correctly so the container can be resized
rSize = (u32) mpStream->PeekShort(); rSize = (mArchiveVersion < eArVer_32BitBinarySize ? (u32) mpStream->PeekShort() : mpStream->PeekLong());
} }
virtual void SerializeAbstractObjectType(u32& rType) virtual void SerializeAbstractObjectType(u32& rType)

44
src/Common/Serialization/CBinaryWriter.h
View File

@ -9,32 +9,39 @@ class CBinaryWriter : public IArchive
struct SParameter struct SParameter
{ {
u32 Offset; u32 Offset;
u16 NumSubParams; u32 NumSubParams;
bool Abstract; bool Abstract;
}; };
std::vector<SParameter> mParamStack; std::vector<SParameter> mParamStack;
IOutputStream *mpStream; IOutputStream *mpStream;
u32 mMagic;
bool mOwnsStream; bool mOwnsStream;
public: public:
CBinaryWriter(const TString& rkFilename, u16 FileVersion, EGame Game = eUnknownGame) CBinaryWriter(const TString& rkFilename, u32 Magic, u16 FileVersion, EGame Game)
: IArchive(false, true) : IArchive(false, true)
, mMagic(Magic)
, mOwnsStream(true) , mOwnsStream(true)
{ {
mpStream = new CFileOutStream(rkFilename, IOUtil::eBigEndian); mpStream = new CFileOutStream(rkFilename, IOUtil::eBigEndian);
ASSERT(mpStream->IsValid());
if (mpStream->IsValid())
{
mpStream->WriteLong(0); // Magic is written after the rest of the file has been successfully written
SetVersion(skCurrentArchiveVersion, FileVersion, Game); SetVersion(skCurrentArchiveVersion, FileVersion, Game);
GetVersionInfo().Write(*mpStream); GetVersionInfo().Write(*mpStream);
}
InitParamStack(); InitParamStack();
} }
CBinaryWriter(IOutputStream *pStream, u16 FileVersion, EGame Game = eUnknownGame) CBinaryWriter(IOutputStream *pStream, u16 FileVersion, EGame Game)
: IArchive(false, true) : IArchive(false, true)
, mMagic(0)
, mOwnsStream(false) , mOwnsStream(false)
{ {
ASSERT(pStream->IsValid()); ASSERT(pStream && pStream->IsValid());
mpStream = pStream; mpStream = pStream;
SetVersion(skCurrentArchiveVersion, FileVersion, Game); SetVersion(skCurrentArchiveVersion, FileVersion, Game);
InitParamStack(); InitParamStack();
@ -42,9 +49,10 @@ public:
CBinaryWriter(IOutputStream *pStream, const CSerialVersion& rkVersion) CBinaryWriter(IOutputStream *pStream, const CSerialVersion& rkVersion)
: IArchive(false, true) : IArchive(false, true)
, mMagic(0)
, mOwnsStream(false) , mOwnsStream(false)
{ {
ASSERT(pStream->IsValid()); ASSERT(pStream && pStream->IsValid());
mpStream = pStream; mpStream = pStream;
SetVersion(rkVersion); SetVersion(rkVersion);
InitParamStack(); InitParamStack();
@ -58,17 +66,23 @@ public:
// Finish root param // Finish root param
ParamEnd(); ParamEnd();
// Delete stream // Write magic and delete stream
if (mOwnsStream) if (mOwnsStream)
{
mpStream->GoTo(0);
mpStream->WriteLong(mMagic);
delete mpStream; delete mpStream;
} }
}
inline bool IsValid() const { return mpStream->IsValid(); }
private: private:
void InitParamStack() void InitParamStack()
{ {
mParamStack.reserve(20); mParamStack.reserve(20);
mpStream->WriteLong(0xFFFFFFFF); mpStream->WriteLong(0xFFFFFFFF);
mpStream->WriteShort(0); // Size filler mpStream->WriteLong(0); // Size filler
mParamStack.push_back( SParameter { mpStream->Tell(), 0, false } ); mParamStack.push_back( SParameter { mpStream->Tell(), 0, false } );
} }
@ -80,12 +94,12 @@ public:
mParamStack.back().NumSubParams++; mParamStack.back().NumSubParams++;
if (mParamStack.back().NumSubParams == 1) if (mParamStack.back().NumSubParams == 1)
mpStream->WriteShort(-1); // Sub-param count filler mpStream->WriteLong(-1); // Sub-param count filler
// Write param metadata // Write param metadata
u32 ParamID = TString(pkName).Hash32(); u32 ParamID = TString(pkName).Hash32();
mpStream->WriteLong(ParamID); mpStream->WriteLong(ParamID);
mpStream->WriteShort((u16) 0xFFFF); // Param size filler mpStream->WriteLong(-1); // Param size filler
// Add new param to the stack // Add new param to the stack
mParamStack.push_back( SParameter { mpStream->Tell(), 0, false } ); mParamStack.push_back( SParameter { mpStream->Tell(), 0, false } );
@ -99,16 +113,16 @@ public:
SParameter& rParam = mParamStack.back(); SParameter& rParam = mParamStack.back();
u32 StartOffset = rParam.Offset; u32 StartOffset = rParam.Offset;
u32 EndOffset = mpStream->Tell(); u32 EndOffset = mpStream->Tell();
u16 ParamSize = (u16) (EndOffset - StartOffset); u32 ParamSize = (EndOffset - StartOffset);
mpStream->GoTo(StartOffset - 2); mpStream->GoTo(StartOffset - 4);
mpStream->WriteShort(ParamSize); mpStream->WriteLong(ParamSize);
// Write param child count // Write param child count
if (rParam.NumSubParams > 0 || mParamStack.size() == 1) if (rParam.NumSubParams > 0 || mParamStack.size() == 1)
{ {
if (rParam.Abstract) mpStream->Skip(4); if (rParam.Abstract) mpStream->Skip(4);
mpStream->WriteShort(rParam.NumSubParams); mpStream->WriteLong(rParam.NumSubParams);
} }
mpStream->GoTo(EndOffset); mpStream->GoTo(EndOffset);
@ -119,7 +133,7 @@ public:
{ {
// Normally handled by ParamBegin and ParamEnd but we need to do something here to account for zero-sized containers // Normally handled by ParamBegin and ParamEnd but we need to do something here to account for zero-sized containers
if (rSize == 0) if (rSize == 0)
mpStream->WriteShort(0); mpStream->WriteLong(0);
} }
virtual void SerializeAbstractObjectType(u32& rType) virtual void SerializeAbstractObjectType(u32& rType)

9
src/Common/Serialization/IArchive.h
View File

@ -178,7 +178,14 @@ protected:
bool mIsWriter; bool mIsWriter;
public: public:
static const u32 skCurrentArchiveVersion = 0; enum EArchiveVersion
{
eArVer_Initial,
eArVer_32BitBinarySize,
// Insert new versions before this line
eArVer_Max
};
static const u32 skCurrentArchiveVersion = (eArVer_Max - 1);
IArchive(bool IsReader, bool IsWriter) IArchive(bool IsReader, bool IsWriter)
: mFileVersion(0) : mFileVersion(0)

8
src/Core/GameProject/CGameExporter.cpp
View File

@ -542,11 +542,11 @@ void CGameExporter::ExportResourceEditorData()
// All resources should have dependencies generated, so save the project files // All resources should have dependencies generated, so save the project files
SCOPED_TIMER(SaveResourceDatabase); SCOPED_TIMER(SaveResourceDatabase);
#if EXPORT_COOKED #if EXPORT_COOKED
mpStore->SaveResourceDatabase(); bool ResDBSaveSuccess = mpStore->SaveDatabaseCache();
ASSERT(ResDBSaveSuccess);
#endif #endif
bool SaveSuccess = mpProject->Save(); bool ProjectSaveSuccess = mpProject->Save();
ASSERT(SaveSuccess); ASSERT(ProjectSaveSuccess);
mpStore->SaveCacheFile();
} }
} }

4
src/Core/GameProject/CGameProject.cpp
View File

@ -9,7 +9,7 @@ CGameProject::~CGameProject()
{ {
if (mpResourceStore) if (mpResourceStore)
{ {
ASSERT(!mpResourceStore->IsDirty()); ASSERT(!mpResourceStore->IsCacheDirty());
if (gpResourceStore == mpResourceStore) if (gpResourceStore == mpResourceStore)
gpResourceStore = nullptr; gpResourceStore = nullptr;
@ -238,7 +238,7 @@ CGameProject* CGameProject::LoadProject(const TString& rkProjPath, IProgressNoti
// Load resource database // Load resource database
pProgress->Report("Loading resource database"); pProgress->Report("Loading resource database");
pProj->mpResourceStore = new CResourceStore(pProj); pProj->mpResourceStore = new CResourceStore(pProj);
LoadSuccess = pProj->mpResourceStore->LoadResourceDatabase(); LoadSuccess = pProj->mpResourceStore->LoadDatabaseCache();
// Validate resource database // Validate resource database
if (LoadSuccess) if (LoadSuccess)

142
src/Core/GameProject/CResourceEntry.cpp
View File

@ -10,24 +10,70 @@
#include <Common/Serialization/CXMLReader.h> #include <Common/Serialization/CXMLReader.h>
#include <Common/Serialization/CXMLWriter.h> #include <Common/Serialization/CXMLWriter.h>
CResourceEntry::CResourceEntry(CResourceStore *pStore, const CAssetID& rkID, CResourceEntry::CResourceEntry(CResourceStore *pStore)
const TString& rkDir, const TString& rkFilename,
EResType Type)
: mpResource(nullptr) : mpResource(nullptr)
, mpTypeInfo(nullptr)
, mpStore(pStore) , mpStore(pStore)
, mpDependencies(nullptr) , mpDependencies(nullptr)
, mID(rkID) , mID( CAssetID::InvalidID(pStore->Game()) )
, mpDirectory(nullptr) , mpDirectory(nullptr)
, mName(rkFilename)
, mMetadataDirty(false) , mMetadataDirty(false)
, mCachedSize(-1) , mCachedSize(-1)
, mCachedUppercaseName(rkFilename.ToUpper()) {}
{
mpTypeInfo = CResTypeInfo::FindTypeInfo(Type);
ASSERT(mpTypeInfo);
mpDirectory = mpStore->GetVirtualDirectory(rkDir, true); // Static constructors
if (mpDirectory) mpDirectory->AddChild("", this); CResourceEntry* CResourceEntry::CreateNewResource(CResourceStore *pStore, const CAssetID& rkID,
const TString& rkDir, const TString& rkName,
EResType Type)
{
// Initialize all entry info with the input data.
CResourceEntry *pEntry = new CResourceEntry(pStore);
pEntry->mID = rkID;
pEntry->mName = rkName;
pEntry->mCachedUppercaseName = rkName.ToUpper();
pEntry->mpTypeInfo = CResTypeInfo::FindTypeInfo(Type);
ASSERT(pEntry->mpTypeInfo);
pEntry->mpDirectory = pStore->GetVirtualDirectory(rkDir, true);
ASSERT(pEntry->mpDirectory);
pEntry->mpDirectory->AddChild("", pEntry);
pEntry->mMetadataDirty = true;
return pEntry;
}
CResourceEntry* CResourceEntry::BuildFromArchive(CResourceStore *pStore, IArchive& rArc)
{
// Load all entry info from the archive.
CResourceEntry *pEntry = new CResourceEntry(pStore);
pEntry->SerializeEntryInfo(rArc, false);
ASSERT(pEntry->mpTypeInfo);
ASSERT(pEntry->mpDirectory);
return pEntry;
}
CResourceEntry* CResourceEntry::BuildFromDirectory(CResourceStore *pStore, CResTypeInfo *pTypeInfo,
const TString& rkDirPath, const TString& rkName)
{
// Initialize as much entry info as possible from the input data, then load the rest from the metadata file.
ASSERT(pTypeInfo);
CResourceEntry *pEntry = new CResourceEntry(pStore);
pEntry->mpTypeInfo = pTypeInfo;
pEntry->mName = rkName;
pEntry->mCachedUppercaseName = rkName.ToUpper();
pEntry->mpDirectory = pStore->GetVirtualDirectory(rkDirPath, true);
ASSERT(pEntry->mpDirectory);
pEntry->mpDirectory->AddChild("", pEntry);
// Make sure we're valid, then load the remaining data from the metadata file
ASSERT(pEntry->HasCookedVersion() || pEntry->HasRawVersion());
bool Success = pEntry->LoadMetadata();
ASSERT(Success);
return pEntry;
} }
CResourceEntry::~CResourceEntry() CResourceEntry::~CResourceEntry()
@ -39,25 +85,18 @@ CResourceEntry::~CResourceEntry()
bool CResourceEntry::LoadMetadata() bool CResourceEntry::LoadMetadata()
{ {
ASSERT(!mMetadataDirty); ASSERT(!mMetadataDirty);
TString Path = MetadataFilePath(); TString Path = MetadataFilePath();
CBinaryReader MetaFile(Path, FOURCC('META'));
if (FileUtil::Exists(Path)) if (MetaFile.IsValid())
{ {
// Validate file SerializeEntryInfo(MetaFile, true);
CFileInStream MetaFile(Path, IOUtil::eBigEndian);
u32 Magic = MetaFile.ReadLong();
if (Magic == FOURCC('META'))
{
CSerialVersion Version(MetaFile);
CBinaryReader Reader(&MetaFile, Version);
SerializeMetadata(Reader);
return true; return true;
} }
else else
{ {
Log::Error(Path + ": Failed to load metadata file, invalid magic: " + CFourCC(Magic).ToString()); Log::Error(Path + ": Failed to load metadata file!");
}
} }
return false; return false;
@ -71,24 +110,11 @@ bool CResourceEntry::SaveMetadata(bool ForceSave /*= false*/)
TString Dir = Path.GetFileDirectory(); TString Dir = Path.GetFileDirectory();
FileUtil::MakeDirectory(Dir); FileUtil::MakeDirectory(Dir);
CFileOutStream MetaFile(Path, IOUtil::eBigEndian); CBinaryWriter MetaFile(Path, FOURCC('META'), 0, Game());
if (MetaFile.IsValid()) if (MetaFile.IsValid())
{ {
MetaFile.WriteLong(0); // Magic dummy SerializeEntryInfo(MetaFile, true);
CSerialVersion Version(IArchive::skCurrentArchiveVersion, 0, Game());
Version.Write(MetaFile);
// Scope the binary writer to ensure it finishes before we go back to write the magic value
{
CBinaryWriter Writer(&MetaFile, Version);
SerializeMetadata(Writer);
}
MetaFile.GoTo(0);
MetaFile.WriteLong(FOURCC('META'));
mMetadataDirty = false; mMetadataDirty = false;
return true; return true;
} }
@ -97,30 +123,34 @@ bool CResourceEntry::SaveMetadata(bool ForceSave /*= false*/)
return false; return false;
} }
void CResourceEntry::SerializeMetadata(IArchive& rArc) void CResourceEntry::SerializeEntryInfo(IArchive& rArc, bool MetadataOnly)
{ {
// Serialize ID. If we already have a valid ID then don't allow the file to override it.
CAssetID ID = mID; CAssetID ID = mID;
rArc << SERIAL("AssetID", ID);
rArc << SERIAL("AssetID", ID)
<< SERIAL("Type", mpTypeInfo)
<< SERIAL("Flags", mFlags);
// Don't allow the file to override our asset ID if we already have a valid one.
if (rArc.IsReader() && !mID.IsValid()) if (rArc.IsReader() && !mID.IsValid())
mID = ID; mID = ID;
// Serialize type // Serialize extra data that we exclude from the metadata file
rArc << SERIAL("Type", mpTypeInfo); if (!MetadataOnly)
{
TString Dir = (mpDirectory ? mpDirectory->FullPath() : "");
// Serialize flags rArc << SERIAL("Name", mName)
u32 Flags = mFlags & eREF_SavedFlags; << SERIAL("Directory", Dir)
rArc << SERIAL_AUTO(Flags); << SERIAL_ABSTRACT("Dependencies", mpDependencies, &gDependencyNodeFactory);
if (rArc.IsReader()) mFlags = Flags & eREF_SavedFlags;
}
void CResourceEntry::SerializeCacheData(IArchive& rArc) if (rArc.IsReader())
{ {
// Note: If the dependency tree format is changed this should be adjusted so that mpDirectory = mpStore->GetVirtualDirectory(Dir, true);
// we regenerate the dependencies from scratch instead of reading the tree if the mpDirectory->AddChild("", this);
// file version number is too low mCachedUppercaseName = mName.ToUpper();
rArc << SERIAL_ABSTRACT("Dependencies", mpDependencies, &gDependencyNodeFactory); }
}
} }
void CResourceEntry::UpdateDependencies() void CResourceEntry::UpdateDependencies()
@ -150,7 +180,7 @@ void CResourceEntry::UpdateDependencies()
} }
mpDependencies = mpResource->BuildDependencyTree(); mpDependencies = mpResource->BuildDependencyTree();
mpStore->SetCacheDataDirty(); mpStore->SetCacheDirty();
if (!WasLoaded) if (!WasLoaded)
mpStore->DestroyUnreferencedResources(); mpStore->DestroyUnreferencedResources();
@ -550,7 +580,7 @@ bool CResourceEntry::Move(const TString& rkDir, const TString& rkName, bool IsAu
SetFlagEnabled(eREF_AutoResName, IsAutoGenName); SetFlagEnabled(eREF_AutoResName, IsAutoGenName);
} }
mpStore->SetDatabaseDirty(); mpStore->SetCacheDirty();
mCachedUppercaseName = rkName.ToUpper(); mCachedUppercaseName = rkName.ToUpper();
FileUtil::DeleteFile(OldRawPath); FileUtil::DeleteFile(OldRawPath);
FileUtil::DeleteFile(OldCookedPath); FileUtil::DeleteFile(OldCookedPath);

16
src/Core/GameProject/CResourceEntry.h
View File

@ -22,9 +22,6 @@ enum EResEntryFlag
eREF_HasBeenModified = 0x00000008, // Resource has been modified and resaved by the user eREF_HasBeenModified = 0x00000008, // Resource has been modified and resaved by the user
eREF_AutoResName = 0x00000010, // Resource name is auto-generated eREF_AutoResName = 0x00000010, // Resource name is auto-generated
eREF_AutoResDir = 0x00000020, // Resource directory name is auto-generated eREF_AutoResDir = 0x00000020, // Resource directory name is auto-generated
// Flags that save to the cache file
eREF_SavedFlags = eREF_NeedsRecook | eREF_IsBaseGameResource | eREF_Hidden | eREF_HasBeenModified |
eREF_AutoResName | eREF_AutoResDir
}; };
DECLARE_FLAGS(EResEntryFlag, FResEntryFlags) DECLARE_FLAGS(EResEntryFlag, FResEntryFlags)
@ -43,16 +40,21 @@ class CResourceEntry
mutable u64 mCachedSize; mutable u64 mCachedSize;
mutable TString mCachedUppercaseName; // This is used to speed up case-insensitive sorting and filtering. mutable TString mCachedUppercaseName; // This is used to speed up case-insensitive sorting and filtering.
// Private constructor
CResourceEntry(CResourceStore *pStore);
public: public:
CResourceEntry(CResourceStore *pStore, const CAssetID& rkID, static CResourceEntry* CreateNewResource(CResourceStore *pStore, const CAssetID& rkID,
const TString& rkDir, const TString& rkFilename, const TString& rkDir, const TString& rkName,
EResType Type); EResType Type);
static CResourceEntry* BuildFromArchive(CResourceStore *pStore, IArchive& rArc);
static CResourceEntry* BuildFromDirectory(CResourceStore *pStore, CResTypeInfo *pTypeInfo,
const TString& rkDirPath, const TString& rkName);
~CResourceEntry(); ~CResourceEntry();
bool LoadMetadata(); bool LoadMetadata();
bool SaveMetadata(bool ForceSave = false); bool SaveMetadata(bool ForceSave = false);
void SerializeMetadata(IArchive& rArc); void SerializeEntryInfo(IArchive& rArc, bool MetadataOnly);
void SerializeCacheData(IArchive& rArc);
void UpdateDependencies(); void UpdateDependencies();
bool HasRawVersion() const; bool HasRawVersion() const;

254
src/Core/GameProject/CResourceStore.cpp
View File

@ -2,6 +2,7 @@
#include "CGameExporter.h" #include "CGameExporter.h"
#include "CGameProject.h" #include "CGameProject.h"
#include "CResourceIterator.h" #include "CResourceIterator.h"
#include "Core/IUIRelay.h"
#include "Core/Resource/CResource.h" #include "Core/Resource/CResource.h"
#include <Common/AssertMacro.h> #include <Common/AssertMacro.h>
#include <Common/FileUtil.h> #include <Common/FileUtil.h>
@ -17,13 +18,12 @@ CResourceStore *gpEditorStore = nullptr;
// Constructor for editor store // Constructor for editor store
CResourceStore::CResourceStore(const TString& rkDatabasePath) CResourceStore::CResourceStore(const TString& rkDatabasePath)
: mpProj(nullptr) : mpProj(nullptr)
, mGame(eUnknownGame) , mGame(ePrime)
, mDatabaseDirty(false) , mDatabaseCacheDirty(false)
, mCacheFileDirty(false)
{ {
mpDatabaseRoot = new CVirtualDirectory(this); mpDatabaseRoot = new CVirtualDirectory(this);
mDatabasePath = FileUtil::MakeAbsolute(rkDatabasePath.GetFileDirectory()); mDatabasePath = FileUtil::MakeAbsolute(rkDatabasePath.GetFileDirectory());
mDatabaseName = rkDatabasePath.GetFileName(); LoadDatabaseCache();
} }
// Main constructor for game projects and game exporter // Main constructor for game projects and game exporter
@ -31,8 +31,7 @@ CResourceStore::CResourceStore(CGameProject *pProject)
: mpProj(nullptr) : mpProj(nullptr)
, mGame(eUnknownGame) , mGame(eUnknownGame)
, mpDatabaseRoot(nullptr) , mpDatabaseRoot(nullptr)
, mDatabaseDirty(false) , mDatabaseCacheDirty(false)
, mCacheFileDirty(false)
{ {
SetProject(pProject); SetProject(pProject);
} }
@ -60,58 +59,60 @@ void RecursiveGetListOfEmptyDirectories(CVirtualDirectory *pDir, TStringList& rO
} }
} }
bool CResourceStore::SerializeResourceDatabase(IArchive& rArc) bool CResourceStore::SerializeDatabaseCache(IArchive& rArc)
{ {
struct SDatabaseResource // Serialize resources
if (rArc.ParamBegin("Resources"))
{ {
CAssetID ID; // Serialize resources
CResTypeInfo *pType; u32 ResourceCount = mResourceEntries.size();
TString Directory; rArc << SERIAL_AUTO(ResourceCount);
TString Name;
void Serialize(IArchive& rArc) if (rArc.IsReader())
{ {
rArc << SERIAL_AUTO(ID) << SERIAL("Type", pType) << SERIAL_AUTO(Directory) << SERIAL_AUTO(Name); for (u32 ResIdx = 0; ResIdx < ResourceCount; ResIdx++)
{
if (rArc.ParamBegin("Resource"))
{
CResourceEntry *pEntry = CResourceEntry::BuildFromArchive(this, rArc);
ASSERT( FindEntry(pEntry->ID()) == nullptr );
mResourceEntries[pEntry->ID()] = pEntry;
rArc.ParamEnd();
} }
}; }
std::vector<SDatabaseResource> Resources; }
else
// Populate resource list
if (!rArc.IsReader())
{ {
Resources.reserve(mResourceEntries.size());
for (CResourceIterator It(this); It; ++It) for (CResourceIterator It(this); It; ++It)
Resources.push_back( SDatabaseResource { It->ID(), It->TypeInfo(), It->Directory()->FullPath(), It->Name() } ); {
if (rArc.ParamBegin("Resource"))
{
It->SerializeEntryInfo(rArc, false);
rArc.ParamEnd();
}
}
}
rArc.ParamEnd();
} }
// Populate directory list // Serialize empty directory list
TStringList EmptyDirectories; TStringList EmptyDirectories;
if (!rArc.IsReader()) if (!rArc.IsReader())
RecursiveGetListOfEmptyDirectories(mpDatabaseRoot, EmptyDirectories); RecursiveGetListOfEmptyDirectories(mpDatabaseRoot, EmptyDirectories);
// Serialize rArc << SERIAL_CONTAINER_AUTO(EmptyDirectories, "Directory");
rArc << SERIAL_CONTAINER_AUTO(Resources, "Resource")
<< SERIAL_CONTAINER_AUTO(EmptyDirectories, "Directory");
// Register resources
if (rArc.IsReader()) if (rArc.IsReader())
{ {
for (auto Iter = EmptyDirectories.begin(); Iter != EmptyDirectories.end(); Iter++) for (auto Iter = EmptyDirectories.begin(); Iter != EmptyDirectories.end(); Iter++)
CreateVirtualDirectory(*Iter); CreateVirtualDirectory(*Iter);
for (auto Iter = Resources.begin(); Iter != Resources.end(); Iter++)
{
SDatabaseResource& rRes = *Iter;
RegisterResource(rRes.ID, rRes.pType->Type(), rRes.Directory, rRes.Name);
}
} }
return true; return true;
} }
bool CResourceStore::LoadResourceDatabase() bool CResourceStore::LoadDatabaseCache()
{ {
ASSERT(!mDatabasePath.IsEmpty()); ASSERT(!mDatabasePath.IsEmpty());
TString Path = DatabasePath(); TString Path = DatabasePath();
@ -119,139 +120,46 @@ bool CResourceStore::LoadResourceDatabase()
if (!mpDatabaseRoot) if (!mpDatabaseRoot)
mpDatabaseRoot = new CVirtualDirectory(this); mpDatabaseRoot = new CVirtualDirectory(this);
CXMLReader Reader(Path); // Load the resource database
CBinaryReader Reader(Path, FOURCC('CACH'));
if (!Reader.IsValid()) if (!Reader.IsValid() || !SerializeDatabaseCache(Reader))
{ {
Log::Error("Failed to open resource database for load: " + Path); if (gpUIRelay->AskYesNoQuestion("Error", "Failed to load the resource database. Attempt to build from the directory? (This may take a while.)"))
{
if (!BuildFromDirectory(true))
return false; return false;
} }
else return false;
}
else
{
// Database is succesfully loaded at this point
if (mpProj) if (mpProj)
ASSERT(mpProj->Game() == Reader.Game()); ASSERT(mpProj->Game() == Reader.Game());
}
mGame = Reader.Game(); mGame = Reader.Game();
if (!SerializeResourceDatabase(Reader)) return false;
return LoadCacheFile();
}
bool CResourceStore::SaveResourceDatabase()
{
TString Path = DatabasePath();
CXMLWriter Writer(Path, "ResourceDB", 0, mGame);
SerializeResourceDatabase(Writer);
bool SaveSuccess = Writer.Save();
if (SaveSuccess)
mDatabaseDirty = false;
else
Log::Error("Failed to save resource database: " + Path);
return SaveSuccess;
}
bool CResourceStore::LoadCacheFile()
{
TString CachePath = CacheDataPath();
CFileInStream CacheFile(CachePath, IOUtil::eBigEndian);
if (!CacheFile.IsValid())
{
Log::Error("Failed to open cache file for load: " + CachePath);
return false;
}
// Cache header
CFourCC Magic(CacheFile);
if (Magic != FOURCC('CACH'))
{
Log::Error("Invalid resource cache data magic: " + Magic.ToString());
return false;
}
CSerialVersion Version(CacheFile);
u32 NumResources = CacheFile.ReadLong();
for (u32 iRes = 0; iRes < NumResources; iRes++)
{
CAssetID ID(CacheFile, Version.Game());
u32 EntryCacheSize = CacheFile.ReadLong();
u32 EntryCacheEnd = CacheFile.Tell() + EntryCacheSize;
CResourceEntry *pEntry = FindEntry(ID);
if (pEntry)
{
CBasicBinaryReader Reader(&CacheFile, Version);
if (Reader.ParamBegin("EntryCache"))
{
pEntry->SerializeCacheData(Reader);
Reader.ParamEnd();
}
}
CacheFile.Seek(EntryCacheEnd, SEEK_SET);
}
return true; return true;
} }
bool CResourceStore::SaveCacheFile() bool CResourceStore::SaveDatabaseCache()
{ {
TString CachePath = CacheDataPath(); TString Path = DatabasePath();
CFileOutStream CacheFile(CachePath, IOUtil::eBigEndian);
if (!CacheFile.IsValid()) CBinaryWriter Writer(Path, FOURCC('CACH'), 0, mGame);
{
Log::Error("Failed to open cache file for save: " + CachePath); if (!Writer.IsValid())
return false; return false;
}
// Cache header SerializeDatabaseCache(Writer);
CacheFile.WriteLong(0); // Magic dummy. Magic isn't written until the rest of the file is saved successfully. mDatabaseCacheDirty = false;
CSerialVersion Version(IArchive::skCurrentArchiveVersion, 0, mGame);
Version.Write(CacheFile);
u32 ResCountOffset = CacheFile.Tell();
u32 ResCount = 0;
CacheFile.WriteLong(0); // Resource count dummy - fill in when we know the real count
// Save entry cache data
// Structure: Entry Asset ID -> Entry Cache Size -> Serialized Entry Cache Data
for (CResourceIterator It(this); It; ++It)
{
ResCount++;
It->ID().Write(CacheFile);
u32 SizeOffset = CacheFile.Tell();
CacheFile.WriteLong(0);
CBasicBinaryWriter Writer(&CacheFile, Version.FileVersion(), Version.Game());
if (Writer.ParamBegin("EntryCache"))
{
It->SerializeCacheData(Writer);
Writer.ParamEnd();
}
u32 EntryCacheEnd = CacheFile.Tell();
CacheFile.Seek(SizeOffset, SEEK_SET);
CacheFile.WriteLong(EntryCacheEnd - SizeOffset - 4);
CacheFile.Seek(EntryCacheEnd, SEEK_SET);
}
CacheFile.Seek(ResCountOffset, SEEK_SET);
CacheFile.WriteLong(ResCount);
CacheFile.Seek(0, SEEK_SET);
CacheFile.WriteLong( FOURCC('CACH') );
mCacheFileDirty = false;
return true; return true;
} }
void CResourceStore::ConditionalSaveStore() void CResourceStore::ConditionalSaveStore()
{ {
if (mDatabaseDirty) SaveResourceDatabase(); if (mDatabaseCacheDirty) SaveDatabaseCache();
if (mCacheFileDirty) SaveCacheFile();
} }
void CResourceStore::SetProject(CGameProject *pProj) void CResourceStore::SetProject(CGameProject *pProj)
@ -267,7 +175,6 @@ void CResourceStore::SetProject(CGameProject *pProj)
{ {
TString DatabasePath = mpProj->ResourceDBPath(false); TString DatabasePath = mpProj->ResourceDBPath(false);
mDatabasePath = DatabasePath.GetFileDirectory(); mDatabasePath = DatabasePath.GetFileDirectory();
mDatabaseName = DatabasePath.GetFileName();
mpDatabaseRoot = new CVirtualDirectory(this); mpDatabaseRoot = new CVirtualDirectory(this);
mGame = mpProj->Game(); mGame = mpProj->Game();
} }
@ -377,13 +284,11 @@ void CResourceStore::ClearDatabase()
delete mpDatabaseRoot; delete mpDatabaseRoot;
mpDatabaseRoot = new CVirtualDirectory(this); mpDatabaseRoot = new CVirtualDirectory(this);
mDatabaseDirty = true; mDatabaseCacheDirty = true;
mCacheFileDirty = true;
} }
void CResourceStore::BuildFromDirectory() bool CResourceStore::BuildFromDirectory(bool ShouldGenerateCacheFile)
{ {
ASSERT(mpProj != nullptr);
ASSERT(mResourceEntries.empty()); ASSERT(mResourceEntries.empty());
// Get list of resources // Get list of resources
@ -415,8 +320,7 @@ void CResourceStore::BuildFromDirectory()
} }
// Create resource entry // Create resource entry
CResourceEntry *pEntry = new CResourceEntry(this, CAssetID::InvalidID(mGame), DirPath, ResName, pTypeInfo->Type()); CResourceEntry *pEntry = CResourceEntry::BuildFromDirectory(this, pTypeInfo, DirPath, ResName);
pEntry->LoadMetadata();
// Validate the entry // Validate the entry
CAssetID ID = pEntry->ID(); CAssetID ID = pEntry->ID();
@ -429,6 +333,31 @@ void CResourceStore::BuildFromDirectory()
else if (FileUtil::IsDirectory(Path)) else if (FileUtil::IsDirectory(Path))
CreateVirtualDirectory(RelPath); CreateVirtualDirectory(RelPath);
} }
// Generate new cache file
if (ShouldGenerateCacheFile)
{
// Make sure gpResourceStore points to this store
CResourceStore *pOldStore = gpResourceStore;
gpResourceStore = this;
// Make sure audio manager is loaded correctly so AGSC dependencies can be looked up
if (mpProj)
mpProj->AudioManager()->LoadAssets();
// Update dependencies
for (CResourceIterator It(this); It; ++It)
It->UpdateDependencies();
// Update database file
mDatabaseCacheDirty = true;
ConditionalSaveStore();
// Restore old gpResourceStore
gpResourceStore = pOldStore;
}
return true;
} }
void CResourceStore::RebuildFromDirectory() void CResourceStore::RebuildFromDirectory()
@ -436,19 +365,7 @@ void CResourceStore::RebuildFromDirectory()
ASSERT(mpProj != nullptr); ASSERT(mpProj != nullptr);
mpProj->AudioManager()->ClearAssets(); mpProj->AudioManager()->ClearAssets();
ClearDatabase(); ClearDatabase();
BuildFromDirectory(); BuildFromDirectory(true);
// Make sure audio manager is loaded correctly so AGSC dependencies can be looked up
mpProj->AudioManager()->LoadAssets();
// Update dependencies
for (CResourceIterator It(this); It; ++It)
It->UpdateDependencies();
// Update database files
mDatabaseDirty = true;
mCacheFileDirty = true;
ConditionalSaveStore();
} }
bool CResourceStore::IsResourceRegistered(const CAssetID& rkID) const bool CResourceStore::IsResourceRegistered(const CAssetID& rkID) const
@ -468,8 +385,7 @@ CResourceEntry* CResourceStore::RegisterResource(const CAssetID& rkID, EResType
// Validate directory // Validate directory
if (IsValidResourcePath(rkDir, rkName)) if (IsValidResourcePath(rkDir, rkName))
{ {
pEntry = new CResourceEntry(this, rkID, rkDir, rkName, Type); pEntry = CResourceEntry::CreateNewResource(this, rkID, rkDir, rkName, Type);
pEntry->LoadMetadata();
mResourceEntries[rkID] = pEntry; mResourceEntries[rkID] = pEntry;
} }

22
src/Core/GameProject/CResourceStore.h
View File

@ -24,12 +24,10 @@ class CResourceStore
CVirtualDirectory *mpDatabaseRoot; CVirtualDirectory *mpDatabaseRoot;
std::map<CAssetID, CResourceEntry*> mResourceEntries; std::map<CAssetID, CResourceEntry*> mResourceEntries;
std::map<CAssetID, CResourceEntry*> mLoadedResources; std::map<CAssetID, CResourceEntry*> mLoadedResources;
bool mDatabaseDirty; bool mDatabaseCacheDirty;
bool mCacheFileDirty;
// Directory paths // Directory paths
TString mDatabasePath; TString mDatabasePath;
TString mDatabaseName;
enum EDatabaseVersion enum EDatabaseVersion
{ {
@ -43,11 +41,9 @@ public:
CResourceStore(const TString& rkDatabasePath); CResourceStore(const TString& rkDatabasePath);
CResourceStore(CGameProject *pProject); CResourceStore(CGameProject *pProject);
~CResourceStore(); ~CResourceStore();
bool SerializeResourceDatabase(IArchive& rArc); bool SerializeDatabaseCache(IArchive& rArc);
bool LoadResourceDatabase(); bool LoadDatabaseCache();
bool SaveResourceDatabase(); bool SaveDatabaseCache();
bool LoadCacheFile();
bool SaveCacheFile();
void ConditionalSaveStore(); void ConditionalSaveStore();
void SetProject(CGameProject *pProj); void SetProject(CGameProject *pProj);
void CloseProject(); void CloseProject();
@ -61,7 +57,7 @@ public:
CResourceEntry* FindEntry(const TString& rkPath) const; CResourceEntry* FindEntry(const TString& rkPath) const;
bool AreAllEntriesValid() const; bool AreAllEntriesValid() const;
void ClearDatabase(); void ClearDatabase();
void BuildFromDirectory(); bool BuildFromDirectory(bool ShouldGenerateCacheFile);
void RebuildFromDirectory(); void RebuildFromDirectory();
template<typename ResType> ResType* LoadResource(const CAssetID& rkID) { return static_cast<ResType*>(LoadResource(rkID, ResType::StaticType())); } template<typename ResType> ResType* LoadResource(const CAssetID& rkID) { return static_cast<ResType*>(LoadResource(rkID, ResType::StaticType())); }
@ -81,15 +77,13 @@ public:
inline EGame Game() const { return mGame; } inline EGame Game() const { return mGame; }
inline TString DatabaseRootPath() const { return mDatabasePath; } inline TString DatabaseRootPath() const { return mDatabasePath; }
inline TString ResourcesDir() const { return IsEditorStore() ? DatabaseRootPath() : DatabaseRootPath() + "Resources/"; } inline TString ResourcesDir() const { return IsEditorStore() ? DatabaseRootPath() : DatabaseRootPath() + "Resources/"; }
inline TString DatabasePath() const { return DatabaseRootPath() + "ResourceDatabase.xml"; } inline TString DatabasePath() const { return DatabaseRootPath() + "ResourceDatabaseCache.bin"; }
inline TString CacheDataPath() const { return DatabaseRootPath() + "ResourceCacheData.bin"; }
inline CVirtualDirectory* RootDirectory() const { return mpDatabaseRoot; } inline CVirtualDirectory* RootDirectory() const { return mpDatabaseRoot; }
inline u32 NumTotalResources() const { return mResourceEntries.size(); } inline u32 NumTotalResources() const { return mResourceEntries.size(); }
inline u32 NumLoadedResources() const { return mLoadedResources.size(); } inline u32 NumLoadedResources() const { return mLoadedResources.size(); }
inline bool IsDirty() const { return mDatabaseDirty || mCacheFileDirty; } inline bool IsCacheDirty() const { return mDatabaseCacheDirty; }
inline void SetDatabaseDirty() { mDatabaseDirty = true; } inline void SetCacheDirty() { mDatabaseCacheDirty = true; }
inline void SetCacheDataDirty() { mCacheFileDirty = true; }
inline bool IsEditorStore() const { return mpProj == nullptr; } inline bool IsEditorStore() const { return mpProj == nullptr; }
}; };

2
src/Editor/CEditorApplication.cpp
View File

@ -253,7 +253,7 @@ void CEditorApplication::TickEditors()
double DeltaTime = mLastUpdate - LastUpdate; double DeltaTime = mLastUpdate - LastUpdate;
// The resource store should NOT be dirty at the beginning of a tick - this indicates we forgot to save it after updating somewhere // The resource store should NOT be dirty at the beginning of a tick - this indicates we forgot to save it after updating somewhere
if (gpResourceStore && gpResourceStore->IsDirty()) if (gpResourceStore && gpResourceStore->IsCacheDirty())
{ {
Log::Error("Resource store is dirty at the beginning of a tick! Call ConditionalSaveStore() after making any significant changes to assets!"); Log::Error("Resource store is dirty at the beginning of a tick! Call ConditionalSaveStore() after making any significant changes to assets!");
DEBUG_BREAK; DEBUG_BREAK;

10
src/Editor/CUIRelay.h
View File

@ -6,10 +6,18 @@
#include "WorldEditor/CWorldEditor.h" #include "WorldEditor/CWorldEditor.h"
#include "UICommon.h" #include "UICommon.h"
#include <QThread>
class CUIRelay : public QObject, public IUIRelay class CUIRelay : public QObject, public IUIRelay
{ {
Q_OBJECT Q_OBJECT
Qt::ConnectionType GetConnectionType()
{
bool IsUIThread = (QThread::currentThread() == gpEdApp->thread());
return IsUIThread ? Qt::DirectConnection : Qt::BlockingQueuedConnection;
}
public: public:
explicit CUIRelay(QObject *pParent = 0) explicit CUIRelay(QObject *pParent = 0)
: QObject(pParent) : QObject(pParent)
@ -20,7 +28,7 @@ public:
virtual bool AskYesNoQuestion(const TString& rkInfoBoxTitle, const TString& rkQuestion) virtual bool AskYesNoQuestion(const TString& rkInfoBoxTitle, const TString& rkQuestion)
{ {
bool RetVal; bool RetVal;
QMetaObject::invokeMethod(this, "AskYesNoQuestionSlot", Qt::BlockingQueuedConnection, QMetaObject::invokeMethod(this, "AskYesNoQuestionSlot", GetConnectionType(),
Q_RETURN_ARG(bool, RetVal), Q_RETURN_ARG(bool, RetVal),
Q_ARG(QString, TO_QSTRING(rkInfoBoxTitle)), Q_ARG(QString, TO_QSTRING(rkInfoBoxTitle)),
Q_ARG(QString, TO_QSTRING(rkQuestion)) ); Q_ARG(QString, TO_QSTRING(rkQuestion)) );

23
src/Editor/main.cpp
View File

@ -32,22 +32,10 @@ int main(int argc, char *argv[])
App.setOrganizationName("Aruki"); App.setOrganizationName("Aruki");
App.setWindowIcon(QIcon(":/icons/AppIcon.ico")); App.setWindowIcon(QIcon(":/icons/AppIcon.ico"));
// Init log
bool Initialized = Log::InitLog("primeworldeditor.log");
if (!Initialized) QMessageBox::warning(0, "Error", "Couldn't open log file. Logging will not work for this session.");
qInstallMessageHandler(QtLogRedirect);
// Create UI relay // Create UI relay
CUIRelay UIRelay(&App); CUIRelay UIRelay(&App);
gpUIRelay = &UIRelay; gpUIRelay = &UIRelay;
// Create editor resource store
gpEditorStore = new CResourceStore("../resources/EditorResourceDB.rdb");
gpEditorStore->LoadResourceDatabase();
// Load templates
CTemplateLoader::LoadGameList();
// Set up dark theme // Set up dark theme
qApp->setStyle(QStyleFactory::create("Fusion")); qApp->setStyle(QStyleFactory::create("Fusion"));
QPalette DarkPalette; QPalette DarkPalette;
@ -66,6 +54,17 @@ int main(int argc, char *argv[])
DarkPalette.setColor(QPalette::HighlightedText, Qt::white); DarkPalette.setColor(QPalette::HighlightedText, Qt::white);
qApp->setPalette(DarkPalette); qApp->setPalette(DarkPalette);
// Init log
bool Initialized = Log::InitLog("primeworldeditor.log");
if (!Initialized) QMessageBox::warning(0, "Error", "Couldn't open log file. Logging will not work for this session.");
qInstallMessageHandler(QtLogRedirect);
// Create editor resource store
gpEditorStore = new CResourceStore("../resources/");
// Load templates
CTemplateLoader::LoadGameList();
// Execute application // Execute application
App.InitEditor(); App.InitEditor();
return App.exec(); return App.exec();