Initial implementation of world/area exports, added support for asset lists to specify a path/name for resources

src/Core/Resource/CResCache.cpp

@@ -14,12 +14,14 @@
 #include "Core/Resource/Factory/CWorldLoader.h"
 
 #include <FileIO/FileIO.h>
+#include <Common/AssertMacro.h>
 #include <Common/FileUtil.h>
 #include <Common/Log.h>
 #include <Common/TString.h>
 #include <iostream>
 
 CResCache::CResCache()
+    : mpGameExporter(nullptr)
 {
 }
 
@@ -71,8 +73,33 @@ TString CResCache::GetSourcePath()
 CResource* CResCache::GetResource(CUniqueID ResID, CFourCC Type)
 {
     if (!ResID.IsValid()) return nullptr;
-    TString Source = mResDir + ResID.ToString() + "." + Type.ToString();
-    return GetResource(Source);
+    TString StringName = ResID.ToString() + "." + Type.ToString();
+
+    // With Game Exporter - get data buffer from exporter
+    if (mpGameExporter)
+    {
+        // Check if we already have resource loaded
+        auto Got = mResourceCache.find(ResID.ToLongLong());
+        if (Got != mResourceCache.end())
+            return Got->second;
+
+        // Otherwise load resource
+        std::vector<u8> DataBuffer;
+        mpGameExporter->LoadResource(ResID, DataBuffer);
+        if (DataBuffer.empty()) return nullptr;
+
+        CMemoryInStream MemStream(DataBuffer.data(), DataBuffer.size(), IOUtil::eBigEndian);
+        CResource *pRes = InternalLoadResource(MemStream, ResID, Type);
+        pRes->mResSource = StringName;
+        return pRes;
+    }
+
+    // Without Game Exporter - load from file
+    else
+    {
+        TString Source = mResDir + StringName;
+        return GetResource(Source);
+    }
 }
 
 CResource* CResCache::GetResource(const TString& rkResPath)
@@ -98,32 +125,11 @@ CResource* CResCache::GetResource(const TString& rkResPath)
     mResDir = rkResPath.GetFileDirectory();
 
     // Load resource
-    CResource *pRes = nullptr;
     CFourCC Type = rkResPath.GetFileExtension().ToUpper();
-    bool SupportedFormat = true;
-
-    if      (Type == "CMDL") pRes = CModelLoader::LoadCMDL(File);
-    else if (Type == "TXTR") pRes = CTextureDecoder::LoadTXTR(File);
-    else if (Type == "ANCS") pRes = CAnimSetLoader::LoadANCS(File);
-    else if (Type == "CHAR") pRes = CAnimSetLoader::LoadCHAR(File);
-    else if (Type == "MREA") pRes = CAreaLoader::LoadMREA(File);
-    else if (Type == "MLVL") pRes = CWorldLoader::LoadMLVL(File);
-    else if (Type == "STRG") pRes = CStringLoader::LoadSTRG(File);
-    else if (Type == "FONT") pRes = CFontLoader::LoadFONT(File);
-    else if (Type == "SCAN") pRes = CScanLoader::LoadSCAN(File);
-    else if (Type == "DCLN") pRes = CCollisionLoader::LoadDCLN(File);
-    else if (Type == "EGMC") pRes = CPoiToWorldLoader::LoadEGMC(File);
-    else if (Type == "CINF") pRes = CSkeletonLoader::LoadCINF(File);
-    else if (Type == "ANIM") pRes = CAnimationLoader::LoadANIM(File);
-    else if (Type == "CSKR") pRes = CSkinLoader::LoadCSKR(File);
-    else SupportedFormat = false;
-
-    if (!pRes) pRes = new CResource(); // Default for unsupported formats
+    CResource *pRes = InternalLoadResource(File, ResID, Type);
+    pRes->mResSource = rkResPath;
 
     // Add to cache and cleanup
-    pRes->mID = *rkResPath;
-    pRes->mResSource = rkResPath;
-    mResourceCache[ResID.ToLongLong()] = pRes;
     mResDir = OldResDir;
     return pRes;
 }
@@ -168,4 +174,36 @@ void CResCache::DeleteResource(CUniqueID ResID)
     }
 }
 
+// ************ PROTECTED ************
+CResource* CResCache::InternalLoadResource(IInputStream& rInput, const CUniqueID& rkID, CFourCC Type)
+{
+    // todo - need some sort of auto-registration of loaders to avoid this if-else mess
+    ASSERT(mResourceCache.find(rkID.ToLongLong()) == mResourceCache.end()); // this test should be done before calling this func!
+    CResource *pRes = nullptr;
+
+    // Load resource
+    if      (Type == "CMDL") pRes = CModelLoader::LoadCMDL(rInput);
+    else if (Type == "TXTR") pRes = CTextureDecoder::LoadTXTR(rInput);
+    else if (Type == "ANCS") pRes = CAnimSetLoader::LoadANCS(rInput);
+    else if (Type == "CHAR") pRes = CAnimSetLoader::LoadCHAR(rInput);
+    else if (Type == "MREA") pRes = CAreaLoader::LoadMREA(rInput);
+    else if (Type == "MLVL") pRes = CWorldLoader::LoadMLVL(rInput);
+    else if (Type == "STRG") pRes = CStringLoader::LoadSTRG(rInput);
+    else if (Type == "FONT") pRes = CFontLoader::LoadFONT(rInput);
+    else if (Type == "SCAN") pRes = CScanLoader::LoadSCAN(rInput);
+    else if (Type == "DCLN") pRes = CCollisionLoader::LoadDCLN(rInput);
+    else if (Type == "EGMC") pRes = CPoiToWorldLoader::LoadEGMC(rInput);
+    else if (Type == "CINF") pRes = CSkeletonLoader::LoadCINF(rInput);
+    else if (Type == "ANIM") pRes = CAnimationLoader::LoadANIM(rInput);
+    else if (Type == "CSKR") pRes = CSkinLoader::LoadCSKR(rInput);
+    if (!pRes) pRes = new CResource(); // Default for unsupported formats
+
+    ASSERT(pRes->mRefCount == 0);
+
+    // Cache and return
+    pRes->mID = rkID;
+    mResourceCache[rkID.ToLongLong()] = pRes;
+    return pRes;
+}
+
 CResCache gResCache;

src/Core/Resource/CResCache.h

@@ -2,6 +2,7 @@
 #define CRESCACHE_H
 
 #include "CResource.h"
+#include "Core/GameProject/CGameExporter.h"
 #include <Common/types.h>
 #include <Common/TString.h>
 #include <unordered_map>
@@ -10,6 +11,7 @@ class CResCache
 {
     std::unordered_map<u64, CResource*> mResourceCache;
     TString mResDir;
+    CGameExporter *mpGameExporter;
 
 public:
     CResCache();
@@ -22,6 +24,11 @@ public:
     CFourCC FindResourceType(CUniqueID ResID, const TStringList& rkPossibleTypes);
     void CacheResource(CResource *pRes);
     void DeleteResource(CUniqueID ResID);
+
+    inline void SetGameExporter(CGameExporter *pExporter)   { mpGameExporter = pExporter; }
+
+protected:
+    CResource* InternalLoadResource(IInputStream& rInput, const CUniqueID& rkID, CFourCC Type);
 };
 
 extern CResCache gResCache;

src/Core/Resource/CResource.cpp

@@ -120,7 +120,7 @@ REGISTER_RESOURCE_TYPE(MREA, eArea, ePrimeDemo, eReturns)
 REGISTER_RESOURCE_TYPE(NTWK, eTweak, eEchoesDemo, eReturns)
 REGISTER_RESOURCE_TYPE(PAK , ePackage, ePrimeDemo, eReturns)
 REGISTER_RESOURCE_TYPE(PART, eParticle, ePrimeDemo, eReturns)
-REGISTER_RESOURCE_TYPE(PATH, eNavMesh, ePrimeDemo, eCorruption)
+REGISTER_RESOURCE_TYPE(PATH, ePathfinding, ePrimeDemo, eCorruption)
 REGISTER_RESOURCE_TYPE(PTLA, ePortalArea, eEchoesDemo, eCorruption)
 REGISTER_RESOURCE_TYPE(RULE, eRuleSet, eEchoesDemo, eReturns)
 REGISTER_RESOURCE_TYPE(SAND, eSourceAnimData, eCorruptionProto, eCorruption)

src/Core/Resource/EResType.h

@@ -14,7 +14,6 @@ enum EResType
     eAudioMacro,
     eAudioGroupSet,
     eAudioSample,
-    eStreamedAudio,
     eAudioLookupTable,
     eBinaryData,
     eBurstFireData,
@@ -25,14 +24,12 @@ enum EResType
     eGuiFrame,
     eGuiKeyFrame,
     eHintSystem,
-    eInvalidResType,
     eMapArea,
     eMapWorld,
     eMapUniverse,
     eMidi,
     eModel,
     eMusicTrack,
-    eNavMesh,
     ePackage,
     eParticle,
     eParticleCollisionResponse,
@@ -43,6 +40,7 @@ enum EResType
     eParticleSwoosh,
     eParticleTransform,
     eParticleWeapon,
+    ePathfinding,
     ePortalArea,
     eResource,
     eRuleSet,
@@ -56,6 +54,7 @@ enum EResType
     eStateMachine,
     eStateMachine2, // For distinguishing AFSM/FSM2
     eStaticGeometryMap,
+    eStreamedAudio,
     eStringList,
     eStringTable,
     eTexture,
@@ -63,7 +62,9 @@ enum EResType
     eUnknown_CAAD,
     eUserEvaluatorData,
     eVideo,
-    eWorld
+    eWorld,
+
+    eInvalidResType = -1
 };
 
 // defined in CResource.cpp

src/Core/Resource/Factory/CTextureDecoder.cpp

@@ -201,6 +201,10 @@ void CTextureDecoder::ReadDDS(IInputStream& rDDS)
 // ************ DECODE ************
 void CTextureDecoder::PartialDecodeGXTexture(IInputStream& TXTR)
 {
+    // TODO: This function doesn't handle very small mipmaps correctly.
+    // The format applies padding when the size of a mipmap is less than the block size for that format.
+    // The decode needs to be adjusted to account for the padding and skip over it (since we don't have padding in OpenGL).
+
     // Get image data size, create output buffer
     u32 ImageStart = TXTR.Tell();
     TXTR.Seek(0x0, SEEK_END);
@@ -233,11 +237,22 @@ void CTextureDecoder::PartialDecodeGXTexture(IInputStream& TXTR)
         MipH /= 4;
     }
 
+    // This value set to true if we hit the end of the file earlier than expected.
+    // This is necessary due to a mistake Retro made in their cooker for I8 textures where very small mipmaps are cut off early, resulting in an out-of-bounds memory access.
+    // This affects one texture that I know of - Echoes 3bb2c034.TXTR
+    bool BreakEarly = false;
+
     for (u32 iMip = 0; iMip < mNumMipMaps; iMip++)
     {
+        if (MipW < BWidth) MipW = BWidth;
+        if (MipH < BHeight) MipH = BHeight;
+
         for (u32 iBlockY = 0; iBlockY < MipH; iBlockY += BHeight)
-            for (u32 iBlockX = 0; iBlockX < MipW; iBlockX += BWidth) {
-                for (u32 iImgY = iBlockY; iImgY < iBlockY + BHeight; iImgY++) {
+        {
+            for (u32 iBlockX = 0; iBlockX < MipW; iBlockX += BWidth)
+            {
+                for (u32 iImgY = iBlockY; iImgY < iBlockY + BHeight; iImgY++)
+                {
                     for (u32 iImgX = iBlockX; iImgX < iBlockX + BWidth; iImgX++)
                     {
                         u32 DstPos = ((iImgY * MipW) + iImgX) * PixelStride;
@@ -256,10 +271,17 @@ void CTextureDecoder::PartialDecodeGXTexture(IInputStream& TXTR)
 
                         // I4 and C4 have 4bpp images, so I'm forced to read two pixels at a time.
                         if ((mTexelFormat == eGX_I4) || (mTexelFormat == eGX_C4)) iImgX++;
+
+                        // Check if we're at the end of the file.
+                        if (TXTR.EoF()) BreakEarly = true;
                     }
+                    if (BreakEarly) break;
                 }
                 if (mTexelFormat == eGX_RGBA8) TXTR.Seek(0x20, SEEK_CUR);
+                if (BreakEarly) break;
             }
+            if (BreakEarly) break;
+        }
 
         u32 MipSize = (u32) (MipW * MipH * gskPixelsToBytes[mTexelFormat]);
         if (mTexelFormat == eGX_CMPR) MipSize *= 16; // Since we're pretending the image is 1/4 its actual size, we have to multiply the size by 16 to get the correct offset
@@ -267,8 +289,8 @@ void CTextureDecoder::PartialDecodeGXTexture(IInputStream& TXTR)
         MipOffset += MipSize;
         MipW /= 2;
         MipH /= 2;
-        if (MipW < BWidth) MipW = BWidth;
-        if (MipH < BHeight) MipH = BHeight;
+
+        if (BreakEarly) break;
     }
 }
 

src/Core/Resource/Script/CScriptObject.h

@@ -23,7 +23,7 @@ class CScriptObject
     friend class CAreaLoader;
 
     CScriptTemplate *mpTemplate;
-    TResPtr<CGameArea> mpArea;
+    CGameArea *mpArea;
     CScriptLayer *mpLayer;
     u32 mVersion;