Added ability to rebuild the resource database from the project resources folder. Editor can detect if the resource database is corrupt on load and if so prompts the user to repair it.

2025-12-17 00:47:05 +00:00 · 2017-07-04 04:59:22 -06:00
parent 1f3df14b02
commit 9a52fe52d4
29 changed files with 425 additions and 93 deletions
--- a/src/Core/GameProject/AssetNameGeneration.cpp
+++ b/src/Core/GameProject/AssetNameGeneration.cpp
@@ -76,12 +76,8 @@ void ApplyGeneratedName(CResourceEntry *pEntry, const TString& rkDir, const TStr
    if (pEntry->Directory() == pNewDir && pEntry->Name() == NewName) return;

    // Perform the move
-    CVirtualDirectory *pOldDir = pEntry->Directory();
    bool Success = pEntry->Move(pNewDir->FullPath(), NewName, true, true);
    ASSERT(Success);
-
-    // If the old directory is now empty, delete it
-    pEntry->ResourceStore()->ConditionalDeleteDirectory(pOldDir);
 }

 void GenerateAssetNames(CGameProject *pProj)
@@ -657,6 +653,7 @@ void GenerateAssetNames(CGameProject *pProj)
    }
 #endif

-    Log::Write("*** Asset Name Generation FINISHED ***");
+    pStore->RootDirectory()->RemoveEmptySubdirectories();
    pStore->ConditionalSaveStore();
+    Log::Write("*** Asset Name Generation FINISHED ***");
 }
--- a/src/Core/GameProject/CGameProject.cpp
+++ b/src/Core/GameProject/CGameProject.cpp
@@ -1,4 +1,5 @@
 #include "CGameProject.h"
+#include "IUIRelay.h"
 #include "Core/Resource/Factory/CTemplateLoader.h"
 #include "Core/Resource/Script/CMasterTemplate.h"
 #include <Common/Serialization/XML.h>
@@ -33,7 +34,7 @@ bool CGameProject::Save()
    return SaveSuccess;
 }

-void CGameProject::Serialize(IArchive& rArc)
+bool CGameProject::Serialize(IArchive& rArc)
 {
    rArc << SERIAL("Name", mProjectName)
         << SERIAL("Region", mRegion)
@@ -50,7 +51,7 @@ void CGameProject::Serialize(IArchive& rArc)

    rArc << SERIAL("ResourceDB", mResourceDBPath);

-    // Packages
+    // Serialize package list
    std::vector<TString> PackageList;

    if (!rArc.IsReader())
@@ -61,38 +62,29 @@ void CGameProject::Serialize(IArchive& rArc)

    rArc << SERIAL_CONTAINER("Packages", PackageList, "Package");

+    // Load packages
    if (rArc.IsReader())
    {
-        // Load resource store
-        ASSERT(mpResourceStore == nullptr);
-        mpResourceStore = new CResourceStore(this);
+        ASSERT(mPackages.empty());

-        if (!mpResourceStore->LoadResourceDatabase())
-            mLoadSuccess = false;
-
-        else
+        for (u32 iPkg = 0; iPkg < PackageList.size(); iPkg++)
        {
-            // Load packages
-            ASSERT(mPackages.empty());
+            const TString& rkPackagePath = PackageList[iPkg];
+            TString PackageName = rkPackagePath.GetFileName(false);
+            TString PackageDir = rkPackagePath.GetFileDirectory();

-            for (u32 iPkg = 0; iPkg < PackageList.size(); iPkg++)
+            CPackage *pPackage = new CPackage(this, PackageName, PackageDir);
+            bool PackageLoadSuccess = pPackage->Load();
+            mPackages.push_back(pPackage);
+
+            if (!PackageLoadSuccess)
            {
-                const TString& rkPackagePath = PackageList[iPkg];
-                TString PackageName = rkPackagePath.GetFileName(false);
-                TString PackageDir = rkPackagePath.GetFileDirectory();
-
-                CPackage *pPackage = new CPackage(this, PackageName, PackageDir);
-                bool PackageLoadSuccess = pPackage->Load();
-                mPackages.push_back(pPackage);
-
-                if (!PackageLoadSuccess)
-                {
-                    mLoadSuccess = false;
-                    break;
-                }
+                return false;
            }
        }
    }
+
+    return true;
 }

 bool CGameProject::BuildISO(const TString& rkIsoPath, IProgressNotifier *pProgress)
@@ -213,16 +205,23 @@ CGameProject* CGameProject::CreateProjectForExport(
    pProj->mProjectRoot.Replace("\\", "/");
    pProj->mpResourceStore = new CResourceStore(pProj);
    pProj->mpGameInfo->LoadGameInfo(Game);
-    pProj->mLoadSuccess = true;
    return pProj;
 }

-CGameProject* CGameProject::LoadProject(const TString& rkProjPath)
+CGameProject* CGameProject::LoadProject(const TString& rkProjPath, IProgressNotifier *pProgress)
 {
+    // Init project
    CGameProject *pProj = new CGameProject;
    pProj->mProjectRoot = rkProjPath.GetFileDirectory();
    pProj->mProjectRoot.Replace("\\", "/");

+    // Init progress
+    pProgress->SetTask(0, "Loading project: " + rkProjPath.GetFileName());
+
+    // Load main project file
+    pProgress->Report("Loading project settings");
+    bool LoadSuccess = false;
+
    TString ProjPath = rkProjPath;
    CXMLReader Reader(ProjPath);

@@ -233,9 +232,37 @@ CGameProject* CGameProject::LoadProject(const TString& rkProjPath)
    }

    pProj->mGame = Reader.Game();
-    pProj->Serialize(Reader);

-    if (!pProj->mLoadSuccess)
+    if (pProj->Serialize(Reader))
+    {
+        // Load resource database
+        pProgress->Report("Loading resource database");
+        pProj->mpResourceStore = new CResourceStore(pProj);
+        LoadSuccess = pProj->mpResourceStore->LoadResourceDatabase();
+
+        // Validate resource database
+        if (LoadSuccess)
+        {
+            pProgress->Report("Validating resource database");
+            bool DatabaseIsValid = pProj->mpResourceStore->AreAllEntriesValid();
+
+            // Resource database is corrupt. Ask the user if they want to rebuild it.
+            if (!DatabaseIsValid)
+            {
+                bool ShouldRebuild = gpUIRelay->AskYesNoQuestion("Error", "The resource database is corrupt. Attempt to repair it?");
+
+                if (ShouldRebuild)
+                {
+                    pProgress->Report("Repairing resource database");
+                    pProj->mpResourceStore->RebuildFromDirectory();
+                }
+                else
+                    LoadSuccess = false;
+            }
+        }
+    }
+
+    if (!LoadSuccess)
    {
        delete pProj;
        return nullptr;
--- a/src/Core/GameProject/CGameProject.h
+++ b/src/Core/GameProject/CGameProject.h
@@ -36,7 +36,6 @@ class CGameProject
    // Keep file handle open for the .prj file to prevent users from opening the same project
    // in multiple instances of PWE
    CFileLock mProjFileLock;
-    bool mLoadSuccess;

    enum EProjectVersion
    {
@@ -55,7 +54,6 @@ class CGameProject
        , mBuildVersion(0.f)
        , mResourceDBPath("ResourceDB.rdb")
        , mpResourceStore(nullptr)
-        , mLoadSuccess(true)
    {
        mpGameInfo = new CGameInfo();
        mpAudioManager = new CAudioManager(this);
@@ -65,7 +63,7 @@ public:
    ~CGameProject();

    bool Save();
-    void Serialize(IArchive& rArc);
+    bool Serialize(IArchive& rArc);
    bool BuildISO(const TString& rkIsoPath, IProgressNotifier *pProgress);
    void GetWorldList(std::list<CAssetID>& rOut) const;
    CAssetID FindNamedResource(const TString& rkName) const;
@@ -84,7 +82,7 @@ public:
            u32 FstAddress
        );

-    static CGameProject* LoadProject(const TString& rkProjPath);
+    static CGameProject* LoadProject(const TString& rkProjPath, IProgressNotifier *pProgress);

    // Directory Handling
    inline TString ProjectRoot() const                      { return mProjectRoot; }
--- a/src/Core/GameProject/CResourceEntry.cpp
+++ b/src/Core/GameProject/CResourceEntry.cpp
@@ -566,7 +566,7 @@ bool CResourceEntry::Move(const TString& rkDir, const TString& rkName, bool IsAu
        Log::Error("MOVE FAILED: " + MoveFailReason);
        mpDirectory = pOldDir;
        mName = OldName;
-        mpStore->ConditionalDeleteDirectory(pNewDir);
+        mpStore->ConditionalDeleteDirectory(pNewDir, false);
        return false;
    }
 }
--- a/src/Core/GameProject/CResourceIterator.h
+++ b/src/Core/GameProject/CResourceIterator.h
@@ -7,22 +7,22 @@
 class CResourceIterator
 {
 protected:
-    CResourceStore *mpStore;
-    std::map<CAssetID, CResourceEntry*>::iterator mIter;
+    const CResourceStore *mpkStore;
+    std::map<CAssetID, CResourceEntry*>::const_iterator mIter;
    CResourceEntry *mpCurEntry;

 public:
-    CResourceIterator(CResourceStore *pStore = gpResourceStore)
-        : mpStore(pStore)
+    CResourceIterator(const CResourceStore *pkStore = gpResourceStore)
+        : mpkStore(pkStore)
        , mpCurEntry(nullptr)
    {
-        mIter = mpStore->mResourceEntries.begin();
+        mIter = mpkStore->mResourceEntries.begin();
        Next();
    }

    virtual CResourceEntry* Next()
    {
-        if (mIter != mpStore->mResourceEntries.end())
+        if (mIter != mpkStore->mResourceEntries.end())
        {
            mpCurEntry = mIter->second;
            mIter++;
@@ -73,7 +73,7 @@ public:
    TResourceIterator(CResourceStore *pStore = gpResourceStore)
        : CResourceIterator(pStore)
    {
-        if (mpCurEntry->ResourceType() != ResType)
+        if (mpCurEntry && mpCurEntry->ResourceType() != ResType)
            Next();
    }

--- a/src/Core/GameProject/CResourceStore.cpp
+++ b/src/Core/GameProject/CResourceStore.cpp
@@ -46,7 +46,7 @@ CResourceStore::~CResourceStore()
        delete It->second;
 }

-void CResourceStore::SerializeResourceDatabase(IArchive& rArc)
+bool CResourceStore::SerializeResourceDatabase(IArchive& rArc)
 {
    struct SDatabaseResource
    {
@@ -83,6 +83,8 @@ void CResourceStore::SerializeResourceDatabase(IArchive& rArc)
            RegisterResource(rRes.ID, rRes.pType->Type(), rRes.Directory, rRes.Name);
        }
    }
+
+    return true;
 }

 bool CResourceStore::LoadResourceDatabase()
@@ -105,7 +107,7 @@ bool CResourceStore::LoadResourceDatabase()
        ASSERT(mpProj->Game() == Reader.Game());

    mGame = Reader.Game();
-    SerializeResourceDatabase(Reader);
+    if (!SerializeResourceDatabase(Reader)) return false;
    return LoadCacheFile();
 }

@@ -293,19 +295,17 @@ CVirtualDirectory* CResourceStore::GetVirtualDirectory(const TString& rkPath, bo
        return nullptr;
 }

-void CResourceStore::ConditionalDeleteDirectory(CVirtualDirectory *pDir)
+void CResourceStore::ConditionalDeleteDirectory(CVirtualDirectory *pDir, bool Recurse)
 {
-    if (pDir->IsEmpty())
+    if (pDir->IsEmpty() && !pDir->IsRoot())
    {
-        // If this directory is part of the project, then we should delete the corresponding filesystem directory
-        if (pDir->GetRoot() == mpDatabaseRoot && !pDir->IsRoot())
-        {
-            FileUtil::DeleteDirectory(ResourcesDir() + pDir->FullPath(), true);
-        }
-
        CVirtualDirectory *pParent = pDir->Parent();
        pParent->RemoveChildDirectory(pDir);
-        ConditionalDeleteDirectory(pParent);
+
+        if (Recurse)
+        {
+            ConditionalDeleteDirectory(pParent, true);
+        }
    }
 }

@@ -322,6 +322,98 @@ CResourceEntry* CResourceStore::FindEntry(const TString& rkPath) const
    return (mpDatabaseRoot ? mpDatabaseRoot->FindChildResource(rkPath) : nullptr);
 }

+bool CResourceStore::AreAllEntriesValid() const
+{
+    for (CResourceIterator Iter(this); Iter; ++Iter)
+    {
+        if (!Iter->HasCookedVersion() && !Iter->HasRawVersion())
+            return false;
+    }
+
+    return true;
+}
+
+void CResourceStore::ClearDatabase()
+{
+    // THIS OPERATION REQUIRES THAT ALL RESOURCES ARE UNREFERENCED
+    DestroyUnreferencedResources();
+    ASSERT(mLoadedResources.empty());
+
+    // Clear out existing resource entries and directories
+    for (auto Iter = mResourceEntries.begin(); Iter != mResourceEntries.end(); Iter++)
+        delete Iter->second;
+    mResourceEntries.clear();
+
+    delete mpDatabaseRoot;
+    mpDatabaseRoot = new CVirtualDirectory(this);
+
+    mDatabaseDirty = true;
+    mCacheFileDirty = true;
+}
+
+void CResourceStore::RebuildFromDirectory()
+{
+    ASSERT(mpProj != nullptr);
+    mpProj->AudioManager()->ClearAssets();
+    ClearDatabase();
+
+    // Get list of resources
+    TString ResDir = ResourcesDir();
+    TStringList ResourceList;
+    FileUtil::GetDirectoryContents(ResDir, ResourceList);
+
+    for (auto Iter = ResourceList.begin(); Iter != ResourceList.end(); Iter++)
+    {
+        TString Path = *Iter;
+        TString RelPath = FileUtil::MakeRelative(Path, ResDir);
+
+        if (FileUtil::IsFile(Path) && Path.GetFileExtension() == "rsmeta")
+        {
+            // Determine resource name
+            TString DirPath = RelPath.GetFileDirectory();
+            TString CookedFilename = RelPath.GetFileName(false); // This call removes the .rsmeta extension
+            TString ResName = CookedFilename.GetFileName(false); // This call removes the cooked extension
+            ASSERT( IsValidResourcePath(DirPath, ResName) );
+
+            // Determine resource type
+            TString CookedExtension = CookedFilename.GetFileExtension();
+            CResTypeInfo *pTypeInfo = CResTypeInfo::TypeForCookedExtension( Game(), CFourCC(CookedExtension) );
+
+            if (!pTypeInfo)
+            {
+                Log::Error("Found resource but couldn't register because failed to identify resource type: " + RelPath);
+                continue;
+            }
+
+            // Create resource entry
+            CResourceEntry *pEntry = new CResourceEntry(this, CAssetID::InvalidID(mGame), DirPath, ResName, pTypeInfo->Type());
+            pEntry->LoadMetadata();
+
+            // Validate the entry
+            CAssetID ID = pEntry->ID();
+            ASSERT( mResourceEntries.find(ID) == mResourceEntries.end() );
+            ASSERT( ID.Length() == CAssetID::GameIDLength(mGame) );
+
+            mResourceEntries[ID] = pEntry;
+        }
+
+        else if (FileUtil::IsDirectory(Path))
+            GetVirtualDirectory(RelPath, true);
+    }
+
+    // Make sure audio manager is loaded correctly so AGSC dependencies can be looked up
+    mpProj->AudioManager()->LoadAssets();
+
+    // Update dependencies
+    for (CResourceIterator It(this); It; ++It)
+        It->UpdateDependencies();
+
+    // Update database files
+    mDatabaseDirty = true;
+    mCacheFileDirty = true;
+    ConditionalSaveStore();
+}
+
 bool CResourceStore::IsResourceRegistered(const CAssetID& rkID) const
 {
    return FindEntry(rkID) != nullptr;
--- a/src/Core/GameProject/CResourceStore.h
+++ b/src/Core/GameProject/CResourceStore.h
@@ -43,7 +43,7 @@ public:
    CResourceStore(const TString& rkDatabasePath);
    CResourceStore(CGameProject *pProject);
    ~CResourceStore();
-    void SerializeResourceDatabase(IArchive& rArc);
+    bool SerializeResourceDatabase(IArchive& rArc);
    bool LoadResourceDatabase();
    bool SaveResourceDatabase();
    bool LoadCacheFile();
@@ -52,12 +52,15 @@ public:
    void SetProject(CGameProject *pProj);
    void CloseProject();
    CVirtualDirectory* GetVirtualDirectory(const TString& rkPath, bool AllowCreate);
-    void ConditionalDeleteDirectory(CVirtualDirectory *pDir);
+    void ConditionalDeleteDirectory(CVirtualDirectory *pDir, bool Recurse);

    bool IsResourceRegistered(const CAssetID& rkID) const;
    CResourceEntry* RegisterResource(const CAssetID& rkID, EResType Type, const TString& rkDir, const TString& rkName);
    CResourceEntry* FindEntry(const CAssetID& rkID) const;
    CResourceEntry* FindEntry(const TString& rkPath) const;
+    bool AreAllEntriesValid() const;
+    void ClearDatabase();
+    void RebuildFromDirectory();

    template<typename ResType> ResType* LoadResource(const CAssetID& rkID)  { return static_cast<ResType*>(LoadResource(rkID, ResType::StaticType())); }
    CResource* LoadResource(const CAssetID& rkID);
--- a/src/Core/GameProject/CVirtualDirectory.cpp
+++ b/src/Core/GameProject/CVirtualDirectory.cpp
@@ -155,6 +155,7 @@ bool CVirtualDirectory::AddChild(const TString &rkPath, CResourceEntry *pEntry)
        {
            // Create new subdirectory
            pSubdir = new CVirtualDirectory(this, DirName, mpStore);
+            FileUtil::MakeDirectory(mpStore->ResourcesDir() + pSubdir->FullPath());
            mSubdirectories.push_back(pSubdir);

            std::sort(mSubdirectories.begin(), mSubdirectories.end(), [](CVirtualDirectory *pLeft, CVirtualDirectory *pRight) -> bool {
@@ -199,6 +200,14 @@ bool CVirtualDirectory::RemoveChildDirectory(CVirtualDirectory *pSubdir)
        if (*It == pSubdir)
        {
            mSubdirectories.erase(It);
+
+            // If this is part of the resource store, delete the corresponding filesystem directory
+            if (mpStore && pSubdir->GetRoot() == mpStore->RootDirectory())
+            {
+                TString AbsPath = mpStore->DatabaseRootPath() + pSubdir->FullPath();
+                FileUtil::DeleteDirectory(AbsPath, true);
+            }
+
            delete pSubdir;
            return true;
        }
@@ -221,6 +230,19 @@ bool CVirtualDirectory::RemoveChildResource(CResourceEntry *pEntry)
    return false;
 }

+void CVirtualDirectory::RemoveEmptySubdirectories()
+{
+    for (u32 SubdirIdx = 0; SubdirIdx < mSubdirectories.size(); SubdirIdx++)
+    {
+        CVirtualDirectory *pDir = mSubdirectories[SubdirIdx];
+
+        if (pDir->IsEmpty())
+            RemoveChildDirectory(pDir);
+        else
+            pDir->RemoveEmptySubdirectories();
+    }
+}
+
 // ************ STATIC ************
 bool CVirtualDirectory::IsValidDirectoryName(const TString& rkName)
 {
--- a/src/Core/GameProject/CVirtualDirectory.h
+++ b/src/Core/GameProject/CVirtualDirectory.h
@@ -33,6 +33,7 @@ public:
    bool AddChild(const TString& rkPath, CResourceEntry *pEntry);
    bool RemoveChildDirectory(CVirtualDirectory *pSubdir);
    bool RemoveChildResource(CResourceEntry *pEntry);
+    void RemoveEmptySubdirectories();

    static bool IsValidDirectoryName(const TString& rkName);
    static bool IsValidDirectoryPath(TString Path);