System files added to Wii partition first to keep 32-bit loaders happy

include/NOD/DirectoryEnumerator.hpp

@@ -6,6 +6,29 @@
 namespace NOD
 {
 
+struct CaseInsensitiveCompare
+{
+    bool operator()(const std::string& lhs, const std::string& rhs) const
+    {
+#if _WIN32
+        if (_stricmp(lhs.c_str(), rhs.c_str()) < 0)
+#else
+        if (strcasecmp(lhs.c_str(), rhs.c_str()) < 0)
+#endif
+            return true;
+        return false;
+    }
+
+#if _WIN32
+    bool operator()(const std::wstring& lhs, const std::wstring& rhs) const
+    {
+        if (_wcsicmp(lhs.c_str(), rhs.c_str()) < 0)
+            return true;
+        return false;
+    }
+#endif
+};
+
 class DirectoryEnumerator
 {
 public:

include/NOD/DiscBase.hpp

@@ -4,6 +4,7 @@
 #include <vector>
 #include <memory>
 #include <string>
+#include <unordered_map>
 #include <stdio.h>
 #include <stdint.h>
 #include "Util.hpp"
@@ -300,10 +301,10 @@ public:
 class DiscBuilderBase
 {
 public:
-    class IPartitionBuilder
+    class PartitionBuilderBase
     {
     public:
-        virtual ~IPartitionBuilder() {}
+        virtual ~PartitionBuilderBase() {}
         enum class Kind : uint32_t
         {
             Data,
@@ -311,13 +312,15 @@ public:
             Channel
         };
     protected:
+        std::unordered_map<SystemString, std::pair<uint64_t,uint64_t>> m_fileOffsetsSizes;
         std::vector<FSTNode> m_buildNodes;
         std::vector<std::string> m_buildNames;
         size_t m_buildNameOff = 0;
         virtual uint64_t userAllocate(uint64_t reqSz)=0;
         virtual uint32_t packOffset(uint64_t offset) const=0;
-        void recursiveBuildNodes(const SystemChar* dirIn, uint64_t dolInode,
+        void recursiveBuildNodes(bool system, const SystemChar* dirIn, uint64_t dolInode);
-                                 std::function<void(void)> incParents);
+        void recursiveBuildFST(const SystemChar* dirIn, uint64_t dolInode,
+                               std::function<void(void)> incParents);
         void addBuildName(const SystemString& str)
         {
             SystemUTF8View utf8View(str);
@@ -333,8 +336,8 @@ public:
         uint64_t m_dolOffset = 0;
         uint64_t m_dolSize = 0;
     public:
-        IPartitionBuilder(DiscBuilderBase& parent, Kind kind,
+        PartitionBuilderBase(DiscBuilderBase& parent, Kind kind,
-                          const char gameID[6], const char* gameTitle)
+                             const char gameID[6], const char* gameTitle)
         : m_parent(parent), m_kind(kind), m_gameTitle(gameTitle)
         {
             memcpy(m_gameID, gameID, 6);
@@ -347,7 +350,7 @@ public:
     };
 protected:
     std::unique_ptr<IFileIO> m_fileIO;
-    std::vector<std::unique_ptr<IPartitionBuilder>> m_partitions;
+    std::vector<std::unique_ptr<PartitionBuilderBase>> m_partitions;
 public:
     std::function<void(size_t idx, const SystemString&, size_t)> m_progressCB;
     size_t m_progressIdx = 0;

include/NOD/Util.hpp

@@ -48,6 +48,7 @@ static inline void ToLower(SystemString& str)
 {std::transform(str.begin(), str.end(), str.begin(), towlower);}
 static inline void ToUpper(SystemString& str)
 {std::transform(str.begin(), str.end(), str.begin(), towupper);}
+static inline size_t StrLen(const SystemChar* str) {return _wcslen(str);}
 class SystemUTF8View
 {
     std::string m_utf8;
@@ -82,6 +83,7 @@ static inline void ToLower(SystemString& str)
 {std::transform(str.begin(), str.end(), str.begin(), tolower);}
 static inline void ToUpper(SystemString& str)
 {std::transform(str.begin(), str.end(), str.begin(), toupper);}
+static inline size_t StrLen(const SystemChar* str) {return strlen(str);}
 class SystemUTF8View
 {
     const std::string& m_utf8;
@@ -112,6 +114,24 @@ static inline void Unlink(const SystemChar* file)
 #endif
 }
 
+static inline int StrCmp(const SystemChar* str1, const SystemChar* str2)
+{
+#if HECL_UCS2
+    return _wcscmp(str1, str2);
+#else
+    return strcmp(str1, str2);
+#endif
+}
+
+static inline int StrCaseCmp(const SystemChar* str1, const SystemChar* str2)
+{
+#if HECL_UCS2
+    return _wcsicmp(str1, str2);
+#else
+    return strcasecmp(str1, str2);
+#endif
+}
+
 #undef bswap16
 #undef bswap32
 #undef bswap64

lib/DirectoryEnumerator.cpp

@@ -12,29 +12,6 @@
 namespace NOD
 {
 
-struct CaseInsensitiveCompare
-{
-    bool operator()(const std::string& lhs, const std::string& rhs) const
-    {
-#if _WIN32
-        if (_stricmp(lhs.c_str(), rhs.c_str()) < 0)
-#else
-        if (strcasecmp(lhs.c_str(), rhs.c_str()) < 0)
-#endif
-            return true;
-        return false;
-    }
-
-#if _WIN32
-    bool operator()(const std::wstring& lhs, const std::wstring& rhs) const
-    {
-        if (_wcsicmp(lhs.c_str(), rhs.c_str()) < 0)
-            return true;
-        return false;
-    }
-#endif
-};
-
 DirectoryEnumerator::DirectoryEnumerator(const SystemChar* path, Mode mode,
                                          bool sizeSort, bool reverse, bool noHidden)
 {

lib/DiscBase.cpp

@@ -145,8 +145,8 @@ static uint64_t GetInode(const SystemChar* path)
     BY_HANDLE_FILE_INFORMATION info;
     if (!GetFileInformationByHandle(fp, &info))
         LogModule.report(LogVisor::FatalError, _S("unable to GetFileInformationByHandle %s"), path);
-    inode = info.nFileIndexHigh << 32;
+    inode = uint64_t(info.nFileIndexHigh) << 32;
-    inode |= info.nFileIndexLow;
+    inode |= uint64_t(info.nFileIndexLow);
     CloseHandle(fp);
 #else
     struct stat st;
@@ -157,9 +157,31 @@ static uint64_t GetInode(const SystemChar* path)
     return inode;
 }
 
-void DiscBuilderBase::IPartitionBuilder::recursiveBuildNodes(const SystemChar* dirIn,
+static bool IsSystemFile(const SystemString& name)
-                                                             uint64_t dolInode,
+{
-                                                             std::function<void(void)> incParents)
+    if (name.size() < 4)
+        return false;
+
+    if (!StrCaseCmp((&*name.cend()) - 4, _S(".dol")))
+        return true;
+    if (!StrCaseCmp((&*name.cend()) - 4, _S(".rel")))
+        return true;
+    if (!StrCaseCmp((&*name.cend()) - 4, _S(".rso")))
+        return true;
+    if (!StrCaseCmp((&*name.cend()) - 4, _S(".sel")))
+        return true;
+    if (!StrCaseCmp((&*name.cend()) - 4, _S(".bnr")))
+        return true;
+    if (!StrCaseCmp((&*name.cend()) - 4, _S(".elf")))
+        return true;
+    if (!StrCaseCmp((&*name.cend()) - 4, _S(".wad")))
+        return true;
+
+    return false;
+}
+
+void DiscBuilderBase::PartitionBuilderBase::recursiveBuildNodes(bool system, const SystemChar* dirIn,
+                                                                uint64_t dolInode)
 {
     DirectoryEnumerator dEnum(dirIn, DirectoryEnumerator::Mode::DirsThenFilesSorted, false, false, true);
     for (const DirectoryEnumerator::Entry& e : dEnum)
@@ -167,23 +189,20 @@ void DiscBuilderBase::IPartitionBuilder::recursiveBuildNodes(const SystemChar* d
         if (e.m_isDir)
         {
             size_t dirNodeIdx = m_buildNodes.size();
-            m_buildNodes.emplace_back(true, m_buildNameOff, 0, dirNodeIdx+1);
+            recursiveBuildNodes(system, e.m_path.c_str(), dolInode);
-            addBuildName(e.m_name);
-            incParents();
-            recursiveBuildNodes(e.m_path.c_str(), dolInode, [&](){m_buildNodes[dirNodeIdx].incrementLength(); incParents();});
         }
         else
         {
-            if (dolInode == GetInode(e.m_path.c_str()))
+            bool isSys = IsSystemFile(e.m_name);
-            {
+            if (system ^ isSys)
-                m_buildNodes.emplace_back(false, m_buildNameOff, packOffset(m_dolOffset), m_dolSize);
+                continue;
-                addBuildName(e.m_name);
+
-                incParents();
+            if (dolInode == GetInode(e.m_path.c_str()))
                 continue;
-            }
 
             size_t fileSz = ROUND_UP_32(e.m_fileSz);
             uint64_t fileOff = userAllocate(fileSz);
+            m_fileOffsetsSizes[e.m_path] = std::make_pair(fileOff, fileSz);
             std::unique_ptr<IFileIO::IWriteStream> ws = m_parent.getFileIO().beginWriteStream(fileOff);
             FILE* fp = Fopen(e.m_path.c_str(), _S("rb"), FileLockType::Read);
             if (!fp)
@@ -203,14 +222,42 @@ void DiscBuilderBase::IPartitionBuilder::recursiveBuildNodes(const SystemChar* d
             fclose(fp);
             for (size_t i=0 ; i<fileSz-xferSz ; ++i)
                 ws->write("\xff", 1);
-            m_buildNodes.emplace_back(false, m_buildNameOff, packOffset(fileOff), fileSz);
+        }
+    }
+}
+
+void DiscBuilderBase::PartitionBuilderBase::recursiveBuildFST(const SystemChar* dirIn, uint64_t dolInode,
+                                                              std::function<void(void)> incParents)
+{
+    DirectoryEnumerator dEnum(dirIn, DirectoryEnumerator::Mode::DirsThenFilesSorted, false, false, true);
+    for (const DirectoryEnumerator::Entry& e : dEnum)
+    {
+        if (e.m_isDir)
+        {
+            size_t dirNodeIdx = m_buildNodes.size();
+            m_buildNodes.emplace_back(true, m_buildNameOff, 0, dirNodeIdx+1);
+            addBuildName(e.m_name);
+            incParents();
+            recursiveBuildFST(e.m_path.c_str(), dolInode, [&](){m_buildNodes[dirNodeIdx].incrementLength(); incParents();});
+        }
+        {
+            if (dolInode == GetInode(e.m_path.c_str()))
+            {
+                m_buildNodes.emplace_back(false, m_buildNameOff, packOffset(m_dolOffset), m_dolSize);
+                addBuildName(e.m_name);
+                incParents();
+                continue;
+            }
+
+            std::pair<uint64_t,uint64_t> fileOffSz = m_fileOffsetsSizes.at(e.m_path);
+            m_buildNodes.emplace_back(false, m_buildNameOff, packOffset(fileOffSz.first), fileOffSz.second);
             addBuildName(e.m_name);
             incParents();
         }
     }
 }
 
-bool DiscBuilderBase::IPartitionBuilder::buildFromDirectory(const SystemChar* dirIn,
+bool DiscBuilderBase::PartitionBuilderBase::buildFromDirectory(const SystemChar* dirIn,
                                                             const SystemChar* dolIn,
                                                             const SystemChar* apploaderIn)
 {
@@ -258,7 +305,10 @@ bool DiscBuilderBase::IPartitionBuilder::buildFromDirectory(const SystemChar* di
     }
 
     /* Gather files in root directory */
-    recursiveBuildNodes(dirIn, GetInode(dolIn), [&](){m_buildNodes[0].incrementLength();});
+    uint64_t dolInode = GetInode(dolIn);
+    recursiveBuildNodes(true, dirIn, dolInode);
+    recursiveBuildNodes(false, dirIn, dolInode);
+    recursiveBuildFST(dirIn, dolInode, [&](){m_buildNodes[0].incrementLength();});
 
     return true;
 }

lib/DiscGCN.cpp

@@ -109,14 +109,14 @@ DiscGCN::DiscGCN(std::unique_ptr<IDiscIO>&& dio)
     m_partitions.emplace_back(new PartitionGCN(*this, IPartition::Kind::Data, 0));
 }
 
-class PartitionBuilderGCN : public DiscBuilderBase::IPartitionBuilder
+class PartitionBuilderGCN : public DiscBuilderBase::PartitionBuilderBase
 {
     uint64_t m_curUser = 0x57058000;
     uint32_t m_fstMemoryAddr;
 public:
     PartitionBuilderGCN(DiscBuilderBase& parent, Kind kind,
                         const char gameID[6], const char* gameTitle, uint32_t fstMemoryAddr)
-    : DiscBuilderBase::IPartitionBuilder(parent, kind, gameID, gameTitle), m_fstMemoryAddr(fstMemoryAddr) {}
+    : DiscBuilderBase::PartitionBuilderBase(parent, kind, gameID, gameTitle), m_fstMemoryAddr(fstMemoryAddr) {}
 
     uint64_t userAllocate(uint64_t reqSz)
     {
@@ -137,7 +137,7 @@ public:
 
     bool buildFromDirectory(const SystemChar* dirIn, const SystemChar* dolIn, const SystemChar* apploaderIn)
     {
-        bool result = DiscBuilderBase::IPartitionBuilder::buildFromDirectory(dirIn, dolIn, apploaderIn);
+        bool result = DiscBuilderBase::PartitionBuilderBase::buildFromDirectory(dirIn, dolIn, apploaderIn);
         if (!result)
             return false;
 
@@ -208,7 +208,7 @@ DiscBuilderGCN::DiscBuilderGCN(const SystemChar* outPath, const char gameID[6],
                                uint32_t fstMemoryAddr, std::function<void(size_t, const SystemString&, size_t)> progressCB)
 : DiscBuilderBase(std::move(NewFileIO(outPath)), progressCB)
 {
-    PartitionBuilderGCN* partBuilder = new PartitionBuilderGCN(*this, IPartitionBuilder::Kind::Data,
+    PartitionBuilderGCN* partBuilder = new PartitionBuilderGCN(*this, PartitionBuilderBase::Kind::Data,
                                                                gameID, gameTitle, fstMemoryAddr);
     m_partitions.emplace_back(partBuilder);
 }

lib/DiscWii.cpp

@@ -433,13 +433,13 @@ void DiscWii::writeOutDataPartitionHeader(const SystemChar* pathOut) const
     }
 }
 
-class PartitionBuilderWii : public DiscBuilderBase::IPartitionBuilder
+class PartitionBuilderWii : public DiscBuilderBase::PartitionBuilderBase
 {
     uint64_t m_curUser = 0x1F0000;
 public:
     PartitionBuilderWii(DiscBuilderBase& parent, Kind kind,
                         const char gameID[6], const char* gameTitle)
-    : DiscBuilderBase::IPartitionBuilder(parent, kind, gameID, gameTitle) {}
+    : DiscBuilderBase::PartitionBuilderBase(parent, kind, gameID, gameTitle) {}
 
     uint64_t getCurUserEnd() const {return m_curUser;}
 
@@ -463,7 +463,7 @@ public:
 
     bool buildFromDirectory(const SystemChar* dirIn, const SystemChar* dolIn, const SystemChar* apploaderIn)
     {
-        bool result = DiscBuilderBase::IPartitionBuilder::buildFromDirectory(dirIn, dolIn, apploaderIn);
+        bool result = DiscBuilderBase::PartitionBuilderBase::buildFromDirectory(dirIn, dolIn, apploaderIn);
         if (!result)
             return false;
 
@@ -839,7 +839,7 @@ DiscBuilderWii::DiscBuilderWii(const SystemChar* outPath, const char gameID[6],
                                std::function<void(size_t, const SystemString&, size_t)> progressCB)
 : DiscBuilderBase(std::move(std::unique_ptr<IFileIO>()), progressCB), m_outPath(outPath), m_dualLayer(dualLayer)
 {
-    PartitionBuilderWii* partBuilder = new PartitionBuilderWii(*this, IPartitionBuilder::Kind::Data,
+    PartitionBuilderWii* partBuilder = new PartitionBuilderWii(*this, PartitionBuilderBase::Kind::Data,
                                                                gameID, gameTitle);
     m_partitions.emplace_back(partBuilder);
 }