added pak index DNA records

.gitmodules

@@ -1,6 +1,3 @@
-[submodule "PakLib"]
-	path = PakLib
-	url = https://github.com/RetroView/PakLib.git
 [submodule "NODLib"]
 	path = NODLib
 	url = https://github.com/RetroView/NODLib.git

DataSpec/DNACommon/DNACommon.hpp

@@ -2,8 +2,7 @@
 #define __DNA_COMMON_HPP__
 
 #include <Athena/DNA.hpp>
-#include <CFourCC.hpp>
+#include "HECL/HECL.hpp"
-#include <CUniqueID.hpp>
 
 namespace Retro
 {
@@ -11,50 +10,120 @@ namespace Retro
 /* This comes up a great deal */
 typedef Athena::io::DNA<Athena::BigEndian> BigDNA;
 
-/* FourCC DNA */
+/* PAK 32-bit Unique ID */
-class DNAFourCC : public BigDNA, public CFourCC
+class UniqueID32 : public BigDNA
 {
+    uint32_t m_id;
 public:
     Delete expl;
-    void read(Athena::io::IStreamReader& reader)
+    inline void read(Athena::io::IStreamReader& reader)
-    {_read(reader);}
+    {reader.readUint32();}
-    void write(Athena::io::IStreamWriter& writer) const
+    inline void write(Athena::io::IStreamWriter& writer) const
-    {_write(writer);}
+    {writer.writeUint32(m_id);}
+
+    inline bool operator!=(const UniqueID32& other) const {return m_id != other.m_id;}
+    inline bool operator==(const UniqueID32& other) const {return m_id == other.m_id;}
+    inline std::string toString() const
+    {
+        char buf[9];
+        snprintf(buf, 9, "%08X", m_id);
+        return std::string(buf);
+    }
 };
 
-/* PAK 32-bit Unique ID DNA */
+/* PAK 64-bit Unique ID */
-class DNAUniqueID32 : public BigDNA, public CUniqueID
+class UniqueID64 : public BigDNA
 {
+    uint64_t m_id;
 public:
     Delete expl;
-    void read(Athena::io::IStreamReader& reader)
+    inline void read(Athena::io::IStreamReader& reader)
-    {_read(reader, E_32Bits);}
+    {reader.readUint64();}
-    void write(Athena::io::IStreamWriter& writer) const
+    inline void write(Athena::io::IStreamWriter& writer) const
-    {_write(writer);}
+    {writer.writeUint64(m_id);}
+
+    inline bool operator!=(const UniqueID64& other) const {return m_id != other.m_id;}
+    inline bool operator==(const UniqueID64& other) const {return m_id == other.m_id;}
+    inline std::string toString() const
+    {
+        char buf[17];
+        snprintf(buf, 17, "%16X", m_id);
+        return std::string(buf);
+    }
 };
 
-/* PAK 64-bit Unique ID DNA */
+/* PAK 128-bit Unique ID */
-class DNAUniqueID64 : public BigDNA, public CUniqueID
+class UniqueID128 : public BigDNA
 {
-public:
+    union
-    Delete expl;
+    {
-    void read(Athena::io::IStreamReader& reader)
+        uint64_t m_id[2];
-    {_read(reader, E_64Bits);}
+#if __SSE__
-    void write(Athena::io::IStreamWriter& writer) const
+        __m128i m_id128;
-    {_write(writer);}
+#endif
     };
-
-/* PAK 128-bit Unique ID DNA */
-class DNAUniqueID128 : public BigDNA, public CUniqueID
-{
 public:
     Delete expl;
-    void read(Athena::io::IStreamReader& reader)
+    inline void read(Athena::io::IStreamReader& reader)
-    {_read(reader, E_128Bits);}
+    {
-    void write(Athena::io::IStreamWriter& writer) const
+        m_id[0] = reader.readUint64();
-    {_write(writer);}
+        m_id[1] = reader.readUint64();
+    }
+    inline void write(Athena::io::IStreamWriter& writer) const
+    {
+        writer.writeUint64(m_id[0]);
+        writer.writeUint64(m_id[1]);
+    }
+
+    inline bool operator!=(const UniqueID128& other) const
+    {
+#if __SSE__
+        return m_id128 != other.m_id128;
+#else
+        return (m_id[0] != other.m_id[0]) || (m_id[1] != other.m_id[1]);
+#endif
+    }
+    inline bool operator==(const UniqueID128& other) const
+    {
+#if __SSE__
+        return m_id128 == other.m_id128;
+#else
+        return (m_id[0] == other.m_id[0]) && (m_id[1] == other.m_id[1]);
+#endif
+    }
+    inline std::string toString() const
+    {
+        char buf[33];
+        snprintf(buf, 33, "%16X%16X", m_id[0], m_id[1]);
+        return std::string(buf);
+    }
 };
 
 }
 
+/* Hash template-specializations for UniqueID types */
+namespace std
+{
+template<>
+struct hash<Retro::UniqueID32>
+{
+    inline size_t operator()(const Retro::UniqueID32& id) const
+    {return id.m_id;}
+};
+
+template<>
+struct hash<Retro::UniqueID64>
+{
+    inline size_t operator()(const Retro::UniqueID64& id) const
+    {return id.m_id;}
+};
+
+template<>
+struct hash<Retro::UniqueID128>
+{
+    inline size_t operator()(const Retro::UniqueID128& id) const
+    {return id.m_id[0] ^ id.m_id[1];}
+};
+}
+
 #endif // __DNA_COMMON_HPP__

DataSpec/DNAMP1/DNAMP1.pri

@@ -1,2 +1,3 @@
 HEADERS += \
+    $$PWD/PAK.hpp \
     $$PWD/MLVL.hpp

DataSpec/DNAMP1/PAK.hpp

@@ -0,0 +1,115 @@
+#ifndef __DNAMP1_PAK_HPP__
+#define __DNAMP1_PAK_HPP__
+
+#include <unordered_map>
+
+#include "../Logging.hpp"
+#include "../DNACommon/DNACommon.hpp"
+
+namespace Retro
+{
+namespace DNAMP1
+{
+
+class PAK : public BigDNA
+{
+public:
+    struct NameEntry : public BigDNA
+    {
+        HECL::FourCC type;
+        UniqueID32 id;
+        Value<atUint32> nameLen;
+        String<DNA_COUNT(nameLen)> name;
+    };
+
+    struct Entry : public BigDNA
+    {
+        Value<atUint32> compressed;
+        HECL::FourCC type;
+        UniqueID32 id;
+        Value<atUint32> size;
+        Value<atUint32> offset;
+    };
+
+private:
+    std::vector<NameEntry> m_nameEntries;
+    std::vector<Entry> m_entries;
+    std::unordered_map<UniqueID32, Entry*> m_idMap;
+    std::unordered_map<std::string, Entry*> m_nameMap;
+    Delete expl;
+
+public:
+    void read(Athena::io::IStreamReader& reader)
+    {
+        reader.setEndian(Athena::BigEndian);
+        atUint32 version = reader.readUint32();
+        if (version != 0x00030005)
+            LogModule.report(LogVisor::FatalError, "unexpected PAK magic");
+        reader.readUint32();
+
+        atUint32 nameCount = reader.readUint32();
+        m_nameEntries.clear();
+        m_nameEntries.reserve(nameCount);
+        for (atUint32 n=0 ; n<nameCount ; ++n)
+        {
+            m_nameEntries.emplace_back();
+            m_nameEntries.back().read(reader);
+        }
+
+        atUint32 count = reader.readUint32();
+        m_entries.clear();
+        m_entries.reserve(count);
+        m_idMap.clear();
+        m_idMap.reserve(count);
+        for (atUint32 e=0 ; e<count ; ++e)
+        {
+            m_entries.emplace_back();
+            m_entries.back().read(reader);
+            m_idMap[m_entries.back().id] = &m_entries.back();
+        }
+
+        m_nameMap.clear();
+        m_nameMap.reserve(nameCount);
+        for (NameEntry& entry : m_nameEntries)
+        {
+            Entry* found = m_idMap.find(entry.id);
+            if (found != m_idMap.end())
+                m_nameMap[entry.name] = found;
+        }
+    }
+    void write(Athena::io::IStreamWriter& writer) const
+    {
+        writer.setEndian(Athena::BigEndian);
+        writer.writeUint32(0x00030005);
+        writer.writeUint32(0);
+
+        writer.writeUint32(m_nameEntries.size());
+        for (NameEntry& entry : m_nameEntries)
+            entry.write(writer);
+
+        writer.writeUint32(m_entries.size());
+        for (Entry& entry : m_entries)
+            entry.write(writer);
+    }
+
+    inline const Entry* lookupEntry(const UniqueID32& id) const
+    {
+        Entry* result = m_idMap.find(id);
+        if (result != m_idMap.end())
+            return result;
+        return nullptr;
+    }
+
+    inline const Entry* lookupEntry(const std::string& name) const
+    {
+        Entry* result = m_nameMap.find(name);
+        if (result != m_nameMap.end())
+            return result;
+        return nullptr;
+    }
+};
+
+}
+}
+
+#endif // __DNAMP1_PAK_HPP__

DataSpec/DNAMP2/DNAMP2.pri

@@ -0,0 +1,2 @@
+HEADERS += \
+    $$PWD/PAK.hpp

DataSpec/DNAMP2/PAK.hpp

@@ -0,0 +1,17 @@
+#ifndef __DNAMP2_PAK_HPP__
+#define __DNAMP2_PAK_HPP__
+
+#include "../DNAMP1/PAK.hpp"
+
+namespace Retro
+{
+namespace DNAMP2
+{
+
+/* Same PAK format as MP1 */
+using PAK = DNAMP1::PAK;
+
+}
+}
+
+#endif // __DNAMP2_PAK_HPP__

DataSpec/DNAMP3/DNAMP3.pri

@@ -0,0 +1,2 @@
+HEADERS += \
+    $$PWD/PAK.hpp

DataSpec/DNAMP3/PAK.hpp

@@ -0,0 +1,156 @@
+#ifndef __DNAMP3_PAK_HPP__
+#define __DNAMP3_PAK_HPP__
+
+#include <unordered_map>
+
+#include "../Logging.hpp"
+#include "../DNACommon/DNACommon.hpp"
+
+namespace Retro
+{
+namespace DNAMP3
+{
+
+class PAK : public BigDNA
+{
+public:
+    struct Header : public BigDNA
+    {
+        Value<atUint32> version;
+        Value<atUint32> headSz;
+        Value<atUint8> md5sum[16];
+        Seek<40, Athena::Current> seek;
+    } m_header;
+
+    struct NameEntry : public BigDNA
+    {
+        String<-1> name;
+        HECL::FourCC type;
+        UniqueID64 id;
+    };
+
+    struct Entry : public BigDNA
+    {
+        atUint32 compressed;
+        HECL::FourCC type;
+        UniqueID64 id;
+        atUint32 size;
+        atUint32 offset;
+    };
+
+private:
+    std::vector<NameEntry> m_nameEntries;
+    std::vector<Entry> m_entries;
+    std::unordered_map<UniqueID64, Entry*> m_idMap;
+    std::unordered_map<std::string, Entry*> m_nameMap;
+    size_t m_dataOffset = 0;
+    Delete expl;
+
+public:
+    void read(Athena::io::IStreamReader& reader)
+    {
+        reader.setEndian(Athena::BigEndian);
+        m_header.read(reader);
+        if (m_header.version != 2)
+            LogModule.report(LogVisor::FatalError, "unexpected PAK magic");
+
+        reader.seek(8, Athena::Current);
+        atUint32 strgSz = reader.readUint32();
+        reader.seek(4, Athena::Current);
+        atUint32 rshdSz = reader.readUint32();
+        reader.seek(44, Athena::Current);
+        m_dataOffset = 128 + strgSz + rshdSz;
+
+        atUint32 nameCount = reader.readUint32();
+        m_nameEntries.clear();
+        m_nameEntries.reserve(nameCount);
+        for (atUint32 n=0 ; n<nameCount ; ++n)
+        {
+            m_nameEntries.emplace_back();
+            m_nameEntries.back().read(reader);
+        }
+        reader.seek((reader.position() + 63) & ~63, Athena::Begin);
+
+        atUint32 count = reader.readUint32();
+        m_entries.clear();
+        m_entries.reserve(count);
+        m_idMap.clear();
+        m_idMap.reserve(count);
+        for (atUint32 e=0 ; e<count ; ++e)
+        {
+            m_entries.emplace_back();
+            m_entries.back().read(reader);
+            m_idMap[m_entries.back().id] = &m_entries.back();
+        }
+
+        m_nameMap.clear();
+        m_nameMap.reserve(nameCount);
+        for (NameEntry& entry : m_nameEntries)
+        {
+            Entry* found = m_idMap.find(entry.id);
+            if (found != m_idMap.end())
+                m_nameMap[entry.name] = found;
+        }
+    }
+    void write(Athena::io::IStreamWriter& writer)
+    {
+        writer.setEndian(Athena::BigEndian);
+        m_header.write(writer);
+
+        HECL::FourCC("STRG").write(writer);
+        atUint32 strgSz = 4;
+        for (NameEntry& entry : m_nameEntries)
+            strgSz += entry.name.size() + 13;
+        atUint32 strgPad = ((strgSz + 63) & ~63) - strgSz;
+        strgSz += strgPad;
+        writer.writeUint32(strgSz);
+
+        HECL::FourCC("RSHD").write(writer);
+        atUint32 rshdSz = 4 + 24 * m_entries.size();
+        atUint32 rshdPad = ((rshdSz + 63) & ~63) - rshdSz;
+        rshdSz += rshdPad;
+        writer.writeUint32(rshdSz);
+
+        HECL::FourCC("DATA").write(writer);
+        atUint32 dataSz = 0;
+        for (Entry& entry : m_entries)
+            dataSz += (entry.size + 63) & ~63;
+        atUint32 dataPad = ((dataSz + 63) & ~63) - dataSz;
+        dataSz += dataPad;
+        writer.writeUint32(dataSz);
+        writer.seek(36, Athena::Current);
+
+        writer.writeUint32(m_nameEntries.size());
+        for (NameEntry& entry : m_nameEntries)
+            entry.write(writer);
+        writer.seek(strgPad, Athena::Current);
+
+        writer.writeUint32(m_entries.size());
+        for (Entry& entry : m_entries)
+            entry.write(writer);
+        writer.seek(rshdPad, Athena::Current);
+    }
+
+    inline const Entry* lookupEntry(const UniqueID64& id) const
+    {
+        Entry* result = m_idMap.find(id);
+        if (result != m_idMap.end())
+            return result;
+        return nullptr;
+    }
+
+    inline const Entry* lookupEntry(const std::string& name) const
+    {
+        Entry* result = m_nameMap.find(name);
+        if (result != m_nameMap.end())
+            return result;
+        return nullptr;
+    }
+
+    inline size_t getDataOffset() const {return m_dataOffset;}
+};
+
+}
+}
+
+#endif // __DNAMP3_PAK_HPP__

DataSpec/DataSpec.pro

@@ -10,14 +10,13 @@ INCLUDEPATH += ../../../include \
     ../../LogVisor/include \
     ../NODLib/include
 
-include(../PakLib/PakLib.pri)
-
 include(DNACommon/DNACommon.pri)
 include(DNAMP1/DNAMP1.pri)
 include(DNAMP2/DNAMP2.pri)
 include(DNAMP3/DNAMP3.pri)
 
 HEADERS += \
+    Logging.hpp \
     SpecBase.hpp
 
 SOURCES += \

DataSpec/Logging.hpp

@@ -0,0 +1,11 @@
+#ifndef __RETRO_DATASPEC_LOGGING__
+#define __RETRO_DATASPEC_LOGGING__
+
+#include <LogVisor/LogVisor.hpp>
+
+namespace Retro
+{
+extern LogVisor::LogModule LogModule;
+}
+
+#endif // __RETRO_DATASPEC_LOGGING__

DataSpec/SpecBase.cpp

@@ -3,13 +3,15 @@
 namespace Retro
 {
 
-bool SpecBase::canExtract(const ExtractPassInfo& info, HECL::SystemString& reasonNo)
+LogVisor::LogModule LogModule("RetroDataSpec");
+
+bool SpecBase::canExtract(const ExtractPassInfo& info)
 {
     bool isWii;
     std::unique_ptr<NOD::DiscBase> disc = NOD::OpenDiscFromImage(info.srcpath.c_str(), isWii);
     if (!disc)
     {
-        reasonNo = _S("Not a valid Nintendo disc image");
+        LogModule.report(LogVisor::Error, _S("'%s' not a valid Nintendo disc image"), info.srcpath.c_str());
         return false;
     }
     const char* gameID = disc->getHeader().gameID;
@@ -29,9 +31,7 @@ bool SpecBase::canExtract(const ExtractPassInfo& info, HECL::SystemString& reaso
             return true;
     }
 
-    HECL::SystemStringView gameIDView(std::string(gameID, 6));
+    LogModule.report(LogVisor::Error, "%.6s (%s) is not supported", gameID, disc->getHeader().gameTitle);
-    HECL::SystemStringView gameNameView(disc->getHeader().gameTitle);
-    reasonNo = gameIDView.sys_str() + _S(" (") + gameNameView.sys_str() + _S(") is not supported");
     return false;
 }
 
@@ -39,8 +39,7 @@ void SpecBase::doExtract(const HECL::Database::Project& project, const ExtractPa
 {
 }
 
-bool SpecBase::canCook(const HECL::Database::Project& project, const CookTaskInfo& info,
+bool SpecBase::canCook(const HECL::Database::Project& project, const CookTaskInfo& info)
-                       HECL::SystemString& reasonNo)
 {
 }
 
@@ -48,8 +47,7 @@ void SpecBase::doCook(const HECL::Database::Project& project, const CookTaskInfo
 {
 }
 
-bool SpecBase::canPackage(const HECL::Database::Project& project, const PackagePassInfo& info,
+bool SpecBase::canPackage(const HECL::Database::Project& project, const PackagePassInfo& info)
-                          HECL::SystemString& reasonNo)
 {
 }
 

DataSpec/SpecBase.hpp

@@ -9,17 +9,17 @@
 namespace Retro
 {
 
+extern LogVisor::LogModule LogModule;
+
 struct SpecBase : public HECL::Database::IDataSpec
 {
-    bool canExtract(const ExtractPassInfo& info, HECL::SystemString& reasonNo);
+    bool canExtract(const ExtractPassInfo& info);
     void doExtract(const HECL::Database::Project& project, const ExtractPassInfo& info);
 
-    bool canCook(const HECL::Database::Project& project, const CookTaskInfo& info,
+    bool canCook(const HECL::Database::Project& project, const CookTaskInfo& info);
-                 HECL::SystemString& reasonNo);
     void doCook(const HECL::Database::Project& project, const CookTaskInfo& info);
 
-    bool canPackage(const HECL::Database::Project& project, const PackagePassInfo& info,
+    bool canPackage(const HECL::Database::Project& project, const PackagePassInfo& info);
-                    HECL::SystemString& reasonNo);
     void gatherDependencies(const HECL::Database::Project& project, const PackagePassInfo& info,
                             std::unordered_set<HECL::ProjectPath>& implicitsOut);
     void doPackage(const HECL::Database::Project& project, const PackagePassInfo& info);

DataSpec/SpecMP1.cpp

@@ -49,12 +49,12 @@ struct SpecMP1 : public SpecBase
     }
 };
 
-static HECL::Database::DataSpecEntry SpecMP1
+HECL::Database::DataSpecEntry SpecMP1 =
-(
+{
     _S("MP1"),
     _S("Data specification for original Metroid Prime engine"),
     [](HECL::Database::DataSpecTool) -> HECL::Database::IDataSpec* {return new struct SpecMP1;}
-);
+};
 
 }
 

NODLib

@@ -1 +1 @@
-Subproject commit f093f633b44270c6b52834d310407f9faeb18b76
+Subproject commit c2227fe826ef8009dba0d7ff3a2c6e5b19b56a2b

PakLib

@@ -1 +0,0 @@
-Subproject commit 6a71c1fdc4b176f1b14d83b87b57a519f8ba534a