Fully working PAK/MLVL/STRG reads

DataSpec/DNACommon/DNACommon.hpp

@@ -3,6 +3,7 @@
 
 #include <Athena/DNA.hpp>
 #include "HECL/HECL.hpp"
+#include "../Logging.hpp"
 
 namespace Retro
 {
@@ -124,20 +125,58 @@ public:
     }
 };
 
+/* Case-insensitive comparator for std::map sorting */
 struct CaseInsensitiveCompare
 {
     inline bool operator()(const std::string& lhs, const std::string& rhs) const
     {
-        std::string lhsl = lhs;
+#if _WIN32
-        std::transform(lhsl.begin(), lhsl.end(), lhsl.begin(), tolower);
+        if (stricmp(lhs.c_str(), rhs.c_str()) < 0)
-        std::string rhsl = rhs;
+#else
-        std::transform(rhsl.begin(), rhsl.end(), rhsl.begin(), tolower);
+        if (strcasecmp(lhs.c_str(), rhs.c_str()) < 0)
-        if (lhsl.compare(rhsl) < 0)
+#endif
             return true;
         return false;
     }
 };
 
+/* PAK entry stream reader */
+class PAKEntryReadStream : public Athena::io::IStreamReader
+{
+    std::unique_ptr<atUint8[]> m_buf;
+    atUint64 m_sz;
+    atUint64 m_pos;
+public:
+    PAKEntryReadStream(std::unique_ptr<atUint8[]>&& buf, atUint64 sz, atUint64 pos)
+    : m_buf(std::move(buf)), m_sz(sz), m_pos(pos)
+    {
+        if (m_pos >= m_sz)
+            LogModule.report(LogVisor::FatalError, "PAK stream cursor overrun");
+    }
+    inline void seek(atInt64 pos, Athena::SeekOrigin origin)
+    {
+        if (origin == Athena::Begin)
+            m_pos = pos;
+        else if (origin == Athena::Current)
+            m_pos += pos;
+        else if (origin == Athena::End)
+            m_pos = m_sz + pos;
+        if (m_pos >= m_sz)
+            LogModule.report(LogVisor::FatalError, "PAK stream cursor overrun");
+    }
+    inline atUint64 position() const {return m_pos;}
+    inline atUint64 length() const {return m_sz;}
+    inline atUint64 readUBytesToBuf(void* buf, atUint64 len)
+    {
+        atUint64 bufEnd = m_pos + len;
+        if (bufEnd > m_sz)
+            len -= bufEnd - m_sz;
+        memcpy(buf, m_buf.get() + m_pos, len);
+        m_pos += len;
+        return len;
+    }
+};
+
 /* Language-identifiers */
 extern const HECL::FourCC ENGL;
 extern const HECL::FourCC FREN;

DataSpec/DNAMP1/MLVL.hpp

@@ -37,7 +37,7 @@ struct MLVL : BigDNA
         Value<atVec4f> transformMtx[3];
         Value<atVec3f> aabb[2];
         UniqueID32 areaMREAId;
-        Value<atUint32> areaId;
+        UniqueID32 areaId;
 
         Value<atUint32> attachedAreaCount;
         Vector<atUint16, DNA_COUNT(attachedAreaCount)> attachedAreas;

DataSpec/DNAMP1/PAK.cpp

@@ -1,3 +1,5 @@
+#include <zlib.h>
+#include <lzo/lzo1x.h>
 #include "PAK.hpp"
 
 namespace Retro
@@ -63,5 +65,68 @@ void PAK::write(Athena::io::IStreamWriter& writer) const
         entry.write(writer);
 }
 
+std::unique_ptr<atUint8[]> PAK::Entry::getBuffer(const NOD::DiscBase::IPartition::Node& pak, atUint64& szOut) const
+{
+    if (compressed)
+    {
+        std::unique_ptr<NOD::IPartReadStream> strm = pak.beginReadStream(offset);
+
+        atUint32 decompSz;
+        strm->read(&decompSz, 4);
+        decompSz = HECL::SBig(decompSz);
+        atUint8* buf = new atUint8[decompSz];
+        atUint8* bufCur = buf;
+
+        atUint16 zlibCheck;
+        strm->read(&zlibCheck, 2);
+        zlibCheck = HECL::SBig(zlibCheck);
+        strm->seek(-2, SEEK_CUR);
+
+        atUint8 compBuf[0x4000];
+        if ((zlibCheck % 31) == 0)
+        {
+            atUint32 compRem = size - 4;
+            z_stream zs;
+            inflateInit(&zs);
+            zs.avail_out = decompSz;
+            zs.next_out = buf;
+            while (zs.avail_out)
+            {
+                atUint64 readSz = strm->read(compBuf, MIN(compRem, 0x4000));
+                compRem -= readSz;
+                zs.avail_in = readSz;
+                zs.next_in = compBuf;
+                inflate(&zs, Z_FINISH);
+            }
+            inflateEnd(&zs);
+        }
+        else
+        {
+            atUint32 rem = decompSz;
+            while (rem)
+            {
+                atUint16 chunkSz;
+                strm->read(&chunkSz, 2);
+                chunkSz = HECL::SBig(chunkSz);
+                strm->read(compBuf, chunkSz);
+                lzo_uint dsz = rem;
+                lzo1x_decompress(compBuf, chunkSz, bufCur, &dsz, nullptr);
+                bufCur += dsz;
+                rem -= dsz;
+            }
+        }
+
+        szOut = decompSz;
+        return std::unique_ptr<atUint8[]>(buf);
+    }
+    else
+    {
+        atUint8* buf = new atUint8[size];
+        pak.beginReadStream(offset)->read(buf, size);
+        szOut = size;
+        return std::unique_ptr<atUint8[]>(buf);
+    }
+}
+
 }
 }

DataSpec/DNAMP1/PAK.hpp

@@ -3,6 +3,7 @@
 
 #include <unordered_map>
 
+#include <NOD/DiscBase.hpp>
 #include "../Logging.hpp"
 #include "../DNACommon/DNACommon.hpp"
 
@@ -32,6 +33,13 @@ struct PAK : BigDNA
         UniqueID32 id;
         Value<atUint32> size;
         Value<atUint32> offset;
+
+        std::unique_ptr<atUint8[]> getBuffer(const NOD::DiscBase::IPartition::Node& pak, atUint64& szOut) const;
+        inline PAKEntryReadStream beginReadStream(const NOD::DiscBase::IPartition::Node& pak, atUint64 off=0) const
+        {
+            atUint64 sz;
+            return PAKEntryReadStream(getBuffer(pak, sz), sz, off);
+        }
     };
 
     std::vector<NameEntry> m_nameEntries;

DataSpec/DNAMP2/MLVL.hpp

@@ -27,7 +27,7 @@ struct MLVL : BigDNA
         Value<atVec4f> transformMtx[3];
         Value<atVec3f> aabb[2];
         UniqueID32 areaMREAId;
-        Value<atUint32> areaId;
+        UniqueID32 areaId;
 
         Value<atUint32> attachedAreaCount;
         Vector<atUint16, DNA_COUNT(attachedAreaCount)> attachedAreas;
@@ -56,7 +56,6 @@ struct MLVL : BigDNA
                 Value<atUint32> areaIdx;
                 Value<atUint32> dockIdx;
             };
-            FourCC type;
             Vector<Endpoint, DNA_COUNT(endpointCount)> endpoints;
 
             Value<atUint32> planeVertCount;

DataSpec/DNAMP3/MLVL.hpp

@@ -26,7 +26,7 @@ struct MLVL : BigDNA
         Value<atVec4f> transformMtx[3];
         Value<atVec3f> aabb[2];
         UniqueID64 areaMREAId;
-        Value<atUint32> areaId;
+        UniqueID64 areaId;
 
         Value<atUint32> attachedAreaCount;
         Vector<atUint16, DNA_COUNT(attachedAreaCount)> attachedAreas;
@@ -42,7 +42,6 @@ struct MLVL : BigDNA
                 Value<atUint32> areaIdx;
                 Value<atUint32> dockIdx;
             };
-            FourCC type;
             Vector<Endpoint, DNA_COUNT(endpointCount)> endpoints;
 
             Value<atUint32> planeVertCount;
@@ -52,6 +51,7 @@ struct MLVL : BigDNA
 
         String<-1> internalAreaName;
     };
+    Vector<Area, DNA_COUNT(areaCount)> areas;
 
     UniqueID64 worldMap;
     Value<atUint8> unknown2;

DataSpec/DNAMP3/PAK.cpp

@@ -5,6 +5,8 @@ namespace Retro
 namespace DNAMP3
 {
 
+const HECL::FourCC CMPD("CMPD");
+
 void PAK::read(Athena::io::IStreamReader& reader)
 {
     reader.setEndian(Athena::BigEndian);
@@ -17,7 +19,7 @@ void PAK::read(Athena::io::IStreamReader& reader)
     reader.seek(4, Athena::Current);
     atUint32 rshdSz = reader.readUint32();
     reader.seek(44, Athena::Current);
-    m_dataOffset = 128 + strgSz + rshdSz;
+    atUint32 dataOffset = 128 + strgSz + rshdSz;
 
     atUint64 strgBase = reader.position();
     atUint32 nameCount = reader.readUint32();
@@ -42,6 +44,7 @@ void PAK::read(Athena::io::IStreamReader& reader)
     {
         m_entries.emplace_back();
         m_entries.back().read(reader);
+        m_entries.back().offset += dataOffset;
     }
     for (Entry& entry : m_entries)
         m_idMap[entry.id] = &entry;
@@ -73,6 +76,7 @@ void PAK::write(Athena::io::IStreamWriter& writer) const
     atUint32 rshdPad = ((rshdSz + 63) & ~63) - rshdSz;
     rshdSz += rshdPad;
     writer.writeUint32(rshdSz);
+    atUint32 dataOffset = 128 + strgSz + rshdSz;
 
     FourCC("DATA").write(writer);
     atUint32 dataSz = 0;
@@ -90,9 +94,91 @@ void PAK::write(Athena::io::IStreamWriter& writer) const
 
     writer.writeUint32(m_entries.size());
     for (const Entry& entry : m_entries)
-        entry.write(writer);
+    {
+        Entry copy = entry;
+        copy.offset -= dataOffset;
+        copy.write(writer);
+    }
     writer.seek(rshdPad, Athena::Current);
 }
 
+std::unique_ptr<atUint8[]> PAK::Entry::getBuffer(const NOD::DiscBase::IPartition::Node& pak, atUint64& szOut) const
+{
+    if (compressed)
+    {
+        std::unique_ptr<NOD::IPartReadStream> strm = pak.beginReadStream(offset);
+        struct
+        {
+            HECL::FourCC magic;
+            atUint32 blockCount;
+        } head;
+        strm->read(&head, 8);
+        if (head.magic != CMPD)
+        {
+            LogModule.report(LogVisor::Error, "invalid CMPD block");
+            return std::unique_ptr<atUint8[]>();
+        }
+        head.blockCount = HECL::SBig(head.blockCount);
+
+        struct Block
+        {
+            atUint32 compSz;
+            atUint32 decompSz;
+        } blocks[head.blockCount];
+        strm->read(blocks, 8 * head.blockCount);
+
+        atUint64 maxBlockSz = 0;
+        atUint64 totalDecompSz = 0;
+        for (atUint32 b=0 ; b<head.blockCount ; ++b)
+        {
+            Block& block = blocks[b];
+            block.compSz = HECL::SBig(block.compSz) & 0xffffff;
+            block.decompSz = HECL::SBig(block.decompSz);
+            if (block.compSz > maxBlockSz)
+                maxBlockSz = block.compSz;
+            totalDecompSz += block.decompSz;
+        }
+
+        std::unique_ptr<atUint8[]> compBuf(new atUint8[maxBlockSz]);
+        atUint8* buf = new atUint8[totalDecompSz];
+        atUint8* bufCur = buf;
+        for (atUint32 b=0 ; b<head.blockCount ; ++b)
+        {
+            Block& block = blocks[b];
+            atUint8* compBufCur = compBuf.get();
+            strm->read(compBufCur, block.compSz);
+            if (block.compSz == block.decompSz)
+            {
+                memcpy(bufCur, compBufCur, block.decompSz);
+                bufCur += block.decompSz;
+            }
+            else
+            {
+                atUint32 rem = block.decompSz;
+                while (rem)
+                {
+                    atUint16 chunkSz = HECL::SBig(*(atUint16*)compBufCur);
+                    compBufCur += 2;
+                    lzo_uint dsz = rem;
+                    lzo1x_decompress(compBufCur, chunkSz, bufCur, &dsz, nullptr);
+                    compBufCur += chunkSz;
+                    bufCur += dsz;
+                    rem -= dsz;
+                }
+            }
+        }
+
+        szOut = totalDecompSz;
+        return std::unique_ptr<atUint8[]>(buf);
+    }
+    else
+    {
+        atUint8* buf = new atUint8[size];
+        pak.beginReadStream(offset)->read(buf, size);
+        szOut = size;
+        return std::unique_ptr<atUint8[]>(buf);
+    }
+}
+
 }
 }

DataSpec/DNAMP3/PAK.hpp

@@ -3,14 +3,18 @@
 
 #include <unordered_map>
 
-#include "../Logging.hpp"
+#include <lzo/lzo1x.h>
+#include <NOD/DiscBase.hpp>
 #include "../DNACommon/DNACommon.hpp"
+#include "../Logging.hpp"
 
 namespace Retro
 {
 namespace DNAMP3
 {
 
+extern const HECL::FourCC CMPD;
+
 struct PAK : BigDNA
 {
     struct Header : BigDNA
@@ -38,13 +42,19 @@ struct PAK : BigDNA
         UniqueID64 id;
         Value<atUint32> size;
         Value<atUint32> offset;
+
+        std::unique_ptr<atUint8[]> getBuffer(const NOD::DiscBase::IPartition::Node& pak, atUint64& szOut) const;
+        inline PAKEntryReadStream beginReadStream(const NOD::DiscBase::IPartition::Node& pak, atUint64 off=0) const
+        {
+            atUint64 sz;
+            return PAKEntryReadStream(getBuffer(pak, sz), sz, off);
+        }
     };
 
     std::vector<NameEntry> m_nameEntries;
     std::vector<Entry> m_entries;
     std::unordered_map<UniqueID64, Entry*> m_idMap;
     std::unordered_map<std::string, Entry*> m_nameMap;
-    size_t m_dataOffset = 0;
 
     DECL_EXPLICIT_DNA
 
@@ -63,8 +73,6 @@ struct PAK : BigDNA
             return result->second;
         return nullptr;
     }
-
-    inline size_t getDataOffset() const {return m_dataOffset;}
 };
 
 }

DataSpec/DNAMP3/STRG.cpp

@@ -13,17 +13,20 @@ void STRG::_read(Athena::io::IStreamReader& reader)
 
     atUint32 nameCount = reader.readUint32();
     atUint32 nameTableSz = reader.readUint32();
-    std::unique_ptr<uint8_t[]> nameTableBuf(new uint8_t[nameTableSz]);
+    if (nameTableSz)
-    reader.readUBytesToBuf(nameTableBuf.get(), nameTableSz);
-    struct NameIdxEntry
     {
-        atUint32 nameOff;
+        std::unique_ptr<uint8_t[]> nameTableBuf(new uint8_t[nameTableSz]);
-        atUint32 strIdx;
+        reader.readUBytesToBuf(nameTableBuf.get(), nameTableSz);
-    }* nameIndex = (NameIdxEntry*)nameTableBuf.get();
+        struct NameIdxEntry
-    for (atUint32 n=0 ; n<nameCount ; ++n)
+        {
-    {
+            atUint32 nameOff;
-        const char* name = (char*)(nameTableBuf.get() + HECL::SBig(nameIndex[n].nameOff));
+            atUint32 strIdx;
-        names[name] = HECL::SBig(nameIndex[n].strIdx);
+        }* nameIndex = (NameIdxEntry*)nameTableBuf.get();
+        for (atUint32 n=0 ; n<nameCount ; ++n)
+        {
+            const char* name = (char*)(nameTableBuf.get() + HECL::SBig(nameIndex[n].nameOff));
+            names[name] = HECL::SBig(nameIndex[n].strIdx);
+        }
     }
 
     std::vector<FourCC> readLangs;
@@ -33,20 +36,28 @@ void STRG::_read(Athena::io::IStreamReader& reader)
         FourCC lang;
         lang.read(reader);
         readLangs.emplace_back(lang);
-        reader.seek(strCount * 4 + 4);
+    }
+    std::unique_ptr<atUint32[]> strOffs(new atUint32[langCount * strCount]);
+    for (atUint32 l=0 ; l<langCount ; ++l)
+    {
+        reader.readUint32();
+        for (atUint32 s=0 ; s<strCount ; ++s)
+            strOffs[l*strCount+s] = reader.readUint32();
     }
 
+    atUint64 strBase = reader.position();
     langs.clear();
     langs.reserve(langCount);
-    for (FourCC& lang : readLangs)
+    for (atUint32 l=0 ; l<langCount ; ++l)
     {
         std::vector<std::string> strs;
         for (atUint32 s=0 ; s<strCount ; ++s)
         {
+            reader.seek(strBase + strOffs[l*strCount+s], Athena::Begin);
             atUint32 len = reader.readUint32();
             strs.emplace_back(reader.readString(len));
         }
-        langs.emplace(lang, strs);
+        langs.emplace(readLangs[l], strs);
     }
 }
 
@@ -55,11 +66,17 @@ void STRG::read(Athena::io::IStreamReader& reader)
     reader.setEndian(Athena::BigEndian);
     atUint32 magic = reader.readUint32();
     if (magic != 0x87654321)
+    {
         LogModule.report(LogVisor::Error, "invalid STRG magic");
+        return;
+    }
 
     atUint32 version = reader.readUint32();
     if (version != 3)
+    {
         LogModule.report(LogVisor::Error, "invalid STRG version");
+        return;
+    }
 
     _read(reader);
 }
@@ -74,43 +91,53 @@ void STRG::write(Athena::io::IStreamWriter& writer) const
     writer.writeUint32(strCount);
 
     atUint32 nameTableSz = names.size() * 8;
-    for (const std::pair<std::string, int32_t>& name : names)
+    for (const auto& name : names)
         nameTableSz += name.first.size() + 1;
     writer.writeUint32(names.size());
     writer.writeUint32(nameTableSz);
     atUint32 offset = names.size() * 8;
-    for (const std::pair<std::string, int32_t>& name : names)
+    for (const auto& name : names)
     {
         writer.writeUint32(offset);
         writer.writeInt32(name.second);
         offset += name.first.size() + 1;
     }
-    for (const std::pair<std::string, int32_t>& name : names)
+    for (const auto& name : names)
         writer.writeString(name.first);
 
-    offset = 0;
+    for (const auto& lang : langs)
-    for (const std::pair<FourCC, std::vector<std::string>>& lang : langs)
-    {
         lang.first.write(writer);
 
+    offset = 0;
+    for (const auto& lang : langs)
+    {
         atUint32 langSz = 0;
         for (const std::string& str : lang.second)
-            langSz += str.size() + 4;
+            langSz += str.size() + 5;
         writer.writeUint32(langSz);
 
         for (const std::string& str : lang.second)
         {
             writer.writeUint32(offset);
-            offset += str.size() + 4;
+            offset += str.size() + 5;
         }
     }
 
-    for (const std::pair<FourCC, std::vector<std::string>>& lang : langs)
+    for (atUint32 s=0 ; s<strCount ; ++s)
     {
-        for (const std::string& str : lang.second)
+        for (const auto& lang : langs)
         {
-            writer.writeUint32(str.size());
+            if (s >= lang.second.size())
-            writer.writeString(str, str.size());
+            {
+                writer.writeUint32(1);
+                writer.writeUByte(0);
+            }
+            else
+            {
+                const std::string& str = lang.second[s];
+                writer.writeUint32(str.size() + 1);
+                writer.writeString(str);
+            }
         }
     }
 }

DataSpec/SpecMP1.cpp

@@ -99,14 +99,14 @@ struct SpecMP1 : SpecBase
             {
                 if (entry.type == MLVL)
                 {
-                    NOD::AthenaPartReadStream rs(item.second->node.beginReadStream(entry.offset));
+                    PAKEntryReadStream rs = entry.beginReadStream(item.second->node);
                     DNAMP1::MLVL mlvl;
                     mlvl.read(rs);
                     const DNAMP1::PAK::Entry* nameEnt = pak.lookupEntry(mlvl.worldNameId);
                     if (nameEnt)
                     {
+                        PAKEntryReadStream rs = nameEnt->beginReadStream(item.second->node);
                         DNAMP1::STRG mlvlName;
-                        NOD::AthenaPartReadStream rs(item.second->node.beginReadStream(nameEnt->offset));
                         mlvlName.read(rs);
                         if (childRep.desc.size())
                             childRep.desc += _S(", ");

DataSpec/SpecMP2.cpp

@@ -99,14 +99,14 @@ struct SpecMP2 : SpecBase
             {
                 if (entry.type == MLVL)
                 {
-                    NOD::AthenaPartReadStream rs(item.second->node.beginReadStream(entry.offset));
+                    PAKEntryReadStream rs = entry.beginReadStream(item.second->node);
                     DNAMP2::MLVL mlvl;
                     mlvl.read(rs);
                     const DNAMP1::PAK::Entry* nameEnt = pak.lookupEntry(mlvl.worldNameId);
                     if (nameEnt)
                     {
+                        PAKEntryReadStream rs = nameEnt->beginReadStream(item.second->node);
                         DNAMP2::STRG mlvlName;
-                        NOD::AthenaPartReadStream rs(item.second->node.beginReadStream(nameEnt->offset));
                         mlvlName.read(rs);
                         if (childRep.desc.size())
                             childRep.desc += _S(", ");

DataSpec/SpecMP3.cpp

@@ -1,4 +1,5 @@
 #include <utility>
+#include <set>
 
 #define NOD_ATHENA 1
 #include "SpecBase.hpp"
@@ -93,27 +94,40 @@ struct SpecMP3 : SpecBase
             rep.childOpts.emplace_back();
             ExtractReport& childRep = rep.childOpts.back();
             childRep.name = item.first;
+            if (!item.first.compare("Worlds.pak"))
+                continue;
+            else if (!item.first.compare("Metroid6.pak"))
+            {
+                childRep.desc = _S("Phaaze");
+                continue;
+            }
 
+            std::set<HECL::SystemString> worldNames;
             DNAMP3::PAK& pak = item.second->pak;
             for (DNAMP3::PAK::Entry& entry : pak.m_entries)
             {
                 if (entry.type == MLVL)
                 {
-                    NOD::AthenaPartReadStream rs(item.second->node.beginReadStream(entry.offset));
+                    PAKEntryReadStream rs = entry.beginReadStream(item.second->node);
                     DNAMP3::MLVL mlvl;
                     mlvl.read(rs);
                     const DNAMP3::PAK::Entry* nameEnt = pak.lookupEntry(mlvl.worldNameId);
                     if (nameEnt)
                     {
+                        PAKEntryReadStream rs = nameEnt->beginReadStream(item.second->node);
                         DNAMP3::STRG mlvlName;
-                        NOD::AthenaPartReadStream rs(item.second->node.beginReadStream(nameEnt->offset));
                         mlvlName.read(rs);
-                        if (childRep.desc.size())
+                        worldNames.emplace(mlvlName.getSystemString(ENGL, 0));
-                            childRep.desc += _S(", ");
-                        childRep.desc += mlvlName.getSystemString(ENGL, 0);
                     }
                 }
             }
+
+            for (const std::string& name : worldNames)
+            {
+                if (childRep.desc.size())
+                    childRep.desc += _S(", ");
+                childRep.desc += name;
+            }
         }
     }
 

NODLib

@@ -1 +1 @@
-Subproject commit 01f269e8e2a55d8b64e1e277d9a1f87f6877c446
+Subproject commit 83f29da294c05c99aba06dfa7a5a095326a1d101