MP3 DEPS support

DataSpec/DNACommon/DNACommon.hpp

@@ -70,7 +70,7 @@ public:
 };
 
 /* PAK 64-bit Unique ID */
-class UniqueID64 : public BigDNA
+class UniqueID64 : public BigYAML
 {
     uint64_t m_id = 0xffffffffffffffff;
 public:
@@ -80,6 +80,10 @@ public:
     {m_id = reader.readUint64Big();}
     void write(Athena::io::IStreamWriter& writer) const
     {writer.writeUint64Big(m_id);}
+    void fromYAML(Athena::io::YAMLDocReader& reader)
+    {m_id = reader.readUint64(nullptr);}
+    void toYAML(Athena::io::YAMLDocWriter& writer) const
+    {writer.writeUint64(nullptr, m_id);}
 
     bool operator!=(const UniqueID64& other) const {return m_id != other.m_id;}
     bool operator==(const UniqueID64& other) const {return m_id == other.m_id;}
@@ -93,7 +97,7 @@ public:
 };
 
 /* PAK 128-bit Unique ID */
-class UniqueID128 : public BigDNA
+class UniqueID128 : public BigYAML
 {
     union
     {
@@ -117,6 +121,20 @@ public:
         writer.writeUint64Big(m_id[0]);
         writer.writeUint64Big(m_id[1]);
     }
+    void fromYAML(Athena::io::YAMLDocReader& reader)
+    {
+        std::string str = reader.readString(nullptr);
+        while (str.size() < 32)
+            str += '0';
+        std::string hStr(str.begin(), str.begin() + 16);
+        std::string lStr(str.begin() + 16, str.begin() + 32);
+        m_id[0] = strtoull(hStr.c_str(), nullptr, 16);
+        m_id[1] = strtoull(lStr.c_str(), nullptr, 16);
+    }
+    void toYAML(Athena::io::YAMLDocWriter& writer) const
+    {
+        writer.writeString(nullptr, toString().c_str());
+    }
 
     bool operator!=(const UniqueID128& other) const
     {

DataSpec/DNAMP2/DNAMP2.cpp

@@ -264,6 +264,8 @@ ResExtractor<PAKBridge> PAKBridge::LookupExtractor(const DNAMP1::PAK::Entry& ent
         return {nullptr, ANCS::Extract, {_S(".yaml"), _S(".blend")}, 2};
     case SBIG('MREA'):
         return {nullptr, MREA::Extract, {_S(".blend")}, 3};
+    case SBIG('MLVL'):
+        return {MLVL::Extract, nullptr, {_S(".yaml")}};
     }
     return {};
 }

DataSpec/DNAMP2/MLVL.hpp

@@ -1,16 +1,16 @@
 #ifndef __DNAMP2_MLVL_HPP__
 #define __DNAMP2_MLVL_HPP__
 
-#include "../DNACommon/DNACommon.hpp"
+#include "../DNACommon/PAK.hpp"
 
 namespace Retro
 {
 namespace DNAMP2
 {
 
-struct MLVL : BigDNA
+struct MLVL : BigYAML
 {
-    DECL_DNA
+    DECL_YAML
     Value<atUint32> magic;
     Value<atUint32> version;
     UniqueID32 worldNameId;
@@ -20,9 +20,9 @@ struct MLVL : BigDNA
     UniqueID32 worldSkyboxId;
 
     Value<atUint32> areaCount;
-    struct Area : BigDNA
+    struct Area : BigYAML
     {
-        DECL_DNA
+        DECL_YAML
         UniqueID32 areaNameId;
         Value<atVec4f> transformMtx[3];
         Value<atVec3f> aabb[2];
@@ -34,9 +34,9 @@ struct MLVL : BigDNA
         Value<atUint32> padding;
 
         Value<atUint32> depCount;
-        struct Dependency : BigDNA
+        struct Dependency : BigYAML
         {
-            DECL_DNA
+            DECL_YAML
             UniqueID32 id;
             DNAFourCC type;
         };
@@ -46,13 +46,13 @@ struct MLVL : BigDNA
         Vector<atUint32, DNA_COUNT(depLayerCount)> depLayers;
 
         Value<atUint32> dockCount;
-        struct Dock : BigDNA
+        struct Dock : BigYAML
         {
-            DECL_DNA
+            DECL_YAML
             Value<atUint32> endpointCount;
-            struct Endpoint : BigDNA
+            struct Endpoint : BigYAML
             {
-                DECL_DNA
+                DECL_YAML
                 Value<atUint32> areaIdx;
                 Value<atUint32> dockIdx;
             };
@@ -77,9 +77,9 @@ struct MLVL : BigDNA
     Value<atUint32> unknown3;
 
     Value<atUint32> layerFlagCount;
-    struct LayerFlags : BigDNA
+    struct LayerFlags : BigYAML
     {
-        DECL_DNA
+        DECL_YAML
         Value<atUint32> layerCount;
         Value<atUint64> flags;
     };
@@ -90,6 +90,16 @@ struct MLVL : BigDNA
 
     Value<atUint32> layerNameOffsetCount;
     Vector<atUint32, DNA_COUNT(layerNameOffsetCount)> layerNameOffsets;
+
+    static bool Extract(PAKEntryReadStream& rs, const HECL::ProjectPath& outPath)
+    {
+        MLVL mlvl;
+        mlvl.read(rs);
+        FILE* fp = HECL::Fopen(outPath.getAbsolutePath().c_str(), _S("wb"));
+        mlvl.toYAMLFile(fp);
+        fclose(fp);
+        return true;
+    }
 };
 
 }

DataSpec/DNAMP3/DNAMP3.cpp

@@ -58,8 +58,194 @@ PAKBridge::PAKBridge(HECL::Database::Project& project, const NOD::DiscBase::IPar
     }
 }
 
+UniqueResult PAKBridge::uniqueCheck(const PAK::Entry& entry)
+{
+    UniqueResult::Type result = UniqueResult::UNIQUE_NOTFOUND;
+    bool foundOneLayer = false;
+    UniqueID64 areaId;
+    unsigned layerIdx;
+    for (const auto& pair : m_areaDeps)
+    {
+        unsigned l=0;
+        for (const auto& layer : pair.second.layers)
+        {
+            if (layer.resources.find(entry.id) != layer.resources.end())
+            {
+                if (foundOneLayer)
+                {
+                    if (areaId == pair.first)
+                        result = UniqueResult::UNIQUE_AREA;
+                    else
+                        return {UniqueResult::UNIQUE_LEVEL};
+                    continue;
+                }
+                else
+                    result = UniqueResult::UNIQUE_LAYER;
+                areaId = pair.first;
+                layerIdx = l;
+                foundOneLayer = true;
+            }
+            ++l;
+        }
+        if (pair.second.resources.find(entry.id) != pair.second.resources.end())
+        {
+            if (foundOneLayer)
+            {
+                if (areaId == pair.first)
+                    result = UniqueResult::UNIQUE_AREA;
+                else
+                    return {UniqueResult::UNIQUE_LEVEL};
+                continue;
+            }
+            else
+                result = UniqueResult::UNIQUE_AREA;
+            areaId = pair.first;
+            foundOneLayer = true;
+        }
+    }
+    UniqueResult retval = {result};
+    if (result == UniqueResult::UNIQUE_LAYER || result == UniqueResult::UNIQUE_AREA)
+    {
+        const PAKBridge::Area& area = m_areaDeps[areaId];
+        retval.areaName = &area.name;
+        if (result == UniqueResult::UNIQUE_LAYER)
+        {
+            const PAKBridge::Area::Layer& layer = area.layers[layerIdx];
+            retval.layerName = &layer.name;
+        }
+    }
+    return retval;
+}
+
+static HECL::SystemString LayerName(const std::string& name)
+{
+#if HECL_UCS2
+    HECL::SystemString ret = HECL::UTF8ToWide(name);
+#else
+    HECL::SystemString ret = name;
+#endif
+    for (auto& ch : ret)
+        if (ch == _S('/') || ch == _S('\\'))
+            ch = _S('-');
+    return ret;
+}
+
 void PAKBridge::build()
 {
+    /* First pass: build per-area/per-layer dependency map */
+    for (const PAK::Entry& entry : m_pak.m_entries)
+    {
+        if (entry.type == FOURCC('MLVL'))
+        {
+            PAKEntryReadStream rs = entry.beginReadStream(m_node);
+            MLVL mlvl;
+            mlvl.read(rs);
+            m_areaDeps.reserve(mlvl.areaCount);
+            unsigned layerIdx = 0;
+
+            /* Pre-pass: find duplicate area names */
+            std::unordered_map<HECL::SystemString, std::pair<atUint32, atUint32>> dupeTracker;
+            dupeTracker.reserve(mlvl.areas.size());
+            for (const MLVL::Area& area : mlvl.areas)
+            {
+                const PAK::Entry* areaNameEnt = m_pak.lookupEntry(area.areaNameId);
+                if (areaNameEnt)
+                {
+                    STRG areaName;
+                    PAKEntryReadStream rs = areaNameEnt->beginReadStream(m_node);
+                    areaName.read(rs);
+                    HECL::SystemString name = areaName.getSystemString(FOURCC('ENGL'), 0);
+                    auto search = dupeTracker.find(name);
+                    if (search != dupeTracker.end())
+                        ++search->second.first;
+                    else
+                        dupeTracker[name] = std::make_pair(1, 1);
+                }
+            }
+
+            /* Main-pass: index areas */
+            auto layerFlagsIt = mlvl.layerFlags.begin();
+            for (const MLVL::Area& area : mlvl.areas)
+            {
+                Area& areaDeps = m_areaDeps[area.areaMREAId];
+                const PAK::Entry* areaNameEnt = m_pak.lookupEntry(area.areaNameId);
+                if (areaNameEnt)
+                {
+                    STRG areaName;
+                    PAKEntryReadStream rs = areaNameEnt->beginReadStream(m_node);
+                    areaName.read(rs);
+                    areaDeps.name = areaName.getSystemString(FOURCC('ENGL'), 0);
+                    auto search = dupeTracker.find(areaDeps.name);
+                    if (search != dupeTracker.end() && search->second.first > 1)
+                    {
+                        char num[16];
+                        snprintf(num, 16, " (%d)", search->second.second++);
+                        areaDeps.name += num;
+                    }
+
+                    /* Trim possible trailing whitespace */
+#if HECL_UCS2
+                    while (areaDeps.name.size() && iswblank(areaDeps.name.back()))
+                        areaDeps.name.pop_back();
+#else
+                    while (areaDeps.name.size() && isblank(areaDeps.name.back()))
+                        areaDeps.name.pop_back();
+#endif
+                }
+                if (areaDeps.name.empty())
+                {
+#if HECL_UCS2
+                    areaDeps.name = HECL::UTF8ToWide(area.internalAreaName);
+#else
+                    areaDeps.name = area.internalAreaName;
+#endif
+                    if (areaDeps.name.empty())
+                    {
+#if HECL_UCS2
+                        areaDeps.name = _S("MREA_") + HECL::UTF8ToWide(area.areaMREAId.toString());
+#else
+                        areaDeps.name = "MREA_" + area.areaMREAId.toString();
+#endif
+                    }
+                }
+
+                const MLVL::LayerFlags& areaLayers = *layerFlagsIt++;
+                if (areaLayers.layerCount)
+                {
+                    areaDeps.layers.reserve(areaLayers.layerCount);
+                    for (unsigned l=0 ; l<areaLayers.layerCount ; ++l)
+                    {
+                        areaDeps.layers.emplace_back();
+                        Area::Layer& layer = areaDeps.layers.back();
+                        layer.name = LayerName(mlvl.layerNames[layerIdx++]);
+                        /* Trim possible trailing whitespace */
+#if HECL_UCS2
+                        while (layer.name.size() && iswblank(layer.name.back()))
+                            layer.name.pop_back();
+#else
+                        while (layer.name.size() && isblank(layer.name.back()))
+                            layer.name.pop_back();
+#endif
+                    }
+                }
+
+                /* Load area DEPS */
+                const PAK::Entry* areaEntry = m_pak.lookupEntry(area.areaMREAId);
+                if (areaEntry)
+                {
+                    PAKEntryReadStream ars = areaEntry->beginReadStream(m_node);
+                    MREA::ExtractLayerDeps(ars, areaDeps);
+                    areaDeps.resources.emplace(area.areaMREAId);
+                }
+            }
+        }
+    }
+
+    /* Second pass: cross-compare uniqueness */
+    for (PAK::Entry& entry : m_pak.m_entries)
+    {
+        entry.unique = uniqueCheck(entry);
+    }
 }
 
 void PAKBridge::addCMDLRigPairs(std::unordered_map<UniqueID64, std::pair<UniqueID64, UniqueID64>>& addTo) const
@@ -84,6 +270,8 @@ ResExtractor<PAKBridge> PAKBridge::LookupExtractor(const PAK::Entry& entry)
         return {nullptr, CMDL::Extract, {_S(".blend")}, 1};
     case SBIG('MREA'):
         return {nullptr, MREA::Extract, {_S(".blend")}, 2};
+    case SBIG('MLVL'):
+        return {MLVL::Extract, nullptr, {_S(".yaml")}};
     }
     return {};
 }

DataSpec/DNAMP3/DNAMP3.hpp

@@ -17,8 +17,22 @@ class PAKBridge
     HECL::Database::Project& m_project;
     const NOD::DiscBase::IPartition::Node& m_node;
     PAK m_pak;
-    HECL::SystemString m_levelString;
+    UniqueResult uniqueCheck(const PAK::Entry& entry);
 public:
+    struct Area
+    {
+        HECL::SystemString name;
+        struct Layer
+        {
+            HECL::SystemString name;
+            std::unordered_set<UniqueID64> resources;
+        };
+        std::vector<Layer> layers;
+        std::unordered_set<UniqueID64> resources;
+    };
+    std::unordered_map<UniqueID64, Area> m_areaDeps;
+    HECL::SystemString m_levelString;
+
     PAKBridge(HECL::Database::Project& project, const NOD::DiscBase::IPartition::Node& node);
     void build();
     static ResExtractor<PAKBridge> LookupExtractor(const PAK::Entry& entry);

DataSpec/DNAMP3/MLVL.hpp

@@ -1,16 +1,16 @@
 #ifndef __DNAMP3_MLVL_HPP__
 #define __DNAMP3_MLVL_HPP__
 
-#include "../DNACommon/DNACommon.hpp"
+#include "../DNACommon/PAK.hpp"
 
 namespace Retro
 {
 namespace DNAMP3
 {
 
-struct MLVL : BigDNA
+struct MLVL : BigYAML
 {
-    DECL_DNA
+    DECL_YAML
     Value<atUint32> magic;
     Value<atUint32> version;
     UniqueID64 worldNameId;
@@ -19,9 +19,9 @@ struct MLVL : BigDNA
     UniqueID64 worldSkyboxId;
 
     Value<atUint32> areaCount;
-    struct Area : BigDNA
+    struct Area : BigYAML
     {
-        DECL_DNA
+        DECL_YAML
         UniqueID64 areaNameId;
         Value<atVec4f> transformMtx[3];
         Value<atVec3f> aabb[2];
@@ -32,13 +32,13 @@ struct MLVL : BigDNA
         Vector<atUint16, DNA_COUNT(attachedAreaCount)> attachedAreas;
 
         Value<atUint32> dockCount;
-        struct Dock : BigDNA
+        struct Dock : BigYAML
         {
-            DECL_DNA
+            DECL_YAML
             Value<atUint32> endpointCount;
-            struct Endpoint : BigDNA
+            struct Endpoint : BigYAML
             {
-                DECL_DNA
+                DECL_YAML
                 Value<atUint32> areaIdx;
                 Value<atUint32> dockIdx;
             };
@@ -58,9 +58,9 @@ struct MLVL : BigDNA
     Value<atUint32> unknown3;
 
     Value<atUint32> layerFlagCount;
-    struct LayerFlags : BigDNA
+    struct LayerFlags : BigYAML
     {
-        DECL_DNA
+        DECL_YAML
         Value<atUint32> layerCount;
         Value<atUint64> flags;
     };
@@ -74,6 +74,16 @@ struct MLVL : BigDNA
 
     Value<atUint32> layerNameOffsetCount;
     Vector<atUint32, DNA_COUNT(layerNameOffsetCount)> layerNameOffsets;
+
+    static bool Extract(PAKEntryReadStream& rs, const HECL::ProjectPath& outPath)
+    {
+        MLVL mlvl;
+        mlvl.read(rs);
+        FILE* fp = HECL::Fopen(outPath.getAbsolutePath().c_str(), _S("wb"));
+        mlvl.toYAMLFile(fp);
+        fclose(fp);
+        return true;
+    }
 };
 
 }

DataSpec/DNAMP3/MREA.cpp

@@ -315,5 +315,40 @@ bool MREA::Extract(const SpecBase& dataSpec,
     return conn.saveBlend();
 }
 
+bool MREA::ExtractLayerDeps(PAKEntryReadStream& rs, PAKBridge::Area& areaOut)
+{
+    /* Do extract */
+    Header head;
+    head.read(rs);
+    rs.seekAlign32();
+
+    /* MREA decompression stream */
+    StreamReader drs(rs, head.compressedBlockCount, head.secIndexCount);
+    for (const std::pair<DNAFourCC, atUint32>& idx : drs.m_secIdxs)
+    {
+        if (idx.first == FOURCC('DEPS'))
+        {
+            drs.seek(head.getSecOffset(idx.second), Athena::Begin);
+            DEPS deps;
+            deps.read(drs);
+
+            unsigned r=0;
+            for (unsigned l=1 ; l<deps.depLayerCount ; ++l)
+            {
+                PAKBridge::Area::Layer& layer = areaOut.layers.at(l-1);
+                layer.resources.reserve(deps.depLayers[l] - r);
+                for (; r<deps.depLayers[l] ; ++r)
+                    layer.resources.emplace(deps.deps[r].id);
+            }
+            areaOut.resources.reserve(deps.depCount - r);
+            for (; r<deps.depCount ; ++r)
+                areaOut.resources.emplace(deps.deps[r].id);
+
+            return true;
+        }
+    }
+    return false;
+}
+
 }
 }

DataSpec/DNAMP3/MREA.hpp

@@ -13,10 +13,9 @@ namespace DNAMP3
 
 struct MREA
 {
-    class StreamReader : public DNAMP2::MREA::StreamReader
+    struct StreamReader : DNAMP2::MREA::StreamReader
     {
         std::vector<std::pair<DNAFourCC, atUint32>> m_secIdxs;
-    public:
         StreamReader(Athena::io::IStreamReader& source,
                      atUint32 blkCount, atUint32 secIdxCount);
         std::vector<std::pair<DNAFourCC, atUint32>>::const_iterator beginSecIdxs()
@@ -39,6 +38,16 @@ struct MREA
         Value<atUint32> secIndexCount;
         Seek<20, Athena::Current> align1;
         Vector<atUint32, DNA_COUNT(secCount)> secSizes;
+
+        atUint32 getSecOffset(atUint32 idx) const
+        {
+            if (idx >= secSizes.size())
+                return -1;
+            atUint32 retval = 0;
+            for (atUint32 i=0 ; i<idx ; ++i)
+                retval += secSizes[i];
+            return retval;
+        }
     };
 
     struct MeshHeader : BigDNA
@@ -53,6 +62,21 @@ struct MREA
         Value<atVec3f> aabb[2];
     };
 
+    struct DEPS : BigDNA
+    {
+        DECL_DNA
+        Value<atUint32> depCount;
+        struct Dependency : BigDNA
+        {
+            DECL_DNA
+            UniqueID64 id;
+            DNAFourCC type;
+        };
+        Vector<Dependency, DNA_COUNT(depCount)> deps;
+        Value<atUint32> depLayerCount;
+        Vector<atUint32, DNA_COUNT(depLayerCount)> depLayers;
+    };
+
     struct BabeDeadLight : BigDNA
     {
         DECL_DNA
@@ -89,6 +113,8 @@ struct MREA
                         const PAK::Entry& entry,
                         bool,
                         std::function<void(const HECL::SystemChar*)>);
+
+    static bool ExtractLayerDeps(PAKEntryReadStream& rs, PAKBridge::Area& areaOut);
 };
 
 }