GameCube spec cook fixes

DataSpec/DNACommon/CMDL.cpp

@@ -775,9 +775,9 @@ atUint32 ReadGeomSectionsToBlender(hecl::blender::PyOutStream& os,
                     size_t normCount = secSizes[s] / 6;
                     for (size_t i=0 ; i<normCount ; ++i)
                     {
-                        float x = reader.readInt16Big() / 16834.0f;
+                        float x = reader.readInt16Big() / 16384.0f;
-                        float y = reader.readInt16Big() / 16834.0f;
+                        float y = reader.readInt16Big() / 16384.0f;
-                        float z = reader.readInt16Big() / 16834.0f;
+                        float z = reader.readInt16Big() / 16384.0f;
                         os.format("norm_list.append((%f,%f,%f))\n",
                                   x, y, z);
                     }
@@ -1218,13 +1218,18 @@ static void WriteDLVal(W& writer, GX::AttrType type, atUint32 val)
 template <class MaterialSet, class SurfaceHeader, atUint32 Version>
 bool WriteCMDL(const hecl::ProjectPath& outPath, const hecl::ProjectPath& inPath, const Mesh& mesh)
 {
+    bool skinned = !mesh.skins.empty();
+
     Header head;
     head.magic = 0xDEADBABE;
     head.version = Version;
+    head.flags.setSkinned(skinned);
+    head.flags.setShortNormals(!skinned);
+    head.flags.setShortUVs(true); /* This just means there's an (empty) short UV section */
     head.aabbMin = mesh.aabbMin.val;
     head.aabbMax = mesh.aabbMax.val;
     head.matSetCount = mesh.materialSets.size();
-    head.secCount = head.matSetCount + 5 + mesh.surfaces.size();
+    head.secCount = head.matSetCount + 6 + mesh.surfaces.size();
     head.secSizes.reserve(head.secCount);
 
     /* Lengths of padding to insert while writing */
@@ -1238,6 +1243,8 @@ bool WriteCMDL(const hecl::ProjectPath& outPath, const hecl::ProjectPath& inPath
         hecl::Frontend::Frontend FE;
         for (const std::vector<Material>& mset : mesh.materialSets)
         {
+            std::unordered_map<uint64_t, int> uniqueMatMap;
+
             matSets.emplace_back();
             MaterialSet& targetMSet = matSets.back();
             std::vector<hecl::ProjectPath> texPaths;
@@ -1245,7 +1252,6 @@ bool WriteCMDL(const hecl::ProjectPath& outPath, const hecl::ProjectPath& inPath
             setBackends.reserve(mset.size());
 
             size_t endOff = 0;
-            atUint32 nextGroupIdx = 0;
             for (const Material& mat : mset)
             {
                 std::string diagName = hecl::Format("%s:%s", inPath.getLastComponentUTF8().data(), mat.name.c_str());
@@ -1254,25 +1260,9 @@ bool WriteCMDL(const hecl::ProjectPath& outPath, const hecl::ProjectPath& inPath
                 hecl::Backend::GX& matGX = setBackends.back();
                 matGX.reset(matIR, FE.getDiagnostics());
 
-                atUint32 groupIdx = -1;
-                if (matSets.size() == 1)
-                {
-                    for (size_t i=0 ; i<setBackends.size()-1 ; ++i)
-                    {
-                        const hecl::Backend::GX& other = setBackends[i];
-                        if (matGX == other)
-                        {
-                            groupIdx = i;
-                            break;
-                        }
-                    }
-                    if (groupIdx == -1)
-                        groupIdx = nextGroupIdx++;
-                }
-
                 targetMSet.materials.emplace_back(matGX, mat.iprops, mat.texs, texPaths,
                                                   mesh.colorLayerCount, mesh.uvLayerCount,
-                                                  false, false, groupIdx);
+                                                  false, false, uniqueMatMap);
 
                 targetMSet.materials.back().binarySize(endOff);
                 targetMSet.head.addMaterialEndOff(endOff);
@@ -1317,7 +1307,7 @@ bool WriteCMDL(const hecl::ProjectPath& outPath, const hecl::ProjectPath& inPath
     paddingSizes.push_back(secSz32 - secSz);
 
     /* Vertex Normals */
-    secSz = mesh.norm.size() * 12;
+    secSz = mesh.norm.size() * (skinned ? 12 : 6);
     secSz32 = ROUND_UP_32(secSz);
     if (secSz32 == 0)
         secSz32 = 32;
@@ -1340,6 +1330,12 @@ bool WriteCMDL(const hecl::ProjectPath& outPath, const hecl::ProjectPath& inPath
     head.secSizes.push_back(secSz32);
     paddingSizes.push_back(secSz32 - secSz);
 
+    /* LUV coords */
+    secSz = 0;
+    secSz32 = ROUND_UP_32(secSz);
+    head.secSizes.push_back(secSz32);
+    paddingSizes.push_back(secSz32 - secSz);
+
     /* Surface index */
     std::vector<size_t> surfEndOffs;
     surfEndOffs.reserve(mesh.surfaces.size());
@@ -1388,7 +1384,21 @@ bool WriteCMDL(const hecl::ProjectPath& outPath, const hecl::ProjectPath& inPath
 
     /* Vertex Normals */
     for (const atVec3f& norm : mesh.norm)
-        writer.writeVec3fBig(norm);
+    {
+        if (skinned)
+        {
+            writer.writeVec3fBig(norm);
+        }
+        else
+        {
+            for (int i = 0; i < 3; ++i)
+            {
+                int tmpV = int(norm.vec[i] * 16384.f);
+                tmpV = zeus::clamp(-32768, tmpV, 32767);
+                writer.writeInt16Big(atInt16(tmpV));
+            }
+        }
+    }
     writer.fill(atUint8(0), *padIt);
     ++padIt;
 
@@ -1407,6 +1417,10 @@ bool WriteCMDL(const hecl::ProjectPath& outPath, const hecl::ProjectPath& inPath
     writer.fill(atUint8(0), *padIt);
     ++padIt;
 
+    /* LUV coords */
+    writer.fill(atUint8(0), *padIt);
+    ++padIt;
+
     /* Surface index */
     writer.writeUint32Big(surfEndOffs.size());
     for (size_t off : surfEndOffs)
@@ -1435,7 +1449,9 @@ bool WriteCMDL(const hecl::ProjectPath& outPath, const hecl::ProjectPath& inPath
         header.reflectionNormal = surf.reflectionNormal;
         header.write(writer);
 
-        writer.writeUByte(prim);
+        /* VAT0 = float normals, float UVs
+         * VAT1 = short normals, float UVs */
+        writer.writeUByte(prim | (skinned ? 0x0 : 0x1));
         writer.writeUint16Big(surf.verts.size());
 
         for (const Mesh::Surface::Vert& vert : surf.verts)
@@ -1680,6 +1696,7 @@ bool WriteMREASecs(std::vector<std::vector<uint8_t>>& secsOut, const hecl::Proje
     MaterialSet matSet;
     {
         MaterialPool matPool;
+        std::unordered_map<uint64_t, int> uniqueMatMap;
 
         size_t surfCount = 0;
         for (const Mesh& mesh : meshes)
@@ -1690,7 +1707,6 @@ bool WriteMREASecs(std::vector<std::vector<uint8_t>>& secsOut, const hecl::Proje
         size_t endOff = 0;
         std::vector<hecl::ProjectPath> texPaths;
         std::vector<hecl::Backend::GX> setBackends;
-        atUint32 nextGroupIdx = 0;
         for (const Mesh& mesh : meshes)
         {
             if (mesh.materialSets.size())
@@ -1727,25 +1743,12 @@ bool WriteMREASecs(std::vector<std::vector<uint8_t>>& secsOut, const hecl::Proje
                     hecl::Backend::GX& matGX = setBackends.back();
                     matGX.reset(matIR, FE.getDiagnostics());
 
-                    atUint32 groupIdx = -1;
-                    for (size_t i=0 ; i<setBackends.size()-1 ; ++i)
-                    {
-                        const hecl::Backend::GX& other = setBackends[i];
-                        if (matGX == other)
-                        {
-                            groupIdx = i;
-                            break;
-                        }
-                    }
-                    if (groupIdx == -1)
-                        groupIdx = nextGroupIdx++;
-
                     auto lightmapped = mat.iprops.find("retro_lightmapped");
                     bool lm = lightmapped != mat.iprops.cend() && lightmapped->second != 0;
 
                     matSet.materials.emplace_back(matGX, mat.iprops, mat.texs, texPaths,
                                                   mesh.colorLayerCount, mesh.uvLayerCount,
-                                                  lm, false, groupIdx);
+                                                  lm, false, uniqueMatMap);
 
                     matSet.materials.back().binarySize(endOff);
                     matSet.head.addMaterialEndOff(endOff);
@@ -1834,7 +1837,7 @@ bool WriteMREASecs(std::vector<std::vector<uint8_t>>& secsOut, const hecl::Proje
                 zeus::CVector3f preXfNorm = (meshXf.basis * zeus::CVector3f(v)).normalized();
                 for (int i=0 ; i<3 ; ++i)
                 {
-                    int tmpV = int(preXfNorm[i] * 16834.f);
+                    int tmpV = int(preXfNorm[i] * 16384.f);
                     tmpV = zeus::clamp(-32768, tmpV, 32767);
                     w.writeInt16Big(atInt16(tmpV));
                 }
@@ -1904,8 +1907,8 @@ bool WriteMREASecs(std::vector<std::vector<uint8_t>>& secsOut, const hecl::Proje
         for (const Mesh::Surface& surf : mesh.surfaces)
         {
             size_t matIdx = *matIt++;
-            const typename MaterialSet::Material::VAFlags& vaFlags =
+            const typename MaterialSet::Material& mat = matSet.materials.at(matIdx);
-                matSet.materials.at(matIdx).getVAFlags();
+            const typename MaterialSet::Material::VAFlags& vaFlags = mat.getVAFlags();
             size_t vertSz = vaFlags.vertDLSize();
 
             SurfaceHeader header;
@@ -1927,7 +1930,9 @@ bool WriteMREASecs(std::vector<std::vector<uint8_t>>& secsOut, const hecl::Proje
             athena::io::MemoryWriter w(secsOut.back().data(), secsOut.back().size());
             header.write(w);
 
-            w.writeUByte(prim);
+            /* VAT1 = short normals, float UVs
+             * VAT2 = short normals, short UVs */
+            w.writeUByte(prim | (mat.flags.lightmapUVArray() ? 0x2 : 0x1));
             w.writeUint16Big(surf.verts.size());
 
             for (const Mesh::Surface::Vert& vert : surf.verts)

DataSpec/DNACommon/CMDL.hpp

@@ -24,6 +24,8 @@ struct Header : BigDNA
     {
         AT_DECL_DNA
         Value<atUint32> flags = 0;
+        bool skinned() const {return (flags & 0x1) != 0;}
+        void setSkinned(bool val) {flags &= ~0x1; flags |= val;}
         bool shortNormals() const {return (flags & 0x2) != 0;}
         void setShortNormals(bool val) {flags &= ~0x2; flags |= val << 1;}
         bool shortUVs() const {return (flags & 0x4) != 0;}

DataSpec/DNACommon/DGRP.hpp

@@ -17,9 +17,28 @@ struct AT_SPECIALIZE_PARMS(DataSpec::UniqueID32, DataSpec::UniqueID64) DGRP : Bi
         AT_DECL_DNA_YAML
         DNAFourCC type;
         Value<IDType> id;
+
+        bool validate() const
+        {
+            if (!id.operator bool())
+                return false;
+            hecl::ProjectPath path = UniqueIDBridge::TranslatePakIdToPath(id);
+            return path && !path.isNone();
+        }
     };
 
     Vector<ObjectTag, AT_DNA_COUNT(dependCount)> depends;
+
+    void validateDeps()
+    {
+        std::vector<ObjectTag> newDeps;
+        newDeps.reserve(depends.size());
+        for (const ObjectTag& tag : depends)
+            if (tag.validate())
+                newDeps.push_back(tag);
+        depends = std::move(newDeps);
+        dependCount = atUint32(depends.size());
+    }
 };
 
 template <class IDType>

DataSpec/DNACommon/DNACommon.cpp

@@ -142,6 +142,22 @@ std::string UniqueID32::toString() const
     return std::string(buf);
 }
 
+template <>
+void UniqueID32Zero::Enumerate<BigDNA::Read>(typename Read::StreamT& reader)
+{UniqueID32::Enumerate<BigDNA::Read>(reader);}
+template <>
+void UniqueID32Zero::Enumerate<BigDNA::Write>(typename Write::StreamT& writer)
+{writer.writeUint32Big(*this ? m_id : 0);}
+template <>
+void UniqueID32Zero::Enumerate<BigDNA::ReadYaml>(typename ReadYaml::StreamT& reader)
+{UniqueID32::Enumerate<BigDNA::ReadYaml>(reader);}
+template <>
+void UniqueID32Zero::Enumerate<BigDNA::WriteYaml>(typename WriteYaml::StreamT& writer)
+{UniqueID32::Enumerate<BigDNA::WriteYaml>(writer);}
+template <>
+void UniqueID32Zero::Enumerate<BigDNA::BinarySize>(typename BinarySize::StreamT& s)
+{UniqueID32::Enumerate<BigDNA::BinarySize>(s);}
+
 AuxiliaryID32& AuxiliaryID32::operator=(const hecl::ProjectPath& path)
 {
     assign(path.ensureAuxInfo(m_auxStr).hash().val32());

DataSpec/DNACommon/DNACommon.hpp

@@ -223,6 +223,15 @@ public:
     static constexpr size_t BinarySize() {return 4;}
 };
 
+/** PAK 32-bit Unique ID - writes zero when invalid */
+class UniqueID32Zero : public UniqueID32
+{
+public:
+    AT_DECL_DNA_YAML
+    Delete __d2;
+    using UniqueID32::UniqueID32;
+};
+
 class AuxiliaryID32 : public UniqueID32
 {
     const hecl::SystemChar* m_auxStr;
@@ -352,6 +361,10 @@ public:
     static constexpr size_t BinarySize() {return 16;}
 };
 
+/** Casts ID type to its null-zero equivalent */
+template <class T>
+using CastIDToZero = typename std::conditional_t<std::is_same_v<T, UniqueID32>, UniqueID32Zero, T>;
+
 /** Word Bitmap reader/writer */
 class WordBitmap
 {

DataSpec/DNACommon/ParticleCommon.hpp

@@ -820,7 +820,7 @@ struct UVEConstant : IUVElement
 {
     AT_DECL_EXPLICIT_DNA_YAML
     AT_SUBDECL_DNA
-    IDType tex;
+    CastIDToZero<IDType> tex;
     const char* ClassID() const {return "CNST";}
 
     void gatherDependencies(std::vector<hecl::ProjectPath>& pathsOut) const
@@ -834,7 +834,7 @@ struct UVEAnimTexture : IUVElement
 {
     AT_DECL_EXPLICIT_DNA_YAML
     AT_SUBDECL_DNA
-    IDType tex;
+    CastIDToZero<IDType> tex;
     IntElementFactory tileW;
     IntElementFactory tileH;
     IntElementFactory strideW;

DataSpec/DNACommon/TXTR.cpp

@@ -621,8 +621,8 @@ static uint8_t* EncodePaletteSPLT(png_structp png, png_infop info, int numEntrie
 
     ((uint32_t*)data)[0] = hecl::SBig(format);
     data += 4;
-    ((uint16_t*)data)[0] = hecl::SBig(uint16_t(numEntries));
+    ((uint16_t*)data)[0] = hecl::SBig(uint16_t(1));
-    ((uint16_t*)data)[1] = hecl::SBig(uint16_t(1));
+    ((uint16_t*)data)[1] = hecl::SBig(uint16_t(numEntries));
     data += 4;
 
     switch (format)
@@ -1415,7 +1415,7 @@ bool TXTR::Cook(const hecl::ProjectPath& inPath, const hecl::ProjectPath& outPat
                   width >= 4 && height >= 4;
 
     /* Read into mip0 image buffer */
-    for (int r=height-1 ; r>=0 ; --r)
+    for (int r=0 ; r<height ; ++r)
     {
         if (colorType == PNG_COLOR_TYPE_RGB)
         {
@@ -1545,6 +1545,7 @@ bool TXTR::Cook(const hecl::ProjectPath& inPath, const hecl::ProjectPath& outPat
         filterHeight = height;
         const uint8_t* rgbaIn = bufOut.get();
         uint8_t* blocksOut = compOut.get();
+        memset(blocksOut, 0, compLen);
         for (size_t i=0 ; i<numMips ; ++i)
         {
             int thisLen = squish::GetStorageRequirements(filterWidth, filterHeight, squish::kDxt1);
@@ -1563,10 +1564,15 @@ bool TXTR::Cook(const hecl::ProjectPath& inPath, const hecl::ProjectPath& outPat
         int filterHeight = height;
         compLen = bufLen;
         if (colorType == PNG_COLOR_TYPE_PALETTE)
+        {
+            if (nPaletteEntries == 16)
+                compLen /= 2;
             compLen += 8 + nPaletteEntries * 2;
+        }
         compOut.reset(new uint8_t[compLen]);
         const uint8_t* rgbaIn = bufOut.get();
         uint8_t* dataOut = compOut.get();
+        memset(dataOut, 0, compLen);
         for (size_t i=0 ; i<numMips ; ++i)
         {
             switch (colorType)

DataSpec/DNAMP1/ANCS.cpp

@@ -432,9 +432,9 @@ void ANCS::CharacterSet::CharacterInfo::Enumerate<BigDNA::Write>(athena::io::ISt
     writer.writeUint16Big(sectionCount);
 
     writer.writeString(name);
-    cmdl.UniqueID32::write(writer);
+    cmdl.write(writer);
-    cskr.UniqueID32::write(writer);
+    cskr.write(writer);
-    cinf.UniqueID32::write(writer);
+    cinf.write(writer);
 
     writer.writeUint32Big(animations.size());
     writer.enumerate(animations);
@@ -472,8 +472,8 @@ void ANCS::CharacterSet::CharacterInfo::Enumerate<BigDNA::Write>(athena::io::ISt
 
     if (sectionCount > 3)
     {
-        cmdlOverlay.UniqueID32::write(writer);
+        cmdlOverlay.write(writer);
-        cskrOverlay.UniqueID32::write(writer);
+        cskrOverlay.write(writer);
     }
 
     if (sectionCount > 4)
@@ -1157,8 +1157,8 @@ bool ANCS::Cook(const hecl::ProjectPath& outPath,
         ch.cmdl = UniqueID32{};
         ch.cskr = UniqueID32{};
         ch.cinf = UniqueID32{};
-        ch.cmdlOverlay = UniqueID32{};
+        ch.cmdlOverlay = UniqueID32Zero{};
-        ch.cskrOverlay = UniqueID32{};
+        ch.cskrOverlay = UniqueID32Zero{};
 
         hecl::SystemStringConv chSysName(ch.name);
         ch.cskr = inPath.ensureAuxInfo(hecl::SystemString(chSysName.sys_str()) + _S(".CSKR"));

DataSpec/DNAMP1/ANCS.hpp

@@ -140,8 +140,8 @@ struct ANCS : BigDNA
             };
             std::vector<Effect> effects;
 
-            UniqueID32 cmdlOverlay;
+            UniqueID32Zero cmdlOverlay;
-            UniqueID32 cskrOverlay;
+            UniqueID32Zero cskrOverlay;
 
             std::vector<atUint32> animIdxs;
         };

DataSpec/DNAMP1/ANIM.hpp

@@ -24,7 +24,7 @@ struct ANIM : BigDNA
         std::vector<DNAANIM::Channel> channels;
         std::vector<std::vector<DNAANIM::Value>> chanKeys;
         float mainInterval = 0.0;
-        UniqueID32 evnt;
+        UniqueID32Zero evnt;
         bool looping = false;
 
         void sendANIMToBlender(hecl::blender::PyOutStream&, const DNAANIM::RigInverter<CINF>& rig) const;
@@ -59,7 +59,7 @@ struct ANIM : BigDNA
         {
             AT_DECL_DNA
             Value<atUint32> scratchSize;
-            UniqueID32 evnt;
+            UniqueID32Zero evnt;
             Value<atUint32> unk0 = 1;
             Value<float> duration;
             Value<float> interval;

DataSpec/DNAMP1/CMDL.cpp

@@ -65,7 +65,7 @@ bool CMDL::Cook(const hecl::ProjectPath& outPath,
                 const hecl::ProjectPath& inPath,
                 const DNACMDL::Mesh& mesh)
 {
-    if (mesh.skins.size())
+    if (!mesh.skins.empty())
     {
         DNACMDL::Mesh skinMesh = mesh.getContiguousSkinningVersion();
         if (!DNACMDL::WriteCMDL<MaterialSet, DNACMDL::SurfaceHeader_1, 2>(outPath, inPath, skinMesh))

DataSpec/DNAMP1/CMDLMaterials.cpp

@@ -933,8 +933,11 @@ MaterialSet::Material::Material(const hecl::Backend::GX& gx,
                                 int uvCount,
                                 bool lightmapUVs,
                                 bool matrixSkinning,
-                                atUint32 setIdxIn)
+                                std::unordered_map<uint64_t, int>& uniqueMap)
 {
+    XXH64_state_t xxHash;
+    XXH64_reset(&xxHash, 0);
+
     if (gx.m_kcolorCount)
     {
         flags.setKonstValuesEnabled(true);
@@ -1010,6 +1013,8 @@ MaterialSet::Material::Material(const hecl::Backend::GX& gx,
     }
     flags.setTextureSlots(texFlags);
 
+    XXH64_update(&xxHash, &flags.flags, sizeof(flags.flags));
+
     vaFlags.setPosition(GX::INDEX16);
     vaFlags.setNormal(GX::INDEX16);
 
@@ -1052,17 +1057,27 @@ MaterialSet::Material::Material(const hecl::Backend::GX& gx,
             vaFlags.setTex6MatIdx(GX::DIRECT);
     }
 
-    groupIdx = setIdxIn;
+    XXH64_update(&xxHash, &vaFlags.vaFlags, sizeof(vaFlags.vaFlags));
 
+    XXH64_update(&xxHash, &gx.m_kcolorCount, sizeof(gx.m_kcolorCount));
     for (unsigned i=0 ; i<gx.m_kcolorCount ; ++i)
+    {
         konstColors.emplace_back(gx.m_kcolors[i]);
+        XXH64_update(&xxHash, &gx.m_kcolors[i].num, sizeof(gx.m_kcolors[i].num));
+    }
 
     blendDstFac = BlendFactor(gx.m_blendDst);
+    XXH64_update(&xxHash, &gx.m_blendDst, sizeof(gx.m_blendDst));
     blendSrcFac = BlendFactor(gx.m_blendSrc);
+    XXH64_update(&xxHash, &gx.m_blendSrc, sizeof(gx.m_blendSrc));
     if (flags.samusReflectionIndirectTexture())
+    {
         indTexSlot.push_back(textureIdxs.size());
+        XXH64_update(&xxHash, &indTexSlot.back(), sizeof(indTexSlot.back()));
+    }
 
     colorChannelCount = 1;
+    XXH64_update(&xxHash, &colorChannelCount, sizeof(colorChannelCount));
     colorChannels.emplace_back();
     ColorChannel& ch = colorChannels.back();
     for (unsigned i=0 ; i<gx.m_tevCount ; ++i)
@@ -1074,6 +1089,8 @@ MaterialSet::Material::Material(const hecl::Backend::GX& gx,
                 stage.m_alpha[c] == hecl::Backend::GX::CA_RASA)
             {
                 ch.setLighting(true);
+                uint8_t one = 1;
+                XXH64_update(&xxHash, &one, sizeof(one));
                 break;
             }
         if (ch.lighting())
@@ -1083,6 +1100,7 @@ MaterialSet::Material::Material(const hecl::Backend::GX& gx,
     ch.setAttenuationFn(GX::AF_SPOT);
 
     tevStageCount = gx.m_tevCount;
+    XXH64_update(&xxHash, &tevStageCount, sizeof(tevStageCount));
     tevStages.reserve(gx.m_tevCount);
     for (unsigned i=0 ; i<gx.m_tevCount ; ++i)
     {
@@ -1110,9 +1128,18 @@ MaterialSet::Material::Material(const hecl::Backend::GX& gx,
         target.setAlphaOpOutReg(stage.m_aRegOut);
         target.setKColorIn(stage.m_kColor);
         target.setKAlphaIn(stage.m_kAlpha);
+
+        XXH64_update(&xxHash, &target.ciFlags, sizeof(target.ciFlags));
+        XXH64_update(&xxHash, &target.aiFlags, sizeof(target.aiFlags));
+        XXH64_update(&xxHash, &target.ccFlags, sizeof(target.ccFlags));
+        XXH64_update(&xxHash, &target.acFlags, sizeof(target.acFlags));
+        XXH64_update(&xxHash, &target.kaInput, sizeof(target.kaInput));
+        XXH64_update(&xxHash, &target.kcInput, sizeof(target.kcInput));
+        XXH64_update(&xxHash, &target.rascInput, sizeof(target.rascInput));
     }
 
     tcgCount = gx.m_tcgCount;
+    XXH64_update(&xxHash, &tcgCount, sizeof(tcgCount));
     for (unsigned i=0 ; i<gx.m_tcgCount ; ++i)
     {
         const hecl::Backend::GX::TexCoordGen& tcg = gx.m_tcgs[i];
@@ -1123,6 +1150,8 @@ MaterialSet::Material::Material(const hecl::Backend::GX& gx,
         target.setMtx(tcg.m_mtx);
         target.setNormalize(tcg.m_norm);
         target.setPostMtx(tcg.m_pmtx);
+
+        XXH64_update(&xxHash, &target.flags, sizeof(target.flags));
     }
 
     uvAnimsSize = 4;
@@ -1140,6 +1169,8 @@ MaterialSet::Material::Material(const hecl::Backend::GX& gx,
                 found = true;
                 ++uvAnimsCount;
                 uvAnims.emplace_back(tcg.m_gameFunction, tcg.m_gameArgs);
+                XXH64_update(&xxHash, tcg.m_gameFunction.data(), sizeof(tcg.m_gameFunction.size()));
+                XXH64_update(&xxHash, &tcg.m_gameArgs, sizeof(tcg.m_gameArgs));
                 size_t tmpUvAnimsSize = uvAnimsSize;
                 uvAnims.back().binarySize(tmpUvAnimsSize);
                 uvAnimsSize = tmpUvAnimsSize;
@@ -1149,6 +1180,15 @@ MaterialSet::Material::Material(const hecl::Backend::GX& gx,
         if (!found)
             break;
     }
+
+    XXH64_update(&xxHash, &uvAnimsSize, sizeof(uvAnimsSize));
+    XXH64_update(&xxHash, &uvAnimsCount, sizeof(uvAnimsCount));
+
+    uint64_t hash = XXH64_digest(&xxHash);
+    auto hashSearch = uniqueMap.find(hash);
+    if (hashSearch == uniqueMap.end())
+        hashSearch = uniqueMap.insert(std::make_pair(hash, uniqueMap.size())).first;
+    uniqueIdx = hashSearch->second;
 }
 
 HMDLMaterialSet::Material::Material(hecl::Frontend::Frontend& FE,

DataSpec/DNAMP1/CMDLMaterials.hpp

@@ -130,7 +130,7 @@ struct MaterialSet : BigDNA
             }
         } vaFlags;
         const VAFlags& getVAFlags() const {return vaFlags;}
-        Value<atUint32> groupIdx;
+        Value<atUint32> uniqueIdx;
 
         Vector<atUint32, AT_DNA_COUNT(flags.konstValuesEnabled())> konstCount;
         Vector<GX::Color, AT_DNA_COUNT(flags.konstValuesEnabled() ? konstCount[0] : 0)> konstColors;
@@ -291,7 +291,7 @@ struct MaterialSet : BigDNA
                  int uvCount,
                  bool lightmapUVs,
                  bool matrixSkinning,
-                 atUint32 grpIdx);
+                 std::unordered_map<uint64_t, int>& uniqueMap);
     };
     Vector<Material, AT_DNA_COUNT(head.materialCount)> materials;
 

DataSpec/DNAMP1/MLVL.cpp

@@ -79,24 +79,24 @@ struct LayerResources
 {
     BulkResources& bulkResources;
     std::unordered_map<hecl::Hash, std::pair<size_t, size_t>> addedPaths;
-    std::vector<std::vector<hecl::ProjectPath>> layerPaths;
+    std::vector<std::vector<std::pair<hecl::ProjectPath, bool>>> layerPaths;
     std::unordered_set<hecl::Hash> addedSharedPaths;
-    std::vector<hecl::ProjectPath> sharedPaths;
+    std::vector<std::pair<hecl::ProjectPath, bool>> sharedPaths;
     LayerResources(BulkResources& bulkResources) : bulkResources(bulkResources) {}
     void beginLayer()
     {
         layerPaths.resize(layerPaths.size() + 1);
     }
-    void addSharedPath(const hecl::ProjectPath& path)
+    void addSharedPath(const hecl::ProjectPath& path, bool lazy)
     {
         auto search = addedSharedPaths.find(path.hash());
         if (search == addedSharedPaths.cend())
         {
-            sharedPaths.push_back(path);
+            sharedPaths.emplace_back(path, lazy);
             addedSharedPaths.insert(path.hash());
         }
     }
-    void addPath(const hecl::ProjectPath& path)
+    void addPath(const hecl::ProjectPath& path, bool lazy)
     {
         auto search = addedPaths.find(path.hash());
         if (search != addedPaths.cend())
@@ -105,22 +105,22 @@ struct LayerResources
                 return;
             else
             {
-                hecl::ProjectPath& toMove = layerPaths[search->second.first][search->second.second];
+                auto& toMove = layerPaths[search->second.first][search->second.second];
-                addSharedPath(toMove);
+                addSharedPath(toMove.first, toMove.second);
-                toMove.clear();
+                toMove.first.clear();
             }
         }
         else
         {
-            layerPaths.back().push_back(path);
+            layerPaths.back().emplace_back(path, lazy);
             addedPaths.insert(std::make_pair(path.hash(),
                 std::make_pair(layerPaths.size() - 1, layerPaths.back().size() - 1)));
         }
     }
-    void addBulkPath(const hecl::ProjectPath& path, size_t areaIdx)
+    void addBulkPath(const hecl::ProjectPath& path, size_t areaIdx, bool lazy)
     {
         if (bulkResources.addBulkPath(path, areaIdx))
-            addPath(path);
+            addPath(path, lazy);
     }
 };
 
@@ -293,7 +293,7 @@ bool MLVL::Cook(const hecl::ProjectPath& outPath, const hecl::ProjectPath& inPat
             {
                 g_ThreadBlenderToken.reset(&btok);
                 std::vector<hecl::ProjectPath> depPaths;
-                std::vector<hecl::ProjectPath> bulkPaths;
+                std::vector<hecl::ProjectPath> lazyPaths;
                 for (std::unique_ptr<IScriptObject>& obj : layer.objects)
                 {
                     if (obj->type == int(urde::EScriptObjectType::MemoryRelay))
@@ -325,14 +325,14 @@ bool MLVL::Cook(const hecl::ProjectPath& outPath, const hecl::ProjectPath& inPat
                     }
 
                     obj->gatherDependencies(depPaths);
-                    obj->gatherBulkDependencies(bulkPaths);
+                    obj->gatherLazyDependencies(lazyPaths);
                 }
 
                 /* Cull duplicate paths and add typed hash to list */
                 for (const hecl::ProjectPath& path : depPaths)
-                    layerResources.addBulkPath(path, areaIdx);
+                    layerResources.addBulkPath(path, areaIdx, false);
-                for (const hecl::ProjectPath& path : bulkPaths)
+                for (const hecl::ProjectPath& path : lazyPaths)
-                    layerResources.addBulkPath(path, areaIdx);
+                    layerResources.addBulkPath(path, areaIdx, true);
             }
 
             hecl::SystemUTF8Conv layerU8(layerName);
@@ -349,16 +349,22 @@ bool MLVL::Cook(const hecl::ProjectPath& outPath, const hecl::ProjectPath& inPat
 
         /* Build deplist */
         MLVL::Area& areaOut = mlvl.areas.back();
-        for (const std::vector<hecl::ProjectPath>& layer : layerResources.layerPaths)
+        for (const std::vector<std::pair<hecl::ProjectPath, bool>>& layer : layerResources.layerPaths)
         {
             areaOut.depLayers.push_back(areaOut.deps.size());
-            for (const hecl::ProjectPath& path : layer)
+            for (const std::pair<hecl::ProjectPath, bool>& path : layer)
             {
-                if (path)
+                if (path.first)
                 {
-                    urde::SObjectTag tag = g_curSpec->buildTagFromPath(path, btok);
+                    urde::SObjectTag tag = g_curSpec->buildTagFromPath(path.first, btok);
                     if (tag.id.IsValid())
+                    {
+                        if (path.second)
+                            areaOut.lazyDeps.emplace_back(tag.id.Value(), tag.type);
+                        else
+                            areaOut.lazyDeps.emplace_back(0, FOURCC('NONE'));
                         areaOut.deps.emplace_back(tag.id.Value(), tag.type);
+                    }
                 }
             }
         }
@@ -377,19 +383,28 @@ bool MLVL::Cook(const hecl::ProjectPath& outPath, const hecl::ProjectPath& inPat
             ds.close();
 
             for (const hecl::ProjectPath& path : texs)
-                layerResources.addSharedPath(path);
+                layerResources.addSharedPath(path, false);
 
-            for (const hecl::ProjectPath& path : layerResources.sharedPaths)
+            for (const std::pair<hecl::ProjectPath, bool>& path : layerResources.sharedPaths)
             {
-                urde::SObjectTag tag = g_curSpec->buildTagFromPath(path, btok);
+                urde::SObjectTag tag = g_curSpec->buildTagFromPath(path.first, btok);
                 if (tag.id.IsValid())
+                {
+                    if (path.second)
+                        areaOut.lazyDeps.emplace_back(tag.id.Value(), tag.type);
+                    else
+                        areaOut.lazyDeps.emplace_back(0, FOURCC('NONE'));
                     areaOut.deps.emplace_back(tag.id.Value(), tag.type);
+                }
             }
 
             hecl::ProjectPath pathPath(areaPath.getParentPath(), _S("!path.blend"));
             urde::SObjectTag pathTag = g_curSpec->buildTagFromPath(pathPath, btok);
             if (pathTag.id.IsValid())
+            {
                 areaOut.deps.emplace_back(pathTag.id.Value(), pathTag.type);
+                areaOut.lazyDeps.emplace_back(0, FOURCC('NONE'));
+            }
         }
 
         ++areaIdx;

DataSpec/DNAMP1/MLVL.hpp

@@ -41,9 +41,7 @@ struct MLVL : BigDNA
 
         Value<atUint32> attachedAreaCount;
         Vector<atUint16, AT_DNA_COUNT(attachedAreaCount)> attachedAreas;
-        Value<atUint32> padding;
 
-        Value<atUint32> depCount;
         struct Dependency : BigDNA
         {
             AT_DECL_DNA_YAML
@@ -52,8 +50,13 @@ struct MLVL : BigDNA
 
             Dependency() = default;
             Dependency(const UniqueID32& idin, const hecl::FourCC& fcc)
-            : id(idin), type(fcc) {}
+                : id(idin), type(fcc) {}
         };
+
+        Value<atUint32> lazyDepCount;
+        Vector<Dependency, AT_DNA_COUNT(lazyDepCount)> lazyDeps;
+
+        Value<atUint32> depCount;
         Vector<Dependency, AT_DNA_COUNT(depCount)> deps;
 
         Value<atUint32> depLayerCount;
@@ -85,6 +88,7 @@ struct MLVL : BigDNA
         {
             MLVL::Area& areaLast = areas.back();
             areaLast.attachedAreaCount = areaLast.attachedAreas.size();
+            areaLast.lazyDepCount = areaLast.lazyDeps.size();
             areaLast.depCount = areaLast.deps.size();
             areaLast.depLayerCount = areaLast.depLayers.size();
             areaLast.dockCount = areaLast.docks.size();

DataSpec/DNAMP1/ScriptObjects/DoorArea.hpp

@@ -39,13 +39,9 @@ struct DoorArea : IScriptObject
     }
 
     void gatherDependencies(std::vector<hecl::ProjectPath>& pathsOut) const
-    {
-        actorParameters.depIDs(pathsOut);
-    }
-
-    void gatherBulkDependencies(std::vector<hecl::ProjectPath>& pathsOut) const
     {
         animationParameters.depANCS(pathsOut);
+        actorParameters.depIDs(pathsOut);
     }
 
     void gatherScans(std::vector<Scan>& scansOut) const

DataSpec/DNAMP1/ScriptObjects/IScriptObject.hpp

@@ -45,7 +45,7 @@ struct IScriptObject : BigDNAVYaml
                                  std::unordered_map<UniqueID32, std::pair<UniqueID32, UniqueID32>>&) const {}
     virtual void nameIDs(PAKRouter<PAKBridge>& pakRouter) const {}
     virtual void gatherDependencies(std::vector<hecl::ProjectPath>& pathsOut) const {}
-    virtual void gatherBulkDependencies(std::vector<hecl::ProjectPath>& pathsOut) const {}
+    virtual void gatherLazyDependencies(std::vector<hecl::ProjectPath>& pathsOut) const {}
     virtual void gatherScans(std::vector<Scan>& scansOut) const {}
     virtual zeus::CAABox getVISIAABB(hecl::blender::Token& btok) const { return {}; }
 };

DataSpec/DNAMP1/ScriptObjects/PlayerActor.hpp

@@ -55,7 +55,7 @@ struct PlayerActor : IScriptObject
         actorParameters.depIDs(pathsOut);
     }
 
-    void gatherBulkDependencies(std::vector<hecl::ProjectPath>& pathsOut) const
+    void gatherLazyDependencies(std::vector<hecl::ProjectPath>& pathsOut) const
     {
         animationParameters.depANCSAll(pathsOut);
     }

DataSpec/DNAMP2/CMDLMaterials.hpp

@@ -31,7 +31,7 @@ struct MaterialSet : BigDNA
         const VAFlags& getVAFlags() const {return vaFlags;}
         Value<atUint32> unk0; /* MP2 only */
         Value<atUint32> unk1; /* MP2 only */
-        Value<atUint32> groupIdx;
+        Value<atUint32> uniqueIdx;
 
         Vector<atUint32, AT_DNA_COUNT(flags.konstValuesEnabled())> konstCount;
         Vector<GX::Color, AT_DNA_COUNT(flags.konstValuesEnabled() ? konstCount[0] : 0)> konstColors;

DataSpec/DNAMP3/CMDLMaterials.hpp

@@ -42,7 +42,7 @@ struct MaterialSet : BigDNA
                 void setShadowOccluderMesh(bool enabled) {flags &= ~0x100; flags |= atUint32(enabled) << 8;}
                 bool lightmapUVArray() const {return false;} /* For polymorphic compatibility with MP1/2 */
             } flags;
-            Value<atUint32> groupIdx;
+            Value<atUint32> uniqueIdx;
             Value<atUint32> unk1;
             VAFlags vaFlags;
             Value<atUint32> unk2;

DataSpec/SpecBase.cpp

@@ -231,7 +231,7 @@ static bool IsPathSong(const hecl::ProjectPath& path)
     return true;
 }
 
-bool SpecBase::canCook(const hecl::ProjectPath& path, hecl::blender::Token& btok)
+bool SpecBase::canCook(const hecl::ProjectPath& path, hecl::blender::Token& btok, int cookPass)
 {
     if (!checkPathPrefix(path))
         return false;
@@ -247,10 +247,19 @@ bool SpecBase::canCook(const hecl::ProjectPath& path, hecl::blender::Token& btok
         hecl::blender::Connection& conn = btok.getBlenderConnection();
         if (!conn.openBlend(asBlend))
             return false;
-        if (conn.getBlendType() != hecl::blender::BlendType::None)
+        hecl::blender::BlendType type = conn.getBlendType();
-            return true;
+        if (type != hecl::blender::BlendType::None)
+            return cookPass < 0 ||
+                   (cookPass == 0 && type == hecl::blender::BlendType::Mesh) || // CMDL only
+                   (cookPass == 1 && type != hecl::blender::BlendType::Mesh);   // Non-CMDL only
+        return false;
     }
-    else if (hecl::IsPathPNG(path))
+
+    /* Non-CMDLs shall not pass */
+    if (cookPass == 0)
+        return false;
+
+    if (hecl::IsPathPNG(path))
     {
         return true;
     }
@@ -430,12 +439,18 @@ void SpecBase::flattenDependenciesBlend(const hecl::ProjectPath& in,
         hecl::ProjectPath asGlob = in.getWithExtension(_S(".*"), true);
         hecl::blender::DataStream ds = conn.beginData();
         hecl::blender::Actor actor = ds.compileActorCharacterOnly();
+        auto actNames = ds.getActionNames();
+        ds.close();
 
         auto doSubtype = [&](Actor::Subtype& sub)
         {
             if (sub.armature >= 0)
             {
-                pathsOut.push_back(sub.mesh);
+                if (hecl::IsPathBlend(sub.mesh))
+                {
+                    flattenDependenciesBlend(sub.mesh, pathsOut, btok);
+                    pathsOut.push_back(sub.mesh);
+                }
 
                 hecl::SystemStringConv chSysName(sub.name);
                 pathsOut.push_back(asGlob.ensureAuxInfo(hecl::SystemString(chSysName.sys_str()) + _S(".CSKR")));
@@ -446,7 +461,11 @@ void SpecBase::flattenDependenciesBlend(const hecl::ProjectPath& in,
                 for (const auto& overlay : sub.overlayMeshes)
                 {
                     hecl::SystemStringConv ovelaySys(overlay.first);
-                    pathsOut.push_back(overlay.second);
+                    if (hecl::IsPathBlend(overlay.second))
+                    {
+                        flattenDependenciesBlend(overlay.second, pathsOut, btok);
+                        pathsOut.push_back(overlay.second);
+                    }
                     pathsOut.push_back(asGlob.ensureAuxInfo(hecl::SystemString(chSysName.sys_str()) + _S('.') +
                                                             ovelaySys.c_str() + _S(".CSKR")));
                 }
@@ -458,7 +477,6 @@ void SpecBase::flattenDependenciesBlend(const hecl::ProjectPath& in,
         else if (charIdx < actor.subtypes.size())
             doSubtype(actor.subtypes[charIdx]);
 
-        auto actNames = ds.getActionNames();
         for (const auto& act : actNames)
         {
             hecl::SystemStringConv actSysName(act);
@@ -468,7 +486,6 @@ void SpecBase::flattenDependenciesBlend(const hecl::ProjectPath& in,
             if (evntPath.isFile())
                 pathsOut.push_back(evntPath);
         }
-        ds.close();
 
         hecl::ProjectPath yamlPath = asGlob.getWithExtension(_S(".yaml"), true);
         if (yamlPath.isFile())
@@ -577,7 +594,8 @@ void SpecBase::copyBuildListData(std::vector<std::tuple<size_t, size_t, bool>>&
                                  const std::vector<urde::SObjectTag>& buildList,
                                  const hecl::Database::DataSpecEntry* entry,
                                  bool fast, const hecl::MultiProgressPrinter& progress,
-                                 athena::io::FileWriter& pakOut)
+                                 athena::io::FileWriter& pakOut,
+                                 const std::unordered_map<urde::CAssetId, std::vector<uint8_t>>& mlvlData)
 {
     fileIndex.reserve(buildList.size());
     int loadIdx = 0;
@@ -589,6 +607,23 @@ void SpecBase::copyBuildListData(std::vector<std::tuple<size_t, size_t, bool>>&
 
         fileIndex.emplace_back();
         auto& thisIdx = fileIndex.back();
+
+        if (tag.type == FOURCC('MLVL'))
+        {
+            auto search = mlvlData.find(tag.id);
+            if (search == mlvlData.end())
+                Log.report(logvisor::Fatal, _S("Unable to find MLVL %08X"), tag.id.Value());
+
+            std::get<0>(thisIdx) = pakOut.position();
+            std::get<1>(thisIdx) = ROUND_UP_32(search->second.size());
+            std::get<2>(thisIdx) = false;
+            pakOut.writeUBytes(search->second.data(), search->second.size());
+            for (atUint64 i = search->second.size() ; i < std::get<1>(thisIdx) ; ++i)
+                pakOut.writeUByte(0xff);
+
+            continue;
+        }
+
         hecl::ProjectPath path = pathFromTag(tag);
         hecl::ProjectPath cooked = getCookedPath(path, true);
         athena::io::FileReader r(cooked.getAbsolutePath());
@@ -646,6 +681,7 @@ void SpecBase::doPackage(const hecl::ProjectPath& path, const hecl::Database::Da
     athena::io::FileWriter pakOut(outPath.getAbsolutePath());
     std::vector<urde::SObjectTag> buildList;
     atUint64 resTableOffset = 0;
+    std::unordered_map<urde::CAssetId, std::vector<uint8_t>> mlvlData;
 
     if (path.getPathType() == hecl::ProjectPath::Type::File &&
         !hecl::StrCmp(path.getLastComponent().data(), _S("!world.blend"))) /* World PAK */
@@ -656,7 +692,7 @@ void SpecBase::doPackage(const hecl::ProjectPath& path, const hecl::Database::Da
             cp->waitUntilComplete();
         progress.startNewLine();
         hecl::ProjectPath cooked = getCookedPath(path, true);
-        buildWorldPakList(path, cooked, btok, pakOut, buildList, resTableOffset);
+        buildWorldPakList(path, cooked, btok, pakOut, buildList, resTableOffset, mlvlData);
         if (int64_t rem = pakOut.position() % 32)
             for (int64_t i=0 ; i<32-rem ; ++i)
                 pakOut.writeUByte(0xff);
@@ -671,9 +707,10 @@ void SpecBase::doPackage(const hecl::ProjectPath& path, const hecl::Database::Da
         /* Build name list */
         for (const auto& item : buildList)
         {
-            auto search = m_catalogTagToName.find(item);
+            auto search = m_catalogTagToNames.find(item);
-            if (search != m_catalogTagToName.end())
+            if (search != m_catalogTagToNames.end())
-                nameList.emplace_back(item, search->second);
+                for (const auto& name : search->second)
+                    nameList.emplace_back(item, name);
         }
 
         /* Write resource list structure */
@@ -703,9 +740,10 @@ void SpecBase::doPackage(const hecl::ProjectPath& path, const hecl::Database::Da
         /* Build name list */
         for (const auto& item : buildList)
         {
-            auto search = m_catalogTagToName.find(item);
+            auto search = m_catalogTagToNames.find(item);
-            if (search != m_catalogTagToName.end())
+            if (search != m_catalogTagToNames.end())
-                nameList.emplace_back(item, search->second);
+                for (const auto& name : search->second)
+                    nameList.emplace_back(item, name);
         }
 
         /* Write resource list structure */
@@ -718,24 +756,26 @@ void SpecBase::doPackage(const hecl::ProjectPath& path, const hecl::Database::Da
     /* Async cook resource list if using ClientProcess */
     if (cp)
     {
-        std::unordered_set<urde::SObjectTag> addedTags;
-        addedTags.reserve(buildList.size());
-
         Log.report(logvisor::Info, _S("Validating resources"));
         progress.setMainIndeterminate(true);
-        for (auto& tag : buildList)
+        for (int i=0 ; i<entry->m_numCookPasses ; ++i)
         {
-            if (addedTags.find(tag) != addedTags.end())
+            std::unordered_set<urde::SObjectTag> addedTags;
-                continue;
+            addedTags.reserve(buildList.size());
-            addedTags.insert(tag);
+            for (auto& tag : buildList)
-
-            hecl::ProjectPath depPath = pathFromTag(tag);
-            if (!depPath)
             {
-                Log.report(logvisor::Fatal, _S("Unable to resolve %.4s %08X"),
+                if (addedTags.find(tag) != addedTags.end())
-                           tag.type.getChars(), tag.id.Value());
+                    continue;
+                addedTags.insert(tag);
+
+                hecl::ProjectPath depPath = pathFromTag(tag);
+                if (!depPath)
+                {
+                    Log.report(logvisor::Fatal, _S("Unable to resolve %.4s %08X"),
+                               tag.type.getChars(), tag.id.Value());
+                }
+                m_project.cookPath(depPath, progress, false, false, fast, entry, cp, i);
             }
-            m_project.cookPath(depPath, progress, false, false, fast, entry, cp);
         }
         progress.setMainIndeterminate(false);
         cp->waitUntilComplete();
@@ -745,7 +785,7 @@ void SpecBase::doPackage(const hecl::ProjectPath& path, const hecl::Database::Da
     /* Write resource data and build file index */
     std::vector<std::tuple<size_t, size_t, bool>> fileIndex;
     Log.report(logvisor::Info, _S("Copying data into %s"), outPath.getRelativePath().data());
-    copyBuildListData(fileIndex, buildList, entry, fast, progress, pakOut);
+    copyBuildListData(fileIndex, buildList, entry, fast, progress, pakOut, mlvlData);
 
     /* Write file index */
     writePakFileIndex(pakOut, buildList, fileIndex, resTableOffset);
@@ -833,7 +873,7 @@ void SpecBase::clearTagCache()
     m_tagToPath.clear();
     m_pathToTag.clear();
     m_catalogNameToTag.clear();
-    m_catalogTagToName.clear();
+    m_catalogTagToNames.clear();
 }
 
 hecl::ProjectPath SpecBase::pathFromTag(const urde::SObjectTag& tag) const
@@ -1030,7 +1070,8 @@ void SpecBase::readCatalog(const hecl::ProjectPath& catalogPath,
         {
             std::unique_lock<std::mutex> lk(m_backgroundIndexMutex);
             m_catalogNameToTag[pLower] = pathTag;
-            m_catalogTagToName[pathTag] = p.first;
+            m_catalogTagToNames[pathTag].insert(p.first);
+
             WriteNameTag(nameWriter, pathTag, p.first);
 #if 0
             fprintf(stderr, "%s %s %08X\n",
@@ -1315,7 +1356,7 @@ void SpecBase::backgroundIndexProc()
                 {
                     std::unique_lock<std::mutex> lk(m_backgroundIndexMutex);
                     m_catalogNameToTag.reserve(nameReader.getRootNode()->m_mapChildren.size());
-                    m_catalogTagToName.reserve(nameReader.getRootNode()->m_mapChildren.size());
+                    m_catalogTagToNames.reserve(nameReader.getRootNode()->m_mapChildren.size());
                     for (const auto& child : nameReader.getRootNode()->m_mapChildren)
                     {
                         unsigned long id = strtoul(child.second->m_scalarString.c_str(), nullptr, 16);
@@ -1325,7 +1366,7 @@ void SpecBase::backgroundIndexProc()
                             std::string chLower = child.first;
                             std::transform(chLower.cbegin(), chLower.cend(), chLower.begin(), tolower);
                             m_catalogNameToTag[chLower] = search->first;
-                            m_catalogTagToName[search->first] = child.first;
+                            m_catalogTagToNames[search->first].insert(child.first);
                             WriteNameTag(nameWriter, search->first, child.first);
                         }
                     }
@@ -1342,7 +1383,7 @@ void SpecBase::backgroundIndexProc()
     if (oidsTag)
     {
         m_catalogNameToTag["mp1originalids"] = oidsTag;
-        m_catalogTagToName[oidsTag] = "MP1OriginalIDs";
+        m_catalogTagToNames[oidsTag].insert("MP1OriginalIDs");
     }
 
     Log.report(logvisor::Info, _S("Background index of '%s' started"), getOriginalSpec().m_name.data());

DataSpec/SpecBase.hpp

@@ -41,7 +41,7 @@ struct SpecBase : hecl::Database::IDataSpec
     bool canExtract(const ExtractPassInfo& info, std::vector<ExtractReport>& reps);
     void doExtract(const ExtractPassInfo& info, const hecl::MultiProgressPrinter& progress);
 
-    bool canCook(const hecl::ProjectPath& path, hecl::blender::Token& btok);
+    bool canCook(const hecl::ProjectPath& path, hecl::blender::Token& btok, int cookPass);
     const hecl::Database::DataSpecEntry* overrideDataSpec(const hecl::ProjectPath& path,
                                                           const hecl::Database::DataSpecEntry* oldEntry,
                                                           hecl::blender::Token& btok) const;
@@ -145,7 +145,8 @@ struct SpecBase : hecl::Database::IDataSpec
                                    hecl::blender::Token& btok,
                                    athena::io::FileWriter& w,
                                    std::vector<urde::SObjectTag>& listOut,
-                                   atUint64& resTableOffset) {}
+                                   atUint64& resTableOffset,
+                                   std::unordered_map<urde::CAssetId, std::vector<uint8_t>>& mlvlData) {}
     virtual void buildPakList(hecl::blender::Token& btok,
                               athena::io::FileWriter& w,
                               const std::vector<urde::SObjectTag>& list,
@@ -198,7 +199,7 @@ protected:
     std::unordered_map<urde::SObjectTag, hecl::ProjectPath> m_tagToPath;
     std::unordered_map<hecl::Hash, urde::SObjectTag> m_pathToTag;
     std::unordered_map<std::string, urde::SObjectTag> m_catalogNameToTag;
-    std::unordered_map<urde::SObjectTag, std::string> m_catalogTagToName;
+    std::unordered_map<urde::SObjectTag, std::unordered_set<std::string>> m_catalogTagToNames;
     void clearTagCache();
 
     hecl::blender::Token m_backgroundBlender;
@@ -227,7 +228,8 @@ protected:
                            const std::vector<urde::SObjectTag>& buildList,
                            const hecl::Database::DataSpecEntry* entry,
                            bool fast, const hecl::MultiProgressPrinter& progress,
-                           athena::io::FileWriter& pakOut);
+                           athena::io::FileWriter& pakOut,
+                           const std::unordered_map<urde::CAssetId, std::vector<uint8_t>>& mlvlData);
 
 protected:
     std::unique_ptr<nod::DiscBase> m_disc;

DataSpec/SpecMP1.cpp

@@ -950,6 +950,7 @@ struct SpecMP1 : SpecBase
             {
                 DNADGRP::DGRP<UniqueID32> dgrp;
                 dgrp.read(reader);
+                dgrp.validateDeps();
                 DNADGRP::WriteDGRP(dgrp, out);
             }
             else if (!classStr.compare(DNAFont::FONT<UniqueID32>::DNAType()))
@@ -1188,7 +1189,8 @@ struct SpecMP1 : SpecBase
                            hecl::blender::Token& btok,
                            athena::io::FileWriter& w,
                            std::vector<urde::SObjectTag>& listOut,
-                           atUint64& resTableOffset)
+                           atUint64& resTableOffset,
+                           std::unordered_map<urde::CAssetId, std::vector<uint8_t>>& mlvlData)
     {
         DNAMP1::MLVL mlvl;
         {
@@ -1200,8 +1202,17 @@ struct SpecMP1 : SpecBase
 
         size_t count = 5;
         for (const auto& area : mlvl.areas)
-            for (const auto& dep : area.deps)
+        {
-                ++count;
+            auto lazyIt = area.lazyDeps.cbegin();
+            auto it = area.deps.cbegin();
+            while (it != area.deps.cend())
+            {
+                if (it->id != lazyIt->id)
+                    ++count;
+                ++lazyIt;
+                ++it;
+            }
+        }
         listOut.reserve(count);
 
         urde::SObjectTag worldTag = tagFromPath(worldPath.getWithExtension(_S(".*"), true), btok);
@@ -1218,7 +1229,7 @@ struct SpecMP1 : SpecBase
         nameEnt.name = parentDir.getLastComponentUTF8();
         nameEnt.write(w);
 
-        for (const auto& area : mlvl.areas)
+        for (auto& area : mlvl.areas)
         {
             urde::SObjectTag nameTag(FOURCC('STRG'), originalToNew(area.areaNameId));
             if (nameTag)
@@ -1228,6 +1239,33 @@ struct SpecMP1 : SpecBase
             urde::SObjectTag areaTag(FOURCC('MREA'), originalToNew(area.areaMREAId));
             if (areaTag)
                 listOut.push_back(areaTag);
+
+            std::vector<DNAMP1::MLVL::Area::Dependency> strippedDeps;
+            strippedDeps.reserve(area.deps.size());
+            std::vector<atUint32> strippedDepLayers;
+            strippedDepLayers.reserve(area.depLayers.size());
+            auto lazyIt = area.lazyDeps.cbegin();
+            auto it = area.deps.cbegin();
+            auto layerIt = area.depLayers.cbegin();
+            while (it != area.deps.cend())
+            {
+                if (it - area.deps.cbegin() == *layerIt)
+                {
+                    strippedDepLayers.push_back(atUint32(strippedDeps.size()));
+                    ++layerIt;
+                }
+                if (it->id != lazyIt->id)
+                    strippedDeps.push_back(*it);
+                ++lazyIt;
+                ++it;
+            }
+
+            area.lazyDepCount = 0;
+            area.lazyDeps.clear();
+            area.depCount = strippedDeps.size();
+            area.deps = std::move(strippedDeps);
+            area.depLayerCount = strippedDepLayers.size();
+            area.depLayers = std::move(strippedDepLayers);
         }
 
         urde::SObjectTag nameTag(FOURCC('STRG'), originalToNew(mlvl.worldNameId));
@@ -1302,6 +1340,15 @@ struct SpecMP1 : SpecBase
             ent.offset = 0;
             ent.write(w);
         }
+
+        {
+            std::vector<uint8_t>& mlvlOut = mlvlData[worldTag.id];
+            size_t mlvlSize = 0;
+            mlvl.binarySize(mlvlSize);
+            mlvlOut.resize(mlvlSize);
+            athena::io::MemoryWriter w(&mlvlOut[0], mlvlSize);
+            mlvl.write(w);
+        }
     }
 
     void buildPakList(hecl::blender::Token& btok,
@@ -1429,13 +1476,13 @@ struct SpecMP1 : SpecBase
 };
 
 hecl::Database::DataSpecEntry SpecEntMP1 = {
-    _S("MP1"sv), _S("Data specification for original Metroid Prime engine"sv), _S(".pak"sv),
+    _S("MP1"sv), _S("Data specification for original Metroid Prime engine"sv), _S(".pak"sv), 2,
     [](hecl::Database::Project& project, hecl::Database::DataSpecTool) -> std::unique_ptr<hecl::Database::IDataSpec> {
         return std::make_unique<SpecMP1>(&SpecEntMP1, project, false);
     }};
 
 hecl::Database::DataSpecEntry SpecEntMP1PC = {
-    _S("MP1-PC"sv), _S("Data specification for PC-optimized Metroid Prime engine"sv), _S(".upak"sv),
+    _S("MP1-PC"sv), _S("Data specification for PC-optimized Metroid Prime engine"sv), _S(".upak"sv), 2,
     [](hecl::Database::Project& project, hecl::Database::DataSpecTool tool) -> std::unique_ptr<hecl::Database::IDataSpec> {
         if (tool != hecl::Database::DataSpecTool::Extract)
             return std::make_unique<SpecMP1>(&SpecEntMP1PC, project, true);
@@ -1443,5 +1490,5 @@ hecl::Database::DataSpecEntry SpecEntMP1PC = {
     }};
 
 hecl::Database::DataSpecEntry SpecEntMP1ORIG = {
-    _S("MP1-ORIG"sv), _S("Data specification for unmodified Metroid Prime resources"sv), {}, {}};
+    _S("MP1-ORIG"sv), _S("Data specification for unmodified Metroid Prime resources"sv), {}, 2, {}};
 }

DataSpec/SpecMP2.cpp

@@ -418,7 +418,7 @@ struct SpecMP2 : SpecBase
 hecl::Database::DataSpecEntry SpecEntMP2
 (
     _S("MP2"sv),
-    _S("Data specification for original Metroid Prime 2 engine"sv), _S(".pak"sv),
+    _S("Data specification for original Metroid Prime 2 engine"sv), _S(".pak"sv), 2,
     [](hecl::Database::Project& project, hecl::Database::DataSpecTool)
     -> std::unique_ptr<hecl::Database::IDataSpec> {return std::make_unique<SpecMP2>(&SpecEntMP2, project, false);}
 );
@@ -426,7 +426,7 @@ hecl::Database::DataSpecEntry SpecEntMP2
 hecl::Database::DataSpecEntry SpecEntMP2PC =
 {
     _S("MP2-PC"sv),
-    _S("Data specification for PC-optimized Metroid Prime 2 engine"sv), _S(".upak"sv),
+    _S("Data specification for PC-optimized Metroid Prime 2 engine"sv), _S(".upak"sv), 2,
     [](hecl::Database::Project& project, hecl::Database::DataSpecTool tool)
     -> std::unique_ptr<hecl::Database::IDataSpec>
     {
@@ -440,7 +440,7 @@ hecl::Database::DataSpecEntry SpecEntMP2ORIG =
 {
     _S("MP2-ORIG"sv),
     _S("Data specification for unmodified Metroid Prime 2 resources"sv),
-    {}, {}
+    {}, 2, {}
 };
 
 }

DataSpec/SpecMP3.cpp

@@ -602,7 +602,7 @@ struct SpecMP3 : SpecBase
 hecl::Database::DataSpecEntry SpecEntMP3
 (
     _S("MP3"sv),
-    _S("Data specification for original Metroid Prime 3 engine"sv), _S(".pak"sv),
+    _S("Data specification for original Metroid Prime 3 engine"sv), _S(".pak"sv), 2,
     [](hecl::Database::Project& project, hecl::Database::DataSpecTool)
     -> std::unique_ptr<hecl::Database::IDataSpec> {return std::make_unique<SpecMP3>(&SpecEntMP3, project, false);}
 );
@@ -610,7 +610,7 @@ hecl::Database::DataSpecEntry SpecEntMP3
 hecl::Database::DataSpecEntry SpecEntMP3PC =
 {
     _S("MP3-PC"sv),
-    _S("Data specification for PC-optimized Metroid Prime 3 engine"sv), _S(".upak"sv),
+    _S("Data specification for PC-optimized Metroid Prime 3 engine"sv), _S(".upak"sv), 2,
     [](hecl::Database::Project& project, hecl::Database::DataSpecTool tool)
     -> std::unique_ptr<hecl::Database::IDataSpec>
     {
@@ -624,7 +624,7 @@ hecl::Database::DataSpecEntry SpecEntMP3ORIG =
 {
     _S("MP3-ORIG"sv),
     _S("Data specification for unmodified Metroid Prime 3 resources"sv),
-    {}, {}
+    {}, 2, {}
 };
 
 }

Runtime/World/CPlayer.cpp

@@ -1893,7 +1893,7 @@ void CPlayer::ProcessInput(const CFinalInput& input, CStateManager& mgr)
     }
 
     UpdateCameraState(mgr);
-    UpdateMorphBallState(input, mgr);
+    UpdateMorphBallState(input.DeltaTime(), input, mgr);
     UpdateCameraTimers(input.DeltaTime(), input);
     UpdateFootstepSounds(input, mgr, input.DeltaTime());
     x2a8_timeSinceJump += input.DeltaTime();
@@ -3291,7 +3291,47 @@ void CPlayer::UpdateCameraTimers(float dt, const CFinalInput& input)
         x29c_fallCameraTimer += dt;
 }
 
-void CPlayer::UpdateMorphBallState(const CFinalInput&, CStateManager& mgr) {}
+void CPlayer::UpdateMorphBallState(float dt, const CFinalInput& input, CStateManager& mgr)
+{
+    if (!ControlMapper::GetPressInput(ControlMapper::ECommands::Morph, input))
+        return;
+    switch (x2f8_morphBallState)
+    {
+    case EPlayerMorphBallState::Unmorphed:
+        if (mgr.GetPlayerState()->HasPowerUp(CPlayerState::EItemType::MorphBall) &&
+            CanEnterMorphBallState(mgr, 0.f))
+        {
+            x574_morphTime = 0.f;
+            x578_morphDuration = 1.f;
+            TransitionToMorphBallState(dt, mgr);
+        }
+        else
+        {
+            CSfxHandle hnd = CSfxManager::SfxStart(1781, 1.f, 0.f, true, 0x7f, false, kInvalidAreaId);
+            ApplySubmergedPitchBend(hnd);
+        }
+        break;
+    case EPlayerMorphBallState::Morphed:
+    {
+        zeus::CVector3f posDelta;
+        if (CanLeaveMorphBallState(mgr, posDelta))
+        {
+            SetTranslation(x34_transform.origin + posDelta);
+            x574_morphTime = 0.f;
+            x578_morphDuration = 1.f;
+            TransitionFromMorphBallState(mgr);
+        }
+        else
+        {
+            CSfxHandle hnd = CSfxManager::SfxStart(1781, 1.f, 0.f, true, 0x7f, false, kInvalidAreaId);
+            ApplySubmergedPitchBend(hnd);
+        }
+        break;
+    }
+    default:
+        break;
+    }
+}
 
 CFirstPersonCamera& CPlayer::GetFirstPersonCamera(CStateManager& mgr)
 {

Runtime/World/CPlayer.hpp

@@ -523,7 +523,7 @@ public:
     void UpdateCameraState(CStateManager& mgr);
     void UpdateDebugCamera(CStateManager& mgr);
     void UpdateCameraTimers(float dt, const CFinalInput& input);
-    void UpdateMorphBallState(const CFinalInput&, CStateManager& mgr);
+    void UpdateMorphBallState(float dt, const CFinalInput&, CStateManager& mgr);
     CFirstPersonCamera& GetFirstPersonCamera(CStateManager& mgr);
     void UpdateGunTransform(const zeus::CVector3f&, CStateManager& mgr);
     void UpdateAssistedAiming(const zeus::CTransform& xf, const CStateManager& mgr);

hecl

@@ -1 +1 @@
-Subproject commit e7c678ff572d941fa648ef99c92d90624a582761
+Subproject commit b36da710f38bd67e90cc961bd075976d610f9fc8