#ifndef _DNACOMMON_CMDL_HPP_ #define _DNACOMMON_CMDL_HPP_ #include "PAK.hpp" #include "BlenderConnection.hpp" #include "GX.hpp" #include "TXTR.hpp" namespace Retro { namespace DNACMDL { struct Header : BigDNA { DECL_DNA Value magic; Value version; struct Flags : BigDNA { DECL_DNA Value flags; bool shortNormals() const {return (flags & 0x2) != 0;} void setShortNormals(bool val) {flags &= ~0x2; flags |= val << 1;} bool shortUVs() const {return (flags & 0x4) != 0;} void setShortUVs(bool val) {flags &= ~0x4; flags |= val << 2;} } flags; Value aabbMin; Value aabbMax; Value secCount; Value matSetCount; Vector secSizes; Align<32> align; }; struct SurfaceHeader_1_2 : BigDNA { DECL_DNA Value centroid; Value matIdx; Value qDiv; Value dlSize; Value unk1; Value unk2; Value aabbSz; Value reflectionNormal; Seek seek2; Align<32> align; static constexpr bool UseMatrixSkinning() {return false;} constexpr atInt16 skinMatrixBankIdx() const {return -1;} }; struct SurfaceHeader_3 : BigDNA { DECL_DNA Value centroid; Value matIdx; Value qDiv; Value dlSize; Value unk1; Value unk2; Value aabbSz; Value reflectionNormal; Value skinMtxBankIdx; Value unk3; Seek seek2; Align<32> align; static constexpr bool UseMatrixSkinning() {return true;} atInt16 skinMatrixBankIdx() const {return skinMtxBankIdx;} }; struct VertexAttributes { GX::AttrType pos = GX::NONE; GX::AttrType norm = GX::NONE; GX::AttrType color0 = GX::NONE; GX::AttrType color1 = GX::NONE; unsigned uvCount = 0; GX::AttrType uvs[7] = {GX::NONE}; GX::AttrType pnMtxIdx = GX::NONE; unsigned texMtxIdxCount = 0; GX::AttrType texMtxIdx[7] = {GX::NONE}; bool shortUVs; }; template void GetVertexAttributes(const MaterialSet& matSet, std::vector& attributesOut) { attributesOut.clear(); attributesOut.reserve(matSet.materials.size()); for (const typename MaterialSet::Material& mat : matSet.materials) { const typename MaterialSet::Material::VAFlags& vaFlags = mat.getVAFlags(); attributesOut.emplace_back(); VertexAttributes& va = attributesOut.back(); va.pos = vaFlags.position(); va.norm = vaFlags.normal(); va.color0 = vaFlags.color0(); va.color1 = vaFlags.color1(); if ((va.uvs[0] = vaFlags.tex0())) ++va.uvCount; if ((va.uvs[1] = vaFlags.tex1())) ++va.uvCount; if ((va.uvs[2] = vaFlags.tex2())) ++va.uvCount; if ((va.uvs[3] = vaFlags.tex3())) ++va.uvCount; if ((va.uvs[4] = vaFlags.tex4())) ++va.uvCount; if ((va.uvs[5] = vaFlags.tex5())) ++va.uvCount; if ((va.uvs[6] = vaFlags.tex6())) ++va.uvCount; va.pnMtxIdx = vaFlags.pnMatIdx(); if ((va.texMtxIdx[0] = vaFlags.tex0MatIdx())) ++va.texMtxIdxCount; if ((va.texMtxIdx[1] = vaFlags.tex1MatIdx())) ++va.texMtxIdxCount; if ((va.texMtxIdx[2] = vaFlags.tex2MatIdx())) ++va.texMtxIdxCount; if ((va.texMtxIdx[3] = vaFlags.tex3MatIdx())) ++va.texMtxIdxCount; if ((va.texMtxIdx[4] = vaFlags.tex4MatIdx())) ++va.texMtxIdxCount; if ((va.texMtxIdx[5] = vaFlags.tex5MatIdx())) ++va.texMtxIdxCount; if ((va.texMtxIdx[6] = vaFlags.tex6MatIdx())) ++va.texMtxIdxCount; va.shortUVs = mat.getFlags().lightmapUVArray(); } } template void ReadMaterialSetToBlender_1_2(HECL::BlenderConnection::PyOutStream& os, const MaterialSet& matSet, const PAKRouter& pakRouter, const typename PAKRouter::EntryType& entry, unsigned setIdx) { /* Texmaps */ os << "texmap_list = []\n"; for (const UniqueID32& tex : matSet.head.textureIDs) { std::string texName = pakRouter.getBestEntryName(tex); const NOD::Node* node; const typename PAKRouter::EntryType* texEntry = pakRouter.lookupEntry(tex, &node); HECL::ProjectPath txtrPath = pakRouter.getWorking(texEntry); if (txtrPath.getPathType() == HECL::ProjectPath::PT_NONE) { PAKEntryReadStream rs = texEntry->beginReadStream(*node); TXTR::Extract(rs, txtrPath); } HECL::SystemString resPath = pakRouter.getResourceRelativePath(entry, tex); HECL::SystemUTF8View resPathView(resPath); os.format("if '%s' in bpy.data.textures:\n" " image = bpy.data.images['%s']\n" " texture = bpy.data.textures[image.name]\n" "else:\n" " image = bpy.data.images.load('''//%s''')\n" " image.name = '%s'\n" " texture = bpy.data.textures.new(image.name, 'IMAGE')\n" " texture.image = image\n" "texmap_list.append(texture)\n" "\n", texName.c_str(), texName.c_str(), resPathView.str().c_str(), texName.c_str()); } unsigned m=0; for (const typename MaterialSet::Material& mat : matSet.materials) { MaterialSet::ConstructMaterial(os, mat, setIdx, m++); os << "materials.append(new_material)\n"; } } template void ReadMaterialSetToBlender_3(HECL::BlenderConnection::PyOutStream& os, const MaterialSet& matSet, const PAKRouter& pakRouter, const typename PAKRouter::EntryType& entry, unsigned setIdx) { unsigned m=0; for (const typename MaterialSet::Material& mat : matSet.materials) { MaterialSet::ConstructMaterial(os, pakRouter, entry, mat, setIdx, m++); os << "materials.append(new_material)\n"; } } class DLReader { public: /* Class used for splitting verts with shared positions but different skinning matrices */ class ExtraVertTracker { std::map>> m_extraVerts; atUint16 m_maxBasePos = 0; atUint16 m_nextOverPos = 1; public: atInt16 addPosSkinPair(atUint16 pos, atInt16 skin) { m_maxBasePos = std::max(m_maxBasePos, pos); auto search = m_extraVerts.find(pos); if (search == m_extraVerts.end()) { m_extraVerts[pos] = {std::make_pair(skin, 0)}; return skin; } std::vector>& vertTrack = search->second; for (const std::pair& s : vertTrack) if (s.first == skin) return vertTrack.front().first; vertTrack.push_back(std::make_pair(skin, m_nextOverPos++)); return vertTrack.front().first; } template void sendAdditionalVertsToBlender(HECL::BlenderConnection::PyOutStream& os, const RigPair& rp) const { atUint32 nextVert = 1; while (nextVert < m_nextOverPos) { for (const std::pair>>& ev : m_extraVerts) { for (const std::pair& se : ev.second) { if (se.second == nextVert) { os.format("bm.verts.ensure_lookup_table()\n" "orig_vert = bm.verts[%u]\n" "vert = bm.verts.new(orig_vert.co)\n", ev.first); rp.first->weightVertex(os, *rp.second, se.first); ++nextVert; } } } } } atUint16 lookupVertIdx(atUint16 pos, atInt16 skin) const { auto search = m_extraVerts.find(pos); if (search == m_extraVerts.end()) return -1; const std::vector>& vertTrack = search->second; if (vertTrack.front().first == skin) return pos; for (auto it=vertTrack.begin()+1 ; it!=vertTrack.end() ; ++it) if (it->first == skin) return m_maxBasePos + it->second; return -1; } }; private: const VertexAttributes& m_va; std::unique_ptr m_dl; size_t m_dlSize; ExtraVertTracker& m_evt; const atInt16* m_bankIn; atUint8* m_cur; atUint16 readVal(GX::AttrType type) { atUint16 retval = 0; switch (type) { case GX::DIRECT: case GX::INDEX8: if ((m_cur - m_dl.get()) >= intptr_t(m_dlSize)) return 0; retval = *m_cur; ++m_cur; break; case GX::INDEX16: if ((m_cur - m_dl.get() + 1) >= intptr_t(m_dlSize)) return 0; retval = HECL::SBig(*(atUint16*)m_cur); m_cur += 2; break; default: break; } return retval; } public: DLReader(const VertexAttributes& va, std::unique_ptr&& dl, size_t dlSize, ExtraVertTracker& evt, const atInt16* bankIn=nullptr) : m_va(va), m_dl(std::move(dl)), m_dlSize(dlSize), m_evt(evt), m_bankIn(bankIn) { m_cur = m_dl.get(); } operator bool() { return ((m_cur - m_dl.get()) < intptr_t(m_dlSize)) && *m_cur; } GX::Primitive readPrimitive() { return GX::Primitive(*m_cur++ & 0xf8); } GX::Primitive readPrimitiveAndVat(unsigned& vatOut) { atUint8 val = *m_cur++; vatOut = val & 0x7; return GX::Primitive(val & 0xf8); } atUint16 readVertCount() { atUint16 retval = HECL::SBig(*(atUint16*)m_cur); m_cur += 2; return retval; } struct DLPrimVert { atUint16 pos = 0; atUint16 norm = 0; atUint16 color[2] = {0}; atUint16 uvs[7] = {0}; atUint8 pnMtxIdx = 0; atUint8 texMtxIdx[7] = {0}; }; DLPrimVert readVert(bool peek=false) { atUint8* bakCur = m_cur; DLPrimVert retval; retval.pnMtxIdx = readVal(m_va.pnMtxIdx); retval.texMtxIdx[0] = readVal(m_va.texMtxIdx[0]); retval.texMtxIdx[1] = readVal(m_va.texMtxIdx[1]); retval.texMtxIdx[2] = readVal(m_va.texMtxIdx[2]); retval.texMtxIdx[3] = readVal(m_va.texMtxIdx[3]); retval.texMtxIdx[4] = readVal(m_va.texMtxIdx[4]); retval.texMtxIdx[5] = readVal(m_va.texMtxIdx[5]); retval.texMtxIdx[6] = readVal(m_va.texMtxIdx[6]); if (m_bankIn) { atUint16 posIdx = readVal(m_va.pos); atUint8 mtxIdx = retval.pnMtxIdx / 3; atInt16 skinIdx = -1; if (mtxIdx < 10) skinIdx = m_bankIn[mtxIdx]; retval.pos = m_evt.lookupVertIdx(posIdx, skinIdx); } else retval.pos = readVal(m_va.pos); retval.norm = readVal(m_va.norm); retval.color[0] = readVal(m_va.color0); retval.color[1] = readVal(m_va.color1); retval.uvs[0] = readVal(m_va.uvs[0]); retval.uvs[1] = readVal(m_va.uvs[1]); retval.uvs[2] = readVal(m_va.uvs[2]); retval.uvs[3] = readVal(m_va.uvs[3]); retval.uvs[4] = readVal(m_va.uvs[4]); retval.uvs[5] = readVal(m_va.uvs[5]); retval.uvs[6] = readVal(m_va.uvs[6]); if (peek) m_cur = bakCur; return retval; } void preReadMaxIdxs(DLPrimVert& out) { atUint8* bakCur = m_cur; while (*this) { readPrimitive(); atUint16 vc = readVertCount(); for (atUint16 v=0 ; v& skinOut) { atUint8* bakCur = m_cur; while (*this) { readPrimitive(); atUint16 vc = readVertCount(); for (atUint16 v=0 ; v atUint32 ReadGeomSectionsToBlender(HECL::BlenderConnection::PyOutStream& os, Athena::io::IStreamReader& reader, PAKRouter& pakRouter, const typename PAKRouter::EntryType& entry, const RigPair& rp, bool shortNormals, bool shortUVs, std::vector& vertAttribs, int meshIdx, atUint32 secCount, atUint32 matSetCount, const atUint32* secSizes, atUint32 surfaceCount=0) { os << "# Begin bmesh\n" "bm = bmesh.new()\n" "\n" "# Overdraw-tracking\n" "od_list = []\n" "\n" "orig_pidx_lay = bm.verts.layers.int.new('CMDLOriginalPosIdxs')\n" "orig_nidx_lay = bm.loops.layers.int.new('CMDLOriginalNormIdxs')\n"; if (rp.first) os << "dvert_lay = bm.verts.layers.deform.verify()\n"; /* Pre-read pass to determine maximum used vert indices */ atUint32 matSecCount = 0; if (matSetCount) matSecCount = MaterialSet::OneSection() ? 1 : matSetCount; bool visitedDLOffsets = false; atUint32 lastDlSec = secCount; atUint64 afterHeaderPos = reader.position(); DLReader::DLPrimVert maxIdxs; std::vector skinIndices; DLReader::ExtraVertTracker extraTracker; for (size_t s=0 ; sgetMatrixBank(sHead.skinMatrixBankIdx()); /* Do max index pre-read */ atUint32 realDlSize = secSizes[s] - (reader.position() - secStart); DLReader dl(vertAttribs[sHead.matIdx], reader.readUBytes(realDlSize), realDlSize, extraTracker, bankIn); if (SurfaceHeader::UseMatrixSkinning() && rp.first) dl.preReadMaxIdxs(maxIdxs, skinIndices); else dl.preReadMaxIdxs(maxIdxs); } } } if (s < secCount - 1) reader.seek(secStart + secSizes[s], Athena::Begin); } reader.seek(afterHeaderPos, Athena::Begin); visitedDLOffsets = false; unsigned createdUVLayers = 0; unsigned surfIdx = 0; for (size_t s=0 ; sweightVertex(os, *rp.second, skinIndices[i]); else if (!SurfaceHeader::UseMatrixSkinning()) rp.first->weightVertex(os, *rp.second, i); } } if (rp.first && SurfaceHeader::UseMatrixSkinning() && !skinIndices.empty()) extraTracker.sendAdditionalVertsToBlender(os, rp); break; } case 1: { /* Normals */ os << "norm_list = []\n"; if (shortNormals) { size_t normCount = secSizes[s] / 6; for (size_t i=0 ; igetMatrixBank(sHead.skinMatrixBankIdx()); os.format("materials[%u].pass_index = %u\n", sHead.matIdx, surfIdx++); if (matUVCount > createdUVLayers) { for (unsigned l=createdUVLayers ; l= vertCount - 3); /* Advance one prim vert */ primVerts[c%3] = dl.readVert(peek); ++c; } } else if (ptype == GX::TRIANGLES) { for (int v=0 ; v= vertCount) break; /* Advance 3 Prim Verts */ for (int pv=0 ; pv<3 ; ++pv) primVerts[pv] = dl.readVert(); } } else if (ptype == GX::TRIANGLEFAN) { ++c; for (int v=0 ; v= vertCount) break; /* Advance one prim vert */ primVerts[(c+2)%3] = dl.readVert(); ++c; } } os << "\n"; } } } } if (s < secCount - 1) reader.seek(secStart + secSizes[s], Athena::Begin); } /* Finish Mesh */ FinishBlenderMesh(os, matSetCount, meshIdx); if (rp.first) rp.second->sendVertexGroupsToBlender(os); return lastDlSec; } template bool ReadCMDLToBlender(HECL::BlenderConnection& conn, Athena::io::IStreamReader& reader, PAKRouter& pakRouter, const typename PAKRouter::EntryType& entry, const SpecBase& dataspec, const RigPair& rp) { Header head; head.read(reader); if (head.magic != 0xDEADBABE) { LogDNACommon.report(LogVisor::Error, "invalid CMDL magic"); return false; } if (head.version != Version) { LogDNACommon.report(LogVisor::Error, "invalid CMDL version"); return false; } /* Open Py Stream and read sections */ HECL::BlenderConnection::PyOutStream os = conn.beginPythonOut(true); os.format("import bpy\n" "import bmesh\n" "\n" "bpy.context.scene.name = '%s'\n" "bpy.context.scene.hecl_mesh_obj = bpy.context.scene.name\n", pakRouter.getBestEntryName(entry).c_str()); InitGeomBlenderContext(os, dataspec.getMasterShaderPath()); MaterialSet::RegisterMaterialProps(os); os << "# Materials\n" "materials = []\n" "\n"; std::vector vertAttribs; ReadGeomSectionsToBlender (os, reader, pakRouter, entry, rp, head.flags.shortNormals(), head.flags.shortUVs(), vertAttribs, -1, head.secCount, head.matSetCount, head.secSizes.data()); return true; } } } #endif // _DNACOMMON_CMDL_HPP_