Various space-saving cook optimizations and fixes

This commit is contained in:
Jack Andersen 2018-05-05 10:56:42 -10:00
parent 54d2759ace
commit 70657efa82
12 changed files with 259 additions and 105 deletions

View File

@ -141,6 +141,9 @@ endif()
add_subdirectory(discord-rpc/src)
include_directories(${CMAKE_CURRENT_SOURCE_DIR}/discord-rpc/include)
add_subdirectory(nod)
set(NOD_INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/nod/include)
set(HECL_DLPACKAGE ${URDE_DLPACKAGE})
set(BOO_INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/hecl/extern/boo/include)
@ -194,9 +197,7 @@ include_directories(${BOO_SYS_INCLUDES})
add_subdirectory(amuse)
add_subdirectory(specter)
set(SPECTER_INCLUDE_DIR specter/include specter/freetype2/include)
add_subdirectory(nod)
add_subdirectory(assetnameparser)
set(NOD_INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/nod/include)
add_definitions(-DZE_ATHENA_TYPES=1)
set(ZEUS_INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/specter/zeus/include)
include_directories(${ATHENA_INCLUDE_DIR} ${LOGVISOR_INCLUDE_DIR} ${HECL_INCLUDE_DIR}

View File

@ -7,6 +7,7 @@ namespace DataSpec
logvisor::Module Log("AROTBuilder");
#define AROT_MAX_LEVEL 6
#define AROT_MIN_MODELS 8
#define COLLISION_MIN_NODE_TRIANGLES 16
#define PATH_MIN_NODE_REGIONS 16
@ -348,7 +349,7 @@ void AROTBuilder::build(std::vector<std::vector<uint8_t>>& secs, const zeus::CAA
{
/* Recursively split */
BspNodeType rootType;
rootNode.addChild(0, 1, meshAabbs, fullAabb, rootType);
rootNode.addChild(0, AROT_MIN_MODELS, meshAabbs, fullAabb, rootType);
/* Calculate indexing metrics */
size_t totalNodeCount = 0;

View File

@ -1354,12 +1354,34 @@ bool WriteCMDL(const hecl::ProjectPath& outPath, const hecl::ProjectPath& inPath
/* Surfaces */
size_t endOff = 0;
size_t firstSurfSec = head.secSizes.size();
for (const Mesh::Surface& surf : mesh.surfaces)
{
size_t vertSz = matSets.at(0).materials.at(surf.materialIdx).getVAFlags().vertDLSize();
if (surf.verts.size() > 65536)
LogDNACommon.report(logvisor::Fatal, "GX DisplayList overflow");
size_t secSz = 67 + surf.verts.size() * vertSz;
size_t secSz = 64;
for (auto it = surf.verts.cbegin() ; it != surf.verts.cend() ;)
{
atUint16 vertCount = 0;
auto itEnd = surf.verts.cend();
for (auto it2 = it ; it2 != surf.verts.cend() ; ++it2, ++vertCount)
if (it2->iPos == 0xffffffff)
{
if (vertCount == 3)
{
/* All primitives here on out are triangles */
vertCount = atUint16((surf.verts.cend() - it + 1) * 3 / 4);
break;
}
itEnd = it2;
break;
}
secSz += 3 + vertCount * vertSz;
if (itEnd == surf.verts.cend())
break;
it = itEnd + 1;
}
secSz32 = ROUND_UP_32(secSz);
if (secSz32 == 0)
secSz32 = 32;
@ -1435,33 +1457,55 @@ bool WriteCMDL(const hecl::ProjectPath& outPath, const hecl::ProjectPath& inPath
++padIt;
/* Surfaces */
GX::Primitive prim;
GX::Primitive prim = GX::TRIANGLES;
if (mesh.topology == hecl::HMDLTopology::Triangles)
prim = GX::TRIANGLES;
else if (mesh.topology == hecl::HMDLTopology::TriStrips)
prim = GX::TRIANGLESTRIP;
else
LogDNACommon.report(logvisor::Fatal, "unrecognized mesh output mode");
auto surfSizeIt = head.secSizes.begin() + firstSurfSec;
for (const Mesh::Surface& surf : mesh.surfaces)
{
const typename MaterialSet::Material::VAFlags& vaFlags =
matSets.at(0).materials.at(surf.materialIdx).getVAFlags();
size_t vertSz = vaFlags.vertDLSize();
SurfaceHeader header;
header.centroid = surf.centroid;
header.matIdx = surf.materialIdx;
header.dlSize = ROUND_UP_32(3 + surf.verts.size() * vertSz);
header.dlSize = (*surfSizeIt++ - 64) | 0x80000000;
header.reflectionNormal = surf.reflectionNormal;
header.write(writer);
GX::Primitive usePrim = prim;
for (auto it = surf.verts.cbegin() ; it != surf.verts.cend() ;)
{
atUint16 vertCount = 0;
auto itEnd = surf.verts.cend();
for (auto it2 = it ; it2 != surf.verts.cend() ; ++it2, ++vertCount)
if (it2->iPos == 0xffffffff)
{
if (vertCount == 3)
{
/* All primitives here on out are triangles */
usePrim = GX::TRIANGLES;
vertCount = atUint16((surf.verts.cend() - it + 1) * 3 / 4);
break;
}
itEnd = it2;
break;
}
/* VAT0 = float normals, float UVs
* VAT1 = short normals, float UVs */
writer.writeUByte(prim | (skinned ? 0x0 : 0x1));
writer.writeUint16Big(surf.verts.size());
writer.writeUByte(usePrim | (skinned ? 0x0 : 0x1));
writer.writeUint16Big(vertCount);
for (const Mesh::Surface::Vert& vert : surf.verts)
for (auto it2 = it ; it2 != itEnd ; ++it2)
{
const Mesh::Surface::Vert& vert = *it2;
if (vert.iPos == 0xffffffff)
continue;
atUint32 skinIdx = vert.iBankSkin * 3;
WriteDLVal(writer, vaFlags.pnMatIdx(), skinIdx);
WriteDLVal(writer, vaFlags.tex0MatIdx(), skinIdx);
@ -1484,6 +1528,11 @@ bool WriteCMDL(const hecl::ProjectPath& outPath, const hecl::ProjectPath& inPath
WriteDLVal(writer, vaFlags.tex6(), vert.iUv[6]);
}
if (itEnd == surf.verts.cend())
break;
it = itEnd + 1;
}
writer.fill(atUint8(0), *padIt);
++padIt;
}
@ -1772,7 +1821,6 @@ bool WriteMREASecs(std::vector<std::vector<uint8_t>>& secsOut, const hecl::Proje
/* Iterate meshes */
auto matIt = surfToGlobalMats.cbegin();
int meshIdx = 0;
for (const Mesh& mesh : meshes)
{
zeus::CTransform meshXf(mesh.sceneXf.val);
@ -1808,7 +1856,28 @@ bool WriteMREASecs(std::vector<std::vector<uint8_t>>& secsOut, const hecl::Proje
matSet.materials.at(*smatIt++).getVAFlags();
size_t vertSz = vaFlags.vertDLSize();
endOff += 96 + vertSz * surf.verts.size() + 4;
endOff += 96;
for (auto it = surf.verts.cbegin() ; it != surf.verts.cend() ;)
{
atUint16 vertCount = 0;
auto itEnd = surf.verts.cend();
for (auto it2 = it ; it2 != surf.verts.cend() ; ++it2, ++vertCount)
if (it2->iPos == 0xffffffff)
{
if (vertCount == 3)
{
/* All primitives here on out are triangles */
vertCount = atUint16((surf.verts.cend() - it + 1) * 3 / 4);
break;
}
itEnd = it2;
break;
}
endOff += 3 + vertSz * vertCount;
if (itEnd == surf.verts.cend())
break;
it = itEnd + 1;
}
endOff = ROUND_UP_32(endOff);
surfEndOffs.push_back(endOff);
}
@ -1899,24 +1968,27 @@ bool WriteMREASecs(std::vector<std::vector<uint8_t>>& secsOut, const hecl::Proje
}
/* Surfaces */
GX::Primitive prim;
GX::Primitive prim = GX::TRIANGLES;
if (mesh.topology == hecl::HMDLTopology::Triangles)
prim = GX::TRIANGLES;
else if (mesh.topology == hecl::HMDLTopology::TriStrips)
prim = GX::TRIANGLESTRIP;
else
LogDNACommon.report(logvisor::Fatal, "unrecognized mesh output mode");
auto surfEndOffIt = surfEndOffs.begin();
size_t lastEndOff = 0;
for (const Mesh::Surface& surf : mesh.surfaces)
{
size_t matIdx = *matIt++;
const typename MaterialSet::Material& mat = matSet.materials.at(matIdx);
const typename MaterialSet::Material::VAFlags& vaFlags = mat.getVAFlags();
size_t vertSz = vaFlags.vertDLSize();
SurfaceHeader header;
header.centroid = meshXf * zeus::CVector3f(surf.centroid);
header.matIdx = matIdx;
header.dlSize = ROUND_UP_32(3 + surf.verts.size() * vertSz);
uint32_t dlSize = uint32_t(*surfEndOffIt - lastEndOff - 96);
header.dlSize = dlSize | 0x80000000;
lastEndOff = *surfEndOffIt++;
header.reflectionNormal = (meshXf.basis * zeus::CVector3f(surf.reflectionNormal)).normalized();
header.aabbSz = 24;
zeus::CAABox aabb(zeus::CVector3f(surf.aabbMin), zeus::CVector3f(surf.aabbMax));
@ -1926,19 +1998,41 @@ bool WriteMREASecs(std::vector<std::vector<uint8_t>>& secsOut, const hecl::Proje
size_t secSz = 0;
header.binarySize(secSz);
secSz += 3 + surf.verts.size() * vertSz;
secSz += dlSize;
secSz = ROUND_UP_32(secSz);
secsOut.emplace_back(secSz, 0);
athena::io::MemoryWriter w(secsOut.back().data(), secsOut.back().size());
header.write(w);
GX::Primitive usePrim = prim;
for (auto it = surf.verts.cbegin() ; it != surf.verts.cend() ;)
{
atUint16 vertCount = 0;
auto itEnd = surf.verts.cend();
for (auto it2 = it ; it2 != surf.verts.cend() ; ++it2, ++vertCount)
if (it2->iPos == 0xffffffff)
{
if (vertCount == 3)
{
/* All primitives here on out are triangles */
usePrim = GX::TRIANGLES;
vertCount = atUint16((surf.verts.cend() - it + 1) * 3 / 4);
break;
}
itEnd = it2;
break;
}
/* VAT1 = short normals, float UVs
* VAT2 = short normals, short UVs */
w.writeUByte(prim | (mat.flags.lightmapUVArray() ? 0x2 : 0x1));
w.writeUint16Big(surf.verts.size());
w.writeUByte(usePrim | (mat.flags.lightmapUVArray() ? 0x2 : 0x1));
w.writeUint16Big(vertCount);
for (const Mesh::Surface::Vert& vert : surf.verts)
for (auto it2 = it ; it2 != itEnd ; ++it2)
{
const Mesh::Surface::Vert& vert = *it2;
if (vert.iPos == 0xffffffff)
continue;
atUint32 skinIdx = vert.iBankSkin * 3;
WriteDLVal(w, vaFlags.pnMatIdx(), skinIdx);
WriteDLVal(w, vaFlags.tex0MatIdx(), skinIdx);
@ -1960,6 +2054,11 @@ bool WriteMREASecs(std::vector<std::vector<uint8_t>>& secsOut, const hecl::Proje
WriteDLVal(w, vaFlags.tex5(), vert.iUv[5]);
WriteDLVal(w, vaFlags.tex6(), vert.iUv[6]);
}
if (itEnd == surf.verts.cend())
break;
it = itEnd + 1;
}
}
}
@ -2040,7 +2139,6 @@ bool WriteHMDLMREASecs(std::vector<std::vector<uint8_t>>& secsOut, const hecl::P
/* Iterate meshes */
auto matIt = surfToGlobalMats.cbegin();
int meshIdx = 0;
for (const Mesh& mesh : meshes)
{
zeus::CTransform meshXf(mesh.sceneXf.val);
@ -2151,10 +2249,8 @@ void SurfaceHeader_1::Enumerate<BigDNA::Read>(typename Read::StreamT& reader)
centroid = reader.readVec3fBig();
/* matIdx */
matIdx = reader.readUint32Big();
/* qDiv */
qDiv = reader.readUint16Big();
/* dlSize */
dlSize = reader.readUint16Big();
dlSize = reader.readUint32Big();
/* idxStart */
idxStart = reader.readUint32Big();
/* idxCount */
@ -2183,10 +2279,8 @@ void SurfaceHeader_1::Enumerate<BigDNA::Write>(typename Write::StreamT& writer)
writer.writeVec3fBig(centroid);
/* matIdx */
writer.writeUint32Big(matIdx);
/* qDiv */
writer.writeUint16Big(qDiv);
/* dlSize */
writer.writeUint16Big(dlSize);
writer.writeUint32Big(dlSize);
/* idxStart */
writer.writeUint32Big(idxStart);
/* idxCount */
@ -2220,10 +2314,8 @@ void SurfaceHeader_2::Enumerate<BigDNA::Read>(typename Read::StreamT& reader)
centroid = reader.readVec3fBig();
/* matIdx */
matIdx = reader.readUint32Big();
/* qDiv */
qDiv = reader.readUint16Big();
/* dlSize */
dlSize = reader.readUint16Big();
dlSize = reader.readUint32Big();
/* idxStart */
idxStart = reader.readUint32Big();
/* idxCount */
@ -2256,10 +2348,8 @@ void SurfaceHeader_2::Enumerate<BigDNA::Write>(typename Write::StreamT& writer)
writer.writeVec3fBig(centroid);
/* matIdx */
writer.writeUint32Big(matIdx);
/* qDiv */
writer.writeUint16Big(qDiv);
/* dlSize */
writer.writeUint16Big(dlSize);
writer.writeUint32Big(dlSize);
/* idxStart */
writer.writeUint32Big(idxStart);
/* idxCount */
@ -2297,10 +2387,8 @@ void SurfaceHeader_3::Enumerate<BigDNA::Read>(typename Read::StreamT& reader)
centroid = reader.readVec3fBig();
/* matIdx */
matIdx = reader.readUint32Big();
/* qDiv */
qDiv = reader.readUint16Big();
/* dlSize */
dlSize = reader.readUint16Big();
dlSize = reader.readUint32Big();
/* idxStart */
idxStart = reader.readUint32Big();
/* idxCount */
@ -2335,10 +2423,8 @@ void SurfaceHeader_3::Enumerate<BigDNA::Write>(typename Write::StreamT& writer)
writer.writeVec3fBig(centroid);
/* matIdx */
writer.writeUint32Big(matIdx);
/* qDiv */
writer.writeUint16Big(qDiv);
/* dlSize */
writer.writeUint16Big(dlSize);
writer.writeUint32Big(dlSize);
/* idxStart */
writer.writeUint32Big(idxStart);
/* idxCount */

View File

@ -44,8 +44,7 @@ struct SurfaceHeader_1 : BigDNA
AT_DECL_EXPLICIT_DNA
Value<atVec3f> centroid;
Value<atUint32> matIdx = 0;
Value<atUint16> qDiv = 0x8000;
Value<atUint16> dlSize = 0;
Value<atUint32> dlSize = 0;
Value<atUint32> idxStart = 0; /* Actually used by game to stash CCubeModel pointer */
Value<atUint32> idxCount = 0; /* Actually used by game to stash next CCubeSurface pointer */
Value<atUint32> aabbSz = 0;
@ -62,8 +61,7 @@ struct SurfaceHeader_2 : BigDNA
AT_DECL_EXPLICIT_DNA
Value<atVec3f> centroid;
Value<atUint32> matIdx = 0;
Value<atUint16> qDiv = 0x8000;
Value<atUint16> dlSize = 0;
Value<atUint32> dlSize = 0;
Value<atUint32> idxStart = 0; /* Actually used by game to stash CCubeModel pointer */
Value<atUint32> idxCount = 0; /* Actually used by game to stash next CCubeSurface pointer */
Value<atUint32> aabbSz = 0;
@ -82,8 +80,7 @@ struct SurfaceHeader_3 : BigDNA
AT_DECL_EXPLICIT_DNA
Value<atVec3f> centroid;
Value<atUint32> matIdx = 0;
Value<atUint16> qDiv = 0x8000;
Value<atUint16> dlSize = 0;
Value<atUint32> dlSize = 0;
Value<atUint32> idxStart = 0; /* Actually used by game to stash CCubeModel pointer */
Value<atUint32> idxCount = 0; /* Actually used by game to stash next CCubeSurface pointer */
Value<atUint32> aabbSz = 0;

View File

@ -53,6 +53,20 @@ bool MLVL::Extract(const SpecBase& dataSpec, PAKEntryReadStream& rs, const hecl:
}
if (pakRouter.mreaHasDupeResources(area.areaMREAId))
athena::io::FileWriter(hecl::ProjectPath(areaDir, _S("!duperes")).getAbsolutePath());
if (areaIdx == 0)
{
char path[256];
snprintf(path, 256, "/Users/jacko/Desktop/res/orig_%08X.txt", entry.id.toUint32());
FILE* fp = fopen(path, "w");
for (const auto& dep : area.deps)
{
hecl::ProjectPath depPath = pakRouter.getWorking(dep.id);
fprintf(fp, "%s\n", depPath.getRelativePathUTF8().data());
}
fclose(fp);
}
areaIdx++;
}

View File

@ -36,15 +36,15 @@ struct FishCloud : IScriptObject
Value<atVec4f> unknown17; // CColor
Value<bool> unknown18;
Value<float> unknown19;
Value<atUint32> unknown20;
Value<atUint32> unknown21;
Value<atUint32> unknown22;
Value<atUint32> unknown23;
Value<atUint32> unknown24;
Value<atUint32> unknown25;
Value<atUint32> unknown26;
Value<atUint32> unknown27;
Value<atUint32> unknown28;
UniqueID32 deathParticle1;
Value<atUint32> deathParticle1Frames;
UniqueID32 deathParticle2;
Value<atUint32> deathParticle2Frames;
UniqueID32 deathParticle3;
Value<atUint32> deathParticle3Frames;
UniqueID32 deathParticle4;
Value<atUint32> deathParticle4Frames;
Value<atUint32> deathSFX;
Value<bool> unknown29;
Value<bool> unknown30;
@ -56,12 +56,36 @@ struct FishCloud : IScriptObject
ent->name = name + "_model";
}
animationParameters.nameANCS(pakRouter, name + "_animp");
if (deathParticle1)
{
PAK::Entry* ent = (PAK::Entry*)pakRouter.lookupEntry(deathParticle1);
ent->name = name + "_deathParticle1";
}
if (deathParticle2)
{
PAK::Entry* ent = (PAK::Entry*)pakRouter.lookupEntry(deathParticle2);
ent->name = name + "_deathParticle2";
}
if (deathParticle3)
{
PAK::Entry* ent = (PAK::Entry*)pakRouter.lookupEntry(deathParticle3);
ent->name = name + "_deathParticle3";
}
if (deathParticle4)
{
PAK::Entry* ent = (PAK::Entry*)pakRouter.lookupEntry(deathParticle4);
ent->name = name + "_deathParticle4";
}
}
void gatherDependencies(std::vector<hecl::ProjectPath>& pathsOut) const
{
g_curSpec->flattenDependencies(model, pathsOut);
animationParameters.depANCS(pathsOut);
g_curSpec->flattenDependencies(deathParticle1, pathsOut);
g_curSpec->flattenDependencies(deathParticle2, pathsOut);
g_curSpec->flattenDependencies(deathParticle3, pathsOut);
g_curSpec->flattenDependencies(deathParticle4, pathsOut);
}
};
}

View File

@ -446,7 +446,8 @@ struct ActorParameters : BigDNA
void depIDs(std::vector<hecl::ProjectPath>& pathsOut) const
{
scannableParameters.depIDs(pathsOut);
// Scans aren't included as dependencies
//scannableParameters.depIDs(pathsOut);
g_curSpec->flattenDependencies(xrayModel, pathsOut);
g_curSpec->flattenDependencies(xraySkin, pathsOut);
g_curSpec->flattenDependencies(thermalModel, pathsOut);

View File

@ -23,7 +23,7 @@ struct PointOfInterest : IScriptObject
scannableParameters.nameIDs(pakRouter, name + "_scanp");
}
void gatherDependencies(std::vector<hecl::ProjectPath>& pathsOut) const
void gatherLazyDependencies(std::vector<hecl::ProjectPath>& pathsOut) const
{
scannableParameters.depIDs(pathsOut);
}

View File

@ -34,7 +34,7 @@ struct Steam : IScriptObject
}
}
void gatherDependencies(std::vector<hecl::ProjectPath>& pathsOut) const
void gatherLazyDependencies(std::vector<hecl::ProjectPath>& pathsOut) const
{
g_curSpec->flattenDependencies(texture, pathsOut);
}

View File

@ -90,8 +90,10 @@ struct OriginalIDs
std::unordered_set<UniqueID32> addedIDs;
std::vector<UniqueID32> originalIDs;
pakRouter.enumerateResources([&](const DNAMP1::PAK::Entry* ent) -> bool {
if (ent->type == FOURCC('MLVL') || ent->type == FOURCC('SCAN') ||
pakRouter.enumerateResources([&](const DNAMP1::PAK::Entry* ent) {
if (ent->type == FOURCC('MLVL') ||
ent->type == FOURCC('SCAN') ||
ent->type == FOURCC('MREA') ||
IndividualOrigIDs.find(ent->id.toUint32()) != IndividualOrigIDs.end())
{
if (addedIDs.find(ent->id) == addedIDs.cend())
@ -1281,6 +1283,14 @@ struct SpecMP1 : SpecBase
std::unordered_set<urde::CAssetId> addedTags;
for (auto& area : mlvl.areas)
{
FILE* fp = nullptr;
if (addedTags.empty())
{
char path[256];
snprintf(path, 256, "/Users/jacko/Desktop/res/new_%08X.txt", nameEnt.id.toUint32());
fp = fopen(path, "w");
}
urde::SObjectTag areaTag(FOURCC('MREA'), originalToNew(area.areaMREAId));
bool dupeRes = false;
@ -1328,6 +1338,23 @@ struct SpecMP1 : SpecBase
area.deps = std::move(strippedDeps);
area.depLayerCount = strippedDepLayers.size();
area.depLayers = std::move(strippedDepLayers);
if (fp)
{
//std::unordered_set<urde::CAssetId> tmpAddedTags;
for (const auto& dep : area.deps)
{
urde::CAssetId newId = originalToNew(dep.id);
//if (tmpAddedTags.find(newId) == tmpAddedTags.end())
//{
urde::SObjectTag tag(dep.type, originalToNew(dep.id));
hecl::ProjectPath depPath = pathFromTag(tag);
fprintf(fp, "%s\n", depPath.getRelativePathUTF8().data());
//tmpAddedTags.insert(newId);
//}
}
fclose(fp);
}
}
urde::SObjectTag nameTag(FOURCC('STRG'), originalToNew(mlvl.worldNameId));

2
hecl

@ -1 +1 @@
Subproject commit 343aae42e2ff945ed5436a3c250901a2101db625
Subproject commit 654d7d39802271fc3d33aa134e07285fbc6f943d

View File

@ -110,6 +110,9 @@ bool VISIRenderer::SetupShaders()
glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, m_aabbIBO);
glBufferData(GL_ELEMENT_ARRAY_BUFFER, 20 * 4, AABBIdxs, GL_STATIC_DRAW);
glEnable(GL_PRIMITIVE_RESTART);
glPrimitiveRestartIndex(0xffffffff);
return true;
}