More fixes for GameCube cooking

This commit is contained in:
Jack Andersen 2018-04-07 10:55:57 -10:00
parent 898114d803
commit ed4d7445c7
22 changed files with 198 additions and 74 deletions

View File

@ -1259,8 +1259,7 @@ bool WriteCMDL(const hecl::ProjectPath& outPath, const hecl::ProjectPath& inPath
matGX.reset(matIR, FE.getDiagnostics());
targetMSet.materials.emplace_back(matGX, mat.iprops, mat.texs, texPaths,
mesh.colorLayerCount, mesh.uvLayerCount,
false, false);
mesh.colorLayerCount, false, false);
targetMSet.materials.back().binarySize(endOff);
targetMSet.head.addMaterialEndOff(endOff);
@ -1742,8 +1741,7 @@ bool WriteMREASecs(std::vector<std::vector<uint8_t>>& secsOut, const hecl::Proje
bool lm = lightmapped != mat.iprops.cend() && lightmapped->second != 0;
matSet.materials.emplace_back(matGX, mat.iprops, mat.texs, texPaths,
mesh.colorLayerCount, mesh.uvLayerCount,
lm, false);
mesh.colorLayerCount, lm, false);
matSet.materials.back().binarySize(endOff);
matSet.head.addMaterialEndOff(endOff);

View File

@ -775,6 +775,16 @@ void PAKRouter<BRIDGETYPE>::enumerateResources(const std::function<bool(const En
return;
}
template <class BRIDGETYPE>
bool PAKRouter<BRIDGETYPE>::mreaHasDupeResources(const IDType& id) const
{
const PAKType* pak = m_pak.get();
if (!pak)
LogDNACommon.report(logvisor::Fatal,
"PAKRouter::enterPAKBridge() must be called before PAKRouter::mreaHasDupeResources()");
return pak->mreaHasDupeResources(id);
}
template class PAKRouter<DNAMP1::PAKBridge>;
template class PAKRouter<DNAMP2::PAKBridge>;
template class PAKRouter<DNAMP3::PAKBridge>;

View File

@ -232,6 +232,8 @@ public:
hecl::ProjectPath getAreaLayerCooked(const IDType& areaId, int layerIdx, bool& activeOut) const;
void enumerateResources(const std::function<bool(const EntryType*)>& func);
bool mreaHasDupeResources(const IDType& id) const;
};
}

View File

@ -20,7 +20,7 @@ static int CountBits(uint32_t n)
/* Box filter algorithm (for mipmapping) */
static void BoxFilter(const uint8_t* input, unsigned chanCount,
unsigned inWidth, unsigned inHeight, uint8_t* output)
unsigned inWidth, unsigned inHeight, uint8_t* output, bool dxt1)
{
unsigned mipWidth = 1;
unsigned mipHeight = 1;
@ -45,7 +45,9 @@ static void BoxFilter(const uint8_t* input, unsigned chanCount,
tmp += input[(in1LineBase+(x*2+1))*chanCount+c];
tmp += input[(in2LineBase+(x*2))*chanCount+c];
tmp += input[(in2LineBase+(x*2+1))*chanCount+c];
out[c] = tmp / 4;
out[c] = uint8_t(tmp / 4);
if (c == 3 && dxt1)
out[c] = uint8_t(out[c] ? 0xff : 0x0);
}
}
}
@ -1513,7 +1515,7 @@ bool TXTR::Cook(const hecl::ProjectPath& inPath, const hecl::ProjectPath& outPat
unsigned filterHeight = height;
for (size_t i=1 ; i<numMips ; ++i)
{
BoxFilter(filterIn, nComps, filterWidth, filterHeight, filterOut);
BoxFilter(filterIn, nComps, filterWidth, filterHeight, filterOut, doDXT1);
filterIn += filterWidth * filterHeight * nComps;
filterWidth /= 2;
filterHeight /= 2;
@ -1847,7 +1849,7 @@ bool TXTR::CookPC(const hecl::ProjectPath& inPath, const hecl::ProjectPath& outP
unsigned filterHeight = height;
for (size_t i=1 ; i<numMips ; ++i)
{
BoxFilter(filterIn, nComps, filterWidth, filterHeight, filterOut);
BoxFilter(filterIn, nComps, filterWidth, filterHeight, filterOut, doDXT1);
filterIn += filterWidth * filterHeight * nComps;
filterWidth /= 2;
filterHeight /= 2;

View File

@ -45,6 +45,16 @@ struct AFSM : public BigDNA
athena::io::ToYAMLStream(afsm, writer);
return true;
}
static bool Cook(const hecl::ProjectPath& inPath, const hecl::ProjectPath& outPath)
{
AFSM afsm;
athena::io::FileReader reader(inPath.getAbsolutePath());
athena::io::FromYAMLStream(afsm, reader);
athena::io::FileWriter ws(outPath.getAbsolutePath());
afsm.write(ws);
return true;
}
};
}

View File

@ -930,7 +930,6 @@ MaterialSet::Material::Material(const hecl::Backend::GX& gx,
const std::vector<hecl::ProjectPath>& texPathsIn,
std::vector<hecl::ProjectPath>& texPathsOut,
int colorCount,
int uvCount,
bool lightmapUVs,
bool matrixSkinning)
{

View File

@ -291,7 +291,6 @@ struct MaterialSet : BigDNA
const std::vector<hecl::ProjectPath>& texPathsIn,
std::vector<hecl::ProjectPath>& texPathsOut,
int colorCount,
int uvCount,
bool lightmapUVs,
bool matrixSkinning);
};

View File

@ -51,6 +51,8 @@ bool MLVL::Extract(const SpecBase& dataSpec, PAKEntryReadStream& rs, const hecl:
w.enumerate<atUint32>("memrelays", relayIds);
w.finish(&fw);
}
if (pakRouter.mreaHasDupeResources(area.areaMREAId))
athena::io::FileWriter(hecl::ProjectPath(areaDir, _S("!duperes")).getAbsolutePath());
areaIdx++;
}
@ -60,32 +62,16 @@ bool MLVL::Extract(const SpecBase& dataSpec, PAKEntryReadStream& rs, const hecl:
return DNAMLVL::ReadMLVLToBlender(conn, mlvl, outPath, pakRouter, entry, force, fileChanged);
}
struct BulkResources
{
std::unordered_map<hecl::Hash, size_t> addedBulkPaths;
bool addBulkPath(const hecl::ProjectPath& path, size_t areaIdx)
{
auto search = addedBulkPaths.find(path.hash());
if (search == addedBulkPaths.cend())
{
addedBulkPaths.insert(std::make_pair(path.hash(), areaIdx));
return true;
}
return false;
}
};
struct LayerResources
{
BulkResources& bulkResources;
std::unordered_map<hecl::Hash, std::pair<size_t, size_t>> addedPaths;
std::unordered_set<hecl::Hash> addedPaths;
std::vector<std::vector<std::pair<hecl::ProjectPath, bool>>> layerPaths;
std::unordered_set<hecl::Hash> addedSharedPaths;
std::vector<std::pair<hecl::ProjectPath, bool>> sharedPaths;
LayerResources(BulkResources& bulkResources) : bulkResources(bulkResources) {}
void beginLayer()
{
layerPaths.resize(layerPaths.size() + 1);
addedPaths.clear();
}
void addSharedPath(const hecl::ProjectPath& path, bool lazy)
{
@ -99,29 +85,12 @@ struct LayerResources
void addPath(const hecl::ProjectPath& path, bool lazy)
{
auto search = addedPaths.find(path.hash());
if (search != addedPaths.cend())
{
if (search->second.first == layerPaths.size() - 1)
return;
else
{
auto& toMove = layerPaths[search->second.first][search->second.second];
addSharedPath(toMove.first, toMove.second);
toMove.first.clear();
}
}
else
if (search == addedPaths.cend())
{
layerPaths.back().emplace_back(path, lazy);
addedPaths.insert(std::make_pair(path.hash(),
std::make_pair(layerPaths.size() - 1, layerPaths.back().size() - 1)));
addedPaths.insert(path.hash());
}
}
void addBulkPath(const hecl::ProjectPath& path, size_t areaIdx, bool lazy)
{
if (bulkResources.addBulkPath(path, areaIdx))
addPath(path, lazy);
}
};
bool MLVL::Cook(const hecl::ProjectPath& outPath, const hecl::ProjectPath& inPath, const World& wld,
@ -144,19 +113,18 @@ bool MLVL::Cook(const hecl::ProjectPath& outPath, const hecl::ProjectPath& inPat
size_t areaIdx = 0;
size_t nameOffset = 0;
BulkResources bulkResources;
for (const World::Area& area : wld.areas)
{
if (area.path.getPathType() != hecl::ProjectPath::Type::Directory)
continue;
hecl::ProjectPath areaPath(area.path, _S("/!area.blend"));
hecl::ProjectPath areaPath(area.path, _S("!area.blend"));
if (!areaPath.isFile())
continue;
Log.report(logvisor::Info, _S("Visiting %s"), area.path.getRelativePath().data());
hecl::ProjectPath memRelayPath(area.path, _S("/!memoryrelays.yaml"));
hecl::ProjectPath memRelayPath(area.path, _S("!memoryrelays.yaml"));
std::vector<atUint32> memRelays;
@ -175,7 +143,7 @@ bool MLVL::Cook(const hecl::ProjectPath& outPath, const hecl::ProjectPath& inPat
bool areaInit = false;
size_t layerIdx = 0;
LayerResources layerResources(bulkResources);
LayerResources layerResources;
for (const hecl::DirectoryEnumerator::Entry& e : dEnum)
{
hecl::SystemString layerName;
@ -220,7 +188,7 @@ bool MLVL::Cook(const hecl::ProjectPath& outPath, const hecl::ProjectPath& inPat
mlvl.areas.emplace_back();
MLVL::Area& areaOut = mlvl.areas.back();
hecl::ProjectPath namePath(area.path, _S("/!name.yaml"));
hecl::ProjectPath namePath(area.path, _S("!name.yaml"));
if (namePath.isFile())
areaOut.areaNameId = namePath;
@ -232,7 +200,7 @@ bool MLVL::Cook(const hecl::ProjectPath& outPath, const hecl::ProjectPath& inPat
areaOut.areaMREAId = areaPath;
areaOut.areaId = 0xffffffff;
hecl::ProjectPath memIdPath(area.path, _S("/!memoryid.yaml"));
hecl::ProjectPath memIdPath(area.path, _S("!memoryid.yaml"));
if (memIdPath.isFile())
{
athena::io::FileReader fr(memIdPath.getAbsolutePath());
@ -330,9 +298,9 @@ bool MLVL::Cook(const hecl::ProjectPath& outPath, const hecl::ProjectPath& inPat
/* Cull duplicate paths and add typed hash to list */
for (const hecl::ProjectPath& path : depPaths)
layerResources.addBulkPath(path, areaIdx, false);
layerResources.addPath(path, false);
for (const hecl::ProjectPath& path : lazyPaths)
layerResources.addBulkPath(path, areaIdx, true);
layerResources.addPath(path, true);
}
hecl::SystemUTF8Conv layerU8(layerName);
@ -347,6 +315,9 @@ bool MLVL::Cook(const hecl::ProjectPath& outPath, const hecl::ProjectPath& inPat
++layerIdx;
}
if (!areaInit)
Log.report(logvisor::Info, _S("No layer directories for area %s"), area.path.getRelativePath().data());
/* Build deplist */
MLVL::Area& areaOut = mlvl.areas.back();
for (const std::vector<std::pair<hecl::ProjectPath, bool>>& layer : layerResources.layerPaths)

View File

@ -423,6 +423,7 @@ bool MREA::Cook(const hecl::ProjectPath& outPath,
const ColMesh& cMesh,
const std::vector<Light>& lights,
hecl::blender::Token& btok,
const hecl::blender::Matrix4f* xf,
bool pc)
{
/* Discover area layers */
@ -450,9 +451,18 @@ bool MREA::Cook(const hecl::ProjectPath& outPath,
Header head = {};
head.magic = 0xDEADBEEF;
head.version = pc ? 0x1000F : 0xF;
head.localToWorldMtx[0].vec[0] = 1.f;
head.localToWorldMtx[1].vec[1] = 1.f;
head.localToWorldMtx[2].vec[2] = 1.f;
if (xf)
{
head.localToWorldMtx[0] = xf->val[0];
head.localToWorldMtx[1] = xf->val[1];
head.localToWorldMtx[2] = xf->val[2];
}
else
{
head.localToWorldMtx[0].vec[0] = 1.f;
head.localToWorldMtx[1].vec[1] = 1.f;
head.localToWorldMtx[2].vec[2] = 1.f;
}
head.meshCount = meshes.size();
head.geomSecIdx = 0;
head.arotSecIdx = secCount++;

View File

@ -131,6 +131,7 @@ struct MREA
const ColMesh& cMesh,
const std::vector<Light>& lights,
hecl::blender::Token& btok,
const hecl::blender::Matrix4f* xf,
bool pc);
};

View File

@ -28,10 +28,16 @@ void PAK::Enumerate<BigDNA::Read>(typename Read::StreamT& reader)
m_entries.reserve(count);
m_firstEntries.clear();
m_firstEntries.reserve(count);
std::vector<Entry> entries;
entries.reserve(count);
for (atUint32 e=0 ; e<count ; ++e)
{
Entry entry;
entry.read(reader);
entries.emplace_back();
entries.back().read(reader);
}
for (atUint32 e=0 ; e<count ; ++e)
{
Entry& entry = entries[e];
if (entry.compressed && m_useLzo)
entry.compressed = 2;
@ -41,6 +47,18 @@ void PAK::Enumerate<BigDNA::Read>(typename Read::StreamT& reader)
m_firstEntries.push_back(entry.id);
m_entries[entry.id] = std::move(entry);
}
else
{
/* Find next MREA to record which area has dupes */
for (atUint32 e2=e+1 ; e2<count ; ++e2)
{
Entry& entry2 = entries[e2];
if (entry2.type != FOURCC('MREA'))
continue;
m_dupeMREAs.insert(entry2.id);
break;
}
}
}
m_nameMap.clear();

View File

@ -49,11 +49,15 @@ struct PAK : BigDNA
std::unordered_map<UniqueID32, Entry> m_entries;
std::vector<UniqueID32> m_firstEntries;
std::unordered_map<std::string, UniqueID32> m_nameMap;
std::unordered_set<UniqueID32> m_dupeMREAs;
const Entry* lookupEntry(const UniqueID32& id) const;
const Entry* lookupEntry(std::string_view name) const;
std::string bestEntryName(const Entry& entry, bool& named) const;
bool mreaHasDupeResources(const UniqueID32& id) const
{ return m_dupeMREAs.find(id) != m_dupeMREAs.cend(); }
using IDType = UniqueID32;
};

View File

@ -36,10 +36,16 @@ void PAK::Enumerate<BigDNA::Read>(athena::io::IStreamReader& reader)
m_entries.reserve(count);
m_firstEntries.clear();
m_firstEntries.reserve(count);
std::vector<Entry> entries;
entries.reserve(count);
for (atUint32 e=0 ; e<count ; ++e)
{
Entry entry;
entry.read(reader);
entries.emplace_back();
entries.back().read(reader);
}
for (atUint32 e=0 ; e<count ; ++e)
{
Entry& entry = entries[e];
entry.offset += dataOffset;
auto search = m_entries.find(entry.id);
@ -48,6 +54,18 @@ void PAK::Enumerate<BigDNA::Read>(athena::io::IStreamReader& reader)
m_firstEntries.push_back(entry.id);
m_entries[entry.id] = std::move(entry);
}
else
{
/* Find next MREA to record which area has dupes */
for (atUint32 e2=e+1 ; e2<count ; ++e2)
{
Entry& entry2 = entries[e2];
if (entry2.type != FOURCC('MREA'))
continue;
m_dupeMREAs.insert(entry2.id);
break;
}
}
}
m_nameMap.clear();

View File

@ -58,6 +58,7 @@ struct PAK : BigDNA
std::unordered_map<UniqueID64, Entry> m_entries;
std::vector<UniqueID64> m_firstEntries;
std::unordered_map<std::string, UniqueID64> m_nameMap;
std::unordered_set<UniqueID64> m_dupeMREAs;
AT_DECL_EXPLICIT_DNA
@ -65,6 +66,9 @@ struct PAK : BigDNA
const Entry* lookupEntry(std::string_view name) const;
std::string bestEntryName(const Entry& entry, bool& named) const;
bool mreaHasDupeResources(const UniqueID64& id) const
{ return m_dupeMREAs.find(id) != m_dupeMREAs.cend(); }
typedef UniqueID64 IDType;
};

View File

@ -201,7 +201,7 @@ protected:
std::unordered_map<std::string, urde::SObjectTag> m_catalogNameToTag;
std::unordered_map<urde::SObjectTag, std::unordered_set<std::string>> m_catalogTagToNames;
void clearTagCache();
hecl::blender::Token m_backgroundBlender;
std::thread m_backgroundIndexTh;
std::mutex m_backgroundIndexMutex;
@ -231,7 +231,6 @@ protected:
athena::io::FileWriter& pakOut,
const std::unordered_map<urde::CAssetId, std::vector<uint8_t>>& mlvlData);
protected:
std::unique_ptr<nod::DiscBase> m_disc;
bool m_isWii;
bool m_standalone;

View File

@ -18,6 +18,7 @@
#include "DNAMP1/MAPA.hpp"
#include "DNAMP1/PATH.hpp"
#include "DNAMP1/FRME.hpp"
#include "DNAMP1/AFSM.hpp"
#include "DNACommon/ATBL.hpp"
#include "DNACommon/FONT.hpp"
#include "DNACommon/PART.hpp"
@ -179,6 +180,8 @@ struct SpecMP1 : SpecBase
IDRestorer<UniqueID32> m_idRestorer;
std::unordered_map<hecl::Hash, hecl::blender::Matrix4f> m_mreaPathToXF;
void setThreadProject()
{
SpecBase::setThreadProject();
@ -525,6 +528,8 @@ struct SpecMP1 : SpecBase
return true;
else if (!strcmp(classType, "ATBL"))
return true;
else if (!strcmp(classType, DNAMP1::AFSM::DNAType()))
return true;
else if (!strcmp(classType, "MP1OriginalIDs"))
return true;
return false;
@ -721,6 +726,11 @@ struct SpecMP1 : SpecBase
resTag.type = SBIG('ATBL');
return true;
}
else if (!strcmp(className, DataSpec::DNAMP1::AFSM::DNAType()))
{
resTag.type = SBIG('AFSM');
return true;
}
else if (!strcmp(className, "MP1OriginalIDs"))
{
resTag.type = SBIG('OIDS');
@ -821,6 +831,34 @@ struct SpecMP1 : SpecBase
}
}
void buildAreaXFs(hecl::blender::Token& btok)
{
hecl::blender::Connection& conn = btok.getBlenderConnection();
for (const auto& ent : m_workPath.enumerateDir())
{
if (ent.m_isDir)
{
hecl::ProjectPath pakPath(m_workPath, ent.m_name);
for (const auto& ent2 : pakPath.enumerateDir())
{
if (ent2.m_isDir)
{
hecl::ProjectPath wldPath(pakPath, ent2.m_name + _S("/!world.blend"));
if (wldPath.isFile())
{
if (!conn.openBlend(wldPath))
continue;
hecl::blender::DataStream ds = conn.beginData();
hecl::blender::World world = ds.compileWorld();
for (const auto& area : world.areas)
m_mreaPathToXF[area.path.hash()] = area.transform;
}
}
}
}
}
}
void cookArea(const hecl::ProjectPath& out, const hecl::ProjectPath& in, BlendStream& ds, bool fast,
hecl::blender::Token& btok, FCookProgress progress)
{
@ -851,7 +889,14 @@ struct SpecMP1 : SpecBase
ds.close();
DNAMP1::MREA::Cook(out, in, meshCompiles, *colMesh, lights, btok, m_pc);
if (m_mreaPathToXF.empty())
buildAreaXFs(btok);
const hecl::blender::Matrix4f* xf = nullptr;
auto xfSearch = m_mreaPathToXF.find(in.getParentPath().hash());
if (xfSearch != m_mreaPathToXF.cend())
xf = &xfSearch->second;
DNAMP1::MREA::Cook(out, in, meshCompiles, *colMesh, lights, btok, xf, m_pc);
}
void cookWorld(const hecl::ProjectPath& out, const hecl::ProjectPath& in, BlendStream& ds, bool fast,
@ -1073,6 +1118,10 @@ struct SpecMP1 : SpecBase
{
DNAAudio::ATBL::Cook(in, out);
}
else if (!classStr.compare(DNAMP1::AFSM::DNAType()))
{
DNAMP1::AFSM::Cook(in, out);
}
else if (!classStr.compare("MP1OriginalIDs"))
{
OriginalIDs::Cook(in, out);
@ -1229,14 +1278,27 @@ struct SpecMP1 : SpecBase
nameEnt.name = parentDir.getLastComponentUTF8();
nameEnt.write(w);
std::unordered_set<urde::CAssetId> addedTags;
for (auto& area : mlvl.areas)
{
urde::SObjectTag areaTag(FOURCC('MREA'), originalToNew(area.areaMREAId));
bool dupeRes = false;
if (hecl::ProjectPath areaDir = pathFromTag(areaTag).getParentPath())
dupeRes = hecl::ProjectPath(areaDir, _S("!duperes")).isFile();
urde::SObjectTag nameTag(FOURCC('STRG'), originalToNew(area.areaNameId));
if (nameTag)
listOut.push_back(nameTag);
for (const auto& dep : area.deps)
listOut.push_back({dep.type, originalToNew(dep.id)});
urde::SObjectTag areaTag(FOURCC('MREA'), originalToNew(area.areaMREAId));
{
urde::CAssetId newId = originalToNew(dep.id);
if (dupeRes || addedTags.find(newId) == addedTags.end())
{
listOut.push_back({dep.type, newId});
addedTags.insert(newId);
}
}
if (areaTag)
listOut.push_back(areaTag);

View File

@ -237,7 +237,7 @@ void ViewManager::init(boo::IApplication* app)
{
hecl::SystemString rootPath(root.getAbsolutePath());
hecl::Sstat theStat;
if (!hecl::Stat((rootPath + _S("/out/MP1/!original_ids.upak")).c_str(), &theStat) &&
if (!hecl::Stat((rootPath + _S("/out/files/Metroid1.upak")).c_str(), &theStat) &&
S_ISREG(theStat.st_mode))
m_deferedProject = rootPath + _S("/out");
}

View File

@ -57,6 +57,13 @@ private:
size_t m_posInBuf = 0;
boo::ObjToken<boo::IAudioVoice> m_booVoice;
//void* x4_loadBuf;
//void* x8_rom;
//void* xc_state;
//OSModuleInfo* x10_module = x4_loadBuf;
//void* x14_bss;
//void* x18_prgram;
//void* x1c_wram;
bool x20_gameOver = false;
u8 x21_passwordFromNES[18];
EPasswordEntryState x34_passwordEntryState = EPasswordEntryState::NotPasswordScreen;

View File

@ -117,21 +117,29 @@ void CResFactory::CancelBuild(const SObjectTag& tag)
void CResFactory::LoadOriginalIDs(CSimplePool& sp)
{
//m_origIds = sp.GetObj("MP1OriginalIDs");
#if RUNTIME_ORIGINAL_IDS
m_origIds = sp.GetObj("MP1OriginalIDs");
#endif
}
CAssetId CResFactory::TranslateOriginalToNew(CAssetId id) const
{
#if RUNTIME_ORIGINAL_IDS
return m_origIds->TranslateOriginalToNew(id);
#else
/* The packager will have restored these ahead of time */
return id;
//return m_origIds->TranslateOriginalToNew(id);
#endif
}
CAssetId CResFactory::TranslateNewToOriginal(CAssetId id) const
{
#if RUNTIME_ORIGINAL_IDS
return m_origIds->TranslateNewToOriginal(id);
#else
/* The packager will have restored these ahead of time */
return id;
//return m_origIds->TranslateNewToOriginal(id);
#endif
}
}

View File

@ -17,7 +17,9 @@ class CResFactory : public IFactory
{
CResLoader x4_loader;
CFactoryMgr x5c_factoryMgr;
//TLockedToken<MP1OriginalIDs> m_origIds;
#if RUNTIME_ORIGINAL_IDS
TLockedToken<MP1OriginalIDs> m_origIds;
#endif
public:
struct SLoadingData

2
hecl

@ -1 +1 @@
Subproject commit b0fa3912c5cced6d60f0bca7e7c48568038d176c
Subproject commit 9120c838d48a0660501949c92a6ca47668637096

@ -1 +1 @@
Subproject commit 3aa75f7ae8a271fd03a47939066f2bbd0f956e16
Subproject commit b254cb3803425f79020899ceda2424b46648355a