AxioDL
/
metaforce
mirror of https://github.com/AxioDL/metaforce.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

Fully working PAK/MLVL/STRG reads

This commit is contained in:
Jack Andersen 2015-07-13 14:38:48 -10:00
parent 7876d4c209
commit 31f77497fd
13 changed files with 297 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
DataSpec/DNACommon/DNACommon.hpp
View File

@ -3,6 +3,7 @@
#include <Athena/DNA.hpp>
#include "HECL/HECL.hpp"
#include "../Logging.hpp"
namespace Retro
{
@ -124,20 +125,58 @@ public:
}
};
/* Case-insensitive comparator for std::map sorting */
struct CaseInsensitiveCompare
{
inline bool operator()(const std::string& lhs, const std::string& rhs) const
{
std::string lhsl = lhs;
std::transform(lhsl.begin(), lhsl.end(), lhsl.begin(), tolower);
std::string rhsl = rhs;
std::transform(rhsl.begin(), rhsl.end(), rhsl.begin(), tolower);
if (lhsl.compare(rhsl) < 0)
#if _WIN32
if (stricmp(lhs.c_str(), rhs.c_str()) < 0)
#else
if (strcasecmp(lhs.c_str(), rhs.c_str()) < 0)
#endif
return true;
return false;
}
};
/* PAK entry stream reader */
class PAKEntryReadStream : public Athena::io::IStreamReader
{
std::unique_ptr<atUint8[]> m_buf;
atUint64 m_sz;
atUint64 m_pos;
public:
PAKEntryReadStream(std::unique_ptr<atUint8[]>&& buf, atUint64 sz, atUint64 pos)
: m_buf(std::move(buf)), m_sz(sz), m_pos(pos)
{
if (m_pos >= m_sz)
LogModule.report(LogVisor::FatalError, "PAK stream cursor overrun");
}
inline void seek(atInt64 pos, Athena::SeekOrigin origin)
{
if (origin == Athena::Begin)
m_pos = pos;
else if (origin == Athena::Current)
m_pos += pos;
else if (origin == Athena::End)
m_pos = m_sz + pos;
if (m_pos >= m_sz)
LogModule.report(LogVisor::FatalError, "PAK stream cursor overrun");
}
inline atUint64 position() const {return m_pos;}
inline atUint64 length() const {return m_sz;}
inline atUint64 readUBytesToBuf(void* buf, atUint64 len)
{
atUint64 bufEnd = m_pos + len;
if (bufEnd > m_sz)
len -= bufEnd - m_sz;
memcpy(buf, m_buf.get() + m_pos, len);
m_pos += len;
return len;
}
};
/* Language-identifiers */
extern const HECL::FourCC ENGL;
extern const HECL::FourCC FREN;

2
DataSpec/DNAMP1/MLVL.hpp
View File

@ -37,7 +37,7 @@ struct MLVL : BigDNA
Value<atVec4f> transformMtx[3];
Value<atVec3f> aabb[2];
UniqueID32 areaMREAId;
Value<atUint32> areaId;
UniqueID32 areaId;
Value<atUint32> attachedAreaCount;
Vector<atUint16, DNA_COUNT(attachedAreaCount)> attachedAreas;

65
DataSpec/DNAMP1/PAK.cpp
View File

@ -1,3 +1,5 @@
#include <zlib.h>
#include <lzo/lzo1x.h>
#include "PAK.hpp"
namespace Retro
@ -63,5 +65,68 @@ void PAK::write(Athena::io::IStreamWriter& writer) const
entry.write(writer);
}
std::unique_ptr<atUint8[]> PAK::Entry::getBuffer(const NOD::DiscBase::IPartition::Node& pak, atUint64& szOut) const
{
if (compressed)
{
std::unique_ptr<NOD::IPartReadStream> strm = pak.beginReadStream(offset);
atUint32 decompSz;
strm->read(&decompSz, 4);
decompSz = HECL::SBig(decompSz);
atUint8* buf = new atUint8[decompSz];
atUint8* bufCur = buf;
atUint16 zlibCheck;
strm->read(&zlibCheck, 2);
zlibCheck = HECL::SBig(zlibCheck);
strm->seek(-2, SEEK_CUR);
atUint8 compBuf[0x4000];
if ((zlibCheck % 31) == 0)
{
atUint32 compRem = size - 4;
z_stream zs;
inflateInit(&zs);
zs.avail_out = decompSz;
zs.next_out = buf;
while (zs.avail_out)
{
atUint64 readSz = strm->read(compBuf, MIN(compRem, 0x4000));
compRem -= readSz;
zs.avail_in = readSz;
zs.next_in = compBuf;
inflate(&zs, Z_FINISH);
}
inflateEnd(&zs);
}
else
{
atUint32 rem = decompSz;
while (rem)
{
atUint16 chunkSz;
strm->read(&chunkSz, 2);
chunkSz = HECL::SBig(chunkSz);
strm->read(compBuf, chunkSz);
lzo_uint dsz = rem;
lzo1x_decompress(compBuf, chunkSz, bufCur, &dsz, nullptr);
bufCur += dsz;
rem -= dsz;
}
}
szOut = decompSz;
return std::unique_ptr<atUint8[]>(buf);
}
else
{
atUint8* buf = new atUint8[size];
pak.beginReadStream(offset)->read(buf, size);
szOut = size;
return std::unique_ptr<atUint8[]>(buf);
}
}
}
}

8
DataSpec/DNAMP1/PAK.hpp
View File

@ -3,6 +3,7 @@
#include <unordered_map>
#include <NOD/DiscBase.hpp>
#include "../Logging.hpp"
#include "../DNACommon/DNACommon.hpp"
@ -32,6 +33,13 @@ struct PAK : BigDNA
UniqueID32 id;
Value<atUint32> size;
Value<atUint32> offset;
std::unique_ptr<atUint8[]> getBuffer(const NOD::DiscBase::IPartition::Node& pak, atUint64& szOut) const;
inline PAKEntryReadStream beginReadStream(const NOD::DiscBase::IPartition::Node& pak, atUint64 off=0) const
{
atUint64 sz;
return PAKEntryReadStream(getBuffer(pak, sz), sz, off);
}
};
std::vector<NameEntry> m_nameEntries;

3
DataSpec/DNAMP2/MLVL.hpp
View File

@ -27,7 +27,7 @@ struct MLVL : BigDNA
Value<atVec4f> transformMtx[3];
Value<atVec3f> aabb[2];
UniqueID32 areaMREAId;
Value<atUint32> areaId;
UniqueID32 areaId;
Value<atUint32> attachedAreaCount;
Vector<atUint16, DNA_COUNT(attachedAreaCount)> attachedAreas;
@ -56,7 +56,6 @@ struct MLVL : BigDNA
Value<atUint32> areaIdx;
Value<atUint32> dockIdx;
};
FourCC type;
Vector<Endpoint, DNA_COUNT(endpointCount)> endpoints;
Value<atUint32> planeVertCount;

4
DataSpec/DNAMP3/MLVL.hpp
View File

@ -26,7 +26,7 @@ struct MLVL : BigDNA
Value<atVec4f> transformMtx[3];
Value<atVec3f> aabb[2];
UniqueID64 areaMREAId;
Value<atUint32> areaId;
UniqueID64 areaId;
Value<atUint32> attachedAreaCount;
Vector<atUint16, DNA_COUNT(attachedAreaCount)> attachedAreas;
@ -42,7 +42,6 @@ struct MLVL : BigDNA
Value<atUint32> areaIdx;
Value<atUint32> dockIdx;
};
FourCC type;
Vector<Endpoint, DNA_COUNT(endpointCount)> endpoints;
Value<atUint32> planeVertCount;
@ -52,6 +51,7 @@ struct MLVL : BigDNA
String<-1> internalAreaName;
};
Vector<Area, DNA_COUNT(areaCount)> areas;
UniqueID64 worldMap;
Value<atUint8> unknown2;

90
DataSpec/DNAMP3/PAK.cpp
View File

@ -5,6 +5,8 @@ namespace Retro
namespace DNAMP3
{
const HECL::FourCC CMPD("CMPD");
void PAK::read(Athena::io::IStreamReader& reader)
{
reader.setEndian(Athena::BigEndian);
@ -17,7 +19,7 @@ void PAK::read(Athena::io::IStreamReader& reader)
reader.seek(4, Athena::Current);
atUint32 rshdSz = reader.readUint32();
reader.seek(44, Athena::Current);
m_dataOffset = 128 + strgSz + rshdSz;
atUint32 dataOffset = 128 + strgSz + rshdSz;
atUint64 strgBase = reader.position();
atUint32 nameCount = reader.readUint32();
@ -42,6 +44,7 @@ void PAK::read(Athena::io::IStreamReader& reader)
{
m_entries.emplace_back();
m_entries.back().read(reader);
m_entries.back().offset += dataOffset;
}
for (Entry& entry : m_entries)
m_idMap[entry.id] = &entry;
@ -73,6 +76,7 @@ void PAK::write(Athena::io::IStreamWriter& writer) const
atUint32 rshdPad = ((rshdSz + 63) & ~63) - rshdSz;
rshdSz += rshdPad;
writer.writeUint32(rshdSz);
atUint32 dataOffset = 128 + strgSz + rshdSz;
FourCC("DATA").write(writer);
atUint32 dataSz = 0;
@ -90,9 +94,91 @@ void PAK::write(Athena::io::IStreamWriter& writer) const
writer.writeUint32(m_entries.size());
for (const Entry& entry : m_entries)
entry.write(writer);
{
Entry copy = entry;
copy.offset -= dataOffset;
copy.write(writer);
}
writer.seek(rshdPad, Athena::Current);
}
std::unique_ptr<atUint8[]> PAK::Entry::getBuffer(const NOD::DiscBase::IPartition::Node& pak, atUint64& szOut) const
{
if (compressed)
{
std::unique_ptr<NOD::IPartReadStream> strm = pak.beginReadStream(offset);
struct
{
HECL::FourCC magic;
atUint32 blockCount;
} head;
strm->read(&head, 8);
if (head.magic != CMPD)
{
LogModule.report(LogVisor::Error, "invalid CMPD block");
return std::unique_ptr<atUint8[]>();
}
head.blockCount = HECL::SBig(head.blockCount);
struct Block
{
atUint32 compSz;
atUint32 decompSz;
} blocks[head.blockCount];
strm->read(blocks, 8 * head.blockCount);
atUint64 maxBlockSz = 0;
atUint64 totalDecompSz = 0;
for (atUint32 b=0 ; b<head.blockCount ; ++b)
{
Block& block = blocks[b];
block.compSz = HECL::SBig(block.compSz) & 0xffffff;
block.decompSz = HECL::SBig(block.decompSz);
if (block.compSz > maxBlockSz)
maxBlockSz = block.compSz;
totalDecompSz += block.decompSz;
}
std::unique_ptr<atUint8[]> compBuf(new atUint8[maxBlockSz]);
atUint8* buf = new atUint8[totalDecompSz];
atUint8* bufCur = buf;
for (atUint32 b=0 ; b<head.blockCount ; ++b)
{
Block& block = blocks[b];
atUint8* compBufCur = compBuf.get();
strm->read(compBufCur, block.compSz);
if (block.compSz == block.decompSz)
{
memcpy(bufCur, compBufCur, block.decompSz);
bufCur += block.decompSz;
}
else
{
atUint32 rem = block.decompSz;
while (rem)
{
atUint16 chunkSz = HECL::SBig(*(atUint16*)compBufCur);
compBufCur += 2;
lzo_uint dsz = rem;
lzo1x_decompress(compBufCur, chunkSz, bufCur, &dsz, nullptr);
compBufCur += chunkSz;
bufCur += dsz;
rem -= dsz;
}
}
}
szOut = totalDecompSz;
return std::unique_ptr<atUint8[]>(buf);
}
else
{
atUint8* buf = new atUint8[size];
pak.beginReadStream(offset)->read(buf, size);
szOut = size;
return std::unique_ptr<atUint8[]>(buf);
}
}
}
}

16
DataSpec/DNAMP3/PAK.hpp
View File

@ -3,14 +3,18 @@
#include <unordered_map>
#include "../Logging.hpp"
#include <lzo/lzo1x.h>
#include <NOD/DiscBase.hpp>
#include "../DNACommon/DNACommon.hpp"
#include "../Logging.hpp"
namespace Retro
{
namespace DNAMP3
{
extern const HECL::FourCC CMPD;
struct PAK : BigDNA
{
struct Header : BigDNA
@ -38,13 +42,19 @@ struct PAK : BigDNA
UniqueID64 id;
Value<atUint32> size;
Value<atUint32> offset;
std::unique_ptr<atUint8[]> getBuffer(const NOD::DiscBase::IPartition::Node& pak, atUint64& szOut) const;
inline PAKEntryReadStream beginReadStream(const NOD::DiscBase::IPartition::Node& pak, atUint64 off=0) const
{
atUint64 sz;
return PAKEntryReadStream(getBuffer(pak, sz), sz, off);
}
};
std::vector<NameEntry> m_nameEntries;
std::vector<Entry> m_entries;
std::unordered_map<UniqueID64, Entry*> m_idMap;
std::unordered_map<std::string, Entry*> m_nameMap;
size_t m_dataOffset = 0;
DECL_EXPLICIT_DNA
@ -63,8 +73,6 @@ struct PAK : BigDNA
return result->second;
return nullptr;
}
inline size_t getDataOffset() const {return m_dataOffset;}
};
}

77
DataSpec/DNAMP3/STRG.cpp
View File

@ -13,17 +13,20 @@ void STRG::_read(Athena::io::IStreamReader& reader)
atUint32 nameCount = reader.readUint32();
atUint32 nameTableSz = reader.readUint32();
std::unique_ptr<uint8_t[]> nameTableBuf(new uint8_t[nameTableSz]);
reader.readUBytesToBuf(nameTableBuf.get(), nameTableSz);
struct NameIdxEntry
if (nameTableSz)
{
atUint32 nameOff;
atUint32 strIdx;
}* nameIndex = (NameIdxEntry*)nameTableBuf.get();
for (atUint32 n=0 ; n<nameCount ; ++n)
{
const char* name = (char*)(nameTableBuf.get() + HECL::SBig(nameIndex[n].nameOff));
names[name] = HECL::SBig(nameIndex[n].strIdx);
std::unique_ptr<uint8_t[]> nameTableBuf(new uint8_t[nameTableSz]);
reader.readUBytesToBuf(nameTableBuf.get(), nameTableSz);
struct NameIdxEntry
{
atUint32 nameOff;
atUint32 strIdx;
}* nameIndex = (NameIdxEntry*)nameTableBuf.get();
for (atUint32 n=0 ; n<nameCount ; ++n)
{
const char* name = (char*)(nameTableBuf.get() + HECL::SBig(nameIndex[n].nameOff));
names[name] = HECL::SBig(nameIndex[n].strIdx);
}
}
std::vector<FourCC> readLangs;
@ -33,20 +36,28 @@ void STRG::_read(Athena::io::IStreamReader& reader)
FourCC lang;
lang.read(reader);
readLangs.emplace_back(lang);
reader.seek(strCount * 4 + 4);
}
std::unique_ptr<atUint32[]> strOffs(new atUint32[langCount * strCount]);
for (atUint32 l=0 ; l<langCount ; ++l)
{
reader.readUint32();
for (atUint32 s=0 ; s<strCount ; ++s)
strOffs[l*strCount+s] = reader.readUint32();
}
atUint64 strBase = reader.position();
langs.clear();
langs.reserve(langCount);
for (FourCC& lang : readLangs)
for (atUint32 l=0 ; l<langCount ; ++l)
{
std::vector<std::string> strs;
for (atUint32 s=0 ; s<strCount ; ++s)
{
reader.seek(strBase + strOffs[l*strCount+s], Athena::Begin);
atUint32 len = reader.readUint32();
strs.emplace_back(reader.readString(len));
}
langs.emplace(lang, strs);
langs.emplace(readLangs[l], strs);
}
}
@ -55,11 +66,17 @@ void STRG::read(Athena::io::IStreamReader& reader)
reader.setEndian(Athena::BigEndian);
atUint32 magic = reader.readUint32();
if (magic != 0x87654321)
{
LogModule.report(LogVisor::Error, "invalid STRG magic");
return;
}
atUint32 version = reader.readUint32();
if (version != 3)
{
LogModule.report(LogVisor::Error, "invalid STRG version");
return;
}
_read(reader);
}
@ -74,43 +91,53 @@ void STRG::write(Athena::io::IStreamWriter& writer) const
writer.writeUint32(strCount);
atUint32 nameTableSz = names.size() * 8;
for (const std::pair<std::string, int32_t>& name : names)
for (const auto& name : names)
nameTableSz += name.first.size() + 1;
writer.writeUint32(names.size());
writer.writeUint32(nameTableSz);
atUint32 offset = names.size() * 8;
for (const std::pair<std::string, int32_t>& name : names)
for (const auto& name : names)
{
writer.writeUint32(offset);
writer.writeInt32(name.second);
offset += name.first.size() + 1;
}
for (const std::pair<std::string, int32_t>& name : names)
for (const auto& name : names)
writer.writeString(name.first);
offset = 0;
for (const std::pair<FourCC, std::vector<std::string>>& lang : langs)
{
for (const auto& lang : langs)
lang.first.write(writer);
offset = 0;
for (const auto& lang : langs)
{
atUint32 langSz = 0;
for (const std::string& str : lang.second)
langSz += str.size() + 4;
langSz += str.size() + 5;
writer.writeUint32(langSz);
for (const std::string& str : lang.second)
{
writer.writeUint32(offset);
offset += str.size() + 4;
offset += str.size() + 5;
}
}
for (const std::pair<FourCC, std::vector<std::string>>& lang : langs)
for (atUint32 s=0 ; s<strCount ; ++s)
{
for (const std::string& str : lang.second)
for (const auto& lang : langs)
{
writer.writeUint32(str.size());
writer.writeString(str, str.size());
if (s >= lang.second.size())
{
writer.writeUint32(1);
writer.writeUByte(0);
}
else
{
const std::string& str = lang.second[s];
writer.writeUint32(str.size() + 1);
writer.writeString(str);
}
}
}
}

4
DataSpec/SpecMP1.cpp
View File

@ -99,14 +99,14 @@ struct SpecMP1 : SpecBase
{
if (entry.type == MLVL)
{
NOD::AthenaPartReadStream rs(item.second->node.beginReadStream(entry.offset));
PAKEntryReadStream rs = entry.beginReadStream(item.second->node);
DNAMP1::MLVL mlvl;
mlvl.read(rs);
const DNAMP1::PAK::Entry* nameEnt = pak.lookupEntry(mlvl.worldNameId);
if (nameEnt)
{
PAKEntryReadStream rs = nameEnt->beginReadStream(item.second->node);
DNAMP1::STRG mlvlName;
NOD::AthenaPartReadStream rs(item.second->node.beginReadStream(nameEnt->offset));
mlvlName.read(rs);
if (childRep.desc.size())
childRep.desc += _S(", ");

4
DataSpec/SpecMP2.cpp
View File

@ -99,14 +99,14 @@ struct SpecMP2 : SpecBase
{
if (entry.type == MLVL)
{
NOD::AthenaPartReadStream rs(item.second->node.beginReadStream(entry.offset));
PAKEntryReadStream rs = entry.beginReadStream(item.second->node);
DNAMP2::MLVL mlvl;
mlvl.read(rs);
const DNAMP1::PAK::Entry* nameEnt = pak.lookupEntry(mlvl.worldNameId);
if (nameEnt)
{
PAKEntryReadStream rs = nameEnt->beginReadStream(item.second->node);
DNAMP2::STRG mlvlName;
NOD::AthenaPartReadStream rs(item.second->node.beginReadStream(nameEnt->offset));
mlvlName.read(rs);
if (childRep.desc.size())
childRep.desc += _S(", ");

24
DataSpec/SpecMP3.cpp
View File

@ -1,4 +1,5 @@
#include <utility>
#include <set>
#define NOD_ATHENA 1
#include "SpecBase.hpp"
@ -93,27 +94,40 @@ struct SpecMP3 : SpecBase
rep.childOpts.emplace_back();
ExtractReport& childRep = rep.childOpts.back();
childRep.name = item.first;
if (!item.first.compare("Worlds.pak"))
continue;
else if (!item.first.compare("Metroid6.pak"))
{
childRep.desc = _S("Phaaze");
continue;
}
std::set<HECL::SystemString> worldNames;
DNAMP3::PAK& pak = item.second->pak;
for (DNAMP3::PAK::Entry& entry : pak.m_entries)
{
if (entry.type == MLVL)
{
NOD::AthenaPartReadStream rs(item.second->node.beginReadStream(entry.offset));
PAKEntryReadStream rs = entry.beginReadStream(item.second->node);
DNAMP3::MLVL mlvl;
mlvl.read(rs);
const DNAMP3::PAK::Entry* nameEnt = pak.lookupEntry(mlvl.worldNameId);
if (nameEnt)
{
PAKEntryReadStream rs = nameEnt->beginReadStream(item.second->node);
DNAMP3::STRG mlvlName;
NOD::AthenaPartReadStream rs(item.second->node.beginReadStream(nameEnt->offset));
mlvlName.read(rs);
if (childRep.desc.size())
childRep.desc += _S(", ");
childRep.desc += mlvlName.getSystemString(ENGL, 0);
worldNames.emplace(mlvlName.getSystemString(ENGL, 0));
}
}
}
for (const std::string& name : worldNames)
{
if (childRep.desc.size())
childRep.desc += _S(", ");
childRep.desc += name;
}
}
}

2
NODLib

@ -1 +1 @@
Subproject commit 01f269e8e2a55d8b64e1e277d9a1f87f6877c446
Subproject commit 83f29da294c05c99aba06dfa7a5a095326a1d101