AxioDL/metaforce
2
0
Fork 0
mirror of https://github.com/AxioDL/metaforce.git synced 2025-12-16 04:57:01 +00:00
Code Issues Projects Releases Wiki Activity

Use UTF-8 exclusively internally

Browse Source 
This removes SystemString, SystemChar, etc.
All filepaths and log strings are assumed to be UTF-8,
with conversions to UTF-16 for Windows APIs as appropriate.

Updates amuse, athena, boo, kabufua and nod
This commit is contained in:
Luke Street
2021-06-30 14:20:45 -04:00
parent 6e12554026
commit 9ca1a38171
160 changed files with 2029 additions and 2753 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
DataSpec/DNAMP2/AGSC.cpp
View File

@@ -30,7 +30,7 @@ bool AGSC::Extract(PAKEntryReadStream& rs, const hecl::ProjectPath& dir) {
/* Write out project/pool */
{
auto projd = group.getProj().toYAML();
athena::io::FileWriter fo(hecl::ProjectPath(dir, _SYS_STR("!project.yaml")).getAbsolutePath());
athena::io::FileWriter fo(hecl::ProjectPath(dir, "!project.yaml").getAbsolutePath());
if (fo.hasError())
return false;
fo.writeUBytes(projd.data(), projd.size());
@@ -38,7 +38,7 @@ bool AGSC::Extract(PAKEntryReadStream& rs, const hecl::ProjectPath& dir) {
{
auto poold = group.getPool().toYAML();
athena::io::FileWriter fo(hecl::ProjectPath(dir, _SYS_STR("!pool.yaml")).getAbsolutePath());
athena::io::FileWriter fo(hecl::ProjectPath(dir, "!pool.yaml").getAbsolutePath());
if (fo.hasError())
return false;
fo.writeUBytes(poold.data(), poold.size());
@@ -61,7 +61,7 @@ bool AGSC::Cook(const hecl::ProjectPath& dir, const hecl::ProjectPath& outPath)
auto sdirSamp = group.getSdir().toGCNData(group);
Header head;
head.groupName = dir.getLastComponentUTF8();
head.groupName = dir.getLastComponent();
for (const auto& p : group.getProj().sfxGroups())
head.groupId = p.first.id;
head.poolSz = pool.size();

6
DataSpec/DNAMP2/ANCS.hpp
View File

@@ -199,10 +199,10 @@ struct ANCS : BigDNA {
static bool Extract(const SpecBase& dataSpec, PAKEntryReadStream& rs, const hecl::ProjectPath& outPath,
PAKRouter<PAKBridge>& pakRouter, const DNAMP2::PAK::Entry& entry, bool force,
hecl::blender::Token& btok, std::function<void(const hecl::SystemChar*)> fileChanged) {
hecl::ProjectPath yamlPath = outPath.getWithExtension(_SYS_STR(".yaml"), true);
hecl::blender::Token& btok, std::function<void(const char*)> fileChanged) {
hecl::ProjectPath yamlPath = outPath.getWithExtension(".yaml", true);
hecl::ProjectPath::Type yamlType = yamlPath.getPathType();
hecl::ProjectPath blendPath = outPath.getWithExtension(_SYS_STR(".blend"), true);
hecl::ProjectPath blendPath = outPath.getWithExtension(".blend", true);
hecl::ProjectPath::Type blendType = blendPath.getPathType();
if (force || yamlType == hecl::ProjectPath::Type::None || blendType == hecl::ProjectPath::Type::None) {

6
DataSpec/DNAMP2/CINF.cpp
View File

@@ -176,7 +176,7 @@ CINF::CINF(const Armature& armature, std::unordered_map<std::string, atInt32>& i
template <class PAKBridge>
bool CINF::Extract(const SpecBase& dataSpec, PAKEntryReadStream& rs, const hecl::ProjectPath& outPath,
PAKRouter<PAKBridge>& pakRouter, const typename PAKBridge::PAKType::Entry& entry, bool force,
hecl::blender::Token& btok, std::function<void(const hecl::SystemChar*)> fileChanged) {
hecl::blender::Token& btok, std::function<void(const char*)> fileChanged) {
if (!force && outPath.isFile())
return true;
@@ -207,11 +207,11 @@ bool CINF::Extract(const SpecBase& dataSpec, PAKEntryReadStream& rs, const hecl:
template bool CINF::Extract(const SpecBase& dataSpec, PAKEntryReadStream& rs, const hecl::ProjectPath& outPath,
PAKRouter<PAKBridge>& pakRouter, const typename PAKBridge::PAKType::Entry& entry,
bool force, hecl::blender::Token& btok,
std::function<void(const hecl::SystemChar*)> fileChanged);
std::function<void(const char*)> fileChanged);
template bool CINF::Extract(const SpecBase& dataSpec, PAKEntryReadStream& rs, const hecl::ProjectPath& outPath,
PAKRouter<DNAMP3::PAKBridge>& pakRouter,
const typename DNAMP3::PAKBridge::PAKType::Entry& entry, bool force,
hecl::blender::Token& btok, std::function<void(const hecl::SystemChar*)> fileChanged);
hecl::blender::Token& btok, std::function<void(const char*)> fileChanged);
bool CINF::Cook(const hecl::ProjectPath& outPath, const hecl::ProjectPath& inPath,
const hecl::blender::Armature& armature) {

2
DataSpec/DNAMP2/CINF.hpp
View File

@@ -53,7 +53,7 @@ struct CINF : BigDNA {
template <class PAKBridge>
static bool Extract(const SpecBase& dataSpec, PAKEntryReadStream& rs, const hecl::ProjectPath& outPath,
PAKRouter<PAKBridge>& pakRouter, const typename PAKBridge::PAKType::Entry& entry, bool force,
hecl::blender::Token& btok, std::function<void(const hecl::SystemChar*)> fileChanged);
hecl::blender::Token& btok, std::function<void(const char*)> fileChanged);
static bool Cook(const hecl::ProjectPath& outPath, const hecl::ProjectPath& inPath,
const hecl::blender::Armature& armature);

2
DataSpec/DNAMP2/CMDL.cpp
View File

@@ -5,7 +5,7 @@ namespace DataSpec::DNAMP2 {
bool CMDL::Extract(const SpecBase& dataSpec, PAKEntryReadStream& rs, const hecl::ProjectPath& outPath,
PAKRouter<PAKBridge>& pakRouter, const DNAMP2::PAK::Entry& entry, bool, hecl::blender::Token& btok,
std::function<void(const hecl::SystemChar*)>) {
std::function<void(const char*)>) {
/* Check for RigPair */
CINF cinf;
CSKR cskr;

2
DataSpec/DNAMP2/CMDL.hpp
View File

@@ -12,7 +12,7 @@ namespace DataSpec::DNAMP2 {
struct CMDL {
static bool Extract(const SpecBase& dataSpec, PAKEntryReadStream& rs, const hecl::ProjectPath& outPath,
PAKRouter<PAKBridge>& pakRouter, const DNAMP2::PAK::Entry& entry, bool,
hecl::blender::Token& btok, std::function<void(const hecl::SystemChar*)>);
hecl::blender::Token& btok, std::function<void(const char*)>);
};
} // namespace DataSpec::DNAMP2

66
DataSpec/DNAMP2/DNAMP2.cpp
View File

@@ -52,18 +52,18 @@ PAKBridge::PAKBridge(const nod::Node& node, bool doExtract)
STRG mlvlName;
mlvlName.read(rs);
if (m_levelString.size())
m_levelString += _SYS_STR(", ");
m_levelString += mlvlName.getSystemString(FOURCC('ENGL'), 0);
m_levelString += ", ";
m_levelString += mlvlName.getUTF8(FOURCC('ENGL'), 0);
}
}
}
}
static hecl::SystemString LayerName(std::string_view name) {
hecl::SystemString ret(hecl::SystemStringConv(name).sys_str());
static std::string LayerName(std::string_view name) {
std::string ret(name);
for (auto& ch : ret)
if (ch == _SYS_STR('/') || ch == _SYS_STR('\\'))
ch = _SYS_STR('-');
if (ch == '/' || ch == '\\')
ch = '-';
return ret;
}
@@ -80,8 +80,7 @@ void PAKBridge::build() {
mlvl.read(rs);
}
std::string catalogueName;
std::string bestName = m_pak.bestEntryName(m_node, e, catalogueName);
level.name = hecl::SystemStringConv(bestName).sys_str();
level.name = m_pak.bestEntryName(m_node, e, catalogueName);
level.areas.reserve(mlvl.areaCount);
unsigned layerIdx = 0;
@@ -109,17 +108,16 @@ void PAKBridge::build() {
PAKEntryReadStream rs = areaNameEnt->beginReadStream(m_node);
areaName.read(rs);
}
areaDeps.name = areaName.getSystemString(FOURCC('ENGL'), 0);
areaDeps.name = areaName.getUTF8(FOURCC('ENGL'), 0);
areaDeps.name = hecl::StringUtils::TrimWhitespace(areaDeps.name);
}
if (areaDeps.name.empty()) {
areaDeps.name = hecl::SystemStringConv(area.internalAreaName).sys_str();
areaDeps.name = area.internalAreaName;
if (areaDeps.name.empty()) {
std::string idStr = area.areaMREAId.toString();
areaDeps.name = hecl::SystemString(_SYS_STR("MREA_")) + hecl::SystemStringConv(idStr).c_str();
areaDeps.name = "MREA_" + area.areaMREAId.toString();
}
}
hecl::SystemString num = fmt::format(FMT_STRING(_SYS_STR("{:02d} ")), ai);
std::string num = fmt::format(FMT_STRING("{:02d} "), ai);
areaDeps.name = num + areaDeps.name;
areaDeps.layers.reserve(area.depLayerCount - 1);
@@ -130,7 +128,7 @@ void PAKBridge::build() {
layer.name = LayerName(mlvl.layerNames[layerIdx++]);
layer.active = layerFlags.flags >> (l - 1) & 0x1;
layer.name = hecl::StringUtils::TrimWhitespace(layer.name);
num = fmt::format(FMT_STRING(_SYS_STR("{:02d} ")), l - 1);
num = fmt::format(FMT_STRING("{:02d} "), l - 1);
layer.name = num + layer.name;
layer.resources.reserve(area.depLayers[l] - r);
@@ -202,7 +200,7 @@ void PAKBridge::addMAPATransforms(PAKRouter<PAKBridge>& pakRouter,
if (mlvl.worldNameId.isValid())
pathOverrides[mlvl.worldNameId] =
hecl::ProjectPath(mlvlDirPath, fmt::format(FMT_STRING(_SYS_STR("!name_{}.yaml")), mlvl.worldNameId));
hecl::ProjectPath(mlvlDirPath, fmt::format(FMT_STRING("!name_{}.yaml"), mlvl.worldNameId));
for (const MLVL::Area& area : mlvl.areas) {
{
@@ -218,7 +216,7 @@ void PAKBridge::addMAPATransforms(PAKRouter<PAKBridge>& pakRouter,
hecl::ProjectPath areaDirPath = pakRouter.getWorking(area.areaMREAId).getParentPath();
if (area.areaNameId.isValid())
pathOverrides[area.areaNameId] =
hecl::ProjectPath(areaDirPath, fmt::format(FMT_STRING(_SYS_STR("!name_{}.yaml")), area.areaNameId));
hecl::ProjectPath(areaDirPath, fmt::format(FMT_STRING("!name_{}.yaml"), area.areaNameId));
}
if (mlvl.worldMap.isValid()) {
@@ -249,43 +247,43 @@ ResExtractor<PAKBridge> PAKBridge::LookupExtractor(const nod::Node& pakNode, con
const DNAMP2::PAK::Entry& entry) {
switch (entry.type.toUint32()) {
case SBIG('HINT'):
return {DNAMP1::HINT::Extract, {_SYS_STR(".yaml")}};
return {DNAMP1::HINT::Extract, {".yaml"}};
case SBIG('STRG'):
return {STRG::Extract, {_SYS_STR(".yaml")}};
return {STRG::Extract, {".yaml"}};
case SBIG('TXTR'):
return {TXTR::Extract, {_SYS_STR(".png")}};
return {TXTR::Extract, {".png"}};
case SBIG('AFSM'):
return {AFSM::Extract, {_SYS_STR(".yaml")}};
return {AFSM::Extract, {".yaml"}};
case SBIG('SAVW'):
return {SAVWCommon::ExtractSAVW<SAVW>, {_SYS_STR(".yaml")}};
return {SAVWCommon::ExtractSAVW<SAVW>, {".yaml"}};
case SBIG('CMDL'):
return {CMDL::Extract, {_SYS_STR(".blend")}, 1};
return {CMDL::Extract, {".blend"}, 1};
case SBIG('CINF'):
return {CINF::Extract<PAKBridge>, {_SYS_STR(".blend")}, 1};
return {CINF::Extract<PAKBridge>, {".blend"}, 1};
case SBIG('ANCS'):
return {ANCS::Extract, {_SYS_STR(".yaml"), _SYS_STR(".blend")}, 2};
return {ANCS::Extract, {".yaml", ".blend"}, 2};
case SBIG('MLVL'):
return {MLVL::Extract, {_SYS_STR(".yaml"), _SYS_STR(".blend")}, 3};
return {MLVL::Extract, {".yaml", ".blend"}, 3};
case SBIG('MREA'):
return {MREA::Extract, {_SYS_STR(".blend")}, 4};
return {MREA::Extract, {".blend"}, 4};
case SBIG('MAPA'):
return {MAPA::Extract, {_SYS_STR(".blend")}, 4};
return {MAPA::Extract, {".blend"}, 4};
case SBIG('MAPU'):
return {MAPU::Extract, {_SYS_STR(".blend")}, 5};
return {MAPU::Extract, {".blend"}, 5};
case SBIG('PATH'):
return {PATH::Extract, {_SYS_STR(".blend")}, 5};
return {PATH::Extract, {".blend"}, 5};
case SBIG('FSM2'):
return {DNAFSM2::ExtractFSM2<UniqueID32>, {_SYS_STR(".yaml")}};
return {DNAFSM2::ExtractFSM2<UniqueID32>, {".yaml"}};
case SBIG('FONT'):
return {DNAFont::ExtractFONT<UniqueID32>, {_SYS_STR(".yaml")}};
return {DNAFont::ExtractFONT<UniqueID32>, {".yaml"}};
case SBIG('DGRP'):
return {DNADGRP::ExtractDGRP<UniqueID32>, {_SYS_STR(".yaml")}};
return {DNADGRP::ExtractDGRP<UniqueID32>, {".yaml"}};
case SBIG('AGSC'):
return {AGSC::Extract, {}};
case SBIG('CSNG'):
return {DNAMP1::CSNG::Extract, {_SYS_STR(".mid"), _SYS_STR(".yaml")}};
return {DNAMP1::CSNG::Extract, {".mid", ".yaml"}};
case SBIG('ATBL'):
return {DNAAudio::ATBL::Extract, {_SYS_STR(".yaml")}};
return {DNAAudio::ATBL::Extract, {".yaml"}};
}
return {};
}

4
DataSpec/DNAMP2/DNAMP2.hpp
View File

@@ -16,14 +16,14 @@ public:
bool m_doExtract;
using Level = DataSpec::Level<UniqueID32>;
std::unordered_map<UniqueID32, Level> m_levelDeps;
hecl::SystemString m_levelString;
std::string m_levelString;
PAKBridge(const nod::Node& node, bool doExtract = true);
void build();
static ResExtractor<PAKBridge> LookupExtractor(const nod::Node& pakNode, const DNAMP2::PAK& pak,
const DNAMP2::PAK::Entry& entry);
std::string_view getName() const { return m_node.getName(); }
hecl::SystemStringView getLevelString() const { return m_levelString; }
std::string_view getLevelString() const { return m_levelString; }
using PAKType = DNAMP2::PAK;
const PAKType& getPAK() const { return m_pak; }

2
DataSpec/DNAMP2/MAPA.hpp
View File

@@ -8,7 +8,7 @@ namespace DataSpec::DNAMP2 {
struct MAPA : DNAMAPA::MAPA {
static bool Extract(const SpecBase& dataSpec, PAKEntryReadStream& rs, const hecl::ProjectPath& outPath,
PAKRouter<PAKBridge>& pakRouter, const DNAMP2::PAK::Entry& entry, bool force,
hecl::blender::Token& btok, std::function<void(const hecl::SystemChar*)> fileChanged) {
hecl::blender::Token& btok, std::function<void(const char*)> fileChanged) {
MAPA mapa;
mapa.read(rs);
hecl::blender::Connection& conn = btok.getBlenderConnection();

2
DataSpec/DNAMP2/MAPU.hpp
View File

@@ -11,7 +11,7 @@ namespace DataSpec::DNAMP2 {
struct MAPU : DNAMAPU::MAPU {
static bool Extract(const SpecBase& dataSpec, PAKEntryReadStream& rs, const hecl::ProjectPath& outPath,
PAKRouter<PAKBridge>& pakRouter, const DNAMP2::PAK::Entry& entry, bool force,
hecl::blender::Token& btok, std::function<void(const hecl::SystemChar*)> fileChanged) {
hecl::blender::Token& btok, std::function<void(const char*)> fileChanged) {
MAPU mapu;
mapu.read(rs);
hecl::blender::Connection& conn = btok.getBlenderConnection();

4
DataSpec/DNAMP2/MLVL.hpp
View File

@@ -85,10 +85,10 @@ struct MLVL : BigDNA {
static bool Extract(const SpecBase& dataSpec, PAKEntryReadStream& rs, const hecl::ProjectPath& outPath,
PAKRouter<PAKBridge>& pakRouter, const DNAMP2::PAK::Entry& entry, bool force,
hecl::blender::Token& btok, std::function<void(const hecl::SystemChar*)> fileChanged) {
hecl::blender::Token& btok, std::function<void(const char*)> fileChanged) {
MLVL mlvl;
mlvl.read(rs);
athena::io::FileWriter writer(outPath.getWithExtension(_SYS_STR(".yaml"), true).getAbsolutePath());
athena::io::FileWriter writer(outPath.getWithExtension(".yaml", true).getAbsolutePath());
athena::io::ToYAMLStream(mlvl, writer);
hecl::blender::Connection& conn = btok.getBlenderConnection();
return DNAMLVL::ReadMLVLToBlender(conn, mlvl, outPath, pakRouter, entry, force, fileChanged);

4
DataSpec/DNAMP2/MREA.cpp
View File

@@ -179,7 +179,7 @@ void MREA::StreamReader::writeDecompInfos(athena::io::IStreamWriter& writer) con
bool MREA::Extract(const SpecBase& dataSpec, PAKEntryReadStream& rs, const hecl::ProjectPath& outPath,
PAKRouter<PAKBridge>& pakRouter, const DNAMP2::PAK::Entry& entry, bool force,
hecl::blender::Token& btok, std::function<void(const hecl::SystemChar*)>) {
hecl::blender::Token& btok, std::function<void(const char*)>) {
using RigPair = std::pair<std::pair<UniqueID32, CSKR*>, std::pair<UniqueID32, CINF*>>;
RigPair dummy = {};
@@ -193,7 +193,7 @@ bool MREA::Extract(const SpecBase& dataSpec, PAKEntryReadStream& rs, const hecl:
/* MREA decompression stream */
StreamReader drs(rs, head.compressedBlockCount);
hecl::ProjectPath decompPath = outPath.getCookedPath(SpecEntMP2ORIG).getWithExtension(_SYS_STR(".decomp"));
hecl::ProjectPath decompPath = outPath.getCookedPath(SpecEntMP2ORIG).getWithExtension(".decomp");
decompPath.makeDirChain(false);
athena::io::FileWriter mreaDecompOut(decompPath.getAbsolutePath());
head.write(mreaDecompOut);

2
DataSpec/DNAMP2/MREA.hpp
View File

@@ -123,7 +123,7 @@ struct MREA {
static bool Extract(const SpecBase& dataSpec, PAKEntryReadStream& rs, const hecl::ProjectPath& outPath,
PAKRouter<PAKBridge>& pakRouter, const DNAMP2::PAK::Entry& entry, bool,
hecl::blender::Token& btok, std::function<void(const hecl::SystemChar*)>);
hecl::blender::Token& btok, std::function<void(const char*)>);
};
} // namespace DataSpec::DNAMP2

10
DataSpec/DNAMP2/STRG.hpp
View File

@@ -42,16 +42,6 @@ struct STRG : ISTRG {
return search->second->at(idx);
return std::u16string();
}
hecl::SystemString getSystemString(const FourCC& lang, size_t idx) const override {
auto search = langMap.find(lang);
if (search != langMap.end())
#if HECL_UCS2
return hecl::Char16ToWide(search->second->at(idx));
#else
return hecl::Char16ToUTF8(search->second->at(idx));
#endif
return hecl::SystemString();
}
static bool Extract(PAKEntryReadStream& rs, const hecl::ProjectPath& outPath) {
STRG strg;