From 2503e04b9f41bad730dbdae4ecfc3a46d3fd1268 Mon Sep 17 00:00:00 2001 From: Jack Andersen Date: Fri, 7 Oct 2016 08:44:45 -1000 Subject: [PATCH] DataSpec link structure refactor; UniqueIDBridge cache fix --- CMakeLists.txt | 9 +- DataSpec/CMakeLists.txt | 17 ++ DataSpec/DNACommon/CMakeLists.txt | 76 +++--- DataSpec/DNACommon/DNACommon.cpp | 237 +++++++++++++++++++ DataSpec/DNACommon/DNACommon.hpp | 227 +++--------------- DataSpec/DNAMP1/CMakeLists.txt | 43 ++-- DataSpec/DNAMP1/ScriptObjects/CMakeLists.txt | 12 +- DataSpec/DNAMP2/CMakeLists.txt | 26 +- DataSpec/DNAMP3/CMakeLists.txt | 26 +- Editor/CMakeLists.txt | 3 +- 10 files changed, 384 insertions(+), 292 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4be19f013..2e6355cc1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -58,14 +58,7 @@ set(BOO_INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/hecl/extern/boo/include) set(HECL_INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/hecl/include ${CMAKE_CURRENT_SOURCE_DIR}/hecl/blender) -set(DATA_SPEC_LIBS - RetroDataSpec - DNAMP3 - DNAMP2 - DNAMP1 - ScriptObjectsMP1 - DNACommon - amuse) +set(DATA_SPEC_LIBS RetroDataSpec amuse) set(HECL_DATASPEC_DECLS "/* RetroCommon specs */ namespace DataSpec diff --git a/DataSpec/CMakeLists.txt b/DataSpec/CMakeLists.txt index 8aef913f6..9881617e9 100644 --- a/DataSpec/CMakeLists.txt +++ b/DataSpec/CMakeLists.txt @@ -16,6 +16,19 @@ macro(make_dnalist outlist) endforeach() endmacro() +# Assembles source files together for the main DataSpecCommon library +macro(dataspec_add_list rel_path a_list) + unset(tmp_list) + foreach(path IN LISTS ${a_list}) + if (IS_ABSOLUTE ${path}) + list(APPEND tmp_list "${path}") + else() + list(APPEND tmp_list "${rel_path}/${path}") + endif() + endforeach(path) + set(${a_list} "${tmp_list}" PARENT_SCOPE) +endmacro(dataspec_add_list) + include_directories(${ZLIB_INCLUDE_DIR} ${LZO_INCLUDE_DIR}) # Each game's DNA library @@ -30,9 +43,13 @@ bintoc(RetroMasterShader.c Blender/RetroMasterShader.py RETRO_MASTER_SHADER) # Each game's DataSpec implementation add_library(RetroDataSpec SpecBase.cpp + ${DNACOMMON_SOURCES} SpecMP1.cpp + ${DNAMP1_SOURCES} SpecMP2.cpp + ${DNAMP2_SOURCES} SpecMP3.cpp + ${DNAMP3_SOURCES} Blender/BlenderSupport.hpp Blender/BlenderSupport.cpp Blender/RetroMasterShader.py diff --git a/DataSpec/DNACommon/CMakeLists.txt b/DataSpec/DNACommon/CMakeLists.txt index 4f7713116..6e2581ced 100644 --- a/DataSpec/DNACommon/CMakeLists.txt +++ b/DataSpec/DNACommon/CMakeLists.txt @@ -5,40 +5,42 @@ make_dnalist(liblist SAVWCommon ParticleCommon) -add_library(DNACommon - DNACommon.hpp DNACommon.cpp - ${liblist} - PAK.hpp PAK.cpp - GX.hpp - FSM2.hpp FSM2.cpp - MLVL.hpp MLVL.cpp - CMDL.cpp - MAPA.cpp - STRG.hpp STRG.cpp - TXTR.hpp TXTR.cpp - ANCS.hpp - ANIM.hpp ANIM.cpp - PART.hpp PART.cpp - SWHC.hpp SWHC.cpp - CRSC.hpp CRSC.cpp - ELSC.hpp ELSC.cpp - WPSC.hpp WPSC.cpp - DPSC.hpp DPSC.cpp - ParticleCommon.cpp - FONT.hpp FONT.cpp - DGRP.hpp DGRP.cpp - ATBL.hpp ATBL.cpp - DeafBabe.hpp DeafBabe.cpp - BabeDead.hpp BabeDead.cpp - RigInverter.hpp RigInverter.cpp - AROTBuilder.hpp AROTBuilder.cpp - Tweaks/TweakWriter.hpp - Tweaks/ITweakGame.hpp - Tweaks/ITweakParticle.hpp - Tweaks/ITweakPlayer.hpp - Tweaks/ITweakPlayerControl.hpp - Tweaks/ITweakPlayerGun.hpp - Tweaks/ITweakGunRes.hpp - Tweaks/ITweakPlayerRes.hpp - Tweaks/ITweakGui.hpp - Tweaks/ITweakSlideShow.hpp) +set(DNACOMMON_SOURCES + DNACommon.hpp DNACommon.cpp + ${liblist} + PAK.hpp PAK.cpp + GX.hpp + FSM2.hpp FSM2.cpp + MLVL.hpp MLVL.cpp + CMDL.cpp + MAPA.cpp + STRG.hpp STRG.cpp + TXTR.hpp TXTR.cpp + ANCS.hpp + ANIM.hpp ANIM.cpp + PART.hpp PART.cpp + SWHC.hpp SWHC.cpp + CRSC.hpp CRSC.cpp + ELSC.hpp ELSC.cpp + WPSC.hpp WPSC.cpp + DPSC.hpp DPSC.cpp + ParticleCommon.cpp + FONT.hpp FONT.cpp + DGRP.hpp DGRP.cpp + ATBL.hpp ATBL.cpp + DeafBabe.hpp DeafBabe.cpp + BabeDead.hpp BabeDead.cpp + RigInverter.hpp RigInverter.cpp + AROTBuilder.hpp AROTBuilder.cpp + Tweaks/TweakWriter.hpp + Tweaks/ITweakGame.hpp + Tweaks/ITweakParticle.hpp + Tweaks/ITweakPlayer.hpp + Tweaks/ITweakPlayerControl.hpp + Tweaks/ITweakPlayerGun.hpp + Tweaks/ITweakGunRes.hpp + Tweaks/ITweakPlayerRes.hpp + Tweaks/ITweakGui.hpp + Tweaks/ITweakSlideShow.hpp) + +dataspec_add_list(DNACommon DNACOMMON_SOURCES) diff --git a/DataSpec/DNACommon/DNACommon.cpp b/DataSpec/DNACommon/DNACommon.cpp index c8191b6f7..a4e47644c 100644 --- a/DataSpec/DNACommon/DNACommon.cpp +++ b/DataSpec/DNACommon/DNACommon.cpp @@ -11,4 +11,241 @@ ThreadLocalPtr g_PakRouter; ThreadLocalPtr UniqueIDBridge::s_Project; UniqueID32 UniqueID32::kInvalidId; +template +hecl::ProjectPath UniqueIDBridge::TranslatePakIdToPath(const IDType& id, bool silenceWarnings) +{ + /* Try PAKRouter first (only available at extract) */ + PAKRouterBase* pakRouter = g_PakRouter.get(); + if (pakRouter) + { + hecl::ProjectPath path = pakRouter->getWorking(id, silenceWarnings); + if (path) + return path; + } + + /* Try project cache second (populated with paths read from YAML resources) */ + hecl::Database::Project* project = s_Project.get(); + if (!project) + { + if (pakRouter) + { + if (!silenceWarnings) + LogDNACommon.report(logvisor::Warning, + "unable to translate %s to path", id.toString().c_str()); + return {}; + } + LogDNACommon.report(logvisor::Fatal, + "g_PakRouter or s_Project must be set to non-null before " + "calling UniqueIDBridge::TranslatePakIdToPath"); + } + + const hecl::ProjectPath* search = project->lookupBridgePath(id.toUint64()); + if (!search) + { + if (!silenceWarnings) + LogDNACommon.report(logvisor::Warning, + "unable to translate %s to path", id.toString().c_str()); + return {}; + } + return *search; +} +template +hecl::ProjectPath UniqueIDBridge::TranslatePakIdToPath(const UniqueID32& id, bool silenceWarnings); +template +hecl::ProjectPath UniqueIDBridge::TranslatePakIdToPath(const UniqueID64& id, bool silenceWarnings); + +template +hecl::ProjectPath UniqueIDBridge::MakePathFromString(const std::string& str) +{ + hecl::Database::Project* project = s_Project.get(); + if (!project) + LogDNACommon.report(logvisor::Fatal, + "UniqueIDBridge::setGlobalProject must be called before MakePathFromString"); + hecl::ProjectPath path = hecl::ProjectPath(*project, str); + project->addBridgePathToCache(IDType(path), path); + return path; +} +template +hecl::ProjectPath UniqueIDBridge::MakePathFromString(const std::string& str); +template +hecl::ProjectPath UniqueIDBridge::MakePathFromString(const std::string& str); + +template +void UniqueIDBridge::TransformOldHashToNewHash(IDType& id) +{ + id = TranslatePakIdToPath(id); +} +template +void UniqueIDBridge::TransformOldHashToNewHash(UniqueID32& id); +template +void UniqueIDBridge::TransformOldHashToNewHash(UniqueID64& id); + +void UniqueIDBridge::setThreadProject(hecl::Database::Project& project) +{ + s_Project.reset(&project); +} + +/** PAK 32-bit Unique ID */ +void UniqueID32::read(athena::io::IStreamReader& reader) +{m_id = reader.readUint32Big();} +void UniqueID32::write(athena::io::IStreamWriter& writer) const +{writer.writeUint32Big(m_id);} +void UniqueID32::read(athena::io::YAMLDocReader& reader) +{ + *this = UniqueIDBridge::MakePathFromString(reader.readString(nullptr)); +} +void UniqueID32::write(athena::io::YAMLDocWriter& writer) const +{ + if (!operator bool()) + return; + hecl::ProjectPath path = UniqueIDBridge::TranslatePakIdToPath(*this); + if (!path) + return; + writer.writeString(nullptr, path.getRelativePathUTF8()); +} +size_t UniqueID32::binarySize(size_t __isz) const +{return __isz + 4;} + +std::string UniqueID32::toString() const +{ + char buf[9]; + snprintf(buf, 9, "%08X", m_id); + return std::string(buf); +} + +AuxiliaryID32& AuxiliaryID32::operator=(const hecl::ProjectPath& path) +{ + m_id = path.ensureAuxInfo(m_auxStr).hash().val32(); + return *this; +} + +AuxiliaryID32& AuxiliaryID32::operator=(const UniqueID32& id) +{ + m_baseId = id; + hecl::ProjectPath path = UniqueIDBridge::TranslatePakIdToPath(id); + if (path) + { + if (m_addExtension) + path = path.getWithExtension(m_addExtension); + *this = path; + } + return *this; +} + +void AuxiliaryID32::read(athena::io::IStreamReader& reader) +{ + m_id = reader.readUint32Big(); + m_baseId = *this; +} + +void AuxiliaryID32::write(athena::io::IStreamWriter& writer) const +{ + writer.writeUint32Big(m_id); +} + +void AuxiliaryID32::read(athena::io::YAMLDocReader& reader) +{ + hecl::ProjectPath readPath = UniqueIDBridge::MakePathFromString(reader.readString(nullptr)); + *this = readPath.ensureAuxInfo(m_auxStr); +} + +void AuxiliaryID32::write(athena::io::YAMLDocWriter& writer) const +{ + if (!operator bool()) + return; + hecl::ProjectPath path = UniqueIDBridge::TranslatePakIdToPath(*this, true); + if (!path) + path = UniqueIDBridge::TranslatePakIdToPath(m_baseId); + if (!path) + return; + if (m_addExtension) + path = path.getWithExtension(m_addExtension); + hecl::SystemUTF8View ufx8AuxStr(m_auxStr); + writer.writeString(nullptr, path.getRelativePathUTF8() + '|' + ufx8AuxStr); +} + + +/** PAK 64-bit Unique ID */ +void UniqueID64::read(athena::io::IStreamReader& reader) +{m_id = reader.readUint64Big();} +void UniqueID64::write(athena::io::IStreamWriter& writer) const +{writer.writeUint64Big(m_id);} +void UniqueID64::read(athena::io::YAMLDocReader& reader) +{ + *this = UniqueIDBridge::MakePathFromString(reader.readString(nullptr)); +} +void UniqueID64::write(athena::io::YAMLDocWriter& writer) const +{ + if (!operator bool()) + return; + hecl::ProjectPath path = UniqueIDBridge::TranslatePakIdToPath(*this); + if (!path) + return; + writer.writeString(nullptr, path.getRelativePathUTF8()); +} +size_t UniqueID64::binarySize(size_t __isz) const +{return __isz + 8;} + +std::string UniqueID64::toString() const +{ + char buf[17]; + snprintf(buf, 17, "%016" PRIX64, m_id); + return std::string(buf); +} + +/** PAK 128-bit Unique ID */ +void UniqueID128::read(athena::io::IStreamReader& reader) +{ + m_id[0] = reader.readUint64Big(); + m_id[1] = reader.readUint64Big(); +} +void UniqueID128::write(athena::io::IStreamWriter& writer) const +{ + writer.writeUint64Big(m_id[0]); + writer.writeUint64Big(m_id[1]); +} +void UniqueID128::read(athena::io::YAMLDocReader& reader) +{ + *this = UniqueIDBridge::MakePathFromString(reader.readString(nullptr)); +} +void UniqueID128::write(athena::io::YAMLDocWriter& writer) const +{ + if (!operator bool()) + return; + hecl::ProjectPath path = UniqueIDBridge::TranslatePakIdToPath(*this); + if (!path) + return; + writer.writeString(nullptr, path.getRelativePathUTF8()); +} +size_t UniqueID128::binarySize(size_t __isz) const +{return __isz + 16;} + +std::string UniqueID128::toString() const +{ + char buf[33]; + snprintf(buf, 33, "%016" PRIX64 "%016" PRIX64, m_id[0], m_id[1]); + return std::string(buf); +} + + +/** Word Bitmap reader/writer */ +void WordBitmap::read(athena::io::IStreamReader& reader, size_t bitCount) +{ + m_bitCount = bitCount; + size_t wordCount = (bitCount + 31) / 32; + m_words.clear(); + m_words.reserve(wordCount); + for (size_t w=0 ; w s_Project; public: template - static hecl::ProjectPath TranslatePakIdToPath(const IDType& id, bool silenceWarnings=false) - { - /* Try PAKRouter first (only available at extract) */ - PAKRouterBase* pakRouter = g_PakRouter.get(); - if (pakRouter) - { - hecl::ProjectPath path = pakRouter->getWorking(id, silenceWarnings); - if (path) - return path; - } - - /* Try project cache second (populated with paths read from YAML resources) */ - hecl::Database::Project* project = s_Project.get(); - if (!project) - LogDNACommon.report(logvisor::Fatal, - "g_PakRouter or s_Project must be set to non-null before " - "calling UniqueIDBridge::TranslatePakIdToPath"); - - const hecl::ProjectPath* search = project->lookupBridgePath(id.toUint64()); - if (!search) - { - if (!silenceWarnings) - LogDNACommon.report(logvisor::Warning, - "unable to translate %s to path", id.toString().c_str()); - return {}; - } - return *search; - } + static hecl::ProjectPath TranslatePakIdToPath(const IDType& id, bool silenceWarnings=false); template - static hecl::ProjectPath MakePathFromString(const std::string& str) - { - hecl::Database::Project* project = s_Project.get(); - if (!project) - LogDNACommon.report(logvisor::Fatal, - "UniqueIDBridge::setGlobalProject must be called before MakePathFromString"); - hecl::ProjectPath path = hecl::ProjectPath(*project, str); - project->addBridgePathToCache(IDType(path), path); - return path; - } + static hecl::ProjectPath MakePathFromString(const std::string& str); template - static void TransformOldHashToNewHash(IDType& id) - { - id = TranslatePakIdToPath(id); - } + static void TransformOldHashToNewHash(IDType& id); - static void setThreadProject(hecl::Database::Project& project) - { - s_Project.reset(&project); - } + static void setThreadProject(hecl::Database::Project& project); }; /** PAK 32-bit Unique ID */ @@ -185,25 +143,11 @@ public: static UniqueID32 kInvalidId; Delete expl; operator bool() const {return m_id != 0xffffffff && m_id != 0;} - void read(athena::io::IStreamReader& reader) - {m_id = reader.readUint32Big();} - void write(athena::io::IStreamWriter& writer) const - {writer.writeUint32Big(m_id);} - void read(athena::io::YAMLDocReader& reader) - { - *this = UniqueIDBridge::MakePathFromString(reader.readString(nullptr)); - } - void write(athena::io::YAMLDocWriter& writer) const - { - if (!operator bool()) - return; - hecl::ProjectPath path = UniqueIDBridge::TranslatePakIdToPath(*this); - if (!path) - return; - writer.writeString(nullptr, path.getRelativePathUTF8()); - } - size_t binarySize(size_t __isz) const - {return __isz + 4;} + void read(athena::io::IStreamReader& reader); + void write(athena::io::IStreamWriter& writer) const; + void read(athena::io::YAMLDocReader& reader); + void write(athena::io::YAMLDocWriter& writer) const; + size_t binarySize(size_t __isz) const; UniqueID32& operator=(const hecl::ProjectPath& path) {m_id = path.hash().val32(); return *this;} @@ -212,12 +156,7 @@ public: bool operator==(const UniqueID32& other) const {return m_id == other.m_id;} uint32_t toUint32() const {return m_id;} uint64_t toUint64() const {return m_id;} - std::string toString() const - { - char buf[9]; - snprintf(buf, 9, "%08X", m_id); - return std::string(buf); - } + std::string toString() const; void clear() {m_id = 0xffffffff;} UniqueID32() = default; @@ -252,57 +191,12 @@ public: const hecl::SystemChar* addExtension=nullptr) : m_auxStr(auxStr), m_addExtension(addExtension) {} - AuxiliaryID32& operator=(const hecl::ProjectPath& path) - { - m_id = path.ensureAuxInfo(m_auxStr).hash().val32(); - return *this; - } - - AuxiliaryID32& operator=(const UniqueID32& id) - { - m_baseId = id; - hecl::ProjectPath path = UniqueIDBridge::TranslatePakIdToPath(id); - if (path) - { - if (m_addExtension) - path = path.getWithExtension(m_addExtension); - *this = path; - } - return *this; - } - - void read(athena::io::IStreamReader& reader) - { - m_id = reader.readUint32Big(); - m_baseId = *this; - } - - void write(athena::io::IStreamWriter& writer) const - { - writer.writeUint32Big(m_id); - } - - void read(athena::io::YAMLDocReader& reader) - { - hecl::ProjectPath readPath = UniqueIDBridge::MakePathFromString(reader.readString(nullptr)); - *this = readPath.ensureAuxInfo(m_auxStr); - } - - void write(athena::io::YAMLDocWriter& writer) const - { - if (!operator bool()) - return; - hecl::ProjectPath path = UniqueIDBridge::TranslatePakIdToPath(*this, true); - if (!path) - path = UniqueIDBridge::TranslatePakIdToPath(m_baseId); - if (!path) - return; - if (m_addExtension) - path = path.getWithExtension(m_addExtension); - hecl::SystemUTF8View ufx8AuxStr(m_auxStr); - writer.writeString(nullptr, path.getRelativePathUTF8() + '|' + ufx8AuxStr); - } - + AuxiliaryID32& operator=(const hecl::ProjectPath& path); + AuxiliaryID32& operator=(const UniqueID32& id); + void read(athena::io::IStreamReader& reader); + void write(athena::io::IStreamWriter& writer) const; + void read(athena::io::YAMLDocReader& reader); + void write(athena::io::YAMLDocWriter& writer) const; const UniqueID32& getBaseId() const {return m_baseId;} }; @@ -313,25 +207,11 @@ class UniqueID64 : public BigYAML public: Delete expl; operator bool() const {return m_id != 0xffffffffffffffff && m_id != 0;} - void read(athena::io::IStreamReader& reader) - {m_id = reader.readUint64Big();} - void write(athena::io::IStreamWriter& writer) const - {writer.writeUint64Big(m_id);} - void read(athena::io::YAMLDocReader& reader) - { - *this = UniqueIDBridge::MakePathFromString(reader.readString(nullptr)); - } - void write(athena::io::YAMLDocWriter& writer) const - { - if (!operator bool()) - return; - hecl::ProjectPath path = UniqueIDBridge::TranslatePakIdToPath(*this); - if (!path) - return; - writer.writeString(nullptr, path.getRelativePathUTF8()); - } - size_t binarySize(size_t __isz) const - {return __isz + 8;} + void read(athena::io::IStreamReader& reader); + void write(athena::io::IStreamWriter& writer) const; + void read(athena::io::YAMLDocReader& reader); + void write(athena::io::YAMLDocWriter& writer) const; + size_t binarySize(size_t __isz) const; UniqueID64& operator=(const hecl::ProjectPath& path) {m_id = path.hash().val64(); return *this;} @@ -339,12 +219,7 @@ public: bool operator!=(const UniqueID64& other) const {return m_id != other.m_id;} bool operator==(const UniqueID64& other) const {return m_id == other.m_id;} uint64_t toUint64() const {return m_id;} - std::string toString() const - { - char buf[17]; - snprintf(buf, 17, "%016" PRIX64, m_id); - return std::string(buf); - } + std::string toString() const; void clear() {m_id = 0xffffffffffffffff;} UniqueID64() = default; @@ -392,31 +267,11 @@ public: UniqueID128() {m_id[0]=0xffffffffffffffff; m_id[1]=0xffffffffffffffff;} operator bool() const {return m_id[0] != 0xffffffffffffffff && m_id[0] != 0 && m_id[1] != 0xffffffffffffffff && m_id[1] != 0;} - void read(athena::io::IStreamReader& reader) - { - m_id[0] = reader.readUint64Big(); - m_id[1] = reader.readUint64Big(); - } - void write(athena::io::IStreamWriter& writer) const - { - writer.writeUint64Big(m_id[0]); - writer.writeUint64Big(m_id[1]); - } - void read(athena::io::YAMLDocReader& reader) - { - *this = UniqueIDBridge::MakePathFromString(reader.readString(nullptr)); - } - void write(athena::io::YAMLDocWriter& writer) const - { - if (!operator bool()) - return; - hecl::ProjectPath path = UniqueIDBridge::TranslatePakIdToPath(*this); - if (!path) - return; - writer.writeString(nullptr, path.getRelativePathUTF8()); - } - size_t binarySize(size_t __isz) const - {return __isz + 16;} + void read(athena::io::IStreamReader& reader); + void write(athena::io::IStreamWriter& writer) const; + void read(athena::io::YAMLDocReader& reader); + void write(athena::io::YAMLDocWriter& writer) const; + size_t binarySize(size_t __isz) const; UniqueID128& operator=(const hecl::ProjectPath& path) { @@ -450,12 +305,7 @@ public: uint64_t toUint64() const {return m_id[0];} uint64_t toHighUint64() const {return m_id[0];} uint64_t toLowUint64() const {return m_id[1];} - std::string toString() const - { - char buf[33]; - snprintf(buf, 33, "%016" PRIX64 "%016" PRIX64, m_id[0], m_id[1]); - return std::string(buf); - } + std::string toString() const; static constexpr size_t BinarySize() {return 16;} }; @@ -466,24 +316,9 @@ class WordBitmap std::vector m_words; size_t m_bitCount = 0; public: - void read(athena::io::IStreamReader& reader, size_t bitCount) - { - m_bitCount = bitCount; - size_t wordCount = (bitCount + 31) / 32; - m_words.clear(); - m_words.reserve(wordCount); - for (size_t w=0 ; w