From 6dda293cb9707a575e108c0f9e3d4b9bac5c4592 Mon Sep 17 00:00:00 2001 From: Jack Andersen Date: Wed, 10 Jun 2015 18:55:06 -1000 Subject: [PATCH] lots of dataspec implementation --- hecl/dataspec/dataspec.cpp | 32 ++++++-- hecl/dataspec/hecl/hecl.cpp | 50 +++++++++++++ hecl/dataspec/hecl/hecl.pri | 2 + hecl/dataspec/mp1/CMDL.hpp | 0 hecl/dataspec/mp1/MREA.hpp | 0 hecl/dataspec/mp1/mp1.cpp | 22 ++++++ hecl/dataspec/mp1/mp1.pri | 4 + hecl/dataspec/mp2/CMDL.hpp | 0 hecl/dataspec/mp2/MREA.hpp | 0 hecl/dataspec/mp2/mp2.cpp | 22 ++++++ hecl/dataspec/mp2/mp2.pri | 4 + hecl/dataspec/mp3/CMDL.hpp | 0 hecl/dataspec/mp3/MREA.hpp | 0 hecl/dataspec/mp3/mp3.cpp | 22 ++++++ hecl/dataspec/mp3/mp3.pri | 4 + hecl/driver/ToolBase.hpp | 1 + hecl/driver/ToolSpec.hpp | 109 ++++++++++++++++++++++++++- hecl/driver/driver.pro | 2 +- hecl/driver/main.cpp | 48 +++++++----- hecl/include/HECL.hpp | 48 +++++++----- hecl/include/HECLDatabase.hpp | 137 ++++++++++++++++++++-------------- hecl/lib/ProjectPath.cpp | 93 +++++++++++++++++++++-- hecl/lib/database/Project.cpp | 62 +++++++-------- hecl/lib/lib.pro | 2 +- 24 files changed, 524 insertions(+), 140 deletions(-) create mode 100644 hecl/dataspec/hecl/hecl.pri create mode 100644 hecl/dataspec/mp1/CMDL.hpp create mode 100644 hecl/dataspec/mp1/MREA.hpp create mode 100644 hecl/dataspec/mp2/CMDL.hpp create mode 100644 hecl/dataspec/mp2/MREA.hpp create mode 100644 hecl/dataspec/mp3/CMDL.hpp create mode 100644 hecl/dataspec/mp3/MREA.hpp diff --git a/hecl/dataspec/dataspec.cpp b/hecl/dataspec/dataspec.cpp index 64b1bf9ed..4e714af41 100644 --- a/hecl/dataspec/dataspec.cpp +++ b/hecl/dataspec/dataspec.cpp @@ -1,10 +1,30 @@ #include "HECLDatabase.hpp" -const std::pair DATA_SPECS[] = +namespace HECL { - {"hecl-little", "Targets little-endian pc apps using the HECL runtime"}, - {"hecl-big", "Targets big-endian pc apps using the HECL runtime"}, - {"hecl-revolution", "Targets Wii apps using the HECL runtime"}, - {"hecl-cafe", "Targets Wii U apps using the HECL runtime"}, - {"mp1", ""} +namespace Database +{ + +IDataSpec* NewDataSpec_little(); +IDataSpec* NewDataSpec_big(); +IDataSpec* NewDataSpec_revolution(); +IDataSpec* NewDataSpec_cafe(); + +IDataSpec* NewDataSpec_mp1(); +IDataSpec* NewDataSpec_mp2(); +IDataSpec* NewDataSpec_mp3(); + +const DataSpecEntry DATA_SPEC_REGISTRY[] = +{ + {_S("hecl-little"), _S("Targets little-endian PC apps using the HECL runtime"), NewDataSpec_little}, + {_S("hecl-big"), _S("Targets big-endian PC apps using the HECL runtime"), NewDataSpec_big}, + {_S("hecl-revolution"), _S("Targets Wii apps using the HECL runtime"), NewDataSpec_revolution}, + {_S("hecl-cafe"), _S("Targets Wii U apps using the HECL runtime"), NewDataSpec_cafe}, + {_S("mp1"), _S("Targets original Metroid Prime engine"), NewDataSpec_mp1}, + {_S("mp2"), _S("Targets original Metroid Prime 2 engine"), NewDataSpec_mp2}, + {_S("mp3"), _S("Targets original Metroid Prime 3 engine"), NewDataSpec_mp3}, + {} }; + +} +} diff --git a/hecl/dataspec/hecl/hecl.cpp b/hecl/dataspec/hecl/hecl.cpp index e69de29bb..40d37cf1d 100644 --- a/hecl/dataspec/hecl/hecl.cpp +++ b/hecl/dataspec/hecl/hecl.cpp @@ -0,0 +1,50 @@ +#include "HECLDatabase.hpp" + +namespace HECL +{ +namespace Database +{ + +class HECLBaseDataSpec : public IDataSpec +{ + +}; + +class HECLLittleDataSpec : public HECLBaseDataSpec +{ +}; + +class HECLBigDataSpec : public HECLBaseDataSpec +{ +}; + +class HECLRevolutionDataSpec : public HECLBaseDataSpec +{ +}; + +class HECLCafeDataSpec : public HECLBaseDataSpec +{ +}; + +IDataSpec* NewDataSpec_little() +{ + return new HECLLittleDataSpec(); +} + +IDataSpec* NewDataSpec_big() +{ + return new HECLBigDataSpec(); +} + +IDataSpec* NewDataSpec_revolution() +{ + return new HECLRevolutionDataSpec(); +} + +IDataSpec* NewDataSpec_cafe() +{ + return new HECLCafeDataSpec(); +} + +} +} diff --git a/hecl/dataspec/hecl/hecl.pri b/hecl/dataspec/hecl/hecl.pri new file mode 100644 index 000000000..965c4a673 --- /dev/null +++ b/hecl/dataspec/hecl/hecl.pri @@ -0,0 +1,2 @@ +SOURCES += \ + $$PWD/hecl.cpp diff --git a/hecl/dataspec/mp1/CMDL.hpp b/hecl/dataspec/mp1/CMDL.hpp new file mode 100644 index 000000000..e69de29bb diff --git a/hecl/dataspec/mp1/MREA.hpp b/hecl/dataspec/mp1/MREA.hpp new file mode 100644 index 000000000..e69de29bb diff --git a/hecl/dataspec/mp1/mp1.cpp b/hecl/dataspec/mp1/mp1.cpp index e69de29bb..954b86acc 100644 --- a/hecl/dataspec/mp1/mp1.cpp +++ b/hecl/dataspec/mp1/mp1.cpp @@ -0,0 +1,22 @@ +#include "HECLDatabase.hpp" + +#include "CMDL.hpp" +#include "MREA.hpp" + +namespace HECL +{ +namespace Database +{ + +class MP1DataSpec : public IDataSpec +{ + +}; + +IDataSpec* NewDataSpec_mp1() +{ + return new MP1DataSpec(); +} + +} +} diff --git a/hecl/dataspec/mp1/mp1.pri b/hecl/dataspec/mp1/mp1.pri index f30e8b372..fbd40fca1 100644 --- a/hecl/dataspec/mp1/mp1.pri +++ b/hecl/dataspec/mp1/mp1.pri @@ -1,2 +1,6 @@ SOURCES += \ $$PWD/mp1.cpp + +HEADERS += \ + $$PWD/MREA.hpp \ + $$PWD/CMDL.hpp diff --git a/hecl/dataspec/mp2/CMDL.hpp b/hecl/dataspec/mp2/CMDL.hpp new file mode 100644 index 000000000..e69de29bb diff --git a/hecl/dataspec/mp2/MREA.hpp b/hecl/dataspec/mp2/MREA.hpp new file mode 100644 index 000000000..e69de29bb diff --git a/hecl/dataspec/mp2/mp2.cpp b/hecl/dataspec/mp2/mp2.cpp index e69de29bb..14a610a71 100644 --- a/hecl/dataspec/mp2/mp2.cpp +++ b/hecl/dataspec/mp2/mp2.cpp @@ -0,0 +1,22 @@ +#include "HECLDatabase.hpp" + +#include "CMDL.hpp" +#include "MREA.hpp" + +namespace HECL +{ +namespace Database +{ + +class MP2DataSpec : public IDataSpec +{ + +}; + +IDataSpec* NewDataSpec_mp2() +{ + return new MP2DataSpec(); +} + +} +} diff --git a/hecl/dataspec/mp2/mp2.pri b/hecl/dataspec/mp2/mp2.pri index afaad83b0..11ac5182b 100644 --- a/hecl/dataspec/mp2/mp2.pri +++ b/hecl/dataspec/mp2/mp2.pri @@ -1,2 +1,6 @@ SOURCES += \ $$PWD/mp2.cpp + +HEADERS += \ + $$PWD/MREA.hpp \ + $$PWD/CMDL.hpp diff --git a/hecl/dataspec/mp3/CMDL.hpp b/hecl/dataspec/mp3/CMDL.hpp new file mode 100644 index 000000000..e69de29bb diff --git a/hecl/dataspec/mp3/MREA.hpp b/hecl/dataspec/mp3/MREA.hpp new file mode 100644 index 000000000..e69de29bb diff --git a/hecl/dataspec/mp3/mp3.cpp b/hecl/dataspec/mp3/mp3.cpp index e69de29bb..f60ac54f3 100644 --- a/hecl/dataspec/mp3/mp3.cpp +++ b/hecl/dataspec/mp3/mp3.cpp @@ -0,0 +1,22 @@ +#include "HECLDatabase.hpp" + +#include "CMDL.hpp" +#include "MREA.hpp" + +namespace HECL +{ +namespace Database +{ + +class MP3DataSpec : public IDataSpec +{ + +}; + +IDataSpec* NewDataSpec_mp3() +{ + return new MP3DataSpec(); +} + +} +} diff --git a/hecl/dataspec/mp3/mp3.pri b/hecl/dataspec/mp3/mp3.pri index 50b65534d..502afb0ec 100644 --- a/hecl/dataspec/mp3/mp3.pri +++ b/hecl/dataspec/mp3/mp3.pri @@ -1,2 +1,6 @@ SOURCES += \ $$PWD/mp3.cpp + +HEADERS += \ + $$PWD/MREA.hpp \ + $$PWD/CMDL.hpp diff --git a/hecl/driver/ToolBase.hpp b/hecl/driver/ToolBase.hpp index ada95fe06..61d88b756 100644 --- a/hecl/driver/ToolBase.hpp +++ b/hecl/driver/ToolBase.hpp @@ -15,6 +15,7 @@ struct ToolPassInfo HECL::SystemString cwd; std::vector args; HECL::SystemString output; + HECL::Database::Project* project = NULL; unsigned verbosityLevel = 0; bool force = false; }; diff --git a/hecl/driver/ToolSpec.hpp b/hecl/driver/ToolSpec.hpp index cc39f2290..790ec86cf 100644 --- a/hecl/driver/ToolSpec.hpp +++ b/hecl/driver/ToolSpec.hpp @@ -3,17 +3,59 @@ #include "ToolBase.hpp" #include +#include class ToolSpec final : public ToolBase { + enum Mode + { + MLIST = 0, + MENABLE, + MDISABLE + } mode = MLIST; public: ToolSpec(const ToolPassInfo& info) : ToolBase(info) { - } + if (info.args.empty()) + return; - ~ToolSpec() - { + if (!info.project) + throw HECL::Exception(_S("hecl spec must be ran within a project directory")); + + const auto& specs = info.project->getDataSpecs(); + HECL::SystemString firstArg = info.args[0]; + HECL::ToLower(firstArg); + + static const HECL::SystemString enable(_S("enable")); + static const HECL::SystemString disable(_S("disable")); + if (!firstArg.compare(enable)) + mode = MENABLE; + else if (!firstArg.compare(disable)) + mode = MDISABLE; + else + return; + + if (info.args.size() < 2) + throw HECL::Exception(_S("Speclist argument required")); + + for (auto it = info.args.begin()+1; + it != info.args.end(); + ++it) + { + + bool found = false; + for (auto& spec : specs) + { + if (!spec.first.name.compare(*it)) + { + found = true; + break; + } + } + if (!found) + throw HECL::Exception(_S("'") + *it + _S("' is not found in the dataspec registry")); + } } static void Help(HelpOutput& help) @@ -46,6 +88,67 @@ public: int run() { + if (!m_info.project) + { + for (const HECL::Database::DataSpecEntry* spec = HECL::Database::DATA_SPEC_REGISTRY; + spec->name.size(); + ++spec) + { + if (XTERM_COLOR) + HECL::Printf(_S("" BOLD CYAN "%s" NORMAL "\n"), spec->name.c_str()); + else + HECL::Printf(_S("%s\n"), spec->name.c_str()); + HECL::Printf(_S(" %s\n"), spec->desc.c_str()); + } + return 0; + } + + const auto& specs = m_info.project->getDataSpecs(); + if (mode == MLIST) + { + for (auto& spec : specs) + { + if (XTERM_COLOR) + HECL::Printf(_S("" BOLD CYAN "%s" NORMAL ""), spec.first.name.c_str()); + else + HECL::Printf(_S("%s"), spec.first.name.c_str()); + if (spec.second) + { + if (XTERM_COLOR) + HECL::Printf(_S(" " BOLD GREEN "[ENABLED]" NORMAL "")); + else + HECL::Printf(_S(" [ENABLED]")); + } + HECL::Printf(_S("\n %s\n"), spec.first.desc.c_str()); + } + return 0; + } + + std::vector opSpecs; + for (auto it = m_info.args.begin()+1; + it != m_info.args.end(); + ++it) + { + HECL::SystemString itName = *it; + HECL::ToLower(itName); + for (auto& spec : specs) + { + if (!spec.first.name.compare(itName)) + { + opSpecs.push_back(spec.first.name); + break; + } + } + } + + if (opSpecs.size()) + { + if (mode == MENABLE) + m_info.project->enableDataSpecs(opSpecs); + else if (mode == MDISABLE) + m_info.project->disableDataSpecs(opSpecs); + } + return 0; } }; diff --git a/hecl/driver/driver.pro b/hecl/driver/driver.pro index f81f32755..a5b87bf57 100644 --- a/hecl/driver/driver.pro +++ b/hecl/driver/driver.pro @@ -7,7 +7,7 @@ unix:LIBS += -std=c++11 clang:QMAKE_CXXFLAGS += -stdlib=libc++ clang:LIBS += -stdlib=libc++ -lc++abi -INCLUDEPATH += ../include +INCLUDEPATH += ../include ../extern/Athena/include LIBPATH += $$OUT_PWD/../lib \ $$OUT_PWD/../dataspec \ diff --git a/hecl/driver/main.cpp b/hecl/driver/main.cpp index 490e4a604..abbe0f9aa 100644 --- a/hecl/driver/main.cpp +++ b/hecl/driver/main.cpp @@ -149,34 +149,49 @@ int main(int argc, const char** argv) info.args.push_back(arg); } + /* Attempt to find hecl project */ + HECL::ProjectRootPath* rootPath = HECL::SearchForProject(info.cwd); + std::unique_ptr project; + if (rootPath) + { + try + { + project.reset(new HECL::Database::Project(*rootPath)); + info.project = project.get(); + } + catch (HECL::Exception& ex) + { + HECL::FPrintf(stderr, + _S("Unable to open discovered project at '%s':\n%s\n"), + rootPath->getAbsolutePath().c_str(), ex.swhat()); + return -1; + } + } + /* Construct selected tool */ HECL::SystemString toolName(argv[1]); -#if HECL_UCS2 - std::transform(toolName.begin(), toolName.end(), toolName.begin(), towlower); -#else - std::transform(toolName.begin(), toolName.end(), toolName.begin(), tolower); -#endif - ToolBase* tool = NULL; + HECL::ToLower(toolName); + std::unique_ptr tool; try { if (toolName == _S("init")) - tool = new ToolInit(info); + tool.reset(new ToolInit(info)); else if (toolName == _S("spec")) - tool = new ToolSpec(info); + tool.reset(new ToolSpec(info)); else if (toolName == _S("add")) - tool = new ToolAdd(info); + tool.reset(new ToolAdd(info)); else if (toolName == _S("remove") || toolName == _S("rm")) - tool = new ToolRemove(info); + tool.reset(new ToolRemove(info)); else if (toolName == _S("group")) - tool = new ToolGroup(info); + tool.reset(new ToolGroup(info)); else if (toolName == _S("cook")) - tool = new ToolCook(info); + tool.reset(new ToolCook(info)); else if (toolName == _S("clean")) - tool = new ToolClean(info); + tool.reset(new ToolClean(info)); else if (toolName == _S("package") || toolName == _S("pack")) - tool = new ToolPackage(info); + tool.reset(new ToolPackage(info)); else if (toolName == _S("help")) - tool = new ToolHelp(info); + tool.reset(new ToolHelp(info)); else throw HECL::Exception(_S("unrecognized tool '") + toolName + _S("'")); } @@ -185,7 +200,6 @@ int main(int argc, const char** argv) HECL::FPrintf(stderr, _S("Unable to construct HECL tool '%s':\n%s\n"), toolName.c_str(), ex.swhat()); - delete tool; return -1; } @@ -201,10 +215,8 @@ int main(int argc, const char** argv) catch (HECL::Exception& ex) { HECL::FPrintf(stderr, _S("Error running HECL tool '%s':\n%s\n"), toolName.c_str(), ex.swhat()); - delete tool; return -1; } - delete tool; return retval; } diff --git a/hecl/include/HECL.hpp b/hecl/include/HECL.hpp index e252941b8..590d91ac0 100644 --- a/hecl/include/HECL.hpp +++ b/hecl/include/HECL.hpp @@ -17,6 +17,7 @@ char* win_realpath(const char* name, char* restrict resolved); #include #include #include +#include #include #include "../extern/blowfish/blowfish.h" @@ -33,6 +34,10 @@ std::wstring UTF8ToWide(const std::string& src); #if HECL_UCS2 typedef wchar_t SystemChar; typedef std::wstring SystemString; +static inline void ToLower(SystemString& str) +{std::transform(str.begin(), str.end(), str.begin(), towlower);} +static inline void ToUpper(SystemString& str) +{std::transform(str.begin(), str.end(), str.begin(), towupper);} class SystemUTF8View { std::string m_utf8; @@ -55,6 +60,10 @@ public: #else typedef char SystemChar; typedef std::string SystemString; +static inline void ToLower(SystemString& str) +{std::transform(str.begin(), str.end(), str.begin(), tolower);} +static inline void ToUpper(SystemString& str) +{std::transform(str.begin(), str.end(), str.begin(), toupper);} class SystemUTF8View { const std::string& m_utf8; @@ -232,11 +241,11 @@ public: : num(0) {} FourCC(const char* name) : num(*(uint32_t*)name) {} - inline bool operator==(FourCC& other) {return num == other.num;} - inline bool operator!=(FourCC& other) {return num != other.num;} - inline bool operator==(const char* other) {return num == *(uint32_t*)other;} - inline bool operator!=(const char* other) {return num != *(uint32_t*)other;} - inline std::string toString() {return std::string(fcc, 4);} + inline bool operator==(const FourCC& other) const {return num == other.num;} + inline bool operator!=(const FourCC& other) const {return num != other.num;} + inline bool operator==(const char* other) const {return num == *(uint32_t*)other;} + inline bool operator!=(const char* other) const {return num != *(uint32_t*)other;} + inline std::string toString() const {return std::string(fcc, 4);} }; /** @@ -270,12 +279,12 @@ public: */ class Time final { - uint64_t ts; + time_t ts; public: Time() : ts(time(NULL)) {} - Time(uint64_t ti) : ts(ti) {} + Time(time_t ti) : ts(ti) {} Time(const Time& other) {ts = other.ts;} - inline uint64_t getTs() const {return ts;} + inline time_t getTs() const {return ts;} inline Time& operator=(const Time& other) {ts = other.ts; return *this;} inline bool operator==(const Time& other) const {return ts == other.ts;} inline bool operator!=(const Time& other) const {return ts != other.ts;} @@ -302,7 +311,7 @@ class ProjectPath { protected: SystemString m_absPath; - const SystemChar* m_relPath = NULL; + SystemString m_relPath; size_t m_hash = 0; #if HECL_UCS2 std::string m_utf8AbsPath; @@ -322,7 +331,7 @@ public: * @brief Determine if ProjectPath represents project root directory * @return true if project root directory */ - inline bool isRoot() const {return (m_relPath == NULL);} + inline bool isRoot() const {return m_relPath.empty();} /** * @brief Access fully-canonicalized absolute path @@ -334,11 +343,12 @@ public: * @brief Access fully-canonicalized project-relative path * @return Relative pointer to within absolute-path or "." for project root-directory (use isRoot to detect) */ - inline const SystemChar* getRelativePath() const + inline const SystemString& getRelativePath() const { - if (m_relPath) + if (m_relPath.size()) return m_relPath; - return _S("."); + static const SystemString dot = _S("."); + return dot; } /** @@ -354,7 +364,7 @@ public: #endif } - inline const char* getRelativePathUTF8() const + inline const std::string& getRelativePathUTF8() const { #if HECL_UCS2 return m_utf8RelPath; @@ -416,11 +426,15 @@ class ProjectRootPath : public ProjectPath { public: ProjectRootPath(const SystemString& path) - { - _canonAbsPath(path); - } + {_canonAbsPath(path);} }; +/** + * @brief Search from within provided directory for the project root + * @param path absolute or relative file path to search from + * @return Newly-constructed root path or NULL if not found + */ +ProjectRootPath* SearchForProject(const SystemString& path); /* Type-sensitive byte swappers */ diff --git a/hecl/include/HECLDatabase.hpp b/hecl/include/HECLDatabase.hpp index 98f2bfc5e..302999d3f 100644 --- a/hecl/include/HECLDatabase.hpp +++ b/hecl/include/HECLDatabase.hpp @@ -12,6 +12,8 @@ #include #include +#include + #include "HECL.hpp" namespace HECL @@ -19,6 +21,68 @@ namespace HECL namespace Database { +/** + * @brief Nodegraph class for gathering dependency-resolved objects for packaging + */ +class PackageDepsgraph +{ +public: + struct Node + { + enum + { + NODE_DATA, + NODE_GROUP + } type; + SystemString path; + class ObjectBase* projectObj; + Node* sub; + Node* next; + }; +private: + friend class Project; + std::vector m_nodes; +public: + const Node* getRootNode() const {return &m_nodes[0];} +}; + +/** + * @brief Subclassed by dataspec entries to manage per-game aspects of the data pipeline + * + * The DataSpec class manages interfaces for unpackaging, cooking, and packaging + * of data for interacting with a specific system/game-engine. + */ +class IDataSpec +{ +public: + struct ExtractPassInfo + { + SystemString srcpath; + ProjectPath subpath; + }; + virtual bool canExtract(const ExtractPassInfo& info) {(void)info;return false;} + virtual void doExtract(const ExtractPassInfo& info) {(void)info;} + + struct PackagePassInfo + { + PackageDepsgraph& depsgraph; + ProjectPath subpath; + }; + virtual bool canPackage(const PackagePassInfo& info) {(void)info;return false;} + virtual void doPackage(const PackagePassInfo& info) {(void)info;} +}; + +/** + * @brief IDataSpec registry entry + */ +struct DataSpecEntry +{ + SystemString name; + SystemString desc; + std::function factory; +}; +extern const HECL::Database::DataSpecEntry DATA_SPEC_REGISTRY[]; + /** * @brief Base object to subclass for integrating with key project operations * @@ -31,7 +95,7 @@ namespace Database class ObjectBase { friend class Project; - std::string m_path; + SystemString m_path; protected: /** @@ -86,16 +150,10 @@ protected: {(void)depAdder;} public: - ObjectBase(const std::string& path) + ObjectBase(const SystemString& path) : m_path(path) {} - inline const std::string& getPath() const {return m_path;} - - /** - * @brief Overridable function to verify data at provided path - * @return true if ProjectObject subclass handles data at provided path/subpath - */ - static bool ClaimPath(const std::string& /*path*/, const std::string& /*subpath*/) {return false;} + inline const SystemString& getPath() const {return m_path;} }; @@ -109,7 +167,11 @@ public: */ class Project { +public: + typedef std::vector> CompiledSpecs; +private: ProjectRootPath m_rootPath; + CompiledSpecs m_compiledSpecs; public: Project(const HECL::ProjectRootPath& rootPath); @@ -133,6 +195,9 @@ public: void unlockAndDiscard(); void unlockAndCommit(); }; + ConfigFile m_specs; + ConfigFile m_paths; + ConfigFile m_groups; /** * @brief Index file handle @@ -145,6 +210,7 @@ public: SystemString m_filepath; const Project& m_project; size_t m_maxPathLen = 0; + size_t m_onlyUpdatedMaxPathLen = 0; FILE* m_lockedFile = NULL; public: class Entry @@ -152,12 +218,13 @@ public: friend class IndexFile; ProjectPath m_path; HECL::Time m_lastModtime; - bool m_removed = false; + bool m_updated = false; Entry(const ProjectPath& path, const HECL::Time& lastModtime) : m_path(path), m_lastModtime(lastModtime) {} Entry(const ProjectPath& path); }; private: + size_t m_updatedCount = 0; std::vector m_entryStore; std::unordered_map m_entryLookup; public: @@ -165,10 +232,10 @@ public: const std::vector& lockAndRead(); const std::vector getChangedPaths(); void addOrUpdatePath(const ProjectPath& path); - void removePath(const ProjectPath& path); void unlockAndDiscard(); - void unlockAndCommit(); + void unlockAndCommit(bool onlyUpdated=false); }; + IndexFile m_index; /** * @brief Internal packagePath() exception @@ -257,21 +324,21 @@ public: * @brief Return map populated with dataspecs targetable by this project interface * @return Platform map with name-string keys and enable-status values */ - const std::map& listDataSpecs(); + inline const CompiledSpecs& getDataSpecs() {return m_compiledSpecs;} /** * @brief Enable persistent user preference for particular spec string(s) * @param specs String(s) representing unique spec(s) from listDataSpecs * @return true on success */ - bool enableDataSpecs(const std::vector& specs); + bool enableDataSpecs(const std::vector& specs); /** * @brief Disable persistent user preference for particular spec string(s) * @param specs String(s) representing unique spec(s) from listDataSpecs * @return true on success */ - bool disableDataSpecs(const std::vector& specs); + bool disableDataSpecs(const std::vector& specs); /** * @brief Begin cook process for specified directory @@ -285,7 +352,7 @@ public: * feedback delivered via feedbackCb. */ bool cookPath(const ProjectPath& path, - std::function feedbackCb, + std::function feedbackCb, bool recursive=false); /** @@ -311,31 +378,6 @@ public: */ bool cleanPath(const ProjectPath& path, bool recursive=false); - /** - * @brief Nodegraph class for gathering dependency-resolved objects for packaging - */ - class PackageDepsgraph - { - public: - struct Node - { - enum - { - NODE_DATA, - NODE_GROUP - } type; - std::string path; - ObjectBase* projectObj; - Node* sub; - Node* next; - }; - private: - friend class Project; - std::vector m_nodes; - public: - const Node* getRootNode() const {return &m_nodes[0];} - }; - /** * @brief Constructs a full depsgraph of the project-subpath provided * @param path Subpath of project to root depsgraph at @@ -345,19 +387,6 @@ public: }; - -/** - * @brief Subclassed by dataspec entries to manage per-game aspects of the data pipeline - * - * The DataSpec class manages interfaces for unpackaging, cooking, and packaging - * of data for interacting with a specific system/game-engine. - */ -class IDataSpec -{ -public: - virtual Project::PackageDepsgraph packageData(); -}; - } } diff --git a/hecl/lib/ProjectPath.cpp b/hecl/lib/ProjectPath.cpp index b359df72d..9551808df 100644 --- a/hecl/lib/ProjectPath.cpp +++ b/hecl/lib/ProjectPath.cpp @@ -47,11 +47,10 @@ ProjectPath::ProjectPath(const ProjectRootPath& rootPath, const SystemString& pa /* Copies of the project root are permitted */ return; } - m_relPath = m_absPath.c_str() + ((ProjectPath&)rootPath).m_absPath.size(); - if (m_relPath[0] == _S('/')) - ++m_relPath; - if (m_relPath[0] == _S('\0')) - m_relPath = NULL; + SystemString::iterator beginit = m_absPath.begin() + ((ProjectPath&)rootPath).m_absPath.size(); + if (*beginit == _S('/')) + ++beginit; + m_relPath = SystemString(beginit, m_absPath.end()); std::hash hash_fn; m_hash = hash_fn(std::string(m_relPath)); @@ -81,7 +80,50 @@ ProjectPath::PathType ProjectPath::getPathType() const Time ProjectPath::getModtime() const { - + struct stat theStat; + time_t latestTime = 0; + if (std::regex_search(m_absPath, regGLOB)) + { + std::vector globReults; + getGlobResults(globReults); + for (SystemString& path : globReults) + { + if (!HECL::Stat(path.c_str(), &theStat)) + { + if (S_ISREG(theStat.st_mode) && theStat.st_mtime > latestTime) + latestTime = theStat.st_mtime; + } + } + } + if (!HECL::Stat(m_absPath.c_str(), &theStat)) + { + if (S_ISREG(theStat.st_mode)) + { + return Time(theStat.st_mtime); + } + else if (S_ISDIR(theStat.st_mode)) + { +#if _WIN32 +#else + DIR* dir = opendir(m_absPath.c_str()); + dirent* de; + while ((de = readdir(dir))) + { + if (de->d_name[0] == '.') + continue; + if (!HECL::Stat(de->d_name, &theStat)) + { + if (S_ISREG(theStat.st_mode) && theStat.st_mtime > latestTime) + latestTime = theStat.st_mtime; + } + } + closedir(dir); +#endif + return Time(latestTime); + } + } + throw HECL::Exception(_S("invalid path type")); + return Time(); } static void _recursiveGlob(std::vector& outPaths, @@ -155,4 +197,43 @@ void ProjectPath::getGlobResults(std::vector& outPaths) const _recursiveGlob(outPaths, 1, pathCompMatches, itStr, false); } +ProjectRootPath* SearchForProject(const SystemString& path) +{ + ProjectRootPath testRoot(path); + SystemString::const_iterator begin = testRoot.getAbsolutePath().begin(); + SystemString::const_iterator end = testRoot.getAbsolutePath().end(); + while (begin != end) + { + while (begin != end && *(end-1) != _S('/') && *(end-1) != _S('\\')) + --end; + if (begin == end) + break; + + SystemString testPath(begin, end); + SystemString testIndexPath = testPath + _S("/.hecl/index"); + struct stat theStat; + if (!HECL::Stat(testIndexPath.c_str(), &theStat)) + { + if (S_ISREG(theStat.st_mode)) + { + FILE* fp = HECL::Fopen(testIndexPath.c_str(), _S("rb")); + if (!fp) + continue; + char magic[4]; + size_t readSize = fread(magic, 1, 4, fp); + fclose(fp); + if (readSize != 4) + continue; + static const HECL::FourCC hecl("HECL"); + if (HECL::FourCC(magic) != hecl) + continue; + return new ProjectRootPath(testPath); + } + } + + --end; + } + return NULL; +} + } diff --git a/hecl/lib/database/Project.cpp b/hecl/lib/database/Project.cpp index 3c85c3bab..13082071d 100644 --- a/hecl/lib/database/Project.cpp +++ b/hecl/lib/database/Project.cpp @@ -176,6 +176,8 @@ const std::vector& Project::IndexFile::lockAndRead() return m_entryStore; m_lockedFile = HECL::Fopen(m_filepath.c_str(), _S("r+"), LWRITE); + m_maxPathLen = 0; + m_onlyUpdatedMaxPathLen = 0; SIndexHeader header; if (fread(&header, 1, sizeof(header), m_lockedFile) != sizeof(header)) @@ -208,6 +210,7 @@ const std::vector& Project::IndexFile::lockAndRead() } } delete[] pathBuf; + return m_entryStore; } const std::vector Project::IndexFile::getChangedPaths() @@ -217,12 +220,8 @@ const std::vector Project::IndexFile::getChangedPaths() std::vector retval; for (Project::IndexFile::Entry& ent : m_entryStore) - { - if (ent.m_removed) - continue; if (ent.m_lastModtime != ent.m_path.getModtime()) retval.push_back(&ent.m_path); - } return retval; } @@ -231,27 +230,20 @@ void Project::IndexFile::addOrUpdatePath(const ProjectPath& path) if (!m_lockedFile) throw HECL::Exception(_S("Project::IndexFile::lockAndRead not yet called")); + size_t pathLen = path.getRelativePath().size(); + if (pathLen > m_onlyUpdatedMaxPathLen) + m_onlyUpdatedMaxPathLen = pathLen; + std::unordered_map::iterator it = m_entryLookup.find(path); if (it == m_entryLookup.end()) { m_entryStore.push_back(Project::IndexFile::Entry(path, path.getModtime())); m_entryLookup[path] = &m_entryStore.back(); + m_entryStore.back().m_updated = true; return; } (*it).second->m_lastModtime = path.getModtime(); -} - -void Project::IndexFile::removePath(const ProjectPath& path) -{ - if (!m_lockedFile) - throw HECL::Exception(_S("Project::IndexFile::lockAndRead not yet called")); - - std::unordered_map::iterator it = m_entryLookup.find(path); - if (it != m_entryLookup.end()) - { - (*it).second->m_removed = true; - m_entryLookup.erase(it); - } + (*it).second->m_updated = true; } void Project::IndexFile::unlockAndDiscard() @@ -265,7 +257,7 @@ void Project::IndexFile::unlockAndDiscard() m_lockedFile = NULL; } -void Project::IndexFile::unlockAndCommit() +void Project::IndexFile::unlockAndCommit(bool onlyUpdated) { if (!m_lockedFile) throw HECL::Exception(_S("Project::IndexFile::lockAndRead not yet called")); @@ -279,20 +271,24 @@ void Project::IndexFile::unlockAndCommit() SIndexHeader header = { - HECL::FourCC("HECL"), - 1, (uint32_t)m_entryStore.size(), (uint32_t)m_maxPathLen + HECL::FourCC("HECL"), 1, + (uint32_t)(onlyUpdated ? m_updatedCount : m_entryStore.size()), + (uint32_t)(onlyUpdated ? m_onlyUpdatedMaxPathLen : m_maxPathLen) }; header.swapWithNative(); fwrite(&header, 1, sizeof(header), m_lockedFile); for (Project::IndexFile::Entry& ent : m_entryStore) { - uint64_t mt = ToBig(ent.m_lastModtime.getTs()); - fwrite(&mt, 1, 8, m_lockedFile); - size_t strLen = strlen(ent.m_path.getRelativePathUTF8()); - uint32_t strLenb = ToBig(strLen); - fwrite(&strLenb, 1, 4, m_lockedFile); - fwrite(ent.m_path.getRelativePathUTF8(), 1, strLen, m_lockedFile); + if (!onlyUpdated || ent.m_updated) + { + uint64_t mt = ToBig(ent.m_lastModtime.getTs()); + fwrite(&mt, 1, 8, m_lockedFile); + size_t strLen = ent.m_path.getRelativePathUTF8().size(); + uint32_t strLenb = ToBig(strLen); + fwrite(&strLenb, 1, 4, m_lockedFile); + fwrite(ent.m_path.getRelativePathUTF8().c_str(), 1, strLen, m_lockedFile); + } } m_entryLookup.clear(); @@ -306,7 +302,11 @@ void Project::IndexFile::unlockAndCommit() **********************************************/ Project::Project(const ProjectRootPath& rootPath) -: m_rootPath(rootPath) +: m_rootPath(rootPath), + m_specs(*this, _S("specs")), + m_paths(*this, _S("paths")), + m_groups(*this, _S("groups")), + m_index(*this) { /* Stat for existing project directory (must already exist) */ struct stat myStat; @@ -321,8 +321,6 @@ Project::Project(const ProjectRootPath& rootPath) HECL::MakeDir(m_rootPath.getAbsolutePath() + _S("/.hecl")); HECL::MakeDir(m_rootPath.getAbsolutePath() + _S("/.hecl/cooked")); HECL::MakeDir(m_rootPath.getAbsolutePath() + _S("/.hecl/config")); - - /* Create or open databases */ } void Project::registerLogger(FLogger logger) @@ -345,10 +343,6 @@ bool Project::removeGroup(const ProjectPath& path) { } -const std::map& Project::listDataSpecs() -{ -} - bool Project::enableDataSpecs(const std::vector& specs) { } @@ -371,7 +365,7 @@ bool Project::cleanPath(const ProjectPath& path, bool recursive) { } -Project::PackageDepsgraph Project::buildPackageDepsgraph(const ProjectPath& path) +PackageDepsgraph Project::buildPackageDepsgraph(const ProjectPath& path) { } diff --git a/hecl/lib/lib.pro b/hecl/lib/lib.pro index 86ab2635b..2433ff42b 100644 --- a/hecl/lib/lib.pro +++ b/hecl/lib/lib.pro @@ -9,7 +9,7 @@ unix:LIBS += -std=c++11 clang:QMAKE_CXXFLAGS += -stdlib=libc++ clang:LIBS += -stdlib=libc++ -lc++abi -INCLUDEPATH += $$PWD ../include ../extern +INCLUDEPATH += $$PWD ../include ../extern ../extern/Athena/include include (frontend/frontend.pri) include (backend/backend.pri)