From a6d28c2b2ba94dc4b2b1448613ce0331d11027ac Mon Sep 17 00:00:00 2001 From: Jack Andersen Date: Wed, 10 Jun 2015 13:34:14 -1000 Subject: [PATCH] added index and config file management --- hecl/driver/driver.pro | 3 - hecl/include/HECL.hpp | 95 +++++++++++-- hecl/include/HECLDatabase.hpp | 81 ++++++++--- hecl/lib/ProjectPath.cpp | 8 ++ hecl/lib/database/Project.cpp | 253 +++++++++++++++++++++++++++++----- hecl/lib/lib.pro | 1 + 6 files changed, 375 insertions(+), 66 deletions(-) diff --git a/hecl/driver/driver.pro b/hecl/driver/driver.pro index 42484a63e..f81f32755 100644 --- a/hecl/driver/driver.pro +++ b/hecl/driver/driver.pro @@ -17,9 +17,6 @@ LIBPATH += $$OUT_PWD/../lib \ LIBS += -lhecl -lhecl-dataspec -lhecl-blender -lblowfish -lpng -# Yay!! Athena IO -include(../extern/Athena/AthenaCore.pri) - SOURCES += \ $$PWD/main.cpp diff --git a/hecl/include/HECL.hpp b/hecl/include/HECL.hpp index d1415291d..e252941b8 100644 --- a/hecl/include/HECL.hpp +++ b/hecl/include/HECL.hpp @@ -8,8 +8,10 @@ char* win_realpath(const char* name, char* restrict resolved); #include #include #include +#include #endif +#include #include #include #include @@ -122,7 +124,13 @@ static inline SystemChar* Getcwd(SystemChar* buf, int maxlen) #endif } -static inline FILE* Fopen(const SystemChar* path, const SystemChar* mode) +enum FileLockType +{ + LNONE = 0, + LREAD, + LWRITE +}; +static inline FILE* Fopen(const SystemChar* path, const SystemChar* mode, FileLockType lock=LNONE) { #if HECL_UCS2 FILE* fp = wfopen(path, mode); @@ -132,6 +140,22 @@ static inline FILE* Fopen(const SystemChar* path, const SystemChar* mode) if (!fp) throw std::error_code(errno, std::system_category()); + if (lock) + { +#if _WIN32 + HANDLE fhandle = (HANDLE)fileno(fp); + OVERLAPPED ov = {}; + LockFileEx(fhandle, (lock == LWRITE) ? LOCKFILE_EXCLUSIVE_LOCK : 0, 0, 0, 1, &ov); +#else + struct flock lk = + { + (short)((lock == LREAD) ? F_RDLCK : F_WRLCK), + SEEK_SET, 0, 0, 0 + }; + fcntl(fileno(fp), F_SETLK, &lk); +#endif + } + return fp; } @@ -196,7 +220,7 @@ typedef std::function FLogger; * while fitting comfortably in a 32-bit word. HECL uses a four-char array * to remain endian-independent. */ -class FourCC +class FourCC final { union { @@ -210,6 +234,8 @@ public: : num(*(uint32_t*)name) {} inline bool operator==(FourCC& other) {return num == other.num;} inline bool operator!=(FourCC& other) {return num != other.num;} + inline bool operator==(const char* other) {return num == *(uint32_t*)other;} + inline bool operator!=(const char* other) {return num != *(uint32_t*)other;} inline std::string toString() {return std::string(fcc, 4);} }; @@ -219,7 +245,7 @@ public: * Hashes are used within HECL to avoid redundant storage of objects; * providing a rapid mechanism to compare for equality. */ -class Hash +class Hash final { int64_t hash; public: @@ -229,12 +255,34 @@ public: : hash(Blowfish_hash(str.data(), str.size())) {} Hash(int64_t hashin) : hash(hashin) {} - inline bool operator==(Hash& other) {return hash == other.hash;} - inline bool operator!=(Hash& other) {return hash != other.hash;} - inline bool operator<(Hash& other) {return hash < other.hash;} - inline bool operator>(Hash& other) {return hash > other.hash;} - inline bool operator<=(Hash& other) {return hash <= other.hash;} - inline bool operator>=(Hash& other) {return hash >= other.hash;} + Hash(const Hash& other) {hash = other.hash;} + inline Hash& operator=(const Hash& other) {hash = other.hash; return *this;} + inline bool operator==(const Hash& other) const {return hash == other.hash;} + inline bool operator!=(const Hash& other) const {return hash != other.hash;} + inline bool operator<(const Hash& other) const {return hash < other.hash;} + inline bool operator>(const Hash& other) const {return hash > other.hash;} + inline bool operator<=(const Hash& other) const {return hash <= other.hash;} + inline bool operator>=(const Hash& other) const {return hash >= other.hash;} +}; + +/** + * @brief Timestamp representation used for comparing modtimes of cooked resources + */ +class Time final +{ + uint64_t ts; +public: + Time() : ts(time(NULL)) {} + Time(uint64_t ti) : ts(ti) {} + Time(const Time& other) {ts = other.ts;} + inline uint64_t getTs() const {return ts;} + inline Time& operator=(const Time& other) {ts = other.ts; return *this;} + inline bool operator==(const Time& other) const {return ts == other.ts;} + inline bool operator!=(const Time& other) const {return ts != other.ts;} + inline bool operator<(const Time& other) const {return ts < other.ts;} + inline bool operator>(const Time& other) const {return ts > other.ts;} + inline bool operator<=(const Time& other) const {return ts <= other.ts;} + inline bool operator>=(const Time& other) const {return ts >= other.ts;} }; /** @@ -255,6 +303,7 @@ class ProjectPath protected: SystemString m_absPath; const SystemChar* m_relPath = NULL; + size_t m_hash = 0; #if HECL_UCS2 std::string m_utf8AbsPath; const char* m_utf8RelPath; @@ -331,11 +380,30 @@ public: */ PathType getPathType() const; + /** + * @brief Get time of last modification with special behaviors for directories and glob-paths + * @return Time object representing entity's time of last modification + * + * Regular files simply return their modtime as queried from the OS + * Directories return the latest modtime of all first-level regular files + * Glob-paths return the latest modtime of all matched regular files + */ + Time getModtime() const; + /** * @brief Insert glob matches into existing vector * @param outPaths Vector to add matches to (will not erase existing contents) */ void getGlobResults(std::vector& outPaths) const; + + /** + * @brief C++11 compatible runtime hash (NOT USED IN PACKAGES!!) + * @return System-specific hash value + */ + inline size_t hash() const {return m_hash;} + inline bool operator==(const ProjectPath& other) const {return m_hash == other.m_hash;} + inline bool operator!=(const ProjectPath& other) const {return m_hash != other.m_hash;} + }; /** @@ -434,4 +502,13 @@ static inline uint64_t ToBig(uint64_t val) {return val;} } +namespace std +{ +template <> struct hash +{ + size_t operator()(const HECL::ProjectPath& val) const noexcept + {return val.hash();} +}; +} + #endif // HECL_HPP diff --git a/hecl/include/HECLDatabase.hpp b/hecl/include/HECLDatabase.hpp index 97445f6da..98f2bfc5e 100644 --- a/hecl/include/HECLDatabase.hpp +++ b/hecl/include/HECLDatabase.hpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -112,17 +113,61 @@ class Project public: Project(const HECL::ProjectRootPath& rootPath); + /** + * @brief Configuration file handle + * + * Holds a path to a line-delimited textual configuration file; + * opening a locked handle for read/write transactions + */ class ConfigFile { - const Project& m_project; - const SystemString& m_name; SystemString m_filepath; + std::vector m_lines; + FILE* m_lockedFile = NULL; public: ConfigFile(const Project& project, const SystemString& name); - std::vector readLines(); + const std::vector& lockAndRead(); void addLine(const std::string& line); void removeLine(const std::string& refLine); bool checkForLine(const std::string& refLine); + void unlockAndDiscard(); + void unlockAndCommit(); + }; + + /** + * @brief Index file handle + * + * Holds a path to a binary index file; + * opening a locked handle for read/write transactions + */ + class IndexFile + { + SystemString m_filepath; + const Project& m_project; + size_t m_maxPathLen = 0; + FILE* m_lockedFile = NULL; + public: + class Entry + { + friend class IndexFile; + ProjectPath m_path; + HECL::Time m_lastModtime; + bool m_removed = false; + Entry(const ProjectPath& path, const HECL::Time& lastModtime) + : m_path(path), m_lastModtime(lastModtime) {} + Entry(const ProjectPath& path); + }; + private: + std::vector m_entryStore; + std::unordered_map m_entryLookup; + public: + IndexFile(const Project& project); + const std::vector& lockAndRead(); + const std::vector getChangedPaths(); + void addOrUpdatePath(const ProjectPath& path); + void removePath(const ProjectPath& path); + void unlockAndDiscard(); + void unlockAndCommit(); }; /** @@ -154,7 +199,7 @@ public: * * If this method is never called, all project operations will run silently. */ - virtual void registerLogger(HECL::FLogger logger); + void registerLogger(HECL::FLogger logger); /** * @brief Get the path of the project's root-directory @@ -163,7 +208,7 @@ public: * * Self explanatory */ - virtual const ProjectRootPath& getProjectRootPath(bool absolute=false) const; + inline const ProjectRootPath& getProjectRootPath() const {return m_rootPath;} /** * @brief Add given file(s) to the database @@ -172,7 +217,7 @@ public: * * This method blocks while object hashing takes place */ - virtual bool addPaths(const std::vector& paths); + bool addPaths(const std::vector& paths); /** * @brief Remove a given file or file-pattern from the database @@ -183,7 +228,7 @@ public: * This method will not delete actual working files from the project * directory. It will delete associated cooked objects though. */ - virtual bool removePaths(const std::vector& paths, bool recursive=false); + bool removePaths(const std::vector& paths, bool recursive=false); /** * @brief Register a working sub-directory as a Dependency Group @@ -199,34 +244,34 @@ public: * This contiguous storage makes for optimal loading from slow block-devices * like optical drives. */ - virtual bool addGroup(const ProjectPath& path); + bool addGroup(const ProjectPath& path); /** * @brief Unregister a working sub-directory as a dependency group * @param path directory to unregister as Dependency Group * @return true on success */ - virtual bool removeGroup(const ProjectPath& path); + bool removeGroup(const ProjectPath& path); /** * @brief Return map populated with dataspecs targetable by this project interface * @return Platform map with name-string keys and enable-status values */ - virtual const std::map& listDataSpecs(); + const std::map& listDataSpecs(); /** * @brief Enable persistent user preference for particular spec string(s) * @param specs String(s) representing unique spec(s) from listDataSpecs * @return true on success */ - virtual bool enableDataSpecs(const std::vector& specs); + bool enableDataSpecs(const std::vector& specs); /** * @brief Disable persistent user preference for particular spec string(s) * @param specs String(s) representing unique spec(s) from listDataSpecs * @return true on success */ - virtual bool disableDataSpecs(const std::vector& specs); + bool disableDataSpecs(const std::vector& specs); /** * @brief Begin cook process for specified directory @@ -239,9 +284,9 @@ public: * This method blocks execution during the procedure, with periodic * feedback delivered via feedbackCb. */ - virtual bool cookPath(const ProjectPath& path, - std::function feedbackCb, - bool recursive=false); + bool cookPath(const ProjectPath& path, + std::function feedbackCb, + bool recursive=false); /** * @brief Interrupts a cook in progress (call from SIGINT handler) @@ -253,7 +298,7 @@ public: * Note that this method returns immediately; the resumed cookPath() * call will return as quickly as possible. */ - virtual void interruptCook(); + void interruptCook(); /** * @brief Delete cooked objects for directory @@ -264,7 +309,7 @@ public: * Developers understand how useful 'clean' is. While ideally not required, * it's useful for verifying that a rebuild from ground-up is doable. */ - virtual bool cleanPath(const ProjectPath& path, bool recursive=false); + bool cleanPath(const ProjectPath& path, bool recursive=false); /** * @brief Nodegraph class for gathering dependency-resolved objects for packaging @@ -296,7 +341,7 @@ public: * @param path Subpath of project to root depsgraph at * @return Populated depsgraph ready to traverse */ - virtual PackageDepsgraph buildPackageDepsgraph(const ProjectPath& path); + PackageDepsgraph buildPackageDepsgraph(const ProjectPath& path); }; diff --git a/hecl/lib/ProjectPath.cpp b/hecl/lib/ProjectPath.cpp index 87f2a9628..b359df72d 100644 --- a/hecl/lib/ProjectPath.cpp +++ b/hecl/lib/ProjectPath.cpp @@ -53,6 +53,9 @@ ProjectPath::ProjectPath(const ProjectRootPath& rootPath, const SystemString& pa if (m_relPath[0] == _S('\0')) m_relPath = NULL; + std::hash hash_fn; + m_hash = hash_fn(std::string(m_relPath)); + #if HECL_UCS2 m_utf8AbsPath = WideToUTF8(m_absPath); m_utf8RelPath = m_utf8AbsPath.c_str() + ((ProjectPath&)rootPath).m_utf8AbsPath.size(); @@ -76,6 +79,11 @@ ProjectPath::PathType ProjectPath::getPathType() const #endif } +Time ProjectPath::getModtime() const +{ + +} + static void _recursiveGlob(std::vector& outPaths, size_t level, const SystemRegexMatch& pathCompMatches, diff --git a/hecl/lib/database/Project.cpp b/hecl/lib/database/Project.cpp index bbeb4f2eb..3c85c3bab 100644 --- a/hecl/lib/database/Project.cpp +++ b/hecl/lib/database/Project.cpp @@ -4,6 +4,11 @@ #include #include +#if _WIN32 +#else +#include +#endif + #include "HECLDatabase.hpp" namespace HECL @@ -17,7 +22,7 @@ namespace Database static inline bool CheckNewLineAdvance(std::string::const_iterator& it) { - if (*it == '\n' || *it == '\0') + if (*it == '\n') { it += 1; return true; @@ -29,31 +34,34 @@ static inline bool CheckNewLineAdvance(std::string::const_iterator& it) it += 2; return true; } + it += 1; + return true; } return false; } Project::ConfigFile::ConfigFile(const Project& project, const SystemString& name) -: m_project(project), m_name(name) { m_filepath = project.m_rootPath.getAbsolutePath() + _S("/.hecl/config/") + name; } -std::vector Project::ConfigFile::readLines() +const std::vector& Project::ConfigFile::lockAndRead() { - FILE* fp = HECL::Fopen(m_filepath.c_str(), _S("r")); + if (m_lockedFile) + return m_lines; + + m_lockedFile = HECL::Fopen(m_filepath.c_str(), _S("r+"), LWRITE); std::string mainString; char readBuf[1024]; size_t readSz; - while ((readSz = fread(readBuf, 1, 1024, fp))) + while ((readSz = fread(readBuf, 1, 1024, m_lockedFile))) mainString += std::string(readBuf, readSz); - fclose(fp); std::string::const_iterator begin = mainString.begin(); std::string::const_iterator end = mainString.begin(); - std::vector retval; + m_lines.clear(); while (end != mainString.end()) { std::string::const_iterator origEnd = end; @@ -62,53 +70,44 @@ std::vector Project::ConfigFile::readLines() else if (CheckNewLineAdvance(end)) { if (begin != origEnd) - retval.push_back(std::string(begin, origEnd)); + m_lines.push_back(std::string(begin, origEnd)); begin = end; continue; } ++end; } if (begin != end) - retval.push_back(std::string(begin, end)); + m_lines.push_back(std::string(begin, end)); - return retval; + return m_lines; } void Project::ConfigFile::addLine(const std::string& line) { - std::vector curLines = readLines(); - - FILE* fp = HECL::Fopen(m_filepath.c_str(), _S("w")); - for (std::string& line : curLines) - { - fwrite(line.data(), 1, line.length(), fp); - fwrite("\n", 1, 1, fp); - } - fwrite(line.data(), 1, line.length(), fp); - fwrite("\n", 1, 1, fp); - fclose(fp); + if (!checkForLine(line)) + m_lines.push_back(line); } void Project::ConfigFile::removeLine(const std::string& refLine) { - std::vector curLines = readLines(); + if (!m_lockedFile) + throw HECL::Exception(_S("Project::ConfigFile::lockAndRead not yet called")); - FILE* fp = HECL::Fopen(m_filepath.c_str(), _S("w")); - for (std::string& line : curLines) + for (std::vector::const_iterator it=m_lines.begin(); + it != m_lines.end(); + ++it) { - if (line.compare(refLine)) - { - fwrite(line.data(), 1, line.length(), fp); - fwrite("\n", 1, 1, fp); - } + if (!(*it).compare(refLine)) + it = m_lines.erase(it); } - fclose(fp); } bool Project::ConfigFile::checkForLine(const std::string& refLine) { - std::vector curLines = readLines(); - for (std::string& line : curLines) + if (!m_lockedFile) + throw HECL::Exception(_S("Project::ConfigFile::lockAndRead not yet called")); + + for (const std::string& line : m_lines) { if (!line.compare(refLine)) return true; @@ -116,6 +115,192 @@ bool Project::ConfigFile::checkForLine(const std::string& refLine) return false; } +void Project::ConfigFile::unlockAndDiscard() +{ + if (!m_lockedFile) + throw HECL::Exception(_S("Project::ConfigFile::lockAndRead not yet called")); + + m_lines.clear(); + fclose(m_lockedFile); + m_lockedFile = NULL; +} + +void Project::ConfigFile::unlockAndCommit() +{ + if (!m_lockedFile) + throw HECL::Exception(_S("Project::ConfigFile::lockAndRead not yet called")); + + fseek(m_lockedFile, 0, SEEK_SET); +#if _WIN32 + SetEndOfFile((HANDLE)fileno(m_lockedFile)); +#else + ftruncate(fileno(m_lockedFile), 0); +#endif + for (const std::string& line : m_lines) + { + fwrite(line.c_str(), 1, line.size(), m_lockedFile); + fwrite("\n", 1, 1, m_lockedFile); + } + m_lines.clear(); + fclose(m_lockedFile); + m_lockedFile = NULL; +} + +/********************************************** + * Project::IndexFile + **********************************************/ + +struct SIndexHeader +{ + HECL::FourCC magic; + uint32_t version; + uint32_t entryCount; + uint32_t maxPathLen; + void swapWithNative() + { + version = ToBig(version); + entryCount = ToBig(entryCount); + maxPathLen = ToBig(maxPathLen); + } +}; + +Project::IndexFile::IndexFile(const Project& project) +: m_project(project) +{ + m_filepath = project.m_rootPath.getAbsolutePath() + _S("/.hecl/index"); +} + +const std::vector& Project::IndexFile::lockAndRead() +{ + if (m_lockedFile) + return m_entryStore; + + m_lockedFile = HECL::Fopen(m_filepath.c_str(), _S("r+"), LWRITE); + + SIndexHeader header; + if (fread(&header, 1, sizeof(header), m_lockedFile) != sizeof(header)) + return m_entryStore; + header.swapWithNative(); + if (header.magic != "HECL") + throw HECL::Exception(_S("unrecognized HECL index")); + if (header.version != 1) + throw HECL::Exception(_S("unrecognized HECL version")); + + char* pathBuf = new char[header.maxPathLen]; + for (uint32_t e=0 ; e m_maxPathLen) + m_maxPathLen = strLen; + fread(pathBuf, 1, strLen, m_lockedFile); + std::string pathStr(pathBuf, strLen); + SystemStringView pathView(pathStr); + ProjectPath path(m_project.getProjectRootPath(), pathView.sys_str()); + if (m_entryLookup.find(path) == m_entryLookup.end()) + { + m_entryStore.push_back(Project::IndexFile::Entry(path, mt)); + m_entryLookup[path] = &m_entryStore.back(); + } + } + delete[] pathBuf; +} + +const std::vector Project::IndexFile::getChangedPaths() +{ + if (!m_lockedFile) + throw HECL::Exception(_S("Project::IndexFile::lockAndRead not yet called")); + + std::vector retval; + for (Project::IndexFile::Entry& ent : m_entryStore) + { + if (ent.m_removed) + continue; + if (ent.m_lastModtime != ent.m_path.getModtime()) + retval.push_back(&ent.m_path); + } + return retval; +} + +void Project::IndexFile::addOrUpdatePath(const ProjectPath& path) +{ + if (!m_lockedFile) + throw HECL::Exception(_S("Project::IndexFile::lockAndRead not yet called")); + + std::unordered_map::iterator it = m_entryLookup.find(path); + if (it == m_entryLookup.end()) + { + m_entryStore.push_back(Project::IndexFile::Entry(path, path.getModtime())); + m_entryLookup[path] = &m_entryStore.back(); + return; + } + (*it).second->m_lastModtime = path.getModtime(); +} + +void Project::IndexFile::removePath(const ProjectPath& path) +{ + if (!m_lockedFile) + throw HECL::Exception(_S("Project::IndexFile::lockAndRead not yet called")); + + std::unordered_map::iterator it = m_entryLookup.find(path); + if (it != m_entryLookup.end()) + { + (*it).second->m_removed = true; + m_entryLookup.erase(it); + } +} + +void Project::IndexFile::unlockAndDiscard() +{ + if (!m_lockedFile) + throw HECL::Exception(_S("Project::IndexFile::lockAndRead not yet called")); + + m_entryLookup.clear(); + m_entryStore.clear(); + fclose(m_lockedFile); + m_lockedFile = NULL; +} + +void Project::IndexFile::unlockAndCommit() +{ + if (!m_lockedFile) + throw HECL::Exception(_S("Project::IndexFile::lockAndRead not yet called")); + + fseek(m_lockedFile, 0, SEEK_SET); +#if _WIN32 + SetEndOfFile((HANDLE)fileno(m_lockedFile)); +#else + ftruncate(fileno(m_lockedFile), 0); +#endif + + SIndexHeader header = + { + HECL::FourCC("HECL"), + 1, (uint32_t)m_entryStore.size(), (uint32_t)m_maxPathLen + }; + header.swapWithNative(); + fwrite(&header, 1, sizeof(header), m_lockedFile); + + for (Project::IndexFile::Entry& ent : m_entryStore) + { + uint64_t mt = ToBig(ent.m_lastModtime.getTs()); + fwrite(&mt, 1, 8, m_lockedFile); + size_t strLen = strlen(ent.m_path.getRelativePathUTF8()); + uint32_t strLenb = ToBig(strLen); + fwrite(&strLenb, 1, 4, m_lockedFile); + fwrite(ent.m_path.getRelativePathUTF8(), 1, strLen, m_lockedFile); + } + + m_entryLookup.clear(); + m_entryStore.clear(); + fclose(m_lockedFile); + m_lockedFile = NULL; +} + /********************************************** * Project **********************************************/ @@ -144,10 +329,6 @@ void Project::registerLogger(FLogger logger) { } -const ProjectRootPath& Project::getProjectRootPath(bool absolute) const -{ -} - bool Project::addPaths(const std::vector& paths) { } diff --git a/hecl/lib/lib.pro b/hecl/lib/lib.pro index f3a7d8c58..86ab2635b 100644 --- a/hecl/lib/lib.pro +++ b/hecl/lib/lib.pro @@ -15,6 +15,7 @@ include (frontend/frontend.pri) include (backend/backend.pri) include (database/database.pri) include (runtime/runtime.pri) +include(../extern/Athena/AthenaCore.pri) SOURCES += \ HECL.cpp \