tool tweaks

hecl/driver/ToolInit.hpp

@@ -29,7 +29,7 @@ public:
             return;
         }
 
-        HECL::SystemString testPath = *dir + _S("/.hecl/index");
+        HECL::SystemString testPath = *dir + _S("/.hecl/beacon");
         if (!HECL::Stat(testPath.c_str(), &theStat))
         {
             throw HECL::Exception(_S("project already exists at '") + *dir + _S("'"));

hecl/driver/driver.pro

@@ -7,7 +7,7 @@ unix:LIBS += -std=c++11
 clang:QMAKE_CXXFLAGS += -stdlib=libc++
 clang:LIBS += -stdlib=libc++ -lc++abi
 
-INCLUDEPATH += ../include ../extern/Athena/include
+INCLUDEPATH += ../include ../extern/Athena/include ../extern
 
 LIBPATH += $$OUT_PWD/../lib \
     $$OUT_PWD/../dataspec \

hecl/driver/main.cpp

@@ -152,7 +152,7 @@ int main(int argc, const char** argv)
     }
 
     /* Attempt to find hecl project */
-    std::unique_ptr<HECL::ProjectRootPath> rootPath(HECL::SearchForProject(info.cwd));
+    std::unique_ptr<HECL::ProjectRootPath> rootPath = HECL::SearchForProject(info.cwd);
     std::unique_ptr<HECL::Database::Project> project;
     if (rootPath.get())
     {

hecl/include/HECL.hpp

@@ -431,7 +431,7 @@ public:
  * @param path absolute or relative file path to search from
  * @return Newly-constructed root path or NULL if not found
  */
-ProjectRootPath* SearchForProject(const SystemString& path);
+std::unique_ptr<ProjectRootPath> SearchForProject(const SystemString& path);
 
 
 /* Type-sensitive byte swappers */
@@ -482,33 +482,33 @@ static inline T bswap64(T val)
 
 
 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
-static inline int16_t ToBig(int16_t val) {return bswap16(val);}
+static inline int16_t SBig(int16_t val) {return bswap16(val);}
-static inline uint16_t ToBig(uint16_t val) {return bswap16(val);}
+static inline uint16_t SBig(uint16_t val) {return bswap16(val);}
-static inline int32_t ToBig(int32_t val) {return bswap32(val);}
+static inline int32_t SBig(int32_t val) {return bswap32(val);}
-static inline uint32_t ToBig(uint32_t val) {return bswap32(val);}
+static inline uint32_t SBig(uint32_t val) {return bswap32(val);}
-static inline int64_t ToBig(int64_t val) {return bswap64(val);}
+static inline int64_t SBig(int64_t val) {return bswap64(val);}
-static inline uint64_t ToBig(uint64_t val) {return bswap64(val);}
+static inline uint64_t SBig(uint64_t val) {return bswap64(val);}
 
-static inline int16_t ToLittle(int16_t val) {return val;}
+static inline int16_t SLittle(int16_t val) {return val;}
-static inline uint16_t ToLittle(uint16_t val) {return val;}
+static inline uint16_t SLittle(uint16_t val) {return val;}
-static inline int32_t ToLittle(int32_t val) {return val;}
+static inline int32_t SLittle(int32_t val) {return val;}
-static inline uint32_t ToLittle(uint32_t val) {return val;}
+static inline uint32_t SLittle(uint32_t val) {return val;}
-static inline int64_t ToLittle(int64_t val) {return val;}
+static inline int64_t SLittle(int64_t val) {return val;}
-static inline uint64_t ToLittle(uint64_t val) {return val;}
+static inline uint64_t SLittle(uint64_t val) {return val;}
 #else
-static inline int16_t ToLittle(int16_t val) {return bswap16(val);}
+static inline int16_t SLittle(int16_t val) {return bswap16(val);}
-static inline uint16_t ToLittle(uint16_t val) {return bswap16(val);}
+static inline uint16_t SLittle(uint16_t val) {return bswap16(val);}
-static inline int32_t ToLittle(int32_t val) {return bswap32(val);}
+static inline int32_t SLittle(int32_t val) {return bswap32(val);}
-static inline uint32_t ToLittle(uint32_t val) {return bswap32(val);}
+static inline uint32_t SLittle(uint32_t val) {return bswap32(val);}
-static inline int64_t ToLittle(int64_t val) {return bswap64(val);}
+static inline int64_t SLittle(int64_t val) {return bswap64(val);}
-static inline uint64_t ToLittle(uint64_t val) {return bswap64(val);}
+static inline uint64_t SLittle(uint64_t val) {return bswap64(val);}
 
-static inline int16_t ToBig(int16_t val) {return val;}
+static inline int16_t SBig(int16_t val) {return val;}
-static inline uint16_t ToBig(uint16_t val) {return val;}
+static inline uint16_t SBig(uint16_t val) {return val;}
-static inline int32_t ToBig(int32_t val) {return val;}
+static inline int32_t SBig(int32_t val) {return val;}
-static inline uint32_t ToBig(uint32_t val) {return val;}
+static inline uint32_t SBig(uint32_t val) {return val;}
-static inline int64_t ToBig(int64_t val) {return val;}
+static inline int64_t SBig(int64_t val) {return val;}
-static inline uint64_t ToBig(uint64_t val) {return val;}
+static inline uint64_t SBig(uint64_t val) {return val;}
 #endif
 
 }

hecl/include/HECLDatabase.hpp

@@ -201,44 +201,6 @@ public:
     ConfigFile m_paths;
     ConfigFile m_groups;
 
-    /**
-     * @brief Index file handle
-     *
-     * Holds a path to a binary index file;
-     * opening a locked handle for read/write transactions
-     */
-    class IndexFile
-    {
-        SystemString m_filepath;
-        const Project& m_project;
-        size_t m_maxPathLen = 0;
-        size_t m_onlyUpdatedMaxPathLen = 0;
-        FILE* m_lockedFile = NULL;
-    public:
-        class Entry
-        {
-            friend class IndexFile;
-            ProjectPath m_path;
-            HECL::Time m_lastModtime;
-            bool m_updated = false;
-            Entry(const ProjectPath& path, const HECL::Time& lastModtime)
-            : m_path(path), m_lastModtime(lastModtime) {}
-            Entry(const ProjectPath& path);
-        };
-    private:
-        size_t m_updatedCount = 0;
-        std::vector<Entry> m_entryStore;
-        std::unordered_map<ProjectPath, Entry*> m_entryLookup;
-    public:
-        IndexFile(const Project& project);
-        const std::vector<Entry>& lockAndRead();
-        const std::vector<ProjectPath*> getChangedPaths();
-        void addOrUpdatePath(const ProjectPath& path);
-        void unlockAndDiscard();
-        bool unlockAndCommit(bool onlyUpdated=false);
-    };
-    IndexFile m_index;
-
     /**
      * @brief Internal packagePath() exception
      *
@@ -322,6 +284,14 @@ public:
      */
     bool removeGroup(const ProjectPath& path);
 
+    /**
+     * @brief Re-reads the data store holding user's spec preferences
+     *
+     * Call periodically in a long-term use of the HECL::Database::Project class.
+     * Install filesystem event-hooks if possible.
+     */
+    void rescanDataSpecs();
+
     /**
      * @brief Return map populated with dataspecs targetable by this project interface
      * @return Platform map with name-string keys and enable-status values

hecl/lib/ProjectPath.cpp

@@ -197,18 +197,13 @@ void ProjectPath::getGlobResults(std::vector<SystemString>& outPaths) const
         _recursiveGlob(outPaths, 1, pathCompMatches, itStr, false);
 }
 
-ProjectRootPath* SearchForProject(const SystemString& path)
+std::unique_ptr<ProjectRootPath> SearchForProject(const SystemString& path)
 {
     ProjectRootPath testRoot(path);
     SystemString::const_iterator begin = testRoot.getAbsolutePath().begin();
     SystemString::const_iterator end = testRoot.getAbsolutePath().end();
     while (begin != end)
     {
-        while (begin != end && *(end-1) != _S('/') && *(end-1) != _S('\\'))
-            --end;
-        if (begin == end)
-            break;
-
         SystemString testPath(begin, end);
         SystemString testIndexPath = testPath + _S("/.hecl/index");
         struct stat theStat;
@@ -227,13 +222,15 @@ ProjectRootPath* SearchForProject(const SystemString& path)
                 static const HECL::FourCC hecl("HECL");
                 if (HECL::FourCC(magic) != hecl)
                     continue;
-                return new ProjectRootPath(testPath);
+                return std::unique_ptr<ProjectRootPath>(new ProjectRootPath(testPath));
             }
         }
 
+        while (begin != end && *(end-1) != _S('/') && *(end-1) != _S('\\'))
+            --end;
         --end;
     }
-    return NULL;
+    return std::unique_ptr<ProjectRootPath>();
 }
 
 }

hecl/lib/database/Project.cpp

@@ -162,183 +162,6 @@ bool Project::ConfigFile::unlockAndCommit()
     }
 }
 
-/**********************************************
- * Project::IndexFile
- **********************************************/
-
-struct SIndexHeader
-{
-    HECL::FourCC magic;
-    uint32_t version;
-    uint32_t entryCount;
-    uint32_t maxPathLen;
-    void swapWithNative()
-    {
-        version = ToBig(version);
-        entryCount = ToBig(entryCount);
-        maxPathLen = ToBig(maxPathLen);
-    }
-};
-
-Project::IndexFile::IndexFile(const Project& project)
-: m_project(project)
-{
-    m_filepath = project.m_rootPath.getAbsolutePath() + _S("/.hecl/index");
-}
-
-const std::vector<Project::IndexFile::Entry>& Project::IndexFile::lockAndRead()
-{
-    if (m_lockedFile)
-        return m_entryStore;
-
-    /* Open file and begin lock cycle */
-    m_lockedFile = HECL::Fopen(m_filepath.c_str(), _S("a+b"), LWRITE);
-    m_maxPathLen = 0;
-    m_onlyUpdatedMaxPathLen = 0;
-
-    /* Read index header */
-    SIndexHeader header;
-    if (fread(&header, 1, sizeof(header), m_lockedFile) != sizeof(header))
-        return m_entryStore; /* Not yet written, this commit will take care of it */
-    header.swapWithNative();
-    if (header.magic != "HECL")
-        throw HECL::Exception(_S("unrecognized HECL index"));
-    if (header.version != 1)
-        throw HECL::Exception(_S("unrecognized HECL version"));
-
-    /* Iterate existing index entries */
-    char* pathBuf = new char[header.maxPathLen];
-    for (uint32_t e=0 ; e<header.entryCount ; ++e)
-    {
-        uint64_t mt;
-        fread(&mt, 1, 8, m_lockedFile);
-        mt = ToBig(mt);
-        uint32_t strLen;
-        fread(&strLen, 1, 4, m_lockedFile);
-        strLen = ToBig(strLen);
-        if (strLen > m_maxPathLen)
-            m_maxPathLen = strLen;
-        fread(pathBuf, 1, strLen, m_lockedFile);
-        std::string pathStr(pathBuf, strLen);
-        SystemStringView pathView(pathStr);
-        ProjectPath path(m_project.getProjectRootPath(), pathView.sys_str());
-        if (m_entryLookup.find(path) == m_entryLookup.end())
-        {
-            m_entryStore.push_back(Entry(path, mt));
-            m_entryLookup[path] = &m_entryStore.back();
-        }
-    }
-    delete[] pathBuf;
-    return m_entryStore;
-}
-
-const std::vector<ProjectPath*> Project::IndexFile::getChangedPaths()
-{
-    if (!m_lockedFile)
-        throw HECL::Exception(_S("Project::IndexFile::lockAndRead not yet called"));
-
-    std::vector<ProjectPath*> retval;
-    for (Entry& ent : m_entryStore)
-        if (ent.m_lastModtime != ent.m_path.getModtime())
-            retval.push_back(&ent.m_path);
-    return retval;
-}
-
-void Project::IndexFile::addOrUpdatePath(const ProjectPath& path)
-{
-    if (!m_lockedFile)
-        throw HECL::Exception(_S("Project::IndexFile::lockAndRead not yet called"));
-
-    size_t pathLen = path.getRelativePath().size();
-    if (pathLen > m_onlyUpdatedMaxPathLen)
-        m_onlyUpdatedMaxPathLen = pathLen;
-
-    std::unordered_map<ProjectPath, Entry*>::iterator it = m_entryLookup.find(path);
-    if (it == m_entryLookup.end())
-    {
-        m_entryStore.push_back(Entry(path, path.getModtime()));
-        m_entryLookup[path] = &m_entryStore.back();
-        m_entryStore.back().m_updated = true;
-        return;
-    }
-    (*it).second->m_lastModtime = path.getModtime();
-    (*it).second->m_updated = true;
-}
-
-void Project::IndexFile::unlockAndDiscard()
-{
-    if (!m_lockedFile)
-        throw HECL::Exception(_S("Project::IndexFile::lockAndRead not yet called"));
-
-    m_entryLookup.clear();
-    m_entryStore.clear();
-    fclose(m_lockedFile);
-    m_lockedFile = NULL;
-}
-
-bool Project::IndexFile::unlockAndCommit(bool onlyUpdated)
-{
-    if (!m_lockedFile)
-        throw HECL::Exception(_S("Project::IndexFile::lockAndRead not yet called"));
-
-    SystemString newPath = m_filepath + _S(".part");
-    FILE* newFile = HECL::Fopen(newPath.c_str(), _S("wb"), LWRITE);
-    SIndexHeader header =
-    {
-        HECL::FourCC("HECL"), 1,
-        (uint32_t)(onlyUpdated ? m_updatedCount : m_entryStore.size()),
-        (uint32_t)(onlyUpdated ? m_onlyUpdatedMaxPathLen : m_maxPathLen)
-    };
-    header.swapWithNative();
-    bool fail = false;
-    if (fwrite(&header, 1, sizeof(header), newFile) != sizeof(header))
-        fail = true;
-
-    if (!fail)
-    {
-        for (Entry& ent : m_entryStore)
-        {
-            if (!onlyUpdated || ent.m_updated)
-            {
-                uint64_t mt = ToBig(ent.m_lastModtime.getTs());
-                if (fwrite(&mt, 1, 8, newFile) != 8)
-                {
-                    fail = true;
-                    break;
-                }
-                size_t strLen = ent.m_path.getRelativePathUTF8().size();
-                uint32_t strLenb = ToBig(strLen);
-                if (fwrite(&strLenb, 1, 4, newFile) != 4)
-                {
-                    fail = true;
-                    break;
-                }
-                if (fwrite(ent.m_path.getRelativePathUTF8().c_str(), 1, strLen, newFile) != strLen)
-                {
-                    fail = true;
-                    break;
-                }
-            }
-        }
-    }
-
-    m_entryLookup.clear();
-    m_entryStore.clear();
-    fclose(newFile);
-    fclose(m_lockedFile);
-    m_lockedFile = NULL;
-    if (fail)
-    {
-        unlink(newPath.c_str());
-        return false;
-    }
-    else
-    {
-        rename(newPath.c_str(), m_filepath.c_str());
-        return true;
-    }
-}
-
 /**********************************************
  * Project
  **********************************************/
@@ -347,8 +170,7 @@ Project::Project(const ProjectRootPath& rootPath)
 : m_rootPath(rootPath),
   m_specs(*this, _S("specs")),
   m_paths(*this, _S("paths")),
-  m_groups(*this, _S("groups")),
+  m_groups(*this, _S("groups"))
-  m_index(*this)
 {
     /* Stat for existing project directory (must already exist) */
     struct stat myStat;
@@ -364,11 +186,29 @@ Project::Project(const ProjectRootPath& rootPath)
     HECL::MakeDir(m_rootPath.getAbsolutePath() + _S("/.hecl/cooked"));
     HECL::MakeDir(m_rootPath.getAbsolutePath() + _S("/.hecl/config"));
 
-    /* Ensure index is initialized */
+    /* Ensure beacon is valid or created */
-    if (m_index.lockAndRead().empty())
+    FILE* bf = HECL::Fopen((m_rootPath.getAbsolutePath() + _S("/.hecl/beacon")).c_str(), _S("a+b"));
-        m_index.unlockAndCommit();
+    struct BeaconStruct
-    else
+    {
-        m_index.unlockAndDiscard();
+        FourCC magic;
+        uint32_t version;
+    } beacon;
+#define DATA_VERSION 1
+    static const FourCC hecl("HECL");
+    if (fread(&beacon, 1, sizeof(beacon), bf) != sizeof(beacon))
+    {
+        fseek(bf, 0, SEEK_SET);
+        beacon.magic = hecl;
+        beacon.version = SBig(DATA_VERSION);
+        fwrite(&beacon, 1, sizeof(beacon), bf);
+    }
+    fclose(bf);
+    if (beacon.magic != hecl ||
+        SBig(beacon.version) != DATA_VERSION)
+        throw Exception(_S("incompatible HECL project"));
+
+    /* Compile current dataspec */
+    rescanDataSpecs();
 }
 
 void Project::registerLogger(FLogger logger)
@@ -421,6 +261,20 @@ bool Project::removeGroup(const ProjectPath& path)
     return m_groups.unlockAndCommit();
 }
 
+void Project::rescanDataSpecs()
+{
+    m_compiledSpecs.clear();
+    m_specs.lockAndRead();
+    for (const DataSpecEntry* spec = DATA_SPEC_REGISTRY;
+         spec->name.size();
+         ++spec)
+    {
+        SystemUTF8View specUTF8(spec->name);
+        m_compiledSpecs.push_back({*spec, m_specs.checkForLine(specUTF8.utf8_str()) ? true : false});
+    }
+    m_specs.unlockAndDiscard();
+}
+
 bool Project::enableDataSpecs(const std::vector<SystemString>& specs)
 {
     m_specs.lockAndRead();