added index and config file management

This commit is contained in:
Jack Andersen 2015-06-10 13:34:14 -10:00
parent 5d2d2b010d
commit a6d28c2b2b
6 changed files with 375 additions and 66 deletions

View File

@ -17,9 +17,6 @@ LIBPATH += $$OUT_PWD/../lib \
LIBS += -lhecl -lhecl-dataspec -lhecl-blender -lblowfish -lpng
# Yay!! Athena IO
include(../extern/Athena/AthenaCore.pri)
SOURCES += \
$$PWD/main.cpp

View File

@ -8,8 +8,10 @@ char* win_realpath(const char* name, char* restrict resolved);
#include <sys/param.h>
#include <sys/stat.h>
#include <dirent.h>
#include <fcntl.h>
#endif
#include <time.h>
#include <stdarg.h>
#include <stdio.h>
#include <functional>
@ -122,7 +124,13 @@ static inline SystemChar* Getcwd(SystemChar* buf, int maxlen)
#endif
}
static inline FILE* Fopen(const SystemChar* path, const SystemChar* mode)
enum FileLockType
{
LNONE = 0,
LREAD,
LWRITE
};
static inline FILE* Fopen(const SystemChar* path, const SystemChar* mode, FileLockType lock=LNONE)
{
#if HECL_UCS2
FILE* fp = wfopen(path, mode);
@ -132,6 +140,22 @@ static inline FILE* Fopen(const SystemChar* path, const SystemChar* mode)
if (!fp)
throw std::error_code(errno, std::system_category());
if (lock)
{
#if _WIN32
HANDLE fhandle = (HANDLE)fileno(fp);
OVERLAPPED ov = {};
LockFileEx(fhandle, (lock == LWRITE) ? LOCKFILE_EXCLUSIVE_LOCK : 0, 0, 0, 1, &ov);
#else
struct flock lk =
{
(short)((lock == LREAD) ? F_RDLCK : F_WRLCK),
SEEK_SET, 0, 0, 0
};
fcntl(fileno(fp), F_SETLK, &lk);
#endif
}
return fp;
}
@ -196,7 +220,7 @@ typedef std::function<void(LogType, std::string&)> FLogger;
* while fitting comfortably in a 32-bit word. HECL uses a four-char array
* to remain endian-independent.
*/
class FourCC
class FourCC final
{
union
{
@ -210,6 +234,8 @@ public:
: num(*(uint32_t*)name) {}
inline bool operator==(FourCC& other) {return num == other.num;}
inline bool operator!=(FourCC& other) {return num != other.num;}
inline bool operator==(const char* other) {return num == *(uint32_t*)other;}
inline bool operator!=(const char* other) {return num != *(uint32_t*)other;}
inline std::string toString() {return std::string(fcc, 4);}
};
@ -219,7 +245,7 @@ public:
* Hashes are used within HECL to avoid redundant storage of objects;
* providing a rapid mechanism to compare for equality.
*/
class Hash
class Hash final
{
int64_t hash;
public:
@ -229,12 +255,34 @@ public:
: hash(Blowfish_hash(str.data(), str.size())) {}
Hash(int64_t hashin)
: hash(hashin) {}
inline bool operator==(Hash& other) {return hash == other.hash;}
inline bool operator!=(Hash& other) {return hash != other.hash;}
inline bool operator<(Hash& other) {return hash < other.hash;}
inline bool operator>(Hash& other) {return hash > other.hash;}
inline bool operator<=(Hash& other) {return hash <= other.hash;}
inline bool operator>=(Hash& other) {return hash >= other.hash;}
Hash(const Hash& other) {hash = other.hash;}
inline Hash& operator=(const Hash& other) {hash = other.hash; return *this;}
inline bool operator==(const Hash& other) const {return hash == other.hash;}
inline bool operator!=(const Hash& other) const {return hash != other.hash;}
inline bool operator<(const Hash& other) const {return hash < other.hash;}
inline bool operator>(const Hash& other) const {return hash > other.hash;}
inline bool operator<=(const Hash& other) const {return hash <= other.hash;}
inline bool operator>=(const Hash& other) const {return hash >= other.hash;}
};
/**
* @brief Timestamp representation used for comparing modtimes of cooked resources
*/
class Time final
{
uint64_t ts;
public:
Time() : ts(time(NULL)) {}
Time(uint64_t ti) : ts(ti) {}
Time(const Time& other) {ts = other.ts;}
inline uint64_t getTs() const {return ts;}
inline Time& operator=(const Time& other) {ts = other.ts; return *this;}
inline bool operator==(const Time& other) const {return ts == other.ts;}
inline bool operator!=(const Time& other) const {return ts != other.ts;}
inline bool operator<(const Time& other) const {return ts < other.ts;}
inline bool operator>(const Time& other) const {return ts > other.ts;}
inline bool operator<=(const Time& other) const {return ts <= other.ts;}
inline bool operator>=(const Time& other) const {return ts >= other.ts;}
};
/**
@ -255,6 +303,7 @@ class ProjectPath
protected:
SystemString m_absPath;
const SystemChar* m_relPath = NULL;
size_t m_hash = 0;
#if HECL_UCS2
std::string m_utf8AbsPath;
const char* m_utf8RelPath;
@ -331,11 +380,30 @@ public:
*/
PathType getPathType() const;
/**
* @brief Get time of last modification with special behaviors for directories and glob-paths
* @return Time object representing entity's time of last modification
*
* Regular files simply return their modtime as queried from the OS
* Directories return the latest modtime of all first-level regular files
* Glob-paths return the latest modtime of all matched regular files
*/
Time getModtime() const;
/**
* @brief Insert glob matches into existing vector
* @param outPaths Vector to add matches to (will not erase existing contents)
*/
void getGlobResults(std::vector<SystemString>& outPaths) const;
/**
* @brief C++11 compatible runtime hash (NOT USED IN PACKAGES!!)
* @return System-specific hash value
*/
inline size_t hash() const {return m_hash;}
inline bool operator==(const ProjectPath& other) const {return m_hash == other.m_hash;}
inline bool operator!=(const ProjectPath& other) const {return m_hash != other.m_hash;}
};
/**
@ -434,4 +502,13 @@ static inline uint64_t ToBig(uint64_t val) {return val;}
}
namespace std
{
template <> struct hash<HECL::ProjectPath>
{
size_t operator()(const HECL::ProjectPath& val) const noexcept
{return val.hash();}
};
}
#endif // HECL_HPP

View File

@ -6,6 +6,7 @@
#include <functional>
#include <vector>
#include <map>
#include <unordered_map>
#include <memory>
#include <atomic>
#include <stdexcept>
@ -112,17 +113,61 @@ class Project
public:
Project(const HECL::ProjectRootPath& rootPath);
/**
* @brief Configuration file handle
*
* Holds a path to a line-delimited textual configuration file;
* opening a locked handle for read/write transactions
*/
class ConfigFile
{
const Project& m_project;
const SystemString& m_name;
SystemString m_filepath;
std::vector<std::string> m_lines;
FILE* m_lockedFile = NULL;
public:
ConfigFile(const Project& project, const SystemString& name);
std::vector<std::string> readLines();
const std::vector<std::string>& lockAndRead();
void addLine(const std::string& line);
void removeLine(const std::string& refLine);
bool checkForLine(const std::string& refLine);
void unlockAndDiscard();
void unlockAndCommit();
};
/**
* @brief Index file handle
*
* Holds a path to a binary index file;
* opening a locked handle for read/write transactions
*/
class IndexFile
{
SystemString m_filepath;
const Project& m_project;
size_t m_maxPathLen = 0;
FILE* m_lockedFile = NULL;
public:
class Entry
{
friend class IndexFile;
ProjectPath m_path;
HECL::Time m_lastModtime;
bool m_removed = false;
Entry(const ProjectPath& path, const HECL::Time& lastModtime)
: m_path(path), m_lastModtime(lastModtime) {}
Entry(const ProjectPath& path);
};
private:
std::vector<Entry> m_entryStore;
std::unordered_map<ProjectPath, Entry*> m_entryLookup;
public:
IndexFile(const Project& project);
const std::vector<Entry>& lockAndRead();
const std::vector<ProjectPath*> getChangedPaths();
void addOrUpdatePath(const ProjectPath& path);
void removePath(const ProjectPath& path);
void unlockAndDiscard();
void unlockAndCommit();
};
/**
@ -154,7 +199,7 @@ public:
*
* If this method is never called, all project operations will run silently.
*/
virtual void registerLogger(HECL::FLogger logger);
void registerLogger(HECL::FLogger logger);
/**
* @brief Get the path of the project's root-directory
@ -163,7 +208,7 @@ public:
*
* Self explanatory
*/
virtual const ProjectRootPath& getProjectRootPath(bool absolute=false) const;
inline const ProjectRootPath& getProjectRootPath() const {return m_rootPath;}
/**
* @brief Add given file(s) to the database
@ -172,7 +217,7 @@ public:
*
* This method blocks while object hashing takes place
*/
virtual bool addPaths(const std::vector<ProjectPath>& paths);
bool addPaths(const std::vector<ProjectPath>& paths);
/**
* @brief Remove a given file or file-pattern from the database
@ -183,7 +228,7 @@ public:
* This method will not delete actual working files from the project
* directory. It will delete associated cooked objects though.
*/
virtual bool removePaths(const std::vector<ProjectPath>& paths, bool recursive=false);
bool removePaths(const std::vector<ProjectPath>& paths, bool recursive=false);
/**
* @brief Register a working sub-directory as a Dependency Group
@ -199,34 +244,34 @@ public:
* This contiguous storage makes for optimal loading from slow block-devices
* like optical drives.
*/
virtual bool addGroup(const ProjectPath& path);
bool addGroup(const ProjectPath& path);
/**
* @brief Unregister a working sub-directory as a dependency group
* @param path directory to unregister as Dependency Group
* @return true on success
*/
virtual bool removeGroup(const ProjectPath& path);
bool removeGroup(const ProjectPath& path);
/**
* @brief Return map populated with dataspecs targetable by this project interface
* @return Platform map with name-string keys and enable-status values
*/
virtual const std::map<const std::string, const bool>& listDataSpecs();
const std::map<const std::string, const bool>& listDataSpecs();
/**
* @brief Enable persistent user preference for particular spec string(s)
* @param specs String(s) representing unique spec(s) from listDataSpecs
* @return true on success
*/
virtual bool enableDataSpecs(const std::vector<std::string>& specs);
bool enableDataSpecs(const std::vector<std::string>& specs);
/**
* @brief Disable persistent user preference for particular spec string(s)
* @param specs String(s) representing unique spec(s) from listDataSpecs
* @return true on success
*/
virtual bool disableDataSpecs(const std::vector<std::string>& specs);
bool disableDataSpecs(const std::vector<std::string>& specs);
/**
* @brief Begin cook process for specified directory
@ -239,9 +284,9 @@ public:
* This method blocks execution during the procedure, with periodic
* feedback delivered via feedbackCb.
*/
virtual bool cookPath(const ProjectPath& path,
std::function<void(std::string&, Cost, unsigned)> feedbackCb,
bool recursive=false);
bool cookPath(const ProjectPath& path,
std::function<void(std::string&, Cost, unsigned)> feedbackCb,
bool recursive=false);
/**
* @brief Interrupts a cook in progress (call from SIGINT handler)
@ -253,7 +298,7 @@ public:
* Note that this method returns immediately; the resumed cookPath()
* call will return as quickly as possible.
*/
virtual void interruptCook();
void interruptCook();
/**
* @brief Delete cooked objects for directory
@ -264,7 +309,7 @@ public:
* Developers understand how useful 'clean' is. While ideally not required,
* it's useful for verifying that a rebuild from ground-up is doable.
*/
virtual bool cleanPath(const ProjectPath& path, bool recursive=false);
bool cleanPath(const ProjectPath& path, bool recursive=false);
/**
* @brief Nodegraph class for gathering dependency-resolved objects for packaging
@ -296,7 +341,7 @@ public:
* @param path Subpath of project to root depsgraph at
* @return Populated depsgraph ready to traverse
*/
virtual PackageDepsgraph buildPackageDepsgraph(const ProjectPath& path);
PackageDepsgraph buildPackageDepsgraph(const ProjectPath& path);
};

View File

@ -53,6 +53,9 @@ ProjectPath::ProjectPath(const ProjectRootPath& rootPath, const SystemString& pa
if (m_relPath[0] == _S('\0'))
m_relPath = NULL;
std::hash<std::string> hash_fn;
m_hash = hash_fn(std::string(m_relPath));
#if HECL_UCS2
m_utf8AbsPath = WideToUTF8(m_absPath);
m_utf8RelPath = m_utf8AbsPath.c_str() + ((ProjectPath&)rootPath).m_utf8AbsPath.size();
@ -76,6 +79,11 @@ ProjectPath::PathType ProjectPath::getPathType() const
#endif
}
Time ProjectPath::getModtime() const
{
}
static void _recursiveGlob(std::vector<SystemString>& outPaths,
size_t level,
const SystemRegexMatch& pathCompMatches,

View File

@ -4,6 +4,11 @@
#include <string.h>
#include <system_error>
#if _WIN32
#else
#include <unistd.h>
#endif
#include "HECLDatabase.hpp"
namespace HECL
@ -17,7 +22,7 @@ namespace Database
static inline bool CheckNewLineAdvance(std::string::const_iterator& it)
{
if (*it == '\n' || *it == '\0')
if (*it == '\n')
{
it += 1;
return true;
@ -29,31 +34,34 @@ static inline bool CheckNewLineAdvance(std::string::const_iterator& it)
it += 2;
return true;
}
it += 1;
return true;
}
return false;
}
Project::ConfigFile::ConfigFile(const Project& project, const SystemString& name)
: m_project(project), m_name(name)
{
m_filepath = project.m_rootPath.getAbsolutePath() + _S("/.hecl/config/") + name;
}
std::vector<std::string> Project::ConfigFile::readLines()
const std::vector<std::string>& Project::ConfigFile::lockAndRead()
{
FILE* fp = HECL::Fopen(m_filepath.c_str(), _S("r"));
if (m_lockedFile)
return m_lines;
m_lockedFile = HECL::Fopen(m_filepath.c_str(), _S("r+"), LWRITE);
std::string mainString;
char readBuf[1024];
size_t readSz;
while ((readSz = fread(readBuf, 1, 1024, fp)))
while ((readSz = fread(readBuf, 1, 1024, m_lockedFile)))
mainString += std::string(readBuf, readSz);
fclose(fp);
std::string::const_iterator begin = mainString.begin();
std::string::const_iterator end = mainString.begin();
std::vector<std::string> retval;
m_lines.clear();
while (end != mainString.end())
{
std::string::const_iterator origEnd = end;
@ -62,53 +70,44 @@ std::vector<std::string> Project::ConfigFile::readLines()
else if (CheckNewLineAdvance(end))
{
if (begin != origEnd)
retval.push_back(std::string(begin, origEnd));
m_lines.push_back(std::string(begin, origEnd));
begin = end;
continue;
}
++end;
}
if (begin != end)
retval.push_back(std::string(begin, end));
m_lines.push_back(std::string(begin, end));
return retval;
return m_lines;
}
void Project::ConfigFile::addLine(const std::string& line)
{
std::vector<std::string> curLines = readLines();
FILE* fp = HECL::Fopen(m_filepath.c_str(), _S("w"));
for (std::string& line : curLines)
{
fwrite(line.data(), 1, line.length(), fp);
fwrite("\n", 1, 1, fp);
}
fwrite(line.data(), 1, line.length(), fp);
fwrite("\n", 1, 1, fp);
fclose(fp);
if (!checkForLine(line))
m_lines.push_back(line);
}
void Project::ConfigFile::removeLine(const std::string& refLine)
{
std::vector<std::string> curLines = readLines();
if (!m_lockedFile)
throw HECL::Exception(_S("Project::ConfigFile::lockAndRead not yet called"));
FILE* fp = HECL::Fopen(m_filepath.c_str(), _S("w"));
for (std::string& line : curLines)
for (std::vector<std::string>::const_iterator it=m_lines.begin();
it != m_lines.end();
++it)
{
if (line.compare(refLine))
{
fwrite(line.data(), 1, line.length(), fp);
fwrite("\n", 1, 1, fp);
}
if (!(*it).compare(refLine))
it = m_lines.erase(it);
}
fclose(fp);
}
bool Project::ConfigFile::checkForLine(const std::string& refLine)
{
std::vector<std::string> curLines = readLines();
for (std::string& line : curLines)
if (!m_lockedFile)
throw HECL::Exception(_S("Project::ConfigFile::lockAndRead not yet called"));
for (const std::string& line : m_lines)
{
if (!line.compare(refLine))
return true;
@ -116,6 +115,192 @@ bool Project::ConfigFile::checkForLine(const std::string& refLine)
return false;
}
void Project::ConfigFile::unlockAndDiscard()
{
if (!m_lockedFile)
throw HECL::Exception(_S("Project::ConfigFile::lockAndRead not yet called"));
m_lines.clear();
fclose(m_lockedFile);
m_lockedFile = NULL;
}
void Project::ConfigFile::unlockAndCommit()
{
if (!m_lockedFile)
throw HECL::Exception(_S("Project::ConfigFile::lockAndRead not yet called"));
fseek(m_lockedFile, 0, SEEK_SET);
#if _WIN32
SetEndOfFile((HANDLE)fileno(m_lockedFile));
#else
ftruncate(fileno(m_lockedFile), 0);
#endif
for (const std::string& line : m_lines)
{
fwrite(line.c_str(), 1, line.size(), m_lockedFile);
fwrite("\n", 1, 1, m_lockedFile);
}
m_lines.clear();
fclose(m_lockedFile);
m_lockedFile = NULL;
}
/**********************************************
* Project::IndexFile
**********************************************/
struct SIndexHeader
{
HECL::FourCC magic;
uint32_t version;
uint32_t entryCount;
uint32_t maxPathLen;
void swapWithNative()
{
version = ToBig(version);
entryCount = ToBig(entryCount);
maxPathLen = ToBig(maxPathLen);
}
};
Project::IndexFile::IndexFile(const Project& project)
: m_project(project)
{
m_filepath = project.m_rootPath.getAbsolutePath() + _S("/.hecl/index");
}
const std::vector<Project::IndexFile::Entry>& Project::IndexFile::lockAndRead()
{
if (m_lockedFile)
return m_entryStore;
m_lockedFile = HECL::Fopen(m_filepath.c_str(), _S("r+"), LWRITE);
SIndexHeader header;
if (fread(&header, 1, sizeof(header), m_lockedFile) != sizeof(header))
return m_entryStore;
header.swapWithNative();
if (header.magic != "HECL")
throw HECL::Exception(_S("unrecognized HECL index"));
if (header.version != 1)
throw HECL::Exception(_S("unrecognized HECL version"));
char* pathBuf = new char[header.maxPathLen];
for (uint32_t e=0 ; e<header.entryCount ; ++e)
{
uint64_t mt;
fread(&mt, 1, 8, m_lockedFile);
mt = ToBig(mt);
uint32_t strLen;
fread(&strLen, 1, 4, m_lockedFile);
strLen = ToBig(strLen);
if (strLen > m_maxPathLen)
m_maxPathLen = strLen;
fread(pathBuf, 1, strLen, m_lockedFile);
std::string pathStr(pathBuf, strLen);
SystemStringView pathView(pathStr);
ProjectPath path(m_project.getProjectRootPath(), pathView.sys_str());
if (m_entryLookup.find(path) == m_entryLookup.end())
{
m_entryStore.push_back(Project::IndexFile::Entry(path, mt));
m_entryLookup[path] = &m_entryStore.back();
}
}
delete[] pathBuf;
}
const std::vector<ProjectPath*> Project::IndexFile::getChangedPaths()
{
if (!m_lockedFile)
throw HECL::Exception(_S("Project::IndexFile::lockAndRead not yet called"));
std::vector<ProjectPath*> retval;
for (Project::IndexFile::Entry& ent : m_entryStore)
{
if (ent.m_removed)
continue;
if (ent.m_lastModtime != ent.m_path.getModtime())
retval.push_back(&ent.m_path);
}
return retval;
}
void Project::IndexFile::addOrUpdatePath(const ProjectPath& path)
{
if (!m_lockedFile)
throw HECL::Exception(_S("Project::IndexFile::lockAndRead not yet called"));
std::unordered_map<ProjectPath, Entry*>::iterator it = m_entryLookup.find(path);
if (it == m_entryLookup.end())
{
m_entryStore.push_back(Project::IndexFile::Entry(path, path.getModtime()));
m_entryLookup[path] = &m_entryStore.back();
return;
}
(*it).second->m_lastModtime = path.getModtime();
}
void Project::IndexFile::removePath(const ProjectPath& path)
{
if (!m_lockedFile)
throw HECL::Exception(_S("Project::IndexFile::lockAndRead not yet called"));
std::unordered_map<ProjectPath, Entry*>::iterator it = m_entryLookup.find(path);
if (it != m_entryLookup.end())
{
(*it).second->m_removed = true;
m_entryLookup.erase(it);
}
}
void Project::IndexFile::unlockAndDiscard()
{
if (!m_lockedFile)
throw HECL::Exception(_S("Project::IndexFile::lockAndRead not yet called"));
m_entryLookup.clear();
m_entryStore.clear();
fclose(m_lockedFile);
m_lockedFile = NULL;
}
void Project::IndexFile::unlockAndCommit()
{
if (!m_lockedFile)
throw HECL::Exception(_S("Project::IndexFile::lockAndRead not yet called"));
fseek(m_lockedFile, 0, SEEK_SET);
#if _WIN32
SetEndOfFile((HANDLE)fileno(m_lockedFile));
#else
ftruncate(fileno(m_lockedFile), 0);
#endif
SIndexHeader header =
{
HECL::FourCC("HECL"),
1, (uint32_t)m_entryStore.size(), (uint32_t)m_maxPathLen
};
header.swapWithNative();
fwrite(&header, 1, sizeof(header), m_lockedFile);
for (Project::IndexFile::Entry& ent : m_entryStore)
{
uint64_t mt = ToBig(ent.m_lastModtime.getTs());
fwrite(&mt, 1, 8, m_lockedFile);
size_t strLen = strlen(ent.m_path.getRelativePathUTF8());
uint32_t strLenb = ToBig(strLen);
fwrite(&strLenb, 1, 4, m_lockedFile);
fwrite(ent.m_path.getRelativePathUTF8(), 1, strLen, m_lockedFile);
}
m_entryLookup.clear();
m_entryStore.clear();
fclose(m_lockedFile);
m_lockedFile = NULL;
}
/**********************************************
* Project
**********************************************/
@ -144,10 +329,6 @@ void Project::registerLogger(FLogger logger)
{
}
const ProjectRootPath& Project::getProjectRootPath(bool absolute) const
{
}
bool Project::addPaths(const std::vector<ProjectPath>& paths)
{
}

View File

@ -15,6 +15,7 @@ include (frontend/frontend.pri)
include (backend/backend.pri)
include (database/database.pri)
include (runtime/runtime.pri)
include(../extern/Athena/AthenaCore.pri)
SOURCES += \
HECL.cpp \