mirror of https://github.com/AxioDL/metaforce.git
459 lines
14 KiB
C++
459 lines
14 KiB
C++
#ifndef HECLDATABASE_HPP
|
|
#define HECLDATABASE_HPP
|
|
|
|
#include <iterator>
|
|
#include <string>
|
|
#include <functional>
|
|
#include <vector>
|
|
#include <map>
|
|
#include <list>
|
|
#include <unordered_map>
|
|
#include <unordered_set>
|
|
#include <memory>
|
|
#include <atomic>
|
|
#include <fstream>
|
|
#include <stdint.h>
|
|
#include <assert.h>
|
|
|
|
#include <athena/IStreamReader.hpp>
|
|
#include "logvisor/logvisor.hpp"
|
|
|
|
#include "hecl.hpp"
|
|
|
|
namespace hecl
|
|
{
|
|
namespace Database
|
|
{
|
|
class Project;
|
|
|
|
extern logvisor::Module LogModule;
|
|
|
|
typedef std::function<void(const hecl::SystemChar*, const hecl::SystemChar*, int, float)> FProgress;
|
|
|
|
/**
|
|
* @brief Nodegraph class for gathering dependency-resolved objects for packaging
|
|
*/
|
|
class PackageDepsgraph
|
|
{
|
|
public:
|
|
struct Node
|
|
{
|
|
enum class Type
|
|
{
|
|
Data,
|
|
Group
|
|
} type;
|
|
ProjectPath path;
|
|
ProjectPath cookedPath;
|
|
class ObjectBase* projectObj;
|
|
Node* sub;
|
|
Node* next;
|
|
};
|
|
private:
|
|
friend class Project;
|
|
std::vector<Node> m_nodes;
|
|
public:
|
|
const Node* getRootNode() const {return &m_nodes[0];}
|
|
};
|
|
|
|
/**
|
|
* @brief Subclassed by dataspec entries to manage per-game aspects of the data pipeline
|
|
*
|
|
* The DataSpec class manages interfaces for unpackaging, cooking, and packaging
|
|
* of data for interacting with a specific system/game-engine.
|
|
*/
|
|
class IDataSpec
|
|
{
|
|
public:
|
|
virtual ~IDataSpec() {}
|
|
using FProgress = hecl::Database::FProgress;
|
|
using FCookProgress = std::function<void(const SystemChar*)>;
|
|
|
|
/**
|
|
* @brief Extract Pass Info
|
|
*
|
|
* An extract pass iterates through a source package or image and
|
|
* reverses the cooking process by emitting editable resources
|
|
*/
|
|
struct ExtractPassInfo
|
|
{
|
|
SystemString srcpath;
|
|
std::vector<SystemString> extractArgs;
|
|
bool force;
|
|
};
|
|
|
|
/**
|
|
* @brief Extract Report Representation
|
|
*
|
|
* Constructed by canExtract() to advise the user of the content about
|
|
* to be extracted
|
|
*/
|
|
struct ExtractReport
|
|
{
|
|
SystemString name;
|
|
SystemString desc;
|
|
std::vector<ExtractReport> childOpts;
|
|
};
|
|
|
|
virtual bool canExtract(const ExtractPassInfo& info, std::vector<ExtractReport>& reps)
|
|
{(void)info;(void)reps;LogModule.report(logvisor::Error, "not implemented");return false;}
|
|
virtual void doExtract(const ExtractPassInfo& info, FProgress progress)
|
|
{(void)info;(void)progress;}
|
|
|
|
virtual bool canCook(const ProjectPath& path)
|
|
{(void)path;LogModule.report(logvisor::Error, "not implemented");return false;}
|
|
virtual void doCook(const ProjectPath& path, const ProjectPath& cookedPath,
|
|
bool fast, FCookProgress progress)
|
|
{(void)path;(void)cookedPath;(void)fast;(void)progress;}
|
|
|
|
/**
|
|
* @brief Package Pass Info
|
|
*
|
|
* A package pass performs last-minute queries of source resources,
|
|
* gathers dependencies and packages cooked data together in the
|
|
* most efficient form for the dataspec
|
|
*/
|
|
struct PackagePassInfo
|
|
{
|
|
const PackageDepsgraph& depsgraph;
|
|
ProjectPath subpath;
|
|
ProjectPath outpath;
|
|
};
|
|
virtual bool canPackage(const PackagePassInfo& info,
|
|
SystemString& reasonNo)
|
|
{(void)info;reasonNo=_S("not implemented");return false;}
|
|
virtual void gatherDependencies(const PackagePassInfo& info,
|
|
std::unordered_set<ProjectPath>& implicitsOut)
|
|
{(void)info;(void)implicitsOut;}
|
|
virtual void doPackage(const PackagePassInfo& info)
|
|
{(void)info;}
|
|
};
|
|
|
|
/**
|
|
* @brief Pre-emptive indication of what the constructed DataSpec is used for
|
|
*/
|
|
enum class DataSpecTool
|
|
{
|
|
Extract,
|
|
Cook,
|
|
Package
|
|
};
|
|
|
|
extern std::vector<const struct DataSpecEntry*> DATA_SPEC_REGISTRY;
|
|
|
|
/**
|
|
* @brief IDataSpec registry entry
|
|
*
|
|
* Auto-registers with data spec registry
|
|
*/
|
|
struct DataSpecEntry
|
|
{
|
|
const SystemChar* m_name;
|
|
const SystemChar* m_desc;
|
|
std::function<IDataSpec*(Project&, DataSpecTool)> m_factory;
|
|
|
|
DataSpecEntry(const SystemChar* name, const SystemChar* desc,
|
|
std::function<IDataSpec*(Project& project, DataSpecTool)>&& factory)
|
|
: m_name(name), m_desc(desc), m_factory(std::move(factory)) {}
|
|
};
|
|
|
|
/**
|
|
* @brief Base object to subclass for integrating with key project operations
|
|
*
|
|
* All project objects are provided with IDataObject pointers to their database
|
|
* entries. Subclasses register themselves with a type registry so instances
|
|
* are automatically constructed when performing operations like cooking and packaging.
|
|
*
|
|
* DO NOT CONSTRUCT THIS OR SUBCLASSES DIRECTLY!!
|
|
*/
|
|
class ObjectBase
|
|
{
|
|
friend class Project;
|
|
SystemString m_path;
|
|
protected:
|
|
|
|
/**
|
|
* @brief Byte-order of target system
|
|
*/
|
|
enum class DataEndianness
|
|
{
|
|
None,
|
|
Big, /**< Big-endian (PowerPC) */
|
|
Little /**< Little-endian (Intel) */
|
|
};
|
|
|
|
/**
|
|
* @brief Data-formats of target system
|
|
*/
|
|
enum class DataPlatform
|
|
{
|
|
None,
|
|
Generic, /**< Scanline textures and 3-way shader bundle (GLSL, HLSL, SPIR-V) */
|
|
Revolution, /**< Tiled textures and GX register buffers */
|
|
Cafe /**< Swizzled textures and R700 shader objects */
|
|
};
|
|
|
|
typedef std::function<void(const void* data, size_t len)> FDataAppender;
|
|
|
|
/**
|
|
* @brief Optional private method implemented by subclasses to cook objects
|
|
* @param dataAppender subclass calls this function zero or more times to provide cooked-data linearly
|
|
* @param endianness byte-order to target
|
|
* @param platform data-formats to target
|
|
* @return true if cook succeeded
|
|
*
|
|
* This method is called during IProject::cookPath().
|
|
* Part of the cooking process may include embedding database-refs to dependencies.
|
|
* This method should store the 64-bit value provided by IDataObject::id() when doing this.
|
|
*/
|
|
virtual bool cookObject(FDataAppender dataAppender,
|
|
DataEndianness endianness, DataPlatform platform)
|
|
{(void)dataAppender;(void)endianness;(void)platform;return true;}
|
|
|
|
typedef std::function<void(ObjectBase*)> FDepAdder;
|
|
|
|
/**
|
|
* @brief Optional private method implemented by CProjectObject subclasses to resolve dependencies
|
|
* @param depAdder subclass calls this function zero or more times to register each dependency
|
|
*
|
|
* This method is called during IProject::packagePath().
|
|
* Dependencies registered via this method will eventually have this method called on themselves
|
|
* as well. This is a non-recursive operation, no need for subclasses to implement recursion-control.
|
|
*/
|
|
virtual void gatherDeps(FDepAdder depAdder)
|
|
{(void)depAdder;}
|
|
|
|
/**
|
|
* @brief Get a packagable FourCC representation of the object's type
|
|
* @return FourCC of the type
|
|
*/
|
|
virtual FourCC getType() const
|
|
{return FourCC("NULL");}
|
|
|
|
public:
|
|
ObjectBase(const SystemString& path)
|
|
: m_path(path) {}
|
|
|
|
const SystemString& getPath() const {return m_path;}
|
|
|
|
};
|
|
|
|
|
|
/**
|
|
* @brief Main project interface
|
|
*
|
|
* Projects are intermediate working directories used for staging
|
|
* resources in their ideal editor-formats. This interface exposes all
|
|
* primary operations to perform on a given project.
|
|
*/
|
|
class Project
|
|
{
|
|
public:
|
|
struct ProjectDataSpec
|
|
{
|
|
const DataSpecEntry& spec;
|
|
ProjectPath cookedPath;
|
|
bool active;
|
|
};
|
|
private:
|
|
ProjectRootPath m_rootPath;
|
|
ProjectPath m_workRoot;
|
|
ProjectPath m_dotPath;
|
|
ProjectPath m_cookedRoot;
|
|
std::vector<ProjectDataSpec> m_compiledSpecs;
|
|
bool m_valid = false;
|
|
public:
|
|
Project(const hecl::ProjectRootPath& rootPath);
|
|
operator bool() const {return m_valid;}
|
|
|
|
/**
|
|
* @brief Configuration file handle
|
|
*
|
|
* Holds a path to a line-delimited textual configuration file;
|
|
* opening a locked handle for read/write transactions
|
|
*/
|
|
class ConfigFile
|
|
{
|
|
SystemString m_filepath;
|
|
std::vector<std::string> m_lines;
|
|
FILE* m_lockedFile = NULL;
|
|
public:
|
|
ConfigFile(const Project& project, const SystemString& name,
|
|
const SystemString& subdir=_S("/.hecl/"));
|
|
std::vector<std::string>& lockAndRead();
|
|
void addLine(const std::string& line);
|
|
void removeLine(const std::string& refLine);
|
|
bool checkForLine(const std::string& refLine);
|
|
void unlockAndDiscard();
|
|
bool unlockAndCommit();
|
|
};
|
|
ConfigFile m_specs;
|
|
ConfigFile m_paths;
|
|
ConfigFile m_groups;
|
|
|
|
/**
|
|
* @brief A rough description of how 'expensive' a given cook operation is
|
|
*
|
|
* This is used to provide pretty colors during the cook operation
|
|
*/
|
|
enum class Cost
|
|
{
|
|
None,
|
|
Light,
|
|
Medium,
|
|
Heavy
|
|
};
|
|
|
|
/**
|
|
* @brief Get the path of the project's root-directory
|
|
* @return project root path
|
|
*
|
|
* Self explanatory
|
|
*/
|
|
const ProjectRootPath& getProjectRootPath() const {return m_rootPath;}
|
|
|
|
/**
|
|
* @brief Get the path of project's working directory
|
|
* @return project working path
|
|
*/
|
|
const ProjectPath& getProjectWorkingPath() const {return m_workRoot;}
|
|
|
|
/**
|
|
* @brief Get the path of project's cooked directory for a specific DataSpec
|
|
* @param DataSpec to retrieve path for
|
|
* @return project cooked path
|
|
*
|
|
* The cooked path matches the directory layout of the working directory
|
|
*/
|
|
const ProjectPath& getProjectCookedPath(const DataSpecEntry& spec) const
|
|
{
|
|
for (const ProjectDataSpec& sp : m_compiledSpecs)
|
|
if (&sp.spec == &spec)
|
|
return sp.cookedPath;
|
|
LogModule.report(logvisor::Fatal, "Unable to find spec '%s'", spec.m_name);
|
|
return m_cookedRoot;
|
|
}
|
|
|
|
/**
|
|
* @brief Add given file(s) to the database
|
|
* @param path file or pattern within project
|
|
* @return true on success
|
|
*
|
|
* This method blocks while object hashing takes place
|
|
*/
|
|
bool addPaths(const std::vector<ProjectPath>& paths);
|
|
|
|
/**
|
|
* @brief Remove a given file or file-pattern from the database
|
|
* @param paths file(s) or pattern(s) within project
|
|
* @param recursive traverse into matched subdirectories
|
|
* @return true on success
|
|
*
|
|
* This method will not delete actual working files from the project
|
|
* directory. It will delete associated cooked objects though.
|
|
*/
|
|
bool removePaths(const std::vector<ProjectPath>& paths, bool recursive=false);
|
|
|
|
/**
|
|
* @brief Register a working sub-directory as a Dependency Group
|
|
* @param path directory to register as Dependency Group
|
|
* @return true on success
|
|
*
|
|
* Dependency Groups are used at runtime to stage burst load-transactions.
|
|
* They may only be added to directories and will automatically claim
|
|
* subdirectories as well.
|
|
*
|
|
* Cooked objects in dependency groups will be packaged contiguously
|
|
* and automatically duplicated if shared with other dependency groups.
|
|
* This contiguous storage makes for optimal loading from slow block-devices
|
|
* like optical drives.
|
|
*/
|
|
bool addGroup(const ProjectPath& path);
|
|
|
|
/**
|
|
* @brief Unregister a working sub-directory as a dependency group
|
|
* @param path directory to unregister as Dependency Group
|
|
* @return true on success
|
|
*/
|
|
bool removeGroup(const ProjectPath& path);
|
|
|
|
/**
|
|
* @brief Re-reads the data store holding user's spec preferences
|
|
*
|
|
* Call periodically in a long-term use of the hecl::Database::Project class.
|
|
* Install filesystem event-hooks if possible.
|
|
*/
|
|
void rescanDataSpecs();
|
|
|
|
/**
|
|
* @brief Return map populated with dataspecs targetable by this project interface
|
|
* @return Platform map with name-string keys and enable-status values
|
|
*/
|
|
const std::vector<ProjectDataSpec>& getDataSpecs() {return m_compiledSpecs;}
|
|
|
|
/**
|
|
* @brief Enable persistent user preference for particular spec string(s)
|
|
* @param specs String(s) representing unique spec(s) from listDataSpecs
|
|
* @return true on success
|
|
*/
|
|
bool enableDataSpecs(const std::vector<SystemString>& specs);
|
|
|
|
/**
|
|
* @brief Disable persistent user preference for particular spec string(s)
|
|
* @param specs String(s) representing unique spec(s) from listDataSpecs
|
|
* @return true on success
|
|
*/
|
|
bool disableDataSpecs(const std::vector<SystemString>& specs);
|
|
|
|
/**
|
|
* @brief Begin cook process for specified directory
|
|
* @param path directory of intermediates to cook
|
|
* @param feedbackCb a callback to run reporting cook-progress
|
|
* @param recursive traverse subdirectories to cook as well
|
|
* @param fast enables faster (draft) extraction for supported data types
|
|
* @return true on success
|
|
*
|
|
* Object cooking is generally an expensive process for large projects.
|
|
* This method blocks execution during the procedure, with periodic
|
|
* feedback delivered via feedbackCb.
|
|
*/
|
|
bool cookPath(const ProjectPath& path, FProgress feedbackCb,
|
|
bool recursive=false, bool force=false, bool fast=false);
|
|
|
|
/**
|
|
* @brief Interrupts a cook in progress (call from SIGINT handler)
|
|
*
|
|
* Database corruption is bad! sqlite is pretty robust at avoiding data corruption,
|
|
* but HECL spreads its data objects through the filesystem; this ensures that
|
|
* those objects are cleanly finalized or discarded before stopping.
|
|
*
|
|
* Note that this method returns immediately; the resumed cookPath()
|
|
* call will return as quickly as possible.
|
|
*/
|
|
void interruptCook();
|
|
|
|
/**
|
|
* @brief Delete cooked objects for directory
|
|
* @param path directory of intermediates to clean
|
|
* @param recursive traverse subdirectories to clean as well
|
|
* @return true on success
|
|
*
|
|
* Developers understand how useful 'clean' is. While ideally not required,
|
|
* it's useful for verifying that a rebuild from ground-up is doable.
|
|
*/
|
|
bool cleanPath(const ProjectPath& path, bool recursive=false);
|
|
|
|
/**
|
|
* @brief Constructs a full depsgraph of the project-subpath provided
|
|
* @param path Subpath of project to root depsgraph at
|
|
* @return Populated depsgraph ready to traverse
|
|
*/
|
|
PackageDepsgraph buildPackageDepsgraph(const ProjectPath& path);
|
|
|
|
};
|
|
|
|
}
|
|
}
|
|
|
|
#endif // HECLDATABASE_HPP
|