AxioDL
/
metaforce
mirror of https://github.com/AxioDL/metaforce.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

Initial ShaderCache implementation

This commit is contained in:
Jack Andersen 2015-11-12 16:12:09 -10:00
parent a9fe41d994
commit 7c0206bd39
12 changed files with 440 additions and 209 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hecl/extern/Athena vendored

@ -1 +1 @@
Subproject commit bedcf128acfcc3307e896ccab903a4dcd082639b
Subproject commit b6b54d092130aa9d31e845c768faf8f08b7a30b1

2
hecl/extern/libBoo vendored

@ -1 +1 @@
Subproject commit c9fd0fdbb57e8c5e0d23041b789aa43dff30520a
Subproject commit 83475b4b092af4941924385083cff9c2ecb1f567

11
hecl/include/HECL/HECL.hpp
View File

@ -386,10 +386,12 @@ public:
* Hashes are used within HECL to avoid redundant storage of objects;
* providing a rapid mechanism to compare for equality.
*/
class Hash final
class Hash
{
unsigned long long hash;
unsigned long long hash = 0;
public:
Hash() = default;
operator bool() const {return hash != 0;}
Hash(const void* buf, size_t len)
: hash(XXH64((uint8_t*)buf, len, 0)) {}
Hash(const std::string& str)
@ -980,6 +982,11 @@ template <> struct hash<HECL::ProjectPath>
size_t operator()(const HECL::ProjectPath& val) const NOEXCEPT
{return val.hash().valSizeT();}
};
template <> struct hash<HECL::Hash>
{
size_t operator()(const HECL::Hash& val) const NOEXCEPT
{return val.valSizeT();}
};
}
#endif // HECL_HPP

280
hecl/include/HECL/Runtime.hpp
View File

@ -1,204 +1,136 @@
#ifndef HECLRUNTIME_HPP
#define HECLRUNTIME_HPP
#include <memory>
#include <vector>
#include <atomic>
#include "HECL.hpp"
#include <boo/graphicsdev/IGraphicsDataFactory.hpp>
#include <Athena/DNA.hpp>
#include <Athena/FileReader.hpp>
#include <unordered_map>
namespace HECL
{
namespace Runtime
{
class Entity
/**
* @brief Per-platform file store resolution
*/
class FileStoreManager
{
SystemString m_domain;
SystemString m_storeRoot;
public:
enum Type
FileStoreManager(const SystemString& domain);
const SystemString& getDomain() const {return m_domain;}
const SystemString& getStoreRoot() const {return m_storeRoot;}
};
/**
* @brief Shader formats that may be identified within ShaderHash
*/
enum ShaderFormat : uint8_t
{
ShaderFormatNone,
ShaderFormatGLSL,
ShaderFormatHLSL,
ShaderFormatMetal,
ShaderFormatSpirV
};
/**
* @brief Hash subclass for identifying shaders and their metadata
*/
class ShaderTag : public Hash
{
union
{
ENTITY_NONE,
ENTITY_OBJECT,
ENTITY_GROUP
uint64_t m_meta = 0;
struct
{
ShaderFormat m_fmt;
uint8_t m_colorCount;
uint8_t m_uvCount;
uint8_t m_weightCount;
};
};
private:
Type m_type;
const std::string& m_path;
bool m_loaded = false;
friend class Group;
friend class ObjectBase;
Entity(Type type, const std::string& path)
: m_type(type), m_path(path) {}
public:
/**
* @brief Get type of runtime object
* @return Type enum
*/
inline Type getType() const {return m_type;}
/**
* @brief Get database entity path
* @return Path string
*/
inline const std::string& getPath() const {return m_path;}
/**
* @brief Determine if object is fully loaded and constructed
* @return true if so
*/
inline bool isLoaded() const {return m_loaded;}
ShaderTag() = default;
ShaderTag(const void* buf, size_t len, ShaderFormat fmt, uint8_t c, uint8_t u, uint8_t w)
: Hash(buf, len), m_fmt(fmt), m_colorCount(c), m_uvCount(u), m_weightCount(w) {}
ShaderTag(unsigned long long hashin, uint64_t meta)
: Hash(hashin), m_meta(meta) {}
ShaderTag(const ShaderTag& other) : Hash(other) {}
ShaderFormat getShaderFormat() const {return m_fmt;}
uint8_t getColorCount() const {return m_colorCount;}
uint8_t getUvCount() const {return m_uvCount;}
uint8_t getWeightCount() const {return m_weightCount;}
uint64_t getMetaData() const {return m_meta;}
};
/**
* @brief Interface representing a load-ordered group of runtime objects
*
* HLPK files perform all data retrieval using the notion of 'groups'
* Groups are a collection of data objects that have been sequentially packed
* in the package file and are constructed in the indexed order of the group.
*
* RuntimeGroup objects are internally created and weakly-referenced by CRuntime.
* RuntimeObject objects are weakly-referenced by RuntimeGroup; they're strongly
* referenced by application systems as long as they're needed.
*
* DO NOT CONSTRUCT THIS DIRECTLY!!
* @brief Maintains index/data file pair containing platform-dependent cached shader data
*/
class Group : public Entity
class ShaderCacheManager
{
public:
typedef std::vector<std::weak_ptr<const class RuntimeObjectBase>> GroupObjectsVector;
private:
friend class HECLRuntime;
GroupObjectsVector m_objects;
Group(const std::string& path)
: Entity(ENTITY_GROUP, path) {}
public:
inline const GroupObjectsVector& getObjects() const {return m_objects;}
};
/**
* @brief Base object to subclass for integrating with key runtime operations
*
* All runtime objects are provided with IDataObject pointers to their database
* entries. Subclasses register themselves with a type registry so instances
* are automatically constructed when performing operations like runtime-integration.
*
* DO NOT CONSTRUCT THIS OR SUBCLASSES DIRECTLY!!
*/
class ObjectBase : public Entity
{
std::shared_ptr<const Group> m_parent;
protected:
/**
* @brief Optional subclass method called on background thread or in response to interrupt when data is ready
* @param data fully-loaded data buffer
* @param len length of buffer
* @return true when data is successfully integrated into the runtime
*/
virtual bool _objectFinishedLoading(const void* data, size_t len)
{(void)data;(void)len;return true;}
/**
* @brief Optional subclass method called in response to reference-count dropping to 0
*/
virtual void _objectWillUnload() {}
public:
ObjectBase(const Group* group, const std::string& path)
: Entity(ENTITY_OBJECT, path), m_parent(group) {}
/**
* @brief Get parent group of object
* @return Borrowed pointer of parent RuntimeGroup
*/
inline const Group* getParentGroup() {return m_parent.get();}
};
/**
* @brief Loadable/Bindable runtime texture class
*/
class Texture
{
};
/**
* @brief Bindable runtime material class
*/
class Material
{
};
/**
* @brief Bindable runtime mesh surface
*/
class MeshSurface
{
};
/**
* @brief Loadable data representation containing vertex buffers, surfaces and shader refs
*/
class MeshObject
{
public:
};
/**
* @brief HLPK Runtime data-management root
*
* Interface for controlling runtime data-operations like object lookup
* and burst load-transactions using HLPK packages. The runtime's
* implementation automatically constructs RuntimeObjectBase and
* RuntimeGroup instances as needed.
*/
class Runtime
{
public:
/**
* @brief Constructs the HECL runtime root
* @param hlpkDirectory directory to search for .hlpk files
*/
Runtime(const SystemString& hlpkDirectory);
~Runtime();
/**
* @brief Structure indicating the load status of an object group
*/
struct SGroupLoadStatus
const FileStoreManager& m_storeMgr;
Athena::io::FileReader m_idxFr;
Athena::io::FileReader m_datFr;
struct IndexEntry : Athena::io::DNA<Athena::BigEndian>
{
std::atomic_bool done;
std::atomic_size_t completedObjects;
std::atomic_size_t totalObjects;
DECL_DNA
Value<atUint64> m_hash;
Value<atUint64> m_meta;
Value<atUint64> m_compOffset;
Value<atUint32> m_compSize;
Value<atUint32> m_decompSize;
};
std::vector<IndexEntry> m_entries;
std::unordered_map<Hash, size_t> m_entryLookup;
uint64_t m_loadedRand = 0;
void BootstrapIndex();
public:
ShaderCacheManager(const FileStoreManager& storeMgr)
: m_storeMgr(storeMgr),
m_idxFr(storeMgr.getStoreRoot() + _S("/shadercache.idx")),
m_datFr(storeMgr.getStoreRoot() + _S("/shadercache.dat"))
{reload();}
void reload();
/**
* @brief Begin a synchronous group-load transaction
* @param pathHash Hashed path string to perform lookup
* @return Shared reference to the loading/loaded object
*
* This method blocks until the entire containing-group is loaded.
* Paths to groups or individual objects are accepted.
*/
std::shared_ptr<Entity> loadSync(const Hash& pathHash);
/**
* @brief Begin an asynchronous group-load transaction
* @param pathHash Hashed path string to perform lookup
* @param statusOut Optional atomically-pollable structure updated with status fields
* @return Shared reference to the loading/loaded object
*
* This method returns once all group entity stubs are constructed.
* Paths to groups or individual objects are accepted.
*/
std::shared_ptr<Entity> loadAsync(const Hash& pathHash,
SGroupLoadStatus* statusOut=NULL);
class CachedData
{
friend class ShaderCacheManager;
CachedData() = default;
CachedData(unsigned long long hashin, uint64_t meta, size_t decompSz)
: m_tag(hashin, meta), m_data(new uint8_t[decompSz]), m_sz(decompSz) {}
public:
ShaderTag m_tag;
std::unique_ptr<uint8_t[]> m_data;
size_t m_sz;
operator bool() const {return m_tag.operator bool();}
};
CachedData lookupData(const Hash& hash);
bool addData(const ShaderTag& hash, const void* data, size_t sz);
};
/**
* @brief Integrated reader/constructor/container for HMDL data
*/
class HMDLData
{
public:
HMDLData(boo::IGraphicsDataFactory* factory, const void* data);
};
}
}
namespace std
{
template <> struct hash<HECL::Runtime::ShaderTag>
{
size_t operator()(const HECL::Runtime::ShaderTag& val) const NOEXCEPT
{return val.valSizeT();}
};
}
#endif // HECLRUNTIME_HPP

2
hecl/lib/CMakeLists.txt
View File

@ -8,6 +8,7 @@ list(APPEND PLAT_SRCS winsupport.cpp ../include/HECL/winsupport.hpp)
endif()
atdna(atdna_Frontend.cpp ../include/HECL/Frontend.hpp)
atdna(atdna_Runtime.cpp ../include/HECL/Runtime.hpp)
add_library(HECLCommon
HECL.cpp
@ -22,5 +23,6 @@ add_library(HECLCommon
../include/HECL/Database.hpp
../include/HECL/Runtime.hpp
atdna_Frontend.cpp
atdna_Runtime.cpp
${PLAT_SRCS})

4
hecl/lib/Runtime/CMakeLists.txt
View File

@ -1,2 +1,4 @@
add_library(HECLRuntime
HECLRuntime.cpp)
FileStoreManager.cpp
ShaderCacheManager.cpp
HMDL.cpp)

31
hecl/lib/Runtime/FileStoreManager.cpp Normal file
View File

@ -0,0 +1,31 @@
#include "HECL/Runtime.hpp"
#include <LogVisor/LogVisor.hpp>
namespace HECL
{
namespace Runtime
{
static LogVisor::LogModule Log("FileStoreManager");
FileStoreManager::FileStoreManager(const SystemString& domain)
: m_domain(domain)
{
#if _WIN32
#elif __APPLE__
#else
const char* home = getenv("HOME");
if (!home)
Log.report(LogVisor::FatalError, "unable to locate $HOME for file store");
std::string path(home);
path += "/.heclrun";
if (mkdir(path.c_str(), 0755))
Log.report(LogVisor::FatalError, "unable to mkdir at %s", path.c_str());
path += '/' + domain;
if (mkdir(path.c_str(), 0755))
Log.report(LogVisor::FatalError, "unable to mkdir at %s", path.c_str());
m_storeRoot = path;
#endif
}
}
}

28
hecl/lib/Runtime/HECLRuntime.cpp
View File

@ -1,28 +0,0 @@
#include "HECL/Runtime.hpp"
namespace HECL
{
namespace Runtime
{
Runtime::Runtime(const HECL::SystemString& hlpkDirectory)
{
}
Runtime::~Runtime()
{
}
std::shared_ptr<Entity> Runtime::loadSync(const Hash& pathHash)
{
return std::shared_ptr<Entity>();
}
std::shared_ptr<Entity> Runtime::loadAsync(const Hash& pathHash,
SGroupLoadStatus* statusOut)
{
return std::shared_ptr<Entity>();
}
}
}

13
hecl/lib/Runtime/HMDL.cpp Normal file
View File

@ -0,0 +1,13 @@
#include "HECL/Runtime.hpp"
namespace HECL
{
namespace Runtime
{
HMDLData::HMDLData(boo::IGraphicsDataFactory* factory, const void *data)
{
}
}
}

256
hecl/lib/Runtime/ShaderCacheManager.cpp Normal file
View File

@ -0,0 +1,256 @@
#include "HECL/Runtime.hpp"
#include <Athena/FileReader.hpp>
#include <Athena/FileWriter.hpp>
#include <zlib.h>
#include <algorithm>
namespace HECL
{
namespace Runtime
{
static LogVisor::LogModule Log("ShaderCacheManager");
static uint64_t IDX_MAGIC = SBIG(0xDEADFEEDC001D00D);
static uint64_t DAT_MAGIC = SBIG(0xC001D00DDEADBABE);
static uint64_t ZERO64 = 0;
static uint64_t Random64()
{
uint64_t ret;
#if _WIN32
#else
FILE* fp = fopen("/dev/urandom", "rb");
fread(&ret, 1, 8, fp);
fclose(fp);
#endif
return ret;
}
void ShaderCacheManager::BootstrapIndex()
{
m_loadedRand = Random64();
m_idxFr.close();
m_datFr.close();
FILE* idxFp = HECL::Fopen(m_idxFr.filename().c_str(), _S("wb"));
if (!idxFp)
Log.report(LogVisor::FatalError, _S("unable to write shader cache index at %s"),
m_idxFr.filename().c_str());
fwrite(&IDX_MAGIC, 1, 8, idxFp);
fwrite(&m_loadedRand, 1, 8, idxFp);
fwrite(&ZERO64, 1, 8, idxFp);
fwrite(&ZERO64, 1, 8, idxFp);
fclose(idxFp);
FILE* datFp = HECL::Fopen(m_datFr.filename().c_str(), _S("wb"));
if (!datFp)
Log.report(LogVisor::FatalError, _S("unable to write shader cache data at %s"),
m_datFr.filename().c_str());
fwrite(&DAT_MAGIC, 1, 8, datFp);
fwrite(&m_loadedRand, 1, 8, datFp);
fclose(datFp);
m_idxFr.open();
m_datFr.open();
}
void ShaderCacheManager::reload()
{
m_entries.clear();
m_entryLookup.clear();
m_loadedRand = 0;
/* Attempt to open existing index */
m_idxFr.seek(0, Athena::Begin);
m_datFr.seek(0, Athena::Begin);
if (m_idxFr.hasError() || m_datFr.hasError())
{
BootstrapIndex();
return;
}
else
{
uint64_t idxMagic;
size_t rb = m_idxFr.readUBytesToBuf(&idxMagic, 8);
if (rb != 8 || idxMagic != IDX_MAGIC)
{
BootstrapIndex();
return;
}
uint64_t datMagic;
rb = m_datFr.readUBytesToBuf(&datMagic, 8);
if (rb != 8 || datMagic != DAT_MAGIC)
{
BootstrapIndex();
return;
}
uint64_t idxRand, datRand;
rb = m_idxFr.readUBytesToBuf(&idxRand, 8);
size_t rb2 = m_datFr.readUBytesToBuf(&datRand, 8);
if (rb != 8 || rb2 != 8 || idxRand != datRand)
{
BootstrapIndex();
return;
}
m_loadedRand = idxRand;
}
/* Read existing entries */
atUint64 idxCount = m_idxFr.readUint64Big();
if (m_idxFr.position() != 24)
{
BootstrapIndex();
return;
}
m_entries.reserve(idxCount);
m_entryLookup.reserve(idxCount);
for (atUint64 i=0 ; i<idxCount ; ++i)
{
m_entries.emplace_back();
IndexEntry& ent = m_entries.back();
ent.read(m_idxFr);
m_entryLookup[ent.m_hash] = m_entries.size() - 1;
}
}
ShaderCacheManager::CachedData ShaderCacheManager::lookupData(const Hash& hash)
{
auto search = m_entryLookup.find(hash);
if (search == m_entryLookup.cend())
return CachedData();
const IndexEntry& ent = m_entries[search->second];
if (ent.m_compOffset + ent.m_compSize > m_datFr.length())
{
Log.report(LogVisor::Warning, "shader cache not long enough to read entry, might be corrupt");
return CachedData();
}
/* File-streamed decompression */
m_datFr.seek(ent.m_compOffset, Athena::Begin);
CachedData ret(ent.m_hash, ent.m_meta, ent.m_decompSize);
uint8_t compDat[2048];
z_stream z = {};
inflateInit(&z);
z.avail_out = ent.m_decompSize;
z.next_out = ret.m_data.get();
while (z.avail_out)
{
z.avail_in = std::min(size_t(2048), size_t(ent.m_compSize - z.total_in));
m_datFr.readUBytesToBuf(compDat, z.avail_in);
z.next_in = compDat;
inflate(&z, Z_NO_FLUSH);
}
inflateEnd(&z);
return ret;
}
bool ShaderCacheManager::addData(const ShaderTag& tag, const void* data, size_t sz)
{
m_idxFr.close();
m_datFr.close();
/* Perform one-shot buffer compression */
uLong cBound = compressBound(sz);
void* compBuf = malloc(cBound);
if (compress((Bytef*)compBuf, &cBound, (Bytef*)data, sz) != Z_OK)
Log.report(LogVisor::FatalError, "unable to deflate data");
/* Open index for writing (non overwriting) */
Athena::io::FileWriter idxFw(m_idxFr.filename(), false);
if (idxFw.hasError())
Log.report(LogVisor::FatalError, _S("unable to append shader cache index at %s"),
m_idxFr.filename().c_str());
/* Open data for writing (non overwriting) */
Athena::io::FileWriter datFw(m_datFr.filename(), false);
if (datFw.hasError())
Log.report(LogVisor::FatalError, _S("unable to append shader cache data at %s"),
m_datFr.filename().c_str());
size_t targetOffset = 0;
auto search = m_entryLookup.find(tag);
if (search != m_entryLookup.cend())
{
/* Hash already present, attempt to replace data */
IndexEntry& ent = m_entries[search->second];
if (search->second == m_entries.size() - 1)
{
/* Replacing final entry; simply write-over */
ent.m_meta = tag.getMetaData();
ent.m_compSize = cBound;
ent.m_decompSize = sz;
targetOffset = ent.m_compOffset;
idxFw.seek(search->second * 32 + 32);
ent.write(idxFw);
}
else
{
/* Replacing non-final entry; write into available space */
IndexEntry& nent = m_entries[search->second+1];
size_t space = nent.m_compOffset - ent.m_compOffset;
if (cBound <= space)
{
ent.m_meta = tag.getMetaData();
ent.m_compSize = cBound;
ent.m_decompSize = sz;
targetOffset = ent.m_compOffset;
idxFw.seek(search->second * 32 + 32);
ent.write(idxFw);
}
else
{
/* Not enough space; null-entry and add to end */
ent.m_hash = 0;
ent.m_meta = 0;
ent.m_compOffset = 0;
ent.m_compSize = 0;
ent.m_decompSize = 0;
idxFw.seek(search->second * 32 + 32);
ent.write(idxFw);
}
}
}
if (!targetOffset)
{
/* New index entry at end */
idxFw.seek(16, Athena::Begin);
idxFw.writeUint64Big(m_entries.size() + 1);
idxFw.seek(m_entries.size() * 32 + 32, Athena::Begin);
datFw.seek(0, Athena::End);
m_entryLookup[tag] = m_entries.size();
m_entries.emplace_back();
IndexEntry& ent = m_entries.back();
ent.m_hash = tag.val64();
ent.m_meta = tag.getMetaData();
ent.m_compOffset = datFw.position();
ent.m_compSize = cBound;
ent.m_decompSize = sz;
ent.write(idxFw);
datFw.writeUBytes((atUint8*)compBuf, cBound);
}
else
{
/* Reusing index entry and data space */
datFw.seek(targetOffset, Athena::Begin);
datFw.writeUBytes((atUint8*)compBuf, cBound);
}
free(compBuf);
idxFw.close();
datFw.close();
m_idxFr.open();
m_datFr.open();
return true;
}
}
}

4
hecl/test/CMakeLists.txt
View File

@ -1,4 +1,4 @@
add_executable(heclTest WIN32 main.cpp)
target_link_libraries(heclTest
HECLDatabase HECLBackend HECLFrontend HECLBlender HECLCommon AthenaCore
LogVisor Boo ${BOO_SYS_LIBS})
HECLDatabase HECLRuntime HECLBackend HECLFrontend HECLBlender HECLCommon AthenaCore
LogVisor Boo ${ZLIB_LIBRARIES} ${BOO_SYS_LIBS})

16
hecl/test/main.cpp
View File

@ -1,5 +1,6 @@
#include <boo/boo.hpp>
#include <LogVisor/LogVisor.hpp>
#include "HECL/Runtime.hpp"
struct HECLWindowCallback : boo::IWindowCallback
{
@ -10,6 +11,12 @@ struct HECLWindowCallback : boo::IWindowCallback
m_sizeDirty = true;
m_latestSize = rect;
}
bool m_destroyed = false;
void destroyed()
{
m_destroyed = true;
}
};
struct HECLApplicationCallback : boo::IApplicationCallback
@ -19,6 +26,9 @@ struct HECLApplicationCallback : boo::IApplicationCallback
bool m_running = true;
int appMain(boo::IApplication* app)
{
HECL::Runtime::FileStoreManager fileMgr(app->getUniqueName());
HECL::Runtime::ShaderCacheManager shaderMgr(fileMgr);
m_mainWindow = app->newWindow(_S("HECL Test"));
m_mainWindow->setCallback(&m_windowCb);
boo::IGraphicsCommandQueue* gfxQ = m_mainWindow->getCommandQueue();
@ -30,6 +40,12 @@ struct HECLApplicationCallback : boo::IApplicationCallback
{
m_mainWindow->waitForRetrace();
if (m_windowCb.m_destroyed)
{
m_running = false;
break;
}
if (m_windowCb.m_sizeDirty)
{
gfxQ->resizeRenderTexture(renderTex,