Add direct SDNA parser for blend files

hecl/extern/boo

@@ -1 +1 @@
-Subproject commit 7bda89073daeec487f9c82f220f199a687a1826b
+Subproject commit 3b4d7abae6dc0d11cefa6b3aaf545bff90963ce4

hecl/include/hecl/Blender/SDNARead.hpp

@@ -0,0 +1,72 @@
+#pragma once
+#include "hecl/hecl.hpp"
+#include "hecl/FourCC.hpp"
+#include "athena/DNA.hpp"
+#include "athena/MemoryReader.hpp"
+
+namespace hecl::blender
+{
+
+struct SDNABlock : public athena::io::DNA<athena::Little>
+{
+    AT_DECL_DNA
+    DNAFourCC magic;
+    DNAFourCC nameMagic;
+    Value<atUint32> numNames;
+    Vector<String<-1>, AT_DNA_COUNT(numNames)> names;
+    Align<4> align1;
+    DNAFourCC typeMagic;
+    Value<atUint32> numTypes;
+    Vector<String<-1>, AT_DNA_COUNT(numTypes)> types;
+    Align<4> align2;
+    DNAFourCC tlenMagic;
+    Vector<Value<atUint16>, AT_DNA_COUNT(numTypes)> tlens;
+    Align<4> align3;
+    DNAFourCC strcMagic;
+    Value<atUint32> numStrcs;
+    struct SDNAStruct : public athena::io::DNA<athena::Little>
+    {
+        AT_DECL_DNA
+        Value<atUint16> type;
+        Value<atUint16> numFields;
+        struct SDNAField : public athena::io::DNA<athena::Little>
+        {
+            AT_DECL_DNA
+            Value<atUint16> type;
+            Value<atUint16> name;
+            atUint32 offset;
+        };
+        Vector<SDNAField, AT_DNA_COUNT(numFields)> fields;
+
+        void computeOffsets(const SDNABlock& block);
+        const SDNAField* lookupField(const SDNABlock& block, const char* n) const;
+    };
+    Vector<SDNAStruct, AT_DNA_COUNT(numStrcs)> strcs;
+
+    const SDNAStruct* lookupStruct(const char* n, int& idx) const;
+};
+
+struct FileBlock : public athena::io::DNA<athena::Little>
+{
+    AT_DECL_DNA
+    DNAFourCC type;
+    Value<atUint32> size;
+    Value<atUint64> ptr;
+    Value<atUint32> sdnaIdx;
+    Value<atUint32> count;
+};
+
+class SDNARead
+{
+    std::vector<uint8_t> m_data;
+    SDNABlock m_sdnaBlock;
+public:
+    explicit SDNARead(SystemStringView path);
+    operator bool() const { return !m_data.empty(); }
+    const SDNABlock& sdnaBlock() const { return m_sdnaBlock; }
+    void enumerate(const std::function<bool(const FileBlock& block, athena::io::MemoryReader& r)>& func) const;
+};
+
+BlendType GetBlendType(SystemStringView path);
+
+}

hecl/include/hecl/FourCC.hpp

@@ -3,6 +3,7 @@
 #include <cstdint>
 #include <cstddef>
 #include <string>
+#include "athena/DNA.hpp"
 
 namespace hecl
 {
@@ -47,6 +48,32 @@ public:
 };
 #define FOURCC(chars) FourCC(SBIG(chars))
 
+using BigDNA = athena::io::DNA<athena::Big>;
+
+/** FourCC with DNA read/write */
+class DNAFourCC final : public BigDNA, public FourCC
+{
+public:
+    DNAFourCC() : FourCC() {}
+    DNAFourCC(const FourCC& other)
+    : FourCC() {num = other.toUint32();}
+    DNAFourCC(const char* name)
+    : FourCC(name) {}
+    DNAFourCC(uint32_t n)
+    : FourCC(n) {}
+    AT_DECL_EXPLICIT_DNA_YAML
+};
+template <> inline void DNAFourCC::Enumerate<BigDNA::Read>(typename Read::StreamT& r)
+{ r.readUBytesToBuf(fcc, 4); }
+template <> inline void DNAFourCC::Enumerate<BigDNA::Write>(typename Write::StreamT& w)
+{ w.writeUBytes((atUint8*)fcc, 4); }
+template <> inline void DNAFourCC::Enumerate<BigDNA::ReadYaml>(typename ReadYaml::StreamT& r)
+{ std::string rs = r.readString(nullptr); strncpy(fcc, rs.c_str(), 4); }
+template <> inline void DNAFourCC::Enumerate<BigDNA::WriteYaml>(typename WriteYaml::StreamT& w)
+{ w.writeString(nullptr, std::string(fcc, 4)); }
+template <> inline void DNAFourCC::Enumerate<BigDNA::BinarySize>(typename BinarySize::StreamT& s)
+{ s += 4; }
+
 }
 
 namespace std

hecl/lib/Blender/CMakeLists.txt

@@ -1,5 +1,6 @@
 set(BLENDER_SOURCES
     Connection.cpp
+    SDNARead.cpp
     HMDL.cpp)
 
 hecl_add_list(Blender BLENDER_SOURCES)

hecl/lib/Blender/SDNARead.cpp

@@ -0,0 +1,208 @@
+#include "hecl/Blender/SDNARead.hpp"
+#include "athena/FileReader.hpp"
+#include <zlib.h>
+
+namespace hecl::blender
+{
+
+void SDNABlock::SDNAStruct::computeOffsets(const SDNABlock& block)
+{
+    atUint32 offset = 0;
+    for (SDNAField& f : fields)
+    {
+        const auto& name = block.names[f.name];
+        f.offset = offset;
+        if (name.front() == '*')
+        {
+            offset += 8;
+        }
+        else
+        {
+            atUint32 length = block.tlens[f.type];
+            auto bracket = name.find('[');
+            if (bracket != std::string::npos)
+                length *= strtoul(name.data() + bracket + 1, nullptr, 10);
+            offset += length;
+        }
+    }
+}
+
+const SDNABlock::SDNAStruct::SDNAField* SDNABlock::SDNAStruct::lookupField(const SDNABlock& block, const char* n) const
+{
+    for (const SDNAField& field : fields)
+    {
+        const auto& name = block.names[field.name];
+        auto bracket = name.find('[');
+        if (bracket != std::string::npos)
+        {
+            if (!name.compare(0, bracket, n))
+                return &field;
+        }
+        else if (!name.compare(n))
+            return &field;
+    }
+    return nullptr;
+}
+
+const SDNABlock::SDNAStruct* SDNABlock::lookupStruct(const char* n, int& idx) const
+{
+    idx = 0;
+    for (const SDNAStruct& strc : strcs)
+    {
+        const auto& name = types[strc.type];
+        if (!name.compare(n))
+            return &strc;
+        ++idx;
+    }
+    return nullptr;
+}
+
+void SDNARead::enumerate(const std::function<bool(const FileBlock& block, athena::io::MemoryReader& r)>& func) const
+{
+    athena::io::MemoryReader r(m_data.data(), m_data.size());
+    r.seek(12);
+    while (r.position() < r.length())
+    {
+        FileBlock block;
+        block.read(r);
+        if (block.type == FOURCC('ENDB'))
+            break;
+        athena::io::MemoryReader r2(m_data.data() + r.position(), block.size);
+        if (!func(block, r2))
+            break;
+        r.seek(block.size);
+    }
+}
+
+SDNARead::SDNARead(SystemStringView path)
+{
+    athena::io::FileReader r(path);
+    if (r.hasError())
+        return;
+
+    atUint64 length = r.length();
+    char magicBuf[7];
+    r.readUBytesToBuf(magicBuf, 7);
+    r.seek(0, athena::Begin);
+    if (strncmp(magicBuf, "BLENDER", 7))
+    {
+        /* Try gzip decompression */
+        std::unique_ptr<uint8_t[]> compBuf(new uint8_t[4096]);
+        m_data.resize((length * 2 + 4095) & ~4095);
+        z_stream zstrm = {};
+        inflateInit2(&zstrm, 16 + MAX_WBITS);
+        zstrm.next_out = (Bytef*)m_data.data();
+        zstrm.avail_out = m_data.size();
+        zstrm.total_out = 0;
+
+        atUint64 rs;
+        while ((rs = r.readUBytesToBuf(compBuf.get(), 4096)))
+        {
+            int inflateRet;
+            zstrm.next_in = compBuf.get();
+            zstrm.avail_in = rs;
+            while (zstrm.avail_in)
+            {
+                if (!zstrm.avail_out)
+                {
+                    zstrm.avail_out = m_data.size();
+                    m_data.resize(zstrm.avail_out * 2);
+                    zstrm.next_out = (Bytef*)m_data.data() + zstrm.avail_out;
+                }
+                inflateRet = inflate(&zstrm, Z_NO_FLUSH);
+                if (inflateRet == Z_STREAM_END)
+                    break;
+                if (inflateRet != Z_OK)
+                {
+                    inflateEnd(&zstrm);
+                    m_data = std::vector<uint8_t>();
+                    return;
+                }
+            }
+            if (inflateRet == Z_STREAM_END)
+                break;
+        }
+
+        inflateEnd(&zstrm);
+
+        if (strncmp((char*)m_data.data(), "BLENDER", 7))
+        {
+            m_data = std::vector<uint8_t>();
+            return;
+        }
+    }
+    else
+    {
+        m_data.resize(length);
+        r.readUBytesToBuf(m_data.data(), length);
+    }
+
+    enumerate([this](const FileBlock& block, athena::io::MemoryReader& r)
+    {
+        if (block.type == FOURCC('DNA1'))
+        {
+            m_sdnaBlock.read(r);
+            for (SDNABlock::SDNAStruct& s : m_sdnaBlock.strcs)
+                s.computeOffsets(m_sdnaBlock);
+            return false;
+        }
+        return true;
+    });
+}
+
+BlendType GetBlendType(SystemStringView path)
+{
+    SDNARead r(path);
+    if (!r)
+        return BlendType::None;
+
+    int idPropIdx;
+    const auto* idPropStruct = r.sdnaBlock().lookupStruct("IDProperty", idPropIdx);
+    if (!idPropStruct)
+        return BlendType::None;
+    const auto* typeField = idPropStruct->lookupField(r.sdnaBlock(), "type");
+    if (!typeField)
+        return BlendType::None;
+    atUint32 typeOffset = typeField->offset;
+    const auto* nameField = idPropStruct->lookupField(r.sdnaBlock(), "name");
+    if (!nameField)
+        return BlendType::None;
+    atUint32 nameOffset = nameField->offset;
+    const auto* dataField = idPropStruct->lookupField(r.sdnaBlock(), "data");
+    if (!dataField)
+        return BlendType::None;
+    atUint32 dataOffset = dataField->offset;
+
+    int idPropDataIdx;
+    const auto* idPropDataStruct = r.sdnaBlock().lookupStruct("IDPropertyData", idPropDataIdx);
+    if (!idPropDataStruct)
+        return BlendType::None;
+    const auto* valField = idPropDataStruct->lookupField(r.sdnaBlock(), "val");
+    if (!valField)
+        return BlendType::None;
+    atUint32 valOffset = dataOffset + valField->offset;
+
+    BlendType ret = BlendType::None;
+    r.enumerate([idPropIdx, typeOffset, nameOffset, valOffset, &ret](const FileBlock& block, athena::io::MemoryReader& r)
+    {
+        if (block.type == FOURCC('DATA') && block.sdnaIdx == idPropIdx)
+        {
+            r.seek(typeOffset, athena::Begin);
+            if (r.readUByte() != 1)
+                return true;
+
+            r.seek(nameOffset, athena::Begin);
+            if (r.readString() != "hecl_type")
+                return true;
+
+            r.seek(valOffset, athena::Begin);
+            ret = BlendType(r.readUint32Little());
+            return false;
+        }
+        return true;
+    });
+
+    return ret;
+}
+
+}

hecl/lib/CMakeLists.txt

@@ -20,6 +20,7 @@ endif()
 atdna(atdna_HMDLMeta.cpp ../include/hecl/HMDLMeta.hpp)
 atdna(atdna_Frontend.cpp ../include/hecl/Frontend.hpp)
 atdna(atdna_CVar.cpp ../include/hecl/CVar.hpp)
+atdna(atdna_SDNARead.cpp ../include/hecl/Blender/SDNARead.hpp)
 
 if("${CMAKE_BUILD_TYPE}" STREQUAL "Release" OR "${CMAKE_BUILD_TYPE}" STREQUAL "RelWithDebInfo")
   add_definitions(-DHECL_MULTIPROCESSOR)
@@ -41,6 +42,7 @@ set(HECL_HEADERS
     ../include/hecl/Backend/HLSL.hpp
     ../include/hecl/Backend/Metal.hpp
     ../include/hecl/Blender/Connection.hpp
+    ../include/hecl/Blender/SDNARead.hpp
     ../include/hecl/Blender/Token.hpp
     ../include/hecl/SteamFinder.hpp
     ../include/hecl/Frontend.hpp
@@ -78,6 +80,7 @@ add_library(hecl-full
             ${FRONTEND_SOURCES}
             ${RUNTIME_SOURCES}
             ${BLENDER_SOURCES}
+            atdna_SDNARead.cpp
             ${COMMON_SOURCES}
             ${HECL_HEADERS}
             ${PLAT_SRCS})