mesh cook bug fixes

hecl/blender/BlenderConnection.cpp

@@ -342,13 +342,13 @@ BlenderConnection::~BlenderConnection()
     _closePipe();
 }
 
-static std::string BlendTypeStrs[] =
+static const char* BlendTypeStrs[] =
 {
     "NONE",
     "MESH",
     "ACTOR",
     "AREA",
-    ""
+    nullptr
 };
 
 bool BlenderConnection::createBlend(const SystemString& path, BlendType type)
@@ -377,9 +377,12 @@ BlenderConnection::BlendType BlenderConnection::getBlendType()
     char lineBuf[256];
     _readLine(lineBuf, sizeof(lineBuf));
     unsigned idx = 0;
-    while (BlendTypeStrs[idx].size())
+    while (BlendTypeStrs[idx])
-        if (!BlendTypeStrs[idx].compare(lineBuf))
+    {
+        if (!strcmp(BlendTypeStrs[idx], lineBuf))
             return BlendType(idx);
+        ++idx;
+    }
     return TypeNone;
 }
 
@@ -456,7 +459,7 @@ void BlenderConnection::PyOutStream::linkBlend(const std::string& target,
            objName.c_str(), objName.c_str(), target.c_str(), objName.c_str());
 }
 
-BlenderConnection::DataStream::Mesh::Mesh(BlenderConnection& conn, int maxSkinBanks)
+BlenderConnection::DataStream::Mesh::Mesh(BlenderConnection& conn, int skinSlotCount)
 {
     uint32_t matSetCount;
     conn._readBuf(&matSetCount, 4);
@@ -484,22 +487,16 @@ BlenderConnection::DataStream::Mesh::Mesh(BlenderConnection& conn, int maxSkinBa
         norm.emplace_back(conn);
 
     conn._readBuf(&colorLayerCount, 4);
-    for (int i=0 ; i<colorLayerCount ; ++i)
-    {
     conn._readBuf(&count, 4);
-        color[i].reserve(count);
+    color.reserve(count);
-        for (int j=0 ; j<count ; ++j)
+    for (int i=0 ; i<count ; ++i)
-            color[i].emplace_back(conn);
+        color.emplace_back(conn);
-    }
 
     conn._readBuf(&uvLayerCount, 4);
-    for (int i=0 ; i<uvLayerCount ; ++i)
-    {
     conn._readBuf(&count, 4);
-        uv[i].reserve(count);
+    uv.reserve(count);
-        for (int j=0 ; j<count ; ++j)
+    for (int i=0 ; i<count ; ++i)
-            uv[i].emplace_back(conn);
+        uv.emplace_back(conn);
-    }
 
     conn._readBuf(&count, 4);
     boneNames.reserve(count);
@@ -523,18 +520,36 @@ BlenderConnection::DataStream::Mesh::Mesh(BlenderConnection& conn, int maxSkinBa
             binds.emplace_back(conn);
     }
 
+    /* Assume 16 islands per material for reserve */
+    if (materialSets.size())
+        surfaces.reserve(materialSets.front().size() * 16);
     uint8_t isSurf;
     conn._readBuf(&isSurf, 1);
     while (isSurf)
     {
-        surfaces.emplace_back(conn, *this);
+        surfaces.emplace_back(conn, *this, skinSlotCount);
         conn._readBuf(&isSurf, 1);
     }
 
-    /* Resolve skin banks here */
+    /* Connect skinned verts to bank slots */
     if (boneNames.size())
+    {
         for (Surface& surf : surfaces)
-            skinBanks.addSurface(surf);
+        {
+            std::vector<uint32_t>& bank = skinBanks.banks[surf.skinBankIdx];
+            for (Surface::Vert& vert : surf.verts)
+            {
+                for (uint32_t i=0 ; i<bank.size() ; ++i)
+                {
+                    if (bank[i] == vert.iSkin)
+                    {
+                        vert.iBankSkin = i;
+                        break;
+                    }
+                }
+            }
+        }
+    }
 }
 
 BlenderConnection::DataStream::Mesh::Material::Material
@@ -555,10 +570,14 @@ BlenderConnection::DataStream::Mesh::Material::Material
 }
 
 BlenderConnection::DataStream::Mesh::Surface::Surface
-(BlenderConnection& conn, const Mesh& parent)
+(BlenderConnection& conn, Mesh& parent, int skinSlotCount)
 : centroid(conn), materialIdx(conn), aabbMin(conn), aabbMax(conn),
   reflectionNormal(conn)
 {
+    uint32_t countEstimate;
+    conn._readBuf(&countEstimate, 4);
+    verts.reserve(countEstimate);
+
     uint8_t isVert;
     conn._readBuf(&isVert, 1);
     while (isVert)
@@ -566,6 +585,9 @@ BlenderConnection::DataStream::Mesh::Surface::Surface
         verts.emplace_back(conn, parent);
         conn._readBuf(&isVert, 1);
     }
+
+    if (parent.boneNames.size())
+        skinBankIdx = parent.skinBanks.addSurface(*this, skinSlotCount);
 }
 
 BlenderConnection::DataStream::Mesh::Surface::Vert::Vert
@@ -578,6 +600,63 @@ BlenderConnection::DataStream::Mesh::Surface::Vert::Vert
     if (parent.uvLayerCount)
         conn._readBuf(iUv, 4 * parent.uvLayerCount);
     conn._readBuf(&iSkin, 4);
+    if (parent.pos.size() == 1250)
+        printf("");
+}
+
+static bool VertInBank(const std::vector<uint32_t>& bank, uint32_t sIdx)
+{
+    for (uint32_t idx : bank)
+        if (sIdx == idx)
+            return true;
+    return false;
+}
+
+uint32_t BlenderConnection::DataStream::Mesh::SkinBanks::addSurface
+(const Surface& surf, int skinSlotCount)
+{
+    if (banks.empty())
+        addSkinBank(skinSlotCount);
+    std::vector<uint32_t> toAdd;
+    toAdd.reserve(skinSlotCount);
+    std::vector<std::vector<uint32_t>>::iterator bankIt = banks.begin();
+    for (;;)
+    {
+        bool done = true;
+        for (; bankIt != banks.end() ; ++bankIt)
+        {
+            std::vector<uint32_t>& bank = *bankIt;
+            done = true;
+            for (const Surface::Vert& v : surf.verts)
+            {
+                if (!VertInBank(bank, v.iSkin) && !VertInBank(toAdd, v.iSkin))
+                {
+                    toAdd.push_back(v.iSkin);
+                    if (bank.size() + toAdd.size() > skinSlotCount)
+                    {
+                        toAdd.clear();
+                        done = false;
+                        break;
+                    }
+                }
+            }
+            if (toAdd.size())
+            {
+                for (uint32_t a : toAdd)
+                    bank.push_back(a);
+                toAdd.clear();
+            }
+            if (done)
+                return uint32_t(bankIt - banks.begin());
+        }
+        if (!done)
+        {
+            bankIt = addSkinBank(skinSlotCount);
+            continue;
+        }
+        break;
+    }
+    return uint32_t(-1);
 }
 
 void BlenderConnection::quitBlender()

hecl/blender/BlenderConnection.hpp

@@ -310,9 +310,9 @@ public:
             std::vector<Vector3f> pos;
             std::vector<Vector3f> norm;
             uint32_t colorLayerCount = 0;
-            std::vector<Vector3f> color[4];
+            std::vector<Vector3f> color;
             uint32_t uvLayerCount = 0;
-            std::vector<Vector2f> uv[8];
+            std::vector<Vector2f> uv;
 
             /* Skinning data */
             std::vector<std::string> boneNames;
@@ -342,33 +342,51 @@ public:
                     uint32_t iColor[4] = {uint32_t(-1)};
                     uint32_t iUv[8] = {uint32_t(-1)};
                     uint32_t iSkin;
+                    uint32_t iBankSkin = -1;
 
                     Vert(BlenderConnection& conn, const Mesh& parent);
                 };
                 std::vector<Vert> verts;
 
-                Surface(BlenderConnection& conn, const Mesh& parent);
+                Surface(BlenderConnection& conn, Mesh& parent, int skinSlotCount);
             };
             std::vector<Surface> surfaces;
 
-            class SkinBanks
+            struct SkinBanks
             {
                 std::vector<std::vector<uint32_t>> banks;
-            public:
+                std::vector<std::vector<uint32_t>>::iterator addSkinBank(int skinSlotCount)
-                uint32_t addSurface(const Surface& surf)
                 {
-                    return 0;
+                    banks.emplace_back();
+                    banks.back().reserve(skinSlotCount);
+                    return banks.end() - 1;
                 }
+                uint32_t addSurface(const Surface& surf, int skinSlotCount);
             } skinBanks;
 
-            Mesh(BlenderConnection& conn, int maxSkinBanks);
+            Mesh(BlenderConnection& conn, int skinSlotCount);
         };
 
-        /* Compile mesh by name */
+        /* Compile mesh by context */
-        Mesh compileMesh(const std::string& name, int maxSkinBanks=10)
+        Mesh compileMesh(int skinSlotCount=10)
         {
             char req[128];
-            snprintf(req, 128, "MESHCOMPILE %s %d", name.c_str(), maxSkinBanks);
+            snprintf(req, 128, "MESHCOMPILE %d", skinSlotCount);
+            m_parent->_writeLine(req);
+
+            char readBuf[256];
+            m_parent->_readLine(readBuf, 256);
+            if (strcmp(readBuf, "OK"))
+                BlenderLog.report(LogVisor::FatalError, "unable to cook mesh: %s", readBuf);
+
+            return Mesh(*m_parent, skinSlotCount);
+        }
+
+        /* Compile mesh by name */
+        Mesh compileMesh(const std::string& name, int skinSlotCount=10)
+        {
+            char req[128];
+            snprintf(req, 128, "MESHCOMPILENAME %s %d", name.c_str(), skinSlotCount);
             m_parent->_writeLine(req);
 
             char readBuf[256];
@@ -376,14 +394,14 @@ public:
             if (strcmp(readBuf, "OK"))
                 BlenderLog.report(LogVisor::FatalError, "unable to cook mesh '%s': %s", name.c_str(), readBuf);
 
-            return Mesh(*m_parent, maxSkinBanks);
+            return Mesh(*m_parent, skinSlotCount);
         }
 
         /* Compile all meshes into one */
-        Mesh compileAllMeshes(int maxSkinBanks=10)
+        Mesh compileAllMeshes(int skinSlotCount=10)
         {
             char req[128];
-            snprintf(req, 128, "MESHCOMPILEALL %d", maxSkinBanks);
+            snprintf(req, 128, "MESHCOMPILEALL %d", skinSlotCount);
             m_parent->_writeLine(req);
 
             char readBuf[256];
@@ -391,7 +409,7 @@ public:
             if (strcmp(readBuf, "OK"))
                 BlenderLog.report(LogVisor::FatalError, "unable to cook all meshes: %s", readBuf);
 
-            return Mesh(*m_parent, maxSkinBanks);
+            return Mesh(*m_parent, skinSlotCount);
         }
     };
     DataStream beginData()

hecl/blender/hecl/hmdl/HMDLMesh.py

@@ -8,17 +8,18 @@ from mathutils import Vector
 
 # Class for building unique sets of vertex attributes for VBO generation
 class VertPool:
-    pos = {}
-    norm = {}
-    skin = {}
-    color = []
-    uv = []
-    dlay = None
-    clays = []
-    ulays = []
 
     # Initialize hash-unique index for each available attribute
     def __init__(self, bm):
+        self.pos = {}
+        self.norm = {}
+        self.skin = {}
+        self.color = {}
+        self.uv = {}
+        self.dlay = None
+        self.clays = []
+        self.ulays = []
+
         dlay = None
         if len(bm.verts.layers.deform):
             dlay = bm.verts.layers.deform[0]
@@ -27,13 +28,11 @@ class VertPool:
         clays = []
         for cl in range(len(bm.loops.layers.color)):
             clays.append(bm.loops.layers.color[cl])
-            self.color.append([])
         self.clays = clays
 
         ulays = []
         for ul in range(len(bm.loops.layers.uv)):
             ulays.append(bm.loops.layers.uv[ul])
-            self.uv.append([])
         self.ulays = ulays
 
         # Per-vert pool attributes
@@ -54,12 +53,12 @@ class VertPool:
             for l in f.loops:
                 for cl in range(len(clays)):
                     cf = l[clays[cl]].copy().freeze()
-                    if cf not in self.color[cl]:
+                    if cf not in self.color:
-                        self.color[cl][cf] = len(self.color[cl])
+                        self.color[cf] = len(self.color)
                 for ul in range(len(ulays)):
                     uf = l[ulays[ul]].uv.copy().freeze()
-                    if uf not in self.uv[ul]:
+                    if uf not in self.uv:
-                        self.uv[ul][uf] = len(self.uv[ul])
+                        self.uv[uf] = len(self.uv)
 
     def write_out(self, writebuffunc, vert_groups):
         writebuffunc(struct.pack('I', len(self.pos)))
@@ -70,16 +69,12 @@ class VertPool:
         for n in sorted(self.norm.items(), key=operator.itemgetter(1)):
             writebuffunc(struct.pack('fff', n[0][0], n[0][1], n[0][2]))
 
-        writebuffunc(struct.pack('I', len(self.color)))
+        writebuffunc(struct.pack('II', len(self.clays), len(self.color)))
-        for clay in self.color:
+        for c in sorted(self.color.items(), key=operator.itemgetter(1)):
-            writebuffunc(struct.pack('I', len(clay)))
-            for c in sorted(clay.items(), key=operator.itemgetter(1)):
             writebuffunc(struct.pack('fff', c[0][0], c[0][1], c[0][2]))
 
-        writebuffunc(struct.pack('I', len(self.uv)))
+        writebuffunc(struct.pack('II', len(self.ulays), len(self.uv)))
-        for ulay in self.uv:
+        for u in sorted(self.uv.items(), key=operator.itemgetter(1)):
-            writebuffunc(struct.pack('I', len(ulay)))
-            for u in sorted(ulay.items(), key=operator.itemgetter(1)):
             writebuffunc(struct.pack('ff', u[0][0], u[0][1]))
 
         writebuffunc(struct.pack('I', len(vert_groups)))
@@ -90,23 +85,12 @@ class VertPool:
         for s in sorted(self.skin.items(), key=operator.itemgetter(1)):
             entries = s[0]
             writebuffunc(struct.pack('I', len(entries)))
+            if len(entries):
+                total_len = 0.0
                 for ent in entries:
-                writebuffunc(struct.pack('If', ent[0], ent[1]))
+                    total_len += ent[1]
-
+                for ent in entries:
-    def set_bm_layers(self, bm):
+                    writebuffunc(struct.pack('If', ent[0], ent[1] / total_len))
-        self.dlay = None
-        if len(bm.verts.layers.deform):
-            self.dlay = bm.verts.layers.deform[0]
-
-        clays = []
-        for cl in range(len(bm.loops.layers.color)):
-            clays.append(bm.loops.layers.color[cl])
-        self.clays = clays
-
-        ulays = []
-        for ul in range(len(bm.loops.layers.uv)):
-            ulays.append(bm.loops.layers.uv[ul])
-        self.ulays = ulays
 
     def get_pos_idx(self, vert):
         pf = vert.co.copy().freeze()
@@ -117,24 +101,29 @@ class VertPool:
         return self.norm[nf]
 
     def get_skin_idx(self, vert):
+        if not self.dlay:
+            return 0
         sf = tuple(sorted(vert[self.dlay].items()))
         return self.skin[sf]
 
     def get_color_idx(self, loop, cidx):
-        cf = tuple(sorted(loop[self.clays[cidx]].items()))
+        cf = loop[self.clays[cidx]].copy().freeze()
-        return self.color[cidx][cf]
+        return self.color[cf]
 
     def get_uv_idx(self, loop, uidx):
-        uf = tuple(sorted(loop[self.ulays[uidx]].items()))
+        uf = loop[self.ulays[uidx]].uv.copy().freeze()
-        return self.uv[uidx][uf]
+        return self.uv[uf]
 
     def loop_out(self, writebuffunc, loop):
-        writebuffunc(struct.pack('BII', 1, self.get_pos_idx(loop.vert), self.get_norm_idx(loop.vert)))
+        writebuffunc(struct.pack('B', 1))
+        writebuffunc(struct.pack('II', self.get_pos_idx(loop.vert), self.get_norm_idx(loop.vert)))
         for cl in range(len(self.clays)):
-            writebuffunc('I', self.get_color_idx(loop, cl))
+            writebuffunc(struct.pack('I', self.get_color_idx(loop, cl)))
         for ul in range(len(self.ulays)):
-            writebuffunc('I', self.get_uv_idx(loop, ul))
+            writebuffunc(struct.pack('I', self.get_uv_idx(loop, ul)))
-        writebuffunc(struct.pack('I', self.get_skin_idx(loop.vert)))
+        sp = struct.pack('I', self.get_skin_idx(loop.vert))
+        print(sp)
+        writebuffunc(sp)
 
 def recursive_faces_islands(dlay, list_out, rem_list, skin_slot_set, skin_slot_count, face):
     if face not in rem_list:
@@ -250,7 +239,7 @@ def stripify_primitive(writebuffunc, vert_pool, prim_faces, last_loop, last_idx)
     return last_loop, last_idx
 
 
-def write_out_surface(writebuffunc, vert_pool, bm, island_faces, mat_idx):
+def write_out_surface(writebuffunc, vert_pool, island_faces, mat_idx):
 
     # Centroid of surface
     centroid = Vector()
@@ -282,6 +271,10 @@ def write_out_surface(writebuffunc, vert_pool, bm, island_faces, mat_idx):
     avg_norm.normalize()
     writebuffunc(struct.pack('fff', avg_norm[0], avg_norm[1], avg_norm[2]))
 
+    # Count estimate
+    writebuffunc(struct.pack('I', len(island_faces) * 3))
+    print('EST', len(island_faces) * 3)
+
     # Verts themselves
     last_loop = None
     last_idx = 0

hecl/blender/hecl/hmdl/HMDLShader.py

@@ -34,7 +34,7 @@ def recursive_color_trace(mat_obj, mesh_obj, tex_list, node, socket=None):
         if node.inputs['Color'].is_linked:
             return recursive_color_trace(mat_obj, mesh_obj, tex_list, node.inputs['Color'].links[0].from_node, node.inputs['Color'].links[0].from_socket)
         else:
-            return 'vec3(%f, %f, %f)' % (node.inputs['Color'].default_value[0],
+            return 'vec3(%g,%g,%g)' % (node.inputs['Color'].default_value[0],
                                        node.inputs['Color'].default_value[1],
                                        node.inputs['Color'].default_value[2])
 
@@ -43,14 +43,14 @@ def recursive_color_trace(mat_obj, mesh_obj, tex_list, node, socket=None):
         if node.inputs[1].is_linked:
             a_input = recursive_color_trace(mat_obj, mesh_obj, tex_list, node.inputs[1].links[0].from_node, node.inputs[1].links[0].from_socket)
         else:
-            a_input = 'vec3(%f, %f, %f)' % (node.inputs[1].default_value[0],
+            a_input = 'vec3(%g,%g,%g)' % (node.inputs[1].default_value[0],
                                           node.inputs[1].default_value[1],
                                           node.inputs[1].default_value[2])
 
         if node.inputs[2].is_linked:
             b_input = recursive_color_trace(mat_obj, mesh_obj, tex_list, node.inputs[2].links[0].from_node, node.inputs[2].links[0].from_socket)
         else:
-            b_input = 'vec3(%f, %f, %f)' % (node.inputs[2].default_value[0],
+            b_input = 'vec3(%g,%g,%g)' % (node.inputs[2].default_value[0],
                                           node.inputs[2].default_value[1],
                                           node.inputs[2].default_value[2])
 
@@ -83,10 +83,10 @@ def recursive_color_trace(mat_obj, mesh_obj, tex_list, node, socket=None):
                 if ncomps > 1:
                     matrix_str += 'vec%d(' % ncomps
                     for c in ncomps-1:
-                        matrix_str += '%f, ' % soc.default_value[c]
+                        matrix_str += '%g, ' % soc.default_value[c]
-                    matrix_str += '%f)' % soc.default_value[ncomps-1]
+                    matrix_str += '%g)' % soc.default_value[ncomps-1]
                 else:
-                    matrix_str += '%f' % soc.default_value
+                    matrix_str += '%g' % soc.default_value
 
                 if s == len(soc_from.node.inputs)-2:
                     matrix_str += ')'
@@ -108,20 +108,20 @@ def recursive_color_trace(mat_obj, mesh_obj, tex_list, node, socket=None):
         # Resolve map and matrix index
         node_label = soc_from.node.label
         if not matrix_str and node_label.startswith('MTX_'):
-            matrix_str = 'HECLTexMtx(%%s, %d)' % int(node_label[4:])
+            matrix_str = 'TexMtx(%%s, %d)' % int(node_label[4:])
 
         if soc_from.name == 'UV':
             uv_name = soc_from.node.uv_layer
             uv_idx = mesh_obj.data.uv_layers.find(uv_name)
             if uv_idx == -1:
                 raise RuntimeError('UV Layer "%s" doesn\'t exist' % uv_name)
-            uvsource_str = 'HECLUV(%d)' % uv_idx
+            uvsource_str = 'UV(%d)' % uv_idx
 
         elif soc_from.name == 'Normal':
-            uvsource_str = 'HECLNormal()'
+            uvsource_str = 'Normal()'
 
         elif soc_from.name == 'View':
-            uvsource_str = 'HECLView()'
+            uvsource_str = 'View()'
 
         else:
             raise RuntimeError("Only the 'UV', 'Normal' and 'View' sockets may be used from 'Geometry' nodes")
@@ -148,7 +148,7 @@ def recursive_color_trace(mat_obj, mesh_obj, tex_list, node, socket=None):
                 if input.is_linked:
                     group_str += recursive_color_trace(mat_obj, mesh_obj, tex_list, input.links[0].from_node, input.links[0].from_socket)
                 else:
-                    group_str += 'vec3(%f, %f, %f)' % (input.default_value[0],
+                    group_str += 'vec3(%g,%g,%g)' % (input.default_value[0],
                                                      input.default_value[1],
                                                      input.default_value[2])
                 did_first = True
@@ -159,16 +159,18 @@ def recursive_color_trace(mat_obj, mesh_obj, tex_list, node, socket=None):
 
         if node.label.startswith('DYNAMIC_'):
             dynamic_index = int(node.label[8:])
-            return 'HECLColorReg(%d)' % dynamic_index
+            return 'ColorReg(%d)' % dynamic_index
 
-        return '%f' % node.outputs['Color'].default_value
+        return 'vec3(%g,%g,%g)' % (node.outputs['Color'].default_value[0],
+                                   node.outputs['Color'].default_value[1],
+                                   node.outputs['Color'].default_value[2])
 
     elif node.type == 'MATERIAL':
         
         if mat_obj.use_shadeless:
             return 'vec3(1.0)'
         else:
-            return 'HECLLighting()'
+            return 'Lighting()'
 
     else:
         raise RuntimeError("HMDL is unable to process '{0}' shader nodes in '{1}'".format(node.type, mat_obj.name))
@@ -182,19 +184,19 @@ def recursive_alpha_trace(mat_obj, mesh_obj, tex_list, node, socket=None):
         if node.inputs['Alpha'].is_linked:
             return recursive_alpha_trace(mat_obj, mesh_obj, tex_list, node.inputs['Alpha'].links[0].from_node, node.inputs['Alpha'].links[0].from_socket)
         else:
-            return '%f' % node.inputs['Alpha'].default_value
+            return '%g' % node.inputs['Alpha'].default_value
     
     elif node.type == 'MATH':
     
         if node.inputs[0].is_linked:
             a_input = recursive_alpha_trace(mat_obj, mesh_obj, tex_list, node.inputs[0].links[0].from_node, node.inputs[0].links[0].from_socket)
         else:
-            a_input = '%f' % node.inputs[0].default_value
+            a_input = '%g' % node.inputs[0].default_value
 
         if node.inputs[1].is_linked:
             b_input = recursive_alpha_trace(mat_obj, mesh_obj, tex_list, node.inputs[1].links[0].from_node, node.inputs[1].links[0].from_socket)
         else:
-            b_input = '%f' % node.inputs[1].default_value
+            b_input = '%g' % node.inputs[1].default_value
         
         if node.operation == 'MULTIPLY':
             return '(%s * %s)' % (a_input, b_input)
@@ -225,10 +227,10 @@ def recursive_alpha_trace(mat_obj, mesh_obj, tex_list, node, socket=None):
                 if ncomps > 1:
                     matrix_str += 'vec%d(' % ncomps
                     for c in ncomps-1:
-                        matrix_str += '%f, ' % soc.default_value[c]
+                        matrix_str += '%g, ' % soc.default_value[c]
-                    matrix_str += '%f)' % soc.default_value[ncomps-1]
+                    matrix_str += '%g)' % soc.default_value[ncomps-1]
                 else:
-                    matrix_str += '%f' % soc.default_value
+                    matrix_str += '%g' % soc.default_value
 
                 if s == len(soc_from.node.inputs)-2:
                     matrix_str += ')'
@@ -250,20 +252,20 @@ def recursive_alpha_trace(mat_obj, mesh_obj, tex_list, node, socket=None):
         # Resolve map and matrix index
         node_label = soc_from.node.label
         if not matrix_str and node_label.startswith('MTX_'):
-            matrix_str = 'HECLTexMtx(%%s, %d)' % int(node_label[4:])
+            matrix_str = 'TexMtx(%%s, %d)' % int(node_label[4:])
 
         if soc_from.name == 'UV':
             uv_name = soc_from.node.uv_layer
             uv_idx = mesh_obj.data.uv_layers.find(uv_name)
             if uv_idx == -1:
                 raise RuntimeError('UV Layer "%s" doesn\'t exist' % uv_name)
-            uvsource_str = 'HECLUV(%d)' % uv_idx
+            uvsource_str = 'UV(%d)' % uv_idx
             
         elif soc_from.name == 'Normal':
-            uvsource_str = 'HECLNormal()'
+            uvsource_str = 'Normal()'
 
         elif soc_from.name == 'View':
-            uvsource_str = 'HECLView()'
+            uvsource_str = 'View()'
 
         else:
             raise RuntimeError("Only the 'UV', 'Normal' and 'View' sockets may be used from 'Geometry' nodes")
@@ -286,7 +288,7 @@ def recursive_alpha_trace(mat_obj, mesh_obj, tex_list, node, socket=None):
                 if input.is_linked:
                     group_str += recursive_alpha_trace(mat_obj, mesh_obj, tex_list, input.links[0].from_node, input.links[0].from_socket)
                 else:
-                    group_str += '%f' % input.default_value
+                    group_str += '%g' % input.default_value
                 did_first = True
         group_str += ')'
         return group_str
@@ -295,9 +297,9 @@ def recursive_alpha_trace(mat_obj, mesh_obj, tex_list, node, socket=None):
 
         if node.label.startswith('DYNAMIC_'):
             dynamic_index = int(node.label[8:])
-            return 'HECLColorReg(%d).a' % dynamic_index
+            return 'ColorReg(%d).a' % dynamic_index
 
-        return '%f' % node.outputs['Value'].default_value
+        return '%g' % node.outputs['Value'].default_value
 
     elif node.type == 'MATERIAL':
     

hecl/blender/hecl/hmdl/__init__.py

@@ -25,28 +25,6 @@ import struct, bpy, bmesh
 from mathutils import Vector
 from . import HMDLShader, HMDLSkin, HMDLMesh
 
-def get_3d_context(object_):
-    window = bpy.context.window_manager.windows[0]
-    screen = window.screen
-    for area in screen.areas:
-        if area.type == "VIEW_3D":
-            area3d = area
-            break
-    for region in area3d.regions:
-        if region.type == "WINDOW":
-            region3d = region
-            break
-    override = {
-        "window": window,
-        "screen": screen,
-        "area": area3d,
-        "region": region3d,
-        "object": object_
-    }
-    
-    return override
-
-
 # Generate Skeleton Info structure (free-form tree structure)
 def generate_skeleton_info(armature, endian_char='<'):
     
@@ -95,13 +73,6 @@ def cook(writebuffunc, mesh_obj, max_skin_banks):
     if mesh_obj.type != 'MESH':
         raise RuntimeError("%s is not a mesh" % mesh_obj.name)
     
-    # Normalize all vertex weights
-    override = get_3d_context(mesh_obj)
-    try:
-        bpy.ops.object.vertex_group_normalize_all(override, lock_active=False)
-    except:
-        pass
-    
     # Copy mesh (and apply mesh modifiers)
     copy_name = mesh_obj.name + "_hmdltri"
     copy_mesh = bpy.data.meshes.new(copy_name)
@@ -130,14 +101,14 @@ def cook(writebuffunc, mesh_obj, max_skin_banks):
 
     # Generate shaders
     if mesh_obj.data.hecl_material_count > 0:
-        writebuffunc(struct.pack('I', len(mesh_obj.data.hecl_material_count)))
+        writebuffunc(struct.pack('I', mesh_obj.data.hecl_material_count))
         for grp_idx in range(mesh_obj.data.hecl_material_count):
             writebuffunc(struct.pack('I', len(sorted_material_idxs)))
             for mat_idx in sorted_material_idxs:
                 found = False
                 for mat in bpy.data.materials:
                     if mat.name.endswith('_%u_%u' % (grp_idx, mat_idx)):
-                        hecl_str, texs = hmdl_shader.shader(mat, mesh_obj, bpy.data.filepath)
+                        hecl_str, texs = HMDLShader.shader(mat, mesh_obj)
                         writebuffunc((hecl_str + '\n').encode())
                         writebuffunc(struct.pack('I', len(texs)))
                         for tex in texs:
@@ -150,7 +121,7 @@ def cook(writebuffunc, mesh_obj, max_skin_banks):
         writebuffunc(struct.pack('II', 1, len(sorted_material_idxs)))
         for mat_idx in sorted_material_idxs:
             mat = mesh_obj.data.materials[mat_idx]
-            hecl_str, texs = hmdl_shader.shader(mat, mesh_obj, bpy.data.filepath)
+            hecl_str, texs = HMDLShader.shader(mat, mesh_obj)
             writebuffunc((hecl_str + '\n').encode())
             writebuffunc(struct.pack('I', len(texs)))
             for tex in texs:
@@ -159,44 +130,36 @@ def cook(writebuffunc, mesh_obj, max_skin_banks):
     # Output vert pool
     vert_pool.write_out(writebuffunc, mesh_obj.vertex_groups)
 
+    dlay = None
+    if len(bm_master.verts.layers.deform):
+        dlay = bm_master.verts.layers.deform[0]
+
     # Generate island meshes
     for mat_idx in sorted_material_idxs:
-        # Make special version of mesh with just the relevant material
+        mat_faces_rem = []
-        bm = bm_master.copy()
+        for face in bm_master.faces:
-        to_remove = []
+            if face.material_index == mat_idx:
-        for face in bm.faces:
+                mat_faces_rem.append(face)
-            if face.material_index != mat_idx:
-                to_remove.append(face)
-        bmesh.ops.delete(bm, geom=to_remove, context=5)
-        vert_pool.set_bm_layers(bm)
-
-        dlay = None
-        if len(bm.verts.layers.deform):
-            dlay = bm.verts.layers.deform[0]
-
-        mat_faces_rem = list(bm.faces)
         while len(mat_faces_rem):
             the_list = []
             skin_slot_set = set()
             HMDLMesh.recursive_faces_islands(dlay, the_list, mat_faces_rem, skin_slot_set,
                                              max_skin_banks, mat_faces_rem[0])
             writebuffunc(struct.pack('B', 1))
-            HMDLMesh.write_out_surface(writebuffunc, vert_pool, bm, the_list, mat_idx)
+            HMDLMesh.write_out_surface(writebuffunc, vert_pool, the_list, mat_idx)
-
-        bm.to_mesh(mesh)
-        bm.free()
 
     # No more surfaces
     writebuffunc(struct.pack('B', 0))
 
     # Filter out useless AABB points and generate data array
-    aabb = bytearray()
+    #aabb = bytearray()
-    for comp in copy_obj.bound_box[0]:
+    #for comp in copy_obj.bound_box[0]:
-        aabb += struct.pack('f', comp)
+    #    aabb += struct.pack('f', comp)
-    for comp in copy_obj.bound_box[6]:
+    #for comp in copy_obj.bound_box[6]:
-        aabb += struct.pack('f', comp)
+    #    aabb += struct.pack('f', comp)
 
     # Delete copied mesh from scene
+    bm_master.free()
     bpy.context.scene.objects.unlink(copy_obj)
     bpy.data.objects.remove(copy_obj)
     bpy.data.meshes.remove(copy_mesh)

hecl/blender/hecl_blendershell.py

@@ -24,11 +24,12 @@ def readpipeline():
         retval += ch
 
 def writepipeline(linebytes):
+    print('LINE', linebytes)
     os.write(writefd, linebytes + b'\n')
 
 def writepipebuf(linebytes):
-    writepipeline(b'BUF')
+    #print('BUF', linebytes)
-    os.write(writefd, struct.pack('I', len(linebytes)) + linebytes)
+    os.write(writefd, linebytes)
 
 def quitblender():
     writepipeline(b'QUITTING')
@@ -134,7 +135,8 @@ def dataout_loop():
         print(cmdargs)
 
         if cmdargs[0] == 'DATAEND':
-            break
+            writepipeline(b'DONE')
+            return
 
         elif cmdargs[0] == 'MESHLIST':
             for meshobj in bpy.data.objects:
@@ -142,13 +144,25 @@ def dataout_loop():
                     writepipeline(meshobj.name.encode())
 
         elif cmdargs[0] == 'MESHCOMPILE':
+            maxSkinBanks = int(cmdargs[1])
+
+            meshName = bpy.context.scene.hecl_mesh_obj
+            if meshName not in bpy.data.objects:
+                writepipeline(('mesh %s not found' % meshName).encode())
+                continue
+
+            writepipeline(b'OK')
+            hecl.hmdl.cook(writepipebuf, bpy.data.objects[meshName], maxSkinBanks)
+
+        elif cmdargs[0] == 'MESHCOMPILENAME':
             meshName = cmdargs[1]
             maxSkinBanks = int(cmdargs[2])
 
             if meshName not in bpy.data.objects:
-                writepipeline(b'mesh not found')
+                writepipeline(('mesh %s not found' % meshName).encode())
                 continue
 
+            writepipeline(b'OK')
             hecl.hmdl.cook(writepipebuf, bpy.data.objects[meshName], maxSkinBanks)
 
         elif cmdargs[0] == 'MESHCOMPILEALL':
@@ -162,6 +176,7 @@ def dataout_loop():
             bpy.context.scene.objects.active = join_obj
             bpy.ops.object.join()
 
+            writepipeline(b'OK')
             hecl.hmdl.cook(writepipebuf, join_obj, maxSkinBanks)
 
             bpy.context.scene.objects.unlink(join_obj)

hecl/driver/ToolBase.hpp

@@ -20,7 +20,7 @@ struct ToolPassInfo
 {
     HECL::SystemString pname;
     HECL::SystemString cwd;
-    std::list<HECL::SystemString> args;
+    std::vector<HECL::SystemString> args;
     HECL::SystemString output;
     HECL::Database::Project* project = nullptr;
     unsigned verbosityLevel = 0;
@@ -229,6 +229,7 @@ static HECL::SystemString MakePathArgAbsolute(const HECL::SystemString& arg,
 void ToolPrintProgress(const HECL::SystemChar* message, const HECL::SystemChar* submessage,
                        int lidx, float factor, int& lineIdx)
 {
+    bool blocks = factor >= 0.0;
     factor = std::max(0.0f, std::min(1.0f, factor));
     int iFactor = factor * 100.0;
     if (XTERM_COLOR)
@@ -243,7 +244,11 @@ void ToolPrintProgress(const HECL::SystemChar* message, const HECL::SystemChar*
         HECL::Printf(_S("  "));
 
     int width = HECL::ConsoleWidth();
-    int half = width / 2 - 2;
+    int half;
+    if (blocks)
+        half = width / 2 - 2;
+    else
+        half = width - 4;
 
     if (!message)
         message = _S("");
@@ -278,6 +283,8 @@ void ToolPrintProgress(const HECL::SystemChar* message, const HECL::SystemChar*
         }
     }
 
+    if (blocks)
+    {
         if (XTERM_COLOR)
         {
             size_t blocks = half - 7;
@@ -302,6 +309,7 @@ void ToolPrintProgress(const HECL::SystemChar* message, const HECL::SystemChar*
                 HECL::Printf(_S("-"));
             HECL::Printf(_S("]"));
         }
+    }
 
     HECL::Printf(_S("\r"));
     if (XTERM_COLOR)

hecl/driver/ToolCook.hpp

@@ -6,7 +6,7 @@
 
 class ToolCook final : public ToolBase
 {
-    std::list<HECL::ProjectPath> m_selectedItems;
+    std::vector<HECL::ProjectPath> m_selectedItems;
     std::unique_ptr<HECL::Database::Project> m_fallbackProj;
     HECL::Database::Project* m_useProj;
     bool m_recursive = false;
@@ -18,6 +18,7 @@ public:
         if (info.args.size())
         {
             /* See if project path is supplied via args and use that over the getcwd one */
+            m_selectedItems.reserve(info.args.size());
             for (const HECL::SystemString& arg : info.args)
             {
                 if (arg.empty())
@@ -59,7 +60,11 @@ public:
 
         /* Default case: recursive at root */
         if (m_selectedItems.empty())
-            m_selectedItems.push_back({HECL::ProjectPath(*m_useProj, _S("."))});
+        {
+            m_selectedItems.reserve(1);
+            m_selectedItems.push_back({HECL::ProjectPath(*m_useProj, _S(""))});
+            m_recursive = true;
+        }
     }
 
     static void Help(HelpOutput& help)
@@ -124,11 +129,10 @@ public:
         {
             int lineIdx = 0;
             m_useProj->cookPath(path,
-            [&lineIdx](const HECL::SystemChar* message, const HECL::SystemChar* submessage,
+            [&lineIdx](const HECL::SystemChar* message,
+                       const HECL::SystemChar* submessage,
                        int lidx, float factor)
-            {
+            {ToolPrintProgress(message, submessage, lidx, factor, lineIdx);}, m_recursive);
-                ToolPrintProgress(message, submessage, lidx, factor, lineIdx);
-            }, m_recursive);
         }
         return 0;
     }

hecl/driver/ToolExtract.hpp

@@ -22,8 +22,8 @@ class ToolExtract final : public ToolBase
         SpecExtractPass(const SpecExtractPass& other) = delete;
         SpecExtractPass(SpecExtractPass&& other) = default;
     };
-    std::list<SpecExtractPass> m_specPasses;
+    std::vector<SpecExtractPass> m_specPasses;
-    std::list<HECL::Database::IDataSpec::ExtractReport> m_reps;
+    std::vector<HECL::Database::IDataSpec::ExtractReport> m_reps;
     std::unique_ptr<HECL::Database::Project> m_fallbackProj;
     HECL::Database::Project* m_useProj;
 public:
@@ -64,12 +64,13 @@ public:
 
         m_einfo.srcpath = m_info.args.front();
         m_einfo.force = info.force;
-        std::list<HECL::SystemString>::const_iterator it=info.args.begin();
+        m_einfo.extractArgs.reserve(info.args.size());
+        auto it=info.args.cbegin();
         ++it;
-        for (;it != info.args.end();
+        for (; it != info.args.cend(); ++it)
-             ++it)
             m_einfo.extractArgs.push_back(*it);
 
+        m_specPasses.reserve(HECL::Database::DATA_SPEC_REGISTRY.size());
         for (const HECL::Database::DataSpecEntry* entry : HECL::Database::DATA_SPEC_REGISTRY)
         {
             HECL::Database::IDataSpec* ds = entry->m_factory(*m_useProj, HECL::Database::TOOL_EXTRACT);
@@ -187,11 +188,10 @@ public:
 
             int lineIdx = 0;
             ds.m_instance->doExtract(m_einfo,
-            [&lineIdx](const HECL::SystemChar* message, const HECL::SystemChar* submessage,
+            [&lineIdx](const HECL::SystemChar* message,
+                       const HECL::SystemChar* submessage,
                        int lidx, float factor)
-            {
+            {ToolPrintProgress(message, submessage, lidx, factor, lineIdx);});
-                ToolPrintProgress(message, submessage, lidx, factor, lineIdx);
-            });
             HECL::Printf(_S("\n\n"));
         }
 

hecl/driver/ToolSpec.hpp

@@ -129,14 +129,15 @@ public:
         std::vector<HECL::SystemString> opSpecs;
         auto it = m_info.args.begin();
         ++it;
-        for (;it != m_info.args.end();
+        for (; it != m_info.args.end() ; ++it)
-             ++it)
         {
             HECL::SystemString itName = *it;
             HECL::ToLower(itName);
             for (auto& spec : specs)
             {
-                if (!itName.compare(spec.spec.m_name))
+                HECL::SystemString compName(spec.spec.m_name);
+                HECL::ToLower(compName);
+                if (!itName.compare(compName))
                 {
                     opSpecs.push_back(spec.spec.m_name);
                     break;

hecl/driver/main.cpp

@@ -149,14 +149,15 @@ int main(int argc, const char** argv)
     }
 
     /* Concatenate args */
-    std::list<HECL::SystemString> args;
+    std::vector<HECL::SystemString> args;
+    args.reserve(argc-2);
     for (int i=2 ; i<argc ; ++i)
         args.push_back(HECL::SystemString(argv[i]));
 
     if (!args.empty())
     {
         /* Extract output argument */
-        for (std::list<HECL::SystemString>::const_iterator it = args.begin() ; it != args.end() ;)
+        for (auto it = args.cbegin() ; it != args.cend() ;)
         {
             const HECL::SystemString& arg = *it;
             HECL::SystemRegexMatch oMatch;
@@ -184,7 +185,7 @@ int main(int argc, const char** argv)
         }
 
         /* Count verbosity */
-        for (std::list<HECL::SystemString>::const_iterator it = args.begin() ; it != args.end() ;)
+        for (auto it = args.cbegin() ; it != args.cend() ;)
         {
             const HECL::SystemString& arg = *it;
             HECL::SystemRegexMatch vMatch;
@@ -199,7 +200,7 @@ int main(int argc, const char** argv)
         }
 
         /* Check force argument */
-        for (std::list<HECL::SystemString>::const_iterator it = args.begin() ; it != args.end() ;)
+        for (auto it = args.cbegin() ; it != args.cend() ;)
         {
             const HECL::SystemString& arg = *it;
             if (std::regex_search(arg, regFORCE))
@@ -212,6 +213,7 @@ int main(int argc, const char** argv)
         }
 
         /* Gather remaining args */
+        info.args.reserve(args.size());
         for (const HECL::SystemString& arg : args)
             info.args.push_back(arg);
     }
@@ -270,7 +272,7 @@ int main(int argc, const char** argv)
         {
             /* Shortcut-case: implicit extract */
             fclose(fp);
-            info.args.push_front(argv[1]);
+            info.args.insert(info.args.begin(), argv[1]);
             tool.reset(new ToolExtract(info));
         }
     }

hecl/include/HECL/Database.hpp

@@ -77,7 +77,7 @@ public:
     struct ExtractPassInfo
     {
         SystemString srcpath;
-        std::list<SystemString> extractArgs;
+        std::vector<SystemString> extractArgs;
         bool force;
     };
 
@@ -94,8 +94,8 @@ public:
         std::vector<ExtractReport> childOpts;
     };
 
-    virtual bool canExtract(const ExtractPassInfo& info, std::list<ExtractReport>& reps)
+    virtual bool canExtract(const ExtractPassInfo& info, std::vector<ExtractReport>& reps)
-    {(void)info;LogModule.report(LogVisor::Error, "not implemented");return false;}
+    {(void)info;(void)reps;LogModule.report(LogVisor::Error, "not implemented");return false;}
     virtual void doExtract(const ExtractPassInfo& info, FProgress progress)
     {(void)info;(void)progress;}
 
@@ -271,12 +271,12 @@ public:
     class ConfigFile
     {
         SystemString m_filepath;
-        std::list<std::string> m_lines;
+        std::vector<std::string> m_lines;
         FILE* m_lockedFile = NULL;
     public:
         ConfigFile(const Project& project, const SystemString& name,
                    const SystemString& subdir=_S("/.hecl/"));
-        std::list<std::string>& lockAndRead();
+        std::vector<std::string>& lockAndRead();
         void addLine(const std::string& line);
         void removeLine(const std::string& refLine);
         bool checkForLine(const std::string& refLine);

hecl/include/HECL/HECL.hpp

@@ -554,7 +554,7 @@ public:
 
     /**
      * @brief Return new ProjectPath with extension added
-     * @param ext file extension to add
+     * @param ext file extension to add (nullptr may be passed to remove the extension)
      * @param replace remove existing extension (if any) before appending new extension
      * @return new path with extension
      */
@@ -570,14 +570,17 @@ public:
                 --relIt;
                 --absIt;
             }
-            if (*relIt == _S('.'))
+            if (*relIt == _S('.') && relIt != pp.m_relPath.begin())
             {
                 pp.m_relPath.resize(relIt - pp.m_relPath.begin());
                 pp.m_absPath.resize(absIt - pp.m_absPath.begin());
             }
         }
+        if (ext)
+        {
             pp.m_relPath += ext;
             pp.m_absPath += ext;
+        }
         return pp;
     }
 
@@ -706,13 +709,13 @@ public:
      * @brief Insert directory children into list
      * @param outPaths list to append children to
      */
-    void getDirChildren(std::list<ProjectPath>& outPaths) const;
+    void getDirChildren(std::vector<ProjectPath>& outPaths) const;
 
     /**
      * @brief Insert glob matches into existing vector
      * @param outPaths Vector to add matches to (will not erase existing contents)
      */
-    void getGlobResults(std::list<ProjectPath>& outPaths) const;
+    void getGlobResults(std::vector<ProjectPath>& outPaths) const;
 
     /**
      * @brief Count how many directory levels deep in project path is

hecl/lib/Database/Project.cpp

@@ -48,7 +48,7 @@ Project::ConfigFile::ConfigFile(const Project& project, const SystemString& name
     m_filepath = project.m_rootPath.getAbsolutePath() + subdir + name;
 }
 
-std::list<std::string>& Project::ConfigFile::lockAndRead()
+std::vector<std::string>& Project::ConfigFile::lockAndRead()
 {
     if (m_lockedFile)
         return m_lines;
@@ -122,10 +122,8 @@ bool Project::ConfigFile::checkForLine(const std::string& refLine)
     }
 
     for (const std::string& line : m_lines)
-    {
         if (!line.compare(refLine))
             return true;
-    }
     return false;
 }
 
@@ -262,7 +260,7 @@ bool Project::addPaths(const std::vector<ProjectPath>& paths)
 
 bool Project::removePaths(const std::vector<ProjectPath>& paths, bool recursive)
 {
-    std::list<std::string>& existingPaths = m_paths.lockAndRead();
+    std::vector<std::string>& existingPaths = m_paths.lockAndRead();
     if (recursive)
     {
         for (const ProjectPath& path : paths)
@@ -306,7 +304,8 @@ void Project::rescanDataSpecs()
     m_specs.lockAndRead();
     for (const DataSpecEntry* spec : DATA_SPEC_REGISTRY)
     {
-        SystemUTF8View specUTF8(spec->m_name);
+        HECL::SystemString specStr(spec->m_name);
+        SystemUTF8View specUTF8(specStr);
         m_compiledSpecs.push_back({*spec, ProjectPath(m_cookedRoot, HECL::SystemString(spec->m_name) + _S(".spec")),
                                    m_specs.checkForLine(specUTF8) ? true : false});
     }
@@ -339,116 +338,198 @@ bool Project::disableDataSpecs(const std::vector<SystemString>& specs)
     return result;
 }
 
-static void InsertPath(std::list<std::pair<ProjectPath, std::list<ProjectPath>>>& dirs,
+class CookProgress
-                       ProjectPath&& path)
 {
-    ProjectPath thisDir = path.getParentPath();
+    FProgress& m_progFunc;
-    for (std::pair<ProjectPath, std::list<ProjectPath>>& dir : dirs)
+    const SystemChar* m_dir = nullptr;
+    const SystemChar* m_file = nullptr;
+    int lidx = 0;
+    float m_prog = 0.0;
+public:
+    CookProgress(FProgress& progFunc) : m_progFunc(progFunc) {}
+    void changeDir(const SystemChar* dir) {m_dir = dir; ++lidx;}
+    void changeFile(const SystemChar* file, float prog) {m_file = file; m_prog = prog;}
+    void reportFile(const DataSpecEntry* specEnt)
     {
-        if (dir.first == thisDir)
+        SystemString submsg(m_file);
+        submsg += _S(" (");
+        submsg += specEnt->m_name;
+        submsg += _S(')');
+        m_progFunc(m_dir, submsg.c_str(), lidx, m_prog);
+    }
+    void reportDirComplete() {m_progFunc(m_dir, nullptr, lidx, 1.0);}
+};
+
+using SpecInst = std::pair<const DataSpecEntry*, std::unique_ptr<IDataSpec>>;
+
+static void VisitFile(const ProjectPath& path,
+                      std::vector<SpecInst>& specInsts,
+                      CookProgress& progress)
+{
+    for (SpecInst& spec : specInsts)
     {
-            dir.second.push_back(std::move(path));
+        if (spec.second->canCook(path))
-            return;
+        {
+            ProjectPath cooked = path.getCookedPath(*spec.first);
+            if (path.getModtime() > cooked.getModtime())
+            {
+                progress.reportFile(spec.first);
+                spec.second->doCook(path, cooked);
+            }
         }
     }
-    dirs.emplace_back(std::move(thisDir), std::list<ProjectPath>(std::move(path)));
 }
 
-static void VisitDirectory(std::list<std::pair<ProjectPath, std::list<ProjectPath>>>& allDirs,
+static void VisitDirectory(const ProjectPath& dir, bool recursive,
-                           const ProjectPath& dir, bool recursive)
+                           std::vector<SpecInst>& specInsts,
+                           CookProgress& progress)
 {
-    std::list<ProjectPath> children;
+    std::vector<ProjectPath> children;
     dir.getDirChildren(children);
-    allDirs.emplace_back(dir, std::list<ProjectPath>());
+
-    std::list<ProjectPath>& ch = allDirs.back().second;
+    /* Pass 1: child file count */
+    int childFileCount = 0;
     for (ProjectPath& child : children)
     {
         switch (child.getPathType())
         {
         case ProjectPath::PT_FILE:
         {
-            ch.push_back(std::move(child));
+            ++childFileCount;
-            break;
-        }
-        case ProjectPath::PT_DIRECTORY:
-        {
-            if (recursive)
-                VisitDirectory(allDirs, child, recursive);
             break;
         }
         default: break;
         }
     }
+
+    /* Pass 2: child files */
+    int progNum = 0;
+    float progDenom = childFileCount;
+    progress.changeDir(dir.getLastComponent());
+    for (ProjectPath& child : children)
+    {
+        switch (child.getPathType())
+        {
+        case ProjectPath::PT_FILE:
+        {
+            progress.changeFile(child.getLastComponent(), progNum++/progDenom);
+            VisitFile(child, specInsts, progress);
+            break;
+        }
+        default: break;
+        }
+    }
+    progress.reportDirComplete();
+
+    /* Pass 3: child directories */
+    if (recursive)
+    {
+        for (ProjectPath& child : children)
+        {
+            switch (child.getPathType())
+            {
+            case ProjectPath::PT_DIRECTORY:
+            {
+                VisitDirectory(child, recursive, specInsts, progress);
+                break;
+            }
+            default: break;
+            }
+        }
+    }
 }
 
-bool Project::cookPath(const ProjectPath& path, FProgress feedbackCb, bool recursive)
+static void VisitGlob(const ProjectPath& path, bool recursive,
+                      std::vector<SpecInst>& specInsts,
+                      CookProgress& progress)
+{
+    std::vector<ProjectPath> children;
+    path.getGlobResults(children);
+
+    /* Pass 1: child file count */
+    int childFileCount = 0;
+    for (ProjectPath& child : children)
+    {
+        switch (child.getPathType())
+        {
+        case ProjectPath::PT_FILE:
+        {
+            ++childFileCount;
+            break;
+        }
+        default: break;
+        }
+    }
+
+    /* Pass 2: child files */
+    int progNum = 0;
+    float progDenom = childFileCount;
+    progress.changeDir(path.getLastComponent());
+    for (ProjectPath& child : children)
+    {
+        switch (child.getPathType())
+        {
+        case ProjectPath::PT_FILE:
+        {
+            progress.changeFile(child.getLastComponent(), progNum++/progDenom);
+            VisitFile(child, specInsts, progress);
+            break;
+        }
+        default: break;
+        }
+    }
+    progress.reportDirComplete();
+
+    /* Pass 3: child directories */
+    if (recursive)
+    {
+        for (ProjectPath& child : children)
+        {
+            switch (child.getPathType())
+            {
+            case ProjectPath::PT_DIRECTORY:
+            {
+                VisitDirectory(child, recursive, specInsts, progress);
+                break;
+            }
+            default: break;
+            }
+        }
+    }
+}
+
+bool Project::cookPath(const ProjectPath& path, FProgress progress, bool recursive)
 {
     /* Construct DataSpec instances for cooking */
-    std::list<std::pair<const DataSpecEntry*, std::unique_ptr<IDataSpec>>> specInsts;
+    std::vector<SpecInst> specInsts;
+    specInsts.reserve(m_compiledSpecs.size());
     for (const ProjectDataSpec& spec : m_compiledSpecs)
         if (spec.active)
-            specInsts.emplace_back(&spec.spec, std::unique_ptr<IDataSpec>(spec.spec.m_factory(*this, TOOL_COOK)));
+            specInsts.emplace_back(&spec.spec,
+            std::unique_ptr<IDataSpec>(spec.spec.m_factory(*this, TOOL_COOK)));
 
-    /* Gather complete directory/file list */
+    /* Iterate complete directory/file/glob list */
-    std::list<std::pair<ProjectPath, std::list<ProjectPath>>> allDirs;
+    CookProgress cookProg(progress);
     switch (path.getPathType())
     {
     case ProjectPath::PT_FILE:
     {
-        InsertPath(allDirs, std::move(ProjectPath(path)));
+        cookProg.changeFile(path.getLastComponent(), 0.0);
+        VisitFile(path, specInsts, cookProg);
         break;
     }
     case ProjectPath::PT_DIRECTORY:
     {
-        VisitDirectory(allDirs, path, recursive);
+        VisitDirectory(path, recursive, specInsts, cookProg);
         break;
     }
     case ProjectPath::PT_GLOB:
     {
-        std::list<ProjectPath> results;
+        VisitGlob(path, recursive, specInsts, cookProg);
-        path.getGlobResults(results);
-        for (ProjectPath& result : results)
-        {
-            switch (result.getPathType())
-            {
-            case ProjectPath::PT_FILE:
-            {
-                InsertPath(allDirs, std::move(result));
-                break;
-            }
-            case ProjectPath::PT_DIRECTORY:
-            {
-                VisitDirectory(allDirs, path, recursive);
         break;
     }
     default: break;
     }
-        }
-    }
-    default: break;
-    }
-
-    /* Iterate and cook */
-    int lidx = 0;
-    for (const std::pair<ProjectPath, std::list<ProjectPath>>& dir : allDirs)
-    {
-        float dirSz = dir.second.size();
-        int pidx = 0;
-        for (const ProjectPath& path : dir.second)
-        {
-            feedbackCb(dir.first.getLastComponent(), path.getLastComponent(), lidx, pidx++/dirSz);
-            for (std::pair<const DataSpecEntry*, std::unique_ptr<IDataSpec>>& spec : specInsts)
-            {
-                if (spec.second->canCook(path))
-                {
-                    ProjectPath cooked = path.getCookedPath(*spec.first);
-                    if (path.getModtime() > cooked.getModtime())
-                        spec.second->doCook(path, cooked);
-                }
-            }
-        }
-        feedbackCb(dir.first.getLastComponent(), nullptr, lidx++, 1.0);
-    }
 
     return true;
 }

hecl/lib/ProjectPath.cpp

@@ -32,7 +32,7 @@ static SystemString canonRelPath(const SystemString& path)
             if (comps.empty())
             {
                 /* Unable to resolve outside project */
-                LogModule.report(LogVisor::Error, _S("Unable to resolve outside project root in %s"), path.c_str());
+                LogModule.report(LogVisor::FatalError, _S("Unable to resolve outside project root in %s"), path.c_str());
                 return _S(".");
             }
             comps.pop_back();
@@ -118,7 +118,8 @@ void ProjectPath::assign(const ProjectPath& parentPath, const std::string& path)
 
 ProjectPath ProjectPath::getCookedPath(const Database::DataSpecEntry& spec) const
 {
-    return ProjectPath(m_proj->getProjectCookedPath(spec), m_relPath);
+    ProjectPath woExt = getWithExtension(nullptr, true);
+    return ProjectPath(m_proj->getProjectCookedPath(spec), woExt.getRelativePath());
 }
 
 ProjectPath::PathType ProjectPath::getPathType() const
@@ -141,7 +142,7 @@ Time ProjectPath::getModtime() const
     time_t latestTime = 0;
     if (std::regex_search(m_absPath, regGLOB))
     {
-        std::list<ProjectPath> globResults;
+        std::vector<ProjectPath> globResults;
         getGlobResults(globResults);
         for (ProjectPath& path : globResults)
         {
@@ -179,12 +180,12 @@ Time ProjectPath::getModtime() const
             return Time(latestTime);
         }
     }
-    LogModule.report(LogVisor::Error, _S("invalid path type for computing modtime"));
+    LogModule.report(LogVisor::Error, _S("invalid path type for computing modtime in '%s'"), m_absPath.c_str());
     return Time();
 }
 
 static void _recursiveGlob(Database::Project& proj,
-                           std::list<ProjectPath>& outPaths,
+                           std::vector<ProjectPath>& outPaths,
                            size_t level,
                            const SystemRegexMatch& pathCompMatches,
                            const SystemString& itStr,
@@ -241,10 +242,11 @@ static void _recursiveGlob(Database::Project& proj,
 #endif
 }
 
-void ProjectPath::getDirChildren(std::list<ProjectPath>& outPaths) const
+void ProjectPath::getDirChildren(std::vector<ProjectPath>& outPaths) const
 {
 #if _WIN32
 #else
+    struct dirent* de;
     DIR* dir = opendir(m_absPath.c_str());
     if (!dir)
     {
@@ -252,15 +254,34 @@ void ProjectPath::getDirChildren(std::list<ProjectPath>& outPaths) const
         return;
     }
 
-    struct dirent* de;
+    /* Count elements */
+    size_t count = 0;
     while ((de = readdir(dir)))
+    {
+        if (!strcmp(de->d_name, "."))
+            continue;
+        if (!strcmp(de->d_name, ".."))
+            continue;
+        ++count;
+    }
+    outPaths.reserve(outPaths.size() + count);
+
+    /* Add elements */
+    rewinddir(dir);
+    while ((de = readdir(dir)))
+    {
+        if (!strcmp(de->d_name, "."))
+            continue;
+        if (!strcmp(de->d_name, ".."))
+            continue;
         outPaths.emplace_back(*this, de->d_name);
+    }
 
     closedir(dir);
 #endif
 }
 
-void ProjectPath::getGlobResults(std::list<ProjectPath>& outPaths) const
+void ProjectPath::getGlobResults(std::vector<ProjectPath>& outPaths) const
 {
 #if _WIN32
     SystemString itStr;
@@ -338,8 +359,7 @@ ProjectRootPath SearchForProject(const SystemString& path, SystemString& subpath
                 fclose(fp);
                 if (readSize != 4)
                     continue;
-                static const HECL::FourCC hecl("HECL");
+                if (HECL::FourCC(magic) != FOURCC('HECL'))
-                if (HECL::FourCC(magic) != hecl)
                     continue;
                 ProjectRootPath newRootPath = ProjectRootPath(testPath);
                 SystemString::const_iterator origEnd = testRoot.getAbsolutePath().end();