Work on GCN image building

include/NOD/DirectoryEnumerator.hpp

@@ -0,0 +1,50 @@
+#ifndef __NOD_DIRECTORY_ENUMERATOR__
+#define __NOD_DIRECTORY_ENUMERATOR__
+
+#include "Util.hpp"
+
+namespace NOD
+{
+
+class DirectoryEnumerator
+{
+public:
+    enum class Mode
+    {
+        Native,
+        DirsSorted,
+        FilesSorted,
+        DirsThenFilesSorted
+    };
+    struct Entry
+    {
+        SystemString m_path;
+        SystemString m_name;
+        size_t m_fileSz;
+        bool m_isDir;
+
+    private:
+        friend class DirectoryEnumerator;
+        Entry(SystemString&& path, const SystemChar* name, size_t sz, bool isDir)
+        : m_path(std::move(path)), m_name(name), m_fileSz(sz), m_isDir(isDir) {}
+    };
+
+private:
+    std::vector<Entry> m_entries;
+
+public:
+    DirectoryEnumerator(const SystemString& path, Mode mode=Mode::DirsThenFilesSorted,
+                        bool sizeSort=false, bool reverse=false, bool noHidden=false)
+    : DirectoryEnumerator(path.c_str(), mode, sizeSort, reverse, noHidden) {}
+    DirectoryEnumerator(const SystemChar* path, Mode mode=Mode::DirsThenFilesSorted,
+                        bool sizeSort=false, bool reverse=false, bool noHidden=false);
+
+    operator bool() const {return m_entries.size() != 0;}
+    size_t size() const {return m_entries.size();}
+    std::vector<Entry>::const_iterator begin() const {return m_entries.cbegin();}
+    std::vector<Entry>::const_iterator end() const {return m_entries.cend();}
+};
+
+}
+
+#endif // __NOD_DIRECTORY_ENUMERATOR__

include/NOD/DiscBase.hpp

@@ -13,71 +13,82 @@
 namespace NOD
 {
 
+class FSTNode
+{
+    uint32_t typeAndNameOffset;
+    uint32_t offset;
+    uint32_t length;
+public:
+    FSTNode(bool isDir, uint32_t nameOff, uint32_t off, uint32_t len)
+    {
+        typeAndNameOffset = nameOff & 0xffffff;
+        typeAndNameOffset |= isDir << 24;
+        typeAndNameOffset = SBig(typeAndNameOffset);
+        offset = SBig(off);
+        length = SBig(len);
+    }
+    inline bool isDir() const {return ((SBig(typeAndNameOffset) >> 24) != 0);}
+    inline uint32_t getNameOffset() const {return SBig(typeAndNameOffset) & 0xffffff;}
+    inline uint32_t getOffset() const {return SBig(offset);}
+    inline uint32_t getLength() const {return SBig(length);}
+    void incrementLength()
+    {
+        uint32_t orig = SBig(length);
+        ++orig;
+        length = SBig(orig);
+    }
+};
+
+struct Header
+{
+    char m_gameID[6];
+    char m_discNum;
+    char m_discVersion;
+    char m_audioStreaming;
+    char m_streamBufSz;
+    char m_unk[14];
+    uint32_t m_wiiMagic;
+    uint32_t m_gcnMagic;
+    char m_gameTitle[64];
+
+    Header(IDiscIO& dio)
+    {
+        std::unique_ptr<IDiscIO::IReadStream> s = dio.beginReadStream(0);
+        s->read(this, sizeof(*this));
+        m_wiiMagic = SBig(m_wiiMagic);
+        m_gcnMagic = SBig(m_gcnMagic);
+    }
+
+    Header(const char gameID[6], const char* gameTitle, bool wii, char discNum=0, char discVersion=0,
+           char audioStreaming=1, char streamBufSz=0)
+    {
+        memset(this, 0, sizeof(*this));
+        memcpy(m_gameID, gameID, 6);
+        strncpy(m_gameTitle, gameTitle, 64);
+        m_discNum = discNum;
+        m_discVersion = discVersion;
+        m_audioStreaming = audioStreaming;
+        m_streamBufSz = streamBufSz;
+        if (wii)
+            m_wiiMagic = 0x5D1C9EA3;
+        else
+            m_gcnMagic = 0xC2339F3D;
+    }
+
+    void write(IFileIO::IWriteStream& ws) const
+    {
+        Header hs(*this);
+        hs.m_wiiMagic = SBig(hs.m_wiiMagic);
+        hs.m_gcnMagic = SBig(hs.m_gcnMagic);
+        ws.write(&hs, sizeof(hs));
+    }
+};
+
 struct ExtractionContext;
 class DiscBase
 {
 public:
     virtual ~DiscBase() {}
-    struct Header
-    {
-        char m_gameID[6];
-        char m_discNum;
-        char m_discVersion;
-        char m_audioStreaming;
-        char m_streamBufSz;
-        char m_unk[14];
-        uint32_t m_wiiMagic;
-        uint32_t m_gcnMagic;
-        char m_gameTitle[64];
-
-        Header(IDiscIO& dio)
-        {
-            std::unique_ptr<IDiscIO::IReadStream> s = dio.beginReadStream(0);
-            s->read(this, sizeof(*this));
-            m_wiiMagic = SBig(m_wiiMagic);
-            m_gcnMagic = SBig(m_gcnMagic);
-        }
-
-        Header(const char gameID[6], const char* gameTitle, char discNum=0, char discVersion=0,
-               char audioStreaming=1, char streamBufSz=0)
-        {
-            memset(this, 0, sizeof(*this));
-            memcpy(m_gameID, gameID, 6);
-            strncpy(m_gameTitle, gameTitle, 64);
-            m_discNum = discNum;
-            m_discVersion = discVersion;
-            m_audioStreaming = audioStreaming;
-            m_streamBufSz = streamBufSz;
-            m_gcnMagic = 0xC2339F3D;
-        }
-
-        void write(IDiscIO::IWriteStream& ws) const
-        {
-            Header hs(*this);
-            hs.m_gcnMagic = SBig(hs.m_gcnMagic);
-            ws.write(&hs, sizeof(hs));
-        }
-    };
-
-    class FSTNode
-    {
-        uint32_t typeAndNameOffset;
-        uint32_t offset;
-        uint32_t length;
-    public:
-        FSTNode(bool isDir, uint32_t nameOff, uint32_t off, uint32_t len)
-        {
-            typeAndNameOffset = nameOff & 0xffffff;
-            typeAndNameOffset |= isDir << 24;
-            typeAndNameOffset = SBig(typeAndNameOffset);
-            offset = SBig(off);
-            length = SBig(len);
-        }
-        inline bool isDir() const {return ((SBig(typeAndNameOffset) >> 24) != 0);}
-        inline uint32_t getNameOffset() const {return SBig(typeAndNameOffset) & 0xffffff;}
-        inline uint32_t getOffset() const {return SBig(offset);}
-        inline uint32_t getLength() const {return SBig(length);}
-    };
 
     class IPartition
     {
@@ -202,6 +213,11 @@ public:
         std::vector<Node> m_nodes;
         void parseFST(IPartReadStream& s);
 
+        std::vector<FSTNode> m_buildNodes;
+        std::vector<std::string> m_buildNames;
+        size_t m_buildNameOff = 0;
+        void recursiveBuildNodes(const SystemChar* dirIn, std::function<void(void)> incParents);
+
         uint64_t m_dolSz;
         void parseDOL(IPartReadStream& s);
 
@@ -278,8 +294,68 @@ public:
         for (std::unique_ptr<IPartition>& part : m_partitions)
             part->extractToDirectory(path, ctx);
     }
-    virtual bool packFromDirectory(const SystemChar* dataPath, const SystemChar* updatePath,
-                                   const SystemChar* outPath, const char gameID[6], const char* gameTitle, bool korean=false)=0;
+};
+
+class DiscBuilderBase
+{
+public:
+    class IPartitionBuilder
+    {
+    public:
+        virtual ~IPartitionBuilder() {}
+        enum class Kind : uint32_t
+        {
+            Data,
+            Update,
+            Channel
+        };
+    protected:
+        std::vector<FSTNode> m_buildNodes;
+        std::vector<std::string> m_buildNames;
+        size_t m_buildNameOff = 0;
+        virtual uint64_t userAllocate(uint64_t reqSz)=0;
+        void recursiveBuildNodes(const SystemChar* dirIn, uint64_t dolInode,
+                                 std::function<void(void)> incParents);
+        void addBuildName(const SystemString& str)
+        {
+            SystemUTF8View utf8View(str);
+            m_buildNames.push_back(utf8View.utf8_str());
+            m_buildNameOff += str.size() + 1;
+        }
+
+        DiscBuilderBase& m_parent;
+        Kind m_kind;
+        uint64_t m_offset;
+
+        char m_gameID[6];
+        std::string m_gameTitle;
+        uint32_t m_fstMemoryAddr;
+        uint64_t m_dolOffset = 0;
+    public:
+        IPartitionBuilder(DiscBuilderBase& parent, Kind kind, uint64_t offset,
+                          const char gameID[6], const char* gameTitle, uint32_t fstMemoryAddr)
+        : m_parent(parent), m_kind(kind), m_offset(offset), m_gameTitle(gameTitle), m_fstMemoryAddr(fstMemoryAddr)
+        {
+            memcpy(m_gameID, gameID, 6);
+        }
+        bool buildFromDirectory(const SystemChar* dirIn, const SystemChar* dolIn,
+                                const SystemChar* apploaderIn);
+    };
+protected:
+    std::unique_ptr<IFileIO> m_fileIO;
+    std::vector<std::unique_ptr<IPartitionBuilder>> m_partitions;
+public:
+    std::function<void(size_t idx, const SystemString&, size_t)> m_progressCB;
+    size_t m_progressIdx = 0;
+    virtual ~DiscBuilderBase() {}
+    DiscBuilderBase(std::unique_ptr<IFileIO>&& fio,
+                    std::function<void(size_t idx, const SystemString&, size_t)> progressCB)
+    : m_fileIO(std::move(fio)), m_progressCB(progressCB) {}
+
+    IFileIO& getFileIO() {return *m_fileIO;}
+
+    virtual bool buildFromDirectory(const SystemChar* dirIn, const SystemChar* dolIn,
+                                    const SystemChar* apploaderIn)=0;
 };
 
 using Partition = DiscBase::IPartition;

include/NOD/DiscGCN.hpp

@@ -10,9 +10,15 @@ class DiscGCN : public DiscBase
 {
 public:
     DiscGCN(std::unique_ptr<IDiscIO>&& dio);
-    bool packFromDirectory(const SystemChar* dataPath, const SystemChar* updatePath,
-                           const SystemChar* outPath, const char gameID[6], const char* gameTitle,
-                           bool korean=false);
+};
+
+class DiscBuilderGCN : public DiscBuilderBase
+{
+public:
+    DiscBuilderGCN(const SystemChar* outPath, const char gameID[6], const char* gameTitle,
+                   uint32_t fstMemoryAddr, std::function<void(size_t, const SystemString&, size_t)> progressCB);
+    bool buildFromDirectory(const SystemChar* dirIn, const SystemChar* dolIn,
+                            const SystemChar* apploaderIn);
 };
 
 }

include/NOD/DiscWii.hpp

@@ -10,9 +10,9 @@ class DiscWii : public DiscBase
 {
 public:
     DiscWii(std::unique_ptr<IDiscIO>&& dio);
-    bool packFromDirectory(const SystemChar* dataPath, const SystemChar* updatePath,
-                           const SystemChar* outPath, const char gameID[6], const char* gameTitle,
-                           bool korean=false);
+    DiscWii(const SystemChar* dataPath, const SystemChar* updatePath,
+            const SystemChar* outPath, const char gameID[6], const char* gameTitle,
+            bool korean=false);
 };
 
 }

include/NOD/IFileIO.hpp

@@ -18,10 +18,11 @@ public:
     struct IWriteStream
     {
         virtual ~IWriteStream() {}
-        virtual uint64_t write(void* buf, uint64_t length)=0;
+        virtual uint64_t write(const void* buf, uint64_t length)=0;
         virtual uint64_t copyFromDisc(struct IPartReadStream& discio, uint64_t length)=0;
     };
     virtual std::unique_ptr<IWriteStream> beginWriteStream() const=0;
+    virtual std::unique_ptr<IWriteStream> beginWriteStream(size_t offset) const=0;
 
     struct IReadStream
     {
@@ -30,9 +31,11 @@ public:
         virtual uint64_t copyToDisc(struct IPartWriteStream& discio, uint64_t length)=0;
     };
     virtual std::unique_ptr<IReadStream> beginReadStream() const=0;
+    virtual std::unique_ptr<IReadStream> beginReadStream(size_t offset) const=0;
 };
 
 std::unique_ptr<IFileIO> NewFileIO(const SystemString& path);
+std::unique_ptr<IFileIO> NewFileIO(const SystemChar* path);
 std::unique_ptr<IFileIO> NewMemIO(void* buf, uint64_t size);
 
 }

include/NOD/Util.hpp

@@ -13,6 +13,7 @@
 #include <windows.h>
 #else
 #include <ctype.h>
+#include <sys/file.h>
 #endif
 #include <sys/stat.h>
 
@@ -181,6 +182,43 @@ static inline int64_t SBig(int64_t val) {return val;}
 static inline uint64_t SBig(uint64_t val) {return val;}
 #endif
 
+#ifndef ROUND_UP_32
+#define ROUND_UP_32(val) (((val) + 31) & ~31)
+#define ROUND_UP_16(val) (((val) + 15) & ~15)
+#endif
+
+enum class FileLockType
+{
+    None = 0,
+    Read,
+    Write
+};
+static inline FILE* Fopen(const SystemChar* path, const SystemChar* mode, FileLockType lock=FileLockType::None)
+{
+#if NOD_UCS2
+    FILE* fp = _wfopen(path, mode);
+    if (!fp)
+        return nullptr;
+#else
+    FILE* fp = fopen(path, mode);
+    if (!fp)
+        return nullptr;
+#endif
+
+    if (lock != FileLockType::None)
+    {
+#if _WIN32
+        OVERLAPPED ov = {};
+        LockFileEx((HANDLE)(uintptr_t)_fileno(fp), (lock == FileLockType::Write) ? LOCKFILE_EXCLUSIVE_LOCK : 0, 0, 0, 1, &ov);
+#else
+        if (flock(fileno(fp), ((lock == FileLockType::Write) ? LOCK_EX : LOCK_SH) | LOCK_NB))
+            LogModule.report(LogVisor::Error, "flock %s: %s", path, strerror(errno));
+#endif
+    }
+
+    return fp;
+}
+
 }
 
 #endif // __NOD_UTIL_HPP__