From fafff92874b04f830f1255e4c2d199f2048e0ae8 Mon Sep 17 00:00:00 2001 From: Jack Andersen Date: Wed, 20 Jan 2016 20:30:37 -1000 Subject: [PATCH] Work on GCN image building --- driver/main.cpp | 56 +++++- include/NOD/DirectoryEnumerator.hpp | 50 +++++ include/NOD/DiscBase.hpp | 200 +++++++++++++------ include/NOD/DiscGCN.hpp | 12 +- include/NOD/DiscWii.hpp | 6 +- include/NOD/IFileIO.hpp | 5 +- include/NOD/Util.hpp | 38 ++++ lib/CMakeLists.txt | 1 + lib/DirectoryEnumerator.cpp | 300 ++++++++++++++++++++++++++++ lib/DiscBase.cpp | 127 +++++++++++- lib/DiscGCN.cpp | 96 ++++++++- lib/DiscWii.cpp | 10 - lib/FileIOFILE.cpp | 52 ++++- 13 files changed, 854 insertions(+), 99 deletions(-) create mode 100644 include/NOD/DirectoryEnumerator.hpp create mode 100644 lib/DirectoryEnumerator.cpp diff --git a/driver/main.cpp b/driver/main.cpp index b6f44b4..af97add 100644 --- a/driver/main.cpp +++ b/driver/main.cpp @@ -6,8 +6,9 @@ static void printHelp() { fprintf(stderr, "Usage:\n" - " nodlib extract [-f] []\n" - " nodlib make []\n"); + " nodtool extract [-f] []\n" + " nodtool makegcn []\n" + " nodtool makewii [] [-u ]\n"); } #if NOD_UCS2 @@ -15,12 +16,16 @@ static void printHelp() #undef strcasecmp #endif #define strcasecmp _wcsicmp +#define PRISize "Iu" int wmain(int argc, wchar_t* argv[]) #else +#define PRISize "zu" int main(int argc, char* argv[]) #endif { - if (argc < 3) + if (argc < 3 || + (!strcasecmp(argv[1], _S("makegcn")) && argc < 7) || + (!strcasecmp(argv[1], _S("makewii")) && argc < 7)) { printHelp(); return -1; @@ -61,8 +66,51 @@ int main(int argc, char* argv[]) if (!dataPart->extractToDirectory(outDir, ctx)) return -1; } - else if (!strcasecmp(argv[1], _S("make"))) + else if (!strcasecmp(argv[1], _S("makegcn"))) { +#if NOD_UCS2 + if (_wcslen(argv[2]) < 6) + NOD::LogModule.report(LogVisor::FatalError, "game-id is not at least 6 characters"); +#else + if (strlen(argv[2]) < 6) + NOD::LogModule.report(LogVisor::FatalError, "game-id is not at least 6 characters"); +#endif + + /* Pre-validate paths */ + NOD::Sstat theStat; + if (NOD::Stat(argv[4], &theStat) || !S_ISDIR(theStat.st_mode)) + NOD::LogModule.report(LogVisor::FatalError, "unable to stat %s as directory", argv[4]); + if (NOD::Stat(argv[5], &theStat) || !S_ISREG(theStat.st_mode)) + NOD::LogModule.report(LogVisor::FatalError, "unable to stat %s as file", argv[5]); + if (NOD::Stat(argv[6], &theStat) || !S_ISREG(theStat.st_mode)) + NOD::LogModule.report(LogVisor::FatalError, "unable to stat %s as file", argv[6]); + + size_t lastIdx = -1; + auto progFunc = [&](size_t idx, const NOD::SystemString& name, size_t bytes) + { + if (idx != lastIdx) + { + lastIdx = idx; + printf("\n"); + } + printf("\r%s %" PRISize " B", name.c_str(), bytes); + fflush(stdout); + }; + + if (argc < 8) + { + NOD::SystemString outPath(argv[4]); + outPath.append(_S(".iso")); + NOD::DiscBuilderGCN b(outPath.c_str(), argv[2], argv[3], 0x0003EB60, progFunc); + b.buildFromDirectory(argv[4], argv[5], argv[6]); + } + else + { + NOD::DiscBuilderGCN b(argv[7], argv[2], argv[3], 0x0003EB60, progFunc); + b.buildFromDirectory(argv[4], argv[5], argv[6]); + } + + printf("\n"); } else { diff --git a/include/NOD/DirectoryEnumerator.hpp b/include/NOD/DirectoryEnumerator.hpp new file mode 100644 index 0000000..2badbf6 --- /dev/null +++ b/include/NOD/DirectoryEnumerator.hpp @@ -0,0 +1,50 @@ +#ifndef __NOD_DIRECTORY_ENUMERATOR__ +#define __NOD_DIRECTORY_ENUMERATOR__ + +#include "Util.hpp" + +namespace NOD +{ + +class DirectoryEnumerator +{ +public: + enum class Mode + { + Native, + DirsSorted, + FilesSorted, + DirsThenFilesSorted + }; + struct Entry + { + SystemString m_path; + SystemString m_name; + size_t m_fileSz; + bool m_isDir; + + private: + friend class DirectoryEnumerator; + Entry(SystemString&& path, const SystemChar* name, size_t sz, bool isDir) + : m_path(std::move(path)), m_name(name), m_fileSz(sz), m_isDir(isDir) {} + }; + +private: + std::vector m_entries; + +public: + DirectoryEnumerator(const SystemString& path, Mode mode=Mode::DirsThenFilesSorted, + bool sizeSort=false, bool reverse=false, bool noHidden=false) + : DirectoryEnumerator(path.c_str(), mode, sizeSort, reverse, noHidden) {} + DirectoryEnumerator(const SystemChar* path, Mode mode=Mode::DirsThenFilesSorted, + bool sizeSort=false, bool reverse=false, bool noHidden=false); + + operator bool() const {return m_entries.size() != 0;} + size_t size() const {return m_entries.size();} + std::vector::const_iterator begin() const {return m_entries.cbegin();} + std::vector::const_iterator end() const {return m_entries.cend();} +}; + +} + +#endif // __NOD_DIRECTORY_ENUMERATOR__ diff --git a/include/NOD/DiscBase.hpp b/include/NOD/DiscBase.hpp index ff2f274..465cadc 100644 --- a/include/NOD/DiscBase.hpp +++ b/include/NOD/DiscBase.hpp @@ -13,71 +13,82 @@ namespace NOD { +class FSTNode +{ + uint32_t typeAndNameOffset; + uint32_t offset; + uint32_t length; +public: + FSTNode(bool isDir, uint32_t nameOff, uint32_t off, uint32_t len) + { + typeAndNameOffset = nameOff & 0xffffff; + typeAndNameOffset |= isDir << 24; + typeAndNameOffset = SBig(typeAndNameOffset); + offset = SBig(off); + length = SBig(len); + } + inline bool isDir() const {return ((SBig(typeAndNameOffset) >> 24) != 0);} + inline uint32_t getNameOffset() const {return SBig(typeAndNameOffset) & 0xffffff;} + inline uint32_t getOffset() const {return SBig(offset);} + inline uint32_t getLength() const {return SBig(length);} + void incrementLength() + { + uint32_t orig = SBig(length); + ++orig; + length = SBig(orig); + } +}; + +struct Header +{ + char m_gameID[6]; + char m_discNum; + char m_discVersion; + char m_audioStreaming; + char m_streamBufSz; + char m_unk[14]; + uint32_t m_wiiMagic; + uint32_t m_gcnMagic; + char m_gameTitle[64]; + + Header(IDiscIO& dio) + { + std::unique_ptr s = dio.beginReadStream(0); + s->read(this, sizeof(*this)); + m_wiiMagic = SBig(m_wiiMagic); + m_gcnMagic = SBig(m_gcnMagic); + } + + Header(const char gameID[6], const char* gameTitle, bool wii, char discNum=0, char discVersion=0, + char audioStreaming=1, char streamBufSz=0) + { + memset(this, 0, sizeof(*this)); + memcpy(m_gameID, gameID, 6); + strncpy(m_gameTitle, gameTitle, 64); + m_discNum = discNum; + m_discVersion = discVersion; + m_audioStreaming = audioStreaming; + m_streamBufSz = streamBufSz; + if (wii) + m_wiiMagic = 0x5D1C9EA3; + else + m_gcnMagic = 0xC2339F3D; + } + + void write(IFileIO::IWriteStream& ws) const + { + Header hs(*this); + hs.m_wiiMagic = SBig(hs.m_wiiMagic); + hs.m_gcnMagic = SBig(hs.m_gcnMagic); + ws.write(&hs, sizeof(hs)); + } +}; + struct ExtractionContext; class DiscBase { public: virtual ~DiscBase() {} - struct Header - { - char m_gameID[6]; - char m_discNum; - char m_discVersion; - char m_audioStreaming; - char m_streamBufSz; - char m_unk[14]; - uint32_t m_wiiMagic; - uint32_t m_gcnMagic; - char m_gameTitle[64]; - - Header(IDiscIO& dio) - { - std::unique_ptr s = dio.beginReadStream(0); - s->read(this, sizeof(*this)); - m_wiiMagic = SBig(m_wiiMagic); - m_gcnMagic = SBig(m_gcnMagic); - } - - Header(const char gameID[6], const char* gameTitle, char discNum=0, char discVersion=0, - char audioStreaming=1, char streamBufSz=0) - { - memset(this, 0, sizeof(*this)); - memcpy(m_gameID, gameID, 6); - strncpy(m_gameTitle, gameTitle, 64); - m_discNum = discNum; - m_discVersion = discVersion; - m_audioStreaming = audioStreaming; - m_streamBufSz = streamBufSz; - m_gcnMagic = 0xC2339F3D; - } - - void write(IDiscIO::IWriteStream& ws) const - { - Header hs(*this); - hs.m_gcnMagic = SBig(hs.m_gcnMagic); - ws.write(&hs, sizeof(hs)); - } - }; - - class FSTNode - { - uint32_t typeAndNameOffset; - uint32_t offset; - uint32_t length; - public: - FSTNode(bool isDir, uint32_t nameOff, uint32_t off, uint32_t len) - { - typeAndNameOffset = nameOff & 0xffffff; - typeAndNameOffset |= isDir << 24; - typeAndNameOffset = SBig(typeAndNameOffset); - offset = SBig(off); - length = SBig(len); - } - inline bool isDir() const {return ((SBig(typeAndNameOffset) >> 24) != 0);} - inline uint32_t getNameOffset() const {return SBig(typeAndNameOffset) & 0xffffff;} - inline uint32_t getOffset() const {return SBig(offset);} - inline uint32_t getLength() const {return SBig(length);} - }; class IPartition { @@ -202,6 +213,11 @@ public: std::vector m_nodes; void parseFST(IPartReadStream& s); + std::vector m_buildNodes; + std::vector m_buildNames; + size_t m_buildNameOff = 0; + void recursiveBuildNodes(const SystemChar* dirIn, std::function incParents); + uint64_t m_dolSz; void parseDOL(IPartReadStream& s); @@ -278,8 +294,68 @@ public: for (std::unique_ptr& part : m_partitions) part->extractToDirectory(path, ctx); } - virtual bool packFromDirectory(const SystemChar* dataPath, const SystemChar* updatePath, - const SystemChar* outPath, const char gameID[6], const char* gameTitle, bool korean=false)=0; +}; + +class DiscBuilderBase +{ +public: + class IPartitionBuilder + { + public: + virtual ~IPartitionBuilder() {} + enum class Kind : uint32_t + { + Data, + Update, + Channel + }; + protected: + std::vector m_buildNodes; + std::vector m_buildNames; + size_t m_buildNameOff = 0; + virtual uint64_t userAllocate(uint64_t reqSz)=0; + void recursiveBuildNodes(const SystemChar* dirIn, uint64_t dolInode, + std::function incParents); + void addBuildName(const SystemString& str) + { + SystemUTF8View utf8View(str); + m_buildNames.push_back(utf8View.utf8_str()); + m_buildNameOff += str.size() + 1; + } + + DiscBuilderBase& m_parent; + Kind m_kind; + uint64_t m_offset; + + char m_gameID[6]; + std::string m_gameTitle; + uint32_t m_fstMemoryAddr; + uint64_t m_dolOffset = 0; + public: + IPartitionBuilder(DiscBuilderBase& parent, Kind kind, uint64_t offset, + const char gameID[6], const char* gameTitle, uint32_t fstMemoryAddr) + : m_parent(parent), m_kind(kind), m_offset(offset), m_gameTitle(gameTitle), m_fstMemoryAddr(fstMemoryAddr) + { + memcpy(m_gameID, gameID, 6); + } + bool buildFromDirectory(const SystemChar* dirIn, const SystemChar* dolIn, + const SystemChar* apploaderIn); + }; +protected: + std::unique_ptr m_fileIO; + std::vector> m_partitions; +public: + std::function m_progressCB; + size_t m_progressIdx = 0; + virtual ~DiscBuilderBase() {} + DiscBuilderBase(std::unique_ptr&& fio, + std::function progressCB) + : m_fileIO(std::move(fio)), m_progressCB(progressCB) {} + + IFileIO& getFileIO() {return *m_fileIO;} + + virtual bool buildFromDirectory(const SystemChar* dirIn, const SystemChar* dolIn, + const SystemChar* apploaderIn)=0; }; using Partition = DiscBase::IPartition; diff --git a/include/NOD/DiscGCN.hpp b/include/NOD/DiscGCN.hpp index 1ac1689..565c13c 100644 --- a/include/NOD/DiscGCN.hpp +++ b/include/NOD/DiscGCN.hpp @@ -10,9 +10,15 @@ class DiscGCN : public DiscBase { public: DiscGCN(std::unique_ptr&& dio); - bool packFromDirectory(const SystemChar* dataPath, const SystemChar* updatePath, - const SystemChar* outPath, const char gameID[6], const char* gameTitle, - bool korean=false); +}; + +class DiscBuilderGCN : public DiscBuilderBase +{ +public: + DiscBuilderGCN(const SystemChar* outPath, const char gameID[6], const char* gameTitle, + uint32_t fstMemoryAddr, std::function progressCB); + bool buildFromDirectory(const SystemChar* dirIn, const SystemChar* dolIn, + const SystemChar* apploaderIn); }; } diff --git a/include/NOD/DiscWii.hpp b/include/NOD/DiscWii.hpp index 79d6ab7..c4cb95e 100644 --- a/include/NOD/DiscWii.hpp +++ b/include/NOD/DiscWii.hpp @@ -10,9 +10,9 @@ class DiscWii : public DiscBase { public: DiscWii(std::unique_ptr&& dio); - bool packFromDirectory(const SystemChar* dataPath, const SystemChar* updatePath, - const SystemChar* outPath, const char gameID[6], const char* gameTitle, - bool korean=false); + DiscWii(const SystemChar* dataPath, const SystemChar* updatePath, + const SystemChar* outPath, const char gameID[6], const char* gameTitle, + bool korean=false); }; } diff --git a/include/NOD/IFileIO.hpp b/include/NOD/IFileIO.hpp index d0212ec..8bf736d 100644 --- a/include/NOD/IFileIO.hpp +++ b/include/NOD/IFileIO.hpp @@ -18,10 +18,11 @@ public: struct IWriteStream { virtual ~IWriteStream() {} - virtual uint64_t write(void* buf, uint64_t length)=0; + virtual uint64_t write(const void* buf, uint64_t length)=0; virtual uint64_t copyFromDisc(struct IPartReadStream& discio, uint64_t length)=0; }; virtual std::unique_ptr beginWriteStream() const=0; + virtual std::unique_ptr beginWriteStream(size_t offset) const=0; struct IReadStream { @@ -30,9 +31,11 @@ public: virtual uint64_t copyToDisc(struct IPartWriteStream& discio, uint64_t length)=0; }; virtual std::unique_ptr beginReadStream() const=0; + virtual std::unique_ptr beginReadStream(size_t offset) const=0; }; std::unique_ptr NewFileIO(const SystemString& path); +std::unique_ptr NewFileIO(const SystemChar* path); std::unique_ptr NewMemIO(void* buf, uint64_t size); } diff --git a/include/NOD/Util.hpp b/include/NOD/Util.hpp index a9cf6b3..9f3917c 100644 --- a/include/NOD/Util.hpp +++ b/include/NOD/Util.hpp @@ -13,6 +13,7 @@ #include #else #include +#include #endif #include @@ -181,6 +182,43 @@ static inline int64_t SBig(int64_t val) {return val;} static inline uint64_t SBig(uint64_t val) {return val;} #endif +#ifndef ROUND_UP_32 +#define ROUND_UP_32(val) (((val) + 31) & ~31) +#define ROUND_UP_16(val) (((val) + 15) & ~15) +#endif + +enum class FileLockType +{ + None = 0, + Read, + Write +}; +static inline FILE* Fopen(const SystemChar* path, const SystemChar* mode, FileLockType lock=FileLockType::None) +{ +#if NOD_UCS2 + FILE* fp = _wfopen(path, mode); + if (!fp) + return nullptr; +#else + FILE* fp = fopen(path, mode); + if (!fp) + return nullptr; +#endif + + if (lock != FileLockType::None) + { +#if _WIN32 + OVERLAPPED ov = {}; + LockFileEx((HANDLE)(uintptr_t)_fileno(fp), (lock == FileLockType::Write) ? LOCKFILE_EXCLUSIVE_LOCK : 0, 0, 0, 1, &ov); +#else + if (flock(fileno(fp), ((lock == FileLockType::Write) ? LOCK_EX : LOCK_SH) | LOCK_NB)) + LogModule.report(LogVisor::Error, "flock %s: %s", path, strerror(errno)); +#endif + } + + return fp; +} + } #endif // __NOD_UTIL_HPP__ diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index fd50e7d..7fbb994 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -9,6 +9,7 @@ add_library(NOD DiscIOWBFS.cpp DiscWii.cpp FileIOFILE.cpp + DirectoryEnumerator.cpp NOD.cpp ${NOD_HEADERS}) if(NOT WIN32) diff --git a/lib/DirectoryEnumerator.cpp b/lib/DirectoryEnumerator.cpp new file mode 100644 index 0000000..cb1f914 --- /dev/null +++ b/lib/DirectoryEnumerator.cpp @@ -0,0 +1,300 @@ +#ifdef WIN32 +#include +#else +#include +#include +#endif + +#include + +#include "NOD/DirectoryEnumerator.hpp" + +namespace NOD +{ + +struct CaseInsensitiveCompare +{ + bool operator()(const std::string& lhs, const std::string& rhs) const + { +#if _WIN32 + if (_stricmp(lhs.c_str(), rhs.c_str()) < 0) +#else + if (strcasecmp(lhs.c_str(), rhs.c_str()) < 0) +#endif + return true; + return false; + } + +#if _WIN32 + bool operator()(const std::wstring& lhs, const std::wstring& rhs) const + { + if (_wcsicmp(lhs.c_str(), rhs.c_str()) < 0) + return true; + return false; + } +#endif +}; + +DirectoryEnumerator::DirectoryEnumerator(const SystemChar* path, Mode mode, + bool sizeSort, bool reverse, bool noHidden) +{ + Sstat theStat; + if (Stat(path, &theStat) || !S_ISDIR(theStat.st_mode)) + return; + +#if _WIN32 + SystemString wc(path); + wc += _S("/*"); + WIN32_FIND_DATAW d; + HANDLE dir = FindFirstFileW(wc.c_str(), &d); + if (dir == INVALID_HANDLE_VALUE) + return; + switch (mode) + { + case Mode::Native: + do + { + if (!wcscmp(d.cFileName, _S(".")) || !wcscmp(d.cFileName, _S(".."))) + continue; + if (noHidden && (d.cFileName[0] == L'.' || (d.dwFileAttributes & FILE_ATTRIBUTE_HIDDEN) != 0)) + continue; + SystemString fp(path); + fp += _S('/'); + fp += d.cFileName; + Sstat st; + if (Stat(fp.c_str(), &st)) + continue; + + size_t sz = 0; + bool isDir = false; + if (S_ISDIR(st.st_mode)) + isDir = true; + else if (S_ISREG(st.st_mode)) + sz = st.st_size; + else + continue; + + m_entries.push_back(std::move(Entry(std::move(fp), d.cFileName, sz, isDir))); + } while (FindNextFileW(dir, &d)); + break; + case Mode::DirsThenFilesSorted: + case Mode::DirsSorted: + { + std::map sort; + do + { + if (!wcscmp(d.cFileName, _S(".")) || !wcscmp(d.cFileName, _S(".."))) + continue; + if (noHidden && (d.cFileName[0] == L'.' || (d.dwFileAttributes & FILE_ATTRIBUTE_HIDDEN) != 0)) + continue; + SystemString fp(path); + fp +=_S('/'); + fp += d.cFileName; + Sstat st; + if (Stat(fp.c_str(), &st) || !S_ISDIR(st.st_mode)) + continue; + sort.emplace(std::make_pair(d.cFileName, Entry(std::move(fp), d.cFileName, 0, true))); + } while (FindNextFileW(dir, &d)); + + if (reverse) + for (auto it=sort.crbegin() ; it != sort.crend() ; ++it) + m_entries.push_back(std::move(it->second)); + else + for (auto& e : sort) + m_entries.push_back(std::move(e.second)); + + if (mode == Mode::DirsSorted) + break; + FindClose(dir); + dir = FindFirstFileW(wc.c_str(), &d); + } + case Mode::FilesSorted: + { + if (mode == Mode::FilesSorted) + m_entries.clear(); + + if (sizeSort) + { + std::multimap sort; + do + { + if (!wcscmp(d.cFileName, _S(".")) || !wcscmp(d.cFileName, _S(".."))) + continue; + if (noHidden && (d.cFileName[0] == L'.' || (d.dwFileAttributes & FILE_ATTRIBUTE_HIDDEN) != 0)) + continue; + SystemString fp(path); + fp += _S('/'); + fp += d.cFileName; + Sstat st; + if (Stat(fp.c_str(), &st) || !S_ISREG(st.st_mode)) + continue; + sort.emplace(std::make_pair(st.st_size, Entry(std::move(fp), d.cFileName, st.st_size, false))); + } while (FindNextFileW(dir, &d)); + + if (reverse) + for (auto it=sort.crbegin() ; it != sort.crend() ; ++it) + m_entries.push_back(std::move(it->second)); + else + for (auto& e : sort) + m_entries.push_back(std::move(e.second)); + } + else + { + std::map sort; + do + { + if (!wcscmp(d.cFileName, _S(".")) || !wcscmp(d.cFileName, _S(".."))) + continue; + if (noHidden && (d.cFileName[0] == L'.' || (d.dwFileAttributes & FILE_ATTRIBUTE_HIDDEN) != 0)) + continue; + SystemString fp(path); + fp += _S('/'); + fp += d.cFileName; + Sstat st; + if (Stat(fp.c_str(), &st) || !S_ISREG(st.st_mode)) + continue; + sort.emplace(std::make_pair(d.cFileName, Entry(std::move(fp), d.cFileName, st.st_size, false))); + } while (FindNextFileW(dir, &d)); + + if (reverse) + for (auto it=sort.crbegin() ; it != sort.crend() ; ++it) + m_entries.push_back(std::move(it->second)); + else + for (auto& e : sort) + m_entries.push_back(std::move(e.second)); + } + + break; + } + } + FindClose(dir); + +#else + + DIR* dir = opendir(path); + if (!dir) + return; + const dirent* d; + switch (mode) + { + case Mode::Native: + while ((d = readdir(dir))) + { + if (!strcmp(d->d_name, ".") || !strcmp(d->d_name, "..")) + continue; + if (noHidden && d->d_name[0] == '.') + continue; + SystemString fp(path); + fp += '/'; + fp += d->d_name; + Sstat st; + if (Stat(fp.c_str(), &st)) + continue; + + size_t sz = 0; + bool isDir = false; + if (S_ISDIR(st.st_mode)) + isDir = true; + else if (S_ISREG(st.st_mode)) + sz = st.st_size; + else + continue; + + m_entries.push_back(std::move(Entry(std::move(fp), d->d_name, sz, isDir))); + } + break; + case Mode::DirsThenFilesSorted: + case Mode::DirsSorted: + { + std::map sort; + while ((d = readdir(dir))) + { + if (!strcmp(d->d_name, ".") || !strcmp(d->d_name, "..")) + continue; + if (noHidden && d->d_name[0] == '.') + continue; + SystemString fp(path); + fp += '/'; + fp += d->d_name; + Sstat st; + if (Stat(fp.c_str(), &st) || !S_ISDIR(st.st_mode)) + continue; + sort.emplace(std::make_pair(d->d_name, Entry(std::move(fp), d->d_name, 0, true))); + } + + if (reverse) + for (auto it=sort.crbegin() ; it != sort.crend() ; ++it) + m_entries.push_back(std::move(it->second)); + else + for (auto& e : sort) + m_entries.push_back(std::move(e.second)); + + if (mode == Mode::DirsSorted) + break; + rewinddir(dir); + } + case Mode::FilesSorted: + { + if (mode == Mode::FilesSorted) + m_entries.clear(); + + if (sizeSort) + { + std::multimap sort; + while ((d = readdir(dir))) + { + if (!strcmp(d->d_name, ".") || !strcmp(d->d_name, "..")) + continue; + if (noHidden && d->d_name[0] == '.') + continue; + SystemString fp(path); + fp += '/'; + fp += d->d_name; + Sstat st; + if (Stat(fp.c_str(), &st) || !S_ISREG(st.st_mode)) + continue; + sort.emplace(std::make_pair(st.st_size, Entry(std::move(fp), d->d_name, st.st_size, false))); + } + + if (reverse) + for (auto it=sort.crbegin() ; it != sort.crend() ; ++it) + m_entries.push_back(std::move(it->second)); + else + for (auto& e : sort) + m_entries.push_back(std::move(e.second)); + } + else + { + std::map sort; + while ((d = readdir(dir))) + { + if (!strcmp(d->d_name, ".") || !strcmp(d->d_name, "..")) + continue; + if (noHidden && d->d_name[0] == '.') + continue; + SystemString fp(path); + fp += '/'; + fp += d->d_name; + Sstat st; + if (Stat(fp.c_str(), &st) || !S_ISREG(st.st_mode)) + continue; + sort.emplace(std::make_pair(d->d_name, Entry(std::move(fp), d->d_name, st.st_size, false))); + } + + if (reverse) + for (auto it=sort.crbegin() ; it != sort.crend() ; ++it) + m_entries.push_back(std::move(it->second)); + else + for (auto& e : sort) + m_entries.push_back(std::move(e.second)); + } + + break; + } + } + closedir(dir); + +#endif +} + +} diff --git a/lib/DiscBase.cpp b/lib/DiscBase.cpp index f4718ee..2f43ba9 100644 --- a/lib/DiscBase.cpp +++ b/lib/DiscBase.cpp @@ -1,12 +1,16 @@ #include "NOD/DiscBase.hpp" #include "NOD/IFileIO.hpp" +#include "NOD/DirectoryEnumerator.hpp" #include "NOD/NOD.hpp" +#include #include #ifndef _WIN32 #include #endif +#include + namespace NOD { @@ -61,7 +65,8 @@ void DiscBase::IPartition::parseDOL(IPartReadStream& s) m_dolSz = dolSize; } -bool DiscBase::IPartition::Node::extractToDirectory(const SystemString& basePath, const ExtractionContext& ctx) const +bool DiscBase::IPartition::Node::extractToDirectory(const SystemString& basePath, + const ExtractionContext& ctx) const { SystemStringView nameView(getName()); SystemString path = basePath + _S("/") + nameView.sys_str(); @@ -95,7 +100,8 @@ bool DiscBase::IPartition::Node::extractToDirectory(const SystemString& basePath return true; } -bool DiscBase::IPartition::extractToDirectory(const SystemString& path, const ExtractionContext& ctx) +bool DiscBase::IPartition::extractToDirectory(const SystemString& path, + const ExtractionContext& ctx) { Sstat theStat; if (Mkdir(path.c_str(), 0755) && errno != EEXIST) @@ -128,4 +134,121 @@ bool DiscBase::IPartition::extractToDirectory(const SystemString& path, const Ex return m_nodes[0].extractToDirectory(path, ctx); } +static uint64_t GetInode(const SystemChar* path) +{ + uint64_t inode; +#if _WIN32 + OFSTRUCT ofs; + HFILE fp = OpenFile(path, &ofs, OF_READ); + if (fp == HFILE_ERROR) + LogModule.report(LogVisor::FatalError, "unable to open %s", path); + BY_HANDLE_FILE_INFORMATION info; + if (!GetFileInformationByHandle(fp, &info)) + LogModule.report(LogVisor::FatalError, "unable to GetFileInformationByHandle %s", path); + inode = info.nFileIndexHigh << 32; + inode |= info.nFileIndexLow; + CloseHandle(fp); +#else + struct stat st; + if (stat(path, &st)) + LogModule.report(LogVisor::FatalError, "unable to stat %s", path); + inode = uint64_t(st.st_ino); +#endif + return inode; +} + +void DiscBuilderBase::IPartitionBuilder::recursiveBuildNodes(const SystemChar* dirIn, + uint64_t dolInode, + std::function incParents) +{ + DirectoryEnumerator dEnum(dirIn, DirectoryEnumerator::Mode::DirsThenFilesSorted, false, false, true); + for (const DirectoryEnumerator::Entry& e : dEnum) + { + if (e.m_isDir) + { + size_t dirNodeIdx = m_buildNodes.size(); + m_buildNodes.emplace_back(true, m_buildNameOff, 0, 1); + addBuildName(e.m_name); + incParents(); + recursiveBuildNodes(e.m_path.c_str(), dolInode, [&](){m_buildNodes[dirNodeIdx].incrementLength(); incParents();}); + } + else + { + size_t fileSz = ROUND_UP_32(e.m_fileSz); + uint64_t fileOff = userAllocate(fileSz); + if (dolInode == GetInode(e.m_path.c_str())) + m_dolOffset = fileOff; + std::unique_ptr ws = m_parent.getFileIO().beginWriteStream(fileOff); + FILE* fp = Fopen(e.m_path.c_str(), _S("rb"), FileLockType::Read); + if (!fp) + LogModule.report(LogVisor::FatalError, "unable to open '%s' for reading", e.m_path.c_str()); + char buf[8192]; + size_t xferSz = 0; + ++m_parent.m_progressIdx; + while (xferSz < e.m_fileSz) + { + size_t rdSz = fread(buf, 1, std::min(8192ul, e.m_fileSz - xferSz), fp); + if (!rdSz) + break; + ws->write(buf, rdSz); + xferSz += rdSz; + m_parent.m_progressCB(m_parent.m_progressIdx, e.m_name, xferSz); + } + fclose(fp); + for (size_t i=0 ; iwrite("\xff", 1); + m_buildNodes.emplace_back(false, m_buildNameOff, fileOff, fileSz); + addBuildName(e.m_name); + incParents(); + } + } +} + +bool DiscBuilderBase::IPartitionBuilder::buildFromDirectory(const SystemChar* dirIn, + const SystemChar* dolIn, + const SystemChar* apploaderIn) +{ + if (!dirIn || !dolIn || !apploaderIn) + LogModule.report(LogVisor::FatalError, "all arguments must be supplied to buildFromDirectory()"); + + /* Clear file */ + m_parent.getFileIO().beginWriteStream(); + + m_buildNodes.emplace_back(true, m_buildNameOff, 0, 1); + addBuildName(_S("")); + recursiveBuildNodes(dirIn, GetInode(dolIn), [&](){m_buildNodes[0].incrementLength();}); + + if (!m_dolOffset) + { + Sstat dolStat; + if (Stat(dolIn, &dolStat)) + LogModule.report(LogVisor::FatalError, "unable to stat %s", dolIn); + size_t fileSz = ROUND_UP_32(dolStat.st_size); + uint64_t fileOff = userAllocate(fileSz); + m_dolOffset = fileOff; + std::unique_ptr ws = m_parent.getFileIO().beginWriteStream(fileOff); + FILE* fp = Fopen(dolIn, _S("rb"), FileLockType::Read); + if (!fp) + LogModule.report(LogVisor::FatalError, "unable to open '%s' for reading", dolIn); + char buf[8192]; + size_t xferSz = 0; + SystemString dolName(dolIn); + ++m_parent.m_progressIdx; + while (xferSz < dolStat.st_size) + { + size_t rdSz = fread(buf, 1, std::min(8192ul, dolStat.st_size - xferSz), fp); + if (!rdSz) + break; + ws->write(buf, rdSz); + xferSz += rdSz; + m_parent.m_progressCB(m_parent.m_progressIdx, dolName, xferSz); + } + fclose(fp); + for (size_t i=0 ; iwrite("\xff", 1); + } + + return true; +} + } diff --git a/lib/DiscGCN.cpp b/lib/DiscGCN.cpp index a45e84e..e664b20 100644 --- a/lib/DiscGCN.cpp +++ b/lib/DiscGCN.cpp @@ -109,17 +109,97 @@ DiscGCN::DiscGCN(std::unique_ptr&& dio) m_partitions.emplace_back(new PartitionGCN(*this, IPartition::Kind::Data, 0)); } -bool DiscGCN::packFromDirectory(const SystemChar* dataPath, const SystemChar* updatePath, - const SystemChar* outPath, const char gameID[6], const char* gameTitle, - bool korean) +class PartitionBuilderGCN : public DiscBuilderBase::IPartitionBuilder { - std::unique_ptr ws = m_discIO->beginWriteStream(0); - Header header(gameID, gameTitle); - header.write(*ws); + uint64_t m_curUser = 0x57058000; +public: + PartitionBuilderGCN(DiscBuilderBase& parent, Kind kind, uint64_t offset, + const char gameID[6], const char* gameTitle, uint32_t fstMemoryAddr) + : DiscBuilderBase::IPartitionBuilder(parent, kind, offset, gameID, gameTitle, fstMemoryAddr) {} - ws = m_discIO->beginWriteStream(0x420); + uint64_t userAllocate(uint64_t reqSz) + { + m_curUser -= reqSz; + m_curUser &= 0xfffffffffffffff0; + if (m_curUser < 0x30000) + { + LogModule.report(LogVisor::FatalError, "user area low mark reached"); + return -1; + } + return m_curUser; + } - return false; + bool buildFromDirectory(const SystemChar* dirIn, const SystemChar* dolIn, const SystemChar* apploaderIn) + { + bool result = DiscBuilderBase::IPartitionBuilder::buildFromDirectory(dirIn, dolIn, apploaderIn); + if (!result) + return false; + + std::unique_ptr ws = m_parent.getFileIO().beginWriteStream(0); + Header header(m_gameID, m_gameTitle.c_str(), false); + header.write(*ws); + + ws = m_parent.getFileIO().beginWriteStream(0x2440); + FILE* fp = Fopen(apploaderIn, _S("rb"), FileLockType::Read); + if (!fp) + LogModule.report(LogVisor::FatalError, "unable to open %s for reading", apploaderIn); + char buf[8192]; + size_t xferSz = 0; + SystemString apploaderName(apploaderIn); + ++m_parent.m_progressIdx; + while (true) + { + size_t rdSz = fread(buf, 1, 8192, fp); + if (!rdSz) + break; + ws->write(buf, rdSz); + xferSz += rdSz; + if (0x2440 + xferSz >= m_curUser) + LogModule.report(LogVisor::FatalError, + "apploader flows into user area (one or the other is too big)"); + m_parent.m_progressCB(m_parent.m_progressIdx, apploaderName, xferSz); + } + fclose(fp); + + size_t fstOff = ROUND_UP_32(xferSz); + size_t fstSz = sizeof(FSTNode) * m_buildNodes.size(); + for (size_t i=0 ; iwrite("\xff", 1); + ws->write(m_buildNodes.data(), fstSz); + for (const std::string& str : m_buildNames) + ws->write(str.data(), str.size()+1); + fstSz += m_buildNameOff; + fstSz = ROUND_UP_32(fstSz); + + ws = m_parent.getFileIO().beginWriteStream(0x420); + uint32_t vals[7]; + vals[0] = SBig(uint32_t(m_dolOffset)); + vals[1] = SBig(uint32_t(fstOff)); + vals[2] = SBig(uint32_t(fstSz)); + vals[3] = SBig(uint32_t(fstSz)); + vals[4] = SBig(uint32_t(m_fstMemoryAddr)); + vals[5] = SBig(uint32_t(m_curUser)); + vals[6] = SBig(uint32_t(0x57058000 - m_curUser)); + ws->write(vals, sizeof(vals)); + + return true; + } +}; + +bool DiscBuilderGCN::buildFromDirectory(const SystemChar* dirIn, const SystemChar* dolIn, + const SystemChar* apploaderIn) +{ + PartitionBuilderGCN& pb = static_cast(*m_partitions[0]); + return pb.buildFromDirectory(dirIn, dolIn, apploaderIn); +} + +DiscBuilderGCN::DiscBuilderGCN(const SystemChar* outPath, const char gameID[6], const char* gameTitle, + uint32_t fstMemoryAddr, std::function progressCB) +: DiscBuilderBase(std::move(NewFileIO(outPath)), progressCB) +{ + PartitionBuilderGCN* partBuilder = new PartitionBuilderGCN(*this, IPartitionBuilder::Kind::Data, 0, + gameID, gameTitle, fstMemoryAddr); + m_partitions.emplace_back(partBuilder); } } diff --git a/lib/DiscWii.cpp b/lib/DiscWii.cpp index d177210..7f009d8 100644 --- a/lib/DiscWii.cpp +++ b/lib/DiscWii.cpp @@ -390,14 +390,4 @@ DiscWii::DiscWii(std::unique_ptr&& dio) } } -bool DiscWii::packFromDirectory(const SystemChar* dataPath, const SystemChar* updatePath, - const SystemChar* outPath, const char gameID[6], const char* gameTitle, - bool korean) -{ - std::unique_ptr s = m_discIO->beginWriteStream(0x420); - - - return false; -} - } diff --git a/lib/FileIOFILE.cpp b/lib/FileIOFILE.cpp index 7a4250d..ffbc732 100644 --- a/lib/FileIOFILE.cpp +++ b/lib/FileIOFILE.cpp @@ -21,6 +21,8 @@ class FileIOFILE : public IFileIO public: FileIOFILE(const SystemString& path) : m_path(path) {} + FileIOFILE(const SystemChar* path) + : m_path(path) {} uint64_t size() { @@ -39,8 +41,8 @@ public: struct WriteStream : public IFileIO::IWriteStream { - FILE* fp; uint8_t buf[0x7c00]; + FILE* fp; WriteStream(const SystemString& path) { #if NOD_UCS2 @@ -51,9 +53,25 @@ public: if (!fp) LogModule.report(LogVisor::Error, _S("unable to open '%s' for writing"), path.c_str()); } - ~WriteStream() {fclose(fp);} - uint64_t write(void* buf, uint64_t length) - {return fwrite(buf, 1, length, fp);} + WriteStream(const SystemString& path, size_t offset) + { +#if NOD_UCS2 + fp = _wfopen(path.c_str(), L"r+b"); +#else + fp = fopen(path.c_str(), "r+b"); +#endif + if (!fp) + LogModule.report(LogVisor::Error, _S("unable to open '%s' for writing"), path.c_str()); + fseek(fp, offset, SEEK_SET); + } + ~WriteStream() + { + fclose(fp); + } + uint64_t write(const void* buf, uint64_t length) + { + return fwrite(buf, 1, length, fp); + } uint64_t copyFromDisc(IPartReadStream& discio, uint64_t length) { uint64_t read = 0; @@ -78,7 +96,13 @@ public: } }; std::unique_ptr beginWriteStream() const - {return std::unique_ptr(new WriteStream(m_path));} + { + return std::unique_ptr(new WriteStream(m_path)); + } + std::unique_ptr beginWriteStream(size_t offset) const + { + return std::unique_ptr(new WriteStream(m_path, offset)); + } struct ReadStream : public IFileIO::IReadStream { @@ -94,6 +118,11 @@ public: if (!fp) LogModule.report(LogVisor::Error, _S("unable to open '%s' for reading"), path.c_str()); } + ReadStream(const SystemString& path, size_t offset) + : ReadStream(path) + { + fseek(fp, offset, SEEK_SET); + } ~ReadStream() {fclose(fp);} uint64_t read(void* buf, uint64_t length) {return fread(buf, 1, length, fp);} @@ -120,7 +149,13 @@ public: } }; std::unique_ptr beginReadStream() const - {return std::unique_ptr(new ReadStream(m_path));} + { + return std::unique_ptr(new ReadStream(m_path)); + } + std::unique_ptr beginReadStream(size_t offset) const + { + return std::unique_ptr(new ReadStream(m_path, offset)); + } }; std::unique_ptr NewFileIO(const SystemString& path) @@ -128,4 +163,9 @@ std::unique_ptr NewFileIO(const SystemString& path) return std::unique_ptr(new FileIOFILE(path)); } +std::unique_ptr NewFileIO(const SystemChar* path) +{ + return std::unique_ptr(new FileIOFILE(path)); +} + }