AxioDL
/
nod
mirror of https://github.com/AxioDL/nod.git
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Huge compile performance refactor

This commit is contained in:
Jack Andersen 2017-12-28 21:57:54 -10:00
parent 34de6276b0
commit a557f86974
16 changed files with 281 additions and 280 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
driver/main.cpp
View File

@ -1,5 +1,5 @@
#include <stdio.h>
#include <string.h>
#include <cstdio>
#include <cstring>
#include "logvisor/logvisor.hpp"
#include "nod/nod.hpp"

96
include/nod/DiscBase.hpp
View File

@ -5,8 +5,8 @@
#include <memory>
#include <string>
#include <unordered_map>
#include <stdio.h>
#include <stdint.h>
#include <cstdio>
#include <cstdint>
#include <functional>
#include "Util.hpp"
#include "IDiscIO.hpp"
@ -186,27 +186,8 @@ struct BI2Header
};
struct ExtractionContext;
class DiscBase
{
public:
virtual ~DiscBase() = default;
class IPartition
{
public:
virtual ~IPartition() = default;
struct DOLHeader
{
uint32_t textOff[7];
uint32_t dataOff[11];
uint32_t textStarts[7];
uint32_t dataStarts[11];
uint32_t textSizes[7];
uint32_t dataSizes[11];
uint32_t bssStart;
uint32_t bssSize;
uint32_t entryPoint;
};
class IPartition;
class DiscBase;
class Node
{
@ -229,35 +210,12 @@ public:
std::vector<Node>::iterator m_childrenEnd;
public:
Node(const IPartition& parent, const FSTNode& node, std::string_view name)
: m_parent(parent),
m_kind(node.isDir() ? Kind::Directory : Kind::File),
m_discOffset(parent.normalizeOffset(node.getOffset())),
m_discLength(node.getLength()),
m_name(name) {}
Node(const IPartition& parent, const FSTNode& node, std::string_view name);
inline Kind getKind() const {return m_kind;}
inline std::string_view getName() const {return m_name;}
inline uint64_t size() const {return m_discLength;}
std::unique_ptr<IPartReadStream> beginReadStream(uint64_t offset=0) const
{
if (m_kind != Kind::File)
{
LogModule.report(logvisor::Error, "unable to stream a non-file %s", m_name.c_str());
return std::unique_ptr<IPartReadStream>();
}
return m_parent.beginReadStream(m_discOffset + offset);
}
std::unique_ptr<uint8_t[]> getBuf() const
{
if (m_kind != Kind::File)
{
LogModule.report(logvisor::Error, "unable to buffer a non-file %s", m_name.c_str());
return std::unique_ptr<uint8_t[]>();
}
uint8_t* buf = new uint8_t[m_discLength];
beginReadStream()->read(buf, m_discLength);
return std::unique_ptr<uint8_t[]>(buf);
}
std::unique_ptr<IPartReadStream> beginReadStream(uint64_t offset=0) const;
std::unique_ptr<uint8_t[]> getBuf() const;
inline std::vector<Node>::iterator rawBegin() const {return m_childrenBegin;}
inline std::vector<Node>::iterator rawEnd() const {return m_childrenEnd;}
@ -265,8 +223,7 @@ public:
{
friend class Node;
std::vector<Node>::iterator m_it;
DirectoryIterator(const std::vector<Node>::iterator& it)
: m_it(it) {}
DirectoryIterator(const std::vector<Node>::iterator& it) : m_it(it) {}
public:
using iterator_category = std::forward_iterator_tag;
using value_type = Node;
@ -306,6 +263,24 @@ public:
bool extractToDirectory(SystemStringView basePath, const ExtractionContext& ctx) const;
};
class IPartition
{
public:
virtual ~IPartition() = default;
struct DOLHeader
{
uint32_t textOff[7];
uint32_t dataOff[11];
uint32_t textStarts[7];
uint32_t dataStarts[11];
uint32_t textSizes[7];
uint32_t dataSizes[11];
uint32_t bssStart;
uint32_t bssSize;
uint32_t entryPoint;
};
protected:
Header m_header;
BI2Header m_bi2Header;
@ -389,6 +364,10 @@ public:
bool extractSysFiles(SystemStringView path, const ExtractionContext& ctx) const;
};
class DiscBase
{
public:
virtual ~DiscBase() = default;
protected:
std::unique_ptr<IDiscIO> m_discIO;
Header m_header;
@ -454,16 +433,16 @@ public:
std::function<void(void)> incParents,
size_t parentDirIdx);
void recursiveMergeNodesPre(const DiscBase::IPartition::Node* nodeIn, SystemStringView dirIn);
void recursiveMergeNodesPre(const Node* nodeIn, SystemStringView dirIn);
bool recursiveMergeNodes(IPartWriteStream& ws, bool system,
const DiscBase::IPartition::Node* nodeIn, SystemStringView dirIn,
const Node* nodeIn, SystemStringView dirIn,
SystemStringView keyPath);
bool recursiveMergeFST(const DiscBase::IPartition::Node* nodeIn,
bool recursiveMergeFST(const Node* nodeIn,
SystemStringView dirIn, std::function<void(void)> incParents,
SystemStringView keyPath);
static bool RecursiveCalculateTotalSize(uint64_t& totalSz,
const DiscBase::IPartition::Node* nodeIn,
const Node* nodeIn,
SystemStringView dirIn);
void addBuildName(SystemStringView str)
@ -485,8 +464,8 @@ public:
virtual std::unique_ptr<IPartWriteStream> beginWriteStream(uint64_t offset)=0;
bool buildFromDirectory(IPartWriteStream& ws, SystemStringView dirIn);
static uint64_t CalculateTotalSizeBuild(SystemStringView dirIn, PartitionKind kind, bool isWii);
bool mergeFromDirectory(IPartWriteStream& ws, const DiscBase::IPartition* partIn, SystemStringView dirIn);
static uint64_t CalculateTotalSizeMerge(const DiscBase::IPartition* partIn, SystemStringView dirIn);
bool mergeFromDirectory(IPartWriteStream& ws, const IPartition* partIn, SystemStringView dirIn);
static uint64_t CalculateTotalSizeMerge(const IPartition* partIn, SystemStringView dirIn);
};
protected:
SystemString m_outPath;
@ -523,9 +502,6 @@ public:
IFileIO& getFileIO() { return *m_fileIO; }
};
using Partition = DiscBase::IPartition;
using Node = Partition::Node;
}
#endif // __NOD_DISC_BASE__

6
include/nod/IDiscIO.hpp
View File

@ -2,9 +2,9 @@
#define __NOD_IDISC_IO__
#include <memory>
#include <stdlib.h>
#include <stdio.h>
#include <stdint.h>
#include <cstdlib>
#include <cstdio>
#include <cstdint>
#if NOD_ATHENA
#include <athena/IStreamReader.hpp>

2
include/nod/IFileIO.hpp
View File

@ -3,7 +3,7 @@
#include <memory>
#include <functional>
#include <stdlib.h>
#include <cstdlib>
#include "IDiscIO.hpp"
#include "Util.hpp"

2
include/nod/Util.hpp
View File

@ -20,7 +20,7 @@
#include <ctype.h>
#include <sys/file.h>
#include <unistd.h>
#include <errno.h>
#include <cerrno>
#include <sys/param.h>
#include <sys/statvfs.h>

4
include/nod/aes.hpp
View File

@ -1,8 +1,8 @@
#ifndef __AES_HPP__
#define __AES_HPP__
#include <stdint.h>
#include <stdlib.h>
#include <cstdint>
#include <cstdlib>
#include <memory>
namespace nod

107
lib/DiscBase.cpp
View File

@ -3,8 +3,8 @@
#include "nod/DirectoryEnumerator.hpp"
#include "nod/nod.hpp"
#include <stdio.h>
#include <errno.h>
#include <cstdio>
#include <cerrno>
#ifndef _WIN32
#include <unistd.h>
@ -53,7 +53,7 @@ const SystemChar* getKindString(PartitionKind kind)
}
}
void DiscBase::IPartition::parseFST(IPartReadStream& s)
void IPartition::parseFST(IPartReadStream& s)
{
std::unique_ptr<uint8_t[]> fst(new uint8_t[m_fstSz]);
s.seek(m_fstOff);
@ -88,7 +88,7 @@ void DiscBase::IPartition::parseFST(IPartReadStream& s)
}
}
void DiscBase::IPartition::parseDOL(IPartReadStream& s)
void IPartition::parseDOL(IPartReadStream& s)
{
/* Read Dol header */
DOLHeader dolHeader;
@ -104,8 +104,36 @@ void DiscBase::IPartition::parseDOL(IPartReadStream& s)
m_dolSz = dolSize;
}
bool DiscBase::IPartition::Node::extractToDirectory(SystemStringView basePath,
const ExtractionContext& ctx) const
Node::Node(const IPartition& parent, const FSTNode& node, std::string_view name)
: m_parent(parent),
m_kind(node.isDir() ? Kind::Directory : Kind::File),
m_discOffset(parent.normalizeOffset(node.getOffset())),
m_discLength(node.getLength()),
m_name(name) {}
std::unique_ptr<IPartReadStream> Node::beginReadStream(uint64_t offset) const
{
if (m_kind != Kind::File)
{
LogModule.report(logvisor::Error, "unable to stream a non-file %s", m_name.c_str());
return std::unique_ptr<IPartReadStream>();
}
return m_parent.beginReadStream(m_discOffset + offset);
}
std::unique_ptr<uint8_t[]> Node::getBuf() const
{
if (m_kind != Kind::File)
{
LogModule.report(logvisor::Error, "unable to buffer a non-file %s", m_name.c_str());
return std::unique_ptr<uint8_t[]>();
}
uint8_t* buf = new uint8_t[m_discLength];
beginReadStream()->read(buf, m_discLength);
return std::unique_ptr<uint8_t[]>(buf);
}
bool Node::extractToDirectory(SystemStringView basePath, const ExtractionContext& ctx) const
{
SystemStringConv nameView(getName());
SystemString path = SystemString(basePath) + _S('/') + nameView.sys_str().data();
@ -148,8 +176,7 @@ bool DiscBase::IPartition::Node::extractToDirectory(SystemStringView basePath,
return true;
}
bool DiscBase::IPartition::extractToDirectory(SystemStringView path,
const ExtractionContext& ctx)
bool IPartition::extractToDirectory(SystemStringView path, const ExtractionContext& ctx)
{
m_curNodeIdx = 0;
if (Mkdir(path.data(), 0755) && errno != EEXIST)
@ -190,7 +217,7 @@ bool DiscBase::IPartition::extractToDirectory(SystemStringView path,
return m_nodes[0].extractToDirectory(fsPath, ctx);
}
bool DiscBase::IPartition::extractSysFiles(SystemStringView basePath, const ExtractionContext& ctx) const
bool IPartition::extractSysFiles(SystemStringView basePath, const ExtractionContext& ctx) const
{
SystemString basePathStr(basePath);
if (Mkdir((basePathStr + _S("/sys")).c_str(), 0755) && errno != EEXIST)
@ -403,21 +430,20 @@ bool DiscBuilderBase::PartitionBuilderBase::recursiveBuildFST(SystemStringView f
return true;
}
void DiscBuilderBase::PartitionBuilderBase::recursiveMergeNodesPre(const DiscBase::IPartition::Node* nodeIn,
SystemStringView dirIn)
void DiscBuilderBase::PartitionBuilderBase::recursiveMergeNodesPre(const Node* nodeIn, SystemStringView dirIn)
{
/* Build map of existing nodes to write-through later */
std::unordered_map<std::string, const Partition::Node*> fileNodes;
std::unordered_map<std::string, const Partition::Node*> dirNodes;
std::unordered_map<std::string, const Node*> fileNodes;
std::unordered_map<std::string, const Node*> dirNodes;
if (nodeIn)
{
fileNodes.reserve(nodeIn->size());
dirNodes.reserve(nodeIn->size());
for (const Partition::Node& ch : *nodeIn)
for (const Node& ch : *nodeIn)
{
if (ch.getKind() == Partition::Node::Kind::File)
if (ch.getKind() == Node::Kind::File)
fileNodes.insert(std::make_pair(ch.getName(), &ch));
else if (ch.getKind() == Partition::Node::Kind::Directory)
else if (ch.getKind() == Node::Kind::Directory)
dirNodes.insert(std::make_pair(ch.getName(), &ch));
}
}
@ -463,22 +489,22 @@ void DiscBuilderBase::PartitionBuilderBase::recursiveMergeNodesPre(const DiscBas
bool DiscBuilderBase::PartitionBuilderBase::recursiveMergeNodes(IPartWriteStream& ws,
bool system,
const DiscBase::IPartition::Node* nodeIn,
const Node* nodeIn,
SystemStringView dirIn,
SystemStringView keyPath)
{
/* Build map of existing nodes to write-through later */
std::unordered_map<std::string, const Partition::Node*> fileNodes;
std::unordered_map<std::string, const Partition::Node*> dirNodes;
std::unordered_map<std::string, const Node*> fileNodes;
std::unordered_map<std::string, const Node*> dirNodes;
if (nodeIn)
{
fileNodes.reserve(nodeIn->size());
dirNodes.reserve(nodeIn->size());
for (const Partition::Node& ch : *nodeIn)
for (const Node& ch : *nodeIn)
{
if (ch.getKind() == Partition::Node::Kind::File)
if (ch.getKind() == Node::Kind::File)
fileNodes.insert(std::make_pair(ch.getName(), &ch));
else if (ch.getKind() == Partition::Node::Kind::Directory)
else if (ch.getKind() == Node::Kind::Directory)
dirNodes.insert(std::make_pair(ch.getName(), &ch));
}
}
@ -565,7 +591,7 @@ bool DiscBuilderBase::PartitionBuilderBase::recursiveMergeNodes(IPartWriteStream
/* Write-through remaining file nodes */
for (const auto& p : fileNodes)
{
const Partition::Node& ch = *p.second;
const Node& ch = *p.second;
SystemStringConv sysName(ch.getName());
SystemString chKeyPath = SystemString(keyPath) + _S('/') + sysName.c_str();
@ -615,23 +641,23 @@ bool DiscBuilderBase::PartitionBuilderBase::recursiveMergeNodes(IPartWriteStream
return true;
}
bool DiscBuilderBase::PartitionBuilderBase::recursiveMergeFST(const Partition::Node* nodeIn,
bool DiscBuilderBase::PartitionBuilderBase::recursiveMergeFST(const Node* nodeIn,
SystemStringView dirIn,
std::function<void(void)> incParents,
SystemStringView keyPath)
{
/* Build map of existing nodes to write-through later */
std::unordered_map<std::string, const Partition::Node*> fileNodes;
std::unordered_map<std::string, const Partition::Node*> dirNodes;
std::unordered_map<std::string, const Node*> fileNodes;
std::unordered_map<std::string, const Node*> dirNodes;
if (nodeIn)
{
fileNodes.reserve(nodeIn->size());
dirNodes.reserve(nodeIn->size());
for (const Partition::Node& ch : *nodeIn)
for (const Node& ch : *nodeIn)
{
if (ch.getKind() == Partition::Node::Kind::File)
if (ch.getKind() == Node::Kind::File)
fileNodes.insert(std::make_pair(ch.getName(), &ch));
else if (ch.getKind() == Partition::Node::Kind::Directory)
else if (ch.getKind() == Node::Kind::Directory)
dirNodes.insert(std::make_pair(ch.getName(), &ch));
}
}
@ -701,7 +727,7 @@ bool DiscBuilderBase::PartitionBuilderBase::recursiveMergeFST(const Partition::N
/* Write-through remaining file nodes */
for (const auto& p : fileNodes)
{
const Partition::Node& ch = *p.second;
const Node& ch = *p.second;
SystemStringConv sysName(ch.getName());
SystemString chKeyPath = SystemString(keyPath) + _S('/') + sysName.sys_str().data();
@ -714,23 +740,22 @@ bool DiscBuilderBase::PartitionBuilderBase::recursiveMergeFST(const Partition::N
return true;
}
bool DiscBuilderBase::PartitionBuilderBase::RecursiveCalculateTotalSize(
uint64_t& totalSz,
const DiscBase::IPartition::Node* nodeIn,
bool DiscBuilderBase::PartitionBuilderBase::RecursiveCalculateTotalSize(uint64_t& totalSz,
const Node* nodeIn,
SystemStringView dirIn)
{
/* Build map of existing nodes to write-through later */
std::unordered_map<std::string, const Partition::Node*> fileNodes;
std::unordered_map<std::string, const Partition::Node*> dirNodes;
std::unordered_map<std::string, const Node*> fileNodes;
std::unordered_map<std::string, const Node*> dirNodes;
if (nodeIn)
{
fileNodes.reserve(nodeIn->size());
dirNodes.reserve(nodeIn->size());
for (const Partition::Node& ch : *nodeIn)
for (const Node& ch : *nodeIn)
{
if (ch.getKind() == Partition::Node::Kind::File)
if (ch.getKind() == Node::Kind::File)
fileNodes.insert(std::make_pair(ch.getName(), &ch));
else if (ch.getKind() == Partition::Node::Kind::Directory)
else if (ch.getKind() == Node::Kind::Directory)
dirNodes.insert(std::make_pair(ch.getName(), &ch));
}
}
@ -776,7 +801,7 @@ bool DiscBuilderBase::PartitionBuilderBase::RecursiveCalculateTotalSize(
/* Write-through remaining file nodes */
for (const auto& p : fileNodes)
{
const Partition::Node& ch = *p.second;
const Node& ch = *p.second;
totalSz += ROUND_UP_32(ch.size());
}
@ -867,7 +892,7 @@ uint64_t DiscBuilderBase::PartitionBuilderBase::CalculateTotalSizeBuild(SystemSt
}
bool DiscBuilderBase::PartitionBuilderBase::mergeFromDirectory(IPartWriteStream& ws,
const nod::Partition* partIn,
const IPartition* partIn,
SystemStringView dirIn)
{
if (dirIn.empty())
@ -924,7 +949,7 @@ bool DiscBuilderBase::PartitionBuilderBase::mergeFromDirectory(IPartWriteStream&
return true;
}
uint64_t DiscBuilderBase::PartitionBuilderBase::CalculateTotalSizeMerge(const DiscBase::IPartition* partIn,
uint64_t DiscBuilderBase::PartitionBuilderBase::CalculateTotalSizeMerge(const IPartition* partIn,
SystemStringView dirIn)
{
SystemString dirStr(dirIn);

4
lib/DiscGCN.cpp
View File

@ -1,12 +1,12 @@
#include "nod/DiscGCN.hpp"
#include "nod/nod.hpp"
#include <inttypes.h>
#include <cinttypes>
#define BUFFER_SZ 0x8000
namespace nod
{
class PartitionGCN : public DiscBase::IPartition
class PartitionGCN : public IPartition
{
public:
PartitionGCN(const DiscGCN& parent, uint64_t offset, bool& err)

2
lib/DiscIOISO.cpp
View File

@ -1,4 +1,4 @@
#include <stdio.h>
#include <cstdio>
#include "nod/Util.hpp"
#include "nod/IDiscIO.hpp"
#include "nod/IFileIO.hpp"

4
lib/DiscIOWBFS.cpp
View File

@ -1,5 +1,5 @@
#include <stdio.h>
#include <inttypes.h>
#include <cstdio>
#include <cinttypes>
#include "nod/Util.hpp"
#include "nod/IDiscIO.hpp"
#include "nod/IFileIO.hpp"

8
lib/DiscWii.cpp
View File

@ -1,7 +1,7 @@
#include <stdio.h>
#include <string.h>
#include <cstdio>
#include <cstring>
#include <cstdlib>
#include <inttypes.h>
#include <cinttypes>
#include "nod/DiscWii.hpp"
#include "nod/aes.hpp"
#include "nod/sha1.h"
@ -24,7 +24,7 @@ static const uint8_t COMMON_KEYS[2][16] =
0xba, 0x4c, 0x9b, 0x7e}
};
class PartitionWii : public DiscBase::IPartition
class PartitionWii : public IPartition
{
enum class SigType : uint32_t
{

6
lib/FileIOFILE.cpp
View File

@ -1,6 +1,6 @@
#include <stdio.h>
#include <stdlib.h>
#include <inttypes.h>
#include <cstdio>
#include <cstdlib>
#include <cinttypes>
#include "nod/Util.hpp"
#include "nod/IFileIO.hpp"

6
lib/FileIOWin32.cpp
View File

@ -1,6 +1,6 @@
#include <stdio.h>
#include <stdlib.h>
#include <inttypes.h>
#include <cstdio>
#include <cstdlib>
#include <cinttypes>
#include "nod/Util.hpp"
#include "nod/IFileIO.hpp"

4
lib/aes.cpp
View File

@ -1,6 +1,6 @@
#include "nod/aes.hpp"
#include <stdio.h>
#include <string.h>
#include <cstdio>
#include <cstring>
#if _WIN32
#include <intrin.h>

2
lib/nod.cpp
View File

@ -1,4 +1,4 @@
#include <stdio.h>
#include <cstdio>
#include "nod/nod.hpp"
#include "nod/DiscBase.hpp"

2
logvisor

@ -1 +1 @@
Subproject commit 408ae926d76128fa23a12607d0a1e0a970ab3554
Subproject commit beee8b397056efcdc33020a26cf69d1039f0f802