2016-05-21 23:58:52 -07:00
# include "CGameExporter.h"
2017-01-31 10:23:28 -08:00
# include "CGameInfo.h"
# include "CResourceIterator.h"
# include "CResourceStore.h"
2017-03-31 22:42:41 -07:00
# include "Core/CompressionUtil.h"
2016-06-05 00:57:31 -07:00
# include "Core/Resource/CWorld.h"
2018-09-22 22:54:34 -07:00
# include "Core/Resource/Script/CGameTemplate.h"
2018-12-11 21:50:46 -08:00
# include <Common/Macros.h>
2016-05-30 23:45:30 -07:00
# include <Common/CScopedTimer.h>
2017-05-04 11:18:52 -07:00
# include <Common/FileIO.h>
2016-05-21 23:58:52 -07:00
# include <Common/FileUtil.h>
2016-08-09 20:58:27 -07:00
# include <Common/Serialization/CXMLWriter.h>
2018-07-07 16:17:33 -07:00
# include <nod/nod.hpp>
2019-11-24 18:46:57 -08:00
# include <nod/DiscBase.hpp>
2016-06-05 00:57:31 -07:00
# include <tinyxml2.h>
2016-05-21 23:58:52 -07:00
# define LOAD_PAKS 1
2016-07-05 00:45:42 -07:00
# define SAVE_PACKAGE_DEFINITIONS 1
2017-01-31 10:23:28 -08:00
# define USE_ASSET_NAME_MAP 1
2016-07-05 19:09:21 -07:00
# define EXPORT_COOKED 1
2016-05-21 23:58:52 -07:00
2019-05-25 23:24:13 -07:00
# if NOD_UCS2
2019-05-26 18:29:10 -07:00
# define TStringToNodString(string) ToWChar(string)
2019-05-25 23:24:13 -07:00
# else
2019-05-26 18:29:10 -07:00
# define TStringToNodString(string) *string
2019-05-25 23:24:13 -07:00
# endif
2017-07-24 20:08:12 -07:00
CGameExporter : : CGameExporter ( EDiscType DiscType , EGame Game , bool FrontEnd , ERegion Region , const TString & rkGameName , const TString & rkGameID , float BuildVersion )
2017-02-06 08:20:18 -08:00
: mGame ( Game )
, mRegion ( Region )
, mGameName ( rkGameName )
, mGameID ( rkGameID )
, mBuildVersion ( BuildVersion )
2017-07-24 20:08:12 -07:00
, mDiscType ( DiscType )
, mFrontEnd ( FrontEnd )
2017-05-21 17:01:09 -07:00
, mpProgress ( nullptr )
2016-05-21 23:58:52 -07:00
{
2018-10-07 16:53:19 -07:00
ASSERT ( mGame ! = EGame : : Invalid ) ;
2018-09-20 12:11:42 -07:00
ASSERT ( mRegion ! = ERegion : : Unknown ) ;
2016-05-21 23:58:52 -07:00
}
2017-05-21 17:01:09 -07:00
bool CGameExporter : : Export ( nod : : DiscBase * pDisc , const TString & rkOutputDir , CAssetNameMap * pNameMap , CGameInfo * pGameInfo , IProgressNotifier * pProgress )
2016-05-21 23:58:52 -07:00
{
2016-05-30 23:45:30 -07:00
SCOPED_TIMER ( ExportGame ) ;
2016-07-04 19:28:17 -07:00
2017-02-06 08:20:18 -08:00
mpDisc = pDisc ;
mpNameMap = pNameMap ;
mpGameInfo = pGameInfo ;
mExportDir = FileUtil : : MakeAbsolute ( rkOutputDir ) ;
2017-05-05 13:06:42 -07:00
mDiscDir = " Disc/ " ;
mWorldsDirName = " Worlds/ " ;
2017-02-06 08:20:18 -08:00
2017-07-26 02:42:57 -07:00
// Export directory must be empty!
if ( FileUtil : : Exists ( mExportDir ) & & ! FileUtil : : IsEmpty ( mExportDir ) )
return false ;
FileUtil : : MakeDirectory ( mExportDir ) ;
2017-05-21 17:01:09 -07:00
// Init progress
mpProgress = pProgress ;
mpProgress - > SetNumTasks ( eES_NumSteps ) ;
2017-02-06 08:20:18 -08:00
// Extract disc
if ( ! ExtractDiscData ( ) )
return false ;
2016-09-16 01:47:46 -07:00
// Create project
2017-02-06 08:20:18 -08:00
mpProject = CGameProject : : CreateProjectForExport (
mExportDir ,
mGame ,
mRegion ,
mGameID ,
2017-07-09 12:44:06 -07:00
mBuildVersion ) ;
2017-02-06 08:20:18 -08:00
mpProject - > SetProjectName ( mGameName ) ;
2016-09-16 01:47:46 -07:00
mpStore = mpProject - > ResourceStore ( ) ;
2017-07-01 18:03:56 -07:00
mResourcesDir = mpStore - > ResourcesDir ( ) ;
2016-08-31 01:09:13 -07:00
2017-02-11 17:35:33 -08:00
CResourceStore * pOldStore = gpResourceStore ;
gpResourceStore = mpStore ;
2017-05-21 17:01:09 -07:00
// Export cooked data
2016-05-21 23:58:52 -07:00
LoadPaks ( ) ;
ExportCookedResources ( ) ;
2017-05-21 17:01:09 -07:00
// Export editor data
if ( ! mpProgress - > ShouldCancel ( ) )
{
mpProject - > AudioManager ( ) - > LoadAssets ( ) ;
ExportResourceEditorData ( ) ;
}
2016-06-05 00:57:31 -07:00
2016-09-16 01:47:46 -07:00
// Export finished!
2017-02-06 08:20:18 -08:00
mProjectPath = mpProject - > ProjectPath ( ) ;
2016-09-16 01:47:46 -07:00
delete mpProject ;
2017-02-11 17:35:33 -08:00
if ( pOldStore ) gpResourceStore = pOldStore ;
2017-05-21 17:01:09 -07:00
return ! mpProgress - > ShouldCancel ( ) ;
2016-05-21 23:58:52 -07:00
}
2018-12-11 21:50:46 -08:00
void CGameExporter : : LoadResource ( const CAssetID & rkID , std : : vector < uint8 > & rBuffer )
2016-06-05 00:57:31 -07:00
{
SResourceInstance * pInst = FindResourceInstance ( rkID ) ;
if ( pInst ) LoadResource ( * pInst , rBuffer ) ;
}
2017-07-24 20:08:12 -07:00
bool CGameExporter : : ShouldExportDiscNode ( const nod : : Node * pkNode , bool IsInRoot )
{
2018-12-16 13:00:40 -08:00
if ( IsInRoot & & mDiscType ! = EDiscType : : Normal )
2017-07-24 20:08:12 -07:00
{
// Directories - exclude the filesystem for other games
if ( pkNode - > getKind ( ) = = nod : : Node : : Kind : : Directory )
{
// Frontend is always included; this is for compatibility with Dolphin
if ( pkNode - > getName ( ) = = " fe " )
return true ;
else if ( mFrontEnd )
return false ;
switch ( mGame )
{
2018-10-07 16:53:19 -07:00
case EGame : : Prime :
2018-12-16 13:00:40 -08:00
return ( ( mDiscType = = EDiscType : : WiiDeAsobu & & pkNode - > getName ( ) = = " MP1JPN " ) | |
( mDiscType = = EDiscType : : Trilogy & & pkNode - > getName ( ) = = " MP1 " ) ) ;
2017-07-24 20:08:12 -07:00
2018-10-07 16:53:19 -07:00
case EGame : : Echoes :
2018-12-16 13:00:40 -08:00
return ( ( mDiscType = = EDiscType : : WiiDeAsobu & & pkNode - > getName ( ) = = " MP2JPN " ) | |
( mDiscType = = EDiscType : : Trilogy & & pkNode - > getName ( ) = = " MP2 " ) ) ;
2017-07-24 20:08:12 -07:00
2018-10-07 16:53:19 -07:00
case EGame : : Corruption :
2018-12-16 13:00:40 -08:00
return ( mDiscType = = EDiscType : : Trilogy & & pkNode - > getName ( ) = = " MP3 " ) ;
2017-07-24 20:08:12 -07:00
default :
return false ;
}
}
// Files - exclude the DOLs for other games
else
{
// Again - always include frontend. Always include opening.bnr as well.
if ( pkNode - > getName ( ) = = " rs5fe_p.dol " | | pkNode - > getName ( ) = = " opening.bnr " )
return true ;
else if ( mFrontEnd )
return false ;
switch ( mGame )
{
2018-10-07 16:53:19 -07:00
case EGame : : Prime :
2018-12-16 13:00:40 -08:00
return ( ( mDiscType = = EDiscType : : WiiDeAsobu & & pkNode - > getName ( ) = = " rs5mp1jpn_p.dol " ) | |
( mDiscType = = EDiscType : : Trilogy & & pkNode - > getName ( ) = = " rs5mp1_p.dol " ) ) ;
2017-07-24 20:08:12 -07:00
2018-10-07 16:53:19 -07:00
case EGame : : Echoes :
2018-12-16 13:00:40 -08:00
return ( ( mDiscType = = EDiscType : : WiiDeAsobu & & pkNode - > getName ( ) = = " rs5mp2jpn_p.dol " ) | |
( mDiscType = = EDiscType : : Trilogy & & pkNode - > getName ( ) = = " rs5mp2_p.dol " ) ) ;
2017-07-24 20:08:12 -07:00
2018-10-07 16:53:19 -07:00
case EGame : : Corruption :
2018-12-16 13:00:40 -08:00
return ( mDiscType = = EDiscType : : Trilogy & & pkNode - > getName ( ) = = " rs5mp3_p.dol " ) ;
2017-07-24 20:08:12 -07:00
default :
return false ;
}
}
}
return true ;
}
2016-05-21 23:58:52 -07:00
// ************ PROTECTED ************
2017-02-06 08:20:18 -08:00
bool CGameExporter : : ExtractDiscData ( )
2016-05-21 23:58:52 -07:00
{
2017-02-06 08:20:18 -08:00
// todo: handle dol, apploader, multiple partitions, wii ticket blob
SCOPED_TIMER ( ExtractDiscData ) ;
2016-05-30 23:45:30 -07:00
2017-05-21 17:01:09 -07:00
// Init progress
mpProgress - > SetTask ( eES_ExtractDisc , " Extracting disc files " ) ;
2016-05-21 23:58:52 -07:00
// Create Disc output folder
2017-05-04 14:43:25 -07:00
TString AbsDiscDir = mExportDir + mDiscDir ;
2017-07-09 12:44:06 -07:00
bool IsWii = ( mBuildVersion > = 3.f ) ;
if ( IsWii ) AbsDiscDir + = " DATA/ " ;
2017-02-06 08:20:18 -08:00
FileUtil : : MakeDirectory ( AbsDiscDir ) ;
// Extract disc filesystem
2018-07-07 16:17:33 -07:00
nod : : IPartition * pDataPartition = mpDisc - > getDataPartition ( ) ;
2017-02-06 08:20:18 -08:00
nod : : ExtractionContext Context ;
Context . force = false ;
2018-07-07 16:17:33 -07:00
Context . progressCB = [ & ] ( const std : : string_view rkDesc , float ProgressPercent ) {
mpProgress - > Report ( ( int ) ( ProgressPercent * 10000 ) , 10000 , rkDesc . data ( ) ) ;
2017-05-21 17:01:09 -07:00
} ;
2017-07-09 12:44:06 -07:00
TString FilesDir = AbsDiscDir + " files/ " ;
FileUtil : : MakeDirectory ( FilesDir ) ;
2017-07-24 20:08:12 -07:00
bool Success = ExtractDiscNodeRecursive ( & pDataPartition - > getFSTRoot ( ) , FilesDir , true , Context ) ;
2017-02-06 08:20:18 -08:00
if ( ! Success ) return false ;
2017-05-21 17:01:09 -07:00
if ( ! mpProgress - > ShouldCancel ( ) )
{
2017-07-09 12:44:06 -07:00
Context . progressCB = nullptr ;
2017-02-06 08:20:18 -08:00
2017-05-21 17:01:09 -07:00
if ( IsWii )
{
2017-07-09 12:44:06 -07:00
// Extract crypto files
2019-05-25 23:24:13 -07:00
if ( ! pDataPartition - > extractCryptoFiles ( TStringToNodString ( AbsDiscDir ) , Context ) )
2017-07-09 12:44:06 -07:00
return false ;
// Extract disc header files
2019-05-25 23:24:13 -07:00
if ( ! mpDisc - > extractDiscHeaderFiles ( TStringToNodString ( AbsDiscDir ) , Context ) )
2017-07-09 12:44:06 -07:00
return false ;
2017-05-21 17:01:09 -07:00
}
2017-07-09 12:44:06 -07:00
// Extract system files
2019-05-25 23:24:13 -07:00
if ( ! pDataPartition - > extractSysFiles ( TStringToNodString ( AbsDiscDir ) , Context ) )
2017-07-09 12:44:06 -07:00
return false ;
2017-05-21 17:01:09 -07:00
return true ;
2016-05-21 23:58:52 -07:00
}
2017-02-06 08:20:18 -08:00
else
2017-05-21 17:01:09 -07:00
return false ;
2016-05-21 23:58:52 -07:00
}
2017-07-24 20:08:12 -07:00
bool CGameExporter : : ExtractDiscNodeRecursive ( const nod : : Node * pkNode , const TString & rkDir , bool RootNode , const nod : : ExtractionContext & rkContext )
2017-01-31 10:23:28 -08:00
{
2017-02-06 08:20:18 -08:00
for ( nod : : Node : : DirectoryIterator Iter = pkNode - > begin ( ) ; Iter ! = pkNode - > end ( ) ; + + Iter )
2017-01-31 10:23:28 -08:00
{
2017-07-24 20:08:12 -07:00
if ( ! ShouldExportDiscNode ( & * Iter , RootNode ) )
continue ;
2017-02-06 08:20:18 -08:00
if ( Iter - > getKind ( ) = = nod : : Node : : Kind : : File )
{
2018-07-07 16:17:33 -07:00
TString FilePath = rkDir + Iter - > getName ( ) . data ( ) ;
2019-05-25 23:24:13 -07:00
bool Success = Iter - > extractToDirectory ( TStringToNodString ( rkDir ) , rkContext ) ;
2017-02-06 08:20:18 -08:00
if ( ! Success ) return false ;
2017-01-31 10:23:28 -08:00
2017-07-24 20:08:12 -07:00
if ( FilePath . GetFileExtension ( ) . CaseInsensitiveCompare ( " pak " ) )
{
// For multi-game Wii discs, don't track packages for frontend unless we're exporting frontend
2018-12-16 13:00:40 -08:00
if ( mDiscType = = EDiscType : : Normal | | mFrontEnd | | pkNode - > getName ( ) ! = " fe " )
2017-07-24 20:08:12 -07:00
mPaks . push_back ( FilePath ) ;
}
2017-02-06 08:20:18 -08:00
}
2017-01-31 10:23:28 -08:00
2017-02-06 08:20:18 -08:00
else
2017-01-31 10:23:28 -08:00
{
2018-07-07 16:17:33 -07:00
TString Subdir = rkDir + Iter - > getName ( ) . data ( ) + " / " ;
2017-02-06 08:20:18 -08:00
bool Success = FileUtil : : MakeDirectory ( Subdir ) ;
if ( ! Success ) return false ;
2017-01-31 10:23:28 -08:00
2017-07-24 20:08:12 -07:00
Success = ExtractDiscNodeRecursive ( & * Iter , Subdir , false , rkContext ) ;
2017-02-06 08:20:18 -08:00
if ( ! Success ) return false ;
2017-01-31 10:23:28 -08:00
}
}
2017-02-06 08:20:18 -08:00
return true ;
2017-01-31 10:23:28 -08:00
}
2016-05-21 23:58:52 -07:00
// ************ RESOURCE LOADING ************
void CGameExporter : : LoadPaks ( )
{
# if LOAD_PAKS
2016-05-30 23:45:30 -07:00
SCOPED_TIMER ( LoadPaks ) ;
2017-05-04 14:43:25 -07:00
mPaks . sort ( [ ] ( const TString & rkLeft , const TString & rkRight ) - > bool {
2017-02-06 08:20:18 -08:00
return rkLeft . ToUpper ( ) < rkRight . ToUpper ( ) ;
} ) ;
2016-07-05 19:09:21 -07:00
for ( auto It = mPaks . begin ( ) ; It ! = mPaks . end ( ) ; It + + )
2016-05-21 23:58:52 -07:00
{
2017-05-04 14:43:25 -07:00
TString PakPath = * It ;
2018-12-16 13:00:40 -08:00
CFileInStream Pak ( PakPath , EEndian : : BigEndian ) ;
2016-05-21 23:58:52 -07:00
2016-07-05 19:09:21 -07:00
if ( ! Pak . IsValid ( ) )
2016-05-21 23:58:52 -07:00
{
2018-12-11 21:50:46 -08:00
errorf ( " Couldn't open pak: %s " , * PakPath ) ;
2016-07-05 19:09:21 -07:00
continue ;
}
2016-05-21 23:58:52 -07:00
2017-07-09 12:44:06 -07:00
TString RelPakPath = FileUtil : : MakeRelative ( PakPath . GetFileDirectory ( ) , mpProject - > DiscFilesystemRoot ( false ) ) ;
CPackage * pPackage = new CPackage ( mpProject , PakPath . GetFileName ( false ) , RelPakPath ) ;
2016-07-05 19:09:21 -07:00
// MP1-MP3Proto
2018-10-07 16:53:19 -07:00
if ( mGame < EGame : : Corruption )
2016-07-05 19:09:21 -07:00
{
2018-12-11 21:50:46 -08:00
uint32 PakVersion = Pak . ReadLong ( ) ;
2016-07-05 19:09:21 -07:00
Pak . Seek ( 0x4 , SEEK_CUR ) ;
ASSERT ( PakVersion = = 0x00030005 ) ;
// Echoes demo disc has a pak that ends right here.
if ( ! Pak . EoF ( ) )
2016-05-21 23:58:52 -07:00
{
2018-12-11 21:50:46 -08:00
uint32 NumNamedResources = Pak . ReadLong ( ) ;
2016-07-05 19:09:21 -07:00
ASSERT ( NumNamedResources > 0 ) ;
2018-12-11 21:50:46 -08:00
for ( uint32 iName = 0 ; iName < NumNamedResources ; iName + + )
2016-07-05 19:09:21 -07:00
{
CFourCC ResType = Pak . ReadLong ( ) ;
2016-09-16 01:47:46 -07:00
CAssetID ResID ( Pak , mGame ) ;
2018-12-11 21:50:46 -08:00
uint32 NameLen = Pak . ReadLong ( ) ;
2016-07-05 19:09:21 -07:00
TString Name = Pak . ReadString ( NameLen ) ;
2017-02-14 18:20:22 -08:00
pPackage - > AddResource ( Name , ResID , ResType ) ;
2016-07-05 19:09:21 -07:00
}
2018-12-11 21:50:46 -08:00
uint32 NumResources = Pak . ReadLong ( ) ;
2016-07-05 19:09:21 -07:00
2016-08-12 03:27:19 -07:00
// Keep track of which areas have duplicate resources
std : : set < CAssetID > PakResourceSet ;
bool AreaHasDuplicates = true ; // Default to true so that first area is always considered as having duplicates
2018-12-11 21:50:46 -08:00
for ( uint32 iRes = 0 ; iRes < NumResources ; iRes + + )
2016-07-05 19:09:21 -07:00
{
bool Compressed = ( Pak . ReadLong ( ) = = 1 ) ;
CFourCC ResType = Pak . ReadLong ( ) ;
2016-09-16 01:47:46 -07:00
CAssetID ResID ( Pak , mGame ) ;
2018-12-11 21:50:46 -08:00
uint32 ResSize = Pak . ReadLong ( ) ;
uint32 ResOffset = Pak . ReadLong ( ) ;
2016-07-05 19:09:21 -07:00
2016-08-12 03:27:19 -07:00
if ( mResourceMap . find ( ResID ) = = mResourceMap . end ( ) )
mResourceMap [ ResID ] = SResourceInstance { PakPath , ResID , ResType , ResOffset , ResSize , Compressed , false } ;
// Check for duplicate resources
if ( ResType = = " MREA " )
{
mAreaDuplicateMap [ ResID ] = AreaHasDuplicates ;
AreaHasDuplicates = false ;
}
else if ( ! AreaHasDuplicates & & PakResourceSet . find ( ResID ) ! = PakResourceSet . end ( ) )
AreaHasDuplicates = true ;
else
PakResourceSet . insert ( ResID ) ;
2016-07-05 19:09:21 -07:00
}
2016-05-21 23:58:52 -07:00
}
2016-07-05 19:09:21 -07:00
}
// MP3 + DKCR
else
{
2018-12-11 21:50:46 -08:00
uint32 PakVersion = Pak . ReadLong ( ) ;
uint32 PakHeaderLen = Pak . ReadLong ( ) ;
2016-07-05 19:09:21 -07:00
Pak . Seek ( PakHeaderLen - 0x8 , SEEK_CUR ) ;
ASSERT ( PakVersion = = 2 ) ;
struct SPakSection {
2018-12-11 21:50:46 -08:00
CFourCC Type ; uint32 Size ;
2016-07-05 19:09:21 -07:00
} ;
std : : vector < SPakSection > PakSections ;
2016-05-21 23:58:52 -07:00
2018-12-11 21:50:46 -08:00
uint32 NumPakSections = Pak . ReadLong ( ) ;
2016-07-05 19:09:21 -07:00
ASSERT ( NumPakSections = = 3 ) ;
2016-05-21 23:58:52 -07:00
2018-12-11 21:50:46 -08:00
for ( uint32 iSec = 0 ; iSec < NumPakSections ; iSec + + )
2016-05-21 23:58:52 -07:00
{
2016-07-05 19:09:21 -07:00
CFourCC Type = Pak . ReadLong ( ) ;
2018-12-11 21:50:46 -08:00
uint32 Size = Pak . ReadLong ( ) ;
2016-07-05 19:09:21 -07:00
PakSections . push_back ( SPakSection { Type , Size } ) ;
}
Pak . SeekToBoundary ( 64 ) ;
2016-05-21 23:58:52 -07:00
2018-12-11 21:50:46 -08:00
for ( uint32 iSec = 0 ; iSec < NumPakSections ; iSec + + )
2016-07-05 19:09:21 -07:00
{
2018-12-11 21:50:46 -08:00
uint32 Next = Pak . Tell ( ) + PakSections [ iSec ] . Size ;
2016-07-05 19:09:21 -07:00
// Named Resources
if ( PakSections [ iSec ] . Type = = " STRG " )
2016-05-21 23:58:52 -07:00
{
2018-12-11 21:50:46 -08:00
uint32 NumNamedResources = Pak . ReadLong ( ) ;
2016-05-21 23:58:52 -07:00
2018-12-11 21:50:46 -08:00
for ( uint32 iName = 0 ; iName < NumNamedResources ; iName + + )
2016-05-30 23:45:30 -07:00
{
2016-07-05 19:09:21 -07:00
TString Name = Pak . ReadString ( ) ;
2016-06-05 00:57:31 -07:00
CFourCC ResType = Pak . ReadLong ( ) ;
2016-09-16 01:47:46 -07:00
CAssetID ResID ( Pak , mGame ) ;
2017-02-14 18:20:22 -08:00
pPackage - > AddResource ( Name , ResID , ResType ) ;
2016-05-30 23:45:30 -07:00
}
2016-07-05 19:09:21 -07:00
}
2016-05-21 23:58:52 -07:00
2016-07-05 19:09:21 -07:00
else if ( PakSections [ iSec ] . Type = = " RSHD " )
{
ASSERT ( PakSections [ iSec + 1 ] . Type = = " DATA " ) ;
2018-12-11 21:50:46 -08:00
uint32 DataStart = Next ;
uint32 NumResources = Pak . ReadLong ( ) ;
2016-05-30 23:45:30 -07:00
2016-08-12 03:27:19 -07:00
// Keep track of which areas have duplicate resources
std : : set < CAssetID > PakResourceSet ;
bool AreaHasDuplicates = true ; // Default to true so that first area is always considered as having duplicates
2018-12-11 21:50:46 -08:00
for ( uint32 iRes = 0 ; iRes < NumResources ; iRes + + )
2016-05-30 23:45:30 -07:00
{
bool Compressed = ( Pak . ReadLong ( ) = = 1 ) ;
2016-07-05 19:09:21 -07:00
CFourCC Type = Pak . ReadLong ( ) ;
2016-09-16 01:47:46 -07:00
CAssetID ResID ( Pak , mGame ) ;
2018-12-11 21:50:46 -08:00
uint32 Size = Pak . ReadLong ( ) ;
uint32 Offset = DataStart + Pak . ReadLong ( ) ;
2016-05-30 23:45:30 -07:00
2016-08-12 03:27:19 -07:00
if ( mResourceMap . find ( ResID ) = = mResourceMap . end ( ) )
mResourceMap [ ResID ] = SResourceInstance { PakPath , ResID , Type , Offset , Size , Compressed , false } ;
// Check for duplicate resources (unnecessary for DKCR)
2018-10-07 16:53:19 -07:00
if ( mGame ! = EGame : : DKCReturns )
2016-08-12 03:27:19 -07:00
{
if ( Type = = " MREA " )
{
mAreaDuplicateMap [ ResID ] = AreaHasDuplicates ;
AreaHasDuplicates = false ;
}
else if ( ! AreaHasDuplicates & & PakResourceSet . find ( ResID ) ! = PakResourceSet . end ( ) )
AreaHasDuplicates = true ;
else
PakResourceSet . insert ( ResID ) ;
}
2016-05-30 23:45:30 -07:00
}
2016-05-21 23:58:52 -07:00
}
2016-07-05 19:09:21 -07:00
Pak . Seek ( Next , SEEK_SET ) ;
2016-05-21 23:58:52 -07:00
}
2016-07-05 19:09:21 -07:00
}
2016-05-21 23:58:52 -07:00
2016-07-05 19:09:21 -07:00
// Add package to project and save
mpProject - > AddPackage ( pPackage ) ;
2016-07-05 00:45:42 -07:00
# if SAVE_PACKAGE_DEFINITIONS
2017-02-10 13:52:47 -08:00
bool SaveSuccess = pPackage - > Save ( ) ;
ASSERT ( SaveSuccess ) ;
2016-07-05 00:45:42 -07:00
# endif
2016-05-21 23:58:52 -07:00
}
# endif
}
2018-12-11 21:50:46 -08:00
void CGameExporter : : LoadResource ( const SResourceInstance & rkResource , std : : vector < uint8 > & rBuffer )
2016-05-21 23:58:52 -07:00
{
2018-12-16 13:00:40 -08:00
CFileInStream Pak ( rkResource . PakFile , EEndian : : BigEndian ) ;
2016-05-21 23:58:52 -07:00
if ( Pak . IsValid ( ) )
{
Pak . Seek ( rkResource . PakOffset , SEEK_SET ) ;
// Handle compression
if ( rkResource . Compressed )
{
2018-10-07 16:53:19 -07:00
bool ZlibCompressed = ( mGame < = EGame : : EchoesDemo | | mGame = = EGame : : DKCReturns ) ;
2016-05-21 23:58:52 -07:00
2018-10-07 16:53:19 -07:00
if ( mGame < = EGame : : CorruptionProto )
2016-05-21 23:58:52 -07:00
{
2018-12-11 21:50:46 -08:00
std : : vector < uint8 > CompressedData ( rkResource . PakSize ) ;
2016-05-21 23:58:52 -07:00
2018-12-11 21:50:46 -08:00
uint32 UncompressedSize = Pak . ReadLong ( ) ;
2016-05-21 23:58:52 -07:00
rBuffer . resize ( UncompressedSize ) ;
Pak . ReadBytes ( CompressedData . data ( ) , CompressedData . size ( ) ) ;
if ( ZlibCompressed )
{
2018-12-11 21:50:46 -08:00
uint32 TotalOut ;
2016-05-21 23:58:52 -07:00
CompressionUtil : : DecompressZlib ( CompressedData . data ( ) , CompressedData . size ( ) , rBuffer . data ( ) , rBuffer . size ( ) , TotalOut ) ;
}
else
{
CompressionUtil : : DecompressSegmentedData ( CompressedData . data ( ) , CompressedData . size ( ) , rBuffer . data ( ) , rBuffer . size ( ) ) ;
}
}
else
{
CFourCC Magic = Pak . ReadLong ( ) ;
ASSERT ( Magic = = " CMPD " ) ;
2018-12-11 21:50:46 -08:00
uint32 NumBlocks = Pak . ReadLong ( ) ;
2016-05-21 23:58:52 -07:00
struct SCompressedBlock {
2018-12-11 21:50:46 -08:00
uint32 CompressedSize ; uint32 UncompressedSize ;
2016-05-21 23:58:52 -07:00
} ;
std : : vector < SCompressedBlock > CompressedBlocks ;
2018-12-11 21:50:46 -08:00
uint32 TotalUncompressedSize = 0 ;
for ( uint32 iBlock = 0 ; iBlock < NumBlocks ; iBlock + + )
2016-05-21 23:58:52 -07:00
{
2018-12-11 21:50:46 -08:00
uint32 CompressedSize = ( Pak . ReadLong ( ) & 0x00FFFFFF ) ;
uint32 UncompressedSize = Pak . ReadLong ( ) ;
2016-05-21 23:58:52 -07:00
TotalUncompressedSize + = UncompressedSize ;
CompressedBlocks . push_back ( SCompressedBlock { CompressedSize , UncompressedSize } ) ;
}
rBuffer . resize ( TotalUncompressedSize ) ;
2018-12-11 21:50:46 -08:00
uint32 Offset = 0 ;
2016-05-21 23:58:52 -07:00
2018-12-11 21:50:46 -08:00
for ( uint32 iBlock = 0 ; iBlock < NumBlocks ; iBlock + + )
2016-05-21 23:58:52 -07:00
{
2018-12-11 21:50:46 -08:00
uint32 CompressedSize = CompressedBlocks [ iBlock ] . CompressedSize ;
uint32 UncompressedSize = CompressedBlocks [ iBlock ] . UncompressedSize ;
2016-05-21 23:58:52 -07:00
// Block is compressed
if ( CompressedSize ! = UncompressedSize )
{
2018-12-11 21:50:46 -08:00
std : : vector < uint8 > CompressedData ( CompressedBlocks [ iBlock ] . CompressedSize ) ;
2016-05-21 23:58:52 -07:00
Pak . ReadBytes ( CompressedData . data ( ) , CompressedData . size ( ) ) ;
if ( ZlibCompressed )
{
2018-12-11 21:50:46 -08:00
uint32 TotalOut ;
2016-05-21 23:58:52 -07:00
CompressionUtil : : DecompressZlib ( CompressedData . data ( ) , CompressedData . size ( ) , rBuffer . data ( ) + Offset , UncompressedSize , TotalOut ) ;
}
else
{
CompressionUtil : : DecompressSegmentedData ( CompressedData . data ( ) , CompressedData . size ( ) , rBuffer . data ( ) + Offset , UncompressedSize ) ;
}
}
// Block is uncompressed
else
Pak . ReadBytes ( rBuffer . data ( ) + Offset , UncompressedSize ) ;
Offset + = UncompressedSize ;
}
}
}
// Handle uncompressed
else
{
rBuffer . resize ( rkResource . PakSize ) ;
Pak . ReadBytes ( rBuffer . data ( ) , rBuffer . size ( ) ) ;
}
}
}
void CGameExporter : : ExportCookedResources ( )
{
2017-05-03 02:07:34 -07:00
SCOPED_TIMER ( ExportCookedResources ) ;
2017-07-01 18:03:56 -07:00
FileUtil : : MakeDirectory ( mResourcesDir ) ;
2016-05-30 23:45:30 -07:00
2017-05-21 17:01:09 -07:00
mpProgress - > SetTask ( eES_ExportCooked , " Unpacking cooked assets " ) ;
int ResIndex = 0 ;
for ( auto It = mResourceMap . begin ( ) ; It ! = mResourceMap . end ( ) & & ! mpProgress - > ShouldCancel ( ) ; It + + , ResIndex + + )
2016-05-30 23:45:30 -07:00
{
2017-05-03 02:07:34 -07:00
SResourceInstance & rRes = It - > second ;
2017-05-21 17:01:09 -07:00
// Update progress
if ( ( ResIndex & 0x3 ) = = 0 )
mpProgress - > Report ( ResIndex , mResourceMap . size ( ) , TString : : Format ( " Unpacking asset %d/%d " , ResIndex , mResourceMap . size ( ) ) ) ;
// Export resource
2017-05-03 02:07:34 -07:00
ExportResource ( rRes ) ;
2016-07-05 19:09:21 -07:00
}
2016-09-16 01:47:46 -07:00
}
void CGameExporter : : ExportResourceEditorData ( )
{
2016-07-25 01:12:30 -07:00
{
2016-08-09 20:58:27 -07:00
// Save raw versions of resources + resource cache data files
// Note this has to be done after all cooked resources are exported
// because we have to load the resource to build its dependency tree and
// some resources will fail to load if their dependencies don't exist
SCOPED_TIMER ( SaveRawResources ) ;
2017-05-21 17:01:09 -07:00
mpProgress - > SetTask ( eES_GenerateRaw , " Generating editor data " ) ;
int ResIndex = 0 ;
2016-07-25 01:12:30 -07:00
2016-08-26 18:33:33 -07:00
// todo: we're wasting a ton of time loading the same resources over and over because most resources automatically
// load all their dependencies and then we just clear it out from memory even though we'll need it again later. we
// should really be doing this by dependency order instead of by ID order.
2017-05-21 17:01:09 -07:00
for ( CResourceIterator It ( mpStore ) ; It & & ! mpProgress - > ShouldCancel ( ) ; + + It , + + ResIndex )
2016-07-25 01:12:30 -07:00
{
2017-05-21 17:01:09 -07:00
// Update progress
2018-12-16 13:00:40 -08:00
if ( ( ResIndex & 0x3 ) = = 0 | | It - > ResourceType ( ) = = EResourceType : : Area )
2017-05-21 17:01:09 -07:00
mpProgress - > Report ( ResIndex , mpStore - > NumTotalResources ( ) , TString : : Format ( " Processing asset %d/%d: %s " ,
ResIndex , mpStore - > NumTotalResources ( ) , * It - > CookedAssetPath ( true ) . GetFileName ( ) ) ) ;
2017-05-07 19:29:33 -07:00
// Worlds need some info we can only get from the pak at export time; namely, which areas can
// have duplicates, as well as the world's internal name.
2018-12-16 13:00:40 -08:00
if ( It - > ResourceType ( ) = = EResourceType : : World )
2016-08-12 03:27:19 -07:00
{
2017-05-07 19:29:33 -07:00
CWorld * pWorld = ( CWorld * ) It - > Load ( ) ;
2016-08-12 03:27:19 -07:00
2017-05-07 19:29:33 -07:00
// Set area duplicate flags
2018-12-11 21:50:46 -08:00
for ( uint32 iArea = 0 ; iArea < pWorld - > NumAreas ( ) ; iArea + + )
2017-05-07 19:29:33 -07:00
{
CAssetID AreaID = pWorld - > AreaResourceID ( iArea ) ;
auto Find = mAreaDuplicateMap . find ( AreaID ) ;
2017-02-20 02:42:04 -08:00
2017-05-07 19:29:33 -07:00
if ( Find ! = mAreaDuplicateMap . end ( ) )
pWorld - > SetAreaAllowsPakDuplicates ( iArea , Find - > second ) ;
2016-08-12 03:27:19 -07:00
}
2017-05-07 19:29:33 -07:00
// Set world name
TString WorldName = MakeWorldName ( pWorld - > ID ( ) ) ;
pWorld - > SetName ( WorldName ) ;
2016-08-12 03:27:19 -07:00
}
2017-05-07 19:29:33 -07:00
// Save raw resource + generate dependencies
if ( It - > TypeInfo ( ) - > CanBeSerialized ( ) )
It - > Save ( true ) ;
else
It - > UpdateDependencies ( ) ;
2017-07-02 01:17:04 -07:00
// Set flags, save metadata
It - > SaveMetadata ( true ) ;
2016-07-25 01:12:30 -07:00
}
}
2017-05-21 17:01:09 -07:00
if ( ! mpProgress - > ShouldCancel ( ) )
2016-08-26 18:33:33 -07:00
{
2017-05-03 02:07:34 -07:00
// All resources should have dependencies generated, so save the project files
SCOPED_TIMER ( SaveResourceDatabase ) ;
# if EXPORT_COOKED
2017-07-04 18:02:56 -07:00
bool ResDBSaveSuccess = mpStore - > SaveDatabaseCache ( ) ;
ASSERT ( ResDBSaveSuccess ) ;
2017-05-03 02:07:34 -07:00
# endif
2017-07-04 18:02:56 -07:00
bool ProjectSaveSuccess = mpProject - > Save ( ) ;
ASSERT ( ProjectSaveSuccess ) ;
2016-08-26 18:33:33 -07:00
}
2016-05-21 23:58:52 -07:00
}
2016-06-05 00:57:31 -07:00
void CGameExporter : : ExportResource ( SResourceInstance & rRes )
{
if ( ! rRes . Exported )
{
2018-12-11 21:50:46 -08:00
std : : vector < uint8 > ResourceData ;
2016-06-05 00:57:31 -07:00
LoadResource ( rRes , ResourceData ) ;
2016-07-04 19:28:17 -07:00
// Register resource and write to file
2016-12-12 00:33:46 -08:00
TString Directory , Name ;
2017-07-02 14:56:30 -07:00
bool AutoDir , AutoName ;
2017-02-06 08:20:18 -08:00
# if USE_ASSET_NAME_MAP
2017-07-02 14:56:30 -07:00
mpNameMap - > GetNameInfo ( rRes . ResourceID , Directory , Name , AutoDir , AutoName ) ;
2017-02-06 08:20:18 -08:00
# else
2017-07-08 12:38:03 -07:00
Directory = mpStore - > DefaultAssetDirectoryPath ( mpStore - > Game ( ) ) ;
2017-02-06 08:20:18 -08:00
Name = rRes . ResourceID . ToString ( ) ;
# endif
2019-02-04 22:15:20 -08:00
CResourceEntry * pEntry = mpStore - > CreateNewResource ( rRes . ResourceID ,
CResTypeInfo : : TypeForCookedExtension ( mGame , rRes . ResourceType ) - > Type ( ) ,
Directory , Name , true ) ;
2016-06-05 00:57:31 -07:00
2017-07-02 14:56:30 -07:00
// Set flags
2018-12-16 13:00:40 -08:00
pEntry - > SetFlag ( EResEntryFlag : : IsBaseGameResource ) ;
pEntry - > SetFlagEnabled ( EResEntryFlag : : AutoResDir , AutoDir ) ;
pEntry - > SetFlagEnabled ( EResEntryFlag : : AutoResName , AutoName ) ;
2017-07-02 14:56:30 -07:00
2016-07-25 01:12:30 -07:00
# if EXPORT_COOKED
2016-08-09 20:58:27 -07:00
// Save cooked asset
2017-05-04 14:43:25 -07:00
TString OutCookedPath = pEntry - > CookedAssetPath ( ) ;
2017-02-06 08:20:18 -08:00
FileUtil : : MakeDirectory ( OutCookedPath . GetFileDirectory ( ) ) ;
2018-12-16 13:00:40 -08:00
CFileOutStream Out ( OutCookedPath , EEndian : : BigEndian ) ;
2016-06-05 00:57:31 -07:00
if ( Out . IsValid ( ) )
Out . WriteBytes ( ResourceData . data ( ) , ResourceData . size ( ) ) ;
2016-07-04 19:28:17 -07:00
ASSERT ( pEntry - > HasCookedVersion ( ) ) ;
2016-07-25 01:12:30 -07:00
# endif
2016-08-09 20:58:27 -07:00
rRes . Exported = true ;
2016-06-05 00:57:31 -07:00
}
}
2017-02-20 02:42:04 -08:00
TString CGameExporter : : MakeWorldName ( CAssetID WorldID )
{
CResourceEntry * pWorldEntry = mpStore - > FindEntry ( WorldID ) ;
2018-12-16 13:00:40 -08:00
ASSERT ( pWorldEntry & & pWorldEntry - > ResourceType ( ) = = EResourceType : : World ) ;
2017-02-20 02:42:04 -08:00
// Find the original world name in the package resource names
TString WorldName ;
2018-12-11 21:50:46 -08:00
for ( uint32 iPkg = 0 ; iPkg < mpProject - > NumPackages ( ) ; iPkg + + )
2017-02-20 02:42:04 -08:00
{
CPackage * pPkg = mpProject - > PackageByIndex ( iPkg ) ;
2018-12-11 21:50:46 -08:00
for ( uint32 iRes = 0 ; iRes < pPkg - > NumNamedResources ( ) ; iRes + + )
2017-02-20 02:42:04 -08:00
{
const SNamedResource & rkRes = pPkg - > NamedResourceByIndex ( iRes ) ;
2017-07-08 12:38:03 -07:00
if ( rkRes . ID = = WorldID )
2017-02-20 02:42:04 -08:00
{
WorldName = rkRes . Name ;
2017-07-08 12:38:03 -07:00
if ( WorldName . EndsWith ( " _NODEPEND " ) )
WorldName = WorldName . ChopBack ( 9 ) ;
2017-02-20 02:42:04 -08:00
break ;
}
}
if ( ! WorldName . IsEmpty ( ) ) break ;
}
// Fix up the name; remove date/time, leading exclamation points, etc
if ( ! WorldName . IsEmpty ( ) )
{
// World names are basically formatted differently in every game...
// MP1 demo - Remove ! from the beginning
2018-10-07 16:53:19 -07:00
if ( mGame = = EGame : : PrimeDemo )
2017-02-20 02:42:04 -08:00
{
if ( WorldName . StartsWith ( ' ! ' ) )
WorldName = WorldName . ChopFront ( 1 ) ;
}
// MP1 - Remove prefix characters and ending date
2018-10-07 16:53:19 -07:00
else if ( mGame = = EGame : : Prime )
2017-02-20 02:42:04 -08:00
{
WorldName = WorldName . ChopFront ( 2 ) ;
bool StartedDate = false ;
while ( ! WorldName . IsEmpty ( ) )
{
char Chr = WorldName . Back ( ) ;
if ( ! StartedDate & & Chr > = ' 0 ' & & Chr < = ' 9 ' )
StartedDate = true ;
else if ( StartedDate & & Chr ! = ' _ ' & & ( Chr < ' 0 ' | | Chr > ' 9 ' ) )
break ;
WorldName = WorldName . ChopBack ( 1 ) ;
}
}
// MP2 demo - Use text between the first and second underscores
2018-10-07 16:53:19 -07:00
else if ( mGame = = EGame : : EchoesDemo )
2017-02-20 02:42:04 -08:00
{
2018-12-11 21:50:46 -08:00
uint32 UnderscoreA = WorldName . IndexOf ( ' _ ' ) ;
uint32 UnderscoreB = WorldName . IndexOf ( ' _ ' , UnderscoreA + 1 ) ;
2017-02-20 02:42:04 -08:00
if ( UnderscoreA ! = UnderscoreB & & UnderscoreA ! = - 1 & & UnderscoreB ! = - 1 )
WorldName = WorldName . SubString ( UnderscoreA + 1 , UnderscoreB - UnderscoreA - 1 ) ;
}
// MP2 - Remove text before first underscore and after last underscore, strip remaining underscores (except multiplayer maps, which have one underscore)
2018-10-07 16:53:19 -07:00
else if ( mGame = = EGame : : Echoes )
2017-02-20 02:42:04 -08:00
{
2018-12-11 21:50:46 -08:00
uint32 FirstUnderscore = WorldName . IndexOf ( ' _ ' ) ;
uint32 LastUnderscore = WorldName . LastIndexOf ( ' _ ' ) ;
2017-02-20 02:42:04 -08:00
if ( FirstUnderscore ! = LastUnderscore & & FirstUnderscore ! = - 1 & & LastUnderscore ! = - 1 )
{
WorldName = WorldName . ChopBack ( WorldName . Size ( ) - LastUnderscore ) ;
WorldName = WorldName . ChopFront ( FirstUnderscore + 1 ) ;
WorldName . Remove ( ' _ ' ) ;
}
}
// MP3 proto - Remove ! from the beginning and all text after last underscore
2018-10-07 16:53:19 -07:00
else if ( mGame = = EGame : : CorruptionProto )
2017-02-20 02:42:04 -08:00
{
if ( WorldName . StartsWith ( ' ! ' ) )
WorldName = WorldName . ChopFront ( 1 ) ;
2018-12-11 21:50:46 -08:00
uint32 LastUnderscore = WorldName . LastIndexOf ( ' _ ' ) ;
2017-02-20 02:42:04 -08:00
WorldName = WorldName . ChopBack ( WorldName . Size ( ) - LastUnderscore ) ;
}
// MP3 - Remove text after last underscore
2018-10-07 16:53:19 -07:00
else if ( mGame = = EGame : : Corruption )
2017-02-20 02:42:04 -08:00
{
2018-12-11 21:50:46 -08:00
uint32 LastUnderscore = WorldName . LastIndexOf ( ' _ ' ) ;
2017-02-20 02:42:04 -08:00
2017-07-08 12:38:03 -07:00
if ( LastUnderscore ! = - 1 & & ! WorldName . StartsWith ( " front_end_ " ) )
2017-02-20 02:42:04 -08:00
WorldName = WorldName . ChopBack ( WorldName . Size ( ) - LastUnderscore ) ;
}
2017-07-08 12:38:03 -07:00
// DKCR - Remove text prior to first underscore
2018-10-07 16:53:19 -07:00
else if ( mGame = = EGame : : DKCReturns )
2017-02-20 02:42:04 -08:00
{
2018-12-11 21:50:46 -08:00
uint32 Underscore = WorldName . IndexOf ( ' _ ' ) ;
2017-07-08 12:38:03 -07:00
WorldName = WorldName . ChopFront ( Underscore + 1 ) ;
2017-02-20 02:42:04 -08:00
}
}
return WorldName ;
}