diff --git a/.gitignore b/.gitignore index a6d3f3b8f5..aa4a3a5c41 100644 --- a/.gitignore +++ b/.gitignore @@ -33,6 +33,7 @@ tools/ido_recomp/* binary ctx.c graphs/ *.c.m2c +tools/**/*dSYM/ tools/decomp-permuter/ tools/mips_to_c/ diff --git a/Makefile b/Makefile index 082203bae0..5c4cdd549c 100644 --- a/Makefile +++ b/Makefile @@ -36,12 +36,7 @@ else endif endif -# Threads to compress and extract assets with, TODO improve later -ifeq ($(DETECTED_OS),linux) - N_THREADS ?= $(shell nproc) -else - N_THREADS ?= 1 -endif +N_THREADS ?= $(shell nproc) #### Tools #### ifeq ($(shell type mips-linux-gnu-ld >/dev/null 2>/dev/null; echo $$?), 0) @@ -246,7 +241,7 @@ setup: $(MAKE) -C tools python3 tools/fixbaserom.py python3 tools/extract_baserom.py - python3 extract_assets.py -t $(N_THREADS) + python3 extract_assets.py -j $(N_THREADS) ## Assembly generation disasm: diff --git a/README.md b/README.md index 80d1e5d549..845a10cc67 100644 --- a/README.md +++ b/README.md @@ -23,26 +23,35 @@ in an experimental and research phase and cannot currently be used traditionally source code base for general changes. ``` -**This repo does not include any assets or code necessary for compiling the ROM. A prior copy of the game is required to extract the required assets.** +This is a WIP **decompilation** of ***The Legend of Zelda: Majora's Mask***. The purpose of the project is to recreate a source code base for the game from scratch, using information found inside the game along with static and/or dynamic analysis. **It is not producing a PC port.** For more information you can get in touch with the team on our [Discord server](https://discord.zelda64.dev). -This is a decompilation of Legend of Zelda: Majora's Mask (US) 1.0 +The only build currently supported is N64 US, but other versions are planned to be supported. -It builds the following ROM: +It currently builds the following ROM: * mm.us.rev1.rom.z64 `md5: 2a0a8acb61538235bc1094d297fb6556` +**This repo does not include any assets or assembly code necessary for compiling the ROM. A prior copy of the game is required to extract the required assets.** + Please refer to the following for more information: - [Website](https://zelda64.dev/) - [Discord](https://discord.zelda64.dev/) - [How to Contribute](CONTRIBUTING.md) + ## Installation ### Windows For Windows 10, install WSL and a distribution by following this [Windows Subsystem for Linux Installation Guide](https://docs.microsoft.com/en-us/windows/wsl/install-win10). -We recommend using Debian or Ubuntu 18.04 Linux distributions. +We recommend using Debian or Ubuntu 20.04 Linux distributions. + + +### MacOS + +Preparation is covered in a [separate document](docsBUILDING_MACOS.md). + ### Linux (Native or under WSL / VM) @@ -110,30 +119,23 @@ This means that something is wrong with the ROM's contents. Either the baserom f Running `make init` will also make the `./expected` directory and copy all of the files there, which will be useful when running the diff script. The diff script is useful in decompiling functions and can be ran with this command: `./tools/asm-differ/diff.py -wmo3 ` -**Note**: to speed up the build, you can either: -* Pass `-jN` to `make setup` and `make`, where N is the number of threads to use in the build, e.g. `make -j4`. The generally-accepted wisdom is to use the number of virtual cores your computer has. -* Pass `-j` to `make setup` and `make`, to use as many threads as possible, but beware that this can use too much memory on lower-end systems. -Both of these have the disadvantage that the ordering of the terminal output is scrambled, so for debugging it is best to stick to one thread (i.e. not pass `-j` or `-jN`). +**Note**: to speed up the build, you can pass `-jN` to `make setup` and `make`, where N is the number of threads to use in the build, e.g. `make -j4`. The generally-accepted wisdom is to use the number of virtual cores your computer has, which is the output of `nproc` (which should be installed as part of `coreutils`). +The disadvantage that the ordering of the terminal output is scrambled, so for debugging it is best to stick to one thread (i.e. not pass `-jN`). +(`-j` also exists, which uses unlimited jobs, but is generally slower.) -**Note**: if you rename symbols, it is recommended that you use the `tools/rename_sym.sh` to ensure that you cover all instances, including the tables which are used to generate the `asm/` directory. - -Usage: `tools/rename_sym.sh old_name new_name`. Example: - -```bash -tools/rename_sym.sh func_808A3428 EnTorch2_UpdateIdle -``` ## Contributing All contributions are welcome. This is a group effort, and even small contributions can make a difference. Some tasks also don't require much knowledge to get started. -Anyone who wishes to contribute to the OOT or MM projects **must not have accessed leaked source code at any point in time** for Nintendo 64 SDK, iQue player SDK, libultra, Ocarina of Time, Majora's Mask, Animal Crossing/Animal Forest, or any other game that shares the same game engine or significant portions of code to a Zelda 64 game or any other console similar to the Nintendo 64. +Please note that is is our strict policy that *Anyone who wishes to contribute to the OOT or MM projects **must not have accessed leaked source code at any point in time** for Nintendo 64 SDK, iQue player SDK, libultra, Ocarina of Time, Majora's Mask, Animal Crossing/Animal Forest, or any other game that shares the same game engine or significant portions of code to a Zelda 64 game or any other console similar to the Nintendo 64.* Most discussions happen on our [Discord Server](https://discord.zelda64.dev), where you are welcome to ask if you need help getting started, or if you have any questions regarding this project and other decompilation projects. For more information on getting started, see our [Contributing Guide](CONTRIBUTING.md) and our [Code Review Guidelines](REVIEWING.md) to see what code quality guidelines we follow. + ## FAQ ### Q: Why does MM use transient assembly? diff --git a/docs/BUILDING_MACOS.md b/docs/BUILDING_MACOS.md new file mode 100644 index 0000000000..17b683442c --- /dev/null +++ b/docs/BUILDING_MACOS.md @@ -0,0 +1,83 @@ +# Building on macOS + +**N.B. C++17 is required to build the asset processing program that we use (ZAPD), so check your OS version can support this before proceeding** + + +## Dependencies + +For macOS, use Homebrew to install the following dependencies: + +* coreutils +* make +* python3 +* libpng +* bash +* clang-format 11 + +You can install them with the following commands: + +```bash +brew update +brew install coreutils make python3 libpng bash clang-format@11 +``` + +(The repository expects Homebrew-installed programs to be either linked correctly in `$PATH` etc. or in their default locations.) + + +## Building mips-linux-binutils + +The following instructions are written for MacOS users but should apply to any Unix-like system, with maybe some modifications at the end regarding the bash_profile. + +Create destination dir for binutils +```bash +sudo mkdir -p /opt/cross +``` + +Create and enter local working dir +```bash +mkdir ~/binutils-tmp +cd ~/binutils-tmp +``` + +Get and extract binutils source +```bash +wget https://ftp.gnu.org/gnu/binutils/binutils-2.35.tar.bz2 +tar xjf binutils-2.35.tar.bz2 +``` +(You may find this command does not work: if so, just access the URL in a browser and save it to `~/binutils-tmp`.) + +Create and enter a build directory +```bash +mkdir build-binutils +cd build-binutils +``` + +Configure the build +```bash +../binutils-2.35/configure --target=mips-linux-gnu --prefix=/opt/cross --disable-gprof --disable-gdb --disable-werror +``` + +Make and install binutils +```bash +make -j +sudo make install +``` + +Edit your `~/.bash_profile`/`~/.zsh_profile` (or whichever shell you use) to add the new binutils binaries to the system PATH +```bash +echo "export PATH=$PATH:/opt/cross/bin" >> ~/.bash_profile +``` + +Reload ~/.bash_profile (or just launch a new terminal tab) +```bash +source ~/.bash_profile +``` + +If this worked, you can now delete the temporary directory `~/binutils-tmp`. + + +## Final note + +Apple's version of `make` is very out-of-date, so you should use the brew-installed `gmake` in place of `make` in this repo from now on. + +You should now be able to continue from [step 2](README.md#2-clone-the-repository) of the Linux instructions. diff --git a/extract_assets.py b/extract_assets.py index a7825af1fe..dda5efaec7 100755 --- a/extract_assets.py +++ b/extract_assets.py @@ -1,9 +1,8 @@ #!/usr/bin/env python3 -import argparse, json, os, signal, time, colorama -from multiprocessing import Pool, Event, Manager +import argparse, json, os, signal, time, colorama, multiprocessing -colorama.init(); +colorama.init() EXTRACTED_ASSETS_NAMEFILE = ".extracted-assets.json" @@ -71,33 +70,33 @@ def initializeWorker(abort, unaccounted: bool, extractedAssetsTracker: dict, man def main(): parser = argparse.ArgumentParser(description="baserom asset extractor") parser.add_argument("-s", "--single", help="asset path relative to assets/, e.g. objects/gameplay_keep") - parser.add_argument("-t", "--threads", help="Number of cpu cores to extract with.") parser.add_argument("-f", "--force", help="Force the extraction of every xml instead of checking the touched ones.", action="store_true") + parser.add_argument("-j", "--jobs", help="Number of cpu cores to extract with.") parser.add_argument("-u", "--unaccounted", help="Enables ZAPD unaccounted detector warning system.", action="store_true") parser.add_argument("-Z", help="Pass the argument on to ZAPD, e.g. `-ZWunaccounted` to warn about unaccounted blocks in XMLs. Each argument should be passed separately, *without* the leading dash.", metavar="ZAPD_ARG", action="append") args = parser.parse_args() - global ZAPDArgs; - ZAPDArgs = ""; + global ZAPDArgs + ZAPDArgs = "" if args.Z is not None: - badZAPDArg = False; + badZAPDArg = False for i in range(len(args.Z)): z = args.Z[i] if z[0] == '-': - print(f"{colorama.Fore.LIGHTRED_EX}error{colorama.Fore.RESET}: argument \"{z}\" starts with \"-\", which is not supported.", file=os.sys.stderr); - badZAPDArg = True; + print(f"{colorama.Fore.LIGHTRED_EX}error{colorama.Fore.RESET}: argument \"{z}\" starts with \"-\", which is not supported.", file=os.sys.stderr) + badZAPDArg = True else: - args.Z[i] = "-" + z; + args.Z[i] = "-" + z if badZAPDArg: - exit(1); + exit(1) - ZAPDArgs = " ".join(args.Z); - print("Using extra ZAPD arguments: " + ZAPDArgs); + ZAPDArgs = " ".join(args.Z) + print("Using extra ZAPD arguments: " + ZAPDArgs) global mainAbort - mainAbort = Event() - manager = Manager() + mainAbort = multiprocessing.Event() + manager = multiprocessing.Manager() signal.signal(signal.SIGINT, SignalHandler) extractedAssetsTracker = manager.dict() @@ -125,12 +124,20 @@ def main(): if file.endswith(".xml"): xmlFiles.append(fullPath) - numCores = int(args.threads or 0) - if numCores <= 0: - numCores = 1 - print("Extracting assets with " + str(numCores) + " CPU core" + ("s" if numCores > 1 else "") + ".") - with Pool(numCores, initializer=initializeWorker, initargs=(mainAbort, args.unaccounted, extractedAssetsTracker, manager)) as p: - p.map(ExtractFunc, xmlFiles) + try: + numCores = int(args.jobs or 0) + if numCores <= 0: + numCores = 1 + print("Extracting assets with " + str(numCores) + " CPU core" + ("s" if numCores > 1 else "") + ".") + with multiprocessing.get_context("fork").Pool(numCores, initializer=initializeWorker, initargs=(mainAbort, args.unaccounted, extractedAssetsTracker, manager)) as p: + p.map(ExtractFunc, xmlFiles) + except (multiprocessing.ProcessError, TypeError): + print("Warning: Multiprocessing exception ocurred.", file=os.sys.stderr) + print("Disabling mutliprocessing.", file=os.sys.stderr) + + initializeWorker(mainAbort, args.unaccounted, extractedAssetsTracker, manager) + for singlePath in xmlFiles: + ExtractFunc(singlePath) with open(EXTRACTED_ASSETS_NAMEFILE, 'w', encoding='utf-8') as f: serializableDict = dict() diff --git a/format.sh b/format.sh index 89a0906ad1..1c9389a9ac 100755 --- a/format.sh +++ b/format.sh @@ -5,6 +5,16 @@ FORMAT_OPTS="-i -style=file" TIDY_OPTS="-p . --fix --fix-errors" COMPILER_OPTS="-fno-builtin -std=gnu90 -Iinclude -Isrc -D_LANGUAGE_C -DNON_MATCHING" +# https://backreference.org/2010/05/23/sanitizing-files-with-no-trailing-newline/index.html +# "gets the last character of the file pipes it into read, which will exit with a nonzero exit code if it encounters EOF before newline (so, if the last character of the file isn't a newline). If read exits nonzero, then append a newline onto the file using echo (if read exits 0, that satisfies the ||, so the echo command isn't run)." (https://stackoverflow.com/a/34865616) +function add_final_newline () { + for file in "$@" + do + tail -c1 $file | read -r _ || echo >> $file + done +} +export -f add_final_newline + shopt -s globstar if [ -z `command -v clang-format-${FORMAT_VER}` ] @@ -20,7 +30,7 @@ if (( $# > 0 )); then echo "Running clang-tidy..." clang-tidy ${TIDY_OPTS} "$@" -- ${COMPILER_OPTS} &> /dev/null echo "Adding missing final new lines..." - sed -i -e '$a\' "$@" + add_final_newline "$@" echo "Done formatting file(s) $*" exit fi @@ -31,6 +41,6 @@ clang-format-${FORMAT_VER} ${FORMAT_OPTS} src/**/*.c echo "Running clang-tidy..." clang-tidy ${TIDY_OPTS} src/**/*.c -- ${COMPILER_OPTS} &> /dev/null echo "Adding missing final new lines..." -find src/ -type f -name "*.c" -exec sed -i -e '$a\' {} \; -find assets/xml/ -type f -name "*.xml" -exec sed -i -e '$a\' {} \; +find src/ -type f -name "*.c" -exec bash -c 'add_final_newline "$@"' bash {} + +find assets/xml/ -type f -name "*.xml" -exec bash -c 'add_final_newline "$@"' bash {} + echo "Done formatting all files." diff --git a/include/functions.h b/include/functions.h index 629514d140..d8b284149d 100644 --- a/include/functions.h +++ b/include/functions.h @@ -97,7 +97,7 @@ void FaultDrawer_SetCharPad(s8 padW, s8 padH); void FaultDrawer_SetCursor(s32 x, s32 y); void FaultDrawer_FillScreen(void); void* FaultDrawer_FormatStringFunc(void* arg, const char* str, size_t count); -void FaultDrawer_VPrintf(const char* str, char* args); +void FaultDrawer_VPrintf(const char* fmt, va_list ap); void FaultDrawer_Printf(const char* fmt, ...); void FaultDrawer_DrawText(s32 x, s32 y, const char* fmt, ...); void FaultDrawer_SetDrawerFB(void* fb, u16 w, u16 h); diff --git a/src/boot_O2/boot_80086760.c b/src/boot_O2/boot_80086760.c index 1a9b374538..3c75bcd3f7 100644 --- a/src/boot_O2/boot_80086760.c +++ b/src/boot_O2/boot_80086760.c @@ -24,30 +24,30 @@ f32 func_80086760(f32 x) { // Unused // Math_FFloorF f32 func_80086794(f32 x) { - func_80086C70(x); + return func_80086C70(x); } // Unused // Math_FCeilF f32 func_800867B4(f32 x) { - func_80086CA8(x); + return func_80086CA8(x); } // Unused // Math_FRoundF f32 func_800867D4(f32 x) { - func_80086D50(x); + return func_80086D50(x); } // Unused // Math_FTruncF f32 func_800867F4(f32 x) { - func_80086CE0(x); + return func_80086CE0(x); } // Math_FNearbyIntF f32 func_80086814(f32 x) { - func_80086D18(x); + return func_80086D18(x); } /** diff --git a/tools/ZAPD/.gitrepo b/tools/ZAPD/.gitrepo index a942638a2e..ccd6c7d649 100644 --- a/tools/ZAPD/.gitrepo +++ b/tools/ZAPD/.gitrepo @@ -6,7 +6,7 @@ [subrepo] remote = https://github.com/zeldaret/ZAPD.git branch = master - commit = 50242eca96a9c36fd84f438bab24548e73b42303 - parent = 744955732b1fc9e8a835c62440aa97799cfe8940 + commit = fd5a7f434171a33b1b3eb2a3e112fdcee6d9262d + parent = 844c24e39c96b7a3f3a40035058bda6c91ed4bbf method = merge cmdver = 0.4.3 diff --git a/tools/ZAPD/Jenkinsfile b/tools/ZAPD/Jenkinsfile index 8db69a1e30..ec9b56ac5a 100644 --- a/tools/ZAPD/Jenkinsfile +++ b/tools/ZAPD/Jenkinsfile @@ -22,13 +22,13 @@ pipeline { } } - // stage('Checkout mm') { - // steps{ - // dir('mm') { - // git url: 'https://github.com/zeldaret/mm.git' - // } - // } - // } + stage('Checkout mm') { + steps{ + dir('mm') { + git url: 'https://github.com/zeldaret/mm.git' + } + } + } } } @@ -51,20 +51,20 @@ pipeline { } } - // stage('Setup MM') { - // steps { - // dir('mm') { - // sh 'cp /usr/local/etc/roms/mm.us.rev1.z64 baserom.mm.us.rev1.z64' + stage('Setup MM') { + steps { + dir('mm') { + sh 'cp /usr/local/etc/roms/mm.us.rev1.z64 baserom.mm.us.rev1.z64' - // // Identical to `make setup` except for copying our newer ZAPD.out into mm - // sh 'make -C tools' - // sh 'cp ../ZAPD.out tools/ZAPD/' - // sh 'python3 tools/fixbaserom.py' - // sh 'python3 tools/extract_baserom.py' - // sh 'python3 extract_assets.py -t 4' - // } - // } - // } + // Identical to `make setup` except for copying our newer ZAPD.out into mm + sh 'make -C tools' + sh 'cp ../ZAPD.out tools/ZAPD/' + sh 'python3 tools/fixbaserom.py' + sh 'python3 tools/extract_baserom.py' + sh 'python3 extract_assets.py -t$(nproc)' + } + } + } } } @@ -78,14 +78,14 @@ pipeline { } } } - // stage('Build mm') { - // steps { - // dir('mm') { - // sh 'make -j disasm' - // sh 'make -j all' - // } - // } - // } + stage('Build mm') { + steps { + dir('mm') { + sh 'make -j disasm' + sh 'make -j all' + } + } + } } } } diff --git a/tools/ZAPD/README.md b/tools/ZAPD/README.md index 44d26b8b3e..448aea0339 100644 --- a/tools/ZAPD/README.md +++ b/tools/ZAPD/README.md @@ -119,6 +119,7 @@ ZAPD also accepts the following list of extra parameters: - `-tm MODE`: Test Mode (enables certain experimental features). To enable it, set `MODE` to `1`. - `-se` / `--set-exporter` : Sets which exporter to use. - `--gcc-compat` : Enables GCC compatibly mode. Slower. +- `-us` / `--unaccounted-static` : Mark unaccounted data as `static` - `-s` / `--static` : Mark every asset as `static`. - This behaviour can be overridden per asset using `Static=` in the respective XML node. - `-W...`: warning flags, see below diff --git a/tools/ZAPD/ZAPD/Declaration.h b/tools/ZAPD/ZAPD/Declaration.h index b7bb0d30d2..4a743b50fa 100644 --- a/tools/ZAPD/ZAPD/Declaration.h +++ b/tools/ZAPD/ZAPD/Declaration.h @@ -12,8 +12,7 @@ typedef uint32_t offset_t; enum class DeclarationAlignment { Align4, - Align8, - Align16 + Align8 }; enum class StaticConfig diff --git a/tools/ZAPD/ZAPD/Globals.cpp b/tools/ZAPD/ZAPD/Globals.cpp index 528a09d256..a10705e9af 100644 --- a/tools/ZAPD/ZAPD/Globals.cpp +++ b/tools/ZAPD/ZAPD/Globals.cpp @@ -152,8 +152,8 @@ bool Globals::GetSegmentedPtrName(segptr_t segAddress, ZFile* currentFile, } } - const auto& symbolFromMap = Globals::Instance->symbolMap.find(segAddress); - if (symbolFromMap != Globals::Instance->symbolMap.end()) + const auto& symbolFromMap = Globals::Instance->cfg.symbolMap.find(segAddress); + if (symbolFromMap != Globals::Instance->cfg.symbolMap.end()) { declName = "&" + symbolFromMap->second; return true; diff --git a/tools/ZAPD/ZAPD/Globals.h b/tools/ZAPD/ZAPD/Globals.h index 19e193f123..1ae753e204 100644 --- a/tools/ZAPD/ZAPD/Globals.h +++ b/tools/ZAPD/ZAPD/Globals.h @@ -58,11 +58,11 @@ public: bool verboseUnaccounted = false; bool gccCompat = false; bool forceStatic = false; + bool forceUnaccountedStatic = false; std::vector files; std::vector externalFiles; std::vector segments; - std::map symbolMap; std::string currentExporter; static std::map& GetExporterMap(); diff --git a/tools/ZAPD/ZAPD/Main.cpp b/tools/ZAPD/ZAPD/Main.cpp index fd2ed06dc0..048338fb70 100644 --- a/tools/ZAPD/ZAPD/Main.cpp +++ b/tools/ZAPD/ZAPD/Main.cpp @@ -48,7 +48,7 @@ void ErrorHandler(int sig) const char* crashEasterEgg[] = { "\tYou've met with a terrible fate, haven't you?", "\tSEA BEARS FOAM. SLEEP BEARS DREAMS. \n\tBOTH END IN THE SAME WAY: CRASSSH!", - "ZAPD has fallen and cannot get up." + "\tZAPD has fallen and cannot get up.", }; srand(time(nullptr)); @@ -216,6 +216,10 @@ int main(int argc, char* argv[]) { Globals::Instance->forceStatic = true; } + else if (arg == "-us" || arg == "--unaccounted-static") + { + Globals::Instance->forceUnaccountedStatic = true; + } } // Parse File Mode diff --git a/tools/ZAPD/ZAPD/OtherStructs/SkinLimbStructs.cpp b/tools/ZAPD/ZAPD/OtherStructs/SkinLimbStructs.cpp index 9d16de4389..0879861997 100644 --- a/tools/ZAPD/ZAPD/OtherStructs/SkinLimbStructs.cpp +++ b/tools/ZAPD/ZAPD/OtherStructs/SkinLimbStructs.cpp @@ -263,7 +263,7 @@ void Struct_800A5E28::DeclareReferences(const std::string& prefix) ZResource::DeclareReferences(varPrefix); - if (unk_4 != 0 && GETSEGNUM(unk_4) == parent->segment) + if (unk_4 != SEGMENTED_NULL && GETSEGNUM(unk_4) == parent->segment) { const auto& res = unk_4_arr.at(0); std::string unk_4_Str = res.GetDefaultName(varPrefix); @@ -293,7 +293,7 @@ void Struct_800A5E28::DeclareReferences(const std::string& prefix) decl->text = entryStr; } - if (unk_8 != 0 && GETSEGNUM(unk_8) == parent->segment) + if (unk_8 != SEGMENTED_NULL && GETSEGNUM(unk_8) == parent->segment) { uint32_t unk_8_Offset = Seg2Filespace(unk_8, parent->baseAddress); @@ -306,6 +306,8 @@ void Struct_800A5E28::DeclareReferences(const std::string& prefix) std::string dListStr = StringHelper::Sprintf("%sSkinLimbDL_%06X", varPrefix.c_str(), unk_8_Offset); unk_8_dlist->SetName(dListStr); + unk_8_dlist->DeclareVar(varPrefix, ""); + unk_8_dlist->DeclareReferences(varPrefix); parent->AddResource(unk_8_dlist); } } diff --git a/tools/ZAPD/ZAPD/Overlays/ZOverlay.cpp b/tools/ZAPD/ZAPD/Overlays/ZOverlay.cpp index 510de19aba..113c4ef339 100644 --- a/tools/ZAPD/ZAPD/Overlays/ZOverlay.cpp +++ b/tools/ZAPD/ZAPD/Overlays/ZOverlay.cpp @@ -90,7 +90,8 @@ ZOverlay* ZOverlay::FromBuild(fs::path buildPath, fs::path cfgFolderPath) std::vector readers; for (size_t i = 1; i < cfgLines.size(); i++) { - std::string elfPath = (buildPath / (cfgLines[i].substr(0, cfgLines[i].size() - 2) + ".o")).string(); + std::string elfPath = + (buildPath / (cfgLines[i].substr(0, cfgLines[i].size() - 2) + ".o")).string(); elfio* reader = new elfio(); if (!reader->load(elfPath)) diff --git a/tools/ZAPD/ZAPD/ZAnimation.cpp b/tools/ZAPD/ZAPD/ZAnimation.cpp index adb0ae7c9a..4d9266b091 100644 --- a/tools/ZAPD/ZAPD/ZAnimation.cpp +++ b/tools/ZAPD/ZAPD/ZAnimation.cpp @@ -104,7 +104,7 @@ void ZNormalAnimation::DeclareReferences(const std::string& prefix) valuesStr += "\n "; } - parent->AddDeclarationArray(rotationValuesOffset, DeclarationAlignment::Align16, + parent->AddDeclarationArray(rotationValuesOffset, DeclarationAlignment::Align4, rotationValues.size() * 2, "s16", StringHelper::Sprintf("%sFrameData", defaultPrefix.c_str()), rotationValues.size(), valuesStr); @@ -118,7 +118,7 @@ void ZNormalAnimation::DeclareReferences(const std::string& prefix) indicesStr += "\n"; } - parent->AddDeclarationArray(rotationIndicesOffset, DeclarationAlignment::Align16, + parent->AddDeclarationArray(rotationIndicesOffset, DeclarationAlignment::Align4, rotationIndices.size() * 6, "JointIndex", StringHelper::Sprintf("%sJointIndices", defaultPrefix.c_str()), rotationIndices.size(), indicesStr); @@ -385,7 +385,7 @@ size_t ZCurveAnimation::GetRawDataSize() const DeclarationAlignment ZCurveAnimation::GetDeclarationAlignment() const { - return DeclarationAlignment::Align16; + return DeclarationAlignment::Align4; } std::string ZCurveAnimation::GetSourceTypeName() const diff --git a/tools/ZAPD/ZAPD/ZDisplayList.cpp b/tools/ZAPD/ZAPD/ZDisplayList.cpp index bdb7374fe1..5e9b1b205e 100644 --- a/tools/ZAPD/ZAPD/ZDisplayList.cpp +++ b/tools/ZAPD/ZAPD/ZDisplayList.cpp @@ -27,8 +27,8 @@ ZDisplayList::ZDisplayList(ZFile* nParent) : ZResource(nParent) lastTexSizTest = F3DZEXTexSizes::G_IM_SIZ_16b; lastTexLoaded = false; lastTexIsPalette = false; - name = ""; dListType = Globals::Instance->game == ZGame::OOT_SW97 ? DListType::F3DEX : DListType::F3DZEX; + RegisterOptionalAttribute("Ucode"); } ZDisplayList::~ZDisplayList() @@ -44,13 +44,29 @@ void ZDisplayList::ExtractFromXML(tinyxml2::XMLElement* reader, uint32_t nRawDat { rawDataIndex = nRawDataIndex; ParseXML(reader); + // TODO add error handling here + bool ucodeSet = registeredAttributes.at("Ucode").wasSet; + std::string ucodeValue = registeredAttributes.at("Ucode").value; + if ((Globals::Instance->game == ZGame::OOT_SW97) || (ucodeValue == "f3dex")) + { + dListType = DListType::F3DEX; + } + else if (!ucodeSet || ucodeValue == "f3dex2") + { + dListType = DListType::F3DZEX; + } + else + { + HANDLE_ERROR_RESOURCE( + WarningType::InvalidAttributeValue, parent, this, rawDataIndex, + StringHelper::Sprintf("Invalid ucode type in node: %s\n", reader->Name()), ""); + } // Don't parse raw data of external files if (parent->GetMode() != ZFileMode::ExternalFile) { - int32_t rawDataSize = ZDisplayList::GetDListLength( - parent->GetRawData(), rawDataIndex, - Globals::Instance->game == ZGame::OOT_SW97 ? DListType::F3DEX : DListType::F3DZEX); + int32_t rawDataSize = + ZDisplayList::GetDListLength(parent->GetRawData(), rawDataIndex, dListType); numInstructions = rawDataSize / 8; ParseRawData(); } @@ -693,7 +709,8 @@ void ZDisplayList::Opcode_G_DL(uint64_t data, const std::string& prefix, char* l else if (dListDecl != nullptr) sprintf(line, "gsSPBranchList(%s),", dListDecl->varName.c_str()); else - sprintf(line, "gsSPBranchList(%sDlist0x%06" PRIX64 "),", prefix.c_str(), GETSEGOFFSET(data)); + sprintf(line, "gsSPBranchList(%sDlist0x%06" PRIX64 "),", prefix.c_str(), + GETSEGOFFSET(data)); } else { @@ -702,7 +719,8 @@ void ZDisplayList::Opcode_G_DL(uint64_t data, const std::string& prefix, char* l else if (dListDecl != nullptr) sprintf(line, "gsSPDisplayList(%s),", dListDecl->varName.c_str()); else - sprintf(line, "gsSPDisplayList(%sDlist0x%06" PRIX64 "),", prefix.c_str(), GETSEGOFFSET(data)); + sprintf(line, "gsSPDisplayList(%sDlist0x%06" PRIX64 "),", prefix.c_str(), + GETSEGOFFSET(data)); } // if (segNum == 8 || segNum == 9 || segNum == 10 || segNum == 11 || segNum == 12 || segNum == @@ -832,7 +850,7 @@ void ZDisplayList::Opcode_G_VTX(uint64_t data, char* line) { segptr_t segmented = data & 0xFFFFFFFF; references.push_back(segmented); - parent->AddDeclaration(segmented, DeclarationAlignment::Align16, 16, "Vtx", + parent->AddDeclaration(segmented, DeclarationAlignment::Align8, 16, "Vtx", StringHelper::Sprintf("0x%08X", segmented), ""); return; } @@ -1748,7 +1766,7 @@ void ZDisplayList::DeclareReferences(const std::string& prefix) if (vertices.size() > 0) { std::vector>> verticesSorted(vertices.begin(), - vertices.end()); + vertices.end()); for (size_t i = 0; i < verticesSorted.size() - 1; i++) { @@ -1796,7 +1814,7 @@ void ZDisplayList::DeclareReferences(const std::string& prefix) if (vertices.size() > 0) { std::vector>> verticesSorted(vertices.begin(), - vertices.end()); + vertices.end()); for (size_t i = 0; i < verticesSorted.size() - 1; i++) { diff --git a/tools/ZAPD/ZAPD/ZFile.cpp b/tools/ZAPD/ZAPD/ZFile.cpp index 77387fc72c..22a6a45058 100644 --- a/tools/ZAPD/ZAPD/ZFile.cpp +++ b/tools/ZAPD/ZAPD/ZFile.cpp @@ -8,6 +8,7 @@ #include "Globals.h" #include "OutputFormatter.h" #include "Utils/BinaryWriter.h" +#include "Utils/BitConverter.h" #include "Utils/Directory.h" #include "Utils/File.h" #include "Utils/MemoryStream.h" @@ -192,7 +193,7 @@ void ZFile::ParseXML(tinyxml2::XMLElement* reader, const std::string& filename) rawData = File::ReadAllBytes((basePath / name).string()); if (reader->Attribute("RangeEnd") == nullptr) - rangeEnd = rawData.size(); + rangeEnd = rawData.size(); } std::unordered_set nameSet; @@ -378,7 +379,7 @@ void ZFile::ExtractResources() ZResourceExporter* exporter = Globals::Instance->GetExporter(res->GetResourceType()); if (exporter != nullptr) { - //exporter->Save(res, Globals::Instance->outputPath.string(), &writerFile); + // exporter->Save(res, Globals::Instance->outputPath.string(), &writerFile); exporter->Save(res, Globals::Instance->outputPath.string(), &writerRes); } @@ -391,7 +392,7 @@ void ZFile::ExtractResources() File::WriteAllBytes(StringHelper::Sprintf("%s%s.bin", Globals::Instance->outputPath.string().c_str(), GetName().c_str()), - memStreamFile->ToVector()); + memStreamFile->ToVector()); } writerFile.Close(); @@ -949,9 +950,6 @@ std::string ZFile::ProcessDeclarations() defines += ProcessTextureIntersections(name); - // Account for padding/alignment - uint32_t lastAddr = 0; - // printf("RANGE START: 0x%06X - RANGE END: 0x%06X\n", rangeStart, rangeEnd); // Optimization: See if there are any arrays side by side that can be merged... @@ -1002,43 +1000,6 @@ std::string ZFile::ProcessDeclarations() { while (item.second->size % 4 != 0) item.second->size++; - - if (lastAddr != 0) - { - if (item.second->alignment == DeclarationAlignment::Align16) - { - int32_t curPtr = lastAddr + declarations[lastAddr]->size; - - while (curPtr % 4 != 0) - { - declarations[lastAddr]->size++; - curPtr++; - } - } - else if (item.second->alignment == DeclarationAlignment::Align8) - { - size_t curPtr = lastAddr + declarations[lastAddr]->size; - - while (curPtr % 4 != 0) - { - declarations[lastAddr]->size++; - curPtr++; - } - - while (curPtr % 8 != 0) - { - char buffer[2048]; - - sprintf(buffer, "u32 %s_align%02zX = 0;\n", name.c_str(), curPtr); - item.second->preText = buffer + item.second->preText; - - declarations[lastAddr]->size += 4; - curPtr += 4; - } - } - } - - lastAddr = item.first; } HandleUnaccountedData(); @@ -1199,14 +1160,15 @@ void ZFile::HandleUnaccountedData() { uint32_t lastAddr = 0; uint32_t lastSize = 0; - std::vector declsAddresses; + std::vector declsAddresses; + for (const auto& item : declarations) { declsAddresses.push_back(item.first); } bool breakLoop = false; - for (uint32_t currentAddress : declsAddresses) + for (offset_t currentAddress : declsAddresses) { if (currentAddress >= rangeEnd) { @@ -1235,7 +1197,7 @@ void ZFile::HandleUnaccountedData() } } -bool ZFile::HandleUnaccountedAddress(uint32_t currentAddress, uint32_t lastAddr, uint32_t& lastSize) +bool ZFile::HandleUnaccountedAddress(offset_t currentAddress, offset_t lastAddr, uint32_t& lastSize) { if (currentAddress != lastAddr && declarations.find(lastAddr) != declarations.end()) { @@ -1275,6 +1237,29 @@ bool ZFile::HandleUnaccountedAddress(uint32_t currentAddress, uint32_t lastAddr, xmlFilePath.c_str(), currentAddress, name.c_str(), rawData.size())); } + // Handle Align8 + if (currentAddress % 8 == 0 && diff % 8 != 0) + { + Declaration* currentDecl = GetDeclaration(currentAddress); + + if (currentDecl != nullptr) + { + if (currentDecl->alignment == DeclarationAlignment::Align8) + { + // Check removed bytes are zeroes + if (BitConverter::ToUInt32BE(rawData, unaccountedAddress + diff - 4) == 0) + { + diff -= 4; + } + } + + if (diff == 0) + { + return false; + } + } + } + for (int i = 0; i < diff; i++) { uint8_t val = rawData.at(unaccountedAddress + i); @@ -1320,7 +1305,10 @@ bool ZFile::HandleUnaccountedAddress(uint32_t currentAddress, uint32_t lastAddr, StringHelper::Sprintf("%s_%s_%06X", name.c_str(), unaccountedPrefix.c_str(), unaccountedAddress), diff, src); + decl->isUnaccounted = true; + if (Globals::Instance->forceUnaccountedStatic) + decl->staticConf = StaticConfig::On; if (nonZeroUnaccounted) { diff --git a/tools/ZAPD/ZAPD/ZFile.h b/tools/ZAPD/ZAPD/ZFile.h index 7918d5f595..ac4062d5b1 100644 --- a/tools/ZAPD/ZAPD/ZFile.h +++ b/tools/ZAPD/ZAPD/ZFile.h @@ -133,5 +133,5 @@ protected: std::string ProcessTextureIntersections(const std::string& prefix); void HandleUnaccountedData(); - bool HandleUnaccountedAddress(uint32_t currentAddress, uint32_t lastAddr, uint32_t& lastSize); + bool HandleUnaccountedAddress(offset_t currentAddress, offset_t lastAddr, uint32_t& lastSize); }; diff --git a/tools/ZAPD/ZAPD/ZLimb.cpp b/tools/ZAPD/ZAPD/ZLimb.cpp index a47615d6c6..330fbaf7ce 100644 --- a/tools/ZAPD/ZAPD/ZLimb.cpp +++ b/tools/ZAPD/ZAPD/ZLimb.cpp @@ -228,8 +228,8 @@ std::string ZLimb::GetBodySourceCode() const { std::string childName; std::string siblingName; - Globals::Instance->GetSegmentedPtrName(childPtr, parent, "Gfx", childName); - Globals::Instance->GetSegmentedPtrName(siblingPtr, parent, "Gfx", siblingName); + Globals::Instance->GetSegmentedPtrName(childPtr, parent, "LegacyLimb", childName); + Globals::Instance->GetSegmentedPtrName(siblingPtr, parent, "LegacyLimb", siblingName); entryStr += StringHelper::Sprintf("%s,\n", dListStr.c_str()); entryStr += diff --git a/tools/ZAPD/ZAPD/ZRoom/Commands/SetMesh.cpp b/tools/ZAPD/ZAPD/ZRoom/Commands/SetMesh.cpp index 69668c49c1..ba0bbe2c24 100644 --- a/tools/ZAPD/ZAPD/ZRoom/Commands/SetMesh.cpp +++ b/tools/ZAPD/ZAPD/ZRoom/Commands/SetMesh.cpp @@ -152,7 +152,6 @@ void PolygonDlist::GetSourceOutputCode(const std::string& prefix) DeclareVar(prefix, bodyStr); else decl->text = bodyStr; - } std::string PolygonDlist::GetSourceTypeName() const diff --git a/tools/ZAPD/ZAPD/ZRoom/Commands/SetSpecialObjects.cpp b/tools/ZAPD/ZAPD/ZRoom/Commands/SetSpecialObjects.cpp index 0c24a2f328..696a3de014 100644 --- a/tools/ZAPD/ZAPD/ZRoom/Commands/SetSpecialObjects.cpp +++ b/tools/ZAPD/ZAPD/ZRoom/Commands/SetSpecialObjects.cpp @@ -1,6 +1,8 @@ #include "SetSpecialObjects.h" + #include "Utils/BitConverter.h" #include "Utils/StringHelper.h" +#include "ZRoom/ZNames.h" SetSpecialObjects::SetSpecialObjects(ZFile* nParent) : ZRoomCommand(nParent) { @@ -15,8 +17,10 @@ void SetSpecialObjects::ParseRawData() std::string SetSpecialObjects::GetBodySourceCode() const { - return StringHelper::Sprintf("SCENE_CMD_SPECIAL_FILES(0x%02X, 0x%04X)", elfMessage, - globalObject); + std::string objectName = ZNames::GetObjectName(globalObject); + + return StringHelper::Sprintf("SCENE_CMD_SPECIAL_FILES(0x%02X, %s)", elfMessage, + objectName.c_str()); } std::string SetSpecialObjects::GetCommandCName() const diff --git a/tools/ZAPD/ZAPD/ZRoom/ZRoom.cpp b/tools/ZAPD/ZAPD/ZRoom/ZRoom.cpp index d47a2e75cb..5831eaa56b 100644 --- a/tools/ZAPD/ZAPD/ZRoom/ZRoom.cpp +++ b/tools/ZAPD/ZAPD/ZRoom/ZRoom.cpp @@ -261,7 +261,8 @@ void ZRoom::ParseRawData() if (Globals::Instance->profile) { auto end = std::chrono::steady_clock::now(); - int64_t diff = std::chrono::duration_cast(end - start).count(); + int64_t diff = + std::chrono::duration_cast(end - start).count(); if (diff > 50) printf("OP: %s, TIME: %" PRIi64 "ms\n", cmd->GetCommandCName().c_str(), diff); } diff --git a/tools/ZAPD/ZAPD/ZSkeleton.cpp b/tools/ZAPD/ZAPD/ZSkeleton.cpp index 1a2f93ff7b..4467c96320 100644 --- a/tools/ZAPD/ZAPD/ZSkeleton.cpp +++ b/tools/ZAPD/ZAPD/ZSkeleton.cpp @@ -139,7 +139,7 @@ ZResourceType ZSkeleton::GetResourceType() const DeclarationAlignment ZSkeleton::GetDeclarationAlignment() const { - return DeclarationAlignment::Align16; + return DeclarationAlignment::Align4; } uint8_t ZSkeleton::GetLimbCount() diff --git a/tools/ZAPD/ZAPD/ZVtx.cpp b/tools/ZAPD/ZAPD/ZVtx.cpp index d5214e840a..e4b3d97968 100644 --- a/tools/ZAPD/ZAPD/ZVtx.cpp +++ b/tools/ZAPD/ZAPD/ZVtx.cpp @@ -82,5 +82,5 @@ std::string ZVtx::GetExternalExtension() const DeclarationAlignment ZVtx::GetDeclarationAlignment() const { - return DeclarationAlignment::Align16; + return DeclarationAlignment::Align8; } diff --git a/tools/ZAPD/ZAPDUtils/Utils/StringHelper.h b/tools/ZAPD/ZAPDUtils/Utils/StringHelper.h index 0b0d676429..6c9e541190 100644 --- a/tools/ZAPD/ZAPDUtils/Utils/StringHelper.h +++ b/tools/ZAPD/ZAPDUtils/Utils/StringHelper.h @@ -109,6 +109,6 @@ public: static bool IEquals(const std::string& a, const std::string& b) { return std::equal(a.begin(), a.end(), b.begin(), b.end(), - [](char a, char b) { return tolower(a) == tolower(b); }); + [](char a, char b) { return tolower(a) == tolower(b); }); } }; diff --git a/tools/disasm/disasm.py b/tools/disasm/disasm.py index 5f8aae37d9..42473379e2 100755 --- a/tools/disasm/disasm.py +++ b/tools/disasm/disasm.py @@ -3,7 +3,7 @@ import argparse, ast, math, os, re, struct import bisect from mips_isa import * -from multiprocessing import Pool +from multiprocessing import * parser = argparse.ArgumentParser() parser.add_argument('-j', dest='jobs', type=int, default=1, help='number of processes to run at once') @@ -1170,7 +1170,7 @@ for segment in files_spec: # Construct variable_addrs, now that variable_addrs is fully constructed variable_addrs = sorted(variables_ast.keys()) -pool = Pool(jobs) +pool = get_context("fork").Pool(jobs) # Find symbols for each segment for segment in files_spec: if segment[2] == 'makerom': @@ -1385,7 +1385,7 @@ print("Disassembling Segments") disassemble_makerom(next(segment for segment in files_spec if segment[2] == 'makerom')) # Textual disassembly for each segment -with Pool(jobs) as p: +with get_context("fork").Pool(jobs) as p: p.map(disassemble_segment, (segment for segment in files_spec if segment[2] != 'makerom')) print("Splitting text and migrating rodata") diff --git a/tools/vtxdis.c b/tools/vtxdis.c index 1ba6969635..8debd89e0c 100644 --- a/tools/vtxdis.c +++ b/tools/vtxdis.c @@ -254,4 +254,4 @@ int main(int argc, char **argv) } return 0; -} \ No newline at end of file +}