From 3fc83fd0512226e4e28e4977d55d9371088e71a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9o=20Lam?= Date: Fri, 23 Jul 2021 00:35:03 +0200 Subject: [PATCH] Streamline project setup by automating NSO conversion (if needed) --- .gitignore | 3 +- .gitmodules | 3 + Contributing.md | 6 -- README.md | 21 ++++-- tools/nx-decomp-tools-binaries | 1 + tools/setup.py | 117 +++++++++++++++++++++++++++++++-- 6 files changed, 135 insertions(+), 16 deletions(-) create mode 160000 tools/nx-decomp-tools-binaries diff --git a/.gitignore b/.gitignore index a6609dec..ec90d25f 100644 --- a/.gitignore +++ b/.gitignore @@ -21,7 +21,8 @@ bin/ *.nam *.til -main.elf +data/*.elf +data/*.nso perf.mData perf.mData.old diff --git a/.gitmodules b/.gitmodules index 67907369..0cbb85c4 100644 --- a/.gitmodules +++ b/.gitmodules @@ -16,3 +16,6 @@ [submodule "toolchain/musl"] path = toolchain/musl url = https://github.com/open-ead/botw-lib-musl +[submodule "tools/nx-decomp-tools-binaries"] + path = tools/nx-decomp-tools-binaries + url = https://github.com/open-ead/nx-decomp-tools-binaries diff --git a/Contributing.md b/Contributing.md index 58f3b901..fa689135 100644 --- a/Contributing.md +++ b/Contributing.md @@ -5,12 +5,6 @@ To contribute to the project, you will need: * A disassembler or a decompiler such as Hex-Rays or Ghidra. * Python 3 and pip for the diff script * These Python modules: `capstone colorama cxxfilt pyelftools` (install them with `pip install ...`) -* The original 1.5.0 `main` NSO executable, converted to ELF format with [nx2elf](https://github.com/shuffle2/nx2elf). - * To dump it, follow [the instructions on the wiki](https://zeldamods.org/wiki/Help:Dumping_games#Dumping_binaries_.28executable_files.29). If you only have 1.6.0 on your console, you should still dump it. - * Decompress the `main` NSO with [hactool](https://github.com/SciresM/hactool). - * If you have a 1.6.0 dump, use xdelta3 on the decompressed NSO to turn it into a 1.5.0 NSO. [The patch is available here.](https://s.botw.link/v150_downgrade/v160_to_v150.patch) - * The uncompressed NSO has the following SHA256 hash: `d9fa308d0ee7c0ab081c66d987523385e1afe06f66731bbfa32628438521c106` - * Copy it to data/main.elf -- it is used for the diff script and other tools. Experience with reverse engineering optimized C++ code is very useful but not necessary if you already know how to decompile C code. diff --git a/README.md b/README.md index 715bebb7..1d24937d 100644 --- a/README.md +++ b/README.md @@ -148,12 +148,25 @@ Ubuntu users can install those dependencies by running: sudo apt install python3 ninja-build cmake ccache ``` -### 2. Set up the repository +### 2. Set up the project 1. Clone this repository. If you are using WSL, please clone the repo *inside* WSL, *not* on the Windows side (for performance reasons). + 2. Run `git submodule update --init --recursive` -3. Run `tools/setup.py` - * This will set up [Clang 4.0.1](https://releases.llvm.org/download.html#4.0.1) and create a build directory in `build/`. + + Next, you'll need to acquire the **original 1.5.0 or 1.6.0 `main` NSO executable**. + + * To dump it from a Switch, follow [the instructions on the wiki](https://zeldamods.org/wiki/Help:Dumping_games#Dumping_binaries_.28executable_files.29). + * You do not need to dump the entire game (RomFS + ExeFS + DLC). Just dumping the 1.5.0 or 1.6.0 ExeFS is sufficient. + * The decompressed 1.5.0 NSO has the following SHA256 hash: `d9fa308d0ee7c0ab081c66d987523385e1afe06f66731bbfa32628438521c106` + * If you have a compressed NSO or a 1.6.0 executable, don't worry about this. + +3. Run `tools/setup.py [path to the NSO]` + * This will: + * convert the executable if necessary + * set up [Clang 4.0.1](https://releases.llvm.org/download.html#4.0.1) by downloading it from the official LLVM website + * create a build directory in `build/` + * If something goes wrong, follow the instructions given to you by the script. ### 3. Build @@ -169,7 +182,7 @@ To check whether everything built correctly, just run `tools/check.py` after the ## Contributing -Follow the [contributing guidelines here](Contributing.md). +Follow the [contributing guidelines here](Contributing.md). Feel free to join the [Zelda Decompilation](https://discord.zelda64.dev/) Discord server if you have any questions. ## Resources diff --git a/tools/nx-decomp-tools-binaries b/tools/nx-decomp-tools-binaries new file mode 160000 index 00000000..011c369b --- /dev/null +++ b/tools/nx-decomp-tools-binaries @@ -0,0 +1 @@ +Subproject commit 011c369bc4bead403d650dd1d1eeb69c04eb19c7 diff --git a/tools/setup.py b/tools/setup.py index 773f591a..227fd84b 100755 --- a/tools/setup.py +++ b/tools/setup.py @@ -1,5 +1,7 @@ #!/usr/bin/env python3 +import argparse +import hashlib import platform from pathlib import Path import subprocess @@ -12,14 +14,112 @@ ROOT = Path(__file__).parent.parent def fail(error: str): - print(error) + print(">>> " + error) sys.exit(1) +def _get_tool_binary_path(): + base = ROOT / "tools" / "nx-decomp-tools-binaries" + system = platform.system() + if system == "Linux": + return str(base / "linux") + "/" + if system == "Darwin": + return str(base / "macos") + "/" + return "" + + +def _convert_nso_to_elf(nso_path: Path): + print(">>>> converting NSO to ELF...") + binpath = _get_tool_binary_path() + subprocess.check_call([binpath + "nx2elf", str(nso_path)]) + + +def _decompress_nso(nso_path: Path, dest_path: Path): + print(">>>> decompressing NSO...") + binpath = _get_tool_binary_path() + subprocess.check_call([binpath + "hactool", "-tnso", + "--uncompressed=" + str(dest_path), str(nso_path)]) + + +def _download_v160_to_v150_patch(dest: Path): + print(">>>> downloading patch...") + urllib.request.urlretrieve("https://s.botw.link/v150_downgrade/v160_to_v150.patch", dest) + + +def _apply_xdelta3_patch(input: Path, patch: Path, dest: Path): + print(">>>> applying patch...") + try: + subprocess.check_call(["xdelta3", "-d", "-s", str(input), str(patch), str(dest)]) + except FileNotFoundError: + fail("error: install xdelta3 and try again") + + +def prepare_executable(original_nso: Path): + COMPRESSED_V150_HASH = "898dc199301f7c419be5144bb5cb27e2fc346e22b27345ba3fb40c0060c2baf8" + UNCOMPRESSED_V150_HASH = "d9fa308d0ee7c0ab081c66d987523385e1afe06f66731bbfa32628438521c106" + COMPRESSED_V160_HASH = "15cfca7b89348956f85d945fade2e215a6af5991ed1071e181f97ca72f7ae20b" + UNCOMPRESSED_V160_HASH = "8a2fc8b1111a35a76fd2d53a8670599da4a7a9706a3d91215d30fd62149f00c1" + + # The uncompressed v1.5.0 main NSO. + TARGET_HASH = UNCOMPRESSED_V150_HASH + TARGET_PATH = ROOT / "data" / "main.nso" + TARGET_ELF_PATH = ROOT / "data" / "main.elf" + + if TARGET_PATH.is_file() and hashlib.sha256(TARGET_PATH.read_bytes()).hexdigest() == TARGET_HASH and TARGET_ELF_PATH.is_file(): + print(">>> NSO is already set up") + return + + if not original_nso.is_file(): + fail(f"{original_nso} is not a file") + + nso_data = original_nso.read_bytes() + nso_hash = hashlib.sha256(nso_data).hexdigest() + + if nso_hash == UNCOMPRESSED_V150_HASH: + print(">>> found uncompressed 1.5.0 NSO") + TARGET_PATH.write_bytes(nso_data) + + elif nso_hash == COMPRESSED_V150_HASH: + print(">>> found compressed 1.5.0 NSO") + _decompress_nso(original_nso, TARGET_PATH) + + elif nso_hash == UNCOMPRESSED_V160_HASH: + print(">>> found uncompressed 1.6.0 NSO") + + with tempfile.TemporaryDirectory() as tmpdir: + patch_path = Path(tmpdir) / "patch" + _download_v160_to_v150_patch(patch_path) + _apply_xdelta3_patch(original_nso, patch_path, TARGET_PATH) + + elif nso_hash == COMPRESSED_V160_HASH: + print(">>> found compressed 1.6.0 NSO") + + with tempfile.TemporaryDirectory() as tmpdir: + patch_path = Path(tmpdir) / "patch" + decompressed_nso_path = Path(tmpdir) / "v160.nso" + + _decompress_nso(original_nso, decompressed_nso_path) + _download_v160_to_v150_patch(patch_path) + _apply_xdelta3_patch(decompressed_nso_path, patch_path, TARGET_PATH) + + else: + fail(f"unknown executable: {nso_hash}") + + if not TARGET_PATH.is_file(): + fail("internal error while preparing executable (missing NSO); please report") + if hashlib.sha256(TARGET_PATH.read_bytes()).hexdigest() != TARGET_HASH: + fail("internal error while preparing executable (wrong NSO hash); please report") + + _convert_nso_to_elf(TARGET_PATH) + + if not TARGET_ELF_PATH.is_file(): + fail("internal error while preparing executable (missing ELF); please report") + + def set_up_compiler(): compiler_dir = ROOT / "toolchain" / "clang" if compiler_dir.is_dir(): - print("clang is already set up: nothing to do") + print(">>> clang is already set up: nothing to do") return system = platform.system() @@ -55,12 +155,12 @@ def set_up_compiler(): url: str = build_info["url"] dir_name: str = build_info["dir_name"] - print(f"downloading Clang from {url}...") + print(f">>> downloading Clang from {url}...") with tempfile.TemporaryDirectory() as tmpdir: path = tmpdir + "/" + url.split("/")[-1] urllib.request.urlretrieve(url, path) - print(f"extracting Clang...") + print(f">>> extracting Clang...") with tarfile.open(path) as f: f.extractall(compiler_dir.parent) (compiler_dir.parent / dir_name).rename(compiler_dir) @@ -71,7 +171,7 @@ def set_up_compiler(): def create_build_dir(): build_dir = ROOT / "build" if build_dir.is_dir(): - print("build directory already exists: nothing to do") + print(">>> build directory already exists: nothing to do") return subprocess.check_call( @@ -80,6 +180,13 @@ def create_build_dir(): def main(): + parser = argparse.ArgumentParser( + "setup.py", description="Set up the Breath of the Wild decompilation project") + parser.add_argument("original_nso", type=Path, + help="Path to the original NSO (1.5.0 or 1.6.0, compressed or not)") + args = parser.parse_args() + + prepare_executable(args.original_nso) set_up_compiler() create_build_dir()