Streamline project setup by automating NSO conversion (if needed)

.gitignore

@@ -21,7 +21,8 @@ bin/
 *.nam
 *.til
 
-main.elf
+data/*.elf
+data/*.nso
 
 perf.mData
 perf.mData.old

.gitmodules

@@ -16,3 +16,6 @@
 [submodule "toolchain/musl"]
 	path = toolchain/musl
 	url = https://github.com/open-ead/botw-lib-musl
+[submodule "tools/nx-decomp-tools-binaries"]
+	path = tools/nx-decomp-tools-binaries
+	url = https://github.com/open-ead/nx-decomp-tools-binaries

Contributing.md

@@ -5,12 +5,6 @@ To contribute to the project, you will need:
 * A disassembler or a decompiler such as Hex-Rays or Ghidra.
 * Python 3 and pip for the diff script
 * These Python modules: `capstone colorama cxxfilt pyelftools` (install them with `pip install ...`)
-* The original 1.5.0 `main` NSO executable, converted to ELF format with [nx2elf](https://github.com/shuffle2/nx2elf).
-    * To dump it, follow [the instructions on the wiki](https://zeldamods.org/wiki/Help:Dumping_games#Dumping_binaries_.28executable_files.29). If you only have 1.6.0 on your console, you should still dump it.
-    * Decompress the `main` NSO with [hactool](https://github.com/SciresM/hactool).
-    * If you have a 1.6.0 dump, use xdelta3 on the decompressed NSO to turn it into a 1.5.0 NSO. [The patch is available here.](https://s.botw.link/v150_downgrade/v160_to_v150.patch)
-    * The uncompressed NSO has the following SHA256 hash: `d9fa308d0ee7c0ab081c66d987523385e1afe06f66731bbfa32628438521c106`
-    * Copy it to data/main.elf -- it is used for the diff script and other tools.
 
 Experience with reverse engineering optimized C++ code is very useful but not necessary if you already know how to decompile C code.
 

README.md

@@ -148,12 +148,25 @@ Ubuntu users can install those dependencies by running:
 sudo apt install python3 ninja-build cmake ccache
 ```
 
-### 2. Set up the repository
+### 2. Set up the project
 
 1. Clone this repository. If you are using WSL, please clone the repo *inside* WSL, *not* on the Windows side (for performance reasons).
+
 2. Run `git submodule update --init --recursive`
-3. Run `tools/setup.py`
+
-    * This will set up [Clang 4.0.1](https://releases.llvm.org/download.html#4.0.1) and create a build directory in `build/`.
+    Next, you'll need to acquire the **original 1.5.0 or 1.6.0 `main` NSO executable**.
+
+    * To dump it from a Switch, follow [the instructions on the wiki](https://zeldamods.org/wiki/Help:Dumping_games#Dumping_binaries_.28executable_files.29).
+    * You do not need to dump the entire game (RomFS + ExeFS + DLC). Just dumping the 1.5.0 or 1.6.0 ExeFS is sufficient.
+    * The decompressed 1.5.0 NSO has the following SHA256 hash: `d9fa308d0ee7c0ab081c66d987523385e1afe06f66731bbfa32628438521c106`
+        * If you have a compressed NSO or a 1.6.0 executable, don't worry about this.
+
+3. Run `tools/setup.py [path to the NSO]`
+    * This will:
+        * convert the executable if necessary
+        * set up [Clang 4.0.1](https://releases.llvm.org/download.html#4.0.1) by downloading it from the official LLVM website
+        * create a build directory in `build/`
+    * If something goes wrong, follow the instructions given to you by the script.
 
 ### 3. Build
 
@@ -169,7 +182,7 @@ To check whether everything built correctly, just run `tools/check.py` after the
 
 ## Contributing
 
-Follow the [contributing guidelines here](Contributing.md).
+Follow the [contributing guidelines here](Contributing.md). Feel free to join the [Zelda Decompilation](https://discord.zelda64.dev/) Discord server if you have any questions.
 
 ## Resources
 

tools/nx-decomp-tools-binaries

@@ -0,0 +1 @@
+Subproject commit 011c369bc4bead403d650dd1d1eeb69c04eb19c7

tools/setup.py

@@ -1,5 +1,7 @@
 #!/usr/bin/env python3
 
+import argparse
+import hashlib
 import platform
 from pathlib import Path
 import subprocess
@@ -12,14 +14,112 @@ ROOT = Path(__file__).parent.parent
 
 
 def fail(error: str):
-    print(error)
+    print(">>> " + error)
     sys.exit(1)
 
 
+def _get_tool_binary_path():
+    base = ROOT / "tools" / "nx-decomp-tools-binaries"
+    system = platform.system()
+    if system == "Linux":
+        return str(base / "linux") + "/"
+    if system == "Darwin":
+        return str(base / "macos") + "/"
+    return ""
+
+
+def _convert_nso_to_elf(nso_path: Path):
+    print(">>>> converting NSO to ELF...")
+    binpath = _get_tool_binary_path()
+    subprocess.check_call([binpath + "nx2elf", str(nso_path)])
+
+
+def _decompress_nso(nso_path: Path, dest_path: Path):
+    print(">>>> decompressing NSO...")
+    binpath = _get_tool_binary_path()
+    subprocess.check_call([binpath + "hactool", "-tnso",
+                           "--uncompressed=" + str(dest_path), str(nso_path)])
+
+
+def _download_v160_to_v150_patch(dest: Path):
+    print(">>>> downloading patch...")
+    urllib.request.urlretrieve("https://s.botw.link/v150_downgrade/v160_to_v150.patch", dest)
+
+
+def _apply_xdelta3_patch(input: Path, patch: Path, dest: Path):
+    print(">>>> applying patch...")
+    try:
+        subprocess.check_call(["xdelta3", "-d", "-s", str(input), str(patch), str(dest)])
+    except FileNotFoundError:
+        fail("error: install xdelta3 and try again")
+
+
+def prepare_executable(original_nso: Path):
+    COMPRESSED_V150_HASH = "898dc199301f7c419be5144bb5cb27e2fc346e22b27345ba3fb40c0060c2baf8"
+    UNCOMPRESSED_V150_HASH = "d9fa308d0ee7c0ab081c66d987523385e1afe06f66731bbfa32628438521c106"
+    COMPRESSED_V160_HASH = "15cfca7b89348956f85d945fade2e215a6af5991ed1071e181f97ca72f7ae20b"
+    UNCOMPRESSED_V160_HASH = "8a2fc8b1111a35a76fd2d53a8670599da4a7a9706a3d91215d30fd62149f00c1"
+
+    # The uncompressed v1.5.0 main NSO.
+    TARGET_HASH = UNCOMPRESSED_V150_HASH
+    TARGET_PATH = ROOT / "data" / "main.nso"
+    TARGET_ELF_PATH = ROOT / "data" / "main.elf"
+
+    if TARGET_PATH.is_file() and hashlib.sha256(TARGET_PATH.read_bytes()).hexdigest() == TARGET_HASH and TARGET_ELF_PATH.is_file():
+        print(">>> NSO is already set up")
+        return
+
+    if not original_nso.is_file():
+        fail(f"{original_nso} is not a file")
+
+    nso_data = original_nso.read_bytes()
+    nso_hash = hashlib.sha256(nso_data).hexdigest()
+
+    if nso_hash == UNCOMPRESSED_V150_HASH:
+        print(">>> found uncompressed 1.5.0 NSO")
+        TARGET_PATH.write_bytes(nso_data)
+
+    elif nso_hash == COMPRESSED_V150_HASH:
+        print(">>> found compressed 1.5.0 NSO")
+        _decompress_nso(original_nso, TARGET_PATH)
+
+    elif nso_hash == UNCOMPRESSED_V160_HASH:
+        print(">>> found uncompressed 1.6.0 NSO")
+
+        with tempfile.TemporaryDirectory() as tmpdir:
+            patch_path = Path(tmpdir) / "patch"
+            _download_v160_to_v150_patch(patch_path)
+            _apply_xdelta3_patch(original_nso, patch_path, TARGET_PATH)
+
+    elif nso_hash == COMPRESSED_V160_HASH:
+        print(">>> found compressed 1.6.0 NSO")
+
+        with tempfile.TemporaryDirectory() as tmpdir:
+            patch_path = Path(tmpdir) / "patch"
+            decompressed_nso_path = Path(tmpdir) / "v160.nso"
+
+            _decompress_nso(original_nso, decompressed_nso_path)
+            _download_v160_to_v150_patch(patch_path)
+            _apply_xdelta3_patch(decompressed_nso_path, patch_path, TARGET_PATH)
+
+    else:
+        fail(f"unknown executable: {nso_hash}")
+
+    if not TARGET_PATH.is_file():
+        fail("internal error while preparing executable (missing NSO); please report")
+    if hashlib.sha256(TARGET_PATH.read_bytes()).hexdigest() != TARGET_HASH:
+        fail("internal error while preparing executable (wrong NSO hash); please report")
+
+    _convert_nso_to_elf(TARGET_PATH)
+
+    if not TARGET_ELF_PATH.is_file():
+        fail("internal error while preparing executable (missing ELF); please report")
+
+
 def set_up_compiler():
     compiler_dir = ROOT / "toolchain" / "clang"
     if compiler_dir.is_dir():
-        print("clang is already set up: nothing to do")
+        print(">>> clang is already set up: nothing to do")
         return
 
     system = platform.system()
@@ -55,12 +155,12 @@ def set_up_compiler():
     url: str = build_info["url"]
     dir_name: str = build_info["dir_name"]
 
-    print(f"downloading Clang from {url}...")
+    print(f">>> downloading Clang from {url}...")
     with tempfile.TemporaryDirectory() as tmpdir:
         path = tmpdir + "/" + url.split("/")[-1]
         urllib.request.urlretrieve(url, path)
 
-        print(f"extracting Clang...")
+        print(f">>> extracting Clang...")
         with tarfile.open(path) as f:
             f.extractall(compiler_dir.parent)
             (compiler_dir.parent / dir_name).rename(compiler_dir)
@@ -71,7 +171,7 @@ def set_up_compiler():
 def create_build_dir():
     build_dir = ROOT / "build"
     if build_dir.is_dir():
-        print("build directory already exists: nothing to do")
+        print(">>> build directory already exists: nothing to do")
         return
 
     subprocess.check_call(
@@ -80,6 +180,13 @@ def create_build_dir():
 
 
 def main():
+    parser = argparse.ArgumentParser(
+        "setup.py", description="Set up the Breath of the Wild decompilation project")
+    parser.add_argument("original_nso", type=Path,
+                        help="Path to the original NSO (1.5.0 or 1.6.0, compressed or not)")
+    args = parser.parse_args()
+
+    prepare_executable(args.original_nso)
     set_up_compiler()
     create_build_dir()