diff --git a/.gitignore b/.gitignore index 343c32e3..a6609dec 100644 --- a/.gitignore +++ b/.gitignore @@ -28,4 +28,5 @@ perf.mData.old .gdb_history .DS_Store -tools/aarch64-none-elf-objdump \ No newline at end of file +tools/aarch64-none-elf-objdump +toolchain/clang/ diff --git a/.gitmodules b/.gitmodules index 08dc5599..67907369 100644 --- a/.gitmodules +++ b/.gitmodules @@ -13,3 +13,6 @@ [submodule "lib/EventFlow"] path = lib/EventFlow url = https://github.com/open-ead/EventFlow +[submodule "toolchain/musl"] + path = toolchain/musl + url = https://github.com/open-ead/botw-lib-musl diff --git a/CMakeLists.txt b/CMakeLists.txt index a3fdf1bd..35455146 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,7 +2,8 @@ cmake_minimum_required(VERSION 3.13) project(uking CXX) set(CMAKE_EXPORT_COMPILE_COMMANDS ON) -set(CMAKE_NINJA_FORCE_RESPONSE_FILE ON) +# Use response files when linking objects because of the large number of source files +set(CMAKE_CXX_USE_RESPONSE_FILE_FOR_OBJECTS ON) if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") add_compile_options(-fdiagnostics-color=always) elseif ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 038229a7..00000000 --- a/Dockerfile +++ /dev/null @@ -1,37 +0,0 @@ -FROM ubuntu:latest - -MAINTAINER Léo - -# devkitpro/devkita64:latest - -ENV DEBIAN_FRONTEND=noninteractive - -RUN apt-get update && \ - apt-get install -y --no-install-recommends apt-utils && \ - apt-get install -y --no-install-recommends sudo ca-certificates pkg-config curl wget xz-utils make git gnupg && \ - apt-get install -y --no-install-recommends gdebi-core && \ - apt-get install -y --no-install-recommends cmake && \ - apt-get install -y --no-install-recommends libncurses5 ninja-build && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - -RUN wget https://github.com/devkitPro/pacman/releases/latest/download/devkitpro-pacman.amd64.deb && \ - gdebi -n devkitpro-pacman.amd64.deb && \ - rm devkitpro-pacman.amd64.deb && \ - dkp-pacman -Scc --noconfirm - -RUN dkp-pacman -Syyu --noconfirm switch-dev && \ - dkp-pacman -Scc --noconfirm - -ENV DEVKITPRO=/opt/devkitpro -ENV DEVKITA64=/opt/devkitpro/devkitA64 -ENV PATH=${DEVKITPRO}/tools/bin:$DEVKITA64/bin:$PATH - -# BotW build - -RUN wget https://releases.llvm.org/4.0.1/clang+llvm-4.0.1-x86_64-linux-gnu-debian8.tar.xz && \ - tar xf clang+llvm-4.0.1-x86_64-linux-gnu-debian8.tar.xz && \ - rm clang+llvm-4.0.1-x86_64-linux-gnu-debian8.tar.xz && \ - mv clang+llvm-4.0.1-x86_64-linux-gnu-debian8 /opt/clang-4.0.1 - -ENV UKING_CLANG=/opt/clang-4.0.1 diff --git a/README.md b/README.md index a624af9a..0b5ccbfd 100644 --- a/README.md +++ b/README.md @@ -122,36 +122,55 @@ Just like other game decompilations, this project is probably in a legal gray zo ### Alright, how do I start contributing? -First, set up the build environment by following the instructions below. Then follow the [contributing guidelines here](Contributing.md). +First, set up the build environment by following the instructions below. ## Building -### Dependencies - -* Clang 4.0.1 - * Download [Clang 4.0.1](https://releases.llvm.org/download.html#4.0.1) and extract the archive to a convenient location of your choice. - * You might also need libtinfo5. For Ubuntu or Debian users, install it with `sudo apt install libtinfo5` -* Ninja - * For Ubuntu or Debian users, install it with `sudo apt install ninja-build` -* CMake 3.13+ - * If you are on Ubuntu 18.04, you must [update CMake by using the official CMake APT repository](https://apt.kitware.com/). -* devkitA64 - * [Follow this guide to set it up.](https://switchbrew.org/wiki/Setting_up_Development_Environment#Setup) - -Using Linux (or WSL) is recommended but not required. The rest of this guide assumes that you are using a Linux environment, though. - -### Building for Switch - -1. After cloning this repository, run: `git submodule update --init --recursive` -2. Run: `env UKING_CLANG=$1 DEVKITA64=$2 cmake -GNinja -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_TOOLCHAIN_FILE=../ToolchainNX64.cmake -B build` - * Replace `$1` with the path to the extracted Clang archive, such that `$1/bin/clang` exists. This should be an absolute path; use `/home/` instead of `~`. - * Replace `$2` with the path to devkitA64. On Linux, this is typically `/opt/devkitpro/devkitA64`. -3. Start the build by running: `ninja -C build` - -On subsequent builds, just run `ninja -C build` from the project root. - Reminder: **this will not produce a playable game.** This project will not allow you to play the game if you don't already own it on a Switch. +### For Windows users + +While Linux is not a hard requirement, it is strongly advised to [set up WSL](https://docs.microsoft.com/en-us/windows/wsl/install-win10) to simplify the setup process. Ubuntu 20.04 is usually a good choice. + +The instructions below assume that you are using Linux (native or WSL) or macOS. + +### 1. Set up dependencies + +* Python 3.6 or newer +* Ninja +* CMake 3.13+ + * If you are on Ubuntu 18.04, you must first [update CMake by using the official CMake APT repository](https://apt.kitware.com/). +* [Optional] ccache (to speed up builds) + +Ubuntu users can install those dependencies by running: + +```shell +sudo apt install python3 ninja-build cmake ccache +``` + +### 2. Set up the repository + +1. Clone this repository. +2. Run `git submodule update --init --recursive` +3. Run `tools/setup.py` + * This will set up [Clang 4.0.1](https://releases.llvm.org/download.html#4.0.1) and create a build directory in `build/`. + +### 3. Build + +To start the build, just run + +```shell +ninja -C build +``` + +By default, Ninja will perform a multithreaded build. There is no need to pass -j manually. + +To check whether everything built correctly, just run `tools/check.py` after the build completes. + +## Contributing + +Follow the [contributing guidelines here](Contributing.md). + ## Resources #### Breath of the Wild documentation and datamining diff --git a/ToolchainNX64.cmake b/ToolchainNX64.cmake deleted file mode 100644 index d9af3beb..00000000 --- a/ToolchainNX64.cmake +++ /dev/null @@ -1,45 +0,0 @@ -if (NOT DEFINED ENV{UKING_CLANG}) - message(FATAL_ERROR "Please define the UKING_CLANG env variable. It should point to a path such that $UKING_CLANG/bin/clang exists") -endif() - -if (NOT DEFINED ENV{DEVKITA64}) - message(FATAL_ERROR "Please define the DEVKITA64 env variable.") -endif() - -set(UKING_CLANG "$ENV{UKING_CLANG}") -set(DEVKITA64 "$ENV{DEVKITA64}") -set(NX64_OPT_FLAGS "-O3 -g") -set(triple aarch64-none-elf) - -set(CMAKE_SYSTEM_NAME Generic) -set(CMAKE_SYSTEM_VERSION 1) -set(CMAKE_SYSTEM_PROCESSOR aarch64) - -set(CMAKE_SYSROOT ${UKING_CLANG}) -set(CMAKE_C_COMPILER "${UKING_CLANG}/bin/clang") -set(CMAKE_C_COMPILER_TARGET ${triple}) -set(CMAKE_CXX_COMPILER "${UKING_CLANG}/bin/clang++") -set(CMAKE_CXX_COMPILER_TARGET ${triple}) - -set(CMAKE_C_FLAGS_RELEASE ${NX64_OPT_FLAGS}) -set(CMAKE_CXX_FLAGS_RELEASE ${NX64_OPT_FLAGS}) -set(CMAKE_C_FLAGS_RELWITHDEBINFO ${NX64_OPT_FLAGS}) -set(CMAKE_CXX_FLAGS_RELWITHDEBINFO ${NX64_OPT_FLAGS}) - -set(ARCH "-mcpu=cortex-a57+fp+simd+crypto+crc") -set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${ARCH} -isystem ${DEVKITA64}/aarch64-none-elf/include") -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -isystem ${UKING_CLANG}/include/c++/v1 -D _LIBCPP_HAS_THREAD_API_PTHREAD ${CMAKE_C_FLAGS}") -set(CMAKE_ASM_FLAGS "${CMAKE_ASM_FLAGS} -x assembler-with-cpp ${ARCH}") - -add_compile_options(-fPIC -stdlib=libc++ -mno-implicit-float) -add_link_options(-B ${DEVKITA64}/bin -fPIC -Wl,-Bsymbolic-functions -shared -nodefaultlibs) -if(EXISTS "${DEVKITA64}/bin/ld.lld") - add_link_options(-fuse-ld=lld -Wl,-z,notext) -endif() -add_definitions(-D SWITCH -D __DEVKITA64__ -D __ELF__) -add_definitions(-D NNSDK) -add_definitions(-D MATCHING_HACK_NX_CLANG) - -# Helps with matching as this causes Clang to emit debug type info even for dynamic classes -# with undefined vtables. -add_compile_options(-fstandalone-debug) diff --git a/toolchain/ToolchainNX64.cmake b/toolchain/ToolchainNX64.cmake new file mode 100644 index 00000000..bc967750 --- /dev/null +++ b/toolchain/ToolchainNX64.cmake @@ -0,0 +1,42 @@ +if (DEFINED ENV{UKING_CLANG}) + set(UKING_CLANG "$ENV{UKING_CLANG}") +else() + set(UKING_CLANG "toolchain/clang") +endif() + +set(NX64_OPT_FLAGS "-O3 -g") +set(NX64_TRIPLE aarch64-linux-elf) + +set(CMAKE_SYSTEM_NAME Generic) +set(CMAKE_SYSTEM_VERSION 1) +set(CMAKE_SYSTEM_PROCESSOR aarch64) + +set(CMAKE_SYSROOT ${CMAKE_CURRENT_LIST_DIR}/musl) +set(CMAKE_C_COMPILER "${UKING_CLANG}/bin/clang") +set(CMAKE_C_COMPILER_TARGET ${NX64_TRIPLE}) +set(CMAKE_CXX_COMPILER "${UKING_CLANG}/bin/clang++") +set(CMAKE_CXX_COMPILER_TARGET ${NX64_TRIPLE}) + +set(CMAKE_C_FLAGS_RELEASE ${NX64_OPT_FLAGS}) +set(CMAKE_CXX_FLAGS_RELEASE ${NX64_OPT_FLAGS}) +set(CMAKE_C_FLAGS_RELWITHDEBINFO ${NX64_OPT_FLAGS}) +set(CMAKE_CXX_FLAGS_RELWITHDEBINFO ${NX64_OPT_FLAGS}) + +# Target options +add_compile_options(-mcpu=cortex-a57+fp+simd+crypto+crc) +add_compile_options(-mno-implicit-float) +# Environment +add_compile_options(-stdlib=libc++) +add_compile_options(-fPIC) +# Helps with matching as this causes Clang to emit debug type info even for dynamic classes +# with undefined vtables. +add_compile_options(-fstandalone-debug) + +add_definitions(-D SWITCH) +add_definitions(-D NNSDK) +add_definitions(-D MATCHING_HACK_NX_CLANG) + +add_link_options(-stdlib=libc++ -nostdlib) +add_link_options(-fPIC -Wl,-Bsymbolic-functions -shared) +# Use lld for performance reasons (and because we don't want a dependency on GNU tools) +add_link_options(-fuse-ld=lld) diff --git a/toolchain/musl b/toolchain/musl new file mode 160000 index 00000000..9a9252a5 --- /dev/null +++ b/toolchain/musl @@ -0,0 +1 @@ +Subproject commit 9a9252a54a67f54c066966f8f0599c450391bd44 diff --git a/tools/setup.py b/tools/setup.py new file mode 100755 index 00000000..b495e89a --- /dev/null +++ b/tools/setup.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 + +import os +import platform +from pathlib import Path +import sys +import tarfile +import tempfile +import urllib.request + +ROOT = Path(__file__).parent.parent + + +def fail(error: str): + print(error) + sys.exit(1) + + +def set_up_compiler(): + compiler_dir = ROOT / "toolchain" / "clang" + if compiler_dir.is_dir(): + print("clang is already set up: nothing to do") + return + + system = platform.system() + machine = platform.machine() + + builds = { + # Linux + ("Linux", "x86_64"): { + "url": "https://releases.llvm.org/4.0.1/clang+llvm-4.0.1-x86_64-linux-gnu-Fedora-25.tar.xz", + "dir_name": "clang+llvm-4.0.1-x86_64-linux-gnu-Fedora-25", + }, + ("Linux", "aarch64"): { + "url": "https://releases.llvm.org/4.0.1/clang+llvm-4.0.1-aarch64-linux-gnu.tar.xz", + "dir_name": "clang+llvm-4.0.1-aarch64-linux-gnu", + }, + + # macOS + ("Darwin", "x86_64"): { + "url": "https://releases.llvm.org/4.0.1/clang+llvm-4.0.1-x86_64-apple-darwin.tar.xz", + "dir_name": "clang+llvm-4.0.1-x86_64-apple-darwin", + }, + ("Darwin", "aarch64"): { + "url": "https://releases.llvm.org/4.0.1/clang+llvm-4.0.1-x86_64-apple-darwin.tar.xz", + "dir_name": "clang+llvm-4.0.1-x86_64-apple-darwin", + }, + } + + build_info = builds.get((system, machine)) + if build_info is None: + fail( + f"unknown platform: {platform.platform()} (please report if you are on Linux and macOS)") + + url: str = build_info["url"] + dir_name: str = build_info["dir_name"] + + print(f"downloading Clang from {url}...") + with tempfile.TemporaryDirectory() as tmpdir: + path = tmpdir + "/" + url.split("/")[-1] + urllib.request.urlretrieve(url, path) + + print(f"extracting Clang...") + with tarfile.open(path) as f: + f.extractall(compiler_dir.parent) + (compiler_dir.parent / dir_name).rename(compiler_dir) + + print(">>> successfully set up Clang") + + +def create_build_dir(): + build_dir = ROOT / "build" + if build_dir.is_dir(): + print("build directory already exists: nothing to do") + return + + os.system("cmake -GNinja -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_TOOLCHAIN_FILE=toolchain/ToolchainNX64.cmake -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -B build/") + print(">>> created build directory") + + +def main(): + set_up_compiler() + create_build_dir() + + +if __name__ == "__main__": + main()