Merge pull request #46 from leoetlino/build-cleanup

Clean up build toolchain (and use musl for libc) and simplify setup

.gitignore

@@ -29,3 +29,4 @@ perf.mData.old
 
 .DS_Store
 tools/aarch64-none-elf-objdump
+toolchain/clang/

.gitmodules

@@ -13,3 +13,6 @@
 [submodule "lib/EventFlow"]
 	path = lib/EventFlow
 	url = https://github.com/open-ead/EventFlow
+[submodule "toolchain/musl"]
+	path = toolchain/musl
+	url = https://github.com/open-ead/botw-lib-musl

CMakeLists.txt

@@ -2,7 +2,8 @@ cmake_minimum_required(VERSION 3.13)
 project(uking CXX)
 
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
-set(CMAKE_NINJA_FORCE_RESPONSE_FILE ON)
+# Use response files when linking objects because of the large number of source files
+set(CMAKE_CXX_USE_RESPONSE_FILE_FOR_OBJECTS ON)
 if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU")
   add_compile_options(-fdiagnostics-color=always)
 elseif ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")

Dockerfile

@@ -1,37 +0,0 @@
-FROM ubuntu:latest
-
-MAINTAINER Léo <leo@leolam.fr>
-
-# devkitpro/devkita64:latest
-
-ENV DEBIAN_FRONTEND=noninteractive
-
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends apt-utils && \
-    apt-get install -y --no-install-recommends sudo ca-certificates pkg-config curl wget xz-utils make git gnupg && \
-    apt-get install -y --no-install-recommends gdebi-core && \
-    apt-get install -y --no-install-recommends cmake && \
-    apt-get install -y --no-install-recommends libncurses5 ninja-build && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/*
-
-RUN wget https://github.com/devkitPro/pacman/releases/latest/download/devkitpro-pacman.amd64.deb && \
-    gdebi -n devkitpro-pacman.amd64.deb && \
-    rm devkitpro-pacman.amd64.deb && \
-    dkp-pacman -Scc --noconfirm
-
-RUN dkp-pacman -Syyu --noconfirm switch-dev && \
-    dkp-pacman -Scc --noconfirm
-
-ENV DEVKITPRO=/opt/devkitpro
-ENV DEVKITA64=/opt/devkitpro/devkitA64
-ENV PATH=${DEVKITPRO}/tools/bin:$DEVKITA64/bin:$PATH
-
-# BotW build
-
-RUN wget https://releases.llvm.org/4.0.1/clang+llvm-4.0.1-x86_64-linux-gnu-debian8.tar.xz && \
-    tar xf clang+llvm-4.0.1-x86_64-linux-gnu-debian8.tar.xz && \
-    rm clang+llvm-4.0.1-x86_64-linux-gnu-debian8.tar.xz && \
-    mv clang+llvm-4.0.1-x86_64-linux-gnu-debian8 /opt/clang-4.0.1
-
-ENV UKING_CLANG=/opt/clang-4.0.1

README.md

@@ -122,36 +122,55 @@ Just like other game decompilations, this project is probably in a legal gray zo
 
 ### Alright, how do I start contributing?
 
-First, set up the build environment by following the instructions below. Then follow the [contributing guidelines here](Contributing.md).
+First, set up the build environment by following the instructions below.
 
 ## Building
 
-### Dependencies
-
-* Clang 4.0.1
-    * Download [Clang 4.0.1](https://releases.llvm.org/download.html#4.0.1) and extract the archive to a convenient location of your choice.
-    * You might also need libtinfo5. For Ubuntu or Debian users, install it with `sudo apt install libtinfo5`
-* Ninja
-    * For Ubuntu or Debian users, install it with `sudo apt install ninja-build`
-* CMake 3.13+
-    * If you are on Ubuntu 18.04, you must [update CMake by using the official CMake APT repository](https://apt.kitware.com/).
-* devkitA64
-    * [Follow this guide to set it up.](https://switchbrew.org/wiki/Setting_up_Development_Environment#Setup)
-
-Using Linux (or WSL) is recommended but not required. The rest of this guide assumes that you are using a Linux environment, though.
-
-### Building for Switch
-
-1. After cloning this repository, run: `git submodule update --init --recursive`
-2. Run: `env UKING_CLANG=$1 DEVKITA64=$2 cmake -GNinja -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_TOOLCHAIN_FILE=../ToolchainNX64.cmake -B build`
-    * Replace `$1` with the path to the extracted Clang archive, such that `$1/bin/clang` exists. This should be an absolute path; use `/home/<name>` instead of `~`.
-    * Replace `$2` with the path to devkitA64. On Linux, this is typically `/opt/devkitpro/devkitA64`.
-3. Start the build by running: `ninja -C build`
-
-On subsequent builds, just run `ninja -C build` from the project root.
-
 Reminder: **this will not produce a playable game.** This project will not allow you to play the game if you don't already own it on a Switch.
 
+### For Windows users
+
+While Linux is not a hard requirement, it is strongly advised to [set up WSL](https://docs.microsoft.com/en-us/windows/wsl/install-win10) to simplify the setup process. Ubuntu 20.04 is usually a good choice.
+
+The instructions below assume that you are using Linux (native or WSL) or macOS.
+
+### 1. Set up dependencies
+
+* Python 3.6 or newer
+* Ninja
+* CMake 3.13+
+    * If you are on Ubuntu 18.04, you must first [update CMake by using the official CMake APT repository](https://apt.kitware.com/).
+* [Optional] ccache (to speed up builds)
+
+Ubuntu users can install those dependencies by running:
+
+```shell
+sudo apt install python3 ninja-build cmake ccache
+```
+
+### 2. Set up the repository
+
+1. Clone this repository.
+2. Run `git submodule update --init --recursive`
+3. Run `tools/setup.py`
+    * This will set up [Clang 4.0.1](https://releases.llvm.org/download.html#4.0.1) and create a build directory in `build/`.
+
+### 3. Build
+
+To start the build, just run
+
+```shell
+ninja -C build
+```
+
+By default, Ninja will perform a multithreaded build. There is no need to pass -j manually.
+
+To check whether everything built correctly, just run `tools/check.py` after the build completes.
+
+## Contributing
+
+Follow the [contributing guidelines here](Contributing.md).
+
 ## Resources
 
 #### Breath of the Wild documentation and datamining

ToolchainNX64.cmake

@@ -1,45 +0,0 @@
-if (NOT DEFINED ENV{UKING_CLANG})
-    message(FATAL_ERROR "Please define the UKING_CLANG env variable. It should point to a path such that $UKING_CLANG/bin/clang exists")
-endif()
-
-if (NOT DEFINED ENV{DEVKITA64})
-    message(FATAL_ERROR "Please define the DEVKITA64 env variable.")
-endif()
-
-set(UKING_CLANG "$ENV{UKING_CLANG}")
-set(DEVKITA64 "$ENV{DEVKITA64}")
-set(NX64_OPT_FLAGS "-O3 -g")
-set(triple aarch64-none-elf)
-
-set(CMAKE_SYSTEM_NAME Generic)
-set(CMAKE_SYSTEM_VERSION 1)
-set(CMAKE_SYSTEM_PROCESSOR aarch64)
-
-set(CMAKE_SYSROOT ${UKING_CLANG})
-set(CMAKE_C_COMPILER "${UKING_CLANG}/bin/clang")
-set(CMAKE_C_COMPILER_TARGET ${triple})
-set(CMAKE_CXX_COMPILER "${UKING_CLANG}/bin/clang++")
-set(CMAKE_CXX_COMPILER_TARGET ${triple})
-
-set(CMAKE_C_FLAGS_RELEASE ${NX64_OPT_FLAGS})
-set(CMAKE_CXX_FLAGS_RELEASE ${NX64_OPT_FLAGS})
-set(CMAKE_C_FLAGS_RELWITHDEBINFO ${NX64_OPT_FLAGS})
-set(CMAKE_CXX_FLAGS_RELWITHDEBINFO ${NX64_OPT_FLAGS})
-
-set(ARCH "-mcpu=cortex-a57+fp+simd+crypto+crc")
-set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${ARCH} -isystem ${DEVKITA64}/aarch64-none-elf/include")
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -isystem ${UKING_CLANG}/include/c++/v1 -D _LIBCPP_HAS_THREAD_API_PTHREAD ${CMAKE_C_FLAGS}")
-set(CMAKE_ASM_FLAGS "${CMAKE_ASM_FLAGS} -x assembler-with-cpp ${ARCH}")
-
-add_compile_options(-fPIC -stdlib=libc++ -mno-implicit-float)
-add_link_options(-B ${DEVKITA64}/bin -fPIC -Wl,-Bsymbolic-functions -shared -nodefaultlibs)
-if(EXISTS "${DEVKITA64}/bin/ld.lld")
-    add_link_options(-fuse-ld=lld -Wl,-z,notext)
-endif()
-add_definitions(-D SWITCH -D __DEVKITA64__ -D __ELF__)
-add_definitions(-D NNSDK)
-add_definitions(-D MATCHING_HACK_NX_CLANG)
-
-# Helps with matching as this causes Clang to emit debug type info even for dynamic classes
-# with undefined vtables.
-add_compile_options(-fstandalone-debug)

toolchain/ToolchainNX64.cmake

@@ -0,0 +1,42 @@
+if (DEFINED ENV{UKING_CLANG})
+    set(UKING_CLANG "$ENV{UKING_CLANG}")
+else()
+    set(UKING_CLANG "toolchain/clang")
+endif()
+
+set(NX64_OPT_FLAGS "-O3 -g")
+set(NX64_TRIPLE aarch64-linux-elf)
+
+set(CMAKE_SYSTEM_NAME Generic)
+set(CMAKE_SYSTEM_VERSION 1)
+set(CMAKE_SYSTEM_PROCESSOR aarch64)
+
+set(CMAKE_SYSROOT ${CMAKE_CURRENT_LIST_DIR}/musl)
+set(CMAKE_C_COMPILER "${UKING_CLANG}/bin/clang")
+set(CMAKE_C_COMPILER_TARGET ${NX64_TRIPLE})
+set(CMAKE_CXX_COMPILER "${UKING_CLANG}/bin/clang++")
+set(CMAKE_CXX_COMPILER_TARGET ${NX64_TRIPLE})
+
+set(CMAKE_C_FLAGS_RELEASE ${NX64_OPT_FLAGS})
+set(CMAKE_CXX_FLAGS_RELEASE ${NX64_OPT_FLAGS})
+set(CMAKE_C_FLAGS_RELWITHDEBINFO ${NX64_OPT_FLAGS})
+set(CMAKE_CXX_FLAGS_RELWITHDEBINFO ${NX64_OPT_FLAGS})
+
+# Target options
+add_compile_options(-mcpu=cortex-a57+fp+simd+crypto+crc)
+add_compile_options(-mno-implicit-float)
+# Environment
+add_compile_options(-stdlib=libc++)
+add_compile_options(-fPIC)
+# Helps with matching as this causes Clang to emit debug type info even for dynamic classes
+# with undefined vtables.
+add_compile_options(-fstandalone-debug)
+
+add_definitions(-D SWITCH)
+add_definitions(-D NNSDK)
+add_definitions(-D MATCHING_HACK_NX_CLANG)
+
+add_link_options(-stdlib=libc++ -nostdlib)
+add_link_options(-fPIC -Wl,-Bsymbolic-functions -shared)
+# Use lld for performance reasons (and because we don't want a dependency on GNU tools)
+add_link_options(-fuse-ld=lld)

toolchain/musl

@@ -0,0 +1 @@
+Subproject commit 9a9252a54a67f54c066966f8f0599c450391bd44

tools/setup.py

@@ -0,0 +1,87 @@
+#!/usr/bin/env python3
+
+import os
+import platform
+from pathlib import Path
+import sys
+import tarfile
+import tempfile
+import urllib.request
+
+ROOT = Path(__file__).parent.parent
+
+
+def fail(error: str):
+    print(error)
+    sys.exit(1)
+
+
+def set_up_compiler():
+    compiler_dir = ROOT / "toolchain" / "clang"
+    if compiler_dir.is_dir():
+        print("clang is already set up: nothing to do")
+        return
+
+    system = platform.system()
+    machine = platform.machine()
+
+    builds = {
+        # Linux
+        ("Linux", "x86_64"): {
+            "url": "https://releases.llvm.org/4.0.1/clang+llvm-4.0.1-x86_64-linux-gnu-Fedora-25.tar.xz",
+            "dir_name": "clang+llvm-4.0.1-x86_64-linux-gnu-Fedora-25",
+        },
+        ("Linux", "aarch64"): {
+            "url": "https://releases.llvm.org/4.0.1/clang+llvm-4.0.1-aarch64-linux-gnu.tar.xz",
+            "dir_name": "clang+llvm-4.0.1-aarch64-linux-gnu",
+        },
+
+        # macOS
+        ("Darwin", "x86_64"): {
+            "url": "https://releases.llvm.org/4.0.1/clang+llvm-4.0.1-x86_64-apple-darwin.tar.xz",
+            "dir_name": "clang+llvm-4.0.1-x86_64-apple-darwin",
+        },
+        ("Darwin", "aarch64"): {
+            "url": "https://releases.llvm.org/4.0.1/clang+llvm-4.0.1-x86_64-apple-darwin.tar.xz",
+            "dir_name": "clang+llvm-4.0.1-x86_64-apple-darwin",
+        },
+    }
+
+    build_info = builds.get((system, machine))
+    if build_info is None:
+        fail(
+            f"unknown platform: {platform.platform()} (please report if you are on Linux and macOS)")
+
+    url: str = build_info["url"]
+    dir_name: str = build_info["dir_name"]
+
+    print(f"downloading Clang from {url}...")
+    with tempfile.TemporaryDirectory() as tmpdir:
+        path = tmpdir + "/" + url.split("/")[-1]
+        urllib.request.urlretrieve(url, path)
+
+        print(f"extracting Clang...")
+        with tarfile.open(path) as f:
+            f.extractall(compiler_dir.parent)
+            (compiler_dir.parent / dir_name).rename(compiler_dir)
+
+    print(">>> successfully set up Clang")
+
+
+def create_build_dir():
+    build_dir = ROOT / "build"
+    if build_dir.is_dir():
+        print("build directory already exists: nothing to do")
+        return
+
+    os.system("cmake -GNinja -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_TOOLCHAIN_FILE=toolchain/ToolchainNX64.cmake -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -B build/")
+    print(">>> created build directory")
+
+
+def main():
+    set_up_compiler()
+    create_build_dir()
+
+
+if __name__ == "__main__":
+    main()