Update dtk-template & fix source paths (#2231)

README.md

@@ -76,12 +76,9 @@ Building
   git clone https://github.com/zeldaret/tp.git
   ```
 
-- Copy your disc image to `orig/GZ2E01`.  
+- Copy your game's disc image to `orig/GZ2E01`.
-  Supported formats: ISO (GCM), RVZ, WIA, WBFS, CISO, NFS, GCZ, TGC.
+  - Supported formats: ISO (GCM), RVZ, WIA, WBFS, CISO, NFS, GCZ, TGC.
-  - To save space, extract the disc image and keep only the following files:
+  - After the initial build, the disc image can be deleted to save space.
-    - `sys/main.dol`
-    - `files/RELS.arc`
-    - `files/rel/**/*.rel`
 
 - Configure:
 

config/GZ2E01/splits.txt

@@ -3326,7 +3326,7 @@ PowerPC_EABI_Support/Runtime/Src/Gecko_ExceptionPPC.cp:
 	.text       start:0x803628AC end:0x80362914
 	.bss        start:0x8044D430 end:0x8044D440
 
-PowerPC_EABI_Support/Runtime/Src/GCN_mem_alloc.c:
+PowerPC_EABI_Support/Runtime/Src/GCN_Mem_Alloc.c:
 	.text       start:0x80362914 end:0x803629CC
 	.rodata     start:0x803A21A8 end:0x803A2220
 
@@ -3364,7 +3364,7 @@ PowerPC_EABI_Support/MSL/MSL_C/MSL_Common/Src/buffer_io.c:
 PowerPC_EABI_Support/MSL/MSL_C/MSL_Common/Src/char_io.c:
 	.text       start:0x803651D8 end:0x80365464
 
-PowerPC_EABI_Support/MSL/MSL_C/PPC_EABI/SRC/critical_regions.gamecube.c:
+PowerPC_EABI_Support/MSL/MSL_C/PPC_EABI/Src/critical_regions.gamecube.c:
 	.text       start:0x80365464 end:0x80365470
 
 PowerPC_EABI_Support/MSL/MSL_C/MSL_Common/Src/ctype.c:
@@ -3420,7 +3420,7 @@ PowerPC_EABI_Support/MSL/MSL_C/MSL_Common/Src/strtoul.c:
 PowerPC_EABI_Support/MSL/MSL_C/MSL_Common/Src/wchar_io.c:
 	.text       start:0x80369114 end:0x8036919C
 
-PowerPC_EABI_Support/MSL/MSL_C/PPC_EABI/SRC/uart_console_io_gcn.c:
+PowerPC_EABI_Support/MSL/MSL_C/PPC_EABI/Src/uart_console_io_gcn.c:
 	.text       start:0x8036919C end:0x80369274
 	.sbss       start:0x804519B0 end:0x804519B8
 
@@ -3534,7 +3534,7 @@ PowerPC_EABI_Support/MSL/MSL_C/MSL_Common_Embedded/Math/Double_precision/e_sqrt.
 	.text       start:0x8036C7A0 end:0x8036C9C4
 	.sdata2     start:0x80456B48 end:0x80456B50
 
-PowerPC_EABI_Support/MSL/MSL_C/PPC_EABI/SRC/math_ppc.c:
+PowerPC_EABI_Support/MSL/MSL_C/PPC_EABI/Src/math_ppc.c:
 	.text       start:0x8036C9C4 end:0x8036CA54
 
 PowerPC_EABI_Support/MSL/MSL_C/MSL_Common_Embedded/Math/Double_precision/w_sqrt.c:

configure.py

@@ -151,9 +151,9 @@ if not config.non_matching:
 # Tool versions
 config.binutils_tag = "2.42-1"
 config.compilers_tag = "20240706"
-config.dtk_tag = "v1.1.2"
+config.dtk_tag = "v1.2.0"
-config.objdiff_tag = "v2.3.2"
+config.objdiff_tag = "v2.3.4"
-config.sjiswrap_tag = "v1.1.1"
+config.sjiswrap_tag = "v1.2.0"
 config.wibo_tag = "0.6.11"
 
 # Project
@@ -1200,7 +1200,7 @@ config.libs = [
             Object(Matching, "PowerPC_EABI_Support/Runtime/Src/runtime.c"),
             Object(NonMatching, "PowerPC_EABI_Support/Runtime/Src/__init_cpp_exceptions.cpp"),
             Object(Matching, "PowerPC_EABI_Support/Runtime/Src/Gecko_ExceptionPPC.cp"),
-            Object(Matching, "PowerPC_EABI_Support/Runtime/Src/GCN_mem_alloc.c", extra_cflags=["-str reuse,nopool,readonly"]),
+            Object(Matching, "PowerPC_EABI_Support/Runtime/Src/GCN_Mem_Alloc.c", extra_cflags=["-str reuse,nopool,readonly"]),
         ],
     },
     {
@@ -1218,7 +1218,7 @@ config.libs = [
             Object(Matching, "PowerPC_EABI_Support/MSL/MSL_C/MSL_Common/Src/arith.c"),
             Object(Matching, "PowerPC_EABI_Support/MSL/MSL_C/MSL_Common/Src/buffer_io.c"),
             Object(Matching, "PowerPC_EABI_Support/MSL/MSL_C/MSL_Common/Src/char_io.c"),
-            Object(Matching, "PowerPC_EABI_Support/MSL/MSL_C/PPC_EABI/SRC/critical_regions.gamecube.c"),
+            Object(Matching, "PowerPC_EABI_Support/MSL/MSL_C/PPC_EABI/Src/critical_regions.gamecube.c"),
             Object(Matching, "PowerPC_EABI_Support/MSL/MSL_C/MSL_Common/Src/ctype.c"),
             Object(Matching, "PowerPC_EABI_Support/MSL/MSL_C/MSL_Common/Src/direct_io.c"),
             Object(Matching, "PowerPC_EABI_Support/MSL/MSL_C/MSL_Common/Src/file_io.c"),
@@ -1234,7 +1234,7 @@ config.libs = [
             Object(Matching, "PowerPC_EABI_Support/MSL/MSL_C/MSL_Common/Src/string.c"),
             Object(Matching, "PowerPC_EABI_Support/MSL/MSL_C/MSL_Common/Src/strtoul.c"),
             Object(Matching, "PowerPC_EABI_Support/MSL/MSL_C/MSL_Common/Src/wchar_io.c"),
-            Object(Matching, "PowerPC_EABI_Support/MSL/MSL_C/PPC_EABI/SRC/uart_console_io_gcn.c"),
+            Object(Matching, "PowerPC_EABI_Support/MSL/MSL_C/PPC_EABI/Src/uart_console_io_gcn.c"),
             Object(Matching, "PowerPC_EABI_Support/MSL/MSL_C/MSL_Common_Embedded/Math/Double_precision/e_acos.c"),
             Object(Matching, "PowerPC_EABI_Support/MSL/MSL_C/MSL_Common_Embedded/Math/Double_precision/e_asin.c"),
             Object(Matching, "PowerPC_EABI_Support/MSL/MSL_C/MSL_Common_Embedded/Math/Double_precision/e_atan2.c"),
@@ -1263,7 +1263,7 @@ config.libs = [
             Object(Matching, "PowerPC_EABI_Support/MSL/MSL_C/MSL_Common_Embedded/Math/Double_precision/w_fmod.c"),
             Object(Matching, "PowerPC_EABI_Support/MSL/MSL_C/MSL_Common_Embedded/Math/Double_precision/w_pow.c"),
             Object(Matching, "PowerPC_EABI_Support/MSL/MSL_C/MSL_Common_Embedded/Math/Double_precision/e_sqrt.c"),
-            Object(Matching, "PowerPC_EABI_Support/MSL/MSL_C/PPC_EABI/SRC/math_ppc.c"),
+            Object(Matching, "PowerPC_EABI_Support/MSL/MSL_C/PPC_EABI/Src/math_ppc.c"),
             Object(Matching, "PowerPC_EABI_Support/MSL/MSL_C/MSL_Common_Embedded/Math/Double_precision/w_sqrt.c"),
             Object(Matching, "PowerPC_EABI_Support/MSL/MSL_C/MSL_Common/Src/extras.c"),
         ],

tools/download_tool.py

@@ -55,6 +55,7 @@ def dtk_url(tag: str) -> str:
     repo = "https://github.com/encounter/decomp-toolkit"
     return f"{repo}/releases/download/{tag}/dtk-{system}-{arch}{suffix}"
 
+
 def objdiff_cli_url(tag: str) -> str:
     uname = platform.uname()
     suffix = ""

tools/ninja_syntax.py

@@ -24,17 +24,10 @@ import textwrap
 import os
 from io import StringIO
 from pathlib import Path
-from typing import Dict, List, Match, Optional, Tuple, Union
+from typing import Dict, Iterable, List, Match, Optional, Tuple, Union
 
 NinjaPath = Union[str, Path]
-NinjaPaths = Union[
+NinjaPaths = Iterable[Optional[NinjaPath]]
-    List[str],
-    List[Path],
-    List[NinjaPath],
-    List[Optional[str]],
-    List[Optional[Path]],
-    List[Optional[NinjaPath]],
-]
 NinjaPathOrPaths = Union[NinjaPath, NinjaPaths]
 
 
@@ -118,8 +111,8 @@ class Writer(object):
         pool: Optional[str] = None,
         dyndep: Optional[NinjaPath] = None,
     ) -> List[str]:
-        outputs = serialize_paths(outputs)
+        str_outputs = serialize_paths(outputs)
-        out_outputs = [escape_path(x) for x in outputs]
+        out_outputs = [escape_path(x) for x in str_outputs]
         all_inputs = [escape_path(x) for x in serialize_paths(inputs)]
 
         if implicit:
@@ -154,7 +147,7 @@ class Writer(object):
             for key, val in iterator:
                 self.variable(key, val, indent=1)
 
-        return outputs
+        return str_outputs
 
     def include(self, path: str) -> None:
         self._line("include %s" % path)
@@ -225,9 +218,11 @@ def serialize_path(input: Optional[NinjaPath]) -> str:
 
 
 def serialize_paths(input: Optional[NinjaPathOrPaths]) -> List[str]:
-    if isinstance(input, list):
+    if isinstance(input, str) or isinstance(input, Path):
-        return [serialize_path(path) for path in input if path]
         return [serialize_path(input)] if input else []
+    elif input is not None:
+        return [serialize_path(path) for path in input if path]
+    return []
 
 
 def escape(string: str) -> str:

tools/project.py

@@ -633,7 +633,7 @@ def generate_build_ninja(
         )
         n.newline()
 
-    def write_custom_step(step: str) -> List[str | Path]:
+    def write_custom_step(step: str, prev_step: Optional[str] = None) -> None:
         implicit: List[str | Path] = []
         if config.custom_build_steps and step in config.custom_build_steps:
             n.comment(f"Custom build steps ({step})")
@@ -657,7 +657,12 @@ def generate_build_ninja(
                     dyndep=custom_step.get("dyndep", None),
                 )
                 n.newline()
-        return implicit
+        n.build(
+            outputs=step,
+            rule="phony",
+            inputs=implicit,
+            order_only=prev_step,
+        )
 
     n.comment("Host build")
     n.variable("host_cflags", "-I include -Wno-trigraphs")
@@ -678,7 +683,7 @@ def generate_build_ninja(
     n.newline()
 
     # Add all build steps needed before we compile (e.g. processing assets)
-    precompile_implicit = write_custom_step("pre-compile")
+    write_custom_step("pre-compile")
 
     ###
     # Source files
@@ -726,13 +731,12 @@ def generate_build_ninja(
                     rule="link",
                     inputs=self.inputs,
                     implicit=[
-                        *precompile_implicit,
                         self.ldscript,
                         *mwld_implicit,
-                        *postcompile_implicit,
                     ],
                     implicit_outputs=elf_map,
                     variables={"ldflags": elf_ldflags},
+                    order_only="post-compile",
                 )
             else:
                 preplf_path = build_path / self.name / f"{self.name}.preplf"
@@ -759,6 +763,7 @@ def generate_build_ninja(
                     implicit=mwld_implicit,
                     implicit_outputs=preplf_map,
                     variables={"ldflags": preplf_ldflags},
+                    order_only="post-compile",
                 )
                 n.build(
                     outputs=plf_path,
@@ -767,6 +772,7 @@ def generate_build_ninja(
                     implicit=[self.ldscript, preplf_path, *mwld_implicit],
                     implicit_outputs=plf_map,
                     variables={"ldflags": plf_ldflags},
+                    order_only="post-compile",
                 )
             n.newline()
 
@@ -822,6 +828,7 @@ def generate_build_ninja(
                 implicit=(
                     mwcc_sjis_implicit if obj.options["shift_jis"] else mwcc_implicit
                 ),
+                order_only="pre-compile",
             )
 
             # Add ctx build rule
@@ -843,6 +850,7 @@ def generate_build_ninja(
                         "basedir": os.path.dirname(obj.host_obj_path),
                         "basefile": obj.host_obj_path.with_suffix(""),
                     },
+                    order_only="pre-compile",
                 )
                 if obj.options["add_to_all"]:
                     host_source_inputs.append(obj.host_obj_path)
@@ -877,6 +885,7 @@ def generate_build_ninja(
                 inputs=src_path,
                 variables={"asflags": asflags_str},
                 implicit=gnu_as_implicit,
+                order_only="pre-compile",
             )
             n.newline()
 
@@ -966,7 +975,7 @@ def generate_build_ninja(
             sys.exit(f"Linker {mw_path} does not exist")
 
         # Add all build steps needed before we link and after compiling objects
-        postcompile_implicit = write_custom_step("post-compile")
+        write_custom_step("post-compile", "pre-compile")
 
         ###
         # Link
@@ -977,7 +986,7 @@ def generate_build_ninja(
         n.newline()
 
         # Add all build steps needed after linking and before GC/Wii native format generation
-        postlink_implicit = write_custom_step("post-link")
+        write_custom_step("post-link", "post-compile")
 
         ###
         # Generate DOL
@@ -986,7 +995,8 @@ def generate_build_ninja(
             outputs=link_steps[0].output(),
             rule="elf2dol",
             inputs=link_steps[0].partial_output(),
-            implicit=[*postlink_implicit, dtk],
+            implicit=dtk,
+            order_only="post-link",
         )
 
         ###
@@ -1048,11 +1058,12 @@ def generate_build_ninja(
                     "rspfile": config.out_path() / f"rel{idx}.rsp",
                     "names": rel_names_arg,
                 },
+                order_only="post-link",
             )
             n.newline()
 
         # Add all build steps needed post-build (re-building archives and such)
-        postbuild_implicit = write_custom_step("post-build")
+        write_custom_step("post-build", "post-link")
 
         ###
         # Helper rule for building all source files
@@ -1091,7 +1102,8 @@ def generate_build_ninja(
             outputs=ok_path,
             rule="check",
             inputs=config.check_sha_path,
-            implicit=[dtk, *link_outputs, *postbuild_implicit],
+            implicit=[dtk, *link_outputs],
+            order_only="post-build",
         )
         n.newline()
 
@@ -1113,6 +1125,7 @@ def generate_build_ninja(
                 python_lib,
                 report_path,
             ],
+            order_only="post-build",
         )
 
         ###
@@ -1124,11 +1137,11 @@ def generate_build_ninja(
             command=f"{objdiff} report generate -o $out",
             description="REPORT",
         )
-        report_implicit: List[str | Path] = [objdiff, "all_source"]
         n.build(
             outputs=report_path,
             rule="report",
-            implicit=report_implicit,
+            implicit=[objdiff, "all_source"],
+            order_only="post-build",
         )
 
         ###
@@ -1386,7 +1399,7 @@ def generate_objdiff_config(
             progress_categories.append(category_opt)
         unit_config["metadata"].update(
             {
-                "complete": obj.completed,
+                "complete": obj.completed if src_exists else None,
                 "reverse_fn_order": reverse_fn_order,
                 "progress_categories": progress_categories,
             }