Cleanup: Pass all paths to tools rather than tools constructing them (#1669)

* Pass all paths to tools rather than tools constructing them

* review

* fix

Makefile

@@ -448,22 +448,22 @@ venv:
 
 setup:
 	$(MAKE) -C tools
-	$(PYTHON) tools/buildtools/decompress_baserom.py $(VERSION)
+	$(PYTHON) tools/buildtools/decompress_baserom.py -v $(VERSION)
-	$(PYTHON) tools/buildtools/extract_baserom.py $(BASEROM_DIR)/baserom-decompressed.z64 -o $(EXTRACTED_DIR)/baserom --dmadata-start `cat $(BASEROM_DIR)/dmadata_start.txt` --dmadata-names $(BASEROM_DIR)/dmadata_names.txt
+	$(PYTHON) tools/buildtools/extract_baserom.py $(BASEROM_DIR)/baserom-decompressed.z64 $(EXTRACTED_DIR)/baserom --dmadata-start `cat $(BASEROM_DIR)/dmadata_start.txt` --dmadata-names $(BASEROM_DIR)/dmadata_names.txt
-	$(PYTHON) tools/buildtools/extract_yars.py $(VERSION)
+	$(PYTHON) tools/buildtools/extract_yars.py $(EXTRACTED_DIR)/baserom -v $(VERSION)
 
 # TODO this is a temporary rule for testing audio, to be removed
 setup-audio:
 	$(AUDIO_EXTRACT) -o $(EXTRACTED_DIR) -v $(VERSION) --read-xml
 
 assets:
-	$(PYTHON) extract_assets.py -j $(N_THREADS) -Z Wno-hardcoded-pointer
+	$(PYTHON) extract_assets.py $(EXTRACTED_DIR)/baserom assets -j$(N_THREADS) -Z Wno-hardcoded-pointer
 	$(AUDIO_EXTRACT) -o $(EXTRACTED_DIR) -v $(VERSION) --read-xml
 
 ## Assembly generation
 disasm:
 	$(RM) -r asm data
-	$(PYTHON) tools/disasm/disasm.py -j $(N_THREADS) $(DISASM_FLAGS)
+	$(PYTHON) tools/disasm/disasm.py $(EXTRACTED_DIR)/baserom -j $(N_THREADS) $(DISASM_FLAGS)
 
 diff-init: rom
 	$(RM) -r expected/

extract_assets.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 
 import argparse, json, os, signal, time, colorama, multiprocessing
+from pathlib import Path
 
 colorama.init()
 
@@ -27,7 +28,7 @@ def ExtractFile(xmlPath, outputPath, outputSourcePath):
             generateSourceFile = "0"
             break
 
-    execStr = f"tools/ZAPD/ZAPD.out e -eh -i {xmlPath} -b extracted/n64-us/baserom -o {outputPath} -osf {outputSourcePath} -gsf {generateSourceFile} -rconf tools/ZAPDConfigs/MM/Config.xml {ZAPDArgs}"
+    execStr = f"tools/ZAPD/ZAPD.out e -eh -i {xmlPath} -b {globalBaseromSegmentsDir} -o {outputPath} -osf {outputSourcePath} -gsf {generateSourceFile} -rconf tools/ZAPDConfigs/MM/Config.xml {ZAPDArgs}"
 
     if globalUnaccounted:
         execStr += " -Wunaccounted"
@@ -46,9 +47,9 @@ def ExtractFunc(fullPath):
     objectName = os.path.splitext(xmlName)[0]
 
     if "scenes" in pathList:
-        outPath = os.path.join("assets", *pathList[2:])
+        outPath = os.path.join(globalOutputDir, *pathList[2:])
     else:
-        outPath = os.path.join("assets", *pathList[2:], objectName)
+        outPath = os.path.join(globalOutputDir, *pathList[2:], objectName)
     outSourcePath = outPath
 
     if fullPath in globalExtractedAssetsTracker:
@@ -68,18 +69,32 @@ def ExtractFunc(fullPath):
             globalExtractedAssetsTracker[fullPath] = globalManager.dict()
         globalExtractedAssetsTracker[fullPath]["timestamp"] = currentTimeStamp
 
-def initializeWorker(abort, unaccounted: bool, extractedAssetsTracker: dict, manager):
+def initializeWorker(abort, unaccounted: bool, extractedAssetsTracker: dict, manager, baseromSegmentsDir: Path, outputDir: Path):
     global globalAbort
     global globalUnaccounted
     global globalExtractedAssetsTracker
     global globalManager
+    global globalBaseromSegmentsDir
+    global globalOutputDir
     globalAbort = abort
     globalUnaccounted = unaccounted
     globalExtractedAssetsTracker = extractedAssetsTracker
     globalManager = manager
+    globalBaseromSegmentsDir = baseromSegmentsDir
+    globalOutputDir = outputDir
 
 def main():
     parser = argparse.ArgumentParser(description="baserom asset extractor")
+    parser.add_argument(
+        "baserom_segments_dir",
+        type=Path,
+        help="Directory of uncompressed ROM segments",
+    )
+    parser.add_argument(
+        "output_dir",
+        type=Path,
+        help="Output directory to place files in",
+    )
     parser.add_argument("-s", "--single", help="asset path relative to assets/, e.g. objects/gameplay_keep")
     parser.add_argument("-f", "--force", help="Force the extraction of every xml instead of checking the touched ones.", action="store_true")
     parser.add_argument("-j", "--jobs", help="Number of cpu cores to extract with.")
@@ -87,6 +102,9 @@ def main():
     parser.add_argument("-Z", help="Pass the argument on to ZAPD, e.g. `-ZWunaccounted` to warn about unaccounted blocks in XMLs. Each argument should be passed separately, *without* the leading dash.", metavar="ZAPD_ARG", action="append")
     args = parser.parse_args()
 
+    baseromSegmentsDir: Path = args.baserom_segments_dir
+    outputDir: Path = args.output_dir
+
     global ZAPDArgs
     ZAPDArgs = ""
     if args.Z is not None:
@@ -115,8 +133,8 @@ def main():
         with open(EXTRACTED_ASSETS_NAMEFILE, encoding='utf-8') as f:
             extractedAssetsTracker.update(json.load(f, object_hook=manager.dict))
 
-    extract_text_path = "assets/text/message_data.h"
+    extract_text_path = outputDir / "text/message_data.h"
-    extract_staff_text_path = "assets/text/staff_message_data.h"
+    extract_staff_text_path = outputDir / "text/staff_message_data.h"
 
     asset_path = args.single
     if asset_path is not None:
@@ -134,7 +152,7 @@ def main():
                 print(f"Error. File {fullPath} does not exist.", file=os.sys.stderr)
                 exit(1)
 
-            initializeWorker(mainAbort, args.unaccounted, extractedAssetsTracker, manager)
+            initializeWorker(mainAbort, args.unaccounted, extractedAssetsTracker, manager, baseromSegmentsDir, outputDir)
             # Always extract if -s is used.
             if fullPath in extractedAssetsTracker:
                 del extractedAssetsTracker[fullPath]
@@ -144,12 +162,12 @@ def main():
         if args.force or not os.path.isfile(extract_text_path):
             from tools.msg.nes import msgdisNES
             print("Extracting message_data")
-            msgdisNES.main(extract_text_path)
+            msgdisNES.main(baseromSegmentsDir, extract_text_path)
 
         if args.force or not os.path.isfile(extract_staff_text_path):
             from tools.msg.staff import msgdisStaff
             print("Extracting staff_message_data")
-            msgdisStaff.main(extract_staff_text_path)
+            msgdisStaff.main(baseromSegmentsDir, extract_staff_text_path)
 
         xmlFiles = []
         for currentPath, _, files in os.walk(os.path.join("assets", "xml")):
@@ -163,13 +181,13 @@ def main():
             if numCores <= 0:
                 numCores = 1
             print("Extracting assets with " + str(numCores) + " CPU core" + ("s" if numCores > 1 else "") + ".")
-            with multiprocessing.get_context("fork").Pool(numCores,  initializer=initializeWorker, initargs=(mainAbort, args.unaccounted, extractedAssetsTracker, manager)) as p:
+            with multiprocessing.get_context("fork").Pool(numCores,  initializer=initializeWorker, initargs=(mainAbort, args.unaccounted, extractedAssetsTracker, manager, baseromSegmentsDir, outputDir)) as p:
                 p.map(ExtractFunc, xmlFiles)
         except (multiprocessing.ProcessError, TypeError):
             print("Warning: Multiprocessing exception ocurred.", file=os.sys.stderr)
             print("Disabling mutliprocessing.", file=os.sys.stderr)
 
-            initializeWorker(mainAbort, args.unaccounted, extractedAssetsTracker, manager)
+            initializeWorker(mainAbort, args.unaccounted, extractedAssetsTracker, manager, baseromSegmentsDir, outputDir)
             for singlePath in xmlFiles:
                 ExtractFunc(singlePath)
 

tools/buildtools/decompress_baserom.py

@@ -157,8 +157,10 @@ def main():
         description="Convert a rom that uses dmadata to an uncompressed one."
     )
     parser.add_argument(
-        "version",
+        "-v",
+        "--version",
         help="Version of the game to decompress.",
+        default="n64-us",
     )
 
     args = parser.parse_args()

tools/buildtools/extract_baserom.py

@@ -20,10 +20,8 @@ def main():
         "rom", metavar="ROM", type=Path, help="Path to uncompressed ROM"
     )
     parser.add_argument(
-        "-o",
+        "output_dir",
-        "--output-dir",
         type=Path,
-        required=True,
         help="Output directory for segments",
     )
     parser.add_argument(

tools/buildtools/extract_yars.py

@@ -100,20 +100,33 @@ def extractArchive(archivePath: Path, outPath: Path):
 
 def main():
     parser = argparse.ArgumentParser(description="MM archives extractor")
-    parser.add_argument("version", help="version to process", default="n64-us")
+    parser.add_argument(
+        "baserom_segments_dir",
+        type=Path,
+        help="Directory of uncompressed ROM segments",
+    )
+    parser.add_argument(
+        "-v",
+        "--version",
+        help="version to process",
+        default="n64-us",
+    )
     parser.add_argument("--xml", help="Generate xml to stdout", action="store_true")
 
     args = parser.parse_args()
 
+    baseromSegmentsDir: Path = args.baserom_segments_dir
+    version: str = args.version
+
     global PRINT_XML
     PRINT_XML = args.xml
 
-    archivesCsvPath = Path(f"tools/filelists/{args.version}/archives.csv")
+    archivesCsvPath = Path(f"tools/filelists/{version}/archives.csv")
 
     with archivesCsvPath.open() as f:
         for line in f:
             archiveName = line.strip().split(",")[1]
-            archivePath = Path(f"extracted/{args.version}/baserom/{archiveName}")
+            archivePath = baseromSegmentsDir / archiveName
 
             extractedPath = Path(str(archivePath) + ".unarchive")
             extractArchive(archivePath, extractedPath)

tools/disasm/disasm.py

@@ -13,6 +13,11 @@ fpr_name_options = {
 }
 
 parser = argparse.ArgumentParser()
+parser.add_argument(
+    "baserom_segments_dir",
+    type=Path,
+    help="Directory of uncompressed ROM segments",
+)
 parser.add_argument(
     "-j", dest="jobs", type=int, default=1, help="number of processes to run at once"
 )
@@ -39,6 +44,8 @@ parser.add_argument(
 args = parser.parse_args()
 jobs = args.jobs
 
+baserom_segments_dir: Path = args.baserom_segments_dir
+
 rabbitizer.config.regNames_fprAbiNames = rabbitizer.Abi.fromStr(args.reg_names)
 rabbitizer.config.regNames_userFpcCsr = False
 
@@ -1873,7 +1880,7 @@ def disassemble_makerom(section):
 
     elif section[-1]["type"] == "ipl3":
         # TODO disassemble this eventually, low priority
-        out = f"{asm_header('.text')}\n.incbin \"extracted/n64-us/baserom/makerom\", 0x40, 0xFC0\n"
+        out = f"{asm_header('.text')}\n.incbin \"{baserom_segments_dir}/makerom\", 0x40, 0xFC0\n"
 
         with open(ASM_OUT + "/makerom/ipl3.s", "w") as outfile:
             outfile.write(out)
@@ -2154,7 +2161,7 @@ for var in sorted(variables_ast.keys()):
 # Read in binary and relocation data for each segment
 for seg, segment in enumerate(files_spec):
     binary = None
-    with open(segment[1] + "/" + segment[0], "rb") as infile:
+    with (baserom_segments_dir / segment[0]).open("rb") as infile:
         binary = bytes(infile.read())
 
     if segment[2] == "overlay":

tools/msg/nes/msgdisNES.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 
 import argparse
+from pathlib import Path
 import sys
 import struct
 
@@ -250,9 +251,9 @@ class MessageNES:
                 prevNewline = False
                 prevCmd = True
 
-def parseTable(start):
+def parseTable(baseromSegmentsDir: Path, start):
     table = {}
-    with open("extracted/n64-us/baserom/code","rb") as f:
+    with open(baseromSegmentsDir / "code","rb") as f:
         f.seek(start)
         buf = f.read(8)
         textId, typePos, segment = struct.unpack(">HBxI", buf)
@@ -266,12 +267,10 @@ def parseTable(start):
 NES_MESSAGE_TABLE_ADDR = 0x1210D8 # Location of NES message table in extracted/n64-us/baserom/code
 NES_SEGMENT_ADDR = 0x08000000
 
-def main(outfile):
+def main(baseromSegmentsDir: Path, outfile):
-    msgTable = parseTable(NES_MESSAGE_TABLE_ADDR)
+    msgTable = parseTable(baseromSegmentsDir, NES_MESSAGE_TABLE_ADDR)
 
-    buf = []
+    buf = (baseromSegmentsDir / "message_data_static").read_bytes()
-    with open("extracted/n64-us/baserom/message_data_static", "rb") as f:
-        buf = f.read()
 
     bufLen = len(buf)
     i = 0
@@ -310,7 +309,14 @@ def main(outfile):
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(description="Extract message_data_static text")
+    parser.add_argument(
+        "--baserom-segments",
+        dest="baserom_segments_dir",
+        type=Path,
+        required=True,
+        help="Directory of uncompressed ROM segments",
+    )
     parser.add_argument('-o', '--outfile', help='output file to write to. None for stdout')
     args = parser.parse_args()
 
-    main(args.outfile)
+    main(args.baserom_segments_dir, args.outfile)

tools/msg/staff/msgdisStaff.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 
 import argparse
+from pathlib import Path
 import sys
 import struct
 
@@ -170,9 +171,9 @@ class MessageCredits:
                 prevNewline = False
                 prevCmd = True
 
-def parseTable(start):
+def parseTable(baseromSegmentsDir: Path, start):
     table = {}
-    with open("extracted/n64-us/baserom/code","rb") as f:
+    with open(baseromSegmentsDir / "code","rb") as f:
         f.seek(start)
         buf = f.read(8)
         textId, typePos, segment = struct.unpack(">HBxI", buf)
@@ -186,12 +187,10 @@ def parseTable(start):
 STAFF_MESSAGE_TABLE_ADDR = 0x12A048 # Location of Staff message table in extracted/n64-us/baserom/code
 STAFF_SEGMENT_ADDR = 0x07000000
 
-def main(outfile):
+def main(baseromSegmentsDir: Path, outfile):
-    msgTable = parseTable(STAFF_MESSAGE_TABLE_ADDR)
+    msgTable = parseTable(baseromSegmentsDir, STAFF_MESSAGE_TABLE_ADDR)
 
-    buf = []
+    buf = (baseromSegmentsDir / "staff_message_data_static").read_bytes()
-    with open("extracted/n64-us/baserom/staff_message_data_static", "rb") as f:
-        buf = f.read()
 
     bufLen = len(buf)
     i = 0
@@ -225,7 +224,14 @@ def main(outfile):
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(description="Extract staff_message_data_static text")
+    parser.add_argument(
+        "--baserom-segments",
+        dest="baserom_segments_dir",
+        type=Path,
+        required=True,
+        help="Directory of uncompressed ROM segments",
+    )
     parser.add_argument('-o', '--outfile', help='output file to write to. None for stdout')
     args = parser.parse_args()
 
-    main(args.outfile)
+    main(args.baserom_segments_dir, args.outfile)