From 76c7e51f4a9f7062b173f47dc6509b35484a7c40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Fa=C3=9Fbender?= <31699028+Brotenko@users.noreply.github.com> Date: Sun, 2 Jan 2022 13:10:49 +0100 Subject: [PATCH] Update and move tools (#596) * update scripts and move outdated ones to /old/ * updates * resolve issues --- tools/asm_sizes.py | 56 ++++++++++++++++-------- tools/disasm_hud_element_animation.py | 5 ++- tools/find_duplicates.py | 63 +++++++++++++++------------ tools/get_variable.py | 8 +++- tools/{ => old}/bootstrap_effects.py | 2 +- tools/{ => old}/codescan.py | 2 +- tools/{ => old}/create_renames.py | 0 tools/{ => old}/fix_s_filenames.py | 2 +- tools/{ => old}/m_map_funcs.py | 0 tools/{ => old}/make_npc_structs.py | 0 tools/{ => old}/new_lines.py | 0 tools/{ => old}/set_reorder_asm.sh | 0 tools/rename.py | 2 +- tools/star_rod_enum_to_decomp.py | 18 ++++---- 14 files changed, 96 insertions(+), 62 deletions(-) rename tools/{ => old}/bootstrap_effects.py (97%) rename tools/{ => old}/codescan.py (92%) rename tools/{ => old}/create_renames.py (100%) rename tools/{ => old}/fix_s_filenames.py (96%) rename tools/{ => old}/m_map_funcs.py (100%) rename tools/{ => old}/make_npc_structs.py (100%) rename tools/{ => old}/new_lines.py (100%) rename tools/{ => old}/set_reorder_asm.sh (100%) diff --git a/tools/asm_sizes.py b/tools/asm_sizes.py index d223abc236..7ebc991b10 100755 --- a/tools/asm_sizes.py +++ b/tools/asm_sizes.py @@ -3,13 +3,21 @@ import json import glob import os +import argparse +from enum import IntEnum -print_funcs = True + +script_dir = os.path.dirname(os.path.realpath(__file__)) +asm_dir = script_dir + "/../ver/current/asm/nonmatchings" + +modes = [ "min", "max", "avg", "total", "size" ] sizes = {} funcs = {} + +# Calculate the number of instructions in a .s file def calc_insns(f_path): ret = 0 with open(f_path) as f: @@ -20,9 +28,11 @@ def calc_insns(f_path): funcs[f_path.split("/")[-1][:-2]] = ret return ret + +# Calculate different data points for each .c files and store them as a Tuple def do_dir(root, dir): max = 0 - min = None + min = 0 total = 0 files = glob.glob(os.path.join(root, dir) + "/*.s") @@ -31,27 +41,39 @@ def do_dir(root, dir): amt = calc_insns(f) if amt > max: max = amt - if min is None or amt < min: + if min == 0 or amt < min: min = amt total += amt avg = 0 if len(files) == 0 else total / len(files) - sizes[dir] = ((min, max, total, avg, len(files))) + sizes[root + "/" + dir] = ((min, max, total, avg, len(files))) -script_dir = os.path.dirname(os.path.realpath(__file__)) -asm_dir = script_dir + "/../ver/current/asm/nonmatchings" +parser = argparse.ArgumentParser(description="A tool to receive information about the number of non-matching .s files " + +"per .c file, or the size of .s files, measured by their number of instructions. " + +"Option -p is used by default if no option is specified.") +group = parser.add_mutually_exclusive_group() +group.add_argument("-f", "--files", help="Default. Print the number of non-matching .s files per .c file, ordered by size.", action='store_true', required=False) +group.add_argument("-a", "--alphabetical", help="Print the size of .s files, ordered by name.", action='store_true', required=False) +group.add_argument("-s", "--size", help="Print the size of .s files, ordered by size.", action='store_true', required=False) +parser.add_argument("-l", "--limit", help="Only print the .c --files that are greater than or equal to the value.", type=int, default=0, required=False) +parser.add_argument("-m", "--mode", help="Switches between output modes for --files. Allowed values are: {min, max, avg, total, size}.", choices=modes, default="size", metavar='', required=False) -for root, dirs, files in os.walk(asm_dir): - for asm_dir in dirs: - if "/os" not in root and "/world/" not in root: +args = parser.parse_args() + + +if __name__ == "__main__": + for root, dirs, files in os.walk(asm_dir): + for asm_dir in dirs: do_dir(root, asm_dir) - -for thing in sorted(sizes.keys(), key=lambda x: sizes[x][4]): - val = sizes[thing][4] - if val > 0: - print(thing.ljust(25) + str(val)) - -if print_funcs: - print(json.dumps(dict(sorted(funcs.items(), key=lambda f: f[1])), indent=4)) + + if args.alphabetical: + print(json.dumps(dict(sorted(funcs.items(), key=lambda f: f[0])), indent=4)) + elif args.size: + print(json.dumps(dict(sorted(funcs.items(), key=lambda f: f[1])), indent=4)) + else: + for thing in sorted(sizes.keys(), key=lambda x: sizes[x][modes.index(args.mode)]): + val = sizes[thing][modes.index(args.mode)] + if val > args.limit: + print(thing.split("nonmatchings/")[1].ljust(50) + str(val)) diff --git a/tools/disasm_hud_element_animation.py b/tools/disasm_hud_element_animation.py index 65de0cec78..ca314600a4 100755 --- a/tools/disasm_hud_element_animation.py +++ b/tools/disasm_hud_element_animation.py @@ -1,6 +1,7 @@ #! /usr/bin/python3 import struct +import argparse def fmt_size(val): if val == 0: @@ -56,6 +57,7 @@ def fmt_size(val): else: return val + class HudElementScript(): def __init__(self, symbol): self.symbol = symbol @@ -141,9 +143,8 @@ class HudElementScript(): print("};\n") -if __name__ == "__main__": - import argparse +if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("file", type=str, help=".data.s file to dissassemble") diff --git a/tools/find_duplicates.py b/tools/find_duplicates.py index e5eba1c33b..bbe223da12 100755 --- a/tools/find_duplicates.py +++ b/tools/find_duplicates.py @@ -294,35 +294,42 @@ def do_cross_query(): print(ccount.most_common(100)) -parser = argparse.ArgumentParser(description="Tools to assist with decomp") -parser.add_argument("query", help="function or file") -parser.add_argument("--threshold", help="score threshold between 0 and 1 (higher is more restrictive)", type=float, default=0.9, required=False) -parser.add_argument("--num-out", help="number of functions to display", type=int, default=100, required=False) -parser.add_argument("--generate-templates", help="automatically generate templates in `all` and `short` mode", action='store_true', required=False) +parser = argparse.ArgumentParser(description="Tool to find duplicates for a specific function or to find all duplicates across the codebase.") +group = parser.add_mutually_exclusive_group() +group.add_argument("-a", "--all", help="find ALL duplicates and output them into a file", action='store_true', required=False) +group.add_argument("-c", "--cross", help="do a cross query over the codebase", action='store_true', required=False) +group.add_argument("-s", "--short", help="find MOST duplicates besides some very small duplicates. Cuts the runtime in half with minimal loss", action='store_true', required=False) +parser.add_argument("query", help="function or file", nargs='?', default=None) +parser.add_argument("-t", "--threshold", help="score threshold between 0 and 1 (higher is more restrictive)", type=float, default=0.9, required=False) +parser.add_argument("-n", "--num-out", help="number of functions to display", type=int, default=100, required=False) args = parser.parse_args() -rom_bytes = read_rom() -map_syms = parse_map(os.path.join(root_dir, "ver", "current", "build", "papermario.map")) -map_offsets = get_map_offsets(map_syms) - -s_files = get_all_s_files() - -query_dir = find_dir(args.query) - -if query_dir is not None: - files = os.listdir(query_dir) - for f_name in files: - do_query(f_name[:-2]) -else: - if args.query == "cross": - args.threshold = 0.985 - do_cross_query() - elif args.query == "all": - args.threshold = 0.985 - all_matches(True) - elif args.query == "short": - args.threshold = 0.985 - all_matches(False) +if __name__ == "__main__": + rom_bytes = read_rom() + map_syms = parse_map(os.path.join(root_dir, "ver", "current", "build", "papermario.map")) + map_offsets = get_map_offsets(map_syms) + + s_files = get_all_s_files() + + query_dir = find_dir(args.query) + + if query_dir is not None: + files = os.listdir(query_dir) + for f_name in files: + do_query(f_name[:-2]) else: - do_query(args.query) + if args.cross: + args.threshold = 0.985 + do_cross_query() + elif args.all: + args.threshold = 0.985 + all_matches(True) + elif args.short: + args.threshold = 0.985 + all_matches(False) + else: + if args.query is None: + parser.print_help() + else: + do_query(args.query) diff --git a/tools/get_variable.py b/tools/get_variable.py index cc232b06d5..92ed2536ba 100755 --- a/tools/get_variable.py +++ b/tools/get_variable.py @@ -1,5 +1,7 @@ #!/usr/bin/python3 +import sys + def get_variable(arg): v = arg - 2**32 # convert to s32 if v > -250000000: @@ -25,5 +27,7 @@ def get_variable(arg): return f"{arg}" if __name__ == "__main__": - import sys - print(get_variable(int(sys.argv[1], 0))) + try: + print(get_variable(int(sys.argv[1], 0))) + except: + print("Invalid literal for numeric operation. Please input a hex/dec number.") diff --git a/tools/bootstrap_effects.py b/tools/old/bootstrap_effects.py similarity index 97% rename from tools/bootstrap_effects.py rename to tools/old/bootstrap_effects.py index c5eeb2a0d0..430af0eb9c 100755 --- a/tools/bootstrap_effects.py +++ b/tools/old/bootstrap_effects.py @@ -5,7 +5,7 @@ import re from pathlib import Path script_dir = os.path.dirname(os.path.realpath(__file__)) -root_dir = script_dir + "/../" +root_dir = script_dir + "/../../" asm_dir = root_dir + "ver/current/asm/" asm_effects_dir = asm_dir + "nonmatchings/effects/" diff --git a/tools/codescan.py b/tools/old/codescan.py similarity index 92% rename from tools/codescan.py rename to tools/old/codescan.py index 844ec9ce39..6b6bfdd60d 100644 --- a/tools/codescan.py +++ b/tools/old/codescan.py @@ -8,7 +8,7 @@ import subprocess from pathlib import Path script_dir = os.path.dirname(os.path.realpath(__file__)) -root_dir = os.path.abspath(os.path.join(script_dir, "..")) +root_dir = os.path.abspath(os.path.join(script_dir, "../..")) import glob, os os.chdir(root_dir) diff --git a/tools/create_renames.py b/tools/old/create_renames.py similarity index 100% rename from tools/create_renames.py rename to tools/old/create_renames.py diff --git a/tools/fix_s_filenames.py b/tools/old/fix_s_filenames.py similarity index 96% rename from tools/fix_s_filenames.py rename to tools/old/fix_s_filenames.py index f18b3c649a..9f0424e5c0 100755 --- a/tools/fix_s_filenames.py +++ b/tools/old/fix_s_filenames.py @@ -7,7 +7,7 @@ import os import re script_dir = os.path.dirname(os.path.realpath(__file__)) -root_dir = script_dir + "/../" +root_dir = script_dir + "/../../" asm_dir = root_dir + "ver/current/asm/nonmatchings/" for root, dirs, files in os.walk(asm_dir): diff --git a/tools/m_map_funcs.py b/tools/old/m_map_funcs.py similarity index 100% rename from tools/m_map_funcs.py rename to tools/old/m_map_funcs.py diff --git a/tools/make_npc_structs.py b/tools/old/make_npc_structs.py similarity index 100% rename from tools/make_npc_structs.py rename to tools/old/make_npc_structs.py diff --git a/tools/new_lines.py b/tools/old/new_lines.py similarity index 100% rename from tools/new_lines.py rename to tools/old/new_lines.py diff --git a/tools/set_reorder_asm.sh b/tools/old/set_reorder_asm.sh similarity index 100% rename from tools/set_reorder_asm.sh rename to tools/old/set_reorder_asm.sh diff --git a/tools/rename.py b/tools/rename.py index f2048b0840..e9c47de827 100755 --- a/tools/rename.py +++ b/tools/rename.py @@ -29,7 +29,7 @@ def handle_file(f_path, try_rename_file=False): f.write(f_text) -# Read Star Rod's output file +# Read Star Rod's output file (one rename per line, old and new, delimited by a space) with open(os.path.join(script_dir, "to_rename.txt")) as f: renames_text = f.readlines() diff --git a/tools/star_rod_enum_to_decomp.py b/tools/star_rod_enum_to_decomp.py index 73953afc83..434c023a04 100755 --- a/tools/star_rod_enum_to_decomp.py +++ b/tools/star_rod_enum_to_decomp.py @@ -6,13 +6,6 @@ import re import os from glob import glob -parser = argparse.ArgumentParser(description='Convert a StarRod enum into an enum that is in decomp format') -parser.add_argument("query", help="StarRod enum file") -#parser.add_argument("-p", "--prefix", help="specify a prefix for the enum elements", type=str, required=False) -parser.add_argument("-r", "--recursive", help="recursively convert all files to enums", type=bool, required=False) - -args = parser.parse_args() - def create_enum(file_content, prefix, ordering): ret = "" @@ -34,11 +27,11 @@ def create_enum(file_content, prefix, ordering): for (key, value) in re.findall(r'(\S+)\s+=\s+(\S+)', file_content): if ordering: - key = '_'.join(re.sub(r'([A-Z])', r' \1', key).split()).replace("H_P", "HP").replace("F_P", "FP").replace("J_P", "JP").replace("N_P_C", "NPC").replace("__", "_").replace("-", "") + key = '_'.join(re.sub(r'([A-Z]{1,2})', r' \1', key).split()).replace("N_P_C", "NPC").replace("__", "_").replace("-", "") key = prefix.upper() + '_{:<{width}}'.format(key, width=max_size + 2).upper() ret += " " + key + " = 0x" + '{:>{fill}{width}}'.format(value, fill=0, width=8) + ",\n" else: - value = '_'.join(re.sub(r'([A-Z])', r' \1', value).split()).replace("H_P", "HP").replace("F_P", "FP").replace("J_P", "JP").replace("N_P_C", "NPC").replace("__", "_").replace("-", "") + value = '_'.join(re.sub(r'([A-Z]{1,2})', r' \1', value).split()).replace("N_P_C", "NPC").replace("__", "_").replace("-", "") value = prefix.upper() + '_{:<{width}}'.format(value, width=max_size + 2).upper() ret += " " + value + " = 0x" + '{:>{fill}{width}}'.format(key, fill=0, width=8) + ",\n" @@ -113,5 +106,12 @@ def main(args): file.close() +parser = argparse.ArgumentParser(description='Convert a StarRod enum into an enum that is in a decomp compatible format') +parser.add_argument("query", help="StarRod enum file or folder") +parser.add_argument("-r", "--recursive", help="recursively convert all files to enums", type=bool, required=False) + +args = parser.parse_args() + + if __name__ == "__main__": main(parser.parse_args())