Compare non-matching functions against expected output

This makes it possible to catch regressions for non-matching functions,
especially those that only have minor issues.

This also reclassifies some minor non-matchings as major non-matchings
whenever it's really not obvious to see that they are equivalent.

data/uking_functions.csv

@@ -44229,7 +44229,7 @@
 0x00000071006dfd8c,DamageMgrBase::ctor,76,_ZN4ksys3dmg17DamageManagerBaseC1EPNS_3act5ActorE?
 0x00000071006dfdd8,DamageMgrBase::m20,96,
 0x00000071006dfe38,DamageMgrBase::resetStuff,20,
-0x00000071006dfe4c,DamageMgrBase::applyDamage,500,_ZN4ksys3dmg17DamageManagerBase11applyDamageERi?
+0x00000071006dfe4c,DamageMgrBase::applyDamage,500,_ZN4ksys3dmg17DamageManagerBase11applyDamageERi!
 0x00000071006e0040,DamageMgrBase::m46,204,_ZN4ksys3dmg17DamageManagerBase21handleDamageForPlayerEPjS2_S2_S2_S2_
 0x00000071006e010c,DamageMgrBase::addDamage,72,_ZN4ksys3dmg17DamageManagerBase9addDamageEliiiiii
 0x00000071006e0154,DamageMgrBase::isSlowTime,4,
@@ -61527,7 +61527,7 @@
 0x0000007100af9be4,nullsub_5537,4,
 0x0000007100af9be8,nullsub_3155,4,
 0x0000007100af9bec,nullsub_3156,4,
-0x0000007100af9bf0,_ZN4sead13CalculateTaskC1ERKNS_16TaskConstructArgEPKc,296,_ZN4sead13CalculateTaskC1ERKNS_16TaskConstructArgEPKc?
+0x0000007100af9bf0,_ZN4sead13CalculateTaskC1ERKNS_16TaskConstructArgEPKc,296,_ZN4sead13CalculateTaskC1ERKNS_16TaskConstructArgEPKc!
 0x0000007100af9d18,_ZN4sead13CalculateTaskD2Ev,84,_ZN4sead13CalculateTaskD1Ev
 0x0000007100af9d6c,sead::CalculateTask::dtorDelete,92,_ZN4sead13CalculateTaskD0Ev
 0x0000007100af9dc8,sead::CalculateTask::getCorrespondingMethodTreeMgrTypeInfo,92,
@@ -63403,7 +63403,7 @@
 0x0000007100b50bc8,sub_7100B50BC8,8,_ZN3agl3utl14ParameterCurveILj2EE7typePtrEv
 0x0000007100b50bd0,sub_7100B50BD0,8,_ZNK3agl3utl14ParameterCurveILj2EE4sizeEv
 0x0000007100b50bd8,_ZNK3agl3utl14ParameterCurveILj2EE5cloneEPN4sead4HeapEPNS0_13IParameterObjE,500,_ZNK3agl3utl14ParameterCurveILj2EE5cloneEPN4sead4HeapEPNS0_13IParameterObjE?
-0x0000007100b50dcc,_ZN3agl3utl14ParameterCurveILj2EE18postApplyResource_EPKvm,140,_ZN3agl3utl14ParameterCurveILj2EE18postApplyResource_EPKvm?
+0x0000007100b50dcc,_ZN3agl3utl14ParameterCurveILj2EE18postApplyResource_EPKvm,140,_ZN3agl3utl14ParameterCurveILj2EE18postApplyResource_EPKvm!
 0x0000007100b50e58,_ZN3agl3utl13ParameterBaseD2Ev,20,_ZN3agl3utl13ParameterBaseD2Ev
 0x0000007100b50e6c,j__ZdlPv_647,4,_ZN3agl3utl14ParameterCurveILj4EED0Ev
 0x0000007100b50e70,_ZN3agl3utl14ParameterCurveILj4EE4copyERKNS0_13ParameterBaseE,112,_ZN3agl3utl14ParameterCurveILj4EE4copyERKNS0_13ParameterBaseE
@@ -63417,7 +63417,7 @@
 0x0000007100b51bec,sub_7100B51BEC,8,_ZN3agl3utl14ParameterCurveILj4EE7typePtrEv
 0x0000007100b51bf4,sub_7100B51BF4,8,_ZNK3agl3utl14ParameterCurveILj4EE4sizeEv
 0x0000007100b51bfc,_ZNK3agl3utl14ParameterCurveILj4EE5cloneEPN4sead4HeapEPNS0_13IParameterObjE,448,_ZNK3agl3utl14ParameterCurveILj4EE5cloneEPN4sead4HeapEPNS0_13IParameterObjE?
-0x0000007100b51dbc,_ZN3agl3utl14ParameterCurveILj4EE18postApplyResource_EPKvm,220,_ZN3agl3utl14ParameterCurveILj4EE18postApplyResource_EPKvm?
+0x0000007100b51dbc,_ZN3agl3utl14ParameterCurveILj4EE18postApplyResource_EPKvm,220,_ZN3agl3utl14ParameterCurveILj4EE18postApplyResource_EPKvm!
 0x0000007100b51e98,sub_7100B51E98,84,
 0x0000007100b51eec,_ZN3agl3utl15ParameterBufferIiED0Ev,68,
 0x0000007100b51f30,sub_7100B51F30,8,
@@ -88726,7 +88726,7 @@
 0x0000007101166b64,sub_7101166B64,556,_ZN4ksys3res9ActorLink5UsersC2Ev
 0x0000007101166d90,nullsub_4611,4,_ZN4ksys3res9ActorLink9doCreate_EPhjPN4sead4HeapE
 0x0000007101166d94,nullsub_4612,4,_ZThn632_N4ksys3res9ActorLink9doCreate_EPhjPN4sead4HeapE
-0x0000007101166d98,Bxml::parse,428,_ZN4ksys3res9ActorLink6parse_EPhmPN4sead4HeapE?
+0x0000007101166d98,Bxml::parse,428,_ZN4ksys3res9ActorLink6parse_EPhmPN4sead4HeapE!
 0x0000007101166f44,ResourceBxml::parse,28,_ZThn632_N4ksys3res9ActorLink6parse_EPhmPN4sead4HeapE
 0x0000007101166f60,sub_7101166F60,124,_ZN4ksys3res9ActorLink9finalize_Ev
 0x0000007101166fdc,sub_7101166FDC,124,_ZThn632_N4ksys3res9ActorLink9finalize_Ev
@@ -89479,7 +89479,7 @@
 0x00000071011b9d00,ActorBase::finalizeInit,164,_ZN4ksys3act8BaseProc13finalizeInit_EPNS1_11InitContextE
 0x00000071011b9da4,ActorBase::deleteLater,284,_ZN4ksys3act8BaseProc11deleteLaterENS1_12DeleteReasonE
 0x00000071011b9ec0,ActorBase::init,196,_ZN4ksys3act8BaseProc4initEPN4sead4HeapEb
-0x00000071011b9f84,ActorBase::startPreparingForPreDelete_,108,_ZN4ksys3act8BaseProc27startPreparingForPreDelete_Ev?
+0x00000071011b9f84,ActorBase::startPreparingForPreDelete_,108,_ZN4ksys3act8BaseProc27startPreparingForPreDelete_Ev!
 0x00000071011b9ff0,ActorBase::free,96,_ZN4ksys3act8BaseProc9destruct_Ei
 0x00000071011ba050,ActorBase::requestPreDelete,324,_ZN4ksys3act8BaseProc16processPreDeleteEv
 0x00000071011ba194,ActorBase::freeLinkData,24,
@@ -89535,7 +89535,7 @@
 0x00000071011bbc0c,BaseProcHandle::wakeUpActorAndReleaseUnit,184,
 0x00000071011bbcc4,BaseProcHandle::getBaseProcEvent,48,
 0x00000071011bbcf4,BaseProcHandle::allocUnit,228,
-0x00000071011bbdd8,BaseProcUnit::setActor,540,_ZN4ksys3act12BaseProcUnit7setProcEPNS0_8BaseProcE?
+0x00000071011bbdd8,BaseProcUnit::setActor,540,_ZN4ksys3act12BaseProcUnit7setProcEPNS0_8BaseProcE!
 0x00000071011bbff4,BaseProcUnit::cleanUp,512,
 0x00000071011bc1f4,BaseProcUnit::unlinkActor,412,
 0x00000071011bc390,BaseProcUnit::isParentHandleDefault,24,_ZNK4ksys3act12BaseProcUnit21isParentHandleDefaultEv
@@ -90701,7 +90701,7 @@
 0x00000071011fc9c0,OverlayArenaSystem::Struct1::dtor,52,_ZN4ksys12OverlayArenaD1Ev
 0x00000071011fc9f4,OverlayArenaSystem::Struct1::destroy,268,
 0x00000071011fcb00,OverlayArenaSystem::Struct1::dtorDelete,64,_ZN4ksys12OverlayArenaD0Ev
-0x00000071011fcb40,OverlayArena::makeHeap,384,_ZN4ksys12OverlayArena4initERKNS0_7InitArgE?
+0x00000071011fcb40,OverlayArena::makeHeap,384,_ZN4ksys12OverlayArena4initERKNS0_7InitArgE!
 0x00000071011fccc0,nullsub_4690,4,_ZN4ksys12OverlayArena7stubbedEv
 0x00000071011fccc4,sub_71011FCCC4,48,_ZN4ksys12OverlayArena11updateFlag8Eb
 0x00000071011fccf4,OverlayArenaSystem::Struct1::callResMgrClearCacheForSync,180,_ZN4ksys12OverlayArena10clearUnitsEv
@@ -96921,7 +96921,7 @@
 0x000000710136e9d4,_ZN4sead8JobQueue6FINISHENS_6CoreIdE,116,_ZN4sead8JobQueue6FINISHENS_6CoreIdE
 0x000000710136ea48,_ZN4sead11FixedSizeJQ3runEjPjPNS_6WorkerE,712,_ZN4sead11FixedSizeJQ3runEjPjPNS_6WorkerE!
 0x000000710136ed10,_ZNK4sead11FixedSizeJQ10getNumJobsEv,8,_ZNK4sead11FixedSizeJQ10getNumJobsEv
-0x000000710136ed18,_ZN4sead6WorkerC1EPNS_9WorkerMgrEjiiRKNS_14SafeStringBaseIcEE,236,_ZN4sead6WorkerC1EPNS_9WorkerMgrEjiiRKNS_14SafeStringBaseIcEE?
+0x000000710136ed18,_ZN4sead6WorkerC1EPNS_9WorkerMgrEjiiRKNS_14SafeStringBaseIcEE,236,_ZN4sead6WorkerC1EPNS_9WorkerMgrEjiiRKNS_14SafeStringBaseIcEE!
 0x000000710136ee04,_ZN4sead6Worker10clearJobQQEv,72,_ZN4sead6Worker10clearJobQQEv
 0x000000710136ee4c,sead::Worker::calc_,24,_ZN4sead6Worker5calc_El
 0x000000710136ee64,_ZN4sead6Worker12pushJobQueueEPKcPNS_8JobQueueENS_16JobQueuePushTypeE,192,_ZN4sead6Worker12pushJobQueueEPKcPNS_8JobQueueENS_16JobQueuePushTypeE

expected/_ZN4ksys3res7CounterD0Ev.bin

@@ -0,0 +1 @@
+в▓Ч

tools/check.py

@@ -11,8 +11,9 @@ import utils
 config: Dict[str, Any] = {}
 diff_settings.apply(config, {})
 
-base_elf = ELFFile((Path(__file__).parent.parent / config["baseimg"]).open("rb"))
+root = Path(__file__).parent.parent
-my_elf = ELFFile((Path(__file__).parent.parent / config["myimg"]).open("rb"))
+base_elf = ELFFile((root / config["baseimg"]).open("rb"))
+my_elf = ELFFile((root / config["myimg"]).open("rb"))
 my_symtab = my_elf.get_section_by_name(".symtab")
 if not my_symtab:
     utils.fail(f'{config["myimg"]} has no symbol table')
@@ -46,15 +47,7 @@ def get_fn_from_my_elf(name: str, size: int) -> bytes:
     return my_elf.stream.read(size)
 
 
-def check_function(addr: int, size: int, name: str) -> bool:
+def check_function_ex(addr: int, size: int, base_fn: bytes, my_fn: bytes) -> bool:
-    try:
-        base_fn = get_fn_from_base_elf(addr, size)
-    except KeyError:
-        utils.print_error(f"couldn't find base function 0x{addr:016x} for {utils.format_symbol_name_for_msg(name)}")
-        return False
-
-    my_fn = get_fn_from_my_elf(name, size)
-
     md = cs.Cs(cs.CS_ARCH_ARM64, cs.CS_MODE_ARM)
     md.detail = True
     adrp_pair_registers: Set[int] = set()
@@ -123,8 +116,23 @@ def check_function(addr: int, size: int, name: str) -> bool:
     return True
 
 
+def check_function(addr: int, size: int, name: str, base_fn=None) -> bool:
+    if base_fn is None:
+        try:
+            base_fn = get_fn_from_base_elf(addr, size)
+        except KeyError:
+            utils.print_error(f"couldn't find base function 0x{addr:016x} for {utils.format_symbol_name_for_msg(name)}")
+            return False
+
+    my_fn = get_fn_from_my_elf(name, size)
+    return check_function_ex(addr, size, base_fn, my_fn)
+
+
 def main() -> None:
     failed = False
+
+    nonmatching_fns_with_dump = {p.stem: p.read_bytes() for p in (root / "expected").glob("*.bin")}
+
     for func in utils.get_functions():
         if not func.decomp_name:
             continue
@@ -145,6 +153,12 @@ def main() -> None:
                 utils.print_note(
                     f"function {utils.format_symbol_name_for_msg(func.decomp_name)} is marked as non-matching but matches")
 
+            fn_dump = nonmatching_fns_with_dump.get(func.decomp_name, None)
+            if fn_dump is not None and not check_function(func.addr, len(fn_dump), func.decomp_name, fn_dump):
+                utils.print_error(
+                    f"function {utils.format_symbol_name_for_msg(func.decomp_name)} does not match expected output")
+                failed = True
+
     if failed:
         sys.exit(1)
 

tools/dump_function.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+
+import argparse
+from elftools.elf.elffile import ELFFile
+import diff_settings
+from pathlib import Path
+from typing import Any, Dict
+import utils
+
+config: Dict[str, Any] = {}
+diff_settings.apply(config, {})
+
+root = Path(__file__).parent.parent
+my_elf = ELFFile((root / config["myimg"]).open("rb"))
+my_symtab = my_elf.get_section_by_name(".symtab")
+if not my_symtab:
+    utils.fail(f'{config["myimg"]} has no symbol table')
+
+
+def get_file_offset(elf, addr: int) -> int:
+    for seg in elf.iter_segments():
+        if seg.header["p_type"] != "PT_LOAD":
+            continue
+        if seg["p_vaddr"] <= addr < seg["p_vaddr"] + seg["p_filesz"]:
+            return addr - seg["p_vaddr"] + seg["p_offset"]
+    assert False
+
+
+def get_symbol_file_offset_and_size(elf, table, name: str) -> (int, int):
+    syms = table.get_symbol_by_name(name)
+    if not syms or len(syms) != 1:
+        raise KeyError(name)
+    return get_file_offset(elf, syms[0]["st_value"]), syms[0]["st_size"]
+
+
+def get_fn_from_my_elf(name: str) -> bytes:
+    offset, size = get_symbol_file_offset_and_size(my_elf, my_symtab, name)
+    my_elf.stream.seek(offset)
+    return my_elf.stream.read(size)
+
+
+def dump_fn(name: str) -> None:
+    expected_dir = root / "expected"
+    try:
+        fn = get_fn_from_my_elf(name)
+        path = expected_dir / f"{name}.bin"
+        path.parent.mkdir(exist_ok=True)
+        path.write_bytes(fn)
+    except KeyError:
+        utils.fail("could not find function")
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("function_name", help="Name of the function to dump")
+    args = parser.parse_args()
+
+    dump_fn(args.function_name)
+
+
+if __name__ == "__main__":
+    main()

tools/progress.py

@@ -3,6 +3,7 @@ import argparse
 from collections import defaultdict
 from colorama import Back, Fore, Style
 import enum
+from pathlib import Path
 import utils
 from utils import FunctionStatus
 import typing as tp
@@ -14,6 +15,8 @@ parser.add_argument("--print-eq", "-e", action="store_true",
                     help="Print non-matching functions with minor issues")
 parser.add_argument("--print-ok", "-m", action="store_true",
                     help="Print matching functions")
+parser.add_argument("--hide-nonmatchings-with-dumps", "-H", help="Hide non-matching functions that have expected "
+                                                                 "output dumps", action="store_true")
 args = parser.parse_args()
 
 
@@ -31,6 +34,15 @@ counts: tp.DefaultDict[FunctionStatus, int] = defaultdict(int)
 ai_counts: tp.DefaultDict[AIClassType, int] = defaultdict(int)
 ai_counts_done: tp.DefaultDict[AIClassType, int] = defaultdict(int)
 
+nonmatching_fns_with_dump = {p.stem for p in (Path(__file__).parent.parent / "expected").glob("*.bin")}
+
+
+def should_hide_nonmatching(name: str) -> bool:
+    if not args.hide_nonmatchings_with_dumps:
+        return False
+    return name in nonmatching_fns_with_dump
+
+
 for info in utils.get_functions():
     code_size_total += info.size
     num_total += 1
@@ -57,10 +69,10 @@ for info in utils.get_functions():
     code_size[info.status] += info.size
 
     if info.status == FunctionStatus.NonMatching:
-        if args.print_nm:
+        if args.print_nm and not should_hide_nonmatching(info.decomp_name):
             print(f"{Fore.RED}NM{Fore.RESET} {utils.format_symbol_name(info.decomp_name)}")
     elif info.status == FunctionStatus.Equivalent:
-        if args.print_eq:
+        if args.print_eq and not should_hide_nonmatching(info.decomp_name):
             print(f"{Fore.YELLOW}EQ{Fore.RESET} {utils.format_symbol_name(info.decomp_name)}")
     elif info.status == FunctionStatus.Matching:
         if args.print_ok: