mirror of https://github.com/zeldaret/mm.git
Bss script update (#1601)
* port back AF's version * Update to use mapfile_parser 2 * Allow to specify other sections * hardcode map path * eprint * -r * review * review * Use fixed version * Update docs/tutorial/vscode.md Co-authored-by: Derek Hensley <hensley.derek58@gmail.com> * review * Update tools/check_reordering.py Co-authored-by: Derek Hensley <hensley.derek58@gmail.com> * update build path --------- Co-authored-by: Derek Hensley <hensley.derek58@gmail.com>
This commit is contained in:
Anghelo Carvajal
GitHub
parent
ef6323b9cd
commit
f4e8ec14e9
|
|
@ -12,7 +12,7 @@ pyelftools>=0.26
|
||||||
# diff
|
# diff
|
||||||
watchdog>=0.10.2
|
watchdog>=0.10.2
|
||||||
argcomplete
|
argcomplete
|
||||||
python-Levenshtein
|
Levenshtein
|
||||||
cxxfilt
|
cxxfilt
|
||||||
|
|
||||||
# permuter
|
# permuter
|
||||||
|
|
@ -20,4 +20,4 @@ pycparser
|
||||||
toml
|
toml
|
||||||
|
|
||||||
# map parsing
|
# map parsing
|
||||||
mapfile-parser>=1.2.1,<2.0.0
|
mapfile-parser>=2.3.5,<3.0.0
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,168 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
|
# SPDX-FileCopyrightText: © 2024 ZeldaRET
|
||||||
|
# SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
import colorama
|
||||||
|
colorama.init()
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import sys
|
||||||
|
import mapfile_parser
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
def eprint(*args, **kwargs):
|
||||||
|
print(*args, file=sys.stderr, **kwargs)
|
||||||
|
|
||||||
|
def mapPathToSource(origName: Path) -> Path:
|
||||||
|
# Try to map built path to the source path
|
||||||
|
parts = origName.parts
|
||||||
|
if parts[0] == "build":
|
||||||
|
parts = parts[1:]
|
||||||
|
|
||||||
|
path = Path(*parts)
|
||||||
|
# Assume every file in the asm folder has .s extension, while everything else has .c extension
|
||||||
|
if path.parts[0] == "asm":
|
||||||
|
path = path.with_suffix(".s")
|
||||||
|
else:
|
||||||
|
path = path.with_suffix(".c")
|
||||||
|
return path
|
||||||
|
|
||||||
|
def compareMapFiles(mapFileBuild: Path, mapFileExpected: Path, section: str=".bss", reverseCheck: bool=True) -> mapfile_parser.MapsComparisonInfo:
|
||||||
|
eprint(f"Build mapfile: {mapFileBuild}")
|
||||||
|
eprint(f"Expected mapfile: {mapFileExpected}")
|
||||||
|
eprint("")
|
||||||
|
|
||||||
|
if not mapFileBuild.exists():
|
||||||
|
eprint(f"{colorama.Fore.LIGHTRED_EX}error{colorama.Fore.RESET}: mapfile not found at {mapFileBuild}. Did you forget to build the rom?")
|
||||||
|
exit(1)
|
||||||
|
|
||||||
|
if not mapFileExpected.exists():
|
||||||
|
eprint(f"{colorama.Fore.LIGHTRED_EX}error{colorama.Fore.RESET}: expected mapfile not found at {mapFileExpected}. A mapfile under the 'expected' folder is required")
|
||||||
|
exit(1)
|
||||||
|
|
||||||
|
buildMap = mapfile_parser.MapFile()
|
||||||
|
buildMap.readMapFile(mapFileBuild)
|
||||||
|
if section != "all":
|
||||||
|
buildMap = buildMap.filterBySectionType(section)
|
||||||
|
|
||||||
|
expectedMap = mapfile_parser.MapFile()
|
||||||
|
expectedMap.readMapFile(mapFileExpected)
|
||||||
|
if section != "all":
|
||||||
|
expectedMap = expectedMap.filterBySectionType(section)
|
||||||
|
|
||||||
|
return buildMap.compareFilesAndSymbols(expectedMap, checkOtherOnSelf=reverseCheck)
|
||||||
|
|
||||||
|
|
||||||
|
def printSymbolComparison(comparisonInfo: mapfile_parser.MapsComparisonInfo, printAll = True):
|
||||||
|
print("Symbol Name,Build Address,Build File,Expected Address,Expected File,Difference,GOOD/BAD/MISSING")
|
||||||
|
|
||||||
|
# If it's bad or missing, don't need to do anything special.
|
||||||
|
# If it's good, check for if it's in a file with bad or missing stuff, and check if print all is on. If none of these, print it.
|
||||||
|
|
||||||
|
for symbolInfo in comparisonInfo.comparedList:
|
||||||
|
buildFile = symbolInfo.buildFile.filepath if symbolInfo.buildFile is not None else None
|
||||||
|
expectedFile = symbolInfo.expectedFile.filepath if symbolInfo.expectedFile is not None else None
|
||||||
|
|
||||||
|
buildFileName = ""
|
||||||
|
if buildFile is not None:
|
||||||
|
buildFileName = mapPathToSource(buildFile)
|
||||||
|
|
||||||
|
expectedFileName = ""
|
||||||
|
if expectedFile is not None:
|
||||||
|
expectedFileName = mapPathToSource(expectedFile)
|
||||||
|
|
||||||
|
symbolGood = colorama.Fore.RED + "BAD" + colorama.Fore.RESET
|
||||||
|
if symbolInfo.diff is None:
|
||||||
|
symbolGood = colorama.Fore.YELLOW + "MISSING" + colorama.Fore.RESET
|
||||||
|
print(f"{symbolInfo.symbol.name},{symbolInfo.buildAddress:X},{buildFileName},{symbolInfo.expectedAddress:X},{expectedFileName},{symbolInfo.diff},{symbolGood}")
|
||||||
|
continue
|
||||||
|
|
||||||
|
if symbolInfo.diff == 0:
|
||||||
|
symbolGood = colorama.Fore.GREEN + "GOOD" + colorama.Fore.RESET
|
||||||
|
if not buildFile in comparisonInfo.badFiles and not expectedFile in comparisonInfo.badFiles:
|
||||||
|
if not buildFile in comparisonInfo.badFiles and not expectedFile in comparisonInfo.badFiles:
|
||||||
|
if not printAll:
|
||||||
|
continue
|
||||||
|
|
||||||
|
if buildFile != expectedFile:
|
||||||
|
symbolGood += colorama.Fore.CYAN + " MOVED" + colorama.Fore.RESET
|
||||||
|
print(f"{symbolInfo.symbol.name},{symbolInfo.buildAddress:X},{buildFileName},{symbolInfo.expectedAddress:X},{expectedFileName},{symbolInfo.diff:X},{symbolGood}")
|
||||||
|
|
||||||
|
|
||||||
|
def printFileComparison(comparisonInfo: mapfile_parser.MapsComparisonInfo, fun_allowed: bool = True):
|
||||||
|
eprint("")
|
||||||
|
|
||||||
|
if len(comparisonInfo.badFiles) != 0:
|
||||||
|
print(colorama.Fore.RED + " BAD" + colorama.Style.RESET_ALL)
|
||||||
|
|
||||||
|
for file in comparisonInfo.badFiles:
|
||||||
|
eprint(f"Symbol reordering in {mapPathToSource(file.filepath)}")
|
||||||
|
eprint("")
|
||||||
|
|
||||||
|
if fun_allowed:
|
||||||
|
eprint(colorama.Fore.LIGHTWHITE_EX +
|
||||||
|
" Symbols are REORDERED!!\n"
|
||||||
|
" Oh! MY GOD!!"
|
||||||
|
+ colorama.Style.RESET_ALL)
|
||||||
|
eprint("")
|
||||||
|
|
||||||
|
if len(comparisonInfo.missingFiles) != 0:
|
||||||
|
print(colorama.Fore.YELLOW + " MISSING" + colorama.Style.RESET_ALL)
|
||||||
|
|
||||||
|
for file in comparisonInfo.missingFiles:
|
||||||
|
eprint(f"Symbols missing from {mapPathToSource(file.filepath)}")
|
||||||
|
eprint("")
|
||||||
|
|
||||||
|
if fun_allowed:
|
||||||
|
eprint(colorama.Fore.LIGHTWHITE_EX + " Error, should (not) be in here " + colorama.Style.RESET_ALL)
|
||||||
|
eprint("")
|
||||||
|
|
||||||
|
eprint("Some files appear to be missing symbols. Have they been renamed or declared as static? You may need to remake 'expected'")
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
description = "Check that globally visible symbols has not been reordered."
|
||||||
|
epilog = """\
|
||||||
|
N.B. Since this script reads the map files, it can only see globally visible symbols; in-function static symbols must be examined with other tools.
|
||||||
|
"""
|
||||||
|
|
||||||
|
parser = argparse.ArgumentParser(description=description, epilog=epilog, formatter_class=argparse.RawTextHelpFormatter)
|
||||||
|
parser.add_argument("-v", "--version", help="MM version to check. Defaults to n64-us", default="n64-us")
|
||||||
|
parser.add_argument("-a", "--print-all", help="Print all symbols of the section, not just non-matching.", action="store_true")
|
||||||
|
parser.add_argument("-n", "--no-fun-allowed", help="Remove amusing messages.", action="store_true")
|
||||||
|
parser.add_argument("-r", "--no-reverse-check", help="Disable looking for symbols on the expected map that are missing on the built map file.", action="store_true")
|
||||||
|
parser.add_argument("-s", "--section", help="Specify which section should be checked for reordered symbols. Use `all` to check all sections. Defaults to .bss", default=".bss", choices=[".text", ".data", ".rodata", ".bss", "all"])
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
mapfilePath = Path(f"build/{args.version}/mm-{args.version}.map")
|
||||||
|
mapfileExpectedPath = "expected" / mapfilePath
|
||||||
|
reverseCheck: bool = not args.no_reverse_check
|
||||||
|
section: str = args.section
|
||||||
|
|
||||||
|
comparisonInfo = compareMapFiles(mapfilePath, mapfileExpectedPath, section, reverseCheck)
|
||||||
|
printSymbolComparison(comparisonInfo, args.print_all)
|
||||||
|
|
||||||
|
if len(comparisonInfo.badFiles) + len(comparisonInfo.missingFiles) != 0:
|
||||||
|
printFileComparison(comparisonInfo, not args.no_fun_allowed)
|
||||||
|
return 1
|
||||||
|
|
||||||
|
eprint("")
|
||||||
|
eprint(colorama.Fore.GREEN + " GOOD" + colorama.Style.RESET_ALL)
|
||||||
|
|
||||||
|
if args.no_fun_allowed:
|
||||||
|
return 0
|
||||||
|
|
||||||
|
eprint("\n" + colorama.Fore.LIGHTWHITE_EX +
|
||||||
|
colorama.Back.RED + f" " + colorama.Back.RESET + "\n" +
|
||||||
|
colorama.Back.RED + f" CONGRATURATIONS! " + colorama.Back.RESET + "\n" +
|
||||||
|
colorama.Back.RED + "{:^34}".format(f"All Global {section if section != 'all' else 'Symbols'} is correct.") + colorama.Back.RESET + "\n" +
|
||||||
|
colorama.Back.RED + f" THANK YOU! " + colorama.Back.RESET + "\n" +
|
||||||
|
colorama.Back.RED + f" You are great decomper! " + colorama.Back.RESET + "\n" +
|
||||||
|
colorama.Back.RED + f" " + colorama.Style.RESET_ALL )
|
||||||
|
|
||||||
|
return 0
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
ret = main()
|
||||||
|
exit(ret)
|
||||||
|
|
@ -1,207 +0,0 @@
|
||||||
#!/usr/bin/env python3
|
|
||||||
import colorama
|
|
||||||
colorama.init()
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import os
|
|
||||||
import re
|
|
||||||
import collections
|
|
||||||
|
|
||||||
regex_fileDataEntry = re.compile(r"^\s+(?P<section>[^\s]+)\s+(?P<vram>0x[^\s]+)\s+(?P<size>0x[^\s]+)\s+(?P<name>[^\s]+)$")
|
|
||||||
regex_bssEntry = re.compile(r"^\s+(?P<vram>0x[^\s]+)\s+(?P<name>[^\s]+)$")
|
|
||||||
regex_label = re.compile(r"^(?P<name>L[0-9A-F]{8})$")
|
|
||||||
|
|
||||||
VarInfo = collections.namedtuple("VarInfo", ["file", "vram"])
|
|
||||||
|
|
||||||
def parseMapFile(mapPath: str):
|
|
||||||
with open(mapPath) as f:
|
|
||||||
mapData = f.read()
|
|
||||||
startIndex = mapData.find("..makerom")
|
|
||||||
mapData = mapData[startIndex:]
|
|
||||||
# print(len(mapData))
|
|
||||||
|
|
||||||
symbolsDict = collections.OrderedDict()
|
|
||||||
|
|
||||||
inFile = False
|
|
||||||
currentFile = ""
|
|
||||||
|
|
||||||
mapLines = mapData.split("\n")
|
|
||||||
for line in mapLines:
|
|
||||||
if inFile:
|
|
||||||
if line.startswith(" "):
|
|
||||||
entryMatch = regex_bssEntry.search(line)
|
|
||||||
|
|
||||||
# Find variable
|
|
||||||
if entryMatch is not None:
|
|
||||||
varName = entryMatch["name"]
|
|
||||||
varVram = int(entryMatch["vram"], 16)
|
|
||||||
|
|
||||||
# Filter out jump table labels
|
|
||||||
labelMatch = regex_label.search(varName)
|
|
||||||
if labelMatch is None:
|
|
||||||
symbolsDict[varName] = VarInfo( currentFile, varVram )
|
|
||||||
# print( symbolsDict[varName] )
|
|
||||||
|
|
||||||
else:
|
|
||||||
inFile = False
|
|
||||||
else:
|
|
||||||
if line.startswith(" .bss "):
|
|
||||||
inFile = False
|
|
||||||
entryMatch = regex_fileDataEntry.search(line)
|
|
||||||
|
|
||||||
# Find file
|
|
||||||
if entryMatch is not None:
|
|
||||||
name = "/".join(entryMatch["name"].split("/")[1:])
|
|
||||||
|
|
||||||
# mapfile only contains .o files, so just strip the last character to replace it
|
|
||||||
# we assume all the .c files are in the src folder, and all others are .s (true for OoT/MM)
|
|
||||||
if name.split("/")[0] == "src":
|
|
||||||
name = name[:-1] + "c"
|
|
||||||
else:
|
|
||||||
name = name[:-1] + "s"
|
|
||||||
|
|
||||||
size = int(entryMatch["size"], 16)
|
|
||||||
vram = int(entryMatch["vram"], 16)
|
|
||||||
|
|
||||||
if size > 0:
|
|
||||||
inFile = True
|
|
||||||
currentFile = name
|
|
||||||
|
|
||||||
return symbolsDict
|
|
||||||
|
|
||||||
|
|
||||||
Compared = collections.namedtuple("Compared", [ "buildAddress", "buildFile", "expectedAddress", "expectedFile", "diff"])
|
|
||||||
|
|
||||||
def compareMapFiles(mapFileBuild: str, mapFileExpected: str):
|
|
||||||
badFiles = set()
|
|
||||||
missingFiles = set()
|
|
||||||
|
|
||||||
print("Build mapfile: " + mapFileBuild, file=os.sys.stderr)
|
|
||||||
print("Expected mapfile: " + mapFileExpected, file=os.sys.stderr)
|
|
||||||
print("", file=os.sys.stderr)
|
|
||||||
|
|
||||||
if not os.path.exists(mapFileBuild):
|
|
||||||
print(f"{colorama.Fore.LIGHTRED_EX}error{colorama.Fore.RESET}: mapfile not found at {mapFileBuild}. Did you enter the correct path?", file=os.sys.stderr)
|
|
||||||
exit(1)
|
|
||||||
|
|
||||||
if not os.path.exists(mapFileExpected):
|
|
||||||
print(f"{colorama.Fore.LIGHTRED_EX}error{colorama.Fore.RESET}: expected mapfile not found at {mapFileExpected}. Is 'expected' missing or in a different folder?", file=os.sys.stderr)
|
|
||||||
exit(1)
|
|
||||||
|
|
||||||
buildMap = parseMapFile(mapFileBuild)
|
|
||||||
expectedMap = parseMapFile(mapFileExpected)
|
|
||||||
|
|
||||||
comparedDict = collections.OrderedDict()
|
|
||||||
|
|
||||||
for symbol in buildMap:
|
|
||||||
if symbol in expectedMap:
|
|
||||||
comparedDict[symbol] = Compared( buildMap[symbol].vram, buildMap[symbol].file, expectedMap[symbol].vram, expectedMap[symbol].file, buildMap[symbol].vram - expectedMap[symbol].vram )
|
|
||||||
if comparedDict[symbol].diff != 0:
|
|
||||||
badFiles.add(buildMap[symbol].file)
|
|
||||||
|
|
||||||
else:
|
|
||||||
missingFiles.add(buildMap[symbol].file)
|
|
||||||
comparedDict[symbol] = Compared( buildMap[symbol].vram, buildMap[symbol].file, -1, "", "Unknown" )
|
|
||||||
|
|
||||||
for symbol in expectedMap:
|
|
||||||
if not symbol in buildMap:
|
|
||||||
missingFiles.add(expectedMap[symbol].file)
|
|
||||||
comparedDict[symbol] = Compared( -1, "", expectedMap[symbol].vram, expectedMap[symbol].file, "Unknown" )
|
|
||||||
|
|
||||||
return badFiles, missingFiles, comparedDict
|
|
||||||
|
|
||||||
|
|
||||||
def printCsv(badFiles, missingFiles, comparedDict, printAll = True):
|
|
||||||
print("Symbol Name,Build Address,Build File,Expected Address,Expected File,Difference,GOOD/BAD/MISSING")
|
|
||||||
|
|
||||||
# If it's bad or missing, don't need to do anything special.
|
|
||||||
# If it's good, check for if it's in a file with bad or missing stuff, and check if print all is on. If none of these, print it.
|
|
||||||
|
|
||||||
for symbol in comparedDict:
|
|
||||||
symbolInfo = comparedDict[symbol]
|
|
||||||
symbolGood = colorama.Fore.RED + "BAD" + colorama.Fore.RESET
|
|
||||||
if type(symbolInfo.diff) != int:
|
|
||||||
symbolGood = colorama.Fore.YELLOW + "MISSING" + colorama.Fore.RESET
|
|
||||||
print(f"{symbol},{symbolInfo.buildAddress:X},{symbolInfo.buildFile},{symbolInfo.expectedAddress:X},{symbolInfo.expectedFile},{symbolInfo.diff},{symbolGood}")
|
|
||||||
continue
|
|
||||||
|
|
||||||
if symbolInfo.diff == 0:
|
|
||||||
symbolGood = colorama.Fore.GREEN + "GOOD" + colorama.Fore.RESET
|
|
||||||
if (not symbolInfo.buildFile in badFiles and not symbolInfo.expectedFile in badFiles) and (not symbolInfo.buildFile in badFiles and not symbolInfo.expectedFile in badFiles) and not printAll:
|
|
||||||
continue
|
|
||||||
|
|
||||||
if symbolInfo.buildFile != symbolInfo.expectedFile:
|
|
||||||
symbolGood += colorama.Fore.CYAN + " MOVED" + colorama.Fore.RESET
|
|
||||||
print(f"{symbol},{symbolInfo.buildAddress:X},{symbolInfo.buildFile},{symbolInfo.expectedAddress:X},{symbolInfo.expectedFile},{symbolInfo.diff:X},{symbolGood}")
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
|
||||||
description = "Check that globally visible bss has not been reordered."
|
|
||||||
epilog = """\
|
|
||||||
N.B. Since this script reads the map files, it can only see globally visible bss; in-function static bss must be examined with other tools.
|
|
||||||
"""
|
|
||||||
|
|
||||||
parser = argparse.ArgumentParser(description=description, epilog=epilog, formatter_class=argparse.RawTextHelpFormatter)
|
|
||||||
parser.add_argument("mapFile", help="Path to a map file.")
|
|
||||||
parser.add_argument("mapFileExpected", help="Path to the expected map file. Optional, default is 'expected/mapFile'.", nargs="?", default="")
|
|
||||||
parser.add_argument("-a", "--print-all", help="Print all bss, not just non-matching.", action="store_true")
|
|
||||||
parser.add_argument("-n", "--no-fun-allowed", help="Remove amusing messages.", action="store_true")
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
if args.mapFileExpected == "":
|
|
||||||
args.mapFileExpected = os.path.join("expected", args.mapFile)
|
|
||||||
|
|
||||||
badFiles, missingFiles, comparedDict = compareMapFiles(args.mapFile, args.mapFileExpected)
|
|
||||||
printCsv(badFiles, missingFiles, comparedDict, args.print_all)
|
|
||||||
|
|
||||||
if len(badFiles) + len(missingFiles) != 0:
|
|
||||||
print("", file=os.sys.stderr)
|
|
||||||
|
|
||||||
if len(badFiles) != 0:
|
|
||||||
print(colorama.Fore.RED + " BAD" + colorama.Style.RESET_ALL)
|
|
||||||
|
|
||||||
for file in badFiles:
|
|
||||||
print(f"bss reordering in {file}", file=os.sys.stderr)
|
|
||||||
print("", file=os.sys.stderr)
|
|
||||||
|
|
||||||
if not args.no_fun_allowed:
|
|
||||||
print(colorama.Fore.LIGHTWHITE_EX +
|
|
||||||
" BSS is REORDERED!!\n"
|
|
||||||
" Oh! MY GOD!!"
|
|
||||||
+ colorama.Style.RESET_ALL, file=os.sys.stderr)
|
|
||||||
print("", file=os.sys.stderr)
|
|
||||||
|
|
||||||
if len(missingFiles) != 0:
|
|
||||||
print(colorama.Fore.YELLOW + " MISSING" + colorama.Style.RESET_ALL)
|
|
||||||
|
|
||||||
for file in missingFiles:
|
|
||||||
print(f"Symbols missing from {file}", file=os.sys.stderr)
|
|
||||||
print("", file=os.sys.stderr)
|
|
||||||
|
|
||||||
if not args.no_fun_allowed:
|
|
||||||
print(colorama.Fore.LIGHTWHITE_EX + " Error, should (not) be in here " + colorama.Style.RESET_ALL, file=os.sys.stderr)
|
|
||||||
print("", file=os.sys.stderr)
|
|
||||||
|
|
||||||
print("Some files appear to be missing symbols. Have they been renamed or declared as static? You may need to remake 'expected'", file=os.sys.stderr)
|
|
||||||
|
|
||||||
return 1
|
|
||||||
|
|
||||||
print("", file=os.sys.stderr)
|
|
||||||
print(colorama.Fore.GREEN + " GOOD" + colorama.Style.RESET_ALL, file=os.sys.stderr)
|
|
||||||
|
|
||||||
if args.no_fun_allowed:
|
|
||||||
return 0
|
|
||||||
|
|
||||||
print("\n" + colorama.Fore.LIGHTWHITE_EX +
|
|
||||||
colorama.Back.RED + " " + colorama.Back.RESET + "\n" +
|
|
||||||
colorama.Back.RED + " CONGRATURATIONS! " + colorama.Back.RESET + "\n" +
|
|
||||||
colorama.Back.RED + " All Global BSS is correct. " + colorama.Back.RESET + "\n" +
|
|
||||||
colorama.Back.RED + " THANK YOU! " + colorama.Back.RESET + "\n" +
|
|
||||||
colorama.Back.RED + " You are great decomper! " + colorama.Back.RESET + "\n" +
|
|
||||||
colorama.Back.RED + " " + colorama.Style.RESET_ALL , file=os.sys.stderr)
|
|
||||||
|
|
||||||
return 0
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
ret = main()
|
|
||||||
exit(ret)
|
|
||||||
Loading…
Reference in New Issue