#! /usr/bin/env python3 import struct import sys import xml.etree.ElementTree as ET from dataclasses import dataclass, field from pathlib import Path from typing import Any, Dict, List, Optional, Set, Tuple import png # type: ignore import yaml as yaml_loader from n64img.image import CI4 from segtypes.n64.segment import N64Segment from util import options from util.color import unpack_color from util.n64.Yay0decompress import Yay0Decompressor sys.path.insert(0, str(Path(__file__).parent)) from sprite_common import AnimComponent, iter_in_groups, read_offset_list # TODO move into yaml PLAYER_PAL_TO_RASTER: Dict[str, int] = { "8bit": 0x57C90, "BareCake": 0x63F10, "BerryCake": 0x65D10, "Bowl": 0x63910, "Bowl_Dark": 0x63910, "Bowl_Poisoned": 0x63910, "Bowl_Shocked": 0x63910, "Burnt": 0x38390, "Burnt_Crouch": 0x0C380, "Butter": 0x63010, "CakeBatter": 0x63D90, "CakeBowl": 0x63910, "CakeMixed": 0x63A90, "CakePan": 0x63C10, "Cleanser": 0x63190, "CompleteCake": 0x63790, "Cream": 0x62D10, "Dark": 0x00B80, "Default": 0x00B80, "Egg": 0x63610, "Flour": 0x63490, "Frozen_Dark": 0x00B80, "Glove": 0x89610, "Glove_Dark": 0x89610, "Glove_Poisoned": 0x89610, "Glove_Shocked": 0x89610, "Golden": 0x00B80, "Golden_19": 0x00B80, "Golden_Dark": 0x00B80, "Golden_Poisoned": 0x00B80, "Golden_Shocked": 0x00B80, "Hammer1": 0x2C950, "Hammer1_Dark": 0x2C950, "Hammer1_Poisoned": 0x2C950, "Hammer1_Shocked": 0x2C950, "Hammer2": 0x2DE50, "Hammer2_Dark": 0x2DE50, "Hammer2_Poisoned": 0x2DE50, "Hammer2_Shocked": 0x2DE50, "Hammer3": 0x2F350, "Hammer3_Dark": 0x2F350, "Hammer3_Poisoned": 0x2F350, "Hammer3_Shocked": 0x2F350, "IcingCake": 0x65B90, "Milk": 0x63310, "Parasol": 0x8D410, "Parasol_Dark": 0x8D410, "Parasol_Poisoned": 0x8D410, "Parasol_Shocked": 0x8D410, "Peach": 0x58B10, "Peach_Dark": 0x58B10, "Peach_Poisoned": 0x58B10, "Peach_Shocked": 0x58B10, "Poisoned": 0x00B80, "Red": 0x00B80, "Red_Dark": 0x00B80, "Red_Poisoned": 0x00B80, "Red_Shocked": 0x00B80, "Salt": 0x63490, "Shocked": 0x00B80, "Strawberry": 0x62E90, "Sugar": 0x63490, "Water": 0x63310, } PLAYER_SPRITE_MEDADATA_XML_FILENAME = "player.xml" NPC_SPRITE_MEDADATA_XML_FILENAME = "npc.xml" MAX_COMPONENTS_XML = "maxComponents" PALETTE_GROUPS_XML = "paletteGroups" PALETTE_XML = "palette" BACK_PALETTE_XML = "backPalette" SPECIAL_RASTER = 0x1F880 LIST_END_BYTES = b"\xFF\xFF\xFF\xFF" @dataclass class RasterTableEntry: offset: int size: int height: int = 0 width: int = 0 raster_bytes: bytes = field(default_factory=bytes) palette: Optional[bytes] = None def write_png( self, raster_buffer: bytes, path: Path, palette: Optional[bytes] = None ): if self.height == 0 or self.width == 0: raise ValueError("Raster size has not been set") if palette is None: palette = self.palette if palette is None: raise ValueError("Palette has not been set") if self.raster_bytes is not None: self.raster_bytes = raster_buffer[ self.offset : self.offset + (self.width * self.height // 2) ] img = CI4(self.raster_bytes, self.width, self.height) img.set_palette(palette) img.write(path) @dataclass class PlayerSpriteRasterSet: start: int count: int raster_offsets: List[int] = field(default_factory=list) raster_positions: List[int] = field(default_factory=list) raster_table_entries: List[RasterTableEntry] = field(default_factory=list) loaded_position: int = 0 def add_table_entry(self, raster: RasterTableEntry): self.raster_table_entries.append(raster) self.raster_offsets.append(raster.offset) self.raster_positions.append(self.loaded_position) self.loaded_position += raster.size @dataclass class PlayerRaster: offset: int width: int height: int palette_idx: int is_special: bool @staticmethod def from_bytes( metadata: bytes, raster_set: PlayerSpriteRasterSet, palettes: List[bytes], img_idx: int, ) -> "PlayerRaster": offset, width, height, default_palette = struct.unpack(">iBBB", metadata) assert offset == raster_set.raster_positions[img_idx] is_special = raster_set.raster_offsets[img_idx] == SPECIAL_RASTER if not is_special: rte = raster_set.raster_table_entries[img_idx] rte.width = width rte.height = height rte.palette = palettes[default_palette] return PlayerRaster(offset, width, height, default_palette, is_special) @dataclass class PlayerSprite: max_components: int num_variations: int animations: List[List[AnimComponent]] palettes: List[bytes] palette_indexes: List[int] rasters: List[PlayerRaster] @staticmethod def from_bytes(data: bytes, raster_set: PlayerSpriteRasterSet) -> "PlayerSprite": raster_list_offset = int.from_bytes(data[0x0:0x4], byteorder="big") palette_list_offset = int.from_bytes(data[0x4:0x8], byteorder="big") max_components = int.from_bytes(data[0x8:0xC], "big") num_variations = int.from_bytes(data[0xC:0x10], "big") raster_offsets = read_offset_list(data[raster_list_offset:]) palette_offsets = read_offset_list(data[palette_list_offset:]) animation_offsets = read_offset_list(data[0x10:]) palettes: List[bytes] = [] palette_indexes: List[int] = [] rasters: List[PlayerRaster] = [] animations: List[List[AnimComponent]] = [] # Read palettes for i, pal_offset in enumerate(palette_offsets): pal = data[pal_offset : pal_offset + 0x20] palettes.append(pal) palette_indexes.append(i) # Read rasters for i, metadata_offset in enumerate(raster_offsets): raster = PlayerRaster.from_bytes( data[metadata_offset : metadata_offset + 7], raster_set, palettes, i, ) rasters.append(raster) # Read animations for anim_offset in animation_offsets: anim: List[AnimComponent] = [] for comp_offset in read_offset_list(data[anim_offset:]): anim.append(AnimComponent.from_bytes(data[comp_offset:], data)) animations.append(anim) return PlayerSprite( max_components, num_variations, animations, palettes, palette_indexes, rasters, ) def extract_raster_table_entries( data: bytes, raster_sets: List[PlayerSpriteRasterSet] ) -> Dict[int, RasterTableEntry]: ret: Dict[int, RasterTableEntry] = {} current_section_pos = 0 current_section = 0 for i in range(0, len(data) - 4, 4): packed = int.from_bytes(data[i : i + 4], "big") size = (packed >> 20) << 4 offset = packed & 0xFFFFF rte = RasterTableEntry(offset, size) if offset in ret: assert ret[offset].size == size else: ret[offset] = rte if current_section > len(raster_sets) - 1: continue # skip the last one, because it's weird...? raster_sets[current_section].add_table_entry(rte) current_section_pos += 1 if current_section_pos == raster_sets[current_section].count: current_section += 1 current_section_pos = 0 return ret def extract_sprites( yay0_data: bytes, raster_sets: List[PlayerSpriteRasterSet] ) -> List[PlayerSprite]: yay0_splits = [] for i in range(14): yay0_splits.append(int.from_bytes(yay0_data[i * 4 : i * 4 + 4], "big")) yay0_sprite_data = [] for i in range(0, len(yay0_splits) - 1): yay0_sprite_data.append(yay0_data[yay0_splits[i] : yay0_splits[i + 1]]) ret: List[PlayerSprite] = [] for i, yay0_piece in enumerate(yay0_sprite_data): sprite_data = Yay0Decompressor.decompress(yay0_piece, "big") sprite = PlayerSprite.from_bytes(sprite_data, raster_sets[i]) ret.append(sprite) return ret def write_player_metadata( out_path: Path, cfg: Any, raster_names: List[str], build_date: str, ) -> None: Names = ET.Element("Names") BuildDate = ET.SubElement(Names, "BuildDate") BuildDate.text = build_date Sprites = ET.SubElement(Names, "Sprites") for sprite_name in cfg: ET.SubElement( Sprites, "Sprite", name=sprite_name, ) Rasters = ET.SubElement(Names, "Rasters") for raster_name in raster_names: ET.SubElement( Rasters, "Raster", name=raster_name, ) xml = ET.ElementTree(Names) # pretty print (Python 3.9+) if hasattr(ET, "indent"): ET.indent(xml, " ") xml.write(str(out_path / PLAYER_SPRITE_MEDADATA_XML_FILENAME), encoding="unicode") def write_npc_metadata( out_path: Path, cfg: Any, ) -> None: Names = ET.Element("Names") Sprites = ET.SubElement(Names, "Sprites") for sprite_name in cfg: ET.SubElement( Sprites, "Sprite", name=sprite_name, ) xml = ET.ElementTree(Names) # pretty print (Python 3.9+) if hasattr(ET, "indent"): ET.indent(xml, " ") xml.write(str(out_path / NPC_SPRITE_MEDADATA_XML_FILENAME), encoding="unicode") def write_player_xmls( out_path: Path, cfg: Any, sprites: List[PlayerSprite], sprite_names: List[str], raster_sets: List[PlayerSpriteRasterSet], raster_table_entry_dict: Dict[int, RasterTableEntry], raster_names: List[str], ) -> None: def get_sprite_name_from_offset( offset: int, offsets: List[int], names: List[str] ) -> str: return names[offsets.index(offset)] sprite_idx = 0 num_sprite_cfgs = len(list(cfg.keys())) sprite_offsets: list[int] = list(raster_table_entry_dict.keys()) for cfg_idx in range(num_sprite_cfgs): cur_sprite_name = sprite_names[sprite_idx] cur_sprite: PlayerSprite = sprites[sprite_idx] cur_sprite_back: Optional[PlayerSprite] = None has_back = cfg[cur_sprite_name].get("has_back", False) if has_back: if cfg_idx == num_sprite_cfgs - 1: print("ERROR: Last sprite has back, but no sprite to pair with") sys.exit(1) cur_sprite_back = sprites[sprite_idx + 1] SpriteSheet = ET.Element( "SpriteSheet", { MAX_COMPONENTS_XML: str(cur_sprite.max_components), PALETTE_GROUPS_XML: str(cur_sprite.num_variations), }, ) PaletteList = ET.SubElement(SpriteSheet, "PaletteList") RasterList = ET.SubElement(SpriteSheet, "RasterList") AnimationList = ET.SubElement(SpriteSheet, "AnimationList") for i, raster in enumerate(cur_sprite.rasters): name_offset = raster_sets[sprite_idx].raster_offsets[i] raster_attributes = { "id": f"{i:X}", "name": f"{i:02X}", PALETTE_XML: f"{raster.palette_idx:X}", "src": f"{get_sprite_name_from_offset(name_offset, sprite_offsets, raster_names)}.png", } if has_back: assert cur_sprite_back is not None back_raster = cur_sprite_back.rasters[i] if back_raster.is_special: raster_attributes[ "special" ] = f"{back_raster.width & 0xFF:X},{back_raster.height & 0xFF:X}" else: back_name_offset = raster_sets[sprite_idx + 1].raster_offsets[i] raster_attributes[ "back" ] = f"{get_sprite_name_from_offset(back_name_offset, sprite_offsets, raster_names)}.png" if back_raster.palette_idx != raster.palette_idx: raster_attributes[BACK_PALETTE_XML] = f"{back_raster.palette_idx:X}" ET.SubElement(RasterList, "Raster", raster_attributes) palette_names = cfg[cur_sprite_name].get("palettes") for i, pal in enumerate(palette_names): front_only = False if isinstance(pal, str): name = pal elif isinstance(pal, dict): name = str(pal["name"]) front_only = pal.get("front_only", False) else: raise Exception("Invalid palette format for palette: " + pal) pal_attributes = { "id": f"{i:X}", "name": name, "src": name + ".png", } if front_only: pal_attributes["front_only"] = "True" ET.SubElement(PaletteList, "Palette", pal_attributes) animation_names = cfg[cur_sprite_name].get("animations") for i, components in enumerate(cur_sprite.animations): Animation = ET.SubElement( AnimationList, "Animation", { "name": animation_names[i] if animation_names else f"Anim{i:02X}", }, ) for j, comp in enumerate(components): Component = ET.SubElement( Animation, "Component", { "name": f"Comp_{j:X}", "xyz": ",".join(map(str, [comp.x, comp.y, comp.z])), }, ) for anim in comp.animations: ET.SubElement( Component, anim.__class__.__name__, anim.get_attributes() ) xml = ET.ElementTree(SpriteSheet) # pretty print (Python 3.9+) if hasattr(ET, "indent"): ET.indent(xml, " ") xml.write(str(out_path / f"{cur_sprite_name}.xml"), encoding="unicode") if has_back: sprite_idx += 2 else: sprite_idx += 1 def write_player_rasters( out_path: Path, raster_table_entry_dict: Dict[int, RasterTableEntry], raster_data: bytes, raster_names: List[str], ) -> None: base_path = out_path / "rasters" base_path.mkdir(parents=True, exist_ok=True) for i, (offset, rte) in enumerate(raster_table_entry_dict.items()): if offset == SPECIAL_RASTER: with open(base_path / f"{raster_names[i]}.png", "wb") as f: f.write(b"\x00" * 0x10) continue # Last raster... weird for some reason if offset == 0x9CD50: with open(base_path / f"{raster_names[i]}.png", "wb") as f: f.write(b"\x00" * 0x10) continue rte.write_png(raster_data, base_path / f"{raster_names[i]}.png") def write_player_palettes( out_path: Path, cfg: Any, sprites: List[PlayerSprite], sprite_names: List[str], raster_table_entry_dict: Dict[int, RasterTableEntry], raster_data: bytes, ) -> None: dumped_palettes: Set[str] = set() for i, sprite in enumerate(sprites): sprite_name = sprite_names[i] path = out_path / "palettes" path.mkdir(parents=True, exist_ok=True) for i, palette in enumerate(sprite.palettes): pal = cfg[sprite_name]["palettes"][sprite.palette_indexes[i]] if isinstance(pal, str): pal_name = pal elif isinstance(pal, dict): pal_name = str(pal["name"]) else: raise Exception("Invalid palette format for palette: " + pal) if pal_name not in dumped_palettes: offset = PLAYER_PAL_TO_RASTER[pal_name] if pal_name not in PLAYER_PAL_TO_RASTER: print( f"WARNING: Palette {pal_name} has no specified raster, not dumping!" ) raster_table_entry_dict[offset].write_png( raster_data, path / (pal_name + ".png"), palette ) ########### ### NPC ### ########### @dataclass class NpcRaster: width: int height: int palette_index: int raster: bytearray @staticmethod def from_bytes(data, sprite_data) -> "NpcRaster": raster_offset = int.from_bytes(data[0:4], byteorder="big") width = data[4] & 0xFF height = data[5] & 0xFF palette_index = data[6] assert data[7] == 0xFF # CI4 raster = bytearray() for i in range(width * height // 2): raster.append(sprite_data[raster_offset + i] >> 4) raster.append(sprite_data[raster_offset + i] & 0xF) return NpcRaster(width, height, palette_index, raster) def write(self, path, palette): w = png.Writer(self.width, self.height, palette=palette) with open(path, "wb") as f: w.write_array(f, self.raster) @dataclass class NpcSprite: max_components: int num_variations: int animations: List[List[AnimComponent]] palettes: List[List[Tuple[int, int, int, int]]] images: List[NpcRaster] image_names: List[str] = field(default_factory=list) palette_names: List[str] = field(default_factory=list) animation_names: List[str] = field(default_factory=list) variation_names: List[str] = field(default_factory=list) @staticmethod def from_bytes(data: bytearray): image_offsets = read_offset_list( data[int.from_bytes(data[0:4], byteorder="big") :] ) palette_offsets = read_offset_list( data[int.from_bytes(data[4:8], byteorder="big") :] ) max_components = int.from_bytes(data[8:0xC], byteorder="big") num_variations = int.from_bytes(data[0xC:0x10], byteorder="big") animation_offsets = read_offset_list(data[0x10:]) palettes = [] for offset in palette_offsets: # 16 colors color_data = data[offset : offset + 16 * 2] palettes.append([unpack_color(c) for c in iter_in_groups(color_data, 2)]) images = [] for offset in image_offsets: img = NpcRaster.from_bytes(data[offset:], data) images.append(img) animations = [] for offset in animation_offsets: anim = [] for comp_offset in read_offset_list(data[offset:]): comp = AnimComponent.from_bytes(data[comp_offset:], data) anim.append(comp) animations.append(anim) return NpcSprite(max_components, num_variations, animations, palettes, images) def write_to_dir(self, path): if len(self.variation_names) > 1: SpriteSheet = ET.Element( "SpriteSheet", { MAX_COMPONENTS_XML: str(self.max_components), PALETTE_GROUPS_XML: str(self.num_variations), "variations": ",".join(self.variation_names), }, ) else: SpriteSheet = ET.Element( "SpriteSheet", { MAX_COMPONENTS_XML: str(self.max_components), PALETTE_GROUPS_XML: str(self.num_variations), }, ) PaletteList = ET.SubElement(SpriteSheet, "PaletteList") RasterList = ET.SubElement(SpriteSheet, "RasterList") AnimationList = ET.SubElement(SpriteSheet, "AnimationList") palette_to_raster = {} for i, image in enumerate(self.images): name = self.image_names[i] if self.image_names else f"Raster{i:02X}" image.write(path / (name + ".png"), self.palettes[image.palette_index]) if image.palette_index not in palette_to_raster: palette_to_raster[image.palette_index] = [] palette_to_raster[image.palette_index].append(image) ET.SubElement( RasterList, "Raster", { "id": f"{i:X}", "palette": f"{image.palette_index:X}", "src": name + ".png", }, ) for i, palette in enumerate(self.palettes): name = ( self.palette_names[i] if (self.palette_names and i < len(self.palette_names)) else f"Pal{i:02X}" ) if i in palette_to_raster: img = palette_to_raster[i][0] else: img = self.images[0] img.write(path / (name + ".png"), palette) ET.SubElement( PaletteList, "Palette", { "id": f"{i:X}", "src": name + ".png", }, ) for i, components in enumerate(self.animations): Animation = ET.SubElement( AnimationList, "Animation", { "name": self.animation_names[i] if self.animation_names else f"Anim{i:02X}", }, ) for j, comp in enumerate(components): Component = ET.SubElement( Animation, "Component", { "name": f"Comp_{j:X}", "xyz": ",".join(map(str, [comp.x, comp.y, comp.z])), }, ) for anim in comp.animations: ET.SubElement( Component, anim.__class__.__name__, anim.get_attributes() ) xml = ET.ElementTree(SpriteSheet) # pretty print (Python 3.9+) if hasattr(ET, "indent"): ET.indent(xml, " ") xml.write(str(path / "SpriteSheet.xml"), encoding="unicode") class N64SegPm_sprites(N64Segment): DEFAULT_NPC_SPRITE_NAMES = [f"{i:02X}" for i in range(0xEA)] def __init__(self, rom_start, rom_end, type, name, vram_start, args, yaml) -> None: super().__init__( rom_start, rom_end, type, name, vram_start, args=args, yaml=yaml ) with (Path(__file__).parent / f"npc_sprite_names.yaml").open("r") as f: self.npc_cfg = yaml_loader.load(f.read(), Loader=yaml_loader.SafeLoader) with (Path(__file__).parent / f"player_sprite_names.yaml").open("r") as f: self.player_cfg = yaml_loader.load(f.read(), Loader=yaml_loader.SafeLoader) def out_path(self): return options.opts.asset_path / "sprite" / "sprites" def split_player( self, build_date: str, player_raster_data: bytes, player_yay0_data: bytes ) -> None: player_sprite_cfg = self.player_cfg["player_sprites"] player_raster_names: List[str] = self.player_cfg["player_rasters"] player_sprite_names = [] for sprite_name in player_sprite_cfg.keys(): player_sprite_names.append(sprite_name) if player_sprite_cfg[sprite_name].get("has_back", False): player_sprite_names.append(sprite_name) # Header parsing index_ranges_offset = int.from_bytes(player_raster_data[0:0x4], "big") raster_info_offset = int.from_bytes(player_raster_data[0x4:0x8], "big") ci4_raster_data_offset = int.from_bytes(player_raster_data[0x8:0xC], "big") index_ranges = player_raster_data[index_ranges_offset:raster_info_offset] raster_info = player_raster_data[raster_info_offset:ci4_raster_data_offset] # ci4_raster_data = player_raster_data[ci4_raster_data_offset:] # Parse raster sets (readSpriteSections) raster_sets: List[PlayerSpriteRasterSet] = [] for i in range(0, len(index_ranges) - 4, 4): start = int.from_bytes(index_ranges[i : i + 4], "big") end = int.from_bytes(index_ranges[i + 4 : i + 8], "big") raster_sets.append(PlayerSpriteRasterSet(start, end - start)) raster_table_entry_dict = extract_raster_table_entries(raster_info, raster_sets) player_sprites = extract_sprites(player_yay0_data, raster_sets) ######### # Writing ######### player_out_path = self.out_path().parent / "player" player_out_path.mkdir(parents=True, exist_ok=True) write_player_metadata( self.out_path().parent, player_sprite_cfg, player_raster_names, build_date, ) write_player_xmls( player_out_path, player_sprite_cfg, player_sprites, player_sprite_names, raster_sets, raster_table_entry_dict, player_raster_names, ) write_player_rasters( player_out_path, raster_table_entry_dict, player_raster_data, player_raster_names, ) write_player_palettes( player_out_path, player_sprite_cfg, player_sprites, player_sprite_names, raster_table_entry_dict, player_raster_data, ) def split_npc(self, data: bytes) -> None: out_dir = self.out_path().parent / "npc" write_npc_metadata( self.out_path().parent, self.npc_cfg, ) for i, sprite_name in enumerate(self.npc_cfg): sprite_dir = out_dir / sprite_name sprite_dir.mkdir(parents=True, exist_ok=True) start = int.from_bytes(data[i * 4 : (i + 1) * 4], byteorder="big") end = int.from_bytes(data[(i + 1) * 4 : (i + 2) * 4], byteorder="big") sprite_data = Yay0Decompressor.decompress(data[start:end], "big") sprite = NpcSprite.from_bytes(sprite_data) sprite.image_names = self.npc_cfg[sprite_name].get("frames", []) sprite.palette_names = self.npc_cfg[sprite_name].get("palettes", []) sprite.animation_names = self.npc_cfg[sprite_name].get("animations", []) sprite.variation_names = self.npc_cfg[sprite_name].get("variations", []) sprite.write_to_dir(sprite_dir) def split(self, rom_bytes) -> None: sprite_in_bytes = rom_bytes[self.rom_start : self.rom_end] build_date = sprite_in_bytes[0:0x10].decode("ascii").rstrip("\0") player_raster_offset = int.from_bytes(sprite_in_bytes[0x10:0x14], "big") + 0x10 player_yay0_offset = int.from_bytes(sprite_in_bytes[0x14:0x18], "big") + 0x10 npc_yay0_offset = int.from_bytes(sprite_in_bytes[0x18:0x1C], "big") + 0x10 sprite_end_offset = int.from_bytes(sprite_in_bytes[0x1C:0x20], "big") + 0x10 player_raster_data: bytes = sprite_in_bytes[ player_raster_offset:player_yay0_offset ] player_yay0_data: bytes = sprite_in_bytes[player_yay0_offset:npc_yay0_offset] npc_yay0_data: bytes = sprite_in_bytes[npc_yay0_offset:sprite_end_offset] self.split_player(build_date, player_raster_data, player_yay0_data) self.split_npc(npc_yay0_data) def get_linker_entries(self): from segtypes.linker_entry import LinkerEntry src_paths = [options.opts.asset_path / "sprite"] # for NPC src_paths += [ options.opts.asset_path / "sprite" / "npc" / sprite_name for sprite_name in self.npc_cfg ] return [ LinkerEntry(self, src_paths, self.out_path(), self.get_linker_section()) ] def cache(self): return (self.yaml, self.rom_end, self.player_cfg, self.npc_cfg)