From 194c99d80b07cdc4b4e2ae92fe0cee18a22615ec Mon Sep 17 00:00:00 2001 From: Anghelo Carvajal Date: Sun, 16 Jan 2022 15:30:31 -0300 Subject: [PATCH] Subrepos update (#613) * a * git subrepo pull tools/asm-differ --force subrepo: subdir: "tools/asm-differ" merged: "6f8f80b71" upstream: origin: "https://github.com/simonlindholm/asm-differ" branch: "main" commit: "6f8f80b71" git-subrepo: version: "0.4.3" origin: "https://github.com/ingydotnet/git-subrepo.git" commit: "2f68596" * git subrepo pull tools/z64compress --force subrepo: subdir: "tools/z64compress" merged: "9e7a6dbfa" upstream: origin: "https://github.com/z64me/z64compress.git" branch: "main" commit: "9e7a6dbfa" git-subrepo: version: "0.4.3" origin: "https://github.com/ingydotnet/git-subrepo.git" commit: "2f68596" * git subrepo pull tools/ZAPD --force subrepo: subdir: "tools/ZAPD" merged: "bf16ff7c4" upstream: origin: "https://github.com/zeldaret/ZAPD.git" branch: "master" commit: "bf16ff7c4" git-subrepo: version: "0.4.3" origin: "https://github.com/ingydotnet/git-subrepo.git" commit: "2f68596" * Revert "git subrepo pull tools/z64compress --force" This reverts commit a7cc87394ed4afb5ebe1f2318526545b6bbe205d. --- tools/ZAPD/.gitrepo | 4 +- tools/ZAPD/Jenkinsfile | 2 +- tools/ZAPD/ZAPD/Declaration.cpp | 2 +- tools/ZAPD/ZAPD/Main.cpp | 4 + tools/ZAPD/ZAPD/ZArray.cpp | 13 ++- tools/ZAPD/ZAPD/ZCollision.cpp | 89 ++++++++++++------- tools/ZAPD/ZAPD/ZCollision.h | 3 +- tools/ZAPD/ZAPD/ZPointer.cpp | 57 ++++++++++++ tools/ZAPD/ZAPD/ZPointer.h | 22 +++++ tools/ZAPD/ZAPD/ZResource.h | 1 + tools/ZAPD/ZAPD/ZTexture.cpp | 53 +++++++---- tools/ZAPD/ZAPD/ZTexture.h | 1 + .../docs/zapd_extraction_xml_reference.md | 25 +++++- tools/asm-differ/.gitrepo | 6 +- tools/asm-differ/diff.py | 48 +++++----- 15 files changed, 240 insertions(+), 90 deletions(-) create mode 100644 tools/ZAPD/ZAPD/ZPointer.cpp create mode 100644 tools/ZAPD/ZAPD/ZPointer.h diff --git a/tools/ZAPD/.gitrepo b/tools/ZAPD/.gitrepo index ccd6c7d649..7576be8e0c 100644 --- a/tools/ZAPD/.gitrepo +++ b/tools/ZAPD/.gitrepo @@ -6,7 +6,7 @@ [subrepo] remote = https://github.com/zeldaret/ZAPD.git branch = master - commit = fd5a7f434171a33b1b3eb2a3e112fdcee6d9262d - parent = 844c24e39c96b7a3f3a40035058bda6c91ed4bbf + commit = bf16ff7c4c409358a53793b72260b1d8e474cf0c + parent = a7cc87394ed4afb5ebe1f2318526545b6bbe205d method = merge cmdver = 0.4.3 diff --git a/tools/ZAPD/Jenkinsfile b/tools/ZAPD/Jenkinsfile index ec9b56ac5a..9c4dde7122 100644 --- a/tools/ZAPD/Jenkinsfile +++ b/tools/ZAPD/Jenkinsfile @@ -61,7 +61,7 @@ pipeline { sh 'cp ../ZAPD.out tools/ZAPD/' sh 'python3 tools/fixbaserom.py' sh 'python3 tools/extract_baserom.py' - sh 'python3 extract_assets.py -t$(nproc)' + sh 'python3 extract_assets.py -j $(nproc)' } } } diff --git a/tools/ZAPD/ZAPD/Declaration.cpp b/tools/ZAPD/ZAPD/Declaration.cpp index be06b0bea7..34ab953517 100644 --- a/tools/ZAPD/ZAPD/Declaration.cpp +++ b/tools/ZAPD/ZAPD/Declaration.cpp @@ -171,7 +171,7 @@ std::string Declaration::GetExternalDeclarationStr() const std::string Declaration::GetExternStr() const { - if (IsStatic() || varType == "") + if (IsStatic() || varType == "" || isUnaccounted) { return ""; } diff --git a/tools/ZAPD/ZAPD/Main.cpp b/tools/ZAPD/ZAPD/Main.cpp index 048338fb70..531edc0946 100644 --- a/tools/ZAPD/ZAPD/Main.cpp +++ b/tools/ZAPD/ZAPD/Main.cpp @@ -434,6 +434,10 @@ void BuildAssetTexture(const fs::path& pngFilePath, TextureType texType, const f std::string name = outPath.stem().string(); ZTexture tex(nullptr); + + if (name.find("u32") != std::string::npos) + tex.dWordAligned = false; + tex.FromPNG(pngFilePath.string(), texType); std::string cfgPath = StringHelper::Split(pngFilePath.string(), ".")[0] + ".cfg"; diff --git a/tools/ZAPD/ZAPD/ZArray.cpp b/tools/ZAPD/ZAPD/ZArray.cpp index ebfb13ee74..b6d9e50e50 100644 --- a/tools/ZAPD/ZAPD/ZArray.cpp +++ b/tools/ZAPD/ZAPD/ZArray.cpp @@ -101,11 +101,18 @@ std::string ZArray::GetBodySourceCode() const const auto& res = resList[i]; output += "\t"; - if (res->GetResourceType() == ZResourceType::Scalar || - res->GetResourceType() == ZResourceType::Vertex) + switch (res->GetResourceType()) + { + case ZResourceType::Pointer: + case ZResourceType::Scalar: + case ZResourceType::Vertex: output += resList.at(i)->GetBodySourceCode(); - else + break; + + default: output += StringHelper::Sprintf("{ %s }", resList.at(i)->GetBodySourceCode().c_str()); + break; + } if (i < arrayCnt - 1 || res->IsExternalResource()) output += ",\n"; diff --git a/tools/ZAPD/ZAPD/ZCollision.cpp b/tools/ZAPD/ZAPD/ZCollision.cpp index 1dfa46b1dd..6554005eca 100644 --- a/tools/ZAPD/ZAPD/ZCollision.cpp +++ b/tools/ZAPD/ZAPD/ZCollision.cpp @@ -1,5 +1,6 @@ #include "ZCollision.h" +#include #include #include @@ -76,9 +77,42 @@ void ZCollisionHeader::ParseRawData() polygonTypes.push_back( BitConverter::ToUInt64BE(rawData, polyTypeDefSegmentOffset + (i * 8))); - if (camDataAddress != 0) - camData = new CameraDataList(parent, name, rawData, camDataSegmentOffset, - polyTypeDefSegmentOffset, polygonTypes.size()); + if (camDataAddress != SEGMENTED_NULL) + { + // Try to guess how many elements the CamDataList array has. + // The "guessing algorithm" is basically a "best effort" one and it + // is error-prone. + // This is based mostly on observation of how CollisionHeader data is + // usually ordered. If for some reason the data was in some other funny + // order, this would probably break. + // The most common ordering is: + // - *CamData* + // - SurfaceType + // - CollisionPoly + // - Vertices + // - WaterBoxes + // - CollisionHeader + offset_t upperCameraBoundary = polyTypeDefSegmentOffset; + if (upperCameraBoundary == 0) + { + upperCameraBoundary = polySegmentOffset; + } + if (upperCameraBoundary == 0) + { + upperCameraBoundary = vtxSegmentOffset; + } + if (upperCameraBoundary == 0) + { + upperCameraBoundary = waterBoxSegmentOffset; + } + if (upperCameraBoundary == 0) + { + upperCameraBoundary = rawDataIndex; + } + + camData = + new CameraDataList(parent, name, rawData, camDataSegmentOffset, upperCameraBoundary); + } for (uint16_t i = 0; i < numWaterBoxes; i++) waterBoxes.push_back(WaterBoxHeader( @@ -106,8 +140,7 @@ void ZCollisionHeader::DeclareReferences(const std::string& prefix) parent->AddDeclarationArray( waterBoxSegmentOffset, DeclarationAlignment::Align4, 16 * waterBoxes.size(), "WaterBox", - StringHelper::Sprintf("%s_waterBoxes_%06X", auxName.c_str(), waterBoxSegmentOffset), - waterBoxes.size(), declaration); + StringHelper::Sprintf("%sWaterBoxes", auxName.c_str()), waterBoxes.size(), declaration); } if (polygons.size() > 0) @@ -126,8 +159,7 @@ void ZCollisionHeader::DeclareReferences(const std::string& prefix) parent->AddDeclarationArray( polySegmentOffset, DeclarationAlignment::Align4, polygons.size() * 16, "CollisionPoly", - StringHelper::Sprintf("%s_polygons_%08X", auxName.c_str(), polySegmentOffset), - polygons.size(), declaration); + StringHelper::Sprintf("%sPolygons", auxName.c_str()), polygons.size(), declaration); } declaration.clear(); @@ -141,11 +173,10 @@ void ZCollisionHeader::DeclareReferences(const std::string& prefix) } if (polyTypeDefAddress != 0) - parent->AddDeclarationArray( - polyTypeDefSegmentOffset, DeclarationAlignment::Align4, polygonTypes.size() * 8, - "SurfaceType", - StringHelper::Sprintf("%s_surfaceType_%08X", auxName.c_str(), polyTypeDefSegmentOffset), - polygonTypes.size(), declaration); + parent->AddDeclarationArray(polyTypeDefSegmentOffset, DeclarationAlignment::Align4, + polygonTypes.size() * 8, "SurfaceType", + StringHelper::Sprintf("%sSurfaceType", auxName.c_str()), + polygonTypes.size(), declaration); declaration.clear(); @@ -167,8 +198,7 @@ void ZCollisionHeader::DeclareReferences(const std::string& prefix) parent->AddDeclarationArray( vtxSegmentOffset, first.GetDeclarationAlignment(), vertices.size() * first.GetRawDataSize(), first.GetSourceTypeName(), - StringHelper::Sprintf("%s_vtx_%08X", auxName.c_str(), vtxSegmentOffset), - vertices.size(), declaration); + StringHelper::Sprintf("%sVertices", auxName.c_str()), vertices.size(), declaration); } } @@ -183,11 +213,11 @@ std::string ZCollisionHeader::GetBodySourceCode() const std::string vtxName; Globals::Instance->GetSegmentedPtrName(vtxAddress, parent, "Vec3s", vtxName); - declaration += StringHelper::Sprintf("\t%i,\n\t%s,\n", numVerts, vtxName.c_str()); + declaration += StringHelper::Sprintf("\t%i, %s,\n", numVerts, vtxName.c_str()); std::string polyName; Globals::Instance->GetSegmentedPtrName(polyAddress, parent, "CollisionPoly", polyName); - declaration += StringHelper::Sprintf("\t%i,\n\t%s,\n", numPolygons, polyName.c_str()); + declaration += StringHelper::Sprintf("\t%i, %s,\n", numPolygons, polyName.c_str()); std::string surfaceName; Globals::Instance->GetSegmentedPtrName(polyTypeDefAddress, parent, "SurfaceType", surfaceName); @@ -199,7 +229,7 @@ std::string ZCollisionHeader::GetBodySourceCode() const std::string waterBoxName; Globals::Instance->GetSegmentedPtrName(waterBoxAddress, parent, "WaterBox", waterBoxName); - declaration += StringHelper::Sprintf("\t%i,\n\t%s\n", numWaterBoxes, waterBoxName.c_str()); + declaration += StringHelper::Sprintf("\t%i, %s\n", numWaterBoxes, waterBoxName.c_str()); return declaration; } @@ -261,16 +291,17 @@ std::string WaterBoxHeader::GetBodySourceCode() const } CameraDataList::CameraDataList(ZFile* parent, const std::string& prefix, - const std::vector& rawData, uint32_t rawDataIndex, - uint32_t polyTypeDefSegmentOffset, - [[maybe_unused]] uint32_t polygonTypesCnt) + const std::vector& rawData, offset_t rawDataIndex, + offset_t upperCameraBoundary) { std::string declaration; // Parse CameraDataEntries - int32_t numElements = (polyTypeDefSegmentOffset - rawDataIndex) / 8; - uint32_t cameraPosDataSeg = rawDataIndex; - for (int32_t i = 0; i < numElements; i++) + size_t numElements = (upperCameraBoundary - rawDataIndex) / 8; + assert(numElements < 10000); + + offset_t cameraPosDataSeg = rawDataIndex; + for (size_t i = 0; i < numElements; i++) { CameraDataEntry* entry = new CameraDataEntry(); @@ -302,8 +333,7 @@ CameraDataList::CameraDataList(ZFile* parent, const std::string& prefix, { int32_t index = ((entries[i]->cameraPosDataSeg & 0x00FFFFFF) - cameraPosDataOffset) / 0x6; - sprintf(camSegLine, "&%s_camPosData_%08X[%i]", prefix.c_str(), cameraPosDataOffset, - index); + sprintf(camSegLine, "&%sCamPosData[%i]", prefix.c_str(), index); } else sprintf(camSegLine, "NULL"); @@ -318,7 +348,7 @@ CameraDataList::CameraDataList(ZFile* parent, const std::string& prefix, parent->AddDeclarationArray( rawDataIndex, DeclarationAlignment::Align4, entries.size() * 8, "CamData", - StringHelper::Sprintf("%s_camDataList_%08X", prefix.c_str(), rawDataIndex), entries.size(), + StringHelper::Sprintf("%sCamDataList", prefix.c_str(), rawDataIndex), entries.size(), declaration); uint32_t numDataTotal = (rawDataIndex - cameraPosDataOffset) / 0x6; @@ -339,10 +369,9 @@ CameraDataList::CameraDataList(ZFile* parent, const std::string& prefix, int32_t cameraPosDataIndex = GETSEGOFFSET(cameraPosDataSeg); uint32_t entrySize = numDataTotal * 0x6; - parent->AddDeclarationArray( - cameraPosDataIndex, DeclarationAlignment::Align4, entrySize, "Vec3s", - StringHelper::Sprintf("%s_camPosData_%08X", prefix.c_str(), cameraPosDataIndex), - numDataTotal, declaration); + parent->AddDeclarationArray(cameraPosDataIndex, DeclarationAlignment::Align4, entrySize, + "Vec3s", StringHelper::Sprintf("%sCamPosData", prefix.c_str()), + numDataTotal, declaration); } } diff --git a/tools/ZAPD/ZAPD/ZCollision.h b/tools/ZAPD/ZAPD/ZCollision.h index d0325bfb95..222fd25f9a 100644 --- a/tools/ZAPD/ZAPD/ZCollision.h +++ b/tools/ZAPD/ZAPD/ZCollision.h @@ -55,8 +55,7 @@ public: std::vector cameraPositionData; CameraDataList(ZFile* parent, const std::string& prefix, const std::vector& rawData, - uint32_t rawDataIndex, uint32_t polyTypeDefSegmentOffset, - uint32_t polygonTypesCnt); + offset_t rawDataIndex, offset_t upperCameraBoundary); ~CameraDataList(); }; diff --git a/tools/ZAPD/ZAPD/ZPointer.cpp b/tools/ZAPD/ZAPD/ZPointer.cpp new file mode 100644 index 0000000000..6e6945cada --- /dev/null +++ b/tools/ZAPD/ZAPD/ZPointer.cpp @@ -0,0 +1,57 @@ +#include "ZPointer.h" + +#include "Globals.h" +#include "Utils/BitConverter.h" +#include "Utils/StringHelper.h" +#include "WarningHandler.h" +#include "ZFile.h" + +REGISTER_ZFILENODE(Pointer, ZPointer); + +ZPointer::ZPointer(ZFile* nParent) : ZResource(nParent) +{ + RegisterRequiredAttribute("Type"); +} + +void ZPointer::ParseXML(tinyxml2::XMLElement* reader) +{ + ZResource::ParseXML(reader); + + type = registeredAttributes.at("Type").value; +} + +void ZPointer::ParseRawData() +{ + auto& rawData = parent->GetRawData(); + + ptr = BitConverter::ToUInt32BE(rawData, rawDataIndex); +} + +std::string ZPointer::GetBodySourceCode() const +{ + std::string ptrName; + + Globals::Instance->GetSegmentedPtrName(ptr, parent, "", ptrName); + + return ptrName; +} + +bool ZPointer::DoesSupportArray() const +{ + return true; +} + +std::string ZPointer::GetSourceTypeName() const +{ + return type + "*"; +} + +ZResourceType ZPointer::GetResourceType() const +{ + return ZResourceType::Pointer; +} + +size_t ZPointer::GetRawDataSize() const +{ + return 0x04; +} diff --git a/tools/ZAPD/ZAPD/ZPointer.h b/tools/ZAPD/ZAPD/ZPointer.h new file mode 100644 index 0000000000..8fb5889339 --- /dev/null +++ b/tools/ZAPD/ZAPD/ZPointer.h @@ -0,0 +1,22 @@ +#pragma once + +#include "ZResource.h" + +class ZPointer : public ZResource +{ +public: + segptr_t ptr = SEGMENTED_NULL; + std::string type; + + ZPointer(ZFile* nParent); + + void ParseXML(tinyxml2::XMLElement* reader) override; + void ParseRawData() override; + std::string GetBodySourceCode() const override; + + bool DoesSupportArray() const override; + std::string GetSourceTypeName() const override; + ZResourceType GetResourceType() const override; + + size_t GetRawDataSize() const override; +}; diff --git a/tools/ZAPD/ZAPD/ZResource.h b/tools/ZAPD/ZAPD/ZResource.h index 4dad398955..c65c3c31b4 100644 --- a/tools/ZAPD/ZAPD/ZResource.h +++ b/tools/ZAPD/ZAPD/ZResource.h @@ -38,6 +38,7 @@ enum class ZResourceType Mtx, Path, PlayerAnimationData, + Pointer, Room, RoomCommand, Scalar, diff --git a/tools/ZAPD/ZAPD/ZTexture.cpp b/tools/ZAPD/ZAPD/ZTexture.cpp index 7bd31438b4..0d72d9c5a6 100644 --- a/tools/ZAPD/ZAPD/ZTexture.cpp +++ b/tools/ZAPD/ZAPD/ZTexture.cpp @@ -16,6 +16,7 @@ ZTexture::ZTexture(ZFile* nParent) : ZResource(nParent) { width = 0; height = 0; + dWordAligned = true; RegisterRequiredAttribute("Width"); RegisterRequiredAttribute("Height"); @@ -105,10 +106,7 @@ void ZTexture::ParseXML(tinyxml2::XMLElement* reader) void ZTexture::ParseRawData() { if (rawDataIndex % 8 != 0) - { - HANDLE_WARNING_RESOURCE(WarningType::NotImplemented, parent, this, rawDataIndex, - "this texture is not 64-bit aligned", ""); - } + dWordAligned = false; switch (format) { @@ -724,7 +722,12 @@ void ZTexture::Save(const fs::path& outFolder) if (!Directory::Exists(outPath.string())) Directory::CreateDirectory(outPath.string()); - auto outFileName = outPath / (outName + "." + GetExternalExtension() + ".png"); + std::filesystem::__cxx11::path outFileName; + + if (!dWordAligned) + outFileName = outPath / (outName + ".u32" + "." + GetExternalExtension() + ".png"); + else + outFileName = outPath / (outName + +"." + GetExternalExtension() + ".png"); #ifdef TEXTURE_DEBUG printf("Saving PNG: %s\n", outFileName.c_str()); @@ -745,7 +748,7 @@ Declaration* ZTexture::DeclareVar(const std::string& prefix, { std::string auxName = name; std::string auxOutName = outName; - + std::string incStr; if (auxName == "") auxName = GetDefaultName(prefix); @@ -754,8 +757,12 @@ Declaration* ZTexture::DeclareVar(const std::string& prefix, auto filepath = Globals::Instance->outputPath / fs::path(auxOutName).stem(); - std::string incStr = - StringHelper::Sprintf("%s.%s.inc.c", filepath.c_str(), GetExternalExtension().c_str()); + if (dWordAligned) + incStr = + StringHelper::Sprintf("%s.%s.inc.c", filepath.c_str(), GetExternalExtension().c_str()); + else + incStr = StringHelper::Sprintf("%s.u32.%s.inc.c", filepath.c_str(), + GetExternalExtension().c_str()); if (!Globals::Instance->cfg.texturePool.empty()) { @@ -765,13 +772,19 @@ Declaration* ZTexture::DeclareVar(const std::string& prefix, const auto& poolEntry = Globals::Instance->cfg.texturePool.find(hash); if (poolEntry != Globals::Instance->cfg.texturePool.end()) { - incStr = StringHelper::Sprintf("%s.%s.inc.c", poolEntry->second.path.c_str(), - GetExternalExtension().c_str()); + if (dWordAligned) + incStr = StringHelper::Sprintf("%s.%s.inc.c", poolEntry->second.path.c_str(), + GetExternalExtension().c_str()); + else + incStr = StringHelper::Sprintf("%s.u32.%s.inc.c", poolEntry->second.path.c_str(), + GetExternalExtension().c_str()); } } + size_t texSizeDivisor = (dWordAligned) ? 8 : 4; - Declaration* decl = parent->AddDeclarationIncludeArray( - rawDataIndex, incStr, GetRawDataSize(), GetSourceTypeName(), auxName, GetRawDataSize() / 8); + Declaration* decl = parent->AddDeclarationIncludeArray(rawDataIndex, incStr, GetRawDataSize(), + GetSourceTypeName(), auxName, + GetRawDataSize() / texSizeDivisor); decl->staticConf = staticConf; return decl; } @@ -779,15 +792,17 @@ Declaration* ZTexture::DeclareVar(const std::string& prefix, std::string ZTexture::GetBodySourceCode() const { std::string sourceOutput; - - for (size_t i = 0; i < textureDataRaw.size(); i += 8) + size_t texSizeInc = (dWordAligned) ? 8 : 4; + for (size_t i = 0; i < textureDataRaw.size(); i += texSizeInc) { if (i % 32 == 0) sourceOutput += " "; - - sourceOutput += - StringHelper::Sprintf("0x%016llX, ", BitConverter::ToUInt64BE(textureDataRaw, i)); - + if (dWordAligned) + sourceOutput += + StringHelper::Sprintf("0x%016llX, ", BitConverter::ToUInt64BE(textureDataRaw, i)); + else + sourceOutput += + StringHelper::Sprintf("0x%08llX, ", BitConverter::ToUInt32BE(textureDataRaw, i)); if (i % 32 == 24) sourceOutput += StringHelper::Sprintf(" // 0x%06X \n", rawDataIndex + ((i / 32) * 32)); } @@ -812,7 +827,7 @@ ZResourceType ZTexture::GetResourceType() const std::string ZTexture::GetSourceTypeName() const { - return "u64"; + return dWordAligned ? "u64" : "u32"; } void ZTexture::CalcHash() diff --git a/tools/ZAPD/ZAPD/ZTexture.h b/tools/ZAPD/ZAPD/ZTexture.h index ef67c3f6ef..29c3fbc0a1 100644 --- a/tools/ZAPD/ZAPD/ZTexture.h +++ b/tools/ZAPD/ZAPD/ZTexture.h @@ -54,6 +54,7 @@ public: ZTexture(ZFile* nParent); bool isPalette = false; + bool dWordAligned = true; void ExtractFromBinary(uint32_t nRawDataIndex, int32_t nWidth, int32_t nHeight, TextureType nType, bool nIsPalette); diff --git a/tools/ZAPD/docs/zapd_extraction_xml_reference.md b/tools/ZAPD/docs/zapd_extraction_xml_reference.md index 06b95bb575..b3f5575c15 100644 --- a/tools/ZAPD/docs/zapd_extraction_xml_reference.md +++ b/tools/ZAPD/docs/zapd_extraction_xml_reference.md @@ -9,6 +9,7 @@ This document aims to be a small reference of how to create a compatible xml fil - [Basic XML](#basic-xml) - [Resources types](#resources-types) - [File](#file) + - [ExternalFile](#externalfile) - [Texture](#texture) - [Background](#background) - [Blob](#blob) @@ -33,6 +34,7 @@ This document aims to be a small reference of how to create a compatible xml fil - [Array](#array) - [Path](#path) - [PlayerAnimationData](#playeranimationdata) + - [Pointer](#pointer) ## Basic XML @@ -589,7 +591,7 @@ Vec3s D_04002040[24] = { The `Array` element is special, because it needs an inner element to work. It will declare an array of its inner element. -Currently, only [`Scalar`](#scalar), [`Vector`](#vector) and [`Vtx`](#vtx) support being wrapped in an array. +Currently, only [`Pointer`](#pointer), [`Scalar`](#scalar), [`Vector`](#vector) and [`Vtx`](#vtx) support being wrapped in an array. - Example: @@ -637,3 +639,24 @@ Allows the extraction of the specific data of the player animations which are fo - `FrameCount`: Required. The length of the animation in frames. It must be a positive integer. ------------------------- + +### Pointer + +Allows the extraction of a variable that contains a pointer + +- Example: + +```xml + + + +``` + +- Attributes: + + - `Name`: Required. + - `Type`: Required. The type of the extracted pointer. + +※ Can be wrapped in an [`Array`](#array) tag. + +------------------------- diff --git a/tools/asm-differ/.gitrepo b/tools/asm-differ/.gitrepo index 241c0ad6b6..48e215c08c 100644 --- a/tools/asm-differ/.gitrepo +++ b/tools/asm-differ/.gitrepo @@ -4,9 +4,9 @@ ; git-subrepo command. See https://github.com/git-commands/git-subrepo#readme ; [subrepo] - remote = https://github.com/simonlindholm/asm-differ.git + remote = https://github.com/simonlindholm/asm-differ branch = main - commit = 70c33cc125666495f84f8c7bee60d3158b4b1164 - parent = 62341d8db0c29d1d4f0c360196e428309ac373b6 + commit = 6f8f80b719359d018a2b734288c977aae6538870 + parent = 91a6e9f647d2035eba281d286c78f089f70f269f method = merge cmdver = 0.4.3 diff --git a/tools/asm-differ/diff.py b/tools/asm-differ/diff.py index 2ec117a126..9781420bb3 100755 --- a/tools/asm-differ/diff.py +++ b/tools/asm-differ/diff.py @@ -1198,6 +1198,9 @@ def parse_elf_data_references(data: bytes) -> List[Tuple[int, int, str]]: # Skip .text -> .text references continue sec_name = sec_names[s.sh_info].decode("latin1") + if sec_name == ".mwcats.text": + # Skip Metrowerks CATS Utility section + continue sec_base = sections[s.sh_info].sh_offset for i in range(0, s.sh_size, s.sh_entsize): if s.sh_type == SHT_REL: @@ -1330,11 +1333,6 @@ def dump_binary( (objdump_flags + flags2, project.myimg, None), ) -DATA_POOL_PLACEHOLDER = "DATA_POOL_PLACEHOLDER-OFFSET_{}" -DATA_POOL_PLACEHOLDER_PATTERN = re.compile( - r"DATA_POOL_PLACEHOLDER-OFFSET_([a-zA-z0-9]+)" -) - # Example: "ldr r4, [pc, #56] ; (4c )" ARM32_LOAD_POOL_PATTERN = r"(ldr\s+r([0-9]|1[0-3]),\s+\[pc,.*;\s*)(\([a-fA-F0-9]+.*\))" @@ -1357,7 +1355,7 @@ class AsmProcessor: def _normalize_arch_specific(self, mnemonic: str, row: str) -> str: return row - + def post_process(self, lines: List["Line"]) -> None: return @@ -1460,37 +1458,21 @@ class AsmProcessorARM32(AsmProcessor): def _normalize_data_pool(self, row: str) -> str: pool_match = re.search(ARM32_LOAD_POOL_PATTERN, row) - if pool_match: - offset = pool_match.group(3).split(" ")[0][1:] - repl = DATA_POOL_PLACEHOLDER.format(offset) - return pool_match.group(1) + repl - return row + return pool_match.group(1) if pool_match else row def post_process(self, lines: List["Line"]) -> None: lines_by_line_number = {} for line in lines: lines_by_line_number[line.line_num] = line for line in lines: - reloc_match = re.search( - DATA_POOL_PLACEHOLDER_PATTERN, line.normalized_original - ) - if reloc_match is None: + if line.data_pool_addr is None: continue - # Get value at relocation - reloc = reloc_match.group(0) - line_number = re.search( - DATA_POOL_PLACEHOLDER_PATTERN, reloc).group(1) - line_original = lines_by_line_number[int(line_number, 16)].original + # Add data symbol and its address to the line. + line_original = lines_by_line_number[line.data_pool_addr].original value = line_original.split()[1] - - # Replace relocation placeholder with value - replaced = re.sub( - DATA_POOL_PLACEHOLDER_PATTERN, - f"={value} ({line_number})", - line.normalized_original, - ) - line.original = replaced + addr = "{:x}".format(line.data_pool_addr) + line.original = line.normalized_original + f"={value} ({addr})" class AsmProcessorAArch64(AsmProcessor): @@ -1795,6 +1777,7 @@ class Line: scorable_line: str line_num: Optional[int] = None branch_target: Optional[int] = None + data_pool_addr: Optional[int] = None source_filename: Optional[str] = None source_line_num: Optional[int] = None source_lines: List[str] = field(default_factory=list) @@ -1856,6 +1839,14 @@ def process(dump: str, config: Config) -> List[Line]: source_lines.append(row) continue + # If the instructions loads a data pool symbol, extract the address of + # the symbol. + data_pool_addr = None + pool_match = re.search(ARM32_LOAD_POOL_PATTERN, row) + if pool_match: + offset = pool_match.group(3).split(" ")[0][1:] + data_pool_addr = int(offset, 16) + m_comment = re.search(arch.re_comment, row) comment = m_comment[0] if m_comment else None row = re.sub(arch.re_comment, "", row) @@ -1946,6 +1937,7 @@ def process(dump: str, config: Config) -> List[Line]: scorable_line=scorable_line, line_num=line_num, branch_target=branch_target, + data_pool_addr=data_pool_addr, source_filename=source_filename, source_line_num=source_line_num, source_lines=source_lines,