Improve compilation time of asset_processor

By splitting it up into object files and letting them be compiled in
parallel. Also only rebuild the parts that changed.

Makefile

@@ -89,6 +89,7 @@ SCANINC := tools/scaninc/scaninc
 # TODO: use charmap? 
 PREPROC := tools/preproc/preproc
 FIX := tools/gbafix/gbafix
+ASSET_PROCESSOR := tools/asset_processor/asset_processor
 
 # Clear the default suffixes
 .SUFFIXES:
@@ -168,13 +169,14 @@ compare: $(ROM)
 
 setup: $(TOOLDIRS)
 
+# Automatically extract binary data
 build/extracted_assets_%: assets/assets.json assets/gfx.json assets/map.json assets/samples.json assets/sounds.json
-	tools/asset_processor/asset_processor extract $(GAME_VERSION) $(ASSET_BUILDDIR)
+	$(ASSET_PROCESSOR) extract $(GAME_VERSION) $(ASSET_BUILDDIR)
 	touch $@
 
 # Extract assets to human readable form
 extractassets:
-	tools/asset_processor/asset_processor convert $(GAME_VERSION) $(ASSET_BUILDDIR)
+	$(ASSET_PROCESSOR) convert $(GAME_VERSION) $(ASSET_BUILDDIR)
 
 $(TOOLDIRS):
 	@$(MAKE) -C $@

tools/asset_processor/.gitignore

@@ -1 +1,2 @@
 asset_processor
+build/

tools/asset_processor/Makefile

@@ -3,19 +3,32 @@ CXX = g++
 CXXFLAGS = -O3 -Wall -Wextra -std=c++17
 #CXXFLAGS += -g # debug
 
+BUILD_FOLDER=build
+
 SRCS = $(wildcard *.cpp)
-HEADERS = $(wildcard *.h)
 SRCS += $(wildcard assets/*.cpp)
-HEADERS += $(wildcard assets/*.h)
+
+OBJS := $(patsubst %.cpp,$(BUILD_FOLDER)/%.o,$(SRCS))
 
 INCLUDES = -I./
 
+# Create build dirs
+$(shell mkdir -p $(dir $(OBJS)) >/dev/null)
+
 .PHONY: all clean
 
 all: asset_processor
 
-asset_processor: $(SRCS) $(HEADERS)
-	$(CXX) $(CXXFLAGS) $(INCLUDES) -o asset_processor $(SRCS) -lstdc++fs
+asset_processor: $(OBJS)
+	$(CXX) -o asset_processor $(OBJS) -lstdc++fs
+
+$(BUILD_FOLDER)/%.o: %.cpp
+	$(CXX) $(CXXFLAGS) $(INCLUDES) -c -o $@ $<
 
 clean:
 	$(RM) asset_processor asset_processor.exe
+	$(RM) -r $(BUILD_FOLDER)
+
+# Automatic dependencies
+CXXFLAGS += -MMD
+-include $(OBJS:.o=.d)

tools/asset_processor/assets/asset.h

@@ -3,13 +3,13 @@
 
 #include "util.h"
 #include <filesystem>
-#include <json.hpp>
+#include <json_fwd.hpp>
 #include <string>
 #include <iostream>
 
 class BaseAsset {
   public:
-    BaseAsset(std::filesystem::path path, int start, int size, nlohmann::json asset)
+    BaseAsset(std::filesystem::path path, int start, int size, const nlohmann::json& asset)
         : path(path), start(start), size(size), asset(asset) {
     }
 
@@ -62,7 +62,7 @@ class BaseAsset {
     std::filesystem::path buildPath;
     int start;
     int size;
-    nlohmann::json asset;
+    const nlohmann::json& asset;
 
   private:
     virtual std::filesystem::path generateAssetPath() {

tools/asset_processor/assets/gfx.cpp

@@ -1,5 +1,6 @@
 #include "gfx.h"
 #include "util.h"
+#include <json.hpp>
 
 std::filesystem::path GfxAsset::generateAssetPath() {
     std::filesystem::path pngPath = this->path;
@@ -18,9 +19,11 @@ void GfxAsset::convertToHumanReadable(const std::vector<char>& baserom) {
     cmd.push_back(toolsPath / "gbagfx" / "gbagfx");
     cmd.push_back(this->path);
     cmd.push_back(this->assetPath);
-    for (const auto& it : this->asset["options"].items()) {
-        cmd.push_back("-" + it.key());
-        cmd.push_back(to_string(it.value()));
+    if (this->asset.contains("options")) {
+        for (const auto& it : this->asset["options"].items()) {
+            cmd.push_back("-" + it.key());
+            cmd.push_back(to_string(it.value()));
+        }
     }
     check_call(cmd);
 }

tools/asset_processor/assets/midi.cpp

@@ -1,10 +1,12 @@
 #include "midi.h"
-#include "main.h"
 #include "reader.h"
 #include "util.h"
 #include <filesystem>
 #include <iostream>
 #include <fstream>
+#include <json.hpp>
+
+extern std::string gBaseromPath;
 
 std::filesystem::path MidiAsset::generateAssetPath() {
     std::filesystem::path path = this->path;

tools/asset_processor/json_fwd.hpp

@@ -0,0 +1,78 @@
+#ifndef INCLUDE_NLOHMANN_JSON_FWD_HPP_
+#define INCLUDE_NLOHMANN_JSON_FWD_HPP_
+
+#include <cstdint> // int64_t, uint64_t
+#include <map> // map
+#include <memory> // allocator
+#include <string> // string
+#include <vector> // vector
+
+/*!
+@brief namespace for Niels Lohmann
+@see https://github.com/nlohmann
+@since version 1.0.0
+*/
+namespace nlohmann
+{
+/*!
+@brief default JSONSerializer template argument
+
+This serializer ignores the template arguments and uses ADL
+([argument-dependent lookup](https://en.cppreference.com/w/cpp/language/adl))
+for serialization.
+*/
+template<typename T = void, typename SFINAE = void>
+struct adl_serializer;
+
+template<template<typename U, typename V, typename... Args> class ObjectType =
+         std::map,
+         template<typename U, typename... Args> class ArrayType = std::vector,
+         class StringType = std::string, class BooleanType = bool,
+         class NumberIntegerType = std::int64_t,
+         class NumberUnsignedType = std::uint64_t,
+         class NumberFloatType = double,
+         template<typename U> class AllocatorType = std::allocator,
+         template<typename T, typename SFINAE = void> class JSONSerializer =
+         adl_serializer,
+         class BinaryType = std::vector<std::uint8_t>>
+class basic_json;
+
+/*!
+@brief JSON Pointer
+
+A JSON pointer defines a string syntax for identifying a specific value
+within a JSON document. It can be used with functions `at` and
+`operator[]`. Furthermore, JSON pointers are the base for JSON patches.
+
+@sa [RFC 6901](https://tools.ietf.org/html/rfc6901)
+
+@since version 2.0.0
+*/
+template<typename BasicJsonType>
+class json_pointer;
+
+/*!
+@brief default JSON class
+
+This type is the default specialization of the @ref basic_json class which
+uses the standard template types.
+
+@since version 1.0.0
+*/
+using json = basic_json<>;
+
+template<class Key, class T, class IgnoredLess, class Allocator>
+struct ordered_map;
+
+/*!
+@brief ordered JSON class
+
+This type preserves the insertion order of object keys.
+
+@since version 3.9.0
+*/
+using ordered_json = basic_json<nlohmann::ordered_map>;
+
+}  // namespace nlohmann
+
+#endif  // INCLUDE_NLOHMANN_JSON_FWD_HPP_

tools/asset_processor/util.cpp

@@ -18,12 +18,3 @@ void check_call(const std::vector<std::string>& cmd) {
         std::exit(1);
     }
 }
-
-// https://github.com/nlohmann/json/issues/642#issuecomment-311937344
-std::string to_string(const nlohmann::json& j) {
-    if (j.type() == nlohmann::json::value_t::string) {
-        return j.get<std::string>();
-    }
-
-    return j.dump();
-}

tools/asset_processor/util.h

@@ -1,7 +1,7 @@
 #ifndef UTIL_H
 #define UTIL_H
 
-#include <json.hpp>
+#include <json_fwd.hpp>
 #include <memory>
 #include <stdexcept>
 #include <stdint.h>
@@ -20,8 +20,6 @@ typedef int64_t s64;
 
 void check_call(const std::vector<std::string>& cmd);
 
-std::string to_string(const nlohmann::json& j);
-
 template <typename... Args> std::string string_format(const std::string& format, Args... args) {
     int size_s = std::snprintf(nullptr, 0, format.c_str(), args...) + 1; // Extra space for '\0'
     if (size_s <= 0) {

tools/mid2agb/midi.h

@@ -33,15 +33,9 @@ enum class EventType
 {
     EndOfTie = 0x01,
     Label = 0x11,
-    LoopEnd = 0x38,// To place it last if at the same time as other meta events, but before notes on the same frame
+    LoopEnd = 0x38, // To place it last if at the same time as other meta events, but before notes on the same frame
     LoopEndBegin = 0x13,
-    LoopBegin = 0x24, 
-    // Original: 0x14
-    // TODO sfx1AA wants a LoopBegin before a Volume Change -> < 0x22
-    // bgmFestivalApproach wants a LoopBegin after a Tempo -> >0x19  -> 0x20
-    // sfxSparkles wants a LoopBegin after a VOL --> NOT POSSIBLE
-    // bgmCuccoMinigame as well
-    // To place it last if at the same time as other meta events, but before notes on the same frame
+    LoopBegin = 0x24, // To place it last if at the same time as other meta events, but before notes on the same frame
     OriginalTimeSignature = 0x15,
     WholeNoteMark = 0x16,
     Pattern = 0x17,