mirrors/N64Recomp
1
0
Fork 0
mirror of https://github.com/Mr-Wiseguy/N64Recomp.git synced 2024-12-26 17:36:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

Modding Support PR 2 (Finished mod tool base feature set and improvements for use in N64ModernRuntime) (#93)

Browse source 
* Remove reference context from parse_mod_symbols argument

* Add support for special dependency names (self and base recomp), fix non-compliant offline mod recompiler output

* Fix export names not being set on functions when parsing mod syms, add missing returns to mod parsing

* Switch offline mod recompilation to use a base global event index instead of per-event global indices

* Add support for creating events in normal recompilation

* Output recomp API version in offline mod recompiler

* Removed dependency version from mod symbols (moved to manifest)

* Added mod manifest generation to mod tool

* Implement mod file creation in Windows

* Fixed some error prints not using stderr

* Implement mod file creation on posix systems

* De-hardcode symbol file path for offline mod recompiler

* Fix duplicate import symbols issue and prevent emitting unused imports
This commit is contained in:
Wiseguy 2024-09-09 22:49:57 -04:00 committed by GitHub
parent 5b17bf8bb5
commit cc71b31b09
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 755 additions and 244 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

42
OfflineModRecomp/main.cpp
View file

@ -24,17 +24,9 @@ static std::vector<uint8_t> read_file(const std::filesystem::path& path, bool& f
return ret;
}
const std::filesystem::path func_reference_syms_file_path {
"C:/n64/MMRecompTestMod/Zelda64RecompSyms/mm.us.rev1.syms.toml"
};
const std::vector<std::filesystem::path> data_reference_syms_file_paths {
"C:/n64/MMRecompTestMod/Zelda64RecompSyms/mm.us.rev1.datasyms.toml",
"C:/n64/MMRecompTestMod/Zelda64RecompSyms/mm.us.rev1.datasyms_static.toml"
};
int main(int argc, const char** argv) {
if (argc != 4) {
printf("Usage: %s [mod symbol file] [ROM] [output C file]\n", argv[0]);
if (argc != 5) {
printf("Usage: %s [mod symbol file] [mod binary file] [recomp symbols file] [output C file]\n", argv[0]);
return EXIT_SUCCESS;
}
bool found;
@ -54,7 +46,7 @@ int main(int argc, const char** argv) {
std::vector<uint8_t> dummy_rom{};
N64Recomp::Context reference_context{};
if (!N64Recomp::Context::from_symbol_file(func_reference_syms_file_path, std::move(dummy_rom), reference_context, false)) {
if (!N64Recomp::Context::from_symbol_file(argv[3], std::move(dummy_rom), reference_context, false)) {
printf("Failed to load provided function reference symbol file\n");
return EXIT_FAILURE;
}
@ -73,12 +65,14 @@ int main(int argc, const char** argv) {
N64Recomp::Context mod_context;
N64Recomp::ModSymbolsError error = N64Recomp::parse_mod_symbols(symbol_data_span, rom_data, sections_by_vrom, reference_context, mod_context);
N64Recomp::ModSymbolsError error = N64Recomp::parse_mod_symbols(symbol_data_span, rom_data, sections_by_vrom, mod_context);
if (error != N64Recomp::ModSymbolsError::Good) {
fprintf(stderr, "Error parsing mod symbols: %d\n", (int)error);
return EXIT_FAILURE;
}
mod_context.import_reference_context(reference_context);
// Populate R_MIPS_26 reloc symbol indices. Start by building a map of vram address to matching reference symbols.
std::unordered_map<uint32_t, std::vector<size_t>> reference_symbols_by_vram{};
for (size_t reference_symbol_index = 0; reference_symbol_index < mod_context.num_regular_reference_symbols(); reference_symbol_index++) {
@ -124,7 +118,7 @@ int main(int argc, const char** argv) {
std::vector<std::vector<uint32_t>> static_funcs_by_section{};
static_funcs_by_section.resize(mod_context.sections.size());
std::ofstream output_file { argv[3] };
std::ofstream output_file { argv[4] };
RabbitizerConfig_Cfg.pseudos.pseudoMove = false;
RabbitizerConfig_Cfg.pseudos.pseudoBeqz = false;
@ -134,6 +128,9 @@ int main(int argc, const char** argv) {
output_file << "#include \"mod_recomp.h\"\n\n";
// Write the API version.
output_file << "RECOMP_EXPORT uint32_t recomp_api_version = 1;\n\n";
output_file << "// Values populated by the runtime:\n\n";
// Write import function pointer array and defines (i.e. `#define testmod_inner_import imported_funcs[0]`)
@ -143,7 +140,8 @@ int main(int argc, const char** argv) {
const auto& import = mod_context.import_symbols[import_index];
output_file << "#define " << import.base.name << " imported_funcs[" << import_index << "]\n";
}
output_file << "RECOMP_EXPORT recomp_func_t* imported_funcs[" << num_imports << "] = {};\n";
output_file << "RECOMP_EXPORT recomp_func_t* imported_funcs[" << std::max(size_t{1}, num_imports) << "] = {0};\n";
output_file << "\n";
// Use reloc list to write reference symbol function pointer array and defines (i.e. `#define func_80102468 reference_symbol_funcs[0]`)
@ -158,11 +156,12 @@ int main(int argc, const char** argv) {
}
}
}
output_file << "RECOMP_EXPORT recomp_func_t* reference_symbol_funcs[" << num_reference_symbols << "] = {};\n\n";
// C doesn't allow 0-sized arrays, so always add at least one member to all arrays. The actual size will be pulled from the mod symbols.
output_file << "RECOMP_EXPORT recomp_func_t* reference_symbol_funcs[" << std::max(size_t{1},num_reference_symbols) << "] = {0};\n\n";
// Write provided event array (maps internal event indices to global ones).
output_file << "// Mapping of internal event indices to global ones.\n";
output_file << "RECOMP_EXPORT uint32_t event_indices[" << mod_context.event_symbols.size() <<"] = {};\n\n";
output_file << "// Base global event index for this mod's events.\n";
output_file << "RECOMP_EXPORT uint32_t base_event_index;\n\n";
// Write the event trigger function pointer.
output_file << "// Pointer to the runtime function for triggering events.\n";
@ -179,11 +178,16 @@ int main(int argc, const char** argv) {
// Write the local section addresses pointer array.
size_t num_sections = mod_context.sections.size();
output_file << "// Array of this mod's loaded section addresses.\n";
output_file << "RECOMP_EXPORT int32_t section_addresses[" << num_sections << "] = {};\n\n";
output_file << "RECOMP_EXPORT int32_t section_addresses[" << std::max(size_t{1}, num_sections) << "] = {0};\n\n";
// Create a set of the export indices to avoid renaming them.
std::unordered_set<size_t> export_indices{mod_context.exported_funcs.begin(), mod_context.exported_funcs.end()};
for (size_t func_index = 0; func_index < mod_context.functions.size(); func_index++) {
auto& func = mod_context.functions[func_index];
func.name = "mod_func_" + std::to_string(func_index);
if (!export_indices.contains(func_index)) {
func.name = "mod_func_" + std::to_string(func_index);
}
N64Recomp::recompile_function(mod_context, func, output_file, static_funcs_by_section, true);
}

693
RecompModTool/main.cpp
View file

@ -3,30 +3,64 @@
#include <filesystem>
#include <iostream>
#include <numeric>
#include <cctype>
#include "fmt/format.h"
#include "fmt/ostream.h"
#include "n64recomp.h"
#include <toml++/toml.hpp>
struct ModConfig {
std::filesystem::path output_syms_path;
std::filesystem::path output_binary_path;
#ifdef _WIN32
#define WIN32_LEAN_AND_MEAN
#include <Windows.h>
#else
#include <unistd.h>
#include <sys/wait.h>
#endif
constexpr std::string_view symbol_filename = "mod_syms.bin";
constexpr std::string_view binary_filename = "mod_binary.bin";
constexpr std::string_view manifest_filename = "manifest.json";
struct ModManifest {
std::string mod_id;
std::string version_string;
std::vector<std::string> authors;
std::string game_id;
std::string minimum_recomp_version;
std::unordered_map<std::string, std::vector<std::string>> native_libraries;
std::vector<std::string> dependencies;
std::vector<std::string> full_dependency_strings;
};
struct ModInputs {
std::filesystem::path elf_path;
std::filesystem::path func_reference_syms_file_path;
std::vector<std::filesystem::path> data_reference_syms_file_paths;
std::vector<N64Recomp::Dependency> dependencies;
std::vector<std::filesystem::path> additional_files;
};
struct ModConfig {
ModManifest manifest;
ModInputs inputs;
};
static std::filesystem::path concat_if_not_empty(const std::filesystem::path& parent, const std::filesystem::path& child) {
if (child.is_absolute()) {
return child;
}
if (!child.empty()) {
return parent / child;
}
return child;
}
static bool parse_version_string(std::string_view str, uint8_t& major, uint8_t& minor, uint8_t& patch) {
static bool validate_version_string(std::string_view str, bool& has_label) {
std::array<size_t, 2> period_indices;
size_t num_periods = 0;
size_t cur_pos = 0;
uint16_t major;
uint16_t minor;
uint16_t patch;
// Find the 2 required periods.
cur_pos = str.find('.', cur_pos);
@ -47,65 +81,273 @@ static bool parse_version_string(std::string_view str, uint8_t& major, uint8_t&
parse_results[1] = std::from_chars(str.data() + parse_starts[1], str.data() + parse_ends[1], minor);
parse_results[2] = std::from_chars(str.data() + parse_starts[2], str.data() + parse_ends[2], patch);
// Check that all 3 parsed correctly.
// Check that the first two parsed correctly.
auto did_parse = [&](size_t i) {
return parse_results[i].ec == std::errc{} && parse_results[i].ptr == str.data() + parse_ends[i];
};
if (!did_parse(0) || !did_parse(1) || !did_parse(2)) {
if (!did_parse(0) || !did_parse(1)) {
return false;
}
return true;
}
static bool parse_dependency_string(const std::string& val, N64Recomp::Dependency& dep) {
N64Recomp::Dependency ret;
size_t id_pos = 0;
size_t id_length = 0;
size_t colon_pos = val.find(':');
if (colon_pos == std::string::npos) {
id_length = val.size();
ret.major_version = 0;
ret.minor_version = 0;
ret.patch_version = 0;
// Check that the third had a successful parse, but not necessarily read all the characters.
if (parse_results[2].ec != std::errc{}) {
return false;
}
else {
id_length = colon_pos;
uint8_t major, minor, patch;
if (!parse_version_string(std::string_view{val.begin() + colon_pos + 1, val.end()}, major, minor, patch)) {
// Allow a plus or minus directly after the third number.
if (parse_results[2].ptr != str.data() + parse_ends[2]) {
has_label = true;
if (*parse_results[2].ptr != '+' && *parse_results[2].ptr != '-') {
// Failed to parse, as nothing is allowed directly after the last number besides a plus or minus.
return false;
}
ret.major_version = major;
ret.minor_version = minor;
ret.patch_version = patch;
}
else {
has_label = false;
}
ret.mod_id = val.substr(id_pos, id_length);
dep = std::move(ret);
return true;
}
static std::vector<std::filesystem::path> get_toml_path_array(const toml::array* toml_array, const std::filesystem::path& basedir) {
static bool validate_dependency_string(const std::string& val, size_t& name_length, bool& has_label) {
std::string ret;
size_t name_length_temp;
// Don't allow an empty dependency name.
if (val.size() == 0) {
return false;
}
bool validated_name;
bool validated_version;
// Check if there's a version number specified.
size_t colon_pos = val.find(':');
if (colon_pos == std::string::npos) {
// No version present, so just validate the dependency's id.
validated_name = N64Recomp::validate_mod_id(std::string_view{val});
name_length_temp = val.size();
validated_version = true;
has_label = false;
}
else {
// Version present, validate it.
// Don't allow an empty dependency name after accounting for the colon.
if (colon_pos == 0) {
return false;
}
name_length_temp = colon_pos;
// Validate the dependency's id and version.
validated_name = N64Recomp::validate_mod_id(std::string_view{val.begin(), val.begin() + colon_pos});
validated_version = validate_version_string(std::string_view{val.begin() + colon_pos + 1, val.end()}, has_label);
}
if (validated_name && validated_version) {
name_length = name_length_temp;
return true;
}
return false;
}
template <typename T>
static T read_toml_value(const toml::table& data, std::string_view key, bool required) {
const toml::node* value_node = data.get(key);
if (value_node == nullptr) {
if (required) {
throw toml::parse_error(("Missing required field " + std::string{key}).c_str(), data.source());
}
else {
return T{};
}
}
std::optional<T> opt = value_node->value_exact<T>();
if (opt.has_value()) {
return opt.value();
}
else {
throw toml::parse_error(("Incorrect type for field " + std::string{key}).c_str(), data.source());
}
}
static const toml::array& read_toml_array(const toml::table& data, std::string_view key, bool required) {
static const toml::array empty_array = toml::array{};
const toml::node* value_node = data.get(key);
if (value_node == nullptr) {
if (required) {
throw toml::parse_error(("Missing required field " + std::string{ key }).c_str(), data.source());
}
else {
return empty_array;
}
}
if (!value_node->is_array()) {
throw toml::parse_error(("Incorrect type for field " + std::string{ key }).c_str(), value_node->source());
}
return *value_node->as_array();
}
static std::vector<std::filesystem::path> get_toml_path_array(const toml::array& toml_array, const std::filesystem::path& basedir) {
std::vector<std::filesystem::path> ret;
// Reserve room for all the funcs in the map.
ret.reserve(toml_array->size());
toml_array->for_each([&ret, &basedir](auto&& el) {
ret.reserve(toml_array.size());
toml_array.for_each([&ret, &basedir](auto&& el) {
if constexpr (toml::is_string<decltype(el)>) {
ret.emplace_back(concat_if_not_empty(basedir, el.ref<std::string>()));
ret.emplace_back(concat_if_not_empty(basedir, el.template ref<std::string>()));
}
else {
throw toml::parse_error("Invalid type for data reference symbol file entry", el.source());
throw toml::parse_error("Invalid type for file entry", el.source());
}
});
return ret;
}
ModManifest parse_mod_config_manifest(const std::filesystem::path& basedir, const toml::table& manifest_table) {
ModManifest ret;
// Mod ID
ret.mod_id = read_toml_value<std::string_view>(manifest_table, "id", true);
// Mod version
ret.version_string = read_toml_value<std::string_view>(manifest_table, "version", true);
bool version_has_label;
if (!validate_version_string(ret.version_string, version_has_label)) {
throw toml::parse_error("Invalid mod version", manifest_table["version"].node()->source());
}
// Authors
const toml::array& authors_array = read_toml_array(manifest_table, "authors", true);
authors_array.for_each([&ret](auto&& el) {
if constexpr (toml::is_string<decltype(el)>) {
ret.authors.emplace_back(el.template ref<std::string>());
}
else {
throw toml::parse_error("Invalid type for author entry", el.source());
}
});
// Game ID
ret.game_id = read_toml_value<std::string_view>(manifest_table, "game_id", true);
// Minimum recomp version
ret.minimum_recomp_version = read_toml_value<std::string_view>(manifest_table, "minimum_recomp_version", true);
bool minimum_recomp_version_has_label;
if (!validate_version_string(ret.minimum_recomp_version, minimum_recomp_version_has_label)) {
throw toml::parse_error("Invalid minimum recomp version", manifest_table["minimum_recomp_version"].node()->source());
}
if (minimum_recomp_version_has_label) {
throw toml::parse_error("Minimum recomp version may not have a label", manifest_table["minimum_recomp_version"].node()->source());
}
// Native libraries (optional)
const toml::array& native_libraries = read_toml_array(manifest_table, "native_libraries", false);
if (!native_libraries.empty()) {
native_libraries.for_each([&ret](const auto& el) {
if constexpr (toml::is_table<decltype(el)>) {
const toml::table& el_table = *el.as_table();
std::string_view library_name = read_toml_value<std::string_view>(el_table, "name", true);
const toml::array funcs_array = read_toml_array(el_table, "funcs", true);
std::vector<std::string> cur_funcs{};
funcs_array.for_each([&ret, &cur_funcs](const auto& func_el) {
if constexpr (toml::is_string<decltype(func_el)>) {
cur_funcs.emplace_back(func_el.template ref<std::string>());
}
else {
throw toml::parse_error("Invalid type for native library function entry", func_el.source());
}
});
ret.native_libraries.emplace(std::string{library_name}, std::move(cur_funcs));
}
else {
throw toml::parse_error("Invalid type for native library entry", el.source());
}
});
}
// Dependency list (optional)
const toml::array& dependency_array = read_toml_array(manifest_table, "dependencies", false);
if (!dependency_array.empty()) {
// Reserve room for all the dependencies.
ret.dependencies.reserve(dependency_array.size());
dependency_array.for_each([&ret](const auto& el) {
if constexpr (toml::is_string<decltype(el)>) {
size_t dependency_id_length;
bool dependency_version_has_label;
if (!validate_dependency_string(el.template ref<std::string>(), dependency_id_length, dependency_version_has_label)) {
throw toml::parse_error("Invalid dependency entry", el.source());
}
if (dependency_version_has_label) {
throw toml::parse_error("Dependency versions may not have labels", el.source());
}
std::string dependency_id = el.template ref<std::string>().substr(0, dependency_id_length);
ret.dependencies.emplace_back(dependency_id);
ret.full_dependency_strings.emplace_back(el.template ref<std::string>());
}
else {
throw toml::parse_error("Invalid type for dependency entry", el.source());
}
});
}
return ret;
}
ModInputs parse_mod_config_inputs(const std::filesystem::path& basedir, const toml::table& inputs_table) {
ModInputs ret;
// Elf file
std::optional<std::string> elf_path_opt = inputs_table["elf_path"].value<std::string>();
if (elf_path_opt.has_value()) {
ret.elf_path = concat_if_not_empty(basedir, elf_path_opt.value());
}
else {
throw toml::parse_error("Mod toml input section is missing elf file", inputs_table.source());
}
// Function reference symbols file
std::optional<std::string> func_reference_syms_file_opt = inputs_table["func_reference_syms_file"].value<std::string>();
if (func_reference_syms_file_opt.has_value()) {
ret.func_reference_syms_file_path = concat_if_not_empty(basedir, func_reference_syms_file_opt.value());
}
else {
throw toml::parse_error("Mod toml input section is missing function reference symbol file", inputs_table.source());
}
// Data reference symbols files
toml::node_view data_reference_syms_file_data = inputs_table["data_reference_syms_files"];
if (data_reference_syms_file_data.is_array()) {
const toml::array& array = *data_reference_syms_file_data.as_array();
ret.data_reference_syms_file_paths = get_toml_path_array(array, basedir);
}
else {
if (data_reference_syms_file_data) {
throw toml::parse_error("Mod toml input section is missing data reference symbol file list", inputs_table.source());
}
else {
throw toml::parse_error("Invalid data reference symbol file list", data_reference_syms_file_data.node()->source());
}
}
// Additional files (optional)
const toml::array& additional_files_array = read_toml_array(inputs_table, "additional_files", false);
if (!additional_files_array.empty()) {
ret.additional_files = get_toml_path_array(additional_files_array, basedir);
}
return ret;
}
ModConfig parse_mod_config(const std::filesystem::path& config_path, bool& good) {
ModConfig ret{};
good = false;
@ -116,84 +358,33 @@ ModConfig parse_mod_config(const std::filesystem::path& config_path, bool& good)
toml_data = toml::parse_file(config_path.native());
std::filesystem::path basedir = config_path.parent_path();
const auto config_data = toml_data["config"];
// Find the manifest section and validate its type.
const toml::node* manifest_data_ptr = toml_data.get("manifest");
if (manifest_data_ptr == nullptr) {
throw toml::parse_error("Mod toml is missing manifest section", toml::source_region{});
}
if (!manifest_data_ptr->is_table()) {
throw toml::parse_error("Incorrect type for mod toml manifest section", manifest_data_ptr->source());
}
const toml::table& manifest_table = *manifest_data_ptr->as_table();
// Output symbol file path
std::optional<std::string> output_syms_path_opt = config_data["output_syms_path"].value<std::string>();
if (output_syms_path_opt.has_value()) {
ret.output_syms_path = concat_if_not_empty(basedir, output_syms_path_opt.value());
// Find the inputs section and validate its type.
const toml::node* inputs_data_ptr = toml_data.get("inputs");
if (inputs_data_ptr == nullptr) {
throw toml::parse_error("Mod toml is missing inputs section", toml::source_region{});
}
else {
throw toml::parse_error("Mod toml is missing output symbol file path", config_data.node()->source());
if (!inputs_data_ptr->is_table()) {
throw toml::parse_error("Incorrect type for mod toml inputs section", inputs_data_ptr->source());
}
const toml::table& inputs_table = *inputs_data_ptr->as_table();
// Output binary file path
std::optional<std::string> output_binary_path_opt = config_data["output_binary_path"].value<std::string>();
if (output_binary_path_opt.has_value()) {
ret.output_binary_path = concat_if_not_empty(basedir, output_binary_path_opt.value());
}
else {
throw toml::parse_error("Mod toml is missing output binary file path", config_data.node()->source());
}
// Elf file
std::optional<std::string> elf_path_opt = config_data["elf_path"].value<std::string>();
if (elf_path_opt.has_value()) {
ret.elf_path = concat_if_not_empty(basedir, elf_path_opt.value());
}
else {
throw toml::parse_error("Mod toml is missing elf file", config_data.node()->source());
}
// Function reference symbols file
std::optional<std::string> func_reference_syms_file_opt = config_data["func_reference_syms_file"].value<std::string>();
if (func_reference_syms_file_opt.has_value()) {
ret.func_reference_syms_file_path = concat_if_not_empty(basedir, func_reference_syms_file_opt.value());
}
else {
throw toml::parse_error("Mod toml is missing function reference symbol file", config_data.node()->source());
}
// Data reference symbols files
toml::node_view data_reference_syms_file_data = config_data["data_reference_syms_files"];
if (data_reference_syms_file_data.is_array()) {
const toml::array* array = data_reference_syms_file_data.as_array();
ret.data_reference_syms_file_paths = get_toml_path_array(array, basedir);
}
else {
if (data_reference_syms_file_data) {
throw toml::parse_error("Mod toml is missing data reference symbol file list", config_data.node()->source());
}
else {
throw toml::parse_error("Invalid data reference symbol file list", data_reference_syms_file_data.node()->source());
}
}
// Dependency list (optional)
toml::node_view dependency_data = config_data["dependencies"];
if (dependency_data.is_array()) {
const toml::array* dependency_array = dependency_data.as_array();
// Reserve room for all the dependencies.
ret.dependencies.reserve(dependency_array->size());
dependency_array->for_each([&ret](auto&& el) {
if constexpr (toml::is_string<decltype(el)>) {
N64Recomp::Dependency dep;
if (!parse_dependency_string(el.ref<std::string>(), dep)) {
throw toml::parse_error("Invalid dependency entry", el.source());
}
ret.dependencies.emplace_back(std::move(dep));
}
else {
throw toml::parse_error("Invalid toml type for dependency", el.source());
}
});
}
else if (dependency_data) {
throw toml::parse_error("Invalid mod dependency list", dependency_data.node()->source());
}
// Parse the manifest.
ret.manifest = parse_mod_config_manifest(basedir, manifest_table);
// Parse the inputs.
ret.inputs = parse_mod_config_inputs(basedir, inputs_table);
}
catch (const toml::parse_error& err) {
std::cerr << "Syntax error parsing toml: " << *err.source().path << " (" << err.source().begin << "):\n" << err.description() << std::endl;
std::cerr << "Syntax error parsing toml: " << config_path << " (" << err.source().begin << "):\n" << err.description() << std::endl;
return {};
}
@ -215,7 +406,7 @@ bool parse_callback_name(std::string_view data, std::string& dependency_name, st
std::string_view dependency_name_view = std::string_view{data}.substr(0, period_pos);
std::string_view event_name_view = std::string_view{data}.substr(period_pos + 1);
if (!N64Recomp::validate_mod_name(dependency_name_view)) {
if (!N64Recomp::validate_mod_id(dependency_name_view)) {
return false;
}
@ -224,6 +415,59 @@ bool parse_callback_name(std::string_view data, std::string& dependency_name, st
return true;
}
void print_vector_elements(std::ostream& output_file, const std::vector<std::string>& vec, bool compact) {
char separator = compact ? ' ' : '\n';
for (size_t i = 0; i < vec.size(); i++) {
const std::string& val = vec[i];
fmt::print(output_file, "{}\"{}\"{}{}",
compact ? "" : " ", val, i == vec.size() - 1 ? "" : ",", separator);
}
}
void write_manifest(const std::filesystem::path& path, const ModManifest& manifest) {
std::ofstream output_file(path);
fmt::print(output_file,
"{{\n"
" \"game_id\": \"{}\",\n"
" \"id\": \"{}\",\n"
" \"version\": \"{}\",\n"
" \"authors\": [\n",
manifest.game_id, manifest.mod_id, manifest.version_string);
print_vector_elements(output_file, manifest.authors, false);
fmt::print(output_file,
" ],\n"
" \"minimum_recomp_version\": \"{}\"",
manifest.minimum_recomp_version);
if (!manifest.native_libraries.empty()) {
fmt::print(output_file, ",\n"
" \"native_libraries\": {{\n");
size_t library_index = 0;
for (const auto& [library, funcs] : manifest.native_libraries) {
fmt::print(output_file, " \"{}\": [ ",
library);
print_vector_elements(output_file, funcs, true);
fmt::print(output_file, "]{}\n",
library_index == manifest.native_libraries.size() - 1 ? "" : ",");
library_index++;
}
fmt::print(output_file, " }}");
}
if (!manifest.full_dependency_strings.empty()) {
fmt::print(output_file, ",\n"
" \"dependencies\": [\n");
print_vector_elements(output_file, manifest.full_dependency_strings, false);
fmt::print(output_file, " ]");
}
fmt::print(output_file, "\n}}\n");
}
N64Recomp::Context build_mod_context(const N64Recomp::Context& input_context, bool& good) {
N64Recomp::Context ret{};
good = false;
@ -253,7 +497,6 @@ N64Recomp::Context build_mod_context(const N64Recomp::Context& input_context, bo
ret.rom = input_context.rom;
// Copy the dependency data from the input context.
ret.dependencies = input_context.dependencies;
ret.dependencies_by_name = input_context.dependencies_by_name;
ret.import_symbols = input_context.import_symbols;
ret.dependency_events = input_context.dependency_events;
@ -325,8 +568,11 @@ N64Recomp::Context build_mod_context(const N64Recomp::Context& input_context, bo
const auto& cur_section_funcs = input_context.section_functions[section_index];
// Skip the functions and relocs in this section if it's the event section, instead opting to create
// event functions from the section's functions.
// Skip the functions and relocs in this section if it's the event section, instead opting to create event functions from the section's functions.
// This has to be done to find events that are never called, which may pop up as a valid use case for maintaining backwards compatibility
// if a mod removes a call to an event but doesn't want to break mods that reference it. If this code wasn't present, then only events that are actually
// triggered would show up in the mod's symbol file.
if (event_section) {
// Create event reference symbols for any functions in the event section. Ignore functions that already
// have a symbol, since relocs from previous sections may have triggered creation of the event's reference symbol already.
@ -340,30 +586,9 @@ N64Recomp::Context build_mod_context(const N64Recomp::Context& input_context, bo
}
}
}
// Skip the functions and relocs in this section if it's an import section, instead opting to create
// import symbols from the section's functions.
else if (import_section) {
for (const auto& input_func_index : cur_section_funcs) {
const auto& cur_func = input_context.functions[input_func_index];
std::string dependency_name = cur_section.name.substr(N64Recomp::ImportSectionPrefix.size());
if (!N64Recomp::validate_mod_name(dependency_name)) {
fmt::print("Failed to import function {} as {} is an invalid mod name.\n",
cur_func.name, dependency_name);
return {};
}
size_t dependency_index;
if (!ret.find_dependency(dependency_name, dependency_index)) {
fmt::print("Failed to import function {} from mod {} as the mod is not a registered dependency.\n",
cur_func.name, dependency_name);
return {};
}
ret.add_import_symbol(cur_func.name, dependency_index);
}
}
// Normal section, copy the functions and relocs over.
else {
// Otherwise, copy the functions and relocs over from this section into the output context.
// Import sections can be skipped, as those only contain dummy functions. Imports will be found while scanning relocs.
else if (!import_section) {
for (size_t section_function_index = 0; section_function_index < cur_section_funcs.size(); section_function_index++) {
size_t output_func_index = ret.functions.size();
size_t input_func_index = cur_section_funcs[section_function_index];
@ -377,14 +602,14 @@ N64Recomp::Context build_mod_context(const N64Recomp::Context& input_context, bo
// Check that the function being patched exists in the original reference symbols.
if (!original_func_exists) {
fmt::print("Function {} is marked as a patch but doesn't exist in the original ROM.\n", cur_func.name);
fmt::print(stderr, "Function {} is marked as a patch but doesn't exist in the original ROM.\n", cur_func.name);
return {};
}
// Check that the reference symbol is actually a function.
const auto& reference_symbol = input_context.get_reference_symbol(cur_reference);
if (!reference_symbol.is_function) {
fmt::print("Function {0} is marked as a patch, but {0} was a variable in the original ROM.\n", cur_func.name);
fmt::print(stderr, "Function {0} is marked as a patch, but {0} was a variable in the original ROM.\n", cur_func.name);
return {};
}
@ -413,27 +638,27 @@ N64Recomp::Context build_mod_context(const N64Recomp::Context& input_context, bo
if (callback_section) {
std::string dependency_name, event_name;
if (!parse_callback_name(std::string_view{ cur_section.name }.substr(N64Recomp::CallbackSectionPrefix.size()), dependency_name, event_name)) {
fmt::print("Invalid mod name or event name for callback function {}.\n",
fmt::print(stderr, "Invalid mod name or event name for callback function {}.\n",
cur_func.name);
return {};
}
size_t dependency_index;
if (!ret.find_dependency(dependency_name, dependency_index)) {
fmt::print("Failed to register callback {} to event {} from mod {} as the mod is not a registered dependency.\n",
fmt::print(stderr, "Failed to register callback {} to event {} from mod {} as the mod is not a registered dependency.\n",
cur_func.name, event_name, dependency_name);
return {};
}
size_t event_index;
if (!ret.add_dependency_event(event_name, dependency_index, event_index)) {
fmt::print("Internal error: Failed to register event {} for dependency {}. Please report this issue.\n",
fmt::print(stderr, "Internal error: Failed to register event {} for dependency {}. Please report this issue.\n",
event_name, dependency_name);
return {};
}
if (!ret.add_callback(event_index, output_func_index)) {
fmt::print("Internal error: Failed to add callback {} to event {} in dependency {}. Please report this issue.\n",
fmt::print(stderr, "Internal error: Failed to add callback {} to event {} in dependency {}. Please report this issue.\n",
cur_func.name, event_name, dependency_name);
return {};
}
@ -499,7 +724,7 @@ N64Recomp::Context build_mod_context(const N64Recomp::Context& input_context, bo
reloc_word |= reloc_target_address & 0xFFFF;
break;
default:
fmt::print("Unsupported or unknown relocation type {} in reloc at address 0x{:08X} in section {}.\n",
fmt::print(stderr, "Unsupported or unknown relocation type {} in reloc at address 0x{:08X} in section {}.\n",
(int)cur_reloc.type, cur_reloc.address, cur_section.name);
return {};
}
@ -519,7 +744,7 @@ N64Recomp::Context build_mod_context(const N64Recomp::Context& input_context, bo
// to the event symbol, creating the event symbol if necessary.
if (target_section.name == N64Recomp::EventSectionName) {
if (cur_reloc.type != N64Recomp::RelocType::R_MIPS_26) {
fmt::print("Symbol {} is an event and cannot have its address taken.\n",
fmt::print(stderr, "Symbol {} is an event and cannot have its address taken.\n",
cur_section.name);
return {};
}
@ -527,7 +752,7 @@ N64Recomp::Context build_mod_context(const N64Recomp::Context& input_context, bo
uint32_t target_function_vram = cur_reloc.target_section_offset + target_section.ram_addr;
size_t target_function_index = input_context.find_function_by_vram_section(target_function_vram, cur_reloc.target_section);
if (target_function_index == (size_t)-1) {
fmt::print("Internal error: Failed to find event symbol in section {} with offset 0x{:08X} (vram 0x{:08X}). Please report this issue.\n",
fmt::print(stderr, "Internal error: Failed to find event symbol in section {} with offset 0x{:08X} (vram 0x{:08X}). Please report this issue.\n",
target_section.name, cur_reloc.target_section_offset, target_function_vram);
return {};
}
@ -556,7 +781,7 @@ N64Recomp::Context build_mod_context(const N64Recomp::Context& input_context, bo
// to the import symbol, creating the import symbol if necessary.
else if (target_section.name.starts_with(N64Recomp::ImportSectionPrefix)) {
if (cur_reloc.type != N64Recomp::RelocType::R_MIPS_26) {
fmt::print("Symbol {} is an import and cannot have its address taken.\n",
fmt::print(stderr, "Symbol {} is an import and cannot have its address taken.\n",
cur_section.name);
return {};
}
@ -564,7 +789,7 @@ N64Recomp::Context build_mod_context(const N64Recomp::Context& input_context, bo
uint32_t target_function_vram = cur_reloc.target_section_offset + target_section.ram_addr;
size_t target_function_index = input_context.find_function_by_vram_section(target_function_vram, cur_reloc.target_section);
if (target_function_index == (size_t)-1) {
fmt::print("Internal error: Failed to find import symbol in section {} with offset 0x{:08X} (vram 0x{:08X}). Please report this issue.\n",
fmt::print(stderr, "Internal error: Failed to find import symbol in section {} with offset 0x{:08X} (vram 0x{:08X}). Please report this issue.\n",
target_section.name, cur_reloc.target_section_offset, target_function_vram);
return {};
}
@ -575,7 +800,7 @@ N64Recomp::Context build_mod_context(const N64Recomp::Context& input_context, bo
std::string dependency_name = target_section.name.substr(N64Recomp::ImportSectionPrefix.size());
size_t dependency_index;
if (!ret.find_dependency(dependency_name, dependency_index)) {
fmt::print("Failed to import function {} from mod {} as the mod is not a registered dependency.\n",
fmt::print(stderr, "Failed to import function {} from mod {} as the mod is not a registered dependency.\n",
target_function.name, dependency_name);
return {};
}
@ -603,7 +828,7 @@ N64Recomp::Context build_mod_context(const N64Recomp::Context& input_context, bo
uint32_t target_rom_to_ram = target_section.ram_addr - target_section.rom_addr;
bool is_noload = target_section.rom_addr == (uint32_t)-1;
if (!is_noload && target_rom_to_ram != cur_rom_to_ram) {
fmt::print("Reloc at address 0x{:08X} in section {} points to a different section.\n",
fmt::print(stderr, "Reloc at address 0x{:08X} in section {} points to a different section.\n",
cur_reloc.address, cur_section.name);
return {};
}
@ -630,7 +855,7 @@ N64Recomp::Context build_mod_context(const N64Recomp::Context& input_context, bo
uint16_t input_section_index = reloc.target_section;
auto find_it = input_section_to_output_section.find(input_section_index);
if (find_it == input_section_to_output_section.end()) {
fmt::print("Reloc at address 0x{:08X} references section {}, which didn't get mapped to an output section\n",
fmt::print(stderr, "Reloc at address 0x{:08X} references section {}, which didn't get mapped to an output section\n",
reloc.address, input_context.sections[input_section_index].name);
return {};
}
@ -652,13 +877,140 @@ N64Recomp::Context build_mod_context(const N64Recomp::Context& input_context, bo
return ret;
}
bool create_mod_zip(const std::filesystem::path& output_dir, const ModConfig& config) {
std::filesystem::path output_path = output_dir / (config.manifest.mod_id + "-" + config.manifest.version_string + ".nrm");
#ifdef _WIN32
std::filesystem::path temp_zip_path = output_path;
temp_zip_path.replace_extension(".zip");
std::string command_string = fmt::format("powershell -command Compress-Archive -Force -CompressionLevel Optimal -DestinationPath \"{}\" -Path \"{}\",\"{}\",\"{}\"",
temp_zip_path.string(), (output_dir / symbol_filename).string(), (output_dir / binary_filename).string(), (output_dir / manifest_filename).string());
for (const auto& cur_file : config.inputs.additional_files) {
command_string += fmt::format(",\"{}\"", cur_file.string());
}
STARTUPINFOA si{};
PROCESS_INFORMATION pi{};
ZeroMemory( &si, sizeof(si) );
si.cb = sizeof(si);
ZeroMemory( &pi, sizeof(pi) );
std::vector<char> command_string_buffer;
command_string_buffer.resize(command_string.size() + 1);
std::copy(command_string.begin(), command_string.end(), command_string_buffer.begin());
command_string_buffer[command_string.size()] = '\x00';
if (!CreateProcessA(NULL, command_string_buffer.data(), NULL, NULL, FALSE, 0, NULL, NULL, &si, &pi)) {
fmt::print(stderr, "Process creation failed {}\n", GetLastError());
return false;
}
WaitForSingleObject(pi.hProcess, INFINITE);
DWORD ec;
GetExitCodeProcess(pi.hProcess, &ec);
CloseHandle(pi.hProcess);
CloseHandle(pi.hThread);
if (ec != EXIT_SUCCESS) {
fmt::print(stderr, "Compress-Archive failed with exit code {}\n", ec);
return false;
}
std::error_code rename_ec;
std::filesystem::rename(temp_zip_path, output_path, rename_ec);
if (rename_ec != std::error_code{}) {
fmt::print(stderr, "Failed to rename temporary zip to output path\n");
return false;
}
#else
std::string args_string{};
std::vector<size_t> arg_positions{};
// Adds an argument with a null terminator to args_string, which is used as a buffer to hold null terminated arguments.
// Also adds the argument's offset into the string into arg_positions for creating the array of character pointers for the exec.
auto add_arg = [&args_string, &arg_positions](const std::string& arg){
arg_positions.emplace_back(args_string.size());
args_string += (arg + '\x00');
};
add_arg("zip"); // The program name (argv[0]).
add_arg("-q"); // Quiet mode.
add_arg("-9"); // Maximum compression level.
add_arg("-MM"); // Error if any files aren't found.
add_arg("-j"); // Junk the paths (store just as the provided filename).
add_arg("-T"); // Test zip integrity.
add_arg(output_path.string());
add_arg((output_dir / symbol_filename).string());
add_arg((output_dir / binary_filename).string());
add_arg((output_dir / manifest_filename).string());
// Add arguments for every additional file in the archive.
for (const auto& cur_file : config.inputs.additional_files) {
add_arg(cur_file.string());
}
// Build the argument char* array in a vector.
std::vector<char*> arg_pointers{};
for (size_t arg_index = 0; arg_index < arg_positions.size(); arg_index++) {
arg_pointers.emplace_back(args_string.data() + arg_positions[arg_index]);
}
// Termimate the argument list with a null pointer.
arg_pointers.emplace_back(nullptr);
// Delete the output file if it exists already.
std::filesystem::remove(output_path);
// Fork-exec to run zip.
pid_t pid = fork();
if (pid == -1) {
fmt::print(stderr, "Failed to run \"zip\"\n");
return false;
}
else if (pid == 0) {
// This is the child process, so exec zip with the arguments.
execvp(arg_pointers[0], arg_pointers.data());
}
else {
// This is the parent process, so wait for the child process to complete and check its exit code.
int status;
if (waitpid(pid, &status, 0) == (pid_t)-1) {
fmt::print(stderr, "Waiting for \"zip\" failed\n");
return false;
}
if (status != EXIT_SUCCESS) {
fmt::print(stderr, "\"zip\" failed with exit code {}\n", status);
return false;
}
}
#endif
return true;
}
int main(int argc, const char** argv) {
if (argc != 2) {
fmt::print("Usage: {} [mod toml]\n", argv[0]);
if (argc != 3) {
fmt::print("Usage: {} [mod toml] [output folder]\n", argv[0]);
return EXIT_SUCCESS;
}
bool config_good;
std::filesystem::path output_dir{ argv[2] };
if (!std::filesystem::exists(output_dir)) {
fmt::print(stderr, "Specified output folder does not exist!\n");
return EXIT_FAILURE;
}
if (!std::filesystem::is_directory(output_dir)) {
fmt::print(stderr, "Specified output folder is not a folder!\n");
return EXIT_FAILURE;
}
ModConfig config = parse_mod_config(argv[1], config_good);
if (!config_good) {
@ -673,7 +1025,7 @@ int main(int argc, const char** argv) {
// Create a new temporary context to read the function reference symbol file into, since it's the same format as the recompilation symbol file.
std::vector<uint8_t> dummy_rom{};
N64Recomp::Context reference_context{};
if (!N64Recomp::Context::from_symbol_file(config.func_reference_syms_file_path, std::move(dummy_rom), reference_context, false)) {
if (!N64Recomp::Context::from_symbol_file(config.inputs.func_reference_syms_file_path, std::move(dummy_rom), reference_context, false)) {
fmt::print(stderr, "Failed to load provided function reference symbol file\n");
return EXIT_FAILURE;
}
@ -685,7 +1037,7 @@ int main(int argc, const char** argv) {
}
}
for (const std::filesystem::path& cur_data_sym_path : config.data_reference_syms_file_paths) {
for (const std::filesystem::path& cur_data_sym_path : config.inputs.data_reference_syms_file_paths) {
if (!context.read_data_reference_syms(cur_data_sym_path)) {
fmt::print(stderr, "Failed to load provided data reference symbol file: {}\n", cur_data_sym_path.string());
return EXIT_FAILURE;
@ -693,7 +1045,7 @@ int main(int argc, const char** argv) {
}
// Copy the dependencies from the config into the context.
context.add_dependencies(config.dependencies);
context.add_dependencies(config.manifest.dependencies);
N64Recomp::ElfParsingConfig elf_config {
.bss_section_suffix = {},
@ -707,7 +1059,7 @@ int main(int argc, const char** argv) {
};
bool dummy_found_entrypoint;
N64Recomp::DataSymbolMap dummy_syms_map;
bool elf_good = N64Recomp::Context::from_elf_file(config.elf_path, context, elf_config, false, dummy_syms_map, dummy_found_entrypoint);
bool elf_good = N64Recomp::Context::from_elf_file(config.inputs.elf_path, context, elf_config, false, dummy_syms_map, dummy_found_entrypoint);
if (!elf_good) {
fmt::print(stderr, "Failed to parse mod elf\n");
@ -727,11 +1079,30 @@ int main(int argc, const char** argv) {
return EXIT_FAILURE;
}
std::ofstream output_syms_file{ config.output_syms_path, std::ios::binary };
output_syms_file.write(reinterpret_cast<const char*>(symbols_bin.data()), symbols_bin.size());
std::filesystem::path output_syms_path = output_dir / symbol_filename;
std::filesystem::path output_binary_path = output_dir / binary_filename;
std::filesystem::path output_manifest_path = output_dir / manifest_filename;
std::ofstream output_binary_file{ config.output_binary_path, std::ios::binary };
output_binary_file.write(reinterpret_cast<const char*>(mod_context.rom.data()), mod_context.rom.size());
// Write the symbol file.
{
std::ofstream output_syms_file{ output_syms_path, std::ios::binary };
output_syms_file.write(reinterpret_cast<const char*>(symbols_bin.data()), symbols_bin.size());
}
// Write the binary file.
{
std::ofstream output_binary_file{ output_binary_path, std::ios::binary };
output_binary_file.write(reinterpret_cast<const char*>(mod_context.rom.data()), mod_context.rom.size());
}
// Write the manifest.
write_manifest(output_manifest_path, config.manifest);
// Create the zip.
if (!create_mod_zip(output_dir, config)) {
fmt::print(stderr, "Failed to create mod file.\n");
return EXIT_FAILURE;
}
return EXIT_SUCCESS;
}

99
include/n64recomp.h
View file

@ -70,9 +70,9 @@ namespace N64Recomp {
constexpr std::string_view ImportSectionPrefix = ".recomp_import.";
constexpr std::string_view CallbackSectionPrefix = ".recomp_callback.";
// Special mod names.
constexpr std::string_view ModSelf = ".";
constexpr std::string_view ModBaseRecomp = "*";
// Special dependency names.
constexpr std::string_view DependencySelf = ".";
constexpr std::string_view DependencyBaseRecomp = "*";
struct Section {
uint32_t rom_addr = 0;
@ -128,13 +128,6 @@ namespace N64Recomp {
extern const std::unordered_set<std::string> ignored_funcs;
extern const std::unordered_set<std::string> renamed_funcs;
struct Dependency {
uint8_t major_version;
uint8_t minor_version;
uint8_t patch_version;
std::string mod_id;
};
struct ImportSymbol {
ReferenceSymbol base;
size_t dependency_index;
@ -202,8 +195,6 @@ namespace N64Recomp {
//// Mod dependencies and their symbols
//// Imported values
// List of dependencies.
std::vector<Dependency> dependencies;
// Mapping of dependency name to dependency index.
std::unordered_map<std::string, size_t> dependencies_by_name;
// List of symbols imported from dependencies.
@ -235,54 +226,55 @@ namespace N64Recomp {
Context() = default;
bool add_dependency(const std::string& id, uint8_t major_version, uint8_t minor_version, uint8_t patch_version) {
bool add_dependency(const std::string& id) {
if (dependencies_by_name.contains(id)) {
return false;
}
size_t dependency_index = dependencies.size();
dependencies.emplace_back(N64Recomp::Dependency {
.major_version = major_version,
.minor_version = minor_version,
.patch_version = patch_version,
.mod_id = id
});
size_t dependency_index = dependencies_by_name.size();
dependencies_by_name.emplace(id, dependency_index);
dependency_events_by_name.resize(dependencies.size());
dependency_imports_by_name.resize(dependencies.size());
dependency_events_by_name.resize(dependencies_by_name.size());
dependency_imports_by_name.resize(dependencies_by_name.size());
return true;
}
bool add_dependencies(const std::vector<Dependency>& new_dependencies) {
dependencies.reserve(dependencies.size() + new_dependencies.size());
bool add_dependencies(const std::vector<std::string>& new_dependencies) {
dependencies_by_name.reserve(dependencies_by_name.size() + new_dependencies.size());
// Check if any of the dependencies already exist and fail if so.
for (const Dependency& dep : new_dependencies) {
if (dependencies_by_name.contains(dep.mod_id)) {
for (const std::string& dep : new_dependencies) {
if (dependencies_by_name.contains(dep)) {
return false;
}
}
for (const Dependency& dep : new_dependencies) {
size_t dependency_index = dependencies.size();
dependencies.emplace_back(dep);
dependencies_by_name.emplace(dep.mod_id, dependency_index);
for (const std::string& dep : new_dependencies) {
size_t dependency_index = dependencies_by_name.size();
dependencies_by_name.emplace(dep, dependency_index);
}
dependency_events_by_name.resize(dependencies.size());
dependency_imports_by_name.resize(dependencies.size());
dependency_events_by_name.resize(dependencies_by_name.size());
dependency_imports_by_name.resize(dependencies_by_name.size());
return true;
}
bool find_dependency(const std::string& mod_id, size_t& dependency_index) {
auto find_it = dependencies_by_name.find(mod_id);
if (find_it == dependencies_by_name.end()) {
return false;
if (find_it != dependencies_by_name.end()) {
dependency_index = find_it->second;
}
else {
// Handle special dependency names.
if (mod_id == DependencySelf || mod_id == DependencyBaseRecomp) {
add_dependency(mod_id);
dependency_index = dependencies_by_name[mod_id];
}
else {
return false;
}
}
dependency_index = find_it->second;
return true;
}
@ -419,7 +411,7 @@ namespace N64Recomp {
}
bool find_import_symbol(const std::string& symbol_name, size_t dependency_index, SymbolReference& ref_out) const {
if (dependency_index >= dependencies.size()) {
if (dependency_index >= dependencies_by_name.size()) {
return false;
}
@ -467,7 +459,7 @@ namespace N64Recomp {
}
bool add_dependency_event(const std::string& event_name, size_t dependency_index, size_t& dependency_event_index) {
if (dependency_index >= dependencies.size()) {
if (dependency_index >= dependencies_by_name.size()) {
return false;
}
@ -535,21 +527,40 @@ namespace N64Recomp {
FunctionOutOfBounds,
};
ModSymbolsError parse_mod_symbols(std::span<const char> data, std::span<const uint8_t> binary, const std::unordered_map<uint32_t, uint16_t>& sections_by_vrom, const Context& reference_context, Context& context_out);
ModSymbolsError parse_mod_symbols(std::span<const char> data, std::span<const uint8_t> binary, const std::unordered_map<uint32_t, uint16_t>& sections_by_vrom, Context& context_out);
std::vector<uint8_t> symbols_to_bin_v1(const Context& mod_context);
inline bool validate_mod_name(std::string_view str) {
// Disallow mod names with a colon in them, since you can't specify that in a dependency string orin callbacks.
for (char c : str) {
if (c == ':') {
inline bool validate_mod_id(std::string_view str) {
// Disallow empty ids.
if (str.size() == 0) {
return false;
}
// Allow special dependency ids.
if (str == N64Recomp::DependencySelf || str == N64Recomp::DependencyBaseRecomp) {
return true;
}
// These following rules basically describe C identifiers. There's no specific reason to enforce them besides colon (currently),
// so this is just to prevent "weird" mod ids.
// Check the first character, which must be alphabetical or an underscore.
if (!isalpha(str[0]) && str[0] != '_') {
return false;
}
// Check the remaining characters, which can be alphanumeric or underscore.
for (char c : str.substr(1)) {
if (!isalnum(c) && c != '_') {
return false;
}
}
return true;
}
inline bool validate_mod_name(const std::string& str) {
return validate_mod_name(std::string_view{str});
inline bool validate_mod_id(const std::string& str) {
return validate_mod_id(std::string_view{str});
}
}

106
src/main.cpp
View file

@ -559,6 +559,16 @@ int main(int argc, char** argv) {
"#include \"funcs.h\"\n"
"\n",
config.recomp_include);
// Print the extern for the base event index and the define to rename it if exports are allowed.
if (config.allow_exports) {
fmt::print(current_output_file,
"extern uint32_t builtin_base_event_index;\n"
"#define base_event_index builtin_base_event_index\n"
"\n"
);
}
cur_file_function_count = 0;
output_file_count++;
};
@ -571,11 +581,86 @@ int main(int argc, char** argv) {
"#include \"funcs.h\"\n"
"\n",
config.recomp_include);
// Print the extern for the base event index and the define to rename it if exports are allowed.
if (config.allow_exports) {
fmt::print(current_output_file,
"extern uint32_t builtin_base_event_index;\n"
"#define base_event_index builtin_base_event_index\n"
"\n"
);
}
}
else if (config.functions_per_output_file > 1) {
open_new_output_file();
}
std::unordered_map<size_t, size_t> function_index_to_event_index{};
// If exports are enabled, scan all the relocs and modify ones that point to an event function.
if (config.allow_exports) {
// First, find the event section by scanning for a section with the special name.
bool event_section_found = false;
size_t event_section_index = 0;
uint32_t event_section_vram = 0;
for (size_t section_index = 0; section_index < context.sections.size(); section_index++) {
const auto& section = context.sections[section_index];
if (section.name == N64Recomp::EventSectionName) {
event_section_found = true;
event_section_index = section_index;
event_section_vram = section.ram_addr;
break;
}
}
// If an event section was found, proceed with the reloc scanning.
if (event_section_found) {
for (auto& section : context.sections) {
for (auto& reloc : section.relocs) {
// Event symbols aren't reference symbols, since they come from the elf itself.
// Therefore, skip reference symbol relocs.
if (reloc.reference_symbol) {
continue;
}
// Check if the reloc points to the event section.
if (reloc.target_section == event_section_index) {
// It does, so find the function it's pointing at.
size_t func_index = context.find_function_by_vram_section(reloc.target_section_offset + event_section_vram, event_section_index);
if (func_index == (size_t)-1) {
exit_failure(fmt::format("Failed to find event function with vram {}.\n", reloc.target_section_offset + event_section_vram));
}
// Ensure the reloc is a MIPS_R_26 one before modifying it, since those are the only type allowed to reference
if (reloc.type != N64Recomp::RelocType::R_MIPS_26) {
const auto& function = context.functions[func_index];
exit_failure(fmt::format("Function {} is an import and cannot have its address taken.\n",
function.name));
}
// Check if this function has been assigned an event index already, and assign it if not.
size_t event_index;
auto find_event_it = function_index_to_event_index.find(func_index);
if (find_event_it != function_index_to_event_index.end()) {
event_index = find_event_it->second;
}
else {
event_index = function_index_to_event_index.size();
function_index_to_event_index.emplace(func_index, event_index);
}
// Modify the reloc's fields accordingly.
reloc.target_section_offset = 0;
reloc.symbol_index = event_index;
reloc.target_section = N64Recomp::SectionEvent;
reloc.reference_symbol = true;
}
}
}
}
}
std::vector<size_t> export_function_indices{};
bool failed_strict_mode = false;
@ -840,19 +925,36 @@ int main(int argc, char** argv) {
fmt::print(overlay_file, "}};\n");
if (config.allow_exports) {
// Emit the exported function table.
fmt::print(overlay_file,
"\n"
"static FunctionExport export_table[] = {{\n"
);
for (size_t func_index : export_function_indices) {
const auto& func = context.functions[func_index];
fmt::print(overlay_file, " {{ \"{}\", 0x{:08X} }},\n", func.name, func.vram);
}
// Add a dummy element at the end to ensure the array has a valid length because C doesn't allow zero-size arrays.
fmt::print(overlay_file, " {{ NULL, 0 }}\n");
fmt::print(overlay_file, "}};\n");
// Emit the event table.
std::vector<size_t> functions_by_event{};
functions_by_event.resize(function_index_to_event_index.size());
for (auto [func_index, event_index] : function_index_to_event_index) {
functions_by_event[event_index] = func_index;
}
fmt::print(overlay_file,
"\n"
"static const char* event_names[] = {{\n"
);
for (size_t func_index : functions_by_event) {
const auto& func = context.functions[func_index];
fmt::print(overlay_file, " \"{}\",\n", func.name);
}
// Add a dummy element at the end to ensure the array has a valid length because C doesn't allow zero-size arrays.
fmt::print(overlay_file, " NULL\n");
fmt::print(overlay_file, "}};\n");
}
}

55
src/mod_symbols.cpp
View file

@ -49,9 +49,6 @@ struct RelocV1 {
};
struct DependencyV1 {
uint8_t major_version;
uint8_t minor_version;
uint8_t patch_version;
uint8_t reserved;
uint32_t mod_id_start;
uint32_t mod_id_size;
@ -143,7 +140,6 @@ bool parse_v1(std::span<const char> data, const std::unordered_map<uint32_t, uin
// TODO add proper creation methods for the remaining vectors and change these to reserves instead.
mod_context.sections.resize(num_sections); // Add method
mod_context.dependencies.reserve(num_dependencies);
mod_context.dependencies_by_name.reserve(num_dependencies);
mod_context.import_symbols.reserve(num_imports);
mod_context.dependency_events.reserve(num_dependency_events);
@ -234,6 +230,7 @@ bool parse_v1(std::span<const char> data, const std::unordered_map<uint32_t, uin
if (reloc_target_section >= mod_context.sections.size()) {
printf("Reloc %zu in section %zu references local section %u, but only %zu exist\n",
reloc_index, section_index, reloc_target_section, mod_context.sections.size());
return false;
}
}
else {
@ -269,10 +266,11 @@ bool parse_v1(std::span<const char> data, const std::unordered_map<uint32_t, uin
if (mod_id_start + mod_id_size > string_data_size) {
printf("Dependency %zu has a name start of %u and size of %u, which extend beyond the string data's total size of %zu\n",
dependency_index, mod_id_start, mod_id_size, string_data_size);
return false;
}
std::string_view mod_id{ string_data + mod_id_start, string_data + mod_id_start + mod_id_size };
mod_context.add_dependency(std::string{mod_id}, dependency_in.major_version, dependency_in.minor_version, dependency_in.patch_version);
mod_context.add_dependency(std::string{mod_id});
}
const ImportV1* imports = reinterpret_data<ImportV1>(data, offset, num_imports);
@ -290,11 +288,13 @@ bool parse_v1(std::span<const char> data, const std::unordered_map<uint32_t, uin
if (name_start + name_size > string_data_size) {
printf("Import %zu has a name start of %u and size of %u, which extend beyond the string data's total size of %zu\n",
import_index, name_start, name_size, string_data_size);
return false;
}
if (dependency_index >= num_dependencies) {
printf("Import %zu belongs to dependency %u, but only %zu dependencies were specified\n",
import_index, dependency_index, num_dependencies);
return false;
}
std::string_view import_name{ string_data + name_start, string_data + name_start + name_size };
@ -317,6 +317,7 @@ bool parse_v1(std::span<const char> data, const std::unordered_map<uint32_t, uin
if (name_start + name_size > string_data_size) {
printf("Dependency event %zu has a name start of %u and size of %u, which extend beyond the string data's total size of %zu\n",
dependency_event_index, name_start, name_size, string_data_size);
return false;
}
std::string_view dependency_event_name{ string_data + name_start, string_data + name_start + name_size };
@ -355,15 +356,29 @@ bool parse_v1(std::span<const char> data, const std::unordered_map<uint32_t, uin
if (func_index >= mod_context.functions.size()) {
printf("Export %zu has a function index of %u, but the symbol file only has %zu functions\n",
export_index, func_index, mod_context.functions.size());
return false;
}
if (name_start + name_size > string_data_size) {
printf("Export %zu has a name start of %u and size of %u, which extend beyond the string data's total size of %zu\n",
export_index, name_start, name_size, string_data_size);
return false;
}
std::string_view export_name_view{ string_data + name_start, string_data + name_start + name_size };
std::string export_name{export_name_view};
if (!mod_context.functions[func_index].name.empty()) {
printf("Function %u is exported twice (%s and %s)\n",
func_index, mod_context.functions[func_index].name.c_str(), export_name.c_str());
return false;
}
// Add the function to the exported function list.
mod_context.exported_funcs[export_index] = func_index;
// Set the function's name to the export name.
mod_context.functions[func_index].name = std::move(export_name);
}
const CallbackV1* callbacks = reinterpret_data<CallbackV1>(data, offset, num_callbacks);
@ -380,15 +395,18 @@ bool parse_v1(std::span<const char> data, const std::unordered_map<uint32_t, uin
if (dependency_event_index >= num_dependency_events) {
printf("Callback %zu is connected to dependency event %u, but only %zu dependency events were specified\n",
callback_index, dependency_event_index, num_dependency_events);
return false;
}
if (function_index >= mod_context.functions.size()) {
printf("Callback %zu uses function %u, but only %zu functions were specified\n",
callback_index, function_index, mod_context.functions.size());
return false;
}
if (!mod_context.add_callback(dependency_event_index, function_index)) {
printf("Failed to add callback %zu\n", callback_index);
return false;
}
}
@ -406,6 +424,7 @@ bool parse_v1(std::span<const char> data, const std::unordered_map<uint32_t, uin
if (name_start + name_size > string_data_size) {
printf("Event %zu has a name start of %u and size of %u, which extend beyond the string data's total size of %zu\n",
event_index, name_start, name_size, string_data_size);
return false;
}
std::string_view import_name{ string_data + name_start, string_data + name_start + name_size };
@ -416,13 +435,11 @@ bool parse_v1(std::span<const char> data, const std::unordered_map<uint32_t, uin
return offset == data.size();
}
N64Recomp::ModSymbolsError N64Recomp::parse_mod_symbols(std::span<const char> data, std::span<const uint8_t> binary, const std::unordered_map<uint32_t, uint16_t>& sections_by_vrom, const Context& reference_context, Context& mod_context_out) {
N64Recomp::ModSymbolsError N64Recomp::parse_mod_symbols(std::span<const char> data, std::span<const uint8_t> binary, const std::unordered_map<uint32_t, uint16_t>& sections_by_vrom, Context& mod_context_out) {
size_t offset = 0;
mod_context_out = {};
const FileHeader* header = reinterpret_data<FileHeader>(data, offset);
mod_context_out.import_reference_context(reference_context);
if (header == nullptr) {
return ModSymbolsError::NotASymbolFile;
}
@ -485,7 +502,7 @@ std::vector<uint8_t> N64Recomp::symbols_to_bin_v1(const N64Recomp::Context& cont
vec_put(ret, &header);
size_t num_dependencies = context.dependencies.size();
size_t num_dependencies = context.dependencies_by_name.size();
size_t num_imported_funcs = context.import_symbols.size();
size_t num_dependency_events = context.dependency_events.size();
@ -512,15 +529,24 @@ std::vector<uint8_t> N64Recomp::symbols_to_bin_v1(const N64Recomp::Context& cont
// Build the string data from the exports and imports.
size_t strings_start = ret.size();
// Order the dependencies by their index. This isn't necessary, but it makes the dependency name order
// in the symbol file match the indices of the dependencies makes debugging easier.
std::vector<std::string> dependencies_ordered{};
dependencies_ordered.resize(context.dependencies_by_name.size());
for (const auto& [dependency, dependency_index] : context.dependencies_by_name) {
dependencies_ordered[dependency_index] = dependency;
}
// Track the start of every dependency's name in the string data.
std::vector<uint32_t> dependency_name_positions{};
dependency_name_positions.resize(num_dependencies);
for (size_t dependency_index = 0; dependency_index < num_dependencies; dependency_index++) {
const Dependency& dependency = context.dependencies[dependency_index];
const std::string& dependency = dependencies_ordered[dependency_index];
dependency_name_positions[dependency_index] = static_cast<uint32_t>(ret.size() - strings_start);
vec_put(ret, dependency.mod_id);
vec_put(ret, dependency);
}
// Track the start of every imported function's name in the string data.
@ -637,14 +663,11 @@ std::vector<uint8_t> N64Recomp::symbols_to_bin_v1(const N64Recomp::Context& cont
// Write the dependencies.
for (size_t dependency_index = 0; dependency_index < num_dependencies; dependency_index++) {
const Dependency& dependency = context.dependencies[dependency_index];
const std::string& dependency = dependencies_ordered[dependency_index];
DependencyV1 dependency_out {
.major_version = dependency.major_version,
.minor_version = dependency.minor_version,
.patch_version = dependency.patch_version,
.mod_id_start = dependency_name_positions[dependency_index],
.mod_id_size = static_cast<uint32_t>(dependency.mod_id.size())
.mod_id_size = static_cast<uint32_t>(dependency.size())
};
vec_put(ret, &dependency_out);

4
src/recompilation.cpp
View file

@ -242,11 +242,11 @@ bool process_instruction(const N64Recomp::Context& context, const N64Recomp::Fun
if (reloc_section == N64Recomp::SectionEvent) {
needs_link_branch = link_branch;
if (indent) {
if (!print_unconditional_branch(" recomp_trigger_event(rdram, ctx, event_indices[{}])", reloc_reference_symbol)) {
if (!print_unconditional_branch(" recomp_trigger_event(rdram, ctx, base_event_index + {})", reloc_reference_symbol)) {
return false;
}
} else {
if (!print_unconditional_branch("recomp_trigger_event(rdram, ctx, event_indices[{}])", reloc_reference_symbol)) {
if (!print_unconditional_branch("recomp_trigger_event(rdram, ctx, base_event_index + {})", reloc_reference_symbol)) {
return false;
}
}