From 94be71101cf5a81685949a6025a3870f2d798284 Mon Sep 17 00:00:00 2001 From: inspectredc Date: Thu, 9 Jan 2025 00:15:16 +0000 Subject: [PATCH] O2R Support --- CMakeLists.txt | 20 ++++++- src/Companion.cpp | 54 +++++++++++++++--- src/Companion.h | 24 +++++--- src/archive/BinaryWrapper.cpp | 3 + src/archive/BinaryWrapper.h | 19 +++++++ src/{storm => archive}/SWrapper.cpp | 28 ++++++--- src/{storm => archive}/SWrapper.h | 9 +-- src/archive/ZWrapper.cpp | 79 ++++++++++++++++++++++++++ src/archive/ZWrapper.h | 17 ++++++ src/factories/sf64/ScriptFactory.cpp | 2 +- src/factories/sf64/SkeletonFactory.cpp | 2 +- src/main.cpp | 49 ++++++++++++---- 12 files changed, 263 insertions(+), 43 deletions(-) create mode 100644 src/archive/BinaryWrapper.cpp create mode 100644 src/archive/BinaryWrapper.h rename src/{storm => archive}/SWrapper.cpp (80%) rename src/{storm => archive}/SWrapper.h (64%) create mode 100644 src/archive/ZWrapper.cpp create mode 100644 src/archive/ZWrapper.h diff --git a/CMakeLists.txt b/CMakeLists.txt index ce9dd62a..d34e285d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -164,6 +164,24 @@ FetchContent_Declare( FetchContent_MakeAvailable(spdlog) +# Link LibZip + +set(CMAKE_POLICY_DEFAULT_CMP0077 NEW) +set(BUILD_TOOLS OFF) +set(BUILD_REGRESS OFF) +set(BUILD_EXAMPLES OFF) +set(BUILD_DOC OFF) +set(BUILD_OSSFUZZ OFF) +set(BUILD_SHARED_LIBS OFF) +FetchContent_Declare( + libzip + GIT_REPOSITORY https://github.com/nih-at/libzip.git + GIT_TAG v1.10.1 + OVERRIDE_FIND_PACKAGE +) +FetchContent_MakeAvailable(libzip) +target_include_directories(${PROJECT_NAME} PRIVATE ${libzip_SOURCE_DIR}/lib ${libzip_BINARY_DIR}) + # Link TinyXML2 set(tinyxml2_BUILD_TESTING OFF) FetchContent_Declare( @@ -174,7 +192,7 @@ FetchContent_Declare( ) FetchContent_MakeAvailable(tinyxml2) -target_link_libraries(${PROJECT_NAME} PRIVATE spdlog tinyxml2 yaml-cpp storm N64Graphics BinaryTools) +target_link_libraries(${PROJECT_NAME} PRIVATE spdlog tinyxml2 yaml-cpp storm N64Graphics BinaryTools libzip::zip) if((CMAKE_SYSTEM_NAME MATCHES "Windows") AND ("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang")) include(../cmake/HandleCompilerRT.cmake) diff --git a/src/Companion.cpp b/src/Companion.cpp index d7aa11dd..4e9eefdc 100644 --- a/src/Companion.cpp +++ b/src/Companion.cpp @@ -2,7 +2,8 @@ #include "utils/Decompressor.h" #include "utils/TorchUtils.h" -#include "storm/SWrapper.h" +#include "archive/SWrapper.h" +#include "archive/ZWrapper.h" #include "spdlog/spdlog.h" #include "hj/sha1.h" @@ -1022,7 +1023,18 @@ void Companion::Process() { this->gConfig.moddingPath = modding_path; switch (this->gConfig.exporterType) { case ExportType::Binary: { - this->gConfig.outputPath = opath && opath["binary"] ? opath["binary"].as() : "generic.otr"; + std::string extension = ""; + switch (this->gConfig.otrMode) { + case ArchiveType::OTR: + extension = ".otr"; + break; + case ArchiveType::O2R: + extension = ".o2r"; + break; + default: + throw std::runtime_error("Invalid archive type for export type Binary"); + } + this->gConfig.outputPath = opath && opath["binary"] ? opath["binary"].as() : ("generic" + extension); break; } case ExportType::Header: { @@ -1138,7 +1150,24 @@ void Companion::Process() { } AudioManager::Instance = new AudioManager(); - auto wrapper = this->gConfig.exporterType == ExportType::Binary ? new SWrapper(this->gConfig.outputPath) : nullptr; + BinaryWrapper* wrapper = nullptr; + + if (this->gConfig.exporterType == ExportType::Binary) { + switch (this->gConfig.otrMode) { + case ArchiveType::OTR: + wrapper = new SWrapper(this->gConfig.outputPath); + break; + case ArchiveType::O2R: + wrapper = new ZWrapper(this->gConfig.outputPath); + break; + default: + throw std::runtime_error("Invalid archive type for export type Binary"); + } + } + + if (wrapper) { + wrapper->CreateArchive(); + } this->gCurrentWrapper = wrapper; auto vWriter = LUS::BinaryWriter(); @@ -1201,7 +1230,7 @@ void Companion::Process() { Instance = nullptr; } -void Companion::Pack(const std::string& folder, const std::string& output) { +void Companion::Pack(const std::string& folder, const std::string& output, const ArchiveType otrMode) { spdlog::set_level(spdlog::level::debug); spdlog::set_pattern("[%Y-%m-%d %H:%M:%S.%e] [%l] %v"); @@ -1225,14 +1254,25 @@ void Companion::Pack(const std::string& folder, const std::string& output) { files[entry.path().generic_string()] = data; } - auto wrapper = SWrapper(output); + std::unique_ptr wrapper; + switch (otrMode) { + case ArchiveType::OTR: + wrapper.reset(new SWrapper(output)); + break; + case ArchiveType::O2R: + wrapper.reset(new ZWrapper(output)); + break; + default: + throw std::runtime_error("Invalid archive type for export type Binary"); + } + wrapper->CreateArchive(); for(auto& [path, data] : files){ std::string normalized = path; std::replace(normalized.begin(), normalized.end(), '\\', '/'); // Remove parent folder normalized = normalized.substr(folder.length() + 1); - wrapper.CreateFile(normalized, data); + wrapper->CreateFile(normalized, data); SPDLOG_INFO("> Added {}", normalized); } @@ -1242,7 +1282,7 @@ void Companion::Pack(const std::string& folder, const std::string& output) { spdlog::set_pattern("[%Y-%m-%d %H:%M:%S.%e] [%l] %v"); SPDLOG_INFO("------------------------------------------------"); - wrapper.Close(); + wrapper->Close(); } std::optional> Companion::RegisterAsset(const std::string& name, YAML::Node& node) { diff --git a/src/Companion.h b/src/Companion.h index 749e318a..2cebae45 100644 --- a/src/Companion.h +++ b/src/Companion.h @@ -13,7 +13,7 @@ #include "utils/Decompressor.h" #include "factories/TextureFactory.h" -class SWrapper; +class BinaryWrapper; namespace fs = std::filesystem; enum class ParseMode { @@ -40,6 +40,12 @@ enum class TableMode { Append }; +enum class ArchiveType { + None, + OTR, + O2R, +}; + struct SegmentConfig { std::unordered_map global; std::unordered_map local; @@ -79,7 +85,7 @@ struct TorchConfig { std::string moddingPath; ExportType exporterType; ParseMode parseMode; - bool otrMode; + ArchiveType otrMode; bool debug; bool modding; }; @@ -103,14 +109,14 @@ class Companion { public: static Companion* Instance; - explicit Companion(std::filesystem::path rom, const bool otr, const bool debug, const bool modding = false) : gCartridge(nullptr) { + explicit Companion(std::filesystem::path rom, const ArchiveType otr, const bool debug, const bool modding = false) : gCartridge(nullptr) { this->gRomPath = rom; this->gConfig.otrMode = otr; this->gConfig.debug = debug; this->gConfig.modding = modding; } - explicit Companion(std::vector rom, const bool otr, const bool debug, const bool modding = false) : gCartridge(nullptr) { + explicit Companion(std::vector rom, const ArchiveType otr, const bool debug, const bool modding = false) : gCartridge(nullptr) { this->gRomData = rom; this->gConfig.otrMode = otr; this->gConfig.debug = debug; @@ -123,7 +129,7 @@ class Companion { void Process(); - bool IsOTRMode() const { return this->gConfig.otrMode; } + bool IsOTRMode() const { return (this->gConfig.otrMode != ArchiveType::None); } bool IsDebug() const { return this->gConfig.debug; } N64::Cartridge* GetCartridge() const { return this->gCartridge.get(); } @@ -151,13 +157,13 @@ class Companion { std::optional SearchTable(uint32_t addr); static std::string CalculateHash(const std::vector& data); - static void Pack(const std::string& folder, const std::string& output); + static void Pack(const std::string& folder, const std::string& output, const ArchiveType otrMode); std::string NormalizeAsset(const std::string& name) const; std::string RelativePath(const std::string& path) const; void RegisterCompanionFile(const std::string path, std::vector data); TorchConfig& GetConfig() { return this->gConfig; } - SWrapper* GetCurrentWrapper() { return this->gCurrentWrapper; } + BinaryWrapper* GetCurrentWrapper() { return this->gCurrentWrapper; } std::optional> RegisterAsset(const std::string& name, YAML::Node& node); std::optional AddAsset(YAML::Node asset); @@ -175,7 +181,7 @@ class Companion { std::shared_ptr gCartridge; std::unordered_map> gCourseMetadata; std::unordered_map> gEnums; - SWrapper* gCurrentWrapper; + BinaryWrapper* gCurrentWrapper; // Temporal Variables std::string gCurrentFile; @@ -206,7 +212,7 @@ class Companion { void ParseModdingConfig(); void ParseCurrentFileConfig(YAML::Node node); void RegisterFactory(const std::string& type, const std::shared_ptr& factory); - void ExtractNode(YAML::Node& node, std::string& name, SWrapper* binary); + void ExtractNode(YAML::Node& node, std::string& name, BinaryWrapper* binary); void ProcessTables(YAML::Node& rom); void LoadYAMLRecursively(const std::string &dirPath, std::vector &result, bool skipRoot); std::optional ParseNode(YAML::Node& node, std::string& name); diff --git a/src/archive/BinaryWrapper.cpp b/src/archive/BinaryWrapper.cpp new file mode 100644 index 00000000..0a4b06d7 --- /dev/null +++ b/src/archive/BinaryWrapper.cpp @@ -0,0 +1,3 @@ +#include "BinaryWrapper.h" + +BinaryWrapper::BinaryWrapper(const std::string& path) : mPath(path) {} diff --git a/src/archive/BinaryWrapper.h b/src/archive/BinaryWrapper.h new file mode 100644 index 00000000..b88e8b08 --- /dev/null +++ b/src/archive/BinaryWrapper.h @@ -0,0 +1,19 @@ +#pragma once + +#include +#include +#include + +class BinaryWrapper { +public: + BinaryWrapper() {} + BinaryWrapper(const std::string& path); + virtual ~BinaryWrapper() = default; + + virtual int32_t CreateArchive(void) = 0; + virtual bool CreateFile(const std::string& path, std::vector data) = 0; + virtual int32_t Close(void) = 0; +protected: + std::mutex mMutex; + std::string mPath; +}; diff --git a/src/storm/SWrapper.cpp b/src/archive/SWrapper.cpp similarity index 80% rename from src/storm/SWrapper.cpp rename to src/archive/SWrapper.cpp index 7b00aa00..087d4019 100644 --- a/src/storm/SWrapper.cpp +++ b/src/archive/SWrapper.cpp @@ -9,14 +9,20 @@ namespace fs = std::filesystem; SWrapper::SWrapper(const std::string& path) { - if(fs::exists(path)) { - fs::remove(path); + mPath = path; +} + +int32_t SWrapper::CreateArchive() { + if(fs::exists(mPath)) { + fs::remove(mPath); } - if(!SFileCreateArchive(path.c_str(), MPQ_CREATE_LISTFILE | MPQ_CREATE_ATTRIBUTES | MPQ_CREATE_ARCHIVE_V2, 16 * 1024, &this->hMpq)){ - SPDLOG_ERROR("Failed to create archive {} with error code {}", path, GetLastError()); - return; + if(!SFileCreateArchive(mPath.c_str(), MPQ_CREATE_LISTFILE | MPQ_CREATE_ATTRIBUTES | MPQ_CREATE_ARCHIVE_V2, 16 * 1024, &this->hMpq)){ + SPDLOG_ERROR("Failed to create archive {} with error code {}", mPath, GetLastError()); + return -1; } + + return 0; } bool SWrapper::CreateFile(const std::string& path, std::vector data) { @@ -70,10 +76,14 @@ bool SWrapper::CreateFile(const std::string& path, std::vector data) { return true; } -void SWrapper::Close() { +int32_t SWrapper::Close(void) { if(this->hMpq == nullptr) { SPDLOG_ERROR("Archive already closed"); - return; + return -1; } - SFileCloseArchive(this->hMpq); -} \ No newline at end of file + if (SFileCloseArchive(this->hMpq)) { + SPDLOG_ERROR("Error closing archive"); + return -1; + } + return 0; +} diff --git a/src/storm/SWrapper.h b/src/archive/SWrapper.h similarity index 64% rename from src/storm/SWrapper.h rename to src/archive/SWrapper.h index e713c922..338af146 100644 --- a/src/storm/SWrapper.h +++ b/src/archive/SWrapper.h @@ -3,14 +3,15 @@ #include #include #include +#include "BinaryWrapper.h" -class SWrapper { +class SWrapper : public BinaryWrapper { public: SWrapper(const std::string& path); - std::vector ReadFile(std::string path); + int32_t CreateArchive(void); bool CreateFile(const std::string& path, std::vector data); - void Close(); + int32_t Close(void); private: HANDLE hMpq{}; -}; \ No newline at end of file +}; diff --git a/src/archive/ZWrapper.cpp b/src/archive/ZWrapper.cpp new file mode 100644 index 00000000..936256e4 --- /dev/null +++ b/src/archive/ZWrapper.cpp @@ -0,0 +1,79 @@ +#include "ZWrapper.h" +#include +#include +#include + +#include "spdlog/spdlog.h" +#include + +namespace fs = std::filesystem; + +ZWrapper::ZWrapper(const std::string& path) { + mPath = path; +} + +int32_t ZWrapper::CreateArchive(void) { + int openErr; + zip_t* zip; + + if(fs::exists(mPath)) { + fs::remove(mPath); + } + + { + const std::lock_guard lock(mMutex); + zip = zip_open(mPath.c_str(), ZIP_CREATE, &openErr); + } + + if (zip == nullptr) { + zip_error_t error; + zip_error_init_with_code(&error, openErr); + SPDLOG_ERROR("Failed to create ZIP (O2R) file. Error: {}", zip_error_strerror(&error)); + zip_error_fini(&error); + return -1; + } + mZip = zip; + SPDLOG_INFO("Loaded ZIP (O2R) archive: {}", mPath.c_str()); + return 0; +} + +bool ZWrapper::CreateFile(const std::string& path, std::vector data) { + char* fileData = data.data(); + size_t fileSize = data.size(); + zip_source_t* source = zip_source_buffer(mZip, fileData, fileSize, 0); + + if (source == nullptr) { + zip_error_t* zipError = zip_get_error(mZip); + SPDLOG_ERROR("Failed to create ZIP source. Error: {}", zip_error_strerror(zipError)); + zip_source_free(source); + zip_error_fini(zipError); + return false; + } + + if (zip_file_add(mZip, path.c_str(), source, ZIP_FL_OVERWRITE | ZIP_FL_ENC_UTF_8) < 0) { + zip_error_t* zipError = zip_get_error(mZip); + SPDLOG_ERROR("Failed to add file to ZIP. Error: {}", zip_error_strerror(zipError)); + zip_source_free(source); + zip_error_fini(zipError); + return false; + } + + return true; +} + +int32_t ZWrapper::Close(void) { + int err; + { + const std::lock_guard lock(mMutex); + err = zip_close(mZip); + } + if (err < 0) { + zip_error_t* zipError = zip_get_error(mZip); + SPDLOG_ERROR("Failed to close ZIP (O2R) file. Error: {}", zip_error_strerror(zipError)); + printf("fail\n"); + zip_error_fini(zipError); + return -1; + } + + return 0; +} diff --git a/src/archive/ZWrapper.h b/src/archive/ZWrapper.h new file mode 100644 index 00000000..7cd3db32 --- /dev/null +++ b/src/archive/ZWrapper.h @@ -0,0 +1,17 @@ +#pragma once + +#include +#include +#include +#include "BinaryWrapper.h" + +class ZWrapper : public BinaryWrapper { +public: + ZWrapper(const std::string& path); + + int32_t CreateArchive(void); + bool CreateFile(const std::string& path, std::vector data); + int32_t Close(void); +private: + zip_t* mZip; +}; diff --git a/src/factories/sf64/ScriptFactory.cpp b/src/factories/sf64/ScriptFactory.cpp index 71c4772d..b3c49db6 100644 --- a/src/factories/sf64/ScriptFactory.cpp +++ b/src/factories/sf64/ScriptFactory.cpp @@ -8,7 +8,7 @@ #include #include -#include "storm/SWrapper.h" +#include "archive/SWrapper.h" #define CLAMP_MAX(val, max) (((val) < (max)) ? (val) : (max)) #define MIN(a, b) (((a) < (b)) ? (a) : (b)) diff --git a/src/factories/sf64/SkeletonFactory.cpp b/src/factories/sf64/SkeletonFactory.cpp index ff1a7623..db247e94 100644 --- a/src/factories/sf64/SkeletonFactory.cpp +++ b/src/factories/sf64/SkeletonFactory.cpp @@ -5,7 +5,7 @@ #include "utils/Decompressor.h" #include "utils/TorchUtils.h" -#include "storm/SWrapper.h" +#include "archive/SWrapper.h" #define NUM(x, w) std::dec << std::setfill(' ') << std::setw(w) << x // #define NUM_JOINT(x) std::dec << std::setfill(' ') << std::setw(5) << x diff --git a/src/main.cpp b/src/main.cpp index d3b6ffb4..53c2c521 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -15,7 +15,9 @@ int main(int argc, char *argv[]) { std::string filename; std::string target; std::string folder; - bool otrMode = false; + std::string archive; + ArchiveType otrMode = ArchiveType::None; + bool otrModeSelected = false; bool xmlMode = false; bool debug = false; @@ -28,7 +30,18 @@ int main(int argc, char *argv[]) { otr->add_flag("-v,--verbose", debug, "Verbose Debug Mode"); otr->parse_complete_callback([&] { - const auto instance = Companion::Instance = new Companion(filename, true, debug); + const auto instance = Companion::Instance = new Companion(filename, ArchiveType::OTR, debug); + instance->Init(ExportType::Binary); + }); + + /* Generate an O2R */ + const auto o2r = app.add_subcommand("o2r", "O2R - Generates an o2r\n"); + + o2r->add_option("", filename, "")->required()->check(CLI::ExistingFile); + o2r->add_flag("-v,--verbose", debug, "Verbose Debug Mode"); + + o2r->parse_complete_callback([&] { + const auto instance = Companion::Instance = new Companion(filename, ArchiveType::O2R, debug); instance->Init(ExportType::Binary); }); @@ -39,7 +52,7 @@ int main(int argc, char *argv[]) { code->add_flag("-v,--verbose", debug, "Verbose Debug Mode; adds offsets to C code"); code->parse_complete_callback([&]() { - const auto instance = Companion::Instance = new Companion(filename, false, debug); + const auto instance = Companion::Instance = new Companion(filename, ArchiveType::None, debug); instance->Init(ExportType::Code); }); @@ -49,7 +62,7 @@ int main(int argc, char *argv[]) { binary->add_option("", filename, "")->required()->check(CLI::ExistingFile); binary->parse_complete_callback([&] { - const auto instance = Companion::Instance = new Companion(filename, false, debug); + const auto instance = Companion::Instance = new Companion(filename, ArchiveType::None, debug); instance->Init(ExportType::Binary); }); @@ -57,22 +70,36 @@ int main(int argc, char *argv[]) { const auto header = app.add_subcommand("header", "Header - Generates headers only\n"); header->add_option("", filename, "")->required()->check(CLI::ExistingFile); - header->add_flag("-o,--otr", otrMode, "OTR Mode"); + header->add_flag("-o,--otr", otrModeSelected, "OTR/O2R Mode"); + + if (otrModeSelected) { + otrMode = ArchiveType::OTR; + } header->parse_complete_callback([&] { const auto instance = Companion::Instance = new Companion(filename, otrMode, debug); instance->Init(ExportType::Header); }); - /* Pack an otr from a folder */ - const auto pack = app.add_subcommand("pack", "Pack - Packs an otr from a folder\n"); + /* Pack an archive from a folder */ + const auto pack = app.add_subcommand("pack", "Pack - Packs an archive from a folder\n"); pack->add_option("", folder, "Generate OTR from a directory of assets")->required()->check(CLI::ExistingDirectory); - pack->add_option("", target, "OTR output destination")->required(); + pack->add_option("", target, "Archive output destination")->required(); + pack->add_option("", archive, "Archive type: otr or o2r")->required(); + pack->parse_complete_callback([&] { + if (archive == "otr") { + otrMode = ArchiveType::OTR; + } else if (archive == "o2r") { + otrMode = ArchiveType::O2R; + } else { + std::cout << "Invalid archive type" << std::endl; + } + if (!folder.empty()) { - Companion::Pack(folder, target); + Companion::Pack(folder, target, otrMode); } else { std::cout << "The folder is empty" << std::endl; } @@ -87,7 +114,7 @@ int main(int argc, char *argv[]) { modding_import->add_option("", filename, "")->required()->check(CLI::ExistingFile); modding_import->parse_complete_callback([&] { - const auto instance = Companion::Instance = new Companion(filename, false, debug, true); + const auto instance = Companion::Instance = new Companion(filename, ArchiveType::None, debug, true); if (mode == "code") { instance->Init(ExportType::Code); } else if (mode == "binary") { @@ -103,7 +130,7 @@ int main(int argc, char *argv[]) { modding_export->add_option("", filename, "")->required()->check(CLI::ExistingFile); modding_export->parse_complete_callback([&] { - const auto instance = Companion::Instance = new Companion(filename, false, debug); + const auto instance = Companion::Instance = new Companion(filename, ArchiveType::None, debug); if (xmlMode) { instance->Init(ExportType::XML); } else {