Skip to content

Commit

Permalink
Added WIP Cache System
Browse files Browse the repository at this point in the history
  • Loading branch information
@KiritoDv
KiritoDv committed Mar 25, 2024
1 parent 81f4371 commit 2907da7
Show file tree
Hide file tree
Showing 3 changed files with 99 additions and 21 deletions.
95 changes: 76 additions & 19 deletions src/Companion.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -185,26 +185,35 @@ void Companion::ExtractNode(YAML::Node& node, std::string& name, SWrapper* binar
}

std::optional<std::shared_ptr<IParsedData>> result;
if(this->gConfig.modding) {
if(impl->SupportModdedAssets() && this->gModdedAssetPaths.contains(name)) {

auto path = fs::path(this->gConfig.moddingPath) / this->gModdedAssetPaths[name];
if(!fs::exists(path)) {
SPDLOG_ERROR("Modded asset {} not found", this->gModdedAssetPaths[name]);
return;
}
if(this->gConfig.modding && impl->SupportModdedAssets() && this->gModdedAssetPaths.contains(name)) {
auto path = fs::path(this->gConfig.moddingPath) / this->gModdedAssetPaths[name];
if(!fs::exists(path)) {
SPDLOG_ERROR("Modded asset {} not found", this->gModdedAssetPaths[name]);
return;
}

std::ifstream input(path, std::ios::binary);
std::vector<uint8_t> data = std::vector<uint8_t>( std::istreambuf_iterator( input ), {});
input.close();
std::ifstream input(path, std::ios::binary);
std::vector<uint8_t> data = std::vector<uint8_t>( std::istreambuf_iterator( input ), {});
input.close();

result = factory->get()->parse_modding(data, node);
result = impl->parse_modding(data, node);
} else if(this->gNodeIsCacheable && impl->IsCacheable()) {
if(this->gCacheData.contains(name)) {
result = std::make_shared<IParsedData>();
result->get()->FromCacheBuffer(this->gCacheData[name]);
} else {
result = factory->get()->parse(this->gRomData, node);
result = impl->parse(this->gRomData, node);
if(result.has_value()) {
auto cache = result.value()->ToCacheBuffer();
if(cache.has_value()) {
this->StoreCache(cache.value());
}
}
}
} else {
result = factory->get()->parse(this->gRomData, node);
result = impl->parse(this->gRomData, node);
}

if(!result.has_value()){
SPDLOG_ERROR("Failed to process {}", name);
return;
Expand Down Expand Up @@ -394,6 +403,46 @@ void Companion::ParseCurrentFileConfig(YAML::Node node) {
}

this->gEnablePadGen = GetSafeNode<bool>(node, "autopads", true);
this->gNodeIsCacheable = GetSafeNode<bool>(node, "cacheable", true);
}

void Companion::PrepareCache(const std::string& path) {
std::ifstream yaml(path);
const std::vector<uint8_t> data = std::vector<uint8_t>(std::istreambuf_iterator( yaml ), {});
this->gCurrentCacheHash = CalculateHash(data);

const std::string out = "torch.cache.yml";
YAML::Node root;

if(fs::exists(out)) {
root = YAML::LoadFile(out);
} else {
root = YAML::Node();
}

if(root[this->gCurrentCacheHash]) {
const auto cache = GetSafeNode<std::string>(root, this->gCurrentCacheHash);
if(cache == this->gCurrentCacheHash) {
std::ifstream file("cache/" + cache, std::ios::binary);
this->gCacheData[cache] = std::vector<uint8_t>(std::istreambuf_iterator( file ), {});
file.close();
} else {
root[path] = this->gCurrentCacheHash;
fs::remove("cache/" + cache);
}
} else {
root[path] = this->gCurrentCacheHash;
}

std::ofstream file(out, std::ios::binary);
file << root;
file.close();
}

void Companion::StoreCache(const std::vector<uint8_t>& value) {
std::ofstream file("cache/" + this->gCurrentCacheHash, std::ios::binary);
file.write(reinterpret_cast<const char*>(value.data()), value.size());
file.close();
}

void Companion::Process() {
Expand Down Expand Up @@ -570,6 +619,8 @@ void Companion::Process() {
this->gCurrentDirectory = relative(entry.path(), path).replace_extension("");
this->gCurrentFile = yamlPath;

this->PrepareCache(yamlPath);

// Set compressed file offsets and compression type
if (auto segments = root[":config"]["segments"]) {
if (segments.IsSequence() && segments.size() > 0) {
Expand Down Expand Up @@ -656,6 +707,10 @@ void Companion::Process() {
this->ParseCurrentFileConfig(root[":config"]);
}

if(this->gNodeIsCacheable) {
this->PrepareCache(yamlPath);
}

spdlog::set_pattern(regular);
SPDLOG_INFO("------------------------------------------------");
spdlog::set_pattern(line);
Expand All @@ -669,7 +724,6 @@ void Companion::Process() {
continue;
}


// Parse horizontal assets
if(assetNode["files"]){
auto segment = assetNode["segment"] ? assetNode["segment"].as<uint8_t>() : -1;
Expand Down Expand Up @@ -1079,11 +1133,14 @@ std::optional<YAML::Node> Companion::AddAsset(YAML::Node asset) {
}

auto rom = this->GetRomData();
auto factory = this->GetFactory(type)->get();
auto factory = this->GetFactory(type);

if(!factory.has_value()) {
return std::nullopt;
}

std::string output;
std::string typeId = ConvertType(type);
int index;

if(symbol != "") {
output = symbol;
Expand All @@ -1095,10 +1152,10 @@ std::optional<YAML::Node> Companion::AddAsset(YAML::Node asset) {
asset["autogen"] = true;
asset["symbol"] = output;

auto result = factory->parse(rom, asset);
auto result = this->RegisterAsset(output, asset);

if(result.has_value()){
return std::get<1>(this->RegisterAsset(output, asset).value());
return std::get<1>(result.value());
}

return std::nullopt;
Expand Down
9 changes: 9 additions & 0 deletions src/Companion.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,9 @@ class Companion {
}

void Init(ExportType type);

void PrepareCache(const std::string& string);

void Process();

bool IsOTRMode() const { return this->gConfig.otrMode; }
Expand Down Expand Up @@ -122,8 +125,10 @@ class Companion {
TorchConfig gConfig;
YAML::Node gModdingConfig;
fs::path gCurrentDirectory;
std::string gCurrentCacheHash;
std::vector<uint8_t> gRomData;
std::filesystem::path gRomPath;
bool gNodeIsCacheable;
std::shared_ptr<N64::Cartridge> gCartridge;
std::unordered_map<std::string, std::unordered_map<int32_t, std::string>> gEnums;

Expand All @@ -140,6 +145,7 @@ class Companion {
std::variant<std::vector<std::string>, std::string> gWriteOrder;
std::unordered_map<std::string, std::shared_ptr<BaseFactory>> gFactories;
std::unordered_map<std::string, std::string> gModdedAssetPaths;
std::unordered_map<std::string, std::vector<uint8_t>> gCacheData;
std::unordered_map<std::string, std::map<std::string, std::vector<WriteEntry>>> gWriteMap;
std::unordered_map<std::string, std::map<std::string, std::pair<YAML::Node, bool>>> gAssetDependencies;
std::unordered_map<std::string, std::unordered_map<uint32_t, std::tuple<std::string, YAML::Node>>> gAddrMap;
Expand All @@ -148,5 +154,8 @@ class Companion {
void ParseModdingConfig();
void ParseCurrentFileConfig(YAML::Node node);
void RegisterFactory(const std::string& type, const std::shared_ptr<BaseFactory>& factory);

void StoreCache(const std::vector<uint8_t>& value);

void ExtractNode(YAML::Node& node, std::string& name, SWrapper* binary);
};
16 changes: 14 additions & 2 deletions src/factories/BaseFactory.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,16 @@ T GetSafeNode(YAML::Node& node, const std::string& key, const T& def) {
return node[key].as<T>();
}

class IParsedData {};
class IParsedData {
public:
virtual std::optional<std::vector<uint8_t>> ToCacheBuffer() {
return std::nullopt;
}

virtual void FromCacheBuffer(std::vector<uint8_t>& buffer) {

}
};

template<typename T>
class GenericData : public IParsedData {
Expand Down Expand Up @@ -103,7 +112,10 @@ class BaseFactory {
}
virtual uint32_t GetAlignment() {
return 4;
};
}
virtual bool IsCacheable() {
return false;
}
private:
virtual std::unordered_map<ExportType, std::shared_ptr<BaseExporter>> GetExporters() = 0;
};

0 comments on commit 2907da7

Please sign in to comment.