Skip to content

Commit

Permalink
Merge pull request #12 from moreal/PDX-369
Browse files Browse the repository at this point in the history 
PDX-395: Refactor initialization process
  • Loading branch information
@moreal
moreal authored Mar 15, 2024
2 parents e09438e + e5e10cb commit 46fbb43
Show file tree
Hide file tree
Showing 13 changed files with 335 additions and 341 deletions.
18 changes: 15 additions & 3 deletions NineChroniclesUtilBackend.Store/Client/EmptyChronicleClient.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,24 @@ public EmptyChronicleClient(string baseUrl)
_httpClient = new HttpClient();
}

public async Task<StateResponse> GetStateByAddressAsync(string address, string? accountAddress = null)
public async Task<StateResponse> GetStateByAddressAsync(
string address, string? accountAddress = null, long? blockIndex = null)
{
var url = $"{_baseUrl}/api/states/{address}/raw";
var url = $"{_baseUrl}/api/states/{address}";
var queryParams = new List<string>();
if (accountAddress != null)
{
url += $"?account={Uri.EscapeDataString(accountAddress)}";
queryParams.Add($"account={Uri.EscapeDataString(accountAddress)}");
}

if (blockIndex is { } bi)
{
queryParams.Add($"blockIndex={bi}");
}

if (queryParams.Count > 0)
{
url += "?" + string.Join("&", queryParams);
}

var response = await _httpClient.GetAsync(url);
Expand Down
39 changes: 39 additions & 0 deletions NineChroniclesUtilBackend.Store/Initializer.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
using NineChroniclesUtilBackend.Store.Scrapper;
using NineChroniclesUtilBackend.Store.Client;
using NineChroniclesUtilBackend.Store.Services;

namespace NineChroniclesUtilBackend.Store;

public class Initializer : BackgroundService
{
private readonly MongoDbStore _store;
private readonly ArenaScrapper _scrapper;
private readonly ILogger<Initializer> _logger;
private readonly IStateService _stateService;

public Initializer(
ILogger<Initializer> logger,
ILogger<ArenaScrapper> scrapperLogger,
IStateService stateService,
MongoDbStore store,
EmptyChronicleClient client)
{
_logger = logger;
_stateService = stateService;
_store = store;
_scrapper = new ArenaScrapper(scrapperLogger, _stateService, client, _store);
}

protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
var started = DateTime.UtcNow;

if (!await _store.IsInitialized())
{
await _scrapper.ExecuteAsync();
}

var totalElapsedMinutes = DateTime.UtcNow.Subtract(started).Minutes;
_logger.LogInformation($"Finished Initializer background service. Elapsed {totalElapsedMinutes} minutes.");
}
}
2 changes: 1 addition & 1 deletion NineChroniclesUtilBackend.Store/Program.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@

builder.Services.AddSingleton<IStateService, EmptyChronicleStateService>();

builder.Services.AddHostedService<Worker>();
builder.Services.AddHostedService<Initializer>();

var host = builder.Build();
host.Run();
113 changes: 40 additions & 73 deletions NineChroniclesUtilBackend.Store/Scrapper/ArenaScrapper.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,92 +7,59 @@

namespace NineChroniclesUtilBackend.Store.Scrapper;

public class ArenaScrapper
public class ArenaScrapper(ILogger<ArenaScrapper> logger, IStateService service, EmptyChronicleClient client, MongoDbStore store)
{
private readonly ILogger<ArenaScrapper> _logger;
public readonly ScrapperResult Result = new ScrapperResult();
private readonly ILogger<ArenaScrapper> _logger = logger;

private StateGetter _stateGetter;
private EmptyChronicleClient _client;
public event EventHandler<ArenaDataCollectedEventArgs> OnDataCollected;

public ArenaScrapper(ILogger<ArenaScrapper> logger, IStateService service, EmptyChronicleClient client)
{
_stateGetter = new StateGetter(service);
_logger = logger;
_client = client;
}

protected virtual void RaiseDataCollected(ArenaData arenaData, AvatarData avatarData)
{
OnDataCollected?.Invoke(this, new ArenaDataCollectedEventArgs(arenaData, avatarData));
}
private readonly IStateService _stateService = service;
private readonly EmptyChronicleClient _client = client;
private readonly MongoDbStore _store = store;

public async Task ExecuteAsync()
{
Result.StartTime = DateTime.UtcNow;
var latestBlock = await _client.GetLatestBlock();
var stateGetter = _stateService.At(latestBlock.Index);
var roundData = await stateGetter.GetArenaRoundData(latestBlock.Index);
var arenaParticipants = await stateGetter.GetArenaParticipantsState(roundData.ChampionshipId, roundData.Round);

var roundData = await GetArenaRoundData(latestBlock.Index);

var arenaParticipants = await _stateGetter.GetArenaParticipantsState(roundData.ChampionshipId, roundData.Round);

foreach(var avatarAddress in arenaParticipants.AvatarAddresses)
await _store.WithTransaction((async (storage, ct) =>
{
var arenaData = await GetArenaData(roundData, avatarAddress);
var avatarData = await GetAvatarData(avatarAddress);

if (arenaData != null && avatarData != null)
var buffer = new List<(Address AvatarAddress, ArenaData Arena, AvatarData Avatar)>();
const int maxBufferSize = 50;
async Task FlushBufferAsync()
{
RaiseDataCollected(arenaData, avatarData);
await storage.AddArenaData(buffer.Select(x => x.Arena).ToList());
await storage.AddAvatarData(buffer.Select(x => x.Avatar).ToList());
foreach (var pair in buffer)
{
await storage.LinkAvatarWithArenaAsync(pair.AvatarAddress);
}
buffer.Clear();
}
}

Result.TotalElapsedMinutes = DateTime.UtcNow.Subtract(Result.StartTime).Minutes;
}

public async Task<ArenaSheet.RoundData> GetArenaRoundData(long index)
{
var arenaSheet = await _stateGetter.GetSheet<ArenaSheet>();
var roundData = arenaSheet.GetRoundByBlockIndex(index);

return roundData;
}
await storage.UpdateLatestBlockIndex(latestBlock.Index);
public async Task<ArenaData?> GetArenaData(ArenaSheet.RoundData roundData, Address avatarAddress)
{
try
{
var arenaScore = await _stateGetter.GetArenaScoreState(avatarAddress, roundData.ChampionshipId, roundData.Round);
var arenaInfo = await _stateGetter.GetArenaInfoState(avatarAddress, roundData.ChampionshipId, roundData.Round);
foreach (var avatarAddress in arenaParticipants.AvatarAddresses)
{
var arenaData = await stateGetter.GetArenaData(roundData, avatarAddress);
var avatarData = await stateGetter.GetAvatarData(avatarAddress);
Result.ArenaScrappedCount += 1;
return new ArenaData(arenaScore, arenaInfo, roundData, avatarAddress);
}
catch (Exception ex)
{
_logger.LogError($"An error occurred during GetArenaData: {ex.Message}");
Result.FailedArenaAddresses.Add(avatarAddress);
return null;
}
}
if (arenaData != null && avatarData != null)
{
buffer.Add((arenaData, avatarData));
}
public async Task<AvatarData?> GetAvatarData(Address avatarAddress)
{
try
{
var avatarState = await _stateGetter.GetAvatarState(avatarAddress);
var avatarItemSlotState = await _stateGetter.GetItemSlotState(avatarAddress);
var avatarRuneStates = await _stateGetter.GetRuneStates(avatarAddress);
if (buffer.Count >= maxBufferSize)
{
await FlushBufferAsync();
}
}
Result.AvatarScrappedCount += 1;
return new AvatarData(avatarState, avatarItemSlotState, avatarRuneStates);
}
catch (Exception ex)
{
_logger.LogError($"An error occurred during GetAvatarData: {ex.Message}");
Result.FailedAvatarAddresses.Add(avatarAddress);
return null;
}
if (buffer.Count > 0)
{
await FlushBufferAsync();
}
}));
}
}
72 changes: 59 additions & 13 deletions NineChroniclesUtilBackend.Store/Scrapper/StateGetter.cs
View file
Original file line number Diff line number Diff line change
@@ -1,29 +1,35 @@
using NineChroniclesUtilBackend.Store.Services;
using Libplanet.Crypto;
using Bencodex.Types;
using BTAI;
using Nekoyume.TableData;
using Nekoyume;
using Nekoyume.Action;
using Nekoyume.Model.EnumType;
using Nekoyume.Model.Item;
using Nekoyume.Model.State;
using Nekoyume.Model.Arena;
using NineChroniclesUtilBackend.Store.Models;

namespace NineChroniclesUtilBackend.Store.Scrapper;

public class StateGetter
{
private readonly ILogger<StateGetter> _logger;
private readonly IStateService _service;

public StateGetter(IStateService service)
private readonly long? _blockIndex;

public StateGetter(IStateService service, long? blockIndex)
{
_logger = LoggerFactory.Create(builder => builder.AddConsole()).CreateLogger<StateGetter>();
_service = service;
_blockIndex = blockIndex;
}

public async Task<T> GetSheet<T>()
where T : ISheet, new()
{
var sheetState = await _service.GetState(Addresses.TableSheet.Derive(typeof(T).Name));
var sheetState = await _service.GetState(Addresses.TableSheet.Derive(typeof(T).Name), _blockIndex);
if (sheetState is not Text sheetValue)
{
throw new ArgumentException(nameof(T));
Expand All @@ -38,7 +44,7 @@ public async Task<ArenaParticipants> GetArenaParticipantsState(int championshipI
{
var arenaParticipantsAddress =
ArenaParticipants.DeriveAddress(championshipId, roundId);
var state = await _service.GetState(arenaParticipantsAddress);
var state = await _service.GetState(arenaParticipantsAddress, _blockIndex);
return state switch
{
List list => new ArenaParticipants(list),
Expand All @@ -50,7 +56,7 @@ public async Task<ArenaScore> GetArenaScoreState(Address avatarAddress, int cham
{
var arenaScoreAddress =
ArenaScore.DeriveAddress(avatarAddress, championshipId, roundId);
var state = await _service.GetState(arenaScoreAddress);
var state = await _service.GetState(arenaScoreAddress, _blockIndex);
return state switch
{
List list => new ArenaScore(list),
Expand All @@ -62,7 +68,7 @@ public async Task<ArenaInformation> GetArenaInfoState(Address avatarAddress, int
{
var arenaInfoAddress =
ArenaInformation.DeriveAddress(avatarAddress, championshipId, roundId);
var state = await _service.GetState(arenaInfoAddress);
var state = await _service.GetState(arenaInfoAddress, _blockIndex);
return state switch
{
List list => new ArenaInformation(list),
Expand Down Expand Up @@ -116,7 +122,7 @@ public async Task<Inventory> GetInventoryState(Address avatarAddress)
public async Task<ItemSlotState> GetItemSlotState(Address avatarAddress)
{
var state = await _service.GetState(
ItemSlotState.DeriveAddress(avatarAddress, BattleType.Arena));
ItemSlotState.DeriveAddress(avatarAddress, BattleType.Arena), _blockIndex);
return state switch
{
List list => new ItemSlotState(list),
Expand All @@ -128,7 +134,7 @@ public async Task<ItemSlotState> GetItemSlotState(Address avatarAddress)
public async Task<List<RuneState>> GetRuneStates(Address avatarAddress)
{
var state = await _service.GetState(
RuneSlotState.DeriveAddress(avatarAddress, BattleType.Arena));
RuneSlotState.DeriveAddress(avatarAddress, BattleType.Arena), _blockIndex);
var runeSlotState = state switch
{
List list => new RuneSlotState(list),
Expand All @@ -139,7 +145,7 @@ public async Task<List<RuneState>> GetRuneStates(Address avatarAddress)
var runes = new List<RuneState>();
foreach (var runeStateAddress in runeSlotState.GetEquippedRuneSlotInfos().Select(info => RuneState.DeriveAddress(avatarAddress, info.RuneId)))
{
if (await _service.GetState(runeStateAddress) is List list)
if (await _service.GetState(runeStateAddress, _blockIndex) is List list)
{
runes.Add(new RuneState(list));
}
Expand All @@ -152,11 +158,11 @@ public async Task<List<RuneState>> GetRuneStates(Address avatarAddress)
{
try
{
return await _service.GetState(address, accountAddress);
return await _service.GetState(address, accountAddress, _blockIndex);
}
catch (HttpRequestException ex) when (ex.StatusCode == System.Net.HttpStatusCode.NotFound)
{
return await _service.GetState(address);
return await _service.GetState(address, _blockIndex);
}
}

Expand All @@ -165,13 +171,53 @@ public async Task<List<RuneState>> GetRuneStates(Address avatarAddress)
IValue? rawState;
try
{
rawState = await _service.GetState(avatarAddress, accountAddress);
rawState = await _service.GetState(avatarAddress, accountAddress, _blockIndex);
}
catch (HttpRequestException ex) when (ex.StatusCode == System.Net.HttpStatusCode.NotFound)
{
rawState = await _service.GetState(legacyAddress);
rawState = await _service.GetState(legacyAddress, _blockIndex);
}
return rawState;
}

public async Task<ArenaSheet.RoundData> GetArenaRoundData(long index)
{
var arenaSheet = await GetSheet<ArenaSheet>();
var roundData = arenaSheet.GetRoundByBlockIndex(index);

return roundData;
}

public async Task<ArenaData?> GetArenaData(ArenaSheet.RoundData roundData, Address avatarAddress)
{
try
{
var arenaScore = await GetArenaScoreState(avatarAddress, roundData.ChampionshipId, roundData.Round);
var arenaInfo = await GetArenaInfoState(avatarAddress, roundData.ChampionshipId, roundData.Round);

return new ArenaData(arenaScore, arenaInfo, roundData, avatarAddress);
}
catch (Exception ex)
{
_logger.LogError($"An error occurred during GetArenaData: {ex.Message}");
return null;
}
}

public async Task<AvatarData?> GetAvatarData(Address avatarAddress)
{
try
{
var avatarState = await GetAvatarState(avatarAddress);
var avatarItemSlotState = await GetItemSlotState(avatarAddress);
var avatarRuneStates = await GetRuneStates(avatarAddress);

return new AvatarData(avatarState, avatarItemSlotState, avatarRuneStates);
}
catch (Exception ex)
{
_logger.LogError($"An error occurred during GetAvatarData: {ex.Message}");
return null;
}
}
}
13 changes: 9 additions & 4 deletions NineChroniclesUtilBackend.Store/Services/EmptyChronicleStateService.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,14 +17,19 @@ public EmptyChronicleStateService(EmptyChronicleClient client)
this.client = client;
}

public async Task<IValue?> GetState(Address address)
public Task<IValue?> GetState(Address address, long? blockIndex=null)
{
return await GetState(address, ReservedAddresses.LegacyAccount);
return GetState(address, ReservedAddresses.LegacyAccount, blockIndex);
}

public async Task<IValue?> GetState(Address address, Address accountAddress)
public Task<IValue?[]> GetStates(Address[] addresses)
{
var result = await client.GetStateByAddressAsync(address.ToString(), accountAddress.ToString());
return Task.WhenAll(addresses.Select(addr => GetState(addr)));
}

public async Task<IValue?> GetState(Address address, Address accountAddress, long? blockIndex=null)
{
var result = await client.GetStateByAddressAsync(address.ToString(), accountAddress.ToString(), blockIndex);

if (result.Value is null)
{
Expand Down
Loading

0 comments on commit 46fbb43

Please sign in to comment.