Merge Nine chronicles util backend.store (#9)

* Init

* Add big integer converter

* Add 0c restapi client

* Configuration

* Add state service

* Update vscode debug setting files

* Add ArenaScrapper

* Add Store service

* Implement bulk store

* Add Store, Scrapper logging

* Fix link avatar bug

* Get latest block information

.vscode/launch.json

@@ -1,15 +1,25 @@
 {
-    "version": "0.2.0",
-    "configurations": [
-        {
-          "name": "Debug NineChroniclesUtilBackend",
-          "type": "coreclr",
-          "request": "launch",
-          "program": "${workspaceFolder}/NineChroniclesUtilBackend/bin/Debug/net8.0/NineChroniclesUtilBackend.dll",
-          "cwd": "${workspaceFolder}/NineChroniclesUtilBackend",
-          "stopAtEntry": false,
-          "console": "internalConsole",
-          "preLaunchTask": "build",
-        }
-    ]
-}
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "name": "Debug NineChroniclesUtilBackend",
+      "type": "coreclr",
+      "request": "launch",
+      "program": "${workspaceFolder}/NineChroniclesUtilBackend/bin/Debug/net8.0/NineChroniclesUtilBackend.dll",
+      "cwd": "${workspaceFolder}/NineChroniclesUtilBackend",
+      "stopAtEntry": false,
+      "console": "internalConsole",
+      "preLaunchTask": "build-backend"
+    },
+    {
+      "name": "Debug NineChroniclesUtilBackend.Store",
+      "type": "coreclr",
+      "request": "launch",
+      "program": "${workspaceFolder}/NineChroniclesUtilBackend.Store/bin/Debug/net8.0/NineChroniclesUtilBackend.Store.dll",
+      "cwd": "${workspaceFolder}/NineChroniclesUtilBackend.Store",
+      "stopAtEntry": false,
+      "console": "internalConsole",
+      "preLaunchTask": "build-store"
+    }
+  ]
+}

.vscode/tasks.json

@@ -2,7 +2,7 @@
     "version": "2.0.0",
     "tasks": [
         {
-            "label": "build",
+            "label": "build-backend",
             "command": "dotnet",
             "type": "process",
             "args": [
@@ -14,7 +14,7 @@
             "problemMatcher": "$msCompile"
         },
         {
-            "label": "publish",
+            "label": "publish-backend",
             "command": "dotnet",
             "type": "process",
             "args": [
@@ -26,7 +26,7 @@
             "problemMatcher": "$msCompile"
         },
         {
-            "label": "watch",
+            "label": "watch-backend",
             "command": "dotnet",
             "type": "process",
             "args": [
@@ -36,6 +36,42 @@
                 "${workspaceFolder}/NineChroniclesUtilBackend/NineChroniclesUtilBackend.csproj"
             ],
             "problemMatcher": "$msCompile"
+        },
+        {
+            "label": "build-store",
+            "command": "dotnet",
+            "type": "process",
+            "args": [
+                "build",
+                "${workspaceFolder}/NineChroniclesUtilBackend.Store/NineChroniclesUtilBackend.Store.csproj",
+                "/property:GenerateFullPaths=true",
+                "/consoleloggerparameters:NoSummary"
+            ],
+            "problemMatcher": "$msCompile"
+        },
+        {
+            "label": "publish-store",
+            "command": "dotnet",
+            "type": "process",
+            "args": [
+                "publish",
+                "${workspaceFolder}/NineChroniclesUtilBackend.Store/NineChroniclesUtilBackend.Store.csproj",
+                "/property:GenerateFullPaths=true",
+                "/consoleloggerparameters:NoSummary"
+            ],
+            "problemMatcher": "$msCompile"
+        },
+        {
+            "label": "watch-store",
+            "command": "dotnet",
+            "type": "process",
+            "args": [
+                "watch",
+                "run",
+                "--project",
+                "${workspaceFolder}/NineChroniclesUtilBackend.Store/NineChroniclesUtilBackend.Store.csproj"
+            ],
+            "problemMatcher": "$msCompile"
         }
     ]
 }

NineChroniclesUtilBackend.Store/Client/EmptyChronicleClient.cs

@@ -0,0 +1,63 @@
+using Newtonsoft.Json;
+using NineChroniclesUtilBackend.Store.Models;
+
+namespace NineChroniclesUtilBackend.Store.Client;
+
+public class EmptyChronicleClient
+{
+    private readonly HttpClient _httpClient;
+    private readonly string _baseUrl;
+
+    public EmptyChronicleClient(string baseUrl)
+    {
+        _baseUrl = baseUrl;
+        _httpClient = new HttpClient();
+    }
+
+    public async Task<StateResponse> GetStateByAddressAsync(string address, string? accountAddress = null)
+    {
+        var url = $"{_baseUrl}/api/states/{address}/raw";
+        if (accountAddress != null)
+        {
+            url += $"?account={Uri.EscapeDataString(accountAddress)}";
+        }
+
+        var response = await _httpClient.GetAsync(url);
+
+        response.EnsureSuccessStatusCode();
+
+        var content = await response.Content.ReadAsStringAsync();
+
+        var stateResponse = JsonConvert.DeserializeObject<StateResponse>(content);
+        if (stateResponse == null)
+        {
+            throw new InvalidOperationException("StateResponse is null.");
+        }
+
+        return stateResponse;
+    }
+
+    public async Task<BlockResponse> GetLatestBlock()
+    {
+        var url = $"{_baseUrl}/api/blocks/latest";
+
+        var response = await _httpClient.GetAsync(url);
+
+        response.EnsureSuccessStatusCode();
+
+        var content = await response.Content.ReadAsStringAsync();
+
+        var stateResponse = JsonConvert.DeserializeObject<BlockResponse>(content);
+        if (stateResponse == null)
+        {
+            throw new InvalidOperationException("StateResponse is null.");
+        }
+
+        return stateResponse;
+    }
+
+    // public async Task<StateResponse> GetBlock(int index)
+    // {
+    //     return stateResponse;
+    // }
+}

NineChroniclesUtilBackend.Store/Configuration.cs

@@ -0,0 +1,8 @@
+namespace NineChroniclesUtilBackend.Store;
+
+public class Configuration
+{
+    public string EmptyChronicleBaseUrl { get; init; }
+    public string MongoDbConnectionString { get; init; }
+    public string DatabaseName { get; set; }
+}

NineChroniclesUtilBackend.Store/Events/ArenaDataCollectedEventArgs.cs

@@ -0,0 +1,15 @@
+using NineChroniclesUtilBackend.Store.Models;
+
+namespace NineChroniclesUtilBackend.Store.Events;
+
+public class ArenaDataCollectedEventArgs : EventArgs
+{
+    public ArenaData ArenaData { get; set; }
+    public AvatarData AvatarData { get; set; }
+
+    public ArenaDataCollectedEventArgs(ArenaData arenaData, AvatarData avatarData)
+    {
+        ArenaData = arenaData;
+        AvatarData = avatarData;
+    }
+}

NineChroniclesUtilBackend.Store/Models/BlockResponse.cs

@@ -0,0 +1,13 @@
+namespace NineChroniclesUtilBackend.Store.Models;
+
+public class BlockResponse
+{
+    public string Hash { get; }
+    public long Index { get; }
+
+    public BlockResponse(string hash, long index)
+    {
+        Hash = hash;
+        Index = index;
+    }
+}

NineChroniclesUtilBackend.Store/Models/ScrapperResult.cs

@@ -0,0 +1,26 @@
+using Libplanet.Crypto;
+
+namespace NineChroniclesUtilBackend.Store.Models;
+
+public class ScrapperResult
+{
+    public DateTime StartTime { get; set; }
+    public int TotalElapsedMinutes { get; set; }
+    public int AvatarScrappedCount { get; set; }
+    public int ArenaScrappedCount { get; set; }
+    public List<Address> FailedAvatarAddresses { get; } = new List<Address>();
+    public List<Address> FailedArenaAddresses { get; } = new List<Address>();
+
+    public override string ToString()
+    {
+        var failedAvatarAddresses = string.Join(", ", FailedAvatarAddresses.Select(a => a.ToString()));
+        var failedArenaAddresses = string.Join(", ", FailedArenaAddresses.Select(a => a.ToString()));
+
+        return $"StartTime: {StartTime}, " +
+                $"TotalElapsedMinutes: {TotalElapsedMinutes}, " +
+                $"AvatarScrappedCount: {AvatarScrappedCount}, " +
+                $"ArenaScrappedCount: {ArenaScrappedCount}, " +
+                $"FailedAvatarAddresses: [{failedAvatarAddresses}], " +
+                $"FailedArenaAddresses: [{failedArenaAddresses}]";
+    }
+}

NineChroniclesUtilBackend.Store/Models/State/ArenaData.cs

@@ -0,0 +1,21 @@
+using Nekoyume.Model.Arena;
+using Nekoyume.TableData;
+using Libplanet.Crypto;
+
+namespace NineChroniclesUtilBackend.Store.Models;
+
+public class ArenaData : BaseData
+{
+    public ArenaScore Score { get; }
+    public ArenaInformation Information { get; }
+    public ArenaSheet.RoundData RoundData { get; }
+    public Address AvatarAddress { get; }
+
+    public ArenaData(ArenaScore score, ArenaInformation information, ArenaSheet.RoundData roundData, Address avatarAddress)
+    {
+        Score = score;
+        Information = information;
+        RoundData = roundData;
+        AvatarAddress = avatarAddress;
+    }
+}

NineChroniclesUtilBackend.Store/Models/State/AvataData.cs

@@ -0,0 +1,17 @@
+using Nekoyume.Model.State;
+
+namespace NineChroniclesUtilBackend.Store.Models;
+
+public class AvatarData : BaseData
+{
+    public AvatarState Avatar { get; }
+    public ItemSlotState ItemSlot { get; }
+    public List<RuneState> RuneSlot { get; }
+
+    public AvatarData(AvatarState avatar, ItemSlotState itemSlot, List<RuneState> runeSlot)
+    {
+        Avatar = avatar;
+        ItemSlot = itemSlot;
+        RuneSlot = runeSlot;
+    }
+}

NineChroniclesUtilBackend.Store/Models/State/BaseData.cs

@@ -0,0 +1,20 @@
+using Newtonsoft.Json;
+using NineChroniclesUtilBackend.Store.Util;
+
+namespace NineChroniclesUtilBackend.Store.Models;
+
+public class BaseData
+{
+    protected static JsonSerializerSettings JsonSerializerSettings => new JsonSerializerSettings
+    {
+        Converters = new[] { new BigIntegerToStringConverter() },
+        Formatting = Formatting.Indented,
+        // ContractResolver = new Newtonsoft.Json.Serialization.CamelCasePropertyNamesContractResolver(),
+        NullValueHandling = NullValueHandling.Ignore
+    };
+
+    public string ToJson()
+    {
+        return JsonConvert.SerializeObject(this, JsonSerializerSettings);
+    }
+}

NineChroniclesUtilBackend.Store/Models/StateResponse.cs

@@ -0,0 +1,15 @@
+namespace NineChroniclesUtilBackend.Store.Models;
+
+public class StateResponse
+{
+    public string Address { get; }
+    public string AccountAddress { get; }
+    public string Value { get; }
+
+    public StateResponse(string address, string accountAddress, string value)
+    {
+        Address = address;
+        AccountAddress = accountAddress;
+        Value = value;
+    }
+}

NineChroniclesUtilBackend.Store/Models/StoreResult.cs

@@ -0,0 +1,30 @@
+using Libplanet.Crypto;
+
+namespace NineChroniclesUtilBackend.Store.Models;
+
+public class StoreResult
+{
+    public DateTime StartTime { get; set; }
+    public int TotalElapsedMinutes { get; set; }
+    public int StoreArenaRequestCount { get; set; }
+    public int StoreAvatarRequestCount { get; set; }
+    public int AvatarStoredCount { get; set; }
+    public int ArenaStoredCount { get; set; }
+    public List<Address> FailedAvatarAddresses { get; } = new List<Address>();
+    public List<Address> FailedArenaAddresses { get; } = new List<Address>();
+
+    public override string ToString()
+    {
+        var failedAvatarAddresses = string.Join(", ", FailedAvatarAddresses.Select(a => a.ToString()));
+        var failedArenaAddresses = string.Join(", ", FailedArenaAddresses.Select(a => a.ToString()));
+
+        return $"StartTime: {StartTime}, " +
+                $"TotalElapsedMinutes: {TotalElapsedMinutes}, " +
+                $"StoreArenaRequestCount: {StoreArenaRequestCount}, " +
+                $"StoreAvatarRequestCount: {StoreAvatarRequestCount}, " +
+                $"AvatarStoredCount: {AvatarStoredCount}, " +
+                $"ArenaStoredCount: {ArenaStoredCount}, " +
+                $"FailedAvatarAddresses: [{failedAvatarAddresses}], " +
+                $"FailedArenaAddresses: [{failedArenaAddresses}]";
+    }
+}

NineChroniclesUtilBackend.Store/NineChroniclesUtilBackend.Store.csproj

@@ -0,0 +1,17 @@
+<Project Sdk="Microsoft.NET.Sdk.Worker">
+
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <Nullable>enable</Nullable>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <UserSecretsId>dotnet-NineChroniclesUtilBackend.Store-bccda56f-4d38-484b-ab03-ebb26065c837</UserSecretsId>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.Extensions.Hosting" Version="8.0.0" />
+    <PackageReference Include="Lib9c" Version="1.8.0-dev.4e2f5d9bfed5366a94e67b88c4b0e7171c0cd0d6" />
+    <PackageReference Include="Libplanet" Version="4.1.0-dev.20242745721" />
+    <PackageReference Include="MongoDB.Driver" Version="2.24.0" />
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
+  </ItemGroup>
+</Project>