rework account locker database schema.

CHANGELOG.md

@@ -49,6 +49,7 @@ Release built: _not released yet_
 
 ### API Changes
 - Restored previously removed `total_count` property to `/state/key-value-store/keys` endpoint.
+- Added missing value for `total_count` property to the `/state/account-locker/page/vaults` endpoint.
 
 ### Database changes
 - Refactored multiple aggregates. Queries follow a similar strategy as key value stores and utilize `_entry_definition`, `_entry_history`, and `_totals_history` tables to return data
@@ -72,6 +73,9 @@ Release built: _not released yet_
         - Renamed `entity_vault_history` to `vault_balance_history`. Holds information about vault content (amount of fungibles or count of non fungible ids inside vault) at a given state version.
     - Key value store
         - New `key_value_store_totals_history` table, which holds total count of all keys under a given store at a given state version.
+    - Account lockers
+        - New `account_locker_totals_history` table, which holds the total number of accounts that have key value stores under the given account locker.
+        - New `account_locker_entry_resource_vault_totals_history` table, which holds the total number of resources/vaults under the given account in an account locker.
 - Changed `receipt_state_updates` in the `ledger_transactions` table to be nullable.
 - Moved all `receipt_event_*` columns from the `ledger_transactions` table to a new separate `ledger_transaction_events` table.
 - Renamed `origin_type` marker type (`ledger_transaction_markers`) to `transaction_type` with possible values:

src/RadixDlt.NetworkGateway.GatewayApi/Validators/RadixAddressValidator.cs

@@ -63,7 +63,6 @@
  */
 
 using FluentValidation;
-using RadixDlt.NetworkGateway.Abstractions;
 using RadixDlt.NetworkGateway.Abstractions.Network;
 using System;
 

src/RadixDlt.NetworkGateway.PostgresIntegration/CommonDbContext.cs

@@ -103,6 +103,8 @@ internal abstract class CommonDbContext : DbContext
 
     public DbSet<AccountLockerEntryDefinition> AccountLockerDefinition => Set<AccountLockerEntryDefinition>();
 
+    public DbSet<AccountLockerTotalsHistory> AccountLockerTotalsHistory => Set<AccountLockerTotalsHistory>();
+
     public DbSet<AccountLockerEntryResourceVaultDefinition> AccountLockerEntryResourceVaultDefinition => Set<AccountLockerEntryResourceVaultDefinition>();
 
     public DbSet<AccountLockerEntryTouchHistory> AccountLockerEntryTouchHistory => Set<AccountLockerEntryTouchHistory>();
@@ -422,6 +424,14 @@ private static void HookupDefinitions(ModelBuilder modelBuilder)
             .Entity<AccountLockerEntryResourceVaultDefinition>()
             .HasIndex(e => new { e.AccountLockerDefinitionId, e.FromStateVersion });
 
+        modelBuilder
+            .Entity<AccountLockerTotalsHistory>()
+            .HasIndex(e => new { e.AccountLockerEntityId, e.FromStateVersion });
+
+        modelBuilder
+            .Entity<AccountLockerEntryResourceVaultTotalsHistory>()
+            .HasIndex(e => new { e.AccountLockerDefinitionId, e.FromStateVersion });
+
         modelBuilder
             .Entity<EntityResourceEntryDefinition>()
             .HasIndex(e => new { e.EntityId, e.FromStateVersion });

src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/AccountLockerProcessor.cs

@@ -90,10 +90,14 @@ private record ObservedVault(long AccountLockerEntityId, long AccountEntityId, l
     private readonly HashSet<ObservedTouch> _observedTouchHistory = new();
     private readonly List<ObservedVault> _observedVaultDefinitions = new();
     private readonly Dictionary<AccountLockerEntryDbLookup, AccountLockerEntryDefinition> _existingEntryDefinitions = new();
+    private readonly Dictionary<long, AccountLockerTotalsHistory> _existingTotalsHistory = new();
+    private readonly Dictionary<long, AccountLockerEntryResourceVaultTotalsHistory> _existingResourceVaultTotalsHistory = new();
 
     private readonly List<AccountLockerEntryDefinition> _definitionsToAdd = new();
     private readonly List<AccountLockerEntryResourceVaultDefinition> _resourceVaultDefinitionsToAdd = new();
     private readonly List<AccountLockerEntryTouchHistory> _touchHistoryToAdd = new();
+    private readonly List<AccountLockerTotalsHistory> _totalsHistoryToAdd = new();
+    private readonly List<AccountLockerEntryResourceVaultTotalsHistory> _resourceVaultTotalsHistoryToAdd = new();
 
     public AccountLockerProcessor(ProcessorContext context, ReferencedEntityDictionary referencedEntities)
     {
@@ -161,11 +165,14 @@ public void VisitUpsert(CoreModel.IUpsertedSubstate substate, ReferencedEntity r
     public async Task LoadDependenciesAsync()
     {
         _existingEntryDefinitions.AddRange(await ExistingAccountLockerEntryDefinitions());
+        _existingTotalsHistory.AddRange(await ExistingAccountLockerTotalsHistory());
+        _existingResourceVaultTotalsHistory.AddRange(await ExistingAccountLockerEntryResourceVaultTotalsHistory());
     }
 
     public void ProcessChanges()
     {
         _existingEntryDefinitions.AddRange(_definitionsToAdd.ToDictionary(e => new AccountLockerEntryDbLookup(e.AccountLockerEntityId, e.AccountEntityId)));
+
         _resourceVaultDefinitionsToAdd.AddRange(
             _observedVaultDefinitions.Select(
                 rv => new AccountLockerEntryResourceVaultDefinition
@@ -176,6 +183,7 @@ public void ProcessChanges()
                     ResourceEntityId = _referencedEntities.Get(rv.ResourceAddress).DatabaseId,
                     VaultEntityId = _referencedEntities.Get(rv.VaultAddress).DatabaseId,
                 }));
+
         _touchHistoryToAdd.AddRange(
             _observedTouchHistory.Select(
                 th => new AccountLockerEntryTouchHistory
@@ -184,6 +192,38 @@ public void ProcessChanges()
                     FromStateVersion = th.StateVersion,
                     AccountLockerDefinitionId = _existingEntryDefinitions[new AccountLockerEntryDbLookup(th.AccountLockerEntityId, th.AccountEntityId)].Id,
                 }));
+
+        foreach (var newDefinition in _definitionsToAdd)
+        {
+            var totalsExists = _existingTotalsHistory.TryGetValue(newDefinition.AccountEntityId, out var existingTotals);
+
+            var newTotals = new AccountLockerTotalsHistory
+            {
+                Id = _context.Sequences.AccountLockerTotalsHistorySequence++,
+                FromStateVersion = newDefinition.FromStateVersion,
+                AccountLockerEntityId = newDefinition.AccountEntityId,
+                TotalAccounts = totalsExists ? existingTotals!.TotalAccounts + 1 : 1,
+            };
+
+            _totalsHistoryToAdd.Add(newTotals);
+            _existingTotalsHistory[newDefinition.AccountEntityId] = newTotals;
+        }
+
+        foreach (var newVaultDefinition in _resourceVaultDefinitionsToAdd)
+        {
+            var totalsExists = _existingResourceVaultTotalsHistory.TryGetValue(newVaultDefinition.AccountLockerDefinitionId, out var existingTotals);
+
+            var newTotals = new AccountLockerEntryResourceVaultTotalsHistory
+            {
+                Id = _context.Sequences.AccountLockerEntryResourceVaultTotalsHistorySequence++,
+                FromStateVersion = newVaultDefinition.FromStateVersion,
+                AccountLockerDefinitionId = newVaultDefinition.AccountLockerDefinitionId,
+                TotalResources = totalsExists ? existingTotals!.TotalResources + 1 : 1,
+            };
+
+            _resourceVaultTotalsHistoryToAdd.Add(newTotals);
+            _existingResourceVaultTotalsHistory[newVaultDefinition.AccountLockerDefinitionId] = newTotals;
+        }
     }
 
     public async Task<int> SaveEntitiesAsync()
@@ -193,6 +233,8 @@ public async Task<int> SaveEntitiesAsync()
         rowsInserted += await CopyAccountLockerEntryDefinition();
         rowsInserted += await CopyAccountLockerEntryResourceVaultDefinition();
         rowsInserted += await CopyAccountLockerEntryTouchHistory();
+        rowsInserted += await CopyAccountLockerTotalsHistory();
+        rowsInserted += await CopyAccountLockerResourceVaultTotalsHistory();
 
         return rowsInserted;
     }
@@ -215,6 +257,62 @@ FROM account_locker_entry_definition
             e => new AccountLockerEntryDbLookup(e.AccountLockerEntityId, e.AccountEntityId));
     }
 
+    private async Task<IDictionary<long, AccountLockerTotalsHistory>> ExistingAccountLockerTotalsHistory()
+    {
+        var accountLockerEntityIds = _observedVaultDefinitions
+            .Select(x => x.AccountLockerEntityId)
+            .ToHashSet();
+
+        if (accountLockerEntityIds.Count == 0)
+        {
+            return ImmutableDictionary<long, AccountLockerTotalsHistory>.Empty;
+        }
+
+        return await _context.ReadHelper.LoadDependencies<long, AccountLockerTotalsHistory>(
+            @$"
+WITH variables (account_locker_entity_id) AS (
+    SELECT UNNEST({accountLockerEntityIds})
+)
+SELECT alth.*
+FROM variables
+INNER JOIN LATERAL (
+    SELECT *
+    FROM account_locker_totals_history
+    WHERE account_locker_entity_id = variables.account_locker_entity_id
+    ORDER BY from_state_version DESC
+    LIMIT 1
+) alth ON true;",
+            e => e.AccountLockerEntityId);
+    }
+
+    private async Task<IDictionary<long, AccountLockerEntryResourceVaultTotalsHistory>> ExistingAccountLockerEntryResourceVaultTotalsHistory()
+    {
+        var accountLockerDefinitionIds = _existingEntryDefinitions.Values
+            .Select(x => x.Id)
+            .ToHashSet();
+
+        if (accountLockerDefinitionIds.Count == 0)
+        {
+            return ImmutableDictionary<long, AccountLockerEntryResourceVaultTotalsHistory>.Empty;
+        }
+
+        return await _context.ReadHelper.LoadDependencies<long, AccountLockerEntryResourceVaultTotalsHistory>(
+            @$"
+WITH variables (account_locker_definition_id) AS (
+    SELECT UNNEST({accountLockerDefinitionIds})
+)
+SELECT alrvth.*
+FROM variables
+INNER JOIN LATERAL (
+    SELECT *
+    FROM account_locker_entry_resource_vault_totals_history
+    WHERE account_locker_definition_id = variables.account_locker_definition_id
+    ORDER BY from_state_version DESC
+    LIMIT 1
+) alrvth ON true;",
+            e => e.AccountLockerDefinitionId);
+    }
+
     private Task<int> CopyAccountLockerEntryDefinition() => _context.WriteHelper.Copy(
         _definitionsToAdd,
         "COPY account_locker_entry_definition (id, from_state_version, account_locker_entity_id, account_entity_id, key_value_store_entity_id) FROM STDIN (FORMAT BINARY)",
@@ -248,4 +346,26 @@ private Task<int> CopyAccountLockerEntryTouchHistory() => _context.WriteHelper.C
             await writer.WriteAsync(e.FromStateVersion, NpgsqlDbType.Bigint, token);
             await writer.WriteAsync(e.AccountLockerDefinitionId, NpgsqlDbType.Bigint, token);
         });
+
+    private Task<int> CopyAccountLockerTotalsHistory() => _context.WriteHelper.Copy(
+        _totalsHistoryToAdd,
+        "COPY account_locker_totals_history (id, from_state_version, account_locker_entity_id, total_accounts) FROM STDIN (FORMAT BINARY)",
+        async (writer, e, token) =>
+        {
+            await writer.WriteAsync(e.Id, NpgsqlDbType.Bigint, token);
+            await writer.WriteAsync(e.FromStateVersion, NpgsqlDbType.Bigint, token);
+            await writer.WriteAsync(e.AccountLockerEntityId, NpgsqlDbType.Bigint, token);
+            await writer.WriteAsync(e.TotalAccounts, NpgsqlDbType.Bigint, token);
+        });
+
+    private Task<int> CopyAccountLockerResourceVaultTotalsHistory() => _context.WriteHelper.Copy(
+        _resourceVaultTotalsHistoryToAdd,
+        "COPY account_locker_entry_resource_vault_totals_history (id, from_state_version, account_locker_definition_id, total_resources) FROM STDIN (FORMAT BINARY)",
+        async (writer, e, token) =>
+        {
+            await writer.WriteAsync(e.Id, NpgsqlDbType.Bigint, token);
+            await writer.WriteAsync(e.FromStateVersion, NpgsqlDbType.Bigint, token);
+            await writer.WriteAsync(e.AccountLockerDefinitionId, NpgsqlDbType.Bigint, token);
+            await writer.WriteAsync(e.TotalResources, NpgsqlDbType.Bigint, token);
+        });
 }

src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/ReadHelper.cs

@@ -149,6 +149,8 @@ public async Task<SequencesHolder> LoadSequences(CancellationToken token)
     nextval('account_locker_entry_definition_id_seq') AS AccountLockerEntryDefinitionSequence,
     nextval('account_locker_entry_resource_vault_definition_id_seq') AS AccountLockerEntryResourceVaultDefinitionSequence,
     nextval('account_locker_entry_touch_history_id_seq') AS AccountLockerEntryTouchHistorySequence,
+    nextval('account_locker_totals_history_id_seq') AS AccountLockerTotalsHistorySequence,
+    nextval('account_locker_entry_resource_vault_totals_history_id_seq') AS AccountLockerEntryResourceVaultTotalsHistorySequence,
     nextval('account_default_deposit_rule_history_id_seq') AS AccountDefaultDepositRuleHistorySequence,
     nextval('account_resource_preference_rule_entry_history_id_seq') AS AccountResourcePreferenceRuleEntryHistorySequence,
     nextval('account_resource_preference_rule_aggregate_history_id_seq') AS AccountResourcePreferenceRuleAggregateHistorySequence,

src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/SequencesHolder.cs

@@ -72,6 +72,10 @@ internal class SequencesHolder
 
     public long AccountLockerEntryTouchHistorySequence { get; set; }
 
+    public long AccountLockerTotalsHistorySequence { get; set; }
+
+    public long AccountLockerEntryResourceVaultTotalsHistorySequence { get; set; }
+
     public long AccountDefaultDepositRuleHistorySequence { get; set; }
 
     public long AccountResourcePreferenceRuleEntryHistorySequence { get; set; }

src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/WriteHelper.cs

@@ -205,6 +205,8 @@ public async Task UpdateSequences(SequencesHolder sequences, CancellationToken t
             accountLockerEntryDefinitionSequence = sequences.AccountLockerEntryDefinitionSequence,
             accountLockerEntryResourceVaultDefinitionSequence = sequences.AccountLockerEntryResourceVaultDefinitionSequence,
             accountLockerEntryTouchHistorySequence = sequences.AccountLockerEntryTouchHistorySequence,
+            accountLockerTotalsHistorySequence = sequences.AccountLockerTotalsHistorySequence,
+            accountLockerEntryResourceVaultTotalsHistorySequence = sequences.AccountLockerEntryResourceVaultTotalsHistorySequence,
             accountDefaultDepositRuleHistorySequence = sequences.AccountDefaultDepositRuleHistorySequence,
             accountResourcePreferenceRuleEntryHistorySequence = sequences.AccountResourcePreferenceRuleEntryHistorySequence,
             accountResourcePreferenceRuleAggregateHistorySequence = sequences.AccountResourcePreferenceRuleAggregateHistorySequence,
@@ -257,6 +259,8 @@ public async Task UpdateSequences(SequencesHolder sequences, CancellationToken t
     setval('account_locker_entry_definition_id_seq', @accountLockerEntryDefinitionSequence),
     setval('account_locker_entry_resource_vault_definition_id_seq', @accountLockerEntryResourceVaultDefinitionSequence),
     setval('account_locker_entry_touch_history_id_seq', @accountLockerEntryTouchHistorySequence),
+    setval('account_locker_totals_history_id_seq', @accountLockerTotalsHistorySequence),
+    setval('account_locker_entry_resource_vault_totals_history_id_seq', @accountLockerEntryResourceVaultTotalsHistorySequence),
     setval('account_default_deposit_rule_history_id_seq', @accountDefaultDepositRuleHistorySequence),
     setval('account_resource_preference_rule_entry_history_id_seq', @accountResourcePreferenceRuleEntryHistorySequence),
     setval('account_resource_preference_rule_aggregate_history_id_seq', @accountResourcePreferenceRuleAggregateHistorySequence),