From 7035ed68a9ba893c4cf0852a2a3a4bb526445b2c Mon Sep 17 00:00:00 2001 From: Kukks Date: Thu, 9 May 2024 18:02:49 +0200 Subject: [PATCH] Import utxo --- NBXplorer.Client/ExplorerClient.cs | 9 ++ NBXplorer.Client/Models/ImportUTXORequest.cs | 10 ++ NBXplorer.Tests/UnitTest1.cs | 122 +++++++++++++++++++ NBXplorer/Controllers/MainController.cs | 51 ++++++++ NBXplorer/RPCClientExtensions.cs | 17 +++ NBXplorer/wwwroot/api.json | 41 ++++++- 6 files changed, 249 insertions(+), 1 deletion(-) create mode 100644 NBXplorer.Client/Models/ImportUTXORequest.cs diff --git a/NBXplorer.Client/ExplorerClient.cs b/NBXplorer.Client/ExplorerClient.cs index 06ace3044..fb9f479dd 100644 --- a/NBXplorer.Client/ExplorerClient.cs +++ b/NBXplorer.Client/ExplorerClient.cs @@ -583,6 +583,15 @@ public Task AddGroupAddressAsync(string cryptoCode, string groupId, string[] add { return SendAsync(HttpMethod.Post, addresses, $"v1/cryptos/{cryptoCode}/groups/{groupId}/addresses", cancellationToken); } + + public async Task ImportUTXOs(string cryptoCode, ImportUTXORequest request, CancellationToken cancellation = default) + { + if (request == null) + throw new ArgumentNullException(nameof(request)); + if (cryptoCode == null) + throw new ArgumentNullException(nameof(cryptoCode)); + await SendAsync(HttpMethod.Post, request, $"v1/cryptos/{cryptoCode}/rescan-utxos", cancellation); + } private static readonly HttpClient SharedClient = new HttpClient(); internal HttpClient Client = SharedClient; diff --git a/NBXplorer.Client/Models/ImportUTXORequest.cs b/NBXplorer.Client/Models/ImportUTXORequest.cs new file mode 100644 index 000000000..25c5cba3f --- /dev/null +++ b/NBXplorer.Client/Models/ImportUTXORequest.cs @@ -0,0 +1,10 @@ +using NBitcoin; +using Newtonsoft.Json; + +namespace NBXplorer.Models; + +public class ImportUTXORequest +{ + [JsonProperty("UTXOs")] + public OutPoint[] Utxos { get; set; } +} \ No newline at end of file diff --git a/NBXplorer.Tests/UnitTest1.cs b/NBXplorer.Tests/UnitTest1.cs index ac55f91a6..c91ee601e 100644 --- a/NBXplorer.Tests/UnitTest1.cs +++ b/NBXplorer.Tests/UnitTest1.cs @@ -4269,5 +4269,127 @@ public async Task IsTrackedTests() group = new GroupTrackedSource((await tester.Client.CreateGroupAsync(Cancel)).GroupId); Assert.True(await tester.Client.IsTrackedAsync(group, Cancel)); } + [Fact] + public async Task CanImportUTXOs() + { + using var tester = ServerTester.Create(); + + var wallet1 = await tester.Client.CreateGroupAsync(); + var wallet1TS = new GroupTrackedSource(wallet1.GroupId); + + var k = new Key(); + var kAddress = k.GetAddress(ScriptPubKeyType.Segwit, tester.Network); + + // We use this one because it allows us to use WaitForTransaction later + var legacy = new AddressTrackedSource(new Key().GetAddress(ScriptPubKeyType.Legacy, tester.Network)); + await tester.Client.TrackAsync(legacy); + + var kScript = kAddress.ScriptPubKey; + + // test 1: create a script and send 2 utxos to it(from diff txs), without confirming + // import the first one, verify it is unconfirmed, confirm, then the second one and see it is confirmed + + var tx = await tester.RPC.SendToAddressAsync(kAddress, Money.Coins(1.0m)); + var tx2 = await tester.RPC.SendToAddressAsync(kAddress, Money.Coins(1.0m)); + var rawTx = await tester.RPC.GetRawTransactionAsync(tx); + var rawTx2 = await tester.RPC.GetRawTransactionAsync(tx2); + var utxo = rawTx.Outputs.AsIndexedOutputs().First(o => o.TxOut.ScriptPubKey == kScript); + var utxo2 = rawTx2.Outputs.AsIndexedOutputs().First(o => o.TxOut.ScriptPubKey == kScript); + + // Making sure that tx and tx2 are processed before continuing + var tx3 = await tester.RPC.SendToAddressAsync(legacy.Address, Money.Coins(1.0m)); + var notif = tester.Notifications.WaitForTransaction(legacy.Address, tx3); + Assert.Equal(legacy, notif.TrackedSource); + + await tester.Client.AddGroupAddressAsync("BTC", wallet1.GroupId, new[] { kAddress.ToString() }); + await tester.Client.ImportUTXOs("BTC", new ImportUTXORequest() + { + Utxos = [utxo.ToCoin().Outpoint] + }); + + var utxos = await tester.Client.GetUTXOsAsync(wallet1TS); + var matched = Assert.Single(utxos.Unconfirmed.UTXOs); + Assert.Equal(kAddress, matched.Address); + + // tx2 didn't matched when it was in the mempool, so the block will not match it either because of the cache. + tester.GetService().GetRepository("BTC").RemoveFromCache(new[] { tx2 }); + tester.Notifications.WaitForBlocks(await tester.RPC.GenerateAsync(1)); + utxos = await tester.Client.GetUTXOsAsync(wallet1TS); + Assert.Equal(2, utxos.Confirmed.UTXOs.Count); + Assert.Contains(tx, utxos.Confirmed.UTXOs.Select(u => u.Outpoint.Hash)); + Assert.Contains(tx2, utxos.Confirmed.UTXOs.Select(u => u.Outpoint.Hash)); + + await tester.Client.ImportUTXOs("BTC", new ImportUTXORequest() + { + Utxos = [utxo2.ToCoin().Outpoint] + }); + + utxos = await tester.Client.GetUTXOsAsync(wallet1TS); + Assert.Equal(2, utxos.Confirmed.UTXOs.Count); + //utxo2 may be confirmed but we should have saved the timestamp based on block time or current date + var utxoInfo = utxos.Confirmed.UTXOs.First(u => u.ScriptPubKey == utxo2.TxOut.ScriptPubKey); + Assert.NotEqual(NBitcoin.Utils.UnixTimeToDateTime(0), utxoInfo.Timestamp); + + //test2: try adding in fake utxos or spent ones + var fakescript = new Key().PubKey.GetAddress(ScriptPubKeyType.Segwit, tester.Network).ScriptPubKey; + var fakeUtxo = new Coin(new OutPoint(uint256.One, 1), new TxOut(Money.Coins(1.0m), fakescript)); + var kToSpend = new Key(); + var kToSpendAddress = kToSpend.GetAddress(ScriptPubKeyType.Segwit, tester.Network); + var tospendtx = await tester.RPC.SendToAddressAsync(kToSpendAddress, Money.Coins(1.0m)); + var tospendrawtx = await tester.RPC.GetRawTransactionAsync(tospendtx); + var tospendutxo = tospendrawtx.Outputs.AsIndexedOutputs().First(o => o.TxOut.ScriptPubKey == kToSpendAddress.ScriptPubKey); + var validScript = new Key().PubKey.GetAddress(ScriptPubKeyType.Segwit, tester.Network).ScriptPubKey; + var spendingtx = tester.Network.CreateTransactionBuilder() + .AddKeys(kToSpend) + .AddCoins(new Coin(tospendutxo)) + .SendEstimatedFees(new FeeRate(100m)) + .SendAll(validScript).BuildTransaction(true); + await tester.RPC.SendRawTransactionAsync(spendingtx); + + var validScriptUtxo = spendingtx.Outputs.AsIndexedOutputs().First(o => o.TxOut.ScriptPubKey == validScript); + + await tester.Client.ImportUTXOs("BTC", new ImportUTXORequest() + { + Utxos = + [ + fakeUtxo.Outpoint, + tospendutxo.ToCoin().Outpoint, + validScriptUtxo.ToCoin().Outpoint + ] + }); + + utxos = await tester.Client.GetUTXOsAsync(wallet1TS); + Assert.Empty(utxos.Unconfirmed.UTXOs); + + // let's test add an utxo after it has been mined + var yoScript = new Key().PubKey.GetAddress(ScriptPubKeyType.Segwit, tester.Network); + var yoTxId = await tester.SendToAddressAsync(yoScript, Money.Coins(1.0m)); + var yoTx = await tester.RPC.GetRawTransactionAsync(yoTxId); + var yoUtxo = yoTx.Outputs.AsIndexedOutputs().First(o => o.TxOut.ScriptPubKey == yoScript.ScriptPubKey); + + await tester.Client.ImportUTXOs("BTC", new ImportUTXORequest() + { + Utxos = [yoUtxo.ToCoin().Outpoint] + }); + + utxos = await tester.Client.GetUTXOsAsync(wallet1TS); + Assert.Empty(utxos.Unconfirmed.UTXOs); + + var aaa = await tester.RPC.GenerateAsync(1); + tester.Notifications.WaitForBlocks(aaa); + utxos = await tester.Client.GetUTXOsAsync(wallet1TS); + Assert.Empty(utxos.Unconfirmed.UTXOs); + + await tester.Client.AddGroupAddressAsync("BTC", wallet1.GroupId, new[] { yoScript.ToString() }); + await tester.Client.ImportUTXOs("BTC", new ImportUTXORequest() + { + Utxos = [yoUtxo.ToCoin().Outpoint] + }); + + utxos = await tester.Client.GetUTXOsAsync(wallet1TS); + var confirmedUtxo = utxos.Confirmed.UTXOs.Single(utxo1 => utxo1.ScriptPubKey == yoScript.ScriptPubKey); + Assert.Equal(1, confirmedUtxo.Confirmations); + Assert.NotEqual(NBitcoin.Utils.UnixTimeToDateTime(0), confirmedUtxo.Timestamp); + } } } diff --git a/NBXplorer/Controllers/MainController.cs b/NBXplorer/Controllers/MainController.cs index a5ef897cf..74775062f 100644 --- a/NBXplorer/Controllers/MainController.cs +++ b/NBXplorer/Controllers/MainController.cs @@ -733,6 +733,57 @@ public async Task Rescan(TrackedSourceContext trackedSourceContex } } + [HttpPost] + [Route($"{CommonRoutes.BaseCryptoEndpoint}/rescan-utxos")] + [TrackedSourceContext.TrackedSourceContextRequirement(false, false, true)] + public async Task ImportUTXOs(TrackedSourceContext trackedSourceContext, [FromBody] ImportUTXORequest request, CancellationToken cancellationToken = default) + { + var repo = trackedSourceContext.Repository; + if (request.Utxos?.Any() is not true) + return Ok(); + + var rpc = trackedSourceContext.RpcClient; + + var coinToTxOut = await rpc.GetTxOuts(request.Utxos); + var bestBlocksToFetch = coinToTxOut.Select(c => c.Value.BestBlock).ToHashSet().ToList(); + var bestBlocks = await rpc.GetBlockHeadersAsync(bestBlocksToFetch, cancellationToken); + var coinsWithHeights = coinToTxOut + .Select(c => new + { + BestBlock = bestBlocks.ByHashes.TryGet(c.Value.BestBlock), + Outpoint = c.Key, + RPCTxOut = c.Value + }) + .Select(c => new + { + Height = c.RPCTxOut.Confirmations == 0 ? null : new int?(c.BestBlock.Height - c.RPCTxOut.Confirmations + 1), + c.Outpoint, + c.RPCTxOut + }) + .ToList(); + var blocks = coinsWithHeights.Where(c => c.Height.HasValue).Select(c => c.Height.Value).Distinct().ToList(); + var blockHeaders = await rpc.GetBlockHeadersAsync(blocks, cancellationToken); + + var now = DateTimeOffset.UtcNow; + var records = new List(); + MatchQuery query = new MatchQuery(coinsWithHeights.Select(c => new Coin(c.Outpoint, c.RPCTxOut.TxOut))); + foreach (var c in coinsWithHeights) + { + var block = c.Height is int h ? blockHeaders.ByHeight.TryGet(h) : null; + var record = SaveTransactionRecord.Create( + txHash: c.Outpoint.Hash, + slimBlock: block?.ToSlimChainedBlock(), + seenAt: Extensions.MinDate(block?.Time ?? now, now)); + records.Add(record); + } + await repo.SaveBlocks(blockHeaders); + repo.RemoveFromCache(records.Select(r => r.Id)); + var trackedTransactions = await repo.SaveMatches(query, records.ToArray()); + _ = AddressPoolService.GenerateAddresses(trackedSourceContext.Network, trackedTransactions); + + return Ok(); + } + internal async Task GetAnnotatedTransactions(Repository repo, TrackedSource trackedSource, bool includeTransaction, uint256 txId = null) { var transactions = await repo.GetTransactions(trackedSource, txId, includeTransaction, this.HttpContext?.RequestAborted ?? default); diff --git a/NBXplorer/RPCClientExtensions.cs b/NBXplorer/RPCClientExtensions.cs index 4a2d577e8..a9a53b1cb 100644 --- a/NBXplorer/RPCClientExtensions.cs +++ b/NBXplorer/RPCClientExtensions.cs @@ -379,5 +379,22 @@ public async static Task ImportDescriptors(this RPCClient rpc, string descriptor } throw new NotSupportedException($"Bug of NBXplorer (ERR 3083), please notify the developers"); } + public static async Task> GetTxOuts(this RPCClient rpc, IList outpoints) + { + var batch = rpc.PrepareBatch(); + var txOuts = outpoints.Select(o => batch.GetTxOutAsync(o.Hash, (int)o.N, true)).ToArray(); + await batch.SendBatchAsync(); + var result = new Dictionary(); + int i = 0; + foreach (var txOut in txOuts) + { + var outpoint = outpoints[i]; + var r = await txOut; + if (r != null) + result.TryAdd(outpoint, r); + i++; + } + return result; + } } } diff --git a/NBXplorer/wwwroot/api.json b/NBXplorer/wwwroot/api.json index 979c7f3c8..1a7f50696 100644 --- a/NBXplorer/wwwroot/api.json +++ b/NBXplorer/wwwroot/api.json @@ -2020,6 +2020,45 @@ } } }, + "/v1/cryptos/{cryptoCode}/rescan-utxos": { + "post": { + "summary": "Rescan utxos", + "description": "Verifies that the UTXOs are unspent and save matches to wallets tracked by this server", + "tags": [ + "Blockchain" + ], + "parameters": [ + { + "$ref": "#/components/parameters/CryptoCode" + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "UTXOs": { + "type": "array", + "description": "The outpoints to rescan", + "items": { + "type": "string", + "example": "f3e34c3ae29c79ddf807de0ab3b40da7862adb1b175b7421d89ec78446a52b13-1" + }, + "example": [ "f3e34c3ae29c79ddf807de0ab3b40da7862adb1b175b7421d89ec78446a52b13-1", "fd97fb1de63fcdb9fa1614a74775e84818b2dbd79fe36aef9e0c18f9fad03742-2" ] + } + } + } + } + } + }, + "responses": { + "200": { + "description": "Rescan initiated successfully." + } + } + } + }, "/v1/cryptos/{cryptoCode}/psbt/update": { "post": { "summary": "Update Partially Signed Bitcoin Transaction", @@ -3728,4 +3767,4 @@ } } } -} \ No newline at end of file +}