Import utxo

NBXplorer.Client/ExplorerClient.cs

@@ -583,6 +583,15 @@ public Task AddGroupAddressAsync(string cryptoCode, string groupId, string[] add
 		{
 			return SendAsync<GroupInformation>(HttpMethod.Post, addresses, $"v1/cryptos/{cryptoCode}/groups/{groupId}/addresses", cancellationToken);
 		}
+
+		public async Task ImportUTXOs(string cryptoCode, ImportUTXORequest request, CancellationToken cancellation = default)
+		{
+			if (request == null)
+				throw new ArgumentNullException(nameof(request));
+			if (cryptoCode == null)
+				throw new ArgumentNullException(nameof(cryptoCode));
+			await SendAsync(HttpMethod.Post, request, $"v1/cryptos/{cryptoCode}/rescan-utxos", cancellation);
+		}
 
 		private static readonly HttpClient SharedClient = new HttpClient();
 		internal HttpClient Client = SharedClient;

NBXplorer.Client/Models/ImportUTXORequest.cs

@@ -0,0 +1,10 @@
+using NBitcoin;
+using Newtonsoft.Json;
+
+namespace NBXplorer.Models;
+
+public class ImportUTXORequest
+{
+	[JsonProperty("UTXOs")]
+	public OutPoint[] Utxos { get; set; }
+}

NBXplorer.Tests/UnitTest1.cs

@@ -4269,5 +4269,127 @@ public async Task IsTrackedTests()
 			group = new GroupTrackedSource((await tester.Client.CreateGroupAsync(Cancel)).GroupId);
 			Assert.True(await tester.Client.IsTrackedAsync(group, Cancel));
 		}
+		[Fact]
+		public async Task CanImportUTXOs()
+		{
+			using var tester = ServerTester.Create();
+
+			var wallet1 = await tester.Client.CreateGroupAsync();
+			var wallet1TS = new GroupTrackedSource(wallet1.GroupId);
+
+			var k = new Key();
+			var kAddress = k.GetAddress(ScriptPubKeyType.Segwit, tester.Network);
+
+			// We use this one because it allows us to use WaitForTransaction later
+			var legacy = new AddressTrackedSource(new Key().GetAddress(ScriptPubKeyType.Legacy, tester.Network));
+			await tester.Client.TrackAsync(legacy);
+
+			var kScript = kAddress.ScriptPubKey;
+
+			// test 1: create a script and send 2 utxos to it(from diff txs), without confirming
+			// import the first one, verify it is unconfirmed, confirm, then the second one and see it is confirmed
+
+			var tx = await tester.RPC.SendToAddressAsync(kAddress, Money.Coins(1.0m));
+			var tx2 = await tester.RPC.SendToAddressAsync(kAddress, Money.Coins(1.0m));
+			var rawTx = await tester.RPC.GetRawTransactionAsync(tx);
+			var rawTx2 = await tester.RPC.GetRawTransactionAsync(tx2);
+			var utxo = rawTx.Outputs.AsIndexedOutputs().First(o => o.TxOut.ScriptPubKey == kScript);
+			var utxo2 = rawTx2.Outputs.AsIndexedOutputs().First(o => o.TxOut.ScriptPubKey == kScript);
+
+			// Making sure that tx and tx2 are processed before continuing
+			var tx3 = await tester.RPC.SendToAddressAsync(legacy.Address, Money.Coins(1.0m));
+			var notif = tester.Notifications.WaitForTransaction(legacy.Address, tx3);
+			Assert.Equal(legacy, notif.TrackedSource);
+
+			await tester.Client.AddGroupAddressAsync("BTC", wallet1.GroupId, new[] { kAddress.ToString() });
+			await tester.Client.ImportUTXOs("BTC", new ImportUTXORequest()
+			{
+				Utxos = [utxo.ToCoin().Outpoint]
+			});
+
+			var utxos = await tester.Client.GetUTXOsAsync(wallet1TS);
+			var matched = Assert.Single(utxos.Unconfirmed.UTXOs);
+			Assert.Equal(kAddress, matched.Address);
+
+			// tx2 didn't matched when it was in the mempool, so the block will not match it either because of the cache.
+			tester.GetService<RepositoryProvider>().GetRepository("BTC").RemoveFromCache(new[] { tx2 });
+			tester.Notifications.WaitForBlocks(await tester.RPC.GenerateAsync(1));
+			utxos = await tester.Client.GetUTXOsAsync(wallet1TS);
+			Assert.Equal(2, utxos.Confirmed.UTXOs.Count);
+			Assert.Contains(tx, utxos.Confirmed.UTXOs.Select(u => u.Outpoint.Hash));
+			Assert.Contains(tx2, utxos.Confirmed.UTXOs.Select(u => u.Outpoint.Hash));
+
+			await tester.Client.ImportUTXOs("BTC", new ImportUTXORequest()
+			{
+				Utxos = [utxo2.ToCoin().Outpoint]
+			});
+
+			utxos = await tester.Client.GetUTXOsAsync(wallet1TS);
+			Assert.Equal(2, utxos.Confirmed.UTXOs.Count);
+			//utxo2 may be confirmed but we should have saved the timestamp based on block time or current date
+			var utxoInfo = utxos.Confirmed.UTXOs.First(u => u.ScriptPubKey == utxo2.TxOut.ScriptPubKey);
+			Assert.NotEqual(NBitcoin.Utils.UnixTimeToDateTime(0), utxoInfo.Timestamp);
+
+			//test2: try adding in fake utxos or spent ones
+			var fakescript = new Key().PubKey.GetAddress(ScriptPubKeyType.Segwit, tester.Network).ScriptPubKey;
+			var fakeUtxo = new Coin(new OutPoint(uint256.One, 1), new TxOut(Money.Coins(1.0m), fakescript));
+			var kToSpend = new Key();
+			var kToSpendAddress = kToSpend.GetAddress(ScriptPubKeyType.Segwit, tester.Network);
+			var tospendtx = await tester.RPC.SendToAddressAsync(kToSpendAddress, Money.Coins(1.0m));
+			var tospendrawtx = await tester.RPC.GetRawTransactionAsync(tospendtx);
+			var tospendutxo = tospendrawtx.Outputs.AsIndexedOutputs().First(o => o.TxOut.ScriptPubKey == kToSpendAddress.ScriptPubKey);
+			var validScript = new Key().PubKey.GetAddress(ScriptPubKeyType.Segwit, tester.Network).ScriptPubKey;
+			var spendingtx = tester.Network.CreateTransactionBuilder()
+				.AddKeys(kToSpend)
+				.AddCoins(new Coin(tospendutxo))
+				.SendEstimatedFees(new FeeRate(100m))
+				.SendAll(validScript).BuildTransaction(true);
+			await tester.RPC.SendRawTransactionAsync(spendingtx);
+
+			var validScriptUtxo = spendingtx.Outputs.AsIndexedOutputs().First(o => o.TxOut.ScriptPubKey == validScript);
+
+			await tester.Client.ImportUTXOs("BTC", new ImportUTXORequest()
+			{
+				Utxos =
+				[
+					fakeUtxo.Outpoint,
+					tospendutxo.ToCoin().Outpoint,
+					validScriptUtxo.ToCoin().Outpoint
+				]
+			});
+
+			utxos = await tester.Client.GetUTXOsAsync(wallet1TS);
+			Assert.Empty(utxos.Unconfirmed.UTXOs);
+
+			// let's test add an utxo after it has been mined
+			var yoScript = new Key().PubKey.GetAddress(ScriptPubKeyType.Segwit, tester.Network);
+			var yoTxId = await tester.SendToAddressAsync(yoScript, Money.Coins(1.0m));
+			var yoTx = await tester.RPC.GetRawTransactionAsync(yoTxId);
+			var yoUtxo = yoTx.Outputs.AsIndexedOutputs().First(o => o.TxOut.ScriptPubKey == yoScript.ScriptPubKey);
+
+			await tester.Client.ImportUTXOs("BTC", new ImportUTXORequest()
+			{
+				Utxos = [yoUtxo.ToCoin().Outpoint]
+			});
+
+			utxos = await tester.Client.GetUTXOsAsync(wallet1TS);
+			Assert.Empty(utxos.Unconfirmed.UTXOs);
+
+			var aaa = await tester.RPC.GenerateAsync(1);
+			tester.Notifications.WaitForBlocks(aaa);
+			utxos = await tester.Client.GetUTXOsAsync(wallet1TS);
+			Assert.Empty(utxos.Unconfirmed.UTXOs);
+
+			await tester.Client.AddGroupAddressAsync("BTC", wallet1.GroupId, new[] { yoScript.ToString() });
+			await tester.Client.ImportUTXOs("BTC", new ImportUTXORequest()
+			{
+				Utxos = [yoUtxo.ToCoin().Outpoint]
+			});
+
+			utxos = await tester.Client.GetUTXOsAsync(wallet1TS);
+			var confirmedUtxo = utxos.Confirmed.UTXOs.Single(utxo1 => utxo1.ScriptPubKey == yoScript.ScriptPubKey);
+			Assert.Equal(1, confirmedUtxo.Confirmations);
+			Assert.NotEqual(NBitcoin.Utils.UnixTimeToDateTime(0), confirmedUtxo.Timestamp);
+		}
 	}
 }

NBXplorer/Controllers/MainController.cs

@@ -733,6 +733,57 @@ public async Task<IActionResult> Rescan(TrackedSourceContext trackedSourceContex
 			}
 		}
 
+		[HttpPost]
+		[Route($"{CommonRoutes.BaseCryptoEndpoint}/rescan-utxos")]
+		[TrackedSourceContext.TrackedSourceContextRequirement(false, false, true)]
+		public async Task<IActionResult> ImportUTXOs(TrackedSourceContext trackedSourceContext, [FromBody] ImportUTXORequest request, CancellationToken cancellationToken = default)
+		{
+			var repo = trackedSourceContext.Repository;
+			if (request.Utxos?.Any() is not true)
+				return Ok();
+
+			var rpc = trackedSourceContext.RpcClient;
+
+			var coinToTxOut = await rpc.GetTxOuts(request.Utxos);
+			var bestBlocksToFetch = coinToTxOut.Select(c => c.Value.BestBlock).ToHashSet().ToList();
+			var bestBlocks = await rpc.GetBlockHeadersAsync(bestBlocksToFetch, cancellationToken);
+			var coinsWithHeights = coinToTxOut
+				.Select(c => new
+				{
+					BestBlock = bestBlocks.ByHashes.TryGet(c.Value.BestBlock),
+					Outpoint = c.Key,
+					RPCTxOut = c.Value
+				})
+				.Select(c => new
+				{
+					Height = c.RPCTxOut.Confirmations == 0 ? null : new int?(c.BestBlock.Height - c.RPCTxOut.Confirmations + 1),
+					c.Outpoint,
+					c.RPCTxOut
+				})
+				.ToList();
+			var blocks = coinsWithHeights.Where(c => c.Height.HasValue).Select(c => c.Height.Value).Distinct().ToList();
+			var blockHeaders = await rpc.GetBlockHeadersAsync(blocks, cancellationToken);
+
+			var now = DateTimeOffset.UtcNow;
+			var records = new List<SaveTransactionRecord>();
+			MatchQuery query = new MatchQuery(coinsWithHeights.Select(c => new Coin(c.Outpoint, c.RPCTxOut.TxOut)));
+			foreach (var c in coinsWithHeights)
+			{
+				var block = c.Height is int h ? blockHeaders.ByHeight.TryGet(h) : null;
+				var record = SaveTransactionRecord.Create(
+									txHash: c.Outpoint.Hash,
+									slimBlock: block?.ToSlimChainedBlock(),
+									seenAt: Extensions.MinDate(block?.Time ?? now, now));
+				records.Add(record);
+			}
+			await repo.SaveBlocks(blockHeaders);
+			repo.RemoveFromCache(records.Select(r => r.Id));
+			var trackedTransactions = await repo.SaveMatches(query, records.ToArray());
+			_ = AddressPoolService.GenerateAddresses(trackedSourceContext.Network, trackedTransactions);
+
+			return Ok();
+		}
+
 		internal async Task<AnnotatedTransactionCollection> GetAnnotatedTransactions(Repository repo, TrackedSource trackedSource, bool includeTransaction, uint256 txId = null)
 		{
 			var transactions = await repo.GetTransactions(trackedSource, txId, includeTransaction, this.HttpContext?.RequestAborted ?? default);

NBXplorer/RPCClientExtensions.cs

@@ -379,5 +379,22 @@ public async static Task ImportDescriptors(this RPCClient rpc, string descriptor
 			}
 			throw new NotSupportedException($"Bug of NBXplorer (ERR 3083), please notify the developers");
 		}
+		public static async Task<Dictionary<OutPoint, GetTxOutResponse>> GetTxOuts(this RPCClient rpc, IList<OutPoint> outpoints)
+		{
+			var batch = rpc.PrepareBatch();
+			var txOuts = outpoints.Select(o => batch.GetTxOutAsync(o.Hash, (int)o.N, true)).ToArray();
+			await batch.SendBatchAsync();
+			var result = new Dictionary<OutPoint, GetTxOutResponse>();
+			int i = 0;
+			foreach (var txOut in txOuts)
+			{
+				var outpoint = outpoints[i];
+				var r = await txOut;
+				if (r != null)
+					result.TryAdd(outpoint, r);
+				i++;
+			}
+			return result;
+		}
 	}
 }

NBXplorer/wwwroot/api.json

@@ -2020,6 +2020,45 @@
         }
       }
     },
+    "/v1/cryptos/{cryptoCode}/rescan-utxos": {
+      "post": {
+        "summary": "Rescan utxos",
+        "description": "Verifies that the UTXOs are unspent and save matches to wallets tracked by this server",
+        "tags": [
+          "Blockchain"
+        ],
+        "parameters": [
+          {
+            "$ref": "#/components/parameters/CryptoCode"
+          }
+        ],
+        "requestBody": {
+          "content": {
+            "application/json": {
+              "schema": {
+                "type": "object",
+                "properties": {
+                  "UTXOs": {
+                    "type": "array",
+                    "description": "The outpoints to rescan",
+                    "items": {
+                      "type": "string",
+                      "example": "f3e34c3ae29c79ddf807de0ab3b40da7862adb1b175b7421d89ec78446a52b13-1"
+                    },
+                    "example": [ "f3e34c3ae29c79ddf807de0ab3b40da7862adb1b175b7421d89ec78446a52b13-1", "fd97fb1de63fcdb9fa1614a74775e84818b2dbd79fe36aef9e0c18f9fad03742-2" ]
+                  }
+                }
+              }
+            }
+          }
+        },
+        "responses": {
+          "200": {
+            "description": "Rescan initiated successfully."
+          }
+        }
+      }
+    },
     "/v1/cryptos/{cryptoCode}/psbt/update": {
       "post": {
         "summary": "Update Partially Signed Bitcoin Transaction",
@@ -3728,4 +3767,4 @@
       }
     }
   }
-}
+}