From f648cde47f6201b9ec1050d555cdb3c33a10b79f Mon Sep 17 00:00:00 2001 From: Enzo Cioppettini Date: Thu, 11 Jan 2024 17:41:25 -0300 Subject: [PATCH 01/14] transaction history: add input addresses, metadata and slots filter --- .../TransactionHistoryController.ts | 35 ++++++- .../slotBoundsPagination.queries.ts | 57 +++++++++++ .../pagination/slotBoundsPagination.sql | 25 +++++ .../sqlHistoryForAddresses.queries.ts | 14 ++- .../transaction/sqlHistoryForAddresses.sql | 8 +- .../sqlHistoryForCredentials.queries.ts | 14 ++- .../transaction/sqlHistoryForCredentials.sql | 10 +- .../app/services/TransactionHistoryService.ts | 98 +++++++++++++++++++ webserver/shared/models/TransactionHistory.ts | 18 ++++ 9 files changed, 270 insertions(+), 9 deletions(-) create mode 100644 webserver/server/app/models/pagination/slotBoundsPagination.queries.ts create mode 100644 webserver/server/app/models/pagination/slotBoundsPagination.sql diff --git a/webserver/server/app/controllers/TransactionHistoryController.ts b/webserver/server/app/controllers/TransactionHistoryController.ts index 36609e73..7278487e 100644 --- a/webserver/server/app/controllers/TransactionHistoryController.ts +++ b/webserver/server/app/controllers/TransactionHistoryController.ts @@ -15,6 +15,7 @@ import { Routes } from '../../../shared/routes'; import sortBy from 'lodash/sortBy'; import { getAddressTypes } from '../models/utils'; import { RelationFilterType } from '../../../shared/models/common'; +import { slotBoundsPagination } from '../models/pagination/slotBoundsPagination.queries'; const route = Routes.transactionHistory; @@ -62,7 +63,7 @@ export class TransactionHistoryController extends Controller { const cardanoTxs = await tx< ErrorShape | [TransactionHistoryResponse, TransactionHistoryResponse] >(pool, async dbTx => { - const [until, pageStart] = await Promise.all([ + const [until, pageStart, slotBounds] = await Promise.all([ resolveUntilTransaction({ block_hash: Buffer.from(requestBody.untilBlock, 'hex'), dbTx, @@ -74,6 +75,12 @@ export class TransactionHistoryController extends Controller { after_tx: Buffer.from(requestBody.after.tx, 'hex'), dbTx, }), + !requestBody.slotLimits + ? Promise.resolve(undefined) + : slotBoundsPagination.run( + { low: requestBody.slotLimits.from, high: requestBody.slotLimits.to }, + dbTx + ), ]); if (until == null) { return genErrorMessage(Errors.BlockHashNotFound, { @@ -87,8 +94,32 @@ export class TransactionHistoryController extends Controller { }); } + let pageStartWithSlot = pageStart; + + if (requestBody.slotLimits) { + const bounds = slotBounds ? slotBounds[0] : { min_tx_id: -1, max_tx_id: -2 }; + + const minTxId = Number(bounds.min_tx_id); + + if (!pageStartWithSlot) { + pageStartWithSlot = { + block_id: -1, + tx_id: minTxId, + }; + } else { + if (minTxId > pageStartWithSlot.tx_id) { + pageStartWithSlot.tx_id = minTxId; + } + } + + const maxTxId = Number(bounds.max_tx_id); + if (maxTxId < until.tx_id) { + until.tx_id = maxTxId; + } + } + const commonRequest = { - after: pageStart, + after: pageStartWithSlot, limit: requestBody.limit ?? ADDRESS_LIMIT.RESPONSE, until, dbTx, diff --git a/webserver/server/app/models/pagination/slotBoundsPagination.queries.ts b/webserver/server/app/models/pagination/slotBoundsPagination.queries.ts new file mode 100644 index 00000000..ba3c7103 --- /dev/null +++ b/webserver/server/app/models/pagination/slotBoundsPagination.queries.ts @@ -0,0 +1,57 @@ +/** Types generated for queries found in "app/models/pagination/slotBoundsPagination.sql" */ +import { PreparedQuery } from '@pgtyped/runtime'; + +/** 'SlotBoundsPagination' parameters type */ +export interface ISlotBoundsPaginationParams { + high: number; + low: number; +} + +/** 'SlotBoundsPagination' return type */ +export interface ISlotBoundsPaginationResult { + max_slot: number; + max_tx_id: string | null; + min_slot: number; + min_tx_id: string | null; +} + +/** 'SlotBoundsPagination' query type */ +export interface ISlotBoundsPaginationQuery { + params: ISlotBoundsPaginationParams; + result: ISlotBoundsPaginationResult; +} + +const slotBoundsPaginationIR: any = {"usedParamSet":{"low":true,"high":true},"params":[{"name":"low","required":true,"transform":{"type":"scalar"},"locs":[{"a":193,"b":197}]},{"name":"high","required":true,"transform":{"type":"scalar"},"locs":[{"a":449,"b":454}]}],"statement":"WITH MIN_HASH AS\n\t(SELECT COALESCE(\"Transaction\".ID,\n\n\t\t\t\t\t\t\t\t\t\t-1) AS MIN_TX_ID,\n\t\t\tSLOT AS MIN_SLOT\n\t\tFROM \"Transaction\"\n\t\tJOIN \"Block\" ON \"Block\".ID = \"Transaction\".BLOCK_ID\n\t\tWHERE SLOT <= :low!\n\t\tORDER BY \"Block\".ID DESC, \"Transaction\".ID DESC\n\t\tLIMIT 1),\n\tMAX_HASH AS\n\t(SELECT SLOT AS MAX_SLOT,\n\t\t\tCOALESCE(MAX(\"Transaction\".ID),\n\n\t\t\t\t-2) AS MAX_TX_ID\n\t\tFROM \"Transaction\"\n\t\tJOIN \"Block\" ON \"Transaction\".BLOCK_ID = \"Block\".ID\n\t\tWHERE SLOT <= :high!\n\t\tGROUP BY \"Block\".ID\n\t\tORDER BY \"Block\".ID DESC\n\t\tLIMIT 1)\nSELECT *\nFROM MIN_HASH\nLEFT JOIN MAX_HASH ON 1 = 1"}; + +/** + * Query generated from SQL: + * ``` + * WITH MIN_HASH AS + * (SELECT COALESCE("Transaction".ID, + * + * -1) AS MIN_TX_ID, + * SLOT AS MIN_SLOT + * FROM "Transaction" + * JOIN "Block" ON "Block".ID = "Transaction".BLOCK_ID + * WHERE SLOT <= :low! + * ORDER BY "Block".ID DESC, "Transaction".ID DESC + * LIMIT 1), + * MAX_HASH AS + * (SELECT SLOT AS MAX_SLOT, + * COALESCE(MAX("Transaction".ID), + * + * -2) AS MAX_TX_ID + * FROM "Transaction" + * JOIN "Block" ON "Transaction".BLOCK_ID = "Block".ID + * WHERE SLOT <= :high! + * GROUP BY "Block".ID + * ORDER BY "Block".ID DESC + * LIMIT 1) + * SELECT * + * FROM MIN_HASH + * LEFT JOIN MAX_HASH ON 1 = 1 + * ``` + */ +export const slotBoundsPagination = new PreparedQuery(slotBoundsPaginationIR); + + diff --git a/webserver/server/app/models/pagination/slotBoundsPagination.sql b/webserver/server/app/models/pagination/slotBoundsPagination.sql new file mode 100644 index 00000000..c0b19c05 --- /dev/null +++ b/webserver/server/app/models/pagination/slotBoundsPagination.sql @@ -0,0 +1,25 @@ +/* @name slotBoundsPagination */ +WITH MIN_HASH AS + (SELECT COALESCE("Transaction".ID, + + -1) AS MIN_TX_ID, + SLOT AS MIN_SLOT + FROM "Transaction" + JOIN "Block" ON "Block".ID = "Transaction".BLOCK_ID + WHERE SLOT <= :low! + ORDER BY "Block".ID DESC, "Transaction".ID DESC + LIMIT 1), + MAX_HASH AS + (SELECT SLOT AS MAX_SLOT, + COALESCE(MAX("Transaction".ID), + + -2) AS MAX_TX_ID + FROM "Transaction" + JOIN "Block" ON "Transaction".BLOCK_ID = "Block".ID + WHERE SLOT <= :high! + GROUP BY "Block".ID + ORDER BY "Block".ID DESC + LIMIT 1) +SELECT * +FROM MIN_HASH +LEFT JOIN MAX_HASH ON 1 = 1; \ No newline at end of file diff --git a/webserver/server/app/models/transaction/sqlHistoryForAddresses.queries.ts b/webserver/server/app/models/transaction/sqlHistoryForAddresses.queries.ts index 10c81309..c9b8f040 100644 --- a/webserver/server/app/models/transaction/sqlHistoryForAddresses.queries.ts +++ b/webserver/server/app/models/transaction/sqlHistoryForAddresses.queries.ts @@ -3,6 +3,8 @@ import { PreparedQuery } from '@pgtyped/runtime'; export type BufferArray = (Buffer)[]; +export type Json = null | boolean | number | string | Json[] | { [key: string]: Json }; + export type NumberOrString = number | string; /** 'SqlHistoryForAddresses' parameters type */ @@ -21,7 +23,9 @@ export interface ISqlHistoryForAddressesResult { hash: Buffer; height: number; id: string; + input_addresses: Json | null; is_valid: boolean; + metadata: Buffer; payload: Buffer; slot: number; tx_index: number; @@ -33,7 +37,7 @@ export interface ISqlHistoryForAddressesQuery { result: ISqlHistoryForAddressesResult; } -const sqlHistoryForAddressesIR: any = {"usedParamSet":{"addresses":true,"until_tx_id":true,"after_tx_id":true,"limit":true},"params":[{"name":"addresses","required":false,"transform":{"type":"scalar"},"locs":[{"a":91,"b":100}]},{"name":"until_tx_id","required":false,"transform":{"type":"scalar"},"locs":[{"a":373,"b":384},{"a":788,"b":799},{"a":1250,"b":1261}]},{"name":"after_tx_id","required":false,"transform":{"type":"scalar"},"locs":[{"a":440,"b":451},{"a":854,"b":865},{"a":1325,"b":1336}]},{"name":"limit","required":false,"transform":{"type":"scalar"},"locs":[{"a":516,"b":521},{"a":929,"b":934},{"a":1409,"b":1414},{"a":1924,"b":1929}]}],"statement":"WITH\n address_row AS (\n SELECT *\n FROM \"Address\"\n WHERE \"Address\".payload = ANY (:addresses)\n ),\n outputs AS (\n SELECT DISTINCT ON (\"TransactionOutput\".tx_id) \"TransactionOutput\".tx_id\n FROM \"TransactionOutput\"\n INNER JOIN address_row ON \"TransactionOutput\".address_id = address_row.id\n WHERE\n \"TransactionOutput\".tx_id <= (:until_tx_id)\n AND\n \"TransactionOutput\".tx_id > (:after_tx_id)\n ORDER BY \"TransactionOutput\".tx_id ASC\n LIMIT (:limit)\n ),\n inputs AS (\n SELECT DISTINCT ON (\"TransactionInput\".tx_id) \"TransactionInput\".tx_id\n FROM \"TransactionInput\"\n INNER JOIN address_row ON \"TransactionInput\".address_id = address_row.id\n WHERE\n \"TransactionInput\".tx_id <= (:until_tx_id)\n AND\n \"TransactionInput\".tx_id > (:after_tx_id)\n ORDER BY \"TransactionInput\".tx_id ASC\n LIMIT (:limit)\n ),\n ref_inputs AS (\n SELECT DISTINCT ON (\"TransactionReferenceInput\".tx_id) \"TransactionReferenceInput\".tx_id\n FROM \"TransactionReferenceInput\"\n INNER JOIN address_row ON \"TransactionReferenceInput\".address_id = address_row.id\n WHERE\n \"TransactionReferenceInput\".tx_id <= (:until_tx_id)\n AND\n \"TransactionReferenceInput\".tx_id > (:after_tx_id)\n ORDER BY \"TransactionReferenceInput\".tx_id ASC\n LIMIT (:limit)\n )\nSELECT \"Transaction\".id,\n \"Transaction\".payload,\n \"Transaction\".hash,\n \"Transaction\".tx_index,\n \"Transaction\".is_valid,\n \"Block\".hash AS block_hash,\n \"Block\".epoch,\n \"Block\".slot,\n \"Block\".era,\n \"Block\".height\nFROM \"Transaction\"\nINNER JOIN \"Block\" ON \"Transaction\".block_id = \"Block\".id\nWHERE \"Transaction\".id IN (SELECT * FROM inputs UNION ALL SELECT * from ref_inputs UNION ALL SELECT * from outputs)\nORDER BY \"Transaction\".id ASC\nLIMIT (:limit)"}; +const sqlHistoryForAddressesIR: any = {"usedParamSet":{"addresses":true,"until_tx_id":true,"after_tx_id":true,"limit":true},"params":[{"name":"addresses","required":false,"transform":{"type":"scalar"},"locs":[{"a":91,"b":100}]},{"name":"until_tx_id","required":false,"transform":{"type":"scalar"},"locs":[{"a":373,"b":384},{"a":788,"b":799},{"a":1250,"b":1261}]},{"name":"after_tx_id","required":false,"transform":{"type":"scalar"},"locs":[{"a":440,"b":451},{"a":854,"b":865},{"a":1325,"b":1336}]},{"name":"limit","required":false,"transform":{"type":"scalar"},"locs":[{"a":516,"b":521},{"a":929,"b":934},{"a":1409,"b":1414},{"a":2329,"b":2334}]}],"statement":"WITH\n address_row AS (\n SELECT *\n FROM \"Address\"\n WHERE \"Address\".payload = ANY (:addresses)\n ),\n outputs AS (\n SELECT DISTINCT ON (\"TransactionOutput\".tx_id) \"TransactionOutput\".tx_id\n FROM \"TransactionOutput\"\n INNER JOIN address_row ON \"TransactionOutput\".address_id = address_row.id\n WHERE\n \"TransactionOutput\".tx_id <= (:until_tx_id)\n AND\n \"TransactionOutput\".tx_id > (:after_tx_id)\n ORDER BY \"TransactionOutput\".tx_id ASC\n LIMIT (:limit)\n ),\n inputs AS (\n SELECT DISTINCT ON (\"TransactionInput\".tx_id) \"TransactionInput\".tx_id\n FROM \"TransactionInput\"\n INNER JOIN address_row ON \"TransactionInput\".address_id = address_row.id\n WHERE\n \"TransactionInput\".tx_id <= (:until_tx_id)\n AND\n \"TransactionInput\".tx_id > (:after_tx_id)\n ORDER BY \"TransactionInput\".tx_id ASC\n LIMIT (:limit)\n ),\n ref_inputs AS (\n SELECT DISTINCT ON (\"TransactionReferenceInput\".tx_id) \"TransactionReferenceInput\".tx_id\n FROM \"TransactionReferenceInput\"\n INNER JOIN address_row ON \"TransactionReferenceInput\".address_id = address_row.id\n WHERE\n \"TransactionReferenceInput\".tx_id <= (:until_tx_id)\n AND\n \"TransactionReferenceInput\".tx_id > (:after_tx_id)\n ORDER BY \"TransactionReferenceInput\".tx_id ASC\n LIMIT (:limit)\n )\nSELECT \"Transaction\".id,\n \"Transaction\".payload,\n \"Transaction\".hash,\n \"Transaction\".tx_index,\n \"Transaction\".is_valid,\n \"Block\".hash AS block_hash,\n \"Block\".epoch,\n \"Block\".slot,\n \"Block\".era,\n \"Block\".height,\n \"TransactionMetadata\".payload AS metadata,\n json_agg(DISTINCT \"Address\".PAYLOAD) input_addresses\nFROM \"Transaction\"\nINNER JOIN \"Block\" ON \"Transaction\".block_id = \"Block\".id\nINNER JOIN \"TransactionInput\" ON \"TransactionInput\".tx_id = \"Transaction\".id\nINNER JOIN \"Address\" ON \"Address\".id = \"TransactionInput\".address_id\nLEFT JOIN \"TransactionMetadata\" ON \"Transaction\".id = \"TransactionMetadata\".tx_id\nWHERE \"Transaction\".id IN (SELECT * FROM inputs UNION ALL SELECT * from ref_inputs UNION ALL SELECT * from outputs)\nGROUP BY \"Transaction\".id, \"Block\".id, \"TransactionMetadata\".id\nORDER BY \"Transaction\".id ASC\nLIMIT (:limit)"}; /** * Query generated from SQL: @@ -86,10 +90,16 @@ const sqlHistoryForAddressesIR: any = {"usedParamSet":{"addresses":true,"until_t * "Block".epoch, * "Block".slot, * "Block".era, - * "Block".height + * "Block".height, + * "TransactionMetadata".payload AS metadata, + * json_agg(DISTINCT "Address".PAYLOAD) input_addresses * FROM "Transaction" * INNER JOIN "Block" ON "Transaction".block_id = "Block".id + * INNER JOIN "TransactionInput" ON "TransactionInput".tx_id = "Transaction".id + * INNER JOIN "Address" ON "Address".id = "TransactionInput".address_id + * LEFT JOIN "TransactionMetadata" ON "Transaction".id = "TransactionMetadata".tx_id * WHERE "Transaction".id IN (SELECT * FROM inputs UNION ALL SELECT * from ref_inputs UNION ALL SELECT * from outputs) + * GROUP BY "Transaction".id, "Block".id, "TransactionMetadata".id * ORDER BY "Transaction".id ASC * LIMIT (:limit) * ``` diff --git a/webserver/server/app/models/transaction/sqlHistoryForAddresses.sql b/webserver/server/app/models/transaction/sqlHistoryForAddresses.sql index b57b7fcd..ac3d7d51 100644 --- a/webserver/server/app/models/transaction/sqlHistoryForAddresses.sql +++ b/webserver/server/app/models/transaction/sqlHistoryForAddresses.sql @@ -47,9 +47,15 @@ SELECT "Transaction".id, "Block".epoch, "Block".slot, "Block".era, - "Block".height + "Block".height, + "TransactionMetadata".payload AS metadata, + json_agg(DISTINCT "Address".PAYLOAD) input_addresses FROM "Transaction" INNER JOIN "Block" ON "Transaction".block_id = "Block".id +INNER JOIN "TransactionInput" ON "TransactionInput".tx_id = "Transaction".id +INNER JOIN "Address" ON "Address".id = "TransactionInput".address_id +LEFT JOIN "TransactionMetadata" ON "Transaction".id = "TransactionMetadata".tx_id WHERE "Transaction".id IN (SELECT * FROM inputs UNION ALL SELECT * from ref_inputs UNION ALL SELECT * from outputs) +GROUP BY "Transaction".id, "Block".id, "TransactionMetadata".id ORDER BY "Transaction".id ASC LIMIT (:limit); \ No newline at end of file diff --git a/webserver/server/app/models/transaction/sqlHistoryForCredentials.queries.ts b/webserver/server/app/models/transaction/sqlHistoryForCredentials.queries.ts index e66b4f15..50c042d1 100644 --- a/webserver/server/app/models/transaction/sqlHistoryForCredentials.queries.ts +++ b/webserver/server/app/models/transaction/sqlHistoryForCredentials.queries.ts @@ -3,6 +3,8 @@ import { PreparedQuery } from '@pgtyped/runtime'; export type BufferArray = (Buffer)[]; +export type Json = null | boolean | number | string | Json[] | { [key: string]: Json }; + export type NumberOrString = number | string; /** 'SqlHistoryForCredentials' parameters type */ @@ -22,7 +24,9 @@ export interface ISqlHistoryForCredentialsResult { hash: Buffer; height: number; id: string; + input_addresses: Json | null; is_valid: boolean; + metadata: Buffer; payload: Buffer; slot: number; tx_index: number; @@ -34,7 +38,7 @@ export interface ISqlHistoryForCredentialsQuery { result: ISqlHistoryForCredentialsResult; } -const sqlHistoryForCredentialsIR: any = {"usedParamSet":{"credentials":true,"relation":true,"until_tx_id":true,"after_tx_id":true,"limit":true},"params":[{"name":"credentials","required":false,"transform":{"type":"scalar"},"locs":[{"a":288,"b":299}]},{"name":"relation","required":false,"transform":{"type":"scalar"},"locs":[{"a":354,"b":362}]},{"name":"until_tx_id","required":false,"transform":{"type":"scalar"},"locs":[{"a":464,"b":475}]},{"name":"after_tx_id","required":false,"transform":{"type":"scalar"},"locs":[{"a":527,"b":538}]},{"name":"limit","required":false,"transform":{"type":"scalar"},"locs":[{"a":598,"b":603}]}],"statement":"WITH\n tx_relations AS (\n SELECT DISTINCT ON (\"TxCredentialRelation\".tx_id) \"TxCredentialRelation\".tx_id\n FROM \"StakeCredential\"\n INNER JOIN \"TxCredentialRelation\" ON \"TxCredentialRelation\".credential_id = \"StakeCredential\".id\n WHERE\n \"StakeCredential\".credential = ANY (:credentials)\n AND\n (\"TxCredentialRelation\".relation & (:relation)) > 0\n AND\n \n \"TxCredentialRelation\".tx_id <= (:until_tx_id)\n AND \n \"TxCredentialRelation\".tx_id > (:after_tx_id)\n ORDER BY \"TxCredentialRelation\".tx_id ASC\n LIMIT (:limit)\n )\nSELECT \"Transaction\".id,\n \"Transaction\".payload,\n \"Transaction\".hash,\n \"Transaction\".tx_index,\n \"Transaction\".is_valid,\n \"Block\".hash AS block_hash,\n \"Block\".epoch,\n \"Block\".slot,\n \"Block\".era,\n \"Block\".height\nFROM tx_relations\nINNER JOIN \"Transaction\" ON tx_relations.tx_id = \"Transaction\".id\nINNER JOIN \"Block\" ON \"Transaction\".block_id = \"Block\".id"}; +const sqlHistoryForCredentialsIR: any = {"usedParamSet":{"credentials":true,"relation":true,"until_tx_id":true,"after_tx_id":true,"limit":true},"params":[{"name":"credentials","required":false,"transform":{"type":"scalar"},"locs":[{"a":288,"b":299}]},{"name":"relation","required":false,"transform":{"type":"scalar"},"locs":[{"a":354,"b":362}]},{"name":"until_tx_id","required":false,"transform":{"type":"scalar"},"locs":[{"a":464,"b":475}]},{"name":"after_tx_id","required":false,"transform":{"type":"scalar"},"locs":[{"a":527,"b":538}]},{"name":"limit","required":false,"transform":{"type":"scalar"},"locs":[{"a":598,"b":603}]}],"statement":"WITH\n tx_relations AS (\n SELECT DISTINCT ON (\"TxCredentialRelation\".tx_id) \"TxCredentialRelation\".tx_id\n FROM \"StakeCredential\"\n INNER JOIN \"TxCredentialRelation\" ON \"TxCredentialRelation\".credential_id = \"StakeCredential\".id\n WHERE\n \"StakeCredential\".credential = ANY (:credentials)\n AND\n (\"TxCredentialRelation\".relation & (:relation)) > 0\n AND\n \n \"TxCredentialRelation\".tx_id <= (:until_tx_id)\n AND \n \"TxCredentialRelation\".tx_id > (:after_tx_id)\n ORDER BY \"TxCredentialRelation\".tx_id ASC\n LIMIT (:limit)\n )\nSELECT \"Transaction\".id,\n \"Transaction\".payload,\n \"Transaction\".hash,\n \"Transaction\".tx_index,\n \"Transaction\".is_valid,\n \"Block\".hash AS block_hash,\n \"Block\".epoch,\n \"Block\".slot,\n \"Block\".era,\n \"Block\".height,\n \"TransactionMetadata\".payload AS metadata,\n json_agg(DISTINCT \"Address\".PAYLOAD) input_addresses\nFROM tx_relations\nINNER JOIN \"Transaction\" ON tx_relations.tx_id = \"Transaction\".id\nINNER JOIN \"TransactionInput\" ON \"TransactionInput\".tx_id = \"Transaction\".id\nINNER JOIN \"Address\" ON \"Address\".id = \"TransactionInput\".address_id\nLEFT JOIN \"TransactionMetadata\" ON \"Transaction\".id = \"TransactionMetadata\".tx_id\nINNER JOIN \"Block\" ON \"Transaction\".block_id = \"Block\".id\nGROUP BY \"Transaction\".id, \"Block\".id, \"TransactionMetadata\".id"}; /** * Query generated from SQL: @@ -65,10 +69,16 @@ const sqlHistoryForCredentialsIR: any = {"usedParamSet":{"credentials":true,"rel * "Block".epoch, * "Block".slot, * "Block".era, - * "Block".height + * "Block".height, + * "TransactionMetadata".payload AS metadata, + * json_agg(DISTINCT "Address".PAYLOAD) input_addresses * FROM tx_relations * INNER JOIN "Transaction" ON tx_relations.tx_id = "Transaction".id + * INNER JOIN "TransactionInput" ON "TransactionInput".tx_id = "Transaction".id + * INNER JOIN "Address" ON "Address".id = "TransactionInput".address_id + * LEFT JOIN "TransactionMetadata" ON "Transaction".id = "TransactionMetadata".tx_id * INNER JOIN "Block" ON "Transaction".block_id = "Block".id + * GROUP BY "Transaction".id, "Block".id, "TransactionMetadata".id * ``` */ export const sqlHistoryForCredentials = new PreparedQuery(sqlHistoryForCredentialsIR); diff --git a/webserver/server/app/models/transaction/sqlHistoryForCredentials.sql b/webserver/server/app/models/transaction/sqlHistoryForCredentials.sql index 822388a3..78558ada 100644 --- a/webserver/server/app/models/transaction/sqlHistoryForCredentials.sql +++ b/webserver/server/app/models/transaction/sqlHistoryForCredentials.sql @@ -25,8 +25,14 @@ SELECT "Transaction".id, "Block".epoch, "Block".slot, "Block".era, - "Block".height + "Block".height, + "TransactionMetadata".payload AS metadata, + json_agg(DISTINCT "Address".PAYLOAD) input_addresses FROM tx_relations INNER JOIN "Transaction" ON tx_relations.tx_id = "Transaction".id -INNER JOIN "Block" ON "Transaction".block_id = "Block".id; +INNER JOIN "TransactionInput" ON "TransactionInput".tx_id = "Transaction".id +INNER JOIN "Address" ON "Address".id = "TransactionInput".address_id +LEFT JOIN "TransactionMetadata" ON "Transaction".id = "TransactionMetadata".tx_id +INNER JOIN "Block" ON "Transaction".block_id = "Block".id +GROUP BY "Transaction".id, "Block".id, "TransactionMetadata".id; diff --git a/webserver/server/app/services/TransactionHistoryService.ts b/webserver/server/app/services/TransactionHistoryService.ts index c8de3597..fa0d1643 100644 --- a/webserver/server/app/services/TransactionHistoryService.ts +++ b/webserver/server/app/services/TransactionHistoryService.ts @@ -4,6 +4,7 @@ import { sqlHistoryForAddresses } from '../models/transaction/sqlHistoryForAddre import type { PoolClient } from 'pg'; import type { TransactionPaginationType } from './PaginationService'; import type { RelationFilter } from '../../../shared/models/common'; +import { Address, Transaction } from '@dcspark/cardano-multiplatform-lib-nodejs'; export async function historyForCredentials( request: TransactionPaginationType & { @@ -39,6 +40,11 @@ export async function historyForCredentials( transaction: { hash: entry.hash.toString('hex'), payload: entry.payload.toString('hex'), + outputs: computeOutputs(entry.payload), + metadata: entry.metadata && entry.metadata.toString('hex'), + inputCredentials: entry.input_addresses + ? (entry.input_addresses as string[]).map(getPaymentCred) + : [], }, })), }; @@ -76,7 +82,99 @@ export async function historyForAddresses( transaction: { hash: entry.hash.toString('hex'), payload: entry.payload.toString('hex'), + outputs: computeOutputs(entry.payload), + metadata: entry.metadata && entry.metadata.toString('hex'), + inputCredentials: entry.input_addresses + ? (entry.input_addresses as string[]).map(getPaymentCred) + : [], }, })), }; } + +function computeOutputs( + tx: Buffer +): { asset: { policyId: string; assetName: string } | null; amount: string; address: string }[] { + const transaction = Transaction.from_bytes(tx); + + const rawOutputs = transaction.body().outputs(); + + const outputs = []; + + for (let i = 0; i < rawOutputs.len(); i++) { + const output = rawOutputs.get(i); + + const rawAddress = output.address(); + const address = rawAddress.to_bech32(); + rawAddress.free(); + + const amount = output.amount(); + const ma = amount.multiasset(); + + if (ma) { + const policyIds = ma.keys(); + + for (let j = 0; j < policyIds.len(); j++) { + const policyId = policyIds.get(j); + + const assets = ma.get(policyId); + + if (!assets) { + continue; + } + + const assetNames = assets.keys(); + + for (let k = 0; k < assetNames.len(); k++) { + const assetName = assetNames.get(k); + + const amount = assets.get(assetName); + + if (amount === undefined) { + continue; + } + + outputs.push({ + amount: amount.to_str(), + asset: { + policyId: policyId.to_hex(), + assetName: Buffer.from(assetName.to_bytes()).toString('hex'), + }, + address + }); + + assetName.free(); + } + + assetNames.free(); + assets.free(); + policyId.free(); + } + + policyIds.free(); + ma.free(); + } + + outputs.push({ amount: amount.coin().to_str(), asset: null, address }); + + amount.free(); + output.free(); + } + + rawOutputs.free(); + transaction.free(); + + return outputs; +} + +function getPaymentCred(addressRaw: string): string { + const address = Address.from_bytes(Buffer.from(addressRaw.slice(2), 'hex')); + + const paymentCred = address.payment_cred(); + const addressBytes = paymentCred?.to_bytes(); + + address.free(); + paymentCred?.free(); + + return Buffer.from(addressBytes as Uint8Array).toString('hex'); +} diff --git a/webserver/shared/models/TransactionHistory.ts b/webserver/shared/models/TransactionHistory.ts index 73935e5d..70f262ea 100644 --- a/webserver/shared/models/TransactionHistory.ts +++ b/webserver/shared/models/TransactionHistory.ts @@ -1,5 +1,6 @@ import type { Address } from "./Address"; import type { BlockSubset } from "./BlockLatest"; +import { AssetName, PolicyId } from "./PolicyIdAssetMap"; import type { Pagination, RelationFilter } from "./common"; export type TransactionHistoryRequest = { @@ -8,6 +9,8 @@ export type TransactionHistoryRequest = { relationFilter?: RelationFilter; /** Defaults to `ADDRESS_LIMIT.RESPONSE` */ limit?: number; + + slotLimits?: SlotLimits; } & Pagination; export type BlockInfo = BlockSubset & { @@ -33,6 +36,16 @@ export type TransactionInfo = { * @example "84a500818258209cb4f8c2eecccc9f1e13768046f37ef56dcb5a4dc44f58907fe4ae21d7cf621d020181825839019cb581f4337a6142e477af6e00fe41b1fc4a5944a575681b8499a3c0bd07ce733b5911eb657e7aff5d35f8b0682fe0380f7621af2bbcb2f71b0000000586321393021a0002a389031a004b418c048183028200581cbd07ce733b5911eb657e7aff5d35f8b0682fe0380f7621af2bbcb2f7581c53215c471b7ac752e3ddf8f2c4c1e6ed111857bfaa675d5e31ce8bcea1008282582073e584cda9fe483fbefb81c251e616018a2b493ef56820f0095b63adede54ff758404f13df42ef1684a3fd55255d8368c9ecbd15b55e2761a2991cc4f401a753c16d6da1da158e84b87b4de9715af7d9adc0d79a7c1f2c3097228e02b20be4616a0c82582066c606974819f457ceface78ee3c4d181a84ca9927a3cfc92ef8c0b6dd4576e8584014ae9ee9ed5eb5700b6c5ac270543671f5d4f943d4726f4614dc061174ee29db44b9e7fc58e6c98c13fad8594f2633c5ec70a9a87f5cbf130308a42edb553001f5f6" */ payload: string; + + outputs: { + asset: { policyId: PolicyId; assetName: AssetName } | null; + amount: string; + address: string; + }[]; + + metadata: string | null; + + inputCredentials: string[]; }; export type TxAndBlockInfo = { @@ -42,3 +55,8 @@ export type TxAndBlockInfo = { export type TransactionHistoryResponse = { transactions: TxAndBlockInfo[]; }; + +export type SlotLimits = { + from: number; + to: number; +}; From 368050d975fd3f7f4a98abf8a985dc7443be18e9 Mon Sep 17 00:00:00 2001 From: Enzo Cioppettini Date: Mon, 29 Jan 2024 18:11:56 -0300 Subject: [PATCH 02/14] remove outputs field from the response moved to the client --- .../TransactionHistoryController.ts | 4 +- .../app/services/TransactionHistoryService.ts | 79 +------------------ webserver/shared/models/TransactionHistory.ts | 6 -- 3 files changed, 4 insertions(+), 85 deletions(-) diff --git a/webserver/server/app/controllers/TransactionHistoryController.ts b/webserver/server/app/controllers/TransactionHistoryController.ts index 7278487e..a136be13 100644 --- a/webserver/server/app/controllers/TransactionHistoryController.ts +++ b/webserver/server/app/controllers/TransactionHistoryController.ts @@ -33,7 +33,9 @@ export class TransactionHistoryController extends Controller { requestBody: EndpointTypes[typeof route]['input'], @Res() errorResponse: TsoaResponse< - StatusCodes.CONFLICT | StatusCodes.BAD_REQUEST | StatusCodes.UNPROCESSABLE_ENTITY, + | StatusCodes.CONFLICT + | StatusCodes.BAD_REQUEST + | StatusCodes.UNPROCESSABLE_ENTITY, ErrorShape > ): Promise { diff --git a/webserver/server/app/services/TransactionHistoryService.ts b/webserver/server/app/services/TransactionHistoryService.ts index fa0d1643..eef02778 100644 --- a/webserver/server/app/services/TransactionHistoryService.ts +++ b/webserver/server/app/services/TransactionHistoryService.ts @@ -4,7 +4,7 @@ import { sqlHistoryForAddresses } from '../models/transaction/sqlHistoryForAddre import type { PoolClient } from 'pg'; import type { TransactionPaginationType } from './PaginationService'; import type { RelationFilter } from '../../../shared/models/common'; -import { Address, Transaction } from '@dcspark/cardano-multiplatform-lib-nodejs'; +import { Address } from '@dcspark/cardano-multiplatform-lib-nodejs'; export async function historyForCredentials( request: TransactionPaginationType & { @@ -40,7 +40,6 @@ export async function historyForCredentials( transaction: { hash: entry.hash.toString('hex'), payload: entry.payload.toString('hex'), - outputs: computeOutputs(entry.payload), metadata: entry.metadata && entry.metadata.toString('hex'), inputCredentials: entry.input_addresses ? (entry.input_addresses as string[]).map(getPaymentCred) @@ -82,7 +81,6 @@ export async function historyForAddresses( transaction: { hash: entry.hash.toString('hex'), payload: entry.payload.toString('hex'), - outputs: computeOutputs(entry.payload), metadata: entry.metadata && entry.metadata.toString('hex'), inputCredentials: entry.input_addresses ? (entry.input_addresses as string[]).map(getPaymentCred) @@ -92,81 +90,6 @@ export async function historyForAddresses( }; } -function computeOutputs( - tx: Buffer -): { asset: { policyId: string; assetName: string } | null; amount: string; address: string }[] { - const transaction = Transaction.from_bytes(tx); - - const rawOutputs = transaction.body().outputs(); - - const outputs = []; - - for (let i = 0; i < rawOutputs.len(); i++) { - const output = rawOutputs.get(i); - - const rawAddress = output.address(); - const address = rawAddress.to_bech32(); - rawAddress.free(); - - const amount = output.amount(); - const ma = amount.multiasset(); - - if (ma) { - const policyIds = ma.keys(); - - for (let j = 0; j < policyIds.len(); j++) { - const policyId = policyIds.get(j); - - const assets = ma.get(policyId); - - if (!assets) { - continue; - } - - const assetNames = assets.keys(); - - for (let k = 0; k < assetNames.len(); k++) { - const assetName = assetNames.get(k); - - const amount = assets.get(assetName); - - if (amount === undefined) { - continue; - } - - outputs.push({ - amount: amount.to_str(), - asset: { - policyId: policyId.to_hex(), - assetName: Buffer.from(assetName.to_bytes()).toString('hex'), - }, - address - }); - - assetName.free(); - } - - assetNames.free(); - assets.free(); - policyId.free(); - } - - policyIds.free(); - ma.free(); - } - - outputs.push({ amount: amount.coin().to_str(), asset: null, address }); - - amount.free(); - output.free(); - } - - rawOutputs.free(); - transaction.free(); - - return outputs; -} - function getPaymentCred(addressRaw: string): string { const address = Address.from_bytes(Buffer.from(addressRaw.slice(2), 'hex')); diff --git a/webserver/shared/models/TransactionHistory.ts b/webserver/shared/models/TransactionHistory.ts index 70f262ea..e5b1de3d 100644 --- a/webserver/shared/models/TransactionHistory.ts +++ b/webserver/shared/models/TransactionHistory.ts @@ -37,12 +37,6 @@ export type TransactionInfo = { */ payload: string; - outputs: { - asset: { policyId: PolicyId; assetName: AssetName } | null; - amount: string; - address: string; - }[]; - metadata: string | null; inputCredentials: string[]; From 5e736bfa20b24481a6e4fb7b3b920719b9d3c931 Mon Sep 17 00:00:00 2001 From: Enzo Cioppettini Date: Mon, 29 Jan 2024 21:59:29 -0300 Subject: [PATCH 03/14] update .nvmrc --- webserver/server/.nvmrc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webserver/server/.nvmrc b/webserver/server/.nvmrc index cab13a79..f274881e 100644 --- a/webserver/server/.nvmrc +++ b/webserver/server/.nvmrc @@ -1 +1 @@ -v14.17.0 +v16.16.0 From 945981bfa46e536d5935ad89abac8f961884bab2 Mon Sep 17 00:00:00 2001 From: Enzo Cioppettini Date: Tue, 30 Jan 2024 01:15:38 -0300 Subject: [PATCH 04/14] make input addresses and metadata optional (as an arg) --- .../TransactionHistoryController.ts | 1 + .../sqlHistoryForAddresses.queries.ts | 73 +++++++++++++------ .../transaction/sqlHistoryForAddresses.sql | 66 +++++++++++------ .../sqlHistoryForCredentials.queries.ts | 66 +++++++++++------ .../transaction/sqlHistoryForCredentials.sql | 59 ++++++++++----- .../app/services/TransactionHistoryService.ts | 36 ++++++--- webserver/shared/models/TransactionHistory.ts | 5 +- 7 files changed, 206 insertions(+), 100 deletions(-) diff --git a/webserver/server/app/controllers/TransactionHistoryController.ts b/webserver/server/app/controllers/TransactionHistoryController.ts index a136be13..9c9a1c0a 100644 --- a/webserver/server/app/controllers/TransactionHistoryController.ts +++ b/webserver/server/app/controllers/TransactionHistoryController.ts @@ -125,6 +125,7 @@ export class TransactionHistoryController extends Controller { limit: requestBody.limit ?? ADDRESS_LIMIT.RESPONSE, until, dbTx, + withInputContext: !!requestBody.withInputContext }; const result = await Promise.all([ historyForCredentials({ diff --git a/webserver/server/app/models/transaction/sqlHistoryForAddresses.queries.ts b/webserver/server/app/models/transaction/sqlHistoryForAddresses.queries.ts index c9b8f040..53899d00 100644 --- a/webserver/server/app/models/transaction/sqlHistoryForAddresses.queries.ts +++ b/webserver/server/app/models/transaction/sqlHistoryForAddresses.queries.ts @@ -13,6 +13,7 @@ export interface ISqlHistoryForAddressesParams { after_tx_id?: NumberOrString | null | void; limit?: NumberOrString | null | void; until_tx_id?: NumberOrString | null | void; + with_input_context: boolean; } /** 'SqlHistoryForAddresses' return type */ @@ -22,10 +23,10 @@ export interface ISqlHistoryForAddressesResult { era: number; hash: Buffer; height: number; - id: string; + id: string | null; input_addresses: Json | null; is_valid: boolean; - metadata: Buffer; + metadata: Buffer | null; payload: Buffer; slot: number; tx_index: number; @@ -37,7 +38,7 @@ export interface ISqlHistoryForAddressesQuery { result: ISqlHistoryForAddressesResult; } -const sqlHistoryForAddressesIR: any = {"usedParamSet":{"addresses":true,"until_tx_id":true,"after_tx_id":true,"limit":true},"params":[{"name":"addresses","required":false,"transform":{"type":"scalar"},"locs":[{"a":91,"b":100}]},{"name":"until_tx_id","required":false,"transform":{"type":"scalar"},"locs":[{"a":373,"b":384},{"a":788,"b":799},{"a":1250,"b":1261}]},{"name":"after_tx_id","required":false,"transform":{"type":"scalar"},"locs":[{"a":440,"b":451},{"a":854,"b":865},{"a":1325,"b":1336}]},{"name":"limit","required":false,"transform":{"type":"scalar"},"locs":[{"a":516,"b":521},{"a":929,"b":934},{"a":1409,"b":1414},{"a":2329,"b":2334}]}],"statement":"WITH\n address_row AS (\n SELECT *\n FROM \"Address\"\n WHERE \"Address\".payload = ANY (:addresses)\n ),\n outputs AS (\n SELECT DISTINCT ON (\"TransactionOutput\".tx_id) \"TransactionOutput\".tx_id\n FROM \"TransactionOutput\"\n INNER JOIN address_row ON \"TransactionOutput\".address_id = address_row.id\n WHERE\n \"TransactionOutput\".tx_id <= (:until_tx_id)\n AND\n \"TransactionOutput\".tx_id > (:after_tx_id)\n ORDER BY \"TransactionOutput\".tx_id ASC\n LIMIT (:limit)\n ),\n inputs AS (\n SELECT DISTINCT ON (\"TransactionInput\".tx_id) \"TransactionInput\".tx_id\n FROM \"TransactionInput\"\n INNER JOIN address_row ON \"TransactionInput\".address_id = address_row.id\n WHERE\n \"TransactionInput\".tx_id <= (:until_tx_id)\n AND\n \"TransactionInput\".tx_id > (:after_tx_id)\n ORDER BY \"TransactionInput\".tx_id ASC\n LIMIT (:limit)\n ),\n ref_inputs AS (\n SELECT DISTINCT ON (\"TransactionReferenceInput\".tx_id) \"TransactionReferenceInput\".tx_id\n FROM \"TransactionReferenceInput\"\n INNER JOIN address_row ON \"TransactionReferenceInput\".address_id = address_row.id\n WHERE\n \"TransactionReferenceInput\".tx_id <= (:until_tx_id)\n AND\n \"TransactionReferenceInput\".tx_id > (:after_tx_id)\n ORDER BY \"TransactionReferenceInput\".tx_id ASC\n LIMIT (:limit)\n )\nSELECT \"Transaction\".id,\n \"Transaction\".payload,\n \"Transaction\".hash,\n \"Transaction\".tx_index,\n \"Transaction\".is_valid,\n \"Block\".hash AS block_hash,\n \"Block\".epoch,\n \"Block\".slot,\n \"Block\".era,\n \"Block\".height,\n \"TransactionMetadata\".payload AS metadata,\n json_agg(DISTINCT \"Address\".PAYLOAD) input_addresses\nFROM \"Transaction\"\nINNER JOIN \"Block\" ON \"Transaction\".block_id = \"Block\".id\nINNER JOIN \"TransactionInput\" ON \"TransactionInput\".tx_id = \"Transaction\".id\nINNER JOIN \"Address\" ON \"Address\".id = \"TransactionInput\".address_id\nLEFT JOIN \"TransactionMetadata\" ON \"Transaction\".id = \"TransactionMetadata\".tx_id\nWHERE \"Transaction\".id IN (SELECT * FROM inputs UNION ALL SELECT * from ref_inputs UNION ALL SELECT * from outputs)\nGROUP BY \"Transaction\".id, \"Block\".id, \"TransactionMetadata\".id\nORDER BY \"Transaction\".id ASC\nLIMIT (:limit)"}; +const sqlHistoryForAddressesIR: any = {"usedParamSet":{"addresses":true,"until_tx_id":true,"after_tx_id":true,"limit":true,"with_input_context":true},"params":[{"name":"addresses","required":false,"transform":{"type":"scalar"},"locs":[{"a":91,"b":100}]},{"name":"until_tx_id","required":false,"transform":{"type":"scalar"},"locs":[{"a":373,"b":384},{"a":788,"b":799},{"a":1250,"b":1261}]},{"name":"after_tx_id","required":false,"transform":{"type":"scalar"},"locs":[{"a":440,"b":451},{"a":854,"b":865},{"a":1325,"b":1336}]},{"name":"limit","required":false,"transform":{"type":"scalar"},"locs":[{"a":516,"b":521},{"a":929,"b":934},{"a":1409,"b":1414},{"a":2215,"b":2220},{"a":3446,"b":3451}]},{"name":"with_input_context","required":true,"transform":{"type":"scalar"},"locs":[{"a":3493,"b":3512},{"a":3576,"b":3595}]}],"statement":"WITH\n address_row AS (\n SELECT *\n FROM \"Address\"\n WHERE \"Address\".payload = ANY (:addresses)\n ),\n outputs AS (\n SELECT DISTINCT ON (\"TransactionOutput\".tx_id) \"TransactionOutput\".tx_id\n FROM \"TransactionOutput\"\n INNER JOIN address_row ON \"TransactionOutput\".address_id = address_row.id\n WHERE\n \"TransactionOutput\".tx_id <= (:until_tx_id)\n AND\n \"TransactionOutput\".tx_id > (:after_tx_id)\n ORDER BY \"TransactionOutput\".tx_id ASC\n LIMIT (:limit)\n ),\n inputs AS (\n SELECT DISTINCT ON (\"TransactionInput\".tx_id) \"TransactionInput\".tx_id\n FROM \"TransactionInput\"\n INNER JOIN address_row ON \"TransactionInput\".address_id = address_row.id\n WHERE\n \"TransactionInput\".tx_id <= (:until_tx_id)\n AND\n \"TransactionInput\".tx_id > (:after_tx_id)\n ORDER BY \"TransactionInput\".tx_id ASC\n LIMIT (:limit)\n ),\n ref_inputs AS (\n SELECT DISTINCT ON (\"TransactionReferenceInput\".tx_id) \"TransactionReferenceInput\".tx_id\n FROM \"TransactionReferenceInput\"\n INNER JOIN address_row ON \"TransactionReferenceInput\".address_id = address_row.id\n WHERE\n \"TransactionReferenceInput\".tx_id <= (:until_tx_id)\n AND\n \"TransactionReferenceInput\".tx_id > (:after_tx_id)\n ORDER BY \"TransactionReferenceInput\".tx_id ASC\n LIMIT (:limit)\n ),\n base_query AS (\n SELECT \"Transaction\".id,\n \"Transaction\".payload as \"payload!\",\n \"Transaction\".hash as \"hash!\",\n \"Transaction\".tx_index as \"tx_index!\",\n \"Transaction\".is_valid as \"is_valid!\",\n \"Block\".hash AS \"block_hash!\",\n \"Block\".epoch as \"epoch!\",\n \"Block\".slot as \"slot!\",\n \"Block\".era as \"era!\",\n \"Block\".height as \"height!\",\n NULL :: bytea as metadata,\n NULL :: json as input_addresses\n FROM \"Transaction\"\n INNER JOIN \"Block\" ON \"Transaction\".block_id = \"Block\".id\n WHERE \"Transaction\".id IN (SELECT * FROM inputs UNION ALL SELECT * from ref_inputs UNION ALL SELECT * from outputs)\n ORDER BY \"Transaction\".id ASC\n LIMIT (:limit)\n ),\n query_with_inputs_and_metadata AS (\n SELECT \"Transaction\".id,\n \"Transaction\".payload as \"payload!\",\n \"Transaction\".hash as \"hash!\",\n \"Transaction\".tx_index as \"tx_index!\",\n \"Transaction\".is_valid as \"is_valid!\",\n \"Block\".hash AS \"block_hash!\",\n \"Block\".epoch as \"epoch!\",\n \"Block\".slot as \"slot!\",\n \"Block\".era as \"era!\",\n \"Block\".height as \"height!\",\n \"TransactionMetadata\".payload AS metadata,\n json_agg(DISTINCT \"Address\".PAYLOAD) input_addresses\n FROM \"Transaction\"\n INNER JOIN \"Block\" ON \"Transaction\".block_id = \"Block\".id\n INNER JOIN \"TransactionInput\" ON \"TransactionInput\".tx_id = \"Transaction\".id\n INNER JOIN \"Address\" ON \"Address\".id = \"TransactionInput\".address_id\n LEFT JOIN \"TransactionMetadata\" ON \"Transaction\".id = \"TransactionMetadata\".tx_id\n WHERE \"Transaction\".id IN (SELECT * FROM inputs UNION ALL SELECT * from ref_inputs UNION ALL SELECT * from outputs)\n GROUP BY \"Transaction\".id, \"Block\".id, \"TransactionMetadata\".id\n ORDER BY \"Transaction\".id ASC\n LIMIT (:limit)\n )\nSELECT * FROM base_query WHERE NOT :with_input_context!\nUNION ALL\n(SELECT * from query_with_inputs_and_metadata WHERE :with_input_context!)"}; /** * Query generated from SQL: @@ -80,28 +81,52 @@ const sqlHistoryForAddressesIR: any = {"usedParamSet":{"addresses":true,"until_t * "TransactionReferenceInput".tx_id > (:after_tx_id) * ORDER BY "TransactionReferenceInput".tx_id ASC * LIMIT (:limit) + * ), + * base_query AS ( + * SELECT "Transaction".id, + * "Transaction".payload as "payload!", + * "Transaction".hash as "hash!", + * "Transaction".tx_index as "tx_index!", + * "Transaction".is_valid as "is_valid!", + * "Block".hash AS "block_hash!", + * "Block".epoch as "epoch!", + * "Block".slot as "slot!", + * "Block".era as "era!", + * "Block".height as "height!", + * NULL :: bytea as metadata, + * NULL :: json as input_addresses + * FROM "Transaction" + * INNER JOIN "Block" ON "Transaction".block_id = "Block".id + * WHERE "Transaction".id IN (SELECT * FROM inputs UNION ALL SELECT * from ref_inputs UNION ALL SELECT * from outputs) + * ORDER BY "Transaction".id ASC + * LIMIT (:limit) + * ), + * query_with_inputs_and_metadata AS ( + * SELECT "Transaction".id, + * "Transaction".payload as "payload!", + * "Transaction".hash as "hash!", + * "Transaction".tx_index as "tx_index!", + * "Transaction".is_valid as "is_valid!", + * "Block".hash AS "block_hash!", + * "Block".epoch as "epoch!", + * "Block".slot as "slot!", + * "Block".era as "era!", + * "Block".height as "height!", + * "TransactionMetadata".payload AS metadata, + * json_agg(DISTINCT "Address".PAYLOAD) input_addresses + * FROM "Transaction" + * INNER JOIN "Block" ON "Transaction".block_id = "Block".id + * INNER JOIN "TransactionInput" ON "TransactionInput".tx_id = "Transaction".id + * INNER JOIN "Address" ON "Address".id = "TransactionInput".address_id + * LEFT JOIN "TransactionMetadata" ON "Transaction".id = "TransactionMetadata".tx_id + * WHERE "Transaction".id IN (SELECT * FROM inputs UNION ALL SELECT * from ref_inputs UNION ALL SELECT * from outputs) + * GROUP BY "Transaction".id, "Block".id, "TransactionMetadata".id + * ORDER BY "Transaction".id ASC + * LIMIT (:limit) * ) - * SELECT "Transaction".id, - * "Transaction".payload, - * "Transaction".hash, - * "Transaction".tx_index, - * "Transaction".is_valid, - * "Block".hash AS block_hash, - * "Block".epoch, - * "Block".slot, - * "Block".era, - * "Block".height, - * "TransactionMetadata".payload AS metadata, - * json_agg(DISTINCT "Address".PAYLOAD) input_addresses - * FROM "Transaction" - * INNER JOIN "Block" ON "Transaction".block_id = "Block".id - * INNER JOIN "TransactionInput" ON "TransactionInput".tx_id = "Transaction".id - * INNER JOIN "Address" ON "Address".id = "TransactionInput".address_id - * LEFT JOIN "TransactionMetadata" ON "Transaction".id = "TransactionMetadata".tx_id - * WHERE "Transaction".id IN (SELECT * FROM inputs UNION ALL SELECT * from ref_inputs UNION ALL SELECT * from outputs) - * GROUP BY "Transaction".id, "Block".id, "TransactionMetadata".id - * ORDER BY "Transaction".id ASC - * LIMIT (:limit) + * SELECT * FROM base_query WHERE NOT :with_input_context! + * UNION ALL + * (SELECT * from query_with_inputs_and_metadata WHERE :with_input_context!) * ``` */ export const sqlHistoryForAddresses = new PreparedQuery(sqlHistoryForAddressesIR); diff --git a/webserver/server/app/models/transaction/sqlHistoryForAddresses.sql b/webserver/server/app/models/transaction/sqlHistoryForAddresses.sql index ac3d7d51..34658c77 100644 --- a/webserver/server/app/models/transaction/sqlHistoryForAddresses.sql +++ b/webserver/server/app/models/transaction/sqlHistoryForAddresses.sql @@ -37,25 +37,49 @@ WITH "TransactionReferenceInput".tx_id > (:after_tx_id) ORDER BY "TransactionReferenceInput".tx_id ASC LIMIT (:limit) + ), + base_query AS ( + SELECT "Transaction".id, + "Transaction".payload as "payload!", + "Transaction".hash as "hash!", + "Transaction".tx_index as "tx_index!", + "Transaction".is_valid as "is_valid!", + "Block".hash AS "block_hash!", + "Block".epoch as "epoch!", + "Block".slot as "slot!", + "Block".era as "era!", + "Block".height as "height!", + NULL :: bytea as metadata, + NULL :: json as input_addresses + FROM "Transaction" + INNER JOIN "Block" ON "Transaction".block_id = "Block".id + WHERE "Transaction".id IN (SELECT * FROM inputs UNION ALL SELECT * from ref_inputs UNION ALL SELECT * from outputs) + ORDER BY "Transaction".id ASC + LIMIT (:limit) + ), + query_with_inputs_and_metadata AS ( + SELECT "Transaction".id, + "Transaction".payload as "payload!", + "Transaction".hash as "hash!", + "Transaction".tx_index as "tx_index!", + "Transaction".is_valid as "is_valid!", + "Block".hash AS "block_hash!", + "Block".epoch as "epoch!", + "Block".slot as "slot!", + "Block".era as "era!", + "Block".height as "height!", + "TransactionMetadata".payload AS metadata, + json_agg(DISTINCT "Address".PAYLOAD) input_addresses + FROM "Transaction" + INNER JOIN "Block" ON "Transaction".block_id = "Block".id + INNER JOIN "TransactionInput" ON "TransactionInput".tx_id = "Transaction".id + INNER JOIN "Address" ON "Address".id = "TransactionInput".address_id + LEFT JOIN "TransactionMetadata" ON "Transaction".id = "TransactionMetadata".tx_id + WHERE "Transaction".id IN (SELECT * FROM inputs UNION ALL SELECT * from ref_inputs UNION ALL SELECT * from outputs) + GROUP BY "Transaction".id, "Block".id, "TransactionMetadata".id + ORDER BY "Transaction".id ASC + LIMIT (:limit) ) -SELECT "Transaction".id, - "Transaction".payload, - "Transaction".hash, - "Transaction".tx_index, - "Transaction".is_valid, - "Block".hash AS block_hash, - "Block".epoch, - "Block".slot, - "Block".era, - "Block".height, - "TransactionMetadata".payload AS metadata, - json_agg(DISTINCT "Address".PAYLOAD) input_addresses -FROM "Transaction" -INNER JOIN "Block" ON "Transaction".block_id = "Block".id -INNER JOIN "TransactionInput" ON "TransactionInput".tx_id = "Transaction".id -INNER JOIN "Address" ON "Address".id = "TransactionInput".address_id -LEFT JOIN "TransactionMetadata" ON "Transaction".id = "TransactionMetadata".tx_id -WHERE "Transaction".id IN (SELECT * FROM inputs UNION ALL SELECT * from ref_inputs UNION ALL SELECT * from outputs) -GROUP BY "Transaction".id, "Block".id, "TransactionMetadata".id -ORDER BY "Transaction".id ASC -LIMIT (:limit); \ No newline at end of file +SELECT * FROM base_query WHERE NOT :with_input_context! +UNION ALL +(SELECT * from query_with_inputs_and_metadata WHERE :with_input_context!); \ No newline at end of file diff --git a/webserver/server/app/models/transaction/sqlHistoryForCredentials.queries.ts b/webserver/server/app/models/transaction/sqlHistoryForCredentials.queries.ts index 50c042d1..e02270de 100644 --- a/webserver/server/app/models/transaction/sqlHistoryForCredentials.queries.ts +++ b/webserver/server/app/models/transaction/sqlHistoryForCredentials.queries.ts @@ -14,6 +14,7 @@ export interface ISqlHistoryForCredentialsParams { limit?: NumberOrString | null | void; relation?: number | null | void; until_tx_id?: NumberOrString | null | void; + with_input_context: boolean; } /** 'SqlHistoryForCredentials' return type */ @@ -23,10 +24,10 @@ export interface ISqlHistoryForCredentialsResult { era: number; hash: Buffer; height: number; - id: string; + id: string | null; input_addresses: Json | null; is_valid: boolean; - metadata: Buffer; + metadata: Buffer | null; payload: Buffer; slot: number; tx_index: number; @@ -38,7 +39,7 @@ export interface ISqlHistoryForCredentialsQuery { result: ISqlHistoryForCredentialsResult; } -const sqlHistoryForCredentialsIR: any = {"usedParamSet":{"credentials":true,"relation":true,"until_tx_id":true,"after_tx_id":true,"limit":true},"params":[{"name":"credentials","required":false,"transform":{"type":"scalar"},"locs":[{"a":288,"b":299}]},{"name":"relation","required":false,"transform":{"type":"scalar"},"locs":[{"a":354,"b":362}]},{"name":"until_tx_id","required":false,"transform":{"type":"scalar"},"locs":[{"a":464,"b":475}]},{"name":"after_tx_id","required":false,"transform":{"type":"scalar"},"locs":[{"a":527,"b":538}]},{"name":"limit","required":false,"transform":{"type":"scalar"},"locs":[{"a":598,"b":603}]}],"statement":"WITH\n tx_relations AS (\n SELECT DISTINCT ON (\"TxCredentialRelation\".tx_id) \"TxCredentialRelation\".tx_id\n FROM \"StakeCredential\"\n INNER JOIN \"TxCredentialRelation\" ON \"TxCredentialRelation\".credential_id = \"StakeCredential\".id\n WHERE\n \"StakeCredential\".credential = ANY (:credentials)\n AND\n (\"TxCredentialRelation\".relation & (:relation)) > 0\n AND\n \n \"TxCredentialRelation\".tx_id <= (:until_tx_id)\n AND \n \"TxCredentialRelation\".tx_id > (:after_tx_id)\n ORDER BY \"TxCredentialRelation\".tx_id ASC\n LIMIT (:limit)\n )\nSELECT \"Transaction\".id,\n \"Transaction\".payload,\n \"Transaction\".hash,\n \"Transaction\".tx_index,\n \"Transaction\".is_valid,\n \"Block\".hash AS block_hash,\n \"Block\".epoch,\n \"Block\".slot,\n \"Block\".era,\n \"Block\".height,\n \"TransactionMetadata\".payload AS metadata,\n json_agg(DISTINCT \"Address\".PAYLOAD) input_addresses\nFROM tx_relations\nINNER JOIN \"Transaction\" ON tx_relations.tx_id = \"Transaction\".id\nINNER JOIN \"TransactionInput\" ON \"TransactionInput\".tx_id = \"Transaction\".id\nINNER JOIN \"Address\" ON \"Address\".id = \"TransactionInput\".address_id\nLEFT JOIN \"TransactionMetadata\" ON \"Transaction\".id = \"TransactionMetadata\".tx_id\nINNER JOIN \"Block\" ON \"Transaction\".block_id = \"Block\".id\nGROUP BY \"Transaction\".id, \"Block\".id, \"TransactionMetadata\".id"}; +const sqlHistoryForCredentialsIR: any = {"usedParamSet":{"credentials":true,"relation":true,"until_tx_id":true,"after_tx_id":true,"limit":true,"with_input_context":true},"params":[{"name":"credentials","required":false,"transform":{"type":"scalar"},"locs":[{"a":288,"b":299}]},{"name":"relation","required":false,"transform":{"type":"scalar"},"locs":[{"a":354,"b":362}]},{"name":"until_tx_id","required":false,"transform":{"type":"scalar"},"locs":[{"a":464,"b":475}]},{"name":"after_tx_id","required":false,"transform":{"type":"scalar"},"locs":[{"a":527,"b":538}]},{"name":"limit","required":false,"transform":{"type":"scalar"},"locs":[{"a":598,"b":603}]},{"name":"with_input_context","required":true,"transform":{"type":"scalar"},"locs":[{"a":2446,"b":2465},{"a":2529,"b":2548}]}],"statement":"WITH\n tx_relations AS (\n SELECT DISTINCT ON (\"TxCredentialRelation\".tx_id) \"TxCredentialRelation\".tx_id\n FROM \"StakeCredential\"\n INNER JOIN \"TxCredentialRelation\" ON \"TxCredentialRelation\".credential_id = \"StakeCredential\".id\n WHERE\n \"StakeCredential\".credential = ANY (:credentials)\n AND\n (\"TxCredentialRelation\".relation & (:relation)) > 0\n AND\n \n \"TxCredentialRelation\".tx_id <= (:until_tx_id)\n AND \n \"TxCredentialRelation\".tx_id > (:after_tx_id)\n ORDER BY \"TxCredentialRelation\".tx_id ASC\n LIMIT (:limit)\n ),\n base_query AS (\n SELECT \"Transaction\".id,\n \"Transaction\".payload as \"payload!\",\n \"Transaction\".hash as \"hash!\",\n \"Transaction\".tx_index as \"tx_index!\",\n \"Transaction\".is_valid as \"is_valid!\",\n \"Block\".hash AS \"block_hash!\",\n \"Block\".epoch as \"epoch!\",\n \"Block\".slot as \"slot!\",\n \"Block\".era as \"era!\",\n \"Block\".height as \"height!\",\n NULL :: bytea as metadata,\n NULL :: json as input_addresses\n FROM tx_relations\n INNER JOIN \"Transaction\" ON tx_relations.tx_id = \"Transaction\".id\n INNER JOIN \"Block\" ON \"Transaction\".block_id = \"Block\".id\n ),\n query_with_inputs_and_metadata AS (\n SELECT \"Transaction\".id,\n \"Transaction\".payload as \"payload!\",\n \"Transaction\".hash as \"hash!\",\n \"Transaction\".tx_index as \"tx_index!\",\n \"Transaction\".is_valid as \"is_valid!\",\n \"Block\".hash as \"block_hash!\",\n \"Block\".epoch as \"epoch!\",\n \"Block\".slot as \"slot!\",\n \"Block\".era as \"era!\",\n \"Block\".height as \"height!\",\n \"TransactionMetadata\".payload AS metadata,\n json_agg(DISTINCT \"Address\".PAYLOAD) input_addresses\n FROM tx_relations\n INNER JOIN \"Transaction\" ON tx_relations.tx_id = \"Transaction\".id\n INNER JOIN \"TransactionInput\" ON \"TransactionInput\".tx_id = \"Transaction\".id\n INNER JOIN \"Address\" ON \"Address\".id = \"TransactionInput\".address_id\n LEFT JOIN \"TransactionMetadata\" ON \"Transaction\".id = \"TransactionMetadata\".tx_id\n INNER JOIN \"Block\" ON \"Transaction\".block_id = \"Block\".id\n GROUP BY \"Transaction\".id, \"Block\".id, \"TransactionMetadata\".id\n )\nSELECT * FROM base_query WHERE NOT :with_input_context!\nUNION ALL (SELECT * FROM query_with_inputs_and_metadata WHERE :with_input_context!)"}; /** * Query generated from SQL: @@ -59,26 +60,47 @@ const sqlHistoryForCredentialsIR: any = {"usedParamSet":{"credentials":true,"rel * "TxCredentialRelation".tx_id > (:after_tx_id) * ORDER BY "TxCredentialRelation".tx_id ASC * LIMIT (:limit) + * ), + * base_query AS ( + * SELECT "Transaction".id, + * "Transaction".payload as "payload!", + * "Transaction".hash as "hash!", + * "Transaction".tx_index as "tx_index!", + * "Transaction".is_valid as "is_valid!", + * "Block".hash AS "block_hash!", + * "Block".epoch as "epoch!", + * "Block".slot as "slot!", + * "Block".era as "era!", + * "Block".height as "height!", + * NULL :: bytea as metadata, + * NULL :: json as input_addresses + * FROM tx_relations + * INNER JOIN "Transaction" ON tx_relations.tx_id = "Transaction".id + * INNER JOIN "Block" ON "Transaction".block_id = "Block".id + * ), + * query_with_inputs_and_metadata AS ( + * SELECT "Transaction".id, + * "Transaction".payload as "payload!", + * "Transaction".hash as "hash!", + * "Transaction".tx_index as "tx_index!", + * "Transaction".is_valid as "is_valid!", + * "Block".hash as "block_hash!", + * "Block".epoch as "epoch!", + * "Block".slot as "slot!", + * "Block".era as "era!", + * "Block".height as "height!", + * "TransactionMetadata".payload AS metadata, + * json_agg(DISTINCT "Address".PAYLOAD) input_addresses + * FROM tx_relations + * INNER JOIN "Transaction" ON tx_relations.tx_id = "Transaction".id + * INNER JOIN "TransactionInput" ON "TransactionInput".tx_id = "Transaction".id + * INNER JOIN "Address" ON "Address".id = "TransactionInput".address_id + * LEFT JOIN "TransactionMetadata" ON "Transaction".id = "TransactionMetadata".tx_id + * INNER JOIN "Block" ON "Transaction".block_id = "Block".id + * GROUP BY "Transaction".id, "Block".id, "TransactionMetadata".id * ) - * SELECT "Transaction".id, - * "Transaction".payload, - * "Transaction".hash, - * "Transaction".tx_index, - * "Transaction".is_valid, - * "Block".hash AS block_hash, - * "Block".epoch, - * "Block".slot, - * "Block".era, - * "Block".height, - * "TransactionMetadata".payload AS metadata, - * json_agg(DISTINCT "Address".PAYLOAD) input_addresses - * FROM tx_relations - * INNER JOIN "Transaction" ON tx_relations.tx_id = "Transaction".id - * INNER JOIN "TransactionInput" ON "TransactionInput".tx_id = "Transaction".id - * INNER JOIN "Address" ON "Address".id = "TransactionInput".address_id - * LEFT JOIN "TransactionMetadata" ON "Transaction".id = "TransactionMetadata".tx_id - * INNER JOIN "Block" ON "Transaction".block_id = "Block".id - * GROUP BY "Transaction".id, "Block".id, "TransactionMetadata".id + * SELECT * FROM base_query WHERE NOT :with_input_context! + * UNION ALL (SELECT * FROM query_with_inputs_and_metadata WHERE :with_input_context!) * ``` */ export const sqlHistoryForCredentials = new PreparedQuery(sqlHistoryForCredentialsIR); diff --git a/webserver/server/app/models/transaction/sqlHistoryForCredentials.sql b/webserver/server/app/models/transaction/sqlHistoryForCredentials.sql index 78558ada..fc2b1c3f 100644 --- a/webserver/server/app/models/transaction/sqlHistoryForCredentials.sql +++ b/webserver/server/app/models/transaction/sqlHistoryForCredentials.sql @@ -15,24 +15,45 @@ WITH "TxCredentialRelation".tx_id > (:after_tx_id) ORDER BY "TxCredentialRelation".tx_id ASC LIMIT (:limit) + ), + base_query AS ( + SELECT "Transaction".id, + "Transaction".payload as "payload!", + "Transaction".hash as "hash!", + "Transaction".tx_index as "tx_index!", + "Transaction".is_valid as "is_valid!", + "Block".hash AS "block_hash!", + "Block".epoch as "epoch!", + "Block".slot as "slot!", + "Block".era as "era!", + "Block".height as "height!", + NULL :: bytea as metadata, + NULL :: json as input_addresses + FROM tx_relations + INNER JOIN "Transaction" ON tx_relations.tx_id = "Transaction".id + INNER JOIN "Block" ON "Transaction".block_id = "Block".id + ), + query_with_inputs_and_metadata AS ( + SELECT "Transaction".id, + "Transaction".payload as "payload!", + "Transaction".hash as "hash!", + "Transaction".tx_index as "tx_index!", + "Transaction".is_valid as "is_valid!", + "Block".hash as "block_hash!", + "Block".epoch as "epoch!", + "Block".slot as "slot!", + "Block".era as "era!", + "Block".height as "height!", + "TransactionMetadata".payload AS metadata, + json_agg(DISTINCT "Address".PAYLOAD) input_addresses + FROM tx_relations + INNER JOIN "Transaction" ON tx_relations.tx_id = "Transaction".id + INNER JOIN "TransactionInput" ON "TransactionInput".tx_id = "Transaction".id + INNER JOIN "Address" ON "Address".id = "TransactionInput".address_id + LEFT JOIN "TransactionMetadata" ON "Transaction".id = "TransactionMetadata".tx_id + INNER JOIN "Block" ON "Transaction".block_id = "Block".id + GROUP BY "Transaction".id, "Block".id, "TransactionMetadata".id ) -SELECT "Transaction".id, - "Transaction".payload, - "Transaction".hash, - "Transaction".tx_index, - "Transaction".is_valid, - "Block".hash AS block_hash, - "Block".epoch, - "Block".slot, - "Block".era, - "Block".height, - "TransactionMetadata".payload AS metadata, - json_agg(DISTINCT "Address".PAYLOAD) input_addresses -FROM tx_relations -INNER JOIN "Transaction" ON tx_relations.tx_id = "Transaction".id -INNER JOIN "TransactionInput" ON "TransactionInput".tx_id = "Transaction".id -INNER JOIN "Address" ON "Address".id = "TransactionInput".address_id -LEFT JOIN "TransactionMetadata" ON "Transaction".id = "TransactionMetadata".tx_id -INNER JOIN "Block" ON "Transaction".block_id = "Block".id -GROUP BY "Transaction".id, "Block".id, "TransactionMetadata".id; +SELECT * FROM base_query WHERE NOT :with_input_context! +UNION ALL (SELECT * FROM query_with_inputs_and_metadata WHERE :with_input_context!); diff --git a/webserver/server/app/services/TransactionHistoryService.ts b/webserver/server/app/services/TransactionHistoryService.ts index eef02778..417bf6b1 100644 --- a/webserver/server/app/services/TransactionHistoryService.ts +++ b/webserver/server/app/services/TransactionHistoryService.ts @@ -12,6 +12,7 @@ export async function historyForCredentials( stakeCredentials: Buffer[]; relationFilter: RelationFilter; limit: number; + withInputContext?: boolean; } ): Promise { if (request.stakeCredentials.length === 0) return { transactions: [] }; @@ -22,6 +23,7 @@ export async function historyForCredentials( limit: request.limit.toString(), until_tx_id: request.until.tx_id.toString(), relation: request.relationFilter, + with_input_context: !!request.withInputContext, }, request.dbTx ); @@ -38,12 +40,16 @@ export async function historyForCredentials( }, transaction: { - hash: entry.hash.toString('hex'), - payload: entry.payload.toString('hex'), - metadata: entry.metadata && entry.metadata.toString('hex'), - inputCredentials: entry.input_addresses - ? (entry.input_addresses as string[]).map(getPaymentCred) - : [], + ...{ + hash: entry.hash.toString('hex'), + payload: entry.payload.toString('hex'), + }, + ...(request.withInputContext && { + metadata: entry.metadata && entry.metadata.toString('hex'), + inputCredentials: entry.input_addresses + ? (entry.input_addresses as string[]).map(getPaymentCred) + : [], + }), }, })), }; @@ -54,6 +60,7 @@ export async function historyForAddresses( addresses: Buffer[]; dbTx: PoolClient; limit: number; + withInputContext?: boolean; } ): Promise { if (request.addresses?.length === 0) return { transactions: [] }; @@ -63,6 +70,7 @@ export async function historyForAddresses( after_tx_id: (request.after?.tx_id ?? -1)?.toString(), limit: request.limit.toString(), until_tx_id: request.until.tx_id.toString(), + with_input_context: !!request.withInputContext, }, request.dbTx ); @@ -79,12 +87,16 @@ export async function historyForAddresses( }, transaction: { - hash: entry.hash.toString('hex'), - payload: entry.payload.toString('hex'), - metadata: entry.metadata && entry.metadata.toString('hex'), - inputCredentials: entry.input_addresses - ? (entry.input_addresses as string[]).map(getPaymentCred) - : [], + ...{ + hash: entry.hash.toString('hex'), + payload: entry.payload.toString('hex'), + }, + ...(request.withInputContext && { + metadata: entry.metadata && entry.metadata.toString('hex'), + inputCredentials: entry.input_addresses + ? (entry.input_addresses as string[]).map(getPaymentCred) + : [], + }), }, })), }; diff --git a/webserver/shared/models/TransactionHistory.ts b/webserver/shared/models/TransactionHistory.ts index e5b1de3d..39a3e863 100644 --- a/webserver/shared/models/TransactionHistory.ts +++ b/webserver/shared/models/TransactionHistory.ts @@ -11,6 +11,7 @@ export type TransactionHistoryRequest = { limit?: number; slotLimits?: SlotLimits; + withInputContext?: boolean; } & Pagination; export type BlockInfo = BlockSubset & { @@ -37,9 +38,9 @@ export type TransactionInfo = { */ payload: string; - metadata: string | null; + metadata?: string | null; - inputCredentials: string[]; + inputCredentials?: string[]; }; export type TxAndBlockInfo = { From 1e83422fe28652c0e7ab82ab3bbc02c0b70f588d Mon Sep 17 00:00:00 2001 From: Enzo Cioppettini <48031343+ecioppettini@users.noreply.github.com> Date: Tue, 30 Jan 2024 11:20:26 -0300 Subject: [PATCH 05/14] reformat slotBoundsPagination.sql Co-authored-by: Sebastien Guillemot --- .../pagination/slotBoundsPagination.sql | 55 ++++++++++--------- 1 file changed, 30 insertions(+), 25 deletions(-) diff --git a/webserver/server/app/models/pagination/slotBoundsPagination.sql b/webserver/server/app/models/pagination/slotBoundsPagination.sql index c0b19c05..e4a795bb 100644 --- a/webserver/server/app/models/pagination/slotBoundsPagination.sql +++ b/webserver/server/app/models/pagination/slotBoundsPagination.sql @@ -1,25 +1,30 @@ -/* @name slotBoundsPagination */ -WITH MIN_HASH AS - (SELECT COALESCE("Transaction".ID, - - -1) AS MIN_TX_ID, - SLOT AS MIN_SLOT - FROM "Transaction" - JOIN "Block" ON "Block".ID = "Transaction".BLOCK_ID - WHERE SLOT <= :low! - ORDER BY "Block".ID DESC, "Transaction".ID DESC - LIMIT 1), - MAX_HASH AS - (SELECT SLOT AS MAX_SLOT, - COALESCE(MAX("Transaction".ID), - - -2) AS MAX_TX_ID - FROM "Transaction" - JOIN "Block" ON "Transaction".BLOCK_ID = "Block".ID - WHERE SLOT <= :high! - GROUP BY "Block".ID - ORDER BY "Block".ID DESC - LIMIT 1) -SELECT * -FROM MIN_HASH -LEFT JOIN MAX_HASH ON 1 = 1; \ No newline at end of file + /* @name slotBoundsPagination */ +WITH +min_hash AS +( + SELECT COALESCE("Transaction".id, -1) AS min_tx_id, + slot AS min_slot + FROM "Transaction" + JOIN "Block" + ON "Block".id = "Transaction".block_id + WHERE slot <= :low! + ORDER BY "Block".id DESC, + "Transaction".id DESC + LIMIT 1 +), +max_hash AS +( + SELECT slot AS max_slot, + COALESCE(Max("Transaction".id), -2) AS max_tx_id + FROM "Transaction" + JOIN "Block" + ON "Transaction".block_id = "Block".id + WHERE slot <= :high! + GROUP BY "Block".id + ORDER BY "Block".id DESC + LIMIT 1 +) +SELECT * +FROM min_hash +LEFT JOIN max_hash +ON 1 = 1; \ No newline at end of file From 1eb8a0c4daf93e679482c3612fbd9e7c9a9a669e Mon Sep 17 00:00:00 2001 From: Enzo Cioppettini Date: Tue, 30 Jan 2024 12:05:40 -0300 Subject: [PATCH 06/14] add more comments in the code --- .../controllers/TransactionHistoryController.ts | 16 +++++++++------- webserver/shared/models/TransactionHistory.ts | 7 ++++++- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/webserver/server/app/controllers/TransactionHistoryController.ts b/webserver/server/app/controllers/TransactionHistoryController.ts index 9c9a1c0a..33f4297e 100644 --- a/webserver/server/app/controllers/TransactionHistoryController.ts +++ b/webserver/server/app/controllers/TransactionHistoryController.ts @@ -98,6 +98,8 @@ export class TransactionHistoryController extends Controller { let pageStartWithSlot = pageStart; + // if the slotLimits field is set, this shrinks the tx id range + // accordingly if necessary. if (requestBody.slotLimits) { const bounds = slotBounds ? slotBounds[0] : { min_tx_id: -1, max_tx_id: -2 }; @@ -105,19 +107,19 @@ export class TransactionHistoryController extends Controller { if (!pageStartWithSlot) { pageStartWithSlot = { + // block_id is not really used by this query. block_id: -1, + // if no *after* argument is provided, this starts the pagination + // from the corresponding slot. This allows skipping slots you are + // not interested in. If there is also no slotLimits specified this + // starts from the first tx because of the default of -1. tx_id: minTxId, }; } else { - if (minTxId > pageStartWithSlot.tx_id) { - pageStartWithSlot.tx_id = minTxId; - } + pageStartWithSlot.tx_id = Math.max(Number(bounds.min_tx_id), pageStartWithSlot.tx_id); } - const maxTxId = Number(bounds.max_tx_id); - if (maxTxId < until.tx_id) { - until.tx_id = maxTxId; - } + until.tx_id = Math.min(until.tx_id, Number(bounds.max_tx_id)); } const commonRequest = { diff --git a/webserver/shared/models/TransactionHistory.ts b/webserver/shared/models/TransactionHistory.ts index 39a3e863..bab38426 100644 --- a/webserver/shared/models/TransactionHistory.ts +++ b/webserver/shared/models/TransactionHistory.ts @@ -1,6 +1,5 @@ import type { Address } from "./Address"; import type { BlockSubset } from "./BlockLatest"; -import { AssetName, PolicyId } from "./PolicyIdAssetMap"; import type { Pagination, RelationFilter } from "./common"; export type TransactionHistoryRequest = { @@ -10,7 +9,11 @@ export type TransactionHistoryRequest = { /** Defaults to `ADDRESS_LIMIT.RESPONSE` */ limit?: number; + /** This limits the transactions in the result to this range of slots. + * Everything else is filtered out */ slotLimits?: SlotLimits; + /** If this is set to true, the result includes the input addresses (which are + * not part of the tx), and the metadata (if any) */ withInputContext?: boolean; } & Pagination; @@ -52,6 +55,8 @@ export type TransactionHistoryResponse = { }; export type SlotLimits = { + // this is exclusive from: number; + // this is inclusive to: number; }; From d6834468523c7c181e7f5b5e474bf5771ed7ca02 Mon Sep 17 00:00:00 2001 From: Enzo Cioppettini Date: Tue, 30 Jan 2024 12:20:48 -0300 Subject: [PATCH 07/14] update slotBOundsPagination.queries.ts after reformatting --- .../slotBoundsPagination.queries.ts | 55 ++++++++++--------- 1 file changed, 30 insertions(+), 25 deletions(-) diff --git a/webserver/server/app/models/pagination/slotBoundsPagination.queries.ts b/webserver/server/app/models/pagination/slotBoundsPagination.queries.ts index ba3c7103..eade5b78 100644 --- a/webserver/server/app/models/pagination/slotBoundsPagination.queries.ts +++ b/webserver/server/app/models/pagination/slotBoundsPagination.queries.ts @@ -21,35 +21,40 @@ export interface ISlotBoundsPaginationQuery { result: ISlotBoundsPaginationResult; } -const slotBoundsPaginationIR: any = {"usedParamSet":{"low":true,"high":true},"params":[{"name":"low","required":true,"transform":{"type":"scalar"},"locs":[{"a":193,"b":197}]},{"name":"high","required":true,"transform":{"type":"scalar"},"locs":[{"a":449,"b":454}]}],"statement":"WITH MIN_HASH AS\n\t(SELECT COALESCE(\"Transaction\".ID,\n\n\t\t\t\t\t\t\t\t\t\t-1) AS MIN_TX_ID,\n\t\t\tSLOT AS MIN_SLOT\n\t\tFROM \"Transaction\"\n\t\tJOIN \"Block\" ON \"Block\".ID = \"Transaction\".BLOCK_ID\n\t\tWHERE SLOT <= :low!\n\t\tORDER BY \"Block\".ID DESC, \"Transaction\".ID DESC\n\t\tLIMIT 1),\n\tMAX_HASH AS\n\t(SELECT SLOT AS MAX_SLOT,\n\t\t\tCOALESCE(MAX(\"Transaction\".ID),\n\n\t\t\t\t-2) AS MAX_TX_ID\n\t\tFROM \"Transaction\"\n\t\tJOIN \"Block\" ON \"Transaction\".BLOCK_ID = \"Block\".ID\n\t\tWHERE SLOT <= :high!\n\t\tGROUP BY \"Block\".ID\n\t\tORDER BY \"Block\".ID DESC\n\t\tLIMIT 1)\nSELECT *\nFROM MIN_HASH\nLEFT JOIN MAX_HASH ON 1 = 1"}; +const slotBoundsPaginationIR: any = {"usedParamSet":{"low":true,"high":true},"params":[{"name":"low","required":true,"transform":{"type":"scalar"},"locs":[{"a":281,"b":285}]},{"name":"high","required":true,"transform":{"type":"scalar"},"locs":[{"a":668,"b":673}]}],"statement":"WITH\nmin_hash AS\n(\n SELECT COALESCE(\"Transaction\".id, -1) AS min_tx_id,\n slot AS min_slot\n FROM \"Transaction\"\n JOIN \"Block\"\n ON \"Block\".id = \"Transaction\".block_id\n WHERE slot <= :low!\n ORDER BY \"Block\".id DESC,\n \"Transaction\".id DESC\n LIMIT 1\n),\nmax_hash AS\n(\n SELECT slot AS max_slot,\n COALESCE(Max(\"Transaction\".id), -2) AS max_tx_id\n FROM \"Transaction\"\n JOIN \"Block\"\n ON \"Transaction\".block_id = \"Block\".id\n WHERE slot <= :high!\n GROUP BY \"Block\".id\n ORDER BY \"Block\".id DESC\n LIMIT 1\n)\nSELECT *\nFROM min_hash\nLEFT JOIN max_hash\nON 1 = 1"}; /** * Query generated from SQL: * ``` - * WITH MIN_HASH AS - * (SELECT COALESCE("Transaction".ID, - * - * -1) AS MIN_TX_ID, - * SLOT AS MIN_SLOT - * FROM "Transaction" - * JOIN "Block" ON "Block".ID = "Transaction".BLOCK_ID - * WHERE SLOT <= :low! - * ORDER BY "Block".ID DESC, "Transaction".ID DESC - * LIMIT 1), - * MAX_HASH AS - * (SELECT SLOT AS MAX_SLOT, - * COALESCE(MAX("Transaction".ID), - * - * -2) AS MAX_TX_ID - * FROM "Transaction" - * JOIN "Block" ON "Transaction".BLOCK_ID = "Block".ID - * WHERE SLOT <= :high! - * GROUP BY "Block".ID - * ORDER BY "Block".ID DESC - * LIMIT 1) - * SELECT * - * FROM MIN_HASH - * LEFT JOIN MAX_HASH ON 1 = 1 + * WITH + * min_hash AS + * ( + * SELECT COALESCE("Transaction".id, -1) AS min_tx_id, + * slot AS min_slot + * FROM "Transaction" + * JOIN "Block" + * ON "Block".id = "Transaction".block_id + * WHERE slot <= :low! + * ORDER BY "Block".id DESC, + * "Transaction".id DESC + * LIMIT 1 + * ), + * max_hash AS + * ( + * SELECT slot AS max_slot, + * COALESCE(Max("Transaction".id), -2) AS max_tx_id + * FROM "Transaction" + * JOIN "Block" + * ON "Transaction".block_id = "Block".id + * WHERE slot <= :high! + * GROUP BY "Block".id + * ORDER BY "Block".id DESC + * LIMIT 1 + * ) + * SELECT * + * FROM min_hash + * LEFT JOIN max_hash + * ON 1 = 1 * ``` */ export const slotBoundsPagination = new PreparedQuery(slotBoundsPaginationIR); From 28538c0a441364b187741cc05b459c986f6f86b4 Mon Sep 17 00:00:00 2001 From: Enzo Cioppettini Date: Tue, 30 Jan 2024 13:46:44 -0300 Subject: [PATCH 08/14] update .nvmrc to lts/hydrogen --- webserver/server/.nvmrc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webserver/server/.nvmrc b/webserver/server/.nvmrc index f274881e..a77793ec 100644 --- a/webserver/server/.nvmrc +++ b/webserver/server/.nvmrc @@ -1 +1 @@ -v16.16.0 +lts/hydrogen From b0702f899e743fced0450f8b64500d6c9fb422ad Mon Sep 17 00:00:00 2001 From: Enzo Cioppettini Date: Thu, 29 Feb 2024 02:02:31 -0300 Subject: [PATCH 09/14] optimize slot filter tx filter query --- indexer/entity/src/block.rs | 1 + indexer/migration/src/lib.rs | 2 + ...240229_000019_add_block_tx_count_column.rs | 36 ++++++++ indexer/tasks/src/multiera/multiera_block.rs | 1 + .../models/block/sqlBlockLatest.queries.ts | 1 + .../slotBoundsPagination.queries.ts | 82 ++++++++++++------- .../pagination/slotBoundsPagination.sql | 82 ++++++++++++------- 7 files changed, 147 insertions(+), 58 deletions(-) create mode 100644 indexer/migration/src/m20240229_000019_add_block_tx_count_column.rs diff --git a/indexer/entity/src/block.rs b/indexer/entity/src/block.rs index 000219a8..3516c0c8 100644 --- a/indexer/entity/src/block.rs +++ b/indexer/entity/src/block.rs @@ -12,6 +12,7 @@ pub struct Model { pub epoch: i32, pub slot: i32, pub payload: Option>, + pub tx_count: i32, } #[derive(Copy, Clone, Debug, DeriveRelation, EnumIter)] diff --git a/indexer/migration/src/lib.rs b/indexer/migration/src/lib.rs index 11f27ea8..fa57a67c 100644 --- a/indexer/migration/src/lib.rs +++ b/indexer/migration/src/lib.rs @@ -20,6 +20,7 @@ mod m20230223_000015_modify_block_table; mod m20230927_000016_create_stake_delegation_table; mod m20231025_000017_projected_nft; mod m20231220_000018_asset_utxo_table; +mod m20240229_000019_add_block_tx_count_column; pub struct Migrator; @@ -47,6 +48,7 @@ impl MigratorTrait for Migrator { Box::new(m20230927_000016_create_stake_delegation_table::Migration), Box::new(m20231025_000017_projected_nft::Migration), Box::new(m20231220_000018_asset_utxo_table::Migration), + Box::new(m20240229_000019_add_block_tx_count_column::Migration), ] } } diff --git a/indexer/migration/src/m20240229_000019_add_block_tx_count_column.rs b/indexer/migration/src/m20240229_000019_add_block_tx_count_column.rs new file mode 100644 index 00000000..eeae529f --- /dev/null +++ b/indexer/migration/src/m20240229_000019_add_block_tx_count_column.rs @@ -0,0 +1,36 @@ +use sea_schema::migration::prelude::*; + +use entity::block::*; + +pub struct Migration; + +impl MigrationName for Migration { + fn name(&self) -> &str { + "m20240229_000019_add_block_tx_count_column" + } +} + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .alter_table( + Table::alter() + .table(Entity) + .add_column(ColumnDef::new(Column::TxCount).integer()) + .to_owned(), + ) + .await + } + + async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .alter_table( + Table::alter() + .table(Entity) + .drop_column(Column::TxCount) + .to_owned(), + ) + .await + } +} diff --git a/indexer/tasks/src/multiera/multiera_block.rs b/indexer/tasks/src/multiera/multiera_block.rs index 5f74d611..b5c390ff 100644 --- a/indexer/tasks/src/multiera/multiera_block.rs +++ b/indexer/tasks/src/multiera/multiera_block.rs @@ -49,6 +49,7 @@ async fn handle_block( epoch: Set(block.2.epoch.unwrap() as i32), slot: Set(block.1.header().slot() as i32), payload: Set(Some(block_payload)), + tx_count: Set(block.1.txs().len() as i32), ..Default::default() }; block.insert(db_tx).await diff --git a/webserver/server/app/models/block/sqlBlockLatest.queries.ts b/webserver/server/app/models/block/sqlBlockLatest.queries.ts index b072f9b9..163271c6 100644 --- a/webserver/server/app/models/block/sqlBlockLatest.queries.ts +++ b/webserver/server/app/models/block/sqlBlockLatest.queries.ts @@ -17,6 +17,7 @@ export interface ISqlBlockLatestResult { id: number; payload: Buffer | null; slot: number; + tx_count: number | null; } /** 'SqlBlockLatest' query type */ diff --git a/webserver/server/app/models/pagination/slotBoundsPagination.queries.ts b/webserver/server/app/models/pagination/slotBoundsPagination.queries.ts index eade5b78..0d903eaa 100644 --- a/webserver/server/app/models/pagination/slotBoundsPagination.queries.ts +++ b/webserver/server/app/models/pagination/slotBoundsPagination.queries.ts @@ -21,40 +21,64 @@ export interface ISlotBoundsPaginationQuery { result: ISlotBoundsPaginationResult; } -const slotBoundsPaginationIR: any = {"usedParamSet":{"low":true,"high":true},"params":[{"name":"low","required":true,"transform":{"type":"scalar"},"locs":[{"a":281,"b":285}]},{"name":"high","required":true,"transform":{"type":"scalar"},"locs":[{"a":668,"b":673}]}],"statement":"WITH\nmin_hash AS\n(\n SELECT COALESCE(\"Transaction\".id, -1) AS min_tx_id,\n slot AS min_slot\n FROM \"Transaction\"\n JOIN \"Block\"\n ON \"Block\".id = \"Transaction\".block_id\n WHERE slot <= :low!\n ORDER BY \"Block\".id DESC,\n \"Transaction\".id DESC\n LIMIT 1\n),\nmax_hash AS\n(\n SELECT slot AS max_slot,\n COALESCE(Max(\"Transaction\".id), -2) AS max_tx_id\n FROM \"Transaction\"\n JOIN \"Block\"\n ON \"Transaction\".block_id = \"Block\".id\n WHERE slot <= :high!\n GROUP BY \"Block\".id\n ORDER BY \"Block\".id DESC\n LIMIT 1\n)\nSELECT *\nFROM min_hash\nLEFT JOIN max_hash\nON 1 = 1"}; +const slotBoundsPaginationIR: any = {"usedParamSet":{"low":true,"high":true},"params":[{"name":"low","required":true,"transform":{"type":"scalar"},"locs":[{"a":155,"b":159}]},{"name":"high","required":true,"transform":{"type":"scalar"},"locs":[{"a":409,"b":414}]}],"statement":"WITH\n low_block AS (\n SELECT\n \"Block\".id,\n \"Block\".slot\n FROM\n \"Block\"\n WHERE\n slot <= :low! AND tx_count > 0\n ORDER BY\n \"Block\".id DESC\n LIMIT\n 1\n ),\n high_block AS (\n SELECT\n \"Block\".id,\n \"Block\".slot\n FROM\n \"Block\"\n WHERE\n slot <= :high! AND tx_count > 0\n ORDER BY\n \"Block\".id DESC\n LIMIT\n 1\n ),\n min_hash AS (\n SELECT\n COALESCE(MAX(\"Transaction\".id), -1) AS min_tx_id,\n slot AS min_slot\n FROM\n \"Transaction\"\n JOIN low_block ON \"Transaction\".block_id = low_block.id\n GROUP BY\n low_block.slot\n LIMIT\n 1\n ),\n max_hash AS (\n SELECT\n COALESCE(MAX(\"Transaction\".id), -2) AS max_tx_id,\n slot AS max_slot\n FROM\n \"Transaction\"\n JOIN high_block ON \"Transaction\".block_id = high_block.id\n GROUP BY\n high_block.slot\n )\nSELECT\n *\nFROM min_hash\nLEFT JOIN max_hash ON 1 = 1"}; /** * Query generated from SQL: * ``` * WITH - * min_hash AS - * ( - * SELECT COALESCE("Transaction".id, -1) AS min_tx_id, - * slot AS min_slot - * FROM "Transaction" - * JOIN "Block" - * ON "Block".id = "Transaction".block_id - * WHERE slot <= :low! - * ORDER BY "Block".id DESC, - * "Transaction".id DESC - * LIMIT 1 - * ), - * max_hash AS - * ( - * SELECT slot AS max_slot, - * COALESCE(Max("Transaction".id), -2) AS max_tx_id - * FROM "Transaction" - * JOIN "Block" - * ON "Transaction".block_id = "Block".id - * WHERE slot <= :high! - * GROUP BY "Block".id - * ORDER BY "Block".id DESC - * LIMIT 1 - * ) - * SELECT * - * FROM min_hash - * LEFT JOIN max_hash - * ON 1 = 1 + * low_block AS ( + * SELECT + * "Block".id, + * "Block".slot + * FROM + * "Block" + * WHERE + * slot <= :low! AND tx_count > 0 + * ORDER BY + * "Block".id DESC + * LIMIT + * 1 + * ), + * high_block AS ( + * SELECT + * "Block".id, + * "Block".slot + * FROM + * "Block" + * WHERE + * slot <= :high! AND tx_count > 0 + * ORDER BY + * "Block".id DESC + * LIMIT + * 1 + * ), + * min_hash AS ( + * SELECT + * COALESCE(MAX("Transaction".id), -1) AS min_tx_id, + * slot AS min_slot + * FROM + * "Transaction" + * JOIN low_block ON "Transaction".block_id = low_block.id + * GROUP BY + * low_block.slot + * LIMIT + * 1 + * ), + * max_hash AS ( + * SELECT + * COALESCE(MAX("Transaction".id), -2) AS max_tx_id, + * slot AS max_slot + * FROM + * "Transaction" + * JOIN high_block ON "Transaction".block_id = high_block.id + * GROUP BY + * high_block.slot + * ) + * SELECT + * * + * FROM min_hash + * LEFT JOIN max_hash ON 1 = 1 * ``` */ export const slotBoundsPagination = new PreparedQuery(slotBoundsPaginationIR); diff --git a/webserver/server/app/models/pagination/slotBoundsPagination.sql b/webserver/server/app/models/pagination/slotBoundsPagination.sql index e4a795bb..a8c3a69b 100644 --- a/webserver/server/app/models/pagination/slotBoundsPagination.sql +++ b/webserver/server/app/models/pagination/slotBoundsPagination.sql @@ -1,30 +1,54 @@ - /* @name slotBoundsPagination */ +/* @name slotBoundsPagination */ WITH -min_hash AS -( - SELECT COALESCE("Transaction".id, -1) AS min_tx_id, - slot AS min_slot - FROM "Transaction" - JOIN "Block" - ON "Block".id = "Transaction".block_id - WHERE slot <= :low! - ORDER BY "Block".id DESC, - "Transaction".id DESC - LIMIT 1 -), -max_hash AS -( - SELECT slot AS max_slot, - COALESCE(Max("Transaction".id), -2) AS max_tx_id - FROM "Transaction" - JOIN "Block" - ON "Transaction".block_id = "Block".id - WHERE slot <= :high! - GROUP BY "Block".id - ORDER BY "Block".id DESC - LIMIT 1 -) -SELECT * -FROM min_hash -LEFT JOIN max_hash -ON 1 = 1; \ No newline at end of file + low_block AS ( + SELECT + "Block".id, + "Block".slot + FROM + "Block" + WHERE + slot <= :low! AND tx_count > 0 + ORDER BY + "Block".id DESC + LIMIT + 1 + ), + high_block AS ( + SELECT + "Block".id, + "Block".slot + FROM + "Block" + WHERE + slot <= :high! AND tx_count > 0 + ORDER BY + "Block".id DESC + LIMIT + 1 + ), + min_hash AS ( + SELECT + COALESCE(MAX("Transaction".id), -1) AS min_tx_id, + slot AS min_slot + FROM + "Transaction" + JOIN low_block ON "Transaction".block_id = low_block.id + GROUP BY + low_block.slot + LIMIT + 1 + ), + max_hash AS ( + SELECT + COALESCE(MAX("Transaction".id), -2) AS max_tx_id, + slot AS max_slot + FROM + "Transaction" + JOIN high_block ON "Transaction".block_id = high_block.id + GROUP BY + high_block.slot + ) +SELECT + * +FROM min_hash +LEFT JOIN max_hash ON 1 = 1; \ No newline at end of file From e1bca8e74371561510373a8ffc72298625344aba Mon Sep 17 00:00:00 2001 From: Enzo Cioppettini Date: Fri, 1 Mar 2024 22:00:05 -0300 Subject: [PATCH 10/14] add support to lower slot limit as -1 to get the full range --- .../slotBoundsPagination.queries.ts | 20 +++++++++---------- .../pagination/slotBoundsPagination.sql | 16 ++++++++------- 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/webserver/server/app/models/pagination/slotBoundsPagination.queries.ts b/webserver/server/app/models/pagination/slotBoundsPagination.queries.ts index 0d903eaa..c3de9024 100644 --- a/webserver/server/app/models/pagination/slotBoundsPagination.queries.ts +++ b/webserver/server/app/models/pagination/slotBoundsPagination.queries.ts @@ -9,9 +9,7 @@ export interface ISlotBoundsPaginationParams { /** 'SlotBoundsPagination' return type */ export interface ISlotBoundsPaginationResult { - max_slot: number; max_tx_id: string | null; - min_slot: number; min_tx_id: string | null; } @@ -21,7 +19,7 @@ export interface ISlotBoundsPaginationQuery { result: ISlotBoundsPaginationResult; } -const slotBoundsPaginationIR: any = {"usedParamSet":{"low":true,"high":true},"params":[{"name":"low","required":true,"transform":{"type":"scalar"},"locs":[{"a":155,"b":159}]},{"name":"high","required":true,"transform":{"type":"scalar"},"locs":[{"a":409,"b":414}]}],"statement":"WITH\n low_block AS (\n SELECT\n \"Block\".id,\n \"Block\".slot\n FROM\n \"Block\"\n WHERE\n slot <= :low! AND tx_count > 0\n ORDER BY\n \"Block\".id DESC\n LIMIT\n 1\n ),\n high_block AS (\n SELECT\n \"Block\".id,\n \"Block\".slot\n FROM\n \"Block\"\n WHERE\n slot <= :high! AND tx_count > 0\n ORDER BY\n \"Block\".id DESC\n LIMIT\n 1\n ),\n min_hash AS (\n SELECT\n COALESCE(MAX(\"Transaction\".id), -1) AS min_tx_id,\n slot AS min_slot\n FROM\n \"Transaction\"\n JOIN low_block ON \"Transaction\".block_id = low_block.id\n GROUP BY\n low_block.slot\n LIMIT\n 1\n ),\n max_hash AS (\n SELECT\n COALESCE(MAX(\"Transaction\".id), -2) AS max_tx_id,\n slot AS max_slot\n FROM\n \"Transaction\"\n JOIN high_block ON \"Transaction\".block_id = high_block.id\n GROUP BY\n high_block.slot\n )\nSELECT\n *\nFROM min_hash\nLEFT JOIN max_hash ON 1 = 1"}; +const slotBoundsPaginationIR: any = {"usedParamSet":{"low":true,"high":true},"params":[{"name":"low","required":true,"transform":{"type":"scalar"},"locs":[{"a":155,"b":159}]},{"name":"high","required":true,"transform":{"type":"scalar"},"locs":[{"a":409,"b":414}]}],"statement":"WITH\n low_block AS (\n SELECT\n \"Block\".id,\n \"Block\".slot\n FROM\n \"Block\"\n WHERE\n slot <= :low! AND tx_count > 0\n ORDER BY\n \"Block\".id DESC\n LIMIT\n 1\n ),\n high_block AS (\n SELECT\n \"Block\".id,\n \"Block\".slot\n FROM\n \"Block\"\n WHERE\n slot <= :high! AND tx_count > 0\n ORDER BY\n \"Block\".id DESC\n LIMIT 1\n ),\n min_hash AS (\n (SELECT\n COALESCE(MAX(\"Transaction\".id), -1) AS min_tx_id\n FROM\n \"Transaction\"\n JOIN low_block ON \"Transaction\".block_id = low_block.id\n GROUP BY\n low_block.slot\n LIMIT\n 1\n )\n UNION (SELECT min_tx_id FROM (values(-1)) s(min_tx_id))\n ORDER BY min_tx_id DESC\n LIMIT\n 1\n ),\n max_hash AS (\n SELECT\n COALESCE(MAX(\"Transaction\".id), -2) AS max_tx_id\n FROM\n \"Transaction\"\n JOIN high_block ON \"Transaction\".block_id = high_block.id\n GROUP BY\n high_block.slot\n )\nSELECT\n *\nFROM min_hash\nLEFT JOIN max_hash ON 1 = 1"}; /** * Query generated from SQL: @@ -50,13 +48,11 @@ const slotBoundsPaginationIR: any = {"usedParamSet":{"low":true,"high":true},"pa * slot <= :high! AND tx_count > 0 * ORDER BY * "Block".id DESC - * LIMIT - * 1 + * LIMIT 1 * ), * min_hash AS ( - * SELECT - * COALESCE(MAX("Transaction".id), -1) AS min_tx_id, - * slot AS min_slot + * (SELECT + * COALESCE(MAX("Transaction".id), -1) AS min_tx_id * FROM * "Transaction" * JOIN low_block ON "Transaction".block_id = low_block.id @@ -64,11 +60,15 @@ const slotBoundsPaginationIR: any = {"usedParamSet":{"low":true,"high":true},"pa * low_block.slot * LIMIT * 1 + * ) + * UNION (SELECT min_tx_id FROM (values(-1)) s(min_tx_id)) + * ORDER BY min_tx_id DESC + * LIMIT + * 1 * ), * max_hash AS ( * SELECT - * COALESCE(MAX("Transaction".id), -2) AS max_tx_id, - * slot AS max_slot + * COALESCE(MAX("Transaction".id), -2) AS max_tx_id * FROM * "Transaction" * JOIN high_block ON "Transaction".block_id = high_block.id diff --git a/webserver/server/app/models/pagination/slotBoundsPagination.sql b/webserver/server/app/models/pagination/slotBoundsPagination.sql index a8c3a69b..6fbaa75e 100644 --- a/webserver/server/app/models/pagination/slotBoundsPagination.sql +++ b/webserver/server/app/models/pagination/slotBoundsPagination.sql @@ -23,13 +23,11 @@ WITH slot <= :high! AND tx_count > 0 ORDER BY "Block".id DESC - LIMIT - 1 + LIMIT 1 ), min_hash AS ( - SELECT - COALESCE(MAX("Transaction".id), -1) AS min_tx_id, - slot AS min_slot + (SELECT + COALESCE(MAX("Transaction".id), -1) AS min_tx_id FROM "Transaction" JOIN low_block ON "Transaction".block_id = low_block.id @@ -37,11 +35,15 @@ WITH low_block.slot LIMIT 1 + ) + UNION (SELECT min_tx_id FROM (values(-1)) s(min_tx_id)) + ORDER BY min_tx_id DESC + LIMIT + 1 ), max_hash AS ( SELECT - COALESCE(MAX("Transaction".id), -2) AS max_tx_id, - slot AS max_slot + COALESCE(MAX("Transaction".id), -2) AS max_tx_id FROM "Transaction" JOIN high_block ON "Transaction".block_id = high_block.id From cde3ec45d82ce7addecdf935e355901d527f37ed Mon Sep 17 00:00:00 2001 From: Enzo Cioppettini Date: Sat, 2 Mar 2024 13:06:29 -0300 Subject: [PATCH 11/14] update openapi.json --- docs/bin/openapi.json | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/docs/bin/openapi.json b/docs/bin/openapi.json index 42950f27..bbd79a6f 100644 --- a/docs/bin/openapi.json +++ b/docs/bin/openapi.json @@ -1111,6 +1111,16 @@ }, "TransactionInfo": { "properties": { + "inputCredentials": { + "items": { + "type": "string" + }, + "type": "array" + }, + "metadata": { + "type": "string", + "nullable": true + }, "payload": { "type": "string", "description": "cbor-encoded transaction", @@ -1166,10 +1176,35 @@ "description": "Filter which uses of the address are considered relevant for the query.\n\nThis is a bitmask, so you can combine multiple options\nex: `RelationFilterType.Input & RelationFilterType.Output`\n\nNote: relations only apply to credentials and not to full bech32 addresses", "pattern": "([01]?[0-9]?[0-9]|2[0-4][0-9]|25[0-5])" }, + "SlotLimits": { + "properties": { + "to": { + "type": "number", + "format": "double" + }, + "from": { + "type": "number", + "format": "double" + } + }, + "required": [ + "to", + "from" + ], + "type": "object" + }, "TransactionHistoryRequest": { "allOf": [ { "properties": { + "withInputContext": { + "type": "boolean", + "description": "If this is set to true, the result includes the input addresses (which are\nnot part of the tx), and the metadata (if any)" + }, + "slotLimits": { + "$ref": "#/components/schemas/SlotLimits", + "description": "This limits the transactions in the result to this range of slots.\nEverything else is filtered out" + }, "limit": { "type": "number", "format": "double", From 6568a1f3dd40e533725edb271aec96a3d6e4eaea Mon Sep 17 00:00:00 2001 From: Enzo Cioppettini Date: Tue, 5 Mar 2024 00:41:39 -0300 Subject: [PATCH 12/14] update tx_count change to cml multiera after rebase --- indexer/tasks/src/multiera/multiera_block.rs | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/indexer/tasks/src/multiera/multiera_block.rs b/indexer/tasks/src/multiera/multiera_block.rs index b5c390ff..d919f85e 100644 --- a/indexer/tasks/src/multiera/multiera_block.rs +++ b/indexer/tasks/src/multiera/multiera_block.rs @@ -49,8 +49,25 @@ async fn handle_block( epoch: Set(block.2.epoch.unwrap() as i32), slot: Set(block.1.header().slot() as i32), payload: Set(Some(block_payload)), - tx_count: Set(block.1.txs().len() as i32), + tx_count: Set(block_tx_count(block.1) as i32), ..Default::default() }; block.insert(db_tx).await } + +fn block_tx_count(block: &cml_multi_era::MultiEraBlock) -> usize { + match block { + cml_multi_era::MultiEraBlock::Byron( + cml_multi_era::byron::block::ByronBlock::EpochBoundary(_), + ) => 0, + cml_multi_era::MultiEraBlock::Byron(cml_multi_era::byron::block::ByronBlock::Main( + block, + )) => block.body.tx_payload.len(), + cml_multi_era::MultiEraBlock::Shelley(block) => block.transaction_bodies.len(), + cml_multi_era::MultiEraBlock::Allegra(block) => block.transaction_bodies.len(), + cml_multi_era::MultiEraBlock::Mary(block) => block.transaction_bodies.len(), + cml_multi_era::MultiEraBlock::Alonzo(block) => block.transaction_bodies.len(), + cml_multi_era::MultiEraBlock::Babbage(block) => block.transaction_bodies.len(), + cml_multi_era::MultiEraBlock::Conway(block) => block.transaction_bodies.len(), + } +} From 9906695637007f8a99bdd1b6603e4006b7ccfb41 Mon Sep 17 00:00:00 2001 From: Enzo Cioppettini Date: Tue, 5 Mar 2024 12:42:41 -0300 Subject: [PATCH 13/14] fix genesis_block task: missing tx_count --- indexer/tasks/src/genesis/genesis_block.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/indexer/tasks/src/genesis/genesis_block.rs b/indexer/tasks/src/genesis/genesis_block.rs index c2dabbce..a19f7c23 100644 --- a/indexer/tasks/src/genesis/genesis_block.rs +++ b/indexer/tasks/src/genesis/genesis_block.rs @@ -47,6 +47,7 @@ async fn handle_block( epoch: Set(0), slot: Set(0), payload: Set(Some(block_payload)), + tx_count: Set(0), ..Default::default() }; From 9ece15633c32b752a2bad2cca2f9605118ca18c1 Mon Sep 17 00:00:00 2001 From: Enzo Cioppettini Date: Tue, 12 Mar 2024 11:02:37 -0300 Subject: [PATCH 14/14] add comment explaining the <= :low condition --- .../pagination/slotBoundsPagination.queries.ts | 3 ++- .../models/pagination/slotBoundsPagination.sql | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/webserver/server/app/models/pagination/slotBoundsPagination.queries.ts b/webserver/server/app/models/pagination/slotBoundsPagination.queries.ts index c3de9024..4587b218 100644 --- a/webserver/server/app/models/pagination/slotBoundsPagination.queries.ts +++ b/webserver/server/app/models/pagination/slotBoundsPagination.queries.ts @@ -19,7 +19,7 @@ export interface ISlotBoundsPaginationQuery { result: ISlotBoundsPaginationResult; } -const slotBoundsPaginationIR: any = {"usedParamSet":{"low":true,"high":true},"params":[{"name":"low","required":true,"transform":{"type":"scalar"},"locs":[{"a":155,"b":159}]},{"name":"high","required":true,"transform":{"type":"scalar"},"locs":[{"a":409,"b":414}]}],"statement":"WITH\n low_block AS (\n SELECT\n \"Block\".id,\n \"Block\".slot\n FROM\n \"Block\"\n WHERE\n slot <= :low! AND tx_count > 0\n ORDER BY\n \"Block\".id DESC\n LIMIT\n 1\n ),\n high_block AS (\n SELECT\n \"Block\".id,\n \"Block\".slot\n FROM\n \"Block\"\n WHERE\n slot <= :high! AND tx_count > 0\n ORDER BY\n \"Block\".id DESC\n LIMIT 1\n ),\n min_hash AS (\n (SELECT\n COALESCE(MAX(\"Transaction\".id), -1) AS min_tx_id\n FROM\n \"Transaction\"\n JOIN low_block ON \"Transaction\".block_id = low_block.id\n GROUP BY\n low_block.slot\n LIMIT\n 1\n )\n UNION (SELECT min_tx_id FROM (values(-1)) s(min_tx_id))\n ORDER BY min_tx_id DESC\n LIMIT\n 1\n ),\n max_hash AS (\n SELECT\n COALESCE(MAX(\"Transaction\".id), -2) AS max_tx_id\n FROM\n \"Transaction\"\n JOIN high_block ON \"Transaction\".block_id = high_block.id\n GROUP BY\n high_block.slot\n )\nSELECT\n *\nFROM min_hash\nLEFT JOIN max_hash ON 1 = 1"}; +const slotBoundsPaginationIR: any = {"usedParamSet":{"low":true,"high":true},"params":[{"name":"low","required":true,"transform":{"type":"scalar"},"locs":[{"a":782,"b":786}]},{"name":"high","required":true,"transform":{"type":"scalar"},"locs":[{"a":1036,"b":1041}]}],"statement":"WITH\n low_block AS (\n SELECT\n \"Block\".id,\n \"Block\".slot\n FROM\n \"Block\"\n WHERE\n \n slot <= :low! AND tx_count > 0\n ORDER BY\n \"Block\".id DESC\n LIMIT\n 1\n ),\n high_block AS (\n SELECT\n \"Block\".id,\n \"Block\".slot\n FROM\n \"Block\"\n WHERE\n slot <= :high! AND tx_count > 0\n ORDER BY\n \"Block\".id DESC\n LIMIT 1\n ),\n min_hash AS (\n (SELECT\n COALESCE(MAX(\"Transaction\".id), -1) AS min_tx_id\n FROM\n \"Transaction\"\n JOIN low_block ON \"Transaction\".block_id = low_block.id\n GROUP BY\n low_block.slot\n LIMIT\n 1\n )\n UNION (SELECT min_tx_id FROM (values(-1)) s(min_tx_id))\n ORDER BY min_tx_id DESC\n LIMIT\n 1\n ),\n max_hash AS (\n SELECT\n COALESCE(MAX(\"Transaction\".id), -2) AS max_tx_id\n FROM\n \"Transaction\"\n JOIN high_block ON \"Transaction\".block_id = high_block.id\n GROUP BY\n high_block.slot\n )\nSELECT\n *\nFROM min_hash\nLEFT JOIN max_hash ON 1 = 1"}; /** * Query generated from SQL: @@ -32,6 +32,7 @@ const slotBoundsPaginationIR: any = {"usedParamSet":{"low":true,"high":true},"pa * FROM * "Block" * WHERE + * * slot <= :low! AND tx_count > 0 * ORDER BY * "Block".id DESC diff --git a/webserver/server/app/models/pagination/slotBoundsPagination.sql b/webserver/server/app/models/pagination/slotBoundsPagination.sql index 6fbaa75e..4527bccb 100644 --- a/webserver/server/app/models/pagination/slotBoundsPagination.sql +++ b/webserver/server/app/models/pagination/slotBoundsPagination.sql @@ -7,6 +7,21 @@ WITH FROM "Block" WHERE + /* + We use <= here even though slot filter parameter is exclusive on the + lower bound. This is because the tx that we find here (after joining + with min_hash) is used later in a condition of the form: + + "Transaction".id > :after_tx_id! + + For example. + + Lets say :low is 1, and there is a block with txs at this slot. This + means we want to find _at least_ the first tx in slot 2. + + So what we want in this query is to find the last tx in slot 1. + Then, when we use the > comparator we would get the right tx. + */ slot <= :low! AND tx_count > 0 ORDER BY "Block".id DESC