Merge branch 'dev' into task/604_address-formatting

catapult-sdk/package.json

@@ -17,17 +17,16 @@
   "license": "ISC",
   "devDependencies": {
     "chai": "^4.2.0",
-    "coveralls": "^3.0.9",
+    "coveralls": "^3.1.1",
     "cross-env": "^5.2.0",
     "eslint": "^6.8.0",
     "eslint-config-airbnb": "^18.1.0",
     "eslint-plugin-import": "^2.19.1",
     "eslint-plugin-jsx-a11y": "^6.2.3",
     "eslint-plugin-react": "^7.19.0",
     "eslint-plugin-react-hooks": "^2.5.0",
-    "mocha": "^6.1.4",
-    "mocha-jenkins-reporter": "^0.4.3",
-    "ncp": "^2.0.0",
+    "mocha": "^9.1.0",
+    "mocha-jenkins-reporter": "^0.4.6",
     "nyc": "^14.1.1",
     "rimraf": "^2.6.3"
   },

catapult-sdk/src/plugins/receipts.js

@@ -43,6 +43,7 @@ const receiptsPlugin = {
 			const schemaName = `${statementType}Statement`;
 			builder.addSchema(schemaName, {
 				id: ModelType.objectId,
+				meta: { type: ModelType.object, schemaName: 'statement.meta' },
 				statement: { type: ModelType.object, schemaName: `${schemaName}.statement` }
 			});
 			builder.addSchema(`${schemaName}.statement`, schema);
@@ -64,6 +65,10 @@ const receiptsPlugin = {
 			receipts: { type: ModelType.array, schemaName: entity => getBasicReceiptType(entity.type) }
 		});
 
+		builder.addSchema('statement.meta', {
+			timestamp: ModelType.uint64
+		});
+
 		// addressResolution statements
 		builder.addSchema('receipts.entry.address', {
 			source: { type: ModelType.object, schemaName: 'receipts.source' },

catapult-sdk/test/crypto/merkleAuditProof_spec.js

@@ -22,7 +22,7 @@
 const {
 	indexOfLeafWithHash, buildAuditPath, NodePosition, siblingOf, HashNotFoundError,
 	InvalidTree, evenify
-} = require('../../src/crypto/merkleAuditProof.js');
+} = require('../../src/crypto/merkleAuditProof');
 const convert = require('../../src/utils/convert');
 const { expect } = require('chai');
 

catapult-sdk/test/plugins/receipts_spec.js

@@ -37,14 +37,15 @@ describe('receipts plugin', () => {
 			const modelSchema = builder.build();
 
 			// Assert:
-			expect(Object.keys(modelSchema).length).to.equal(numDefaultKeys + 14);
+			expect(Object.keys(modelSchema).length).to.equal(numDefaultKeys + 15);
 			expect(modelSchema).to.contain.all.keys([
 				'addressResolutionStatement',
 				'addressResolutionStatement.statement',
 				'mosaicResolutionStatement',
 				'mosaicResolutionStatement.statement',
 				'transactionStatement',
 				'transactionStatement.statement',
+				'statement.meta',
 				'receipts.balanceChange',
 				'receipts.balanceTransfer',
 				'receipts.artifactExpiry',
@@ -59,7 +60,7 @@ describe('receipts plugin', () => {
 			// - mosaicResolutionStatement
 			// - transactionStatement
 			['addressResolution', 'mosaicResolution', 'transaction'].forEach(statementType => {
-				expect(Object.keys(modelSchema[`${statementType}Statement`]).length).to.equal(2);
+				expect(Object.keys(modelSchema[`${statementType}Statement`])).deep.equal(['id', 'meta', 'statement']);
 				expect(modelSchema[`${statementType}Statement`]).to.contain.all.keys(['statement']);
 			});
 
@@ -91,6 +92,10 @@ describe('receipts plugin', () => {
 				'source', 'resolved'
 			]);
 
+			expect(modelSchema['statement.meta']).to.contain.all.keys([
+				'timestamp'
+			]);
+
 			// - receipts.entry.mosaic
 			expect(Object.keys(modelSchema['receipts.entry.mosaic']).length).to.equal(2);
 			expect(modelSchema['receipts.entry.mosaic']).to.contain.all.keys([

rest/bootstrap-preset-mainnet.yml

@@ -0,0 +1,20 @@
+#nodeEqualityStrategy: public-key
+logLevel: 'Debug'
+throttlingBurst: 35
+throttlingRate: 1000
+databases:
+  - name: db
+    openPort: true
+nodes:
+  - trustedHosts: '127.0.0.1, 172.20.0.25, 172.20.0.1'
+    localNetworks: '127.0.0.1, 172.20.0.25, 172.20.0.1'
+    openPort: true
+    brokerOpenPort: 7902
+gateways:
+  - excludeDockerService: true
+    name: rest
+    apiNodeConfigPath: target/gateways/rest/api-node-config
+    restLoggingFilename: target/rest.log
+    databaseHost: localhost
+    apiNodeHost: localhost
+

rest/package.json

@@ -15,9 +15,10 @@
     "lint:fix": "eslint src test --fix",
     "lint:jenkins": "eslint -o tests.catapult.lint.xml -f junit src test || exit 0",
     "bootstrap-clean": "symbol-bootstrap clean",
-    "bootstrap-start": "symbol-bootstrap start -a light -c bootstrap-preset.yml --healthCheck",
-    "bootstrap-start-testnet": "symbol-bootstrap start -p testnet -a dual -c bootstrap-preset-testnet.yml --healthCheck",
-    "bootstrap-start-detached": "symbol-bootstrap start -a light -c bootstrap-preset.yml --detached --healthCheck --pullImages",
+    "bootstrap-start": "symbol-bootstrap start -a light -c bootstrap-preset.yml --healthCheck --noPassword",
+    "bootstrap-start-testnet": "symbol-bootstrap start -p testnet -a dual -c bootstrap-preset-testnet.yml --healthCheck --noPassword",
+    "bootstrap-start-mainnet": "symbol-bootstrap start -p testnet -a dual -c bootstrap-preset-mainnet.yml --healthCheck --noPassword",
+    "bootstrap-start-detached": "symbol-bootstrap start -a light -c bootstrap-preset.yml --detached --healthCheck --noPassword",
     "bootstrap-stop": "symbol-bootstrap stop",
     "bootstrap-run": "symbol-bootstrap run",
     "bootstrap-run-detached": "symbol-bootstrap run --detached --healthCheck",
@@ -30,7 +31,7 @@
   "devDependencies": {
     "chai": "^4.2.0",
     "cross-env": "^5.2.0",
-    "coveralls": "^3.0.9",
+    "coveralls": "^3.1.1",
     "eslint": "^6.8.0",
     "eslint-config-airbnb": "^18.1.0",
     "eslint-plugin-import": "^2.19.1",
@@ -39,19 +40,18 @@
     "eslint-plugin-react-hooks": "^2.5.0",
     "hippie": "^0.6.1",
     "minimist": "^1.2.0",
-    "mocha": "^6.1.4",
+    "mocha": "^9.1.0",
     "mocha-jenkins-reporter": "^0.4.3",
-    "ncp": "^2.0.0",
     "nodemon": "^2.0.6",
+    "nyc": "^14.1.1",
     "rimraf": "^2.6.3",
     "sinon": "^7.3.2",
-    "symbol-bootstrap": "1.0.4"
+    "symbol-bootstrap": "^1.0.8-alpha-202108171450"
   },
   "dependencies": {
     "catapult-sdk": "link:../catapult-sdk",
     "ini": "^1.3.5",
     "mongodb": "^3.3.0-beta2",
-    "nyc": "^14.1.1",
     "restify": "^8.3.3",
     "restify-errors": "^8.0.0",
     "sshpk": "1.16.1",

rest/src/db/CatapultDb.js

@@ -225,6 +225,23 @@ class CatapultDb {
 		).then(this.sanitizer.renameId);
 	}
 
+	/**
+	 * Returns the blocks (or a projection of the blocks) for the given heights.
+	 * @param {Long[]} heights the array of long heights
+	 * @param {object} projection optional projection, by default it excludes the transactionMerkleTree and statementMerkleTree fields
+	 * @returns {Promise} with the blocks objects or projections.
+	 */
+	blocksAtHeights(heights, projection) {
+		if (!heights.length)
+			return Promise.resolve([]);
+
+		return this.queryDocumentsAndCopyIds(
+			'blocks',
+			{ 'block.height': { $in: heights } },
+			{ projection: projection || { 'meta.transactionMerkleTree': 0, 'meta.statementMerkleTree': 0 } }
+		);
+	}
+
 	blockWithMerkleTreeAtHeight(height, merkleTreeName) {
 		const blockMerkleTreeNames = ['transactionMerkleTree', 'statementMerkleTree'];
 		const excludedMerkleTrees = {};

rest/src/db/dbUtils.js

@@ -94,4 +94,12 @@ const dbUtils = {
 	}
 };
 
+	/**
+	 * Creates copy of the array without duplicated longs.
+	 * @param {Long[]} duplicatedIds of {Long} objects.
+	 * @returns {Long[]} copy of the original list without duplicated values.
+	 */
+	uniqueLongList: duplicatedIds => duplicatedIds.filter((height, index) =>
+		index === duplicatedIds.findIndex(anotherHeight => anotherHeight.equals(height)))
+};
 module.exports = dbUtils;

rest/src/plugins/receipts/ReceiptsDb.js

@@ -19,7 +19,7 @@
  * along with Catapult.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-const { convertToLong, buildOffsetCondition } = require('../../db/dbUtils');
+const { convertToLong, buildOffsetCondition, uniqueLongList } = require('../../db/dbUtils');
 const catapult = require('catapult-sdk');
 
 const { convert, uint64 } = catapult.utils;
@@ -84,7 +84,9 @@ class ReceiptsDb {
 		}
 
 		const sortConditions = { [sortingOptions[options.sortField]]: options.sortDirection };
-		return this.catapultDb.queryPagedDocuments(conditions, [], sortConditions, 'transactionStatements', options);
+
+		return this.catapultDb.queryPagedDocuments(conditions, [], sortConditions,
+			'transactionStatements', options).then(page => this.addBlockMeta(page));
 	}
 
 	/**
@@ -109,7 +111,44 @@ class ReceiptsDb {
 			conditions['statement.height'] = convertToLong(height);
 
 		const sortConditions = { [sortingOptions[options.sortField]]: options.sortDirection };
-		return this.catapultDb.queryPagedDocuments(conditions, [], sortConditions, `${artifact}ResolutionStatements`, options);
+		return this.catapultDb.queryPagedDocuments(conditions, [], sortConditions,
+			`${artifact}ResolutionStatements`, options).then(page => this.addBlockMeta(page));
+	}
+
+	/**
+	 * It retrives and adds the blocks information to the statements' meta.
+	 *
+	 * The block information includes the its timestamp
+	 *
+	 * @param {object} page the page without meta in the items.
+	 * @returns {Promise<{pagination, data}>} the page with the added block's meta to the items.
+	 */
+	async addBlockMeta(page) {
+		const blockHeights = uniqueLongList(
+			page.data.map(pageItem => pageItem.statement.height)
+		);
+		const blocks = await this.catapultDb.blocksAtHeights(blockHeights,
+			{ 'block.timestamp': 1, 'block.height': 1 });
+		const data = page.data.map(pageItem => {
+			const statementBlock = blocks.find(
+				blockInfo => blockInfo.block.height.equals(
+					pageItem.statement.height
+				)
+			);
+			if (!statementBlock) {
+				throw new Error(
+					`Cannot find block with height ${pageItem.statement.height.toString()}`
+				);
+			}
+			if (!statementBlock.block.timestamp) {
+				throw new Error(
+					`Cannot find timestamp in block with height ${pageItem.statement.height.toString()}`
+				);
+			}
+			// creates a copy of the page item with the added timestamp to the meta field.
+			return { meta: { ...pageItem.meta, timestamp: statementBlock.block.timestamp }, ...pageItem };
+		});
+		return { data, pagination: page.pagination };
 	}
 }
 

rest/test/db/CatapultDb_spec.js

@@ -394,17 +394,15 @@ describe('catapult db', () => {
 			});
 		});
 	});
-
+	const runBlockAtHeightDbTest = (dbEntities, issueDbCommand, assertDbCommandResult) => {
+		const db = new CatapultDb(Object.assign({ networkId: Mijin_Test_Network }, DefaultPagingOptions));
+		return db.connect(testDbOptions.url, 'test', testDbOptions.connectionPoolSize)
+			.then(() => test.db.populateDatabase(db, dbEntities))
+			.then(() => issueDbCommand(db))
+			.then(assertDbCommandResult)
+			.then(() => db.close());
+	};
 	describe('block at height', () => {
-		const runBlockAtHeightDbTest = (dbEntities, issueDbCommand, assertDbCommandResult) => {
-			const db = new CatapultDb(Object.assign({ networkId: Mijin_Test_Network }, DefaultPagingOptions));
-			return db.connect(testDbOptions.url, 'test', testDbOptions.connectionPoolSize)
-				.then(() => test.db.populateDatabase(db, dbEntities))
-				.then(() => issueDbCommand(db))
-				.then(assertDbCommandResult)
-				.then(() => db.close());
-		};
-
 		it('undefined is returned for block at unknown height', () =>
 			// Assert:
 			runDbTest(
@@ -445,6 +443,72 @@ describe('catapult db', () => {
 		});
 	});
 
+	describe('blocks at heights', () => {
+		it('can retrieve empty blocks when no heights', () => {
+			// Arrange:
+			const seedBlock1 = test.db.createDbBlock(Default_Height);
+			const blockTransactions = test.db.createDbTransactions(2, test.random.publicKey(), test.random.address());
+
+			// Assert:
+			return runBlockAtHeightDbTest(
+				{ blocks: [seedBlock1], transactions: blockTransactions },
+				db => db.blocksAtHeights(
+					[]
+				),
+				blocks => expect(blocks).to.deep.equal(
+					[]
+				)
+			);
+		});
+
+		it('can retrieve blocks with transactions at heights', () => {
+			// Arrange:
+			const seedBlock1 = test.db.createDbBlock(Default_Height);
+			const seedBlock2 = test.db.createDbBlock(Default_Height + 1);
+			const seedBlock3 = test.db.createDbBlock(Default_Height + 2);
+			const blockTransactions = test.db.createDbTransactions(2, test.random.publicKey(), test.random.address());
+
+			// Assert:
+			return runBlockAtHeightDbTest(
+				{ blocks: [seedBlock1, seedBlock2, seedBlock3], transactions: blockTransactions },
+				db => db.blocksAtHeights(
+					[Long.fromNumber(Default_Height), Long.fromNumber(Default_Height + 1)]
+				),
+				blocks => expect(blocks).to.deep.equal(
+					[stripExtraneousBlockInformation(renameId(seedBlock1)), stripExtraneousBlockInformation(renameId(seedBlock2))]
+				)
+			);
+		});
+
+		it('can retrieve blocks with transactions at heights with projection', () => {
+			// Arrange:
+			const seedBlock1 = test.db.createDbBlock(Default_Height);
+			const seedBlock2 = test.db.createDbBlock(Default_Height + 1);
+			const seedBlock3 = test.db.createDbBlock(Default_Height + 2);
+			const blockTransactions = test.db.createDbTransactions(2, test.random.publicKey(), test.random.address());
+
+			// Assert:
+			const projectBlock = block => ({
+				id: block._id,
+				block: {
+					height: block.block.height,
+					timestamp: block.block.timestamp,
+					feeMultiplier: block.block.feeMultiplier
+				}
+			});
+			return runBlockAtHeightDbTest(
+				{ blocks: [seedBlock1, seedBlock2, seedBlock3], transactions: blockTransactions },
+				db => db.blocksAtHeights(
+					[Long.fromNumber(Default_Height), Long.fromNumber(Default_Height + 1)],
+					{ 'block.timestamp': 1, 'block.height': 1, 'block.feeMultiplier': 1 }
+				),
+				blocks => expect(blocks).to.deep.equal(
+					[projectBlock(seedBlock1), projectBlock(seedBlock2)]
+				)
+			);
+		});
+	});
+
 	describe('block at height with statement merkle tree', () => {
 		it('undefined is returned for block at unknown height', () =>
 			// Assert:

rest/test/db/dbUtils_spec.js

@@ -90,6 +90,24 @@ describe('db utils', () => {
 			expect(() => dbUtils.longToUint64('abc')).to.throw('abc has an invalid format: not long');
 		});
 	});
+	describe('uniqueLongList', () => {
+		it('unique list empty', () => {
+			// Act + Assert
+			expect(dbUtils.uniqueLongList([])).to.deep.equal([]);
+		});
+
+		it('unique list not duplicated', () => {
+			// Act + Assert
+			expect(dbUtils.uniqueLongList([convertToLong(1), convertToLong(2), convertToLong(3)]))
+				.to.deep.equal([convertToLong(1), convertToLong(2), convertToLong(3)]);
+		});
+
+		it('unique list duplicated', () => {
+			// Act + Assert
+			expect(dbUtils.uniqueLongList([convertToLong(3), convertToLong(1), convertToLong(3)]))
+				.to.deep.equal([convertToLong(3), convertToLong(1)]);
+		});
+	});
 
 	describe('buildOffsetCondition', () => {
 		it('undefined offset', () => {