Update migrator to read the node snapshot

.jenkins/Jenkinsfile.ci

@@ -16,7 +16,7 @@ pipeline {
  nvm(readFile(".nvmrc").trim()) {
  sh '''
  npm install --global yarn
- yarn --frozen-lockfile --ignore-engines
+ yarn --frozen-lockfile
  yarn build
  '''
  }

docs/migration.md

@@ -79,21 +79,18 @@ FLAGS
  -c, --config=config Custom configuration file path.
  -d, --lisk-core-v3-data-path=lisk-core-v3-data-path Path where the lisk-core v3.x instance is running. Current home directory will be considered the default if not provided.
  -h, --help Shows CLI help.
- -m, --min-compatible-version=min-compatible-version [default: >=3.0.4 <=3.0] Minimum compatible version required to run the migrator.
  -o, --output=output File path to write the genesis block json. If not provided, it will default to cwd/genesis_block.json.
- -p, --snapshot-path=snapshot-path Path where the state snapshot will be created. When not provided it defaults to the current directory.
  -s, --snapshot-height=snapshot-height (Required) The height at which the re-genesis block will be generated. Can be specified with the SNAPSHOT_HEIGHT as well.
  -v, --version Shows the CLI version.
  --auto-download-lisk-core-v4 Download lisk-core v4 automatically. Default to false.
  --auto-migrate-config Migrate user configuration automatically. Default to false.
  --auto-start-lisk-core-v4 Start lisk-core v4 automatically. Default to false.
  --snapshot-time-gap=snapshot-time-gap The number of seconds elapsed between the block at height HEIGHT_SNAPSHOT and the snapshot block.
- --use-existing-snapshot Use the existing database snapshot (Temporary flag, will be removed once the `createSnapshot` command is available on Lisk Core v3.x).
 
 EXAMPLES
- lisk-migrator --snapshot-path /path/to/snapshot --snapshot-height 20931763 --lisk-core-path /path/to/data-dir
+ lisk-migrator --snapshot-height 20931763 --lisk-core-path /path/to/data-dir
 
- lisk-migrator --snapshot-path /path/to/snapshot --snapshot-height 20931763 --lisk-core-path /path/to/data-dir --auto-download-lisk-core-v4 --auto-start-lisk-core-v4 --auto-migrate-config
+ lisk-migrator --snapshot-height 20931763 --lisk-core-path /path/to/data-dir --auto-download-lisk-core-v4 --auto-start-lisk-core-v4 --auto-migrate-config
 ```
 
 <!--

package.json

@@ -59,11 +59,11 @@
  "/docs"
  ],
  "dependencies": {
- "@liskhq/lisk-api-client": "5.1.6",
- "@liskhq/lisk-chain": "0.3.4",
+ "@liskhq/lisk-api-client": "5.1.7-alpha.2",
+ "@liskhq/lisk-chain": "0.3.5-alpha.2",
  "@liskhq/lisk-codec": "0.3.0-beta.4",
- "@liskhq/lisk-cryptography": "3.2.1",
- "@liskhq/lisk-db": "0.2.1",
+ "@liskhq/lisk-cryptography": "3.2.2-alpha.1",
+ "@liskhq/lisk-db": "0.3.6",
  "@liskhq/lisk-utils": "0.3.0-beta.3",
  "@liskhq/lisk-validator": "0.7.0-beta.4",
  "@oclif/command": "1.8.21",

src/assets/pos.ts

@@ -12,7 +12,7 @@
  * Removal or modification of this copyright notice is prohibited.
  */
 import { posGenesisStoreSchema } from 'lisk-framework';
-import { KVStore, formatInt } from '@liskhq/lisk-db';
+import { Database } from '@liskhq/lisk-db';
 import { codec } from '@liskhq/lisk-codec';
 import { BlockHeader, Transaction } from '@liskhq/lisk-chain';
 import { getLisk32AddressFromAddress, getAddressFromPublicKey } from '@liskhq/lisk-cryptography';
@@ -55,23 +55,42 @@ const ceiling = (a: number, b: number) => {
  return Math.floor((a + b - 1) / b);
 };
 
+// TODO: Remove method once exported from lisk-db
+export const formatInt = (num: number | bigint): string => {
+ let buf: Buffer;
+ if (typeof num === 'bigint') {
+ if (num < BigInt(0)) {
+ throw new Error('Negative number cannot be formatted');
+ }
+ buf = Buffer.alloc(8);
+ buf.writeBigUInt64BE(num);
+ } else {
+ if (num < 0) {
+ throw new Error('Negative number cannot be formatted');
+ }
+ buf = Buffer.alloc(4);
+ buf.writeUInt32BE(num, 0);
+ }
+ return buf.toString('binary');
+};
+
 export const getValidatorKeys = async (
  accounts: Account[],
  snapshotHeight: number,
  snapshotHeightPrevious: number,
- db: KVStore,
+ db: Database,
 ): Promise<Record<string, string>> => {
  const keys: Record<string, string> = {};
 
  const blocksStream = db.createReadStream({
- gte: `${DB_KEY_BLOCKS_HEIGHT}:${formatInt(snapshotHeightPrevious + 1)}`,
- lte: `${DB_KEY_BLOCKS_HEIGHT}:${formatInt(snapshotHeight)}`,
+ gte: Buffer.from(`${DB_KEY_BLOCKS_HEIGHT}:${formatInt(snapshotHeightPrevious + 1)}`),
+ lte: Buffer.from(`${DB_KEY_BLOCKS_HEIGHT}:${formatInt(snapshotHeight)}`),
  });
 
  const blockIDs: Buffer[] = await getBlocksIDsFromDBStream(blocksStream);
 
  for (const id of blockIDs) {
- const blockHeaderBuffer = await db.get(`${DB_KEY_BLOCKS_ID}:${keyString(id)}`);
+ const blockHeaderBuffer = await db.get(Buffer.from(`${DB_KEY_BLOCKS_ID}:${keyString(id)}`));
  const blockHeader: BlockHeader = codec.decode(blockHeaderSchema, blockHeaderBuffer);
  const payload = ((await getTransactions(id, db)) as unknown) as Transaction[];
 

src/constants.ts

@@ -85,6 +85,8 @@ export const BINARY_ADDRESS_LENGTH = 20;
 export const TRANSACTION_ID_LENGTH = SHA_256_HASH_LENGTH;
 
 export const DEFAULT_DATA_DIR = 'data';
-export const EXTRACTED_SNAPSHOT_DIR = 'blockchain.db';
+export const SNAPSHOT_DIR = `${DEFAULT_DATA_DIR}/backup`;
+export const MIN_SUPPORTED_LISK_CORE_VERSION = '3.0.5';
+export const DEFAULT_LISK_CORE_PATH = '~/.lisk/lisk-core';
 
 export const DEFAULT_VERSION = '0.1.0';

src/createAsset.ts

@@ -12,7 +12,7 @@
  * Removal or modification of this copyright notice is prohibited.
  */
 import { codec } from '@liskhq/lisk-codec';
-import { KVStore } from '@liskhq/lisk-db';
+import { Database } from '@liskhq/lisk-db';
 import { getLisk32AddressFromAddress } from '@liskhq/lisk-cryptography';
 
 import {
@@ -68,9 +68,9 @@ const addressComparator = (
  b: StakerBuffer | ValidatorEntryBuffer,
 ) => a.address.compare(b.address);
 export class CreateAsset {
- private readonly _db: KVStore;
+ private readonly _db: Database;
 
- public constructor(db: KVStore) {
+ public constructor(db: Database) {
  this._db = db;
  }
 
@@ -88,12 +88,12 @@ export class CreateAsset {
  const stakers: StakerBuffer[] = [];
 
  const encodedUnregisteredAddresses = await this._db.get(
- `${DB_KEY_CHAIN_STATE}:${CHAIN_STATE_UNREGISTERED_ADDRESSES}`,
+ Buffer.from(`${DB_KEY_CHAIN_STATE}:${CHAIN_STATE_UNREGISTERED_ADDRESSES}`),
  );
 
  const accountStream = await this._db.createReadStream({
- gte: `${DB_KEY_ACCOUNTS_ADDRESS}:${Buffer.alloc(20, 0).toString('binary')}`,
- lte: `${DB_KEY_ACCOUNTS_ADDRESS}:${Buffer.alloc(20, 255).toString('binary')}`,
+ gte: Buffer.from(`${DB_KEY_ACCOUNTS_ADDRESS}:${Buffer.alloc(20, 0).toString('binary')}`),
+ lte: Buffer.from(`${DB_KEY_ACCOUNTS_ADDRESS}:${Buffer.alloc(20, 255).toString('binary')}`),
  });
 
  const allAccounts = ((await getDataFromDBStream(
@@ -206,7 +206,7 @@ export class CreateAsset {
  }));
 
  const encodedDelegatesVoteWeights = await this._db.get(
- `${DB_KEY_CHAIN_STATE}:${CHAIN_STATE_DELEGATE_VOTE_WEIGHTS}`,
+ Buffer.from(`${DB_KEY_CHAIN_STATE}:${CHAIN_STATE_DELEGATE_VOTE_WEIGHTS}`),
  );
 
  const decodedDelegatesVoteWeights: VoteWeightsWrapper = await codec.decode(

src/index.ts

@@ -15,16 +15,17 @@ import util from 'util';
 import * as fs from 'fs-extra';
 import { join } from 'path';
 import { Application, ApplicationConfig, PartialApplicationConfig } from 'lisk-framework';
-import { KVStore } from '@liskhq/lisk-db';
+import { Database } from '@liskhq/lisk-db';
 import * as semver from 'semver';
 import { Command, flags as flagsParser } from '@oclif/command';
 import cli from 'cli-ux';
 import { Block } from '@liskhq/lisk-chain';
 import {
  NETWORK_CONSTANT,
  ROUND_LENGTH,
- DEFAULT_DATA_DIR,
- EXTRACTED_SNAPSHOT_DIR,
+ SNAPSHOT_DIR,
+ MIN_SUPPORTED_LISK_CORE_VERSION,
+ DEFAULT_LISK_CORE_PATH,
 } from './constants';
 import { getAPIClient } from './client';
 import {
@@ -37,28 +38,17 @@ import {
 } from './utils/config';
 import {
  observeChainHeight,
- setBlockIDAtSnapshotHeight,
- getBlockIDAtSnapshotHeight,
- getBlockIDAtHeight,
  getTokenIDLsk,
  getHeightPrevSnapshotBlock,
  setTokenIDLskByNetID,
  setHeightPrevSnapshotBlockByNetID,
 } from './utils/chain';
 import { createGenesisBlock, writeGenesisBlock } from './utils/genesis_block';
 import { CreateAsset } from './createAsset';
-import { ApplicationConfigV3, NetworkConfigLocal } from './types';
+import { ApplicationConfigV3, NetworkConfigLocal, NodeInfo } from './types';
 import { installLiskCore, startLiskCore } from './utils/node';
-import { extractTarBall } from './utils/fs';
 
 let finalConfigCorev4: PartialApplicationConfig;
-
-// TODO: Import snapshot command from core once implemented
-const createSnapshot = async (liskCorePath: string, snapshotPath: string) => ({
- liskCorePath,
- snapshotPath,
-});
-
 class LiskMigrator extends Command {
  public static description = 'Migrate Lisk Core to latest version';
 
@@ -67,13 +57,6 @@ class LiskMigrator extends Command {
  version: flagsParser.version({ char: 'v' }),
  help: flagsParser.help({ char: 'h' }),
 
- // Custom flags
- 'min-compatible-version': flagsParser.string({
- char: 'm',
- required: false,
- description: 'Minimum compatible version required to run the migrator.',
- default: '>=3.0.4 <=3.0', // TODO: Update version to '3.0.5
- }),
  output: flagsParser.string({
  char: 'o',
  required: false,
@@ -122,62 +105,38 @@ class LiskMigrator extends Command {
  description: 'Start lisk core v4 automatically. Default to false.',
  default: false,
  }),
- 'snapshot-path': flagsParser.string({
- char: 'p',
- required: false,
- description:
- 'Path where the state snapshot will be created. When not supplied, defaults to the current directory.',
- }),
- // TODO: Remove once createSnapshot command is available
- 'use-existing-snapshot': flagsParser.boolean({
- required: false,
- env: 'USE_EXISTING_SNAPSHOT',
- description:
- 'Use the existing database snapshot (Temporary flag, will be removed once the `createSnapshot` command is available on Lisk Core v3.x).',
- default: false,
- }),
  };
 
  public async run(): Promise<void> {
  try {
  const { flags } = this.parse(LiskMigrator);
- const liskCorePath = flags['lisk-core-v3-data-path'] ?? process.cwd();
+ const liskCoreV3Path = flags['lisk-core-v3-data-path'] ?? DEFAULT_LISK_CORE_PATH;
  const outputPath = flags.output ?? join(__dirname, '..', 'output');
  const snapshotHeight = flags['snapshot-height'];
  const customConfigPath = flags.config;
  const autoMigrateUserConfig = flags['auto-migrate-config'] ?? false;
- const compatibleVersions = flags['min-compatible-version'];
- const useExistingSnapshot = flags['use-existing-snapshot'];
- const snapshotPath = ((useExistingSnapshot
- ? flags['snapshot-path']
- : flags['snapshot-path'] ?? process.cwd()) as unknown) as string;
  const autoDownloadLiskCoreV4 = flags['auto-download-lisk-core-v4'];
  const autoStartLiskCoreV4 = flags['auto-start-lisk-core-v4'];
 
- if (useExistingSnapshot) {
- if (!snapshotPath) {
- this.error("Snapshot path is required when 'use-existing-snapshot' set to true");
- } else if (!snapshotPath.endsWith('.tar.gz')) {
- this.error('Snapshot should always end with ".tar.gz"');
- }
- }
+ const client = await getAPIClient(liskCoreV3Path);
+ const nodeInfo = (await client.node.getNodeInfo()) as NodeInfo;
+ const { version: appVersion, networkIdentifier } = nodeInfo;
 
- const dataDir = join(__dirname, '..', DEFAULT_DATA_DIR);
- cli.action.start(`Extracting snapshot at ${dataDir}`);
- await extractTarBall(snapshotPath, dataDir);
- cli.action.stop();
+ cli.action.start('Verifying if backup height from node config matches snapshot height');
+ if (snapshotHeight !== nodeInfo.backup.height) {
+ this.error(
+ `Snapshot height ${snapshotHeight} does not matches backup height ${nodeInfo.backup.height}.`,
+ );
+ }
+ cli.action.stop('Snapshot height matches backup height');
 
  cli.action.start(
  `Verifying snapshot height to be multiples of round length i.e ${ROUND_LENGTH}`,
  );
  if (snapshotHeight % ROUND_LENGTH !== 0) {
  this.error(`Invalid Snapshot Height: ${snapshotHeight}.`);
  }
- cli.action.stop('Snapshot Height is valid');
-
- const client = await getAPIClient(liskCorePath);
- const nodeInfo = await client.node.getNodeInfo();
- const { version: appVersion, networkIdentifier } = nodeInfo;
+ cli.action.stop('Snapshot height is valid');
 
  const networkConstant = NETWORK_CONSTANT[networkIdentifier] as NetworkConfigLocal;
  const networkDir = `${outputPath}/${networkIdentifier}`;
@@ -190,68 +149,40 @@ class LiskMigrator extends Command {
  }
  }
 
- cli.action.start('Verifying Lisk-Core version');
+ cli.action.start('Verifying Lisk Core version');
  const liskCoreVersion = semver.coerce(appVersion);
  if (!liskCoreVersion) {
  this.error(
- `Unsupported lisk-core version detected. Supported version range ${compatibleVersions}.`,
+ `Unsupported lisk-core version detected. Supported version range ${MIN_SUPPORTED_LISK_CORE_VERSION}.`,
  );
  }
- if (!semver.satisfies(liskCoreVersion, compatibleVersions)) {
+ if (!semver.gte(MIN_SUPPORTED_LISK_CORE_VERSION, liskCoreVersion)) {
  this.error(
- `Lisk-Migrator utility is not compatible for lisk-core version ${liskCoreVersion.version}. Compatible versions range is: ${compatibleVersions}.`,
+ `Lisk Migrator utility is not compatible for lisk-core version ${liskCoreVersion.version}. The minimum compatible version is: ${MIN_SUPPORTED_LISK_CORE_VERSION}.`,
  );
  }
  cli.action.stop(`${liskCoreVersion.version} detected`);
 
  // User specified custom config file
  const configV3: ApplicationConfigV3 = customConfigPath
- ? await getConfig(liskCorePath, customConfigPath)
- : await getConfig(liskCorePath);
+ ? await getConfig(liskCoreV3Path, customConfigPath)
+ : await getConfig(liskCoreV3Path);
 
  await setTokenIDLskByNetID(networkIdentifier);
  await setHeightPrevSnapshotBlockByNetID(networkIdentifier);
 
- if (!useExistingSnapshot) {
- await observeChainHeight({
- label: 'Waiting for snapshot height',
- liskCorePath,
- height: snapshotHeight,
- delay: 500,
- isFinal: false,
- });
-
- await setBlockIDAtSnapshotHeight(liskCorePath, snapshotHeight);
-
- // TODO: Placeholder to issue createSnapshot command from lisk-core
- cli.action.start('Creating snapshot');
- await createSnapshot(liskCorePath, snapshotPath);
- cli.action.stop();
-
- await observeChainHeight({
- label: 'Waiting for snapshot height to be finalized',
- liskCorePath,
- height: snapshotHeight,
- delay: 500,
- isFinal: true,
- });
-
- const blockID = getBlockIDAtSnapshotHeight();
- const finalizedBlockID = await getBlockIDAtHeight(liskCorePath, snapshotHeight);
-
- cli.action.start('Verifying blockID');
- if (blockID !== finalizedBlockID) {
- this.error('Snapshotted blockID does not match with the finalized blockID.');
- }
- cli.action.stop();
- }
-
- // TODO: Stop Lisk Core v3 automatically when the application management is implemented
+ await observeChainHeight({
+ label: 'Waiting for snapshot height to be finalized',
+ liskCoreV3Path,
+ height: snapshotHeight,
+ delay: 500,
+ isFinal: true,
+ });
 
  // Create new DB instance based on the snapshot path
  cli.action.start('Creating database instance');
- const snapshotFilePathExtracted = join(dataDir, EXTRACTED_SNAPSHOT_DIR);
- const db = new KVStore(snapshotFilePathExtracted);
+ const snapshotDirPath = join(liskCoreV3Path, SNAPSHOT_DIR);
+ const db = new Database(snapshotDirPath);
  cli.action.stop();
 
  // Create genesis assets
@@ -331,7 +262,7 @@ class LiskMigrator extends Command {
  if (isUserConfirmed) {
  cli.action.start('Starting lisk-core v4');
  const network = networkConstant.name as string;
- await startLiskCore(this, finalConfigCorev4, appVersion, liskCorePath, network);
+ await startLiskCore(this, finalConfigCorev4, appVersion, liskCoreV3Path, network);
  this.log('Started Lisk Core v4 at default data directory.');
  cli.action.stop();
  } else {