Add GCP support for express cli

.env.example

@@ -1,21 +1,12 @@
-# terraform variables
+# Terraform variables - AWS specific
 TF_VAR_AWS_PROFILE=default # do not change this
-TF_VAR_VM_NAME=YOUR_IDENTIFIER # default "polygon-user". It can be any string, used to discriminate between instances
-TF_VAR_DOCKERIZED=no # default "no", otherwise only one VM is created and the Polygon devnet will run in docker containers
-TF_VAR_BOR_DISK_SIZE_GB=20 # size of the disk in GB for Bor client (default is 100GB)
-TF_VAR_ERIGON_DISK_SIZE_GB=20 # size of the disk in GB for Erigon client (default is 100GB)
-TF_VAR_BOR_ARCHIVE_DISK_SIZE_GB=100 # size of the disk in GB in Bor archive node (default is 100GB)
-TF_VAR_ERIGON_ARCHIVE_DISK_SIZE_GB=100 # size of the disk in GB in Erigon archive node (default is 100GB)
+TF_VAR_AWS_REGION=us-west-2 # AWS region to deploy to (default is us-west-2)
+TF_VAR_INSTANCE_AMI=ami-017fecd1353bcc96e # ami instance type (default is ami-017fecd1353bcc96e, running ubuntu 22.04)
 TF_VAR_BOR_IOPS=3000 # Amount of provisioned IOPS for Bor client
 TF_VAR_ERIGON_IOPS=3000 # Amount of provisioned IOPS for Erigon client
 TF_VAR_BOR_ARCHIVE_IOPS=15000 # Amount of provisioned IOPS in Bor archive node
 TF_VAR_ERIGON_ARCHIVE_IOPS=15000 # Amount of provisioned IOPS in Erigon archive node
-TF_VAR_BOR_VALIDATOR_COUNT=2 # number of Bor validator nodes (default is 2). Note that while spinning up a public network (mainnet/mumbai) node, this will serve as a non-validator
-TF_VAR_BOR_SENTRY_COUNT=1 # number of non-validator Bor sentry nodes (default is 1)
-TF_VAR_BOR_ARCHIVE_COUNT=0 # number of Bor archive nodes (default is 0)
-TF_VAR_ERIGON_VALIDATOR_COUNT=0 # number of Erigon validator nodes (default is 0). Note that while spinning up a public network (mainnet/mumbai) node, this will serve as a non-validator
-TF_VAR_ERIGON_SENTRY_COUNT=0 # number of non-validator Erigon sentry nodes (default is 0)
-TF_VAR_ERIGON_ARCHIVE_COUNT=0 # number of Erigon archive nodes (default is 0)
+TF_VAR_PEM_FILE=aws-key # name of the certificate (.pem file) previously generated (default is aws-key)
 TF_VAR_BOR_INSTANCE_TYPE=t2.xlarge # type of the EC2 VM instance for Bor client (default is t2.xlarge)
 TF_VAR_ERIGON_INSTANCE_TYPE=r5b.large # type of the EC2 VM instance for Erigon client (default is r5b.large)
 TF_VAR_BOR_ARCHIVE_INSTANCE_TYPE=t2.xlarge # default t2.xlarge
@@ -24,9 +15,38 @@ TF_VAR_BOR_VOLUME_TYPE=gp3 # default gp3
 TF_VAR_ERIGON_VOLUME_TYPE=gp3 # default gp3
 TF_VAR_BOR_ARCHIVE_VOLUME_TYPE=io1 # type of EBS volume for Bor archive nodes (default is io1)
 TF_VAR_ERIGON_ARCHIVE_VOLUME_TYPE=io1 # type of EBS volume for Erigon archive nodes (default is io1)
-TF_VAR_INSTANCE_AMI=ami-017fecd1353bcc96e # ami instance type (default is ami-017fecd1353bcc96e, running ubuntu 22.04)
-TF_VAR_PEM_FILE=aws-key # name of the certificate (.pem file) previously generated (default is aws-key)
-TF_VAR_REGION=us-west-2 # AWS region to deploy to (default is us-west-2)
+
+# Terraform variables - GCP specific
+TF_VAR_PROJECT_ID=YOUR_PROJECT_ID # GCP Project ID
+TF_VAR_GCP_REGION=europe-west2  # GCP region to deploy to (default is europe-west2)
+TF_VAR_ZONE=europe-west2-a # Zone to provision the GCP compute resources
+TF_VAR_SUBNET_CIDR_RANGE=10.2.0.0/16
+TF_VAR_FW_RULE_SUFFIX=matic # Suffix to be used in firewall rule name.
+TF_VAR_INSTANCE_IMAGE=ubuntu-2204-jammy-v20230302 #  instance image os (default is ubuntu-2204-jammy-v20230302, running ubuntu 22.04)
+TF_VAR_GCE_PUB_KEY_FILE=/absolute/path/to/your/gcp-key.pub # absolute path pointing to the public key file
+TF_VAR_USER=ubuntu 
+TF_VAR_BOR_MACHINE_TYPE=n2d-standard-4  # type of the EC2 VM instance for Bor client (default is n2d-standard-4)
+TF_VAR_ERIGON_MACHINE_TYPE=n2d-standard-4  # type of the EC2 VM instance for Erigon client (default is n2d-standard-4)
+TF_VAR_BOR_ARCHIVE_MACHINE_TYPE=n2d-standard-4  # default n2d-standard-4 
+TF_VAR_ERIGON_ARCHIVE_MACHINE_TYPE=n2d-standard-4  # default n2d-standard-4
+TF_VAR_BOR_VOLUME_TYPE_GCP=pd-ssd # default pd-ssd
+TF_VAR_ERIGON_VOLUME_TYPE_GCP=pd-ssd # default pd-ssd
+TF_VAR_BOR_ARCHIVE_VOLUME_TYPE_GCP=pd-balanced # type of EBS volume for Bor archive nodes (default is pd-balanced)
+TF_VAR_ERIGON_ARCHIVE_VOLUME_TYPE_GCP=pd-balanced # type of EBS volume for Erigon archive nodes (default is pd-balanced)
+
+# Terraform variables - Common for both AWS and GCP
+TF_VAR_VM_NAME=polygon-user # default "polygon-user". It can be any string, used to discriminate between instances
+TF_VAR_DOCKERIZED=no # default "no", otherwise only one VM is created and the Polygon devnet will run in docker containers
+TF_VAR_BOR_DISK_SIZE_GB=20 # size of the disk in GB for Bor client (default is 100GB)
+TF_VAR_ERIGON_DISK_SIZE_GB=20 # size of the disk in GB for Erigon client (default is 100GB)
+TF_VAR_BOR_ARCHIVE_DISK_SIZE_GB=100 # size of the disk in GB in Bor archive node (default is 100GB)
+TF_VAR_ERIGON_ARCHIVE_DISK_SIZE_GB=100 # size of the disk in GB in Erigon archive node (default is 100GB)
+TF_VAR_BOR_VALIDATOR_COUNT=2 # number of Bor validator nodes (default is 2). Note that while spinning up a public network (mainnet/mumbai) node, this will serve as a non-validator
+TF_VAR_BOR_SENTRY_COUNT=1 # number of non-validator Bor sentry nodes (default is 1)
+TF_VAR_BOR_ARCHIVE_COUNT=0 # number of Bor archive nodes (default is 0)
+TF_VAR_ERIGON_VALIDATOR_COUNT=0 # number of Erigon validator nodes (default is 0). Note that while spinning up a public network (mainnet/mumbai) node, this will serve as a non-validator
+TF_VAR_ERIGON_SENTRY_COUNT=0 # number of non-validator Erigon sentry nodes (default is 0)
+TF_VAR_ERIGON_ARCHIVE_COUNT=0 # number of Erigon archive nodes (default is 0)
 PEM_FILE_PATH=/absolute/path/to/your/aws-key.pem # absolute path pointing to the certificate previously downloaded
 
 # Polygon network based variables (see configs/README.md) for more detailed info
@@ -60,7 +80,9 @@ VERBOSE=true # if set to true will print logs also from remote machines
 DD_API_KEY=DATADOG_API_KEY # Datadog API key
 
 # Bor, Erigon and Heimdall Snapshot related variables (when running mainnet/testnet node)
-SNAPSHOT=true # If set true, bor/erigon and heimdall snapshots will be downloaded
+BOR_SNAPSHOT_URL=https://snapshot-download.polygon.technology/bor-mumbai-fullnode-2023-03-18.tar.zst # URL for bor data snapshot (refer: https://snapshot.polygon.technology/)
+HEIMDALL_SNAPSHOT_URL=https://snapshot-download.polygon.technology/heimdall-mumbai-fullnode-2023-03-18.tar.zst # URL for heimdall data snapshot (refer: https://snapshot.polygon.technology/)
+ERIGON_SNAPSHOT_URL= # URL for Erigon data snapshot (refer: https://snapshot.polygon.technology/)
 
 #Stress test variables (used to run stress tests against the remote nodes)
 MNEMONIC="clock radar mass judge dismiss just intact mind resemble fringe diary casino" #random mnemonic

README.md

@@ -21,16 +21,16 @@ Please, refer to the section of this file you are more interested in (`express-c
 
 To use the `express-cli` you have to execute the following steps.
 
-- [install aws cli](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html)
+- [install aws cli](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html) or [install gcloud tool](https://cloud.google.com/sdk/docs/install)
 - [install terraform](https://learn.hashicorp.com/tutorials/terraform/install-cli) on your local machine
 - use [nvm](https://github.com/nvm-sh/nvm#installing-and-updating) to switch to the proper `node` version, `v16.17.1`,
   by running `nvm use` from the root folder
 - install `express-cli` and `matic-cli` locally with command `npm i`
-- generate a keypair on AWS EC2 and download its certificate locally (`.pem` file)
+- generate a keypair on AWS EC2 and download its certificate locally (`.pem` file). If you are on GCP, you can use your existing keypair or use `ssh-keygen` to generate. Check the [GCP guide](./docs/gcp_dev_guide.md).
 - copy `secret.tfvars.example` to `secret.tfvar` with command `cp secret.tfvars.example secret.tfvars` and check the commented file for details
 - **If you are a Polygon employee**, connect to the company VPN
 - modify `secret.tfvar` with addresses of the allowed IPs (as specified in `secret.tfvars.example` file)
-- copy `.env.example` to `.env` with command `cp .env.example .env` and check the heavily commented file for details
+- copy `.env.example` to `.env` with command `cp .env.example .env` and check the heavily commented file for details. If you're using GCP, you can ignore AWS specific terraform varibales and vice versa.
 - make sure `PEM_FILE_PATH` points to a correct AWS key certificate, the one you downloaded in the previous steps
 - define the number of nodes (`TF_VAR_VALIDATOR_COUNT` and `TF_VAR_SENTRY_COUNT`) and adjust the `DEVNET_BOR_USERS`
   accordingly
@@ -41,6 +41,9 @@ To use the `express-cli` you have to execute the following steps.
   be shown
 - **If you are a Polygon employee**, please refer to [this page](https://www.notion.so/polygontechnology/Testing-Toolkit-d47e098641d14c80b2e9a90b3b1b88d9) for more info
 
+In case you plan to utilize express-cli for Google Cloud, you will need to make few modifications like SSH keys. It's important to note that express-cli is not fully tested yet on GCP, and not all features are accessible. Check the [GCP dev guide](./docs/gcp_dev_guide.md).
+
+
 ### Auth Configuration
 
 As a prerequisite, you need to configure authentication on `aws`  
@@ -111,19 +114,32 @@ aws sso login
 
 Congrats! You're all set to use `express-cli` commands.
 
+If you are using Google cloud platform, you need to configure authentication on `gcloud`.
+
+If you have downloaded the service account credentials, you can use the `GOOGLE_APPLICATION_CREDENTIALS` environment variable to provide the location of thst credential JSON file. 
+
+```bash
+gcloud auth application-default login
+
+# OR
+
+export GOOGLE_APPLICATION_CREDENTIALS='/absolute/path/to/sa/creds.json'
+```
+
 ### Commands
 
 Instructions to run `express-cli`.
 For the list of commands, please run `express-cli --help`
 First off, you need to `--init` terraform on your local machine, by executing the following command.
 
-- `./bin/express-cli --init`
+- `./bin/express-cli --init <aws|gcp>`
 
   - Initializes a new devnet folder with terraform and creates some git-ignored files locally. This step is mandatory
     before running any other command. The new devnet folder created will be `devnet-<id>` where `id` is a monotonically
     increasing count for the devnets. Once created, you can `cd deployments/devnet-<id>` and run the other commands.
     This allows you to work with multiple devnets at once.
     Then, a remote devnet can be created with the `--start` command, as follows.
+  - You should specify the cloud provider. Currently the supported values are `aws` and `gcp`.
 
 - `../../bin/express-cli --start`
 
@@ -215,11 +231,11 @@ The `express-cli` also comes with additional utility commands, listed below. Som
 
 - ` ../../bin/express-cli --instances-stop`
 
-  - Stop the AWS EC2 VM instances associated with the deployed devnet.
+  - Stop the cloud VM instances associated with the deployed devnet.
 
 - ` ../../bin/express-cli --instances-start`
 
-  - Start the (previously stopped) AWS EC2 VM instances associated with the deployed devnet. Also, it starts all services, such as ganache, heimdall, and bor
+  - Start the (previously stopped) VM instances associated with the deployed devnet. Also, it starts all services, such as ganache, heimdall, and bor
 
 - `../../bin/express-cli --stress [fund]`
 
@@ -247,13 +263,13 @@ The `express-cli` also comes with additional utility commands, listed below. Som
   - Executes a test to send EIP 1559 tx. In case of a non-dockerized devnet, if an integer [index] is specified, it will use
     that VM to send the tx. Otherwise, it will target the first VM.
 
-- `../../bin/express-cli --aws-key-add`
+- `../../bin/express-cli --ssh-key-add`
 
-  - Generates an additional `aws` key-pair remotely and stores it locally in the devnet folder. The public key is added to the ssh authorized keys of the devnet's machines. The key can be shared - on a secure channel! - with other devs to grant them access to the remote devnet.
+  - Generates an additional ssh key-pair remotely and stores it locally in the devnet folder. The public key is added to the ssh authorized keys of the devnet's machines. The key can be shared - on a secure channel! - with other devs to grant them access to the remote devnet.
 
-- `../../bin/express-cli --aws-key-des [keyName]`
+- `../../bin/express-cli --ssh-key-des [keyName]`
 
-  - Destroys an `aws` key-pair given its `keyName`. The key gets deleted remotely from `aws`, cancelled from the authorized ssh keys of the devnet's machines and removed from local devnet folder.
+  - Destroys an ssh key-pair given its `keyName`. The key gets deleted remotely from `aws` or `gcp`, cancelled from the authorized ssh keys of the devnet's machines and removed from local devnet folder.
 
 - `../../bin/express-cli --reorg-start [split]`
 

docs/gcp_dev_guide.md

@@ -0,0 +1,26 @@
+# express-cli on GCP
+
+* Make to you have installed `gcloud` tool for authentication
+* You also need `ssh-keygen` tool. It's mostly available in all distributions.
+* You should have a public and private keypair for running ssh commands. You can generate one using following commands
+    ```sh
+    ssh-keygen -f ~/ubuntu.pem -N ""
+    ```
+The above command will generates the public key named _ubuntu.pem.pub_ and the private key named _ubuntu.pem_ in home directory.
+* In `.env` file, you have update with correct details. You can ignore AWS specific terraform variables.
+
+## GCP - SSH configuration
+- If you don't have a keypair, run the following command to generate an SSH key pair: `ssh-keygen -f ubuntu.pem -N ""`
+- The above command generates two files. _ubuntu.pem_ (private keyfile) and _ubuntu.pem.pub_ (public key)
+- These two files have to be updated in _.env_ file.
+- Locate the `TF_VAR_GCE_PUB_KEY_FILE` variable in .env file and set its value to the absolute path pointing to the public key file you generated.
+- Locate the `PEM_FILE_PATH` variable in .env file and set its value to the absolute path pointing to the private key file you generated.
+
+## Resource Naming Convention
+- GCP identifies the resources using names. So every resource should have a unique name.
+- When deploying resources in GCP, we are using `TF_VAR_VM_NAME` names as prefix for each resource to ensure uniqueness and avoid conflicts.
+- So for every devnet you provision, make sure to update the `TF_VAR_VM_NAME` in _.env_.
+
+
+
+

src/express-cli.js

@@ -39,12 +39,22 @@ import { milestoneBase } from './express/commands/milestone-base'
 import { milestonePartition } from './express/commands/milestone-partition'
 import { shadow } from './express/commands/shadow'
 import { relay } from './express/commands/relay'
-import { awsKeypairAdd } from './express/commands/aws-keypair-add'
-import { awsKeypairDestroy } from './express/commands/aws-keypair-destroy'
+import { keypairAdd } from './express/commands/keypair-add'
+import { keypairDestroy } from './express/commands/keypair-destroy'
 import { rpcTest } from '../tests/rpc-tests/rpc-test'
+import constants from './express/common/constants'
+
+function checkCloudProvider(provider, _) {
+  const supportedClouds = [constants.cloud.AWS, constants.cloud.GCP];
+  if (supportedClouds.includes(provider.toLowerCase())) {
+    return provider.toLowerCase()
+  }
+  console.log('❌Invalid cloud provider. Choose from: ' + supportedClouds)
+  process.exit(1)
+}
 
 program
-  .option('-i, --init', 'Initiate the terraform setup')
+  .option('-i, --init <gcp|aws>', 'Initiate the terraform setup to specified cloud', checkCloudProvider)
   .option('-s, --start', 'Start the setup')
   .option('-d, --destroy', 'Destroy the setup')
   .option(
@@ -134,12 +144,12 @@ program
     'Rewind the chain by a given number of blocks'
   )
   .option(
-    '-key-a, --aws-key-add',
-    'Generate additional aws keypair for the devnet'
+    '-key-a, --ssh-key-add',
+    'Generate additional ssh keypair for the devnet'
   )
   .option(
-    '-key-d, --aws-key-des [keyName]',
-    'Destroy aws keypair from devnet, given its keyName'
+    '-key-d, --ssh-key-des [keyName]',
+    'Destroy ssh keypair from devnet, given its keyName'
   )
   .option(
     '-sf, --shadow-fork [blockNumber]',
@@ -157,16 +167,15 @@ export async function cli() {
 
   program.parse(process.argv)
   const options = program.opts()
-
   if (options.init) {
-    console.log('📍Command --init')
+    console.log('📍Command --init ' + options.init)
     if (!checkDir(true)) {
       console.log(
         "❌ The init command is supposed to be executed from the project root directory, named 'matic-cli'!"
       )
       process.exit(1)
     }
-    await terraformInit()
+    await terraformInit(options.init)
   } else if (options.start) {
     console.log('📍Command --start')
     if (!checkDir(false)) {
@@ -509,26 +518,26 @@ export async function cli() {
 
     await timer(3000)
     await rewind(options.rewind)
-  } else if (options.awsKeyAdd) {
-    console.log('📍 Command --aws-key-add')
+  } else if (options.sshKeyAdd) {
+    console.log('📍 Command --ssh-key-add')
     if (!checkDir(false)) {
       console.log(
         '❌ The command is not called from the appropriate devnet directory!'
       )
       process.exit(1)
     }
 
-    await awsKeypairAdd()
-  } else if (options.awsKeyDes) {
-    console.log('📍 Command --aws-key-des')
+    await keypairAdd()
+  } else if (options.sshKeyDes) {
+    console.log('📍 Command --ssh-key-des')
     if (!checkDir(false)) {
       console.log(
         '❌ The command is not called from the appropriate devnet directory!'
       )
       process.exit(1)
     }
 
-    await awsKeypairDestroy(options.awsKeyDes)
+    await keypairDestroy(options.sshKeyDes)
   } else if (options.reorgStart) {
     console.log('📍Command --reorg-start [split]')
 

src/express/commands/aws-instances-start.js

@@ -4,6 +4,7 @@ import { loadDevnetConfig } from '../common/config-utils'
 import { restartAll } from './restart'
 import { maxRetries, runSshCommand } from '../common/remote-worker'
 import { timer } from '../common/time-utils'
+import constants from '../common/constants'
 
 const shell = require('shelljs')
 
@@ -26,9 +27,22 @@ export async function startInstances() {
   const devnetType =
     process.env.TF_VAR_DOCKERIZED === 'yes' ? 'docker' : 'remote'
   const doc = await loadDevnetConfig(devnetType)
-  const instances = doc.instancesIds.toString().replace(/,/g, ' ')
+  const cloud = doc.cloud.toString()
 
-  shell.exec(`aws ec2 start-instances --instance-ids ${instances}`)
+  if (cloud === constants.cloud.GCP) {
+    const project = doc.instancesIds[0].split('/')[1].toString()
+    const zone = doc.instancesIds[0].split('/')[3].toString()
+    const instances = doc.instancesIds.map(x => x.split('/').at(-1)).toString().replace(/,/g, ' ')
+    shell.exec(`gcloud compute instances start ${instances} --zone ${zone} --project ${project}`)
+  } else if (cloud === constants.cloud.AWS ) {
+    const instances = doc.instancesIds.toString().replace(/,/g, ' ')
+    const region = doc.devnetRegion.toString()
+    shell.exec(`aws ec2 start-instances  --region ${region} --instance-ids ${instances}`)
+  } else {
+    console.log(`❌ Unsupported cloud provider ${cloud}`)
+    process.exit(1);
+  }
+
   if (shell.error() !== null) {
     console.log(
       `📍Starting instances ${doc.instancesIds.toString()} didn't work. Please check AWS manually`