From 93000cdaf72510631d5168a7d988ec0c963d19e6 Mon Sep 17 00:00:00 2001 From: Nicolas Ochem Date: Mon, 10 Jan 2022 17:28:46 -0800 Subject: [PATCH 1/4] delete old state --- .../Dockerfile.template | 5 - .../polkadot-archive-downloader/entrypoint.sh | 27 ---- .../Dockerfile.template | 13 -- .../entrypoint.sh | 30 ---- .../polkadot-private-node/Dockerfile.template | 13 -- docker/polkadot-private-node/entrypoint.sh | 50 ------ docs/pre-existing-cluster.md | 9 -- docs/production-hardening.md | 33 ---- .../empty_module/empty.tf | 52 ------ terraform-no-cluster-create/gcp.tf | 17 -- terraform-no-cluster-create/k8s.tf | 1 - terraform-no-cluster-create/k8s_override.tf | 5 - terraform-no-cluster-create/variables.tf | 1 - terraform/gcp.tf | 23 --- terraform/k8s.tf | 145 ----------------- terraform/variables.tf | 148 ------------------ 16 files changed, 572 deletions(-) delete mode 100644 docker/polkadot-archive-downloader/Dockerfile.template delete mode 100755 docker/polkadot-archive-downloader/entrypoint.sh delete mode 100644 docker/polkadot-node-key-configurator/Dockerfile.template delete mode 100755 docker/polkadot-node-key-configurator/entrypoint.sh delete mode 100644 docker/polkadot-private-node/Dockerfile.template delete mode 100755 docker/polkadot-private-node/entrypoint.sh delete mode 100644 docs/pre-existing-cluster.md delete mode 100644 docs/production-hardening.md delete mode 100644 terraform-no-cluster-create/empty_module/empty.tf delete mode 100644 terraform-no-cluster-create/gcp.tf delete mode 120000 terraform-no-cluster-create/k8s.tf delete mode 100644 terraform-no-cluster-create/k8s_override.tf delete mode 120000 terraform-no-cluster-create/variables.tf delete mode 100644 terraform/gcp.tf delete mode 100644 terraform/k8s.tf delete mode 100644 terraform/variables.tf diff --git a/docker/polkadot-archive-downloader/Dockerfile.template b/docker/polkadot-archive-downloader/Dockerfile.template deleted file mode 100644 index 85e2d89..0000000 --- a/docker/polkadot-archive-downloader/Dockerfile.template +++ /dev/null @@ -1,5 +0,0 @@ -FROM alpine -RUN apk add curl p7zip bash -COPY entrypoint.sh / -ENTRYPOINT ["/entrypoint.sh"] -CMD [] diff --git a/docker/polkadot-archive-downloader/entrypoint.sh b/docker/polkadot-archive-downloader/entrypoint.sh deleted file mode 100755 index 462ad98..0000000 --- a/docker/polkadot-archive-downloader/entrypoint.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash - -set -e - -if [ "${CHAIN}" == "polkadot" ]; then - chain_dir=polkadot -else - chain_dir=ksmcc3 -fi - -if [ -d /polkadot/.local/share/polkadot/chains/${chain_dir}/db/ ]; then - echo "Blockchain database already exists, no need to import, exiting" - exit 0 -elif [ -z "$ARCHIVE_URL" ]; then - echo "No archive download url specified, exiting" - exit 0 -else - echo "Did not find pre-existing data, importing blockchain" - mkdir -p /polkadot/.local/share/polkadot/chains/${chain_dir}/ - echo "Will download $ARCHIVE_URL" - curl -L $ARCHIVE_URL -o /polkadot/polkadot_archive.7z - 7z x /polkadot/polkadot_archive.7z -o/polkadot/.local/share/polkadot/chains/${chain_dir} - rm -v /polkadot/polkadot_archive.7z - chmod -R 777 /polkadot/.local/ - chown -R 1000:1000 /polkadot/.local/ - find /polkadot/.local/share/ -fi diff --git a/docker/polkadot-node-key-configurator/Dockerfile.template b/docker/polkadot-node-key-configurator/Dockerfile.template deleted file mode 100644 index 5223a57..0000000 --- a/docker/polkadot-node-key-configurator/Dockerfile.template +++ /dev/null @@ -1,13 +0,0 @@ -FROM parity/subkey:2.0.0 -USER root -# install tools and dependencies -RUN apt-get update --allow-insecure-repositories && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y \ - xxd && \ - apt-get autoremove -y && \ - apt-get clean && \ - find /var/lib/apt/lists/ -type f -not -name lock -delete; -COPY entrypoint.sh / -ENTRYPOINT ["/entrypoint.sh"] -CMD [] - diff --git a/docker/polkadot-node-key-configurator/entrypoint.sh b/docker/polkadot-node-key-configurator/entrypoint.sh deleted file mode 100755 index 6f06b19..0000000 --- a/docker/polkadot-node-key-configurator/entrypoint.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash - -set -e -set -x - -rm -vf /polkadot/k8s_local_node_key -rm -rvf /polkadot/k8s_node_ids/ -mkdir -p /polkadot/k8s_node_ids - -# write private key for this node only and protect it -cat /polkadot-node-keys/${KUBERNETES_NAME_PREFIX} | xxd -r -p > /polkadot/k8s_local_node_key -# move owner to polkadot -chown 1000 /polkadot/k8s_local_node_key -chmod 400 /polkadot/k8s_local_node_key - - -# write public keys for all peers in an env file, to be sourced by polkadot startup script -local_peers=("${LOCAL_PEERS}") -rm -rvf /polkadot/k8s_local_peer_cmd -for node in ${local_peers[@]} -do - # do not peer with myself - if [ "${node}" != "${KUBERNETES_NAME_PREFIX}" ] - then - if [ ! -f /polkadot/k8s_local_peer_cmd ]; then - echo "--reserved-nodes " > /polkadot/k8s_local_peer_cmd - fi - echo "/dns4/${node}-private-node-0.${node}-private-node.${node}/tcp/30333/p2p/$(subkey inspect-node-key --file /polkadot-node-keys/$node) " >> /polkadot/k8s_local_peer_cmd - fi -done diff --git a/docker/polkadot-private-node/Dockerfile.template b/docker/polkadot-private-node/Dockerfile.template deleted file mode 100644 index 7f310e4..0000000 --- a/docker/polkadot-private-node/Dockerfile.template +++ /dev/null @@ -1,13 +0,0 @@ -FROM parity/polkadot:((polkadot_version)) -USER root -# install tools and dependencies -RUN apt-get update --allow-insecure-repositories && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y \ - jq xxd && \ - apt-get autoremove -y && \ - apt-get clean && \ - find /var/lib/apt/lists/ -type f -not -name lock -delete; -USER polkadot -COPY entrypoint.sh / -ENTRYPOINT ["/entrypoint.sh"] -CMD [] diff --git a/docker/polkadot-private-node/entrypoint.sh b/docker/polkadot-private-node/entrypoint.sh deleted file mode 100755 index c2e3391..0000000 --- a/docker/polkadot-private-node/entrypoint.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/bash - -set -e -set -x - -if [ -e /polkadot/k8s_local_node_key ]; then - node_key_param="--node-key-file /polkadot/k8s_local_node_key" -fi - -if [ -e /polkadot/k8s_local_peer_cmd ]; then - local_peer_param="$(cat /polkadot/k8s_local_peer_cmd)" -fi - -if [ ! -z "$VALIDATOR_NAME" ]; then - name_param="--name \"$VALIDATOR_NAME\"" -fi - -if [ ! -z "$CHAIN" ]; then - chain_param="--chain \"$CHAIN\"" -fi - -if [ ! -z "$IN_PEERS" ]; then - in_peers_param="--in-peers=${IN_PEERS}" -fi - -if [ ! -z "$OUT_PEERS" ]; then - out_peers_param="--out-peers=${OUT_PEERS}" -fi - -if [ ! -z "$TELEMETRY_URL" ]; then - telemetry_url_param="--telemetry-url \"$TELEMETRY_URL 0\"" -fi - -if [ ! -z "$PUBLIC_MULTIADDR" ]; then - public_address_param="--public-addr=${PUBLIC_MULTIADDR}" -fi - -# unsafe flags are due to polkadot panic alerter needing to connect to the node with rpc -eval /usr/bin/polkadot --validator --wasm-execution Compiled \ - --unsafe-pruning \ - --pruning=1000 \ - --prometheus-external \ - $out_peers_param \ - $in_peers_param \ - $node_key_param \ - $name_param \ - $telemetry_url_param \ - $chain_param \ - $public_address_param \ - $local_peer_param diff --git a/docs/pre-existing-cluster.md b/docs/pre-existing-cluster.md deleted file mode 100644 index 04c0229..0000000 --- a/docs/pre-existing-cluster.md +++ /dev/null @@ -1,9 +0,0 @@ -# Pre-existing kubernetes cluster - -You may want to deploy the validator setup in a kubernetes cluster that already exists. - -In that case: - -* set the `kubernetes_config_context` variable to the context of your target cluster. To list local contexts, do `kubectl config get-contexts` or look at `~/.kube/config` -* set the `project` variable to the GCP project where the cluster is located -* `terraform init` / `terraform plan` / `terraform apply` diff --git a/docs/production-hardening.md b/docs/production-hardening.md deleted file mode 100644 index f2fe627..0000000 --- a/docs/production-hardening.md +++ /dev/null @@ -1,33 +0,0 @@ -# Production hardening - -Follow these instructions for production hardening of the deployment. - -## Terraform service account - -Usage of [Google Default Application Credentials](https://cloud.google.com/docs/authentication/production) is not recommended in a production environment. - -Instead: - -* ensure that you have set up an Organization - that can be done by registering a domain name and adding it to gcloud -* create a Terraform Admin Project, Terraform Service Account and Service Account Credentials following [this Google guide](https://cloud.google.com/community/tutorials/managing-gcp-projects-with-terraform) -* do not pass `project` as a variable when deploying the resources. Instead, pass `org_id` and `billing_account` as variables -* pass the service account credentials json file `serviceAccount:terraform@${TF_ADMIN}.iam.gserviceaccount.com` as `terraform_service_account_credentials` terraform variable - -This is [how to download the service account credentials json file](https://cloud.google.com/iam/docs/creating-managing-service-account-keys). - -That will create the cluster in a new project, created by the terraform service account. - -You may then grant people in your organization access to the project. It is recommended to write more terraform manifests to do so. - -## Multiple operators - -The repository is optimized for quick spinup with one operator: secrets are stored locally in a `terraform.tfvars` file. - -A production validator should be operated with an on-call rotation, meaning several operators have access to the setup. - -Specifically: - -* secrets should be moved from a file in the operator workspace to a production secret store such as [Hashicorp Vault](vaultproject.io) -* terraform state should be stored centrally (in a google storage bucket) -* terraform deploys should be done by a CI system -* any manual change in the kubernetes environment should be recorded in an audit log and committed in the code (see [Gitops](https://www.weave.works/technologies/gitops/)). diff --git a/terraform-no-cluster-create/empty_module/empty.tf b/terraform-no-cluster-create/empty_module/empty.tf deleted file mode 100644 index 3082b87..0000000 --- a/terraform-no-cluster-create/empty_module/empty.tf +++ /dev/null @@ -1,52 +0,0 @@ -# An empty module. -# We do not want cluster creation to take place, so this is a placeholder for the module that creates a cluster. - -variable "project" { - type = string - description = "project name" - default = "" -} - -variable "region" { - type = string -} - -variable "kubernetes_endpoint" { - type = string -} - -variable "cluster_ca_certificate" { - type = string -} - -variable "kubernetes_access_token" { - type = string -} - -variable "cluster_name" { - type = string -} - -output "name" { - value = var.cluster_name -} - -output "kubernetes_endpoint" { - value = var.kubernetes_endpoint -} - -output "cluster_ca_certificate" { - value = var.cluster_ca_certificate -} - -output "kubernetes_access_token" { - value = var.kubernetes_access_token -} - -output "location" { - value = var.region -} - -output "project" { - value = var.project -} diff --git a/terraform-no-cluster-create/gcp.tf b/terraform-no-cluster-create/gcp.tf deleted file mode 100644 index f2d0106..0000000 --- a/terraform-no-cluster-create/gcp.tf +++ /dev/null @@ -1,17 +0,0 @@ -# This file contains all the interactions with Kubernetes -provider "kubernetes" { - host = module.terraform-gke-blockchain.kubernetes_endpoint - cluster_ca_certificate = module.terraform-gke-blockchain.cluster_ca_certificate - token = module.terraform-gke-blockchain.kubernetes_access_token -} - - -module "terraform-gke-blockchain" { - source = "./empty_module" - project = var.project - region = var.region - kubernetes_endpoint = var.kubernetes_endpoint - cluster_ca_certificate = var.cluster_ca_certificate - cluster_name = var.cluster_name - kubernetes_access_token = var.kubernetes_access_token -} diff --git a/terraform-no-cluster-create/k8s.tf b/terraform-no-cluster-create/k8s.tf deleted file mode 120000 index da8b465..0000000 --- a/terraform-no-cluster-create/k8s.tf +++ /dev/null @@ -1 +0,0 @@ -../terraform/k8s.tf \ No newline at end of file diff --git a/terraform-no-cluster-create/k8s_override.tf b/terraform-no-cluster-create/k8s_override.tf deleted file mode 100644 index 735d511..0000000 --- a/terraform-no-cluster-create/k8s_override.tf +++ /dev/null @@ -1,5 +0,0 @@ -resource "null_resource" "push_containers" { - - triggers = { - } -} diff --git a/terraform-no-cluster-create/variables.tf b/terraform-no-cluster-create/variables.tf deleted file mode 120000 index 2e8fc16..0000000 --- a/terraform-no-cluster-create/variables.tf +++ /dev/null @@ -1 +0,0 @@ -../terraform/variables.tf \ No newline at end of file diff --git a/terraform/gcp.tf b/terraform/gcp.tf deleted file mode 100644 index 5ed2e81..0000000 --- a/terraform/gcp.tf +++ /dev/null @@ -1,23 +0,0 @@ -module "terraform-gke-blockchain" { - source = "github.com/midl-dev/terraform-gke-blockchain" - org_id = var.org_id - billing_account = var.billing_account - terraform_service_account_credentials = var.terraform_service_account_credentials - project = var.project - project_prefix = "polkadot" - region = var.region - node_locations = var.node_locations - monitoring_slack_url = var.monitoring_slack_url -} - -# Query the client configuration for our current service account, which should -# have permission to talk to the GKE cluster since it created it. -data "google_client_config" "current" { -} - -# This file contains all the interactions with Kubernetes -provider "kubernetes" { - host = module.terraform-gke-blockchain.kubernetes_endpoint - cluster_ca_certificate = module.terraform-gke-blockchain.cluster_ca_certificate - token = data.google_client_config.current.access_token -} diff --git a/terraform/k8s.tf b/terraform/k8s.tf deleted file mode 100644 index 2d7a138..0000000 --- a/terraform/k8s.tf +++ /dev/null @@ -1,145 +0,0 @@ -resource "null_resource" "push_containers" { - - triggers = { - host = md5(module.terraform-gke-blockchain.kubernetes_endpoint) - cluster_ca_certificate = md5( - module.terraform-gke-blockchain.cluster_ca_certificate, - ) - } - provisioner "local-exec" { - interpreter = [ "/bin/bash", "-c" ] - command = < cloudbuild.yaml -steps: -- name: 'gcr.io/cloud-builders/docker' - args: ['build', '-t', "gcr.io/${module.terraform-gke-blockchain.project}/$container:${var.kubernetes_namespace}-latest", '.'] -images: ["gcr.io/${module.terraform-gke-blockchain.project}/$container:${var.kubernetes_namespace}-latest"] -EOY - gcloud builds submit --project ${module.terraform-gke-blockchain.project} --config cloudbuild.yaml . - rm -v Dockerfile - rm cloudbuild.yaml -} -export -f build_container -find ${path.module}/../docker -mindepth 1 -maxdepth 1 -type d -exec bash -c 'build_container "$0"' {} \; -printf '%f\n' -EOF - } -} - -# generate node keys if they are not passed as parameters -# conventiently, ed25519 is happy with random bytes as private key -# unfortunately, terraform does not support generation of sensitive hex data, so we have -# to hack the "random_password" resource to generate a hex -resource "random_password" "private-node-0-key" { - count = contains(keys(var.polkadot_node_keys), "polkadot-private-node-0") ? 0 : 1 - length = 64 - override_special = "abcdef1234567890" - upper = false - lower = false - number = false -} - -resource "kubernetes_namespace" "polkadot_namespace" { - metadata { - name = var.kubernetes_namespace - } - depends_on = [ module.terraform-gke-blockchain ] -} - -# FIXME this is a bug in kustomize where it will not prepend characters to the storageClass requirement -# to address it, we define it here. At some point, later, it will stop being needed. -resource "kubernetes_storage_class" "local-ssd" { - count = var.kubernetes_name_prefix == "dot" ? 1 : 0 - metadata { - name = "local-ssd" - } - storage_provisioner = "kubernetes.io/gce-pd" - parameters = { - type = "pd-ssd" - } - depends_on = [ kubernetes_namespace.polkadot_namespace ] -} - -# FIXME this is a bug in kustomize where it will not prepend characters to the storageClass requirement -# to address it, we define it here. At some point, later, it will stop being needed. -resource "kubernetes_storage_class" "repd-europe-west1-b-d" { - count = var.kubernetes_name_prefix == "dot" ? 1 : 0 - metadata { - name = "repd-europe-west1-b-d" - } - storage_provisioner = "kubernetes.io/gce-pd" - parameters = { - type = "pd-ssd" - replication-type = "regional-pd" - zones = "europe-west1-b, europe-west1-d" - } - depends_on = [ kubernetes_namespace.polkadot_namespace ] -} - -resource "kubernetes_secret" "polkadot_node_keys" { - metadata { - name = "polkadot-node-keys" - namespace = var.kubernetes_namespace - } - data = var.polkadot_node_keys - depends_on = [ kubernetes_namespace.polkadot_namespace ] -} - -resource "null_resource" "apply" { - provisioner "local-exec" { - - interpreter = [ "/bin/bash", "-c" ] - command = < kustomization.yaml -${templatefile("${path.module}/../k8s/kustomization.yaml.tmpl", - { "project" : module.terraform-gke-blockchain.project, - "polkadot_archive_url": var.polkadot_archive_url, - "polkadot_telemetry_url": var.polkadot_telemetry_url, - "polkadot_validator_name": var.polkadot_validator_name, - "chain": var.chain, - "out_peers": var.out_peers, - "in_peers": var.in_peers, - "local_peers": join(" ", keys(var.polkadot_node_keys)), - "p2p_port": var.p2p_port, - "p2p_ip": var.p2p_ip, - "kubernetes_namespace": var.kubernetes_namespace, - "kubernetes_name_prefix": var.kubernetes_name_prefix})} -EOK -cat < prefixedpv.yaml -${templatefile("${path.module}/../k8s/prefixedpv.yaml.tmpl", - { "kubernetes_name_prefix": var.kubernetes_name_prefix, - "storage_size": var.storage_size})} -EOK -cat < regionalpvpatch.yaml -${templatefile("${path.module}/../k8s/regionalpvpatch.yaml.tmpl", - { "regional_pd_zones" : join(", ", var.node_locations), - "kubernetes_name_prefix": var.kubernetes_name_prefix})} -EORPP -cat < nodepool.yaml -${templatefile("${path.module}/../k8s/nodepool.yaml.tmpl", {"kubernetes_pool_name": var.kubernetes_pool_name})} -EONPN -cat < lb_ip_patch.yaml -${templatefile("${path.module}/../k8s/lb_ip_patch.yaml.tmpl", {"p2p_ip": var.p2p_ip, "p2p_port": var.p2p_port})} -EOLIA -kubectl apply -k . -popd -rm -rvf ${path.module}/k8s-${var.kubernetes_namespace} -EOF - - } - depends_on = [ null_resource.push_containers, kubernetes_namespace.polkadot_namespace ] -} diff --git a/terraform/variables.tf b/terraform/variables.tf deleted file mode 100644 index 1796994..0000000 --- a/terraform/variables.tf +++ /dev/null @@ -1,148 +0,0 @@ -terraform { - required_version = ">= 0.12" -} - -variable "chain" { - type = string - description = "The chain (can be polkadot, kusama)" - default = "polkadot" -} - -variable "polkadot_archive_url" { - type = string - description = "archive url" - default = "https://dot-rocksdb.polkashots.io/snapshot" -} - -variable "polkadot_telemetry_url" { - type = string - description = "url of the telemetry server the polkadot nodes report to" -} - -variable "in_peers" { - type = string - description = "number of incoming connections to be established from remote peers" - default = "5" -} - -variable "out_peers" { - type = string - description = "number of outgoing connections to be established with remote peers" - default = "" -} - -variable "polkadot_validator_name" { - type = string - description = "name of the validator shown on the public telemetry server" -} - -variable "polkadot_node_keys" { - type = map - description = "map between hostname of polkadot nodes and their node keys" - default = {} -} - -variable "polkadot_version" { - type = string - description = "Version of the polkadot containers to use" -} - -variable "project" { - type = string - default = "" - description = "Project ID where Terraform is authenticated to run to create additional projects. If provided, Terraform will create the GKE cluster inside this project. If not given, Terraform will generate a new project." -} - -variable "org_id" { - type = string - description = "Organization ID." - default = "" -} - -variable "region" { - type = string - description = "GCP Region. Only necessary when creating cluster manually" - default = "us-central1" -} - -variable "node_locations" { - type = list - default = [ "us-central1-b", "us-central1-f" ] - description = "Zones in which to create the nodes" -} - -variable "billing_account" { - type = string - description = "Billing account ID." - default = "" -} - -variable "kubernetes_namespace" { - type = string - description = "kubernetes namespace to deploy the resource into" - default = "polkadot" -} - -variable "kubernetes_name_prefix" { - type = string - description = "kubernetes name prefix to prepend to all resources (should be short, like DOT)" - default = "dot" -} - -variable "kubernetes_endpoint" { - type = string - description = "name of the kubernetes endpoint" - default = "" -} - -variable "cluster_ca_certificate" { - type = string - description = "kubernetes cluster certificate" - default = "" -} - -variable "cluster_name" { - type = string - description = "name of the kubernetes cluster" - default = "" -} - -variable "kubernetes_access_token" { - type = string - description = "name of the kubernetes endpoint" - default = "" -} - -variable "terraform_service_account_credentials" { - type = string - description = "path to terraform service account file, created following the instructions in https://cloud.google.com/community/tutorials/managing-gcp-projects-with-terraform" - default = "~/.config/gcloud/application_default_credentials.json" -} - -variable "monitoring_slack_url" { - type = string - default = "" - description = "slack api url to send prometheus alerts to" -} - -variable "kubernetes_pool_name" { - type = string - description = "when kubernetes cluster has several node pools, specify which ones to deploy the baking setup into. only effective when deploying on an external cluster with terraform_no_cluster_create" - default = "blockchain-pool" -} - -variable "p2p_ip" { - type = string - description = "p2p ip to pass to load balancer" -} - -variable "p2p_port" { - type = string - description = "p2p port to pass to load balancer" -} - -variable "storage_size" { - type = string - description = "Size of backend storage for validator in Gi" - default = "25" -} From 769a1c1618f2231a94f5aef6107a3397636c8f10 Mon Sep 17 00:00:00 2001 From: Nicolas Ochem Date: Mon, 10 Jan 2022 17:48:10 -0800 Subject: [PATCH 2/4] pivot to helm --- charts/polkadot/.helmignore | 23 ++++ charts/polkadot/Chart.yaml | 23 ++++ charts/polkadot/README.md | 42 ++++++++ charts/polkadot/scripts/polkadot-node.sh | 51 +++++++++ charts/polkadot/templates/configmap.yaml | 15 +++ charts/polkadot/templates/networkpolicy.yaml | 41 +++++++ charts/polkadot/templates/secrets.yaml | 10 ++ charts/polkadot/templates/services.yaml | 17 +++ charts/polkadot/templates/statefulset.yaml | 106 +++++++++++++++++++ charts/polkadot/values.yaml | 39 +++++++ polkadot-archive-downloader/Dockerfile | 5 + polkadot-archive-downloader/entrypoint.sh | 27 +++++ polkadot-node-key-configurator/Dockerfile | 12 +++ polkadot-node-key-configurator/entrypoint.sh | 30 ++++++ polkadot-sidecar/Dockerfile | 6 ++ polkadot-sidecar/polkadot_sidecar.py | 65 ++++++++++++ polkadot-sidecar/requirements.txt | 4 + polkadot-sidecar/wsgi.py | 4 + 18 files changed, 520 insertions(+) create mode 100644 charts/polkadot/.helmignore create mode 100644 charts/polkadot/Chart.yaml create mode 100644 charts/polkadot/README.md create mode 100755 charts/polkadot/scripts/polkadot-node.sh create mode 100644 charts/polkadot/templates/configmap.yaml create mode 100644 charts/polkadot/templates/networkpolicy.yaml create mode 100644 charts/polkadot/templates/secrets.yaml create mode 100644 charts/polkadot/templates/services.yaml create mode 100644 charts/polkadot/templates/statefulset.yaml create mode 100644 charts/polkadot/values.yaml create mode 100644 polkadot-archive-downloader/Dockerfile create mode 100755 polkadot-archive-downloader/entrypoint.sh create mode 100644 polkadot-node-key-configurator/Dockerfile create mode 100755 polkadot-node-key-configurator/entrypoint.sh create mode 100644 polkadot-sidecar/Dockerfile create mode 100644 polkadot-sidecar/polkadot_sidecar.py create mode 100644 polkadot-sidecar/requirements.txt create mode 100644 polkadot-sidecar/wsgi.py diff --git a/charts/polkadot/.helmignore b/charts/polkadot/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/charts/polkadot/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/polkadot/Chart.yaml b/charts/polkadot/Chart.yaml new file mode 100644 index 0000000..ca6eb88 --- /dev/null +++ b/charts/polkadot/Chart.yaml @@ -0,0 +1,23 @@ +apiVersion: v2 +name: polkadot +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +appVersion: 1.16.0 diff --git a/charts/polkadot/README.md b/charts/polkadot/README.md new file mode 100644 index 0000000..f2f4fd8 --- /dev/null +++ b/charts/polkadot/README.md @@ -0,0 +1,42 @@ +# Polkadot helm chart + +This deploys a polkadot node. + +It can: + +* sync from a filesystem archive +* configure its own host key passed as parameter +* connect to other nodes in different namespaces in a full mesh +* deploy a network load balancer for p2p ingress in a cloud setup + +## How to deploy + +See values.yaml comments for detailed explanation of all possible values. + +## Example + +Deploy val1 chart in namespace `val1`: + +``` +polkadot_validator_name: "val1" +polkadot_archive_url: null +p2p_ip: 10.0.0.1 +p2p_port: 30333 +local_nodes: + nico: 9cd2bad53ae93f45ae19d62f7961679972b9099935ce29d00c2e23efbf2c40bf + nico2: 9cd2bad53ae93f45ae19d62f7961679972b9099935ce29d00c2e23efbf2c40be +``` + +Deploy val2 chart in namespace `val2`: + +``` +polkadot_validator_name: "val2" +polkadot_archive_url: null +p2p_ip: 10.0.0.1 +p2p_port: 30334 +local_nodes: + nico: 9cd2bad53ae93f45ae19d62f7961679972b9099935ce29d00c2e23efbf2c40bf + nico2: 9cd2bad53ae93f45ae19d62f7961679972b9099935ce29d00c2e23efbf2c40be +``` + +These 2 validators will extablish p2p peering with each other. diff --git a/charts/polkadot/scripts/polkadot-node.sh b/charts/polkadot/scripts/polkadot-node.sh new file mode 100755 index 0000000..850e73c --- /dev/null +++ b/charts/polkadot/scripts/polkadot-node.sh @@ -0,0 +1,51 @@ +#!/bin/bash + +set -e +set -x + +if [ -e /polkadot/k8s_local_node_key ]; then + node_key_param="--node-key-file /polkadot/k8s_local_node_key" +fi + +if [ -e /polkadot/k8s_local_peer_cmd ]; then + local_peer_param="$(cat /polkadot/k8s_local_peer_cmd)" +fi + +if [ ! -z "$VALIDATOR_NAME" ]; then + name_param="--name \"$VALIDATOR_NAME\"" +fi + +if [ ! -z "$CHAIN" ]; then + chain_param="--chain \"$CHAIN\"" +fi + +if [ ! -z "$IN_PEERS" ]; then + in_peers_param="--in-peers=${IN_PEERS}" +fi + +if [ ! -z "$OUT_PEERS" ]; then + out_peers_param="--out-peers=${OUT_PEERS}" +fi + +if [ ! -z "$TELEMETRY_URL" ]; then + telemetry_url_param="--telemetry-url \"$TELEMETRY_URL 0\"" +fi + +if [ ! -z "$PUBLIC_MULTIADDR" ]; then + public_address_param="--public-addr=${PUBLIC_MULTIADDR}" +fi + +# sleep 1000 +eval /usr/bin/polkadot --validator --wasm-execution Compiled \ + --unsafe-pruning \ + --pruning=1000 \ + --prometheus-external \ + --execution native \ + $out_peers_param \ + $in_peers_param \ + $node_key_param \ + $name_param \ + $telemetry_url_param \ + $chain_param \ + $public_address_param \ + $local_peer_param diff --git a/charts/polkadot/templates/configmap.yaml b/charts/polkadot/templates/configmap.yaml new file mode 100644 index 0000000..2f91331 --- /dev/null +++ b/charts/polkadot/templates/configmap.yaml @@ -0,0 +1,15 @@ +--- +apiVersion: v1 +data: + ARCHIVE_URL: "{{ .Values.polkadot_archive_url }}" + TELEMETRY_URL: "{{ .Values.polkadot_telemetry_url }}" + VALIDATOR_NAME: "{{ .Values.polkadot_validator_name }}" + OUT_PEERS: "{{ .Values.number_of_out_peers }}" + IN_PEERS: "{{ .Values.number_of_in_peers }}" + CHAIN: "{{ .Values.chain}}" + NAMESPACE: "{{ .Release.Namespace }}" + PUBLIC_MULTIADDR: "/ip4/{{ .Values.p2p_ip }}/tcp/{{ .Values.p2p_port }}" +kind: ConfigMap +metadata: + name: polkadot-configmap + namespace: {{ .Release.Namespace }} diff --git a/charts/polkadot/templates/networkpolicy.yaml b/charts/polkadot/templates/networkpolicy.yaml new file mode 100644 index 0000000..8bcef34 --- /dev/null +++ b/charts/polkadot/templates/networkpolicy.yaml @@ -0,0 +1,41 @@ +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: polkadot-private-node-policy + namespace: default + namespace: {{ .Release.Namespace }} +spec: + podSelector: + matchLabels: + app: private-node + policyTypes: + - Ingress + - Egress + egress: + - ports: + - port: 53 + protocol: TCP + - port: 53 + protocol: UDP + - port: 443 + protocol: TCP + - port: 30333 + protocol: TCP + - port: 30334 + protocol: TCP + - port: 30100 + protocol: TCP + ingress: + - ports: + - port: {{ .Values.p2p_port }} + protocol: TCP + - ports: + - port: 9615 + protocol: TCP + from: + - namespaceSelector: + matchLabels: {} + podSelector: + matchLabels: + app: prometheus diff --git a/charts/polkadot/templates/secrets.yaml b/charts/polkadot/templates/secrets.yaml new file mode 100644 index 0000000..2739360 --- /dev/null +++ b/charts/polkadot/templates/secrets.yaml @@ -0,0 +1,10 @@ +--- +apiVersion: v1 +data: +{{- range $k, $v := .Values.local_nodes }} + {{ $k }}: {{ $v | b64enc }} +{{- end }} +kind: Secret +metadata: + name: polkadot-node-keys + namespace: {{ .Release.Namespace }} diff --git a/charts/polkadot/templates/services.yaml b/charts/polkadot/templates/services.yaml new file mode 100644 index 0000000..48be27c --- /dev/null +++ b/charts/polkadot/templates/services.yaml @@ -0,0 +1,17 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: private-node + namespace: {{ .Release.Namespace }} + labels: + app: polkadot-node +spec: + ports: + - port: 9933 + name: rpc + - port: 9615 + name: metrics + selector: + app: polkadot-node + clusterIP: None diff --git a/charts/polkadot/templates/statefulset.yaml b/charts/polkadot/templates/statefulset.yaml new file mode 100644 index 0000000..05ea8ed --- /dev/null +++ b/charts/polkadot/templates/statefulset.yaml @@ -0,0 +1,106 @@ +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: polkadot-node + namespace: {{ .Release.Namespace }} +spec: + selector: + matchLabels: + app: polkadot-node + serviceName: polkadot-node + replicas: 1 + template: + metadata: + labels: + app: polkadot-node + spec: + nodeSelector: + {{ toYaml .Values.node_selector | indent 8 }} + securityContext: + fsGroup: 1000 + containers: + - name: polkadot-node + image: {{ .Values.images.polkadot_node }} + command: + - /bin/bash + args: + - "-c" + - | +{{ tpl (.Files.Get "scripts/polkadot-node.sh") . | indent 12 }} + ports: + - containerPort: 9933 + name: dot-rpc-port + - containerPort: 9615 + name: metrics + - containerPort: {{ .Values.p2p_port }} + name: dot-p2p-port + volumeMounts: + - name: polkadot-node-pv-claim + mountPath: /polkadot + envFrom: + - configMapRef: + name: polkadot-configmap + resources: + limits: + cpu: 0 + imagePullPolicy: IfNotPresent + readinessProbe: + periodSeconds: 30 + timeoutSeconds: 10 + httpGet: + path: /is_synced + port: 31764 + - name: polkadot-sidecar + image: {{ .Values.polkadot_k8s_images.polkadot_sidecar }} + resources: + limits: + cpu: 0 + imagePullPolicy: Always + initContainers: + - name: polkadot-node-key-configurator + image: {{ .Values.polkadot_k8s_images.polkadot_node_key_configurator }} + volumeMounts: + - name: polkadot-node-pv-claim + mountPath: /polkadot + - name: polkadot-node-keys + mountPath: /polkadot-node-keys + envFrom: + - configMapRef: + name: polkadot-configmap + imagePullPolicy: IfNotPresent + resources: + limits: + cpu: 0 + - name: polkadot-archive-downloader + image: {{ .Values.polkadot_k8s_images.polkadot_archive_downloader }} + volumeMounts: + - name: polkadot-node-pv-claim + mountPath: /polkadot + env: + - name: CHAIN + valueFrom: + configMapKeyRef: + name: polkadot-configmap + key: CHAIN + - name: ARCHIVE_URL + valueFrom: + configMapKeyRef: + name: polkadot-configmap + key: ARCHIVE_URL + imagePullPolicy: IfNotPresent + resources: + limits: + cpu: 0 + volumes: + - name: polkadot-node-keys + secret: + secretName: polkadot-node-keys + volumeClaimTemplates: + - metadata: + name: polkadot-node-pv-claim + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: {{ .Values.vol_size }}Gi diff --git a/charts/polkadot/values.yaml b/charts/polkadot/values.yaml new file mode 100644 index 0000000..432476f --- /dev/null +++ b/charts/polkadot/values.yaml @@ -0,0 +1,39 @@ +# Images not part of the tezos-k8s repo go here +images: + polkadot_node: "paritytech/polkadot:latest" +# Images that are part of the polkadot-k8s repo go here with 'latest' tag +polkadot_k8s_images: + polkadot_archive_downloader: midl/polkadot_archive_downloader + polkadot_node_key_configurator: midl/polkadot_node_key_configurator + polkadot_sidecar: midl/polkadot-sidecar:latest + +polkadot_archive_url: https://ksm-rocksdb.polkashots.io/snapshot + +polkadot_telemetry_url: null + +polkadot_validator_name: polkadot_k8s_pulumi + +number_of_out_peers: 10 + +number_of_in_peers: 10 + +chain: kusama + +# list of peers to always connect to. could be polkadot nodes in different namespaces +# should be a list of key-value pairs with the key as namespace name and the value as private network key +local_nodes: {} + +# if provided, this will be passed as the public ip/port combination of the node +# If you have an ingress network load balancer ip sending p2p traffic to the node, set this to its address/port +p2p_ip: null +p2p_port: 30333 + +# to deploy in a specific node pool, put label here +node_selector: {} + +# vol size (in Gi) +# Applies to the statefulset's PVC template. +# consequently this value only applies for creation. +# It is not possible to resize the volume by editing this value. +# Instead, you must resize the pvc directly. +vol_size: 50 diff --git a/polkadot-archive-downloader/Dockerfile b/polkadot-archive-downloader/Dockerfile new file mode 100644 index 0000000..33abfc4 --- /dev/null +++ b/polkadot-archive-downloader/Dockerfile @@ -0,0 +1,5 @@ +FROM alpine +RUN apk add curl lz4 bash +COPY entrypoint.sh / +ENTRYPOINT ["/entrypoint.sh"] +CMD [] diff --git a/polkadot-archive-downloader/entrypoint.sh b/polkadot-archive-downloader/entrypoint.sh new file mode 100755 index 0000000..462ad98 --- /dev/null +++ b/polkadot-archive-downloader/entrypoint.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +set -e + +if [ "${CHAIN}" == "polkadot" ]; then + chain_dir=polkadot +else + chain_dir=ksmcc3 +fi + +if [ -d /polkadot/.local/share/polkadot/chains/${chain_dir}/db/ ]; then + echo "Blockchain database already exists, no need to import, exiting" + exit 0 +elif [ -z "$ARCHIVE_URL" ]; then + echo "No archive download url specified, exiting" + exit 0 +else + echo "Did not find pre-existing data, importing blockchain" + mkdir -p /polkadot/.local/share/polkadot/chains/${chain_dir}/ + echo "Will download $ARCHIVE_URL" + curl -L $ARCHIVE_URL -o /polkadot/polkadot_archive.7z + 7z x /polkadot/polkadot_archive.7z -o/polkadot/.local/share/polkadot/chains/${chain_dir} + rm -v /polkadot/polkadot_archive.7z + chmod -R 777 /polkadot/.local/ + chown -R 1000:1000 /polkadot/.local/ + find /polkadot/.local/share/ +fi diff --git a/polkadot-node-key-configurator/Dockerfile b/polkadot-node-key-configurator/Dockerfile new file mode 100644 index 0000000..1ae1e56 --- /dev/null +++ b/polkadot-node-key-configurator/Dockerfile @@ -0,0 +1,12 @@ +FROM parity/subkey:2.0.0 +USER root +# install tools and dependencies +RUN apt-get update --allow-insecure-repositories && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y \ + xxd && \ + apt-get autoremove -y && \ + apt-get clean && \ + find /var/lib/apt/lists/ -type f -not -name lock -delete; +COPY entrypoint.sh / +ENTRYPOINT ["/entrypoint.sh"] +CMD [] diff --git a/polkadot-node-key-configurator/entrypoint.sh b/polkadot-node-key-configurator/entrypoint.sh new file mode 100755 index 0000000..2ec1d51 --- /dev/null +++ b/polkadot-node-key-configurator/entrypoint.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +set -e +set -x + +rm -vf /polkadot/k8s_local_node_key +rm -rvf /polkadot/k8s_node_ids/ +mkdir -p /polkadot/k8s_node_ids + +# write private key for this node only and protect it +if [ -f /polkadot-node-keys/${NAMESPACE} ]; then + cat /polkadot-node-keys/${NAMESPACE} | xxd -r -p > /polkadot/k8s_local_node_key + # move owner to polkadot + chown 1000 /polkadot/k8s_local_node_key + chmod 400 /polkadot/k8s_local_node_key +fi + + +for node in $(ls /polkadot-node-keys) +do + # do not peer with myself + if [ "${node}" != "${NAMESPACE}" ] + then + if [ ! -f /polkadot/k8s_local_peer_cmd ]; then + # write public keys for all peers in an env file, to be sourced by polkadot startup script + echo "--reserved-nodes " > /polkadot/k8s_local_peer_cmd + fi + echo "/dns4/${CHAIN}-node-0.${CHAIN}-node.${node}/tcp/30333/p2p/$(subkey inspect-node-key --file /polkadot-node-keys/$node) " >> /polkadot/k8s_local_peer_cmd + fi +done diff --git a/polkadot-sidecar/Dockerfile b/polkadot-sidecar/Dockerfile new file mode 100644 index 0000000..e0d35ff --- /dev/null +++ b/polkadot-sidecar/Dockerfile @@ -0,0 +1,6 @@ +FROM python:alpine +WORKDIR /build +COPY . /build +ENV PYTHONUNBUFFERED=1 +RUN pip install -r requirements.txt +ENTRYPOINT ["gunicorn", "-b", ":31764", "wsgi" ] diff --git a/polkadot-sidecar/polkadot_sidecar.py b/polkadot-sidecar/polkadot_sidecar.py new file mode 100644 index 0000000..560a565 --- /dev/null +++ b/polkadot-sidecar/polkadot_sidecar.py @@ -0,0 +1,65 @@ +from flask import Flask, escape, request +import requests +import time + +application = Flask(__name__) + +@application.route('/is_synced') +def sync_checker(): + try: + system_health = requests.post('http://127.0.0.1:9933', json={"id":1, "jsonrpc":"2.0", "method": "system_health", "params":[]}).json()["result"] + + if system_health["isSyncing"]: + err = "Still syncing", 500 + print(err) + return err + + current_block_height = requests.post('http://127.0.0.1:9933', json={"id":1, "jsonrpc":"2.0", "method": "system_syncState", "params":[]}).json()["result"]["currentBlock"] + + current_block_hash = requests.post('http://127.0.0.1:9933', json={"id":1, "jsonrpc":"2.0", "method": "chain_getBlockHash", "params":[current_block_height]}).json()["result"] + current_block = requests.post('http://127.0.0.1:9933', json={"id":1, "jsonrpc":"2.0", "method": "chain_getBlock", "params":[current_block_hash]}).json()["result"]["block"] + except requests.exceptions.RequestException as e: + err = "Could not connect to node, %s" % repr(e), 500 + print(err) + return err + + timestamp_extrinsic = current_block["extrinsics"][0].split("0x")[1] + # take bytes from fourth to end + timestamp = bytes.fromhex(timestamp_extrinsic[10:]) + # bytes are little endian + timestamp_int = int(int.from_bytes(timestamp, "little")/1000) + time_now = int(time.time()) + last_block_age = time_now - timestamp_int + + PROBLEM_DELAY = 180 + if last_block_age > PROBLEM_DELAY: + err = f"last block is more than {PROBLEM_DELAY} seconds old" + print(err) + return err + return "chain is synced" + + + +# timestamp is encoded with scale codec +# https://substrate.dev/docs/en/knowledgebase/advanced/codec +# from riot discussion +# https://matrix.to/#/!LhjZccBOqFNYKLdmbb:polkadot.builders/$1589987466245024AEJsU:matrix.org?via=matrix.parity.io&via=matrix.org&via=corepaper.org + +# +#nuevax +#it is still unclear to me, the second half of the string always begins with 0b, which indicates an integer, however it does not have the two bits after that indicative for one of the integer formats from the Codec definition. +#Jaco +#0x0b = 0b1011 - 11 (lower 2 bits as per spec) indicates 4 bytes mode, so number of byte following is 4 (as indicated) + 0b10 bytes - so total of 6 bytes. +#nuevax +#I tried all kinds of 6 bytes hex to dec, but to me it seems the 10/13 digit dec timestamp cannot be generated out of 6 bytes even though it says compact. +#nuevax +#Message deleted +#nuevax +#0x280402000bb1f14e327201 this is the string from the extrinsic, 0bb1f14e327201 the second half, after 0b none of the codec specifications. +#Jaco +#:Looks valid, manually decoded it, +# +#1 +#2 +#console.log(new Date(1589981934001)) +#Wed May 20 2020 15:38:54 GMT+0200 (Central European Summer Time) diff --git a/polkadot-sidecar/requirements.txt b/polkadot-sidecar/requirements.txt new file mode 100644 index 0000000..14c8011 --- /dev/null +++ b/polkadot-sidecar/requirements.txt @@ -0,0 +1,4 @@ +flask +requests +gunicorn +python-dateutil diff --git a/polkadot-sidecar/wsgi.py b/polkadot-sidecar/wsgi.py new file mode 100644 index 0000000..59039e3 --- /dev/null +++ b/polkadot-sidecar/wsgi.py @@ -0,0 +1,4 @@ +from polkadot_sidecar import application + +if __name__ == "__main__": + application.run() From ac8528817c9a3f21ef9562cceb2ce232f267b3c8 Mon Sep 17 00:00:00 2001 From: Nicolas Ochem Date: Mon, 10 Jan 2022 19:17:07 -0800 Subject: [PATCH 3/4] add github actions to store charts --- .github/workflows/ci.yml | 126 ++++++++++++++++++++++++++++++++++++ charts/polkadot/values.yaml | 6 +- 2 files changed, 129 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..4d3abe3 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,126 @@ +name: CI + +on: + push: + # Trigger CI on all branch pushes but... + branches: + - "**" + # don't double trigger on new tag push when creating release. Should only + # trigger once for the release. + tags-ignore: + - "*.*.*" + pull_request: + release: + types: [created] + +jobs: + + list_containers_to_publish: + # based on + # https://stackoverflow.com/a/62953566/207209 + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.gen-containers-matrix.outputs.matrix }} + steps: + - uses: actions/checkout@v2 + + - id: gen-containers-matrix + run: | + container_list=$(jq -c -n --arg cont "$(find -name 'Dockerfile' -printf '%h\n' | sort -u | sed 's/.\///')" '{ container: $cont | split("\n")}') + echo "Dynamically generated container list based on subdirectories of the repo with a dockerfile in it. The following list will be passed to next build step:" + echo $container_list + echo "::set-output name=matrix::$container_list" + + publish_containers: + # based on + # https://github.com/docker/build-push-action#usage + runs-on: ubuntu-latest + needs: list_containers_to_publish + if: github.event_name == 'release' && github.event.action == 'created' + strategy: + matrix: ${{fromJson(needs.list_containers_to_publish.outputs.matrix)}} + + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Login to registry + run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin + - name: Docker meta + id: meta + uses: docker/metadata-action@v3 + with: + images: ghcr.io/${{ github.repository_owner }}/${{ matrix.container }} + tags: | + type=ref,event=branch + type=ref,event=pr + type=match,pattern=v(.*),group=1 + + - name: Push to GHCR + uses: docker/build-push-action@v2 + with: + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + + lint_helm_charts: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Set up Helm + uses: azure/setup-helm@v1 + with: + version: v3.4.2 + + - name: Lint Helm Charts + run: helm lint charts/* + + publish_helm_charts: + runs-on: ubuntu-latest + needs: [lint_helm_charts, publish_containers] + if: github.event_name == 'release' && github.event.action == 'created' + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Install yq + run: | + sudo wget -q https://github.com/mikefarah/yq/releases/download/v4.2.0/yq_linux_amd64 -O /usr/bin/yq + sudo chmod +x /usr/bin/yq + + - name: Get Release Version + id: get_release_version + run: echo "::set-output name=RELEASE_VERSION::${GITHUB_REF/refs\/tags\//}" + + - name: Set Helm Chart and Image Versions + run: | + set -x + + RELEASE_VERSION=${{ steps.get_release_version.outputs.RELEASE_VERSION }} + + for chart in charts/*; do + [[ ! -d "$chart" ]] && continue + echo $chart + + # Update Chart.yaml with release version + yq e ".version = \"$RELEASE_VERSION\"" -i "$chart/Chart.yaml" + + # Get midl-dev/polkadot-k8s images specified in values.yaml + custom_images=$(yq e '(.polkadot_k8s_images[]) | path | .[-1]' "$chart/values.yaml") + + # Update the release version of each of polkadot-k8s images + for image in $custom_images; do + image_name=$(yq e ".polkadot_k8s_images.$image" $chart/values.yaml | sed -E "s/polkadot-k8s-(.*):.*/\1/") + yq e ".polkadot_k8s_images.$image = \"midl-dev/polkadot-k8s-$image_name:$RELEASE_VERSION\"" -i $chart/values.yaml + done + done + + - name: Publish Helm charts + uses: stefanprodan/helm-gh-pages@master + with: + linting: off # We already linted in a job before + token: ${{ secrets.CI_GITHUB_TOKEN }} + branch: main + owner: ${{ github.repository_owner }} + repository: charts diff --git a/charts/polkadot/values.yaml b/charts/polkadot/values.yaml index 432476f..71f18e7 100644 --- a/charts/polkadot/values.yaml +++ b/charts/polkadot/values.yaml @@ -3,9 +3,9 @@ images: polkadot_node: "paritytech/polkadot:latest" # Images that are part of the polkadot-k8s repo go here with 'latest' tag polkadot_k8s_images: - polkadot_archive_downloader: midl/polkadot_archive_downloader - polkadot_node_key_configurator: midl/polkadot_node_key_configurator - polkadot_sidecar: midl/polkadot-sidecar:latest + polkadot_archive_downloader: polkadot-archive-downloader:dev + polkadot_node_key_configurator: polkadot-node-key-configurator:dev + polkadot_sidecar: polkadot-sidecar:dev polkadot_archive_url: https://ksm-rocksdb.polkashots.io/snapshot From 8d84aa822035ca9bdc6c25441267c2278e0e1821 Mon Sep 17 00:00:00 2001 From: Nicolas Ochem Date: Mon, 10 Jan 2022 19:34:16 -0800 Subject: [PATCH 4/4] slim down readme --- README.md | 222 +----------------------------------------------------- 1 file changed, 2 insertions(+), 220 deletions(-) diff --git a/README.md b/README.md index 69a1765..f925a4f 100644 --- a/README.md +++ b/README.md @@ -1,17 +1,11 @@ # Polkadot-k8s -This project deploys a fully featured, best practices [Polkadot](https://polkadot.network) or [Kusama](https://kusama.network) validator setup on Google Kubernetes Engine. +This project is a collection of helm charts to help you deploy [Polkadot](https://polkadot.network) or [Kusama](https://kusama.network) validation infrastructure. Features: * compatible with Kusama and Polkadot -* high availability and geographical distribution -* download and import a [pre-synced database](https://dotleap.com/how-to-import-a-pre-synced-kusama-database/) for faster synchronization of the node -* deploy everything in just one command - no prior knowledge of Kubernetes required - -TODO: - -* support for on-prem remote signer [whenever available](https://github.com/paritytech/substrate/issues/4689) +* download and import a [pre-synced database](https://polkashots.io) for faster synchronization of the node Brought to you by MIDL.dev -------------------------- @@ -19,215 +13,3 @@ Brought to you by MIDL.dev MIDL.dev We help you deploy and manage a complete Polkadot or Kusama validator infrastructure for you. [Hire us](https://midl.dev). - -Architecture ------------- - -This is a Kubernetes private cluster with two nodes located in two Google Cloud zones, in the same region. - -The validator node uses a [Regional Persistent Disk](https://cloud.google.com/compute/docs/disks/#repds) so it can be respun quickly in the other node from the pool if the first node goes offline for any reason, for example base OS upgrade. - -The setup is production hardened: -* usage of kubernetes secrets to store sensitive values such as node keys. They are created securely from terraform variables, -* network policies to restrict communication between pods. For example, only sentries can peer with the validator node. - -## Costs - -Deploying will incur Google Compute Engine charges, specifically: - -* virtual machines -* regional persistent SSD storage -* network ingress -* NAT forwarding - -# How to deploy - -*WARNING: Polkadot/Kusama tokens have value. Use judgement and care in your network interactions, otherwise loss of funds may occur.* - -## Prerequisites - -1. Download and install [Terraform](https://terraform.io) - -1. Download, install, and configure the [Google Cloud SDK](https://cloud.google.com/sdk/). - -1. Install the [kubernetes - CLI](https://kubernetes.io/docs/tasks/tools/install-kubectl/) (aka - `kubectl`) - -## Authentication - -Using your Google account, active your Google Cloud access. - -Login to gcloud using `gcloud auth login` - -Set up [Google Default Application Credentials](https://cloud.google.com/docs/authentication/production) by issuing the command: - -``` -gcloud auth application-default login -``` - -NOTE: for production deployments, the method above is not recommended. Instead, you should use a Terraform service account following [these instructions](docs/production-hardening.md). - -## Bond your tokens - -Follow [these instructions](https://wiki.polkadot.network/docs/en/maintain-guides-how-to-validate-kusama#bond-ksm) to bond your KSM. - -NOTE: the link above points to the official guide about [how to validate on Kusama](https://wiki.polkadot.network/docs/en/maintain-guides-how-to-validate-kusama). Not every action in this guide needs to be performed. For example, there is no need to build binaries. - -## Populate terraform variables - -All custom values unique to your deployment are set as terraform variables. You must populate these variables manually before deploying the setup. - -A simple way is to populate a file called `terraform.tfvars` in the `terraform` folder. - -NOTE: `terraform.tfvars` is not recommended for a production deployment. See [production hardening](docs/production-hardening.md). - -### Network (libp2p) keys (optional) - -These keys are needed for the validator to communicate to its sentries. You may pass them as a variabe, or Terraform will generate them for you. - -If you want to pass them, the syntax is: - -``` -polkadot_node_keys = { - "polkadot-private-node-0": "b5ca09a5dccb48d5c7915f24223454fe1a557383ba0b1560cc3ed919a6e9dec5", - "polkadot-sentry-node-0": "dcf609b50868ffe379d4a992cf866deba8ad84ecca24853bacba1171ae7cdf22", - "polkadot-sentry-node-1": "ca62cb1bae8c84090f2e20dae81c79f8843fb75df065c8026e4603b85b48729f" -} -``` - -### Polkadot utility parameters - -Set the `polkadot_version` to the desired version of polkadot container. - -Set the `chain` variable to the desired network you want to launch (`polkadot` or `kusama`). - -Set the `polkadot_telemetry_url` variable to the telemetry server websocket endpoint (that you would pass to polkadot's `--telemetry-url` option) - -Set the `polkadot_validator_name` to your validator name as you want it to appear on telemetry (maps to polkadot's `--name` parameter). - -### Archive URL (optional) - -If you have an archive of the node storage, you can put the URL here. It will make the initial deployment of the nodes faster. It must be in `7z` format. - -See [a resource on how to get a pre-synced archive databsae for Kusama](https://dotleap.com/how-to-import-a-pre-synced-kusama-database/). - - -### Payout account - -Every era, you can pay your nominators automatically by calling the payoutStakers() extrinsic *from any account*. A Kubernetes cronjob will do this for you if you pass the address and its associated private key. - -This should be a dust account. The private key is stored in a Kubernetes secret but should not be considered secure. Do not put meaningful amounts in this account, just enough to pay fees. - -### Google Cloud project - -A default Google Cloud project should have been created when you activated your account. Verify its ID with `gcloud projects list`. You may also create a dedicated project to deploy the cluster. - -Set the project id in the `project` terraform variable. - -NOTE: if you created a [terraform service account](docs/production-hardening.md), leave this variable empty. - -### Recap : full example of terraform.tfvars file - -``` -project="beaming-essence-301841" -polkadot_archive_url="https://ipfs.io/ipfs/Qma3fM33cw4PGiw28SidqhFi3CXRa2tpywqLmhYveabEYQ?filename=Qma3fM33cw4PGiw28SidqhFi3CXRa2tpywqLmhYveabEYQ" -polkadot_validator_name="Hello from k8s!" -polkadot_version="v0.8.0" -chain="kusama" -polkadot_telemetry_url="wss://telemetry-backend.w3f.community/submit" -``` - -The example above would: -* deploy a validator setup in the Google Cloud project named `beaming-essence-301841` -* download a kusama snapshot from IPFS -* report to telemetry server `w3f.community` under the name `Hello from k8s!` - -## Deploy! - -1. Run the following: - -``` -cd terraform - -terraform init -terraform plan -out plan.out -terraform apply plan.out -``` - -This will take time as it will: -* create a Kubernetes cluster -* build the necessary containers -* download and unzip the archives if applicable -* spin up the sentry and validator nodes -* sync the network - -### Connect to the cluster - -After apply is complete, your `kubectl` command should point to the correct cluster. You can then issue `kubectl get pods` and observe that your Polkadot nodes are now alive and syncing. - -When you display the logs of your private node, you will see it syncing: - -``` -kubectl logs -f polkadot-private-node-0 --tail=10 -``` - -### How to check your validator node is running ? - -* connect to your telemetry server and search for your node by name -* set up a websocket tunnel to your local host - -``` -kubectl port-forward polkadot-private-node-0 9944:9944 -``` - -Then go to the [Polkadot Js app](https://polkadot.js.org/apps/#/) and configure it to point to `localhost:9944`. You should see your node syncing. - -### Inject session keys - -[Follow instructions](https://wiki.polkadot.network/docs/en/maintain-guides-how-to-validate-kusama#set-session-keys) to inject session keys using the Polkadot Js app. - -### Validate - -[Follow instructions](https://wiki.polkadot.network/docs/en/maintain-guides-how-to-validate-kusama#validate) - - -## I have a kubernetes cluster already, I just want to deploy to it - -[Instructions here](docs/pre-existing-cluster.md) - -Apply an update ---------------- - -If you have pulled the most recent version of `polkadot-k8s` and wish to apply updates, issue the following commands: - -``` -terraform taint null_resource.push_containers && terraform taint null_resource.apply && terraform plan -out plan.out -terraform apply plan.out -``` - -This will rebuild the containers, then apply the kubernetes changes. - -If you want to upgrade the polkadot containers version, edit the version number in `terraform.tfvars`, then issue the commands above. - -The pods may or may not restart on their own, depending on what changed since last deployment. To force a restart, issue: - -``` -kubectl delete pod polkadot-sentry-node-0 -kubectl delete pod polkadot-sentry-node-1 -kubectl delete pod polkadot-private-node-0 -``` - -Look at the logs and ensure each pod comes back online before deleting the next one. - -NOTE: since these pods are part of Deployment/StatefulSet, kubernetes auto-restarts them when you delete them. Their storage is persisted so they restart where they left off. - -## Wrapping up - -To delete everything and terminate all the charges, issue the command: - -``` -terraform destroy -``` - -Alternatively, go to the GCP console and delete the project.