From a73d0817e1bb178c72e91a0336ecdb9a5b21b255 Mon Sep 17 00:00:00 2001 From: Polkachu Date: Sun, 25 Dec 2022 18:44:28 -0800 Subject: [PATCH] update --- .gitignore | 5 +- README.md | 94 +++---------------- all_apt_update.yml | 10 -- ansible.cfg | 2 +- group_vars/all.yml | 21 +---- group_vars/kusama.yml | 1 - group_vars/polkadot.yml | 1 - group_vars/validators.yml | 3 - inventory.sample | 25 ----- inventory.sample.ini | 17 ++++ polkadot_rotate_key.yml => key_rotation.yml | 2 +- node_exporter.yml | 7 -- process_exporter.yml => polkadot.yml | 4 +- polkadot_backup_keystore.yml | 7 -- polkadot_clean_logs.yml | 7 -- polkadot_full_archive_node_setup.yml | 29 ------ polkadot_full_setup.yml | 25 ----- polkadot_restart.yml | 7 -- polkadot_restore.yml | 7 -- polkadot_stop.yml | 7 -- polkadot_update.yml | 7 -- polkadot_prepare.yml => prepare.yml | 6 +- promtail.yml | 7 -- .../tasks/main.yml | 5 +- roles/node_exporter/tasks/main.yml | 20 ++-- roles/polkadot/tasks/main.yml | 47 ++++++++++ roles/polkadot/templates/polkadot.service.j2 | 27 ++++++ .../files/journald.conf | 0 .../files/nginx.conf | 0 .../nginx.yml => prepare/tasks/main.yml} | 29 ++++-- .../files/process_exporter.service | 12 --- .../files/process_exporter_config.yml | 4 - roles/process_exporter/tasks/main.yml | 56 ----------- roles/promtail/tasks/main.yml | 29 +++--- roles/system_update/tasks/main.yml | 25 ----- .../tasks/main.yml | 6 -- .../tasks/main.yml | 7 -- .../files/polkadot_metrics.nginx.conf | 6 -- .../tasks/firewall.yml | 41 -------- .../tasks/journald.yml | 13 --- .../validator_polkadot_prepare/tasks/main.yml | 25 ----- .../tasks/service.yml | 8 -- .../validator_polkadot_prepare/tasks/user.yml | 23 ----- .../templates/polkadot.service.j2 | 43 --------- .../validator_polkadot_restart/tasks/main.yml | 21 ----- .../validator_polkadot_restore/tasks/main.yml | 58 ------------ roles/validator_polkadot_stop/tasks/main.yml | 4 - .../validator_polkadot_update/tasks/main.yml | 30 ------ system_update.yml | 5 - 49 files changed, 169 insertions(+), 676 deletions(-) delete mode 100644 all_apt_update.yml delete mode 100644 group_vars/validators.yml delete mode 100644 inventory.sample create mode 100644 inventory.sample.ini rename polkadot_rotate_key.yml => key_rotation.yml (72%) delete mode 100644 node_exporter.yml rename process_exporter.yml => polkadot.yml (58%) delete mode 100644 polkadot_backup_keystore.yml delete mode 100644 polkadot_clean_logs.yml delete mode 100644 polkadot_full_archive_node_setup.yml delete mode 100644 polkadot_full_setup.yml delete mode 100644 polkadot_restart.yml delete mode 100644 polkadot_restore.yml delete mode 100644 polkadot_stop.yml delete mode 100644 polkadot_update.yml rename polkadot_prepare.yml => prepare.yml (50%) delete mode 100644 promtail.yml rename roles/{validator_polkadot_rotate_key => key_rotation}/tasks/main.yml (75%) create mode 100644 roles/polkadot/tasks/main.yml create mode 100644 roles/polkadot/templates/polkadot.service.j2 rename roles/{validator_polkadot_prepare => prepare}/files/journald.conf (100%) rename roles/{validator_polkadot_prepare => prepare}/files/nginx.conf (100%) rename roles/{validator_polkadot_prepare/tasks/nginx.yml => prepare/tasks/main.yml} (51%) delete mode 100644 roles/process_exporter/files/process_exporter.service delete mode 100644 roles/process_exporter/files/process_exporter_config.yml delete mode 100644 roles/process_exporter/tasks/main.yml delete mode 100644 roles/system_update/tasks/main.yml delete mode 100644 roles/validator_polkadot_backup_keystore/tasks/main.yml delete mode 100644 roles/validator_polkadot_clean_logs/tasks/main.yml delete mode 100644 roles/validator_polkadot_prepare/files/polkadot_metrics.nginx.conf delete mode 100644 roles/validator_polkadot_prepare/tasks/firewall.yml delete mode 100644 roles/validator_polkadot_prepare/tasks/journald.yml delete mode 100644 roles/validator_polkadot_prepare/tasks/main.yml delete mode 100644 roles/validator_polkadot_prepare/tasks/service.yml delete mode 100644 roles/validator_polkadot_prepare/tasks/user.yml delete mode 100644 roles/validator_polkadot_prepare/templates/polkadot.service.j2 delete mode 100644 roles/validator_polkadot_restart/tasks/main.yml delete mode 100644 roles/validator_polkadot_restore/tasks/main.yml delete mode 100644 roles/validator_polkadot_stop/tasks/main.yml delete mode 100644 roles/validator_polkadot_update/tasks/main.yml delete mode 100644 system_update.yml diff --git a/.gitignore b/.gitignore index 3934a66..10da9c7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1 @@ -.vagrant/ -.vscode/ -inventory -polkadot_debug.yml \ No newline at end of file +inventory.ini diff --git a/README.md b/README.md index 36b2907..c04e60b 100644 --- a/README.md +++ b/README.md @@ -4,19 +4,15 @@ This repo is to set up the Polkadot Validation node. This repo is heavily influe ## Motivation -While the official setup is very comprehensive, it can be overwhelming for "small" validators (myself included) who do not care much about using Terraform on the infrastructure layer. I took the Ansible part of the script and updated it: - -1. The setup is more opinionated, thus the script is simpler by avoiding many "if" statements. It is tailored for Ubuntu only, but you should be able to get it working on other Linux distributions with some revisions. -2. It is more opinionated about node monitoring by recommending Node Exporter, Processor Exporter, and Promtail (for centralized log monitoring). I also have a companion Ansible script (https://github.com/polkachu/server-monitoring) that installs Prometheus, Grafana, and Loki to set up such a centralized monitoring server. This setup will make your life easier if you eventually move from a "small" validator to running a cluster of Polkadot/Kusama nodes. -3. The setup assumes that you will start from an archived node snapshot provided by https://polkashots.io. It is much simpler and less error-prone than Rust compiling. Highly recommended. In fact, we at Polkachu are currently planning to offer such archived node snapshots to provide redundancy to the community. -4. Since it has happened twice already, I have included a configuration to help you roll back to version `0.8.30` in the `group_vars/polkadot.yml` file. +While the official setup is very comprehensive, it can be overwhelming for "small" validators who do not care much about using Terraform on the infrastructure layer. ## Summary -You run one playbook and set up a Kusama/Polkadot node. Boom! +You run one playbook to prepare a node with Node Exporter and Promtail, and run one more playbook to launch a Kusama/Polkadot node. Boom! ```bash -ansible-playbook -i inventory polkadot_full_setup.yml -e "target=VALIDATOR_TARGET" +ansible-playbook prepare.yml -e "target=VALIDATOR_TARGET" +ansible-playbook polkadot.yml -e "target=VALIDATOR_TARGET" ``` But before you rush with this easy setup, you probably want to read on so you understand the structure of this Ansible program and all the features it offers. @@ -28,27 +24,14 @@ First of all, some preparation is in order. Make sure that you have a production inventory file with your confidential server info. You will start by copying the sample inventory file (included in the repo). The sample file gives you a good idea on how to define the inventory. ```bash -cp inventory.sample inventory +cp inventory.sample.ini inventory.ini ``` Needless to say, you need to update the dummy values in the inventory file. For each Kusama/Polkadot node, you need to update: -1. Server IP: Your server public IP -2. validator_name: This is the node name that will show up on telemetry monitoring board. It is especially important if you want to participate in the Thousand Validators Program. For us, we use something like `polkachu-kusama-01` and `polkachu-polkadot-02` to keep it unique and organized. -3. log_name: This is for your internal central monitoring server. We just use something like `kusama1` and `polkadot2` to keep it simple. -4. telemetryUrl: Most likely you will use `wss://telemetry-backend.w3f.community/submit/` -5. archive_node (optional): Set this to true if you want to run an archive node. An archive node is not required for a validator. An archive node has the complete chain data and requires much larger storage space. Most validators do not need an archive node. -6. chain_path (optional): You can set an alternative path to store chain data. This is especially useful when you run an archive node and want to store chain data on a mounted disk. A mounted disk offers more flexibility when you want to wrap disk, increase or decrease disk size, etc. -7. parity_db (optional): You can specify if you prefer to use the experimental ParityDB option in stead of the default RocksDB. - -You will also need to update: - -1. ansible_user: The sample file assumes `ansible`, but you might have another username. Make sure that the user has `sudo` privilege. -2. ansible_port: The sample file assumes `22`. But if you are like me, you will have a different ssh port other than `22` to avoid port sniffing. -3. ansible_ssh_private_key_file: The sample file assumes `~/.ssh/id_rsa`, but you might have a different key location. -4. log_monitor: Enter your monitor server IP. It is most likely a private IP address if you use a firewall around your private virtual cloud (VPC). - -It is beyond the scope of this guide to help you create a sudo user, alternate ssh port, create a private key, install Ansible on your machine, etc. You can do a quick online search and find the answers. In my experience, Digital Ocean have some quality guides on these topics. Stack Overflow can help you trouble-shoot if you are stuck. +1. ansible_host: Your server public IP +1. validator_name: This is the node name that will show up on telemetry monitoring board. It is especially important if you want to participate in the Thousand Validators Program. For us, we use something like `polkachu-kusama-01` and `polkachu-polkadot-02` to keep it unique and organized. +1. port_prefix: This allows you to install multiple nodes on the same server without port conflict ## Basic Cluster Structure @@ -56,77 +39,30 @@ The basic cluster structure is: 1. Name each Kusama node as `kusama1`, `kusama2`, etc. Group all Kusama nodes into `kusama` group. 2. Name each Polkadot node as `polkadot1`, `polkadot2`, etc. Group all Polkadot nodes into `polkadot` group. -3. Group all nodes into a `validators` group. The structure allows you to target `vars` to each node, or either Kusama or Polkadot cluster, or the whole cluster. -Make sure that you are familiar with the files in the `group_vars` folder. They follow this clustered structure closely. The files in this folder often need to be changed to stay up to date with the latest releases. I, for one, bump these program versions religiously so I live on the cutting edge! - ## Main Playbook to Set Up a Kusama/Polkadot Validator (Pruned Node) -The key Ansible playbook is `polkadot_full_setup.yml`. It will set up a fresh validator from scratch. Notice that it will restore from a snapshot from https://polkashots.io. It is very possible that you will get an error on the checksum of data to restore in your first attempt because the snapshot is updated regularly. When this happens, update the files accordingly. - The main setup playbook is: ```bash -ansible-playbook -i inventory polkadot_full_setup.yml -e "target=VALIDATOR_TARGET" +ansible-playbook -i inventory polkadot.yml -e "target=VALIDATOR_TARGET" ``` -Notice that you need to specify a target when you run this playbook (and other playbooks in this repo, as described in the next section). `VALIDATOR_TARGET` is a placeholder that could be a host (`kusama1`, `kusama2`, `polkadot1`, `polkadot2`, etc), a group (`kusama`, `polkadot`), or all validators (`validators`). This is intentionally designed to: +Notice that you need to specify a target when you run this playbook (and other playbooks in this repo, as described in the next section). `VALIDATOR_TARGET` is a placeholder that could be a host (`kusama1`, `kusama2`, `polkadot1`, `polkadot2`, etc), or a group (`kusama`, `polkadot`). This is intentionally designed to: 1. Prevent you from updating all nodes by mistake 2. Allow you to experiment a move on a low-risk node before rolling out to the whole cluster -## Main Playbook to Set Up a Kusama/Polkadot Archive Node - -The main setup playbook is: - -```bash -ansible-playbook -i inventory polkadot_full_archive_node_setup.yml -e "target=VALIDATOR_TARGET" -``` - -Most validators DO NOT need archive node. - -## A Pitfall - -We introduced pruned node / archive node toggle in the version 0.2.0 release. The database for pruned node and archive node is not compatible. If you have trouble start your `polkadot` service, a simple trouble-shooting method is just to delete the whole polkadot `db` directory. - ## Other Playbooks for Different Purposes The most commonly used playbooks are: -| Playbook | Description | -| ------------------------- | ------------------------------------------------------------------------------------------ | -| `polkadot_full_setup.yml` | Run the initial full setup | -| `polkadot_prepare.yml ` | Do the prep work, such as firewall, set up a proxy, copy service files, create users, etc. | -| `polkadot_update.yml` | Update the Polkadot binary and restart the service. You probably need to use it regularly | -| `polkadot_restore.yml` | Restore the Polkadot database with a screenshot. Only useful for initial setup | -| `node_exporter.yml` | Update Node Exporter | -| `process_exporter.yml` | Update Process Exporter | -| `promtail.yml` | Update Promtail | - -The less commonly used playbooks are: - -| Playbook | Description | -| ------------------------------ | ------------------------------------------------------------------------------------- | -| `polkadot_backup_keystore.yml` | Backup Keystore (Not sure about use case) | -| `polkadot_clean_logs.yml` | Clean journal logs (Probably useful when the disk is full) | -| `polkadot_restart.yml` | Restart Polkadot ad hoc (Probably useful when server runs wild for no obvious reason) | -| `polkadot_stop.yml` | Stop Polkadot ad hoc | -| `polkadot_rotate_key.yml` | Rotate session keys the easy way without you ssh into the server yourself | -| `snapshot_script.yml` | If you intend to use the node to take snapshot, then this script is for you | - -## Update All Servers - -One more thing! Sometimes you want to install all apt patches on all machines. I provide you with a simple playbook. Just run: - -```bash -ansible-playbook -i inventory all_apt_update.yml -``` +| Playbook | Description | +| ------------------ | ------------------------------------------------------------------------- | +| `prepare.yml ` | Do the prep work, such as ufw, node_exporter and promtail | +| `polkadot.yml` | Install Kusama/Polkadot node | +| `key_rotation.yml` | Rotate session keys the easy way without you ssh into the server yourself | That's it, folks! - -## Tips/Nominations Accepted - -- DOT: `15ym3MDSG4WPABNoEtx2rAzBB1EYWJDWbWYpNg1BwuWRAQcY` -- KSM: `CsKvJ4fdesaRALc5swo5iknFDpop7YUwKPJHdmUvBsUcMGb` diff --git a/all_apt_update.yml b/all_apt_update.yml deleted file mode 100644 index 24ac878..0000000 --- a/all_apt_update.yml +++ /dev/null @@ -1,10 +0,0 @@ ---- -- name: Update all servers - hosts: all - become: true - gather_facts: false - tasks: - - name: update and upgrade - apt: - upgrade: dist - update_cache: true diff --git a/ansible.cfg b/ansible.cfg index b625ed2..afc12c3 100644 --- a/ansible.cfg +++ b/ansible.cfg @@ -1,2 +1,2 @@ [defaults] -inventory = inventory \ No newline at end of file +inventory = inventory.ini \ No newline at end of file diff --git a/group_vars/all.yml b/group_vars/all.yml index 8c82b3c..2b20db6 100644 --- a/group_vars/all.yml +++ b/group_vars/all.yml @@ -1,17 +1,4 @@ -# Node Exporter -node_exporter_enabled: true -node_exporter_version: '1.1.2' -node_exporter_checksum: '8c1f6a317457a658e0ae68ad710f6b4098db2cad10204649b51e3c043aa3e70d' - -# Process Exporter -process_exporter_enabled: true -process_exporter_version: '0.7.5' -process_exporter_checksum: '27f133596205654a67b4a3e3af11db640f7d4609a457f48c155901835bd349c6' - -# Promtail -promtail_version: 2.2.1 -promtail_checksum: 40d8d414b44baa78c5010cb7575d74eea035b6b00adb78e9676a045d6730a16f - -# Digital Ocean Space for Snapshots (You can ignore if you do not plan to take snapshot from the node) -snapshot_endpoint: 'https://fra1.digitaloceanspaces.com' -snapshot_space: 'polkachu' +--- +node_exporter_version: '1.5.0' +promtail_version: '2.7.0' +polkadot_version: '0.9.36' diff --git a/group_vars/kusama.yml b/group_vars/kusama.yml index cf079a0..0c071a7 100644 --- a/group_vars/kusama.yml +++ b/group_vars/kusama.yml @@ -1,4 +1,3 @@ --- polkadot_network_id: ksmcc3 chain: kusama -polkadot_db_snapshot_url: 'https://substrate-snapshots.polkachu.xyz/kusama/kusama_12160135.tar.lz4' diff --git a/group_vars/polkadot.yml b/group_vars/polkadot.yml index e4febe7..f02a8b5 100644 --- a/group_vars/polkadot.yml +++ b/group_vars/polkadot.yml @@ -1,4 +1,3 @@ --- polkadot_network_id: polkadot chain: polkadot -polkadot_db_snapshot_url: 'https://substrate-snapshots.polkachu.xyz/polkadot_paritydb/polkadot_9494121.tar.lz4' diff --git a/group_vars/validators.yml b/group_vars/validators.yml deleted file mode 100644 index 47a2a2f..0000000 --- a/group_vars/validators.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -p2p_port: 30333 -polkadot_version: '0.9.36' diff --git a/inventory.sample b/inventory.sample deleted file mode 100644 index cee7c3e..0000000 --- a/inventory.sample +++ /dev/null @@ -1,25 +0,0 @@ -[kusama1] -192.168.1.10 validator_name=polkachu-kusama-test-0 log_name=kusama1 telemetryUrl=wss://telemetry.polkadot.io/submit/ - -[kusama1_paritydb] -192.168.1.11 validator_name=polkachu-kusama-test-0 parity_db=true log_name=kusama1 telemetryUrl=wss://telemetry.polkadot.io/submit/ - -[polkadot1] -192.168.1.20 validator_name=polkachu-polkadot-test-0 log_name=polkadot1 telemetryUrl=wss://telemetry.polkadot.io/submit/ chain_path="your_alternative_path_to_store_chain_data' archive_node=true - -[kusama:children] -kusama1 -kusama1_paritydb - -[polkadot:children] -polkadot1 - -[validators:children] -kusama -polkadot - -[all:vars] -ansible_user=ansible -ansible_port=22 -ansible_ssh_private_key_file="~/.ssh/id_rsa" -log_monitor=http://1.1.1.1:3100 \ No newline at end of file diff --git a/inventory.sample.ini b/inventory.sample.ini new file mode 100644 index 0000000..6442548 --- /dev/null +++ b/inventory.sample.ini @@ -0,0 +1,17 @@ +[kusama] +kusama01 ansible_host=10.0.0.1 validator_name=kusama01 port_prefix="100" +kusama02 ansible_host=10.0.0.1 validator_name=kusama01 port_prefix="101" + +[polkadot] +polkadot01 ansible_host=10.0.0.2 validator_name=polkadot01 port_prefix="102" +polkadot02 ansible_host=10.0.0.2 validator_name=polkadot01 port_prefix="103" + +[all:vars] +ansible_user=ubuntu +ansible_port=22 +ansible_ssh_private_key_file="~/.ssh/id_rsa" +log_monitor='YOUR_MONITOR_SERVER' +telemetryUrl=wss://telemetry-backend.w3f.community/submit/ +user_dir='/home/{{ ansible_user }}' +base_path='{{ user_dir}}/.{{ inventory_hostname }}' +log_name='{{ inventory_hostname }}' \ No newline at end of file diff --git a/polkadot_rotate_key.yml b/key_rotation.yml similarity index 72% rename from polkadot_rotate_key.yml rename to key_rotation.yml index c363dbf..12ac0d4 100644 --- a/polkadot_rotate_key.yml +++ b/key_rotation.yml @@ -4,4 +4,4 @@ become: true gather_facts: false roles: - - validator_polkadot_rotate_key + - key_rotation diff --git a/node_exporter.yml b/node_exporter.yml deleted file mode 100644 index 7ae79d6..0000000 --- a/node_exporter.yml +++ /dev/null @@ -1,7 +0,0 @@ ---- -- name: Update Node Exporter - hosts: "{{ target }}" - become: true - gather_facts: false - roles: - - node_exporter diff --git a/process_exporter.yml b/polkadot.yml similarity index 58% rename from process_exporter.yml rename to polkadot.yml index 4fedb6a..54e15d7 100644 --- a/process_exporter.yml +++ b/polkadot.yml @@ -1,7 +1,7 @@ --- -- name: Update Node Exporter +- name: Validator Full Setup hosts: '{{ target }}' become: true gather_facts: false roles: - - process_exporter + - polkadot diff --git a/polkadot_backup_keystore.yml b/polkadot_backup_keystore.yml deleted file mode 100644 index 2332d57..0000000 --- a/polkadot_backup_keystore.yml +++ /dev/null @@ -1,7 +0,0 @@ ---- -- name: Back up keystore - hosts: "{{ target }}" - become: true - gather_facts: false - roles: - - validator_polkadot_backup_keystore diff --git a/polkadot_clean_logs.yml b/polkadot_clean_logs.yml deleted file mode 100644 index 322bc06..0000000 --- a/polkadot_clean_logs.yml +++ /dev/null @@ -1,7 +0,0 @@ ---- -- name: Back up keystore - hosts: "{{ target }}" - become: true - gather_facts: false - roles: - - validator_polkadot_clean_logs diff --git a/polkadot_full_archive_node_setup.yml b/polkadot_full_archive_node_setup.yml deleted file mode 100644 index 08b546a..0000000 --- a/polkadot_full_archive_node_setup.yml +++ /dev/null @@ -1,29 +0,0 @@ ---- -- name: Wait for system ready - hosts: '{{ target }}' - gather_facts: false - become: true - tasks: - - name: check archive node flag - fail: - msg: 'you need to set the archive_node flag as true' - when: archive_node is not defined - - - name: Wait for nodes to become reachable - wait_for_connection: - - - name: Update and upgrade the machine - apt: - upgrade: dist - update_cache: true - -- name: Validator Archive Node Setup - hosts: '{{ target }}' - become: true - gather_facts: false - roles: - - validator_polkadot_prepare - - process_exporter - - node_exporter - - promtail - - validator_polkadot_update diff --git a/polkadot_full_setup.yml b/polkadot_full_setup.yml deleted file mode 100644 index 9cdcbb2..0000000 --- a/polkadot_full_setup.yml +++ /dev/null @@ -1,25 +0,0 @@ ---- -- name: Wait for system ready - hosts: '{{ target }}' - gather_facts: false - become: true - tasks: - - name: Wait for nodes to become reachable - wait_for_connection: - - - name: Update and upgrade the machine - apt: - upgrade: dist - update_cache: true - -- name: Validator Full Setup - hosts: '{{ target }}' - become: true - gather_facts: false - roles: - - validator_polkadot_prepare - - process_exporter - - node_exporter - - promtail - - validator_polkadot_update - - validator_polkadot_restore diff --git a/polkadot_restart.yml b/polkadot_restart.yml deleted file mode 100644 index 1c0ce71..0000000 --- a/polkadot_restart.yml +++ /dev/null @@ -1,7 +0,0 @@ ---- -- name: Polkadot Restart - hosts: "{{ target }}" - become: true - gather_facts: false - roles: - - validator_polkadot_restart diff --git a/polkadot_restore.yml b/polkadot_restore.yml deleted file mode 100644 index a92f834..0000000 --- a/polkadot_restore.yml +++ /dev/null @@ -1,7 +0,0 @@ ---- -- name: Polkadot Restore - hosts: '{{ target }}' - become: true - gather_facts: false - roles: - - validator_polkadot_restore diff --git a/polkadot_stop.yml b/polkadot_stop.yml deleted file mode 100644 index 8a61834..0000000 --- a/polkadot_stop.yml +++ /dev/null @@ -1,7 +0,0 @@ ---- -- name: Polkadot Stop - hosts: '{{ target }}' - become: true - gather_facts: false - roles: - - validator_polkadot_stop diff --git a/polkadot_update.yml b/polkadot_update.yml deleted file mode 100644 index 0cb3ba3..0000000 --- a/polkadot_update.yml +++ /dev/null @@ -1,7 +0,0 @@ ---- -- name: Validator Setup - hosts: '{{ target }}' - become: true - gather_facts: false - roles: - - validator_polkadot_update diff --git a/polkadot_prepare.yml b/prepare.yml similarity index 50% rename from polkadot_prepare.yml rename to prepare.yml index 0fd4b3e..06c28fa 100644 --- a/polkadot_prepare.yml +++ b/prepare.yml @@ -1,7 +1,9 @@ --- -- name: Validator preparation +- name: Node preparation hosts: '{{ target }}' become: true gather_facts: false roles: - - validator_polkadot_prepare + - prepare + - node_exporter + - promtail diff --git a/promtail.yml b/promtail.yml deleted file mode 100644 index e6a5ba1..0000000 --- a/promtail.yml +++ /dev/null @@ -1,7 +0,0 @@ ---- -- name: Update Promtail - hosts: "{{ target }}" - become: true - gather_facts: false - roles: - - promtail diff --git a/roles/validator_polkadot_rotate_key/tasks/main.yml b/roles/key_rotation/tasks/main.yml similarity index 75% rename from roles/validator_polkadot_rotate_key/tasks/main.yml rename to roles/key_rotation/tasks/main.yml index 31cad93..0cc9975 100644 --- a/roles/validator_polkadot_rotate_key/tasks/main.yml +++ b/roles/key_rotation/tasks/main.yml @@ -1,7 +1,7 @@ #curl -H "Content-Type: application/json" -d '{"id":1, "jsonrpc":"2.0", "method": "author_rotateKeys", "params":[]}' http://localhost:9933 - name: rotate server keys uri: - url: http://localhost:9933 + url: 'http://localhost:{{ port_prefix }}33' method: 'POST' body_format: json body: | @@ -15,4 +15,5 @@ - name: save rotateKeys output copy: content: '{{ rotate_keys.json }}' - dest: '/home/polkadot/rotate_keys.log' + dest: '{{ user_dir }}/{{ inventory_hostname }}_key_rotation.log' + owner: '{{ ansible_user }}' diff --git a/roles/node_exporter/tasks/main.yml b/roles/node_exporter/tasks/main.yml index 7d6eca6..989c2dd 100644 --- a/roles/node_exporter/tasks/main.yml +++ b/roles/node_exporter/tasks/main.yml @@ -1,20 +1,19 @@ - name: download node_exporter get_url: - url: "https://github.com/prometheus/node_exporter/releases/download/v{{ node_exporter_version }}/node_exporter-{{ node_exporter_version }}.linux-amd64.tar.gz" - checksum: "sha256:{{ node_exporter_checksum }}" - dest: "/tmp/node_exporter-{{ node_exporter_version }}.linux-amd64.tar.gz" - mode: "0700" + url: 'https://github.com/prometheus/node_exporter/releases/download/v{{ node_exporter_version }}/node_exporter-{{ node_exporter_version }}.linux-amd64.tar.gz' + dest: '/tmp/node_exporter-{{ node_exporter_version }}.linux-amd64.tar.gz' + mode: '0700' - name: unarchive node_exporter unarchive: - src: "/tmp/node_exporter-{{ node_exporter_version }}.linux-amd64.tar.gz" + src: '/tmp/node_exporter-{{ node_exporter_version }}.linux-amd64.tar.gz' remote_src: yes - dest: "/tmp" + dest: '/tmp' - name: copy node_exporter binary copy: - src: "/tmp/node_exporter-{{ node_exporter_version }}.linux-amd64/node_exporter" - dest: "/usr/local/bin/" + src: '/tmp/node_exporter-{{ node_exporter_version }}.linux-amd64/node_exporter' + dest: '/usr/local/bin/' remote_src: true owner: root group: root @@ -26,7 +25,10 @@ dest: /etc/systemd/system/node_exporter.service owner: root group: root - mode: "600" + mode: '600' + +- name: open node_exporter port + command: ufw allow 9100/tcp - name: start node_exporter service systemd: diff --git a/roles/polkadot/tasks/main.yml b/roles/polkadot/tasks/main.yml new file mode 100644 index 0000000..6c9d638 --- /dev/null +++ b/roles/polkadot/tasks/main.yml @@ -0,0 +1,47 @@ +--- +- name: Ensure base path exists + file: + path: '{{ base_path }}' + state: directory + mode: '0755' + owner: '{{ ansible_user }}' + group: '{{ ansible_user }}' + +- name: download polkadot binary + get_url: + url: 'https://github.com/paritytech/polkadot/releases/download/v{{polkadot_version}}/polkadot' + dest: '/tmp/polkadot' + mode: '0700' + +- name: copy polkadot binary + copy: + src: '/tmp/polkadot' + dest: '/usr/local/bin/' + remote_src: true + owner: '{{ ansible_user }}' + mode: 755 + +- name: create polkadot service file + template: + src: polkadot.service.j2 + dest: '/etc/systemd/system/{{ inventory_hostname }}.service' + owner: root + group: root + mode: 0744 + +- name: open prometheus and p2p ports + ufw: + rule: allow + port: '{{ item }}' + proto: tcp + loop: + - '{{ port_prefix }}16' + - '{{ port_prefix }}00' + +- name: start service + become: true + systemd: + name: '{{ inventory_hostname }}' + state: restarted + daemon_reload: yes + enabled: yes diff --git a/roles/polkadot/templates/polkadot.service.j2 b/roles/polkadot/templates/polkadot.service.j2 new file mode 100644 index 0000000..a08308c --- /dev/null +++ b/roles/polkadot/templates/polkadot.service.j2 @@ -0,0 +1,27 @@ + +[Unit] +Description=Polkadot Node + +[Service] +User={{ ansible_user }} +ExecStart=/usr/local/bin/polkadot \ + --base-path {{ base_path }} \ + --validator \ + --rpc-methods=Unsafe \ + --state-pruning=1000 \ + --blocks-pruning=1000 \ + -lsync=warn,afg=warn,babe=warn \ + --name {{ validator_name }} \ + --chain={{ chain }} \ + --port {{ port_prefix }}00 \ + --rpc-port {{ port_prefix}}33 \ + --ws-port {{ port_prefix}}44 \ + --prometheus-port {{ port_prefix}}16 \ + --prometheus-external \ + --telemetry-url '{{ telemetryUrl }} 1' + +Restart=always +RestartSec=60 + +[Install] +WantedBy=multi-user.target diff --git a/roles/validator_polkadot_prepare/files/journald.conf b/roles/prepare/files/journald.conf similarity index 100% rename from roles/validator_polkadot_prepare/files/journald.conf rename to roles/prepare/files/journald.conf diff --git a/roles/validator_polkadot_prepare/files/nginx.conf b/roles/prepare/files/nginx.conf similarity index 100% rename from roles/validator_polkadot_prepare/files/nginx.conf rename to roles/prepare/files/nginx.conf diff --git a/roles/validator_polkadot_prepare/tasks/nginx.yml b/roles/prepare/tasks/main.yml similarity index 51% rename from roles/validator_polkadot_prepare/tasks/nginx.yml rename to roles/prepare/tasks/main.yml index b728d82..156772e 100644 --- a/roles/validator_polkadot_prepare/tasks/nginx.yml +++ b/roles/prepare/tasks/main.yml @@ -1,9 +1,12 @@ --- -- name: install nginx package +- name: install packages apt: - name: nginx + name: '{{ item }}' state: present update_cache: yes + loop: + - ufw + - nginx - name: create nginx config copy: @@ -15,11 +18,6 @@ path: /etc/nginx/sites-enabled/default state: absent -- name: create polkadot_metrics proxy conf - copy: - src: polkadot_metrics.nginx.conf - dest: /etc/nginx/sites-enabled/polkadot_metrics.conf - - name: restart nginx service systemd: name: nginx @@ -27,3 +25,20 @@ daemon_reload: yes enabled: yes changed_when: false + +- name: Enable firewall + ufw: + state: enabled + +- name: copy journald config + copy: + src: journald.conf + dest: /etc/systemd/journald.conf + +- name: restart journald + systemd: + name: systemd-journald + state: restarted + daemon_reload: yes + enabled: yes + changed_when: false diff --git a/roles/process_exporter/files/process_exporter.service b/roles/process_exporter/files/process_exporter.service deleted file mode 100644 index 4b2ee14..0000000 --- a/roles/process_exporter/files/process_exporter.service +++ /dev/null @@ -1,12 +0,0 @@ -[Unit] -Description=Node Exporter - -[Service] -User=root -Group=root -ExecStart=/usr/local/bin/process-exporter --config.path /etc/process-exporter/config.yml - -Restart=always - -[Install] -WantedBy=multi-user.target diff --git a/roles/process_exporter/files/process_exporter_config.yml b/roles/process_exporter/files/process_exporter_config.yml deleted file mode 100644 index 0c78115..0000000 --- a/roles/process_exporter/files/process_exporter_config.yml +++ /dev/null @@ -1,4 +0,0 @@ -process_names: - - name: "{{.Comm}}" - cmdline: - - ".+" diff --git a/roles/process_exporter/tasks/main.yml b/roles/process_exporter/tasks/main.yml deleted file mode 100644 index b19dcc3..0000000 --- a/roles/process_exporter/tasks/main.yml +++ /dev/null @@ -1,56 +0,0 @@ -- name: download process_exporter - get_url: - url: "https://github.com/ncabatoff/process-exporter/releases/download/v{{ process_exporter_version }}/process-exporter-{{ process_exporter_version }}.linux-amd64.tar.gz" - checksum: "sha256:{{ process_exporter_checksum }}" - dest: "/tmp/process-exporter-{{ process_exporter_version }}.linux-amd64.tar.gz" - mode: "0700" - -- name: unarchive process_exporter - unarchive: - src: "/tmp/process-exporter-{{ process_exporter_version }}.linux-amd64.tar.gz" - remote_src: yes - dest: "/tmp" - -- name: copy process_exporter binary - copy: - src: "/tmp/process-exporter-{{ process_exporter_version }}.linux-amd64/process-exporter" - dest: "/usr/local/bin/" - remote_src: true - owner: root - group: root - mode: 755 - -- name: create process_exporter systemd unit - copy: - src: process_exporter.service - dest: /etc/systemd/system/process_exporter.service - owner: root - group: root - mode: "600" - -- name: Creates Process Exporter directory - file: - path: "/etc/process-exporter" - state: directory - owner: root - group: root - recurse: true - mode: 0755 - changed_when: false - -- name: copy config file - copy: - src: process_exporter_config.yml - dest: /etc/process-exporter/config.yml - owner: root - group: root - mode: "600" - changed_when: false - -- name: start process_exporter service - systemd: - name: process_exporter - state: restarted - daemon_reload: yes - enabled: yes - changed_when: false diff --git a/roles/promtail/tasks/main.yml b/roles/promtail/tasks/main.yml index cb5fc8b..346b3f0 100644 --- a/roles/promtail/tasks/main.yml +++ b/roles/promtail/tasks/main.yml @@ -2,14 +2,14 @@ - name: Create Promtail user user: name: promtail - comment: "Promtail user" + comment: 'Promtail user' shell: /usr/sbin/nologin - groups: ["systemd-journal", "adm"] + groups: ['systemd-journal', 'adm'] state: present - name: Creates Promtail directory file: - path: "/etc/promtail" + path: '/etc/promtail' state: directory owner: promtail group: promtail @@ -24,21 +24,20 @@ - name: download Promtail get_url: - url: "https://github.com/grafana/loki/releases/download/v{{ promtail_version }}/promtail-linux-amd64.zip" - checksum: "sha256:{{ promtail_checksum }}" - dest: "/tmp/promtail-linux-amd64.zip" - mode: "0700" + url: 'https://github.com/grafana/loki/releases/download/v{{ promtail_version }}/promtail-linux-amd64.zip' + dest: '/tmp/promtail-linux-amd64.zip' + mode: '0700' - name: unarchive promtail unarchive: - src: "/tmp/promtail-linux-amd64.zip" + src: '/tmp/promtail-linux-amd64.zip' remote_src: yes - dest: "/tmp" + dest: '/tmp' - name: Copy binary file copy: - src: "/tmp/promtail-linux-amd64" - dest: "/usr/local/bin/promtail" + src: '/tmp/promtail-linux-amd64' + dest: '/usr/local/bin/promtail' remote_src: true owner: promtail group: promtail @@ -46,15 +45,15 @@ - name: Copy promtail config template: - src: "promtail.yml.j2" - dest: "/etc/promtail/promtail.yml" + src: 'promtail.yml.j2' + dest: '/etc/promtail/promtail.yml' owner: promtail group: promtail - name: Copy promtail service file copy: - src: "promtail.service" - dest: "/etc/systemd/system/promtail.service" + src: 'promtail.service' + dest: '/etc/systemd/system/promtail.service' owner: root group: root mode: 600 diff --git a/roles/system_update/tasks/main.yml b/roles/system_update/tasks/main.yml deleted file mode 100644 index f2781f3..0000000 --- a/roles/system_update/tasks/main.yml +++ /dev/null @@ -1,25 +0,0 @@ ---- -- name: Update apt repo and cache - apt: - update_cache: true - cache_valid_time: 3600 - -- name: Upgrade all packages on servers - apt: - upgrade: dist - -- name: Check if a reboot is needed on all servers - stat: - path: '/var/run/reboot-required' - get_md5: no - register: reboot_required_file - -- name: Reboot the box if kernel updated - reboot: - msg: 'Reboot initiated by Ansible for kernel updates' - connect_timeout: 5 - reboot_timeout: 300 - pre_reboot_delay: 0 - post_reboot_delay: 30 - test_command: uptime - when: reboot_required_file.stat.exists diff --git a/roles/validator_polkadot_backup_keystore/tasks/main.yml b/roles/validator_polkadot_backup_keystore/tasks/main.yml deleted file mode 100644 index 2b44bb6..0000000 --- a/roles/validator_polkadot_backup_keystore/tasks/main.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- -- name: backup keystore - copy: - src: "{{ chain_path | default('/home/polkadot/.local/share/polkadot') }}/chains/{{ polkadot_network_id }}/keystore" - dest: '/home/polkadot/keystore' - remote_src: yes diff --git a/roles/validator_polkadot_clean_logs/tasks/main.yml b/roles/validator_polkadot_clean_logs/tasks/main.yml deleted file mode 100644 index 6ec61a8..0000000 --- a/roles/validator_polkadot_clean_logs/tasks/main.yml +++ /dev/null @@ -1,7 +0,0 @@ ---- -- name: journalctl vacuum - shell: | - set -o pipefail - journalctl --vacuum-time=2d - args: - executable: /bin/bash diff --git a/roles/validator_polkadot_prepare/files/polkadot_metrics.nginx.conf b/roles/validator_polkadot_prepare/files/polkadot_metrics.nginx.conf deleted file mode 100644 index 837ce32..0000000 --- a/roles/validator_polkadot_prepare/files/polkadot_metrics.nginx.conf +++ /dev/null @@ -1,6 +0,0 @@ -server { - listen 0.0.0.0:9616; - location / { - proxy_pass http://localhost:9615/; - } -} \ No newline at end of file diff --git a/roles/validator_polkadot_prepare/tasks/firewall.yml b/roles/validator_polkadot_prepare/tasks/firewall.yml deleted file mode 100644 index 91d43d1..0000000 --- a/roles/validator_polkadot_prepare/tasks/firewall.yml +++ /dev/null @@ -1,41 +0,0 @@ ---- -- name: install packages - apt: - name: ufw - state: present - update_cache: yes - -- name: ufw already enabled - command: ufw status verbose - register: ufw_status_result - changed_when: False - -- name: open node_exporter port - command: ufw allow 9100/tcp - when: - - node_exporter_enabled|default(false)|bool - - not ufw_status_result.stdout is search("9100/tcp.*ALLOW IN.*Anywhere") - -- name: open process_exporter port - command: ufw allow 9256/tcp - when: - - process_exporter_enabled|default(false)|bool - - not ufw_status_result.stdout is search("9256/tcp.*ALLOW IN.*Anywhere") - -- name: open polkadot_metrics port - command: ufw allow 9616/tcp - when: - - not ufw_status_result.stdout is search("9616/tcp.*ALLOW IN.*Anywhere") - -- name: open p2p port - command: ufw allow {{ p2p_port }}/tcp - when: - - not ufw_status_result.stdout is search(p2p_port ~ "/tcp.*ALLOW IN.*Anywhere") - -- name: enable firewall - shell: | - set -o pipefail - echo "y" | ufw enable - args: - executable: /bin/bash - when: not ufw_status_result.stdout is search("Status.* active") diff --git a/roles/validator_polkadot_prepare/tasks/journald.yml b/roles/validator_polkadot_prepare/tasks/journald.yml deleted file mode 100644 index 4ae2064..0000000 --- a/roles/validator_polkadot_prepare/tasks/journald.yml +++ /dev/null @@ -1,13 +0,0 @@ ---- -- name: copy journald config - copy: - src: journald.conf - dest: /etc/systemd/journald.conf - -- name: restart journald - systemd: - name: systemd-journald - state: restarted - daemon_reload: yes - enabled: yes - changed_when: false diff --git a/roles/validator_polkadot_prepare/tasks/main.yml b/roles/validator_polkadot_prepare/tasks/main.yml deleted file mode 100644 index 1732a12..0000000 --- a/roles/validator_polkadot_prepare/tasks/main.yml +++ /dev/null @@ -1,25 +0,0 @@ -- name: Update and upgrade the machine - apt: - upgrade: dist - update_cache: true - -- name: Install nvstat - apt: - name: vnstat - state: present - update_cache: yes - -- name: firewall setup - import_tasks: firewall.yml - -- name: journald config - import_tasks: journald.yml - -- name: user and group - import_tasks: user.yml - -- name: Nginx setup - import_tasks: nginx.yml - -- name: service setup - import_tasks: service.yml diff --git a/roles/validator_polkadot_prepare/tasks/service.yml b/roles/validator_polkadot_prepare/tasks/service.yml deleted file mode 100644 index 1e5cc09..0000000 --- a/roles/validator_polkadot_prepare/tasks/service.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -- name: create polkadot service file - template: - src: polkadot.service.j2 - dest: /etc/systemd/system/polkadot.service - owner: root - group: root - mode: 0600 diff --git a/roles/validator_polkadot_prepare/tasks/user.yml b/roles/validator_polkadot_prepare/tasks/user.yml deleted file mode 100644 index 8800040..0000000 --- a/roles/validator_polkadot_prepare/tasks/user.yml +++ /dev/null @@ -1,23 +0,0 @@ ---- -- name: create polkadot group - group: - name: polkadot - state: present - -- name: add polkadot user - user: - name: polkadot - groups: ['polkadot'] - append: yes - shell: /bin/bash - -- name: set folder permission if there is an alternative base_path for polkadot - file: - path: '{{ chain_path }}' - state: directory - owner: polkadot - group: polkadot - recurse: true - mode: 0755 - when: chain_path is defined - changed_when: false diff --git a/roles/validator_polkadot_prepare/templates/polkadot.service.j2 b/roles/validator_polkadot_prepare/templates/polkadot.service.j2 deleted file mode 100644 index 5b44951..0000000 --- a/roles/validator_polkadot_prepare/templates/polkadot.service.j2 +++ /dev/null @@ -1,43 +0,0 @@ -{% set execution = hostvars[inventory_hostname].execution|default(None) %} -{% set wasm_execution = hostvars[inventory_hostname].wasm_execution|default(None) %} -{% set archive_node = hostvars[inventory_hostname].archive_node|default(false) %} -{% set chain_path = hostvars[inventory_hostname].chain_path|default(None) %} -{% set parity_db = hostvars[inventory_hostname].parity_db|default(false) %} - -[Unit] -Description=Polkadot Node - -[Service] -User=polkadot -Group=polkadot -ExecStart=/usr/local/bin/polkadot \ - {% if execution is not none and execution|length %} - --execution {{ execution }} \ - {% endif %} - {% if wasm_execution is not none and wasm_execution|length %} - --wasm-execution {{ wasm_execution }} \ - {% endif %} - {% if chain_path is not none and chain_path|length %} - --base-path {{ chain_path }} \ - {% endif %} - --validator \ - --rpc-methods=Unsafe \ - {% if archive_node is not none and archive_node | bool is sameas true %} - --pruning=archive \ - {% else %} - --state-pruning=1000 \ - --blocks-pruning=1000 \ - {% endif %} - {% if parity_db is not none and parity_db | bool is sameas true %} - --database paritydb-experimental \ - {% endif %} - -lsync=warn,afg=warn,babe=warn \ - --name {{ validator_name }} \ - --chain={{ chain }} \ - --telemetry-url '{{ telemetryUrl }} 1' - -Restart=always -RestartSec=60 - -[Install] -WantedBy=multi-user.target diff --git a/roles/validator_polkadot_restart/tasks/main.yml b/roles/validator_polkadot_restart/tasks/main.yml deleted file mode 100644 index e262fa0..0000000 --- a/roles/validator_polkadot_restart/tasks/main.yml +++ /dev/null @@ -1,21 +0,0 @@ -- name: restart polkadot service - systemd: - name: polkadot.service - state: restarted - daemon_reload: yes - enabled: yes - changed_when: false - -#curl -H "Content-Type: application/json" -d '{"id":1, "jsonrpc":"2.0", "method": "system_health", "params":[]}' http://localhost:9933 -- name: wait for polkadot service started - uri: - url: http://localhost:9933 - status_code: '200' - method: 'POST' - body_format: json - body: | - { "jsonrpc":"2.0", "method":"system_health", "params":[], "id":1 } - register: result - until: result.status == 200 - retries: 12 - delay: 5 diff --git a/roles/validator_polkadot_restore/tasks/main.yml b/roles/validator_polkadot_restore/tasks/main.yml deleted file mode 100644 index eb22b6d..0000000 --- a/roles/validator_polkadot_restore/tasks/main.yml +++ /dev/null @@ -1,58 +0,0 @@ -- name: Install snapd - apt: - name: snapd - state: present - update_cache: yes - -- name: Install lz4 - snap: - name: lz4 - state: present - -- name: db_dest_path as fact - set_fact: - db_dest_path: "{{ chain_path | default('/home/polkadot/.local/share/polkadot') }}/chains/{{ polkadot_network_id }}" - -- name: download db - get_url: - url: '{{ polkadot_db_snapshot_url }}' - dest: '{{ db_dest_path }}/db.tar.lz4' - mode: '0700' - owner: 'polkadot' - group: 'polkadot' - -- name: stop polkadot service - systemd: - name: polkadot.service - state: stopped - -- name: unpack db if rocksDB - shell: | - set -o pipefail - cd {{ db_dest_path }} - mv db db.back - lz4 -c -d db.tar.lz4 | tar -x -C . - rm db.tar.lz4 - chown -R polkadot:polkadot db/ - args: - executable: /bin/bash - when: - - not parity_db|default(false)|bool - -- name: unpack db if paritydb - shell: | - set -o pipefail - cd {{ db_dest_path }} - mv paritydb paritydb.back - lz4 -c -d db.tar.lz4 | tar -x -C . - rm db.tar.lz4 - chown -R polkadot:polkadot paritydb/ - args: - executable: /bin/bash - when: - - parity_db|default(false)|bool - -- name: start polkadot service - systemd: - name: polkadot.service - state: started diff --git a/roles/validator_polkadot_stop/tasks/main.yml b/roles/validator_polkadot_stop/tasks/main.yml deleted file mode 100644 index db73dd6..0000000 --- a/roles/validator_polkadot_stop/tasks/main.yml +++ /dev/null @@ -1,4 +0,0 @@ -- name: Stop polkadot service - systemd: - name: polkadot.service - state: stopped diff --git a/roles/validator_polkadot_update/tasks/main.yml b/roles/validator_polkadot_update/tasks/main.yml deleted file mode 100644 index 747a748..0000000 --- a/roles/validator_polkadot_update/tasks/main.yml +++ /dev/null @@ -1,30 +0,0 @@ -- name: download polkadot binary - get_url: - url: 'https://github.com/paritytech/polkadot/releases/download/v{{polkadot_version}}/polkadot' - dest: /usr/local/bin/polkadot-new - force: true - mode: '0700' - owner: 'polkadot' - group: 'polkadot' - -- name: stop polkadot service - systemd: - name: polkadot.service - state: stopped - changed_when: false - -- name: substitute new polkadot binary - shell: | - set -o pipefail - if [ -f /usr/local/bin/polkadot-new ]; then - cp /usr/local/bin/polkadot-new /usr/local/bin/polkadot - chown polkadot:polkadot /usr/local/bin/polkadot - chmod 755 /usr/local/bin/polkadot - fi - args: - executable: /bin/bash - changed_when: False - -- name: restart polkadot service - import_role: - name: validator_polkadot_restart diff --git a/system_update.yml b/system_update.yml deleted file mode 100644 index 5e307b1..0000000 --- a/system_update.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -- hosts: '{{ target }}' - become: true - roles: - - system_update