From 409c8704c51ef5af0d81f9186ccf29cf6a80d4a6 Mon Sep 17 00:00:00 2001 From: Songhua Hu Date: Sat, 3 Jul 2021 14:08:36 -0700 Subject: [PATCH] Add archive_node toggle and chain_path option --- .gitignore | 3 +- README.md | 46 +++++++++++++------ group_vars/polkadot.yml | 4 +- inventory.sample | 2 +- polkadot_full_archive_node_setup.yml | 29 ++++++++++++ polkadot_full_setup.yml | 6 +-- .../tasks/main.yml | 4 +- .../validator_polkadot_prepare/tasks/user.yml | 13 +++++- .../templates/polkadot.service.j2 | 9 ++++ .../validator_polkadot_restart/tasks/main.yml | 4 +- .../validator_polkadot_restore/tasks/main.yml | 14 +++--- .../tasks/main.yml | 6 +-- .../validator_polkadot_update/tasks/main.yml | 10 ++-- 13 files changed, 108 insertions(+), 42 deletions(-) create mode 100644 polkadot_full_archive_node_setup.yml diff --git a/.gitignore b/.gitignore index d040382..d3d0b26 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ .vagrant/ -inventory \ No newline at end of file +inventory +polkadot_debug.yml \ No newline at end of file diff --git a/README.md b/README.md index 9719723..b8b0097 100644 --- a/README.md +++ b/README.md @@ -33,10 +33,12 @@ cp inventory.sample inventory Needless to say, you need to update the dummy values in the inventory file. For each Kusama/Polkadot node, you need to update: -1. Server IP -2. validator_name (especially important if you want to participate in the Thousand Validators Program) -3. log_name (so your central monitoring server knows which node it is) -4. telemetryUrl +1. Server IP: Your server public IP +2. validator_name: This is the node name that will show up on telemtry monitoring board. It is especially important if you want to participate in the Thousand Validators Program. For us, we use something like `polkachu-kusama-01` and `polkachu-polkadot-02` to keep it unique and organized. +3. log_name: This is for your internal central monitoring server. We just use something like `kusama1` and `polkadot2` to keep it simple. +4. telemetryUrl: Most likely you will use `wss://telemetry-backend.w3f.community/submit/` +5. archive_node (optional): Set this to true if you want to run an archive node. An archive node is not required for a validator. An archive node has the complete chain data and requires much larger storage space. Most validators do not need an archive node. +6. chain_path (optional): You can set an alternative path to store chain data. This is especially useful when you run an archive node and want to store chain data on a mounted disk. A mounted disk offers more flexibility when you want to wrap disk, increase or decrease disk size, etc. You will also need to update: @@ -59,7 +61,7 @@ The structure allows you to target `vars` to each node, or either Kusama or Polk Make sure that you are familiar with the files in the `group_vars` folder. They follow this clustered structure closely. The files in this folder often need to be changed to stay up to date with the latest releases. I, for one, bump these program versions religiously so I live on the cutting edge! -## Main Playbook to Set Up a Kusama/Polkadot Validator (Archive Node) +## Main Playbook to Set Up a Kusama/Polkadot Validator (Pruned Node) The key Ansible playbook is `polkadot_full_setup.yml`. It will set up a fresh validator from scratch. Notice that it will restore from a snapshot from https://polkashots.io. It is very possible that you will get an error on the checksum of data to restore in your first attempt because the snapshot is updated regularly. When this happens, update the files accordingly. @@ -74,19 +76,33 @@ Notice that you need to specify a target when you run this playbook (and other p 1. Prevent you from updating all nodes by mistake 2. Allow you to experiment a move on a low-risk node before rolling out to the whole cluster -## Various Playbooks for Different Purposes +## Main Playbook to Set Up a Kusama/Polkadot Pruned Node + +The main setup playbook is: + +```bash +ansible-playbook -i inventory polkadot_full_archive_node_setup.yml -e "target=VALIDATOR_TARGET" +``` + +Most validators DO NOT need archive node. + +## A Pitfall + +We introduced pruned node / archive node toggle in the version 0.2.0 release. The database for pruned node and archive node is not compatible. If you have trouble start your `polkadot` service, a simple trouble-shooting method is just to delete the whole polkadot `db` directory. + +## Other Playbooks for Different Purposes The most commonly used playbooks are: -| Playbook | Description | -| ------------------------- | ----------------------------------------------------------------------------------------- | -| `polkadot_full_setup.yml` | Run the initial full setup | -| `polkadot_prepare.yml ` | Do the prep work, such as firewall, set up a proxy, copy service files, create users, etc.| -| `polkadot_update.yml` | Update the Polkadot binary and restart the service. You probably need to use it regularly | -| `polkadot_restore.yml` | Restore the Polkadot database with a screenshot. Only useful for initial setup | -| `node_exporter.yml` | Update Node Exporter | -| `process_exporter.yml` | Update Process Exporter | -| `promtail.yml` | Update Promtail | +| Playbook | Description | +| ------------------------- | ------------------------------------------------------------------------------------------ | +| `polkadot_full_setup.yml` | Run the initial full setup | +| `polkadot_prepare.yml ` | Do the prep work, such as firewall, set up a proxy, copy service files, create users, etc. | +| `polkadot_update.yml` | Update the Polkadot binary and restart the service. You probably need to use it regularly | +| `polkadot_restore.yml` | Restore the Polkadot database with a screenshot. Only useful for initial setup | +| `node_exporter.yml` | Update Node Exporter | +| `process_exporter.yml` | Update Process Exporter | +| `promtail.yml` | Update Promtail | The less commonly used playbooks are: diff --git a/group_vars/polkadot.yml b/group_vars/polkadot.yml index 12f3acd..97f6840 100644 --- a/group_vars/polkadot.yml +++ b/group_vars/polkadot.yml @@ -1,8 +1,8 @@ --- polkadot_network_id: polkadot chain: polkadot -polkadot_db_snapshot_url: 'https://dot-rocksdb.polkashots.io/polkadot-5547438.RocksDb.7z' -polkadot_db_snapshot_checksum: '441f6a623a1f68e2afa558b3432e504955bae640d2155a05ddf7460452472dec' +polkadot_db_snapshot_url: 'https://dot-rocksdb.polkashots.io/polkadot-5771552.RocksDb.7z' +polkadot_db_snapshot_checksum: '02e6bff302ff9c43db649fb08a3576fee8adcc38c0c787ae08386a9ee3fdec25' # Emergence use when we need to downgrade polkadot # polkadot_version: "0.8.30" # polkadot_checksum: "9dddd2ede827865c6e81684a138b0f282319e07f717c166b92834699f43274cd" diff --git a/inventory.sample b/inventory.sample index 4032fdf..9339481 100644 --- a/inventory.sample +++ b/inventory.sample @@ -2,7 +2,7 @@ 192.168.1.10 validator_name=polkachu-kusama-test-0 log_name=kusama1 telemetryUrl=wss://telemetry.polkadot.io/submit/ [polkadot1] -192.168.1.20 validator_name=polkachu-polkadot-test-0 log_name=polkadot1 telemetryUrl=wss://telemetry.polkadot.io/submit/ +192.168.1.20 validator_name=polkachu-polkadot-test-0 log_name=polkadot1 telemetryUrl=wss://telemetry.polkadot.io/submit/ chain_path="your_alternative_path_to_store_chain_data' archive_node=true [kusama:children] kusama1 diff --git a/polkadot_full_archive_node_setup.yml b/polkadot_full_archive_node_setup.yml new file mode 100644 index 0000000..08b546a --- /dev/null +++ b/polkadot_full_archive_node_setup.yml @@ -0,0 +1,29 @@ +--- +- name: Wait for system ready + hosts: '{{ target }}' + gather_facts: false + become: true + tasks: + - name: check archive node flag + fail: + msg: 'you need to set the archive_node flag as true' + when: archive_node is not defined + + - name: Wait for nodes to become reachable + wait_for_connection: + + - name: Update and upgrade the machine + apt: + upgrade: dist + update_cache: true + +- name: Validator Archive Node Setup + hosts: '{{ target }}' + become: true + gather_facts: false + roles: + - validator_polkadot_prepare + - process_exporter + - node_exporter + - promtail + - validator_polkadot_update diff --git a/polkadot_full_setup.yml b/polkadot_full_setup.yml index 387959f..9cdcbb2 100644 --- a/polkadot_full_setup.yml +++ b/polkadot_full_setup.yml @@ -1,6 +1,6 @@ --- - name: Wait for system ready - hosts: "{{ target }}" + hosts: '{{ target }}' gather_facts: false become: true tasks: @@ -13,13 +13,13 @@ update_cache: true - name: Validator Full Setup - hosts: "{{ target }}" + hosts: '{{ target }}' become: true gather_facts: false roles: - validator_polkadot_prepare - - validator_polkadot_update - process_exporter - node_exporter - promtail + - validator_polkadot_update - validator_polkadot_restore diff --git a/roles/validator_polkadot_backup_keystore/tasks/main.yml b/roles/validator_polkadot_backup_keystore/tasks/main.yml index 4bbaf48..2b44bb6 100644 --- a/roles/validator_polkadot_backup_keystore/tasks/main.yml +++ b/roles/validator_polkadot_backup_keystore/tasks/main.yml @@ -1,6 +1,6 @@ --- - name: backup keystore copy: - src: "/home/polkadot/.local/share/polkadot/chains/{{ polkadot_network_id }}/keystore" - dest: /home/polkadot/keystore + src: "{{ chain_path | default('/home/polkadot/.local/share/polkadot') }}/chains/{{ polkadot_network_id }}/keystore" + dest: '/home/polkadot/keystore' remote_src: yes diff --git a/roles/validator_polkadot_prepare/tasks/user.yml b/roles/validator_polkadot_prepare/tasks/user.yml index 4c3aad3..8800040 100644 --- a/roles/validator_polkadot_prepare/tasks/user.yml +++ b/roles/validator_polkadot_prepare/tasks/user.yml @@ -7,6 +7,17 @@ - name: add polkadot user user: name: polkadot - groups: ["polkadot"] + groups: ['polkadot'] append: yes shell: /bin/bash + +- name: set folder permission if there is an alternative base_path for polkadot + file: + path: '{{ chain_path }}' + state: directory + owner: polkadot + group: polkadot + recurse: true + mode: 0755 + when: chain_path is defined + changed_when: false diff --git a/roles/validator_polkadot_prepare/templates/polkadot.service.j2 b/roles/validator_polkadot_prepare/templates/polkadot.service.j2 index 2f6c822..956e122 100644 --- a/roles/validator_polkadot_prepare/templates/polkadot.service.j2 +++ b/roles/validator_polkadot_prepare/templates/polkadot.service.j2 @@ -1,5 +1,7 @@ {% set execution = hostvars[inventory_hostname].execution|default(None) %} {% set wasm_execution = hostvars[inventory_hostname].wasm_execution|default(None) %} +{% set archive_node = hostvars[inventory_hostname].archive_node|default(false) %} +{% set chain_path = hostvars[inventory_hostname].chain_path|default(None) %} [Unit] Description=Polkadot Node @@ -14,10 +16,17 @@ ExecStart=/usr/local/bin/polkadot \ {% if wasm_execution is not none and wasm_execution|length %} --wasm-execution {{ wasm_execution }} \ {% endif %} + {% if chain_path is not none and chain_path|length %} + --base-path {{ chain_path }} \ + {% endif %} --validator \ --rpc-methods=Unsafe \ + {% if archive_node is not none and archive_node | bool is sameas true %} + --pruning=archive \ + {% else %} --unsafe-pruning \ --pruning=1000 \ + {% endif %} -lsync=warn,afg=warn,babe=warn \ --public-addr=/ip4/{{ hostvars[inventory_hostname].public_ip.json.ip }}/tcp/{{ proxy_port }} \ --listen-addr=/ip4/127.0.0.1/tcp/{{ p2p_port }} \ diff --git a/roles/validator_polkadot_restart/tasks/main.yml b/roles/validator_polkadot_restart/tasks/main.yml index 12ccbfa..e262fa0 100644 --- a/roles/validator_polkadot_restart/tasks/main.yml +++ b/roles/validator_polkadot_restart/tasks/main.yml @@ -10,8 +10,8 @@ - name: wait for polkadot service started uri: url: http://localhost:9933 - status_code: "200" - method: "POST" + status_code: '200' + method: 'POST' body_format: json body: | { "jsonrpc":"2.0", "method":"system_health", "params":[], "id":1 } diff --git a/roles/validator_polkadot_restore/tasks/main.yml b/roles/validator_polkadot_restore/tasks/main.yml index 00a5931..f56a005 100644 --- a/roles/validator_polkadot_restore/tasks/main.yml +++ b/roles/validator_polkadot_restore/tasks/main.yml @@ -6,16 +6,16 @@ - name: db_dest_path as fact set_fact: - db_dest_path: "/home/polkadot/.local/share/polkadot/chains/{{ polkadot_network_id }}" + db_dest_path: "{{ chain_path | default('/home/polkadot/.local/share/polkadot') }}/chains/{{ polkadot_network_id }}" - name: download db get_url: - url: "{{ polkadot_db_snapshot_url }}" - checksum: "sha256:{{ polkadot_db_snapshot_checksum | quote }}" - dest: "{{ db_dest_path }}/db.new.7z" - mode: "0700" - owner: "polkadot" - group: "polkadot" + url: '{{ polkadot_db_snapshot_url }}' + checksum: 'sha256:{{ polkadot_db_snapshot_checksum | quote }}' + dest: '{{ db_dest_path }}/db.new.7z' + mode: '0700' + owner: 'polkadot' + group: 'polkadot' - name: stop polkadot service systemd: diff --git a/roles/validator_polkadot_rotate_key/tasks/main.yml b/roles/validator_polkadot_rotate_key/tasks/main.yml index fdd91f9..31cad93 100644 --- a/roles/validator_polkadot_rotate_key/tasks/main.yml +++ b/roles/validator_polkadot_rotate_key/tasks/main.yml @@ -2,7 +2,7 @@ - name: rotate server keys uri: url: http://localhost:9933 - method: "POST" + method: 'POST' body_format: json body: | { "jsonrpc":"2.0", "method":"author_rotateKeys", "params":[], "id":1 } @@ -14,5 +14,5 @@ - name: save rotateKeys output copy: - content: "{{ rotate_keys.json }}" - dest: /home/polkadot/rotate_keys.log + content: '{{ rotate_keys.json }}' + dest: '/home/polkadot/rotate_keys.log' diff --git a/roles/validator_polkadot_update/tasks/main.yml b/roles/validator_polkadot_update/tasks/main.yml index b7ca2a2..1cb044f 100644 --- a/roles/validator_polkadot_update/tasks/main.yml +++ b/roles/validator_polkadot_update/tasks/main.yml @@ -1,12 +1,12 @@ - name: download polkadot binary get_url: - url: "https://github.com/paritytech/polkadot/releases/download/v{{polkadot_version}}/polkadot" - checksum: "sha256:{{ polkadot_checksum }}" + url: 'https://github.com/paritytech/polkadot/releases/download/v{{polkadot_version}}/polkadot' + checksum: 'sha256:{{ polkadot_checksum }}' dest: /usr/local/bin/polkadot-new force: true - mode: "0700" - owner: "polkadot" - group: "polkadot" + mode: '0700' + owner: 'polkadot' + group: 'polkadot' - name: stop polkadot service systemd: