Restore nodes from individual EBS snapshots (#44)

* Add ability to restore from a snapshots Co-authored-by: Mike Krasnianskyi <[email protected]>
sky-uk · Mar 27, 2020 · ebce75b · ebce75b
1 parent 27e132e
commit ebce75b
Show file tree

Hide file tree

Showing 3 changed files with 41 additions and 1 deletion.
diff --git a/cluster_hosts/tasks/aws.yml b/cluster_hosts/tasks/aws.yml
@@ -145,3 +145,42 @@
         {% endif %}
       {%- endfor %}
       {{ res }}
+
+- block:
+  - name: Get snapshots info
+    ec2_snapshot_info:
+      aws_access_key: "{{cluster_vars[buildenv].aws_access_key}}"
+      aws_secret_key: "{{cluster_vars[buildenv].aws_secret_key}}"
+      region: "{{ cluster_vars.region }}"
+      filters: "{{ _snapshot_tags[0] }}"
+    register: r__ebs_snapshots
+
+  - name: Assert that snapshots exists
+    assert:
+      that:
+        - item in _available_snapshots
+      quiet: true
+      fail_msg: "{{ item }}  not in available snapshots {{ _available_snapshots }}"
+    loop: "{{ _configuration_snapshots }}"
+    vars:
+      _available_snapshots: "{{ r__ebs_snapshots.snapshots|json_query('[].tags.backup_id')|unique }}"
+      _configuration_snapshots: "{{ cluster_vars[buildenv].hosttype_vars|json_query('*.auto_volumes[].snapshot_tags.*[]')|unique }}"
+
+  - name: update cluster_hosts_flat with snapshot_id 
+    set_fact:
+      cluster_hosts_flat: |
+        {%- for host in cluster_hosts_flat -%}
+          {%- set cluster_host_topology = host.hostname | regex_replace('^.*(-.*?-).*$', '\\1') -%}
+          {%- for vol in host.auto_volumes -%}
+            {%- set cur_snapshot = r__ebs_snapshots | default([]) | to_json | from_json | json_query('snapshots[?contains(tags.Name, \'' + cluster_host_topology + '\')]') -%}
+            {%- if cur_snapshot and 'snapshot_tags' in vol.keys() -%}
+              {%- set _dummy = vol.update({'shapshot_id': cur_snapshot[0].snapshot_id}) -%}
+              {%- set _dummy = vol.pop('snapshot_tags') -%}
+            {%- endif %}
+          {%- endfor %}
+        {%- endfor %}
+        {{ cluster_hosts_flat }}
+  vars:
+    _snapshot_tags: "{{ cluster_vars[buildenv].hosttype_vars|json_query('*.auto_volumes[].snapshot_tags')  }}"
+  when: 
+    - _snapshot_tags|length > 0
diff --git a/cluster_hosts/tasks/main.yml b/cluster_hosts/tasks/main.yml
@@ -16,6 +16,7 @@
               'hostname': cluster_name + '-' + hostttype + '-' + azname + azcount|string,
               'az_name': azname|string,
               'flavor': cluster_vars[buildenv].hosttype_vars[hostttype].flavor,
+              'auto_volumes': cluster_vars[buildenv].hosttype_vars[hostttype].auto_volumes,
               'release': release_version
               }]) -%}
           {%- endfor %}

diff --git a/create/tasks/aws.yml b/create/tasks/aws.yml
@@ -43,7 +43,7 @@
           owner: "{{ lookup('env','USER')| lower }}"
           maintenance_mode: "{%- if prometheus_set_unset_maintenance_mode|bool -%}true{%- else -%}false{%- endif -%}"
         termination_protection: "{{cluster_vars[buildenv].termination_protection}}"
-        volumes: "{{cluster_vars[buildenv].hosttype_vars[item.hosttype].auto_volumes | default([])}}"
+        volumes: "{{ item.auto_volumes | default([]) }}"
         count_tag:
           Name: "{{item.hostname}}"
         exact_count: 1