Merge pull request #533 from deNBI/staging

Staging

VirtualMachineService/VirtualMachineHandler.py

@@ -23,6 +23,7 @@
         ALL_TEMPLATES,
         RSTUDIO,
         JUPYTERNOTEBOOK,
+        CWLAB,
     )
 
 except Exception:
@@ -44,6 +45,7 @@
         ALL_TEMPLATES,
         RSTUDIO,
         JUPYTERNOTEBOOK,
+        CWLAB,
     )
 
 import datetime
@@ -961,6 +963,15 @@ def prepare_security_groups_new_server(self, resenv, servername, http, https):
                     ssh=False,
                 ).name
             )
+        if CWLAB in resenv:
+            custom_security_groups.append(
+                self.create_security_group(
+                    name=servername + "_cwlab",
+                    resenv=resenv,
+                    description="CWLab",
+                    ssh=False,
+                ).name
+            )
         if JUPYTERNOTEBOOK in resenv:
             custom_security_groups.append(
                 self.create_security_group(
@@ -2610,6 +2621,16 @@ def create_security_group(
                 port_range_min=8787,
                 security_group_id=new_security_group["id"],
             )
+        if CWLAB in resenv:
+            LOG.info("Add cwlab rule to security group {}".format(name))
+
+            self.conn.network.create_security_group_rule(
+                direction="ingress",
+                protocol="tcp",
+                port_range_max=80,
+                port_range_min=80,
+                security_group_id=new_security_group["id"],
+            )
         if JUPYTERNOTEBOOK in resenv:
             LOG.info("Add jupyternotebook rule to security group {}".format(name))
 

VirtualMachineService/ancon/Playbook.py

@@ -12,10 +12,11 @@
 RSTUDIO = "rstudio"
 GUACAMOLE = "guacamole"
 JUPYTERNOTEBOOK = "jupyternotebook"
+CWLAB = "cwlab"
 OPTIONAL = "optional"
 MOSH = "mosh"
 
-ALL_TEMPLATES = [BIOCONDA, THEIA, RSTUDIO, GUACAMOLE, JUPYTERNOTEBOOK]
+ALL_TEMPLATES = [BIOCONDA, THEIA, RSTUDIO, GUACAMOLE, JUPYTERNOTEBOOK, CWLAB]
 
 LOG = logging.getLogger(__name__)
 LOG.setLevel(logging.DEBUG)
@@ -135,24 +136,35 @@ def load_vars():
                         for p in p_array:
                             p_dict.update({p[0]: {"version": p[1], "build": p[2]}})
                         data[playbook_name + "_tools"][k] = p_dict
-            if playbook_name == THEIA:
-                for k, v in playbook_vars.items():
-                    if k == "template_version":
-                        data[playbook_name + "_vars"][k] = v
-                    if k == "create_only_backend":
-                        data[playbook_name + "_vars"][k] = v
-            if playbook_name == RSTUDIO:
-                for k, v in playbook_vars.items():
-                    if k == "template_version":
-                        data[playbook_name + "_vars"][k] = v
-                    if k == "create_only_backend":
-                        data[playbook_name + "_vars"][k] = v
-            if playbook_name == GUACAMOLE:
+            if (
+                playbook_name == THEIA
+                or playbook_name == RSTUDIO
+                or playbook_name == GUACAMOLE
+                or playbook_name == CWLAB
+            ):
                 for k, v in playbook_vars.items():
                     if k == "template_version":
                         data[playbook_name + "_vars"][k] = v
                     if k == "create_only_backend":
                         data[playbook_name + "_vars"][k] = v
+            # if playbook_name == RSTUDIO:
+            #     for k, v in playbook_vars.items():
+            #         if k == "template_version":
+            #             data[playbook_name + "_vars"][k] = v
+            #         if k == "create_only_backend":
+            #             data[playbook_name + "_vars"][k] = v
+            # if playbook_name == GUACAMOLE:
+            #     for k, v in playbook_vars.items():
+            #         if k == "template_version":
+            #             data[playbook_name + "_vars"][k] = v
+            #         if k == "create_only_backend":
+            #             data[playbook_name + "_vars"][k] = v
+            # if playbook_name == CWLAB:
+            #     for k, v in playbook_vars.items():
+            #         if k == "template_version":
+            #             data[playbook_name + "_vars"][k] = v
+            #         if k == "create_only_backend":
+            #             data[playbook_name + "_vars"][k] = v
             if playbook_name == OPTIONAL:
                 for k, v in playbook_vars.items():
                     if k == MOSH:

VirtualMachineService/ancon/playbooks/cwlab.yml

@@ -0,0 +1,40 @@
+- name: Wait for automatic system updates 1
+  shell: while sudo fuser /var/lib/dpkg/lock >/dev/null 2>&1; do sleep 1; done;
+
+- name: Wait for automatic system updates 2
+  shell: while sudo fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1; do sleep 1; done;
+
+- name: Update apt cache
+  apt:
+    update_cache: true
+  become: yes
+
+- name: Install docker.io package
+  apt:
+    name: docker.io
+  become: yes
+
+- name: Add ubuntu user to docker group
+  user:
+    name: ubuntu
+    groups: docker
+    append: yes
+  become: yes
+
+- name: Install docker python dev package
+  apt:
+    name: python3-docker
+  become: yes
+
+- name: Launch cwlab container
+  community.general.docker_container:
+    name: cwlab
+    image: compepigen/cwlab:dev
+    ports:
+      - "{{ cwlab_vars.exposed_port }}:5000"
+    volumes:
+      - "{{ cwlab_vars.data_volume_path }}:/data"
+    restart_policy: always
+    user: 1000:1000
+    recreate: yes
+    container_default_behavior: no_defaults

VirtualMachineService/ancon/playbooks/cwlab_vars_file.yml

@@ -0,0 +1,5 @@
+cwlab_vars:
+  exposed_port: 80
+  data_volume_path: /home/ubuntu/cwlab-data/
+  template_version:
+  create_only_backend:

VirtualMachineService/config/config.yml

@@ -40,6 +40,8 @@ bibigrid:
 forc:
   forc_url: https://proxy-dev.bi.denbi.de:5000/
   forc_allowed:
+    cwlab:
+      - v01
     theiaide:
       - v02
       - v01

VirtualMachineService/scripts/bash/mount.sh

@@ -31,9 +31,9 @@ for id in "${volumes_new[@]}"; do
   fi
   cd /dev/disk/by-id || exit
   sudo mkfs.ext4 "$id"
-  sudo mkdir -p /mnt/${paths_new[ITER]}
-  sudo chmod 777 /mnt/${paths_new[ITER]}/
-  sudo mount "$id" /mnt/${paths_new[ITER]}
+  sudo mkdir -p /vol//${paths_new[ITER]}
+  sudo chmod 777 /vol//${paths_new[ITER]}/
+  sudo mount "$id" /vol//${paths_new[ITER]}
   ((ITER++))
 done
 ITER=0
@@ -61,8 +61,8 @@ for id in "${volumes_attach[@]}"; do
     continue
   fi
   cd /dev/disk/by-id || exit
-  sudo mkdir -p /mnt/${paths_attach[ITER]}
-  sudo chmod 777 /mnt/${paths_attach[ITER]}/
-  sudo mount "$id" /mnt/${paths_attach[ITER]}
+  sudo mkdir -p /vol//${paths_attach[ITER]}
+  sudo chmod 777 /vol//${paths_attach[ITER]}/
+  sudo mount "$id" /vol//${paths_attach[ITER]}
   ((ITER++))
 done

plays/setup_basics.yml

@@ -3,6 +3,7 @@
   become: yes
   vars_files:
     - general_variables.yml
+
 
   roles:
     - role: geerlingguy.docker
@@ -47,7 +48,7 @@
 
     - name: Install list of packages
       apt:
-        name:  ['libffi-dev', 'libssl-dev', 'jq']
+        name:  ['libffi-dev', 'libssl-dev', 'jq', 'python3-setuptools']
         state: present
         update_cache: yes
       tags: always

requirements.txt

@@ -10,4 +10,4 @@ ruamel.yaml<0.17.0
 paramiko==2.7.2
 pyvim==3.0.2
 redis==3.5.3
-requests==2.24.0
+requests==2.25.0

scripts/gateway/gateway-setup.yml

@@ -22,37 +22,37 @@
     - name: Copy TCP script from repository
       copy:
         src: "{{ general.REPO_PATH }}/scripts/gateway/{{ general.TCP_SCRIPT_PERSISTENT_NAME }}"
-        dest: "{{ SCRIPT_PERSISTENT_PATH }}/{{ general.TCP_SCRIPT_PERSISTENT_NAME }}"
+        dest: "{{ general.SCRIPT_PERSISTENT_PATH }}/{{ general.TCP_SCRIPT_PERSISTENT_NAME }}"
         mode: 0774
         remote_src: yes
       when: tcp_script_in is not defined
 
     - name: Copy UDP script from repository
       copy:
         src: "{{ general.REPO_PATH }}/scripts/gateway/{{ general.UDP_SCRIPT_PERSISTENT_NAME }}"
-        dest: "{{ SCRIPT_PERSISTENT_PATH }}/{{ general.UDP_SCRIPT_PERSISTENT_NAME }}"
+        dest: "{{ general.SCRIPT_PERSISTENT_PATH }}/{{ general.UDP_SCRIPT_PERSISTENT_NAME }}"
         mode: 0774
         remote_src: yes
       when: udp_script_in is not defined
 
     - name: Copy specified TCP script
       copy:
         src: "{{ tcp_script_in }}"
-        dest: "{{ SCRIPT_PERSISTENT_PATH }}/{{ general.TCP_SCRIPT_PERSISTENT_NAME }}"
+        dest: "{{ general.SCRIPT_PERSISTENT_PATH }}/{{ general.TCP_SCRIPT_PERSISTENT_NAME }}"
         mode: 0774
       when: tcp_script_in is defined
 
     - name: Copy specified UDP script
       copy:
         src: "{{ udp_script_in }}"
-        dest: "{{ SCRIPT_PERSISTENT_PATH }}/{{ general.UDP_SCRIPT_PERSISTENT_NAME }}"
+        dest: "{{ general.SCRIPT_PERSISTENT_PATH }}/{{ general.UDP_SCRIPT_PERSISTENT_NAME }}"
         mode: 0774
       when: udp_script_in is defined
 
     - name: Run TCP script
       become: yes
-      command: "{{ SCRIPT_PERSISTENT_PATH }}/{{ general.TCP_SCRIPT_PERSISTENT_NAME }}"
+      command: "{{ general.SCRIPT_PERSISTENT_PATH }}/{{ general.TCP_SCRIPT_PERSISTENT_NAME }}"
 
     - name: Run UDP script
       become: yes
-      command: "{{ SCRIPT_PERSISTENT_PATH }}/{{ general.UDP_SCRIPT_PERSISTENT_NAME }}"
+      command: "{{ general.SCRIPT_PERSISTENT_PATH }}/{{ general.UDP_SCRIPT_PERSISTENT_NAME }}"