Merge branch 'dev'

.dependabot/config.yml

@@ -4,8 +4,16 @@ update_configs:
     directory: "/"
     update_schedule: "weekly"
     target_branch: "dev"
+    default_reviewers:
+      - "dweinholz"
+      - "eKatchko"
+      - "vktrrdk"
   - package_manager: "docker"
     directory: "/"
     update_schedule: "weekly"
     target_branch: "dev"
+    default_reviewers:
+      - "dweinholz"
+      - "eKatchko"
+      - "vktrrdk"
 

.env

@@ -0,0 +1,11 @@
+CLOUD_CLIENT_TAG=0.1.0-beta.0.8.17
+ELASTIC_USER=elasticboi
+FILEBEAT_TAG=7.1.0
+ELASTIC_URL=https://portal-dev.denbi.de:443
+
+OS_AUTH_URL=https://openstack.cebitec.uni-bielefeld.de:5000/v3/
+OS_PROJECT_ID=3e552e42945c40aab02af3be9bc67a23
+OS_PROJECT_NAME=portal-pool-dev
+OS_USERNAME=portal-user-dev
+OS_USER_DOMAIN_NAME=Default
+OS_PROJECT_DOMAIN_ID=default

.github/workflows/main.yml

@@ -15,7 +15,7 @@ jobs:
     - name: Publish to Registry
       uses: elgohr/Publish-Docker-Github-Action@master
       with:
-        name: denbicloud/cloud-api
+        name: denbicloud/cloud-portal-client
         username: ${{ secrets.DOCKER_USERNAME }}
         password: ${{ secrets.DOCKER_PASSWORD }}
         dockerfile: Dockerfile

.gitignore

@@ -79,6 +79,7 @@ celerybeat-schedule
 *.sage.py
 
 # Environments
+.secrets
 .env
 .venv
 env/

.secrets.in

@@ -0,0 +1,3 @@
+OS_PASSWORD=
+FORC_API_KEY=
+ELASTIC_PASSWORD=

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.7.7-slim
+FROM python:3.8.2-slim
 ADD . /code
 WORKDIR /code
 RUN  apt-get update -y 

README.md

@@ -171,3 +171,84 @@ if __name__ == '__main__':
     server = TServer.TSimpleServer(processor, transport, tfactory, pfactory)
     server.serve()
 ```
+
+### Deployment via ansible
+
+#### 1.Create your inventory file:
+
+Example:
+
+~~~BASH
+[test]
+REMOTE_IP ansible_user=ubuntu ansible_ssh_private_key_file=PATH_TO_SSH_FILE ansible_python_interpreter=/usr/bin/python3
+~~~
+
+where
+
+  * REMOTE_IP is the IP of your staging machine
+
+  * PATH_TO_SSH_FILE is the path to the ssh key of the virtual machine
+
+#### 2.Set SSH keyforwarding
+
+In order to checkout the GitHub project you will have to enable
+SSH Key forwarding in your `~/.ssh/config` file.
+
+~~~BASH
+Host IP 
+ ForwardAgent yes
+~~~
+
+where `IP` is the IP of the machine you want to start the portal.
+
+#### 3.Set SSH Key in Github
+
+The GitHub repository will cloned using an ssh key you set for the GitHub repository.
+You can read how to set an ssh key for the cloud-portal repository on [this website](https://help.github.com/articles/adding-a-new-ssh-key-to-your-github-account/).
+
+
+#### 4.Install needed libraries
+
+~~~BASH
+ansible-galaxy install -r ansible_requirements.yml
+~~~
+
+#### 5 Create your own  secrets file
+
+Copy the `.secrets.in` to `.secrets`.
+
+#### 6.Set all variables
+
+Set all variables that can be found in  `.env` and `.secrets` file.
+
+#### 8.Run the playbook
+
+You can run the playbook using the following command:
+
+~~~BASH
+ansible-playbook  -i inventory_openstack site.yml
+~~~
+
+where 
+
+  * inventory_openstack is your inventory file which you created in the first step.
+
+
+**Choose  different files**
+
+You can also specify different .env , .secrets and server.pem files.
+
+You can also specify branch, tag, commit that should be checked out with `--extra-vars`.
+
+For Example:
+
+~~~BASH
+ansible-playbook -i inventory_openstack --extra-vars "repo_version=master" site.yml
+~~~
+Optional Keys are:
++ repo_version
++ env_file
++ secrets_file
++ client_server_pem
+
+**Note:** Default repository is always master. Also by default, the files are taken from the folders as for local start.

VirtualMachineService/VirtualMachineHandler.py

@@ -168,6 +168,7 @@ def __init__(self, config):
             try:
                 self.BIBIGRID_URL = cfg["bibigrid"]["bibigrid_url"]
                 self.SUB_NETWORK = cfg["bibigrid"]["sub_network"]
+                self.BIBIGRID_MODES = cfg["bibigrid"]["bibigrid_modes"]
                 self.logger.info(
                     msg="Bibigrd url loaded: {0}".format(self.BIBIGRID_URL)
                 )
@@ -1697,6 +1698,23 @@ def add_udp_security_group(self, server_id):
 
         return True
 
+    def detach_ip_from_server(self, server_id, floating_ip):
+        self.logger.info(
+            "Detaching floating ip {} from server {}".format(floating_ip, server_id)
+        )
+        try:
+            self.conn.compute.remove_floating_ip_from_server(
+                server=server_id, address=floating_ip
+            )
+            return True
+        except Exception:
+            self.logger.exception(
+                "Could not detach floating ip {} from server {}".format(
+                    floating_ip, server_id
+                )
+            )
+            return False
+
     def get_servers_by_bibigrid_id(self, bibigrid_id):
         filters = {"bibigrid_id": bibigrid_id, "name": bibigrid_id}
         servers = self.conn.list_servers(filters=filters)
@@ -1740,20 +1758,33 @@ def get_cluster_status(self, cluster_id):
         response = req.get(
             url=request_url, json=body, headers=headers, verify=self.PRODUCTION
         )
-        self.logger.info("Cluster {} status: ".format(cluster_id, response.content))
+        self.logger.info("Cluster {} status: {} ".format(cluster_id, response.content))
         return response.json()
 
-    def get_cluster_info(self, cluster_id):
+
+    def bibigrid_available(self):
+        if not self.BIBIGRID_URL:
+            return False
+        try:
+            self.get_clusters_info()
+            return True
+        except Exception:
+            self.logger.exception("Bibigrid is offline")
+            return False
+
+    def get_clusters_info(self):
         headers = {"content-Type": "application/json"}
         body = {"mode": "openstack"}
         request_url = self.BIBIGRID_URL + "list"
-        self.logger.info(request_url)
-
         response = req.get(
             url=request_url, json=body, headers=headers, verify=self.PRODUCTION
         )
         self.logger.info(response.json())
         infos = response.json()["info"]
+        return infos
+
+    def get_cluster_info(self, cluster_id):
+        infos = self.get_clusters_info()
         for info in infos:
             self.logger.info(cluster_id)
             self.logger.info(info)
@@ -1802,7 +1833,10 @@ def start_cluster(self, public_key, master_instance, worker_instances, user):
             "availabilityZone": self.AVAIALABILITY_ZONE,
             "masterInstance": master_instance,
             "workerInstances": wI,
+            "useMasterWithPublicIp": False
         }
+        for mode in self.BIBIGRID_MODES:
+            body.update({mode: True})
         request_url = self.BIBIGRID_URL + "create"
         response = req.post(
             url=request_url, json=body, headers=headers, verify=self.PRODUCTION

VirtualMachineService/VirtualMachineService-remote

@@ -43,6 +43,8 @@ if len(sys.argv) <= 1 or sys.argv[1] == '--help':
     print('  string add_floating_ip_to_server(string openstack_id, string network)')
     print('  bool create_connection(string username, string password, string auth_url, string user_domain_name, string project_domain_name)')
     print('   start_server_without_playbook(string flavor, string image, string public_key, string servername,  metadata, bool https, bool http,  resenv,  volume_ids_path_new,  volume_ids_path_attach)')
+    print('  bool bibigrid_available()')
+    print('  bool detach_ip_from_server(string server_id, string floating_ip)')
     print('   start_server_with_mounted_volume(string flavor, string image, string public_key, string servername,  metadata, bool https, bool http,  resenv,  volume_ids_path_new,  volume_ids_path_attach)')
     print('   start_server(string flavor, string image, string public_key, string servername,  metadata, string diskspace, string volumename, bool https, bool http,  resenv)')
     print('   start_server_with_custom_key(string flavor, string image, string servername,  metadata, bool http, bool https,  resenv,  volume_ids_path_new,  volume_ids_path_attach)')
@@ -275,6 +277,18 @@ elif cmd == 'start_server_without_playbook':
         sys.exit(1)
     pp.pprint(client.start_server_without_playbook(args[0], args[1], args[2], args[3], eval(args[4]), eval(args[5]), eval(args[6]), eval(args[7]), eval(args[8]), eval(args[9]),))
 
+elif cmd == 'bibigrid_available':
+    if len(args) != 0:
+        print('bibigrid_available requires 0 args')
+        sys.exit(1)
+    pp.pprint(client.bibigrid_available())
+
+elif cmd == 'detach_ip_from_server':
+    if len(args) != 2:
+        print('detach_ip_from_server requires 2 args')
+        sys.exit(1)
+    pp.pprint(client.detach_ip_from_server(args[0], args[1],))
+
 elif cmd == 'start_server_with_mounted_volume':
     if len(args) != 10:
         print('start_server_with_mounted_volume requires 10 args')