diff --git a/.clog.toml b/.clog.toml new file mode 100644 index 00000000..0ecdf4a3 --- /dev/null +++ b/.clog.toml @@ -0,0 +1,3 @@ +[clog] +repository = "https://github.com/deNBI/cloud-portal-client" +from-latest-tag = true diff --git a/.env b/.env deleted file mode 100644 index dd3f9f2b..00000000 --- a/.env +++ /dev/null @@ -1,16 +0,0 @@ -CLOUD_CLIENT_TAG=0.1.0-beta.0.13.0 -ELASTIC_USER=elasticboi -FILEBEAT_TAG=7.1.0 -ELASTIC_URL=https://portal-dev.denbi.de:443 -BIBIGRID_TAG=dev-30-9 -CLIENT_PORT=9090 - - -OS_AUTH_URL=https://openstack.cebitec.uni-bielefeld.de:5000/v3/ -OS_PROJECT_ID=3e552e42945c40aab02af3be9bc67a23 -OS_PROJECT_NAME=portal-pool-dev -OS_USERNAME=portal-user-dev -OS_USER_DOMAIN_NAME=Default -OS_PROJECT_DOMAIN_ID=default - -GITHUB_PLAYBOOKS_REPO=https://api.github.com/repos/deNBI/resenvs/contents/ \ No newline at end of file diff --git a/.env.in b/.env.in index 84fb1e0c..a5997b02 100644 --- a/.env.in +++ b/.env.in @@ -1,19 +1,18 @@ -CLOUD_CLIENT_TAG=0.1.0-beta.0.13.0 -BIBIGRID_TAG=dev-30-9 +# Client and Bibigrid +CLOUD_CLIENT_TAG=0.1.0-beta.0.15.1 +BIBIGRID_TAG=bibigrid-rest-2.3 CLIENT_PORT= -FORC_API_KEY= -ELASTIC_USER= -ELASTIC_PASSWORD= -FILEBEAT_TAG=7.1.0 -HOST=portal-dev.denbi.de -REGION=Giessen-Staging - OS_AUTH_URL= OS_PROJECT_ID= OS_PROJECT_NAME= OS_USERNAME= OS_USER_DOMAIN_NAME= OS_PROJECT_DOMAIN_ID= -OS_PROJECT_DOMAIN_NAME= +OS_PASSWORD= -GITHUB_PLAYBOOKS_REPO=https://api.github.com/repos/deNBI/resenvs/contents/ +# Filebeat +FILEBEAT_TAG=7.10.0 +ELASTIC_USER= +ELASTIC_PASSWORD= +HOST=portal-dev.denbi.de +REGION=Giessen-Staging diff --git a/.gitignore b/.gitignore index 32e21699..7eb6318b 100644 --- a/.gitignore +++ b/.gitignore @@ -5,7 +5,6 @@ __pycache__/ # C extensions *.so -.env # Distribution / packaging .Python build/ @@ -155,3 +154,5 @@ fabric.properties # additional data *.pem #*.sh +.env_* +VirtualMachineService/config/config_* diff --git a/.secrets.in b/.secrets.in deleted file mode 100644 index 19f658ee..00000000 --- a/.secrets.in +++ /dev/null @@ -1,3 +0,0 @@ -OS_PASSWORD= -FORC_API_KEY= -ELASTIC_PASSWORD= diff --git a/Makefile b/Makefile index 9760d98b..2bdb377b 100644 --- a/Makefile +++ b/Makefile @@ -59,7 +59,17 @@ bibigrid_logs: ## Logs from Bibigrid enter_client_container: ## Enter Client container docker exec -it client_portal-client_1 bash +check_env: ## Checks if your .env contains every key set in .env.in. + python3 check_env.py .env.in .env env +check_manual_env: ## Checks if your specified .env_* contains every key set in .env.in. + python3 check_env.py .env.in $(env-file) env + +check_local_config: ## Check if your config_local.yml contains every key set in config.yml + python3 check_env.py VirtualMachineService/config/config.yml VirtualMachineService/config/config_local.yml config + +check_manual_config: ## Check if your specified config_*.yml contains every key set in config.yml + python3 check_env.py VirtualMachineService/config/config.yml VirtualMachineService/config/$(config-file) config .PHONY: help lint docs thrift_py diff --git a/README.md b/README.md index 57706cec..35104952 100644 --- a/README.md +++ b/README.md @@ -24,10 +24,14 @@ source NameOfRcFile.sh #### Configuration You can view (almost) all existing parameters in the [yaml file](VirtualMachineService/config/config.yml). +For local development: +Please copy this file and rename it to `config_local.yml` and fill in missing parameters. +For staging/production setup: +Please copy this file and rename it to `config_YOUR_LOCATION.yml` and fill in missing parameters. Also you need to provide the path to your config file as the first param when starting a server. -Furthermore there are some parameters you can set in the [.env.in](.env.in) file, which are read only when starting with docker. -Important: You need to rename .env.in to .env in order for it to be read by docker. +Furthermore there are some parameters you must set in the .env file. Copy the [.env.in](.env.in) to .env and +fill in the missing parameters. When starting with commandline you will need to export some of them manually. #### Security Groups @@ -221,20 +225,21 @@ You can read how to set an ssh key for the cloud-portal repository on [this webs ansible-galaxy install -r ansible_requirements.yml ~~~ -#### 5 Create your own secrets file - -Copy the `.secrets.in` to `.secrets`. - #### 6.Set all variables -Set all variables that can be found in `.env` and `.secrets` file. +Set all variables that can be found in `.env` and `VirtualMachineService/config/config.yml` file. +You can have more than one `.env` file (`.env` and `.env_*` are not tracked by git) and specify which you want to copy +by using the `env_file` variable. +You can have more than one `VirtualMachineService/config/config.yml` file (`VirtualMachineService/config/config_*` are +not tracked by git) and specify which you want to copy by using the `client_config` variable. +These options are useful when maintaining multiple client sites. -#### 8.Run the playbook +#### 7.Run the playbook You can run the playbook using the following command: ~~~BASH -ansible-playbook --tags "client_only" -i inventory_openstack site.yml +ansible-playbook -i inventory_openstack site.yml ~~~ where @@ -244,7 +249,7 @@ where * If you also want to start bibigrid use the tag "bibigrid" **Choose different files** -You can also specify different .env , .secrets and server.pem files. +You can also specify different .env, config.yml and server.pem files. You can also specify branch, tag, commit that should be checked out with `--extra-vars`. @@ -256,7 +261,6 @@ ansible-playbook -i inventory_openstack --extra-vars "repo_version=master" site. Optional Keys are: + repo_version + env_file -+ secrets_file + client_server_pem + client_config diff --git a/VirtualMachineService/VirtualMachineHandler.py b/VirtualMachineService/VirtualMachineHandler.py index baa2c183..23291378 100644 --- a/VirtualMachineService/VirtualMachineHandler.py +++ b/VirtualMachineService/VirtualMachineHandler.py @@ -77,7 +77,6 @@ ch.setFormatter(formatter) LOG.addHandler(fh) LOG.addHandler(ch) -GITHUB_PLAYBOOKS_REPO = os.environ["GITHUB_PLAYBOOKS_REPO"] PLAYBOOKS_DIR = "/code/VirtualMachineService/ancon/playbooks/" PORT = "port" @@ -194,15 +193,32 @@ def __init__(self, config): try: self.RE_BACKEND_URL = cfg["forc"]["forc_url"] - self.FORC_API_KEY = os.environ["FORC_API_KEY"] + self.FORC_API_KEY = cfg["forc"]["forc_api_key"] self.FORC_ALLOWED = {} + self.GITHUB_PLAYBOOKS_REPO = cfg["forc"]["github_playbooks_repo"] + if ( + not self.RE_BACKEND_URL + or not self.FORC_API_KEY + or not self.GITHUB_PLAYBOOKS_REPO + ): + raise ValueError LOG.info(msg="Forc-Backend url loaded: {0}".format(self.RE_BACKEND_URL)) + except ValueError as ve: + LOG.exception(ve) + LOG.info( + "Forc-Backend not loaded as one of the configurations was empty." + ) + self.RE_BACKEND_URL = None + self.FORC_API_KEY = None + self.FORC_ALLOWED = None + self.GITHUB_PLAYBOOKS_REPO = None except Exception as e: LOG.exception(e) LOG.info("Forc-Backend not loaded.") self.RE_BACKEND_URL = None self.FORC_API_KEY = None self.FORC_ALLOWED = None + self.GITHUB_PLAYBOOKS_REPO = None if self.USE_GATEWAY: self.GATEWAY_IP = cfg["openstack_connection"]["gateway_ip"] self.SSH_FORMULAR = cfg["openstack_connection"][ @@ -2665,8 +2681,13 @@ def get_limits(self): } def update_playbooks(self): - LOG.info("STARTED update") - r = req.get(GITHUB_PLAYBOOKS_REPO) + if self.GITHUB_PLAYBOOKS_REPO is None: + LOG.info( + "Github playbooks repo url is None. Aborting download of playbooks." + ) + return + LOG.info("STARTED update of playbooks") + r = req.get(self.GITHUB_PLAYBOOKS_REPO) contents = json.loads(r.content) # Todo maybe clone entire direcotry for f in contents: diff --git a/VirtualMachineService/config/config.yml b/VirtualMachineService/config/config.yml index a6d28404..d5e2213b 100644 --- a/VirtualMachineService/config/config.yml +++ b/VirtualMachineService/config/config.yml @@ -39,6 +39,8 @@ bibigrid: - slurm forc: - forc_url: https://proxy-dev.bi.denbi.de:5000/ + forc_url: + forc_api_key: + github_playbooks_repo: -cloud_site: giessen +cloud_site: bielefeld \ No newline at end of file diff --git a/check_env.py b/check_env.py new file mode 100644 index 00000000..a88530a5 --- /dev/null +++ b/check_env.py @@ -0,0 +1,70 @@ +import sys + + +def get_keys_from_file(file_name): + all_keys = {} + try: + with open(file_name, "r") as stream: + lines = stream.readlines() + for line in lines: + split_line = line.split("=") + if len(split_line) == 2: + if split_line[1] == "\n": + all_keys[split_line[0]] = False + else: + all_keys[split_line[0]] = True + except FileNotFoundError: + print(f"Could not find file {file_name}. Aborting.") + sys.exit(1) + return all_keys + + +def get_keys_from_config_file(file_name): + all_keys = {} + try: + with open(file_name, "r") as stream: + lines = stream.readlines() + for line in lines: + split_line = line.split(":") + if len(split_line) >= 2: + if split_line[1] == "\n": + all_keys[split_line[0].strip()] = False + else: + all_keys[split_line[0].strip()] = True + except FileNotFoundError: + print(f"Could not find file {file_name}. Aborting.") + sys.exit(1) + return all_keys + + +if __name__ == "__main__": + if sys.argv[3] == "env": + env_in = get_keys_from_file(sys.argv[1]) + env_to_check = get_keys_from_file(sys.argv[2]) + for k, v in env_in.items(): + if k in env_to_check: + if env_to_check[k]: + continue + else: + print(f"{k} is not set in your env file.") + else: + print(f"{k} is missing as a key in your env file.") + elif sys.argv[3] == "config": + config_in = get_keys_from_config_file(sys.argv[1]) + config_to_check = get_keys_from_config_file(sys.argv[2]) + for k, v in config_in.items(): + if k in config_to_check: + if config_to_check[k]: + continue + else: + if not v: + print( + f"{k} is not set in your config file. Could also be a yml key." + ) + else: + print(f"{k} is not set in your config file.") + else: + print(f"{k} is missing as a key in your config file.") + else: + print("No information which files to check. Aborting.") + sys.exit(1) diff --git a/docker-compose.bibigrid.yml b/docker-compose.bibigrid.yml index a5260649..a1568c13 100644 --- a/docker-compose.bibigrid.yml +++ b/docker-compose.bibigrid.yml @@ -3,28 +3,16 @@ services: portal-client: image: denbicloud/cloud-portal-client:${CLOUD_CLIENT_TAG} ports: - - ${CLIENT_PORT}:9090 - + - ${CLIENT_PORT}:9090 env_file: - - .env - - environment: - - OS_AUTH_URL - - OS_PROJECT_ID - - OS_PROJECT_NAME - - OS_USERNAME - - OS_PASSWORD - - OS_USER_DOMAIN_NAME - - OS_PROJECT_DOMAIN_ID - - FORC_API_KEY - + - .env volumes: - - ${client_PERSISTENT_PATH}/config.yml:/code/VirtualMachineService/config.yml + - ${client_PERSISTENT_PATH}/config.yml:/code/VirtualMachineService/local_config.yml - ${client_PERSISTENT_PATH}/server.pem:/code/VirtualMachineService/keys/server.pem - ./logs/client:/code/VirtualMachineService/log:rw command: python3 VirtualMachineServer.py /code/VirtualMachineService/config.yml networks: - - portal + - portal redis: image: redis:6.0.9 @@ -38,14 +26,6 @@ services: env_file: - .env environment: - - OS_AUTH_URL - - OS_PROJECT_ID - - OS_PROJECT_NAME - - OS_USERNAME - - OS_PASSWORD - - OS_USER_DOMAIN_NAME - - OS_PROJECT_DOMAIN_ID - - OS_PROJECT_DOMAIN_NAME - server.enableHttps=false - server.enableHttp=true expose: @@ -58,10 +38,6 @@ services: image: docker.elastic.co/beats/filebeat:${FILEBEAT_TAG} env_file: - .env - environment: - - ELASTIC_USER - - ELASTIC_PASSWORD - - HOST volumes: - ./filebeat/filebeat.yml:/usr/share/filebeat/filebeat.yml - ./logs:/usr/share/filebeat/log:ro diff --git a/docker-compose.dev.bibigrid.yml b/docker-compose.dev.bibigrid.yml index a666f09c..4e56af03 100644 --- a/docker-compose.dev.bibigrid.yml +++ b/docker-compose.dev.bibigrid.yml @@ -2,30 +2,18 @@ version: '3.5' services: portal-client: build: - context: . - dockerfile: Dockerfile + context: . + dockerfile: Dockerfile ports: - - ${CLIENT_PORT}:9090 - + - ${CLIENT_PORT}:9090 env_file: - - .env - - environment: - - OS_AUTH_URL - - OS_PROJECT_ID - - OS_PROJECT_NAME - - OS_USERNAME - - OS_PASSWORD - - OS_USER_DOMAIN_NAME - - OS_PROJECT_DOMAIN_ID - - FORC_API_KEY - + - .env volumes: - - ./VirtualMachineService/config/config.yml:/code/VirtualMachineService/config.yml + - ./VirtualMachineService/config/config_local.yml:/code/VirtualMachineService/config.yml - ./VirtualMachineService/keys/localhost/server.pem:/code/VirtualMachineService/keys/server.pem command: python3 VirtualMachineServer.py /code/VirtualMachineService/config.yml networks: - - portal + - portal redis: image: redis:6.0.9 @@ -36,25 +24,16 @@ services: bibigrid: image: bibiserv/bibigrid:${BIBIGRID_TAG} + env_file: + - .env environment: - - OS_AUTH_URL - - OS_PROJECT_ID - - OS_PROJECT_NAME - - OS_USERNAME - - OS_PASSWORD - - OS_USER_DOMAIN_NAME - - OS_PROJECT_DOMAIN_ID - - OS_PROJECT_DOMAIN_NAME - server.enableHttps=false - server.enableHttp=true - expose: - "8080" networks: - portal - - networks: portal: name: portal_default diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index a95a9717..bc3d628b 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -2,30 +2,18 @@ version: '3.5' services: portal-client: build: - context: . - dockerfile: Dockerfile + context: . + dockerfile: Dockerfile ports: - - ${CLIENT_PORT}:9090 - + - ${CLIENT_PORT}:9090 env_file: - - .env - - environment: - - OS_AUTH_URL - - OS_PROJECT_ID - - OS_PROJECT_NAME - - OS_USERNAME - - OS_PASSWORD - - OS_USER_DOMAIN_NAME - - OS_PROJECT_DOMAIN_ID - - FORC_API_KEY - + - .env volumes: - - ./VirtualMachineService/config/config.yml:/code/VirtualMachineService/config.yml + - ./VirtualMachineService/config/config_local.yml:/code/VirtualMachineService/config.yml - ./VirtualMachineService/keys/localhost/server.pem:/code/VirtualMachineService/keys/server.pem command: python3 VirtualMachineServer.py /code/VirtualMachineService/config.yml networks: - - portal + - portal redis: image: redis:6.0.9 @@ -34,7 +22,6 @@ services: networks: - portal - networks: portal: name: portal_default diff --git a/docker-compose.yml b/docker-compose.yml index ac63cba2..1f2a72a2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,30 +3,17 @@ services: portal-client: image: denbicloud/cloud-portal-client:${CLOUD_CLIENT_TAG} ports: - - ${CLIENT_PORT}:9090 - + - ${CLIENT_PORT}:9090 env_file: - - .env - - environment: - - OS_AUTH_URL - - OS_PROJECT_ID - - OS_PROJECT_NAME - - OS_USERNAME - - OS_PASSWORD - - OS_USER_DOMAIN_NAME - - OS_PROJECT_DOMAIN_ID - - FORC_API_KEY - + - .env volumes: - - ${client_PERSISTENT_PATH}/config.yml:/code/VirtualMachineService/config.yml + - ${client_PERSISTENT_PATH}/config.yml:/code/VirtualMachineService/local_config.yml - ${client_PERSISTENT_PATH}/server.pem:/code/VirtualMachineService/keys/server.pem - ./logs/client:/code/VirtualMachineService/log:rw command: python3 VirtualMachineServer.py /code/VirtualMachineService/config.yml networks: - portal - redis: image: redis:6.0.9 expose: @@ -39,10 +26,6 @@ services: image: docker.elastic.co/beats/filebeat:${FILEBEAT_TAG} env_file: - .env - environment: - - ELASTIC_USER - - ELASTIC_PASSWORD - - HOST volumes: - ./filebeat/filebeat.yml:/usr/share/filebeat/filebeat.yml - ./logs:/usr/share/filebeat/log:ro diff --git a/plays/general_variables.yml b/plays/general_variables.yml index bf8cbf2f..9007b2b4 100644 --- a/plays/general_variables.yml +++ b/plays/general_variables.yml @@ -4,4 +4,3 @@ client: CERT_FILE_PATH: /home/ubuntu/client/VirtualMachineService/keys REPO_PATH: /home/ubuntu/client FILEBEAT_DATA_APPEND: filebeat/data - diff --git a/plays/setup_client.yml b/plays/setup_client.yml index 1c40f701..fd060fe1 100644 --- a/plays/setup_client.yml +++ b/plays/setup_client.yml @@ -78,22 +78,6 @@ when: env_file is defined tags: always - - name: Copy secrets file default - copy: - src: "../.secrets" - dest: "{{ client.REPO_PATH }}/.secrets" - backup: yes - when: secrets_file is not defined - tags: always - - - name: Copy secrets file - copy: - src: "{{secrets_file}}" - dest: "{{ client.REPO_PATH }}/.secrets" - backup: yes - when: secrets_file is defined - tags: always - - name: Create variable files copy: content: "{{ item.content | dict2items | map('to_json') | map('regex_replace', '\"key\":\\s\"(.*)\"', lookup('vars', 'regex_env')) | map('from_json') | list}}" @@ -109,10 +93,6 @@ shell: rm -f "{{ client.REPO_PATH }}"/.ansible_environment && jq -r '.[]|"\(.key)=\(.value)"' "{{ client.REPO_PATH }}"/.ansible_environment_*.json >> "{{ client.REPO_PATH }}"/.ansible_environment tags: always - - name: Append secrets to env file - shell: cat "{{ client.REPO_PATH }}"/.secrets "{{ client.REPO_PATH }}"/.ansible_environment >> "{{ client.REPO_PATH }}"/.env - tags: always - # Start - name: Start client docker container without bibigrid docker_compose: diff --git a/requirements.txt b/requirements.txt index 8981b1b0..89140541 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,12 +1,12 @@ -setuptools==54.1.1 +setuptools==54.2.0 thrift >= 0.11.0,<0.20.0 python-keystoneclient -openstacksdk ==0.54.0 -deprecated == 1.2.11 +openstacksdk ==0.55.0 +deprecated == 1.2.12 Click==7.1.2 -flake8==3.8.4 -ansible==3.0.0 -ruamel.yaml<0.17.0 +ansible==3.1.0 +flake8==3.9.0 +ruamel.yaml<0.18.0 paramiko==2.7.2 pyvim==3.0.2 redis==3.5.3 diff --git a/requirements.yml b/requirements.yml index 409f8aa8..ffea6ca1 100644 --- a/requirements.yml +++ b/requirements.yml @@ -1,2 +1,2 @@ - src: oefenweb.rstudio_server -- src: oefenweb.r +- src: oefenweb.latest_r