From f15b0911b878f0c35e577105035a9313014366c0 Mon Sep 17 00:00:00 2001 From: Zarquan Date: Tue, 12 Sep 2023 16:08:03 +0100 Subject: [PATCH 1/2] Notes on git catchup --- notes/zrq/20230912-01-git-catchup.txt | 81 +++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 notes/zrq/20230912-01-git-catchup.txt diff --git a/notes/zrq/20230912-01-git-catchup.txt b/notes/zrq/20230912-01-git-catchup.txt new file mode 100644 index 00000000..fbec8539 --- /dev/null +++ b/notes/zrq/20230912-01-git-catchup.txt @@ -0,0 +1,81 @@ +# +# +# +# Copyright (c) 2023, ROE (http://www.roe.ac.uk/) +# +# This information is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This information is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# +# +#zrq-notes-time +#zrq-notes-indent +#zrq-notes-crypto +#zrq-notes-ansible +#zrq-notes-osformat +#zrq-notes-zeppelin +# +# AIMetrics: [] +# + + Target: + + Bring the local copy up to date. + + Result: + + Work in progress ... + +# ----------------------------------------------------- +# Merge upstream changes. +#[user@desktop] + + source "${HOME:?}/aglais.env" + pushd "${AGLAIS_CODE}" + + git checkout master + + git pull + + git fetch upstream + + git merge upstream/master + + git status + + git push + + popd + + +# ----------------------------------------------------- +# Start a new branch. +#[user@desktop] + + branchname=infra-notes + + source "${HOME:?}/aglais.env" + pushd "${AGLAIS_CODE}" + + newbranch=$(date '+%Y%m%d')-zrq-${branchname:?} + + git checkout master + + git checkout -b "${newbranch:?}" + + git push --set-upstream 'origin' "$(git branch --show-current)" + + popd + + + From 532de9f6e2aa5642094675292c564aa1c22a37ed Mon Sep 17 00:00:00 2001 From: Zarquan Date: Tue, 12 Sep 2023 17:43:00 +0100 Subject: [PATCH 2/2] Notes on infra-ops deployment --- notes/zrq/20230912-01-git-catchup.txt | 24 +++ notes/zrq/20230912-02-infra-notes.txt | 248 ++++++++++++++++++++++++++ notes/zrq/20230912-03-backups.txt | 110 ++++++++++++ 3 files changed, 382 insertions(+) create mode 100644 notes/zrq/20230912-02-infra-notes.txt create mode 100644 notes/zrq/20230912-03-backups.txt diff --git a/notes/zrq/20230912-01-git-catchup.txt b/notes/zrq/20230912-01-git-catchup.txt index fbec8539..ae3af1a4 100644 --- a/notes/zrq/20230912-01-git-catchup.txt +++ b/notes/zrq/20230912-01-git-catchup.txt @@ -78,4 +78,28 @@ popd +# ----------------------------------------------------- +# Add the new notes. +#[user@desktop] + + source "${HOME:?}/aglais.env" + pushd "${AGLAIS_CODE}" + + gedit notes/zrq/20230912-01-git-catchup.txt & + + .... + .... + + gedit notes/zrq/20230912-02-infra-notes.txt + + .... + .... + + gedit notes/zrq/20230912-03-backups.txt + + .... + .... + + + diff --git a/notes/zrq/20230912-02-infra-notes.txt b/notes/zrq/20230912-02-infra-notes.txt new file mode 100644 index 00000000..7271c4c7 --- /dev/null +++ b/notes/zrq/20230912-02-infra-notes.txt @@ -0,0 +1,248 @@ +# +# +# +# Copyright (c) 2023, ROE (http://www.roe.ac.uk/) +# +# This information is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This information is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# +# +#zrq-notes-time +#zrq-notes-indent +#zrq-notes-crypto +#zrq-notes-ansible +#zrq-notes-osformat +#zrq-notes-zeppelin +# +# AIMetrics: [] +# + + Target: + + Notes on how the infrastructure nodes were created. + + Result: + + Work in progress ... + +# ----------------------------------------------------- + + Code to deploy the node(s) is in /deployments/infra-ops + https://github.com/wfau/gaia-dmp/tree/master/deployments/infra-ops/ansible + + This part of the source code is a work in progress. + The original aim was to deploy 2 separate hosts, 'celatum' for secrets and 'gitstore' for a git repository. + https://github.com/wfau/gaia-dmp/blob/master/deployments/infra-ops/ansible/hosts.yml + + Ended up only deploying 'gitstore' .. and then using it to store our secrets. + Ho hum. + + The 'create-all' playbook creates sshkeys and network, and then creates the 'gitstore' node. + https://github.com/wfau/gaia-dmp/blob/1c12107ed41ec35f51ccf5c88e5154fa56bdab5a/deployments/infra-ops/ansible/create-all.yml#L23-L27 + + Notes on using the Ansible scripts are here: + https://github.com/wfau/gaia-dmp/blob/master/notes/zrq/20220221-01-infraops-ansible.txt + https://github.com/wfau/gaia-dmp/blob/master/notes/zrq/20220411-01-infra-server.txt + + Looks like we got this far and then got diverted. + + Since then the 'gitstore' node has been assigned a DNS record of data.gaia-dmp.uk, + making it our 'data' node. + +# ----------------------------------------------------- +#[user@desktop] + + host data.gaia-dmp.uk + + > data.gaia-dmp.uk is an alias for iris-gaia-data.duckdns.org. + > iris-gaia-data.duckdns.org has address 128.232.222.153 + + +# ----------------------------------------------------- +# Our 'gitstore' node stores our project secrets. +# These can be acessed from any location via ssh. +#[user@gitstore] + + cat /home/fedora/secrets + + > secrets: + > + > example: + > fish: "Silver Fish" + > bird: "Yellow Bird" + > + > devops: + > + > duckdns: + > token: "...." + > + > stfc: + > echo: + > endpoint: "s3.echo.stfc.ac.uk" + > template: "s3.echo.stfc.ac.uk/%(bucket)" + > zip_secret: "...." + > access_key: "...." + > secret_key: "...." + + + cat /home/fedora/bin/getsecret + + > #!/bin/sh + > key=${1:?'key required'} + > + > yq " + > .secrets.${key} // \"\" + > " "${HOME}/secrets" + + + getsecret 'example.bird' + + > Yellow Bird + + +# ----------------------------------------------------- +# Our 'gitstore' node stores our password hashes. +# These are loaded into the Zeppelin Shiro database on each deployment. +# These are the only part of the user passwords that we keep. +# ** If a user's passhash is blank, the deploy scripts will generate a new password for them. ** +#[user@gitstore] + + cat /home/fedora/passhashes + + > users: + > passhash: + > "DCrake": "........" + > "NHambly": "........" + > "SVoutsinas": "........" + > "DMorris": "........" + > .... + > .... + + + cat /home/fedora/bin/getpasshash + + > #!/bin/sh + > key=${1:?} + > yq ' + > .users.passhash.'${key}' // "" + > ' '/home/fedora/passhashes' + + + getpasshash 'DMorris' + + > "........" + + +# ----------------------------------------------------- +# Our 'gitstore' node stores our SSL certificates. +#[user@gitstore] + + ls -al /home/fedora/certs + + > .... + > .... + > drwxrwxr-x. 2 fedora fedora 4096 Mar 15 19:00 20230123 + > drwxrwxr-x. 2 fedora fedora 4096 Jun 14 10:42 20230314 + > drwxrwxr-x. 2 fedora fedora 4096 Sep 11 13:54 20230614 + > drwxrwxr-x. 2 fedora fedora 4096 Sep 12 10:43 20230905 + > drwxrwxr-x. 2 fedora fedora 4096 Sep 12 12:13 20230912 + > -rw-r--r--. 1 fedora fedora 63293 Sep 12 12:13 certs.tar.gz + > lrwxrwxrwx. 1 fedora fedora 27 Sep 12 10:44 latest -> /home/fedora/certs/20230912 + + + ls -al /home/fedora/certs/20230123 + + > -rw-r--r--. 1 fedora fedora 22782 Mar 15 19:00 certs.tar.gz + + + ls -al /home/fedora/certs/20230314 + + > lrwxrwxrwx. 1 fedora fedora 9 Jun 14 10:42 20230614 -> 20230614/ + > -rw-rw-r--. 1 fedora fedora 55319 Mar 15 19:14 certs.tar.gz + + + ls -al /home/fedora/certs/20230614 + + > lrwxrwxrwx. 1 fedora fedora 18 Sep 5 15:09 certs -> /home/fedora/certs + > -rw-rw-r--. 1 fedora fedora 55926 Jun 14 10:41 certs.tar.gz + + + ls -al /home/fedora/certs/20230905 + + > lrwxrwxrwx. 1 fedora fedora 8 Sep 12 10:43 20230912 -> 20230912 + > -rw-r--r--. 1 fedora fedora 37522 Sep 11 13:57 certs.tar.gz + + + ls -al /home/fedora/certs/20230912 + + > -rw-r--r--. 1 fedora fedora 63293 Sep 12 12:13 certs.tar.gz + + +# ----------------------------------------------------- +# Our 'gitstore' node stores a backup of our users' notebooks. +#[user@gitstore] + + ls -al /var/local/backups/notebooks/ + + > drwxrwxr-x. 3 fedora fedora 4096 Nov 2 2022 20221102-050416-live.gaia-dmp.uk-notebooks + > drwxrwxr-x. 3 fedora fedora 4096 Dec 4 2022 20221204-141216-live.gaia-dmp.uk-notebooks + > .... + > .... + > drwxrwxr-x. 3 fedora fedora 4096 Sep 12 15:50 20230912-155030-live.gaia-dmp.uk-notebooks + > lrwxrwxrwx. 1 fedora fedora 42 Sep 12 15:50 latest -> 20230912-155030-live.gaia-dmp.uk-notebooks + + + du -h -d 1 /var/local/backups/notebooks/ + + > 103M /var/local/backups/notebooks/20221102-050416-live.gaia-dmp.uk-notebooks + > 110M /var/local/backups/notebooks/20221204-141216-live.gaia-dmp.uk-notebooks + > .... + > .... + > 188M /var/local/backups/notebooks/20230912-155030-live.gaia-dmp.uk-notebooks + > 3.1G /var/local/backups/notebooks/ + + +# ----------------------------------------------------- +# Our 'gitstore' node has an old backup of some users' home directories. +# ** These are no longer used and the data is out of date ** +#[user@gitstore] + + ls -al /var/local/backups/homedirs/ + + > drwxr-xr-x. 12 fedora fedora 4096 Jul 4 2022 20220711-120030-green-homedirs + > drwxr-xr-x. 9 fedora fedora 4096 Jul 8 2022 20220711-120215-blue-homedirs + > .... + > .... + > drwxr-xr-x. 9 fedora fedora 4096 Jul 8 2022 20220727-103930-blue-homedirs + > lrwxrwxrwx. 1 fedora fedora 29 Jul 27 2022 latest -> 20220727-103930-blue-homedirs + + +# ----------------------------------------------------- +# Our 'gitstore' node has an old backup of some logs. +# ** These are no longer used and the data is out of date ** +#[user@gitstore] + + ls -al /var/local/backups/logs/2023/20230720/iris-gaia-blue/logs/ + + > -r--r--r--. 1 fedora fedora 3719 Jul 20 11:45 zeppelin-fedora-iris-gaia-blue-20230619-zeppelin.log + > -r--r--r--. 1 fedora fedora 262513 Jul 20 11:45 zeppelin-fedora-iris-gaia-blue-20230619-zeppelin.log.2023-06-19 + > -r--r--r--. 1 fedora fedora 489883 Jul 20 11:45 zeppelin-fedora-iris-gaia-blue-20230619-zeppelin.log.2023-06-20 + > .... + > .... + > -r--r--r--. 1 fedora fedora 491881 Jul 20 11:45 zeppelin-fedora-iris-gaia-blue-20230619-zeppelin.log.2023-07-18 + > -r--r--r--. 1 fedora fedora 141689 Jul 20 11:45 zeppelin-fedora-iris-gaia-blue-20230619-zeppelin.log.2023-07-19 + > -r--r--r--. 1 fedora fedora 6231 Jul 20 11:45 zeppelin-fedora-iris-gaia-blue-20230619-zeppelin.out + > -r--r--r--. 1 fedora fedora 177 Jul 20 11:45 zeppelin-interpreter-md-Evison-Evison-fedora-iris-gaia-blue-20230619-zeppelin.log + > -r--r--r--. 1 fedora fedora 76781315 Jul 20 11:45 zeppelin-interpreter-spark-Evison-Evison-fedora-iris-gaia-blue-20230619-zeppelin.log + diff --git a/notes/zrq/20230912-03-backups.txt b/notes/zrq/20230912-03-backups.txt new file mode 100644 index 00000000..6eab5f11 --- /dev/null +++ b/notes/zrq/20230912-03-backups.txt @@ -0,0 +1,110 @@ +# +# +# +# Copyright (c) 2023, ROE (http://www.roe.ac.uk/) +# +# This information is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This information is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# +# +#zrq-notes-time +#zrq-notes-indent +#zrq-notes-crypto +#zrq-notes-ansible +#zrq-notes-osformat +#zrq-notes-zeppelin +# +# AIMetrics: [] +# + + Target: + + Offsite backups. + + Result: + + Work in progress ... + +# ----------------------------------------------------- +# Make an offsite backup of our notebooks. +#[user@desktop] + + sshuser=fedora + sshhost=live.gaia-dmp.uk + + pushd /var/local/backups/aglais/notebooks + + datetime=$(date '+%Y%m%d-%H%M%S') + backname="${datetime:?}-${sshhost:?}-notebooks" + + mkdir "${backname}" + + rsync \ + --perms \ + --times \ + --group \ + --owner \ + --stats \ + --progress \ + --exclude '~Trash' \ + --human-readable \ + --checksum \ + --recursive \ + --rsync-path 'sudo rsync' \ + "${sshuser:?}@${sshhost:?}:/home/fedora/zeppelin/notebook" \ + "${backname:?}" + + rsync \ + --perms \ + --times \ + --group \ + --owner \ + --stats \ + --progress \ + --human-readable \ + --checksum \ + --recursive \ + --rsync-path 'sudo rsync' \ + "${sshuser:?}@${sshhost:?}:/home/fedora/zeppelin/conf/notebook-authorization.json" \ + "${backname:?}" + + popd + + +# ----------------------------------------------------- +# Make an offsite backup of our infraops data. +#[user@desktop] + + sshuser=fedora + sshhost=data.gaia-dmp.uk + + pushd /var/local/backups/aglais/ + + mkdir infraops + pushd infraops + + datetime=$(date '+%Y%m%d-%H%M%S') + backname="${datetime:?}-${sshhost:?}" + + mkdir "${backname}" + pushd "${backname}" + + scp "${sshuser:?}@${sshhost:?}:secrets" . + scp "${sshuser:?}@${sshhost:?}:passhashes" . + scp "${sshuser:?}@${sshhost:?}:certs/latest/certs.tar.gz" . + + popd + popd + popd +