From c0112887a19a199021c85c1efd81e24f4447a76d Mon Sep 17 00:00:00 2001 From: Jason Vanderhoof Date: Thu, 10 Nov 2022 12:10:41 -0700 Subject: [PATCH 1/3] Replace shared volumes with file copy These changes set the stage for adding "Re-enroll failed master after failover event". These changes replace shared volumes with file copies to ensure the failed master can be wiped clean before being re-provisioned. --- .gitignore | 5 ++++- bin/dap | 17 ++++++++++++++--- docker-compose.yml | 10 +--------- 3 files changed, 19 insertions(+), 13 deletions(-) diff --git a/.gitignore b/.gitignore index 6fe0f5c7..f14f5cf1 100644 --- a/.gitignore +++ b/.gitignore @@ -28,4 +28,7 @@ files/haproxy/master/haproxy.cfg **/certificates/root_ca/ # Versions plugin backup file (use git history instead) -**/pom.xml.versionsBackup \ No newline at end of file +**/pom.xml.versionsBackup + +# Ignore HAProxy SSL files +system/haproxy/certs/** diff --git a/bin/dap b/bin/dap index 7e40e2a2..5c993748 100755 --- a/bin/dap +++ b/bin/dap @@ -183,9 +183,7 @@ function _start_master { docker-compose pull fi - _set_master_single_node_proxy_config - docker-compose up -d --no-deps conjur-master.mycompany.local \ - conjur-master-1.mycompany.local + docker-compose up -d --no-deps conjur-master-1.mycompany.local fi } @@ -196,6 +194,9 @@ function _start_l7_load_balancer { } function _configure_master { + # Copy DH Param + docker-compose cp files/dhparam.pem conjur-master-1.mycompany.local:${DHPATH}/dhparam.pem + _cmd="evoke configure master" _cmd="$_cmd --accept-eula" _cmd="$_cmd --hostname conjur-master.mycompany.local" @@ -261,6 +262,13 @@ function _perform_promotion { "evoke replication rebase conjur-master-2.mycompany.local" } +function deploy_proxy { + _set_master_single_node_proxy_config + # Copy Conjur generated certificates to HAProxy + docker-compose cp conjur-master-1.mycompany.local:/opt/conjur/etc/ssl ./system/haproxy/certs + docker-compose up -d --no-deps conjur-master.mycompany.local +} + function _single_master { _start_master _configure_master @@ -299,6 +307,7 @@ function _stop { _command rm -rf cli_cache _command rm -rf system/backup _command rm -rf system/logs + _command rm -rf system/haproxy/certs _command rm files/haproxy/master/haproxy.cfg || true echo "stopped" exit @@ -353,6 +362,8 @@ _set_cluster_policy_file() { cat << EOF > policy/cluster.yml - !policy id: production + annotations: + ttl: 30 body: - !layer diff --git a/docker-compose.yml b/docker-compose.yml index 92d64507..36f9677a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,7 +12,7 @@ services: - "1999" - "5432" volumes: - - master-certs:/etc/ssl/certs + - ./system/haproxy/certs:/etc/ssl/certs - ./files/haproxy/master:/usr/local/etc/haproxy conjur-master-1.mycompany.local: @@ -33,13 +33,10 @@ services: - "seccomp:unconfined" volumes: - seeds:/opt/cyberark/dap/seeds - - master-certs:/opt/conjur/etc/ssl/ - ./system/backup:/opt/conjur/backup:Z - ./system/configuration:/opt/cyberark/dap/configuration:Z - ./system/logs/master-1:/var/log/conjur:Z - ./files:/conjur_files - - ./files/dhparam.pem:${DHPATH}/dhparam.pem - conjur-master-2.mycompany.local: image: registry.tld/conjur-appliance:${VERSION} networks: @@ -56,7 +53,6 @@ services: - "seccomp:unconfined" volumes: - seeds:/opt/cyberark/dap/seeds - - master-certs:/opt/conjur/etc/ssl/ - ./system/backup:/opt/conjur/backup:Z - ./system/configuration:/opt/cyberark/dap/configuration:Z - ./system/logs/master-2:/var/log/conjur:Z @@ -78,7 +74,6 @@ services: - "seccomp:unconfined" volumes: - seeds:/opt/cyberark/dap/seeds - - master-certs:/opt/conjur/etc/ssl/ - ./system/backup:/opt/conjur/backup:Z - ./system/configuration:/opt/cyberark/dap/configuration:Z - ./system/logs/master-3:/var/log/conjur:Z @@ -100,7 +95,6 @@ services: - "seccomp:unconfined" volumes: - seeds:/opt/cyberark/dap/seeds - - master-certs:/opt/conjur/etc/ssl/ - ./system/backup:/opt/conjur/backup:Z - ./system/configuration:/opt/cyberark/dap/configuration:Z - ./system/logs/master-4:/var/log/conjur:Z @@ -122,7 +116,6 @@ services: - "seccomp:unconfined" volumes: - seeds:/opt/cyberark/dap/seeds - - master-certs:/opt/conjur/etc/ssl/ - ./system/backup:/opt/conjur/backup:Z - ./system/configuration:/opt/cyberark/dap/configuration:Z - ./system/logs/master-5:/var/log/conjur:Z @@ -209,7 +202,6 @@ services: volumes: seeds: follower-certs: - master-certs: networks: dap_net: From 867560ff74d08a1c446b5f56c440692f2987e5b4 Mon Sep 17 00:00:00 2001 From: Jason Vanderhoof Date: Thu, 10 Nov 2022 12:29:38 -0700 Subject: [PATCH 2/3] Adds a ability to re-enroll a failed leader --- bin/dap | 128 ++++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 97 insertions(+), 31 deletions(-) diff --git a/bin/dap b/bin/dap index 5c993748..a59c667c 100755 --- a/bin/dap +++ b/bin/dap @@ -13,24 +13,62 @@ Synopsis: bin/dap [command options] Usage: bin/dap [options]: - --create-backup Creates a backup|Requires configured master - --dry-run Print configuration commands with executing - --enable-auto-failover Configures Master cluster with auto-failover (Requires configured master and standbys) - --generate-dh Don't mount pre-generated DH params into the appliance containers (will cause a _lot_ more CPU consumption) - --h, --help Shows this help message - --import-custom-certificates Imports pre-generated 3rd-party certificates (Requires configured master) - --promote-standby Stops the current master and promotes a standby (Requires configured standbys and no auto-failover) - --provision-follower Configures follower behind a Layer 7 load balancer (Requires configured master) - --provision-master Configures a DAP Master with account `demo` and password `MySecretP@ss1` behind a Layer 4 load balancer - --provision-standbys Deploys and configures two standbys (Requires configured master) - --restore-from-backup Restores a master from backup|Requires a previously created backup - --stop Stops all containers and cleans up cached files - --trigger-failover Stops current master (Requires an auto-failover cluster) - --trust-follower-proxy Adds Follower load balancer as a trusted proxy (Requires a configured follower) - --upgrade-master Restores master from backup (Requires configured master) - --wait-for-master Blocks until the Master is healthy - --version Version of DAP to use (defaults to latest build) - --standby-count Number of Standbys to deploy (defaults to 2) + --create-backup Creates a backup (Requires configured master). + + --dry-run Print configuration commands with executing. + + --enable-auto-failover Configures Master cluster with auto-failover + (Requires configured master and standbys). + + --generate-dh Don't mount pre-generated DH params into the + appliance containers (will cause a _lot_ more + CPU consumption). + + --h, --help Shows this help message. + + --import-custom-certificates Imports pre-generated 3rd-party certificates + (Requires configured master). + + --promote-standby Stops the current master and promotes a standby + (Requires configured standbys and no auto-failover). + + --provision-follower Configures follower behind a Layer 7 load balancer + (Requires configured master). + + --provision-master Configures a DAP Master with account `demo` and + password `MySecretP@ss1` behind a Layer 4 load + balancer. + + --provision-standbys Deploys and configures two standbys (Requires + configured master). + + --reenroll-failed-leader Performs a re-enroll of the previous master + following a failover event. + + --restore-from-backup Restores a master from backup|Requires a previously + created backup. + + --rotate-custom-certificates Regenerates custom certificates and applies the new + certificates to each node. + + --standby-count Number of Standbys to deploy (defaults to 2). + + --stop Stops all containers and cleans up cached files. + + --trigger-failover Stops current master (Requires an auto-failover + cluster). + + --trust-follower-proxy Adds Follower load balancer as a trusted proxy + (Requires a configured follower). + + --upgrade-master Restores master from backup (Requires configured + master). + + --wait-for-master Blocks until the Master is healthy. + + --version Version of DAP to use (defaults to latest build). + + --wait-for-master Waits for Conjur Master to become healthy. EOF @@ -468,6 +506,33 @@ function _trigger_master_failover_failover { echo 'Auto-failover takes about a minute to complete.' } +function _reenroll_failed_leader { + old_leader_number=$1 + new_leader_number=$2 + + _run "conjur-master-$new_leader_number.mycompany.local" "evoke cluster member remove conjur-master-$old_leader_number.mycompany.local" + docker-compose rm --stop --force conjur-master-$old_leader_number.mycompany.local + + docker-compose up --no-deps --detach "conjur-master-$old_leader_number.mycompany.local" + + # Generate a Seed File + _run "conjur-master-$new_leader_number.mycompany.local" \ + "evoke seed standby conjur-master-$old_leader_number.mycompany.local conjur-master-$new_leader_number.mycompany.local > /opt/cyberark/dap/seeds/standby-seed-$old_leader_number.tar" + + # Unpack and Configure + _run "conjur-master-$old_leader_number.mycompany.local" \ + "evoke unpack seed /opt/cyberark/dap/seeds/standby-seed-1.tar && evoke configure standby" + + _run "conjur-master-$new_leader_number.mycompany.local" \ + "evoke cluster member add conjur-master-$old_leader_number.mycompany.local" + + _run "conjur-master-$old_leader_number.mycompany.local" \ + "evoke cluster enroll --reenroll --cluster-machine-name conjur-master-$old_leader_number.mycompany.local --master-name conjur-master-$new_leader_number.mycompany.local production" + + _run "conjur-master-$new_leader_number.mycompany.local" \ + "evoke replication sync start" +} + function _import_certificates { bin/generate-certs @@ -603,25 +668,26 @@ STANDBY_COUNT=2 while true ; do case "$1" in - --provision-master ) CMD='_single_master' ; shift ;; - --promote-standby ) CMD='_perform_promotion' ; shift ;; - --import-custom-certificates ) CMD='_import_certificates' ; shift ;; - --rotate-custom-certificates ) CMD='_rotate_certificates' ; shift ;; - --provision-standbys ) CMD='_setup_standbys' ; shift ;; + --create-backup ) CMD='_create_backup' ; shift ;; + --dry-run ) DRY_RUN=true ; shift ;; --enable-auto-failover ) CMD='_enable_autofailover' ; shift ;; + --generate-dh ) export DHPATH=/tmp ; shift ;; + -h | --help ) _print_help ; shift ;; + --import-custom-certificates ) CMD='_import_certificates' ; shift ;; + --promote-standby ) CMD='_perform_promotion' ; shift ;; --provision-follower ) CMD='_setup_follower' ; shift ;; - --upgrade-master ) shift ; CMD="_upgrade_via_backup_restore $1" ; shift ;; - --version ) shift ; TAG=$1 ; shift ;; + --provision-master ) CMD='_single_master' ; shift ;; + --provision-standbys ) CMD='_setup_standbys' ; shift ;; + --reenroll-failed-leader ) shift ; CMD="_reenroll_failed_leader $1 $2" ; shift ; shift ;; + --restore-from-backup ) CMD='_restore_from_backup' ; shift ;; + --rotate-custom-certificates ) CMD='_rotate_certificates' ; shift ;; --standby-count ) shift; STANDBY_COUNT=$1 ; shift ;; - --dry-run ) DRY_RUN=true ; shift ;; - -h | --help ) _print_help ; shift ;; + --stop ) _stop ; shift ;; --trigger-failover ) CMD='_trigger_master_failover_failover' ; shift ;; - --create-backup ) CMD='_create_backup' ; shift ;; - --restore-from-backup ) CMD='_restore_from_backup' ; shift ;; --trust-follower-proxy ) CMD='_add_follower_proxy' ; shift ;; + --upgrade-master ) shift ; CMD="_upgrade_via_backup_restore $1" ; shift ;; + --version ) shift ; TAG=$1 ; shift ;; --wait-for-master ) CMD='_wait_for_master' ; shift ;; - --generate-dh ) export DHPATH=/tmp ; shift ;; - --stop ) _stop ; shift ;; * ) if [ -z "$1" ]; then break; else echo "$1 is not a valid option"; exit 1; fi;; esac done From 092b05624aacd837a8eaf7759e1bb9e728660e63 Mon Sep 17 00:00:00 2001 From: Jason Vanderhoof Date: Mon, 5 Dec 2022 15:45:28 -0700 Subject: [PATCH 3/3] Adds example using host factories for host creation --- artifacts/api-client/Dockerfile | 2 +- artifacts/api-client/api-script | 31 ++++++++++++++++++++++++++++- policy/sychronizer/synchronizer.yml | 7 +++++++ 3 files changed, 38 insertions(+), 2 deletions(-) create mode 100644 policy/sychronizer/synchronizer.yml diff --git a/artifacts/api-client/Dockerfile b/artifacts/api-client/Dockerfile index 817d7c31..b1577805 100644 --- a/artifacts/api-client/Dockerfile +++ b/artifacts/api-client/Dockerfile @@ -1,3 +1,3 @@ FROM alpine:3 -RUN apk add --no-cache curl jq +RUN apk add --no-cache curl jq coreutils diff --git a/artifacts/api-client/api-script b/artifacts/api-client/api-script index 40587a2b..d6ceedb6 100755 --- a/artifacts/api-client/api-script +++ b/artifacts/api-client/api-script @@ -12,7 +12,8 @@ Usage: bin/api [options] --enable-seed-service Enables Seed Service for Standbys and Followers --fetch-secrets Fetches previously set variables (use '--against-master' if only master is available) -h, --help Shows this help message - --load-policy + --host-factory-demo Provisions a layer with a host factory and enrolls a new host using a Host Factory + --load-policy Loads a policy into the provided namespace --load-sample-policy Loads a default set of policies to mimic a customer experience --load-sample-policy-and-values Loads policy and sets variable values (equivalent to running '--load-policy' and '--set-secrets') -p, --password Password used for authentication (default is 'MySecretP@ss1') @@ -173,6 +174,33 @@ enable_seed_service() { load_policy 'conjur' 'policy/seed-service/seed.yml' } +demonstrate_host_factory_enrollement() { + load_policy 'root' 'policy/sychronizer/synchronizer.yml' + local token=$(authenticate_user $USER $PASSWORD $URL) + # echo "token: $token" + local timestamp=$(date +"%Y-%m-%dT%H:%M:%S%:z" -d "+1 day") + # echo "timestamp: '$timestamp'" + local hf_token=$(curl \ + --header "Authorization: Token token=\"$token\"" \ + --insecure \ + --request POST \ + --data-urlencode "expiration=${timestamp}" \ + --data-urlencode "host_factory=demo:host_factory:synchronizers" \ + "$master_url/host_factory_tokens" | jq -r .[0].token) + # echo "HF: '$hf_token'" + + local host_name="new-host-$RANDOM" + # echo "host name: '$host_name'" + local host=$(curl \ + --header "Authorization: Token token=\"$hf_token\"" \ + --insecure \ + --request POST \ + --data-urlencode "id=$host_name" \ + "$master_url/host_factories/hosts" | jq .) + echo "host: $host" +} + + follower_url='http://conjur-follower.mycompany.local' master_url='https://conjur-master.mycompany.local' URL=$follower_url @@ -198,6 +226,7 @@ while true ; do --view-roles ) cmd="view_roles" ; shift ;; --rotate-api-key ) cmd="rotate_api_key" ;; --enable-seed-service ) shift ; cmd='enable_seed_service' ;; + --host-factory-demo ) shift ; cmd='demonstrate_host_factory_enrollement' ;; * ) if [ -z "$1" ]; then break; else echo "$1 is not a valid option"; exit 1; fi;; esac done diff --git a/policy/sychronizer/synchronizer.yml b/policy/sychronizer/synchronizer.yml new file mode 100644 index 00000000..9b53a01e --- /dev/null +++ b/policy/sychronizer/synchronizer.yml @@ -0,0 +1,7 @@ +- !policy + id: synchronizers + body: + - !layer + + - !host-factory + layer: [ !layer ]