From 6eabb51a493ba30f75a42ae8047a47e0e732e692 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 30 Jul 2024 08:33:11 -0500 Subject: [PATCH 01/33] PYTHON-4330 Add Kubernetes Support for OIDC --- .evergreen/config.yml | 93 ++++++++++++++-------- .evergreen/run-mongodb-oidc-test.sh | 2 + .evergreen/run-mongodb-remote-oidc-test.sh | 59 ++++++++++++++ pymongo/auth_oidc_shared.py | 11 +++ pymongo/auth_shared.py | 4 + test/auth/legacy/connection-string.json | 20 +++++ 6 files changed, 157 insertions(+), 32 deletions(-) create mode 100755 .evergreen/run-mongodb-remote-oidc-test.sh diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 45ac6c4140..f4922f484b 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -549,6 +549,16 @@ functions: args: - .evergreen/run-mongodb-oidc-test.sh + "run oidc remote test": + - command: subprocess.exec + type: test + params: + working_dir: "src" + binary: bash + include_expansions_in_env: ["DRIVERS_TOOLS", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN", "OIDC_ENV", "K8S_VARIANT"] + args: + - .evergreen/run-mongodb-oidc-remote-test.sh + "run aws auth test with aws credentials as environment variables": - command: shell.exec type: test @@ -1033,6 +1043,28 @@ task_groups: tasks: - oidc-auth-test-gcp + - name: testk8soidc_task_group + setup_group: + - func: fetch source + - func: prepare resources + - func: fix absolute paths + - func: make files executable + - command: subprocess.exec + params: + binary: bash + args: + - ${DRIVERS_TOOLS}/.evergreen/auth_oidc/k8s/setup.sh + teardown_task: + - command: subprocess.exec + params: + binary: bash + args: + - ${DRIVERS_TOOLS}/.evergreen/auth_oidc/k8s/teardown.sh + setup_group_can_fail_task: true + setup_group_timeout_secs: 1800 + tasks: + - oidc-auth-test-k8s + - name: testoidc_task_group setup_group: - func: fetch source @@ -2029,44 +2061,34 @@ tasks: - name: "oidc-auth-test" commands: - - func: "run oidc auth test with test credentials" + - func: "run oidc auth test" - name: "oidc-auth-test-azure" commands: - - command: shell.exec - type: test - params: - shell: bash - script: |- - set -o errexit - ${PREPARE_SHELL} - cd src - git add . - git commit -m "add files" - export AZUREOIDC_DRIVERS_TAR_FILE=/tmp/mongo-python-driver.tgz - git archive -o $AZUREOIDC_DRIVERS_TAR_FILE HEAD - export AZUREOIDC_TEST_CMD="OIDC_ENV=azure ./.evergreen/run-mongodb-oidc-test.sh" - bash $DRIVERS_TOOLS/.evergreen/auth_oidc/azure/run-driver-test.sh + - func: "run oidc remote test" + vars: + OIDC_ENV: azure - name: "oidc-auth-test-gcp" commands: - - command: shell.exec - type: test - params: - shell: bash - script: |- - set -o errexit - ${PREPARE_SHELL} - cd src - git add . - git commit -m "add files" - export GCPOIDC_DRIVERS_TAR_FILE=/tmp/mongo-python-driver.tgz - git archive -o $GCPOIDC_DRIVERS_TAR_FILE HEAD - # Define the command to run on the VM. - # Ensure that we source the environment file created for us, set up any other variables we need, - # and then run our test suite on the vm. - export GCPOIDC_TEST_CMD="OIDC_ENV=gcp ./.evergreen/run-mongodb-oidc-test.sh" - bash $DRIVERS_TOOLS/.evergreen/auth_oidc/gcp/run-driver-test.sh + - func: "run oidc remote test" + vars: + OIDC_ENV: gcp + + - name: "oidc-auth-test-k8s" + commands: + - func: "run oidc remote test" + vars: + K8S_VARIANT: eks + OIDC_ENV: k8s + - func: "run oidc remote test" + vars: + K8S_VARIANT: gke + OIDC_ENV: k8s + # - func: "run oidc remote test" + # vars: + # K8S_VARIANT: aks + # OIDC_ENV: k8s - name: "test-fips-standalone" tags: ["fips"] @@ -3089,6 +3111,13 @@ buildvariants: - name: testgcpoidc_task_group batchtime: 20160 # Use a batchtime of 14 days as suggested by the CSFLE test README +- name: testk8soidc-variant + display_name: "OIDC Auth K8S" + run_on: ubuntu2204-small + tasks: + - name: testk8soidc_task_group + batchtime: 20160 # Use a batchtime of 14 days as suggested by the CSFLE test README + - matrix_name: "aws-auth-test" matrix_spec: platform: [ubuntu-20.04] diff --git a/.evergreen/run-mongodb-oidc-test.sh b/.evergreen/run-mongodb-oidc-test.sh index 0c34912c8a..dc4f5eefbc 100755 --- a/.evergreen/run-mongodb-oidc-test.sh +++ b/.evergreen/run-mongodb-oidc-test.sh @@ -21,6 +21,8 @@ elif [ $OIDC_ENV == "azure" ]; then elif [ $OIDC_ENV == "gcp" ]; then source ./secrets-export.sh +elif [ $OIDC_ENV == "k8s" ]; then + # Nothing extra to do. else echo "Unrecognized OIDC_ENV $OIDC_ENV" exit 1 diff --git a/.evergreen/run-mongodb-remote-oidc-test.sh b/.evergreen/run-mongodb-remote-oidc-test.sh new file mode 100755 index 0000000000..e46e742983 --- /dev/null +++ b/.evergreen/run-mongodb-remote-oidc-test.sh @@ -0,0 +1,59 @@ +#!/bin/bash + +set +x # Disable debug trace +set -eu + +echo "Running MONGODB-OIDC remote tests" + +OIDC_ENV=${OIDC_ENV:-"test"} + +# Make sure DRIVERS_TOOLS is set. +if [ -z "$DRIVERS_TOOLS" ]; then + echo "Must specify DRIVERS_TOOLS" + exit 1 +fi + +# Set up the remote files to test. +git add . +git commit -m "add files" +export TEST_TAR_FILE=/tmp/mongo-python-driver.tgz +git archive -o $TEST_TAR_FILE HEAD + +pushd $DRIVERS_TOOLS + +if [ $OIDC_ENV == "test" ]; then + echo "Test OIDC environment does not support remote test!" + exit 1 + +elif [ $OIDC_ENV == "azure" ]; then + export AZUREOIDC_DRIVERS_TAR_FILE=$TEST_TAR_FILE + export AZUREOIDC_TEST_CMD="OIDC_ENV=azure ./.evergreen/run-mongodb-oidc-test.sh" + bash ./.evergreen/auth_oidc/azure/run-driver-test.sh + +elif [ $OIDC_ENV == "gcp" ]; then + export GCPOIDC_DRIVERS_TAR_FILE=$TEST_TAR_FILE + export GCPOIDC_TEST_CMD="OIDC_ENV=gcp ./.evergreen/run-mongodb-oidc-test.sh" + bash ./.evergreen/auth_oidc/gcp/run-driver-test.sh + +elif [ $OIDC_ENV == "k8s" ]; then + # Make sure K8S_VARIANT is set. + if [ -z "$K8S_VARIANT" ]; then + echo "Must specify K8S_VARIANT" + popd + exit 1 + fi + + bash ./.evergreen/auth_oidc/k8s/setup-pod.sh + bash ./.evergreen/auth_oidc/k8s/run-self-test.sh + export K8S_DRIVERS_TAR_FILE=$TEST_TAR_FILE + export K8S_TEST_CMD="OIDC_ENV=k8s ./.evergreen/run-mongodb-oidc-test.sh" + bash ./.evergreen/auth_oidc/k8s/run-driver-test.sh + bash ./.evergreen/auth_oidc/k8s/teardown-pod.sh + +else + echo "Unrecognized OIDC_ENV $OIDC_ENV" + pod + exit 1 +fi + +popd diff --git a/pymongo/auth_oidc_shared.py b/pymongo/auth_oidc_shared.py index 5e3603fa31..bd38709700 100644 --- a/pymongo/auth_oidc_shared.py +++ b/pymongo/auth_oidc_shared.py @@ -116,3 +116,14 @@ def __init__(self, token_resource: str) -> None: def fetch(self, context: OIDCCallbackContext) -> OIDCCallbackResult: resp = _get_gcp_response(self.token_resource, context.timeout_seconds) return OIDCCallbackResult(access_token=resp["access_token"]) + + +class _OIDCK8SCallback(OIDCCallback): + def fetch(self, context: OIDCCallbackContext) -> OIDCCallbackResult: + fname = "/var/run/secrets/kubernetes.io/serviceaccount/token" + for key in ["AZURE_FEDERATED_TOKEN_FILE", "AWS_WEB_IDENTITY_TOKEN_FILE"]: + if key in os.environ: + fname = os.environ[key] + with open(fname) as fid: + token = fid.read() + return OIDCCallbackResult(access_token=token) diff --git a/pymongo/auth_shared.py b/pymongo/auth_shared.py index 7e3acd9dfb..f454a2704a 100644 --- a/pymongo/auth_shared.py +++ b/pymongo/auth_shared.py @@ -26,6 +26,7 @@ from pymongo.auth_oidc_shared import ( _OIDCAzureCallback, _OIDCGCPCallback, + _OIDCK8SCallback, _OIDCProperties, _OIDCTestCallback, ) @@ -180,6 +181,9 @@ def _build_credentials_tuple( "GCP provider for MONGODB-OIDC requires a TOKEN_RESOURCE auth mechanism property" ) callback = _OIDCGCPCallback(token_resource) + elif environ == "k8s": + passwd = None + callback = _OIDCK8SCallback() else: raise ConfigurationError(f"unrecognized ENVIRONMENT for MONGODB-OIDC: {environ}") else: diff --git a/test/auth/legacy/connection-string.json b/test/auth/legacy/connection-string.json index 57fd9d4a11..0abc1d005f 100644 --- a/test/auth/legacy/connection-string.json +++ b/test/auth/legacy/connection-string.json @@ -625,6 +625,26 @@ "uri": "mongodb://user:pass@localhost/?authMechanism=MONGODB-OIDC&authMechanismProperties=ENVIRONMENT:gcp", "valid": false, "credential": null + }, + { + "description": "should recognise the mechanism with k8s provider (MONGODB-OIDC)", + "uri": "mongodb://localhost/?authMechanism=MONGODB-OIDC&authMechanismProperties=ENVIRONMENT:k8s", + "valid": true, + "credential": { + "username": null, + "password": null, + "source": "$external", + "mechanism": "MONGODB-OIDC", + "mechanism_properties": { + "ENVIRONMENT": "k8s" + } + } + }, + { + "description": "should throw an error for a username and password with k8s provider (MONGODB-OIDC)", + "uri": "mongodb://user:pass@localhost/?authMechanism=MONGODB-OIDC&authMechanismProperties=ENVIRONMENT:k8s", + "valid": false, + "credential": null } ] } \ No newline at end of file From 0cb2a177263be3496fc0a2fb9d2ca5c832eb5f43 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 30 Jul 2024 08:34:12 -0500 Subject: [PATCH 02/33] fix config --- .evergreen/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index f4922f484b..fe12b8dbee 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -2061,7 +2061,7 @@ tasks: - name: "oidc-auth-test" commands: - - func: "run oidc auth test" + - func: "run oidc auth test with credentials" - name: "oidc-auth-test-azure" commands: From 77a2409faebdb84c3506924844e88214b81a40f3 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 30 Jul 2024 08:34:31 -0500 Subject: [PATCH 03/33] fix config --- .evergreen/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index fe12b8dbee..0c57f7f7c6 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -2061,7 +2061,7 @@ tasks: - name: "oidc-auth-test" commands: - - func: "run oidc auth test with credentials" + - func: "run oidc auth test with test credentials" - name: "oidc-auth-test-azure" commands: From c1f6c58321371b803864282029ee82693754c603 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 30 Jul 2024 08:49:04 -0500 Subject: [PATCH 04/33] rename file --- ...ongodb-remote-oidc-test.sh => run-mongodb-oidc-remote-test.sh} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .evergreen/{run-mongodb-remote-oidc-test.sh => run-mongodb-oidc-remote-test.sh} (100%) diff --git a/.evergreen/run-mongodb-remote-oidc-test.sh b/.evergreen/run-mongodb-oidc-remote-test.sh similarity index 100% rename from .evergreen/run-mongodb-remote-oidc-test.sh rename to .evergreen/run-mongodb-oidc-remote-test.sh From 5649291460a0073d972bdf85dcf352cb83e2d3f7 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 30 Jul 2024 10:24:52 -0500 Subject: [PATCH 05/33] fix syntax --- .evergreen/run-mongodb-oidc-test.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.evergreen/run-mongodb-oidc-test.sh b/.evergreen/run-mongodb-oidc-test.sh index dc4f5eefbc..22864528c0 100755 --- a/.evergreen/run-mongodb-oidc-test.sh +++ b/.evergreen/run-mongodb-oidc-test.sh @@ -22,7 +22,8 @@ elif [ $OIDC_ENV == "gcp" ]; then source ./secrets-export.sh elif [ $OIDC_ENV == "k8s" ]; then - # Nothing extra to do. + echo "Running oidc on k8s" + else echo "Unrecognized OIDC_ENV $OIDC_ENV" exit 1 From a55ab75f157353fa4089cf27df0c6ccf104deba6 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 30 Jul 2024 12:37:36 -0500 Subject: [PATCH 06/33] assume role --- .evergreen/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 0c57f7f7c6..6759ad5027 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -1049,6 +1049,7 @@ task_groups: - func: prepare resources - func: fix absolute paths - func: make files executable + - func: assume ec2 role - command: subprocess.exec params: binary: bash From 8cd4a8f8bdace6eb5b3774fbb29f5c7b0cd9c249 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 30 Jul 2024 12:53:51 -0500 Subject: [PATCH 07/33] add mongodb uri --- .evergreen/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 6759ad5027..c5c698918a 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -555,7 +555,7 @@ functions: params: working_dir: "src" binary: bash - include_expansions_in_env: ["DRIVERS_TOOLS", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN", "OIDC_ENV", "K8S_VARIANT"] + include_expansions_in_env: ["DRIVERS_TOOLS", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN", "OIDC_ENV", "K8S_VARIANT", "MONGODB_URI"] args: - .evergreen/run-mongodb-oidc-remote-test.sh From 5e387a67e108161677384e11943b4aad8964134c Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 30 Jul 2024 13:31:02 -0500 Subject: [PATCH 08/33] fix mongodb uri handling --- .evergreen/config.yml | 2 +- .evergreen/run-mongodb-oidc-remote-test.sh | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index c5c698918a..6759ad5027 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -555,7 +555,7 @@ functions: params: working_dir: "src" binary: bash - include_expansions_in_env: ["DRIVERS_TOOLS", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN", "OIDC_ENV", "K8S_VARIANT", "MONGODB_URI"] + include_expansions_in_env: ["DRIVERS_TOOLS", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN", "OIDC_ENV", "K8S_VARIANT"] args: - .evergreen/run-mongodb-oidc-remote-test.sh diff --git a/.evergreen/run-mongodb-oidc-remote-test.sh b/.evergreen/run-mongodb-oidc-remote-test.sh index e46e742983..5841f89b26 100755 --- a/.evergreen/run-mongodb-oidc-remote-test.sh +++ b/.evergreen/run-mongodb-oidc-remote-test.sh @@ -47,6 +47,7 @@ elif [ $OIDC_ENV == "k8s" ]; then bash ./.evergreen/auth_oidc/k8s/run-self-test.sh export K8S_DRIVERS_TAR_FILE=$TEST_TAR_FILE export K8S_TEST_CMD="OIDC_ENV=k8s ./.evergreen/run-mongodb-oidc-test.sh" + source ./.evergreen/auth_oidc/k8s/secrets-export.sh # for MONGODB_URI bash ./.evergreen/auth_oidc/k8s/run-driver-test.sh bash ./.evergreen/auth_oidc/k8s/teardown-pod.sh From 12985c5a3b97e87d5935be0c4bafa003fa549a10 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 30 Jul 2024 19:31:51 -0500 Subject: [PATCH 09/33] use branch --- .evergreen/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 6759ad5027..a40183b6ee 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -95,7 +95,7 @@ functions: # If this was a patch build, doing a fresh clone would not actually test the patch cp -R ${PROJECT_DIRECTORY}/ $DRIVERS_TOOLS else - git clone https://github.com/mongodb-labs/drivers-evergreen-tools.git $DRIVERS_TOOLS + git clone --branch DRIVERS-2882-driver-test https://github.com/blink1073/drivers-evergreen-tools.git $DRIVERS_TOOLS fi echo "{ \"releases\": { \"default\": \"$MONGODB_BINARIES\" }}" > $MONGO_ORCHESTRATION_HOME/orchestration.config From 31eb136b2f8a681feef077f43b7d869608a21519 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 30 Jul 2024 21:30:48 -0500 Subject: [PATCH 10/33] fix placeholder map --- test/unified_format.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/unified_format.py b/test/unified_format.py index 90f36c4a77..ca4558c011 100644 --- a/test/unified_format.py +++ b/test/unified_format.py @@ -179,6 +179,8 @@ "ENVIRONMENT": "gcp", "TOKEN_RESOURCE": os.environ["GCPOIDC_AUDIENCE"], } +elif OIDC_ENV == "k8s": + PLACEHOLDER_MAP["/uriOptions/authMechanismProperties"] = {"ENVIRONMENT": "k8s"} def interrupt_loop(): From 05f3ab9e6799af32620316bcfc3a3a4400919186 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 31 Jul 2024 09:27:59 -0500 Subject: [PATCH 11/33] handle prose tests --- pymongo/auth_oidc_shared.py | 17 ++++++++++------- test/auth_oidc/test_auth_oidc.py | 11 ++++++----- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/pymongo/auth_oidc_shared.py b/pymongo/auth_oidc_shared.py index bd38709700..9e0acaf6c8 100644 --- a/pymongo/auth_oidc_shared.py +++ b/pymongo/auth_oidc_shared.py @@ -120,10 +120,13 @@ def fetch(self, context: OIDCCallbackContext) -> OIDCCallbackResult: class _OIDCK8SCallback(OIDCCallback): def fetch(self, context: OIDCCallbackContext) -> OIDCCallbackResult: - fname = "/var/run/secrets/kubernetes.io/serviceaccount/token" - for key in ["AZURE_FEDERATED_TOKEN_FILE", "AWS_WEB_IDENTITY_TOKEN_FILE"]: - if key in os.environ: - fname = os.environ[key] - with open(fname) as fid: - token = fid.read() - return OIDCCallbackResult(access_token=token) + return OIDCCallbackResult(access_token=_get_k8s_token()) + + +def _get_k8s_token() -> str: + fname = "/var/run/secrets/kubernetes.io/serviceaccount/token" + for key in ["AZURE_FEDERATED_TOKEN_FILE", "AWS_WEB_IDENTITY_TOKEN_FILE"]: + if key in os.environ: + fname = os.environ[key] + with open(fname) as fid: + return fid.read() diff --git a/test/auth_oidc/test_auth_oidc.py b/test/auth_oidc/test_auth_oidc.py index 406ea5ec1b..28fd94174f 100644 --- a/test/auth_oidc/test_auth_oidc.py +++ b/test/auth_oidc/test_auth_oidc.py @@ -36,15 +36,12 @@ from pymongo import MongoClient from pymongo._azure_helpers import _get_azure_response from pymongo._gcp_helpers import _get_gcp_response +from pymongo.auth_oidc_shared import _get_k8s_token from pymongo.cursor_shared import CursorType from pymongo.errors import AutoReconnect, ConfigurationError, OperationFailure from pymongo.hello import HelloCompat from pymongo.operations import InsertOne -from pymongo.synchronous.auth_oidc import ( - OIDCCallback, - OIDCCallbackContext, - OIDCCallbackResult, -) +from pymongo.synchronous.auth_oidc import OIDCCallback, OIDCCallbackContext, OIDCCallbackResult from pymongo.uri_parser import parse_uri ROOT = Path(__file__).parent.parent.resolve() @@ -87,6 +84,10 @@ def get_token(self, username=None): opts = parse_uri(self.uri_single)["options"] token_aud = opts["authmechanismproperties"]["TOKEN_RESOURCE"] return _get_gcp_response(token_aud, username)["access_token"] + elif ENVIRON == "k8s": + return _get_k8s_token() + else: + raise ValueError(f"Unknown ENVIRON: {ENVIRON}") @contextmanager def fail_point(self, command_args): From 0acf73baa3eaf356b546b2c42474866d22c140b6 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 31 Jul 2024 09:45:41 -0500 Subject: [PATCH 12/33] fix archive creation --- .evergreen/run-mongodb-oidc-remote-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen/run-mongodb-oidc-remote-test.sh b/.evergreen/run-mongodb-oidc-remote-test.sh index 5841f89b26..bb90bddf07 100755 --- a/.evergreen/run-mongodb-oidc-remote-test.sh +++ b/.evergreen/run-mongodb-oidc-remote-test.sh @@ -15,7 +15,7 @@ fi # Set up the remote files to test. git add . -git commit -m "add files" +git commit -m "add files" || true export TEST_TAR_FILE=/tmp/mongo-python-driver.tgz git archive -o $TEST_TAR_FILE HEAD From cd2dee03503bc5041c1e46c76b8b14ff2bc9e8f2 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 31 Jul 2024 12:22:35 -0500 Subject: [PATCH 13/33] use upstream --- .evergreen/config.yml | 94 +++++++++++++++---------------------------- 1 file changed, 32 insertions(+), 62 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index a40183b6ee..45ac6c4140 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -95,7 +95,7 @@ functions: # If this was a patch build, doing a fresh clone would not actually test the patch cp -R ${PROJECT_DIRECTORY}/ $DRIVERS_TOOLS else - git clone --branch DRIVERS-2882-driver-test https://github.com/blink1073/drivers-evergreen-tools.git $DRIVERS_TOOLS + git clone https://github.com/mongodb-labs/drivers-evergreen-tools.git $DRIVERS_TOOLS fi echo "{ \"releases\": { \"default\": \"$MONGODB_BINARIES\" }}" > $MONGO_ORCHESTRATION_HOME/orchestration.config @@ -549,16 +549,6 @@ functions: args: - .evergreen/run-mongodb-oidc-test.sh - "run oidc remote test": - - command: subprocess.exec - type: test - params: - working_dir: "src" - binary: bash - include_expansions_in_env: ["DRIVERS_TOOLS", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN", "OIDC_ENV", "K8S_VARIANT"] - args: - - .evergreen/run-mongodb-oidc-remote-test.sh - "run aws auth test with aws credentials as environment variables": - command: shell.exec type: test @@ -1043,29 +1033,6 @@ task_groups: tasks: - oidc-auth-test-gcp - - name: testk8soidc_task_group - setup_group: - - func: fetch source - - func: prepare resources - - func: fix absolute paths - - func: make files executable - - func: assume ec2 role - - command: subprocess.exec - params: - binary: bash - args: - - ${DRIVERS_TOOLS}/.evergreen/auth_oidc/k8s/setup.sh - teardown_task: - - command: subprocess.exec - params: - binary: bash - args: - - ${DRIVERS_TOOLS}/.evergreen/auth_oidc/k8s/teardown.sh - setup_group_can_fail_task: true - setup_group_timeout_secs: 1800 - tasks: - - oidc-auth-test-k8s - - name: testoidc_task_group setup_group: - func: fetch source @@ -2066,30 +2033,40 @@ tasks: - name: "oidc-auth-test-azure" commands: - - func: "run oidc remote test" - vars: - OIDC_ENV: azure + - command: shell.exec + type: test + params: + shell: bash + script: |- + set -o errexit + ${PREPARE_SHELL} + cd src + git add . + git commit -m "add files" + export AZUREOIDC_DRIVERS_TAR_FILE=/tmp/mongo-python-driver.tgz + git archive -o $AZUREOIDC_DRIVERS_TAR_FILE HEAD + export AZUREOIDC_TEST_CMD="OIDC_ENV=azure ./.evergreen/run-mongodb-oidc-test.sh" + bash $DRIVERS_TOOLS/.evergreen/auth_oidc/azure/run-driver-test.sh - name: "oidc-auth-test-gcp" commands: - - func: "run oidc remote test" - vars: - OIDC_ENV: gcp - - - name: "oidc-auth-test-k8s" - commands: - - func: "run oidc remote test" - vars: - K8S_VARIANT: eks - OIDC_ENV: k8s - - func: "run oidc remote test" - vars: - K8S_VARIANT: gke - OIDC_ENV: k8s - # - func: "run oidc remote test" - # vars: - # K8S_VARIANT: aks - # OIDC_ENV: k8s + - command: shell.exec + type: test + params: + shell: bash + script: |- + set -o errexit + ${PREPARE_SHELL} + cd src + git add . + git commit -m "add files" + export GCPOIDC_DRIVERS_TAR_FILE=/tmp/mongo-python-driver.tgz + git archive -o $GCPOIDC_DRIVERS_TAR_FILE HEAD + # Define the command to run on the VM. + # Ensure that we source the environment file created for us, set up any other variables we need, + # and then run our test suite on the vm. + export GCPOIDC_TEST_CMD="OIDC_ENV=gcp ./.evergreen/run-mongodb-oidc-test.sh" + bash $DRIVERS_TOOLS/.evergreen/auth_oidc/gcp/run-driver-test.sh - name: "test-fips-standalone" tags: ["fips"] @@ -3112,13 +3089,6 @@ buildvariants: - name: testgcpoidc_task_group batchtime: 20160 # Use a batchtime of 14 days as suggested by the CSFLE test README -- name: testk8soidc-variant - display_name: "OIDC Auth K8S" - run_on: ubuntu2204-small - tasks: - - name: testk8soidc_task_group - batchtime: 20160 # Use a batchtime of 14 days as suggested by the CSFLE test README - - matrix_name: "aws-auth-test" matrix_spec: platform: [ubuntu-20.04] From 6973b35075bdc4cd0827369180354bb132e9675d Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 2 Oct 2024 06:56:55 -0500 Subject: [PATCH 14/33] add k8s test --- .evergreen/config.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 14e3426b32..cff47e200b 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -511,7 +511,7 @@ functions: params: working_dir: "src" binary: bash - include_expansions_in_env: ["DRIVERS_TOOLS", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] + include_expansions_in_env: ["DRIVERS_TOOLS", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN", "OIDC_ENV"] args: - .evergreen/run-mongodb-oidc-test.sh @@ -1846,6 +1846,9 @@ tasks: - name: "oidc-auth-test" commands: - func: "run oidc auth test with test credentials" + - func: "run oidc auth test with test credentials" + vars: + OIDC_ENV: "k8s" - name: "oidc-auth-test-azure" commands: From d6435a759e771bce989236da4c1b9844b1876b6a Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 2 Oct 2024 07:27:00 -0500 Subject: [PATCH 15/33] use remove test script --- .evergreen/config.yml | 88 +++++++++++++++---------- test/auth/legacy/connection-string.json | 30 ++++----- 2 files changed, 70 insertions(+), 48 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index cff47e200b..c1e096c527 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -511,7 +511,7 @@ functions: params: working_dir: "src" binary: bash - include_expansions_in_env: ["DRIVERS_TOOLS", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN", "OIDC_ENV"] + include_expansions_in_env: ["DRIVERS_TOOLS", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] args: - .evergreen/run-mongodb-oidc-test.sh @@ -915,6 +915,31 @@ task_groups: tasks: - oidc-auth-test-gcp + - name: testk8soidc_task_group + setup_group: + - func: fetch source + - func: prepare resources + - func: fix absolute paths + - func: make files executable + - command: subprocess.exec + params: + binary: bash + env: + GCPOIDC_VMNAME_PREFIX: "PYTHON_DRIVER" + args: + - ${DRIVERS_TOOLS}/.evergreen/auth_oidc/k8s/setup.sh + - local + teardown_task: + - command: subprocess.exec + params: + binary: bash + args: + - ${DRIVERS_TOOLS}/.evergreen/auth_oidc/k8s/teardown.sh + setup_group_can_fail_task: true + setup_group_timeout_secs: 1800 + tasks: + - oidc-auth-test-k8s + - name: testoidc_task_group setup_group: - func: fetch source @@ -1846,46 +1871,36 @@ tasks: - name: "oidc-auth-test" commands: - func: "run oidc auth test with test credentials" - - func: "run oidc auth test with test credentials" - vars: - OIDC_ENV: "k8s" - name: "oidc-auth-test-azure" commands: - - command: shell.exec + - command: subprocess.exec type: test - params: - shell: bash - script: |- - set -o errexit - . src/.evergreen/scripts/env.sh - cd src - git add . - git commit -m "add files" - export AZUREOIDC_DRIVERS_TAR_FILE=/tmp/mongo-python-driver.tgz - git archive -o $AZUREOIDC_DRIVERS_TAR_FILE HEAD - export AZUREOIDC_TEST_CMD="OIDC_ENV=azure ./.evergreen/run-mongodb-oidc-test.sh" - bash $DRIVERS_TOOLS/.evergreen/auth_oidc/azure/run-driver-test.sh + binary: bash + env: + OIDC_ENV: azure + args: + - ${PROJECT_DIRECTORY}/.evergreen/run-mongodb-oidc-remote-test.sh - name: "oidc-auth-test-gcp" commands: - - command: shell.exec + - command: subprocess.exec type: test - params: - shell: bash - script: |- - set -o errexit - . src/.evergreen/scripts/env.sh - cd src - git add . - git commit -m "add files" - export GCPOIDC_DRIVERS_TAR_FILE=/tmp/mongo-python-driver.tgz - git archive -o $GCPOIDC_DRIVERS_TAR_FILE HEAD - # Define the command to run on the VM. - # Ensure that we source the environment file created for us, set up any other variables we need, - # and then run our test suite on the vm. - export GCPOIDC_TEST_CMD="OIDC_ENV=gcp ./.evergreen/run-mongodb-oidc-test.sh" - bash $DRIVERS_TOOLS/.evergreen/auth_oidc/gcp/run-driver-test.sh + binary: bash + env: + OIDC_ENV: gcp + args: + - ${PROJECT_DIRECTORY}/.evergreen/run-mongodb-oidc-remote-test.sh + + - name: "oidc-auth-test-k8s" + commands: + - command: subprocess.exec + type: test + binary: bash + env: + OIDC_ENV: k8s + args: + - ${PROJECT_DIRECTORY}/.evergreen/run-mongodb-oidc-remote-test.sh - name: "test-fips-standalone" tags: ["fips"] @@ -2955,6 +2970,13 @@ buildvariants: - name: testgcpoidc_task_group batchtime: 20160 # Use a batchtime of 14 days as suggested by the CSFLE test README +- name: testk8soidc-variant + display_name: "OIDC Auth K8S" + run_on: ubuntu2204-small + tasks: + - name: testk8soidc_task_group + batchtime: 20160 # Use a batchtime of 14 days as suggested by the CSFLE test README + - matrix_name: "aws-auth-test" matrix_spec: platform: [ubuntu-20.04] diff --git a/test/auth/legacy/connection-string.json b/test/auth/legacy/connection-string.json index 0abc1d005f..61f9e548c8 100644 --- a/test/auth/legacy/connection-string.json +++ b/test/auth/legacy/connection-string.json @@ -627,24 +627,24 @@ "credential": null }, { - "description": "should recognise the mechanism with k8s provider (MONGODB-OIDC)", - "uri": "mongodb://localhost/?authMechanism=MONGODB-OIDC&authMechanismProperties=ENVIRONMENT:k8s", - "valid": true, - "credential": { - "username": null, - "password": null, - "source": "$external", - "mechanism": "MONGODB-OIDC", - "mechanism_properties": { - "ENVIRONMENT": "k8s" - } + "description": "should recognise the mechanism with k8s provider (MONGODB-OIDC)", + "uri": "mongodb://localhost/?authMechanism=MONGODB-OIDC&authMechanismProperties=ENVIRONMENT:k8s", + "valid": true, + "credential": { + "username": null, + "password": null, + "source": "$external", + "mechanism": "MONGODB-OIDC", + "mechanism_properties": { + "ENVIRONMENT": "k8s" } + } }, { - "description": "should throw an error for a username and password with k8s provider (MONGODB-OIDC)", - "uri": "mongodb://user:pass@localhost/?authMechanism=MONGODB-OIDC&authMechanismProperties=ENVIRONMENT:k8s", - "valid": false, - "credential": null + "description": "should throw an error for a username and password with k8s provider (MONGODB-OIDC)", + "uri": "mongodb://user:pass@localhost/?authMechanism=MONGODB-OIDC&authMechanismProperties=ENVIRONMENT:k8s", + "valid": false, + "credential": null } ] } \ No newline at end of file From 2a1ef08178f71c8a5ce0f206dd12abe60edaeb3e Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 2 Oct 2024 07:28:15 -0500 Subject: [PATCH 16/33] remove local --- .evergreen/config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index c1e096c527..4034007884 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -928,7 +928,6 @@ task_groups: GCPOIDC_VMNAME_PREFIX: "PYTHON_DRIVER" args: - ${DRIVERS_TOOLS}/.evergreen/auth_oidc/k8s/setup.sh - - local teardown_task: - command: subprocess.exec params: From 55caf3ad9778959cfb5ecf3799dbc015ef9bdc81 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 2 Oct 2024 07:28:25 -0500 Subject: [PATCH 17/33] cleanup --- .evergreen/config.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 4034007884..ffb2caa3f8 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -924,8 +924,6 @@ task_groups: - command: subprocess.exec params: binary: bash - env: - GCPOIDC_VMNAME_PREFIX: "PYTHON_DRIVER" args: - ${DRIVERS_TOOLS}/.evergreen/auth_oidc/k8s/setup.sh teardown_task: From ca7a7b1b485d67f85e5847d8bc4f85075abf4085 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 2 Oct 2024 07:47:56 -0500 Subject: [PATCH 18/33] fix syntax --- .evergreen/config.yml | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index ffb2caa3f8..ed97f93c1f 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -1873,31 +1873,34 @@ tasks: commands: - command: subprocess.exec type: test - binary: bash - env: - OIDC_ENV: azure - args: - - ${PROJECT_DIRECTORY}/.evergreen/run-mongodb-oidc-remote-test.sh + params: + binary: bash + env: + OIDC_ENV: azure + args: + - ${PROJECT_DIRECTORY}/.evergreen/run-mongodb-oidc-remote-test.sh - name: "oidc-auth-test-gcp" commands: - command: subprocess.exec type: test - binary: bash - env: - OIDC_ENV: gcp - args: - - ${PROJECT_DIRECTORY}/.evergreen/run-mongodb-oidc-remote-test.sh + params: + binary: bash + env: + OIDC_ENV: gcp + args: + - ${PROJECT_DIRECTORY}/.evergreen/run-mongodb-oidc-remote-test.sh - name: "oidc-auth-test-k8s" commands: - command: subprocess.exec type: test - binary: bash - env: - OIDC_ENV: k8s - args: - - ${PROJECT_DIRECTORY}/.evergreen/run-mongodb-oidc-remote-test.sh + params: + binary: bash + env: + OIDC_ENV: k8s + args: + - ${PROJECT_DIRECTORY}/.evergreen/run-mongodb-oidc-remote-test.sh - name: "test-fips-standalone" tags: ["fips"] From f80ab402cf61b3ffbef3945b1209ccd7fb83cfd0 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 2 Oct 2024 08:00:20 -0500 Subject: [PATCH 19/33] include expansions --- .evergreen/config.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index ed97f93c1f..3a2a42e626 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -1877,6 +1877,7 @@ tasks: binary: bash env: OIDC_ENV: azure + include_expansions_in_env: ["DRIVERS_TOOLS"] args: - ${PROJECT_DIRECTORY}/.evergreen/run-mongodb-oidc-remote-test.sh @@ -1888,6 +1889,7 @@ tasks: binary: bash env: OIDC_ENV: gcp + include_expansions_in_env: ["DRIVERS_TOOLS"] args: - ${PROJECT_DIRECTORY}/.evergreen/run-mongodb-oidc-remote-test.sh @@ -1899,6 +1901,7 @@ tasks: binary: bash env: OIDC_ENV: k8s + include_expansions_in_env: ["DRIVERS_TOOLS"] args: - ${PROJECT_DIRECTORY}/.evergreen/run-mongodb-oidc-remote-test.sh From 0edaf01f749c1d7cdd9149cd6f9cfb5028a6faf3 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 2 Oct 2024 08:09:26 -0500 Subject: [PATCH 20/33] fix working dir --- .evergreen/config.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 3a2a42e626..352d619840 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -1875,6 +1875,7 @@ tasks: type: test params: binary: bash + working_dir: src env: OIDC_ENV: azure include_expansions_in_env: ["DRIVERS_TOOLS"] @@ -1887,6 +1888,7 @@ tasks: type: test params: binary: bash + working_dir: src env: OIDC_ENV: gcp include_expansions_in_env: ["DRIVERS_TOOLS"] @@ -1899,6 +1901,7 @@ tasks: type: test params: binary: bash + working_dir: src env: OIDC_ENV: k8s include_expansions_in_env: ["DRIVERS_TOOLS"] From ade3f4f8e3cc9d5efc6f6b2e3f4c42d344a3e6f5 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 2 Oct 2024 10:02:26 -0500 Subject: [PATCH 21/33] set k8s variant --- .evergreen/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 352d619840..99f41a322d 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -1904,6 +1904,7 @@ tasks: working_dir: src env: OIDC_ENV: k8s + K8S_VARIANT: eks include_expansions_in_env: ["DRIVERS_TOOLS"] args: - ${PROJECT_DIRECTORY}/.evergreen/run-mongodb-oidc-remote-test.sh From 6743b6a7c472918502b71e6fc5ff2f23b416a30a Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 2 Oct 2024 10:20:44 -0500 Subject: [PATCH 22/33] assume role --- .evergreen/config.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 99f41a322d..128eb03a90 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -921,6 +921,9 @@ task_groups: - func: prepare resources - func: fix absolute paths - func: make files executable + - command: ec2.assume_role + params: + role_arn: ${assume_role_arn} - command: subprocess.exec params: binary: bash From 4b357d242327c33f16cabc4b1b6895130b14a4bd Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 2 Oct 2024 10:36:05 -0500 Subject: [PATCH 23/33] fix usage of variables --- .evergreen/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 128eb03a90..c3bb3a4813 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -1908,7 +1908,7 @@ tasks: env: OIDC_ENV: k8s K8S_VARIANT: eks - include_expansions_in_env: ["DRIVERS_TOOLS"] + include_expansions_in_env: ["DRIVERS_TOOLS", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] args: - ${PROJECT_DIRECTORY}/.evergreen/run-mongodb-oidc-remote-test.sh From 2ec6fc8cdf3a9243be2dd671082feb369180c86b Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 2 Oct 2024 11:00:32 -0500 Subject: [PATCH 24/33] fix role --- .evergreen/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index c3bb3a4813..48cb55fa51 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -923,7 +923,7 @@ task_groups: - func: make files executable - command: ec2.assume_role params: - role_arn: ${assume_role_arn} + role_arn: ${aws_test_secrets_role} - command: subprocess.exec params: binary: bash From ff9ac07d0404a974cc6cc23012495490163f0592 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 2 Oct 2024 11:15:52 -0500 Subject: [PATCH 25/33] test eks and gke --- .evergreen/config.yml | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 48cb55fa51..ccd2c86462 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -938,7 +938,16 @@ task_groups: setup_group_can_fail_task: true setup_group_timeout_secs: 1800 tasks: - - oidc-auth-test-k8s + - task: oidc-auth-test-k8s + vars: + K8S_VARIANT: eks + - task: oidc-auth-test-k8s + vars: + K8S_VARIANT: gke + # AKS testing is currently blocked until 8.0.1 is available in Atlas. + # - task: oidc-auth-test-k8s + # vars: + # K8S_VARIANT: aks - name: testoidc_task_group setup_group: @@ -1907,8 +1916,7 @@ tasks: working_dir: src env: OIDC_ENV: k8s - K8S_VARIANT: eks - include_expansions_in_env: ["DRIVERS_TOOLS", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] + include_expansions_in_env: ["DRIVERS_TOOLS", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN", "K8S_VARIANT"] args: - ${PROJECT_DIRECTORY}/.evergreen/run-mongodb-oidc-remote-test.sh From a3faa33307f54fa5b5b99825b89e07f4cdeae996 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 2 Oct 2024 11:18:33 -0500 Subject: [PATCH 26/33] test eks and gke --- .evergreen/config.yml | 41 ++++++++++++++++++++++------------------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index ccd2c86462..aa30acca6b 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -515,6 +515,18 @@ functions: args: - .evergreen/run-mongodb-oidc-test.sh + "run oidc k8s auth test": + - command: subprocess.exec + type: test + params: + binary: bash + working_dir: src + env: + OIDC_ENV: k8s + include_expansions_in_env: ["DRIVERS_TOOLS", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN", "K8S_VARIANT"] + args: + - ${PROJECT_DIRECTORY}/.evergreen/run-mongodb-oidc-remote-test.sh + "run aws auth test with aws credentials as environment variables": - command: shell.exec type: test @@ -939,15 +951,6 @@ task_groups: setup_group_timeout_secs: 1800 tasks: - task: oidc-auth-test-k8s - vars: - K8S_VARIANT: eks - - task: oidc-auth-test-k8s - vars: - K8S_VARIANT: gke - # AKS testing is currently blocked until 8.0.1 is available in Atlas. - # - task: oidc-auth-test-k8s - # vars: - # K8S_VARIANT: aks - name: testoidc_task_group setup_group: @@ -1909,16 +1912,16 @@ tasks: - name: "oidc-auth-test-k8s" commands: - - command: subprocess.exec - type: test - params: - binary: bash - working_dir: src - env: - OIDC_ENV: k8s - include_expansions_in_env: ["DRIVERS_TOOLS", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN", "K8S_VARIANT"] - args: - - ${PROJECT_DIRECTORY}/.evergreen/run-mongodb-oidc-remote-test.sh + - func: "run oidc k8s auth test" + vars: + K8S_VARIANT: eks + - func: "run oidc k8s auth test" + vars: + K8S_VARIANT: gke + # AKS testing is currently blocked until 8.0.1 is available in Atlas. + # - func: "run oidc k8s auth test" + # vars: + # K8S_VARIANT: aks - name: "test-fips-standalone" tags: ["fips"] From f6cc79b3623c0b729b4f727c9af15e8aef465f0b Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 2 Oct 2024 11:18:54 -0500 Subject: [PATCH 27/33] lint --- .evergreen/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index aa30acca6b..18fe3792c6 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -950,7 +950,7 @@ task_groups: setup_group_can_fail_task: true setup_group_timeout_secs: 1800 tasks: - - task: oidc-auth-test-k8s + - oidc-auth-test-k8s - name: testoidc_task_group setup_group: From b22c34c1559b8bf0d84a88771131ac2ccf308e69 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 25 Oct 2024 06:50:29 -0500 Subject: [PATCH 28/33] fix tasks --- .evergreen/config.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 2f2afca129..5da718ecef 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -1592,6 +1592,19 @@ tasks: commands: - func: "run oidc auth test with test credentials" + - name: "oidc-auth-test-azure" + commands: + - command: subprocess.exec + type: test + params: + binary: bash + working_dir: src + env: + OIDC_ENV: azure + include_expansions_in_env: ["DRIVERS_TOOLS"] + args: + - ${PROJECT_DIRECTORY}/.evergreen/run-mongodb-oidc-remote-test.sh + - name: "oidc-auth-test-azure" commands: - command: subprocess.exec From 24f9fa74530ecbd4c1ef449f55ae2f6edfadcc3f Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 25 Oct 2024 06:50:49 -0500 Subject: [PATCH 29/33] fix tasks --- .evergreen/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 5da718ecef..dfd5716b7d 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -1605,7 +1605,7 @@ tasks: args: - ${PROJECT_DIRECTORY}/.evergreen/run-mongodb-oidc-remote-test.sh - - name: "oidc-auth-test-azure" + - name: "oidc-auth-test-gcp" commands: - command: subprocess.exec type: test From 58c1d2f92fb67b52756d078d8b251e56962c7204 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 25 Oct 2024 07:37:09 -0500 Subject: [PATCH 30/33] fixup tests --- test/auth_oidc/test_auth_oidc.py | 41 +++----------------------------- 1 file changed, 3 insertions(+), 38 deletions(-) diff --git a/test/auth_oidc/test_auth_oidc.py b/test/auth_oidc/test_auth_oidc.py index 7716e7300e..03350f1b5b 100644 --- a/test/auth_oidc/test_auth_oidc.py +++ b/test/auth_oidc/test_auth_oidc.py @@ -763,7 +763,9 @@ def create_client(self, *args, **kwargs): kwargs["retryReads"] = False if not len(args): args = [self.uri_single] - return MongoClient(*args, authmechanismproperties=props, **kwargs) + client = MongoClient(*args, authmechanismproperties=props, **kwargs) + self.addCleanup(client.close) + return client def test_1_1_callback_is_called_during_reauthentication(self): # Create a ``MongoClient`` configured with a custom OIDC callback that @@ -773,8 +775,6 @@ def test_1_1_callback_is_called_during_reauthentication(self): client.test.test.find_one() # Assert that the callback was called 1 time. self.assertEqual(self.request_called, 1) - # Close the client. - client.close() def test_1_2_callback_is_called_once_for_multiple_connections(self): # Create a ``MongoClient`` configured with a custom OIDC callback that @@ -795,8 +795,6 @@ def target(): thread.join() # Assert that the callback was called 1 time. self.assertEqual(self.request_called, 1) - # Close the client. - client.close() def test_2_1_valid_callback_inputs(self): # Create a MongoClient configured with an OIDC callback that validates its inputs and returns a valid access token. @@ -805,8 +803,6 @@ def test_2_1_valid_callback_inputs(self): client.test.test.find_one() # Assert that the OIDC callback was called with the appropriate inputs, including the timeout parameter if possible. Ensure that there are no unexpected fields. self.assertEqual(self.request_called, 1) - # Close the client. - client.close() def test_2_2_oidc_callback_returns_null(self): # Create a MongoClient configured with an OIDC callback that returns null. @@ -818,8 +814,6 @@ def fetch(self, a): # Perform a find operation that fails. with self.assertRaises(ValueError): client.test.test.find_one() - # Close the client. - client.close() def test_2_3_oidc_callback_returns_missing_data(self): # Create a MongoClient configured with an OIDC callback that returns data not conforming to the OIDCCredential with missing fields. @@ -834,8 +828,6 @@ def fetch(self, a): # Perform a find operation that fails. with self.assertRaises(ValueError): client.test.test.find_one() - # Close the client. - client.close() def test_2_4_invalid_client_configuration_with_callback(self): # Create a MongoClient configured with an OIDC callback and auth mechanism property ENVIRONMENT:test. @@ -875,8 +867,6 @@ def test_3_1_authentication_failure_with_cached_tokens_fetch_a_new_token_and_ret client.test.test.find_one() # Verify that the callback was called 1 time. self.assertEqual(self.request_called, 1) - # Close the client. - client.close() def test_3_2_authentication_failures_without_cached_tokens_returns_an_error(self): # Create a MongoClient configured with retryReads=false and an OIDC callback that always returns invalid access tokens. @@ -894,8 +884,6 @@ def fetch(self, a): client.test.test.find_one() # Verify that the callback was called 1 time. self.assertEqual(callback.count, 1) - # Close the client. - client.close() def test_3_3_unexpected_error_code_does_not_clear_cache(self): # Create a ``MongoClient`` with a human callback that returns a valid token @@ -921,9 +909,6 @@ def test_3_3_unexpected_error_code_does_not_clear_cache(self): # Assert that the callback has been called once. self.assertEqual(self.request_called, 1) - # Close the client. - client.close() - def test_4_1_reauthentication_succeds(self): # Create a ``MongoClient`` configured with a custom OIDC callback that # implements the provider logic. @@ -943,9 +928,6 @@ def test_4_1_reauthentication_succeds(self): # handshake, and again during reauthentication). self.assertEqual(self.request_called, 2) - # Close the client. - client.close() - def test_4_2_read_commands_fail_if_reauthentication_fails(self): # Create a ``MongoClient`` whose OIDC callback returns one good token and then # bad tokens after the first call. @@ -982,9 +964,6 @@ def fetch(self, _): # Verify that the callback was called 2 times. self.assertEqual(callback.count, 2) - # Close the client. - client.close() - def test_4_3_write_commands_fail_if_reauthentication_fails(self): # Create a ``MongoClient`` whose OIDC callback returns one good token and then # bad token after the first call. @@ -1021,9 +1000,6 @@ def fetch(self, _): # Verify that the callback was called 2 times. self.assertEqual(callback.count, 2) - # Close the client. - client.close() - def test_4_4_speculative_authentication_should_be_ignored_on_reauthentication(self): # Create an OIDC configured client that can listen for `SaslStart` commands. listener = OvertCommandListener() @@ -1066,9 +1042,6 @@ def test_4_4_speculative_authentication_should_be_ignored_on_reauthentication(se # Assert there were `SaslStart` commands executed. assert any(event.command_name.lower() == "saslstart" for event in listener.started_events) - # Close the client. - client.close() - def test_5_1_azure_with_no_username(self): if ENVIRON != "azure": raise unittest.SkipTest("Test is only supported on Azure") @@ -1078,7 +1051,6 @@ def test_5_1_azure_with_no_username(self): props = dict(TOKEN_RESOURCE=resource, ENVIRONMENT="azure") client = self.create_client(authMechanismProperties=props) client.test.test.find_one() - client.close() def test_5_2_azure_with_bad_username(self): if ENVIRON != "azure": @@ -1091,7 +1063,6 @@ def test_5_2_azure_with_bad_username(self): client = self.create_client(username="bad", authmechanismproperties=props) with self.assertRaises(ValueError): client.test.test.find_one() - client.close() def test_speculative_auth_success(self): client1 = self.create_client() @@ -1113,10 +1084,6 @@ def test_speculative_auth_success(self): # Perform a find operation. client2.test.test.find_one() - # Close the clients. - client2.close() - client1.close() - def test_reauthentication_succeeds_multiple_connections(self): client1 = self.create_client() client2 = self.create_client() @@ -1156,8 +1123,6 @@ def test_reauthentication_succeeds_multiple_connections(self): client2.test.test.find_one() self.assertEqual(self.request_called, 3) - client1.close() - client2.close() if __name__ == "__main__": From a333fa233f85f24626f92bc0fe1c8332a3065875 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 25 Oct 2024 08:30:28 -0500 Subject: [PATCH 31/33] fix test --- test/auth_oidc/test_auth_oidc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/auth_oidc/test_auth_oidc.py b/test/auth_oidc/test_auth_oidc.py index 03350f1b5b..a0127304c1 100644 --- a/test/auth_oidc/test_auth_oidc.py +++ b/test/auth_oidc/test_auth_oidc.py @@ -1002,7 +1002,7 @@ def fetch(self, _): def test_4_4_speculative_authentication_should_be_ignored_on_reauthentication(self): # Create an OIDC configured client that can listen for `SaslStart` commands. - listener = OvertCommandListener() + listener = EventListener() client = self.create_client(event_listeners=[listener]) # Preload the *Client Cache* with a valid access token to enforce Speculative Authentication. From a8148d7cd8e33eb03ed1f2faebd8a0b81de48d08 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 25 Oct 2024 08:51:13 -0500 Subject: [PATCH 32/33] increase timeout --- .evergreen/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index dfd5716b7d..53c103c39e 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -894,6 +894,7 @@ task_groups: - command: ec2.assume_role params: role_arn: ${aws_test_secrets_role} + duration_seconds: 1800 - command: subprocess.exec params: binary: bash From 46845819ea9c83d112ddb91b96195ad670aa0045 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 25 Oct 2024 09:05:47 -0500 Subject: [PATCH 33/33] restructure variants --- .evergreen/config.yml | 21 --------------------- .evergreen/generated_configs/variants.yml | 9 ++++++--- .evergreen/scripts/generate_config.py | 8 ++++++-- 3 files changed, 12 insertions(+), 26 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 53c103c39e..d2bfd4c920 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -1788,27 +1788,6 @@ buildvariants: tasks: - name: "coverage-report" -- name: testazureoidc-variant - display_name: "OIDC Auth Azure" - run_on: ubuntu2204-small - tasks: - - name: testazureoidc_task_group - batchtime: 20160 # Use a batchtime of 14 days as suggested by the CSFLE test README - -- name: testgcpoidc-variant - display_name: "OIDC Auth GCP" - run_on: ubuntu2204-small - tasks: - - name: testgcpoidc_task_group - batchtime: 20160 # Use a batchtime of 14 days as suggested by the CSFLE test README - -- name: testk8soidc-variant - display_name: "OIDC Auth K8S" - run_on: ubuntu2204-small - tasks: - - name: testk8soidc_task_group - batchtime: 20160 # Use a batchtime of 14 days as suggested by the CSFLE test README - - name: testgcpkms-variant display_name: "GCP KMS" run_on: diff --git a/.evergreen/generated_configs/variants.yml b/.evergreen/generated_configs/variants.yml index 327becc249..944bfdae6d 100644 --- a/.evergreen/generated_configs/variants.yml +++ b/.evergreen/generated_configs/variants.yml @@ -1096,12 +1096,15 @@ buildvariants: VERSION: "8.0" # Oidc auth tests - - name: oidc-auth-rhel8 + - name: oidc-auth-ubuntu-22 tasks: - name: testoidc_task_group - display_name: OIDC Auth RHEL8 + - name: testazureoidc_task_group + - name: testgcpoidc_task_group + - name: testk8soidc_task_group + display_name: OIDC Auth Ubuntu-22 run_on: - - rhel87-small + - ubuntu2204-small batchtime: 20160 - name: oidc-auth-macos tasks: diff --git a/.evergreen/scripts/generate_config.py b/.evergreen/scripts/generate_config.py index b8b8fa367c..9797ef1937 100644 --- a/.evergreen/scripts/generate_config.py +++ b/.evergreen/scripts/generate_config.py @@ -619,10 +619,14 @@ def create_serverless_variants(): def create_oidc_auth_variants(): variants = [] - for host in ["rhel8", "macos", "win64"]: + other_tasks = ["testazureoidc_task_group", "testgcpoidc_task_group", "testk8soidc_task_group"] + for host in ["ubuntu22", "macos", "win64"]: + tasks = ["testoidc_task_group"] + if host == "ubuntu22": + tasks += other_tasks variants.append( create_variant( - ["testoidc_task_group"], + tasks, get_display_name("OIDC Auth", host), host=host, batchtime=BATCHTIME_WEEK * 2,