From f87a4125c2e9ee6a0418fcb84a1188574524348e Mon Sep 17 00:00:00 2001 From: Steven Silvester <steven.silvester@ieee.org> Date: Wed, 5 Mar 2025 07:51:41 -0600 Subject: [PATCH 01/19] PYTHON-5157 Convert aws tests to use python scripts (#622) --- .evergreen/auth_aws/aws_setup.sh | 101 +++--------------- .evergreen/auth_aws/aws_tester.py | 70 ++++++++++-- .../lib/aws_assign_instance_profile.py | 18 ++-- .../orchestration/drivers_orchestration.py | 15 ++- .evergreen/run-orchestration.sh | 1 + .gitignore | 1 + 6 files changed, 95 insertions(+), 111 deletions(-) diff --git a/.evergreen/auth_aws/aws_setup.sh b/.evergreen/auth_aws/aws_setup.sh index fd091bd1e..9fcde9265 100755 --- a/.evergreen/auth_aws/aws_setup.sh +++ b/.evergreen/auth_aws/aws_setup.sh @@ -6,108 +6,33 @@ # . ./aws_setup.sh <test-name> # # Handles AWS credential setup and exports relevant environment variables. -# Assumes you have already set up secrets. +# Sets up secrets if they have not already been set up. set -eu SCRIPT_DIR=$(dirname ${BASH_SOURCE[0]}) . $SCRIPT_DIR/../handle-paths.sh pushd $SCRIPT_DIR -# Ensure that secrets have already been set up. -if [ ! -f "secrets-export.sh" ]; then - echo "ERROR: please run './setup-secrets.sh' in this folder" -fi - # Activate the venv and source the secrets file. . ./activate-authawsvenv.sh -source secrets-export.sh -if [ "$1" == "web-identity" ]; then - export AWS_WEB_IDENTITY_TOKEN_FILE="./token_file.txt" -fi - -# Handle the test setup if not using env variables. -case $1 in - session-creds) - echo "Running aws_tester.py with assume-role" - # Set up credentials with assume-role to create user in MongoDB and write AWS credentials. - python aws_tester.py "assume-role" - ;; - env-creds) - echo "Running aws_tester.py with regular" - # Set up credentials with regular to create user in MongoDB and write AWS credentials. - python aws_tester.py "regular" - ;; - *) - python aws_tester.py "$1" - ;; -esac - -# If this is ecs, exit now. -if [ "$1" == "ecs" ]; then - exit 0 +# Ensure that secrets have already been set up. +if [ ! -f "./secrets-export.sh" ]; then + bash ./setup-secrets.sh fi -# Convenience functions. -urlencode () { - python -c "import sys, urllib.parse as ulp; sys.stdout.write(ulp.quote_plus(sys.argv[1]))" "$1" -} +# Remove any AWS creds that might be set in the parent env. +unset AWS_ACCESS_KEY_ID +unset AWS_SECRET_ACCESS_KEY +unset AWS_SESSION_TOKEN -jsonkey () { - python -c "import json,sys;sys.stdout.write(json.load(sys.stdin)[sys.argv[1]])" "$1" < ./creds.json -} +source ./secrets-export.sh -# Handle extra vars based on auth type. -USER="" -case $1 in - assume-role) - USER=$(jsonkey AccessKeyId) - USER=$(urlencode "$USER") - PASS=$(jsonkey SecretAccessKey) - PASS=$(urlencode "$PASS") - SESSION_TOKEN=$(jsonkey SessionToken) - SESSION_TOKEN=$(urlencode "$SESSION_TOKEN") - ;; - - session-creds) - AWS_ACCESS_KEY_ID=$(jsonkey AccessKeyId) - AWS_SECRET_ACCESS_KEY=$(jsonkey SecretAccessKey) - AWS_SESSION_TOKEN=$(jsonkey SessionToken) - - export AWS_ACCESS_KEY_ID - export AWS_SECRET_ACCESS_KEY - export AWS_SESSION_TOKEN - ;; - - web-identity) - export AWS_ROLE_ARN=$IAM_AUTH_ASSUME_WEB_ROLE_NAME - export AWS_WEB_IDENTITY_TOKEN_FILE="$SCRIPT_DIR/$AWS_WEB_IDENTITY_TOKEN_FILE" - ;; - - regular) - USER=$(urlencode "${IAM_AUTH_ECS_ACCOUNT}") - PASS=$(urlencode "${IAM_AUTH_ECS_SECRET_ACCESS_KEY}") - ;; - - env-creds) - export AWS_ACCESS_KEY_ID=$IAM_AUTH_ECS_ACCOUNT - export AWS_SECRET_ACCESS_KEY=$IAM_AUTH_ECS_SECRET_ACCESS_KEY - ;; -esac - -# Handle the URI. -if [ -n "$USER" ]; then - MONGODB_URI="mongodb://$USER:$PASS@localhost" - export USER - export PASS -else - MONGODB_URI="mongodb://localhost" -fi -MONGODB_URI="${MONGODB_URI}/aws?authMechanism=MONGODB-AWS" -if [[ -n ${SESSION_TOKEN:-} ]]; then - MONGODB_URI="${MONGODB_URI}&authMechanismProperties=AWS_SESSION_TOKEN:${SESSION_TOKEN}" +if [ -f $SCRIPT_DIR/test-env.sh ]; then + rm $SCRIPT_DIR/test-env.sh fi -export MONGODB_URI="$MONGODB_URI" +python aws_tester.py "$1" +source $SCRIPT_DIR/test-env.sh popd diff --git a/.evergreen/auth_aws/aws_tester.py b/.evergreen/auth_aws/aws_tester.py index 62e6b7dbe..2fa13fa58 100755 --- a/.evergreen/auth_aws/aws_tester.py +++ b/.evergreen/auth_aws/aws_tester.py @@ -5,23 +5,27 @@ import argparse import json +import logging import os import subprocess import sys from functools import partial +from pathlib import Path from urllib.parse import quote_plus from pymongo import MongoClient from pymongo.errors import OperationFailure -HERE = os.path.abspath(os.path.dirname(__file__)) +HERE = Path(__file__).absolute().parent +LOGGER = logging.getLogger(__name__) +logging.basicConfig(level=logging.INFO, format="%(levelname)-8s %(message)s") def join(*parts): return os.path.join(*parts).replace(os.sep, "/") -sys.path.insert(0, join(HERE, "lib")) +sys.path.insert(0, str(HERE / "lib")) from aws_assign_instance_profile import _assign_instance_policy from aws_assume_role import _assume_role from aws_assume_web_role import _assume_role_with_web_identity @@ -35,7 +39,7 @@ def join(*parts): _USE_AWS_SECRETS = False try: - with open(join(HERE, "aws_e2e_setup.json")) as fid: + with (HERE / "aws_e2e_setup.json").open() as fid: CONFIG = json.load(fid) get_key = partial(_get_key, uppercase=False) except FileNotFoundError: @@ -51,7 +55,7 @@ def run(args, env): def create_user(user, kwargs): """Create a user and verify access.""" - print("Creating user", user) + LOGGER.info("Creating user %s", user) client = MongoClient(username="bob", password="pwd123") db = client["$external"] try: @@ -76,7 +80,7 @@ def setup_assume_role(): role_name = CONFIG[get_key("iam_auth_assume_role_name")] creds = _assume_role(role_name, quiet=True) - with open(join(HERE, "creds.json"), "w") as fid: + with (HERE / "creds.json").open("w") as fid: json.dump(creds, fid) # Create the user. @@ -87,6 +91,11 @@ def setup_assume_role(): authmechanismproperties=f"AWS_SESSION_TOKEN:{token}", ) create_user(ASSUMED_ROLE, kwargs) + return dict( + USER=kwargs["username"], + PASS=kwargs["password"], + SESSION_TOKEN=creds["SessionToken"], + ) def setup_ec2(): @@ -95,6 +104,7 @@ def setup_ec2(): os.environ.pop("AWS_ACCESS_KEY_ID", None) os.environ.pop("AWS_SECRET_ACCESS_KEY", None) create_user(AWS_ACCOUNT_ARN, dict()) + return dict() def setup_ecs(): @@ -138,6 +148,8 @@ def setup_ecs(): # Run the test in a container subprocess.check_call(["/bin/sh", "-c", run_test_command], env=env) + return dict() + def setup_regular(): # Create the user. @@ -147,6 +159,8 @@ def setup_regular(): ) create_user(CONFIG[get_key("iam_auth_ecs_account_arn")], kwargs) + return dict(USER=kwargs["username"], PASS=kwargs["password"]) + def setup_web_identity(): # Unassign the instance profile. @@ -161,7 +175,7 @@ def setup_web_identity(): raise RuntimeError("Request limit exceeded for AWS API") if ret != 0: - print("ret was", ret) + LOGGER.debug("return code was %s", ret) raise RuntimeError( "Failed to unassign an instance profile from the current machine" ) @@ -186,10 +200,11 @@ def setup_web_identity(): # Assume the web role to get temp credentials. os.environ["AWS_WEB_IDENTITY_TOKEN_FILE"] = token_file - os.environ["AWS_ROLE_ARN"] = CONFIG[get_key("iam_auth_assume_web_role_name")] + role_arn = CONFIG[get_key("iam_auth_assume_web_role_name")] + os.environ["AWS_ROLE_ARN"] = role_arn creds = _assume_role_with_web_identity(True) - with open(join(HERE, "creds.json"), "w") as fid: + with (HERE / "creds.json").open("w") as fid: json.dump(creds, fid) # Create the user. @@ -201,12 +216,39 @@ def setup_web_identity(): ) create_user(ASSUMED_WEB_ROLE, kwargs) + return dict(AWS_WEB_IDENTITY_TOKEN_FILE=token_file, AWS_ROLE_ARN=role_arn) + + +def handle_creds(creds: dict): + if "USER" in creds: + USER = quote_plus(creds["USER"]) + PASS = quote_plus(creds["PASS"]) + MONGODB_URI = f"mongodb://{USER}:{PASS}@localhost" + else: + MONGODB_URI = "mongodb://localhost" + MONGODB_URI = f"{MONGODB_URI}/aws?authMechanism=MONGODB-AWS" + if "SESSION_TOKEN" in creds: + SESSION_TOKEN = quote_plus(creds["SESSION_TOKEN"]) + MONGODB_URI = ( + f"{MONGODB_URI}&authMechanismProperties=AWS_SESSION_TOKEN:{SESSION_TOKEN}" + ) + with (HERE / "test-env.sh").open("w", newline="\n") as fid: + fid.write("#!/usr/bin/env bash\n\n") + fid.write("set +x\n") + for key, value in creds.items(): + if key in ["USER", "PASS", "SESSION_TOKEN"]: + value = quote_plus(value) # noqa: PLW2901 + fid.write(f"export {key}={value}\n") + fid.write(f"export MONGODB_URI={MONGODB_URI}\n") + def main(): parser = argparse.ArgumentParser(description="MONGODB-AWS tester.") sub = parser.add_subparsers(title="Tester subcommands", help="sub-command help") - run_assume_role_cmd = sub.add_parser("assume-role", help="Assume role test") + run_assume_role_cmd = sub.add_parser( + "assume-role", aliases=["session-creds"], help="Assume role test" + ) run_assume_role_cmd.set_defaults(func=setup_assume_role) run_ec2_cmd = sub.add_parser("ec2", help="EC2 test") @@ -215,14 +257,20 @@ def main(): run_ecs_cmd = sub.add_parser("ecs", help="ECS test") run_ecs_cmd.set_defaults(func=setup_ecs) - run_regular_cmd = sub.add_parser("regular", help="Regular credentials test") + run_regular_cmd = sub.add_parser( + "regular", aliases=["env-creds"], help="Regular credentials test" + ) run_regular_cmd.set_defaults(func=setup_regular) run_web_identity_cmd = sub.add_parser("web-identity", help="Web identity test") run_web_identity_cmd.set_defaults(func=setup_web_identity) args = parser.parse_args() - args.func() + func_name = args.func.__name__.replace("setup_", "") + LOGGER.info("Running aws_tester.py with %s...", func_name) + creds = args.func() + handle_creds(creds) + LOGGER.info("Running aws_tester.py with %s... done.", func_name) if __name__ == "__main__": diff --git a/.evergreen/auth_aws/lib/aws_assign_instance_profile.py b/.evergreen/auth_aws/lib/aws_assign_instance_profile.py index be3028a6b..c0bb344f9 100755 --- a/.evergreen/auth_aws/lib/aws_assign_instance_profile.py +++ b/.evergreen/auth_aws/lib/aws_assign_instance_profile.py @@ -33,23 +33,23 @@ def _get_local_instance_id(): def _has_instance_profile(): base_url = "http://169.254.169.254/latest/meta-data/iam/security-credentials/" try: - print("Reading: " + base_url) + LOGGER.info("Reading: " + base_url) iam_role = urllib.request.urlopen(base_url).read().decode() except urllib.error.HTTPError as e: - print(e) if e.code == 404: return False + LOGGER.error(e) raise e try: url = base_url + iam_role - print("Reading: " + url) + LOGGER.info("Reading: " + url) _ = urllib.request.urlopen(url) - print("Assigned " + iam_role) + LOGGER.info("Assigned " + iam_role) except urllib.error.HTTPError as e: - print(e) if e.code == 404: return False + LOGGER.error(e) raise e return True @@ -85,7 +85,7 @@ def _handle_config(): ) return CONFIG[get_key("iam_auth_ec2_instance_profile")] except Exception as e: - print(e) + LOGGER.error(e) return "" @@ -94,7 +94,7 @@ def _handle_config(): def _assign_instance_policy(iam_instance_arn=DEFAULT_ARN): if _has_instance_profile(): - print( + LOGGER.warning( "IMPORTANT: Found machine already has instance profile, skipping the assignment" ) return @@ -112,14 +112,14 @@ def _assign_instance_policy(iam_instance_arn=DEFAULT_ARN): InstanceId=instance_id, ) - print(response) + LOGGER.debug(response) # Wait for the instance profile to be assigned by polling the local instance metadata service _wait_instance_profile() except botocore.exceptions.ClientError as ce: if ce.response["Error"]["Code"] == "RequestLimitExceeded": - print("WARNING: RequestLimitExceeded, exiting with error code 2") + LOGGER.warning("WARNING: RequestLimitExceeded, exiting with error code 2") sys.exit(2) raise diff --git a/.evergreen/orchestration/drivers_orchestration.py b/.evergreen/orchestration/drivers_orchestration.py index 7e89c14d3..0a698b318 100644 --- a/.evergreen/orchestration/drivers_orchestration.py +++ b/.evergreen/orchestration/drivers_orchestration.py @@ -55,14 +55,13 @@ def get_options(): parser.add_argument( "--version", default="latest", - help='The version to download (Required). Use "latest" to download ' + help='The version to download. Use "latest" to download ' "the newest available version (including release candidates).", ) parser.add_argument( "--topology", choices=["standalone", "replica_set", "sharded_cluster"], - default="standalone", - help="The topology of the server deployment", + help="The topology of the server deployment (defaults to standalone unless another flag like load_balancer is set)", ) parser.add_argument( "--auth", action="store_true", help="Whether to add authentication" @@ -78,6 +77,9 @@ def get_options(): other_group.add_argument( "--load-balancer", action="store_true", help="Whether to use a load balancer" ) + other_group.add_argument( + "--auth-aws", action="store_true", help="Whether to use MONGODB-AWS auth" + ) other_group.add_argument( "--skip-crypt-shared", action="store_true", @@ -148,6 +150,11 @@ def get_options(): opts.mongo_orchestration_home = DRIVERS_TOOLS / ".evergreen/orchestration" if opts.mongodb_binaries is None: opts.mongodb_binaries = DRIVERS_TOOLS / "mongodb/bin" + if not opts.topology and opts.load_balancer: + opts.topology = "sharded_cluster" + if opts.auth_aws: + opts.auth = True + opts.orchestration_file = "auth-aws.json" if opts.topology == "standalone" or not opts.topology: opts.topology = "server" if not opts.version: @@ -395,6 +402,8 @@ def run(opts): MO_EXPANSION_YML.write_text( MO_EXPANSION_YML.read_text() + f'\nMONGODB_URI: "{uri}"' ) + MO_EXPANSION_SH.touch() + MO_EXPANSION_SH.write_text(MO_EXPANSION_SH.read_text() + f'\nMONGODB_URI="{uri}"') URI_TXT.write_text(uri) LOGGER.info(f"Cluster URI: {uri}") diff --git a/.evergreen/run-orchestration.sh b/.evergreen/run-orchestration.sh index de925ddec..a1f872332 100755 --- a/.evergreen/run-orchestration.sh +++ b/.evergreen/run-orchestration.sh @@ -15,6 +15,7 @@ set -eu # SKIP_CRYPT_SHARED Set to a non-empty string to skip downloading crypt_shared # MONGODB_BINARIES Set the path to the MONGODB_BINARIES for mongo orchestration. # LOAD_BALANCER Set to a non-empty string to enable load balancer. Only supported for sharded clusters. +# AUTH_AWS Set to a non-empty string to enable MONGODB-AWS authentication. # PYTHON Set the Python binary to use. # INSTALL_LEGACY_SHELL Set to a non-empty string to install the legacy mongo shell. # TLS_CERT_KEY_FILE Set a .pem file to be used as the tlsCertificateKeyFile option in mongo-orchestration diff --git a/.gitignore b/.gitignore index 8479ac011..82a4cc0dd 100644 --- a/.gitignore +++ b/.gitignore @@ -88,6 +88,7 @@ evergreen_config_generator/dist *-expansion.yml secrets-export.sh token_file.txt +test-env.sh # Virtual envs .venv From ce744efdedcd4defc602b2f0227e133b11d41b88 Mon Sep 17 00:00:00 2001 From: Steven Silvester <steven.silvester@ieee.org> Date: Wed, 5 Mar 2025 07:51:41 -0600 Subject: [PATCH 02/19] PYTHON-5157 Convert aws tests to use python scripts --- .evergreen/auth_aws/aws_tester.py | 32 +++++++++++++++----- .evergreen/config.yml | 20 +++++++++++++ .evergreen/tests/test-aws-setup.sh | 48 ++++++++++++++++++++++++++++++ 3 files changed, 93 insertions(+), 7 deletions(-) create mode 100755 .evergreen/tests/test-aws-setup.sh diff --git a/.evergreen/auth_aws/aws_tester.py b/.evergreen/auth_aws/aws_tester.py index 2fa13fa58..a36c20b17 100755 --- a/.evergreen/auth_aws/aws_tester.py +++ b/.evergreen/auth_aws/aws_tester.py @@ -151,6 +151,16 @@ def setup_ecs(): return dict() +def setup_session_creds(): + # Set up the assume role user, and export the aws vars. + creds = setup_assume_role() + return dict( + AWS_ACCESS_KEY_ID=creds["USER"], + AWS_ACCOUNT_ARN=creds["PASS"], + AWS_SESSION_TOKEN=creds["SESSION_TOKEN"], + ) + + def setup_regular(): # Create the user. kwargs = dict( @@ -162,6 +172,12 @@ def setup_regular(): return dict(USER=kwargs["username"], PASS=kwargs["password"]) +def setup_env_creds(): + # Set up the regular user, but export the creds as environment vars. + creds = setup_regular() + return dict(AWS_ACCESS_KEY_ID=creds["USER"], AWS_ACCOUNT_ARN=creds["PASS"]) + + def setup_web_identity(): # Unassign the instance profile. env = dict( @@ -246,9 +262,7 @@ def main(): parser = argparse.ArgumentParser(description="MONGODB-AWS tester.") sub = parser.add_subparsers(title="Tester subcommands", help="sub-command help") - run_assume_role_cmd = sub.add_parser( - "assume-role", aliases=["session-creds"], help="Assume role test" - ) + run_assume_role_cmd = sub.add_parser("assume-role", help="Assume role test") run_assume_role_cmd.set_defaults(func=setup_assume_role) run_ec2_cmd = sub.add_parser("ec2", help="EC2 test") @@ -257,16 +271,20 @@ def main(): run_ecs_cmd = sub.add_parser("ecs", help="ECS test") run_ecs_cmd.set_defaults(func=setup_ecs) - run_regular_cmd = sub.add_parser( - "regular", aliases=["env-creds"], help="Regular credentials test" - ) + run_regular_cmd = sub.add_parser("regular", help="Regular credentials test") run_regular_cmd.set_defaults(func=setup_regular) + run_session_creds_cmd = sub.add_parser("session-creds", help="Session credentials") + run_session_creds_cmd.set_defaults(func=setup_session_creds) + + run_env_creds_cmd = sub.add_parser("env-creds", help="Environment credentials") + run_env_creds_cmd.set_defaults(func=setup_env_creds) + run_web_identity_cmd = sub.add_parser("web-identity", help="Web identity test") run_web_identity_cmd.set_defaults(func=setup_web_identity) args = parser.parse_args() - func_name = args.func.__name__.replace("setup_", "") + func_name = args.func.__name__.replace("setup_", "").replace("_", "-") LOGGER.info("Running aws_tester.py with %s...", func_name) creds = args.func() handle_creds(creds) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 4edfa53c7..acf87d5c1 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -512,6 +512,14 @@ functions: cd ${PROJECT_DIRECTORY} make test + "run aws test": + - command: subprocess.exec + type: test + params: + binary: bash + include_expansions_in_env: [AWS_SECRET_ACCESS_KEY, AWS_ACCESS_KEY_ID, AWS_SESSION_TOKEN] + args: [src/.evergreen/tests/test-aws.sh] + "run csfle test": - command: ec2.assume_role params: @@ -1005,6 +1013,11 @@ tasks: commands: - func: "run csfle test" + - name: "test-aws" + tags: ["pr", "aws"] + commands: + - func: "run aws test" + - name: "test-cli-full" tags: ["pr"] commands: @@ -1399,6 +1412,13 @@ buildvariants: tasks: - ".docker" # Run all tasks with the "docker" tag +- name: tests-aws + display_name: AWS + run_on: + - ubuntu2204-small + tasks: + - ".aws" + - name: tests-oidc display_name: OIDC run_on: ubuntu2204-small diff --git a/.evergreen/tests/test-aws-setup.sh b/.evergreen/tests/test-aws-setup.sh new file mode 100755 index 000000000..c6d8fe675 --- /dev/null +++ b/.evergreen/tests/test-aws-setup.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash + +# Test aws setup function for different inputs. +set -eu + +# Start a server with aws auth enabled. +AUTH="auth" ORCHESTRATION_FILE="auth-aws.json" make run-server + +SCRIPT_DIR=$(dirname ${BASH_SOURCE[0]}) +. $SCRIPT_DIR/../handle-paths.sh + +pushd $SCRIPT_DIR/../auth_aws + +bash aws_setup.sh assume-role +cat test-env.sh | grep -q USER +cat test-env.sh | grep -q PASS +cat test-env.sh | grep -q SESSION_TOKEN +cat test-env.sh | grep -q MONGODB_URI + +bash aws_setup.sh ec2 +cat test-env.sh | grep -q MONGODB_URI + +bash aws_setup.sh regular +cat test-env.sh | grep -q USER +cat test-env.sh | grep -q PASS +cat test-env.sh | grep -q MONGODB_URI + +bash aws_setup.sh session-creds +cat test-env.sh | grep -q AWS_ACCESS_KEY_ID +cat test-env.sh | grep -q AWS_ACCOUNT_ARN +cat test-env.sh | grep -q AWS_SESSION_TOKEN +cat test-env.sh | grep -q MONGODB_URI + +bash aws_setup.sh env-creds +cat test-env.sh | grep -q AWS_ACCESS_KEY_ID +cat test-env.sh | grep -q AWS_ACCOUNT_ARN +cat test-env.sh | grep -q MONGODB_URI + +bash aws_setup.sh web-identity +cat test-env.sh | grep -q AWS_WEB_IDENTITY_TOKEN_FILE +cat test-env.sh | AWS_ROLE_ARN + +bash ./teardown.sh + +make stop-server + +popd +make -C ${DRIVERS_TOOLS} test From 6d5126f5d6af995033612efc6717f1af5bed3f4b Mon Sep 17 00:00:00 2001 From: Steven Silvester <steve.silvester@mongodb.com> Date: Wed, 5 Mar 2025 13:37:17 -0600 Subject: [PATCH 03/19] fix test name --- .evergreen/tests/{test-aws-setup.sh => test-aws.sh} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .evergreen/tests/{test-aws-setup.sh => test-aws.sh} (100%) diff --git a/.evergreen/tests/test-aws-setup.sh b/.evergreen/tests/test-aws.sh similarity index 100% rename from .evergreen/tests/test-aws-setup.sh rename to .evergreen/tests/test-aws.sh From 8128796cb5e7f0a81868dde67eb59c84d3ebefc2 Mon Sep 17 00:00:00 2001 From: Steven Silvester <steve.silvester@mongodb.com> Date: Wed, 5 Mar 2025 13:40:53 -0600 Subject: [PATCH 04/19] fix tests --- .evergreen/tests/test-aws.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.evergreen/tests/test-aws.sh b/.evergreen/tests/test-aws.sh index c6d8fe675..f78cb28cd 100755 --- a/.evergreen/tests/test-aws.sh +++ b/.evergreen/tests/test-aws.sh @@ -3,12 +3,12 @@ # Test aws setup function for different inputs. set -eu -# Start a server with aws auth enabled. -AUTH="auth" ORCHESTRATION_FILE="auth-aws.json" make run-server - SCRIPT_DIR=$(dirname ${BASH_SOURCE[0]}) . $SCRIPT_DIR/../handle-paths.sh +# Start a server with aws auth enabled. +AUTH="auth" ORCHESTRATION_FILE="auth-aws.json" make -C ${DRIVERS_TOOLS} run-server + pushd $SCRIPT_DIR/../auth_aws bash aws_setup.sh assume-role @@ -42,7 +42,7 @@ cat test-env.sh | AWS_ROLE_ARN bash ./teardown.sh -make stop-server - popd + +make -C ${DRIVERS_TOOLS} stop-server make -C ${DRIVERS_TOOLS} test From 3f39b1fffece1aa4a2acc46ede8b85cbd121ade2 Mon Sep 17 00:00:00 2001 From: Steven Silvester <steve.silvester@mongodb.com> Date: Wed, 5 Mar 2025 13:45:43 -0600 Subject: [PATCH 05/19] fix test --- .evergreen/config.yml | 4 ++-- .evergreen/tests/test-aws.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index acf87d5c1..b6978c1e4 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -1413,14 +1413,14 @@ buildvariants: - ".docker" # Run all tasks with the "docker" tag - name: tests-aws - display_name: AWS + display_name: Auth AWS run_on: - ubuntu2204-small tasks: - ".aws" - name: tests-oidc - display_name: OIDC + display_name: Auth OIDC run_on: ubuntu2204-small tasks: - "test-oidc-local" diff --git a/.evergreen/tests/test-aws.sh b/.evergreen/tests/test-aws.sh index f78cb28cd..1fef3b5c2 100755 --- a/.evergreen/tests/test-aws.sh +++ b/.evergreen/tests/test-aws.sh @@ -38,7 +38,7 @@ cat test-env.sh | grep -q MONGODB_URI bash aws_setup.sh web-identity cat test-env.sh | grep -q AWS_WEB_IDENTITY_TOKEN_FILE -cat test-env.sh | AWS_ROLE_ARN +cat test-env.sh | grep -q AWS_ROLE_ARN bash ./teardown.sh From a3d62db15498c9bdb191caf1e7ce0fbf1aa17eae Mon Sep 17 00:00:00 2001 From: Steven Silvester <steve.silvester@mongodb.com> Date: Wed, 5 Mar 2025 13:47:57 -0600 Subject: [PATCH 06/19] update task name --- .evergreen/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index b6978c1e4..fbf0d45ac 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -1013,7 +1013,7 @@ tasks: commands: - func: "run csfle test" - - name: "test-aws" + - name: "test-auth-aws" tags: ["pr", "aws"] commands: - func: "run aws test" From c93edc420cc7304fbb565321943a3680ab30d6d5 Mon Sep 17 00:00:00 2001 From: Steven Silvester <steve.silvester@mongodb.com> Date: Wed, 5 Mar 2025 14:18:51 -0600 Subject: [PATCH 07/19] Fix logic and update test --- .evergreen/auth_aws/aws_tester.py | 4 ++-- .evergreen/tests/test-aws.sh | 10 ++++++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/.evergreen/auth_aws/aws_tester.py b/.evergreen/auth_aws/aws_tester.py index a36c20b17..d7505172d 100755 --- a/.evergreen/auth_aws/aws_tester.py +++ b/.evergreen/auth_aws/aws_tester.py @@ -156,7 +156,7 @@ def setup_session_creds(): creds = setup_assume_role() return dict( AWS_ACCESS_KEY_ID=creds["USER"], - AWS_ACCOUNT_ARN=creds["PASS"], + AWS_SECRET_ACCESS_KEY=creds["PASS"], AWS_SESSION_TOKEN=creds["SESSION_TOKEN"], ) @@ -175,7 +175,7 @@ def setup_regular(): def setup_env_creds(): # Set up the regular user, but export the creds as environment vars. creds = setup_regular() - return dict(AWS_ACCESS_KEY_ID=creds["USER"], AWS_ACCOUNT_ARN=creds["PASS"]) + return dict(AWS_ACCESS_KEY_ID=creds["USER"], AWS_SECRET_ACCESS_KEY=creds["PASS"]) def setup_web_identity(): diff --git a/.evergreen/tests/test-aws.sh b/.evergreen/tests/test-aws.sh index 1fef3b5c2..db299e72a 100755 --- a/.evergreen/tests/test-aws.sh +++ b/.evergreen/tests/test-aws.sh @@ -16,29 +16,35 @@ cat test-env.sh | grep -q USER cat test-env.sh | grep -q PASS cat test-env.sh | grep -q SESSION_TOKEN cat test-env.sh | grep -q MONGODB_URI +rm test-env.sh bash aws_setup.sh ec2 cat test-env.sh | grep -q MONGODB_URI +rm test-env.sh bash aws_setup.sh regular cat test-env.sh | grep -q USER cat test-env.sh | grep -q PASS cat test-env.sh | grep -q MONGODB_URI +rm test-env.sh bash aws_setup.sh session-creds cat test-env.sh | grep -q AWS_ACCESS_KEY_ID -cat test-env.sh | grep -q AWS_ACCOUNT_ARN +cat test-env.sh | grep -q AWS_SECRET_ACCESS_KEY cat test-env.sh | grep -q AWS_SESSION_TOKEN cat test-env.sh | grep -q MONGODB_URI +rm test-env.sh bash aws_setup.sh env-creds cat test-env.sh | grep -q AWS_ACCESS_KEY_ID -cat test-env.sh | grep -q AWS_ACCOUNT_ARN +cat test-env.sh | grep -q AWS_SECRET_ACCESS_KEY cat test-env.sh | grep -q MONGODB_URI +rm test-env.sh bash aws_setup.sh web-identity cat test-env.sh | grep -q AWS_WEB_IDENTITY_TOKEN_FILE cat test-env.sh | grep -q AWS_ROLE_ARN +rm test-env.sh bash ./teardown.sh From 961c46f00b9fbe32fedb6186fb73936fdebabd77 Mon Sep 17 00:00:00 2001 From: Steven Silvester <steve.silvester@mongodb.com> Date: Wed, 5 Mar 2025 14:24:04 -0600 Subject: [PATCH 08/19] update test --- .evergreen/tests/test-aws.sh | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/.evergreen/tests/test-aws.sh b/.evergreen/tests/test-aws.sh index db299e72a..20ce0dbcf 100755 --- a/.evergreen/tests/test-aws.sh +++ b/.evergreen/tests/test-aws.sh @@ -15,35 +15,38 @@ bash aws_setup.sh assume-role cat test-env.sh | grep -q USER cat test-env.sh | grep -q PASS cat test-env.sh | grep -q SESSION_TOKEN -cat test-env.sh | grep -q MONGODB_URI +cat test-env.sh | grep MONGODB_URI | grep -q "@" rm test-env.sh bash aws_setup.sh ec2 -cat test-env.sh | grep -q MONGODB_URI +cat test-env.sh | grep -q MONGODB_URI | grep -c -q "@" rm test-env.sh bash aws_setup.sh regular cat test-env.sh | grep -q USER cat test-env.sh | grep -q PASS -cat test-env.sh | grep -q MONGODB_URI +cat test-env.sh | grep -q -c SESSION_TOKEN +cat test-env.sh | grep MONGODB_URI | grep -q "@" rm test-env.sh bash aws_setup.sh session-creds cat test-env.sh | grep -q AWS_ACCESS_KEY_ID cat test-env.sh | grep -q AWS_SECRET_ACCESS_KEY cat test-env.sh | grep -q AWS_SESSION_TOKEN -cat test-env.sh | grep -q MONGODB_URI +cat test-env.sh | grep -q MONGODB_URI | grep -c -q "@" rm test-env.sh bash aws_setup.sh env-creds cat test-env.sh | grep -q AWS_ACCESS_KEY_ID cat test-env.sh | grep -q AWS_SECRET_ACCESS_KEY -cat test-env.sh | grep -q MONGODB_URI +cat test-env.sh | grep -c -q AWS_SESSION_TOKEN +cat test-env.sh | grep -q MONGODB_URI | grep -c -q "@" rm test-env.sh bash aws_setup.sh web-identity cat test-env.sh | grep -q AWS_WEB_IDENTITY_TOKEN_FILE cat test-env.sh | grep -q AWS_ROLE_ARN +cat test-env.sh | grep -q MONGODB_URI | grep -c -q "@" rm test-env.sh bash ./teardown.sh From 2d3d7ffe7e9ee2ce2e8296732b7e16e0826242b1 Mon Sep 17 00:00:00 2001 From: Steven Silvester <steve.silvester@mongodb.com> Date: Wed, 5 Mar 2025 14:30:15 -0600 Subject: [PATCH 09/19] fix test --- .evergreen/tests/test-aws.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.evergreen/tests/test-aws.sh b/.evergreen/tests/test-aws.sh index 20ce0dbcf..1b56ab697 100755 --- a/.evergreen/tests/test-aws.sh +++ b/.evergreen/tests/test-aws.sh @@ -15,11 +15,13 @@ bash aws_setup.sh assume-role cat test-env.sh | grep -q USER cat test-env.sh | grep -q PASS cat test-env.sh | grep -q SESSION_TOKEN +# Ensure there is a password in the URI. cat test-env.sh | grep MONGODB_URI | grep -q "@" rm test-env.sh bash aws_setup.sh ec2 -cat test-env.sh | grep -q MONGODB_URI | grep -c -q "@" +# Ensure there is no password in the URI. +cat test-env.sh | grep -q MONGODB_URI | grep -v -q "@" rm test-env.sh bash aws_setup.sh regular @@ -33,20 +35,20 @@ bash aws_setup.sh session-creds cat test-env.sh | grep -q AWS_ACCESS_KEY_ID cat test-env.sh | grep -q AWS_SECRET_ACCESS_KEY cat test-env.sh | grep -q AWS_SESSION_TOKEN -cat test-env.sh | grep -q MONGODB_URI | grep -c -q "@" +cat test-env.sh | grep -q MONGODB_URI | grep -v -q "@" rm test-env.sh bash aws_setup.sh env-creds cat test-env.sh | grep -q AWS_ACCESS_KEY_ID cat test-env.sh | grep -q AWS_SECRET_ACCESS_KEY cat test-env.sh | grep -c -q AWS_SESSION_TOKEN -cat test-env.sh | grep -q MONGODB_URI | grep -c -q "@" +cat test-env.sh | grep -q MONGODB_URI | grep -v -q "@" rm test-env.sh bash aws_setup.sh web-identity cat test-env.sh | grep -q AWS_WEB_IDENTITY_TOKEN_FILE cat test-env.sh | grep -q AWS_ROLE_ARN -cat test-env.sh | grep -q MONGODB_URI | grep -c -q "@" +cat test-env.sh | grep -q MONGODB_URI | grep -v -q "@" rm test-env.sh bash ./teardown.sh From 925236de42881ce89c1cd0562981d1f14f1b6cd2 Mon Sep 17 00:00:00 2001 From: Steven Silvester <steve.silvester@mongodb.com> Date: Wed, 5 Mar 2025 14:35:41 -0600 Subject: [PATCH 10/19] debug --- .evergreen/tests/test-aws.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.evergreen/tests/test-aws.sh b/.evergreen/tests/test-aws.sh index 1b56ab697..cda8f25be 100755 --- a/.evergreen/tests/test-aws.sh +++ b/.evergreen/tests/test-aws.sh @@ -21,6 +21,7 @@ rm test-env.sh bash aws_setup.sh ec2 # Ensure there is no password in the URI. +cat test-env.sh cat test-env.sh | grep -q MONGODB_URI | grep -v -q "@" rm test-env.sh From 5ac2d54611fa624e5e22c33a98df104b19058d39 Mon Sep 17 00:00:00 2001 From: Steven Silvester <steve.silvester@mongodb.com> Date: Wed, 5 Mar 2025 14:42:09 -0600 Subject: [PATCH 11/19] fix test --- .evergreen/tests/test-aws.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.evergreen/tests/test-aws.sh b/.evergreen/tests/test-aws.sh index cda8f25be..f3f25adf8 100755 --- a/.evergreen/tests/test-aws.sh +++ b/.evergreen/tests/test-aws.sh @@ -22,7 +22,7 @@ rm test-env.sh bash aws_setup.sh ec2 # Ensure there is no password in the URI. cat test-env.sh -cat test-env.sh | grep -q MONGODB_URI | grep -v -q "@" +cat test-env.sh | grep MONGODB_URI | grep -v -q "@" rm test-env.sh bash aws_setup.sh regular @@ -36,20 +36,20 @@ bash aws_setup.sh session-creds cat test-env.sh | grep -q AWS_ACCESS_KEY_ID cat test-env.sh | grep -q AWS_SECRET_ACCESS_KEY cat test-env.sh | grep -q AWS_SESSION_TOKEN -cat test-env.sh | grep -q MONGODB_URI | grep -v -q "@" +cat test-env.sh | grep MONGODB_URI | grep -v -q "@" rm test-env.sh bash aws_setup.sh env-creds cat test-env.sh | grep -q AWS_ACCESS_KEY_ID cat test-env.sh | grep -q AWS_SECRET_ACCESS_KEY -cat test-env.sh | grep -c -q AWS_SESSION_TOKEN -cat test-env.sh | grep -q MONGODB_URI | grep -v -q "@" +cat test-env.sh | grep -v -q AWS_SESSION_TOKEN +cat test-env.sh | grep MONGODB_URI | grep -v -q "@" rm test-env.sh bash aws_setup.sh web-identity cat test-env.sh | grep -q AWS_WEB_IDENTITY_TOKEN_FILE cat test-env.sh | grep -q AWS_ROLE_ARN -cat test-env.sh | grep -q MONGODB_URI | grep -v -q "@" +cat test-env.sh | grep MONGODB_URI | grep -v -q "@" rm test-env.sh bash ./teardown.sh From 07c7ae33034ef132e137395ca2472d6086285563 Mon Sep 17 00:00:00 2001 From: Steven Silvester <steve.silvester@mongodb.com> Date: Wed, 5 Mar 2025 14:42:20 -0600 Subject: [PATCH 12/19] fix test --- .evergreen/tests/test-aws.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/.evergreen/tests/test-aws.sh b/.evergreen/tests/test-aws.sh index f3f25adf8..8eb25a326 100755 --- a/.evergreen/tests/test-aws.sh +++ b/.evergreen/tests/test-aws.sh @@ -21,7 +21,6 @@ rm test-env.sh bash aws_setup.sh ec2 # Ensure there is no password in the URI. -cat test-env.sh cat test-env.sh | grep MONGODB_URI | grep -v -q "@" rm test-env.sh From d8cd31fe05d084b53cea6e1fb3ab352279f7b228 Mon Sep 17 00:00:00 2001 From: Steven Silvester <steve.silvester@mongodb.com> Date: Wed, 5 Mar 2025 14:47:10 -0600 Subject: [PATCH 13/19] fix test --- .evergreen/tests/test-aws.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen/tests/test-aws.sh b/.evergreen/tests/test-aws.sh index 8eb25a326..e66d5fee7 100755 --- a/.evergreen/tests/test-aws.sh +++ b/.evergreen/tests/test-aws.sh @@ -27,7 +27,7 @@ rm test-env.sh bash aws_setup.sh regular cat test-env.sh | grep -q USER cat test-env.sh | grep -q PASS -cat test-env.sh | grep -q -c SESSION_TOKEN +cat test-env.sh | grep -v -q SESSION_TOKEN cat test-env.sh | grep MONGODB_URI | grep -q "@" rm test-env.sh From a77825c9f5a22d0d8e924912804f4fd435fc8bd6 Mon Sep 17 00:00:00 2001 From: Steven Silvester <steven.silvester@ieee.org> Date: Wed, 5 Mar 2025 17:09:07 -0600 Subject: [PATCH 14/19] Update .evergreen/auth_aws/aws_setup.sh Co-authored-by: Ezra Chung <88335979+eramongodb@users.noreply.github.com> --- .evergreen/auth_aws/aws_setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen/auth_aws/aws_setup.sh b/.evergreen/auth_aws/aws_setup.sh index 9fcde9265..4b64aae5c 100755 --- a/.evergreen/auth_aws/aws_setup.sh +++ b/.evergreen/auth_aws/aws_setup.sh @@ -32,7 +32,7 @@ if [ -f $SCRIPT_DIR/test-env.sh ]; then rm $SCRIPT_DIR/test-env.sh fi -python aws_tester.py "$1" +python aws_tester.py "$@" source $SCRIPT_DIR/test-env.sh popd From 5a5ab46d3768dc63bf0c399d47e7f5d3105253d9 Mon Sep 17 00:00:00 2001 From: Steven Silvester <steven.silvester@ieee.org> Date: Wed, 5 Mar 2025 17:10:05 -0600 Subject: [PATCH 15/19] Update .evergreen/auth_aws/aws_tester.py Co-authored-by: Ezra Chung <88335979+eramongodb@users.noreply.github.com> --- .evergreen/auth_aws/aws_tester.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.evergreen/auth_aws/aws_tester.py b/.evergreen/auth_aws/aws_tester.py index d7505172d..274ed66e5 100755 --- a/.evergreen/auth_aws/aws_tester.py +++ b/.evergreen/auth_aws/aws_tester.py @@ -238,8 +238,11 @@ def setup_web_identity(): def handle_creds(creds: dict): if "USER" in creds: USER = quote_plus(creds["USER"]) - PASS = quote_plus(creds["PASS"]) - MONGODB_URI = f"mongodb://{USER}:{PASS}@localhost" + if "PASS" in creds: + PASS = quote_plus(creds["PASS"]) + MONGODB_URI = f"mongodb://{USER}:{PASS}@localhost" + else: + MONGODB_URI = f"mongodb://{USER}@localhost" else: MONGODB_URI = "mongodb://localhost" MONGODB_URI = f"{MONGODB_URI}/aws?authMechanism=MONGODB-AWS" From bb2e4153fb1d5c1db167ffb160c734eb3d15115f Mon Sep 17 00:00:00 2001 From: Steven Silvester <steve.silvester@mongodb.com> Date: Wed, 5 Mar 2025 17:15:03 -0600 Subject: [PATCH 16/19] debug --- .evergreen/tests/test-aws.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.evergreen/tests/test-aws.sh b/.evergreen/tests/test-aws.sh index e66d5fee7..f3d86130a 100755 --- a/.evergreen/tests/test-aws.sh +++ b/.evergreen/tests/test-aws.sh @@ -21,6 +21,7 @@ rm test-env.sh bash aws_setup.sh ec2 # Ensure there is no password in the URI. +cat test-env.sh cat test-env.sh | grep MONGODB_URI | grep -v -q "@" rm test-env.sh From f1660aba6bc068cb5108a5e9ef0d0f93a6afec88 Mon Sep 17 00:00:00 2001 From: Steven Silvester <steve.silvester@mongodb.com> Date: Wed, 5 Mar 2025 19:23:00 -0600 Subject: [PATCH 17/19] debug --- .evergreen/tests/test-aws.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/.evergreen/tests/test-aws.sh b/.evergreen/tests/test-aws.sh index f3d86130a..e66d5fee7 100755 --- a/.evergreen/tests/test-aws.sh +++ b/.evergreen/tests/test-aws.sh @@ -21,7 +21,6 @@ rm test-env.sh bash aws_setup.sh ec2 # Ensure there is no password in the URI. -cat test-env.sh cat test-env.sh | grep MONGODB_URI | grep -v -q "@" rm test-env.sh From 148b5a6b5efb83fa6f94748e44f0892248cc52a2 Mon Sep 17 00:00:00 2001 From: Steven Silvester <steve.silvester@mongodb.com> Date: Wed, 5 Mar 2025 19:32:06 -0600 Subject: [PATCH 18/19] debug --- .evergreen/tests/test-aws.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.evergreen/tests/test-aws.sh b/.evergreen/tests/test-aws.sh index e66d5fee7..4aa28a829 100755 --- a/.evergreen/tests/test-aws.sh +++ b/.evergreen/tests/test-aws.sh @@ -21,6 +21,8 @@ rm test-env.sh bash aws_setup.sh ec2 # Ensure there is no password in the URI. +source test-env.sh +echo ${MONGODB_URI} cat test-env.sh | grep MONGODB_URI | grep -v -q "@" rm test-env.sh From 17e40293b0b2795c1dbf44b7e6c6431f60b2fdda Mon Sep 17 00:00:00 2001 From: Steven Silvester <steve.silvester@mongodb.com> Date: Wed, 5 Mar 2025 19:41:38 -0600 Subject: [PATCH 19/19] remove debug --- .evergreen/tests/test-aws.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/.evergreen/tests/test-aws.sh b/.evergreen/tests/test-aws.sh index 4aa28a829..e66d5fee7 100755 --- a/.evergreen/tests/test-aws.sh +++ b/.evergreen/tests/test-aws.sh @@ -21,8 +21,6 @@ rm test-env.sh bash aws_setup.sh ec2 # Ensure there is no password in the URI. -source test-env.sh -echo ${MONGODB_URI} cat test-env.sh | grep MONGODB_URI | grep -v -q "@" rm test-env.sh