From a9c52a9115b7c1eebe90d1ecfafb6f0a1027e70b Mon Sep 17 00:00:00 2001 From: Ruslan Temirkhanov Date: Mon, 24 Aug 2026 18:02:39 -0400 Subject: [PATCH 1/3] feat(test): Add --v2 RQTS local contract test runner Add an opt-in --v2 flag to cfn test that runs contract tests through the RQTS (CTv2) executor container instead of the pytest-based suite. Without --v2 the existing path is unchanged. The new rpdk.core.rqts package resolves and pulls the pinned public executor image with a bounded retry and a cached-image fallback, aggregates Docker, artifact and credential preconditions into a single error, builds the docker run argv, and orchestrates the run: mint temporary credentials, stream container output, and map the container exit code to the CLI result. Credential values reach Docker only through the client process environment. The argv carries name-only -e flags, so secrets never appear in the command line or in debug logs. Handler invocation uses DirectJar mode, so no host networking and no handler endpoint are configured, and scenario selection is owned by the executor image. Also add --rqts-image to override the image reference when testing pre-release executor builds. --- src/rpdk/core/rqts/__init__.py | 1 + src/rpdk/core/rqts/argv.py | 157 ++++++++++++ src/rpdk/core/rqts/constants.py | 52 ++++ src/rpdk/core/rqts/image.py | 159 ++++++++++++ src/rpdk/core/rqts/preconditions.py | 119 +++++++++ src/rpdk/core/rqts/runner.py | 257 +++++++++++++++++++ src/rpdk/core/test.py | 26 ++ tests/rqts/__init__.py | 0 tests/rqts/test_argv.py | 373 ++++++++++++++++++++++++++++ tests/rqts/test_image.py | 288 +++++++++++++++++++++ tests/rqts/test_preconditions.py | 189 ++++++++++++++ tests/rqts/test_runner.py | 371 +++++++++++++++++++++++++++ tests/test_test.py | 194 ++++++++++++++- 13 files changed, 2185 insertions(+), 1 deletion(-) create mode 100644 src/rpdk/core/rqts/__init__.py create mode 100644 src/rpdk/core/rqts/argv.py create mode 100644 src/rpdk/core/rqts/constants.py create mode 100644 src/rpdk/core/rqts/image.py create mode 100644 src/rpdk/core/rqts/preconditions.py create mode 100644 src/rpdk/core/rqts/runner.py create mode 100644 tests/rqts/__init__.py create mode 100644 tests/rqts/test_argv.py create mode 100644 tests/rqts/test_image.py create mode 100644 tests/rqts/test_preconditions.py create mode 100644 tests/rqts/test_runner.py diff --git a/src/rpdk/core/rqts/__init__.py b/src/rpdk/core/rqts/__init__.py new file mode 100644 index 00000000..3e07489a --- /dev/null +++ b/src/rpdk/core/rqts/__init__.py @@ -0,0 +1 @@ +"""RQTS (CTv2 local) contract test execution support for ``cfn test --v2``.""" diff --git a/src/rpdk/core/rqts/argv.py b/src/rpdk/core/rqts/argv.py new file mode 100644 index 00000000..d4503afd --- /dev/null +++ b/src/rpdk/core/rqts/argv.py @@ -0,0 +1,157 @@ +"""Pure argv construction for the RQTS (CTv2 local) test runner. + +This module is deliberately **side-effect free**: it performs no subprocess +calls, no filesystem or network I/O, and no Docker/AWS interaction. It only +transforms already-resolved inputs (image ref, project, region) into a +``docker run`` argument list, plus the separate process-environment mapping +(:func:`build_container_env`) that carries the credential values. Keeping both +pure makes them the units that the feature's property-based tests exercise +directly. Credential secrets never enter the argv: the ``-e`` flags are +name-only and docker resolves the values from the client process environment. + +``cfn test --v2`` targets the published executor image's DirectJar handler +mode. The container command produced is:: + + --extension contract-tests run-tests /work/.zip --direct-jar \ + -r -o + +DirectJar loads the handler JAR directly into the executor JVM, so there is no +handler endpoint, no host networking, and no ``--handler-jar``/``-h``/``-tn`` +flags. Inputs are packaged in the artifact zip and resolved by the executor, so +``-i`` is not emitted here. Scenario selection is owned by the executor image +(capability conditions plus namespace gating of the 1P-oriented ``tagging-*`` +scenarios), so no ``-s``/``--scenarios`` and no exclusion flags are emitted. +""" + +from .constants import CONTAINER_OUTPUT_DIR, CONTAINER_WORKDIR, EXTENSION_CONTRACT_TESTS + +# Environment variable names the RQTS executor reads for AWS access. These are +# emitted as NAME-ONLY ``-e`` flags: docker resolves each value from the docker +# client process environment (supplied via ``run_container``'s ``env``), so no +# credential value ever appears in the argv - which makes the DEBUG-logged +# command line and ``ps`` output safe by construction. +# The B105 suppressions below are false positives: these literals are +# environment-variable NAMES and credential-dict KEYS, never secret values. +_ENV_ACCESS_KEY_ID = "AWS_ACCESS_KEY_ID" +_ENV_SECRET_ACCESS_KEY = "AWS_SECRET_ACCESS_KEY" # nosec B105 +_ENV_SESSION_TOKEN = "AWS_SESSION_TOKEN" # nosec B105 +_ENV_REGION = "AWS_REGION" + +# Keys used by ``boto_helpers.get_temporary_credentials`` (its default +# ``BOTO_CRED_KEYS``) to describe minted temporary credentials. +_CRED_ACCESS_KEY_ID = "aws_access_key_id" +_CRED_SECRET_ACCESS_KEY = "aws_secret_access_key" # nosec B105 +_CRED_SESSION_TOKEN = "aws_session_token" # nosec B105 + + +def _env_args(): + """Return the name-only ``-e`` flags for the AWS credential/region variables. + + Values are deliberately NOT embedded: docker inherits each variable from + the docker client process environment, which the runner populates via + :func:`build_container_env`. + """ + return [ + "-e", + _ENV_ACCESS_KEY_ID, + "-e", + _ENV_SECRET_ACCESS_KEY, + "-e", + _ENV_SESSION_TOKEN, + "-e", + _ENV_REGION, + ] + + +def build_container_env(creds, region): + """Build the environment mapping the docker client process must export. + + This is the ONLY place credential secret values are threaded: the runner + merges this mapping into the spawned ``docker run`` process environment, + and the argv references the variables by name only. + + :param creds: a mapping of minted temporary credentials using + ``boto_helpers`` default keys (``aws_access_key_id``, + ``aws_secret_access_key``, ``aws_session_token``). + :param str region: the effective AWS region. + :returns: dict of environment variable name -> value for the docker client. + """ + return { + _ENV_ACCESS_KEY_ID: creds[_CRED_ACCESS_KEY_ID], + _ENV_SECRET_ACCESS_KEY: creds[_CRED_SECRET_ACCESS_KEY], + _ENV_SESSION_TOKEN: creds[_CRED_SESSION_TOKEN], + _ENV_REGION: region, + } + + +def build_docker_argv( + image_ref, + project, + region, + workdir=None, + artifact_name=None, + output_dir=CONTAINER_OUTPUT_DIR, +): + """Build the full ``docker run`` argv for the RQTS container (DirectJar). + + The argv contains NO credential values by construction: the ``-e`` flags + are name-only, and the values travel through the docker client process + environment built by :func:`build_container_env`. This keeps the argv safe + to log at DEBUG (Req 4.10) and invisible to ``ps``. + + Composition:: + + docker run --rm + -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY + -e AWS_SESSION_TOKEN -e AWS_REGION + -v :/work + + --extension contract-tests run-tests /work/ + --direct-jar -r -o + + :param str image_ref: the resolved RQTS container image reference. + :param project: the loaded :class:`rpdk.core.project.Project`. + ``root`` is the default bind-mount source and ``hypenated_name`` names + the artifact zip. + :param str region: the effective AWS region (exported as ``AWS_REGION`` via + the process environment and passed via ``-r``). + :param workdir: host directory to bind-mount onto :data:`CONTAINER_WORKDIR`. + Defaults to ``str(project.root)``. + :param artifact_name: the handler artifact zip filename inside the mount. + Defaults to ``f"{project.hypenated_name}.zip"``. + :param output_dir: container-internal output directory passed via ``-o``. + Defaults to :data:`CONTAINER_OUTPUT_DIR`. + :returns: the complete ``docker run`` argument list. + """ + if workdir is None: + workdir = str(project.root) + if artifact_name is None: + artifact_name = f"{project.hypenated_name}.zip" + + artifact_path_in_container = f"{CONTAINER_WORKDIR}/{artifact_name}" + + argv = ["docker", "run", "--rm"] + # Credential/region variables by NAME only; values come from the process env. + argv += _env_args() + # Bind mount the working directory so the artifact zip is readable in-container. + argv += ["-v", f"{workdir}:{CONTAINER_WORKDIR}"] + # The image to run. + argv.append(image_ref) + # The executor command: top-level --extension selector, then the run-tests + # subcommand with the positional artifact path and DirectJar handler mode. + argv += [ + "--extension", + EXTENSION_CONTRACT_TESTS, + "run-tests", + artifact_path_in_container, + "--direct-jar", + "-r", + region, + "-o", + output_dir, + ] + # No -s/--scenarios and no --exclude-* flags: the executor image owns + # scenario selection (capability conditions + first-party namespace gating + # of the tagging-* scenarios), so 3P resources get the non-tagging subset + # automatically and 1P resources run the full suite. + return argv diff --git a/src/rpdk/core/rqts/constants.py b/src/rpdk/core/rqts/constants.py new file mode 100644 index 00000000..e0e9ca2e --- /dev/null +++ b/src/rpdk/core/rqts/constants.py @@ -0,0 +1,52 @@ +"""Constants for the RQTS (CTv2 local) contract test executor. + +These values are owned by the CLI and pin the behavior of ``cfn test --v2``: +the RQTS Docker image reference, the executor extension selector, the image +pull retry policy, and the container-internal bind-mount root and output +directory. + +``cfn test --v2`` targets the published executor image's DirectJar handler +mode, whose CLI contract is:: + + --extension contract-tests run-tests --direct-jar -r -o + +Input subsetting (``-i``) is intentionally NOT emitted: the executor uses the +inputs packaged inside the artifact zip. Scenario subsetting (``-s``) is NOT +emitted either: scenario selection is owned by the executor image, which gates +1P-oriented scenarios (``tagging-*``) by resource-type namespace, so the CLI +passes no scenario list and no exclusions. +""" + +# Fully-qualified, CLI-pinned RQTS image on the ECR Public Gallery. +# ``s5r7m5i4`` is the permanent default registry alias of the publishing +# account. The mutable ``latest`` tag is followed deliberately: the cloud +# contract-test path always runs the latest published image, and the CLI keeps +# parity by attempting a pull on every run (falling back to the local cache +# when the registry is unreachable) instead of pinning per release. +RQTS_IMAGE_REFERENCE = "public.ecr.aws/s5r7m5i4/cfn-rqts-executor-external:latest" + +# The executor top-level extension selector for contract tests. Passed as the +# top-level ``--extension`` option BEFORE the ``run-tests`` subcommand. +EXTENSION_CONTRACT_TESTS = "contract-tests" + +# SAM Local handler port. Retained for reference; unused by the DirectJar mode +# that ``cfn test --v2`` targets (DirectJar loads the handler in-process and +# needs no handler endpoint). +DEFAULT_HANDLER_PORT = 3031 + +# Image pull retry policy (Req 2.4). +PULL_MAX_ATTEMPTS = 3 +PULL_ATTEMPT_TIMEOUT_SECONDS = 120 + +# Container-internal bind-mount root. +CONTAINER_WORKDIR = "/work" + +# Container-internal directory the executor writes its output to (under the +# bind mount, so results are visible on the host). +CONTAINER_OUTPUT_DIR = "/work/rqts-output" + +# NOTE: the CLI deliberately owns NO scenario set. The executor image decides +# which scenarios run: capability conditions (taggable/creatable/...) plus +# namespace gating that restricts the 1P-oriented ``tagging-*`` scenarios to +# reserved first-party namespaces. 3P resources therefore get the non-tagging +# subset automatically, with no scenario or exclusion flags from the CLI. diff --git a/src/rpdk/core/rqts/image.py b/src/rpdk/core/rqts/image.py new file mode 100644 index 00000000..55d91cfe --- /dev/null +++ b/src/rpdk/core/rqts/image.py @@ -0,0 +1,159 @@ +# have to skip B404, subprocess is required to drive the local Docker CLI +# have to skip B603/B607, docker is invoked with a fixed, non-shell argv +"""RQTS image resolution and pull-with-retry. + +This module owns how ``cfn test --v2`` decides *which* RQTS container image to +run and how it makes that image available in the local Docker image store: + +* :func:`resolve_image` picks the effective image reference (an explicit + ``--rqts-image`` override, otherwise the CLI-pinned default). +* :func:`image_present_locally` reports whether the image is already in the + local Docker image store. +* :func:`ensure_image` attempts an anonymous ``docker pull`` on every run (the + pinned reference is a mutable ``latest`` tag, and the cloud contract-test + path always runs the latest published image, so local runs follow it for + parity) with a bounded per-attempt timeout and a capped number of attempts. + When every attempt fails it falls back to the local image store with a + warning if the image is cached, and raises + :class:`~rpdk.core.exceptions.SysExitRecommendedError` only when no cached + copy exists. + +Docker is driven through the ``docker`` CLI via :mod:`subprocess`. The pull is +deliberately anonymous: no AWS credentials are ever supplied to it, because the +RQTS image is published to the ECR Public Gallery and is anonymously pullable. +""" +import logging +import subprocess # nosec B404 + +from rpdk.core.exceptions import SysExitRecommendedError + +from .constants import ( + PULL_ATTEMPT_TIMEOUT_SECONDS, + PULL_MAX_ATTEMPTS, + RQTS_IMAGE_REFERENCE, +) + +LOG = logging.getLogger(__name__) + + +def resolve_image(args): + """Resolve the effective RQTS image reference. + + Returns the ``--rqts-image`` override when it is provided and non-empty, + otherwise the CLI-pinned :data:`RQTS_IMAGE_REFERENCE`. + + :param args: parsed CLI arguments (expects an ``rqts_image`` attribute) + :return: the image reference to run + :rtype: str + """ + override = getattr(args, "rqts_image", None) + if override: + return override + return RQTS_IMAGE_REFERENCE + + +def _run_docker(docker_args, timeout=None): + """Run a ``docker`` subcommand with a fixed, non-shell argv. + + Kept as a small internal seam so tests can patch a single call site. No AWS + credentials are ever injected here; the child inherits the ambient + environment only. + + :param list docker_args: arguments following ``docker`` (for example + ``["image", "inspect", ref]``) + :param timeout: optional per-attempt timeout in seconds + :return: the completed process + :rtype: subprocess.CompletedProcess + """ + return subprocess.run( # nosec B603 B607 + ["docker", *docker_args], + check=False, + capture_output=True, + timeout=timeout, + ) + + +def image_present_locally(image_ref): + """Return whether ``image_ref`` is present in the local Docker image store. + + Uses ``docker image inspect ``, which exits non-zero when the image is + not present locally. + + :param str image_ref: the image reference to look up + :return: ``True`` when the image is available locally, ``False`` otherwise + :rtype: bool + """ + try: + completed = _run_docker(["image", "inspect", image_ref]) + except (OSError, subprocess.SubprocessError) as err: + LOG.debug("Local image inspect for %s failed: %s", image_ref, err) + return False + return completed.returncode == 0 + + +def ensure_image(image_ref): + """Ensure ``image_ref`` is available locally, refreshing it when possible. + + A pull is attempted on **every** invocation so that a moved mutable tag + (for example ``latest``) is picked up without manual ``docker pull``. + The pull is anonymous (no AWS credentials) with a bounded per-attempt + timeout of :data:`PULL_ATTEMPT_TIMEOUT_SECONDS`, retrying up to + :data:`PULL_MAX_ATTEMPTS` times and stopping on the first success. An + up-to-date image costs only a manifest check; no layers are re-downloaded. + + When every attempt fails, the run falls back to the local image store if + the image is cached there (with a warning that it may be stale); only when + no cached copy exists is the failure fatal. + + :param str image_ref: the resolved RQTS image reference + :raises SysExitRecommendedError: if the pull fails on every attempt and + the image is not present in the local image store + """ + LOG.info("Pulling RQTS image %s (anonymous pull)", image_ref) + last_error = None + for attempt in range(1, PULL_MAX_ATTEMPTS + 1): + LOG.debug( + "Pulling RQTS image %s (attempt %d of %d)", + image_ref, + attempt, + PULL_MAX_ATTEMPTS, + ) + try: + completed = _run_docker( + ["pull", image_ref], timeout=PULL_ATTEMPT_TIMEOUT_SECONDS + ) + except subprocess.TimeoutExpired: + last_error = f"timed out after {PULL_ATTEMPT_TIMEOUT_SECONDS}s" + LOG.debug("Pull attempt %d for %s %s", attempt, image_ref, last_error) + continue + except (OSError, subprocess.SubprocessError) as err: + last_error = str(err) + LOG.debug( + "Pull attempt %d for %s failed: %s", attempt, image_ref, last_error + ) + continue + + if completed.returncode == 0: + LOG.debug( + "Successfully pulled RQTS image %s on attempt %d", image_ref, attempt + ) + return + + last_error = (completed.stderr or b"").decode("utf-8", "replace").strip() or ( + f"docker pull exited with code {completed.returncode}" + ) + LOG.debug("Pull attempt %d for %s failed: %s", attempt, image_ref, last_error) + + if image_present_locally(image_ref): + LOG.warning( + "Could not pull the RQTS image '%s' (%s); using the locally cached " + "copy, which may be stale", + image_ref, + last_error, + ) + return + + raise SysExitRecommendedError( + f"Failed to pull the RQTS image '{image_ref}' after " + f"{PULL_MAX_ATTEMPTS} attempts: {last_error}" + ) diff --git a/src/rpdk/core/rqts/preconditions.py b/src/rpdk/core/rqts/preconditions.py new file mode 100644 index 00000000..1351e977 --- /dev/null +++ b/src/rpdk/core/rqts/preconditions.py @@ -0,0 +1,119 @@ +"""Precondition checks for the RQTS (CTv2 local) contract test runner. + +``cfn test --v2`` requires several runtime and project prerequisites before the +RQTS container can run: a working Docker runtime, a built artifact package, and +valid AWS credentials and region. + +The DirectJar handler mode loads the handler JAR directly into the executor +JVM, so there is no SAM Local handler endpoint to probe and no separate input +resolution: inputs are packaged inside the artifact zip and read from there by +the executor. + +Each check in this module is independent and side-effect-free with respect to +the others. A check appends a single human-readable message on failure and +NEVER raises, so the caller (``RqtsRunner``) can aggregate every unmet +precondition into one error rather than failing on the first problem +(Requirement 3.7). ``check_preconditions`` returns the aggregated list of +failure messages; an empty list means all preconditions are met. +""" + +import logging +import shutil +import subprocess # nosec B404 + +from ..boto_helpers import create_sdk_session + +LOG = logging.getLogger(__name__) + +# Bounded timeout (seconds) for the Docker daemon ping so a hung daemon cannot +# stall the precondition phase. +_DOCKER_INFO_TIMEOUT_SECONDS = 10 + + +def _check_docker(): + """Return a failure message if Docker is unavailable, else ``None``. + + Docker is available only when the ``docker`` CLI is present on PATH AND the + Docker daemon is reachable (probed with ``docker info``) (Requirement 3.2). + """ + if shutil.which("docker") is None: + return ( + "Docker is required and must be running: the 'docker' CLI was not " + "found on PATH." + ) + + try: + result = subprocess.run( # nosec B603, B607 + ["docker", "info"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=_DOCKER_INFO_TIMEOUT_SECONDS, + check=False, + ) + except (OSError, subprocess.SubprocessError) as e: + LOG.debug("Docker daemon ping failed", exc_info=e) + return ( + "Docker is required and must be running: the Docker daemon could " + "not be reached." + ) + + if result.returncode != 0: + return ( + "Docker is required and must be running: the Docker daemon could " + "not be reached." + ) + + return None + + +def _check_artifact(project): + """Return a failure message if the built artifact package is missing. + + Mirrors ``Project._get_zip_file_path()``: the package lives at + ``project.root / f"{project.hypenated_name}.zip"`` (Requirement 3.3). + """ + artifact_name = f"{project.hypenated_name}.zip" + artifact_path = project.root / artifact_name + if not artifact_path.is_file(): + return ( + f"artifact package '{artifact_name}' not found; build the project " "first." + ) + return None + + +def _check_credentials(args): + """Return a failure message if AWS credentials or region are unavailable. + + Reuses ``boto_helpers.create_sdk_session`` which raises when the region or + credentials are missing; that exception is caught and converted to a failure + message so the check never raises (Requirement 3.5). + """ + try: + create_sdk_session(args.region, args.profile) + except Exception as e: # pylint: disable=broad-except + LOG.debug("AWS session could not be created", exc_info=e) + return "valid AWS credentials and a region are required." + return None + + +def check_preconditions(args, project): + """Verify all ``cfn test --v2`` preconditions and aggregate any failures. + + Runs each independent precondition check and collects the human-readable + failure message from every unmet one. No check raises; the caller turns a + non-empty list into a single ``SysExitRecommendedError`` (Requirements 3.1, + 3.7). + + :param args: parsed CLI arguments (uses ``args.region`` and ``args.profile``) + :param project: loaded ``rpdk.core.project.Project`` + :returns: list of failure messages; empty means all preconditions are met + """ + failures = [] + for message in ( + _check_docker(), + _check_artifact(project), + _check_credentials(args), + ): + if message is not None: + failures.append(message) + return failures diff --git a/src/rpdk/core/rqts/runner.py b/src/rpdk/core/rqts/runner.py new file mode 100644 index 00000000..375923b9 --- /dev/null +++ b/src/rpdk/core/rqts/runner.py @@ -0,0 +1,257 @@ +# have to skip B404, subprocess is required to drive the local Docker CLI +# have to skip B603, docker is invoked with a fixed, non-shell argv +"""RQTS (CTv2 local) contract test runner. + +This module drives the ``docker run`` step of ``cfn test --v2`` and maps the +container's exit code onto the CLI's exit-code contract: + +* :class:`RqtsRunner` orchestrates the full ``--v2`` pipeline: it guards the + project artifact type, aggregates and enforces preconditions, mints temporary + AWS credentials, resolves and ensures the RQTS image, builds the ``docker + run`` argv for the executor's DirectJar handler mode, runs the container, and + maps its exit code. Any failure raises + :class:`~rpdk.core.exceptions.SysExitRecommendedError`. +* :func:`run_container` spawns the ``docker run`` process, streams its + stdout/stderr to the terminal live (as produced, without buffering to + completion), and returns the container's exit code. A failure to even start + the container is surfaced as a + :class:`~rpdk.core.exceptions.SysExitRecommendedError`. +* :func:`map_exit_code` turns that exit code into the CLI's contract: ``0`` + reports a pass and returns; any non-zero code raises + :class:`~rpdk.core.exceptions.SysExitRecommendedError`, which ``cli.py`` + already maps to ``SystemExit(1)``. It defers the human-readable pass/fail + summary to :func:`report_result` so the summary and the exit-code mapping + stay single-sourced. +* :func:`report_result` renders the concise overall pass/fail summary, + consistent with the existing pytest-based ``cfn test`` reporting. The + per-scenario/test outcomes themselves are streamed live by + :func:`run_container`, so this summary does not re-parse or duplicate them. + +The DirectJar handler mode loads the handler JAR directly into the executor +JVM, so there is no handler endpoint to probe and no host networking to +configure. Docker is driven through the ``docker`` CLI via :mod:`subprocess`. +The child inherits the parent's stdout/stderr so RQTS results appear live in +the developer's terminal exactly as the executor emits them. +""" +import logging +import os +import subprocess # nosec B404 +from pathlib import Path + +from rpdk.core.exceptions import SysExitRecommendedError + +from ..boto_helpers import BOTO_CRED_KEYS, create_sdk_session, get_temporary_credentials +from ..project import ARTIFACT_TYPE_HOOK, ARTIFACT_TYPE_RESOURCE +from .argv import build_container_env, build_docker_argv +from .image import ensure_image, resolve_image +from .preconditions import check_preconditions + +LOG = logging.getLogger(__name__) + +# Host-side directory (under the project root, i.e. under the bind mount) that +# the executor writes its output to. Created before the container runs so the +# bind-mounted output path exists on the host. +HOST_OUTPUT_DIRNAME = "rqts-output" + +# Overall summary lines for a completed RQTS run. Phrasing is kept consistent +# with the existing pytest-based ``cfn test`` reporting, whose failure path +# raises ``SysExitRecommendedError("One or more contract tests failed")`` in +# ``test.invoke_test``; the RQTS variants name the runner explicitly. The +# container streams the per-scenario/test outcomes live (Requirement 6.2), so +# these are deliberately concise overall summaries (Requirement 6.1). +# B105 false positive: a log summary string, not a password (bandit matches +# the "PASS" in the variable name). +PASS_SUMMARY = "RQTS contract tests passed" # nosec B105 +FAIL_SUMMARY = "One or more RQTS contract tests failed" + + +def run_container(argv, env=None): + """Spawn ``docker run`` and stream its output live, returning the exit code. + + The child process inherits the parent's stdout/stderr, so the RQTS + executor's output is streamed to the terminal as it is produced rather than + buffered until the process completes (Requirement 5.1). This function blocks + until the container exits and returns its exit code. + + Credential values reach docker exclusively through ``env`` (merged over the + ambient environment): the argv's ``-e`` flags are name-only, so the command + line is safe to log and never carries secrets. + + :param list argv: the full ``docker run`` argv (as built by + :func:`rpdk.core.rqts.argv.build_docker_argv`) + :param env: optional mapping of extra environment variables (for example + from :func:`rpdk.core.rqts.argv.build_container_env`) merged over + ``os.environ`` for the docker client process + :return: the container's exit code + :rtype: int + :raises SysExitRecommendedError: if the container process cannot be spawned + """ + LOG.debug("Running RQTS container: %s", " ".join(argv)) + process_env = {**os.environ, **env} if env else None + try: + # stdout/stderr default to None, so the child inherits the parent's + # terminal and streams output live without buffering (Requirement 5.1). + with subprocess.Popen(argv, env=process_env) as process: # nosec B603 + return process.wait() + except OSError as err: + LOG.debug("Failed to start the RQTS container", exc_info=err) + raise SysExitRecommendedError( + f"the RQTS container could not be started: {err}" + ) from err + + +def report_result(code): + """Surface the overall pass/fail summary for a completed RQTS run. + + The RQTS container already streams its per-scenario/test outcomes to the + terminal live via :func:`run_container` (Requirements 5.1, 6.2); this helper + adds only the concise overall summary that mirrors the existing + pytest-based ``cfn test`` reporting (Requirement 6.1), without re-parsing or + duplicating the streamed output. A ``0`` code logs an informational pass + summary; any non-zero code raises :class:`SysExitRecommendedError` whose + message names the RQTS runner, consistent with the existing "One or more + contract tests failed" phrasing used by the pytest path. + + This helper only renders the summary; the exit-code interpretation itself is + owned by :func:`map_exit_code`, which calls this helper so the summary and + the exit-code mapping stay single-sourced (Requirement 6.3). + + :param int code: the container's exit code + :raises SysExitRecommendedError: if ``code`` is non-zero + """ + if code == 0: + LOG.info(PASS_SUMMARY) + return + raise SysExitRecommendedError(FAIL_SUMMARY) + + +def map_exit_code(code): + """Map the RQTS container exit code onto the CLI's exit-code contract. + + A ``0`` exit code means every RQTS contract test passed: an informational + summary is logged and the function returns normally so the CLI exits ``0`` + (Requirements 5.2, 6.1). Any non-zero exit code means one or more contract + tests failed and raises :class:`SysExitRecommendedError`, which ``cli.py`` + maps to ``SystemExit(1)`` (Requirements 5.3, 6.3). + + The pass/fail summary is delegated to :func:`report_result` so that the + reporting and the exit-code mapping remain single-sourced; this function is + the exit-code contract entry point invoked by :meth:`RqtsRunner.run`. + + :param int code: the container's exit code + :raises SysExitRecommendedError: if ``code`` is non-zero + """ + report_result(code) + + +class RqtsRunner: + """Orchestrates the ``cfn test --v2`` RQTS pipeline. + + A single instance owns the parsed CLI ``args`` and the loaded + :class:`~rpdk.core.project.Project` and drives the fixed pipeline in + :meth:`run`: artifact-type guard, precondition aggregation, credential + minting, image resolve/ensure, ``docker run`` argv construction (for the + executor's DirectJar handler mode), container execution, and exit-code + mapping. + + Module projects are short-circuited upstream in ``test()`` before the runner + is constructed, so this class only handles resource (the supported case), + hook, and indeterminate artifact types. + """ + + def __init__(self, args, project): + """Store the parsed CLI arguments and the loaded project. + + :param args: parsed CLI arguments (an argparse ``Namespace``). The + runner reads ``region``, ``profile``, ``role_arn``, + ``source_account``, ``source_arn`` and ``rqts_image``. + :param project: the loaded :class:`~rpdk.core.project.Project`. + """ + self.args = args + self.project = project + + def _guard_artifact_type(self): + """Fail fast unless the project is a supported resource type. + + Hook projects are unsupported by the RQTS local runner (Requirement + 7.2); any artifact type that is neither a resource nor a hook is treated + as indeterminate (Requirement 7.5). Module projects are handled upstream + in ``test()`` and never reach this method. + + :raises SysExitRecommendedError: for hook or indeterminate artifact + types + """ + artifact_type = self.project.artifact_type + if artifact_type == ARTIFACT_TYPE_HOOK: + raise SysExitRecommendedError( + "the RQTS local test runner supports resource types only" + ) + if artifact_type != ARTIFACT_TYPE_RESOURCE: + raise SysExitRecommendedError( + "could not determine the project artifact type" + ) + + def _mint_credentials(self): + """Mint temporary AWS credentials for the container to use. + + Reuses the exact ``boto_helpers`` pattern used by the contract-test + path: create a session from the effective region/profile, then request + temporary credentials (assuming ``--role-arn`` when supplied) with the + confused-deputy ``--source-account`` / ``--source-arn`` headers + (Requirement 3.5). ``BOTO_CRED_KEYS`` is used so the returned mapping's + keys match what :func:`rpdk.core.rqts.argv.build_docker_argv` expects. + + :return: a mapping of temporary credentials keyed by ``BOTO_CRED_KEYS`` + :rtype: dict + """ + session = create_sdk_session(self.args.region, self.args.profile) + return get_temporary_credentials( + session, + BOTO_CRED_KEYS, + self.args.role_arn, + headers={ + "account_id": self.args.source_account, + "source_arn": self.args.source_arn, + }, + ) + + def run(self): + """Orchestrate the full ``--v2`` pipeline. + + Raises :class:`~rpdk.core.exceptions.SysExitRecommendedError` on any + failure (guard, preconditions, image pull, container start, or a + non-zero container exit code); returns normally when every RQTS contract + test passes. + """ + self._guard_artifact_type() + + failures = check_preconditions(self.args, self.project) + if failures: + raise SysExitRecommendedError( + "cannot run 'cfn test --v2'; the following preconditions were " + "not met:\n" + "\n".join(f" - {failure}" for failure in failures) + ) + + creds = self._mint_credentials() + + image_ref = resolve_image(self.args) + ensure_image(image_ref) + + # Ensure the host-side output directory (under the bind mount) exists so + # the container's -o path is present when docker mounts it. + output_dir = Path(self.project.root) / HOST_OUTPUT_DIRNAME + output_dir.mkdir(parents=True, exist_ok=True) + + # The argv is credential-free (name-only -e flags), so logging the full + # command line at DEBUG (Req 4.10) cannot leak secrets; the values + # travel only through the docker client process environment. + argv = build_docker_argv( + image_ref, + self.project, + self.args.region, + ) + LOG.debug("RQTS docker command: %s", " ".join(argv)) + + container_env = build_container_env(creds, self.args.region) + exit_code = run_container(argv, env=container_env) + map_exit_code(exit_code) diff --git a/src/rpdk/core/test.py b/src/rpdk/core/test.py index e0f909c3..83a23fe2 100644 --- a/src/rpdk/core/test.py +++ b/src/rpdk/core/test.py @@ -418,6 +418,13 @@ def test(args): LOG.warning("The test command is not supported in a module project") return + if args.v2: + # local import keeps the pytest path import-light + from .rqts.runner import RqtsRunner # pylint: disable=import-outside-toplevel + + RqtsRunner(args, project).run() + return + if project.artifact_type == ARTIFACT_TYPE_HOOK: overrides = get_hook_overrides( project.root, @@ -545,6 +552,25 @@ def setup_subparser(subparsers, parents): help="Source Type Version Arn key used for Assume Role to Run Contract Tests", ) + parser.add_argument( + "--v2", + action="store_true", + default=False, + help=( + "Opt-in: run the RQTS local test runner (CTv2) in a Docker container " + "instead of the default pytest-based contract tests." + ), + ) + + parser.add_argument( + "--rqts-image", + default=None, + help=( + "Override the RQTS container image reference (for testing or pre-release " + "images). Defaults to the CLI-pinned public.ecr.aws image." + ), + ) + def _sam_arguments(parser): parser.add_argument( diff --git a/tests/rqts/__init__.py b/tests/rqts/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/rqts/test_argv.py b/tests/rqts/test_argv.py new file mode 100644 index 00000000..54626500 --- /dev/null +++ b/tests/rqts/test_argv.py @@ -0,0 +1,373 @@ +"""Property-based tests for the pure argv construction module. + +These tests exercise ``rpdk.core.rqts.argv`` for the executor's DirectJar +handler mode. The module under test is side-effect free, so no Docker or AWS +interaction is required: the tests only assert structural invariants of the +``docker run`` argument list it produces. + +The container command produced for local DirectJar is:: + + docker run --rm -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY + -e AWS_SESSION_TOKEN -e AWS_REGION -v :/work + --extension contract-tests run-tests /work/.zip + --direct-jar -r -o + +Scenario selection is owned by the executor image (capability conditions plus +first-party namespace gating), so the argv carries NO ``-s``/``--scenarios`` +and NO ``--exclude-*`` flags. + +The ``-e`` flags are name-only: credential values travel exclusively through +the container env mapping (``build_container_env``), never through the argv. + +Library: Hypothesis (the standard Python property-based testing library). Each +property test runs at least 100 generated examples via +``@settings(max_examples=100)`` and is tagged with a comment referencing the +design property it validates. +""" +from types import SimpleNamespace + +from hypothesis import given, settings, strategies as st + +from rpdk.core.rqts.argv import build_container_env, build_docker_argv +from rpdk.core.rqts.constants import ( + CONTAINER_OUTPUT_DIR, + CONTAINER_WORKDIR, + EXTENSION_CONTRACT_TESTS, +) + +# The CLI default region: ``cfn test`` defines ``--region`` with this default, so +# the effective region is always populated even when the user omits ``--region``. +CLI_DEFAULT_REGION = "us-east-1" + +# Flags the DirectJar contract must NEVER emit (they belong to the old SAM Local +# / remote-lambda shapes). +FORBIDDEN_FLAGS = ("--handler-jar", "-h", "-tn", "--sam-local", "--remote-lambda") + +# The tagging scenarios that ``cfn test --v2`` must NEVER run: none of these may +# appear anywhere in the argv (per product decision: no tagging tests). +FORBIDDEN_TAGGING_SCENARIOS = ( + "tagging-oob", + "tagging-permission", + "tagging-stack", + "tagging-system", +) + +# --------------------------------------------------------------------------- +# Strategies +# --------------------------------------------------------------------------- + +# A distinctive marker prefix used for generated credential secret values. It is +# built from characters that never appear in the type names, regions, or names +# generated below, so a secret value can be searched for reliably across the +# whole argv (Property: credentials only in env). +_SECRET_MARKER = "SECRETMARKER" + +# Alphabet for type-name / region / name segments. +_SEGMENT_ALPHABET = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" + +_segments = st.text(alphabet=_SEGMENT_ALPHABET, min_size=1, max_size=12) + + +@st.composite +def hyphenated_names(draw): + """Generate realistic ``org-service-resource`` style hyphenated names.""" + parts = draw(st.lists(_segments, min_size=3, max_size=3)) + return "-".join(parts).lower() + + +# Regions: a mix of realistic region strings (including the CLI default) and +# freely generated ones. +_KNOWN_REGIONS = [ + CLI_DEFAULT_REGION, + "us-west-2", + "eu-west-1", + "ap-southeast-2", + "eu-central-1", +] +regions = st.one_of( + st.sampled_from(_KNOWN_REGIONS), + st.text(alphabet=_SEGMENT_ALPHABET + "-", min_size=1, max_size=20), +) + + +# Distinctive, searchable secret values. Each carries the marker prefix so it can +# be located anywhere in the argv without colliding with other generated data. +_secret_values = st.text(alphabet=_SEGMENT_ALPHABET, min_size=6, max_size=24).map( + lambda body: _SECRET_MARKER + body +) + + +@st.composite +def credentials(draw): + """Generate a credentials dict using the boto_helpers default keys.""" + return { + "aws_access_key_id": draw(_secret_values), + "aws_secret_access_key": draw(_secret_values), + "aws_session_token": draw(_secret_values), + } + + +# Working directory paths for the bind mount. +workdirs = st.text(alphabet=_SEGMENT_ALPHABET + "/", min_size=1, max_size=40).map( + lambda p: "/" + p.strip("/") +) + + +def _make_project(hypenated_name, root="/project/root"): + """Lightweight stand-in for a loaded Project (only .hypenated_name/.root used).""" + return SimpleNamespace(hypenated_name=hypenated_name, root=root) + + +def _env_entries(argv): + """Return the list of ``-e`` environment entry tokens (the token after each -e).""" + entries = [] + for i, token in enumerate(argv): + if token == "-e" and i + 1 < len(argv): + entries.append(argv[i + 1]) + return entries + + +# --------------------------------------------------------------------------- +# Property 4 +# --------------------------------------------------------------------------- +# Feature: cfn-test-v2-flag, Property 4: docker run bind-mounts the working directory onto /work +@settings(max_examples=100) +@given( + name=hyphenated_names(), + region=regions, + workdir=workdirs, +) +def test_property_4_docker_run_bind_mounts_workdir(name, region, workdir): + project = _make_project(name) + argv = build_docker_argv("img:ref", project, region, workdir=workdir) + + # The argv must be a `docker run --rm` invocation containing a -v bind mount + # that maps the working directory onto the container workdir. + assert argv[:3] == ["docker", "run", "--rm"] + expected_mount = f"{workdir}:{CONTAINER_WORKDIR}" + assert "-v" in argv + v_index = argv.index("-v") + assert argv[v_index + 1] == expected_mount + + +# --------------------------------------------------------------------------- +# Property 5 +# --------------------------------------------------------------------------- +# Feature: cfn-test-v2-flag, Property 5: credential values never enter the argv; +# they travel only through the container env mapping +@settings(max_examples=100) +@given( + name=hyphenated_names(), + region=regions, + creds=credentials(), +) +def test_property_5_credentials_only_in_env_mapping_never_in_argv(name, region, creds): + project = _make_project(name) + argv = build_docker_argv("img:ref", project, region, workdir="/work/dir") + + env_entries = _env_entries(argv) + + # The -e flags are NAME-ONLY: the variable names are referenced so docker + # inherits them from the client process environment; no `=` and no values. + assert env_entries == [ + "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY", + "AWS_SESSION_TOKEN", + "AWS_REGION", + ] + + # No credential secret value appears ANYWHERE in the argv (this is what + # makes the DEBUG-logged command line and `ps` output safe). + secret_values = [ + creds["aws_access_key_id"], + creds["aws_secret_access_key"], + creds["aws_session_token"], + ] + for token in argv: + for secret in secret_values: + assert secret not in token + assert "=" not in token or not token.startswith("AWS_") + + # The values travel exclusively through the container env mapping. + env = build_container_env(creds, region) + assert env == { + "AWS_ACCESS_KEY_ID": creds["aws_access_key_id"], + "AWS_SECRET_ACCESS_KEY": creds["aws_secret_access_key"], + "AWS_SESSION_TOKEN": creds["aws_session_token"], + "AWS_REGION": region, + } + + +# --------------------------------------------------------------------------- +# Property 6 +# --------------------------------------------------------------------------- +# Feature: cfn-test-v2-flag, Property 6: run-tests receives the positional artifact path inside the mount +@settings(max_examples=100) +@given( + name=hyphenated_names(), + region=regions, +) +def test_property_6_extension_run_tests_and_artifact_path(name, region): + project = _make_project(name) + argv = build_docker_argv("img:ref", project, region) + + # The top-level extension selector precedes the run-tests subcommand. + assert "--extension" in argv + ext_index = argv.index("--extension") + assert argv[ext_index + 1] == EXTENSION_CONTRACT_TESTS + assert argv[ext_index + 2] == "run-tests" + + # The positional artifact path immediately follows run-tests and is addressed + # under the container working directory. + artifact_path = argv[ext_index + 3] + assert artifact_path == f"{CONTAINER_WORKDIR}/{name}.zip" + assert artifact_path.startswith(CONTAINER_WORKDIR) + + +# --------------------------------------------------------------------------- +# Property 7 +# --------------------------------------------------------------------------- +# Feature: cfn-test-v2-flag, Property 7: DirectJar handler mode is always selected +@settings(max_examples=100) +@given( + name=hyphenated_names(), + region=regions, +) +def test_property_7_direct_jar_mode_always_present(name, region): + project = _make_project(name) + argv = build_docker_argv("img:ref", project, region) + + assert "--direct-jar" in argv + # And the artifact path is not itself the --direct-jar token. + assert argv.count("--direct-jar") == 1 + + +# --------------------------------------------------------------------------- +# Property 8 +# --------------------------------------------------------------------------- +# Feature: cfn-test-v2-flag, Property 8: region argument always present with the effective region +@settings(max_examples=100) +@given( + name=hyphenated_names(), + # Include the default-region case explicitly by drawing a boolean that forces + # the CLI default, mirroring "--region not supplied". + use_default=st.booleans(), + region=regions, +) +def test_property_8_region_argument_always_present(name, use_default, region): + effective_region = CLI_DEFAULT_REGION if use_default else region + project = _make_project(name) + argv = build_docker_argv("img:ref", project, effective_region) + + assert "-r" in argv + r_index = argv.index("-r") + assert argv[r_index + 1] == effective_region + + +# --------------------------------------------------------------------------- +# Property 9 +# --------------------------------------------------------------------------- +# Feature: cfn-test-v2-flag, Property 9: the old SAM Local / remote-lambda flags are never emitted +@settings(max_examples=100) +@given( + name=hyphenated_names(), + region=regions, + workdir=workdirs, +) +def test_property_9_forbidden_flags_never_emitted(name, region, workdir): + project = _make_project(name) + argv = build_docker_argv("img:ref", project, region, workdir=workdir) + + for flag in FORBIDDEN_FLAGS: + assert flag not in argv + + +# --------------------------------------------------------------------------- +# Property 10 +# --------------------------------------------------------------------------- +# Feature: cfn-test-v2-flag, Property 10: no host networking is configured (DirectJar needs none) +@settings(max_examples=100) +@given( + name=hyphenated_names(), + region=regions, + workdir=workdirs, +) +def test_property_10_no_host_networking(name, region, workdir): + project = _make_project(name) + argv = build_docker_argv("img:ref", project, region, workdir=workdir) + + argv_str = " ".join(argv) + assert "--network" not in argv + assert "--add-host" not in argv + assert "host.docker.internal" not in argv_str + + +# --------------------------------------------------------------------------- +# Property 11 +# --------------------------------------------------------------------------- +# Feature: cfn-test-v2-flag, Property 11: output directory is emitted via -o under the mount +@settings(max_examples=100) +@given( + name=hyphenated_names(), + region=regions, +) +def test_property_11_output_dir_emitted(name, region): + project = _make_project(name) + argv = build_docker_argv("img:ref", project, region) + + assert "-o" in argv + o_index = argv.index("-o") + assert argv[o_index + 1] == CONTAINER_OUTPUT_DIR + # The default output dir lives under the container working directory mount. + assert CONTAINER_OUTPUT_DIR.startswith(CONTAINER_WORKDIR) + + +# --------------------------------------------------------------------------- +# Property 12 +# --------------------------------------------------------------------------- +# Feature: cfn-test-v2-flag, Property 12: no scenario-selection or exclusion flags +# are ever emitted; scenario selection is owned by the executor image +@settings(max_examples=100) +@given( + name=hyphenated_names(), + region=regions, + workdir=workdirs, +) +def test_property_12_no_scenario_selection_or_exclusion_flags(name, region, workdir): + project = _make_project(name) + argv = build_docker_argv("img:ref", project, region, workdir=workdir) + + # No scenario-selection tokens: the executor decides what runs (capability + # conditions + first-party namespace gating of the tagging-* scenarios). + assert "-s" not in argv + assert "--scenarios" not in argv + + # No test/check exclusion flags of any kind. + for token in argv: + assert not token.startswith("--exclude-") + + # No scenario name (tagging or otherwise) appears anywhere in the argv. + for tagging_scenario in FORBIDDEN_TAGGING_SCENARIOS: + assert tagging_scenario not in argv + + # The executor command therefore ends at the output directory. + assert argv[-2:] == ["-o", CONTAINER_OUTPUT_DIR] + + +# --------------------------------------------------------------------------- +# Defaults +# --------------------------------------------------------------------------- +def test_defaults_use_project_root_and_hyphenated_name(): + """workdir defaults to str(project.root); artifact_name to .zip.""" + project = _make_project("aws-foo-bar", root="/my/project/root") + argv = build_docker_argv("img:ref", project, "us-east-1") + + v_index = argv.index("-v") + assert argv[v_index + 1] == f"/my/project/root:{CONTAINER_WORKDIR}" + + ext_index = argv.index("--extension") + assert argv[ext_index + 3] == f"{CONTAINER_WORKDIR}/aws-foo-bar.zip" + + # No scenario selection: the argv ends at the output directory. + assert "-s" not in argv + assert argv[-2:] == ["-o", CONTAINER_OUTPUT_DIR] diff --git a/tests/rqts/test_image.py b/tests/rqts/test_image.py new file mode 100644 index 00000000..6dd099a4 --- /dev/null +++ b/tests/rqts/test_image.py @@ -0,0 +1,288 @@ +"""Tests for RQTS image resolution and pull-with-retry. + +These tests exercise ``rpdk.core.rqts.image`` (design Properties 1-2, plus the +pull-exhaustion edge cases and the cached-image fallback branch). The pull is +attempted on every run (mutable ``latest`` semantics); on exhaustion the run +falls back to a cached local image with a warning and fails only when no +cached copy exists. Docker is never actually invoked: the single internal +``_run_docker`` seam and ``image_present_locally`` are patched so the retry +policy can be driven deterministically and kept fast. + +Library: Hypothesis (the standard Python property-based testing library). Each +property test runs at least 100 generated examples via +``@settings(max_examples=100)`` and is tagged with a comment referencing the +design property it validates. +""" +import subprocess +from types import SimpleNamespace +from unittest import mock + +import pytest +from hypothesis import given, settings, strategies as st + +from rpdk.core.exceptions import SysExitRecommendedError +from rpdk.core.rqts import image as image_module +from rpdk.core.rqts.constants import PULL_MAX_ATTEMPTS, RQTS_IMAGE_REFERENCE + +# A marker used to build distinctive, searchable fake credential values. It is +# made of characters that never appear in the generated image references, so the +# credential values can be located reliably (or, as Property 2 requires, +# confirmed absent) anywhere in the recorded docker invocations. +_SECRET_MARKER = "AWSSECRETMARKER" + +# AWS credential environment variable names that must never be threaded into an +# anonymous ``docker pull``. +_AWS_CRED_TOKENS = ( + "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY", + "AWS_SESSION_TOKEN", + _SECRET_MARKER, +) + +_IMAGE_ALPHABET = "abcdefghijklmnopqrstuvwxyz0123456789./:-" + +# Image references, including the empty string and freely generated refs, so the +# resolution property covers both the override and the pinned-default cases. +image_refs = st.text(alphabet=_IMAGE_ALPHABET, min_size=1, max_size=60) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +class _RecordingDocker: + """A fake ``_run_docker`` that records calls and replays a fixed result seq. + + Each queued result is either a ``returncode`` int (wrapped in a + ``CompletedProcess``) or an exception instance to raise, modelling docker + pull failures/timeouts and successes. + """ + + def __init__(self, results): + self._results = list(results) + self.calls = [] + + def __call__(self, docker_args, timeout=None): + self.calls.append( + SimpleNamespace(docker_args=list(docker_args), timeout=timeout) + ) + result = self._results[len(self.calls) - 1] + if isinstance(result, BaseException): + raise result + return subprocess.CompletedProcess( + args=["docker", *docker_args], + returncode=result, + stdout=b"", + stderr=b"" if result == 0 else b"boom", + ) + + +def _make_args(rqts_image=None): + return SimpleNamespace(rqts_image=rqts_image) + + +# =========================================================================== +# Task 4.2 -> Property 1 +# =========================================================================== +# Feature: cfn-test-v2-flag, Property 1: Image resolution honors override, else pinned default +@settings(max_examples=100) +@given(override=st.one_of(st.none(), st.just(""), image_refs)) +def test_property_1_image_resolution_override_else_default(override): + """resolve_image returns the override when non-empty, else the pinned default. + + Validates: Requirements 2.1, 2.2 + """ + args = _make_args(rqts_image=override) + + resolved = image_module.resolve_image(args) + + if override: + assert resolved == override + else: + assert resolved == RQTS_IMAGE_REFERENCE + + +def test_resolve_image_missing_attribute_uses_default(): + """resolve_image tolerates args without an ``rqts_image`` attribute.""" + assert image_module.resolve_image(SimpleNamespace()) == RQTS_IMAGE_REFERENCE + + +# =========================================================================== +# Task 4.3 -> Property 2 +# =========================================================================== +def _expected_attempts(results): + """Model the bounded retry policy: attempt until first success, capped.""" + for index, result in enumerate(results[:PULL_MAX_ATTEMPTS]): + if result == 0: + return index + 1, True + return min(len(results), PULL_MAX_ATTEMPTS), False + + +@st.composite +def pull_result_sequences(draw): + """Generate a sequence of per-attempt pull outcomes. + + Each element is a ``0`` (success), a non-zero return code (failure), or an + exception instance (timeout / spawn error). The sequence is padded to at + least ``PULL_MAX_ATTEMPTS`` so ``ensure_image`` always has an outcome to + consume even when every attempt fails. + """ + outcome = st.one_of( + st.just(0), + st.integers(min_value=1, max_value=255), + st.just(subprocess.TimeoutExpired(cmd="docker pull", timeout=1)), + st.just(OSError("docker not found")), + ) + seq = draw( + st.lists(outcome, min_size=PULL_MAX_ATTEMPTS, max_size=PULL_MAX_ATTEMPTS + 4) + ) + return seq + + +# Feature: cfn-test-v2-flag, Property 2: Image pull is bounded and anonymous +@settings(max_examples=100) +@given(results=pull_result_sequences(), image_ref=image_refs, cached=st.booleans()) +def test_property_2_pull_is_bounded_and_anonymous(results, image_ref, cached): + """ensure_image pulls at most PULL_MAX_ATTEMPTS regardless of the local + cache state, stops on first success, never supplies AWS credentials to the + pull, falls back to a cached image on exhaustion, and raises only when no + cached copy exists. + + Validates: Requirements 2.3, 2.4, 2.5, 2.6 + """ + fake_docker = _RecordingDocker(results) + expected_attempts, should_succeed = _expected_attempts(results) + + # ``mock.patch`` context managers are used (rather than the monkeypatch + # fixture) so the patches are applied and reset for every generated example. + with mock.patch.object( + image_module, "image_present_locally", lambda ref: cached + ), mock.patch.object(image_module, "_run_docker", fake_docker): + if should_succeed or cached: + image_module.ensure_image(image_ref) + else: + with pytest.raises(SysExitRecommendedError): + image_module.ensure_image(image_ref) + + # Bounded: pull is attempted at most PULL_MAX_ATTEMPTS times, and exactly the + # number of attempts the retry policy predicts (stopping on first success). + assert len(fake_docker.calls) <= PULL_MAX_ATTEMPTS + assert len(fake_docker.calls) == expected_attempts + + for call in fake_docker.calls: + # Every invocation is an anonymous ``docker pull `` for this image. + assert call.docker_args[0] == "pull" + assert image_ref in call.docker_args + + # No AWS credential is ever threaded through: neither positionally in the + # docker args nor via any keyword. ensure_image only passes ``timeout``. + for token in call.docker_args: + for cred in _AWS_CRED_TOKENS: + assert cred not in token + + +# =========================================================================== +# Task 4.4 -> pull-exhaustion edge cases +# =========================================================================== +def test_pull_exhaustion_without_cached_image_raises(monkeypatch): + """Pull fails on all attempts with no cached image -> SysExitRecommendedError + after exactly PULL_MAX_ATTEMPTS attempts. + + Validates: Requirements 2.6 + """ + # Every attempt fails with a non-zero return code. + fake_docker = _RecordingDocker([1] * (PULL_MAX_ATTEMPTS + 2)) + monkeypatch.setattr(image_module, "image_present_locally", lambda ref: False) + monkeypatch.setattr(image_module, "_run_docker", fake_docker) + + with pytest.raises(SysExitRecommendedError): + image_module.ensure_image("some/image:tag") + + assert len(fake_docker.calls) == PULL_MAX_ATTEMPTS + + +def test_pull_exhaustion_with_cached_image_falls_back_with_warning(monkeypatch, caplog): + """Pull fails on all attempts but the image is cached locally -> warn and + run from the local store instead of raising. + + Validates: Requirements 2.5 + """ + fake_docker = _RecordingDocker([1] * PULL_MAX_ATTEMPTS) + monkeypatch.setattr(image_module, "image_present_locally", lambda ref: True) + monkeypatch.setattr(image_module, "_run_docker", fake_docker) + + with caplog.at_level("WARNING", logger=image_module.__name__): + image_module.ensure_image("cached/image:tag") + + # All attempts consumed, then fallback: no exception raised. + assert len(fake_docker.calls) == PULL_MAX_ATTEMPTS + assert any( + "cached" in record.getMessage() and "cached/image:tag" in record.getMessage() + for record in caplog.records + ) + + +def test_pull_exhaustion_on_repeated_timeouts(monkeypatch): + """Repeated per-attempt timeouts also exhaust after exactly PULL_MAX_ATTEMPTS.""" + timeouts = [ + subprocess.TimeoutExpired(cmd="docker pull", timeout=1) + for _ in range(PULL_MAX_ATTEMPTS + 1) + ] + fake_docker = _RecordingDocker(timeouts) + monkeypatch.setattr(image_module, "image_present_locally", lambda ref: False) + monkeypatch.setattr(image_module, "_run_docker", fake_docker) + + with pytest.raises(SysExitRecommendedError): + image_module.ensure_image("some/image:tag") + + assert len(fake_docker.calls) == PULL_MAX_ATTEMPTS + + +# =========================================================================== +# Task 4.5 -> pull-first behavior +# =========================================================================== +def test_ensure_image_present_locally_still_pulls(monkeypatch): + """Image present locally -> ensure_image STILL attempts the pull, so a moved + mutable tag (e.g. latest) is refreshed without manual docker pull. + + Validates: Requirements 2.3 + """ + fake_docker = _RecordingDocker([0]) + monkeypatch.setattr(image_module, "image_present_locally", lambda ref: True) + monkeypatch.setattr(image_module, "_run_docker", fake_docker) + + image_module.ensure_image("present/image:tag") + + # Exactly one successful pull attempt; presence never short-circuits it. + assert len(fake_docker.calls) == 1 + assert fake_docker.calls[0].docker_args[0] == "pull" + assert "present/image:tag" in fake_docker.calls[0].docker_args + + +def test_ensure_image_absent_pulls_then_returns(monkeypatch): + """Image absent -> ensure_image pulls, then returns on success. + + Validates: Requirements 2.3, 2.4 + """ + fake_docker = _RecordingDocker([0]) + monkeypatch.setattr(image_module, "image_present_locally", lambda ref: False) + monkeypatch.setattr(image_module, "_run_docker", fake_docker) + + image_module.ensure_image("absent/image:tag") + + assert len(fake_docker.calls) == 1 + assert fake_docker.calls[0].docker_args[0] == "pull" + assert "absent/image:tag" in fake_docker.calls[0].docker_args + + +def test_image_present_locally_uses_docker_image_inspect(monkeypatch): + """image_present_locally maps ``docker image inspect`` exit status to bool.""" + present = _RecordingDocker([0]) + monkeypatch.setattr(image_module, "_run_docker", present) + assert image_module.image_present_locally("ref:tag") is True + assert present.calls[0].docker_args == ["image", "inspect", "ref:tag"] + + absent = _RecordingDocker([1]) + monkeypatch.setattr(image_module, "_run_docker", absent) + assert image_module.image_present_locally("ref:tag") is False diff --git a/tests/rqts/test_preconditions.py b/tests/rqts/test_preconditions.py new file mode 100644 index 00000000..9900fd71 --- /dev/null +++ b/tests/rqts/test_preconditions.py @@ -0,0 +1,189 @@ +"""Tests for ``rpdk.core.rqts.preconditions``. + +Covers: +- Property 3: precondition failures are aggregated exactly (over the three + DirectJar checks: Docker runtime, artifact package, credentials+region). +- Each precondition failing in isolation produces its own message and prevents + any container run. + +The DirectJar handler mode has no SAM Local endpoint to probe and reads inputs +from the packaged artifact zip, so there is no endpoint or inputs check. + +Every check is controlled independently by patching within +``rpdk.core.rqts.preconditions`` (``shutil.which`` + ``subprocess.run`` for +Docker, ``create_sdk_session`` for credentials) and by toggling the artifact +package file under a ``tmp_path`` working directory. +""" + +import contextlib +from unittest import mock + +from hypothesis import HealthCheck, given, settings, strategies as st + +from rpdk.core.exceptions import CLIMisconfiguredError +from rpdk.core.rqts.preconditions import check_preconditions + +PRECONDITIONS_MODULE = "rpdk.core.rqts.preconditions" + +# The three checks, in the order check_preconditions runs them, keyed to the +# distinctive substring of the failure message each one emits. +CHECK_NAMES = ("docker", "artifact", "credentials") +MESSAGE_SUBSTRINGS = { + "docker": "Docker is required", + "artifact": "artifact package", + "credentials": "valid AWS credentials and a region", +} + + +class FakeProject: + """Minimal stand-in for ``rpdk.core.project.Project``. + + Only the attributes the precondition checks read are provided: ``root`` and + ``hypenated_name``. + """ + + def __init__(self, root): + self.root = root + self.hypenated_name = "aws-foo-bar" + + +def _make_args(): + args = mock.Mock() + args.region = "us-east-1" + args.profile = None + return args + + +@contextlib.contextmanager +def configured_env(work_dir, states): + """Force each precondition to pass/fail per ``states``. + + ``states`` maps check name -> bool, where ``True`` means the check should + PASS and ``False`` means it should FAIL. Yields ``(args, project)`` wired so + that ``check_preconditions`` observes exactly those outcomes. + """ + project = FakeProject(work_dir) + + # Artifact package presence (real filesystem toggle). + artifact_path = work_dir / f"{project.hypenated_name}.zip" + if states["artifact"]: + artifact_path.write_bytes(b"zip") + elif artifact_path.exists(): + artifact_path.unlink() + + # Docker: which() present + `docker info` returncode 0 => pass. + which_return = "/usr/bin/docker" if states["docker"] else None + docker_info_result = mock.Mock(returncode=0 if states["docker"] else 1) + + # Credentials: create_sdk_session succeeds => pass, raises => fail. + session_side_effect = ( + None if states["credentials"] else CLIMisconfiguredError("no creds") + ) + + with contextlib.ExitStack() as stack: + stack.enter_context( + mock.patch( + f"{PRECONDITIONS_MODULE}.shutil.which", return_value=which_return + ) + ) + stack.enter_context( + mock.patch( + f"{PRECONDITIONS_MODULE}.subprocess.run", + return_value=docker_info_result, + ) + ) + stack.enter_context( + mock.patch( + f"{PRECONDITIONS_MODULE}.create_sdk_session", + return_value=mock.Mock(), + side_effect=session_side_effect, + ) + ) + yield _make_args(), project + + +def _matched_checks(failures): + """Return the set of check names whose message substring appears in ``failures``.""" + matched = set() + for name, substring in MESSAGE_SUBSTRINGS.items(): + if any(substring in message for message in failures): + matched.add(name) + return matched + + +# Feature: cfn-test-v2-flag, Property 3: For any subset of the precondition +# checks (Docker runtime, artifact package, credentials+region) forced to fail, +# check_preconditions returns a failure list whose messages correspond to +# exactly that subset - every unmet precondition is named and every met +# precondition is absent. +@settings(max_examples=200, suppress_health_check=[HealthCheck.function_scoped_fixture]) +@given(pass_flags=st.fixed_dictionaries({name: st.booleans() for name in CHECK_NAMES})) +def test_precondition_failures_aggregated_exactly(tmp_path, pass_flags): + """Validates: Requirements 3.7""" + expected_failures = {name for name, passed in pass_flags.items() if not passed} + + with configured_env(tmp_path, pass_flags) as (args, project): + failures = check_preconditions(args, project) + + # Every unmet precondition is named exactly once, and no met precondition is. + assert _matched_checks(failures) == expected_failures + assert len(failures) == len(expected_failures) + + +# --------------------------------------------------------------------------- +# Each precondition failing in isolation. +# --------------------------------------------------------------------------- + +ALL_PASS = dict.fromkeys(CHECK_NAMES, True) + + +def _states_with_only_failing(check): + states = dict(ALL_PASS) + states[check] = False + return states + + +def test_all_preconditions_met_returns_empty(tmp_path): + """Sanity baseline: when every check passes, no failures are returned.""" + with configured_env(tmp_path, dict(ALL_PASS)) as (args, project): + assert not check_preconditions(args, project) + + +def test_docker_unavailable_in_isolation(tmp_path): + """Requirement 3.2: Docker unavailable yields its message and blocks the run.""" + with configured_env(tmp_path, _states_with_only_failing("docker")) as ( + args, + project, + ): + failures = check_preconditions(args, project) + + assert len(failures) == 1 + assert MESSAGE_SUBSTRINGS["docker"] in failures[0] + # A non-empty failure list prevents the caller from ever running a container. + assert failures + + +def test_artifact_missing_in_isolation(tmp_path): + """Requirement 3.3: missing artifact package yields its message and blocks the run.""" + with configured_env(tmp_path, _states_with_only_failing("artifact")) as ( + args, + project, + ): + failures = check_preconditions(args, project) + + assert len(failures) == 1 + assert MESSAGE_SUBSTRINGS["artifact"] in failures[0] + assert failures + + +def test_credentials_unavailable_in_isolation(tmp_path): + """Requirement 3.5: missing credentials/region yields the message and blocks the run.""" + with configured_env(tmp_path, _states_with_only_failing("credentials")) as ( + args, + project, + ): + failures = check_preconditions(args, project) + + assert len(failures) == 1 + assert MESSAGE_SUBSTRINGS["credentials"] in failures[0] + assert failures diff --git a/tests/rqts/test_runner.py b/tests/rqts/test_runner.py new file mode 100644 index 00000000..36b02b77 --- /dev/null +++ b/tests/rqts/test_runner.py @@ -0,0 +1,371 @@ +"""Tests for ``rpdk.core.rqts.runner``. + +Covers: +- Property 12: exit-code mapping (``map_exit_code``) over arbitrary exit codes. +- Orchestration guards and container spawn failure edge cases + (``run_container`` spawn failure, ``RqtsRunner._guard_artifact_type`` for hook + and indeterminate artifact types). +- Live output streaming integration for ``run_container`` using a fake child + process (no real Docker). +- Result reporting (``report_result``) pass/fail summaries and the + ``RqtsRunner.run`` DEBUG log of the full ``docker run`` command line. + +Docker and AWS are never actually invoked: ``subprocess.Popen`` is patched with +a deterministic fake, and the sibling functions imported into ``runner`` are +patched at ``rpdk.core.rqts.runner`` so the happy path exercises the +orchestration wiring without any external calls. + +Library: Hypothesis (the standard Python property-based testing library). The +property test runs at least 100 generated examples via +``@settings(max_examples=100)`` and is tagged with a comment referencing the +design property it validates. +""" +import logging +from types import SimpleNamespace +from unittest import mock + +import pytest +from hypothesis import given, settings, strategies as st + +from rpdk.core.exceptions import SysExitRecommendedError +from rpdk.core.project import ARTIFACT_TYPE_HOOK, ARTIFACT_TYPE_RESOURCE +from rpdk.core.rqts.runner import ( + FAIL_SUMMARY, + PASS_SUMMARY, + RqtsRunner, + map_exit_code, + report_result, + run_container, +) + +RUNNER_MODULE = "rpdk.core.rqts.runner" + + +# --------------------------------------------------------------------------- +# Fakes +# --------------------------------------------------------------------------- + + +class FakePopen: + """Deterministic stand-in for ``subprocess.Popen``. + + Supports the context-manager protocol used by ``run_container`` and records + that ``wait()`` was called before the return code is surfaced, so the test + can assert the child was awaited (i.e. streamed to completion) rather than + abandoned. Optionally emits incremental "live" output lines to a sink to + model streaming. + """ + + def __init__(self, argv, return_code=0, output_lines=None, output_sink=None): + self.argv = argv + self._return_code = return_code + self._output_lines = list(output_lines or []) + self._output_sink = output_sink + self.waited = False + self.entered = False + + def __enter__(self): + self.entered = True + return self + + def __exit__(self, *_exc): + return False + + def wait(self): + # Model live streaming: output is surfaced as the process runs, before + # wait() returns the exit code. + if self._output_sink is not None: + for line in self._output_lines: + self._output_sink.append(line) + self.waited = True + return self._return_code + + +def _make_resource_project(root): + """Build a minimal resource project the happy-path runner can drive.""" + return SimpleNamespace( + artifact_type=ARTIFACT_TYPE_RESOURCE, + type_name="AWS::Foo::Bar", + hypenated_name="aws-foo-bar", + root=root, + ) + + +def _make_args(): + return SimpleNamespace( + region="us-east-1", + profile=None, + role_arn=None, + source_account=None, + source_arn=None, + rqts_image=None, + ) + + +# =========================================================================== +# Property 12: exit-code mapping +# =========================================================================== +# Feature: cfn-test-v2-flag, Property 12: Exit code mapping +@settings(max_examples=100) +@given( + code=st.one_of( + st.just(0), + st.integers(min_value=1, max_value=255), + st.integers(min_value=-255, max_value=-1), + st.integers(), + ) +) +def test_property_12_exit_code_mapping(code): + """map_exit_code(0) returns None without raising; any non-zero code raises + SysExitRecommendedError. + + Validates: Requirements 5.2, 5.3, 6.3 + """ + if code == 0: + assert map_exit_code(code) is None + else: + with pytest.raises(SysExitRecommendedError): + map_exit_code(code) + + +# =========================================================================== +# Orchestration guards and spawn failure +# =========================================================================== +@pytest.mark.parametrize( + "spawn_error", [FileNotFoundError("no docker"), OSError("boom")] +) +def test_run_container_spawn_failure_raises(spawn_error): + """Container spawn failure -> SysExitRecommendedError naming the start failure. + + Validates: Requirements 5.4 + """ + with mock.patch(f"{RUNNER_MODULE}.subprocess.Popen", side_effect=spawn_error): + with pytest.raises(SysExitRecommendedError) as excinfo: + run_container(["docker", "run", "--rm", "image"]) + + assert "could not be started" in str(excinfo.value) + + +def test_guard_artifact_type_hook_rejected_and_does_not_proceed(): + """Hook project -> SysExitRecommendedError 'resources only'; pipeline halts. + + Validates: Requirements 7.2 + """ + project = SimpleNamespace(artifact_type=ARTIFACT_TYPE_HOOK) + rqts_runner = RqtsRunner(_make_args(), project) + + with mock.patch(f"{RUNNER_MODULE}.check_preconditions") as check, mock.patch( + f"{RUNNER_MODULE}.run_container" + ) as run: + with pytest.raises(SysExitRecommendedError) as excinfo: + rqts_runner.run() + + assert "resource types only" in str(excinfo.value) + # The guard fails fast: no downstream stage runs. + check.assert_not_called() + run.assert_not_called() + + +def test_guard_artifact_type_indeterminate_rejected_and_does_not_proceed(): + """Indeterminate artifact type -> 'could not determine' error; pipeline halts. + + Validates: Requirements 7.5 + """ + project = SimpleNamespace(artifact_type=None) + rqts_runner = RqtsRunner(_make_args(), project) + + with mock.patch(f"{RUNNER_MODULE}.check_preconditions") as check, mock.patch( + f"{RUNNER_MODULE}.run_container" + ) as run: + with pytest.raises(SysExitRecommendedError) as excinfo: + rqts_runner.run() + + assert "could not determine the project artifact type" in str(excinfo.value) + check.assert_not_called() + run.assert_not_called() + + +# =========================================================================== +# Live output streaming integration +# =========================================================================== +def test_run_container_streams_output_and_returns_code(): + """run_container surfaces incremental child output and returns the exit code. + + A fake child process emits incremental lines as it runs (before wait() + returns), modelling the inherited-stdio live streaming; run_container + returns the child's exit code. + + Validates: Requirements 5.1 + """ + argv = [ + "docker", + "run", + "--rm", + "image:tag", + "--extension", + "contract-tests", + "run-tests", + "/work/aws-foo-bar.zip", + "--direct-jar", + ] + streamed = [] + process = FakePopen( + argv, + return_code=0, + output_lines=["scenario create: PASS", "scenario delete: PASS"], + output_sink=streamed, + ) + + def factory(passed_argv, *_args, **_kwargs): + # run_container must spawn docker with exactly the argv it was given. + assert passed_argv == argv + return process + + with mock.patch(f"{RUNNER_MODULE}.subprocess.Popen", side_effect=factory): + code = run_container(argv) + + assert code == 0 + # Output was surfaced live (during the run) and the child was awaited. + assert streamed == ["scenario create: PASS", "scenario delete: PASS"] + assert process.waited is True + + +def test_run_container_returns_nonzero_code(): + """run_container returns a non-zero child exit code unchanged (no raise).""" + argv = ["docker", "run", "--rm", "image:tag"] + + with mock.patch( + f"{RUNNER_MODULE}.subprocess.Popen", + side_effect=lambda passed_argv, *a, **k: FakePopen(passed_argv, return_code=7), + ): + assert run_container(argv) == 7 + + +def test_run_container_merges_env_over_ambient_environment(monkeypatch): + """run_container spawns docker with the supplied env merged over os.environ, + and inherits the ambient environment untouched (env=None) when none is given. + + This is the delivery half of the name-only ``-e`` contract: credential + values reach docker exclusively through the process environment. + """ + argv = ["docker", "run", "--rm", "image:tag"] + captured = {} + + def factory(passed_argv, *_args, **kwargs): + captured["env"] = kwargs.get("env") + return FakePopen(passed_argv, return_code=0) + + monkeypatch.setenv("SOME_AMBIENT_VAR", "ambient") + + with mock.patch(f"{RUNNER_MODULE}.subprocess.Popen", side_effect=factory): + run_container(argv, env={"AWS_ACCESS_KEY_ID": "AKID"}) + assert captured["env"]["AWS_ACCESS_KEY_ID"] == "AKID" + assert captured["env"]["SOME_AMBIENT_VAR"] == "ambient" + + with mock.patch(f"{RUNNER_MODULE}.subprocess.Popen", side_effect=factory): + run_container(argv) + assert captured["env"] is None + + +# =========================================================================== +# Result reporting and DEBUG logging +# =========================================================================== +def test_report_result_pass_logs_summary_and_does_not_raise(caplog): + """report_result(0) logs PASS_SUMMARY at INFO and does not raise. + + Validates: Requirements 6.1 + """ + with caplog.at_level(logging.INFO, logger=RUNNER_MODULE): + assert report_result(0) is None + + assert PASS_SUMMARY in caplog.text + + +def test_report_result_fail_raises_with_fail_summary(): + """report_result(non-zero) raises SysExitRecommendedError with FAIL_SUMMARY. + + Validates: Requirements 6.2 + """ + with pytest.raises(SysExitRecommendedError) as excinfo: + report_result(1) + + assert str(excinfo.value) == FAIL_SUMMARY + + +def test_run_logs_full_docker_command_at_debug_on_happy_path(tmp_path, caplog): + """RqtsRunner.run logs the full docker run command at DEBUG and passes. + + A fully-mocked happy path (no preconditions failures, stubbed credentials, + image resolve/ensure, and a zero container exit code) drives the runner and + asserts the full docker command line is emitted at DEBUG and the run reports + a pass. It also asserts check_preconditions is called with (args, project). + + Validates: Requirements 4.10, 6.1 + """ + project = _make_resource_project(str(tmp_path)) + args = _make_args() + rqts_runner = RqtsRunner(args, project) + docker_argv = [ + "docker", + "run", + "--rm", + "image:tag", + "--extension", + "contract-tests", + "run-tests", + "/work/aws-foo-bar.zip", + "--direct-jar", + ] + + creds = { + "aws_access_key_id": "AKID", + "aws_secret_access_key": "SECRET", + "aws_session_token": "TOKEN", + } + + with mock.patch( + f"{RUNNER_MODULE}.check_preconditions", return_value=[] + ) as check, mock.patch(f"{RUNNER_MODULE}.create_sdk_session"), mock.patch( + f"{RUNNER_MODULE}.get_temporary_credentials", return_value=creds + ), mock.patch( + f"{RUNNER_MODULE}.resolve_image", return_value="image:tag" + ), mock.patch( + f"{RUNNER_MODULE}.ensure_image" + ), mock.patch( + f"{RUNNER_MODULE}.build_docker_argv", return_value=docker_argv + ), mock.patch( + f"{RUNNER_MODULE}.run_container", return_value=0 + ) as run: + with caplog.at_level(logging.DEBUG, logger=RUNNER_MODULE): + # Returns normally (pass) on a zero container exit code. + assert rqts_runner.run() is None + + # check_preconditions is called with exactly (args, project) - no host arg. + check.assert_called_once_with(args, project) + # run_container receives the credential-free argv plus the env mapping that + # carries the credential values (never present in the argv itself). + run.assert_called_once_with( + docker_argv, + env={ + "AWS_ACCESS_KEY_ID": "AKID", + "AWS_SECRET_ACCESS_KEY": "SECRET", + "AWS_SESSION_TOKEN": "TOKEN", + "AWS_REGION": args.region, + }, + ) + # The DEBUG-logged command line contains no credential values. + for record in caplog.records: + for secret in creds.values(): + assert secret not in record.getMessage() + # The host-side output directory was created under the project root. + assert (tmp_path / "rqts-output").is_dir() + # The full docker run command line is logged at DEBUG (Req 4.10). + debug_records = [ + record.getMessage() + for record in caplog.records + if record.levelno == logging.DEBUG + ] + joined_command = " ".join(docker_argv) + assert any(joined_command in message for message in debug_records) + # The pass summary is surfaced (Req 6.1). + assert PASS_SUMMARY in caplog.text diff --git a/tests/test_test.py b/tests/test_test.py index 031feb83..be1ea927 100644 --- a/tests/test_test.py +++ b/tests/test_test.py @@ -1,5 +1,6 @@ # fixture and parameter have the same name -# pylint: disable=protected-access,redefined-outer-name +# pylint: disable=protected-access,redefined-outer-name,too-many-lines +import argparse import json import os from contextlib import contextmanager @@ -31,6 +32,7 @@ get_marker_options, get_overrides, get_type, + setup_subparser, temporary_ini_file, ) from rpdk.core.utils.handler_utils import generate_handler_name @@ -838,3 +840,193 @@ def test_input_files_read_safely(self, base): result = get_inputs(base, DEFAULT_REGION, DEFAULT_ENDPOINT, 1, None, None, {}) assert result is not None and "CREATE" in result + + +# --v2 flag registration and RQTS routing tests +# (Task 8.3: flag registration; Task 8.4: routing + backward compatibility) + + +def _build_test_parser(): + """Build a parser wired up exactly the way cli.py does for the test command. + + Mirrors rpdk.core.cli.main: a root parser with a shared base subparser + (providing -v/--verbose) as the only parent, and the test subcommand + registered via setup_subparser. + """ + parser = argparse.ArgumentParser() + base_subparser = argparse.ArgumentParser(add_help=False) + base_subparser.add_argument("-v", "--verbose", action="count", default=0) + subparsers = parser.add_subparsers(dest="subparser_name") + setup_subparser(subparsers, [base_subparser]) + return parser, subparsers + + +# Task 8.3 - Requirements 1.1, 1.5, 4.11 + + +def test_v2_flag_defaults_to_false(): + parser, _subparsers = _build_test_parser() + args = parser.parse_args(["test"]) + assert args.v2 is False + + +def test_v2_flag_present_sets_true(): + parser, _subparsers = _build_test_parser() + args = parser.parse_args(["test", "--v2"]) + assert args.v2 is True + + +def test_rqts_image_defaults_to_none(): + parser, _subparsers = _build_test_parser() + args = parser.parse_args(["test"]) + assert args.rqts_image is None + + +def test_rqts_image_can_be_overridden(): + parser, _subparsers = _build_test_parser() + args = parser.parse_args(["test", "--rqts-image", "foo:bar"]) + assert args.rqts_image == "foo:bar" + + +def test_v2_help_text_identifies_opt_in_rqts_runner(): + _parser, subparsers = _build_test_parser() + help_text = subparsers.choices["test"].format_help() + lowered = help_text.lower() + assert "opt-in" in lowered + assert "rqts local test runner" in lowered + + +def test_no_scenario_selection_option_on_parser(): + parser, _subparsers = _build_test_parser() + # Scenario selection is owned by the executor image and is not + # user-selectable through the CLI (Req 4.11); a scenario-selection option + # is not a recognized argument. + with pytest.raises(SystemExit): + parser.parse_args(["test", "--scenarios", "x"]) + + +# Task 8.4 - Requirements 1.2, 1.3, 1.4, 7.1, 7.3, 7.4 + + +def test_v2_absent_uses_pytest_path_and_does_not_construct_runner(base): + create_input_file(base, '{"a": 1}', '{"a": 2}', '{"b": 1}') + mock_project = Mock(spec=Project) + mock_project.schema = RESOURCE_SCHEMA + mock_project.root = base + mock_project.executable_entrypoint = None + mock_project.artifact_type = ARTIFACT_TYPE_RESOURCE + + patch_project = patch( + "rpdk.core.test.Project", autospec=True, return_value=mock_project + ) + patch_plugin = patch("rpdk.core.test.ContractPlugin", autospec=True) + patch_resource_client = patch("rpdk.core.test.ResourceClient", autospec=True) + patch_pytest = patch("rpdk.core.test.pytest.main", autospec=True, return_value=0) + patch_ini = patch( + "rpdk.core.test.temporary_ini_file", side_effect=mock_temporary_ini_file + ) + patch_runner = patch("rpdk.core.rqts.runner.RqtsRunner", autospec=True) + # fmt: off + with patch_project, \ + patch_plugin, \ + patch_resource_client, \ + patch_ini, \ + patch_pytest as mock_pytest, \ + patch_runner as mock_runner: + main(args_in=["test"]) + # fmt: on + + # The pytest path is exercised and the RQTS runner is never constructed. + mock_pytest.assert_called_once() + mock_runner.assert_not_called() + + +def test_v2_absent_nonzero_pytest_return_still_raises(base): + create_input_file(base, '{"a": 1}', '{"a": 2}', '{"b": 1}') + mock_project = Mock(spec=Project) + mock_project.schema = RESOURCE_SCHEMA + mock_project.root = base + mock_project.executable_entrypoint = None + mock_project.artifact_type = ARTIFACT_TYPE_RESOURCE + + patch_project = patch( + "rpdk.core.test.Project", autospec=True, return_value=mock_project + ) + patch_plugin = patch("rpdk.core.test.ContractPlugin", autospec=True) + patch_resource_client = patch("rpdk.core.test.ResourceClient", autospec=True) + patch_pytest = patch("rpdk.core.test.pytest.main", autospec=True, return_value=1) + patch_ini = patch( + "rpdk.core.test.temporary_ini_file", side_effect=mock_temporary_ini_file + ) + patch_runner = patch("rpdk.core.rqts.runner.RqtsRunner", autospec=True) + # fmt: off + with patch_project, \ + patch_plugin, \ + patch_resource_client, \ + patch_ini, \ + patch_pytest as mock_pytest, \ + patch_runner as mock_runner: + with pytest.raises(SystemExit) as excinfo: + main(args_in=["test"]) + # fmt: on + + # Existing backward-compatible behavior: a non-zero pytest return raises + # SysExitRecommendedError, mapped by cli.py to a non-unhandled SystemExit. + assert excinfo.value.code != EXIT_UNHANDLED_EXCEPTION + # The --v2 branch did not alter forwarding to the pytest path. + mock_pytest.assert_called_once() + mock_runner.assert_not_called() + + +def test_v2_resource_project_constructs_and_runs_runner(base): + mock_project = Mock(spec=Project) + mock_project.schema = RESOURCE_SCHEMA + mock_project.root = base + mock_project.executable_entrypoint = None + mock_project.artifact_type = ARTIFACT_TYPE_RESOURCE + + patch_project = patch( + "rpdk.core.test.Project", autospec=True, return_value=mock_project + ) + patch_pytest = patch("rpdk.core.test.pytest.main", autospec=True, return_value=0) + # test() imports RqtsRunner locally (from .rqts.runner import RqtsRunner), + # so patch it at its source module. + patch_runner = patch("rpdk.core.rqts.runner.RqtsRunner", autospec=True) + # fmt: off + with patch_project, \ + patch_pytest as mock_pytest, \ + patch_runner as mock_runner: + main(args_in=["test", "--v2"]) + # fmt: on + + # The runner is constructed with the parsed args and loaded project, and + # run() is invoked exactly once; the pytest path is not exercised. + mock_runner.assert_called_once() + called_args = mock_runner.call_args[0] + assert called_args[1] is mock_project + mock_runner.return_value.run.assert_called_once_with() + mock_pytest.assert_not_called() + + +def test_v2_module_project_warns_and_does_not_construct_runner(capsys): + mock_project = Mock(spec=Project) + mock_project.artifact_type = ARTIFACT_TYPE_MODULE + + patch_project = patch( + "rpdk.core.test.Project", autospec=True, return_value=mock_project + ) + patch_pytest = patch("rpdk.core.test.pytest.main", autospec=True, return_value=0) + patch_runner = patch("rpdk.core.rqts.runner.RqtsRunner", autospec=True) + # fmt: off + with patch_project, \ + patch_pytest as mock_pytest, \ + patch_runner as mock_runner: + # The module short-circuit precedes the --v2 branch: clean return, no + # SystemExit raised. + main(args_in=["test", "--v2"]) + # fmt: on + + mock_runner.assert_not_called() + mock_pytest.assert_not_called() + out, err = capsys.readouterr() + assert "module" in (out + err).lower() From a663063dc7c543edce613e43e88b4286613f6440 Mon Sep 17 00:00:00 2001 From: Ruslan Temirkhanov Date: Mon, 24 Aug 2026 18:35:12 -0400 Subject: [PATCH 2/3] chore(rqts): Drop unused handler port constant DEFAULT_HANDLER_PORT described the SAM Local handler endpoint, which the DirectJar mode targeted by cfn test --v2 never uses: the handler JAR is loaded in-process, so no port is bound and nothing referenced the value. --- src/rpdk/core/rqts/constants.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/rpdk/core/rqts/constants.py b/src/rpdk/core/rqts/constants.py index e0e9ca2e..628cd5f3 100644 --- a/src/rpdk/core/rqts/constants.py +++ b/src/rpdk/core/rqts/constants.py @@ -29,12 +29,7 @@ # top-level ``--extension`` option BEFORE the ``run-tests`` subcommand. EXTENSION_CONTRACT_TESTS = "contract-tests" -# SAM Local handler port. Retained for reference; unused by the DirectJar mode -# that ``cfn test --v2`` targets (DirectJar loads the handler in-process and -# needs no handler endpoint). -DEFAULT_HANDLER_PORT = 3031 - -# Image pull retry policy (Req 2.4). +# Image pull retry policy. PULL_MAX_ATTEMPTS = 3 PULL_ATTEMPT_TIMEOUT_SECONDS = 120 From 52dd7dc0c747bb91715f8b49b199e166d8378531 Mon Sep 17 00:00:00 2001 From: Ruslan Temirkhanov Date: Mon, 24 Aug 2026 18:35:12 -0400 Subject: [PATCH 3/3] test(rqts): Cover remaining RQTS error paths Bring the rqts package to full statement and branch coverage by exercising the paths the existing tests bypassed: - _run_docker reaching subprocess.run with a fixed, non-shell argv - image_present_locally when docker inspect cannot run at all - the Docker daemon ping exiting non-zero, raising, or timing out - RqtsRunner.run aggregating unmet preconditions into one error - build_docker_argv with explicit workdir, artifact and output overrides --- tests/rqts/test_argv.py | 30 +++++++++++++ tests/rqts/test_image.py | 45 ++++++++++++++++++++ tests/rqts/test_preconditions.py | 73 ++++++++++++++++++++++++++++++++ tests/rqts/test_runner.py | 36 ++++++++++++++++ 4 files changed, 184 insertions(+) diff --git a/tests/rqts/test_argv.py b/tests/rqts/test_argv.py index 54626500..1773b062 100644 --- a/tests/rqts/test_argv.py +++ b/tests/rqts/test_argv.py @@ -371,3 +371,33 @@ def test_defaults_use_project_root_and_hyphenated_name(): # No scenario selection: the argv ends at the output directory. assert "-s" not in argv assert argv[-2:] == ["-o", CONTAINER_OUTPUT_DIR] + + +# =========================================================================== +# Explicit overrides of the project-derived defaults +# =========================================================================== +def test_explicit_workdir_artifact_and_output_dir_override_defaults(): + """Supplied workdir/artifact_name/output_dir replace the project defaults. + + The defaults are derived from the project (``root`` and + ``hypenated_name``); when a caller passes them explicitly, none of the + project-derived values appear in the argv. + """ + project = _make_project("aws-foo-bar", root="/project/root") + + argv = build_docker_argv( + "img:ref", + project, + "us-west-2", + workdir="/elsewhere", + artifact_name="custom-artifact.zip", + output_dir="/work/custom-output", + ) + + assert f"/elsewhere:{CONTAINER_WORKDIR}" in argv + assert f"{CONTAINER_WORKDIR}/custom-artifact.zip" in argv + assert argv[-2:] == ["-o", "/work/custom-output"] + # None of the project-derived defaults leak in. + assert f"/project/root:{CONTAINER_WORKDIR}" not in argv + assert f"{CONTAINER_WORKDIR}/aws-foo-bar.zip" not in argv + assert CONTAINER_OUTPUT_DIR not in argv diff --git a/tests/rqts/test_image.py b/tests/rqts/test_image.py index 6dd099a4..8861e564 100644 --- a/tests/rqts/test_image.py +++ b/tests/rqts/test_image.py @@ -286,3 +286,48 @@ def test_image_present_locally_uses_docker_image_inspect(monkeypatch): absent = _RecordingDocker([1]) monkeypatch.setattr(image_module, "_run_docker", absent) assert image_module.image_present_locally("ref:tag") is False + + +# =========================================================================== +# Internal docker seam and inspect failure handling +# =========================================================================== +def test_run_docker_invokes_docker_cli_with_fixed_argv(): + """_run_docker drives the docker CLI with a fixed, non-shell argv. + + This is the one place that actually reaches ``subprocess.run``; it is + patched here so no docker process is spawned. A fixed argv list (never a + shell string) is what makes the image reference safe to pass through. + """ + completed = subprocess.CompletedProcess(args=["docker", "info"], returncode=0) + + with mock.patch.object( + image_module.subprocess, "run", return_value=completed + ) as run: + result = image_module._run_docker( # pylint: disable=protected-access + ["image", "inspect", "ref:tag"], timeout=7 + ) + + assert result is completed + run.assert_called_once() + assert run.call_args[0][0] == ["docker", "image", "inspect", "ref:tag"] + assert run.call_args[1]["check"] is False + assert run.call_args[1]["capture_output"] is True + assert run.call_args[1]["timeout"] == 7 + + +@pytest.mark.parametrize( + "docker_error", [OSError("docker not found"), subprocess.SubprocessError("boom")] +) +def test_image_present_locally_false_when_inspect_cannot_run(monkeypatch, docker_error): + """An inspect that cannot even run is treated as 'not present locally'. + + Keeps ``ensure_image``'s fallback decision safe when the docker CLI is + missing or unusable: absent, rather than assumed cached. + """ + + def raise_error(_docker_args, timeout=None): # pylint: disable=unused-argument + raise docker_error + + monkeypatch.setattr(image_module, "_run_docker", raise_error) + + assert image_module.image_present_locally("ref:tag") is False diff --git a/tests/rqts/test_preconditions.py b/tests/rqts/test_preconditions.py index 9900fd71..069d3996 100644 --- a/tests/rqts/test_preconditions.py +++ b/tests/rqts/test_preconditions.py @@ -16,8 +16,10 @@ """ import contextlib +import subprocess from unittest import mock +import pytest from hypothesis import HealthCheck, given, settings, strategies as st from rpdk.core.exceptions import CLIMisconfiguredError @@ -187,3 +189,74 @@ def test_credentials_unavailable_in_isolation(tmp_path): assert len(failures) == 1 assert MESSAGE_SUBSTRINGS["credentials"] in failures[0] assert failures + + +# --------------------------------------------------------------------------- +# Docker daemon ping failure modes (docker CLI present, daemon unreachable). +# --------------------------------------------------------------------------- + + +@contextlib.contextmanager +def docker_ping_env(work_dir, **run_kwargs): + """Yield ``(args, project)`` with docker on PATH and ``docker info`` stubbed. + + The artifact and credential checks are forced to pass, so any failure the + caller observes comes from the Docker daemon ping alone. ``run_kwargs`` is + forwarded to ``mock.patch`` for ``subprocess.run`` (``return_value`` for an + exit status, ``side_effect`` to raise). + """ + project = FakeProject(work_dir) + (work_dir / f"{project.hypenated_name}.zip").write_bytes(b"zip") + + with contextlib.ExitStack() as stack: + stack.enter_context( + mock.patch( + f"{PRECONDITIONS_MODULE}.shutil.which", return_value="/usr/bin/docker" + ) + ) + stack.enter_context( + mock.patch(f"{PRECONDITIONS_MODULE}.subprocess.run", **run_kwargs) + ) + stack.enter_context( + mock.patch( + f"{PRECONDITIONS_MODULE}.create_sdk_session", return_value=mock.Mock() + ) + ) + yield _make_args(), project + + +def test_docker_daemon_ping_nonzero_exit_reports_unreachable(tmp_path): + """docker CLI present but ``docker info`` exits non-zero -> unreachable daemon. + + Distinct from the missing-CLI case: the binary exists, so the ping itself is + what fails. + + Validates: Requirements 3.2 + """ + with docker_ping_env(tmp_path, return_value=mock.Mock(returncode=1)) as ( + args, + project, + ): + failures = check_preconditions(args, project) + + assert len(failures) == 1 + assert MESSAGE_SUBSTRINGS["docker"] in failures[0] + + +@pytest.mark.parametrize( + "ping_error", + [OSError("cannot exec"), subprocess.TimeoutExpired(cmd="docker info", timeout=10)], +) +def test_docker_daemon_ping_error_reports_unreachable(tmp_path, ping_error): + """A ping that raises (spawn failure or timeout) -> unreachable daemon. + + The check converts the exception into a message rather than propagating it, + so a hung or broken daemon still aggregates with other failures. + + Validates: Requirements 3.2 + """ + with docker_ping_env(tmp_path, side_effect=ping_error) as (args, project): + failures = check_preconditions(args, project) + + assert len(failures) == 1 + assert MESSAGE_SUBSTRINGS["docker"] in failures[0] diff --git a/tests/rqts/test_runner.py b/tests/rqts/test_runner.py index 36b02b77..17605ddd 100644 --- a/tests/rqts/test_runner.py +++ b/tests/rqts/test_runner.py @@ -369,3 +369,39 @@ def test_run_logs_full_docker_command_at_debug_on_happy_path(tmp_path, caplog): assert any(joined_command in message for message in debug_records) # The pass summary is surfaced (Req 6.1). assert PASS_SUMMARY in caplog.text + + +# =========================================================================== +# Precondition enforcement inside the pipeline +# =========================================================================== +def test_run_unmet_preconditions_aggregated_and_halts(tmp_path): + """Unmet preconditions -> a single error naming every failure; nothing runs. + + The runner turns the aggregated list from ``check_preconditions`` into one + ``SysExitRecommendedError`` instead of failing on the first problem, and + halts before the image is ensured or any container is started. + + Validates: Requirements 3.1, 3.5 + """ + project = _make_resource_project(str(tmp_path)) + rqts_runner = RqtsRunner(_make_args(), project) + failures = [ + "Docker is required and must be running: the Docker daemon could not " + "be reached.", + "artifact package 'aws-foo-bar.zip' not found; build the project first.", + ] + + with mock.patch( + f"{RUNNER_MODULE}.check_preconditions", return_value=failures + ), mock.patch(f"{RUNNER_MODULE}.ensure_image") as ensure, mock.patch( + f"{RUNNER_MODULE}.run_container" + ) as run: + with pytest.raises(SysExitRecommendedError) as excinfo: + rqts_runner.run() + + message = str(excinfo.value) + assert "preconditions were not met" in message + for failure in failures: + assert failure in message + ensure.assert_not_called() + run.assert_not_called()