diff --git a/src/rpdk/core/rqts/__init__.py b/src/rpdk/core/rqts/__init__.py new file mode 100644 index 00000000..3e07489a --- /dev/null +++ b/src/rpdk/core/rqts/__init__.py @@ -0,0 +1 @@ +"""RQTS (CTv2 local) contract test execution support for ``cfn test --v2``.""" diff --git a/src/rpdk/core/rqts/argv.py b/src/rpdk/core/rqts/argv.py new file mode 100644 index 00000000..d4503afd --- /dev/null +++ b/src/rpdk/core/rqts/argv.py @@ -0,0 +1,157 @@ +"""Pure argv construction for the RQTS (CTv2 local) test runner. + +This module is deliberately **side-effect free**: it performs no subprocess +calls, no filesystem or network I/O, and no Docker/AWS interaction. It only +transforms already-resolved inputs (image ref, project, region) into a +``docker run`` argument list, plus the separate process-environment mapping +(:func:`build_container_env`) that carries the credential values. Keeping both +pure makes them the units that the feature's property-based tests exercise +directly. Credential secrets never enter the argv: the ``-e`` flags are +name-only and docker resolves the values from the client process environment. + +``cfn test --v2`` targets the published executor image's DirectJar handler +mode. The container command produced is:: + + --extension contract-tests run-tests /work/.zip --direct-jar \ + -r -o + +DirectJar loads the handler JAR directly into the executor JVM, so there is no +handler endpoint, no host networking, and no ``--handler-jar``/``-h``/``-tn`` +flags. Inputs are packaged in the artifact zip and resolved by the executor, so +``-i`` is not emitted here. Scenario selection is owned by the executor image +(capability conditions plus namespace gating of the 1P-oriented ``tagging-*`` +scenarios), so no ``-s``/``--scenarios`` and no exclusion flags are emitted. +""" + +from .constants import CONTAINER_OUTPUT_DIR, CONTAINER_WORKDIR, EXTENSION_CONTRACT_TESTS + +# Environment variable names the RQTS executor reads for AWS access. These are +# emitted as NAME-ONLY ``-e`` flags: docker resolves each value from the docker +# client process environment (supplied via ``run_container``'s ``env``), so no +# credential value ever appears in the argv - which makes the DEBUG-logged +# command line and ``ps`` output safe by construction. +# The B105 suppressions below are false positives: these literals are +# environment-variable NAMES and credential-dict KEYS, never secret values. +_ENV_ACCESS_KEY_ID = "AWS_ACCESS_KEY_ID" +_ENV_SECRET_ACCESS_KEY = "AWS_SECRET_ACCESS_KEY" # nosec B105 +_ENV_SESSION_TOKEN = "AWS_SESSION_TOKEN" # nosec B105 +_ENV_REGION = "AWS_REGION" + +# Keys used by ``boto_helpers.get_temporary_credentials`` (its default +# ``BOTO_CRED_KEYS``) to describe minted temporary credentials. +_CRED_ACCESS_KEY_ID = "aws_access_key_id" +_CRED_SECRET_ACCESS_KEY = "aws_secret_access_key" # nosec B105 +_CRED_SESSION_TOKEN = "aws_session_token" # nosec B105 + + +def _env_args(): + """Return the name-only ``-e`` flags for the AWS credential/region variables. + + Values are deliberately NOT embedded: docker inherits each variable from + the docker client process environment, which the runner populates via + :func:`build_container_env`. + """ + return [ + "-e", + _ENV_ACCESS_KEY_ID, + "-e", + _ENV_SECRET_ACCESS_KEY, + "-e", + _ENV_SESSION_TOKEN, + "-e", + _ENV_REGION, + ] + + +def build_container_env(creds, region): + """Build the environment mapping the docker client process must export. + + This is the ONLY place credential secret values are threaded: the runner + merges this mapping into the spawned ``docker run`` process environment, + and the argv references the variables by name only. + + :param creds: a mapping of minted temporary credentials using + ``boto_helpers`` default keys (``aws_access_key_id``, + ``aws_secret_access_key``, ``aws_session_token``). + :param str region: the effective AWS region. + :returns: dict of environment variable name -> value for the docker client. + """ + return { + _ENV_ACCESS_KEY_ID: creds[_CRED_ACCESS_KEY_ID], + _ENV_SECRET_ACCESS_KEY: creds[_CRED_SECRET_ACCESS_KEY], + _ENV_SESSION_TOKEN: creds[_CRED_SESSION_TOKEN], + _ENV_REGION: region, + } + + +def build_docker_argv( + image_ref, + project, + region, + workdir=None, + artifact_name=None, + output_dir=CONTAINER_OUTPUT_DIR, +): + """Build the full ``docker run`` argv for the RQTS container (DirectJar). + + The argv contains NO credential values by construction: the ``-e`` flags + are name-only, and the values travel through the docker client process + environment built by :func:`build_container_env`. This keeps the argv safe + to log at DEBUG (Req 4.10) and invisible to ``ps``. + + Composition:: + + docker run --rm + -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY + -e AWS_SESSION_TOKEN -e AWS_REGION + -v :/work + + --extension contract-tests run-tests /work/ + --direct-jar -r -o + + :param str image_ref: the resolved RQTS container image reference. + :param project: the loaded :class:`rpdk.core.project.Project`. + ``root`` is the default bind-mount source and ``hypenated_name`` names + the artifact zip. + :param str region: the effective AWS region (exported as ``AWS_REGION`` via + the process environment and passed via ``-r``). + :param workdir: host directory to bind-mount onto :data:`CONTAINER_WORKDIR`. + Defaults to ``str(project.root)``. + :param artifact_name: the handler artifact zip filename inside the mount. + Defaults to ``f"{project.hypenated_name}.zip"``. + :param output_dir: container-internal output directory passed via ``-o``. + Defaults to :data:`CONTAINER_OUTPUT_DIR`. + :returns: the complete ``docker run`` argument list. + """ + if workdir is None: + workdir = str(project.root) + if artifact_name is None: + artifact_name = f"{project.hypenated_name}.zip" + + artifact_path_in_container = f"{CONTAINER_WORKDIR}/{artifact_name}" + + argv = ["docker", "run", "--rm"] + # Credential/region variables by NAME only; values come from the process env. + argv += _env_args() + # Bind mount the working directory so the artifact zip is readable in-container. + argv += ["-v", f"{workdir}:{CONTAINER_WORKDIR}"] + # The image to run. + argv.append(image_ref) + # The executor command: top-level --extension selector, then the run-tests + # subcommand with the positional artifact path and DirectJar handler mode. + argv += [ + "--extension", + EXTENSION_CONTRACT_TESTS, + "run-tests", + artifact_path_in_container, + "--direct-jar", + "-r", + region, + "-o", + output_dir, + ] + # No -s/--scenarios and no --exclude-* flags: the executor image owns + # scenario selection (capability conditions + first-party namespace gating + # of the tagging-* scenarios), so 3P resources get the non-tagging subset + # automatically and 1P resources run the full suite. + return argv diff --git a/src/rpdk/core/rqts/constants.py b/src/rpdk/core/rqts/constants.py new file mode 100644 index 00000000..628cd5f3 --- /dev/null +++ b/src/rpdk/core/rqts/constants.py @@ -0,0 +1,47 @@ +"""Constants for the RQTS (CTv2 local) contract test executor. + +These values are owned by the CLI and pin the behavior of ``cfn test --v2``: +the RQTS Docker image reference, the executor extension selector, the image +pull retry policy, and the container-internal bind-mount root and output +directory. + +``cfn test --v2`` targets the published executor image's DirectJar handler +mode, whose CLI contract is:: + + --extension contract-tests run-tests --direct-jar -r -o + +Input subsetting (``-i``) is intentionally NOT emitted: the executor uses the +inputs packaged inside the artifact zip. Scenario subsetting (``-s``) is NOT +emitted either: scenario selection is owned by the executor image, which gates +1P-oriented scenarios (``tagging-*``) by resource-type namespace, so the CLI +passes no scenario list and no exclusions. +""" + +# Fully-qualified, CLI-pinned RQTS image on the ECR Public Gallery. +# ``s5r7m5i4`` is the permanent default registry alias of the publishing +# account. The mutable ``latest`` tag is followed deliberately: the cloud +# contract-test path always runs the latest published image, and the CLI keeps +# parity by attempting a pull on every run (falling back to the local cache +# when the registry is unreachable) instead of pinning per release. +RQTS_IMAGE_REFERENCE = "public.ecr.aws/s5r7m5i4/cfn-rqts-executor-external:latest" + +# The executor top-level extension selector for contract tests. Passed as the +# top-level ``--extension`` option BEFORE the ``run-tests`` subcommand. +EXTENSION_CONTRACT_TESTS = "contract-tests" + +# Image pull retry policy. +PULL_MAX_ATTEMPTS = 3 +PULL_ATTEMPT_TIMEOUT_SECONDS = 120 + +# Container-internal bind-mount root. +CONTAINER_WORKDIR = "/work" + +# Container-internal directory the executor writes its output to (under the +# bind mount, so results are visible on the host). +CONTAINER_OUTPUT_DIR = "/work/rqts-output" + +# NOTE: the CLI deliberately owns NO scenario set. The executor image decides +# which scenarios run: capability conditions (taggable/creatable/...) plus +# namespace gating that restricts the 1P-oriented ``tagging-*`` scenarios to +# reserved first-party namespaces. 3P resources therefore get the non-tagging +# subset automatically, with no scenario or exclusion flags from the CLI. diff --git a/src/rpdk/core/rqts/image.py b/src/rpdk/core/rqts/image.py new file mode 100644 index 00000000..55d91cfe --- /dev/null +++ b/src/rpdk/core/rqts/image.py @@ -0,0 +1,159 @@ +# have to skip B404, subprocess is required to drive the local Docker CLI +# have to skip B603/B607, docker is invoked with a fixed, non-shell argv +"""RQTS image resolution and pull-with-retry. + +This module owns how ``cfn test --v2`` decides *which* RQTS container image to +run and how it makes that image available in the local Docker image store: + +* :func:`resolve_image` picks the effective image reference (an explicit + ``--rqts-image`` override, otherwise the CLI-pinned default). +* :func:`image_present_locally` reports whether the image is already in the + local Docker image store. +* :func:`ensure_image` attempts an anonymous ``docker pull`` on every run (the + pinned reference is a mutable ``latest`` tag, and the cloud contract-test + path always runs the latest published image, so local runs follow it for + parity) with a bounded per-attempt timeout and a capped number of attempts. + When every attempt fails it falls back to the local image store with a + warning if the image is cached, and raises + :class:`~rpdk.core.exceptions.SysExitRecommendedError` only when no cached + copy exists. + +Docker is driven through the ``docker`` CLI via :mod:`subprocess`. The pull is +deliberately anonymous: no AWS credentials are ever supplied to it, because the +RQTS image is published to the ECR Public Gallery and is anonymously pullable. +""" +import logging +import subprocess # nosec B404 + +from rpdk.core.exceptions import SysExitRecommendedError + +from .constants import ( + PULL_ATTEMPT_TIMEOUT_SECONDS, + PULL_MAX_ATTEMPTS, + RQTS_IMAGE_REFERENCE, +) + +LOG = logging.getLogger(__name__) + + +def resolve_image(args): + """Resolve the effective RQTS image reference. + + Returns the ``--rqts-image`` override when it is provided and non-empty, + otherwise the CLI-pinned :data:`RQTS_IMAGE_REFERENCE`. + + :param args: parsed CLI arguments (expects an ``rqts_image`` attribute) + :return: the image reference to run + :rtype: str + """ + override = getattr(args, "rqts_image", None) + if override: + return override + return RQTS_IMAGE_REFERENCE + + +def _run_docker(docker_args, timeout=None): + """Run a ``docker`` subcommand with a fixed, non-shell argv. + + Kept as a small internal seam so tests can patch a single call site. No AWS + credentials are ever injected here; the child inherits the ambient + environment only. + + :param list docker_args: arguments following ``docker`` (for example + ``["image", "inspect", ref]``) + :param timeout: optional per-attempt timeout in seconds + :return: the completed process + :rtype: subprocess.CompletedProcess + """ + return subprocess.run( # nosec B603 B607 + ["docker", *docker_args], + check=False, + capture_output=True, + timeout=timeout, + ) + + +def image_present_locally(image_ref): + """Return whether ``image_ref`` is present in the local Docker image store. + + Uses ``docker image inspect ``, which exits non-zero when the image is + not present locally. + + :param str image_ref: the image reference to look up + :return: ``True`` when the image is available locally, ``False`` otherwise + :rtype: bool + """ + try: + completed = _run_docker(["image", "inspect", image_ref]) + except (OSError, subprocess.SubprocessError) as err: + LOG.debug("Local image inspect for %s failed: %s", image_ref, err) + return False + return completed.returncode == 0 + + +def ensure_image(image_ref): + """Ensure ``image_ref`` is available locally, refreshing it when possible. + + A pull is attempted on **every** invocation so that a moved mutable tag + (for example ``latest``) is picked up without manual ``docker pull``. + The pull is anonymous (no AWS credentials) with a bounded per-attempt + timeout of :data:`PULL_ATTEMPT_TIMEOUT_SECONDS`, retrying up to + :data:`PULL_MAX_ATTEMPTS` times and stopping on the first success. An + up-to-date image costs only a manifest check; no layers are re-downloaded. + + When every attempt fails, the run falls back to the local image store if + the image is cached there (with a warning that it may be stale); only when + no cached copy exists is the failure fatal. + + :param str image_ref: the resolved RQTS image reference + :raises SysExitRecommendedError: if the pull fails on every attempt and + the image is not present in the local image store + """ + LOG.info("Pulling RQTS image %s (anonymous pull)", image_ref) + last_error = None + for attempt in range(1, PULL_MAX_ATTEMPTS + 1): + LOG.debug( + "Pulling RQTS image %s (attempt %d of %d)", + image_ref, + attempt, + PULL_MAX_ATTEMPTS, + ) + try: + completed = _run_docker( + ["pull", image_ref], timeout=PULL_ATTEMPT_TIMEOUT_SECONDS + ) + except subprocess.TimeoutExpired: + last_error = f"timed out after {PULL_ATTEMPT_TIMEOUT_SECONDS}s" + LOG.debug("Pull attempt %d for %s %s", attempt, image_ref, last_error) + continue + except (OSError, subprocess.SubprocessError) as err: + last_error = str(err) + LOG.debug( + "Pull attempt %d for %s failed: %s", attempt, image_ref, last_error + ) + continue + + if completed.returncode == 0: + LOG.debug( + "Successfully pulled RQTS image %s on attempt %d", image_ref, attempt + ) + return + + last_error = (completed.stderr or b"").decode("utf-8", "replace").strip() or ( + f"docker pull exited with code {completed.returncode}" + ) + LOG.debug("Pull attempt %d for %s failed: %s", attempt, image_ref, last_error) + + if image_present_locally(image_ref): + LOG.warning( + "Could not pull the RQTS image '%s' (%s); using the locally cached " + "copy, which may be stale", + image_ref, + last_error, + ) + return + + raise SysExitRecommendedError( + f"Failed to pull the RQTS image '{image_ref}' after " + f"{PULL_MAX_ATTEMPTS} attempts: {last_error}" + ) diff --git a/src/rpdk/core/rqts/preconditions.py b/src/rpdk/core/rqts/preconditions.py new file mode 100644 index 00000000..1351e977 --- /dev/null +++ b/src/rpdk/core/rqts/preconditions.py @@ -0,0 +1,119 @@ +"""Precondition checks for the RQTS (CTv2 local) contract test runner. + +``cfn test --v2`` requires several runtime and project prerequisites before the +RQTS container can run: a working Docker runtime, a built artifact package, and +valid AWS credentials and region. + +The DirectJar handler mode loads the handler JAR directly into the executor +JVM, so there is no SAM Local handler endpoint to probe and no separate input +resolution: inputs are packaged inside the artifact zip and read from there by +the executor. + +Each check in this module is independent and side-effect-free with respect to +the others. A check appends a single human-readable message on failure and +NEVER raises, so the caller (``RqtsRunner``) can aggregate every unmet +precondition into one error rather than failing on the first problem +(Requirement 3.7). ``check_preconditions`` returns the aggregated list of +failure messages; an empty list means all preconditions are met. +""" + +import logging +import shutil +import subprocess # nosec B404 + +from ..boto_helpers import create_sdk_session + +LOG = logging.getLogger(__name__) + +# Bounded timeout (seconds) for the Docker daemon ping so a hung daemon cannot +# stall the precondition phase. +_DOCKER_INFO_TIMEOUT_SECONDS = 10 + + +def _check_docker(): + """Return a failure message if Docker is unavailable, else ``None``. + + Docker is available only when the ``docker`` CLI is present on PATH AND the + Docker daemon is reachable (probed with ``docker info``) (Requirement 3.2). + """ + if shutil.which("docker") is None: + return ( + "Docker is required and must be running: the 'docker' CLI was not " + "found on PATH." + ) + + try: + result = subprocess.run( # nosec B603, B607 + ["docker", "info"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=_DOCKER_INFO_TIMEOUT_SECONDS, + check=False, + ) + except (OSError, subprocess.SubprocessError) as e: + LOG.debug("Docker daemon ping failed", exc_info=e) + return ( + "Docker is required and must be running: the Docker daemon could " + "not be reached." + ) + + if result.returncode != 0: + return ( + "Docker is required and must be running: the Docker daemon could " + "not be reached." + ) + + return None + + +def _check_artifact(project): + """Return a failure message if the built artifact package is missing. + + Mirrors ``Project._get_zip_file_path()``: the package lives at + ``project.root / f"{project.hypenated_name}.zip"`` (Requirement 3.3). + """ + artifact_name = f"{project.hypenated_name}.zip" + artifact_path = project.root / artifact_name + if not artifact_path.is_file(): + return ( + f"artifact package '{artifact_name}' not found; build the project " "first." + ) + return None + + +def _check_credentials(args): + """Return a failure message if AWS credentials or region are unavailable. + + Reuses ``boto_helpers.create_sdk_session`` which raises when the region or + credentials are missing; that exception is caught and converted to a failure + message so the check never raises (Requirement 3.5). + """ + try: + create_sdk_session(args.region, args.profile) + except Exception as e: # pylint: disable=broad-except + LOG.debug("AWS session could not be created", exc_info=e) + return "valid AWS credentials and a region are required." + return None + + +def check_preconditions(args, project): + """Verify all ``cfn test --v2`` preconditions and aggregate any failures. + + Runs each independent precondition check and collects the human-readable + failure message from every unmet one. No check raises; the caller turns a + non-empty list into a single ``SysExitRecommendedError`` (Requirements 3.1, + 3.7). + + :param args: parsed CLI arguments (uses ``args.region`` and ``args.profile``) + :param project: loaded ``rpdk.core.project.Project`` + :returns: list of failure messages; empty means all preconditions are met + """ + failures = [] + for message in ( + _check_docker(), + _check_artifact(project), + _check_credentials(args), + ): + if message is not None: + failures.append(message) + return failures diff --git a/src/rpdk/core/rqts/runner.py b/src/rpdk/core/rqts/runner.py new file mode 100644 index 00000000..375923b9 --- /dev/null +++ b/src/rpdk/core/rqts/runner.py @@ -0,0 +1,257 @@ +# have to skip B404, subprocess is required to drive the local Docker CLI +# have to skip B603, docker is invoked with a fixed, non-shell argv +"""RQTS (CTv2 local) contract test runner. + +This module drives the ``docker run`` step of ``cfn test --v2`` and maps the +container's exit code onto the CLI's exit-code contract: + +* :class:`RqtsRunner` orchestrates the full ``--v2`` pipeline: it guards the + project artifact type, aggregates and enforces preconditions, mints temporary + AWS credentials, resolves and ensures the RQTS image, builds the ``docker + run`` argv for the executor's DirectJar handler mode, runs the container, and + maps its exit code. Any failure raises + :class:`~rpdk.core.exceptions.SysExitRecommendedError`. +* :func:`run_container` spawns the ``docker run`` process, streams its + stdout/stderr to the terminal live (as produced, without buffering to + completion), and returns the container's exit code. A failure to even start + the container is surfaced as a + :class:`~rpdk.core.exceptions.SysExitRecommendedError`. +* :func:`map_exit_code` turns that exit code into the CLI's contract: ``0`` + reports a pass and returns; any non-zero code raises + :class:`~rpdk.core.exceptions.SysExitRecommendedError`, which ``cli.py`` + already maps to ``SystemExit(1)``. It defers the human-readable pass/fail + summary to :func:`report_result` so the summary and the exit-code mapping + stay single-sourced. +* :func:`report_result` renders the concise overall pass/fail summary, + consistent with the existing pytest-based ``cfn test`` reporting. The + per-scenario/test outcomes themselves are streamed live by + :func:`run_container`, so this summary does not re-parse or duplicate them. + +The DirectJar handler mode loads the handler JAR directly into the executor +JVM, so there is no handler endpoint to probe and no host networking to +configure. Docker is driven through the ``docker`` CLI via :mod:`subprocess`. +The child inherits the parent's stdout/stderr so RQTS results appear live in +the developer's terminal exactly as the executor emits them. +""" +import logging +import os +import subprocess # nosec B404 +from pathlib import Path + +from rpdk.core.exceptions import SysExitRecommendedError + +from ..boto_helpers import BOTO_CRED_KEYS, create_sdk_session, get_temporary_credentials +from ..project import ARTIFACT_TYPE_HOOK, ARTIFACT_TYPE_RESOURCE +from .argv import build_container_env, build_docker_argv +from .image import ensure_image, resolve_image +from .preconditions import check_preconditions + +LOG = logging.getLogger(__name__) + +# Host-side directory (under the project root, i.e. under the bind mount) that +# the executor writes its output to. Created before the container runs so the +# bind-mounted output path exists on the host. +HOST_OUTPUT_DIRNAME = "rqts-output" + +# Overall summary lines for a completed RQTS run. Phrasing is kept consistent +# with the existing pytest-based ``cfn test`` reporting, whose failure path +# raises ``SysExitRecommendedError("One or more contract tests failed")`` in +# ``test.invoke_test``; the RQTS variants name the runner explicitly. The +# container streams the per-scenario/test outcomes live (Requirement 6.2), so +# these are deliberately concise overall summaries (Requirement 6.1). +# B105 false positive: a log summary string, not a password (bandit matches +# the "PASS" in the variable name). +PASS_SUMMARY = "RQTS contract tests passed" # nosec B105 +FAIL_SUMMARY = "One or more RQTS contract tests failed" + + +def run_container(argv, env=None): + """Spawn ``docker run`` and stream its output live, returning the exit code. + + The child process inherits the parent's stdout/stderr, so the RQTS + executor's output is streamed to the terminal as it is produced rather than + buffered until the process completes (Requirement 5.1). This function blocks + until the container exits and returns its exit code. + + Credential values reach docker exclusively through ``env`` (merged over the + ambient environment): the argv's ``-e`` flags are name-only, so the command + line is safe to log and never carries secrets. + + :param list argv: the full ``docker run`` argv (as built by + :func:`rpdk.core.rqts.argv.build_docker_argv`) + :param env: optional mapping of extra environment variables (for example + from :func:`rpdk.core.rqts.argv.build_container_env`) merged over + ``os.environ`` for the docker client process + :return: the container's exit code + :rtype: int + :raises SysExitRecommendedError: if the container process cannot be spawned + """ + LOG.debug("Running RQTS container: %s", " ".join(argv)) + process_env = {**os.environ, **env} if env else None + try: + # stdout/stderr default to None, so the child inherits the parent's + # terminal and streams output live without buffering (Requirement 5.1). + with subprocess.Popen(argv, env=process_env) as process: # nosec B603 + return process.wait() + except OSError as err: + LOG.debug("Failed to start the RQTS container", exc_info=err) + raise SysExitRecommendedError( + f"the RQTS container could not be started: {err}" + ) from err + + +def report_result(code): + """Surface the overall pass/fail summary for a completed RQTS run. + + The RQTS container already streams its per-scenario/test outcomes to the + terminal live via :func:`run_container` (Requirements 5.1, 6.2); this helper + adds only the concise overall summary that mirrors the existing + pytest-based ``cfn test`` reporting (Requirement 6.1), without re-parsing or + duplicating the streamed output. A ``0`` code logs an informational pass + summary; any non-zero code raises :class:`SysExitRecommendedError` whose + message names the RQTS runner, consistent with the existing "One or more + contract tests failed" phrasing used by the pytest path. + + This helper only renders the summary; the exit-code interpretation itself is + owned by :func:`map_exit_code`, which calls this helper so the summary and + the exit-code mapping stay single-sourced (Requirement 6.3). + + :param int code: the container's exit code + :raises SysExitRecommendedError: if ``code`` is non-zero + """ + if code == 0: + LOG.info(PASS_SUMMARY) + return + raise SysExitRecommendedError(FAIL_SUMMARY) + + +def map_exit_code(code): + """Map the RQTS container exit code onto the CLI's exit-code contract. + + A ``0`` exit code means every RQTS contract test passed: an informational + summary is logged and the function returns normally so the CLI exits ``0`` + (Requirements 5.2, 6.1). Any non-zero exit code means one or more contract + tests failed and raises :class:`SysExitRecommendedError`, which ``cli.py`` + maps to ``SystemExit(1)`` (Requirements 5.3, 6.3). + + The pass/fail summary is delegated to :func:`report_result` so that the + reporting and the exit-code mapping remain single-sourced; this function is + the exit-code contract entry point invoked by :meth:`RqtsRunner.run`. + + :param int code: the container's exit code + :raises SysExitRecommendedError: if ``code`` is non-zero + """ + report_result(code) + + +class RqtsRunner: + """Orchestrates the ``cfn test --v2`` RQTS pipeline. + + A single instance owns the parsed CLI ``args`` and the loaded + :class:`~rpdk.core.project.Project` and drives the fixed pipeline in + :meth:`run`: artifact-type guard, precondition aggregation, credential + minting, image resolve/ensure, ``docker run`` argv construction (for the + executor's DirectJar handler mode), container execution, and exit-code + mapping. + + Module projects are short-circuited upstream in ``test()`` before the runner + is constructed, so this class only handles resource (the supported case), + hook, and indeterminate artifact types. + """ + + def __init__(self, args, project): + """Store the parsed CLI arguments and the loaded project. + + :param args: parsed CLI arguments (an argparse ``Namespace``). The + runner reads ``region``, ``profile``, ``role_arn``, + ``source_account``, ``source_arn`` and ``rqts_image``. + :param project: the loaded :class:`~rpdk.core.project.Project`. + """ + self.args = args + self.project = project + + def _guard_artifact_type(self): + """Fail fast unless the project is a supported resource type. + + Hook projects are unsupported by the RQTS local runner (Requirement + 7.2); any artifact type that is neither a resource nor a hook is treated + as indeterminate (Requirement 7.5). Module projects are handled upstream + in ``test()`` and never reach this method. + + :raises SysExitRecommendedError: for hook or indeterminate artifact + types + """ + artifact_type = self.project.artifact_type + if artifact_type == ARTIFACT_TYPE_HOOK: + raise SysExitRecommendedError( + "the RQTS local test runner supports resource types only" + ) + if artifact_type != ARTIFACT_TYPE_RESOURCE: + raise SysExitRecommendedError( + "could not determine the project artifact type" + ) + + def _mint_credentials(self): + """Mint temporary AWS credentials for the container to use. + + Reuses the exact ``boto_helpers`` pattern used by the contract-test + path: create a session from the effective region/profile, then request + temporary credentials (assuming ``--role-arn`` when supplied) with the + confused-deputy ``--source-account`` / ``--source-arn`` headers + (Requirement 3.5). ``BOTO_CRED_KEYS`` is used so the returned mapping's + keys match what :func:`rpdk.core.rqts.argv.build_docker_argv` expects. + + :return: a mapping of temporary credentials keyed by ``BOTO_CRED_KEYS`` + :rtype: dict + """ + session = create_sdk_session(self.args.region, self.args.profile) + return get_temporary_credentials( + session, + BOTO_CRED_KEYS, + self.args.role_arn, + headers={ + "account_id": self.args.source_account, + "source_arn": self.args.source_arn, + }, + ) + + def run(self): + """Orchestrate the full ``--v2`` pipeline. + + Raises :class:`~rpdk.core.exceptions.SysExitRecommendedError` on any + failure (guard, preconditions, image pull, container start, or a + non-zero container exit code); returns normally when every RQTS contract + test passes. + """ + self._guard_artifact_type() + + failures = check_preconditions(self.args, self.project) + if failures: + raise SysExitRecommendedError( + "cannot run 'cfn test --v2'; the following preconditions were " + "not met:\n" + "\n".join(f" - {failure}" for failure in failures) + ) + + creds = self._mint_credentials() + + image_ref = resolve_image(self.args) + ensure_image(image_ref) + + # Ensure the host-side output directory (under the bind mount) exists so + # the container's -o path is present when docker mounts it. + output_dir = Path(self.project.root) / HOST_OUTPUT_DIRNAME + output_dir.mkdir(parents=True, exist_ok=True) + + # The argv is credential-free (name-only -e flags), so logging the full + # command line at DEBUG (Req 4.10) cannot leak secrets; the values + # travel only through the docker client process environment. + argv = build_docker_argv( + image_ref, + self.project, + self.args.region, + ) + LOG.debug("RQTS docker command: %s", " ".join(argv)) + + container_env = build_container_env(creds, self.args.region) + exit_code = run_container(argv, env=container_env) + map_exit_code(exit_code) diff --git a/src/rpdk/core/test.py b/src/rpdk/core/test.py index e0f909c3..83a23fe2 100644 --- a/src/rpdk/core/test.py +++ b/src/rpdk/core/test.py @@ -418,6 +418,13 @@ def test(args): LOG.warning("The test command is not supported in a module project") return + if args.v2: + # local import keeps the pytest path import-light + from .rqts.runner import RqtsRunner # pylint: disable=import-outside-toplevel + + RqtsRunner(args, project).run() + return + if project.artifact_type == ARTIFACT_TYPE_HOOK: overrides = get_hook_overrides( project.root, @@ -545,6 +552,25 @@ def setup_subparser(subparsers, parents): help="Source Type Version Arn key used for Assume Role to Run Contract Tests", ) + parser.add_argument( + "--v2", + action="store_true", + default=False, + help=( + "Opt-in: run the RQTS local test runner (CTv2) in a Docker container " + "instead of the default pytest-based contract tests." + ), + ) + + parser.add_argument( + "--rqts-image", + default=None, + help=( + "Override the RQTS container image reference (for testing or pre-release " + "images). Defaults to the CLI-pinned public.ecr.aws image." + ), + ) + def _sam_arguments(parser): parser.add_argument( diff --git a/tests/rqts/__init__.py b/tests/rqts/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/rqts/test_argv.py b/tests/rqts/test_argv.py new file mode 100644 index 00000000..1773b062 --- /dev/null +++ b/tests/rqts/test_argv.py @@ -0,0 +1,403 @@ +"""Property-based tests for the pure argv construction module. + +These tests exercise ``rpdk.core.rqts.argv`` for the executor's DirectJar +handler mode. The module under test is side-effect free, so no Docker or AWS +interaction is required: the tests only assert structural invariants of the +``docker run`` argument list it produces. + +The container command produced for local DirectJar is:: + + docker run --rm -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY + -e AWS_SESSION_TOKEN -e AWS_REGION -v :/work + --extension contract-tests run-tests /work/.zip + --direct-jar -r -o + +Scenario selection is owned by the executor image (capability conditions plus +first-party namespace gating), so the argv carries NO ``-s``/``--scenarios`` +and NO ``--exclude-*`` flags. + +The ``-e`` flags are name-only: credential values travel exclusively through +the container env mapping (``build_container_env``), never through the argv. + +Library: Hypothesis (the standard Python property-based testing library). Each +property test runs at least 100 generated examples via +``@settings(max_examples=100)`` and is tagged with a comment referencing the +design property it validates. +""" +from types import SimpleNamespace + +from hypothesis import given, settings, strategies as st + +from rpdk.core.rqts.argv import build_container_env, build_docker_argv +from rpdk.core.rqts.constants import ( + CONTAINER_OUTPUT_DIR, + CONTAINER_WORKDIR, + EXTENSION_CONTRACT_TESTS, +) + +# The CLI default region: ``cfn test`` defines ``--region`` with this default, so +# the effective region is always populated even when the user omits ``--region``. +CLI_DEFAULT_REGION = "us-east-1" + +# Flags the DirectJar contract must NEVER emit (they belong to the old SAM Local +# / remote-lambda shapes). +FORBIDDEN_FLAGS = ("--handler-jar", "-h", "-tn", "--sam-local", "--remote-lambda") + +# The tagging scenarios that ``cfn test --v2`` must NEVER run: none of these may +# appear anywhere in the argv (per product decision: no tagging tests). +FORBIDDEN_TAGGING_SCENARIOS = ( + "tagging-oob", + "tagging-permission", + "tagging-stack", + "tagging-system", +) + +# --------------------------------------------------------------------------- +# Strategies +# --------------------------------------------------------------------------- + +# A distinctive marker prefix used for generated credential secret values. It is +# built from characters that never appear in the type names, regions, or names +# generated below, so a secret value can be searched for reliably across the +# whole argv (Property: credentials only in env). +_SECRET_MARKER = "SECRETMARKER" + +# Alphabet for type-name / region / name segments. +_SEGMENT_ALPHABET = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" + +_segments = st.text(alphabet=_SEGMENT_ALPHABET, min_size=1, max_size=12) + + +@st.composite +def hyphenated_names(draw): + """Generate realistic ``org-service-resource`` style hyphenated names.""" + parts = draw(st.lists(_segments, min_size=3, max_size=3)) + return "-".join(parts).lower() + + +# Regions: a mix of realistic region strings (including the CLI default) and +# freely generated ones. +_KNOWN_REGIONS = [ + CLI_DEFAULT_REGION, + "us-west-2", + "eu-west-1", + "ap-southeast-2", + "eu-central-1", +] +regions = st.one_of( + st.sampled_from(_KNOWN_REGIONS), + st.text(alphabet=_SEGMENT_ALPHABET + "-", min_size=1, max_size=20), +) + + +# Distinctive, searchable secret values. Each carries the marker prefix so it can +# be located anywhere in the argv without colliding with other generated data. +_secret_values = st.text(alphabet=_SEGMENT_ALPHABET, min_size=6, max_size=24).map( + lambda body: _SECRET_MARKER + body +) + + +@st.composite +def credentials(draw): + """Generate a credentials dict using the boto_helpers default keys.""" + return { + "aws_access_key_id": draw(_secret_values), + "aws_secret_access_key": draw(_secret_values), + "aws_session_token": draw(_secret_values), + } + + +# Working directory paths for the bind mount. +workdirs = st.text(alphabet=_SEGMENT_ALPHABET + "/", min_size=1, max_size=40).map( + lambda p: "/" + p.strip("/") +) + + +def _make_project(hypenated_name, root="/project/root"): + """Lightweight stand-in for a loaded Project (only .hypenated_name/.root used).""" + return SimpleNamespace(hypenated_name=hypenated_name, root=root) + + +def _env_entries(argv): + """Return the list of ``-e`` environment entry tokens (the token after each -e).""" + entries = [] + for i, token in enumerate(argv): + if token == "-e" and i + 1 < len(argv): + entries.append(argv[i + 1]) + return entries + + +# --------------------------------------------------------------------------- +# Property 4 +# --------------------------------------------------------------------------- +# Feature: cfn-test-v2-flag, Property 4: docker run bind-mounts the working directory onto /work +@settings(max_examples=100) +@given( + name=hyphenated_names(), + region=regions, + workdir=workdirs, +) +def test_property_4_docker_run_bind_mounts_workdir(name, region, workdir): + project = _make_project(name) + argv = build_docker_argv("img:ref", project, region, workdir=workdir) + + # The argv must be a `docker run --rm` invocation containing a -v bind mount + # that maps the working directory onto the container workdir. + assert argv[:3] == ["docker", "run", "--rm"] + expected_mount = f"{workdir}:{CONTAINER_WORKDIR}" + assert "-v" in argv + v_index = argv.index("-v") + assert argv[v_index + 1] == expected_mount + + +# --------------------------------------------------------------------------- +# Property 5 +# --------------------------------------------------------------------------- +# Feature: cfn-test-v2-flag, Property 5: credential values never enter the argv; +# they travel only through the container env mapping +@settings(max_examples=100) +@given( + name=hyphenated_names(), + region=regions, + creds=credentials(), +) +def test_property_5_credentials_only_in_env_mapping_never_in_argv(name, region, creds): + project = _make_project(name) + argv = build_docker_argv("img:ref", project, region, workdir="/work/dir") + + env_entries = _env_entries(argv) + + # The -e flags are NAME-ONLY: the variable names are referenced so docker + # inherits them from the client process environment; no `=` and no values. + assert env_entries == [ + "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY", + "AWS_SESSION_TOKEN", + "AWS_REGION", + ] + + # No credential secret value appears ANYWHERE in the argv (this is what + # makes the DEBUG-logged command line and `ps` output safe). + secret_values = [ + creds["aws_access_key_id"], + creds["aws_secret_access_key"], + creds["aws_session_token"], + ] + for token in argv: + for secret in secret_values: + assert secret not in token + assert "=" not in token or not token.startswith("AWS_") + + # The values travel exclusively through the container env mapping. + env = build_container_env(creds, region) + assert env == { + "AWS_ACCESS_KEY_ID": creds["aws_access_key_id"], + "AWS_SECRET_ACCESS_KEY": creds["aws_secret_access_key"], + "AWS_SESSION_TOKEN": creds["aws_session_token"], + "AWS_REGION": region, + } + + +# --------------------------------------------------------------------------- +# Property 6 +# --------------------------------------------------------------------------- +# Feature: cfn-test-v2-flag, Property 6: run-tests receives the positional artifact path inside the mount +@settings(max_examples=100) +@given( + name=hyphenated_names(), + region=regions, +) +def test_property_6_extension_run_tests_and_artifact_path(name, region): + project = _make_project(name) + argv = build_docker_argv("img:ref", project, region) + + # The top-level extension selector precedes the run-tests subcommand. + assert "--extension" in argv + ext_index = argv.index("--extension") + assert argv[ext_index + 1] == EXTENSION_CONTRACT_TESTS + assert argv[ext_index + 2] == "run-tests" + + # The positional artifact path immediately follows run-tests and is addressed + # under the container working directory. + artifact_path = argv[ext_index + 3] + assert artifact_path == f"{CONTAINER_WORKDIR}/{name}.zip" + assert artifact_path.startswith(CONTAINER_WORKDIR) + + +# --------------------------------------------------------------------------- +# Property 7 +# --------------------------------------------------------------------------- +# Feature: cfn-test-v2-flag, Property 7: DirectJar handler mode is always selected +@settings(max_examples=100) +@given( + name=hyphenated_names(), + region=regions, +) +def test_property_7_direct_jar_mode_always_present(name, region): + project = _make_project(name) + argv = build_docker_argv("img:ref", project, region) + + assert "--direct-jar" in argv + # And the artifact path is not itself the --direct-jar token. + assert argv.count("--direct-jar") == 1 + + +# --------------------------------------------------------------------------- +# Property 8 +# --------------------------------------------------------------------------- +# Feature: cfn-test-v2-flag, Property 8: region argument always present with the effective region +@settings(max_examples=100) +@given( + name=hyphenated_names(), + # Include the default-region case explicitly by drawing a boolean that forces + # the CLI default, mirroring "--region not supplied". + use_default=st.booleans(), + region=regions, +) +def test_property_8_region_argument_always_present(name, use_default, region): + effective_region = CLI_DEFAULT_REGION if use_default else region + project = _make_project(name) + argv = build_docker_argv("img:ref", project, effective_region) + + assert "-r" in argv + r_index = argv.index("-r") + assert argv[r_index + 1] == effective_region + + +# --------------------------------------------------------------------------- +# Property 9 +# --------------------------------------------------------------------------- +# Feature: cfn-test-v2-flag, Property 9: the old SAM Local / remote-lambda flags are never emitted +@settings(max_examples=100) +@given( + name=hyphenated_names(), + region=regions, + workdir=workdirs, +) +def test_property_9_forbidden_flags_never_emitted(name, region, workdir): + project = _make_project(name) + argv = build_docker_argv("img:ref", project, region, workdir=workdir) + + for flag in FORBIDDEN_FLAGS: + assert flag not in argv + + +# --------------------------------------------------------------------------- +# Property 10 +# --------------------------------------------------------------------------- +# Feature: cfn-test-v2-flag, Property 10: no host networking is configured (DirectJar needs none) +@settings(max_examples=100) +@given( + name=hyphenated_names(), + region=regions, + workdir=workdirs, +) +def test_property_10_no_host_networking(name, region, workdir): + project = _make_project(name) + argv = build_docker_argv("img:ref", project, region, workdir=workdir) + + argv_str = " ".join(argv) + assert "--network" not in argv + assert "--add-host" not in argv + assert "host.docker.internal" not in argv_str + + +# --------------------------------------------------------------------------- +# Property 11 +# --------------------------------------------------------------------------- +# Feature: cfn-test-v2-flag, Property 11: output directory is emitted via -o under the mount +@settings(max_examples=100) +@given( + name=hyphenated_names(), + region=regions, +) +def test_property_11_output_dir_emitted(name, region): + project = _make_project(name) + argv = build_docker_argv("img:ref", project, region) + + assert "-o" in argv + o_index = argv.index("-o") + assert argv[o_index + 1] == CONTAINER_OUTPUT_DIR + # The default output dir lives under the container working directory mount. + assert CONTAINER_OUTPUT_DIR.startswith(CONTAINER_WORKDIR) + + +# --------------------------------------------------------------------------- +# Property 12 +# --------------------------------------------------------------------------- +# Feature: cfn-test-v2-flag, Property 12: no scenario-selection or exclusion flags +# are ever emitted; scenario selection is owned by the executor image +@settings(max_examples=100) +@given( + name=hyphenated_names(), + region=regions, + workdir=workdirs, +) +def test_property_12_no_scenario_selection_or_exclusion_flags(name, region, workdir): + project = _make_project(name) + argv = build_docker_argv("img:ref", project, region, workdir=workdir) + + # No scenario-selection tokens: the executor decides what runs (capability + # conditions + first-party namespace gating of the tagging-* scenarios). + assert "-s" not in argv + assert "--scenarios" not in argv + + # No test/check exclusion flags of any kind. + for token in argv: + assert not token.startswith("--exclude-") + + # No scenario name (tagging or otherwise) appears anywhere in the argv. + for tagging_scenario in FORBIDDEN_TAGGING_SCENARIOS: + assert tagging_scenario not in argv + + # The executor command therefore ends at the output directory. + assert argv[-2:] == ["-o", CONTAINER_OUTPUT_DIR] + + +# --------------------------------------------------------------------------- +# Defaults +# --------------------------------------------------------------------------- +def test_defaults_use_project_root_and_hyphenated_name(): + """workdir defaults to str(project.root); artifact_name to .zip.""" + project = _make_project("aws-foo-bar", root="/my/project/root") + argv = build_docker_argv("img:ref", project, "us-east-1") + + v_index = argv.index("-v") + assert argv[v_index + 1] == f"/my/project/root:{CONTAINER_WORKDIR}" + + ext_index = argv.index("--extension") + assert argv[ext_index + 3] == f"{CONTAINER_WORKDIR}/aws-foo-bar.zip" + + # No scenario selection: the argv ends at the output directory. + assert "-s" not in argv + assert argv[-2:] == ["-o", CONTAINER_OUTPUT_DIR] + + +# =========================================================================== +# Explicit overrides of the project-derived defaults +# =========================================================================== +def test_explicit_workdir_artifact_and_output_dir_override_defaults(): + """Supplied workdir/artifact_name/output_dir replace the project defaults. + + The defaults are derived from the project (``root`` and + ``hypenated_name``); when a caller passes them explicitly, none of the + project-derived values appear in the argv. + """ + project = _make_project("aws-foo-bar", root="/project/root") + + argv = build_docker_argv( + "img:ref", + project, + "us-west-2", + workdir="/elsewhere", + artifact_name="custom-artifact.zip", + output_dir="/work/custom-output", + ) + + assert f"/elsewhere:{CONTAINER_WORKDIR}" in argv + assert f"{CONTAINER_WORKDIR}/custom-artifact.zip" in argv + assert argv[-2:] == ["-o", "/work/custom-output"] + # None of the project-derived defaults leak in. + assert f"/project/root:{CONTAINER_WORKDIR}" not in argv + assert f"{CONTAINER_WORKDIR}/aws-foo-bar.zip" not in argv + assert CONTAINER_OUTPUT_DIR not in argv diff --git a/tests/rqts/test_image.py b/tests/rqts/test_image.py new file mode 100644 index 00000000..8861e564 --- /dev/null +++ b/tests/rqts/test_image.py @@ -0,0 +1,333 @@ +"""Tests for RQTS image resolution and pull-with-retry. + +These tests exercise ``rpdk.core.rqts.image`` (design Properties 1-2, plus the +pull-exhaustion edge cases and the cached-image fallback branch). The pull is +attempted on every run (mutable ``latest`` semantics); on exhaustion the run +falls back to a cached local image with a warning and fails only when no +cached copy exists. Docker is never actually invoked: the single internal +``_run_docker`` seam and ``image_present_locally`` are patched so the retry +policy can be driven deterministically and kept fast. + +Library: Hypothesis (the standard Python property-based testing library). Each +property test runs at least 100 generated examples via +``@settings(max_examples=100)`` and is tagged with a comment referencing the +design property it validates. +""" +import subprocess +from types import SimpleNamespace +from unittest import mock + +import pytest +from hypothesis import given, settings, strategies as st + +from rpdk.core.exceptions import SysExitRecommendedError +from rpdk.core.rqts import image as image_module +from rpdk.core.rqts.constants import PULL_MAX_ATTEMPTS, RQTS_IMAGE_REFERENCE + +# A marker used to build distinctive, searchable fake credential values. It is +# made of characters that never appear in the generated image references, so the +# credential values can be located reliably (or, as Property 2 requires, +# confirmed absent) anywhere in the recorded docker invocations. +_SECRET_MARKER = "AWSSECRETMARKER" + +# AWS credential environment variable names that must never be threaded into an +# anonymous ``docker pull``. +_AWS_CRED_TOKENS = ( + "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY", + "AWS_SESSION_TOKEN", + _SECRET_MARKER, +) + +_IMAGE_ALPHABET = "abcdefghijklmnopqrstuvwxyz0123456789./:-" + +# Image references, including the empty string and freely generated refs, so the +# resolution property covers both the override and the pinned-default cases. +image_refs = st.text(alphabet=_IMAGE_ALPHABET, min_size=1, max_size=60) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +class _RecordingDocker: + """A fake ``_run_docker`` that records calls and replays a fixed result seq. + + Each queued result is either a ``returncode`` int (wrapped in a + ``CompletedProcess``) or an exception instance to raise, modelling docker + pull failures/timeouts and successes. + """ + + def __init__(self, results): + self._results = list(results) + self.calls = [] + + def __call__(self, docker_args, timeout=None): + self.calls.append( + SimpleNamespace(docker_args=list(docker_args), timeout=timeout) + ) + result = self._results[len(self.calls) - 1] + if isinstance(result, BaseException): + raise result + return subprocess.CompletedProcess( + args=["docker", *docker_args], + returncode=result, + stdout=b"", + stderr=b"" if result == 0 else b"boom", + ) + + +def _make_args(rqts_image=None): + return SimpleNamespace(rqts_image=rqts_image) + + +# =========================================================================== +# Task 4.2 -> Property 1 +# =========================================================================== +# Feature: cfn-test-v2-flag, Property 1: Image resolution honors override, else pinned default +@settings(max_examples=100) +@given(override=st.one_of(st.none(), st.just(""), image_refs)) +def test_property_1_image_resolution_override_else_default(override): + """resolve_image returns the override when non-empty, else the pinned default. + + Validates: Requirements 2.1, 2.2 + """ + args = _make_args(rqts_image=override) + + resolved = image_module.resolve_image(args) + + if override: + assert resolved == override + else: + assert resolved == RQTS_IMAGE_REFERENCE + + +def test_resolve_image_missing_attribute_uses_default(): + """resolve_image tolerates args without an ``rqts_image`` attribute.""" + assert image_module.resolve_image(SimpleNamespace()) == RQTS_IMAGE_REFERENCE + + +# =========================================================================== +# Task 4.3 -> Property 2 +# =========================================================================== +def _expected_attempts(results): + """Model the bounded retry policy: attempt until first success, capped.""" + for index, result in enumerate(results[:PULL_MAX_ATTEMPTS]): + if result == 0: + return index + 1, True + return min(len(results), PULL_MAX_ATTEMPTS), False + + +@st.composite +def pull_result_sequences(draw): + """Generate a sequence of per-attempt pull outcomes. + + Each element is a ``0`` (success), a non-zero return code (failure), or an + exception instance (timeout / spawn error). The sequence is padded to at + least ``PULL_MAX_ATTEMPTS`` so ``ensure_image`` always has an outcome to + consume even when every attempt fails. + """ + outcome = st.one_of( + st.just(0), + st.integers(min_value=1, max_value=255), + st.just(subprocess.TimeoutExpired(cmd="docker pull", timeout=1)), + st.just(OSError("docker not found")), + ) + seq = draw( + st.lists(outcome, min_size=PULL_MAX_ATTEMPTS, max_size=PULL_MAX_ATTEMPTS + 4) + ) + return seq + + +# Feature: cfn-test-v2-flag, Property 2: Image pull is bounded and anonymous +@settings(max_examples=100) +@given(results=pull_result_sequences(), image_ref=image_refs, cached=st.booleans()) +def test_property_2_pull_is_bounded_and_anonymous(results, image_ref, cached): + """ensure_image pulls at most PULL_MAX_ATTEMPTS regardless of the local + cache state, stops on first success, never supplies AWS credentials to the + pull, falls back to a cached image on exhaustion, and raises only when no + cached copy exists. + + Validates: Requirements 2.3, 2.4, 2.5, 2.6 + """ + fake_docker = _RecordingDocker(results) + expected_attempts, should_succeed = _expected_attempts(results) + + # ``mock.patch`` context managers are used (rather than the monkeypatch + # fixture) so the patches are applied and reset for every generated example. + with mock.patch.object( + image_module, "image_present_locally", lambda ref: cached + ), mock.patch.object(image_module, "_run_docker", fake_docker): + if should_succeed or cached: + image_module.ensure_image(image_ref) + else: + with pytest.raises(SysExitRecommendedError): + image_module.ensure_image(image_ref) + + # Bounded: pull is attempted at most PULL_MAX_ATTEMPTS times, and exactly the + # number of attempts the retry policy predicts (stopping on first success). + assert len(fake_docker.calls) <= PULL_MAX_ATTEMPTS + assert len(fake_docker.calls) == expected_attempts + + for call in fake_docker.calls: + # Every invocation is an anonymous ``docker pull `` for this image. + assert call.docker_args[0] == "pull" + assert image_ref in call.docker_args + + # No AWS credential is ever threaded through: neither positionally in the + # docker args nor via any keyword. ensure_image only passes ``timeout``. + for token in call.docker_args: + for cred in _AWS_CRED_TOKENS: + assert cred not in token + + +# =========================================================================== +# Task 4.4 -> pull-exhaustion edge cases +# =========================================================================== +def test_pull_exhaustion_without_cached_image_raises(monkeypatch): + """Pull fails on all attempts with no cached image -> SysExitRecommendedError + after exactly PULL_MAX_ATTEMPTS attempts. + + Validates: Requirements 2.6 + """ + # Every attempt fails with a non-zero return code. + fake_docker = _RecordingDocker([1] * (PULL_MAX_ATTEMPTS + 2)) + monkeypatch.setattr(image_module, "image_present_locally", lambda ref: False) + monkeypatch.setattr(image_module, "_run_docker", fake_docker) + + with pytest.raises(SysExitRecommendedError): + image_module.ensure_image("some/image:tag") + + assert len(fake_docker.calls) == PULL_MAX_ATTEMPTS + + +def test_pull_exhaustion_with_cached_image_falls_back_with_warning(monkeypatch, caplog): + """Pull fails on all attempts but the image is cached locally -> warn and + run from the local store instead of raising. + + Validates: Requirements 2.5 + """ + fake_docker = _RecordingDocker([1] * PULL_MAX_ATTEMPTS) + monkeypatch.setattr(image_module, "image_present_locally", lambda ref: True) + monkeypatch.setattr(image_module, "_run_docker", fake_docker) + + with caplog.at_level("WARNING", logger=image_module.__name__): + image_module.ensure_image("cached/image:tag") + + # All attempts consumed, then fallback: no exception raised. + assert len(fake_docker.calls) == PULL_MAX_ATTEMPTS + assert any( + "cached" in record.getMessage() and "cached/image:tag" in record.getMessage() + for record in caplog.records + ) + + +def test_pull_exhaustion_on_repeated_timeouts(monkeypatch): + """Repeated per-attempt timeouts also exhaust after exactly PULL_MAX_ATTEMPTS.""" + timeouts = [ + subprocess.TimeoutExpired(cmd="docker pull", timeout=1) + for _ in range(PULL_MAX_ATTEMPTS + 1) + ] + fake_docker = _RecordingDocker(timeouts) + monkeypatch.setattr(image_module, "image_present_locally", lambda ref: False) + monkeypatch.setattr(image_module, "_run_docker", fake_docker) + + with pytest.raises(SysExitRecommendedError): + image_module.ensure_image("some/image:tag") + + assert len(fake_docker.calls) == PULL_MAX_ATTEMPTS + + +# =========================================================================== +# Task 4.5 -> pull-first behavior +# =========================================================================== +def test_ensure_image_present_locally_still_pulls(monkeypatch): + """Image present locally -> ensure_image STILL attempts the pull, so a moved + mutable tag (e.g. latest) is refreshed without manual docker pull. + + Validates: Requirements 2.3 + """ + fake_docker = _RecordingDocker([0]) + monkeypatch.setattr(image_module, "image_present_locally", lambda ref: True) + monkeypatch.setattr(image_module, "_run_docker", fake_docker) + + image_module.ensure_image("present/image:tag") + + # Exactly one successful pull attempt; presence never short-circuits it. + assert len(fake_docker.calls) == 1 + assert fake_docker.calls[0].docker_args[0] == "pull" + assert "present/image:tag" in fake_docker.calls[0].docker_args + + +def test_ensure_image_absent_pulls_then_returns(monkeypatch): + """Image absent -> ensure_image pulls, then returns on success. + + Validates: Requirements 2.3, 2.4 + """ + fake_docker = _RecordingDocker([0]) + monkeypatch.setattr(image_module, "image_present_locally", lambda ref: False) + monkeypatch.setattr(image_module, "_run_docker", fake_docker) + + image_module.ensure_image("absent/image:tag") + + assert len(fake_docker.calls) == 1 + assert fake_docker.calls[0].docker_args[0] == "pull" + assert "absent/image:tag" in fake_docker.calls[0].docker_args + + +def test_image_present_locally_uses_docker_image_inspect(monkeypatch): + """image_present_locally maps ``docker image inspect`` exit status to bool.""" + present = _RecordingDocker([0]) + monkeypatch.setattr(image_module, "_run_docker", present) + assert image_module.image_present_locally("ref:tag") is True + assert present.calls[0].docker_args == ["image", "inspect", "ref:tag"] + + absent = _RecordingDocker([1]) + monkeypatch.setattr(image_module, "_run_docker", absent) + assert image_module.image_present_locally("ref:tag") is False + + +# =========================================================================== +# Internal docker seam and inspect failure handling +# =========================================================================== +def test_run_docker_invokes_docker_cli_with_fixed_argv(): + """_run_docker drives the docker CLI with a fixed, non-shell argv. + + This is the one place that actually reaches ``subprocess.run``; it is + patched here so no docker process is spawned. A fixed argv list (never a + shell string) is what makes the image reference safe to pass through. + """ + completed = subprocess.CompletedProcess(args=["docker", "info"], returncode=0) + + with mock.patch.object( + image_module.subprocess, "run", return_value=completed + ) as run: + result = image_module._run_docker( # pylint: disable=protected-access + ["image", "inspect", "ref:tag"], timeout=7 + ) + + assert result is completed + run.assert_called_once() + assert run.call_args[0][0] == ["docker", "image", "inspect", "ref:tag"] + assert run.call_args[1]["check"] is False + assert run.call_args[1]["capture_output"] is True + assert run.call_args[1]["timeout"] == 7 + + +@pytest.mark.parametrize( + "docker_error", [OSError("docker not found"), subprocess.SubprocessError("boom")] +) +def test_image_present_locally_false_when_inspect_cannot_run(monkeypatch, docker_error): + """An inspect that cannot even run is treated as 'not present locally'. + + Keeps ``ensure_image``'s fallback decision safe when the docker CLI is + missing or unusable: absent, rather than assumed cached. + """ + + def raise_error(_docker_args, timeout=None): # pylint: disable=unused-argument + raise docker_error + + monkeypatch.setattr(image_module, "_run_docker", raise_error) + + assert image_module.image_present_locally("ref:tag") is False diff --git a/tests/rqts/test_preconditions.py b/tests/rqts/test_preconditions.py new file mode 100644 index 00000000..069d3996 --- /dev/null +++ b/tests/rqts/test_preconditions.py @@ -0,0 +1,262 @@ +"""Tests for ``rpdk.core.rqts.preconditions``. + +Covers: +- Property 3: precondition failures are aggregated exactly (over the three + DirectJar checks: Docker runtime, artifact package, credentials+region). +- Each precondition failing in isolation produces its own message and prevents + any container run. + +The DirectJar handler mode has no SAM Local endpoint to probe and reads inputs +from the packaged artifact zip, so there is no endpoint or inputs check. + +Every check is controlled independently by patching within +``rpdk.core.rqts.preconditions`` (``shutil.which`` + ``subprocess.run`` for +Docker, ``create_sdk_session`` for credentials) and by toggling the artifact +package file under a ``tmp_path`` working directory. +""" + +import contextlib +import subprocess +from unittest import mock + +import pytest +from hypothesis import HealthCheck, given, settings, strategies as st + +from rpdk.core.exceptions import CLIMisconfiguredError +from rpdk.core.rqts.preconditions import check_preconditions + +PRECONDITIONS_MODULE = "rpdk.core.rqts.preconditions" + +# The three checks, in the order check_preconditions runs them, keyed to the +# distinctive substring of the failure message each one emits. +CHECK_NAMES = ("docker", "artifact", "credentials") +MESSAGE_SUBSTRINGS = { + "docker": "Docker is required", + "artifact": "artifact package", + "credentials": "valid AWS credentials and a region", +} + + +class FakeProject: + """Minimal stand-in for ``rpdk.core.project.Project``. + + Only the attributes the precondition checks read are provided: ``root`` and + ``hypenated_name``. + """ + + def __init__(self, root): + self.root = root + self.hypenated_name = "aws-foo-bar" + + +def _make_args(): + args = mock.Mock() + args.region = "us-east-1" + args.profile = None + return args + + +@contextlib.contextmanager +def configured_env(work_dir, states): + """Force each precondition to pass/fail per ``states``. + + ``states`` maps check name -> bool, where ``True`` means the check should + PASS and ``False`` means it should FAIL. Yields ``(args, project)`` wired so + that ``check_preconditions`` observes exactly those outcomes. + """ + project = FakeProject(work_dir) + + # Artifact package presence (real filesystem toggle). + artifact_path = work_dir / f"{project.hypenated_name}.zip" + if states["artifact"]: + artifact_path.write_bytes(b"zip") + elif artifact_path.exists(): + artifact_path.unlink() + + # Docker: which() present + `docker info` returncode 0 => pass. + which_return = "/usr/bin/docker" if states["docker"] else None + docker_info_result = mock.Mock(returncode=0 if states["docker"] else 1) + + # Credentials: create_sdk_session succeeds => pass, raises => fail. + session_side_effect = ( + None if states["credentials"] else CLIMisconfiguredError("no creds") + ) + + with contextlib.ExitStack() as stack: + stack.enter_context( + mock.patch( + f"{PRECONDITIONS_MODULE}.shutil.which", return_value=which_return + ) + ) + stack.enter_context( + mock.patch( + f"{PRECONDITIONS_MODULE}.subprocess.run", + return_value=docker_info_result, + ) + ) + stack.enter_context( + mock.patch( + f"{PRECONDITIONS_MODULE}.create_sdk_session", + return_value=mock.Mock(), + side_effect=session_side_effect, + ) + ) + yield _make_args(), project + + +def _matched_checks(failures): + """Return the set of check names whose message substring appears in ``failures``.""" + matched = set() + for name, substring in MESSAGE_SUBSTRINGS.items(): + if any(substring in message for message in failures): + matched.add(name) + return matched + + +# Feature: cfn-test-v2-flag, Property 3: For any subset of the precondition +# checks (Docker runtime, artifact package, credentials+region) forced to fail, +# check_preconditions returns a failure list whose messages correspond to +# exactly that subset - every unmet precondition is named and every met +# precondition is absent. +@settings(max_examples=200, suppress_health_check=[HealthCheck.function_scoped_fixture]) +@given(pass_flags=st.fixed_dictionaries({name: st.booleans() for name in CHECK_NAMES})) +def test_precondition_failures_aggregated_exactly(tmp_path, pass_flags): + """Validates: Requirements 3.7""" + expected_failures = {name for name, passed in pass_flags.items() if not passed} + + with configured_env(tmp_path, pass_flags) as (args, project): + failures = check_preconditions(args, project) + + # Every unmet precondition is named exactly once, and no met precondition is. + assert _matched_checks(failures) == expected_failures + assert len(failures) == len(expected_failures) + + +# --------------------------------------------------------------------------- +# Each precondition failing in isolation. +# --------------------------------------------------------------------------- + +ALL_PASS = dict.fromkeys(CHECK_NAMES, True) + + +def _states_with_only_failing(check): + states = dict(ALL_PASS) + states[check] = False + return states + + +def test_all_preconditions_met_returns_empty(tmp_path): + """Sanity baseline: when every check passes, no failures are returned.""" + with configured_env(tmp_path, dict(ALL_PASS)) as (args, project): + assert not check_preconditions(args, project) + + +def test_docker_unavailable_in_isolation(tmp_path): + """Requirement 3.2: Docker unavailable yields its message and blocks the run.""" + with configured_env(tmp_path, _states_with_only_failing("docker")) as ( + args, + project, + ): + failures = check_preconditions(args, project) + + assert len(failures) == 1 + assert MESSAGE_SUBSTRINGS["docker"] in failures[0] + # A non-empty failure list prevents the caller from ever running a container. + assert failures + + +def test_artifact_missing_in_isolation(tmp_path): + """Requirement 3.3: missing artifact package yields its message and blocks the run.""" + with configured_env(tmp_path, _states_with_only_failing("artifact")) as ( + args, + project, + ): + failures = check_preconditions(args, project) + + assert len(failures) == 1 + assert MESSAGE_SUBSTRINGS["artifact"] in failures[0] + assert failures + + +def test_credentials_unavailable_in_isolation(tmp_path): + """Requirement 3.5: missing credentials/region yields the message and blocks the run.""" + with configured_env(tmp_path, _states_with_only_failing("credentials")) as ( + args, + project, + ): + failures = check_preconditions(args, project) + + assert len(failures) == 1 + assert MESSAGE_SUBSTRINGS["credentials"] in failures[0] + assert failures + + +# --------------------------------------------------------------------------- +# Docker daemon ping failure modes (docker CLI present, daemon unreachable). +# --------------------------------------------------------------------------- + + +@contextlib.contextmanager +def docker_ping_env(work_dir, **run_kwargs): + """Yield ``(args, project)`` with docker on PATH and ``docker info`` stubbed. + + The artifact and credential checks are forced to pass, so any failure the + caller observes comes from the Docker daemon ping alone. ``run_kwargs`` is + forwarded to ``mock.patch`` for ``subprocess.run`` (``return_value`` for an + exit status, ``side_effect`` to raise). + """ + project = FakeProject(work_dir) + (work_dir / f"{project.hypenated_name}.zip").write_bytes(b"zip") + + with contextlib.ExitStack() as stack: + stack.enter_context( + mock.patch( + f"{PRECONDITIONS_MODULE}.shutil.which", return_value="/usr/bin/docker" + ) + ) + stack.enter_context( + mock.patch(f"{PRECONDITIONS_MODULE}.subprocess.run", **run_kwargs) + ) + stack.enter_context( + mock.patch( + f"{PRECONDITIONS_MODULE}.create_sdk_session", return_value=mock.Mock() + ) + ) + yield _make_args(), project + + +def test_docker_daemon_ping_nonzero_exit_reports_unreachable(tmp_path): + """docker CLI present but ``docker info`` exits non-zero -> unreachable daemon. + + Distinct from the missing-CLI case: the binary exists, so the ping itself is + what fails. + + Validates: Requirements 3.2 + """ + with docker_ping_env(tmp_path, return_value=mock.Mock(returncode=1)) as ( + args, + project, + ): + failures = check_preconditions(args, project) + + assert len(failures) == 1 + assert MESSAGE_SUBSTRINGS["docker"] in failures[0] + + +@pytest.mark.parametrize( + "ping_error", + [OSError("cannot exec"), subprocess.TimeoutExpired(cmd="docker info", timeout=10)], +) +def test_docker_daemon_ping_error_reports_unreachable(tmp_path, ping_error): + """A ping that raises (spawn failure or timeout) -> unreachable daemon. + + The check converts the exception into a message rather than propagating it, + so a hung or broken daemon still aggregates with other failures. + + Validates: Requirements 3.2 + """ + with docker_ping_env(tmp_path, side_effect=ping_error) as (args, project): + failures = check_preconditions(args, project) + + assert len(failures) == 1 + assert MESSAGE_SUBSTRINGS["docker"] in failures[0] diff --git a/tests/rqts/test_runner.py b/tests/rqts/test_runner.py new file mode 100644 index 00000000..17605ddd --- /dev/null +++ b/tests/rqts/test_runner.py @@ -0,0 +1,407 @@ +"""Tests for ``rpdk.core.rqts.runner``. + +Covers: +- Property 12: exit-code mapping (``map_exit_code``) over arbitrary exit codes. +- Orchestration guards and container spawn failure edge cases + (``run_container`` spawn failure, ``RqtsRunner._guard_artifact_type`` for hook + and indeterminate artifact types). +- Live output streaming integration for ``run_container`` using a fake child + process (no real Docker). +- Result reporting (``report_result``) pass/fail summaries and the + ``RqtsRunner.run`` DEBUG log of the full ``docker run`` command line. + +Docker and AWS are never actually invoked: ``subprocess.Popen`` is patched with +a deterministic fake, and the sibling functions imported into ``runner`` are +patched at ``rpdk.core.rqts.runner`` so the happy path exercises the +orchestration wiring without any external calls. + +Library: Hypothesis (the standard Python property-based testing library). The +property test runs at least 100 generated examples via +``@settings(max_examples=100)`` and is tagged with a comment referencing the +design property it validates. +""" +import logging +from types import SimpleNamespace +from unittest import mock + +import pytest +from hypothesis import given, settings, strategies as st + +from rpdk.core.exceptions import SysExitRecommendedError +from rpdk.core.project import ARTIFACT_TYPE_HOOK, ARTIFACT_TYPE_RESOURCE +from rpdk.core.rqts.runner import ( + FAIL_SUMMARY, + PASS_SUMMARY, + RqtsRunner, + map_exit_code, + report_result, + run_container, +) + +RUNNER_MODULE = "rpdk.core.rqts.runner" + + +# --------------------------------------------------------------------------- +# Fakes +# --------------------------------------------------------------------------- + + +class FakePopen: + """Deterministic stand-in for ``subprocess.Popen``. + + Supports the context-manager protocol used by ``run_container`` and records + that ``wait()`` was called before the return code is surfaced, so the test + can assert the child was awaited (i.e. streamed to completion) rather than + abandoned. Optionally emits incremental "live" output lines to a sink to + model streaming. + """ + + def __init__(self, argv, return_code=0, output_lines=None, output_sink=None): + self.argv = argv + self._return_code = return_code + self._output_lines = list(output_lines or []) + self._output_sink = output_sink + self.waited = False + self.entered = False + + def __enter__(self): + self.entered = True + return self + + def __exit__(self, *_exc): + return False + + def wait(self): + # Model live streaming: output is surfaced as the process runs, before + # wait() returns the exit code. + if self._output_sink is not None: + for line in self._output_lines: + self._output_sink.append(line) + self.waited = True + return self._return_code + + +def _make_resource_project(root): + """Build a minimal resource project the happy-path runner can drive.""" + return SimpleNamespace( + artifact_type=ARTIFACT_TYPE_RESOURCE, + type_name="AWS::Foo::Bar", + hypenated_name="aws-foo-bar", + root=root, + ) + + +def _make_args(): + return SimpleNamespace( + region="us-east-1", + profile=None, + role_arn=None, + source_account=None, + source_arn=None, + rqts_image=None, + ) + + +# =========================================================================== +# Property 12: exit-code mapping +# =========================================================================== +# Feature: cfn-test-v2-flag, Property 12: Exit code mapping +@settings(max_examples=100) +@given( + code=st.one_of( + st.just(0), + st.integers(min_value=1, max_value=255), + st.integers(min_value=-255, max_value=-1), + st.integers(), + ) +) +def test_property_12_exit_code_mapping(code): + """map_exit_code(0) returns None without raising; any non-zero code raises + SysExitRecommendedError. + + Validates: Requirements 5.2, 5.3, 6.3 + """ + if code == 0: + assert map_exit_code(code) is None + else: + with pytest.raises(SysExitRecommendedError): + map_exit_code(code) + + +# =========================================================================== +# Orchestration guards and spawn failure +# =========================================================================== +@pytest.mark.parametrize( + "spawn_error", [FileNotFoundError("no docker"), OSError("boom")] +) +def test_run_container_spawn_failure_raises(spawn_error): + """Container spawn failure -> SysExitRecommendedError naming the start failure. + + Validates: Requirements 5.4 + """ + with mock.patch(f"{RUNNER_MODULE}.subprocess.Popen", side_effect=spawn_error): + with pytest.raises(SysExitRecommendedError) as excinfo: + run_container(["docker", "run", "--rm", "image"]) + + assert "could not be started" in str(excinfo.value) + + +def test_guard_artifact_type_hook_rejected_and_does_not_proceed(): + """Hook project -> SysExitRecommendedError 'resources only'; pipeline halts. + + Validates: Requirements 7.2 + """ + project = SimpleNamespace(artifact_type=ARTIFACT_TYPE_HOOK) + rqts_runner = RqtsRunner(_make_args(), project) + + with mock.patch(f"{RUNNER_MODULE}.check_preconditions") as check, mock.patch( + f"{RUNNER_MODULE}.run_container" + ) as run: + with pytest.raises(SysExitRecommendedError) as excinfo: + rqts_runner.run() + + assert "resource types only" in str(excinfo.value) + # The guard fails fast: no downstream stage runs. + check.assert_not_called() + run.assert_not_called() + + +def test_guard_artifact_type_indeterminate_rejected_and_does_not_proceed(): + """Indeterminate artifact type -> 'could not determine' error; pipeline halts. + + Validates: Requirements 7.5 + """ + project = SimpleNamespace(artifact_type=None) + rqts_runner = RqtsRunner(_make_args(), project) + + with mock.patch(f"{RUNNER_MODULE}.check_preconditions") as check, mock.patch( + f"{RUNNER_MODULE}.run_container" + ) as run: + with pytest.raises(SysExitRecommendedError) as excinfo: + rqts_runner.run() + + assert "could not determine the project artifact type" in str(excinfo.value) + check.assert_not_called() + run.assert_not_called() + + +# =========================================================================== +# Live output streaming integration +# =========================================================================== +def test_run_container_streams_output_and_returns_code(): + """run_container surfaces incremental child output and returns the exit code. + + A fake child process emits incremental lines as it runs (before wait() + returns), modelling the inherited-stdio live streaming; run_container + returns the child's exit code. + + Validates: Requirements 5.1 + """ + argv = [ + "docker", + "run", + "--rm", + "image:tag", + "--extension", + "contract-tests", + "run-tests", + "/work/aws-foo-bar.zip", + "--direct-jar", + ] + streamed = [] + process = FakePopen( + argv, + return_code=0, + output_lines=["scenario create: PASS", "scenario delete: PASS"], + output_sink=streamed, + ) + + def factory(passed_argv, *_args, **_kwargs): + # run_container must spawn docker with exactly the argv it was given. + assert passed_argv == argv + return process + + with mock.patch(f"{RUNNER_MODULE}.subprocess.Popen", side_effect=factory): + code = run_container(argv) + + assert code == 0 + # Output was surfaced live (during the run) and the child was awaited. + assert streamed == ["scenario create: PASS", "scenario delete: PASS"] + assert process.waited is True + + +def test_run_container_returns_nonzero_code(): + """run_container returns a non-zero child exit code unchanged (no raise).""" + argv = ["docker", "run", "--rm", "image:tag"] + + with mock.patch( + f"{RUNNER_MODULE}.subprocess.Popen", + side_effect=lambda passed_argv, *a, **k: FakePopen(passed_argv, return_code=7), + ): + assert run_container(argv) == 7 + + +def test_run_container_merges_env_over_ambient_environment(monkeypatch): + """run_container spawns docker with the supplied env merged over os.environ, + and inherits the ambient environment untouched (env=None) when none is given. + + This is the delivery half of the name-only ``-e`` contract: credential + values reach docker exclusively through the process environment. + """ + argv = ["docker", "run", "--rm", "image:tag"] + captured = {} + + def factory(passed_argv, *_args, **kwargs): + captured["env"] = kwargs.get("env") + return FakePopen(passed_argv, return_code=0) + + monkeypatch.setenv("SOME_AMBIENT_VAR", "ambient") + + with mock.patch(f"{RUNNER_MODULE}.subprocess.Popen", side_effect=factory): + run_container(argv, env={"AWS_ACCESS_KEY_ID": "AKID"}) + assert captured["env"]["AWS_ACCESS_KEY_ID"] == "AKID" + assert captured["env"]["SOME_AMBIENT_VAR"] == "ambient" + + with mock.patch(f"{RUNNER_MODULE}.subprocess.Popen", side_effect=factory): + run_container(argv) + assert captured["env"] is None + + +# =========================================================================== +# Result reporting and DEBUG logging +# =========================================================================== +def test_report_result_pass_logs_summary_and_does_not_raise(caplog): + """report_result(0) logs PASS_SUMMARY at INFO and does not raise. + + Validates: Requirements 6.1 + """ + with caplog.at_level(logging.INFO, logger=RUNNER_MODULE): + assert report_result(0) is None + + assert PASS_SUMMARY in caplog.text + + +def test_report_result_fail_raises_with_fail_summary(): + """report_result(non-zero) raises SysExitRecommendedError with FAIL_SUMMARY. + + Validates: Requirements 6.2 + """ + with pytest.raises(SysExitRecommendedError) as excinfo: + report_result(1) + + assert str(excinfo.value) == FAIL_SUMMARY + + +def test_run_logs_full_docker_command_at_debug_on_happy_path(tmp_path, caplog): + """RqtsRunner.run logs the full docker run command at DEBUG and passes. + + A fully-mocked happy path (no preconditions failures, stubbed credentials, + image resolve/ensure, and a zero container exit code) drives the runner and + asserts the full docker command line is emitted at DEBUG and the run reports + a pass. It also asserts check_preconditions is called with (args, project). + + Validates: Requirements 4.10, 6.1 + """ + project = _make_resource_project(str(tmp_path)) + args = _make_args() + rqts_runner = RqtsRunner(args, project) + docker_argv = [ + "docker", + "run", + "--rm", + "image:tag", + "--extension", + "contract-tests", + "run-tests", + "/work/aws-foo-bar.zip", + "--direct-jar", + ] + + creds = { + "aws_access_key_id": "AKID", + "aws_secret_access_key": "SECRET", + "aws_session_token": "TOKEN", + } + + with mock.patch( + f"{RUNNER_MODULE}.check_preconditions", return_value=[] + ) as check, mock.patch(f"{RUNNER_MODULE}.create_sdk_session"), mock.patch( + f"{RUNNER_MODULE}.get_temporary_credentials", return_value=creds + ), mock.patch( + f"{RUNNER_MODULE}.resolve_image", return_value="image:tag" + ), mock.patch( + f"{RUNNER_MODULE}.ensure_image" + ), mock.patch( + f"{RUNNER_MODULE}.build_docker_argv", return_value=docker_argv + ), mock.patch( + f"{RUNNER_MODULE}.run_container", return_value=0 + ) as run: + with caplog.at_level(logging.DEBUG, logger=RUNNER_MODULE): + # Returns normally (pass) on a zero container exit code. + assert rqts_runner.run() is None + + # check_preconditions is called with exactly (args, project) - no host arg. + check.assert_called_once_with(args, project) + # run_container receives the credential-free argv plus the env mapping that + # carries the credential values (never present in the argv itself). + run.assert_called_once_with( + docker_argv, + env={ + "AWS_ACCESS_KEY_ID": "AKID", + "AWS_SECRET_ACCESS_KEY": "SECRET", + "AWS_SESSION_TOKEN": "TOKEN", + "AWS_REGION": args.region, + }, + ) + # The DEBUG-logged command line contains no credential values. + for record in caplog.records: + for secret in creds.values(): + assert secret not in record.getMessage() + # The host-side output directory was created under the project root. + assert (tmp_path / "rqts-output").is_dir() + # The full docker run command line is logged at DEBUG (Req 4.10). + debug_records = [ + record.getMessage() + for record in caplog.records + if record.levelno == logging.DEBUG + ] + joined_command = " ".join(docker_argv) + assert any(joined_command in message for message in debug_records) + # The pass summary is surfaced (Req 6.1). + assert PASS_SUMMARY in caplog.text + + +# =========================================================================== +# Precondition enforcement inside the pipeline +# =========================================================================== +def test_run_unmet_preconditions_aggregated_and_halts(tmp_path): + """Unmet preconditions -> a single error naming every failure; nothing runs. + + The runner turns the aggregated list from ``check_preconditions`` into one + ``SysExitRecommendedError`` instead of failing on the first problem, and + halts before the image is ensured or any container is started. + + Validates: Requirements 3.1, 3.5 + """ + project = _make_resource_project(str(tmp_path)) + rqts_runner = RqtsRunner(_make_args(), project) + failures = [ + "Docker is required and must be running: the Docker daemon could not " + "be reached.", + "artifact package 'aws-foo-bar.zip' not found; build the project first.", + ] + + with mock.patch( + f"{RUNNER_MODULE}.check_preconditions", return_value=failures + ), mock.patch(f"{RUNNER_MODULE}.ensure_image") as ensure, mock.patch( + f"{RUNNER_MODULE}.run_container" + ) as run: + with pytest.raises(SysExitRecommendedError) as excinfo: + rqts_runner.run() + + message = str(excinfo.value) + assert "preconditions were not met" in message + for failure in failures: + assert failure in message + ensure.assert_not_called() + run.assert_not_called() diff --git a/tests/test_test.py b/tests/test_test.py index 031feb83..be1ea927 100644 --- a/tests/test_test.py +++ b/tests/test_test.py @@ -1,5 +1,6 @@ # fixture and parameter have the same name -# pylint: disable=protected-access,redefined-outer-name +# pylint: disable=protected-access,redefined-outer-name,too-many-lines +import argparse import json import os from contextlib import contextmanager @@ -31,6 +32,7 @@ get_marker_options, get_overrides, get_type, + setup_subparser, temporary_ini_file, ) from rpdk.core.utils.handler_utils import generate_handler_name @@ -838,3 +840,193 @@ def test_input_files_read_safely(self, base): result = get_inputs(base, DEFAULT_REGION, DEFAULT_ENDPOINT, 1, None, None, {}) assert result is not None and "CREATE" in result + + +# --v2 flag registration and RQTS routing tests +# (Task 8.3: flag registration; Task 8.4: routing + backward compatibility) + + +def _build_test_parser(): + """Build a parser wired up exactly the way cli.py does for the test command. + + Mirrors rpdk.core.cli.main: a root parser with a shared base subparser + (providing -v/--verbose) as the only parent, and the test subcommand + registered via setup_subparser. + """ + parser = argparse.ArgumentParser() + base_subparser = argparse.ArgumentParser(add_help=False) + base_subparser.add_argument("-v", "--verbose", action="count", default=0) + subparsers = parser.add_subparsers(dest="subparser_name") + setup_subparser(subparsers, [base_subparser]) + return parser, subparsers + + +# Task 8.3 - Requirements 1.1, 1.5, 4.11 + + +def test_v2_flag_defaults_to_false(): + parser, _subparsers = _build_test_parser() + args = parser.parse_args(["test"]) + assert args.v2 is False + + +def test_v2_flag_present_sets_true(): + parser, _subparsers = _build_test_parser() + args = parser.parse_args(["test", "--v2"]) + assert args.v2 is True + + +def test_rqts_image_defaults_to_none(): + parser, _subparsers = _build_test_parser() + args = parser.parse_args(["test"]) + assert args.rqts_image is None + + +def test_rqts_image_can_be_overridden(): + parser, _subparsers = _build_test_parser() + args = parser.parse_args(["test", "--rqts-image", "foo:bar"]) + assert args.rqts_image == "foo:bar" + + +def test_v2_help_text_identifies_opt_in_rqts_runner(): + _parser, subparsers = _build_test_parser() + help_text = subparsers.choices["test"].format_help() + lowered = help_text.lower() + assert "opt-in" in lowered + assert "rqts local test runner" in lowered + + +def test_no_scenario_selection_option_on_parser(): + parser, _subparsers = _build_test_parser() + # Scenario selection is owned by the executor image and is not + # user-selectable through the CLI (Req 4.11); a scenario-selection option + # is not a recognized argument. + with pytest.raises(SystemExit): + parser.parse_args(["test", "--scenarios", "x"]) + + +# Task 8.4 - Requirements 1.2, 1.3, 1.4, 7.1, 7.3, 7.4 + + +def test_v2_absent_uses_pytest_path_and_does_not_construct_runner(base): + create_input_file(base, '{"a": 1}', '{"a": 2}', '{"b": 1}') + mock_project = Mock(spec=Project) + mock_project.schema = RESOURCE_SCHEMA + mock_project.root = base + mock_project.executable_entrypoint = None + mock_project.artifact_type = ARTIFACT_TYPE_RESOURCE + + patch_project = patch( + "rpdk.core.test.Project", autospec=True, return_value=mock_project + ) + patch_plugin = patch("rpdk.core.test.ContractPlugin", autospec=True) + patch_resource_client = patch("rpdk.core.test.ResourceClient", autospec=True) + patch_pytest = patch("rpdk.core.test.pytest.main", autospec=True, return_value=0) + patch_ini = patch( + "rpdk.core.test.temporary_ini_file", side_effect=mock_temporary_ini_file + ) + patch_runner = patch("rpdk.core.rqts.runner.RqtsRunner", autospec=True) + # fmt: off + with patch_project, \ + patch_plugin, \ + patch_resource_client, \ + patch_ini, \ + patch_pytest as mock_pytest, \ + patch_runner as mock_runner: + main(args_in=["test"]) + # fmt: on + + # The pytest path is exercised and the RQTS runner is never constructed. + mock_pytest.assert_called_once() + mock_runner.assert_not_called() + + +def test_v2_absent_nonzero_pytest_return_still_raises(base): + create_input_file(base, '{"a": 1}', '{"a": 2}', '{"b": 1}') + mock_project = Mock(spec=Project) + mock_project.schema = RESOURCE_SCHEMA + mock_project.root = base + mock_project.executable_entrypoint = None + mock_project.artifact_type = ARTIFACT_TYPE_RESOURCE + + patch_project = patch( + "rpdk.core.test.Project", autospec=True, return_value=mock_project + ) + patch_plugin = patch("rpdk.core.test.ContractPlugin", autospec=True) + patch_resource_client = patch("rpdk.core.test.ResourceClient", autospec=True) + patch_pytest = patch("rpdk.core.test.pytest.main", autospec=True, return_value=1) + patch_ini = patch( + "rpdk.core.test.temporary_ini_file", side_effect=mock_temporary_ini_file + ) + patch_runner = patch("rpdk.core.rqts.runner.RqtsRunner", autospec=True) + # fmt: off + with patch_project, \ + patch_plugin, \ + patch_resource_client, \ + patch_ini, \ + patch_pytest as mock_pytest, \ + patch_runner as mock_runner: + with pytest.raises(SystemExit) as excinfo: + main(args_in=["test"]) + # fmt: on + + # Existing backward-compatible behavior: a non-zero pytest return raises + # SysExitRecommendedError, mapped by cli.py to a non-unhandled SystemExit. + assert excinfo.value.code != EXIT_UNHANDLED_EXCEPTION + # The --v2 branch did not alter forwarding to the pytest path. + mock_pytest.assert_called_once() + mock_runner.assert_not_called() + + +def test_v2_resource_project_constructs_and_runs_runner(base): + mock_project = Mock(spec=Project) + mock_project.schema = RESOURCE_SCHEMA + mock_project.root = base + mock_project.executable_entrypoint = None + mock_project.artifact_type = ARTIFACT_TYPE_RESOURCE + + patch_project = patch( + "rpdk.core.test.Project", autospec=True, return_value=mock_project + ) + patch_pytest = patch("rpdk.core.test.pytest.main", autospec=True, return_value=0) + # test() imports RqtsRunner locally (from .rqts.runner import RqtsRunner), + # so patch it at its source module. + patch_runner = patch("rpdk.core.rqts.runner.RqtsRunner", autospec=True) + # fmt: off + with patch_project, \ + patch_pytest as mock_pytest, \ + patch_runner as mock_runner: + main(args_in=["test", "--v2"]) + # fmt: on + + # The runner is constructed with the parsed args and loaded project, and + # run() is invoked exactly once; the pytest path is not exercised. + mock_runner.assert_called_once() + called_args = mock_runner.call_args[0] + assert called_args[1] is mock_project + mock_runner.return_value.run.assert_called_once_with() + mock_pytest.assert_not_called() + + +def test_v2_module_project_warns_and_does_not_construct_runner(capsys): + mock_project = Mock(spec=Project) + mock_project.artifact_type = ARTIFACT_TYPE_MODULE + + patch_project = patch( + "rpdk.core.test.Project", autospec=True, return_value=mock_project + ) + patch_pytest = patch("rpdk.core.test.pytest.main", autospec=True, return_value=0) + patch_runner = patch("rpdk.core.rqts.runner.RqtsRunner", autospec=True) + # fmt: off + with patch_project, \ + patch_pytest as mock_pytest, \ + patch_runner as mock_runner: + # The module short-circuit precedes the --v2 branch: clean return, no + # SystemExit raised. + main(args_in=["test", "--v2"]) + # fmt: on + + mock_runner.assert_not_called() + mock_pytest.assert_not_called() + out, err = capsys.readouterr() + assert "module" in (out + err).lower()