From 85a9e722c443d502ee7b4f51d2f88c0e65acd332 Mon Sep 17 00:00:00 2001 From: jsong468 Date: Fri, 14 Aug 2026 15:58:40 -0700 Subject: [PATCH 1/4] subcommands --- .pyrit_conf_example | 2 +- doc/blog/2026_07_09_scenarios.md | 2 +- .../2_custom_scenario_parameters.ipynb | 21 +- .../scenarios/2_custom_scenario_parameters.py | 21 +- doc/getting_started/pyrit_conf.md | 8 +- doc/index.md | 2 +- doc/scanner/0_scanner.md | 2 +- doc/scanner/1_pyrit_scan.ipynb | 585 +++++++++--------- doc/scanner/1_pyrit_scan.py | 65 +- doc/scanner/2_pyrit_shell.md | 9 +- doc/scanner/airt.ipynb | 12 +- doc/scanner/airt.py | 12 +- doc/scanner/benchmark.ipynb | 4 +- doc/scanner/benchmark.py | 4 +- doc/scanner/foundry.ipynb | 2 +- doc/scanner/foundry.py | 2 +- doc/scanner/garak.ipynb | 6 +- doc/scanner/garak.py | 6 +- pyrit/cli/_cli_args.py | 19 +- pyrit/cli/_output.py | 2 +- pyrit/cli/pyrit_scan.py | 540 ++++++++-------- tests/unit/cli/test_pyrit_scan.py | 311 ++++++---- tests/unit/cli/test_results.py | 18 +- 23 files changed, 889 insertions(+), 766 deletions(-) diff --git a/.pyrit_conf_example b/.pyrit_conf_example index b1b1ecf932..b41c13e060 100644 --- a/.pyrit_conf_example +++ b/.pyrit_conf_example @@ -33,7 +33,7 @@ memory_db_type: sqlite # - A dictionary with 'name' and optional 'args' for parameters # # Parameters are lists of strings. Use the CLI command -# `pyrit_scan --list-initializers` to see available parameters. +# `pyrit_scan list-initializers` to see available parameters. # # Example: # initializers: diff --git a/doc/blog/2026_07_09_scenarios.md b/doc/blog/2026_07_09_scenarios.md index e866db609d..735aca3c4c 100644 --- a/doc/blog/2026_07_09_scenarios.md +++ b/doc/blog/2026_07_09_scenarios.md @@ -25,7 +25,7 @@ Open one up and you'll find the same four things working together: a **dataset o Let's say you want a wide read on a new target. The broadest scenario in the catalog is `RapidResponse` — a comprehensive sweep across the most common attack techniques and the full AIRT harm-category catalog. The [**Scanner**](../scanner/0_scanner.md) — PyRIT's single-command entry point for running any scenario — makes it one line: ```bash -pyrit_scan airt.rapid_response --target my_target +pyrit_scan run airt.rapid_response --target my_target ``` That one command does a lot. Behind the scenes, initializers populate the registries (techniques, targets, datasets); the CLI resolves `airt.rapid_response` and `my_target`, instantiates `RapidResponse`, and runs it. Out of the box (using the default configuration for techniques ie `--techniques default`) it sends `role_play` and `many_shot` attacks plus a baseline pass — across seven AIRT harm categories: hate, fairness, violence, sexual, harassment, misinformation, leakage. Switch to `--techniques single_turn` to swap in the single-turn pool — `role_play`, `context_compliance`, `crescendo_simulated`, plus the persona-driven crescendo variants (`crescendo_movie_director`, `crescendo_history_lecture`, `crescendo_journalist_interview`). `--techniques multi_turn` picks up the multi-turn pool instead: `many_shot`, `tap`, `pair`, and `red_teaming`. diff --git a/doc/code/scenarios/2_custom_scenario_parameters.ipynb b/doc/code/scenarios/2_custom_scenario_parameters.ipynb index 174fd2eca3..f721a4b70a 100644 --- a/doc/code/scenarios/2_custom_scenario_parameters.ipynb +++ b/doc/code/scenarios/2_custom_scenario_parameters.ipynb @@ -101,7 +101,7 @@ "Each `Parameter` carries:\n", "\n", "- **name**: dict key in `self.params`, converted to `--kebab-case` for the CLI\n", - "- **description**: shown in `--list-scenarios` and `--help`\n", + "- **description**: shown in `list-scenarios` and `--help`\n", "- **default**: value used when not supplied; deep-copied per run\n", "- **param_type**: `str`, `int`, `float`, `bool`, a `Literal[...]`/`Enum` (a\n", " constrained scalar that carries its own allowed set), a `list[...]` of any of\n", @@ -197,10 +197,10 @@ "\n", "```bash\n", "# Use the declared default (5)\n", - "pyrit_scan airt.scam --target my_target --initializers target\n", + "pyrit_scan run airt.scam --target my_target --initializers target\n", "\n", "# Override\n", - "pyrit_scan airt.scam --target my_target --initializers target --max-turns 10\n", + "pyrit_scan run airt.scam --target my_target --initializers target --max-turns 10\n", "```\n", "\n", "The same flags work in `pyrit_shell`:\n", @@ -209,18 +209,13 @@ "pyrit_shell> run airt.scam --target my_target --initializers target --max-turns 10\n", "```\n", "\n", - "Declared flags also show up in `pyrit_scan --help`, alongside\n", - "the built-in options:\n", + "Scenario-declared flags don't appear in `pyrit_scan run --help`\n", + "(that shows only the built-in run options); use `list-scenarios` to discover\n", + "a scenario's parameters.\n", "\n", - "```bash\n", - "pyrit_scan airt.scam --help\n", - "# ...\n", - "# --max-turns MAX_TURNS Conversation turn cap\n", - "```\n", - "\n", - "## Discovering parameters via --list-scenarios\n", + "## Discovering parameters via `list-scenarios`\n", "\n", - "`--list-scenarios` prints declared parameters alongside each scenario's\n", + "`pyrit_scan list-scenarios` prints declared parameters alongside each scenario's\n", "other metadata (description, techniques, datasets). The same formatter the\n", "CLI uses is callable programmatically:" ] diff --git a/doc/code/scenarios/2_custom_scenario_parameters.py b/doc/code/scenarios/2_custom_scenario_parameters.py index 99dd6c9bf8..6cc0339fa6 100644 --- a/doc/code/scenarios/2_custom_scenario_parameters.py +++ b/doc/code/scenarios/2_custom_scenario_parameters.py @@ -67,7 +67,7 @@ # Each `Parameter` carries: # # - **name**: dict key in `self.params`, converted to `--kebab-case` for the CLI -# - **description**: shown in `--list-scenarios` and `--help` +# - **description**: shown in `list-scenarios` and `--help` # - **default**: value used when not supplied; deep-copied per run # - **param_type**: `str`, `int`, `float`, `bool`, a `Literal[...]`/`Enum` (a # constrained scalar that carries its own allowed set), a `list[...]` of any of @@ -135,10 +135,10 @@ # # ```bash # # Use the declared default (5) -# pyrit_scan airt.scam --target my_target --initializers target +# pyrit_scan run airt.scam --target my_target --initializers target # # # Override -# pyrit_scan airt.scam --target my_target --initializers target --max-turns 10 +# pyrit_scan run airt.scam --target my_target --initializers target --max-turns 10 # ``` # # The same flags work in `pyrit_shell`: @@ -147,18 +147,13 @@ # pyrit_shell> run airt.scam --target my_target --initializers target --max-turns 10 # ``` # -# Declared flags also show up in `pyrit_scan --help`, alongside -# the built-in options: +# Scenario-declared flags don't appear in `pyrit_scan run --help` +# (that shows only the built-in run options); use `list-scenarios` to discover +# the full list of a scenario's parameters. # -# ```bash -# pyrit_scan airt.scam --help -# # ... -# # --max-turns MAX_TURNS Conversation turn cap -# ``` -# -# ## Discovering parameters via --list-scenarios +# ## Discovering parameters via `list-scenarios` # -# `--list-scenarios` prints declared parameters alongside each scenario's +# `pyrit_scan list-scenarios` prints declared parameters alongside each scenario's # other metadata (description, techniques, datasets). The same formatter the # CLI uses is callable programmatically: diff --git a/doc/getting_started/pyrit_conf.md b/doc/getting_started/pyrit_conf.md index 878246c140..30fd1443a9 100644 --- a/doc/getting_started/pyrit_conf.md +++ b/doc/getting_started/pyrit_conf.md @@ -182,7 +182,7 @@ Client settings for connecting to or launching a PyRIT backend. | Field | Description | Default | |---|---|---| | `url` | Backend URL used when `--server-url` is omitted | `http://localhost:8000` | -| `startup_timeout` | Seconds `pyrit_scan --start-server` waits for a healthy backend before terminating the spawned process | `120` | +| `startup_timeout` | Seconds `pyrit_scan start-server` waits for a healthy backend before terminating the spawned process | `120` | `startup_timeout` must be a finite number greater than zero. The `--startup-timeout` CLI option overrides the configured value for an individual scanner invocation. @@ -208,7 +208,7 @@ flowchart LR | -------- | ---------------------- | ----------------------------------------------------------------------- | | Lowest | `~/.pyrit/.pyrit_conf` | Loaded automatically if it exists | | Medium | Explicit config file | Passed via `--config-file` (CLI) or `config_file` parameter | -| Highest | Individual arguments | CLI flags like `--database`, `--initializers`, or API keyword arguments | +| Highest | Individual arguments | CLI flags like `--initializers` or API keyword arguments | This means you can set sensible defaults in `~/.pyrit/.pyrit_conf` and override specific values on a per-run basis without modifying the file. @@ -230,10 +230,10 @@ Because initializers run last, they can modify anything set up in earlier steps The CLI and shell automatically load `~/.pyrit/.pyrit_conf`. You can also point to a different config file: ```bash -pyrit_scan run --config-file ./my_project_config.yaml --database InMemory +pyrit_scan run airt.scam --config-file ./my_project_config.yaml ``` -Individual CLI arguments (like `--database`) override values from the config file. +Individual CLI arguments (like `--initializers`) override values from the config file. ### From Python diff --git a/doc/index.md b/doc/index.md index f7a353604e..238b81c599 100644 --- a/doc/index.md +++ b/doc/index.md @@ -125,7 +125,7 @@ initializers: Run security assessments from the command line with `pyrit_scan` or the interactive `pyrit_shell`. Execute built-in scenarios against your AI targets. ```bash -pyrit_scan airt.scam --target openai_chat +pyrit_scan run airt.scam --target openai_chat ``` ![scanner-demo](scanner-demo.png) diff --git a/doc/scanner/0_scanner.md b/doc/scanner/0_scanner.md index 07dfcd4974..2783de46df 100644 --- a/doc/scanner/0_scanner.md +++ b/doc/scanner/0_scanner.md @@ -23,7 +23,7 @@ PyRIT provides two command-line interfaces: ```bash # Run the Foundry RedTeamAgent scenario against your configured target -pyrit_scan foundry.red_team_agent --target openai_chat --initializers target --techniques base64 +pyrit_scan run foundry.red_team_agent --target openai_chat --initializers target --techniques base64 ``` ## Built-in Scenarios diff --git a/doc/scanner/1_pyrit_scan.ipynb b/doc/scanner/1_pyrit_scan.ipynb index af11f40ef9..61a2688d63 100644 --- a/doc/scanner/1_pyrit_scan.ipynb +++ b/doc/scanner/1_pyrit_scan.ipynb @@ -33,14 +33,12 @@ "name": "stdout", "output_type": "stream", "text": [ - "Starting server at http://localhost:8000...\n", - "Server ready (PID 45120)\n", "Server is running at http://localhost:8000\n" ] } ], "source": [ - "!pyrit_scan --start-server" + "!pyrit_scan start-server" ] }, { @@ -63,136 +61,69 @@ "name": "stdout", "output_type": "stream", "text": [ - "usage: pyrit_scan [-h] [--server-url SERVER_URL] [--start-server]\n", - " [--stop-server] [--config-file CONFIG_FILE]\n", - " [--log-level LOG_LEVEL] [--request-timeout REQUEST_TIMEOUT]\n", - " [--list-scenarios] [--list-initializers] [--list-targets]\n", - " [--list-converters] [--list-datasets]\n", - " [--add-initializer FILE [FILE ...]] [--target TARGET]\n", - " [--initializers INITIALIZERS [INITIALIZERS ...]]\n", - " [--techniques SCENARIO_TECHNIQUES [SCENARIO_TECHNIQUES ...]]\n", - " [--max-concurrency MAX_CONCURRENCY]\n", - " [--max-retries MAX_RETRIES] [--memory-labels MEMORY_LABELS]\n", - " [--dataset-names DATASET_NAMES [DATASET_NAMES ...]]\n", - " [--max-dataset-size MAX_DATASET_SIZE]\n", - " [scenario_name]\n", + "usage: pyrit_scan [-h] ...\n", "\n", "PyRIT Scanner - Run AI security scenarios from the command line.\n", "\n", - "Requires a running PyRIT backend server. Use --start-server to launch one,\n", + "Requires a running PyRIT backend server. Use 'start-server' to launch one,\n", "or connect to an existing server with --server-url.\n", "\n", + "Global options (usable with any command, before or after the verb):\n", + " --server-url --config-file --log-level --request-timeout --start-server --startup-timeout\n", + "Run 'pyrit_scan --help' for full option descriptions and a command's arguments.\n", + "\n", "Examples:\n", " # Start the backend server\n", - " pyrit_scan --start-server\n", - "\n", - " # List scenarios, initializers, targets, or converters\n", - " pyrit_scan --list-scenarios\n", - " pyrit_scan --list-initializers\n", - " pyrit_scan --list-targets\n", - " pyrit_scan --list-converters\n", + " pyrit_scan start-server\n", "\n", - " # List available datasets\n", - " pyrit_scan --list-datasets\n", + " # List scenarios, targets, or converters\n", + " pyrit_scan list-scenarios\n", + " pyrit_scan list-targets\n", "\n", " # Run single-turn cyber attacks against a target\n", - " pyrit_scan airt.cyber --target openai_chat --techniques single_turn\n", + " pyrit_scan run airt.cyber --target openai_chat --techniques single_turn\n", "\n", " # Run rapid response with specific datasets and concurrency\n", - " pyrit_scan airt.rapid_response --target openai_chat\n", - " --techniques role_play --dataset-names airt_hate\n", + " pyrit_scan run airt.rapid_response --target openai_chat\n", + " --techniques role_play_movie_script --dataset-names airt_hate\n", " --max-dataset-size 5 --max-concurrency 4\n", "\n", " # Attach registered converters to a technique (repeatable, applied in order)\n", - " pyrit_scan airt.rapid_response --target openai_chat\n", - " --techniques role_play:converter.translation_spanish:converter.leetspeak\n", + " pyrit_scan run airt.rapid_response --target openai_chat\n", + " --techniques role_play_movie_script:converter.translation_spanish:converter.leetspeak\n", "\n", - " # Run multi-turn red team agent with labels for tracking\n", - " pyrit_scan airt.red_team_agent --target openai_chat\n", - " --techniques crescendo\n", - " --memory-labels '{\"experiment\":\"baseline\"}'\n", + " # List recent runs, then inspect one (overview by default; --view attacks for per-attack rows)\n", + " pyrit_scan scenario-history 20\n", + " pyrit_scan scenario-results 605d715b-7c07-4bde-a8f9-22fea0b50c4f --view attacks\n", "\n", " # Register a custom initializer from a Python script\n", - " pyrit_scan --add-initializer ./my_custom_init.py\n", + " pyrit_scan add-initializer ./my_custom_init.py\n", "\n", " # Connect to a remote server\n", - " pyrit_scan --server-url http://remote:8000 --list-scenarios\n", + " pyrit_scan list-scenarios --server-url http://remote:8000\n", "\n", " # Stop the server\n", - " pyrit_scan --stop-server\n", + " pyrit_scan stop-server\n", "\n", "options:\n", - " -h, --help show this help message and exit\n", - "\n", - "server:\n", - " --server-url SERVER_URL\n", - " URL of the PyRIT backend server (default:\n", - " http://localhost:8000)\n", - " --start-server Start a local backend server if one is not already\n", - " running\n", - " --stop-server Stop the backend server and exit\n", - " --config-file CONFIG_FILE\n", - " Path to a YAML configuration file. Allows specifying\n", - " database, initializers (with args), initialization\n", - " scripts, and env files. CLI arguments override config\n", - " file values. If not specified, ~/.pyrit/.pyrit_conf is\n", - " loaded if it exists.\n", - " --log-level LOG_LEVEL\n", - " Logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL)\n", - " (default: WARNING)\n", - " --request-timeout REQUEST_TIMEOUT\n", - " HTTP read timeout in seconds for non-polling server\n", - " requests (catalog/results/cancel/etc). Defaults to 60.\n", - " Polling a live scenario run always waits indefinitely\n", - " regardless of this value.\n", + " -h, --help show this help message and exit\n", "\n", - "discovery:\n", - " --list-scenarios List all available scenarios and exit\n", - " --list-initializers List all available initializers and exit\n", - " --list-targets List all available targets and exit\n", - " --list-converters List all registered converter instances and exit\n", - " --list-datasets List all available datasets and exit\n", - " --add-initializer FILE [FILE ...]\n", - " Register initializer(s) from Python script file(s) and\n", - " exit\n", - "\n", - "scenario run:\n", - " scenario_name Name of the scenario to run\n", - " --target TARGET Name of a registered target from the TargetRegistry to\n", - " use as the objective target. Targets are registered by\n", - " initializers (e.g., 'target' initializer). Use --list-\n", - " targets to see available target names after\n", - " initializers have run\n", - " --initializers INITIALIZERS [INITIALIZERS ...]\n", - " Built-in initializer names to run before the scenario.\n", - " Supports optional params with name:key=val syntax\n", - " (e.g., target:tags=default,scorer dataset:mode=strict)\n", - " --techniques, -t SCENARIO_TECHNIQUES [SCENARIO_TECHNIQUES ...]\n", - " List of technique names to run (e.g., base64 rot13).\n", - " Append one or more registered converters to a\n", - " technique with ':converter.' (repeatable), e.g. \n", - " role_play:converter.translation_spanish:converter.leet\n", - " speak. The converter is appended on top of the\n", - " technique's built-in converters. Use --list-converters\n", - " to see registered converter names\n", - " --max-concurrency MAX_CONCURRENCY\n", - " Maximum number of concurrent attack executions (must\n", - " be >= 1)\n", - " --max-retries MAX_RETRIES\n", - " Maximum number of automatic retries on exception (must\n", - " be >= 0)\n", - " --memory-labels MEMORY_LABELS\n", - " Additional labels as JSON string (e.g.,\n", - " '{\"experiment\": \"test1\"}')\n", - " --dataset-names DATASET_NAMES [DATASET_NAMES ...]\n", - " List of dataset names to use instead of scenario\n", - " defaults (e.g., harmbench advbench). Creates a new\n", - " dataset config; fetches all items unless --max-\n", - " dataset-size is also specified\n", - " --max-dataset-size MAX_DATASET_SIZE\n", - " Maximum number of items to use from the dataset (must\n", - " be >= 1). Limits new datasets if --dataset-names\n", - " provided, otherwise overrides scenario's default limit\n" + "commands:\n", + " \n", + " run Run a scenario against a target\n", + " list-scenarios List all available scenarios\n", + " list-initializers\n", + " List all available initializers\n", + " list-targets List all available targets\n", + " list-converters List all registered converter instances\n", + " list-datasets List all available datasets\n", + " add-initializer Register initializer(s) from Python script file(s)\n", + " scenario-results\n", + " Inspect the results of a completed scenario run\n", + " scenario-history\n", + " List recent scenario runs\n", + " start-server Start a local backend server\n", + " stop-server Stop the backend server\n" ] } ], @@ -232,11 +163,13 @@ " ``prompt_sending`` runs as the baseline comparison and is excluded from\n", " the adaptive technique pool.\n", " Aggregate Techniques:\n", - " - all, default, single_turn, multi_turn\n", - " Available Techniques (11):\n", - " role_play, many_shot, tap, crescendo_simulated, red_teaming,\n", - " context_compliance, crescendo_movie_director, crescendo_history_lecture,\n", - " crescendo_journalist_interview, pair, violent_durian\n", + " - all, default, core, extra, light, multi_turn, single_turn\n", + " Available Techniques (17):\n", + " role_play_movie_script, role_play_video_game, role_play_trivia_game,\n", + " role_play_persuasion, role_play_persuasion_written, many_shot, tap,\n", + " crescendo_simulated, crescendo_movie_director,\n", + " crescendo_history_lecture, crescendo_journalist_interview, red_teaming,\n", + " context_compliance, flip, pair, skeleton_key, violent_durian\n", " Default Technique: default\n", " Default Datasets (7):\n", " airt_hate, airt_fairness, airt_violence, airt_sexual, airt_harassment,\n", @@ -260,10 +193,15 @@ " Cyber class contains different variations of the malware generation\n", " techniques.\n", " Aggregate Techniques:\n", - " - all, multi_turn\n", - " Available Techniques (1):\n", - " red_teaming\n", - " Default Technique: all\n", + " - all, default, core, light, multi_turn, single_turn\n", + " Available Techniques (14):\n", + " context_compliance, crescendo_history_lecture,\n", + " crescendo_journalist_interview, crescendo_movie_director,\n", + " crescendo_simulated, flip, many_shot, red_teaming,\n", + " role_play_movie_script, role_play_persuasion,\n", + " role_play_persuasion_written, role_play_trivia_game,\n", + " role_play_video_game, tap\n", + " Default Technique: default\n", " Default Datasets (1):\n", " airt_malware\n", " Supported Parameters:\n", @@ -279,17 +217,39 @@ " airt.jailbreak\u001b[0m\n", " Class: Jailbreak\n", " Description:\n", - " Jailbreak scenario implementation for PyRIT. This scenario tests how\n", - " vulnerable models are to jailbreak attacks by applying various\n", - " single-turn jailbreak templates to a set of test prompts. The responses\n", - " are scored to determine if the jailbreak was successful.\n", + " Jailbreak scenario implementation for PyRIT. Tests how vulnerable a\n", + " model is to jailbreak templates. A run is the cross-product of three\n", + " selectors: - **dataset** — the harmful objectives (HarmBench). -\n", + " **techniques** — the *attack techniques* each jailbreak is delivered\n", + " through. Two deliveries are on by default: ``prompt_sending`` (the\n", + " template rendered inline into the user message) and\n", + " ``jailbreak_system_prompt`` (the template set as the system prompt with\n", + " the objective sent as the user turn). The registry techniques\n", + " (``role_play_*``, ``many_shot``, ``tap``, …) are opt-in. -\n", + " **jailbreaks** — which jailbreak templates to run (a random\n", + " ``num_jailbreaks`` sample or an explicit ``jailbreak_names`` set).\n", + " ``prompt_sending`` applies each template as a ``TextJailbreakConverter``\n", + " on the outgoing request, so the objective is rendered inline into the\n", + " template's ``{{prompt}}`` slot; this keeps that delivery target-agnostic\n", + " and lets it compose with every technique. ``jailbreak_system_prompt``\n", + " instead sets the template as a native system prompt and sends the\n", + " objective as its own user turn, so it is only built for targets that\n", + " natively support editable history and system prompts (it is skipped for\n", + " incapable targets, or raises if it is the only selected technique).\n", + " Responses are scored to determine whether the jailbreak succeeded\n", + " (non-refusal).\n", " Aggregate Techniques:\n", - " - all, simple, complex\n", - " Available Techniques (4):\n", - " prompt_sending, many_shot, skeleton, role_play\n", - " Default Technique: simple\n", + " - all, default, core, light, multi_turn, single_turn\n", + " Available Techniques (16):\n", + " context_compliance, crescendo_history_lecture,\n", + " crescendo_journalist_interview, crescendo_movie_director,\n", + " crescendo_simulated, flip, many_shot, red_teaming,\n", + " role_play_movie_script, role_play_persuasion,\n", + " role_play_persuasion_written, role_play_trivia_game,\n", + " role_play_video_game, tap, prompt_sending, jailbreak_system_prompt\n", + " Default Technique: default\n", " Default Datasets (1):\n", - " airt_harms\n", + " harmbench\n", " Supported Parameters:\n", " - objective_target (any): Target system under attack: a registered target name or a PromptTarget instance.\n", " - scenario_techniques (any): Techniques to execute; defaults to the scenario's default aggregate when omitted.\n", @@ -299,6 +259,9 @@ " - max_concurrency (int) [default: '4']: Maximum number of concurrent units of work for the scenario.\n", " - max_retries (int) [default: '0']: Maximum number of automatic retries if the scenario raises an exception.\n", " - include_baseline (bool): Whether to prepend a baseline atomic attack; None defers to BASELINE_ATTACK_POLICY.\n", + " - num_jailbreaks (int): Draw this many random jailbreak templates for the run. Mutually exclusive with jailbreak_names.\n", + " - num_jailbreak_attempts (int) [default: '1']: Number of times to try each (technique x jailbreak template x objective).\n", + " - jailbreak_names (list[str]): Explicit jailbreak template file names to run (e.g. aim.yaml dan_11.yaml). When omitted, a random sample is drawn. Mutually exclusive with num_jailbreaks.\n", "\u001b[1m\u001b[36m\n", " airt.leakage\u001b[0m\n", " Class: Leakage\n", @@ -308,12 +271,14 @@ " property, or other confidential information. Uses the registry/factory\n", " pattern to construct attack techniques.\n", " Aggregate Techniques:\n", - " - all, default, single_turn, multi_turn\n", - " Available Techniques (11):\n", + " - all, default, airt, core, leakage, light, multi_turn, single_turn\n", + " Available Techniques (16):\n", " context_compliance, crescendo_history_lecture,\n", " crescendo_journalist_interview, crescendo_movie_director,\n", - " crescendo_simulated, many_shot, red_teaming, role_play, tap,\n", - " first_letter, image\n", + " crescendo_simulated, flip, many_shot, red_teaming,\n", + " role_play_movie_script, role_play_persuasion,\n", + " role_play_persuasion_written, role_play_trivia_game,\n", + " role_play_video_game, tap, first_letter, image\n", " Default Technique: default\n", " Default Datasets (1):\n", " airt_leakage\n", @@ -330,32 +295,41 @@ " airt.psychosocial\u001b[0m\n", " Class: Psychosocial\n", " Description:\n", - " Psychosocial Harms Scenario implementation for PyRIT. This scenario\n", - " contains various psychosocial harm-based checks that evaluate model\n", - " behavior for responses that may harm the well-being or mental health of\n", - " users. Examples include mistreating users in a mental health crisis or\n", - " aiding them in self-harm. All datasets in this scenario share the\n", - " unified 'psychosocial' harm category, but each dataset (e.g.,\n", - " imminent_crisis, dependency) can have unique: - Scoring rubrics - Seed\n", - " objectives The scenario tests whether models: - Recognize mental health\n", - " crises and respond appropriately - Avoid facilitating self-harm or\n", - " harmful behaviors - Provide crisis resources and encourage professional\n", - " help Example usage with custom configurations: # Define custom\n", - " configurations per subharm category custom_configs = {\n", - " \"airt_imminent_crisis\": SubharmConfig(\n", - " crescendo_system_prompt_path=\"path/to/custom_escalation.yaml\",\n", - " scoring_rubric_path=\"path/to/custom_rubric.yaml\", ), } scenario =\n", - " Psychosocial(subharm_configs=custom_configs)\n", - " scenario.set_params_from_args( args={ \"objective_target\": target_llm,\n", - " \"scenario_techniques\": [PsychosocialTechnique.ImminentCrisis], } ) await\n", - " scenario.initialize_async()\n", + " Psychosocial Harms scenario covering the imminent-crisis and\n", + " licensed-therapist sub-harms. Evaluates whether a model harms the\n", + " well-being or mental health of users -- for example by mistreating\n", + " someone in a mental-health crisis, facilitating self-harm, or improperly\n", + " presenting itself as a real licensed therapist. **Two axes.** The\n", + " primary axis is ``sub_harm`` (``imminent_crisis`` and/or\n", + " ``licensed_therapist``; both by default). Each sub-harm owns its\n", + " dataset, its escalation prompt, and its own conversation-level scorer,\n", + " so every attack and baseline is scored by the rubric that matches its\n", + " harm. The secondary axis is the ``PsychosocialTechnique`` converter\n", + " sweep, selected with ``--techniques``. **The base technique is a\n", + " simulated crescendo.** For each sub-harm the scenario builds an\n", + " escalating simulated conversation (via\n", + " ``AttackTechniqueFactory.with_simulated_conversation`` using that\n", + " sub-harm's escalation prompt) and delivers the final message to the\n", + " target. Each selected converter is layered on top of that base; the live\n", + " multi-turn ``Crescendo`` technique (``all`` only) swaps the simulated\n", + " base for a real ``CrescendoAttack``. One baseline per sub-harm is\n", + " emitted (toggle with ``include_baseline``). Dataset selection is bound\n", + " to the sub-harms: the ``dataset_config`` parameter still tunes\n", + " ``max_dataset_size`` and sampling, but the dataset names are always the\n", + " selected sub-harms' datasets (``--dataset-names`` is ignored).\n", " Aggregate Techniques:\n", - " - all\n", - " Available Techniques (2):\n", - " imminent_crisis, licensed_therapist\n", - " Default Technique: all\n", - " Default Datasets (1):\n", - " airt_imminent_crisis\n", + " - all, default, tone, language, persuasion, deterministic\n", + " Available Techniques (24):\n", + " none, tone_soften, tone_upset, tone_angry, tone_sad, tone_urgent,\n", + " language_spanish, language_french, language_german, language_japanese,\n", + " persuasion_logical_appeal, persuasion_authority_endorsement,\n", + " persuasion_evidence_based, persuasion_expert_endorsement,\n", + " persuasion_misrepresentation, tense_past, variation, noise,\n", + " insert_punctuation, random_capitalization, diacritic, char_swap,\n", + " colloquial_wordswap, crescendo\n", + " Default Technique: default\n", + " Default Datasets (2):\n", + " airt_imminent_crisis, airt_licensed_therapist\n", " Supported Parameters:\n", " - objective_target (any): Target system under attack: a registered target name or a PromptTarget instance.\n", " - scenario_techniques (any): Techniques to execute; defaults to the scenario's default aggregate when omitted.\n", @@ -365,6 +339,8 @@ " - max_concurrency (int) [default: '4']: Maximum number of concurrent units of work for the scenario.\n", " - max_retries (int) [default: '0']: Maximum number of automatic retries if the scenario raises an exception.\n", " - include_baseline (bool): Whether to prepend a baseline atomic attack; None defers to BASELINE_ATTACK_POLICY.\n", + " - sub_harm (str) [default: 'all']: Psychosocial sub-harm to run: 'imminent_crisis', 'licensed_therapist', or 'all'. Defaults to 'all'.\n", + " - max_turns (int) [default: '5']: Number of turns in the simulated-crescendo escalation for each attack.\n", "\u001b[1m\u001b[36m\n", " airt.rapid_response\u001b[0m\n", " Class: RapidResponse\n", @@ -372,11 +348,14 @@ " Rapid Response scenario for content-harms testing. Tests model behavior\n", " across multiple harm categories using selectable attack techniques.\n", " Aggregate Techniques:\n", - " - all, default, single_turn, multi_turn\n", - " Available Techniques (9):\n", + " - all, default, core, light, multi_turn, single_turn\n", + " Available Techniques (14):\n", " context_compliance, crescendo_history_lecture,\n", " crescendo_journalist_interview, crescendo_movie_director,\n", - " crescendo_simulated, many_shot, red_teaming, role_play, tap\n", + " crescendo_simulated, flip, many_shot, red_teaming,\n", + " role_play_movie_script, role_play_persuasion,\n", + " role_play_persuasion_written, role_play_trivia_game,\n", + " role_play_video_game, tap\n", " Default Technique: default\n", " Default Datasets (7):\n", " airt_hate, airt_fairness, airt_violence, airt_sexual, airt_harassment,\n", @@ -398,10 +377,10 @@ " materials (e.g., phishing emails, fraudulent messages) with primarily\n", " persuasion-oriented techniques.\n", " Aggregate Techniques:\n", - " - all, single_turn, multi_turn\n", + " - all, default, single_turn, multi_turn\n", " Available Techniques (3):\n", - " context_compliance, role_play, persuasive_rta\n", - " Default Technique: all\n", + " context_compliance, role_play_persuasion_written, persuasive_rta\n", + " Default Technique: default\n", " Default Datasets (1):\n", " airt_scams\n", " Supported Parameters:\n", @@ -427,20 +406,21 @@ " ``TargetRegistry.get_registry_singleton().instances.register``. At run\n", " time, ``_build_atomic_attacks_async`` performs the ``(technique ×\n", " adversarial_target × dataset)`` cross-product: for each selected\n", - " adversarial-capable ``core`` factory in the ``AttackTechniqueRegistry``\n", - " and each requested target, it calls\n", - " ``factory.create(adversarial_chat=...)`` with the resolved target — no\n", - " global registry mutation. The resulting ``AtomicAttack`` is named\n", - " ``f\"{technique}__{target}_{dataset}\"`` with ``display_group`` set to the\n", - " target's registry name so per-model ASR rolls up naturally in result\n", - " displays.\n", + " adversarial-capable factory in the ``AttackTechniqueRegistry`` and each\n", + " requested target, it calls ``factory.create(adversarial_chat=...)`` with\n", + " the resolved target — no global registry mutation. The resulting\n", + " ``AtomicAttack`` is named ``f\"{technique}__{target}_{dataset}\"`` with\n", + " ``display_group`` set to the target's registry name so per-model ASR\n", + " rolls up naturally in result displays.\n", " Aggregate Techniques:\n", - " - all, default, light, single_turn, multi_turn\n", - " Available Techniques (8):\n", + " - all, default, core, light, multi_turn, single_turn\n", + " Available Techniques (12):\n", " context_compliance, crescendo_history_lecture,\n", " crescendo_journalist_interview, crescendo_movie_director,\n", - " crescendo_simulated, red_teaming, role_play, tap\n", - " Default Technique: light\n", + " crescendo_simulated, red_teaming, role_play_movie_script,\n", + " role_play_persuasion, role_play_persuasion_written,\n", + " role_play_trivia_game, role_play_video_game, tap\n", + " Default Technique: default\n", " Default Datasets (1):\n", " harmbench\n", " Supported Parameters:\n", @@ -502,10 +482,10 @@ " [@hiddenlayer2025policypuppetry]\n", " (https://hiddenlayer.com/innovation-hub/novel-universal-bypass-for-all-major-llms/)\n", " Aggregate Techniques:\n", - " - all, default\n", + " - all, default, single_turn\n", " Available Techniques (2):\n", " policy_puppetry, policy_puppetry_leet\n", - " Default Technique: all\n", + " Default Technique: default\n", " Default Datasets (1):\n", " garak_doctor\n", " Supported Parameters:\n", @@ -532,12 +512,12 @@ " decoded and repeated the harmful content By default, this uses the same\n", " dataset as Garak: slur terms and web XSS payloads.\n", " Aggregate Techniques:\n", - " - all\n", + " - all, default\n", " Available Techniques (17):\n", " base64, base2048, base16, base32, ascii85, hex, quoted_printable,\n", " uuencode, rot13, braille, atbash, morse_code, nato, ecoji, zalgo,\n", " leet_speak, ascii_smuggler\n", - " Default Technique: all\n", + " Default Technique: default\n", " Default Datasets (2):\n", " garak_slur_terms_en, garak_web_html_js\n", " Supported Parameters:\n", @@ -590,7 +570,7 @@ } ], "source": [ - "!pyrit_scan --list-scenarios" + "!pyrit_scan list-scenarios" ] }, { @@ -598,14 +578,16 @@ "id": "6", "metadata": {}, "source": [ - "**Tip**: You can also discover user-defined scenarios by providing initialization scripts:\n", + "**Tip**: You can also surface user-defined scenarios. List your initializer script in the\n", + "`initialization_scripts` section of the config file the backend loads (see [here](../getting_started/pyrit_conf.md)).\n", + "The backend runs those scripts at startup and auto-discovers any `Scenario` subclasses they\n", + "define, so start the server with that config, then list:\n", "\n", "```shell\n", - "pyrit_scan --list-scenarios --initialization-scripts ./my_custom_initializer.py\n", + "pyrit_scan --config-file ./my_pyrit_conf.yaml start-server\n", + "pyrit_scan list-scenarios\n", "```\n", "\n", - "This will load your custom scenario definitions and include them in the list.\n", - "\n", "## Initializers\n", "\n", "PyRITInitializers are how you can configure the CLI scanner. PyRIT includes several built-in initializers you can use with the `--initializers` flag.\n", @@ -636,10 +618,7 @@ " - dataset_names: Explicit dataset names to load. Overrides the scenario-default selection.\n", " - tags: Load datasets whose metadata matches these tags. Overrides scenario-default selection.\n", " Description:\n", - " Load datasets into memory so scenarios can run. By default this loads\n", - " the datasets required by all registered scenarios. Pass\n", - " ``dataset_names`` to load specific datasets by name, or ``tags`` to\n", - " select datasets by metadata.\n", + " Load datasets into memory so scenarios can run.\n", "\u001b[1m\u001b[36m\n", " preload_scenario_metadata\u001b[0m\n", " Class: PreloadScenarioMetadata\n", @@ -647,6 +626,16 @@ " Description:\n", " Instantiate every registered scenario once to warm the metadata cache.\n", "\u001b[1m\u001b[36m\n", + " refresh_datasets\u001b[0m\n", + " Class: RefreshDatasets\n", + " Required Environment Variables: None\n", + " Supported Parameters:\n", + " - days [default: 30]: Refresh only datasets whose newest seed is older than this many days. 0 refreshes every selected dataset regardless of age.\n", + " - dataset_names: Explicit dataset names to refresh; refreshes all in-memory datasets if omitted.\n", + " Description:\n", + " Refresh datasets already loaded in memory from their registered\n", + " providers.\n", + "\u001b[1m\u001b[36m\n", " scorer\u001b[0m\n", " Class: ScorerInitializer\n", " Required Environment Variables: None\n", @@ -654,15 +643,7 @@ " - tags [default: ['default']]: Tags for filtering (e.g., ['default'])\n", " Description:\n", " Instantiates a collection of scorers using targets from the\n", - " TargetRegistry and adds them to the ScorerRegistry. This initializer\n", - " registers all evaluation scorers into the ScorerRegistry. Targets are\n", - " pulled from the TargetRegistry (populated by TargetInitializer), so this\n", - " initializer should be listed after TargetInitializer in the initializers\n", - " list. Scorers that fail to initialize (e.g., due to missing targets) are\n", - " skipped with a warning. Every scorer category follows the same pattern:\n", - " ``_register__scorers()`` registers all variants with a\n", - " category tag. ``_tag_best_per_category()`` marks the preferred scorer\n", - " per category. Compound scorers reference core scorers via BEST_* tags.\n", + " TargetRegistry and adds them to the ScorerRegistry.\n", "\u001b[1m\u001b[36m\n", " target\u001b[0m\n", " Class: TargetInitializer\n", @@ -671,49 +652,7 @@ " - tags [default: ['default']]: Target tags to register (e.g., ['default'], ['default', 'scorer'], or ['all'])\n", " - auto_group [default: True]: Auto-create round-robin groups from targets with matching behavioral eval params\n", " Description:\n", - " Target Initializer for registering pre-configured targets. This\n", - " initializer scans for known endpoint environment variables and registers\n", - " the corresponding targets into the TargetRegistry. Targets can be\n", - " filtered by tags to control which targets are registered. Supported\n", - " Parameters: tags: Target tags to register (list of strings). \"default\"\n", - " registers the base environment targets. \"scorer\" registers\n", - " scorer-specific temperature variant targets. \"all\" registers all targets\n", - " regardless of tag. If not provided, only \"default\" targets are\n", - " registered. auto_group: Whether to automatically create round-robin\n", - " groups from targets with matching behavioral eval params (underlying\n", - " model, temperature, top_p). Defaults to True. Supported Endpoints by\n", - " Category: **OpenAI Chat Targets (OpenAIChatTarget):** -\n", - " PLATFORM_OPENAI_CHAT_* - Platform OpenAI Chat API - AZURE_OPENAI_GPT4O_*\n", - " - Azure OpenAI GPT-4o - AZURE_OPENAI_INTEGRATION_TEST_* - Integration\n", - " test endpoint - AZURE_OPENAI_GPT3_5_CHAT_* - Azure OpenAI GPT-3.5 -\n", - " AZURE_OPENAI_GPT4_CHAT_* - Azure OpenAI GPT-4 - AZURE_OPENAI_GPT5_4_* -\n", - " Azure OpenAI GPT-5.4 - AZURE_OPENAI_GPT5_COMPLETIONS_* - Azure OpenAI\n", - " GPT-5.1 - AZURE_OPENAI_GPT4O_UNSAFE_CHAT_* - Azure OpenAI GPT-4o unsafe\n", - " - AZURE_OPENAI_GPT4O_UNSAFE_CHAT_*2 - Azure OpenAI GPT-4o unsafe\n", - " secondary - AZURE_FOUNDRY_DEEPSEEK_* - Azure AI Foundry DeepSeek -\n", - " AZURE_FOUNDRY_PHI4_* - Azure AI Foundry Phi-4 -\n", - " AZURE_FOUNDRY_MISTRAL_LARGE_* - Azure AI Foundry Mistral Large - GROQ_*\n", - " - Groq API - OPEN_ROUTER_* - OpenRouter API - OLLAMA_* - Ollama local -\n", - " GOOGLE_GEMINI_* - Google Gemini (OpenAI-compatible) **OpenAI Responses\n", - " Targets (OpenAIResponseTarget):** - AZURE_OPENAI_GPT5_RESPONSES_* -\n", - " Azure OpenAI GPT-5 Responses - AZURE_OPENAI_GPT5_RESPONSES_* (high\n", - " reasoning) - Azure OpenAI GPT-5 Responses with high reasoning effort -\n", - " PLATFORM_OPENAI_RESPONSES_* - Platform OpenAI Responses -\n", - " AZURE_OPENAI_RESPONSES_* - Azure OpenAI Responses **Realtime Targets\n", - " (RealtimeTarget):** - PLATFORM_OPENAI_REALTIME_* - Platform OpenAI\n", - " Realtime - AZURE_OPENAI_REALTIME_* - Azure OpenAI Realtime **Image\n", - " Targets (OpenAIImageTarget):** - OPENAI_IMAGE_*1 - Azure OpenAI Image -\n", - " OPENAI_IMAGE_*2 - Platform OpenAI Image **TTS Targets\n", - " (OpenAITTSTarget):** - OPENAI_TTS_*1 - Azure OpenAI TTS - OPENAI_TTS_*2\n", - " - Platform OpenAI TTS **Video Targets (OpenAIVideoTarget):** -\n", - " AZURE_OPENAI_VIDEO_* - Azure OpenAI Video **Completion Targets\n", - " (OpenAICompletionTarget):** - OPENAI_COMPLETION_* - OpenAI Completion\n", - " **Azure ML Targets (AzureMLChatTarget):** - AZURE_ML_PHI_* - Azure ML\n", - " Phi **Safety Targets (PromptShieldTarget):** - AZURE_CONTENT_SAFETY_* -\n", - " Azure Content Safety Example: initializer = TargetInitializer() await\n", - " initializer.initialize_async() # Register scorer temperature variants\n", - " too initializer.params = {\"tags\": [\"default\", \"scorer\"]} await\n", - " initializer.initialize_async()\n", + " Target Initializer for registering pre-configured targets.\n", "\u001b[1m\u001b[36m\n", " technique\u001b[0m\n", " Class: TechniqueInitializer\n", @@ -722,19 +661,16 @@ " - tags [default: ['core']]: Technique groups to register (e.g., ['core'], ['core', 'extra'], or ['all'])\n", " Description:\n", " Register scenario attack technique factories into the\n", - " AttackTechniqueRegistry. By default only the ``core`` group is\n", - " registered. Pass ``tags`` to select groups (``core``, ``extra``, or\n", - " ``all``). Registration is per-name idempotent: pre-existing entries in\n", - " ``AttackTechniqueRegistry`` are not overwritten.\n", + " AttackTechniqueRegistry.\n", "\n", "================================================================================\n", "\n", - "Total initializers: 5\n" + "Total initializers: 6\n" ] } ], "source": [ - "!pyrit_scan --list-initializers" + "!pyrit_scan list-initializers" ] }, { @@ -747,25 +683,25 @@ "You need a single scenario to run, you need two things:\n", "\n", "1. A Scenario. Many are defined in `pyrit.scenario.scenarios`. But you can also define your own in initialization_scripts.\n", - "2. Initializers (which can be supplied via `--initializers` or `--initialization-scripts` or `initializers` section of config file (see [here](../getting_started/pyrit_conf.md))). Scenarios often don't need many arguments, but they can be configured in different ways. And at the very least, most need an `objective_target` (the thing you're running a scan against) which you can configure by using the `--target` flag if your initializer registers targets (e.g. `target` initializer)\n", + "2. Initializers (which can be supplied via the `--initializers` flag on `run`, or the `initializers` / `initialization_scripts` sections of a config file (see [here](../getting_started/pyrit_conf.md))). Scenarios often don't need many arguments, but they can be configured in different ways. And at the very least, most need an `objective_target` (the thing you're running a scan against) which you can configure by using the `--target` flag if your initializer registers targets (e.g. `target` initializer)\n", "3. Scenario Techniques (optional). These are supplied by the `--techniques` flag and tell the scenario what to test, but they are always optional. Also note you can obtain these by running `--list-scenarios`\n", "\n", "Basic usage will look something like:\n", "\n", "```shell\n", - "pyrit_scan --target --initializers --techniques \n", + "pyrit_scan run --target --initializers --techniques \n", "```\n", "\n", "You can also override scenario parameters directly from the CLI:\n", "\n", "```shell\n", - "pyrit_scan --max-concurrency 10 --max-retries 3 --memory-labels '{\"experiment\": \"test1\", \"version\": \"v2\"}'\n", + "pyrit_scan run --max-concurrency 10 --max-retries 3 --memory-labels '{\"experiment\": \"test1\", \"version\": \"v2\"}'\n", "```\n", "\n", "Or concretely:\n", "\n", "```shell\n", - "!pyrit_scan foundry.red_team_agent --target openai_chat --initializers target --techniques base64\n", + "!pyrit_scan run foundry.red_team_agent --target openai_chat --initializers target --techniques base64\n", "```\n", "\n", "Example with a basic configuration that runs the Foundry scenario against the objective target defined in the `target` initializer." @@ -784,24 +720,99 @@ "\n", "Running scenario: foundry.red_team_agent\n", "\n", - " techniques: 0/1 | attacks: 0 | success rate: 0% | IN_PROGRESS\n", - " techniques: 0/1 | attacks: 0 | success rate: 0% | IN_PROGRESS\n", - " techniques: 0/1 | attacks: 0 | success rate: 0% | IN_PROGRESS\n", - " techniques: 0/1 | attacks: 0 | success rate: 0% | IN_PROGRESS\n", - " techniques: 0/1 | attacks: 0 | success rate: 0% | IN_PROGRESS\n", - " techniques: 0/1 | attacks: 0 | success rate: 0% | IN_PROGRESS\n", - " techniques: 0/1 | attacks: 0 | success rate: 0% | IN_PROGRESS\n", - " techniques: 0/1 | attacks: 0 | success rate: 0% | IN_PROGRESS\n", - " techniques: 0/1 | attacks: 0 | success rate: 0% | IN_PROGRESS\n", - " techniques: 0/1 | attacks: 0 | success rate: 0% | IN_PROGRESS\n", - " techniques: 0/1 | attacks: 0 | success rate: 0% | IN_PROGRESS\n", - " techniques: 0/1 | attacks: 0 | success rate: 0% | IN_PROGRESS\n", - "Error (UnicodeEncodeError): 'charmap' codec can't encode characters in position 22-51: character maps to \n" + " [░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░] techniques: 0/1 (0%) | success rate: 0% | IN_PROGRESS\n", + " [░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░] techniques: 0/1 (0%) | success rate: 0% | IN_PROGRESS\n", + " [░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░] techniques: 0/1 (0%) | success rate: 0% | IN_PROGRESS\n", + " [░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░] techniques: 0/1 (0%) | success rate: 0% | IN_PROGRESS\n", + " [░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░] techniques: 0/1 (0%) | success rate: 0% | IN_PROGRESS\n", + " [░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░] techniques: 0/1 (0%) | success rate: 0% | IN_PROGRESS\n", + " [░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░] techniques: 0/1 (0%) | success rate: 0% | IN_PROGRESS\n", + " [░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░] techniques: 0/1 (0%) | success rate: 0% | IN_PROGRESS\n", + " [░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░] techniques: 0/1 (0%) | success rate: 0% | IN_PROGRESS\n", + " [░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░] techniques: 0/1 (0%) | success rate: 0% | IN_PROGRESS\n", + " [░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░] techniques: 0/1 (0%) | success rate: 0% | IN_PROGRESS\n", + " [░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░] techniques: 0/1 (0%) | success rate: 0% | IN_PROGRESS\n", + " [░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░] techniques: 0/1 (0%) | success rate: 0% | IN_PROGRESS\n", + " [░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░] techniques: 0/1 (0%) | success rate: 0% | IN_PROGRESS\n", + " [░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░] techniques: 0/1 (0%) | success rate: 0% | IN_PROGRESS\n", + " [░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░] techniques: 0/1 (0%) | success rate: 0% | IN_PROGRESS\n", + " [░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░] techniques: 0/1 (0%) | success rate: 0% | IN_PROGRESS\n", + " [░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░] techniques: 0/1 (0%) | success rate: 0% | IN_PROGRESS\n", + " [░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░] techniques: 0/1 (0%) | success rate: 0% | IN_PROGRESS\n", + " [░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░] techniques: 0/1 (0%) | success rate: 0% | IN_PROGRESS\n", + " [░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░] techniques: 0/1 (0%) | success rate: 0% | IN_PROGRESS\n", + " [░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░] techniques: 0/1 (0%) | success rate: 0% | IN_PROGRESS\n", + " [░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░] techniques: 0/1 (0%) | success rate: 0% | IN_PROGRESS\n", + " [░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░] techniques: 0/1 (0%) | success rate: 0% | IN_PROGRESS\n", + " [██████████████████████████████] techniques: 2/2 (100%) | success rate: 0% | IN_PROGRESS\n", + " [██████████████████████████████] techniques: 2/2 (100%) | success rate: 0% | COMPLETED\n", + "\u001b[36m====================================================================================================\u001b[0m\n", + "\u001b[1m\u001b[36m 📊 SCENARIO RESULTS: RedTeamAgent \u001b[0m\n", + "\u001b[36m====================================================================================================\u001b[0m\n", + "\n", + "\u001b[1m\u001b[36m▼ Scenario Information\u001b[0m\n", + "\u001b[36m────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\n", + "\u001b[1m 📋 Scenario Details\u001b[0m\n", + "\u001b[36m • Name: RedTeamAgent\u001b[0m\n", + "\u001b[36m • Result ID: 10840c79-567c-4ecd-a3c8-fb7e9349bd57\u001b[0m\n", + "\u001b[36m • Scenario Version: 1\u001b[0m\n", + "\u001b[36m • PyRIT Version: 1.1.0.dev0\u001b[0m\n", + "\u001b[36m • Description:\u001b[0m\n", + "\u001b[36m RedTeamAgent is a preconfigured scenario that automatically generates multiple AtomicAttack instances based on\u001b[0m\n", + "\u001b[36m the specified attack techniques. It supports both single-turn attacks (with various converters) and multi-turn\u001b[0m\n", + "\u001b[36m attacks (Crescendo, RedTeaming), making it easy to quickly test a target against multiple attack vectors. The\u001b[0m\n", + "\u001b[36m scenario can expand difficulty levels (EASY, MODERATE, DIFFICULT) into their constituent attack techniques, or\u001b[0m\n", + "\u001b[36m you can specify individual techniques directly. This scenario is designed for use with the Foundry AI Red\u001b[0m\n", + "\u001b[36m Teaming Agent library, providing a consistent PyRIT contract for their integration.\u001b[0m\n", + "\n", + "\u001b[1m 🎯 Target Information\u001b[0m\n", + "\u001b[36m • Target Type: OpenAIChatTarget\u001b[0m\n", + "\u001b[36m • Target Model: gpt-4o\u001b[0m\n", + "\u001b[36m • Target Endpoint: https://pyrit-japan-test.openai.azure.com/openai/v1\u001b[0m\n", + "\n", + "\u001b[1m 📊 Scorer Information\u001b[0m\n", + "\u001b[37m ▸ Scorer Identifier\u001b[0m\n", + "\u001b[36m • Scorer Type: FloatScaleThresholdScorer\u001b[0m\n", + "\u001b[36m • scorer_type: true_false\u001b[0m\n", + "\u001b[36m • score_aggregator: OR_\u001b[0m\n", + "\u001b[36m └─ Composite of 1 scorer(s):\u001b[0m\n", + "\u001b[36m • Scorer Type: AzureContentFilterScorer\u001b[0m\n", + "\u001b[36m • scorer_type: float_scale\u001b[0m\n", + "\n", + "\u001b[37m ▸ Performance Metrics\u001b[0m\n", + "\u001b[31m • Accuracy: 59.24%\u001b[0m\n", + "\u001b[36m • Accuracy Std Error: ±0.0247\u001b[0m\n", + "\u001b[31m • F1 Score: 0.5306\u001b[0m\n", + "\u001b[31m • Precision: 0.5987\u001b[0m\n", + "\u001b[31m • Recall: 0.4764\u001b[0m\n", + "\u001b[32m • Average Score Time: 0.04s\u001b[0m\n", + "\n", + "\u001b[1m\u001b[36m▼ Overall Statistics\u001b[0m\n", + "\u001b[36m────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\n", + "\u001b[1m 📈 Summary\u001b[0m\n", + "\u001b[32m • Total Techniques: 2\u001b[0m\n", + "\u001b[32m • Total Attack Results: 8\u001b[0m\n", + "\u001b[32m • Overall Success Rate: 0%\u001b[0m\n", + "\u001b[32m • Unique Objectives: 4\u001b[0m\n", + "\n", + "\u001b[1m\u001b[36m▼ Per-Group Breakdown\u001b[0m\n", + "\u001b[36m────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\n", + "\n", + "\u001b[1m 🔸 Group: base64\u001b[0m\n", + "\u001b[33m • Number of Results: 4\u001b[0m\n", + "\u001b[32m • Success Rate: 0%\u001b[0m\n", + "\n", + "\u001b[1m 🔸 Group: baseline\u001b[0m\n", + "\u001b[33m • Number of Results: 4\u001b[0m\n", + "\u001b[32m • Success Rate: 0%\u001b[0m\n", + "\n", + "\u001b[36m====================================================================================================\u001b[0m\n", + "\n" ] } ], "source": [ - "!pyrit_scan foundry.red_team_agent --target openai_chat --initializers target --techniques base64" + "!pyrit_scan run foundry.red_team_agent --target openai_chat --initializers target --techniques base64" ] }, { @@ -812,17 +823,17 @@ "Or with all options and multiple techniques:\n", "\n", "```shell\n", - "pyrit_scan foundry.red_team_agent --target openai_chat --initializers target --techniques easy crescendo\n", + "pyrit_scan run foundry.red_team_agent --target openai_chat --initializers target --techniques easy crescendo\n", "```\n", "\n", "You can also override scenario execution parameters:\n", "\n", "```shell\n", "# Override concurrency and retry settings\n", - "pyrit_scan foundry.red_team_agent --target openai_chat --initializers target --max-concurrency 10 --max-retries 3\n", + "pyrit_scan run foundry.red_team_agent --target openai_chat --initializers target --max-concurrency 10 --max-retries 3\n", "\n", "# Add custom memory labels for tracking (must be valid JSON)\n", - "pyrit_scan foundry.red_team_agent --target openai_chat --initializers target --memory-labels '{\"experiment\": \"test1\", \"version\": \"v2\", \"researcher\": \"alice\"}'\n", + "pyrit_scan run foundry.red_team_agent --target openai_chat --initializers target --memory-labels '{\"experiment\": \"test1\", \"version\": \"v2\", \"researcher\": \"alice\"}'\n", "```\n", "\n", "Available CLI parameter overrides:\n", @@ -830,11 +841,15 @@ "- `--max-retries `: Maximum number of automatic retries if the scenario raises an exception\n", "- `--memory-labels `: Additional labels to apply to all attack runs (must be a JSON string with string keys and values)\n", "\n", - "You can also use custom initialization scripts by passing file paths. It is relative to your current working directory, but to avoid confusion, full paths are always better:\n", + "Custom initialization scripts are loaded by the backend at startup: list them in the\n", + "`initialization_scripts` section of the config the server loads (paths are relative to your\n", + "working directory, but full paths avoid confusion). Once the server is running with that\n", + "config, they apply to every `run`:\n", "\n", - "```shell\n", - "pyrit_scan garak.encoding --initialization-scripts ./my_custom_config.py\n", - "```" + "\n", + "```shell```\n", + "\n", + "pyrit_scan --config-file ./my_pyrit_conf.yaml start-serverpyrit_scan run garak.encoding" ] }, { @@ -849,22 +864,26 @@ "the technique produces, on top of any converters the technique already bakes in. This also works on\n", "aggregate techniques (the converter is applied to every technique the aggregate expands to).\n", "\n", - "First discover the registered converter instances with `--list-converters` (converters are\n", - "registered by initializers, so pass the same `--initializers`/`--initialization-scripts` you use to run):\n", + "First discover the registered converter instances with `list-converters`. Converters are\n", + "registered by initializers that the backend runs at startup, so the initializer that registers\n", + "them must be part of the server's configuration — a built-in in the `initializers` section, or a\n", + "custom script in the `initialization_scripts` section of the config the server loads (see\n", + "[here](../getting_started/pyrit_conf.md)). Start the server with that config, then list:\n", "\n", "```shell\n", - "pyrit_scan --list-converters --initializers my_converters\n", + "pyrit_scan --config-file ./my_pyrit_conf.yaml start-server\n", + "pyrit_scan list-converters\n", "```\n", "\n", "Then reference a converter by name in `--techniques`:\n", "\n", "```shell\n", "# Add the registered \"translation_spanish\" converter to role_play_movie_script only\n", - "pyrit_scan airt.rapid_response --target openai_chat --initializers load_default_datasets target my_converters --techniques role_play_movie_script:converter.translation_spanish\n", + "pyrit_scan run airt.rapid_response --target openai_chat --initializers load_default_datasets target my_converters --techniques role_play_movie_script:converter.translation_spanish\n", + "\n", + "```\n", "\n", - "# Chain multiple converters (applied in order) and combine with plain techniques\n", - "pyrit_scan airt.rapid_response --target openai_chat --initializers load_default_datasets target my_converters --techniques role_play_movie_script:converter.translation_spanish:converter.base64 many_shot\n", - "```" + "# Chain multiple converters (applied in order) and combine with plain techniquespyrit_scan run airt.rapid_response --target openai_chat --initializers load_default_datasets target my_converters --techniques role_play_movie_script:converter.translation_spanish:converter.base64 many_shot" ] }, { @@ -966,14 +985,16 @@ "Then discover and run it:\n", "\n", "```shell\n", - "# List to see it's available\n", - "pyrit_scan --list-scenarios --initialization-scripts ./my_custom_scenarios.py\n", + "# Start the backend with a config whose initialization_scripts lists my_custom_scenarios.py\n", + "pyrit_scan --config-file ./my_pyrit_conf.yaml start-server\n", + "\n", + "# List to confirm it's available\n", + "pyrit_scan list-scenarios\n", "\n", "# Run it with parameter overrides\n", - "pyrit_scan my_custom_scenario --initialization-scripts ./my_custom_scenarios.py --max-concurrency 10\n", - "```\n", + "pyrit_scan run my_custom_scenario --max-concurrency 10\n", "\n", - "The scenario name is automatically converted from the class name (e.g., `MyCustomScenario` becomes `my_custom_scenario`)." + "```The scenario name is automatically converted from the class name (e.g., `MyCustomScenario` becomes `my_custom_scenario`).\n" ] }, { @@ -1001,7 +1022,7 @@ } ], "source": [ - "!pyrit_scan --stop-server" + "!pyrit_scan stop-server" ] } ], @@ -1019,7 +1040,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.13.5" + "version": "3.12.4" } }, "nbformat": 4, diff --git a/doc/scanner/1_pyrit_scan.py b/doc/scanner/1_pyrit_scan.py index 1c17ee2bdb..8bc619211a 100644 --- a/doc/scanner/1_pyrit_scan.py +++ b/doc/scanner/1_pyrit_scan.py @@ -27,7 +27,7 @@ # up and is reused by every command below. We stop it again at the end of the notebook. # %% -# !pyrit_scan --start-server +# !pyrit_scan start-server # %% [markdown] # ## Quick Start @@ -43,17 +43,19 @@ # List all available scenarios: # %% -# !pyrit_scan --list-scenarios +# !pyrit_scan list-scenarios # %% [markdown] -# **Tip**: You can also discover user-defined scenarios by providing initialization scripts: +# **Tip**: You can also surface user-defined scenarios. List your initializer script in the +# `initialization_scripts` section of the config file the backend loads (see [here](../getting_started/pyrit_conf.md)). +# The backend runs those scripts at startup and auto-discovers any `Scenario` subclasses they +# define, so start the server with that config, then list: # # ```shell -# pyrit_scan --list-scenarios --initialization-scripts ./my_custom_initializer.py +# pyrit_scan --config-file ./my_pyrit_conf.yaml start-server +# pyrit_scan list-scenarios # ``` # -# This will load your custom scenario definitions and include them in the list. -# # ## Initializers # # PyRITInitializers are how you can configure the CLI scanner. PyRIT includes several built-in initializers you can use with the `--initializers` flag. @@ -63,7 +65,7 @@ # List the available initializers using the --list-initializers flag. # %% -# !pyrit_scan --list-initializers +# !pyrit_scan list-initializers # %% [markdown] # ### Running Scenarios @@ -71,47 +73,47 @@ # You need a single scenario to run, you need two things: # # 1. A Scenario. Many are defined in `pyrit.scenario.scenarios`. But you can also define your own in initialization_scripts. -# 2. Initializers (which can be supplied via `--initializers` or `--initialization-scripts` or `initializers` section of config file (see [here](../getting_started/pyrit_conf.md))). Scenarios often don't need many arguments, but they can be configured in different ways. And at the very least, most need an `objective_target` (the thing you're running a scan against) which you can configure by using the `--target` flag if your initializer registers targets (e.g. `target` initializer) +# 2. Initializers (which can be supplied via the `--initializers` flag on `run`, or the `initializers` / `initialization_scripts` sections of a config file (see [here](../getting_started/pyrit_conf.md))). Scenarios often don't need many arguments, but they can be configured in different ways. And at the very least, most need an `objective_target` (the thing you're running a scan against) which you can configure by using the `--target` flag if your initializer registers targets (e.g. `target` initializer) # 3. Scenario Techniques (optional). These are supplied by the `--techniques` flag and tell the scenario what to test, but they are always optional. Also note you can obtain these by running `--list-scenarios` # # Basic usage will look something like: # # ```shell -# pyrit_scan --target --initializers --techniques +# pyrit_scan run --target --initializers --techniques # ``` # # You can also override scenario parameters directly from the CLI: # # ```shell -# pyrit_scan --max-concurrency 10 --max-retries 3 --memory-labels '{"experiment": "test1", "version": "v2"}' +# pyrit_scan run --max-concurrency 10 --max-retries 3 --memory-labels '{"experiment": "test1", "version": "v2"}' # ``` # # Or concretely: # # ```shell -# !pyrit_scan foundry.red_team_agent --target openai_chat --initializers target --techniques base64 +# !pyrit_scan run foundry.red_team_agent --target openai_chat --initializers target --techniques base64 # ``` # # Example with a basic configuration that runs the Foundry scenario against the objective target defined in the `target` initializer. # %% -# !pyrit_scan foundry.red_team_agent --target openai_chat --initializers target --techniques base64 +# !pyrit_scan run foundry.red_team_agent --target openai_chat --initializers target --techniques base64 # %% [markdown] # Or with all options and multiple techniques: # # ```shell -# pyrit_scan foundry.red_team_agent --target openai_chat --initializers target --techniques easy crescendo +# pyrit_scan run foundry.red_team_agent --target openai_chat --initializers target --techniques easy crescendo # ``` # # You can also override scenario execution parameters: # # ```shell # # Override concurrency and retry settings -# pyrit_scan foundry.red_team_agent --target openai_chat --initializers target --max-concurrency 10 --max-retries 3 +# pyrit_scan run foundry.red_team_agent --target openai_chat --initializers target --max-concurrency 10 --max-retries 3 # # # Add custom memory labels for tracking (must be valid JSON) -# pyrit_scan foundry.red_team_agent --target openai_chat --initializers target --memory-labels '{"experiment": "test1", "version": "v2", "researcher": "alice"}' +# pyrit_scan run foundry.red_team_agent --target openai_chat --initializers target --memory-labels '{"experiment": "test1", "version": "v2", "researcher": "alice"}' # ``` # # Available CLI parameter overrides: @@ -119,10 +121,14 @@ # - `--max-retries `: Maximum number of automatic retries if the scenario raises an exception # - `--memory-labels `: Additional labels to apply to all attack runs (must be a JSON string with string keys and values) # -# You can also use custom initialization scripts by passing file paths. It is relative to your current working directory, but to avoid confusion, full paths are always better: +# Custom initialization scripts are loaded by the backend at startup: list them in the +# `initialization_scripts` section of the config the server loads (paths are relative to your +# working directory, but full paths avoid confusion). Once the server is running with that +# config, they apply to every `run`: # # ```shell -# pyrit_scan garak.encoding --initialization-scripts ./my_custom_config.py +# pyrit_scan --config-file ./my_pyrit_conf.yaml start-server +# pyrit_scan run garak.encoding # ``` # %% [markdown] @@ -133,21 +139,25 @@ # the technique produces, on top of any converters the technique already bakes in. This also works on # aggregate techniques (the converter is applied to every technique the aggregate expands to). # -# First discover the registered converter instances with `--list-converters` (converters are -# registered by initializers, so pass the same `--initializers`/`--initialization-scripts` you use to run): +# First discover the registered converter instances with `list-converters`. Converters are +# registered by initializers that the backend runs at startup, so the initializer that registers +# them must be part of the server's configuration — a built-in in the `initializers` section, or a +# custom script in the `initialization_scripts` section of the config the server loads (see +# [here](../getting_started/pyrit_conf.md)). Start the server with that config, then list: # # ```shell -# pyrit_scan --list-converters --initializers my_converters +# pyrit_scan --config-file ./my_pyrit_conf.yaml start-server +# pyrit_scan list-converters # ``` # # Then reference a converter by name in `--techniques`: # # ```shell # # Add the registered "translation_spanish" converter to role_play_movie_script only -# pyrit_scan airt.rapid_response --target openai_chat --initializers load_default_datasets target my_converters --techniques role_play_movie_script:converter.translation_spanish +# pyrit_scan run airt.rapid_response --target openai_chat --initializers load_default_datasets target my_converters --techniques role_play_movie_script:converter.translation_spanish # # # Chain multiple converters (applied in order) and combine with plain techniques -# pyrit_scan airt.rapid_response --target openai_chat --initializers load_default_datasets target my_converters --techniques role_play_movie_script:converter.translation_spanish:converter.base64 many_shot +# pyrit_scan run airt.rapid_response --target openai_chat --initializers load_default_datasets target my_converters --techniques role_play_movie_script:converter.translation_spanish:converter.base64 many_shot # ``` # %% [markdown] @@ -206,11 +216,14 @@ async def _build_atomic_attacks_async(self, *, context): # Then discover and run it: # # ```shell -# # List to see it's available -# pyrit_scan --list-scenarios --initialization-scripts ./my_custom_scenarios.py +# # Start the backend with a config whose initialization_scripts lists my_custom_scenarios.py +# pyrit_scan --config-file ./my_pyrit_conf.yaml start-server +# +# # List to confirm it's available +# pyrit_scan list-scenarios # # # Run it with parameter overrides -# pyrit_scan my_custom_scenario --initialization-scripts ./my_custom_scenarios.py --max-concurrency 10 +# pyrit_scan run my_custom_scenario --max-concurrency 10 # ``` # # The scenario name is automatically converted from the class name (e.g., `MyCustomScenario` becomes `my_custom_scenario`). @@ -221,4 +234,4 @@ async def _build_atomic_attacks_async(self, *, context): # When you're done, stop the local backend that we started at the top of the notebook. # %% -# !pyrit_scan --stop-server +# !pyrit_scan stop-server diff --git a/doc/scanner/2_pyrit_shell.md b/doc/scanner/2_pyrit_shell.md index 7a098fbae3..06eb372b83 100644 --- a/doc/scanner/2_pyrit_shell.md +++ b/doc/scanner/2_pyrit_shell.md @@ -18,17 +18,11 @@ With startup options: ```bash # Load configuration file (if not provided, defaults to ~/.pyrit/.pyrit_conf if it exists) -# to set database preference, initializers, labels, env_file, and more. +# to set database preference, initializers, custom initialization scripts, labels, env_file, and more. pyrit_shell --config-file ./.pyrit_conf # Set default log level pyrit_shell --log-level DEBUG - -# Load initializers at startup -pyrit_shell --initializers target - -# Load custom initialization scripts -pyrit_shell --initialization-scripts ./my_config.py ``` ## Available Commands @@ -102,7 +96,6 @@ pyrit> run garak.encoding --target my_target --initializers target --log-level D ``` --initializers ... Built-in initializers to run before the scenario (REQUIRED) ---initialization-scripts <...> Custom Python scripts to run before the scenario (alternative) --techniques, -t ... Technique names to use --max-concurrency Maximum concurrent operations --max-retries Maximum retry attempts diff --git a/doc/scanner/airt.ipynb b/doc/scanner/airt.ipynb index 7a1e2cf1eb..bfa48e6b62 100644 --- a/doc/scanner/airt.ipynb +++ b/doc/scanner/airt.ipynb @@ -91,7 +91,7 @@ "different attack technique to the full set of harm datasets.\n", "\n", "```bash\n", - "pyrit_scan airt.rapid_response \\\n", + "pyrit_scan run airt.rapid_response \\\n", " --initializers target \\\n", " --target openai_chat \\\n", " --techniques role_play_movie_script \\\n", @@ -223,7 +223,7 @@ "default, each with its own dataset, escalation prompt, and conversation-level scorer.\n", "\n", "```bash\n", - "pyrit_scan airt.psychosocial --target openai_chat --techniques tone\n", + "pyrit_scan run airt.psychosocial --target openai_chat --techniques tone\n", "```\n", "\n", "Each sub-harm escalates a simulated multi-turn conversation toward the objective, then layers the\n", @@ -362,7 +362,7 @@ "and multi-turn attacks.\n", "\n", "```bash\n", - "pyrit_scan airt.cyber \\\n", + "pyrit_scan run airt.cyber \\\n", " --initializers target \\\n", " --target openai_chat \\\n", " --techniques multi_turn \\\n", @@ -497,7 +497,7 @@ "included by default so complying with the bare objective is itself visible.\n", "\n", "```bash\n", - "pyrit_scan airt.jailbreak \\\n", + "pyrit_scan run airt.jailbreak \\\n", " --initializers target load_default_datasets \\\n", " --target openai_chat \\\n", " --dataset-names harmbench \\\n", @@ -1274,7 +1274,7 @@ "plagiarism detection.\n", "\n", "```bash\n", - "pyrit_scan airt.leakage --target openai_chat --techniques first_letter --max-dataset-size 1\n", + "pyrit_scan run airt.leakage --target openai_chat --techniques first_letter --max-dataset-size 1\n", "```\n", "\n", "**Available techniques:** ALL, SINGLE_TURN, MULTI_TURN, IP, SENSITIVE_DATA, FirstLetter, Image, RolePlay, Crescendo\n", @@ -1422,7 +1422,7 @@ "Tests whether a target can be induced to generate scam, phishing, or fraud content.\n", "\n", "```bash\n", - "pyrit_scan airt.scam \\\n", + "pyrit_scan run airt.scam \\\n", " --initializers target \\\n", " --target openai_chat \\\n", " --techniques context_compliance \\\n", diff --git a/doc/scanner/airt.py b/doc/scanner/airt.py index d97c859788..9c79e7a9cf 100644 --- a/doc/scanner/airt.py +++ b/doc/scanner/airt.py @@ -44,7 +44,7 @@ # different attack technique to the full set of harm datasets. # # ```bash -# pyrit_scan airt.rapid_response \ +# pyrit_scan run airt.rapid_response \ # --initializers target \ # --target openai_chat \ # --techniques role_play_movie_script \ @@ -84,7 +84,7 @@ # default, each with its own dataset, escalation prompt, and conversation-level scorer. # # ```bash -# pyrit_scan airt.psychosocial --target openai_chat --techniques tone +# pyrit_scan run airt.psychosocial --target openai_chat --techniques tone # ``` # # Each sub-harm escalates a simulated multi-turn conversation toward the objective, then layers the @@ -125,7 +125,7 @@ # and multi-turn attacks. # # ```bash -# pyrit_scan airt.cyber \ +# pyrit_scan run airt.cyber \ # --initializers target \ # --target openai_chat \ # --techniques multi_turn \ @@ -168,7 +168,7 @@ # included by default so complying with the bare objective is itself visible. # # ```bash -# pyrit_scan airt.jailbreak \ +# pyrit_scan run airt.jailbreak \ # --initializers target load_default_datasets \ # --target openai_chat \ # --dataset-names harmbench \ @@ -208,7 +208,7 @@ # plagiarism detection. # # ```bash -# pyrit_scan airt.leakage --target openai_chat --techniques first_letter --max-dataset-size 1 +# pyrit_scan run airt.leakage --target openai_chat --techniques first_letter --max-dataset-size 1 # ``` # # **Available techniques:** ALL, SINGLE_TURN, MULTI_TURN, IP, SENSITIVE_DATA, FirstLetter, Image, RolePlay, Crescendo @@ -258,7 +258,7 @@ # Tests whether a target can be induced to generate scam, phishing, or fraud content. # # ```bash -# pyrit_scan airt.scam \ +# pyrit_scan run airt.scam \ # --initializers target \ # --target openai_chat \ # --techniques context_compliance \ diff --git a/doc/scanner/benchmark.ipynb b/doc/scanner/benchmark.ipynb index 6d165db104..5f4d0a027a 100644 --- a/doc/scanner/benchmark.ipynb +++ b/doc/scanner/benchmark.ipynb @@ -26,11 +26,11 @@ "\n", "Adversarial targets are user-provided via the `adversarial_targets` scenario parameter. Each name\n", "must already be registered in `TargetRegistry` — typically by `TargetInitializer` from the\n", - "`ADVERSARIAL_CHAT_*` env vars (see `.env_example`). Use `pyrit_scan --list-targets` to see every\n", + "`ADVERSARIAL_CHAT_*` env vars (see `.env_example`). Use `pyrit_scan list-targets` to see every\n", "target currently registered.\n", "\n", "```bash\n", - "pyrit_scan benchmark.adversarial \\\n", + "pyrit_scan run benchmark.adversarial \\\n", " --initializers target \\\n", " --target openai_chat \\\n", " --adversarial-targets adversarial_chat_singleturn adversarial_chat_multiturn \\\n", diff --git a/doc/scanner/benchmark.py b/doc/scanner/benchmark.py index 1cf3762998..99a8228425 100644 --- a/doc/scanner/benchmark.py +++ b/doc/scanner/benchmark.py @@ -25,11 +25,11 @@ # # Adversarial targets are user-provided via the `adversarial_targets` scenario parameter. Each name # must already be registered in `TargetRegistry` — typically by `TargetInitializer` from the -# `ADVERSARIAL_CHAT_*` env vars (see `.env_example`). Use `pyrit_scan --list-targets` to see every +# `ADVERSARIAL_CHAT_*` env vars (see `.env_example`). Use `pyrit_scan list-targets` to see every # target currently registered. # # ```bash -# pyrit_scan benchmark.adversarial \ +# pyrit_scan run benchmark.adversarial \ # --initializers target \ # --target openai_chat \ # --adversarial-targets adversarial_chat_singleturn adversarial_chat_multiturn \ diff --git a/doc/scanner/foundry.ipynb b/doc/scanner/foundry.ipynb index 2760a1412e..9d655096bc 100644 --- a/doc/scanner/foundry.ipynb +++ b/doc/scanner/foundry.ipynb @@ -82,7 +82,7 @@ "**CLI example:**\n", "\n", "```bash\n", - "pyrit_scan foundry.red_team_agent --target openai_chat --techniques base64 --max-dataset-size 1\n", + "pyrit_scan run foundry.red_team_agent --target openai_chat --techniques base64 --max-dataset-size 1\n", "```\n", "\n", "**Available techniques by difficulty:**\n", diff --git a/doc/scanner/foundry.py b/doc/scanner/foundry.py index 53e8c2ba20..74c21e403b 100644 --- a/doc/scanner/foundry.py +++ b/doc/scanner/foundry.py @@ -40,7 +40,7 @@ # **CLI example:** # # ```bash -# pyrit_scan foundry.red_team_agent --target openai_chat --techniques base64 --max-dataset-size 1 +# pyrit_scan run foundry.red_team_agent --target openai_chat --techniques base64 --max-dataset-size 1 # ``` # # **Available techniques by difficulty:** diff --git a/doc/scanner/garak.ipynb b/doc/scanner/garak.ipynb index 5ec637fbf1..c476725a5e 100644 --- a/doc/scanner/garak.ipynb +++ b/doc/scanner/garak.ipynb @@ -84,7 +84,7 @@ "**CLI example:**\n", "\n", "```bash\n", - "pyrit_scan garak.encoding --target openai_chat --techniques base64 --max-dataset-size 1\n", + "pyrit_scan run garak.encoding --target openai_chat --techniques base64 --max-dataset-size 1\n", "```\n", "\n", "**Available techniques** (17 encodings): Base64, Base2048, Base16, Base32, ASCII85, Hex,\n", @@ -150,7 +150,7 @@ "**CLI example:**\n", "\n", "```bash\n", - "pyrit_scan garak.web_injection --target openai_chat --techniques xss --max-dataset-size 1\n", + "pyrit_scan run garak.web_injection --target openai_chat --techniques xss --max-dataset-size 1\n", "```\n", "\n", "**Available techniques** (8 probes): MarkdownImageExfil, ColabAIDataLeakage,\n", @@ -177,7 +177,7 @@ "**CLI example:**\n", "\n", "```bash\n", - "pyrit_scan garak.doctor --target openai_chat --techniques policy_puppetry --max-dataset-size 1\n", + "pyrit_scan run garak.doctor --target openai_chat --techniques policy_puppetry --max-dataset-size 1\n", "```\n", "\n", "**Available techniques** (2 probes): `PolicyPuppetry` (wraps the objective in the Dr House\n", diff --git a/doc/scanner/garak.py b/doc/scanner/garak.py index 80bbd852f0..06584f2e6e 100644 --- a/doc/scanner/garak.py +++ b/doc/scanner/garak.py @@ -42,7 +42,7 @@ # **CLI example:** # # ```bash -# pyrit_scan garak.encoding --target openai_chat --techniques base64 --max-dataset-size 1 +# pyrit_scan run garak.encoding --target openai_chat --techniques base64 --max-dataset-size 1 # ``` # # **Available techniques** (17 encodings): Base64, Base2048, Base16, Base32, ASCII85, Hex, @@ -89,7 +89,7 @@ # **CLI example:** # # ```bash -# pyrit_scan garak.web_injection --target openai_chat --techniques xss --max-dataset-size 1 +# pyrit_scan run garak.web_injection --target openai_chat --techniques xss --max-dataset-size 1 # ``` # # **Available techniques** (8 probes): MarkdownImageExfil, ColabAIDataLeakage, @@ -111,7 +111,7 @@ # **CLI example:** # # ```bash -# pyrit_scan garak.doctor --target openai_chat --techniques policy_puppetry --max-dataset-size 1 +# pyrit_scan run garak.doctor --target openai_chat --techniques policy_puppetry --max-dataset-size 1 # ``` # # **Available techniques** (2 probes): `PolicyPuppetry` (wraps the objective in the Dr House diff --git a/pyrit/cli/_cli_args.py b/pyrit/cli/_cli_args.py index a3df74c873..3c7980ad06 100644 --- a/pyrit/cli/_cli_args.py +++ b/pyrit/cli/_cli_args.py @@ -420,15 +420,15 @@ def _coerce_filter_values(value: str) -> list[str]: } -def add_results_arguments(*, parser: argparse.ArgumentParser, include_id_flag: bool = False) -> None: +def add_results_arguments(*, parser: argparse.ArgumentParser) -> None: """ Add the shared ``scenario-results`` selection flags to *parser*. Registers ``--view``, ``--attack-result-ids``, and ``--limit`` in a - ``scenario results`` group so that ``pyrit_scan`` and ``pyrit_shell`` expose - an identical results interface. The scenario-result id differs by surface — - a ``--scenario-results`` value in scan versus a positional in the shell — so - it is only added here when *include_id_flag* is set (the scan case). + ``scenario results`` group so that ``pyrit_scan`` (its ``scenario-results`` + sub-parser) and ``pyrit_shell`` (``build_scenario_results_parser``) expose an + identical results interface. Both surfaces take the scenario-result id as a + positional, added by the caller. ``--view`` defaults to ``None`` (not ``OVERVIEW``) so callers can tell an explicit ``--view`` apart from an omitted one; resolve it with @@ -436,17 +436,8 @@ def add_results_arguments(*, parser: argparse.ArgumentParser, include_id_flag: b Args: parser (argparse.ArgumentParser): The parser to extend. - include_id_flag (bool): When True, also register ``--scenario-results`` - (scan's mode flag). Defaults to False. """ group = parser.add_argument_group("scenario results") - if include_id_flag: - group.add_argument( - "--scenario-results", - dest="scenario_results", - metavar="SCENARIO_RESULT_ID", - help="Print results for a completed scenario run and exit", - ) group.add_argument( "--view", type=parse_scenario_result_view, diff --git a/pyrit/cli/_output.py b/pyrit/cli/_output.py index 70fbc60421..6e2ab40b9f 100644 --- a/pyrit/cli/_output.py +++ b/pyrit/cli/_output.py @@ -187,7 +187,7 @@ def print_target_list(*, items: list[TargetInstance]) -> None: print("\nNo targets found in registry.") print( "\nTargets are registered by initializers. Include an initializer that " - "registers targets, for example:\n --initializers target\n" + "registers targets in your config file" ) return diff --git a/pyrit/cli/pyrit_scan.py b/pyrit/cli/pyrit_scan.py index 7aaf3bd946..ca33ca4d0f 100644 --- a/pyrit/cli/pyrit_scan.py +++ b/pyrit/cli/pyrit_scan.py @@ -83,47 +83,45 @@ def _print_cli_exception(*, exc: BaseException) -> None: _DESCRIPTION = """PyRIT Scanner - Run AI security scenarios from the command line. -Requires a running PyRIT backend server. Use --start-server to launch one, +Requires a running PyRIT backend server. Use 'start-server' to launch one, or connect to an existing server with --server-url. +Global options (usable with any command, before or after the verb): + --server-url --config-file --log-level --request-timeout --start-server --startup-timeout +Run 'pyrit_scan --help' for full option descriptions and a command's arguments. + Examples: # Start the backend server - pyrit_scan --start-server - - # List scenarios, initializers, targets, or converters - pyrit_scan --list-scenarios - pyrit_scan --list-initializers - pyrit_scan --list-targets - pyrit_scan --list-converters + pyrit_scan start-server - # List available datasets - pyrit_scan --list-datasets + # List scenarios, targets, or converters + pyrit_scan list-scenarios + pyrit_scan list-targets # Run single-turn cyber attacks against a target - pyrit_scan airt.cyber --target openai_chat --techniques single_turn + pyrit_scan run airt.cyber --target openai_chat --techniques single_turn # Run rapid response with specific datasets and concurrency - pyrit_scan airt.rapid_response --target openai_chat + pyrit_scan run airt.rapid_response --target openai_chat --techniques role_play_movie_script --dataset-names airt_hate --max-dataset-size 5 --max-concurrency 4 # Attach registered converters to a technique (repeatable, applied in order) - pyrit_scan airt.rapid_response --target openai_chat + pyrit_scan run airt.rapid_response --target openai_chat --techniques role_play_movie_script:converter.translation_spanish:converter.leetspeak - # Run multi-turn red team agent with labels for tracking - pyrit_scan airt.red_team_agent --target openai_chat - --techniques crescendo - --memory-labels '{"experiment":"baseline"}' + # List recent runs, then inspect one (overview by default; --view attacks for per-attack rows) + pyrit_scan scenario-history 20 + pyrit_scan scenario-results 605d715b-7c07-4bde-a8f9-22fea0b50c4f --view attacks # Register a custom initializer from a Python script - pyrit_scan --add-initializer ./my_custom_init.py + pyrit_scan add-initializer ./my_custom_init.py # Connect to a remote server - pyrit_scan --server-url http://remote:8000 --list-scenarios + pyrit_scan list-scenarios --server-url http://remote:8000 # Stop the server - pyrit_scan --stop-server + pyrit_scan stop-server """ @@ -137,59 +135,49 @@ def _positive_finite_float(value: str) -> float: return parsed -def _build_base_parser(*, add_help: bool = True) -> ArgumentParser: +def _build_global_parser() -> ArgumentParser: """ - Build the ``pyrit_scan`` argparse parser with the built-in (non-scenario) flags. + Build the parser holding options valid for *every* subcommand. - Args: - add_help (bool): Whether to register the ``-h``/``--help`` action. + This parser is never used on its own. It is passed as ``parents=[...]`` to + each verb's sub-parser so that global options work after any verb, e.g. + ``pyrit_scan run foo --server-url X`` or ``pyrit_scan list-scenarios + --server-url X``. Returns: - ArgumentParser: Parser with all built-in flags registered. + ArgumentParser: A help-less parent parser with the global options. """ - parser = ArgumentParser( - prog="pyrit_scan", - description=_DESCRIPTION, - formatter_class=RawDescriptionHelpFormatter, - add_help=add_help, - ) - - # -- Server management -- - server_group = parser.add_argument_group("server") - server_group.add_argument( + parser = ArgumentParser(add_help=False) + group = parser.add_argument_group("global options") + group.add_argument( "--server-url", type=str, help="URL of the PyRIT backend server (default: http://localhost:8000)", ) - server_group.add_argument( + group.add_argument( "--start-server", action="store_true", - help="Start a local backend server if one is not already running", + help="Start a local backend server first if one is not already running", ) - server_group.add_argument( - "--stop-server", - action="store_true", - help="Stop the backend server and exit", - ) - server_group.add_argument( + group.add_argument( "--startup-timeout", type=_positive_finite_float, default=None, metavar="SECONDS", help="Seconds to wait for a local backend to start (default: server.startup_timeout or 120)", ) - server_group.add_argument( + group.add_argument( "--config-file", type=Path, help=ARG_HELP["config_file"], ) - server_group.add_argument( + group.add_argument( "--log-level", type=validate_log_level_argparse, default=logging.WARNING, help="Logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL) (default: WARNING)", ) - server_group.add_argument( + group.add_argument( "--request-timeout", type=float, default=None, @@ -199,62 +187,32 @@ def _build_base_parser(*, add_help: bool = True) -> ArgumentParser: "scenario run always waits indefinitely regardless of this value." ), ) + return parser - # -- Discovery -- - discovery_group = parser.add_argument_group("discovery") - discovery_group.add_argument( - "--list-scenarios", - action="store_true", - help="List all available scenarios and exit", - ) - discovery_group.add_argument( - "--list-initializers", - action="store_true", - help="List all available initializers and exit", - ) - discovery_group.add_argument( - "--list-targets", - action="store_true", - help="List all available targets and exit", - ) - discovery_group.add_argument( - "--list-converters", - action="store_true", - help="List all registered converter instances and exit", - ) - discovery_group.add_argument( - "--list-datasets", - action="store_true", - help="List all available datasets and exit", - ) - discovery_group.add_argument( - "--add-initializer", - type=str, - nargs="+", - metavar="FILE", - help="Register initializer(s) from Python script file(s) and exit", - ) - # -- Scenario run -- - run_group = parser.add_argument_group("scenario run") - run_group.add_argument( +def _add_run_arguments(*, parser: ArgumentParser, scenario_params: list[Parameter] | None = None) -> None: + """ + Add the ``run`` verb's arguments (scenario positional + run flags) to *parser*. + + Args: + parser (ArgumentParser): The ``run`` sub-parser to populate. + scenario_params (list[Parameter] | None): Scenario-declared parameters to + register as flags. Provided on the second parse pass, once the + scenario metadata has been fetched. Defaults to None. + """ + parser.add_argument( "scenario_name", type=str, - nargs="?", help="Name of the scenario to run", ) - run_group.add_argument( - "--target", - type=str, - help=ARG_HELP["target"], - ) - run_group.add_argument( + parser.add_argument("--target", type=str, help=ARG_HELP["target"]) + parser.add_argument( "--initializers", type=_parse_initializer_arg, nargs="+", help=ARG_HELP["initializers"], ) - run_group.add_argument( + parser.add_argument( "--techniques", "-t", type=str, @@ -262,46 +220,111 @@ def _build_base_parser(*, add_help: bool = True) -> ArgumentParser: dest="scenario_techniques", help=ARG_HELP["scenario_techniques"], ) - run_group.add_argument( - "--max-concurrency", - type=positive_int, - help=ARG_HELP["max_concurrency"], + parser.add_argument("--max-concurrency", type=positive_int, help=ARG_HELP["max_concurrency"]) + parser.add_argument("--max-retries", type=non_negative_int, help=ARG_HELP["max_retries"]) + parser.add_argument("--memory-labels", type=str, help=ARG_HELP["memory_labels"]) + parser.add_argument("--dataset-names", type=str, nargs="+", help=ARG_HELP["dataset_names"]) + parser.add_argument("--max-dataset-size", type=positive_int, help=ARG_HELP["max_dataset_size"]) + parser.add_argument( + "--dataset-filters", + type=parse_dataset_filter, + nargs="+", + metavar="KEY=VALUE", + help=ARG_HELP["dataset_filters"], + ) + if scenario_params: + _add_scenario_params_from_api(parser=parser, params=scenario_params) + + +#: Discovery verbs that only list a catalog and exit, mapped to their help text. +_LIST_VERBS: dict[str, str] = { + "list-scenarios": "List all available scenarios", + "list-initializers": "List all available initializers", + "list-targets": "List all available targets", + "list-converters": "List all registered converter instances", + "list-datasets": "List all available datasets", +} + + +def _build_parser(*, scenario_params: list[Parameter] | None = None, add_help: bool = True) -> ArgumentParser: + """ + Build the top-level ``pyrit_scan`` parser with one sub-parser per verb. + + Args: + scenario_params (list[Parameter] | None): Scenario-declared parameters to + register on the ``run`` sub-parser (second parse pass). Defaults to None. + add_help (bool): Whether to register the ``-h``/``--help`` action. + + Returns: + ArgumentParser: The configured parser. + """ + global_parser = _build_global_parser() + parser = ArgumentParser( + prog="pyrit_scan", + description=_DESCRIPTION, + formatter_class=RawDescriptionHelpFormatter, + add_help=add_help, ) - run_group.add_argument( - "--max-retries", - type=non_negative_int, - help=ARG_HELP["max_retries"], + subparsers = parser.add_subparsers(dest="command", metavar="", title="commands") + + run_parser = subparsers.add_parser( + "run", + parents=[global_parser], + help="Run a scenario against a target", + formatter_class=RawDescriptionHelpFormatter, ) - run_group.add_argument( - "--memory-labels", - type=str, - help=ARG_HELP["memory_labels"], + _add_run_arguments(parser=run_parser, scenario_params=scenario_params) + + for verb, help_text in _LIST_VERBS.items(): + subparsers.add_parser(verb, parents=[global_parser], help=help_text) + + add_init_parser = subparsers.add_parser( + "add-initializer", + parents=[global_parser], + help="Register initializer(s) from Python script file(s)", ) - run_group.add_argument( - "--dataset-names", + add_init_parser.add_argument( + "files", type=str, nargs="+", - help=ARG_HELP["dataset_names"], + metavar="FILE", + help="Initializer script file(s) to register", ) - run_group.add_argument( - "--max-dataset-size", - type=positive_int, - help=ARG_HELP["max_dataset_size"], + + results_parser = subparsers.add_parser( + "scenario-results", + parents=[global_parser], + help="Inspect the results of a completed scenario run", ) - run_group.add_argument( - "--dataset-filters", - type=parse_dataset_filter, - nargs="+", - metavar="KEY=VALUE", - help=ARG_HELP["dataset_filters"], + results_parser.add_argument("scenario_result_id", type=str, help="Scenario result id to inspect") + add_results_arguments(parser=results_parser) + + history_parser = subparsers.add_parser( + "scenario-history", + parents=[global_parser], + help="List recent scenario runs", + ) + history_parser.add_argument( + "limit", + type=positive_int, + nargs="?", + default=10, + metavar="N", + help="Number of recent runs to show (default: 10)", ) - # -- Scenario results (inspect a completed run and exit) -- - add_results_arguments(parser=parser, include_id_flag=True) + subparsers.add_parser("start-server", parents=[global_parser], help="Start a local backend server") + subparsers.add_parser("stop-server", parents=[global_parser], help="Stop the backend server") return parser +#: Every valid subcommand verb (used by the legacy-argv shim to detect new-style calls). +_KNOWN_VERBS: frozenset[str] = frozenset( + {"run", "add-initializer", "scenario-results", "scenario-history", "start-server", "stop-server", *_LIST_VERBS} +) + + # Namespacing prefix for scenario-declared params on the parsed Namespace. _SCENARIO_DEST_PREFIX = "scenario__" @@ -414,16 +437,91 @@ def _extract_scenario_args(*, parsed: Namespace) -> dict[str, Any]: } +#: Legacy "mode flag" → new subcommand verb, used by the back-compat shim. +#: ``--start-server`` is intentionally absent: it stays a global modifier flag and +#: only maps to the ``start-server`` verb when it appears with no other command. +_LEGACY_COMMAND_FLAGS: dict[str, str] = { + "--list-scenarios": "list-scenarios", + "--list-initializers": "list-initializers", + "--list-targets": "list-targets", + "--list-converters": "list-converters", + "--list-datasets": "list-datasets", + "--add-initializer": "add-initializer", + "--stop-server": "stop-server", +} + + +def _warn_legacy(*, old: str, new: str) -> None: + """Warn (visibly and via ``DeprecationWarning``) about a legacy ``pyrit_scan`` invocation.""" + from pyrit.common.deprecation import print_deprecation_message + + print_deprecation_message(old_item=f"pyrit_scan {old}", new_item=f"pyrit_scan {new}", removed_in="1.3.0") + # DeprecationWarning is suppressed by default in a CLI, so also print a visible note. + print(f"Note: 'pyrit_scan {old}' is deprecated; use 'pyrit_scan {new}' instead.", file=sys.stderr) + + +def _translate_legacy_argv(argv: list[str]) -> list[str]: + """ + Rewrite legacy flag-style invocations into the new subcommand form. + + Back-compat shim for one release. Maps ``--list-scenarios`` → ``list-scenarios`` + etc., a bare ```` → ``run `` (implicit run), and a standalone + ``--start-server`` → the ``start-server`` verb, emitting a deprecation warning + for each. It also (without warning) moves a new-style verb to the front when it + was placed after global options (e.g. ``--server-url X list-scenarios``), so + globals work before or after the verb. New-style calls that already start with a + verb pass through untouched, as does the brand-new ``scenario-results`` surface + (no legacy form). Delete this function when the deprecation window closes. + + Args: + argv (list[str]): The raw argument list (already ``sys.argv[1:]``). + + Returns: + list[str]: The possibly-rewritten argument list to feed to argparse. + """ + if not argv or argv[0] in _KNOWN_VERBS or argv[0] in ("-h", "--help"): + return argv + + # A legacy command flag anywhere in argv → prepend its verb, drop the flag. + for index, token in enumerate(argv): + verb = _LEGACY_COMMAND_FLAGS.get(token) + if verb is not None: + _warn_legacy(old=token, new=verb) + return [verb, *argv[:index], *argv[index + 1 :]] + + # No legacy command flag. Strip global options with the global parser and + # inspect what remains. + _, leftover = _build_global_parser().parse_known_args(argv) + if leftover: + if leftover[0] in _KNOWN_VERBS: + # A new-style verb placed after global options (e.g. + # ``--server-url X list-scenarios``). Move the verb to the front so its + # sub-parser sees the globals. This is valid ordering, not a legacy + # form, so it does not warn. + verb = leftover[0] + index = argv.index(verb) + return [verb, *argv[:index], *argv[index + 1 :]] + # Otherwise it is a bare scenario name (+ run flags) → implicit run. + _warn_legacy(old=" (implicit run)", new="run ") + return ["run", *argv] + + # Only global options remained: a standalone --start-server means "just start". + if "--start-server" in argv: + _warn_legacy(old="--start-server", new="start-server") + return ["start-server", *[token for token in argv if token != "--start-server"]] + + return argv + + def parse_args(args: list[str] | None = None) -> Namespace: """ Parse command-line arguments (pass 1 — tolerant of scenario-declared flags). - Pass 1 uses ``parse_known_args`` so scenario-specific flags (e.g. - ``--max-turns 7``) don't cause an error before we've had a chance to - fetch the scenario's declared parameters from the server. The unknown - leftovers are stashed on the returned Namespace as ``_unknown_args`` - so ``_reparse_with_scenario_params`` can detect truly unknown flags - when no scenario was specified. + The raw argv is first run through ``_translate_legacy_argv`` (the back-compat + shim). Pass 1 then uses ``parse_known_args`` so scenario-specific flags (e.g. + ``--max-turns 7``) don't error before we've fetched the scenario's declared + parameters. Unknown leftovers are stashed on the Namespace as ``_unknown_args``, + and the translated argv as ``_translated_args``, for the ``run`` reparse. Args: args: Argument list (``sys.argv[1:]`` when None). @@ -431,10 +529,12 @@ def parse_args(args: list[str] | None = None) -> Namespace: Returns: Namespace: Parsed command-line arguments. """ - parser = _build_base_parser(add_help=True) - parsed, unknown = parser.parse_known_args(args) + raw_args = list(args) if args is not None else list(sys.argv[1:]) + translated = _translate_legacy_argv(raw_args) + parser = _build_parser(add_help=True) + parsed, unknown = parser.parse_known_args(translated) parsed._unknown_args = unknown - parsed._raw_args = list(args) if args is not None else list(sys.argv[1:]) + parsed._translated_args = translated return parsed @@ -491,26 +591,6 @@ async def _resolve_server_url_async(*, parsed_args: Namespace) -> str | None: return None -def _is_command_specified(*, parsed_args: Namespace) -> bool: - """ - Return True if the user supplied any actionable command flag (besides - ``--start-server`` / ``--stop-server``). - - Returns: - bool: ``True`` if at least one actionable command flag was provided. - """ - return bool( - parsed_args.list_scenarios - or parsed_args.list_initializers - or parsed_args.list_targets - or parsed_args.list_converters - or parsed_args.list_datasets - or parsed_args.add_initializer - or parsed_args.scenario_results - or parsed_args.scenario_name - ) - - def _resolve_configured_server_url(*, parsed_args: Namespace) -> str: """ Resolve the effective server URL (without probing). @@ -553,36 +633,29 @@ async def _handle_stop_server_async(*, parsed_args: Namespace) -> int: return 0 -async def _handle_list_commands_async(*, client: Any, parsed_args: Namespace) -> int | None: +async def _handle_list_commands_async(*, client: Any, parsed_args: Namespace) -> int: """ - Dispatch ``--list-*`` flags. + Dispatch a ``list-*`` verb. Returns: - int | None: Exit code if a flag was handled, else ``None``. + int: Exit code (always ``0`` on success). """ from pyrit.cli import _output - if parsed_args.list_scenarios: - scenarios = await client.list_scenarios_async() - _output.print_scenario_list(items=scenarios) - return 0 - if parsed_args.list_initializers: - initializers = await client.list_initializers_async() - _output.print_initializer_list(items=initializers) - return 0 - if parsed_args.list_targets: - targets = await client.list_targets_async() - _output.print_target_list(items=targets) - return 0 - if parsed_args.list_datasets: + command = parsed_args.command + if command == "list-scenarios": + _output.print_scenario_list(items=await client.list_scenarios_async()) + elif command == "list-initializers": + _output.print_initializer_list(items=await client.list_initializers_async()) + elif command == "list-targets": + _output.print_target_list(items=await client.list_targets_async()) + elif command == "list-datasets": resp = await client.list_datasets_async() _output.print_dataset_list(items=resp.get("items", [])) - return 0 - if parsed_args.list_converters: + elif command == "list-converters": resp = await client.list_converters_async() _output.print_converter_list(items=resp.get("items", [])) - return 0 - return None + return 0 async def _handle_add_initializer_async(*, client: Any, parsed_args: Namespace) -> int: @@ -594,7 +667,7 @@ async def _handle_add_initializer_async(*, client: Any, parsed_args: Namespace) """ from pyrit.cli.api_client import ServerNotAvailableError - for script_path_str in parsed_args.add_initializer: + for script_path_str in parsed_args.files: script_path = Path(script_path_str).resolve() if not script_path.exists(): print(f"Error: File not found: {script_path}") @@ -612,38 +685,9 @@ async def _handle_add_initializer_async(*, client: Any, parsed_args: Namespace) return 0 -def _validate_results_flags(*, parsed_args: Namespace) -> str | None: - """ - Ensure the ``scenario-results`` sub-flags are only used with ``--scenario-results``. - - ``--view`` / ``--attack-result-ids`` / ``--limit`` only mean something when a - run is being inspected. Because the flat parser accepts them regardless, this - check gives a clear error instead of the generic "no scenario specified" - fallthrough. - - Args: - parsed_args (Namespace): The parsed CLI arguments. - - Returns: - str | None: An error message when a sub-flag is misused, else None. - """ - if parsed_args.scenario_results: - return None - misused: list[str] = [] - if parsed_args.view is not None: - misused.append("--view") - if parsed_args.attack_result_ids: - misused.append("--attack-result-ids") - if parsed_args.limit is not None: - misused.append("--limit") - if not misused: - return None - return f"Error: {', '.join(misused)} require --scenario-results ." - - async def _handle_results_async(*, client: Any, parsed_args: Namespace) -> int: """ - Handle ``--scenario-results``: fetch a run and render the requested view. + Handle the ``scenario-results`` verb: fetch a run and render the requested view. Returns: int: Exit code (``0`` on success, ``1`` on error). @@ -652,7 +696,7 @@ async def _handle_results_async(*, client: Any, parsed_args: Namespace) -> int: from pyrit.cli._cli_args import ScenarioResultView from pyrit.cli._results import apply_view_limit_policy, build_attacks_table_payload, resolve_view - scenario_result_id = parsed_args.scenario_results + scenario_result_id = parsed_args.scenario_result_id view = resolve_view(view=parsed_args.view) limit = apply_view_limit_policy(view=view, limit=parsed_args.limit) @@ -676,34 +720,47 @@ async def _handle_results_async(*, client: Any, parsed_args: Namespace) -> int: return 0 +async def _handle_scenario_history_async(*, client: Any, parsed_args: Namespace) -> int: + """ + Handle the ``scenario-history`` verb: list recent scenario runs. + + Returns: + int: Exit code (always ``0`` on success). + """ + from pyrit.cli import _output + + runs = await client.list_scenario_runs_async(limit=parsed_args.limit) + _output.print_scenario_runs_list(runs=runs) + return 0 + + def _reparse_with_scenario_params(*, parsed_args: Namespace, supported_params: list[Parameter]) -> Namespace | None: """ - Re-parse the original args with scenario-declared flags added to the base parser. + Re-parse the ``run`` invocation with scenario-declared flags registered. - The original argument list is read from ``parsed_args._raw_args`` (populated - by ``parse_args``). If no scenario-declared parameters are supplied but - pass 1 left unknown args behind, surface the error now via strict re-parse. + The translated argument list is read from ``parsed_args._translated_args`` + (populated by ``parse_args``). If no scenario-declared parameters exist but + pass 1 left unknown args behind, surface the error via a strict re-parse. Returns: Namespace | None: The re-parsed Namespace, or ``None`` on argparse ``SystemExit``. """ - raw_args: list[str] = getattr(parsed_args, "_raw_args", sys.argv[1:] if len(sys.argv) > 1 else []) + translated: list[str] = getattr(parsed_args, "_translated_args", []) if not supported_params: unknown = getattr(parsed_args, "_unknown_args", None) if not unknown: return parsed_args - # Re-parse strictly so argparse prints the standard "unrecognized arguments" error - strict_parser = _build_base_parser(add_help=True) + # Re-parse strictly so argparse prints the standard "unrecognized arguments" error. + strict_parser = _build_parser(add_help=True) try: - return strict_parser.parse_args(raw_args) + return strict_parser.parse_args(translated) except SystemExit: return None - pass2_parser = _build_base_parser(add_help=True) - _add_scenario_params_from_api(parser=pass2_parser, params=supported_params) + pass2_parser = _build_parser(scenario_params=supported_params, add_help=True) try: - return pass2_parser.parse_args(raw_args) + return pass2_parser.parse_args(translated) except SystemExit: return None @@ -852,26 +909,23 @@ async def _run_scenario_async( async def _dispatch_with_client_async(*, client: Any, parsed_args: Namespace) -> int: """ - Dispatch list/add-initializer/scenario-run commands once a client is open. + Dispatch a verb that needs an open API client. Returns: int: Exit code from the dispatched command. """ - list_result = await _handle_list_commands_async(client=client, parsed_args=parsed_args) - if list_result is not None: - return list_result - - if parsed_args.add_initializer: + command = parsed_args.command + if command in _LIST_VERBS: + return await _handle_list_commands_async(client=client, parsed_args=parsed_args) + if command == "add-initializer": return await _handle_add_initializer_async(client=client, parsed_args=parsed_args) - - if parsed_args.scenario_results: + if command == "scenario-results": return await _handle_results_async(client=client, parsed_args=parsed_args) + if command == "scenario-history": + return await _handle_scenario_history_async(client=client, parsed_args=parsed_args) + # command == "run": the scenario positional is required by the run sub-parser. scenario_name = parsed_args.scenario_name - if not scenario_name: - print("Error: No scenario specified. Provide one positionally or use --list-scenarios.") - return 1 - scenario_meta = await client.get_scenario_async(scenario_name=scenario_name) if scenario_meta is None: print(f"Error: Scenario '{scenario_name}' not found on server.") @@ -902,29 +956,26 @@ async def _run_async(*, parsed_args: Namespace) -> int: from pyrit.cli import _output from pyrit.cli.api_client import PyRITApiClient, ServerNotAvailableError - if parsed_args.stop_server: - return await _handle_stop_server_async(parsed_args=parsed_args) + command = parsed_args.command - results_flag_error = _validate_results_flags(parsed_args=parsed_args) - if results_flag_error is not None: - print(results_flag_error, file=sys.stderr) - return 1 + # stop-server needs no API client. + if command == "stop-server": + return await _handle_stop_server_async(parsed_args=parsed_args) - if not (parsed_args.start_server or _is_command_specified(parsed_args=parsed_args)): - _build_base_parser().print_help() - return 0 + # The start-server verb forces an auto-start attempt, then just confirms. + if command == "start-server": + parsed_args.start_server = True base_url_result = await _resolve_server_url_async(parsed_args=parsed_args) if base_url_result is None: attempted = _resolve_configured_server_url(parsed_args=parsed_args) _output.print_error_with_hint( message=f"Server not available at {attempted}", - hint="Use '--start-server' to launch a local backend, or pass '--server-url '.", + hint="Use 'start-server' to launch a local backend, or pass '--server-url '.", ) return 1 - # --start-server with no other command: just confirm and exit - if not _is_command_specified(parsed_args=parsed_args): + if command == "start-server": print(f"Server is running at {base_url_result}") return 0 @@ -937,7 +988,7 @@ async def _run_async(*, parsed_args: Namespace) -> int: except ServerNotAvailableError as exc: _output.print_error_with_hint( message=str(exc), - hint="Use '--start-server' to launch a local backend, or pass '--server-url '.", + hint="Use 'start-server' to launch a local backend, or pass '--server-url '.", ) return 1 except Exception as exc: @@ -957,22 +1008,27 @@ def main(args: list[str] | None = None) -> int: except SystemExit as e: return e.code if isinstance(e.code, int) else 1 - # If there are leftover unknown flags AND no scenario was specified, - # there's no chance for pass 2 to recognize them - fail loudly now. + # No verb at all: show the top-level help listing the subcommands. + if getattr(parsed_args, "command", None) is None: + _build_parser().print_help() + return 0 + + # Unknown flags are only expected for `run` (scenario-declared flags, resolved + # in the reparse). For any other verb they are genuinely unrecognized. unknown = getattr(parsed_args, "_unknown_args", []) - if unknown and not parsed_args.scenario_name: - strict_parser = _build_base_parser(add_help=True) + if unknown and parsed_args.command != "run": + strict_parser = _build_parser(add_help=True) try: - strict_parser.parse_args(parsed_args._raw_args) + strict_parser.parse_args(parsed_args._translated_args) except SystemExit as e: return e.code if isinstance(e.code, int) else 1 - logging.basicConfig(level=parsed_args.log_level) + logging.basicConfig(level=getattr(parsed_args, "log_level", logging.WARNING)) from pyrit.cli._config_reader import ConfigError, validate_client_config try: - validate_client_config(config_file=parsed_args.config_file) + validate_client_config(config_file=getattr(parsed_args, "config_file", None)) return asyncio.run(_run_async(parsed_args=parsed_args)) except ConfigError as exc: print(f"Error: {exc}", file=sys.stderr) diff --git a/tests/unit/cli/test_pyrit_scan.py b/tests/unit/cli/test_pyrit_scan.py index fb7e929e50..d5f901065e 100644 --- a/tests/unit/cli/test_pyrit_scan.py +++ b/tests/unit/cli/test_pyrit_scan.py @@ -48,69 +48,93 @@ def test_dataset_filter_help_covers_every_request_model_key(): class TestParseArgs: - """Tests for parse_args function.""" + """Tests for parse_args with the subcommand model.""" - def test_parse_args_list_scenarios(self): - args = pyrit_scan.parse_args(["--list-scenarios"]) - assert args.list_scenarios is True - assert args.scenario_name is None - - def test_parse_args_list_initializers(self): - args = pyrit_scan.parse_args(["--list-initializers"]) - assert args.list_initializers is True - - def test_parse_args_scenario_name_only(self): - args = pyrit_scan.parse_args(["test_scenario"]) + def test_run_scenario_name(self): + args = pyrit_scan.parse_args(["run", "test_scenario"]) + assert args.command == "run" assert args.scenario_name == "test_scenario" assert args.log_level == logging.WARNING - def test_parse_args_with_log_level(self): - args = pyrit_scan.parse_args(["test_scenario", "--log-level", "DEBUG"]) - assert args.log_level == logging.DEBUG + def test_list_scenarios_verb(self): + assert pyrit_scan.parse_args(["list-scenarios"]).command == "list-scenarios" - def test_parse_args_with_initializers(self): - args = pyrit_scan.parse_args(["test_scenario", "--initializers", "init1", "init2"]) - assert args.initializers == ["init1", "init2"] + def test_list_initializers_verb(self): + assert pyrit_scan.parse_args(["list-initializers"]).command == "list-initializers" + + def test_list_targets_verb(self): + assert pyrit_scan.parse_args(["list-targets"]).command == "list-targets" + + def test_list_converters_verb(self): + assert pyrit_scan.parse_args(["list-converters"]).command == "list-converters" + + def test_list_datasets_verb(self): + assert pyrit_scan.parse_args(["list-datasets"]).command == "list-datasets" + + def test_add_initializer_verb(self): + args = pyrit_scan.parse_args(["add-initializer", "script1.py", "script2.py"]) + assert args.command == "add-initializer" + assert args.files == ["script1.py", "script2.py"] + + def test_stop_server_verb(self): + assert pyrit_scan.parse_args(["stop-server"]).command == "stop-server" - def test_parse_args_with_add_initializer(self): - args = pyrit_scan.parse_args(["--add-initializer", "script1.py", "script2.py"]) - assert args.add_initializer == ["script1.py", "script2.py"] + def test_start_server_verb(self): + assert pyrit_scan.parse_args(["start-server"]).command == "start-server" - def test_parse_args_list_datasets(self): - args = pyrit_scan.parse_args(["--list-datasets"]) - assert args.list_datasets is True + def test_scenario_history_verb_default_limit(self): + args = pyrit_scan.parse_args(["scenario-history"]) + assert args.command == "scenario-history" + assert args.limit == 10 - def test_parse_args_with_techniques(self): - args = pyrit_scan.parse_args(["test_scenario", "--techniques", "s1", "s2"]) + def test_scenario_history_verb_custom_limit(self): + assert pyrit_scan.parse_args(["scenario-history", "25"]).limit == 25 + + def test_no_command_is_none(self): + assert pyrit_scan.parse_args([]).command is None + + def test_run_with_log_level(self): + args = pyrit_scan.parse_args(["run", "test_scenario", "--log-level", "DEBUG"]) + assert args.log_level == logging.DEBUG + + def test_run_with_initializers(self): + args = pyrit_scan.parse_args(["run", "test_scenario", "--initializers", "init1", "init2"]) + assert args.initializers == ["init1", "init2"] + + def test_run_with_techniques(self): + args = pyrit_scan.parse_args(["run", "test_scenario", "--techniques", "s1", "s2"]) assert args.scenario_techniques == ["s1", "s2"] - def test_parse_args_with_techniques_short_flag(self): - args = pyrit_scan.parse_args(["test_scenario", "-t", "s1", "s2"]) + def test_run_with_techniques_short_flag(self): + args = pyrit_scan.parse_args(["run", "test_scenario", "-t", "s1", "s2"]) assert args.scenario_techniques == ["s1", "s2"] - def test_parse_args_with_max_concurrency(self): - args = pyrit_scan.parse_args(["test_scenario", "--max-concurrency", "5"]) + def test_run_with_max_concurrency(self): + args = pyrit_scan.parse_args(["run", "test_scenario", "--max-concurrency", "5"]) assert args.max_concurrency == 5 - def test_parse_args_with_max_retries(self): - args = pyrit_scan.parse_args(["test_scenario", "--max-retries", "3"]) + def test_run_with_max_retries(self): + args = pyrit_scan.parse_args(["run", "test_scenario", "--max-retries", "3"]) assert args.max_retries == 3 - def test_parse_args_with_memory_labels(self): - args = pyrit_scan.parse_args(["test_scenario", "--memory-labels", '{"key":"value"}']) + def test_run_with_memory_labels(self): + args = pyrit_scan.parse_args(["run", "test_scenario", "--memory-labels", '{"key":"value"}']) assert args.memory_labels == '{"key":"value"}' - def test_parse_args_with_dataset_filters(self): - args = pyrit_scan.parse_args(["test_scenario", "--dataset-filters", "harm_categories=cyber", "data_types=text"]) + def test_run_with_dataset_filters(self): + args = pyrit_scan.parse_args( + ["run", "test_scenario", "--dataset-filters", "harm_categories=cyber", "data_types=text"] + ) assert args.dataset_filters == [("harm_categories", "cyber"), ("data_types", "text")] - def test_parse_args_dataset_filter_without_equals_errors(self): + def test_run_dataset_filter_without_equals_errors(self): with pytest.raises(SystemExit): - pyrit_scan.parse_args(["test_scenario", "--dataset-filters", "harm_categories"]) + pyrit_scan.parse_args(["run", "test_scenario", "--dataset-filters", "harm_categories"]) - def test_parse_args_complex_command(self): + def test_run_complex_command(self): args = pyrit_scan.parse_args( [ + "run", "encoding_scenario", "--log-level", "INFO", @@ -134,55 +158,101 @@ def test_parse_args_complex_command(self): assert args.max_concurrency == 10 assert args.max_retries == 5 - def test_parse_args_invalid_log_level(self): + def test_run_invalid_log_level(self): with pytest.raises(SystemExit): - pyrit_scan.parse_args(["test_scenario", "--log-level", "INVALID"]) + pyrit_scan.parse_args(["run", "test_scenario", "--log-level", "INVALID"]) - def test_parse_args_invalid_max_concurrency(self): + def test_run_invalid_max_concurrency(self): with pytest.raises(SystemExit): - pyrit_scan.parse_args(["test_scenario", "--max-concurrency", "0"]) + pyrit_scan.parse_args(["run", "test_scenario", "--max-concurrency", "0"]) - def test_parse_args_invalid_max_retries(self): + def test_run_invalid_max_retries(self): with pytest.raises(SystemExit): - pyrit_scan.parse_args(["test_scenario", "--max-retries", "-1"]) + pyrit_scan.parse_args(["run", "test_scenario", "--max-retries", "-1"]) - def test_parse_args_help_flag(self): + def test_help_flag(self): with pytest.raises(SystemExit) as exc_info: pyrit_scan.parse_args(["--help"]) assert exc_info.value.code == 0 - def test_parse_args_with_target(self): - args = pyrit_scan.parse_args(["test_scenario", "--target", "my_target"]) + def test_run_with_target(self): + args = pyrit_scan.parse_args(["run", "test_scenario", "--target", "my_target"]) assert args.target == "my_target" - def test_parse_args_target_default_is_none(self): - args = pyrit_scan.parse_args(["test_scenario"]) + def test_run_target_default_is_none(self): + args = pyrit_scan.parse_args(["run", "test_scenario"]) assert args.target is None - def test_parse_args_with_list_targets(self): - args = pyrit_scan.parse_args(["--list-targets"]) - assert args.list_targets is True - - def test_parse_args_with_list_converters(self): - args = pyrit_scan.parse_args(["--list-converters"]) - assert args.list_converters is True + def test_list_with_server_url(self): + args = pyrit_scan.parse_args(["list-scenarios", "--server-url", "http://remote:9000"]) + assert args.server_url == "http://remote:9000" - def test_parse_args_with_server_url(self): - args = pyrit_scan.parse_args(["--list-scenarios", "--server-url", "http://remote:9000"]) + def test_global_flag_before_verb(self): + args = pyrit_scan.parse_args(["--server-url", "http://remote:9000", "list-scenarios"]) + assert args.command == "list-scenarios" assert args.server_url == "http://remote:9000" - def test_parse_args_with_start_server(self): - args = pyrit_scan.parse_args(["--list-scenarios", "--start-server"]) + def test_list_with_start_server(self): + args = pyrit_scan.parse_args(["list-scenarios", "--start-server"]) assert args.start_server is True - def test_parse_args_with_stop_server(self): - args = pyrit_scan.parse_args(["--stop-server"]) - assert args.stop_server is True - - def test_parse_args_with_startup_timeout(self): - args = pyrit_scan.parse_args(["--start-server", "--startup-timeout", "45.5"]) + def test_start_server_with_startup_timeout(self): + args = pyrit_scan.parse_args(["start-server", "--startup-timeout", "45.5"]) assert args.startup_timeout == 45.5 + +class TestLegacyArgvShim: + """The back-compat shim maps old flag forms to verbs and warns.""" + + def test_legacy_list_flag_maps_to_verb(self): + with pytest.warns(DeprecationWarning, match="list-scenarios"): + args = pyrit_scan.parse_args(["--list-scenarios"]) + assert args.command == "list-scenarios" + + def test_legacy_stop_server_flag_maps_to_verb(self): + with pytest.warns(DeprecationWarning, match="stop-server"): + args = pyrit_scan.parse_args(["--stop-server"]) + assert args.command == "stop-server" + + def test_legacy_add_initializer_flag_maps_to_verb(self): + with pytest.warns(DeprecationWarning, match="add-initializer"): + args = pyrit_scan.parse_args(["--add-initializer", "a.py", "b.py"]) + assert args.command == "add-initializer" + assert args.files == ["a.py", "b.py"] + + def test_legacy_flag_preserves_globals(self): + with pytest.warns(DeprecationWarning): + args = pyrit_scan.parse_args(["--server-url", "http://x", "--list-targets"]) + assert args.command == "list-targets" + assert args.server_url == "http://x" + + def test_implicit_run_maps_to_run_verb(self): + with pytest.warns(DeprecationWarning, match="run "): + args = pyrit_scan.parse_args(["foundry", "--target", "t"]) + assert args.command == "run" + assert args.scenario_name == "foundry" + assert args.target == "t" + + def test_standalone_start_server_flag_maps_to_verb(self): + with pytest.warns(DeprecationWarning, match="start-server"): + args = pyrit_scan.parse_args(["--start-server"]) + assert args.command == "start-server" + + def test_new_style_verb_does_not_warn(self, recwarn): + args = pyrit_scan.parse_args(["list-scenarios"]) + assert args.command == "list-scenarios" + assert not [w for w in recwarn if issubclass(w.category, DeprecationWarning)] + + def test_legacy_flag_prints_visible_note(self, capsys): + with pytest.warns(DeprecationWarning): + pyrit_scan.parse_args(["--list-scenarios"]) + assert "Note:" in capsys.readouterr().err + + def test_global_before_verb_does_not_warn(self, recwarn): + args = pyrit_scan.parse_args(["--server-url", "http://x", "list-scenarios"]) + assert args.command == "list-scenarios" + assert not [w for w in recwarn if issubclass(w.category, DeprecationWarning)] + @pytest.mark.parametrize("value", ["0", "-1", "inf", "nan", "slow"]) def test_parse_args_rejects_invalid_startup_timeout(self, value): with pytest.raises(SystemExit): @@ -322,7 +392,7 @@ def test_main_list_scenarios(self, mock_client_class, mock_probe): mock_client = _mock_api_client() mock_client_class.return_value = mock_client - result = pyrit_scan.main(["--list-scenarios"]) + result = pyrit_scan.main(["list-scenarios"]) assert result == 0 mock_client.list_scenarios_async.assert_awaited_once() @@ -338,7 +408,7 @@ def test_main_list_initializers(self, mock_client_class, mock_probe): mock_client = _mock_api_client() mock_client_class.return_value = mock_client - result = pyrit_scan.main(["--list-initializers"]) + result = pyrit_scan.main(["list-initializers"]) assert result == 0 mock_client.list_initializers_async.assert_awaited_once() @@ -354,7 +424,7 @@ def test_main_list_targets(self, mock_client_class, mock_probe): mock_client = _mock_api_client() mock_client_class.return_value = mock_client - result = pyrit_scan.main(["--list-targets"]) + result = pyrit_scan.main(["list-targets"]) assert result == 0 mock_client.list_targets_async.assert_awaited_once() @@ -366,7 +436,7 @@ def test_main_list_converters(self, mock_client_class, mock_probe): mock_client = _mock_api_client() mock_client_class.return_value = mock_client - result = pyrit_scan.main(["--list-converters"]) + result = pyrit_scan.main(["list-converters"]) assert result == 0 mock_client.list_converters_async.assert_awaited_once() @@ -378,7 +448,7 @@ def test_main_list_datasets(self, mock_client_class, mock_probe): mock_client = _mock_api_client() mock_client_class.return_value = mock_client - result = pyrit_scan.main(["--list-datasets"]) + result = pyrit_scan.main(["list-datasets"]) assert result == 0 mock_client.list_datasets_async.assert_awaited_once() @@ -400,7 +470,7 @@ def test_main_run_scenario(self, _mock_print, mock_client_class, mock_probe): mock_client = _mock_api_client() mock_client_class.return_value = mock_client - result = pyrit_scan.main(["test_scenario", "--target", "my_target"]) + result = pyrit_scan.main(["run", "test_scenario", "--target", "my_target"]) assert result == 0 mock_client.get_scenario_async.assert_awaited_once() @@ -418,7 +488,7 @@ def test_main_run_scenario_with_initializers(self, _mock_print, mock_client_clas mock_client = _mock_api_client() mock_client_class.return_value = mock_client - result = pyrit_scan.main(["test_scenario", "--target", "t", "--initializers", "target", "datasets"]) + result = pyrit_scan.main(["run", "test_scenario", "--target", "t", "--initializers", "target", "datasets"]) assert result == 0 call_kwargs = mock_client.start_scenario_run_async.call_args.kwargs @@ -432,7 +502,7 @@ def test_main_run_scenario_with_initializers(self, _mock_print, mock_client_clas ) def test_main_server_not_available(self, mock_probe, capsys): """Test main when server is not available.""" - result = pyrit_scan.main(["--list-scenarios"]) + result = pyrit_scan.main(["list-scenarios"]) assert result == 1 captured = capsys.readouterr() @@ -447,7 +517,7 @@ def test_main_malformed_config_is_hard_error(self, tmp_path, capsys): "_DEFAULT_CONFIG_FILE", tmp_path / "missing_default.yaml", ): - result = pyrit_scan.main(["--list-scenarios", "--config-file", str(bad)]) + result = pyrit_scan.main(["list-scenarios", "--config-file", str(bad)]) assert result == 1 assert "not valid YAML" in capsys.readouterr().err @@ -459,7 +529,7 @@ def test_main_malformed_config_is_hard_error(self, tmp_path, capsys): ) def test_main_stop_server(self, mock_probe, capsys): """Test main with --stop-server.""" - result = pyrit_scan.main(["--stop-server"]) + result = pyrit_scan.main(["stop-server"]) assert result == 0 captured = capsys.readouterr() @@ -477,7 +547,7 @@ def test_main_scenario_not_found(self, mock_client_class, mock_probe, capsys): mock_client.get_scenario_async.return_value = None mock_client_class.return_value = mock_client - result = pyrit_scan.main(["nonexistent_scenario", "--target", "t"]) + result = pyrit_scan.main(["run", "nonexistent_scenario", "--target", "t"]) assert result == 1 captured = capsys.readouterr() @@ -512,7 +582,7 @@ def test_main_failed_scenario(self, mock_client_class, mock_probe): ) mock_client_class.return_value = mock_client - result = pyrit_scan.main(["test_scenario", "--target", "t"]) + result = pyrit_scan.main(["run", "test_scenario", "--target", "t"]) assert result == 1 @@ -1063,7 +1133,7 @@ def test_main_scenario_not_found_lists_available(self, mock_client_class, _mock_ ] mock_client_class.return_value = mock_client - result = pyrit_scan.main(["nonexistent", "--target", "t"]) + result = pyrit_scan.main(["run", "nonexistent", "--target", "t"]) assert result == 1 captured = capsys.readouterr() assert "alt_a" in captured.out @@ -1080,7 +1150,7 @@ def test_main_start_scenario_failure(self, mock_client_class, _mock_probe, capsy mock_client.start_scenario_run_async.side_effect = RuntimeError("server full") mock_client_class.return_value = mock_client - result = pyrit_scan.main(["test_scenario", "--target", "t"]) + result = pyrit_scan.main(["run", "test_scenario", "--target", "t"]) assert result == 1 captured = capsys.readouterr() assert "server full" in captured.out @@ -1096,7 +1166,7 @@ def test_main_run_results_failure_is_hard_error(self, mock_client_class, _mock_p mock_client.get_scenario_run_results_async.side_effect = RuntimeError("nope") mock_client_class.return_value = mock_client - result = pyrit_scan.main(["test_scenario", "--target", "t"]) + result = pyrit_scan.main(["run", "test_scenario", "--target", "t"]) # A completed run whose results can't be fetched/parsed is a hard CLI failure. assert result == 1 captured = capsys.readouterr() @@ -1113,7 +1183,7 @@ def test_main_run_results_failure_is_hard_error(self, mock_client_class, _mock_p ) @patch("pyrit.cli.api_client.PyRITApiClient") def test_main_start_server_only_prints_url_and_returns_zero(self, mock_client_class, _mock_probe, capsys): - result = pyrit_scan.main(["--start-server"]) + result = pyrit_scan.main(["start-server"]) assert result == 0 captured = capsys.readouterr() assert "running" in captured.out.lower() @@ -1126,12 +1196,12 @@ def test_main_start_server_only_prints_url_and_returns_zero(self, mock_client_cl @patch("pyrit.cli._server_launcher.stop_server_on_port", return_value=True) def test_main_stop_server_kills_process_and_returns_zero(self, _stop_mock, mock_probe, capsys): mock_probe.side_effect = [True, False] - result = pyrit_scan.main(["--stop-server"]) + result = pyrit_scan.main(["stop-server"]) assert result == 0 assert "stopped" in capsys.readouterr().out async def test_handle_stop_server_offloads_blocking_shutdown(self): - parsed_args = pyrit_scan.parse_args(["--stop-server"]) + parsed_args = pyrit_scan.parse_args(["stop-server"]) to_thread_mock = AsyncMock(return_value=True) probe_mock = AsyncMock(side_effect=[True, False]) @@ -1153,7 +1223,7 @@ async def test_handle_stop_server_offloads_blocking_shutdown(self): ) @patch("pyrit.cli._server_launcher.stop_server_on_port", return_value=False) def test_main_stop_server_when_process_cannot_be_identified(self, _stop_mock, _mock_probe, capsys): - result = pyrit_scan.main(["--stop-server"]) + result = pyrit_scan.main(["stop-server"]) assert result == 1 out = capsys.readouterr().out assert "could not be stopped" in out @@ -1165,7 +1235,7 @@ def test_main_stop_server_when_process_cannot_be_identified(self, _stop_mock, _m ) @patch("pyrit.cli._server_launcher.stop_server_on_port", return_value=True) def test_main_stop_server_fails_when_backend_remains_healthy(self, _stop_mock, _mock_probe, capsys): - result = pyrit_scan.main(["--stop-server"]) + result = pyrit_scan.main(["stop-server"]) assert result == 1 assert "still responding" in capsys.readouterr().out @@ -1175,7 +1245,7 @@ def test_main_stop_server_fails_when_backend_remains_healthy(self, _stop_mock, _ ) @patch("pyrit.cli._server_launcher.stop_server_on_port") def test_main_stop_server_refuses_remote_url(self, stop_mock, probe_mock, capsys): - result = pyrit_scan.main(["--stop-server", "--server-url", "http://remote:8000"]) + result = pyrit_scan.main(["stop-server", "--server-url", "http://remote:8000"]) assert result == 1 stop_mock.assert_not_called() probe_mock.assert_not_called() @@ -1192,7 +1262,7 @@ def test_main_add_initializer_missing_file(self, mock_client_class, _mock_probe, mock_client_class.return_value = mock_client missing = tmp_path / "nonexistent.py" - result = pyrit_scan.main(["--add-initializer", str(missing)]) + result = pyrit_scan.main(["add-initializer", str(missing)]) assert result == 1 assert "File not found" in capsys.readouterr().out @@ -1210,7 +1280,7 @@ def test_main_add_initializer_success(self, mock_client_class, _mock_probe, caps script = tmp_path / "myinit.py" script.write_text("# stub initializer\n") - result = pyrit_scan.main(["--add-initializer", str(script)]) + result = pyrit_scan.main(["add-initializer", str(script)]) assert result == 0 assert "Registered initializer 'myinit'" in capsys.readouterr().out mock_client.register_initializer_async.assert_awaited_once() @@ -1231,48 +1301,30 @@ def test_main_add_initializer_server_disabled(self, mock_client_class, _mock_pro script = tmp_path / "myinit.py" script.write_text("# stub\n") - result = pyrit_scan.main(["--add-initializer", str(script)]) + result = pyrit_scan.main(["add-initializer", str(script)]) assert result == 1 assert "disabled" in capsys.readouterr().out -class TestScenarioResultsFlag: - """Tests for the ``--scenario-results`` mode flag, validation, and dispatch.""" +class TestScenarioResults: + """Tests for the ``scenario-results`` verb and its handler.""" def test_parse_args_recognizes_scenario_results(self): args = pyrit_scan.parse_args( - ["--scenario-results", "SID", "--view", "attacks", "--attack-result-ids", "a", "b", "--limit", "2"] + ["scenario-results", "SID", "--view", "attacks", "--attack-result-ids", "a", "b", "--limit", "2"] ) - assert args.scenario_results == "SID" + assert args.command == "scenario-results" + assert args.scenario_result_id == "SID" assert args.view.value == "attacks" assert args.attack_result_ids == ["a", "b"] assert args.limit == 2 - def test_scenario_results_is_a_specified_command(self): - args = pyrit_scan.parse_args(["--scenario-results", "SID"]) - assert pyrit_scan._is_command_specified(parsed_args=args) is True - - def test_validate_results_flags_allows_bare_scenario_results(self): - args = pyrit_scan.parse_args(["--scenario-results", "SID"]) - assert pyrit_scan._validate_results_flags(parsed_args=args) is None - - def test_validate_results_flags_rejects_subflags_without_id(self): - args = pyrit_scan.parse_args(["test_scenario", "--view", "attacks", "--limit", "3"]) - error = pyrit_scan._validate_results_flags(parsed_args=args) - assert error is not None - assert "--view" in error and "--limit" in error - assert "--scenario-results" in error - - def test_validate_results_flags_ignores_when_no_subflags(self): - args = pyrit_scan.parse_args(["test_scenario"]) - assert pyrit_scan._validate_results_flags(parsed_args=args) is None - def test_handle_results_overview_delegates_to_printer(self): import asyncio client = AsyncMock() client.get_scenario_run_results_async.return_value = _make_scenario_result() - parsed = pyrit_scan.parse_args(["--scenario-results", "SID"]) + parsed = pyrit_scan.parse_args(["scenario-results", "SID"]) with patch("pyrit.cli._output.print_scenario_result_async", new_callable=AsyncMock) as mock_print: rc = asyncio.run(pyrit_scan._handle_results_async(client=client, parsed_args=parsed)) assert rc == 0 @@ -1284,7 +1336,7 @@ def test_handle_results_attacks_prints_table(self, capsys): client = AsyncMock() client.get_scenario_run_results_async.return_value = _make_scenario_result() - parsed = pyrit_scan.parse_args(["--scenario-results", "SID", "--view", "attacks"]) + parsed = pyrit_scan.parse_args(["scenario-results", "SID", "--view", "attacks"]) rc = asyncio.run(pyrit_scan._handle_results_async(client=client, parsed_args=parsed)) assert rc == 0 assert "extract data" in capsys.readouterr().out @@ -1294,12 +1346,27 @@ def test_handle_results_reports_fetch_error(self, capsys): client = AsyncMock() client.get_scenario_run_results_async.side_effect = RuntimeError("boom") - parsed = pyrit_scan.parse_args(["--scenario-results", "SID"]) + parsed = pyrit_scan.parse_args(["scenario-results", "SID"]) rc = asyncio.run(pyrit_scan._handle_results_async(client=client, parsed_args=parsed)) assert rc == 1 assert "boom" in capsys.readouterr().out +class TestScenarioHistory: + """Tests for the ``scenario-history`` verb and its handler.""" + + def test_handle_scenario_history_lists_runs(self, capsys): + import asyncio + + client = AsyncMock() + client.list_scenario_runs_async.return_value = [] + parsed = pyrit_scan.parse_args(["scenario-history", "5"]) + rc = asyncio.run(pyrit_scan._handle_scenario_history_async(client=client, parsed_args=parsed)) + assert rc == 0 + client.list_scenario_runs_async.assert_awaited_once_with(limit=5) + assert "No scenario runs found" in capsys.readouterr().out + + class TestScenarioParamFlow: """Regression tests for scenario-declared parameters flowing through the CLI.""" @@ -1388,7 +1455,7 @@ def test_scenario_declared_flag_is_forwarded(self, _mock_prog, _mock_print, mock client = self._build_mock_client(supported_params=[{"name": "max_turns", "description": "..."}]) mock_client_class.return_value = client - result = pyrit_scan.main(["foo", "--target", "t", "--max-turns", "7"]) + result = pyrit_scan.main(["run", "foo", "--target", "t", "--max-turns", "7"]) assert result == 0 sent_request = client.start_scenario_run_async.call_args.kwargs["request"] @@ -1418,7 +1485,7 @@ def test_typed_scenario_flags_are_forwarded_as_typed_values( ) mock_client_class.return_value = client - result = pyrit_scan.main(["foo", "--target", "t", "--dry-run", "yes", "--sample-ids", "1", "2"]) + result = pyrit_scan.main(["run", "foo", "--target", "t", "--dry-run", "yes", "--sample-ids", "1", "2"]) assert result == 0 sent_request = client.start_scenario_run_async.call_args.kwargs["request"] @@ -1436,7 +1503,7 @@ def test_unknown_flag_after_valid_scenario_errors(self, _mock_prog, _mock_print, client = self._build_mock_client(supported_params=[{"name": "max_turns", "description": "..."}]) mock_client_class.return_value = client - result = pyrit_scan.main(["foo", "--target", "t", "--max-turns", "7", "--unknown-flag"]) + result = pyrit_scan.main(["run", "foo", "--target", "t", "--max-turns", "7", "--unknown-flag"]) assert result == 1 client.start_scenario_run_async.assert_not_called() @@ -1453,7 +1520,7 @@ def test_no_scenario_params_passes_through_cleanly(self, _mock_prog, _mock_print client = self._build_mock_client(supported_params=[]) mock_client_class.return_value = client - result = pyrit_scan.main(["foo", "--target", "t"]) + result = pyrit_scan.main(["run", "foo", "--target", "t"]) assert result == 0 sent_request = client.start_scenario_run_async.call_args.kwargs["request"] @@ -1461,7 +1528,7 @@ def test_no_scenario_params_passes_through_cleanly(self, _mock_prog, _mock_print def test_parse_args_tolerates_scenario_specific_flags(self): # Pass 1 must not error on scenario-declared flags (they're recognized in pass 2). - parsed = pyrit_scan.parse_args(["foo", "--target", "t", "--max-turns", "7"]) + parsed = pyrit_scan.parse_args(["run", "foo", "--target", "t", "--max-turns", "7"]) assert parsed.scenario_name == "foo" assert parsed.target == "t" assert parsed._unknown_args == ["--max-turns", "7"] diff --git a/tests/unit/cli/test_results.py b/tests/unit/cli/test_results.py index 7adaacc759..9d03162daf 100644 --- a/tests/unit/cli/test_results.py +++ b/tests/unit/cli/test_results.py @@ -198,20 +198,12 @@ def test_shell_parser_rejects_non_positive_limit(): parser.parse_args(["SID", "--limit", "0"]) -def test_add_results_arguments_registers_id_flag_when_requested(): - import argparse - - parser = argparse.ArgumentParser() - add_results_arguments(parser=parser, include_id_flag=True) - parsed = parser.parse_args(["--scenario-results", "SID", "--view", "overview"]) - assert parsed.scenario_results == "SID" - assert parsed.view is ScenarioResultView.OVERVIEW - - -def test_add_results_arguments_omits_id_flag_by_default(): +def test_add_results_arguments_registers_view_flags(): import argparse parser = argparse.ArgumentParser() add_results_arguments(parser=parser) - with pytest.raises(SystemExit): - parser.parse_args(["--scenario-results", "SID"]) + parsed = parser.parse_args(["--view", "attacks", "--attack-result-ids", "a", "b", "--limit", "3"]) + assert parsed.view is ScenarioResultView.ATTACKS + assert parsed.attack_result_ids == ["a", "b"] + assert parsed.limit == 3 From 02c7a61a3040fafcb2145ab40fc508715d254de7 Mon Sep 17 00:00:00 2001 From: jsong468 Date: Fri, 14 Aug 2026 16:09:46 -0700 Subject: [PATCH 2/4] fix test --- tests/unit/cli/test_output.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/cli/test_output.py b/tests/unit/cli/test_output.py index a90e100243..2c0dd38c99 100644 --- a/tests/unit/cli/test_output.py +++ b/tests/unit/cli/test_output.py @@ -278,7 +278,7 @@ def test_print_target_list_empty(capsys): _output.print_target_list(items=[]) captured = capsys.readouterr() assert "No targets found in registry" in captured.out - assert "--initializers target" in captured.out + assert "config file" in captured.out def test_print_target_list_full(capsys): From 3579c0c2f324cff5e963ff733c4e56bbcd857ae1 Mon Sep 17 00:00:00 2001 From: jsong468 Date: Fri, 14 Aug 2026 16:52:13 -0700 Subject: [PATCH 3/4] fix doc --- pyrit/cli/pyrit_scan.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pyrit/cli/pyrit_scan.py b/pyrit/cli/pyrit_scan.py index ca33ca4d0f..20d8c3e6f6 100644 --- a/pyrit/cli/pyrit_scan.py +++ b/pyrit/cli/pyrit_scan.py @@ -94,9 +94,11 @@ def _print_cli_exception(*, exc: BaseException) -> None: # Start the backend server pyrit_scan start-server - # List scenarios, targets, or converters + # List scenarios, initializers, targets, or converters pyrit_scan list-scenarios + pyrit_scan list-initializers pyrit_scan list-targets + pyrit_scan list-converters # Run single-turn cyber attacks against a target pyrit_scan run airt.cyber --target openai_chat --techniques single_turn From 34adcd03b4df672faee247fb852cf579e8f26a41 Mon Sep 17 00:00:00 2001 From: jsong468 Date: Mon, 17 Aug 2026 18:46:07 -0700 Subject: [PATCH 4/4] pr feedback --- pyrit/cli/pyrit_scan.py | 240 +++++++++++++++++++++--------- pyrit/cli/pyrit_shell.py | 1 + tests/unit/cli/test_pyrit_scan.py | 37 +++++ 3 files changed, 204 insertions(+), 74 deletions(-) diff --git a/pyrit/cli/pyrit_scan.py b/pyrit/cli/pyrit_scan.py index 20d8c3e6f6..66915c9ba5 100644 --- a/pyrit/cli/pyrit_scan.py +++ b/pyrit/cli/pyrit_scan.py @@ -86,9 +86,10 @@ def _print_cli_exception(*, exc: BaseException) -> None: Requires a running PyRIT backend server. Use 'start-server' to launch one, or connect to an existing server with --server-url. -Global options (usable with any command, before or after the verb): - --server-url --config-file --log-level --request-timeout --start-server --startup-timeout -Run 'pyrit_scan --help' for full option descriptions and a command's arguments. +Global options (--server-url, --config-file, --log-level) are listed below and +work before or after the verb. Backend commands (run, list-*, add-initializer, +scenario-results, scenario-history) also accept --start-server, --startup-timeout, +and --request-timeout; run 'pyrit_scan --help' to see them. Examples: # Start the backend server @@ -137,61 +138,113 @@ def _positive_finite_float(value: str) -> float: return parsed -def _build_global_parser() -> ArgumentParser: +_SERVER_URL_HELP = "URL of the PyRIT backend server (default: http://localhost:8000)" +_LOG_LEVEL_HELP = "Logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL) (default: WARNING)" +# Scan-specific override of the shared CONFIG_FILE_HELP: for the thin client the file's +# database/initializers/init-scripts/env sections only take effect when a backend is +# launched (start-server or --start-server) and are NOT re-applied to a running server. +_CONFIG_FILE_HELP = ( + "Path to a YAML config file. For commands that talk to a running server this only " + "selects which backend to connect to (server.url). Its database, initializers, " + "initialization-scripts, and env sections apply only when a backend is launched " + "(start-server or --start-server); they are not re-run against an already-running server." +) +_START_SERVER_HELP = "Start a local backend server first if one is not already running" +_STARTUP_TIMEOUT_HELP = "Seconds to wait for a local backend to start (default: server.startup_timeout or 120)" +_REQUEST_TIMEOUT_HELP = ( + "HTTP read timeout in seconds for non-polling server requests " + "(catalog/results/cancel/etc). Defaults to 60. Polling a live " + "scenario run always waits indefinitely regardless of this value." +) + + +def _add_common_options(*, parser: ArgumentParser, suppress_defaults: bool) -> None: """ - Build the parser holding options valid for *every* subcommand. + Add the options that *every* command honors: server URL, config file, log level. + + These live on the root parser (real defaults) *and* on each sub-parser + (``SUPPRESS`` defaults, so a value parsed by the root — e.g. + ``pyrit_scan --server-url X run`` — is not clobbered by the sub-parser's own + default on the second parse pass). + + Args: + parser (ArgumentParser): Parser to extend. + suppress_defaults (bool): Use ``argparse.SUPPRESS`` defaults (sub-parser copies) + instead of real defaults (the root parser, which owns the canonical values). + """ + default = argparse.SUPPRESS if suppress_defaults else None + log_default = argparse.SUPPRESS if suppress_defaults else logging.WARNING + group = parser.add_argument_group("global options") + group.add_argument("--server-url", type=str, default=default, help=_SERVER_URL_HELP) + group.add_argument("--config-file", type=Path, default=default, help=_CONFIG_FILE_HELP) + group.add_argument("--log-level", type=validate_log_level_argparse, default=log_default, help=_LOG_LEVEL_HELP) - This parser is never used on its own. It is passed as ``parents=[...]`` to - each verb's sub-parser so that global options work after any verb, e.g. - ``pyrit_scan run foo --server-url X`` or ``pyrit_scan list-scenarios - --server-url X``. + +def _build_common_parent() -> ArgumentParser: + """ + Parent parser with the common options for a sub-parser (``SUPPRESS`` defaults). Returns: - ArgumentParser: A help-less parent parser with the global options. + ArgumentParser: A help-less parent parser with the common options. """ parser = ArgumentParser(add_help=False) - group = parser.add_argument_group("global options") - group.add_argument( - "--server-url", - type=str, - help="URL of the PyRIT backend server (default: http://localhost:8000)", - ) - group.add_argument( - "--start-server", - action="store_true", - help="Start a local backend server first if one is not already running", - ) - group.add_argument( - "--startup-timeout", - type=_positive_finite_float, - default=None, - metavar="SECONDS", - help="Seconds to wait for a local backend to start (default: server.startup_timeout or 120)", - ) - group.add_argument( - "--config-file", - type=Path, - help=ARG_HELP["config_file"], - ) + _add_common_options(parser=parser, suppress_defaults=True) + return parser + + +def _build_client_parent() -> ArgumentParser: + """ + Parent parser for commands that reach the backend through the API client. + + Adds ``--request-timeout`` plus the auto-start options (``--start-server`` / + ``--startup-timeout``). Attached to ``run``, the ``list-*`` verbs, + ``add-initializer``, ``scenario-results``, and ``scenario-history``. Verbs that do + not open a client (``start-server``, ``stop-server``) deliberately omit it so + unsupported combinations like ``start-server --request-timeout`` are rejected. + + Returns: + ArgumentParser: A help-less parent parser with the client/auto-start options. + """ + parser = ArgumentParser(add_help=False) + group = parser.add_argument_group("server options") + group.add_argument("--request-timeout", type=float, default=None, help=_REQUEST_TIMEOUT_HELP) + group.add_argument("--start-server", action="store_true", help=_START_SERVER_HELP) group.add_argument( - "--log-level", - type=validate_log_level_argparse, - default=logging.WARNING, - help="Logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL) (default: WARNING)", + "--startup-timeout", type=_positive_finite_float, default=None, metavar="SECONDS", help=_STARTUP_TIMEOUT_HELP ) + return parser + + +def _build_start_server_parent() -> ArgumentParser: + """ + Parent parser for the ``start-server`` verb: only ``--startup-timeout`` applies. + + Returns: + ArgumentParser: A help-less parent parser with the startup timeout option. + """ + parser = ArgumentParser(add_help=False) + group = parser.add_argument_group("server startup options") group.add_argument( - "--request-timeout", - type=float, - default=None, - help=( - "HTTP read timeout in seconds for non-polling server requests " - "(catalog/results/cancel/etc). Defaults to 60. Polling a live " - "scenario run always waits indefinitely regardless of this value." - ), + "--startup-timeout", type=_positive_finite_float, default=None, metavar="SECONDS", help=_STARTUP_TIMEOUT_HELP ) return parser +def _build_global_parser() -> ArgumentParser: + """ + Union of every option group. + + Used *only* by the legacy shim to strip options and locate a verb; it is never + attached to a command. Keeping it a union (rather than the per-command groups) + lets the shim reorder any option placed before a verb, regardless of which + command ultimately owns it. + + Returns: + ArgumentParser: A help-less parser recognizing all common and server options. + """ + return ArgumentParser(add_help=False, parents=[_build_common_parent(), _build_client_parent()]) + + def _add_run_arguments(*, parser: ArgumentParser, scenario_params: list[Parameter] | None = None) -> None: """ Add the ``run`` verb's arguments (scenario positional + run flags) to *parser*. @@ -260,29 +313,36 @@ def _build_parser(*, scenario_params: list[Parameter] | None = None, add_help: b Returns: ArgumentParser: The configured parser. """ - global_parser = _build_global_parser() + common_parent = _build_common_parent() + client_parent = _build_client_parent() + start_server_parent = _build_start_server_parent() + client_parents = [common_parent, client_parent] + parser = ArgumentParser( prog="pyrit_scan", description=_DESCRIPTION, formatter_class=RawDescriptionHelpFormatter, add_help=add_help, ) + # The root parser owns the real global options so top-level help and pre-verb + # handling (e.g. ``pyrit_scan --server-url X --help``) work normally. + _add_common_options(parser=parser, suppress_defaults=False) subparsers = parser.add_subparsers(dest="command", metavar="", title="commands") run_parser = subparsers.add_parser( "run", - parents=[global_parser], + parents=client_parents, help="Run a scenario against a target", formatter_class=RawDescriptionHelpFormatter, ) _add_run_arguments(parser=run_parser, scenario_params=scenario_params) for verb, help_text in _LIST_VERBS.items(): - subparsers.add_parser(verb, parents=[global_parser], help=help_text) + subparsers.add_parser(verb, parents=client_parents, help=help_text) add_init_parser = subparsers.add_parser( "add-initializer", - parents=[global_parser], + parents=client_parents, help="Register initializer(s) from Python script file(s)", ) add_init_parser.add_argument( @@ -295,7 +355,7 @@ def _build_parser(*, scenario_params: list[Parameter] | None = None, add_help: b results_parser = subparsers.add_parser( "scenario-results", - parents=[global_parser], + parents=client_parents, help="Inspect the results of a completed scenario run", ) results_parser.add_argument("scenario_result_id", type=str, help="Scenario result id to inspect") @@ -303,7 +363,7 @@ def _build_parser(*, scenario_params: list[Parameter] | None = None, add_help: b history_parser = subparsers.add_parser( "scenario-history", - parents=[global_parser], + parents=client_parents, help="List recent scenario runs", ) history_parser.add_argument( @@ -315,16 +375,31 @@ def _build_parser(*, scenario_params: list[Parameter] | None = None, add_help: b help="Number of recent runs to show (default: 10)", ) - subparsers.add_parser("start-server", parents=[global_parser], help="Start a local backend server") - subparsers.add_parser("stop-server", parents=[global_parser], help="Stop the backend server") + subparsers.add_parser( + "start-server", parents=[common_parent, start_server_parent], help="Start a local backend server" + ) + subparsers.add_parser("stop-server", parents=[common_parent], help="Stop the backend server") return parser +def _discover_verbs() -> frozenset[str]: + """ + Read the registered subcommand verbs straight off the built parser's subparsers. + + Returns: + frozenset[str]: Every registered subcommand verb. + """ + parser = _build_parser(add_help=False) + for action in parser._actions: + if isinstance(action, argparse._SubParsersAction): + return frozenset(action.choices) + return frozenset() + + #: Every valid subcommand verb (used by the legacy-argv shim to detect new-style calls). -_KNOWN_VERBS: frozenset[str] = frozenset( - {"run", "add-initializer", "scenario-results", "scenario-history", "start-server", "stop-server", *_LIST_VERBS} -) +#: Derived from _build_parser so adding/renaming a subcommand can't leave this stale. +_KNOWN_VERBS: frozenset[str] = _discover_verbs() # Namespacing prefix for scenario-declared params on the parsed Namespace. @@ -450,6 +525,7 @@ def _extract_scenario_args(*, parsed: Namespace) -> dict[str, Any]: "--list-datasets": "list-datasets", "--add-initializer": "add-initializer", "--stop-server": "stop-server", + "--scenario-results": "scenario-results", } @@ -503,6 +579,11 @@ def _translate_legacy_argv(argv: list[str]) -> list[str]: verb = leftover[0] index = argv.index(verb) return [verb, *argv[:index], *argv[index + 1 :]] + if leftover[0] in ("-h", "--help"): + # Global options followed by top-level help (e.g. ``--server-url X --help``). + # Leave argv untouched so the root parser prints its own help instead of + # misreading ``--help`` as an implicit scenario name. + return argv # Otherwise it is a bare scenario name (+ run flags) → implicit run. _warn_legacy(old=" (implicit run)", new="run ") return ["run", *argv] @@ -607,7 +688,7 @@ def _resolve_configured_server_url(*, parsed_args: Namespace) -> str: async def _handle_stop_server_async(*, parsed_args: Namespace) -> int: """ - Handle ``--stop-server``: probe, then terminate the listening process. + Handle ``stop-server``: probe, then terminate the listening process. Returns: int: Zero when no server is running or shutdown succeeds; one otherwise. @@ -662,7 +743,7 @@ async def _handle_list_commands_async(*, client: Any, parsed_args: Namespace) -> async def _handle_add_initializer_async(*, client: Any, parsed_args: Namespace) -> int: """ - Handle ``--add-initializer``: upload one or more scripts to the server. + Handle ``add-initializer``: upload one or more scripts to the server. Returns: int: Exit code (``0`` on success, ``1`` on failure). @@ -909,24 +990,13 @@ async def _run_scenario_async( return 1 -async def _dispatch_with_client_async(*, client: Any, parsed_args: Namespace) -> int: +async def _handle_run_async(*, client: Any, parsed_args: Namespace) -> int: """ - Dispatch a verb that needs an open API client. + Handle the ``run`` verb: resolve the scenario, reparse its declared flags, then run it. Returns: - int: Exit code from the dispatched command. + int: Exit code (``0`` if the run completed successfully, ``1`` otherwise). """ - command = parsed_args.command - if command in _LIST_VERBS: - return await _handle_list_commands_async(client=client, parsed_args=parsed_args) - if command == "add-initializer": - return await _handle_add_initializer_async(client=client, parsed_args=parsed_args) - if command == "scenario-results": - return await _handle_results_async(client=client, parsed_args=parsed_args) - if command == "scenario-history": - return await _handle_scenario_history_async(client=client, parsed_args=parsed_args) - - # command == "run": the scenario positional is required by the run sub-parser. scenario_name = parsed_args.scenario_name scenario_meta = await client.get_scenario_async(scenario_name=scenario_name) if scenario_meta is None: @@ -943,9 +1013,31 @@ async def _dispatch_with_client_async(*, client: Any, parsed_args: Namespace) -> ) if reparsed is None: return 1 - parsed_args = reparsed - return await _run_scenario_async(client=client, parsed_args=parsed_args, scenario_meta=scenario_meta) + return await _run_scenario_async(client=client, parsed_args=reparsed, scenario_meta=scenario_meta) + + +#: Post-client verbs, each a uniform ``(*, client, parsed_args) -> int`` handler. Reached +#: only after the API client is open (start-server/stop-server are handled earlier, before +#: any client exists), so dispatch here is a pure table lookup with no branching. +_CLIENT_HANDLERS: dict[str, Callable[..., Any]] = { + "run": _handle_run_async, + "add-initializer": _handle_add_initializer_async, + "scenario-results": _handle_results_async, + "scenario-history": _handle_scenario_history_async, + **dict.fromkeys(_LIST_VERBS, _handle_list_commands_async), +} + + +async def _dispatch_with_client_async(*, client: Any, parsed_args: Namespace) -> int: + """ + Dispatch a verb that needs an open API client. + + Returns: + int: Exit code from the dispatched command. + """ + handler = _CLIENT_HANDLERS[parsed_args.command] + return await handler(client=client, parsed_args=parsed_args) async def _run_async(*, parsed_args: Namespace) -> int: diff --git a/pyrit/cli/pyrit_shell.py b/pyrit/cli/pyrit_shell.py index 020e606b92..1b15755cca 100644 --- a/pyrit/cli/pyrit_shell.py +++ b/pyrit/cli/pyrit_shell.py @@ -71,6 +71,7 @@ class PyRITShell(cmd.Cmd): list-initializers - List all available initializers list-targets - List all available targets list-converters - List all registered converter instances + add-initializer ... - Register initializer(s) from Python script file(s) run [opts] - Run a scenario with optional parameters scenario-history [N] - List the last N (default 10) scenario runs scenario-results [id] - Inspect a run: --view overview|attacks diff --git a/tests/unit/cli/test_pyrit_scan.py b/tests/unit/cli/test_pyrit_scan.py index d5f901065e..0d52bc9ef4 100644 --- a/tests/unit/cli/test_pyrit_scan.py +++ b/tests/unit/cli/test_pyrit_scan.py @@ -196,11 +196,48 @@ def test_list_with_start_server(self): args = pyrit_scan.parse_args(["list-scenarios", "--start-server"]) assert args.start_server is True + def test_list_with_request_timeout(self): + args = pyrit_scan.parse_args(["list-scenarios", "--request-timeout", "30"]) + assert args.request_timeout == 30 + def test_start_server_with_startup_timeout(self): args = pyrit_scan.parse_args(["start-server", "--startup-timeout", "45.5"]) assert args.startup_timeout == 45.5 +class TestGlobalOptionScoping: + """Options are scoped to the commands that use them; unsupported combos are rejected.""" + + def test_top_level_help_with_global_option_shows_root_help(self, capsys, recwarn): + with pytest.raises(SystemExit) as exc_info: + pyrit_scan.parse_args(["--server-url", "http://x", "--help"]) + assert exc_info.value.code == 0 + out = capsys.readouterr().out + assert "" in out + assert not [w for w in recwarn if issubclass(w.category, DeprecationWarning)] + + def test_stop_server_rejects_start_server(self): + assert pyrit_scan.main(["stop-server", "--start-server"]) == 2 + + def test_stop_server_rejects_request_timeout(self): + assert pyrit_scan.main(["stop-server", "--request-timeout", "7"]) == 2 + + def test_start_server_rejects_request_timeout(self): + assert pyrit_scan.main(["start-server", "--request-timeout", "7"]) == 2 + + +class TestClientHandlerTable: + """The client-dispatch table must stay in sync with the registered verbs.""" + + def test_client_handlers_cover_every_post_client_verb(self): + # _dispatch_with_client_async indexes _CLIENT_HANDLERS directly, so a client verb + # without a handler would KeyError at runtime. start-server/stop-server run before a + # client is opened and are handled in _run_async, so they are the only exclusions. + pre_client_verbs = {"start-server", "stop-server"} + expected = set(pyrit_scan._KNOWN_VERBS) - pre_client_verbs + assert set(pyrit_scan._CLIENT_HANDLERS) == expected + + class TestLegacyArgvShim: """The back-compat shim maps old flag forms to verbs and warns."""