From 5dbf90310acc0c6562a6ce346be22bae2abe8145 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 09:04:49 +0900 Subject: [PATCH 01/15] fix: separate Strix baseline provider exhaustion --- .github/workflows/strix.yml | 9 +++ CHANGELOG.md | 5 ++ .../strix-pr-baseline-provider-exhaustion.md | 29 +++++++++ scripts/ci/strix_quick_gate.sh | 12 ++++ scripts/ci/test_strix_quick_gate.sh | 65 +++++++++++++++++++ ...est_strix_nvidia_nim_not_found_fallback.py | 10 +++ 6 files changed, 130 insertions(+) create mode 100644 docs/doctoring/strix-pr-baseline-provider-exhaustion.md diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index 4155c7346..8ec9c51c2 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -858,6 +858,15 @@ jobs: exit 0 fi + # Exit 3 is emitted only by the trusted gate after its exact PR-scope + # mapper excludes every reported finding as unchanged and all model + # providers are unavailable. Vulnerability text remains in the audit + # log, so this cannot use the generic no-finding regex path below. + if [ "$strix_rc" -eq 3 ]; then + echo "::warning title=Strix backend unavailable after baseline scan::Strix excluded all reported findings from the pull-request diff, then exhausted its model providers. Treating this exact-head run as a neutral infrastructure skip; changed or unmapped findings still fail closed." + exit 0 + fi + # Preserve configuration failures (exit 2) and any unexpected exit # code as hard failures — only the scan-failure code (1) can be an # infrastructure/backend-unavailability outcome. diff --git a/CHANGELOG.md b/CHANGELOG.md index 89199da27..bba2745b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,11 @@ Semantic Versioning where the repository publishes a release. ### Changed +- Preserve the trusted PR-scope decision when Strix finds only unchanged-file + baseline issues and then exhausts every LLM provider, so provider retirement + or outage is a neutral infrastructure result while changed and unmapped + findings still fail closed; the reproduced incident and gate boundary are + recorded in `docs/doctoring/strix-pr-baseline-provider-exhaustion.md`. - Emit completed repository pull-list requests as they finish in the five-minute agent-mention sweep, while retaining the four-worker ceiling, rotation, and exact-name dispatch ledger, so one slow repository cannot hide ready sibling diff --git a/docs/doctoring/strix-pr-baseline-provider-exhaustion.md b/docs/doctoring/strix-pr-baseline-provider-exhaustion.md new file mode 100644 index 000000000..1b08cf00e --- /dev/null +++ b/docs/doctoring/strix-pr-baseline-provider-exhaustion.md @@ -0,0 +1,29 @@ +# Strix PR baseline/provider-exhaustion incident + +## Observed failure + +LineageWeave PR 392 run `32530198775` reported a critical secret in the +nonexistent `frontend/src/config.ts`. The trusted changed-file mapper correctly +classified that report as unchanged, but later fallback attempts ended in +provider HTTP 410 retirement brownouts. The gate retained the earlier severity +rank and returned the same exit code used for changed-file findings, so the +outer workflow could not distinguish the cleared baseline report from a real +pull-request vulnerability. + +## Root cause and repair + +The quick gate already owns the exact PR-head changed-file mapping decision. +After it has classified every report as `allow_baseline`, a later provider +exhaustion now returns the dedicated status 3. The trusted reusable workflow +maps only that status to a neutral infrastructure warning. Changed, unmapped, +or manifest findings still return the blocking status, and configuration or +unexpected statuses still fail closed. + +## Verification + +- A three-attempt regression reproduces an unchanged critical report followed + by two provider failures and requires status 3. +- Existing source tests require changed findings to remain blocking and clean + unchanged findings to remain admissible. +- The central Python suite, native workflow validation, Bash syntax checks, and + the complete Strix shell regression suite run on the final tree. diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index 0f37f3460..208f6756e 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -4040,6 +4040,18 @@ run_current_target_scan() { return 1 fi + # The trusted PR-scope mapper has already proved these reports belong only + # to unchanged files. Keep that decision distinct from a real changed-file + # finding when every remaining provider is unavailable. + if [ "$PR_FINDINGS_DECISION" = "allow_baseline" ] && [ "$INFRA_ERROR_DETECTED" -eq 1 ]; then + if is_vertex_model "$PRIMARY_MODEL"; then + echo "Configured Vertex model and fallback models were unavailable after unchanged-file findings were excluded." >&2 + else + echo "Configured model and fallback models were unavailable after unchanged-file findings were excluded." >&2 + fi + return 3 + fi + local threshold_rank threshold_rank="$(severity_rank "$STRIX_FAIL_ON_MIN_SEVERITY")" if [ "${STRIX_MAX_SEVERITY_RANK:--1}" -ge "$threshold_rank" ]; then diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index 1c05feb6f..831cb6094 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -4650,6 +4650,28 @@ EOS echo "Penetration test failed: baseline critical finding" exit 1 ;; + pr-baseline-provider-exhausted) + case "${STRIX_LLM:-}" in + vertex_ai/baseline-primary) + mkdir -p "$STRIX_REPORTS_DIR/fake-pr-baseline-provider/vulnerabilities" + cat >"$STRIX_REPORTS_DIR/fake-pr-baseline-provider/vulnerabilities/vuln-0001.md" <<'EOS' +Severity: CRITICAL +Location 1: +sync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/service/impl/SysUserServiceImpl.java:5 +EOS + echo "litellm.APIConnectionError: GeminiException LLM CONNECTION FAILED after baseline report" + exit 1 + ;; + vertex_ai/fallback-one | vertex_ai/fallback-two) + echo "litellm.APIConnectionError: GeminiException LLM CONNECTION FAILED: provider unavailable" + exit 1 + ;; + *) + echo "Error: pr-baseline-provider-exhausted unexpected model (${STRIX_LLM:-})" >&2 + exit 35 + ;; + esac + ;; pr-critical-changed) mkdir -p "$STRIX_REPORTS_DIR/fake-pr-changed/vulnerabilities" cat >"$STRIX_REPORTS_DIR/fake-pr-changed/vulnerabilities/vuln-0001.md" <<'EOS' @@ -6276,6 +6298,28 @@ run_filtered_gate_case_if_requested() { "Materialized PR-head changed-file scope" \ "repository_dispatch" ;; + pr-baseline-provider-exhausted) + run_gate_case "pr-baseline-provider-exhausted" \ + "vertex_ai/baseline-primary" \ + "vertex_ai/fallback-one vertex_ai/fallback-two" \ + "3" \ + "Configured Vertex model and fallback models were unavailable after unchanged-file findings were excluded." \ + "3" \ + "vertex_ai/baseline-primary|vertex_ai/fallback-one|vertex_ai/fallback-two" \ + "||" \ + "vertex_ai" \ + "__DEFAULT__" \ + "" \ + "0" \ + "CRITICAL" \ + "0" \ + "" \ + "" \ + "1200" \ + "0" \ + "pull_request" \ + "sync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/controller/SysPositionController.java" + ;; *) record_failure "unknown STRIX_TEST_CASE_FILTER '${STRIX_TEST_CASE_FILTER:-}'" ;; @@ -10909,6 +10953,27 @@ run_gate_case "pr-baseline-critical-unchanged" \ "pull_request" \ "sync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/controller/SysPositionController.java" +run_gate_case "pr-baseline-provider-exhausted" \ + "vertex_ai/baseline-primary" \ + "vertex_ai/fallback-one vertex_ai/fallback-two" \ + "3" \ + "Configured Vertex model and fallback models were unavailable after unchanged-file findings were excluded." \ + "3" \ + "vertex_ai/baseline-primary|vertex_ai/fallback-one|vertex_ai/fallback-two" \ + "||" \ + "vertex_ai" \ + "__DEFAULT__" \ + "" \ + "0" \ + "CRITICAL" \ + "0" \ + "" \ + "" \ + "1200" \ + "0" \ + "pull_request" \ + "sync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/controller/SysPositionController.java" + run_gate_case "pr-baseline-critical-absolute-target" \ "openai/gpt-4o-mini" \ "" \ diff --git a/tests/test_strix_nvidia_nim_not_found_fallback.py b/tests/test_strix_nvidia_nim_not_found_fallback.py index a48f3092d..4c067c822 100644 --- a/tests/test_strix_nvidia_nim_not_found_fallback.py +++ b/tests/test_strix_nvidia_nim_not_found_fallback.py @@ -256,6 +256,16 @@ def test_workflow_neutralizes_only_nvidia_404_without_findings(self) -> None: workflow, ) + def test_workflow_accepts_trusted_unchanged_finding_provider_exhaustion(self) -> None: + """Keep gate-mapped baseline findings separate from provider exhaustion.""" + + workflow = STRIX_WORKFLOW.read_text(encoding="utf-8") + self.assertIn('if [ "$strix_rc" -eq 3 ]; then', workflow) + self.assertIn( + "Strix excluded all reported findings from the pull-request diff", + workflow, + ) + if __name__ == "__main__": unittest.main() From d065e69fa48fbf96251433f8be77934ccf03b159 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 09:37:59 +0900 Subject: [PATCH 02/15] fix(strix): neutralize empty Caido bootstrap outages --- .github/workflows/strix.yml | 6 +++--- CHANGELOG.md | 8 +++++--- .../strix-pr-baseline-provider-exhaustion.md | 12 ++++++++++++ tests/test_strix_nvidia_nim_not_found_fallback.py | 10 ++++++++++ 4 files changed, 30 insertions(+), 6 deletions(-) diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index 8ec9c51c2..be9ced67c 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -874,8 +874,8 @@ jobs: exit "$strix_rc" fi - # Recognized signals that the LLM backend was unavailable / starved. - backend_unavailable_signal='RateLimitError|Too many requests\. For more on scraping GitHub|exceeded your current quota|insufficient_quota|billing details|"status"[[:space:]]*:[[:space:]]*"RESOURCE_EXHAUSTED"|tokens_limit_reached|Request body too large|Max size:[[:space:]]*[0-9]+[[:space:]]+tokens|Error code:[[:space:]]*413|LLM CONNECTION FAILED|Could not establish connection to the language model|LLM warm-up failed|Configured model and fallback models were unavailable|Configured Vertex model and fallback models were unavailable|emitted provider infrastructure or failure-signal output|before provider infrastructure failure|litellm(\.exceptions)?\.NotFoundError[^[:cntrl:]]*Nvidia_nimException[^[:cntrl:]]*Error code:[[:space:]]*404' + # Recognized signals that the scanner or LLM backend was unavailable. + backend_unavailable_signal='RateLimitError|Too many requests\. For more on scraping GitHub|exceeded your current quota|insufficient_quota|billing details|"status"[[:space:]]*:[[:space:]]*"RESOURCE_EXHAUSTED"|tokens_limit_reached|Request body too large|Max size:[[:space:]]*[0-9]+[[:space:]]+tokens|Error code:[[:space:]]*413|LLM CONNECTION FAILED|Could not establish connection to the language model|LLM warm-up failed|Configured model and fallback models were unavailable|Configured Vertex model and fallback models were unavailable|emitted provider infrastructure or failure-signal output|before provider infrastructure failure|litellm(\.exceptions)?\.NotFoundError[^[:cntrl:]]*Nvidia_nimException[^[:cntrl:]]*Error code:[[:space:]]*404|RuntimeError:[[:space:]]*loginAsGuest failed after [0-9]+ attempts:[^[:cntrl:]]*curl exit 7:[^[:cntrl:]]*Failed to connect to 127\.0\.0\.1 port 48080' # Any evidence that a vulnerability was actually reported. Its presence # forces a hard failure so real findings are NEVER downgraded. Keep the # severity branch anchored away from identifiers so environment lines @@ -888,7 +888,7 @@ jobs: # from blocking current-head merge progress. if grep -Eiq "$backend_unavailable_signal" "$strix_run_log" \ && ! grep -Eiq "$reported_vulnerability_signal" "$strix_run_log"; then - echo "::warning title=Strix backend unavailable::Strix could not complete because its LLM backend was unavailable (rate limit / token cap / connection or warm-up failure) before producing a vulnerability report. Treating as a neutral skip so an infrastructure outage does not block merges; genuine findings still fail the check. See the strix-reports artifact and the run log." + echo "::warning title=Strix infrastructure unavailable::Strix could not complete because its scanner bootstrap or LLM backend was unavailable before producing a vulnerability report. Treating as a neutral skip so an infrastructure outage does not block merges; genuine findings still fail the check. See the strix-reports artifact and the run log." exit 0 fi diff --git a/CHANGELOG.md b/CHANGELOG.md index bba2745b0..b291cb988 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,9 +24,11 @@ Semantic Versioning where the repository publishes a release. - Preserve the trusted PR-scope decision when Strix finds only unchanged-file baseline issues and then exhausts every LLM provider, so provider retirement - or outage is a neutral infrastructure result while changed and unmapped - findings still fail closed; the reproduced incident and gate boundary are - recorded in `docs/doctoring/strix-pr-baseline-provider-exhaustion.md`. + or outage is a neutral infrastructure result; recognize the exact local Caido + bootstrap connection failure the same way only when no vulnerability exists, + while changed and unmapped findings and all other runtime failures still fail + closed. The reproduced incidents and gate boundary are recorded in + `docs/doctoring/strix-pr-baseline-provider-exhaustion.md`. - Emit completed repository pull-list requests as they finish in the five-minute agent-mention sweep, while retaining the four-worker ceiling, rotation, and exact-name dispatch ledger, so one slow repository cannot hide ready sibling diff --git a/docs/doctoring/strix-pr-baseline-provider-exhaustion.md b/docs/doctoring/strix-pr-baseline-provider-exhaustion.md index 1b08cf00e..3add57692 100644 --- a/docs/doctoring/strix-pr-baseline-provider-exhaustion.md +++ b/docs/doctoring/strix-pr-baseline-provider-exhaustion.md @@ -10,6 +10,11 @@ rank and returned the same exit code used for changed-file findings, so the outer workflow could not distinguish the cleared baseline report from a real pull-request vulnerability. +A later central PR run reported zero vulnerabilities and then failed before +scanning when Strix's local Caido process did not accept connections on +`127.0.0.1:48080`; the outer workflow did not yet recognize that exact scanner +bootstrap outage as infrastructure. + ## Root cause and repair The quick gate already owns the exact PR-head changed-file mapping decision. @@ -19,11 +24,18 @@ maps only that status to a neutral infrastructure warning. Changed, unmapped, or manifest findings still return the blocking status, and configuration or unexpected statuses still fail closed. +The outer workflow also recognizes only the observed `loginAsGuest` retry +exhaustion with curl exit 7 against Strix's fixed local Caido port. It is neutral +only when no positive vulnerability or severity signal exists; every other +runtime failure remains blocking. + ## Verification - A three-attempt regression reproduces an unchanged critical report followed by two provider failures and requires status 3. - Existing source tests require changed findings to remain blocking and clean unchanged findings to remain admissible. +- A workflow regression requires the exact Caido bootstrap outage to be neutral + with zero findings and blocking when any vulnerability is reported. - The central Python suite, native workflow validation, Bash syntax checks, and the complete Strix shell regression suite run on the final tree. diff --git a/tests/test_strix_nvidia_nim_not_found_fallback.py b/tests/test_strix_nvidia_nim_not_found_fallback.py index 4c067c822..b1c8275f9 100644 --- a/tests/test_strix_nvidia_nim_not_found_fallback.py +++ b/tests/test_strix_nvidia_nim_not_found_fallback.py @@ -243,6 +243,16 @@ def test_outer_workflow_never_neutralizes_reported_vulnerabilities(self) -> None ) ) + def test_outer_workflow_neutralizes_only_empty_caido_bootstrap_failure(self) -> None: + """Treat the exact scanner-local Caido outage as infrastructure only.""" + + caido_error = ( + "RuntimeError: loginAsGuest failed after 10 attempts: curl exit 7: " + "curl: (7) Failed to connect to 127.0.0.1 port 48080\n" + ) + self.assertTrue(_workflow_neutralizes(caido_error + "Vulnerabilities 0\n")) + self.assertFalse(_workflow_neutralizes(caido_error + "Vulnerabilities 1\n")) + def test_workflow_neutralizes_only_nvidia_404_without_findings(self) -> None: """Retain the static fail-closed vulnerability evidence contract.""" From 7179abb91f0069c49e7efbf5f158f1e0cc4231af Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 15:55:00 +0900 Subject: [PATCH 03/15] fix(strix): preserve primary-only baseline result --- CHANGELOG.md | 9 ++-- .../strix-pr-baseline-provider-exhaustion.md | 13 +++--- scripts/ci/strix_quick_gate.sh | 32 +++++++------ scripts/ci/test_strix_quick_gate.sh | 45 ++++++++++++++++++- 4 files changed, 76 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f89ef8cb..b9219dc4b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,10 +24,11 @@ Semantic Versioning where the repository publishes a release. - Preserve the trusted PR-scope decision when Strix finds only unchanged-file baseline issues and then exhausts every LLM provider, so provider retirement - or outage is a neutral infrastructure result; recognize the exact local Caido - bootstrap connection failure the same way only when no vulnerability exists, - while changed and unmapped findings and all other runtime failures still fail - closed. The reproduced incidents and gate boundary are recorded in + or outage is a neutral infrastructure result even when no distinct fallback + is configured; recognize the exact local Caido bootstrap connection failure + the same way only when no vulnerability exists, while changed and unmapped + findings and all other runtime failures still fail closed. The reproduced + incidents and gate boundary are recorded in `docs/doctoring/strix-pr-baseline-provider-exhaustion.md`. - Emit completed repository pull-list requests as they finish in the five-minute agent-mention sweep, while retaining the four-worker ceiling, rotation, and diff --git a/docs/doctoring/strix-pr-baseline-provider-exhaustion.md b/docs/doctoring/strix-pr-baseline-provider-exhaustion.md index 3add57692..010af2cad 100644 --- a/docs/doctoring/strix-pr-baseline-provider-exhaustion.md +++ b/docs/doctoring/strix-pr-baseline-provider-exhaustion.md @@ -18,11 +18,12 @@ bootstrap outage as infrastructure. ## Root cause and repair The quick gate already owns the exact PR-head changed-file mapping decision. -After it has classified every report as `allow_baseline`, a later provider -exhaustion now returns the dedicated status 3. The trusted reusable workflow -maps only that status to a neutral infrastructure warning. Changed, unmapped, -or manifest findings still return the blocking status, and configuration or -unexpected statuses still fail closed. +After it has classified every report as `allow_baseline`, provider exhaustion +now returns the dedicated status 3, including a deployment with no distinct +fallback configured. The trusted reusable workflow maps only that status to a +neutral infrastructure warning. Changed, unmapped, or manifest findings still +return the blocking status, and configuration or unexpected statuses still +fail closed. The outer workflow also recognizes only the observed `loginAsGuest` retry exhaustion with curl exit 7 against Strix's fixed local Caido port. It is neutral @@ -33,6 +34,8 @@ runtime failure remains blocking. - A three-attempt regression reproduces an unchanged critical report followed by two provider failures and requires status 3. +- A primary-only regression requires the same trusted baseline outcome without + treating the absent fallback as a pull-request finding. - Existing source tests require changed findings to remain blocking and clean unchanged findings to remain admissible. - A workflow regression requires the exact Caido bootstrap outage to be neutral diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index 208f6756e..38f04de07 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -3894,7 +3894,7 @@ run_current_target_scan() { local strict_primary_provider_fallback=0 if [ "$INFRA_ERROR_DETECTED" -eq 1 ] && provider_signal_fail_closed_enabled; then - if is_model_retryable_error "$PRIMARY_MODEL" && has_distinct_fallback_model_for_model "$PRIMARY_MODEL"; then + if is_model_retryable_error "$PRIMARY_MODEL"; then strict_primary_provider_fallback=1 else echo "Strix scan failed after provider infrastructure or failure-signal output; failing closed." >&2 @@ -4024,6 +4024,24 @@ run_current_target_scan() { return 1 fi + # The trusted PR-scope mapper has already proved these reports belong only + # to unchanged files. Keep that decision distinct from a real changed-file + # finding when every configured model path is unavailable. + if [ "$PR_FINDINGS_DECISION" = "allow_baseline" ] && [ "$INFRA_ERROR_DETECTED" -eq 1 ]; then + if [ "$fallback_tried" -eq 0 ]; then + if is_vertex_model "$PRIMARY_MODEL"; then + echo "Configured Vertex model was unavailable after unchanged-file findings were excluded." >&2 + else + echo "Configured model was unavailable after unchanged-file findings were excluded." >&2 + fi + elif is_vertex_model "$PRIMARY_MODEL"; then + echo "Configured Vertex model and fallback models were unavailable after unchanged-file findings were excluded." >&2 + else + echo "Configured model and fallback models were unavailable after unchanged-file findings were excluded." >&2 + fi + return 3 + fi + if [ "$fallback_tried" -eq 0 ]; then local fallback_config_name fallback_config_name="$(fallback_models_config_name_for_model "$PRIMARY_MODEL")" @@ -4040,18 +4058,6 @@ run_current_target_scan() { return 1 fi - # The trusted PR-scope mapper has already proved these reports belong only - # to unchanged files. Keep that decision distinct from a real changed-file - # finding when every remaining provider is unavailable. - if [ "$PR_FINDINGS_DECISION" = "allow_baseline" ] && [ "$INFRA_ERROR_DETECTED" -eq 1 ]; then - if is_vertex_model "$PRIMARY_MODEL"; then - echo "Configured Vertex model and fallback models were unavailable after unchanged-file findings were excluded." >&2 - else - echo "Configured model and fallback models were unavailable after unchanged-file findings were excluded." >&2 - fi - return 3 - fi - local threshold_rank threshold_rank="$(severity_rank "$STRIX_FAIL_ON_MIN_SEVERITY")" if [ "${STRIX_MAX_SEVERITY_RANK:--1}" -ge "$threshold_rank" ]; then diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index 831cb6094..432dcee71 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -4650,7 +4650,7 @@ EOS echo "Penetration test failed: baseline critical finding" exit 1 ;; - pr-baseline-provider-exhausted) + pr-baseline-provider-exhausted | pr-baseline-provider-exhausted-no-fallback) case "${STRIX_LLM:-}" in vertex_ai/baseline-primary) mkdir -p "$STRIX_REPORTS_DIR/fake-pr-baseline-provider/vulnerabilities" @@ -6320,6 +6320,28 @@ run_filtered_gate_case_if_requested() { "pull_request" \ "sync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/controller/SysPositionController.java" ;; + pr-baseline-provider-exhausted-no-fallback) + run_gate_case "pr-baseline-provider-exhausted-no-fallback" \ + "vertex_ai/baseline-primary" \ + " " \ + "3" \ + "Configured Vertex model was unavailable after unchanged-file findings were excluded." \ + "1" \ + "vertex_ai/baseline-primary" \ + "" \ + "vertex_ai" \ + "__DEFAULT__" \ + "" \ + "0" \ + "CRITICAL" \ + "0" \ + "" \ + "" \ + "1200" \ + "0" \ + "pull_request" \ + "sync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/controller/SysPositionController.java" + ;; *) record_failure "unknown STRIX_TEST_CASE_FILTER '${STRIX_TEST_CASE_FILTER:-}'" ;; @@ -10974,6 +10996,27 @@ run_gate_case "pr-baseline-provider-exhausted" \ "pull_request" \ "sync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/controller/SysPositionController.java" +run_gate_case "pr-baseline-provider-exhausted-no-fallback" \ + "vertex_ai/baseline-primary" \ + " " \ + "3" \ + "Configured Vertex model was unavailable after unchanged-file findings were excluded." \ + "1" \ + "vertex_ai/baseline-primary" \ + "" \ + "vertex_ai" \ + "__DEFAULT__" \ + "" \ + "0" \ + "CRITICAL" \ + "0" \ + "" \ + "" \ + "1200" \ + "0" \ + "pull_request" \ + "sync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/controller/SysPositionController.java" + run_gate_case "pr-baseline-critical-absolute-target" \ "openai/gpt-4o-mini" \ "" \ From 94eafa4ca107f7e6b57e013e312223d4944faefc Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 16:03:48 +0900 Subject: [PATCH 04/15] refactor(strix): remove obsolete fallback probe --- scripts/ci/strix_quick_gate.sh | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index 38f04de07..996328b9f 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -2212,28 +2212,6 @@ fallback_models_config_name_for_model() { printf '%s\n' "STRIX_FALLBACK_MODELS" } -has_distinct_fallback_model_for_model() { - local model="$1" - local fallback_models_raw - fallback_models_raw="$(fallback_models_raw_for_model "$model")" - fallback_models_raw="${fallback_models_raw//$'\r'/ }" - fallback_models_raw="${fallback_models_raw//$'\n'/ }" - - local fallback_models=() - read -r -a fallback_models <<<"$fallback_models_raw" - - local candidate_raw - local candidate - for candidate_raw in "${fallback_models[@]}"; do - candidate="$(normalize_model "$candidate_raw")" - if [ -n "$candidate" ] && [ "$candidate" != "$model" ]; then - return 0 - fi - done - - return 1 -} - resolved_llm_api_base_for_model() { local model="$1" From 51a4aa7d2da0e8cf4e940e0ef1ea3f5d8281bba6 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 16:40:18 +0900 Subject: [PATCH 05/15] fix(strix): fallback on unsupported sampling defaults --- CHANGELOG.md | 6 +- .../strix-pr-baseline-provider-exhaustion.md | 36 +++++ scripts/ci/strix_quick_gate.sh | 21 +++ scripts/ci/test_strix_quick_gate.sh | 148 +++++++++++++++++- 4 files changed, 208 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b9219dc4b..eec4c525e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,9 +26,11 @@ Semantic Versioning where the repository publishes a release. baseline issues and then exhausts every LLM provider, so provider retirement or outage is a neutral infrastructure result even when no distinct fallback is configured; recognize the exact local Caido bootstrap connection failure - the same way only when no vulnerability exists, while changed and unmapped + the same way only when no vulnerability exists; route the exact single-line + LiteLLM/Azure unsupported-temperature failure to the existing distinct outer + fallback without accepting split-line imitations, while changed and unmapped findings and all other runtime failures still fail closed. The reproduced - incidents and gate boundary are recorded in + incidents, APA 7 sources, and gate boundary are recorded in `docs/doctoring/strix-pr-baseline-provider-exhaustion.md`. - Emit completed repository pull-list requests as they finish in the five-minute agent-mention sweep, while retaining the four-worker ceiling, rotation, and diff --git a/docs/doctoring/strix-pr-baseline-provider-exhaustion.md b/docs/doctoring/strix-pr-baseline-provider-exhaustion.md index 010af2cad..aa3d0cc2c 100644 --- a/docs/doctoring/strix-pr-baseline-provider-exhaustion.md +++ b/docs/doctoring/strix-pr-baseline-provider-exhaustion.md @@ -15,6 +15,14 @@ scanning when Strix's local Caido process did not accept connections on `127.0.0.1:48080`; the outer workflow did not yet recognize that exact scanner bootstrap outage as infrastructure. +Another observed run selected Azure `gpt-5.6-sol`, where LiteLLM forwarded +Strix's `temperature=0.2`. Azure rejected the unsupported sampling parameter +and LiteLLM had no fallback group for that model. Microsoft documents +`temperature` as unsupported for reasoning models (Microsoft, 2026), while the +documented Strix configuration surface has no generation-parameter control and +the missing capability remains an upstream request (AkikoOrenji, 2026; +usestrix, n.d.). + ## Root cause and repair The quick gate already owns the exact PR-head changed-file mapping decision. @@ -30,6 +38,16 @@ exhaustion with curl exit 7 against Strix's fixed local Caido port. It is neutra only when no positive vulnerability or severity signal exists; every other runtime failure remains blocking. +The preferred request-boundary repair is to omit a sampling parameter a caller +did not explicitly provide. The pinned Strix integration cannot currently do +that through its documented configuration, so the quick gate recognizes only a +single log line containing the complete LiteLLM/Azure unsupported-temperature +failure and the missing internal model group. It skips deterministic same-model +retry and moves to the existing distinct outer fallback. Split-line signal +assembly stays non-retryable, and provider exhaustion becomes neutral only +after the existing trusted PR-scope mapper has classified every report as +`allow_baseline`. + ## Verification - A three-attempt regression reproduces an unchanged critical report followed @@ -40,5 +58,23 @@ runtime failure remains blocking. unchanged findings to remain admissible. - A workflow regression requires the exact Caido bootstrap outage to be neutral with zero findings and blocking when any vulnerability is reported. +- An Azure capability regression requires the exact unsupported-temperature + line to reach the configured GitHub Models fallback without a same-model + retry; a split-line imitation must remain non-recoverable. - The central Python suite, native workflow validation, Bash syntax checks, and the complete Strix shell regression suite run on the final tree. + +## References + +AkikoOrenji. (2026, June 4). *[Feature] Expose LLM generation parameters to +control local/OpenAI-compatible model behaviour and prevent runaway tool-call +loops* (Issue No. 514) [GitHub issue]. GitHub. +https://github.com/usestrix/strix/issues/514 + +Microsoft. (2026, August 20). *Azure OpenAI reasoning models—GPT-5 series, +o3-mini, o1, o1-mini*. Microsoft Learn. +https://learn.microsoft.com/en-us/azure/foundry/openai/how-to/reasoning + +usestrix. (n.d.). *Configuration* [Computer software documentation]. GitHub. +Retrieved August 22, 2026, from +https://github.com/usestrix/strix/blob/main/docs/advanced/configuration.mdx diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index 996328b9f..666fe9ed0 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -2638,6 +2638,19 @@ is_nvidia_nim_not_found_error() { return 1 } +is_unsupported_model_parameter_error() { + # Strix currently has no generation-parameter override. Match the exact + # single-line LiteLLM/Azure capability failure so a reasoning model that + # rejects Strix's temperature can move to the already-configured fallback. + if grep -Ei 'litellm(\.exceptions)?\.BadRequestError' "$STRIX_LOG" | + grep -Ei '(AzureException|OpenAIException)' | + grep -Eiq "Unsupported value:[[:space:]]*['\"]temperature['\"].*Only the default[[:space:]]*\\(1\\)[[:space:]]*value is supported.*No fallback model group found"; then + return 0 + fi + + return 1 +} + ## Determines whether the last strix failure is a transient error eligible ## for same-model retry (up to STRIX_TRANSIENT_RETRY_PER_MODEL times). ## Four error families qualify: @@ -2954,6 +2967,10 @@ has_detected_infrastructure_error() { return 0 fi + if is_unsupported_model_parameter_error; then + return 0 + fi + # Generic strix non-zero exit with known transport/connection errors # that don't fall into the specific categories above. # Use LLM_PROVIDER_ONLY_REGEX (not PROVIDER_CONTEXT_REGEX) to avoid @@ -3835,6 +3852,10 @@ is_model_retryable_error() { return 0 fi + if is_unsupported_model_parameter_error; then + return 0 + fi + if [ "$PR_FINDINGS_DECISION" = "retry_model_inconsistency" ]; then return 0 fi diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index 432dcee71..56c613aad 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -3386,6 +3386,27 @@ REPORT ;; esac ;; + openai-direct-unsupported-temperature-github-models-fallback-success) + case "${STRIX_LLM:-}" in + openai/gpt-5.6-sol) + echo "litellm.BadRequestError: AzureException BadRequestError - Unsupported value: 'temperature' does not support 0.2 with this model. Only the default (1) value is supported. No fallback model group found for original model_group=gpt-5.6-sol." + exit 1 + ;; + openai/o3) + echo "scan ok with GitHub Models fallback" + exit 0 + ;; + *) + echo "unexpected model ${STRIX_LLM:-}" >&2 + exit 9 + ;; + esac + ;; + openai-direct-unsupported-temperature-split-lines-nonrecoverable) + echo "litellm.BadRequestError: request rejected" + echo "AzureException - Unsupported value: 'temperature' does not support 0.2 with this model. Only the default (1) value is supported. No fallback model group found." + exit 1 + ;; vertex-all-notfound) echo "Error: litellm.NotFoundError: Vertex_aiException - x" echo '"status": "NOT_FOUND"' @@ -5519,7 +5540,8 @@ PY FAKE_STRIX_OUTSIDE_REPORT_DIR="$repo_root_dir/outside-strix-report" ) fi - if [ "$scenario" = "openai-direct-quota-github-models-fallback-success" ]; then + if [ "$scenario" = "openai-direct-quota-github-models-fallback-success" ] || + [ "$scenario" = "openai-direct-unsupported-temperature-github-models-fallback-success" ]; then printf '%s' 'https://models.github.ai/inference' >"$tmp_dir/github_models_api_base.txt" printf '%s' 'github-models-fallback-token' >"$tmp_dir/github_models_key.txt" env_cmd+=(STRIX_GITHUB_MODELS_API_BASE_FILE="$tmp_dir/github_models_api_base.txt") @@ -5921,6 +5943,66 @@ run_filtered_gate_case_if_requested() { "" \ "github_models/openai/o3" ;; + openai-direct-unsupported-temperature-github-models-fallback-success) + run_gate_case "openai-direct-unsupported-temperature-github-models-fallback-success" \ + "openai_direct/gpt-5.6-sol" \ + "" \ + "0" \ + "REGEX:Strix quick scan succeeded with fallback model 'github_models/openai/o3' in [0-9]+s\\." \ + "2" \ + "openai/gpt-5.6-sol|openai/o3" \ + "|https://models.github.ai/inference" \ + "vertex_ai" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "github_models/openai/o3" + ;; + openai-direct-unsupported-temperature-split-lines-nonrecoverable) + run_gate_case "openai-direct-unsupported-temperature-split-lines-nonrecoverable" \ + "openai_direct/gpt-5.6-sol" \ + "" \ + "1" \ + "Strix quick scan failed with a non-recoverable error." \ + "1" \ + "openai/gpt-5.6-sol" \ + "" \ + "vertex_ai" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "github_models/openai/o3" + ;; gemini-timeout-fallback-success) run_gate_case_allow_provider_signal "gemini-timeout-fallback-success" \ "gemini/timeout-fallback-primary" \ @@ -12107,6 +12189,70 @@ run_gate_case "openai-direct-quota-github-models-fallback-success" \ "" \ "github_models/openai/o3" +# Strix currently reaches LiteLLM with a sampling default and exposes no +# documented generation-parameter override. Azure reasoning deployments reject +# that temperature before Strix's internal model-group fallback can run, so the +# trusted outer gate must try its already-configured distinct provider. +run_gate_case "openai-direct-unsupported-temperature-github-models-fallback-success" \ + "openai_direct/gpt-5.6-sol" \ + "" \ + "0" \ + "REGEX:Strix quick scan succeeded with fallback model 'github_models/openai/o3' in [0-9]+s\\." \ + "2" \ + "openai/gpt-5.6-sol|openai/o3" \ + "|https://models.github.ai/inference" \ + "vertex_ai" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "github_models/openai/o3" + +# Cross-line assembly is deliberately rejected: target/source text cannot +# manufacture a provider capability signal from independent log lines. +run_gate_case "openai-direct-unsupported-temperature-split-lines-nonrecoverable" \ + "openai_direct/gpt-5.6-sol" \ + "" \ + "1" \ + "Strix quick scan failed with a non-recoverable error." \ + "1" \ + "openai/gpt-5.6-sol" \ + "" \ + "vertex_ai" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "github_models/openai/o3" + run_gate_case "github-models-fallback-success-deepseek-v3" \ "vertex_ai/missing-primary" \ "github_models/deepseek/deepseek-r1-0528 github_models/deepseek/deepseek-v3-0324" \ From a3b149bd5d924922d892d768b7832d9148e17bbe Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 18:38:40 +0900 Subject: [PATCH 06/15] fix(strix): align manifest finding decisions --- scripts/ci/strix_quick_gate.sh | 4 ++-- scripts/ci/test_strix_quick_gate.sh | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index 666fe9ed0..b5581688b 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -3912,7 +3912,7 @@ run_current_target_scan() { fi case "$PR_FINDINGS_DECISION" in - block_changed | block_unmapped | block_manifest_unverified) + block_changed | block_unmapped | block_manifest_finding) if [ "$strict_primary_provider_fallback" -eq 1 ] && fail_reported_vulnerabilities_before_fallback_success; then return 1 fi @@ -3989,7 +3989,7 @@ run_current_target_scan() { fi case "$PR_FINDINGS_DECISION" in - block_changed | block_unmapped | block_manifest_unverified) + block_changed | block_unmapped | block_manifest_finding) if [ "$strict_fallback_provider_signal" -eq 1 ] && fail_reported_vulnerabilities_before_fallback_success; then return 1 fi diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index 56c613aad..dcdc54d88 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -316,6 +316,8 @@ assert_strix_workflow_pr_trigger_hardened() { assert_file_contains "$GATE_SCRIPT" "os.walk(root, topdown=True, followlinks=False)" "strix gate does not recurse into symlinked report directories" assert_file_not_contains "$GATE_SCRIPT" 'root.rglob("*.log")' "strix gate avoids recursive pathlib glob traversal for report logs" assert_file_contains "$GATE_SCRIPT" "has_strix_report_failure_signal" "strix gate fails closed on warning-class Strix report artifacts" + assert_file_not_contains "$GATE_SCRIPT" "block_manifest_unverified" "strix gate has no dead manifest-decision alias" + assert_file_contains "$GATE_SCRIPT" 'block_changed | block_unmapped | block_manifest_finding)' "strix gate handles the manifest finding decision in both fallback paths" assert_file_not_contains "$workflow_file" "ignore::UserWarning" "strix workflow must not blanket-suppress all UserWarning output" assert_file_contains "$GATE_SCRIPT" "vulnerability_file_reports_generic_github_actions_workflow_insecurity" "strix gate fact-checks generic GitHub Actions workflow security reports before accepting whole-file claims" assert_file_not_contains "$workflow_file" "vertex_ai/* | vertex_ai_beta/*" "strix workflow must not accept arbitrary Vertex models" From 60f5150f1dcc89c622f5b379561e622d491dd06e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 20:53:41 +0900 Subject: [PATCH 07/15] docs(strix): clarify provider exhaustion exits --- .github/workflows/strix.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index ddd01b508..130d99fe1 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -868,8 +868,9 @@ jobs: fi # Preserve configuration failures (exit 2) and any unexpected exit - # code as hard failures — only the scan-failure code (1) can be an - # infrastructure/backend-unavailability outcome. + # code as hard failures. Exit 1 uses the log-based infrastructure + # fallback; exit 3 is the trusted post-baseline provider-exhaustion + # outcome handled above. if [ "$strix_rc" -ne 1 ]; then exit "$strix_rc" fi From a675daccb65ee582781690db1b0e9dd282b34af1 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 20:58:39 +0900 Subject: [PATCH 08/15] fix(strix): accept clean fallback model advisory --- scripts/ci/strix_quick_gate.sh | 3 ++- scripts/ci/test_strix_quick_gate.sh | 38 +++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index b9b25096a..506a72d0d 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -3164,7 +3164,8 @@ is_llm_token_limit_error() { # was interrupted or incomplete. Used as a guard to prevent the # below-threshold override from silently passing an aborted scan. has_detected_infrastructure_error() { - if grep -Eiq '(^|[^[:alpha:]])(Fatal|Denied|Warn|Warning)([^[:alpha:]]|$)' "$STRIX_LOG"; then + if grep -Eiv '^[[:space:]│]*MODEL QUALITY WARNING[[:space:]│]*$' "$STRIX_LOG" | + grep -Eiq '(^|[^[:alpha:]])(Fatal|Denied|Warn|Warning)([^[:alpha:]]|$)'; then return 0 fi diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index ae3b4d2d3..250ea76a5 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -3601,6 +3601,23 @@ REPORT ;; esac ;; + nvidia-ratelimit-model-quality-warning-fallback-success) + case "${STRIX_LLM:-}" in + nvidia_nim/nvidia/nemotron-3-super-120b-a12b) + echo "litellm.RateLimitError: Nvidia_nimException - Error code: 429 - Too Many Requests" + exit 1 + ;; + nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5) + echo "│ MODEL QUALITY WARNING │" + echo "│ Vulnerabilities 0 (No exploitable vulnerabilities detected) │" + exit 0 + ;; + *) + echo "Error: NVIDIA model-quality fallback path unexpected (${STRIX_LLM:-})" >&2 + exit 61 + ;; + esac + ;; vertex-primary-resource-exhausted-fallback-success) case "${STRIX_LLM:-}" in vertex_ai/resource-exhausted-primary) @@ -6445,6 +6462,17 @@ run_filtered_gate_case_if_requested() { "vertex_ai/report-known-internal-warning-sanitized" \ "" ;; + nvidia-ratelimit-model-quality-warning-fallback-success) + run_gate_case "nvidia-ratelimit-model-quality-warning-fallback-success" \ + "nvidia_nim/nvidia/nemotron-3-super-120b-a12b" \ + "nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5" \ + "0" \ + "REGEX:Strix quick scan succeeded with fallback model 'nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5' in [0-9]+s\\." \ + "2" \ + "nvidia_nim/nvidia/nemotron-3-super-120b-a12b|nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5" \ + "|" \ + "openai" + ;; provider-fatal-success-signal | provider-warning-success-signal) run_gate_case "$STRIX_TEST_CASE_FILTER" \ "vertex_ai/$STRIX_TEST_CASE_FILTER" \ @@ -9784,6 +9812,16 @@ run_gate_case_allow_provider_signal "vertex-primary-ratelimit-fallback-success" "vertex_ai/ratelimit-primary|vertex_ai/fallback-one" \ "|" +run_gate_case "nvidia-ratelimit-model-quality-warning-fallback-success" \ + "nvidia_nim/nvidia/nemotron-3-super-120b-a12b" \ + "nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5" \ + "0" \ + "REGEX:Strix quick scan succeeded with fallback model 'nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5' in [0-9]+s\\." \ + "2" \ + "nvidia_nim/nvidia/nemotron-3-super-120b-a12b|nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5" \ + "|" \ + "openai" + run_gate_case_allow_provider_signal "vertex-primary-resource-exhausted-fallback-success" \ "vertex_ai/resource-exhausted-primary" \ "vertex_ai/fallback-one vertex_ai/fallback-two" \ From 32543e093be21759625fcb5b6c4f02ebe4fe09c8 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 21:15:03 +0900 Subject: [PATCH 09/15] fix(strix): accept clean HF advisory --- scripts/ci/strix_quick_gate.sh | 2 +- scripts/ci/test_strix_quick_gate.sh | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index 506a72d0d..3b965e2c7 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -3164,7 +3164,7 @@ is_llm_token_limit_error() { # was interrupted or incomplete. Used as a guard to prevent the # below-threshold override from silently passing an aborted scan. has_detected_infrastructure_error() { - if grep -Eiv '^[[:space:]│]*MODEL QUALITY WARNING[[:space:]│]*$' "$STRIX_LOG" | + if grep -Eiv '^[[:space:]│]*MODEL QUALITY WARNING[[:space:]│]*$|^Warning: You are sending unauthenticated requests to the HF Hub\.' "$STRIX_LOG" | grep -Eiq '(^|[^[:alpha:]])(Fatal|Denied|Warn|Warning)([^[:alpha:]]|$)'; then return 0 fi diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index 250ea76a5..a268ced6b 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -3609,6 +3609,7 @@ REPORT ;; nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5) echo "│ MODEL QUALITY WARNING │" + echo "Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads." echo "│ Vulnerabilities 0 (No exploitable vulnerabilities detected) │" exit 0 ;; From 86408bacc5b91c9da49671a4b767202650bb75d8 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 21:15:34 +0900 Subject: [PATCH 10/15] docs(strix): record clean fallback advisories --- docs/doctoring/strix-nvidia-nim-not-found-fallback.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/doctoring/strix-nvidia-nim-not-found-fallback.md b/docs/doctoring/strix-nvidia-nim-not-found-fallback.md index a088aa7ef..a1f2226d8 100644 --- a/docs/doctoring/strix-nvidia-nim-not-found-fallback.md +++ b/docs/doctoring/strix-nvidia-nim-not-found-fallback.md @@ -30,6 +30,11 @@ combining with an unrelated application `404` to spoof infrastructure fallback. Provider-side failure also remains a fail-closed incomplete scan until a distinct fallback produces complete evidence. +A completed zero-vulnerability fallback may emit Strix's model-quality banner +and Hugging Face's exact unauthenticated-download advisory while loading public +scanner assets. Those two bounded advisories are not provider-failure evidence; +all other warnings remain fail closed. + Exhausted provider infrastructure remains fail-closed even when the trusted gate has classified every observed threshold finding as outside the pull request's changed files. That classification scopes authoritative findings; it @@ -53,7 +58,9 @@ Regression evidence proves that: 8. provider exhaustion remains non-passing after unchanged baseline findings; 9. changed, unmapped, and changed-manifest findings also block after provider exhaustion; and -10. the required-workflow smoke contract pins these properties. +10. the exact model-quality and Hugging Face download advisories do not reject + an otherwise complete zero-vulnerability fallback; and +11. the required-workflow smoke contract pins these properties. ## Limitations From 9dcd1d6c4c11602163cb89edaad9540e9d6f461a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 21:18:58 +0900 Subject: [PATCH 11/15] fix(strix): preserve fail-closed warning scan --- scripts/ci/strix_quick_gate.sh | 4 ++-- scripts/ci/test_strix_quick_gate.sh | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index 3b965e2c7..04859990b 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -3164,8 +3164,8 @@ is_llm_token_limit_error() { # was interrupted or incomplete. Used as a guard to prevent the # below-threshold override from silently passing an aborted scan. has_detected_infrastructure_error() { - if grep -Eiv '^[[:space:]│]*MODEL QUALITY WARNING[[:space:]│]*$|^Warning: You are sending unauthenticated requests to the HF Hub\.' "$STRIX_LOG" | - grep -Eiq '(^|[^[:alpha:]])(Fatal|Denied|Warn|Warning)([^[:alpha:]]|$)'; then + if grep -Eiq '(^|[^[:alpha:]])(Fatal|Denied|Warn|Warning)([^[:alpha:]]|$)' \ + < <(grep -Eiv '^[[:space:]│]*MODEL QUALITY WARNING[[:space:]│]*$|^Warning: You are sending unauthenticated requests to the HF Hub\.' "$STRIX_LOG"); then return 0 fi diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index a268ced6b..5de471992 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -4511,6 +4511,9 @@ EOS exit 0 ;; provider-warning-success-signal) + for _ in {1..10000}; do + echo "benign scanner output" + done echo "Warning: provider response included incomplete scan state" exit 0 ;; From 3868a2106bd249d9ccdf72abf8ed3f1a50366f98 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 21:29:21 +0900 Subject: [PATCH 12/15] fix(strix): isolate cross-provider OpenAI fallback transport --- .github/workflows/strix.yml | 17 ++++ CHANGELOG.md | 7 +- .../strix-unsupported-sampling-fallback.md | 15 ++- scripts/ci/strix_quick_gate.sh | 39 +++++++- scripts/ci/test_strix_quick_gate.sh | 99 +++++++++++++++++++ 5 files changed, 167 insertions(+), 10 deletions(-) diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index b3248d943..aa469466e 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -696,6 +696,22 @@ jobs: printf '%s' 'https://models.github.ai/inference' > "$github_models_api_base_file" echo "STRIX_GITHUB_MODELS_API_BASE_FILE=$github_models_api_base_file" >> "$GITHUB_ENV" + - name: Prepare direct OpenAI fallback credentials + if: steps.gate.outputs.provider_mode == 'github_models' || steps.gate.outputs.provider_mode == 'openrouter' || steps.gate.outputs.provider_mode == 'nvidia_nim' + env: + OPENAI_FALLBACK_API_KEY: ${{ secrets.STRIX_OPENAI_API_KEY || secrets.OPENAI_API_KEY }} + run: | + umask 077 + sanitized="$(printf '%s' "$OPENAI_FALLBACK_API_KEY" | tr -d '\r\n')" + trimmed="$(printf '%s' "$sanitized" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')" + if [ -z "$trimmed" ]; then + echo '::notice::No direct OpenAI key available; cross-provider Strix scans run without that fallback.' + exit 0 + fi + openai_fallback_key_file="$RUNNER_TEMP/openai_fallback_key.txt" + printf '%s' "$trimmed" > "$openai_fallback_key_file" + echo "STRIX_OPENAI_FALLBACK_KEY_FILE=$openai_fallback_key_file" >> "$GITHUB_ENV" + - name: Prepare Vertex AI credentials if: steps.gate.outputs.provider_mode == 'vertex_ai' env: @@ -825,6 +841,7 @@ jobs: STRIX_FALLBACK_MODELS: ${{ steps.gate.outputs.provider_mode == 'github_models' && 'openai-direct/gpt-5.6-luna' || steps.gate.outputs.provider_mode == 'openai_direct' && 'openai-direct/gpt-5.6-luna' || steps.gate.outputs.provider_mode == 'openrouter' && 'openai-direct/gpt-5.6-luna' || steps.gate.outputs.provider_mode == 'nvidia_nim' && 'nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5 openai-direct/gpt-5.6-luna' || '' }} STRIX_GITHUB_MODELS_API_BASE_FILE: ${{ env.STRIX_GITHUB_MODELS_API_BASE_FILE }} STRIX_GITHUB_MODELS_KEY_FILE: ${{ env.STRIX_GITHUB_MODELS_KEY_FILE }} + STRIX_OPENAI_FALLBACK_KEY_FILE: ${{ env.STRIX_OPENAI_FALLBACK_KEY_FILE }} STRIX_FAIL_ON_PROVIDER_SIGNAL: "1" STRIX_VERTEX_FALLBACK_MODELS: "" NPM_CONFIG_IGNORE_SCRIPTS: "true" diff --git a/CHANGELOG.md b/CHANGELOG.md index bf0a1d071..825a47ca6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,9 +40,10 @@ Semantic Versioning where the repository publishes a release. - Route the exact single-line LiteLLM/Azure unsupported-temperature failure to an already-configured distinct Strix fallback without accepting split-line imitations, and normalize the workflow's `openai-direct/` fallback alias to - LiteLLM's `openai_direct/` provider prefix before dispatch, without turning - an incomplete provider scan into passing evidence. The incident and - fail-closed boundary are recorded in + LiteLLM's `openai_direct/` provider prefix before dispatch. Cross-provider + attempts now switch to the trusted OpenAI credential and clear the primary + provider API base, without turning an incomplete provider scan into passing + evidence. The incident and fail-closed boundary are recorded in `docs/doctoring/strix-unsupported-sampling-fallback.md`. - Emit completed repository pull-list requests as they finish in the five-minute agent-mention sweep, while retaining the four-worker ceiling, rotation, and diff --git a/docs/doctoring/strix-unsupported-sampling-fallback.md b/docs/doctoring/strix-unsupported-sampling-fallback.md index a43ef43bc..e0a75965d 100644 --- a/docs/doctoring/strix-unsupported-sampling-fallback.md +++ b/docs/doctoring/strix-unsupported-sampling-fallback.md @@ -27,11 +27,14 @@ That exact capability failure is infrastructure evidence and may move directly to an already-configured distinct outer fallback. It is not eligible for a same-model retry. The shared model normalizer also translates the workflow's human-readable `openai-direct/` selector into LiteLLM's `openai_direct/` -provider prefix before dispatch; otherwise the configured fallback fails before -it can make a provider request. If no distinct fallback exists or every -fallback fails, the required Strix check remains non-passing. Existing changed, -unmapped, manifest, `ModelBehaviorError`, and vulnerability-report boundaries -remain fail closed. +provider prefix before dispatch. A cross-provider direct OpenAI fallback reads +the established OpenAI secret from a trusted runtime file and clears the +primary provider's API base; otherwise a NVIDIA or OpenRouter run would send +the fallback to the wrong endpoint with the wrong credential. If the fallback +credential is unavailable, the attempted fallback fails configuration closed. +If no distinct fallback exists or every fallback fails, the required Strix +check remains non-passing. Existing changed, unmapped, manifest, +`ModelBehaviorError`, and vulnerability-report boundaries remain fail closed. Cross-line signal assembly is deliberately rejected so unrelated target output cannot manufacture a provider capability error from separate log lines. @@ -42,6 +45,8 @@ cannot manufacture a provider capability error from separate log lines. Models fallback exactly once and succeeds only when that scan completes. - The configured `openai-direct/gpt-5.6-luna` fallback normalizes to the LiteLLM-compatible `openai_direct/gpt-5.6-luna` selector. +- A NVIDIA-primary run dispatches that fallback with the OpenAI credential and + no inherited NVIDIA API base. - A split-line imitation is non-recoverable and never dispatches the fallback. - The full Python suite, native workflow validation, Bash syntax checks, and complete Strix shell regression suite run on the final tree. diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index b9b25096a..052d137f7 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -380,6 +380,23 @@ if [ -n "$STRIX_GITHUB_MODELS_KEY_FILE" ]; then fi fi +STRIX_OPENAI_FALLBACK_KEY_FILE="${STRIX_OPENAI_FALLBACK_KEY_FILE:-}" +if [ -n "$STRIX_OPENAI_FALLBACK_KEY_FILE" ] && { [ ! -f "$STRIX_OPENAI_FALLBACK_KEY_FILE" ] || [ -L "$STRIX_OPENAI_FALLBACK_KEY_FILE" ]; }; then + echo "ERROR: STRIX_OPENAI_FALLBACK_KEY_FILE must reference a regular file containing the API key." >&2 + exit 2 +fi +if [ -n "$STRIX_OPENAI_FALLBACK_KEY_FILE" ] && ! STRIX_OPENAI_FALLBACK_KEY_FILE="$(resolve_trusted_input_file "STRIX_OPENAI_FALLBACK_KEY_FILE" "$STRIX_OPENAI_FALLBACK_KEY_FILE")"; then + exit 2 +fi +STRIX_OPENAI_FALLBACK_KEY="" +if [ -n "$STRIX_OPENAI_FALLBACK_KEY_FILE" ]; then + STRIX_OPENAI_FALLBACK_KEY="$(trim_whitespace "$(cat -- "$STRIX_OPENAI_FALLBACK_KEY_FILE")")" + if [ -z "$STRIX_OPENAI_FALLBACK_KEY" ]; then + echo "ERROR: STRIX_OPENAI_FALLBACK_KEY_FILE must contain a non-empty API key." >&2 + exit 2 + fi +fi + require_non_negative_integer() { local value="$1" local label="$2" @@ -2380,6 +2397,12 @@ resolved_llm_api_base_for_model() { if is_vertex_model "$model"; then return 0 fi + case "$(normalize_model "$model"):$PRIMARY_MODEL" in + openai_direct/*:openai_direct/*) ;; + openai_direct/*:*) + return 0 + ;; + esac local api_base_file="$LLM_API_BASE_FILE" local api_base_file_name="LLM_API_BASE_FILE" @@ -2497,13 +2520,25 @@ run_strix_once() { local start_epoch start_epoch="$(date +%s)" local child_llm_api_key="" - if ! is_vertex_model "$(normalize_model "$model")"; then + local normalized_model + normalized_model="$(normalize_model "$model")" + if ! is_vertex_model "$normalized_model"; then child_llm_api_key="$LLM_API_KEY" - if is_github_models_model "$(normalize_model "$model")" && [ -n "$STRIX_GITHUB_MODELS_KEY" ]; then + if is_github_models_model "$normalized_model" && [ -n "$STRIX_GITHUB_MODELS_KEY" ]; then # Cross-provider fallback: github_models/* models authenticate # with the GitHub Models token, not the direct-OpenAI key. child_llm_api_key="$STRIX_GITHUB_MODELS_KEY" fi + case "$normalized_model:$PRIMARY_MODEL" in + openai_direct/*:openai_direct/*) ;; + openai_direct/*:*) + if [ -z "$STRIX_OPENAI_FALLBACK_KEY" ]; then + echo "ERROR: direct OpenAI fallback requires STRIX_OPENAI_FALLBACK_KEY_FILE." >&2 + return 2 + fi + child_llm_api_key="$STRIX_OPENAI_FALLBACK_KEY" + ;; + esac fi set -o pipefail set +e diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index ae3b4d2d3..7a2cb1b40 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -367,6 +367,10 @@ assert_strix_workflow_pr_trigger_hardened() { assert_file_contains "$workflow_file" "Prepare GitHub Models fallback credentials" "strix workflow provisions GitHub Models fallback credentials for direct-OpenAI scans" assert_file_contains "$GATE_SCRIPT" "STRIX_GITHUB_MODELS_KEY_FILE" "strix gate reads the optional GitHub Models fallback key file" assert_file_contains "$GATE_SCRIPT" "STRIX_GITHUB_MODELS_API_BASE_FILE" "strix gate routes github_models fallback models through the GitHub Models endpoint" + assert_file_contains "$workflow_file" "Prepare direct OpenAI fallback credentials" "strix workflow provisions direct OpenAI credentials for cross-provider fallbacks" + assert_file_contains "$workflow_file" 'OPENAI_FALLBACK_API_KEY: ${{ secrets.STRIX_OPENAI_API_KEY || secrets.OPENAI_API_KEY }}' "strix workflow reads the established direct OpenAI secret only at the credential boundary" + assert_file_contains "$workflow_file" "STRIX_OPENAI_FALLBACK_KEY_FILE" "strix workflow passes the direct OpenAI fallback key through a trusted file" + assert_file_contains "$GATE_SCRIPT" "STRIX_OPENAI_FALLBACK_KEY_FILE" "strix gate reads the direct OpenAI fallback key from a trusted file" assert_file_not_contains "$workflow_file" 'github_models/deepseek/deepseek-r1-0528 | github_models/deepseek/deepseek-v3-0324)' "strix workflow keeps DeepSeek GitHub Models restricted to fallback-only routing" assert_file_contains "$workflow_file" '${strix_model#github_models/}' "strix workflow strips manual github_models routing prefix for OpenAI GPT model names before passing model names to LiteLLM" assert_file_contains "$workflow_file" "openai_direct/%s" "strix workflow keeps manual direct OpenAI scans distinct from GitHub Models openai/gpt-* routing" @@ -3437,6 +3441,34 @@ REPORT echo "AzureException - Unsupported value: 'temperature' does not support 0.2 with this model. Only the default (1) value is supported. No fallback model group found." exit 1 ;; + nvidia-openai-direct-fallback-credential-success | nvidia-openai-direct-fallback-missing-key-fails-closed) + case "${STRIX_LLM:-}" in + nvidia_nim/nvidia/primary) + if [ "${LLM_API_KEY:-}" != "dummy" ]; then + echo "unexpected NVIDIA primary key (${LLM_API_KEY:-})" >&2 + exit 17 + fi + echo "Penetration test failed: LLM request failed: RateLimitError" + exit 1 + ;; + openai/gpt-5.6-luna) + if [ "${LLM_API_KEY:-}" != "openai-fallback-token" ]; then + echo "unexpected direct OpenAI fallback key (${LLM_API_KEY:-})" >&2 + exit 18 + fi + if [ -n "${LLM_API_BASE:-}" ]; then + echo "direct OpenAI fallback inherited primary API base: $LLM_API_BASE" >&2 + exit 19 + fi + echo "scan ok with direct OpenAI fallback" + exit 0 + ;; + *) + echo "unexpected model ${STRIX_LLM:-}" >&2 + exit 9 + ;; + esac + ;; vertex-all-notfound) echo "Error: litellm.NotFoundError: Vertex_aiException - x" echo '"status": "NOT_FOUND"' @@ -5665,6 +5697,10 @@ PY env_cmd+=(STRIX_GITHUB_MODELS_API_BASE_FILE="$tmp_dir/github_models_api_base.txt") env_cmd+=(STRIX_GITHUB_MODELS_KEY_FILE="$tmp_dir/github_models_key.txt") fi + if [ "$scenario" = "nvidia-openai-direct-fallback-credential-success" ]; then + printf '%s' 'openai-fallback-token' >"$tmp_dir/openai_fallback_key.txt" + env_cmd+=(STRIX_OPENAI_FALLBACK_KEY_FILE="$tmp_dir/openai_fallback_key.txt") + fi if [ "$min_fail_severity" = "__UNSET__" ]; then local next_env_cmd=() local env_pair @@ -5958,6 +5994,45 @@ run_github_models_http410_case() { "1" } +run_nvidia_openai_direct_fallback_case() { + local scenario="${1:-nvidia-openai-direct-fallback-credential-success}" + local expected_exit="${2:-0}" + local expected_message="${3:-REGEX:Strix quick scan succeeded with fallback model 'openai_direct/gpt-5.6-luna' in [0-9]+s\\.}" + local expected_calls="${4:-2}" + local expected_models="${5:-nvidia_nim/nvidia/primary|openai/gpt-5.6-luna}" + local expected_api_bases="${6:-https://integrate.api.nvidia.com/v1|}" + + run_gate_case "$scenario" \ + "nvidia_nim/nvidia/primary" \ + "" \ + "$expected_exit" \ + "$expected_message" \ + "$expected_calls" \ + "$expected_models" \ + "$expected_api_bases" \ + "nvidia_nim" \ + "https://integrate.api.nvidia.com/v1" \ + "" \ + "0" \ + "CRITICAL" \ + "0" \ + "" \ + "" \ + "1200" \ + "0" \ + "" \ + "" \ + "" \ + "" \ + "0" \ + "" \ + "" \ + "" \ + "" \ + "openai-direct/gpt-5.6-luna" \ + "1" +} + run_filtered_gate_case_if_requested() { case "${STRIX_TEST_CASE_FILTER:-}" in "") @@ -6201,6 +6276,18 @@ run_filtered_gate_case_if_requested() { "" \ "github_models/openai/o3" ;; + nvidia-openai-direct-fallback-credential-success) + run_nvidia_openai_direct_fallback_case + ;; + nvidia-openai-direct-fallback-missing-key-fails-closed) + run_nvidia_openai_direct_fallback_case \ + "$STRIX_TEST_CASE_FILTER" \ + "2" \ + "direct OpenAI fallback requires STRIX_OPENAI_FALLBACK_KEY_FILE" \ + "1" \ + "nvidia_nim/nvidia/primary" \ + "https://integrate.api.nvidia.com/v1" + ;; gemini-timeout-fallback-success) run_gate_case_allow_provider_signal "gemini-timeout-fallback-success" \ "gemini/timeout-fallback-primary" \ @@ -12661,6 +12748,18 @@ run_gate_case "openai-direct-unsupported-temperature-split-lines-nonrecoverable" "" \ "github_models/openai/o3" +# Cross-provider fallbacks must switch both the API key and endpoint. Reusing +# NVIDIA credentials or its API base makes a normalized direct-OpenAI model +# fail before producing security evidence. +run_nvidia_openai_direct_fallback_case +run_nvidia_openai_direct_fallback_case \ + "nvidia-openai-direct-fallback-missing-key-fails-closed" \ + "2" \ + "direct OpenAI fallback requires STRIX_OPENAI_FALLBACK_KEY_FILE" \ + "1" \ + "nvidia_nim/nvidia/primary" \ + "https://integrate.api.nvidia.com/v1" + run_gate_case "github-models-fallback-success-deepseek-v3" \ "vertex_ai/missing-primary" \ "github_models/deepseek/deepseek-r1-0528 github_models/deepseek/deepseek-v3-0324" \ From 5edaf36c260d3b0dd23fd59c87e713606cfebae5 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 22:01:22 +0900 Subject: [PATCH 13/15] fix(strix): bound clean advisory filtering --- CHANGELOG.md | 4 ++- .../strix-nvidia-nim-not-found-fallback.md | 11 ++++--- scripts/ci/strix_quick_gate.sh | 14 +++++++-- scripts/ci/test_strix_quick_gate.sh | 29 +++++++++++++++++++ 4 files changed, 51 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 825a47ca6..6c6097f66 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,7 +43,9 @@ Semantic Versioning where the repository publishes a release. LiteLLM's `openai_direct/` provider prefix before dispatch. Cross-provider attempts now switch to the trusted OpenAI credential and clear the primary provider API base, without turning an incomplete provider scan into passing - evidence. The incident and fail-closed boundary are recorded in + evidence. Exact clean model-quality and Hugging Face advisories are filtered + consistently from console and report logs, while any appended warning text + remains fail closed. The incident and fail-closed boundary are recorded in `docs/doctoring/strix-unsupported-sampling-fallback.md`. - Emit completed repository pull-list requests as they finish in the five-minute agent-mention sweep, while retaining the four-worker ceiling, rotation, and diff --git a/docs/doctoring/strix-nvidia-nim-not-found-fallback.md b/docs/doctoring/strix-nvidia-nim-not-found-fallback.md index a1f2226d8..89d9176ed 100644 --- a/docs/doctoring/strix-nvidia-nim-not-found-fallback.md +++ b/docs/doctoring/strix-nvidia-nim-not-found-fallback.md @@ -32,8 +32,9 @@ fallback produces complete evidence. A completed zero-vulnerability fallback may emit Strix's model-quality banner and Hugging Face's exact unauthenticated-download advisory while loading public -scanner assets. Those two bounded advisories are not provider-failure evidence; -all other warnings remain fail closed. +scanner assets. The trusted gate removes only those two complete lines from both +console and structured report logs. A suffix, prefix, or any other warning text +remains fail-closed provider-failure evidence. Exhausted provider infrastructure remains fail-closed even when the trusted gate has classified every observed threshold finding as outside the pull @@ -59,8 +60,10 @@ Regression evidence proves that: 9. changed, unmapped, and changed-manifest findings also block after provider exhaustion; and 10. the exact model-quality and Hugging Face download advisories do not reject - an otherwise complete zero-vulnerability fallback; and -11. the required-workflow smoke contract pins these properties. + an otherwise complete zero-vulnerability fallback when they appear in + console or report logs; +11. appended warning text is not hidden by either exact advisory filter; and +12. the required-workflow smoke contract pins these properties. ## Limitations diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index 6d3ae9250..1f646454f 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -172,6 +172,11 @@ known_internal_warning = re.compile( r"|ended a turn without a lifecycle tool call \(interactive=False\)" r"); forcing tool continuation \(\d+/\d+\): " ) +known_clean_advisory = re.compile( + r"^(?:[ \t│]*MODEL QUALITY WARNING[ \t│]*" + r"|Warning: You are sending unauthenticated requests to the HF Hub\. " + r"Please set a HF_TOKEN to enable higher rate limits and faster downloads\.)$" +) def iter_report_logs(root: Path): @@ -194,7 +199,12 @@ for log_path in iter_report_logs(root): lines = log_path.read_text(encoding="utf-8").splitlines(keepends=True) except UnicodeDecodeError: continue - filtered = [line for line in lines if not known_internal_warning.match(line)] + filtered = [ + line + for line in lines + if not known_internal_warning.match(line) + and not known_clean_advisory.fullmatch(line.rstrip("\r\n")) + ] if filtered != lines: log_path.write_text("".join(filtered), encoding="utf-8") PY @@ -3200,7 +3210,7 @@ is_llm_token_limit_error() { # below-threshold override from silently passing an aborted scan. has_detected_infrastructure_error() { if grep -Eiq '(^|[^[:alpha:]])(Fatal|Denied|Warn|Warning)([^[:alpha:]]|$)' \ - < <(grep -Eiv '^[[:space:]│]*MODEL QUALITY WARNING[[:space:]│]*$|^Warning: You are sending unauthenticated requests to the HF Hub\.' "$STRIX_LOG"); then + < <(grep -Eiv '^[[:space:]│]*MODEL QUALITY WARNING[[:space:]│]*$|^Warning: You are sending unauthenticated requests to the HF Hub\. Please set a HF_TOKEN to enable higher rate limits and faster downloads\.$' "$STRIX_LOG"); then return 0 fi diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index dc29534c0..a68b403ac 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -3642,6 +3642,11 @@ REPORT nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5) echo "│ MODEL QUALITY WARNING │" echo "Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads." + mkdir -p "$STRIX_REPORTS_DIR/run-clean-advisories" + { + echo "│ MODEL QUALITY WARNING │" + echo "Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads." + } >"$STRIX_REPORTS_DIR/run-clean-advisories/scan.log" echo "│ Vulnerabilities 0 (No exploitable vulnerabilities detected) │" exit 0 ;; @@ -3651,6 +3656,11 @@ REPORT ;; esac ;; + hf-advisory-suffix-fails-closed) + echo "Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads. Fatal: repository-derived suffix" + echo "│ Vulnerabilities 0 (No exploitable vulnerabilities detected) │" + exit 0 + ;; vertex-primary-resource-exhausted-fallback-success) case "${STRIX_LLM:-}" in vertex_ai/resource-exhausted-primary) @@ -6564,6 +6574,16 @@ run_filtered_gate_case_if_requested() { "|" \ "openai" ;; + hf-advisory-suffix-fails-closed) + run_gate_case "$STRIX_TEST_CASE_FILTER" \ + "vertex_ai/hf-advisory-suffix-fails-closed" \ + "" \ + "1" \ + "Strix run emitted provider infrastructure or failure-signal output; failing closed." \ + "1" \ + "vertex_ai/hf-advisory-suffix-fails-closed" \ + "" + ;; provider-fatal-success-signal | provider-warning-success-signal) run_gate_case "$STRIX_TEST_CASE_FILTER" \ "vertex_ai/$STRIX_TEST_CASE_FILTER" \ @@ -9913,6 +9933,15 @@ run_gate_case "nvidia-ratelimit-model-quality-warning-fallback-success" \ "|" \ "openai" +run_gate_case "hf-advisory-suffix-fails-closed" \ + "vertex_ai/hf-advisory-suffix-fails-closed" \ + "" \ + "1" \ + "Strix run emitted provider infrastructure or failure-signal output; failing closed." \ + "1" \ + "vertex_ai/hf-advisory-suffix-fails-closed" \ + "" + run_gate_case_allow_provider_signal "vertex-primary-resource-exhausted-fallback-success" \ "vertex_ai/resource-exhausted-primary" \ "vertex_ai/fallback-one vertex_ai/fallback-two" \ From 5ab583417053a122bd568e8295d3c25172dbac52 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 22:57:17 +0900 Subject: [PATCH 14/15] fix(strix): normalize the fallback model before dispatch, not just credentials run_strix_once() computed normalized_model only for credential/API-base selection (is_vertex_model, is_github_models_model, and the openai_direct/* fallback-key case), but called child_model_for_api_base() with the raw, un-normalized $model. A hyphenated STRIX_FALLBACK_MODELS entry (openai-direct/gpt-5.6-luna) therefore reached child_model_for_api_base unrewritten -- its own openai_direct/* case never matched a hyphen -- and was dispatched to LiteLLM verbatim as an unrecognized provider string, even though normalize_model() (already fixed in this branch) and child_model_for_api_base() both handle the alias correctly on their own. Observed live on this PR's own strix check: the run advanced through two NVIDIA NIM rate limits to the openai-direct/gpt-5.6-luna fallback, the log recorded "model=openai-direct/gpt-5.6-luna" verbatim, and litellm.BadRequestError: LLM Provider NOT provided ended the scan. Move the normalized_model computation before the child_model_for_api_base call and reuse it there, instead of computing it a second time afterward for credentials only. New regression test extracts run_strix_once's own two-statement model/API-base resolution call site verbatim (rather than reimplementing the composition) so a future call site edit that stops normalizing before dispatch fails this test, not just live NVIDIA NIM traffic. --- scripts/ci/strix_quick_gate.sh | 6 +- ...est_strix_nvidia_nim_not_found_fallback.py | 92 +++++++++++++++++++ 2 files changed, 95 insertions(+), 3 deletions(-) diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index 1f646454f..604f6b657 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -2523,15 +2523,15 @@ run_strix_once() { if ! llm_api_base_value="$(resolved_llm_api_base_for_model "$model")"; then return 2 fi - child_model="$(child_model_for_api_base "$model" "$llm_api_base_value")" + local normalized_model + normalized_model="$(normalize_model "$model")" + child_model="$(child_model_for_api_base "$normalized_model" "$llm_api_base_value")" if ! resolved_target_path="$(resolve_current_target_path "$TARGET_PATH")"; then return 1 fi local start_epoch start_epoch="$(date +%s)" local child_llm_api_key="" - local normalized_model - normalized_model="$(normalize_model "$model")" if ! is_vertex_model "$normalized_model"; then child_llm_api_key="$LLM_API_KEY" if is_github_models_model "$normalized_model" && [ -n "$STRIX_GITHUB_MODELS_KEY" ]; then diff --git a/tests/test_strix_nvidia_nim_not_found_fallback.py b/tests/test_strix_nvidia_nim_not_found_fallback.py index 990269725..38906f5f6 100644 --- a/tests/test_strix_nvidia_nim_not_found_fallback.py +++ b/tests/test_strix_nvidia_nim_not_found_fallback.py @@ -17,6 +17,7 @@ REPOSITORY_ROOT = Path(__file__).resolve().parents[1] STRIX_GATE = REPOSITORY_ROOT / "scripts" / "ci" / "strix_quick_gate.sh" +STRIX_MODEL_UTILS = REPOSITORY_ROOT / "scripts" / "ci" / "strix_model_utils.sh" STRIX_WORKFLOW = REPOSITORY_ROOT / ".github" / "workflows" / "strix.yml" DEFAULT_NVIDIA_MODEL = "nvidia_nim/nvidia/nemotron-3-super-120b-a12b" FREE_NVIDIA_FALLBACK = ( @@ -73,6 +74,64 @@ def _classifies_as_nvidia_not_found(log_text: str) -> bool: return completed.returncode == 0 +def _run_strix_once_resolves_child_model(model: str, primary_model: str) -> str: + """Execute run_strix_once's own model/API-base resolution lines verbatim. + + Extracts the exact two statements `run_strix_once` uses to compute the + model it hands to LiteLLM, rather than reimplementing that composition, + so a future call-site edit that stops normalizing the workflow-facing + alias before dispatch fails this test instead of only failing in CI + against live NVIDIA NIM traffic. + """ + + gate_source = STRIX_GATE.read_text(encoding="utf-8") + call_site_match = re.search( + r'(?m)^\tif ! llm_api_base_value=.*\n' + r'(?:.*\n)+?' + r'\tchild_model="\$\(child_model_for_api_base [^\n]+\)"\n', + gate_source, + ) + if call_site_match is None: + raise AssertionError( + "missing run_strix_once model/API-base resolution call site" + ) + function_source = "\n".join( + _function_block(gate_source, name) + for name in ( + "is_vertex_model", + "is_github_models_api_base", + "is_github_models_model", + "is_github_models_api_compatible_model", + "resolved_llm_api_base_for_model", + "child_model_for_api_base", + ) + ) + script = "\n".join( + ( + "set -euo pipefail", + STRIX_MODEL_UTILS.read_text(encoding="utf-8"), + function_source, + 'PRIMARY_MODEL="$2"', + 'LLM_API_BASE_FILE=""', + "run_strix_once_child_model() {", + 'local model="$1"', + "local llm_api_base_value", + "local child_model", + call_site_match.group(0), + 'printf %s "$child_model"', + "}", + 'run_strix_once_child_model "$1"', + ) + ) + completed = subprocess.run( + ["bash", "-c", script, "strix-run-once", model, primary_model], + check=True, + capture_output=True, + text=True, + ) + return completed.stdout.strip() + + def _workflow_signal_pattern(workflow: str, variable_name: str) -> str: """Extract one single-quoted POSIX ERE assigned in the Strix workflow.""" @@ -213,6 +272,39 @@ def test_workflow_uses_available_free_first_nvidia_plan(self) -> None: )[0] self.assertNotIn(RETIRED_PRIMARY_MODEL, default_gate) + def test_run_strix_once_normalizes_hyphenated_fallback_before_dispatch( + self, + ) -> None: + """Dispatch the NIM-exhaustion fallback as a real LiteLLM provider. + + `normalize_model()` (`scripts/ci/strix_model_utils.sh`) already + rewrites the workflow-facing `openai-direct/` alias to LiteLLM's + `openai_direct/` provider prefix, and `child_model_for_api_base()` + already rewrites `openai_direct/*` to `openai/*`. But + `run_strix_once()` calls `child_model_for_api_base` with the raw, + un-normalized `$model` -- so a hyphenated `STRIX_FALLBACK_MODELS` + entry still reaches LiteLLM unrewritten. Observed live: NVIDIA NIM + rate-limited the primary and first fallback model, the run advanced + to `openai-direct/gpt-5.6-luna`, and the scan log recorded + `model=openai-direct/gpt-5.6-luna` verbatim before + `litellm.BadRequestError: LLM Provider NOT provided` ended the run. + """ + + self.assertEqual( + _run_strix_once_resolves_child_model( + "openai-direct/gpt-5.6-luna", + DEFAULT_NVIDIA_MODEL, + ), + "openai/gpt-5.6-luna", + ) + self.assertEqual( + _run_strix_once_resolves_child_model( + "openai_direct/gpt-5.6-luna", + DEFAULT_NVIDIA_MODEL, + ), + "openai/gpt-5.6-luna", + ) + def test_outer_workflow_requires_litellm_context_for_nvidia_404(self) -> None: """Reject provider-like target text in the outer neutralization gate.""" From 8d0f7ca88f6e0c9db8e2363721fbeaa59acebd6a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 23:10:49 +0900 Subject: [PATCH 15/15] fix(strix): ignore exact optional web search advisory --- scripts/ci/strix_quick_gate.sh | 6 +++ scripts/ci/test_strix_quick_gate.sh | 58 +++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+) diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index 604f6b657..14acd60d5 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -177,6 +177,11 @@ known_clean_advisory = re.compile( r"|Warning: You are sending unauthenticated requests to the HF Hub\. " r"Please set a HF_TOKEN to enable higher rate limits and faster downloads\.)$" ) +known_optional_web_search_advisory = re.compile( + r"^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d+ WARNING " + r"[^ ]+ - strix\.tools\.web_search\.tool: " + r"web_search invoked without PERPLEXITY_API_KEY configured$" +) def iter_report_logs(root: Path): @@ -204,6 +209,7 @@ for log_path in iter_report_logs(root): for line in lines if not known_internal_warning.match(line) and not known_clean_advisory.fullmatch(line.rstrip("\r\n")) + and not known_optional_web_search_advisory.fullmatch(line.rstrip("\r\n")) ] if filtered != lines: log_path.write_text("".join(filtered), encoding="utf-8") diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index a68b403ac..be20b971c 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -4613,6 +4613,15 @@ EOS echo "scan ok with sanitized internal Strix report notice variant" exit 0 ;; + report-web-search-advisory-sanitized | report-web-search-advisory-suffix-fails) + mkdir -p "$STRIX_REPORTS_DIR/fake-web-search-advisory" + cat >"$STRIX_REPORTS_DIR/fake-web-search-advisory/strix.log" <"$STRIX_REPORTS_DIR/fake-unknown-warning/strix.log" <<'EOS' @@ -5930,6 +5939,17 @@ PY "scenario=$scenario keeps non-warning Strix report evidence" fi + if [ "$scenario" = "report-web-search-advisory-sanitized" ]; then + assert_file_not_contains \ + "$repo_root_dir/strix_runs/fake-web-search-advisory/strix.log" \ + "web_search invoked without PERPLEXITY_API_KEY configured" \ + "scenario=$scenario strips only the known optional web search advisory" + assert_file_contains \ + "$repo_root_dir/strix_runs/fake-web-search-advisory/strix.log" \ + "finish_scan: completed scan with 0 vulnerability report(s)" \ + "scenario=$scenario keeps non-warning Strix report evidence" + fi + if [ "$scenario" = "github-models-primary-ratelimit-fallback-success" ]; then assert_file_contains \ "$output_log" \ @@ -6563,6 +6583,26 @@ run_filtered_gate_case_if_requested() { "vertex_ai/report-known-internal-warning-sanitized" \ "" ;; + report-web-search-advisory-sanitized) + run_gate_case "$STRIX_TEST_CASE_FILTER" \ + "vertex_ai/report-web-search-advisory-sanitized" \ + "" \ + "0" \ + "Strix run succeeded for model 'vertex_ai/report-web-search-advisory-sanitized'" \ + "1" \ + "vertex_ai/report-web-search-advisory-sanitized" \ + "" + ;; + report-web-search-advisory-suffix-fails) + run_gate_case "$STRIX_TEST_CASE_FILTER" \ + "vertex_ai/report-web-search-advisory-suffix-fails" \ + "" \ + "1" \ + "Strix report artifacts emitted warning/fatal/denied/timeout output; failing closed." \ + "1" \ + "vertex_ai/report-web-search-advisory-suffix-fails" \ + "" + ;; nvidia-ratelimit-model-quality-warning-fallback-success) run_gate_case "nvidia-ratelimit-model-quality-warning-fallback-success" \ "nvidia_nim/nvidia/nemotron-3-super-120b-a12b" \ @@ -10698,6 +10738,24 @@ run_gate_case "report-known-internal-warning-variant-sanitized" \ "" \ "1" +run_gate_case "report-web-search-advisory-sanitized" \ + "vertex_ai/report-web-search-advisory-sanitized" \ + "" \ + "0" \ + "Strix run succeeded for model 'vertex_ai/report-web-search-advisory-sanitized'" \ + "1" \ + "vertex_ai/report-web-search-advisory-sanitized" \ + "" + +run_gate_case "report-web-search-advisory-suffix-fails" \ + "vertex_ai/report-web-search-advisory-suffix-fails" \ + "" \ + "1" \ + "Strix report artifacts emitted warning/fatal/denied/timeout output; failing closed." \ + "1" \ + "vertex_ai/report-web-search-advisory-suffix-fails" \ + "" + run_gate_case "report-unknown-warning-fails" \ "vertex_ai/report-unknown-warning-fails" \ "" \