diff --git a/.env.example b/.env.example index e494300..cd055af 100644 --- a/.env.example +++ b/.env.example @@ -2,18 +2,19 @@ # cb-testing Docker image overrides # --------------------------------------------------------------------------- # Copy this file to .env and customize for your local images. -# All values have defaults in generate_kurtosis_configs.py, so this file -# is entirely optional — only needed when using custom Docker images. +# All values have baked defaults in `sim generate` (Images::default), so this +# file is entirely optional — only needed when using custom Docker images. # --------------------------------------------------------------------------- -# Helix relay image (your custom relay build) -# Default: helix-relay:kurtosis -# HELIX_RELAY_IMAGE=helix-relay:kurtosis +# Helix relay image +# Default: ghcr.io/gattaca-com/helix-relay:main +# HELIX_RELAY_IMAGE=ghcr.io/gattaca-com/helix-relay:main -# mev-boost relay image (used in multi-relay scenarios) +# flashbots mev-boost-relay image — NO LONGER USED as a relay (multi-relay now runs +# two helix instances; see docs/fork-delta.md). Still emitted into configs but inert. # Default: ethpandaops/mev-boost-relay:main # MEV_RELAY_IMAGE=ethpandaops/mev-boost-relay:main -# Commit-Boost PBS image (sub latest for your local build) -# Default: commit-boost/commit-boost:latest -# MEV_BOOST_IMAGE=commit-boost/commit-boost:latest +# Commit-Boost sidecar image (sub your local `just build-all ` build) +# Default: commit-boost/commit-boost:kurtosis +# MEV_BOOST_IMAGE=commit-boost/commit-boost:kurtosis diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 4cac47b..a4c1ae8 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -14,16 +14,34 @@ on: description: 'Observation window in epochs' required: false default: '1' - -env: - ENCLAVE: cb-ci-${{ github.run_id }} + cb_image: + description: 'Commit-Boost PBS image (MEV_BOOST_IMAGE). Override to pin a specific tag/digest.' + required: false + default: 'ghcr.io/commit-boost/pbs:latest' jobs: verify: - name: devnet + verify + name: devnet + verify (${{ matrix.scenario }}) + # RUNNER MEMORY: a full devnet (geth+lighthouse + helix relay(s) + reth-rbuilder + # + commit-boost + dora/spamoor/prometheus) needs ~15-20GB; cb-mux runs TWO + # helix instances (8GB cap each) → ~25GB+. A GitHub-hosted ubuntu-latest has + # only ~7GB RAM, so this nightly requires a LARGE or SELF-HOSTED runner. Change + # runs-on accordingly before relying on it (cb-mux will OOM on the hosted runner). runs-on: ubuntu-latest timeout-minutes: 90 + strategy: + # Don't cancel cb-mux just because cb-basic failed (or vice versa); we + # want independent e2e signal from each scenario every night. + fail-fast: false + matrix: + scenario: [cb-basic, cb-mux] + + env: + # One enclave per (run, scenario) so matrix legs never collide and each + # leg's Teardown removes exactly the enclave it launched. + ENCLAVE: cb-ci-${{ github.run_id }}-${{ matrix.scenario }} + steps: - name: Checkout uses: actions/checkout@v6 @@ -56,21 +74,44 @@ jobs: kurtosis analytics disable echo "$(dirname $(which kurtosis))" >> $GITHUB_PATH + - name: Generate ${{ matrix.scenario }} config + # The Python generator + its checked-in example config were retired (P2); + # `sim generate` is now the source. The nightly builds no commit-boost + # image, so it pins the public CB image via .env (MEV_BOOST_IMAGE). + # + # REPRODUCIBILITY GAP: the default below is a moving `:latest` tag, so a + # replay can silently pick up a different PBS binary. commit-boost-client + # is NOT checked out here (only ethereum-package is a submodule), so we + # can't build the image from `main` in-workflow without adding that repo. + # For a reproducible run, override the `cb_image` workflow_dispatch input + # with a pinned tag/digest, e.g. + # ghcr.io/commit-boost/pbs@sha256: + # TODO: promote a pinned digest to the default here once the SSZ-current + # PBS image is confirmed (see the review NOTE this replaced), so the + # scheduled nightly is reproducible too, not just manual dispatches. + env: + # On schedule there are no inputs, so fall back to the pinned default. + CB_IMAGE: ${{ github.event.inputs.cb_image || 'ghcr.io/commit-boost/pbs:latest' }} + run: | + echo "MEV_BOOST_IMAGE=${CB_IMAGE}" > .env + cargo run --bin sim -- generate ${{ matrix.scenario }} + - name: Launch devnet + verify timeout-minutes: 60 run: | ./scripts/run-and-verify.sh \ - --config configs/example-kurtosis-config.yml \ + --enclave "$ENCLAVE" \ + --config configs/generated/${{ matrix.scenario }}.yml \ --json \ --live-metrics \ --min-epochs "${{ github.event.inputs.min_epochs }}" \ - --target-epoch "${{ github.event.inputs.target_epoch }}" \ + --target-epoch "${{ github.event.inputs.target_epoch }}" - name: Upload artifacts if: always() uses: actions/upload-artifact@v5 with: - name: verify-${{ github.run_id }} + name: verify-${{ matrix.scenario }}-${{ github.run_id }} path: | verify-report.json kurtosis-run.log diff --git a/.gitignore b/.gitignore index 4bc7c38..080eb2a 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,6 @@ kurtosis-configs/ scripts/kurtosis-configs/ .env */__pycache__ + +CB-Testnet.json +.agent/ diff --git a/.gitmodules b/.gitmodules index bea06ae..6f95033 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "ethereum-package"] path = ethereum-package - url = https://github.com/JasonVranek/ethereum-package.git + url = https://github.com/Commit-Boost/ethereum-package.git diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..ce1e845 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,275 @@ +# cb-testing - agent orientation + +An **opinionated block-building simulation substrate for Commit-Boost**. It stands up a real Ethereum +devnet (helix relay + reth-rbuilder builder + the CB sidecar) via Kurtosis, exercises one PBS feature end +to end, and returns a **trustworthy verdict**. The point is not "the devnet booted"; it is a verdict you +can gate a release on. Nothing here is a mock. + +Rust workspace (one lib, three bins): + +| Target | Path | Role | +|---|---|---| +| `cb_testnet_verifier` (lib) | `src/lib.rs` | shared: `beacon`, `relay`, `metrics`, `discovery`, `checks/*`, `report` | +| `cb-verify` (bin, default-run) | `src/main.rs` | the verifier: discover -> observe -> run checks -> report -> exit code | +| `sim` (bin) | `src/bin/sim/main.rs` | `generate` \| `preflight` \| `triage` \| `checks` \| `doctor` \| `diff` | +| `cb-orchestrator` (bin) | `src/orchestrator.rs` | parallel multi-enclave runner (`just test-all`) | + +## Authoritative docs (read these first, this file is the ROUTER) +- **docs/DESIGN.md** - WHY the repo exists + **the 7 design laws** (each prevents a named smell). + Read when a change feels like it is fighting the grain. Law 1 real-schema configs, Law 3 every scenario + asserts its feature FIRED, Law 4 verdict logic is TDD-able without a devnet, Law 5 observability is a + system property (never build an agent-only tool surface), Law 7 coverage is a matrix (EL/CL pairs). +- **docs/ARCH.md** - HOW it fits: end-to-end flow, module map, the config <-> fork seam, the verdict model. +- **docs/CHECKS.md** - the authoritative **per-check contract** (tier, source, pass/warn/fail/skip conditions). +- **docs/DEVELOPING.md** - the dev loop + how to add a check / a scenario. +- **docs/fork-delta.md** - what our `ethereum-package` fork changes vs upstream, file by file. +- **docs/local-kurtosis-e2e.md** - the operational runbook + the paid-for incidents behind half the design. +- **README.md** - user-facing quick start (user-facing quick start). + +Internal back-room (agent working material, not part of the public docs surface): +> `.agent/` is a **local, gitignored working area** — present when you develop against this repo, NOT in a fresh public clone. Agents write working notes here; nothing in it ships. + +- **.agent/SWEEP-BACKLOG.md** - the **live** queue AND the findings log (every live-devnet result gets recorded + there). Highest-density source of hard-won facts; read the tail before starting anything. +- **.agent/plans/INDEX.md** - status of every plan (`live` steers, `landed` is history). +- **.agent/NORTH-STAR.md** - the internal staged plan (P0-P5), ratified directions, and open scars behind DESIGN. + +## Runtime facts +- **Rust 1.91+, edition 2024.** No Node, no Python. `cargo test` is pure, hermetic, seconds (no docker). +- **Kurtosis CLI pinned to 1.18.1.** The parsers read its human text tables (1.18.1 has no `--format json` + for them) and a newer CLI writes a `config-version: 9` `~/.config/kurtosis/kurtosis-config.yml` that + 1.18.1 cannot read. `sim doctor` checks this for you. +- **No Kurtosis Rust SDK exists.** All enclave ops are `std::process::Command` + text parsing + (`discovery.rs`, `triage.rs`). Inherent, not a rewrite candidate. +- `sim` is deliberately **sync / no tokio** (it shells out and blocks); `cb-verify` and `cb-orchestrator` + are tokio (concurrent HTTP polling / parallel enclaves). +- **The CB sidecar image is BUILT, not pulled** (`just build-cb-image` -> `commit-boost/commit-boost:kurtosis` + from the sibling `../commit-boost-client`). helix / reth-rbuilder / lighthouse are public pulls. + +## USING it + +`sim` is a workspace bin, not a PATH command: invoke it as `cargo run --quiet --bin sim -- ` or, after +`cargo build --release`, as `./target/release/sim `. Written `sim ` below for brevity. + +```bash +sim doctor # host preflight: kurtosis 1.18.1, docker, memory, CB image, submodule +git submodule update --init # the forked ethereum-package (empty otherwise; load-bearing) +just build-cb-image # once: builds commit-boost/commit-boost:kurtosis from ../commit-boost-client +just e2e # cb-basic, end to end +just e2e configs/generated/cb-mux.yml # any scenario +``` + +`just e2e` = `generate-configs` + `pull-images` + `just testnet `, which calls the launcher: + +```bash +./scripts/run-and-verify.sh --config configs/generated/cb-basic.yml \ + --json --live-metrics --min-epochs 1 --target-epoch 1 --keep --skip-finalization -v +``` +Launcher flags (all real, `scripts/run-and-verify.sh`): `--config --enclave --package --keep --json +--json-dir DIR --strict --live-metrics --skip-finalization --timeout --min-epochs --target-epoch -v`. +It (0) pre-builds `cb-verify` OFF the critical path, (1) removes the stale enclave, (1b) runs +`sim preflight` as a gate, (1c) warns on low host memory (`LOW_MEM_ABORT=1` to abort instead), +(2) `kurtosis run` (auto-fires `sim triage` on launch failure), (3) runs the pre-built `cb-verify`. +With `--json` and no `--json-dir` it saves the report to `/.json`. + +`cb-verify` direct (`src/main.rs`): `--enclave --config --min-epochs --target-epoch --timeout +--mev-threshold --json --verbose/-v --strict --live-metrics --show-logs --skip-finalization-check +--output-dir`. Note the launcher's flag is `--skip-finalization`, the binary's is `--skip-finalization-check`. + +Against an already-running enclave (no observation window): `just verify-now `, +`just verify-with-config `, `just test-mux `, `just show-logs `. + +### Scenarios and config generation +`configs/generated/*.yml` are **gitignored build products** of the typed generator; the tracked truth is +`tests/fixtures/golden-configs/`. Nine scenarios (`Scenario::ALL`, `src/bin/sim/genmodel/scenario.rs`): +`cb-basic`, `cb-basic-nethermind-prysm` (Law 7 alt EL/CL pair), `cb-multiple-relays` (two helix instances, +divergent per-relay subsidies), `cb-skip-sigverify`, `cb-sigverify-diff` + `cb-sigverify-diff-control` +(a real ON/OFF differential), `cb-timing-games`, `cb-extra-validation`, `cb-mux` (256 validators split +across two relays). + +```bash +sim generate # all nine -> configs/generated/ (= just generate-configs) +sim generate cb-mux --out-dir /tmp/x +sim generate --check # drift gate: nonzero if on-disk configs != what the generator emits +sim preflight configs/generated/cb-mux.yml # ~1s: parse the rendered config with the REAL helix image +sim checks --list [--json] # the check contract, machine-readable +sim diff a.json b.json [--json] # verdict/provenance regression gate between two reports +sim triage # each dead service's ROOT panic, as JSON +sim --log-format json # structured event stream for agents (default: pretty) +``` +`.env` (gitignored, see `.env.example`) overrides the embedded docker images and is read only at the +`sim generate` CLI boundary, so assembly stays pure. + +## THE VERDICT MODEL (load-bearing) +A run emits a `VerificationReport` (`src/report.rs`) plus an exit code. Each `CheckResult` has `id`, +`tier` (1/2/3), `result` (`PASS`/`FAIL`/`WARN`/`SKIP`), `detail`, `data`. + +**The exit code keys ONLY on a tier-1 FAIL** (`report::exit_code`): `2` = no tier-1 check ran at all +(setup/discovery failure), `1` = some tier-1 check FAILed, `0` otherwise. `WARN` and `SKIP` are +**non-fatal at every tier** and never move the exit code or the overall result. + +**The consequence a consumer MUST internalize:** several checks that exist precisely to catch an anomaly +report it as `WARN` (relay equivocation in `payload_hash_match`, unverifiable routing in `mux.routing`, +a best-bid shortfall in `relay.best_bid`). A run that hits them **still exits 0**. Parse the JSON and +inspect each check's `result`; never gate on the exit code alone. `--strict` promotes selected soft +warnings to FAIL. + +**Escalation:** `cb_metrics` checks whose id ends in `_matrix`, plus `cb_relay_v2_unsupported`, are +authored at tier 2 but **promoted to tier 1 when they FAIL** (`src/checks/cb_metrics.rs`), because a real +relay 5xx or a lost blinded-block submission is a genuine pipeline failure. So a matrix FAIL does gate +the exit code despite its nominal tier. + +Per-check contract, data sources, and death-mode behavior: **docs/CHECKS.md**. `sim preflight` has its own +narrower 3-valued verdict (`Pass` / `Fail{field}` / `Inconclusive`) and only its `Fail` aborts a launch. + +## DEBUGGING with it (the highest-value section) +**The method that has repeatedly paid off here: get the RAW evidence before believing a check's summary.** +Every misdiagnosis in this repo's history was a check's `detail` string read as fact. `data` is closer to +truth than `detail`; raw logs and raw counters are closer still. + +```bash +sim doctor # is it me or the box? +sim preflight # ~1s real-image config parse, BEFORE a ~10min spend +sim triage # broken enclave -> per-service root panic (skips masking lines, + # falls back to `docker logs` when kurtosis's broker masks it) +kurtosis enclave ls # ALWAYS check for a live run before touching anything +kurtosis enclave inspect # services, statuses, mapped ports +kurtosis service logs -n 200000 | sed 's/\x1b\[[0-9;]*m//g' +kurtosis port print +``` +Run with `--keep` (the `just testnet` default) so the enclave survives for inspection; tear down explicitly +with `kurtosis enclave rm -f `. + +- **Logs need ANSI stripping.** CB logs are colored; the parsers use `strip_ansi` internally, you need the + `sed` above by hand. Services are `commit-boost-001`, `helix-relay-N`, `el-{i}-{el}-{cl}`, + `cl-{i}-{cl}-{el}` (the EL/CL pair is in the name; see Law 7). +- **Scrape CB's Prometheus counters directly for the RAW per-code view.** The report BUCKETS status codes + (2xx/4xx/5xx/timeout/transport), which hides which exact code fired - that bucketing is what produced a + "47.5% relay 5xx" panic that was entirely CB's synthetic 555. Get the truth: + ```bash + kurtosis service exec commit-boost-001 "curl -s http://localhost:9090/metrics" \ + | grep -E 'cb_pbs_relay_status_code_total|cb_pbs_beacon_node_status_code_total' + ``` + Other counters worth reading raw: `pbs_submit_block_v2_unsupported_total`, + `cb_pbs_submit_block_v2_fallback_to_v1_total`, `cb_pbs_relay_latency`. +- **Read the report JSON, not the terminal render**: `.json` at the repo root after a `--json` run + (e.g. `CB-Testnet.json`). Then `sim diff old.json new.json` for the delta. +- **Cross-check the two sides of a matrix.** `relay_side` vs `beacon_side` in a `cb_*_matrix` check's `data` + is what distinguishes "the relay rejected us" from "the CL never asked": relay 4xx + beacon 5xx means CB + forwarded and the relay refused, then returned 502 to the CL. +- Middle ground before a full run: `just verify-now`, `just test-mux`, `sim preflight`. **A ~10-minute e2e is + the final confirmation, never the debugger.** + +## Known traps (hard-won; do not re-derive) + +**The signer runs as uid 10001 and the devnet's `secrets/` dir is mode 600 root-owned.** Kurtosis does +NOT chown files-artifacts on mount, so a non-root container cannot even traverse it, and CB's keystore +loaders skip unreadable entries with `warn!` rather than failing - producing a healthy signer holding +ZERO keys. Use the `teku-keys` + `teku-secrets` pair (755/777), which is also what the package's own +web3signer launcher relies on. Verified live; six package launchers force `User(uid=0)` for the same +reason. + +**Never let a grep's SILENCE mean success.** `cargo test ... | grep "test result:"` prints nothing +when the build fails, which reads identically to "no output, fine". A broken test suite was committed +this way once. Gate on the EXIT CODE, not on matched lines: +```bash +set -o pipefail +cargo test --all-targets 2>&1 | grep -E "^test result:"; echo "TEST_EXIT=$?" +cargo clippy --all-targets -- -D warnings >/dev/null 2>&1; echo "CLIPPY_EXIT=$?" +cargo fmt --check >/dev/null 2>&1; echo "FMT_EXIT=$?" +``` +This is the same defect class the harness keeps finding in itself: a check that cannot distinguish +"no signal" from "bad signal". Apply it to devnet runs too - an empty log tail is not a passing run. + +- **CB's synthetic status codes 555 and 556 are NOT relay-served.** 555 = `TIMEOUT_ERROR_CODE`, CB's own + client-side deadline cancellation; 556 = WS transport error. They bucket separately (`timeout`, + `transport`) and are excluded from the 5xx denominator. Counting them as relay 5xx tier-1-FAILed a fully + green timing-games run. High rates WARN, never FAIL, not even under `--strict`. +- **helix `router_config.enabled_routes` gates routes, and a missing route looks like a client bug.** + `GetPayloadV2` was absent from our generated helix config, so helix 404'd `/eth/v2/builder/blinded_blocks`, + CB (correctly) refused to downgrade to v1, and prysm's entire MEV path died. Hours were spent suspecting + nethermind/prysm; it was OUR config. `cb_relay_v2_unsupported` now names it in one line. +- **A check's `detail` string can be flat wrong.** `cb_submit_blinded_block_matrix` reported "proposer never + chose a builder block" while the relay had rejected 26 blinded blocks - the opposite of reality. Fixed, but + the lesson generalizes: **trust `data`, verify against raw logs and counters.** A diagnosis naming the + wrong component is worse than no diagnosis. +- **Kurtosis pinned at 1.18.1**; a newer CLI's `config-version: 9` config file breaks it (delete the file on + downgrade, then `kurtosis analytics disable`). +- **Pass `kurtosis run` an ABSOLUTE package path from a script.** `run-and-verify.sh` resolves + `$REPO_DIR/ethereum-package`; a relative `./ethereum-package` resolves against the caller's cwd and fails + with a confusing "no kurtosis.yml" error. +- **`ethereum-package/` is a FORK** (`Commit-Boost/ethereum-package`, branch `cb-testing`), pinned as a + detached-HEAD submodule and load-bearing (empty without `--init`). Changes there must be **pushed** or + every other clone breaks. There is no `upstream` remote configured. See docs/fork-delta.md. +- **`configs/generated/` is gitignored, but `src/bin/sim/render.rs` and `genmodel/scenario.rs` + `include_str!` `configs/generated/cb-basic.yml` at COMPILE time** (a code comment even calls it "TRACKED"; + it is not). A fresh clone therefore cannot build `sim` until that file exists, and `sim` is the generator. + Recovery: `cp tests/fixtures/golden-configs/cb-basic.yml configs/generated/` (verified byte-identical), + then `just generate-configs`. +- **helix's config schema drifts with the moving `:main` tag.** The source of truth is the running binary's + serde metadata, not any checked-in mirror. Reconcile by parsing against the real image (that is exactly + what `sim preflight` is), never by editing to match a local helix checkout. +- **NEVER pattern-kill processes.** No `pkill -f` / `killall`: the pattern matches your own shell and you + kill your session. Read the PID (`ps -eo pid,args | grep `) and kill by explicit number. +- **A devnet may be live on this box.** `kurtosis enclave ls` first; never `kurtosis clean -a` or remove an + enclave you did not create. + +## DEVELOPING +**The gates, all four, before any commit** (mirror of CI plus the drift gate): +```bash +cargo test --all-targets && cargo clippy --all-targets -- -D warnings && cargo fmt --check +sim generate --check # config-generation drift gate (NOT in CI today; run it yourself) +``` +`just ci` = `check` + `test` + `lint`. Green `just ci` says nothing about the devnet path. + +**Adding a check - the Law-4 pure-`classify_*` seam (non-negotiable).** Split every judgement into +(a) a **pure classifier** taking already-fetched data and returning a `CheckResult`, unit-tested on **both +sides of every boundary**, and (b) a **thin async fetch shell** holding zero verdict logic. Worked examples: +`cb_metrics::{collect_endpoint_stats, classify_endpoint}` (the cleanest seam in the repo), +`feature_fired::{classify_marker_feature, classify_skip_sigverify}`, `payload_matching::classify_payload_matches`, +`best_bid::classify_best_bid`, `mux_routing::classify_mux_routing`. Then wire the fetch fn into +`run_verification` in `src/main.rs`. The anti-pattern is welding the verdict into the `await` path: +`chain_health` and `relay_pipeline` still do this and are the standing gap. **Pick the tier deliberately** +(it is the severity contract) and prefer **WARN over a silent PASS** when the check could not actually +verify anything - that is how false-greens ship. Full recipe: docs/DEVELOPING.md §2. + +**Adding a scenario:** add the variant to `Scenario` + `Scenario::ALL` in `src/bin/sim/genmodel/scenario.rs`, +fill `name()`/`comment()`/`relays()`/`cb_block()`/`network_params()`, build the CB TOML via `CbParams` + +`cb_toml` in `genmodel/cb.rs` (the helix block in `genmodel/helix.rs` is shared and byte-identical across +scenarios), run `just generate-configs`, then copy the output into `tests/fixtures/golden-configs/`. +**The golden-fixture byte-identity test is the oracle** (`every_scenario_matches_its_golden`): it proves the +generator reproduces its own output deterministically, NOT that the config is a working devnet - only a real +run proves that. And **Law 3: ship the scenario with a check that positively asserts its feature FIRED** +(see `src/checks/feature_fired.rs`), or it is a non-test. Full recipe: docs/DEVELOPING.md §3. + +**Where things live:** `src/checks/` (the verdicts), `src/bin/sim/` (generate/preflight/triage/checks/ +doctor/diff + `genmodel/`), `src/report.rs` (report shape + `exit_code`), `src/discovery.rs` (kurtosis text +parsing), `tests/fixtures/` (log fixtures + golden configs), `scripts/run-and-verify.sh` (the launcher). + +## Documentation discipline (KEEP THIS FILE UPDATED) +**If you change behavior, flags, check ids/tiers/verdicts, scenario names, or the config-generation contract, +you MUST update this file AND its companion doc IN THE SAME COMMIT:** + +| You changed | Also update, same commit | +|---|---| +| a check's id, tier, verdict conditions, or data source | `docs/CHECKS.md` **and** `src/bin/sim/checks_catalog.rs` | +| a CLI flag, a `just` target, the launcher | this file + `README.md` | +| a scenario, the generator, the helix/CB blocks | `tests/fixtures/golden-configs/` (regenerate) + this file's scenario list | +| a design law | `docs/DESIGN.md` | +| a ratified direction, a staged-plan status | `.agent/NORTH-STAR.md` | +| a live-devnet result or a new defect | `.agent/SWEEP-BACKLOG.md` (findings log) | +| a plan shipped | `.agent/plans/INDEX.md` (`live` -> `landed`) | + +**`sim checks --list` is a hand-maintained static catalog with NO compiler enforcement** (`checks_catalog.rs` +says so at the top): the checks are constructed imperatively across several modules with no registry to +reflect on, so drift is caught only by review. This is not hypothetical - `cb_relay_v2_unsupported` shipped +in `src/checks/cb_metrics.rs` while missing from BOTH `checks_catalog.rs` and `docs/CHECKS.md`, in the very +commit that added it (caught on the next read, now synced). Adding a check is a THREE-file +change. If a real registry ever lands in `src/checks`, derive the catalog from it and delete this note. + +**New hard-won gotchas belong in the "Known traps" section above**, with the evidence that bought them. +A trap that lives only in a commit message will be paid for twice. + +## Coding preferences +No em dashes in prose, plain language, hypothesis-driven iteration, many small files. Pure cores with tests +on both sides of every boundary; thin IO shells around them. diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..8b7cbf4 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1 @@ +See [AGENTS.md](AGENTS.md). diff --git a/Cargo.lock b/Cargo.lock index 24d22d8..f1ca0f2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -675,17 +675,22 @@ version = "0.1.0" dependencies = [ "alloy-primitives", "alloy-rpc-types-beacon", + "base64", "chrono", "clap", "color-eyre", "colored", "eyre", + "futures", + "hmac", "prometheus-parse", "reqwest", "serde", "serde_json", "serde_yaml", + "sha2", "tokio", + "toml", "tracing", "tracing-subscriber", ] @@ -1222,35 +1227,90 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" +[[package]] +name = "futures" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a88cf1f829d945f548cf8fec32c61b1f202b6d93b45848602fc02af4b12ad218" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + [[package]] name = "futures-channel" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +checksum = "262590f4fe6afeb0bc83be1daa64e52657fe185690a958af7f3ad0e92085c5ae" dependencies = [ "futures-core", + "futures-sink", ] [[package]] name = "futures-core" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" +checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7" + +[[package]] +name = "futures-executor" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6754879cc9f2c66f88c6e5c35344bb0bdb0708b0352b1201815667c7eabc7458" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4577ecaa3c4f96589d473f679a71b596316f6641bc350038b962a5daf0085d7a" + +[[package]] +name = "futures-macro" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6d3cde68c518367be28956066ddfef33813991b77a55005a69dae04bf3b10b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "futures-sink" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e34418ac499d6305c2fb5ad0ed2f6ac998c5f8ca209b4510f7f94242c647e307" [[package]] name = "futures-task" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" +checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109" [[package]] name = "futures-util" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa" dependencies = [ + "futures-channel", "futures-core", + "futures-io", + "futures-macro", + "futures-sink", "futures-task", + "memchr", "pin-project-lite", "slab", ] @@ -1802,15 +1862,6 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" -[[package]] -name = "lock_api" -version = "0.4.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" -dependencies = [ - "scopeguard", -] - [[package]] name = "log" version = "0.4.29" @@ -1979,29 +2030,6 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "parking_lot" -version = "0.12.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" -dependencies = [ - "lock_api", - "parking_lot_core", -] - -[[package]] -name = "parking_lot_core" -version = "0.9.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" -dependencies = [ - "cfg-if", - "libc", - "redox_syscall", - "smallvec", - "windows-link", -] - [[package]] name = "paste" version = "1.0.15" @@ -2091,7 +2119,7 @@ version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" dependencies = [ - "toml_edit", + "toml_edit 0.25.11+spec-1.1.0", ] [[package]] @@ -2302,15 +2330,6 @@ dependencies = [ "rustversion", ] -[[package]] -name = "redox_syscall" -version = "0.5.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" -dependencies = [ - "bitflags", -] - [[package]] name = "regex" version = "1.12.3" @@ -2554,12 +2573,6 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" -[[package]] -name = "scopeguard" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" - [[package]] name = "sec1" version = "0.7.3" @@ -2663,6 +2676,15 @@ dependencies = [ "zmij", ] +[[package]] +name = "serde_spanned" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +dependencies = [ + "serde", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -3023,7 +3045,6 @@ dependencies = [ "bytes", "libc", "mio", - "parking_lot", "pin-project-lite", "signal-hook-registry", "socket2", @@ -3052,6 +3073,27 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime 0.6.11", + "toml_edit 0.22.27", +] + +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +dependencies = [ + "serde", +] + [[package]] name = "toml_datetime" version = "1.1.1+spec-1.1.0" @@ -3061,6 +3103,20 @@ dependencies = [ "serde_core", ] +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime 0.6.11", + "toml_write", + "winnow 0.7.15", +] + [[package]] name = "toml_edit" version = "0.25.11+spec-1.1.0" @@ -3068,9 +3124,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b59c4d22ed448339746c59b905d24568fcbb3ab65a500494f7b8c3e97739f2b" dependencies = [ "indexmap", - "toml_datetime", + "toml_datetime 1.1.1+spec-1.1.0", "toml_parser", - "winnow", + "winnow 1.0.1", ] [[package]] @@ -3079,9 +3135,15 @@ version = "1.1.2+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" dependencies = [ - "winnow", + "winnow 1.0.1", ] +[[package]] +name = "toml_write" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" + [[package]] name = "tower" version = "0.5.3" @@ -3180,6 +3242,16 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-serde" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704b1aeb7be0d0a84fc9828cae51dab5970fee5088f83d1dd7ee6f6246fc6ff1" +dependencies = [ + "serde", + "tracing-core", +] + [[package]] name = "tracing-subscriber" version = "0.3.23" @@ -3190,12 +3262,15 @@ dependencies = [ "nu-ansi-term", "once_cell", "regex-automata", + "serde", + "serde_json", "sharded-slab", "smallvec", "thread_local", "tracing", "tracing-core", "tracing-log", + "tracing-serde", ] [[package]] @@ -3675,6 +3750,15 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" +[[package]] +name = "winnow" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" +dependencies = [ + "memchr", +] + [[package]] name = "winnow" version = "1.0.1" diff --git a/Cargo.toml b/Cargo.toml index 87b31ed..2a6d1f9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,6 +7,10 @@ rust-version = "1.91" description = "Automated verification for Commit-Boost Kurtosis testnets" license = "MIT" +[lib] +name = "cb_testnet_verifier" +path = "src/lib.rs" + [[bin]] name = "cb-verify" path = "src/main.rs" @@ -16,12 +20,8 @@ name = "cb-orchestrator" path = "src/orchestrator.rs" [[bin]] -name = "test-mux" -path = "src/bin/test_mux.rs" - -[[bin]] -name = "test-relay" -path = "src/bin/test_relay.rs" +name = "sim" +path = "src/bin/sim/main.rs" [dependencies] @@ -36,11 +36,20 @@ prometheus-parse = "0.2" reqwest = { version = "0.12", features = ["json", "rustls-tls"], default-features = false } serde = { version = "1", features = ["derive"] } serde_json = "1" -tokio = { version = "1", features = ["full"] } +# HS256 JWT minting for the Commit-Boost signer module API. Hand-rolled rather +# than pulling `jsonwebtoken`: CB binds the JWT to the exact request ROUTE and to +# a keccak256 payload hash that must be NULL when there is no body, and getting +# those claims exactly right matters more than the ~20 lines saved. +hmac = "0.12" +sha2 = "0.10" +base64 = "0.22" +futures = "0.3" +tokio = { version = "1", features = ["rt-multi-thread", "macros", "time", "process", "net", "signal", "sync"] } tracing = "0.1" -tracing-subscriber = { version = "0.3", features = ["env-filter"] } +tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] } serde_yaml = "0.9.34" +toml = "0.8" [profile.release] strip = true -lto = true +lto = "thin" diff --git a/README.md b/README.md index 2e0379b..319650f 100644 --- a/README.md +++ b/README.md @@ -4,12 +4,18 @@ Automated verification for [Commit-Boost](https://github.com/Commit-Boost/commit ## Prerequisites -- [Kurtosis CLI](https://docs.kurtosis.com/install) (>= 0.90) -- [Rust toolchain](https://rustup.rs/) (1.91+) +- [Kurtosis CLI](https://docs.kurtosis.com/install) — **pin 1.18.1** (the proven-good version; the parsers + rely on its text-table output, and a newer CLI, e.g. 1.20.0, writes an incompatible + `~/.config/kurtosis/kurtosis-config.yml` — see `docs/local-kurtosis-e2e.md`) +- [Rust toolchain](https://rustup.rs/) (1.91+, edition 2024) - Docker (for Kurtosis) +- The forked `ethereum-package` submodule: `git submodule update --init` -If you're testing a local CB build, you also need: -- The [commit-boost-client](https://github.com/Commit-Boost/commit-boost-client) repo cloned +If you're testing a local CB build (the default — the CB sidecar image is built, not pulled), you also need: +- The [commit-boost-client](https://github.com/Commit-Boost/commit-boost-client) repo cloned as a sibling + (`../commit-boost-client`, overridable — see `just build-cb-image`) + +> Contributing? See **[`docs/DEVELOPING.md`](docs/DEVELOPING.md)** for the dev loop and how to add checks + scenarios. ## Docker image configuration @@ -24,14 +30,14 @@ cp .env.example .env | Variable | Default | Purpose | |---|---|---| -| `HELIX_RELAY_IMAGE` | `helix-relay:kurtosis` | Custom Helix relay image | -| `MEV_RELAY_IMAGE` | `ethpandaops/mev-boost-relay:main` | mev-boost relay (multi-relay scenarios) | -| `MEV_BOOST_IMAGE` | `commit-boost/pbs:kurtosis` | Commit-Boost PBS image | +| `HELIX_RELAY_IMAGE` | `ghcr.io/gattaca-com/helix-relay:main` | Helix relay image | +| `MEV_RELAY_IMAGE` | `ethpandaops/mev-boost-relay:main` | flashbots mev-boost-relay — **no longer used** (multi-relay now runs two helix instances); still emitted into configs but inert | +| `MEV_BOOST_IMAGE` | `commit-boost/commit-boost:kurtosis` | Commit-Boost sidecar image | | `BUILDER_CL_IMAGE` | `sigp/lighthouse:latest` | Builder consensus client | | `BUILDER_EL_IMAGE` | `ethpandaops/reth-rbuilder:develop` | Builder execution client | -The `.env` file is read automatically by `generate_kurtosis_configs.py`. -It is gitignored — do not commit it. Use `.env.example` as the reference. +The `.env` file is read automatically by `just generate-configs` (the `sim generate` +command). It is gitignored — do not commit it. Use `.env.example` as the reference. ## Kurtosis setup / gotchas @@ -45,9 +51,9 @@ The fork generalizes hardcoded patterns from upstream, enabling configs like com ### Kurtosis configs -Kurtosis uses a default Commit-Boost config that can be overridden by inlining it into the kurtosis config — see `configs/example-kurtosis-config.yml`. Every generated test config uses this pattern. +Kurtosis uses a default Commit-Boost config that can be overridden by inlining it into the kurtosis config. Every generated test config uses this pattern: the helix and commit-boost configs are embedded as the two `|` block scalars under `mev_params`. -`generate_kurtosis_configs.py` generates test scenarios from `.env`: +`sim generate` (via `just generate-configs`) builds the test scenarios, applying any `.env` image overrides: ```bash just generate-configs @@ -58,8 +64,8 @@ Six scenarios are generated: | Config | What it tests | |---|---| | `cb-basic.yml` | Single relay (helix), default CB config | -| `cb-multiple-relays.yml` | Two relays (helix + flashbots), aggregated bidding | -| `cb-mux.yml` | Mux routing — 128 validators to helix, 128 to flashbots | +| `cb-multiple-relays.yml` | Two helix relay instances, aggregated bidding | +| `cb-mux.yml` | Mux routing — 128 validators to helix-1, 128 to helix-2 | | `cb-skip-sigverify.yml` | Fast path with BLS signature verification disabled | | `cb-timing-games.yml` | Aggressive per-relay timing overrides for late bidding | | `cb-extra-validation.yml` | Extra get_header validation via local EL RPC | @@ -67,6 +73,19 @@ Six scenarios are generated: ## Quick start ```bash +# ONE-TIME (from scratch): init the forked ethereum-package, then build the +# Commit-Boost image the devnet runs (from the sibling commit-boost-client repo) +git submodule update --init +just build-cb-image # -> commit-boost/commit-boost:kurtosis + +# Generate configs, pull public images, launch + verify. Prints the tiered +# report and exits 0 (pass) / 1 (tier-1 FAIL) / 2 (setup failure). Add --json +# to the verifier for the machine-readable verdict (see docs/CHECKS.md). +just e2e # cb-basic +just e2e configs/generated/cb-mux.yml + +# --- or the individual steps --- + # Generate configs from .env just generate-configs @@ -84,13 +103,15 @@ just show-logs CB-Testnet # Quick mux routing diagnostic just test-mux CB-Testnet configs/generated/cb-mux.yml - -# Test relay API endpoints -cargo run --release --bin test-relay -- http://127.0.0.1:PORT 128 160 ``` ## What it checks +> The tables below are a quick reference. **[`docs/CHECKS.md`](docs/CHECKS.md) is the authoritative +> catalog** — per-check pass/warn/fail contract, data source, and the load-bearing verdict rule: the +> process exit code keys **only on a tier-1 FAIL**; WARN and SKIP are non-fatal, so a consumer gating on +> a trust-critical anomaly must read the JSON `result:"WARN"`, not just the exit code. + ### Tier 1: Pipeline health (must pass) | Check | What it verifies | @@ -110,6 +131,7 @@ cargo run --release --bin test-relay -- http://127.0.0.1:PORT 128 160 | `relay.builder_blocks_received` | Builder submitted blocks to relay | > 0 | | `relay.mev_delivery_rate` | Slots using relay-built blocks vs local | >= 30% | | `relay.validator_registrations` | All validators registered on relay | 100% | +| `relay.best_bid` | CB delivered >= the best bid it was offered across relays | competition + delivered | ### Tier 3: CB metrics @@ -142,24 +164,26 @@ Options: --live-metrics Poll :9090/metrics during observation --show-logs Print raw CB PBS logs, no checks --output-dir Save JSON reports (requires --json) + --skip-finalization-check Skip the finality check (for early/short windows) ``` -### test-mux +### sim (generate | preflight | triage) ``` -test-mux - -Fetches CB PBS logs, parses mux events, verifies routing against config. -No observation window. Completes in seconds. +sim generate [SCENARIO] [--out-dir DIR] [--check] # typed Rust config generator + # --check: verify on-disk configs match (CI drift gate) +sim preflight # validate the config against the real helix image (~1s) + # exit 1 only on a genuine config-drift Fail +sim triage # extract each dead service's root panic (JSON) ``` -### test-relay +### test-mux ``` -test-relay [pubkey] +just test-mux -Tests relay data API endpoints with slot filtering. -Verifies delivered payloads, builder blocks, validator registration. +Runs cb-verify with --config against a running enclave: fetches CB PBS logs, +parses mux events, verifies routing against config. No observation window. ``` ## How it works @@ -192,26 +216,22 @@ Verifies delivered payloads, builder blocks, validator registration. ``` cb-testing/ - Cargo.toml # Workspace: cb-verify, test-mux, test-relay + Cargo.toml # Workspace: cb-verify, cb-orchestrator, sim justfile # Build/test/launch commands README.md .env.example # Docker image overrides scripts/ - run-and-verify.sh # Attached mode launcher - generate_kurtosis_configs.py # Config generator + run-and-verify.sh # Attached mode launcher (preflight-gated) configs/ - generated/ # Pre-generated test scenarios - example-kurtosis-config.yml + generated/ # Test scenarios, emitted by `sim generate` src/ main.rs # cb-verify binary + bin/sim/ # sim: generate | preflight | triage checks/ chain_health.rs # Finality, sync, missed slots relay_pipeline.rs # Delivery, registration, MEV rate payload_matching.rs # Hash matching mux_routing.rs # Mux config parsing, log analysis cb_metrics.rs # Prometheus metrics checks - bin/ - test_mux.rs # Mux diagnostic binary - test_relay.rs # Relay API diagnostic binary ethereum-package/ # Forked Kurtosis package (submodule) ``` diff --git a/configs/example-kurtosis-config.yml b/configs/example-kurtosis-config.yml deleted file mode 100644 index 7244098..0000000 --- a/configs/example-kurtosis-config.yml +++ /dev/null @@ -1,67 +0,0 @@ - -# cb-basic: Single relay (helix) with default Commit-Boost config. -# -# Tests the core MEV pipeline through Commit-Boost with a single Helix -# relay as the only relay endpoint. - -participants: - - el_type: geth - cl_type: lighthouse - -additional_services: - - dora - - spamoor - - prometheus - -mev_type: custom - -mev_params: - mev_relay: helix - mev_sidecar: commit-boost - mev_builder: flashbots - - helix_relay_image: ghcr.io/gattaca-com/helix-relay:main - mev_boost_image: ghcr.io/commit-boost/pbs:latest - mev_builder_image: ethpandaops/reth-rbuilder:develop - mev_builder_cl_image: sigp/lighthouse:latest - mev_builder_subsidy: 1 - - - commit_boost_config: | - chain = { genesis_time_secs = {{ .Timestamp }}, path = "{{ .Network }}" } - - [pbs] - host = "0.0.0.0" - port = {{ .Port }} - timeout_get_header_ms = 950 - timeout_get_payload_ms = 4000 - late_in_slot_time_ms = 2000 - - [metrics] - host = "0.0.0.0" - port = 9090 - - {{ range $index, $relay := .Relays }} - [[relays]] - id = "mev_relay_{{$index}}" - url = "{{ $relay }}" - {{- end }} - - [logs.stdout] - level = "debug" - - [logs.file] - enabled = false - -network_params: - network: kurtosis - network_id: "3151908" - deposit_contract_address: "0x00000000219ab540356cBB839Cbe05303d7705Fa" - seconds_per_slot: 12 - slot_duration_ms: 12000 - num_validator_keys_per_node: 128 - preregistered_validator_keys_mnemonic: - "giant issue aisle success illegal bike spike - question tent bar rely arctic volcano long crawl hungry vocal artwork sniff fantasy - very lucky have athlete" - prefunded_accounts: '{"0xb9e79d19f651a941757b35830232e7efc77e1c79": {"balance": "100000ETH"}}' diff --git a/docs/ARCH.md b/docs/ARCH.md new file mode 100644 index 0000000..066bac2 --- /dev/null +++ b/docs/ARCH.md @@ -0,0 +1,240 @@ +# cb-testing — ARCHITECTURE + +How the pieces fit together. `docs/DESIGN.md` says WHY this repo exists and where it's going; +this doc is the HOW — the map a newcomer needs so they don't have to reverse-engineer ~10k lines of +Rust. Keep it small and load-bearing: WHAT each thing does lives in the code + tests; WHY a given +design was chosen lives here and in `docs/DESIGN.md`. + +Scope note: the target is a single library-first Rust app (`sim generate | preflight | run | verify | +triage`). Today it is **most** of the way there — the verifier, config generation, preflight, and +triage are all Rust; a shell launcher (`run-and-verify.sh`) still glues launch+verify together and is +the last thing slated to fold into `sim run`. This doc describes what exists now and flags the seams. + +--- + +## 1. End-to-end flow + +One devnet run, from typed config to verdict. Names below are the actual binaries / scripts / +functions. + +``` + sim generate (Rust, src/bin/sim/generate.rs → genmodel/*) + │ typed Scenario + Images → args_file_in() assembles the Kurtosis YAML + │ writes configs/generated/.yml (byte-identical to golden fixtures) + ▼ + just e2e / just testnet ──► scripts/run-and-verify.sh (the attached-mode launcher) + │ + ├─(0) cargo build --release --bin cb-verify pre-build OFF the critical path + │ (no multi-GB compile while 10 services are live) + ├─(1) kurtosis enclave rm -f clear any stale enclave + ├─(1b) sim preflight src/bin/sim/preflight.rs + │ render.rs pulls the two | block scalars, substitutes dummy runtime vars, + │ runs the REAL helix image against the rendered config (~1s docker probe), + │ classify_helix_probe() → Pass / Fail{field} / Inconclusive. + │ Exits nonzero ONLY on Fail (genuine config drift) → aborts the run early. + ├─(1c) check_host_memory advisory OOM warning before a ~10-min spend + ├─(2) kurtosis run ./ethereum-package \ launch the devnet (the forked package) + │ --args-file --image-download always + │ → geth + lighthouse, N helix relays, reth-rbuilder builder, + │ commit-boost sidecar, dora + spamoor + prometheus + │ on launch failure → sim triage (root-cause capture, then exit) + ▼ + cb-verify --enclave --config … (src/main.rs; the pre-built binary) + │ discovery::discover() kurtosis inspect/port print → beacon/relay/cb/metrics URLs + │ health::probe_all() Tier-0 reachability preflight (dead service → abort or postmortem) + │ wait_for_slot() wait until head passes the observation window's end slot + │ checks::* chain_health, relay_pipeline, payload_matching, best_bid, + │ cb_metrics, mux_routing → Vec + │ report::print_report() + exit_code() verdict keyed on any Tier-1 FAIL + ▼ + VerificationReport (human render OR --json) + exit 0 / 1 / 2 +``` + +On any service crash mid-run, or a launch failure, `sim triage` (`triage.rs` → `diagnose.rs`) +attaches each dead service's root panic to the output — observability as a property of the run, not a +separate tool (DESIGN Law 5). + +`just` verbs are thin wrappers: `generate-configs` → `sim generate`; `e2e`/`testnet` → +`run-and-verify.sh`; `verify*` → `cb-verify` directly; `test-all` → `cb-orchestrator`. + +--- + +## 2. Two binaries + a shared library (+ `sim`) + +The crate is **library-first** (the target architecture; see `docs/DESIGN.md`): the mature verifier modules live in +`src/lib.rs` (`cb_testnet_verifier`) so every binary imports them instead of re-declaring or +re-implementing. `Cargo.toml` declares one lib and five bins. + +| Target | Path | Role | +|---|---|---| +| `cb_testnet_verifier` (lib) | `src/lib.rs` | Shared modules: `beacon`, `relay`, `metrics`, `checks`, `discovery`, `report`. Imported by every bin. | +| `cb-verify` (bin, default-run) | `src/main.rs` | The single-enclave verifier. Discover → preflight → observe → run checks → report → exit code. `health`/`live` are private to this bin. | +| `cb-orchestrator` (bin) | `src/orchestrator.rs` | Parallel multi-enclave runner (`just test-all`). Spawns a launch→wait→observe→check→teardown pipeline per config, bounded by a `--jobs` semaphore, then shells the built `cb-verify` binary for the checks and aggregates a `BatchReport`. This is the "one entry at `--jobs 1`" that `sim run` will eventually absorb. | +| `sim` (bin) | `src/bin/sim/main.rs` | Separate app for `generate` / `preflight` / `triage`. Reuses the lib but owns its own submodule tree. Sync-only (see §6). | +| `test-mux`, `test-relay` (bins) | `src/bin/test_{mux,relay}.rs` | Standalone diagnostics (quick mux-routing / relay-API probes). Slated for retirement into the checks they duplicate. | + +Two verdict surfaces coexist: `cb-verify`/orchestrator produce a **`VerificationReport`** (tiered +checks); `sim` produces its own small JSON reports (`PreflightReport`, `TriageReport`). They are +different report types because `sim` runs before/around a devnet, not against a healthy one. + +--- + +## 3. Module map of `src/` + +(Replaces the stale tree in `README.md`.) + +### Shared library (`src/lib.rs`) + +| Module | Responsibility | +|---|---| +| `discovery.rs` | Shell `kurtosis enclave inspect` / `port print`, parse the text tables → `EnclaveServices` (beacon / relay / cb-pbs / cb-metrics URLs, relay identities, cb service names, prometheus). Also `query_mev_relay_postgres` post-mortem. Pure helpers `parse_services`, `split_on_multi_space`, `extract_ports`, `matches_pattern`, `is_relay_api_service`, `relay_identity` are unit-tested. | +| `beacon.rs` | `BeaconClient` — async reqwest wrapper over the standard Beacon API (head slot, finalized epoch, syncing, genesis time, header/block-hash by slot, active validator pubkeys); returns alloy beacon types. Only the endpoints verification needs. | +| `relay.rs` | `RelayClient` — async wrapper over the Flashbots relay Data API (`ping`, cursor-paginated `get_payloads_delivered`, `get_builder_blocks_received`, `is_validator_registered`). | +| `metrics.rs` | Fetch + parse CB Prometheus text (`prometheus-parse`) via HTTP or a `kurtosis service exec` fallback; sample-aggregation helpers (`sum_metric`, `metric_values`, `has_metric`). | +| `report.rs` | `VerificationReport { enclave, timestamp, observation_window, result, checks }` + `ObservationWindow`; `print_report` (color or `--json`), `save_json_report`, and `exit_code` — the authoritative verdict→process-code mapping (see §5). | +| `checks/mod.rs` | `CheckResult { id, tier: u8, status (serde `result`), detail, data: Value }` + `CheckStatus` (Pass/Fail/Warn/Skip, serialized UPPERCASE). Constructors `pass`/`fail`/`warn`/`skip(id, tier, detail)` + `with_data`. | +| `checks/chain_health.rs` | Tier-1/2 chain checks: finality, sync status, missed-slot rate. | +| `checks/relay_pipeline.rs` | Relay-side checks: payloads delivered, builder blocks received, MEV delivery rate, validator registrations. | +| `checks/payload_matching.rs` | Tier-1: delivered payload `block_hash` matches the on-chain block. | +| `checks/best_bid.rs` | Aggregated-bidding check: compares bid values across relays (not union-by-slot — DESIGN Law 3). | +| `checks/mux_routing.rs` | Parse `[[mux]]` sections from the CB config, fetch + parse CB PBS logs (ANSI-aware), verify each proposer pubkey routed to its assigned relay. Also `fetch_service_logs` / `parse_cb_log_line`. | +| `checks/cb_metrics.rs` | CB Prometheus status-code matrices (get_header / register_validator / submit_blinded_block / status), v1→v2 fallback, get_header latency p95. Emitted at tier 2, but a `*_matrix` FAIL (a 5xx = real pipeline failure) is **escalated to tier 1** so it gates the exit code. | + +### `cb-verify`-private (`src/`) + +| Module | Responsibility | +|---|---| +| `health.rs` | `HealthTarget` / `ServiceKind` + `probe_all` — the Tier-0 reachability preflight and mid-wait liveness probes. | +| `live.rs` | Live-metrics deltas during the observation window (`compute_deltas`, `format_delta_{json,log}`, `LIVE_METRICS_FILTER`). | + +### `sim` submodules (`src/bin/sim/`) + +| Module | Responsibility | +|---|---| +| `cli.rs` | clap surface: `Cli` + `Command { Preflight, Triage, Generate }`, global `--log-format pretty|json`. | +| `main.rs` | Dispatch to the three verbs; `tracing` init; nonzero exits on error. | +| `generate.rs` | IO boundary for `sim generate`: `.env` image overrides, atomic assemble-then-write, and `--check` drift gate (`generate::check`). Pure assembly lives in `genmodel`. | +| `genmodel/mod.rs` | Golden-fixture harness (`golden`, `assert_matches_golden`, `extract_block_scalar`) — the byte-identity oracle for the verbatim config port. | +| `genmodel/scenario.rs` | `Scenario` enum (six scenarios) + `Images` map; `args_file_in()` joins the static fragments + helix const + CB block into a full args-file; `build_mev_params`. | +| `genmodel/helix.rs` | `HELIX_RELAY_CONFIG` — the helix YAML block, byte-identical across all six scenarios (const). | +| `genmodel/cb.rs` | The CB TOML block: `cb_toml(CbParams)` + `cb_toml_mux(node0, node1)` — verbatim port of the Python builders, generate-time knobs injected by string building. | +| `render.rs` | The **compatibility contract** with the fork: `extract_config_blocks` (pull the two `|` scalars), `substitute_runtime_vars` (strip the `{{ range }}` loop, fill `{{ .VAR }}` dummies), `default_dummies`. Pure. | +| `preflight.rs` | `sim preflight`: render + run the real helix image + `classify_helix_probe` (pure, 3-valued). | +| `triage.rs` | `sim triage`: `parse_service_statuses` + `services_to_triage` (pure) + process I/O to collect logs (kurtosis→docker fallback for the masking bug). | +| `diagnose.rs` | `extract_root_cause` (pure): pattern-based root-panic extraction from log text, skipping broker/grpc masking lines. Shared by `preflight` and `triage`. | + +--- + +## 4. Config generation ↔ the fork coupling + +This is the load-bearing seam. `sim generate` emits a Kurtosis args-file whose `mev_params` carries +**two `|` block scalars** that the forked ethereum-package parses and fills at launch: + +- `helix_relay_config: |` — the helix relay's YAML config (byte-identical across all six scenarios). +- `commit_boost_config: |` — the commit-boost sidecar's TOML config (varies ≤ ~7 lines per scenario). + +Both blocks are **opaque scalars to YAML** and contain unrendered **Go-template holes** that only the +ethereum-package fills at `kurtosis run` time (the generator never sees the runtime values — postgres +host/port, the actual beacon/blocksim URLs, genesis timestamp, the real relay URL list): + +| Block | Runtime `{{ }}` holes | +|---|---| +| helix (YAML) | `.POSTGRES_HOST_NAME`, `.POSTGRES_PORT`, `.POSTGRES_DB`, `.POSTGRES_USER`, `.POSTGRES_PASS`, `.BEACON_URI`, `.BLOCKSIM_URI` | +| commit-boost (TOML) | `.Timestamp`, `.Network`, `.Port`, and the `{{ range $index, $relay := .Relays }} … {{- end }}` loop (plus `{{ index .Relays N }}` in the mux `[[mux.relays]]`) | + +**`render.rs` is the compatibility contract.** For preflight (§5), it must turn a template-holed block +back into something the real image can parse: `strip_range_blocks` drops the `.Relays` loop (valid +because relays are `#[serde(default)]` downstream) and `replace_simple_vars` fills each `{{ .VAR }}` +from `default_dummies()`. `default_dummies` therefore has to cover **every** hole the args-file uses — +if the fork adds a template var, this map is where the contract breaks, and the preflight tests +(`substituted helix is valid YAML` / `substituted CB is valid TOML`) are what catch it. + +**Typing lives only at the assembly layer** (`Scenario` + `Images`), not in the block bodies. The +bodies are ported **verbatim** from the retired Python `generate_kurtosis_configs.py` into `const` +strings / string builders — the P2 grill killed the "build from `cb_common` structs / typed helix +mirror" plan (helix types aren't importable; the serde-sentinel mechanism was fragile; the templates +weren't actually duplicated).. + +**The golden-fixture byte-identity guard is the oracle.** `tests/fixtures/golden-configs/.yml` +snapshots the proven-good output; `every_scenario_matches_its_golden` asserts `sim generate` reproduces +each **byte-for-byte** with the baked-default images. A separate test guards the tracked +`configs/generated/cb-basic.yml` (which `render.rs` and `sim preflight` consume as a fixture) against +silently drifting from the generator. `sim generate --check` is the CI/agent form of the same guard. + +The one image map (`Images::default()`) is also where the historical four-way +`commit-boost/pbs` vs `commit-boost/commit-boost` drift was killed; `.env` overrides are applied only +at the CLI boundary (`generate::run`), keeping assembly pure and hermetically testable. + +--- + +## 5. The verdict model + +Checks are pure functions over already-fetched data; each returns a `CheckResult { name, tier, status, +detail }` where `status ∈ {Pass, Fail, Warn, Skip}` and `tier ∈ {1, 2, 3}`. `cb-verify` collects every +check into `Vec`, then the report verdict is: **`Fail` iff any Tier-1 check is `Fail`, +else `Pass`.** `report::exit_code` computes the process code by filtering to `tier == 1` checks: +**no tier-1 check present → 2** (nothing gating ran — a setup/infra failure; the discovery/preflight/ +timeout paths in `main.rs` also `return 2` directly), **any tier-1 `Fail` → 1**, **else → 0**. +Tier-2/Tier-3 `Fail`s and all `Warn`s are **non-fatal** +by design; `--strict` promotes selected soft warnings to `Fail`. `Skip` never fails the run (a check +skips when its inputs are unavailable — e.g. no metrics port, no validator pubkeys, no `[[mux]]` +sections). + +`sim preflight` has its own narrower 3-valued verdict (`Pass` / `Fail{field}` / `Inconclusive`) and +only its `Fail` aborts a launch — an `Inconclusive` (slow pull, docker down, pre-genesis panic) must +never be scored as config drift. + +The per-check catalog (names, tiers, thresholds, what each asserts) lives in **`docs/CHECKS.md`**. + +--- + +## 6. Key architectural decisions + rationale + +- **Shell out to kurtosis, synchronously, and parse text.** There is **no Kurtosis Rust SDK** + (`discovery.rs` TODO; only a Go SDK exists), so enclave ops are `std::process::Command` + + text-parsing — inherent, not fixable by the rewrite. And kurtosis 1.18.1 has **no `--format json`** + for the tables we read, so `discovery`/`triage` parse the human `enclave inspect` output column by + column (`split_on_multi_space`). The `sim` app is deliberately **sync / no-tokio** to match this + (verbs shell out and block; a bounded `timeout` coreutil supplies the wall-clock cap that sync std + lacks). `cb-verify` and `cb-orchestrator` *are* tokio (they do concurrent HTTP polling / parallel + enclaves), but `sim` and the shared discovery layer are not. + +- **The forked `ethereum-package` exists on purpose.** Upstream treats out-of-protocol block building + as bespoke, hard-coded convenience and injects the VC `--builder` flag deep inside + `enrich_mev_extra_params` via a naming-convention URL, with no external-builder hook. The fork carries + a general `(relay, sidecar, builder)` component model (`mev_resolver.star`) + a `mev_type: custom` + config API — which is exactly what lets a config say "helix relay + commit-boost sidecar + + reth-rbuilder builder" and, later, swap in an ePBS builder. A pure shim over today's upstream would + have to reimplement a brittle 7-client flag matrix — worse than the fork. cb-testing is the fork's + consumer/dogfood (DESIGN Law 6). ONE fork; do not maintain two. + +- **The pure `classify_*` / pure-core seam.** Every verb that makes a judgement splits into a **pure + classifier** (data in, verdict out — unit-testable against fixture logs, no devnet, no docker) and a + thin **process-I/O shell** (smoke-checked by hand with the real tools). Named seams: + `preflight::classify_helix_probe`, `triage::{parse_service_statuses, services_to_triage}`, + `diagnose::extract_root_cause`, `payload_matching::classify_payload_matches`, + `best_bid::classify_best_bid` (+ `value_eth_to_wei`), `mux_routing::classify_mux_routing` (+ + `parse_cb_log_line`, `extract_mux_from_config`), `cb_metrics::{collect_endpoint_stats, + classify_endpoint, check_v2_fallback, check_relay_latency, histogram_quantile}`, and + `live::compute_deltas`. These pure cores (generic over the hash/value type, taking pre-fetched + `BTreeMap`s) are the unit-tested surface; the fixtures under `tests/fixtures/` are the real test + inputs. This is DESIGN Law 4 ("verdict logic is TDD-able without a devnet"). The two exceptions: + `chain_health` and `relay_pipeline` inline their verdicts in the async check fns and have **no** + factored-out pure classifier — the standing gap that P3 + closes. + +- **Preflight-first, observability-as-a-property.** Validating a rendered config against the real image + in ~1s (before a ~10-min devnet spend) is the single biggest agent-friendliness win; auto-triage on + any failure means a run emits its own root cause. Both are normal outputs of a normal run, one source + of truth (`--log-format json` for agents, pretty for humans) — never a separate agent-only tool + surface. See DESIGN's thesis + Laws 1 and 5. + +--- + +## See also + +- `docs/DESIGN.md` — the mission and the design laws. +- `docs/CHECKS.md` — the per-check catalog (tiers, thresholds, feature-assertion status). +- `docs/local-kurtosis-e2e.md` — the runbook + the paid-for incident behind half the design. +- the design-rationale plans (kept in the local `.agent/` working area) — the + grilled rationale for each slice (internal). diff --git a/docs/CHECKS.md b/docs/CHECKS.md new file mode 100644 index 0000000..b382112 --- /dev/null +++ b/docs/CHECKS.md @@ -0,0 +1,459 @@ +# cb-verify checks — the authoritative catalog + +This is the per-check contract for `cb-verify`: what each check asserts, its tier, its +pass/warn/fail/skip conditions, and where its data comes from. It is meant to be read by both a +human and a CI/agent consumer of the verdict. Facts here are sourced from the code +(`src/checks/*.rs`, `src/report.rs`, `src/main.rs`); when in doubt, the code wins. + +## The verdict model (read this first — it is load-bearing) + +A run emits a `VerificationReport` (`src/report.rs`): an `enclave`, a `timestamp`, an +`observation_window`, an overall `result`, and a list of per-check `CheckResult`s. Each +`CheckResult` (`src/checks/mod.rs:26`) carries an `id`, a `tier` (`u8`), a `result` (serialized +name of `CheckStatus`: `PASS` / `FAIL` / `WARN` / `SKIP`), a `detail` string, and a `data` object. + +**Tiers = severity contract:** + +| Tier | Meaning | Effect on exit code / overall result | +|------|---------|--------------------------------------| +| 1 | **must** — a real pipeline invariant | A tier-1 `FAIL` fails the whole run | +| 2 | **should** — health signal | Never fails the run; annotative | +| 3 | **informational** | Never fails the run; annotative | + +**The crucial contract — only a tier-1 FAIL is fatal.** `report::exit_code` +(`src/report.rs:131-145`) returns: + +- `2` if **no** tier-1 checks ran at all (setup/discovery failure), +- `1` if **any** tier-1 check is `FAIL`, +- `0` otherwise. + +The overall `report.result` is computed the same way (`src/main.rs:559-571`): it is `FAIL` iff some +tier-1 check is `FAIL`, else `PASS`. `WARN` and `SKIP` are **non-fatal and annotative** — they never +change the exit code and never change the overall result, at any tier. + +**Consequence a consumer MUST internalize:** several checks that exist specifically to catch an +anomaly report that anomaly as `WARN`, not `FAIL` (relay equivocation in `payload_hash_match`, +unverifiable routing in `mux.routing`, a best-bid shortfall in `relay.best_bid`). A run in which +those fire **still exits 0**. A CI job or agent that trusts only the process exit code will call a +misbehaving pipeline green. **Parse the JSON and inspect each check's `result` field** — do not gate +on the exit code alone. If you want any of these `WARN`s to be fatal, that is a policy decision that +has to be made explicitly (see the P3 notes and Known gaps below); today they are not. + +### `inconclusive`: armed and unmeasured (`--require-feature-proof`) + +One class of `WARN` is not an anomaly at all, it is a **failure to measure**. A Law 3 feature check +that arms a differential and then observes nothing has proved nothing about the feature, yet tier-1 +`WARN` is non-fatal, so the scenario exits 0 and a sweep counts it as a win. Those checks now carry +`"inconclusive": true` in their JSON, and **`--require-feature-proof`** makes a tier-1 inconclusive +check exit `1`. + +The flag is **off by default**, so the contract above is unchanged for existing callers. Turn it on +in sweeps. The three sites it covers: + +| check | inconclusive when | +| --- | --- | +| `feature.` | enabled in CB config, ZERO proof markers in CB debug logs | +| `feature.skip_sigverify` | differential ARMED (wrong-pubkey relay url) but zero auction winners | +| `feature.min_bid` | floor set but ZERO bids rejected | + +Deliberately **not** marked: `feature.skip_sigverify` in a plain (unpoisoned) scenario. That is a +negative codepath which emits nothing when it fires, so it is structurally unconfirmable rather than +unmeasured, and marking it would turn every scenario carrying `skip_sigverify` permanently red. +Relay equivocation in `payload_hash_match` also stays a plain `WARN`: it is a real observation, not +the absence of one. + +One escalation exists: the `cb_metrics` matrix checks are authored at tier 2 but are **promoted to +tier 1 when they FAIL** (`src/checks/cb_metrics.rs:636-641`), because a 5xx from a relay is a real +pipeline failure. So a matrix 5xx does gate the exit code even though the check's nominal tier is 2. + +**Setup / preflight failures** (bad args, discovery failure, no beacon node, dead services, chain +never reached the window) short-circuit before the check phase and produce a single synthetic tier-1 +`FAIL` check (`setup`), exit code `2` (`src/main.rs`, `make_error_report`). + +## Data-source robustness + +Checks split by where their evidence comes from, which determines how they behave when a service +dies mid-run: + +- **CB-log-based / kurtosis-inspect-based — robust to relay death.** Evidence is in the CB + container logs or `kurtosis enclave inspect`, which survive a relay crash: `cb_running`, + `mux.routing`, and the *offered-bid* half of `relay.best_bid`. +- **CB Prometheus metrics — robust to relay death, but usually absent.** `cb_*_matrix`, + `cb_v2_fallback`, `cb_relay_latency`. These SKIP wholesale unless CB was configured to expose + metrics; the default kurtosis PBS mode does **not** set metrics config, so they SKIP by default. +- **Relay data-API-based — fragile.** If the relay dies before check time these `FAIL` or `SKIP`: + `relay.builder_blocks_received`, `relay.payloads_delivered_multi`, `relay.mev_delivery_rate`, + `relay.validator_registrations`, `payload_hash_match`, and the *delivered-value* half of + `relay.best_bid`. `run_relay_checks` pings each relay first and emits a single `SKIP` per + downstream check when all relays are unreachable (rather than a pile of request errors). +- **Beacon-API-based.** `chain_finality`, `sync_status`, `missed_slots`, plus the on-chain half of + `relay.mev_delivery_rate` and `payload_hash_match`. + +## Catalog + +| id | tier | source | asserts | +|----|------|--------|---------| +| `chain_finality` | 1 | beacon API | finalized epoch ≥ 2 (conditionally run) | +| `sync_status` | 1 | beacon API | beacon node is done syncing | +| `cb_running` | 1 | kurtosis inspect | ≥1 commit-boost service is `running` | +| `missed_slots` | 2 | beacon API | missed-slot rate < 10% over the window | +| `relay.payloads_delivered_multi` | 1 | relay data API | ≥1 payload delivered across relays | +| `relay.builder_blocks_received` | 2 | relay data API | ≥1 builder block received by a relay | +| `relay.mev_delivery_rate` | 2 | relay data API + beacon | MEV-delivered block fraction ≥ threshold (0.30) | +| `relay.validator_registrations` | 3 | relay data API | validators are registered with the relay | +| `payload_hash_match` | 1 | relay data API + beacon | relay-delivered hashes match on-chain, no cross-relay conflict | +| `relay.best_bid` | 2 | CB logs + relay data API | CB delivered ≥ the best per-relay bid it was offered | +| `mux.routing` | 1 | CB logs | every checked getHeader routed per the `[[mux]]` config | +| `feature.timing_games` | 1 | CB logs | timing-games codepath fired (≥1 `TG:` debug line); config-gated | +| `feature.extra_validation` | 1 | CB logs | extra-validation codepath fired (≥1 parent-block fetch); config-gated | +| `feature.min_bid` | 1 | CB logs | the `min_bid_eth` floor dropped bids; FAIL if a winner is under it; config-gated | +| `feature.skip_sigverify` | 1 | CB logs | skip-sigverify fired (differential: wrong-pubkey relay + auction winners); WARN in plain scenarios | +| `signer.pubkeys` | 1 | CB signer API | the signer loaded the devnet's validator keys (JWT-authed count); config-gated | +| `cb_get_header_matrix` | 2 → 1 on FAIL | CB Prometheus | get_header status-code distribution healthy | +| `cb_register_validator_matrix` | 2 → 1 on FAIL | CB Prometheus | register_validator acceptance healthy | +| `cb_submit_blinded_block_matrix` | 2 → 1 on FAIL | CB Prometheus | ≥1 blinded-block delivery (200/202) | +| `cb_status_matrix` | 2 → 1 on FAIL | CB Prometheus | status endpoint answering 200 | +| `cb_relay_v2_unsupported` | 2 → 1 on FAIL | CB Prometheus | no v2 submit_block lost to a relay 404ing the v2 route | +| `cb_v2_fallback` | 2 | CB Prometheus | no v2→v1 submitBlindedBlock fallbacks | +| `cb_relay_latency` | 2 | CB Prometheus | p95 relay latency < 500 ms | + +Note: `relay.validator_registrations` (tier 3) is only added to the report when active validator +pubkeys were successfully fetched; if the pubkey fetch fails, the check is omitted entirely (it +does not even SKIP). `cb_v2_fallback` and `cb_relay_latency` are produced by the metrics phase but +were not part of the original catalog request; they are included here for completeness. + +--- + +### `chain_finality` — tier 1 (beacon API) + +Asserts the beacon chain has finalized past epoch 2. Source: `beacon.get_finalized_epoch()`. + +- **PASS** — finalized epoch ≥ 2. +- **FAIL** — finalized epoch < 2, or the beacon query errored. +- **SKIP** — the surrounding `run_chain_health_checks` skips this check (and injects a tier-1 SKIP) + when `--skip-finalization-check` is set, **or** when the observation window ends before slot 96 + (epoch 3), because the justification cascade has not had time to finalize epoch 2 yet. +- No WARN state. + +### `sync_status` — tier 1 (beacon API) + +Asserts the beacon node is not syncing. Source: `beacon.is_syncing()`. + +- **PASS** — node reports not syncing. +- **FAIL** — node still syncing, or the query errored. +- No WARN/SKIP. + +### `cb_running` — tier 1 (kurtosis inspect) + +Asserts commit-boost is actually up. Source: `kurtosis enclave inspect `, grepped +case-insensitively for the service pattern `commit-boost` and the word `running`. + +- **PASS** — ≥1 matching service line also contains `running`. +- **FAIL** — matching services exist but none are `running`; **or** no matching services found; + **or** the `kurtosis` CLI errored / returned non-zero. +- No WARN/SKIP. + +### `missed_slots` — tier 2 (beacon API) + +Asserts the miss rate over the window is under threshold. Source: `beacon.get_header(slot)` for each +slot in `[start, end)`; a `None` header or an error counts as missed. Threshold is hardcoded to +`0.10` by `run_chain_health_checks`. + +- **PASS** — miss rate < 10%. +- **WARN** — miss rate ≥ 10%. +- **SKIP** — single-slot window (`start == end`): no interior slots to measure, so it deliberately + SKIPs rather than PASS on zero data. +- **FAIL** — inverted range (`start > end`), treated as nonsense input. + +### `relay.payloads_delivered_multi` — tier 1 (relay data API) + +Asserts the MEV pipeline delivered at least one payload. Source: `get_payloads_delivered(start,end)` +across all live relays, unioned by slot. + +- **PASS** — ≥1 delivered payload (across any relay). +- **FAIL** — zero delivered payloads across all relays. +- **SKIP** — all relays unreachable at check time (fragile-source SKIP from `run_relay_checks`). + +### `relay.builder_blocks_received` — tier 2 (relay data API) + +Asserts a relay received builder blocks. Source: `get_builder_blocks_received(slot)` sampled at ~10 +slots across the window (the data API requires a filter param, so it samples). Aggregated across live +relays: PASS if any relay received blocks. + +- **PASS** — ≥1 builder block received by at least one relay. +- **FAIL** — no builder blocks at any sampled slot on any relay. +- **SKIP** — all relays unreachable at check time. + +### `relay.mev_delivery_rate` — tier 2 (relay data API + beacon) + +Asserts a healthy fraction of on-chain blocks came from the relay. Source: delivered block hashes +from the first relay whose data API responds, intersected with on-chain block hashes +(`beacon.get_block_hash(slot)`) over the window. Threshold is `--mev-threshold` (default `0.30`). + +- **PASS** — `mev_blocks / total_blocks` ≥ threshold. +- **WARN** — rate below threshold. +- **FAIL** — no proposed (on-chain) blocks found in the window (`total_blocks == 0`). +- **SKIP** — no relay supports the data API, or all relays unreachable. + +### `relay.validator_registrations` — tier 3 (relay data API) + +Asserts validators are registered with the relay. Source: `GET +/relay/v1/data/validator_registration?pubkey=…` for each active validator pubkey, per live relay, +aggregated to the worst status. + +- **PASS** — all pubkeys registered. +- **WARN** — some registered, some missing. +- **FAIL** — none registered (`0/total`). +- **SKIP** — pubkey list empty, or all relays unreachable. If the upstream pubkey fetch failed + entirely, the check is **omitted** from the report rather than SKIPped. + +### `payload_hash_match` — tier 1 (relay data API + beacon) + +Cross-checks each relay's delivered `block_hash` against the on-chain hash per slot, and detects +cross-relay disagreement. Source: per-(relay, slot) delivered hashes (kept un-deduped, on purpose) + +`beacon.get_block_hash(slot)`. + +- **PASS** — every observed slot has a relay hash matching the on-chain hash, and no slot has two + relays reporting divergent hashes. +- **WARN** — any slot where no relay hash matched the on-chain hash (`mismatched > 0`, possible + reorg) **or** any slot with a cross-relay hash conflict (`cross_relay_conflicts > 0`, relay + equivocation). The offending slots and relays are named in `data`. +- **SKIP** — no delivered payloads to compare (this check explicitly defers the "was anything + delivered" signal to `relay.payloads_delivered_multi` rather than PASS on zero comparisons). +- `missed` (a delivered slot with no on-chain block) is counted but does **not** downgrade the + verdict — informational only. + +### `relay.best_bid` — tier 2 (CB logs + relay data API) + +Asserts CB delivered at least the best bid it was actually offered across relays (aggregated +bidding). Offered bids come from CB's own `received new header` getHeader log events (`relay_id`, +`slot`, `value_eth` parsed decimal→wei exactly, no float) — the bids CB itself compared. Delivered +values come from the relay data API (the winning payload's value). Comparison is exact wei vs exact +wei. + +- **SKIP** — fewer than 2 relays: no cross-relay aggregation is possible to verify. +- **WARN (not exercised)** — no slot had ≥2 distinct relays offering bids; aggregation never + happened, so nothing is asserted. +- **WARN (not verified)** — competitive slots exist but none had a delivered payload to compare + against (out of window / missed slot); the Law-3 guard against greening having compared nothing. +- **WARN (suboptimal)** — a verified competitive slot delivered **less** than the best offered bid + (value left on the table — late, rejected, or ineligible header). The slots are named in `data`. +- **PASS** — ≥1 verified competitive slot, and every one delivered ≥ its best offered bid. + +### `mux.routing` — tier 1 (CB logs) + +Asserts CB routed each getHeader to the mux/relay the `[[mux]]` config specifies. Only runs when +`--config` was given and the CB config contains `[[mux]]` sections. Source: CB PBS container logs +(`using mux config` DEBUG events carry `mux_id` + `validator`; a routing decision is only "verified" +when a known pubkey is seen with its mux_id). + +- **PASS** — ≥1 routing decision was actually checked and all checked decisions routed to the + expected mux. +- **FAIL** — a checked decision routed a pubkey to the wrong mux/relay (misrouting). Details name + the pubkey, the routed vs expected mux and relay. +- **WARN (no events)** — no mux-related log lines at all; routing could not be observed. +- **WARN (nothing verified)** — mux log events exist but zero routing decisions were verifiable + (`routing_decisions_verified == 0`), typically because CB debug logging is off so there is no + `using mux config` line. Requires `[logs.stdout] level = "debug"`. +- **SKIP** — no `[[mux]]` entries to verify. (A config that fails to parse produces a tier-1 FAIL + from `main.rs`, not a SKIP.) + +### `feature.*` — tier 1 (CB logs), Law-3 feature-fired assertions + +One check per CB feature the `--config` enables, proving the feature's codepath actually fired at +runtime (not just that generic health passed). Source: CB PBS debug logs (the toggle scenarios set +`[logs.stdout] level = "debug"`). Detected by scanning the CB config template for ` = true`. +Each is emitted **only when its feature is enabled** — an off feature produces no check at all. + +- **`feature.timing_games`** (`enable_timing_games`) — **PASS** on ≥1 `TG:` debug line + (`send_timed_get_header`), else **WARN** (enabled but unobserved — maybe no getHeader in the window). +- **`feature.extra_validation`** (`extra_validation_enabled`) — **PASS** on ≥1 `fetched parent block` + / `fetching parent block` line, else **WARN**. +- **`feature.skip_sigverify`** (`skip_sigverify`) — a *negative* codepath (sigverify simply not + called; no success log or metric), indistinguishable from OFF on the happy path — so in plain + scenarios it stays an honest **WARN**. The **cb-sigverify-diff scenario arms a real differential**: + CB's `[[relays]]` url carries a valid-but-WRONG pubkey (a mnemonic validator key, not helix's + `DEFAULT_MEV_PUBKEY`), so `validate_signature` would reject every bid (PubkeyMismatch) — with the + poison detected and ≥1 "auction winner" in CB logs (winners are post-validation), the check + **PASSes**: bids winning is only possible if the skip fired. `cb-sigverify-diff-control` (same + poison, skip OFF) is the expected-FAIL control arm; `sim diff` between the two runs shows the flip. + +A marker feature enabled-but-unobserved is WARN, never FAIL (no-false-red — the same discipline as +`mux.routing`). All three are non-fatal (only a tier-1 FAIL fails the run). + +### `feature.min_bid` — tier 1 (CB logs), config-gated + +Emitted only when the CB config sets `min_bid_eth > 0`. Counts CB's `bid below minimum` rejections +(`ValidationError::BidTooLow`) and the `value_eth` of every `auction winner`. + +- **FAIL** — any auction winner's value is BELOW the floor. That can only happen if the floor was not + applied, and it is the definitive falsifier. **`[pbs]` has no `deny_unknown_fields`** (it must + `#[serde(flatten)]` `PbsConfig`), so a renamed or misspelled key there is *silently ignored* rather + than rejected - this check is the canary for that whole class. +- **PASS** — ≥1 rejection and no sub-floor winner. +- **WARN** — nothing rejected: cannot distinguish "the key was ignored" from "every bid legitimately + cleared the floor", so no false red. + +**The scenario must run with the builder subsidy OFF.** With `mev_builder_subsidy: 1` real bids land +near 1.04 ETH, and CB validates `min_bid_wei < 1 ETH`, so no LEGAL floor could ever reject one and the +scenario would silently prove nothing. `cb-min-bid` therefore sets subsidy `0` (bids ≈ 0.04 ETH of +spamoor MEV) against a 0.5 ETH floor. + +### `signer.pubkeys` — tier 1 (CB signer API), only when a signer is running + +Emitted only when discovery finds a `cb-signer-*` service. Mints an HS256 module JWT and calls +`GET /signer/v1/get_pubkeys`, asserting the KEY COUNT against the devnet's active validator set. + +- **PASS** — the signer loaded every expected key and authenticated the module JWT. +- **WARN** — a partial load: CB warns and continues per keystore, so some were skipped. +- **FAIL** — **zero keys**, or a non-200 answer. Zero is this feature's signature failure: CB's + keystore loaders are `filter_map` + `warn!`, so an unreadable mount yields a perfectly healthy + signer holding nothing. The detail names the likely cause (the devnet's `secrets/` dir is mode 600 + and root-owned; the `teku-keys`/`teku-secrets` pair is the readable one). + +**Why not `/status`.** It is `Ok(StatusCode::OK)` with no logic — 200 with zero keys loaded — and the +metrics server exposes a *second* unconditional `/status`, so probing the wrong port is an even +emptier green. The startup log's `loaded_consensus=N` is log-only (the signer registers exactly one +metric, `signer_status_code_total`, with no key-count gauge) and is ANSI-colored, so the field is not +a contiguous substring. One JWT-authed `get_pubkeys` subsumes liveness, module registration, auth and +key loading. + +### `cb_*_matrix` — tier 2, escalates to tier 1 on FAIL (CB Prometheus) + +Four checks, one per endpoint, built from CB's status-code counters +`cb_pbs_relay_status_code_total` (codes CB received from relays, the source of truth) and +`cb_pbs_beacon_node_status_code_total` (codes CB returned to the CL, surfaced for cross-boundary +diagnosis). Codes bucket into `200 / 202 / 204 / 4xx / 5xx / timeout / transport / other`. **`timeout` is CB's +synthetic code 555** (`TIMEOUT_ERROR_CODE`) and **`transport` is its code 556** (`TRANSPORT_ERROR_CODE`, +introduced with WS get_header streaming: connect refused / dns / tls / stream broke) — neither is a +relay-served status; 555 is CB cancelling its own +request at its deadline; it must never count as relay 5xx (live-confirmed 2026-08-03: timing-games +produced 42% 555s with ZERO real relay 5xx, and the old bucketing tier-1-failed the run). Metrics are +fetched over HTTP, falling back to `kurtosis exec`; if neither works (the usual case — default +kurtosis PBS mode sets no metrics config), **all** matrix checks plus `cb_v2_fallback` and +`cb_relay_latency` SKIP. + +Shared rules across all four: **relay-side 5xx FAILs when it exceeds 25% of COMPLETED responses** +(timeouts excluded from the denominator, so a real error storm still fails amid heavy timeout +polling); at or below the rate it's a transient-warmup WARN, promoted to FAIL under `--strict`. +**CB client-side codes (555 timeouts + 556 ws transport errors) above 25% combined → WARN, never FAIL +— not even under `--strict`** (client-side +deadline policy, e.g. timing-games cancelling late polls by design, or a slow relay). Any matrix FAIL +is escalated from tier 2 to tier 1 so it gates the exit code. **No samples for the endpoint → SKIP.** + +- **`cb_get_header_matrix`** — PASS if any 200 (bids delivered, timeout count noted); WARN if only + 204s (relay alive, no bid — promoted to **FAIL under `--strict`**); FAIL if only 4xx. +- **`cb_register_validator_matrix`** — PASS if 200s and zero 4xx (100% accepted); WARN if a mix of + 200 and 4xx (some batches rejected — normal early on; the beacon-side 502 translation is surfaced); + FAIL if only 4xx; SKIP if no registrations observed; FAIL on any 5xx. +- **`cb_submit_blinded_block_matrix`** — judged on the **BEACON side** (what CB returned to the CL), + not the relay side. PASS if CB served any (200 + 202) to the beacon node; **FAIL on a beacon-side + 5xx** (the proposer did not get its payload); WARN on zero deliveries. Relay-side codes are reported + as context only. + **Why the exception:** CB asks EVERY configured relay for the payload, but only the auction winner + has it, so the losing relays answer 4xx/5xx *by construction*. On a 2-relay run with divergent + subsidies (one relay wins every slot) that produced a 29.7% relay-side 5xx rate and FAILED a run + which delivered 65/65 payloads with 100% MEV rate and 0 missed slots, while the beacon side was + 220x 202 with zero failures. The discriminator still catches the real failure: on nethermind+prysm + the beacon side was 26x 5xx (CB returning 502 to the CL). Falls back to relay-side logic when no + beacon-side samples exist. +- **`cb_status_matrix`** — PASS if any 200; SKIP if no 200s; FAIL on any 5xx. + +### `cb_relay_v2_unsupported` — tier 2, escalates to tier 1 on FAIL (CB Prometheus) + +Reads `pbs_submit_block_v2_unsupported_total{relay_id}`, which CB increments when a relay 404s the +**v2** `submit_block` route. CB deliberately does NOT downgrade to v1 there (in v2 the relay publishes +the block after an empty 202, so forwarding a v1 payload would be silently dropped by the beacon +node), so every affected submission is LOST and the slot is typically missed. + +- **PASS** — counter zero or absent (Prometheus omits never-incremented families). +- **FAIL** (escalated to tier 1) — any nonzero count, naming the relay(s). + +**Read a FAIL as a relay CONFIG problem first.** Found live on nethermind+prysm (2026-08-04): prysm +submits to `/eth/v2/builder/blinded_blocks`, helix 404'd it, and every builder block was lost (11 +events, 11 missed slots) - but helix supports v2 fine; our generated +`router_config.enabled_routes` simply omitted its `GetPayloadV2` route. Lighthouse never triggers this +because it submits via v1. + +### `cb_v2_fallback` — tier 2 (CB Prometheus) + +Asserts relays support submitBlindedBlockV2. Source: +`cb_pbs_submit_block_v2_fallback_to_v1_total`. A missing counter = never incremented = zero +fallbacks. + +- **PASS** — total fallbacks == 0 (includes the missing-counter case). +- **WARN** — any v2→v1 fallback; a relay is behind on the builder-specs v2 upgrade. Never FAIL (v1 + still works), unaffected by `--strict`. + +### `cb_relay_latency` — tier 2 (CB Prometheus) + +Asserts p95 relay latency under threshold. Source: the `cb_pbs_relay_latency` histogram, aggregated +across all `{endpoint, relay_id}` dimensions; standard `histogram_quantile` at q=0.95. Threshold is +hardcoded to 500 ms. + +- **PASS** — p95 < 500 ms. +- **WARN** — p95 ≥ 500 ms. +- **SKIP** — histogram not exposed, zero observations, or degenerate buckets. + +--- + +## The P3 trust-fix notes (the load-bearing WHY behind three WARN gates) + +Three checks were rewritten (DESIGN Law 3 "a +harness that lies green is worse than an ugly one") to kill a **false green** — a PASS reported while +the check had verified nothing. Each now WARNs instead of passing-on-nothing: + +- **`mux.routing`** — the old pass-gate keyed on `total_events` (raw log lines) and PASSed whenever + there were no violations, even with **zero** parseable routing decisions (CB debug logging off). It + reported "all N routing decisions verified" where N was log lines, not decisions. Fixed to gate on + `routing_decisions_verified`; zero verified → WARN, and debug logging is required for mux scenarios. +- **`payload_hash_match`** — the old code was a first-wins union by slot + (`by_slot.entry(slot).or_insert(hash)`) that kept only the first relay's hash and **dropped + cross-relay disagreement** before the on-chain compare, making the verdict order-dependent (honest + relay first → false PASS). Fixed to keep every (relay, slot) hash and detect divergence → WARN, + naming the slot and relays. +- **`relay.best_bid`** — the old check was a first-wins union that counted distinct delivered slots + and never compared bid **values** across relays, so one delivering relay scored identically to + genuine two-relay aggregation. A first rewrite sourced offered bids from + `get_builder_blocks_received`, but adversarial review found that source unsound (it includes builder + submissions that failed simulation and were never offered to the proposer — a false alarm on correct + runs), so it was backed out. The shipped version sources offered bids from CB's `received new + header` log events (what CB actually compared) and WARNs whenever aggregation was not exercised or no + competitive slot could be verified, rather than PASS. + +All three land as `WARN`, which per the verdict model is **non-fatal** — see the consumer caveat in +the intro. + +## Known gaps and caveats (factual, from the code) + +- **Feature-fired assertions (Law 3) — mostly closed.** DESIGN Law 3 wants every scenario to + positively assert its feature's codepath fired. Now five checks do: `mux.routing`, `relay.best_bid`, + and the config-gated `feature.timing_games` (≥1 `TG:` debug line), `feature.extra_validation` (≥1 + parent-block fetch log). The residual gap is **`skip_sigverify`**: it is a *negative* codepath + (signature verification simply not called, with no success log or metric), indistinguishable from + OFF on the happy path with a valid-signature mock relay. `feature.skip_sigverify` therefore reports + an honest **WARN** ("not runtime-confirmable") rather than a false green. Closing it fully needs a + bad-signature-injecting relay in the helix mock (then: ON delivers the bad-sig bid, OFF rejects it). + A marker feature that is enabled but unobserved WARNs (could be no getHeader in the window), never + FAILs — the no-false-red discipline. +- **`relay.best_bid` is inert or degenerate in common setups.** It SKIPs any single-relay run (no + aggregation to verify), and in mux mode — where each mux typically points a validator at exactly one + relay — no slot sees ≥2 relays competing, so it WARNs "aggregation not exercised" rather than + verifying anything. With an identical two-relay setup (e.g. two helix instances serving the same + bid) the comparison is technically competitive but degenerate. +- **The `cb_*_matrix` counters are cumulative, not windowed.** The H2 fix made the 5xx verdict + rate-based (FAIL only above 25% of completed responses; below = transient-warmup WARN), and code 555 + now buckets as `timeout` (WARN-only), so neither a warmup blip nor a designed deadline-cancellation + fails a run anymore. But the counters still cover the container's whole life, not the observation + window — a sustained pre-window error burst can still dominate the rate. True windowing (delta + against a baseline scrape, as `--live-metrics` already takes) remains future work. +- **A real anomaly can still exit 0.** As stated in the intro: relay equivocation, unverifiable + routing, and best-bid shortfall are all `WARN`. Gate on the JSON `result` per check, not the exit + code, if you care about these. +- **Metrics are usually absent.** The default kurtosis PBS mode does not configure CB metrics, so the + six Prometheus-based checks SKIP unless metrics are explicitly enabled. +- **`payload_hash_match` leniency (left as-is under P3 scope).** A delivered-but-not-on-chain slot + (`missed`) is indistinguishable from a transient beacon error and never downgrades the verdict; a + hash mismatch is WARN, not FAIL. + + diff --git a/docs/DESIGN.md b/docs/DESIGN.md new file mode 100644 index 0000000..b12169c --- /dev/null +++ b/docs/DESIGN.md @@ -0,0 +1,141 @@ +# cb-testing — design + +Why this project exists, what it is trying to be, and the engineering laws it holds to. Read this when +a change feels like it is fighting the grain. `docs/ARCH.md` is the companion HOW (module map, seams); +this doc is the WHY and the principles everything else cites as "Law N". + +## What cb-testing is + +An **opinionated block-building simulation substrate for [Commit-Boost](https://github.com/Commit-Boost/commit-boost-client)** +— the layer that stands up a real Ethereum devnet with a relay + builder + commit-boost sidecar, +exercises a specific PBS/ePBS feature end to end, and returns a trustworthy verdict you can gate a +release on. Nothing here is a mock; the point is not "the devnet booted," it is a verdict. + +Mainline `ethpandaops/ethereum-package` treats out-of-protocol block building as a bespoke, hard-coded +convenience; it has no incentive to keep commit-boost first-class, and **ePBS will churn exactly this +surface** (relay / builder / sidecar wiring). So cb-testing maintains it, opinionated about +commit-boost, and aims to be the integration layer the release flow never had. + +## The problem it solves + +The release loop for a sidecar like this is otherwise: unit tests → ship → wait for testnet feedback → +release. Weak coverage, slow. The integration layer that should sit in the middle is this repo, but the +earlier version was built before test-driven discipline, so it was neither trustworthy nor fast enough +to lean on. As ePBS clients ship, the issue rate rises, and the human-in-the-loop version (paste +terminal output, `docker logs` a container by hand, isolate the real panic under three masking errors) +does not scale. **The win is a loop that can be driven end to end: launch → triage → diagnose → report, +autonomously — legible to a human and to an automated agent from the same output.** + +## The thesis (the mechanism, not a rewrite) + +The leverage is NOT "rewrite it in Rust." It is three properties, in priority order: + +1. **Preflight against the real images.** Validate a rendered config by having the actual relay / + commit-boost image parse it in ~1s, before any 10-minute devnet spend. This is the generalized, + productized form of the manual `docker run --rm ` loop that first found real schema + drift. It is the single biggest legibility win and it retires the whole class of "schema drift + discovered 5 minutes into a launch as a masked runtime panic." +2. **Observable by default (not agent-only tooling).** Legibility is a system property: structured + `tracing` events on everything, a durable verdict report, and AUTOMATIC root-cause capture on any + failure, emitted as the normal output of a normal run. A human reads a pretty rendering of that + stream; an automated consumer reads the JSON; ONE source of truth, not a separate agent surface. So + root-cause capture is a PROPERTY OF THE RUN (when a service dies, the harness attaches its + container's root panic to the event stream automatically) — the `triage` verb is only the + after-the-fact entry point into that same data, not the mechanism. This is strictly better for a + human too (root panic inline vs `docker logs` by hand). +3. **Feature-asserting scenarios.** Every scenario positively asserts its feature's codepath FIRED + (skip-sigverify counter > 0, timing-game poll count, extra-validation RPC hit, cross-relay best-bid + comparison), not merely "the pipeline didn't crash." A scenario that passes while the feature + silently no-oped is a non-test. + +Type-reuse from the real config crates is a supporting move (see Law 1), valuable but scoped: it makes +commit-boost config drift a compile error; it does NOT solve helix drift (Law 1 caveat). Preflight is +what covers both halves. + +## Design laws + +Non-negotiable; each prevents a named smell. Numbered and quotable — other docs and code cite them as +"Law N." + + +### Law 1 — Configs come from the real schemas, never hand-mirrored strings + +A string-template generator that reverse-engineers a service's serde layout from binary panics is the +root smell and does not survive. Commit-boost config is built from `cb_common` config structs and +serialized, so a renamed field is a compile error / `deny_unknown_fields` deserialize error. + +*Caveat:* helix types are NOT reusably importable (divergent branch, different org, a `teloxide` type +graph), and helix `CoresConfig` is the field that drifts most. So helix gets an owned, thin +`HelixRelayConfig` mirror pinned in lockstep with the `HELIX_RELAY_IMAGE` tag, guarded by the Preflight +law. The few kurtosis-runtime template holes (`{{ .Timestamp/.Port/.Network/.Relays }}`) are the one +place string-patching survives; isolate them. (Note: because a hand-written mirror is still +hand-mirrored, preflight — not the mirror — is the real guard for helix; see Law 1's interaction with +the config-generation history in `docs/ARCH.md` §4.) + + +### Law 2 — No config, image name, or scenario truth exists in two hand-edited places + +One image map (no four-way `commit-boost/pbs` vs `commit-boost/commit-boost` drift). No checked-in +stale example config that hand-copies generator output; generate on demand instead. + + +### Law 3 — Every scenario asserts its feature fired + +No green from mere block delivery. A pass-gate must key on the signal that proves the feature ran +(e.g. `pubkeys_verified`, not raw `total_events`); a best-bid check must compare bid values across +relays, not union-by-slot (one delivering relay must not pass identically to genuine two-relay +aggregation). A harness that lies green is worse than an ugly one. + + +### Law 4 — Verdict logic is unit-tested and TDD-able without a devnet + +The checks are pure functions over already-fetched data; inject fixture beacon/relay responses and +test the math. Every judgement splits into a pure classifier (data in, verdict out, unit-tested on both +sides of every boundary) and a thin I/O shell holding zero verdict logic. CI runs the generator and +schema-validates every output. + + +### Law 5 — Observability is a first-class system property, not an agent bolt-on + +Everything emits structured `tracing` events + a durable verdict report; failures auto-attach their +root cause to that stream. Humans and automated consumers tap the SAME surface (pretty rendering vs +JSON) — never a separate agent-only tool category; build good logging on everything and let both +consume it. The `VerificationReport` is the model to extend, not replace. + + +### Law 6 — Dogfood the abstraction in one fork; upstreaming is optional icing later + +The `(relay, sidecar, builder)` component model (`mev_resolver.star`) + the `mev_type: custom` config +API already exist in the fork, and cb-testing already consumes them — so this is maturing what exists, +not a new build, and there is no rush to upstream. Topology: ONE fork carries the abstraction; +cb-testing is its consumer/dogfood (do NOT maintain two forks — the abstraction-vs-usage seam is the +fork/consumer boundary, not two forks of one repo). "Do it properly" means: + +- (a) refine the abstraction to be clean and GENERAL (reads like an API a stranger would use: any relay + × any sidecar × any builder, already spans epbs/buildoor/mev-rs); +- (b) add the external-sidecar hook (let VCs point at an externally-supplied builder URL; make each + component independently `none`-able) — the one missing piece that later enables a thin + compose-over-UNMODIFIED-upstream shim; +- (c) keep the fork delta minimal and PR-shaped as you touch it. + +OPTIONAL LATER: upstream the proven code (a medium PR — the `mev_resolver.star` module + a refactor of +`main.star`'s mev dispatch + `input_parser.star`'s per-client builder-flag matrix), then cb-testing +repoints its `import_module` from the fork to upstream in one line. A pure shim is not possible on +today's upstream because upstream injects the VC `--builder` flag inside `enrich_mev_extra_params`, +triggered only by a native `mev_type` via a naming-convention URL, with no external-builder hook — a +shim would otherwise reimplement the brittle per-client flag matrix, worse than the fork. (The `#1384` +"exit" referenced in earlier audits is UNVERIFIED — check upstream HEAD before any PR.) + + +### Law 7 — Coverage is a matrix, not a point + +Scenarios parametrize over EL/CL client pairs. A regression specific to nethermind+prysm is invisible +if everything hardcodes geth+lighthouse. + +## Where to go next + +- `docs/ARCH.md` — how the pieces fit (module map, the config↔fork seam, the verdict model). +- `docs/CHECKS.md` — the per-check catalog (tiers, thresholds, feature-assertion status). +- `docs/DEVELOPING.md` — the dev loop + how to add a check or a scenario. +- `docs/fork-delta.md` — what the `ethereum-package` fork changes vs upstream, file by file. +- `docs/local-kurtosis-e2e.md` — the operational runbook. diff --git a/docs/DEVELOPING.md b/docs/DEVELOPING.md new file mode 100644 index 0000000..0517f85 --- /dev/null +++ b/docs/DEVELOPING.md @@ -0,0 +1,207 @@ +# cb-testing — contributor guide + +How to get the repo running and how to extend it — add a **check** (a new verdict on the pipeline) or a +**scenario** (a new devnet configuration that exercises a CB feature). This is the how-to-develop doc; it +does not re-explain the architecture or the check catalog: + +- **[`docs/ARCH.md`](ARCH.md)** — how the pieces fit (module map, the config↔fork seam, the verdict model). +- **[`docs/CHECKS.md`](CHECKS.md)** — the authoritative per-check catalog + the verdict contract for consumers. +- **[`docs/DESIGN.md`](DESIGN.md)** — why the repo exists + the design laws referenced below (Law 1 + real-schema configs, Law 3 feature-asserting scenarios, Law 4 TDD-able verdicts, Law 5 observability). +- **[`docs/local-kurtosis-e2e.md`](local-kurtosis-e2e.md)** — the operational runbook + kurtosis pin. + +--- + +## 1. Dev loop / prerequisites + +### Toolchain + +- **Rust 1.91+**, edition 2024 (`Cargo.toml` pins `rust-version = "1.91"`). No Node, no bun — this is a pure + Rust workspace (one lib + five bins: `cb-verify`, `cb-orchestrator`, `sim`, `test-mux`, `test-relay`). +- **Docker + Kurtosis CLI 1.18.1** — only needed for the devnet e2e, not for unit tests. Pin 1.18.1 (the + parsers read its human text tables; a newer CLI has a config-version clash — see the runbook). +- The forked `ethereum-package` submodule: `git submodule update --init` (only needed to launch a devnet). + +### The `just` recipes (the whole dev loop) + +`just` is the task runner (`justfile` at the repo root). The inner loop is pure-Rust and hermetic — no +Docker, no network, sub-second: + +| Recipe | What it runs | When | +|---|---|---| +| `just check` | `cargo check --all-targets` | fast compile check, no codegen | +| `just test` | `cargo test` | all unit tests — **pure, fast, hermetic** (no devnet, no docker) | +| `just fmt` / `just fmt-check` | `cargo fmt` / `cargo fmt --check` | format / CI format gate | +| `just clippy` | `cargo clippy --all-targets -- -D warnings` | strict lint (**warnings are errors**) | +| `just lint` | `fmt-check` + `clippy` | the pre-commit gate | +| `just ci` | `check` + `test` + `lint` | the full local CI pipeline — run this before pushing | +| `just build-release` | `cargo build --release` | release binary | +| `just generate-configs` | `sim generate` | regenerate `configs/generated/*.yml` from the typed model | + +The two gates that CI enforces (mirror them locally): **`cargo fmt --check`** and **`clippy -D warnings`**. +A `just ci` green means the pure surface is good; it says nothing about the devnet path. + +### Unit tests vs the devnet e2e + +- **`cargo test` / `just test` — the everyday loop.** Every verdict, parser, and config assembler has a pure + core that is unit-tested against fixture data (`tests/fixtures/`, `#[cfg(test)]` modules in each source + file). No devnet, no Docker, no images. This is where you do TDD. Runs in seconds. +- **`just e2e` — the full devnet confirmation.** Regenerates configs, pulls the public images, launches a + Kurtosis enclave (geth + lighthouse + N helix relays + reth-rbuilder + the CB sidecar + dora/spamoor/ + prometheus), observes ~1 epoch, runs every check, prints the report. Needs Docker + Kurtosis 1.18.1 and a + **locally-built CB image** — run `just build-cb-image` once first (it builds `commit-boost/commit-boost:kurtosis` + from the sibling `../commit-boost-client` repo; the helix + reth images are public and pulled). A run is + ~10 minutes; it is the *final* confirmation, never the debugger. See [`docs/local-kurtosis-e2e.md`](local-kurtosis-e2e.md). + +Middle ground: `just test-mux`, `just verify-now`, and `sim preflight ` (the ~1s real-image config +probe) run against a live enclave without a full observation window — useful for iterating on a running devnet. + +--- + +## 2. How to add a CHECK + +A check is a pure verdict over already-fetched pipeline data. The pattern (**DESIGN Law 4** — "verdict +logic is TDD-able without a devnet"; the check-trustworthiness plan calls it the `classify_*` seam): + +### The two-part shape + +**(a) Write the pure classifier — the seam.** A free function that takes *already-fetched* data (maps, +counts, parsed log events — never a network client) and returns a `CheckResult`: + +```rust +pub fn classify_(data: &AlreadyFetched) -> CheckResult { … } +``` + +`CheckResult` and its constructors live in [`src/checks/mod.rs`](../src/checks/mod.rs): +`CheckResult::{pass, fail, warn, skip}(id, tier, detail)` plus `.with_data(json)`. `id` is the check's stable +name (also its key in the JSON report), `tier ∈ {1,2,3}` (§ tier choice below), `detail` is a human string, +`data` is a `serde_json::Value` for machine consumers. `CheckStatus` serializes UPPERCASE under the JSON key +`result` (`PASS`/`FAIL`/`WARN`/`SKIP`). + +Make the classifier generic over the value type where it helps testing — e.g. +`classify_payload_matches` takes `BTreeMap>` so tests pass a trivial `u64` in place +of a real `B256` hash. + +**(b) Unit-test both sides of every boundary.** For each Pass/Warn/Fail/Skip transition, write a test with +fixture data that lands just inside and just outside the boundary. The worked example +[`src/checks/payload_matching.rs`](../src/checks/payload_matching.rs) tests: clean single-relay match → PASS, +cross-relay conflict → WARN, no-relay-matches-chain → WARN, missed-not-downgraded → PASS, empty → SKIP. Build +the fixture maps with small helpers (`by_slot(...)`, `chain(...)`) so each test is one line of intent. + +**(c) Write the thin async fetch shell.** A separate `async fn` that does the I/O (calls `BeaconClient` / +`RelayClient` / metrics / `kurtosis logs`), assembles the same data shape, and calls the classifier. It holds +**no verdict logic** — it just gathers and delegates. In `payload_matching.rs` that is +`check_payload_hash_match(...)`: fetch per-(relay,slot) hashes + on-chain hashes, then `classify_payload_matches(...)`. + +**(d) Wire it into `run_verification`.** In [`src/main.rs`](../src/main.rs) (~lines 414-526) the checks are +collected into one `Vec`. Add your fetch fn there, either via a module `run_*` that returns +`Vec` (`all_checks.extend(...)`, as chain_health / relay_pipeline / payload_matching / +cb_metrics do) or a single `all_checks.push(...)` (as best_bid and mux_routing do). Register the module in +`src/checks/mod.rs` if it is new. + +### Worked examples (read these, don't invent a new shape) + +- **[`src/checks/cb_metrics.rs`](../src/checks/cb_metrics.rs)** — `collect_endpoint_stats(scrape, endpoint) → + EndpointStats` (pure gather over parsed Prometheus) then `classify_endpoint(endpoint, &stats, strict) → + CheckResult` (pure verdict, ~19 decision tests, no known false-greens). The cleanest seam in the repo. +- **[`src/checks/mux_routing.rs`](../src/checks/mux_routing.rs)** — `parse_cb_log_line` + + `extract_mux_from_config` + `classify_mux_routing` (pure); the async shell fetches CB logs. +- **[`src/checks/best_bid.rs`](../src/checks/best_bid.rs)** — `classify_best_bid` (+ `value_eth_to_wei`), + fed by offered bids parsed from CB's own getHeader log events. + +### Pick the tier deliberately + +The tier is the **severity contract** — it decides whether your check can fail the run (full table: +[`docs/CHECKS.md`](CHECKS.md)): + +- **Tier 1 = must** — a real pipeline invariant. A tier-1 `FAIL` fails the whole run (exit code 1). Use only + for "the pipeline is genuinely broken." +- **Tier 2 = should** — a health signal. Never fails the run on its own. (`cb_metrics` matrix checks are the + one exception: authored tier 2 but *escalated to tier 1 on FAIL* because a relay 5xx is a real failure.) +- **Tier 3 = informational** — annotative only. + +Note the trust rule: a check that exists to *catch an anomaly* should prefer **WARN**, not a silent PASS, +when it could not actually verify anything (the P3 false-green fixes: `mux.routing`, `payload_hash_match`, +`relay.best_bid` all WARN rather than pass-on-nothing). WARN is non-fatal, so surface it in `data` and let the +consumer gate on the JSON `result` (§4). + +### The anti-pattern (why the seam is non-negotiable) + +**Do not weld the verdict into the async fetch fn.** `chain_health` and `relay_pipeline` inline their verdicts +in the async check fns and have *no* factored-out classifier — which is exactly the standing gap the +check-trustworthiness plan exists to close. A verdict +tangled with `await` calls cannot be unit-tested without a devnet, so its pass/fail boundaries go unproven — +which is how false-greens ship. Pure classifier first, thin I/O shell second, always. + +--- + +## 3. How to add a SCENARIO + +A scenario is a typed devnet configuration that assembles into a Kurtosis args-file. Everything lives under +[`src/bin/sim/genmodel/`](../src/bin/sim/genmodel/); the assembly is pure and guarded by byte-identity golden +fixtures. The config↔fork coupling (the two `|` block scalars, the runtime template holes) is explained in +[`docs/ARCH.md`](ARCH.md) §4 — read it before touching the block bodies. + +### Steps + +1. **Add the variant** to the `Scenario` enum in + [`src/bin/sim/genmodel/scenario.rs`](../src/bin/sim/genmodel/scenario.rs), and add it to `Scenario::ALL` + (order = emission order — match the intent; the array is what `sim generate` iterates). +2. **Fill the match arms** for the new variant: `name()` (the canonical basename, e.g. `cb-myfeature`), + `comment()` (the leading doc block), `relays()` (`&["helix"]` single-relay vs `&["helix", "helix"]` + multi-relay — this also toggles the scalar-vs-list `mev_relay` form and `mev_relay_image` emission), + `cb_block()` (the CB TOML — see next step), and `network_params()` (only if you need a different validator + count; `Mux` is the sole scenario using `MUX_NETWORK_PARAMS` = 256 keys). +3. **Build the CB TOML.** Most scenarios just construct a `CbParams` in `cb_block()` and call + `cb_toml(¶ms)` — the knobs are `timeout_get_header_ms`, `timeout_get_payload_ms`, `extra_pbs_lines` + (injected into `[pbs]`), `per_relay_lines` (injected inside the `{{ range }}` relay loop). See + [`src/bin/sim/genmodel/cb.rs`](../src/bin/sim/genmodel/cb.rs): skip-sigverify adds one `[pbs]` line, + extra-validation adds two, timing-games sets short timeouts + three per-relay lines. Only add a whole new + builder (like `cb_toml_mux`) when the TOML *structure* changes (mux moves the range loop and adds `[[mux]]` + blocks). Generate-time knobs are injected by plain string-building, **not serde** — no quoting/sentinel hazard. +4. **The helix block is shared.** `HELIX_RELAY_CONFIG` in + [`src/bin/sim/genmodel/helix.rs`](../src/bin/sim/genmodel/helix.rs) is byte-identical across all scenarios. + Only touch it if the *relay* config itself must change — and if you do, keep it pinned in lockstep with the + `HELIX_RELAY_IMAGE` tag (Law 1 caveat: helix types aren't importable, so this const *is* the contract) and + preserve the hard-won comments verbatim. +5. **Regenerate + add the golden fixture.** Run `just generate-configs`, then copy the new + `configs/generated/cb-myfeature.yml` to `tests/fixtures/golden-configs/cb-myfeature.yml`. The test + `every_scenario_matches_its_golden` (in `scenario.rs`) then asserts `sim generate` reproduces it + **byte-for-byte** with the default images. +6. **The drift gate.** `sim generate --check` (`generate::check`) is the CI/agent form of the same guard — it + fails if the on-disk configs no longer match the generator. Run it after any generator change. + +### Two caveats + +- **Multi-relay goldens are self-generated.** The byte-identity oracle only proves the generator reproduces + *its own* output; it does **not** prove the config is a valid, working devnet. Validate a new scenario on a + real devnet: `sim preflight configs/generated/cb-myfeature.yml` (~1s real-image parse) and then + `just e2e configs/generated/cb-myfeature.yml`. +- **Assert the feature fired (Law 3).** A scenario that passes while its feature silently no-oped is a + non-test. A new scenario should ship with a check that *positively asserts its codepath fired* — a + skip-sigverify counter > 0, a timing-game poll count, an extra-validation RPC hit — not just the generic + pipeline checks. Today only `mux.routing` and `relay.best_bid` do this (the gap is documented in + [`docs/CHECKS.md`](CHECKS.md) "Known gaps"); adding the assertion is § 2 above. + +--- + +## 4. The verdict contract (for consumers) + +A run emits a `VerificationReport` (human-rendered, or `--json`) and an exit code. **The exit code keys only +on a tier-1 FAIL**: `0` = no tier-1 failure, `1` = some tier-1 check FAILed, `2` = no tier-1 check ran at all +(a setup/discovery/preflight failure). `WARN` and `SKIP` are **non-fatal at every tier** and never move the +exit code. The load-bearing consequence: several checks that exist to catch a real anomaly report it as +`WARN` (relay equivocation, unverifiable mux routing, best-bid shortfall) — a run that hits them **still exits +0**. A CI job or agent that cares about those must **parse the JSON and inspect each check's `result` field**, +not gate on the exit code alone. Full contract, per-check pass/warn/fail conditions, and the one tier-2→tier-1 +escalation: [`docs/CHECKS.md`](CHECKS.md). + +--- + +## 5. Repo map — where things live + +Do not reverse-engineer the tree; the module-by-module map is **[`docs/ARCH.md`](ARCH.md) §2–3** (shared lib +`src/lib.rs`, the `cb-verify` binary `src/main.rs`, the `sim` submodules under `src/bin/sim/`, and the +config↔fork seam). The check catalog is [`docs/CHECKS.md`](CHECKS.md); the fork divergence is +[`docs/fork-delta.md`](fork-delta.md); the current backlog of what to build next is the internal +the local `.agent/` working area (backlog + plans index). diff --git a/docs/fork-delta.md b/docs/fork-delta.md new file mode 100644 index 0000000..9c0c372 --- /dev/null +++ b/docs/fork-delta.md @@ -0,0 +1,98 @@ +# ethereum-package fork delta + +The vendored submodule at `ethereum-package/` is a fork of `ethpandaops/ethereum-package` +(fork origin: `github.com/Commit-Boost/ethereum-package`, currently detached at `fbe3141`). +The fork's own README / CHANGELOG / architecture read as stock upstream, so the divergence is +recoverable only from `git log`. This file makes that delta legible for a future rebase or +upstream PR. Cited against real commits and files; nothing here is committed by the doc itself. + +## 1. Why the fork exists + +Mainline `ethpandaops/ethereum-package` treats out-of-protocol block building as a bespoke, +hard-coded convenience. It has no incentive to keep commit-boost first-class, and **ePBS will +churn exactly this surface** (relay / builder / sidecar wiring). cb-testing owns the opinionated +block-building simulation substrate for commit-boost, so it maintains this fork opinionated about +commit-boost rather than waiting on upstream. See `docs/DESIGN.md` ("What cb-testing is") and Law 6. +The bet is explicitly "own it"; whether the fork investment is worth it long-term vs waiting on +ethpandaops#1384 is an open question — revisit if #1384 lands. + +## 2. The delta, file by file + +Fork-authored commits, newest first: `fbe3141` (N relay instances + 8GB cap), `43fe436` +(helix wait-for-genesis), `4844f88` + `1dbaa38` (disable zkboost), `022951e` (commit-boost +prometheus), `7efe6fe` (the custom-mev component model — the core IP). `1ecc324` is the last +`upstream/main` merge; `7efe6fe`'s parent `eac08c0` is upstream, so `7efe6fe` is the first fork +commit onto the upstream base. + +| File | Commit | What changed | Why | +|---|---|---|---| +| `src/package_io/mev_resolver.star` (**new, 168 lines — the core IP**) | `7efe6fe` | Adds the `(relay, sidecar, builder)` component decomposition. `resolve_mev_components(mev_type, mev_params)` expands a preset `mev_type` via `MEV_PRESETS` OR, for `mev_type: "custom"`, reads explicit `mev_params.{mev_relay,mev_sidecar,mev_builder}`. Each component is independently `none`-able. Validates against `VALID_RELAYS/SIDECARS/BUILDERS`, normalizes relay to a list, and hard-errors impossible combos (e.g. `builder=flashbots` with all relays `none`). Helpers `get_sidecar_service_prefix`, `get_relay_image`. | Lets any relay × any sidecar × any builder mix without patching code (e.g. helix relay + commit-boost sidecar). Presets (`flashbots`/`helix`/`commit-boost`/`mev-rs`/`mock`/`buildoor`/`epbs`) stay as shortcuts; `custom` is the general API. | +| `main.star` | `7efe6fe` | Rewrote the MEV dispatch (~350 lines changed / net ~-43). Reads `args_with_right_defaults.mev_components`; a **relay-launch loop** iterates `mev_components.relay` with a `relay_index` counter, launching each relay service (helix/flashbots/mev-rs) at `index = num_participants + relay_index`; then a **per-validator sidecar loop** keyed on `mev_components.sidecar` (`mev-boost`/`commit-boost`/`mev-rs`/`none`); `builder == "buildoor"` and `sidecar == "none"` (ePBS) are branches. | Replaces the old single-hardcoded `mev_type` switch with the component model. The `num_participants + relay_index` suffix is the seam the 2-helix design relies on (§3). | +| `src/package_io/input_parser.star` | `7efe6fe` | Calls `mev_resolver.resolve_mev_components(...)` and threads `result["mev_components"]` through; surfaces `mev_relay`/`mev_sidecar`/`mev_builder` params. (~182 lines touched.) | Wires the resolver into the parsed args so `main.star` consumes a resolved struct, not raw `mev_type`. | +| `src/package_io/constants.star` | `7efe6fe` | Adds `CUSTOM_MEV_TYPE = "custom"`, `EPBS_MEV_TYPE = "epbs"`. `DEFAULT_HELIX_RELAY_IMAGE = "ghcr.io/gattaca-com/helix-relay:main"` (untagged `:main`). | New mev_type identifiers. The `:main` pin is why the helix config schema source-of-truth is the running binary's serde metadata, not any checked-in mirror (§4). | +| `src/mev/helix/helix_relay_launcher.star` | `43fe436`, then `fbe3141` | (a) **wait-for-genesis wrapper**: passes `GENESIS_TIME` env and sets `entrypoint=["sh","-c"]` + a cmd that `until [ "$(date +%s)" -ge "$GENESIS_TIME" ]; do sleep 1; done; exec /app/helix-relay --config ...`. (b) **N-instance suffixing**: service, postgres (`helix-relay-postgres-{index}`), and config-artifact names all suffixed by the per-instance `index`. (c) `RELAY_MAX_MEMORY` 4096 → **8192**. | (a) Latest `:main` helix **panics in `HousekeeperTile::new -> current_slot().unwrap()`** if it boots before genesis; the shell wrapper blocks until genesis (sh + date exist in image; `exec` preserves PID 1 / signals). (b) Lets two helix entries launch as `helix-relay-N`/`helix-relay-N+1` without name collision. (c) Both relays were **cgroup-OOM-killed (CONSTRAINT_MEMCG)** at the 4GB cap ~9min into a spamoor devnet; 8GB clears the window. | +| `src/mev/flashbots/mev_builder/mev_builder_launcher.star` | `fbe3141` | rbuilder config template now takes `participant_count`; emits **per-instance helix targets** with `Name=helix-{suffix}` / `Service=helix-relay-{suffix}` where `suffix = participant_count + relay_index`, mirroring `main.star`'s relay-launch loop verbatim (relay_index increments for every non-`none` relay). `Priority` keyed on `relay_index`. | flashbots rbuilder is kept as the BUILDER even when helix is the relay; its submission targets must resolve to the actually-launched helix service names. | +| `src/mev/commit-boost/mev_boost/mev_boost_launcher.star` | `022951e` (+ minor `7efe6fe`) | Adds a `metrics` port (9090), `CB_METRICS_PORT=9090` env. | Expose commit-boost prometheus metrics. | +| `static_files/mev/commit-boost/cb-config.toml.tmpl` | `022951e` | Adds `[metrics] enabled=true host="0.0.0.0" start_port=9090`. | Same — turn CB metrics on in the rendered config. | +| `main.star` (prometheus scrape jobs) | `022951e` | When `mev_components.sidecar == "commit-boost"`, appends a `commit-boost-{idx}` scrape job (`{ip}:9090/metrics`, 15s) per mev-boost context. | Prometheus actually scrapes the CB sidecars. | +| `main.star` (zkboost import + dispatch) | `1dbaa38`, `4844f88` | The `zkboost` import and its `additional_service == "zkboost"` launch branch are **commented out / stubbed** (`GpuConfig` is undefined on the upstream base). | zkboost is dead weight for this fork; kept commented (not deleted) to minimize rebase conflict churn. Flag for rebase: this is a stub, not a feature. | +| `static_files/mev/helix/config.yaml.tmpl` | `7efe6fe` | Trimmed (~-12 lines). | Config template reconciled with the `:main` helix serde layout. | +| `.github/tests/mev-custom-helix-cb.yaml` (**new**) | `7efe6fe` | Test scenario exercising `mev_type: custom` = helix relay + commit-boost sidecar. | Regression coverage for the component API. | +| `README.md`, `network_params.yaml`, `sanity_check.star`, `reth_launcher.star`, `participant_network.star` | `7efe6fe` | Doc/param/sanity plumbing for the new mev fields. | Supporting edits for the component model. | + +## 3. The 2-helix design + +The default topology drops the flashbots **relay** (its mev-boost-relay leaks ~825MB/min under +spamoor) and runs **two helix relays**, while still using flashbots **rbuilder** as the builder. +It works because everything downstream of the relay is positional and index-threaded: + +- `main.star`'s relay-launch loop launches each non-`none` relay at + `index = num_participants + relay_index`, so two `helix` entries in `mev_components.relay` + become services `helix-relay-N` and `helix-relay-N+1` (with matching `helix-relay-postgres-N` + and `helix-relay-config-N` artifacts) — no collision (`fbe3141` + `7efe6fe`). +- Ports / relay URLs / the commit-boost `[[relays]]` list / the mux routing were already + positional, so no per-instance divergence is needed there. +- The **critical invariant**: `mev_builder_launcher.star` recomputes the exact same suffix + (`participant_count + relay_index`, mirroring the launch loop) so the rbuilder `Service` names + (`helix-relay-{suffix}`) resolve to the real launched services. If the two loops ever drift in + how they count `relay_index` (note: a `mev-rs` relay still *consumes* an index slot but is not + emitted into rbuilder config), block submission silently targets a nonexistent service. + +Validated (`fbe3141` msg): the `cb-multiple-relays` devnet brings up `helix-relay-2` + +`helix-relay-3`, both survive, CB sees 2 relays (33 competitive bid slots). + +## 4. Rebase / maintenance notes + +- **No `upstream` remote is configured.** `git remote -v` shows only `origin = + Commit-Boost/ethereum-package`. A rebase today has nothing to rebase against without first + `git remote add upstream https://github.com/ethpandaops/ethereum-package`. +- **The fork is not tag-pinned.** The submodule is a bare detached HEAD at `fbe3141`; the many + `git tag` entries are inherited upstream release tags, not a fork pin. +- **The planned fork diet + treadmill** wants an `upstream` remote + a tagged pin, and wants + that pin moved **in lockstep with `HELIX_RELAY_IMAGE`** (`DEFAULT_HELIX_RELAY_IMAGE = + ...helix-relay:main` in `constants.star`). +- **Schema source-of-truth for helix is the `:main` binary's serde metadata, not the fork + checkout.** Helix types are not reusably importable (divergent branch / different org), and the + helix config drifts against whatever `:main` currently deserializes — the wait-for-genesis and + config-template fixes exist precisely because a checked-in mirror lags the deployed binary + (DESIGN Law 1 caveat). Reconcile config changes by parsing against the actual + image (the Preflight law), not by editing to match a stale local checkout. + +## 5. What's already upstream-PR-shaped + +**DESIGN Law 6** describes a **medium PR**: the `mev_resolver.star` component module + the +`main.star` mev-dispatch refactor + the `input_parser.star` per-client builder-flag matrix. +That code already exists here (`7efe6fe`) and cb-testing already consumes it, so upstreaming is +maturing-what-exists, not a new build — and there is no rush (Law 6, RATIFIED). + +**Law 6b** — the external-builder hook + independent `none`-ability of each component — is +**already present**: `resolve_mev_components` lets `relay`/`sidecar`/`builder` each be `"none"` +independently (see `VALID_*` lists and the `epbs` preset `{relay:none, sidecar:none, +builder:buildoor}`), which is the missing piece Law 6b called out as the enabler for a future +thin compose-over-unmodified-upstream shim. What is *not* yet done: a true external-supplied +builder URL hook (VCs pointing at an arbitrary external builder) — `get_relay_image` / the +builder branches still resolve known images. + +Note (Law 6): the `#1384` "exit" referenced in earlier audits is **unverified** — check upstream +HEAD before opening any PR, and confirm upstream still injects the VC `--builder` flag inside +`enrich_mev_extra_params` (the reason a pure shim isn't possible on today's upstream). diff --git a/docs/local-kurtosis-e2e.md b/docs/local-kurtosis-e2e.md new file mode 100644 index 0000000..0c0a152 --- /dev/null +++ b/docs/local-kurtosis-e2e.md @@ -0,0 +1,141 @@ +# Local Kurtosis e2e for commit-boost (PBS / ePBS) — setup runbook + +> Living doc. Captures the *actual* working steps to build a local commit-boost image, +> deploy a Kurtosis devnet, run a PBS simulation, and verify it end-to-end on this box. +> Written as we prove each step, so it reflects reality, not theory. +> Reference box: Linux, Docker 29.6, 32 cores / 60 GB. Status: **WORKING — cb-basic PASSES**. +> +> First green e2e run: v0.11.0 SSZ commit-boost image (`commit-boost/commit-boost:kurtosis` from `main`), +> kurtosis 1.18.1, cb-basic. cb-verify overall = PASS (13 PASS / 1 SKIP finality / 1 WARN relay-latency +> p95 914ms on a loaded box / 0 FAIL): 33 payloads delivered + 33/33 hash-matched on-chain, 62 get_header +> bids, 62 submit_blinded_block 200s, 100% MEV delivery, 128/128 validators registered, 0/32 missed slots. +> Getting here required the kurtosis 1.18.1 pin + 3 latest-helix fixes (network_config, cores, pre-genesis +> wait) — all below. + +## The two flows (pick one) +- **cb-testing (this repo) — the verification harness.** `cb-verify` Rust binary with tiered + pass/fail checks, 6 scenarios, forked `Commit-Boost/ethereum-package` submodule, helix relay. + This is the authoritative flow (success criteria baked in). **We use this.** +- **commit-boost-client `just kurtosis-*`** — lighter alt: upstream `ethereum-package`, + `mev_type: commit-boost`, mev-boost-relay, no pass/fail harness. Good for a quick smoke. + +## Prerequisites (this box) +- Docker + buildx: PRESENT (29.6). +- Rust 1.91 (cb-testing pins `edition 2024`, `rust-version 1.91`): repo toolchain present. +- Kurtosis CLI >= 0.90: **TODO install** (see below). + +## Step 0 — Install Kurtosis CLI (STATUS: pending approval) +Documented + CI-proven recipe (from cb-testing `.github/workflows/integration.yml`): +```bash +echo "deb [trusted=yes] https://sdk.kurtosis.com/kurtosis-cli-release-artifacts/ /" \ + | sudo tee /etc/apt/sources.list.d/kurtosis.list +sudo apt update && sudo apt install -y kurtosis-cli +kurtosis analytics disable +kurtosis version # verify; record the version here: ____ +``` + +## Step 1 — Build the commit-boost image from `main` (STATUS: done, CONFIRMED) +`main` = the v0.11.0 SSZ content (#467/#468/#481/#482 SSZ + #465 Stader + #480 Dirk), tip `635384f`. +```bash +cd ../commit-boost-client && git checkout main && git pull --ff-only origin main +just build-all kurtosis # -> image commit-boost/commit-boost:kurtosis (crate=commit-boost) +``` +Image name is `commit-boost/commit-boost:` (NOT `pbs:*`; the `.env.example` confirms +`commit-boost/commit-boost:latest` is the default). Long first build (full Rust release compile in +docker buildx). Verify: `docker image inspect commit-boost/commit-boost:kurtosis`. + +## Step 2 — Prepare cb-testing (STATUS: done, CONFIRMED) +```bash +cd . +git submodule update --init --recursive # pull forked ethereum-package @ 4844f884 (was EMPTY) +# .env — override BOTH images (see gotcha): +# MEV_BOOST_IMAGE=commit-boost/commit-boost:kurtosis (local CB build) +# HELIX_RELAY_IMAGE=ghcr.io/gattaca-com/helix-relay:main (public; we don't build helix) +just generate-configs # -> configs/generated/*.yml +``` +**GOTCHA (confirmed):** the generator defaults `helix_relay_image` to the LOCAL tag +`helix-relay:kurtosis`, which does not exist unless you build helix (helix lives in the ws-workspace +meta-repo, not cb-testing). For a PBS-only run, override `HELIX_RELAY_IMAGE` to the public +`ghcr.io/gattaca-com/helix-relay:main` in `.env`. Confirmed cb-basic then references only the local +CB image + public helix/reth-rbuilder/lighthouse. +Pre-pull the public images so the run doesn't stall: +`docker pull ghcr.io/gattaca-com/helix-relay:main; docker pull ethpandaops/reth-rbuilder:develop; docker pull sigp/lighthouse:latest` + +cb-testing state: NOT stale-broken — J confirmed it's version-independent and in use; treat +sim failures as real signal. (Refactoring/improving cb-testing is a separate parallel subtask.) + +## Helix pre-genesis panic (3rd helix fix — CONFIRMED + fixed) +After the config fixes, latest `:main` helix still crashed at runtime: +`panicked at chain_info.rs: called Option::unwrap() on a None value` in `ChainInfo::current_slot` +(from `HousekeeperTile::new` at boot). ROOT CAUSE: helix eagerly computes `current_slot()` at startup, +which is `None` before genesis. Confirmed timing: helix started 19s BEFORE genesis (devnet +`genesis_delay = 20`). FIX: wrap helix's launch to wait for genesis in +`ethereum-package/src/mev/helix/helix_relay_launcher.star` — add `"GENESIS_TIME": str(genesis_timestamp)` +to the service `env_vars`, and set `entrypoint=["sh","-c"]` + +`cmd=['until [ "$(date +%s)" -ge "$GENESIS_TIME" ]; do sleep 1; done; exec /app/helix-relay --config ']`. +(sh+date are in the image; the ~20s wait fits kurtosis's 60s port-check.) RESULT: full stack deploys — +helix-relay RUNNING, commit-boost RUNNING, dora/spamoor/prometheus up; the run reaches `cb-verify`. + +## Step 3 — Run the sim (STATUS: full stack deploys; cb-verify runs) +```bash +just testnet configs/generated/cb-basic.yml +# = run-and-verify.sh: rm stale enclave -> kurtosis run CB-Testnet -> cb-verify +``` + +## Step 4 — Verify (the pass bar) (STATUS: pending) +cb-verify tiers (exit 0 = all pass): +- Tier 1 (must): chain_finality, sync_status, cb_running, relay.payloads_delivered_multi, + payload_hash_match, mux.routing. +- Tier 2 (should): missed_slots <10%, builder_blocks_received >0, mev_delivery_rate >=30%, + validator_registrations =100%. +- Tier 3: CB Prometheus `cb_pbs_relay_status_code_total{endpoint=get_header|submit_blinded_block|...}`. +Monitor live: dora block explorer (additional_service), `just show-logs CB-Testnet`, +`kurtosis enclave inspect CB-Testnet`, `just kurtosis-logs `. + +## Cleanup +```bash +kurtosis enclave rm -f CB-Testnet # single enclave +kurtosis clean -a # full wipe +``` + +## Gotchas (fill in as hit) +- **helix `:main` drift → grpc invalid-UTF8 at `Adding service 'helix-relay'` (CONFIRMED blocker).** + Symptom: `kurtosis run` aborts at the helix-relay service add with + `grpc: error while marshaling: string field contains invalid UTF-8`, half-building the enclave (no + commit-boost / helix-relay-api). ROOT CAUSE (via `docker logs` on the exited helix container, NOT the + kurtosis error): helix panics on startup — `config.rs: failed to parse config file: + network_config: untagged and internally tagged enums do not support enum input`. The public + `ghcr.io/gattaca-com/helix-relay:main` (moving tag) drifted and its config schema no longer matches the + `helix_relay_config` the pinned fork (`ethereum-package @ 4844f884`) renders. The UTF-8 grpc error is a + SECONDARY symptom of kurtosis streaming the crashing container. Reproduced on kurtosis 1.20.0 AND 1.18.1 + (NOT a kurtosis-version issue). FIX (chosen: track latest helix): reconcile the embedded + `HELIX_RELAY_CONFIG` const in `src/bin/sim/genmodel/helix.rs` to current `:main`. Source of truth = + the `:main` BINARY's serde metadata (`docker create` + `docker cp /app/helix-relay` + `strings`), NOT + the ws-workspace helix checkout (a divergent WS branch, unreliable). Two drifts fixed: + 1. Deleted `network_config: !Custom {dir_path, genesis_validator_root, genesis_time}` — `:main` removed + it; the relay now fetches chain spec + genesis from the beacon node at startup. + 2. `cores:` (CoresConfig) is now 10 fields: dropped `sub_workers`; added `decoder: [0]` (Vec) + + `simulator`/`top_bid`/`data_gatherer`/`block_merging`/`housekeeper` (usize `0`). The panic + mis-reported this as top-level `missing field \`decoder\`` (a `#[serde(flatten)]` artifact). + FAST iteration loop (seconds, not 10-min kurtosis runs): `docker run --rm -v cfg:/app/config.yaml + ` panics ~1s per bad field; success = it reaches the beacon-fetch stage past config parse. + PIN vs LATEST: pin via `HELIX_RELAY_IMAGE=` in `.env` (no template change). Template shape is + COUPLED to helix version, so a pinned OLD helix needs the OLD template. For both (relevant to the + WebSocket-vs-helix work), add a schema switch in the generator keyed off the image tag. Core VALUES (all + core 0) are a smoke-test choice, not a realistic perf layout. +- **kurtosis version + config-version clash:** J runs 1.18.1. A newer CLI (1.20.0) writes + `~/.config/kurtosis/kurtosis-config.yml` at `config-version: 9`, which 1.18.1 can't read + (`ConfigVersion(9) ... newer than ConfigVersion_v7`). Fix on downgrade: `rm` that file (engine restart + regenerates it), then `kurtosis analytics disable`. (This box: pinned to 1.18.1.) +- Forked ethereum-package submodule is load-bearing + must be `--init`ed (empty otherwise). +- `--image-download always` still uses purely-local tags if they aren't registry refs. +- Image-tag mismatch across generator / example config / justfile (see Step 1). +- (add machine-specific / version-pin issues here) + +## What ePBS e2e would additionally need (scaffold — not this run) +- ePBS-aware relay + builder (stock helix/reth-rbuilder speak legacy PBS, not + getExecutionPayloadBid / in-protocol bids). +- A gloas-capable ethereum-package (fork bump). +- A CB image built from the `epbs` branch. +- cb-verify: add an endpoint arm to the `cb_metrics.rs` status-code matrix for the ePBS + endpoints, and a beacon-side check analogous to `payload_matching` for the envelope flow. diff --git a/ethereum-package b/ethereum-package index 4844f88..1b255a4 160000 --- a/ethereum-package +++ b/ethereum-package @@ -1 +1 @@ -Subproject commit 4844f884cb06daab30dd2cc1693328d55168720e +Subproject commit 1b255a4c5ecdaad0e54be2e939d7a0a12c52f60b diff --git a/justfile b/justfile index 55fda26..ee07771 100644 --- a/justfile +++ b/justfile @@ -77,12 +77,36 @@ show-logs enclave="CB-Testnet": # Quick mux routing check (no observation window, just fetch logs and check) test-mux enclave="CB-Testnet" config="configs/generated/cb-mux.yml": - cargo run --release --bin test-mux -- {{enclave}} {{config}} + cargo run --release --bin cb-verify -- \ + --enclave {{enclave}} \ + --config {{config}} \ + --min-epochs 0 \ + --timeout 300 -# Generate Kurtosis YAML configs from templates into configs/generated/ -# Loads optional .env for Docker image overrides (see .env.example). +# Generate Kurtosis YAML configs into configs/generated/ (the typed `sim` +# generator). Loads optional .env for Docker image overrides (see .env.example). generate-configs: - python3 scripts/generate_kurtosis_configs.py + cargo run --quiet --bin sim -- generate + +# Build the Commit-Boost image the devnet runs, from the sibling commit-boost repo +# (default ../commit-boost-client). Produces commit-boost/commit-boost:{{tag}}; +# keep it in sync with MEV_BOOST_IMAGE in .env. Helix is a PUBLIC image (not built). +build-cb-image tag="kurtosis" cb_dir="../commit-boost-client": + cd {{cb_dir}} && just build-all {{tag}} + +# Pre-pull the public images the devnet needs so `kurtosis run` doesn't stall. +# (The CB sidecar image is built locally — see build-cb-image.) +pull-images: + docker pull ghcr.io/gattaca-com/helix-relay:main + docker pull ethpandaops/reth-rbuilder:develop + docker pull sigp/lighthouse:latest + +# One-command e2e: (re)generate configs, pull public images, launch + verify. +# PREREQ (once): `just build-cb-image` — the CB image must exist locally. +# Usage: just e2e (cb-basic) +# just e2e configs/generated/cb-mux.yml +e2e config="configs/generated/cb-basic.yml": generate-configs pull-images + just testnet {{config}} # Run kurtosis testnet with verification on target `config`. # Observes 1 epoch starting at target_epoch. Chain just needs to reach diff --git a/scripts/generate_kurtosis_configs.py b/scripts/generate_kurtosis_configs.py deleted file mode 100644 index c8c7613..0000000 --- a/scripts/generate_kurtosis_configs.py +++ /dev/null @@ -1,563 +0,0 @@ -#!/usr/bin/env python3 -"""Generate Kurtosis YAML configs for Commit-Boost testing scenarios. - -Reads optional .env file from the project root for Docker image overrides. -See .env.example for all available variables and defaults. -""" - -import argparse -import json -import os -import sys - - -SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) -PROJECT_DIR = os.path.normpath(os.path.join(SCRIPT_DIR, "..")) -KEYS_CONFIGS_DIR = os.path.join(PROJECT_DIR, "keys") - -# --------------------------------------------------------------------------- -# Load .env overrides from cb-testing/.env -# --------------------------------------------------------------------------- - -def load_env(): - """Load key=value pairs from .env file in the project root. - - Simple parser: no variable expansion, no quoting tricks. Just - strips comments and blank lines. Missing file = not an error. - Returns a dict of (key, value) pairs. - """ - env_path = os.path.join(PROJECT_DIR, ".env") - result = {} - if not os.path.isfile(env_path): - return result - with open(env_path) as f: - for line in f: - stripped = line.strip() - if not stripped or stripped.startswith("#"): - continue - if "=" not in stripped: - continue - key, _, value = stripped.partition("=") - result[key.strip()] = value.strip() - return result - - -ENV = load_env() - -# Image defaults (overridable via .env) -HELIX_RELAY_IMAGE = ENV.get("HELIX_RELAY_IMAGE", "helix-relay:kurtosis") -MEV_RELAY_IMAGE = ENV.get("MEV_RELAY_IMAGE", "ethpandaops/mev-boost-relay:main") -MEV_BOOST_IMAGE = ENV.get("MEV_BOOST_IMAGE", "commit-boost/pbs:kurtosis") -BUILDER_CL_IMAGE = ENV.get("BUILDER_CL_IMAGE", "sigp/lighthouse:latest") -BUILDER_EL_IMAGE = ENV.get("BUILDER_EL_IMAGE", "ethpandaops/reth-rbuilder:develop") - -# --------------------------------------------------------------------------- -# Shared YAML fragments -# --------------------------------------------------------------------------- - -COMMON_PARTICIPANTS = """\ -participants: - - el_type: geth - cl_type: lighthouse""" - -COMMON_ADDITIONAL_SERVICES = """\ -additional_services: - - dora - - spamoor - - prometheus""" - -COMMON_NETWORK_PARAMS = ( - "network_params:\n" - ' network: kurtosis\n' - ' network_id: "3151908"\n' - ' deposit_contract_address: "0x00000000219ab540356cBB839Cbe05303d7705Fa"\n' - " seconds_per_slot: 12\n" - " slot_duration_ms: 12000\n" - " num_validator_keys_per_node: 128\n" - " preregistered_validator_keys_mnemonic:\n" - ' "giant issue aisle success illegal bike spike\n' - " question tent bar rely arctic volcano long crawl hungry vocal artwork sniff fantasy\n" - ' very lucky have athlete"\n' - ' prefunded_accounts: \'{"0xb9e79d19f651a941757b35830232E7EFC77E1c79": {"balance": "100000ETH"}}\'\n' -) - -MUX_NETWORK_PARAMS = ( - "network_params:\n" - ' network: kurtosis\n' - ' network_id: "3151908"\n' - ' deposit_contract_address: "0x00000000219ab540356cBB839Cbe05303d7705Fa"\n' - " seconds_per_slot: 12\n" - " slot_duration_ms: 12000\n" - " num_validator_keys_per_node: 256\n" - " preregistered_validator_keys_mnemonic:\n" - ' "giant issue aisle success illegal bike spike\n' - " question tent bar rely arctic volcano long crawl hungry vocal artwork sniff fantasy\n" - ' very lucky have athlete"\n' - " prefunded_accounts: '{\"0xb9e79d19f651a941757b35830232E7EFC77E1c79\": {\"balance\": \"100000ETH\"}}'\n" -) -# --------------------------------------------------------------------------- - -def load_pubkeys(filename): - path = os.path.join(KEYS_CONFIGS_DIR, filename) - if not os.path.isfile(path): - print(f"Error: missing pubkey file {path}", file=sys.stderr) - sys.exit(1) - with open(path, "r") as f: - return json.load(f) - - -def format_pubkey_list(pubkeys): - """Return a multiline list literal with 4-space entry indentation. - - When placed inside a YAML literal block that is itself indented 4 spaces, - the entries end up at 8 spaces total — matching the ground truth. - """ - lines = ["["] - for i, pk in enumerate(pubkeys): - comma = "" if i == len(pubkeys) - 1 else "," - lines.append(f' "{pk}"{comma}') - lines.append("]") - return "\n".join(lines) - - -# --------------------------------------------------------------------------- -# TOML builders (raw; get indented 4 spaces by build_mev_params) -# --------------------------------------------------------------------------- - -def build_cb_toml_basic(timeout_get_header_ms, timeout_get_payload_ms, - extra_pbs_lines=None, per_relay_lines=None): - lines = [ - 'chain = { genesis_time_secs = {{ .Timestamp }}, path = "{{ .Network }}" }', - "", - "[pbs]", - 'host = "0.0.0.0"', - "port = {{ .Port }}", - f"timeout_get_header_ms = {timeout_get_header_ms}", - f"timeout_get_payload_ms = {timeout_get_payload_ms}", - "late_in_slot_time_ms = 2000", - ] - - if extra_pbs_lines: - # Insert after port (idx 4), before timeouts (idx 5) - insert_idx = 5 - for line in extra_pbs_lines: - lines.insert(insert_idx, line) - insert_idx += 1 - - lines.append("") - lines.append("") - lines.append("[metrics]") - lines.append("enabled = true") - lines.append('host = "0.0.0.0"') - lines.append("start_port = 9090") - lines.append("") - lines.append("{{ range $index, $relay := .Relays }}") - lines.append("[[relays]]") - lines.append('id = "mev_relay_{{$index}}"') - lines.append('url = "{{ $relay }}"') - - if per_relay_lines: - for line in per_relay_lines: - lines.append(line) - - lines.append("{{- end }}") - lines.append("") - lines.append("[logs.stdout]") - lines.append('level = "debug"') - lines.append("") - lines.append("[logs.file]") - lines.append("enabled = false") - - return "\n".join(lines) - - -def build_cb_toml_mux(pubkeys_node0, pubkeys_node1): - node0_list = format_pubkey_list(pubkeys_node0) - node1_list = format_pubkey_list(pubkeys_node1) - - lines = [ - 'chain = { genesis_time_secs = {{ .Timestamp }}, path = "{{ .Network }}" }', - "", - "[pbs]", - 'host = "0.0.0.0"', - "port = {{ .Port }}", - "timeout_get_header_ms = 950", - "timeout_get_payload_ms = 4000", - "late_in_slot_time_ms = 2000", - "", - "{{ range $index, $relay := .Relays }}", - "[[relays]]", - 'id = "mev_relay_{{$index}}"', - 'url = "{{ $relay }}"', - "{{- end }}", - "", - "[metrics]", - "enabled = true", - 'host = "0.0.0.0"', - "start_port = 9090", - "", - "[[mux]]", - 'id = "node_0_to_helix"', - f"validator_pubkeys = {node0_list}", - "timeout_get_header_ms = 900", - "[[mux.relays]]", - 'id = "mux_helix"', - 'url = "{{ index .Relays 0 }}"', - "", - "[[mux]]", - 'id = "node_1_to_flashbots"', - f"validator_pubkeys = {node1_list}", - "timeout_get_header_ms = 900", - "[[mux.relays]]", - 'id = "mux_flashbots"', - 'url = "{{ index .Relays 1 }}"', - "", - - "[logs.stdout]", - 'level = "debug"', - "", - "[logs.file]", - "enabled = false", - ] - - return "\n".join(lines) - - -# --------------------------------------------------------------------------- -# MEV params builder -# --------------------------------------------------------------------------- - -def build_helix_relay_config(): - """Return the Helix relay YAML literal block as a string. - - The caller indents each line 4 spaces to fit under mev_params. - Includes the simulators block which improves relay reliability. - """ - return """instance_id: "helix-kurtosis-test" - -network_config: !Custom - dir_path: "{{ .GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER }}/config.json" - genesis_validator_root: "{{ .GENESIS_VALIDATORS_ROOT }}" - genesis_time: {{ .GENESIS_TIME }} - -postgres: - hostname: "{{ .POSTGRES_HOST_NAME }}" - port: {{ .POSTGRES_PORT }} - db_name: "{{ .POSTGRES_DB }}" - user: "{{ .POSTGRES_USER }}" - password: "{{ .POSTGRES_PASS }}" - region: 0 - region_name: "LOCAL" - -beacon_clients: - - url: "{{ .BEACON_URI }}" - -gossip_payload_on_header: false - -simulators: - - url: "{{ .BLOCKSIM_URI }}" - namespace: flashbots - is_merging_simulator: false - max_concurrent_tasks: 32 - -router_config: - enabled_routes: - - route: GetValidators - - route: SubmitBlock - - route: GetTopBid - - route: GetHeader - rate_limit: - replenish_ms: 50 - burst_size: 20 - - route: GetPayload - - route: RegisterValidators - - route: Status - - route: ProposerPayloadDelivered - - route: BuilderBidsReceived - - route: ValidatorRegistration - shutdown_delay_ms: 12000 - -timing_game_config: - max_header_delay_ms: 400 - latest_header_delay_ms_in_slot: 1500 - default_client_latency_ms: 50 - -target_get_payload_propagation_duration_ms: 500 - -is_submission_instance: true -is_registration_instance: true - -admin_token: "test_admin_token" - -logging: - type: Console - -cores: - auctioneer: 0 - sub_workers: [0] - reg_workers: [0] - tokio: [0] - tcp_bid_submissions_tile: 2 - -is_local_dev: false""" - - -def build_mev_params(relays, images, toml_block, helix_relay_yaml=None): - lines = ["mev_params:"] - - if isinstance(relays, list): - lines.append(" mev_relay:") - for r in relays: - lines.append(f" - {r}") - else: - lines.append(f" mev_relay: {relays}") - - lines.append(" mev_sidecar: commit-boost") - lines.append(" mev_builder: flashbots") - lines.append("") - - for key, val in images.items(): - lines.append(f" {key}: {val}") - - lines.append("") - lines.append(" mev_builder_subsidy: 1") - lines.append("") - - if helix_relay_yaml: - lines.append(" helix_relay_config: |") - for line in helix_relay_yaml.splitlines(): - if line.strip(): - lines.append(f" {line}") - else: - lines.append("") - lines.append("") - - lines.append(" commit_boost_config: |") - # Indent every non-empty TOML line by 4 spaces; keep blanks truly empty - for line in toml_block.splitlines(): - if line.strip(): - lines.append(f" {line}") - else: - lines.append("") - # NB: no trailing empty line here — that is handled by the caller's join - - return "\n".join(lines) - - -# --------------------------------------------------------------------------- -# Scenario generators -# --------------------------------------------------------------------------- - -def generate_basic(): - comment = ( - "# cb-basic: Single relay (helix) with default Commit-Boost config.\n" - "#\n" - "# Tests the core MEV pipeline through Commit-Boost with a single Helix\n" - "# relay as the only relay endpoint." - ) - images = { - "helix_relay_image": HELIX_RELAY_IMAGE, - "mev_boost_image": MEV_BOOST_IMAGE, - "mev_builder_image": BUILDER_EL_IMAGE, - "mev_builder_cl_image": BUILDER_CL_IMAGE, - } - toml = build_cb_toml_basic(950, 4000) - mev_params = build_mev_params("helix", images, toml, helix_relay_yaml=build_helix_relay_config()) - return "\n\n".join([ - comment, - COMMON_PARTICIPANTS, - COMMON_ADDITIONAL_SERVICES, - "mev_type: custom", - mev_params, - COMMON_NETWORK_PARAMS, - ]) + "\n" - - -def generate_multiple_relays(): - comment = ( - "# cb-multiple-relays: Two relays (helix + flashbots) behind a single\n" - "# Commit-Boost sidecar.\n" - "#\n" - "# Tests that CB correctly routes get_header requests to both relays,\n" - "# aggregating responses and selecting the best bid." - ) - images = { - "helix_relay_image": HELIX_RELAY_IMAGE, - "mev_relay_image": MEV_RELAY_IMAGE, - "mev_boost_image": MEV_BOOST_IMAGE, - "mev_builder_image": BUILDER_EL_IMAGE, - "mev_builder_cl_image": BUILDER_CL_IMAGE, - } - toml = build_cb_toml_basic(950, 4000) - mev_params = build_mev_params(["helix", "flashbots"], images, toml, helix_relay_yaml=build_helix_relay_config()) - return "\n\n".join([ - comment, - COMMON_PARTICIPANTS, - COMMON_ADDITIONAL_SERVICES, - "mev_type: custom", - mev_params, - COMMON_NETWORK_PARAMS, - ]) + "\n" - - -def generate_skip_sigverify(): - comment = ( - "# cb-skip-sigverify: Signature verification disabled for header responses.\n" - "#\n" - "# Tests the CB fast path where BLS verification is skipped. This trades\n" - "# correctness for speed — useful to verify that the path exists and is\n" - "# reachable under load." - ) - images = { - "helix_relay_image": HELIX_RELAY_IMAGE, - "mev_boost_image": MEV_BOOST_IMAGE, - "mev_builder_image": BUILDER_EL_IMAGE, - "mev_builder_cl_image": BUILDER_CL_IMAGE, - } - toml = build_cb_toml_basic(950, 4000, extra_pbs_lines=["skip_sigverify = true"]) - mev_params = build_mev_params("helix", images, toml, helix_relay_yaml=build_helix_relay_config()) - return "\n\n".join([ - comment, - COMMON_PARTICIPANTS, - COMMON_ADDITIONAL_SERVICES, - "mev_type: custom", - mev_params, - COMMON_NETWORK_PARAMS, - ]) + "\n" - - -def generate_timing_games(): - comment = ( - "# cb-timing-games: Aggressive timing game configuration.\n" - "#\n" - "# Tests CB's ability to orchestrate repeated get_header polls with\n" - "# short timeouts in order to arrive at the best bid as late as possible\n" - "# in the slot. Per-relay timing overrides are enabled for all relays." - ) - images = { - "helix_relay_image": HELIX_RELAY_IMAGE, - "mev_relay_image": MEV_RELAY_IMAGE, - "mev_boost_image": MEV_BOOST_IMAGE, - "mev_builder_image": BUILDER_EL_IMAGE, - "mev_builder_cl_image": BUILDER_CL_IMAGE, - } - toml = build_cb_toml_basic( - 400, - 2000, - per_relay_lines=[ - "enable_timing_games = true", - "target_first_request_ms = 100", - "frequency_get_header_ms = 200", - ], - ) - mev_params = build_mev_params(["helix", "flashbots"], images, toml, helix_relay_yaml=build_helix_relay_config()) - return "\n\n".join([ - comment, - COMMON_PARTICIPANTS, - COMMON_ADDITIONAL_SERVICES, - "mev_type: custom", - mev_params, - COMMON_NETWORK_PARAMS, - ]) + "\n" - - -def generate_extra_validation(): - comment = ( - "# cb-extra-validation: Enable extra validation of get_header responses\n" - "# via a local execution layer client.\n" - "#\n" - "# Tests that CB will RPC-call the execution client to verify block\n" - "# parameters before returning a header to the beacon node." - ) - images = { - "helix_relay_image": HELIX_RELAY_IMAGE, - "mev_boost_image": MEV_BOOST_IMAGE, - "mev_builder_image": BUILDER_EL_IMAGE, - "mev_builder_cl_image": BUILDER_CL_IMAGE, - } - toml = build_cb_toml_basic( - 950, - 4000, - extra_pbs_lines=[ - "extra_validation_enabled = true", - 'rpc_url = "http://el-1-geth-lighthouse:8545"', - ], - ) - mev_params = build_mev_params("helix", images, toml, helix_relay_yaml=build_helix_relay_config()) - return "\n\n".join([ - comment, - COMMON_PARTICIPANTS, - COMMON_ADDITIONAL_SERVICES, - "mev_type: custom", - mev_params, - COMMON_NETWORK_PARAMS, - ]) + "\n" - - -def generate_mux(pubkeys_node0, pubkeys_node1): - comment = ( - "# cb-mux: Multiplexed relay routing per validator node.\n" - "#\n" - "# Routes all 128 validators from node-0 exclusively to the Helix relay and\n" - "# all 128 validators from node-1 exclusively to the Flashbots relay.\n" - "# This tests CB's ability to partition the validator set and apply\n" - "# per-mux timeout and relay configurations." - ) - images = { - "helix_relay_image": HELIX_RELAY_IMAGE, - "mev_relay_image": MEV_RELAY_IMAGE, - "mev_boost_image": MEV_BOOST_IMAGE, - "mev_builder_image": BUILDER_EL_IMAGE, - "mev_builder_cl_image": BUILDER_CL_IMAGE, - } - toml = build_cb_toml_mux(pubkeys_node0, pubkeys_node1) - mev_params = build_mev_params(["helix", "flashbots"], images, toml, helix_relay_yaml=build_helix_relay_config()) - return "\n\n".join([ - comment, - COMMON_PARTICIPANTS, - COMMON_ADDITIONAL_SERVICES, - "mev_type: custom", - mev_params, - MUX_NETWORK_PARAMS, - ]) + "\n" - - -# --------------------------------------------------------------------------- -# Main -# --------------------------------------------------------------------------- - -def main(): - parser = argparse.ArgumentParser( - description="Generate Kurtosis YAML configs for Commit-Boost testing." - ) - parser.add_argument( - "--output-dir", - default="configs/generated", - help="Directory to write generated YAML configs (default: kurtosis-configs).", - ) - args = parser.parse_args() - - output_dir = os.path.abspath(args.output_dir) - os.makedirs(output_dir, exist_ok=True) - - pubkeys_node0 = load_pubkeys("node-0-pubkeys.json") - pubkeys_node1 = load_pubkeys("node-1-pubkeys.json") - - scenarios = { - "cb-basic.yml": generate_basic(), - "cb-multiple-relays.yml": generate_multiple_relays(), - "cb-skip-sigverify.yml": generate_skip_sigverify(), - "cb-timing-games.yml": generate_timing_games(), - "cb-extra-validation.yml": generate_extra_validation(), - "cb-mux.yml": generate_mux(pubkeys_node0, pubkeys_node1), - } - - for filename, content in scenarios.items(): - path = os.path.join(output_dir, filename) - with open(path, "w") as f: - f.write(content) - print(f"Generated {path}") - - -if __name__ == "__main__": - main() diff --git a/scripts/run-and-verify.sh b/scripts/run-and-verify.sh index 1fef1b4..d6da4d3 100755 --- a/scripts/run-and-verify.sh +++ b/scripts/run-and-verify.sh @@ -23,6 +23,7 @@ JSON_DIR_FLAG="" STRICT_FLAG="" LIVE_METRICS_FLAG="" SKIP_FINALIZATION_FLAG="" +REQUIRE_FEATURE_PROOF_FLAG="" TIMEOUT=3600 MIN_EPOCHS=2 TARGET_EPOCH=7 @@ -44,6 +45,7 @@ usage() { echo " --timeout SECS Readiness timeout (default: 1500)" echo " --min-epochs N Observation window in epochs (default: 2)" echo " --target-epoch N Observation window starts at this epoch (default: 5)" + echo " --require-feature-proof Fail when an armed tier-1 feature check proves nothing (Law 3)" echo " -v, --verbose Verbose logging" echo " -h, --help Show this help" exit 0 @@ -60,6 +62,7 @@ while [[ $# -gt 0 ]]; do --strict) STRICT_FLAG="--strict"; shift;; --live-metrics) LIVE_METRICS_FLAG="--live-metrics"; shift;; --skip-finalization) SKIP_FINALIZATION_FLAG="--skip-finalization-check"; shift;; + --require-feature-proof) REQUIRE_FEATURE_PROOF_FLAG="--require-feature-proof"; shift;; --timeout) TIMEOUT="$2"; shift 2;; --min-epochs) MIN_EPOCHS="$2"; shift 2;; --target-epoch) TARGET_EPOCH="$2"; shift 2;; @@ -100,10 +103,70 @@ cleanup() { } trap cleanup EXIT +# Advisory: report host memory before a ~10-min run and warn if the box is +# genuinely tight (a truly exhausted host can stall the kurtosis launch or thrash +# the whole run). NOTE: this is NOT what killed the relays in the 2026-07-31 run — +# that was a PER-CONTAINER cgroup OOM (CONSTRAINT_MEMCG) at the relays' own +# RELAY_MAX_MEMORY cap, fixed by raising that cap in the ethereum-package +# launchers, independent of host memory. Non-blocking; set LOW_MEM_ABORT=1 to abort. +check_host_memory() { + local need_mb=24000 # ~2x8GB relays + ~10 services + headroom + local avail_mb swap_total_mb swap_free_mb swap_used_mb + avail_mb=$(awk '/MemAvailable/ {print int($2/1024)}' /proc/meminfo) + swap_total_mb=$(awk '/SwapTotal/ {print int($2/1024)}' /proc/meminfo) + swap_free_mb=$(awk '/SwapFree/ {print int($2/1024)}' /proc/meminfo) + swap_used_mb=$(( swap_total_mb - swap_free_mb )) + echo "Host memory: ${avail_mb}MB available; swap ${swap_used_mb}/${swap_total_mb}MB used." + if (( avail_mb < need_mb )); then + echo "" >&2 + echo "⚠️ HOST MEMORY LOW — a devnet wants ~${need_mb}MB available; the launch may stall" >&2 + echo " or the run may thrash. Top memory consumers to consider freeing:" >&2 + ps -eo rss,comm --sort=-rss 2>/dev/null | awk 'NR>1 && NR<=6 {printf " %5.1f GB %s\n", $1/1024/1024, $2}' >&2 + if [[ "${LOW_MEM_ABORT:-0}" == "1" ]]; then + echo " Aborting (LOW_MEM_ABORT=1)." >&2 + exit 2 + fi + echo " Proceeding anyway (set LOW_MEM_ABORT=1 to abort instead)." >&2 + echo "" >&2 + fi +} + +# Step 0: Pre-build the verifier BEFORE the devnet is up — a `cargo run --release` +# on cb-verify is a multi-GB compile; keeping it off the critical path (it used to +# run mid-devnet) means the box isn't compiling while 10 services are live. Compile +# while idle, then invoke the built binary. +echo "Building cb-verify (release) before launch..." +cargo build --release --bin cb-verify --manifest-path "$REPO_DIR/Cargo.toml" +CB_VERIFY_BIN="$REPO_DIR/target/release/cb-verify" + # Step 1: Clean any stale enclave with the same name echo "Cleaning stale enclave '$ENCLAVE' (if any)..." kurtosis enclave rm -f "$ENCLAVE" 2>/dev/null || true +# Step 1b: Preflight gate — validate the config against the real images (~1s) BEFORE the ~10-min run. +# `sim preflight` exits nonzero ONLY on a genuine config-drift Fail (Inconclusive/Pass proceed), so a +# schema drift is caught here as a labeled failure instead of a masked runtime panic minutes into launch. +# Best-effort: if the `sim` bin can't be built/run, warn and proceed (don't block the run on tooling). +# LIMITATION (P1): preflight parses with whatever `:main` image is cached LOCALLY, while the run below pulls +# with `--image-download always` — a `:main` that drifts between the two is a false-green window. Closing it +# needs pull-then-pin-by-digest, which belongs to `sim run` (P2) that owns the pull. +echo "Preflighting config against real images..." +if cargo run --quiet --bin sim --manifest-path "$REPO_DIR/Cargo.toml" -- preflight "$CONFIG"; then + echo "Preflight OK (no config drift)." +else + pf_rc=$? + if [[ $pf_rc -eq 1 ]]; then + echo "PREFLIGHT FAILED: config drift detected (see the Fail{field} above). Aborting launch." >&2 + exit 1 + fi + echo "Preflight could not run (rc=$pf_rc); proceeding without the gate." >&2 +fi +echo "" + +# Step 1c: Host memory check (warn before spending ~10min on a run the OOM-killer +# would wreck). +check_host_memory + # Step 2: Launch the devnet echo "Launching devnet..." echo " Package: $PACKAGE" @@ -111,17 +174,23 @@ echo " Config: $CONFIG" echo " Enclave: $ENCLAVE" echo "" -kurtosis run "$PACKAGE" \ +# On any launch failure, auto-fire triage (root-cause capture as a run property) before exiting. +if ! kurtosis run "$PACKAGE" \ --enclave "$ENCLAVE" \ --args-file "$CONFIG" \ - --image-download always + --image-download always; then + echo "" + echo "LAUNCH FAILED — triaging crashed services (structured root-cause capture)..." >&2 + cargo run --quiet --bin sim --manifest-path "$REPO_DIR/Cargo.toml" -- triage "$ENCLAVE" || true + exit 1 +fi echo "" echo "Enclave '$ENCLAVE' is up. Starting verification..." echo "" -# Step 3: Run verification -cargo run --bin cb-verify --manifest-path "$REPO_DIR/Cargo.toml" --release -- \ +# Step 3: Run verification (pre-built binary from Step 0 — no mid-devnet compile). +"$CB_VERIFY_BIN" \ --enclave "$ENCLAVE" \ --config "$CONFIG" \ --timeout "$TIMEOUT" \ @@ -132,4 +201,5 @@ cargo run --bin cb-verify --manifest-path "$REPO_DIR/Cargo.toml" --release -- \ $STRICT_FLAG \ $LIVE_METRICS_FLAG \ $SKIP_FINALIZATION_FLAG \ + $REQUIRE_FEATURE_PROOF_FLAG \ $VERBOSE diff --git a/src/beacon.rs b/src/beacon.rs index d2d3dcc..b62223f 100644 --- a/src/beacon.rs +++ b/src/beacon.rs @@ -7,12 +7,11 @@ use std::time::Duration; use alloy_primitives::B256; use alloy_rpc_types_beacon::{ - block::BlockResponse, config::SpecResponse, genesis::GenesisResponse, header::HeaderResponse, - node::SyncStatus, state::FinalityCheckpointsResponse, + block::BlockResponse, header::HeaderResponse, node::SyncStatus, + state::FinalityCheckpointsResponse, }; use eyre::{Result, WrapErr}; use serde::Deserialize; -use tracing::warn; const REQUEST_TIMEOUT: Duration = Duration::from_secs(10); @@ -97,47 +96,6 @@ impl BeaconClient { Ok(resp.data.is_syncing) } - /// GET /eth/v1/beacon/genesis -> genesis_time - pub async fn get_genesis_time(&self) -> Result { - let resp: GenesisResponse = self - .client - .get(format!("{}/eth/v1/beacon/genesis", self.base_url)) - .send() - .await? - .error_for_status()? - .json() - .await?; - - Ok(resp.data.genesis_time) - } - - /// GET /eth/v1/config/spec -> SECONDS_PER_SLOT (defaults to 12) - pub async fn get_seconds_per_slot(&self) -> u64 { - match self.try_get_seconds_per_slot().await { - Ok(sps) => sps, - Err(e) => { - warn!("Failed to get SECONDS_PER_SLOT, defaulting to 12: {e}"); - 12 - } - } - } - - async fn try_get_seconds_per_slot(&self) -> Result { - let resp: SpecResponse = self - .client - .get(format!("{}/eth/v1/config/spec", self.base_url)) - .send() - .await? - .error_for_status()? - .json() - .await?; - - resp.data - .get("SECONDS_PER_SLOT") - .and_then(|v| v.parse().ok()) - .ok_or_else(|| eyre::eyre!("SECONDS_PER_SLOT not found in spec")) - } - /// GET /eth/v1/beacon/headers/{slot} -> Some(header) or None if 404 pub async fn get_header(&self, slot: u64) -> Result> { let resp = self @@ -211,3 +169,128 @@ impl BeaconClient { Ok(resp.data.into_iter().map(|e| e.validator.pubkey).collect()) } } + +#[cfg(test)] +mod tests { + use super::*; + + /// The shape a real beacon node returns for `/eth/v2/beacon/blocks/{slot}`, + /// captured live from **prysm v7.1.8** on the nethermind+prysm devnet + /// (2026-08-04) and trimmed to the fields around the one we extract. The + /// point is the fields we do NOT model: a block body carries ~12 more keys + /// and grows every fork, so the parse must ignore unknown fields rather + /// than fail. A regression here does not error loudly - it silently yields + /// `None`, which `payload_hash_match` reports as "missed", i.e. a real + /// mismatch would be indistinguishable from a missing block. + fn prysm_block_json(extra_body_fields: bool) -> String { + let extra = if extra_body_fields { + r#""randao_reveal": "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "eth1_data": {"deposit_root": "0xbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"}, + "graffiti": "0x0000000000000000000000000000000000000000000000000000000000000000", "proposer_slashings": [], "attester_slashings": [], + "attestations": [], "deposits": [], "voluntary_exits": [], + "sync_aggregate": {"sync_committee_bits": "0x00"}, + "bls_to_execution_changes": [], "blob_kzg_commitments": [],"# + } else { + "" + }; + format!( + r#"{{ + "version": "fulu", + "execution_optimistic": false, + "finalized": true, + "data": {{ + "message": {{ + "slot": "93", + "proposer_index": "42", + "parent_root": "0xcccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", + "state_root": "0xdddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", + "body": {{ + {extra} + "execution_payload": {{ + "parent_hash": "0x1111111111111111111111111111111111111111111111111111111111111111", + "fee_recipient": "0x0000000000000000000000000000000000000000", + "block_number": "75", + "gas_limit": "60000000", + "block_hash": "0xba639ff997222ed1521e1474ae80094ed4dccad19b5d2ac1b596e7fbe248cf1b", + "transactions": [] + }} + }} + }}, + "signature": "0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" + }} + }}"# + ) + } + + #[test] + fn parses_block_hash_from_a_real_prysm_response() { + let json = prysm_block_json(true); + let block: BlockResponse = + serde_json::from_str(&json).expect("real prysm block must parse"); + assert_eq!( + block + .data + .message + .body + .execution_payload + .map(|ep| ep.block_hash), + Some( + "0xba639ff997222ed1521e1474ae80094ed4dccad19b5d2ac1b596e7fbe248cf1b" + .parse::() + .unwrap() + ) + ); + } + + #[test] + fn unknown_body_and_payload_fields_are_ignored() { + // Same block with the sibling body fields stripped: the extraction must + // be insensitive to which of them are present, so a new fork adding or + // removing body fields cannot silently break hash comparison. + let with = prysm_block_json(true); + let without = prysm_block_json(false); + let a: BlockResponse = serde_json::from_str(&with).unwrap(); + let b: BlockResponse = serde_json::from_str(&without).unwrap(); + assert_eq!( + a.data.message.body.execution_payload.map(|e| e.block_hash), + b.data.message.body.execution_payload.map(|e| e.block_hash), + ); + } + + #[test] + fn block_without_execution_payload_yields_none_not_an_error() { + // A pre-merge / phase0 block has no execution_payload. That must be a + // clean `None` (the caller reports "missed"), never a parse error that + // would abort the whole slot scan. + let json = r#"{ + "version": "phase0", + "data": { + "message": { + "slot": "1", "proposer_index": "0", + "parent_root": "0xcccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", "state_root": "0xdddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", + "body": { "randao_reveal": "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" } + }, + "signature": "0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" + } + }"#; + let block: BlockResponse = + serde_json::from_str(json).expect("payload-less block must still parse"); + assert!(block.data.message.body.execution_payload.is_none()); + } + + #[test] + fn data_wrapper_unwraps_the_envelope() { + // Every beacon endpoint we call wraps its payload in {"data": ...}. + let w: DataWrapper> = serde_json::from_str(r#"{"data":[1,2,3]}"#).unwrap(); + assert_eq!(w.data, vec![1, 2, 3]); + } + + #[test] + fn base_url_trailing_slash_is_normalized() { + // URLs are built by string concat, so a trailing slash would produce a + // double slash and a 404 on some clients. + let c = BeaconClient::new("http://beacon:5052/"); + assert_eq!(c.base_url, "http://beacon:5052"); + let c2 = BeaconClient::new("http://beacon:5052"); + assert_eq!(c2.base_url, "http://beacon:5052"); + } +} diff --git a/src/bin/sim/checks_catalog.rs b/src/bin/sim/checks_catalog.rs new file mode 100644 index 0000000..ebcf6bd --- /dev/null +++ b/src/bin/sim/checks_catalog.rs @@ -0,0 +1,405 @@ +//! `sim checks --list [--json]` — the machine-readable catalog of what `cb-verify` +//! asserts, so an agent can discover the harness's contract WITHOUT reading +//! `src/checks/*` or `docs/CHECKS.md`. +//! +//! IMPORTANT — this is a STATIC, hand-maintained catalog. It mirrors two sources +//! of truth and MUST be kept in sync with BOTH when a check is added, removed, +//! retiered, or its data-source changes: +//! * the check ids + tiers in `src/checks/*.rs` (the code that emits them), and +//! * the per-check contract in `docs/CHECKS.md` (the prose catalog). +//! +//! There is no derivation today (the checks are constructed imperatively across +//! several modules with no single registry to reflect on), so drift is caught by +//! review, not the compiler. If a clean registry is ever introduced in +//! `src/checks`, prefer deriving this from it and delete the hand-maintenance note. + +use eyre::Result; +use serde::Serialize; + +/// Where a check's evidence comes from. Determines how it behaves when a service +/// dies mid-run (see `docs/CHECKS.md` "Data-source robustness"). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +pub enum DataSource { + /// Commit-boost PBS container logs (survive a relay crash). + #[serde(rename = "cb-logs")] + CbLogs, + /// Relay data API (`/relay/v1/data/...`) — fragile; dies with the relay. + #[serde(rename = "relay-data-api")] + RelayDataApi, + /// Beacon node HTTP API. + #[serde(rename = "beacon-api")] + BeaconApi, + /// Commit-boost Prometheus metrics (usually absent in default PBS mode). + #[serde(rename = "cb-prometheus")] + CbPrometheus, + /// `kurtosis enclave inspect` (survives a relay crash). + #[serde(rename = "kurtosis-inspect")] + KurtosisInspect, +} + +impl DataSource { + /// The stable kebab-case token used in the table + JSON. + pub fn as_str(self) -> &'static str { + match self { + DataSource::CbLogs => "cb-logs", + DataSource::RelayDataApi => "relay-data-api", + DataSource::BeaconApi => "beacon-api", + DataSource::CbPrometheus => "cb-prometheus", + DataSource::KurtosisInspect => "kurtosis-inspect", + } + } +} + +/// One entry in the check catalog: the discoverable contract for a single check. +#[derive(Debug, Clone, Serialize)] +pub struct CatalogEntry { + /// The check id as emitted in the JSON report (`CheckResult.id`). + pub id: &'static str, + /// Severity tier: 1 (must / invariant), 2 (should / health), 3 (info). + pub tier: u8, + /// One-line statement of what a PASS asserts. + pub title: &'static str, + /// Where the evidence comes from. + pub data_source: DataSource, + /// True iff this check positively asserts that a scenario's FEATURE codepath + /// fired (per `docs/CHECKS.md`, only `mux.routing` and `relay.best_bid` do). + pub feature_asserted: bool, + /// Terse note on the check's WARN/SKIP/escalation quirks — the gotchas a + /// consumer must internalize (e.g. a tier-1 anomaly that lands as a non-fatal + /// WARN, or a tier-2 matrix that escalates to tier 1 on FAIL). + pub severity_note: &'static str, +} + +/// The verdict rule a consumer must internalize, printed as the header and worth +/// stating up front: the process exit code keys ONLY on a tier-1 FAIL. +pub const VERDICT_RULE: &str = "verdict: exit code keys ONLY on a tier-1 FAIL \ +(exit 1); WARN/SKIP are non-fatal at every tier; no tier-1 check ran = exit 2. \ +Several anomaly detectors report as WARN — gate on each check's `result`, not the \ +exit code."; + +/// The static catalog. Keep in sync with `src/checks/*` + `docs/CHECKS.md`. +pub fn catalog() -> Vec { + use DataSource::*; + vec![ + CatalogEntry { + id: "chain_finality", + tier: 1, + title: "finalized epoch >= 2 (the chain is finalizing)", + data_source: BeaconApi, + feature_asserted: false, + severity_note: "tier-1 FAIL fails the run; SKIPs before epoch 3 or under --skip-finalization-check", + }, + CatalogEntry { + id: "sync_status", + tier: 1, + title: "the beacon node is done syncing", + data_source: BeaconApi, + feature_asserted: false, + severity_note: "tier-1 FAIL fails the run; no WARN/SKIP state", + }, + CatalogEntry { + id: "cb_running", + tier: 1, + title: "at least one commit-boost service is running", + data_source: KurtosisInspect, + feature_asserted: false, + severity_note: "tier-1 FAIL fails the run; no WARN/SKIP state", + }, + CatalogEntry { + id: "missed_slots", + tier: 2, + title: "missed-slot rate < 10% over the window", + data_source: BeaconApi, + feature_asserted: false, + severity_note: "WARN over threshold; non-fatal (SKIP on a single-slot window)", + }, + CatalogEntry { + id: "relay.payloads_delivered_multi", + tier: 1, + title: "at least one payload delivered across relays", + data_source: RelayDataApi, + feature_asserted: false, + severity_note: "tier-1 FAIL fails the run; SKIP if all relays unreachable", + }, + CatalogEntry { + id: "relay.builder_blocks_received", + tier: 2, + title: "at least one builder block received by a relay", + data_source: RelayDataApi, + feature_asserted: false, + severity_note: "annotative; non-fatal (SKIP if all relays unreachable)", + }, + CatalogEntry { + id: "relay.mev_delivery_rate", + tier: 2, + title: "MEV-delivered on-chain block fraction >= 0.30", + data_source: RelayDataApi, + feature_asserted: false, + severity_note: "WARN below threshold; FAIL only if no on-chain blocks — tier 2, non-fatal", + }, + CatalogEntry { + id: "relay.validator_registrations", + tier: 3, + title: "validators are registered with the relay", + data_source: RelayDataApi, + feature_asserted: false, + severity_note: "informational; OMITTED entirely (not even SKIP) if the pubkey fetch failed", + }, + CatalogEntry { + id: "payload_hash_match", + tier: 1, + title: "relay-delivered hashes match on-chain, no cross-relay conflict", + data_source: RelayDataApi, + feature_asserted: false, + severity_note: "ANOMALY (reorg / relay equivocation) is reported as WARN — NON-FATAL despite tier 1; gate on JSON, not exit code", + }, + CatalogEntry { + id: "relay.best_bid", + tier: 2, + title: "CB delivered >= the best per-relay bid it was offered", + data_source: CbLogs, + feature_asserted: true, + severity_note: "feature check (cross-relay bid aggregation); suboptimal delivery is WARN; SKIP with < 2 relays", + }, + CatalogEntry { + id: "mux.routing", + tier: 1, + title: "every checked getHeader routed per the [[mux]] config", + data_source: CbLogs, + feature_asserted: true, + severity_note: "feature check; misrouting FAILs (fatal), but unverifiable routing is WARN — needs CB [logs.stdout] level = debug", + }, + CatalogEntry { + id: "feature.timing_games", + tier: 1, + title: "timing-games codepath fired (TG: debug logs seen)", + data_source: CbLogs, + feature_asserted: true, + severity_note: "emitted only when the config enables enable_timing_games; PASS on >=1 TG: log line, WARN (non-fatal) if none seen", + }, + CatalogEntry { + id: "feature.extra_validation", + tier: 1, + title: "extra-validation codepath fired (parent-block fetch logs seen)", + data_source: CbLogs, + feature_asserted: true, + severity_note: "emitted only when the config enables extra_validation_enabled; PASS on >=1 'fetched parent block' log, WARN (non-fatal) if none", + }, + CatalogEntry { + id: "feature.min_bid", + tier: 1, + title: "the min_bid_eth floor actually dropped bids", + data_source: CbLogs, + feature_asserted: true, + severity_note: "emitted only when min_bid_eth > 0; FAIL if any auction winner is BELOW the floor (proof the key was silently ignored - [pbs] has no deny_unknown_fields); PASS on >=1 rejection; WARN if nothing was rejected (cannot tell 'ignored' from 'all bids cleared it')", + }, + CatalogEntry { + id: "feature.skip_sigverify", + tier: 1, + title: "skip-sigverify codepath fired (differential via wrong-pubkey relay)", + data_source: CbLogs, + feature_asserted: true, + severity_note: "emitted only when skip_sigverify is enabled; PASS only in the cb-sigverify-diff scenario (wrong-pubkey relay url + >=1 auction winner proves the skip); plain scenarios stay an honest WARN (negative codepath, no positive signal)", + }, + CatalogEntry { + id: "signer.pubkeys", + tier: 1, + title: "the CB signer loaded the devnet's validator keys and authenticated a module JWT", + data_source: CbLogs, + feature_asserted: true, + severity_note: "emitted only when a cb-signer-* service exists; ZERO keys FAILs (CB's loaders skip unreadable keystores with warn!, so a bad mount yields a healthy signer holding nothing); a partial load WARNs. Asserted over JWT-authed get_pubkeys, NOT /status (which is an unconditional 200)", + }, + CatalogEntry { + id: "cb_get_header_matrix", + tier: 2, + title: "get_header status-code distribution healthy", + data_source: CbPrometheus, + feature_asserted: false, + severity_note: "tier 2 -> ESCALATES to tier 1 on FAIL (relay 5xx over the 25% rate); CB client-side codes (555 timeout, 556 ws transport) bucket separately and WARN only; SKIP if metrics absent (the default)", + }, + CatalogEntry { + id: "cb_register_validator_matrix", + tier: 2, + title: "register_validator acceptance healthy", + data_source: CbPrometheus, + feature_asserted: false, + severity_note: "tier 2 -> ESCALATES to tier 1 on FAIL (5xx); SKIP if metrics absent", + }, + CatalogEntry { + id: "cb_submit_blinded_block_matrix", + tier: 2, + title: "at least one blinded-block delivery (200/202)", + data_source: CbPrometheus, + feature_asserted: false, + severity_note: "judged on the BEACON side (what CB returned to the CL), NOT the relay side: losing relays cannot serve a payload they never won and 4xx/5xx from them is expected. FAIL on a beacon-side 5xx; escalates to tier 1", + }, + CatalogEntry { + id: "cb_status_matrix", + tier: 2, + title: "status endpoint answering 200", + data_source: CbPrometheus, + feature_asserted: false, + severity_note: "tier 2 -> ESCALATES to tier 1 on FAIL (5xx); SKIP if no 200s / metrics absent", + }, + CatalogEntry { + id: "cb_relay_v2_unsupported", + tier: 2, + title: "no v2 submit_block lost to a relay that 404s the v2 route", + data_source: CbPrometheus, + feature_asserted: false, + severity_note: "tier 2 -> ESCALATES to tier 1 on FAIL: every builder block the proposer chose is LOST (CB will not downgrade v2->v1). Usually the relay's route config, not a capability gap (helix needs GetPayloadV2 in enabled_routes); SKIP/PASS if metrics absent", + }, + CatalogEntry { + id: "cb_v2_fallback", + tier: 2, + title: "no v2->v1 submitBlindedBlock fallbacks", + data_source: CbPrometheus, + feature_asserted: false, + severity_note: "INERT - always SKIP: commit-boost registers no v2->v1 fallback counter, so the check could only ever PASS. Relay v2 support is owned by cb_relay_v2_unsupported", + }, + CatalogEntry { + id: "cb_relay_latency", + tier: 2, + title: "p95 relay latency < 500 ms", + data_source: CbPrometheus, + feature_asserted: false, + severity_note: "WARN over threshold; SKIP if the histogram is absent / degenerate", + }, + ] +} + +/// Entry point for `sim checks --list [--json]`. +/// +/// `--list` emits the catalog (a readable table, or JSON with `--json`). Without +/// `--list` there is nothing else to do, so it points the caller at the flag. +pub fn run(list: bool, json: bool) -> Result<()> { + if !list { + println!( + "sim checks: pass --list to emit the check catalog (add --json for machine output)." + ); + return Ok(()); + } + let entries = catalog(); + if json { + // Wrap the array in an envelope so the verdict rule travels WITH the data. + let doc = serde_json::json!({ + "verdict_rule": VERDICT_RULE, + "checks": entries, + }); + println!("{}", serde_json::to_string_pretty(&doc)?); + } else { + print_table(&entries); + } + Ok(()) +} + +/// Render the catalog as an aligned, human-readable table. +fn print_table(entries: &[CatalogEntry]) { + println!("cb-verify check catalog ({} checks)", entries.len()); + println!("{VERDICT_RULE}"); + println!(); + + let id_w = entries.iter().map(|e| e.id.len()).max().unwrap_or(2).max(2); + let src_w = entries + .iter() + .map(|e| e.data_source.as_str().len()) + .max() + .unwrap_or(6) + .max(6); + + println!( + "{:4} {:>4} {:4} {:>4} {: = entries.iter().map(|e| e.id).collect(); + ids.sort_unstable(); + let mut deduped = ids.clone(); + deduped.dedup(); + assert_eq!(ids, deduped, "catalog has duplicate check ids"); + } + + #[test] + fn only_the_feature_checks_assert_a_feature() { + // Guards the CHECKS.md invariant: exactly these checks positively assert + // that a scenario's feature codepath fired — the two runtime-behaviour + // checks (best_bid, mux.routing) plus the three Law-3 feature-fired + // checks (skip_sigverify is a WARN-only honest report). + let feature_ids: Vec<&str> = catalog() + .iter() + .filter(|e| e.feature_asserted) + .map(|e| e.id) + .collect(); + assert_eq!( + feature_ids, + vec![ + "relay.best_bid", + "mux.routing", + "feature.timing_games", + "feature.extra_validation", + "feature.min_bid", + "feature.skip_sigverify", + "signer.pubkeys", + ] + ); + } + + #[test] + fn json_envelope_carries_the_verdict_rule() { + let doc = serde_json::json!({ + "verdict_rule": VERDICT_RULE, + "checks": catalog(), + }); + let s = serde_json::to_string(&doc).unwrap(); + assert!(s.contains("tier-1 FAIL"), "verdict rule must be present"); + assert!(s.contains("chain_finality"), "checks must serialize"); + assert!( + s.contains("relay-data-api"), + "data-source token must serialize" + ); + } +} diff --git a/src/bin/sim/cli.rs b/src/bin/sim/cli.rs new file mode 100644 index 0000000..f21dcea --- /dev/null +++ b/src/bin/sim/cli.rs @@ -0,0 +1,81 @@ +//! `sim` CLI surface (clap derive). +//! +//! Task 0 scaffold: the argument shape is wired; the subcommand bodies are +//! stubs implemented in later tasks (preflight = Task 3, triage = Task 2). + +use std::path::PathBuf; + +use clap::{Parser, Subcommand, ValueEnum}; + +/// Structured preflight + triage for Commit-Boost Kurtosis testnets. +#[derive(Debug, Parser)] +#[command(name = "sim", about = "helix preflight + triage for the sim harness")] +pub struct Cli { + /// Output format for the structured `tracing` stream. + #[arg(long, value_enum, global = true, default_value_t = LogFormat::Pretty)] + pub log_format: LogFormat, + + #[command(subcommand)] + pub command: Command, +} + +/// How the structured event stream is rendered. +#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)] +pub enum LogFormat { + /// Human-readable rendering (default). + Pretty, + /// One JSON object per event (for agents / machine consumption). + Json, +} + +#[derive(Debug, Subcommand)] +pub enum Command { + /// Validate a launch args-file BEFORE running the testnet (helix config-parse). + Preflight { + /// Path to the kurtosis args-file to validate. + args_file: PathBuf, + }, + /// Attach to an already-broken enclave and extract each service's root cause. + Triage { + /// Name of the kurtosis enclave to triage. + enclave: String, + }, + /// Emit the machine-readable catalog of what `cb-verify` asserts, so an agent + /// can discover the harness's contract without reading source or CHECKS.md. + Checks { + /// Emit the check catalog (required — without it there is nothing to do). + #[arg(long)] + list: bool, + /// Emit the catalog as JSON instead of a readable table. + #[arg(long)] + json: bool, + }, + /// Host-prerequisite preflight for a devnet: kurtosis, docker, memory + /// headroom, the CB image, and the ethereum-package submodule. + Doctor, + /// Compare two verification reports (JSON) and surface the verdict delta. + /// Exits nonzero if any check regressed — usable as a CI regression gate + /// after an image bump. + Diff { + /// The baseline report (the "from" side). + from: PathBuf, + /// The new report (the "to" side). + to: PathBuf, + /// Emit the structured diff as JSON instead of a readable summary. + #[arg(long)] + json: bool, + }, + /// Generate Kurtosis args-files for the CB test scenarios (Rust port of the + /// retired `generate_kurtosis_configs.py`). + Generate { + /// Scenario name (e.g. `cb-basic`); omit to generate all six. + scenario: Option, + /// Directory to write the generated `.yml` files into. + #[arg(long, default_value = "configs/generated")] + out_dir: PathBuf, + /// Don't write; instead verify the on-disk configs already match what the + /// generator would produce, and exit nonzero on any drift (CI / agent gate). + #[arg(long)] + check: bool, + }, +} diff --git a/src/bin/sim/diagnose.rs b/src/bin/sim/diagnose.rs new file mode 100644 index 0000000..b5cce23 --- /dev/null +++ b/src/bin/sim/diagnose.rs @@ -0,0 +1,360 @@ +//! Root-cause extraction from a crashed service's logs (Task 2, the heart). +//! +//! `extract_root_cause` is PURE: log text in, an optional structured cause out. +//! It is pattern-based, not string-match-based — it captures the varying field +//! name / message so held-out crashes (a field it has never seen) are diagnosed +//! the same as the ones we captured today. The `triage` entry point owns all the +//! process I/O (kurtosis / docker); this module never touches the outside world. +//! +//! Why this exists: `kurtosis service logs` routes through a broker that MASKS +//! the real Rust panic behind a grpc/marshaling error. The extractor must skip +//! such masking lines and return the innermost app-code panic. + +// The public API is wired into `triage::run`; the tests below exercise it. + +use serde::Serialize; + +/// The kind of failure we recognised in a service's logs. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +pub enum CauseKind { + /// A Rust `panicked at …` — the strongest, most specific signal. + Panic, + /// The process was killed with no panic message (OOM / SIGKILL). + Killed, + /// A non-panic fatal error (bind failure, connection refused, `os error`). + Fatal, +} + +/// A structured root cause extracted from a log stream. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct RootCause { + pub kind: CauseKind, + /// `file:line:col` for a panic; `None` for a fatal / kill. + pub location: Option, + /// The panic / error message (the captured field name lives here). + pub message: String, + /// A small trailing slice of the (ANSI-stripped) log for context. + pub log_tail: String, +} + +/// Extract the root cause from a service's log text. +/// +/// Precedence: an app-code Rust panic (root, not the first masking line) > +/// a non-panic fatal (`os error`, address-in-use, connection refused) > +/// a bare kill (OOM / SIGKILL) > `None` (clean logs). +pub fn extract_root_cause(logs: &str) -> Option { + let clean = strip_ansi(logs); + let lines: Vec<&str> = clean.lines().collect(); + let tail = log_tail(&lines); + + // 1) Rust panics win. Collect them all, then pick the ROOT: prefer an + // app-code (`.rs`) location, and among those the LAST (innermost) — so a + // grpc/broker masking line that precedes the real panic never wins. + let panics: Vec = (0..lines.len()) + .filter_map(|i| parse_panic_at(&lines, i)) + .collect(); + if let Some(p) = choose_root_panic(&panics) { + return Some(RootCause { + kind: CauseKind::Panic, + location: Some(p.location.clone()), + message: p.message.clone(), + log_tail: tail, + }); + } + + // 2) A non-panic fatal (bind failure / refused / `os error`). + if let Some(msg) = find_fatal(&lines) { + return Some(RootCause { + kind: CauseKind::Fatal, + location: None, + message: msg, + log_tail: tail, + }); + } + + // 3) A bare kill with no message (OOM / SIGKILL). Do NOT fabricate a panic. + if let Some(msg) = find_kill(&lines) { + return Some(RootCause { + kind: CauseKind::Killed, + location: None, + message: msg, + log_tail: tail, + }); + } + + None +} + +/// A single parsed `panicked at` occurrence. +struct Panic { + location: String, + message: String, +} + +/// Parse a panic anchored at `lines[i]` if that line contains `panicked at`. +/// +/// New-format panics read `… panicked at :::` with the message +/// either inline after the trailing colon or on the next non-empty line. +fn parse_panic_at(lines: &[&str], i: usize) -> Option { + let line = lines[i]; + let anchor = line.find("panicked at ")?; + let rem = &line[anchor + "panicked at ".len()..]; + + let loc_colon = find_location_end(rem)?; + let location = rem[..loc_colon].trim().to_string(); + let inline = rem[loc_colon + 1..].trim(); + + let message = if !inline.is_empty() { + inline.to_string() + } else { + // The message is the next non-empty line (skip blank lines). + lines + .iter() + .skip(i + 1) + .map(|l| l.trim()) + .find(|l| !l.is_empty()) + .unwrap_or("") + .to_string() + }; + + Some(Panic { location, message }) +} + +/// Find the trailing colon of a `:::` location inside `rem`, +/// returning that colon's byte index. Purely structural — captures whatever the +/// path is, so a never-seen crate path is handled the same as a known one. +fn find_location_end(rem: &str) -> Option { + let b = rem.as_bytes(); + let mut i = 0; + while i < b.len() { + if b[i] != b':' { + i += 1; + continue; + } + // Expect ::: starting at i. + let mut j = i + 1; + let d1 = j; + while j < b.len() && b[j].is_ascii_digit() { + j += 1; + } + if j == d1 || j >= b.len() || b[j] != b':' { + i += 1; + continue; + } + j += 1; + let d2 = j; + while j < b.len() && b[j].is_ascii_digit() { + j += 1; + } + if j == d2 || j >= b.len() || b[j] != b':' { + i += 1; + continue; + } + // b[j] is the trailing colon after the column number. + return Some(j); + } + None +} + +/// Choose the root panic: prefer app-code (`.rs`) locations, then the innermost +/// (last) — never the first, which may be a broker/CLI masking frame. +fn choose_root_panic(panics: &[Panic]) -> Option<&Panic> { + panics + .iter() + .rev() + .find(|p| p.location.contains(".rs")) + .or_else(|| panics.last()) +} + +/// Match a non-panic fatal line (bind failure, refused, generic `os error`). +fn find_fatal(lines: &[&str]) -> Option { + const NEEDLES: [&str; 3] = ["Address already in use", "connection refused", "os error"]; + lines.iter().find_map(|line| { + let lower = line.to_ascii_lowercase(); + NEEDLES + .iter() + .any(|n| lower.contains(&n.to_ascii_lowercase())) + .then(|| line.trim().to_string()) + }) +} + +/// Detect a bare process kill (OOM / SIGKILL) with no panic or fatal message. +fn find_kill(lines: &[&str]) -> Option { + const NEEDLES: [&str; 4] = ["killed", "sigkill", "signal: 9", "out of memory"]; + lines.iter().rev().find_map(|line| { + let lower = line.to_ascii_lowercase(); + NEEDLES + .iter() + .any(|n| lower.contains(n)) + .then(|| line.trim().to_string()) + }) +} + +/// Strip ANSI CSI escape sequences (`ESC [ … `) from a string. +fn strip_ansi(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + let mut chars = s.chars().peekable(); + while let Some(c) = chars.next() { + if c != '\u{1b}' { + out.push(c); + continue; + } + // ESC: consume a `[ … ` CSI sequence. + if chars.peek() == Some(&'[') { + chars.next(); + for e in chars.by_ref() { + if ('\u{40}'..='\u{7e}').contains(&e) { + break; + } + } + } + // A lone ESC (or non-CSI) is simply dropped. + } + out +} + +/// Keep a small trailing slice (last ~20 lines) of the log for context. +fn log_tail(lines: &[&str]) -> String { + const TAIL: usize = 20; + let start = lines.len().saturating_sub(TAIL); + lines[start..].join("\n") +} + +#[cfg(test)] +mod tests { + use super::*; + + const SERDE_MISSING: &str = + include_str!("../../../tests/fixtures/helix_serde_missing_field.log"); + const PREGENESIS: &str = include_str!("../../../tests/fixtures/helix_pregenesis_unwrap.log"); + const INVENTED: &str = include_str!("../../../tests/fixtures/invented_field.log"); + const MULTI_MASKED: &str = include_str!("../../../tests/fixtures/multi_masked.log"); + const ANSI: &str = include_str!("../../../tests/fixtures/ansi_colored.log"); + const NEXT_LINE: &str = include_str!("../../../tests/fixtures/next_line_message.log"); + const OOM: &str = include_str!("../../../tests/fixtures/oom_killed.log"); + const BIND: &str = include_str!("../../../tests/fixtures/bind_error.log"); + const CLEAN: &str = include_str!("../../../tests/fixtures/clean.log"); + + #[test] + fn serde_missing_field_panic() { + let rc = extract_root_cause(SERDE_MISSING).expect("should find a panic"); + assert_eq!(rc.kind, CauseKind::Panic); + assert_eq!( + rc.location.as_deref(), + Some("/app/crates/common/src/config.rs:203:51") + ); + // Field captured from the message, not string-matched. + assert!( + rc.message.contains("decoder"), + "message should carry the missing field name: {}", + rc.message + ); + } + + #[test] + fn pregenesis_unwrap_panic() { + let rc = extract_root_cause(PREGENESIS).expect("should find a panic"); + assert_eq!(rc.kind, CauseKind::Panic); + assert_eq!( + rc.location.as_deref(), + Some("crates/common/src/chain_info.rs:63:26") + ); + assert!( + rc.message.contains("unwrap()") && rc.message.contains("None"), + "message should be the unwrap-on-None panic: {}", + rc.message + ); + } + + #[test] + fn held_out_field_is_captured_not_matched() { + // A field name we have never seen — proves pattern capture, not memory. + let rc = extract_root_cause(INVENTED).expect("should find a panic"); + assert_eq!(rc.kind, CauseKind::Panic); + assert!( + rc.message.contains("foobar"), + "held-out field name must be captured: {}", + rc.message + ); + } + + #[test] + fn picks_root_panic_over_masking_line() { + // The grpc/marshaling masking lines come FIRST; the real panic is later. + let rc = extract_root_cause(MULTI_MASKED).expect("should find a panic"); + assert_eq!(rc.kind, CauseKind::Panic); + assert_eq!( + rc.location.as_deref(), + Some("/app/crates/common/src/config.rs:203:51") + ); + assert!( + rc.message.contains("network_config"), + "should return the ROOT panic message, not the masking line: {}", + rc.message + ); + // The masking noise must not leak into the message. + assert!( + !rc.message.contains("UTF-8"), + "masking line must not be the reported cause: {}", + rc.message + ); + } + + #[test] + fn strips_ansi_escape_codes() { + let rc = extract_root_cause(ANSI).expect("should find a panic"); + assert_eq!(rc.kind, CauseKind::Panic); + assert_eq!(rc.location.as_deref(), Some("src/main.rs:10:5")); + assert!( + !rc.message.contains('\u{1b}'), + "ANSI escape must be stripped from the message: {:?}", + rc.message + ); + assert!( + rc.message.contains("explicit panic"), + "message should survive ANSI stripping: {}", + rc.message + ); + } + + #[test] + fn panic_message_on_next_line() { + let rc = extract_root_cause(NEXT_LINE).expect("should find a panic"); + assert_eq!(rc.kind, CauseKind::Panic); + assert_eq!(rc.location.as_deref(), Some("src/worker.rs:88:12")); + assert!( + rc.message.contains("something went terribly wrong"), + "next-line message should be captured: {}", + rc.message + ); + } + + #[test] + fn oom_returns_killed_not_a_fake_panic() { + let rc = extract_root_cause(OOM).expect("should report the kill"); + assert_eq!(rc.kind, CauseKind::Killed); + assert!(rc.location.is_none(), "a kill has no source location"); + } + + #[test] + fn non_rust_fatal_is_matched() { + let rc = extract_root_cause(BIND).expect("should find a fatal"); + assert_eq!(rc.kind, CauseKind::Fatal); + assert!( + rc.message.contains("os error 98") || rc.message.contains("Address already in use"), + "fatal message should name the bind error: {}", + rc.message + ); + } + + #[test] + fn clean_logs_return_none() { + assert!(extract_root_cause(CLEAN).is_none()); + } + + #[test] + fn log_tail_is_populated_for_a_cause() { + let rc = extract_root_cause(SERDE_MISSING).expect("panic"); + assert!(!rc.log_tail.is_empty(), "log_tail should carry context"); + } +} diff --git a/src/bin/sim/diff.rs b/src/bin/sim/diff.rs new file mode 100644 index 0000000..8434269 --- /dev/null +++ b/src/bin/sim/diff.rs @@ -0,0 +1,531 @@ +//! `sim diff`: compare two verification reports and surface the verdict delta. +//! +//! The report is a round-trippable JSON interchange format (the `report` types +//! derive `Deserialize`), and each run records its [`Provenance`] (config hash + +//! resolved image ids). This verb answers "what changed between run A and run +//! B, and did anything regress?" — the CI shape being: bump an image, re-run, +//! `sim diff old.json new.json`, fail the pipeline if a check regressed. +//! +//! A regression is a check whose severity INCREASED among real verdicts +//! (Fail > Warn > Pass). Transitions in and out of `Skip` are COVERAGE changes, +//! not severity changes — a Skip carries no verdict information, so neither +//! direction is a regression, but both are printed so a human sees coverage +//! move. The report's own tier-1 gate, not this diff, owns pass/fail. + +use std::collections::BTreeMap; +use std::path::Path; + +use cb_testnet_verifier::checks::CheckStatus; +use cb_testnet_verifier::report::VerificationReport; +use serde::Serialize; + +/// Direction of a single check's verdict change. +/// +/// NOTE the `Skip` asymmetry, found by dogfooding this tool on the sigverify +/// differential (2026-08-04): `CheckStatus`'s `Ord` ranks `Skip` BELOW `Pass`, +/// which is right for worst-status AGGREGATION (a Skip must not win a fold over +/// a Pass) but wrong for TRANSITIONS — a Skip is not "better than a Pass", it is +/// NO INFORMATION. Ordering `SKIP -> PASS` as a severity increase reported a +/// check that started running and passed as REGRESSED, and dragged a run that +/// went FAIL -> PASS to an overall REGRESSION verdict. So transitions in and out +/// of `Skip` are their own category: coverage, not severity. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum Direction { + /// Severity increased among real verdicts (e.g. Pass -> Fail). + Regressed, + /// Severity decreased among real verdicts (e.g. Fail -> Pass). + Improved, + /// A check STOPPED running (real verdict -> Skip). Not a regression (it is + /// not a FAIL), but surfaced: the run lost coverage it used to have. + CoverageLost, + /// A check STARTED running (Skip -> real verdict). Never a regression, even + /// when it starts by failing — the failure was there before, just unmeasured. + CoverageGained, +} + +/// Classify a verdict transition. `from != to` is the caller's precondition. +fn direction_of(from: CheckStatus, to: CheckStatus) -> Direction { + match (from, to) { + (CheckStatus::Skip, _) => Direction::CoverageGained, + (_, CheckStatus::Skip) => Direction::CoverageLost, + // Both are real verdicts: severity ordering applies as intended. + _ if to > from => Direction::Regressed, + _ => Direction::Improved, + } +} + +/// A check whose verdict changed between the two reports. +#[derive(Debug, Clone, Serialize)] +pub struct CheckDelta { + pub id: String, + pub from: CheckStatus, + pub to: CheckStatus, + pub direction: Direction, +} + +/// A check present in only one of the two reports. +#[derive(Debug, Clone, Serialize)] +pub struct CheckPresence { + pub id: String, + pub status: CheckStatus, +} + +/// One image role whose resolved name or id changed between the two reports. +#[derive(Debug, Clone, Serialize)] +pub struct ImageDelta { + pub role: String, + pub from_name: Option, + pub to_name: Option, + pub from_id: Option, + pub to_id: Option, +} + +/// The full structured diff of two reports. +#[derive(Debug, Clone, Serialize)] +pub struct DiffReport { + pub from_overall: CheckStatus, + pub to_overall: CheckStatus, + pub overall_regressed: bool, + /// Checks whose verdict changed, sorted by id. + pub changed: Vec, + /// Checks in B but not A, sorted by id. + pub added: Vec, + /// Checks in A but not B, sorted by id. + pub removed: Vec, + /// `(from, to)` config hashes, present only when they differ. + pub config_hash: Option<(Option, Option)>, + /// Image roles whose name or id changed, sorted by role. + pub images: Vec, +} + +impl DiffReport { + /// True iff anything got a strictly-worse verdict: the overall result + /// regressed, or any individual check regressed among real verdicts. + /// Coverage changes (either direction across `Skip`) never count. + pub fn has_regression(&self) -> bool { + self.overall_regressed + || self + .changed + .iter() + .any(|c| c.direction == Direction::Regressed) + } +} + +/// Map each check id to its status. Later duplicates win (a report should not +/// have duplicate ids; if it does, the last is what a reader would see printed). +fn status_by_id(report: &VerificationReport) -> BTreeMap { + report + .checks + .iter() + .map(|c| (c.id.clone(), c.status)) + .collect() +} + +/// Pure verdict-diff logic (the Law 4 test seam; `run` only does file IO). +pub fn diff_reports(a: &VerificationReport, b: &VerificationReport) -> DiffReport { + let from = status_by_id(a); + let to = status_by_id(b); + + let mut changed = Vec::new(); + let mut removed = Vec::new(); + for (id, &from_status) in &from { + match to.get(id) { + Some(&to_status) if to_status != from_status => { + let direction = direction_of(from_status, to_status); + changed.push(CheckDelta { + id: id.clone(), + from: from_status, + to: to_status, + direction, + }); + } + Some(_) => {} + None => removed.push(CheckPresence { + id: id.clone(), + status: from_status, + }), + } + } + + let added: Vec = to + .iter() + .filter(|(id, _)| !from.contains_key(*id)) + .map(|(id, &status)| CheckPresence { + id: id.clone(), + status, + }) + .collect(); + + // Same classification as per-check: an overall result that moves across + // Skip is a coverage change, not a regression. (In practice `report.result` + // is only ever Pass or Fail, so this matches the raw severity compare on + // real reports — but it keeps the one rule in one place.) + let overall_regressed = + a.result != b.result && direction_of(a.result, b.result) == Direction::Regressed; + + DiffReport { + from_overall: a.result, + to_overall: b.result, + overall_regressed, + changed, + added, + removed, + config_hash: diff_config_hash(a, b), + images: diff_images(a, b), + } +} + +/// `(from, to)` config hashes if they differ, else `None`. A `None` provenance +/// on either side yields a `None` hash on that side. +fn diff_config_hash( + a: &VerificationReport, + b: &VerificationReport, +) -> Option<(Option, Option)> { + let ah = a.provenance.as_ref().and_then(|p| p.config_hash.clone()); + let bh = b.provenance.as_ref().and_then(|p| p.config_hash.clone()); + if ah != bh { Some((ah, bh)) } else { None } +} + +/// Image roles whose configured name or resolved id changed. Roles present on +/// only one side are reported with the missing side's fields as `None`. +fn diff_images(a: &VerificationReport, b: &VerificationReport) -> Vec { + let by_role = |r: &VerificationReport| -> BTreeMap)> { + r.provenance + .as_ref() + .map(|p| { + p.images + .iter() + .map(|i| (i.role.clone(), (i.name.clone(), i.id.clone()))) + .collect() + }) + .unwrap_or_default() + }; + let am = by_role(a); + let bm = by_role(b); + + let mut roles: Vec<&String> = am.keys().chain(bm.keys()).collect(); + roles.sort(); + roles.dedup(); + + let mut out = Vec::new(); + for role in roles { + let av = am.get(role); + let bv = bm.get(role); + let from_name = av.map(|(n, _)| n.clone()); + let to_name = bv.map(|(n, _)| n.clone()); + let from_id = av.and_then(|(_, i)| i.clone()); + let to_id = bv.and_then(|(_, i)| i.clone()); + if from_name != to_name || from_id != to_id { + out.push(ImageDelta { + role: role.clone(), + from_name, + to_name, + from_id, + to_id, + }); + } + } + out +} + +// --------------------------------------------------------------------------- +// IO wrapper +// --------------------------------------------------------------------------- + +fn load(path: &Path) -> eyre::Result { + let bytes = std::fs::read(path) + .map_err(|e| eyre::eyre!("failed to read report '{}': {e}", path.display()))?; + serde_json::from_slice(&bytes) + .map_err(|e| eyre::eyre!("failed to parse report '{}' as JSON: {e}", path.display())) +} + +/// Load two reports, diff them, print the result, and exit nonzero if a check +/// regressed (so the diff is usable as a CI gate). `Err` is reserved for IO / +/// parse failures; a regression is a clean nonzero exit, not an error. +pub fn run(from: &Path, to: &Path, json: bool) -> eyre::Result<()> { + let a = load(from)?; + let b = load(to)?; + let diff = diff_reports(&a, &b); + + if json { + println!("{}", serde_json::to_string_pretty(&diff)?); + } else { + print_pretty(&a, &b, &diff); + } + + if diff.has_regression() { + std::process::exit(1); + } + Ok(()) +} + +fn print_pretty(a: &VerificationReport, b: &VerificationReport, diff: &DiffReport) { + println!( + "Report diff: {}@{} -> {}@{}", + a.enclave, a.timestamp, b.enclave, b.timestamp + ); + let flag = if diff.overall_regressed { + " [REGRESSION]" + } else { + "" + }; + println!( + "Overall: {} -> {}{}", + diff.from_overall, diff.to_overall, flag + ); + + if diff.changed.is_empty() && diff.added.is_empty() && diff.removed.is_empty() { + println!("\nNo per-check verdict changes."); + } else { + println!("\nCheck changes:"); + for c in &diff.changed { + let tag = match c.direction { + Direction::Regressed => "REGRESSED", + Direction::Improved => "improved ", + Direction::CoverageLost => "cov-lost ", + Direction::CoverageGained => "cov-gain ", + }; + println!(" [{tag}] {}: {} -> {}", c.id, c.from, c.to); + } + for p in &diff.added { + println!(" [added] + {}: {}", p.id, p.status); + } + for p in &diff.removed { + println!(" [removed] - {}: {}", p.id, p.status); + } + } + + if diff.config_hash.is_some() || !diff.images.is_empty() { + println!("\nProvenance:"); + if let Some((from, to)) = &diff.config_hash { + let show = |h: &Option| h.clone().unwrap_or_else(|| "none".to_string()); + println!(" config_hash: {} -> {}", show(from), show(to)); + } + for img in &diff.images { + let show = |o: &Option| o.clone().unwrap_or_else(|| "none".to_string()); + println!( + " image[{}]: name {} -> {}, id {} -> {}", + img.role, + show(&img.from_name), + show(&img.to_name), + show(&img.from_id), + show(&img.to_id), + ); + } + } + + println!(); + if diff.has_regression() { + println!("Verdict: REGRESSION (a check got a worse verdict)"); + } else { + println!("Verdict: no regression"); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use cb_testnet_verifier::checks::CheckResult; + use cb_testnet_verifier::report::{ImageRef, Provenance}; + + /// Build a report from `(id, status)` pairs + optional provenance. + fn report( + checks: &[(&str, CheckStatus)], + provenance: Option, + ) -> VerificationReport { + let checks: Vec = checks + .iter() + .map(|(id, status)| match status { + CheckStatus::Pass => CheckResult::pass(*id, 1, "ok"), + CheckStatus::Fail => CheckResult::fail(*id, 1, "bad"), + CheckStatus::Warn => CheckResult::warn(*id, 1, "meh"), + CheckStatus::Skip => CheckResult::skip(*id, 1, "n/a"), + }) + .collect(); + // The overall result mirrors the worst check severity, matching how a + // real report is minted (not load-bearing for these tests, which set it + // explicitly via the pairs' worst status). + let result = checks_worst(&checks); + VerificationReport { + enclave: "e".to_string(), + timestamp: "t".to_string(), + observation_window: None, + result, + checks, + provenance, + } + } + + fn checks_worst(checks: &[CheckResult]) -> CheckStatus { + checks + .iter() + .map(|c| c.status) + .max() + .unwrap_or(CheckStatus::Pass) + } + + fn img(role: &str, name: &str, id: Option<&str>) -> ImageRef { + ImageRef { + role: role.to_string(), + name: name.to_string(), + id: id.map(str::to_string), + } + } + + fn prov(hash: Option<&str>, images: Vec) -> Provenance { + Provenance { + config_path: None, + config_hash: hash.map(str::to_string), + images, + } + } + + #[test] + fn identical_reports_have_no_changes_and_no_regression() { + let a = report(&[("x", CheckStatus::Pass), ("y", CheckStatus::Warn)], None); + let b = report(&[("x", CheckStatus::Pass), ("y", CheckStatus::Warn)], None); + let d = diff_reports(&a, &b); + assert!(d.changed.is_empty()); + assert!(d.added.is_empty()); + assert!(d.removed.is_empty()); + assert!(!d.has_regression()); + } + + #[test] + fn pass_to_fail_is_a_regression() { + let a = report(&[("x", CheckStatus::Pass)], None); + let b = report(&[("x", CheckStatus::Fail)], None); + let d = diff_reports(&a, &b); + assert_eq!(d.changed.len(), 1); + assert_eq!(d.changed[0].direction, Direction::Regressed); + assert!(d.has_regression()); + assert!(d.overall_regressed); + } + + #[test] + fn fail_to_pass_is_an_improvement_not_a_regression() { + let a = report(&[("x", CheckStatus::Fail)], None); + let b = report(&[("x", CheckStatus::Pass)], None); + let d = diff_reports(&a, &b); + assert_eq!(d.changed[0].direction, Direction::Improved); + assert!(!d.has_regression()); + } + + #[test] + fn skip_to_pass_is_coverage_gained_not_a_regression() { + // THE DOGFOOD BUG (2026-08-04): CheckStatus::Ord ranks Skip below Pass + // (right for worst-status folds), so this transition read as a severity + // INCREASE and reported a check that started running and passed as + // REGRESSED — dragging a run that went FAIL -> PASS to an overall + // REGRESSION verdict. Starting to run is never a regression. + let a = report(&[("x", CheckStatus::Skip)], None); + let b = report(&[("x", CheckStatus::Pass)], None); + let d = diff_reports(&a, &b); + assert_eq!(d.changed[0].direction, Direction::CoverageGained); + assert!(!d.has_regression(), "SKIP -> PASS must not be a regression"); + } + + #[test] + fn skip_to_fail_is_coverage_gained_not_a_regression() { + // Subtle: a check that starts running and immediately fails is NOT a + // regression either — the failure existed before, just unmeasured. + // (The report's own tier-1 gate still fails the run on its own terms.) + let a = report(&[("x", CheckStatus::Skip)], None); + let b = report(&[("x", CheckStatus::Fail)], None); + let d = diff_reports(&a, &b); + assert_eq!(d.changed[0].direction, Direction::CoverageGained); + assert!( + !d.has_regression(), + "newly-measured failure is not a regression" + ); + } + + #[test] + fn pass_to_skip_is_shown_but_is_not_a_regression() { + // Coverage loss: a check stopped running. Severity DROPPED (Skip < Pass) + // so it is Improved-by-severity and does NOT fail the gate — the report's + // own tier-1 gate owns pass/fail, not the diff. But it IS surfaced. + let a = report(&[("x", CheckStatus::Pass)], None); + let b = report(&[("x", CheckStatus::Skip)], None); + let d = diff_reports(&a, &b); + assert_eq!(d.changed.len(), 1, "the coverage change is still surfaced"); + assert_eq!(d.changed[0].to, CheckStatus::Skip); + assert_eq!(d.changed[0].direction, Direction::CoverageLost); + assert!(!d.has_regression(), "Skip is not a FAIL"); + } + + #[test] + fn added_and_removed_checks_are_tracked() { + let a = report(&[("only_a", CheckStatus::Pass)], None); + let b = report(&[("only_b", CheckStatus::Pass)], None); + let d = diff_reports(&a, &b); + assert_eq!(d.removed.len(), 1); + assert_eq!(d.removed[0].id, "only_a"); + assert_eq!(d.added.len(), 1); + assert_eq!(d.added[0].id, "only_b"); + assert!(!d.has_regression(), "presence changes are not regressions"); + } + + #[test] + fn config_hash_change_is_reported_only_when_it_differs() { + let same_a = report(&[("x", CheckStatus::Pass)], Some(prov(Some("abc"), vec![]))); + let same_b = report(&[("x", CheckStatus::Pass)], Some(prov(Some("abc"), vec![]))); + assert!(diff_reports(&same_a, &same_b).config_hash.is_none()); + + let a = report(&[("x", CheckStatus::Pass)], Some(prov(Some("abc"), vec![]))); + let b = report(&[("x", CheckStatus::Pass)], Some(prov(Some("def"), vec![]))); + let d = diff_reports(&a, &b); + assert_eq!( + d.config_hash, + Some((Some("abc".to_string()), Some("def".to_string()))) + ); + } + + #[test] + fn image_id_change_is_reported_name_unchanged() { + let a = report( + &[("x", CheckStatus::Pass)], + Some(prov( + None, + vec![img("mev_boost", "cb:kurtosis", Some("sha256:aa"))], + )), + ); + let b = report( + &[("x", CheckStatus::Pass)], + Some(prov( + None, + vec![img("mev_boost", "cb:kurtosis", Some("sha256:bb"))], + )), + ); + let d = diff_reports(&a, &b); + assert_eq!(d.images.len(), 1); + assert_eq!(d.images[0].role, "mev_boost"); + assert_eq!(d.images[0].from_id, Some("sha256:aa".to_string())); + assert_eq!(d.images[0].to_id, Some("sha256:bb".to_string())); + assert_eq!(d.images[0].from_name, d.images[0].to_name, "name unchanged"); + } + + #[test] + fn unchanged_image_is_not_reported() { + let same = vec![img("mev_boost", "cb:kurtosis", Some("sha256:aa"))]; + let a = report(&[("x", CheckStatus::Pass)], Some(prov(None, same.clone()))); + let b = report(&[("x", CheckStatus::Pass)], Some(prov(None, same))); + assert!(diff_reports(&a, &b).images.is_empty()); + } + + #[test] + fn a_regressed_report_round_trips_through_json() { + // The whole point of the Deserialize derive: a serialized report can be + // read back and diffed. Serialize A and B, deserialize, diff. + let a = report(&[("x", CheckStatus::Pass)], None); + let b = report(&[("x", CheckStatus::Fail)], None); + let a_json = serde_json::to_string(&a).unwrap(); + let b_json = serde_json::to_string(&b).unwrap(); + let a2: VerificationReport = serde_json::from_str(&a_json).unwrap(); + let b2: VerificationReport = serde_json::from_str(&b_json).unwrap(); + let d = diff_reports(&a2, &b2); + assert!(d.has_regression()); + assert_eq!(d.changed[0].to, CheckStatus::Fail); + } +} diff --git a/src/bin/sim/doctor.rs b/src/bin/sim/doctor.rs new file mode 100644 index 0000000..d546ada --- /dev/null +++ b/src/bin/sim/doctor.rs @@ -0,0 +1,490 @@ +//! `sim doctor` — host-prerequisite preflight for a Kurtosis devnet. +//! +//! Turns the hard-won gotchas in `docs/local-kurtosis-e2e.md` into one command: +//! is kurtosis installed (and the config-version-9-safe 1.18.1 pin), is docker +//! reachable, does the host have memory headroom for a ~10-min devnet, is the CB +//! image built, and is the forked `ethereum-package` submodule initialized. +//! +//! Structure mirrors the other verbs: a PURE classifier (`classify`, given probe +//! results -> verdict) that is unit-tested, and a thin IO layer (`gather_probes`) +//! that shells `kurtosis`/`docker` with `std::process::Command` and reads +//! `/proc/meminfo`. Sync only; no tokio. + +use std::fs; +use std::path::Path; +use std::process::Command; + +use eyre::Result; + +/// The default CB image tag (matches `.env.example`); overridable via +/// `MEV_BOOST_IMAGE` in `.env`. +const DEFAULT_CB_IMAGE: &str = "commit-boost/commit-boost:kurtosis"; + +/// The kurtosis version this box is pinned to. A NEWER CLI writes a +/// `config-version: 9` config file that 1.18.1 can't read (the clash documented +/// in `docs/local-kurtosis-e2e.md`), so anything but this is a WARN. +const PINNED_KURTOSIS: (u64, u64, u64) = (1, 18, 1); + +/// Memory a devnet wants available (MemAvailable + SwapFree), in MB. The task +/// floor is ~18GB; `scripts/run-and-verify.sh` uses 24000 with more headroom. +const NEED_MB: u64 = 18_000; + +/// Per-item verdict. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Status { + Ok, + Warn, + Fail, +} + +impl Status { + fn glyph(self) -> char { + match self { + Status::Ok => '\u{2713}', // ✓ + Status::Warn => '\u{26a0}', // ⚠ + Status::Fail => '\u{2717}', // ✗ + } + } +} + +/// One checklist line. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Item { + pub name: &'static str, + pub status: Status, + /// Whether this is a HARD prerequisite (a Fail exits nonzero). Only kurtosis + /// and docker are hard; memory/image/submodule are advisory warnings. + pub hard: bool, + pub detail: String, +} + +/// The raw probe results — the ONLY input to the pure classifier. `gather_probes` +/// fills this from the host; tests construct it directly. +#[derive(Debug, Clone)] +pub struct Probes { + /// Raw `kurtosis version` output, or `None` if the CLI is not installed. + pub kurtosis_version_raw: Option, + /// `docker info` returned exit 0. + pub docker_ok: bool, + /// `/proc/meminfo` MemAvailable, in MB. + pub mem_available_mb: u64, + /// `/proc/meminfo` SwapFree, in MB. + pub swap_free_mb: u64, + /// The CB image tag we looked for (from `.env` or the default). + pub cb_image: String, + /// `docker image inspect ` returned exit 0. + pub cb_image_present: bool, + /// The `ethereum-package` submodule directory is initialized (non-empty). + pub submodule_initialized: bool, +} + +/// The full classified report. +#[derive(Debug, Clone)] +pub struct Report { + pub items: Vec, + /// 0 = all hard prerequisites present; 1 = a hard prerequisite is missing. + pub exit_code: i32, +} + +/// Parse a `(major, minor, patch)` semver out of arbitrary text (e.g. +/// `"CLI Version: 1.18.1"`). Returns the first `N.N.N` run found. +pub fn parse_semver(text: &str) -> Option<(u64, u64, u64)> { + for token in text.split(|c: char| !(c.is_ascii_digit() || c == '.')) { + let mut parts = token.split('.'); + let (Some(a), Some(b), Some(c)) = (parts.next(), parts.next(), parts.next()) else { + continue; + }; + if parts.next().is_some() { + continue; // more than 3 components — not a plain semver + } + if let (Ok(a), Ok(b), Ok(c)) = (a.parse(), b.parse(), c.parse()) { + return Some((a, b, c)); + } + } + None +} + +/// The PURE decision: probe results -> checklist + exit code. No IO. +pub fn classify(p: &Probes) -> Report { + let mut items = Vec::new(); + + // (a) kurtosis installed + version pin. Missing = hard fail. + items.push(match &p.kurtosis_version_raw { + None => Item { + name: "kurtosis CLI", + status: Status::Fail, + hard: true, + detail: "not installed / not on PATH — install kurtosis-cli (see docs/local-kurtosis-e2e.md Step 0)".to_string(), + }, + Some(raw) => match parse_semver(raw) { + Some(v) if v == PINNED_KURTOSIS => Item { + name: "kurtosis CLI", + status: Status::Ok, + hard: true, + detail: format!("{}.{}.{} (pinned)", v.0, v.1, v.2), + }, + Some(v) if v > PINNED_KURTOSIS => Item { + name: "kurtosis CLI", + status: Status::Warn, + hard: true, + detail: format!( + "{}.{}.{} is newer than the pinned {}.{}.{} — risks the config-version-9 clash (1.18.1 can't read it); see docs/local-kurtosis-e2e.md", + v.0, v.1, v.2, PINNED_KURTOSIS.0, PINNED_KURTOSIS.1, PINNED_KURTOSIS.2, + ), + }, + Some(v) => Item { + name: "kurtosis CLI", + status: Status::Warn, + hard: true, + detail: format!( + "{}.{}.{} is older than the pinned {}.{}.{} — untested here", + v.0, v.1, v.2, PINNED_KURTOSIS.0, PINNED_KURTOSIS.1, PINNED_KURTOSIS.2, + ), + }, + None => Item { + name: "kurtosis CLI", + status: Status::Warn, + hard: true, + detail: format!("installed, but could not parse version from {raw:?}"), + }, + }, + }); + + // (b) docker daemon reachable. Missing = hard fail. + items.push(if p.docker_ok { + Item { + name: "docker daemon", + status: Status::Ok, + hard: true, + detail: "reachable (`docker info` ok)".to_string(), + } + } else { + Item { + name: "docker daemon", + status: Status::Fail, + hard: true, + detail: "unreachable — `docker info` failed; is the daemon running and are you in the docker group?".to_string(), + } + }); + + // (c) host memory headroom. Advisory warning only. + let usable = p.mem_available_mb + p.swap_free_mb; + items.push(if usable >= NEED_MB { + Item { + name: "host memory", + status: Status::Ok, + hard: false, + detail: format!( + "{usable}MB usable (avail {} + swapfree {}) >= {NEED_MB}MB", + p.mem_available_mb, p.swap_free_mb + ), + } + } else { + Item { + name: "host memory", + status: Status::Warn, + hard: false, + detail: format!( + "only {usable}MB usable (avail {} + swapfree {}) < {NEED_MB}MB — a devnet may stall or thrash", + p.mem_available_mb, p.swap_free_mb + ), + } + }); + + // (d) CB image present. Advisory warning only. + items.push(if p.cb_image_present { + Item { + name: "CB image", + status: Status::Ok, + hard: false, + detail: format!("{} present", p.cb_image), + } + } else { + Item { + name: "CB image", + status: Status::Warn, + hard: false, + detail: format!( + "{} not found locally — build it (`just build-all kurtosis`) or set MEV_BOOST_IMAGE in .env", + p.cb_image + ), + } + }); + + // (e) ethereum-package submodule initialized. Advisory warning only. + items.push(if p.submodule_initialized { + Item { + name: "ethereum-package submodule", + status: Status::Ok, + hard: false, + detail: "initialized (non-empty)".to_string(), + } + } else { + Item { + name: "ethereum-package submodule", + status: Status::Warn, + hard: false, + detail: "empty — run `git submodule update --init --recursive`".to_string(), + } + }); + + let exit_code = if items.iter().any(|i| i.hard && i.status == Status::Fail) { + 1 + } else { + 0 + }; + Report { items, exit_code } +} + +/// Entry point for `sim doctor`. Probes the host, prints the checklist, and +/// exits nonzero if a HARD prerequisite (kurtosis, docker) is missing. +pub fn run() -> Result<()> { + let probes = gather_probes(); + let report = classify(&probes); + + println!("sim doctor — devnet host preflight"); + println!(); + for item in &report.items { + println!(" {} {}: {}", item.status.glyph(), item.name, item.detail); + } + println!(); + if report.exit_code == 0 { + println!("hard prerequisites OK (warnings above are advisory)."); + } else { + println!("MISSING a hard prerequisite (kurtosis / docker) — fix the ✗ items above."); + } + + if report.exit_code != 0 { + std::process::exit(report.exit_code); + } + Ok(()) +} + +/// The IO layer: run the shell probes + read `/proc/meminfo`. Best-effort — a +/// failed probe becomes a negative/absent result, never a panic. +fn gather_probes() -> Probes { + let (mem_available_mb, swap_free_mb) = read_meminfo(Path::new("/proc/meminfo")); + let cb_image = cb_image_from_env(Path::new(".env")); + Probes { + kurtosis_version_raw: probe_kurtosis_version(), + docker_ok: cmd_ok("docker", &["info"]), + mem_available_mb, + swap_free_mb, + cb_image_present: cmd_ok("docker", &["image", "inspect", &cb_image]), + cb_image, + submodule_initialized: dir_non_empty(Path::new("ethereum-package")), + } +} + +/// `kurtosis version` stdout, or `None` if the binary isn't runnable. +fn probe_kurtosis_version() -> Option { + let out = Command::new("kurtosis").arg("version").output().ok()?; + let text = format!( + "{}{}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ); + if text.trim().is_empty() { + None + } else { + Some(text) + } +} + +/// True iff ` ` spawned and exited 0. +fn cmd_ok(prog: &str, args: &[&str]) -> bool { + Command::new(prog) + .args(args) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .status() + .map(|s| s.success()) + .unwrap_or(false) +} + +/// Read `MemAvailable` + `SwapFree` (KB) from a `/proc/meminfo`-shaped file, in +/// MB. A missing file / field reads as 0 (which will trip the low-memory WARN). +fn read_meminfo(path: &Path) -> (u64, u64) { + let contents = fs::read_to_string(path).unwrap_or_default(); + let field_mb = |key: &str| -> u64 { + contents + .lines() + .find_map(|l| l.strip_prefix(key)) + .and_then(|rest| rest.split_whitespace().next()) + .and_then(|kb| kb.parse::().ok()) + .map(|kb| kb / 1024) + .unwrap_or(0) + }; + (field_mb("MemAvailable:"), field_mb("SwapFree:")) +} + +/// The CB image tag from `.env` (`MEV_BOOST_IMAGE`), else the default. +fn cb_image_from_env(env_path: &Path) -> String { + let Ok(contents) = fs::read_to_string(env_path) else { + return DEFAULT_CB_IMAGE.to_string(); + }; + for line in contents.lines() { + let stripped = line.trim(); + if stripped.is_empty() || stripped.starts_with('#') { + continue; + } + if let Some((key, value)) = stripped.split_once('=') + && key.trim() == "MEV_BOOST_IMAGE" + { + let v = value.trim(); + if !v.is_empty() { + return v.to_string(); + } + } + } + DEFAULT_CB_IMAGE.to_string() +} + +/// True iff `path` is a directory with at least one entry. +fn dir_non_empty(path: &Path) -> bool { + fs::read_dir(path) + .map(|mut it| it.next().is_some()) + .unwrap_or(false) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn healthy_probes() -> Probes { + Probes { + kurtosis_version_raw: Some("CLI Version: 1.18.1\n".to_string()), + docker_ok: true, + mem_available_mb: 30_000, + swap_free_mb: 0, + cb_image: DEFAULT_CB_IMAGE.to_string(), + cb_image_present: true, + submodule_initialized: true, + } + } + + #[test] + fn parse_semver_extracts_the_version() { + assert_eq!(parse_semver("CLI Version: 1.18.1"), Some((1, 18, 1))); + assert_eq!(parse_semver("kurtosis version 1.20.0\n"), Some((1, 20, 0))); + assert_eq!(parse_semver("no version here"), None); + // A 4-component build string is not a plain semver. + assert_eq!(parse_semver("1.2.3.4"), None); + } + + #[test] + fn all_healthy_exits_zero_and_all_ok() { + let report = classify(&healthy_probes()); + assert_eq!(report.exit_code, 0); + assert!(report.items.iter().all(|i| i.status == Status::Ok)); + assert_eq!(report.items.len(), 5); + } + + #[test] + fn missing_kurtosis_is_a_hard_fail() { + let mut p = healthy_probes(); + p.kurtosis_version_raw = None; + let report = classify(&p); + assert_eq!(report.exit_code, 1); + let k = &report.items[0]; + assert_eq!(k.name, "kurtosis CLI"); + assert_eq!(k.status, Status::Fail); + assert!(k.hard); + } + + #[test] + fn missing_docker_is_a_hard_fail() { + let mut p = healthy_probes(); + p.docker_ok = false; + assert_eq!(classify(&p).exit_code, 1); + } + + #[test] + fn newer_kurtosis_warns_but_is_not_fatal() { + let mut p = healthy_probes(); + p.kurtosis_version_raw = Some("1.20.0".to_string()); + let report = classify(&p); + assert_eq!( + report.exit_code, 0, + "a version warning must not fail the run" + ); + assert_eq!(report.items[0].status, Status::Warn); + assert!(report.items[0].detail.contains("config-version-9")); + } + + #[test] + fn low_memory_warns_but_does_not_fail() { + let mut p = healthy_probes(); + p.mem_available_mb = 4_000; + p.swap_free_mb = 1_000; + let report = classify(&p); + assert_eq!(report.exit_code, 0); + let mem = report + .items + .iter() + .find(|i| i.name == "host memory") + .unwrap(); + assert_eq!(mem.status, Status::Warn); + assert!(!mem.hard); + } + + #[test] + fn swap_counts_toward_memory_headroom() { + let mut p = healthy_probes(); + p.mem_available_mb = 10_000; + p.swap_free_mb = 10_000; // 20_000 >= NEED_MB + let mem = classify(&p) + .items + .into_iter() + .find(|i| i.name == "host memory") + .unwrap(); + assert_eq!(mem.status, Status::Ok); + } + + #[test] + fn missing_image_and_submodule_are_warnings_only() { + let mut p = healthy_probes(); + p.cb_image_present = false; + p.submodule_initialized = false; + let report = classify(&p); + assert_eq!(report.exit_code, 0, "image/submodule are advisory"); + assert!( + report + .items + .iter() + .filter(|i| i.status == Status::Warn) + .count() + >= 2 + ); + } + + #[test] + fn read_meminfo_parses_kb_to_mb() { + let dir = std::env::temp_dir().join(format!("sim-doctor-mem-{}", std::process::id())); + fs::create_dir_all(&dir).unwrap(); + let f = dir.join("meminfo"); + fs::write( + &f, + "MemTotal: 65808360 kB\nMemAvailable: 20480000 kB\nSwapTotal: 8000000 kB\nSwapFree: 1048576 kB\n", + ) + .unwrap(); + let (avail, swap) = read_meminfo(&f); + assert_eq!(avail, 20_000); // 20480000 / 1024 + assert_eq!(swap, 1024); // 1048576 / 1024 + let _ = fs::remove_dir_all(&dir); + } + + #[test] + fn cb_image_from_env_reads_override_else_default() { + let dir = std::env::temp_dir().join(format!("sim-doctor-env-{}", std::process::id())); + fs::create_dir_all(&dir).unwrap(); + let env = dir.join(".env"); + fs::write(&env, "# c\nMEV_BOOST_IMAGE=my/cb:tag\n").unwrap(); + assert_eq!(cb_image_from_env(&env), "my/cb:tag"); + assert_eq!( + cb_image_from_env(Path::new("/no/such/.env")), + DEFAULT_CB_IMAGE + ); + let _ = fs::remove_dir_all(&dir); + } +} diff --git a/src/bin/sim/generate.rs b/src/bin/sim/generate.rs new file mode 100644 index 0000000..157e1ca --- /dev/null +++ b/src/bin/sim/generate.rs @@ -0,0 +1,225 @@ +//! `sim generate` — write Kurtosis args-files for the CB test scenarios. +//! +//! The pure assembly lives in `genmodel::scenario`; this module is the IO +//! boundary: it applies `.env` image overrides (read-only), resolves the output +//! directory, and writes `configs/generated/.yml`. Applying overrides +//! HERE (not inside `args_file`) keeps assembly pure and hermetically testable +//! against the golden fixtures. + +use std::fs; +use std::path::Path; + +use eyre::{Result, WrapErr, eyre}; + +use crate::genmodel::scenario::{Images, Scenario}; + +/// Generate one scenario (by name) or all six (`None`) into `out_dir`. Reads +/// `keys/` and `.env` relative to the CWD (the repo root — how `just +/// generate-configs` and the Python generator both run). +pub fn run(scenario: Option<&str>, out_dir: &Path) -> Result<()> { + run_in(scenario, out_dir, Path::new("keys"), Path::new(".env")) +} + +/// Testable core with the two IO roots injected. Assembles ALL bodies (reading + +/// validating the mux key files) BEFORE writing anything, so a missing/malformed +/// keys file fails cleanly with nothing written — no partial output. This mirrors +/// the Python's pre-write `load_pubkeys` + `sys.exit(1)` all-or-nothing contract. +fn run_in(scenario: Option<&str>, out_dir: &Path, keys_dir: &Path, env_path: &Path) -> Result<()> { + let images = images_from_env(env_path); + let outputs = assemble(scenario, &images, keys_dir)?; + + fs::create_dir_all(out_dir) + .wrap_err_with(|| format!("creating output dir {}", out_dir.display()))?; + + for (name, body) in &outputs { + let path = out_dir.join(format!("{name}.yml")); + fs::write(&path, body).wrap_err_with(|| format!("writing {}", path.display()))?; + tracing::info!(scenario = name, path = %path.display(), "generated config"); + println!("Generated {}", path.display()); + } + + Ok(()) +} + +/// Verify the on-disk configs already match what the generator would produce, +/// WITHOUT writing (CI / agent drift gate). Errors (nonzero exit) on any drift. +pub fn check(scenario: Option<&str>, out_dir: &Path) -> Result<()> { + check_in(scenario, out_dir, Path::new("keys"), Path::new(".env")) +} + +fn check_in( + scenario: Option<&str>, + out_dir: &Path, + keys_dir: &Path, + env_path: &Path, +) -> Result<()> { + let images = images_from_env(env_path); + let outputs = assemble(scenario, &images, keys_dir)?; + + let mut drift: Vec = Vec::new(); + for (name, body) in &outputs { + let path = out_dir.join(format!("{name}.yml")); + match fs::read_to_string(&path) { + Ok(on_disk) if &on_disk == body => println!("ok {}", path.display()), + Ok(_) => drift.push(format!("{} differs from `sim generate`", path.display())), + Err(_) => drift.push(format!("{} missing (would be created)", path.display())), + } + } + + if drift.is_empty() { + Ok(()) + } else { + Err(eyre!( + "{} config(s) out of date — run `just generate-configs`:\n {}", + drift.len(), + drift.join("\n ") + )) + } +} + +/// Select scenarios and assemble each `(name, body)`. Reads mux key files (the +/// only fallible step) — shared by `run` and `check` so both fail identically +/// before touching the filesystem. +fn assemble( + scenario: Option<&str>, + images: &Images, + keys_dir: &Path, +) -> Result> { + let scenarios: Vec = match scenario { + Some(name) => vec![ + Scenario::from_name(name) + .ok_or_else(|| eyre!("unknown scenario {name:?}; expected one of {:?}", names()))?, + ], + None => Scenario::ALL.to_vec(), + }; + scenarios + .iter() + .map(|s| Ok((s.name().to_string(), s.args_file_in(images, keys_dir)?))) + .collect() +} + +fn names() -> Vec<&'static str> { + Scenario::ALL.iter().map(|s| s.name()).collect() +} + +/// Build the image map from defaults, overridden by `.env` if present. Mirrors +/// the Python `.env` key names. A missing `.env` is not an error (defaults win). +fn images_from_env(env_path: &Path) -> Images { + let mut images = Images::default(); + let Ok(contents) = fs::read_to_string(env_path) else { + return images; + }; + for (key, value) in parse_env(&contents) { + match key.as_str() { + "HELIX_RELAY_IMAGE" => images.helix_relay = value, + "MEV_RELAY_IMAGE" => images.mev_relay = value, + "MEV_BOOST_IMAGE" => images.mev_boost = value, + "BUILDER_EL_IMAGE" => images.builder_el = value, + "BUILDER_CL_IMAGE" => images.builder_cl = value, + _ => {} + } + } + images +} + +/// Parse `KEY=VALUE` pairs, skipping blanks and `#` comments. No quoting/expansion +/// (matches the Python `load_env`). +fn parse_env(contents: &str) -> Vec<(String, String)> { + let mut out = Vec::new(); + for line in contents.lines() { + let stripped = line.trim(); + if stripped.is_empty() || stripped.starts_with('#') { + continue; + } + if let Some((key, value)) = stripped.split_once('=') { + out.push((key.trim().to_string(), value.trim().to_string())); + } + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn env_overrides_apply_over_defaults() { + let dir = std::env::temp_dir().join(format!("sim-env-{}", std::process::id())); + fs::create_dir_all(&dir).unwrap(); + let env = dir.join(".env"); + fs::write( + &env, + "# c\nHELIX_RELAY_IMAGE=custom/helix:dev\n\nMEV_BOOST_IMAGE=x/y:z\n", + ) + .unwrap(); + let images = images_from_env(&env); + assert_eq!(images.helix_relay, "custom/helix:dev"); + assert_eq!(images.mev_boost, "x/y:z"); + // Untouched keys keep defaults. + assert_eq!(images.builder_cl, "sigp/lighthouse:latest"); + } + + #[test] + fn missing_env_yields_defaults() { + let images = images_from_env(Path::new("/no/such/.env")); + assert_eq!(images.helix_relay, Images::default().helix_relay); + } + + /// IO faithfulness: `run` writes all six files, each byte-equal to the pure + /// assembly for the resolved image set. (The hermetic GOLDEN byte-match lives + /// in `scenario.rs::every_scenario_matches_its_golden`, on default images; + /// this only proves the IO layer writes what assembly produced.) + #[test] + fn run_writes_all_six_matching_assembly() { + let dir = std::env::temp_dir().join(format!("sim-gen-{}", std::process::id())); + let _ = fs::remove_dir_all(&dir); + run(None, &dir).expect("generate all"); + let images = images_from_env(Path::new(".env")); + for s in Scenario::ALL { + let produced = fs::read_to_string(dir.join(format!("{}.yml", s.name()))).unwrap(); + let expected = s.args_file_in(&images, Path::new("keys")).unwrap(); + assert_eq!(produced, expected, "{} on-disk body", s.name()); + } + let _ = fs::remove_dir_all(&dir); + } + + /// `--check` passes when on-disk configs match the generator, and fails + /// (Err → nonzero exit) when one has drifted. + #[test] + fn check_passes_when_current_and_fails_on_drift() { + let dir = std::env::temp_dir().join(format!("sim-check-{}", std::process::id())); + let _ = fs::remove_dir_all(&dir); + run(None, &dir).expect("seed"); + // Fresh output → check is clean. + check(None, &dir).expect("check should pass on freshly-generated configs"); + // Mutate one file → check must fail. + let f = dir.join("cb-basic.yml"); + let mut body = fs::read_to_string(&f).unwrap(); + body.push_str("\n# hand-edit\n"); + fs::write(&f, body).unwrap(); + let err = check(None, &dir).unwrap_err(); + assert!(err.to_string().contains("out of date"), "got: {err}"); + assert!( + err.to_string().contains("cb-basic.yml"), + "names the drifted file: {err}" + ); + let _ = fs::remove_dir_all(&dir); + } + + /// Atomicity: a missing keys dir (mux can't load pubkeys) must fail with + /// NOTHING written — the output dir is not even created. + #[test] + fn run_is_atomic_on_missing_keys() { + let dir = std::env::temp_dir().join(format!("sim-atomic-{}", std::process::id())); + let _ = fs::remove_dir_all(&dir); + let err = run_in( + None, + &dir, + Path::new("/no/such/keys"), + Path::new("/no/such/.env"), + ) + .unwrap_err(); + assert!(err.to_string().contains("pubkey file"), "got: {err}"); + assert!(!dir.exists(), "no output dir should be created on failure"); + } +} diff --git a/src/bin/sim/genmodel/cb.rs b/src/bin/sim/genmodel/cb.rs new file mode 100644 index 0000000..9bb4f06 --- /dev/null +++ b/src/bin/sim/genmodel/cb.rs @@ -0,0 +1,386 @@ +//! The commit-boost TOML block — a verbatim port of the Python +//! `build_cb_toml_basic` / `build_cb_toml_mux` +//! (`scripts/generate_kurtosis_configs.py`). +//! +//! The `chain = {...}`, `port = {{ .Port }}`, and +//! `{{ range $index, $relay := .Relays }} … {{- end }}` are runtime holes filled +//! by the ethereum-package at launch — literal text here. Generate-time knobs +//! (timeouts, extra `[pbs]` lines, per-relay lines) are injected by plain string +//! building — NOT serde — so there is no quoting/sentinel hazard. + +/// Generate-time knobs for the basic CB template (`build_cb_toml_basic`). +#[derive(Debug, Clone)] +pub struct CbParams { + pub timeout_get_header_ms: u32, + pub timeout_get_payload_ms: u32, + /// Extra `[pbs]` lines, inserted after `port` and before the timeouts. + pub extra_pbs_lines: Vec, + /// Extra lines appended inside the `{{ range }}` relay loop (per relay). + pub per_relay_lines: Vec, + /// When `Some`, the `{{ range }}` relay loop is REPLACED by a single + /// literal `[[relays]]` block with this exact url — the fault-injection + /// seam for the sigverify differential (a wrong-but-valid pubkey in the + /// url makes CB's signature validation reject every bid from the real + /// relay; `skip_sigverify = true` is then the only way bids flow). + /// Kurtosis service DNS (`helix-relay-N:4040`) makes the literal url + /// resolvable in-enclave without knowing the relay's IP at generate time. + pub literal_relay_url: Option, + /// When `Some`, append the `[signer]` + `[[modules]]` blocks. OPT-IN so the + /// nine existing golden fixtures stay byte-identical: the signer sections + /// are appended AFTER `[logs.file]`, which is valid TOML (interleaving is + /// not, once `[[relays]]` has opened an array-of-tables). + pub signer: Option, +} + +/// The `[signer]` + `[[modules]]` knobs. Deliberately minimal: everything CB +/// does not require is left at its default. +/// +/// The key PATHS are intentionally absent - CB reads them from +/// `CB_SIGNER_LOADER_KEYS_DIR` / `CB_SIGNER_LOADER_SECRETS_DIR`, which override +/// the TOML (`signer/loader.rs`). That matters because the paths are +/// per-participant (`node--keystores/...`) and the config template only +/// carries `.Network/.Port/.Relays/.Timestamp`, so they could not be templated +/// in anyway. The TOML keeps placeholder paths purely to satisfy the schema. +#[derive(Debug, Clone)] +pub struct SignerParams { + /// Listen port. CB defaults to 20000. + pub port: u16, + /// Keystore format. **Must be `teku`** on a Kurtosis devnet: the + /// ethereum-package's `secrets/` dir is `chmod 0600 -R` (mode 600, + /// root-owned, NO execute bit - verified live), so the CB container's uid + /// 10001 cannot traverse it and would load ZERO keys while looking healthy. + /// `teku-secrets` is 755 and `teku-keys` is 777, which is also the pair the + /// package's own web3signer launcher uses. + pub keys_format: String, + /// The single commit module. At least one `[[modules]]` entry is REQUIRED: + /// with none, `load_module_signing_configs` bails loudly, and with an empty + /// list the service exits 0 silently. + pub module_id: String, + /// 32-byte hex, non-zero, unique per module; mixed into the signing root. + pub module_signing_id: String, +} + +impl SignerParams { + /// The devnet defaults: port 20000, teku keystores, one commit module. + pub fn devnet() -> Self { + Self { + port: 20000, + keys_format: "teku".to_string(), + module_id: "TEST_MODULE".to_string(), + // Arbitrary but fixed: a stable signing_id keeps BLS signatures + // deterministic across runs, which is what the signature + // differential assertion relies on. + module_signing_id: "0x6a33a23ef26a4836979edff86c493a69b26ccf0b4a16491a815a13787657431b" + .to_string(), + } + } +} + +impl CbParams { + /// The default basic knobs (950 / 4000, no extra lines) — cb-basic and + /// cb-multiple-relays. + pub fn basic() -> Self { + Self { + timeout_get_header_ms: 950, + timeout_get_payload_ms: 4000, + extra_pbs_lines: Vec::new(), + per_relay_lines: Vec::new(), + literal_relay_url: None, + signer: None, + } + } +} + +/// Reproduce `build_cb_toml_basic`: base template + Rust-side injection of the +/// timeouts, `extra_pbs_lines` (after `port`, before the timeouts) and +/// `per_relay_lines` (inside the range loop). +pub fn cb_toml(p: &CbParams) -> String { + let mut lines: Vec = vec![ + r#"chain = { genesis_time_secs = {{ .Timestamp }}, path = "{{ .Network }}" }"#.to_string(), + String::new(), + "[pbs]".to_string(), + r#"host = "0.0.0.0""#.to_string(), + "port = {{ .Port }}".to_string(), + format!("timeout_get_header_ms = {}", p.timeout_get_header_ms), + format!("timeout_get_payload_ms = {}", p.timeout_get_payload_ms), + "late_in_slot_time_ms = 2000".to_string(), + ]; + + // Insert after port (idx 4), before the timeouts (idx 5) — matches Python. + for (offset, line) in p.extra_pbs_lines.iter().enumerate() { + lines.insert(5 + offset, line.clone()); + } + + lines.push(String::new()); + lines.push(String::new()); + lines.push("[metrics]".to_string()); + lines.push("enabled = true".to_string()); + lines.push(r#"host = "0.0.0.0""#.to_string()); + lines.push("start_port = 9090".to_string()); + lines.push(String::new()); + match &p.literal_relay_url { + // Fault-injection: one literal [[relays]] entry, no template loop. + Some(url) => { + lines.push("[[relays]]".to_string()); + lines.push(r#"id = "mev_relay_0""#.to_string()); + lines.push(format!(r#"url = "{url}""#)); + for line in &p.per_relay_lines { + lines.push(line.clone()); + } + } + None => { + lines.push("{{ range $index, $relay := .Relays }}".to_string()); + lines.push("[[relays]]".to_string()); + lines.push(r#"id = "mev_relay_{{$index}}""#.to_string()); + lines.push(r#"url = "{{ $relay }}""#.to_string()); + + for line in &p.per_relay_lines { + lines.push(line.clone()); + } + + lines.push("{{- end }}".to_string()); + } + } + lines.push(String::new()); + lines.push("[logs.stdout]".to_string()); + lines.push(r#"level = "debug""#.to_string()); + lines.push(String::new()); + lines.push("[logs.file]".to_string()); + lines.push("enabled = false".to_string()); + + if let Some(sg) = &p.signer { + lines.push(String::new()); + lines.push("[signer]".to_string()); + lines.push(format!("port = {}", sg.port)); + lines.push(String::new()); + lines.push("[signer.local.loader]".to_string()); + // Placeholders only: CB_SIGNER_LOADER_{KEYS,SECRETS}_DIR override these + // at runtime with the real per-participant artifact paths. + lines.push(format!(r#"format = "{}""#, sg.keys_format)); + lines.push(r#"keys_path = "/keystores/teku-keys""#.to_string()); + lines.push(r#"secrets_path = "/keystores/teku-secrets""#.to_string()); + lines.push(String::new()); + lines.push("[[modules]]".to_string()); + lines.push(format!(r#"id = "{}""#, sg.module_id)); + lines.push(r#"type = "commit""#.to_string()); + lines.push(format!(r#"signing_id = "{}""#, sg.module_signing_id)); + // Required field, never read by the running signer (only by `cb init`). + lines.push(r#"docker_image = "unused""#.to_string()); + } + + lines.join("\n") +} + +/// Format a pubkey list as a multiline literal with 4-space entry indentation, +/// matching Python `format_pubkey_list`. Inside the 4-space-indented block +/// scalar the entries land at 8 spaces total. +fn format_pubkey_list(pubkeys: &[String]) -> String { + let mut lines = vec!["[".to_string()]; + let last = pubkeys.len().saturating_sub(1); + for (i, pk) in pubkeys.iter().enumerate() { + let comma = if i == last { "" } else { "," }; + lines.push(format!(" \"{pk}\"{comma}")); + } + lines.push("]".to_string()); + lines.join("\n") +} + +/// Reproduce `build_cb_toml_mux`: the range loop precedes `[metrics]`, and two +/// `[[mux]]` blocks (per-node `validator_pubkeys` + a `[[mux.relays]]`) sit +/// between `[metrics]` and `[logs]`. +pub fn cb_toml_mux(pubkeys_node0: &[String], pubkeys_node1: &[String]) -> String { + let node0_list = format_pubkey_list(pubkeys_node0); + let node1_list = format_pubkey_list(pubkeys_node1); + + let lines: Vec = vec![ + r#"chain = { genesis_time_secs = {{ .Timestamp }}, path = "{{ .Network }}" }"#.to_string(), + String::new(), + "[pbs]".to_string(), + r#"host = "0.0.0.0""#.to_string(), + "port = {{ .Port }}".to_string(), + "timeout_get_header_ms = 950".to_string(), + "timeout_get_payload_ms = 4000".to_string(), + "late_in_slot_time_ms = 2000".to_string(), + String::new(), + "{{ range $index, $relay := .Relays }}".to_string(), + "[[relays]]".to_string(), + r#"id = "mev_relay_{{$index}}""#.to_string(), + r#"url = "{{ $relay }}""#.to_string(), + "{{- end }}".to_string(), + String::new(), + "[metrics]".to_string(), + "enabled = true".to_string(), + r#"host = "0.0.0.0""#.to_string(), + "start_port = 9090".to_string(), + String::new(), + "[[mux]]".to_string(), + r#"id = "node_0_to_helix""#.to_string(), + format!("validator_pubkeys = {node0_list}"), + "timeout_get_header_ms = 900".to_string(), + "[[mux.relays]]".to_string(), + r#"id = "mux_helix""#.to_string(), + r#"url = "{{ index .Relays 0 }}""#.to_string(), + String::new(), + "[[mux]]".to_string(), + r#"id = "node_1_to_helix""#.to_string(), + format!("validator_pubkeys = {node1_list}"), + "timeout_get_header_ms = 900".to_string(), + "[[mux.relays]]".to_string(), + r#"id = "mux_helix_1""#.to_string(), + r#"url = "{{ index .Relays 1 }}""#.to_string(), + String::new(), + "[logs.stdout]".to_string(), + r#"level = "debug""#.to_string(), + String::new(), + "[logs.file]".to_string(), + "enabled = false".to_string(), + ]; + + lines.join("\n") +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::genmodel::{extract_block_scalar, golden}; + + #[test] + fn basic_cb_matches_golden_block() { + let block = extract_block_scalar(golden("cb-basic"), "commit_boost_config"); + assert_eq!(cb_toml(&CbParams::basic()), block); + } + + #[test] + fn timing_games_cb_matches_golden_block() { + // 400/2000 timeouts + 3 per-relay lines INSIDE the range loop. + let params = CbParams { + timeout_get_header_ms: 400, + timeout_get_payload_ms: 2000, + extra_pbs_lines: Vec::new(), + per_relay_lines: vec![ + "enable_timing_games = true".to_string(), + "target_first_request_ms = 100".to_string(), + "frequency_get_header_ms = 200".to_string(), + ], + literal_relay_url: None, + signer: None, + }; + let block = extract_block_scalar(golden("cb-timing-games"), "commit_boost_config"); + assert_eq!(cb_toml(¶ms), block); + } + + #[test] + fn skip_sigverify_cb_matches_golden_block() { + let params = CbParams { + extra_pbs_lines: vec!["skip_sigverify = true".to_string()], + ..CbParams::basic() + }; + let block = extract_block_scalar(golden("cb-skip-sigverify"), "commit_boost_config"); + assert_eq!(cb_toml(¶ms), block); + } + + #[test] + fn extra_validation_cb_matches_golden_block() { + let params = CbParams { + extra_pbs_lines: vec![ + "extra_validation_enabled = true".to_string(), + r#"rpc_url = "http://el-1-geth-lighthouse:8545""#.to_string(), + ], + ..CbParams::basic() + }; + let block = extract_block_scalar(golden("cb-extra-validation"), "commit_boost_config"); + assert_eq!(cb_toml(¶ms), block); + } + + #[test] + fn signer_blocks_are_opt_in_and_change_nothing_when_absent() { + // The whole point of making it Option: the nine pre-existing goldens + // must stay byte-identical, so a signer scenario cannot churn them. + let without = cb_toml(&CbParams::basic()); + assert!(!without.contains("[signer]")); + assert!(!without.contains("[[modules]]")); + } + + #[test] + fn signer_blocks_render_after_the_logs_sections() { + // TOML ordering is load-bearing: once [[relays]] has opened an + // array-of-tables, interleaving new top-level tables is invalid. + // Appending after [logs.file] is the only safe placement. + let out = cb_toml(&CbParams { + signer: Some(SignerParams::devnet()), + ..CbParams::basic() + }); + let relays_at = out.find("[[relays]]").expect("relays present"); + let logs_at = out.find("[logs.file]").expect("logs present"); + let signer_at = out.find("[signer]").expect("signer present"); + let modules_at = out.find("[[modules]]").expect("modules present"); + assert!(relays_at < logs_at, "relays before logs"); + assert!(logs_at < signer_at, "signer AFTER the logs sections"); + assert!(signer_at < modules_at, "[signer] before [[modules]]"); + } + + #[test] + fn signer_uses_teku_keystores_not_the_unreadable_lighthouse_pair() { + // Verified live: the package's secrets/ dir is mode 600 root-owned (no + // execute bit), so CB's uid 10001 cannot traverse it and would start + // healthy holding ZERO keys. teku-secrets is 755, teku-keys 777. + let out = cb_toml(&CbParams { + signer: Some(SignerParams::devnet()), + ..CbParams::basic() + }); + assert!( + out.contains(r#"format = "teku""#), + "must be the teku format" + ); + assert!(out.contains("teku-keys") && out.contains("teku-secrets")); + assert!( + !out.contains(r#"keys_path = "/keystores/keys""#), + "must NOT use the unreadable lighthouse keys/secrets pair" + ); + } + + #[test] + fn signer_emits_a_module_because_none_is_fatal() { + // With NO [[modules]] the signer bails loudly; with an EMPTY list it + // exits 0 silently. Either way a module entry is mandatory, and every + // field of it is required (no serde defaults). + let out = cb_toml(&CbParams { + signer: Some(SignerParams::devnet()), + ..CbParams::basic() + }); + assert!(out.contains(r#"id = "TEST_MODULE""#)); + assert!(out.contains(r#"type = "commit""#)); + assert!( + out.contains("signing_id = \"0x"), + "signing_id must be present and hex" + ); + assert!( + out.contains(r#"docker_image = "unused""#), + "required even though unread" + ); + // signing_id must be non-zero: CB rejects the zero value. + assert!(!out.contains(&format!("signing_id = \"0x{}\"", "0".repeat(64)))); + } + + #[test] + fn mux_cb_has_the_right_structure() { + // STRUCTURE-only unit test with 2 synthetic keys/node (NOT 256). + let node0 = vec!["0xdead".to_string(), "0xbeef".to_string()]; + let node1 = vec!["0xcafe".to_string(), "0xf00d".to_string()]; + let out = cb_toml_mux(&node0, &node1); + + assert_eq!(out.matches("[[mux]]").count(), 2, "two [[mux]] blocks"); + assert_eq!(out.matches("[[mux.relays]]").count(), 2); + assert!(out.contains(r#"id = "node_0_to_helix""#)); + assert!(out.contains(r#"id = "node_1_to_helix""#)); + // per-node validator_pubkeys lists, entries indented 4 spaces. + assert!(out.contains("validator_pubkeys = [\n \"0xdead\",\n \"0xbeef\"\n]")); + assert!(out.contains("validator_pubkeys = [\n \"0xcafe\",\n \"0xf00d\"\n]")); + assert!(out.contains(r#"url = "{{ index .Relays 0 }}""#)); + assert!(out.contains(r#"url = "{{ index .Relays 1 }}""#)); + // range loop precedes [metrics] in mux (unlike basic). + let range_at = out.find("{{ range").unwrap(); + let metrics_at = out.find("[metrics]").unwrap(); + assert!(range_at < metrics_at, "range loop before [metrics] in mux"); + } +} diff --git a/src/bin/sim/genmodel/helix.rs b/src/bin/sim/genmodel/helix.rs new file mode 100644 index 0000000..aa93f22 --- /dev/null +++ b/src/bin/sim/genmodel/helix.rs @@ -0,0 +1,123 @@ +//! The helix relay YAML block — a verbatim port of the Python +//! `build_helix_relay_config()` output (`scripts/generate_kurtosis_configs.py`). +//! +//! This block is byte-IDENTICAL across all 6 scenarios (verified by diff), so it +//! is a single `const`. The `{{ .POSTGRES_* }}`, `{{ .BEACON_URI }}`, and +//! `{{ .BLOCKSIM_URI }}` are runtime holes filled by the ethereum-package at +//! launch — they stay as literal text here. +//! +//! The ~40 lines of comments are the expensive, hard-won knowledge (the +//! `network_config` removal, the binary-verified 10-field `CoresConfig`) — PRESERVE +//! them verbatim. Do not "clean them up". + +/// The helix relay config, de-indented (no leading 4-space block-scalar indent). +/// `scenario::args_file` re-indents it 4 spaces when embedding it as the +/// `helix_relay_config: |` block scalar. No trailing newline (matches Python). +pub const HELIX_RELAY_CONFIG: &str = r#"instance_id: "helix-kurtosis-test" + +# NOTE: no network/genesis section. Current helix-relay:main removed the old +# `network_config: !Custom {dir_path, genesis_validator_root, genesis_time}` +# field entirely; the relay now fetches the chain spec + genesis from the +# beacon node at startup (GET eth/v1/config/spec + eth/v1/beacon/genesis, see +# beacon_client.get_chain_info -> main.rs load_chain_info). A `!Custom` YAML +# tag on that now-unknown key makes serde_yaml panic with +# "untagged and internally tagged enums do not support enum input". + +postgres: + hostname: "{{ .POSTGRES_HOST_NAME }}" + port: {{ .POSTGRES_PORT }} + db_name: "{{ .POSTGRES_DB }}" + user: "{{ .POSTGRES_USER }}" + password: "{{ .POSTGRES_PASS }}" + region: 0 + region_name: "LOCAL" + +beacon_clients: + - url: "{{ .BEACON_URI }}" + +gossip_payload_on_header: false + +simulators: + - url: "{{ .BLOCKSIM_URI }}" + namespace: flashbots + is_merging_simulator: false + max_concurrent_tasks: 32 + +router_config: + enabled_routes: + - route: GetValidators + - route: SubmitBlock + - route: GetTopBid + - route: GetHeader + rate_limit: + replenish_ms: 50 + burst_size: 20 + - route: GetPayload + # The builder-spec v2 proposer route (submitBlindedBlockV2). REQUIRED for + # any CL that submits via v2 -- prysm does. Without it helix 404s + # /eth/v2/builder/blinded_blocks, CB (correctly) refuses to downgrade to v1 + # (v2 semantics: the relay publishes the block, so a v1 payload would be + # silently dropped), returns 502, and EVERY builder block the proposer + # chose is lost. That read as "prysm can't do MEV" until the route list was + # checked (Law 7 first dividend). + - route: GetPayloadV2 + - route: HeaderStream + - route: RegisterValidators + - route: Status + - route: ProposerPayloadDelivered + - route: BuilderBidsReceived + - route: ValidatorRegistration + shutdown_delay_ms: 12000 + +timing_game_config: + max_header_delay_ms: 400 + latest_header_delay_ms_in_slot: 1500 + default_client_latency_ms: 50 + +target_get_payload_propagation_duration_ms: 500 + +is_submission_instance: true +is_registration_instance: true + +admin_token: "test_admin_token" + +logging: + type: Console + +# CoresConfig (helix_common::config CoresConfig, 10 fields in current +# :main). The old block used `sub_workers: [0]` and omitted the per-tile +# core assignments; current :main removed sub_workers and added +# decoder/simulator/top_bid/data_gatherer/block_merging/housekeeper. The +# outer RelayConfigExt flattens RelayConfig, so a wrong/missing cores field +# surfaces as a top-level "missing field `decoder`" / "invalid type ... +# expected usize|sequence" serde error. Verified against the binary: +# `decoder` is Vec ([0]); the other five new tile fields are usize. +cores: + auctioneer: 0 + tokio: [0] + reg_workers: [0] + tcp_bid_submissions_tile: 2 + decoder: [0] + simulator: 0 + top_bid: 0 + data_gatherer: 0 + block_merging: 0 + housekeeper: 0 + +is_local_dev: false"#; + +#[cfg(test)] +mod tests { + use super::*; + use crate::genmodel::extract_block_scalar; + + /// The const must equal the `helix_relay_config: |` block of the cb-basic + /// golden, de-indented 4 spaces. (The block is identical in every golden, so + /// one is enough.) + #[test] + fn helix_const_matches_golden_block() { + let golden = crate::genmodel::golden("cb-basic"); + let block = extract_block_scalar(golden, "helix_relay_config"); + assert_eq!(HELIX_RELAY_CONFIG, block); + } +} diff --git a/src/bin/sim/genmodel/mod.rs b/src/bin/sim/genmodel/mod.rs new file mode 100644 index 0000000..c827b23 --- /dev/null +++ b/src/bin/sim/genmodel/mod.rs @@ -0,0 +1,193 @@ +//! `genmodel`: Rust-native Kurtosis config generation — the P2 replacement for +//! `scripts/generate_kurtosis_configs.py`. +//! +//! The config *bodies* (the helix YAML block, the commit-boost TOML block) are +//! ported VERBATIM from the Python templates into `const` strings: they are +//! already DRY there (helix is byte-identical across all 6 scenarios; CB is one +//! parameterized template), and the `{{ }}` runtime holes are filled by the +//! ethereum-package at launch, so they stay as literal text. Typing lives only +//! at the assembly layer (`Scenario` + `Images`) where it actually pays. See +//! the P2 config-gen consolidation rationale. +//! +//! Task 0 lands only the golden-fixture regression harness; the generator bodies +//! (`helix`, `cb`, `scenario` submodules) land in Task 1. + +pub mod cb; +pub mod helix; +pub mod scenario; + +/// Extract a YAML `|` block scalar (named `key`, at 2-space indent) from `yaml`, +/// de-indented 4 spaces, with trailing blank lines removed. Test-only oracle +/// helper: it recovers the raw body the generator embeds so a body port can be +/// byte-compared against the golden independent of the surrounding assembly. +#[cfg(test)] +pub fn extract_block_scalar(yaml: &str, key: &str) -> String { + let header = format!(" {key}: |"); + let mut lines = yaml.lines(); + for line in lines.by_ref() { + if line == header { + break; + } + } + let mut body: Vec = Vec::new(); + for line in lines { + if line.is_empty() { + body.push(String::new()); + } else if let Some(rest) = line.strip_prefix(" ") { + body.push(rest.to_string()); + } else { + // A less-indented non-empty line ends the block scalar. + break; + } + } + while body.last().is_some_and(|l| l.is_empty()) { + body.pop(); + } + body.join("\n") +} + +/// The golden configs (one per scenario). The three single-relay ones (cb-basic, +/// cb-skip-sigverify, cb-extra-validation) are the exact Python output that +/// produced the green e2e run. The three multi-relay ones (cb-multiple-relays, +/// cb-timing-games, cb-mux) were REGENERATED for the intended two-Helix-instance +/// topology (the flashbots RELAY was dropped; the flashbots BUILDER stays), so +/// they are the intended output, not the old Python baseline. All are snapshotted +/// with the baked-default images. `sim generate` must reproduce each +/// byte-for-byte. Depth-independent path via `CARGO_MANIFEST_DIR`. +#[cfg(test)] +pub fn golden(scenario: &str) -> &'static str { + match scenario { + "cb-basic" => include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/golden-configs/cb-basic.yml" + )), + "cb-multiple-relays" => include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/golden-configs/cb-multiple-relays.yml" + )), + "cb-basic-nethermind-prysm" => include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/golden-configs/cb-basic-nethermind-prysm.yml" + )), + "cb-signer" => include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/golden-configs/cb-signer.yml" + )), + "cb-min-bid" => include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/golden-configs/cb-min-bid.yml" + )), + "cb-skip-sigverify" => include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/golden-configs/cb-skip-sigverify.yml" + )), + "cb-sigverify-diff" => include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/golden-configs/cb-sigverify-diff.yml" + )), + "cb-sigverify-diff-control" => include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/golden-configs/cb-sigverify-diff-control.yml" + )), + "cb-extra-validation" => include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/golden-configs/cb-extra-validation.yml" + )), + "cb-timing-games" => include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/golden-configs/cb-timing-games.yml" + )), + "cb-ws-stream" => include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/golden-configs/cb-ws-stream.yml" + )), + "cb-ws-stream-nokey" => include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/golden-configs/cb-ws-stream-nokey.yml" + )), + "cb-mux" => include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/golden-configs/cb-mux.yml" + )), + other => panic!("no golden fixture for scenario {other:?}"), + } +} + +/// Byte-diff `produced` against the golden for `scenario`. On mismatch, panics +/// naming the first differing line with a little context — the acceptance oracle +/// for the verbatim port (a byte-exact port makes byte-identity the right test). +#[cfg(test)] +pub fn assert_matches_golden(scenario: &str, produced: &str) { + let expected = golden(scenario); + if produced == expected { + return; + } + let exp_lines: Vec<&str> = expected.lines().collect(); + let got_lines: Vec<&str> = produced.lines().collect(); + let max = exp_lines.len().max(got_lines.len()); + for i in 0..max { + let e = exp_lines.get(i).copied(); + let g = got_lines.get(i).copied(); + if e != g { + let ctx_start = i.saturating_sub(2); + let mut ctx = String::new(); + for (j, line) in exp_lines.iter().enumerate().take(i).skip(ctx_start) { + ctx.push_str(&format!(" {:>4} {}\n", j + 1, line)); + } + panic!( + "{scenario}: output differs from golden at line {}:\n{ctx} {:>4}- {}\n {:>4}+ {}\n\ + (expected {} lines, got {} lines)", + i + 1, + i + 1, + e.unwrap_or(""), + i + 1, + g.unwrap_or(""), + exp_lines.len(), + got_lines.len(), + ); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const ALL: [&str; 6] = [ + "cb-basic", + "cb-multiple-relays", + "cb-skip-sigverify", + "cb-extra-validation", + "cb-timing-games", + "cb-mux", + ]; + + #[test] + fn every_golden_matches_itself() { + for s in ALL { + assert_matches_golden(s, golden(s)); + } + } + + #[test] + #[should_panic(expected = "differs from golden at line")] + fn a_flipped_line_is_caught() { + // Flip one line of the basic golden; the harness must reject it. + let mutated = golden("cb-basic").replacen("mev_type: custom", "mev_type: BOGUS", 1); + assert_matches_golden("cb-basic", &mutated); + } + + #[test] + fn golden_images_are_the_baked_defaults() { + // Guards hermeticity: the fixtures must encode the baked-default images + // (the proven-good values), not a box-specific .env. Task 1's + // Images::defaults() must reproduce exactly these. + let basic = golden("cb-basic"); + assert!(basic.contains("mev_boost_image: commit-boost/commit-boost:kurtosis")); + assert!( + !basic.contains("commit-boost/pbs:kurtosis"), + "the pbs bug must be gone" + ); + assert!(basic.contains("helix_relay_image: ghcr.io/gattaca-com/helix-relay:main")); + } +} diff --git a/src/bin/sim/genmodel/scenario.rs b/src/bin/sim/genmodel/scenario.rs new file mode 100644 index 0000000..564e8bf --- /dev/null +++ b/src/bin/sim/genmodel/scenario.rs @@ -0,0 +1,654 @@ +//! Scenario assembly — the `Scenario` enum + `Images` map that join the vetted +//! static fragments (participants, additional_services, network_params) with the +//! helix const and the CB block into a full Kurtosis args-file. +//! +//! Ports the Python `generate_*()` assemblers + `build_mev_params` +//! (`scripts/generate_kurtosis_configs.py`). The static fragments are kept as +//! verbatim `const` strings (they never drift; out of the port's typing scope). + +use std::path::Path; + +use eyre::{Result, WrapErr}; + +use super::cb::{CbParams, SignerParams, cb_toml, cb_toml_mux}; +use super::helix::HELIX_RELAY_CONFIG; + +// --- Vetted static fragments (verbatim from Python) ------------------------- + +/// An execution/consensus client pair. Law 7 ("coverage is a matrix, not a +/// point"): everything used to hardcode geth+lighthouse, so a CB regression +/// specific to another pair was invisible. The pair is threaded through BOTH +/// the participants block and every service name derived from it — notably +/// extra-validation's `rpc_url`, which the ethereum-package names +/// `el-{index}-{el}-{cl}` (`src/el/el_launcher.star:177`). That coupling is +/// exactly why a hardcoded rpc_url silently no-ops on an alternate pair. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ElCl { + pub el: &'static str, + pub cl: &'static str, +} + +impl ElCl { + /// The baked default pair every scenario used before Law 7. + pub const DEFAULT: ElCl = ElCl { + el: "geth", + cl: "lighthouse", + }; + /// The alternate pair (the P3 slice: prove the parametrization is real). + pub const ALT: ElCl = ElCl { + el: "nethermind", + cl: "prysm", + }; + + /// The `participants:` fragment for this pair. + fn participants(&self) -> String { + format!( + "participants:\n - el_type: {}\n cl_type: {}", + self.el, self.cl + ) + } + + /// The first participant's EL RPC endpoint, as the ethereum-package names + /// it (`el-{index}-{el}-{cl}`, 1-indexed). + fn el_rpc_url(&self) -> String { + format!("http://el-1-{}-{}:8545", self.el, self.cl) + } +} + +const COMMON_ADDITIONAL_SERVICES: &str = + "additional_services:\n - dora\n - spamoor\n - prometheus"; + +const COMMON_NETWORK_PARAMS: &str = r#"network_params: + network: kurtosis + network_id: "3151908" + deposit_contract_address: "0x00000000219ab540356cBB839Cbe05303d7705Fa" + seconds_per_slot: 12 + slot_duration_ms: 12000 + num_validator_keys_per_node: 128 + preregistered_validator_keys_mnemonic: + "giant issue aisle success illegal bike spike + question tent bar rely arctic volcano long crawl hungry vocal artwork sniff fantasy + very lucky have athlete" + prefunded_accounts: '{"0xb9e79d19f651a941757b35830232E7EFC77E1c79": {"balance": "100000ETH"}}' +"#; + +const MUX_NETWORK_PARAMS: &str = r#"network_params: + network: kurtosis + network_id: "3151908" + deposit_contract_address: "0x00000000219ab540356cBB839Cbe05303d7705Fa" + seconds_per_slot: 12 + slot_duration_ms: 12000 + num_validator_keys_per_node: 256 + preregistered_validator_keys_mnemonic: + "giant issue aisle success illegal bike spike + question tent bar rely arctic volcano long crawl hungry vocal artwork sniff fantasy + very lucky have athlete" + prefunded_accounts: '{"0xb9e79d19f651a941757b35830232E7EFC77E1c79": {"balance": "100000ETH"}}' +"#; + +// --- Images (the ONE image map) --------------------------------------------- + +/// The unified Docker-image map. Defaults are the baked, proven-good values — +/// note `mev_boost` = `commit-boost/commit-boost:kurtosis` (the Python's +/// `commit-boost/pbs:kurtosis` default was the bug this consolidation fixes). +/// `.env` overrides are applied at the CLI boundary (`generate::run`), never here. +#[derive(Debug, Clone)] +pub struct Images { + pub helix_relay: String, + pub mev_relay: String, + pub mev_boost: String, + pub builder_el: String, + pub builder_cl: String, +} + +impl Default for Images { + fn default() -> Self { + Self { + helix_relay: "ghcr.io/gattaca-com/helix-relay:main".to_string(), + mev_relay: "ethpandaops/mev-boost-relay:main".to_string(), + mev_boost: "commit-boost/commit-boost:kurtosis".to_string(), + builder_el: "ethpandaops/reth-rbuilder:develop".to_string(), + builder_cl: "sigp/lighthouse:latest".to_string(), + } + } +} + +// --- Scenario --------------------------------------------------------------- + +/// The six Commit-Boost test scenarios. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Scenario { + Basic, + MultipleRelays, + BasicAltClients, + MinBid, + Signer, + SkipSigverify, + SigverifyDiff, + SigverifyDiffControl, + ExtraValidation, + TimingGames, + /// getHeader over the websocket bid stream (CB `get_header = "stream"` + + /// helix HeaderStream route). The api key rides relay `headers`, so + /// registration TOFU-binds it and the stream authenticates. + WsStream, + /// NEGATIVE CONTROL: stream configured but NO api key. helix refuses every + /// handshake, every slot falls back to HTTP, MEV stays green - and + /// feature.ws_header_stream goes INCONCLUSIVE, which is the point: this + /// scenario exists to prove the criteria discriminate. Expected to FAIL + /// under --require-feature-proof; not part of the green sweep. + WsStreamNoKey, + Mux, +} + +impl Scenario { + /// All six scenarios, in the Python emission order (the `scenarios` dict at + /// `generate_kurtosis_configs.py:562`: timing-games precedes extra-validation). + pub const ALL: [Scenario; 13] = [ + Scenario::Basic, + Scenario::BasicAltClients, + Scenario::MultipleRelays, + Scenario::MinBid, + Scenario::Signer, + Scenario::SkipSigverify, + Scenario::SigverifyDiff, + Scenario::SigverifyDiffControl, + Scenario::TimingGames, + Scenario::ExtraValidation, + Scenario::WsStream, + Scenario::WsStreamNoKey, + Scenario::Mux, + ]; + + /// The scenario's canonical name (also the golden fixture / output basename). + pub fn name(&self) -> &'static str { + match self { + Scenario::Basic => "cb-basic", + Scenario::MultipleRelays => "cb-multiple-relays", + Scenario::BasicAltClients => "cb-basic-nethermind-prysm", + Scenario::MinBid => "cb-min-bid", + Scenario::Signer => "cb-signer", + Scenario::SkipSigverify => "cb-skip-sigverify", + Scenario::SigverifyDiff => "cb-sigverify-diff", + Scenario::SigverifyDiffControl => "cb-sigverify-diff-control", + Scenario::ExtraValidation => "cb-extra-validation", + Scenario::TimingGames => "cb-timing-games", + Scenario::WsStream => "cb-ws-stream", + Scenario::WsStreamNoKey => "cb-ws-stream-nokey", + Scenario::Mux => "cb-mux", + } + } + + /// Parse a scenario by name; `None` if unknown. + pub fn from_name(name: &str) -> Option { + Scenario::ALL.into_iter().find(|s| s.name() == name) + } + + /// The EL/CL client pair this scenario runs on (Law 7). Everything else + /// stays on the baked default pair; the alt-clients scenario is the P3 + /// slice proving the parametrization is real end to end. + pub fn el_cl(&self) -> ElCl { + match self { + Scenario::BasicAltClients => ElCl::ALT, + _ => ElCl::DEFAULT, + } + } + + /// The leading comment block (verbatim from Python). + fn comment(&self) -> &'static str { + match self { + Scenario::Basic => { + "# cb-basic: Single relay (helix) with default Commit-Boost config.\n\ + #\n\ + # Tests the core MEV pipeline through Commit-Boost with a single Helix\n\ + # relay as the only relay endpoint." + } + Scenario::MultipleRelays => { + "# cb-multiple-relays: Two Helix relay instances behind a single\n\ + # Commit-Boost sidecar.\n\ + #\n\ + # Tests that CB correctly routes get_header requests to both relays,\n\ + # aggregating responses and selecting the best bid. The per-relay\n\ + # subsidy list [1, 2] makes the builder submit DIVERGENT bid values\n\ + # (rbuilder [[subsidy_overrides]]), so the best-bid selection is a\n\ + # real discrimination, not a tie between identical bids." + } + Scenario::WsStream => { + "# cb-ws-stream: getHeader over the websocket bid stream.\n\ + #\n\ + # CB `get_header = \"stream\"` + helix HeaderStream route. The relay's\n\ + # X-Api-Key header rides validator registration, helix TOFU-binds it,\n\ + # and the stream authenticates. THE TRAP this scenario guards: the HTTP\n\ + # fallback keeps every MEV check green when the stream is broken, so\n\ + # feature.ws_header_stream (proof markers) is the real assertion - run\n\ + # under --require-feature-proof." + } + Scenario::WsStreamNoKey => { + "# cb-ws-stream-nokey: NEGATIVE CONTROL for the ws criteria.\n\ + #\n\ + # Stream configured with NO api key: helix refuses every handshake\n\ + # (\"no api key registered for this proposer\"), every slot falls back\n\ + # to HTTP, MEV stays green - and feature.ws_header_stream goes\n\ + # INCONCLUSIVE. EXPECTED to fail under --require-feature-proof; that\n\ + # failure is this scenario's proof that the criteria discriminate.\n\ + # Not part of the green sweep." + } + Scenario::BasicAltClients => { + "# cb-basic-nethermind-prysm: cb-basic on an ALTERNATE EL/CL pair.\n\ + #\n\ + # Law 7 (coverage is a matrix, not a point): every other scenario runs\n\ + # geth+lighthouse, so a CB regression specific to another client pair is\n\ + # invisible. Same MEV pipeline assertions as cb-basic, different clients." + } + Scenario::MinBid => { + "# cb-min-bid: the min_bid_eth floor actually drops bids.\n\ + #\n\ + # min_bid_eth = 0.5 with the builder subsidy OFF: real devnet bids are\n\ + # ~0.04 ETH of spamoor MEV, so EVERY bid must be rejected with\n\ + # \"bid below minimum\" and zero auctions won. The subsidy must be 0 or\n\ + # bids land near 1.04 ETH and no LEGAL floor could reject them - CB\n\ + # validates min_bid_wei < 1 ETH.\n\ + #\n\ + # Doubles as a canary for CB's silent-flatten trap: [pbs] has no\n\ + # deny_unknown_fields, so a renamed/misspelled key is IGNORED rather\n\ + # than rejected. If bids still win here, the key was silently dropped." + } + Scenario::Signer => { + "# cb-signer: the Commit-Boost SIGNER module, which has never been\n\ + # testable on Kurtosis (the ethereum-package had no config support).\n\ + #\n\ + # Adds [signer] + [[modules]] to the CB config. The signer container\n\ + # reuses the devnet's existing validator keystores in TEKU format:\n\ + # secrets/ is chmod 0600 root-owned (no execute bit), so CB's uid 10001\n\ + # cannot traverse it and would load ZERO keys while looking healthy;\n\ + # teku-secrets is 755 and teku-keys 777 (verified on a live enclave).\n\ + #\n\ + # PBS reads the same file: [signer]/[[modules]] are Option fields it\n\ + # parses and drops, and pbs.with_signer is dead code in the shipped\n\ + # binary, so nothing about the PBS path changes." + } + Scenario::SkipSigverify => { + "# cb-skip-sigverify: Signature verification disabled for header responses.\n\ + #\n\ + # Tests the CB fast path where BLS verification is skipped. This trades\n\ + # correctness for speed — useful to verify that the path exists and is\n\ + # reachable under load." + } + Scenario::SigverifyDiff => { + "# cb-sigverify-diff: the skip_sigverify DIFFERENTIAL (treatment arm).\n\ + #\n\ + # CB's [[relays]] entry is a LITERAL url whose pubkey is a valid BLS\n\ + # key that is NOT the helix relay's signing key, so CB's signature\n\ + # validation would reject every bid. With skip_sigverify = true the\n\ + # validation is skipped and bids flow anyway - an auction winner in\n\ + # this scenario is positive proof the skip codepath fired. Compare\n\ + # with cb-sigverify-diff-control (same poison, skip OFF, zero bids)." + } + Scenario::SigverifyDiffControl => { + "# cb-sigverify-diff-control: the skip_sigverify differential (control\n\ + # arm). Same wrong-pubkey literal relay url as cb-sigverify-diff but\n\ + # withOUT skip_sigverify - CB rejects every bid (PubkeyMismatch), so\n\ + # the run is EXPECTED to fail payload delivery. `sim diff` against the\n\ + # treatment run shows the flip that proves the feature discriminates." + } + Scenario::ExtraValidation => { + "# cb-extra-validation: Enable extra validation of get_header responses\n\ + # via a local execution layer client.\n\ + #\n\ + # Tests that CB will RPC-call the execution client to verify block\n\ + # parameters before returning a header to the beacon node." + } + Scenario::TimingGames => { + "# cb-timing-games: Aggressive timing game configuration.\n\ + #\n\ + # Tests CB's ability to orchestrate repeated get_header polls with\n\ + # short timeouts in order to arrive at the best bid as late as possible\n\ + # in the slot. Per-relay timing overrides are enabled for all relays." + } + Scenario::Mux => { + "# cb-mux: Multiplexed relay routing per validator node.\n\ + #\n\ + # Routes all 128 validators from node-0 exclusively to the first Helix\n\ + # relay instance and all 128 validators from node-1 exclusively to the\n\ + # second Helix relay instance. This tests CB's ability to partition the\n\ + # validator set and apply per-mux timeout and relay configurations." + } + } + } + + /// The relay list. Single-relay scenarios emit `mev_relay: ` (scalar) + /// and OMIT `mev_relay_image`; multi-relay scenarios emit a list and INCLUDE + /// `mev_relay_image` — matching the Python scenario dicts. + fn relays(&self) -> &'static [&'static str] { + match self { + Scenario::Basic + | Scenario::BasicAltClients + | Scenario::MinBid + | Scenario::Signer + | Scenario::SkipSigverify + | Scenario::SigverifyDiff + | Scenario::SigverifyDiffControl + | Scenario::ExtraValidation + | Scenario::WsStream + | Scenario::WsStreamNoKey => &["helix"], + Scenario::MultipleRelays | Scenario::TimingGames | Scenario::Mux => &["helix", "helix"], + } + } + + /// The commit-boost TOML block for this scenario. `keys_dir` is only read for + /// the mux scenario (the 256 per-node pubkey lists). + fn cb_block(&self, keys_dir: &Path) -> Result { + Ok(match self { + Scenario::Basic | Scenario::BasicAltClients | Scenario::MultipleRelays => { + cb_toml(&CbParams::basic()) + } + Scenario::Signer => cb_toml(&CbParams { + signer: Some(SignerParams::devnet()), + ..CbParams::basic() + }), + Scenario::MinBid => cb_toml(&CbParams { + extra_pbs_lines: vec!["min_bid_eth = 0.5".to_string()], + ..CbParams::basic() + }), + Scenario::SkipSigverify => cb_toml(&CbParams { + extra_pbs_lines: vec!["skip_sigverify = true".to_string()], + ..CbParams::basic() + }), + Scenario::SigverifyDiff => cb_toml(&CbParams { + extra_pbs_lines: vec!["skip_sigverify = true".to_string()], + literal_relay_url: Some(poisoned_relay_url()), + ..CbParams::basic() + }), + Scenario::SigverifyDiffControl => cb_toml(&CbParams { + literal_relay_url: Some(poisoned_relay_url()), + ..CbParams::basic() + }), + Scenario::ExtraValidation => cb_toml(&CbParams { + extra_pbs_lines: vec![ + "extra_validation_enabled = true".to_string(), + format!(r#"rpc_url = "{}""#, self.el_cl().el_rpc_url()), + ], + ..CbParams::basic() + }), + Scenario::TimingGames => cb_toml(&CbParams { + timeout_get_header_ms: 400, + timeout_get_payload_ms: 2000, + extra_pbs_lines: Vec::new(), + per_relay_lines: vec![ + "enable_timing_games = true".to_string(), + "target_first_request_ms = 100".to_string(), + "frequency_get_header_ms = 200".to_string(), + ], + literal_relay_url: None, + signer: None, + }), + Scenario::WsStream => cb_toml(&CbParams { + per_relay_lines: vec![ + r#"get_header = "stream""#.to_string(), + r#"headers = { X-Api-Key = "9d5c2f4e-1b7a-4c3d-8e6f-0a1b2c3d4e5f" }"# + .to_string(), + ], + ..CbParams::basic() + }), + Scenario::WsStreamNoKey => cb_toml(&CbParams { + per_relay_lines: vec![r#"get_header = "stream""#.to_string()], + ..CbParams::basic() + }), + Scenario::Mux => { + let node0 = load_pubkeys(keys_dir, 0)?; + let node1 = load_pubkeys(keys_dir, 1)?; + cb_toml_mux(&node0, &node1) + } + }) + } + + fn network_params(&self) -> &'static str { + match self { + Scenario::Mux => MUX_NETWORK_PARAMS, + _ => COMMON_NETWORK_PARAMS, + } + } + + /// The builder subsidy YAML value. cb-multiple-relays uses the per-relay + /// LIST form ([1, 2]: relay 0 gets subsidy 1, relay 1 gets 2) so the two + /// helix instances offer DIVERGENT bid values on the same slot and CB's + /// best-bid aggregation is actually discriminated — with the scalar form + /// one shared builder submits the identical bid to both relays and the + /// comparison is degenerate. Other scenarios keep the historical scalar. + fn builder_subsidy(&self) -> &'static str { + match self { + Scenario::MultipleRelays => "[1, 2]", + // MUST be 0: with a 1 ETH subsidy every bid lands near 1.04 ETH and + // CB caps min_bid_wei below 1 ETH, so no legal floor could reject + // one and the scenario would silently prove nothing. + Scenario::MinBid => "0", + _ => "1", + } + } + + /// Assemble the full Kurtosis args-file for this scenario. Reads + /// `keys/node-{0,1}-pubkeys.json` under `keys_dir` for the mux scenario only. + pub fn args_file_in(&self, images: &Images, keys_dir: &Path) -> Result { + let cb_block = self.cb_block(keys_dir)?; + let mev_params = build_mev_params( + self.relays(), + images, + &cb_block, + self.builder_subsidy(), + matches!(self, Scenario::Signer), + ); + Ok([ + self.comment(), + &self.el_cl().participants(), + COMMON_ADDITIONAL_SERVICES, + "mev_type: custom", + &mev_params, + self.network_params(), + ] + .join("\n\n") + + "\n") + } +} + +// --- sigverify differential fault injection --------------------------------- + +/// A VALID BLS pubkey (validator key from the standard preregistered mnemonic) +/// that is NOT the helix relay's signing key (`DEFAULT_MEV_PUBKEY` in the +/// ethereum-package, 0xa55c1285...). Putting it in CB's [[relays]] url makes +/// validate_signature reject every bid from the real relay (PubkeyMismatch) - +/// unless skip_sigverify is on. Must be a real curve point or CB fails at +/// config parse; a mnemonic validator key is guaranteed valid. +pub const WRONG_RELAY_PUBKEY: &str = "0xaaf6c1251e73fb600624937760fef218aace5b253bf068ed45398aeb29d821e4d2899343ddcbbe37cb3f6cf500dff26c"; + +/// The literal poisoned relay url. Service DNS: with the 1-participant common +/// scenario + the auto-appended builder participant, main.star launches the +/// single helix instance as `helix-relay-2` (index = participant_count 2 + +/// relay_index 0), listening on the fixed in-enclave port 4040 - confirmed by +/// the live 2-helix runs (helix-relay-2/-3). +fn poisoned_relay_url() -> String { + format!("http://{WRONG_RELAY_PUBKEY}@helix-relay-2:4040") +} + +// --- mev_params assembly (ports build_mev_params) --------------------------- + +fn build_mev_params( + relays: &[&str], + images: &Images, + cb_block: &str, + subsidy: &str, + signer: bool, +) -> String { + let mut lines: Vec = vec!["mev_params:".to_string()]; + + if relays.len() > 1 { + lines.push(" mev_relay:".to_string()); + for r in relays { + lines.push(format!(" - {r}")); + } + } else { + lines.push(format!(" mev_relay: {}", relays[0])); + } + + lines.push(" mev_sidecar: commit-boost".to_string()); + lines.push(" mev_builder: flashbots".to_string()); + lines.push(String::new()); + + // Image map. Single-relay scenarios omit mev_relay_image (correlates with the + // scalar relay form above). + lines.push(format!(" helix_relay_image: {}", images.helix_relay)); + if relays.len() > 1 { + lines.push(format!(" mev_relay_image: {}", images.mev_relay)); + } + lines.push(format!(" mev_boost_image: {}", images.mev_boost)); + lines.push(format!(" mev_builder_image: {}", images.builder_el)); + lines.push(format!(" mev_builder_cl_image: {}", images.builder_cl)); + + lines.push(String::new()); + lines.push(format!(" mev_builder_subsidy: {subsidy}")); + if signer { + // Opt-in knob consumed by the fork's main.star: launch a CB SIGNER + // container beside the PBS sidecar, reusing this participant's + // validator keystores. + lines.push(" commit_boost_signer: true".to_string()); + } + lines.push(String::new()); + + // helix block scalar: indent non-empty lines 4 spaces, blanks stay empty. + lines.push(" helix_relay_config: |".to_string()); + push_block_scalar(&mut lines, HELIX_RELAY_CONFIG); + lines.push(String::new()); + + // commit-boost block scalar. + lines.push(" commit_boost_config: |".to_string()); + push_block_scalar(&mut lines, cb_block); + + lines.join("\n") +} + +/// Append `body` as a YAML `|` block scalar body: non-empty lines get a 4-space +/// indent, blank lines stay truly empty (matches Python `build_mev_params`). +fn push_block_scalar(lines: &mut Vec, body: &str) { + for line in body.lines() { + if line.trim().is_empty() { + lines.push(String::new()); + } else { + lines.push(format!(" {line}")); + } + } +} + +/// Load `keys_dir/node-{node}-pubkeys.json` as a list of pubkey strings. Fallible +/// so a missing/malformed keys file surfaces as a clean `run` error (and lets the +/// caller validate BEFORE writing anything), matching the Python's pre-write +/// `load_pubkeys` + `sys.exit(1)` rather than panicking mid-generation. +fn load_pubkeys(keys_dir: &Path, node: u8) -> Result> { + let path = keys_dir.join(format!("node-{node}-pubkeys.json")); + let raw = std::fs::read_to_string(&path) + .wrap_err_with(|| format!("reading pubkey file {}", path.display()))?; + serde_json::from_str(&raw).wrap_err_with(|| format!("parsing pubkey JSON {}", path.display())) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::genmodel::assert_matches_golden; + + /// Headline test: every scenario assembled with the default images must be + /// byte-identical to its golden fixture. The mux scenario exercises the full + /// 256-key path (real `keys/*.json`). + #[test] + fn every_scenario_matches_its_golden() { + let images = Images::default(); + for s in Scenario::ALL { + let produced = s.args_file_in(&images, Path::new("keys")).unwrap(); + assert_matches_golden(s.name(), &produced); + } + } + + #[test] + fn alt_client_pair_flows_into_participants() { + // Law 7: the pair is real config, not a label. + let out = Scenario::BasicAltClients + .args_file_in(&Images::default(), Path::new("keys")) + .unwrap(); + assert!( + out.contains("el_type: nethermind"), + "alt EL in participants" + ); + assert!(out.contains("cl_type: prysm"), "alt CL in participants"); + // Every other scenario stays on the baked default pair. + let basic = Scenario::Basic + .args_file_in(&Images::default(), Path::new("keys")) + .unwrap(); + assert!(basic.contains("el_type: geth") && basic.contains("cl_type: lighthouse")); + } + + #[test] + fn extra_validation_rpc_url_derives_from_the_pair() { + // The coupling Law 7 exists to catch: the ethereum-package names the EL + // service el-{index}-{el}-{cl}, so a HARDCODED rpc_url silently points + // at a nonexistent service on any other pair (extra validation then + // no-ops, and feature.extra_validation would WARN). Derive it instead. + assert_eq!( + ElCl::DEFAULT.el_rpc_url(), + "http://el-1-geth-lighthouse:8545" + ); + assert_eq!(ElCl::ALT.el_rpc_url(), "http://el-1-nethermind-prysm:8545"); + // The default-pair scenario's rendered config still carries the + // original url byte-for-byte (no silent drift from the refactor). + let out = Scenario::ExtraValidation + .args_file_in(&Images::default(), Path::new("keys")) + .unwrap(); + assert!(out.contains(r#"rpc_url = "http://el-1-geth-lighthouse:8545""#)); + } + + #[test] + fn from_name_round_trips() { + for s in Scenario::ALL { + assert_eq!(Scenario::from_name(s.name()), Some(s)); + } + assert_eq!(Scenario::from_name("nope"), None); + } + + #[test] + fn tracked_cb_basic_config_stays_in_sync_with_sim_generate() { + // configs/generated/cb-basic.yml is TRACKED (render.rs's fixture + what + // `sim preflight` validates). Guard it against silently drifting from the + // generator — the staleness class that rotted the old example config. + let tracked = include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/configs/generated/cb-basic.yml" + )); + let produced = Scenario::Basic + .args_file_in(&Images::default(), Path::new("keys")) + .unwrap(); + assert_eq!( + produced, tracked, + "configs/generated/cb-basic.yml is stale — run `just generate-configs`" + ); + } + + #[test] + fn mux_with_missing_keys_is_a_clean_error_not_a_panic() { + // The mux scenario reads keys/*.json; a missing dir must surface as an + // Err (which `run` turns into a clean exit), never a panic mid-generation. + let err = Scenario::Mux + .args_file_in(&Images::default(), Path::new("/no/such/keys")) + .unwrap_err(); + assert!(err.to_string().contains("pubkey file"), "got: {err}"); + } + + #[test] + fn non_mux_scenarios_need_no_keys() { + // Everything except mux must assemble regardless of the keys dir. + for s in Scenario::ALL { + if s == Scenario::Mux { + continue; + } + assert!( + s.args_file_in(&Images::default(), Path::new("/no/such/keys")) + .is_ok() + ); + } + } +} diff --git a/src/bin/sim/main.rs b/src/bin/sim/main.rs new file mode 100644 index 0000000..724c365 --- /dev/null +++ b/src/bin/sim/main.rs @@ -0,0 +1,117 @@ +//! `sim`: structured helix preflight + triage for Commit-Boost Kurtosis testnets. +//! +//! Task 0 scaffold. This bin reuses the shared library (`cb_testnet_verifier`) +//! rather than re-declaring modules. The subcommand bodies are stubs; the real +//! implementations land in later tasks (preflight = Task 3, triage = Task 2). +//! +//! Sync only: the verbs shell `kurtosis`/`docker` with `std::process::Command`, +//! matching `discovery.rs`. No tokio. + +use std::path::Path; + +use clap::Parser; + +mod checks_catalog; +mod cli; +mod diagnose; +mod diff; +mod doctor; +mod generate; +mod genmodel; +mod preflight; +mod render; +mod triage; + +use cli::{Cli, Command, LogFormat}; + +fn main() { + let cli = Cli::parse(); + init_tracing(cli.log_format); + + match cli.command { + Command::Preflight { args_file } => preflight(&args_file), + Command::Triage { enclave } => triage(&enclave), + Command::Checks { list, json } => checks(list, json), + Command::Doctor => doctor(), + Command::Diff { from, to, json } => diff_reports(&from, &to, json), + Command::Generate { + scenario, + out_dir, + check, + } => generate(scenario.as_deref(), &out_dir, check), + } +} + +/// Generate Kurtosis args-files (Task 1), or `--check` them (P2 drift gate). +/// Implemented in `generate::run` / `generate::check`. +fn generate(scenario: Option<&str>, out_dir: &Path, check: bool) { + let result = if check { + generate::check(scenario, out_dir) + } else { + generate::run(scenario, out_dir) + }; + if let Err(e) = result { + tracing::error!(error = %e, "sim generate failed"); + eprintln!("generate error: {e:?}"); + std::process::exit(1); + } +} + +/// Emit the machine-readable check catalog (`sim checks --list [--json]`). +/// Implemented in `checks_catalog::run`. +fn checks(list: bool, json: bool) { + if let Err(e) = checks_catalog::run(list, json) { + tracing::error!(error = %e, "sim checks failed"); + eprintln!("checks error: {e:?}"); + std::process::exit(1); + } +} + +/// Compare two verification reports (`sim diff`). Implemented in `diff::run`; +/// exits nonzero if any check regressed (usable as a CI regression gate). +fn diff_reports(from: &Path, to: &Path, json: bool) { + if let Err(e) = diff::run(from, to, json) { + tracing::error!(error = %e, "sim diff failed"); + eprintln!("diff error: {e:?}"); + std::process::exit(1); + } +} + +/// Host-prerequisite preflight for a devnet (`sim doctor`). Implemented in +/// `doctor::run`; exits nonzero if a hard prerequisite (kurtosis/docker) is missing. +fn doctor() { + if let Err(e) = doctor::run() { + tracing::error!(error = %e, "sim doctor failed"); + eprintln!("doctor error: {e:?}"); + std::process::exit(1); + } +} + +/// Initialize the `tracing` subscriber. `--log-format json` emits one JSON +/// object per event; otherwise a pretty human rendering. +fn init_tracing(format: LogFormat) { + match format { + LogFormat::Json => tracing_subscriber::fmt().json().init(), + LogFormat::Pretty => tracing_subscriber::fmt().init(), + } +} + +/// Validate a launch args-file's helix config against the real image before a +/// run (Task 3). Implemented in `preflight::run`; exits nonzero on a `Fail`. +fn preflight(args_file: &Path) { + if let Err(e) = preflight::run(args_file) { + tracing::error!(args_file = %args_file.display(), error = %e, "sim preflight failed"); + eprintln!("preflight error: {e:?}"); + std::process::exit(1); + } +} + +/// Attach to an already-broken enclave and extract each dead service's root +/// cause (Task 2). Implemented in `triage::run`. +fn triage(enclave: &str) { + if let Err(e) = triage::run(enclave) { + tracing::error!(enclave, error = %e, "sim triage failed"); + eprintln!("triage error: {e:?}"); + std::process::exit(1); + } +} diff --git a/src/bin/sim/preflight.rs b/src/bin/sim/preflight.rs new file mode 100644 index 0000000..7f8cbbf --- /dev/null +++ b/src/bin/sim/preflight.rs @@ -0,0 +1,470 @@ +//! `sim preflight ` — the config-drift gate (Task 3), HELIX ONLY. +//! +//! Extract the two embedded config blocks (`render`), validate the HELIX block by +//! running the real helix image against it (~1s), and emit a **3-valued** verdict. +//! The 3-value part is a hard requirement: a slow image pull, docker being down, +//! or a pre-genesis runtime panic must NOT be scored as config drift ("pilot +//! breaks the instrument"). The CB block is stubbed `Inconclusive` — typed +//! validation lands in P2. +//! +//! Layering, mirroring `triage`: the classifier (`classify_helix_probe`) is PURE +//! and fixture-tested; the process I/O (`preflight_helix`, `run`) is smoke-checked +//! manually with Docker + the real image, NOT in `cargo test`. +//! +//! Why key on the panic LOCATION, not "reached fetch": the same run can both +//! reach the fetch stage AND later panic for a non-config reason, and a +//! config-parse failure has a stable, recognisable location (`config.rs` / a +//! serde parse error). Keying on location lets us separate genuine schema drift +//! (Fail) from env/timing/infra noise (Inconclusive) rather than guessing from +//! how far the process got. + +use std::fs; +use std::path::Path; +use std::process::Command; +use std::time::Duration; + +use eyre::{Result, WrapErr, eyre}; +use serde::Serialize; + +use crate::diagnose::{CauseKind, extract_root_cause}; +use crate::render::{self, default_dummies}; + +/// Wall-clock bound for the helix config probe. Config parse + reaching the +/// beacon-fetch stage takes ~1s; a clean config then blocks on the (absent) +/// beacon, which we cut off. helix IGNORES SIGTERM, so we `timeout --signal=KILL` +/// to stop the probe promptly rather than let it run to its own 1-minute panic. +const PROBE_TIMEOUT_SECS: u64 = 8; + +/// The 3-valued config verdict for one block. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(tag = "verdict", rename_all = "snake_case")] +pub enum ConfigVerdict { + /// The config parsed cleanly (the probe got past config load). + Pass, + /// Genuine config drift: a config-parse panic naming the offending field. + Fail { field: String, detail: String }, + /// Could not decide (env/timing/infra) — must NOT fail the gate. + Inconclusive { reason: String }, +} + +/// The preflight report: one verdict per config block. +#[derive(Debug, Clone, Serialize)] +pub struct PreflightReport { + pub helix: ConfigVerdict, + pub commit_boost: ConfigVerdict, +} + +/// Classify a helix config probe's outcome into a 3-valued verdict. +/// +/// Keys on the panic LOCATION (via `diagnose::extract_root_cause`), not on how +/// far the process got: +/// - a `config.rs` panic / serde parse error → `Fail` (real config drift), with +/// the offending field captured from the message. +/// - reached the beacon-fetch stage with no config panic → `Pass` (config +/// parsed; the missing beacon is expected in a probe). +/// - a pre-genesis panic (`chain_info.rs` / `HousekeeperTile` / `current_slot` / +/// unwrap-on-None), a missing `GENESIS_*` env, a kill, or an infra signal +/// (image-pull error, docker daemon down, timeout) → `Inconclusive`. +/// - nothing recognisable → `Inconclusive`. +pub fn classify_helix_probe(exit_status: Option, logs: &str) -> ConfigVerdict { + let cause = extract_root_cause(logs); + + // Panic-keyed buckets. ORDER MATTERS: `config.rs` hosts BOTH the serde config + // parse AND env-var reads / runtime construction, so the non-drift buckets + // (env-missing, pre-genesis) must be checked BEFORE the config-drift Fail — + // otherwise a missing `RELAY_KEY` env read (which panics inside config.rs) is + // mis-scored as schema drift. The "pilot breaks the instrument" trap. + if let Some(rc) = &cause { + let loc = rc.location.as_deref().unwrap_or(""); + let msg = &rc.message; + + // 1) A missing env PREREQUISITE (e.g. `RELAY_KEY should be set: + // NotPresent`) — happens inside config.rs but is not schema drift. + if is_env_missing(msg) { + return ConfigVerdict::Inconclusive { + reason: format!("missing env prerequisite (not config drift): {msg}"), + }; + } + + // 2) Pre-genesis / runtime panic (HousekeeperTile current_slot unwrap) — + // NOT config drift. + let pregenesis = loc.contains("chain_info.rs") + || msg.contains("HousekeeperTile") + || msg.contains("current_slot") + || (msg.contains("unwrap()") && msg.contains("None")); + if pregenesis { + return ConfigVerdict::Inconclusive { + reason: format!("pre-genesis runtime panic (not config): {msg}"), + }; + } + + // 3) Real config drift: a serde parse signature, or a config.rs parse + // panic — captures the offending field name from the message. + if is_serde_parse_error(msg) || loc.contains("config.rs") { + return ConfigVerdict::Fail { + field: capture_field(msg), + detail: msg.clone(), + }; + } + } + + // 4) Config parsed and the relay reached the beacon-fetch stage → Pass. This + // is checked BEFORE the timeout/kill bucket: our probe supplies no beacon, + // so a config-clean relay reaches fetch and then either retries until our + // wall-clock kill or panics with "failed fetching chain info for 1 minute" + // — both mean the config parsed, so both are a Pass, not a kill. + if reached_fetch(logs) && !is_config_panic(&cause) { + return ConfigVerdict::Pass; + } + + // 5) Infra signals in the logs — image pull, docker daemon, timeout marker. + if let Some(reason) = infra_signal(logs) { + return ConfigVerdict::Inconclusive { reason }; + } + + // 6) A bare kill (from the extracted cause or a SIGKILL/timeout exit code) + // with no fetch-stage evidence — we could not tell whether config parsed. + if matches!(cause.as_ref().map(|c| c.kind), Some(CauseKind::Killed)) + || matches!(exit_status, Some(137) | Some(124) | Some(143)) + { + return ConfigVerdict::Inconclusive { + reason: "probe killed (timeout / OOM) before reaching a config verdict".to_string(), + }; + } + + // 7) Missing genesis env — a runtime prerequisite, not a config schema issue. + if logs.contains("GENESIS_") && logs.to_ascii_lowercase().contains("not set") + || logs.contains("missing GENESIS") + { + return ConfigVerdict::Inconclusive { + reason: "missing GENESIS_* env (runtime prerequisite, not config)".to_string(), + }; + } + + // 8) Nothing recognisable. + ConfigVerdict::Inconclusive { + reason: "no recognisable config-parse or fetch-stage signal in probe logs".to_string(), + } +} + +/// Does this message look like a missing ENV prerequisite (e.g. `RELAY_KEY +/// should be set: NotPresent`)? Such reads live inside `config.rs` but are a +/// runtime prerequisite, not config-schema drift. +fn is_env_missing(message: &str) -> bool { + const NEEDLES: [&str; 4] = [ + "NotPresent", + "should be set", + "environment variable", + "VarError", + ]; + NEEDLES.iter().any(|n| message.contains(n)) +} + +/// Does this message look like a serde/config parse failure (schema drift)? +fn is_serde_parse_error(message: &str) -> bool { + const NEEDLES: [&str; 5] = [ + "missing field", + "unknown field", + "untagged", + "failed to parse config", + "did not match any variant", + ]; + NEEDLES.iter().any(|n| message.contains(n)) +} + +/// True if the extracted cause is a config-parse panic (used to guard `Pass`). +fn is_config_panic(cause: &Option) -> bool { + cause.as_ref().is_some_and(|rc| { + rc.location.as_deref().unwrap_or("").contains("config.rs") + || is_serde_parse_error(&rc.message) + }) +} + +/// Capture the offending field name from a serde error message. +/// +/// Serde renders the field between backticks (`missing field \`decoder\``), so we +/// pull the first backtick-quoted token. Pattern-based — a never-seen field is +/// captured the same as a known one. Falls back to `"unknown"`. +fn capture_field(message: &str) -> String { + let mut parts = message.split('`'); + // parts: [before, FIELD, after, …] — the first quoted token is index 1. + if let (Some(_), Some(field)) = (parts.next(), parts.next()) + && !field.is_empty() + { + return field.to_string(); + } + "unknown".to_string() +} + +/// Did the relay reach the beacon-fetch stage (proving config parsed)? +fn reached_fetch(logs: &str) -> bool { + const NEEDLES: [&str; 3] = [ + "get_chain_info", + "failed fetching chain info", + "starting metrics server", + ]; + NEEDLES.iter().any(|n| logs.contains(n)) +} + +/// Recognise an infra failure (image pull / docker daemon / timeout marker). +fn infra_signal(logs: &str) -> Option { + let lower = logs.to_ascii_lowercase(); + const NEEDLES: [(&str, &str); 6] = [ + ( + "manifest unknown", + "image not available in registry (manifest unknown)", + ), + ( + "no such image", + "image not present locally / pull failed (no such image)", + ), + ( + "unable to find image", + "image not present locally / pull in progress", + ), + ( + "cannot connect to the docker daemon", + "docker daemon not reachable", + ), + ( + "is the docker daemon running", + "docker daemon not reachable", + ), + ("__sim_probe_timeout__", "probe hit the wall-clock timeout"), + ]; + NEEDLES + .iter() + .find(|(needle, _)| lower.contains(needle)) + .map(|(_, reason)| reason.to_string()) +} + +/// Validate the helix config block by running the real helix image against it. +/// +/// Substitutes the block's runtime vars, writes it to a tmp dir, runs +/// `docker run --rm --entrypoint sh -c 'exec /app/helix-relay --config +/// /cfg/config.yaml'` with the dir mounted at `/cfg` under a bounded timeout, +/// captures combined output, and classifies. Smoke-checked manually (needs Docker +/// + the image), NOT a `cargo test`. +pub fn preflight_helix(image: &str, yaml_block: &str) -> Result { + let rendered = render::substitute_runtime_vars(yaml_block, &default_dummies()); + + // A private tmp dir mounted read-only into the container. + let tmp = std::env::temp_dir().join(format!("sim-preflight-{}", std::process::id())); + fs::create_dir_all(&tmp).wrap_err("create preflight tmp dir")?; + let cfg_path = tmp.join("config.yaml"); + fs::write(&cfg_path, rendered).wrap_err("write rendered helix config")?; + + let container = format!("sim-preflight-{}", std::process::id()); + let verdict = run_probe(image, &tmp, &container); + + // Best-effort cleanup of the tmp file and any orphan container. + let _ = fs::remove_file(&cfg_path); + let _ = fs::remove_dir(&tmp); + let _ = Command::new("docker") + .args(["rm", "-f", &container]) + .output(); + + verdict +} + +/// The dummy env the real helix `:main` image reads at boot BEFORE it parses the +/// config file (from `helix_relay_launcher.star`). Without these the relay panics +/// on an env read inside `config.rs` and never reaches the YAML parse we want to +/// exercise. Values are the launcher's own dummies (a throwaway secret key, etc.); +/// a past `GENESIS_TIME` lets the relay compute `current_slot()` and reach the +/// beacon-fetch stage instead of the pre-genesis unwrap. +const PROBE_ENV: [(&str, &str); 4] = [ + ( + "RELAY_KEY", + "0x607a11b45a7219cc61a3d9c5fd08c7eebd602a6a19a977f8d3771d5711a550f2", + ), + ("POSTGRES_PASSWORD", "postgres"), + ("ADMIN_TOKEN", "admin_token"), + ("GENESIS_TIME", "1700000000"), +]; + +/// Run the bounded docker probe and classify its combined output. +fn run_probe(image: &str, cfg_dir: &Path, container: &str) -> Result { + let mount = format!("{}:/cfg:ro", cfg_dir.display()); + let secs = Duration::from_secs(PROBE_TIMEOUT_SECS) + .as_secs() + .to_string(); + + // `timeout` bounds the shell (sync std has no wait-with-timeout), matching + // `triage`. `--signal=KILL` because helix ignores SIGTERM; on the kill it + // exits 137 (a plain timeout kill would be 124); a missing docker → 127. + let mut cmd = Command::new("timeout"); + cmd.args(["--signal=KILL", &secs]) + .args(["docker", "run", "--rm", "--name", container, "-v", &mount]); + for (k, v) in PROBE_ENV { + cmd.args(["-e", &format!("{k}={v}")]); + } + let output = cmd + .args(["--entrypoint", "sh", image, "-c"]) + .arg("exec /app/helix-relay --config /cfg/config.yaml") + .output() + .wrap_err("failed to spawn `docker` (via `timeout`). Is docker installed and on PATH?")?; + + if output.status.code() == Some(127) { + return Ok(ConfigVerdict::Inconclusive { + reason: "docker not found on PATH (needed by `sim preflight`)".to_string(), + }); + } + + let mut logs = String::new(); + logs.push_str(&String::from_utf8_lossy(&output.stdout)); + if !output.stderr.is_empty() { + logs.push('\n'); + logs.push_str(&String::from_utf8_lossy(&output.stderr)); + } + // On a wall-clock kill (124 plain / 137 SIGKILL / no code = signal), mark the + // logs so the classifier can distinguish a timeout from a clean exit. + if matches!(output.status.code(), Some(124) | Some(137) | None) { + logs.push_str("\n__sim_probe_timeout__\n"); + } + + Ok(classify_helix_probe(output.status.code(), &logs)) +} + +/// Entry point for `sim preflight `. +/// +/// Extracts both config blocks, validates the helix block against its image, +/// stubs the CB block as `Inconclusive` (P2), prints the JSON report, and exits +/// nonzero ONLY on a `Fail` (an `Inconclusive` must not break the gate on a slow +/// pull). +pub fn run(args_file: &Path) -> Result<()> { + let contents = fs::read_to_string(args_file) + .wrap_err_with(|| format!("read args-file {}", args_file.display()))?; + let blocks = render::extract_config_blocks(&contents)?; + let image = helix_image(&contents)?; + + tracing::info!(image = %image, "preflighting helix config"); + let helix = preflight_helix(&image, &blocks.helix)?; + + // The CB block IS extracted, but typed validation is deferred to P2 (a + // 409-crate dep, broken as specified). Surface that we saw it, then stub. + tracing::info!( + cb_config_bytes = blocks.commit_boost.len(), + "commit-boost config extracted; typed validation deferred to P2" + ); + let commit_boost = ConfigVerdict::Inconclusive { + reason: "typed validation lands in P2".to_string(), + }; + + let report = PreflightReport { + helix, + commit_boost, + }; + let json = serde_json::to_string_pretty(&report).wrap_err("serialize preflight report")?; + println!("{json}"); + + // Exit nonzero ONLY on a genuine Fail — Inconclusive must not fail the gate. + if matches!(report.helix, ConfigVerdict::Fail { .. }) + || matches!(report.commit_boost, ConfigVerdict::Fail { .. }) + { + std::process::exit(1); + } + Ok(()) +} + +/// Read the helix image id from the args-file (`mev_params.helix_relay_image`). +fn helix_image(args_file_contents: &str) -> Result { + let root: serde_yaml::Value = serde_yaml::from_str(args_file_contents)?; + root.get("mev_params") + .and_then(|m| m.get("helix_relay_image")) + .and_then(serde_yaml::Value::as_str) + .map(str::to_string) + .ok_or_else(|| eyre!("args-file has no `mev_params.helix_relay_image`")) +} + +#[cfg(test)] +mod tests { + use super::*; + + const SERDE_MISSING: &str = + include_str!("../../../tests/fixtures/helix_serde_missing_field.log"); + const PREGENESIS: &str = include_str!("../../../tests/fixtures/helix_pregenesis_unwrap.log"); + const INVENTED: &str = include_str!("../../../tests/fixtures/invented_field.log"); + const REACHED_FETCH: &str = include_str!("../../../tests/fixtures/helix_reached_fetch.log"); + const DOCKER_PULL: &str = include_str!("../../../tests/fixtures/docker_pull_error.log"); + const OOM: &str = include_str!("../../../tests/fixtures/oom_killed.log"); + const CLEAN: &str = include_str!("../../../tests/fixtures/clean.log"); + + #[test] + fn serde_missing_field_is_fail_naming_the_field() { + let v = classify_helix_probe(Some(101), SERDE_MISSING); + match v { + ConfigVerdict::Fail { field, .. } => { + // Captured from the message (generalization), not hardcoded-matched. + assert_eq!(field, "decoder", "should capture the missing field name"); + } + other => panic!("expected Fail, got {other:?}"), + } + } + + #[test] + fn held_out_field_is_captured_as_fail() { + // A field name we have never seen — proves pattern capture, not memory. + let v = classify_helix_probe(Some(101), INVENTED); + match v { + ConfigVerdict::Fail { field, .. } => assert_eq!(field, "foobar"), + other => panic!("expected Fail for held-out field, got {other:?}"), + } + } + + #[test] + fn pregenesis_unwrap_is_inconclusive_not_fail() { + // The "pilot breaks the instrument" trap: a runtime panic must NOT be + // scored as config drift. + let v = classify_helix_probe(Some(101), PREGENESIS); + assert!( + matches!(v, ConfigVerdict::Inconclusive { .. }), + "pre-genesis unwrap must be Inconclusive, got {v:?}" + ); + } + + #[test] + fn reached_fetch_is_pass() { + // Config parsed; the relay got to the beacon-fetch stage. The missing + // beacon in a probe is expected — that is a Pass, not a failure. + let v = classify_helix_probe(None, REACHED_FETCH); + assert_eq!(v, ConfigVerdict::Pass); + } + + #[test] + fn docker_pull_error_is_inconclusive() { + // An image-pull / registry problem is infra, not config drift. + let v = classify_helix_probe(Some(125), DOCKER_PULL); + assert!( + matches!(v, ConfigVerdict::Inconclusive { .. }), + "docker pull error must be Inconclusive, got {v:?}" + ); + } + + #[test] + fn oom_kill_is_inconclusive() { + let v = classify_helix_probe(Some(137), OOM); + assert!( + matches!(v, ConfigVerdict::Inconclusive { .. }), + "a kill must be Inconclusive, got {v:?}" + ); + } + + #[test] + fn unrecognised_logs_are_inconclusive() { + let v = classify_helix_probe(None, CLEAN); + assert!( + matches!(v, ConfigVerdict::Inconclusive { .. }), + "no recognisable signal must be Inconclusive, got {v:?}" + ); + } + + #[test] + fn timeout_exit_is_inconclusive() { + let v = classify_helix_probe(Some(124), "some partial output\n__sim_probe_timeout__\n"); + assert!( + matches!(v, ConfigVerdict::Inconclusive { .. }), + "a timeout kill must be Inconclusive, got {v:?}" + ); + } +} diff --git a/src/bin/sim/render.rs b/src/bin/sim/render.rs new file mode 100644 index 0000000..a81ec53 --- /dev/null +++ b/src/bin/sim/render.rs @@ -0,0 +1,252 @@ +//! Shared runtime-var substitution + config-block extraction for the `sim` +//! preflight (Task 1). +//! +//! The kurtosis args-file embeds two configs as YAML `|` block scalars under +//! `mev_params`: `helix_relay_config` (YAML) and `commit_boost_config` (TOML). +//! Both carry unrendered Go-template vars (`{{ .VAR }}`), and the CB block has a +//! `{{ range $i, $r := .Relays }} … {{- end }}` loop. Un-substituted, neither +//! block is parseable. This module renders dummy values in so later tasks can +//! validate the shapes. +//! +//! Pure: string in, string out. No fs/process (the test harness reads the real +//! fixture; the module itself never touches I/O). + +use std::collections::BTreeMap; + +/// The two embedded config bodies pulled out of the args-file. +pub struct ConfigBlocks { + /// The `helix_relay_config` body (YAML, with template vars unrendered). + pub helix: String, + /// The `commit_boost_config` body (TOML, with template vars + a `.Relays` + /// range loop unrendered). + pub commit_boost: String, +} + +/// Pull the two `|` block scalars out of the kurtosis args-file. +/// +/// The `|` bodies are opaque scalars to YAML, so `serde_yaml` hands them back as +/// plain strings — exactly the un-rendered template text we want to substitute +/// into. +pub fn extract_config_blocks(args_file_contents: &str) -> eyre::Result { + let root: serde_yaml::Value = serde_yaml::from_str(args_file_contents)?; + let mev = root + .get("mev_params") + .ok_or_else(|| eyre::eyre!("args-file has no `mev_params` mapping"))?; + + let helix = mev + .get("helix_relay_config") + .and_then(serde_yaml::Value::as_str) + .ok_or_else(|| eyre::eyre!("`mev_params.helix_relay_config` missing or not a string"))? + .to_string(); + + let commit_boost = mev + .get("commit_boost_config") + .and_then(serde_yaml::Value::as_str) + .ok_or_else(|| eyre::eyre!("`mev_params.commit_boost_config` missing or not a string"))? + .to_string(); + + Ok(ConfigBlocks { + helix, + commit_boost, + }) +} + +/// Render a config block: strip any `{{ range … }} … {{- end }}` loop, then +/// replace every `{{ .VAR }}` with its dummy. +/// +/// Whitespace inside the markers is tolerated (`{{.VAR}}`, `{{ .VAR }}`). A +/// `{{ .VAR }}` with no dummy is left verbatim on purpose: the caller's +/// `!contains("{{")` check then flags it as a missing dummy rather than silently +/// emitting a broken value. +pub fn substitute_runtime_vars(block: &str, dummies: &BTreeMap<&str, String>) -> String { + let stripped = strip_range_blocks(block); + replace_simple_vars(&stripped, dummies) +} + +/// Remove whole `{{ range … }} … {{- end }}` (or `{{ end }}`) line ranges. +/// +/// Go's `{{-` trim marker only affects surrounding whitespace, which line-range +/// removal already discards, so we treat `{{- end }}` and `{{ end }}` alike. The +/// `.Relays` are `#[serde(default)]` downstream, so dropping the loop yields a +/// valid empty-relays config. +fn strip_range_blocks(block: &str) -> String { + let mut kept: Vec<&str> = Vec::new(); + let mut skipping = false; + + for line in block.lines() { + if skipping { + // Inside a range block: drop everything up to and including `end`. + if is_range_end(line) { + skipping = false; + } + continue; + } + + if is_range_start(line) { + // A degenerate one-line `{{ range … }}{{ end }}` closes immediately. + skipping = !is_range_end(line); + continue; + } + + kept.push(line); + } + + let mut out = kept.join("\n"); + if block.ends_with('\n') { + out.push('\n'); + } + out +} + +fn is_range_start(line: &str) -> bool { + line.contains("{{") && line.contains("range") +} + +fn is_range_end(line: &str) -> bool { + line.contains("end") && line.contains("}}") +} + +/// Replace each `{{ .VAR }}` marker with its dummy, tolerating internal +/// whitespace. Markers whose key has no dummy are emitted verbatim. +fn replace_simple_vars(block: &str, dummies: &BTreeMap<&str, String>) -> String { + let mut out = String::with_capacity(block.len()); + let mut rest = block; + + while let Some(open) = rest.find("{{") { + out.push_str(&rest[..open]); + let after_open = &rest[open + 2..]; + + let Some(close_rel) = after_open.find("}}") else { + // Unbalanced marker: keep the tail untouched and stop. + out.push_str(&rest[open..]); + return out; + }; + + let inner = &after_open[..close_rel]; + let key = inner.trim().trim_start_matches('.'); + + match dummies.get(key) { + Some(val) => out.push_str(val), + // Unknown var: keep the raw marker so the caller's check catches it. + None => out.push_str(&rest[open..open + 2 + close_rel + 2]), + } + + rest = &after_open[close_rel + 2..]; + } + + out.push_str(rest); + out +} + +/// Type-correct dummy values for every runtime var the real args-file uses. +/// +/// Numeric-context vars (`Port`, `POSTGRES_PORT`, `Timestamp`) are bare digit +/// strings so they render unquoted (`port: 5432`, `genesis_time_secs = 170…`); +/// URI vars are valid URLs; the rest are plain names that fit their quoted slot. +pub fn default_dummies() -> BTreeMap<&'static str, String> { + let mut m = BTreeMap::new(); + // Helix (YAML) vars. + m.insert("POSTGRES_HOST_NAME", "postgres".to_string()); + m.insert("POSTGRES_PORT", "5432".to_string()); + m.insert("POSTGRES_DB", "helix".to_string()); + m.insert("POSTGRES_USER", "helix".to_string()); + m.insert("POSTGRES_PASS", "helixpass".to_string()); + m.insert("BEACON_URI", "http://127.0.0.1:5052".to_string()); + m.insert("BLOCKSIM_URI", "http://127.0.0.1:8545".to_string()); + // Commit-Boost (TOML) vars. + m.insert("Timestamp", "1700000000".to_string()); + m.insert("Network", "mainnet".to_string()); + m.insert("Port", "18550".to_string()); + m +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The real, checked-in args-file — the only trustworthy fixture for the + /// exact template shapes we must handle. + const ARGS_FILE: &str = include_str!("../../../configs/generated/cb-basic.yml"); + + #[test] + fn test_extract_both_blocks() { + let blocks = extract_config_blocks(ARGS_FILE).expect("extract"); + + // helix body starts with the helix config's first line. + assert!( + blocks.helix.trim_start().starts_with("instance_id:"), + "helix block should start with instance_id, got:\n{}", + &blocks.helix[..blocks.helix.len().min(80)] + ); + // commit_boost body carries the [pbs] table. + assert!( + blocks.commit_boost.contains("[pbs]"), + "commit_boost block should contain [pbs]" + ); + // Neither block bleeds into the other. + assert!( + !blocks.helix.contains("[pbs]"), + "helix block must not contain the CB key [pbs]" + ); + assert!( + !blocks.commit_boost.contains("instance_id"), + "commit_boost block must not contain the helix key instance_id" + ); + } + + #[test] + fn test_substitute_covers_all_vars() { + let blocks = extract_config_blocks(ARGS_FILE).expect("extract"); + let dummies = default_dummies(); + + for (name, block) in [ + ("helix", &blocks.helix), + ("commit_boost", &blocks.commit_boost), + ] { + let rendered = substitute_runtime_vars(block, &dummies); + assert!( + !rendered.contains("{{"), + "{name} block still has an opening template marker after substitution:\n{rendered}" + ); + assert!( + !rendered.contains("}}"), + "{name} block still has a closing template marker after substitution:\n{rendered}" + ); + } + } + + #[test] + fn test_range_block_stripped() { + let blocks = extract_config_blocks(ARGS_FILE).expect("extract"); + let rendered = substitute_runtime_vars(&blocks.commit_boost, &default_dummies()); + + assert!( + !rendered.contains("[[relays]]"), + "the .Relays range body should be stripped, got:\n{rendered}" + ); + assert!( + !rendered.contains("range"), + "no `range` residue should survive" + ); + assert!(!rendered.contains("end"), "no `end` residue should survive"); + } + + #[test] + fn test_substituted_helix_is_valid_yaml() { + let blocks = extract_config_blocks(ARGS_FILE).expect("extract"); + let rendered = substitute_runtime_vars(&blocks.helix, &default_dummies()); + + serde_yaml::from_str::(&rendered).unwrap_or_else(|e| { + panic!("substituted helix is not valid YAML: {e}\n---\n{rendered}") + }); + } + + #[test] + fn test_substituted_cb_is_valid_toml() { + let blocks = extract_config_blocks(ARGS_FILE).expect("extract"); + let rendered = substitute_runtime_vars(&blocks.commit_boost, &default_dummies()); + + toml::from_str::(&rendered) + .unwrap_or_else(|e| panic!("substituted CB is not valid TOML: {e}\n---\n{rendered}")); + } +} diff --git a/src/bin/sim/triage.rs b/src/bin/sim/triage.rs new file mode 100644 index 0000000..9c4563f --- /dev/null +++ b/src/bin/sim/triage.rs @@ -0,0 +1,321 @@ +//! `sim triage ` — attach to a broken enclave, extract each dead +//! service's ROOT cause, emit a structured JSON report (Task 2, the wiring). +//! +//! Design (J): observability is a property of the run. The JSON `TriageReport` +//! is the single source of truth — a human reads a pretty rendering, an agent +//! reads the JSON, both off the same surface. `triage` is just one entry point. +//! +//! The masking fix: `kurtosis service logs` routes through a broker that can +//! surface a grpc UTF-8 error instead of the real panic, or return empty on a +//! fast-exit race / when a service-add aborted (leaving the service +//! UNREGISTERED). So we try `kurtosis service logs` first and FALL BACK to +//! `docker logs` on the resolved container whenever that is empty / errors / +//! yields no root cause. +//! +//! Only this module does process I/O; `diagnose::extract_root_cause` stays pure. +//! The `run` path needs kurtosis + docker, so it is a manual/Docker-gated smoke +//! check, NOT a `cargo test` (only the pure `parse_service_statuses` / +//! `services_to_triage` cores are unit-tested). + +use std::process::Command; +use std::time::Duration; + +use eyre::{Result, WrapErr, eyre}; +use serde::Serialize; + +use cb_testnet_verifier::discovery::split_on_multi_space; + +use crate::diagnose::{RootCause, extract_root_cause}; + +/// Wall-clock bound for every shell we run (kurtosis/docker can hang). +const SHELL_TIMEOUT_SECS: u64 = 30; + +/// One service row from `kurtosis enclave inspect`, keeping the STATUS column +/// that `discovery::parse_services` discards. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct ServiceStatus { + pub name: String, + pub status: String, +} + +/// A crashed service plus its extracted root cause. +#[derive(Debug, Clone, Serialize)] +pub struct FailedService { + pub service: String, + pub status: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub root_cause: Option, +} + +/// The structured triage output (the source of truth for human + agent). +#[derive(Debug, Clone, Serialize)] +pub struct TriageReport { + pub enclave: String, + pub failed: Vec, +} + +/// Parse the STATUS column out of `kurtosis enclave inspect` text. +/// +/// Reuses `discovery::split_on_multi_space` for the exact column split used to +/// read the `User Services` table. The status is the LAST column of each row +/// (`UUID Name Ports… Status`); ports may themselves span multiple columns, +/// so we key off the row's last field rather than a fixed index. +pub fn parse_service_statuses(inspect_output: &str) -> Vec { + let mut out = Vec::new(); + let mut in_services = false; + let mut header_seen = false; + + for line in inspect_output.lines() { + let stripped = line.trim(); + + if stripped.contains("User Services") { + in_services = true; + header_seen = false; + continue; + } + if !in_services { + continue; + } + // A new top-level section ends the User Services block. + if header_seen && stripped.contains("====") && !stripped.contains("User Services") { + break; + } + if stripped.starts_with("====") || stripped.starts_with("----") || stripped.is_empty() { + continue; + } + if stripped.contains("UUID") && stripped.contains("Name") { + header_seen = true; + continue; + } + if !header_seen { + continue; + } + + let parts = split_on_multi_space(stripped); + // Need at least UUID, Name, Status. + if parts.len() < 3 { + continue; + } + let name = parts[1].trim().to_string(); + let status = parts[parts.len() - 1].trim().to_string(); + if name.is_empty() || status.is_empty() { + continue; + } + out.push(ServiceStatus { name, status }); + } + + out +} + +/// Decide which services need triage: every non-RUNNING service from inspect, +/// UNIONed with any `known_crashed` name that inspect never listed (the +/// half-built-enclave case, where a service-add aborted before registration). +/// +/// Pure so it can be unit-tested; `run` supplies the process-derived inputs. +pub fn services_to_triage( + statuses: &[ServiceStatus], + known_crashed: &[&str], +) -> Vec { + let mut result: Vec = statuses + .iter() + .filter(|s| !s.status.eq_ignore_ascii_case("RUNNING")) + .cloned() + .collect(); + + for &name in known_crashed { + let listed = statuses.iter().any(|s| s.name == name); + if !listed { + result.push(ServiceStatus { + name: name.to_string(), + status: "UNREGISTERED".to_string(), + }); + } + } + + result +} + +/// Entry point for `sim triage `. +pub fn run(enclave: &str) -> Result<()> { + let report = triage(enclave, &[])?; + let json = serde_json::to_string_pretty(&report).wrap_err("serialize triage report")?; + println!("{json}"); + Ok(()) +} + +/// Build the triage report for an enclave, optionally forcing triage of +/// `known_crashed` services that inspect may not list (half-built enclave). +fn triage(enclave: &str, known_crashed: &[&str]) -> Result { + let inspect = run_inspect(enclave)?; + let statuses = parse_service_statuses(&inspect); + let targets = services_to_triage(&statuses, known_crashed); + + let mut failed = Vec::new(); + for target in targets { + tracing::info!(service = %target.name, status = %target.status, "triaging service"); + let logs = collect_logs(enclave, &target.name, &inspect); + let root_cause = logs.as_deref().and_then(extract_root_cause); + failed.push(FailedService { + service: target.name, + status: target.status, + root_cause, + }); + } + + Ok(TriageReport { + enclave: enclave.to_string(), + failed, + }) +} + +/// `kurtosis enclave inspect --full-uuids ` (stdout, best effort). +fn run_inspect(enclave: &str) -> Result { + let out = sh_capture("kurtosis", &["enclave", "inspect", "--full-uuids", enclave])?; + Ok(out.stdout) +} + +/// Get a service's logs with the masking fix: try `kurtosis service logs`; if it +/// errors / is empty / carries no extractable root cause, fall back to +/// `docker logs` on the resolved container. +fn collect_logs(enclave: &str, service: &str, inspect: &str) -> Option { + if let Ok(out) = sh_capture("kurtosis", &["service", "logs", enclave, service]) { + let combined = out.combined(); + if !combined.trim().is_empty() && extract_root_cause(&combined).is_some() { + return Some(combined); + } + } + + // Masking / empty / race: go straight to the container. + if let Some(container) = resolve_container(service, inspect) + && let Ok(out) = sh_capture("docker", &["logs", &container]) + { + let combined = out.combined(); + if !combined.trim().is_empty() { + return Some(combined); + } + } + + None +} + +/// Resolve a docker container name for a kurtosis service. Best effort: kurtosis +/// container names embed the service name, so we match it against `docker ps -a`. +fn resolve_container(service: &str, _inspect: &str) -> Option { + let out = sh_capture("docker", &["ps", "-a", "--format", "{{.Names}}"]).ok()?; + out.stdout + .lines() + .map(str::trim) + .find(|name| name.contains(service)) + .map(str::to_string) +} + +/// Captured output of a bounded shell. +struct ShellOutput { + stdout: String, + stderr: String, +} + +impl ShellOutput { + /// stdout + stderr (crash panics can land on either stream). + fn combined(&self) -> String { + if self.stderr.trim().is_empty() { + self.stdout.clone() + } else if self.stdout.trim().is_empty() { + self.stderr.clone() + } else { + format!("{}\n{}", self.stdout, self.stderr) + } + } +} + +/// Run `program args…` under a wall-clock bound, capturing output. +/// +/// Bounded via the `timeout` coreutil (sync std has no wait-with-timeout). In +/// the style of `discovery::run_kurtosis`, a missing tool is wrapped with a +/// clear "is it installed / on PATH?" message rather than a raw OS error. +fn sh_capture(program: &str, args: &[&str]) -> Result { + let secs = Duration::from_secs(SHELL_TIMEOUT_SECS) + .as_secs() + .to_string(); + let output = Command::new("timeout") + .arg(&secs) + .arg(program) + .args(args) + .output() + .wrap_err_with(|| { + format!( + "failed to spawn `{program}` (via `timeout`). Is `{program}` installed and on PATH?" + ) + })?; + + // `timeout` exits 124 on timeout, 127 when the inner tool is not found. + match output.status.code() { + Some(124) => { + return Err(eyre!( + "`{program} {}` timed out after {secs}s", + args.join(" ") + )); + } + Some(127) => { + return Err(eyre!( + "`{program}` not found on PATH (needed by `sim triage`)" + )); + } + _ => {} + } + + Ok(ShellOutput { + stdout: String::from_utf8_lossy(&output.stdout).into_owned(), + stderr: String::from_utf8_lossy(&output.stderr).into_owned(), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + const INSPECT: &str = include_str!("../../../tests/fixtures/enclave_inspect.txt"); + + #[test] + fn parses_status_column_keeping_running_and_stopped() { + let statuses = parse_service_statuses(INSPECT); + assert_eq!(statuses.len(), 2, "two user services, got {statuses:?}"); + + let running = statuses + .iter() + .find(|s| s.name == "cl-1-lighthouse-geth") + .expect("running service present"); + assert_eq!(running.status, "RUNNING"); + + let stopped = statuses + .iter() + .find(|s| s.name == "mev-relay-helix") + .expect("stopped service present"); + assert_eq!(stopped.status, "STOPPED"); + } + + #[test] + fn services_to_triage_picks_non_running() { + let statuses = parse_service_statuses(INSPECT); + let targets = services_to_triage(&statuses, &[]); + assert_eq!(targets.len(), 1); + assert_eq!(targets[0].name, "mev-relay-helix"); + assert_eq!(targets[0].status, "STOPPED"); + } + + #[test] + fn services_to_triage_adds_unregistered_known_crash() { + // A service the launch tried to add but that aborted before registering: + // inspect never lists it, yet we still want it triaged. + let statuses = parse_service_statuses(INSPECT); + let targets = services_to_triage(&statuses, &["ghost-service", "cl-1-lighthouse-geth"]); + // stopped mev-relay + the unregistered ghost; the RUNNING known name is + // already listed, so it is not double-added. + let names: Vec<&str> = targets.iter().map(|s| s.name.as_str()).collect(); + assert!(names.contains(&"mev-relay-helix")); + assert!(names.contains(&"ghost-service")); + assert!(!names.contains(&"cl-1-lighthouse-geth")); + let ghost = targets.iter().find(|s| s.name == "ghost-service").unwrap(); + assert_eq!(ghost.status, "UNREGISTERED"); + } +} diff --git a/src/bin/test_mux.rs b/src/bin/test_mux.rs deleted file mode 100644 index e22120d..0000000 --- a/src/bin/test_mux.rs +++ /dev/null @@ -1,475 +0,0 @@ -//! Quick mux routing diagnostic. -//! -//! Fetches CB PBS logs from a running enclave, parses them, and checks -//! mux routing against the provided config. No observation window, no -//! epoch waiting. Just: fetch → parse → check. -//! -//! Usage: -//! cargo run --release --bin test_mux -- -//! -//! Example: -//! cargo run --release --bin test_mux -- CB-Testnet configs/generated/cb-mux.yml - -use std::process::Command; - -fn main() { - let args: Vec = std::env::args().collect(); - if args.len() != 3 { - eprintln!("Usage: {} ", args[0]); - eprintln!("Example: {} CB-Testnet configs/generated/cb-mux.yml", args[0]); - std::process::exit(1); - } - - let enclave = &args[1]; - let config_path = &args[2]; - - // Step 1: Parse mux config - println!("=== Parsing mux config: {config_path} ==="); - let entries = match parse_mux_config(config_path) { - Ok(Some(e)) => { - println!("Found {} mux entries:", e.len()); - for entry in &e { - println!(" {} → relay={} pubkeys={}", entry.id, entry.relay_identity, entry.validator_pubkeys.len()); - } - e - } - Ok(None) => { - println!("No [[mux]] sections found in config. Nothing to check."); - std::process::exit(0); - } - Err(e) => { - eprintln!("ERROR parsing config: {e}"); - std::process::exit(1); - } - }; - - // Step 2: Discover CB PBS services - println!("\n=== Discovering CB PBS services in enclave: {enclave} ==="); - let services = match discover_services(enclave) { - Ok(s) => s, - Err(e) => { - eprintln!("ERROR discovering services: {e}"); - std::process::exit(1); - } - }; - println!("Found {} CB PBS service(s): {:?}", services.len(), services); - - if services.is_empty() { - eprintln!("ERROR: No CB PBS services found"); - std::process::exit(1); - } - - // Step 3: Fetch and parse logs from each service - println!("\n=== Fetching logs ==="); - let mut all_events: Vec = Vec::new(); - let log_file = format!("/tmp/test_mux_{}.log", enclave); - - for service in &services { - println!("\n--- {service} ---"); - match fetch_logs(enclave, service) { - Ok(logs) => { - if logs.is_empty() { - println!(" (no relevant log lines found)"); - continue; - } - // Write raw logs to file for debugging - if let Ok(mut f) = std::fs::OpenOptions::new().create(true).append(true).open(&log_file) { - use std::io::Write; - let _ = writeln!(f, "=== {service} ==="); - let _ = writeln!(f, "{}", logs); - } - - let mut parsed = 0; - let mut failed = 0; - for line in logs.lines() { - match parse_line(line) { - Some(event) => { - parsed += 1; - all_events.push(event); - } - None => { - failed += 1; - if failed <= 3 { - println!(" PARSE FAIL: {}", &line[..line.len().min(120)]); - } - } - } - } - println!(" Parsed: {parsed} lines, Failed: {failed} lines"); - } - Err(e) => { - println!(" ERROR: {e}"); - } - } - } - println!("\nRaw logs written to: {log_file}"); - - // Step 4: Print sample of ALL parsed events and collect unique messages - let mut all_messages: Vec = all_events.iter().map(|e| e.message.clone()).collect(); - all_messages.sort(); - all_messages.dedup(); - println!("\n=== Unique messages found ({} total) ===", all_messages.len()); - for msg in &all_messages { - let count = all_events.iter().filter(|e| &e.message == msg).count(); - println!(" ({}) {}", count, msg); - } - - println!("\n=== Sample of all parsed events (first 10) ==="); - for (i, event) in all_events.iter().take(10).enumerate() { - let pk_short = event.validator.as_ref().map(|v| if v.len() > 16 { &v[..16] } else { v }); - println!(" #{} msg={:?} slot={:?} mux={:?} relay={:?} val={:?}", - i, event.message, event.slot, event.mux_id, event.relay_id, pk_short); - } - if all_events.len() > 10 { - println!(" ... and {} more", all_events.len() - 10); - } - - // Step 5: Filter to mux events - println!("\n=== Mux Events ==="); - let mux_events: Vec<&CbEvent> = all_events - .iter() - .filter(|e| { - e.message.starts_with("using mux") - || e.message.starts_with("received new header") - || e.message.starts_with("auction winner") - }) - .collect(); - - println!("Total mux events: {}", mux_events.len()); - for event in &mux_events { - let pk_short = event.validator.as_ref().map(|v| if v.len() > 20 { &v[..20] } else { v }); - println!( - " [{}] slot={:?} mux={:?} relay={:?} val={:?}", - event.message, - event.slot, - event.mux_id, - event.relay_id, - pk_short - ); - } - - // Step 5: Check mux routing - println!("\n=== Mux Routing Check ==="); - let mut violations = 0; - let mut checked = 0; - - for event in &mux_events { - if let Some(ref pk) = event.validator { - let pk_norm = pk.to_lowercase(); - for entry in &entries { - if entry.validator_pubkeys.iter().any(|e| e.to_lowercase() == pk_norm) { - checked += 1; - if let Some(ref actual_mux) = event.mux_id { - if actual_mux != &entry.id { - violations += 1; - println!( - " VIOLATION: pubkey {} should route to '{}' but routed to '{}'", - &pk[..20.min(pk.len())], - entry.id, - actual_mux - ); - } - } - } - } - } - } - - println!("\n=== Result ==="); - if violations > 0 { - println!("FAIL: {violations} routing violation(s) out of {checked} checked"); - std::process::exit(1); - } else if checked == 0 { - println!("WARN: No mux events matched to config pubkeys. Events may not have proposer_pubkey fields."); - std::process::exit(0); - } else { - println!("PASS: All {checked} mux routing decisions are correct"); - std::process::exit(0); - } -} - -// --------------------------------------------------------------------------- -// Types -// --------------------------------------------------------------------------- - -struct MuxEntry { - id: String, - relay_identity: String, - validator_pubkeys: Vec, -} - -struct CbEvent { - message: String, - slot: Option, - validator: Option, - relay_id: Option, - mux_id: Option, -} - -// --------------------------------------------------------------------------- -// Config parsing -// --------------------------------------------------------------------------- - -fn parse_mux_config(path: &str) -> Result>, Box> { - let raw = std::fs::read_to_string(path)?; - - let template = if path.ends_with(".yml") || path.ends_with(".yaml") { - let parsed: serde_yaml::Value = serde_yaml::from_str(&raw)?; - parsed - .get("mev_params") - .and_then(|p| p.get("commit_boost_config")) - .and_then(|c| c.as_str()) - .ok_or("No mev_params.commit_boost_config found")? - .to_string() - } else { - raw - }; - - if !template.contains("[[mux]]") { - return Ok(None); - } - - let mut entries = Vec::new(); - let mut lines = template.lines().peekable(); - - while let Some(line) = lines.next() { - if line.trim() == "[[mux]]" { - let entry = parse_mux_section(&mut lines); - entries.push(entry); - } - } - - if entries.is_empty() { - return Ok(None); - } - - Ok(Some(entries)) -} - -fn parse_mux_section<'a>(lines: &mut std::iter::Peekable>) -> MuxEntry { - let mut id = None; - let mut pubkeys = None; - - loop { - let is_header = lines.peek().map(|l| l.trim().starts_with("[[")).unwrap_or(false); - if is_header { - let header = lines.peek().unwrap().trim().to_string(); - if header.starts_with("[[mux.relays]]") { - let _ = lines.next(); - continue; - } - break; - } - - let Some(line) = lines.next() else { break }; - let trimmed = line.trim(); - if trimmed.is_empty() || trimmed.starts_with('#') { - continue; - } - - if let Some((key, val)) = trimmed.split_once('=') { - let key = key.trim(); - let val = val.trim(); - match key { - "id" => id = Some(val.trim_matches('"').to_string()), - "validator_pubkeys" => pubkeys = Some(parse_pubkey_array(val, lines)), - _ => {} - } - } - } - - let id = id.unwrap_or_default(); - let relay_identity = if let Some(pos) = id.rfind("to_") { - let ident = id[pos + 3..].trim().to_string(); - if !ident.is_empty() { ident } else { id.clone() } - } else { - id.clone() - }; - let pubkeys = pubkeys.unwrap_or_default(); - - MuxEntry { id, relay_identity, validator_pubkeys: pubkeys } -} - -fn parse_pubkey_array(rest: &str, lines: &mut std::iter::Peekable) -> Vec { - let mut accum = rest.to_string(); - if !accum.trim_end().ends_with(']') { - loop { - let Some(next) = lines.next() else { break }; - accum.push('\n'); - accum.push_str(next); - if next.trim().ends_with(']') { break; } - } - } - let raw = accum.trim(); - let start = raw.find('[').unwrap_or(0); - let end = raw.rfind(']').unwrap_or(raw.len()); - raw[start + 1..end] - .split(',') - .map(|s| s.trim().trim_matches('"').to_string()) - .filter(|s| !s.is_empty()) - .collect() -} - -// --------------------------------------------------------------------------- -// Service discovery -// --------------------------------------------------------------------------- - -fn discover_services(enclave: &str) -> Result, Box> { - let output = Command::new("kurtosis") - .args(["enclave", "inspect", "--full-uuids", enclave]) - .output()?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - return Err(format!("kurtosis enclave inspect failed: {stderr}").into()); - } - - let stdout = String::from_utf8_lossy(&output.stdout); - let mut services = Vec::new(); - - for line in stdout.lines() { - let lower = line.to_lowercase(); - if lower.contains("commit-boost") && lower.contains("running") { - // Extract service name (first column) - if let Some(name) = line.split_whitespace().next() { - services.push(name.to_string()); - } - } - } - - Ok(services) -} - -// --------------------------------------------------------------------------- -// Log fetching -// --------------------------------------------------------------------------- - -fn fetch_logs(enclave: &str, service: &str) -> Result> { - let output = Command::new("kurtosis") - .args(["service", "logs", enclave, service, "-n", "200000"]) - .output()?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - return Err(format!("kurtosis service logs failed: {stderr}").into()); - } - - let stdout = String::from_utf8_lossy(&output.stdout); - - // Filter to relevant lines - let result: String = stdout - .lines() - .filter(|line| { - line.contains("using mux config") - || line.contains("received new header") - || line.contains("auction winner") - || line.contains("received unblinded block") - }) - .collect::>() - .join("\n"); - - Ok(result) -} - -// --------------------------------------------------------------------------- -// Log parsing -// --------------------------------------------------------------------------- - -fn strip_ansi_codes(s: &str) -> String { - let mut result = String::with_capacity(s.len()); - let mut chars = s.chars().peekable(); - while let Some(c) = chars.next() { - if c == '\x1b' { - if chars.peek() == Some(&'[') { - chars.next(); - while let Some(&ch) = chars.peek() { - chars.next(); - if ch.is_ascii_alphabetic() { break; } - } - } - } else { - result.push(c); - } - } - result -} - -fn parse_line(line: &str) -> Option { - let line = line.trim(); - if line.is_empty() { return None; } - - // Strip kurtosis prefix: "[service-name] rest" - let line = if line.starts_with('[') { - if let Some(pos) = line.find(']') { - line[pos + 1..].trim_start() - } else { line } - } else { line }; - - // Strip ANSI escape codes - let line = strip_ansi_codes(&line); - - // Find message after "LEVEL : " or "LEVEL " - let after_level: String = if let Some(pos) = line.find(" : ") { - line[pos + 3..].to_string() - } else { - let levels = ["TRACE", "DEBUG", "INFO", "WARN", "ERROR"]; - let mut found = line.clone(); - for lvl in &levels { - if let Some(pos) = line.find(&format!(" {} ", lvl)) { - found = line[pos + lvl.len() + 2..].to_string(); - break; - } - if line.starts_with(lvl) { - found = line[lvl.len()..].trim_start().to_string(); - break; - } - } - found - }; - - // Find message/key boundary: first " key=" where key is a valid identifier - let mut message_end = after_level.len(); - let bytes = after_level.as_bytes(); - for i in 0..bytes.len() { - if bytes[i] == b' ' { - let rest = &after_level[i + 1..]; - if rest.is_empty() { continue; } - let first = rest.as_bytes()[0]; - if first.is_ascii_alphabetic() || first == b'_' { - if let Some(eq_pos) = rest.find('=') { - let key = &rest[..eq_pos]; - if key.chars().all(|c| c.is_alphanumeric() || c == '_') { - let after_eq = &rest[eq_pos + 1..]; - if !after_eq.is_empty() { - message_end = i; - break; - } - } - } - } - } - } - - let message = after_level[..message_end].trim().to_string(); - let kv_part = &after_level[message_end..]; - - let mut slot = None; - let mut validator = None; - let mut relay_id = None; - let mut mux_id = None; - - for kv in kv_part.split_whitespace() { - if let Some((key, val)) = kv.split_once('=') { - let val = val.trim_matches('"'); - match key { - "slot" => { slot = val.parse().ok(); } - "validator" | "pubkey" => { validator = Some(val.to_string()); } - "relay_id" => { relay_id = Some(val.to_string()); } - "mux_id" => { mux_id = Some(val.to_string()); } - _ => {} - } - } - } - - Some(CbEvent { message, slot, validator, relay_id, mux_id }) -} diff --git a/src/bin/test_relay.rs b/src/bin/test_relay.rs deleted file mode 100644 index 17e6876..0000000 --- a/src/bin/test_relay.rs +++ /dev/null @@ -1,226 +0,0 @@ -//! Quick relay API diagnostic. -//! -//! Tests all relay data API endpoints with proper slot filtering. -//! No observation window needed — just query the relay directly. -//! -//! Usage: -//! cargo run --release --bin test_relay -- [pubkey] -//! -//! Examples: -//! cargo run --release --bin test_relay -- http://127.0.0.1:59945 128 160 -//! cargo run --release --bin test_relay -- http://127.0.0.1:59945 128 160 0x889dbdf3bd68af1f6fd84cb6173b1fa1f7c5e6ba63297dc1e2f45cd1a82bb6231ba832adc5228143c5cff3ef0b1caae2 - -use std::time::Duration; - -#[tokio::main] -async fn main() { - let args: Vec = std::env::args().collect(); - if args.len() < 4 { - eprintln!("Usage: {} [pubkey]", args[0]); - eprintln!("Example: {} http://127.0.0.1:59945 128 160", args[0]); - std::process::exit(1); - } - - let relay_url = &args[1]; - let start_slot: u64 = args[2].parse().expect("invalid start_slot"); - let end_slot: u64 = args[3].parse().expect("invalid end_slot"); - let pubkey = args.get(4).map(|s| s.as_str()); - - let client = reqwest::Client::builder() - .timeout(Duration::from_secs(10)) - .build() - .expect("failed to build HTTP client"); - - let base = relay_url.trim_end_matches('/'); - - // === 0. Check if relay has ANY delivered payloads at all === - println!("=== 0. Latest delivered payloads (no slot filter) ==="); - { - let url = format!("{base}/relay/v1/data/bidtraces/proposer_payload_delivered"); - let req = client.get(&url).query(&[("limit", "5")]); - match send::>(req).await { - Ok(payloads) => { - println!(" Total payloads available: {}", payloads.len()); - for p in &payloads { - println!(" slot={} value={} proposer={}...", p.slot, p.value, &p.proposer_pubkey[..20.min(p.proposer_pubkey.len())]); - } - } - Err(e) => println!(" FAIL: {e}"), - } - } - - // === 1. Delivered payloads filtered by slot === - println!("=== 1. Delivered payloads (slot {start_slot}..={end_slot}) ==="); - { - let url = format!("{base}/relay/v1/data/bidtraces/proposer_payload_delivered"); - let mut all = Vec::new(); - let mut cursor: Option = None; - let mut page = 0; - let limit_str = String::from("200"); - let start_slot_str = start_slot.to_string(); - - loop { - page += 1; - let mut params: Vec<(&str, String)> = vec![ - ("limit", limit_str.clone()), - ("slot", start_slot_str.clone()), - ]; - if let Some(ref c) = cursor { - params.push(("cursor", c.clone())); - } - - let req = client.get(&url).query(¶ms); - match send::>(req).await { - Ok(payloads) => { - if payloads.is_empty() { - println!(" Page {page}: empty, done"); - break; - } - let min_slot = payloads.iter().filter_map(|p| p.slot.parse::().ok()).min().unwrap_or(0); - let max_slot = payloads.iter().filter_map(|p| p.slot.parse::().ok()).max().unwrap_or(0); - let page_len = payloads.len(); - let in_range: Vec<_> = payloads.into_iter().filter(|p| { - let s: u64 = p.slot.parse().unwrap_or(0); - s >= start_slot && s <= end_slot - }).collect(); - let count = in_range.len(); - all.extend(in_range); - println!(" Page {page}: {page_len} payloads (slots {min_slot}..={max_slot}), {count} in range, {} total", - all.len()); - if min_slot < start_slot { break; } - cursor = all.last().map(|p: &PayloadDelivered| p.block_number.clone()); - } - Err(e) => { - println!(" FAIL: {e}"); - break; - } - } - if page >= 50 { break; } - } - println!(" Total delivered in range: {}", all.len()); - for p in all.iter().take(5) { - let pk_short = if p.proposer_pubkey.len() > 20 { &p.proposer_pubkey[..20] } else { &p.proposer_pubkey }; - println!(" slot={} value={} proposer={}...", p.slot, p.value, pk_short); - } - } - - // === 2. Builder blocks received filtered by slot === - println!("\n=== 2. Builder blocks received (slot {start_slot}..={end_slot}) ==="); - { - let url = format!("{base}/relay/v1/data/bidtraces/builder_blocks_received"); - let mut all = Vec::new(); - - // Query each slot individually (the API supports slot filter) - for slot in start_slot..=end_slot { - let slot_str = slot.to_string(); - let limit_str = "200".to_string(); - let req = client.get(&url).query(&[("slot", &slot_str), ("limit", &limit_str)]); - match send::>(req).await { - Ok(blocks) => { - if !blocks.is_empty() { - println!(" Slot {slot}: {} blocks", blocks.len()); - for b in &blocks { - let val_short = if b.value.len() > 12 { &b.value[..12] } else { &b.value }; - println!(" builder={}... value={val_short}", &b.builder_pubkey[..20.min(b.builder_pubkey.len())]); - } - all.extend(blocks); - } - } - Err(e) => { - println!(" Slot {slot}: FAIL: {e}"); - } - } - } - println!(" Total builder blocks in range: {}", all.len()); - } - - // === 3. Validator registration check === - println!("\n=== 3. Validator registration ==="); - if let Some(pk) = pubkey { - let url = format!("{base}/relay/v1/data/validator_registration"); - let req = client.get(&url).query(&[("pubkey", pk)]); - match send::(req).await { - Ok(reg) => { - println!(" Registered: YES"); - if let Some(msg) = reg.get("message") { - println!(" Fee recipient: {}", msg.get("fee_recipient").and_then(|v| v.as_str()).unwrap_or("?")); - println!(" Gas limit: {}", msg.get("gas_limit").and_then(|v| v.as_str()).unwrap_or("?")); - println!(" Timestamp: {}", msg.get("timestamp").and_then(|v| v.as_str()).unwrap_or("?")); - } - } - Err(e) => { - println!(" Registered: NO ({e})"); - } - } - } else { - println!(" (skipped — no pubkey provided)"); - } - - // === 4. Summary === - println!("\n=== Summary ==="); - println!("Relay: {base}"); - println!("Slot range: {start_slot}..={end_slot} ({} slots)", end_slot - start_slot + 1); - println!("All queries completed successfully"); -} - -async fn send(req: reqwest::RequestBuilder) -> Result { - let resp = req.send().await.map_err(|e| format!("HTTP error: {e}"))?; - let status = resp.status(); - if !status.is_success() { - let body = resp.text().await.unwrap_or_default(); - return Err(format!("HTTP {}: {}", status, body)); - } - resp.json::().await.map_err(|e| format!("JSON error: {e}")) -} - -#[derive(serde::Deserialize, Debug, Clone)] -struct PayloadDelivered { - slot: String, - block_hash: String, - value: String, - #[serde(default)] - proposer_pubkey: String, - #[serde(default)] - builder_pubkey: String, - #[serde(default)] - block_number: String, - #[serde(default)] - parent_hash: String, - #[serde(default)] - proposer_fee_recipient: String, - #[serde(default)] - gas_limit: String, - #[serde(default)] - gas_used: String, - #[serde(default)] - num_tx: String, -} - -#[derive(serde::Deserialize, Debug, Clone)] -struct BuilderBlock { - slot: String, - block_hash: String, - value: String, - #[serde(default)] - builder_pubkey: String, - #[serde(default)] - proposer_pubkey: String, - #[serde(default)] - block_number: String, - #[serde(default)] - parent_hash: String, - #[serde(default)] - proposer_fee_recipient: String, - #[serde(default)] - gas_limit: String, - #[serde(default)] - gas_used: String, - #[serde(default)] - num_tx: String, - #[serde(default)] - timestamp: String, - #[serde(default)] - timestamp_ms: String, -} - -// Add this as test 0 at the start of main(), before the slot range tests diff --git a/src/checks/best_bid.rs b/src/checks/best_bid.rs new file mode 100644 index 0000000..00672d0 --- /dev/null +++ b/src/checks/best_bid.rs @@ -0,0 +1,341 @@ +//! Aggregated-bidding verification: did Commit-Boost deliver at least the best +//! bid it was OFFERED across relays? +//! +//! Data source (the point of this module vs the reverted first attempt): the +//! per-relay bids come from CB's own "received new header" getHeader log events +//! (`relay_id` + `slot` + `value_eth`) — i.e. exactly what CB compared when it +//! picked a winner — NOT the relay data API's `builder_blocks_received`, which +//! also includes builder submissions that failed simulation and were never +//! offered to the proposer (that source over-stated the bid and false-alarmed). +//! These log events are full-coverage (every getHeader is logged), so there is no +//! slot sampling. The delivered value comes from the relay data API (the actually +//! delivered payload). Both `value_eth` (parsed decimal->wei, exactly, no float) +//! and the delivered U256 are exact wei of the same quantity, so they compare +//! directly — a correct selection delivers the winning bid's exact value. + +use std::collections::BTreeMap; + +use alloy_primitives::U256; + +use crate::checks::CheckResult; +use crate::checks::mux_routing::{fetch_service_logs, parse_cb_log_line}; +use crate::relay::RelayClient; + +/// Parse a `value_eth` decimal string (e.g. "0.050439063999832000") to integer +/// wei, WITHOUT floating point (exact). Returns None on a malformed value. +pub fn value_eth_to_wei(s: &str) -> Option { + let s = s.trim().trim_matches('"'); + let (whole, frac) = match s.split_once('.') { + Some((w, f)) => (w, f), + None => (s, ""), + }; + if whole.is_empty() && frac.is_empty() { + return None; + } + if !whole.chars().all(|c| c.is_ascii_digit()) || !frac.chars().all(|c| c.is_ascii_digit()) { + return None; + } + // 18 decimal places = wei. Pad/truncate the fractional part to 18 digits. + let mut frac18 = String::with_capacity(18); + frac18.push_str(frac); + frac18.truncate(18); + while frac18.len() < 18 { + frac18.push('0'); + } + let combined = format!("{whole}{frac18}"); + // Genuine zero and (physically impossible) u128 overflow both collapse to 0, + // which understates a bid — the safe direction (never a false shortfall). + combined + .trim_start_matches('0') + .parse::() + .ok() + .or(Some(0)) +} + +fn u256_wei_to_u128(v: U256) -> u128 { + v.try_into().unwrap_or(u128::MAX) +} + +/// Verify aggregated bidding for a multi-relay run. SKIPs single-relay runs (no +/// competition possible). Fetches per-relay offered bids from CB logs + delivered +/// values from the relays, then classifies. +pub async fn check_best_bid_selection( + enclave: &str, + cb_service_names: &[String], + relays: &[RelayClient], + start_slot: u64, + end_slot: u64, +) -> CheckResult { + if relays.len() < 2 { + return CheckResult::skip( + "relay.best_bid", + 2, + "Single-relay scenario — no cross-relay aggregation to verify", + ); + } + + // Per-relay offered bids (wei) from CB "received new header" getHeader logs, + // restricted to the observation window so they line up with delivered data. + let mut bids_by_slot: BTreeMap> = BTreeMap::new(); + for service in cb_service_names { + let logs = match fetch_service_logs(enclave, service) { + Ok(l) => l, + Err(_) => continue, + }; + for line in logs.lines() { + let Some(ev) = parse_cb_log_line(line) else { + continue; + }; + if !ev.message.starts_with("received new header") { + continue; + } + let (Some(slot), Some(relay_id)) = (ev.slot, ev.relay_id.as_ref()) else { + continue; + }; + if slot < start_slot || slot > end_slot { + continue; + } + let Some(wei) = ev.fields.get("value_eth").and_then(|v| value_eth_to_wei(v)) else { + continue; + }; + bids_by_slot + .entry(slot) + .or_default() + .push((relay_id.clone(), wei)); + } + } + + // Delivered (winning) value per slot (wei) from the relay data API. + let mut delivered_by_slot: BTreeMap = BTreeMap::new(); + for relay in relays { + if let Ok(payloads) = relay.get_payloads_delivered(start_slot, end_slot).await { + for p in payloads { + let w = u256_wei_to_u128(p.value); + let e = delivered_by_slot.entry(p.slot).or_insert(w); + if w > *e { + *e = w; + } + } + } + } + + classify_best_bid(&bids_by_slot, &delivered_by_slot) +} + +/// Pure verdict logic (Law 4 seam; generic over the value type so tests use a +/// trivial `V`). Contract: +/// - NO slot had >=2 relays offering bids → WARN: aggregation was never exercised. +/// - competitive slots exist but NONE had a delivered payload to compare against → +/// WARN: nothing was actually verified (a competitive slot only counts as +/// verified when we have a delivered value for it — otherwise we'd green having +/// compared nothing, the Law 3 false-green). +/// - a VERIFIED competitive slot where delivered < the best OFFERED bid → +/// suboptimal selection → recorded, verdict WARN. +/// - otherwise → PASS, over the verified competitive slots. +pub fn classify_best_bid( + bids_by_slot: &BTreeMap>, + delivered_by_slot: &BTreeMap, +) -> CheckResult +where + V: Copy + Ord + std::fmt::Display, +{ + let competitive: Vec<(u64, &Vec<(String, V)>)> = bids_by_slot + .iter() + .filter(|(_, bids)| { + bids.iter() + .map(|(r, _)| r) + .collect::>() + .len() + >= 2 + }) + .map(|(s, b)| (*s, b)) + .collect(); + + if competitive.is_empty() { + return CheckResult::warn( + "relay.best_bid", + 2, + format!( + "No multi-relay bid competition observed across {} slot(s) with bids — aggregated \ + bidding was NOT exercised (relays did not offer bids on overlapping slots). Not \ + asserting best-bid selection.", + bids_by_slot.len() + ), + ) + .with_data( + serde_json::json!({ "competitive_slots": 0, "slots_with_bids": bids_by_slot.len() }), + ); + } + + // Only a competitive slot WITH a delivered value can actually be verified. + // `divergent` counts competitive slots whose offered values are NOT all + // equal — the discrimination-strength signal: with identical bids (one + // builder, one subsidy) "delivered >= best" is a tie and proves nothing + // about selection; only a divergent slot shows CB actually picked a winner. + let mut verified = 0usize; + let mut divergent = 0usize; + let mut suboptimal = Vec::new(); + for (slot, bids) in &competitive { + let first = bids[0].1; + if bids.iter().any(|(_, v)| *v != first) { + divergent += 1; + } + let Some(delivered) = delivered_by_slot.get(slot) else { + continue; + }; + verified += 1; + let best = bids.iter().map(|(_, v)| *v).max().unwrap(); // competitive => non-empty + if *delivered < best { + suboptimal.push(serde_json::json!({ + "slot": slot, + "best_offered_bid": best.to_string(), + "delivered": delivered.to_string(), + })); + } + } + + let n = competitive.len(); + let data = serde_json::json!({ + "competitive_slots": n, + "verified_slots": verified, + "divergent_slots": divergent, + "unverified_slots": n - verified, + "suboptimal_count": suboptimal.len(), + "suboptimal": suboptimal, + }); + + if verified == 0 { + CheckResult::warn( + "relay.best_bid", + 2, + format!( + "{n} competitive slot(s) had multi-relay bids but NONE had a delivered payload to \ + compare against (out of window, or the slot was missed) — best-bid selection was \ + not actually verified." + ), + ) + .with_data(data) + } else if suboptimal.is_empty() { + let strength = if divergent > 0 { + format!("{divergent} slot(s) had DIVERGENT offered values (real discrimination)") + } else { + "all offered values were identical (degenerate tie — selection not discriminated)" + .to_string() + }; + CheckResult::pass( + "relay.best_bid", + 2, + format!( + "Aggregated bidding verified across {verified} competitive slot(s) with delivered \ + payloads: CB delivered >= the best offered per-relay bid; {strength}." + ), + ) + .with_data(data) + } else { + CheckResult::warn( + "relay.best_bid", + 2, + format!( + "{} of {verified} verified competitive slot(s) delivered LESS than the best offered \ + bid (may be a late, rejected, or ineligible header — value left on the table).", + suboptimal.len() + ), + ) + .with_data(data) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::checks::CheckStatus; + + #[test] + fn value_eth_parses_to_wei_exactly() { + assert_eq!(value_eth_to_wei("1"), Some(1_000_000_000_000_000_000)); + assert_eq!(value_eth_to_wei("0.000000001"), Some(1_000_000_000)); // 1 gwei + assert_eq!( + value_eth_to_wei("\"0.050439063999832000\""), + Some(50_439_063_999_832_000) + ); + assert_eq!(value_eth_to_wei("0"), Some(0)); + assert_eq!(value_eth_to_wei("abc"), None); + } + + fn bids(pairs: &[(u64, &[(&str, u128)])]) -> BTreeMap> { + pairs + .iter() + .map(|(slot, rs)| (*slot, rs.iter().map(|(r, v)| (r.to_string(), *v)).collect())) + .collect() + } + fn delivered(pairs: &[(u64, u128)]) -> BTreeMap { + pairs.iter().copied().collect() + } + + #[test] + fn best_bid_warn_when_no_multi_relay_competition() { + let b = bids(&[(5, &[("relay-a", 100)]), (6, &[("relay-a", 200)])]); + let d = delivered(&[(5, 100), (6, 200)]); + let r = classify_best_bid(&b, &d); + assert_eq!(r.status, CheckStatus::Warn); + assert_eq!(r.data["competitive_slots"], 0); + } + + #[test] + fn divergent_slots_counted_when_offers_differ() { + // Two relays offering DIFFERENT values (the [1, 2] subsidy setup): the + // slot is divergent — real discrimination, and the detail says so. + let b = bids(&[(5, &[("relay-a", 100), ("relay-b", 150)])]); + let d = delivered(&[(5, 150)]); + let r = classify_best_bid(&b, &d); + assert_eq!(r.data["divergent_slots"], 1); + assert!(r.detail.contains("DIVERGENT"), "detail: {}", r.detail); + } + + #[test] + fn identical_offers_are_a_degenerate_tie_not_divergent() { + // Two relays offering the SAME value (one builder, one subsidy): passes + // but is flagged as a degenerate tie — selection was not discriminated. + let b = bids(&[(5, &[("relay-a", 100), ("relay-b", 100)])]); + let d = delivered(&[(5, 100)]); + let r = classify_best_bid(&b, &d); + assert_eq!(r.status, CheckStatus::Pass); + assert_eq!(r.data["divergent_slots"], 0); + assert!(r.detail.contains("degenerate"), "detail: {}", r.detail); + } + + #[test] + fn best_bid_pass_when_delivered_matches_best() { + let b = bids(&[(5, &[("relay-a", 100), ("relay-b", 150)])]); + let d = delivered(&[(5, 150)]); + let r = classify_best_bid(&b, &d); + assert_eq!(r.status, CheckStatus::Pass); + assert_eq!(r.data["verified_slots"], 1); + assert_eq!(r.data["suboptimal_count"], 0); + } + + #[test] + fn best_bid_warn_when_delivered_below_best() { + let b = bids(&[(5, &[("relay-a", 100), ("relay-b", 150)])]); + let d = delivered(&[(5, 100)]); + let r = classify_best_bid(&b, &d); + assert_eq!(r.status, CheckStatus::Warn); + assert_eq!(r.data["suboptimal_count"], 1); + } + + #[test] + fn best_bid_warn_when_competitive_but_nothing_delivered_to_compare() { + // The Law 3 guard: a competitive slot with NO delivered payload (out of + // window / missed) must NOT count as verified → WARN, never PASS. + let b = bids(&[(5, &[("relay-a", 100), ("relay-b", 150)])]); + let d = delivered(&[(9999, 100)]); // different slot; slot 5 has no delivery + let r = classify_best_bid(&b, &d); + assert_eq!( + r.status, + CheckStatus::Warn, + "must not PASS having verified nothing" + ); + assert_eq!(r.data["verified_slots"], 0); + assert_eq!(r.data["unverified_slots"], 1); + } +} diff --git a/src/checks/cb_metrics.rs b/src/checks/cb_metrics.rs index 41cf40e..81d4e59 100644 --- a/src/checks/cb_metrics.rs +++ b/src/checks/cb_metrics.rs @@ -17,7 +17,14 @@ //! | | 202 | v2 path: relay publishes unblinded block itself | //! | `status`, `reload` | 200 | Endpoint alive / reload succeeded | //! -//! 4xx = client bug. 5xx = relay or CB internal failure. +//! 4xx = client bug. 5xx = relay or CB internal failure. 555 = NOT an HTTP +//! response at all: CB's synthetic client-side timeout marker (commit-boost +//! `constants.rs` `TIMEOUT_ERROR_CODE`), incremented when CB cancels its own +//! request at its deadline. It gets its own `timeout` bucket — counting it as +//! relay 5xx made the timing-games scenario (which cancels late polls at the +//! 400ms budget BY DESIGN) tier-1-fail a run whose relays served zero errors +//! (live-confirmed 2026-08-03: raw counter 200x48/204x6/400x4/555x42, zero +//! real 5xx in helix logs). //! //! # Output shape //! @@ -33,6 +40,17 @@ use crate::metrics; const METRICS_PORT: u16 = 9090; +/// A relay/CB-to-relay 5xx FRACTION above this fails the matrix check; at or below +/// it, a nonzero 5xx count is a transient WARN (warmup noise). See classify_endpoint +/// (the H2 warmup-5xx false-red fix). `--strict` ignores this and fails on any 5xx. +const MAX_5XX_RATE: f64 = 0.25; + +/// A CB client-side timeout (code 555) FRACTION above this WARNs — high timeout +/// rates are either an aggressive timing config (timing-games cancels late polls +/// at its budget by design) or a slow relay, both worth surfacing. Never FAIL: +/// a 555 is CB's own deadline policy firing, not a relay-served error. +const MAX_TIMEOUT_RATE: f64 = 0.25; + /// Status-code counts for a single endpoint, split by observation side. /// /// CB emits the same HTTP-level event at two layers: @@ -130,6 +148,16 @@ impl EndpointStats { fn bucket_code(code: &str) -> String { match code { "200" | "202" | "204" => code.to_string(), + // CB's synthetic client-side timeout marker (TIMEOUT_ERROR_CODE = 555). + // Not a relay-served status — MUST NOT land in the 5xx bucket, or a + // designed deadline-cancellation counts as a relay failure. + "555" => "timeout".to_string(), + // CB's synthetic transport-error marker (TRANSPORT_ERROR_CODE = 556, + // introduced with WS get_header streaming: connect refused / dns / tls + // / stream broke mid-window). Also client-observed, not relay-served — + // must not count toward the relay 5xx error rate. Relay reachability + // has its own tier-1 owner (the relay_pipeline death checks). + "556" => "transport".to_string(), c if c.starts_with('4') && c.len() == 3 => "4xx".to_string(), c if c.starts_with('5') && c.len() == 3 => "5xx".to_string(), _ => "other".to_string(), @@ -220,27 +248,113 @@ pub fn classify_endpoint(endpoint: &str, stats: &EndpointStats, strict: bool) -> let r204 = stats.relay_get("204"); let r4xx = stats.relay_get("4xx"); let r5xx = stats.relay_get("5xx"); + let rtimeout = stats.relay_get("timeout"); + let rtransport = stats.relay_get("transport"); let b5xx = stats.beacon_get("5xx"); - // Relay 5xx always fails. No warnings here -- this is the relay or the - // network between CB and relay actually breaking. + // Relay 5xx handling (H2 fix). These are absolute CUMULATIVE counters that + // include the pre/early-window warmup phase, where a handful of 5xx are normal + // (relays/builders not yet ready). FAILing on any 5xx > 0 made a genuinely + // healthy run red. Classify on the 5xx RATE instead: a materially broken + // relay/CB-to-relay link produces a high fraction of 5xx (FAIL); a few 5xx + // against many good requests is transient warmup (WARN, surfaced but non-fatal). + // `--strict` still promotes the transient case to FAIL for zero-tolerance CI. + // submit_blinded_block is judged on the BEACON side, not the relay side. + // CB asks EVERY configured relay for the payload, but only the relay that + // won the auction has it - the others answer 4xx/5xx by construction, and + // that is not a pipeline failure. Measured on a healthy 2-relay run where + // one relay wins every auction by design (divergent subsidies): + // mev_relay_0 (always loses): 202x1, 4xx x219, 5xx x185 + // mev_relay_1 (always wins): 202x219, 4xx x1, 5xx x1 + // beacon side: 202x220 <- CB served the CL every time + // The relay-side rate was 29.7% and FAILED a run that delivered 65/65 + // payloads with 100% MEV rate and 0 missed slots. The beacon side is the + // signal that actually matters: did the proposer get its payload? It also + // still catches the real failure - on the nethermind+prysm run the beacon + // side was 26x 5xx (CB returning 502 to the CL), which must FAIL. + if endpoint == "submit_blinded_block" && stats.beacon_totals.values().sum::() > 0.0 { + return classify_submit_blinded_block_beacon_side(id, tier, stats, data); + } + if r5xx > 0.0 { - return CheckResult::fail( + // Timeouts (555) are deliberately EXCLUDED from this denominator: the + // 5xx rate is "fraction of COMPLETED relay responses that were errors", + // so a real 5xx storm still FAILs even amid heavy timeout polling. + let total = r200 + r202 + r204 + r4xx + r5xx; + let rate = if total > 0.0 { r5xx / total } else { 1.0 }; + let pct = rate * 100.0; + if rate > MAX_5XX_RATE || strict { + return CheckResult::fail( + id, + tier, + format!( + "{endpoint}: {r5xx:.0}/{total:.0} 5xx ({pct:.1}%) from relay(s) -- relay or \ + CB-to-relay failure (>{:.0}% threshold{})", + MAX_5XX_RATE * 100.0, + if strict { ", --strict" } else { "" } + ), + ) + .with_data(data); + } + return CheckResult::warn( id, tier, - format!("{endpoint}: {r5xx:.0} 5xx from relay(s) -- relay or CB-to-relay failure"), + format!( + "{endpoint}: {r5xx:.0}/{total:.0} transient 5xx ({pct:.1}%) -- likely warmup, below \ + the {:.0}% FAIL threshold", + MAX_5XX_RATE * 100.0 + ), ) .with_data(data); } + // CB client-side synthetic codes: 555 (deadline timeout) and 556 (WS + // transport error). A high rate of either is surfaced as WARN — 555 means + // an aggressive timing config cancelling late polls by design + // (timing-games) or a slow relay; 556 means the stream relay is + // unreachable/breaking (relay reachability's FAIL owner is the tier-1 + // relay_pipeline death check, not this matrix). Never FAIL here, and never + // under --strict either: neither is a relay-served pipeline error. Below + // the threshold, fall through to normal classification (the counts stay + // visible in `data`). + let rsynthetic = rtimeout + rtransport; + if rsynthetic > 0.0 { + let completed = r200 + r202 + r204 + r4xx + r5xx; + let rate = rsynthetic / (completed + rsynthetic); + if rate > MAX_TIMEOUT_RATE { + let pct = rate * 100.0; + return CheckResult::warn( + id, + tier, + format!( + "{endpoint}: {rsynthetic:.0}/{:.0} CB client-side failures ({pct:.1}%: \ + {rtimeout:.0} deadline timeouts (555), {rtransport:.0} ws transport errors \ + (556)) -- not relay-served errors ({r200:.0} bids still delivered). 555 = \ + aggressive timing config or slow relay; 556 = stream relay \ + unreachable/breaking (relay death is the tier-1 relay checks' call)", + completed + rsynthetic + ), + ) + .with_data(data); + } + } + match endpoint { "get_header" => { if r200 > 0.0 { + let mut timeout_note = if rtimeout > 0.0 { + format!(", {rtimeout:.0} CB-deadline timeout (555)") + } else { + String::new() + }; + if rtransport > 0.0 { + timeout_note.push_str(&format!(", {rtransport:.0} ws transport error (556)")); + } CheckResult::pass( id, tier, format!( - "get_header: {r200:.0} bids delivered, {r204:.0} no-bid (204), {r4xx:.0} 4xx" + "get_header: {r200:.0} bids delivered, {r204:.0} no-bid (204), {r4xx:.0} 4xx{timeout_note}" ), ) .with_data(data) @@ -329,8 +443,22 @@ pub fn classify_endpoint(endpoint: &str, stats: &EndpointStats, strict: bool) -> ), ) .with_data(data) + } else if r4xx > 0.0 { + // The proposer DID choose builder blocks — the relay refused + // them. Diagnosing this as "proposer never chose" (which the + // old 200+202==0 branch did) sends an operator to the wrong + // component entirely. Found live on the nethermind+prysm pair + // (2026-08-04): 26 blinded blocks forwarded, 26 relay 4xx. + CheckResult::fail( + id, + tier, + format!( + "submit_blinded_block: the relay REJECTED all {r4xx:.0} blinded block(s) the proposer submitted (0 deliveries, {r4xx:.0} 4xx from relay) -- the proposer DID choose builder blocks; the break is relay-side (block invalid/late/unsigned as the relay sees it), not proposer-side" + ), + ) + .with_data(data) } else { - let msg = "submit_blinded_block: 0 deliveries (200+202=0); proposer never chose a builder block. Pass --strict to treat as failure".to_string(); + let msg = "submit_blinded_block: 0 deliveries and 0 submissions -- the proposer never chose a builder block. Pass --strict to treat as failure".to_string(); if strict { CheckResult::fail(id, tier, msg.replace("Pass --strict ", "(--strict) ")) .with_data(data) @@ -351,23 +479,107 @@ pub fn classify_endpoint(endpoint: &str, stats: &EndpointStats, strict: bool) -> } } -/// Check the v2 -> v1 fallback counter. +/// Verdict for `submit_blinded_block` from the BEACON side: what CB returned to +/// the consensus client. Delivered means the proposer got its payload. +/// +/// Relay-side codes are reported as diagnostic context but never gate the +/// verdict - see the call site for why (losing relays error by construction). +fn classify_submit_blinded_block_beacon_side( + id: String, + tier: u8, + stats: &EndpointStats, + data: serde_json::Value, +) -> CheckResult { + let b200 = stats.beacon_get("200"); + let b202 = stats.beacon_get("202"); + let b4xx = stats.beacon_get("4xx"); + let b5xx = stats.beacon_get("5xx"); + let delivered = b200 + b202; + let relay_5xx = stats.relay_get("5xx"); + let relay_4xx = stats.relay_get("4xx"); + let ctx = format!( + "(relay-side {relay_4xx:.0} 4xx / {relay_5xx:.0} 5xx are the non-winning relays, expected)" + ); + + if b5xx > 0.0 { + return CheckResult::fail( + id, + tier, + format!( + "submit_blinded_block: CB returned {b5xx:.0} 5xx to the beacon node - the proposer \ + did NOT get its payload for those slots ({delivered:.0} delivered) {ctx}" + ), + ) + .with_data(data); + } + if delivered > 0.0 { + return CheckResult::pass( + id, + tier, + format!( + "submit_blinded_block: {delivered:.0} payload(s) served to the beacon node \ + ({b200:.0} v1 200, {b202:.0} v2 202), 0 failures {ctx}" + ), + ) + .with_data(data); + } + CheckResult::warn( + id, + tier, + format!( + "submit_blinded_block: 0 payloads served to the beacon node ({b4xx:.0} 4xx) - the \ + proposer never chose a builder block {ctx}" + ), + ) + .with_data(data) +} + +/// The EXPOSED name of CB's v2-unsupported counter. +/// +/// Note the doubled `pbs_`: the PBS registry is `Registry::new_custom(Some( +/// "cb_pbs"))`, which prefixes every metric, and this counter is *registered* +/// as `pbs_submit_block_v2_unsupported_total` - unlike its siblings, which are +/// registered bare (`relay_status_code_total` -> `cb_pbs_relay_status_code_total`). +const V2_UNSUPPORTED_METRIC: &str = "cb_pbs_pbs_submit_block_v2_unsupported_total"; + +/// Check the v2-unsupported counter: v2 submissions a relay could not serve. /// -/// `cb_pbs_submit_block_v2_fallback_to_v1_total{relay_id}` ticks when CB -/// tried the v2 endpoint and got 404, falling back to v1. A non-zero value -/// means the relay is behind on the builder-specs v2 upgrade. +/// `pbs_submit_block_v2_unsupported_total{relay_id}` ticks when a relay 404s +/// the v2 `submit_block` route. CB deliberately does NOT downgrade to v1 there +/// (in v2 the relay publishes the block after an empty 202, so a v1 payload +/// would be silently dropped by the beacon node) — it fails the submission. /// -/// Missing counter == zero fallbacks == PASS. (Prometheus doesn't emit -/// counter families that never incremented, so absence is the success case.) +/// This is FATAL to the MEV pipeline for any CL that submits via v2: every +/// builder block that proposer chooses is lost, and the slot is typically +/// missed. Found live on nethermind+prysm (2026-08-04): prysm submits to +/// `/eth/v2/builder/blinded_blocks` at ~256ms into the slot, helix 404s the v2 +/// route, CB returns 502, and 11 v2-unsupported events lined up with 11 missed +/// slots. Lighthouse never triggers it because it submits via v1 — the exact +/// class of client-pair-specific break Law 7 exists to surface. /// -/// Always WARN (never FAIL): this is infrastructure drift, not a pipeline -/// failure. Strict mode doesn't change it because the v1 fallback still works. -pub fn check_v2_fallback(scrape: &Scrape) -> CheckResult { - let id = "cb_v2_fallback"; +/// ROOT CAUSE THAT TIME WAS OUR OWN CONFIG, not a helix limitation: helix has a +/// `GetPayloadV2` route and our generated `router_config.enabled_routes` listed +/// only `GetPayload`. Read a 404 on the v2 route as "v2 is DISABLED at the +/// relay" first, and "the relay cannot do v2" only after the route list has +/// actually been checked. +/// +/// Missing counter == zero == PASS (Prometheus omits never-incremented +/// families). Tier 2 -> escalated to tier 1 on FAIL by the caller, like the +/// matrix checks: a relay that cannot serve the proposer's submissions is a +/// real pipeline failure, not an annotation. +/// +/// **The metric name has a doubled `pbs_`** (see [`V2_UNSUPPORTED_METRIC`]). +/// Matching CB's *registered* name instead of its *exposed* name made this +/// check structurally unable to fire: it reported PASS on a run where CB had +/// logged 11 v2-unsupported events, and that false PASS was read as evidence +/// that a relay-route fix had worked. Verify metric names against a real +/// scrape, never against the registration constant in CB's source. +pub fn check_v2_unsupported(scrape: &Scrape) -> CheckResult { + let id = "cb_relay_v2_unsupported"; let mut by_relay: BTreeMap = BTreeMap::new(); for s in &scrape.samples { - if s.metric != "cb_pbs_submit_block_v2_fallback_to_v1_total" { + if s.metric != V2_UNSUPPORTED_METRIC { continue; } let relay = s @@ -388,22 +600,47 @@ pub fn check_v2_fallback(scrape: &Scrape) -> CheckResult { let data = serde_json::json!({ "by_relay": by_relay, "total": total as u64 }); if total == 0.0 { - // Covers both "counter exists and equals 0" and "counter missing - // (never incremented)". Prometheus suppresses counter families with - // no observations, so missing == zero. - CheckResult::pass(id, 2, "No v2->v1 fallbacks (relays support v2)").with_data(data) + CheckResult::pass(id, 2, "No v2-unsupported submissions").with_data(data) } else { - CheckResult::warn( + let relays: Vec<&str> = by_relay.keys().map(|s| s.as_str()).collect(); + CheckResult::fail( id, 2, format!( - "{total:.0} v2 submits fell back to v1; at least one relay doesn't support submitBlindedBlockV2" + "{total:.0} v2 submit_block(s) LOST: relay(s) {} 404 the v2 route and CB will not downgrade to v1 (v2 = the relay publishes the block; a v1 payload would be silently dropped). Every builder block the proposer chose was lost -- expect missed slots. CHECK THE RELAY ROUTE CONFIG FIRST: helix has a GetPayloadV2 route that must be listed in router_config.enabled_routes -- a 404 here usually means v2 is merely DISABLED, not unsupported", + relays.join(", ") ), ) .with_data(data) } } +/// Check the v2 -> v1 fallback counter. +/// +/// **This check is INERT and reports SKIP.** It reads +/// `cb_pbs_submit_block_v2_fallback_to_v1_total`, and no such counter exists in +/// commit-boost: nothing named `*fallback*` is registered anywhere in +/// `crates/pbs/src/metrics.rs`. Because the check treated "counter absent" as +/// "zero fallbacks == PASS", it returned PASS on every run since it was written: +/// a check that cannot fail, which is worse than no check, and which also +/// claimed "relays support v2" on a run where the relay was 404ing v2. +/// +/// It is kept (rather than deleted) as a SKIP so the id stays in the report and +/// the reason travels with it. CB does NOT downgrade v2 -> v1 on a 404 by +/// design (v2 semantics: the relay publishes the block, so a v1 payload would +/// be silently dropped) - it fails loud and increments the v2-UNSUPPORTED +/// counter instead, which is what [`check_v2_unsupported`] reads. If a real +/// fallback counter ever lands, restore the logic and re-point the name. +pub fn check_v2_fallback(scrape: &Scrape) -> CheckResult { + let _ = scrape; + CheckResult::skip( + "cb_v2_fallback", + 2, + "inert: commit-boost registers no v2->v1 fallback counter, so this check could only ever \ + PASS. Relay v2 support is owned by cb_relay_v2_unsupported", + ) +} + /// Standard Prometheus-style histogram_quantile: find bucket where cumulative /// count >= q*total, linearly interpolate between lower-le and le. /// @@ -629,13 +866,18 @@ fn run_checks_on_scrape(scrape: &Scrape, strict: bool) -> Vec { .collect(); out.push(check_v2_fallback(scrape)); + out.push(check_v2_unsupported(scrape)); out.push(check_relay_latency(scrape, 500.0)); // Tier-1 escalation: any matrix FAIL (from 5xx) should fail the overall // run. The matrix checks are tier 2, but 5xx is a real pipeline failure - // -- escalate it to tier 1 so report::exit_code sees it. + // -- escalate it to tier 1 so report::exit_code sees it. The same applies + // to v2-unsupported: every builder block the proposer chose was LOST, which + // is at least as fatal as a 5xx. for c in out.iter_mut() { - if c.status == CheckStatus::Fail && c.id.ends_with("_matrix") { + if c.status == CheckStatus::Fail + && (c.id.ends_with("_matrix") || c.id == "cb_relay_v2_unsupported") + { c.tier = 1; } } @@ -647,11 +889,53 @@ fn run_checks_on_scrape(scrape: &Scrape, strict: bool) -> Vec { mod tests { use super::*; + /// Shorthand: the endpoint's own classifier via the public entry point. + fn classify_submit_blinded_block(s: &EndpointStats) -> CheckResult { + classify_endpoint("submit_blinded_block", s, false) + } + fn parse(text: &str) -> Scrape { let lines = text.lines().map(|l| Ok(l.to_owned())); Scrape::parse(lines).expect("valid prometheus text") } + /// Every commit-boost metric name these checks depend on, and how it is + /// derived. Audited end-to-end on 2026-08-04 after TWO checks were found + /// reading names that could never exist. + /// + /// The rule: CB builds its registries with `Registry::new_custom(Some(..))`, + /// which prefixes EVERY metric at gather time. Most PBS metrics are + /// registered bare (`relay_status_code_total` -> `cb_pbs_relay_status_code_total`), + /// but two carry their own prefix and therefore end up DOUBLED: + /// `cb_pbs` + `pbs_submit_block_v2_unsupported_total` + /// -> cb_pbs_pbs_submit_block_v2_unsupported_total + /// `cb_signer` + `signer_status_code_total` + /// -> cb_signer_signer_status_code_total (not read yet - + /// remember this if a signer metrics check is ever added) + /// + /// This test pins the names so an "obvious tidy-up" of the doubled prefix + /// breaks loudly instead of silently disabling a check. It cannot detect a + /// rename on CB's side - only a real scrape can, which is why the rule is: + /// verify metric names against a scrape, never against CB's source constant. + #[test] + fn metric_names_match_cb_exposed_names() { + assert_eq!( + V2_UNSUPPORTED_METRIC, "cb_pbs_pbs_submit_block_v2_unsupported_total", + "the doubled pbs_ is CORRECT: prefix cb_pbs + registered name pbs_submit_block_..." + ); + // The three read by collect_endpoint_stats / check_relay_latency. + for name in [ + "cb_pbs_relay_status_code_total", + "cb_pbs_beacon_node_status_code_total", + "cb_pbs_relay_latency", + ] { + assert!( + name.starts_with("cb_pbs_"), + "{name} must carry the registry prefix" + ); + } + } + #[test] fn bucket_code_categories() { assert_eq!(bucket_code("200"), "200"); @@ -663,6 +947,127 @@ mod tests { assert_eq!(bucket_code("502"), "5xx"); assert_eq!(bucket_code("201"), "other"); assert_eq!(bucket_code("garbage"), "other"); + // CB's synthetic client-timeout marker MUST NOT bucket as relay 5xx — + // that misattribution tier-1-failed a run whose relays served 0 errors. + assert_eq!(bucket_code("555"), "timeout"); + // CB's synthetic WS transport-error marker (PR-483 streaming): also + // client-observed, same misattribution class as 555. + assert_eq!(bucket_code("556"), "transport"); + // A real (unusual) HTTP 5xx that is neither synthetic code stays 5xx. + assert_eq!(bucket_code("550"), "5xx"); + assert_eq!(bucket_code("557"), "5xx"); + } + + // --- timeout (555) classification: the timing-games false-red fix -------- + + #[test] + fn classify_heavy_timeouts_warn_not_fail() { + // The live-captured timing-games shape (2026-08-03): healthy bids plus + // ~42% CB-deadline cancellations, ZERO relay-served 5xx. Old bucketing + // called this "47.5% relay 5xx" -> tier-1 FAIL. Must be WARN. + let mut s = EndpointStats::default(); + s.add_relay("r0", "200", 27.0); + s.add_relay("r0", "204", 4.0); + s.add_relay("r0", "400", 2.0); + s.add_relay("r0", "555", 17.0); + s.add_relay("r1", "200", 21.0); + s.add_relay("r1", "204", 2.0); + s.add_relay("r1", "400", 2.0); + s.add_relay("r1", "555", 25.0); + let r = classify_endpoint("get_header", &s, false); + assert_eq!(r.status, CheckStatus::Warn, "timeouts must not FAIL"); + assert!( + r.detail.contains("555"), + "detail names the marker: {}", + r.detail + ); + assert!(r.detail.contains("not relay-served")); + } + + #[test] + fn classify_heavy_timeouts_warn_even_under_strict() { + // --strict is zero-tolerance for PIPELINE errors; a 555 is CB's own + // deadline policy, so strict must not promote it to FAIL. + let mut s = EndpointStats::default(); + s.add_relay("r0", "200", 10.0); + s.add_relay("r0", "555", 30.0); + let r = classify_endpoint("get_header", &s, true); + assert_eq!(r.status, CheckStatus::Warn); + } + + #[test] + fn classify_few_timeouts_pass_with_note() { + // Below MAX_TIMEOUT_RATE: normal classification, timeout count noted. + let mut s = EndpointStats::default(); + s.add_relay("r0", "200", 90.0); + s.add_relay("r0", "555", 5.0); + let r = classify_endpoint("get_header", &s, false); + assert_eq!(r.status, CheckStatus::Pass); + assert!(r.detail.contains("timeout"), "detail: {}", r.detail); + } + + #[test] + fn classify_heavy_transport_errors_warn_not_fail() { + // A stream relay refusing connections every slot: heavy 556, zero real + // 5xx. Must WARN (relay death is the tier-1 relay checks' call), never + // FAIL, not even under --strict. + let mut s = EndpointStats::default(); + s.add_relay("r0", "200", 10.0); + s.add_relay("r0", "556", 30.0); + let r = classify_endpoint("get_header", &s, false); + assert_eq!(r.status, CheckStatus::Warn, "556 must not FAIL"); + assert!(r.detail.contains("556"), "detail names 556: {}", r.detail); + let strict = classify_endpoint("get_header", &s, true); + assert_eq!(strict.status, CheckStatus::Warn, "strict must not promote"); + } + + #[test] + fn mixed_555_and_556_aggregate_into_one_warn() { + // Timing-games over a flaky stream: both synthetic codes present. One + // WARN naming both counts, not two competing verdicts. + let mut s = EndpointStats::default(); + s.add_relay("r0", "200", 10.0); + s.add_relay("r0", "555", 10.0); + s.add_relay("r0", "556", 10.0); + let r = classify_endpoint("get_header", &s, false); + assert_eq!(r.status, CheckStatus::Warn); + assert!(r.detail.contains("555") && r.detail.contains("556")); + } + + #[test] + fn few_transport_errors_pass_with_note() { + let mut s = EndpointStats::default(); + s.add_relay("r0", "200", 90.0); + s.add_relay("r0", "556", 5.0); + let r = classify_endpoint("get_header", &s, false); + assert_eq!(r.status, CheckStatus::Pass); + assert!(r.detail.contains("556"), "detail: {}", r.detail); + } + + #[test] + fn real_5xx_still_fails_despite_heavy_transport_errors() { + // 556 excluded from the 5xx denominator, same as 555: a genuine relay + // error storm FAILs even amid heavy stream-transport failures. + let mut s = EndpointStats::default(); + s.add_relay("r0", "200", 4.0); + s.add_relay("r0", "500", 10.0); + s.add_relay("r0", "556", 90.0); + let r = classify_endpoint("get_header", &s, false); + assert_eq!(r.status, CheckStatus::Fail, "real 5xx must keep failing"); + } + + #[test] + fn real_5xx_still_fails_despite_heavy_timeouts() { + // Timeouts are excluded from the 5xx denominator, so a genuine relay + // error storm FAILs even when timeout polling dominates raw counts: + // 10 x 500 vs 14 completed responses = 71% > 25%, regardless of 90 x 555. + let mut s = EndpointStats::default(); + s.add_relay("r0", "200", 4.0); + s.add_relay("r0", "500", 10.0); + s.add_relay("r0", "555", 90.0); + let r = classify_endpoint("get_header", &s, false); + assert_eq!(r.status, CheckStatus::Fail, "real 5xx must keep failing"); + assert!(r.detail.contains("5xx")); } #[test] @@ -723,16 +1128,27 @@ cb_pbs_beacon_node_status_code_total{http_status_code="204",endpoint="get_header } #[test] - fn classify_get_header_5xx_always_fails() { - let mut s = EndpointStats::default(); - s.add_relay("r0", "200", 10.0); - s.add_relay("r0", "500", 1.0); + fn classify_5xx_transient_warns_but_strict_and_high_rate_fail() { + // H2: a few 5xx against many good requests (1/11 = 9% < 25%) is warmup + // noise → WARN (not the old FAIL), but --strict still FAILs. + let mut low = EndpointStats::default(); + low.add_relay("r0", "200", 10.0); + low.add_relay("r0", "500", 1.0); assert_eq!( - classify_endpoint("get_header", &s, false).status, + classify_endpoint("get_header", &low, false).status, + CheckStatus::Warn + ); + assert_eq!( + classify_endpoint("get_header", &low, true).status, CheckStatus::Fail ); + + // A materially broken relay (5/6 = 83% > 25%) FAILs even without --strict. + let mut high = EndpointStats::default(); + high.add_relay("r0", "200", 1.0); + high.add_relay("r0", "500", 5.0); assert_eq!( - classify_endpoint("get_header", &s, true).status, + classify_endpoint("get_header", &high, false).status, CheckStatus::Fail ); } @@ -766,22 +1182,114 @@ cb_pbs_beacon_node_status_code_total{http_status_code="204",endpoint="get_header } #[test] - fn classify_submit_blinded_block_zero_warn() { + fn classify_submit_blinded_block_no_submissions_warn() { + // Genuinely zero submissions: the proposer never chose a builder block. + // (204s are the "no bid" shape; no 4xx = nothing was ever submitted.) let mut s = EndpointStats::default(); - s.add_relay("r0", "400", 1.0); + s.add_relay("r0", "204", 1.0); let r = classify_endpoint("submit_blinded_block", &s, false); assert_eq!(r.status, CheckStatus::Warn); + assert!(r.detail.contains("never chose"), "detail: {}", r.detail); assert!(r.detail.contains("--strict")); } #[test] - fn classify_submit_blinded_block_zero_strict_fails() { + fn classify_submit_blinded_block_no_submissions_strict_fails() { let mut s = EndpointStats::default(); - s.add_relay("r0", "400", 1.0); + s.add_relay("r0", "204", 1.0); let r = classify_endpoint("submit_blinded_block", &s, true); assert_eq!(r.status, CheckStatus::Fail); } + // --- submit_blinded_block is judged on the BEACON side ------------------ + // Both fixtures below are REAL data from live runs with opposite outcomes, + // which is what makes this discriminator trustworthy rather than a guess. + + #[test] + fn multi_relay_losing_relay_errors_do_not_fail_a_healthy_run() { + // Measured on a 2-relay run where relay_1 wins every auction by design + // (divergent subsidies), so relay_0 cannot serve any payload it never + // won. Relay-side rate was 29.7% and FAILED a run that delivered 65/65 + // payloads, 100% MEV rate, 0 missed slots. + let mut s = EndpointStats::default(); + s.add_relay("mev_relay_0", "202", 1.0); + s.add_relay("mev_relay_0", "400", 219.0); + s.add_relay("mev_relay_0", "500", 185.0); + s.add_relay("mev_relay_1", "202", 219.0); + s.add_relay("mev_relay_1", "400", 1.0); + s.add_relay("mev_relay_1", "500", 1.0); + s.add_beacon("202", 220.0); + + let r = classify_submit_blinded_block(&s); + assert_eq!( + r.status, + CheckStatus::Pass, + "CB served the CL 220 times; losing-relay errors are expected: {}", + r.detail + ); + assert!(r.detail.contains("220 payload(s) served")); + assert!( + r.detail.contains("expected"), + "explains the relay-side noise" + ); + } + + #[test] + fn beacon_side_5xx_still_fails() { + // The nethermind+prysm shape: CB returned 502 to the CL 26 times, i.e. + // the proposer genuinely did not get its payload. Must FAIL - this is + // what proves the exemption did not blind the check. + let mut s = EndpointStats::default(); + s.add_relay("mev_relay_0", "400", 26.0); + s.add_beacon("502", 26.0); + + let r = classify_submit_blinded_block(&s); + assert_eq!(r.status, CheckStatus::Fail); + assert!(r.detail.contains("did NOT get its payload")); + } + + #[test] + fn beacon_side_no_deliveries_warns() { + let mut s = EndpointStats::default(); + s.add_beacon("404", 5.0); + let r = classify_submit_blinded_block(&s); + assert_eq!(r.status, CheckStatus::Warn); + assert!(r.detail.contains("never chose")); + } + + #[test] + fn without_beacon_samples_it_falls_back_to_the_relay_side() { + // Metrics can be partial; with no beacon-side data the old relay-side + // logic still applies rather than silently passing. + let mut s = EndpointStats::default(); + s.add_relay("mev_relay_0", "400", 26.0); + let r = classify_submit_blinded_block(&s); + assert_eq!(r.status, CheckStatus::Fail, "relay rejected everything"); + assert!(r.detail.contains("REJECTED")); + } + + #[test] + fn classify_submit_blinded_block_relay_rejected_is_not_never_chose() { + // The live nethermind+prysm shape: 26 blinded blocks submitted, ALL + // rejected 4xx by the relay. The old code called this "proposer never + // chose a builder block" — a wrong diagnosis pointing at the wrong + // component. Must FAIL and name the relay as the rejecter. + let mut s = EndpointStats::default(); + s.add_relay("mev_relay_0", "400", 26.0); + let r = classify_endpoint("submit_blinded_block", &s, false); + assert_eq!( + r.status, + CheckStatus::Fail, + "relay refusing every block is not a WARN" + ); + assert!(r.detail.contains("REJECTED"), "detail: {}", r.detail); + assert!( + !r.detail.contains("never chose"), + "must NOT misdiagnose as proposer-side: {}", + r.detail + ); + } + #[test] fn classify_register_validator_all_accepted_pass() { let mut s = EndpointStats::default(); @@ -835,30 +1343,66 @@ cb_pbs_beacon_node_status_code_total{http_status_code="204",endpoint="get_header } #[test] - fn v2_fallback_zero_passes() { - let text = r#"# HELP cb_pbs_submit_block_v2_fallback_to_v1_total x -# TYPE cb_pbs_submit_block_v2_fallback_to_v1_total counter -cb_pbs_submit_block_v2_fallback_to_v1_total{relay_id="r0"} 0 -"#; - let scrape = parse(text); - assert_eq!(check_v2_fallback(&scrape).status, CheckStatus::Pass); + fn v2_fallback_is_inert_and_says_so() { + // The metric it read does not exist in commit-boost, so the old logic + // could only ever return PASS. An always-green check is worse than no + // check: it also asserted "relays support v2" on a run where the relay + // was 404ing every v2 submission. + let r = check_v2_fallback(&parse("")); + assert_eq!(r.status, CheckStatus::Skip); + assert!(r.detail.contains("inert"), "detail: {}", r.detail); + assert!( + r.detail.contains("cb_relay_v2_unsupported"), + "must point at the check that actually owns v2 support" + ); } #[test] - fn v2_fallback_nonzero_warns() { - let text = r#"# HELP cb_pbs_submit_block_v2_fallback_to_v1_total x -# TYPE cb_pbs_submit_block_v2_fallback_to_v1_total counter -cb_pbs_submit_block_v2_fallback_to_v1_total{relay_id="r0"} 5 + fn v2_unsupported_nonzero_fails_and_names_the_relay() { + // The live nethermind+prysm shape: prysm submits via v2, helix 404s the + // v2 route, CB refuses to downgrade -> every builder block is lost. + let text = r#"# HELP cb_pbs_pbs_submit_block_v2_unsupported_total x +# TYPE cb_pbs_pbs_submit_block_v2_unsupported_total counter +cb_pbs_pbs_submit_block_v2_unsupported_total{relay_id="mev_relay_0"} 11 "#; let scrape = parse(text); - assert_eq!(check_v2_fallback(&scrape).status, CheckStatus::Warn); + let r = check_v2_unsupported(&scrape); + assert_eq!( + r.status, + CheckStatus::Fail, + "lost submissions are not a WARN" + ); + assert!( + r.detail.contains("mev_relay_0"), + "names the relay: {}", + r.detail + ); + assert_eq!(r.data["total"], 11); + } + + #[test] + fn v2_unsupported_missing_counter_passes() { + // Prometheus omits never-incremented families; absence == zero == fine. + assert_eq!(check_v2_unsupported(&parse("")).status, CheckStatus::Pass); } #[test] - fn v2_fallback_missing_counter_passes() { - // Missing counter == never incremented == zero fallbacks == PASS. - let scrape = parse(""); - assert_eq!(check_v2_fallback(&scrape).status, CheckStatus::Pass); + fn v2_unsupported_escalates_to_tier1_on_fail() { + // A relay that cannot serve the proposer's submissions must gate the + // exit code, like a matrix 5xx. + let text = r#"# HELP cb_pbs_pbs_submit_block_v2_unsupported_total x +# TYPE cb_pbs_pbs_submit_block_v2_unsupported_total counter +cb_pbs_pbs_submit_block_v2_unsupported_total{relay_id="mev_relay_0"} 3 +"#; + let mut c = check_v2_unsupported(&parse(text)); + assert_eq!(c.tier, 2, "authored at tier 2"); + // Mirror run_metrics_checks' escalation rule. + if c.status == CheckStatus::Fail + && (c.id.ends_with("_matrix") || c.id == "cb_relay_v2_unsupported") + { + c.tier = 1; + } + assert_eq!(c.tier, 1, "must escalate so the run fails"); } #[test] diff --git a/src/checks/chain_health.rs b/src/checks/chain_health.rs index 592a960..70657e9 100644 --- a/src/checks/chain_health.rs +++ b/src/checks/chain_health.rs @@ -2,22 +2,42 @@ use std::process::Command; +use futures::StreamExt; + use crate::beacon::BeaconClient; use crate::checks::CheckResult; -/// Check if the beacon chain has finalized past epoch 2. -pub async fn check_finality(beacon: &BeaconClient) -> CheckResult { - match beacon.get_finalized_epoch().await { - Ok(epoch) if epoch >= 2 => { - CheckResult::pass("chain_finality", 1, format!("Finalized epoch: {epoch}")) - .with_data(serde_json::json!({ "finalized_epoch": epoch })) - } - Ok(epoch) => CheckResult::fail( +/// Classify a finalized epoch: >= 2 PASSes, anything lower FAILs. +/// +/// Pure decision core extracted from [`check_finality`] so it can be unit +/// tested without a live beacon (the P3 Law-4 pattern — see +/// `cb_metrics::classify_endpoint`). +pub fn classify_finality(finalized_epoch: u64) -> CheckResult { + let data = serde_json::json!({ "finalized_epoch": finalized_epoch }); + if finalized_epoch >= 2 { + CheckResult::pass( + "chain_finality", + 1, + format!("Finalized epoch: {finalized_epoch}"), + ) + .with_data(data) + } else { + CheckResult::fail( "chain_finality", 1, - format!("Finalized epoch too low: {epoch} (need >= 2)"), + format!("Finalized epoch too low: {finalized_epoch} (need >= 2)"), ) - .with_data(serde_json::json!({ "finalized_epoch": epoch })), + .with_data(data) + } +} + +/// Check if the beacon chain has finalized past epoch 2. +/// +/// Thin IO wrapper: fetches the finalized epoch, then defers to +/// [`classify_finality`]. +pub async fn check_finality(beacon: &BeaconClient) -> CheckResult { + match beacon.get_finalized_epoch().await { + Ok(epoch) => classify_finality(epoch), Err(e) => CheckResult::fail("chain_finality", 1, format!("Error checking finality: {e}")), } } @@ -37,28 +57,55 @@ pub async fn check_missed_slots( return CheckResult::skip( "missed_slots", 2, - format!("Single-slot window (slot {}), skipping missed slot check", start_slot), + format!( + "Single-slot window (slot {}), skipping missed slot check", + start_slot + ), ); } let total = end_slot - start_slot; + // Gather the per-slot headers concurrently (bounded). The miss-count fold is + // identical to the old serial loop — `missed` is an order-independent counter, + // so buffer_unordered's out-of-order completion does not change the result. + let fetched: Vec<_> = futures::stream::iter(start_slot..end_slot) + .map(|slot| async move { (slot, beacon.get_header(slot).await) }) + .buffer_unordered(16) + .collect() + .await; + let mut missed = 0u64; - for slot in start_slot..end_slot { - match beacon.get_header(slot).await { + for (_slot, res) in fetched { + match res { Ok(None) => missed += 1, Err(_) => missed += 1, Ok(Some(_)) => {} } } + // Attach the slot bounds to the classifier's data payload (they're context + // the pure rate logic doesn't need to reach a verdict, but the report does). + let mut result = classify_missed_slots(missed, total, threshold); + if let Some(obj) = result.data.as_object_mut() { + obj.insert("start_slot".to_string(), serde_json::json!(start_slot)); + obj.insert("end_slot".to_string(), serde_json::json!(end_slot)); + } + result +} + +/// Classify a miss rate: strictly BELOW `threshold` PASSes, at-or-above WARNs. +/// +/// Pure decision core extracted from [`check_missed_slots`] (the boundary is +/// intentionally strict — `rate < threshold` — so a rate exactly at the +/// threshold warns). Callers guarantee `total > 0` (the inverted-range and +/// single-slot windows are handled upstream before any measurement). +pub fn classify_missed_slots(missed: u64, total: u64, threshold: f64) -> CheckResult { let rate = missed as f64 / total as f64; let data = serde_json::json!({ "missed": missed, "total": total, "rate": (rate * 10000.0).round() / 10000.0, "threshold": threshold, - "start_slot": start_slot, - "end_slot": end_slot, }); if rate < threshold { @@ -119,24 +166,33 @@ pub fn check_cb_running(enclave: &str, service_pattern: &str) -> CheckResult { } let stdout = String::from_utf8_lossy(&output.stdout); + classify_cb_running(&stdout, service_pattern) +} + +/// Classify `kurtosis enclave inspect` stdout for a service pattern. +/// +/// Pure string logic extracted from [`check_cb_running`]: +/// - at least one matching line marked `running` => PASS +/// - matching line(s) exist but none running => FAIL +/// - no matching line at all => FAIL +/// +/// Case-insensitive on both the pattern and the `running` marker. +pub fn classify_cb_running(inspect_stdout: &str, service_pattern: &str) -> CheckResult { let pat_lc = service_pattern.to_lowercase(); - let cb_lines: Vec<&str> = stdout + let cb_lines: Vec<&str> = inspect_stdout .lines() .filter(|l| l.to_lowercase().contains(&pat_lc)) .collect(); - let running: Vec<&&str> = cb_lines + let running = cb_lines .iter() .filter(|l| l.to_lowercase().contains("running")) - .collect(); + .count(); - if !running.is_empty() { + if running > 0 { CheckResult::pass( "cb_running", 1, - format!( - "Found {} {service_pattern} service(s) running", - running.len() - ), + format!("Found {running} {service_pattern} service(s) running"), ) } else if !cb_lines.is_empty() { CheckResult::fail( @@ -151,7 +207,7 @@ pub fn check_cb_running(enclave: &str, service_pattern: &str) -> CheckResult { CheckResult::fail( "cb_running", 1, - format!("No {service_pattern} services found in enclave '{enclave}'"), + format!("No {service_pattern} services found"), ) } } @@ -159,7 +215,8 @@ pub fn check_cb_running(enclave: &str, service_pattern: &str) -> CheckResult { /// Run all chain health checks. /// /// Finalization check (`chain_finality`) is included only when: -/// - `end_slot >= 96` (epoch 3+ — justification cascade has time to finalize epoch 2) +/// - `end_slot >= 160` (epoch 5 — first slot where `finalized_epoch >= 2` is +/// reliably reached; see [`classify_finality`] for the >= 2 threshold) /// - `skip_finalization` is `false` /// /// Otherwise the check is skipped with a reason. @@ -170,10 +227,15 @@ pub async fn run_chain_health_checks( enclave: &str, skip_finalization: bool, ) -> Vec { - // Finalization needs ~3 epochs from genesis for the justification - // cascade to finalize epoch 2. Skip if the observation window ends - // before slot 96 (end of epoch 3). - const FINALITY_POSSIBLE_AFTER_SLOT: u64 = 96; + // The finality check demands `finalized_epoch >= 2`, but finality lags the + // chain head by ~2 epochs: epoch N justifies at N+1 and finalizes at N+2. + // So epoch 2 does not finalize until the END of epoch 4 (~slot 160), which + // is the first slot where `finalized_epoch >= 2` is reliably reached. + // Gating on the old value 96 (end of epoch 3, where only epoch 1 has + // finalized) ran the >= 2 check too early and FAILed healthy chains whose + // window ended in ~[96, 160). Skip until epoch 5 (5 * 32) so the demanded + // finalization has actually had time to happen. + const FINALITY_POSSIBLE_AFTER_SLOT: u64 = 5 * 32; let mut checks = vec![ check_missed_slots(beacon, start_slot, end_slot, 0.10).await, @@ -207,3 +269,157 @@ pub async fn run_chain_health_checks( checks } + +#[cfg(test)] +mod tests { + use super::*; + use crate::checks::CheckStatus; + + // A BeaconClient constructor is pure (it only builds a reqwest::Client, no + // I/O). The check_missed_slots guard branches below return BEFORE any await, + // so this client is never used for a request — the tests exercise pure + // decision logic, no devnet required. + fn dummy_beacon() -> BeaconClient { + BeaconClient::new("http://127.0.0.1:0") + } + + // Contract: an inverted slot range (start > end) is nonsense input and must + // FAIL fast without consulting the beacon. + #[tokio::test] + async fn missed_slots_inverted_range_fails() { + let beacon = dummy_beacon(); + let r = check_missed_slots(&beacon, 10, 5, 0.10).await; + assert_eq!(r.status, CheckStatus::Fail); + assert_eq!(r.id, "missed_slots"); + assert_eq!(r.tier, 2); + } + + // Contract: a single-slot window (start == end) has no interior slots to + // measure a miss rate over, so it must SKIP (not silently PASS on zero + // data — that would be a false green). + #[tokio::test] + async fn missed_slots_single_slot_window_skips() { + let beacon = dummy_beacon(); + let r = check_missed_slots(&beacon, 5, 5, 0.10).await; + assert_eq!(r.status, CheckStatus::Skip); + assert_eq!(r.id, "missed_slots"); + assert_eq!(r.tier, 2); + assert!(r.detail.contains("Single-slot")); + } + + // --- classify_finality (seam 1) ------------------------------------- + + // Contract: finalized epoch >= 2 is the healthy state and PASSes. + #[test] + fn finality_at_threshold_passes() { + let r = classify_finality(2); + assert_eq!(r.status, CheckStatus::Pass); + assert_eq!(r.id, "chain_finality"); + assert_eq!(r.tier, 1); + assert_eq!(r.data["finalized_epoch"], 2); + } + + #[test] + fn finality_above_threshold_passes() { + let r = classify_finality(7); + assert_eq!(r.status, CheckStatus::Pass); + assert_eq!(r.data["finalized_epoch"], 7); + } + + // Contract: finalized epoch < 2 FAILs, and the epoch is surfaced in data. + #[test] + fn finality_below_threshold_fails() { + let r = classify_finality(1); + assert_eq!(r.status, CheckStatus::Fail); + assert_eq!(r.id, "chain_finality"); + assert!(r.detail.contains("too low")); + assert_eq!(r.data["finalized_epoch"], 1); + } + + #[test] + fn finality_zero_fails() { + let r = classify_finality(0); + assert_eq!(r.status, CheckStatus::Fail); + assert_eq!(r.data["finalized_epoch"], 0); + } + + // --- classify_missed_slots (seam 2) --------------------------------- + + // Contract: a miss rate strictly BELOW threshold PASSes. + #[test] + fn missed_slots_below_threshold_passes() { + // 5/100 = 5% < 10% + let r = classify_missed_slots(5, 100, 0.10); + assert_eq!(r.status, CheckStatus::Pass); + assert_eq!(r.id, "missed_slots"); + assert_eq!(r.tier, 2); + assert_eq!(r.data["missed"], 5); + assert_eq!(r.data["total"], 100); + } + + // Contract: a miss rate exactly AT threshold is not "under" it, so it WARNs + // (the boundary belongs to the warn side — `rate < threshold` is strict). + #[test] + fn missed_slots_at_threshold_warns() { + // 10/100 = 10% == 10% + let r = classify_missed_slots(10, 100, 0.10); + assert_eq!(r.status, CheckStatus::Warn); + assert!(r.detail.contains("above")); + } + + // Contract: a miss rate ABOVE threshold WARNs. + #[test] + fn missed_slots_above_threshold_warns() { + // 25/100 = 25% > 10% + let r = classify_missed_slots(25, 100, 0.10); + assert_eq!(r.status, CheckStatus::Warn); + assert_eq!(r.data["missed"], 25); + } + + // --- classify_cb_running (seam 3) ----------------------------------- + + // A minimal `kurtosis enclave inspect` snippet with a running CB service. + const INSPECT_RUNNING: &str = "\ +========================================== User Services ========================================== +UUID Name Ports Status +abc123 cl-1-lighthouse-geth http: 4000/tcp -> ... RUNNING +def456 commit-boost-pbs api: 18550/tcp -> ... RUNNING +"; + + const INSPECT_STOPPED: &str = "\ +UUID Name Ports Status +abc123 cl-1-lighthouse-geth http: 4000/tcp -> ... RUNNING +def456 commit-boost-pbs api: 18550/tcp -> ... STOPPED +"; + + const INSPECT_NONE: &str = "\ +UUID Name Ports Status +abc123 cl-1-lighthouse-geth http: 4000/tcp -> ... RUNNING +"; + + // Contract: at least one matching line marked RUNNING => PASS. + #[test] + fn cb_running_present_passes() { + let r = classify_cb_running(INSPECT_RUNNING, "commit-boost"); + assert_eq!(r.status, CheckStatus::Pass); + assert_eq!(r.id, "cb_running"); + assert_eq!(r.tier, 1); + assert!(r.detail.contains("running")); + } + + // Contract: a matching service exists but none is RUNNING => FAIL. + #[test] + fn cb_running_found_but_stopped_fails() { + let r = classify_cb_running(INSPECT_STOPPED, "commit-boost"); + assert_eq!(r.status, CheckStatus::Fail); + assert!(r.detail.contains("none running")); + } + + // Contract: no matching service line at all => FAIL. + #[test] + fn cb_running_none_found_fails() { + let r = classify_cb_running(INSPECT_NONE, "commit-boost"); + assert_eq!(r.status, CheckStatus::Fail); + assert!(r.detail.contains("No")); + } +} diff --git a/src/checks/feature_fired.rs b/src/checks/feature_fired.rs new file mode 100644 index 0000000..6d8a7c2 --- /dev/null +++ b/src/checks/feature_fired.rs @@ -0,0 +1,792 @@ +//! Feature-fired assertions (Law 3): prove a toggled Commit-Boost feature's +//! codepath actually EXECUTED at runtime, not merely that the generic health +//! checks passed. Without these, the skip-sigverify / extra-validation / +//! timing-games scenarios are non-tests — they enable a feature and then only +//! assert the same things cb-basic does. +//! +//! The proof is CB debug logs (the scenarios all set `[logs.stdout] level = +//! "debug"`). Each feature that leaves a unique log marker gets a positive +//! assertion; `skip_sigverify` leaves NO positive trace on the happy path (it +//! is a *negative* codepath — a function simply not called, with no success log +//! or metric), so it is honestly reported as un-verifiable at runtime rather +//! than falsely green. See the per-variant docs on [`Feature`]. +//! +//! Shape mirrors `mux_routing`: detect the feature from the CB config, fetch CB +//! logs, look for the marker; a missing marker WARNs (could be no getHeader in +//! the window) rather than FAILs — the same no-false-red discipline as the mux +//! check. The verdict logic is a pure seam (`classify_*`) for unit testing. + +use crate::checks::CheckResult; +use crate::checks::mux_routing::fetch_filtered_logs; +use tracing::warn; + +/// A CB feature whose activation we try to confirm fired at runtime. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Feature { + /// `enable_timing_games = true` (per-relay). Delays/repeats getHeader; emits + /// DEBUG logs prefixed `TG:` on no other codepath. + TimingGames, + /// `extra_validation_enabled = true` (`[pbs]`). Fetches the parent block to + /// validate the header; emits `"fetching parent block"` / `"fetched parent + /// block"` DEBUG logs on no other codepath. + ExtraValidation, + /// `get_header = "stream"` (per-relay). getHeader rides a websocket bid + /// stream instead of HTTP polling; CB logs + /// `"received new header from ws stream"` on no other codepath. The HTTP + /// FALLBACK (merged with the feature) means a broken stream still passes + /// every MEV check silently - this marker check is the only discriminator. + WsHeaderStream, + /// `skip_sigverify = true` (`[pbs]`). A negative codepath: signature + /// verification is simply not called, with NO success log or metric. On the + /// happy path (valid mock-relay signatures) ON is indistinguishable from OFF + /// in the logs — so this cannot be positively confirmed without a + /// bad-signature-injecting relay. Reported honestly, never falsely green. + SkipSigverify, +} + +/// Every feature we know how to check, in report order. +pub const ALL_FEATURES: [Feature; 4] = [ + Feature::TimingGames, + Feature::ExtraValidation, + Feature::WsHeaderStream, + Feature::SkipSigverify, +]; + +impl Feature { + /// The check id emitted for this feature. + pub fn id(self) -> &'static str { + match self { + Feature::TimingGames => "feature.timing_games", + Feature::ExtraValidation => "feature.extra_validation", + Feature::WsHeaderStream => "feature.ws_header_stream", + Feature::SkipSigverify => "feature.skip_sigverify", + } + } + + /// The boolean config key that enables the feature. + pub fn config_key(self) -> &'static str { + match self { + Feature::TimingGames => "enable_timing_games", + Feature::ExtraValidation => "extra_validation_enabled", + Feature::WsHeaderStream => "get_header", + Feature::SkipSigverify => "skip_sigverify", + } + } + + /// The config VALUE that arms the feature. Most features are boolean + /// toggles; `get_header` is a string transport selector. + pub fn config_value(self) -> &'static str { + match self { + Feature::WsHeaderStream => "stream", + _ => "true", + } + } + + /// CB log-line markers that PROVE the codepath fired. Empty when no positive + /// runtime proof exists (`skip_sigverify`). + fn proof_markers(self) -> &'static [&'static str] { + match self { + Feature::TimingGames => &["TG:"], + Feature::ExtraValidation => &["fetched parent block", "fetching parent block"], + Feature::WsHeaderStream => &[WS_STREAM_MARKER], + Feature::SkipSigverify => &[], + } + } + + /// Human name used in check details. + fn label(self) -> &'static str { + match self { + Feature::TimingGames => "timing games", + Feature::ExtraValidation => "extra validation", + Feature::WsHeaderStream => "ws header stream", + Feature::SkipSigverify => "skip sigverify", + } + } +} + +/// Detect which features a CB config template enables (pure). Scans for a +/// ` = true` line, ignoring comments. +pub fn detect_enabled_features(template: &str) -> Vec { + ALL_FEATURES + .into_iter() + .filter(|f| config_enables(template, f.config_key(), f.config_value())) + .collect() +} + +/// True iff `template` has an uncommented `key = ` line. +fn config_enables(template: &str, key: &str, value: &str) -> bool { + template.lines().any(|line| { + let t = line.trim(); + if t.starts_with('#') { + return false; + } + match t.split_once('=') { + Some((k, v)) => k.trim() == key && v.trim().trim_matches('"') == value, + None => false, + } + }) +} + +/// CB's proof line for a bid received over the websocket stream — a stream +/// header can ONLY exist because the relay delivered it, so this single marker +/// proves the full relay->CB stream path (no separate relay-side check needed). +pub const WS_STREAM_MARKER: &str = "received new header from ws stream"; +/// CB's warn line when a stream attempt degrades to HTTP. +pub const WS_FALLBACK_MARKER: &str = "falling back to http get_header"; + +/// Pure verdict for the stream-vs-fallback balance (Law 4 seam). +/// +/// The HTTP fallback makes a broken stream invisible to every MEV check, so +/// the COUNT is the signal, not delivery. Thresholds are measured, not +/// guessed: a healthy 220-slot run (CB e622a5e + helix :main) +/// showed exactly ONE fallback — the first slot's registration-TOFU race +/// ("proposer not registered"), gone 12s later. +/// +/// - 0 fallbacks -> PASS +/// - 1 fallback, stream served -> PASS (the startup race; count reported) +/// - >1 fallback, stream served -> WARN, degraded stream +/// - fallbacks, stream NEVER served -> WARN (annotative only: the marker check +/// already carries `inconclusive` for the armed-but-unproven feature, so the +/// red under --require-feature-proof comes from there, not double-flagged) +pub fn classify_ws_fallback(streamed: usize, fallbacks: usize) -> CheckResult { + let id = "feature.ws_stream_fallback"; + let data = serde_json::json!({ + "streamed_headers": streamed, + "fallbacks": fallbacks, + "fallback_marker": WS_FALLBACK_MARKER, + }); + match (streamed, fallbacks) { + (_, 0) => CheckResult::pass(id, 2, "stream transport: zero HTTP fallbacks ✓"), + (s, 1) if s > 0 => CheckResult::pass( + id, + 2, + format!( + "stream served {s} header(s) with 1 HTTP fallback (the startup registration race; expected)" + ), + ), + (s, f) if s > 0 => CheckResult::warn( + id, + 2, + format!( + "stream DEGRADED: {f} HTTP fallbacks alongside {s} streamed header(s) - the stream is flapping; MEV checks stay green via the fallback, so this count is the only signal" + ), + ), + (_, f) => CheckResult::warn( + id, + 2, + format!( + "stream NEVER served: all getHeader traffic degraded to HTTP ({f} fallback warn(s)). The feature.ws_header_stream check carries the inconclusive flag for this run" + ), + ), + } + .with_data(data) +} + +/// Pure verdict for a log-marker feature (timing-games / extra-validation). +/// `proof_count` = CB log lines matching the feature's proof markers. +/// +/// - proof_count > 0 → PASS (the codepath demonstrably fired) +/// - proof_count == 0 → WARN (feature enabled in config but no proof marker +/// seen — the codepath may not have run, or no getHeader landed in the +/// window, or debug logging is off). NOT a FAIL: no-false-red. +pub fn classify_marker_feature(feature: Feature, proof_count: usize) -> CheckResult { + let id = feature.id(); + let label = feature.label(); + let data = serde_json::json!({ + "feature": label, + "config_key": feature.config_key(), + "proof_markers": feature.proof_markers(), + "proof_lines_seen": proof_count, + }); + if proof_count > 0 { + CheckResult::pass( + id, + 1, + format!( + "{label} fired ✓ {proof_count} matching CB debug log line(s) prove the codepath ran" + ), + ) + .with_data(data) + } else { + CheckResult::warn( + id, + 1, + format!( + "{label} is enabled in the CB config but ZERO proof markers ({:?}) were seen in \ + CB debug logs — the codepath may not have fired (no getHeader in window?) or \ + `[logs.stdout] level = \"debug\"` is off. NOT asserting the feature ran.", + feature.proof_markers() + ), + ) + .with_data(data) + .mark_inconclusive() + } +} + +/// The helix relay's signing pubkey (`DEFAULT_MEV_PUBKEY` in the +/// ethereum-package fork — a fixed constant of the devnet topology). A CB +/// `[[relays]]` url carrying any OTHER pubkey is the sigverify-differential +/// fault injection: CB's validate_signature would reject every bid from the +/// real relay, so a bid winning the auction proves the skip fired. +const HELIX_RELAY_PUBKEY: &str = "0xa55c1285d84ba83a5ad26420cd5ad3091e49c55a813eee651cd467db38a8c8e63192f47955e9376f6b42f6d190571cb5"; + +/// Detect the fault injection (pure): does any `[[relays]]` url in the CB +/// config template carry a pubkey that is NOT the helix relay's signing key? +pub fn has_poisoned_relay_pubkey(template: &str) -> bool { + let helix = HELIX_RELAY_PUBKEY.to_lowercase(); + template.lines().any(|line| { + let t = line.trim(); + let Some(rest) = t.strip_prefix("url = ") else { + return false; + }; + let url = rest.trim_matches('"'); + // scheme://@host — extract the userinfo if present. + let Some((_, after_scheme)) = url.split_once("://") else { + return false; + }; + match after_scheme.split_once('@') { + Some((pubkey, _)) => pubkey.to_lowercase() != helix, + None => false, + } + }) +} + +/// Pure verdict for `skip_sigverify` (Law 4 seam). +/// +/// Without the fault injection (`poisoned = false`) the feature is a negative +/// codepath with no positive runtime signal — honest WARN, never a false green. +/// +/// With a poisoned relay pubkey in the CB config, the differential becomes +/// real: CB's validate_signature would reject every bid from the real relay +/// (PubkeyMismatch), so `auction_winners > 0` is positive proof the skip +/// codepath fired — with sigverify on, zero bids could have won. +pub fn classify_skip_sigverify(poisoned: bool, auction_winners: usize) -> CheckResult { + let id = Feature::SkipSigverify.id(); + if !poisoned { + return CheckResult::warn( + id, + 1, + "skip_sigverify is enabled, but it is a negative codepath (signature verification is \ + simply not called) that emits no success log or metric. On the happy path (valid \ + relay signatures) it is indistinguishable from OFF, so it cannot be positively \ + confirmed at runtime. Run the cb-sigverify-diff scenario (wrong-pubkey relay url) \ + for a real differential. Not asserting either way.", + ) + .with_data(serde_json::json!({ + "feature": "skip sigverify", + "config_key": "skip_sigverify", + "verifiable_at_runtime": false, + "poisoned_relay": false, + })); + } + + let data = serde_json::json!({ + "feature": "skip sigverify", + "config_key": "skip_sigverify", + "verifiable_at_runtime": true, + "poisoned_relay": true, + "auction_winners": auction_winners, + }); + if auction_winners > 0 { + CheckResult::pass( + id, + 1, + format!( + "skip_sigverify fired ✓ {auction_winners} auction winner(s) despite a \ + wrong-pubkey relay url — with signature verification ON every bid would have \ + been rejected (PubkeyMismatch), so bids winning proves the skip codepath ran" + ), + ) + .with_data(data) + } else { + CheckResult::warn( + id, + 1, + "skip_sigverify enabled with the wrong-pubkey relay url (differential armed) but \ + ZERO auction winners observed — cannot distinguish 'skip did not fire' from 'no \ + bids in the window'. NOT asserting the feature ran.", + ) + .with_data(data) + .mark_inconclusive() + } +} + +/// CB's rejection marker when a bid is under `min_bid_eth` +/// (`ValidationError::BidTooLow`, rendered by its `thiserror` Display and +/// surfaced by the `error!(%err, relay_id)` at the get_header call site). +const BID_TOO_LOW_MARKER: &str = "bid below minimum"; + +/// Parse `min_bid_eth = ` out of the CB config template (pure). Returns the +/// floor in ETH, or `None` when the key is absent or zero (zero = no floor). +pub fn detect_min_bid_eth(template: &str) -> Option { + template.lines().find_map(|line| { + let t = line.trim(); + if t.starts_with('#') { + return None; + } + let (k, v) = t.split_once('=')?; + if k.trim() != "min_bid_eth" { + return None; + } + let val: f64 = v.trim().trim_matches('"').parse().ok()?; + (val > 0.0).then_some(val) + }) +} + +/// Pure verdict for the `min_bid_eth` floor (Law 4 seam). +/// +/// `rejections` = CB log lines carrying [`BID_TOO_LOW_MARKER`]. +/// `winner_values_eth` = the `value_eth` of every `auction winner` line. +/// +/// The definitive falsifier is a WINNER BELOW THE FLOOR: that can only happen +/// if the floor was not applied, which is the failure mode that matters here. +/// `[pbs]` has no `deny_unknown_fields` (it must `#[serde(flatten)]`), so a +/// renamed or misspelled key is SILENTLY IGNORED rather than rejected - this +/// check is the canary for that whole class. +/// +/// Absence of rejections is NOT a failure on its own: every bid legitimately +/// clearing the floor looks the same as a dropped key, so that is a WARN naming +/// both possibilities rather than a false red. +pub fn classify_min_bid( + floor_eth: f64, + rejections: usize, + winner_values_eth: &[f64], +) -> CheckResult { + let id = "feature.min_bid"; + let below: Vec = winner_values_eth + .iter() + .copied() + .filter(|v| *v < floor_eth) + .collect(); + let data = serde_json::json!({ + "feature": "min bid", + "config_key": "min_bid_eth", + "floor_eth": floor_eth, + "rejections": rejections, + "auction_winners": winner_values_eth.len(), + "winners_below_floor": below.len(), + }); + + if !below.is_empty() { + return CheckResult::fail( + id, + 1, + format!( + "{} auction winner(s) had a value BELOW the {floor_eth} ETH floor (lowest {:.6}) -- min_bid_eth was not applied. `[pbs]` silently ignores unknown keys, so suspect a renamed/misspelled field before suspecting CB", + below.len(), + below.iter().cloned().fold(f64::INFINITY, f64::min) + ), + ) + .with_data(data); + } + if rejections > 0 { + return CheckResult::pass( + id, + 1, + format!( + "min_bid_eth enforced ✓ {rejections} bid(s) rejected below the {floor_eth} ETH floor, and no winner was under it" + ), + ) + .with_data(data); + } + CheckResult::warn( + id, + 1, + format!( + "min_bid_eth = {floor_eth} is set but ZERO bids were rejected -- cannot distinguish 'the floor was silently ignored' from 'every bid legitimately cleared it' (is the builder subsidy raising bids above the floor?). NOT asserting the floor was applied" + ), + ) + .with_data(data) + .mark_inconclusive() +} + +/// Run the feature-fired checks for every feature the CB config enables. +/// +/// `template` is the CB config TOML (via `mux_routing::read_cb_config_template`). +/// Emits one CheckResult per enabled feature; features not enabled are silent. +pub async fn run_feature_checks( + enclave: &str, + cb_service_names: &[String], + template: &str, +) -> Vec { + let mut out = Vec::new(); + for feature in detect_enabled_features(template) { + out.push(check_one(enclave, cb_service_names, feature, template).await); + } + // The stream/fallback balance is a COUNT comparison, not a marker check, + // so it sits beside the per-feature loop. + if detect_enabled_features(template).contains(&Feature::WsHeaderStream) { + let streamed = count_log_lines(enclave, cb_service_names, &[WS_STREAM_MARKER]); + let fallbacks = count_log_lines(enclave, cb_service_names, &[WS_FALLBACK_MARKER]); + out.push(classify_ws_fallback(streamed, fallbacks)); + } + // min_bid_eth is a VALUE knob, not a boolean toggle, so it sits outside the + // Feature enum; it is only emitted when a floor is actually configured. + if let Some(floor) = detect_min_bid_eth(template) { + let rejections = count_log_lines(enclave, cb_service_names, &[BID_TOO_LOW_MARKER]); + let winners = auction_winner_values_eth(enclave, cb_service_names); + out.push(classify_min_bid(floor, rejections, &winners)); + } + out +} + +/// Collect the `value_eth` of every `auction winner` CB logged, as f64 ETH. +/// Lines whose value will not parse are skipped rather than failing the check. +fn auction_winner_values_eth(enclave: &str, cb_service_names: &[String]) -> Vec { + let mut out = Vec::new(); + for service in cb_service_names { + let Ok(logs) = fetch_filtered_logs(enclave, service, &["auction winner"]) else { + continue; + }; + for line in logs.lines() { + if let Some(ev) = crate::checks::mux_routing::parse_cb_log_line(line) + && ev.message.starts_with("auction winner") + && let Some(v) = ev + .fields + .get("value_eth") + .and_then(|v| v.parse::().ok()) + { + out.push(v); + } + } + } + out +} + +async fn check_one( + enclave: &str, + cb_service_names: &[String], + feature: Feature, + template: &str, +) -> CheckResult { + if feature == Feature::SkipSigverify { + let poisoned = has_poisoned_relay_pubkey(template); + // An auction winner is a bid that SURVIVED validation (CB only logs + // "auction winner" for responses that made it out of validation), so + // with a poisoned relay pubkey it is the positive skip-fired signal. + let winners = if poisoned { + count_log_lines(enclave, cb_service_names, &["auction winner"]) + } else { + 0 + }; + return classify_skip_sigverify(poisoned, winners); + } + + let proof_count = count_log_lines(enclave, cb_service_names, feature.proof_markers()); + classify_marker_feature(feature, proof_count) +} + +/// Count non-empty CB log lines matching any of `keywords` across services. +fn count_log_lines(enclave: &str, cb_service_names: &[String], keywords: &[&str]) -> usize { + let mut count = 0usize; + for service in cb_service_names { + match fetch_filtered_logs(enclave, service, keywords) { + Ok(logs) => { + count += logs.lines().filter(|l| !l.trim().is_empty()).count(); + } + Err(e) => warn!("feature check: failed to fetch logs from '{service}': {e}"), + } + } + count +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::checks::CheckStatus; + + #[test] + fn detects_each_feature_from_its_key() { + assert_eq!( + detect_enabled_features("[pbs]\nskip_sigverify = true\n"), + vec![Feature::SkipSigverify] + ); + assert_eq!( + detect_enabled_features("extra_validation_enabled = true\nrpc_url = \"x\"\n"), + vec![Feature::ExtraValidation] + ); + assert_eq!( + detect_enabled_features("enable_timing_games = true\n"), + vec![Feature::TimingGames] + ); + } + + #[test] + fn detects_multiple_features_in_report_order() { + let template = "enable_timing_games = true\nskip_sigverify = true\n"; + // ALL_FEATURES order is timing, extra, skip — timing before skip. + assert_eq!( + detect_enabled_features(template), + vec![Feature::TimingGames, Feature::SkipSigverify] + ); + } + + #[test] + fn baseline_config_enables_nothing() { + let template = "[pbs]\nport = 18550\ntimeout_get_header_ms = 950\n"; + assert!(detect_enabled_features(template).is_empty()); + } + + #[test] + fn false_and_commented_keys_do_not_count() { + assert!(detect_enabled_features("skip_sigverify = false\n").is_empty()); + assert!(detect_enabled_features("# skip_sigverify = true\n").is_empty()); + // A key whose name is a superstring must not match. + assert!(detect_enabled_features("not_skip_sigverify = true\n").is_empty()); + } + + #[test] + fn quoted_true_value_still_counts() { + // Tolerate `key = "true"` as well as `key = true`. + assert_eq!( + detect_enabled_features("skip_sigverify = \"true\"\n"), + vec![Feature::SkipSigverify] + ); + } + + #[test] + fn marker_feature_passes_when_proof_seen() { + let r = classify_marker_feature(Feature::TimingGames, 3); + assert_eq!(r.status, CheckStatus::Pass); + assert_eq!(r.id, "feature.timing_games"); + assert_eq!(r.data["proof_lines_seen"], 3); + } + + #[test] + fn marker_feature_warns_when_no_proof() { + // The Law-3 anti-false-green: enabled but unobserved is WARN, not PASS. + let r = classify_marker_feature(Feature::ExtraValidation, 0); + assert_eq!(r.status, CheckStatus::Warn); + assert_eq!(r.id, "feature.extra_validation"); + assert!(r.detail.contains("NOT asserting")); + } + + #[test] + fn skip_sigverify_unpoisoned_is_an_honest_warn() { + // Without the fault injection there is still no positive signal. + let r = classify_skip_sigverify(false, 0); + assert_eq!(r.status, CheckStatus::Warn); + assert_eq!(r.id, "feature.skip_sigverify"); + assert_eq!(r.data["verifiable_at_runtime"], false); + // Even a nonzero winner count proves nothing when unpoisoned (valid + // signatures win auctions with sigverify ON too). + assert_eq!(classify_skip_sigverify(false, 33).status, CheckStatus::Warn); + } + + #[test] + fn skip_sigverify_poisoned_with_winners_is_positive_proof() { + // The differential: wrong-pubkey relay + bids winning = skip fired. + let r = classify_skip_sigverify(true, 12); + assert_eq!(r.status, CheckStatus::Pass); + assert_eq!(r.data["auction_winners"], 12); + assert!(r.detail.contains("fired"), "detail: {}", r.detail); + } + + #[test] + fn skip_sigverify_poisoned_without_winners_warns() { + // Armed but unobserved: could be no traffic, not a false green. + let r = classify_skip_sigverify(true, 0); + assert_eq!(r.status, CheckStatus::Warn); + assert!(r.detail.contains("NOT asserting"), "detail: {}", r.detail); + } + + #[test] + fn poisoned_relay_detection_both_sides() { + // The real helix pubkey → not poisoned. + let clean = + format!("[[relays]]\nurl = \"http://{HELIX_RELAY_PUBKEY}@helix-relay-2:4040\"\n"); + assert!(!has_poisoned_relay_pubkey(&clean)); + // Any other pubkey → poisoned (this is the cb-sigverify-diff shape). + let poisoned = "[[relays]]\nurl = \"http://0xaaf6c1251e73fb600624937760fef218aace5b253bf068ed45398aeb29d821e4d2899343ddcbbe37cb3f6cf500dff26c@helix-relay-2:4040\"\n"; + assert!(has_poisoned_relay_pubkey(poisoned)); + // Case-insensitive on the pubkey hex. + let upper = format!( + "url = \"http://{}@helix-relay-2:4040\"", + HELIX_RELAY_PUBKEY.to_uppercase().replace("0X", "0x") + ); + assert!(!has_poisoned_relay_pubkey(&upper)); + // A templated url ({{ $relay }}) has no userinfo → not poisoned. + assert!(!has_poisoned_relay_pubkey("url = \"{{ $relay }}\"")); + // No relays at all → not poisoned. + assert!(!has_poisoned_relay_pubkey("[pbs]\nport = 18550\n")); + } + + // --- min_bid: the floor knob + the silent-flatten canary ---------------- + + #[test] + fn detects_min_bid_floor_and_ignores_zero_or_absent() { + assert_eq!(detect_min_bid_eth("[pbs]\nmin_bid_eth = 0.5\n"), Some(0.5)); + assert_eq!(detect_min_bid_eth("min_bid_eth = \"0.25\"\n"), Some(0.25)); + // zero means "no floor" - must not emit a check at all + assert_eq!(detect_min_bid_eth("min_bid_eth = 0\n"), None); + assert_eq!(detect_min_bid_eth("[pbs]\nport = 18550\n"), None); + assert_eq!(detect_min_bid_eth("# min_bid_eth = 0.5\n"), None); + } + + #[test] + fn min_bid_passes_when_bids_were_rejected_and_no_winner_is_under_the_floor() { + let r = classify_min_bid(0.5, 30, &[]); + assert_eq!(r.status, CheckStatus::Pass); + assert_eq!(r.data["rejections"], 30); + } + + #[test] + fn min_bid_fails_when_a_winner_is_below_the_floor() { + // The definitive falsifier: a sub-floor bid winning can ONLY happen if + // the floor was not applied - the silent-flatten trap firing. + let r = classify_min_bid(0.5, 0, &[0.04, 0.9]); + assert_eq!(r.status, CheckStatus::Fail); + assert_eq!(r.data["winners_below_floor"], 1); + assert!( + r.detail.contains("silently ignores"), + "names the trap: {}", + r.detail + ); + } + + #[test] + fn min_bid_fail_beats_rejections() { + // Even with rejections present, a single sub-floor winner is fatal. + let r = classify_min_bid(0.5, 10, &[0.01]); + assert_eq!(r.status, CheckStatus::Fail); + } + + #[test] + fn min_bid_warns_when_nothing_was_rejected() { + // Cannot distinguish "key ignored" from "every bid cleared the floor" + // (e.g. the builder subsidy lifting bids) - no false red. + let r = classify_min_bid(0.5, 0, &[1.04, 2.04]); + assert_eq!(r.status, CheckStatus::Warn); + assert!(r.detail.contains("NOT asserting")); + assert_eq!(r.data["winners_below_floor"], 0); + } + + #[test] + fn feature_ids_and_keys_are_stable() { + assert_eq!(Feature::TimingGames.id(), "feature.timing_games"); + assert_eq!( + Feature::ExtraValidation.config_key(), + "extra_validation_enabled" + ); + assert_eq!(Feature::SkipSigverify.config_key(), "skip_sigverify"); + } + + // --- inconclusive marking (Law 3) ----------------------------------- + // + // These verdicts are tier-1 WARN, which `exit_code` treats as pass. The + // `inconclusive` flag is what lets `--require-feature-proof` tell "the + // experiment produced no signal" apart from "a benign anomaly was noted", + // so which sites carry it IS the contract. + + // Contract: feature enabled but ZERO proof markers = armed and unmeasured. + // Contract: `get_header = "stream"` (a string knob, unlike the boolean + // features) arms WsHeaderStream; plain http does not. + #[test] + fn stream_transport_arms_ws_feature() { + let t = "[pbs]\nport = 1\n[[relays]]\nget_header = \"stream\"\n"; + assert!(detect_enabled_features(t).contains(&Feature::WsHeaderStream)); + let t2 = "[[relays]]\nget_header = \"http\"\n"; + assert!(!detect_enabled_features(t2).contains(&Feature::WsHeaderStream)); + // commented-out lines never arm + let t3 = "# get_header = \"stream\"\n"; + assert!(!detect_enabled_features(t3).contains(&Feature::WsHeaderStream)); + } + + // Contract: the fallback verdict thresholds, measured on a 220-slot + // healthy run = exactly one startup-race fallback). + #[test] + fn ws_fallback_thresholds() { + use crate::checks::CheckStatus; + // zero fallbacks: clean pass + assert_eq!(classify_ws_fallback(220, 0).status, CheckStatus::Pass); + // the startup race: still a pass, count reported + let r = classify_ws_fallback(220, 1); + assert_eq!(r.status, CheckStatus::Pass); + assert_eq!(r.data["fallbacks"], 1); + // flapping stream: warn, NOT inconclusive (real observation) + let r = classify_ws_fallback(200, 7); + assert_eq!(r.status, CheckStatus::Warn); + assert!(!r.inconclusive); + // stream never served: warn, and NOT inconclusive here - the marker + // check owns the inconclusive flag, this must not double-flag + let r = classify_ws_fallback(0, 64); + assert_eq!(r.status, CheckStatus::Warn); + assert!(!r.inconclusive, "double-flagging would make one failure two"); + } + + // Contract: the marker strings match CB main's actual log lines (pinned + // from a live run; if CB rewords them, this is the place that + // must fail). + #[test] + fn ws_marker_strings_pinned() { + assert_eq!(WS_STREAM_MARKER, "received new header from ws stream"); + assert_eq!(WS_FALLBACK_MARKER, "falling back to http get_header"); + } + + #[test] + fn zero_proof_markers_is_inconclusive() { + let r = classify_marker_feature(Feature::ExtraValidation, 0); + assert_eq!(r.status, CheckStatus::Warn); + assert!( + r.inconclusive, + "zero proof markers proves nothing: {}", + r.detail + ); + } + + // Contract: proof markers seen = a real positive assertion. + #[test] + fn seen_proof_markers_is_conclusive() { + let r = classify_marker_feature(Feature::ExtraValidation, 3); + assert_eq!(r.status, CheckStatus::Pass); + assert!(!r.inconclusive); + } + + // Contract: the differential was ARMED (poisoned relay) and saw no winners, + // so it measured nothing. + #[test] + fn armed_sigverify_differential_with_no_winners_is_inconclusive() { + let r = classify_skip_sigverify(true, 0); + assert_eq!(r.status, CheckStatus::Warn); + assert!(r.inconclusive); + } + + // Contract: the UNPOISONED case is structurally unconfirmable, not a failure + // to measure. It must NOT be marked, or every plain scenario carrying + // skip_sigverify would turn red under --require-feature-proof. + #[test] + fn unpoisoned_sigverify_is_warn_but_not_inconclusive() { + let r = classify_skip_sigverify(false, 0); + assert_eq!(r.status, CheckStatus::Warn); + assert!( + !r.inconclusive, + "a negative codepath that cannot be observed is an honest WARN, not an unmeasured one" + ); + } + + // Contract: a floor with zero rejections cannot separate "silently ignored" + // from "every bid legitimately cleared it". + #[test] + fn min_bid_with_zero_rejections_is_inconclusive() { + let r = classify_min_bid(0.5, 0, &[1.0, 2.0]); + assert_eq!(r.status, CheckStatus::Warn); + assert!(r.inconclusive); + } + + // Contract: rejections observed = the floor demonstrably applied. + #[test] + fn min_bid_with_rejections_is_conclusive() { + let r = classify_min_bid(0.5, 4, &[1.0]); + assert_eq!(r.status, CheckStatus::Pass); + assert!(!r.inconclusive); + } + + // Contract: a winner UNDER the floor is a hard FAIL and never inconclusive. + // That is evidence, not the absence of it. + #[test] + fn min_bid_violation_is_a_fail_not_inconclusive() { + let r = classify_min_bid(0.5, 0, &[0.1]); + assert_eq!(r.status, CheckStatus::Fail); + assert!(!r.inconclusive); + } +} diff --git a/src/checks/mod.rs b/src/checks/mod.rs index b08bd54..b34c9aa 100644 --- a/src/checks/mod.rs +++ b/src/checks/mod.rs @@ -1,9 +1,9 @@ //! Verification check infrastructure: result types and status enum. -use serde::Serialize; +use serde::{Deserialize, Serialize}; /// Status of a single verification check. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "UPPERCASE")] pub enum CheckStatus { Pass, @@ -23,15 +23,61 @@ impl std::fmt::Display for CheckStatus { } } +impl CheckStatus { + /// Severity rank for worst-status aggregation: `Fail > Warn > Pass > Skip`. + /// + /// This is the order the hand-rolled worst-status folds in + /// `relay_pipeline::run_relay_checks` implied (a `Fail` from any relay must + /// win the aggregate; `Skip` is the least severe). Deriving `Ord` would use + /// declaration order (`Pass, Fail, Warn, Skip`) which is NOT this order, so + /// the rank is spelled out explicitly. + fn severity(self) -> u8 { + match self { + Self::Skip => 0, + Self::Pass => 1, + Self::Warn => 2, + Self::Fail => 3, + } + } +} + +impl Ord for CheckStatus { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.severity().cmp(&other.severity()) + } +} + +impl PartialOrd for CheckStatus { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + /// Result of a single verification check. -#[derive(Debug, Clone, Serialize)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct CheckResult { pub id: String, pub tier: u8, #[serde(rename = "result")] pub status: CheckStatus, pub detail: String, + #[serde(default = "empty_data")] pub data: serde_json::Value, + + /// The check was ARMED and produced no evidence either way. + /// + /// Distinct from an annotative WARN. A Law 3 feature check that sets up a + /// differential and then observes nothing has not found a benign anomaly, it + /// has failed to measure: the scenario ran, proved nothing, and would + /// otherwise exit 0 because tier-1 WARN is non-fatal. `--require-feature-proof` + /// makes a tier-1 inconclusive check fail the run. + /// + /// Do NOT set this for a check that is structurally unable to confirm its + /// feature (e.g. `skip_sigverify` on the happy path, a negative codepath that + /// emits nothing when it fires). That is an honest WARN, not a failure to + /// measure, and flagging it would make the scenario permanently red. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub inconclusive: bool, } fn empty_data() -> serde_json::Value { @@ -46,6 +92,7 @@ impl CheckResult { status: CheckStatus::Pass, detail: detail.into(), data: empty_data(), + inconclusive: false, } } @@ -56,6 +103,7 @@ impl CheckResult { status: CheckStatus::Fail, detail: detail.into(), data: empty_data(), + inconclusive: false, } } @@ -66,6 +114,7 @@ impl CheckResult { status: CheckStatus::Warn, detail: detail.into(), data: empty_data(), + inconclusive: false, } } @@ -76,6 +125,7 @@ impl CheckResult { status: CheckStatus::Skip, detail: detail.into(), data: empty_data(), + inconclusive: false, } } @@ -83,10 +133,78 @@ impl CheckResult { self.data = data; self } + + /// Mark this check as armed-but-unmeasured. See [`CheckResult::inconclusive`]. + pub fn mark_inconclusive(mut self) -> Self { + self.inconclusive = true; + self + } } +pub mod best_bid; pub mod cb_metrics; pub mod chain_health; +pub mod feature_fired; pub mod mux_routing; pub mod payload_matching; pub mod relay_pipeline; +pub mod signer; + +#[cfg(test)] +mod status_ord_tests { + use super::CheckStatus; + + // Contract: the worst-status ordering is Fail > Warn > Pass > Skip. This is + // the rank the hand-rolled folds in run_relay_checks aggregate by, so + // `.max()` over an iterator of statuses reproduces "the worst wins". + #[test] + fn severity_order_is_fail_warn_pass_skip() { + assert!(CheckStatus::Fail > CheckStatus::Warn); + assert!(CheckStatus::Warn > CheckStatus::Pass); + assert!(CheckStatus::Pass > CheckStatus::Skip); + // Transitively, Fail is the maximum and Skip the minimum. + assert!(CheckStatus::Fail > CheckStatus::Skip); + } + + // Contract: Fail beats Warn beats Pass when aggregating a mixed set. + #[test] + fn max_picks_fail_over_warn_over_pass() { + let statuses = [CheckStatus::Pass, CheckStatus::Warn, CheckStatus::Fail]; + assert_eq!(statuses.into_iter().max(), Some(CheckStatus::Fail)); + + let no_fail = [CheckStatus::Pass, CheckStatus::Warn, CheckStatus::Pass]; + assert_eq!(no_fail.into_iter().max(), Some(CheckStatus::Warn)); + + let all_pass = [CheckStatus::Pass, CheckStatus::Pass]; + assert_eq!(all_pass.into_iter().max(), Some(CheckStatus::Pass)); + } + + // Contract: Skip is the least severe, so a Skip mixed with any real verdict + // never wins the aggregate (mirrors the registrations fold: Skip+Pass=Pass). + #[test] + fn skip_is_least_severe() { + assert_eq!( + [CheckStatus::Skip, CheckStatus::Pass].into_iter().max(), + Some(CheckStatus::Pass) + ); + assert_eq!( + [CheckStatus::Skip, CheckStatus::Warn].into_iter().max(), + Some(CheckStatus::Warn) + ); + // An all-Skip set aggregates to Skip (this input is unreachable in the + // registrations fold, which is why that fold's old init-Pass and this + // rule differ only on the impossible case — see the fold's comment). + assert_eq!( + [CheckStatus::Skip, CheckStatus::Skip].into_iter().max(), + Some(CheckStatus::Skip) + ); + } + + // Contract: an empty iterator yields None; callers pick their own default + // (run_relay_checks guards non-emptiness before aggregating). + #[test] + fn empty_iter_max_is_none() { + let empty: [CheckStatus; 0] = []; + assert_eq!(empty.into_iter().max(), None); + } +} diff --git a/src/checks/mux_routing.rs b/src/checks/mux_routing.rs index c71e526..2246fb6 100644 --- a/src/checks/mux_routing.rs +++ b/src/checks/mux_routing.rs @@ -65,20 +65,28 @@ pub struct CbEvent { /// `Ok(None)` if no mux sections (check will SKIP), /// `Err` if parsing fails. pub fn extract_mux_from_config(path: &str) -> eyre::Result>> { + let template = read_cb_config_template(path)?; + parse_mux_from_toml_template(&template) +} + +/// Read the Commit-Boost config TOML template from a config path. +/// +/// Supports `.toml` (raw CB config) and `.yml`/`.yaml` (Kurtosis config with the +/// CB config embedded at `mev_params.commit_boost_config`). Shared by the mux +/// check and the feature-fired checks, both of which scan the same template. +pub fn read_cb_config_template(path: &str) -> eyre::Result { let raw = std::fs::read_to_string(path) .map_err(|e| eyre::eyre!("Failed to read config '{path}': {e}"))?; - let template = if path.ends_with(".toml") { - raw + if path.ends_with(".toml") { + Ok(raw) } else if path.ends_with(".yml") || path.ends_with(".yaml") { - extract_commit_boost_config_from_yaml(&raw)? + extract_commit_boost_config_from_yaml(&raw) } else { - return Err(eyre::eyre!( + Err(eyre::eyre!( "Unrecognized config format. Expected .toml (CB config) or .yml/.yaml (Kurtosis config), got: {path}" - )); - }; - - parse_mux_from_toml_template(&template) + )) + } } fn extract_commit_boost_config_from_yaml(raw: &str) -> eyre::Result { @@ -92,9 +100,7 @@ fn extract_commit_boost_config_from_yaml(raw: &str) -> eyre::Result { .and_then(|p| p.get("commit_boost_config")) .and_then(|c| c.as_str()) .ok_or_else(|| { - eyre::eyre!( - "No mev_params.commit_boost_config found in Kurtosis YAML config" - ) + eyre::eyre!("No mev_params.commit_boost_config found in Kurtosis YAML config") })?; Ok(template.to_string()) @@ -190,8 +196,8 @@ fn parse_one_mux_section<'a>( let id = id.ok_or_else(|| eyre::eyre!("[[mux]] section missing 'id' field"))?; let relay_identity = relay_identity_from_mux_id(&id); - let pubkeys = pubkeys - .ok_or_else(|| eyre::eyre!("[[mux]] section '{id}' missing 'validator_pubkeys'"))?; + let pubkeys = + pubkeys.ok_or_else(|| eyre::eyre!("[[mux]] section '{id}' missing 'validator_pubkeys'"))?; Ok(Some(MuxEntry { id, @@ -221,11 +227,11 @@ fn parse_mux_relay_body( continue; } - if let Some((key, raw_val)) = parse_key_value(trimmed) { - if key == "url" { - let val = raw_val.trim_matches('"'); - return Ok(parse_relay_index_from_template(val)); - } + if let Some((key, raw_val)) = parse_key_value(trimmed) + && key == "url" + { + let val = raw_val.trim_matches('"'); + return Ok(parse_relay_index_from_template(val)); } } @@ -234,10 +240,7 @@ fn parse_mux_relay_body( fn parse_relay_index_from_template(val: &str) -> Option { let val = val.trim(); - let stripped = val - .trim_start_matches("{{") - .trim_end_matches("}}") - .trim(); + let stripped = val.trim_start_matches("{{").trim_end_matches("}}").trim(); let parts: Vec<&str> = stripped.split_whitespace().collect(); if parts.len() >= 3 && parts[0] == "index" && parts[1] == ".Relays" { parts[2].parse::().ok() @@ -260,10 +263,7 @@ fn parse_pubkey_array( let mut accum = rest.to_string(); if !accum.trim_end().ends_with(']') { - loop { - let Some(next) = lines.next() else { - break; - }; + for next in lines.by_ref() { accum.push('\n'); accum.push_str(next); if next.trim().ends_with(']') { @@ -273,12 +273,12 @@ fn parse_pubkey_array( } let raw = accum.trim(); - let start = raw.find('[').ok_or_else(|| { - eyre::eyre!("Could not find opening '[' in pubkey array: {raw:.50}...") - })?; - let end = raw.rfind(']').ok_or_else(|| { - eyre::eyre!("Could not find closing ']' in pubkey array: {raw:.50}...") - })?; + let start = raw + .find('[') + .ok_or_else(|| eyre::eyre!("Could not find opening '[' in pubkey array: {raw:.50}..."))?; + let end = raw + .rfind(']') + .ok_or_else(|| eyre::eyre!("Could not find closing ']' in pubkey array: {raw:.50}..."))?; let inner = &raw[start + 1..end]; let mut pubkeys = Vec::new(); @@ -368,8 +368,8 @@ pub fn parse_cb_log_line(line: &str) -> Option { found = line[pos + lvl.len() + 2..].to_string(); break; } - if line.starts_with(lvl) { - found = line[lvl.len()..].trim_start().to_string(); + if let Some(stripped) = line.strip_prefix(lvl) { + found = stripped.trim_start().to_string(); break; } } @@ -422,10 +422,18 @@ pub fn parse_cb_log_line(line: &str) -> Option { fields.insert(key.to_string(), val.clone()); match key { - "slot" => { slot = val.parse().ok(); } - "validator" | "pubkey" => { validator = Some(normalize_pubkey(&val)); } - "relay_id" => { relay_id = Some(val); } - "mux_id" => { mux_id = Some(val); } + "slot" => { + slot = val.parse().ok(); + } + "validator" | "pubkey" => { + validator = Some(normalize_pubkey(&val)); + } + "relay_id" => { + relay_id = Some(val); + } + "mux_id" => { + mux_id = Some(val); + } _ => {} } } @@ -441,8 +449,6 @@ pub fn parse_cb_log_line(line: &str) -> Option { }) } - - // --------------------------------------------------------------------------- // Log fetching // --------------------------------------------------------------------------- @@ -452,15 +458,51 @@ pub fn parse_cb_log_line(line: &str) -> Option { /// Fetches all logs and filters client-side. The `--regex-match` flag is /// tried first as an optimization, but some kurtosis versions ignore it. pub fn fetch_service_logs(enclave: &str, service: &str) -> eyre::Result { - info!( - "mux check: fetching logs from service '{service}' (enclave={enclave})..." - ); + info!("mux check: fetching logs from service '{service}' (enclave={enclave})..."); + + // Filter to mux-relevant lines client-side. + let result = fetch_filtered_logs( + enclave, + service, + &[ + "using mux config", + "received new header", + "auction winner", + "received unblinded block", + "CRITICAL: no payload", + ], + )?; + + if result.is_empty() { + // The empty-log warning wants a sample of what WAS there, so re-fetch + // the raw logs. This only runs on the (rare) empty path. + let all_logs = fetch_raw_logs(enclave, service).unwrap_or_default(); + let sample: String = all_logs + .lines() + .filter(|l| !l.trim().is_empty()) + .take(3) + .collect::>() + .join("\n"); + warn!( + "mux check: service '{service}' returned no relevant log lines. \ + Total: {} bytes. Sample:\n{}", + all_logs.len(), + sample + ); + } else { + info!( + "mux check: service '{service}' returned {} relevant log line(s)", + result.lines().count() + ); + } + Ok(result) +} + +/// Fetch a service's raw logs (stdout+stderr combined), no filtering. +fn fetch_raw_logs(enclave: &str, service: &str) -> eyre::Result { let output = std::process::Command::new("kurtosis") - .args([ - "service", "logs", enclave, service, - "-n", "200000", - ]) + .args(["service", "logs", enclave, service, "-n", "200000"]) .output() .map_err(|e| eyre::eyre!("Failed to run 'kurtosis service logs': {e}"))?; @@ -476,41 +518,25 @@ pub fn fetch_service_logs(enclave: &str, service: &str) -> eyre::Result // Combine stdout and stderr — kurtosis writes to either depending on version. let stdout = String::from_utf8_lossy(&output.stdout); let stderr = String::from_utf8_lossy(&output.stderr); - let all_logs = format!("{}\n{}", stdout, stderr); + Ok(format!("{}\n{}", stdout, stderr)) +} - // Filter to relevant lines client-side. +/// Fetch a service's logs and keep only lines containing one of `keywords`. +/// +/// The shared log-fetch primitive: mux and the feature-fired checks both scan +/// CB debug logs for a small set of marker strings. Returns the matching lines +/// joined by newlines (empty string if none matched). +pub fn fetch_filtered_logs( + enclave: &str, + service: &str, + keywords: &[&str], +) -> eyre::Result { + let all_logs = fetch_raw_logs(enclave, service)?; let result: String = all_logs .lines() - .filter(|line| { - line.contains("using mux config") - || line.contains("received new header") - || line.contains("auction winner") - || line.contains("received unblinded block") - || line.contains("CRITICAL: no payload") - }) + .filter(|line| keywords.iter().any(|kw| line.contains(kw))) .collect::>() .join("\n"); - - if result.is_empty() { - let sample: String = all_logs - .lines() - .filter(|l| !l.trim().is_empty()) - .take(3) - .collect::>() - .join("\n"); - warn!( - "mux check: service '{service}' returned no relevant log lines. Total: {} bytes stdout, {} bytes stderr. Sample:\n{}", - stdout.len(), - stderr.len(), - sample - ); - } else { - info!( - "mux check: service '{service}' returned {} relevant log line(s)", - result.lines().count() - ); - } - Ok(result) } @@ -573,6 +599,27 @@ pub async fn check_mux_routing( } } + classify_mux_routing(entries, &expected_mux, &all_events) +} + +/// Pure verdict logic for mux routing (the Law 4 test seam; the async wrapper +/// above only gathers the logs). Contract: +/// - no `[[mux]]` entries → SKIP +/// - no mux-related log events at all → WARN (couldn't observe routing) +/// - events seen but ZERO routing DECISIONS actually checked (no "using mux +/// config" DEBUG event for a known pubkey) → WARN, NOT pass. This is the Law 3 +/// fix: we no longer report "verified" having verified nothing when CB debug +/// logging is off. `[logs.stdout] level = "debug"` is required for mux scenarios. +/// - a checked decision routed to the wrong mux → FAIL +/// - otherwise → PASS, counting the routing decisions actually verified. +pub fn classify_mux_routing( + entries: &[MuxEntry], + expected_mux: &HashMap, + all_events: &[CbEvent], +) -> CheckResult { + let mux_ids: Vec<&str> = entries.iter().map(|e| e.id.as_str()).collect(); + let mux_detail = format!("muxes=[{}]", mux_ids.join(", ")); + // Filter to events relevant to mux verification. let mux_events: Vec<&CbEvent> = all_events .iter() @@ -587,32 +634,34 @@ pub async fn check_mux_routing( let total_events = mux_events.len(); - let data = serde_json::json!({ - "total_mux_entries": entries.len(), - "total_log_events": total_events, - "pubkeys_verified": 0, - "violations": [], - "violation_count": 0, - "mux_entries_seen": [], - }); - if total_events == 0 { return CheckResult::warn( "mux.routing", 1, format!( - "No mux-related log lines found in any CB PBS service. \ - No getHeader requests were recorded — mux config is valid \ - but routing could not be verified at runtime. muxes=[{}]", - entries.iter().map(|e| e.id.as_str()).collect::>().join(", ") + "No mux-related log lines found in any CB PBS service — routing could not be \ + verified at runtime (mux config parsed fine). {mux_detail}" ), - ).with_data(data); + ) + .with_data(serde_json::json!({ + "total_mux_entries": entries.len(), + "total_log_events": 0, + "pubkeys_verified": 0, + "routing_decisions_verified": 0, + "violations": [], + "violation_count": 0, + "mux_entries_seen": [], + })); } - // Verify: for each "using mux config" event, does the pubkey match? + // Verify: for each "using mux config" event for a KNOWN pubkey, does the + // routed mux match the expected mux? Count how many such decisions we + // actually checked — a match OR a violation both count; an event without a + // mux_id (e.g. "received new header") is NOT a verified routing decision. let mut violations: Vec = Vec::new(); let mut pubkeys_verified: HashSet = HashSet::new(); let mut mux_entries_seen: HashSet = HashSet::new(); + let mut routing_decisions_verified: usize = 0; for event in &mux_events { if let Some(ref mux_id) = event.mux_id { @@ -622,35 +671,37 @@ pub async fn check_mux_routing( if let Some(ref pk_norm) = event.validator { pubkeys_verified.insert(pk_norm.clone()); - if let Some(expected_mux_id) = expected_mux.get(pk_norm) { - if let Some(ref actual_mux_id) = event.mux_id { - if actual_mux_id != expected_mux_id { - let expected_relay = entries - .iter() - .find(|e| e.id == *expected_mux_id) - .map(|e| e.relay_identity.as_str()) - .unwrap_or("?"); - let actual_relay = entries - .iter() - .find(|e| e.id == *actual_mux_id) - .map(|e| e.relay_identity.as_str()) - .unwrap_or("?"); - - violations.push(serde_json::json!({ - "slot": event.slot, - "proposer_pubkey": format!("0x{pk_norm}"), - "routed_to_mux": actual_mux_id, - "routed_to_relay": actual_relay, - "expected_mux": expected_mux_id, - "expected_relay": expected_relay, - })); - - warn!( - "mux check: MISROUTING — pubkey 0x{pk_norm}.. should route to \ - '{expected_mux_id}' ({expected_relay}) but was routed to \ - '{actual_mux_id}' ({actual_relay})" - ); - } + if let Some(expected_mux_id) = expected_mux.get(pk_norm) + && let Some(ref actual_mux_id) = event.mux_id + { + routing_decisions_verified += 1; + + if actual_mux_id != expected_mux_id { + let expected_relay = entries + .iter() + .find(|e| e.id == *expected_mux_id) + .map(|e| e.relay_identity.as_str()) + .unwrap_or("?"); + let actual_relay = entries + .iter() + .find(|e| e.id == *actual_mux_id) + .map(|e| e.relay_identity.as_str()) + .unwrap_or("?"); + + violations.push(serde_json::json!({ + "slot": event.slot, + "proposer_pubkey": format!("0x{pk_norm}"), + "routed_to_mux": actual_mux_id, + "routed_to_relay": actual_relay, + "expected_mux": expected_mux_id, + "expected_relay": expected_relay, + })); + + warn!( + "mux check: MISROUTING — pubkey 0x{pk_norm}.. should route to \ + '{expected_mux_id}' ({expected_relay}) but was routed to \ + '{actual_mux_id}' ({actual_relay})" + ); } } } @@ -660,14 +711,12 @@ pub async fn check_mux_routing( "total_mux_entries": entries.len(), "total_log_events": total_events, "pubkeys_verified": pubkeys_verified.len(), + "routing_decisions_verified": routing_decisions_verified, "violations": violations, "violation_count": violations.len(), "mux_entries_seen": mux_entries_seen.iter().cloned().collect::>(), }); - let mux_ids: Vec<&str> = entries.iter().map(|e| e.id.as_str()).collect(); - let mux_detail = format!("muxes=[{}]", mux_ids.join(", ")); - if !violations.is_empty() { CheckResult::fail( "mux.routing", @@ -679,15 +728,25 @@ pub async fn check_mux_routing( ), ) .with_data(data) + } else if routing_decisions_verified == 0 { + CheckResult::warn( + "mux.routing", + 1, + format!( + "{total_events} mux log event(s) seen but ZERO routing decisions could be \ + verified — need CB \"using mux config\" DEBUG logs (is `[logs.stdout] level = \ + \"debug\"` set, and are proposer pubkeys covered by the mux config?). NOT \ + asserting routing correctness. {mux_detail}" + ), + ) + .with_data(data) } else { CheckResult::pass( "mux.routing", 1, format!( - "All {} mux routing decision(s) verified ✓ CB PBS correctly routed \ - every getHeader request according to mux config. {}", - total_events, - mux_detail, + "All {routing_decisions_verified} mux routing decision(s) verified ✓ CB PBS routed \ + every checked getHeader request per mux config. {mux_detail}" ), ) .with_data(data) @@ -701,11 +760,95 @@ pub async fn check_mux_routing( #[cfg(test)] mod tests { use super::*; + use crate::checks::CheckStatus; + + // --- classify_mux_routing verdict tests (Law 3/4: no false green) -------- + + fn entry(id: &str, relay: &str, pubkeys: &[&str]) -> MuxEntry { + MuxEntry { + id: id.to_string(), + relay_identity: relay.to_string(), + validator_pubkeys: pubkeys.iter().map(|p| p.to_string()).collect(), + } + } + + fn event(message: &str, validator: Option<&str>, mux_id: Option<&str>) -> CbEvent { + CbEvent { + message: message.to_string(), + fields: HashMap::new(), + slot: Some(1), + validator: validator.map(|v| v.to_string()), + relay_id: None, + mux_id: mux_id.map(|m| m.to_string()), + } + } + + fn expected(pairs: &[(&str, &str)]) -> HashMap { + pairs + .iter() + .map(|(pk, mux)| (pk.to_string(), mux.to_string())) + .collect() + } + + #[test] + fn mux_pass_when_a_routing_decision_is_verified() { + let entries = [entry("mux_0", "helix", &["aa"])]; + let exp = expected(&[("aa", "mux_0")]); + // A "using mux config" event: known pubkey routed to its expected mux. + let events = [event("using mux config", Some("aa"), Some("mux_0"))]; + let r = classify_mux_routing(&entries, &exp, &events); + assert_eq!(r.status, CheckStatus::Pass); + assert_eq!(r.data["routing_decisions_verified"], 1); + } + + #[test] + fn mux_warn_when_events_seen_but_no_decision_verifiable() { + // The false-green case: log events exist ("received new header" has a + // validator but no mux_id), so NO routing decision is actually checked. + // Old code PASSed here; the fix WARNs. + let entries = [entry("mux_0", "helix", &["aa"])]; + let exp = expected(&[("aa", "mux_0")]); + let events = [event("received new header", Some("aa"), None)]; + let r = classify_mux_routing(&entries, &exp, &events); + assert_eq!( + r.status, + CheckStatus::Warn, + "must NOT pass on zero verified decisions" + ); + assert_eq!(r.data["routing_decisions_verified"], 0); + assert!(r.detail.contains("DEBUG")); + } + + #[test] + fn mux_warn_when_no_events_at_all() { + let entries = [entry("mux_0", "helix", &["aa"])]; + let exp = expected(&[("aa", "mux_0")]); + let r = classify_mux_routing(&entries, &exp, &[]); + assert_eq!(r.status, CheckStatus::Warn); + assert_eq!(r.data["total_log_events"], 0); + } + + #[test] + fn mux_fail_on_misroute() { + let entries = [ + entry("mux_0", "helix", &["aa"]), + entry("mux_1", "flashbots", &["bb"]), + ]; + let exp = expected(&[("aa", "mux_0"), ("bb", "mux_1")]); + // pubkey aa expected at mux_0 but routed to mux_1 → violation. + let events = [event("using mux config", Some("aa"), Some("mux_1"))]; + let r = classify_mux_routing(&entries, &exp, &events); + assert_eq!(r.status, CheckStatus::Fail); + assert_eq!(r.data["violation_count"], 1); + } #[test] fn test_relay_identity_from_mux_id() { assert_eq!(relay_identity_from_mux_id("node_0_to_helix"), "helix"); - assert_eq!(relay_identity_from_mux_id("node_1_to_flashbots"), "flashbots"); + assert_eq!( + relay_identity_from_mux_id("node_1_to_flashbots"), + "flashbots" + ); assert_eq!(relay_identity_from_mux_id("my_mux_entry"), "my_mux_entry"); assert_eq!(relay_identity_from_mux_id("to_"), "to_"); } @@ -839,7 +982,10 @@ additional_services: assert_eq!(event.mux_id, Some("node_1_to_flashbots".to_string())); assert_eq!(event.slot, Some(160)); assert!(event.validator.is_some()); - assert_eq!(event.validator.unwrap(), "b2ad1574eaca33f1555308e24b27a095d24aed8f4af5302ea2c6ba2e50936d25ffea7047be94065eac630693c7f86757"); + assert_eq!( + event.validator.unwrap(), + "b2ad1574eaca33f1555308e24b27a095d24aed8f4af5302ea2c6ba2e50936d25ffea7047be94065eac630693c7f86757" + ); } #[test] @@ -863,8 +1009,14 @@ additional_services: assert_eq!(event.message, "received new header"); assert_eq!(event.relay_id, Some("mux_helix".to_string())); assert_eq!(event.slot, Some(521)); - assert_eq!(event.fields.get("header_size_bytes"), Some(&"2891".to_string())); - assert_eq!(event.fields.get("value_eth"), Some(&"0.050439063999832000".to_string())); + assert_eq!( + event.fields.get("header_size_bytes"), + Some(&"2891".to_string()) + ); + assert_eq!( + event.fields.get("value_eth"), + Some(&"0.050439063999832000".to_string()) + ); } #[test] @@ -876,8 +1028,41 @@ additional_services: assert_eq!(event.relay_id, Some("mux_helix".to_string())); assert_eq!(event.slot, Some(34)); assert!(event.validator.is_some()); - assert_eq!(event.fields.get("header_size_bytes"), Some(&"3099".to_string())); - assert_eq!(event.fields.get("value_eth"), Some(&"0.042701386561497000".to_string())); + assert_eq!( + event.fields.get("header_size_bytes"), + Some(&"3099".to_string()) + ); + assert_eq!( + event.fields.get("value_eth"), + Some(&"0.042701386561497000".to_string()) + ); + } + + #[test] + fn test_parse_cb_log_line_ws_stream_variant() { + // CB with WS get_header streaming (commit-boost PR #483) logs + // "received new header from ws stream" — a SUPERSTRING of the HTTP + // message with different latency fields (connect_latency / + // first_bid_latency instead of latency) and no content_type. Our + // consumers filter with starts_with("received new header"), so the + // variant must (a) match that prefix and (b) yield the fields + // best_bid needs: relay_id, slot (span-rendered), value_eth. + let line = r#"2026-08-04T10:00:00.000000Z INFO : received new header from ws stream relay_id="mux_helix" header_size_bytes=2891 connect_latency=12.3ms first_bid_latency=88.1ms validate_latency=1.2ms version=Fulu value_eth="0.050439063999832000" block_hash=0x15cd5f31333e1a8d42f0207cf1a61c65baf3d938836b07877a3a76b1cb890d11 updates=7 invalid_frames=0 req_id=8e5020cb-a893-42b3-a2f5-8f4c3f400c9e slot=521"#; + + let event = parse_cb_log_line(line).expect("ws variant should parse"); + assert!( + event.message.starts_with("received new header"), + "prefix filter must match the ws variant, got: {:?}", + event.message + ); + assert_eq!(event.relay_id, Some("mux_helix".to_string())); + assert_eq!(event.slot, Some(521)); + assert_eq!( + event.fields.get("value_eth"), + Some(&"0.050439063999832000".to_string()) + ); + // The HTTP-only fields are absent, not misparsed. + assert!(!event.fields.contains_key("latency")); } #[test] @@ -918,8 +1103,13 @@ mod log_file_tests { ]; for line in &lines { - let event = parse_cb_log_line(line).expect(&format!("should parse: {}", &line[..80])); - assert!(event.message.starts_with("using mux"), "message should start with 'using mux', got: {:?}", event.message); + let event = + parse_cb_log_line(line).unwrap_or_else(|| panic!("should parse: {}", &line[..80])); + assert!( + event.message.starts_with("using mux"), + "message should start with 'using mux', got: {:?}", + event.message + ); assert!(event.mux_id.is_some(), "mux_id should be Some"); assert!(event.slot.is_some(), "slot should be Some"); assert!(event.validator.is_some(), "validator should be Some"); @@ -933,7 +1123,11 @@ mod log_file_tests { let event = parse_cb_log_line(line).expect("should parse line with ANSI codes"); // The message should contain "using mux" (may have trailing ANSI codes) - assert!(event.message.contains("using mux"), "message should contain 'using mux', got: {:?}", event.message); + assert!( + event.message.contains("using mux"), + "message should contain 'using mux', got: {:?}", + event.message + ); // mux_id should be parsed correctly despite ANSI codes assert_eq!(event.mux_id, Some("node_1_to_flashbots".to_string())); assert_eq!(event.slot, Some(2)); @@ -945,7 +1139,11 @@ mod log_file_tests { let line = "[16eac416a3014ec191173b9e95cc11a6] \x1b[2m2026-05-07T04:28:26.009013Z\x1b[0m \x1b[32mINFO\x1b[0m \x1b[1m\x1b[0m: received new header \x1b[3mrelay_id\x1b[0m\x1b[2m=\x1b[0m\"mux_helix\" \x1b[3mheader_size_bytes\x1b[0m\x1b[2m=\x1b[0m2891 \x1b[3mlatency\x1b[0m\x1b[2m=\x1b[0m6.1415ms \x1b[3mversion\x1b[0m\x1b[2m=\x1b[0mFulu \x1b[3mvalue_eth\x1b[0m\x1b[2m=\x1b[0m\"0.050439063999832000\" \x1b[2m\x1b[3mmethod\x1b[0m\x1b[2m=\x1b[0m/eth/v1/builder/header/{slot}/{parent_hash}/{pubkey} \x1b[3mreq_id\x1b[0m\x1b[2m=\x1b[0m8e5020cb \x1b[3mslot\x1b[0m\x1b[2m=\x1b[0m521 \x1b[3mparent_hash\x1b[0m\x1b[2m=\x1b[0m0x969f22b3 \x1b[3mvalidator\x1b[0m\x1b[2m=\x1b[0m0x98213294\x1b[0m"; let event = parse_cb_log_line(line).expect("should parse"); - assert!(event.message.contains("received new header"), "message: {:?}", event.message); + assert!( + event.message.contains("received new header"), + "message: {:?}", + event.message + ); assert_eq!(event.relay_id, Some("mux_helix".to_string())); assert_eq!(event.slot, Some(521)); } @@ -953,7 +1151,7 @@ mod log_file_tests { #[test] fn test_mux_event_filter() { // Test that the filter used in check_mux_routing matches parsed events - let lines = vec![ + let lines = [ "2026-05-07T04:28:26.004744Z DEBUG : using mux config mux_id=\"node_1_to_flashbots\" relays=1 pubkey=0x8ca49f0c slot=2 validator=0x8ca49f0c", "2026-05-07T04:28:26.009013Z INFO : received new header relay_id=\"mux_helix\" header_size_bytes=2891 slot=521 validator=0x98213294", "2026-05-07T04:28:26.011040Z INFO : auction winner relay_id=\"mux_helix\" value_eth=\"0.050439063999832000\" block_hash=0x15cd5f31 slot=521", @@ -972,6 +1170,10 @@ mod log_file_tests { }) .collect(); - assert_eq!(mux_events.len(), 3, "3 of 4 events should be mux-related (not 'received header')"); + assert_eq!( + mux_events.len(), + 3, + "3 of 4 events should be mux-related (not 'received header')" + ); } } diff --git a/src/checks/payload_matching.rs b/src/checks/payload_matching.rs index f140ca9..7401722 100644 --- a/src/checks/payload_matching.rs +++ b/src/checks/payload_matching.rs @@ -1,5 +1,9 @@ //! Cross-references relay delivered payloads with on-chain beacon blocks. +use std::collections::BTreeMap; + +use alloy_primitives::B256; +use futures::StreamExt; use tracing::warn; use crate::beacon::BeaconClient; @@ -7,19 +11,27 @@ use crate::checks::CheckResult; use crate::relay::RelayClient; /// Compare relay delivered payload hashes against on-chain beacon block hashes. +/// +/// Collects the block_hash EACH relay reported per slot (NOT deduped/first-wins), +/// fetches the on-chain hash per slot, then classifies. Splitting IO from the +/// verdict lets `classify_payload_matches` be unit-tested (Law 4). pub async fn check_payload_hash_match( relays: &[RelayClient], beacon: &BeaconClient, start_slot: u64, end_slot: u64, ) -> CheckResult { - // Collect delivered payloads from all relays, deduped by slot - let mut by_slot = std::collections::HashMap::new(); + // Per slot, every (relay, block_hash) reported. Keeping all of them (instead + // of first-wins `or_insert`) is what lets us DETECT cross-relay disagreement. + let mut by_slot: BTreeMap> = BTreeMap::new(); for relay in relays { match relay.get_payloads_delivered(start_slot, end_slot).await { Ok(payloads) => { for p in payloads { - by_slot.entry(p.slot).or_insert(p.block_hash); + by_slot + .entry(p.slot) + .or_default() + .push((relay.base_url().to_string(), p.block_hash)); } } Err(e) => { @@ -36,51 +48,111 @@ pub async fn check_payload_hash_match( ); } + // Fetch the on-chain hash for each observed slot (None = missing/error), + // concurrently but bounded. The result map is keyed by slot, so out-of-order + // completion cannot change it; the Err -> warn+None mapping is preserved. + let slots: Vec = by_slot.keys().copied().collect(); + let fetched: Vec<_> = futures::stream::iter(slots) + .map(|slot| async move { (slot, beacon.get_block_hash(slot).await) }) + .buffer_unordered(16) + .collect() + .await; + + let mut chain: BTreeMap> = BTreeMap::new(); + for (slot, res) in fetched { + let h = match res { + Ok(v) => v, + Err(e) => { + warn!("Failed to get block for slot {slot}: {e}"); + None + } + }; + chain.insert(slot, h); + } + + classify_payload_matches(&by_slot, &chain) +} + +/// Pure verdict logic (Law 4 seam). Generic over the hash type so tests use a +/// trivial `H`. WARNs when a slot has divergent relay hashes (relay equivocation) +/// OR when no relay hash matches the on-chain hash — the first-wins union used to +/// silently drop the cross-relay disagreement and could PASS order-dependently. +pub fn classify_payload_matches( + by_slot: &BTreeMap>, + chain: &BTreeMap>, +) -> CheckResult +where + H: Copy + Eq + std::hash::Hash + std::fmt::LowerHex, +{ + if by_slot.is_empty() { + return CheckResult::skip( + "payload_hash_match", + 1, + "No delivered payloads to compare (upstream check owns this signal)", + ); + } + let mut matched = 0u64; let mut mismatched = 0u64; let mut missed = 0u64; let mut mismatches = Vec::new(); + let mut conflicts = Vec::new(); - for (&slot, relay_hash) in &by_slot { - match beacon.get_block_hash(slot).await { - Ok(None) => { - missed += 1; - } - Err(e) => { - warn!("Failed to get block for slot {slot}: {e}"); - missed += 1; - } - Ok(Some(chain_hash)) => { - if *relay_hash == chain_hash { + for (&slot, relay_hashes) in by_slot { + // Cross-relay disagreement: >1 distinct block_hash reported for one slot. + let distinct: std::collections::HashSet = relay_hashes.iter().map(|(_, h)| *h).collect(); + if distinct.len() > 1 { + conflicts.push(serde_json::json!({ + "slot": slot, + "relays": relay_hashes + .iter() + .map(|(r, h)| serde_json::json!({ "relay": r, "block_hash": format!("{h:#x}") })) + .collect::>(), + })); + warn!( + "Payload hash conflict at slot {slot}: {} distinct block hashes reported \ + (relay equivocation or bug)", + distinct.len() + ); + } + + match chain.get(&slot) { + Some(Some(chain_hash)) => { + if distinct.iter().any(|h| h == chain_hash) { matched += 1; } else { mismatched += 1; mismatches.push(serde_json::json!({ "slot": slot, - "relay_hash": format!("{:#x}", relay_hash), - "chain_hash": format!("{:#x}", chain_hash), + "relay_hashes": distinct.iter().map(|h| format!("{h:#x}")).collect::>(), + "chain_hash": format!("{chain_hash:#x}"), })); warn!( - "Hash mismatch at slot {slot}: relay={:#x} chain={:#x} (possible reorg)", - relay_hash, chain_hash + "Hash mismatch at slot {slot}: no relay hash matched chain {chain_hash:#x} \ + (possible reorg)" ); } } + _ => missed += 1, } } let total = by_slot.len(); + let conflict_count = conflicts.len(); let detail = format!( - "{matched} matched, {mismatched} mismatched, {missed} missed out of {total} delivered" + "{matched} matched, {mismatched} mismatched, {conflict_count} cross-relay conflict(s), \ + {missed} missed out of {total} delivered" ); let data = serde_json::json!({ "matched": matched, "mismatched": mismatched, "missed": missed, + "cross_relay_conflicts": conflict_count, + "conflicts": conflicts, "mismatches": mismatches, }); - if mismatched > 0 { + if mismatched > 0 || conflict_count > 0 { CheckResult::warn("payload_hash_match", 1, detail).with_data(data) } else { CheckResult::pass("payload_hash_match", 1, detail).with_data(data) @@ -96,3 +168,88 @@ pub async fn run_payload_checks( ) -> Vec { vec![check_payload_hash_match(relays, beacon, start_slot, end_slot).await] } + +#[cfg(test)] +mod tests { + use super::*; + use crate::checks::CheckStatus; + + // --- classify_payload_matches verdict tests (u64 stands in for B256) ------ + + fn by_slot(pairs: &[(u64, &[(&str, u64)])]) -> BTreeMap> { + pairs + .iter() + .map(|(slot, relays)| { + ( + *slot, + relays.iter().map(|(r, h)| (r.to_string(), *h)).collect(), + ) + }) + .collect() + } + + fn chain(pairs: &[(u64, Option)]) -> BTreeMap> { + pairs.iter().copied().collect() + } + + #[test] + fn payload_pass_on_clean_single_relay_match() { + let bs = by_slot(&[(5, &[("relay-a", 0xaa)])]); + let ch = chain(&[(5, Some(0xaa))]); + let r = classify_payload_matches(&bs, &ch); + assert_eq!(r.status, CheckStatus::Pass); + assert_eq!(r.data["matched"], 1); + } + + #[test] + fn payload_warn_on_cross_relay_conflict() { + // The false-green: two relays report DIFFERENT hashes for slot 5. The old + // first-wins union dropped one and could PASS; now it's detected → WARN. + let bs = by_slot(&[(5, &[("relay-a", 0xaa), ("relay-b", 0xbb)])]); + let ch = chain(&[(5, Some(0xaa))]); // one relay even matches chain + let r = classify_payload_matches(&bs, &ch); + assert_eq!(r.status, CheckStatus::Warn, "conflict must not pass"); + assert_eq!(r.data["cross_relay_conflicts"], 1); + } + + #[test] + fn payload_warn_when_no_relay_matches_chain() { + let bs = by_slot(&[(5, &[("relay-a", 0xaa)])]); + let ch = chain(&[(5, Some(0xbb))]); + let r = classify_payload_matches(&bs, &ch); + assert_eq!(r.status, CheckStatus::Warn); + assert_eq!(r.data["mismatched"], 1); + } + + #[test] + fn payload_missed_does_not_downgrade_verdict() { + // A delivered slot with no on-chain block is 'missed', informational only. + let bs = by_slot(&[(5, &[("relay-a", 0xaa)])]); + let ch = chain(&[(5, None)]); + let r = classify_payload_matches(&bs, &ch); + assert_eq!(r.status, CheckStatus::Pass); + assert_eq!(r.data["missed"], 1); + } + + #[test] + fn payload_empty_skips() { + let bs: BTreeMap> = BTreeMap::new(); + let ch: BTreeMap> = BTreeMap::new(); + assert_eq!(classify_payload_matches(&bs, &ch).status, CheckStatus::Skip); + } + + // Contract: with no relays there are no delivered payloads to cross-check, + // so the check must SKIP (it explicitly defers this signal to an upstream + // check) rather than PASS on zero comparisons. With an empty relay slice the + // collection loop never runs and the beacon is never queried, so this is a + // pure, network-free assertion of the empty-input contract. + #[tokio::test] + async fn no_relays_skips_not_passes() { + let beacon = BeaconClient::new("http://127.0.0.1:0"); + let relays: [RelayClient; 0] = []; + let r = check_payload_hash_match(&relays, &beacon, 0, 10).await; + assert_eq!(r.status, CheckStatus::Skip); + assert_eq!(r.id, "payload_hash_match"); + assert_eq!(r.tier, 1); + } +} diff --git a/src/checks/relay_pipeline.rs b/src/checks/relay_pipeline.rs index 1e58a4c..e979b46 100644 --- a/src/checks/relay_pipeline.rs +++ b/src/checks/relay_pipeline.rs @@ -2,6 +2,8 @@ use std::collections::HashMap; +use futures::StreamExt; + use crate::beacon::BeaconClient; use crate::checks::{CheckResult, CheckStatus}; use crate::relay::RelayClient; @@ -109,34 +111,36 @@ pub async fn check_mev_delivery_rate( end_slot: u64, threshold: f64, ) -> CheckResult { - // Get delivered payload block hashes — try each relay until one succeeds. - // Some relays (e.g., mev-boost-relay) don't expose the data API. + // UNION delivered payloads across ALL relays (not just the first that answers). + // Under mux each relay holds only the half of deliveries it won, so taking the + // first relay undercounts the MEV rate and spuriously WARNs (H3). We only SKIP + // if NO relay answered the data API at all. let mut delivered = Vec::new(); + let mut any_ok = false; let mut last_error = None; for relay in relays { match relay.get_payloads_delivered(start_slot, end_slot).await { Ok(p) => { - delivered = p; - break; + delivered.extend(p); + any_ok = true; } Err(e) => { tracing::warn!( - "Relay {} doesn't support data API ({}), trying next...", - relay.base_url(), - e + "Relay {} doesn't support the data API ({e})", + relay.base_url() ); last_error = Some(e); } } } - if delivered.is_empty() && last_error.is_some() { + if !any_ok { + let err = last_error + .map(|e| e.to_string()) + .unwrap_or_else(|| "no relays".to_string()); return CheckResult::skip( "relay.mev_delivery_rate", 2, - format!( - "No relay supports the data API. Last error: {}", - last_error.unwrap() - ), + format!("No relay supports the data API. Last error: {err}"), ); } @@ -147,8 +151,17 @@ pub async fn check_mev_delivery_rate( let mut total_blocks = 0u64; let mut missed = 0u64; - for slot in start_slot..=end_slot { - match beacon.get_block_hash(slot).await { + // Gather the per-slot block hashes concurrently (bounded). The counting fold + // below is identical to the old serial loop — order-independent counters, so + // buffer_unordered's out-of-order completion does not change the result. + let fetched: Vec<_> = futures::stream::iter(start_slot..=end_slot) + .map(|slot| async move { (slot, beacon.get_block_hash(slot).await) }) + .buffer_unordered(16) + .collect() + .await; + + for (_slot, res) in fetched { + match res { Ok(None) => missed += 1, Err(_) => missed += 1, Ok(Some(hash)) => { @@ -160,6 +173,28 @@ pub async fn check_mev_delivery_rate( } } + // Delegate the verdict to the pure classifier, then attach the missed-slot + // count (context the rate logic doesn't need to reach a verdict, but the + // report records — mirrors chain_health::check_missed_slots). + let mut result = classify_mev_rate(mev_blocks, total_blocks, threshold); + if let Some(obj) = result.data.as_object_mut() { + obj.insert("missed_slots".to_string(), serde_json::json!(missed)); + } + result +} + +/// Classify the MEV delivery rate verdict from the delivered/proposed counts. +/// +/// Pure decision core extracted from [`check_mev_delivery_rate`] (the P3 pattern +/// — see `chain_health::classify_missed_slots`): +/// - `total_blocks == 0` => FAIL (no proposed blocks to measure against) +/// - `rate >= threshold` => PASS +/// - otherwise => WARN +/// +/// `rate = mev_blocks / total_blocks` (0.0 when there are no proposed blocks). +/// The caller ([`check_mev_delivery_rate`]) attaches the `missed_slots` count to +/// the returned `data` payload afterward. +pub fn classify_mev_rate(mev_blocks: u64, total_blocks: u64, threshold: f64) -> CheckResult { let rate = if total_blocks > 0 { mev_blocks as f64 / total_blocks as f64 } else { @@ -169,7 +204,6 @@ pub async fn check_mev_delivery_rate( let data = serde_json::json!({ "mev_blocks": mev_blocks, "total_blocks": total_blocks, - "missed_slots": missed, "rate": (rate * 10000.0).round() / 10000.0, }); @@ -202,9 +236,10 @@ pub async fn check_mev_delivery_rate( /// Check validator registrations with the relay (tier 3). /// -/// For each pubkey, calls `is_validator_registered`. PASS if all registered, -/// WARN if some missing, FAIL if none registered. The caller should SKIP -/// outright if the pubkey list is empty (we also handle that defensively). +/// For each pubkey, queries the relay's validator_registration endpoint. PASS +/// if all registered, WARN if some missing, FAIL if none registered. The caller +/// should SKIP outright if the pubkey list is empty (we also handle that +/// defensively). pub async fn check_validator_registrations( relay_url: &str, client: &reqwest::Client, @@ -249,39 +284,210 @@ pub async fn check_validator_registrations( "missing": missing, }); - if reg_count == total { + // The verdict + detail come from the pure classifier; the caller owns the + // `data` payload (it holds the missing-pubkey vector the counts can't carry). + classify_registrations(reg_count, missing.len()).with_data(data) +} + +/// Classify the validator-registration verdict from the registered/missing counts. +/// +/// Pure decision core extracted from [`check_validator_registrations`] (the P3 +/// pattern). `total = registered + missing`: +/// - all registered (`missing == 0`) => PASS +/// - some registered, some missing => WARN +/// - none registered => FAIL +/// +/// Returns the verdict with an empty `data` payload; the caller attaches the +/// full payload (which includes the list of missing pubkeys) via `with_data`. +pub fn classify_registrations(registered: usize, missing: usize) -> CheckResult { + let total = registered + missing; + + if registered == total { CheckResult::pass( "relay.validator_registrations", 3, format!("All {total} validator(s) registered on relay"), ) - .with_data(data) - } else if reg_count > 0 { + } else if registered > 0 { CheckResult::warn( "relay.validator_registrations", 3, - format!( - "{reg_count}/{total} validator(s) registered; {} missing", - missing.len() - ), + format!("{registered}/{total} validator(s) registered; {missing} missing"), ) - .with_data(data) } else { CheckResult::fail( "relay.validator_registrations", 3, format!("No validators registered on relay (0/{total})"), ) - .with_data(data) } } +/// Verdicts when EVERY relay in the enclave is unreachable at check time. This is +/// NOT benign: the relays were launched and the chain observed a full epoch, so +/// all-dead means the MEV pipeline died mid-run (e.g. relay OOM). The tier-1 +/// delivery check must therefore FAIL, not SKIP — a tier-1 SKIP is treated as +/// PASS by `report::exit_code`, which would green a run whose relays died (the C1 +/// false-green). The tier-2/3 checks stay SKIP (the tier-1 FAIL already gates). +fn all_relays_dead_results( + total: usize, + dead_urls: &[String], + has_pubkeys: bool, +) -> Vec { + let detail = format!( + "All {total} relay(s) unreachable at check time ({}) — the MEV pipeline is down: relays died \ + or never served during the observation window", + dead_urls.join(", ") + ); + let mut out = vec![ + CheckResult::fail("relay.payloads_delivered_multi", 1, detail.clone()), + CheckResult::skip("relay.builder_blocks_received", 2, detail.clone()), + CheckResult::skip("relay.mev_delivery_rate", 2, detail.clone()), + ]; + if has_pubkeys { + out.push(CheckResult::skip( + "relay.validator_registrations", + 3, + detail, + )); + } + out +} + /// Run all relay pipeline checks. /// -/// Probes each relay before running the check batch. Unreachable relays -/// produce a single SKIP per downstream check instead of multiple FAILs -/// with "error sending request" noise. This catches mid-run relay crashes -/// that the startup preflight couldn't. +/// Probes each relay before running the check batch. If ALL relays are unreachable +/// at check time (mid-run death the startup preflight couldn't catch), the tier-1 +/// delivery check FAILs (see `all_relays_dead_results`); otherwise the batch runs +/// against the live relays. +/// Aggregate per-relay `builder_blocks_received` results (pure, Law 4 seam). +/// +/// PASS if ANY relay received blocks (the builder only submits to the relays it +/// is configured for, so one silent relay is not a pipeline failure), summing +/// the counts. Otherwise surface the WORST result via `CheckStatus: Ord` +/// (Fail > Warn > Pass > Skip). Empty input cannot happen from the caller (it is +/// guarded by a non-empty live-relay list) but is handled as a SKIP rather than +/// panicking on `.max().unwrap()`. +pub fn aggregate_builder_blocks(results: Vec) -> CheckResult { + if results.is_empty() { + return CheckResult::skip( + "relay.builder_blocks_received", + 2, + "No live relays to query", + ); + } + if results.iter().any(|r| r.status == CheckStatus::Pass) { + let total: u64 = results + .iter() + .map(|r| r.data.get("count").and_then(|c| c.as_u64()).unwrap_or(0)) + .sum(); + let details: Vec<&str> = results.iter().map(|r| r.detail.as_str()).collect(); + CheckResult::pass("relay.builder_blocks_received", 2, details.join("; ")) + .with_data(serde_json::json!({ "count": total })) + } else { + // No Pass present by construction; `.max()` picks Fail over Warn over Skip. + results.into_iter().max_by_key(|r| r.status).unwrap() + } +} + +/// Aggregate `mev_delivery_rate` results (pure, Law 4 seam). +/// +/// This is a BEST-of aggregation (Pass wins), the OPPOSITE of the worst-status +/// `CheckStatus: Ord`, and it deliberately collapses Skip => Fail: a Skip here +/// means no relay could answer the data API at all, which is a failure to +/// measure MEV delivery, not a benign no-op. Kept separate from `.max()` for +/// exactly those two reasons. +pub fn aggregate_mev_rate(results: Vec) -> CheckResult { + let best = if results.iter().any(|r| r.status == CheckStatus::Pass) { + CheckStatus::Pass + } else if results.iter().any(|r| r.status == CheckStatus::Warn) { + CheckStatus::Warn + } else { + CheckStatus::Fail + }; + let sum = |key: &str| -> u64 { + results + .iter() + .map(|r| r.data.get(key).and_then(|c| c.as_u64()).unwrap_or(0)) + .sum() + }; + let total_mev = sum("mev_blocks"); + let total_blocks = sum("total_blocks"); + let details: Vec<&str> = results.iter().map(|r| r.detail.as_str()).collect(); + let data = serde_json::json!({ + "mev_blocks": total_mev, + "total_blocks": total_blocks, + "rate": if total_blocks > 0 { + (total_mev as f64 / total_blocks as f64 * 10000.0).round() / 10000.0 + } else { 0.0 }, + }); + let joined = details.join("; "); + match best { + CheckStatus::Pass => CheckResult::pass( + "relay.mev_delivery_rate", + 2, + format!("MEV delivery rate across all relays: {joined}"), + ), + CheckStatus::Warn => CheckResult::warn( + "relay.mev_delivery_rate", + 2, + format!("MEV delivery rate below threshold: {joined}"), + ), + _ => CheckResult::fail( + "relay.mev_delivery_rate", + 2, + format!("No MEV deliveries across any relay: {joined}"), + ), + } + .with_data(data) +} + +/// Aggregate per-relay `validator_registrations` to the WORST status (pure, +/// Law 4 seam). `urls` is parallel to `per_relay` and only labels the detail. +/// Returns `None` for empty input (the caller then emits no check at all, which +/// is the documented "omitted entirely, not even SKIP" behavior). +pub fn aggregate_registrations(per_relay: &[CheckResult], urls: &[String]) -> Option { + if per_relay.is_empty() { + return None; + } + let worst = per_relay + .iter() + .map(|r| r.status) + .max() + .unwrap_or(CheckStatus::Pass); + let combined_detail = per_relay + .iter() + .enumerate() + .map(|(i, r)| { + let url = urls.get(i).map(String::as_str).unwrap_or("?"); + format!("[{url}] {}", r.detail) + }) + .collect::>() + .join("; "); + let data = serde_json::json!({ + "per_relay": per_relay + .iter() + .enumerate() + .map(|(i, r)| serde_json::json!({ + "relay": urls.get(i).map(String::as_str).unwrap_or("?"), + "status": r.status.to_string(), + "detail": r.detail, + "data": r.data, + })) + .collect::>(), + }); + let id = "relay.validator_registrations"; + Some( + match worst { + CheckStatus::Pass => CheckResult::pass(id, 3, combined_detail), + CheckStatus::Warn => CheckResult::warn(id, 3, combined_detail), + CheckStatus::Fail => CheckResult::fail(id, 3, combined_detail), + CheckStatus::Skip => CheckResult::skip(id, 3, combined_detail), + } + .with_data(data), + ) +} + pub async fn run_relay_checks( relays: &[RelayClient], beacon: &BeaconClient, @@ -307,29 +513,11 @@ pub async fn run_relay_checks( } if live_relays.is_empty() && !relays.is_empty() { - let detail = format!( - "All {} relay(s) unreachable at check time: {}", + results.extend(all_relays_dead_results( relays.len(), - dead_urls.join(", ") - ); - results.push(CheckResult::skip( - "relay.builder_blocks_received", - 2, - &detail, + &dead_urls, + !pubkeys.is_empty(), )); - results.push(CheckResult::skip( - "relay.payloads_delivered_multi", - 1, - &detail, - )); - results.push(CheckResult::skip("relay.mev_delivery_rate", 2, &detail)); - if !pubkeys.is_empty() { - results.push(CheckResult::skip( - "relay.validator_registrations", - 3, - &detail, - )); - } return results; } @@ -346,29 +534,7 @@ pub async fn run_relay_checks( for relay in &live { bb_results.push(check_builder_blocks_received(relay, start_slot, end_slot).await); } - let any_pass = bb_results.iter().any(|r| r.status == CheckStatus::Pass); - if any_pass { - let total: usize = bb_results - .iter() - .map(|r| r.data.get("count").and_then(|c| c.as_u64()).unwrap_or(0) as usize) - .sum(); - let details: Vec<&str> = bb_results.iter().map(|r| r.detail.as_str()).collect(); - results.push( - CheckResult::pass( - "relay.builder_blocks_received", - 2, - format!("{}", details.join("; ")), - ) - .with_data(serde_json::json!({"count": total})), - ); - } else { - let worst = bb_results.into_iter().max_by_key(|r| match r.status { - CheckStatus::Fail => 2, - CheckStatus::Warn => 1, - _ => 0, - }).unwrap(); - results.push(worst); - } + results.push(aggregate_builder_blocks(bb_results)); } results.push(check_payloads_delivered_multi(&live, start_slot, end_slot).await); @@ -381,47 +547,7 @@ pub async fn run_relay_checks( mv_results.push( check_mev_delivery_rate(&live, beacon, start_slot, end_slot, mev_threshold).await, ); - let any_pass = mv_results.iter().any(|r| r.status == CheckStatus::Pass); - let best_status = if any_pass { - CheckStatus::Pass - } else if mv_results.iter().any(|r| r.status == CheckStatus::Warn) { - CheckStatus::Warn - } else { - CheckStatus::Fail - }; - let total_mev: u64 = mv_results - .iter() - .map(|r| r.data.get("mev_blocks").and_then(|c| c.as_u64()).unwrap_or(0)) - .sum(); - let total_blocks: u64 = mv_results - .iter() - .map(|r| r.data.get("total_blocks").and_then(|c| c.as_u64()).unwrap_or(0)) - .sum(); - let details: Vec<&str> = mv_results.iter().map(|r| r.detail.as_str()).collect(); - let data = serde_json::json!({ - "mev_blocks": total_mev, - "total_blocks": total_blocks, - "rate": if total_blocks > 0 { - (total_mev as f64 / total_blocks as f64 * 10000.0).round() / 10000.0 - } else { 0.0 }, - }); - results.push(match best_status { - CheckStatus::Pass => CheckResult::pass( - "relay.mev_delivery_rate", - 2, - format!("MEV delivery rate across all relays: {}", details.join("; ")), - ), - CheckStatus::Warn => CheckResult::warn( - "relay.mev_delivery_rate", - 2, - format!("MEV delivery rate below threshold: {}", details.join("; ")), - ), - _ => CheckResult::fail( - "relay.mev_delivery_rate", - 2, - format!("No MEV deliveries across any relay: {}", details.join("; ")), - ), - }.with_data(data)); + results.push(aggregate_mev_rate(mv_results)); } // Tier 3: per-relay validator registration, aggregated to the worst status. @@ -431,55 +557,302 @@ pub async fn run_relay_checks( per_relay .push(check_validator_registrations(relay.base_url(), http_client, pubkeys).await); } - // Aggregate: FAIL > WARN > PASS; details comma-joined. - if !per_relay.is_empty() { - let worst = - per_relay - .iter() - .map(|r| r.status) - .fold(CheckStatus::Pass, |acc, s| match (acc, s) { - (CheckStatus::Fail, _) | (_, CheckStatus::Fail) => CheckStatus::Fail, - (CheckStatus::Warn, _) | (_, CheckStatus::Warn) => CheckStatus::Warn, - (CheckStatus::Skip, CheckStatus::Pass) - | (CheckStatus::Pass, CheckStatus::Skip) => CheckStatus::Pass, - (a, _) => a, - }); - let combined_detail = per_relay - .iter() - .enumerate() - .map(|(i, r)| format!("[{}] {}", live[i].base_url(), r.detail)) - .collect::>() - .join("; "); - let data = serde_json::json!({ - "per_relay": per_relay - .iter() - .zip(live.iter()) - .map(|(r, relay)| serde_json::json!({ - "relay": relay.base_url(), - "status": r.status.to_string(), - "detail": r.detail, - "data": r.data, - })) - .collect::>(), - }); - let agg = match worst { - CheckStatus::Pass => { - CheckResult::pass("relay.validator_registrations", 3, combined_detail) - } - CheckStatus::Warn => { - CheckResult::warn("relay.validator_registrations", 3, combined_detail) - } - CheckStatus::Fail => { - CheckResult::fail("relay.validator_registrations", 3, combined_detail) - } - CheckStatus::Skip => { - CheckResult::skip("relay.validator_registrations", 3, combined_detail) - } - } - .with_data(data); + let urls: Vec = live.iter().map(|r| r.base_url().to_string()).collect(); + if let Some(agg) = aggregate_registrations(&per_relay, &urls) { results.push(agg); } } results } + +#[cfg(test)] +mod tests { + use super::*; + + // --- aggregation seams (Law 4): these decide tier-1/2 verdicts across + // relays and were previously unreachable without a live devnet ---------- + + fn res(id: &str, st: CheckStatus, detail: &str, data: serde_json::Value) -> CheckResult { + let r = match st { + CheckStatus::Pass => CheckResult::pass(id, 2, detail), + CheckStatus::Fail => CheckResult::fail(id, 2, detail), + CheckStatus::Warn => CheckResult::warn(id, 2, detail), + CheckStatus::Skip => CheckResult::skip(id, 2, detail), + }; + r.with_data(data) + } + + #[test] + fn builder_blocks_one_pass_wins_and_sums_counts() { + // The builder only submits to the relays it is configured for, so one + // silent relay must not fail the check; counts sum across relays. + let out = aggregate_builder_blocks(vec![ + res( + "x", + CheckStatus::Pass, + "r0: 10", + serde_json::json!({"count": 10}), + ), + res( + "x", + CheckStatus::Fail, + "r1: none", + serde_json::json!({"count": 0}), + ), + ]); + assert_eq!(out.status, CheckStatus::Pass); + assert_eq!(out.data["count"], 10); + assert!( + out.detail.contains("r0") && out.detail.contains("r1"), + "both detailed" + ); + } + + #[test] + fn builder_blocks_no_pass_surfaces_the_worst() { + let out = aggregate_builder_blocks(vec![ + res("x", CheckStatus::Warn, "r0 warn", serde_json::json!({})), + res("x", CheckStatus::Fail, "r1 fail", serde_json::json!({})), + ]); + assert_eq!(out.status, CheckStatus::Fail, "Fail beats Warn"); + assert_eq!(out.detail, "r1 fail"); + } + + #[test] + fn builder_blocks_empty_skips_instead_of_panicking() { + // Guards the old `.max().unwrap()` on an empty vec. + let out = aggregate_builder_blocks(vec![]); + assert_eq!(out.status, CheckStatus::Skip); + } + + #[test] + fn mev_rate_is_best_of_not_worst_of() { + // Deliberately the OPPOSITE of CheckStatus::Ord: one relay measuring a + // healthy rate is enough. + let out = aggregate_mev_rate(vec![ + res( + "m", + CheckStatus::Fail, + "r0 none", + serde_json::json!({"mev_blocks": 0, "total_blocks": 10}), + ), + res( + "m", + CheckStatus::Pass, + "r1 ok", + serde_json::json!({"mev_blocks": 8, "total_blocks": 10}), + ), + ]); + assert_eq!(out.status, CheckStatus::Pass, "Pass wins in best-of"); + assert_eq!(out.data["mev_blocks"], 8); + assert_eq!(out.data["total_blocks"], 20); + assert_eq!(out.data["rate"], 0.4); + } + + #[test] + fn mev_rate_collapses_skip_to_fail() { + // A Skip means NO relay could answer the data API: that is a failure to + // measure MEV delivery, not a benign no-op. This is why the seam is not + // unified with the worst-status `.max()` (which would rank Skip lowest). + let out = aggregate_mev_rate(vec![res( + "m", + CheckStatus::Skip, + "no data api", + serde_json::json!({}), + )]); + assert_eq!(out.status, CheckStatus::Fail); + assert!(out.detail.contains("No MEV deliveries")); + } + + #[test] + fn mev_rate_warn_when_only_warns() { + let out = aggregate_mev_rate(vec![res( + "m", + CheckStatus::Warn, + "below threshold", + serde_json::json!({"mev_blocks": 1, "total_blocks": 10}), + )]); + assert_eq!(out.status, CheckStatus::Warn); + assert_eq!(out.data["rate"], 0.1); + } + + #[test] + fn mev_rate_zero_blocks_does_not_divide_by_zero() { + let out = aggregate_mev_rate(vec![res( + "m", + CheckStatus::Fail, + "nothing", + serde_json::json!({"mev_blocks": 0, "total_blocks": 0}), + )]); + assert_eq!(out.data["rate"], 0.0); + } + + #[test] + fn registrations_takes_the_worst_and_labels_each_relay() { + let urls = vec!["http://a".to_string(), "http://b".to_string()]; + let out = aggregate_registrations( + &[ + res( + "v", + CheckStatus::Pass, + "all registered", + serde_json::json!({}), + ), + res("v", CheckStatus::Warn, "3 missing", serde_json::json!({})), + ], + &urls, + ) + .expect("non-empty input yields a check"); + assert_eq!(out.status, CheckStatus::Warn, "worst wins"); + assert!(out.detail.contains("[http://a]") && out.detail.contains("[http://b]")); + assert_eq!(out.data["per_relay"].as_array().unwrap().len(), 2); + assert_eq!(out.tier, 3); + } + + #[test] + fn registrations_fail_beats_warn() { + let urls = vec!["http://a".to_string(), "http://b".to_string()]; + let out = aggregate_registrations( + &[ + res( + "v", + CheckStatus::Warn, + "some missing", + serde_json::json!({}), + ), + res( + "v", + CheckStatus::Fail, + "none registered", + serde_json::json!({}), + ), + ], + &urls, + ) + .unwrap(); + assert_eq!(out.status, CheckStatus::Fail); + } + + #[test] + fn registrations_empty_emits_no_check_at_all() { + // Documented behavior: omitted entirely, not even a SKIP. + assert!(aggregate_registrations(&[], &[]).is_none()); + } + + #[test] + fn registrations_tolerates_short_url_list() { + // Defensive: a urls/per_relay length mismatch must not panic. + let out = aggregate_registrations( + &[res("v", CheckStatus::Pass, "ok", serde_json::json!({}))], + &[], + ) + .unwrap(); + assert!(out.detail.contains("[?]")); + } + + #[test] + fn all_relays_dead_fails_tier1_not_skip() { + // C1: relays that die mid-run must FAIL the tier-1 delivery check, not SKIP + // (a tier-1 SKIP is treated as pass by exit_code → false green). + let r = all_relays_dead_results(2, &["u1".into(), "u2".into()], true); + let t1 = r + .iter() + .find(|c| c.id == "relay.payloads_delivered_multi") + .expect("tier-1 delivery check present"); + assert_eq!(t1.tier, 1); + assert_eq!(t1.status, CheckStatus::Fail); + // The others stay SKIP (tier-1 FAIL already gates the run). + assert!( + r.iter() + .filter(|c| c.tier != 1) + .all(|c| c.status == CheckStatus::Skip) + ); + } + + // --- classify_mev_rate (seam 1) ------------------------------------- + + // Contract: no proposed blocks (total == 0) FAILs — there's nothing to + // measure a delivery rate against, so a 0-rate must not silently PASS/WARN. + #[test] + fn mev_rate_zero_total_fails() { + let r = classify_mev_rate(0, 0, 0.5); + assert_eq!(r.status, CheckStatus::Fail); + assert_eq!(r.id, "relay.mev_delivery_rate"); + assert_eq!(r.tier, 2); + assert_eq!(r.detail, "No proposed blocks found"); + assert_eq!(r.data["mev_blocks"], 0); + assert_eq!(r.data["total_blocks"], 0); + assert_eq!(r.data["rate"], 0.0); + } + + // Contract: a rate exactly AT the threshold PASSes (boundary is `>=`). + #[test] + fn mev_rate_at_threshold_passes() { + // 5/10 = 0.5 == 0.5 + let r = classify_mev_rate(5, 10, 0.5); + assert_eq!(r.status, CheckStatus::Pass); + assert!(r.detail.contains(">=")); + assert_eq!(r.data["mev_blocks"], 5); + assert_eq!(r.data["total_blocks"], 10); + assert_eq!(r.data["rate"], 0.5); + } + + // Contract: a rate ABOVE the threshold PASSes. + #[test] + fn mev_rate_above_threshold_passes() { + // 9/10 = 0.9 > 0.5 + let r = classify_mev_rate(9, 10, 0.5); + assert_eq!(r.status, CheckStatus::Pass); + assert_eq!(r.data["rate"], 0.9); + } + + // Contract: a rate BELOW the threshold WARNs (not FAIL — deliveries exist, + // just under target). + #[test] + fn mev_rate_below_threshold_warns() { + // 2/10 = 0.2 < 0.5 + let r = classify_mev_rate(2, 10, 0.5); + assert_eq!(r.status, CheckStatus::Warn); + assert!(r.detail.contains("below")); + assert_eq!(r.data["rate"], 0.2); + } + + // Contract: the classifier's data payload carries no `missed_slots` (the + // caller, check_mev_delivery_rate, attaches it) but everything else matches. + #[test] + fn mev_rate_classifier_omits_missed_slots() { + let r = classify_mev_rate(1, 4, 0.5); + assert!(r.data.get("missed_slots").is_none()); + assert_eq!(r.data["rate"], 0.25); + } + + // --- classify_registrations (seam 2) -------------------------------- + + // Contract: every validator registered (missing == 0) PASSes. + #[test] + fn registrations_all_registered_passes() { + let r = classify_registrations(3, 0); + assert_eq!(r.status, CheckStatus::Pass); + assert_eq!(r.id, "relay.validator_registrations"); + assert_eq!(r.tier, 3); + assert!(r.detail.contains("All 3 validator(s) registered")); + } + + // Contract: some registered, some missing WARNs, and the counts appear. + #[test] + fn registrations_some_missing_warns() { + let r = classify_registrations(2, 1); + assert_eq!(r.status, CheckStatus::Warn); + // total = registered + missing = 3 + assert!(r.detail.contains("2/3 validator(s) registered; 1 missing")); + } + + // Contract: none registered FAILs (0/total), even when total > 0. + #[test] + fn registrations_none_registered_fails() { + let r = classify_registrations(0, 4); + assert_eq!(r.status, CheckStatus::Fail); + assert!(r.detail.contains("No validators registered on relay (0/4)")); + } +} diff --git a/src/checks/signer.rs b/src/checks/signer.rs new file mode 100644 index 0000000..f10b605 --- /dev/null +++ b/src/checks/signer.rs @@ -0,0 +1,288 @@ +//! Commit-Boost SIGNER module checks. +//! +//! The signer has never been runnable on a Kurtosis devnet (the ethereum-package +//! had no config support for it), so this is the first assertion of it here. +//! +//! **What we deliberately do NOT assert.** `GET /status` is +//! `Ok(StatusCode::OK)` with no logic - it returns 200 with zero keys loaded - +//! and the metrics server exposes a SECOND unconditional `/status`, so probing +//! the wrong port is an even emptier green. The startup log's +//! `loaded_consensus=N` is log-only (the signer registers exactly one metric, +//! `signer_status_code_total`, with no key-count gauge) and is ANSI-colored by +//! default, so the field is not even a contiguous substring. +//! +//! **What we assert instead:** a JWT-authenticated `GET /signer/v1/get_pubkeys` +//! with a COUNT assertion. One HTTP call subsumes liveness, module registration, +//! JWT auth AND key loading - and it is by construction the assertion that fails +//! if the keystore mount is unreadable, which is the failure this whole feature +//! is most likely to hit (CB's loader is `filter_map` + `warn!`, so a permissions +//! problem yields a healthy process holding zero keys). + +use base64::Engine; +use hmac::{Hmac, Mac}; +use serde::Deserialize; +use sha2::Sha256; + +use crate::checks::CheckResult; + +/// CB's module-API route for listing consensus pubkeys. The JWT is bound to +/// this exact string; a mismatch is rejected. +pub const GET_PUBKEYS_ROUTE: &str = "/signer/v1/get_pubkeys"; + +/// CB's JWT lifetime (`SIGNER_JWT_EXPIRATION`); validation allows 10s leeway. +const JWT_EXPIRATION_SECS: u64 = 300; + +fn b64(bytes: &[u8]) -> String { + base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(bytes) +} + +/// Mint the HS256 JWT a commit module presents to the signer (pure). +/// +/// Claims are `{module, route, exp, payload_hash}`: +/// - `route` MUST equal the exact request path, else the signer rejects it. +/// - `payload_hash` MUST be `null` when the request has no body, and +/// `keccak256(body)` when it does. Both directions are enforced by CB, so a +/// "harmless" empty-string default would fail auth. +/// +/// `now_secs` is a parameter rather than read from the clock so the output is +/// deterministic and testable. +pub fn mint_module_jwt( + module_id: &str, + secret: &str, + route: &str, + payload_hash: Option<&str>, + now_secs: u64, +) -> String { + let header = br#"{"alg":"HS256","typ":"JWT"}"#; + let claims = serde_json::json!({ + "module": module_id, + "route": route, + "exp": now_secs + JWT_EXPIRATION_SECS, + "payload_hash": payload_hash, + }); + let signing_input = format!("{}.{}", b64(header), b64(claims.to_string().as_bytes())); + + let mut mac = >::new_from_slice(secret.as_bytes()) + .expect("HMAC accepts a key of any length"); + mac.update(signing_input.as_bytes()); + let sig = mac.finalize().into_bytes(); + + format!("{signing_input}.{}", b64(&sig)) +} + +/// The shape of `GET /signer/v1/get_pubkeys`. +#[derive(Debug, Deserialize)] +pub struct PubkeysResponse { + pub keys: Vec, +} + +#[derive(Debug, Deserialize)] +pub struct PubkeyEntry { + pub consensus: String, +} + +/// Pure verdict for the signer key-loading assertion (Law 4 seam). +/// +/// `expected` is how many validator keys the participant's keystore artifact +/// holds; `got` is what the signer actually reports. +/// +/// A count of ZERO is the signature failure mode of this feature: CB's keystore +/// loaders skip unreadable or malformed entries with `filter_map` + `warn!`, so +/// a permissions problem (the devnet's `secrets/` dir is mode 600 and +/// root-owned) produces a perfectly healthy signer that has loaded nothing. +pub fn classify_signer_pubkeys(expected: usize, got: usize) -> CheckResult { + let id = "signer.pubkeys"; + let data = serde_json::json!({ "expected_keys": expected, "loaded_keys": got }); + + if got == 0 { + return CheckResult::fail( + id, + 1, + format!( + "the signer authenticated but loaded ZERO keys (expected {expected}). CB's keystore \ + loaders skip unreadable entries silently, so suspect the mount: the devnet's \ + `secrets/` dir is mode 600 root-owned and unreadable by the container's uid 10001 \ + - use the teku-keys/teku-secrets pair" + ), + ) + .with_data(data); + } + if got != expected { + return CheckResult::warn( + id, + 1, + format!( + "signer loaded {got} key(s) but the keystore artifact holds {expected} - some \ + keystores were skipped (CB warns and continues per key)" + ), + ) + .with_data(data); + } + CheckResult::pass( + id, + 1, + format!("signer loaded all {got} validator key(s) and authenticated the module JWT ✓"), + ) + .with_data(data) +} + +/// Pure verdict for the JWT negative control. +/// +/// Run this LAST: CB rate-limits a source IP after `jwt_auth_fail_limit` (3) +/// failures for `jwt_auth_fail_timeout_seconds`, and every harness request +/// arrives from the same NAT address, so probing it early would 429 the positive +/// assertions that follow. +pub fn classify_jwt_rejection(status: u16) -> CheckResult { + let id = "signer.jwt_auth"; + let data = serde_json::json!({ "status": status }); + match status { + 401 => CheckResult::pass(id, 2, "a bad module JWT is rejected with 401 ✓").with_data(data), + 429 => CheckResult::warn( + id, + 2, + "rate-limited (429) before the negative control could be observed - run negative \ + probes last, and lower CB_SIGNER_JWT_AUTH_FAIL_TIMEOUT_SECONDS", + ) + .with_data(data), + 200 => CheckResult::fail( + id, + 2, + "a BAD module JWT was ACCEPTED (200) - signer authentication is not enforced", + ) + .with_data(data), + other => CheckResult::warn( + id, + 2, + format!("unexpected status {other} for a bad JWT (expected 401)"), + ) + .with_data(data), + } +} + +/// Ask the signer for its pubkeys with a freshly minted module JWT. +pub async fn fetch_pubkeys( + client: &reqwest::Client, + signer_url: &str, + module_id: &str, + secret: &str, + now_secs: u64, +) -> eyre::Result<(u16, Option)> { + // No body on this route, so payload_hash MUST be null. + let jwt = mint_module_jwt(module_id, secret, GET_PUBKEYS_ROUTE, None, now_secs); + let resp = client + .get(format!( + "{}{GET_PUBKEYS_ROUTE}", + signer_url.trim_end_matches('/') + )) + .bearer_auth(jwt) + .timeout(std::time::Duration::from_secs(5)) + .send() + .await?; + let status = resp.status().as_u16(); + if status != 200 { + return Ok((status, None)); + } + Ok((status, Some(resp.json().await?))) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn decode_claims(jwt: &str) -> serde_json::Value { + let payload = jwt.split('.').nth(1).expect("jwt has 3 parts"); + let raw = base64::engine::general_purpose::URL_SAFE_NO_PAD + .decode(payload) + .expect("payload is base64url"); + serde_json::from_slice(&raw).expect("payload is json") + } + + #[test] + fn jwt_has_three_parts_and_is_deterministic() { + let a = mint_module_jwt("TEST_MODULE", "secret", GET_PUBKEYS_ROUTE, None, 1000); + let b = mint_module_jwt("TEST_MODULE", "secret", GET_PUBKEYS_ROUTE, None, 1000); + assert_eq!(a, b, "same inputs must give the same token"); + assert_eq!(a.split('.').count(), 3); + } + + #[test] + fn jwt_binds_the_exact_route() { + // CB compares `route` against the request path and rejects a mismatch, + // so a token minted for one route cannot be replayed on another. + let c = decode_claims(&mint_module_jwt("m", "s", GET_PUBKEYS_ROUTE, None, 0)); + assert_eq!(c["route"], GET_PUBKEYS_ROUTE); + let other = decode_claims(&mint_module_jwt("m", "s", "/signer/v1/other", None, 0)); + assert_eq!(other["route"], "/signer/v1/other"); + } + + #[test] + fn payload_hash_is_null_when_there_is_no_body() { + // Enforced in BOTH directions by CB: a non-null hash on a bodyless + // request is rejected just as a missing one on a request with a body is. + let c = decode_claims(&mint_module_jwt("m", "s", GET_PUBKEYS_ROUTE, None, 0)); + assert!( + c["payload_hash"].is_null(), + "must be null, not \"\" or absent" + ); + + let c2 = decode_claims(&mint_module_jwt("m", "s", "/r", Some("0xabc"), 0)); + assert_eq!(c2["payload_hash"], "0xabc"); + } + + #[test] + fn jwt_expiry_is_five_minutes_out() { + let c = decode_claims(&mint_module_jwt("m", "s", "/r", None, 1_000_000)); + assert_eq!(c["exp"], 1_000_000 + 300); + } + + #[test] + fn jwt_signature_changes_with_the_secret() { + let a = mint_module_jwt("m", "secret-a", "/r", None, 0); + let b = mint_module_jwt("m", "secret-b", "/r", None, 0); + assert_ne!( + a, b, + "a different module secret must produce a different token" + ); + // Only the signature differs; the claims are identical. + assert_eq!(a.rsplit_once('.').unwrap().0, b.rsplit_once('.').unwrap().0); + } + + #[test] + fn zero_keys_fails_and_names_the_permissions_trap() { + // The signature failure of this feature: healthy process, no keys. + let r = classify_signer_pubkeys(128, 0); + assert_eq!(r.status, crate::checks::CheckStatus::Fail); + assert!(r.detail.contains("ZERO keys")); + assert!(r.detail.contains("teku"), "points at the fix: {}", r.detail); + } + + #[test] + fn partial_key_load_warns() { + let r = classify_signer_pubkeys(128, 100); + assert_eq!(r.status, crate::checks::CheckStatus::Warn); + assert_eq!(r.data["loaded_keys"], 100); + } + + #[test] + fn full_key_load_passes() { + let r = classify_signer_pubkeys(128, 128); + assert_eq!(r.status, crate::checks::CheckStatus::Pass); + } + + #[test] + fn jwt_negative_control_verdicts() { + assert_eq!( + classify_jwt_rejection(401).status, + crate::checks::CheckStatus::Pass + ); + // Accepting a bad JWT is the security-relevant failure. + assert_eq!( + classify_jwt_rejection(200).status, + crate::checks::CheckStatus::Fail + ); + // 429 means we poisoned ourselves by probing too early. + let limited = classify_jwt_rejection(429); + assert_eq!(limited.status, crate::checks::CheckStatus::Warn); + assert!(limited.detail.contains("run negative probes last")); + } +} diff --git a/src/discovery.rs b/src/discovery.rs index 0759a94..4d3dd42 100644 --- a/src/discovery.rs +++ b/src/discovery.rs @@ -12,14 +12,6 @@ use std::process::Command; use eyre::{Result, WrapErr, bail}; use tracing::{debug, info, warn}; -/// A single payload-delivered record from post-mortem Postgres query. -#[derive(Debug, Clone, serde::Serialize)] -pub struct PostMortemRecord { - pub slot: u64, - pub block_hash: String, - pub value: String, -} - /// Derive a relay identity from the Kurtosis service name. /// /// Returns a short string like "helix", "flashbots", or "mev-rs" @@ -43,13 +35,14 @@ pub fn relay_identity(service_name: &str) -> Option { pub struct EnclaveServices { pub beacon_urls: Vec, pub relay_urls: Vec, - /// Parallel to relay_urls: identity string per relay - /// ("helix", "flashbots", "mev-rs", etc.) - pub relay_identities: Vec, pub cb_pbs_urls: Vec, pub cb_metrics_urls: Vec, pub cb_service_names: Vec, pub prometheus_url: Option, + /// Commit-Boost SIGNER module endpoints (`cb-signer-*`). Deliberately a + /// SEPARATE pattern from `commit-boost-*`: that glob feeds cb_service_names, + /// which three checks iterate while shelling out a 200k-line log fetch each. + pub signer_urls: Vec, } /// Run a kurtosis CLI command and return stdout. @@ -96,7 +89,7 @@ fn port_print(enclave: &str, service: &str, port_name: &str) -> Option { } /// A parsed service from kurtosis inspect output. -struct ParsedService { +pub struct ParsedService { name: String, ports: Vec<(String, String)>, // (port_name, url) } @@ -157,7 +150,10 @@ fn parse_services(inspect_output: &str) -> Vec { } /// Split a string on runs of 2+ whitespace characters. -fn split_on_multi_space(s: &str) -> Vec<&str> { +/// +/// Public so other bins (e.g. `sim triage`) can reuse the exact column split +/// used to read the kurtosis `User Services` table, rather than re-deriving it. +pub fn split_on_multi_space(s: &str) -> Vec<&str> { let mut result = Vec::new(); let mut start = None; let mut space_count = 0; @@ -287,102 +283,121 @@ fn matches_pattern(name: &str, pattern: &str) -> bool { /// Discover all relevant services in a Kurtosis enclave. pub fn discover(enclave: &str) -> Result { - let mut result = EnclaveServices::default(); - let inspect_output = run_kurtosis(&["enclave", "inspect", "--full-uuids", enclave]) .wrap_err_with(|| format!("Could not inspect enclave '{enclave}'"))?; let services = parse_services(&inspect_output); if services.is_empty() { warn!("No services found in enclave '{enclave}'"); - return Ok(result); + return Ok(EnclaveServices::default()); } - info!("Found {} services in enclave '{enclave}'", services.len()); - for svc in &services { + // The only IO in the selection below: a `kurtosis port print` fallback for + // ports the inspect-table parse missed. + Ok(classify_services(&services, |svc, port| { + port_print(enclave, svc, port) + })) +} + +/// Decide which discovered services are the beacon nodes, relay APIs, CB +/// sidecars and prometheus, and pick a URL for each (pure, Law 4 seam). +/// +/// `port_fallback(service, port_name)` is consulted ONLY when the port was not +/// already parsed out of the `enclave inspect` table; tests pass a stub, the +/// real caller passes `kurtosis port print`. Splitting it this way makes the +/// classification - which decides WHAT gets checked, and therefore silently +/// invalidates every downstream check when it is wrong - testable without a +/// live enclave. It matters more since Law 7: service names carry the client +/// pair (`cl-1-prysm-nethermind` vs `cl-1-lighthouse-geth`), so the patterns +/// must not accidentally encode one pair. +pub fn classify_services( + services: &[ParsedService], + port_fallback: impl Fn(&str, &str) -> Option, +) -> EnclaveServices { + let mut result = EnclaveServices::default(); + + for svc in services { let find_port = |name: &str| -> Option { svc.ports .iter() .find(|(pn, _)| pn == name) .map(|(_, url)| url.clone()) }; + // Try EVERY already-parsed port name first, and only then fall back to + // the injected lookup. Interleaving them per-name would shell out to + // `kurtosis port print` for an early name before trying a later name + // that the inspect table already carried - a subprocess per relay per + // run, which is the precedence bug an earlier perf pass removed. + let pick = |names: &[&str]| -> Option { + names + .iter() + .find_map(|n| find_port(n)) + .or_else(|| names.iter().find_map(|n| port_fallback(&svc.name, n))) + }; - // Beacon API: cl-* services, port 'http' if matches_pattern(&svc.name, "cl-*") { - let url = port_print(enclave, &svc.name, "http").or_else(|| find_port("http")); - if let Some(url) = url { - info!("Beacon API: {} -> {url}", svc.name); - result.beacon_urls.push(url); - } else { - warn!("Beacon '{}': no http port", svc.name); + match pick(&["http"]) { + Some(url) => { + info!("Beacon API: {} -> {url}", svc.name); + result.beacon_urls.push(url); + } + None => warn!("Beacon '{}': no http port", svc.name), } } - // Relay Data API: match any relay service by name heuristics. - // - // Different relay implementations use different service names and port IDs: - // flashbots: "mev-relay-api" — port "http" (9067) - // helix: "helix-relay" — port "endpoint" (4040) - // mev-rs: "mev-rs-relay" — port "http" (28545) - // Exclude supporting services (postgres, redis, website, housekeeper). + // Relay implementations differ in service name AND port id: + // flashbots "mev-relay-api" http/9067, helix "helix-relay" + // endpoint/4040, mev-rs "mev-rs-relay" http/28545. Supporting + // services (postgres/redis/website/housekeeper) are excluded. if is_relay_api_service(&svc.name) { - let url = port_print(enclave, &svc.name, "http") - .or_else(|| port_print(enclave, &svc.name, "endpoint")) - .or_else(|| find_port("http")) - .or_else(|| find_port("endpoint")); - if let Some(url) = url { - let identity = relay_identity(&svc.name).unwrap_or_else(|| "unknown".to_string()); - info!("Relay API: {} -> {url} (identity={identity})", svc.name); - result.relay_urls.push(url); - result.relay_identities.push(identity); - } else { - warn!("Relay '{}': no http/endpoint port", svc.name); + match pick(&["http", "endpoint"]) { + Some(url) => { + let identity = + relay_identity(&svc.name).unwrap_or_else(|| "unknown".to_string()); + info!("Relay API: {} -> {url} (identity={identity})", svc.name); + result.relay_urls.push(url); + } + None => warn!("Relay '{}': no http/endpoint port", svc.name), } } - // Commit-Boost: commit-boost-* services - // - // The kurtosis ethereum-package publishes CB's PBS port under the - // name "http" (port 18550), not "pbs". Metrics are only exposed - // when commit_boost_config enables [metrics] AND the yaml publishes - // the port -- see configs/pbs-metrics.yml. Absent that, matrix - // checks will SKIP gracefully. if matches_pattern(&svc.name, "commit-boost-*") { result.cb_service_names.push(svc.name.clone()); - - // Try "pbs" first (older configs / custom setups), fall back to - // "http" (ethereum-package default). - let pbs_url = port_print(enclave, &svc.name, "pbs") - .or_else(|| find_port("pbs")) - .or_else(|| port_print(enclave, &svc.name, "http")) - .or_else(|| find_port("http")); - if let Some(url) = pbs_url { - info!("CB PBS: {} -> {url}", svc.name); - result.cb_pbs_urls.push(url); - } else { - warn!("CB '{}': no pbs/http port exposed", svc.name); + // "pbs" first (older/custom configs), then "http" (the + // ethereum-package default, 18550). + match pick(&["pbs", "http"]) { + Some(url) => { + info!("CB PBS: {} -> {url}", svc.name); + result.cb_pbs_urls.push(url); + } + None => warn!("CB '{}': no pbs/http port exposed", svc.name), } - - // Metrics port may be named "metrics" or "http-metrics". - let metrics_url = port_print(enclave, &svc.name, "metrics") - .or_else(|| find_port("metrics")) - .or_else(|| port_print(enclave, &svc.name, "http-metrics")) - .or_else(|| find_port("http-metrics")); - if let Some(url) = metrics_url { + // Metrics only exist when commit_boost_config enables [metrics] AND + // the yaml publishes the port; absent that the matrix checks SKIP. + if let Some(url) = pick(&["metrics", "http-metrics"]) { info!("CB metrics: {} -> {url}", svc.name); result.cb_metrics_urls.push(url); } } - // Prometheus + if matches_pattern(&svc.name, "cb-signer-*") { + match pick(&["signer", "http"]) { + Some(url) => { + info!("CB signer: {} -> {url}", svc.name); + result.signer_urls.push(url); + } + None => warn!("CB signer '{}': no signer/http port", svc.name), + } + } + if svc.name == "prometheus" { - if let Some(url) = port_print(enclave, &svc.name, "http").or_else(|| find_port("http")) - { - info!("Prometheus: {} -> {url}", svc.name); - result.prometheus_url = Some(url); - } else { - warn!("Prometheus service found but no http port available."); + match pick(&["http"]) { + Some(url) => { + info!("Prometheus: {} -> {url}", svc.name); + result.prometheus_url = Some(url); + } + None => warn!("Prometheus service found but no http port available."), } } } @@ -400,7 +415,6 @@ pub fn discover(enclave: &str) -> Result { "no" }, ); - if result.beacon_urls.is_empty() { warn!("No beacon API services (cl-*) found"); } @@ -411,122 +425,7 @@ pub fn discover(enclave: &str) -> Result { warn!("No Commit-Boost services found"); } - Ok(result) -} - -/// Post-mortem: query `mev-relay-postgres` directly when the relay API is dead. -/// -/// Shells out to `kurtosis service exec` to run a psql query inside the relay's -/// Postgres container. Returns payload-delivered records found. If the Postgres -/// container doesn't exist or the query fails, returns an empty Vec. -/// -/// This salvages a verdict when the relay crashed mid-run: if the pipeline -/// worked before the crash, Postgres still has the evidence. -pub fn query_mev_relay_postgres(enclave: &str) -> Vec { - debug!("Running post-mortem Postgres query in enclave '{enclave}'"); - - let query = "SELECT slot, block_hash, value FROM mainnet_payload_delivered ORDER BY slot DESC LIMIT 20;"; - - // `kurtosis service exec` returns output on stdout. The psql output - // includes a header row and separator line before data rows. - match Command::new("kurtosis") - .args([ - "service", - "exec", - enclave, - "mev-relay-postgres", - "--", - "psql", - "-U", - "postgres", - "-d", - "mev_boost_relay", - "-c", - query, - ]) - .output() - { - Ok(output) => { - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - debug!( - "Post-mortem query failed (exit {}): {}", - output.status, - stderr.trim() - ); - return Vec::new(); - } - let stdout = String::from_utf8_lossy(&output.stdout); - parse_postmortem_output(&stdout) - } - Err(e) => { - debug!("Post-mortem: kurtosis CLI not available: {e}"); - Vec::new() - } - } -} - -/// Parse psql tabular output into PostMortemRecords. -/// -/// Expected format: -/// ```text -/// slot | block_hash | value -/// ------+------------+------- -/// 224 | 0xabc... | 12345 -/// 223 | 0xdef... | 67890 -/// (2 rows) -/// ``` -/// -/// Skips header rows (first 2 lines: column names + separator) and the -/// trailing "(N rows)" line. -fn parse_postmortem_output(output: &str) -> Vec { - let mut records = Vec::new(); - let mut seen_header = false; - let mut seen_sep = false; - - for line in output.lines() { - let stripped = line.trim(); - - if stripped.is_empty() { - continue; - } - - // Skip the column-name header and separator line. - if !seen_header { - seen_header = true; - continue; - } - if !seen_sep { - seen_sep = true; - continue; - } - - // Skip trailing "(N rows)" line. - if stripped.starts_with('(') && stripped.ends_with(')') { - continue; - } - - // Data rows: " 224 | 0xabc... | 12345" - let parts: Vec<&str> = stripped.split('|').map(|s| s.trim()).collect(); - if parts.len() < 3 { - continue; - } - - let slot: u64 = match parts[0].parse() { - Ok(s) => s, - Err(_) => continue, - }; - let block_hash = parts[1].to_string(); - let value = parts[2].to_string(); - - records.push(PostMortemRecord { - slot, - block_hash, - value, - }); - } - - records + result } /// Heuristic check: is this service name a relay API endpoint? @@ -539,7 +438,12 @@ fn is_relay_api_service(name: &str) -> bool { let lower = name.to_lowercase(); let known_non_api = ["-postgres", "-redis", "-website", "-housekeeper"]; let is_relay = lower.contains("relay"); - let is_non_api = known_non_api.iter().any(|suffix| lower.ends_with(suffix)); + // CONTAINS, not ends_with: the N-relay-instance topology suffixes every + // service with its index, so the support services are named + // `helix-relay-postgres-2`, which does not END with "-postgres". With + // ends_with, a relay's POSTGRES container was classified as a relay data + // API (found by test, 2026-08-04). + let is_non_api = known_non_api.iter().any(|marker| lower.contains(marker)); is_relay && !is_non_api } @@ -547,11 +451,237 @@ fn is_relay_api_service(name: &str) -> bool { mod tests { use super::relay_identity; + fn svc(name: &str, ports: &[(&str, &str)]) -> ParsedService { + ParsedService { + name: name.to_string(), + ports: ports + .iter() + .map(|(a, b)| (a.to_string(), b.to_string())) + .collect(), + } + } + + /// A fallback that never resolves anything. NOT a panic-stub: the fallback + /// is legitimately consulted for OPTIONAL lookups (CB metrics are absent in + /// the default devnet shape). Precedence is asserted separately, by + /// counting calls, in `parsed_ports_win_over_the_fallback`. + fn no_fallback(_svc: &str, _port: &str) -> Option { + None + } + + #[test] + fn parsed_ports_win_over_the_fallback() { + // The perf contract: a port already in the inspect table must never + // cost a `kurtosis port print` subprocess. Counted, not asserted by + // panic, so optional lookups (metrics) do not confuse the signal. + use std::cell::Cell; + let calls = Cell::new(0u32); + let out = classify_services( + &[svc("cl-1-lighthouse-geth", &[("http", "http://parsed:1")])], + |_, _| { + calls.set(calls.get() + 1); + Some("http://shelled-out:1".to_string()) + }, + ); + assert_eq!(out.beacon_urls, vec!["http://parsed:1"]); + assert_eq!(calls.get(), 0, "parsed port must not shell out"); + } + + #[test] + fn relay_endpoint_port_costs_no_subprocess() { + // Helix exposes only "endpoint". Trying "http" first must NOT shell out + // before "endpoint" is tried against the parsed table - that was a real + // regression (one subprocess per relay per run). + use std::cell::Cell; + let calls = Cell::new(0u32); + let out = classify_services( + &[svc("helix-relay-2", &[("endpoint", "http://h:1")])], + |_, _| { + calls.set(calls.get() + 1); + None + }, + ); + assert_eq!(out.relay_urls, vec!["http://h:1"]); + assert_eq!( + calls.get(), + 0, + "parsed 'endpoint' must not cost a port print" + ); + } + + #[test] + fn classify_picks_beacon_relay_cb_and_prometheus() { + let services = vec![ + svc("cl-1-lighthouse-geth", &[("http", "http://127.0.0.1:1111")]), + svc("el-1-geth-lighthouse", &[("rpc", "http://127.0.0.1:2222")]), + svc("helix-relay-2", &[("endpoint", "http://127.0.0.1:3333")]), + svc( + "commit-boost-1-lighthouse-geth", + &[("http", "http://127.0.0.1:4444")], + ), + svc("prometheus", &[("http", "http://127.0.0.1:5555")]), + ]; + let out = classify_services(&services, no_fallback); + assert_eq!(out.beacon_urls, vec!["http://127.0.0.1:1111"]); + assert_eq!( + out.relay_urls, + vec!["http://127.0.0.1:3333"], + "helix uses 'endpoint'" + ); + assert_eq!(out.cb_pbs_urls, vec!["http://127.0.0.1:4444"]); + assert_eq!(out.cb_service_names, vec!["commit-boost-1-lighthouse-geth"]); + assert_eq!(out.prometheus_url.as_deref(), Some("http://127.0.0.1:5555")); + // The EL is not a beacon, a relay, or a CB service. + assert_eq!(out.beacon_urls.len(), 1); + } + + #[test] + fn classify_finds_the_signer_outside_the_commit_boost_glob() { + // The signer MUST NOT land in cb_service_names: three checks iterate + // that list shelling out `kurtosis service logs -n 200000` per name. + let out = classify_services( + &[ + svc( + "commit-boost-1-lighthouse-geth", + &[("http", "http://cb:18550")], + ), + svc( + "cb-signer-1-lighthouse-geth", + &[("signer", "http://sg:20000")], + ), + ], + no_fallback, + ); + assert_eq!(out.signer_urls, vec!["http://sg:20000"]); + assert_eq!( + out.cb_service_names, + vec!["commit-boost-1-lighthouse-geth"], + "the signer must NOT be swept into cb_service_names" + ); + assert_eq!(out.cb_pbs_urls.len(), 1, "and must not be treated as a PBS"); + } + + #[test] + fn no_signer_service_is_not_an_error() { + // Every scenario except cb-signer runs without one. + let out = classify_services( + &[svc( + "commit-boost-1-lighthouse-geth", + &[("http", "http://cb:1")], + )], + no_fallback, + ); + assert!(out.signer_urls.is_empty()); + } + + #[test] + fn classify_is_client_pair_agnostic() { + // Law 7: service names carry the pair. Patterns must not encode one. + let lh = classify_services( + &[svc("cl-1-lighthouse-geth", &[("http", "http://a:1")])], + no_fallback, + ); + let prysm = classify_services( + &[svc("cl-1-prysm-nethermind", &[("http", "http://b:1")])], + no_fallback, + ); + assert_eq!(lh.beacon_urls.len(), 1); + assert_eq!( + prysm.beacon_urls.len(), + 1, + "prysm+nethermind must classify too" + ); + } + + #[test] + fn classify_finds_every_relay_flavour_and_excludes_support_services() { + let services = vec![ + svc("mev-relay-api", &[("http", "http://f:1")]), + svc("helix-relay-2", &[("endpoint", "http://h:1")]), + svc("mev-rs-relay", &[("http", "http://m:1")]), + // Supporting services that must NOT be treated as relay APIs: + svc("helix-relay-postgres-2", &[("http", "http://p:1")]), + svc("mev-relay-website", &[("http", "http://w:1")]), + svc("mev-relay-housekeeper", &[("http", "http://k:1")]), + ]; + let out = classify_services(&services, no_fallback); + assert_eq!( + out.relay_urls.len(), + 3, + "3 relay APIs, 3 support services excluded" + ); + assert!(out.relay_urls.contains(&"http://h:1".to_string())); + assert!(!out.relay_urls.contains(&"http://p:1".to_string())); + } + + #[test] + fn classify_prefers_pbs_over_http_for_cb() { + let out = classify_services( + &[svc( + "commit-boost-1-lighthouse-geth", + &[("http", "http://x:18550"), ("pbs", "http://x:9999")], + )], + no_fallback, + ); + assert_eq!(out.cb_pbs_urls, vec!["http://x:9999"], "pbs wins over http"); + } + + #[test] + fn classify_uses_the_fallback_only_when_the_port_was_not_parsed() { + // A service whose ports the inspect table did not carry: the injected + // lookup supplies it. This is the ONLY place IO happens in discovery. + let out = classify_services(&[svc("cl-1-lighthouse-geth", &[])], |svc, port| { + assert_eq!((svc, port), ("cl-1-lighthouse-geth", "http")); + Some("http://fallback:1".to_string()) + }); + assert_eq!(out.beacon_urls, vec!["http://fallback:1"]); + } + + #[test] + fn classify_skips_a_service_with_no_usable_port() { + // Must warn and continue, never panic or emit a bogus URL. + let out = classify_services(&[svc("cl-1-lighthouse-geth", &[])], |_, _| None); + assert!(out.beacon_urls.is_empty()); + } + + #[test] + fn classify_cb_metrics_are_optional() { + // Metrics absent is the DEFAULT devnet shape (matrix checks then SKIP); + // it must not stop the PBS url from being discovered. + let out = classify_services( + &[svc( + "commit-boost-1-lighthouse-geth", + &[("http", "http://x:1")], + )], + |_, _| None, + ); + assert_eq!(out.cb_pbs_urls.len(), 1); + assert!(out.cb_metrics_urls.is_empty()); + } + + #[test] + fn classify_handles_multi_relay_and_multi_cb() { + let out = classify_services( + &[ + svc("helix-relay-2", &[("endpoint", "http://h2:1")]), + svc("helix-relay-3", &[("endpoint", "http://h3:1")]), + svc("commit-boost-1-lighthouse-geth", &[("http", "http://c1:1")]), + svc("commit-boost-2-lighthouse-geth", &[("http", "http://c2:1")]), + ], + no_fallback, + ); + assert_eq!(out.relay_urls.len(), 2, "2-helix topology"); + assert_eq!(out.cb_service_names.len(), 2); + } + #[test] fn test_relay_identity() { assert_eq!(relay_identity("helix-relay").as_deref(), Some("helix")); assert_eq!(relay_identity("Helix-Relay").as_deref(), Some("helix")); - assert_eq!(relay_identity("mev-relay-api").as_deref(), Some("flashbots")); + assert_eq!( + relay_identity("mev-relay-api").as_deref(), + Some("flashbots") + ); assert_eq!(relay_identity("mev-rs-relay").as_deref(), Some("mev-rs")); // Non-relay services: function should not be called for these // in practice (is_relay_api_service filters them), but they @@ -613,33 +743,54 @@ mod tests { assert_eq!(parts[1], "cl-1-lighthouse"); } + // Parse the real `kurtosis enclave inspect` fixture (a prime format-drift + // trap). Asserts the User Services table is read into service names + ports, + // that the header/separator/Files-Artifacts noise is skipped, and that a + // `` ports column yields no parsed ports. + // + // Note: the shared fixture (also consumed by `sim triage`) contains only a + // beacon (cl-*) and a relay (mev-relay-*) service; it has no commit-boost + // service, so cb-name/port extraction isn't exercised here (extending the + // fixture would break the triage test's `len == 2` assertion). #[test] - fn test_parse_postmortem_output() { - let output = concat!( - " slot | block_hash | value\n", - "------+------------+-------\n", - " 224 | 0xabc123 | 12345\n", - " 223 | 0xdef456 | 67890\n", - "(2 rows)\n", + fn test_parse_services_from_fixture() { + const INSPECT: &str = include_str!("../tests/fixtures/enclave_inspect.txt"); + let services = parse_services(INSPECT); + + // Exactly the two User Services rows (the Files Artifacts table and all + // header/separator lines are skipped). + assert_eq!( + services.len(), + 2, + "expected 2 user services, got {:?}", + services.iter().map(|s| &s.name).collect::>() ); - let records = parse_postmortem_output(output); - assert_eq!(records.len(), 2); - assert_eq!(records[0].slot, 224); - assert_eq!(records[0].block_hash, "0xabc123"); - assert_eq!(records[0].value, "12345"); - assert_eq!(records[1].slot, 223); - assert_eq!(records[1].block_hash, "0xdef456"); - assert_eq!(records[1].value, "67890"); - } - #[test] - fn test_parse_postmortem_output_empty() { - let output = concat!( - " slot | block_hash | value\n", - "------+------------+-------\n", - "(0 rows)\n", + // Beacon service: name + a parsed `http` port URL. + let beacon = services + .iter() + .find(|s| s.name == "cl-1-lighthouse-geth") + .expect("beacon service parsed"); + let (_, http_url) = beacon + .ports + .iter() + .find(|(name, _)| name == "http") + .expect("beacon http port parsed"); + assert!( + http_url.contains("127.0.0.1:32811"), + "unexpected http url: {http_url}" + ); + + // Relay service: name is extracted even though its ports column is + // `` (a stopped service), which parses to zero ports. + let relay = services + .iter() + .find(|s| s.name == "mev-relay-helix") + .expect("relay service parsed"); + assert!( + relay.ports.is_empty(), + "a `` ports column must parse to no ports, got {:?}", + relay.ports ); - let records = parse_postmortem_output(output); - assert!(records.is_empty()); } } diff --git a/src/health.rs b/src/health.rs index f19aacf..5768755 100644 --- a/src/health.rs +++ b/src/health.rs @@ -91,3 +91,71 @@ pub async fn probe_all( } dead } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn probe_urls_match_each_service_kind() { + // These paths are the liveness contract with three different servers. + // A wrong path returns 404 - which `probe` counts as ALIVE (only + // transport errors mean death), so a typo here would make the death + // detector permanently blind rather than fail loudly. + assert_eq!( + HealthTarget::new("b", "http://cl:5052", ServiceKind::Beacon).probe_url(), + "http://cl:5052/eth/v1/node/health" + ); + assert_eq!( + HealthTarget::new("r", "http://relay:4040", ServiceKind::Relay).probe_url(), + "http://relay:4040/relay/v1/data/bidtraces/proposer_payload_delivered?limit=1" + ); + assert_eq!( + HealthTarget::new("c", "http://cb:18550", ServiceKind::CbPbs).probe_url(), + "http://cb:18550/eth/v1/builder/status" + ); + } + + #[test] + fn trailing_slash_never_produces_a_double_slash() { + // URLs are built by concatenation; `//` 404s on some servers, which + // would again read as "alive" and blind the detector. + let t = HealthTarget::new("b", "http://cl:5052/", ServiceKind::Beacon); + assert_eq!(t.base_url, "http://cl:5052"); + assert!(!t.probe_url().contains("5052//")); + } + + #[tokio::test] + async fn probe_reports_transport_failure_as_death() { + // Port 1 on localhost refuses instantly: the one condition that must + // count as dead. No network dependency beyond loopback. + let client = reqwest::Client::new(); + let t = HealthTarget::new("dead", "http://127.0.0.1:1", ServiceKind::Beacon); + assert!(probe(&client, &t).await.is_err()); + } + + #[tokio::test] + async fn probe_all_returns_the_labels_that_failed() { + let client = reqwest::Client::new(); + let targets = vec![ + HealthTarget::new("dead-a", "http://127.0.0.1:1", ServiceKind::Beacon), + HealthTarget::new("dead-b", "http://127.0.0.1:1", ServiceKind::Relay), + ]; + // probe_all returns (label, error) pairs so the caller can report WHY. + let failed = probe_all(&client, &targets).await; + assert_eq!(failed.len(), 2); + let labels: Vec<&str> = failed.iter().map(|(l, _)| l.as_str()).collect(); + assert!(labels.contains(&"dead-a")); + assert!(labels.contains(&"dead-b")); + assert!( + !failed[0].1.to_string().is_empty(), + "the death reason must be carried, not discarded" + ); + } + + #[tokio::test] + async fn probe_all_with_no_targets_reports_nothing_dead() { + let client = reqwest::Client::new(); + assert!(probe_all(&client, &[]).await.is_empty()); + } +} diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..e3beae6 --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,13 @@ +//! cb-testnet-verifier: shared library. +//! +//! The mature modules that discover services, probe beacon/relay endpoints, +//! run verification checks, and build structured reports live here so every +//! binary in the package (`cb-verify`, `cb-orchestrator`, `sim`, …) can reuse +//! them by import instead of re-declaring or re-implementing them. + +pub mod beacon; +pub mod checks; +pub mod discovery; +pub mod metrics; +pub mod relay; +pub mod report; diff --git a/src/main.rs b/src/main.rs index b67a7a8..dfab00e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,23 +3,17 @@ //! Discovers services in a running enclave, polls for readiness, //! runs verification checks, and produces a structured report. -#![allow(unused_imports)] -#![allow(dead_code)] - use std::time::{Duration, Instant}; use clap::Parser; use eyre::Result; use tracing::{debug, error, info, warn}; -mod beacon; -mod checks; -mod discovery; +// Shared modules now live in the library crate; health/live are private to this bin. +use cb_testnet_verifier::{beacon, checks, discovery, metrics, relay, report}; + mod health; mod live; -mod metrics; -mod relay; -mod report; use beacon::BeaconClient; use checks::{CheckResult, CheckStatus}; @@ -107,6 +101,18 @@ struct Cli { #[arg(short, long)] verbose: bool, + /// Fail the run when a tier-1 feature check armed a differential and then + /// observed nothing (Law 3). + /// + /// Off by default to preserve the documented exit contract (tier-1 WARN is + /// non-fatal). But that contract means a scenario can report "NOT asserting + /// the feature ran" and still exit 0, so a sweep counts a vacuous scenario as + /// a win. Turn this on in sweeps. It does NOT affect checks that are + /// structurally unable to confirm their feature, e.g. skip_sigverify on the + /// happy path. + #[arg(long)] + require_feature_proof: bool, + /// Strict mode: promote soft warnings to FAIL. Affects: /// /// - get_header with zero 200s but some 204s (relay alive but no bids @@ -174,9 +180,10 @@ async fn main() -> Result<()> { /// The enclave name is required. The config is optional and used for /// mux verification. fn resolve_enclave_and_config(cli: &Cli) -> Result<(String, Option)> { - let enclave = cli.enclave.clone().ok_or_else(|| { - eyre::eyre!("Must provide --enclave to specify a running enclave") - })?; + let enclave = cli + .enclave + .clone() + .ok_or_else(|| eyre::eyre!("Must provide --enclave to specify a running enclave"))?; Ok((enclave, cli.config.clone())) } @@ -233,7 +240,13 @@ async fn run_verification(cli: &Cli) -> i32 { // --show-logs mode: print raw CB PBS logs and exit if cli.show_logs { - return show_cb_logs(&enclave_name, &services.cb_service_names, &now, cli.json, &save_report); + return show_cb_logs( + &enclave_name, + &services.cb_service_names, + &now, + cli.json, + &save_report, + ); } if relays.is_empty() { @@ -261,7 +274,11 @@ async fn run_verification(cli: &Cli) -> i32 { Ok(s) => s, Err(e) => { error!("Failed to get current slot: {e}"); - let report = make_error_report(&enclave_name, &now, &format!("Failed to get current slot: {e}")); + let report = make_error_report( + &enclave_name, + &now, + &format!("Failed to get current slot: {e}"), + ); report::print_report(&report, cli.json); save_report(&report); return 2; @@ -315,9 +332,12 @@ async fn run_verification(cli: &Cli) -> i32 { let summary: Vec = dead_at_preflight.iter().map(|(l, _)| l.clone()).collect(); - // When ONLY relay targets are dead, try post-mortem: query the relay's - // Postgres directly. If the pipeline worked before the crash, Postgres - // still has the evidence. Salvage the verdict instead of hard-failing. + // When ONLY relay targets are dead, PROCEED to the observation window + // rather than bail: the relay checks handle dead relays and FAIL the + // tier-1 delivery check (all_relays_dead_results / C1), which is the + // correct verdict for a relay that died. (A flashbots-Postgres post-mortem + // salvage lived here; it hardcoded the mev-boost-relay's service/schema and + // was dead after the 2-helix migration — dropped, see M5.) let all_are_relays = dead_at_preflight .iter() .all(|(l, _)| l.starts_with("relay[")); @@ -326,46 +346,13 @@ async fn run_verification(cli: &Cli) -> i32 { .any(|(l, _)| l.starts_with("relay[")); if all_are_relays && relay_died { - info!("Relay Data API unreachable — attempting post-mortem via Postgres..."); - let postmortem = discovery::query_mev_relay_postgres(&enclave_name); - if !postmortem.is_empty() { - info!( - "Post-mortem: found {} payload(s) in relay Postgres before crash:", - postmortem.len() - ); - for r in &postmortem { - let hash_short = if r.block_hash.len() > 28 { - &r.block_hash[..28] - } else { - &r.block_hash - }; - info!(" slot={} hash={}... value={}", r.slot, hash_short, r.value); - } - info!( - "Pipeline worked before relay crash. Proceeding with non-relay checks \ - (relay API checks will SKIP)." - ); - // Fall through to Step 3 — wait for window. Relay checks - // will naturally SKIP because the relay URLs are unreachable. - } else { - error!("Post-mortem: no delivery records found in relay Postgres."); - let report = make_error_report( - &enclave_name, - &now, - &format!( - "Preflight failed ({} of {} services): {}. Relay API unreachable \ - and post-mortem Postgres query found no delivery records. \ - Try: kurtosis enclave inspect {} ; docker ps -a", - dead_at_preflight.len(), - targets.len(), - summary.join(", "), - &enclave_name - ), - ); - report::print_report(&report, cli.json); - save_report(&report); - return 2; - } + warn!( + "Relay(s) unreachable at preflight ({}). Non-relay services are up; proceeding to \ + the observation window — the relay checks report the failure \ + (relay.payloads_delivered_multi FAILs if they stay down).", + summary.join(", ") + ); + // Fall through to Step 3. } else { error!( "{} of {} service(s) unreachable: {:?}", @@ -418,7 +405,10 @@ async fn run_verification(cli: &Cli) -> i32 { let report = make_error_report( &enclave_name, &now, - &format!("Chain did not reach slot {end_slot} within {}s", cli.timeout), + &format!( + "Chain did not reach slot {end_slot} within {}s", + cli.timeout + ), ); report::print_report(&report, cli.json); save_report(&report); @@ -487,6 +477,18 @@ async fn run_verification(cli: &Cli) -> i32 { .await, ); + info!("Running best-bid (aggregated bidding) check..."); + all_checks.push( + checks::best_bid::check_best_bid_selection( + &enclave_name, + &services.cb_service_names, + &relays, + window.start_slot, + window.end_slot, + ) + .await, + ); + info!("Running CB metrics checks..."); all_checks.extend( checks::cb_metrics::run_metrics_checks( @@ -532,26 +534,99 @@ async fn run_verification(cli: &Cli) -> i32 { info!("No --cb-config provided — skipping MUX routing check"); } - // Step 5: Report - let tier1_failed = all_checks - .iter() - .any(|c| c.tier == 1 && c.status == CheckStatus::Fail); + // Feature-fired assertions (Law 3): for each CB feature the config enables + // (skip_sigverify / extra_validation / timing_games), assert its codepath + // actually fired at runtime. Same config gate as mux. + if let Some(ref cb_path) = cb_config { + match checks::mux_routing::read_cb_config_template(cb_path) { + Ok(template) => { + all_checks.extend( + checks::feature_fired::run_feature_checks( + &enclave_name, + &services.cb_service_names, + &template, + ) + .await, + ); + } + Err(e) => { + warn!("Could not read CB config for feature-fired checks: {e}"); + } + } + } + + // Commit-Boost SIGNER module. Only runs when a cb-signer-* service was + // discovered, so every other scenario is unaffected. + // + // The assertion is deliberately the key COUNT over a JWT-authed + // get_pubkeys, not a /status probe: /status is an unconditional 200 with + // zero logic, so it stays green with no keys loaded - and zero-keys is this + // feature's most likely failure, since CB's keystore loaders skip + // unreadable entries with warn! rather than failing startup. + for signer_url in &services.signer_urls { + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0); + // Must match the fork's signer_launcher defaults (CB_JWTS). + let module_id = "TEST_MODULE"; + let module_jwt = "b3d1f5a4c8e2079b6d4f1a3c5e7b9d2f"; + match checks::signer::fetch_pubkeys(&http_client, signer_url, module_id, module_jwt, now) + .await + { + Ok((200, Some(resp))) => { + all_checks.push(checks::signer::classify_signer_pubkeys( + validator_pubkeys.len(), + resp.keys.len(), + )); + } + Ok((status, _)) => { + all_checks.push(CheckResult::fail( + "signer.pubkeys", + 1, + format!( + "signer at {signer_url} answered {status} to an authenticated get_pubkeys (expected 200)" + ), + )); + } + Err(e) => { + all_checks.push(CheckResult::fail( + "signer.pubkeys", + 1, + format!("signer at {signer_url} unreachable: {e}"), + )); + } + } + } + // Best-effort provenance: WHAT was tested (config hash + resolved Docker + // image IDs). Never fails the run — docker unreachable or an unparseable + // config just yields None. + let provenance = report::gather_provenance(cb_config.as_deref()); + if provenance.is_none() { + debug!( + "provenance unavailable (docker unreachable or config could not be parsed); \ + continuing without it" + ); + } + + // Step 5: Report let report = VerificationReport { enclave: enclave_name.clone(), timestamp: now, observation_window: Some(window), - result: if tier1_failed { + result: if report::tier1_failed(&all_checks) { CheckStatus::Fail } else { CheckStatus::Pass }, checks: all_checks, + provenance, }; report::print_report(&report, cli.json); save_report(&report); - report::exit_code(&report) + report::exit_code_with_policy(&report, cli.require_feature_proof) } /// Poll the beacon node until the devnet is ready for verification. @@ -577,7 +652,9 @@ async fn wait_for_slot( timeout: u64, live_opts: WaitLiveOpts<'_>, ) -> bool { - info!("Waiting for chain to reach slot {end_slot} (verification starts at slot {start_slot}, timeout {timeout}s)..."); + info!( + "Waiting for chain to reach slot {end_slot} (verification starts at slot {start_slot}, timeout {timeout}s)..." + ); let wait_start = Instant::now(); let timeout_dur = Duration::from_secs(timeout); @@ -638,7 +715,9 @@ async fn wait_for_slot( } }, None => { - warn!("--live-metrics requested but metrics not HTTP-reachable; skipping live deltas"); + warn!( + "--live-metrics requested but metrics not HTTP-reachable; skipping live deltas" + ); } } } @@ -662,9 +741,7 @@ async fn wait_for_slot( } // Live metrics: scrape, compute deltas vs previous, log. - if live_started - && let Some(url) = live_opts.metrics_url - { + if live_started && let Some(url) = live_opts.metrics_url { match metrics::fetch_metrics(http, url).await { Ok(curr) => { let deltas = @@ -700,7 +777,7 @@ fn show_cb_logs( json_mode: bool, save_report: &dyn Fn(&VerificationReport), ) -> i32 { - use crate::checks::mux_routing::{parse_cb_log_line, fetch_service_logs}; + use crate::checks::mux_routing::{fetch_service_logs, parse_cb_log_line}; println!("\n=== CB PBS Service Logs ==="); println!("Enclave: {enclave_name}"); @@ -721,7 +798,14 @@ fn show_cb_logs( total_events += 1; if let Some(event) = parse_cb_log_line(line) { parsed_events += 1; - print!(" [{}] {}", event.message, event.slot.map(|s| format!("slot={}", s)).unwrap_or_default()); + print!( + " [{}] {}", + event.message, + event + .slot + .map(|s| format!("slot={}", s)) + .unwrap_or_default() + ); if let Some(ref mux) = event.mux_id { print!(" mux={}", mux); } @@ -746,7 +830,10 @@ fn show_cb_logs( } } - println!("\nTotal: {} log lines, {} parsed successfully", total_events, parsed_events); + println!( + "\nTotal: {} log lines, {} parsed successfully", + total_events, parsed_events + ); let report = VerificationReport { enclave: enclave_name.to_string(), @@ -756,8 +843,13 @@ fn show_cb_logs( checks: vec![CheckResult::pass( "logs", 1, - format!("Fetched {} log lines from {} service(s)", total_events, cb_service_names.len()), + format!( + "Fetched {} log lines from {} service(s)", + total_events, + cb_service_names.len() + ), )], + provenance: None, }; report::print_report(&report, json_mode); save_report(&report); @@ -771,5 +863,6 @@ fn make_error_report(enclave: &str, timestamp: &str, detail: &str) -> Verificati observation_window: None, result: CheckStatus::Fail, checks: vec![CheckResult::fail("setup", 1, detail)], + provenance: None, } } diff --git a/src/metrics.rs b/src/metrics.rs index 2a745e8..ccddc78 100644 --- a/src/metrics.rs +++ b/src/metrics.rs @@ -6,8 +6,7 @@ use std::process::Command; use eyre::{Result, WrapErr, bail}; -use prometheus_parse::{Scrape, Value}; -use tracing::{debug, warn}; +use prometheus_parse::Scrape; /// Fetch and parse Prometheus metrics from an HTTP endpoint. pub async fn fetch_metrics(client: &reqwest::Client, url: &str) -> Result { @@ -52,47 +51,66 @@ fn parse_metrics(text: &str) -> Result { Scrape::parse(lines).wrap_err("failed to parse Prometheus metrics") } -/// Helper: sum all samples matching a metric name and optional label filter. -pub fn sum_metric(scrape: &Scrape, name: &str, label_filter: Option<(&str, &str)>) -> f64 { - scrape - .samples - .iter() - .filter(|s| s.metric == name) - .filter(|s| { - if let Some((key, val)) = label_filter { - s.labels.get(key) == Some(val) - } else { - true - } - }) - .map(|s| match &s.value { - Value::Counter(v) | Value::Gauge(v) | Value::Untyped(v) => *v, - _ => 0.0, - }) - .sum() -} +#[cfg(test)] +mod tests { + use super::*; -/// Helper: check if any samples exist for a metric. -pub fn has_metric(scrape: &Scrape, name: &str) -> bool { - scrape.samples.iter().any(|s| s.metric == name) -} + /// The exact shape CB exposes, captured from a live devnet scrape. Every + /// matrix check reads these three families, so a parse regression here + /// makes them all SKIP - silently, since "metrics absent" is the normal + /// devnet state and SKIP is non-fatal. + const CB_SCRAPE: &str = r#"# HELP cb_pbs_relay_status_code_total relay status codes +# TYPE cb_pbs_relay_status_code_total counter +cb_pbs_relay_status_code_total{endpoint="get_header",http_status_code="200",relay_id="mev_relay_0"} 27 +cb_pbs_relay_status_code_total{endpoint="get_header",http_status_code="555",relay_id="mev_relay_0"} 17 +# HELP pbs_submit_block_v2_unsupported_total v2 unsupported +# TYPE pbs_submit_block_v2_unsupported_total counter +pbs_submit_block_v2_unsupported_total{relay_id="mev_relay_0"} 11 +# HELP cb_pbs_relay_latency HTTP latency by relay +# TYPE cb_pbs_relay_latency histogram +cb_pbs_relay_latency_bucket{endpoint="get_header",relay_id="mev_relay_0",le="0.05"} 12 +cb_pbs_relay_latency_bucket{endpoint="get_header",relay_id="mev_relay_0",le="+Inf"} 27 +cb_pbs_relay_latency_sum{endpoint="get_header",relay_id="mev_relay_0"} 1.5 +cb_pbs_relay_latency_count{endpoint="get_header",relay_id="mev_relay_0"} 27 +"#; -/// Helper: get all sample values for a metric, optionally filtered by label. -pub fn metric_values(scrape: &Scrape, name: &str, label_filter: Option<(&str, &str)>) -> Vec { - scrape - .samples - .iter() - .filter(|s| s.metric == name) - .filter(|s| { - if let Some((key, val)) = label_filter { - s.labels.get(key) == Some(val) - } else { - true - } - }) - .filter_map(|s| match &s.value { - Value::Counter(v) | Value::Gauge(v) | Value::Untyped(v) => Some(*v), - _ => None, - }) - .collect() + #[test] + fn parses_a_real_cb_scrape_with_labels_and_histograms() { + let scrape = parse_metrics(CB_SCRAPE).expect("real CB scrape must parse"); + let names: Vec<&str> = scrape.samples.iter().map(|s| s.metric.as_str()).collect(); + assert!(names.contains(&"cb_pbs_relay_status_code_total")); + assert!(names.contains(&"pbs_submit_block_v2_unsupported_total")); + // Labels must survive: every check keys on endpoint/http_status_code/relay_id. + let s = scrape + .samples + .iter() + .find(|s| { + s.metric == "cb_pbs_relay_status_code_total" + && s.labels.get("http_status_code") == Some("555") + }) + .expect("the synthetic 555 sample must be addressable by label"); + assert_eq!(s.labels.get("relay_id"), Some("mev_relay_0")); + } + + #[test] + fn empty_scrape_parses_to_no_samples_rather_than_erroring() { + // The default devnet exposes no metrics; that must be an empty scrape + // (checks then SKIP), never a hard error that fails the run. + let scrape = parse_metrics("").expect("empty body must parse"); + assert!(scrape.samples.is_empty()); + } + + #[test] + fn comments_only_scrape_is_empty() { + let scrape = parse_metrics("# HELP x nothing\n# TYPE x counter\n").unwrap(); + assert!(scrape.samples.is_empty()); + } + + #[tokio::test] + async fn fetch_from_a_dead_endpoint_errors_instead_of_hanging() { + // Port 1 refuses instantly. The caller turns this into a SKIP; it must + // never panic or block the run. + let client = reqwest::Client::new(); + assert!(fetch_metrics(&client, "http://127.0.0.1:1").await.is_err()); + } } diff --git a/src/orchestrator.rs b/src/orchestrator.rs index 459a357..03ea6ae 100644 --- a/src/orchestrator.rs +++ b/src/orchestrator.rs @@ -16,7 +16,7 @@ use std::process::Command; use std::time::{Duration, Instant}; use clap::Parser; -use eyre::{bail, Context, Result}; +use eyre::{Context, Result, bail}; use serde::Serialize; use tokio::sync::Semaphore; use tokio::task::JoinSet; @@ -87,29 +87,11 @@ struct Cli { // Types // --------------------------------------------------------------------------- -/// The lifecycle state of a single enclave run. -#[derive(Debug, Clone, PartialEq)] -enum EnclaveState { - /// `kurtosis run` has been launched, waiting for containers to start. - Launching, - /// Containers are up, waiting for beacon to reach target_epoch. - WaitingForReadiness, - /// Beacon is ready, observing for min_epochs. - Observing, - /// Running cb-verify checks. - Checking, - /// All checks complete. - Done, - /// Failed at some point. - Failed(String), -} - /// Per-enclave status tracked by the orchestrator. #[derive(Debug, Clone)] struct EnclaveStatus { name: String, config: PathBuf, - state: EnclaveState, /// Set when the enclave process has been launched. launched_at: Option, /// Set when the enclave becomes ready for observation. @@ -190,7 +172,6 @@ async fn main() -> Result<()> { EnclaveStatus { name, config: config.clone(), - state: EnclaveState::Launching, launched_at: None, ready_at: None, observed_at: None, @@ -252,10 +233,7 @@ async fn main() -> Result<()> { Ok((idx, result)) => results.push((idx, result)), Err(e) => { error!("Task panicked: {e}"); - results.push(( - usize::MAX, - Err(eyre::eyre!("Task panicked: {e}")), - )); + results.push((usize::MAX, Err(eyre::eyre!("Task panicked: {e}")))); } } } @@ -345,6 +323,9 @@ async fn main() -> Result<()> { // Per-enclave pipeline // --------------------------------------------------------------------------- +// Launcher fn: threading these as individual params reads clearer than a +// bespoke options struct that exists only for this one call site. +#[allow(clippy::too_many_arguments)] async fn run_enclave_pipeline( mut enc: EnclaveStatus, package: &str, @@ -360,53 +341,61 @@ async fn run_enclave_pipeline( let start = Instant::now(); // Phase 1: Launch - info!("[{}] Launching enclave with config {}...", enc.name, enc.config.display()); - enc.state = EnclaveState::Launching; + info!( + "[{}] Launching enclave with config {}...", + enc.name, + enc.config.display() + ); enc.launched_at = Some(Instant::now()); if let Err(e) = launch_enclave(&enc.name, &enc.config, package).await { let msg = format!("Launch failed: {e}"); error!("[{}] {}", enc.name, msg); - enc.state = EnclaveState::Failed(msg.clone()); // Try to clean up - if !keep { let _ = teardown_enclave(&enc.name); } + if !keep { + let _ = teardown_enclave(&enc.name); + } bail!(msg); } // Phase 2: Wait for readiness - info!("[{}] Waiting for readiness (target epoch {target_epoch})...", enc.name); - enc.state = EnclaveState::WaitingForReadiness; + info!( + "[{}] Waiting for readiness (target epoch {target_epoch})...", + enc.name + ); if let Err(e) = wait_for_enclave_readiness(&enc.name, target_epoch, timeout).await { let msg = format!("Readiness timeout: {e}"); error!("[{}] {}", enc.name, msg); - enc.state = EnclaveState::Failed(msg.clone()); - if !keep { let _ = teardown_enclave(&enc.name); } + if !keep { + let _ = teardown_enclave(&enc.name); + } bail!(msg); } enc.ready_at = Some(Instant::now()); info!( "[{}] Enclave ready after {:?}", enc.name, - enc.ready_at.unwrap().duration_since(enc.launched_at.unwrap()) + enc.ready_at + .unwrap() + .duration_since(enc.launched_at.unwrap()) ); // Phase 3: Observe info!("[{}] Observing for {min_epochs} epoch(s)...", enc.name); - enc.state = EnclaveState::Observing; if let Err(e) = observe_enclave(&enc.name, min_epochs, target_epoch).await { let msg = format!("Observation failed: {e}"); error!("[{}] {}", enc.name, msg); - enc.state = EnclaveState::Failed(msg.clone()); - if !keep { let _ = teardown_enclave(&enc.name); } + if !keep { + let _ = teardown_enclave(&enc.name); + } bail!(msg); } enc.observed_at = Some(Instant::now()); // Phase 4: Run checks info!("[{}] Running checks...", enc.name); - enc.state = EnclaveState::Checking; let check_result = run_checks( &enc.name, @@ -424,13 +413,16 @@ async fn run_enclave_pipeline( Ok(summary) => { let result_str = summary.result.clone(); enc.check_result = Some(summary); - enc.state = EnclaveState::Done; - info!("[{}] Checks complete: {} (total {:?})", enc.name, result_str, enc.checked_at.unwrap().duration_since(start)); + info!( + "[{}] Checks complete: {} (total {:?})", + enc.name, + result_str, + enc.checked_at.unwrap().duration_since(start) + ); } Err(e) => { let msg = format!("Check execution failed: {e}"); error!("[{}] {}", enc.name, msg); - enc.state = EnclaveState::Failed(msg); } } @@ -502,24 +494,23 @@ async fn wait_for_enclave_readiness( let url = format!("{beacon_url}/eth/v1/beacon/headers/head"); match client.get(&url).send().await { Ok(resp) => { - if let Ok(json) = resp.json::().await { - if let Some(slot) = json + if let Ok(json) = resp.json::().await + && let Some(slot) = json .get("data") .and_then(|d| d.get("header")) .and_then(|h| h.get("message")) .and_then(|m| m.get("slot")) .and_then(|s| s.as_str()) .and_then(|s| s.parse::().ok()) - { - let epoch = slot / 32; - if epoch >= target_epoch { - return Ok(()); - } - tracing::debug!( - "[{}] Beacon at epoch {epoch}, waiting for {target_epoch}...", - name - ); + { + let epoch = slot / 32; + if epoch >= target_epoch { + return Ok(()); } + tracing::debug!( + "[{}] Beacon at epoch {epoch}, waiting for {target_epoch}...", + name + ); } } Err(e) => { @@ -561,20 +552,21 @@ async fn observe_enclave(name: &str, min_epochs: u64, target_epoch: u64) -> Resu let url = format!("{beacon_url}/eth/v1/beacon/headers/head"); match client.get(&url).send().await { Ok(resp) => { - if let Ok(json) = resp.json::().await { - if let Some(slot) = json + if let Ok(json) = resp.json::().await + && let Some(slot) = json .get("data") .and_then(|d| d.get("header")) .and_then(|h| h.get("message")) .and_then(|m| m.get("slot")) .and_then(|s| s.as_str()) .and_then(|s| s.parse::().ok()) - { - if slot >= target_slot { - info!("[{}] Observation complete: slot {start_slot} -> {slot}", name); - return Ok(()); - } - } + && slot >= target_slot + { + info!( + "[{}] Observation complete: slot {start_slot} -> {slot}", + name + ); + return Ok(()); } } Err(e) => { @@ -602,12 +594,15 @@ async fn run_checks( let binary_path = manifest_path.join("target/release/cb-verify"); if !binary_path.exists() { - bail!("cb-verify binary not found at {}. Run 'cargo build --release' first.", binary_path.display()); + bail!( + "cb-verify binary not found at {}. Run 'cargo build --release' first.", + binary_path.display() + ); } let mut cmd = tokio::process::Command::new(&binary_path); cmd.arg("--enclave").arg(name); - cmd.arg("--cb-config").arg(config); + cmd.arg("--config").arg(config); cmd.arg("--json"); cmd.arg("--timeout").arg("3600"); cmd.arg("--min-epochs").arg("0"); // Already observed @@ -626,10 +621,7 @@ async fn run_checks( cmd.arg("-v"); } - let output = cmd - .output() - .await - .wrap_err("Failed to run cb-verify")?; + let output = cmd.output().await.wrap_err("Failed to run cb-verify")?; // Parse the JSON report from stdout let stdout = String::from_utf8_lossy(&output.stdout); @@ -641,8 +633,8 @@ async fn run_checks( .unwrap_or(0); let json_str = &stdout[json_start..]; - let report: serde_json::Value = serde_json::from_str(json_str) - .wrap_err("Failed to parse cb-verify JSON output")?; + let report: serde_json::Value = + serde_json::from_str(json_str).wrap_err("Failed to parse cb-verify JSON output")?; let result = report .get("result") @@ -707,7 +699,13 @@ fn teardown_enclave(name: &str) -> Result<()> { /// Discover the beacon HTTP URL for an enclave by querying kurtosis port print. async fn discover_beacon_url(enclave: &str) -> Result { // Try common beacon service names - let beacon_names = ["cl-1-lighthouse", "cl-1-prysm", "cl-1-teku", "cl-1-nimbus", "cl-1-lodestar"]; + let beacon_names = [ + "cl-1-lighthouse", + "cl-1-prysm", + "cl-1-teku", + "cl-1-nimbus", + "cl-1-lodestar", + ]; for name_prefix in &beacon_names { // Try to find the full service name @@ -748,8 +746,9 @@ async fn discover_beacon_url(enclave: &str) -> Result { .output() .await; if let Ok(port_out2) = port_output2 { - let url2 = - String::from_utf8_lossy(&port_out2.stdout).trim().to_string(); + let url2 = String::from_utf8_lossy(&port_out2.stdout) + .trim() + .to_string(); if !url2.is_empty() { return Ok(url2); } @@ -831,22 +830,10 @@ fn print_batch_summary(batch: &BatchReport) { println!("╔══════════════════════════════════════════════════════════════╗"); println!("║ Batch Verification Report ║"); println!("╠══════════════════════════════════════════════════════════════╣"); - println!( - "║ Time: {:48} ║", - batch.timestamp - ); - println!( - "║ Total: {:48} ║", - batch.total - ); - println!( - "║ Passed: {:48} ║", - batch.passed.to_string().green() - ); - println!( - "║ Failed: {:48} ║", - batch.failed.to_string().red() - ); + println!("║ Time: {:48} ║", batch.timestamp); + println!("║ Total: {:48} ║", batch.total); + println!("║ Passed: {:48} ║", batch.passed.to_string().green()); + println!("║ Failed: {:48} ║", batch.failed.to_string().red()); println!("╠══════════════════════════════════════════════════════════════╣"); for result in &batch.results { @@ -861,7 +848,13 @@ fn print_batch_summary(batch: &BatchReport) { .unwrap_or(&result.config); println!( "║ {} {:20} {:6} ({}p / {}f / {}w / {}s) ║", - status_icon, name, result.result, result.passed, result.failed, result.warnings, result.skipped + status_icon, + name, + result.result, + result.passed, + result.failed, + result.warnings, + result.skipped ); } diff --git a/src/relay.rs b/src/relay.rs index f5ec668..a36a33b 100644 --- a/src/relay.rs +++ b/src/relay.rs @@ -1,16 +1,59 @@ //! Relay Data API client. //! //! Thin async wrapper over reqwest returning alloy relay types. -//! Implements the Flashbots relay data API endpoints needed for verification. +//! Implements the helix relay data API endpoints needed for verification +//! (the relays run `ghcr.io/gattaca-com/helix-relay`, not the Flashbots +//! reference relay, so helix's response contract is what we target here). +use std::collections::HashSet; use std::time::Duration; +use alloy_primitives::B256; use alloy_rpc_types_beacon::relay::{BuilderBlockReceived, ProposerPayloadDelivered}; use eyre::Result; -use tracing::warn; const REQUEST_TIMEOUT: Duration = Duration::from_secs(10); +/// The relay caps `limit` at 200 rows per page. +const PAGE_LIMIT: usize = 200; + +/// Safety bound on pagination: 50 × 200 = 10,000 payloads max. +const MAX_PAGES: usize = 50; + +/// Decide whether to keep paging after fetching a page. +/// +/// This is the pure loop-termination seam for `get_payloads_delivered`. It is +/// deliberately order-agnostic (works whether the relay returns rows ascending +/// or descending by slot) and defensive against a relay that ignores the +/// `cursor` query param. +/// +/// Returns `true` to keep paging, `false` to stop. +/// +/// - **No-progress / ignored-cursor guard:** if the next cursor equals the +/// previous one, the relay did not advance the page (e.g. helix ignored the +/// cursor and re-served the same 200 rows). Stop rather than refetch the +/// same page up to `MAX_PAGES` times. +/// - **Range-complete guard:** once we have collected at least one in-range row +/// (`prev_seen_count > 0`), a page that adds no new in-range rows means we +/// have paged past the requested window in whichever direction the relay +/// orders results. Stop. While `prev_seen_count == 0` we keep paging so that +/// an ascending relay whose earliest pages sit below the window still reaches +/// the window instead of terminating after page 1. +fn page_made_progress( + prev_cursor: Option<&str>, + new_cursor: Option<&str>, + prev_seen_count: usize, + new_seen_count: usize, +) -> bool { + if new_cursor == prev_cursor { + return false; + } + if prev_seen_count > 0 && new_seen_count == prev_seen_count { + return false; + } + true +} + /// Relay data API client. pub struct RelayClient { client: reqwest::Client, @@ -56,60 +99,77 @@ impl RelayClient { /// /// Returns payloads delivered in the given slot range. /// - /// The relay enforces a maximum limit of 200. We paginate using `cursor` - /// (which is an opaque DB ID from the last item's `block_number` field) until - /// we've fetched all payloads in the slot range or the relay returns no more results. + /// The relay caps `limit` at 200, so we paginate with `cursor` (an opaque + /// DB id we source from the last row's `block_number`). We make **no** + /// assumption about how helix orders results (ascending or descending by + /// slot) or whether it honors `cursor` at all: + /// + /// - Termination is order-agnostic: we stop once a page adds no new + /// in-range rows after we have already collected some (see + /// [`page_made_progress`]), which terminates correctly in either + /// direction without undercounting deliveries past the first page. + /// - If the relay ignores `cursor` and re-serves the same page, the cursor + /// does not advance and we stop after 2 pages instead of refetching the + /// same rows `MAX_PAGES` times. + /// - Rows are de-duplicated by `block_hash`, so a refetched page cannot + /// double-count a delivery. pub async fn get_payloads_delivered( &self, start_slot: u64, end_slot: u64, ) -> Result> { let mut all = Vec::new(); + let mut seen: HashSet = HashSet::new(); let mut cursor: Option = None; - let max_pages = 50; // safety: 50 × 200 = 10,000 payloads max - for _ in 0..max_pages { + for _ in 0..MAX_PAGES { let url = format!( "{}/relay/v1/data/bidtraces/proposer_payload_delivered", self.base_url ); - let mut req = self.client.get(&url).query(&[("limit", "200")]); + let mut req = self + .client + .get(&url) + .query(&[("limit", PAGE_LIMIT.to_string())]); if let Some(ref c) = cursor { req = req.query(&[("cursor", c)]); } - let resp: Vec = req - .send() - .await? - .error_for_status()? - .json() - .await?; + let resp: Vec = + req.send().await?.error_for_status()?.json().await?; if resp.is_empty() { break; } - // Check if we've gone past our slot range - let min_slot = resp.iter().map(|p| p.slot).min().unwrap_or(0); - let _max_slot = resp.iter().map(|p| p.slot).max().unwrap_or(0); + let prev_seen_count = seen.len(); - // Filter to our slot range + // Collect in-range rows, de-duplicated by block hash so a relay that + // ignores the cursor (and re-serves the same page) cannot + // double-count a delivery. for p in &resp { - if p.slot >= start_slot && p.slot <= end_slot { + if p.slot >= start_slot && p.slot <= end_slot && seen.insert(p.block_hash) { all.push(p.clone()); } } + let new_seen_count = seen.len(); - // If the oldest result is before our range, we can stop - if min_slot < start_slot { + // A short page means the relay had nothing more for this query; this + // is the only stop condition the normal small-window case hits, so + // it still completes in a single request. + if resp.len() < PAGE_LIMIT { break; } - // Use the last item's block_number as cursor for pagination - if let Some(last) = resp.last() { - cursor = Some(last.block_number.to_string()); - } else { + let new_cursor = resp.last().map(|last| last.block_number.to_string()); + if !page_made_progress( + cursor.as_deref(), + new_cursor.as_deref(), + prev_seen_count, + new_seen_count, + ) { break; } + cursor = new_cursor; } Ok(all) @@ -138,27 +198,47 @@ impl RelayClient { Ok(entries) } +} - /// Check if a validator is registered with the relay. - /// - /// GET /relay/v1/data/validator_registration?pubkey={pubkey} - /// Returns true if 200, false otherwise. - pub async fn is_validator_registered(&self, pubkey: &str) -> bool { - match self - .client - .get(format!( - "{}/relay/v1/data/validator_registration", - self.base_url - )) - .query(&[("pubkey", pubkey)]) - .send() - .await - { - Ok(resp) => resp.status().is_success(), - Err(e) => { - warn!("Failed to check registration for {pubkey}: {e}"); - false - } - } +#[cfg(test)] +mod tests { + use super::page_made_progress; + + #[test] + fn continues_when_cursor_advances_and_coverage_grows() { + // Normal forward progress: new cursor, new in-range rows. + assert!(page_made_progress(Some("100"), Some("101"), 5, 12)); + } + + #[test] + fn continues_on_first_page() { + // Page 1: no previous cursor, first in-range rows collected. + assert!(page_made_progress(None, Some("42"), 0, 10)); + } + + #[test] + fn stops_when_cursor_does_not_advance() { + // Relay ignored the cursor and re-served the same page (same last + // block_number). Bounds the ignore-cursor case to 2 pages. + assert!(!page_made_progress(Some("77"), Some("77"), 3, 3)); + // ...even if it somehow reported "new" rows, an unchanged cursor stops us. + assert!(!page_made_progress(Some("77"), Some("77"), 3, 9)); + } + + #[test] + fn stops_when_in_range_and_page_adds_nothing_new() { + // We already collected in-range rows and this (cursor-advanced) page + // adds none: we've paged past the window. This is the order-agnostic + // range-complete termination (holds for ascending or descending). + assert!(!page_made_progress(Some("50"), Some("40"), 11, 11)); // descending + assert!(!page_made_progress(Some("50"), Some("60"), 11, 11)); // ascending + } + + #[test] + fn keeps_paging_below_range_before_entering_window() { + // Ascending relay whose early pages sit entirely below the window: + // zero in-range rows yet, so we must keep paging to reach the window + // rather than terminate after page 1 (the old descending-only bug). + assert!(page_made_progress(Some("10"), Some("20"), 0, 0)); } } diff --git a/src/report.rs b/src/report.rs index 7290037..a9df664 100644 --- a/src/report.rs +++ b/src/report.rs @@ -1,26 +1,60 @@ //! Verification report formatting: terminal (colored) and JSON output. use colored::Colorize; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use crate::checks::{CheckResult, CheckStatus}; /// Observation window: slot range that was verified. -#[derive(Debug, Clone, Serialize)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct ObservationWindow { pub start_slot: u64, pub end_slot: u64, } +/// A single Docker image the run used, with its resolved image ID. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ImageRef { + /// Role of the image in the pipeline (helix_relay, mev_boost, ...). + pub role: String, + /// Image name/tag as configured (e.g. `ghcr.io/gattaca-com/helix-relay:main`). + pub name: String, + /// Resolved Docker image ID (`sha256:...`), or `null` when the image was + /// not present locally / could not be inspected. Deliberately serialized as + /// `null` (not omitted) so a report records that the image was unresolved. + #[serde(default)] + pub id: Option, +} + +/// Provenance: WHAT was tested. Makes a report self-describing so two runs can +/// be compared for regression detection (consumed by `sim diff`). +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Provenance { + /// Path to the Kurtosis config the run used (as passed on the CLI). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub config_path: Option, + /// Short content fingerprint of the config file bytes (first 12 hex of a + /// std `DefaultHasher`). `None` if no config was read. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub config_hash: Option, + /// Resolved Docker image IDs for the images the devnet ran. + #[serde(default)] + pub images: Vec, +} + /// Full verification report. -#[derive(Debug, Clone, Serialize)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct VerificationReport { pub enclave: String, pub timestamp: String, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default, skip_serializing_if = "Option::is_none")] pub observation_window: Option, pub result: CheckStatus, pub checks: Vec, + /// What was tested (config + resolved image IDs). Best-effort: `None` when + /// docker is unreachable or the config could not be parsed. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub provenance: Option, } /// Print the report to stdout in terminal-colored format. @@ -106,7 +140,11 @@ pub fn print_report(report: &VerificationReport, json_mode: bool) { /// Logs a warning on failure but does not return an error — the /// verification itself has already completed. pub fn save_json_report(report: &VerificationReport, output_dir: &str) { - let report_path = format!("{}/{}.json", output_dir.trim_end_matches('/'), report.enclave); + let report_path = format!( + "{}/{}.json", + output_dir.trim_end_matches('/'), + report.enclave + ); match serde_json::to_string_pretty(report) { Ok(json) => match std::fs::write(&report_path, &json) { Ok(_) => { @@ -124,18 +162,424 @@ pub fn save_json_report(report: &VerificationReport, output_dir: &str) { } } +/// The single tier-1 failure predicate shared by the report's overall `result` +/// and by [`exit_code`], so the two verdicts can never diverge. +/// +/// True iff any tier-1 check has status `Fail`. Note a tier-1 `Skip` is NOT a +/// failure — this is what the C1 fix relies on: it makes the relay check FAIL, +/// it does not lean on exit_code treating Skip as a failure. +pub fn tier1_failed(checks: &[CheckResult]) -> bool { + checks + .iter() + .any(|c| c.tier == 1 && c.status == CheckStatus::Fail) +} + /// Determine process exit code from the report. /// /// - 0: all tier-1 checks passed /// - 1: any tier-1 check failed /// - 2: setup/discovery failure (no tier-1 checks ran) pub fn exit_code(report: &VerificationReport) -> i32 { - let tier1: Vec<_> = report.checks.iter().filter(|c| c.tier == 1).collect(); - if tier1.is_empty() { + exit_code_with_policy(report, false) +} + +/// A tier-1 check that was armed and measured nothing. See +/// [`crate::checks::CheckResult::inconclusive`]. +pub fn tier1_inconclusive(checks: &[CheckResult]) -> bool { + checks.iter().any(|c| c.tier == 1 && c.inconclusive) +} + +/// `exit_code` plus the opt-in Law 3 rule. +/// +/// With `require_feature_proof`, a tier-1 check that armed a differential and +/// then observed nothing fails the run. Without it that check is an annotative +/// WARN and the run exits 0 — the historical contract in docs/CHECKS.md. +/// +/// This exists because tier-1 WARN is non-fatal, so a scenario whose whole +/// purpose is to exercise one feature could report "NOT asserting the feature +/// ran" and still exit 0, and a sweep would count it as a win. +pub fn exit_code_with_policy(report: &VerificationReport, require_feature_proof: bool) -> i32 { + let has_tier1 = report.checks.iter().any(|c| c.tier == 1); + if !has_tier1 { return 2; } - if tier1.iter().any(|c| c.status == CheckStatus::Fail) { + if tier1_failed(&report.checks) { + return 1; + } + if require_feature_proof && tier1_inconclusive(&report.checks) { return 1; } 0 } + +// --------------------------------------------------------------------------- +// Provenance: record WHAT was tested (config + resolved Docker image IDs). +// --------------------------------------------------------------------------- + +// Baked image defaults — mirror `sim`'s `Images::default()` (that map lives in +// the `sim` binary crate, which this binary can't import). Used when a config +// does not pin the image, or when no config was supplied. +const DEFAULT_HELIX_RELAY_IMAGE: &str = "ghcr.io/gattaca-com/helix-relay:main"; +const DEFAULT_MEV_BOOST_IMAGE: &str = "commit-boost/commit-boost:kurtosis"; +const DEFAULT_MEV_BUILDER_IMAGE: &str = "ethpandaops/reth-rbuilder:develop"; +const DEFAULT_MEV_BUILDER_CL_IMAGE: &str = "sigp/lighthouse:latest"; + +/// Short content fingerprint of `bytes`: first 12 hex chars of a std +/// `DefaultHasher`. Not cryptographic — a cheap change-detector for `sim diff`. +fn short_hash(bytes: &[u8]) -> String { + use std::hash::{Hash, Hasher}; + let mut h = std::collections::hash_map::DefaultHasher::new(); + bytes.hash(&mut h); + // 64-bit digest => 16 hex chars; take the leading 12. + format!("{:016x}", h.finish())[..12].to_string() +} + +/// Marker: `docker` itself was unreachable (binary missing / daemon down), as +/// distinct from an image simply not being present locally. +struct DockerUnavailable; + +/// Resolve a Docker image name to its ID via `docker image inspect`. +/// +/// - `Ok(Some(id))` — image present, ID resolved. +/// - `Ok(None)` — docker works but the image isn't present (=> null in report). +/// - `Err(DockerUnavailable)` — docker binary missing / daemon unreachable. +fn resolve_image_id(image: &str) -> Result, DockerUnavailable> { + let output = std::process::Command::new("docker") + .args(["image", "inspect", "--format", "{{.Id}}", image]) + .output(); + + let out = match output { + Ok(o) => o, + // Spawn failed (docker not on PATH) => docker unavailable. + Err(_) => return Err(DockerUnavailable), + }; + + if out.status.code() == Some(127) { + return Err(DockerUnavailable); + } + + if out.status.success() { + let id = String::from_utf8_lossy(&out.stdout).trim().to_string(); + return Ok(if id.is_empty() { None } else { Some(id) }); + } + + // Non-zero exit: distinguish "daemon down" (unavailable) from "no such + // image" (a present-but-empty null, best-effort). + let stderr = String::from_utf8_lossy(&out.stderr); + if stderr.contains("Cannot connect to the Docker daemon") || stderr.contains("daemon") { + Err(DockerUnavailable) + } else { + Ok(None) + } +} + +/// Gather run provenance: the config path + content hash, and the resolved +/// Docker image IDs for the images the devnet used. +/// +/// Best-effort and side-effect-free on failure: +/// - a provided-but-unreadable/unparseable config => `None` (we won't attach +/// misleading default image names to a run that used a bespoke config); +/// - docker unreachable => `None`; +/// - an individual image not present locally => that image's `id` is `null`. +/// +/// Image names come from the config's `mev_params.{helix_relay,mev_boost, +/// mev_builder,mev_builder_cl}_image` fields, falling back to the baked defaults. +pub fn gather_provenance(config_path: Option<&str>) -> Option { + let (mev_params, config_hash) = match config_path { + Some(path) => { + let bytes = std::fs::read(path).ok()?; + let root: serde_yaml::Value = serde_yaml::from_slice(&bytes).ok()?; + (root.get("mev_params").cloned(), Some(short_hash(&bytes))) + } + None => (None, None), + }; + + // (role, config key, baked default) + let specs = [ + ( + "helix_relay", + "helix_relay_image", + DEFAULT_HELIX_RELAY_IMAGE, + ), + ("mev_boost", "mev_boost_image", DEFAULT_MEV_BOOST_IMAGE), + ( + "mev_builder", + "mev_builder_image", + DEFAULT_MEV_BUILDER_IMAGE, + ), + ( + "mev_builder_cl", + "mev_builder_cl_image", + DEFAULT_MEV_BUILDER_CL_IMAGE, + ), + ]; + + let mut images = Vec::with_capacity(specs.len()); + for (role, key, default) in specs { + let name = mev_params + .as_ref() + .and_then(|m| m.get(key)) + .and_then(|v| v.as_str()) + .map(str::to_string) + .unwrap_or_else(|| default.to_string()); + match resolve_image_id(&name) { + Ok(id) => images.push(ImageRef { + role: role.to_string(), + name, + id, + }), + // If docker itself is down we can't trust ANY id — abandon provenance. + Err(DockerUnavailable) => return None, + } + } + + Some(Provenance { + config_path: config_path.map(str::to_string), + config_hash, + images, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::checks::CheckResult; + + /// Build a report from `(tier, status)` pairs — just enough to exercise the + /// exit-code / result verdict contract. + fn report_from(checks: Vec<(u8, CheckStatus)>) -> VerificationReport { + let checks = checks + .into_iter() + .enumerate() + .map(|(i, (tier, status))| { + let id = format!("check-{i}"); + match status { + CheckStatus::Pass => CheckResult::pass(id, tier, "ok"), + CheckStatus::Fail => CheckResult::fail(id, tier, "bad"), + CheckStatus::Warn => CheckResult::warn(id, tier, "meh"), + CheckStatus::Skip => CheckResult::skip(id, tier, "n/a"), + } + }) + .collect(); + VerificationReport { + enclave: "test".to_string(), + timestamp: "1970-01-01T00:00:00Z".to_string(), + observation_window: None, + result: CheckStatus::Pass, + checks, + provenance: None, + } + } + + /// The interchange contract: a report written to disk by a verification run + /// must be readable back by `sim diff`. These two live in different crates + /// (lib writes, the sim bin reads) and are only connected by serde derives, + /// so nothing but a round-trip proves the pipeline actually composes. + #[test] + fn saved_report_round_trips_back_into_a_report() { + let dir = + std::env::temp_dir().join(format!("cb-report-rt-{}-{}", std::process::id(), line!())); + std::fs::create_dir_all(&dir).unwrap(); + + let mut report = report_from(vec![(1, CheckStatus::Pass), (2, CheckStatus::Warn)]); + report.enclave = "rt-enclave".to_string(); + report.observation_window = Some(ObservationWindow { + start_slot: 160, + end_slot: 224, + }); + report.provenance = Some(Provenance { + config_path: Some("configs/generated/cb-basic.yml".to_string()), + config_hash: Some("abc123def456".to_string()), + images: vec![ImageRef { + role: "mev_boost".to_string(), + name: "commit-boost/commit-boost:kurtosis".to_string(), + id: Some("sha256:deadbeef".to_string()), + }], + }); + + save_json_report(&report, dir.to_str().unwrap()); + + let path = dir.join("rt-enclave.json"); + let raw = + std::fs::read_to_string(&path).expect("save_json_report must write .json"); + let back: VerificationReport = serde_json::from_str(&raw) + .expect("a saved report must deserialize (sim diff reads it)"); + + assert_eq!(back.enclave, "rt-enclave"); + assert_eq!(back.result, report.result); + assert_eq!(back.checks.len(), 2); + assert_eq!(back.observation_window.unwrap().end_slot, 224); + let prov = back + .provenance + .expect("provenance must survive the round trip"); + assert_eq!(prov.config_hash.as_deref(), Some("abc123def456")); + assert_eq!(prov.images[0].id.as_deref(), Some("sha256:deadbeef")); + // The wire name is `result`, not `status` - sim diff and any external + // consumer key on it. + assert!( + raw.contains("\"result\""), + "checks serialize their status as `result`" + ); + + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + fn save_is_a_no_op_when_the_directory_does_not_exist() { + // Documented behavior: saving never fails the run, it warns. A verify + // that has already finished must not die on a bad --output-dir. + let report = report_from(vec![(1, CheckStatus::Pass)]); + save_json_report(&report, "/nonexistent/dir/for/cb/report"); + // Reaching here without panicking IS the assertion. + } + + #[test] + fn short_hash_is_deterministic_and_fixed_width() { + let a = short_hash(b"the same bytes"); + let b = short_hash(b"the same bytes"); + let c = short_hash(b"different bytes"); + assert_eq!(a, b, "same input must give the same config_hash"); + assert_ne!(a, c, "different configs must be distinguishable"); + assert_eq!(a.len(), 12, "12 hex chars, as `sim diff` prints them"); + assert!(a.chars().all(|ch| ch.is_ascii_hexdigit())); + } + + #[test] + fn short_hash_handles_empty_input() { + assert_eq!(short_hash(b"").len(), 12); + } + + #[test] + fn print_report_json_and_terminal_both_run() { + // Smoke: neither renderer may panic on a report carrying every status, + // a FAIL with data (the terminal path prints data lines for fails), an + // observation window and provenance. + let mut report = report_from(vec![ + (1, CheckStatus::Pass), + (1, CheckStatus::Fail), + (2, CheckStatus::Warn), + (3, CheckStatus::Skip), + ]); + report.observation_window = Some(ObservationWindow { + start_slot: 1, + end_slot: 2, + }); + print_report(&report, true); + print_report(&report, false); + } + + #[test] + fn exit_code_no_tier1_checks_is_2() { + // No tier-1 checks ran at all => setup/discovery failure. + let report = report_from(vec![(2, CheckStatus::Pass), (3, CheckStatus::Fail)]); + assert_eq!(exit_code(&report), 2); + } + + #[test] + fn exit_code_empty_report_is_2() { + let report = report_from(vec![]); + assert_eq!(exit_code(&report), 2); + } + + #[test] + fn exit_code_tier1_fail_is_1() { + let report = report_from(vec![(1, CheckStatus::Pass), (1, CheckStatus::Fail)]); + assert_eq!(exit_code(&report), 1); + } + + #[test] + fn exit_code_all_tier1_non_fail_is_0() { + // Pass / Warn / Skip on tier-1 checks are all a green verdict. + let report = report_from(vec![ + (1, CheckStatus::Pass), + (1, CheckStatus::Warn), + (1, CheckStatus::Skip), + ]); + assert_eq!(exit_code(&report), 0); + } + + #[test] + fn exit_code_tier1_skip_alone_is_0() { + // The subtle one the C1 fix relies on: a tier-1 SKIP does NOT fail the + // run. exit_code only fails on Fail — so the C1 fix works by making the + // relay check FAIL, not by changing exit_code's treatment of Skip. + let report = report_from(vec![(1, CheckStatus::Skip)]); + assert_eq!(exit_code(&report), 0); + } + + #[test] + fn exit_code_ignores_non_tier1_fail_when_tier1_passes() { + // A tier-1 check exists and passes; a tier-2 Fail must not flip to 1. + let report = report_from(vec![(1, CheckStatus::Pass), (2, CheckStatus::Fail)]); + assert_eq!(exit_code(&report), 0); + } + + /// Build a report whose single tier-1 WARN is marked inconclusive: the + /// "armed a differential and measured nothing" shape. + fn report_with_inconclusive_tier1() -> VerificationReport { + let mut report = report_from(vec![(1, CheckStatus::Pass), (1, CheckStatus::Warn)]); + report.checks[1] = report.checks[1].clone().mark_inconclusive(); + report + } + + // Contract: default policy is unchanged. A tier-1 inconclusive WARN still + // exits 0, so existing callers and the docs/CHECKS.md contract are intact. + #[test] + fn inconclusive_tier1_is_not_fatal_by_default() { + let report = report_with_inconclusive_tier1(); + assert_eq!(exit_code(&report), 0); + assert_eq!(exit_code_with_policy(&report, false), 0); + } + + // Contract: with --require-feature-proof, an armed-but-unmeasured tier-1 + // check fails the run. This is the Law 3 rule: a scenario that proved + // nothing must not be counted as a win. + #[test] + fn inconclusive_tier1_fails_under_require_feature_proof() { + let report = report_with_inconclusive_tier1(); + assert_eq!(exit_code_with_policy(&report, true), 1); + assert!(tier1_inconclusive(&report.checks)); + } + + // Contract: the flag only promotes INCONCLUSIVE. A plain tier-1 WARN (an + // annotative anomaly like relay equivocation) stays non-fatal even under the + // strict policy, so turning the flag on does not redden honest warnings. + #[test] + fn plain_tier1_warn_stays_green_under_require_feature_proof() { + let report = report_from(vec![(1, CheckStatus::Pass), (1, CheckStatus::Warn)]); + assert!(!tier1_inconclusive(&report.checks)); + assert_eq!(exit_code_with_policy(&report, true), 0); + } + + // Contract: an inconclusive check at tier 2/3 is never fatal — the rule is + // scoped to tier 1, matching the rest of the exit contract. + #[test] + fn inconclusive_below_tier1_is_never_fatal() { + let mut report = report_from(vec![(1, CheckStatus::Pass), (2, CheckStatus::Warn)]); + report.checks[1] = report.checks[1].clone().mark_inconclusive(); + assert!(!tier1_inconclusive(&report.checks)); + assert_eq!(exit_code_with_policy(&report, true), 0); + } + + // Contract: a real tier-1 FAIL still wins over the inconclusive rule, and + // "no tier-1 checks ran" is still 2 rather than being masked by the flag. + #[test] + fn fail_and_no_tier1_precedence_unchanged_under_the_flag() { + let mut failing = report_from(vec![(1, CheckStatus::Fail)]); + failing.checks[0] = failing.checks[0].clone().mark_inconclusive(); + assert_eq!(exit_code_with_policy(&failing, true), 1); + + let no_tier1 = report_from(vec![(2, CheckStatus::Warn)]); + assert_eq!(exit_code_with_policy(&no_tier1, true), 2); + } + + #[test] + fn tier1_failed_matches_exit_code_predicate() { + // The shared predicate and exit_code agree on the fail case. + let checks: Vec = report_from(vec![(1, CheckStatus::Fail)]).checks; + assert!(tier1_failed(&checks)); + + let clean: Vec = report_from(vec![(1, CheckStatus::Skip)]).checks; + assert!(!tier1_failed(&clean)); + } +} diff --git a/tests/fixtures/ansi_colored.log b/tests/fixtures/ansi_colored.log new file mode 100644 index 0000000..48eb6a6 --- /dev/null +++ b/tests/fixtures/ansi_colored.log @@ -0,0 +1,5 @@ +2026-07-30T13:00:00.000Z INFO relay: booting +2026-07-30T13:00:00.500Z INFO relay: parsing flags +thread 'main' panicked at src/main.rs:10:5: +explicit panic: invalid --cores value: expected integer +note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace diff --git a/tests/fixtures/bind_error.log b/tests/fixtures/bind_error.log new file mode 100644 index 0000000..272a8de --- /dev/null +++ b/tests/fixtures/bind_error.log @@ -0,0 +1,2 @@ +2026-07-30T16:00:00.000Z INFO relay: starting HTTP server on 0.0.0.0:18550 +Error: Address already in use (os error 98) diff --git a/tests/fixtures/clean.log b/tests/fixtures/clean.log new file mode 100644 index 0000000..c586ef5 --- /dev/null +++ b/tests/fixtures/clean.log @@ -0,0 +1,4 @@ +2026-07-30T17:00:00.000Z INFO relay: service started +2026-07-30T17:00:01.000Z INFO relay: listening on 0.0.0.0:8080 +2026-07-30T17:00:02.000Z INFO relay: ready to accept connections +2026-07-30T17:00:03.000Z INFO relay: registered with beacon node diff --git a/tests/fixtures/docker_pull_error.log b/tests/fixtures/docker_pull_error.log new file mode 100644 index 0000000..0c77438 --- /dev/null +++ b/tests/fixtures/docker_pull_error.log @@ -0,0 +1,4 @@ +Unable to find image 'ghcr.io/gattaca-com/helix-relay:main' locally +docker: Error response from daemon: manifest unknown: manifest unknown. +See 'docker run --help'. +Error: No such image: ghcr.io/gattaca-com/helix-relay:main diff --git a/tests/fixtures/enclave_inspect.txt b/tests/fixtures/enclave_inspect.txt new file mode 100644 index 0000000..bfeebdc --- /dev/null +++ b/tests/fixtures/enclave_inspect.txt @@ -0,0 +1,15 @@ +Name: CB-Testnet +UUID: 8f3c1a2b4d5e6f708192a3b4c5d6e7f8 +Status: RUNNING +Creation Time: Thu, 30 Jul 2026 10:00:00 UTC +Flags: + +========================================== Files Artifacts ========================================== +UUID Name +1a2b3c4d genesis-data +5e6f7a8b cl-genesis-data + +========================================== User Services ========================================== +UUID Name Ports Status +aaa111bbb222ccc333ddd444eee555f01 cl-1-lighthouse-geth http: 4000/tcp -> http://127.0.0.1:32811 RUNNING +bbb222ccc333ddd444eee555fff666a02 mev-relay-helix STOPPED diff --git a/tests/fixtures/golden-configs/cb-basic-nethermind-prysm.yml b/tests/fixtures/golden-configs/cb-basic-nethermind-prysm.yml new file mode 100644 index 0000000..68a1fa2 --- /dev/null +++ b/tests/fixtures/golden-configs/cb-basic-nethermind-prysm.yml @@ -0,0 +1,164 @@ +# cb-basic-nethermind-prysm: cb-basic on an ALTERNATE EL/CL pair. +# +# Law 7 (coverage is a matrix, not a point): every other scenario runs +# geth+lighthouse, so a CB regression specific to another client pair is +# invisible. Same MEV pipeline assertions as cb-basic, different clients. + +participants: + - el_type: nethermind + cl_type: prysm + +additional_services: + - dora + - spamoor + - prometheus + +mev_type: custom + +mev_params: + mev_relay: helix + mev_sidecar: commit-boost + mev_builder: flashbots + + helix_relay_image: ghcr.io/gattaca-com/helix-relay:main + mev_boost_image: commit-boost/commit-boost:kurtosis + mev_builder_image: ethpandaops/reth-rbuilder:develop + mev_builder_cl_image: sigp/lighthouse:latest + + mev_builder_subsidy: 1 + + helix_relay_config: | + instance_id: "helix-kurtosis-test" + + # NOTE: no network/genesis section. Current helix-relay:main removed the old + # `network_config: !Custom {dir_path, genesis_validator_root, genesis_time}` + # field entirely; the relay now fetches the chain spec + genesis from the + # beacon node at startup (GET eth/v1/config/spec + eth/v1/beacon/genesis, see + # beacon_client.get_chain_info -> main.rs load_chain_info). A `!Custom` YAML + # tag on that now-unknown key makes serde_yaml panic with + # "untagged and internally tagged enums do not support enum input". + + postgres: + hostname: "{{ .POSTGRES_HOST_NAME }}" + port: {{ .POSTGRES_PORT }} + db_name: "{{ .POSTGRES_DB }}" + user: "{{ .POSTGRES_USER }}" + password: "{{ .POSTGRES_PASS }}" + region: 0 + region_name: "LOCAL" + + beacon_clients: + - url: "{{ .BEACON_URI }}" + + gossip_payload_on_header: false + + simulators: + - url: "{{ .BLOCKSIM_URI }}" + namespace: flashbots + is_merging_simulator: false + max_concurrent_tasks: 32 + + router_config: + enabled_routes: + - route: GetValidators + - route: SubmitBlock + - route: GetTopBid + - route: GetHeader + rate_limit: + replenish_ms: 50 + burst_size: 20 + - route: GetPayload + # The builder-spec v2 proposer route (submitBlindedBlockV2). REQUIRED for + # any CL that submits via v2 -- prysm does. Without it helix 404s + # /eth/v2/builder/blinded_blocks, CB (correctly) refuses to downgrade to v1 + # (v2 semantics: the relay publishes the block, so a v1 payload would be + # silently dropped), returns 502, and EVERY builder block the proposer + # chose is lost. That read as "prysm can't do MEV" until the route list was + # checked -- see .agent/SWEEP-BACKLOG.md (Law 7 first dividend). + - route: GetPayloadV2 + - route: HeaderStream + - route: RegisterValidators + - route: Status + - route: ProposerPayloadDelivered + - route: BuilderBidsReceived + - route: ValidatorRegistration + shutdown_delay_ms: 12000 + + timing_game_config: + max_header_delay_ms: 400 + latest_header_delay_ms_in_slot: 1500 + default_client_latency_ms: 50 + + target_get_payload_propagation_duration_ms: 500 + + is_submission_instance: true + is_registration_instance: true + + admin_token: "test_admin_token" + + logging: + type: Console + + # CoresConfig (helix_common::config CoresConfig, 10 fields in current + # :main). The old block used `sub_workers: [0]` and omitted the per-tile + # core assignments; current :main removed sub_workers and added + # decoder/simulator/top_bid/data_gatherer/block_merging/housekeeper. The + # outer RelayConfigExt flattens RelayConfig, so a wrong/missing cores field + # surfaces as a top-level "missing field `decoder`" / "invalid type ... + # expected usize|sequence" serde error. Verified against the binary: + # `decoder` is Vec ([0]); the other five new tile fields are usize. + cores: + auctioneer: 0 + tokio: [0] + reg_workers: [0] + tcp_bid_submissions_tile: 2 + decoder: [0] + simulator: 0 + top_bid: 0 + data_gatherer: 0 + block_merging: 0 + housekeeper: 0 + + is_local_dev: false + + commit_boost_config: | + chain = { genesis_time_secs = {{ .Timestamp }}, path = "{{ .Network }}" } + + [pbs] + host = "0.0.0.0" + port = {{ .Port }} + timeout_get_header_ms = 950 + timeout_get_payload_ms = 4000 + late_in_slot_time_ms = 2000 + + + [metrics] + enabled = true + host = "0.0.0.0" + start_port = 9090 + + {{ range $index, $relay := .Relays }} + [[relays]] + id = "mev_relay_{{$index}}" + url = "{{ $relay }}" + {{- end }} + + [logs.stdout] + level = "debug" + + [logs.file] + enabled = false + +network_params: + network: kurtosis + network_id: "3151908" + deposit_contract_address: "0x00000000219ab540356cBB839Cbe05303d7705Fa" + seconds_per_slot: 12 + slot_duration_ms: 12000 + num_validator_keys_per_node: 128 + preregistered_validator_keys_mnemonic: + "giant issue aisle success illegal bike spike + question tent bar rely arctic volcano long crawl hungry vocal artwork sniff fantasy + very lucky have athlete" + prefunded_accounts: '{"0xb9e79d19f651a941757b35830232E7EFC77E1c79": {"balance": "100000ETH"}}' + diff --git a/tests/fixtures/golden-configs/cb-basic.yml b/tests/fixtures/golden-configs/cb-basic.yml new file mode 100644 index 0000000..b5ddd33 --- /dev/null +++ b/tests/fixtures/golden-configs/cb-basic.yml @@ -0,0 +1,163 @@ +# cb-basic: Single relay (helix) with default Commit-Boost config. +# +# Tests the core MEV pipeline through Commit-Boost with a single Helix +# relay as the only relay endpoint. + +participants: + - el_type: geth + cl_type: lighthouse + +additional_services: + - dora + - spamoor + - prometheus + +mev_type: custom + +mev_params: + mev_relay: helix + mev_sidecar: commit-boost + mev_builder: flashbots + + helix_relay_image: ghcr.io/gattaca-com/helix-relay:main + mev_boost_image: commit-boost/commit-boost:kurtosis + mev_builder_image: ethpandaops/reth-rbuilder:develop + mev_builder_cl_image: sigp/lighthouse:latest + + mev_builder_subsidy: 1 + + helix_relay_config: | + instance_id: "helix-kurtosis-test" + + # NOTE: no network/genesis section. Current helix-relay:main removed the old + # `network_config: !Custom {dir_path, genesis_validator_root, genesis_time}` + # field entirely; the relay now fetches the chain spec + genesis from the + # beacon node at startup (GET eth/v1/config/spec + eth/v1/beacon/genesis, see + # beacon_client.get_chain_info -> main.rs load_chain_info). A `!Custom` YAML + # tag on that now-unknown key makes serde_yaml panic with + # "untagged and internally tagged enums do not support enum input". + + postgres: + hostname: "{{ .POSTGRES_HOST_NAME }}" + port: {{ .POSTGRES_PORT }} + db_name: "{{ .POSTGRES_DB }}" + user: "{{ .POSTGRES_USER }}" + password: "{{ .POSTGRES_PASS }}" + region: 0 + region_name: "LOCAL" + + beacon_clients: + - url: "{{ .BEACON_URI }}" + + gossip_payload_on_header: false + + simulators: + - url: "{{ .BLOCKSIM_URI }}" + namespace: flashbots + is_merging_simulator: false + max_concurrent_tasks: 32 + + router_config: + enabled_routes: + - route: GetValidators + - route: SubmitBlock + - route: GetTopBid + - route: GetHeader + rate_limit: + replenish_ms: 50 + burst_size: 20 + - route: GetPayload + # The builder-spec v2 proposer route (submitBlindedBlockV2). REQUIRED for + # any CL that submits via v2 -- prysm does. Without it helix 404s + # /eth/v2/builder/blinded_blocks, CB (correctly) refuses to downgrade to v1 + # (v2 semantics: the relay publishes the block, so a v1 payload would be + # silently dropped), returns 502, and EVERY builder block the proposer + # chose is lost. That read as "prysm can't do MEV" until the route list was + # checked -- see .agent/SWEEP-BACKLOG.md (Law 7 first dividend). + - route: GetPayloadV2 + - route: HeaderStream + - route: RegisterValidators + - route: Status + - route: ProposerPayloadDelivered + - route: BuilderBidsReceived + - route: ValidatorRegistration + shutdown_delay_ms: 12000 + + timing_game_config: + max_header_delay_ms: 400 + latest_header_delay_ms_in_slot: 1500 + default_client_latency_ms: 50 + + target_get_payload_propagation_duration_ms: 500 + + is_submission_instance: true + is_registration_instance: true + + admin_token: "test_admin_token" + + logging: + type: Console + + # CoresConfig (helix_common::config CoresConfig, 10 fields in current + # :main). The old block used `sub_workers: [0]` and omitted the per-tile + # core assignments; current :main removed sub_workers and added + # decoder/simulator/top_bid/data_gatherer/block_merging/housekeeper. The + # outer RelayConfigExt flattens RelayConfig, so a wrong/missing cores field + # surfaces as a top-level "missing field `decoder`" / "invalid type ... + # expected usize|sequence" serde error. Verified against the binary: + # `decoder` is Vec ([0]); the other five new tile fields are usize. + cores: + auctioneer: 0 + tokio: [0] + reg_workers: [0] + tcp_bid_submissions_tile: 2 + decoder: [0] + simulator: 0 + top_bid: 0 + data_gatherer: 0 + block_merging: 0 + housekeeper: 0 + + is_local_dev: false + + commit_boost_config: | + chain = { genesis_time_secs = {{ .Timestamp }}, path = "{{ .Network }}" } + + [pbs] + host = "0.0.0.0" + port = {{ .Port }} + timeout_get_header_ms = 950 + timeout_get_payload_ms = 4000 + late_in_slot_time_ms = 2000 + + + [metrics] + enabled = true + host = "0.0.0.0" + start_port = 9090 + + {{ range $index, $relay := .Relays }} + [[relays]] + id = "mev_relay_{{$index}}" + url = "{{ $relay }}" + {{- end }} + + [logs.stdout] + level = "debug" + + [logs.file] + enabled = false + +network_params: + network: kurtosis + network_id: "3151908" + deposit_contract_address: "0x00000000219ab540356cBB839Cbe05303d7705Fa" + seconds_per_slot: 12 + slot_duration_ms: 12000 + num_validator_keys_per_node: 128 + preregistered_validator_keys_mnemonic: + "giant issue aisle success illegal bike spike + question tent bar rely arctic volcano long crawl hungry vocal artwork sniff fantasy + very lucky have athlete" + prefunded_accounts: '{"0xb9e79d19f651a941757b35830232E7EFC77E1c79": {"balance": "100000ETH"}}' + diff --git a/tests/fixtures/golden-configs/cb-extra-validation.yml b/tests/fixtures/golden-configs/cb-extra-validation.yml new file mode 100644 index 0000000..8b8b9a7 --- /dev/null +++ b/tests/fixtures/golden-configs/cb-extra-validation.yml @@ -0,0 +1,166 @@ +# cb-extra-validation: Enable extra validation of get_header responses +# via a local execution layer client. +# +# Tests that CB will RPC-call the execution client to verify block +# parameters before returning a header to the beacon node. + +participants: + - el_type: geth + cl_type: lighthouse + +additional_services: + - dora + - spamoor + - prometheus + +mev_type: custom + +mev_params: + mev_relay: helix + mev_sidecar: commit-boost + mev_builder: flashbots + + helix_relay_image: ghcr.io/gattaca-com/helix-relay:main + mev_boost_image: commit-boost/commit-boost:kurtosis + mev_builder_image: ethpandaops/reth-rbuilder:develop + mev_builder_cl_image: sigp/lighthouse:latest + + mev_builder_subsidy: 1 + + helix_relay_config: | + instance_id: "helix-kurtosis-test" + + # NOTE: no network/genesis section. Current helix-relay:main removed the old + # `network_config: !Custom {dir_path, genesis_validator_root, genesis_time}` + # field entirely; the relay now fetches the chain spec + genesis from the + # beacon node at startup (GET eth/v1/config/spec + eth/v1/beacon/genesis, see + # beacon_client.get_chain_info -> main.rs load_chain_info). A `!Custom` YAML + # tag on that now-unknown key makes serde_yaml panic with + # "untagged and internally tagged enums do not support enum input". + + postgres: + hostname: "{{ .POSTGRES_HOST_NAME }}" + port: {{ .POSTGRES_PORT }} + db_name: "{{ .POSTGRES_DB }}" + user: "{{ .POSTGRES_USER }}" + password: "{{ .POSTGRES_PASS }}" + region: 0 + region_name: "LOCAL" + + beacon_clients: + - url: "{{ .BEACON_URI }}" + + gossip_payload_on_header: false + + simulators: + - url: "{{ .BLOCKSIM_URI }}" + namespace: flashbots + is_merging_simulator: false + max_concurrent_tasks: 32 + + router_config: + enabled_routes: + - route: GetValidators + - route: SubmitBlock + - route: GetTopBid + - route: GetHeader + rate_limit: + replenish_ms: 50 + burst_size: 20 + - route: GetPayload + # The builder-spec v2 proposer route (submitBlindedBlockV2). REQUIRED for + # any CL that submits via v2 -- prysm does. Without it helix 404s + # /eth/v2/builder/blinded_blocks, CB (correctly) refuses to downgrade to v1 + # (v2 semantics: the relay publishes the block, so a v1 payload would be + # silently dropped), returns 502, and EVERY builder block the proposer + # chose is lost. That read as "prysm can't do MEV" until the route list was + # checked -- see .agent/SWEEP-BACKLOG.md (Law 7 first dividend). + - route: GetPayloadV2 + - route: HeaderStream + - route: RegisterValidators + - route: Status + - route: ProposerPayloadDelivered + - route: BuilderBidsReceived + - route: ValidatorRegistration + shutdown_delay_ms: 12000 + + timing_game_config: + max_header_delay_ms: 400 + latest_header_delay_ms_in_slot: 1500 + default_client_latency_ms: 50 + + target_get_payload_propagation_duration_ms: 500 + + is_submission_instance: true + is_registration_instance: true + + admin_token: "test_admin_token" + + logging: + type: Console + + # CoresConfig (helix_common::config CoresConfig, 10 fields in current + # :main). The old block used `sub_workers: [0]` and omitted the per-tile + # core assignments; current :main removed sub_workers and added + # decoder/simulator/top_bid/data_gatherer/block_merging/housekeeper. The + # outer RelayConfigExt flattens RelayConfig, so a wrong/missing cores field + # surfaces as a top-level "missing field `decoder`" / "invalid type ... + # expected usize|sequence" serde error. Verified against the binary: + # `decoder` is Vec ([0]); the other five new tile fields are usize. + cores: + auctioneer: 0 + tokio: [0] + reg_workers: [0] + tcp_bid_submissions_tile: 2 + decoder: [0] + simulator: 0 + top_bid: 0 + data_gatherer: 0 + block_merging: 0 + housekeeper: 0 + + is_local_dev: false + + commit_boost_config: | + chain = { genesis_time_secs = {{ .Timestamp }}, path = "{{ .Network }}" } + + [pbs] + host = "0.0.0.0" + port = {{ .Port }} + extra_validation_enabled = true + rpc_url = "http://el-1-geth-lighthouse:8545" + timeout_get_header_ms = 950 + timeout_get_payload_ms = 4000 + late_in_slot_time_ms = 2000 + + + [metrics] + enabled = true + host = "0.0.0.0" + start_port = 9090 + + {{ range $index, $relay := .Relays }} + [[relays]] + id = "mev_relay_{{$index}}" + url = "{{ $relay }}" + {{- end }} + + [logs.stdout] + level = "debug" + + [logs.file] + enabled = false + +network_params: + network: kurtosis + network_id: "3151908" + deposit_contract_address: "0x00000000219ab540356cBB839Cbe05303d7705Fa" + seconds_per_slot: 12 + slot_duration_ms: 12000 + num_validator_keys_per_node: 128 + preregistered_validator_keys_mnemonic: + "giant issue aisle success illegal bike spike + question tent bar rely arctic volcano long crawl hungry vocal artwork sniff fantasy + very lucky have athlete" + prefunded_accounts: '{"0xb9e79d19f651a941757b35830232E7EFC77E1c79": {"balance": "100000ETH"}}' + diff --git a/tests/fixtures/golden-configs/cb-min-bid.yml b/tests/fixtures/golden-configs/cb-min-bid.yml new file mode 100644 index 0000000..e6ead9c --- /dev/null +++ b/tests/fixtures/golden-configs/cb-min-bid.yml @@ -0,0 +1,171 @@ +# cb-min-bid: the min_bid_eth floor actually drops bids. +# +# min_bid_eth = 0.5 with the builder subsidy OFF: real devnet bids are +# ~0.04 ETH of spamoor MEV, so EVERY bid must be rejected with +# "bid below minimum" and zero auctions won. The subsidy must be 0 or +# bids land near 1.04 ETH and no LEGAL floor could reject them - CB +# validates min_bid_wei < 1 ETH. +# +# Doubles as a canary for CB's silent-flatten trap: [pbs] has no +# deny_unknown_fields, so a renamed/misspelled key is IGNORED rather +# than rejected. If bids still win here, the key was silently dropped. + +participants: + - el_type: geth + cl_type: lighthouse + +additional_services: + - dora + - spamoor + - prometheus + +mev_type: custom + +mev_params: + mev_relay: helix + mev_sidecar: commit-boost + mev_builder: flashbots + + helix_relay_image: ghcr.io/gattaca-com/helix-relay:main + mev_boost_image: commit-boost/commit-boost:kurtosis + mev_builder_image: ethpandaops/reth-rbuilder:develop + mev_builder_cl_image: sigp/lighthouse:latest + + mev_builder_subsidy: 0 + + helix_relay_config: | + instance_id: "helix-kurtosis-test" + + # NOTE: no network/genesis section. Current helix-relay:main removed the old + # `network_config: !Custom {dir_path, genesis_validator_root, genesis_time}` + # field entirely; the relay now fetches the chain spec + genesis from the + # beacon node at startup (GET eth/v1/config/spec + eth/v1/beacon/genesis, see + # beacon_client.get_chain_info -> main.rs load_chain_info). A `!Custom` YAML + # tag on that now-unknown key makes serde_yaml panic with + # "untagged and internally tagged enums do not support enum input". + + postgres: + hostname: "{{ .POSTGRES_HOST_NAME }}" + port: {{ .POSTGRES_PORT }} + db_name: "{{ .POSTGRES_DB }}" + user: "{{ .POSTGRES_USER }}" + password: "{{ .POSTGRES_PASS }}" + region: 0 + region_name: "LOCAL" + + beacon_clients: + - url: "{{ .BEACON_URI }}" + + gossip_payload_on_header: false + + simulators: + - url: "{{ .BLOCKSIM_URI }}" + namespace: flashbots + is_merging_simulator: false + max_concurrent_tasks: 32 + + router_config: + enabled_routes: + - route: GetValidators + - route: SubmitBlock + - route: GetTopBid + - route: GetHeader + rate_limit: + replenish_ms: 50 + burst_size: 20 + - route: GetPayload + # The builder-spec v2 proposer route (submitBlindedBlockV2). REQUIRED for + # any CL that submits via v2 -- prysm does. Without it helix 404s + # /eth/v2/builder/blinded_blocks, CB (correctly) refuses to downgrade to v1 + # (v2 semantics: the relay publishes the block, so a v1 payload would be + # silently dropped), returns 502, and EVERY builder block the proposer + # chose is lost. That read as "prysm can't do MEV" until the route list was + # checked -- see .agent/SWEEP-BACKLOG.md (Law 7 first dividend). + - route: GetPayloadV2 + - route: HeaderStream + - route: RegisterValidators + - route: Status + - route: ProposerPayloadDelivered + - route: BuilderBidsReceived + - route: ValidatorRegistration + shutdown_delay_ms: 12000 + + timing_game_config: + max_header_delay_ms: 400 + latest_header_delay_ms_in_slot: 1500 + default_client_latency_ms: 50 + + target_get_payload_propagation_duration_ms: 500 + + is_submission_instance: true + is_registration_instance: true + + admin_token: "test_admin_token" + + logging: + type: Console + + # CoresConfig (helix_common::config CoresConfig, 10 fields in current + # :main). The old block used `sub_workers: [0]` and omitted the per-tile + # core assignments; current :main removed sub_workers and added + # decoder/simulator/top_bid/data_gatherer/block_merging/housekeeper. The + # outer RelayConfigExt flattens RelayConfig, so a wrong/missing cores field + # surfaces as a top-level "missing field `decoder`" / "invalid type ... + # expected usize|sequence" serde error. Verified against the binary: + # `decoder` is Vec ([0]); the other five new tile fields are usize. + cores: + auctioneer: 0 + tokio: [0] + reg_workers: [0] + tcp_bid_submissions_tile: 2 + decoder: [0] + simulator: 0 + top_bid: 0 + data_gatherer: 0 + block_merging: 0 + housekeeper: 0 + + is_local_dev: false + + commit_boost_config: | + chain = { genesis_time_secs = {{ .Timestamp }}, path = "{{ .Network }}" } + + [pbs] + host = "0.0.0.0" + port = {{ .Port }} + min_bid_eth = 0.5 + timeout_get_header_ms = 950 + timeout_get_payload_ms = 4000 + late_in_slot_time_ms = 2000 + + + [metrics] + enabled = true + host = "0.0.0.0" + start_port = 9090 + + {{ range $index, $relay := .Relays }} + [[relays]] + id = "mev_relay_{{$index}}" + url = "{{ $relay }}" + {{- end }} + + [logs.stdout] + level = "debug" + + [logs.file] + enabled = false + +network_params: + network: kurtosis + network_id: "3151908" + deposit_contract_address: "0x00000000219ab540356cBB839Cbe05303d7705Fa" + seconds_per_slot: 12 + slot_duration_ms: 12000 + num_validator_keys_per_node: 128 + preregistered_validator_keys_mnemonic: + "giant issue aisle success illegal bike spike + question tent bar rely arctic volcano long crawl hungry vocal artwork sniff fantasy + very lucky have athlete" + prefunded_accounts: '{"0xb9e79d19f651a941757b35830232E7EFC77E1c79": {"balance": "100000ETH"}}' + diff --git a/tests/fixtures/golden-configs/cb-multiple-relays.yml b/tests/fixtures/golden-configs/cb-multiple-relays.yml new file mode 100644 index 0000000..b1d5e67 --- /dev/null +++ b/tests/fixtures/golden-configs/cb-multiple-relays.yml @@ -0,0 +1,170 @@ +# cb-multiple-relays: Two Helix relay instances behind a single +# Commit-Boost sidecar. +# +# Tests that CB correctly routes get_header requests to both relays, +# aggregating responses and selecting the best bid. The per-relay +# subsidy list [1, 2] makes the builder submit DIVERGENT bid values +# (rbuilder [[subsidy_overrides]]), so the best-bid selection is a +# real discrimination, not a tie between identical bids. + +participants: + - el_type: geth + cl_type: lighthouse + +additional_services: + - dora + - spamoor + - prometheus + +mev_type: custom + +mev_params: + mev_relay: + - helix + - helix + mev_sidecar: commit-boost + mev_builder: flashbots + + helix_relay_image: ghcr.io/gattaca-com/helix-relay:main + mev_relay_image: ethpandaops/mev-boost-relay:main + mev_boost_image: commit-boost/commit-boost:kurtosis + mev_builder_image: ethpandaops/reth-rbuilder:develop + mev_builder_cl_image: sigp/lighthouse:latest + + mev_builder_subsidy: [1, 2] + + helix_relay_config: | + instance_id: "helix-kurtosis-test" + + # NOTE: no network/genesis section. Current helix-relay:main removed the old + # `network_config: !Custom {dir_path, genesis_validator_root, genesis_time}` + # field entirely; the relay now fetches the chain spec + genesis from the + # beacon node at startup (GET eth/v1/config/spec + eth/v1/beacon/genesis, see + # beacon_client.get_chain_info -> main.rs load_chain_info). A `!Custom` YAML + # tag on that now-unknown key makes serde_yaml panic with + # "untagged and internally tagged enums do not support enum input". + + postgres: + hostname: "{{ .POSTGRES_HOST_NAME }}" + port: {{ .POSTGRES_PORT }} + db_name: "{{ .POSTGRES_DB }}" + user: "{{ .POSTGRES_USER }}" + password: "{{ .POSTGRES_PASS }}" + region: 0 + region_name: "LOCAL" + + beacon_clients: + - url: "{{ .BEACON_URI }}" + + gossip_payload_on_header: false + + simulators: + - url: "{{ .BLOCKSIM_URI }}" + namespace: flashbots + is_merging_simulator: false + max_concurrent_tasks: 32 + + router_config: + enabled_routes: + - route: GetValidators + - route: SubmitBlock + - route: GetTopBid + - route: GetHeader + rate_limit: + replenish_ms: 50 + burst_size: 20 + - route: GetPayload + # The builder-spec v2 proposer route (submitBlindedBlockV2). REQUIRED for + # any CL that submits via v2 -- prysm does. Without it helix 404s + # /eth/v2/builder/blinded_blocks, CB (correctly) refuses to downgrade to v1 + # (v2 semantics: the relay publishes the block, so a v1 payload would be + # silently dropped), returns 502, and EVERY builder block the proposer + # chose is lost. That read as "prysm can't do MEV" until the route list was + # checked -- see .agent/SWEEP-BACKLOG.md (Law 7 first dividend). + - route: GetPayloadV2 + - route: HeaderStream + - route: RegisterValidators + - route: Status + - route: ProposerPayloadDelivered + - route: BuilderBidsReceived + - route: ValidatorRegistration + shutdown_delay_ms: 12000 + + timing_game_config: + max_header_delay_ms: 400 + latest_header_delay_ms_in_slot: 1500 + default_client_latency_ms: 50 + + target_get_payload_propagation_duration_ms: 500 + + is_submission_instance: true + is_registration_instance: true + + admin_token: "test_admin_token" + + logging: + type: Console + + # CoresConfig (helix_common::config CoresConfig, 10 fields in current + # :main). The old block used `sub_workers: [0]` and omitted the per-tile + # core assignments; current :main removed sub_workers and added + # decoder/simulator/top_bid/data_gatherer/block_merging/housekeeper. The + # outer RelayConfigExt flattens RelayConfig, so a wrong/missing cores field + # surfaces as a top-level "missing field `decoder`" / "invalid type ... + # expected usize|sequence" serde error. Verified against the binary: + # `decoder` is Vec ([0]); the other five new tile fields are usize. + cores: + auctioneer: 0 + tokio: [0] + reg_workers: [0] + tcp_bid_submissions_tile: 2 + decoder: [0] + simulator: 0 + top_bid: 0 + data_gatherer: 0 + block_merging: 0 + housekeeper: 0 + + is_local_dev: false + + commit_boost_config: | + chain = { genesis_time_secs = {{ .Timestamp }}, path = "{{ .Network }}" } + + [pbs] + host = "0.0.0.0" + port = {{ .Port }} + timeout_get_header_ms = 950 + timeout_get_payload_ms = 4000 + late_in_slot_time_ms = 2000 + + + [metrics] + enabled = true + host = "0.0.0.0" + start_port = 9090 + + {{ range $index, $relay := .Relays }} + [[relays]] + id = "mev_relay_{{$index}}" + url = "{{ $relay }}" + {{- end }} + + [logs.stdout] + level = "debug" + + [logs.file] + enabled = false + +network_params: + network: kurtosis + network_id: "3151908" + deposit_contract_address: "0x00000000219ab540356cBB839Cbe05303d7705Fa" + seconds_per_slot: 12 + slot_duration_ms: 12000 + num_validator_keys_per_node: 128 + preregistered_validator_keys_mnemonic: + "giant issue aisle success illegal bike spike + question tent bar rely arctic volcano long crawl hungry vocal artwork sniff fantasy + very lucky have athlete" + prefunded_accounts: '{"0xb9e79d19f651a941757b35830232E7EFC77E1c79": {"balance": "100000ETH"}}' + diff --git a/tests/fixtures/golden-configs/cb-mux.yml b/tests/fixtures/golden-configs/cb-mux.yml new file mode 100644 index 0000000..87b4d4a --- /dev/null +++ b/tests/fixtures/golden-configs/cb-mux.yml @@ -0,0 +1,441 @@ +# cb-mux: Multiplexed relay routing per validator node. +# +# Routes all 128 validators from node-0 exclusively to the first Helix +# relay instance and all 128 validators from node-1 exclusively to the +# second Helix relay instance. This tests CB's ability to partition the +# validator set and apply per-mux timeout and relay configurations. + +participants: + - el_type: geth + cl_type: lighthouse + +additional_services: + - dora + - spamoor + - prometheus + +mev_type: custom + +mev_params: + mev_relay: + - helix + - helix + mev_sidecar: commit-boost + mev_builder: flashbots + + helix_relay_image: ghcr.io/gattaca-com/helix-relay:main + mev_relay_image: ethpandaops/mev-boost-relay:main + mev_boost_image: commit-boost/commit-boost:kurtosis + mev_builder_image: ethpandaops/reth-rbuilder:develop + mev_builder_cl_image: sigp/lighthouse:latest + + mev_builder_subsidy: 1 + + helix_relay_config: | + instance_id: "helix-kurtosis-test" + + # NOTE: no network/genesis section. Current helix-relay:main removed the old + # `network_config: !Custom {dir_path, genesis_validator_root, genesis_time}` + # field entirely; the relay now fetches the chain spec + genesis from the + # beacon node at startup (GET eth/v1/config/spec + eth/v1/beacon/genesis, see + # beacon_client.get_chain_info -> main.rs load_chain_info). A `!Custom` YAML + # tag on that now-unknown key makes serde_yaml panic with + # "untagged and internally tagged enums do not support enum input". + + postgres: + hostname: "{{ .POSTGRES_HOST_NAME }}" + port: {{ .POSTGRES_PORT }} + db_name: "{{ .POSTGRES_DB }}" + user: "{{ .POSTGRES_USER }}" + password: "{{ .POSTGRES_PASS }}" + region: 0 + region_name: "LOCAL" + + beacon_clients: + - url: "{{ .BEACON_URI }}" + + gossip_payload_on_header: false + + simulators: + - url: "{{ .BLOCKSIM_URI }}" + namespace: flashbots + is_merging_simulator: false + max_concurrent_tasks: 32 + + router_config: + enabled_routes: + - route: GetValidators + - route: SubmitBlock + - route: GetTopBid + - route: GetHeader + rate_limit: + replenish_ms: 50 + burst_size: 20 + - route: GetPayload + # The builder-spec v2 proposer route (submitBlindedBlockV2). REQUIRED for + # any CL that submits via v2 -- prysm does. Without it helix 404s + # /eth/v2/builder/blinded_blocks, CB (correctly) refuses to downgrade to v1 + # (v2 semantics: the relay publishes the block, so a v1 payload would be + # silently dropped), returns 502, and EVERY builder block the proposer + # chose is lost. That read as "prysm can't do MEV" until the route list was + # checked -- see .agent/SWEEP-BACKLOG.md (Law 7 first dividend). + - route: GetPayloadV2 + - route: HeaderStream + - route: RegisterValidators + - route: Status + - route: ProposerPayloadDelivered + - route: BuilderBidsReceived + - route: ValidatorRegistration + shutdown_delay_ms: 12000 + + timing_game_config: + max_header_delay_ms: 400 + latest_header_delay_ms_in_slot: 1500 + default_client_latency_ms: 50 + + target_get_payload_propagation_duration_ms: 500 + + is_submission_instance: true + is_registration_instance: true + + admin_token: "test_admin_token" + + logging: + type: Console + + # CoresConfig (helix_common::config CoresConfig, 10 fields in current + # :main). The old block used `sub_workers: [0]` and omitted the per-tile + # core assignments; current :main removed sub_workers and added + # decoder/simulator/top_bid/data_gatherer/block_merging/housekeeper. The + # outer RelayConfigExt flattens RelayConfig, so a wrong/missing cores field + # surfaces as a top-level "missing field `decoder`" / "invalid type ... + # expected usize|sequence" serde error. Verified against the binary: + # `decoder` is Vec ([0]); the other five new tile fields are usize. + cores: + auctioneer: 0 + tokio: [0] + reg_workers: [0] + tcp_bid_submissions_tile: 2 + decoder: [0] + simulator: 0 + top_bid: 0 + data_gatherer: 0 + block_merging: 0 + housekeeper: 0 + + is_local_dev: false + + commit_boost_config: | + chain = { genesis_time_secs = {{ .Timestamp }}, path = "{{ .Network }}" } + + [pbs] + host = "0.0.0.0" + port = {{ .Port }} + timeout_get_header_ms = 950 + timeout_get_payload_ms = 4000 + late_in_slot_time_ms = 2000 + + {{ range $index, $relay := .Relays }} + [[relays]] + id = "mev_relay_{{$index}}" + url = "{{ $relay }}" + {{- end }} + + [metrics] + enabled = true + host = "0.0.0.0" + start_port = 9090 + + [[mux]] + id = "node_0_to_helix" + validator_pubkeys = [ + "0xaaf6c1251e73fb600624937760fef218aace5b253bf068ed45398aeb29d821e4d2899343ddcbbe37cb3f6cf500dff26c", + "0x8aa5bbee21e98c7b9e7a4c8ea45aa99f89e22992fa4fc2d73869d77da4cc8a05b25b61931ff521986677dd7f7159e8e6", + "0x996323af7e545fb6363ace53f1538c7ddc3eb0d985b2479da3ee4ace10cbc393b518bf02d1a2ddb2f5bdf09b473933ea", + "0xa1584dfe1573df8ec88c7b74d76726b4821bfe84bf886dd3c0e3f74c2ea18aa62ca44c871fb1c63971fccf6937e6501f", + "0xac69ae9e6c385a368df71d11ac68f45f05e005306df3c2bf98ed3577708256bd97f8c09d3f72115444077a9bb711d8d1", + "0xa54fe5c26059ed60b4f0b66ef7b0bf167580504525f83c169507dc812816df41b1da6128341c23977300dffd32a32f41", + "0xad9222dec71ff8ee6bc0426ffe7b5e66f96738225db281dd20027a1556d089fdebd040abfbc2041d6c1a0d8fdcfce183", + "0x87231421a08ed28e7d357e2b37a26a458155c8d822d829344bd1029e5d175b5edfaa78f16f784f724a2caef124944c4f", + "0xb72cb106b7bc1ecae219e0ae1830a509ed18a042b56a2779f4033419de69ba8ae8017090caed1f5377bfa68506157360", + "0xb27ad13afc8ff30e087797b344c8382bb0a84447549f1b0274059ddd652276e7b148ba8808a10cc45746762957d4efbe", + "0xaaddb0cb69ca18f14aed7054e98a24df0ff606aeff919d489f7884fd1bd183bcb46ea54bc363146e1a88db36dc20a7a4", + "0x996d10c3026b9344532b06c70a596f972a1e779a1f6106d3da9f6ba376bbf7ec82d2f52629e5dbf3f7d03b00f6b862af", + "0x91709ee06497b9ac049325853d64947290189a8c2322e3a500d91e23ea02dc158b6db63ae558b3b7670357a151cd6071", + "0xa03c2a82374e04b2e0594c4ce14fb3f225b46f13188f0d8002a523c7dcfb939ae4856053c2c9c695374d7c3685df1ca5", + "0xab72cbc6575c3179680a58c0ecd5de46d2678ccbafc016746348ee5688edcb21b4e15bd37c70c508e3ea73103c2d566b", + "0xafa10af166a0dbf3a25ff86cd6f8e44cccc818c5e70cd70e4e98e226b158f3563450b3fb184d2649adbb11e53080d1ca", + "0xabd12678c73463ecea5867a80caf256d5c5e6ba53ff188b143a4d5be83365ad257edf39eaa1ba8753c4cdf4c632ff99e", + "0xa35c6004f387430c3797ab0157af7b824c8fe106241c7cdeb897d900c0f9e4bb945ff2a6b88cbd10e35ec48aaa554ecb", + "0x8a8bb292bcc481070d3afdbbc8789e2ab4b29c9603936e6d85f5ff71e23fc5b6d61009f0fa636b5d5b2dc309d39e3d75", + "0xae940a07850cf904b44f31cbf0e44824bae5ec36dcfdb7fad858f2a39dba38de82ca12b0ae939a34fce7a02e4b9789f8", + "0xa75ca9447dca3a3745ada36731187ddd1f6a152cf15d7446b785eab381e5c8562c1202a6e7a24080bc6b619a161113db", + "0x84a687ffdf21a0ad754d0164d1e2c03035613ab76359e7f5cf51ea4a425a6ee026725ec0a0dbd336f7dab759596f0bf8", + "0x96947de9e6068c22a7716656a2755a9551b0b66c2d1a741bf84a088fe1e840e992dc39861bf8ba3e8d5b6d21e8f57e64", + "0xb570dde8ee80512e3d031caf22e775c60f7f5a6cbdeb3e52e24cf8c867d38569a53dd19cdc36a03a1bbb3a8d94b03670", + "0xae5302796cfeca685eaf37ffd5baeb32121f2f07415bee26cc0051ee513ff3932d2c365e3d9f87b0949a5980445cb64c", + "0x8de5a6200cebb09b2198e69fed84bcd512ec5cf317c5f1ee99aad03d2a9a8564bf3807c08da2664222268d59c34a06e4", + "0xa1d9840eda3036fbf63eeea40146e4548553e6e1b2a653ab349b376f31b367c40d71fb59ff8e94b91daa99c262ec8b52", + "0x8419cf00f2783c430dc861a710984d0429d3b3a7f6db849b4f5c05e0d87339704c5c7f5eede6adfc8776d666587b5932", + "0x8d46e9aa0c1986056e407efc7013b7f271027d3c98ce96667faa98074ab0588a61681faf78644c11819a459a95689dab", + "0x930743bfc7e18d3bd7351eaa74f477505268c1e4e1fd1ca3ccccdefb2595517343bbb8f5589c435c3c39323a4c0080f8", + "0x81ea9f74ef7d935b807474e38954ae3934856219a23e074954b2e860c5a3c400f9aedb42cd27cb4ceb697ca36d1e58cb", + "0xa804e4fa8d1391a9d078aa93985a12503b84ce4f6f1f9e70ab7fca421e1cf972538666299d4c1bfc39327b469b2db7a8", + "0xab40dc1cfe273ad0da700c64f8fc94f91db253ca3acf20e336d9bd09de67eec5c7d3506285d83c7bb6a08d64b77e5f2d", + "0x8dfa86c051edd28c3554a30e40531c898e5936ad3002711616ddd1b27054bc39caedd505a200c3d23a1c3f6b26c50ae9", + "0x81fa222737fe818b43f55f209f42adaee135b2801d02709617fc88c2871852358260ace97cf323e761b5cc18bc7325b3", + "0xa4ee6d37dc259cbb5237e4265429a9fd8ab5643af81628cc101e0d8b4a333ef2618a37df89ea3f92b5ea4333d8cda393", + "0xa759f6bcca8f35fcaadc406cc4b828c016c0ed23882987a79f52f2933b5cedefe24e31df6fd0d38e8a802dbafd750d01", + "0x8d028a021c5c31a1aa1e18eda74cfaf0fba1c454c17c2e0fc730dd07a19d0c77f7a905d54017292f3e800ca06b6977cd", + "0xa2e2d8384fc87a512ee34eb43405fd82572c9d7cd96e155a382cda284e8df9eb7189c25b7473d89c63ea4e6080e10ff8", + "0x81b676591b823270a3284ace7d81cbce2d6cdce55bb0e053874d7e3a08f729453009d3e662ec3130379f43c0f3210b6d", + "0xb5e898a1fc06d51c695712928f44646d15451340d1b3e480a40f03250160bc07d3b6691ec94361dd524d59d9df7f76d3", + "0x84dc37ca3cd621d3da0fbdd11ca84021e0cd81a73d772dd6fcf19775b72eb64af4e573213378ccee0915dde92ac83ba6", + "0x84d08d58c31bcd3cddf93e13d6f50203897384afa34644bff1135efe8e01c81c6a91ca6c234bb1e51ca32e41b828aaf9", + "0xb2225575d5e70da1257db7a0d1222c5041b52aac61cf161e8fc8126a3fdf5eb4f0867d98dfe272199c36cf8f02661b3d", + "0xa8fa3584a92b079c8c73ed1553e5e161a0b21325fc2fc4e24a892354a899c7fc0bfb436a97a7ed1fc71bccda438ea715", + "0x9918433b8f0bc5e126da3fdef8d7b71456492dae6d2d07f2e10c7a7f852046f84ed0ce6d3bfec42200670db27dcf3037", + "0xb24391aa97bfff29adc935d06a2b6d583433caf82f92de1980e0192d3b270323bdbf24b86dc61520a40c419dde3df4b3", + "0xa62c0205fb22df8535c0b70076486e69dfa908feddae79e4a94a9d47b97ed190d228e1c6217e84a59882bb992dacae30", + "0xb63f327df68581cdc02a66c1c65e906a06a1a3a8d7a6e38f7b6da944e8e6cc2db85fced5327d8c12945ceb33018272ca", + "0x8aec5129a518010912215e1887191da94be419b4e75904c2ea745e2d253d707c088fa5b2c46dade1d162affe9f7ab17b", + "0x8725b32751419f22a54485790f8187d1ba52d84a31ad45738a93777fcd1ccbec1652229923f82f37793ce0fc2763fb4c", + "0xab64f900c770e2b99de6b86b4390bbd1579bd48dccec55800adbcf52e006f22128e9971bbf3a92cc0105b0974849935a", + "0xa0485d71f1f5e177f7d5bc9d98c5248a6a2d0de4554c2eaf02abae48f5a3e273b2ee7765784cf2a4cb7df84f617177c9", + "0xb09cb155daf2022afd18114a352e506a84065c80573cb0c7c310cbe92e2706cdcf91f74bbd9e464f74e3d831386d5033", + "0x99d83a0ba33161d8c6bbe80929fd9046d4dfdac43477ff85fea5bae925e6c179ad28eb338375ee2417acbd6576ee670a", + "0x958c2692b86b4d20eaea3bb45e9447ebbc5b93ccaf8d21ef659d0cefedf5c4371b31b460ae40e8243682bde505abac1e", + "0x8d8985e5dd341c9035b37bf7391c5944c28131b47c7d5359d18fca598010ba9a63e27c55e6b421a807038c320564db17", + "0xaf89ab00a0eab1131645292a9cfba583a69a1e3ac58b210e262494853e67385aeb50d4af428bdd577b9399daa96d8b20", + "0x896a51e0b0de0f29029af38b796db1f1e6d0f9f9085ade40a313a60cb723fa3d58f6587175570086c4fbf0fe5331f1c8", + "0x9763dde1b8028136a3ffd6dafd1f450e2cafb2819c7fa901f7c6e9cde8f2897ee7e9a45da6947fde1ad0d3836188eab5", + "0x8fda66b8607af873f4c2c8218dd3ffc7940d411047eb199b5cd010156af4845d21dd2e65b0e44cfffb5e78271e9bb29d", + "0x86e014747c7922ccfc2b9d4bf6c1ecf0dc800197037858d0b85ab1944b4c3c14b95e0ed325bc42a6f467bc47ec27bc7b", + "0x8c0d15baa72bfcd317e9b9402ca9bb6e7ae1db35ffce7faccae0bd19b3c8e5de7d5524aef0377770b3a90626627a9304", + "0xaf61f263addfb41c46d66e60ecfb598a5942f648f58718b6b4e4c92019fdb12328efbff98703134bcf28e9c1fab4bb60", + "0x8de7ec501d574152f52a962bf588573df2fc3563fd0c6077651208ed20f24f3d8572425706b343117b48bdca56808416", + "0xb97ecbcfe8c52b9bcdca9e75da13c5650b751b037c570934ea6b6441ff32de6566c50dafc0557e63105b2ee7e8cbb39e", + "0xac30aacd9e91cb0727c34ca6b40fbfd4d255b998471e25c443cf6cf777d6bb823a58e162958f32c3c5ca80453387a5d2", + "0xb43ccb05317c2b666470ab251e987d6bf31f5ead6b5edac5fe007dd334ae6ce1a92e24c19e5ab387cd8fa253b63bb78c", + "0x966c488d807b3208bb1b10a1af422bac8d363c8015cda4e24d214549ced019cd3dd575545dd887461cae3f70d95cb061", + "0xb3faeebfbebd085b9123ae0e09af9cd15d3b1db6a25f3e82d8b48b68e53522b41b342a3a3c8b008897df356048862d98", + "0xb8c6663371dc37bf083134dea26a20115ccc52b7c15a662bcfa33435e4ade14c6bc9714a5cdee492530accf8a327b2aa", + "0xa39731d5cb52838d02d4ff897ab908c0f76a9ef837f9288c634ed3091a1f69d5347dc65cd2c8009a5207a369a4c6bdae", + "0xb97dbe4add8aefd96c575ae9de19d1ac590bb7d92f23a9e4e113f7271c2243cf689e7645879efbb546d58ec44f5263d6", + "0x95833097520df43a5cb013e97f80041a7a0b7d84a4ec79e2f16baeeb6edfbcf62ede97becfde73883831bb65e1415dc0", + "0xaee3fbc60f939c125877a4f4529517edcf114fdb83715f9f4041eccebb91323ce8c4784ff87815c38752517b3d2e2725", + "0x92b68717b3b88b77716884d492966713d902eb35196cf71faf1fb5625327ddbdebae94786c77dde207b5666ffa6dff98", + "0x91da49dbfb1a4a339ee8e1b902bf18302c3ca948da1dacfecfe9934d231013544dbd97689c2996f22b456f7e408a138c", + "0x93d491d7211af181ccab8353567ad10c065a6e991c9b70def7215c62fa3bf843a177e18b00580deead4b5678f46d4d39", + "0x8890da2859ccb2afd0742c1c791075104d7acc207b6fa478bb1f94fd6665ac75e5dff4d2cac0f81e4168448bc3a4c90d", + "0xb320e188ef282109ba8dc3d2573f9edc33831c1025a29844c86e7fc60a25627e507964725ec8dbeccfcbaa12e7fb5e1a", + "0xa90dab71ad924fec01a579727736fd9a19a147bf57ef471255527b55a4702cfd54bb0300623c506823e723e7c30cf4dd", + "0x95d7b5b39931578c673a6ffdcd5b3618ddf08b40909fd0ee04a74c70aae4e88c55a43d5b71cdf40c514195a7805019b3", + "0x978eb7a3aee4238207d1e78684a6769e3d71f1a3b9e42ac53c347c97e15eec3ac0082abf65ad9cfd126c687c51436cc1", + "0x98213294b82bc66ee39e95a678472fb41df846ec2863c5be53e1fd56b6ff0fe1bfd5b2bd8c534dd97acbe597ad119cc7", + "0xabd3280a86fbb8d736717d8bd950920873ff7bb8a68ac4d7c339bb3c783c6f4e6e29912f6a3528e04b13aef57d4290a1", + "0xb0b263c298eb4d09de14dd71005fc683c4405efdb230c6661b9697d3a0978e5ed7736e76729b2233341e75ea46ecff1f", + "0x85fc722fa6d2c54b9610307ef86f0752d1428bdec6dc7e46ae14318bd9203f32df1ca8d0a420c36973067a50930b7720", + "0x958b57a4b16322f680eb4eebf37c538f1b5bf96400e51f99df6a4a439b75b9acd8866caf4a091d02f25510d3bba1aa26", + "0x889dbdf3bd68af1f6fd84cb6173b1fa1f7c5e6ba63297dc1e2f45cd1a82bb6231ba832adc5228143c5cff3ef0b1caae2", + "0x99377a407f49949e88651b57160044764930ddd9fc404f4b610a581c5edf9e906f3edaad611364d240336a3eaf4dab82", + "0xb6cd22959866607d91e13122bd34b050a7da426dad5e9779bcbb99e4a2e7bd5d18ce39f266bc61ef7524a2e2adfaf765", + "0x8548d74ab33e8dd285e72b1ea3c15eb08b66d555493bcf39aa4355af06ccf0f4469e09f110544dc04b1f3d0e7880ac1d", + "0x8457b3220923283912ea67a58687eab0fd1747497c94a8f73c4f75147fcc548acbe662e7d2542f5fc3c94749f2f6bd6f", + "0x961a7d85c4e61428f07c4c4d41cc03115ec423bf40172b21f9db161edec282c950782ade73a6dfa915da1c39c716eb1f", + "0xb26fae53e4f7d2088de3fd67ce06fde88a84a7963af20ff8901613bd8be30e9660b244e5e0bbcc39c940c814f6f3f318", + "0x986a201d308cce68d381d103680725fbce0fd9f618de12d371fd5c909049ba3081de693de9e0560364c201505b2fabaa", + "0xb165f7ea9728a1e55dcebc59c89321586290bb3c6aad2114217e81990104863b779ffac9728ed736b6069a36a26aad47", + "0x8e8d4d32b9c20bfa05ae904524dd1fcffe5ef1cfa451d6f0289eb54844a6ba033da0c656c1b69a508a45bb572797e195", + "0xa9fdcd176de75a9a1ec07d553a267413ec5406ea5f46a65d924519da3fa4b3009e2bb350fed52a51bd8683fca5d6184a", + "0x889fd954b35f31ea3f3eca4496ba13b9678a527e4d3137c8aea484b4d741886bdfff59f99a89721165f5ba7c9f5a366b", + "0xb4147399957d387dd0ad99bc9fbd7ddd3ad85809c9658d121879099c602db62c4a4ce4d122915b6d8135bbd16c13ffbc", + "0xb9882d1217732ebc2e6e2eaf42a9fc606ccd61e6a28a01f77c0d6829e255b51e50cebb38cc86254b37bcab1fe817e7b5", + "0xb8f3af1b7f9ea13cb73348da2fa847c6e0b89bb415dae2b62ea29e294060ea5c73c51bf00c50d8b76af2df9b35ca8c45", + "0xac97a570a795d24af13ef32709d71a37c0fb90a49581e735d635721581c5553c10eb15f43531121b6026f55d603d73f7", + "0xb0d4372ed0f55fa767a5fefc734c155a33821ffb6e6be4f628955a6477cf1d4f12c1d0e82426a8ca14b9e92ef094472f", + "0xa11c83b69a43111201fe36f54212afe6f12cd3a8ad551b586061901775a8205b816cc8638956c57a339228d61b520aa7", + "0x867e89563df1501ac7dc5a369e6713cebab2aa1b676ea6d97fcb62802488866ae1223b4ed6c00718ee895d7e8e650cac", + "0xb4cb3c19c6974ff1ff486b97cfc43eccba5e61e288ff008349cc99d1adb317ad2e9e28adafc48057322998eef132ab26", + "0xb1f2588848bded71a0ad83d124b5ff2d1ab964068e4eab97edc678eddc9aaa6175b51f561c4ec96c818ad5f6c59aa936", + "0xb4c9e24cb284d23c40bccfe377f9963dcdc1d22d5daf0766a6291354eb4ff3b4d1c21913dadae80fad218bc02778f5f2", + "0xac2d58a25bea23bb5160dc19a9e3a936dcf276e81079326e2925704ac9ba43907d653b841ebbdbbfbabd0f1ea1fac717", + "0xaa080afda88d384d10c98431dfec91c4072d6c9a4b43f302e5918d6292fdca1969f37a26cb05af494b6e46b3d61eb053", + "0xa7ed4d8afeea6c020adddf320ed86a863ec5d048236c28e56f61f83b4603cb9e4f2f2bf3f6dc10864936c98c0784c038", + "0x899f89c23e08b2e89a5416a7c6c9e76c7e1a064c49cb2f0825699d4412784bd4e34799c06b3faf0539b6fb4ab10c104e", + "0xa878b6b608d51a556d4d599810a70dac104d7971a8d99ac72341ff2685fa3a75561ca1fc4e5ffb4b73b5dcfce372350c", + "0xa8d747a2b2602fe32095a7dda37a86d94f8a59cc771902be8c45de5bfd9d58fc1bf46fb0cf867054cd5eacf4f331dfd1", + "0x851cb7093041931bbda885ef6d5a0411353e41b2baf7713ff7688b70d505dc97561a6b6482571aca964486d8a9b76a47", + "0x8d9dc5d04f5b105ddd3e84529e16f33748969fea81729ea7dc087fee8f1e3e2faeaaee3ca7ee5dc264a22cbe0ff809b3", + "0xb8f404f4d6965ff42eb6b325a85570ad85c4e8bfa9953d10eda05dc5db0070b8b43242eef8c2fbbe3a549149419a6428", + "0xa8494e9a4ca0b6fb595347fda04978eb76c0520b07b760ebce609c5051d6a7e2b01dc68f2b2eab6aca5a8fdbbdcd9346", + "0xa3baebcb3b1f6364d27e5bb8cab4cdee8b8388f29bf5ea235251742563e171a6b7e02a854ed24691725c58b5a7a88987", + "0xb3a1fbfd06f4d68415d699ced0f6d3f10ab370214fe66e2e4843bde2821bb4b878de8b7c9b2e38b45de941d06a665117", + "0xaa8ec705d7394909c67619fe6ba077508290a2756f0bd39b46f53950da8965c8a2fe8d1f2ce19bca22a8aeb9efbdbff6", + "0xb967afc934f2efd001a6901490e770602f80df9ec0eb3293490ed8c55e14d08f5bebb7bd183f485de29a8423d859a501", + "0xae2dd5d89fc56a0368ebba3891020b24ecf7acbdced259277d10eddf8d812568542ee3df43fab157034cc3620f17b75b", + "0xaffdd643395f3ca4138646f729aa0f5d1bfdd085df8d2559e075f72cbcaa24d9310074491910572390fbca5478d7d369", + "0x81093820fe0770a18a816945494db8fd957f10f7693da18f782e0968ef28ea8f23afefb0dc203925262352925f2739df", + "0xb9e03b94bb696b0e4c7939bce96d9e4fb1938074233d87b290b20cf66d3e48a7f3d852d89969f45c075e3dca91945832" + ] + timeout_get_header_ms = 900 + [[mux.relays]] + id = "mux_helix" + url = "{{ index .Relays 0 }}" + + [[mux]] + id = "node_1_to_helix" + validator_pubkeys = [ + "0xb05cafec5912f22dbd6f15677f25f13d93ecd5ec6f957fddd7cf27d73521b34aaaf6a219f77b21128d18321c2c8d679b", + "0x870342ee85d1d3eda564de4126f20880d59164e2f88652d9dcf3dc93d0bf19e22ca3a11305f1cba1cadaf2d117028936", + "0xb0db46ced0115b365df2d2c1e29ef3333b0bd4ed297288f7a09ca9c1de5e702ef8f2cbeb89d0d70a584cfe991cf7bb65", + "0xa567c07d1f258a7dc4f685b9c45c3217e9e640d8cbb3fde3a875e31b0212df6d48985f8524922205aaf6917a5b577d89", + "0xb7f216edfaf073f84d71ed41b7376d8fe85c88b40297636a2870278d00b452fd37b41af5e357a2dbe1297b53fb027e9a", + "0x9188ee0eb50d3a88a27a2ce4334ea8cac1662593526f4c280cb3049bc91afd8381ddbda124b9fab871aacf378ab5380a", + "0x8bf6583d6de04a89b9ca61c69977de4ca440c4c9b13c7a1c65e205979849376e239d03cb46d8bcdf58afca96a8e0fcfa", + "0xa9f291de2f415ab3a4002206769493d82a094e1934b78d98b79c93f70ffff7389d8ea0962b34b6df04ba999442a0fda5", + "0xb7ec2f481129da715b78d3c6bc1ab2e04bedadb937812497196409c08d7837d133c8bf52aec70689b1b180d8eef2676c", + "0xb0862b7f0788739e315de558384d6f95531088016f18330ab572fe89f3600098284f7a08a1ea8e7e33666904b918b17c", + "0x947e056beba42f6d7fec6712de16458c85ae513391dc4d42120e5877ed21a0681820a6150cb45af4f0cbf98c1d25c8cc", + "0xa03f7aa96fd1dcb385cf9f4f29f9f3ef4c25a47efdb090c18763a51608e92267daa98695f765705a7630269561f9086c", + "0xa62b23a8a25355c20cf4ebf93bc43a7b4076ab247ac7bff133ddaa7cfb9588e598d1022285b0e2f2fdd0ee3fd51a6f39", + "0x84c1b3ec3752b9d7e3ddfe87c68c59443b5fca9d18d78e5441c80e4fb6c0ecec27f8074811c2e5f823364a71e0567394", + "0xb7c37e964efdbe3916e611daa4a1241bad3aeedf6a57a21c87c23bec872452c05816ee9179d021c3cf36843063c687d0", + "0xae2bb170260de9afb23f49d5770474a65b4d380d904aa41cab7c2852074ed8ad9fb94f6d7e5a5a1f6f71c0dce0bddd12", + "0x8faadcfded5c85beb36f1cdc234f0cfd8ebb0820a18899445accf3e6e35efc0ee34419eeb0cfb097dff5fd6462b9ed90", + "0x837d8e7320247799d20afbddb410fda1bb7fcd31e36ccc841012e4dac0d643e8e5e12d2467d8d3219aab33b64280eaea", + "0x859155dd5a22f116ae8f61b1516770f8ff41ec0ea24b8b745171b4cf34981bb7d235e7e1a739a0589e7c7ff69ede9b15", + "0xb869ba1794050a014193ad467452efa3a54bfe6c6d1689bf7de9576eac2d2c2dfd4383a219e2d450b00cd9a70fc5e2a7", + "0xb06cb2010c1167c72840b3149ba92b326799375b1b05a1c0ca38ace5e8f61cba48c1b350a7220938e9c2c0fe6b6c2881", + "0xadcb081ad4dd8f1acfdf1a71360c6d5655bdd58d9bb1f09e4de43b7ff8a6da60b61df9ce65f9ffc951740dfc69812667", + "0x8d1df9e9132058de96486f102ff4ef6e34c988b6dd42a9462954218b8728310bdc25a4251c092eec8128bfdde893049e", + "0x8257c261afa77e79086b503de88dad720443a1d135cbc14f8a6de408a03ac5b9c4263731d6693bf843f0a9657aa3c4e8", + "0x8afc5f7128e998c2a59b9e2dd1805ccda56220d1bb25ea94c13d4f9abb4ab55a522518aabface4f10d54f45d08dffa8d", + "0xadc1c39301fa1fe99678a7b7887e895c8df24e15546b13d2237ab2795cc7004e6b68e69724f2c0922f119d5af8819bc9", + "0x94ef22c9183e15da2e4ad8e05a75c5b9201d52e9ad7f66cb0061b0c68779ba5a1fd0f11b9c365c1721d00199d287923c", + "0x8c3f73416c86d93ae2dbc2a468b5d6cc39d42c2ea7e20cc215212d3dd3b8efbed324bcd4a28de941fe855c54a92c6973", + "0x914dc0bb6af111bc9021887ca4dfb27cc2063c9ebd2be133754d30ca18fa8698787005538f556bd94867cc1c5f7b817a", + "0x95e8e9e5c389be338759c40d2e408b1f0b78ebc0ffbafae360e33b683ada3638414338b83490e9ab1ff067425d25d785", + "0x88ce4d8fee80abef17438d6e1dee0da0087b3fde540c7cd157e169866c603af0af26fe3c3d2ba517dead10376d60df89", + "0xb956f93f164469f3bb6b2e95c7d3db7d979b03457be68c6b227a9bda9b4be639198e89c9a62452162640cf95a00ad339", + "0x902b0898a017b3e98d334cdf49d5411e507f6043fca0624e937945c7dfb1829a3a2e1d0bdba654aef7d2e3c14c76b48d", + "0xb8a434ac1fa9c6a2c2c37da065576261d981d870923222a50b225422097a3e59598f548be8c6bc0b4764c02546f11b7c", + "0x98eeb5417011d88a2924cbe7ff7ed616a6b7dfe273187360757c667f27349f525ac60665f9f7ba2d07d91d2f94566f1e", + "0xa83c93593c32b08e89f0089e1a0892dcf121a856199b751ec40959433b4e64ee7fea260a16f4929261c5e2ebc148042c", + "0x995c28e8767e8677ef93cf6aa49453a8ad6c279622820e321ff9d352ceafffaa7ec082713e4c85e50cf9f7e11439cc16", + "0xb83d8622fe3180d6e8fd95b59d5e5bc6b7c0451040ec3b14588b1405410bbb249c15d449dd9afa4d9b42650b26a58fcc", + "0xa757657d95e795d3460d5454b0f3987885f0a8138c4de92e08d2843709f808adb191c9d2b22399bb9445dbb94e190382", + "0x81f9ac9f60825d682d5ab33098117dbdcf3c5245116c8c03a8c0493a5d441ba578a0b3d069d745cfdff70122c65e421a", + "0x8d19ef6c96e7ea917640ea3ad0c6c6a9ac8320456c3ec046cebaa625d415476f71d854d186a1758c79843925b2d268ac", + "0xa1c646e753dd9d9811cd75f58c371e74ce83f05606bc076d64bdc77adb313b1bfd8ceceae38d8a8b7d4795f10ac68d18", + "0x8cb3b628e5ec89b1cae8fffb946a1277613deec1f3f0e7d75708fe3f6b15c5efb75a0ec8dce885812cf14f3d589b6f91", + "0x933d88e34601df31b7a66bae1d49079548a2ae2d72037c7c3b2fb8925631989c1ae353311c36fe325b2f1fdc1f648194", + "0x8ff3e5eb9c905e42b88cd8fd8593bde4d44e22cbf296d47917a4b0f144fbbe2ab69a4421b2c0ff3500141dcf88a0b007", + "0x9592c95f5c1574b8c510e546759f383779a0364fefa009f84dfcebb2efc4e86b909a5fffb90ed79b5f66980b420db7e4", + "0x9218096756d3ccec228caddd27979b9050b58d7304f2be4f7d4aafb6ae19fdab0bb9149ce29517a383f59facaacf6d39", + "0x8de14b70a834e78b74d6bd46bd0f5f92d878a1b19f9081808dae86391aeab2a048368f0ae5b93ab412561db70482dba0", + "0xa0833e9ca3181e33a9b51d7c31722cd07c8f0a18a34ba083eaeb091529c2a454c3950ec449ab7f6f237995bf1ec0a802", + "0x98249520dcdc8be36b8afed019336e2eb478d44a0f5ebb6d7c994710856abf32ed25313867e060f83b205ea5ebe9f0fc", + "0x8f14d19885ec7d1bf7b6c4668d505283d68d1af78588532a15aa80ecf96a2d8bbeb813f63c25c15cd8f04c10fca4c49f", + "0xa6094298efdad03170daefba1bf92ee8691aa86fb753afcc252090c74c5f97c7d6cdfbbe0017f57e25d406d4ccebdb0e", + "0xa94d36839c1557727aa55fcca24faaab0d1e5f0ecff1dd709e22f0c55408b8b74d22173bc312d27baaa8688ae1b4be2c", + "0xaa228e16f801dfbde52d5f57b44e7e9a613fef33f893fad979ce003eeed012c8f45e9050db814e241be87ae94c0c4011", + "0xa8f3c2f7371294ca58c85e7b1513c155a39e5bd76eedd90f50a4fb50d6ae55b40623d5ba0c5d85c40def89d6c67d5fc3", + "0x8fe444e4b5610d3583a667dcf23af26a9f686db05e7de9ebc03f8ea0a756cc96b4f55611f5788d61f26e5bda1df70f68", + "0xadfdf512e8ac8e3f01ab9a03733c6d0bd0a5403778eebf691d2906380bc01591e93b43cae86c94f657c92c29d5698a06", + "0xaaafd2bc633a130d0798c8f398ebd00f5c1b131c2b4d48cbaddde1c8ed59eae6af8290d27700852df2b3cf23d630d807", + "0x863fb35bdce0573031210c5a6d3521b5a3e11cc99356de9e8adccf3accb4c4d387f954b202d0db647e65c331b5019226", + "0xb1748deeb17775232e5b53f55bc2d1b08c494a80cb727a2f2361af61464afa85a1afcae978e5d802e6c75e3be60965da", + "0x999664783bb5eb59491a99142392f020fb4e3b607203c3f9bec7ccb3537e8f7856d7ddfbede9dc3332cf40bdd4334a64", + "0x84e8581cf13f7df6a96835ad1330593bf5d1f4e5bd6341f7a54063b0233e921250d0c09a48849155953e293cf635d7b6", + "0xa608c06384bc606f723cde2a6f7d64a29de9cd987c0626dabea2414bd5b653647b31953c1156803191484d5ecf4630f1", + "0x81ef1d058664e94bcdd4876d049fc40e9a2d55a104a2fbcd33b63774a55bf994d72e6c7dc641af29bfa57362413db705", + "0x944a35116eaa393571e02f6751f1a2820ae5c4075e9bcb9746143a320c5a6ef5c3f4f939629dd0a2753b27bf5c5317a0", + "0xa1c888d5d2c76f2b388d6a42b42005adeca9af4d454c6342b9b634bcc6944432398823d240b9bd1bc7f37f58b390405d", + "0xa2869de5721730e34b8b074d27e6e4c2c6d09f6bffdf892edc6b3590830362f496e9c41c601a3e38eb80dcaf6f0c65c3", + "0xa069b0ecd8be3a45dc395f87b4ea9ac575f2f39dd922a1201ff0a3b9a5ec551f64a325647e6fa58770d94af853d6d6e5", + "0x8eece2624f5aeefc66c93625462a310c55f8df3c0632543d590db8799df2642ae39a0e77a9842e5d1982701311d46e42", + "0x8b1748a729412116660036812e9196fce6b9c553b47ffd942dc40c60a018bd49ae026e2bf4a1999773521b31bc12e64a", + "0x88cee7a3c398f11dc6eaa039542e9467bc7cfcab47718d92533d7adf7e36021e7cbee54342320837aca410b4e1dd6a33", + "0xade9004ebdbf580c9c29ab6d34b548f489609c98961b95b9f0b1c6bbad66ea4941143398510bf4f1ae0a7c72555cb9e4", + "0x93641e8f9e440bd769fa86db3f72ecf076da7d2cdb01355415c7c54fc90075899e40fe005898d63d706f3f94bfe9373d", + "0xa85336ac11331940f64d8bcded1f4a3dde75808d847311b2ff79bad985af2e2665fa5bf58d67b9eedf001d5663c08030", + "0xa7372f1be3e7bf1aa20000103a787929d2b884c2081392571cfc7f5034a91b79127706c6a2acde2c0277009fbd9943d1", + "0x953ba2a21db4a6b4063eccf6b599003803c9b03d24009ca78cd139a4d2cf26c602c34fb186e106dd0649985a3e5718a3", + "0x8d1c890c3036ed8913e69c6fa9f37816425ff5c06d1821db0d32e338c5d9b8db47e9146dd1327fe2c8358911b4b2fa26", + "0xb2ad1574eaca33f1555308e24b27a095d24aed8f4af5302ea2c6ba2e50936d25ffea7047be94065eac630693c7f86757", + "0x9806dff63a4ea679c0283a72d42a38d10d7949e6e7b5964ec1eced6e7ff1add4f32d7ff92d4097e7b0773633cded5051", + "0xa826dbfe4cccb4198a2689e1b32e59be8b774af0abb56c720ee2fc7472552f3ddc7a1ad15ddb2c2bf4189ecd5ada3761", + "0x8eeeef43e0617ee768138e98103fa9aaadd9491966a3904a24b51e5c955f3ed007a5ae64de4b541533a19246b3357e3a", + "0xa4bacbd1f1195b0ab5253b7437e05d235d663f9e8109dded69ef45a71a5366f337c2787679717ea06f8321cd0982c22d", + "0x8188a5547ef304357dc50d0e7ff3505202b18ed0dacb2f856e0450bdab0be01c509d4659ec44cf518cc5fcade8b3111d", + "0x9320225d3ed15c0fd4c0d9d629a6c8bff13af7eead740f4941aa850918649f00e42e4fcba371ffcdb2864ada1c654c6e", + "0xb44a24c31c8330ab530f2d4066256d26a12fbd3163ebbfa8d3cfe091a3926d16b17a5d2f3b4f87ee5cbe91c2fdfc5f05", + "0xb3abb58d18a587ff17cdbd85be687622eb2add07e80ca5190745badb03a216f8969b59405c85bd5a188008bcdf5a58c5", + "0xa5c521377c52dfe16b14e171bf3cddfceb7a6dbd768a7ce9716447b342466ab23e5c0e77854649735c4e41dd5f981e74", + "0x8ca49f0c11f05aaf0d7f7277fce738c55cf907d9b3a03a7eb61ed31b4c6808397506624310000a6dba2441e96c7b9ce0", + "0x84b2c527a35b380c1d407f89a70f28e5241ab6ebc902f13e3f8dabde85c1c2a4fc5bcb9b16474ff0cbda39dbabcef906", + "0x92a6792fe1ffb91a4ae57c6f8b82fa12d9f5f8485d21232304b7a8a971c2f5c0759839b90833cf561b4be224ccc23018", + "0xa45e18dc474050ec2d4a16975e6d59a9714f7a06458ce995e7b4882e21dbd2f4f1094733f69da85a6a636254b2d228bb", + "0x879336a84d4871909368d36a4896036e4482ea11204aeb57ee703f4bd323307f4521f038f766a1776a0b648d795901cd", + "0xb8716a085bf7915e9fedb32b67c1daf91fa32638a06e2edf1b38966566e8ca10ca06deaa0faec734e3e03899c8c919fc", + "0xa11525182b424d0c561b7e08882e05717cb4c93eb3002927efd7e077f37f20bcfc6f2fbd3f0ca9533d36f499564b279e", + "0xa067c96285c9e48306065cc1b0217e2de81be782fc101b7cb0a1cc1fd25f10e7bd428df87593a04babc7cdb0ceff31aa", + "0xa01ba34e831653404dc9cddc23155e6085f631c713681fa56412ff7c6ed8f5702d384649f42791f6b36843522ca0b8de", + "0xa49bb35ada704f33f70d2d77a72498b3e2fa01fc9b7f8f02b2a6397a825b043b3eaf00e440c48ae522e42119095c88bd", + "0xabe99cd6c855608a1d0d34e724a6f4dea47cb8148544809169d7d323ad47efd55402adffe2d20f346fe17e3f60402e37", + "0xab22c3c322bc0e0594926d5ac48e2fad0eeaa24d7dc616fc9bb0af6652188242d1dc59072f35a1552b3a0bae0ad6b170", + "0x881b0b35d01247b4d7e5a1a3d2b071c40904d0c1892eed4f8b7f1c531b18feba1e5b81acf8c69cd6b6ad0cb31f658333", + "0xb586ce5eee6b78c9d65284be1496d8b6d32080ff573361ad2d69e26859772e8c2e0263b0f173489adf9c7b7cd98bf95f", + "0xb602e252aea22de9c5d5733b8ddcd2b4db8aaf24fe57f73b5584bbc9acb5f89af75717290ec7f7ea2a9fbc1ee7dd82d7", + "0x8a48989a82f473050e42cd9cb58d8538d285e069d775b034df1d3a703fd5821d8ae6e2bc5cc5ee3bc770cb28ba59f6bd", + "0x87c10f9555a9040368909a1c4670de4c2f725b4a485a77716e5e060e65c47c0edb926f28a422d2ec24868bdaf771dbd3", + "0xa38f647bb270cfd59a8567c5afe5e02463eafd2fc9d0379494c0b83b235594cf3283d5cf67ae6ddcca00f78bce7d78d2", + "0x97844029484238a8951e707cd4f46207722e0f9fd15b050fc2798362d87293af7021398188e1fc033423d75a604fd830", + "0xaf49927c935c201799ab1f4961786ce1bc01266bcd6d2adb81b193c3362b73f2dac32db526b597ec5baec292dea920d5", + "0xb79d0e7e5004f1679bd96e89fd94c5cc2ee5c368d9207b4c778b6983f288e5f4cee3bf84c7ff4a16935140da31615b9f", + "0xb7e03f23fac574fc7be4a455aad6807d0c71bf06fd74cd4a7d72cbf1c1db1be5a01f50dbd51457264a16d8874a6a1648", + "0xac1ada1e9db792ed12d314638c5b16e7820eeb0d178aeeeed1e986b6919eb886c7fda24e058768ec0c63e96c1b2c632f", + "0x8ff47ed158f7547c94a40492dd7996ebf2a5223f2d1313219b0e56ef41ad3a7676a59ed91f1870d7c77e0b398d1848fd", + "0x94ef5705ed74f1618a6918d903de2b950df9fa5273f27ce89481155c1f5b084370d59c9ac2df9ab2bacd6a9ff9fd1138", + "0xa25eef6fa1f3b3bd7abb301c28230897aaf57fad66ce0871c54241948328bf40092f343b5f092d89ffa79b462a0ad0f1", + "0x89108f74cef974fd3a3d6263831f54007aab5d272ae9c0e7193575406eb9556445c1bab5f1338a7fefe97d030ce2c0ad", + "0xb92e5259c50d22699af4fd397cdd32177154a01f26c3d2e687489af5a94e86aa7cf342213d0827e4c90801a8cfafaad3", + "0x815e815cf1a66a872ea0f171341d096afd5b026a6f6ef60d176f1598a6dca647379ddb82d00df0b28c0479d291a90b9a", + "0x9491aca6c6afd9319152796ab99fe3eedffcfcb959e7d7da0ba68e04754534d8576c394d80f407d84cfd3f25d90df4a0", + "0x84189eff6ff8fe060064c0bb0c9c50e8680ca4198c14878807ac781f62376662c13d6c8cf4c588428fba90541d12b35e", + "0x9995cd3fe60b1757fb734715aea9941479756e8e1a8912cfd4ab20e1f9e3c2496740a4bfe66c2633876606d5136f73ae", + "0xb9b76b28c192907820d1ea5e0fab9c545e8859cadf0a25e17bbeebeaaa8a01f36b0971124e844ef976fa076689ed03a8", + "0x876df2629991ad5014a9552fe4bef866ad93db67c291fed081254316441b0dc974d11b02d318c2342e2379293d88036e", + "0x84cc6208ab0086c1a439a4ae3787a46a133847a5daeb9f7476814490be581f25c5bb515d43ad6092d4a8a01083917a6f", + "0xad8f20f15d45a0293e58e0188f19cc59e649daabc63e03e66da3ef58306ecfe8f2501d6e0ea3e3017734585ec398507d", + "0xb0d6f1ceefb752039bd31dfd2c26dc4f96a5a0d8f0772e08737438aa6e324f557d09c643891dfc5358b23b25ba3fd310", + "0xae4dc089d92c027f4cc3141c309edb5e23f0bc0935146bf3db2592317479e7757fbc65624ef9c1ecfb54d21602079d06", + "0x91edd2ba701744a581d1dad1b67b862d08e0dad9558f8c31826692cb9eb11a97f25ea32859113300e69136b9b01b4ac2", + "0xb9b781554f467aea192418c18dae0bdc986d447f6a020224d5062143cb96c1bde68d8365d1108c11162492a79238fd96", + "0xa09202a971426cc12fc4609b8c63cf8b1e11e2ba59db9238da5d7eaff9b8df01d7ed6cd927f82685839edd3a621112ed" + ] + timeout_get_header_ms = 900 + [[mux.relays]] + id = "mux_helix_1" + url = "{{ index .Relays 1 }}" + + [logs.stdout] + level = "debug" + + [logs.file] + enabled = false + +network_params: + network: kurtosis + network_id: "3151908" + deposit_contract_address: "0x00000000219ab540356cBB839Cbe05303d7705Fa" + seconds_per_slot: 12 + slot_duration_ms: 12000 + num_validator_keys_per_node: 256 + preregistered_validator_keys_mnemonic: + "giant issue aisle success illegal bike spike + question tent bar rely arctic volcano long crawl hungry vocal artwork sniff fantasy + very lucky have athlete" + prefunded_accounts: '{"0xb9e79d19f651a941757b35830232E7EFC77E1c79": {"balance": "100000ETH"}}' + diff --git a/tests/fixtures/golden-configs/cb-signer.yml b/tests/fixtures/golden-configs/cb-signer.yml new file mode 100644 index 0000000..a520320 --- /dev/null +++ b/tests/fixtures/golden-configs/cb-signer.yml @@ -0,0 +1,186 @@ +# cb-signer: the Commit-Boost SIGNER module, which has never been +# testable on Kurtosis (the ethereum-package had no config support). +# +# Adds [signer] + [[modules]] to the CB config. The signer container +# reuses the devnet's existing validator keystores in TEKU format: +# secrets/ is chmod 0600 root-owned (no execute bit), so CB's uid 10001 +# cannot traverse it and would load ZERO keys while looking healthy; +# teku-secrets is 755 and teku-keys 777 (verified on a live enclave). +# +# PBS reads the same file: [signer]/[[modules]] are Option fields it +# parses and drops, and pbs.with_signer is dead code in the shipped +# binary, so nothing about the PBS path changes. + +participants: + - el_type: geth + cl_type: lighthouse + +additional_services: + - dora + - spamoor + - prometheus + +mev_type: custom + +mev_params: + mev_relay: helix + mev_sidecar: commit-boost + mev_builder: flashbots + + helix_relay_image: ghcr.io/gattaca-com/helix-relay:main + mev_boost_image: commit-boost/commit-boost:kurtosis + mev_builder_image: ethpandaops/reth-rbuilder:develop + mev_builder_cl_image: sigp/lighthouse:latest + + mev_builder_subsidy: 1 + commit_boost_signer: true + + helix_relay_config: | + instance_id: "helix-kurtosis-test" + + # NOTE: no network/genesis section. Current helix-relay:main removed the old + # `network_config: !Custom {dir_path, genesis_validator_root, genesis_time}` + # field entirely; the relay now fetches the chain spec + genesis from the + # beacon node at startup (GET eth/v1/config/spec + eth/v1/beacon/genesis, see + # beacon_client.get_chain_info -> main.rs load_chain_info). A `!Custom` YAML + # tag on that now-unknown key makes serde_yaml panic with + # "untagged and internally tagged enums do not support enum input". + + postgres: + hostname: "{{ .POSTGRES_HOST_NAME }}" + port: {{ .POSTGRES_PORT }} + db_name: "{{ .POSTGRES_DB }}" + user: "{{ .POSTGRES_USER }}" + password: "{{ .POSTGRES_PASS }}" + region: 0 + region_name: "LOCAL" + + beacon_clients: + - url: "{{ .BEACON_URI }}" + + gossip_payload_on_header: false + + simulators: + - url: "{{ .BLOCKSIM_URI }}" + namespace: flashbots + is_merging_simulator: false + max_concurrent_tasks: 32 + + router_config: + enabled_routes: + - route: GetValidators + - route: SubmitBlock + - route: GetTopBid + - route: GetHeader + rate_limit: + replenish_ms: 50 + burst_size: 20 + - route: GetPayload + # The builder-spec v2 proposer route (submitBlindedBlockV2). REQUIRED for + # any CL that submits via v2 -- prysm does. Without it helix 404s + # /eth/v2/builder/blinded_blocks, CB (correctly) refuses to downgrade to v1 + # (v2 semantics: the relay publishes the block, so a v1 payload would be + # silently dropped), returns 502, and EVERY builder block the proposer + # chose is lost. That read as "prysm can't do MEV" until the route list was + # checked -- see .agent/SWEEP-BACKLOG.md (Law 7 first dividend). + - route: GetPayloadV2 + - route: HeaderStream + - route: RegisterValidators + - route: Status + - route: ProposerPayloadDelivered + - route: BuilderBidsReceived + - route: ValidatorRegistration + shutdown_delay_ms: 12000 + + timing_game_config: + max_header_delay_ms: 400 + latest_header_delay_ms_in_slot: 1500 + default_client_latency_ms: 50 + + target_get_payload_propagation_duration_ms: 500 + + is_submission_instance: true + is_registration_instance: true + + admin_token: "test_admin_token" + + logging: + type: Console + + # CoresConfig (helix_common::config CoresConfig, 10 fields in current + # :main). The old block used `sub_workers: [0]` and omitted the per-tile + # core assignments; current :main removed sub_workers and added + # decoder/simulator/top_bid/data_gatherer/block_merging/housekeeper. The + # outer RelayConfigExt flattens RelayConfig, so a wrong/missing cores field + # surfaces as a top-level "missing field `decoder`" / "invalid type ... + # expected usize|sequence" serde error. Verified against the binary: + # `decoder` is Vec ([0]); the other five new tile fields are usize. + cores: + auctioneer: 0 + tokio: [0] + reg_workers: [0] + tcp_bid_submissions_tile: 2 + decoder: [0] + simulator: 0 + top_bid: 0 + data_gatherer: 0 + block_merging: 0 + housekeeper: 0 + + is_local_dev: false + + commit_boost_config: | + chain = { genesis_time_secs = {{ .Timestamp }}, path = "{{ .Network }}" } + + [pbs] + host = "0.0.0.0" + port = {{ .Port }} + timeout_get_header_ms = 950 + timeout_get_payload_ms = 4000 + late_in_slot_time_ms = 2000 + + + [metrics] + enabled = true + host = "0.0.0.0" + start_port = 9090 + + {{ range $index, $relay := .Relays }} + [[relays]] + id = "mev_relay_{{$index}}" + url = "{{ $relay }}" + {{- end }} + + [logs.stdout] + level = "debug" + + [logs.file] + enabled = false + + [signer] + port = 20000 + + [signer.local.loader] + format = "teku" + keys_path = "/keystores/teku-keys" + secrets_path = "/keystores/teku-secrets" + + [[modules]] + id = "TEST_MODULE" + type = "commit" + signing_id = "0x6a33a23ef26a4836979edff86c493a69b26ccf0b4a16491a815a13787657431b" + docker_image = "unused" + +network_params: + network: kurtosis + network_id: "3151908" + deposit_contract_address: "0x00000000219ab540356cBB839Cbe05303d7705Fa" + seconds_per_slot: 12 + slot_duration_ms: 12000 + num_validator_keys_per_node: 128 + preregistered_validator_keys_mnemonic: + "giant issue aisle success illegal bike spike + question tent bar rely arctic volcano long crawl hungry vocal artwork sniff fantasy + very lucky have athlete" + prefunded_accounts: '{"0xb9e79d19f651a941757b35830232E7EFC77E1c79": {"balance": "100000ETH"}}' + diff --git a/tests/fixtures/golden-configs/cb-sigverify-diff-control.yml b/tests/fixtures/golden-configs/cb-sigverify-diff-control.yml new file mode 100644 index 0000000..1dbe5af --- /dev/null +++ b/tests/fixtures/golden-configs/cb-sigverify-diff-control.yml @@ -0,0 +1,162 @@ +# cb-sigverify-diff-control: the skip_sigverify differential (control +# arm). Same wrong-pubkey literal relay url as cb-sigverify-diff but +# withOUT skip_sigverify - CB rejects every bid (PubkeyMismatch), so +# the run is EXPECTED to fail payload delivery. `sim diff` against the +# treatment run shows the flip that proves the feature discriminates. + +participants: + - el_type: geth + cl_type: lighthouse + +additional_services: + - dora + - spamoor + - prometheus + +mev_type: custom + +mev_params: + mev_relay: helix + mev_sidecar: commit-boost + mev_builder: flashbots + + helix_relay_image: ghcr.io/gattaca-com/helix-relay:main + mev_boost_image: commit-boost/commit-boost:kurtosis + mev_builder_image: ethpandaops/reth-rbuilder:develop + mev_builder_cl_image: sigp/lighthouse:latest + + mev_builder_subsidy: 1 + + helix_relay_config: | + instance_id: "helix-kurtosis-test" + + # NOTE: no network/genesis section. Current helix-relay:main removed the old + # `network_config: !Custom {dir_path, genesis_validator_root, genesis_time}` + # field entirely; the relay now fetches the chain spec + genesis from the + # beacon node at startup (GET eth/v1/config/spec + eth/v1/beacon/genesis, see + # beacon_client.get_chain_info -> main.rs load_chain_info). A `!Custom` YAML + # tag on that now-unknown key makes serde_yaml panic with + # "untagged and internally tagged enums do not support enum input". + + postgres: + hostname: "{{ .POSTGRES_HOST_NAME }}" + port: {{ .POSTGRES_PORT }} + db_name: "{{ .POSTGRES_DB }}" + user: "{{ .POSTGRES_USER }}" + password: "{{ .POSTGRES_PASS }}" + region: 0 + region_name: "LOCAL" + + beacon_clients: + - url: "{{ .BEACON_URI }}" + + gossip_payload_on_header: false + + simulators: + - url: "{{ .BLOCKSIM_URI }}" + namespace: flashbots + is_merging_simulator: false + max_concurrent_tasks: 32 + + router_config: + enabled_routes: + - route: GetValidators + - route: SubmitBlock + - route: GetTopBid + - route: GetHeader + rate_limit: + replenish_ms: 50 + burst_size: 20 + - route: GetPayload + # The builder-spec v2 proposer route (submitBlindedBlockV2). REQUIRED for + # any CL that submits via v2 -- prysm does. Without it helix 404s + # /eth/v2/builder/blinded_blocks, CB (correctly) refuses to downgrade to v1 + # (v2 semantics: the relay publishes the block, so a v1 payload would be + # silently dropped), returns 502, and EVERY builder block the proposer + # chose is lost. That read as "prysm can't do MEV" until the route list was + # checked -- see .agent/SWEEP-BACKLOG.md (Law 7 first dividend). + - route: GetPayloadV2 + - route: HeaderStream + - route: RegisterValidators + - route: Status + - route: ProposerPayloadDelivered + - route: BuilderBidsReceived + - route: ValidatorRegistration + shutdown_delay_ms: 12000 + + timing_game_config: + max_header_delay_ms: 400 + latest_header_delay_ms_in_slot: 1500 + default_client_latency_ms: 50 + + target_get_payload_propagation_duration_ms: 500 + + is_submission_instance: true + is_registration_instance: true + + admin_token: "test_admin_token" + + logging: + type: Console + + # CoresConfig (helix_common::config CoresConfig, 10 fields in current + # :main). The old block used `sub_workers: [0]` and omitted the per-tile + # core assignments; current :main removed sub_workers and added + # decoder/simulator/top_bid/data_gatherer/block_merging/housekeeper. The + # outer RelayConfigExt flattens RelayConfig, so a wrong/missing cores field + # surfaces as a top-level "missing field `decoder`" / "invalid type ... + # expected usize|sequence" serde error. Verified against the binary: + # `decoder` is Vec ([0]); the other five new tile fields are usize. + cores: + auctioneer: 0 + tokio: [0] + reg_workers: [0] + tcp_bid_submissions_tile: 2 + decoder: [0] + simulator: 0 + top_bid: 0 + data_gatherer: 0 + block_merging: 0 + housekeeper: 0 + + is_local_dev: false + + commit_boost_config: | + chain = { genesis_time_secs = {{ .Timestamp }}, path = "{{ .Network }}" } + + [pbs] + host = "0.0.0.0" + port = {{ .Port }} + timeout_get_header_ms = 950 + timeout_get_payload_ms = 4000 + late_in_slot_time_ms = 2000 + + + [metrics] + enabled = true + host = "0.0.0.0" + start_port = 9090 + + [[relays]] + id = "mev_relay_0" + url = "http://0xaaf6c1251e73fb600624937760fef218aace5b253bf068ed45398aeb29d821e4d2899343ddcbbe37cb3f6cf500dff26c@helix-relay-2:4040" + + [logs.stdout] + level = "debug" + + [logs.file] + enabled = false + +network_params: + network: kurtosis + network_id: "3151908" + deposit_contract_address: "0x00000000219ab540356cBB839Cbe05303d7705Fa" + seconds_per_slot: 12 + slot_duration_ms: 12000 + num_validator_keys_per_node: 128 + preregistered_validator_keys_mnemonic: + "giant issue aisle success illegal bike spike + question tent bar rely arctic volcano long crawl hungry vocal artwork sniff fantasy + very lucky have athlete" + prefunded_accounts: '{"0xb9e79d19f651a941757b35830232E7EFC77E1c79": {"balance": "100000ETH"}}' + diff --git a/tests/fixtures/golden-configs/cb-sigverify-diff.yml b/tests/fixtures/golden-configs/cb-sigverify-diff.yml new file mode 100644 index 0000000..7aac423 --- /dev/null +++ b/tests/fixtures/golden-configs/cb-sigverify-diff.yml @@ -0,0 +1,166 @@ +# cb-sigverify-diff: the skip_sigverify DIFFERENTIAL (treatment arm). +# +# CB's [[relays]] entry is a LITERAL url whose pubkey is a valid BLS +# key that is NOT the helix relay's signing key, so CB's signature +# validation would reject every bid. With skip_sigverify = true the +# validation is skipped and bids flow anyway - an auction winner in +# this scenario is positive proof the skip codepath fired. Compare +# with cb-sigverify-diff-control (same poison, skip OFF, zero bids). + +participants: + - el_type: geth + cl_type: lighthouse + +additional_services: + - dora + - spamoor + - prometheus + +mev_type: custom + +mev_params: + mev_relay: helix + mev_sidecar: commit-boost + mev_builder: flashbots + + helix_relay_image: ghcr.io/gattaca-com/helix-relay:main + mev_boost_image: commit-boost/commit-boost:kurtosis + mev_builder_image: ethpandaops/reth-rbuilder:develop + mev_builder_cl_image: sigp/lighthouse:latest + + mev_builder_subsidy: 1 + + helix_relay_config: | + instance_id: "helix-kurtosis-test" + + # NOTE: no network/genesis section. Current helix-relay:main removed the old + # `network_config: !Custom {dir_path, genesis_validator_root, genesis_time}` + # field entirely; the relay now fetches the chain spec + genesis from the + # beacon node at startup (GET eth/v1/config/spec + eth/v1/beacon/genesis, see + # beacon_client.get_chain_info -> main.rs load_chain_info). A `!Custom` YAML + # tag on that now-unknown key makes serde_yaml panic with + # "untagged and internally tagged enums do not support enum input". + + postgres: + hostname: "{{ .POSTGRES_HOST_NAME }}" + port: {{ .POSTGRES_PORT }} + db_name: "{{ .POSTGRES_DB }}" + user: "{{ .POSTGRES_USER }}" + password: "{{ .POSTGRES_PASS }}" + region: 0 + region_name: "LOCAL" + + beacon_clients: + - url: "{{ .BEACON_URI }}" + + gossip_payload_on_header: false + + simulators: + - url: "{{ .BLOCKSIM_URI }}" + namespace: flashbots + is_merging_simulator: false + max_concurrent_tasks: 32 + + router_config: + enabled_routes: + - route: GetValidators + - route: SubmitBlock + - route: GetTopBid + - route: GetHeader + rate_limit: + replenish_ms: 50 + burst_size: 20 + - route: GetPayload + # The builder-spec v2 proposer route (submitBlindedBlockV2). REQUIRED for + # any CL that submits via v2 -- prysm does. Without it helix 404s + # /eth/v2/builder/blinded_blocks, CB (correctly) refuses to downgrade to v1 + # (v2 semantics: the relay publishes the block, so a v1 payload would be + # silently dropped), returns 502, and EVERY builder block the proposer + # chose is lost. That read as "prysm can't do MEV" until the route list was + # checked -- see .agent/SWEEP-BACKLOG.md (Law 7 first dividend). + - route: GetPayloadV2 + - route: HeaderStream + - route: RegisterValidators + - route: Status + - route: ProposerPayloadDelivered + - route: BuilderBidsReceived + - route: ValidatorRegistration + shutdown_delay_ms: 12000 + + timing_game_config: + max_header_delay_ms: 400 + latest_header_delay_ms_in_slot: 1500 + default_client_latency_ms: 50 + + target_get_payload_propagation_duration_ms: 500 + + is_submission_instance: true + is_registration_instance: true + + admin_token: "test_admin_token" + + logging: + type: Console + + # CoresConfig (helix_common::config CoresConfig, 10 fields in current + # :main). The old block used `sub_workers: [0]` and omitted the per-tile + # core assignments; current :main removed sub_workers and added + # decoder/simulator/top_bid/data_gatherer/block_merging/housekeeper. The + # outer RelayConfigExt flattens RelayConfig, so a wrong/missing cores field + # surfaces as a top-level "missing field `decoder`" / "invalid type ... + # expected usize|sequence" serde error. Verified against the binary: + # `decoder` is Vec ([0]); the other five new tile fields are usize. + cores: + auctioneer: 0 + tokio: [0] + reg_workers: [0] + tcp_bid_submissions_tile: 2 + decoder: [0] + simulator: 0 + top_bid: 0 + data_gatherer: 0 + block_merging: 0 + housekeeper: 0 + + is_local_dev: false + + commit_boost_config: | + chain = { genesis_time_secs = {{ .Timestamp }}, path = "{{ .Network }}" } + + [pbs] + host = "0.0.0.0" + port = {{ .Port }} + skip_sigverify = true + timeout_get_header_ms = 950 + timeout_get_payload_ms = 4000 + late_in_slot_time_ms = 2000 + + + [metrics] + enabled = true + host = "0.0.0.0" + start_port = 9090 + + [[relays]] + id = "mev_relay_0" + url = "http://0xaaf6c1251e73fb600624937760fef218aace5b253bf068ed45398aeb29d821e4d2899343ddcbbe37cb3f6cf500dff26c@helix-relay-2:4040" + + [logs.stdout] + level = "debug" + + [logs.file] + enabled = false + +network_params: + network: kurtosis + network_id: "3151908" + deposit_contract_address: "0x00000000219ab540356cBB839Cbe05303d7705Fa" + seconds_per_slot: 12 + slot_duration_ms: 12000 + num_validator_keys_per_node: 128 + preregistered_validator_keys_mnemonic: + "giant issue aisle success illegal bike spike + question tent bar rely arctic volcano long crawl hungry vocal artwork sniff fantasy + very lucky have athlete" + prefunded_accounts: '{"0xb9e79d19f651a941757b35830232E7EFC77E1c79": {"balance": "100000ETH"}}' + diff --git a/tests/fixtures/golden-configs/cb-skip-sigverify.yml b/tests/fixtures/golden-configs/cb-skip-sigverify.yml new file mode 100644 index 0000000..a602e46 --- /dev/null +++ b/tests/fixtures/golden-configs/cb-skip-sigverify.yml @@ -0,0 +1,165 @@ +# cb-skip-sigverify: Signature verification disabled for header responses. +# +# Tests the CB fast path where BLS verification is skipped. This trades +# correctness for speed — useful to verify that the path exists and is +# reachable under load. + +participants: + - el_type: geth + cl_type: lighthouse + +additional_services: + - dora + - spamoor + - prometheus + +mev_type: custom + +mev_params: + mev_relay: helix + mev_sidecar: commit-boost + mev_builder: flashbots + + helix_relay_image: ghcr.io/gattaca-com/helix-relay:main + mev_boost_image: commit-boost/commit-boost:kurtosis + mev_builder_image: ethpandaops/reth-rbuilder:develop + mev_builder_cl_image: sigp/lighthouse:latest + + mev_builder_subsidy: 1 + + helix_relay_config: | + instance_id: "helix-kurtosis-test" + + # NOTE: no network/genesis section. Current helix-relay:main removed the old + # `network_config: !Custom {dir_path, genesis_validator_root, genesis_time}` + # field entirely; the relay now fetches the chain spec + genesis from the + # beacon node at startup (GET eth/v1/config/spec + eth/v1/beacon/genesis, see + # beacon_client.get_chain_info -> main.rs load_chain_info). A `!Custom` YAML + # tag on that now-unknown key makes serde_yaml panic with + # "untagged and internally tagged enums do not support enum input". + + postgres: + hostname: "{{ .POSTGRES_HOST_NAME }}" + port: {{ .POSTGRES_PORT }} + db_name: "{{ .POSTGRES_DB }}" + user: "{{ .POSTGRES_USER }}" + password: "{{ .POSTGRES_PASS }}" + region: 0 + region_name: "LOCAL" + + beacon_clients: + - url: "{{ .BEACON_URI }}" + + gossip_payload_on_header: false + + simulators: + - url: "{{ .BLOCKSIM_URI }}" + namespace: flashbots + is_merging_simulator: false + max_concurrent_tasks: 32 + + router_config: + enabled_routes: + - route: GetValidators + - route: SubmitBlock + - route: GetTopBid + - route: GetHeader + rate_limit: + replenish_ms: 50 + burst_size: 20 + - route: GetPayload + # The builder-spec v2 proposer route (submitBlindedBlockV2). REQUIRED for + # any CL that submits via v2 -- prysm does. Without it helix 404s + # /eth/v2/builder/blinded_blocks, CB (correctly) refuses to downgrade to v1 + # (v2 semantics: the relay publishes the block, so a v1 payload would be + # silently dropped), returns 502, and EVERY builder block the proposer + # chose is lost. That read as "prysm can't do MEV" until the route list was + # checked -- see .agent/SWEEP-BACKLOG.md (Law 7 first dividend). + - route: GetPayloadV2 + - route: HeaderStream + - route: RegisterValidators + - route: Status + - route: ProposerPayloadDelivered + - route: BuilderBidsReceived + - route: ValidatorRegistration + shutdown_delay_ms: 12000 + + timing_game_config: + max_header_delay_ms: 400 + latest_header_delay_ms_in_slot: 1500 + default_client_latency_ms: 50 + + target_get_payload_propagation_duration_ms: 500 + + is_submission_instance: true + is_registration_instance: true + + admin_token: "test_admin_token" + + logging: + type: Console + + # CoresConfig (helix_common::config CoresConfig, 10 fields in current + # :main). The old block used `sub_workers: [0]` and omitted the per-tile + # core assignments; current :main removed sub_workers and added + # decoder/simulator/top_bid/data_gatherer/block_merging/housekeeper. The + # outer RelayConfigExt flattens RelayConfig, so a wrong/missing cores field + # surfaces as a top-level "missing field `decoder`" / "invalid type ... + # expected usize|sequence" serde error. Verified against the binary: + # `decoder` is Vec ([0]); the other five new tile fields are usize. + cores: + auctioneer: 0 + tokio: [0] + reg_workers: [0] + tcp_bid_submissions_tile: 2 + decoder: [0] + simulator: 0 + top_bid: 0 + data_gatherer: 0 + block_merging: 0 + housekeeper: 0 + + is_local_dev: false + + commit_boost_config: | + chain = { genesis_time_secs = {{ .Timestamp }}, path = "{{ .Network }}" } + + [pbs] + host = "0.0.0.0" + port = {{ .Port }} + skip_sigverify = true + timeout_get_header_ms = 950 + timeout_get_payload_ms = 4000 + late_in_slot_time_ms = 2000 + + + [metrics] + enabled = true + host = "0.0.0.0" + start_port = 9090 + + {{ range $index, $relay := .Relays }} + [[relays]] + id = "mev_relay_{{$index}}" + url = "{{ $relay }}" + {{- end }} + + [logs.stdout] + level = "debug" + + [logs.file] + enabled = false + +network_params: + network: kurtosis + network_id: "3151908" + deposit_contract_address: "0x00000000219ab540356cBB839Cbe05303d7705Fa" + seconds_per_slot: 12 + slot_duration_ms: 12000 + num_validator_keys_per_node: 128 + preregistered_validator_keys_mnemonic: + "giant issue aisle success illegal bike spike + question tent bar rely arctic volcano long crawl hungry vocal artwork sniff fantasy + very lucky have athlete" + prefunded_accounts: '{"0xb9e79d19f651a941757b35830232E7EFC77E1c79": {"balance": "100000ETH"}}' + diff --git a/tests/fixtures/golden-configs/cb-timing-games.yml b/tests/fixtures/golden-configs/cb-timing-games.yml new file mode 100644 index 0000000..bb6f363 --- /dev/null +++ b/tests/fixtures/golden-configs/cb-timing-games.yml @@ -0,0 +1,170 @@ +# cb-timing-games: Aggressive timing game configuration. +# +# Tests CB's ability to orchestrate repeated get_header polls with +# short timeouts in order to arrive at the best bid as late as possible +# in the slot. Per-relay timing overrides are enabled for all relays. + +participants: + - el_type: geth + cl_type: lighthouse + +additional_services: + - dora + - spamoor + - prometheus + +mev_type: custom + +mev_params: + mev_relay: + - helix + - helix + mev_sidecar: commit-boost + mev_builder: flashbots + + helix_relay_image: ghcr.io/gattaca-com/helix-relay:main + mev_relay_image: ethpandaops/mev-boost-relay:main + mev_boost_image: commit-boost/commit-boost:kurtosis + mev_builder_image: ethpandaops/reth-rbuilder:develop + mev_builder_cl_image: sigp/lighthouse:latest + + mev_builder_subsidy: 1 + + helix_relay_config: | + instance_id: "helix-kurtosis-test" + + # NOTE: no network/genesis section. Current helix-relay:main removed the old + # `network_config: !Custom {dir_path, genesis_validator_root, genesis_time}` + # field entirely; the relay now fetches the chain spec + genesis from the + # beacon node at startup (GET eth/v1/config/spec + eth/v1/beacon/genesis, see + # beacon_client.get_chain_info -> main.rs load_chain_info). A `!Custom` YAML + # tag on that now-unknown key makes serde_yaml panic with + # "untagged and internally tagged enums do not support enum input". + + postgres: + hostname: "{{ .POSTGRES_HOST_NAME }}" + port: {{ .POSTGRES_PORT }} + db_name: "{{ .POSTGRES_DB }}" + user: "{{ .POSTGRES_USER }}" + password: "{{ .POSTGRES_PASS }}" + region: 0 + region_name: "LOCAL" + + beacon_clients: + - url: "{{ .BEACON_URI }}" + + gossip_payload_on_header: false + + simulators: + - url: "{{ .BLOCKSIM_URI }}" + namespace: flashbots + is_merging_simulator: false + max_concurrent_tasks: 32 + + router_config: + enabled_routes: + - route: GetValidators + - route: SubmitBlock + - route: GetTopBid + - route: GetHeader + rate_limit: + replenish_ms: 50 + burst_size: 20 + - route: GetPayload + # The builder-spec v2 proposer route (submitBlindedBlockV2). REQUIRED for + # any CL that submits via v2 -- prysm does. Without it helix 404s + # /eth/v2/builder/blinded_blocks, CB (correctly) refuses to downgrade to v1 + # (v2 semantics: the relay publishes the block, so a v1 payload would be + # silently dropped), returns 502, and EVERY builder block the proposer + # chose is lost. That read as "prysm can't do MEV" until the route list was + # checked -- see .agent/SWEEP-BACKLOG.md (Law 7 first dividend). + - route: GetPayloadV2 + - route: HeaderStream + - route: RegisterValidators + - route: Status + - route: ProposerPayloadDelivered + - route: BuilderBidsReceived + - route: ValidatorRegistration + shutdown_delay_ms: 12000 + + timing_game_config: + max_header_delay_ms: 400 + latest_header_delay_ms_in_slot: 1500 + default_client_latency_ms: 50 + + target_get_payload_propagation_duration_ms: 500 + + is_submission_instance: true + is_registration_instance: true + + admin_token: "test_admin_token" + + logging: + type: Console + + # CoresConfig (helix_common::config CoresConfig, 10 fields in current + # :main). The old block used `sub_workers: [0]` and omitted the per-tile + # core assignments; current :main removed sub_workers and added + # decoder/simulator/top_bid/data_gatherer/block_merging/housekeeper. The + # outer RelayConfigExt flattens RelayConfig, so a wrong/missing cores field + # surfaces as a top-level "missing field `decoder`" / "invalid type ... + # expected usize|sequence" serde error. Verified against the binary: + # `decoder` is Vec ([0]); the other five new tile fields are usize. + cores: + auctioneer: 0 + tokio: [0] + reg_workers: [0] + tcp_bid_submissions_tile: 2 + decoder: [0] + simulator: 0 + top_bid: 0 + data_gatherer: 0 + block_merging: 0 + housekeeper: 0 + + is_local_dev: false + + commit_boost_config: | + chain = { genesis_time_secs = {{ .Timestamp }}, path = "{{ .Network }}" } + + [pbs] + host = "0.0.0.0" + port = {{ .Port }} + timeout_get_header_ms = 400 + timeout_get_payload_ms = 2000 + late_in_slot_time_ms = 2000 + + + [metrics] + enabled = true + host = "0.0.0.0" + start_port = 9090 + + {{ range $index, $relay := .Relays }} + [[relays]] + id = "mev_relay_{{$index}}" + url = "{{ $relay }}" + enable_timing_games = true + target_first_request_ms = 100 + frequency_get_header_ms = 200 + {{- end }} + + [logs.stdout] + level = "debug" + + [logs.file] + enabled = false + +network_params: + network: kurtosis + network_id: "3151908" + deposit_contract_address: "0x00000000219ab540356cBB839Cbe05303d7705Fa" + seconds_per_slot: 12 + slot_duration_ms: 12000 + num_validator_keys_per_node: 128 + preregistered_validator_keys_mnemonic: + "giant issue aisle success illegal bike spike + question tent bar rely arctic volcano long crawl hungry vocal artwork sniff fantasy + very lucky have athlete" + prefunded_accounts: '{"0xb9e79d19f651a941757b35830232E7EFC77E1c79": {"balance": "100000ETH"}}' + diff --git a/tests/fixtures/golden-configs/cb-ws-stream-nokey.yml b/tests/fixtures/golden-configs/cb-ws-stream-nokey.yml new file mode 100644 index 0000000..70092f5 --- /dev/null +++ b/tests/fixtures/golden-configs/cb-ws-stream-nokey.yml @@ -0,0 +1,168 @@ +# cb-ws-stream-nokey: NEGATIVE CONTROL for the ws criteria. +# +# Stream configured with NO api key: helix refuses every handshake +# ("no api key registered for this proposer"), every slot falls back +# to HTTP, MEV stays green - and feature.ws_header_stream goes +# INCONCLUSIVE. EXPECTED to fail under --require-feature-proof; that +# failure is this scenario's proof that the criteria discriminate. +# Not part of the green sweep. + +participants: + - el_type: geth + cl_type: lighthouse + +additional_services: + - dora + - spamoor + - prometheus + +mev_type: custom + +mev_params: + mev_relay: helix + mev_sidecar: commit-boost + mev_builder: flashbots + + helix_relay_image: ghcr.io/gattaca-com/helix-relay:main + mev_boost_image: commit-boost/commit-boost:kurtosis + mev_builder_image: ethpandaops/reth-rbuilder:develop + mev_builder_cl_image: sigp/lighthouse:latest + + mev_builder_subsidy: 1 + + helix_relay_config: | + instance_id: "helix-kurtosis-test" + + # NOTE: no network/genesis section. Current helix-relay:main removed the old + # `network_config: !Custom {dir_path, genesis_validator_root, genesis_time}` + # field entirely; the relay now fetches the chain spec + genesis from the + # beacon node at startup (GET eth/v1/config/spec + eth/v1/beacon/genesis, see + # beacon_client.get_chain_info -> main.rs load_chain_info). A `!Custom` YAML + # tag on that now-unknown key makes serde_yaml panic with + # "untagged and internally tagged enums do not support enum input". + + postgres: + hostname: "{{ .POSTGRES_HOST_NAME }}" + port: {{ .POSTGRES_PORT }} + db_name: "{{ .POSTGRES_DB }}" + user: "{{ .POSTGRES_USER }}" + password: "{{ .POSTGRES_PASS }}" + region: 0 + region_name: "LOCAL" + + beacon_clients: + - url: "{{ .BEACON_URI }}" + + gossip_payload_on_header: false + + simulators: + - url: "{{ .BLOCKSIM_URI }}" + namespace: flashbots + is_merging_simulator: false + max_concurrent_tasks: 32 + + router_config: + enabled_routes: + - route: GetValidators + - route: SubmitBlock + - route: GetTopBid + - route: GetHeader + rate_limit: + replenish_ms: 50 + burst_size: 20 + - route: GetPayload + # The builder-spec v2 proposer route (submitBlindedBlockV2). REQUIRED for + # any CL that submits via v2 -- prysm does. Without it helix 404s + # /eth/v2/builder/blinded_blocks, CB (correctly) refuses to downgrade to v1 + # (v2 semantics: the relay publishes the block, so a v1 payload would be + # silently dropped), returns 502, and EVERY builder block the proposer + # chose is lost. That read as "prysm can't do MEV" until the route list was + # checked -- see .agent/SWEEP-BACKLOG.md (Law 7 first dividend). + - route: GetPayloadV2 + - route: HeaderStream + - route: RegisterValidators + - route: Status + - route: ProposerPayloadDelivered + - route: BuilderBidsReceived + - route: ValidatorRegistration + shutdown_delay_ms: 12000 + + timing_game_config: + max_header_delay_ms: 400 + latest_header_delay_ms_in_slot: 1500 + default_client_latency_ms: 50 + + target_get_payload_propagation_duration_ms: 500 + + is_submission_instance: true + is_registration_instance: true + + admin_token: "test_admin_token" + + logging: + type: Console + + # CoresConfig (helix_common::config CoresConfig, 10 fields in current + # :main). The old block used `sub_workers: [0]` and omitted the per-tile + # core assignments; current :main removed sub_workers and added + # decoder/simulator/top_bid/data_gatherer/block_merging/housekeeper. The + # outer RelayConfigExt flattens RelayConfig, so a wrong/missing cores field + # surfaces as a top-level "missing field `decoder`" / "invalid type ... + # expected usize|sequence" serde error. Verified against the binary: + # `decoder` is Vec ([0]); the other five new tile fields are usize. + cores: + auctioneer: 0 + tokio: [0] + reg_workers: [0] + tcp_bid_submissions_tile: 2 + decoder: [0] + simulator: 0 + top_bid: 0 + data_gatherer: 0 + block_merging: 0 + housekeeper: 0 + + is_local_dev: false + + commit_boost_config: | + chain = { genesis_time_secs = {{ .Timestamp }}, path = "{{ .Network }}" } + + [pbs] + host = "0.0.0.0" + port = {{ .Port }} + timeout_get_header_ms = 950 + timeout_get_payload_ms = 4000 + late_in_slot_time_ms = 2000 + + + [metrics] + enabled = true + host = "0.0.0.0" + start_port = 9090 + + {{ range $index, $relay := .Relays }} + [[relays]] + id = "mev_relay_{{$index}}" + url = "{{ $relay }}" + get_header = "stream" + {{- end }} + + [logs.stdout] + level = "debug" + + [logs.file] + enabled = false + +network_params: + network: kurtosis + network_id: "3151908" + deposit_contract_address: "0x00000000219ab540356cBB839Cbe05303d7705Fa" + seconds_per_slot: 12 + slot_duration_ms: 12000 + num_validator_keys_per_node: 128 + preregistered_validator_keys_mnemonic: + "giant issue aisle success illegal bike spike + question tent bar rely arctic volcano long crawl hungry vocal artwork sniff fantasy + very lucky have athlete" + prefunded_accounts: '{"0xb9e79d19f651a941757b35830232E7EFC77E1c79": {"balance": "100000ETH"}}' + diff --git a/tests/fixtures/golden-configs/cb-ws-stream.yml b/tests/fixtures/golden-configs/cb-ws-stream.yml new file mode 100644 index 0000000..0d83e89 --- /dev/null +++ b/tests/fixtures/golden-configs/cb-ws-stream.yml @@ -0,0 +1,169 @@ +# cb-ws-stream: getHeader over the websocket bid stream. +# +# CB `get_header = "stream"` + helix HeaderStream route. The relay's +# X-Api-Key header rides validator registration, helix TOFU-binds it, +# and the stream authenticates. THE TRAP this scenario guards: the HTTP +# fallback keeps every MEV check green when the stream is broken, so +# feature.ws_header_stream (proof markers) is the real assertion - run +# under --require-feature-proof. + +participants: + - el_type: geth + cl_type: lighthouse + +additional_services: + - dora + - spamoor + - prometheus + +mev_type: custom + +mev_params: + mev_relay: helix + mev_sidecar: commit-boost + mev_builder: flashbots + + helix_relay_image: ghcr.io/gattaca-com/helix-relay:main + mev_boost_image: commit-boost/commit-boost:kurtosis + mev_builder_image: ethpandaops/reth-rbuilder:develop + mev_builder_cl_image: sigp/lighthouse:latest + + mev_builder_subsidy: 1 + + helix_relay_config: | + instance_id: "helix-kurtosis-test" + + # NOTE: no network/genesis section. Current helix-relay:main removed the old + # `network_config: !Custom {dir_path, genesis_validator_root, genesis_time}` + # field entirely; the relay now fetches the chain spec + genesis from the + # beacon node at startup (GET eth/v1/config/spec + eth/v1/beacon/genesis, see + # beacon_client.get_chain_info -> main.rs load_chain_info). A `!Custom` YAML + # tag on that now-unknown key makes serde_yaml panic with + # "untagged and internally tagged enums do not support enum input". + + postgres: + hostname: "{{ .POSTGRES_HOST_NAME }}" + port: {{ .POSTGRES_PORT }} + db_name: "{{ .POSTGRES_DB }}" + user: "{{ .POSTGRES_USER }}" + password: "{{ .POSTGRES_PASS }}" + region: 0 + region_name: "LOCAL" + + beacon_clients: + - url: "{{ .BEACON_URI }}" + + gossip_payload_on_header: false + + simulators: + - url: "{{ .BLOCKSIM_URI }}" + namespace: flashbots + is_merging_simulator: false + max_concurrent_tasks: 32 + + router_config: + enabled_routes: + - route: GetValidators + - route: SubmitBlock + - route: GetTopBid + - route: GetHeader + rate_limit: + replenish_ms: 50 + burst_size: 20 + - route: GetPayload + # The builder-spec v2 proposer route (submitBlindedBlockV2). REQUIRED for + # any CL that submits via v2 -- prysm does. Without it helix 404s + # /eth/v2/builder/blinded_blocks, CB (correctly) refuses to downgrade to v1 + # (v2 semantics: the relay publishes the block, so a v1 payload would be + # silently dropped), returns 502, and EVERY builder block the proposer + # chose is lost. That read as "prysm can't do MEV" until the route list was + # checked -- see .agent/SWEEP-BACKLOG.md (Law 7 first dividend). + - route: GetPayloadV2 + - route: HeaderStream + - route: RegisterValidators + - route: Status + - route: ProposerPayloadDelivered + - route: BuilderBidsReceived + - route: ValidatorRegistration + shutdown_delay_ms: 12000 + + timing_game_config: + max_header_delay_ms: 400 + latest_header_delay_ms_in_slot: 1500 + default_client_latency_ms: 50 + + target_get_payload_propagation_duration_ms: 500 + + is_submission_instance: true + is_registration_instance: true + + admin_token: "test_admin_token" + + logging: + type: Console + + # CoresConfig (helix_common::config CoresConfig, 10 fields in current + # :main). The old block used `sub_workers: [0]` and omitted the per-tile + # core assignments; current :main removed sub_workers and added + # decoder/simulator/top_bid/data_gatherer/block_merging/housekeeper. The + # outer RelayConfigExt flattens RelayConfig, so a wrong/missing cores field + # surfaces as a top-level "missing field `decoder`" / "invalid type ... + # expected usize|sequence" serde error. Verified against the binary: + # `decoder` is Vec ([0]); the other five new tile fields are usize. + cores: + auctioneer: 0 + tokio: [0] + reg_workers: [0] + tcp_bid_submissions_tile: 2 + decoder: [0] + simulator: 0 + top_bid: 0 + data_gatherer: 0 + block_merging: 0 + housekeeper: 0 + + is_local_dev: false + + commit_boost_config: | + chain = { genesis_time_secs = {{ .Timestamp }}, path = "{{ .Network }}" } + + [pbs] + host = "0.0.0.0" + port = {{ .Port }} + timeout_get_header_ms = 950 + timeout_get_payload_ms = 4000 + late_in_slot_time_ms = 2000 + + + [metrics] + enabled = true + host = "0.0.0.0" + start_port = 9090 + + {{ range $index, $relay := .Relays }} + [[relays]] + id = "mev_relay_{{$index}}" + url = "{{ $relay }}" + get_header = "stream" + headers = { X-Api-Key = "9d5c2f4e-1b7a-4c3d-8e6f-0a1b2c3d4e5f" } + {{- end }} + + [logs.stdout] + level = "debug" + + [logs.file] + enabled = false + +network_params: + network: kurtosis + network_id: "3151908" + deposit_contract_address: "0x00000000219ab540356cBB839Cbe05303d7705Fa" + seconds_per_slot: 12 + slot_duration_ms: 12000 + num_validator_keys_per_node: 128 + preregistered_validator_keys_mnemonic: + "giant issue aisle success illegal bike spike + question tent bar rely arctic volcano long crawl hungry vocal artwork sniff fantasy + very lucky have athlete" + prefunded_accounts: '{"0xb9e79d19f651a941757b35830232E7EFC77E1c79": {"balance": "100000ETH"}}' + diff --git a/tests/fixtures/helix_pregenesis_unwrap.log b/tests/fixtures/helix_pregenesis_unwrap.log new file mode 100644 index 0000000..89604e4 --- /dev/null +++ b/tests/fixtures/helix_pregenesis_unwrap.log @@ -0,0 +1,11 @@ +2026-07-30T10:05:00.001Z INFO helix_relay: starting up +2026-07-30T10:05:00.400Z INFO helix_relay: connected to beacon node +thread 'main' panicked at crates/common/src/chain_info.rs:63:26: +called `Option::unwrap()` on a `None` value +note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace +stack backtrace: + 0: rust_begin_unwind + 1: core::panicking::panic + 2: helix_common::chain_info::ChainInfo::current_slot + 3: helix_housekeeper::HousekeeperTile::new + 4: helix_relay::main diff --git a/tests/fixtures/helix_reached_fetch.log b/tests/fixtures/helix_reached_fetch.log new file mode 100644 index 0000000..694db50 --- /dev/null +++ b/tests/fixtures/helix_reached_fetch.log @@ -0,0 +1,7 @@ +2026-07-30T10:10:00.001Z INFO helix_relay: starting up +2026-07-30T10:10:00.020Z INFO helix_relay: reading config from /cfg/config.yaml +2026-07-30T10:10:00.050Z INFO helix_relay: config parsed, initialising relay +2026-07-30T10:10:00.080Z INFO helix_relay: starting metrics server on 0.0.0.0:9000 +2026-07-30T10:10:00.100Z INFO helix_housekeeper: calling get_chain_info against beacon node http://127.0.0.1:5052 +2026-07-30T10:10:05.400Z WARN helix_housekeeper: failed fetching chain info: error sending request for url (http://127.0.0.1:5052/eth/v1/beacon/genesis): connection refused +2026-07-30T10:10:10.400Z WARN helix_housekeeper: failed fetching chain info: connection refused, retrying diff --git a/tests/fixtures/helix_serde_missing_field.log b/tests/fixtures/helix_serde_missing_field.log new file mode 100644 index 0000000..5fba022 --- /dev/null +++ b/tests/fixtures/helix_serde_missing_field.log @@ -0,0 +1,10 @@ +2026-07-30T10:00:01.123Z INFO helix_relay: starting up +2026-07-30T10:00:01.130Z INFO helix_relay: reading config from /cfg/config.yaml +thread 'main' panicked at /app/crates/common/src/config.rs:203:51: +failed to parse config file: Error("missing field `decoder`", line: 1, column: 1) +note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace +stack backtrace: + 0: rust_begin_unwind + 1: core::panicking::panic_fmt + 2: helix_common::config::Config::load + 3: helix_relay::main diff --git a/tests/fixtures/invented_field.log b/tests/fixtures/invented_field.log new file mode 100644 index 0000000..9652a77 --- /dev/null +++ b/tests/fixtures/invented_field.log @@ -0,0 +1,8 @@ +2026-07-30T11:00:00.000Z INFO some-service: booting +2026-07-30T11:00:00.050Z INFO some-service: loading configuration +thread 'main' panicked at /app/crates/common/src/config.rs:203:51: +failed to parse config file: Error("missing field `foobar`", line: 4, column: 2) +note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace +stack backtrace: + 0: rust_begin_unwind + 1: core::panicking::panic_fmt diff --git a/tests/fixtures/multi_masked.log b/tests/fixtures/multi_masked.log new file mode 100644 index 0000000..15b4dfd --- /dev/null +++ b/tests/fixtures/multi_masked.log @@ -0,0 +1,11 @@ +time="2026-07-30T12:00:00Z" level=error msg="error while marshaling response: proto: string field contains invalid UTF-8" +time="2026-07-30T12:00:00Z" level=warn msg="Client might have cancelled the stream, closing" +rpc error: code = Internal desc = grpc: error while marshaling: invalid UTF-8 +2026-07-30T12:00:01.000Z INFO helix_relay: starting up +2026-07-30T12:00:01.010Z INFO helix_relay: reading config from /cfg/config.yaml +thread 'main' panicked at /app/crates/common/src/config.rs:203:51: +failed to parse config file: Error("missing field `network_config`", line: 1, column: 1) +note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace +stack backtrace: + 0: rust_begin_unwind + 1: core::panicking::panic_fmt diff --git a/tests/fixtures/next_line_message.log b/tests/fixtures/next_line_message.log new file mode 100644 index 0000000..e588a05 --- /dev/null +++ b/tests/fixtures/next_line_message.log @@ -0,0 +1,5 @@ +2026-07-30T14:00:00.000Z INFO worker: start +2026-07-30T14:00:00.900Z INFO worker: entering main loop +thread 'main' panicked at src/worker.rs:88:12: +something went terribly wrong in the worker loop +note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace diff --git a/tests/fixtures/oom_killed.log b/tests/fixtures/oom_killed.log new file mode 100644 index 0000000..feab38b --- /dev/null +++ b/tests/fixtures/oom_killed.log @@ -0,0 +1,4 @@ +2026-07-30T15:00:00.000Z INFO indexer: allocating large in-memory buffer +2026-07-30T15:00:00.500Z INFO indexer: still allocating (2 GiB) +2026-07-30T15:00:01.000Z INFO indexer: still allocating (6 GiB) +Killed