From b61cd2390bfbb121810d37e194dd81afde2707db Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Wed, 12 Aug 2026 12:19:50 -0400 Subject: [PATCH] feat(api): migrate experiments to the API client Migrate experiment lookup, base resolution, and comparison reads to the policy-aware API workflow with safe-read retries and typed responses. Resolve https://github.com/braintrustdata/braintrust-sdk-python/issues/639 by retrying transient comparison failures and raising typed API errors after retries are exhausted instead of presenting failures as empty scores. Represent successful and intentionally skipped summaries explicitly. --- py/src/braintrust/api/__init__.py | 14 +- ...ummarize_end_to_end_with_real_backend.yaml | 803 ++++++++++++++++++ ...xperiment_has_no_base_on_real_backend.yaml | 230 +++++ py/src/braintrust/api/experiments.py | 200 ++++- py/src/braintrust/api/test_experiments.py | 451 ++++++++++ py/src/braintrust/framework.py | 6 +- py/src/braintrust/logger.py | 134 +-- py/src/braintrust/test_framework.py | 81 +- .../braintrust/type_tests/test_api_client.py | 17 +- .../type_tests/test_experiment_summary.py | 32 + 10 files changed, 1865 insertions(+), 103 deletions(-) create mode 100644 py/src/braintrust/api/cassettes/test_experiment_summarize_end_to_end_with_real_backend.yaml create mode 100644 py/src/braintrust/api/cassettes/test_fresh_experiment_has_no_base_on_real_backend.yaml create mode 100644 py/src/braintrust/api/test_experiments.py create mode 100644 py/src/braintrust/type_tests/test_experiment_summary.py diff --git a/py/src/braintrust/api/__init__.py b/py/src/braintrust/api/__init__.py index b83591de..1d1cb5db 100644 --- a/py/src/braintrust/api/__init__.py +++ b/py/src/braintrust/api/__init__.py @@ -14,7 +14,14 @@ BraintrustTransportError, BraintrustTransportRetryExhaustedError, ) -from .experiments import ExperimentsAPI +from .experiments import ( + BaseExperiment, + ExperimentComparison, + ExperimentMetric, + ExperimentRecord, + ExperimentsAPI, + ExperimentScore, +) from .functions import FunctionsAPI from .policies import RetryMode, RetryPolicy from .projects import ProjectsAPI @@ -25,6 +32,7 @@ __all__ = [ "AttachmentsAPI", "AuthAPI", + "BaseExperiment", "BraintrustAPIError", "BraintrustClient", "BraintrustHTTPError", @@ -35,6 +43,10 @@ "ClientContext", "DatasetsAPI", "EndpointRouter", + "ExperimentComparison", + "ExperimentMetric", + "ExperimentRecord", + "ExperimentScore", "ExperimentsAPI", "FunctionsAPI", "LoginResult", diff --git a/py/src/braintrust/api/cassettes/test_experiment_summarize_end_to_end_with_real_backend.yaml b/py/src/braintrust/api/cassettes/test_experiment_summarize_end_to_end_with_real_backend.yaml new file mode 100644 index 00000000..d46b09f4 --- /dev/null +++ b/py/src/braintrust/api/cassettes/test_experiment_summarize_end_to_end_with_real_backend.yaml @@ -0,0 +1,803 @@ +interactions: +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '0' + User-Agent: + - python-requests/2.34.2 + method: POST + uri: https://www.braintrust.dev/api/apikey/login + response: + body: + string: '{"org_info":[{"id":"5abfae3a-7aa7-4653-a9c8-b3efcb18f584","name":"Braintrust + SDKs","api_url":"https://api.braintrust.dev","git_metadata":{"collect":"some","fields":["commit","branch","tag","dirty","author_name","author_email","commit_message","commit_time"]},"is_universal_api":null,"proxy_url":"https://api.braintrust.dev","realtime_url":"wss://realtime.braintrustapi.com"}]}' + headers: + Access-Control-Allow-Credentials: + - 'true' + Access-Control-Allow-Headers: + - X-CSRF-Token, X-Requested-With, Accept, Accept-Version, Content-Length, Content-MD5, + Content-Type, Date, X-Api-Version + Access-Control-Allow-Methods: + - GET,OPTIONS,PATCH,DELETE,POST,PUT + Access-Control-Allow-Origin: + - '*' + Cache-Control: + - public, max-age=0, must-revalidate + Content-Length: + - '376' + Content-Security-Policy: + - 'script-src ''self'' ''unsafe-eval'' ''wasm-unsafe-eval'' ''strict-dynamic'' + ''nonce-NmVjNzc0NzktZDU5Zi00ZjllLTllZWItNDU2Zjk4NGJlZGI3'' *.js.stripe.com + js.stripe.com maps.googleapis.com ; style-src ''self'' ''unsafe-inline'' *.braintrust.dev + btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com + d4tuoctqmanu0.cloudfront.net; font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com + cdn.jsdelivr.net d4tuoctqmanu0.cloudfront.net fonts.googleapis.com mintlify-assets.b-cdn.net + fonts.cdnfonts.com; object-src ''none''; base-uri ''self''; form-action ''self'' + https://www.facebook.com; frame-ancestors ''self''; worker-src ''self'' blob:; + report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16; + report-to csp-endpoint-0' + Content-Type: + - application/json; charset=utf-8 + Date: + - Wed, 12 Aug 2026 15:17:30 GMT + Etag: + - '"13vsc5ye8flag"' + Reporting-Endpoints: + - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16" + Server: + - Vercel + Strict-Transport-Security: + - max-age=63072000 + X-Bt-Was-Udf-Cached: + - 'true' + X-Clerk-Auth-Message: + - Invalid JWT form. A JWT consists of three parts separated by dots. (reason=token-invalid, + token-carrier=header) + X-Clerk-Auth-Reason: + - token-invalid + X-Clerk-Auth-Status: + - signed-out + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-Matched-Path: + - /api/apikey/login + X-Nonce: + - NmVjNzc0NzktZDU5Zi00ZjllLTllZWItNDU2Zjk4NGJlZGI3 + X-Vercel-Cache: + - MISS + X-Vercel-Id: + - yul1::iad1::rzj66-1786547850779-7d0ce895f395 + status: + code: 200 + message: OK +- request: + body: '{"project_name": "python-sdk-api-experiment-service-vcr", "project_id": + null, "org_id": "5abfae3a-7aa7-4653-a9c8-b3efcb18f584", "update": true, "experiment_name": + "experiment-service-base", "repo_info": {"commit": null, "branch": null, "tag": + null, "dirty": null, "author_name": null, "author_email": null, "commit_message": + null, "commit_time": null, "git_diff": null}, "public": false}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '387' + Content-Type: + - application/json + User-Agent: + - python-requests/2.34.2 + method: POST + uri: https://www.braintrust.dev/api/experiment/register + response: + body: + string: '{"project":{"id":"4910159a-7461-43d1-8ce4-7d21a7209ec6","org_id":"5abfae3a-7aa7-4653-a9c8-b3efcb18f584","name":"python-sdk-api-experiment-service-vcr","description":null,"created":"2026-08-12T15:15:05.921Z","deleted_at":null,"user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","settings":null},"experiment":{"id":"6dbe9fca-00ad-4160-9513-21255ca21133","project_id":"4910159a-7461-43d1-8ce4-7d21a7209ec6","name":"experiment-service-base","description":null,"created":"2026-08-12T15:15:05.921Z","repo_info":{},"commit":"abc202c350c7f6f4d6b85e128c9bb3a1013517a8","base_exp_id":null,"deleted_at":null,"dataset_id":null,"dataset_version":null,"internal_metadata":null,"parameters_id":null,"parameters_version":null,"public":false,"user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","metadata":null,"tags":null}}' + headers: + Cache-Control: + - public, max-age=0, must-revalidate + Content-Security-Policy: + - 'script-src ''self'' ''unsafe-eval'' ''wasm-unsafe-eval'' ''strict-dynamic'' + ''nonce-ZGEyYTk0YTktNzE5YS00ODE2LTgwNzctZjcxOGQ0N2NkMmMz'' *.js.stripe.com + js.stripe.com maps.googleapis.com ; style-src ''self'' ''unsafe-inline'' *.braintrust.dev + btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com + d4tuoctqmanu0.cloudfront.net; font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com + cdn.jsdelivr.net d4tuoctqmanu0.cloudfront.net fonts.googleapis.com mintlify-assets.b-cdn.net + fonts.cdnfonts.com; object-src ''none''; base-uri ''self''; form-action ''self'' + https://www.facebook.com; frame-ancestors ''self''; worker-src ''self'' blob:; + report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16; + report-to csp-endpoint-0' + Content-Type: + - application/json; charset=utf-8 + Date: + - Wed, 12 Aug 2026 15:17:31 GMT + Etag: + - W/"ojuc6bokebyv" + Reporting-Endpoints: + - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16" + Server: + - Vercel + Strict-Transport-Security: + - max-age=63072000 + Transfer-Encoding: + - chunked + X-Clerk-Auth-Message: + - Invalid JWT form. A JWT consists of three parts separated by dots. (reason=token-invalid, + token-carrier=header) + X-Clerk-Auth-Reason: + - token-invalid + X-Clerk-Auth-Status: + - signed-out + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-Matched-Path: + - /api/experiment/register + X-Nonce: + - ZGEyYTk0YTktNzE5YS00ODE2LTgwNzctZjcxOGQ0N2NkMmMz + X-Vercel-Cache: + - MISS + X-Vercel-Id: + - yul1::iad1::9sdpq-1786547850953-34c34345069a + content-length: + - '1255' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + User-Agent: + - python-requests/2.34.2 + method: GET + uri: https://api.braintrust.dev/version + response: + body: + string: '{"version":"2.10.0","date_version":"20260811","ff_version":45,"commit":"6fcbc7d3aff9ad4a8a4a6cdc3c7baab433f34deb","deployment_mode":"lambda","deployment_type":"custom","brainstore_default":"force","brainstore_can_contain_row_refs":true,"skip_pg_config":"all","has_realtime_wal_bucket":true,"brainstore_wal_footer_version":"v3","brainstore_wal_use_efficient_format":true,"has_logs2":true,"brainstore_export_enabled":true,"js":true,"universal":true,"code_execution":true,"logs3_payload_max_bytes":5242880,"control_plane_telemetry":["status","memprof","usage"]}' + headers: + Connection: + - keep-alive + Content-Type: + - application/json; charset=utf-8 + Date: + - Wed, 12 Aug 2026 15:17:31 GMT + Via: + - 1.1 e7e881849322d751aeeb9605914b08b4.cloudfront.net (CloudFront), 1.1 cdd327922be1fd75b18f2ae0982269cc.cloudfront.net + (CloudFront) + X-Amz-Cf-Id: + - dq0kqPbWgNFkrZwflsMlK3Wf8JRXceqkizmm2-pjBewdnvSIjEGQpw== + X-Amz-Cf-Pop: + - YTO53-P2 + - YTO50-P2 + X-Amzn-Trace-Id: + - Root=1-6a7c8e8b-7d365f0b1c315ee921b19e93;Parent=69775a51ae54f025;Sampled=0;Lineage=1:24be3d11:0 + X-Cache: + - Miss from cloudfront + access-control-allow-credentials: + - 'true' + access-control-expose-headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + cache-control: + - no-store, no-cache, must-revalidate, proxy-revalidate + content-length: + - '558' + etag: + - W/"22e-B3ODPz4iWAYw9Vfpk+Xl6ij3Ax4" + expires: + - '0' + surrogate-control: + - no-store + vary: + - Origin + x-amz-apigw-id: + - B_s11He-IAMElXg= + x-amzn-Remapped-content-length: + - '558' + x-amzn-RequestId: + - 2f9d9e6b-70dc-4c31-b3b9-6bb780f5e30c + x-bt-internal-trace-id: + - 6a7c8e8b0000000073b0338d7f7ab304 + status: + code: 200 + message: OK +- request: + body: '{"api_version":2,"rows":[{"expected":"4","experiment_id":"6dbe9fca-00ad-4160-9513-21255ca21133","id":"experiment-service-base-row","input":{"question":"What + is 2 + 2?"},"metrics":{},"output":"4","scores":{"exact_match":0.5},"span_attributes":{"exec_counter":1,"name":"root","type":"eval"},"span_parents":null}]}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '914' + User-Agent: + - python-requests/2.34.2 + method: POST + uri: https://api.braintrust.dev/logs3 + response: + body: + string: '{"ids":["experiment-service-base-row"],"xact_id":"1000197675452973914"}' + headers: + Connection: + - keep-alive + Content-Type: + - application/json; charset=utf-8 + Date: + - Wed, 12 Aug 2026 15:17:31 GMT + Via: + - 1.1 e7e881849322d751aeeb9605914b08b4.cloudfront.net (CloudFront), 1.1 5e2f1ed3ba0ab1e08304bb3d134360de.cloudfront.net + (CloudFront) + X-Amz-Cf-Id: + - dq10lBdAwQPoSolTMa0XK0zEiOC4Ty-ebcpbHYb2F4-kJ_M9KciZTQ== + X-Amz-Cf-Pop: + - YTO53-P2 + - YTO50-P2 + X-Amzn-Trace-Id: + - Root=1-6a7c8e8b-079abe9b3adbb8f105a7415b;Parent=4e63f9b08c7bd1c1;Sampled=0;Lineage=1:24be3d11:0 + X-Cache: + - Miss from cloudfront + access-control-allow-credentials: + - 'true' + access-control-expose-headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + cache-control: + - no-store, no-cache, must-revalidate, proxy-revalidate + content-length: + - '71' + etag: + - W/"47-Om0+sCAZNEnWGncBjWJZkQnVbRU" + expires: + - '0' + surrogate-control: + - no-store + vary: + - Origin, Accept-Encoding + x-amz-apigw-id: + - B_s13HZ_oAMEctA= + x-amzn-RequestId: + - 8fc6ca7e-2cce-4bb9-b512-5262a7db4c91 + x-bt-internal-trace-id: + - 6a7c8e8b000000005466ab68b9ac6be6 + status: + code: 200 + message: OK +- request: + body: '{"project_name": "python-sdk-api-experiment-service-vcr", "project_id": + null, "org_id": "5abfae3a-7aa7-4653-a9c8-b3efcb18f584", "update": true, "experiment_name": + "experiment-service-candidate", "repo_info": {"commit": null, "branch": null, + "tag": null, "dirty": null, "author_name": null, "author_email": null, "commit_message": + null, "commit_time": null, "git_diff": null}, "base_exp_id": "6dbe9fca-00ad-4160-9513-21255ca21133", + "public": false}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '447' + Content-Type: + - application/json + User-Agent: + - python-requests/2.34.2 + method: POST + uri: https://www.braintrust.dev/api/experiment/register + response: + body: + string: '{"project":{"id":"4910159a-7461-43d1-8ce4-7d21a7209ec6","org_id":"5abfae3a-7aa7-4653-a9c8-b3efcb18f584","name":"python-sdk-api-experiment-service-vcr","description":null,"created":"2026-08-12T15:15:05.921Z","deleted_at":null,"user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","settings":null},"experiment":{"id":"f7304a70-c5c0-48f3-ba9a-cebf9d502475","project_id":"4910159a-7461-43d1-8ce4-7d21a7209ec6","name":"experiment-service-candidate","description":null,"created":"2026-08-12T15:15:06.993Z","repo_info":{},"commit":"abc202c350c7f6f4d6b85e128c9bb3a1013517a8","base_exp_id":"6dbe9fca-00ad-4160-9513-21255ca21133","deleted_at":null,"dataset_id":null,"dataset_version":null,"internal_metadata":null,"parameters_id":null,"parameters_version":null,"public":false,"user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","metadata":null,"tags":null}}' + headers: + Cache-Control: + - public, max-age=0, must-revalidate + Content-Security-Policy: + - 'script-src ''self'' ''unsafe-eval'' ''wasm-unsafe-eval'' ''strict-dynamic'' + ''nonce-ZTZmYWNlNTEtNjRjOS00NzIyLTk0YTMtNWYxNjgxMzY5YzUy'' *.js.stripe.com + js.stripe.com maps.googleapis.com ; style-src ''self'' ''unsafe-inline'' *.braintrust.dev + btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com + d4tuoctqmanu0.cloudfront.net; font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com + cdn.jsdelivr.net d4tuoctqmanu0.cloudfront.net fonts.googleapis.com mintlify-assets.b-cdn.net + fonts.cdnfonts.com; object-src ''none''; base-uri ''self''; form-action ''self'' + https://www.facebook.com; frame-ancestors ''self''; worker-src ''self'' blob:; + report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16; + report-to csp-endpoint-0' + Content-Type: + - application/json; charset=utf-8 + Date: + - Wed, 12 Aug 2026 15:17:31 GMT + Etag: + - W/"32zcksmh8tzy" + Reporting-Endpoints: + - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16" + Server: + - Vercel + Strict-Transport-Security: + - max-age=63072000 + Transfer-Encoding: + - chunked + X-Clerk-Auth-Message: + - Invalid JWT form. A JWT consists of three parts separated by dots. (reason=token-invalid, + token-carrier=header) + X-Clerk-Auth-Reason: + - token-invalid + X-Clerk-Auth-Status: + - signed-out + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-Matched-Path: + - /api/experiment/register + X-Nonce: + - ZTZmYWNlNTEtNjRjOS00NzIyLTk0YTMtNWYxNjgxMzY5YzUy + X-Vercel-Cache: + - MISS + X-Vercel-Id: + - yul1::iad1::74jch-1786547851644-0c530630d7d9 + content-length: + - '1294' + status: + code: 200 + message: OK +- request: + body: '{"api_version":2,"rows":[{"expected":"4","experiment_id":"f7304a70-c5c0-48f3-ba9a-cebf9d502475","id":"experiment-service-candidate-row","input":{"question":"What + is 2 + 2?"},"metrics":{},"output":"4","scores":{"exact_match":1.0},"span_attributes":{"exec_counter":2,"name":"root","type":"eval"},"span_parents":null}]}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '918' + User-Agent: + - python-requests/2.34.2 + method: POST + uri: https://api.braintrust.dev/logs3 + response: + body: + string: '{"ids":["experiment-service-candidate-row"],"xact_id":"1000197675452976388"}' + headers: + Connection: + - keep-alive + Content-Type: + - application/json; charset=utf-8 + Date: + - Wed, 12 Aug 2026 15:17:32 GMT + Via: + - 1.1 e7e881849322d751aeeb9605914b08b4.cloudfront.net (CloudFront), 1.1 21c66eb5f493a6e3ddbaa803cebfe014.cloudfront.net + (CloudFront) + X-Amz-Cf-Id: + - WMmBNpSLupuFHcOUihBCtfEPdbNCdDokeU2cI_bfxBfpMKp16-sZgg== + X-Amz-Cf-Pop: + - YTO53-P2 + - YTO50-P2 + X-Amzn-Trace-Id: + - Root=1-6a7c8e8b-040b7931598fd358516e4ada;Parent=4299680d71f1c0a9;Sampled=0;Lineage=1:24be3d11:0 + X-Cache: + - Miss from cloudfront + access-control-allow-credentials: + - 'true' + access-control-expose-headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + cache-control: + - no-store, no-cache, must-revalidate, proxy-revalidate + content-length: + - '76' + etag: + - W/"4c-PYAx5sirQbACL80Kg9eG49ioeV4" + expires: + - '0' + surrogate-control: + - no-store + vary: + - Origin, Accept-Encoding + x-amz-apigw-id: + - B_s17HlDIAMEjRw= + x-amzn-RequestId: + - 5d14b41e-2c4f-461b-af7c-298d14e31f8f + x-bt-internal-trace-id: + - 6a7c8e8b000000003d9a57fbc8ad91e7 + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + User-Agent: + - python-requests/2.34.2 + method: GET + uri: https://api.braintrust.dev/v1/experiment/6dbe9fca-00ad-4160-9513-21255ca21133 + response: + body: + string: '{"id":"6dbe9fca-00ad-4160-9513-21255ca21133","project_id":"4910159a-7461-43d1-8ce4-7d21a7209ec6","name":"experiment-service-base","description":null,"created":"2026-08-12T15:15:05.921Z","repo_info":{},"commit":"abc202c350c7f6f4d6b85e128c9bb3a1013517a8","base_exp_id":null,"deleted_at":null,"dataset_id":null,"dataset_version":null,"internal_metadata":null,"parameters_id":null,"parameters_version":null,"public":false,"user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","metadata":null,"tags":null}' + headers: + Connection: + - keep-alive + Content-Type: + - application/json; charset=utf-8 + Date: + - Wed, 12 Aug 2026 15:17:32 GMT + Via: + - 1.1 e7e881849322d751aeeb9605914b08b4.cloudfront.net (CloudFront), 1.1 5e2f1ed3ba0ab1e08304bb3d134360de.cloudfront.net + (CloudFront) + X-Amz-Cf-Id: + - keTOS2fVBp59HnH36QYFK6fGTAZi6ojp4pf_DFeUe0ZcLFxvQfUZXQ== + X-Amz-Cf-Pop: + - YTO53-P2 + - YTO50-P2 + X-Amzn-Trace-Id: + - Root=1-6a7c8e8c-0af69e79609ced955e920b3c;Parent=001104f446e07bbd;Sampled=0;Lineage=1:24be3d11:0 + X-Cache: + - Miss from cloudfront + access-control-allow-credentials: + - 'true' + access-control-expose-headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + cache-control: + - no-store, no-cache, must-revalidate, proxy-revalidate + content-length: + - '950' + etag: + - W/"3b6-Q1FIgFUac3LX+IYqIweYEFXscVg" + expires: + - '0' + surrogate-control: + - no-store + vary: + - Origin, Accept-Encoding + x-amz-apigw-id: + - B_s1-FSjIAMEGfw= + x-amzn-RequestId: + - 11cae360-a8ba-4895-8fb6-e362ecedb12d + x-bt-internal-trace-id: + - 6a7c8e8c000000007281ce7b01b65998 + status: + code: 200 + message: OK +- request: + body: '{"id": "f7304a70-c5c0-48f3-ba9a-cebf9d502475"}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '46' + Content-Type: + - application/json + User-Agent: + - python-requests/2.34.2 + method: POST + uri: https://www.braintrust.dev/api/base_experiment/get_id + response: + body: + string: '{"id":"f7304a70-c5c0-48f3-ba9a-cebf9d502475","project_id":"4910159a-7461-43d1-8ce4-7d21a7209ec6","name":"experiment-service-candidate","base_exp_id":"6dbe9fca-00ad-4160-9513-21255ca21133","base_exp_name":"experiment-service-base"}' + headers: + Cache-Control: + - public, max-age=0, must-revalidate + Content-Length: + - '230' + Content-Security-Policy: + - 'script-src ''self'' ''unsafe-eval'' ''wasm-unsafe-eval'' ''strict-dynamic'' + ''nonce-ODI1ZGQ0NDMtY2NiOC00NGM3LTgxZTktNzRhZTAyOTdlYjA4'' *.js.stripe.com + js.stripe.com maps.googleapis.com ; style-src ''self'' ''unsafe-inline'' *.braintrust.dev + btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com + d4tuoctqmanu0.cloudfront.net; font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com + cdn.jsdelivr.net d4tuoctqmanu0.cloudfront.net fonts.googleapis.com mintlify-assets.b-cdn.net + fonts.cdnfonts.com; object-src ''none''; base-uri ''self''; form-action ''self'' + https://www.facebook.com; frame-ancestors ''self''; worker-src ''self'' blob:; + report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16; + report-to csp-endpoint-0' + Content-Type: + - application/json; charset=utf-8 + Date: + - Wed, 12 Aug 2026 15:17:32 GMT + Etag: + - '"15lpwnw5v2t6e"' + Reporting-Endpoints: + - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16" + Server: + - Vercel + Strict-Transport-Security: + - max-age=63072000 + X-Clerk-Auth-Message: + - Invalid JWT form. A JWT consists of three parts separated by dots. (reason=token-invalid, + token-carrier=header) + X-Clerk-Auth-Reason: + - token-invalid + X-Clerk-Auth-Status: + - signed-out + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-Matched-Path: + - /api/base_experiment/get_id + X-Nonce: + - ODI1ZGQ0NDMtY2NiOC00NGM3LTgxZTktNzRhZTAyOTdlYjA4 + X-Vercel-Cache: + - MISS + X-Vercel-Id: + - yul1::iad1::5kfhb-1786547852344-d356296cacf4 + status: + code: 200 + message: OK +- request: + body: '{"id": "f7304a70-c5c0-48f3-ba9a-cebf9d502475"}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '46' + Content-Type: + - application/json + User-Agent: + - python-requests/2.34.2 + method: POST + uri: https://www.braintrust.dev/api/base_experiment/get_id + response: + body: + string: '{"id":"f7304a70-c5c0-48f3-ba9a-cebf9d502475","project_id":"4910159a-7461-43d1-8ce4-7d21a7209ec6","name":"experiment-service-candidate","base_exp_id":"6dbe9fca-00ad-4160-9513-21255ca21133","base_exp_name":"experiment-service-base"}' + headers: + Cache-Control: + - public, max-age=0, must-revalidate + Content-Length: + - '230' + Content-Security-Policy: + - 'script-src ''self'' ''unsafe-eval'' ''wasm-unsafe-eval'' ''strict-dynamic'' + ''nonce-NjA3YjI3NTItOWIwNC00OTg3LTg3ODEtNjUzNWVjYzExMGEy'' *.js.stripe.com + js.stripe.com maps.googleapis.com ; style-src ''self'' ''unsafe-inline'' *.braintrust.dev + btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com + d4tuoctqmanu0.cloudfront.net; font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com + cdn.jsdelivr.net d4tuoctqmanu0.cloudfront.net fonts.googleapis.com mintlify-assets.b-cdn.net + fonts.cdnfonts.com; object-src ''none''; base-uri ''self''; form-action ''self'' + https://www.facebook.com; frame-ancestors ''self''; worker-src ''self'' blob:; + report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16; + report-to csp-endpoint-0' + Content-Type: + - application/json; charset=utf-8 + Date: + - Wed, 12 Aug 2026 15:17:33 GMT + Etag: + - '"15lpwnw5v2t6e"' + Reporting-Endpoints: + - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16" + Server: + - Vercel + Strict-Transport-Security: + - max-age=63072000 + X-Clerk-Auth-Message: + - Invalid JWT form. A JWT consists of three parts separated by dots. (reason=token-invalid, + token-carrier=header) + X-Clerk-Auth-Reason: + - token-invalid + X-Clerk-Auth-Status: + - signed-out + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-Matched-Path: + - /api/base_experiment/get_id + X-Nonce: + - NjA3YjI3NTItOWIwNC00OTg3LTg3ODEtNjUzNWVjYzExMGEy + X-Vercel-Cache: + - MISS + X-Vercel-Id: + - yul1::iad1::w47ts-1786547852858-a6b009a13441 + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + User-Agent: + - python-requests/2.34.2 + method: GET + uri: https://api.braintrust.dev/experiment-comparison2?experiment_id=f7304a70-c5c0-48f3-ba9a-cebf9d502475&base_experiment_id=6dbe9fca-00ad-4160-9513-21255ca21133 + response: + body: + string: '{"scores":{"exact_match":{"name":"exact_match","score":1,"diff":0.5,"improvements":1,"regressions":0}},"metrics":{"llm_calls":{"name":"llm_calls","metric":0,"unit":"","diff":0,"improvements":0,"regressions":0},"tool_calls":{"name":"tool_calls","metric":0,"unit":"","diff":0,"improvements":0,"regressions":0},"errors":{"name":"errors","metric":0,"unit":"","diff":0,"improvements":0,"regressions":0},"llm_errors":{"name":"llm_errors","metric":0,"unit":"","diff":0,"improvements":0,"regressions":0},"tool_errors":{"name":"tool_errors","metric":0,"unit":"","diff":0,"improvements":0,"regressions":0},"prompt_tokens":{"name":"prompt_tokens","metric":0,"unit":"tok","diff":0,"improvements":0,"regressions":0},"prompt_cached_tokens":{"name":"prompt_cached_tokens","metric":0,"unit":"tok","diff":0,"improvements":0,"regressions":0},"prompt_cache_creation_tokens":{"name":"prompt_cache_creation_tokens","metric":0,"unit":"tok","diff":0,"improvements":0,"regressions":0},"prompt_cache_creation_5m_tokens":{"name":"prompt_cache_creation_5m_tokens","metric":0,"unit":"tok","diff":0,"improvements":0,"regressions":0},"prompt_cache_creation_1h_tokens":{"name":"prompt_cache_creation_1h_tokens","metric":0,"unit":"tok","diff":0,"improvements":0,"regressions":0},"completion_tokens":{"name":"completion_tokens","metric":0,"unit":"tok","diff":0,"improvements":0,"regressions":0},"completion_reasoning_tokens":{"name":"completion_reasoning_tokens","metric":0,"unit":"tok","diff":0,"improvements":0,"regressions":0},"total_tokens":{"name":"total_tokens","metric":0,"unit":"tok","diff":0,"improvements":0,"regressions":0},"duration":{"name":"duration","metric":0.2307281494140625,"unit":"s","diff":-0.37787771224975586,"improvements":1,"regressions":0}}}' + headers: + Connection: + - keep-alive + Content-Type: + - application/json; charset=utf-8 + Date: + - Wed, 12 Aug 2026 15:17:35 GMT + Via: + - 1.1 e7dbdec0a9983cf82c16b332b6b30812.cloudfront.net (CloudFront), 1.1 71eaa9eb77c2eecb57c03cdcdad1cf76.cloudfront.net + (CloudFront) + X-Amz-Cf-Id: + - S5xBYuzdwP8iKpli_p7plAwq1jF0WHcqwEe2_OPNSKFmiUHict6J_A== + X-Amz-Cf-Pop: + - YTO53-P2 + - YTO50-P2 + X-Amzn-Trace-Id: + - Root=1-6a7c8e8d-67d700882911cc554e8b8a38;Parent=1bb003f3ff79e2c2;Sampled=0;Lineage=1:24be3d11:0 + X-Cache: + - Miss from cloudfront + access-control-allow-credentials: + - 'true' + access-control-expose-headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + cache-control: + - no-store, no-cache, must-revalidate, proxy-revalidate + content-length: + - '1734' + etag: + - W/"6c6-3WYJ9rsgEIZKBfB09DmwSO64VXA" + expires: + - '0' + surrogate-control: + - no-store + vary: + - Origin, Accept-Encoding + x-amz-apigw-id: + - B_s2LEGNIAMEXxw= + x-amzn-RequestId: + - 44c3cacd-2b9b-43c0-89c5-c78857117f16 + x-bt-internal-trace-id: + - 6a7c8e8d0000000015490499bfe4311a + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + User-Agent: + - python-requests/2.34.2 + method: GET + uri: https://api.braintrust.dev/v1/experiment/6dbe9fca-00ad-4160-9513-21255ca21133 + response: + body: + string: '{"id":"6dbe9fca-00ad-4160-9513-21255ca21133","project_id":"4910159a-7461-43d1-8ce4-7d21a7209ec6","name":"experiment-service-base","description":null,"created":"2026-08-12T15:15:05.921Z","repo_info":{},"commit":"abc202c350c7f6f4d6b85e128c9bb3a1013517a8","base_exp_id":null,"deleted_at":null,"dataset_id":null,"dataset_version":null,"internal_metadata":null,"parameters_id":null,"parameters_version":null,"public":false,"user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","metadata":null,"tags":null}' + headers: + Connection: + - keep-alive + Content-Type: + - application/json; charset=utf-8 + Date: + - Wed, 12 Aug 2026 15:17:35 GMT + Via: + - 1.1 a9372ab148a2984b992fb2eac84148d0.cloudfront.net (CloudFront), 1.1 12aa3fefbdb5e80269e58f34f94a99e8.cloudfront.net + (CloudFront) + X-Amz-Cf-Id: + - 1nCY8Qn-2co_kOadh0Qc6S6XVqteDY8DnyMFaaNu-n7zmNiIBRBsfQ== + X-Amz-Cf-Pop: + - YTO53-P2 + - YTO50-P2 + X-Amzn-Trace-Id: + - Root=1-6a7c8e8f-5a7c511c198ee4bf5aac8a81;Parent=73bef7a4990e05df;Sampled=0;Lineage=1:24be3d11:0 + X-Cache: + - Miss from cloudfront + access-control-allow-credentials: + - 'true' + access-control-expose-headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + cache-control: + - no-store, no-cache, must-revalidate, proxy-revalidate + content-length: + - '950' + etag: + - W/"3b6-Q1FIgFUac3LX+IYqIweYEFXscVg" + expires: + - '0' + surrogate-control: + - no-store + vary: + - Origin, Accept-Encoding + x-amz-apigw-id: + - B_s2iGZgIAMEM9w= + x-amzn-RequestId: + - 3e790d83-0497-4262-85fe-c999cb7e1f35 + x-bt-internal-trace-id: + - 6a7c8e8f000000000f1a787baa9f5b33 + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + User-Agent: + - python-requests/2.34.2 + method: GET + uri: https://api.braintrust.dev/experiment-comparison2?experiment_id=f7304a70-c5c0-48f3-ba9a-cebf9d502475&base_experiment_id=6dbe9fca-00ad-4160-9513-21255ca21133 + response: + body: + string: '{"scores":{"exact_match":{"name":"exact_match","score":1,"diff":0.5,"improvements":1,"regressions":0}},"metrics":{"llm_calls":{"name":"llm_calls","metric":0,"unit":"","diff":0,"improvements":0,"regressions":0},"tool_calls":{"name":"tool_calls","metric":0,"unit":"","diff":0,"improvements":0,"regressions":0},"errors":{"name":"errors","metric":0,"unit":"","diff":0,"improvements":0,"regressions":0},"llm_errors":{"name":"llm_errors","metric":0,"unit":"","diff":0,"improvements":0,"regressions":0},"tool_errors":{"name":"tool_errors","metric":0,"unit":"","diff":0,"improvements":0,"regressions":0},"prompt_tokens":{"name":"prompt_tokens","metric":0,"unit":"tok","diff":0,"improvements":0,"regressions":0},"prompt_cached_tokens":{"name":"prompt_cached_tokens","metric":0,"unit":"tok","diff":0,"improvements":0,"regressions":0},"prompt_cache_creation_tokens":{"name":"prompt_cache_creation_tokens","metric":0,"unit":"tok","diff":0,"improvements":0,"regressions":0},"prompt_cache_creation_5m_tokens":{"name":"prompt_cache_creation_5m_tokens","metric":0,"unit":"tok","diff":0,"improvements":0,"regressions":0},"prompt_cache_creation_1h_tokens":{"name":"prompt_cache_creation_1h_tokens","metric":0,"unit":"tok","diff":0,"improvements":0,"regressions":0},"completion_tokens":{"name":"completion_tokens","metric":0,"unit":"tok","diff":0,"improvements":0,"regressions":0},"completion_reasoning_tokens":{"name":"completion_reasoning_tokens","metric":0,"unit":"tok","diff":0,"improvements":0,"regressions":0},"total_tokens":{"name":"total_tokens","metric":0,"unit":"tok","diff":0,"improvements":0,"regressions":0},"duration":{"name":"duration","metric":0.2307281494140625,"unit":"s","diff":-0.37787771224975586,"improvements":1,"regressions":0}}}' + headers: + Connection: + - keep-alive + Content-Type: + - application/json; charset=utf-8 + Date: + - Wed, 12 Aug 2026 15:17:38 GMT + Via: + - 1.1 a9372ab148a2984b992fb2eac84148d0.cloudfront.net (CloudFront), 1.1 16808c837fedc33331e77d172952efee.cloudfront.net + (CloudFront) + X-Amz-Cf-Id: + - GhgClZKw2D-fjY5kdTEe3PUMh9s7FM-AozyIpcuSKVy8_BxOTWx58Q== + X-Amz-Cf-Pop: + - YTO53-P2 + - YTO50-P2 + X-Amzn-Trace-Id: + - Root=1-6a7c8e90-27d5a540207370501859e150;Parent=528cdf1b61cc1605;Sampled=0;Lineage=1:24be3d11:0 + X-Cache: + - Miss from cloudfront + access-control-allow-credentials: + - 'true' + access-control-expose-headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + cache-control: + - no-store, no-cache, must-revalidate, proxy-revalidate + content-length: + - '1734' + etag: + - W/"6c6-3WYJ9rsgEIZKBfB09DmwSO64VXA" + expires: + - '0' + surrogate-control: + - no-store + vary: + - Origin, Accept-Encoding + x-amz-apigw-id: + - B_s2lERnIAMEfIw= + x-amzn-RequestId: + - 3905ec46-0177-492d-a48d-baaf004f589b + x-bt-internal-trace-id: + - 6a7c8e9000000000719def5f92cdac89 + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/api/cassettes/test_fresh_experiment_has_no_base_on_real_backend.yaml b/py/src/braintrust/api/cassettes/test_fresh_experiment_has_no_base_on_real_backend.yaml new file mode 100644 index 00000000..6dbcb0c1 --- /dev/null +++ b/py/src/braintrust/api/cassettes/test_fresh_experiment_has_no_base_on_real_backend.yaml @@ -0,0 +1,230 @@ +interactions: +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '0' + User-Agent: + - python-requests/2.34.2 + method: POST + uri: https://www.braintrust.dev/api/apikey/login + response: + body: + string: '{"org_info":[{"id":"5abfae3a-7aa7-4653-a9c8-b3efcb18f584","name":"Braintrust + SDKs","api_url":"https://api.braintrust.dev","git_metadata":{"collect":"some","fields":["commit","branch","tag","dirty","author_name","author_email","commit_message","commit_time"]},"is_universal_api":null,"proxy_url":"https://api.braintrust.dev","realtime_url":"wss://realtime.braintrustapi.com"}]}' + headers: + Access-Control-Allow-Credentials: + - 'true' + Access-Control-Allow-Headers: + - X-CSRF-Token, X-Requested-With, Accept, Accept-Version, Content-Length, Content-MD5, + Content-Type, Date, X-Api-Version + Access-Control-Allow-Methods: + - GET,OPTIONS,PATCH,DELETE,POST,PUT + Access-Control-Allow-Origin: + - '*' + Cache-Control: + - public, max-age=0, must-revalidate + Content-Length: + - '376' + Content-Security-Policy: + - 'script-src ''self'' ''unsafe-eval'' ''wasm-unsafe-eval'' ''strict-dynamic'' + ''nonce-ZTAyYmVhODEtNDU1Mi00ZTAzLWIxMTYtMDIyNmY3MTU1ZDY0'' *.js.stripe.com + js.stripe.com maps.googleapis.com ; style-src ''self'' ''unsafe-inline'' *.braintrust.dev + btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com + d4tuoctqmanu0.cloudfront.net; font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com + cdn.jsdelivr.net d4tuoctqmanu0.cloudfront.net fonts.googleapis.com mintlify-assets.b-cdn.net + fonts.cdnfonts.com; object-src ''none''; base-uri ''self''; form-action ''self'' + https://www.facebook.com; frame-ancestors ''self''; worker-src ''self'' blob:; + report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16; + report-to csp-endpoint-0' + Content-Type: + - application/json; charset=utf-8 + Date: + - Wed, 12 Aug 2026 15:17:38 GMT + Etag: + - '"13vsc5ye8flag"' + Reporting-Endpoints: + - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16" + Server: + - Vercel + Strict-Transport-Security: + - max-age=63072000 + X-Bt-Was-Udf-Cached: + - 'true' + X-Clerk-Auth-Message: + - Invalid JWT form. A JWT consists of three parts separated by dots. (reason=token-invalid, + token-carrier=header) + X-Clerk-Auth-Reason: + - token-invalid + X-Clerk-Auth-Status: + - signed-out + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-Matched-Path: + - /api/apikey/login + X-Nonce: + - ZTAyYmVhODEtNDU1Mi00ZTAzLWIxMTYtMDIyNmY3MTU1ZDY0 + X-Vercel-Cache: + - MISS + X-Vercel-Id: + - yul1::iad1::vrzs6-1786547858136-19160ee5c180 + status: + code: 200 + message: OK +- request: + body: '{"project_name": "python-sdk-api-experiment-service-fresh-vcr", "project_id": + null, "org_id": "5abfae3a-7aa7-4653-a9c8-b3efcb18f584", "update": true, "experiment_name": + "fresh-experiment-without-base", "repo_info": {"commit": null, "branch": null, + "tag": null, "dirty": null, "author_name": null, "author_email": null, "commit_message": + null, "commit_time": null, "git_diff": null}, "public": false}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '399' + Content-Type: + - application/json + User-Agent: + - python-requests/2.34.2 + method: POST + uri: https://www.braintrust.dev/api/experiment/register + response: + body: + string: '{"project":{"id":"eb1d7d8e-9088-4937-9543-58f894306ac7","org_id":"5abfae3a-7aa7-4653-a9c8-b3efcb18f584","name":"python-sdk-api-experiment-service-fresh-vcr","description":null,"created":"2026-08-12T15:14:54.955Z","deleted_at":null,"user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","settings":null},"experiment":{"id":"cbe48deb-473d-498f-af94-facb1c90ac68","project_id":"eb1d7d8e-9088-4937-9543-58f894306ac7","name":"fresh-experiment-without-base","description":null,"created":"2026-08-12T15:14:54.955Z","repo_info":{},"commit":"abc202c350c7f6f4d6b85e128c9bb3a1013517a8","base_exp_id":null,"deleted_at":null,"dataset_id":null,"dataset_version":null,"internal_metadata":null,"parameters_id":null,"parameters_version":null,"public":false,"user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","metadata":null,"tags":null}}' + headers: + Cache-Control: + - public, max-age=0, must-revalidate + Content-Security-Policy: + - 'script-src ''self'' ''unsafe-eval'' ''wasm-unsafe-eval'' ''strict-dynamic'' + ''nonce-OTA2ZDgxMjMtZTVhNC00NmMzLWFlYTAtYzkyYmMyYWVkYWI1'' *.js.stripe.com + js.stripe.com maps.googleapis.com ; style-src ''self'' ''unsafe-inline'' *.braintrust.dev + btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com + d4tuoctqmanu0.cloudfront.net; font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com + cdn.jsdelivr.net d4tuoctqmanu0.cloudfront.net fonts.googleapis.com mintlify-assets.b-cdn.net + fonts.cdnfonts.com; object-src ''none''; base-uri ''self''; form-action ''self'' + https://www.facebook.com; frame-ancestors ''self''; worker-src ''self'' blob:; + report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16; + report-to csp-endpoint-0' + Content-Type: + - application/json; charset=utf-8 + Date: + - Wed, 12 Aug 2026 15:17:38 GMT + Etag: + - W/"12h4qcgnnqaz7" + Reporting-Endpoints: + - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16" + Server: + - Vercel + Strict-Transport-Security: + - max-age=63072000 + Transfer-Encoding: + - chunked + X-Clerk-Auth-Message: + - Invalid JWT form. A JWT consists of three parts separated by dots. (reason=token-invalid, + token-carrier=header) + X-Clerk-Auth-Reason: + - token-invalid + X-Clerk-Auth-Status: + - signed-out + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-Matched-Path: + - /api/experiment/register + X-Nonce: + - OTA2ZDgxMjMtZTVhNC00NmMzLWFlYTAtYzkyYmMyYWVkYWI1 + X-Vercel-Cache: + - MISS + X-Vercel-Id: + - yul1::iad1::v8n4z-1786547858299-2a4f4df48f96 + content-length: + - '1267' + status: + code: 200 + message: OK +- request: + body: '{"id": "cbe48deb-473d-498f-af94-facb1c90ac68"}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '46' + Content-Type: + - application/json + User-Agent: + - python-requests/2.34.2 + method: POST + uri: https://www.braintrust.dev/api/base_experiment/get_id + response: + body: + string: Missing read access to experiment id cbe48deb-473d-498f-af94-facb1c90ac68, + or the experiment does not exist [user_email=redacted] [user_org=Braintrust + SDKs] [timestamp=1786547858.705] [request_id=yul1::xm6c7-1786547858606-847e928224d9] + headers: + Cache-Control: + - public, max-age=0, must-revalidate + Content-Length: + - '254' + Content-Security-Policy: + - 'script-src ''self'' ''unsafe-eval'' ''wasm-unsafe-eval'' ''strict-dynamic'' + ''nonce-NDJiYjMwOTItYmI0Zi00NTBjLWI3MGItMjg5YTM4ZWVlY2Fm'' *.js.stripe.com + js.stripe.com maps.googleapis.com ; style-src ''self'' ''unsafe-inline'' *.braintrust.dev + btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com + d4tuoctqmanu0.cloudfront.net; font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com + cdn.jsdelivr.net d4tuoctqmanu0.cloudfront.net fonts.googleapis.com mintlify-assets.b-cdn.net + fonts.cdnfonts.com; object-src ''none''; base-uri ''self''; form-action ''self'' + https://www.facebook.com; frame-ancestors ''self''; worker-src ''self'' blob:; + report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16; + report-to csp-endpoint-0' + Content-Type: + - text/plain; charset=utf-8 + Date: + - Wed, 12 Aug 2026 15:17:38 GMT + Etag: + - '"slqvuufn5l72"' + Reporting-Endpoints: + - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16" + Server: + - Vercel + Strict-Transport-Security: + - max-age=63072000 + X-Clerk-Auth-Message: + - Invalid JWT form. A JWT consists of three parts separated by dots. (reason=token-invalid, + token-carrier=header) + X-Clerk-Auth-Reason: + - token-invalid + X-Clerk-Auth-Status: + - signed-out + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-Matched-Path: + - /api/base_experiment/get_id + X-Nonce: + - NDJiYjMwOTItYmI0Zi00NTBjLWI3MGItMjg5YTM4ZWVlY2Fm + X-Vercel-Cache: + - MISS + X-Vercel-Id: + - yul1::iad1::xm6c7-1786547858606-847e928224d9 + status: + code: 400 + message: Bad Request +version: 1 diff --git a/py/src/braintrust/api/experiments.py b/py/src/braintrust/api/experiments.py index 6673fd1c..eed7e87a 100644 --- a/py/src/braintrust/api/experiments.py +++ b/py/src/braintrust/api/experiments.py @@ -1,10 +1,202 @@ -"""Experiment API service.""" +"""Experiment API service and response models.""" +from collections.abc import Mapping +from dataclasses import dataclass, field +from types import MappingProxyType +from typing import Any + +from ..util import encode_uri_component +from ._routing import RequestTarget from ._service import ResourceAPI +from .errors import BraintrustHTTPError +from .policies import RetryMode + + +def _empty_mapping() -> Mapping[str, Any]: + return MappingProxyType({}) + + +def _preserve(value: Mapping[str, Any]) -> Mapping[str, Any]: + return MappingProxyType(dict(value)) + + +@dataclass(frozen=True) +class ExperimentRecord: + """An experiment returned by the resource API.""" + + id: str + name: str + project_id: str | None + raw: Mapping[str, Any] = field(default_factory=_empty_mapping) + + @classmethod + def from_dict(cls, value: Mapping[str, Any]) -> "ExperimentRecord": + experiment_id = value.get("id") + name = value.get("name") + project_id = value.get("project_id") + if not isinstance(experiment_id, str) or not isinstance(name, str): + raise ValueError("Experiment data must include string id and name fields") + if project_id is not None and not isinstance(project_id, str): + raise ValueError("Experiment project_id must be a string or null") + return cls(id=experiment_id, name=name, project_id=project_id, raw=_preserve(value)) + + +@dataclass(frozen=True) +class BaseExperiment: + """The persisted baseline selected for an experiment.""" + + id: str + name: str + raw: Mapping[str, Any] = field(default_factory=_empty_mapping) + + @classmethod + def from_dict(cls, value: Mapping[str, Any]) -> "BaseExperiment": + experiment_id = value.get("base_exp_id") + name = value.get("base_exp_name") + if not isinstance(experiment_id, str) or not isinstance(name, str): + raise ValueError("Base experiment data must include string base_exp_id and base_exp_name fields") + return cls(id=experiment_id, name=name, raw=_preserve(value)) + + +@dataclass(frozen=True) +class ExperimentScore: + """One score aggregate in an experiment comparison response.""" + + name: str + score: float + improvements: int | None + regressions: int | None + diff: float | None = None + raw: Mapping[str, Any] = field(default_factory=_empty_mapping) + + @classmethod + def from_dict(cls, key: str, value: Mapping[str, Any]) -> "ExperimentScore": + name = value.get("name", key) + if not isinstance(name, str): + raise ValueError(f"Experiment score {key!r} must include a string name") + return cls( + name=name, + score=value.get("score"), + improvements=value.get("improvements"), + regressions=value.get("regressions"), + diff=value.get("diff"), + raw=_preserve(value), + ) + + +@dataclass(frozen=True) +class ExperimentMetric: + """One metric aggregate in an experiment comparison response.""" + + name: str + metric: float | int + unit: str + improvements: int | None + regressions: int | None + diff: float | None = None + raw: Mapping[str, Any] = field(default_factory=_empty_mapping) + + @classmethod + def from_dict(cls, key: str, value: Mapping[str, Any]) -> "ExperimentMetric": + name = value.get("name", key) + unit = value.get("unit", "") + if not isinstance(name, str) or not isinstance(unit, str): + raise ValueError(f"Experiment metric {key!r} must include string name and unit fields") + return cls( + name=name, + metric=value.get("metric"), + unit=unit, + improvements=value.get("improvements"), + regressions=value.get("regressions"), + diff=value.get("diff"), + raw=_preserve(value), + ) + + +@dataclass(frozen=True) +class ExperimentComparison: + """Scores and metrics returned by the experiment comparison endpoint.""" + + scores: Mapping[str, ExperimentScore] + metrics: Mapping[str, ExperimentMetric] + raw: Mapping[str, Any] = field(default_factory=_empty_mapping) + + @classmethod + def from_dict(cls, value: Mapping[str, Any]) -> "ExperimentComparison": + raw_scores = value.get("scores", {}) + raw_metrics = value.get("metrics", {}) + if not isinstance(raw_scores, Mapping) or not isinstance(raw_metrics, Mapping): + raise ValueError("Experiment comparison scores and metrics must be objects") + + scores = {} + for key, score in raw_scores.items(): + if not isinstance(key, str) or not isinstance(score, Mapping): + raise ValueError("Experiment comparison scores must map string names to objects") + scores[key] = ExperimentScore.from_dict(key, score) + + metrics = {} + for key, metric in raw_metrics.items(): + if not isinstance(key, str) or not isinstance(metric, Mapping): + raise ValueError("Experiment comparison metrics must map string names to objects") + metrics[key] = ExperimentMetric.from_dict(key, metric) + + return cls( + scores=MappingProxyType(scores), + metrics=MappingProxyType(metrics), + raw=_preserve(value), + ) class ExperimentsAPI(ResourceAPI): - """Synchronous experiment operations. + """Synchronous experiment lookup and comparison operations.""" + + def get(self, experiment_id: str) -> ExperimentRecord: + """Fetch an experiment by ID.""" + + response = self._request_json( + RequestTarget.API, + "GET", + f"/v1/experiment/{encode_uri_component(experiment_id)}", + retry_mode=RetryMode.SAFE_READ, + ) + return ExperimentRecord.from_dict(response) + + def get_base(self, experiment_id: str) -> BaseExperiment | None: + """Return an experiment's persisted baseline, or ``None`` when none exists.""" + + try: + response = self._request_json( + RequestTarget.APP, + "POST", + "/api/base_experiment/get_id", + json={"id": experiment_id}, + retry_mode=RetryMode.SAFE_READ, + ) + except BraintrustHTTPError as exc: + if exc.status_code == 400: + return None + raise + + if not response: + return None + return BaseExperiment.from_dict(response) + + def compare( + self, + experiment_id: str, + *, + base_experiment_id: str | None = None, + ) -> ExperimentComparison: + """Fetch score and metric aggregates for an experiment comparison.""" - Endpoint methods are added as experiment call sites migrate to the API client. - """ + response = self._request_json( + RequestTarget.API, + "GET", + "/experiment-comparison2", + params={ + "experiment_id": experiment_id, + "base_experiment_id": base_experiment_id, + }, + retry_mode=RetryMode.SAFE_READ, + ) + return ExperimentComparison.from_dict(response) diff --git a/py/src/braintrust/api/test_experiments.py b/py/src/braintrust/api/test_experiments.py new file mode 100644 index 00000000..b835156d --- /dev/null +++ b/py/src/braintrust/api/test_experiments.py @@ -0,0 +1,451 @@ +import contextlib +import http.server +import json +import os +import re +import socketserver +import threading +from collections.abc import Mapping +from urllib.parse import urlsplit + +import braintrust +import pytest +from braintrust.api import ( + BraintrustClient, + BraintrustRetryExhaustedError, + EndpointRouter, + ExperimentComparison, + ExperimentRecord, +) +from braintrust.api._transport import Transport +from braintrust.conftest import get_vcr_config +from braintrust.framework import EvalCase, Evaluator, run_evaluator +from braintrust.git_fields import RepoInfo +from braintrust.logger import SummarySkipped, SummarySuccess +from braintrust.test_helpers import init_test_exp, with_memory_logger, with_simulate_login # noqa: F401 + + +def _normalize_experiment_request(request): + if not request.body or urlsplit(request.uri).path != "/logs3": + return request + try: + payload = json.loads(request.body) + except (TypeError, ValueError): + return request + + for row in payload.get("rows", []): + row.pop("context", None) + row.pop("created", None) + row.pop("root_span_id", None) + row.pop("span_id", None) + metrics = row.get("metrics", {}) + metrics.pop("start", None) + metrics.pop("end", None) + request.body = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode() + return request + + +def _scrub_experiment_response(response): + body = response.get("body", {}).get("string") + body_was_bytes = isinstance(body, bytes) + if body_was_bytes: + body = body.decode(errors="replace") + if not isinstance(body, str): + return response + + body = re.sub(r"user_email=[^\]]+", "user_email=redacted", body) + try: + payload = json.loads(body) + except ValueError: + pass + else: + + def scrub_repo_info(value): + if isinstance(value, dict): + if "repo_info" in value: + value["repo_info"] = {} + for nested in value.values(): + scrub_repo_info(nested) + elif isinstance(value, list): + for nested in value: + scrub_repo_info(nested) + + scrub_repo_info(payload) + body = json.dumps(payload, separators=(",", ":")) + + response["body"]["string"] = body.encode() if body_was_bytes else body + return response + + +@pytest.fixture(scope="module") +def vcr_config(): + config = get_vcr_config() + config["before_record_request"] = _normalize_experiment_request + scrub_sensitive_headers = config["before_record_response"] + + def scrub_response(response): + return _scrub_experiment_response(scrub_sensitive_headers(response)) + + config["before_record_response"] = scrub_response + return config + + +@contextlib.contextmanager +def experiment_server(routes): + class ExperimentHandler(http.server.BaseHTTPRequestHandler): + requests = [] + route_counts = {} + + def log_message(self, format, *args): + pass + + def do_GET(self): + self._handle() + + def do_POST(self): + self._handle() + + def _handle(self): + path = urlsplit(self.path).path + content_length = int(self.headers.get("Content-Length", "0")) + body = self.rfile.read(content_length) if content_length else b"" + type(self).requests.append((self.command, self.path, body, self.headers.get("Authorization"))) + request_number = type(self).route_counts.get(path, 0) + type(self).route_counts[path] = request_number + 1 + actions = routes[path] + action = actions[min(request_number, len(actions) - 1)] + status, response = action[:2] + response_headers = action[2] if len(action) > 2 else {} + response_body = json.dumps(response).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + for name, value in response_headers.items(): + self.send_header(name, value) + self.send_header("Content-Length", str(len(response_body))) + self.end_headers() + self.wfile.write(response_body) + + server = socketserver.ThreadingTCPServer(("127.0.0.1", 0), ExperimentHandler) + server.daemon_threads = True + thread = threading.Thread(target=server.serve_forever, kwargs={"poll_interval": 0.01}, daemon=True) + thread.start() + try: + yield f"http://127.0.0.1:{server.server_address[1]}", ExperimentHandler + finally: + server.shutdown() + server.server_close() + + +def _client_for_server(url): + return BraintrustClient.from_transport( + transport=Transport(sleep=lambda _delay: None), + router=EndpointRouter(app_url=url, api_url=url), + api_key="test-key", + org_id="test-org-id", + org_name="test-org-name", + ) + + +def _use_client_for_experiment(experiment, client, url): + state = experiment.state + state._client = client + # Keep the legacy destinations local too, so this remains a focused red/green + # regression test while Experiment migrates from HTTPConnection to ExperimentsAPI. + state.app_url = url + state.api_url = url + state._app_conn = None + state._api_conn = None + + +def _comparison_response(score=0.9): + return { + "scores": { + "accuracy": { + "name": "accuracy", + "score": score, + "improvements": 1, + "regressions": 0, + "diff": 0.4, + "new_score_field": {"preserved": True}, + } + }, + "metrics": { + "duration": { + "name": "duration", + "metric": 1.25, + "unit": "s", + "improvements": 0, + "regressions": 1, + "diff": 0.1, + "new_metric_field": ["preserved"], + } + }, + "new_backend_field": {"preserved": True}, + } + + +def test_experiment_service_parses_additive_responses_and_uses_expected_targets(): + routes = { + "/v1/experiment/base-exp-id": [ + ( + 200, + { + "id": "base-exp-id", + "name": "baseline", + "project_id": "project-id", + "new_backend_field": {"preserved": True}, + }, + ) + ], + "/api/base_experiment/get_id": [ + ( + 200, + { + "id": "candidate-exp-id", + "name": "candidate", + "project_id": "project-id", + "base_exp_id": "base-exp-id", + "base_exp_name": "baseline", + "new_backend_field": 42, + }, + ) + ], + "/experiment-comparison2": [(200, _comparison_response())], + } + with experiment_server(routes) as (url, handler): + service = _client_for_server(url).experiments + experiment = service.get("base-exp-id") + base = service.get_base("candidate-exp-id") + comparison = service.compare("candidate-exp-id", base_experiment_id="base-exp-id") + + assert isinstance(experiment, ExperimentRecord) + assert experiment.id == "base-exp-id" + assert experiment.name == "baseline" + assert experiment.raw["new_backend_field"] == {"preserved": True} + assert base is not None + assert base.id == "base-exp-id" + assert base.name == "baseline" + assert base.raw["new_backend_field"] == 42 + assert isinstance(comparison, ExperimentComparison) + assert comparison.scores["accuracy"].score == 0.9 + assert comparison.scores["accuracy"].raw["score"] == 0.9 + assert comparison.scores["accuracy"].raw["new_score_field"] == {"preserved": True} + assert comparison.metrics["duration"].raw["new_metric_field"] == ["preserved"] + assert comparison.raw["new_backend_field"] == {"preserved": True} + + requests = handler.requests + assert [request[0] for request in requests] == ["GET", "POST", "GET"] + assert requests[0][1] == "/v1/experiment/base-exp-id" + assert json.loads(requests[1][2]) == {"id": "candidate-exp-id"} + assert requests[2][1] == ("/experiment-comparison2?experiment_id=candidate-exp-id&base_experiment_id=base-exp-id") + assert all(request[3] == "Bearer test-key" for request in requests) + + +def test_experiment_summarize_retries_transient_comparison_failure_issue_639(with_memory_logger, with_simulate_login): + # Regression test for https://github.com/braintrustdata/braintrust-sdk-python/issues/639 + routes = { + "/v1/experiment/base-exp-id": [(200, {"id": "base-exp-id", "name": "baseline"})], + "/experiment-comparison2": [(503, {"error": "temporary outage"}), (200, _comparison_response())], + } + with experiment_server(routes) as (url, handler): + client = _client_for_server(url) + experiment = init_test_exp("candidate-exp-id", "project-id") + _use_client_for_experiment(experiment, client, url) + + summary = experiment.summarize(comparison_experiment_id="base-exp-id") + + assert summary.comparison_experiment_name == "baseline" + assert isinstance(summary.comparison, SummarySuccess) + assert summary.comparison.scores["accuracy"].score == 0.9 + assert summary.comparison.metrics["duration"].metric == 1.25 + assert handler.route_counts["/experiment-comparison2"] == 2 + + +def test_experiment_summarize_raises_after_retry_exhaustion(with_memory_logger, with_simulate_login): + routes = { + "/v1/experiment/base-exp-id": [(200, {"id": "base-exp-id", "name": "baseline"})], + "/experiment-comparison2": [ + ( + 503, + {"error": "persistent outage"}, + {"x-bt-internal-trace-id": "summary-trace-id", "x-unrelated-header": "excluded"}, + ) + ], + } + with experiment_server(routes) as (url, handler): + experiment = init_test_exp("candidate-exp-id", "project-id") + _use_client_for_experiment(experiment, _client_for_server(url), url) + + with pytest.raises(BraintrustRetryExhaustedError) as exc_info: + experiment.summarize(comparison_experiment_id="base-exp-id") + + error = exc_info.value + assert error.status_code == 503 + assert error.attempts == 4 + assert error.response_body == '{"error": "persistent outage"}' + assert error.request_id == "summary-trace-id" + assert error.response_headers == { + "content-type": "application/json", + "x-bt-internal-trace-id": "summary-trace-id", + } + assert handler.route_counts["/experiment-comparison2"] == 4 + + +def test_experiment_summarize_surfaces_comparison_lookup_failure(with_memory_logger, with_simulate_login): + routes = { + "/v1/experiment/base-exp-id": [(503, {"error": "lookup unavailable"})], + } + with experiment_server(routes) as (url, handler): + experiment = init_test_exp("candidate-exp-id", "project-id") + _use_client_for_experiment(experiment, _client_for_server(url), url) + + with pytest.raises(BraintrustRetryExhaustedError): + experiment.summarize(comparison_experiment_id="base-exp-id") + + assert handler.route_counts["/v1/experiment/base-exp-id"] == 4 + + +def test_experiment_summarize_genuine_empty_comparison_is_success(with_memory_logger, with_simulate_login): + routes = { + "/v1/experiment/base-exp-id": [(200, {"id": "base-exp-id", "name": "baseline"})], + "/experiment-comparison2": [(200, {"scores": {}, "metrics": {}})], + } + with experiment_server(routes) as (url, _handler): + experiment = init_test_exp("candidate-exp-id", "project-id") + _use_client_for_experiment(experiment, _client_for_server(url), url) + + summary = experiment.summarize(comparison_experiment_id="base-exp-id") + + assert isinstance(summary.comparison, SummarySuccess) + assert summary.comparison.scores == {} + assert summary.comparison.metrics == {} + assert summary.as_dict()["comparison"] == { + "scores": {}, + "metrics": {}, + "status": "success", + } + + +def test_experiment_summarize_without_scores_is_marked_skipped(with_memory_logger, with_simulate_login): + experiment = init_test_exp("candidate-exp-id", "project-id") + + summary = experiment.summarize(summarize_scores=False) + + assert isinstance(summary.comparison, SummarySkipped) + assert summary.comparison.reason == "Score summarization was disabled" + assert summary.as_dict()["comparison"] == { + "status": "skipped", + "reason": "Score summarization was disabled", + } + + +@pytest.mark.asyncio +async def test_run_evaluator_raises_on_summary_failure(with_memory_logger, with_simulate_login): + routes = { + "/v1/experiment/base-exp-id": [(200, {"id": "base-exp-id", "name": "baseline"})], + "/experiment-comparison2": [(503, {"error": "persistent outage"})], + } + evaluator = Evaluator( + project_name="project-id", + eval_name="candidate-exp-id", + data=[EvalCase(input="hello", expected="hello")], + task=lambda input_value: input_value, + scores=[lambda input_value, output, expected: 1.0], + experiment_name="candidate-exp-id", + metadata=None, + base_experiment_id="base-exp-id", + ) + with experiment_server(routes) as (url, _handler): + experiment = init_test_exp("candidate-exp-id", "project-id") + _use_client_for_experiment(experiment, _client_for_server(url), url) + + with pytest.raises(BraintrustRetryExhaustedError): + await run_evaluator(experiment, evaluator, position=None, filters=[]) + + +def test_base_experiment_400_is_none_without_retry(): + routes = {"/api/base_experiment/get_id": [(400, {"error": "No base experiment"})]} + with experiment_server(routes) as (url, handler): + base = _client_for_server(url).experiments.get_base("fresh-exp-id") + + assert base is None + assert handler.route_counts["/api/base_experiment/get_id"] == 1 + + +def _api_key(): + return os.environ.get("BRAINTRUST_API_KEY", "sk-dummy-for-vcr-replay") + + +def _log_score(experiment, score): + experiment.log( + id=f"{experiment.name}-row", + input={"question": "What is 2 + 2?"}, + output="4", + expected="4", + scores={"exact_match": score}, + ) + experiment.flush() + + +@pytest.mark.vcr +def test_experiment_summarize_end_to_end_with_real_backend(): + project_name = "python-sdk-api-experiment-service-vcr" + base = braintrust.init( + project=project_name, + experiment="experiment-service-base", + api_key=_api_key(), + update=True, + set_current=False, + repo_info=RepoInfo(), + ) + _log_score(base, 0.5) + + candidate = braintrust.init( + project=project_name, + experiment="experiment-service-candidate", + api_key=_api_key(), + base_experiment_id=base.id, + update=True, + set_current=False, + repo_info=RepoInfo(), + ) + _log_score(candidate, 1.0) + + service = candidate.state.api_client().experiments + record = service.get(base.id) + assert record.id == base.id + assert record.name == base.name + + persisted_base = candidate.fetch_base_experiment() + assert persisted_base is not None + assert persisted_base.id == base.id + assert persisted_base.name == base.name + + automatic_summary = candidate.summarize() + assert automatic_summary.comparison_experiment_name == base.name + assert isinstance(automatic_summary.comparison, SummarySuccess) + assert automatic_summary.comparison.status == "success" + assert automatic_summary.comparison.scores["exact_match"].score == 1.0 + assert automatic_summary.comparison.scores["exact_match"].diff == 0.5 + assert isinstance(automatic_summary.comparison.metrics, Mapping) + assert automatic_summary.as_dict()["comparison"]["status"] == "success" + + explicit_summary = candidate.summarize(comparison_experiment_id=base.id) + assert explicit_summary.comparison_experiment_name == base.name + assert isinstance(explicit_summary.comparison, SummarySuccess) + assert explicit_summary.comparison.scores == automatic_summary.comparison.scores + assert explicit_summary.comparison.metrics == automatic_summary.comparison.metrics + + +@pytest.mark.vcr +def test_fresh_experiment_has_no_base_on_real_backend(): + experiment = braintrust.init( + project="python-sdk-api-experiment-service-fresh-vcr", + experiment="fresh-experiment-without-base", + api_key=_api_key(), + update=True, + set_current=False, + repo_info=RepoInfo(), + ) + + assert experiment.fetch_base_experiment() is None diff --git a/py/src/braintrust/framework.py b/py/src/braintrust/framework.py index 6bc0e255..3482c0fa 100644 --- a/py/src/braintrust/framework.py +++ b/py/src/braintrust/framework.py @@ -37,6 +37,7 @@ Metadata, ScoreSummary, Span, + SummarySuccess, parent_context, start_span, stringify_exception, @@ -829,6 +830,7 @@ async def EvalAsync( """ A function you can use to define an evaluator. This is a convenience wrapper around the `Evaluator` class. + Summary retrieval failures fail the evaluation after retries are exhausted. Use this function over `Eval()` when you are running in an async context, including in a Jupyter notebook. Example: @@ -957,6 +959,7 @@ def Eval( """ A function you can use to define an evaluator. This is a convenience wrapper around the `Evaluator` class. + Summary retrieval failures fail the evaluation after retries are exhausted. For callers running in an async context, use `EvalAsync()` instead. Example: @@ -1947,8 +1950,7 @@ def build_local_summary( project_url=None, experiment_url=None, comparison_experiment_name=None, - scores=avg_scores, - metrics={}, + comparison=SummarySuccess(scores=avg_scores, metrics={}), ) diff --git a/py/src/braintrust/logger.py b/py/src/braintrust/logger.py index 90dd1227..3f81f420 100644 --- a/py/src/braintrust/logger.py +++ b/py/src/braintrust/logger.py @@ -2269,6 +2269,8 @@ def summarize(summarize_scores: bool = True, comparison_experiment_id: str | Non """ Summarize the current experiment, including the scores (compared to the closest reference experiment) and metadata. + Summary retrieval failures are raised after retries are exhausted. + :param summarize_scores: Whether to summarize the scores. If False, only the metadata will be returned. :param comparison_experiment_id: The experiment to compare against. If None, the most recent experiment on the comparison_commit will be used. :returns: `ExperimentSummary` @@ -4204,20 +4206,10 @@ def update_span(self, id: str, **event: Any) -> None: ) def fetch_base_experiment(self) -> ExperimentIdentifier | None: - state = self._get_state() - conn = state.app_conn() - - resp = conn.post("/api/base_experiment/get_id", json={"id": self.id}) - if resp.status_code == 400: - # No base experiment - return None - - response_raise_for_status(resp) - base = resp.json() - if base: - return ExperimentIdentifier(id=base["base_exp_id"], name=base["base_exp_name"]) - else: + base = self._get_state().api_client().experiments.get_base(self.id) + if base is None: return None + return ExperimentIdentifier(id=base.id, name=base.name) def summarize( self, summarize_scores: bool = True, comparison_experiment_id: str | None = None @@ -4225,6 +4217,8 @@ def summarize( """ Summarize the experiment, including the scores (compared to the closest reference experiment) and metadata. + Summary retrieval failures are raised after retries are exhausted. + :param summarize_scores: Whether to summarize the scores. If False, only the metadata will be returned. :param comparison_experiment_id: The experiment to compare against. If None, the most recent experiment on the origin's main branch will be used. :returns: `ExperimentSummary` @@ -4237,10 +4231,10 @@ def summarize( project_url = f"{state.app_public_url}/app/{encode_uri_component(state.org_name)}/p/{encode_uri_component(self.project.name)}" experiment_url = f"{project_url}/experiments/{encode_uri_component(self.name)}" - score_summary = {} - metric_summary = {} comparison_experiment_name = None - if summarize_scores: + if not summarize_scores: + comparison: SummaryResult = SummarySkipped(reason="Score summarization was disabled") + else: # Get the comparison experiment if comparison_experiment_id is None: base_experiment = self.fetch_base_experiment() @@ -4248,38 +4242,43 @@ def summarize( comparison_experiment_id = base_experiment.id comparison_experiment_name = base_experiment.name else: - try: - comparison_experiment = state.api_conn().get_json(f"v1/experiment/{comparison_experiment_id}") - comparison_experiment_name = comparison_experiment.get("name") - except Exception: - pass + comparison_experiment = state.api_client().experiments.get(comparison_experiment_id) + comparison_experiment_name = comparison_experiment.name - try: - summary_items = state.api_conn().get_json( - "experiment-comparison2", - args={ - "experiment_id": self.id, - "base_experiment_id": comparison_experiment_id, - }, - ) - except Exception as e: - _logger.warning( - f"Failed to fetch experiment scores and metrics: {e}\n\nView complete results in Braintrust or run experiment.summarize() again." - ) - summary_items = {} - - score_items = summary_items.get("scores", {}) - metric_items = summary_items.get("metrics", {}) + summary_items = state.api_client().experiments.compare( + self.id, + base_experiment_id=comparison_experiment_id, + ) + score_items = summary_items.scores + metric_items = summary_items.metrics longest_score_name = max(len(k) for k in score_items.keys()) if score_items else 0 score_summary = { - k: ScoreSummary(_longest_score_name=longest_score_name, **v) for (k, v) in score_items.items() + k: ScoreSummary( + name=v.name, + score=v.score, + improvements=v.improvements, + regressions=v.regressions, + diff=v.diff, + _longest_score_name=longest_score_name, + ) + for (k, v) in score_items.items() } longest_metric_name = max(len(k) for k in metric_items.keys()) if metric_items else 0 metric_summary = { - k: MetricSummary(_longest_metric_name=longest_metric_name, **v) for (k, v) in metric_items.items() + k: MetricSummary( + name=v.name, + metric=v.metric, + unit=v.unit, + improvements=v.improvements, + regressions=v.regressions, + diff=v.diff, + _longest_metric_name=longest_metric_name, + ) + for (k, v) in metric_items.items() } + comparison = SummarySuccess(scores=score_summary, metrics=metric_summary) return ExperimentSummary( project_name=self.project.name, @@ -4289,8 +4288,7 @@ def summarize( project_url=project_url, experiment_url=experiment_url, comparison_experiment_name=comparison_experiment_name, - scores=score_summary, - metrics=metric_summary, + comparison=comparison, ) def export(self) -> str: @@ -5858,9 +5856,29 @@ def __str__(self): ) +@dataclasses.dataclass(frozen=True) +class SummarySuccess(SerializableDataClass): + """A successfully retrieved experiment comparison.""" + + scores: dict[str, ScoreSummary] + metrics: dict[str, MetricSummary] + status: Literal["success"] = dataclasses.field(default="success", init=False) + + +@dataclasses.dataclass(frozen=True) +class SummarySkipped(SerializableDataClass): + """An experiment comparison that was intentionally skipped.""" + + reason: str + status: Literal["skipped"] = dataclasses.field(default="skipped", init=False) + + +SummaryResult = SummarySuccess | SummarySkipped + + @dataclasses.dataclass class ExperimentSummary(SerializableDataClass): - """Summary of an experiment's scores and metadata.""" + """Summary of an experiment's comparison and metadata.""" project_name: str """Name of the project that the experiment belongs to.""" @@ -5876,29 +5894,29 @@ class ExperimentSummary(SerializableDataClass): """URL to the experiment's page in the Braintrust app.""" comparison_experiment_name: str | None """The experiment scores are baselined against.""" - scores: dict[str, ScoreSummary] - """Summary of the experiment's scores.""" - metrics: dict[str, MetricSummary] - """Summary of the experiment's metrics.""" + comparison: SummaryResult + """Result of retrieving the experiment comparison.""" def __str__(self): comparison_line = "" if self.comparison_experiment_name: comparison_line = f"""{self.experiment_name} compared to {self.comparison_experiment_name}:\n""" - return ( - f"""\n=========================SUMMARY=========================\n{comparison_line}""" - + "\n".join([str(score) for score in self.scores.values()]) - + ("\n\n" if self.scores else "") - + "\n".join([str(metric) for metric in self.metrics.values()]) - + ("\n\n" if self.metrics else "") - + ( - textwrap.dedent( - f"""\ + if isinstance(self.comparison, SummarySkipped): + result = f"Summary skipped: {self.comparison.reason}\n\n" + else: + result = ( + "\n".join([str(score) for score in self.comparison.scores.values()]) + + ("\n\n" if self.comparison.scores else "") + + "\n".join([str(metric) for metric in self.comparison.metrics.values()]) + + ("\n\n" if self.comparison.metrics else "") + ) + return f"""\n=========================SUMMARY=========================\n{comparison_line}{result}""" + ( + textwrap.dedent( + f"""\ See results for {self.experiment_name} at {self.experiment_url}""" - ) - if self.experiment_url is not None - else "" ) + if self.experiment_url is not None + else "" ) diff --git a/py/src/braintrust/test_framework.py b/py/src/braintrust/test_framework.py index 9f1b390b..74391d60 100644 --- a/py/src/braintrust/test_framework.py +++ b/py/src/braintrust/test_framework.py @@ -4,7 +4,15 @@ from unittest.mock import MagicMock, patch import pytest -from braintrust.logger import BraintrustState, Dataset, ObjectMetadata, ProjectDatasetMetadata +from braintrust.logger import ( + BraintrustState, + Dataset, + ExperimentSummary, + ObjectMetadata, + ProjectDatasetMetadata, + SummarySkipped, + SummarySuccess, +) from braintrust.util import LazyValue from .framework import ( @@ -26,6 +34,25 @@ HAS_PYDANTIC = importlib.util.find_spec("pydantic") is not None +def test_eval_result_serializes_structured_summary(): + summary = ExperimentSummary( + project_name="test-project", + project_id="project-id", + experiment_id="experiment-id", + experiment_name="test-evaluator", + project_url="https://example.com/project", + experiment_url="https://example.com/experiment", + comparison_experiment_name=None, + comparison=SummarySkipped(reason="disabled"), + ) + + serialized = EvalResultWithSummary(summary=summary, results=[]).as_dict()["summary"] + + assert serialized["comparison"] == {"reason": "disabled", "status": "skipped"} + assert "scores" not in serialized + assert "metrics" not in serialized + + def make_dataset(dataset_id, row): project_metadata = ObjectMetadata(id="test-project", name="test-project", full_info={}) dataset_metadata = ObjectMetadata(id=dataset_id, name="test-dataset", full_info={}) @@ -222,8 +249,9 @@ def exact_match(input_value, output, expected): # Verify summary assert result.summary.project_name == "test-project" - assert "exact_match" in result.summary.scores - assert result.summary.scores["exact_match"].score == 1.0 + assert isinstance(result.summary.comparison, SummarySuccess) + assert "exact_match" in result.summary.comparison.scores + assert result.summary.comparison.scores["exact_match"].score == 1.0 @pytest.mark.asyncio @@ -285,33 +313,6 @@ def exact_match(input_value, output, expected): ) -def test_experiment_summarize_resolves_explicit_comparison_name(with_memory_logger, with_simulate_login): - exp = init_test_exp("test-evaluator", "test-project") - mock_conn = MagicMock() - - def get_json(path, args=None): - if path == "v1/experiment/base-exp-id": - return {"name": "base-exp"} - if path == "experiment-comparison2": - return {"scores": {}, "metrics": {}} - raise AssertionError(f"Unexpected get_json call: {path}, {args}") - - mock_conn.get_json.side_effect = get_json - - with patch.object(exp.state, "api_conn", return_value=mock_conn): - summary = exp.summarize(comparison_experiment_id="base-exp-id") - - assert summary.comparison_experiment_name == "base-exp" - mock_conn.get_json.assert_any_call("v1/experiment/base-exp-id") - mock_conn.get_json.assert_any_call( - "experiment-comparison2", - args={ - "experiment_id": "test-evaluator", - "base_experiment_id": "base-exp-id", - }, - ) - - @pytest.mark.asyncio @pytest.mark.skipif(not HAS_PYDANTIC, reason="pydantic not installed") async def test_run_evaluator_exposes_validated_parameter_values_to_hooks(): @@ -439,9 +440,10 @@ def _run_eval_sync(self, *args, **kwargs): assert eval_result.scores[scorer_name] == 1.0 assert result.summary.project_name == "test-project" + assert isinstance(result.summary.comparison, SummarySuccess) for scorer_name in scorer_names: - assert scorer_name in result.summary.scores - assert result.summary.scores[scorer_name].score == 1.0 + assert scorer_name in result.summary.comparison.scores + assert result.summary.comparison.scores[scorer_name].score == 1.0 @pytest.mark.asyncio @@ -480,8 +482,9 @@ def list_dict_scorer(input_value, output, expected): assert eval_result.scores["summary_only"] == 1.0 assert result.summary.project_name == "test-project" - assert result.summary.scores["per_result"].score == 1.0 - assert result.summary.scores["summary_only"].score == 1.0 + assert isinstance(result.summary.comparison, SummarySuccess) + assert result.summary.comparison.scores["per_result"].score == 1.0 + assert result.summary.comparison.scores["summary_only"].score == 1.0 @pytest.mark.asyncio @@ -750,6 +753,7 @@ def purpose_scorer(input_value, output, expected): scores=[purpose_scorer], experiment_name="test-scorer-purpose", metadata=None, + summarize_scores=False, ) # Create experiment so spans get logged @@ -910,6 +914,7 @@ async def test_classifier_spans_are_logged(with_memory_logger, with_simulate_log ], experiment_name="test-classifier-span", metadata=None, + summarize_scores=False, ) exp = init_test_exp("test-classifier-span", "test-project") @@ -966,8 +971,9 @@ def exact_match(input, output, expected): # Verify it builds a local summary (no experiment_url means local run) assert result.summary.project_name == "test-no-logs" assert result.summary.experiment_url is None - assert result.summary.scores["exact_match"].score == 1.0 - assert result.summary.scores["simple_scorer"].score == 0.8 + assert isinstance(result.summary.comparison, SummarySuccess) + assert result.summary.comparison.scores["exact_match"].score == 1.0 + assert result.summary.comparison.scores["simple_scorer"].score == 0.8 # Most importantly: verify that no logs were sent (should be empty) logs = with_memory_logger.pop() @@ -996,7 +1002,8 @@ def sometimes_none_scorer(input, output, expected): ) # Should not crash and should calculate average from non-None scores only - assert result.summary.scores["conditional"].score == 1.0 # Only the second score counts + assert isinstance(result.summary.comparison, SummarySuccess) + assert result.summary.comparison.scores["conditional"].score == 1.0 # Only the second score counts @pytest.mark.asyncio diff --git a/py/src/braintrust/type_tests/test_api_client.py b/py/src/braintrust/type_tests/test_api_client.py index aec402b0..623062c2 100644 --- a/py/src/braintrust/type_tests/test_api_client.py +++ b/py/src/braintrust/type_tests/test_api_client.py @@ -2,7 +2,14 @@ from typing import TYPE_CHECKING -from braintrust.api import BraintrustClient, EndpointRouter, RequestTarget +from braintrust.api import ( + BaseExperiment, + BraintrustClient, + EndpointRouter, + ExperimentComparison, + ExperimentRecord, + RequestTarget, +) if TYPE_CHECKING: @@ -16,6 +23,11 @@ org_id: str = client.org_id org_name: str = client.org_name api_url: str | None = client_with_overrides.router.api_url + experiment: ExperimentRecord = client.experiments.get("experiment-id") + base_experiment: BaseExperiment | None = client.experiments.get_base("experiment-id") + comparison: ExperimentComparison = client.experiments.compare( + "experiment-id", base_experiment_id="base-experiment-id" + ) def test_api_client_public_types() -> None: @@ -23,3 +35,6 @@ def test_api_client_public_types() -> None: assert router.resolve(RequestTarget.API, "ping") == "https://api.example.com/ping" assert BraintrustClient.__name__ == "BraintrustClient" + assert ExperimentRecord.__name__ == "ExperimentRecord" + assert BaseExperiment.__name__ == "BaseExperiment" + assert ExperimentComparison.__name__ == "ExperimentComparison" diff --git a/py/src/braintrust/type_tests/test_experiment_summary.py b/py/src/braintrust/type_tests/test_experiment_summary.py new file mode 100644 index 00000000..47a3b7d6 --- /dev/null +++ b/py/src/braintrust/type_tests/test_experiment_summary.py @@ -0,0 +1,32 @@ +"""Static and runtime checks for structured experiment summaries.""" + +from typing import TYPE_CHECKING + +from braintrust import ExperimentSummary, MetricSummary, ScoreSummary, SummarySkipped + + +if TYPE_CHECKING: + + def check_summary_narrowing(summary: ExperimentSummary) -> None: + comparison = summary.comparison + if comparison.status == "success": + scores: dict[str, ScoreSummary] = comparison.scores + metrics: dict[str, MetricSummary] = comparison.metrics + else: + reason: str = comparison.reason + + +def test_structured_experiment_summary_public_types() -> None: + summary = ExperimentSummary( + project_name="project", + project_id="project-id", + experiment_id="experiment-id", + experiment_name="experiment", + project_url="https://example.com/project", + experiment_url="https://example.com/experiment", + comparison_experiment_name=None, + comparison=SummarySkipped(reason="disabled"), + ) + + assert isinstance(summary.comparison, SummarySkipped) + assert summary.comparison.reason == "disabled"