From 0aa794ed9b759fe921433ac7970ef79370dde8ba Mon Sep 17 00:00:00 2001 From: Erica Pisani Date: Fri, 31 Jul 2026 12:02:07 -0400 Subject: [PATCH 1/3] feat(openai): Gate embeddings input collection on data_collection Honour `_experiments["data_collection"]["gen_ai"]["inputs"]` in `_set_embeddings_input_data`. When the experiment is set, it takes precedence over `send_default_pii` (though `include_prompts=False` still wins); when unset, fall back to the previous PII-based behavior. `gen_ai.operation.name` and the request model are now always set on the span, even when input collection is skipped. Add tests covering the precedence matrix (sync + async) and input shapes under disabled input collection. Refs PY-2588 --- sentry_sdk/integrations/openai.py | 34 +-- tests/integrations/openai/test_openai.py | 326 +++++++++++++++++++++++ 2 files changed, 343 insertions(+), 17 deletions(-) diff --git a/sentry_sdk/integrations/openai.py b/sentry_sdk/integrations/openai.py index a400393237..8baaf5631f 100644 --- a/sentry_sdk/integrations/openai.py +++ b/sentry_sdk/integrations/openai.py @@ -610,31 +610,36 @@ def _set_embeddings_input_data( kwargs: "dict[str, Any]", integration: "OpenAIIntegration", ) -> None: - messages: "Union[str, SequenceNotStr[str], Iterable[int], Iterable[Iterable[int]]]" = kwargs.get( - "input" - ) + + set_data_normalized(span, SPANDATA.GEN_AI_OPERATION_NAME, "embeddings") set_on_span = ( span.set_attribute if isinstance(span, StreamedSpan) else span.set_data ) + model = kwargs.get("model") if model is not None: set_on_span(SPANDATA.GEN_AI_REQUEST_MODEL, model) - if ( - not should_send_default_pii() - or not integration.include_prompts - or messages is None - ): - set_data_normalized(span, SPANDATA.GEN_AI_OPERATION_NAME, "embeddings") + messages: "Union[str, SequenceNotStr[str], Iterable[int], Iterable[Iterable[int]]]" = kwargs.get( + "input" + ) + + client = sentry_sdk.get_client() + if has_data_collection_enabled(client.options): + # This takes precedence over the global data collection settings + if not integration.include_prompts: + return + if not client.options["data_collection"]["gen_ai"]["inputs"]: + return + elif not should_send_default_pii() or not integration.include_prompts: + return + if messages is None: return if isinstance(messages, str): - set_data_normalized(span, SPANDATA.GEN_AI_OPERATION_NAME, "embeddings") - normalized_messages = normalize_message_roles([messages]) # type: ignore - client = sentry_sdk.get_client() scope = sentry_sdk.get_current_scope() messages_data = ( truncate_and_annotate_embedding_inputs(normalized_messages, span, scope) @@ -650,7 +655,6 @@ def _set_embeddings_input_data( # dict special case following https://github.com/openai/openai-python/blob/3e0c05b84a2056870abf3bd6a5e7849020209cc3/src/openai/_utils/_transform.py#L194-L197 if not isinstance(messages, Iterable) or isinstance(messages, dict): - set_data_normalized(span, SPANDATA.GEN_AI_OPERATION_NAME, "embeddings") return messages = list(messages) @@ -658,7 +662,6 @@ def _set_embeddings_input_data( if len(messages) > 0: normalized_messages = normalize_message_roles(messages) - client = sentry_sdk.get_client() scope = sentry_sdk.get_current_scope() messages_data = ( truncate_and_annotate_embedding_inputs(normalized_messages, span, scope) @@ -670,9 +673,6 @@ def _set_embeddings_input_data( span, SPANDATA.GEN_AI_EMBEDDINGS_INPUT, messages_data, unpack=False ) - set_data_normalized(span, SPANDATA.GEN_AI_OPERATION_NAME, "embeddings") - - def _set_common_output_data( span: "Union[Span, StreamedSpan]", response: "Any", diff --git a/tests/integrations/openai/test_openai.py b/tests/integrations/openai/test_openai.py index e03ab2bbcf..e88b67e6d1 100644 --- a/tests/integrations/openai/test_openai.py +++ b/tests/integrations/openai/test_openai.py @@ -2980,6 +2980,206 @@ def test_embeddings_create( assert span["data"]["gen_ai.usage.total_tokens"] == 30 +def _collect_embeddings_span_data( + capture_events, capture_items, span_streaming, stream_gen_ai_spans, create +): + if span_streaming or stream_gen_ai_spans: + items = capture_items("span") + + with start_transaction(name="openai tx"): + response = create() + + assert len(response.data[0].embedding) == 3 + + sentry_sdk.flush() + span = next(item.payload for item in items) + assert span["attributes"]["sentry.op"] == "gen_ai.embeddings" + return span["attributes"] + + events = capture_events() + + with start_transaction(name="openai tx"): + response = create() + + assert len(response.data[0].embedding) == 3 + + tx = events[0] + assert tx["type"] == "transaction" + span = tx["spans"][0] + assert span["op"] == "gen_ai.embeddings" + return span["data"] + + +@pytest.mark.parametrize("span_streaming", [True, False]) +@pytest.mark.parametrize("stream_gen_ai_spans", [True, False]) +@pytest.mark.parametrize( + "data_collection,send_default_pii,include_prompts,expect_input", + [ + pytest.param( + {"gen_ai": {"inputs": True}}, + False, + True, + True, + id="inputs-enabled-overrides-pii-disabled", + ), + pytest.param( + {"gen_ai": {"inputs": False}}, + True, + True, + False, + id="inputs-disabled-overrides-pii-enabled", + ), + pytest.param( + {}, + False, + True, + True, + id="gen-ai-omitted-defaults-to-enabled", + ), + pytest.param( + {"gen_ai": {"inputs": False}}, + False, + True, + False, + id="inputs-disabled-and-pii-disabled", + ), + pytest.param( + {"gen_ai": {"inputs": True}}, + True, + False, + False, + id="include-prompts-disabled-overrides-inputs-enabled", + ), + pytest.param( + None, + False, + True, + False, + id="no-experiment-falls-back-to-pii", + ), + ], +) +def test_embeddings_create_data_collection( + sentry_init, + capture_events, + capture_items, + data_collection, + send_default_pii, + include_prompts, + expect_input, + stream_gen_ai_spans, + span_streaming, +): + init_kwargs = { + "integrations": [OpenAIIntegration(include_prompts=include_prompts)], + "disabled_integrations": [StdlibIntegration], + "traces_sample_rate": 1.0, + "send_default_pii": send_default_pii, + "stream_gen_ai_spans": stream_gen_ai_spans, + "trace_lifecycle": "stream" if span_streaming else "static", + } + + sentry_init_kwargs = dict(init_kwargs) + if data_collection is not None: + sentry_init_kwargs["_experiments"] = {"data_collection": data_collection} + + sentry_init(**sentry_init_kwargs) + + client = OpenAI(api_key="z") + + returned_embedding = CreateEmbeddingResponse( + data=[Embedding(object="embedding", index=0, embedding=[1.0, 2.0, 3.0])], + model="some-model", + object="list", + usage=EmbeddingTokenUsage( + prompt_tokens=20, + total_tokens=30, + ), + ) + + client.embeddings._post = mock.Mock(return_value=returned_embedding) + + span_data = _collect_embeddings_span_data( + capture_events, + capture_items, + span_streaming, + stream_gen_ai_spans, + lambda: client.embeddings.create(input="hello", model="text-embedding-3-large"), + ) + + assert span_data[SPANDATA.GEN_AI_SYSTEM] == "openai" + assert span_data[SPANDATA.GEN_AI_OPERATION_NAME] == "embeddings" + assert span_data[SPANDATA.GEN_AI_REQUEST_MODEL] == "text-embedding-3-large" + + if expect_input: + assert json.loads(span_data[SPANDATA.GEN_AI_EMBEDDINGS_INPUT]) == ["hello"] + else: + assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT not in span_data + + assert span_data["gen_ai.usage.input_tokens"] == 20 + assert span_data["gen_ai.usage.total_tokens"] == 30 + + +@pytest.mark.parametrize("span_streaming", [True, False]) +@pytest.mark.parametrize("stream_gen_ai_spans", [True, False]) +@pytest.mark.parametrize( + "get_input", + [ + lambda: "hello", + lambda: ["First text", "Second text"], + lambda: iter(["First text", "Second text"]), + lambda: [5, 8, 13, 21, 34], + lambda: [[5, 8, 13], [8, 13, 21]], + lambda: {"text": "hello"}, + ], +) +def test_embeddings_create_data_collection_inputs_disabled_input_shapes( + sentry_init, + capture_events, + capture_items, + get_input, + stream_gen_ai_spans, + span_streaming, +): + sentry_init( + integrations=[OpenAIIntegration(include_prompts=True)], + disabled_integrations=[StdlibIntegration], + traces_sample_rate=1.0, + send_default_pii=True, + stream_gen_ai_spans=stream_gen_ai_spans, + trace_lifecycle="stream" if span_streaming else "static", + _experiments={"data_collection": {"gen_ai": {"inputs": False}}}, + ) + + client = OpenAI(api_key="z") + + returned_embedding = CreateEmbeddingResponse( + data=[Embedding(object="embedding", index=0, embedding=[1.0, 2.0, 3.0])], + model="some-model", + object="list", + usage=EmbeddingTokenUsage( + prompt_tokens=20, + total_tokens=30, + ), + ) + + client.embeddings._post = mock.Mock(return_value=returned_embedding) + + span_data = _collect_embeddings_span_data( + capture_events, + capture_items, + span_streaming, + stream_gen_ai_spans, + lambda: client.embeddings.create( + input=get_input(), model="text-embedding-3-large" + ), + ) + + assert span_data[SPANDATA.GEN_AI_OPERATION_NAME] == "embeddings" + assert span_data[SPANDATA.GEN_AI_REQUEST_MODEL] == "text-embedding-3-large" + assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT not in span_data + + @pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize("stream_gen_ai_spans", [True, False]) @pytest.mark.asyncio @@ -3225,6 +3425,132 @@ async def test_embeddings_create_async( assert span["data"]["gen_ai.usage.total_tokens"] == 30 +@pytest.mark.parametrize("span_streaming", [True, False]) +@pytest.mark.parametrize("stream_gen_ai_spans", [True, False]) +@pytest.mark.asyncio +@pytest.mark.parametrize( + "data_collection,send_default_pii,include_prompts,expect_input", + [ + pytest.param( + {"gen_ai": {"inputs": True}}, + False, + True, + True, + id="inputs-enabled-overrides-pii-disabled", + ), + pytest.param( + {"gen_ai": {"inputs": False}}, + True, + True, + False, + id="inputs-disabled-overrides-pii-enabled", + ), + pytest.param( + {}, + False, + True, + True, + id="gen-ai-omitted-defaults-to-enabled", + ), + pytest.param( + {"gen_ai": {"inputs": True}}, + True, + False, + False, + id="include-prompts-disabled-overrides-inputs-enabled", + ), + pytest.param( + None, + False, + True, + False, + id="no-experiment-falls-back-to-pii", + ), + ], +) +async def test_embeddings_create_async_data_collection( + sentry_init, + capture_events, + capture_items, + data_collection, + send_default_pii, + include_prompts, + expect_input, + stream_gen_ai_spans, + span_streaming, +): + init_kwargs = { + "integrations": [OpenAIIntegration(include_prompts=include_prompts)], + "disabled_integrations": [StdlibIntegration], + "traces_sample_rate": 1.0, + "send_default_pii": send_default_pii, + "stream_gen_ai_spans": stream_gen_ai_spans, + "trace_lifecycle": "stream" if span_streaming else "static", + } + + sentry_init_kwargs = dict(init_kwargs) + if data_collection is not None: + sentry_init_kwargs["_experiments"] = {"data_collection": data_collection} + + sentry_init(**sentry_init_kwargs) + + client = AsyncOpenAI(api_key="z") + + returned_embedding = CreateEmbeddingResponse( + data=[Embedding(object="embedding", index=0, embedding=[1.0, 2.0, 3.0])], + model="some-model", + object="list", + usage=EmbeddingTokenUsage( + prompt_tokens=20, + total_tokens=30, + ), + ) + + client.embeddings._post = AsyncMock(return_value=returned_embedding) + + if span_streaming or stream_gen_ai_spans: + items = capture_items("span") + + with start_transaction(name="openai tx"): + response = await client.embeddings.create( + input="hello", model="text-embedding-3-large" + ) + + assert len(response.data[0].embedding) == 3 + + sentry_sdk.flush() + span = next(item.payload for item in items) + assert span["attributes"]["sentry.op"] == "gen_ai.embeddings" + span_data = span["attributes"] + else: + events = capture_events() + + with start_transaction(name="openai tx"): + response = await client.embeddings.create( + input="hello", model="text-embedding-3-large" + ) + + assert len(response.data[0].embedding) == 3 + + tx = events[0] + assert tx["type"] == "transaction" + span = tx["spans"][0] + assert span["op"] == "gen_ai.embeddings" + span_data = span["data"] + + assert span_data[SPANDATA.GEN_AI_SYSTEM] == "openai" + assert span_data[SPANDATA.GEN_AI_OPERATION_NAME] == "embeddings" + assert span_data[SPANDATA.GEN_AI_REQUEST_MODEL] == "text-embedding-3-large" + + if expect_input: + assert json.loads(span_data[SPANDATA.GEN_AI_EMBEDDINGS_INPUT]) == ["hello"] + else: + assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT not in span_data + + assert span_data["gen_ai.usage.input_tokens"] == 20 + assert span_data["gen_ai.usage.total_tokens"] == 30 + + @pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize("stream_gen_ai_spans", [True, False]) @pytest.mark.parametrize( From c41a9ce1575815a1c355df7e6068eec58223dab6 Mon Sep 17 00:00:00 2001 From: Erica Pisani Date: Fri, 31 Jul 2026 12:17:39 -0400 Subject: [PATCH 2/3] lint --- sentry_sdk/integrations/openai.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sentry_sdk/integrations/openai.py b/sentry_sdk/integrations/openai.py index 8baaf5631f..304a3a0899 100644 --- a/sentry_sdk/integrations/openai.py +++ b/sentry_sdk/integrations/openai.py @@ -673,6 +673,7 @@ def _set_embeddings_input_data( span, SPANDATA.GEN_AI_EMBEDDINGS_INPUT, messages_data, unpack=False ) + def _set_common_output_data( span: "Union[Span, StreamedSpan]", response: "Any", From a033d030710cfa8e5930b3a8f4995c31114c7593 Mon Sep 17 00:00:00 2001 From: Erica Pisani Date: Fri, 31 Jul 2026 15:26:18 -0400 Subject: [PATCH 3/3] feat(openai): Make include_prompts override data collection when set Change the default of OpenAIIntegration's include_prompts from True to None (unset). When unset, prompt collection follows the global data_collection.gen_ai.inputs setting; when explicitly set, it takes precedence over the global setting in both directions. The unset default is resolved lazily at call time rather than in __init__, so integrations constructed before sentry_sdk.init() still see the active client's options. Pre-data-collection behavior (defaults to True, subject to send_default_pii) is preserved. Refs PY-2588 --- sentry_sdk/integrations/openai.py | 114 +++++++----- tests/integrations/openai/test_openai.py | 220 ++++++++++++++++++++--- 2 files changed, 271 insertions(+), 63 deletions(-) diff --git a/sentry_sdk/integrations/openai.py b/sentry_sdk/integrations/openai.py index 304a3a0899..c080c0db8d 100644 --- a/sentry_sdk/integrations/openai.py +++ b/sentry_sdk/integrations/openai.py @@ -114,7 +114,7 @@ class OpenAIIntegration(Integration): def __init__( self: "OpenAIIntegration", - include_prompts: bool = True, + include_prompts: "Optional[bool]" = None, tiktoken_encoding_name: "Optional[str]" = None, ) -> None: self.include_prompts = include_prompts @@ -325,6 +325,18 @@ def _calculate_responses_token_usage( ) +def _set_tool_definitions_responses_api( + set_on_span: "Callable[[str, Any], None]", + kwargs: "dict[str, Any]", +) -> None: + tools = kwargs.get("tools") + if tools is not None and _is_given(tools): + set_on_span( + SPANDATA.GEN_AI_TOOL_DEFINITIONS, + json.dumps(_transform_tool_definitions_responses(tools)), + ) + + def _set_responses_api_input_data( span: "Union[Span, StreamedSpan]", kwargs: "dict[str, Any]", @@ -371,33 +383,24 @@ def _set_responses_api_input_data( client_options = sentry_sdk.get_client().options if has_data_collection_enabled(client_options): - if ( - integration.include_prompts - and client_options["data_collection"]["gen_ai"]["inputs"] - ): - tools = kwargs.get("tools") - if tools is not None and _is_given(tools): - set_on_span( - SPANDATA.GEN_AI_TOOL_DEFINITIONS, - json.dumps(_transform_tool_definitions_responses(tools)), - ) + if integration.include_prompts is not None: + if integration.include_prompts: + _set_tool_definitions_responses_api(set_on_span, kwargs) + elif client_options["data_collection"]["gen_ai"]["inputs"]: + _set_tool_definitions_responses_api(set_on_span, kwargs) else: # Pre-data collection this was always set, so this needs to be left here for now until # we deprecate `send_default_pii`. Once we do, this 'else' branch should be removed, # and the above branch placed below the "if not should_send_default_pii() or not integration.include_prompts" # line below - tools = kwargs.get("tools") - if tools is not None and _is_given(tools): - set_on_span( - SPANDATA.GEN_AI_TOOL_DEFINITIONS, - json.dumps(_transform_tool_definitions_responses(tools)), - ) + _set_tool_definitions_responses_api(set_on_span, kwargs) if has_data_collection_enabled(client_options): # This takes precedence over the global data collection settings - if not integration.include_prompts: - return - if not client_options["data_collection"]["gen_ai"]["inputs"]: + if integration.include_prompts is not None: + if not integration.include_prompts: + return + elif not client_options["data_collection"]["gen_ai"]["inputs"]: return elif not should_send_default_pii() or not integration.include_prompts: return @@ -478,6 +481,18 @@ def _set_responses_api_input_data( ) +def _set_tool_definitions_completions_api( + set_on_span: "Callable[[str, Any], None]", + kwargs: "dict[str, Any]", +) -> None: + tools = kwargs.get("tools") + if tools is not None and _is_given(tools): + set_on_span( + SPANDATA.GEN_AI_TOOL_DEFINITIONS, + json.dumps(_transform_tool_definitions_completions(tools)), + ) + + def _set_completions_api_input_data( span: "Union[Span, StreamedSpan]", kwargs: "dict[str, Any]", @@ -519,27 +534,17 @@ def _set_completions_api_input_data( client = sentry_sdk.get_client() if has_data_collection_enabled(client.options): - if ( - integration.include_prompts - and client.options["data_collection"]["gen_ai"]["inputs"] - ): - tools = kwargs.get("tools") - if tools is not None and _is_given(tools): - set_on_span( - SPANDATA.GEN_AI_TOOL_DEFINITIONS, - json.dumps(_transform_tool_definitions_completions(tools)), - ) + if integration.include_prompts is not None: + if integration.include_prompts: + _set_tool_definitions_completions_api(set_on_span, kwargs) + elif client.options["data_collection"]["gen_ai"]["inputs"]: + _set_tool_definitions_completions_api(set_on_span, kwargs) else: # Pre-data collection this was always set, so this needs to be left here for now until # we deprecate `send_default_pii`. Once we do, this 'else' branch should be removed, # and the above branch placed below the "if not should_send_default_pii() or not integration.include_prompts" # line below - tools = kwargs.get("tools") - if tools is not None and _is_given(tools): - set_on_span( - SPANDATA.GEN_AI_TOOL_DEFINITIONS, - json.dumps(_transform_tool_definitions_completions(tools)), - ) + _set_tool_definitions_completions_api(set_on_span, kwargs) messages: "Optional[Union[str, Iterable[ChatCompletionMessageParam]]]" = kwargs.get( "messages" @@ -547,9 +552,10 @@ def _set_completions_api_input_data( if has_data_collection_enabled(client.options): # This takes precedence over the global data collection settings - if not integration.include_prompts: - return - if not client.options["data_collection"]["gen_ai"]["inputs"]: + if integration.include_prompts is not None: + if not integration.include_prompts: + return + elif not client.options["data_collection"]["gen_ai"]["inputs"]: return elif not should_send_default_pii() or not integration.include_prompts: return @@ -628,9 +634,10 @@ def _set_embeddings_input_data( client = sentry_sdk.get_client() if has_data_collection_enabled(client.options): # This takes precedence over the global data collection settings - if not integration.include_prompts: - return - if not client.options["data_collection"]["gen_ai"]["inputs"]: + if integration.include_prompts is not None: + if not integration.include_prompts: + return + elif not client.options["data_collection"]["gen_ai"]["inputs"]: return elif not should_send_default_pii() or not integration.include_prompts: return @@ -763,12 +770,25 @@ def _set_common_output_data( span.__exit__(None, None, None) +def _resolve_include_prompts( + integration: "OpenAIIntegration", options: "Optional[dict[str, Any]]" +) -> None: + if integration.include_prompts is None and not has_data_collection_enabled(options): + # Pre-data-collection behavior: prompts were included by default (subject + # to send_default_pii). Resolve the unset default here, at call time, + # rather than in __init__ so that integrations constructed before + # sentry_sdk.init() still see the active client's options. + integration.include_prompts = True + + def _new_sync_chat_completion(f: "Any", *args: "Any", **kwargs: "Any") -> "Any": client = sentry_sdk.get_client() integration = client.get_integration(OpenAIIntegration) if integration is None: return f(*args, **kwargs) + _resolve_include_prompts(integration, client.options) + if "messages" not in kwargs: # invalid call (in all versions of openai), let it return error return f(*args, **kwargs) @@ -850,6 +870,8 @@ async def _new_async_chat_completion(f: "Any", *args: "Any", **kwargs: "Any") -> if integration is None: return await f(*args, **kwargs) + _resolve_include_prompts(integration, client.options) + if "messages" not in kwargs: # invalid call (in all versions of openai), let it return error return await f(*args, **kwargs) @@ -1280,6 +1302,8 @@ def _new_sync_embeddings_create(f: "Any", *args: "Any", **kwargs: "Any") -> "Any if integration is None: return f(*args, **kwargs) + _resolve_include_prompts(integration, client.options) + model = kwargs.get("model") if has_span_streaming_enabled(client.options): @@ -1338,6 +1362,8 @@ async def _new_async_embeddings_create( if integration is None: return await f(*args, **kwargs) + _resolve_include_prompts(integration, client.options) + model = kwargs.get("model") if has_span_streaming_enabled(client.options): @@ -1418,6 +1444,8 @@ def _new_sync_responses_create(f: "Any", *args: "Any", **kwargs: "Any") -> "Any" if integration is None: return f(*args, **kwargs) + _resolve_include_prompts(integration, client.options) + model = kwargs.get("model") # Same bool handling as in https://github.com/openai/openai-python/blob/acd0c54d8a68efeedde0e5b4e6c310eef1ce7867/src/openai/resources/responses/responses.py#L940 @@ -1488,6 +1516,8 @@ async def _new_async_responses_create(f: "Any", *args: "Any", **kwargs: "Any") - if integration is None: return await f(*args, **kwargs) + _resolve_include_prompts(integration, client.options) + model = kwargs.get("model") # Same bool handling as in https://github.com/openai/openai-python/blob/acd0c54d8a68efeedde0e5b4e6c310eef1ce7867/src/openai/resources/responses/responses.py#L940 diff --git a/tests/integrations/openai/test_openai.py b/tests/integrations/openai/test_openai.py index e88b67e6d1..a929ed9752 100644 --- a/tests/integrations/openai/test_openai.py +++ b/tests/integrations/openai/test_openai.py @@ -158,6 +158,31 @@ async def __call__(self, *args, **kwargs): ] +@pytest.mark.parametrize( + "data_collection_enabled", + [True, False], + ids=["data-collection-enabled", "data-collection-disabled"], +) +@pytest.mark.parametrize("include_prompts", [None, True, False]) +def test_include_prompts_init_stores_passed_value( + sentry_init, data_collection_enabled, include_prompts +): + init_kwargs = { + "traces_sample_rate": 1.0, + "disabled_integrations": [StdlibIntegration], + } + if data_collection_enabled: + init_kwargs["_experiments"] = {"data_collection": {"gen_ai": {"inputs": True}}} + sentry_init(**init_kwargs) + + integration = OpenAIIntegration(include_prompts=include_prompts) + + # The constructor stores the value as passed; the None default is resolved + # lazily at call time based on the active client's options (covered by the + # data_collection parametrizations below). + assert integration.include_prompts is include_prompts + + @pytest.mark.skipif( OPENAI_VERSION <= (2, 10, 0), reason="ChatCompletionCustomToolParam is unavailable before.", @@ -686,13 +711,17 @@ def test_nonstreaming_chat_completion( pytest.param( {"gen_ai": {"inputs": False}}, True, - {}, - [ - SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS, - SPANDATA.GEN_AI_REQUEST_MESSAGES, - SPANDATA.GEN_AI_TOOL_DEFINITIONS, - ], - id="inputs-disabled", + { + SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS: json.dumps( + [{"type": "text", "content": "You are a helpful assistant."}] + ), + SPANDATA.GEN_AI_REQUEST_MESSAGES: safe_serialize( + [{"role": "user", "content": "hello"}] + ), + SPANDATA.GEN_AI_TOOL_DEFINITIONS: safe_serialize(EXAMPLE_TOOLS), + }, + [], + id="include-prompts-enabled-overrides-inputs-disabled", ), pytest.param( {}, @@ -720,6 +749,47 @@ def test_nonstreaming_chat_completion( ], id="include-prompts-disabled-overrides-inputs-enabled", ), + pytest.param( + {"gen_ai": {"inputs": True}}, + None, + { + SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS: json.dumps( + [{"type": "text", "content": "You are a helpful assistant."}] + ), + SPANDATA.GEN_AI_REQUEST_MESSAGES: safe_serialize( + [{"role": "user", "content": "hello"}] + ), + SPANDATA.GEN_AI_TOOL_DEFINITIONS: safe_serialize(EXAMPLE_TOOLS), + }, + [], + id="include-prompts-default-follows-inputs-enabled", + ), + pytest.param( + {"gen_ai": {"inputs": False}}, + None, + {}, + [ + SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS, + SPANDATA.GEN_AI_REQUEST_MESSAGES, + SPANDATA.GEN_AI_TOOL_DEFINITIONS, + ], + id="include-prompts-default-follows-inputs-disabled", + ), + pytest.param( + {}, + None, + { + SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS: json.dumps( + [{"type": "text", "content": "You are a helpful assistant."}] + ), + SPANDATA.GEN_AI_REQUEST_MESSAGES: safe_serialize( + [{"role": "user", "content": "hello"}] + ), + SPANDATA.GEN_AI_TOOL_DEFINITIONS: safe_serialize(EXAMPLE_TOOLS), + }, + [], + id="include-prompts-default-follows-default-enabled", + ), ], ) def test_completions_api_data_collection( @@ -3026,8 +3096,8 @@ def _collect_embeddings_span_data( {"gen_ai": {"inputs": False}}, True, True, - False, - id="inputs-disabled-overrides-pii-enabled", + True, + id="include-prompts-overrides-inputs-disabled", ), pytest.param( {}, @@ -3040,8 +3110,8 @@ def _collect_embeddings_span_data( {"gen_ai": {"inputs": False}}, False, True, - False, - id="inputs-disabled-and-pii-disabled", + True, + id="include-prompts-overrides-inputs-disabled-and-pii-disabled", ), pytest.param( {"gen_ai": {"inputs": True}}, @@ -3057,6 +3127,27 @@ def _collect_embeddings_span_data( False, id="no-experiment-falls-back-to-pii", ), + pytest.param( + {"gen_ai": {"inputs": True}}, + False, + None, + True, + id="include-prompts-default-follows-inputs-enabled", + ), + pytest.param( + {"gen_ai": {"inputs": False}}, + True, + None, + False, + id="include-prompts-default-follows-inputs-disabled", + ), + pytest.param( + {}, + False, + None, + True, + id="include-prompts-default-follows-default-enabled", + ), ], ) def test_embeddings_create_data_collection( @@ -3142,7 +3233,7 @@ def test_embeddings_create_data_collection_inputs_disabled_input_shapes( span_streaming, ): sentry_init( - integrations=[OpenAIIntegration(include_prompts=True)], + integrations=[OpenAIIntegration(include_prompts=False)], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=True, @@ -3442,8 +3533,8 @@ async def test_embeddings_create_async( {"gen_ai": {"inputs": False}}, True, True, - False, - id="inputs-disabled-overrides-pii-enabled", + True, + id="include-prompts-overrides-inputs-disabled", ), pytest.param( {}, @@ -3466,6 +3557,27 @@ async def test_embeddings_create_async( False, id="no-experiment-falls-back-to-pii", ), + pytest.param( + {"gen_ai": {"inputs": True}}, + False, + None, + True, + id="include-prompts-default-follows-inputs-enabled", + ), + pytest.param( + {"gen_ai": {"inputs": False}}, + True, + None, + False, + id="include-prompts-default-follows-inputs-disabled", + ), + pytest.param( + {}, + False, + None, + True, + id="include-prompts-default-follows-default-enabled", + ), ], ) async def test_embeddings_create_async_data_collection( @@ -5293,14 +5405,23 @@ def test_ai_client_span_responses_api( "input": "How do I check if a Python object is an instance of a class?", "tools": EXAMPLE_TOOLS, }, - {}, - [ - SPANDATA.GEN_AI_REQUEST_MESSAGES, - SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS, - SPANDATA.GEN_AI_TOOL_DEFINITIONS, - ], + { + SPANDATA.GEN_AI_REQUEST_MESSAGES: safe_serialize( + ["How do I check if a Python object is an instance of a class?"] + ), + SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS: safe_serialize( + [ + { + "type": "text", + "content": "You are a coding assistant that talks like a pirate.", + } + ] + ), + SPANDATA.GEN_AI_TOOL_DEFINITIONS: safe_serialize(EXAMPLE_TOOLS), + }, + [], True, - id="inputs-disabled", + id="include-prompts-enabled-overrides-inputs-disabled", ), pytest.param( {}, @@ -5346,6 +5467,63 @@ def test_ai_client_span_responses_api( False, id="include-prompts-disabled-overrides-inputs-enabled", ), + pytest.param( + {"gen_ai": {"inputs": True}}, + { + "instructions": "You are a coding assistant that talks like a pirate.", + "input": "How do I check if a Python object is an instance of a class?", + "tools": EXAMPLE_TOOLS, + }, + { + SPANDATA.GEN_AI_REQUEST_MESSAGES: safe_serialize( + ["How do I check if a Python object is an instance of a class?"] + ), + SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS: safe_serialize( + [ + { + "type": "text", + "content": "You are a coding assistant that talks like a pirate.", + } + ] + ), + SPANDATA.GEN_AI_TOOL_DEFINITIONS: safe_serialize(EXAMPLE_TOOLS), + }, + [], + None, + id="include-prompts-default-follows-inputs-enabled", + ), + pytest.param( + {"gen_ai": {"inputs": False}}, + { + "instructions": "You are a coding assistant that talks like a pirate.", + "input": "How do I check if a Python object is an instance of a class?", + "tools": EXAMPLE_TOOLS, + }, + {}, + [ + SPANDATA.GEN_AI_REQUEST_MESSAGES, + SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS, + SPANDATA.GEN_AI_TOOL_DEFINITIONS, + ], + None, + id="include-prompts-default-follows-inputs-disabled", + ), + pytest.param( + {}, + { + "input": "How do I check if a Python object is an instance of a class?", + "tools": EXAMPLE_TOOLS, + }, + { + SPANDATA.GEN_AI_REQUEST_MESSAGES: safe_serialize( + ["How do I check if a Python object is an instance of a class?"] + ), + SPANDATA.GEN_AI_TOOL_DEFINITIONS: safe_serialize(EXAMPLE_TOOLS), + }, + [SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS], + None, + id="include-prompts-default-follows-default-enabled", + ), ], ) @pytest.mark.skipif(SKIP_RESPONSES_TESTS, reason="Responses API not available")