From 22f20458bc48ff4ea4b527ceda60983166ecf406 Mon Sep 17 00:00:00 2001 From: lelia <2418071+lelia@users.noreply.github.com> Date: Tue, 4 Aug 2026 23:50:58 -0400 Subject: [PATCH 1/8] Poll diff-scans endpoints for scan comparison instead of streaming The scan comparison (fullscans.stream_diff) held a single HTTP connection open, fully idle, while the API computed the diff. Network middleboxes with TCP idle timeouts - notably Azure NAT gateways, which default to 4 minutes - kill that connection with a RST, surfacing as intermittent "Connection reset by peer" / blank "API Error:" failures on the final comparison step of long scans (CE-354). The comparison now creates a diff-scan resource (POST /orgs/{org}/diff-scans/from-ids) and polls GET /orgs/{org}/diff-scans/{id}?cached=true with short bounded requests: 202 while the diff is computing, 200 with the result once ready. No request is ever idle long enough to be reaped, and the poll interval backs off 5s -> 30s to stay quota-friendly (each poll costs 1 quota unit). Transient poll failures retry; a 30-minute backstop guards against a diff scan that never completes. Any failure of the new flow (e.g. org tokens missing the diff-scans:create / diff-scans:list / full-scans:list scopes) logs a warning and falls back to the legacy streaming comparison, so the change is transparent to existing users. Requires socketdev>=3.4.0 for diffscans.get query-param/202 support. Co-Authored-By: Claude Fable 5 --- CHANGELOG.md | 19 +++ socketsecurity/core/__init__.py | 179 ++++++++++++++++++++++----- tests/core/conftest.py | 21 ++++ tests/core/test_diff_scan_polling.py | 103 +++++++++++++++ tests/core/test_sdk_methods.py | 32 +++-- 5 files changed, 313 insertions(+), 41 deletions(-) create mode 100644 tests/core/test_diff_scan_polling.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 202c6ed2..d974eee5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,25 @@ ## 2.6.0 +### Changed: scan comparison now polls the diff-scans endpoints + +- Diff mode no longer holds a single idle HTTP connection open while the API + computes the scan comparison. The CLI now creates a diff-scan resource + (`POST /orgs/{org}/diff-scans/from-ids`) and polls + `GET /orgs/{org}/diff-scans/{id}?cached=true` with short, bounded requests + until the comparison is ready (HTTP 200 instead of 202). This fixes + intermittent `Connection reset by peer` failures on the final comparison + step when scans take several minutes to compare and network middleboxes + (e.g. Azure NAT gateways, which default to a 4-minute TCP idle timeout) + reap the idle connection (CE-354). +- The change is transparent: no flags or workflow changes are needed. If the + org API token is missing the `diff-scans:create`, `diff-scans:list` or + `full-scans:list` scopes — or the new flow fails for any other reason — the + CLI logs a warning and falls back to the legacy streaming comparison. +- Requires `socketdev>=3.4.0`. + +## 2.6.0 + ### Changed: pin all Python dependencies - Pinned every runtime dependency in `pyproject.toml` to an exact version; diff --git a/socketsecurity/core/__init__.py b/socketsecurity/core/__init__.py index a372de4d..69220587 100644 --- a/socketsecurity/core/__init__.py +++ b/socketsecurity/core/__init__.py @@ -15,7 +15,7 @@ from socketsecurity.config import CliConfig from socketdev import socketdev from socketdev.exceptions import APIFailure -from socketdev.fullscans import FullScanParams, SocketArtifact +from socketdev.fullscans import DiffArtifacts, FullScanParams, SocketArtifact from socketdev.org import Organization from socketdev.repos import RepositoryInfo import copy @@ -92,6 +92,25 @@ FULL_SCAN_UPLOAD_MAX_ATTEMPTS = len(FULL_SCAN_UPLOAD_BACKOFF_SCHEDULE_SECONDS) FULL_SCAN_UPLOAD_BACKOFF_JITTER_SECONDS = 2.0 +# Diff-scan polling policy. The legacy scan comparison (fullscans.stream_diff) holds a +# single HTTP connection open, fully idle, while the backend computes the diff; network +# middleboxes with TCP idle timeouts (notably Azure NAT gateways, which default to +# 4 minutes) kill that connection with a RST, surfacing as an intermittent +# ConnectionResetError on large scans (CE-354). The diff-scans flow instead creates a +# diff-scan resource and polls its cached endpoint with short bounded requests: the API +# answers 202 while the comparison is still computing and 200 with the result once it is +# ready, so no connection is ever idle long enough to be reaped. +# +# Each poll consumes 1 unit of API quota, so the interval backs off toward +# DIFF_SCAN_POLL_MAX_INTERVAL_SECONDS to stay quota-friendly on comparisons that take +# minutes to compute. The timeout is a backstop against a diff scan that never +# completes; on expiry (or any other failure of this flow) the caller falls back to the +# legacy streaming comparison rather than failing the scan outright. +DIFF_SCAN_POLL_INITIAL_INTERVAL_SECONDS = 5.0 +DIFF_SCAN_POLL_MAX_INTERVAL_SECONDS = 30.0 +DIFF_SCAN_POLL_BACKOFF_MULTIPLIER = 1.5 +DIFF_SCAN_POLL_TIMEOUT_SECONDS = 30 * 60.0 + def _humanize_alert_type(alert_type: str) -> str: """Convert a camelCase/PascalCase alert type into a Title-Cased label. @@ -1303,6 +1322,93 @@ def get_license_text_via_purl(self, packages: dict[str, Package], batch_size: in return packages + def get_diff_scan_artifacts( + self, + head_full_scan_id: str, + new_full_scan_id: str, + include_license_details: bool = False + ) -> DiffArtifacts: + """Compare two full scans via the diff-scans endpoints, polling for the result. + + Creates a diff-scan resource from the two full scan IDs, then polls + ``GET /orgs/{org}/diff-scans/{id}?cached=true`` until the API returns the + computed comparison (200) instead of a processing status (202). Unlike the + legacy ``fullscans.stream_diff`` call, no request is ever left idle while + the backend computes, so the comparison survives network idle timeouts + (CE-354). See the DIFF_SCAN_POLL_* constants for the polling policy. + + Requires an org token with the ``diff-scans:create``, ``diff-scans:list`` + and ``full-scans:list`` scopes; callers are expected to catch failures and + fall back to the legacy streaming comparison. + + Args: + head_full_scan_id: The before/base full scan ID + new_full_scan_id: The after/head full scan ID + include_license_details: Whether to keep embedded per-package license + details in the response (see get_added_and_removed_packages for + why this defaults to False) + + Returns: + DiffArtifacts with the added/removed/unchanged/replaced/updated lists + """ + create_params = { + "before": head_full_scan_id, + "after": new_full_scan_id, + "description": f"Socket Security CLI v{__version__} scan comparison", + # A rerun against the same pair of scans returns the existing diff + # scan instead of failing with a 409. + "on_duplicate": "redirect", + } + result = self.sdk.diffscans.create_from_ids(self.config.org_slug, create_params) + diff_scan = result.get("diff_scan") or {} + diff_scan_id = diff_scan.get("id") + if not diff_scan_id: + raise Exception(f"Error creating diff scan: unexpected response: {str(result)[:500]}") + # An on_duplicate redirect can land on an already-computed diff scan, in + # which case the create response already carries the artifacts. + artifacts_dict = diff_scan.get("artifacts") + + poll_params = { + "cached": "true", + "omit_license_details": "false" if include_license_details else "true", + } + deadline = time.monotonic() + DIFF_SCAN_POLL_TIMEOUT_SECONDS + interval = DIFF_SCAN_POLL_INITIAL_INTERVAL_SECONDS + while artifacts_dict is None: + try: + response = self.sdk.diffscans.get(self.config.org_slug, diff_scan_id, params=poll_params) + except APIFailure as error: + if not error.is_transient_error(): + raise + # A dropped/timed-out poll is retryable: the diff scan keeps + # computing server-side regardless of what happens to any one poll. + log.warning( + f"Transient error polling diff scan {diff_scan_id} " + f"({type(error).__name__}), retrying in {interval:.0f}s" + ) + response = {"status": "processing"} + if response.get("status") != "processing": + scan = response.get("diff_scan") or {} + if scan.get("artifacts") is None: + raise Exception( + f"Error fetching diff scan {diff_scan_id}: unexpected response: {str(response)[:500]}" + ) + artifacts_dict = scan["artifacts"] + break + if time.monotonic() >= deadline: + raise Exception( + f"Timed out waiting for diff scan {diff_scan_id} after " + f"{DIFF_SCAN_POLL_TIMEOUT_SECONDS:.0f} seconds" + ) + log.debug(f"Diff scan {diff_scan_id} still processing, polling again in {interval:.0f}s") + time.sleep(interval) + interval = min(interval * DIFF_SCAN_POLL_BACKOFF_MULTIPLIER, DIFF_SCAN_POLL_MAX_INTERVAL_SECONDS) + + return DiffArtifacts.from_dict({ + key: artifacts_dict.get(key) or [] + for key in ("added", "removed", "unchanged", "replaced", "updated") + }) + def get_added_and_removed_packages( self, head_full_scan_id: str, @@ -1343,39 +1449,56 @@ def get_added_and_removed_packages( log.info(f"Comparing scans - Head scan ID: {head_full_scan_id}, New scan ID: {new_full_scan_id}") diff_start = time.time() + diff_artifacts = None try: - diff_report = ( - self.sdk.fullscans.stream_diff( - self.config.org_slug, - head_full_scan_id, - new_full_scan_id, - use_types=True, - include_license_details=str(include_license_details).lower() - ).data + diff_artifacts = self.get_diff_scan_artifacts( + head_full_scan_id, + new_full_scan_id, + include_license_details=include_license_details ) - except APIFailure as e: - log.error(f"API Error: {e}") - if self.cli_config and self.cli_config.disable_blocking: - sys.exit(0) - sys.exit(1) - except Exception as e: - import traceback - log.error(f"Error getting diff report: {str(e)}") - log.error(f"Stack trace:\n{traceback.format_exc()}") - raise + except Exception as error: + # SDK error messages can span many lines (path + response headers); the + # first line carries the status, which is all the warning needs. + error_summary = str(error).strip().splitlines()[0] if str(error).strip() else "" + log.warning( + f"Diff scan comparison failed with {type(error).__name__}({error_summary}), " + "falling back to the streaming scan comparison" + ) + + if diff_artifacts is None: + try: + diff_artifacts = ( + self.sdk.fullscans.stream_diff( + self.config.org_slug, + head_full_scan_id, + new_full_scan_id, + use_types=True, + include_license_details=str(include_license_details).lower() + ).data.artifacts + ) + except APIFailure as e: + log.error(f"API Error: {e}") + if self.cli_config and self.cli_config.disable_blocking: + sys.exit(0) + sys.exit(1) + except Exception as e: + import traceback + log.error(f"Error getting diff report: {str(e)}") + log.error(f"Stack trace:\n{traceback.format_exc()}") + raise diff_end = time.time() log.info(f"Diff Report Gathered in {diff_end - diff_start:.2f} seconds") log.info("Diff report artifact counts:") - log.info(f"Added: {len(diff_report.artifacts.added)}") - log.info(f"Removed: {len(diff_report.artifacts.removed)}") - log.info(f"Unchanged: {len(diff_report.artifacts.unchanged)}") - log.info(f"Replaced: {len(diff_report.artifacts.replaced)}") - log.info(f"Updated: {len(diff_report.artifacts.updated)}") - - added_artifacts = diff_report.artifacts.added + diff_report.artifacts.updated - removed_artifacts = diff_report.artifacts.removed + diff_report.artifacts.replaced - unchanged_artifacts = diff_report.artifacts.unchanged + log.info(f"Added: {len(diff_artifacts.added)}") + log.info(f"Removed: {len(diff_artifacts.removed)}") + log.info(f"Unchanged: {len(diff_artifacts.unchanged)}") + log.info(f"Replaced: {len(diff_artifacts.replaced)}") + log.info(f"Updated: {len(diff_artifacts.updated)}") + + added_artifacts = diff_artifacts.added + diff_artifacts.updated + removed_artifacts = diff_artifacts.removed + diff_artifacts.replaced + unchanged_artifacts = diff_artifacts.unchanged added_packages: Dict[str, Package] = {} removed_packages: Dict[str, Package] = {} diff --git a/tests/core/conftest.py b/tests/core/conftest.py index 381c2c3f..ae6b10c0 100644 --- a/tests/core/conftest.py +++ b/tests/core/conftest.py @@ -87,6 +87,22 @@ def stream_diff_response(data_dir, load_json): }) +@pytest.fixture +def diff_scan_get_response(data_dir, load_json): + """GET /orgs/{org}/diff-scans/{id} response built from the stream_diff fixture. + + The diff-scans endpoint returns the same artifact shape as the legacy + streaming diff, wrapped in a diff_scan object. + """ + json_data = load_json(data_dir / "fullscans" / "diff" / "stream_diff.json") + return { + "diff_scan": { + "id": "diff-scan-123", + "artifacts": json_data["data"]["artifacts"], + } + } + + @@ -138,6 +154,7 @@ def mock_sdk_with_responses( new_scan_metadata, new_scan_stream, stream_diff_response, + diff_scan_get_response, create_full_scan_response, ): sdk = mock_socket_sdk.return_value @@ -173,4 +190,8 @@ def mock_sdk_with_responses( lambda org_slug, head_id, new_id, **kwargs: stream_diff_response ) + # Diff-scans endpoints (primary scan-comparison path) + sdk.diffscans.create_from_ids.return_value = {"diff_scan": {"id": "diff-scan-123"}} + sdk.diffscans.get.return_value = diff_scan_get_response + return sdk diff --git a/tests/core/test_diff_scan_polling.py b/tests/core/test_diff_scan_polling.py new file mode 100644 index 00000000..b44b227b --- /dev/null +++ b/tests/core/test_diff_scan_polling.py @@ -0,0 +1,103 @@ +"""Tests for the diff-scans polling scan comparison (CE-354). + +The comparison must never hold an idle connection open: it creates a diff-scan +resource and polls the cached endpoint (202 while processing, 200 when ready), +falling back to the legacy streaming diff if the new flow is unavailable. +""" +import pytest +from socketdev.exceptions import APIConnectionError, APIFailure + +import socketsecurity.core as core_module +from socketsecurity.core import Core +from socketsecurity.core.socket_config import SocketConfig + + +@pytest.fixture +def core(mock_sdk_with_responses): + config = SocketConfig(api_key="test_key") + return Core(config=config, sdk=mock_sdk_with_responses) + + +@pytest.fixture +def no_sleep(mocker): + return mocker.patch("socketsecurity.core.time.sleep") + + +def test_polls_until_diff_scan_ready(core, diff_scan_get_response, no_sleep): + """202 processing responses are polled through until the 200 result arrives.""" + processing = {"status": "processing", "id": "diff-scan-123"} + core.sdk.diffscans.get.side_effect = [processing, processing, diff_scan_get_response] + + artifacts = core.get_diff_scan_artifacts("head", "new") + + assert core.sdk.diffscans.get.call_count == 3 + assert no_sleep.call_count == 2 # slept between polls, never during them + assert len(artifacts.added) > 0 + + +def test_poll_interval_backs_off(core, diff_scan_get_response, no_sleep, monkeypatch): + """The poll interval grows toward the max so long comparisons stay quota-friendly.""" + monkeypatch.setattr(core_module, "DIFF_SCAN_POLL_INITIAL_INTERVAL_SECONDS", 4.0) + monkeypatch.setattr(core_module, "DIFF_SCAN_POLL_MAX_INTERVAL_SECONDS", 10.0) + processing = {"status": "processing", "id": "diff-scan-123"} + core.sdk.diffscans.get.side_effect = [processing] * 4 + [diff_scan_get_response] + + core.get_diff_scan_artifacts("head", "new") + + waits = [call.args[0] for call in no_sleep.call_args_list] + assert waits == [4.0, 6.0, 9.0, 10.0] # 1.5x backoff, capped at the max + + +def test_transient_poll_error_is_retried(core, diff_scan_get_response, no_sleep): + """A dropped poll doesn't abandon the flow - the diff keeps computing server-side.""" + core.sdk.diffscans.get.side_effect = [APIConnectionError("reset"), diff_scan_get_response] + + artifacts = core.get_diff_scan_artifacts("head", "new") + + assert core.sdk.diffscans.get.call_count == 2 + assert len(artifacts.added) > 0 + + +def test_non_transient_poll_error_raises(core, no_sleep): + """Deterministic API errors (e.g. 403 missing scopes) propagate to the caller.""" + core.sdk.diffscans.get.side_effect = APIFailure("forbidden", status_code=403) + + with pytest.raises(APIFailure): + core.get_diff_scan_artifacts("head", "new") + + +def test_poll_timeout_raises(core, no_sleep, monkeypatch): + """A diff scan that never completes hits the polling backstop.""" + monkeypatch.setattr(core_module, "DIFF_SCAN_POLL_TIMEOUT_SECONDS", 0.0) + core.sdk.diffscans.get.return_value = {"status": "processing", "id": "diff-scan-123"} + + with pytest.raises(Exception, match="Timed out waiting for diff scan"): + core.get_diff_scan_artifacts("head", "new") + + +def test_duplicate_redirect_uses_embedded_artifacts(core, diff_scan_get_response): + """An on_duplicate redirect can return the computed diff scan straight away.""" + core.sdk.diffscans.create_from_ids.return_value = diff_scan_get_response + + artifacts = core.get_diff_scan_artifacts("head", "new") + + core.sdk.diffscans.get.assert_not_called() + assert len(artifacts.added) > 0 + + +def test_fallback_to_streaming_diff_on_failure(core): + """If the diff-scans flow fails (e.g. token missing the diff-scans scopes), + the comparison falls back to the legacy streaming diff transparently.""" + core.sdk.diffscans.create_from_ids.side_effect = APIFailure("forbidden", status_code=403) + + added, removed, all_packages = core.get_added_and_removed_packages("head", "new") + + core.sdk.fullscans.stream_diff.assert_called_once_with( + core.config.org_slug, + "head", + "new", + use_types=True, + include_license_details="false", + ) + assert "dp3" in added + assert "dp2" in removed diff --git a/tests/core/test_sdk_methods.py b/tests/core/test_sdk_methods.py index 9b1ce449..3532bd00 100644 --- a/tests/core/test_sdk_methods.py +++ b/tests/core/test_sdk_methods.py @@ -225,19 +225,27 @@ def test_get_added_and_removed_packages(core): """Test getting added and removed packages between two scans""" # Get two different scans to compare added, removed, all_packages = core.get_added_and_removed_packages("head", "new") - - # Verify SDK was called correctly. - # include_license_details defaults to "false": the diff path never consumes + + # Verify SDK was called correctly: the comparison goes through the diff-scans + # endpoints (create + poll) rather than the legacy streaming diff, so no + # connection is left idle while the backend computes (CE-354). + create_args = core.sdk.diffscans.create_from_ids.call_args + assert create_args[0][0] == core.config.org_slug + create_params = create_args[0][1] + assert create_params["before"] == "head" + assert create_params["after"] == "new" + assert create_params["on_duplicate"] == "redirect" + + # include_license_details defaults to False: the diff path never consumes # embedded license data (license artifacts come from the PURL endpoint), so # requesting it only bloats the response and risks the truncation # crash on large repos. - core.sdk.fullscans.stream_diff.assert_called_once_with( + core.sdk.diffscans.get.assert_called_once_with( core.config.org_slug, - "head", - "new", - use_types=True, - include_license_details="false", + "diff-scan-123", + params={"cached": "true", "omit_license_details": "true"}, ) + core.sdk.fullscans.stream_diff.assert_not_called() # Verify the results # Added packages @@ -255,12 +263,10 @@ def test_get_added_and_removed_packages_license_override(core): """The include_license_details override seam still works when explicitly requested.""" core.get_added_and_removed_packages("head", "new", include_license_details=True) - core.sdk.fullscans.stream_diff.assert_called_once_with( + core.sdk.diffscans.get.assert_called_once_with( core.config.org_slug, - "head", - "new", - use_types=True, - include_license_details="true", + "diff-scan-123", + params={"cached": "true", "omit_license_details": "false"}, ) def test_empty_alerts_preserved(core): From 314b77adee4dc3396f7162b335d84ac328f063ad Mon Sep 17 00:00:00 2001 From: lelia <2418071+lelia@users.noreply.github.com> Date: Wed, 5 Aug 2026 00:19:36 -0400 Subject: [PATCH 2/8] Drop ignored omit_license_details param from cached diff-scan polls The API ignores omit_license_details when cached=true - cached diff-scan results always embed license details - so sending the param suggested a lean-response guarantee the polling path doesn't have. Document the caveat instead: if the heavier payload ever gets truncated on a huge dependency tree, JSON parsing fails and the caller already falls back to the legacy streaming comparison, which still requests the lean payload. include_license_details now only governs that fallback call. Flagged by Cursor Bugbot on #284. Co-Authored-By: Claude Fable 5 --- socketsecurity/core/__init__.py | 35 +++++++++++++++++++++------------ tests/core/test_sdk_methods.py | 26 +++++++++++++++--------- 2 files changed, 39 insertions(+), 22 deletions(-) diff --git a/socketsecurity/core/__init__.py b/socketsecurity/core/__init__.py index 69220587..70ffd271 100644 --- a/socketsecurity/core/__init__.py +++ b/socketsecurity/core/__init__.py @@ -1325,8 +1325,7 @@ def get_license_text_via_purl(self, packages: dict[str, Package], batch_size: in def get_diff_scan_artifacts( self, head_full_scan_id: str, - new_full_scan_id: str, - include_license_details: bool = False + new_full_scan_id: str ) -> DiffArtifacts: """Compare two full scans via the diff-scans endpoints, polling for the result. @@ -1341,12 +1340,14 @@ def get_diff_scan_artifacts( and ``full-scans:list`` scopes; callers are expected to catch failures and fall back to the legacy streaming comparison. + Note that cached diff-scan responses always embed per-package license + details (the API ignores ``omit_license_details`` when ``cached=true``), + so unlike the legacy streaming comparison there is no lean-response + option here; see the comment on ``poll_params`` below. + Args: head_full_scan_id: The before/base full scan ID new_full_scan_id: The after/head full scan ID - include_license_details: Whether to keep embedded per-package license - details in the response (see get_added_and_removed_packages for - why this defaults to False) Returns: DiffArtifacts with the added/removed/unchanged/replaced/updated lists @@ -1368,10 +1369,15 @@ def get_diff_scan_artifacts( # which case the create response already carries the artifacts. artifacts_dict = diff_scan.get("artifacts") - poll_params = { - "cached": "true", - "omit_license_details": "false" if include_license_details else "true", - } + # cached=true is the polling contract (202 while computing, 200 when + # ready). The API ignores omit_license_details when cached=true - cached + # results always embed license details - so there is no lean-response + # option on this path (unlike stream_diff with + # include_license_details=false, the CE-224 mitigation). If that extra + # payload ever gets a response truncated on a huge dependency tree, + # response.json() fails and the caller falls back to the legacy + # streaming comparison, which still requests the lean payload. + poll_params = {"cached": "true"} deadline = time.monotonic() + DIFF_SCAN_POLL_TIMEOUT_SECONDS interval = DIFF_SCAN_POLL_INITIAL_INTERVAL_SECONDS while artifacts_dict is None: @@ -1421,8 +1427,12 @@ def get_added_and_removed_packages( Args: head_full_scan_id: Previous scan (maybe None if first scan) new_full_scan_id: New scan just created - include_license_details: Whether to ask the diff endpoint to embed - per-package license attribution/details in the response. + include_license_details: Whether to ask the *legacy streaming* diff + endpoint to embed per-package license attribution/details in the + response. Only consulted on the fallback path: the primary + diff-scans path always receives embedded license details, since + the API ignores ``omit_license_details`` for cached reads (see + get_diff_scan_artifacts). Defaults to ``False`` on purpose. The diff endpoint exists to compare alerts between two scans; the license fields it can embed @@ -1453,8 +1463,7 @@ def get_added_and_removed_packages( try: diff_artifacts = self.get_diff_scan_artifacts( head_full_scan_id, - new_full_scan_id, - include_license_details=include_license_details + new_full_scan_id ) except Exception as error: # SDK error messages can span many lines (path + response headers); the diff --git a/tests/core/test_sdk_methods.py b/tests/core/test_sdk_methods.py index 3532bd00..85c6825d 100644 --- a/tests/core/test_sdk_methods.py +++ b/tests/core/test_sdk_methods.py @@ -236,14 +236,14 @@ def test_get_added_and_removed_packages(core): assert create_params["after"] == "new" assert create_params["on_duplicate"] == "redirect" - # include_license_details defaults to False: the diff path never consumes - # embedded license data (license artifacts come from the PURL endpoint), so - # requesting it only bloats the response and risks the truncation - # crash on large repos. + # cached=true is the polling contract (202 while computing, 200 when ready). + # No omit_license_details param: the API ignores it for cached reads (cached + # results always embed license details), so sending it would only suggest a + # leanness guarantee this path doesn't have. core.sdk.diffscans.get.assert_called_once_with( core.config.org_slug, "diff-scan-123", - params={"cached": "true", "omit_license_details": "true"}, + params={"cached": "true"}, ) core.sdk.fullscans.stream_diff.assert_not_called() @@ -260,13 +260,21 @@ def test_get_added_and_removed_packages(core): assert "pypi/direct_package_1@1.6.0" in all_packages # Unchanged package is in full package map def test_get_added_and_removed_packages_license_override(core): - """The include_license_details override seam still works when explicitly requested.""" + """include_license_details only governs the legacy fallback path now: the + diff-scans path always receives embedded license details (the API ignores + omit_license_details for cached reads), so the seam must survive through to + the stream_diff call when the primary path is unavailable.""" + from socketdev.exceptions import APIFailure + + core.sdk.diffscans.create_from_ids.side_effect = APIFailure("forbidden", status_code=403) core.get_added_and_removed_packages("head", "new", include_license_details=True) - core.sdk.diffscans.get.assert_called_once_with( + core.sdk.fullscans.stream_diff.assert_called_once_with( core.config.org_slug, - "diff-scan-123", - params={"cached": "true", "omit_license_details": "false"}, + "head", + "new", + use_types=True, + include_license_details="true", ) def test_empty_alerts_preserved(core): From 25bd351912c24d870057526fe386edba06c5d1a6 Mon Sep 17 00:00:00 2001 From: lelia <2418071+lelia@users.noreply.github.com> Date: Wed, 5 Aug 2026 00:56:49 -0400 Subject: [PATCH 3/8] Keep duplicate diff scans on cached polling --- CHANGELOG.md | 3 +++ socketsecurity/core/__init__.py | 37 ++++++++++++++++++++++------ tests/core/test_diff_scan_polling.py | 27 +++++++++++++++++--- tests/core/test_sdk_methods.py | 2 +- 4 files changed, 56 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d974eee5..10f0b340 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,9 @@ step when scans take several minutes to compare and network middleboxes (e.g. Azure NAT gateways, which default to a 4-minute TCP idle timeout) reap the idle connection (CE-354). +- Duplicate scan pairs are resolved after an HTTP 409 and then polled through + the same cached endpoint. This avoids automatically following the API's 302 + duplicate redirect with an uncached, potentially long-lived GET request. - The change is transparent: no flags or workflow changes are needed. If the org API token is missing the `diff-scans:create`, `diff-scans:list` or `full-scans:list` scopes — or the new flow fails for any other reason — the diff --git a/socketsecurity/core/__init__.py b/socketsecurity/core/__init__.py index 70ffd271..bfe8bcb1 100644 --- a/socketsecurity/core/__init__.py +++ b/socketsecurity/core/__init__.py @@ -1356,17 +1356,38 @@ def get_diff_scan_artifacts( "before": head_full_scan_id, "after": new_full_scan_id, "description": f"Socket Security CLI v{__version__} scan comparison", - # A rerun against the same pair of scans returns the existing diff - # scan instead of failing with a 409. - "on_duplicate": "redirect", } - result = self.sdk.diffscans.create_from_ids(self.config.org_slug, create_params) - diff_scan = result.get("diff_scan") or {} + try: + result = self.sdk.diffscans.create_from_ids(self.config.org_slug, create_params) + diff_scan = result.get("diff_scan") or {} + response_summary = result + except APIFailure as error: + if error.status_code != 409: + raise + + # Do not use on_duplicate=redirect here. The SDK follows that 302 + # automatically with a GET that lacks cached=true, which can leave + # the connection idle while an existing diff scan is still computing. + # Resolve the duplicate resource explicitly so every result fetch + # continues through the bounded cached polling path below. + existing = self.sdk.diffscans.list( + self.config.org_slug, + params={ + "before_full_scan_id": head_full_scan_id, + "after_full_scan_id": new_full_scan_id, + "per_page": 1, + }, + ) + matches = existing.get("results") or [] + diff_scan = matches[0] if matches else {} + response_summary = existing + diff_scan_id = diff_scan.get("id") if not diff_scan_id: - raise Exception(f"Error creating diff scan: unexpected response: {str(result)[:500]}") - # An on_duplicate redirect can land on an already-computed diff scan, in - # which case the create response already carries the artifacts. + raise Exception( + "Error creating or resolving diff scan: " + f"unexpected response: {str(response_summary)[:500]}" + ) artifacts_dict = diff_scan.get("artifacts") # cached=true is the polling contract (202 while computing, 200 when diff --git a/tests/core/test_diff_scan_polling.py b/tests/core/test_diff_scan_polling.py index b44b227b..7c98a510 100644 --- a/tests/core/test_diff_scan_polling.py +++ b/tests/core/test_diff_scan_polling.py @@ -75,13 +75,32 @@ def test_poll_timeout_raises(core, no_sleep, monkeypatch): core.get_diff_scan_artifacts("head", "new") -def test_duplicate_redirect_uses_embedded_artifacts(core, diff_scan_get_response): - """An on_duplicate redirect can return the computed diff scan straight away.""" - core.sdk.diffscans.create_from_ids.return_value = diff_scan_get_response +def test_duplicate_conflict_uses_cached_polling(core, diff_scan_get_response): + """A duplicate is resolved explicitly so the SDK cannot follow an uncached redirect.""" + core.sdk.diffscans.create_from_ids.side_effect = APIFailure( + "duplicate", status_code=409 + ) + core.sdk.diffscans.list.return_value = { + "results": [{"id": "existing-diff-scan"}], + } artifacts = core.get_diff_scan_artifacts("head", "new") - core.sdk.diffscans.get.assert_not_called() + create_params = core.sdk.diffscans.create_from_ids.call_args.args[1] + assert "on_duplicate" not in create_params + core.sdk.diffscans.list.assert_called_once_with( + core.config.org_slug, + params={ + "before_full_scan_id": "head", + "after_full_scan_id": "new", + "per_page": 1, + }, + ) + core.sdk.diffscans.get.assert_called_once_with( + core.config.org_slug, + "existing-diff-scan", + params={"cached": "true"}, + ) assert len(artifacts.added) > 0 diff --git a/tests/core/test_sdk_methods.py b/tests/core/test_sdk_methods.py index 85c6825d..9edcbe9e 100644 --- a/tests/core/test_sdk_methods.py +++ b/tests/core/test_sdk_methods.py @@ -234,7 +234,7 @@ def test_get_added_and_removed_packages(core): create_params = create_args[0][1] assert create_params["before"] == "head" assert create_params["after"] == "new" - assert create_params["on_duplicate"] == "redirect" + assert "on_duplicate" not in create_params # cached=true is the polling contract (202 while computing, 200 when ready). # No omit_license_details param: the API ignores it for cached reads (cached From d147a311f57b674bd375a641e019ffc64db095d0 Mon Sep 17 00:00:00 2001 From: lelia <2418071+lelia@users.noreply.github.com> Date: Wed, 5 Aug 2026 16:54:16 -0400 Subject: [PATCH 4/8] Require bundled socketdev 3.4.2 release --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 10f0b340..f6d22e32 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,7 +20,7 @@ org API token is missing the `diff-scans:create`, `diff-scans:list` or `full-scans:list` scopes — or the new flow fails for any other reason — the CLI logs a warning and falls back to the legacy streaming comparison. -- Requires `socketdev>=3.4.0`. +- Requires `socketdev>=3.4.2`. ## 2.6.0 From a443b624586279439eb61c462b051d7ec7a02b68 Mon Sep 17 00:00:00 2001 From: lelia <2418071+lelia@users.noreply.github.com> Date: Wed, 5 Aug 2026 22:55:37 -0400 Subject: [PATCH 5/8] Stage CLI 2.6.1 --- CHANGELOG.md | 2 +- pyproject.toml | 2 +- socketsecurity/__init__.py | 2 +- uv.lock | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f6d22e32..c3bd578b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## 2.6.0 +## 2.6.1 ### Changed: scan comparison now polls the diff-scans endpoints diff --git a/pyproject.toml b/pyproject.toml index 1b49c2ca..c70b629b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "hatchling.build" [project] name = "socketsecurity" -version = "2.6.0" +version = "2.6.1" requires-python = ">= 3.11" license = {"file" = "LICENSE"} dependencies = [ diff --git a/socketsecurity/__init__.py b/socketsecurity/__init__.py index 2a2ecb9c..312c5053 100644 --- a/socketsecurity/__init__.py +++ b/socketsecurity/__init__.py @@ -1,3 +1,3 @@ __author__ = 'socket.dev' -__version__ = '2.6.0' +__version__ = '2.6.1' USER_AGENT = f'SocketPythonCLI/{__version__}' diff --git a/uv.lock b/uv.lock index fb5a540f..172c2af4 100644 --- a/uv.lock +++ b/uv.lock @@ -1282,7 +1282,7 @@ wheels = [ [[package]] name = "socketsecurity" -version = "2.6.0" +version = "2.6.1" source = { editable = "." } dependencies = [ { name = "beautifulsoup4" }, From 53a808621172c8ceaa6803740a0b3cb027a8575b Mon Sep 17 00:00:00 2001 From: lelia <2418071+lelia@users.noreply.github.com> Date: Wed, 5 Aug 2026 23:51:02 -0400 Subject: [PATCH 6/8] Require socketdev 3.5.0 --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c3bd578b..01fc8ea8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,7 +20,7 @@ org API token is missing the `diff-scans:create`, `diff-scans:list` or `full-scans:list` scopes — or the new flow fails for any other reason — the CLI logs a warning and falls back to the legacy streaming comparison. -- Requires `socketdev>=3.4.2`. +- Requires `socketdev>=3.5.0`. ## 2.6.0 From a1fd65a70c4099e3145076c2857abd57a26a2cf3 Mon Sep 17 00:00:00 2001 From: lelia <2418071+lelia@users.noreply.github.com> Date: Thu, 6 Aug 2026 00:14:05 -0400 Subject: [PATCH 7/8] Drop ticket references from code comments, workflows, and changelog Co-Authored-By: Claude Fable 5 Signed-off-by: lelia <2418071+lelia@users.noreply.github.com> --- CHANGELOG.md | 2 +- socketsecurity/core/__init__.py | 8 ++++---- tests/core/test_diff_scan_polling.py | 2 +- tests/core/test_sdk_methods.py | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 01fc8ea8..248d9f1c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ intermittent `Connection reset by peer` failures on the final comparison step when scans take several minutes to compare and network middleboxes (e.g. Azure NAT gateways, which default to a 4-minute TCP idle timeout) - reap the idle connection (CE-354). + reap the idle connection. - Duplicate scan pairs are resolved after an HTTP 409 and then polled through the same cached endpoint. This avoids automatically following the API's 302 duplicate redirect with an uncached, potentially long-lived GET request. diff --git a/socketsecurity/core/__init__.py b/socketsecurity/core/__init__.py index bfe8bcb1..7bd4a33e 100644 --- a/socketsecurity/core/__init__.py +++ b/socketsecurity/core/__init__.py @@ -96,7 +96,7 @@ # single HTTP connection open, fully idle, while the backend computes the diff; network # middleboxes with TCP idle timeouts (notably Azure NAT gateways, which default to # 4 minutes) kill that connection with a RST, surfacing as an intermittent -# ConnectionResetError on large scans (CE-354). The diff-scans flow instead creates a +# ConnectionResetError on large scans. The diff-scans flow instead creates a # diff-scan resource and polls its cached endpoint with short bounded requests: the API # answers 202 while the comparison is still computing and 200 with the result once it is # ready, so no connection is ever idle long enough to be reaped. @@ -1333,8 +1333,8 @@ def get_diff_scan_artifacts( ``GET /orgs/{org}/diff-scans/{id}?cached=true`` until the API returns the computed comparison (200) instead of a processing status (202). Unlike the legacy ``fullscans.stream_diff`` call, no request is ever left idle while - the backend computes, so the comparison survives network idle timeouts - (CE-354). See the DIFF_SCAN_POLL_* constants for the polling policy. + the backend computes, so the comparison survives network idle timeouts. + See the DIFF_SCAN_POLL_* constants for the polling policy. Requires an org token with the ``diff-scans:create``, ``diff-scans:list`` and ``full-scans:list`` scopes; callers are expected to catch failures and @@ -1394,7 +1394,7 @@ def get_diff_scan_artifacts( # ready). The API ignores omit_license_details when cached=true - cached # results always embed license details - so there is no lean-response # option on this path (unlike stream_diff with - # include_license_details=false, the CE-224 mitigation). If that extra + # include_license_details=false, the lean-payload mitigation). If that extra # payload ever gets a response truncated on a huge dependency tree, # response.json() fails and the caller falls back to the legacy # streaming comparison, which still requests the lean payload. diff --git a/tests/core/test_diff_scan_polling.py b/tests/core/test_diff_scan_polling.py index 7c98a510..be369429 100644 --- a/tests/core/test_diff_scan_polling.py +++ b/tests/core/test_diff_scan_polling.py @@ -1,4 +1,4 @@ -"""Tests for the diff-scans polling scan comparison (CE-354). +"""Tests for the diff-scans polling scan comparison. The comparison must never hold an idle connection open: it creates a diff-scan resource and polls the cached endpoint (202 while processing, 200 when ready), diff --git a/tests/core/test_sdk_methods.py b/tests/core/test_sdk_methods.py index 9edcbe9e..02967315 100644 --- a/tests/core/test_sdk_methods.py +++ b/tests/core/test_sdk_methods.py @@ -228,7 +228,7 @@ def test_get_added_and_removed_packages(core): # Verify SDK was called correctly: the comparison goes through the diff-scans # endpoints (create + poll) rather than the legacy streaming diff, so no - # connection is left idle while the backend computes (CE-354). + # connection is left idle while the backend computes. create_args = core.sdk.diffscans.create_from_ids.call_args assert create_args[0][0] == core.config.org_slug create_params = create_args[0][1] From 0791f41e4ee6900fe8376100f55a1f61574e7565 Mon Sep 17 00:00:00 2001 From: lelia <2418071+lelia@users.noreply.github.com> Date: Thu, 6 Aug 2026 00:26:39 -0400 Subject: [PATCH 8/8] Align changelog with pinned SDK dependency --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 248d9f1c..22976e66 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,7 +20,7 @@ org API token is missing the `diff-scans:create`, `diff-scans:list` or `full-scans:list` scopes — or the new flow fails for any other reason — the CLI logs a warning and falls back to the legacy streaming comparison. -- Requires `socketdev>=3.5.0`. +- Requires the pinned `socketdev==3.5.0` SDK. ## 2.6.0