Fix intermittent connection resets on scan comparison by polling the diff-scans endpoints

.github/workflows/e2e-test.yml

@@ -46,6 +46,11 @@ jobs:
               --enable-debug
             validate: tests/e2e/validate-reachability.sh
             setup-node: "true"
+            # The tier-1 reachability backend intermittently returns empty
+            # results while the CLI reports success (ENG-5093). The probe
+            # exits 0 when the facts file has alerted components; anything
+            # else is retried before validation fails the job.
+            retry-probe: bash tests/e2e/reach-facts-probe.sh tests/e2e/fixtures/simple-npm
 
           - name: gitlab
             args: >-
@@ -96,15 +101,54 @@ jobs:
       - name: Run Socket CLI
         env:
           SOCKET_SECURITY_API_KEY: ${{ secrets.SOCKET_CLI_API_TOKEN }}
+          RETRY_PROBE: ${{ matrix.retry-probe }}
         run: |
           set -o pipefail
-          socketcli ${{ matrix.args }} 2>&1 | tee /tmp/e2e-output.log
+          # Entries with retry-probe get up to 3 attempts: the probe exits 0
+          # when the scan output looks complete, and a run that fails it is
+          # re-run on the assumption of a transient backend failure. A
+          # persistent failure still reaches the validate step, which fails
+          # the job with full context. Retries are surfaced as warning
+          # annotations so flake frequency stays visible instead of being
+          # silently absorbed.
+          max_attempts=3
+          attempt=1
+          while :; do
+            socketcli ${{ matrix.args }} 2>&1 | tee /tmp/e2e-output.log
+            [ -z "$RETRY_PROBE" ] && break
+            if bash -c "$RETRY_PROBE"; then
+              break
+            fi
+            if [ "$attempt" -ge "$max_attempts" ]; then
+              echo "::warning title=e2e-${{ matrix.name }} incomplete results::output still fails the completeness probe after ${max_attempts} attempts; letting validation fail the job"
+              break
+            fi
+            echo "::warning title=e2e-${{ matrix.name }} transient retry::attempt ${attempt} failed the completeness probe (suspected backend transient, see ENG-5093); retrying"
+            echo "e2e-${{ matrix.name }}: retry after attempt ${attempt} — completeness probe failed (suspected transient)" >> "$GITHUB_STEP_SUMMARY"
+            attempt=$((attempt+1))
+            sleep 30
+          done
 
       - name: Validate results
         env:
           SOCKET_SECURITY_API_KEY: ${{ secrets.SOCKET_CLI_API_TOKEN }}
         run: bash ${{ matrix.validate }}
 
+      - name: Upload diagnostics on failure
+        if: failure()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: e2e-${{ matrix.name }}-diagnostics-attempt${{ github.run_attempt }}
+          path: |
+            /tmp/e2e-output.log
+            /tmp/*.sarif
+            tests/e2e/fixtures/simple-npm/.socket.facts.json
+            tests/e2e/fixtures/simple-pypi/.socket.facts.json
+            gl-*.json
+            license_output.json
+          if-no-files-found: ignore
+          retention-days: 14
+
   # Branch protection requires the e2e-* checks, but the `e2e` job above is
   # skipped on PRs that can't access repository secrets -- fork PRs and
   # Dependabot PRs. A job skipped via a job-level `if` never expands its

CHANGELOG.md

@@ -1,5 +1,27 @@
 # Changelog
 
+## 2.6.1
+
+### Changed: scan comparison now polls the diff-scans endpoints
+
+- Diff mode no longer holds a single idle HTTP connection open while the API
+  computes the scan comparison. The CLI now creates a diff-scan resource
+  (`POST /orgs/{org}/diff-scans/from-ids`) and polls
+  `GET /orgs/{org}/diff-scans/{id}?cached=true` with short, bounded requests
+  until the comparison is ready (HTTP 200 instead of 202). This fixes
+  intermittent `Connection reset by peer` failures on the final comparison
+  step when scans take several minutes to compare and network middleboxes
+  (e.g. Azure NAT gateways, which default to a 4-minute TCP idle timeout)
+  reap the idle connection (CE-354).
+- Duplicate scan pairs are resolved after an HTTP 409 and then polled through
+  the same cached endpoint. This avoids automatically following the API's 302
+  duplicate redirect with an uncached, potentially long-lived GET request.
+- The change is transparent: no flags or workflow changes are needed. If the
+  org API token is missing the `diff-scans:create`, `diff-scans:list` or
+  `full-scans:list` scopes — or the new flow fails for any other reason — the
+  CLI logs a warning and falls back to the legacy streaming comparison.
+- Requires `socketdev>=3.5.0`.
+
 ## 2.5.9
 
 ### Changed: bump pinned @coana-tech/cli to 15.10.3

pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "socketsecurity"
-version = "2.5.9"
+version = "2.6.1"
 requires-python = ">= 3.11"
 license = {"file" = "LICENSE"}
 dependencies = [
@@ -16,7 +16,7 @@ dependencies = [
     'GitPython',
     'packaging',
     'python-dotenv',
-    "socketdev>=3.3.0,<4.0.0",
+    "socketdev>=3.5.0,<4.0.0",
     "bs4>=0.0.2",
     "markdown>=3.10",
     "brotli>=1.0.9; platform_python_implementation == 'CPython'",

socketsecurity/__init__.py

@@ -1,3 +1,3 @@
 __author__ = 'socket.dev'
-__version__ = '2.5.9'
+__version__ = '2.6.1'
 USER_AGENT = f'SocketPythonCLI/{__version__}'

socketsecurity/core/__init__.py

@@ -15,7 +15,7 @@
     from socketsecurity.config import CliConfig
 from socketdev import socketdev
 from socketdev.exceptions import APIFailure
-from socketdev.fullscans import FullScanParams, SocketArtifact
+from socketdev.fullscans import DiffArtifacts, FullScanParams, SocketArtifact
 from socketdev.org import Organization
 from socketdev.repos import RepositoryInfo
 import copy
@@ -92,6 +92,25 @@
 FULL_SCAN_UPLOAD_MAX_ATTEMPTS = len(FULL_SCAN_UPLOAD_BACKOFF_SCHEDULE_SECONDS)
 FULL_SCAN_UPLOAD_BACKOFF_JITTER_SECONDS = 2.0
 
+# Diff-scan polling policy. The legacy scan comparison (fullscans.stream_diff) holds a
+# single HTTP connection open, fully idle, while the backend computes the diff; network
+# middleboxes with TCP idle timeouts (notably Azure NAT gateways, which default to
+# 4 minutes) kill that connection with a RST, surfacing as an intermittent
+# ConnectionResetError on large scans (CE-354). The diff-scans flow instead creates a
+# diff-scan resource and polls its cached endpoint with short bounded requests: the API
+# answers 202 while the comparison is still computing and 200 with the result once it is
+# ready, so no connection is ever idle long enough to be reaped.
+#
+# Each poll consumes 1 unit of API quota, so the interval backs off toward
+# DIFF_SCAN_POLL_MAX_INTERVAL_SECONDS to stay quota-friendly on comparisons that take
+# minutes to compute. The timeout is a backstop against a diff scan that never
+# completes; on expiry (or any other failure of this flow) the caller falls back to the
+# legacy streaming comparison rather than failing the scan outright.
+DIFF_SCAN_POLL_INITIAL_INTERVAL_SECONDS = 5.0
+DIFF_SCAN_POLL_MAX_INTERVAL_SECONDS = 30.0
+DIFF_SCAN_POLL_BACKOFF_MULTIPLIER = 1.5
+DIFF_SCAN_POLL_TIMEOUT_SECONDS = 30 * 60.0
+
 
 def _humanize_alert_type(alert_type: str) -> str:
     """Convert a camelCase/PascalCase alert type into a Title-Cased label.
@@ -1303,6 +1322,120 @@ def get_license_text_via_purl(self, packages: dict[str, Package], batch_size: in
 
         return packages
 
+    def get_diff_scan_artifacts(
+            self,
+            head_full_scan_id: str,
+            new_full_scan_id: str
+    ) -> DiffArtifacts:
+        """Compare two full scans via the diff-scans endpoints, polling for the result.
+
+        Creates a diff-scan resource from the two full scan IDs, then polls
+        ``GET /orgs/{org}/diff-scans/{id}?cached=true`` until the API returns the
+        computed comparison (200) instead of a processing status (202). Unlike the
+        legacy ``fullscans.stream_diff`` call, no request is ever left idle while
+        the backend computes, so the comparison survives network idle timeouts
+        (CE-354). See the DIFF_SCAN_POLL_* constants for the polling policy.
+
+        Requires an org token with the ``diff-scans:create``, ``diff-scans:list``
+        and ``full-scans:list`` scopes; callers are expected to catch failures and
+        fall back to the legacy streaming comparison.
+
+        Note that cached diff-scan responses always embed per-package license
+        details (the API ignores ``omit_license_details`` when ``cached=true``),
+        so unlike the legacy streaming comparison there is no lean-response
+        option here; see the comment on ``poll_params`` below.
+
+        Args:
+            head_full_scan_id: The before/base full scan ID
+            new_full_scan_id: The after/head full scan ID
+
+        Returns:
+            DiffArtifacts with the added/removed/unchanged/replaced/updated lists
+        """
+        create_params = {
+            "before": head_full_scan_id,
+            "after": new_full_scan_id,
+            "description": f"Socket Security CLI v{__version__} scan comparison",
+        }
+        try:
+            result = self.sdk.diffscans.create_from_ids(self.config.org_slug, create_params)
+            diff_scan = result.get("diff_scan") or {}
+            response_summary = result
+        except APIFailure as error:
+            if error.status_code != 409:
+                raise
+
+            # Do not use on_duplicate=redirect here. The SDK follows that 302
+            # automatically with a GET that lacks cached=true, which can leave
+            # the connection idle while an existing diff scan is still computing.
+            # Resolve the duplicate resource explicitly so every result fetch
+            # continues through the bounded cached polling path below.
+            existing = self.sdk.diffscans.list(
+                self.config.org_slug,
+                params={
+                    "before_full_scan_id": head_full_scan_id,
+                    "after_full_scan_id": new_full_scan_id,
+                    "per_page": 1,
+                },
+            )
+            matches = existing.get("results") or []
+            diff_scan = matches[0] if matches else {}
+            response_summary = existing
+
+        diff_scan_id = diff_scan.get("id")
+        if not diff_scan_id:
+            raise Exception(
+                "Error creating or resolving diff scan: "
+                f"unexpected response: {str(response_summary)[:500]}"
+            )
+        artifacts_dict = diff_scan.get("artifacts")
+
+        # cached=true is the polling contract (202 while computing, 200 when
+        # ready). The API ignores omit_license_details when cached=true - cached
+        # results always embed license details - so there is no lean-response
+        # option on this path (unlike stream_diff with
+        # include_license_details=false, the CE-224 mitigation). If that extra
+        # payload ever gets a response truncated on a huge dependency tree,
+        # response.json() fails and the caller falls back to the legacy
+        # streaming comparison, which still requests the lean payload.
+        poll_params = {"cached": "true"}
+        deadline = time.monotonic() + DIFF_SCAN_POLL_TIMEOUT_SECONDS
+        interval = DIFF_SCAN_POLL_INITIAL_INTERVAL_SECONDS
+        while artifacts_dict is None:
+            try:
+                response = self.sdk.diffscans.get(self.config.org_slug, diff_scan_id, params=poll_params)
+            except APIFailure as error:
+                if not error.is_transient_error():
+                    raise
+                # A dropped/timed-out poll is retryable: the diff scan keeps
+                # computing server-side regardless of what happens to any one poll.
+                log.warning(
+                    f"Transient error polling diff scan {diff_scan_id} "
+                    f"({type(error).__name__}), retrying in {interval:.0f}s"
+                )
+                response = {"status": "processing"}
+            if response.get("status") != "processing":
+                scan = response.get("diff_scan") or {}
+                if scan.get("artifacts") is None:
+                    raise Exception(
+                        f"Error fetching diff scan {diff_scan_id}: unexpected response: {str(response)[:500]}"
+                    )
+                artifacts_dict = scan["artifacts"]
+                break
+            if time.monotonic() >= deadline:
+                raise Exception(
+                    f"Timed out waiting for diff scan {diff_scan_id} after "
+                    f"{DIFF_SCAN_POLL_TIMEOUT_SECONDS:.0f} seconds"
+                )
+            log.debug(f"Diff scan {diff_scan_id} still processing, polling again in {interval:.0f}s")
+            time.sleep(interval)
+            interval = min(interval * DIFF_SCAN_POLL_BACKOFF_MULTIPLIER, DIFF_SCAN_POLL_MAX_INTERVAL_SECONDS)
+
+        return DiffArtifacts.from_dict({
+            key: artifacts_dict.get(key) or []
+            for key in ("added", "removed", "unchanged", "replaced", "updated")
+        })
+
     def get_added_and_removed_packages(
             self,
             head_full_scan_id: str,
@@ -1315,8 +1448,12 @@ def get_added_and_removed_packages(
         Args:
             head_full_scan_id: Previous scan (maybe None if first scan)
             new_full_scan_id: New scan just created
-            include_license_details: Whether to ask the diff endpoint to embed
-                per-package license attribution/details in the response.
+            include_license_details: Whether to ask the *legacy streaming* diff
+                endpoint to embed per-package license attribution/details in the
+                response. Only consulted on the fallback path: the primary
+                diff-scans path always receives embedded license details, since
+                the API ignores ``omit_license_details`` for cached reads (see
+                get_diff_scan_artifacts).
 
                 Defaults to ``False`` on purpose. The diff endpoint exists to
                 compare alerts between two scans; the license fields it can embed
@@ -1343,39 +1480,55 @@ def get_added_and_removed_packages(
 
         log.info(f"Comparing scans - Head scan ID: {head_full_scan_id}, New scan ID: {new_full_scan_id}")
         diff_start = time.time()
+        diff_artifacts = None
         try:
-            diff_report = (
-                self.sdk.fullscans.stream_diff(
-                    self.config.org_slug,
-                    head_full_scan_id,
-                    new_full_scan_id,
-                    use_types=True,
-                    include_license_details=str(include_license_details).lower()
-                ).data
+            diff_artifacts = self.get_diff_scan_artifacts(
+                head_full_scan_id,
+                new_full_scan_id
             )
-        except APIFailure as e:
-            log.error(f"API Error: {e}")
-            if self.cli_config and self.cli_config.disable_blocking:
-                sys.exit(0)
-            sys.exit(1)
-        except Exception as e:
-            import traceback
-            log.error(f"Error getting diff report: {str(e)}")
-            log.error(f"Stack trace:\n{traceback.format_exc()}")
-            raise
+        except Exception as error:
+            # SDK error messages can span many lines (path + response headers); the
+            # first line carries the status, which is all the warning needs.
+            error_summary = str(error).strip().splitlines()[0] if str(error).strip() else ""
+            log.warning(
+                f"Diff scan comparison failed with {type(error).__name__}({error_summary}), "
+                "falling back to the streaming scan comparison"
+            )
+
+        if diff_artifacts is None:
+            try:
+                diff_artifacts = (
+                    self.sdk.fullscans.stream_diff(
+                        self.config.org_slug,
+                        head_full_scan_id,
+                        new_full_scan_id,
+                        use_types=True,
+                        include_license_details=str(include_license_details).lower()
+                    ).data.artifacts
+                )
+            except APIFailure as e:
+                log.error(f"API Error: {e}")
+                if self.cli_config and self.cli_config.disable_blocking:
+                    sys.exit(0)
+                sys.exit(1)
+            except Exception as e:
+                import traceback
+                log.error(f"Error getting diff report: {str(e)}")
+                log.error(f"Stack trace:\n{traceback.format_exc()}")
+                raise
 
         diff_end = time.time()
         log.info(f"Diff Report Gathered in {diff_end - diff_start:.2f} seconds")
         log.info("Diff report artifact counts:")
-        log.info(f"Added: {len(diff_report.artifacts.added)}")
-        log.info(f"Removed: {len(diff_report.artifacts.removed)}")
-        log.info(f"Unchanged: {len(diff_report.artifacts.unchanged)}")
-        log.info(f"Replaced: {len(diff_report.artifacts.replaced)}")
-        log.info(f"Updated: {len(diff_report.artifacts.updated)}")
-
-        added_artifacts = diff_report.artifacts.added + diff_report.artifacts.updated
-        removed_artifacts = diff_report.artifacts.removed + diff_report.artifacts.replaced
-        unchanged_artifacts = diff_report.artifacts.unchanged
+        log.info(f"Added: {len(diff_artifacts.added)}")
+        log.info(f"Removed: {len(diff_artifacts.removed)}")
+        log.info(f"Unchanged: {len(diff_artifacts.unchanged)}")
+        log.info(f"Replaced: {len(diff_artifacts.replaced)}")
+        log.info(f"Updated: {len(diff_artifacts.updated)}")
+
+        added_artifacts = diff_artifacts.added + diff_artifacts.updated
+        removed_artifacts = diff_artifacts.removed + diff_artifacts.replaced
+        unchanged_artifacts = diff_artifacts.unchanged
 
         added_packages: Dict[str, Package] = {}
         removed_packages: Dict[str, Package] = {}