From e2dd4aa8b60e0a9e74ef93a322aa563258f782fe Mon Sep 17 00:00:00 2001 From: David Larsen Date: Fri, 20 Nov 2026 15:01:51 -0500 Subject: [PATCH 1/3] Fail with exit 3 when a scan comparison returns no artifacts When the diff-scan poll returned a finished comparison whose artifacts were null and whose incomplete flag was unset, get_diff_scan_artifacts raised a plain Exception. get_added_and_removed_packages treated that like any other failure of the diff-scans flow and fell back to the streaming comparison, whose response has no incomplete flag. The run then reported on a comparison nobody had checked for completeness. A finished comparison without an artifacts object now raises IncompleteComparisonError, which skips the fallback and maps to the infrastructure exit code (3, --exit-code-on-api-error, or 0 under --disable-blocking). Failures of the flow itself (missing token scopes, transient errors, the polling timeout, a truncated response) still fall back to streaming as before. --- CHANGELOG.md | 9 ++++++ docs/ci-cd.md | 3 +- pyproject.toml | 2 +- socketsecurity/__init__.py | 2 +- socketsecurity/core/__init__.py | 23 +++++++++++--- socketsecurity/exit_codes.py | 2 +- tests/core/test_exit_codes.py | 56 +++++++++++++++++++++++++++++++++ uv.lock | 2 +- 8 files changed, 89 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ff28e72..75d0e59 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +## 2.13.1 + +### Fixed + +- A scan comparison that the API returns without its package lists fails the run + with the infrastructure-error code (`3`, or the `--exit-code-on-api-error` + value). It used to fall back to the streaming comparison, which can't report an + incomplete result. + ## 2.13.0 ### Upgrade notes diff --git a/docs/ci-cd.md b/docs/ci-cd.md index 88210ed..2c61d49 100644 --- a/docs/ci-cd.md +++ b/docs/ci-cd.md @@ -87,7 +87,8 @@ Findings block only when a [diff comparison runs](#scan-type-by-pipeline). SCM event filters can also skip unsupported events with exit `0`; confirm a report was created when validating a new workflow. `--strict-blocking` also checks existing blocking alerts; an incomplete comparison -or incomplete existing-alert data is an infrastructure failure. See the +(including one the API returns without its package lists) or incomplete +existing-alert data is an infrastructure failure. See the [2.13.0 migration table](../README.md#upgrading-to-2130) when updating existing CI rules. ## Config file usage in CI diff --git a/pyproject.toml b/pyproject.toml index 0924672..2ba92fb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "hatchling.build" [project] name = "socketsecurity" -version = "2.13.0" +version = "2.13.1" requires-python = ">= 3.11" license = {"file" = "LICENSE"} dependencies = [ diff --git a/socketsecurity/__init__.py b/socketsecurity/__init__.py index 08835ec..a3c8f69 100644 --- a/socketsecurity/__init__.py +++ b/socketsecurity/__init__.py @@ -1,4 +1,4 @@ __author__ = 'socket.dev' -__version__ = '2.13.0' +__version__ = '2.13.1' USER_AGENT_NAME = 'socket-python-cli' USER_AGENT = f'{USER_AGENT_NAME}/{__version__}' diff --git a/socketsecurity/core/__init__.py b/socketsecurity/core/__init__.py index 6f6b598..423e2ae 100644 --- a/socketsecurity/core/__init__.py +++ b/socketsecurity/core/__init__.py @@ -136,7 +136,9 @@ # DIFF_SCAN_POLL_MAX_INTERVAL_SECONDS to stay quota-friendly on comparisons that take # minutes to compute. The timeout is a backstop against a diff scan that never # completes; on expiry (or any other failure of this flow) the caller falls back to the -# legacy streaming comparison rather than failing the scan outright. +# legacy streaming comparison rather than failing the scan outright. The exception is a +# comparison the API finished but marked incomplete or returned without artifacts: the +# streaming comparison has no completeness signal, so those fail the run instead. DIFF_SCAN_POLL_INITIAL_INTERVAL_SECONDS = 5.0 DIFF_SCAN_POLL_MAX_INTERVAL_SECONDS = 10.0 DIFF_SCAN_POLL_BACKOFF_MULTIPLIER = 1.5 @@ -1995,7 +1997,9 @@ def get_diff_scan_artifacts( Requires an org token with the ``diff-scans:create``, ``diff-scans:list`` and ``full-scans:list`` scopes; callers are expected to catch failures and - fall back to the legacy streaming comparison. + fall back to the legacy streaming comparison, except + ``IncompleteComparisonError``: a finished comparison marked incomplete, or + returned without an artifacts object, must fail the run. Note that cached diff-scan responses always embed per-package license details (the API ignores ``omit_license_details`` when ``cached=true``), @@ -2115,9 +2119,18 @@ def get_diff_scan_artifacts( "could not compare every package, so alerts may be missing. Failing " "instead of reporting a result that may not cover the whole change." ) - if scan.get("artifacts") is None: - raise Exception( - f"Error fetching diff scan {diff_scan_id}: unexpected response: {str(response)[:500]}" + if not isinstance(scan.get("artifacts"), dict): + # A finished comparison always carries an artifacts object (with + # unchanged set to null, not artifacts, under omit_unchanged). Without + # one the result can't be used, and the streaming comparison is no + # substitute: its response has no incomplete flag, so falling back + # would turn a missing comparison into one nobody checked. + raise IncompleteComparisonError( + f"The scan comparison returned no results (diff scan {diff_scan_id}): " + "the API finished the comparison without its added, removed and " + "unchanged package lists. Failing instead of falling back to the " + "streaming comparison, which can't report whether its result is " + f"complete. Response: {str(response)[:500]}" ) artifacts_dict = scan["artifacts"] log.info( diff --git a/socketsecurity/exit_codes.py b/socketsecurity/exit_codes.py index 4901d01..606a48e 100644 --- a/socketsecurity/exit_codes.py +++ b/socketsecurity/exit_codes.py @@ -35,7 +35,7 @@ class InfrastructureError(Exception): class IncompleteComparisonError(InfrastructureError): - """The API marked a scan comparison incomplete. + """The API marked a scan comparison incomplete, or finished it without results. An incomplete comparison can leave out alerts, so a passing result would not mean the change is clean. diff --git a/tests/core/test_exit_codes.py b/tests/core/test_exit_codes.py index 9ecda02..2ca9c35 100644 --- a/tests/core/test_exit_codes.py +++ b/tests/core/test_exit_codes.py @@ -164,6 +164,62 @@ def test_incomplete_comparison_does_not_fall_back_to_streaming(core, incomplete_ core.sdk.fullscans.stream_diff.assert_not_called() +# A finished (non-202) comparison without an artifacts object, whether or not it is +# marked incomplete. Covers every shape that previously fell through to streaming. +@pytest.fixture(params=[ + pytest.param({"incomplete": False, "artifacts": None}, id="null-artifacts-complete"), + pytest.param({"artifacts": None}, id="null-artifacts-flag-unset"), + pytest.param({}, id="artifacts-absent-flag-unset"), + pytest.param({"artifacts": []}, id="artifacts-not-an-object"), +]) +def resultless_response(request): + return {"diff_scan": {"id": "diff-scan-1", **request.param}} + + +def test_comparison_without_artifacts_raises(core, resultless_response): + core.sdk.diffscans.create_from_ids.return_value = {"diff_scan": {"id": "diff-scan-1"}} + core.sdk.diffscans.get.return_value = resultless_response + + with pytest.raises(IncompleteComparisonError, match="returned no results"): + core.get_diff_scan_artifacts("head", "new") + + +def test_comparison_without_artifacts_does_not_fall_back_to_streaming(core, resultless_response): + """The streaming response has no incomplete flag, so it can't vouch for the result.""" + core.sdk.diffscans.create_from_ids.return_value = {"diff_scan": {"id": "diff-scan-1"}} + core.sdk.diffscans.get.return_value = resultless_response + + with pytest.raises(IncompleteComparisonError): + core.get_added_and_removed_packages("head", "new") + + core.sdk.fullscans.stream_diff.assert_not_called() + + +@pytest.mark.parametrize("argv, expected", [ + ([], 3), + (["--exit-code-on-api-error", "42"], 42), + (["--disable-blocking"], 0), + (["--disable-blocking", "--exit-code-on-api-error", "42"], 0), +]) +def test_comparison_without_artifacts_follows_exit_policy(monkeypatch, argv, expected): + error = IncompleteComparisonError("The scan comparison returned no results") + + assert _run_cli(monkeypatch, argv, error) == expected + + +def test_omitted_unchanged_bucket_is_still_a_result(core): + """omit_unchanged nulls the unchanged list, not the artifacts object.""" + core.sdk.diffscans.create_from_ids.return_value = {"diff_scan": {"id": "diff-scan-1"}} + core.sdk.diffscans.get.return_value = {"diff_scan": { + "id": "diff-scan-1", "incomplete": False, + "artifacts": {"added": [], "removed": [], "unchanged": None, "replaced": [], "updated": []}, + }} + + artifacts = core.get_diff_scan_artifacts("head", "new") + + assert artifacts.unchanged == [] + + def test_complete_comparison_is_returned(core): core.sdk.diffscans.create_from_ids.return_value = {"diff_scan": {"id": "diff-scan-1"}} core.sdk.diffscans.get.return_value = { diff --git a/uv.lock b/uv.lock index a6271a3..aea1ca6 100644 --- a/uv.lock +++ b/uv.lock @@ -1334,7 +1334,7 @@ wheels = [ [[package]] name = "socketsecurity" -version = "2.13.0" +version = "2.13.1" source = { editable = "." } dependencies = [ { name = "beautifulsoup4" }, From 9561f82250d78a8b5363994362d6017e5f14fbb4 Mon Sep 17 00:00:00 2001 From: David Larsen Date: Fri, 20 Nov 2026 17:07:06 -0500 Subject: [PATCH 2/3] Keep the streaming fallback when the response has no diff_scan object The SDK's diffscans.get returns {} for statuses it doesn't raise on (204, an unfollowed 3xx). That response says nothing about the comparison, so it falls back to streaming as it did in 2.13.0 instead of exiting 3 with a message claiming the API finished the comparison. Only a returned diff_scan whose artifacts are null, missing or not an object raises the incomplete-comparison error, and its message now says that. --- CHANGELOG.md | 9 +++---- socketsecurity/core/__init__.py | 37 +++++++++++++++++++---------- socketsecurity/exit_codes.py | 2 +- tests/core/test_exit_codes.py | 42 ++++++++++++++++++++++++++++++++- 4 files changed, 71 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 75d0e59..76eed4c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,10 +4,11 @@ ### Fixed -- A scan comparison that the API returns without its package lists fails the run - with the infrastructure-error code (`3`, or the `--exit-code-on-api-error` - value). It used to fall back to the streaming comparison, which can't report an - incomplete result. +- A diff scan that the API returns without its package lists (`artifacts` null, + missing or not an object) fails the run with the infrastructure-error code (`3`, + or the `--exit-code-on-api-error` value). It used to fall back to the streaming + comparison, which can't report an incomplete result. A response with no diff + scan object at all (for example a `204`) still falls back, as in 2.13.0. ## 2.13.0 diff --git a/socketsecurity/core/__init__.py b/socketsecurity/core/__init__.py index 423e2ae..04a2d2e 100644 --- a/socketsecurity/core/__init__.py +++ b/socketsecurity/core/__init__.py @@ -137,8 +137,8 @@ # minutes to compute. The timeout is a backstop against a diff scan that never # completes; on expiry (or any other failure of this flow) the caller falls back to the # legacy streaming comparison rather than failing the scan outright. The exception is a -# comparison the API finished but marked incomplete or returned without artifacts: the -# streaming comparison has no completeness signal, so those fail the run instead. +# diff scan the API returned marked incomplete, or returned without its artifacts object: +# the streaming comparison has no completeness signal, so those fail the run instead. DIFF_SCAN_POLL_INITIAL_INTERVAL_SECONDS = 5.0 DIFF_SCAN_POLL_MAX_INTERVAL_SECONDS = 10.0 DIFF_SCAN_POLL_BACKOFF_MULTIPLIER = 1.5 @@ -1998,8 +1998,10 @@ def get_diff_scan_artifacts( Requires an org token with the ``diff-scans:create``, ``diff-scans:list`` and ``full-scans:list`` scopes; callers are expected to catch failures and fall back to the legacy streaming comparison, except - ``IncompleteComparisonError``: a finished comparison marked incomplete, or - returned without an artifacts object, must fail the run. + ``IncompleteComparisonError``: a diff scan marked incomplete, or returned + without an artifacts object, must fail the run. A response with no + ``diff_scan`` object at all (the SDK's ``{}`` for a 204 or unfollowed 3xx) + says nothing about the comparison and falls back like any other failure. Note that cached diff-scan responses always embed per-package license details (the API ignores ``omit_license_details`` when ``cached=true``), @@ -2112,25 +2114,34 @@ def get_diff_scan_artifacts( response = {"status": "processing"} polls += 1 if response.get("status") != "processing": - scan = response.get("diff_scan") or {} + envelope = response.get("diff_scan") + scan = envelope if isinstance(envelope, dict) else {} if scan.get("incomplete") or response.get("incomplete"): raise IncompleteComparisonError( f"The scan comparison is incomplete (diff scan {diff_scan_id}): the API " "could not compare every package, so alerts may be missing. Failing " "instead of reporting a result that may not cover the whole change." ) + if not isinstance(envelope, dict): + # No diff_scan object at all. The SDK returns {} for statuses it + # doesn't raise on (204, an unfollowed 3xx), so this says nothing + # about the comparison: fall back like any other failure of this flow. + raise Exception( + f"Error fetching diff scan {diff_scan_id}: response has no diff_scan " + f"object: {str(response)[:500]}" + ) if not isinstance(scan.get("artifacts"), dict): - # A finished comparison always carries an artifacts object (with - # unchanged set to null, not artifacts, under omit_unchanged). Without - # one the result can't be used, and the streaming comparison is no + # The API returned the diff scan, but not its result. A finished diff + # scan always carries an artifacts object (under omit_unchanged only + # artifacts.unchanged is null). The streaming comparison is no # substitute: its response has no incomplete flag, so falling back - # would turn a missing comparison into one nobody checked. + # would turn a missing result into one nobody checked. raise IncompleteComparisonError( f"The scan comparison returned no results (diff scan {diff_scan_id}): " - "the API finished the comparison without its added, removed and " - "unchanged package lists. Failing instead of falling back to the " - "streaming comparison, which can't report whether its result is " - f"complete. Response: {str(response)[:500]}" + "the API returned the diff scan without its artifacts object (the " + "added, removed and unchanged package lists). Failing instead of " + "falling back to the streaming comparison, which can't report " + f"whether its result is complete. Response: {str(response)[:500]}" ) artifacts_dict = scan["artifacts"] log.info( diff --git a/socketsecurity/exit_codes.py b/socketsecurity/exit_codes.py index 606a48e..16f0742 100644 --- a/socketsecurity/exit_codes.py +++ b/socketsecurity/exit_codes.py @@ -35,7 +35,7 @@ class InfrastructureError(Exception): class IncompleteComparisonError(InfrastructureError): - """The API marked a scan comparison incomplete, or finished it without results. + """The API marked a scan comparison incomplete, or returned it without results. An incomplete comparison can leave out alerts, so a passing result would not mean the change is clean. diff --git a/tests/core/test_exit_codes.py b/tests/core/test_exit_codes.py index 2ca9c35..1f06a32 100644 --- a/tests/core/test_exit_codes.py +++ b/tests/core/test_exit_codes.py @@ -9,6 +9,8 @@ from unittest.mock import MagicMock import pytest +import requests +from socketdev.diffscans import DiffScans from socketdev.exceptions import ( APIFailure, APIInsufficientPermissions, @@ -164,7 +166,7 @@ def test_incomplete_comparison_does_not_fall_back_to_streaming(core, incomplete_ core.sdk.fullscans.stream_diff.assert_not_called() -# A finished (non-202) comparison without an artifacts object, whether or not it is +# A finished (non-202) diff scan without an artifacts object, whether or not it is # marked incomplete. Covers every shape that previously fell through to streaming. @pytest.fixture(params=[ pytest.param({"incomplete": False, "artifacts": None}, id="null-artifacts-complete"), @@ -207,6 +209,44 @@ def test_comparison_without_artifacts_follows_exit_policy(monkeypatch, argv, exp assert _run_cli(monkeypatch, argv, error) == expected +class StreamingFallback(Exception): + """Raised by the mocked stream_diff to prove the fallback path was taken.""" + + +@pytest.mark.parametrize("response", [ + pytest.param({}, id="empty-dict"), + pytest.param({"diff_scan": None}, id="diff-scan-null"), +]) +def test_response_without_diff_scan_object_falls_back_to_streaming(core, response): + """No diff_scan object says nothing about the comparison, so 2.13.0's fallback stays.""" + core.sdk.diffscans.create_from_ids.return_value = {"diff_scan": {"id": "diff-scan-1"}} + core.sdk.diffscans.get.return_value = response + core.sdk.fullscans.stream_diff.side_effect = StreamingFallback + + with pytest.raises(Exception, match="no diff_scan object") as error: + core.get_diff_scan_artifacts("head", "new") + assert not isinstance(error.value, IncompleteComparisonError) + + with pytest.raises(StreamingFallback): + core.get_added_and_removed_packages("head", "new") + core.sdk.fullscans.stream_diff.assert_called_once() + + +def test_http_204_through_the_sdk_falls_back_to_streaming(core): + """The SDK's own get() maps a 204 to {}; that must not exit 3.""" + no_content = requests.Response() + no_content.status_code = 204 + no_content._content = b"" + api = MagicMock() + api.do_request.return_value = no_content + core.sdk.diffscans = DiffScans(api) + core.sdk.diffscans.create_from_ids = MagicMock(return_value={"diff_scan": {"id": "diff-scan-1"}}) + core.sdk.fullscans.stream_diff.side_effect = StreamingFallback + + with pytest.raises(StreamingFallback): + core.get_added_and_removed_packages("head", "new") + + def test_omitted_unchanged_bucket_is_still_a_result(core): """omit_unchanged nulls the unchanged list, not the artifacts object.""" core.sdk.diffscans.create_from_ids.return_value = {"diff_scan": {"id": "diff-scan-1"}} From d183dd1833b750f85b2b69b6b0363011c1610e85 Mon Sep 17 00:00:00 2001 From: David Larsen Date: Fri, 20 Nov 2026 18:08:17 -0500 Subject: [PATCH 3/3] Bump version to 2.13.3 2.13.2 was released from main, so this change needs the next patch version. --- pyproject.toml | 2 +- socketsecurity/__init__.py | 2 +- uv.lock | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 5c88c15..82c5bbf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "hatchling.build" [project] name = "socketsecurity" -version = "2.13.2" +version = "2.13.3" requires-python = ">= 3.11" license = {"file" = "LICENSE"} dependencies = [ diff --git a/socketsecurity/__init__.py b/socketsecurity/__init__.py index 7ba49e1..0871ade 100644 --- a/socketsecurity/__init__.py +++ b/socketsecurity/__init__.py @@ -1,4 +1,4 @@ __author__ = 'socket.dev' -__version__ = '2.13.2' +__version__ = '2.13.3' USER_AGENT_NAME = 'socket-python-cli' USER_AGENT = f'{USER_AGENT_NAME}/{__version__}' diff --git a/uv.lock b/uv.lock index 196ae3c..b3681e4 100644 --- a/uv.lock +++ b/uv.lock @@ -1334,7 +1334,7 @@ wheels = [ [[package]] name = "socketsecurity" -version = "2.13.2" +version = "2.13.3" source = { editable = "." } dependencies = [ { name = "beautifulsoup4" },