diff --git a/CHANGELOG.md b/CHANGELOG.md index 165e42c5..9459c9b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,21 @@ # Changelog +## 2.13.3 + +### Fixed + +- With `--scm github`, `pull_request` runs with the `reopened` or + `ready_for_review` action now scan the pull request. They exited `0` without + scanning, so a reopened pull request could pass a required check on a commit an + earlier run failed. +- Skipped GitHub events and actions, and skipped GitLab pipeline sources, log a + warning that no scan ran. `pull_request_target` gets its own message, and a + missing event name or action says it could not be determined. Exit codes are + unchanged. +- `docs/ci-cd.md` explains that the example GitHub workflow's `push` trigger also + runs the job on a pull request's head commit as a full scan that exits `0`, and + how to keep that run from satisfying a required check. + ## 2.13.2 ### Added diff --git a/docs/ci-cd.md b/docs/ci-cd.md index def55470..b0e263b8 100644 --- a/docs/ci-cd.md +++ b/docs/ci-cd.md @@ -84,8 +84,9 @@ they also hide findings and usage errors. Even with exit `0`, a setup/API failur does not mean a clean scan; check the logs for a completed report. Findings block only when a [diff comparison runs](#scan-type-by-pipeline). -SCM event filters can also skip unsupported events with exit `0`; confirm a report -was created when validating a new workflow. +SCM event filters can also skip unsupported events with exit `0` and a warning +(see [Skipped events](#skipped-events)); confirm a report was created when +validating a new workflow. `--strict-blocking` also checks existing blocking alerts; an incomplete comparison or incomplete existing-alert data is an infrastructure failure. See the [2.13.0 migration table](../README.md#upgrading-to-2130) when updating existing CI rules. @@ -520,6 +521,48 @@ runs the comparison without the PR comment adapter. `--generate-license` and `--legal-format fossa` work on both paths; a full scan fetches the package list for them. +### Skipped events + +With `--scm github`, `pull_request` runs scan on the `opened`, `synchronize`, +`reopened` and `ready_for_review` actions. Other actions (`closed`, `labeled`, +`edited`, ...) and events the CLI does not handle (`pull_request_target`, +`merge_group`, `schedule`, ...) are skipped: no scan runs, the CLI logs a warning +and exits `0`. `issue_comment` runs the ignore-command flow, not a scan. With +`--scm gitlab`, pipeline sources other than `web`, `merge_request_event`, `push`, +`api`, `pipeline` and the comment flow are skipped the same way. + +GitHub doesn't document which run a required check uses when a job with the same +name runs more than once on a commit, so a skipped run that passes can satisfy the +check in place of an earlier failure. Subscribe the Socket job only to events and +actions the CLI scans. + +- `merge_group`: if the required check also runs on merge queue events, the job + passes there without scanning. The `pull_request` run is the gate. +- `pull_request_target` is skipped because it runs against the base branch, so it + can't stand in for `pull_request` on fork pull requests. Fork pull requests on + `pull_request` get no repository secrets, so the CLI has no API token and exits + `3` (or the `--exit-code-on-api-error` value). Decide how fork pull requests get + a token before making the check required. + +### Push and pull request runs on the same commit + +The [example workflow](../workflows/github-actions.yml) runs one `socket-security` +job on `push` to every branch and on `pull_request`. A new commit on a pull +request's branch triggers both events on the same head commit. The `pull_request` +run compares against the baseline and can fail. The `push` run is a full scan, +which never blocks and exits `0`. Both runs report a check with the same name, so +a required check can resolve against the passing `push` run. + +If the Socket job is a required check, do one of these: + +- Restrict `push` to the default branch (`branches: [main]`). Default-branch scans + still keep the baseline current; feature branches are scanned only through their + pull requests. +- Run the pull request scan in its own job (for example `socket-pr` with + `if: github.event_name == 'pull_request'`, and + `if: github.event_name != 'pull_request'` on the existing job), and require that + job's check. + ## Pull request and Dashboard association The CLI sends the resolved pull request number with each full scan and attaches diff --git a/pyproject.toml b/pyproject.toml index 5c88c152..82c5bbf4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "hatchling.build" [project] name = "socketsecurity" -version = "2.13.2" +version = "2.13.3" requires-python = ">= 3.11" license = {"file" = "LICENSE"} dependencies = [ diff --git a/socketsecurity/__init__.py b/socketsecurity/__init__.py index 7ba49e1b..0871adeb 100644 --- a/socketsecurity/__init__.py +++ b/socketsecurity/__init__.py @@ -1,4 +1,4 @@ __author__ = 'socket.dev' -__version__ = '2.13.2' +__version__ = '2.13.3' USER_AGENT_NAME = 'socket-python-cli' USER_AGENT = f'{USER_AGENT_NAME}/{__version__}' diff --git a/socketsecurity/core/scm/github.py b/socketsecurity/core/scm/github.py index 5248fc87..7e0f06d2 100644 --- a/socketsecurity/core/scm/github.py +++ b/socketsecurity/core/scm/github.py @@ -14,6 +14,18 @@ from socketsecurity.exit_codes import InfrastructureError from socketsecurity.socketcli import CliClient +# pull_request actions that can leave a PR head without a current Socket result: +# new commits (opened, synchronize) and a PR coming back after it was closed or +# drafted (reopened, ready_for_review). A skipped run exits 0, and GitHub doesn't +# define which run a required check uses when the same job runs more than once on a +# commit, so a skipped run on one of these could satisfy the check in place of an +# earlier failure. Other actions (closed, labeled, edited, ...) are skipped with a +# warning. +PULL_REQUEST_SCAN_ACTIONS = ("opened", "synchronize", "reopened", "ready_for_review") +HANDLED_EVENTS = ( + "push, workflow_dispatch and pull_request (scan); issue_comment (ignore commands)" +) + @dataclass class GithubConfig: @@ -175,19 +187,55 @@ def __init__( raise InfrastructureError("Unable to get Github API Token") def check_event_type(self) -> str: - if self.config.event_name.lower() in ["push", "workflow_dispatch"]: + event_name = self.config.event_name.lower() + if event_name in ["push", "workflow_dispatch"]: if not self.config.pr_number: return "main" return "diff" - elif self.config.event_name.lower() == "pull_request": - if self.config.event_action and self.config.event_action.lower() in ['opened', 'synchronize']: + elif event_name == "pull_request": + action = (self.config.event_action or "").lower() + if action in PULL_REQUEST_SCAN_ACTIONS: return "diff" - log.info(f"Pull Request Action {self.config.event_action} is not a supported type") + if not action: + log.warning( + "Could not determine the pull_request action: EVENT_ACTION is not set " + "and GITHUB_EVENT_PATH has no 'action'. No scan ran; exiting 0. Set " + "EVENT_ACTION (or GITHUB_EVENT_PATH) when running with --scm github " + "outside GitHub Actions." + ) + sys.exit(0) + log.warning( + f"Pull request action '{self.config.event_action}' is not scanned " + f"(scanned actions: {', '.join(PULL_REQUEST_SCAN_ACTIONS)}). No scan ran; " + "exiting 0. If this job is a required check, its success here does not " + "reflect a scan: remove this action from the workflow's pull_request types." + ) sys.exit(0) - elif self.config.event_name.lower() == "issue_comment": + elif event_name == "issue_comment": return "comment" + elif event_name == "pull_request_target": + # Not treated as pull_request on purpose: GITHUB_SHA and GITHUB_REF point at + # the base branch and the default checkout is the base branch, so a "PR scan" + # here would compare the base against itself and pass. + log.warning( + "GitHub event 'pull_request_target' is not scanned: it runs against the " + "base branch, not the pull request head. No scan ran; exiting 0. Trigger " + "the Socket job on pull_request instead." + ) + sys.exit(0) - log.error(f"Unknown event type {self.config.event_name}") + if not event_name: + log.warning( + "Could not determine the GitHub event: GITHUB_EVENT_NAME is not set. " + "No scan ran; exiting 0. Set GITHUB_EVENT_NAME when running with " + "--scm github outside GitHub Actions." + ) + sys.exit(0) + log.warning( + f"GitHub event '{self.config.event_name}' is not scanned (handled events: " + f"{HANDLED_EVENTS}). No scan ran; exiting 0. If this job is a required " + "check, its success here does not reflect a scan." + ) sys.exit(0) def post_comment(self, body: str) -> None: diff --git a/socketsecurity/core/scm/gitlab.py b/socketsecurity/core/scm/gitlab.py index a7b2bd3a..4c9cdbf6 100644 --- a/socketsecurity/core/scm/gitlab.py +++ b/socketsecurity/core/scm/gitlab.py @@ -213,8 +213,20 @@ def check_event_type(self) -> str: return "diff" elif pipeline_source == "issue_comment": return "comment" + elif not pipeline_source: + log.warning( + "Could not determine the GitLab pipeline source: CI_PIPELINE_SOURCE is " + "not set. No scan ran; exiting 0. Set CI_PIPELINE_SOURCE when running " + "with --scm gitlab outside GitLab CI." + ) + sys.exit(0) else: - log.error(f"Unknown event type {pipeline_source}") + log.warning( + f"GitLab pipeline source '{self.config.pipeline_source}' is not scanned " + "(handled sources: web, merge_request_event, push, api and pipeline " + "(scan); issue_comment (ignore commands)). No scan ran; exiting 0. If " + "this job gates merges, its success here does not reflect a scan." + ) sys.exit(0) def post_comment(self, body: str) -> None: diff --git a/tests/core/test_cli_exit_contract.py b/tests/core/test_cli_exit_contract.py index d3988f91..74c4ceef 100644 --- a/tests/core/test_cli_exit_contract.py +++ b/tests/core/test_cli_exit_contract.py @@ -275,14 +275,20 @@ def test_branch_full_scan_does_not_gate_on_findings(cli_runtime, monkeypatch, ou @pytest.mark.parametrize("scm, event, action", [ ("github", "pull_request", "closed"), ("github", "unsupported", ""), - ("gitlab", "unsupported", ""), + ("github", "pull_request_target", "opened"), ("gitlab", "unsupported", ""), ]) -def test_scm_event_filters_skip_without_scanning(cli_runtime, monkeypatch, scm, event, action): +def test_scm_event_filters_skip_without_scanning(cli_runtime, monkeypatch, caplog, scm, event, action): monkeypatch.setenv("GH_API_TOKEN", "test") monkeypatch.setenv("GITLAB_TOKEN", "test") monkeypatch.setenv("GITHUB_EVENT_NAME", event) monkeypatch.setenv("EVENT_ACTION", action) monkeypatch.setenv("CI_PIPELINE_SOURCE", event) - assert cli_runtime.run(["--scm", scm]) == 0 + with caplog.at_level("INFO", logger="socketdev"): + assert cli_runtime.run(["--scm", scm]) == 0 cli_runtime.scan.assert_not_called() + # A skipped run passes, so it has to say so where a required-check owner looks. + assert any( + record.levelname == "WARNING" and "No scan ran" in record.getMessage() + for record in caplog.records + ) diff --git a/tests/unit/test_github_event_filter.py b/tests/unit/test_github_event_filter.py new file mode 100644 index 00000000..5973673c --- /dev/null +++ b/tests/unit/test_github_event_filter.py @@ -0,0 +1,259 @@ +"""Which GitHub events and pull_request actions run a scan. + +A skipped run exits 0, and GitHub doesn't define which run a required check uses +when the same job runs more than once on a commit, so skipping an action that a +workflow subscribes to can satisfy the check with a pass that never scanned +anything. +""" + +import logging +import sys +from contextlib import nullcontext +from unittest.mock import MagicMock, Mock + +import pytest + +from socketsecurity import socketcli +from socketsecurity.core.classes import Diff, Issue +from socketsecurity.core.scm.github import ( + PULL_REQUEST_SCAN_ACTIONS, + Github, + GithubConfig, +) +from socketsecurity.core.scm.gitlab import Gitlab, GitlabConfig +from socketsecurity.core.scm_comments import Comments + +CONTEXT_VARIABLES = ( + "BUILDKITE", + "BUILDKITE_PULL_REQUEST", + "DEFAULT_BRANCH", + "EVENT_ACTION", + "CI_MERGE_REQUEST_IID", + "CI_PIPELINE_SOURCE", + "GITHUB_EVENT_NAME", + "GITHUB_EVENT_PATH", + "GITHUB_REPOSITORY", + "PR_NUMBER", + "SOCKET_SECURITY_API_KEY", + "SOCKET_SECURITY_API_TOKEN", + "SOCKET_API_KEY", + "SOCKET_API_TOKEN", +) + + +@pytest.fixture(autouse=True) +def clear_context(monkeypatch): + for variable in CONTEXT_VARIABLES: + monkeypatch.delenv(variable, raising=False) + monkeypatch.setenv("GH_API_TOKEN", "test-token") + monkeypatch.setenv("GITHUB_REPOSITORY", "acme/repo") + + +def _github(monkeypatch, event_name, action=None, pr_number="7"): + if event_name is not None: + monkeypatch.setenv("GITHUB_EVENT_NAME", event_name) + if action is not None: + monkeypatch.setenv("EVENT_ACTION", action) + return Github(client=object(), config=GithubConfig.from_env(pr_number=pr_number)) + + +def _skip_warnings(scm, caplog) -> list[str]: + with ( + caplog.at_level(logging.INFO, logger="socketdev"), + pytest.raises(SystemExit) as result, + ): + scm.check_event_type() + assert result.value.code == 0 + return [r.getMessage() for r in caplog.records if r.levelno == logging.WARNING] + + +@pytest.mark.parametrize( + "action", ["opened", "synchronize", "reopened", "ready_for_review", "Reopened"] +) +def test_pull_request_actions_that_change_the_head_result_are_scanned( + monkeypatch, action +): + assert _github(monkeypatch, "pull_request", action).check_event_type() == "diff" + + +def test_scanned_actions_are_the_documented_set(): + assert PULL_REQUEST_SCAN_ACTIONS == ( + "opened", + "synchronize", + "reopened", + "ready_for_review", + ) + + +@pytest.mark.parametrize( + "event_name, pr_number, expected", + [ + ("push", None, "main"), + ("push", "7", "diff"), + ("workflow_dispatch", None, "main"), + ("workflow_dispatch", "7", "diff"), + ("issue_comment", "7", "comment"), + ], +) +def test_other_handled_events_are_unchanged( + monkeypatch, event_name, pr_number, expected +): + github = _github(monkeypatch, event_name, pr_number=pr_number) + assert github.check_event_type() == expected + + +@pytest.mark.parametrize("action", ["closed", "labeled", "edited"]) +def test_other_pull_request_actions_skip_with_a_warning(monkeypatch, caplog, action): + warnings = _skip_warnings(_github(monkeypatch, "pull_request", action), caplog) + + assert any( + f"'{action}' is not scanned" in m and "No scan ran" in m for m in warnings + ) + + +def test_missing_pull_request_action_says_it_could_not_be_determined( + monkeypatch, caplog +): + warnings = _skip_warnings(_github(monkeypatch, "pull_request"), caplog) + + assert any( + "Could not determine the pull_request action" in m and "No scan ran" in m + for m in warnings + ) + assert not any("is not scanned" in m for m in warnings) + + +def test_missing_event_name_says_it_could_not_be_determined(monkeypatch, caplog): + warnings = _skip_warnings(_github(monkeypatch, None), caplog) + + assert any( + "GITHUB_EVENT_NAME is not set" in m and "No scan ran" in m for m in warnings + ) + + +@pytest.mark.parametrize( + "event_name, expected", + [ + ("pull_request_target", "runs against the base branch"), + ("merge_group", "'merge_group' is not scanned"), + ("schedule", "'schedule' is not scanned"), + ], +) +def test_unscanned_events_skip_with_a_warning( + monkeypatch, caplog, event_name, expected +): + warnings = _skip_warnings(_github(monkeypatch, event_name, "opened"), caplog) + + assert any(expected in message and "No scan ran" in message for message in warnings) + # issue_comment is handled, but it runs the ignore-command flow, not a scan. + if event_name != "pull_request_target": + assert any("issue_comment (ignore commands)" in message for message in warnings) + + +def _gitlab(monkeypatch, pipeline_source): + monkeypatch.setenv("GITLAB_TOKEN", "test-token") + if pipeline_source is not None: + monkeypatch.setenv("CI_PIPELINE_SOURCE", pipeline_source) + return Gitlab(client=object(), config=GitlabConfig.from_env()) + + +@pytest.mark.parametrize( + "pipeline_source, expected", + [ + ("schedule", "'schedule' is not scanned"), + (None, "CI_PIPELINE_SOURCE is not set"), + ], +) +def test_gitlab_skipped_sources_warn(monkeypatch, caplog, pipeline_source, expected): + warnings = _skip_warnings(_gitlab(monkeypatch, pipeline_source), caplog) + + assert any(expected in message and "No scan ran" in message for message in warnings) + + +def _blocking_alert() -> Issue: + return Issue( + pkg_name="lodash", + pkg_version="4.17.11", + pkg_type="npm", + severity="critical", + title="Critical CVE", + description="", + type="criticalCVE", + url="https://socket.dev/test", + manifests="package.json", + props={}, + key="key", + purl="pkg:npm/lodash@4.17.11", + error=True, + warn=False, + ignore=False, + monitor=False, + ) + + +@pytest.mark.parametrize( + "action, scanned, expected_exit", + [ + ("reopened", True, 1), + ("ready_for_review", True, 1), + ("synchronize", True, 1), + ("closed", False, 0), + ], +) +def test_reopened_pull_request_blocks_like_synchronize( + monkeypatch, action, scanned, expected_exit +): + """A reopened PR with an existing blocker fails instead of passing unscanned.""" + monkeypatch.setattr( + sys, + "argv", + [ + "socketcli", + "--api-token", + "test", + "--scm", + "github", + "--strict-blocking", + "--repo", + "repo", + "--branch", + "feature", + "--pr-number", + "7", + "--files", + '["package.json"]', + ], + ) + monkeypatch.setenv("GITHUB_EVENT_NAME", "pull_request") + monkeypatch.setenv("EVENT_ACTION", action) + monkeypatch.setenv("DEFAULT_BRANCH", "false") + + diff = Diff(id="scan", new_alerts=[], unchanged_alerts=[_blocking_alert()]) + diff.diff_url = "https://socket.dev/diff" + monkeypatch.setattr( + Github, + "get_comments_for_pr", + lambda self: Comments.check_for_socket_comments({}), + ) + posted = Mock() + monkeypatch.setattr(Github, "add_socket_comments", posted) + monkeypatch.setattr(Github, "handle_ignore_reactions", Mock()) + sdk = MagicMock() + sdk.org.get.return_value = {"organizations": {"org-id": {"slug": "acme"}}} + monkeypatch.setattr(socketcli, "build_socket_sdk", Mock(return_value=sdk)) + monkeypatch.setattr(socketcli, "CliClient", Mock(return_value=Mock())) + monkeypatch.setattr( + socketcli, "StreamingLogs", lambda **kwargs: nullcontext(Mock()) + ) + monkeypatch.setattr(socketcli, "apply_git_context", lambda config: (False, None)) + scan = Mock(return_value=diff) + monkeypatch.setattr(socketcli.Core, "create_new_diff", scan) + monkeypatch.setattr(socketcli.Core, "has_manifest_files", Mock(return_value=True)) + monkeypatch.setattr(socketcli.OutputHandler, "handle_output", Mock()) + + with pytest.raises(SystemExit) as result: + socketcli.cli() + + assert result.value.code == expected_exit + assert scan.called is scanned + assert posted.called is scanned diff --git a/uv.lock b/uv.lock index 196ae3c8..b3681e40 100644 --- a/uv.lock +++ b/uv.lock @@ -1334,7 +1334,7 @@ wheels = [ [[package]] name = "socketsecurity" -version = "2.13.2" +version = "2.13.3" source = { editable = "." } dependencies = [ { name = "beautifulsoup4" }, diff --git a/workflows/github-actions.yml b/workflows/github-actions.yml index 422eb253..ffd4c79d 100644 --- a/workflows/github-actions.yml +++ b/workflows/github-actions.yml @@ -7,8 +7,15 @@ run-name: Socket Security Github Action on: push: + # On a branch with an open pull request, this also runs the job on the PR head + # commit as a full scan, which exits 0. If this job is a required check, restrict + # push to the default branch or give the pull_request run its own job name. See + # "Push and pull request runs on the same commit" in docs/ci-cd.md. branches: ['**'] # Run on all branches, all commits pull_request: + # Every listed action runs a scan. Don't add actions the CLI skips (closed, + # labeled, edited, ...): a skipped run exits 0 and can satisfy a required check + # on a commit an earlier run failed. types: [opened, synchronize, reopened] issue_comment: types: [created]