From 4f8b28897c661bee36e2409cd1296ab5a6f83e00 Mon Sep 17 00:00:00 2001 From: jdalton Date: Fri, 20 Nov 2026 20:49:58 -0500 Subject: [PATCH] fix(scan): reduce memory use during directory scans --- CHANGELOG.md | 5 +++ scripts/refresh/snapshots.mts | 2 +- scripts/test/native.mts | 11 ++--- .../manifest/cmd-manifest-cdxgen.test.mts | 1 + src/utils/glob.mts | 9 +++-- src/utils/glob.test.mts | 40 ++++++++++++++++++- test/release-workflow.test.mts | 25 ++++++------ 7 files changed, 70 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 682c8bb4c..ff165b167 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). +## [Unreleased] + +### Fixed +- Directory scans now filter unsupported files before deduplicating matches, reducing memory use in large repositories. + ## [1.16.0](https://github.com/SocketDev/socket-cli/releases/tag/v1.16.0) - 2026-11-19 ### Added diff --git a/scripts/refresh/snapshots.mts b/scripts/refresh/snapshots.mts index cf3485cf4..5e39c983d 100644 --- a/scripts/refresh/snapshots.mts +++ b/scripts/refresh/snapshots.mts @@ -8,9 +8,9 @@ import type { ScriptMeta } from '../lib/run-main.mts' import { runMain } from '../lib/run-main.mts' import type { SnapshotDependencyName } from './snapshots/dependencies.mts' import { + SNAPSHOT_DEPENDENCIES, assertSnapshotDependenciesCurrent, normalizeSnapshotDependencyName, - SNAPSHOT_DEPENDENCIES, } from './snapshots/dependencies.mts' const ROOT = path.resolve(import.meta.dirname, '../..') diff --git a/scripts/test/native.mts b/scripts/test/native.mts index 393dbb95c..9119e8f3b 100644 --- a/scripts/test/native.mts +++ b/scripts/test/native.mts @@ -1,3 +1,5 @@ +import { SEA_VERSION } from '../../src/sea/version.mts' + import type { SpawnSyncOptions } from 'node:child_process' import { spawnSync } from 'node:child_process' import { createHash } from 'node:crypto' @@ -25,7 +27,6 @@ const ROOT = path.resolve(import.meta.dirname, '../..') const PACKAGE_PATH = path.join(ROOT, 'package.json') type PackageMetadata = { name: string - version: string } const BUILD_DIR = path.join(ROOT, 'build/sea') const TRIPLE = `${process.platform === 'win32' ? 'win' : process.platform}-${process.arch}` @@ -72,10 +73,10 @@ async function verifyArtifact( const manifest = JSON.parse(manifestText) as NativeManifest if ( manifest.application?.name !== packageInfo.name || - manifest.application?.version !== packageInfo.version + manifest.application?.version !== SEA_VERSION ) { throw new Error( - `Native manifest is for ${manifest.application?.name}@${manifest.application?.version}; expected ${packageInfo.name}@${packageInfo.version}`, + `Native manifest is for ${manifest.application?.name}@${manifest.application?.version}; expected ${packageInfo.name}@${SEA_VERSION}`, ) } if (manifest.target?.triple !== TRIPLE) { @@ -229,7 +230,7 @@ async function main(): Promise { process.env['SOCKET_CLI_NATIVE_TEST_STAGE'] = stageDir process.env['SOCKET_CLI_NATIVE_TEST_CAPABILITIES'] = manifest.capabilities.available.join(',') - process.env['SOCKET_CLI_NATIVE_TEST_VERSION'] = packageInfo.version + process.env['SOCKET_CLI_NATIVE_TEST_VERSION'] = SEA_VERSION const nodeRuntime = manifest.payloads.find( payload => payload.id === 'node-runtime', ) @@ -292,7 +293,7 @@ async function main(): Promise { } if ( version.status !== 0 || - !version.stdout.trim().startsWith(packageInfo.version) + !version.stdout.trim().startsWith(SEA_VERSION) ) { throw new Error( `Native --version check failed (status ${version.status}): ${version.stdout}${version.stderr}`, diff --git a/src/commands/manifest/cmd-manifest-cdxgen.test.mts b/src/commands/manifest/cmd-manifest-cdxgen.test.mts index c01f736f8..27feeba7c 100644 --- a/src/commands/manifest/cmd-manifest-cdxgen.test.mts +++ b/src/commands/manifest/cmd-manifest-cdxgen.test.mts @@ -117,6 +117,7 @@ describe('socket manifest cdxgen', async () => { '`socket manifest cdxgen`', ) }, + { timeout: 60_000 }, ) it.skipIf(process.platform === 'win32')( diff --git a/src/utils/glob.mts b/src/utils/glob.mts index c01dd31c0..1e4529467 100644 --- a/src/utils/glob.mts +++ b/src/utils/glob.mts @@ -436,6 +436,9 @@ export async function globWithGitIgnore( ] : [...ignores, ...cliMinimatchIgnores].map(stripTrailingSlash), ...additionalOptions, + // fast-glob's uniqueness set retains every match before the streaming + // filter runs. Filtered scans deduplicate only supported paths below. + unique: filter ? false : additionalOptions.unique, // Skip dirs the running user cannot read instead of aborting on the first // `EACCES`. Pinned after `...additionalOptions` so a caller's options bag // cannot flip it off; it is a safety invariant, not a tunable. @@ -483,7 +486,7 @@ export async function globWithGitIgnore( // optional caller filter drops non-matches before they accumulate. On the slow // path each surviving entry is also re-checked against its gitignore ancestor // chain, which carries the full negation support fast-glob lacks. - const results: string[] = [] + const results = new Set() const stream = fastGlob.globStream( patterns as string[], globOptions, @@ -502,9 +505,9 @@ export async function globWithGitIgnore( if (filter && !filter(p)) { continue } - results.push(p) + results.add(p) } - return results + return [...results] } export async function globWorkspace( diff --git a/src/utils/glob.test.mts b/src/utils/glob.test.mts index e86abcece..42e8a0e04 100644 --- a/src/utils/glob.test.mts +++ b/src/utils/glob.test.mts @@ -8,7 +8,8 @@ import { import os from 'node:os' import path from 'node:path' -import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import fastGlob from 'fast-glob' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { normalizePath } from '@socketsecurity/registry/lib/path' @@ -170,6 +171,43 @@ describe('glob utilities', () => { ]) }) + it('disables fast-glob uniqueness when filtering', async () => { + mockTestFs({ [`${mockFixturePath}/package.json`]: '{}' }) + const globStreamSpy = vi.spyOn(fastGlob, 'globStream') + + try { + await globWithGitIgnore(['**/*'], { + cwd: mockFixturePath, + filter: filterJsonFiles, + }) + + expect(globStreamSpy).toHaveBeenLastCalledWith( + ['**/*'], + expect.objectContaining({ unique: false }), + ) + } finally { + globStreamSpy.mockRestore() + } + }) + + it('deduplicates overlapping matches after filtering', async () => { + mockTestFs({ + [`${mockFixturePath}/package.json`]: '{}', + [`${mockFixturePath}/src/package.json`]: '{}', + [`${mockFixturePath}/src/readme.md`]: '', + }) + + const results = await globWithGitIgnore(['**/*.json', 'src/**/*'], { + cwd: mockFixturePath, + filter: filterJsonFiles, + }) + + expect(results.map(normalizePath).sort()).toEqual([ + `${mockFixturePath}/package.json`, + `${mockFixturePath}/src/package.json`, + ]) + }) + it('should skip directories ignored via trailing-slash patterns', async () => { // fast-glob silently drops `ignore` entries that end in `/`, so // the pattern produced for `dist/` (`**/dist/`) used to do diff --git a/test/release-workflow.test.mts b/test/release-workflow.test.mts index 4ef9672b5..82fe7bbb0 100644 --- a/test/release-workflow.test.mts +++ b/test/release-workflow.test.mts @@ -1,10 +1,5 @@ import { execFileSync } from 'node:child_process' -import { - mkdtempSync, - readFileSync, - rmSync, - writeFileSync, -} from 'node:fs' +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import path from 'node:path' @@ -153,9 +148,7 @@ describe('v1 release workflow contract', () => { const stage = workflow.jobs.publish.steps.find( step => step.name === 'Stage release packages', ) - const parser = stage?.run - ?.split("<<'NODE'\n")[1] - ?.split('\nNODE')[0] + const parser = stage?.run?.split("<<'NODE'\n")[1]?.split('\nNODE')[0] expect(parser).toBeDefined() const directory = mkdtempSync(path.join(tmpdir(), 'npm-stage-output-')) @@ -221,7 +214,9 @@ describe('v1 release workflow contract', () => { expect(checkIndex).toBeGreaterThanOrEqual(0) expect(finalizeIndex).toBeGreaterThan(checkIndex) expect(finalize?.run).toContain('make_latest=false') - expect(finalize?.run).toContain('gh release view "${TAG}" --json databaseId') + expect(finalize?.run).toContain( + 'gh release view "${TAG}" --json databaseId', + ) expect(finalize?.run).not.toContain('releases/tags/${TAG}') }) @@ -261,9 +256,13 @@ describe('v1 release workflow contract', () => { const manifest = JSON.parse( readFileSync(new URL('../package.json', import.meta.url), 'utf8'), - ) as { publishConfig?: { tag?: string }; version?: string } - expect(manifest.version).toMatch(/^2\.1\.0-rc\.\d+$/u) - expect(manifest.publishConfig?.tag).toBe('rc-2.1.0') + ) as { publishConfig?: { tag?: string } } + const releasePolicy = JSON.parse( + readFileSync(new URL('../.config/release.json', import.meta.url), 'utf8'), + ) as { identifier: string; versionHint: string } + expect(manifest.publishConfig?.tag).toBe( + `${releasePolicy.identifier}-${releasePolicy.versionHint}`, + ) expect(workflow.on.workflow_dispatch.inputs['release-major']?.default).toBe( '1', )