From ba1aaa7a534ea5e7f175c2de109a179dc4fc0559 Mon Sep 17 00:00:00 2001 From: GENTILHOMME Thomas Date: Wed, 11 Mar 2026 14:26:41 +0100 Subject: [PATCH] feat: implement new OSV strategy --- README.md | 12 +- docs/database/osv.md | 80 ++++++++-- docs/osv.md | 65 ++++++++ package.json | 1 + src/constants.ts | 1 + src/database/index.ts | 5 +- src/database/osv.ts | 66 ++++++-- src/extractor/index.ts | 61 ++++++++ src/formats/osv/index.ts | 4 +- src/formats/standard/index.ts | 2 +- src/formats/standard/mappers.ts | 89 ++++++++++- src/index.ts | 31 +++- src/strategies/github-advisory.ts | 45 +++--- src/strategies/osv.ts | 122 +++++++++++++++ test/database/osv.unit.spec.ts | 55 ++++--- test/extractor/index.unit.spec.ts | 49 ++++++ test/fixtures/extractor/package-lock.json | 28 ++++ test/fixtures/extractor/package.json | 7 + test/strategies/osv/index.unit.spec.ts | 178 ++++++++++++++++++++++ 19 files changed, 814 insertions(+), 87 deletions(-) create mode 100644 docs/osv.md create mode 100644 src/extractor/index.ts create mode 100644 src/strategies/osv.ts create mode 100644 test/extractor/index.unit.spec.ts create mode 100644 test/fixtures/extractor/package-lock.json create mode 100644 test/fixtures/extractor/package.json create mode 100644 test/strategies/osv/index.unit.spec.ts diff --git a/README.md b/README.md index eaf1d6d..7dc670e 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ The **vuln-*era*** has begun! Programmatically fetch security vulnerabilities with one or many strategies. Originally designed to run and analyze [Scanner](https://github.com/NodeSecure/scanner) dependencies it now also runs independently from an npm Manifest. ## Requirements -- [Node.js](https://nodejs.org/en/) v22 or higher +- [Node.js](https://nodejs.org/en/) v24 or higher ## Getting Started @@ -51,13 +51,14 @@ console.log(vulnerabilities); The default strategy is **NONE** which mean no strategy at all (we execute nothing). -[GitHub Advisory](./docs/github_advisory.md) | [Sonatype - OSS Index](./docs/sonatype.md) | Snyk -:-------------------------:|:-------------------------:|:-------------------------: - | | +- [GitHub Advisory](./docs/github_advisory.md) +- [Sonatype OSS Index](./docs/sonatype.md) +- [OSV](./docs/osv.md) +- Snyk Those strategies are described as "string" **type** with the following TypeScript definition: ```ts -type Kind = "github-advisory" | "snyk" | "sonatype" | "none"; +type Kind = "github-advisory" | "snyk" | "sonatype" | "osv" | "none"; ``` To add a strategy or better understand how the code works, please consult [the following guide](./docs/adding_new_strategy.md). @@ -72,6 +73,7 @@ const strategies: Object.freeze({ GITHUB_ADVISORY: "github-advisory", SNYK: "snyk", SONATYPE: "sonatype", + OSV: "osv", NONE: "none" }); diff --git a/docs/database/osv.md b/docs/database/osv.md index e8db3b1..e1db69d 100644 --- a/docs/database/osv.md +++ b/docs/database/osv.md @@ -1,10 +1,10 @@ # OSV -OSV stand for Open Source Vulnerability database. This project is an open, precise, and distributed approach to producing and consuming vulnerability information for open source. +OSV stands for Open Source Vulnerability database. This project is an open, precise, and distributed approach to producing and consuming vulnerability information for open source. All advisories in this database use the [OpenSSF OSV format](https://ossf.github.io/osv-schema/), which was developed in collaboration with open source communities. -Lean more at [osv.dev](https://osv.dev/) +Learn more at [osv.dev](https://osv.dev/) ## Format @@ -26,36 +26,88 @@ export interface OSVOptions { } ``` -### `findOne(parameters: OSVApiParameter): Promise` -Find the vulnerabilities of a given package using available OSV API parameters. +No credentials are required to use the OSV public API. The optional `credential` can be used to attach API key headers if needed. + +### `query(parameters: OSVQueryBatchEntry): Promise` + +Find the vulnerabilities of a given package using available OSV API parameters. Defaults the ecosystem to `npm` if not specified. ```ts -export type OSVApiParameter = { +export type OSVQueryBatchEntry = { version?: string; package: { name: string; /** - * @default npm + * @default "npm" */ ecosystem?: string; }; -} +}; ``` -### `findOneBySpec(spec: string): Promise` -Find the vulnerabilities of a given package using the NPM spec format like `packageName@version`. +Example: ```ts -const vulns = await db.findOneBySpec("01template1"); +import * as vulnera from "@nodesecure/vulnera"; + +const db = new vulnera.Database.OSV(); + +const vulns = await db.query({ + version: "1.0.0", + package: { name: "lodash" } +}); console.log(vulns); ``` -### `findMany(specs: T[]): Promise>` -Find the vulnerabilities of many packages using the spec format. +### `queryBySpec(spec: string): Promise` -Return a Record where keys are equals to the provided specs. +Find the vulnerabilities of a given package using the npm spec format `packageName@version`. ```ts -const vulns = await db.findMany(["express@4.0.0", "lodash@4.17.0"]); +import * as vulnera from "@nodesecure/vulnera"; + +const db = new vulnera.Database.OSV(); + +const vulns = await db.queryBySpec("lodash@4.17.20"); console.log(vulns); ``` + +### `queryBatch(queries: OSVQueryBatchEntry[]): Promise` + +Query multiple packages at once using the `/v1/querybatch` OSV endpoint. Results are returned in the same order as the input queries. + +```ts +export interface OSVQueryBatchResult { + vulns?: OSV[]; +} +``` + +Example: + +```ts +import * as vulnera from "@nodesecure/vulnera"; + +const db = new vulnera.Database.OSV(); + +const results = await db.queryBatch([ + { version: "4.17.20", package: { name: "lodash" } }, + { version: "1.0.0", package: { name: "minimist" } } +]); + +for (const result of results) { + console.log(result.vulns ?? []); +} +``` + +### `findVulnById(id: string): Promise` + +Fetch a single vulnerability entry by its OSV identifier (e.g. `GHSA-xxxx-xxxx-xxxx` or `RUSTSEC-xxxx-xxxx`). + +```ts +import * as vulnera from "@nodesecure/vulnera"; + +const db = new vulnera.Database.OSV(); + +const vuln = await db.findVulnById("GHSA-p6mc-m468-83gw"); +console.log(vuln); +``` diff --git a/docs/osv.md b/docs/osv.md new file mode 100644 index 0000000..4c084e6 --- /dev/null +++ b/docs/osv.md @@ -0,0 +1,65 @@ +# OSV strategy + +The OSV strategy queries the [OSV (Open Source Vulnerability)](https://osv.dev/) public API directly. It uses the `/v1/querybatch` endpoint for efficient batch lookups, resolving vulnerabilities for all dependencies at once. + +No credentials or local database synchronization are required. + +## How it works + +1. Dependencies are extracted from the local project using `NodeDependencyExtractor` (via Arborist), which reads from `node_modules` or falls back to the lockfile. +2. All `name@version` pairs are batched into chunks of up to **1000** entries and sent to the OSV batch API. +3. Results are mapped back to each package and optionally converted to the [Standard](./formats/standard.md) or [OSV](./formats/osv.md) format. + +## Usage + +### `getVulnerabilities(path, options?)` + +Scans a local project directory and returns all found vulnerabilities. + +```js +import * as vulnera from "@nodesecure/vulnera"; + +const definition = vulnera.setStrategy(vulnera.strategies.OSV); + +const vulnerabilities = await definition.getVulnerabilities(process.cwd()); +console.log(vulnerabilities); +``` + +With the Standard NodeSecure format: + +```js +import * as vulnera from "@nodesecure/vulnera"; + +const definition = vulnera.setStrategy(vulnera.strategies.OSV); + +const vulnerabilities = await definition.getVulnerabilities(process.cwd(), { + useFormat: "Standard" +}); +console.log(vulnerabilities); +``` + +### `hydratePayloadDependencies(dependencies, options?)` + +Hydrates a Scanner dependencies `Map` in-place with vulnerability data. + +```js +import * as vulnera from "@nodesecure/vulnera"; + +const dependencies = new Map(); +// ...populate dependencies from Scanner... + +const definition = vulnera.setStrategy(vulnera.strategies.OSV); +await definition.hydratePayloadDependencies(dependencies); +``` + +With the Standard NodeSecure format: + +```js +await definition.hydratePayloadDependencies(dependencies, { + useFormat: "Standard" +}); +``` + +## OSV Database + +The strategy uses the [`OSV` database class](./database/osv.md) internally. You can also use it directly for lower-level access to the OSV API (single queries, batch queries, lookup by ID). diff --git a/package.json b/package.json index 4f492ce..251735b 100644 --- a/package.json +++ b/package.json @@ -55,6 +55,7 @@ "@openally/config.typescript": "1.2.1", "@slimio/is": "^2.0.0", "@types/node": "^25.0.2", + "@types/npmcli__arborist": "6.3.3", "c8": "^11.0.0", "typescript": "^5.4.2" }, diff --git a/src/constants.ts b/src/constants.ts index 04d492b..5fc5cfb 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -5,6 +5,7 @@ export const VULN_MODE = Object.freeze({ GITHUB_ADVISORY: "github-advisory", SNYK: "snyk", SONATYPE: "sonatype", + OSV: "osv", NONE: "none" }); export type Kind = typeof VULN_MODE[keyof typeof VULN_MODE]; diff --git a/src/database/index.ts b/src/database/index.ts index 26e5272..232775f 100644 --- a/src/database/index.ts +++ b/src/database/index.ts @@ -12,7 +12,10 @@ export type { export { OSV } from "./osv.ts"; export type { OSVOptions, - OSVApiParameter + OSVQueryBatchEntry, + OSVQueryBatchRequest, + OSVQueryBatchResult, + OSVQueryBatchResponse } from "./osv.ts"; export { Snyk } from "./snyk.ts"; diff --git a/src/database/osv.ts b/src/database/osv.ts index 2621c18..2f7c19a 100644 --- a/src/database/osv.ts +++ b/src/database/osv.ts @@ -6,7 +6,7 @@ import type { OSV as OSVFormat } from "../formats/osv/index.ts"; import * as utils from "../utils.ts"; import type { ApiCredential } from "../credential.ts"; -export type OSVApiParameter = { +export type OSVQueryBatchEntry = { version?: string; package: { name: string; @@ -17,6 +17,18 @@ export type OSVApiParameter = { }; }; +export interface OSVQueryBatchRequest { + queries: OSVQueryBatchEntry[]; +} + +export interface OSVQueryBatchResult { + vulns?: OSVFormat[]; +} + +export interface OSVQueryBatchResponse { + results: OSVQueryBatchResult[]; +} + export interface OSVOptions { credential?: ApiCredential; } @@ -32,44 +44,68 @@ export class OSV { this.#credential = options.credential; } - async findOne( - parameters: OSVApiParameter + async query( + query: OSVQueryBatchEntry ): Promise { - if (!parameters.package.ecosystem) { - parameters.package.ecosystem = "npm"; + if (!query.package.ecosystem) { + query.package.ecosystem = "npm"; } const { data } = await httpie.post<{ vulns: OSVFormat[]; }>( new URL("v1/query", OSV.ROOT_API), { headers: this.#credential?.headers, - body: parameters + body: query } ); return data.vulns; } - findOneBySpec( + queryBySpec( spec: string ): Promise { const { name, version } = utils.parseNpmSpec(spec); - return this.findOne({ + return this.query({ version, package: { - name + name, + ecosystem: "npm" } }); } - async findMany( - specs: T[] - ): Promise> { - const entries = await Promise.all( - specs.map(async(spec) => [spec, await this.findOneBySpec(spec)] as [T, OSVFormat[]]) + async queryBatch( + queries: OSVQueryBatchEntry[] + ): Promise { + for (const query of queries) { + if (!query.package.ecosystem) { + query.package.ecosystem = "npm"; + } + } + + const { data } = await httpie.post( + new URL("v1/querybatch", OSV.ROOT_API), + { + headers: this.#credential?.headers, + body: { queries } + } + ); + + return data.results; + } + + async findVulnById( + id: string + ): Promise { + const { data } = await httpie.get( + new URL(`v1/vulns/${id}`, OSV.ROOT_API), + { + headers: this.#credential?.headers + } ); - return Object.fromEntries(entries) as Record; + return data; } } diff --git a/src/extractor/index.ts b/src/extractor/index.ts new file mode 100644 index 0000000..a332894 --- /dev/null +++ b/src/extractor/index.ts @@ -0,0 +1,61 @@ +// Import Node.js Dependencies +import fs from "node:fs/promises"; +import nodePath from "node:path"; + +// Import Third-party Dependencies +import Arborist from "@npmcli/arborist"; + +// Import Internal Dependencies +import { NPM_TOKEN } from "../constants.ts"; + +export type PackageSpec = { + name: string; + version: string; +}; + +/** + * Extracts npm package dependencies from a local project using Arborist. + * Tries to load from `node_modules` first (`loadActual`), then falls back + * to the lockfile (`loadVirtual`) when `node_modules` is absent. + */ +export class NodeDependencyExtractor { + async extract( + projectPath: string + ): Promise { + const arborist = new Arborist({ ...NPM_TOKEN, path: projectPath }); + + let root: Arborist.Node; + try { + await fs.access(nodePath.join(projectPath, "node_modules")); + root = await arborist.loadActual(); + } + catch { + root = await arborist.loadVirtual(); + } + + return Array.from( + collectFromTree(root) + ); + } +} + +function* collectFromTree( + root: Arborist.Node +): IterableIterator { + const seen = new Set(); + const queue: Arborist.Node[] = [root]; + + while (queue.length > 0) { + const node = queue.shift()!; + for (const [, child] of node.children) { + if (seen.has(child)) { + continue; + } + seen.add(child); + if (!child.isRoot && !child.isWorkspace && child.version) { + yield { name: child.name, version: child.version }; + } + queue.push(child); + } + } +} diff --git a/src/formats/osv/index.ts b/src/formats/osv/index.ts index 6ddbdba..56bea90 100644 --- a/src/formats/osv/index.ts +++ b/src/formats/osv/index.ts @@ -85,12 +85,12 @@ export interface OSVSeverity { export type OSVKind = keyof typeof OSV_VULN_MAPPERS; export function osvVulnerabilityMapper( - strategy: OSVKind, + strategy: OSVKind | string, vulnerabilities: any[] ): OSV[] { if (!(strategy in OSV_VULN_MAPPERS)) { return []; } - return vulnerabilities.map(OSV_VULN_MAPPERS[strategy]); + return vulnerabilities.map(OSV_VULN_MAPPERS[strategy as OSVKind]); } diff --git a/src/formats/standard/index.ts b/src/formats/standard/index.ts index 42c213d..a395a90 100644 --- a/src/formats/standard/index.ts +++ b/src/formats/standard/index.ts @@ -50,7 +50,7 @@ export interface StandardVulnerability { export type StandardizeKind = keyof typeof STANDARD_VULN_MAPPERS; export function standardVulnerabilityMapper( - strategy: StandardizeKind, + strategy: StandardizeKind | string, vulnerabilities: any[] ): StandardVulnerability[] { if (!(strategy in STANDARD_VULN_MAPPERS)) { diff --git a/src/formats/standard/mappers.ts b/src/formats/standard/mappers.ts index d56d6b2..7ffd0dc 100644 --- a/src/formats/standard/mappers.ts +++ b/src/formats/standard/mappers.ts @@ -8,6 +8,26 @@ import type { PnpmAuditAdvisory, StandardVulnerability } from "../../index.ts"; +type Severity = "info" | "low" | "medium" | "high" | "critical"; + +/** Minimal OSV shape needed by mapFromOSV (avoids circular import chain) */ +interface OSVVulnForMapper { + id: string; + summary: string; + details: string; + aliases?: string[]; + references?: Array<{ type: string; url: string; }>; + severity?: Array<{ type: string; score: string; }>; + affected?: Array<{ + versions?: string[]; + ranges?: Array<{ + type: string; + events: Array<{ introduced?: string; fixed?: string; }>; + }>; + }>; + database_specific?: Record; + package: string; +} function mapFromNPM(vuln: NpmAuditAdvisory): StandardVulnerability { const hasCVSS = typeof vuln.cvss !== "undefined"; @@ -89,10 +109,77 @@ function mapFromSonatype(vuln: SonatypeVulnerability): StandardVulnerability { }; } +function osvSeverityToStandard( + severity: string +): Severity | undefined { + const lower = severity.toLowerCase(); + if (lower === "moderate") { + return "medium"; + } + if (lower === "low" || lower === "medium" || lower === "high" || lower === "critical" || lower === "info") { + return lower; + } + + return undefined; +} + +function mapFromOSV( + vuln: OSVVulnForMapper +): StandardVulnerability { + const advisoryRef = vuln.references?.find((r) => r.type === "ADVISORY") ?? vuln.references?.[0]; + const cves = vuln.aliases?.filter((a) => a.startsWith("CVE-")) ?? []; + + const affected = vuln.affected?.[0]; + const vulnerableVersions = affected?.versions ?? []; + + const semverRanges = affected?.ranges?.filter((r) => r.type === "SEMVER") ?? []; + const vulnerableRanges: string[] = semverRanges.flatMap((range) => { + const ranges: string[] = []; + let intro: string | undefined; + for (const event of range.events) { + if (event.introduced !== undefined) { + intro = event.introduced; + } + else if (event.fixed !== undefined && intro !== undefined) { + ranges.push(`>=${intro} <${event.fixed}`); + intro = undefined; + } + } + + return ranges; + }); + + const patchedVersions = semverRanges + .flatMap((r) => r.events.filter((e) => e.fixed !== undefined).map((e) => e.fixed!)) + .join(" || ") || undefined; + + const cvssEntry = vuln.severity?.[0]; + const cvssVector = cvssEntry?.score; + + const dbSeverity = vuln.database_specific?.severity as string | undefined; + const severity = dbSeverity ? osvSeverityToStandard(dbSeverity) : undefined; + + return { + id: vuln.id, + origin: VULN_MODE.OSV, + package: vuln.package, + title: vuln.summary, + description: vuln.details, + url: advisoryRef?.url, + severity, + cves: cves.length > 0 ? cves : undefined, + cvssVector, + vulnerableVersions, + vulnerableRanges, + patchedVersions + }; +} + export const STANDARD_VULN_MAPPERS = Object.freeze({ [VULN_MODE.GITHUB_ADVISORY]: mapFromNPM, "github-advisory_pnpm": mapFromPnpm, [VULN_MODE.SNYK]: mapFromSnyk, - [VULN_MODE.SONATYPE]: mapFromSonatype + [VULN_MODE.SONATYPE]: mapFromSonatype, + [VULN_MODE.OSV]: mapFromOSV }); diff --git a/src/index.ts b/src/index.ts index d65b05d..23b55af 100644 --- a/src/index.ts +++ b/src/index.ts @@ -25,6 +25,11 @@ import { type NoneStrategyDefinition } from "./strategies/none.ts"; +import { + OSVStrategy, + type OSVStrategyDefinition +} from "./strategies/osv.ts"; + import { VULN_MODE, type Kind @@ -34,7 +39,7 @@ import { ApiCredential, type ApiCredentialOptions } from "./credential.ts"; import { formatVulnsPayload -} from "./formats/index.js"; +} from "./formats/index.ts"; import type { SnykVulnerability @@ -67,6 +72,7 @@ export type AllStrategy = { "github-advisory": GithubAdvisoryStrategyDefinition; snyk: SnykStrategyDefinition; sonatype: SonatypeStrategyDefinition; + osv: OSVStrategyDefinition; }; export type AnyStrategy = AllStrategy[keyof AllStrategy]; @@ -75,6 +81,7 @@ type StrategyOptions = { "github-advisory": undefined; snyk: SnykStrategyOptions; sonatype: SonatypeStrategyOptions; + osv: undefined; }; // CONSTANTS @@ -94,16 +101,29 @@ export function setStrategy( } if (name === VULN_MODE.GITHUB_ADVISORY) { - localVulnerabilityStrategy = Object.seal(GitHubAdvisoryStrategy()); + localVulnerabilityStrategy = Object.seal( + GitHubAdvisoryStrategy() + ); } else if (name === VULN_MODE.SNYK) { - localVulnerabilityStrategy = Object.seal(SnykStrategy(options as SnykStrategyOptions)); + localVulnerabilityStrategy = Object.seal( + SnykStrategy(options as SnykStrategyOptions) + ); } else if (name === VULN_MODE.SONATYPE) { - localVulnerabilityStrategy = Object.seal(SonatypeStrategy(options as SonatypeStrategyOptions)); + localVulnerabilityStrategy = Object.seal( + SonatypeStrategy(options as SonatypeStrategyOptions) + ); + } + else if (name === VULN_MODE.OSV) { + localVulnerabilityStrategy = Object.seal( + OSVStrategy() + ); } else { - localVulnerabilityStrategy = Object.seal(NoneStrategy()); + localVulnerabilityStrategy = Object.seal( + NoneStrategy() + ); } return localVulnerabilityStrategy as AllStrategy[T]; @@ -138,6 +158,7 @@ export type { PnpmAuditAdvisory, SnykVulnerability, SonatypeVulnerability, + OSVStrategyDefinition, OSV, formatVulnsPayload diff --git a/src/strategies/github-advisory.ts b/src/strategies/github-advisory.ts index 55eb96f..5f15f34 100644 --- a/src/strategies/github-advisory.ts +++ b/src/strategies/github-advisory.ts @@ -20,19 +20,9 @@ import type { HydratePayloadDepsOptions } from "./types/api.ts"; -export type NpmAuditAdvisory = { - /** The unique cache key for this vuln or metavuln. **/ - source: number; +export type NpmAuditAdvisory = Omit & { /** Same as source (but seems deprecated now) **/ - id?: number; - /** The name of the package that this vulnerability is about**/ - name: string; - /** For metavulns, the dependency that causes this package to be have a vulnerability. For advisories, the same as name. **/ - dependency: string; - /** The text title of the advisory or metavuln **/ - title: string; - /** The url for the advisory (null for metavulns) **/ - url: string; + id?: string | number; /** Publicly-known vulnerabilities have identification numbers, known as Common Vulnerabilities and Exposures (CVEs) */ cwe?: string[]; /** The Common Vulnerability Scoring System (CVSS) is a method used to supply a qualitative measure of severity. CVSS is not a measure of risk. */ @@ -40,15 +30,9 @@ export type NpmAuditAdvisory = { score: number; vectorString: string; }; - /** The severity level **/ - severity: "info" | "low" | "moderate" | "high" | "critical"; - /** The range that is vulnerable **/ - range: string; - /** The set of versions that are vulnerable **/ - vulnerableVersions?: string[]; }; -export type PnpmAuditAdvisory = Exclude & { +export type PnpmAuditAdvisory = Omit & { github_advisory_id: string; npm_advisory_id: null | number; cwe: string | string[]; @@ -120,7 +104,7 @@ async function hydratePayloadDependencies( await npmAudit(path, registry); for (const packageVulns of vulnerabilities) { - const packageName = (packageVulns as NpmAuditAdvisory).name || (packageVulns as PnpmAuditAdvisory).module_name; + const packageName = "name" in packageVulns ? packageVulns.name : packageVulns.module_name; if (!dependencies.has(packageName)) { continue; } @@ -141,10 +125,16 @@ async function npmAudit( path: string, registry: string ): Promise { - const arborist = new Arborist({ ...NPM_TOKEN, registry, path }); - const { vulnerabilities } = (await arborist.audit()).toJSON() as { vulnerabilities: any[]; }; + const arborist = new Arborist({ + ...NPM_TOKEN, + registry, + path + }); + + const { vulnerabilities } = ( + await arborist.audit() + ).toJSON(); - // TODO: remove Symbols? return Object.values(vulnerabilities) .flatMap((vuln) => (Array.isArray(vuln.via) && typeof vuln.via[0] === "object" ? vuln.via : [])); } @@ -164,8 +154,9 @@ async function pnpmAudit( ignoreIncompatible: false }); - // eslint-disable-next-line - const getAuthHeader = () => (void 0); + function getAuthHeader() { + return void 0; + } const { advisories } = await audit( lockfile!, getAuthHeader, @@ -176,7 +167,9 @@ async function pnpmAudit( return Object.values(advisories) as PnpmAuditAdvisory[]; } -async function hasPnpmLockFile(lockfileDir: string): Promise { +async function hasPnpmLockFile( + lockfileDir: string +): Promise { try { await fs.access( path.join(lockfileDir, "pnpm-lock.yaml"), diff --git a/src/strategies/osv.ts b/src/strategies/osv.ts new file mode 100644 index 0000000..413137e --- /dev/null +++ b/src/strategies/osv.ts @@ -0,0 +1,122 @@ +/* eslint-disable no-empty */ +// Import Internal Dependencies +import { VULN_MODE } from "../constants.ts"; +import type { OSV as OSVFormat } from "../formats/osv/index.ts"; +import type { StandardVulnerability } from "../formats/standard/index.ts"; +import { formatVulnsPayload } from "../formats/index.ts"; +import type { Dependencies } from "./types/scanner.ts"; +import type { + BaseStrategyOptions, + ExtendedStrategy, + HydratePayloadDepsOptions +} from "./types/api.ts"; +import { OSV } from "../database/index.ts"; +import type { OSVQueryBatchEntry } from "../database/osv.ts"; +import { + NodeDependencyExtractor, + type PackageSpec +} from "../extractor/index.ts"; +import * as utils from "../utils.ts"; + +// CONSTANTS +const kBatchSize = 1000; + +export type OSVStrategyDefinition = ExtendedStrategy<"osv", OSVFormat>; + +/** + * Creates an OSV vulnerability scanning strategy that queries the OSV database + * directly using the /v1/querybatch endpoint for efficient batch lookups. + * No credentials are required for the OSV public API. + */ +export function OSVStrategy(): OSVStrategyDefinition { + const db = new OSV(); + + return { + strategy: VULN_MODE.OSV, + hydratePayloadDependencies: hydratePayloadDependencies.bind(null, db), + getVulnerabilities: getVulnerabilities.bind(null, db) + }; +} + +type OSVAnnotatedFormat = OSVFormat & { package: string; }; + +function toQuery( + { name, version }: PackageSpec +): OSVQueryBatchEntry { + return { + version, + package: { name, ecosystem: "npm" } + }; +} + +async function queryAndAnnotate( + db: OSV, + pairs: PackageSpec[] +): Promise { + const queries = pairs.map(toQuery); + const allResults: Awaited> = []; + + for (const chunk of utils.chunkArray(queries, kBatchSize)) { + const results = await db.queryBatch(chunk); + allResults.push(...results); + } + + const annotatedVulns: OSVAnnotatedFormat[] = []; + for (let i = 0; i < allResults.length; i++) { + const result = allResults[i]; + if (!result.vulns) { + continue; + } + const { name } = pairs[i]; + for (const vuln of result.vulns) { + annotatedVulns.push({ ...vuln, package: name }); + } + } + + return annotatedVulns; +} + +async function getVulnerabilities( + db: OSV, + path: string, + options: BaseStrategyOptions = {} +): Promise<(OSVFormat | StandardVulnerability)[]> { + const { useFormat } = options; + + const extractor = new NodeDependencyExtractor(); + const packages = await extractor.extract(path); + const annotatedVulns = await queryAndAnnotate(db, packages); + + return formatVulnsPayload( + useFormat + )(VULN_MODE.OSV, annotatedVulns); +} + +async function hydratePayloadDependencies( + db: OSV, + dependencies: Dependencies, + options: HydratePayloadDepsOptions = {} +): Promise { + const { useFormat } = options; + + const pairs: PackageSpec[] = []; + for (const [name, dep] of dependencies) { + for (const version of Object.keys(dep.versions)) { + pairs.push({ name, version }); + } + } + + try { + const annotatedVulns = await queryAndAnnotate(db, pairs); + const formatVulnerabilities = formatVulnsPayload(useFormat); + + for (const annotated of annotatedVulns) { + const dep = dependencies.get(annotated.package); + if (dep) { + const formatted = formatVulnerabilities(VULN_MODE.OSV, [annotated]); + dep.vulnerabilities.push(...formatted); + } + } + } + catch { } +} diff --git a/test/database/osv.unit.spec.ts b/test/database/osv.unit.spec.ts index ef2e0af..7317ac8 100644 --- a/test/database/osv.unit.spec.ts +++ b/test/database/osv.unit.spec.ts @@ -18,7 +18,7 @@ describe("Database.OSV", () => { restoreHttpAgent(); }); - test(`should send a POST http request to the OSV API using findOne + test(`should send a POST http request to the OSV API using query and then return the 'vulns' property from the JSON response`, async() => { const expectedResponse = { vulns: "hello world" }; mockedHttpClient @@ -29,7 +29,7 @@ describe("Database.OSV", () => { }) .reply(200, expectedResponse, HTTP_CLIENT_HEADERS); - const vulns = await db.findOne({ + const vulns = await db.query({ package: { name: "foobar", ecosystem: "npm" @@ -38,7 +38,7 @@ describe("Database.OSV", () => { assert.strictEqual(vulns, expectedResponse.vulns); }); - test(`should send a POST http request to the OSV API using findOneBySpec + test(`should send a POST http request to the OSV API using queryBySpec and then return the 'vulns' property from the JSON response`, async() => { const expectedResponse = { vulns: "hello world" }; const packageName = "@nodesecure/js-x-ray"; @@ -54,27 +54,48 @@ describe("Database.OSV", () => { }) .reply(200, expectedResponse, HTTP_CLIENT_HEADERS); - const vulns = await db.findOneBySpec(`${packageName}@2.0.0`); + const vulns = await db.queryBySpec(`${packageName}@2.0.0`); assert.strictEqual(vulns, expectedResponse.vulns); }); - test("should send multiple POST http requests to the OSV API using findMany", async() => { - const expectedResponse = { vulns: [1, 2, 3] }; + test("should send a POST http request to /v1/querybatch and return the 'results' array", async() => { + const expectedResults = [ + { vulns: [{ id: "OSV-2021-1" }] }, + {} + ]; + const queries = [ + { package: { name: "foo", ecosystem: "npm" }, version: "1.0.0" }, + { package: { name: "bar", ecosystem: "npm" }, version: "2.0.0" } + ]; mockedHttpClient .intercept({ - path: new URL("/v1/query", OSV.ROOT_API).href, - method: "POST" + path: new URL("/v1/querybatch", OSV.ROOT_API).href, + method: "POST", + body: JSON.stringify({ queries }) }) - .reply(200, expectedResponse, HTTP_CLIENT_HEADERS) - .times(2); + .reply(200, { results: expectedResults }, HTTP_CLIENT_HEADERS); - const result = await db.findMany( - ["foobar", "yoobar"] - ); - assert.deepEqual(result, { - foobar: expectedResponse.vulns, - yoobar: expectedResponse.vulns - }); + const results = await db.queryBatch(queries); + assert.deepEqual(results, expectedResults); + }); + + test("should send a GET http request to /v1/vulns/{id} and return the full OSV object", async() => { + const vulnId = "OSV-2021-1234"; + const expectedVuln = { + id: vulnId, + summary: "Fake vulnerability", + modified: "2021-01-01T00:00:00Z" + }; + + mockedHttpClient + .intercept({ + path: new URL(`/v1/vulns/${vulnId}`, OSV.ROOT_API).href, + method: "GET" + }) + .reply(200, expectedVuln, HTTP_CLIENT_HEADERS); + + const vuln = await db.findVulnById(vulnId); + assert.deepEqual(vuln, expectedVuln); }); }); diff --git a/test/extractor/index.unit.spec.ts b/test/extractor/index.unit.spec.ts new file mode 100644 index 0000000..886e71d --- /dev/null +++ b/test/extractor/index.unit.spec.ts @@ -0,0 +1,49 @@ +// Import Node.js Dependencies +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { test } from "node:test"; +import assert from "node:assert"; + +// Import Internal Dependencies +import { + NodeDependencyExtractor, + type PackageSpec +} from "../../src/extractor/index.ts"; + +// CONSTANTS +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const kFixturesDir = path.join(__dirname, "..", "fixtures"); +const kExtractorFixture = path.join(kFixturesDir, "extractor"); + +// The fixture has exactly two packages: once@1.4.0 and its dep wrappy@1.0.2 +const kExpectedPackages: PackageSpec[] = [ + { name: "once", version: "1.4.0" }, + { name: "wrappy", version: "1.0.2" } +]; + +test("NodeDependencyExtractor: extract() returns one PackageSpec per lockfile entry", async() => { + const extractor = new NodeDependencyExtractor(); + const packages = await extractor.extract(kExtractorFixture); + + assert.strictEqual(packages.length, kExpectedPackages.length); + assert.deepEqual( + packages.slice().sort((a, b) => a.name.localeCompare(b.name)), + kExpectedPackages.slice().sort((a, b) => a.name.localeCompare(b.name)) + ); +}); + +test("NodeDependencyExtractor: extract() does not include the root package", async() => { + const extractor = new NodeDependencyExtractor(); + const packages = await extractor.extract(kExtractorFixture); + + const rootEntry = packages.find((pkg) => pkg.name === "test-extractor-project"); + assert.strictEqual(rootEntry, undefined, "the root package must not appear in results"); +}); + +test("NodeDependencyExtractor: extract() returns no duplicate name@version pairs", async() => { + const extractor = new NodeDependencyExtractor(); + const packages = await extractor.extract(kExtractorFixture); + + const specs = new Set(packages.map((pkg) => `${pkg.name}@${pkg.version}`)); + assert.strictEqual(specs.size, packages.length, "must not contain duplicate name@version entries"); +}); diff --git a/test/fixtures/extractor/package-lock.json b/test/fixtures/extractor/package-lock.json new file mode 100644 index 0000000..b093946 --- /dev/null +++ b/test/fixtures/extractor/package-lock.json @@ -0,0 +1,28 @@ +{ + "name": "test-extractor-project", + "version": "1.0.0", + "lockfileVersion": 2, + "requires": true, + "packages": { + "": { + "name": "test-extractor-project", + "version": "1.0.0", + "dependencies": { + "once": "^1.4.0" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFuh+znXDN34B5COryvXiA+a6fKkQKE8xTOLRLxemN8CRRg==", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" + } + } +} diff --git a/test/fixtures/extractor/package.json b/test/fixtures/extractor/package.json new file mode 100644 index 0000000..7ef84b6 --- /dev/null +++ b/test/fixtures/extractor/package.json @@ -0,0 +1,7 @@ +{ + "name": "test-extractor-project", + "version": "1.0.0", + "dependencies": { + "once": "^1.4.0" + } +} diff --git a/test/strategies/osv/index.unit.spec.ts b/test/strategies/osv/index.unit.spec.ts new file mode 100644 index 0000000..2d6a4cf --- /dev/null +++ b/test/strategies/osv/index.unit.spec.ts @@ -0,0 +1,178 @@ +// Import Node.js Dependencies +import { test } from "node:test"; +import assert from "node:assert"; + +// Import Internal Dependencies +import { OSVStrategy } from "../../../src/strategies/osv.ts"; +import { OSV } from "../../../src/database/index.ts"; +import { + expectVulnToBeNodeSecureStandardCompliant, + HTTP_CLIENT_HEADERS, + setupHttpAgentMock +} from "../utils.ts"; + +// CONSTANTS +const kOSVApiOrigin = OSV.ROOT_API; +const kQueryBatchPath = new URL("/v1/querybatch", kOSVApiOrigin).href; + +const kFakeOSVVuln = { + id: "OSV-2021-1234", + modified: "2021-01-01T00:00:00Z", + published: "2021-01-01T00:00:00Z", + aliases: ["CVE-2021-1234"], + upstream: [], + summary: "Fake vulnerability for testing", + details: "This is a fake vulnerability for testing purposes", + severity: [{ type: "CVSS_V3", score: "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H" }], + affected: [ + { + package: { ecosystem: "npm", name: "fake-pkg", purl: "pkg:npm/fake-pkg" }, + severity: [], + ranges: [ + { + type: "SEMVER", + events: [{ introduced: "1.0.0" }, { fixed: "1.0.1" }], + database_specific: {} + } + ], + versions: ["1.0.0"], + ecosystem_specific: {}, + database_specific: {} + } + ], + references: [{ type: "ADVISORY", url: "https://example.com/vuln/OSV-2021-1234" }], + credits: [], + database_specific: { severity: "high" } +}; + +test("OSVStrategy definition must return three keys", () => { + const definition = OSVStrategy(); + + assert.strictEqual( + definition.strategy, + "osv", + "strategy property must equal 'osv'" + ); + assert.deepEqual( + Object.keys(definition).sort(), + ["getVulnerabilities", "hydratePayloadDependencies", "strategy"].sort() + ); +}); + +test("osv strategy: hydratePayloadDependencies", async() => { + const { hydratePayloadDependencies } = OSVStrategy(); + const [mockedHttpAgent, restoreHttpAgent] = setupHttpAgentMock(); + const mockedHttpClient = mockedHttpAgent.get(kOSVApiOrigin); + + const dependencies = new Map(); + dependencies.set("fake-pkg", { + vulnerabilities: [], + versions: { "1.0.0": { id: 1, description: "package description" } } + }); + + mockedHttpClient + .intercept({ + path: kQueryBatchPath, + method: "POST" + }) + .reply( + 200, + { results: [{ vulns: [kFakeOSVVuln] }] }, + HTTP_CLIENT_HEADERS + ); + + await hydratePayloadDependencies(dependencies, {}); + + assert.strictEqual( + dependencies.size, + 1, + "hydratePayloadDependencies must not add new dependencies by itself" + ); + + const { vulnerabilities } = dependencies.get("fake-pkg"); + assert.strictEqual(vulnerabilities.length, 1); + + const [vuln] = vulnerabilities; + assert.strictEqual(vuln.id, kFakeOSVVuln.id); + assert.strictEqual(vuln.package, "fake-pkg"); + + restoreHttpAgent(); +}); + +test("osv strategy: hydratePayloadDependencies when using NodeSecure standard format", async() => { + const { hydratePayloadDependencies } = OSVStrategy(); + const [mockedHttpAgent, restoreHttpAgent] = setupHttpAgentMock(); + const mockedHttpClient = mockedHttpAgent.get(kOSVApiOrigin); + + const dependencies = new Map(); + dependencies.set("fake-pkg", { + vulnerabilities: [], + versions: { "1.0.0": { id: 1, description: "package description" } } + }); + + mockedHttpClient + .intercept({ + path: kQueryBatchPath, + method: "POST" + }) + .reply( + 200, + { results: [{ vulns: [kFakeOSVVuln] }] }, + HTTP_CLIENT_HEADERS + ); + + await hydratePayloadDependencies(dependencies, { useFormat: "Standard" }); + + assert.strictEqual( + dependencies.size, + 1, + "hydratePayloadDependencies must not add new dependencies by itself" + ); + + const { vulnerabilities } = dependencies.get("fake-pkg"); + assert.strictEqual(vulnerabilities.length, 1); + + const [vulnerability] = vulnerabilities; + expectVulnToBeNodeSecureStandardCompliant(vulnerability); + assert.strictEqual(vulnerability.origin, "osv"); + assert.strictEqual(vulnerability.package, "fake-pkg"); + + restoreHttpAgent(); +}); + +test("osv strategy: hydratePayloadDependencies with > 1000 packages sends two batches", async() => { + const { hydratePayloadDependencies } = OSVStrategy(); + const [mockedHttpAgent, restoreHttpAgent] = setupHttpAgentMock(); + const mockedHttpClient = mockedHttpAgent.get(kOSVApiOrigin); + + const fakeDependencyPayload = { + vulnerabilities: [], + versions: { "1.0.0": { id: 1, description: "package description" } } + }; + const dependencies = new Map(); + + // 1001 packages => two batches (1000 + 1) + Array.from({ length: 1001 }, (_, i) => dependencies.set(`fake-pkg-${i}`, fakeDependencyPayload)); + + mockedHttpClient + .intercept({ + path: kQueryBatchPath, + method: "POST" + }) + .reply(200, { results: Array.from({ length: 1000 }, () => { + return {}; + }) }, HTTP_CLIENT_HEADERS); + + mockedHttpClient + .intercept({ + path: kQueryBatchPath, + method: "POST" + }) + .reply(200, { results: [{}] }, HTTP_CLIENT_HEADERS); + + await hydratePayloadDependencies(dependencies, {}); + + mockedHttpAgent.assertNoPendingInterceptors(); + + restoreHttpAgent(); +});