diff --git a/src/index.ts b/src/index.ts index 9aeb0f0..29edd4a 100644 --- a/src/index.ts +++ b/src/index.ts @@ -24,7 +24,7 @@ import { listScenariosForSpec, listClientScenariosForSpec, getScenarioSpecVersions, - ALL_SPEC_VERSIONS + resolveSpecVersion } from './scenarios'; import type { SpecVersion } from './scenarios'; import { ConformanceCheck } from './types'; @@ -37,15 +37,6 @@ import { import { createTierCheckCommand } from './tier-check'; import packageJson from '../package.json'; -function resolveSpecVersion(value: string): SpecVersion { - if (ALL_SPEC_VERSIONS.includes(value as SpecVersion)) { - return value as SpecVersion; - } - console.error(`Unknown spec version: ${value}`); - console.error(`Valid versions: ${ALL_SPEC_VERSIONS.join(', ')}`); - process.exit(1); -} - // Note on naming: `command` refers to which CLI command is calling this. // The `client` command tests Scenario objects (which test clients), // and the `server` command tests ClientScenario objects (which test servers). diff --git a/src/scenarios/index.ts b/src/scenarios/index.ts index 36c8f32..75ca61d 100644 --- a/src/scenarios/index.ts +++ b/src/scenarios/index.ts @@ -227,6 +227,15 @@ export const ALL_SPEC_VERSIONS: SpecVersion[] = [ 'extension' ]; +export function resolveSpecVersion(value: string): SpecVersion { + if (ALL_SPEC_VERSIONS.includes(value as SpecVersion)) { + return value as SpecVersion; + } + console.error(`Unknown spec version: ${value}`); + console.error(`Valid versions: ${ALL_SPEC_VERSIONS.join(', ')}`); + process.exit(1); +} + export function listScenariosForSpec(version: SpecVersion): string[] { return scenariosList .filter((s) => s.specVersions.includes(version)) diff --git a/src/tier-check/checks/test-conformance-results.ts b/src/tier-check/checks/test-conformance-results.ts index f68d879..2a9ce43 100644 --- a/src/tier-check/checks/test-conformance-results.ts +++ b/src/tier-check/checks/test-conformance-results.ts @@ -6,6 +6,8 @@ import { ConformanceResult } from '../types'; import { listScenarios, listActiveClientScenarios, + listScenariosForSpec, + listClientScenariosForSpec, getScenarioSpecVersions } from '../../scenarios'; import { ConformanceCheck, SpecVersion } from '../../types'; @@ -166,6 +168,7 @@ function reconcileWithExpected( export async function checkConformance(options: { serverUrl?: string; skip?: boolean; + specVersion?: SpecVersion; }): Promise { if (options.skip || !options.serverUrl) { return { @@ -179,23 +182,37 @@ export async function checkConformance(options: { } const outputDir = mkdtempSync(join(tmpdir(), 'tier-check-server-')); + const args = [ + process.argv[1], + 'server', + '--url', + options.serverUrl, + '-o', + outputDir + ]; + if (options.specVersion) { + args.push('--spec-version', options.specVersion); + } try { - execFileSync( - process.execPath, - [process.argv[1], 'server', '--url', options.serverUrl, '-o', outputDir], - { - stdio: ['pipe', 'pipe', 'pipe'], - timeout: 120_000 - } - ); + execFileSync(process.execPath, args, { + stdio: ['pipe', 'pipe', 'pipe'], + timeout: 120_000 + }); } catch { // Non-zero exit is expected when tests fail — results are still in outputDir } + const activeScenarios = new Set(listActiveClientScenarios()); + const expectedScenarios = options.specVersion + ? listClientScenariosForSpec(options.specVersion).filter((s) => + activeScenarios.has(s) + ) + : [...activeScenarios]; + return reconcileWithExpected( parseOutputDir(outputDir), - listActiveClientScenarios(), + expectedScenarios, 'server' ); } @@ -206,6 +223,7 @@ export async function checkConformance(options: { export async function checkClientConformance(options: { clientCmd?: string; skip?: boolean; + specVersion?: SpecVersion; }): Promise { if (options.skip || !options.clientCmd) { return { @@ -219,28 +237,32 @@ export async function checkClientConformance(options: { } const outputDir = mkdtempSync(join(tmpdir(), 'tier-check-client-')); + const args = [ + process.argv[1], + 'client', + '--command', + options.clientCmd, + '--suite', + 'all', + '-o', + outputDir + ]; + if (options.specVersion) { + args.push('--spec-version', options.specVersion); + } try { - execFileSync( - process.execPath, - [ - process.argv[1], - 'client', - '--command', - options.clientCmd, - '--suite', - 'all', - '-o', - outputDir - ], - { - stdio: ['pipe', 'pipe', 'pipe'], - timeout: 120_000 - } - ); + execFileSync(process.execPath, args, { + stdio: ['pipe', 'pipe', 'pipe'], + timeout: 120_000 + }); } catch { // Non-zero exit is expected when tests fail — results are still in outputDir } - return reconcileWithExpected(parseOutputDir(outputDir), listScenarios()); + const expectedScenarios = options.specVersion + ? listScenariosForSpec(options.specVersion) + : listScenarios(); + + return reconcileWithExpected(parseOutputDir(outputDir), expectedScenarios); } diff --git a/src/tier-check/index.ts b/src/tier-check/index.ts index 5416748..1cd32c1 100644 --- a/src/tier-check/index.ts +++ b/src/tier-check/index.ts @@ -13,6 +13,7 @@ import { checkSpecTracking } from './checks/spec-tracking'; import { computeTier } from './tier-logic'; import { formatJson, formatMarkdown, formatTerminal } from './output'; import { TierScorecard } from './types'; +import { resolveSpecVersion } from '../scenarios'; function parseRepo(repo: string): { owner: string; repo: string } { const parts = repo.split('/'); @@ -48,10 +49,18 @@ export function createTierCheckCommand(): Command { '--token ', 'GitHub token (defaults to GITHUB_TOKEN env var)' ) + .option( + '--spec-version ', + 'Only run conformance scenarios for this spec version' + ) .action(async (options) => { const { owner, repo } = parseRepo(options.repo); let token = options.token || process.env.GITHUB_TOKEN; + const specVersion = options.specVersion + ? resolveSpecVersion(options.specVersion) + : undefined; + if (!token) { // Try to get token from GitHub CLI try { @@ -90,14 +99,16 @@ export function createTierCheckCommand(): Command { ] = await Promise.all([ checkConformance({ serverUrl: options.conformanceServerUrl, - skip: options.skipConformance + skip: options.skipConformance, + specVersion }).then((r) => { console.error(' ✓ Server Conformance'); return r; }), checkClientConformance({ clientCmd: options.clientCmd, - skip: options.skipConformance || !options.clientCmd + skip: options.skipConformance || !options.clientCmd, + specVersion }).then((r) => { console.error(' ✓ Client Conformance'); return r;