From a906662ce020f091367638116ab0089ee3410b76 Mon Sep 17 00:00:00 2001 From: Alexander Karan Date: Wed, 22 Apr 2026 12:19:53 +0800 Subject: [PATCH] Added Generate Stats for MPA --- .github/frameworks.json | 14 +- .github/workflows/generate-stats.yml | 3 + .github/workflows/measure-framework.yml | 60 ++++++- .github/workflows/validate-stats.yml | 22 +++ packages/app-astro/ci-stats.json | 10 +- packages/app-next-js/ci-stats.json | 8 +- packages/app-nuxt/ci-stats.json | 14 +- packages/app-react-router/ci-stats.json | 14 +- packages/app-solid-start/ci-stats.json | 14 +- packages/app-sveltekit/ci-stats.json | 8 +- .../app-tanstack-start-react/ci-stats.json | 14 +- packages/stats-generator/package.json | 1 + packages/stats-generator/src/mpa/index.ts | 148 ++++++++++++++++++ .../stats-generator/src/mpa/run-benchmark.ts | 129 +++++++++++++++ packages/stats-generator/src/mpa/types.ts | 25 +++ .../stats-generator/src/run-mpa-benchmark.ts | 78 +++++++++ packages/stats-generator/src/save-ci-stats.ts | 41 +++-- .../stats-generator/src/serve/react-router.ts | 61 ++++++++ packages/stats-generator/src/types.ts | 6 + 19 files changed, 630 insertions(+), 40 deletions(-) create mode 100644 packages/stats-generator/src/mpa/index.ts create mode 100644 packages/stats-generator/src/mpa/run-benchmark.ts create mode 100644 packages/stats-generator/src/mpa/types.ts create mode 100644 packages/stats-generator/src/run-mpa-benchmark.ts create mode 100644 packages/stats-generator/src/serve/react-router.ts diff --git a/.github/frameworks.json b/.github/frameworks.json index 78757004..69b3ebc2 100644 --- a/.github/frameworks.json +++ b/.github/frameworks.json @@ -30,7 +30,7 @@ "package": "app-astro", "buildScript": "build", "buildOutputDir": "dist", - "measurements": [{ "type": "ssr" }, { "type": "spa" }] + "measurements": [{ "type": "ssr" }, { "type": "spa" }, { "type": "mpa" }] } }, { @@ -74,7 +74,7 @@ "package": "app-next-js", "buildScript": "build", "buildOutputDir": ".next", - "measurements": [{ "type": "ssr" }, { "type": "spa" }] + "measurements": [{ "type": "ssr" }, { "type": "spa" }, { "type": "mpa" }] } }, { @@ -96,7 +96,7 @@ "package": "app-nuxt", "buildScript": "build", "buildOutputDir": ".output", - "measurements": [{ "type": "ssr" }, { "type": "spa" }] + "measurements": [{ "type": "ssr" }, { "type": "spa" }, { "type": "mpa" }] } }, { @@ -118,7 +118,7 @@ "package": "app-react-router", "buildScript": "build", "buildOutputDir": "build", - "measurements": [{ "type": "ssr" }, { "type": "spa" }] + "measurements": [{ "type": "ssr" }, { "type": "spa" }, { "type": "mpa" }] } }, { @@ -140,7 +140,7 @@ "package": "app-solid-start", "buildScript": "build", "buildOutputDir": ".output", - "measurements": [{ "type": "ssr" }, { "type": "spa" }] + "measurements": [{ "type": "ssr" }, { "type": "spa" }, { "type": "mpa" }] } }, { @@ -162,7 +162,7 @@ "package": "app-sveltekit", "buildScript": "build", "buildOutputDir": "build", - "measurements": [{ "type": "ssr" }, { "type": "spa" }] + "measurements": [{ "type": "ssr" }, { "type": "spa" }, { "type": "mpa" }] } }, { @@ -184,7 +184,7 @@ "package": "app-tanstack-start-react", "buildScript": "build", "buildOutputDir": ".output", - "measurements": [{ "type": "ssr" }, { "type": "spa" }] + "measurements": [{ "type": "ssr" }, { "type": "spa" }, { "type": "mpa" }] } } ] diff --git a/.github/workflows/generate-stats.yml b/.github/workflows/generate-stats.yml index 1a00a724..a637a177 100644 --- a/.github/workflows/generate-stats.yml +++ b/.github/workflows/generate-stats.yml @@ -27,6 +27,7 @@ jobs: ssr-matrix: ${{ steps.set-matrix.outputs.ssr }} deps-matrix: ${{ steps.set-matrix.outputs.deps }} spa-matrix: ${{ steps.set-matrix.outputs.spa }} + mpa-matrix: ${{ steps.set-matrix.outputs.mpa }} steps: - name: Checkout code uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -40,6 +41,7 @@ jobs: echo "ssr=$(echo "$FRAMEWORKS" | jq -c '[.[] | select(.app) | select(.app.measurements | map(.type) | contains(["ssr"])) | {name, displayName, package: .app.package, buildScript: .app.buildScript, buildOutputDir: .app.buildOutputDir, measurements: .app.measurements}]')" >> $GITHUB_OUTPUT echo "deps=$(echo "$FRAMEWORKS" | jq -c '[.[] | select(.starter) | select(.starter.measurements | map(.type) | contains(["dependencies"])) | {name, displayName, package: .starter.package}]')" >> $GITHUB_OUTPUT echo "spa=$(echo "$FRAMEWORKS" | jq -c '[.[] | select(.app) | select(.app.measurements | map(.type) | contains(["spa"])) | {name, displayName, package: .app.package, buildScript: .app.buildScript, buildOutputDir: .app.buildOutputDir, measurements: .app.measurements}]')" >> $GITHUB_OUTPUT + echo "mpa=$(echo "$FRAMEWORKS" | jq -c '[.[] | select(.app) | select(.app.measurements | map(.type) | contains(["mpa"])) | {name, displayName, package: .app.package, buildScript: .app.buildScript, buildOutputDir: .app.buildOutputDir, measurements: .app.measurements}]')" >> $GITHUB_OUTPUT measure: needs: setup @@ -50,6 +52,7 @@ jobs: ssr-matrix: ${{ needs.setup.outputs.ssr-matrix }} deps-matrix: ${{ needs.setup.outputs.deps-matrix }} spa-matrix: ${{ needs.setup.outputs.spa-matrix }} + mpa-matrix: ${{ needs.setup.outputs.mpa-matrix }} generate-stats: needs: [setup, measure] diff --git a/.github/workflows/measure-framework.yml b/.github/workflows/measure-framework.yml index 5d0dd65a..d51d88cb 100644 --- a/.github/workflows/measure-framework.yml +++ b/.github/workflows/measure-framework.yml @@ -25,6 +25,11 @@ on: type: string required: false default: '[]' + mpa-matrix: + description: 'JSON array of frameworks to measure MPA paint and interaction performance' + type: string + required: false + default: '[]' jobs: measure-install: @@ -205,13 +210,13 @@ jobs: framework: ${{ fromJson(inputs.spa-matrix) }} steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Setup pnpm - uses: pnpm/action-setup@v4 + uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 - name: Setup Node.js - uses: actions/setup-node@v4 + uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 with: node-version: '24' cache: 'pnpm' @@ -243,3 +248,52 @@ jobs: path: packages/${{ matrix.framework.package }}/ci-stats.json retention-days: 1 if-no-files-found: error + + measure-mpa: + if: inputs.mpa-matrix != '[]' + runs-on: ubuntu-latest + container: + image: mcr.microsoft.com/playwright:v1.49.0-noble + options: --ipc=host + strategy: + fail-fast: false + matrix: + framework: ${{ fromJson(inputs.mpa-matrix) }} + steps: + - name: Checkout code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Setup pnpm + uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 + + - name: Setup Node.js + uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 + with: + node-version: '24' + cache: 'pnpm' + + - name: Install workspace dependencies + run: pnpm install --frozen-lockfile + + - name: Install package dependencies + working-directory: ./packages/${{ matrix.framework.package }} + run: pnpm install --frozen-lockfile --ignore-workspace + + - name: Build app + working-directory: ./packages/${{ matrix.framework.package }} + run: pnpm build + + - name: Run MPA benchmark + run: pnpm --filter @framework-tracker/stats-generator run:mpa ${{ matrix.framework.package }} + env: + PLAYWRIGHT_BROWSERS_PATH: /ms-playwright + PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: '1' + RUNNER_LABEL: ubuntu-latest + + - name: Upload MPA stats + uses: actions/upload-artifact@v4 + with: + name: mpa-stats-${{ matrix.framework.name }} + path: packages/${{ matrix.framework.package }}/ci-stats.json + retention-days: 1 + if-no-files-found: error diff --git a/.github/workflows/validate-stats.yml b/.github/workflows/validate-stats.yml index 9a3d1958..fa0d0237 100644 --- a/.github/workflows/validate-stats.yml +++ b/.github/workflows/validate-stats.yml @@ -72,6 +72,28 @@ jobs: (cd packages/$PKG && npx @e18e/cli@0.5.0 analyze --json > e18e-stats.json || true) done + echo "" + echo "=== Running SPA benchmarks ===" + FRAMEWORKS=$(cat .github/frameworks.json) + echo "$FRAMEWORKS" | jq -c '.[] | select(.starter) | select(.starter.measurements | map(.type) | contains(["spa"]))' | while read -r framework; do + PKG=$(echo "$framework" | jq -r '.starter.package') + echo "Building SPA for $PKG..." + (cd packages/$PKG && pnpm build) + echo "Running SPA benchmark for $PKG..." + pnpm --filter @framework-tracker/stats-generator run:spa $PKG + done + + echo "" + echo "=== Running MPA benchmarks ===" + FRAMEWORKS=$(cat .github/frameworks.json) + echo "$FRAMEWORKS" | jq -c '.[] | select(.app) | select(.app.measurements | map(.type) | contains(["mpa"]))' | while read -r framework; do + PKG=$(echo "$framework" | jq -r '.app.package') + echo "Building MPA for $PKG..." + (cd packages/$PKG && pnpm build) + echo "Running MPA benchmark for $PKG..." + pnpm --filter @framework-tracker/stats-generator run:mpa $PKG + done + echo "" echo "=== Validating all outputs ===" pnpm --filter @framework-tracker/stats-generator validate diff --git a/packages/app-astro/ci-stats.json b/packages/app-astro/ci-stats.json index 72b1901c..957eb690 100644 --- a/packages/app-astro/ci-stats.json +++ b/packages/app-astro/ci-stats.json @@ -1,10 +1,14 @@ { - "timingMeasuredAt": "2026-03-08T00:31:57.171Z", - "runner": "ubuntu-latest", + "timingMeasuredAt": "2026-04-21T05:06:35.326Z", + "runner": "local", "frameworkVersion": "5.16.15", "ssrOpsPerSec": 366, "ssrAvgLatencyMs": 2.735, "ssrSamples": 3656, "ssrBodySizeKb": 99.86, - "ssrDuplicationFactor": 1 + "ssrDuplicationFactor": 1, + "mpaFirstPaintMs": 90.33, + "mpaFCPMs": 90.39, + "mpaINPMs": 22.56, + "mpaRuns": 3 } diff --git a/packages/app-next-js/ci-stats.json b/packages/app-next-js/ci-stats.json index 2e66d033..fb124646 100644 --- a/packages/app-next-js/ci-stats.json +++ b/packages/app-next-js/ci-stats.json @@ -1,5 +1,5 @@ { - "timingMeasuredAt": "2026-04-05T04:53:54.987Z", + "timingMeasuredAt": "2026-04-21T05:07:39.298Z", "runner": "local", "frameworkVersion": "16.1.1", "ssrOpsPerSec": 129, @@ -10,5 +10,9 @@ "spaFirstPaintMs": 371, "spaFCPMs": 370.94, "spaINPMs": 23.38, - "spaRuns": 1 + "spaRuns": 1, + "mpaFirstPaintMs": 127.33, + "mpaFCPMs": 127.19, + "mpaINPMs": 20.21, + "mpaRuns": 3 } diff --git a/packages/app-nuxt/ci-stats.json b/packages/app-nuxt/ci-stats.json index 4985917c..dd6d5430 100644 --- a/packages/app-nuxt/ci-stats.json +++ b/packages/app-nuxt/ci-stats.json @@ -1,10 +1,18 @@ { - "timingMeasuredAt": "2026-03-08T00:31:57.171Z", - "runner": "ubuntu-latest", + "timingMeasuredAt": "2026-04-21T05:26:28.141Z", + "runner": "local", "frameworkVersion": "4.2.2", "ssrOpsPerSec": 248, "ssrAvgLatencyMs": 4.037, "ssrSamples": 2478, "ssrBodySizeKb": 201.18, - "ssrDuplicationFactor": 2 + "ssrDuplicationFactor": 2, + "mpaFirstPaintMs": 89.67, + "mpaFCPMs": 89.6, + "mpaINPMs": 24.05, + "mpaRuns": 3, + "spaFirstPaintMs": 111.67, + "spaFCPMs": 111.7, + "spaINPMs": 22.65, + "spaRuns": 3 } diff --git a/packages/app-react-router/ci-stats.json b/packages/app-react-router/ci-stats.json index 518332ae..5b208450 100644 --- a/packages/app-react-router/ci-stats.json +++ b/packages/app-react-router/ci-stats.json @@ -1,10 +1,18 @@ { - "timingMeasuredAt": "2026-03-08T00:31:57.171Z", - "runner": "ubuntu-latest", + "timingMeasuredAt": "2026-04-21T05:29:46.068Z", + "runner": "local", "frameworkVersion": "7.10.1", "ssrOpsPerSec": 64, "ssrAvgLatencyMs": 15.528, "ssrSamples": 644, "ssrBodySizeKb": 211.14, - "ssrDuplicationFactor": 2 + "ssrDuplicationFactor": 2, + "mpaFirstPaintMs": 167.33, + "mpaFCPMs": 167.24, + "mpaINPMs": 24.37, + "mpaRuns": 3, + "spaFirstPaintMs": 121, + "spaFCPMs": 121.16, + "spaINPMs": 22.24, + "spaRuns": 3 } diff --git a/packages/app-solid-start/ci-stats.json b/packages/app-solid-start/ci-stats.json index 0ccb0f04..dd53a411 100644 --- a/packages/app-solid-start/ci-stats.json +++ b/packages/app-solid-start/ci-stats.json @@ -1,10 +1,18 @@ { - "timingMeasuredAt": "2026-03-08T00:31:57.171Z", - "runner": "ubuntu-latest", + "timingMeasuredAt": "2026-04-21T05:30:50.803Z", + "runner": "local", "frameworkVersion": "1.2.1", "ssrOpsPerSec": 234, "ssrAvgLatencyMs": 4.275, "ssrSamples": 2340, "ssrBodySizeKb": 225.49, - "ssrDuplicationFactor": 2 + "ssrDuplicationFactor": 2, + "mpaFirstPaintMs": 106.33, + "mpaFCPMs": 106.31, + "mpaINPMs": 23.79, + "mpaRuns": 3, + "spaFirstPaintMs": 114, + "spaFCPMs": 114.32, + "spaINPMs": 21.33, + "spaRuns": 3 } diff --git a/packages/app-sveltekit/ci-stats.json b/packages/app-sveltekit/ci-stats.json index e53675ed..1474aa3f 100644 --- a/packages/app-sveltekit/ci-stats.json +++ b/packages/app-sveltekit/ci-stats.json @@ -1,5 +1,5 @@ { - "timingMeasuredAt": "2026-04-05T04:55:06.141Z", + "timingMeasuredAt": "2026-04-21T05:11:57.395Z", "runner": "local", "frameworkVersion": "2.49.4", "ssrOpsPerSec": 259, @@ -10,5 +10,9 @@ "spaFirstPaintMs": 93, "spaFCPMs": 93.14, "spaINPMs": 20.37, - "spaRuns": 1 + "spaRuns": 1, + "mpaFirstPaintMs": 109, + "mpaFCPMs": 108.78, + "mpaINPMs": 21.66, + "mpaRuns": 3 } diff --git a/packages/app-tanstack-start-react/ci-stats.json b/packages/app-tanstack-start-react/ci-stats.json index 873033bb..6f24710f 100644 --- a/packages/app-tanstack-start-react/ci-stats.json +++ b/packages/app-tanstack-start-react/ci-stats.json @@ -1,10 +1,18 @@ { - "timingMeasuredAt": "2026-03-08T00:31:57.171Z", - "runner": "ubuntu-latest", + "timingMeasuredAt": "2026-04-21T05:31:57.912Z", + "runner": "local", "frameworkVersion": "1.145.3", "ssrOpsPerSec": 185, "ssrAvgLatencyMs": 5.395, "ssrSamples": 1854, "ssrBodySizeKb": 193.53, - "ssrDuplicationFactor": 2 + "ssrDuplicationFactor": 2, + "mpaFirstPaintMs": 109.33, + "mpaFCPMs": 109.36, + "mpaINPMs": 23.98, + "mpaRuns": 3, + "spaFirstPaintMs": 693, + "spaFCPMs": 693.07, + "spaINPMs": 104.6, + "spaRuns": 3 } diff --git a/packages/stats-generator/package.json b/packages/stats-generator/package.json index fb22d2ee..e357bb2d 100644 --- a/packages/stats-generator/package.json +++ b/packages/stats-generator/package.json @@ -7,6 +7,7 @@ "collect": "node src/collect-stats.ts", "run:ssr": "node src/run-ssr-benchmark.ts", "run:spa": "node src/run-spa-benchmark.ts", + "run:mpa": "node src/run-mpa-benchmark.ts", "run:install": "node src/run-install-benchmark.ts", "run:build": "node src/run-build-benchmark.ts", "run:corejs": "node src/run-corejs-scan.ts", diff --git a/packages/stats-generator/src/mpa/index.ts b/packages/stats-generator/src/mpa/index.ts new file mode 100644 index 00000000..f00e7ee5 --- /dev/null +++ b/packages/stats-generator/src/mpa/index.ts @@ -0,0 +1,148 @@ +import { spawn } from 'node:child_process' +import { join } from 'node:path' +import { fileURLToPath } from 'node:url' +import { packagesDir } from '../constants.ts' +import { runBenchmark } from './run-benchmark.ts' +import type { MPABenchmarkResult } from './types.ts' + +const MPA_PORT = 3002 + +interface MPAFrameworkConfig { + name: string + displayName: string + package: string + serveScript: string + serveArgs?: string[] +} + +const MPA_FRAMEWORKS: MPAFrameworkConfig[] = [ + { + name: 'astro-mpa', + displayName: 'Astro MPA', + package: 'app-astro', + serveScript: 'astro.ts', + }, + { + name: 'next-mpa', + displayName: 'Next.js MPA', + package: 'app-next-js', + serveScript: 'next.ts', + }, + { + name: 'nuxt-mpa', + displayName: 'Nuxt MPA', + package: 'app-nuxt', + serveScript: 'nitro.ts', + }, + { + name: 'react-router-mpa', + displayName: 'React Router MPA', + package: 'app-react-router', + serveScript: 'react-router.ts', + }, + { + name: 'solid-start-mpa', + displayName: 'SolidStart MPA', + package: 'app-solid-start', + serveScript: 'nitro.ts', + }, + { + name: 'sveltekit-mpa', + displayName: 'SvelteKit MPA', + package: 'app-sveltekit', + serveScript: 'sveltekit.ts', + }, + { + name: 'tanstack-start-mpa', + displayName: 'TanStack Start MPA', + package: 'app-tanstack-start-react', + serveScript: 'tanstack-start.ts', + }, +] + +async function waitForServer(url: string, timeoutMs = 30_000): Promise { + const deadline = Date.now() + timeoutMs + while (Date.now() < deadline) { + try { + const res = await fetch(url) + if (res.status === 200) return + } catch { + // not ready yet + } + await new Promise((r) => setTimeout(r, 500)) + } + throw new Error(`Server at ${url} did not become ready within ${timeoutMs}ms`) +} + +async function spawnServer(config: MPAFrameworkConfig): Promise<() => void> { + const appDir = join(packagesDir, config.package) + const scriptPath = fileURLToPath( + new URL(`../serve/${config.serveScript}`, import.meta.url), + ) + const scriptArgs = [scriptPath, appDir, ...(config.serveArgs ?? [])] + + const proc = spawn('node', scriptArgs, { + env: { ...process.env, PORT: String(MPA_PORT) }, + stdio: ['ignore', 'pipe', 'pipe'], + }) + + proc.stdout?.on('data', (chunk: Buffer) => + process.stdout.write(`[${config.package}] ${chunk}`), + ) + proc.stderr?.on('data', (chunk: Buffer) => + process.stderr.write(`[${config.package}] ${chunk}`), + ) + + let exited = false + proc.on('exit', (code) => { + exited = true + if (code != null && code !== 0) { + console.error(`[${config.package}] server exited with code ${code}`) + } + }) + + const exitPromise = new Promise((_, reject) => { + proc.on('exit', (code) => { + if (code != null && code !== 0) { + reject(new Error(`Server process exited with code ${code}`)) + } + }) + }) + + await Promise.race([ + waitForServer(`http://localhost:${MPA_PORT}/mpa`), + exitPromise, + ]) + + return () => { + if (!exited) proc.kill('SIGTERM') + } +} + +export async function runMPABenchmark( + packageName: string, + runs = 5, +): Promise { + const config = MPA_FRAMEWORKS.find((f) => f.package === packageName) + + if (!config) { + throw new Error( + `Unknown MPA package: ${packageName}. Available: ${MPA_FRAMEWORKS.map((f) => f.package).join(', ')}`, + ) + } + + console.info(`Starting server for ${config.displayName}...`) + const killServer = await spawnServer(config) + + try { + console.info(`Running MPA benchmark for ${config.displayName}...`) + return await runBenchmark( + `http://localhost:${MPA_PORT}`, + config.name, + config.displayName, + runs, + ) + } finally { + killServer() + } +} diff --git a/packages/stats-generator/src/mpa/run-benchmark.ts b/packages/stats-generator/src/mpa/run-benchmark.ts new file mode 100644 index 00000000..df751e93 --- /dev/null +++ b/packages/stats-generator/src/mpa/run-benchmark.ts @@ -0,0 +1,129 @@ +import { existsSync, readdirSync } from 'node:fs' +import puppeteer from 'puppeteer-core' +import { startFlow } from 'lighthouse' +import type { MPABenchmarkResult, MPARunResult } from './types.ts' + +const MPA_PATH = '/mpa' + +function findChromium(): string { + if (process.env.CHROME_PATH) return process.env.CHROME_PATH + + const browsersPath = process.env.PLAYWRIGHT_BROWSERS_PATH + if (browsersPath && existsSync(browsersPath)) { + const chromiumDir = readdirSync(browsersPath).find((e) => + e.startsWith('chromium-'), + ) + if (chromiumDir) { + const chromePath = `${browsersPath}/${chromiumDir}/chrome-linux/chrome` + if (existsSync(chromePath)) return chromePath + } + } + + const candidates = [ + '/usr/bin/google-chrome', + '/usr/bin/chromium-browser', + '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome', + ] + for (const candidate of candidates) { + if (existsSync(candidate)) return candidate + } + + throw new Error( + 'Could not find Chromium/Chrome. Set the CHROME_PATH env var.', + ) +} + +async function runOnce( + url: string, + chromiumPath: string, +): Promise { + const browser = await puppeteer.launch({ + executablePath: chromiumPath, + headless: true, + args: ['--no-sandbox', '--disable-setuid-sandbox'], + }) + + try { + const page = await browser.newPage() + + const flow = await startFlow(page, { + name: 'MPA benchmark', + flags: { + throttlingMethod: 'provided', + formFactor: 'desktop', + screenEmulation: { disabled: true }, + }, + }) + + // FP + FCP: navigate to /mpa, wait for the table to render + await flow.navigate(`${url}${MPA_PATH}`) + await page.waitForSelector('table tbody tr', { timeout: 15_000 }) + + // INP: click the first row's detail link + await flow.startTimespan() + await page.click('table tbody tr:first-child a') + await page.waitForSelector('#detail-id', { timeout: 15_000 }) + // Double rAF ensures the paint entry is recorded before the timespan ends. + await page.evaluate( + (): Promise => + new Promise((r) => { + // @ts-expect-error — callback runs in browser context, not Node.js + requestAnimationFrame(() => requestAnimationFrame(r)) + }), + ) + await flow.endTimespan() + + const flowResult = await flow.createFlowResult() + const navLhr = flowResult.steps[0].lhr + const timespanLhr = flowResult.steps[1].lhr + + const metricsItems = ( + navLhr.audits['metrics']?.details as { items?: Record[] } + )?.items?.[0] + const firstPaintMs = metricsItems?.observedFirstPaint ?? null + const fcpMs = navLhr.audits['first-contentful-paint']?.numericValue ?? null + const inpMs = + timespanLhr.audits['interaction-to-next-paint']?.numericValue ?? null + + await page.close() + return { firstPaintMs, fcpMs, inpMs } + } finally { + await browser.close() + } +} + +function avg(values: number[]): number { + if (values.length === 0) return 0 + return Number((values.reduce((a, b) => a + b, 0) / values.length).toFixed(2)) +} + +export async function runBenchmark( + url: string, + packageName: string, + displayName: string, + runs: number, +): Promise { + const chromiumPath = findChromium() + const results: MPARunResult[] = [] + + for (let i = 0; i < runs; i++) { + console.log(` Run ${i + 1}/${runs}...`) + results.push(await runOnce(url, chromiumPath)) + } + + const fp = results + .map((r) => r.firstPaintMs) + .filter((v): v is number => v !== null) + const fcp = results.map((r) => r.fcpMs).filter((v): v is number => v !== null) + const inp = results.map((r) => r.inpMs).filter((v): v is number => v !== null) + + return { + name: packageName, + displayName, + package: packageName, + mpaFirstPaintMs: avg(fp), + mpaFCPMs: avg(fcp), + mpaINPMs: avg(inp), + mpaRuns: results.length, + } +} diff --git a/packages/stats-generator/src/mpa/types.ts b/packages/stats-generator/src/mpa/types.ts new file mode 100644 index 00000000..3d04cb86 --- /dev/null +++ b/packages/stats-generator/src/mpa/types.ts @@ -0,0 +1,25 @@ +export interface MPARunResult { + firstPaintMs: number | null + fcpMs: number | null + inpMs: number | null +} + +export interface MPABenchmarkResult { + name: string + displayName: string + package: string + mpaFirstPaintMs: number + mpaFCPMs: number + mpaINPMs: number + mpaRuns: number +} + +export interface MPAStats { + timingMeasuredAt: string + runner: string + frameworkVersion?: string + mpaFirstPaintMs: number + mpaFCPMs: number + mpaINPMs: number + mpaRuns: number +} diff --git a/packages/stats-generator/src/run-mpa-benchmark.ts b/packages/stats-generator/src/run-mpa-benchmark.ts new file mode 100644 index 00000000..60de7d31 --- /dev/null +++ b/packages/stats-generator/src/run-mpa-benchmark.ts @@ -0,0 +1,78 @@ +import { readFile, writeFile } from 'node:fs/promises' +import { join } from 'node:path' +import { runMPABenchmark } from './mpa/index.ts' +import { packagesDir } from './constants.ts' +import { getFrameworkByPackage, readJsonFile } from './utils.ts' +import type { CIStats } from './types.ts' + +async function getFrameworkVersion( + packageName: string, + frameworkPackage: string, +): Promise { + try { + const pkgJsonPath = join( + packagesDir, + packageName, + 'node_modules', + frameworkPackage, + 'package.json', + ) + const pkgJson = JSON.parse(await readFile(pkgJsonPath, 'utf-8')) + return pkgJson.version + } catch { + console.warn( + `Could not read version for ${frameworkPackage} in ${packageName}`, + ) + return undefined + } +} + +async function main() { + const packageName = process.argv[2] + const runs = process.argv[3] ? parseInt(process.argv[3], 10) : 5 + + if (!packageName) { + console.error('Usage: run-mpa-benchmark [runs]') + console.error('Example: run-mpa-benchmark app-astro 5') + process.exit(1) + } + + console.info(`Running MPA benchmark for ${packageName} (${runs} runs)...\n`) + + const { framework } = await getFrameworkByPackage(packageName) + const frameworkVersion = await getFrameworkVersion( + packageName, + framework.frameworkPackage, + ) + + const result = await runMPABenchmark(packageName, runs) + const timestamp = new Date().toISOString() + const runner = process.env.RUNNER_LABEL || 'local' + + const existingStats = readJsonFile( + join(packagesDir, packageName, 'ci-stats.json'), + ) + + const ciStats: CIStats = { + ...existingStats, + timingMeasuredAt: timestamp, + runner, + frameworkVersion: frameworkVersion ?? existingStats?.frameworkVersion, + mpaFirstPaintMs: result.mpaFirstPaintMs, + mpaFCPMs: result.mpaFCPMs, + mpaINPMs: result.mpaINPMs, + mpaRuns: result.mpaRuns, + } + + const outputPath = join(packagesDir, packageName, 'ci-stats.json') + await writeFile(outputPath, JSON.stringify(ciStats, null, 2), 'utf-8') + + console.info( + `\n✓ Saved ${result.displayName} v${frameworkVersion ?? 'unknown'} (${packageName})`, + ) + console.info(` First Paint: ${result.mpaFirstPaintMs}ms`) + console.info(` FCP: ${result.mpaFCPMs}ms`) + console.info(` INP: ${result.mpaINPMs}ms`) +} + +main().catch(console.error) diff --git a/packages/stats-generator/src/save-ci-stats.ts b/packages/stats-generator/src/save-ci-stats.ts index f8acf24c..23073860 100644 --- a/packages/stats-generator/src/save-ci-stats.ts +++ b/packages/stats-generator/src/save-ci-stats.ts @@ -60,9 +60,7 @@ async function main() { } frameworkVersion = installStats.frameworkVersion } else { - console.info( - ` ⚠ No install stats artifact found at ${installStatsPath}`, - ) + console.warn(`No install stats artifact found at ${installStatsPath}`) } // Load build stats from artifact @@ -82,7 +80,7 @@ async function main() { buildOutputSize: buildStats.buildOutputSize, } } else { - console.info(` ⚠ No build stats artifact found at ${buildStatsPath}`) + console.warn(`No build stats artifact found at ${buildStatsPath}`) } // Load core-js stats from artifact @@ -100,8 +98,8 @@ async function main() { vendoredCoreJsUnnecessaryModules: coreJsStats.unnecessaryModules, } } else { - console.info( - ` ⚠ No core-js stats artifact found at ${coreJsArtifactPath}`, + console.warn( + ` No core-js stats artifact found at ${coreJsArtifactPath}`, ) } @@ -129,7 +127,7 @@ async function main() { e18eMessages: e18eStats.messages, } } else { - console.info(` ⚠ No e18e stats artifact found at ${e18eArtifactPath}`) + console.warn(`No e18e stats artifact found at ${e18eArtifactPath}`) } // Save to ci-stats.json @@ -189,7 +187,7 @@ async function main() { } frameworkVersion = ssrStats.frameworkVersion } else { - console.info(` ⚠ No SSR stats artifact found at ${ssrStatsPath}`) + console.warn(`No SSR stats artifact found at ${ssrStatsPath}`) } // Load SPA stats from artifact (run-spa-benchmark writes directly to ci-stats.json) @@ -210,9 +208,30 @@ async function main() { spaRuns: spaStats.spaRuns, } } else { - console.info( - ` ⚠ No SPA stats artifact found at ${spaStatsArtifactPath}`, - ) + console.warn(`No SPA stats artifact found at ${spaStatsArtifactPath}`) + } + + // Load MPA stats from artifact (run-mpa-benchmark writes directly to ci-stats.json) + const mpaStatsArtifactPath = join( + artifactsDir, + `mpa-stats-${name}`, + 'ci-stats.json', + ) + const mpaStats = readJsonFile(mpaStatsArtifactPath) + + if (mpaStats) { + console.info(` ✓ Found MPA stats artifact`) + stats = { + ...stats, + frameworkVersion: mpaStats.frameworkVersion, + mpaFirstPaintMs: mpaStats.mpaFirstPaintMs, + mpaFCPMs: mpaStats.mpaFCPMs, + mpaINPMs: mpaStats.mpaINPMs, + mpaRuns: mpaStats.mpaRuns, + } + frameworkVersion = mpaStats.frameworkVersion ?? frameworkVersion + } else { + console.warn(`No MPA stats artifact found at ${mpaStatsArtifactPath}`) } // Save to ci-stats.json diff --git a/packages/stats-generator/src/serve/react-router.ts b/packages/stats-generator/src/serve/react-router.ts new file mode 100644 index 00000000..3e9511c9 --- /dev/null +++ b/packages/stats-generator/src/serve/react-router.ts @@ -0,0 +1,61 @@ +import { createServer } from 'node:http' +import { createRequire } from 'node:module' +import { join } from 'node:path' +import { pathToFileURL } from 'node:url' +import { + getPort, + parseAppDir, + tryServeFile, + registerShutdown, +} from './common.ts' + +const appDir = parseAppDir() +const PORT = getPort() +const staticDir = join(appDir, 'build', 'client') +const buildPath = join(appDir, 'build', 'server', 'index.js') +const buildUrl = pathToFileURL(buildPath).href + +// Resolve react-router from the app's own node_modules, not the stats-generator's. +const appRequire = createRequire(join(appDir, 'package.json')) +const rrPath = appRequire.resolve('react-router') +const { createRequestHandler } = await import(pathToFileURL(rrPath).href) +const build = await import(buildUrl) +const handler = createRequestHandler(build, 'production') + +const server = createServer(async (req, res) => { + const { pathname } = new URL(req.url ?? '/', `http://localhost:${PORT}`) + + if (tryServeFile(staticDir, pathname, req, res)) return + + const headers = new Headers( + Object.fromEntries( + Object.entries(req.headers).map(([k, v]) => [ + k, + Array.isArray(v) ? v.join(', ') : (v ?? ''), + ]), + ), + ) + const hasBody = req.method !== 'GET' && req.method !== 'HEAD' + const webReq = new Request(`http://localhost:${PORT}${req.url ?? '/'}`, { + method: req.method, + headers, + body: hasBody ? req : null, + duplex: 'half', + }) + + const webRes: Response = await handler(webReq) + + res.writeHead(webRes.status, Object.fromEntries(webRes.headers.entries())) + + if (webRes.body) { + for await (const chunk of webRes.body) { + res.write(chunk) + } + } + + res.end() +}).listen(PORT, () => { + console.log(`Ready at http://localhost:${PORT}`) +}) + +registerShutdown(server) diff --git a/packages/stats-generator/src/types.ts b/packages/stats-generator/src/types.ts index b2081b96..49dc7e2e 100644 --- a/packages/stats-generator/src/types.ts +++ b/packages/stats-generator/src/types.ts @@ -5,6 +5,7 @@ export type MeasurementType = | 'dependencies' | 'ssr' | 'spa' + | 'mpa' export interface MeasurementConfig { type: MeasurementType @@ -51,6 +52,11 @@ export interface CIStats { spaFCPMs?: number spaINPMs?: number spaRuns?: number + // MPA stats (browser paint + interaction timings) + mpaFirstPaintMs?: number + mpaFCPMs?: number + mpaINPMs?: number + mpaRuns?: number // Core-js vendored polyfill stats vendoredCoreJsSize?: number vendoredCoreJsUnnecessaryModules?: string[]