Replace ReactFlow+ELK diagrams with Next.js+Graphviz and add CI auto-update

.github/workflows/update_pipeline_diagrams.yaml

@@ -0,0 +1,49 @@
+name: Update Pipeline Diagrams
+
+on:
+  push:
+    branches: [main]
+
+permissions:
+  contents: write
+
+jobs:
+  update-diagrams:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '24'
+
+      - name: Install Claude Code CLI
+        run: npm install -g @anthropic-ai/claude-code
+
+      - name: Install diagram dependencies
+        working-directory: docs/pipeline-diagrams
+        run: npm ci
+
+      - name: Run diagram update pipeline
+        working-directory: docs/pipeline-diagrams
+        run: node scripts/update-diagrams.mjs
+        env:
+          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+          DIFF_BASE: ${{ github.event.before }}
+          DIFF_HEAD: ${{ github.event.after }}
+
+      - name: Commit if changed
+        run: |
+          if git diff --quiet docs/pipeline-diagrams/; then
+            echo "No diagram changes detected."
+            exit 0
+          fi
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+          git add docs/pipeline-diagrams/public/dots/ docs/pipeline-diagrams/manifest.json
+          git commit -m "Auto-update pipeline diagrams"
+          git push

Makefile

@@ -1,4 +1,4 @@
-.PHONY: all format test install download upload docker documentation data validate-data calibrate calibrate-build publish-local-area upload-calibration upload-dataset upload-database push-to-modal build-data-modal build-matrices calibrate-modal calibrate-modal-national calibrate-both stage-h5s stage-national-h5 stage-all-h5s pipeline validate-staging validate-staging-full upload-validation check-staging check-sanity clean build paper clean-paper presentations database database-refresh promote-database promote-dataset promote build-h5s validate-local
+.PHONY: all format test install download upload docker documentation data validate-data calibrate calibrate-build publish-local-area upload-calibration upload-dataset upload-database push-to-modal build-data-modal build-matrices calibrate-modal calibrate-modal-national calibrate-both stage-h5s stage-national-h5 stage-all-h5s pipeline validate-staging validate-staging-full upload-validation check-staging check-sanity clean build paper clean-paper presentations database database-refresh promote-database promote-dataset promote build-h5s validate-local diagrams
 
 GPU ?= T4
 EPOCHS ?= 1000
@@ -288,3 +288,6 @@ presentations/nta_2024_11/nta_2024_slides.pdf: presentations/nta_2024_11/main.te
 	cd presentations/nta_2024_11 && \
 		pdflatex -jobname=nta_2024_slides main && \
 		pdflatex -jobname=nta_2024_slides main
+
+diagrams:
+	cd docs/pipeline-diagrams && npm run dev

docs/pipeline-diagrams/.gitignore

@@ -0,0 +1,15 @@
+# Dependencies
+node_modules
+
+# Next.js
+.next
+out
+
+# Logs
+*.log
+
+# Editor
+.DS_Store
+.idea
+.vscode
+*.sw?

docs/pipeline-diagrams/CLAUDE.md

@@ -0,0 +1,71 @@
+# Pipeline Diagram DOT Style Guide
+
+When editing or creating Graphviz DOT files in `public/dots/`, follow these conventions exactly.
+
+## Graph Defaults
+
+```dot
+digraph "Title" {
+  graph [rankdir=TB fontname="Helvetica" bgcolor="transparent" label="" pad="0.5"]
+  node [shape=box fontname="Helvetica" fontsize=11 penwidth=2 margin="0.15,0.1"]
+  edge [fontname="Helvetica" fontsize=9]
+}
+```
+
+## Node Types
+
+Each node type has a specific color and style. Always use `style="filled,rounded"` unless noted.
+
+| Type | Fill | Border | Style | Use for |
+|------|------|--------|-------|---------|
+| Input | `fillcolor="#dbeafe" color="#3b82f6"` | blue | filled,rounded | Data files consumed by this stage |
+| Output | `fillcolor="#dcfce7" color="#22c55e"` | green | filled,rounded | Data files produced by this stage |
+| Process | `fillcolor="#ffedd5" color="#f97316"` | orange | filled,rounded | Computation/transformation steps |
+| Utility | `fillcolor="#f3e8ff" color="#a855f7"` | purple | filled,rounded | Shared helper functions/modules |
+| External | `fillcolor="#fef9c3" color="#eab308"` | yellow | filled,rounded | External services (HuggingFace, Modal, APIs) |
+| US-Specific | `fillcolor="#fce7f3" color="#ec4899"` | pink | filled,rounded | Steps unique to the US pipeline |
+| UK-Specific | `fillcolor="#ccfbf1" color="#14b8a6"` | teal | filled,rounded | Steps unique to the UK pipeline |
+| Missing | `fillcolor="#fee2e2" color="#ef4444"` | red | filled,rounded,dashed | Planned but not yet implemented |
+| Absent | `fillcolor="#f3f4f6" color="#d1d5db"` | gray | filled,rounded,dashed | Exists in the other country's pipeline but not this one |
+
+## Edge Types
+
+| Type | Attributes | Use for |
+|------|-----------|---------|
+| Data flow | `color="#334155" penwidth=2 style="solid"` | Primary data movement between steps |
+| Produces artifact | `color="#16a34a" penwidth=2 style="solid"` | Step producing an output file |
+| Uses utility | `color="#7c3aed" penwidth=1.5 style="dashed"` | Step calling a shared utility |
+| External source | `color="#b45309" penwidth=1.5 style="dotted"` | Data coming from external service |
+| Runs on infra | `color="#dc2626" penwidth=1.5 style="dashed"` | Infrastructure dependency (Modal, GPU) |
+| Informational | `color="#9ca3af" penwidth=1 style="dotted"` | Contextual/optional relationship |
+
+Edge labels use `fontcolor` matching the edge `color`:
+```dot
+  a -> b [color="#334155" penwidth=2 style="solid" label="description" fontcolor="#334155"]
+```
+
+## Node Label Format
+
+Use HTML-like labels with a `<TABLE>` structure:
+
+```dot
+  node_id [label=<<TABLE BORDER="0" CELLBORDER="0" CELLSPACING="0" CELLPADDING="2">
+    <TR><TD><B>Node Title</B></TD></TR>
+    <TR><TD><FONT POINT-SIZE="9" COLOR="#666666"><I>Subtitle or function name</I></FONT></TD></TR>
+    <TR><TD><FONT POINT-SIZE="8" COLOR="#555555">Detail line 1<BR/>Detail line 2</FONT></TD></TR>
+  </TABLE>> fillcolor="#dbeafe" color="#3b82f6" style="filled,rounded"]
+```
+
+- **First row**: Bold title — the node's name
+- **Second row**: Italic subtitle — function name, file path, or brief description (point size 9, color #666666)
+- **Third row** (optional): Details — implementation notes, data formats (point size 8, color #555555). Use `<BR/>` for line breaks.
+
+## Node ID Conventions
+
+- Use snake_case: `in_cps`, `qrf_pass1`, `out_ext`
+- Prefix inputs with `in_`, outputs with `out_`, utilities with `util_`, missing items with `miss_`, absent items with `absent_`
+- Use descriptive but concise IDs
+
+## Manifest
+
+The file `manifest.json` in this directory is the source of truth for which stages exist, their titles, and which Python source files map to each stage. The Sidebar and page components derive their data from this manifest.

docs/pipeline-diagrams/README.md

@@ -0,0 +1,16 @@
+# React + Vite
+
+This template provides a minimal setup to get React working in Vite with HMR and some ESLint rules.
+
+Currently, two official plugins are available:
+
+- [@vitejs/plugin-react](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react) uses [Oxc](https://oxc.rs)
+- [@vitejs/plugin-react-swc](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react-swc) uses [SWC](https://swc.rs/)
+
+## React Compiler
+
+The React Compiler is not enabled on this template because of its impact on dev & build performances. To add it, see [this documentation](https://react.dev/learn/react-compiler/installation).
+
+## Expanding the ESLint configuration
+
+If you are developing a production application, we recommend using TypeScript with type-aware lint rules enabled. Check out the [TS template](https://github.com/vitejs/vite/tree/main/packages/create-vite/template-react-ts) for information on how to integrate TypeScript and [`typescript-eslint`](https://typescript-eslint.io) in your project.

docs/pipeline-diagrams/app/[country]/page.jsx

@@ -0,0 +1,22 @@
+"use client";
+
+import { use } from "react";
+import Diagram from "../../components/Diagram";
+import manifest from "../../manifest.json";
+
+const overviewTitles = Object.fromEntries(
+  Object.entries(manifest.countries).map(([country, data]) => [
+    country,
+    data.overviewTitle,
+  ])
+);
+
+export default function CountryOverview({ params }) {
+  const { country } = use(params);
+  return (
+    <Diagram
+      dotPath={`/dots/${country}/overview.dot`}
+      title={overviewTitles[country] || "Overview"}
+    />
+  );
+}

docs/pipeline-diagrams/app/[country]/stage/[stageId]/page.jsx

@@ -0,0 +1,24 @@
+"use client";
+
+import { use } from "react";
+import Diagram from "../../../../components/Diagram";
+import manifest from "../../../../manifest.json";
+
+const titlesByCountry = Object.fromEntries(
+  Object.entries(manifest.countries).map(([country, data]) => [
+    country,
+    Object.fromEntries(data.stages.map((s) => [s.id, s.title])),
+  ])
+);
+
+export default function StagePage({ params }) {
+  const { country, stageId } = use(params);
+  const titles = titlesByCountry[country] || {};
+  const title = titles[stageId] || `Stage ${stageId}`;
+  return (
+    <Diagram
+      dotPath={`/dots/${country}/stage_${stageId}.dot`}
+      title={title}
+    />
+  );
+}

docs/pipeline-diagrams/app/globals.css

@@ -0,0 +1 @@
+@import "tailwindcss";

docs/pipeline-diagrams/app/layout.jsx

@@ -0,0 +1,17 @@
+import "./globals.css";
+import Sidebar from "../components/Sidebar";
+
+export const metadata = {
+  title: "PolicyEngine Data Pipelines",
+};
+
+export default function RootLayout({ children }) {
+  return (
+    <html lang="en">
+      <body className="flex h-screen overflow-hidden">
+        <Sidebar />
+        <main className="flex-1 overflow-auto bg-white">{children}</main>
+      </body>
+    </html>
+  );
+}

docs/pipeline-diagrams/app/page.jsx

@@ -0,0 +1,5 @@
+import { redirect } from "next/navigation";
+
+export default function Home() {
+  redirect("/us");
+}