improve CI speed

luarss · luarss · commit bf32ddfe4eae · 2026-02-17T10:08:20.000Z
- use caching in `astral-sh/setup-uv`
- use caching in HF dataset
- parallelise init/init-dev targets in root
- general CI to split and parallelise lint/unit tests before docker build

Signed-off-by: Jack Luar &lt;jluar@precisioninno.com&gt;
diff --git a/.github/workflows/ci-secret.yaml b/.github/workflows/ci-secret.yaml
@@ -1,7 +1,7 @@
 name: ORAssistant Secret CI
 run-name: ${{ github.actor }} started CI
 
-on: 
+on:
   push:
     branches:
       - master
@@ -20,25 +20,42 @@ jobs:
   build-backend-docker:
     runs-on: self-hosted
     steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
     - name: Setup python
       uses: actions/setup-python@v5
       with:
-        python-version: '3.12'
+        python-version: '3.13'
     - name: Install uv
-      uses: astral-sh/setup-uv@v6   
-    - name: Checkout code
-      uses: actions/checkout@v4
+      uses: astral-sh/setup-uv@v6
+      with:
+        enable-cache: true
+        cache-dependency-glob: |
+          backend/uv.lock
+          frontend/uv.lock
+          evaluation/uv.lock
     - name: Setup prereqs
       run: |
         make init-dev
     - name: Run formatting checks
       run: |
         make check
+    - name: Cache HuggingFace dataset
+      uses: actions/cache@v4
+      with:
+        path: backend/data/source_list.json
+        key: hf-source-list-v1
+    - name: Download HuggingFace dataset
+      working-directory: backend
+      run: |
+        uv run huggingface-cli download \
+          --repo-type dataset \
+          The-OpenROAD-Project/ORAssistant_RAG_Dataset \
+          --include source_list.json \
+          --local-dir data/
     - name: Run unit tests
       working-directory: backend
       run: |
-        uv pip install huggingface_hub[cli]
-        huggingface-cli download --repo-type dataset The-OpenROAD-Project/ORAssistant_RAG_Dataset --include source_list.json --local-dir data/
         cp .env.test .env
         make test
     - name: Populate environment variables
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -1,7 +1,7 @@
 name: ORAssistant CI
 run-name: ${{ github.actor }} started CI
 
-on: 
+on:
   pull_request:
     paths:
       - 'backend/**'
@@ -15,36 +15,82 @@ defaults:
     shell: bash
 
 jobs:
-  build-backend-docker:
+  lint:
     runs-on: self-hosted
     steps:
-    - name: Setup python
-      uses: actions/setup-python@v5
-      with:
-        python-version: '3.12'
-
-    - name: Install uv
-      uses: astral-sh/setup-uv@v6   
-
-    - name: Checkout code
-      uses: actions/checkout@v4
-
-    - name: Setup prereqs
-      run: |
-        make init-dev
-
-    - name: Run formatting checks
-      run: |
-        make check
-
-    - name: Run unit tests
-      working-directory: backend
-      run: |
-        uv pip install huggingface_hub[cli]
-        huggingface-cli download --repo-type dataset The-OpenROAD-Project/ORAssistant_RAG_Dataset --include source_list.json --local-dir data/
-        cp .env.test .env
-        make test
-
-    - name: Build Docker images
-      run: |
-        docker compose build
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Setup python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.13'
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
+        with:
+          enable-cache: true
+          cache-dependency-glob: |
+            backend/uv.lock
+            frontend/uv.lock
+            evaluation/uv.lock
+
+      - name: Setup prereqs
+        run: make init-dev
+
+      - name: Run formatting checks
+        run: make check
+
+  test:
+    runs-on: self-hosted
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Setup python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.13'
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
+        with:
+          enable-cache: true
+          cache-dependency-glob: |
+            backend/uv.lock
+            frontend/uv.lock
+            evaluation/uv.lock
+
+      - name: Setup prereqs
+        run: make init-dev
+
+      - name: Cache HuggingFace dataset
+        uses: actions/cache@v4
+        with:
+          path: backend/data/source_list.json
+          key: hf-source-list-v1
+
+      - name: Download HuggingFace dataset
+        working-directory: backend
+        run: |
+          uv run huggingface-cli download \
+            --repo-type dataset \
+            The-OpenROAD-Project/ORAssistant_RAG_Dataset \
+            --include source_list.json \
+            --local-dir data/
+
+      - name: Run unit tests
+        working-directory: backend
+        run: |
+          cp .env.test .env
+          make test
+
+  docker-build:
+    runs-on: self-hosted
+    needs: [lint, test]
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Build Docker images
+        run: docker compose build
diff --git a/Makefile b/Makefile
@@ -3,21 +3,27 @@ GOOGLE_SECRET_JSON?=$(HOME)/secret.json
 
 .PHONY: init
 init:
-	@for folder in $(FOLDERS); do (cd $$folder && make init && cd ../); done
+	@pids=""; \
+	 for folder in $(FOLDERS); do (cd $$folder && make init) & pids="$$pids $$!"; done; \
+	 for pid in $$pids; do wait $$pid || exit 1; done
 
 .PHONY: init-dev
 init-dev:
-	@for folder in $(FOLDERS); do (cd $$folder && make init-dev && cd ../); done
+	@pids=""; \
+	 for folder in $(FOLDERS); do (cd $$folder && make init-dev) & pids="$$pids $$!"; done; \
+	 for pid in $$pids; do wait $$pid || exit 1; done
 
 .PHONY: format
 format:
-	@for folder in $(FOLDERS); do (cd $$folder && make format && cd ../); done
+	@pids=""; \
+	 for folder in $(FOLDERS); do (cd $$folder && make format) & pids="$$pids $$!"; done; \
+	 for pid in $$pids; do wait $$pid || exit 1; done
 
 .PHONY: check
 check:
-	@for folder in $(FOLDERS); do \
- 	   (cd $$folder && make check && cd ../) || exit 1; \
-		done
+	@pids=""; \
+	 for folder in $(FOLDERS); do (cd $$folder && make check) & pids="$$pids $$!"; done; \
+	 for pid in $$pids; do wait $$pid || exit 1; done
 	@. ./backend/.venv/bin/activate && \
 		pre-commit run --all-files
 
diff --git a/backend/Dockerfile b/backend/Dockerfile
@@ -2,6 +2,7 @@ FROM ghcr.io/astral-sh/uv:python3.13-bookworm-slim
 
 WORKDIR /ORAssistant-backend
 
+# Layer 1: system deps (rarely changes)
 RUN apt-get update && apt-get install -y \
     build-essential \
     curl \
@@ -15,18 +16,25 @@ RUN apt-get update && apt-get install -y \
     git lfs install && \
     rm -rf /var/lib/apt/lists/*
 
-RUN pip install uv
-
-COPY ./pyproject.toml /ORAssistant-backend/pyproject.toml
-COPY . .
-
-RUN uv venv .venv && uv sync --dev && uv run /ORAssistant-backend/src/post_install.py
-
+# Layer 2: HF dataset (independent of code — cached across code changes)
 RUN git clone https://huggingface.co/datasets/The-OpenROAD-Project/ORAssistant_RAG_Dataset && \
     mkdir -p data && \
     mv ORAssistant_RAG_Dataset/* data/ && \
     rm -rf ORAssistant_RAG_Dataset
 
+# Layer 3: Python dep manifests (changes when deps change, not when code changes)
+COPY ./pyproject.toml /ORAssistant-backend/pyproject.toml
+COPY ./uv.lock /ORAssistant-backend/uv.lock
+
+# Layer 4: Install Python deps (cached unless layer 3 changes)
+RUN uv venv .venv && uv sync --dev
+
+# Layer 5: source code (changes frequently; only busts layers below)
+COPY . .
+
+# Layer 6: lightweight post-install
+RUN uv run /ORAssistant-backend/src/post_install.py
+
 EXPOSE 8000
 
 CMD ["uv", "run", "uvicorn", "src.api.main:app", "--host", "0.0.0.0", "--port", "8000", "--reload"]
