Draft - Build and publish free-threaded Python wheels (cp313t/cp314t) across CI

.github/workflows/build.yml

@@ -24,11 +24,11 @@ on:
   workflow_call:
     inputs:
       build_mode:
-        description: 'Build mode: debug or release'
+        description: "Build mode: debug or release"
         required: true
         type: string
       run_wheels:
-        description: 'Whether to build distribution wheels'
+        description: "Whether to build distribution wheels"
         required: false
         type: boolean
         default: false
@@ -249,14 +249,63 @@ jobs:
     needs: [generate-license, lint-rust, lint-python]
     name: macOS arm64 & Windows
     runs-on: ${{ matrix.os }}
+    continue-on-error: ${{ matrix.experimental || false }}
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.10"]
-        os: [macos-latest, windows-latest]
+        include:
+          - os: macos-latest
+            python-version: "3.10"
+            python-tag: cp310
+          - os: macos-latest
+            python-version: "3.11"
+            python-tag: cp311
+          - os: macos-latest
+            python-version: "3.12"
+            python-tag: cp312
+          - os: macos-latest
+            python-version: "3.13"
+            python-tag: cp313
+          - os: macos-latest
+            python-version: "3.14"
+            python-tag: cp314
+          - os: macos-latest
+            python-version: "3.13t"
+            python-tag: cp313t
+          - os: macos-latest
+            python-version: "3.14t"
+            python-tag: cp314t
+            experimental: true
+          - os: windows-latest
+            python-version: "3.10"
+            python-tag: cp310
+          - os: windows-latest
+            python-version: "3.11"
+            python-tag: cp311
+          - os: windows-latest
+            python-version: "3.12"
+            python-tag: cp312
+          - os: windows-latest
+            python-version: "3.13"
+            python-tag: cp313
+          - os: windows-latest
+            python-version: "3.14"
+            python-tag: cp314
+          - os: windows-latest
+            python-version: "3.13t"
+            python-tag: cp313t
+          - os: windows-latest
+            python-version: "3.14t"
+            python-tag: cp314t
+            experimental: true
     steps:
       - uses: actions/checkout@v6
 
+      - uses: actions/setup-python@v6
+        id: setup-python
+        with:
+          python-version: ${{ matrix.python-version }}
+
       - uses: dtolnay/rust-toolchain@stable
 
       - run: rm LICENSE.txt
@@ -282,6 +331,7 @@ jobs:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Install dependencies
+        if: matrix.python-version != '3.14t'
         run: uv sync --dev --no-install-package datafusion
 
       # Run clippy BEFORE maturin so we can avoid rebuilding. The features must match
@@ -293,11 +343,15 @@ jobs:
 
       - name: Build Python package (release mode)
         if: inputs.build_mode == 'release'
-        run: uv run --no-project maturin build --release --strip --features substrait
+        env:
+          MATURIN_FEATURES: ${{ endsWith(matrix.python-version, 't') && 'substrait' || 'substrait,py-limited-api' }}
+        run: uv run --no-project maturin build --release --strip --features "${MATURIN_FEATURES}" --interpreter "${{ steps.setup-python.outputs.python-path }}"
 
       - name: Build Python package (debug mode)
         if: inputs.build_mode != 'release'
-        run: uv run --no-project maturin build --features substrait
+        env:
+          MATURIN_FEATURES: ${{ endsWith(matrix.python-version, 't') && 'substrait' || 'substrait,py-limited-api' }}
+        run: uv run --no-project maturin build --features "${MATURIN_FEATURES}" --interpreter "${{ steps.setup-python.outputs.python-path }}"
 
       - name: List Windows wheels
         if: matrix.os == 'windows-latest'
@@ -314,7 +368,7 @@ jobs:
         uses: actions/upload-artifact@v6
         if: inputs.build_mode == 'release'
         with:
-          name: dist-${{ matrix.os  }}
+          name: dist-${{ matrix.os }}-${{ matrix.python-tag }}
           path: target/wheels/*
 
   # ============================================
@@ -324,13 +378,34 @@ jobs:
     if: inputs.build_mode == 'release'
     needs: [generate-license, lint-rust, lint-python]
     runs-on: macos-15-intel
+    continue-on-error: ${{ matrix.experimental || false }}
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.10"]
+        include:
+          - python-version: "3.10"
+            python-tag: cp310
+          - python-version: "3.11"
+            python-tag: cp311
+          - python-version: "3.12"
+            python-tag: cp312
+          - python-version: "3.13"
+            python-tag: cp313
+          - python-version: "3.14"
+            python-tag: cp314
+          - python-version: "3.13t"
+            python-tag: cp313t
+          - python-version: "3.14t"
+            python-tag: cp314t
+            experimental: true
     steps:
       - uses: actions/checkout@v6
 
+      - uses: actions/setup-python@v6
+        id: setup-python
+        with:
+          python-version: ${{ matrix.python-version }}
+
       - uses: dtolnay/rust-toolchain@stable
 
       - run: rm LICENSE.txt
@@ -356,19 +431,22 @@ jobs:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Install dependencies
+        if: matrix.python-version != '3.14t'
         run: uv sync --dev --no-install-package datafusion
 
       - name: Build (release mode)
+        env:
+          MATURIN_FEATURES: ${{ endsWith(matrix.python-version, 't') && 'substrait' || 'substrait,py-limited-api' }}
         run: |
-          uv run --no-project maturin build --release --strip --features substrait
+          uv run --no-project maturin build --release --strip --features "${MATURIN_FEATURES}" --interpreter "${{ steps.setup-python.outputs.python-path }}"
 
       - name: List Mac wheels
         run: find target/wheels/
 
       - name: Archive wheels
         uses: actions/upload-artifact@v6
         with:
-          name: dist-macos-aarch64
+          name: dist-macos-aarch64-${{ matrix.python-tag }}
           path: target/wheels/*
 
   # ============================================
@@ -407,9 +485,8 @@ jobs:
         shell: bash
 
   # ============================================
-  # Build - Source Distribution
+  # Build - Merge Artifacts
   # ============================================
-
   merge-build-artifacts:
     runs-on: ubuntu-latest
     name: Merge build artifacts
@@ -434,7 +511,7 @@ jobs:
   build-docs:
     name: Build docs
     runs-on: ubuntu-latest
-    needs: [build-manylinux-x86_64]  # Only need the Linux wheel for docs
+    needs: [build-manylinux-x86_64] # Only need the Linux wheel for docs
     # Only run docs on main branch pushes, tags, or PRs
     if: github.event_name == 'push' || github.event_name == 'pull_request'
     steps:
@@ -477,7 +554,7 @@ jobs:
       - name: Download pre-built Linux wheel
         uses: actions/download-artifact@v7
         with:
-          name: dist-manylinux-x86_64
+          name: dist-manylinux-x86_64-cp310-314
           path: wheels/
 
       # Install from the pre-built wheels

Cargo.toml

@@ -40,6 +40,7 @@ include = [
 default = ["mimalloc"]
 protoc = ["datafusion-substrait/protoc"]
 substrait = ["dep:datafusion-substrait"]
+py-limited-api = ["pyo3/abi3-py310"]
 
 [dependencies]
 tokio = { version = "1.49", features = [
@@ -48,11 +49,7 @@ tokio = { version = "1.49", features = [
   "rt-multi-thread",
   "sync",
 ] }
-pyo3 = { version = "0.26", features = [
-  "extension-module",
-  "abi3",
-  "abi3-py310",
-] }
+pyo3 = { version = "0.26", features = ["extension-module"] }
 pyo3-async-runtimes = { version = "0.26", features = ["tokio-runtime"] }
 pyo3-log = "0.13.3"
 arrow = { version = "57", features = ["pyarrow"] }

dev/release/README.md

@@ -130,6 +130,10 @@ datafusion-22.0.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
 datafusion-22.0.0-cp37-abi3-win_amd64.whl
 ```
 
+Note: PyO3's free-threaded CPython builds (3.13t/3.14t) use a distinct ABI and cannot use the limited API (`abi3`).
+The release workflow enables Cargo feature `py-limited-api` for GIL-enabled wheels and disables it for free-threaded
+builds, producing version-specific `cp313t`/`cp314t` wheels.
+
 Upload the wheels to testpypi.
 
 ```bash
@@ -237,6 +241,12 @@ cargo publish
 
 ### Publishing Python Artifacts to PyPi
 
+GitHub Actions groups wheel artifacts by platform and interpreter tag using the pattern `dist-<platform>-<python-tag>`.
+For example, standard manylinux wheels live under `dist-manylinux-x86_64-cp310-314` while the free-threaded builds
+use `dist-manylinux-x86_64-cp313t-314t`. macOS and Windows jobs publish one artifact per CPython version as well
+(`dist-macos-latest-cp311`, `dist-windows-latest-cp313t`, etc.). Download the exact tags you intend to push to PyPI,
+and remember that the docs workflow currently installs from the `cp310-314` manylinux artifact.
+
 Go to the Test PyPI page of Datafusion, and download
 [all published artifacts](https://test.pypi.org/project/datafusion/#files) under `dist-release/` directory. Then proceed
 uploading them using `twine`:

src/lib.rs

@@ -16,9 +16,9 @@
 // under the License.
 
 // Re-export Apache Arrow DataFusion dependencies
-pub use datafusion;
 pub use datafusion::{
-    common as datafusion_common, logical_expr as datafusion_expr, optimizer, sql as datafusion_sql,
+    self, common as datafusion_common, logical_expr as datafusion_expr, optimizer,
+    sql as datafusion_sql,
 };
 #[cfg(feature = "substrait")]
 pub use datafusion_substrait;