Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 0 additions & 14 deletions .github/workflows/build_infra_images_cache.yml
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@ on:
- 'dev/spark-test-image/sparkr/Dockerfile'
- 'dev/spark-test-image/python-minimum/Dockerfile'
- 'dev/spark-test-image/python-ps-minimum/Dockerfile'
- 'dev/spark-test-image/python-310/Dockerfile'
- 'dev/spark-test-image/python-311/Dockerfile'
- 'dev/spark-test-image/python-312/Dockerfile'
- 'dev/spark-test-image/python-312-classic-only/Dockerfile'
Expand Down Expand Up @@ -139,19 +138,6 @@ jobs:
- name: Image digest (PySpark PS with old dependencies)
if: hashFiles('dev/spark-test-image/python-ps-minimum/Dockerfile') != ''
run: echo ${{ steps.docker_build_pyspark_python_ps_minimum.outputs.digest }}
- name: Build and push (PySpark with Python 3.10)
if: hashFiles('dev/spark-test-image/python-310/Dockerfile') != ''
id: docker_build_pyspark_python_310
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f
with:
context: ./dev/spark-test-image/python-310/
push: true
tags: ghcr.io/apache/spark/apache-spark-github-action-image-pyspark-python-310-cache:${{ github.ref_name }}-static
cache-from: type=registry,ref=ghcr.io/apache/spark/apache-spark-github-action-image-pyspark-python-310-cache:${{ github.ref_name }}
cache-to: type=registry,ref=ghcr.io/apache/spark/apache-spark-github-action-image-pyspark-python-310-cache:${{ github.ref_name }},mode=max
- name: Image digest (PySpark with Python 3.10)
if: hashFiles('dev/spark-test-image/python-310/Dockerfile') != ''
run: echo ${{ steps.docker_build_pyspark_python_310.outputs.digest }}
- name: Build and push (PySpark with Python 3.11)
if: hashFiles('dev/spark-test-image/python-311/Dockerfile') != ''
id: docker_build_pyspark_python_311
Expand Down
47 changes: 0 additions & 47 deletions .github/workflows/build_python_3.10.yml

This file was deleted.

2 changes: 1 addition & 1 deletion .github/workflows/build_python_minimum.yml
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ jobs:
envs: >-
{
"PYSPARK_IMAGE_TO_TEST": "python-minimum",
"PYTHON_TO_TEST": "python3.10"
"PYTHON_TO_TEST": "python3.11"
}
jobs: >-
{
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/build_python_ps_minimum.yml
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ jobs:
envs: >-
{
"PYSPARK_IMAGE_TO_TEST": "python-ps-minimum",
"PYTHON_TO_TEST": "python3.10"
"PYTHON_TO_TEST": "python3.11"
}
jobs: >-
{
Expand Down
1 change: 0 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,6 @@ This README file only contains basic setup instructions.
| | [![GitHub Actions Build](https://github.com/apache/spark/actions/workflows/build_maven_java21_macos26.yml/badge.svg)](https://github.com/apache/spark/actions/workflows/build_maven_java21_macos26.yml) |
| | [![GitHub Actions Build](https://github.com/apache/spark/actions/workflows/build_maven_java21_arm.yml/badge.svg)](https://github.com/apache/spark/actions/workflows/build_maven_java21_arm.yml) |
| | [![GitHub Actions Build](https://github.com/apache/spark/actions/workflows/build_coverage.yml/badge.svg)](https://github.com/apache/spark/actions/workflows/build_coverage.yml) |
| | [![GitHub Actions Build](https://github.com/apache/spark/actions/workflows/build_python_3.10.yml/badge.svg)](https://github.com/apache/spark/actions/workflows/build_python_3.10.yml) |
| | [![GitHub Actions Build](https://github.com/apache/spark/actions/workflows/build_python_3.11.yml/badge.svg)](https://github.com/apache/spark/actions/workflows/build_python_3.11.yml) |
| | [![GitHub Actions Build](https://github.com/apache/spark/actions/workflows/build_python_3.12_classic_only.yml/badge.svg)](https://github.com/apache/spark/actions/workflows/build_python_3.12_classic_only.yml) |
| | [![GitHub Actions Build](https://github.com/apache/spark/actions/workflows/build_python_3.12_arm.yml/badge.svg)](https://github.com/apache/spark/actions/workflows/build_python_3.12_arm.yml) |
Expand Down
2 changes: 1 addition & 1 deletion binder/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
# limitations under the License.
#

FROM python:3.10-slim
FROM python:3.11-slim
# install the notebook package
RUN pip install --no-cache notebook jupyterlab

Expand Down
18 changes: 9 additions & 9 deletions dev/infra/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ RUN apt-get update && apt-get install -y \
openjdk-17-jdk-headless \
pandoc \
pkg-config \
python3.10 \
python3.11 \
python3-psutil \
qpdf \
r-base \
Expand Down Expand Up @@ -94,14 +94,14 @@ ARG BASIC_PIP_PKGS="numpy pyarrow>=18.0.0 six==1.16.0 pandas==2.3.3 scipy plotly
# Python deps for Spark Connect
ARG CONNECT_PIP_PKGS="grpcio==1.76.0 grpcio-status==1.76.0 protobuf==6.33.5 googleapis-common-protos==1.71.0 graphviz==0.20.3"

# Install Python 3.10 packages
RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python3.10
RUN python3.10 -m pip install --ignore-installed 'blinker>=1.6.2' # mlflow needs this
RUN python3.10 -m pip install --ignore-installed 'six==1.16.0' # Avoid `python3-six` installation
RUN python3.10 -m pip install $BASIC_PIP_PKGS unittest-xml-reporting $CONNECT_PIP_PKGS && \
python3.10 -m pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu && \
python3.10 -m pip install deepspeed torcheval && \
python3.10 -m pip cache purge
# Install Python 3.11 packages
RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python3.11
RUN python3.11 -m pip install --ignore-installed 'blinker>=1.6.2' # mlflow needs this
RUN python3.11 -m pip install --ignore-installed 'six==1.16.0' # Avoid `python3-six` installation
RUN python3.11 -m pip install $BASIC_PIP_PKGS unittest-xml-reporting $CONNECT_PIP_PKGS && \
python3.11 -m pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu && \
python3.11 -m pip install deepspeed torcheval && \
python3.11 -m pip cache purge

# Install Python 3.9
RUN add-apt-repository ppa:deadsnakes/ppa
Expand Down
74 changes: 0 additions & 74 deletions dev/spark-test-image/python-310/Dockerfile

This file was deleted.

12 changes: 6 additions & 6 deletions dev/spark-test-image/python-minimum/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -49,22 +49,22 @@ RUN apt-get update && apt-get install -y \
software-properties-common \
zlib1g-dev

# Install Python 3.10
# Install Python 3.11
RUN add-apt-repository ppa:deadsnakes/ppa
RUN apt-get update && apt-get install -y \
python3.10 \
python3.10-venv \
python3.11 \
python3.11-venv \
&& apt-get autoremove --purge -y \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*

# Setup virtual environment
ENV VIRTUAL_ENV=/opt/spark-venv
RUN python3.10 -m venv $VIRTUAL_ENV
RUN python3.11 -m venv $VIRTUAL_ENV
ENV PATH="$VIRTUAL_ENV/bin:$PATH"

ARG BASIC_PIP_PKGS="numpy==1.22.4 pyarrow==18.0.0 pandas==2.2.0 six==1.16.0 scipy scikit-learn coverage unittest-xml-reporting psutil"
ARG CONNECT_PIP_PKGS="grpcio==1.76.0 grpcio-status==1.76.0 googleapis-common-protos==1.71.0 zstandard==0.25.0 graphviz==0.20 protobuf==6.33.5"

RUN python3.10 -m pip install --force $BASIC_PIP_PKGS $CONNECT_PIP_PKGS && \
python3.10 -m pip cache purge
RUN python3.11 -m pip install --force $BASIC_PIP_PKGS $CONNECT_PIP_PKGS && \
python3.11 -m pip cache purge
12 changes: 6 additions & 6 deletions dev/spark-test-image/python-ps-minimum/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -49,22 +49,22 @@ RUN apt-get update && apt-get install -y \
software-properties-common \
zlib1g-dev

# Install Python 3.10
# Install Python 3.11
RUN add-apt-repository ppa:deadsnakes/ppa
RUN apt-get update && apt-get install -y \
python3.10 \
python3.10-venv \
python3.11 \
python3.11-venv \
&& apt-get autoremove --purge -y \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*

# Setup virtual environment
ENV VIRTUAL_ENV=/opt/spark-venv
RUN python3.10 -m venv $VIRTUAL_ENV
RUN python3.11 -m venv $VIRTUAL_ENV
ENV PATH="$VIRTUAL_ENV/bin:$PATH"

ARG BASIC_PIP_PKGS="pyarrow==18.0.0 pandas==2.2.0 six==1.16.0 numpy scipy coverage unittest-xml-reporting psutil"
ARG CONNECT_PIP_PKGS="grpcio==1.76.0 grpcio-status==1.76.0 googleapis-common-protos==1.71.0 zstandard==0.25.0 graphviz==0.20 protobuf==6.33.5"

RUN python3.10 -m pip install --force $BASIC_PIP_PKGS $CONNECT_PIP_PKGS && \
python3.10 -m pip cache purge
RUN python3.11 -m pip install --force $BASIC_PIP_PKGS $CONNECT_PIP_PKGS && \
python3.11 -m pip cache purge
2 changes: 1 addition & 1 deletion docs/index.md
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ source, visit [Building Spark](building-spark.html).

Spark runs on both Windows and UNIX-like systems (e.g. Linux, Mac OS), and it should run on any platform that runs a supported version of Java. This should include JVMs on x86_64 and ARM64. It's easy to run locally on one machine --- all you need is to have `java` installed on your system `PATH`, or the `JAVA_HOME` environment variable pointing to a Java installation.

Spark runs on Java 17/21/25, Scala 2.13, Python 3.10+, and R 3.5+ (Deprecated).
Spark runs on Java 17/21/25, Scala 2.13, Python 3.11+, and R 3.5+ (Deprecated).
Java 25 prior to version 25.0.3 support is deprecated as of Spark 4.2.0.
When using the Scala API, it is necessary for applications to use the same version of Scala that Spark was compiled for. Since Spark 4.0.0, it's Scala 2.13.

Expand Down
2 changes: 1 addition & 1 deletion docs/rdd-programming-guide.md
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ along with if you launch Spark's interactive shell -- either `bin/spark-shell` f

<div data-lang="python" markdown="1">

Spark {{site.SPARK_VERSION}} works with Python 3.10+. It can use the standard CPython interpreter,
Spark {{site.SPARK_VERSION}} works with Python 3.11+. It can use the standard CPython interpreter,
so C libraries like NumPy can be used.

Spark applications in Python can either be run with the `bin/spark-submit` script which includes Spark at runtime, or by including it in your setup.py as:
Expand Down
8 changes: 4 additions & 4 deletions python/docs/source/development/contributing.rst
Original file line number Diff line number Diff line change
Expand Up @@ -129,8 +129,8 @@ If you are using Conda, the development environment can be set as follows.

.. code-block:: bash

# Python 3.10+ is required
conda create --name pyspark-dev-env python=3.10
# Python 3.11+ is required
conda create --name pyspark-dev-env python=3.11
conda activate pyspark-dev-env
pip install --upgrade -r dev/requirements.txt

Expand All @@ -149,15 +149,15 @@ You can use Python's built-in ``venv`` module to create an isolated environment:

.. code-block:: bash

# Python 3.10+ is required
# Python 3.11+ is required
python3 -m venv .venv
source .venv/bin/activate
pip install --upgrade -r dev/requirements.txt

pip
~~~

With Python 3.10+, pip can be used as below to install and set up the development environment.
With Python 3.11+, pip can be used as below to install and set up the development environment.

.. code-block:: bash

Expand Down
4 changes: 2 additions & 2 deletions python/docs/source/getting_started/install.rst
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ and building from the source.
Python Versions Supported
-------------------------

Python 3.10 and above.
Python 3.11 and above.


Using PyPI
Expand Down Expand Up @@ -143,7 +143,7 @@ the same session as pyspark (you can install in several steps too).

.. code-block:: bash

conda install -c conda-forge pyspark # can also add "python=3.10 some_package [etc.]" here
conda install -c conda-forge pyspark # can also add "python=3.11 some_package [etc.]" here

Note that `PySpark for conda <https://anaconda.org/conda-forge/pyspark>`_ is maintained
separately by the community; while new versions generally get packaged quickly, the
Expand Down
2 changes: 1 addition & 1 deletion python/docs/source/tutorial/pandas_on_spark/typehints.rst
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ it as a Spark schema. As an example, you can specify the return type hint as bel
Notice that the function ``pandas_div`` actually takes and outputs a pandas DataFrame instead of
pandas-on-Spark :class:`DataFrame`. So, technically the correct types should be of pandas.

With Python 3.10+, you can specify the type hints by using pandas instances as follows:
With Python 3.11+, you can specify the type hints by using pandas instances as follows:

.. code-block:: python

Expand Down
3 changes: 1 addition & 2 deletions python/packaging/classic/setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -382,10 +382,9 @@ def run(self):
"pyyaml>=%s" % _minimum_pyyaml_version,
],
},
python_requires=">=3.10",
python_requires=">=3.11",
classifiers=[
"Development Status :: 5 - Production/Stable",
"Programming Language :: Python :: 3.10",
"Programming Language :: Python :: 3.11",
"Programming Language :: Python :: 3.12",
"Programming Language :: Python :: 3.13",
Expand Down
3 changes: 1 addition & 2 deletions python/packaging/client/setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -217,10 +217,9 @@
"numpy>=%s" % _minimum_numpy_version,
"pyyaml>=%s" % _minimum_pyyaml_version,
],
python_requires=">=3.10",
python_requires=">=3.11",
classifiers=[
"Development Status :: 5 - Production/Stable",
"Programming Language :: Python :: 3.10",
"Programming Language :: Python :: 3.11",
"Programming Language :: Python :: 3.12",
"Programming Language :: Python :: 3.13",
Expand Down
3 changes: 1 addition & 2 deletions python/packaging/connect/setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -127,10 +127,9 @@
"numpy>=%s" % _minimum_numpy_version,
"pyyaml>=%s" % _minimum_pyyaml_version,
],
python_requires=">=3.10",
python_requires=">=3.11",
classifiers=[
"Development Status :: 5 - Production/Stable",
"Programming Language :: Python :: 3.10",
"Programming Language :: Python :: 3.11",
"Programming Language :: Python :: 3.12",
"Programming Language :: Python :: 3.13",
Expand Down
Loading