From 47f7690b9f8d041d2306ae48052eba1e86fbdd54 Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Fri, 15 May 2026 14:45:06 -0700 Subject: [PATCH] [SPARK-56889][PYTHON][INFRA] Drop Python 3.10 Support --- .../workflows/build_infra_images_cache.yml | 14 ---- .github/workflows/build_python_3.10.yml | 47 ------------ .github/workflows/build_python_minimum.yml | 2 +- .github/workflows/build_python_ps_minimum.yml | 2 +- README.md | 1 - binder/Dockerfile | 2 +- dev/infra/Dockerfile | 18 ++--- dev/spark-test-image/python-310/Dockerfile | 74 ------------------- .../python-minimum/Dockerfile | 12 +-- .../python-ps-minimum/Dockerfile | 12 +-- docs/index.md | 2 +- docs/rdd-programming-guide.md | 2 +- .../docs/source/development/contributing.rst | 8 +- .../docs/source/getting_started/install.rst | 4 +- .../tutorial/pandas_on_spark/typehints.rst | 2 +- python/packaging/classic/setup.py | 3 +- python/packaging/client/setup.py | 3 +- python/packaging/connect/setup.py | 3 +- python/pyspark/pandas/typedef/typehints.py | 24 +++--- python/pyspark/shuffle.py | 7 +- python/run-tests | 4 +- 21 files changed, 48 insertions(+), 198 deletions(-) delete mode 100644 .github/workflows/build_python_3.10.yml delete mode 100644 dev/spark-test-image/python-310/Dockerfile diff --git a/.github/workflows/build_infra_images_cache.yml b/.github/workflows/build_infra_images_cache.yml index 54fa7516fc0f5..78fb1cffaf1b2 100644 --- a/.github/workflows/build_infra_images_cache.yml +++ b/.github/workflows/build_infra_images_cache.yml @@ -32,7 +32,6 @@ on: - 'dev/spark-test-image/sparkr/Dockerfile' - 'dev/spark-test-image/python-minimum/Dockerfile' - 'dev/spark-test-image/python-ps-minimum/Dockerfile' - - 'dev/spark-test-image/python-310/Dockerfile' - 'dev/spark-test-image/python-311/Dockerfile' - 'dev/spark-test-image/python-312/Dockerfile' - 'dev/spark-test-image/python-312-classic-only/Dockerfile' @@ -139,19 +138,6 @@ jobs: - name: Image digest (PySpark PS with old dependencies) if: hashFiles('dev/spark-test-image/python-ps-minimum/Dockerfile') != '' run: echo ${{ steps.docker_build_pyspark_python_ps_minimum.outputs.digest }} - - name: Build and push (PySpark with Python 3.10) - if: hashFiles('dev/spark-test-image/python-310/Dockerfile') != '' - id: docker_build_pyspark_python_310 - uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f - with: - context: ./dev/spark-test-image/python-310/ - push: true - tags: ghcr.io/apache/spark/apache-spark-github-action-image-pyspark-python-310-cache:${{ github.ref_name }}-static - cache-from: type=registry,ref=ghcr.io/apache/spark/apache-spark-github-action-image-pyspark-python-310-cache:${{ github.ref_name }} - cache-to: type=registry,ref=ghcr.io/apache/spark/apache-spark-github-action-image-pyspark-python-310-cache:${{ github.ref_name }},mode=max - - name: Image digest (PySpark with Python 3.10) - if: hashFiles('dev/spark-test-image/python-310/Dockerfile') != '' - run: echo ${{ steps.docker_build_pyspark_python_310.outputs.digest }} - name: Build and push (PySpark with Python 3.11) if: hashFiles('dev/spark-test-image/python-311/Dockerfile') != '' id: docker_build_pyspark_python_311 diff --git a/.github/workflows/build_python_3.10.yml b/.github/workflows/build_python_3.10.yml deleted file mode 100644 index c39853b5cd4af..0000000000000 --- a/.github/workflows/build_python_3.10.yml +++ /dev/null @@ -1,47 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: "Build / Python-only (master, Python 3.10)" - -on: - schedule: - - cron: '0 17 */2 * *' - workflow_dispatch: - -jobs: - run-build: - permissions: - packages: write - name: Run - uses: ./.github/workflows/build_and_test.yml - if: github.repository == 'apache/spark' - with: - java: 17 - branch: master - hadoop: hadoop3 - envs: >- - { - "PYSPARK_IMAGE_TO_TEST": "python-310", - "PYTHON_TO_TEST": "python3.10" - } - jobs: >- - { - "pyspark": "true", - "pyspark-pandas": "true" - } diff --git a/.github/workflows/build_python_minimum.yml b/.github/workflows/build_python_minimum.yml index 0277ca28f4b9a..36bf7f6d7ba00 100644 --- a/.github/workflows/build_python_minimum.yml +++ b/.github/workflows/build_python_minimum.yml @@ -38,7 +38,7 @@ jobs: envs: >- { "PYSPARK_IMAGE_TO_TEST": "python-minimum", - "PYTHON_TO_TEST": "python3.10" + "PYTHON_TO_TEST": "python3.11" } jobs: >- { diff --git a/.github/workflows/build_python_ps_minimum.yml b/.github/workflows/build_python_ps_minimum.yml index 4e6292f289e75..f29b3e1bedd58 100644 --- a/.github/workflows/build_python_ps_minimum.yml +++ b/.github/workflows/build_python_ps_minimum.yml @@ -38,7 +38,7 @@ jobs: envs: >- { "PYSPARK_IMAGE_TO_TEST": "python-ps-minimum", - "PYTHON_TO_TEST": "python3.10" + "PYTHON_TO_TEST": "python3.11" } jobs: >- { diff --git a/README.md b/README.md index e849cfd97f531..ef09e33936d0d 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,6 @@ This README file only contains basic setup instructions. | | [![GitHub Actions Build](https://github.com/apache/spark/actions/workflows/build_maven_java21_macos26.yml/badge.svg)](https://github.com/apache/spark/actions/workflows/build_maven_java21_macos26.yml) | | | [![GitHub Actions Build](https://github.com/apache/spark/actions/workflows/build_maven_java21_arm.yml/badge.svg)](https://github.com/apache/spark/actions/workflows/build_maven_java21_arm.yml) | | | [![GitHub Actions Build](https://github.com/apache/spark/actions/workflows/build_coverage.yml/badge.svg)](https://github.com/apache/spark/actions/workflows/build_coverage.yml) | -| | [![GitHub Actions Build](https://github.com/apache/spark/actions/workflows/build_python_3.10.yml/badge.svg)](https://github.com/apache/spark/actions/workflows/build_python_3.10.yml) | | | [![GitHub Actions Build](https://github.com/apache/spark/actions/workflows/build_python_3.11.yml/badge.svg)](https://github.com/apache/spark/actions/workflows/build_python_3.11.yml) | | | [![GitHub Actions Build](https://github.com/apache/spark/actions/workflows/build_python_3.12_classic_only.yml/badge.svg)](https://github.com/apache/spark/actions/workflows/build_python_3.12_classic_only.yml) | | | [![GitHub Actions Build](https://github.com/apache/spark/actions/workflows/build_python_3.12_arm.yml/badge.svg)](https://github.com/apache/spark/actions/workflows/build_python_3.12_arm.yml) | diff --git a/binder/Dockerfile b/binder/Dockerfile index 2d5c30a9a92e0..60b207fcf7d88 100644 --- a/binder/Dockerfile +++ b/binder/Dockerfile @@ -15,7 +15,7 @@ # limitations under the License. # -FROM python:3.10-slim +FROM python:3.11-slim # install the notebook package RUN pip install --no-cache notebook jupyterlab diff --git a/dev/infra/Dockerfile b/dev/infra/Dockerfile index b848f8eb754da..435d32dacb067 100644 --- a/dev/infra/Dockerfile +++ b/dev/infra/Dockerfile @@ -61,7 +61,7 @@ RUN apt-get update && apt-get install -y \ openjdk-17-jdk-headless \ pandoc \ pkg-config \ - python3.10 \ + python3.11 \ python3-psutil \ qpdf \ r-base \ @@ -94,14 +94,14 @@ ARG BASIC_PIP_PKGS="numpy pyarrow>=18.0.0 six==1.16.0 pandas==2.3.3 scipy plotly # Python deps for Spark Connect ARG CONNECT_PIP_PKGS="grpcio==1.76.0 grpcio-status==1.76.0 protobuf==6.33.5 googleapis-common-protos==1.71.0 graphviz==0.20.3" -# Install Python 3.10 packages -RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python3.10 -RUN python3.10 -m pip install --ignore-installed 'blinker>=1.6.2' # mlflow needs this -RUN python3.10 -m pip install --ignore-installed 'six==1.16.0' # Avoid `python3-six` installation -RUN python3.10 -m pip install $BASIC_PIP_PKGS unittest-xml-reporting $CONNECT_PIP_PKGS && \ - python3.10 -m pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu && \ - python3.10 -m pip install deepspeed torcheval && \ - python3.10 -m pip cache purge +# Install Python 3.11 packages +RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python3.11 +RUN python3.11 -m pip install --ignore-installed 'blinker>=1.6.2' # mlflow needs this +RUN python3.11 -m pip install --ignore-installed 'six==1.16.0' # Avoid `python3-six` installation +RUN python3.11 -m pip install $BASIC_PIP_PKGS unittest-xml-reporting $CONNECT_PIP_PKGS && \ + python3.11 -m pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu && \ + python3.11 -m pip install deepspeed torcheval && \ + python3.11 -m pip cache purge # Install Python 3.9 RUN add-apt-repository ppa:deadsnakes/ppa diff --git a/dev/spark-test-image/python-310/Dockerfile b/dev/spark-test-image/python-310/Dockerfile deleted file mode 100644 index 326a5008ed5b2..0000000000000 --- a/dev/spark-test-image/python-310/Dockerfile +++ /dev/null @@ -1,74 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Image for building and testing Spark branches. Based on Ubuntu 24.04. -# See also in https://hub.docker.com/_/ubuntu -FROM ubuntu:noble -LABEL org.opencontainers.image.authors="Apache Spark project " -LABEL org.opencontainers.image.licenses="Apache-2.0" -LABEL org.opencontainers.image.ref.name="Apache Spark Infra Image For PySpark with Python 3.10" -# Overwrite this label to avoid exposing the underlying Ubuntu OS version label -LABEL org.opencontainers.image.version="" - -ENV FULL_REFRESH_DATE=20260210 - -ENV DEBIAN_FRONTEND=noninteractive -ENV DEBCONF_NONINTERACTIVE_SEEN=true - -RUN printf 'Types: deb\nURIs: https://mirrors.edge.kernel.org/ubuntu\nSuites: noble noble-updates noble-security\nComponents: main restricted universe multiverse\nSigned-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg\n' > /etc/apt/sources.list.d/mirror.sources - -RUN apt-get update && apt-get install -y \ - build-essential \ - ca-certificates \ - curl \ - gfortran \ - git \ - gnupg \ - libgit2-dev \ - liblapack-dev \ - libopenblas-dev \ - libssl-dev \ - libtiff5-dev \ - libwebp-dev \ - libxml2-dev \ - openjdk-17-jdk-headless \ - pkg-config \ - tzdata \ - software-properties-common \ - zlib1g-dev - -# Install Python 3.10 -RUN add-apt-repository ppa:deadsnakes/ppa -RUN apt-get update && apt-get install -y \ - python3.10 \ - python3.10-venv \ - && apt-get autoremove --purge -y \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* - -# Setup virtual environment -ENV VIRTUAL_ENV=/opt/spark-venv -RUN python3.10 -m venv $VIRTUAL_ENV -ENV PATH="$VIRTUAL_ENV/bin:$PATH" - -ARG BASIC_PIP_PKGS="numpy pyarrow>=23.0.0 six==1.16.0 pandas==2.3.3 scipy plotly<6.0.0 mlflow>=2.8.1 coverage matplotlib openpyxl memory-profiler>=0.61.0 scikit-learn>=1.3.2 pystack>=1.6.0 psutil" -ARG CONNECT_PIP_PKGS="grpcio==1.76.0 grpcio-status==1.76.0 protobuf==6.33.5 googleapis-common-protos==1.71.0 zstandard==0.25.0 graphviz==0.20.3" - -RUN python3.10 -m pip install $BASIC_PIP_PKGS unittest-xml-reporting $CONNECT_PIP_PKGS && \ - python3.10 -m pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu && \ - python3.10 -m pip install deepspeed torcheval && \ - python3.10 -m pip cache purge diff --git a/dev/spark-test-image/python-minimum/Dockerfile b/dev/spark-test-image/python-minimum/Dockerfile index 72abd5f5811b9..3d791bcb881eb 100644 --- a/dev/spark-test-image/python-minimum/Dockerfile +++ b/dev/spark-test-image/python-minimum/Dockerfile @@ -49,22 +49,22 @@ RUN apt-get update && apt-get install -y \ software-properties-common \ zlib1g-dev -# Install Python 3.10 +# Install Python 3.11 RUN add-apt-repository ppa:deadsnakes/ppa RUN apt-get update && apt-get install -y \ - python3.10 \ - python3.10-venv \ + python3.11 \ + python3.11-venv \ && apt-get autoremove --purge -y \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* # Setup virtual environment ENV VIRTUAL_ENV=/opt/spark-venv -RUN python3.10 -m venv $VIRTUAL_ENV +RUN python3.11 -m venv $VIRTUAL_ENV ENV PATH="$VIRTUAL_ENV/bin:$PATH" ARG BASIC_PIP_PKGS="numpy==1.22.4 pyarrow==18.0.0 pandas==2.2.0 six==1.16.0 scipy scikit-learn coverage unittest-xml-reporting psutil" ARG CONNECT_PIP_PKGS="grpcio==1.76.0 grpcio-status==1.76.0 googleapis-common-protos==1.71.0 zstandard==0.25.0 graphviz==0.20 protobuf==6.33.5" -RUN python3.10 -m pip install --force $BASIC_PIP_PKGS $CONNECT_PIP_PKGS && \ - python3.10 -m pip cache purge +RUN python3.11 -m pip install --force $BASIC_PIP_PKGS $CONNECT_PIP_PKGS && \ + python3.11 -m pip cache purge diff --git a/dev/spark-test-image/python-ps-minimum/Dockerfile b/dev/spark-test-image/python-ps-minimum/Dockerfile index 0f970202edd52..afbbe5a0d282b 100644 --- a/dev/spark-test-image/python-ps-minimum/Dockerfile +++ b/dev/spark-test-image/python-ps-minimum/Dockerfile @@ -49,22 +49,22 @@ RUN apt-get update && apt-get install -y \ software-properties-common \ zlib1g-dev -# Install Python 3.10 +# Install Python 3.11 RUN add-apt-repository ppa:deadsnakes/ppa RUN apt-get update && apt-get install -y \ - python3.10 \ - python3.10-venv \ + python3.11 \ + python3.11-venv \ && apt-get autoremove --purge -y \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* # Setup virtual environment ENV VIRTUAL_ENV=/opt/spark-venv -RUN python3.10 -m venv $VIRTUAL_ENV +RUN python3.11 -m venv $VIRTUAL_ENV ENV PATH="$VIRTUAL_ENV/bin:$PATH" ARG BASIC_PIP_PKGS="pyarrow==18.0.0 pandas==2.2.0 six==1.16.0 numpy scipy coverage unittest-xml-reporting psutil" ARG CONNECT_PIP_PKGS="grpcio==1.76.0 grpcio-status==1.76.0 googleapis-common-protos==1.71.0 zstandard==0.25.0 graphviz==0.20 protobuf==6.33.5" -RUN python3.10 -m pip install --force $BASIC_PIP_PKGS $CONNECT_PIP_PKGS && \ - python3.10 -m pip cache purge +RUN python3.11 -m pip install --force $BASIC_PIP_PKGS $CONNECT_PIP_PKGS && \ + python3.11 -m pip cache purge diff --git a/docs/index.md b/docs/index.md index 6d590172e9380..c1d6e03386f64 100644 --- a/docs/index.md +++ b/docs/index.md @@ -34,7 +34,7 @@ source, visit [Building Spark](building-spark.html). Spark runs on both Windows and UNIX-like systems (e.g. Linux, Mac OS), and it should run on any platform that runs a supported version of Java. This should include JVMs on x86_64 and ARM64. It's easy to run locally on one machine --- all you need is to have `java` installed on your system `PATH`, or the `JAVA_HOME` environment variable pointing to a Java installation. -Spark runs on Java 17/21/25, Scala 2.13, Python 3.10+, and R 3.5+ (Deprecated). +Spark runs on Java 17/21/25, Scala 2.13, Python 3.11+, and R 3.5+ (Deprecated). Java 25 prior to version 25.0.3 support is deprecated as of Spark 4.2.0. When using the Scala API, it is necessary for applications to use the same version of Scala that Spark was compiled for. Since Spark 4.0.0, it's Scala 2.13. diff --git a/docs/rdd-programming-guide.md b/docs/rdd-programming-guide.md index 8a9a59000792c..deff45ddc852f 100644 --- a/docs/rdd-programming-guide.md +++ b/docs/rdd-programming-guide.md @@ -39,7 +39,7 @@ along with if you launch Spark's interactive shell -- either `bin/spark-shell` f
-Spark {{site.SPARK_VERSION}} works with Python 3.10+. It can use the standard CPython interpreter, +Spark {{site.SPARK_VERSION}} works with Python 3.11+. It can use the standard CPython interpreter, so C libraries like NumPy can be used. Spark applications in Python can either be run with the `bin/spark-submit` script which includes Spark at runtime, or by including it in your setup.py as: diff --git a/python/docs/source/development/contributing.rst b/python/docs/source/development/contributing.rst index ba9d1b2cd1dc2..ddfd776c8f859 100644 --- a/python/docs/source/development/contributing.rst +++ b/python/docs/source/development/contributing.rst @@ -129,8 +129,8 @@ If you are using Conda, the development environment can be set as follows. .. code-block:: bash - # Python 3.10+ is required - conda create --name pyspark-dev-env python=3.10 + # Python 3.11+ is required + conda create --name pyspark-dev-env python=3.11 conda activate pyspark-dev-env pip install --upgrade -r dev/requirements.txt @@ -149,7 +149,7 @@ You can use Python's built-in ``venv`` module to create an isolated environment: .. code-block:: bash - # Python 3.10+ is required + # Python 3.11+ is required python3 -m venv .venv source .venv/bin/activate pip install --upgrade -r dev/requirements.txt @@ -157,7 +157,7 @@ You can use Python's built-in ``venv`` module to create an isolated environment: pip ~~~ -With Python 3.10+, pip can be used as below to install and set up the development environment. +With Python 3.11+, pip can be used as below to install and set up the development environment. .. code-block:: bash diff --git a/python/docs/source/getting_started/install.rst b/python/docs/source/getting_started/install.rst index 1b122e197c494..11c610b69277d 100644 --- a/python/docs/source/getting_started/install.rst +++ b/python/docs/source/getting_started/install.rst @@ -30,7 +30,7 @@ and building from the source. Python Versions Supported ------------------------- -Python 3.10 and above. +Python 3.11 and above. Using PyPI @@ -143,7 +143,7 @@ the same session as pyspark (you can install in several steps too). .. code-block:: bash - conda install -c conda-forge pyspark # can also add "python=3.10 some_package [etc.]" here + conda install -c conda-forge pyspark # can also add "python=3.11 some_package [etc.]" here Note that `PySpark for conda `_ is maintained separately by the community; while new versions generally get packaged quickly, the diff --git a/python/docs/source/tutorial/pandas_on_spark/typehints.rst b/python/docs/source/tutorial/pandas_on_spark/typehints.rst index 7045ce7f75987..46a53ec9d46fb 100644 --- a/python/docs/source/tutorial/pandas_on_spark/typehints.rst +++ b/python/docs/source/tutorial/pandas_on_spark/typehints.rst @@ -62,7 +62,7 @@ it as a Spark schema. As an example, you can specify the return type hint as bel Notice that the function ``pandas_div`` actually takes and outputs a pandas DataFrame instead of pandas-on-Spark :class:`DataFrame`. So, technically the correct types should be of pandas. -With Python 3.10+, you can specify the type hints by using pandas instances as follows: +With Python 3.11+, you can specify the type hints by using pandas instances as follows: .. code-block:: python diff --git a/python/packaging/classic/setup.py b/python/packaging/classic/setup.py index 911c50141e43f..95eea42f5cd95 100755 --- a/python/packaging/classic/setup.py +++ b/python/packaging/classic/setup.py @@ -382,10 +382,9 @@ def run(self): "pyyaml>=%s" % _minimum_pyyaml_version, ], }, - python_requires=">=3.10", + python_requires=">=3.11", classifiers=[ "Development Status :: 5 - Production/Stable", - "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", diff --git a/python/packaging/client/setup.py b/python/packaging/client/setup.py index 17475e9e065ad..b903765ecb63a 100755 --- a/python/packaging/client/setup.py +++ b/python/packaging/client/setup.py @@ -217,10 +217,9 @@ "numpy>=%s" % _minimum_numpy_version, "pyyaml>=%s" % _minimum_pyyaml_version, ], - python_requires=">=3.10", + python_requires=">=3.11", classifiers=[ "Development Status :: 5 - Production/Stable", - "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", diff --git a/python/packaging/connect/setup.py b/python/packaging/connect/setup.py index 1eb8cf1de77eb..0cc7fed6d5f3a 100755 --- a/python/packaging/connect/setup.py +++ b/python/packaging/connect/setup.py @@ -127,10 +127,9 @@ "numpy>=%s" % _minimum_numpy_version, "pyyaml>=%s" % _minimum_pyyaml_version, ], - python_requires=">=3.10", + python_requires=">=3.11", classifiers=[ "Development Status :: 5 - Production/Stable", - "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", diff --git a/python/pyspark/pandas/typedef/typehints.py b/python/pyspark/pandas/typedef/typehints.py index 658f3fb3bc627..69c20288dc124 100644 --- a/python/pyspark/pandas/typedef/typehints.py +++ b/python/pyspark/pandas/typedef/typehints.py @@ -21,7 +21,6 @@ import datetime import decimal -import sys import typing from collections.abc import Iterable from inspect import isclass @@ -863,21 +862,16 @@ def _new_type_holders( isinstance(param, slice) and param.step is None and param.stop is not None for param in params ) - if sys.version_info < (3, 11): - is_unnamed_params = all( - not isinstance(param, slice) and not isinstance(param, Iterable) for param in params - ) - else: - # PEP 646 changes `GenericAlias` instances into iterable ones at Python 3.11 - is_unnamed_params = all( - not isinstance(param, slice) - and ( - not isinstance(param, Iterable) - or isinstance(param, typing.GenericAlias) # type: ignore[attr-defined] - or isinstance(param, typing._GenericAlias) # type: ignore[attr-defined] - ) - for param in params + # PEP 646 changes `GenericAlias` instances into iterable ones at Python 3.11+ + is_unnamed_params = all( + not isinstance(param, slice) + and ( + not isinstance(param, Iterable) + or isinstance(param, typing.GenericAlias) # type: ignore[attr-defined] + or isinstance(param, typing._GenericAlias) # type: ignore[attr-defined] ) + for param in params + ) if is_named_params: # DataFrame["id": int, "A": int] diff --git a/python/pyspark/shuffle.py b/python/pyspark/shuffle.py index 3d52f0e02d0d1..bfdc0b79ce418 100644 --- a/python/pyspark/shuffle.py +++ b/python/pyspark/shuffle.py @@ -101,12 +101,7 @@ def _get_local_dirs(sub: str) -> list[str]: path = os.environ.get("SPARK_LOCAL_DIRS", "/tmp") dirs = path.split(",") if len(dirs) > 1: - if sys.version_info < (3, 11): - # different order in different processes and instances - rnd = random.Random(os.getpid() + id(dirs)) - random.shuffle(dirs, rnd.random) - else: - random.shuffle(dirs) + random.shuffle(dirs) return [os.path.join(d, "python", str(os.getpid()), sub) for d in dirs] diff --git a/python/run-tests b/python/run-tests index c0b0f1c9e5da4..9ef7bb5a3aaf0 100755 --- a/python/run-tests +++ b/python/run-tests @@ -21,9 +21,9 @@ FWDIR="$(cd "`dirname $0`"/..; pwd)" cd "$FWDIR" -PYTHON_VERSION_CHECK=$(python3 -c 'import sys; print(sys.version_info < (3, 10, 0))') +PYTHON_VERSION_CHECK=$(python3 -c 'import sys; print(sys.version_info < (3, 11, 0))') if [[ "$PYTHON_VERSION_CHECK" == "True" ]]; then - echo "Python versions prior to 3.10 are not supported." + echo "Python versions prior to 3.11 are not supported." exit -1 fi