apache · ngachung · May 21, 2026 · May 22, 2026 · May 27, 2026
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - SDAP-534: Added proper support for data variables in NetCDF groups. Can handle multiple data variables across different groups. Coordinates are still required to be in root.
 ### Changed
 - Properly handle skipping of remote collections when reloading the collections config in the CM.
+- SDAP-542: Updated collection manager and granule ingester build to python 3.11.
 ### Deprecated
 ### Removed
 ### Fixed

diff --git a/collection_manager/README.md b/collection_manager/README.md
@@ -9,11 +9,11 @@ Manager service will publish a message to RabbitMQ to be picked up by the Granul
 
 ## Prerequisites
 
-Python 3.7
+Python 3.11
 
 Use a conda environment for example:
 
-    $ conda create -n cmenv python=3.7
+    $ conda create -n cmenv python=3.11
     $ conda activate cmenv    
 
 ## Building the service

diff --git a/collection_manager/collection_manager/services/S3Observer.py b/collection_manager/collection_manager/services/S3Observer.py
@@ -132,7 +132,8 @@ async def _get_s3_files(self, path: str):
         new_cache = {}
 
         start = time.perf_counter()
-        async with aioboto3.resource("s3") as s3:
+        session = aioboto3.Session()
+        async with session.resource("s3") as s3:
             bucket = await s3.Bucket(self._bucket)
 
             n_keys = 0

diff --git a/collection_manager/docker/Dockerfile b/collection_manager/docker/Dockerfile
@@ -13,12 +13,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM python:3.8.13
+FROM python:3.11-slim
 
-RUN apt-get update && apt-get install -y apt-transport-https gnupg2 ca-certificates curl
-RUN mkdir -p -m 755 /etc/apt/keyrings && curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.29/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
-RUN echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.29/deb/ /' | tee /etc/apt/sources.list.d/kubernetes.list
-RUN apt-get update && apt-get install -y kubectl
+ARG KUBECTL_VERSION=v1.29.14
+RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates curl && \
+    curl -fsSLo /usr/local/bin/kubectl "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" && \
+    chmod +x /usr/local/bin/kubectl && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
 
 COPY VERSION.txt /VERSION.txt
 COPY common /common
@@ -28,8 +30,7 @@ COPY collection_manager/requirements.txt /collection_manager/requirements.txt
 COPY collection_manager/README.md /collection_manager/README.md
 COPY collection_manager/docker/entrypoint.sh /entrypoint.sh
 
-RUN cd /common && python setup.py install 
-RUN pip install -U setuptools
-RUN cd /collection_manager && python setup.py install && pip uninstall -y chardet  && pip install boto3==1.16.10 requests==2.26.0
+RUN cd /common && pip install .
+RUN cd /collection_manager && pip install .
 
 ENTRYPOINT ["/bin/bash", "/entrypoint.sh"]
diff --git a/collection_manager/requirements.txt b/collection_manager/requirements.txt
@@ -13,14 +13,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-PyYAML==5.3.1
-pysolr==3.9.0
+PyYAML==6.0.2
+pysolr==3.10.0
 watchdog==0.10.2
 requests==2.31.0
 tenacity==6.2.0
-aioboto3==8.3.0
-aiobotocore==1.2.2
-aiohttp==3.8.3
+aioboto3==11.3.0
+aiobotocore==2.6.0
+boto3==1.28.17
+aiohttp==3.8.5
 aio-pika==6.7.1
-urllib3==1.25.8
+urllib3==1.26.20
 yarl<=1.9.4
diff --git a/collection_manager/setup.py b/collection_manager/setup.py
@@ -43,7 +43,7 @@
         "Operating System :: OS Independent",
         "Development Status :: 4 - Beta",
     ],
-    python_requires='>=3.8',
+    python_requires='>=3.11',
     include_package_data=True,
     install_requires=pip_requirements
 )
diff --git a/common/setup.py b/common/setup.py
@@ -31,6 +31,6 @@
         "Operating System :: OS Independent",
         "Development Status :: 4 - Beta",
     ],
-    python_requires='>=3.7',
+    python_requires='>=3.11',
     include_package_data=True
 )
diff --git a/granule_ingester/README.md b/granule_ingester/README.md
@@ -9,7 +9,7 @@ data to Cassandra and Solr.
 
 ## Prerequisites
 
-Python 3.7
+Python 3.11
 
 ## Building the service
 From `incubator-sdap-ingester`, run:

diff --git a/granule_ingester/docker/Dockerfile b/granule_ingester/docker/Dockerfile
@@ -21,14 +21,14 @@ COPY --chmod=755 granule_ingester/docker/install_python.sh /install_python.sh
 
 # Install python
 RUN /install_python.sh
-RUN ln -s /opt/python/3.8.17/bin/python3.8  /opt/python/3.8.17/bin/python
+RUN ln -s /opt/python/3.11.15/bin/python3.11 /opt/python/3.11.15/bin/python
 
 RUN apt-get update && \
     apt-get install --no-install-recommends -y git && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
-ENV PATH="/opt/python/3.8.17/bin:$PATH"
+ENV PATH="/opt/python/3.11.15/bin:$PATH"
 
 COPY VERSION.txt /VERSION.txt
 COPY common /common
@@ -79,8 +79,6 @@ ARG APACHE_NEXUSPROTO=https://github.com/apache/incubator-sdap-nexusproto.git
 ARG APACHE_NEXUSPROTO_BRANCH=master
 
 RUN ./install_nexusproto.sh $APACHE_NEXUSPROTO $APACHE_NEXUSPROTO_BRANCH
-RUN cd /common && python setup.py install
-
-RUN pip install boto3==1.16.10
+RUN cd /common && pip install .
 
 ENTRYPOINT ["/bin/bash", "/entrypoint.sh"]
diff --git a/granule_ingester/docker/install_python.sh b/granule_ingester/docker/install_python.sh
@@ -27,22 +27,22 @@ rm -rf /var/lib/apt/lists/*
 echo "Downloading & extracting source tarball"
 
 cd /tmp/
-wget https://www.python.org/ftp/python/3.8.17/Python-3.8.17.tgz
-tar xzf Python-3.8.17.tgz
-cd Python-3.8.17
+wget https://www.python.org/ftp/python/3.11.15/Python-3.11.15.tgz
+tar xzf Python-3.11.15.tgz
+cd Python-3.11.15
 
 echo "Running install"
 
-./configure --prefix=/opt/python/3.8.17/ --enable-optimizations --with-lto --with-computed-gotos --with-system-ffi
+./configure --prefix=/opt/python/3.11.15/ --enable-optimizations --with-lto --with-computed-gotos --with-system-ffi
 make -j "$(nproc)"
 make altinstall
 
 echo "Cleaning up install dir"
 
-rm /tmp/Python-3.8.17.tgz
+rm /tmp/Python-3.11.15.tgz
 cd /tmp/
-rm -rf Python-3.8.17
+rm -rf Python-3.11.15
 
 echo "Final steps..."
 
-/opt/python/3.8.17/bin/python3.8 -m pip install --upgrade pip setuptools wheel
+/opt/python/3.11.15/bin/python3.11 -m pip install --upgrade pip setuptools wheel
diff --git a/granule_ingester/granule_ingester/granule_loaders/GranuleLoader.py b/granule_ingester/granule_ingester/granule_loaders/GranuleLoader.py
@@ -99,7 +99,8 @@ async def _download_s3_file(url: str):
         parsed_url = parse.urlparse(url)
         logger.info(
             "Downloading S3 file from bucket '{}' with key '{}'".format(parsed_url.hostname, parsed_url.path[1:]))
-        async with aioboto3.resource("s3") as s3:
+        session = aioboto3.Session()
+        async with session.resource("s3") as s3:
             obj = await s3.Object(bucket_name=parsed_url.hostname, key=parsed_url.path[1:])
             response = await obj.get()
             data = await response['Body'].read()