Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
74 changes: 74 additions & 0 deletions .github/workflows/scripts/detect_changed_eessi_version.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
#!/bin/bash
# Script to detect which EESSI version(s) have easystack files changed in a PR
# Outputs the detected EESSI version(s) to stdout, separated by newlines

set -e

# Get the list of changed files from the PR
# When run in GitHub Actions, this uses the github.event.pull_request.changed_files
# For local testing or other contexts, can use git diff directly

CHANGED_FILES="$1"
BASE_COMMIT="$2"
HEAD_COMMIT="$3"

# EESSI version directories to check
EESSI_VERSIONS=("2023.06" "2025.06")
DETECTED_VERSIONS=()

if [ -n "$CHANGED_FILES" ]; then
# Changed files are passed as argument (newline-separated from GitHub Actions join())
# Handle both actual newlines and literal \n strings
# First, convert literal \n to actual newlines if needed
CHANGED_FILES=$(echo "$CHANGED_FILES" | sed 's/\\n/\n/g')

files_array=()
while IFS= read -r file; do
# Trim whitespace and skip empty lines
file=$(echo "$file" | tr -d '[:space:]')
if [ -n "$file" ]; then
files_array+=("$file")
fi
done <<< "$CHANGED_FILES"
elif [ -n "$BASE_COMMIT" ] && [ -n "$HEAD_COMMIT" ]; then
# Use git diff to get changed files between commits
IFS=$'\n' read -r -d '' -a files_array <<< "$(git diff --name-only "$BASE_COMMIT" "$HEAD_COMMIT")" || true
else
# Try to use GitHub Actions context
if [ -n "$GITHUB_EVENT_PATH" ]; then
# Extract changed files from the pull_request event using jq if available
if command -v jq &> /dev/null; then
CHANGED_FILES_JSON=$(jq -r '.changed_files // empty' "$GITHUB_EVENT_PATH" 2>/dev/null)
if [ -n "$CHANGED_FILES_JSON" ]; then
while IFS= read -r file; do
if [ -n "$file" ]; then
files_array+=("$file")
fi
done <<< "$CHANGED_FILES_JSON"
fi
else
# Fallback to grep-based parsing
CHANGED_FILES_JSON=$(cat "$GITHUB_EVENT_PATH" | grep -o '"changed_files"[[:space:]]*:[[:space:]]*"[^"]*"' | cut -d'"' -f4)
if [ -n "$CHANGED_FILES_JSON" ]; then
IFS=$'\n' read -r -d '' -a files_array <<< "$CHANGED_FILES_JSON" || true
fi
fi
fi
fi

# Check each changed file to see which EESSI version directory it belongs to
for file in "${files_array[@]}"; do
for version in "${EESSI_VERSIONS[@]}"; do
if [[ "$file" == easystacks/software.eessi.io/$version/* ]]; then
# Add version to detected versions if not already present
if [[ ! " ${DETECTED_VERSIONS[*]} " =~ " $version " ]]; then
DETECTED_VERSIONS+=("$version")
fi
fi
done
done

# Output detected versions
for version in "${DETECTED_VERSIONS[@]}"; do
echo "$version"
done
237 changes: 139 additions & 98 deletions .github/workflows/test-software.eessi.io.yml
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,46 @@ env:
# and then allow for special cases for specific architectures
aarch64/a64fx: []
jobs:
detect_eessi_version:
runs-on: ubuntu-latest
outputs:
detected_versions: ${{ steps.detect.outputs.detected_versions }}
steps:
- name: Check out software-layer repository
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
with:
fetch-depth: 0 # Fetch all history for all branches and tags

- name: Detect changed EESSI version(s) in PR
id: detect
run: |
# Get changed files from GitHub Actions context (only for pull requests)
if [ "${{ github.event_name }}" == "pull_request" ]; then
# Use jq to properly parse the changed_files array from GitHub event
changed_files=$(jq -r '.pull_request.changed_files[]' "$GITHUB_EVENT_PATH" 2>/dev/null || echo "")
echo "Changed files in PR:"
echo "$changed_files"
echo ""
else
changed_files=""
fi

# Run the detection script
# For PRs, use base.sha; for other events, use the current commit
base_sha="${{ github.event.pull_request.base.sha || github.sha }}"
detected_versions=$(bash .github/workflows/scripts/detect_changed_eessi_version.sh "$changed_files" "$base_sha" "${{ github.sha }}")

echo "Detected EESSI version(s):"
echo "$detected_versions"
echo ""

# Store detected versions as newline-separated list in output
echo "detected_versions<<EOF" >> $GITHUB_OUTPUT
echo "$detected_versions" >> $GITHUB_OUTPUT
echo "EOF" >> $GITHUB_OUTPUT

check_missing:
needs: detect_eessi_version
strategy:
fail-fast: false
matrix:
Expand Down Expand Up @@ -128,109 +167,111 @@ jobs:
EESSI_SOFTWARE_SUBDIR_OVERRIDE: x86_64/generic
runs-on: ${{ matrix.runs_on }}
steps:
- name: Check out software-layer repository
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
with:
fetch-depth: 0 # Fetch all history for all branches and tags
- name: Check out software-layer repository
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
with:
fetch-depth: 0 # Fetch all history for all branches and tags

- name: Clone EESSI/software-layer-scripts repository
run: |
git clone https://github.com/EESSI/software-layer-scripts
- name: Clone EESSI/software-layer-scripts repository
run: |
git clone https://github.com/EESSI/software-layer-scripts

- name: Show host system info
run: |
echo "/proc/cpuinfo:"
cat /proc/cpuinfo
echo
echo "lscpu:"
lscpu
- name: Show host system info
run: |
echo "/proc/cpuinfo:"
cat /proc/cpuinfo
echo
echo "lscpu:"
lscpu

- name: Mount EESSI CernVM-FS pilot repository
uses: cvmfs-contrib/github-action-cvmfs@55899ca74cf78ab874bdf47f5a804e47c198743c # v4.0
with:
cvmfs_config_package: https://github.com/EESSI/filesystem-layer/releases/download/latest/cvmfs-config-eessi_latest_all.deb
cvmfs_http_proxy: DIRECT
cvmfs_repositories: software.eessi.io
- name: Mount EESSI CernVM-FS pilot repository
uses: cvmfs-contrib/github-action-cvmfs@55899ca74cf78ab874bdf47f5a804e47c198743c # v4.0
with:
cvmfs_config_package: https://github.com/EESSI/filesystem-layer/releases/download/latest/cvmfs-config-eessi_latest_all.deb
cvmfs_http_proxy: DIRECT
cvmfs_repositories: software.eessi.io

- name: Check for missing installlations
run: |
export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}}
source /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/bash
# set $EESSI_CPU_FAMILY to the CPU architecture that corresponds to $EESSI_SOFTWARE_SUBDIR_OVERRIDE (part before the first slash),
# to prevent issues with checks in the Easybuild configuration that use this variable
export EESSI_CPU_FAMILY=${EESSI_SOFTWARE_SUBDIR_OVERRIDE%%/*}
export EESSI_PREFIX=/cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}
export EESSI_OS_TYPE=linux
env | grep ^EESSI | sort
- name: Check for missing installations
if: ${{ github.event_name != 'pull_request' || contains(needs.detect_eessi_version.outputs.detected_versions, matrix.EESSI_VERSION) }}
run: |
export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}}
source /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/bash
# set $EESSI_CPU_FAMILY to the CPU architecture that corresponds to $EESSI_SOFTWARE_SUBDIR_OVERRIDE (part before the first slash),
# to prevent issues with checks in the Easybuild configuration that use this variable
export EESSI_CPU_FAMILY=${EESSI_SOFTWARE_SUBDIR_OVERRIDE%%/*}
export EESSI_PREFIX=/cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}
export EESSI_OS_TYPE=linux
env | grep ^EESSI | sort

# first check the CPU-only builds for this CPU target
echo "first run check_missing_installations.sh for CPU-only builds"
for easystack_file in $(EESSI_VERSION=${{matrix.EESSI_VERSION}} .github/workflows/scripts/only_latest_easystacks.sh); do
eb_version=$(echo ${easystack_file} | sed 's/.*eb-\([0-9.]*\).*.yml/\1/g')
echo "check missing installations for ${easystack_file} with EasyBuild ${eb_version}..."
module purge
module load EasyBuild/${eb_version}
which eb
eb --version
software-layer-scripts/check_missing_installations.sh ${easystack_file}
ec=$?
if [[ ${ec} -ne 0 ]]; then echo "missing installations found for ${easystack_file}!" >&2; exit ${ec}; fi
done
# first check the CPU-only builds for this CPU target
echo "first run check_missing_installations.sh for CPU-only builds"
for easystack_file in $(EESSI_VERSION=${{matrix.EESSI_VERSION}} .github/workflows/scripts/only_latest_easystacks.sh); do
eb_version=$(echo ${easystack_file} | sed 's/.*eb-\([0-9.]*\).*.yml/\1/g')
echo "check missing installations for ${easystack_file} with EasyBuild ${eb_version}..."
module purge
module load EasyBuild/${eb_version}
which eb
eb --version
software-layer-scripts/check_missing_installations.sh ${easystack_file}
ec=$?
if [[ ${ec} -ne 0 ]]; then echo "missing installations found for ${easystack_file}!" >&2; exit ${ec}; fi
done

# now check the accelerator builds for this CPU target
accelerators=$(echo "${EESSI_ACCELERATOR_TARGETS}" | yq ".\"${{matrix.EESSI_VERSION}}\".\"${EESSI_SOFTWARE_SUBDIR_OVERRIDE}\" // .\"${{matrix.EESSI_VERSION}}\".default | .[]")
if [ -z "${accelerators}" ]; then
echo "no accelerator targets defined for ${EESSI_SOFTWARE_SUBDIR_OVERRIDE}"
else
for accel in ${accelerators}; do
module use ${EESSI_SOFTWARE_PATH}/accel/${accel}/modules/all
echo "checking missing installations for accelerator ${accel} using modulepath: ${MODULEPATH}"
for easystack_file in $(EESSI_VERSION=${{matrix.EESSI_VERSION}} ACCEL_EASYSTACKS=1 .github/workflows/scripts/only_latest_easystacks.sh); do
eb_version=$(echo ${easystack_file} | sed 's/.*eb-\([0-9.]*\).*.yml/\1/g')
echo "check missing installations for ${easystack_file} with EasyBuild ${eb_version}..."
module purge
module load EasyBuild/${eb_version}
which eb
eb --version
software-layer-scripts/check_missing_installations.sh ${easystack_file}
ec=$?
if [[ ${ec} -ne 0 ]]; then echo "missing installations found for ${easystack_file}!" >&2; exit ${ec}; fi
done
module unuse ${EESSI_SOFTWARE_PATH}/accel/${accel}/modules/all
done
fi
# now check the accelerator builds for this CPU target
accelerators=$(echo "${EESSI_ACCELERATOR_TARGETS}" | yq ".\"${{matrix.EESSI_VERSION}}\".\"${EESSI_SOFTWARE_SUBDIR_OVERRIDE}\" // .\"${{matrix.EESSI_VERSION}}\".default | .[]")
if [ -z "${accelerators}" ]; then
echo "no accelerator targets defined for ${EESSI_SOFTWARE_SUBDIR_OVERRIDE}"
else
for accel in ${accelerators}; do
module use ${EESSI_SOFTWARE_PATH}/accel/${accel}/modules/all
echo "checking missing installations for accelerator ${accel} using modulepath: ${MODULEPATH}"
for easystack_file in $(EESSI_VERSION=${{matrix.EESSI_VERSION}} ACCEL_EASYSTACKS=1 .github/workflows/scripts/only_latest_easystacks.sh); do
eb_version=$(echo ${easystack_file} | sed 's/.*eb-\([0-9.]*\).*.yml/\1/g')
echo "check missing installations for ${easystack_file} with EasyBuild ${eb_version}..."
module purge
module load EasyBuild/${eb_version}
which eb
eb --version
software-layer-scripts/check_missing_installations.sh ${easystack_file}
ec=$?
if [[ ${ec} -ne 0 ]]; then echo "missing installations found for ${easystack_file}!" >&2; exit ${ec}; fi
done
module unuse ${EESSI_SOFTWARE_PATH}/accel/${accel}/modules/all
done
fi

# make sure that Lmod cache file is present
ls -l ${EESSI_SOFTWARE_PATH}/.lmod/cache/spiderT.lua
# make sure that Lmod cache file is present
ls -l ${EESSI_SOFTWARE_PATH}/.lmod/cache/spiderT.lua

- name: Test check_missing_installations.sh with missing package (GCC/8.3.0)
run: |
export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}}
source /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/bash
# set $EESSI_CPU_FAMILY to the CPU architecture that corresponds to $EESSI_SOFTWARE_SUBDIR_OVERRIDE (part before the first slash),
# to prevent issues with checks in the Easybuild configuration that use this variable
export EESSI_CPU_FAMILY=${EESSI_SOFTWARE_SUBDIR_OVERRIDE%%/*}
module load EasyBuild
which eb
eb --version
export EESSI_PREFIX=/cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}
export EESSI_OS_TYPE=linux
env | grep ^EESSI | sort
# create dummy easystack file with a single entry (something that is not installed in EESSI)
easystack_file="test.yml"
echo "easyconfigs:" > ${easystack_file}
echo " - GCC-8.3.0:" >> ${easystack_file}
echo "created easystack file '${easystack_file}' with a missing installation (GCC/8.3.0):"
cat ${easystack_file}
# note, check_missing_installations.sh exits 1 if a package was
# missing, which is intepreted as false (exit code based, not
# boolean logic), hence when the script exits 0 if no package was
# missing it is interpreted as true, thus the test did not capture
# the missing package
if software-layer-scripts/check_missing_installations.sh ${easystack_file}; then
echo "did NOT capture missing package; test FAILED"
exit 1
else
echo "captured missing package; test PASSED"
exit 0
fi
- name: Test check_missing_installations.sh with missing package (GCC/8.3.0)
if: ${{ github.event_name != 'pull_request' || contains(needs.detect_eessi_version.outputs.detected_versions, matrix.EESSI_VERSION) }}
run: |
export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}}
source /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/bash
# set $EESSI_CPU_FAMILY to the CPU architecture that corresponds to $EESSI_SOFTWARE_SUBDIR_OVERRIDE (part before the first slash),
# to prevent issues with checks in the Easybuild configuration that use this variable
export EESSI_CPU_FAMILY=${EESSI_SOFTWARE_SUBDIR_OVERRIDE%%/*}
module load EasyBuild
which eb
eb --version
export EESSI_PREFIX=/cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}
export EESSI_OS_TYPE=linux
env | grep ^EESSI | sort
# create dummy easystack file with a single entry (something that is not installed in EESSI)
easystack_file="test.yml"
echo "easyconfigs:" > ${easystack_file}
echo " - GCC-8.3.0:" >> ${easystack_file}
echo "created easystack file '${easystack_file}' with a missing installation (GCC/8.3.0):"
cat ${easystack_file}
# note, check_missing_installations.sh exits 1 if a package was
# missing, which is intepreted as false (exit code based, not
# boolean logic), hence when the script exits 0 if no package was
# missing it is interpreted as true, thus the test did not capture
# the missing package
if software-layer-scripts/check_missing_installations.sh ${easystack_file}; then
echo "did NOT capture missing package; test FAILED"
exit 1
else
echo "captured missing package; test PASSED"
exit 0
fi
Original file line number Diff line number Diff line change
Expand Up @@ -18,3 +18,4 @@ easyconfigs:
# - framework PR for fix: https://github.com/easybuilders/easybuild-framework/pull/5142
# - triggered by https://github.com/easybuilders/easybuild-easyblocks/pull/3993
- jupyterlmod-5.2.2-GCCcore-13.3.0.eb
- Test-foss-2024a.eb
Loading