Skip to content

fix(ci): clean up dangling EBS volumes left by Packer builds #2165

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
jfreeland wants to merge 1 commit into
base: develop
Choose a base branch
from
Open

fix(ci): clean up dangling EBS volumes left by Packer builds #2165

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 11 additions & 2 deletions .github/workflows/ami-release-nix-single.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -165,15 +165,24 @@ jobs:
SLACK_MESSAGE: 'Building Postgres AMI failed'
SLACK_FOOTER: ''

- name: configure aws credentials for cleanup
if: ${{ always() }}
uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4.3.1
with:
role-to-assume: ${{ secrets.DEV_AWS_ROLE }}
aws-region: "us-east-1"

- name: Cleanup resources after build
if: ${{ always() }}
run: |
EXECUTION_ID="${{ steps.build-ami.outputs.execution_id }}"
aws ec2 describe-instances --filters "Name=tag:packerExecutionId,Values=${EXECUTION_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --instance-ids
aws ec2 --region us-east-1 describe-instances --filters "Name=tag:packerExecutionId,Values=${EXECUTION_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --region us-east-1 --instance-ids
aws ec2 --region us-east-1 describe-volumes --filters "Name=tag:packerExecutionId,Values=${EXECUTION_ID}" "Name=status,Values=available" --query "Volumes[*].VolumeId" --output text | xargs -r -n1 aws ec2 --region us-east-1 delete-volume --volume-id

- name: Cleanup resources on build cancellation
if: ${{ cancelled() }}
run: |
EXECUTION_ID="${{ steps.build-ami.outputs.execution_id }}"
aws ec2 describe-instances --filters "Name=tag:packerExecutionId,Values=${EXECUTION_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --instance-ids
aws ec2 --region us-east-1 describe-instances --filters "Name=tag:packerExecutionId,Values=${EXECUTION_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --region us-east-1 --instance-ids
aws ec2 --region us-east-1 describe-volumes --filters "Name=tag:packerExecutionId,Values=${EXECUTION_ID}" "Name=status,Values=available" --query "Volumes[*].VolumeId" --output text | xargs -r -n1 aws ec2 --region us-east-1 delete-volume --volume-id

2 changes: 2 additions & 0 deletions .github/workflows/ami-release-nix.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -326,9 +326,11 @@ jobs:
run: |
EXECUTION_ID="${{ env.EXECUTION_ID }}"
aws ec2 --region us-east-1 describe-instances --filters "Name=tag:packerExecutionId,Values=${EXECUTION_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --region us-east-1 --instance-ids
aws ec2 --region us-east-1 describe-volumes --filters "Name=tag:packerExecutionId,Values=${EXECUTION_ID}" "Name=status,Values=available" --query "Volumes[*].VolumeId" --output text | xargs -r -n1 aws ec2 --region us-east-1 delete-volume --volume-id

- name: Cleanup resources on build cancellation
if: ${{ cancelled() }}
run: |
EXECUTION_ID="${{ env.EXECUTION_ID }}"
aws ec2 --region us-east-1 describe-instances --filters "Name=tag:packerExecutionId,Values=${EXECUTION_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --region us-east-1 --instance-ids
aws ec2 --region us-east-1 describe-volumes --filters "Name=tag:packerExecutionId,Values=${EXECUTION_ID}" "Name=status,Values=available" --query "Volumes[*].VolumeId" --output text | xargs -r -n1 aws ec2 --region us-east-1 delete-volume --volume-id
2 changes: 2 additions & 0 deletions .github/workflows/base-image-nightly.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,8 +70,10 @@ jobs:
if: ${{ always() }}
run: |
aws ec2 --region us-east-1 describe-instances --filters "Name=tag:packerExecutionId,Values=${EXECUTION_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --region us-east-1 --instance-ids
aws ec2 --region us-east-1 describe-volumes --filters "Name=tag:packerExecutionId,Values=${EXECUTION_ID}" "Name=status,Values=available" --query "Volumes[*].VolumeId" --output text | xargs -r -n1 aws ec2 --region us-east-1 delete-volume --volume-id

- name: Cleanup resources on build cancellation
if: ${{ cancelled() }}
run: |
aws ec2 --region us-east-1 describe-instances --filters "Name=tag:packerExecutionId,Values=${EXECUTION_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --region us-east-1 --instance-ids
aws ec2 --region us-east-1 describe-volumes --filters "Name=tag:packerExecutionId,Values=${EXECUTION_ID}" "Name=status,Values=available" --query "Volumes[*].VolumeId" --output text | xargs -r -n1 aws ec2 --region us-east-1 delete-volume --volume-id
14 changes: 14 additions & 0 deletions .github/workflows/testinfra-ami-build.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,13 @@ jobs:
else
echo "No packer build instances to clean up"
fi
VOLUME_IDS=$(aws ec2 --region ap-southeast-1 describe-volumes --filters "Name=tag:packerExecutionId,Values=${EXECUTION_ID}" "Name=status,Values=available" --query "Volumes[*].VolumeId" --output text)
if [ -n "$VOLUME_IDS" ]; then
echo "Deleting orphaned packer volumes: $VOLUME_IDS"
echo "$VOLUME_IDS" | xargs -r -n1 aws ec2 --region ap-southeast-1 delete-volume --volume-id
else
echo "No orphaned packer volumes to clean up"
fi

- name: Cleanup resources after build
if: ${{ always() }}
Expand All @@ -132,6 +139,13 @@ jobs:
else
echo "No testinfra instances to clean up"
fi
VOLUME_IDS=$(aws ec2 --region ap-southeast-1 describe-volumes --filters "Name=tag:packerExecutionId,Values=${EXECUTION_ID}" "Name=status,Values=available" --query "Volumes[*].VolumeId" --output text)
if [ -n "$VOLUME_IDS" ]; then
echo "Deleting orphaned packer volumes: $VOLUME_IDS"
echo "$VOLUME_IDS" | xargs -r -n1 aws ec2 --region ap-southeast-1 delete-volume --volume-id || true
else
echo "No orphaned packer volumes to clean up"
fi

- name: Cleanup stage 2 AMI
if: always()
Expand Down
5 changes: 3 additions & 2 deletions amazon-amd64-nix.pkr.hcl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -152,8 +152,9 @@ source "amazon-ebssurrogate" "source" {
packerExecutionId = "${var.packer-execution-id}"
}
run_volume_tags = {
creator = "packer"
appType = "postgres"
creator = "packer"
appType = "postgres"
packerExecutionId = "${var.packer-execution-id}"
}
snapshot_tags = {
creator = "packer"
Expand Down
5 changes: 3 additions & 2 deletions amazon-arm64-nix.pkr.hcl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -152,8 +152,9 @@ source "amazon-ebssurrogate" "source" {
packerExecutionId = "${var.packer-execution-id}"
}
run_volume_tags = {
creator = "packer"
appType = "postgres"
creator = "packer"
appType = "postgres"
packerExecutionId = "${var.packer-execution-id}"
}
snapshot_tags = {
creator = "packer"
Expand Down
Loading