forked from sig_core/toolkit
Add script that syncs EC2 images to AMI account
This commit is contained in:
parent
67d131178e
commit
443ca4c3f9
GPG Key ID:
705BC21EC3C70F34
260
sync/import-snapshot-publish-aws.sh
Executable file
260
sync/import-snapshot-publish-aws.sh
Executable file
@ -0,0 +1,260 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
VERSION="$1"
|
||||
EPOCH=0
|
||||
|
||||
RLVER=$VERSION
|
||||
|
||||
# Source common variables
|
||||
# shellcheck disable=SC2046,1091,1090
|
||||
source "$(dirname "${BASH_SOURCE[0]}")/common"
|
||||
|
||||
|
||||
usage() {
|
||||
echo "usage: $0 VERSION ($0 8.7)"
|
||||
}
|
||||
|
||||
aws() {
|
||||
command aws --region us-east-1 --profile resf-ami --output text $@
|
||||
}
|
||||
|
||||
DATE=$(date +%Y%m%d)
|
||||
|
||||
if [[ -z $VERSION ]]; then
|
||||
usage
|
||||
exit 1
|
||||
fi
|
||||
|
||||
download() {
|
||||
curl -o "$qcow" "https://dl.rockylinux.org/stg/rocky/$VERSION/images/$arch/$qcow"
|
||||
return $?
|
||||
}
|
||||
|
||||
exists() {
|
||||
aws s3 ls $2/$1 &>/dev/null
|
||||
return $?
|
||||
}
|
||||
|
||||
upload() {
|
||||
if exists $raw $BUCKET/$DATE; then
|
||||
echo "Found existing upload in $BUCKET"
|
||||
return 0
|
||||
fi
|
||||
aws s3 cp $raw $BUCKET/$DATE/
|
||||
}
|
||||
|
||||
convert() {
|
||||
qemu-img convert -f qcow2 -O raw "$qcow" $raw
|
||||
return $?
|
||||
}
|
||||
|
||||
BUCKET="s3://resf-prod-import-use1"
|
||||
|
||||
write-container-file() {
|
||||
printf '{"Description": "%s", "Format": "raw", "Url": "%s/%s"}\n' $name "$BUCKET/$DATE" $raw > $json
|
||||
}
|
||||
|
||||
download-convert-upload() {
|
||||
if ! [[ -f $qcow ]]; then
|
||||
echo "Downloading $qcow from cdn"
|
||||
download || exit
|
||||
fi
|
||||
|
||||
if ! [[ -f ${raw} ]]; then
|
||||
echo "Converting qcow2 $qcow to RAW $raw"
|
||||
convert || exit
|
||||
fi
|
||||
|
||||
echo "Uploading raw image back to s3"
|
||||
if ! upload; then
|
||||
echo "Failed to upload"
|
||||
exit
|
||||
fi
|
||||
}
|
||||
|
||||
|
||||
begin-job() {
|
||||
import_task_id="$(aws ec2 import-snapshot --disk-container "file://$PWD/$json" --query 'ImportTaskId')"
|
||||
if [[ -z $import_task_id ]]; then
|
||||
echo "Failed to import $json"
|
||||
exit 5
|
||||
fi
|
||||
echo $import_task_id
|
||||
return
|
||||
}
|
||||
|
||||
is-snapshot-imported() {
|
||||
snapshot_id=$(aws ec2 describe-import-snapshot-tasks --query 'ImportSnapshotTasks[].SnapshotTaskDetail.SnapshotId[]' --import-task-ids $1)
|
||||
if [[ -z "$snapshot_id" ]]; then
|
||||
return 0
|
||||
fi
|
||||
}
|
||||
|
||||
register-image() {
|
||||
# Given a snapshot id, register the image
|
||||
name=$1
|
||||
snapshot_id=$2
|
||||
|
||||
case $(awk -F'.' '{print $NF}'<<<"$name") in
|
||||
x86_64)
|
||||
arch=x86_64;;
|
||||
aarch64)
|
||||
arch=arm64;;
|
||||
esac
|
||||
|
||||
ami_id=$(aws --query "ImageId" ec2 register-image --name "$name" --description "$name" --block-device-mappings DeviceName="/dev/sda1",Ebs={SnapshotId="$snapshot_id"} --root-device-name "/dev/sda1" --virtualization-type hvm --architecture $arch)
|
||||
|
||||
if [[ -z "$ami_id" ]]; then
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
tag-resources() {
|
||||
local resources="$1"
|
||||
local tags="$2"
|
||||
if [[ -z $resources || -z $tags ]]; then
|
||||
echo "Need to provide tags and resources to tag"
|
||||
return 1
|
||||
fi
|
||||
aws ec2 create-tags --resources $resources --tags $tags
|
||||
}
|
||||
|
||||
image-exists() {
|
||||
local RESF_AMI_ACCOUNT_ID=792107900819
|
||||
local query="$(printf 'Images[?Name==`%s`].[ImageId,Name][]' "${1}")"
|
||||
mapfile -t res < <(aws ec2 describe-images --owners $RESF_AMI_ACCOUNT_ID --query "${query}" 2>/dev/null)
|
||||
res=($res)
|
||||
if [[ ${#res[@]} -eq 0 ]]; then
|
||||
# Skip empty results
|
||||
return 1 #not found
|
||||
fi
|
||||
id=${res[0]//\"}
|
||||
name=${res[@]/$id}
|
||||
found_image_id=$id
|
||||
return 0 # found
|
||||
}
|
||||
snapshot-exists() {
|
||||
local RESF_AMI_ACCOUNT_ID=792107900819
|
||||
local filter="$(printf 'Name=tag:Name,Values=%s' "${1}")"
|
||||
local query='Snapshots[].[SnapshotId][]'
|
||||
mapfile -t res < <(aws ec2 describe-snapshots --owner-ids $RESF_AMI_ACCOUNT_ID --filter "${filter}" --query "${query}" 2>/dev/null)
|
||||
res=($res)
|
||||
if [[ ${#res[@]} -eq 0 ]]; then
|
||||
# Skip empty results
|
||||
return 1 #not found
|
||||
fi
|
||||
id=${res[0]//\"}
|
||||
found_snapshot_id=$id
|
||||
return 0 # found
|
||||
}
|
||||
|
||||
declare -A import_jobs
|
||||
declare -A snapshot_ids
|
||||
declare -A ami_ids
|
||||
|
||||
ARCHES=(x86_64 aarch64)
|
||||
VARIANTS=(Base LVM)
|
||||
|
||||
for variant in "${VARIANTS[@]}"; do
|
||||
for arch in "${ARCHES[@]}"; do
|
||||
latest=$(printf "Rocky-%s-EC2-%s.latest.%s" "$VERSION" $variant $arch)
|
||||
name=$(printf "Rocky-%s-EC2-%s-%s-%s.%s.%s" "$VERSION" $variant $REVISION $DATE $EPOCH $arch)
|
||||
qcow=${latest}.qcow2
|
||||
raw=${name}.raw
|
||||
json=${name}.json
|
||||
|
||||
if image-exists $name; then
|
||||
echo "Found existing AMI in us-east-1. Skipping. ($found_image_id,$name)"
|
||||
continue
|
||||
fi
|
||||
|
||||
if snapshot-exists $name; then
|
||||
# If the snapshot exists, we can skip the import task and just do the image registration
|
||||
echo "Found existing snapshot: ($found_snapshot_id,$name)"
|
||||
snapshot_ids[$name]="${found_snapshot_id}"
|
||||
continue
|
||||
fi
|
||||
|
||||
# Download latest artifacts from CDN, convert from qcow2 to raw, and upload to the proper bucket
|
||||
echo "Downloading/converting artifacts for $name"
|
||||
download-convert-upload
|
||||
|
||||
|
||||
echo "Writing disk container json file"
|
||||
write-container-file
|
||||
|
||||
jobid=$(begin-job)
|
||||
echo "Beginning snapshot import task with id $jobid"
|
||||
import_jobs[$name]=$jobid
|
||||
done
|
||||
done
|
||||
|
||||
# wait for all import jobs to complete, then tag the resultant images
|
||||
finished=false
|
||||
while ! $finished; do
|
||||
for name in "${!import_jobs[@]}"; do
|
||||
import_task_id="${import_jobs[${name}]}"
|
||||
if ! is-snapshot-imported $import_task_id; then
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ -z $snapshot_id ]]; then
|
||||
echo "Snapshot ID is null.. continuing"
|
||||
continue
|
||||
fi
|
||||
|
||||
echo "Tagging snapshot with name"
|
||||
tag-resources $snapshot_id "Key=Name,Value=$name"
|
||||
|
||||
unset import_jobs[${name}]
|
||||
snapshot_ids[${name}]=$snapshot_id
|
||||
done
|
||||
# Check if we're done, if so, great!
|
||||
if [[ ${#import_jobs[@]} -gt 0 ]]; then
|
||||
sleep 1m
|
||||
continue
|
||||
else
|
||||
finished=true
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
finished=false
|
||||
while ! $finished; do
|
||||
for name in "${!snapshot_ids[@]}"; do
|
||||
# If the snapshot is imported, turn it into an AMI
|
||||
snapshot_id="${snapshot_ids[${name}]}"
|
||||
|
||||
echo "Creating AMI from snapshot."
|
||||
if ! register-image $name $snapshot_id; then
|
||||
echo "ERROR: Failed to create image for $name with snapshot id $snapshot_id"
|
||||
continue
|
||||
fi
|
||||
|
||||
echo "Tagging AMI - Name=$name"
|
||||
tag-resources $ami_id "Key=Name,Value=$name"
|
||||
|
||||
if [[ -z $ami_id ]]; then
|
||||
echo "AMI ID is null. continuing...";
|
||||
continue
|
||||
fi
|
||||
|
||||
unset snapshot_ids[${name}]
|
||||
ami_ids[$name]=$ami_id
|
||||
done
|
||||
if [[ ${#snapshot_ids[@]} -gt 0 ]]; then
|
||||
sleep 1m
|
||||
continue
|
||||
else
|
||||
finished=true
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
res=""
|
||||
for name in "${!ami_ids[@]}"; do
|
||||
ami_id="${ami_ids[${name}]}"
|
||||
res="${res}\n$(printf '%s\t%s\n' $name $ami_id)"
|
||||
done
|
||||
|
||||
printf "$res\n"
|
||||
Loading…
Reference in New Issue
Block a user