toolkit/sync/import-snapshot-publish-aws.sh

#!/usr/bin/env bash

VERSION="$1"
EPOCH=${EPOCH:-0}

RLVER=$VERSION

# Source common variables
# shellcheck disable=SC2046,1091,1090
source "$(dirname "${BASH_SOURCE[0]}")/common"

usage() {
	echo "usage: $0 VERSION ($0 8.7)"
}

aws() {
	command aws --region us-east-1 --profile resf-ami --output text $@
}

DATE=${DATE:-$(date +%Y%m%d)}

if [[ -z $VERSION ]]; then
	usage
	exit 1
fi

download() {
	curl -o "$qcow" "https://dl.rockylinux.org/stg/rocky/$VERSION/images/$arch/$qcow"
	return $?
}

exists() {
	aws s3 ls $2/$1 &>/dev/null
	return $?
}

upload() {
	if exists $raw $BUCKET/$DATE; then
		echo "Found existing upload in $BUCKET"
		return 0
	fi
	aws s3 cp $raw $BUCKET/$DATE/
}

convert() {
	qemu-img convert -f qcow2 -O raw "$qcow" $raw
	return $?
}

BUCKET="s3://resf-prod-import-use1"

write-container-file() {
	printf '{"Description": "%s", "Format": "raw", "Url": "%s/%s"}\n' $name "$BUCKET/$DATE" $raw >$json
}

download-convert-upload() {
	if ! [[ -f $qcow ]]; then
		echo "Downloading $qcow from cdn"
		download || exit
	fi

	if ! [[ -f ${raw} ]]; then
		echo "Converting qcow2 $qcow to RAW $raw"
		convert || exit
	fi

	echo "Uploading raw image back to s3"
	if ! upload; then
		echo "Failed to upload"
		exit
	fi
}

begin-job() {
	import_task_id="$(aws ec2 import-snapshot --disk-container "file://$PWD/$json" --query 'ImportTaskId')"
	if [[ -z $import_task_id ]]; then
		echo "Failed to import $json"
		return 1
	fi
	echo $import_task_id
	return 0
}

is-snapshot-imported() {
	snapshot_id=$(aws ec2 describe-import-snapshot-tasks --query 'ImportSnapshotTasks[].SnapshotTaskDetail.SnapshotId[]' --import-task-ids $1)
	if [[ -z "$snapshot_id" ]]; then
		return 1
	fi
	return 0
}

register-image() {
	# Given a snapshot id, register the image
	name=$1
	snapshot_id=$2

	case $(awk -F'.' '{print $NF}' <<<"$name") in
	x86_64)
		arch=x86_64
		;;
	aarch64)
		arch=arm64
		;;
	esac

	ami_id=$(aws --query "ImageId" ec2 register-image --name "$name" --description "$name" --block-device-mappings DeviceName="/dev/sda1",Ebs={SnapshotId="$snapshot_id"} --root-device-name "/dev/sda1" --virtualization-type hvm --architecture $arch --ena-support)

	if [[ -z "$ami_id" ]]; then
		return 1
	fi
	return 0
}

tag-resources() {
	local resources="$1"
	local tags="$2"
	if [[ -z $resources || -z $tags ]]; then
		echo "Need to provide tags and resources to tag"
		return 1
	fi
	aws ec2 create-tags --resources $resources --tags $tags
}

image-exists() {
	local RESF_AMI_ACCOUNT_ID=792107900819
	local query="$(printf 'Images[?Name==`%s`].[ImageId,Name][]' "${1}")"
	mapfile -t res < <(aws ec2 describe-images --owners $RESF_AMI_ACCOUNT_ID --query "${query}" 2>/dev/null)
	res=($res)
	if [[ ${#res[@]} -eq 0 ]]; then
		# Skip empty results
		return 1 #not found
	fi
	id=${res[0]//\"/}
	name=${res[@]/$id/}
	found_image_id=$id
	return 0 # found
}
snapshot-exists() {
	local RESF_AMI_ACCOUNT_ID=792107900819
	local filter="$(printf 'Name=tag:Name,Values=%s' "${1}")"
	local query='Snapshots[].[SnapshotId][]'
	mapfile -t res < <(aws ec2 describe-snapshots --owner-ids $RESF_AMI_ACCOUNT_ID --filter "${filter}" --query "${query}" 2>/dev/null)
	res=($res)
	if [[ ${#res[@]} -eq 0 ]]; then
		# Skip empty results
		return 1 #not found
	fi
	id=${res[0]//\"/}
	found_snapshot_id=$id
	return 0 # found
}

declare -A import_jobs
declare -A snapshot_ids
declare -A ami_ids

TARGET_ARCHES=(x86_64 aarch64)
TARGET_VARIANTS=(Base LVM)

for variant in "${TARGET_VARIANTS[@]}"; do
	for arch in "${TARGET_ARCHES[@]}"; do
		latest=$(printf "Rocky-%s-EC2-%s.latest.%s" "$VERSION" $variant $arch)
		name=$(printf "Rocky-%s-EC2-%s-%s-%s.%s.%s" "$VERSION" $variant $REVISION $DATE $EPOCH $arch)
		qcow=${name}.qcow2
		raw=${name}.raw
		json=${name}.json

		if image-exists $name; then
			echo "Found existing AMI in us-east-1. Skipping. ($found_image_id,$name)"
			continue
		fi

		if snapshot-exists $name; then
			# If the snapshot exists, we can skip the import task and just do the image registration
			echo "Found existing snapshot: ($found_snapshot_id,$name)"
			snapshot_ids[$name]="${found_snapshot_id}"
			continue
		fi

		# Download latest artifacts from CDN, convert from qcow2 to raw, and upload to the proper bucket
		echo "Downloading/converting artifacts for $name"
		download-convert-upload

		echo "Writing disk container json file"
		write-container-file

		jobid=$(begin-job)
		echo "Beginning snapshot import task with id $jobid"
		import_jobs[$name]=$jobid
	done
done

# wait for all import jobs to complete, then tag the resultant images
finished=false
while ! $finished; do
	for name in "${!import_jobs[@]}"; do
		import_task_id="${import_jobs[${name}]}"
		if ! is-snapshot-imported $import_task_id; then
			echo "Snapshot for $import_task_id ($name) is not yet finished"
			continue
		fi

		# await finalization
		sleep 2

		if [[ -z $snapshot_id ]]; then
			echo "Snapshot ID is null.. exiting"
			exit 2
		fi

		echo "Tagging snapshot with name"
		tag-resources $snapshot_id "Key=Name,Value=$name"

		unset import_jobs[${name}]
		snapshot_ids[${name}]=$snapshot_id
	done
	# Check if we're done, if so, great!
	if [[ ${#import_jobs[@]} -gt 0 ]]; then
		echo "Sleeping for 1m"
		sleep 1m
		continue
	else
		finished=true
		break
	fi
done

finished=false
while ! $finished; do
	for name in "${!snapshot_ids[@]}"; do
		# If the snapshot is imported, turn it into an AMI
		snapshot_id="${snapshot_ids[${name}]}"

		echo "Creating AMI from snapshot $snapshot_id ($name)"
		if ! register-image $name $snapshot_id; then
			echo "ERROR: Failed to create image for $name with snapshot id $snapshot_id"
			continue
		fi

		echo "Tagging AMI - Name=$name"
		tag-resources $ami_id "Key=Name,Value=$name"

		if [[ -z $ami_id ]]; then
			echo "AMI ID is null. continuing..."
			continue
		fi

		unset snapshot_ids[${name}]
		ami_ids[$name]=$ami_id
	done
	if [[ ${#snapshot_ids[@]} -gt 0 ]]; then
		sleep 1m
		continue
	else
		finished=true
		break
	fi
done

res=""
for name in "${!ami_ids[@]}"; do
	ami_id="${ami_ids[${name}]}"
	res="${res}\n$(printf '%s\t%s\n' $name $ami_id)"
done

printf "$res\n"