formatting, add WSL container variant, support requesting new quotas

2023-05-12 14:00:52 -04:00 · 2023-05-12 14:00:52 -04:00 · ed2a2999ad
commit ed2a2999ad
parent 173b0ff814
7 changed files with 327 additions and 293 deletions
--- a/iso/empanadas/empanadas/common.py
+++ b/iso/empanadas/empanadas/common.py
@ -104,7 +104,7 @@ for conf in glob.iglob(f"{_rootdir}/sig/*.yaml"):

 ALLOWED_TYPE_VARIANTS = {
        "Azure": ["Base", "LVM"],
-        "Container": ["Base", "Minimal", "UBI"],
+        "Container": ["Base", "Minimal", "UBI", "WSL"],
        "EC2": ["Base", "LVM"],
        "GenericCloud": ["Base", "LVM"],
        "Vagrant": ["Libvirt", "Vbox", "VMware"],
--- a/iso/empanadas/empanadas/configs/el8.yaml
+++ b/iso/empanadas/empanadas/configs/el8.yaml
@ -88,7 +88,7 @@
        primary_variant: 'Base'
      Container:
        format: tar.xz
-        variants: [Base, Minimal, UBI]
+        variants: [Base, Minimal, UBI, WSL]
      RPI:
        format: raw.xz
      OCP:
--- a/iso/empanadas/empanadas/templates/kube/Job.tmpl
+++ b/iso/empanadas/empanadas/templates/kube/Job.tmpl
@ -2,7 +2,7 @@
 apiVersion: batch/v1
 kind: Job
 metadata:
-  name: {{ jobname }}-{{ major }}-{{ architecture }}
+  name: {{ jobname }}-{{ major }}-{{ architecture }}-{{ buildTime }}
  namespace: {{ namespace }}
 spec:
  template:
--- a/mangle/quotas.go
+++ b/mangle/quotas.go
@ -7,12 +7,14 @@ import (
 	"sync"

 	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/aws/aws-sdk-go/service/servicequotas"
 )

-func getQuotaCode(sqSvc *servicequotas.ServiceQuotas) string {
+func getQuotaCode(sess *session.Session) string {
+	sqSvc := servicequotas.New(sess)
 	input := &servicequotas.ListServiceQuotasInput{
 		ServiceCode: aws.String("ec2"),
 	}
@ -71,9 +73,13 @@ func getQuotaInfo(sqSvc *servicequotas.ServiceQuotas, quotaCode string, region s

 	output, err := sqSvc.GetServiceQuota(input)
 	if err != nil {
-		log.Printf("Error getting quota info for %s: %s\n", region, err)
+		if awsErr, ok := err.(awserr.Error); ok {
+			if message := awsErr.Code(); message == "UnknownOperationException" {
+				log.Printf("[sdk] Region %s does not appear to support Service Quotas: %v", region, message)
 				return nil
-		// os.Exit(1)
+			}
+			log.Fatalf("[sdk] Error getting quota info for %s: %v\n", region, awsErr)
+		}
 	}

 	currentValue := *output.Quota.Value
@ -116,6 +122,35 @@ func listQuotas(sess *session.Session, quotaCode string, regions []*string) {
 	wg.Wait()
 }

+func requestQuotaIncrease(sess *session.Session, quotaCode string, regions []string, quota float64) {
+	var wg sync.WaitGroup
+	wg.Add(len(regions))
+
+	for _, region := range regions {
+		go func(region string) {
+			defer wg.Done()
+			regionSqSvc := servicequotas.New(sess, &aws.Config{Region: aws.String(region)})
+			quotaInfo := getQuotaInfo(regionSqSvc, quotaCode, region)
+			if quotaInfo.CurrentQuota >= quota {
+				fmt.Printf("Quota for Public AMIs in region %s is already set to %.0f, skipping request.\n", region, quotaInfo.CurrentQuota)
+			} else {
+				input := &servicequotas.RequestServiceQuotaIncreaseInput{
+					ServiceCode:  aws.String("ec2"),
+					QuotaCode:    aws.String(quotaCode),
+					DesiredValue: aws.Float64(quota),
+				}
+				output, err := regionSqSvc.RequestServiceQuotaIncrease(input)
+				if err != nil {
+					fmt.Println("Error requesting quota increase:", err)
+					os.Exit(1)
+				}
+				fmt.Printf("Successfully submitted request with ID: %s\n", aws.StringValue(output.RequestedQuota.Id))
+			}
+		}(region)
+	}
+	wg.Wait()
+}
+
 func main() {
 	// Create session
 	sess := session.Must(session.NewSessionWithOptions(session.Options{
@ -125,11 +160,8 @@ func main() {
 	// Create EC2 client
 	ec2Svc := ec2.New(sess, &aws.Config{Region: aws.String("us-east-1")})

-	// Create Service Quotas client
-	sqSvc := servicequotas.New(sess)
-
 	// Get the quota code for Public AMIs once
-	quotaCode := getQuotaCode(sqSvc)
+	quotaCode := getQuotaCode(sess)

 	// Get all regions
 	regions, err := getRegions(ec2Svc)
@ -142,5 +174,5 @@ func main() {
 	listQuotas(sess, quotaCode, regions)

 	// Request quota increase for all regions
-	// requestQuotaIncrease(sqSvc, quotaCode, regions)
+	// requestQuotaIncrease(sess, quotaCode, regions)
 }
--- a/sync/import-snapshot-publish-aws.sh
+++ b/sync/import-snapshot-publish-aws.sh
@ -9,7 +9,6 @@ RLVER=$VERSION
 # shellcheck disable=SC2046,1091,1090
 source "$(dirname "${BASH_SOURCE[0]}")/common"

-
 usage() {
 	echo "usage: $0 VERSION ($0 8.7)"
 }
@ -18,7 +17,7 @@ aws() {
 	command aws --region us-east-1 --profile resf-ami --output text $@
 }

-DATE=$(date +%Y%m%d)
+DATE=${DATE:-$(date +%Y%m%d)}

 if [[ -z $VERSION ]]; then
 	usage
@ -51,7 +50,7 @@ convert() {
 BUCKET="s3://resf-prod-import-use1"

 write-container-file() {
-  printf '{"Description": "%s", "Format": "raw", "Url": "%s/%s"}\n' $name "$BUCKET/$DATE" $raw > $json
+	printf '{"Description": "%s", "Format": "raw", "Url": "%s/%s"}\n' $name "$BUCKET/$DATE" $raw >$json
 }

 download-convert-upload() {
@ -72,7 +71,6 @@ download-convert-upload() {
 	fi
 }

-
 begin-job() {
 	import_task_id="$(aws ec2 import-snapshot --disk-container "file://$PWD/$json" --query 'ImportTaskId')"
 	if [[ -z $import_task_id ]]; then
@ -96,11 +94,13 @@ register-image() {
 	name=$1
 	snapshot_id=$2

-  case $(awk -F'.' '{print $NF}'<<<"$name") in
+	case $(awk -F'.' '{print $NF}' <<<"$name") in
 	x86_64)
-      arch=x86_64;;
+		arch=x86_64
+		;;
 	aarch64)
-      arch=arm64;;
+		arch=arm64
+		;;
 	esac

 	ami_id=$(aws --query "ImageId" ec2 register-image --name "$name" --description "$name" --block-device-mappings DeviceName="/dev/sda1",Ebs={SnapshotId="$snapshot_id"} --root-device-name "/dev/sda1" --virtualization-type hvm --architecture $arch --ena-support)
@ -130,8 +130,8 @@ image-exists() {
 		# Skip empty results
 		return 1 #not found
 	fi
-  id=${res[0]//\"}
-  name=${res[@]/$id}
+	id=${res[0]//\"/}
+	name=${res[@]/$id/}
 	found_image_id=$id
 	return 0 # found
 }
@ -145,7 +145,7 @@ snapshot-exists() {
 		# Skip empty results
 		return 1 #not found
 	fi
-  id=${res[0]//\"}
+	id=${res[0]//\"/}
 	found_snapshot_id=$id
 	return 0 # found
 }
@ -161,7 +161,7 @@ for variant in "${TARGET_VARIANTS[@]}"; do
 	for arch in "${TARGET_ARCHES[@]}"; do
 		latest=$(printf "Rocky-%s-EC2-%s.latest.%s" "$VERSION" $variant $arch)
 		name=$(printf "Rocky-%s-EC2-%s-%s-%s.%s.%s" "$VERSION" $variant $REVISION $DATE $EPOCH $arch)
-    qcow=${latest}.qcow2
+		qcow=${name}.qcow2
 		raw=${name}.raw
 		json=${name}.json

@ -181,7 +181,6 @@ for variant in "${TARGET_VARIANTS[@]}"; do
 		echo "Downloading/converting artifacts for $name"
 		download-convert-upload

-
 		echo "Writing disk container json file"
 		write-container-file

@ -242,7 +241,7 @@ while ! $finished; do
 		tag-resources $ami_id "Key=Name,Value=$name"

 		if [[ -z $ami_id ]]; then
-      echo "AMI ID is null. continuing...";
+			echo "AMI ID is null. continuing..."
 			continue
 		fi

--- a/sync/oval.sh
+++ b/sync/oval.sh
@ -26,3 +26,4 @@ popd > /dev/null || exit 2
 log "Copying to staging directory $TEMP => $OUTPUT_DIR"
 sudo rsync -vrSHP "$TEMP/" "$OUTPUT_DIR"
 sudo chown -Rv 10004:10005 "$OUTPUT_DIR"
+
--- a/sync/propagate-image.sh
+++ b/sync/propagate-image.sh
@ -14,10 +14,11 @@ REGIONS=$(aws --profile resf-ami ec2 describe-regions \
 	--all-regions \
 	--query "Regions[].{Name:RegionName}" \
 	--output text | grep -vE "$source_region")
+REGIONS="ap-southeast-4"

 SOURCE_AMI_NAME=$(aws --profile resf-ami ec2 describe-images \
-   --region "$source_region" --image-ids "$source_ami" --query 'Images[0].Name'\
-   --output text )
+	--region "$source_region" --image-ids "$source_ami" --query 'Images[0].Name' \
+	--output text)

 # Enforce a name structure
 # Rocky-8-ec2-8.6-20220515.0.x86_64
@ -29,7 +30,7 @@ if [[ ! "${SOURCE_AMI_NAME}" =~ $pat ]]; then
 	exit 1
 fi

-function copy(){
+function copy() {
 	for region in $REGIONS; do
 		if find_image_by_name $region; then
 			echo "Found copy of $source_ami in $region - $found_image_id - Skipping"
@ -56,7 +57,7 @@ function copy(){
 	done
 }

-function change_privacy(){
+function change_privacy() {
 	local status="$1"
 	local launch_permission
 	case $status in
@ -68,6 +69,7 @@ function change_privacy(){
 		;;
 	esac
 	local finished=false
+	ami_ids[${source_region}]="${source_ami}"
 	while ! $finished; do
 		for region in "${!ami_ids[@]}"; do
 			image_id=${ami_ids[$region]}
@ -101,8 +103,8 @@ function change_privacy(){
 		done
 		if [[ ${#ami_ids[@]} -gt 0 ]]; then
 			echo -n "Sleeping for one minute... "
-      for (( i=0; i<60; i++ )); do
-        if [[ $((i%10)) -eq 0 ]]; then 
+			for ((i = 0; i < 60; i++)); do
+				if [[ $((i % 10)) -eq 0 ]]; then
 					echo -n "$i"
 				else
 					echo -n "."
@ -118,22 +120,22 @@ function change_privacy(){
 	echo "Completed!"
 }

-function find_image_by_name(){
+function find_image_by_name() {
 	# found_ami_ids[region]=ami_id
 	# ami-id "name"
 	local query="$(printf 'Images[?Name==`%s`]|[?Public==`true`].[ImageId,Name][]' "${SOURCE_AMI_NAME}")"
 	mapfile -t res < <(
 		aws --profile resf-ami ec2 describe-images --region $region --owners $RESF_AMI_ACCOUNT_ID \
-      --query "${query}" 2>/dev/null \
-      | jq -r '.|@sh'
+			--query "${query}" 2>/dev/null |
+			jq -r '.|@sh'
 	)
 	res=($res)
 	if [[ ${#res[@]} -eq 0 ]]; then
 		# Skip empty results
 		return 1 #not found
 	fi
-  id=${res[0]//\"}
-  name=${res[@]/$id}
+	id=${res[0]//\"/}
+	name=${res[@]/$id/}
 	# printf "Found public image: %s in %s with name '%s'\n" "$id" "$region" "${name//\"}"
 	found_image_id=$id
 	return 0 # found