formatting, add WSL container variant, support requesting new quotas

This commit is contained in:
Neil Hanlon 2023-05-12 14:00:52 -04:00
parent 173b0ff814
commit ed2a2999ad
Signed by untrusted user: neil
GPG Key ID: 705BC21EC3C70F34
7 changed files with 327 additions and 293 deletions

View File

@ -104,7 +104,7 @@ for conf in glob.iglob(f"{_rootdir}/sig/*.yaml"):
ALLOWED_TYPE_VARIANTS = {
"Azure": ["Base", "LVM"],
"Container": ["Base", "Minimal", "UBI"],
"Container": ["Base", "Minimal", "UBI", "WSL"],
"EC2": ["Base", "LVM"],
"GenericCloud": ["Base", "LVM"],
"Vagrant": ["Libvirt", "Vbox", "VMware"],

View File

@ -88,7 +88,7 @@
primary_variant: 'Base'
Container:
format: tar.xz
variants: [Base, Minimal, UBI]
variants: [Base, Minimal, UBI, WSL]
RPI:
format: raw.xz
OCP:

View File

@ -2,7 +2,7 @@
apiVersion: batch/v1
kind: Job
metadata:
name: {{ jobname }}-{{ major }}-{{ architecture }}
name: {{ jobname }}-{{ major }}-{{ architecture }}-{{ buildTime }}
namespace: {{ namespace }}
spec:
template:

View File

@ -7,12 +7,14 @@ import (
"sync"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/aws/aws-sdk-go/service/servicequotas"
)
func getQuotaCode(sqSvc *servicequotas.ServiceQuotas) string {
func getQuotaCode(sess *session.Session) string {
sqSvc := servicequotas.New(sess)
input := &servicequotas.ListServiceQuotasInput{
ServiceCode: aws.String("ec2"),
}
@ -71,9 +73,13 @@ func getQuotaInfo(sqSvc *servicequotas.ServiceQuotas, quotaCode string, region s
output, err := sqSvc.GetServiceQuota(input)
if err != nil {
log.Printf("Error getting quota info for %s: %s\n", region, err)
if awsErr, ok := err.(awserr.Error); ok {
if message := awsErr.Code(); message == "UnknownOperationException" {
log.Printf("[sdk] Region %s does not appear to support Service Quotas: %v", region, message)
return nil
// os.Exit(1)
}
log.Fatalf("[sdk] Error getting quota info for %s: %v\n", region, awsErr)
}
}
currentValue := *output.Quota.Value
@ -116,6 +122,35 @@ func listQuotas(sess *session.Session, quotaCode string, regions []*string) {
wg.Wait()
}
func requestQuotaIncrease(sess *session.Session, quotaCode string, regions []string, quota float64) {
var wg sync.WaitGroup
wg.Add(len(regions))
for _, region := range regions {
go func(region string) {
defer wg.Done()
regionSqSvc := servicequotas.New(sess, &aws.Config{Region: aws.String(region)})
quotaInfo := getQuotaInfo(regionSqSvc, quotaCode, region)
if quotaInfo.CurrentQuota >= quota {
fmt.Printf("Quota for Public AMIs in region %s is already set to %.0f, skipping request.\n", region, quotaInfo.CurrentQuota)
} else {
input := &servicequotas.RequestServiceQuotaIncreaseInput{
ServiceCode: aws.String("ec2"),
QuotaCode: aws.String(quotaCode),
DesiredValue: aws.Float64(quota),
}
output, err := regionSqSvc.RequestServiceQuotaIncrease(input)
if err != nil {
fmt.Println("Error requesting quota increase:", err)
os.Exit(1)
}
fmt.Printf("Successfully submitted request with ID: %s\n", aws.StringValue(output.RequestedQuota.Id))
}
}(region)
}
wg.Wait()
}
func main() {
// Create session
sess := session.Must(session.NewSessionWithOptions(session.Options{
@ -125,11 +160,8 @@ func main() {
// Create EC2 client
ec2Svc := ec2.New(sess, &aws.Config{Region: aws.String("us-east-1")})
// Create Service Quotas client
sqSvc := servicequotas.New(sess)
// Get the quota code for Public AMIs once
quotaCode := getQuotaCode(sqSvc)
quotaCode := getQuotaCode(sess)
// Get all regions
regions, err := getRegions(ec2Svc)
@ -142,5 +174,5 @@ func main() {
listQuotas(sess, quotaCode, regions)
// Request quota increase for all regions
// requestQuotaIncrease(sqSvc, quotaCode, regions)
// requestQuotaIncrease(sess, quotaCode, regions)
}

View File

@ -9,7 +9,6 @@ RLVER=$VERSION
# shellcheck disable=SC2046,1091,1090
source "$(dirname "${BASH_SOURCE[0]}")/common"
usage() {
echo "usage: $0 VERSION ($0 8.7)"
}
@ -18,7 +17,7 @@ aws() {
command aws --region us-east-1 --profile resf-ami --output text $@
}
DATE=$(date +%Y%m%d)
DATE=${DATE:-$(date +%Y%m%d)}
if [[ -z $VERSION ]]; then
usage
@ -51,7 +50,7 @@ convert() {
BUCKET="s3://resf-prod-import-use1"
write-container-file() {
printf '{"Description": "%s", "Format": "raw", "Url": "%s/%s"}\n' $name "$BUCKET/$DATE" $raw > $json
printf '{"Description": "%s", "Format": "raw", "Url": "%s/%s"}\n' $name "$BUCKET/$DATE" $raw >$json
}
download-convert-upload() {
@ -72,7 +71,6 @@ download-convert-upload() {
fi
}
begin-job() {
import_task_id="$(aws ec2 import-snapshot --disk-container "file://$PWD/$json" --query 'ImportTaskId')"
if [[ -z $import_task_id ]]; then
@ -96,11 +94,13 @@ register-image() {
name=$1
snapshot_id=$2
case $(awk -F'.' '{print $NF}'<<<"$name") in
case $(awk -F'.' '{print $NF}' <<<"$name") in
x86_64)
arch=x86_64;;
arch=x86_64
;;
aarch64)
arch=arm64;;
arch=arm64
;;
esac
ami_id=$(aws --query "ImageId" ec2 register-image --name "$name" --description "$name" --block-device-mappings DeviceName="/dev/sda1",Ebs={SnapshotId="$snapshot_id"} --root-device-name "/dev/sda1" --virtualization-type hvm --architecture $arch --ena-support)
@ -130,8 +130,8 @@ image-exists() {
# Skip empty results
return 1 #not found
fi
id=${res[0]//\"}
name=${res[@]/$id}
id=${res[0]//\"/}
name=${res[@]/$id/}
found_image_id=$id
return 0 # found
}
@ -145,7 +145,7 @@ snapshot-exists() {
# Skip empty results
return 1 #not found
fi
id=${res[0]//\"}
id=${res[0]//\"/}
found_snapshot_id=$id
return 0 # found
}
@ -161,7 +161,7 @@ for variant in "${TARGET_VARIANTS[@]}"; do
for arch in "${TARGET_ARCHES[@]}"; do
latest=$(printf "Rocky-%s-EC2-%s.latest.%s" "$VERSION" $variant $arch)
name=$(printf "Rocky-%s-EC2-%s-%s-%s.%s.%s" "$VERSION" $variant $REVISION $DATE $EPOCH $arch)
qcow=${latest}.qcow2
qcow=${name}.qcow2
raw=${name}.raw
json=${name}.json
@ -181,7 +181,6 @@ for variant in "${TARGET_VARIANTS[@]}"; do
echo "Downloading/converting artifacts for $name"
download-convert-upload
echo "Writing disk container json file"
write-container-file
@ -242,7 +241,7 @@ while ! $finished; do
tag-resources $ami_id "Key=Name,Value=$name"
if [[ -z $ami_id ]]; then
echo "AMI ID is null. continuing...";
echo "AMI ID is null. continuing..."
continue
fi

View File

@ -26,3 +26,4 @@ popd > /dev/null || exit 2
log "Copying to staging directory $TEMP => $OUTPUT_DIR"
sudo rsync -vrSHP "$TEMP/" "$OUTPUT_DIR"
sudo chown -Rv 10004:10005 "$OUTPUT_DIR"

View File

@ -14,10 +14,11 @@ REGIONS=$(aws --profile resf-ami ec2 describe-regions \
--all-regions \
--query "Regions[].{Name:RegionName}" \
--output text | grep -vE "$source_region")
REGIONS="ap-southeast-4"
SOURCE_AMI_NAME=$(aws --profile resf-ami ec2 describe-images \
--region "$source_region" --image-ids "$source_ami" --query 'Images[0].Name'\
--output text )
--region "$source_region" --image-ids "$source_ami" --query 'Images[0].Name' \
--output text)
# Enforce a name structure
# Rocky-8-ec2-8.6-20220515.0.x86_64
@ -29,7 +30,7 @@ if [[ ! "${SOURCE_AMI_NAME}" =~ $pat ]]; then
exit 1
fi
function copy(){
function copy() {
for region in $REGIONS; do
if find_image_by_name $region; then
echo "Found copy of $source_ami in $region - $found_image_id - Skipping"
@ -56,7 +57,7 @@ function copy(){
done
}
function change_privacy(){
function change_privacy() {
local status="$1"
local launch_permission
case $status in
@ -68,6 +69,7 @@ function change_privacy(){
;;
esac
local finished=false
ami_ids[${source_region}]="${source_ami}"
while ! $finished; do
for region in "${!ami_ids[@]}"; do
image_id=${ami_ids[$region]}
@ -101,8 +103,8 @@ function change_privacy(){
done
if [[ ${#ami_ids[@]} -gt 0 ]]; then
echo -n "Sleeping for one minute... "
for (( i=0; i<60; i++ )); do
if [[ $((i%10)) -eq 0 ]]; then
for ((i = 0; i < 60; i++)); do
if [[ $((i % 10)) -eq 0 ]]; then
echo -n "$i"
else
echo -n "."
@ -118,22 +120,22 @@ function change_privacy(){
echo "Completed!"
}
function find_image_by_name(){
function find_image_by_name() {
# found_ami_ids[region]=ami_id
# ami-id "name"
local query="$(printf 'Images[?Name==`%s`]|[?Public==`true`].[ImageId,Name][]' "${SOURCE_AMI_NAME}")"
mapfile -t res < <(
aws --profile resf-ami ec2 describe-images --region $region --owners $RESF_AMI_ACCOUNT_ID \
--query "${query}" 2>/dev/null \
| jq -r '.|@sh'
--query "${query}" 2>/dev/null |
jq -r '.|@sh'
)
res=($res)
if [[ ${#res[@]} -eq 0 ]]; then
# Skip empty results
return 1 #not found
fi
id=${res[0]//\"}
name=${res[@]/$id}
id=${res[0]//\"/}
name=${res[@]/$id/}
# printf "Found public image: %s in %s with name '%s'\n" "$id" "$region" "${name//\"}"
found_image_id=$id
return 0 # found