From f152ff5a441939c4eab63c67a8cf40cdbbf88f8b Mon Sep 17 00:00:00 2001 From: Neil Hanlon Date: Fri, 12 May 2023 14:02:39 -0400 Subject: [PATCH] Add golang stubs for scripts to maintain image versions --- sync/latest-images/main.go | 161 +++++++++++++++++++++++++++++++++++++ sync/latest-isos/main.go | 144 +++++++++++++++++++++++++++++++++ 2 files changed, 305 insertions(+) create mode 100644 sync/latest-images/main.go create mode 100644 sync/latest-isos/main.go diff --git a/sync/latest-images/main.go b/sync/latest-images/main.go new file mode 100644 index 0000000..4366ca0 --- /dev/null +++ b/sync/latest-images/main.go @@ -0,0 +1,161 @@ +package main + +import ( + "fmt" + "log" + "regexp" + "strconv" + "strings" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/spf13/cobra" +) + +type Image struct { + Type string + Architecture string + Variant string + Version string + Date time.Time + File string + FilePath string +} + +func FindRecentImages(bucketName string, prefix string, version string, imageType string, variant string, date string, fileType string) ([]Image, error) { + s3session := session.Must(session.NewSession()) + svc := s3.New(s3session, &aws.Config{Region: aws.String("us-east-2")}) + + result := []Image{} + + parts := strings.Split(version, ".") + major := parts[0] + minor := parts[1] + + if prefix == "" { + prefix = fmt.Sprintf("buildimage-%s.%s-", major, minor) + } + + // Build the S3 key prefix for the given parameters + // @todo: support passing the other parameters + keyPrefix := fmt.Sprintf("%s", + prefix) + + log.Printf("Looking for images in the following format: %s", keyPrefix) + + var items []*s3.Object + + // List the objects in the S3 bucket with the given prefix + input := &s3.ListObjectsV2Input{ + Bucket: aws.String(bucketName), + Prefix: aws.String(keyPrefix), + } + err := svc.ListObjectsV2Pages(input, func(page *s3.ListObjectsV2Output, lastPage bool) bool { + items = append(items, page.Contents...) + return true + }) + + log.Printf("Found %d images in our search", len(items)) + + if err != nil { + log.Fatalf("uh oh: %v", err) + } + + pattern := regexp.MustCompile(`(?PRocky-(?P[0-9]+)-(?P\w+)(?:-(?P\w+))?-(?:[0-9]+)\.(?P[0-9])-(?P[0-9]+)\.(?P[0-9]+)\.(?P\w+)/(?P[0-9]+)/(?P(.+\.(?P(box|qcow2|raw|tar\.xz|vhd)))))$`) + // Loop through the objects and find the latest one for each file type + latestByTypeVariant := map[string]map[string]*Image{} + for _, obj := range items { + key := *obj.Key + + match := pattern.FindStringSubmatch(key) + + if len(match) == 0 { + // log.Printf("key did not match pattern, %s : %s\n", key, pattern) + continue + } + + date, err := strconv.ParseInt(match[pattern.SubexpIndex("datestamp")], 10, 64) + if err != nil { + log.Fatalf("uh oh dates are fun! %v", err) + } + imageType := match[pattern.SubexpIndex("type")] + variant := match[pattern.SubexpIndex("variant")] + + image := Image{ + Type: imageType, + Variant: variant, + Version: match[pattern.SubexpIndex("major")], + // FilePath: match[pattern.SubexpIndex("whole")], + File: match[pattern.SubexpIndex("file")], + Architecture: match[pattern.SubexpIndex("architecture")], + Date: time.Unix(date, 0), + } + + typeVariant := image.Type + if image.Variant != "" { + typeVariant = fmt.Sprintf("%s-%s", typeVariant, image.Variant) + } + + // Early images of these two types were named differently.. Skip them + if typeVariant == "GenericCloud" || typeVariant == "EC2" { + continue + } + + latest, ok := latestByTypeVariant[typeVariant][image.Architecture] + + if !ok || image.Date.After(latest.Date) { + if latestByTypeVariant[typeVariant] == nil { + latestByTypeVariant[typeVariant] = map[string]*Image{} + } + latestByTypeVariant[typeVariant][image.Architecture] = &image + } + } + + // Convert the map to an array + for _, architecture := range latestByTypeVariant { + for _, image := range architecture { + result = append(result, *image) + } + } + + return result, nil +} + +func main() { + var bucket, prefix, imageType, variant, fileType, date string + + // Set up the root command with flags + var rootCmd = &cobra.Command{ + Use: "find-latest-images VERSION", + Short: "Lists the most recent S3 images based on certain criteria", + Long: `Lists the most recent S3 images based on the image version, type, variant, and date`, + Args: cobra.ExactArgs(1), + Run: func(cmd *cobra.Command, args []string) { + // version is the only positional arg. + version := args[0] + // Call the function to list the S3 images based on the provided criteria + res, err := FindRecentImages(bucket, prefix, version, imageType, variant, date, fileType) + if err != nil { + fmt.Println(fmt.Errorf("IDK yo: %v", err)) + } + for _, image := range res { + fmt.Println(image) + } + }, + } + + // Set up the command flags + rootCmd.Flags().StringVar(&bucket, "bucket", "resf-empanadas", "The name of the S3 bucket") + rootCmd.Flags().StringVar(&prefix, "prefix", "", "The prefix for the S3 key") + rootCmd.Flags().StringVar(&imageType, "type", "", "The image type") + rootCmd.Flags().StringVar(&variant, "variant", "", "The image variant") + rootCmd.Flags().StringVar(&fileType, "file-type", "", "The image file type") + rootCmd.Flags().StringVar(&date, "date", time.Now().Format("20060102"), "The date of the images to list (in YYYYMMDD format)") + + // Set up the CLI + if err := rootCmd.Execute(); err != nil { + fmt.Println("Error:", err) + } +} diff --git a/sync/latest-isos/main.go b/sync/latest-isos/main.go new file mode 100644 index 0000000..d13a56b --- /dev/null +++ b/sync/latest-isos/main.go @@ -0,0 +1,144 @@ +package main + +import ( + "fmt" + "log" + "regexp" + "strconv" + "strings" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/spf13/cobra" +) + +type LoraxResult struct { + Architecture string + Major string + Minor string + Date time.Time + File string + FilePath string +} + +func FindRecentISOs(bucketName string, prefix string, version string) ([]LoraxResult, error) { + s3session := session.Must(session.NewSession()) + svc := s3.New(s3session, &aws.Config{Region: aws.String("us-east-2")}) + + result := []LoraxResult{} + + parts := strings.Split(version, ".") + major := parts[0] + // minor := parts[1] + + if prefix == "" { + prefix = fmt.Sprintf("buildiso-%s-", major) + } + + // Build the S3 key prefix for the given parameters + // @todo: support passing the other parameters + keyPrefix := fmt.Sprintf("%s", prefix) + + log.Printf("Looking for isos in the following format: %s", keyPrefix) + + var items []*s3.Object + + // List the objects in the S3 bucket with the given prefix + input := &s3.ListObjectsV2Input{ + Bucket: aws.String(bucketName), + Prefix: aws.String(keyPrefix), + } + err := svc.ListObjectsV2Pages(input, func(page *s3.ListObjectsV2Output, lastPage bool) bool { + items = append(items, page.Contents...) + return true + }) + + log.Printf("Found %d matches for prefix", len(items)) + + if err != nil { + log.Fatalf("uh oh: %v", err) + } + + pattern := regexp.MustCompile(`(?Pbuildiso-(\d+)-(\w+)/(?P[0-9]+)/(?Plorax-(?P\d+)\.(?P\d)-(?P\w+).tar.gz))$`) + // Loop through the objects and find the latest one for each file type + latestISOs := map[string]*LoraxResult{} + for _, obj := range items { + key := *obj.Key + + match := pattern.FindStringSubmatch(key) + + if len(match) == 0 { + // log.Printf("key did not match pattern, %s : %s\n", key, pattern) + continue + } + // log.Printf("found a match: %s", key) + + date, err := strconv.ParseInt(match[pattern.SubexpIndex("datestamp")], 10, 64) + if err != nil { + log.Fatalf("uh oh dates are fun! %v", err) + } + + iso := LoraxResult{ + Architecture: match[pattern.SubexpIndex("architecture")], + Major: match[pattern.SubexpIndex("major")], + Minor: match[pattern.SubexpIndex("minor")], + FilePath: match[pattern.SubexpIndex("whole")], + File: match[pattern.SubexpIndex("file")], + Date: time.Unix(date, 0), + } + // log.Printf("iso is: %v", iso) + + latest, ok := latestISOs[iso.Architecture] + + // log.Printf("latest, ok: %v, %t", latest, ok) + + if !ok || iso.Date.After(latest.Date) { + latestISOs[iso.Architecture] = &iso + } + } + + for _, iso := range latestISOs { + result = append(result, *iso) + } + + return result, nil +} + +func main() { + var bucket, prefix, imageType, variant, fileType, date string + + // Set up the root command with flags + var rootCmd = &cobra.Command{ + Use: "find-latest-isos VERSION", + Short: "Lists the most recent S3 images based on certain criteria", + Long: `Lists the most recent S3 images based on the image version, type, variant, and date`, + Args: cobra.ExactArgs(1), + Run: func(cmd *cobra.Command, args []string) { + // version is the only positional arg. + version := args[0] + // Call the function to list the S3 images based on the provided criteria + res, err := FindRecentISOs(bucket, prefix, version) + if err != nil { + fmt.Println(fmt.Errorf("IDK yo: %v", err)) + } + for _, image := range res { + fmt.Println(image) + } + }, + } + + // Set up the command flags + rootCmd.Flags().StringVar(&bucket, "bucket", "resf-empanadas", "The name of the S3 bucket") + rootCmd.Flags().StringVar(&prefix, "prefix", "", "The prefix for the S3 key") + rootCmd.Flags().StringVar(&imageType, "type", "", "The image type") + rootCmd.Flags().StringVar(&variant, "variant", "", "The image variant") + rootCmd.Flags().StringVar(&fileType, "file-type", "", "The image file type") + rootCmd.Flags().StringVar(&date, "date", time.Now().Format("20060102"), "The date of the images to list (in YYYYMMDD format)") + + // Set up the CLI + if err := rootCmd.Execute(); err != nil { + fmt.Println("Error:", err) + } +}