From b3c86f82215f5eb25cf80047407fc562724d7552 Mon Sep 17 00:00:00 2001 From: Neil Hanlon Date: Mon, 11 Mar 2024 11:47:01 -0400 Subject: [PATCH 1/2] feat: allow rpm import to skip upload OR import step n.b. skipping both steps can be accomplished by not running the command :) --- peridot/cmd/v1/peridot/BUILD.bazel | 2 +- peridot/cmd/v1/peridot/build_rpm_import.go | 48 +++++++++++++++++----- 2 files changed, 38 insertions(+), 12 deletions(-) diff --git a/peridot/cmd/v1/peridot/BUILD.bazel b/peridot/cmd/v1/peridot/BUILD.bazel index 070f206b..08220ce4 100644 --- a/peridot/cmd/v1/peridot/BUILD.bazel +++ b/peridot/cmd/v1/peridot/BUILD.bazel @@ -64,7 +64,7 @@ pkg_rpm( srcs = [":peridot-files"], license = "MIT", summary = "Peridot Command Line Interface", - version = "0.2.0", + version = "0.2.1", release = "0", architecture = "x86_64", description = "A command line interface to interact with the Peridot build system", diff --git a/peridot/cmd/v1/peridot/build_rpm_import.go b/peridot/cmd/v1/peridot/build_rpm_import.go index e894ba9f..fe715f43 100644 --- a/peridot/cmd/v1/peridot/build_rpm_import.go +++ b/peridot/cmd/v1/peridot/build_rpm_import.go @@ -32,12 +32,14 @@ package main import ( "encoding/base64" - "github.com/spf13/cobra" - "io/ioutil" "log" - "openapi.peridot.resf.org/peridotopenapi" "os" + "strings" "time" + "crypto/sha256" + "encoding/hex" + "github.com/spf13/cobra" + "openapi.peridot.resf.org/peridotopenapi" ) type LookasideUploadTask struct { @@ -57,9 +59,11 @@ var buildRpmImport = &cobra.Command{ } var buildRpmImportForceOverride bool +var skipStep string func init() { buildRpmImport.Flags().BoolVar(&buildRpmImportForceOverride, "force-override", true, "Force override even if version exists (default: true)") + buildRpmImport.Flags().StringVarP(&skipStep, "skip", "s", "", "which step to skip") } func isFile(path string) bool { @@ -70,10 +74,24 @@ func isFile(path string) bool { return true } -func buildRpmImportMn(_ *cobra.Command, args []string) { +func buildRpmImportMn(c *cobra.Command, args []string) { // Ensure project id exists projectId := mustGetProjectID() + var skipUpload bool = false + var skipImport bool = false + + if skipStep != "" { + switch strings.ToLower(skipStep) { + case "upload": + skipUpload = true + case "import": + skipImport = true + default: + log.Fatalf("invalid skip step: %s", skipStep) + } + } + // Ensure all args are valid files for _, arg := range args { if !isFile(arg) { @@ -85,16 +103,24 @@ func buildRpmImportMn(_ *cobra.Command, args []string) { var blobs []string projectCl := getClient(serviceProject).(peridotopenapi.ProjectServiceApi) for _, arg := range args { - bts, err := ioutil.ReadFile(arg) + bts, err := os.ReadFile(arg) errFatal(err) base64EncodedBytes := base64.StdEncoding.EncodeToString(bts) + hash := sha256.Sum256(bts) + shasum := hex.EncodeToString(hash[:]) - res, _, err := projectCl.LookasideFileUpload(getContext()).Body(peridotopenapi.V1LookasideFileUploadRequest{ - File: &base64EncodedBytes, - }).Execute() - errFatal(err) - log.Printf("Uploaded %s to lookaside", arg) - blobs = append(blobs, res.GetDigest()) + if !skipUpload { + _, _, err := projectCl.LookasideFileUpload(getContext()).Body(peridotopenapi.V1LookasideFileUploadRequest{ + File: &base64EncodedBytes, + }).Execute() + errFatal(err) + log.Printf("Uploaded %s to lookaside", arg) + } + blobs = append(blobs, shasum) + } + + if skipImport { + return } taskCl := getClient(serviceTask).(peridotopenapi.TaskServiceApi) From 3c00db9e36424efac62d85ad3273e83f3b6d1116 Mon Sep 17 00:00:00 2001 From: Neil Hanlon Date: Mon, 11 Mar 2024 12:02:13 -0400 Subject: [PATCH 2/2] fix: don't encode bytes if we don't have to --- peridot/cmd/v1/peridot/BUILD.bazel | 2 +- peridot/cmd/v1/peridot/build_rpm_import.go | 16 ++++++++++------ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/peridot/cmd/v1/peridot/BUILD.bazel b/peridot/cmd/v1/peridot/BUILD.bazel index 08220ce4..d91c4915 100644 --- a/peridot/cmd/v1/peridot/BUILD.bazel +++ b/peridot/cmd/v1/peridot/BUILD.bazel @@ -65,7 +65,7 @@ pkg_rpm( license = "MIT", summary = "Peridot Command Line Interface", version = "0.2.1", - release = "0", + release = "1", architecture = "x86_64", description = "A command line interface to interact with the Peridot build system", source_date_epoch = 0, diff --git a/peridot/cmd/v1/peridot/build_rpm_import.go b/peridot/cmd/v1/peridot/build_rpm_import.go index fe715f43..307ce1ff 100644 --- a/peridot/cmd/v1/peridot/build_rpm_import.go +++ b/peridot/cmd/v1/peridot/build_rpm_import.go @@ -31,13 +31,14 @@ package main import ( + "crypto/sha256" "encoding/base64" + "encoding/hex" "log" "os" "strings" "time" - "crypto/sha256" - "encoding/hex" + "github.com/spf13/cobra" "openapi.peridot.resf.org/peridotopenapi" ) @@ -74,12 +75,12 @@ func isFile(path string) bool { return true } -func buildRpmImportMn(c *cobra.Command, args []string) { +func buildRpmImportMn(_ *cobra.Command, args []string) { // Ensure project id exists projectId := mustGetProjectID() - var skipUpload bool = false - var skipImport bool = false + var skipUpload = false + var skipImport = false if skipStep != "" { switch strings.ToLower(skipStep) { @@ -103,19 +104,22 @@ func buildRpmImportMn(c *cobra.Command, args []string) { var blobs []string projectCl := getClient(serviceProject).(peridotopenapi.ProjectServiceApi) for _, arg := range args { + bts, err := os.ReadFile(arg) errFatal(err) - base64EncodedBytes := base64.StdEncoding.EncodeToString(bts) + hash := sha256.Sum256(bts) shasum := hex.EncodeToString(hash[:]) if !skipUpload { + base64EncodedBytes := base64.StdEncoding.EncodeToString(bts) _, _, err := projectCl.LookasideFileUpload(getContext()).Body(peridotopenapi.V1LookasideFileUploadRequest{ File: &base64EncodedBytes, }).Execute() errFatal(err) log.Printf("Uploaded %s to lookaside", arg) } + log.Printf("Will upload %s to lookaside for %s", shasum, arg) blobs = append(blobs, shasum) }