enhancement: add support for downloading from blob storage instead of http endpoint

cmd/srpmproc/fetch.go

@@ -37,6 +37,8 @@ var cdnUrl string
 func init() {
 	fetch.Flags().StringVar(&cdnUrl, "cdn-url", "", "Path to CDN")
 	_ = fetch.MarkFlagRequired("cdn-url")
+
+	root.AddCommand(fetch)
 }
 
 func runFetch(_ *cobra.Command, _ []string) {
@@ -45,12 +47,8 @@ func runFetch(_ *cobra.Command, _ []string) {
 		log.Fatalf("could not get working directory: %v", err)
 	}
 
-	err = srpmproc.Fetch(os.Stdout, cdnUrl, wd)
+	err = srpmproc.Fetch(os.Stdout, cdnUrl, wd, nil)
 	if err != nil {
 		log.Fatal(err)
 	}
 }
-
-func init() {
-	root.AddCommand(fetch)
-}

pkg/srpmproc/fetch.go

@@ -3,6 +3,7 @@ package srpmproc
 import (
 	"errors"
 	"fmt"
+	"github.com/rocky-linux/srpmproc/pkg/blob"
 	"github.com/rocky-linux/srpmproc/pkg/data"
 	"io"
 	"io/ioutil"
@@ -13,7 +14,7 @@ import (
 	"strings"
 )
 
-func Fetch(logger io.Writer, cdnUrl string, dir string) error {
+func Fetch(logger io.Writer, cdnUrl string, dir string, storage blob.Storage) error {
 	pd := &data.ProcessData{
 		Log: log.New(logger, "", log.LstdFlags),
 	}
@@ -58,26 +59,38 @@ func Fetch(logger io.Writer, cdnUrl string, dir string) error {
 		path := lineInfo[1]
 
 		url := fmt.Sprintf("%s/%s", cdnUrl, hash)
+		if storage != nil {
+			url = hash
+		}
 		pd.Log.Printf("downloading %s", url)
 
-		req, err := http.NewRequest("GET", url, nil)
+		var body []byte
-		if err != nil {
-			return fmt.Errorf("could not create new http request: %v", err)
-		}
-		req.Header.Set("Accept-Encoding", "*")
 
-		resp, err := client.Do(req)
+		if storage != nil {
-		if err != nil {
+			body, err = storage.Read(hash)
-			return fmt.Errorf("could not download dist-git file: %v", err)
+			if err != nil {
-		}
+				return fmt.Errorf("could not read blob: %v", err)
+			}
+		} else {
+			req, err := http.NewRequest("GET", url, nil)
+			if err != nil {
+				return fmt.Errorf("could not create new http request: %v", err)
+			}
+			req.Header.Set("Accept-Encoding", "*")
 
-		body, err := ioutil.ReadAll(resp.Body)
+			resp, err := client.Do(req)
-		if err != nil {
+			if err != nil {
-			return fmt.Errorf("could not read the whole dist-git file: %v", err)
+				return fmt.Errorf("could not download dist-git file: %v", err)
-		}
+			}
-		err = resp.Body.Close()
+
-		if err != nil {
+			body, err = ioutil.ReadAll(resp.Body)
-			return fmt.Errorf("could not close body handle: %v", err)
+			if err != nil {
+				return fmt.Errorf("could not read the whole dist-git file: %v", err)
+			}
+			err = resp.Body.Close()
+			if err != nil {
+				return fmt.Errorf("could not close body handle: %v", err)
+			}
 		}
 
 		hasher := pd.CompareHash(body, hash)