Updated cdn-related commits for style and cleanup

-Skip Grube
This commit is contained in:
Skip Grube 2023-02-23 12:31:31 -05:00
parent 3237794071
commit 2cc3bcc3a3
No known key found for this signature in database
GPG Key ID: D391F8393BEA6D9C
2 changed files with 50 additions and 29 deletions

View File

@ -24,6 +24,7 @@ import (
"bytes" "bytes"
"fmt" "fmt"
"io/ioutil" "io/ioutil"
"log"
"net/http" "net/http"
"path/filepath" "path/filepath"
"sort" "sort"
@ -361,17 +362,14 @@ func (g *GitMode) WriteSource(pd *data.ProcessData, md *data.ModeData) error {
fileName := strings.Split(path, "/")[1] fileName := strings.Split(path, "/")[1]
// Feed our template info to ProcessUrl and transform to the real values: ( {{.Name}}, {{.Branch}}, {{.Hash}}, {{.Hashtype}}, {{.Filename}} ) // Feed our template info to ProcessUrl and transform to the real values: ( {{.Name}}, {{.Branch}}, {{.Hash}}, {{.Hashtype}}, {{.Filename}} )
url, err = ProcessUrl(pd.CdnUrl, md.Name, branchName, hash, hashType, fileName) url, hasTemplate := ProcessUrl(pd.CdnUrl, md.Name, branchName, hash, hashType, fileName)
if err != nil {
return fmt.Errorf("Could not process CDN URL template(s) in string ( {{ .Variable }} )")
}
var req *http.Request var req *http.Request
var resp *http.Response var resp *http.Response
// Download the --cdn-url given, but *only* if it contains template strings ( {{.Name}} , {{.Hash}} , etc. ) // Download the --cdn-url given, but *only* if it contains template strings ( {{.Name}} , {{.Hash}} , etc. )
// Otherwise we need to fall back to the traditional cdn-url patterns // Otherwise we need to fall back to the traditional cdn-url patterns
if strings.Contains(pd.CdnUrl, "{{") && strings.Contains(pd.CdnUrl, "}}") { if hasTemplate {
pd.Log.Printf("downloading %s", url) pd.Log.Printf("downloading %s", url)
req, err := http.NewRequest("GET", url, nil) req, err := http.NewRequest("GET", url, nil)
@ -488,7 +486,7 @@ func (g *GitMode) ImportName(pd *data.ProcessData, md *data.ModeData) string {
// Given a cdnUrl string as input, return same string, but with substituted // Given a cdnUrl string as input, return same string, but with substituted
// template values ( {{.Name}} , {{.Hash}}, {{.Filename}}, etc. ) // template values ( {{.Name}} , {{.Hash}}, {{.Filename}}, etc. )
func ProcessUrl(cdnUrl string, name string, branch string, hash string, hashtype string, filename string) (string, error) { func ProcessUrl(cdnUrl string, name string, branch string, hash string, hashtype string, filename string) (string, bool) {
// These 5 {{ .Value }} items are possible in our templated string: // These 5 {{ .Value }} items are possible in our templated string:
type Lookaside struct { type Lookaside struct {
@ -501,18 +499,18 @@ func ProcessUrl(cdnUrl string, name string, branch string, hash string, hashtype
tmpUrl := Lookaside{name, branch, hash, hashtype, filename} tmpUrl := Lookaside{name, branch, hash, hashtype, filename}
// If we run into trouble with our template parsing, we'll just return the cdnUrl, exactly as we found it
tmpl, err := template.New("").Parse(cdnUrl) tmpl, err := template.New("").Parse(cdnUrl)
if err != nil { if err != nil {
panic(err) return cdnUrl, false
} }
var result bytes.Buffer var result bytes.Buffer
err = tmpl.Execute(&result, tmpUrl) err = tmpl.Execute(&result, tmpUrl)
if err != nil { if err != nil {
panic(err) log.Fatalf("ERROR: Could not process CDN URL template(s) from URL string: %s\n", cdnUrl)
} }
return result.String(), nil return result.String(), true
} }

View File

@ -122,16 +122,46 @@ func gitlabify(str string) string {
// If we find one of these passed as --cdn (ex: "--cdn fedora"), then we override, and assign this URL to be our --cdn-url // If we find one of these passed as --cdn (ex: "--cdn fedora"), then we override, and assign this URL to be our --cdn-url
func StaticLookasides() []LookasidePath { func StaticLookasides() []LookasidePath {
centos := LookasidePath{Distro: "centos", Url: "https://git.centos.org/sources/{{.Name}}/{{.Branch}}/{{.Hash}}"} centos := LookasidePath{
centosStream := LookasidePath{Distro: "centos-stream", Url: "https://sources.stream.centos.org/sources/rpms/{{.Name}}/{{.Filename}}/{{.Hashtype}}/{{.Hash}}/{{.Filename}}"} Distro: "centos",
rocky8 := LookasidePath{Distro: "rocky8", Url: "https://rocky-linux-sources-staging.a1.rockylinux.org/{{.Hash}}"} Url: "https://git.centos.org/sources/{{.Name}}/{{.Branch}}/{{.Hash}}",
rocky := LookasidePath{Distro: "rocky", Url: "https://sources.build.resf.org/{{.Hash}}"} }
fedora := LookasidePath{Distro: "fedora", Url: "https://src.fedoraproject.org/repo/pkgs/{{.Name}}/{{.Filename}}/{{.Hashtype}}/{{.Hash}}/{{.Filename}}"} centosStream := LookasidePath{
Distro: "centos-stream",
Url: "https://sources.stream.centos.org/sources/rpms/{{.Name}}/{{.Filename}}/{{.Hashtype}}/{{.Hash}}/{{.Filename}}",
}
rocky8 := LookasidePath{
Distro: "rocky8",
Url: "https://rocky-linux-sources-staging.a1.rockylinux.org/{{.Hash}}",
}
rocky := LookasidePath{
Distro: "rocky",
Url: "https://sources.build.resf.org/{{.Hash}}",
}
fedora := LookasidePath{
Distro: "fedora",
Url: "https://src.fedoraproject.org/repo/pkgs/{{.Name}}/{{.Filename}}/{{.Hashtype}}/{{.Hash}}/{{.Filename}}",
}
return []LookasidePath{centos, centosStream, rocky8, rocky, fedora} return []LookasidePath{centos, centosStream, rocky8, rocky, fedora}
} }
// Given a "--cdn" entry like "centos", we can search through our struct list of distros, and return the proper lookaside URL
// If we can't find it, we return false and the calling function will error out
func FindDistro(cdn string) (string, bool) {
var cdnUrl = ""
// Loop through each distro in the static list defined, try to find a match with "--cdn":
for _, distro := range StaticLookasides() {
if distro.Distro == strings.ToLower(cdn) {
cdnUrl = distro.Url
return cdnUrl, true
}
}
return "", false
}
func NewProcessData(req *ProcessDataRequest) (*data.ProcessData, error) { func NewProcessData(req *ProcessDataRequest) (*data.ProcessData, error) {
// Build the logger to use for the data import // Build the logger to use for the data import
@ -170,25 +200,18 @@ func NewProcessData(req *ProcessDataRequest) (*data.ProcessData, error) {
req.CdnUrl = "https://git.centos.org/sources" req.CdnUrl = "https://git.centos.org/sources"
} }
// If a Cdn distro is defined, loop through StaticLookasides() array of structs, // If a Cdn distro is defined, we try to find a match from StaticLookasides() array of structs
// see if we have a match to --cdn (matching values are things like fedora, centos, rocky8, etc.) // see if we have a match to --cdn (matching values are things like fedora, centos, rocky8, etc.)
// If we match, then we want to short-circuit the CdnUrl to the assigned distro's one // If we match, then we want to short-circuit the CdnUrl to the assigned distro's one
if req.Cdn != "" { if req.Cdn != "" {
newCdn, foundDistro := FindDistro(req.Cdn)
var foundDistro = false if !foundDistro {
for _, distro := range StaticLookasides() {
if distro.Distro == strings.ToLower(req.Cdn) {
foundDistro = true
req.CdnUrl = distro.Url
logger.Printf("Discovered --cdn distro: %s . Using override CDN URL Pattern: %s", distro.Distro, req.CdnUrl)
break
}
}
if foundDistro == false {
return nil, fmt.Errorf("Error, distro name given as --cdn argument is not valid.") return nil, fmt.Errorf("Error, distro name given as --cdn argument is not valid.")
} }
req.CdnUrl = newCdn
logger.Printf("Discovered --cdn distro: %s . Using override CDN URL Pattern: %s", req.Cdn, req.CdnUrl)
} }
// Validate required // Validate required
@ -406,7 +429,7 @@ func ProcessRPM(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error) {
} }
// If we have no valid branches to consider, then we'll automatically switch to attempt a tagless import: // If we have no valid branches to consider, then we'll automatically switch to attempt a tagless import:
if len(md.Branches) <= 0 { if len(md.Branches) == 0 {
log.Println("No valid tags (refs/tags/imports/*) found in repository! Switching to perform a tagless import.") log.Println("No valid tags (refs/tags/imports/*) found in repository! Switching to perform a tagless import.")
pd.TaglessMode = true pd.TaglessMode = true
result, err := processRPMTagless(pd) result, err := processRPMTagless(pd)