Add commit pinning

cmd/srpmproc/main.go

@@ -40,6 +40,8 @@ var (
 	tmpFsMode           string
 	noStorageDownload   bool
 	noStorageUpload     bool
+	manualCommits       string
+	upstreamPrefixHttps string
 )
 
 var root = &cobra.Command{
@@ -111,6 +113,11 @@ func mn(_ *cobra.Command, _ []string) {
 		}
 	}
 
+	var manualCs []string
+	if strings.TrimSpace(manualCommits) != "" {
+		manualCs = strings.Split(manualCommits, ",")
+	}
+
 	internal.ProcessRPM(&data.ProcessData{
 		Importer:            importer,
 		RpmLocation:         sourceRpmLocation,
@@ -131,6 +138,8 @@ func mn(_ *cobra.Command, _ []string) {
 		TmpFsMode:           tmpFsMode,
 		NoStorageDownload:   noStorageDownload,
 		NoStorageUpload:     noStorageUpload,
+		ManualCommits:       manualCs,
+		UpstreamPrefixHttps: upstreamPrefixHttps,
 		FsCreator:           fsCreator,
 	})
 }
@@ -159,6 +168,8 @@ func main() {
 	root.Flags().StringVar(&tmpFsMode, "tmpfs-mode", "", "If set, packages are imported to path and patched but not pushed")
 	root.Flags().BoolVar(&noStorageDownload, "no-storage-download", false, "If enabled, blobs are always downloaded from upstream")
 	root.Flags().BoolVar(&noStorageUpload, "no-storage-upload", false, "If enabled, blobs are not uploaded to blob storage")
+	root.Flags().StringVar(&manualCommits, "manual-commits", "", "Comma separated branch and commit list for packages with broken release tags (Format: BRANCH:HASH)")
+	root.Flags().StringVar(&upstreamPrefixHttps, "upstream-prefix-https", "", "Web version of upstream prefix. Required if module-mode")
 
 	if err := root.Execute(); err != nil {
 		log.Fatal(err)

internal/data/process.go

@@ -27,5 +27,7 @@ type ProcessData struct {
 	TmpFsMode           string
 	NoStorageDownload   bool
 	NoStorageUpload     bool
+	ManualCommits       []string
+	UpstreamPrefixHttps string
 	FsCreator           func(branch string) billy.Filesystem
 }

internal/fetch.go

@@ -1,8 +0,0 @@
-package internal
-
-import "log"
-
-// FetchRPM downloads the rpm into /tmp/srpmproc
-func FetchRPM(uri string) {
-    log.Fatal("Not supported")
-}

internal/git.go

@@ -112,7 +112,8 @@ func (g *GitMode) RetrieveSource(pd *data.ProcessData) *data.ModeData {
 
 			commit, err := repo.CommitObject(ref.Hash())
 			if err != nil {
-				log.Fatalf("could not get commit object: %v", err)
+				log.Printf("could not get commit object for ref %s: %v", ref.Name().String(), err)
+				continue
 			}
 			_ = tagAdd(&object.Tag{
 				Name:   strings.TrimPrefix(string(ref.Name()), "refs/tags/"),

internal/patch.go

@@ -16,6 +16,7 @@ import (
 	"log"
 	"path/filepath"
 	"strings"
+	"time"
 )
 
 func cfgPatches(pd *data.ProcessData, md *data.ModeData, patchTree *git.Worktree, pushTree *git.Worktree) {
@@ -128,7 +129,7 @@ func executePatchesRpm(pd *data.ProcessData, md *data.ModeData) {
 	}
 }
 
-func getTipStream(pd *data.ProcessData, module string, pushBranch string, origPushBranch string) string {
+func getTipStream(pd *data.ProcessData, module string, pushBranch string, origPushBranch string, tries int) string {
 	repo, err := git.Init(memory.NewStorage(), memfs.New())
 	if err != nil {
 		log.Fatalf("could not init git Repo: %v", err)
@@ -149,6 +150,13 @@ func getTipStream(pd *data.ProcessData, module string, pushBranch string, origPu
 		Auth: pd.Authenticator,
 	})
 	if err != nil {
+		log.Printf("could not import module: %s", module)
+		if tries < 3 {
+			log.Printf("could not get rpm refs. will retry in 3s. %v", err)
+			time.Sleep(3 * time.Second)
+			return getTipStream(pd, module, pushBranch, origPushBranch, tries+1)
+		}
+
 		log.Fatalf("could not get rpm refs. import the rpm before the module: %v", err)
 	}
 
@@ -167,13 +175,12 @@ func getTipStream(pd *data.ProcessData, module string, pushBranch string, origPu
 			}
 		}
 
-		log.Println(prefix, ref.Name().String())
-
 		if strings.HasPrefix(ref.Name().String(), prefix) {
 			tipHash = ref.Hash().String()
 		}
 	}
 
+	if tipHash == "" {
 		for _, ref := range list {
 			prefix := fmt.Sprintf("refs/heads/%s", origPushBranch)
 
@@ -181,8 +188,20 @@ func getTipStream(pd *data.ProcessData, module string, pushBranch string, origPu
 				tipHash = ref.Hash().String()
 			}
 		}
+	}
 
 	if tipHash == "" {
+		for _, ref := range list {
+			if !strings.Contains(ref.Name().String(), "stream") {
+				tipHash = ref.Hash().String()
+			}
+		}
+	}
+
+	if tipHash == "" {
+		for _, ref := range list {
+			log.Println(pushBranch, ref.Name())
+		}
 		log.Fatal("could not find tip hash")
 	}
 
@@ -212,6 +231,11 @@ func patchModuleYaml(pd *data.ProcessData, md *data.ModeData) {
 		log.Fatalf("could not parse modulemd file: %v", err)
 	}
 
+	log.Println("This module contains the following rpms:")
+	for name := range module.Data.Components.Rpms {
+		log.Printf("\t- %s", name)
+	}
+
 	for name, rpm := range module.Data.Components.Rpms {
 		var tipHash string
 		var pushBranch string
@@ -237,7 +261,7 @@ func patchModuleYaml(pd *data.ProcessData, md *data.ModeData) {
 		}
 
 		rpm.Ref = pushBranch
-		tipHash = getTipStream(pd, name, pushBranch, md.PushBranch)
+		tipHash = getTipStream(pd, name, pushBranch, md.PushBranch, 0)
 
 		err = module.Marshal(md.Worktree.Filesystem, mdTxtPath)
 		if err != nil {
@@ -247,6 +271,12 @@ func patchModuleYaml(pd *data.ProcessData, md *data.ModeData) {
 		rpm.Ref = tipHash
 	}
 
+	for name, rpm := range module.Data.Components.Rpms {
+		if name != gitlabify(name) {
+			rpm.Repository = fmt.Sprintf("https://%s/rpms/%s.git", pd.UpstreamPrefixHttps, gitlabify(name))
+		}
+	}
+
 	rootModule := fmt.Sprintf("%s.yaml", md.RpmFile.Name())
 	err = module.Marshal(md.Worktree.Filesystem, rootModule)
 	if err != nil {

internal/process.go

@@ -85,8 +85,22 @@ func ProcessRPM(pd *data.ProcessData) {
 	sourceRepo := *md.Repo
 	sourceWorktree := *md.Worktree
 
+	commitPin := map[string]string{}
+
 	if pd.SingleTag != "" {
 		md.Branches = []string{fmt.Sprintf("refs/tags/%s", pd.SingleTag)}
+	} else if len(pd.ManualCommits) > 0 {
+		md.Branches = []string{}
+		for _, commit := range pd.ManualCommits {
+			branchCommit := strings.Split(commit, ":")
+			if len(branchCommit) != 2 {
+				log.Fatalln("invalid manual commit list")
+			}
+
+			head := fmt.Sprintf("refs/heads/%s", branchCommit[0])
+			md.Branches = append(md.Branches, head)
+			commitPin[head] = branchCommit[1]
+		}
 	}
 
 	for _, branch := range md.Branches {
@@ -168,6 +182,11 @@ func ProcessRPM(pd *data.ProcessData) {
 		refName := plumbing.NewBranchReferenceName(md.PushBranch)
 		log.Printf("set reference to ref: %s", refName)
 
+		var hash plumbing.Hash
+		if commitPin[md.PushBranch] != "" {
+			hash = plumbing.NewHash(commitPin[md.PushBranch])
+		}
+
 		if err != nil {
 			h := plumbing.NewSymbolicReference(plumbing.HEAD, refName)
 			if err := repo.Storer.CheckAndSetReference(h, nil); err != nil {
@@ -176,6 +195,7 @@ func ProcessRPM(pd *data.ProcessData) {
 		} else {
 			err = w.Checkout(&git.CheckoutOptions{
 				Branch: plumbing.NewRemoteReferenceName("origin", md.PushBranch),
+				Hash:   hash,
 				Force:  true,
 			})
 			if err != nil {