Add commit pinning

cmd/srpmproc/main.go

@@ -22,24 +22,26 @@ import (
 )
 
 var (
-	sourceRpm          string
+	sourceRpm           string
-	sshKeyLocation     string
+	sshKeyLocation      string
-	sshUser            string
+	sshUser             string
-	upstreamPrefix     string
+	upstreamPrefix      string
-	version            int
+	version             int
-	storageAddr        string
+	storageAddr         string
-	gitCommitterName   string
+	gitCommitterName    string
-	gitCommitterEmail  string
+	gitCommitterEmail   string
-	modulePrefix       string
+	modulePrefix        string
-	rpmPrefix          string
+	rpmPrefix           string
-	importBranchPrefix string
+	importBranchPrefix  string
-	branchPrefix       string
+	branchPrefix        string
-	singleTag          string
+	singleTag           string
-	noDupMode          bool
+	noDupMode           bool
-	moduleMode         bool
+	moduleMode          bool
-	tmpFsMode          string
+	tmpFsMode           string
-	noStorageDownload  bool
+	noStorageDownload   bool
-	noStorageUpload    bool
+	noStorageUpload     bool
+	manualCommits       string
+	upstreamPrefixHttps string
 )
 
 var root = &cobra.Command{
@@ -111,27 +113,34 @@ func mn(_ *cobra.Command, _ []string) {
 		}
 	}
 
+	var manualCs []string
+	if strings.TrimSpace(manualCommits) != "" {
+		manualCs = strings.Split(manualCommits, ",")
+	}
+
 	internal.ProcessRPM(&data.ProcessData{
-		Importer:           importer,
+		Importer:            importer,
-		RpmLocation:        sourceRpmLocation,
+		RpmLocation:         sourceRpmLocation,
-		UpstreamPrefix:     upstreamPrefix,
+		UpstreamPrefix:      upstreamPrefix,
-		SshKeyLocation:     sshKeyLocation,
+		SshKeyLocation:      sshKeyLocation,
-		SshUser:            sshUser,
+		SshUser:             sshUser,
-		Version:            version,
+		Version:             version,
-		BlobStorage:        blobStorage,
+		BlobStorage:         blobStorage,
-		GitCommitterName:   gitCommitterName,
+		GitCommitterName:    gitCommitterName,
-		GitCommitterEmail:  gitCommitterEmail,
+		GitCommitterEmail:   gitCommitterEmail,
-		ModulePrefix:       modulePrefix,
+		ModulePrefix:        modulePrefix,
-		ImportBranchPrefix: importBranchPrefix,
+		ImportBranchPrefix:  importBranchPrefix,
-		BranchPrefix:       branchPrefix,
+		BranchPrefix:        branchPrefix,
-		SingleTag:          singleTag,
+		SingleTag:           singleTag,
-		Authenticator:      authenticator,
+		Authenticator:       authenticator,
-		NoDupMode:          noDupMode,
+		NoDupMode:           noDupMode,
-		ModuleMode:         moduleMode,
+		ModuleMode:          moduleMode,
-		TmpFsMode:          tmpFsMode,
+		TmpFsMode:           tmpFsMode,
-		NoStorageDownload:  noStorageDownload,
+		NoStorageDownload:   noStorageDownload,
-		NoStorageUpload:    noStorageUpload,
+		NoStorageUpload:     noStorageUpload,
-		FsCreator:          fsCreator,
+		ManualCommits:       manualCs,
+		UpstreamPrefixHttps: upstreamPrefixHttps,
+		FsCreator:           fsCreator,
 	})
 }
 
@@ -159,6 +168,8 @@ func main() {
 	root.Flags().StringVar(&tmpFsMode, "tmpfs-mode", "", "If set, packages are imported to path and patched but not pushed")
 	root.Flags().BoolVar(&noStorageDownload, "no-storage-download", false, "If enabled, blobs are always downloaded from upstream")
 	root.Flags().BoolVar(&noStorageUpload, "no-storage-upload", false, "If enabled, blobs are not uploaded to blob storage")
+	root.Flags().StringVar(&manualCommits, "manual-commits", "", "Comma separated branch and commit list for packages with broken release tags (Format: BRANCH:HASH)")
+	root.Flags().StringVar(&upstreamPrefixHttps, "upstream-prefix-https", "", "Web version of upstream prefix. Required if module-mode")
 
 	if err := root.Execute(); err != nil {
 		log.Fatal(err)

internal/data/process.go

@@ -7,25 +7,27 @@ import (
 )
 
 type ProcessData struct {
-	RpmLocation        string
+	RpmLocation         string
-	UpstreamPrefix     string
+	UpstreamPrefix      string
-	SshKeyLocation     string
+	SshKeyLocation      string
-	SshUser            string
+	SshUser             string
-	Version            int
+	Version             int
-	GitCommitterName   string
+	GitCommitterName    string
-	GitCommitterEmail  string
+	GitCommitterEmail   string
-	Mode               int
+	Mode                int
-	ModulePrefix       string
+	ModulePrefix        string
-	ImportBranchPrefix string
+	ImportBranchPrefix  string
-	BranchPrefix       string
+	BranchPrefix        string
-	SingleTag          string
+	SingleTag           string
-	Authenticator      *ssh.PublicKeys
+	Authenticator       *ssh.PublicKeys
-	Importer           ImportMode
+	Importer            ImportMode
-	BlobStorage        blob.Storage
+	BlobStorage         blob.Storage
-	NoDupMode          bool
+	NoDupMode           bool
-	ModuleMode         bool
+	ModuleMode          bool
-	TmpFsMode          string
+	TmpFsMode           string
-	NoStorageDownload  bool
+	NoStorageDownload   bool
-	NoStorageUpload    bool
+	NoStorageUpload     bool
-	FsCreator          func(branch string) billy.Filesystem
+	ManualCommits       []string
+	UpstreamPrefixHttps string
+	FsCreator           func(branch string) billy.Filesystem
 }

internal/fetch.go

@@ -1,8 +0,0 @@
-package internal
-
-import "log"
-
-// FetchRPM downloads the rpm into /tmp/srpmproc
-func FetchRPM(uri string) {
-    log.Fatal("Not supported")
-}

internal/git.go

@@ -112,7 +112,8 @@ func (g *GitMode) RetrieveSource(pd *data.ProcessData) *data.ModeData {
 
 			commit, err := repo.CommitObject(ref.Hash())
 			if err != nil {
-				log.Fatalf("could not get commit object: %v", err)
+				log.Printf("could not get commit object for ref %s: %v", ref.Name().String(), err)
+				continue
 			}
 			_ = tagAdd(&object.Tag{
 				Name:   strings.TrimPrefix(string(ref.Name()), "refs/tags/"),

internal/patch.go

@@ -16,6 +16,7 @@ import (
 	"log"
 	"path/filepath"
 	"strings"
+	"time"
 )
 
 func cfgPatches(pd *data.ProcessData, md *data.ModeData, patchTree *git.Worktree, pushTree *git.Worktree) {
@@ -128,7 +129,7 @@ func executePatchesRpm(pd *data.ProcessData, md *data.ModeData) {
 	}
 }
 
-func getTipStream(pd *data.ProcessData, module string, pushBranch string, origPushBranch string) string {
+func getTipStream(pd *data.ProcessData, module string, pushBranch string, origPushBranch string, tries int) string {
 	repo, err := git.Init(memory.NewStorage(), memfs.New())
 	if err != nil {
 		log.Fatalf("could not init git Repo: %v", err)
@@ -149,6 +150,13 @@ func getTipStream(pd *data.ProcessData, module string, pushBranch string, origPu
 		Auth: pd.Authenticator,
 	})
 	if err != nil {
+		log.Printf("could not import module: %s", module)
+		if tries < 3 {
+			log.Printf("could not get rpm refs. will retry in 3s. %v", err)
+			time.Sleep(3 * time.Second)
+			return getTipStream(pd, module, pushBranch, origPushBranch, tries+1)
+		}
+
 		log.Fatalf("could not get rpm refs. import the rpm before the module: %v", err)
 	}
 
@@ -167,22 +175,33 @@ func getTipStream(pd *data.ProcessData, module string, pushBranch string, origPu
 			}
 		}
 
-		log.Println(prefix, ref.Name().String())
-
-		if strings.HasPrefix(ref.Name().String(), prefix) {
-			tipHash = ref.Hash().String()
-		}
-	}
-
-	for _, ref := range list {
-		prefix := fmt.Sprintf("refs/heads/%s", origPushBranch)
-
 		if strings.HasPrefix(ref.Name().String(), prefix) {
 			tipHash = ref.Hash().String()
 		}
 	}
 
 	if tipHash == "" {
+		for _, ref := range list {
+			prefix := fmt.Sprintf("refs/heads/%s", origPushBranch)
+
+			if strings.HasPrefix(ref.Name().String(), prefix) {
+				tipHash = ref.Hash().String()
+			}
+		}
+	}
+
+	if tipHash == "" {
+		for _, ref := range list {
+			if !strings.Contains(ref.Name().String(), "stream") {
+				tipHash = ref.Hash().String()
+			}
+		}
+	}
+
+	if tipHash == "" {
+		for _, ref := range list {
+			log.Println(pushBranch, ref.Name())
+		}
 		log.Fatal("could not find tip hash")
 	}
 
@@ -212,6 +231,11 @@ func patchModuleYaml(pd *data.ProcessData, md *data.ModeData) {
 		log.Fatalf("could not parse modulemd file: %v", err)
 	}
 
+	log.Println("This module contains the following rpms:")
+	for name := range module.Data.Components.Rpms {
+		log.Printf("\t- %s", name)
+	}
+
 	for name, rpm := range module.Data.Components.Rpms {
 		var tipHash string
 		var pushBranch string
@@ -237,7 +261,7 @@ func patchModuleYaml(pd *data.ProcessData, md *data.ModeData) {
 		}
 
 		rpm.Ref = pushBranch
-		tipHash = getTipStream(pd, name, pushBranch, md.PushBranch)
+		tipHash = getTipStream(pd, name, pushBranch, md.PushBranch, 0)
 
 		err = module.Marshal(md.Worktree.Filesystem, mdTxtPath)
 		if err != nil {
@@ -247,6 +271,12 @@ func patchModuleYaml(pd *data.ProcessData, md *data.ModeData) {
 		rpm.Ref = tipHash
 	}
 
+	for name, rpm := range module.Data.Components.Rpms {
+		if name != gitlabify(name) {
+			rpm.Repository = fmt.Sprintf("https://%s/rpms/%s.git", pd.UpstreamPrefixHttps, gitlabify(name))
+		}
+	}
+
 	rootModule := fmt.Sprintf("%s.yaml", md.RpmFile.Name())
 	err = module.Marshal(md.Worktree.Filesystem, rootModule)
 	if err != nil {

internal/process.go

@@ -85,8 +85,22 @@ func ProcessRPM(pd *data.ProcessData) {
 	sourceRepo := *md.Repo
 	sourceWorktree := *md.Worktree
 
+	commitPin := map[string]string{}
+
 	if pd.SingleTag != "" {
 		md.Branches = []string{fmt.Sprintf("refs/tags/%s", pd.SingleTag)}
+	} else if len(pd.ManualCommits) > 0 {
+		md.Branches = []string{}
+		for _, commit := range pd.ManualCommits {
+			branchCommit := strings.Split(commit, ":")
+			if len(branchCommit) != 2 {
+				log.Fatalln("invalid manual commit list")
+			}
+
+			head := fmt.Sprintf("refs/heads/%s", branchCommit[0])
+			md.Branches = append(md.Branches, head)
+			commitPin[head] = branchCommit[1]
+		}
 	}
 
 	for _, branch := range md.Branches {
@@ -168,6 +182,11 @@ func ProcessRPM(pd *data.ProcessData) {
 		refName := plumbing.NewBranchReferenceName(md.PushBranch)
 		log.Printf("set reference to ref: %s", refName)
 
+		var hash plumbing.Hash
+		if commitPin[md.PushBranch] != "" {
+			hash = plumbing.NewHash(commitPin[md.PushBranch])
+		}
+
 		if err != nil {
 			h := plumbing.NewSymbolicReference(plumbing.HEAD, refName)
 			if err := repo.Storer.CheckAndSetReference(h, nil); err != nil {
@@ -176,6 +195,7 @@ func ProcessRPM(pd *data.ProcessData) {
 		} else {
 			err = w.Checkout(&git.CheckoutOptions{
 				Branch: plumbing.NewRemoteReferenceName("origin", md.PushBranch),
+				Hash:   hash,
 				Force:  true,
 			})
 			if err != nil {