diff --git a/cmd/srpmproc/main.go b/cmd/srpmproc/main.go index 91d3ed2..816964f 100644 --- a/cmd/srpmproc/main.go +++ b/cmd/srpmproc/main.go @@ -22,24 +22,26 @@ import ( ) var ( - sourceRpm string - sshKeyLocation string - sshUser string - upstreamPrefix string - version int - storageAddr string - gitCommitterName string - gitCommitterEmail string - modulePrefix string - rpmPrefix string - importBranchPrefix string - branchPrefix string - singleTag string - noDupMode bool - moduleMode bool - tmpFsMode string - noStorageDownload bool - noStorageUpload bool + sourceRpm string + sshKeyLocation string + sshUser string + upstreamPrefix string + version int + storageAddr string + gitCommitterName string + gitCommitterEmail string + modulePrefix string + rpmPrefix string + importBranchPrefix string + branchPrefix string + singleTag string + noDupMode bool + moduleMode bool + tmpFsMode string + noStorageDownload bool + noStorageUpload bool + manualCommits string + upstreamPrefixHttps string ) var root = &cobra.Command{ @@ -111,27 +113,34 @@ func mn(_ *cobra.Command, _ []string) { } } + var manualCs []string + if strings.TrimSpace(manualCommits) != "" { + manualCs = strings.Split(manualCommits, ",") + } + internal.ProcessRPM(&data.ProcessData{ - Importer: importer, - RpmLocation: sourceRpmLocation, - UpstreamPrefix: upstreamPrefix, - SshKeyLocation: sshKeyLocation, - SshUser: sshUser, - Version: version, - BlobStorage: blobStorage, - GitCommitterName: gitCommitterName, - GitCommitterEmail: gitCommitterEmail, - ModulePrefix: modulePrefix, - ImportBranchPrefix: importBranchPrefix, - BranchPrefix: branchPrefix, - SingleTag: singleTag, - Authenticator: authenticator, - NoDupMode: noDupMode, - ModuleMode: moduleMode, - TmpFsMode: tmpFsMode, - NoStorageDownload: noStorageDownload, - NoStorageUpload: noStorageUpload, - FsCreator: fsCreator, + Importer: importer, + RpmLocation: sourceRpmLocation, + UpstreamPrefix: upstreamPrefix, + SshKeyLocation: sshKeyLocation, + SshUser: sshUser, + Version: version, + BlobStorage: blobStorage, + GitCommitterName: gitCommitterName, + GitCommitterEmail: gitCommitterEmail, + ModulePrefix: modulePrefix, + ImportBranchPrefix: importBranchPrefix, + BranchPrefix: branchPrefix, + SingleTag: singleTag, + Authenticator: authenticator, + NoDupMode: noDupMode, + ModuleMode: moduleMode, + TmpFsMode: tmpFsMode, + NoStorageDownload: noStorageDownload, + NoStorageUpload: noStorageUpload, + ManualCommits: manualCs, + UpstreamPrefixHttps: upstreamPrefixHttps, + FsCreator: fsCreator, }) } @@ -159,6 +168,8 @@ func main() { root.Flags().StringVar(&tmpFsMode, "tmpfs-mode", "", "If set, packages are imported to path and patched but not pushed") root.Flags().BoolVar(&noStorageDownload, "no-storage-download", false, "If enabled, blobs are always downloaded from upstream") root.Flags().BoolVar(&noStorageUpload, "no-storage-upload", false, "If enabled, blobs are not uploaded to blob storage") + root.Flags().StringVar(&manualCommits, "manual-commits", "", "Comma separated branch and commit list for packages with broken release tags (Format: BRANCH:HASH)") + root.Flags().StringVar(&upstreamPrefixHttps, "upstream-prefix-https", "", "Web version of upstream prefix. Required if module-mode") if err := root.Execute(); err != nil { log.Fatal(err) diff --git a/internal/data/process.go b/internal/data/process.go index abf177f..351cb68 100644 --- a/internal/data/process.go +++ b/internal/data/process.go @@ -7,25 +7,27 @@ import ( ) type ProcessData struct { - RpmLocation string - UpstreamPrefix string - SshKeyLocation string - SshUser string - Version int - GitCommitterName string - GitCommitterEmail string - Mode int - ModulePrefix string - ImportBranchPrefix string - BranchPrefix string - SingleTag string - Authenticator *ssh.PublicKeys - Importer ImportMode - BlobStorage blob.Storage - NoDupMode bool - ModuleMode bool - TmpFsMode string - NoStorageDownload bool - NoStorageUpload bool - FsCreator func(branch string) billy.Filesystem + RpmLocation string + UpstreamPrefix string + SshKeyLocation string + SshUser string + Version int + GitCommitterName string + GitCommitterEmail string + Mode int + ModulePrefix string + ImportBranchPrefix string + BranchPrefix string + SingleTag string + Authenticator *ssh.PublicKeys + Importer ImportMode + BlobStorage blob.Storage + NoDupMode bool + ModuleMode bool + TmpFsMode string + NoStorageDownload bool + NoStorageUpload bool + ManualCommits []string + UpstreamPrefixHttps string + FsCreator func(branch string) billy.Filesystem } diff --git a/internal/fetch.go b/internal/fetch.go deleted file mode 100644 index 4c53e21..0000000 --- a/internal/fetch.go +++ /dev/null @@ -1,8 +0,0 @@ -package internal - -import "log" - -// FetchRPM downloads the rpm into /tmp/srpmproc -func FetchRPM(uri string) { - log.Fatal("Not supported") -} diff --git a/internal/git.go b/internal/git.go index 5b0b989..0aad5f9 100644 --- a/internal/git.go +++ b/internal/git.go @@ -112,7 +112,8 @@ func (g *GitMode) RetrieveSource(pd *data.ProcessData) *data.ModeData { commit, err := repo.CommitObject(ref.Hash()) if err != nil { - log.Fatalf("could not get commit object: %v", err) + log.Printf("could not get commit object for ref %s: %v", ref.Name().String(), err) + continue } _ = tagAdd(&object.Tag{ Name: strings.TrimPrefix(string(ref.Name()), "refs/tags/"), diff --git a/internal/patch.go b/internal/patch.go index 172dd0d..0290b65 100644 --- a/internal/patch.go +++ b/internal/patch.go @@ -16,6 +16,7 @@ import ( "log" "path/filepath" "strings" + "time" ) func cfgPatches(pd *data.ProcessData, md *data.ModeData, patchTree *git.Worktree, pushTree *git.Worktree) { @@ -128,7 +129,7 @@ func executePatchesRpm(pd *data.ProcessData, md *data.ModeData) { } } -func getTipStream(pd *data.ProcessData, module string, pushBranch string, origPushBranch string) string { +func getTipStream(pd *data.ProcessData, module string, pushBranch string, origPushBranch string, tries int) string { repo, err := git.Init(memory.NewStorage(), memfs.New()) if err != nil { log.Fatalf("could not init git Repo: %v", err) @@ -149,6 +150,13 @@ func getTipStream(pd *data.ProcessData, module string, pushBranch string, origPu Auth: pd.Authenticator, }) if err != nil { + log.Printf("could not import module: %s", module) + if tries < 3 { + log.Printf("could not get rpm refs. will retry in 3s. %v", err) + time.Sleep(3 * time.Second) + return getTipStream(pd, module, pushBranch, origPushBranch, tries+1) + } + log.Fatalf("could not get rpm refs. import the rpm before the module: %v", err) } @@ -167,22 +175,33 @@ func getTipStream(pd *data.ProcessData, module string, pushBranch string, origPu } } - log.Println(prefix, ref.Name().String()) - - if strings.HasPrefix(ref.Name().String(), prefix) { - tipHash = ref.Hash().String() - } - } - - for _, ref := range list { - prefix := fmt.Sprintf("refs/heads/%s", origPushBranch) - if strings.HasPrefix(ref.Name().String(), prefix) { tipHash = ref.Hash().String() } } if tipHash == "" { + for _, ref := range list { + prefix := fmt.Sprintf("refs/heads/%s", origPushBranch) + + if strings.HasPrefix(ref.Name().String(), prefix) { + tipHash = ref.Hash().String() + } + } + } + + if tipHash == "" { + for _, ref := range list { + if !strings.Contains(ref.Name().String(), "stream") { + tipHash = ref.Hash().String() + } + } + } + + if tipHash == "" { + for _, ref := range list { + log.Println(pushBranch, ref.Name()) + } log.Fatal("could not find tip hash") } @@ -212,6 +231,11 @@ func patchModuleYaml(pd *data.ProcessData, md *data.ModeData) { log.Fatalf("could not parse modulemd file: %v", err) } + log.Println("This module contains the following rpms:") + for name := range module.Data.Components.Rpms { + log.Printf("\t- %s", name) + } + for name, rpm := range module.Data.Components.Rpms { var tipHash string var pushBranch string @@ -237,7 +261,7 @@ func patchModuleYaml(pd *data.ProcessData, md *data.ModeData) { } rpm.Ref = pushBranch - tipHash = getTipStream(pd, name, pushBranch, md.PushBranch) + tipHash = getTipStream(pd, name, pushBranch, md.PushBranch, 0) err = module.Marshal(md.Worktree.Filesystem, mdTxtPath) if err != nil { @@ -247,6 +271,12 @@ func patchModuleYaml(pd *data.ProcessData, md *data.ModeData) { rpm.Ref = tipHash } + for name, rpm := range module.Data.Components.Rpms { + if name != gitlabify(name) { + rpm.Repository = fmt.Sprintf("https://%s/rpms/%s.git", pd.UpstreamPrefixHttps, gitlabify(name)) + } + } + rootModule := fmt.Sprintf("%s.yaml", md.RpmFile.Name()) err = module.Marshal(md.Worktree.Filesystem, rootModule) if err != nil { diff --git a/internal/process.go b/internal/process.go index f053d23..cea2102 100644 --- a/internal/process.go +++ b/internal/process.go @@ -85,8 +85,22 @@ func ProcessRPM(pd *data.ProcessData) { sourceRepo := *md.Repo sourceWorktree := *md.Worktree + commitPin := map[string]string{} + if pd.SingleTag != "" { md.Branches = []string{fmt.Sprintf("refs/tags/%s", pd.SingleTag)} + } else if len(pd.ManualCommits) > 0 { + md.Branches = []string{} + for _, commit := range pd.ManualCommits { + branchCommit := strings.Split(commit, ":") + if len(branchCommit) != 2 { + log.Fatalln("invalid manual commit list") + } + + head := fmt.Sprintf("refs/heads/%s", branchCommit[0]) + md.Branches = append(md.Branches, head) + commitPin[head] = branchCommit[1] + } } for _, branch := range md.Branches { @@ -168,6 +182,11 @@ func ProcessRPM(pd *data.ProcessData) { refName := plumbing.NewBranchReferenceName(md.PushBranch) log.Printf("set reference to ref: %s", refName) + var hash plumbing.Hash + if commitPin[md.PushBranch] != "" { + hash = plumbing.NewHash(commitPin[md.PushBranch]) + } + if err != nil { h := plumbing.NewSymbolicReference(plumbing.HEAD, refName) if err := repo.Storer.CheckAndSetReference(h, nil); err != nil { @@ -176,6 +195,7 @@ func ProcessRPM(pd *data.ProcessData) { } else { err = w.Checkout(&git.CheckoutOptions{ Branch: plumbing.NewRemoteReferenceName("origin", md.PushBranch), + Hash: hash, Force: true, }) if err != nil {