diff --git a/pkg/misc/regex.go b/pkg/misc/regex.go index 254b6f1..3cc6f0b 100644 --- a/pkg/misc/regex.go +++ b/pkg/misc/regex.go @@ -5,6 +5,7 @@ import ( "github.com/rocky-linux/srpmproc/pkg/data" "path/filepath" "regexp" + "strings" ) func GetTagImportRegex(pd *data.ProcessData) *regexp.Regexp { @@ -31,3 +32,22 @@ func GetTagImportRegex(pd *data.ProcessData) *regexp.Regexp { return regexp.MustCompile(regex) } + +// Given a git reference in tagless mode (like "refs/heads/c9s", or "refs/heads/stream-httpd-2.4-rhel-9.1.0"), determine +// if we are ok with importing that reference. We are looking for the traditional pattern, like "c9s", and also the +// modular "stream---rhel- branch pattern as well +func TaglessRefOk(tag string, pd *data.ProcessData) bool { + + // First case is very easy: if we are "refs/heads/" , then this is def. a branch we should import + if strings.HasPrefix(tag, fmt.Sprintf("refs/heads/%s%d%s", pd.ImportBranchPrefix, pd.Version, pd.BranchSuffix)) { + return true + } + + // Less easy: if a modular branch is present (starts w/ "stream-"), we need to check if it's part of our major version, and return true if it is + // (major version means we look for the text "rhel-X." in the branch name, like "rhel-9.1.0") + if strings.HasPrefix(tag, "refs/heads/stream-") && strings.Contains(tag, fmt.Sprintf("rhel-%d.", pd.Version)) { + return true + } + + return false +} diff --git a/pkg/modes/git.go b/pkg/modes/git.go index b19fa36..d68d7e4 100644 --- a/pkg/modes/git.go +++ b/pkg/modes/git.go @@ -128,22 +128,16 @@ func (g *GitMode) RetrieveSource(pd *data.ProcessData) (*data.ModeData, error) { // In case of "tagless mode", we need to get the head ref of the branch instead // This is a kind of alternative implementation of the above tagAdd assignment refAdd := func(tag *object.Tag) error { - if strings.HasPrefix(tag.Name, fmt.Sprintf("refs/heads/%s%d%s", pd.ImportBranchPrefix, pd.Version, pd.BranchSuffix)) { + if misc.TaglessRefOk(tag.Name, pd) { pd.Log.Printf("Tagless mode: Identified tagless commit for import: %s\n", tag.Name) refSpec := fmt.Sprintf(tag.Name) // We split the string by "/", the branch name we're looking for to pass to latestTags is always last // (ex: "refs/heads/c9s" ---> we want latestTags[c9s] - _tmpRef := strings.Split(refSpec, "/") - _branchName := _tmpRef[(len(_tmpRef) - 1)] + tmpRef := strings.Split(refSpec, "/") + tmpBranchName := tmpRef[(len(tmpRef) - 1)] - // In the case of "strict branch mode" on, the branch name must match *exactly* with our prefix-version-suffix (like "c8" cannot also match "c8-beta") - // If it doesn't, bail out without adding this branch - if pd.StrictBranchMode == true && _branchName != fmt.Sprintf("%s%d%s", pd.ImportBranchPrefix, pd.Version, pd.BranchSuffix) { - return nil - } - - latestTags[_branchName] = &remoteTarget{ + latestTags[tmpBranchName] = &remoteTarget{ remote: refSpec, when: tag.Tagger.When, } @@ -159,7 +153,7 @@ func (g *GitMode) RetrieveSource(pd *data.ProcessData) (*data.ModeData, error) { // tagless mode means we use "refAdd" (add commit by reference) // normal mode means we can rely on "tagAdd" (the tag should be present for us in the source repo) - if pd.TaglessMode == true { + if pd.TaglessMode { _ = tagIter.ForEach(refAdd) } else { _ = tagIter.ForEach(tagAdd) @@ -192,7 +186,7 @@ func (g *GitMode) RetrieveSource(pd *data.ProcessData) (*data.ModeData, error) { } // Call refAdd instead of tagAdd in the case of TaglessMode enabled - if pd.TaglessMode == true { + if pd.TaglessMode { _ = refAdd(&object.Tag{ Name: string(ref.Name()), Tagger: commit.Committer, @@ -230,7 +224,7 @@ func (g *GitMode) WriteSource(pd *data.ProcessData, md *data.ModeData) error { remote, err := md.Repo.Remote("upstream") - if err != nil && pd.TaglessMode == false { + if err != nil && !pd.TaglessMode { return fmt.Errorf("could not get upstream remote: %v", err) } @@ -239,7 +233,7 @@ func (g *GitMode) WriteSource(pd *data.ProcessData, md *data.ModeData) error { // In the case of tagless mode, we already have the transformed repo sitting in the worktree, // and don't need to perform any checkout or fetch operations - if pd.TaglessMode == false { + if !pd.TaglessMode { if strings.HasPrefix(md.TagBranch, "refs/heads") { refspec = config.RefSpec(fmt.Sprintf("+%s:%s", md.TagBranch, md.TagBranch)) branchName = strings.TrimPrefix(md.TagBranch, "refs/heads/") @@ -285,7 +279,7 @@ func (g *GitMode) WriteSource(pd *data.ProcessData, md *data.ModeData) error { } } - if pd.TaglessMode == true { + if pd.TaglessMode { branchName = fmt.Sprintf("%s%d%s", pd.ImportBranchPrefix, pd.Version, pd.BranchSuffix) } @@ -349,7 +343,7 @@ func (g *GitMode) WriteSource(pd *data.ProcessData, md *data.ModeData) error { url := "" // Alternate lookaside logic: if enabled, we pull from a new URL pattern - if pd.AltLookAside == false { + if !pd.AltLookAside { url = fmt.Sprintf("%s/%s/%s/%s", pd.CdnUrl, md.Name, branchName, hash) } else { // We first need the hash algorithm based on length of hash: diff --git a/pkg/srpmproc/patch.go b/pkg/srpmproc/patch.go index b2ada0d..b2f739c 100644 --- a/pkg/srpmproc/patch.go +++ b/pkg/srpmproc/patch.go @@ -32,6 +32,7 @@ import ( "strings" "time" + "github.com/go-git/go-billy/v5" "github.com/go-git/go-billy/v5/memfs" "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/config" @@ -286,13 +287,30 @@ func patchModuleYaml(pd *data.ProcessData, md *data.ModeData) error { return nil } - mdTxtPath := "SOURCES/modulemd.src.txt" - f, err := md.Worktree.Filesystem.Open(mdTxtPath) - if err != nil { - mdTxtPath = "SOURCES/modulemd.txt" + mdTxtPath := "" + var f billy.File + + // tagless mode implies we're looking for CentOS Stream modules, which are generally "SOURCES/NAME.yaml" (copied to SOURCES/ from import) + // if not tagless mode, proceed as usual with SOURCES/modulemd.*.txt + if pd.TaglessMode { + mdTxtPath = fmt.Sprintf("SOURCES/%s.yaml", md.Name) f, err = md.Worktree.Filesystem.Open(mdTxtPath) if err != nil { - return fmt.Errorf("could not open modulemd file: %v", err) + mdTxtPath = fmt.Sprintf("SOURCES/%s.yml", md.Name) + f, err = md.Worktree.Filesystem.Open(mdTxtPath) + if err != nil { + return fmt.Errorf("could not open modulemd file: %v", err) + } + } + } else { + mdTxtPath = "SOURCES/modulemd.src.txt" + f, err = md.Worktree.Filesystem.Open(mdTxtPath) + if err != nil { + mdTxtPath = "SOURCES/modulemd.txt" + f, err = md.Worktree.Filesystem.Open(mdTxtPath) + if err != nil { + return fmt.Errorf("could not open modulemd file: %v", err) + } } } @@ -307,11 +325,13 @@ func patchModuleYaml(pd *data.ProcessData, md *data.ModeData) error { } // Get stream branch from tag - match := misc.GetTagImportRegex(pd).FindStringSubmatch(md.TagBranch) - streamBranch := strings.Split(match[2], "-") - // Force stream to be the same as stream name in branch - module.Data.Stream = streamBranch[len(streamBranch)-1] - + // (in tagless mode we are trusting the "Stream: " text in the source YAML to be accurate) + if !pd.TaglessMode { + match := misc.GetTagImportRegex(pd).FindStringSubmatch(md.TagBranch) + streamBranch := strings.Split(match[2], "-") + // Force stream to be the same as stream name in branch + module.Data.Stream = streamBranch[len(streamBranch)-1] + } log.Println("This module contains the following rpms:") for name := range module.Data.Components.Rpms { pd.Log.Printf("\t- %s", name) diff --git a/pkg/srpmproc/process.go b/pkg/srpmproc/process.go index 309b40b..63e10e4 100644 --- a/pkg/srpmproc/process.go +++ b/pkg/srpmproc/process.go @@ -138,12 +138,12 @@ func NewProcessData(req *ProcessDataRequest) (*data.ProcessData, error) { if req.BranchPrefix == "" { req.BranchPrefix = "r" } - if req.CdnUrl == "" && req.AltLookAside == false { + if req.CdnUrl == "" && !req.AltLookAside { req.CdnUrl = "https://git.centos.org/sources" } // If altlookaside is enabled, and the CdnUrl hasn't been changed, then automatically set it to the default // CentOS Stream (the new pattern very much won't work with the old git.centos.org/sources site) - if (req.CdnUrl == "https://git.centos.org/sources" || req.CdnUrl == "") && req.AltLookAside == true { + if (req.CdnUrl == "https://git.centos.org/sources" || req.CdnUrl == "") && req.AltLookAside { req.CdnUrl = "https://sources.stream.centos.org/sources/rpms" } @@ -290,7 +290,7 @@ func ProcessRPM(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error) { // if we are using "tagless mode", then we need to jump to a completely different import process: // Version info needs to be derived from rpmbuild + spec file, not tags - if pd.TaglessMode == true { + if pd.TaglessMode { result, err := processRPMTagless(pd) return result, err } @@ -780,22 +780,32 @@ func processRPMTagless(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error return nil, fmt.Errorf("Error converting repository into SOURCES + SPECS + .package.metadata format") } - nvrString := getVersionFromSpec(md.Name, localPath, pd.Version) - if nvrString == "" { - return nil, fmt.Errorf("Error using rpm or rpmbuild to build SRPM and determine version info! (tagless mode)") + // call extra function to determine the proper way to convert the tagless branch name. + // c9s becomes r9s (in the usual case), or in the modular case, stream-httpd-2.4-rhel-9.1.0 becomes r9s-stream-httpd-2.4_r9.1.0 + md.PushBranch = taglessBranchName(branch, pd) + + rpmVersion := "" + + // get name-version-release of tagless repo, only if we're not a module repo: + if !pd.ModuleMode { + nvrString := getVersionFromSpec(md.Name, localPath, pd.Version) + if nvrString == "" { + return nil, fmt.Errorf("Error using rpm or rpmbuild to build SRPM and determine version info! (tagless mode)") + } + + // Set version and release fields we extracted (name|version|release are separated by pipes) + pd.PackageVersion = strings.Split(nvrString, "|")[1] + pd.PackageRelease = strings.Split(nvrString, "|")[2] + + // Set full rpm version: name-version-release (for tagging properly) + rpmVersion = fmt.Sprintf("%s-%s-%s", md.Name, pd.PackageVersion, pd.PackageRelease) + + pd.Log.Println("Successfully determined version of tagless checkout: ", rpmVersion) + } else { + // In case of module mode, we just set rpmVersion to the current date - that's what our tag will end up being + rpmVersion = time.Now().Format("2006-01-02") } - // Set version and release fields we extracted (name|version|release are separated by pipes) - pd.PackageVersion = strings.Split(nvrString, "|")[1] - pd.PackageRelease = strings.Split(nvrString, "|")[2] - - // Set full rpm version: name-version-release (for tagging properly) - rpmVersion := fmt.Sprintf("%s-%s-%s", md.Name, pd.PackageVersion, pd.PackageRelease) - - pd.Log.Println("Successfully determined version of tagless checkout: ", rpmVersion) - - md.PushBranch = fmt.Sprintf("%s%d%s", pd.BranchPrefix, pd.Version, pd.BranchSuffix) - // Make an initial repo we will use to push to our target pushRepo, err := git.PlainInit(localPath+"_gitpush", false) if err != nil { @@ -858,7 +868,6 @@ func processRPMTagless(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error // Call function to upload source to target lookaside and // ensure the sources are added to .gitignore - err = processLookasideSources(pd, md, localPath+"_gitpush") if err != nil { return nil, err @@ -883,7 +892,7 @@ func processRPMTagless(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error } status, err := w.Status() - fmt.Println("Current repo status == ", status) + pd.Log.Printf("successfully processed:\n%s", status) // assign tag for our new remote we're about to push (derived from the SRPM version) newTag := "refs/tags/imports/" + md.PushBranch + "/" + rpmVersion @@ -905,7 +914,7 @@ func processRPMTagless(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error } } - if newRepo == true { + if newRepo { pushRefspecs = append(pushRefspecs, config.RefSpec("*:*")) pd.Log.Printf("New remote repo detected, creating new remote branch") } @@ -979,8 +988,6 @@ func processRPMTagless(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error } - fmt.Printf("returning::\n latestHashForBranch == %+v \n\n versionForBranch == %+v\n\n", latestHashForBranch, versionForBranch) - // return struct with all our branch:commit and branch:version+release mappings return &srpmprocpb.ProcessResponse{ BranchCommits: latestHashForBranch, @@ -1302,3 +1309,30 @@ func processLookasideSources(pd *data.ProcessData, md *data.ModeData, localDir s return nil } + +// Given an input branch name to import from, like "refs/heads/c9s", produce the tagless branch name we want to commit to, like "r9s" +// Modular translation of CentOS stream branches i is also done - branch stream-maven-3.8-rhel-9.1.0 ----> r9s-stream-maven-3.8_9.1.0 +func taglessBranchName(fullBranch string, pd *data.ProcessData) string { + + // Split the full branch name "refs/heads/blah" to only get the short name - last entry + tmpBranch := strings.Split(fullBranch, "/") + branch := tmpBranch[len(tmpBranch)-1] + + // Simple case: if our branch is not a modular stream branch, just return the normal pattern + if !strings.HasPrefix(branch, "stream-") { + return fmt.Sprintf("%s%d%s", pd.BranchPrefix, pd.Version, pd.BranchSuffix) + } + + // index where the "-rhel-" starts near the end of the string + rhelSpot := strings.LastIndex(branch, "-rhel-") + + // module name will be everything from the start until that "-rhel-" string (like "stream-httpd-2.4") + moduleString := branch[0:rhelSpot] + + // major minor version is everything after the "-rhel-" string + majorMinor := branch[rhelSpot+6 : len(branch)] + + // return translated modular branch: + return fmt.Sprintf("%s%d%s-%s_%s", pd.BranchPrefix, pd.Version, pd.BranchSuffix, moduleString, majorMinor) + +}