Added modular imports for tagless upstream
- CentOS-Stream import branches are converted from stream-<MODULE_VERSION>-<RHEL_VERSION> to the more familiar r9s-stream-<VERSION> - stream-style YAML is detected and converted for modules, similar to the older modulemd.src.txt files - Various small formatting improvements - This new pattern is for "tagless mode" only, previous tagged imports (from git.centos.org) should not be affected -Skip Grube
This commit is contained in:
Skip Grube
parent
a5ceeeae42
commit
28a8e03470
|
|
@ -5,6 +5,7 @@ import (
|
|||
"github.com/rocky-linux/srpmproc/pkg/data"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func GetTagImportRegex(pd *data.ProcessData) *regexp.Regexp {
|
||||
|
|
@ -31,3 +32,22 @@ func GetTagImportRegex(pd *data.ProcessData) *regexp.Regexp {
|
|||
|
||||
return regexp.MustCompile(regex)
|
||||
}
|
||||
|
||||
// Given a git reference in tagless mode (like "refs/heads/c9s", or "refs/heads/stream-httpd-2.4-rhel-9.1.0"), determine
|
||||
// if we are ok with importing that reference. We are looking for the traditional <prefix><version><suffix> pattern, like "c9s", and also the
|
||||
// modular "stream-<NAME>-<VERSION>-rhel-<VERSION> branch pattern as well
|
||||
func TaglessRefOk(tag string, pd *data.ProcessData) bool {
|
||||
|
||||
// First case is very easy: if we are "refs/heads/<prefix><version><suffix>" , then this is def. a branch we should import
|
||||
if strings.HasPrefix(tag, fmt.Sprintf("refs/heads/%s%d%s", pd.ImportBranchPrefix, pd.Version, pd.BranchSuffix)) {
|
||||
return true
|
||||
}
|
||||
|
||||
// Less easy: if a modular branch is present (starts w/ "stream-"), we need to check if it's part of our major version, and return true if it is
|
||||
// (major version means we look for the text "rhel-X." in the branch name, like "rhel-9.1.0")
|
||||
if strings.HasPrefix(tag, "refs/heads/stream-") && strings.Contains(tag, fmt.Sprintf("rhel-%d.", pd.Version)) {
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
|
|
|||
|
|
@ -128,22 +128,16 @@ func (g *GitMode) RetrieveSource(pd *data.ProcessData) (*data.ModeData, error) {
|
|||
// In case of "tagless mode", we need to get the head ref of the branch instead
|
||||
// This is a kind of alternative implementation of the above tagAdd assignment
|
||||
refAdd := func(tag *object.Tag) error {
|
||||
if strings.HasPrefix(tag.Name, fmt.Sprintf("refs/heads/%s%d%s", pd.ImportBranchPrefix, pd.Version, pd.BranchSuffix)) {
|
||||
if misc.TaglessRefOk(tag.Name, pd) {
|
||||
pd.Log.Printf("Tagless mode: Identified tagless commit for import: %s\n", tag.Name)
|
||||
refSpec := fmt.Sprintf(tag.Name)
|
||||
|
||||
// We split the string by "/", the branch name we're looking for to pass to latestTags is always last
|
||||
// (ex: "refs/heads/c9s" ---> we want latestTags[c9s]
|
||||
_tmpRef := strings.Split(refSpec, "/")
|
||||
_branchName := _tmpRef[(len(_tmpRef) - 1)]
|
||||
tmpRef := strings.Split(refSpec, "/")
|
||||
tmpBranchName := tmpRef[(len(tmpRef) - 1)]
|
||||
|
||||
// In the case of "strict branch mode" on, the branch name must match *exactly* with our prefix-version-suffix (like "c8" cannot also match "c8-beta")
|
||||
// If it doesn't, bail out without adding this branch
|
||||
if pd.StrictBranchMode == true && _branchName != fmt.Sprintf("%s%d%s", pd.ImportBranchPrefix, pd.Version, pd.BranchSuffix) {
|
||||
return nil
|
||||
}
|
||||
|
||||
latestTags[_branchName] = &remoteTarget{
|
||||
latestTags[tmpBranchName] = &remoteTarget{
|
||||
remote: refSpec,
|
||||
when: tag.Tagger.When,
|
||||
}
|
||||
|
|
@ -159,7 +153,7 @@ func (g *GitMode) RetrieveSource(pd *data.ProcessData) (*data.ModeData, error) {
|
|||
|
||||
// tagless mode means we use "refAdd" (add commit by reference)
|
||||
// normal mode means we can rely on "tagAdd" (the tag should be present for us in the source repo)
|
||||
if pd.TaglessMode == true {
|
||||
if pd.TaglessMode {
|
||||
_ = tagIter.ForEach(refAdd)
|
||||
} else {
|
||||
_ = tagIter.ForEach(tagAdd)
|
||||
|
|
@ -192,7 +186,7 @@ func (g *GitMode) RetrieveSource(pd *data.ProcessData) (*data.ModeData, error) {
|
|||
}
|
||||
|
||||
// Call refAdd instead of tagAdd in the case of TaglessMode enabled
|
||||
if pd.TaglessMode == true {
|
||||
if pd.TaglessMode {
|
||||
_ = refAdd(&object.Tag{
|
||||
Name: string(ref.Name()),
|
||||
Tagger: commit.Committer,
|
||||
|
|
@ -230,7 +224,7 @@ func (g *GitMode) WriteSource(pd *data.ProcessData, md *data.ModeData) error {
|
|||
|
||||
remote, err := md.Repo.Remote("upstream")
|
||||
|
||||
if err != nil && pd.TaglessMode == false {
|
||||
if err != nil && !pd.TaglessMode {
|
||||
return fmt.Errorf("could not get upstream remote: %v", err)
|
||||
}
|
||||
|
||||
|
|
@ -239,7 +233,7 @@ func (g *GitMode) WriteSource(pd *data.ProcessData, md *data.ModeData) error {
|
|||
|
||||
// In the case of tagless mode, we already have the transformed repo sitting in the worktree,
|
||||
// and don't need to perform any checkout or fetch operations
|
||||
if pd.TaglessMode == false {
|
||||
if !pd.TaglessMode {
|
||||
if strings.HasPrefix(md.TagBranch, "refs/heads") {
|
||||
refspec = config.RefSpec(fmt.Sprintf("+%s:%s", md.TagBranch, md.TagBranch))
|
||||
branchName = strings.TrimPrefix(md.TagBranch, "refs/heads/")
|
||||
|
|
@ -285,7 +279,7 @@ func (g *GitMode) WriteSource(pd *data.ProcessData, md *data.ModeData) error {
|
|||
}
|
||||
}
|
||||
|
||||
if pd.TaglessMode == true {
|
||||
if pd.TaglessMode {
|
||||
branchName = fmt.Sprintf("%s%d%s", pd.ImportBranchPrefix, pd.Version, pd.BranchSuffix)
|
||||
}
|
||||
|
||||
|
|
@ -349,7 +343,7 @@ func (g *GitMode) WriteSource(pd *data.ProcessData, md *data.ModeData) error {
|
|||
|
||||
url := ""
|
||||
// Alternate lookaside logic: if enabled, we pull from a new URL pattern
|
||||
if pd.AltLookAside == false {
|
||||
if !pd.AltLookAside {
|
||||
url = fmt.Sprintf("%s/%s/%s/%s", pd.CdnUrl, md.Name, branchName, hash)
|
||||
} else {
|
||||
// We first need the hash algorithm based on length of hash:
|
||||
|
|
|
|||
|
|
@ -32,6 +32,7 @@ import (
|
|||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/go-git/go-billy/v5"
|
||||
"github.com/go-git/go-billy/v5/memfs"
|
||||
"github.com/go-git/go-git/v5"
|
||||
"github.com/go-git/go-git/v5/config"
|
||||
|
|
@ -286,13 +287,30 @@ func patchModuleYaml(pd *data.ProcessData, md *data.ModeData) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
mdTxtPath := "SOURCES/modulemd.src.txt"
|
||||
f, err := md.Worktree.Filesystem.Open(mdTxtPath)
|
||||
if err != nil {
|
||||
mdTxtPath = "SOURCES/modulemd.txt"
|
||||
mdTxtPath := ""
|
||||
var f billy.File
|
||||
|
||||
// tagless mode implies we're looking for CentOS Stream modules, which are generally "SOURCES/NAME.yaml" (copied to SOURCES/ from import)
|
||||
// if not tagless mode, proceed as usual with SOURCES/modulemd.*.txt
|
||||
if pd.TaglessMode {
|
||||
mdTxtPath = fmt.Sprintf("SOURCES/%s.yaml", md.Name)
|
||||
f, err = md.Worktree.Filesystem.Open(mdTxtPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not open modulemd file: %v", err)
|
||||
mdTxtPath = fmt.Sprintf("SOURCES/%s.yml", md.Name)
|
||||
f, err = md.Worktree.Filesystem.Open(mdTxtPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not open modulemd file: %v", err)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
mdTxtPath = "SOURCES/modulemd.src.txt"
|
||||
f, err = md.Worktree.Filesystem.Open(mdTxtPath)
|
||||
if err != nil {
|
||||
mdTxtPath = "SOURCES/modulemd.txt"
|
||||
f, err = md.Worktree.Filesystem.Open(mdTxtPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not open modulemd file: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -307,11 +325,13 @@ func patchModuleYaml(pd *data.ProcessData, md *data.ModeData) error {
|
|||
}
|
||||
|
||||
// Get stream branch from tag
|
||||
match := misc.GetTagImportRegex(pd).FindStringSubmatch(md.TagBranch)
|
||||
streamBranch := strings.Split(match[2], "-")
|
||||
// Force stream to be the same as stream name in branch
|
||||
module.Data.Stream = streamBranch[len(streamBranch)-1]
|
||||
|
||||
// (in tagless mode we are trusting the "Stream: <VERSION>" text in the source YAML to be accurate)
|
||||
if !pd.TaglessMode {
|
||||
match := misc.GetTagImportRegex(pd).FindStringSubmatch(md.TagBranch)
|
||||
streamBranch := strings.Split(match[2], "-")
|
||||
// Force stream to be the same as stream name in branch
|
||||
module.Data.Stream = streamBranch[len(streamBranch)-1]
|
||||
}
|
||||
log.Println("This module contains the following rpms:")
|
||||
for name := range module.Data.Components.Rpms {
|
||||
pd.Log.Printf("\t- %s", name)
|
||||
|
|
|
|||
|
|
@ -138,12 +138,12 @@ func NewProcessData(req *ProcessDataRequest) (*data.ProcessData, error) {
|
|||
if req.BranchPrefix == "" {
|
||||
req.BranchPrefix = "r"
|
||||
}
|
||||
if req.CdnUrl == "" && req.AltLookAside == false {
|
||||
if req.CdnUrl == "" && !req.AltLookAside {
|
||||
req.CdnUrl = "https://git.centos.org/sources"
|
||||
}
|
||||
// If altlookaside is enabled, and the CdnUrl hasn't been changed, then automatically set it to the default
|
||||
// CentOS Stream (the new pattern very much won't work with the old git.centos.org/sources site)
|
||||
if (req.CdnUrl == "https://git.centos.org/sources" || req.CdnUrl == "") && req.AltLookAside == true {
|
||||
if (req.CdnUrl == "https://git.centos.org/sources" || req.CdnUrl == "") && req.AltLookAside {
|
||||
req.CdnUrl = "https://sources.stream.centos.org/sources/rpms"
|
||||
}
|
||||
|
||||
|
|
@ -290,7 +290,7 @@ func ProcessRPM(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error) {
|
|||
|
||||
// if we are using "tagless mode", then we need to jump to a completely different import process:
|
||||
// Version info needs to be derived from rpmbuild + spec file, not tags
|
||||
if pd.TaglessMode == true {
|
||||
if pd.TaglessMode {
|
||||
result, err := processRPMTagless(pd)
|
||||
return result, err
|
||||
}
|
||||
|
|
@ -780,22 +780,32 @@ func processRPMTagless(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error
|
|||
return nil, fmt.Errorf("Error converting repository into SOURCES + SPECS + .package.metadata format")
|
||||
}
|
||||
|
||||
nvrString := getVersionFromSpec(md.Name, localPath, pd.Version)
|
||||
if nvrString == "" {
|
||||
return nil, fmt.Errorf("Error using rpm or rpmbuild to build SRPM and determine version info! (tagless mode)")
|
||||
// call extra function to determine the proper way to convert the tagless branch name.
|
||||
// c9s becomes r9s (in the usual case), or in the modular case, stream-httpd-2.4-rhel-9.1.0 becomes r9s-stream-httpd-2.4_r9.1.0
|
||||
md.PushBranch = taglessBranchName(branch, pd)
|
||||
|
||||
rpmVersion := ""
|
||||
|
||||
// get name-version-release of tagless repo, only if we're not a module repo:
|
||||
if !pd.ModuleMode {
|
||||
nvrString := getVersionFromSpec(md.Name, localPath, pd.Version)
|
||||
if nvrString == "" {
|
||||
return nil, fmt.Errorf("Error using rpm or rpmbuild to build SRPM and determine version info! (tagless mode)")
|
||||
}
|
||||
|
||||
// Set version and release fields we extracted (name|version|release are separated by pipes)
|
||||
pd.PackageVersion = strings.Split(nvrString, "|")[1]
|
||||
pd.PackageRelease = strings.Split(nvrString, "|")[2]
|
||||
|
||||
// Set full rpm version: name-version-release (for tagging properly)
|
||||
rpmVersion = fmt.Sprintf("%s-%s-%s", md.Name, pd.PackageVersion, pd.PackageRelease)
|
||||
|
||||
pd.Log.Println("Successfully determined version of tagless checkout: ", rpmVersion)
|
||||
} else {
|
||||
// In case of module mode, we just set rpmVersion to the current date - that's what our tag will end up being
|
||||
rpmVersion = time.Now().Format("2006-01-02")
|
||||
}
|
||||
|
||||
// Set version and release fields we extracted (name|version|release are separated by pipes)
|
||||
pd.PackageVersion = strings.Split(nvrString, "|")[1]
|
||||
pd.PackageRelease = strings.Split(nvrString, "|")[2]
|
||||
|
||||
// Set full rpm version: name-version-release (for tagging properly)
|
||||
rpmVersion := fmt.Sprintf("%s-%s-%s", md.Name, pd.PackageVersion, pd.PackageRelease)
|
||||
|
||||
pd.Log.Println("Successfully determined version of tagless checkout: ", rpmVersion)
|
||||
|
||||
md.PushBranch = fmt.Sprintf("%s%d%s", pd.BranchPrefix, pd.Version, pd.BranchSuffix)
|
||||
|
||||
// Make an initial repo we will use to push to our target
|
||||
pushRepo, err := git.PlainInit(localPath+"_gitpush", false)
|
||||
if err != nil {
|
||||
|
|
@ -858,7 +868,6 @@ func processRPMTagless(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error
|
|||
|
||||
// Call function to upload source to target lookaside and
|
||||
// ensure the sources are added to .gitignore
|
||||
|
||||
err = processLookasideSources(pd, md, localPath+"_gitpush")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
|
@ -883,7 +892,7 @@ func processRPMTagless(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error
|
|||
}
|
||||
|
||||
status, err := w.Status()
|
||||
fmt.Println("Current repo status == ", status)
|
||||
pd.Log.Printf("successfully processed:\n%s", status)
|
||||
|
||||
// assign tag for our new remote we're about to push (derived from the SRPM version)
|
||||
newTag := "refs/tags/imports/" + md.PushBranch + "/" + rpmVersion
|
||||
|
|
@ -905,7 +914,7 @@ func processRPMTagless(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error
|
|||
}
|
||||
}
|
||||
|
||||
if newRepo == true {
|
||||
if newRepo {
|
||||
pushRefspecs = append(pushRefspecs, config.RefSpec("*:*"))
|
||||
pd.Log.Printf("New remote repo detected, creating new remote branch")
|
||||
}
|
||||
|
|
@ -979,8 +988,6 @@ func processRPMTagless(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error
|
|||
|
||||
}
|
||||
|
||||
fmt.Printf("returning::\n latestHashForBranch == %+v \n\n versionForBranch == %+v\n\n", latestHashForBranch, versionForBranch)
|
||||
|
||||
// return struct with all our branch:commit and branch:version+release mappings
|
||||
return &srpmprocpb.ProcessResponse{
|
||||
BranchCommits: latestHashForBranch,
|
||||
|
|
@ -1302,3 +1309,30 @@ func processLookasideSources(pd *data.ProcessData, md *data.ModeData, localDir s
|
|||
return nil
|
||||
|
||||
}
|
||||
|
||||
// Given an input branch name to import from, like "refs/heads/c9s", produce the tagless branch name we want to commit to, like "r9s"
|
||||
// Modular translation of CentOS stream branches i is also done - branch stream-maven-3.8-rhel-9.1.0 ----> r9s-stream-maven-3.8_9.1.0
|
||||
func taglessBranchName(fullBranch string, pd *data.ProcessData) string {
|
||||
|
||||
// Split the full branch name "refs/heads/blah" to only get the short name - last entry
|
||||
tmpBranch := strings.Split(fullBranch, "/")
|
||||
branch := tmpBranch[len(tmpBranch)-1]
|
||||
|
||||
// Simple case: if our branch is not a modular stream branch, just return the normal <prefix><version><suffix> pattern
|
||||
if !strings.HasPrefix(branch, "stream-") {
|
||||
return fmt.Sprintf("%s%d%s", pd.BranchPrefix, pd.Version, pd.BranchSuffix)
|
||||
}
|
||||
|
||||
// index where the "-rhel-" starts near the end of the string
|
||||
rhelSpot := strings.LastIndex(branch, "-rhel-")
|
||||
|
||||
// module name will be everything from the start until that "-rhel-" string (like "stream-httpd-2.4")
|
||||
moduleString := branch[0:rhelSpot]
|
||||
|
||||
// major minor version is everything after the "-rhel-" string
|
||||
majorMinor := branch[rhelSpot+6 : len(branch)]
|
||||
|
||||
// return translated modular branch:
|
||||
return fmt.Sprintf("%s%d%s-%s_%s", pd.BranchPrefix, pd.Version, pd.BranchSuffix, moduleString, majorMinor)
|
||||
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue