diff --git a/cmd/srpmproc/main.go b/cmd/srpmproc/main.go index 38be0da..fe6f966 100644 --- a/cmd/srpmproc/main.go +++ b/cmd/srpmproc/main.go @@ -56,6 +56,8 @@ var ( basicPassword string packageVersion string packageRelease string + taglessMode bool + altLookAside bool ) var root = &cobra.Command{ @@ -92,7 +94,10 @@ func mn(_ *cobra.Command, _ []string) { HttpPassword: basicPassword, PackageVersion: packageVersion, PackageRelease: packageRelease, + TaglessMode: taglessMode, + AltLookAside: altLookAside, }) + if err != nil { log.Fatal(err) } @@ -142,6 +147,8 @@ func main() { root.Flags().StringVar(&basicPassword, "basic-password", "", "Basic auth password") root.Flags().StringVar(&packageVersion, "package-version", "", "Package version to fetch") root.Flags().StringVar(&packageRelease, "package-release", "", "Package release to fetch") + root.Flags().BoolVar(&taglessMode, "taglessmode", false, "Tagless mode: If set, pull the latest commit from a branch, and determine version info from spec file (aka upstream versions aren't tagged)") + root.Flags().BoolVar(&altLookAside, "altlookaside", false, "If set, uses the new CentOS Stream lookaside pattern (https:///////)") if err := root.Execute(); err != nil { log.Fatal(err) diff --git a/pkg/data/process.go b/pkg/data/process.go index bbfd694..72d9a61 100644 --- a/pkg/data/process.go +++ b/pkg/data/process.go @@ -57,4 +57,6 @@ type ProcessData struct { Log *log.Logger PackageVersion string PackageRelease string + TaglessMode bool + AltLookAside bool } diff --git a/pkg/misc/regex.go b/pkg/misc/regex.go index 254b6f1..3cc6f0b 100644 --- a/pkg/misc/regex.go +++ b/pkg/misc/regex.go @@ -5,6 +5,7 @@ import ( "github.com/rocky-linux/srpmproc/pkg/data" "path/filepath" "regexp" + "strings" ) func GetTagImportRegex(pd *data.ProcessData) *regexp.Regexp { @@ -31,3 +32,22 @@ func GetTagImportRegex(pd *data.ProcessData) *regexp.Regexp { return regexp.MustCompile(regex) } + +// Given a git reference in tagless mode (like "refs/heads/c9s", or "refs/heads/stream-httpd-2.4-rhel-9.1.0"), determine +// if we are ok with importing that reference. We are looking for the traditional pattern, like "c9s", and also the +// modular "stream---rhel- branch pattern as well +func TaglessRefOk(tag string, pd *data.ProcessData) bool { + + // First case is very easy: if we are "refs/heads/" , then this is def. a branch we should import + if strings.HasPrefix(tag, fmt.Sprintf("refs/heads/%s%d%s", pd.ImportBranchPrefix, pd.Version, pd.BranchSuffix)) { + return true + } + + // Less easy: if a modular branch is present (starts w/ "stream-"), we need to check if it's part of our major version, and return true if it is + // (major version means we look for the text "rhel-X." in the branch name, like "rhel-9.1.0") + if strings.HasPrefix(tag, "refs/heads/stream-") && strings.Contains(tag, fmt.Sprintf("rhel-%d.", pd.Version)) { + return true + } + + return false +} diff --git a/pkg/modes/git.go b/pkg/modes/git.go index dee39e7..d68d7e4 100644 --- a/pkg/modes/git.go +++ b/pkg/modes/git.go @@ -116,7 +116,6 @@ func (g *GitMode) RetrieveSource(pd *data.ProcessData) (*data.ModeData, error) { if exists != nil && exists.when.After(tag.Tagger.When) { return nil } - latestTags[match[2]] = &remoteTarget{ remote: refSpec, when: tag.Tagger.When, @@ -126,11 +125,39 @@ func (g *GitMode) RetrieveSource(pd *data.ProcessData) (*data.ModeData, error) { return nil } + // In case of "tagless mode", we need to get the head ref of the branch instead + // This is a kind of alternative implementation of the above tagAdd assignment + refAdd := func(tag *object.Tag) error { + if misc.TaglessRefOk(tag.Name, pd) { + pd.Log.Printf("Tagless mode: Identified tagless commit for import: %s\n", tag.Name) + refSpec := fmt.Sprintf(tag.Name) + + // We split the string by "/", the branch name we're looking for to pass to latestTags is always last + // (ex: "refs/heads/c9s" ---> we want latestTags[c9s] + tmpRef := strings.Split(refSpec, "/") + tmpBranchName := tmpRef[(len(tmpRef) - 1)] + + latestTags[tmpBranchName] = &remoteTarget{ + remote: refSpec, + when: tag.Tagger.When, + } + } + return nil + } + tagIter, err := repo.TagObjects() + if err != nil { return nil, fmt.Errorf("could not get tag objects: %v", err) } - _ = tagIter.ForEach(tagAdd) + + // tagless mode means we use "refAdd" (add commit by reference) + // normal mode means we can rely on "tagAdd" (the tag should be present for us in the source repo) + if pd.TaglessMode { + _ = tagIter.ForEach(refAdd) + } else { + _ = tagIter.ForEach(tagAdd) + } listOpts := &git.ListOptions{ Auth: pd.Authenticator, @@ -157,17 +184,26 @@ func (g *GitMode) RetrieveSource(pd *data.ProcessData) (*data.ModeData, error) { if err != nil { continue } - _ = tagAdd(&object.Tag{ - Name: strings.TrimPrefix(string(ref.Name()), "refs/tags/"), - Tagger: commit.Committer, - }) + + // Call refAdd instead of tagAdd in the case of TaglessMode enabled + if pd.TaglessMode { + _ = refAdd(&object.Tag{ + Name: string(ref.Name()), + Tagger: commit.Committer, + }) + } else { + _ = tagAdd(&object.Tag{ + Name: strings.TrimPrefix(string(ref.Name()), "refs/tags/"), + Tagger: commit.Committer, + }) + } + } for _, branch := range latestTags { pd.Log.Printf("tag: %s", strings.TrimPrefix(branch.remote, "refs/tags/")) branches = append(branches, *branch) } - sort.Sort(branches) var sortedBranches []string @@ -185,54 +221,66 @@ func (g *GitMode) RetrieveSource(pd *data.ProcessData) (*data.ModeData, error) { } func (g *GitMode) WriteSource(pd *data.ProcessData, md *data.ModeData) error { + remote, err := md.Repo.Remote("upstream") - if err != nil { + + if err != nil && !pd.TaglessMode { return fmt.Errorf("could not get upstream remote: %v", err) } var refspec config.RefSpec var branchName string - if strings.HasPrefix(md.TagBranch, "refs/heads") { - refspec = config.RefSpec(fmt.Sprintf("+%s:%s", md.TagBranch, md.TagBranch)) - branchName = strings.TrimPrefix(md.TagBranch, "refs/heads/") - } else { - match := misc.GetTagImportRegex(pd).FindStringSubmatch(md.TagBranch) - branchName = match[2] - refspec = config.RefSpec(fmt.Sprintf("+refs/heads/%s:%s", branchName, md.TagBranch)) - } - pd.Log.Printf("checking out upstream refspec %s", refspec) - fetchOpts := &git.FetchOptions{ - Auth: pd.Authenticator, - RemoteName: "upstream", - RefSpecs: []config.RefSpec{refspec}, - Tags: git.AllTags, - Force: true, - } - err = remote.Fetch(fetchOpts) - if err != nil && err != git.NoErrAlreadyUpToDate { - if err == transport.ErrInvalidAuthMethod || err == transport.ErrAuthenticationRequired { - fetchOpts.Auth = nil - err = remote.Fetch(fetchOpts) - if err != nil && err != git.NoErrAlreadyUpToDate { + // In the case of tagless mode, we already have the transformed repo sitting in the worktree, + // and don't need to perform any checkout or fetch operations + if !pd.TaglessMode { + if strings.HasPrefix(md.TagBranch, "refs/heads") { + refspec = config.RefSpec(fmt.Sprintf("+%s:%s", md.TagBranch, md.TagBranch)) + branchName = strings.TrimPrefix(md.TagBranch, "refs/heads/") + } else { + match := misc.GetTagImportRegex(pd).FindStringSubmatch(md.TagBranch) + branchName = match[2] + refspec = config.RefSpec(fmt.Sprintf("+refs/heads/%s:%s", branchName, md.TagBranch)) + fmt.Println("Found branchname that does not start w/ refs/heads :: ", branchName) + } + pd.Log.Printf("checking out upstream refspec %s", refspec) + + fetchOpts := &git.FetchOptions{ + Auth: pd.Authenticator, + RemoteName: "upstream", + RefSpecs: []config.RefSpec{refspec}, + Tags: git.AllTags, + Force: true, + } + err = remote.Fetch(fetchOpts) + if err != nil && err != git.NoErrAlreadyUpToDate { + if err == transport.ErrInvalidAuthMethod || err == transport.ErrAuthenticationRequired { + fetchOpts.Auth = nil + err = remote.Fetch(fetchOpts) + if err != nil && err != git.NoErrAlreadyUpToDate { + return fmt.Errorf("could not fetch upstream: %v", err) + } + } else { return fmt.Errorf("could not fetch upstream: %v", err) } - } else { - return fmt.Errorf("could not fetch upstream: %v", err) + } + + err = md.Worktree.Checkout(&git.CheckoutOptions{ + Branch: plumbing.ReferenceName(md.TagBranch), + Force: true, + }) + if err != nil { + return fmt.Errorf("could not checkout source from git: %v", err) + } + + _, err = md.Worktree.Add(".") + if err != nil { + return fmt.Errorf("could not add Worktree: %v", err) } } - err = md.Worktree.Checkout(&git.CheckoutOptions{ - Branch: plumbing.ReferenceName(md.TagBranch), - Force: true, - }) - if err != nil { - return fmt.Errorf("could not checkout source from git: %v", err) - } - - _, err = md.Worktree.Add(".") - if err != nil { - return fmt.Errorf("could not add Worktree: %v", err) + if pd.TaglessMode { + branchName = fmt.Sprintf("%s%d%s", pd.ImportBranchPrefix, pd.Version, pd.BranchSuffix) } metadataPath := "" @@ -292,7 +340,32 @@ func (g *GitMode) WriteSource(pd *data.ProcessData, md *data.ModeData) error { body = fromBlobStorage pd.Log.Printf("downloading %s from blob storage", hash) } else { - url := fmt.Sprintf("%s/%s/%s/%s", pd.CdnUrl, md.Name, branchName, hash) + + url := "" + // Alternate lookaside logic: if enabled, we pull from a new URL pattern + if !pd.AltLookAside { + url = fmt.Sprintf("%s/%s/%s/%s", pd.CdnUrl, md.Name, branchName, hash) + } else { + // We first need the hash algorithm based on length of hash: + hashType := "sha512" + switch len(hash) { + case 128: + hashType = "sha512" + case 64: + hashType = "sha256" + case 40: + hashType = "sha1" + case 32: + hashType = "md5" + } + + // need the name of the file without "SOURCES/": + fileName := strings.Split(path, "/")[1] + + // Alt. lookaside url is of the form: / / / / / + url = fmt.Sprintf("%s/%s/%s/%s/%s/%s", pd.CdnUrl, md.Name, fileName, hashType, hash, fileName) + } + pd.Log.Printf("downloading %s", url) req, err := http.NewRequest("GET", url, nil) diff --git a/pkg/srpmproc/patch.go b/pkg/srpmproc/patch.go index b2ada0d..b2f739c 100644 --- a/pkg/srpmproc/patch.go +++ b/pkg/srpmproc/patch.go @@ -32,6 +32,7 @@ import ( "strings" "time" + "github.com/go-git/go-billy/v5" "github.com/go-git/go-billy/v5/memfs" "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/config" @@ -286,13 +287,30 @@ func patchModuleYaml(pd *data.ProcessData, md *data.ModeData) error { return nil } - mdTxtPath := "SOURCES/modulemd.src.txt" - f, err := md.Worktree.Filesystem.Open(mdTxtPath) - if err != nil { - mdTxtPath = "SOURCES/modulemd.txt" + mdTxtPath := "" + var f billy.File + + // tagless mode implies we're looking for CentOS Stream modules, which are generally "SOURCES/NAME.yaml" (copied to SOURCES/ from import) + // if not tagless mode, proceed as usual with SOURCES/modulemd.*.txt + if pd.TaglessMode { + mdTxtPath = fmt.Sprintf("SOURCES/%s.yaml", md.Name) f, err = md.Worktree.Filesystem.Open(mdTxtPath) if err != nil { - return fmt.Errorf("could not open modulemd file: %v", err) + mdTxtPath = fmt.Sprintf("SOURCES/%s.yml", md.Name) + f, err = md.Worktree.Filesystem.Open(mdTxtPath) + if err != nil { + return fmt.Errorf("could not open modulemd file: %v", err) + } + } + } else { + mdTxtPath = "SOURCES/modulemd.src.txt" + f, err = md.Worktree.Filesystem.Open(mdTxtPath) + if err != nil { + mdTxtPath = "SOURCES/modulemd.txt" + f, err = md.Worktree.Filesystem.Open(mdTxtPath) + if err != nil { + return fmt.Errorf("could not open modulemd file: %v", err) + } } } @@ -307,11 +325,13 @@ func patchModuleYaml(pd *data.ProcessData, md *data.ModeData) error { } // Get stream branch from tag - match := misc.GetTagImportRegex(pd).FindStringSubmatch(md.TagBranch) - streamBranch := strings.Split(match[2], "-") - // Force stream to be the same as stream name in branch - module.Data.Stream = streamBranch[len(streamBranch)-1] - + // (in tagless mode we are trusting the "Stream: " text in the source YAML to be accurate) + if !pd.TaglessMode { + match := misc.GetTagImportRegex(pd).FindStringSubmatch(md.TagBranch) + streamBranch := strings.Split(match[2], "-") + // Force stream to be the same as stream name in branch + module.Data.Stream = streamBranch[len(streamBranch)-1] + } log.Println("This module contains the following rpms:") for name := range module.Data.Components.Rpms { pd.Log.Printf("\t- %s", name) diff --git a/pkg/srpmproc/process.go b/pkg/srpmproc/process.go index bbba9d8..de118ee 100644 --- a/pkg/srpmproc/process.go +++ b/pkg/srpmproc/process.go @@ -21,10 +21,12 @@ package srpmproc import ( + "bufio" "encoding/hex" "fmt" "github.com/go-git/go-billy/v5" "github.com/go-git/go-billy/v5/osfs" + "github.com/go-git/go-git/v5/plumbing/format/gitignore" "github.com/go-git/go-git/v5/plumbing/transport" "github.com/go-git/go-git/v5/plumbing/transport/http" "github.com/go-git/go-git/v5/plumbing/transport/ssh" @@ -40,6 +42,7 @@ import ( "io/ioutil" "log" "os" + "os/exec" "os/user" "path/filepath" "strings" @@ -96,6 +99,9 @@ type ProcessDataRequest struct { PackageVersion string PackageRelease string + + TaglessMode bool + AltLookAside bool } func gitlabify(str string) string { @@ -132,9 +138,14 @@ func NewProcessData(req *ProcessDataRequest) (*data.ProcessData, error) { if req.BranchPrefix == "" { req.BranchPrefix = "r" } - if req.CdnUrl == "" { + if req.CdnUrl == "" && !req.AltLookAside { req.CdnUrl = "https://git.centos.org/sources" } + // If altlookaside is enabled, and the CdnUrl hasn't been changed, then automatically set it to the default + // CentOS Stream (the new pattern very much won't work with the old git.centos.org/sources site) + if (req.CdnUrl == "https://git.centos.org/sources" || req.CdnUrl == "") && req.AltLookAside { + req.CdnUrl = "https://sources.stream.centos.org/sources/rpms" + } // Validate required if req.Package == "" { @@ -263,6 +274,8 @@ func NewProcessData(req *ProcessDataRequest) (*data.ProcessData, error) { Log: logger, PackageVersion: req.PackageVersion, PackageRelease: req.PackageRelease, + TaglessMode: req.TaglessMode, + AltLookAside: req.AltLookAside, }, nil } @@ -274,6 +287,14 @@ func NewProcessData(req *ProcessDataRequest) (*data.ProcessData, error) { // all files that are remote goes into .gitignore // all ignored files' hash goes into .{Name}.metadata func ProcessRPM(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error) { + + // if we are using "tagless mode", then we need to jump to a completely different import process: + // Version info needs to be derived from rpmbuild + spec file, not tags + if pd.TaglessMode { + result, err := processRPMTagless(pd) + return result, err + } + md, err := pd.Importer.RetrieveSource(pd) if err != nil { return nil, err @@ -372,7 +393,9 @@ func ProcessRPM(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error) { } match := misc.GetTagImportRegex(pd).FindStringSubmatch(matchString) + md.PushBranch = pd.BranchPrefix + strings.TrimPrefix(match[2], pd.ImportBranchPrefix) + newTag := "imports/" + pd.BranchPrefix + strings.TrimPrefix(match[1], "imports/"+pd.ImportBranchPrefix) newTag = strings.Replace(newTag, "%", "_", -1) @@ -656,3 +679,660 @@ func ProcessRPM(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error) { BranchVersions: versionForBranch, }, nil } + +// Process for when we want to import a tagless repo (like from CentOS Stream) +// +func processRPMTagless(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error) { + pd.Log.Println("Tagless mode detected, attempting import of latest commit") + + // In tagless mode, we *automatically* set StrictBranchMode to true + // Only the exact branch should be pulled from the source repo + pd.StrictBranchMode = true + + // our return values: a mapping of branches -> commits (1:1) that we're bringing in, + // and a mapping of branches to: version = X, release = Y + latestHashForBranch := map[string]string{} + versionForBranch := map[string]*srpmprocpb.VersionRelease{} + + md, err := pd.Importer.RetrieveSource(pd) + if err != nil { + pd.Log.Println("Error detected in RetrieveSource!") + return nil, err + } + + md.BlobCache = map[string][]byte{} + + // TODO: add tagless module support + remotePrefix := "rpms" + if pd.ModuleMode { + remotePrefix = "modules" + } + + // Set up our remote URL for pushing our repo to + var tagIgnoreList []string + if pd.NoDupMode { + repo, err := git.Init(memory.NewStorage(), memfs.New()) + if err != nil { + return nil, fmt.Errorf("could not init git repo: %v", err) + } + remoteUrl := fmt.Sprintf("%s/%s/%s.git", pd.UpstreamPrefix, remotePrefix, gitlabify(md.Name)) + refspec := config.RefSpec("+refs/heads/*:refs/remotes/origin/*") + + remote, err := repo.CreateRemote(&config.RemoteConfig{ + Name: "origin", + URLs: []string{remoteUrl}, + Fetch: []config.RefSpec{refspec}, + }) + if err != nil { + return nil, fmt.Errorf("could not create remote: %v", err) + } + + list, err := remote.List(&git.ListOptions{ + Auth: pd.Authenticator, + }) + if err != nil { + log.Println("ignoring no-dup-mode") + } else { + for _, ref := range list { + if !strings.HasPrefix(string(ref.Name()), "refs/tags/imports") { + continue + } + tagIgnoreList = append(tagIgnoreList, string(ref.Name())) + } + } + } + + sourceRepo := *md.Repo + sourceWorktree := *md.Worktree + localPath := "" + + for _, branch := range md.Branches { + md.Repo = &sourceRepo + md.Worktree = &sourceWorktree + md.TagBranch = branch + + for _, source := range md.SourcesToIgnore { + source.Expired = true + } + + // Create a temporary place to check out our tag/branch : /tmp/srpmproctmp_/ + localPath, _ = os.MkdirTemp("/tmp", fmt.Sprintf("srpmproctmp_%s", md.Name)) + + if err := os.RemoveAll(localPath); err != nil { + return nil, fmt.Errorf("Could not remove previous temporary directory: %s", localPath) + } + if err := os.Mkdir(localPath, 0755); err != nil { + return nil, fmt.Errorf("Could not create temporary directory: %s", localPath) + } + + // Clone repo into the temporary path, but only the tag we're interested in: + // (TODO: will probably need to assign this a variable or use the md struct gitrepo object to perform a successful tag+push later) + _, _ = git.PlainClone(localPath, false, &git.CloneOptions{ + URL: pd.RpmLocation, + SingleBranch: true, + ReferenceName: plumbing.ReferenceName(branch), + }) + + // Now that we're cloned into localPath, we need to "covert" the import into the old format + // We want sources to become .PKGNAME.metadata, we want SOURCES and SPECS folders, etc. + repoFixed, _ := convertLocalRepo(md.Name, localPath) + if !repoFixed { + return nil, fmt.Errorf("Error converting repository into SOURCES + SPECS + .package.metadata format") + } + + // call extra function to determine the proper way to convert the tagless branch name. + // c9s becomes r9s (in the usual case), or in the modular case, stream-httpd-2.4-rhel-9.1.0 becomes r9s-stream-httpd-2.4_r9.1.0 + md.PushBranch = taglessBranchName(branch, pd) + + rpmVersion := "" + + // get name-version-release of tagless repo, only if we're not a module repo: + if !pd.ModuleMode { + nvrString := getVersionFromSpec(md.Name, localPath, pd.Version) + if nvrString == "" { + return nil, fmt.Errorf("Error using rpm or rpmbuild to build SRPM and determine version info! (tagless mode)") + } + + // Set version and release fields we extracted (name|version|release are separated by pipes) + pd.PackageVersion = strings.Split(nvrString, "|")[1] + pd.PackageRelease = strings.Split(nvrString, "|")[2] + + // Set full rpm version: name-version-release (for tagging properly) + rpmVersion = fmt.Sprintf("%s-%s-%s", md.Name, pd.PackageVersion, pd.PackageRelease) + + pd.Log.Println("Successfully determined version of tagless checkout: ", rpmVersion) + } else { + // In case of module mode, we just set rpmVersion to the current date - that's what our tag will end up being + rpmVersion = time.Now().Format("2006-01-02") + } + + // Make an initial repo we will use to push to our target + pushRepo, err := git.PlainInit(localPath+"_gitpush", false) + if err != nil { + return nil, fmt.Errorf("could not create new dist Repo: %v", err) + } + + w, err := pushRepo.Worktree() + if err != nil { + return nil, fmt.Errorf("could not get dist Worktree: %v", err) + } + + // Create a remote "origin" in our empty git, make the upstream equal to the branch we want to modify + pushUrl := fmt.Sprintf("%s/%s/%s.git", pd.UpstreamPrefix, remotePrefix, gitlabify(md.Name)) + refspec := config.RefSpec(fmt.Sprintf("+refs/heads/%s:refs/remotes/origin/%s", md.PushBranch, md.PushBranch)) + + // Make our remote repo the target one - the one we want to push our update to + pushRepoRemote, err := pushRepo.CreateRemote(&config.RemoteConfig{ + Name: "origin", + URLs: []string{pushUrl}, + Fetch: []config.RefSpec{refspec}, + }) + if err != nil { + return nil, fmt.Errorf("could not create remote: %v", err) + } + + // fetch our branch data (md.PushBranch) into this new repo + err = pushRepo.Fetch(&git.FetchOptions{ + RemoteName: "origin", + RefSpecs: []config.RefSpec{refspec}, + Auth: pd.Authenticator, + }) + + refName := plumbing.NewBranchReferenceName(md.PushBranch) + + var hash plumbing.Hash + h := plumbing.NewSymbolicReference(plumbing.HEAD, refName) + if err := pushRepo.Storer.CheckAndSetReference(h, nil); err != nil { + return nil, fmt.Errorf("Could not set symbolic reference: %v", err) + } + + err = w.Checkout(&git.CheckoutOptions{ + Branch: plumbing.NewRemoteReferenceName("origin", md.PushBranch), + Hash: hash, + Force: true, + }) + + os.Rename(fmt.Sprintf("%s/SPECS", localPath), fmt.Sprintf("%s_gitpush/SPECS", localPath)) + os.Rename(fmt.Sprintf("%s/SOURCES", localPath), fmt.Sprintf("%s_gitpush/SOURCES", localPath)) + os.Rename(fmt.Sprintf("%s/.gitignore", localPath), fmt.Sprintf("%s_gitpush/.gitignore", localPath)) + os.Rename(fmt.Sprintf("%s/.%s.metadata", localPath, md.Name), fmt.Sprintf("%s_gitpush/.%s.metadata", localPath, md.Name)) + + md.Repo = pushRepo + md.Worktree = w + + // Download lookaside sources (tarballs) into the push git repo: + err = pd.Importer.WriteSource(pd, md) + if err != nil { + return nil, err + } + + // Call function to upload source to target lookaside and + // ensure the sources are added to .gitignore + err = processLookasideSources(pd, md, localPath+"_gitpush") + if err != nil { + return nil, err + } + + // Apply patch(es) if needed: + if pd.ModuleMode { + err := patchModuleYaml(pd, md) + if err != nil { + return nil, err + } + } else { + err := executePatchesRpm(pd, md) + if err != nil { + return nil, err + } + } + + err = w.AddWithOptions(&git.AddOptions{All: true}) + if err != nil { + return nil, fmt.Errorf("Error adding SOURCES/ , SPECS/ or .metadata file to commit list.") + } + + status, err := w.Status() + pd.Log.Printf("successfully processed:\n%s", status) + + // assign tag for our new remote we're about to push (derived from the SRPM version) + newTag := "refs/tags/imports/" + md.PushBranch + "/" + rpmVersion + newTag = strings.Replace(newTag, "%", "_", -1) + + // pushRefspecs is a list of all the references we want to push (tags + heads) + // It's an array of colon-separated strings which map local references to their remote counterparts + var pushRefspecs []config.RefSpec + + // We need to find out if the remote repo already has this branch + // If it doesn't, we want to add *:* to our references for commit. This will allow us to push the new branch + // If it does, we can simply push HEAD:refs/heads/ + newRepo := true + refList, _ := pushRepoRemote.List(&git.ListOptions{Auth: pd.Authenticator}) + for _, ref := range refList { + if strings.HasSuffix(ref.Name().String(), fmt.Sprintf("heads/%s", md.PushBranch)) { + newRepo = false + break + } + } + + if newRepo { + pushRefspecs = append(pushRefspecs, config.RefSpec("*:*")) + pd.Log.Printf("New remote repo detected, creating new remote branch") + } + + // Identify specific references we want to push + // Should be refs/heads/, and a tag called imports// + pushRefspecs = append(pushRefspecs, config.RefSpec(fmt.Sprintf("HEAD:refs/heads/%s", md.PushBranch))) + pushRefspecs = append(pushRefspecs, config.RefSpec(fmt.Sprintf("HEAD:%s", newTag))) + + // Actually do the commit (locally) + commit, err := w.Commit("import from tagless source "+pd.Importer.ImportName(pd, md), &git.CommitOptions{ + Author: &object.Signature{ + Name: pd.GitCommitterName, + Email: pd.GitCommitterEmail, + When: time.Now(), + }, + }) + if err != nil { + return nil, fmt.Errorf("could not commit object: %v", err) + } + + obj, err := pushRepo.CommitObject(commit) + if err != nil { + return nil, fmt.Errorf("could not get commit object: %v", err) + } + + pd.Log.Printf("Committed local repo tagless mode transform:\n%s", obj.String()) + + // After commit, we will now tag our local repo on disk: + _, err = pushRepo.CreateTag(newTag, commit, &git.CreateTagOptions{ + Tagger: &object.Signature{ + Name: pd.GitCommitterName, + Email: pd.GitCommitterEmail, + When: time.Now(), + }, + Message: "import " + md.TagBranch + " from " + pd.RpmLocation + "(import from tagless source)", + SignKey: nil, + }) + if err != nil { + return nil, fmt.Errorf("could not create tag: %v", err) + } + + pd.Log.Printf("Pushing these references to the remote: %+v \n", pushRefspecs) + + // Do the actual push to the remote target repository + err = pushRepo.Push(&git.PushOptions{ + RemoteName: "origin", + Auth: pd.Authenticator, + RefSpecs: pushRefspecs, + Force: true, + }) + + if err != nil { + return nil, fmt.Errorf("could not push to remote: %v", err) + } + + if err := os.RemoveAll(localPath); err != nil { + log.Printf("Error cleaning up temporary git checkout directory %s . Non-fatal, continuing anyway...\n", localPath) + } + if err := os.RemoveAll(fmt.Sprintf("%s_gitpush", localPath)); err != nil { + log.Printf("Error cleaning up temporary git checkout directory %s . Non-fatal, continuing anyway...\n", fmt.Sprintf("%s_gitpush", localPath)) + } + + // append our processed branch to the return structures: + latestHashForBranch[md.PushBranch] = obj.Hash.String() + + versionForBranch[md.PushBranch] = &srpmprocpb.VersionRelease{ + Version: pd.PackageVersion, + Release: pd.PackageRelease, + } + + } + + // return struct with all our branch:commit and branch:version+release mappings + return &srpmprocpb.ProcessResponse{ + BranchCommits: latestHashForBranch, + BranchVersions: versionForBranch, + }, nil + +} + +// Given a local repo on disk, ensure it's in the "traditional" format. This means: +// - metadata file is named .pkgname.metadata +// - metadata file has the old " SOURCES/" format +// - SPECS/ and SOURCES/ exist and are populated correctly +func convertLocalRepo(pkgName string, localRepo string) (bool, error) { + + // Make sure we have a SPECS and SOURCES folder made: + if err := os.MkdirAll(fmt.Sprintf("%s/SOURCES", localRepo), 0755); err != nil { + return false, fmt.Errorf("Could not create SOURCES directory in: %s", localRepo) + } + + if err := os.MkdirAll(fmt.Sprintf("%s/SPECS", localRepo), 0755); err != nil { + return false, fmt.Errorf("Could not create SPECS directory in: %s", localRepo) + } + + // Loop through each file/folder and operate accordingly: + files, err := ioutil.ReadDir(localRepo) + if err != nil { + return false, err + } + + for _, file := range files { + + // We don't want to process SOURCES, SPECS, or any of our .git folders + if file.Name() == "SOURCES" || file.Name() == "SPECS" || strings.HasPrefix(file.Name(), ".git") || file.Name() == "."+pkgName+".metadata" { + continue + } + + // If we have a metadata "sources" file, we need to read it and convert to the old ..metadata format + if file.Name() == "sources" { + convertStatus := convertMetaData(pkgName, localRepo) + + if convertStatus != true { + return false, fmt.Errorf("Error converting sources metadata file to .metadata format") + } + + continue + } + + // Any file that ends in a ".spec" should be put into SPECS/ + if strings.HasSuffix(file.Name(), ".spec") { + err := os.Rename(fmt.Sprintf("%s/%s", localRepo, file.Name()), fmt.Sprintf("%s/SPECS/%s", localRepo, file.Name())) + if err != nil { + return false, fmt.Errorf("Error moving .spec file to SPECS/") + } + } + + // if a file isn't skipped in one of the above checks, then it must be a file that belongs in SOURCES/ + os.Rename(fmt.Sprintf("%s/%s", localRepo, file.Name()), fmt.Sprintf("%s/SOURCES/%s", localRepo, file.Name())) + } + + return true, nil +} + +// Given a local "sources" metadata file (new CentOS Stream format), convert it into the older +// classic CentOS style: " SOURCES/" +func convertMetaData(pkgName string, localRepo string) bool { + + lookAside, err := os.Open(fmt.Sprintf("%s/sources", localRepo)) + if err != nil { + return false + } + + // Split file into lines and start processing: + scanner := bufio.NewScanner(lookAside) + scanner.Split(bufio.ScanLines) + + // convertedLA is our array of new "converted" lookaside lines + var convertedLA []string + + // loop through each line, and: + // - split by whitespace + // - check each line begins with "SHA" or "MD" - validate + // - take the + // Then check + for scanner.Scan() { + + tmpLine := strings.Fields(scanner.Text()) + // make sure line starts with a "SHA" or "MD" before processing - otherwise it might not be a valid format lookaside line! + if !(strings.HasPrefix(tmpLine[0], "SHA") || strings.HasPrefix(tmpLine[0], "MD")) { + continue + } + + // Strip out "( )" characters from file name and prepend SOURCES/ to it + tmpLine[1] = strings.ReplaceAll(tmpLine[1], "(", "") + tmpLine[1] = strings.ReplaceAll(tmpLine[1], ")", "") + tmpLine[1] = fmt.Sprintf("SOURCES/%s", tmpLine[1]) + + convertedLA = append(convertedLA, fmt.Sprintf("%s %s", tmpLine[3], tmpLine[1])) + + } + lookAside.Close() + + // open ..metadata file for writing our old-format lines + lookAside, err = os.OpenFile(fmt.Sprintf("%s/.%s.metadata", localRepo, pkgName), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + fmt.Errorf("Error opening new .metadata file for writing.") + return false + } + + writer := bufio.NewWriter(lookAside) + + for _, convertedLine := range convertedLA { + _, _ = writer.WriteString(convertedLine + "\n") + } + + writer.Flush() + lookAside.Close() + + // Remove old "sources" metadata file - we don't need it now that conversion is complete + os.Remove(fmt.Sprintf("%s/sources", localRepo)) + + return true +} + +// Given a local checked out folder and package name, including SPECS/ , SOURCES/ , and .package.metadata, this will: +// - create a "dummy" SRPM (using dummy sources files we use to populate tarballs from lookaside) +// - extract RPM version info from that SRPM, and return it +// If we are in tagless mode, we need to get a package version somehow! +func getVersionFromSpec(pkgName string, localRepo string, majorVersion int) string { + + // Make sure we have "rpm" and "rpmbuild" and "cp" available in our PATH. Otherwise, this won't work: + _, err := exec.LookPath("rpm") + if err != nil { + return "" + } + + _, err = exec.LookPath("rpmbuild") + if err != nil { + return "" + } + + _, err = exec.LookPath("cp") + if err != nil { + return "" + } + + // create separate temp folder space to do our RPM work - we don't want to accidentally contaminate the main Git area: + rpmBuildPath := fmt.Sprintf("%s_rpm", localRepo) + os.Mkdir(rpmBuildPath, 0755) + + // Copy SOURCES/ and SPECS/ into the temp rpmbuild directory recursively + // Yes, we could create or import an elaborate Go-native way to do this, but damnit this is easier: + cmdArgs := strings.Fields(fmt.Sprintf("cp -rp %s/SOURCES %s/SPECS %s/", localRepo, localRepo, rpmBuildPath)) + if err := exec.Command(cmdArgs[0], cmdArgs[1:]...).Run(); err != nil { + log.Println(err) + return "" + } + + // Loop through ..metadata and get the file names we need to make our SRPM: + lookAside, err := os.Open(fmt.Sprintf("%s/.%s.metadata", localRepo, pkgName)) + if err != nil { + log.Println(err) + return "" + } + + // Split file into lines and start processing: + scanner := bufio.NewScanner(lookAside) + scanner.Split(bufio.ScanLines) + + // loop through each line, and: + // - isolate the SOURCES/filename entry + // - write out a dummy file of the same name to rpmBuildPath/SOURCES + for scanner.Scan() { + + // lookaside source is always the 2nd part of the line (after the long SHA sum) + srcFile := strings.Fields(scanner.Text())[1] + + // write a dummy file of the same name into the rpmbuild SOURCES/ directory: + dummyFile, err := os.OpenFile(fmt.Sprintf("%s/%s", rpmBuildPath, srcFile), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + return "" + } + writer := bufio.NewWriter(dummyFile) + _, _ = writer.WriteString("This is a dummy lookaside file generated by srpmproc. It is only needed to get a working SRPM and extract version information. Please disregard\n") + writer.Flush() + dummyFile.Close() + } + + lookAside.Close() + + // Now, call rpmbuild to produce the dummy src file: + // Example: rpmbuild --define "_topdir /tmp/srpmproctmp_httpd1988142783_rpm" -bs /tmp/srpmproctmp_httpd1988142783_rpm/SPECS/*.spec + cmd := exec.Command("rpmbuild", fmt.Sprintf(`--define=_topdir %s`, rpmBuildPath), fmt.Sprintf(`--define=dist .el%d`, majorVersion), "-bs", fmt.Sprintf("%s/SPECS/%s.spec", rpmBuildPath, pkgName)) + if err := cmd.Run(); err != nil { + log.Println(err) + return "" + } + + // Read the first file from the SRPMS/ folder in rpmBuildPath. It should be the SRPM that rpmbuild produced above + // (there should only be one file - we check that it ends in ".rpm" just to be sure!) + lsTmp, err := ioutil.ReadDir(fmt.Sprintf("%s/SRPMS/", rpmBuildPath)) + if err != nil { + log.Println(err) + return "" + } + + srpmFile := lsTmp[0].Name() + + if !strings.HasSuffix(srpmFile, ".rpm") { + log.Println("Error, file found in dummy SRPMS directory did not have an .rpm extension! Perhaps rpmbuild didn't produce a proper source RPM?") + return "" + } + + // Call the rpm binary to extract the version-release info out of it, and tack on ".el" at the end: + cmd = exec.Command("rpm", "-qp", "--qf", `%{NAME}|%{VERSION}|%{RELEASE}\n`, fmt.Sprintf("%s/SRPMS/%s", rpmBuildPath, srpmFile)) + nvrTmp, err := cmd.CombinedOutput() + if err != nil { + log.Println("Error running rpm command to extract temporary SRPM name-version-release identifiers.") + log.Println("rpmbuild output: ", string(nvrTmp)) + log.Println("rpmbuild command: ", cmd.String()) + return "" + } + + // Pull first line of the rpm command's output to get the name-version-release number (there should only be 1 line) + nvr := string(nvrTmp) + nvr = strings.Fields(nvr)[0] + + // Clean up: delete the temporary directory + if err := os.RemoveAll(rpmBuildPath); err != nil { + log.Printf("Error cleaning up temporary RPM directory %s . Non-fatal, continuing anyway...\n", rpmBuildPath) + } + + // return name-version-release string we derived: + log.Printf("Derived NVR %s from tagless repo via temporary SRPM build\n", nvr) + return nvr + +} + +// We need to loop through the lookaside blob files ("SourcesToIgnore"), +// and upload them to our target storage (usually an S3 bucket, but could be a local folder) +// +// We also need to add the source paths to .gitignore in the git repo, so we don't accidentally commit + push them +func processLookasideSources(pd *data.ProcessData, md *data.ModeData, localDir string) error { + + w := md.Worktree + metadata, err := w.Filesystem.Create(fmt.Sprintf(".%s.metadata", md.Name)) + if err != nil { + return fmt.Errorf("could not create metadata file: %v", err) + } + + // Keep track of files we've already uploaded - don't want duplicates! + var alreadyUploadedBlobs []string + + gitIgnore, err := os.OpenFile(fmt.Sprintf("%s/.gitignore", localDir), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + return err + } + + for _, source := range md.SourcesToIgnore { + + sourcePath := source.Name + _, err := w.Filesystem.Stat(sourcePath) + if source.Expired || err != nil { + continue + } + + sourceFile, err := w.Filesystem.Open(sourcePath) + if err != nil { + return fmt.Errorf("could not open ignored source file %s: %v", sourcePath, err) + } + sourceFileBts, err := ioutil.ReadAll(sourceFile) + if err != nil { + return fmt.Errorf("could not read the whole of ignored source file: %v", err) + } + + source.HashFunction.Reset() + _, err = source.HashFunction.Write(sourceFileBts) + if err != nil { + return fmt.Errorf("could not write bytes to hash function: %v", err) + } + checksum := hex.EncodeToString(source.HashFunction.Sum(nil)) + checksumLine := fmt.Sprintf("%s %s\n", checksum, sourcePath) + _, err = metadata.Write([]byte(checksumLine)) + if err != nil { + return fmt.Errorf("could not write to metadata file: %v", err) + } + + if data.StrContains(alreadyUploadedBlobs, checksum) { + continue + } + exists, err := pd.BlobStorage.Exists(checksum) + if err != nil { + return err + } + if !exists && !pd.NoStorageUpload { + err := pd.BlobStorage.Write(checksum, sourceFileBts) + if err != nil { + return err + } + pd.Log.Printf("wrote %s to blob storage", checksum) + } + alreadyUploadedBlobs = append(alreadyUploadedBlobs, checksum) + + // Add this SOURCES/ lookaside file to be excluded + w.Excludes = append(w.Excludes, gitignore.ParsePattern(sourcePath, nil)) + + // Append the SOURCES/ path to .gitignore: + _, err = gitIgnore.Write([]byte(fmt.Sprintf("%s\n", sourcePath))) + if err != nil { + return err + } + + } + + err = gitIgnore.Close() + if err != nil { + return err + } + + return nil + +} + +// Given an input branch name to import from, like "refs/heads/c9s", produce the tagless branch name we want to commit to, like "r9s" +// Modular translation of CentOS stream branches i is also done - branch stream-maven-3.8-rhel-9.1.0 ----> r9s-stream-maven-3.8_9.1.0 +func taglessBranchName(fullBranch string, pd *data.ProcessData) string { + + // Split the full branch name "refs/heads/blah" to only get the short name - last entry + tmpBranch := strings.Split(fullBranch, "/") + branch := tmpBranch[len(tmpBranch)-1] + + // Simple case: if our branch is not a modular stream branch, just return the normal pattern + if !strings.HasPrefix(branch, "stream-") { + return fmt.Sprintf("%s%d%s", pd.BranchPrefix, pd.Version, pd.BranchSuffix) + } + + // index where the "-rhel-" starts near the end of the string + rhelSpot := strings.LastIndex(branch, "-rhel-") + + // module name will be everything from the start until that "-rhel-" string (like "stream-httpd-2.4") + moduleString := branch[0:rhelSpot] + + // major minor version is everything after the "-rhel-" string + majorMinor := branch[rhelSpot+6 : len(branch)] + + // return translated modular branch: + return fmt.Sprintf("%s%d%s-%s_%s", pd.BranchPrefix, pd.Version, pd.BranchSuffix, moduleString, majorMinor) + +}