From f4085418413d29c4c905a9206d6de19703cd88a4 Mon Sep 17 00:00:00 2001 From: Skip Grube Date: Fri, 9 Sep 2022 17:19:13 -0400 Subject: [PATCH] Cleaned up formatting w/ gofmt (no functional changes) -Skip G. --- cmd/srpmproc/main.go | 11 +- pkg/modes/git.go | 226 +++++----- pkg/srpmproc/process.go | 925 +++++++++++++++++++--------------------- 3 files changed, 551 insertions(+), 611 deletions(-) diff --git a/cmd/srpmproc/main.go b/cmd/srpmproc/main.go index 7a4a80b..fe6f966 100644 --- a/cmd/srpmproc/main.go +++ b/cmd/srpmproc/main.go @@ -56,8 +56,8 @@ var ( basicPassword string packageVersion string packageRelease string - taglessMode bool - altLookAside bool + taglessMode bool + altLookAside bool ) var root = &cobra.Command{ @@ -97,7 +97,7 @@ func mn(_ *cobra.Command, _ []string) { TaglessMode: taglessMode, AltLookAside: altLookAside, }) - + if err != nil { log.Fatal(err) } @@ -147,9 +147,8 @@ func main() { root.Flags().StringVar(&basicPassword, "basic-password", "", "Basic auth password") root.Flags().StringVar(&packageVersion, "package-version", "", "Package version to fetch") root.Flags().StringVar(&packageRelease, "package-release", "", "Package release to fetch") - root.Flags().BoolVar(&taglessMode, "taglessmode", false, "Tagless mode: If set, pull the latest commit from a branch, and determine version info from spec file (aka upstream versions aren't tagged)") - root.Flags().BoolVar(&altLookAside, "altlookaside", false, "If set, uses the new CentOS Stream lookaside pattern (https:///////)") - + root.Flags().BoolVar(&taglessMode, "taglessmode", false, "Tagless mode: If set, pull the latest commit from a branch, and determine version info from spec file (aka upstream versions aren't tagged)") + root.Flags().BoolVar(&altLookAside, "altlookaside", false, "If set, uses the new CentOS Stream lookaside pattern (https:///////)") if err := root.Execute(); err != nil { log.Fatal(err) diff --git a/pkg/modes/git.go b/pkg/modes/git.go index 94d318a..b19fa36 100644 --- a/pkg/modes/git.go +++ b/pkg/modes/git.go @@ -125,52 +125,46 @@ func (g *GitMode) RetrieveSource(pd *data.ProcessData) (*data.ModeData, error) { return nil } - - // In case of "tagless mode", we need to get the head ref of the branch instead - // This is a kind of alternative implementation of the above tagAdd assignment - refAdd := func(tag *object.Tag) error { + // In case of "tagless mode", we need to get the head ref of the branch instead + // This is a kind of alternative implementation of the above tagAdd assignment + refAdd := func(tag *object.Tag) error { if strings.HasPrefix(tag.Name, fmt.Sprintf("refs/heads/%s%d%s", pd.ImportBranchPrefix, pd.Version, pd.BranchSuffix)) { pd.Log.Printf("Tagless mode: Identified tagless commit for import: %s\n", tag.Name) refSpec := fmt.Sprintf(tag.Name) - - + // We split the string by "/", the branch name we're looking for to pass to latestTags is always last // (ex: "refs/heads/c9s" ---> we want latestTags[c9s] _tmpRef := strings.Split(refSpec, "/") _branchName := _tmpRef[(len(_tmpRef) - 1)] - + // In the case of "strict branch mode" on, the branch name must match *exactly* with our prefix-version-suffix (like "c8" cannot also match "c8-beta") // If it doesn't, bail out without adding this branch if pd.StrictBranchMode == true && _branchName != fmt.Sprintf("%s%d%s", pd.ImportBranchPrefix, pd.Version, pd.BranchSuffix) { - return nil + return nil + } + + latestTags[_branchName] = &remoteTarget{ + remote: refSpec, + when: tag.Tagger.When, } - - - latestTags[_branchName] = &remoteTarget{ - remote: refSpec, - when: tag.Tagger.When, - } } return nil } - - tagIter, err := repo.TagObjects() + tagIter, err := repo.TagObjects() - if err != nil { + if err != nil { return nil, fmt.Errorf("could not get tag objects: %v", err) } - - // tagless mode means we use "refAdd" (add commit by reference) - // normal mode means we can rely on "tagAdd" (the tag should be present for us in the source repo) + // tagless mode means we use "refAdd" (add commit by reference) + // normal mode means we can rely on "tagAdd" (the tag should be present for us in the source repo) if pd.TaglessMode == true { - _ = tagIter.ForEach(refAdd) - } else { - _ = tagIter.ForEach(tagAdd) - } - - + _ = tagIter.ForEach(refAdd) + } else { + _ = tagIter.ForEach(tagAdd) + } + listOpts := &git.ListOptions{ Auth: pd.Authenticator, } @@ -196,21 +190,20 @@ func (g *GitMode) RetrieveSource(pd *data.ProcessData) (*data.ModeData, error) { if err != nil { continue } - - + // Call refAdd instead of tagAdd in the case of TaglessMode enabled if pd.TaglessMode == true { - _ = refAdd(&object.Tag{ - Name: string(ref.Name()), - Tagger: commit.Committer, - }) + _ = refAdd(&object.Tag{ + Name: string(ref.Name()), + Tagger: commit.Committer, + }) } else { - _ = tagAdd(&object.Tag{ - Name: strings.TrimPrefix(string(ref.Name()), "refs/tags/"), - Tagger: commit.Committer, - }) + _ = tagAdd(&object.Tag{ + Name: strings.TrimPrefix(string(ref.Name()), "refs/tags/"), + Tagger: commit.Committer, + }) } - + } for _, branch := range latestTags { @@ -223,7 +216,7 @@ func (g *GitMode) RetrieveSource(pd *data.ProcessData) (*data.ModeData, error) { for _, branch := range branches { sortedBranches = append(sortedBranches, branch.remote) } - + return &data.ModeData{ Name: filepath.Base(pd.RpmLocation), Repo: repo, @@ -234,73 +227,68 @@ func (g *GitMode) RetrieveSource(pd *data.ProcessData) (*data.ModeData, error) { } func (g *GitMode) WriteSource(pd *data.ProcessData, md *data.ModeData) error { - + remote, err := md.Repo.Remote("upstream") - + if err != nil && pd.TaglessMode == false { - return fmt.Errorf("could not get upstream remote: %v", err) + return fmt.Errorf("could not get upstream remote: %v", err) } - var refspec config.RefSpec var branchName string - fmt.Printf("pd.AltLookaside == %v , pd.CdnUrl == %s \n", pd.AltLookAside, pd.CdnUrl) - + // In the case of tagless mode, we already have the transformed repo sitting in the worktree, + // and don't need to perform any checkout or fetch operations + if pd.TaglessMode == false { + if strings.HasPrefix(md.TagBranch, "refs/heads") { + refspec = config.RefSpec(fmt.Sprintf("+%s:%s", md.TagBranch, md.TagBranch)) + branchName = strings.TrimPrefix(md.TagBranch, "refs/heads/") + } else { + match := misc.GetTagImportRegex(pd).FindStringSubmatch(md.TagBranch) + branchName = match[2] + refspec = config.RefSpec(fmt.Sprintf("+refs/heads/%s:%s", branchName, md.TagBranch)) + fmt.Println("Found branchname that does not start w/ refs/heads :: ", branchName) + } + pd.Log.Printf("checking out upstream refspec %s", refspec) - // In the case of tagless mode, we already have the transformed repo sitting in the worktree, - // and don't need to perform any checkout or fetch operations - if pd.TaglessMode == false { - if strings.HasPrefix(md.TagBranch, "refs/heads") { - refspec = config.RefSpec(fmt.Sprintf("+%s:%s", md.TagBranch, md.TagBranch)) - branchName = strings.TrimPrefix(md.TagBranch, "refs/heads/") - } else { - match := misc.GetTagImportRegex(pd).FindStringSubmatch(md.TagBranch) - branchName = match[2] - refspec = config.RefSpec(fmt.Sprintf("+refs/heads/%s:%s", branchName, md.TagBranch)) - fmt.Println("Found branchname that does not start w/ refs/heads :: ", branchName) - } - pd.Log.Printf("checking out upstream refspec %s", refspec) - - - fetchOpts := &git.FetchOptions{ - Auth: pd.Authenticator, - RemoteName: "upstream", - RefSpecs: []config.RefSpec{refspec}, - Tags: git.AllTags, - Force: true, - } - err = remote.Fetch(fetchOpts) - if err != nil && err != git.NoErrAlreadyUpToDate { - if err == transport.ErrInvalidAuthMethod || err == transport.ErrAuthenticationRequired { - fetchOpts.Auth = nil - err = remote.Fetch(fetchOpts) - if err != nil && err != git.NoErrAlreadyUpToDate { - return fmt.Errorf("could not fetch upstream: %v", err) - } - } else { - return fmt.Errorf("could not fetch upstream: %v", err) - } - } + fetchOpts := &git.FetchOptions{ + Auth: pd.Authenticator, + RemoteName: "upstream", + RefSpecs: []config.RefSpec{refspec}, + Tags: git.AllTags, + Force: true, + } + err = remote.Fetch(fetchOpts) + if err != nil && err != git.NoErrAlreadyUpToDate { + if err == transport.ErrInvalidAuthMethod || err == transport.ErrAuthenticationRequired { + fetchOpts.Auth = nil + err = remote.Fetch(fetchOpts) + if err != nil && err != git.NoErrAlreadyUpToDate { + return fmt.Errorf("could not fetch upstream: %v", err) + } + } else { + return fmt.Errorf("could not fetch upstream: %v", err) + } + } - err = md.Worktree.Checkout(&git.CheckoutOptions{ - Branch: plumbing.ReferenceName(md.TagBranch), - Force: true, - }) - if err != nil { - return fmt.Errorf("could not checkout source from git: %v", err) - } + err = md.Worktree.Checkout(&git.CheckoutOptions{ + Branch: plumbing.ReferenceName(md.TagBranch), + Force: true, + }) + if err != nil { + return fmt.Errorf("could not checkout source from git: %v", err) + } + + _, err = md.Worktree.Add(".") + if err != nil { + return fmt.Errorf("could not add Worktree: %v", err) + } + } + + if pd.TaglessMode == true { + branchName = fmt.Sprintf("%s%d%s", pd.ImportBranchPrefix, pd.Version, pd.BranchSuffix) + } - _, err = md.Worktree.Add(".") - if err != nil { - return fmt.Errorf("could not add Worktree: %v", err) - } - } - - if pd.TaglessMode == true { - branchName = fmt.Sprintf("%s%d%s", pd.ImportBranchPrefix, pd.Version, pd.BranchSuffix) - } - metadataPath := "" ls, err := md.Worktree.Filesystem.ReadDir(".") if err != nil { @@ -358,32 +346,32 @@ func (g *GitMode) WriteSource(pd *data.ProcessData, md *data.ModeData) error { body = fromBlobStorage pd.Log.Printf("downloading %s from blob storage", hash) } else { - - url := "" - // Alternate lookaside logic: if enabled, we pull from a new URL pattern - if pd.AltLookAside == false { - url = fmt.Sprintf("%s/%s/%s/%s", pd.CdnUrl, md.Name, branchName, hash) + + url := "" + // Alternate lookaside logic: if enabled, we pull from a new URL pattern + if pd.AltLookAside == false { + url = fmt.Sprintf("%s/%s/%s/%s", pd.CdnUrl, md.Name, branchName, hash) } else { - // We first need the hash algorithm based on length of hash: - hashType := "sha512" - switch len(hash) { - case 128: - hashType = "sha512" - case 64: - hashType = "sha256" - case 40: - hashType = "sha1" - case 32: - hashType = "md5" - } - - // need the name of the file without "SOURCES/": - fileName := strings.Split(path, "/")[1] - - // Alt. lookaside url is of the form: / / / / / - url = fmt.Sprintf("%s/%s/%s/%s/%s/%s", pd.CdnUrl, md.Name, fileName, hashType, hash, fileName) + // We first need the hash algorithm based on length of hash: + hashType := "sha512" + switch len(hash) { + case 128: + hashType = "sha512" + case 64: + hashType = "sha256" + case 40: + hashType = "sha1" + case 32: + hashType = "md5" + } + + // need the name of the file without "SOURCES/": + fileName := strings.Split(path, "/")[1] + + // Alt. lookaside url is of the form: / / / / / + url = fmt.Sprintf("%s/%s/%s/%s/%s/%s", pd.CdnUrl, md.Name, fileName, hashType, hash, fileName) } - + pd.Log.Printf("downloading %s", url) req, err := http.NewRequest("GET", url, nil) diff --git a/pkg/srpmproc/process.go b/pkg/srpmproc/process.go index 68665dd..309b40b 100644 --- a/pkg/srpmproc/process.go +++ b/pkg/srpmproc/process.go @@ -21,6 +21,7 @@ package srpmproc import ( + "bufio" "encoding/hex" "fmt" "github.com/go-git/go-billy/v5" @@ -46,7 +47,6 @@ import ( "path/filepath" "strings" "time" - "bufio" "github.com/go-git/go-billy/v5/memfs" "github.com/go-git/go-git/v5" @@ -100,8 +100,8 @@ type ProcessDataRequest struct { PackageVersion string PackageRelease string - TaglessMode bool - AltLookAside bool + TaglessMode bool + AltLookAside bool } func gitlabify(str string) string { @@ -144,7 +144,7 @@ func NewProcessData(req *ProcessDataRequest) (*data.ProcessData, error) { // If altlookaside is enabled, and the CdnUrl hasn't been changed, then automatically set it to the default // CentOS Stream (the new pattern very much won't work with the old git.centos.org/sources site) if (req.CdnUrl == "https://git.centos.org/sources" || req.CdnUrl == "") && req.AltLookAside == true { - req.CdnUrl = "https://sources.stream.centos.org/sources/rpms" + req.CdnUrl = "https://sources.stream.centos.org/sources/rpms" } // Validate required @@ -246,7 +246,7 @@ func NewProcessData(req *ProcessDataRequest) (*data.ProcessData, error) { if strings.TrimSpace(req.ManualCommits) != "" { manualCs = strings.Split(req.ManualCommits, ",") } - + return &data.ProcessData{ Importer: importer, RpmLocation: sourceRpmLocation, @@ -287,14 +287,14 @@ func NewProcessData(req *ProcessDataRequest) (*data.ProcessData, error) { // all files that are remote goes into .gitignore // all ignored files' hash goes into .{Name}.metadata func ProcessRPM(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error) { - + // if we are using "tagless mode", then we need to jump to a completely different import process: // Version info needs to be derived from rpmbuild + spec file, not tags if pd.TaglessMode == true { - result, err := processRPMTagless(pd) - return result, err + result, err := processRPMTagless(pd) + return result, err } - + md, err := pd.Importer.RetrieveSource(pd) if err != nil { return nil, err @@ -366,11 +366,9 @@ func ProcessRPM(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error) { commitPin[head] = branchCommit[1] } } - - - for _, branch := range md.Branches { - md.Repo = &sourceRepo + for _, branch := range md.Branches { + md.Repo = &sourceRepo md.Worktree = &sourceWorktree md.TagBranch = branch for _, source := range md.SourcesToIgnore { @@ -391,20 +389,19 @@ func ProcessRPM(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error) { continue } } else { - matchString = md.TagBranch + matchString = md.TagBranch } - match := misc.GetTagImportRegex(pd).FindStringSubmatch(matchString) - + md.PushBranch = pd.BranchPrefix + strings.TrimPrefix(match[2], pd.ImportBranchPrefix) - + newTag := "imports/" + pd.BranchPrefix + strings.TrimPrefix(match[1], "imports/"+pd.ImportBranchPrefix) newTag = strings.Replace(newTag, "%", "_", -1) createdFs, err := pd.FsCreator(md.PushBranch) if err != nil { - return nil, err + return nil, err } // create new Repo for final dist @@ -498,7 +495,7 @@ func ProcessRPM(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error) { } // get ignored files hash and add to .{Name}.metadata - metadataFile := "" + metadataFile := "" ls, err := md.Worktree.Filesystem.ReadDir(".") if err != nil { return nil, fmt.Errorf("could not read directory: %v", err) @@ -662,7 +659,6 @@ func ProcessRPM(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error) { } pushRefspecs = append(pushRefspecs, config.RefSpec("HEAD:"+plumbing.NewTagReferenceName(newTag))) - err = repo.Push(&git.PushOptions{ RemoteName: "origin", @@ -677,45 +673,42 @@ func ProcessRPM(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error) { hashString := obj.Hash.String() latestHashForBranch[md.PushBranch] = hashString } - + return &srpmprocpb.ProcessResponse{ BranchCommits: latestHashForBranch, BranchVersions: versionForBranch, }, nil } - // Process for when we want to import a tagless repo (like from CentOS Stream) // func processRPMTagless(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error) { - pd.Log.Println("Tagless mode detected, attempting import of latest commit") - - // In tagless mode, we *automatically* set StrictBranchMode to true - // Only the exact branch should be pulled from the source repo - pd.StrictBranchMode = true - - // our return values: a mapping of branches -> commits (1:1) that we're bringing in, - // and a mapping of branches to: version = X, release = Y - latestHashForBranch := map[string]string{} + pd.Log.Println("Tagless mode detected, attempting import of latest commit") + + // In tagless mode, we *automatically* set StrictBranchMode to true + // Only the exact branch should be pulled from the source repo + pd.StrictBranchMode = true + + // our return values: a mapping of branches -> commits (1:1) that we're bringing in, + // and a mapping of branches to: version = X, release = Y + latestHashForBranch := map[string]string{} versionForBranch := map[string]*srpmprocpb.VersionRelease{} - - - md, err := pd.Importer.RetrieveSource(pd) - if err != nil { - pd.Log.Println("Error detected in RetrieveSource!") - return nil, err - } - - md.BlobCache = map[string][]byte{} - // TODO: add tagless module support - remotePrefix := "rpms" - if pd.ModuleMode { - remotePrefix = "modules" - } - + md, err := pd.Importer.RetrieveSource(pd) + if err != nil { + pd.Log.Println("Error detected in RetrieveSource!") + return nil, err + } - // Set up our remote URL for pushing our repo to + md.BlobCache = map[string][]byte{} + + // TODO: add tagless module support + remotePrefix := "rpms" + if pd.ModuleMode { + remotePrefix = "modules" + } + + // Set up our remote URL for pushing our repo to var tagIgnoreList []string if pd.NoDupMode { repo, err := git.Init(memory.NewStorage(), memfs.New()) @@ -749,142 +742,130 @@ func processRPMTagless(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error } } - sourceRepo := *md.Repo - sourceWorktree := *md.Worktree - localPath := "" + sourceWorktree := *md.Worktree + localPath := "" - for _, branch := range md.Branches { - md.Repo = &sourceRepo + for _, branch := range md.Branches { + md.Repo = &sourceRepo md.Worktree = &sourceWorktree md.TagBranch = branch - + for _, source := range md.SourcesToIgnore { source.Expired = true } - - // Create a temporary place to check out our tag/branch : /tmp/srpmproctmp_/ - localPath, _ = os.MkdirTemp("/tmp", fmt.Sprintf("srpmproctmp_%s", md.Name)) - if err := os.RemoveAll(localPath); err != nil { - return nil, fmt.Errorf("Could not remove previous temporary directory: %s", localPath) - } - if err := os.Mkdir(localPath, 0755); err != nil { - return nil, fmt.Errorf("Could not create temporary directory: %s", localPath) - } - - // Clone repo into the temporary path, but only the tag we're interested in: - // (TODO: will probably need to assign this a variable or use the md struct gitrepo object to perform a successful tag+push later) - _, _ = git.PlainClone(localPath, false, &git.CloneOptions{ - URL: pd.RpmLocation, - SingleBranch: true, - ReferenceName: plumbing.ReferenceName(branch), - }) + // Create a temporary place to check out our tag/branch : /tmp/srpmproctmp_/ + localPath, _ = os.MkdirTemp("/tmp", fmt.Sprintf("srpmproctmp_%s", md.Name)) - // Now that we're cloned into localPath, we need to "covert" the import into the old format - // We want sources to become .PKGNAME.metadata, we want SOURCES and SPECS folders, etc. - repoFixed, _ := convertLocalRepo(md.Name, localPath) - if !repoFixed { - return nil, fmt.Errorf("Error converting repository into SOURCES + SPECS + .package.metadata format") - } - - - nvrString := getVersionFromSpec(md.Name, localPath, pd.Version) - if nvrString == "" { - return nil, fmt.Errorf("Error using rpm or rpmbuild to build SRPM and determine version info! (tagless mode)") - } - - // Set version and release fields we extracted (name|version|release are separated by pipes) - pd.PackageVersion = strings.Split(nvrString, "|")[1] - pd.PackageRelease = strings.Split(nvrString, "|")[2] - - - // Set full rpm version: name-version-release (for tagging properly) - rpmVersion := fmt.Sprintf("%s-%s-%s", md.Name, pd.PackageVersion, pd.PackageRelease) - - - pd.Log.Println("Successfully determined version of tagless checkout: ", rpmVersion) - - md.PushBranch = fmt.Sprintf("%s%d%s", pd.BranchPrefix, pd.Version, pd.BranchSuffix) - - - // Make an initial repo we will use to push to our target - pushRepo, err := git.PlainInit(localPath + "_gitpush", false) - if err != nil { + if err := os.RemoveAll(localPath); err != nil { + return nil, fmt.Errorf("Could not remove previous temporary directory: %s", localPath) + } + if err := os.Mkdir(localPath, 0755); err != nil { + return nil, fmt.Errorf("Could not create temporary directory: %s", localPath) + } + + // Clone repo into the temporary path, but only the tag we're interested in: + // (TODO: will probably need to assign this a variable or use the md struct gitrepo object to perform a successful tag+push later) + _, _ = git.PlainClone(localPath, false, &git.CloneOptions{ + URL: pd.RpmLocation, + SingleBranch: true, + ReferenceName: plumbing.ReferenceName(branch), + }) + + // Now that we're cloned into localPath, we need to "covert" the import into the old format + // We want sources to become .PKGNAME.metadata, we want SOURCES and SPECS folders, etc. + repoFixed, _ := convertLocalRepo(md.Name, localPath) + if !repoFixed { + return nil, fmt.Errorf("Error converting repository into SOURCES + SPECS + .package.metadata format") + } + + nvrString := getVersionFromSpec(md.Name, localPath, pd.Version) + if nvrString == "" { + return nil, fmt.Errorf("Error using rpm or rpmbuild to build SRPM and determine version info! (tagless mode)") + } + + // Set version and release fields we extracted (name|version|release are separated by pipes) + pd.PackageVersion = strings.Split(nvrString, "|")[1] + pd.PackageRelease = strings.Split(nvrString, "|")[2] + + // Set full rpm version: name-version-release (for tagging properly) + rpmVersion := fmt.Sprintf("%s-%s-%s", md.Name, pd.PackageVersion, pd.PackageRelease) + + pd.Log.Println("Successfully determined version of tagless checkout: ", rpmVersion) + + md.PushBranch = fmt.Sprintf("%s%d%s", pd.BranchPrefix, pd.Version, pd.BranchSuffix) + + // Make an initial repo we will use to push to our target + pushRepo, err := git.PlainInit(localPath+"_gitpush", false) + if err != nil { return nil, fmt.Errorf("could not create new dist Repo: %v", err) } - w, err := pushRepo.Worktree() + w, err := pushRepo.Worktree() if err != nil { return nil, fmt.Errorf("could not get dist Worktree: %v", err) } - - - // Create a remote "origin" in our empty git, make the upstream equal to the branch we want to modify - pushUrl := fmt.Sprintf("%s/%s/%s.git", pd.UpstreamPrefix, remotePrefix, gitlabify(md.Name)) - refspec := config.RefSpec(fmt.Sprintf("+refs/heads/%s:refs/remotes/origin/%s", md.PushBranch, md.PushBranch)) - - // Make our remote repo the target one - the one we want to push our update to - pushRepoRemote, err := pushRepo.CreateRemote(&config.RemoteConfig{ + + // Create a remote "origin" in our empty git, make the upstream equal to the branch we want to modify + pushUrl := fmt.Sprintf("%s/%s/%s.git", pd.UpstreamPrefix, remotePrefix, gitlabify(md.Name)) + refspec := config.RefSpec(fmt.Sprintf("+refs/heads/%s:refs/remotes/origin/%s", md.PushBranch, md.PushBranch)) + + // Make our remote repo the target one - the one we want to push our update to + pushRepoRemote, err := pushRepo.CreateRemote(&config.RemoteConfig{ Name: "origin", URLs: []string{pushUrl}, Fetch: []config.RefSpec{refspec}, }) - if err != nil { + if err != nil { return nil, fmt.Errorf("could not create remote: %v", err) } - - // fetch our branch data (md.PushBranch) into this new repo - err = pushRepo.Fetch(&git.FetchOptions{ + + // fetch our branch data (md.PushBranch) into this new repo + err = pushRepo.Fetch(&git.FetchOptions{ RemoteName: "origin", RefSpecs: []config.RefSpec{refspec}, Auth: pd.Authenticator, }) - - refName := plumbing.NewBranchReferenceName(md.PushBranch) + refName := plumbing.NewBranchReferenceName(md.PushBranch) var hash plumbing.Hash h := plumbing.NewSymbolicReference(plumbing.HEAD, refName) if err := pushRepo.Storer.CheckAndSetReference(h, nil); err != nil { - return nil, fmt.Errorf("Could not set symbolic reference: %v", err) + return nil, fmt.Errorf("Could not set symbolic reference: %v", err) } - - err = w.Checkout(&git.CheckoutOptions{ - Branch: plumbing.NewRemoteReferenceName("origin", md.PushBranch), - Hash: hash, - Force: true, - }) - - - os.Rename(fmt.Sprintf("%s/SPECS", localPath), fmt.Sprintf("%s_gitpush/SPECS", localPath) ) - os.Rename(fmt.Sprintf("%s/SOURCES", localPath), fmt.Sprintf("%s_gitpush/SOURCES", localPath) ) - os.Rename(fmt.Sprintf("%s/.gitignore", localPath), fmt.Sprintf("%s_gitpush/.gitignore", localPath) ) - os.Rename(fmt.Sprintf("%s/.%s.metadata", localPath, md.Name ), fmt.Sprintf("%s_gitpush/.%s.metadata", localPath, md.Name) ) - - - - md.Repo = pushRepo - md.Worktree = w - // Download lookaside sources (tarballs) into the push git repo: - err = pd.Importer.WriteSource(pd, md) + err = w.Checkout(&git.CheckoutOptions{ + Branch: plumbing.NewRemoteReferenceName("origin", md.PushBranch), + Hash: hash, + Force: true, + }) + + os.Rename(fmt.Sprintf("%s/SPECS", localPath), fmt.Sprintf("%s_gitpush/SPECS", localPath)) + os.Rename(fmt.Sprintf("%s/SOURCES", localPath), fmt.Sprintf("%s_gitpush/SOURCES", localPath)) + os.Rename(fmt.Sprintf("%s/.gitignore", localPath), fmt.Sprintf("%s_gitpush/.gitignore", localPath)) + os.Rename(fmt.Sprintf("%s/.%s.metadata", localPath, md.Name), fmt.Sprintf("%s_gitpush/.%s.metadata", localPath, md.Name)) + + md.Repo = pushRepo + md.Worktree = w + + // Download lookaside sources (tarballs) into the push git repo: + err = pd.Importer.WriteSource(pd, md) if err != nil { return nil, err } - - - // Call function to upload source to target lookaside and - // ensure the sources are added to .gitignore - - err = processLookasideSources(pd, md, localPath + "_gitpush") - if err != nil { - return nil, err - } - - - // Apply patch(es) if needed: - if pd.ModuleMode { + + // Call function to upload source to target lookaside and + // ensure the sources are added to .gitignore + + err = processLookasideSources(pd, md, localPath+"_gitpush") + if err != nil { + return nil, err + } + + // Apply patch(es) if needed: + if pd.ModuleMode { err := patchModuleYaml(pd, md) if err != nil { return nil, err @@ -895,75 +876,66 @@ func processRPMTagless(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error return nil, err } } - - - - err = w.AddWithOptions(&git.AddOptions{All: true} ) - if err != nil { - return nil, fmt.Errorf("Error adding SOURCES/ , SPECS/ or .metadata file to commit list.") - } - - status, err := w.Status() - fmt.Println("Current repo status == ", status) - - - // assign tag for our new remote we're about to push (derived from the SRPM version) - newTag := "refs/tags/imports/" + md.PushBranch + "/" + rpmVersion - newTag = strings.Replace(newTag, "%", "_", -1) - // pushRefspecs is a list of all the references we want to push (tags + heads) - // It's an array of colon-separated strings which map local references to their remote counterparts - var pushRefspecs []config.RefSpec - - - // We need to find out if the remote repo already has this branch - // If it doesn't, we want to add *:* to our references for commit. This will allow us to push the new branch - // If it does, we can simply push HEAD:refs/heads/ - newRepo := true - refList, _ := pushRepoRemote.List(&git.ListOptions{Auth: pd.Authenticator,}) - for _, ref := range refList { - if strings.HasSuffix(ref.Name().String(), fmt.Sprintf("heads/%s", md.PushBranch)) { - newRepo = false - break - } - } + err = w.AddWithOptions(&git.AddOptions{All: true}) + if err != nil { + return nil, fmt.Errorf("Error adding SOURCES/ , SPECS/ or .metadata file to commit list.") + } - if newRepo == true { - pushRefspecs = append(pushRefspecs, config.RefSpec("*:*")) - pd.Log.Printf("New remote repo detected, creating new remote branch") - } - - - - // Identify specific references we want to push - // Should be refs/heads/, and a tag called imports// - pushRefspecs = append(pushRefspecs, config.RefSpec(fmt.Sprintf("HEAD:refs/heads/%s", md.PushBranch)) ) - pushRefspecs = append(pushRefspecs, config.RefSpec(fmt.Sprintf("HEAD:%s", newTag)) ) + status, err := w.Status() + fmt.Println("Current repo status == ", status) - - // Actually do the commit (locally) - commit, err := w.Commit("import from tagless source "+pd.Importer.ImportName(pd, md), &git.CommitOptions{ + // assign tag for our new remote we're about to push (derived from the SRPM version) + newTag := "refs/tags/imports/" + md.PushBranch + "/" + rpmVersion + newTag = strings.Replace(newTag, "%", "_", -1) + + // pushRefspecs is a list of all the references we want to push (tags + heads) + // It's an array of colon-separated strings which map local references to their remote counterparts + var pushRefspecs []config.RefSpec + + // We need to find out if the remote repo already has this branch + // If it doesn't, we want to add *:* to our references for commit. This will allow us to push the new branch + // If it does, we can simply push HEAD:refs/heads/ + newRepo := true + refList, _ := pushRepoRemote.List(&git.ListOptions{Auth: pd.Authenticator}) + for _, ref := range refList { + if strings.HasSuffix(ref.Name().String(), fmt.Sprintf("heads/%s", md.PushBranch)) { + newRepo = false + break + } + } + + if newRepo == true { + pushRefspecs = append(pushRefspecs, config.RefSpec("*:*")) + pd.Log.Printf("New remote repo detected, creating new remote branch") + } + + // Identify specific references we want to push + // Should be refs/heads/, and a tag called imports// + pushRefspecs = append(pushRefspecs, config.RefSpec(fmt.Sprintf("HEAD:refs/heads/%s", md.PushBranch))) + pushRefspecs = append(pushRefspecs, config.RefSpec(fmt.Sprintf("HEAD:%s", newTag))) + + // Actually do the commit (locally) + commit, err := w.Commit("import from tagless source "+pd.Importer.ImportName(pd, md), &git.CommitOptions{ Author: &object.Signature{ Name: pd.GitCommitterName, Email: pd.GitCommitterEmail, When: time.Now(), }, - }) - if err != nil { + }) + if err != nil { return nil, fmt.Errorf("could not commit object: %v", err) } - obj, err := pushRepo.CommitObject(commit) + obj, err := pushRepo.CommitObject(commit) if err != nil { return nil, fmt.Errorf("could not get commit object: %v", err) } pd.Log.Printf("Committed local repo tagless mode transform:\n%s", obj.String()) - - - - // After commit, we will now tag our local repo on disk: - _, err = pushRepo.CreateTag(newTag, commit, &git.CreateTagOptions{ + + // After commit, we will now tag our local repo on disk: + _, err = pushRepo.CreateTag(newTag, commit, &git.CreateTagOptions{ Tagger: &object.Signature{ Name: pd.GitCommitterName, Email: pd.GitCommitterEmail, @@ -972,314 +944,302 @@ func processRPMTagless(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error Message: "import " + md.TagBranch + " from " + pd.RpmLocation + "(import from tagless source)", SignKey: nil, }) - if err != nil { + if err != nil { return nil, fmt.Errorf("could not create tag: %v", err) } - - pd.Log.Printf("Pushing these references to the remote: %+v \n", pushRefspecs) + pd.Log.Printf("Pushing these references to the remote: %+v \n", pushRefspecs) - // Do the actual push to the remote target repository - err = pushRepo.Push(&git.PushOptions{ - RemoteName: "origin", - Auth: pd.Authenticator, - RefSpecs: pushRefspecs, - Force: true, - }) - - if err != nil { + // Do the actual push to the remote target repository + err = pushRepo.Push(&git.PushOptions{ + RemoteName: "origin", + Auth: pd.Authenticator, + RefSpecs: pushRefspecs, + Force: true, + }) + + if err != nil { return nil, fmt.Errorf("could not push to remote: %v", err) } - if err := os.RemoveAll(localPath); err != nil { - log.Printf("Error cleaning up temporary git checkout directory %s . Non-fatal, continuing anyway...\n", localPath) - } - if err := os.RemoveAll(fmt.Sprintf("%s_gitpush", localPath)); err != nil { - log.Printf("Error cleaning up temporary git checkout directory %s . Non-fatal, continuing anyway...\n", fmt.Sprintf("%s_gitpush", localPath)) - } - - // append our processed branch to the return structures: - latestHashForBranch[md.PushBranch] = obj.Hash.String() - - versionForBranch[md.PushBranch] = &srpmprocpb.VersionRelease{ - Version: pd.PackageVersion, - Release: pd.PackageRelease, - } - - } - - fmt.Printf("returning::\n latestHashForBranch == %+v \n\n versionForBranch == %+v\n\n", latestHashForBranch, versionForBranch) - - // return struct with all our branch:commit and branch:version+release mappings - return &srpmprocpb.ProcessResponse{ - BranchCommits: latestHashForBranch, - BranchVersions: versionForBranch, - }, nil - -} + if err := os.RemoveAll(localPath); err != nil { + log.Printf("Error cleaning up temporary git checkout directory %s . Non-fatal, continuing anyway...\n", localPath) + } + if err := os.RemoveAll(fmt.Sprintf("%s_gitpush", localPath)); err != nil { + log.Printf("Error cleaning up temporary git checkout directory %s . Non-fatal, continuing anyway...\n", fmt.Sprintf("%s_gitpush", localPath)) + } + // append our processed branch to the return structures: + latestHashForBranch[md.PushBranch] = obj.Hash.String() + + versionForBranch[md.PushBranch] = &srpmprocpb.VersionRelease{ + Version: pd.PackageVersion, + Release: pd.PackageRelease, + } + + } + + fmt.Printf("returning::\n latestHashForBranch == %+v \n\n versionForBranch == %+v\n\n", latestHashForBranch, versionForBranch) + + // return struct with all our branch:commit and branch:version+release mappings + return &srpmprocpb.ProcessResponse{ + BranchCommits: latestHashForBranch, + BranchVersions: versionForBranch, + }, nil + +} // Given a local repo on disk, ensure it's in the "traditional" format. This means: // - metadata file is named .pkgname.metadata // - metadata file has the old " SOURCES/" format // - SPECS/ and SOURCES/ exist and are populated correctly -func convertLocalRepo(pkgName string, localRepo string) (bool, error) { - - // Make sure we have a SPECS and SOURCES folder made: - if err := os.MkdirAll(fmt.Sprintf("%s/SOURCES", localRepo), 0755); err != nil { - return false, fmt.Errorf("Could not create SOURCES directory in: %s", localRepo) - } - - if err := os.MkdirAll(fmt.Sprintf("%s/SPECS", localRepo), 0755); err != nil { - return false, fmt.Errorf("Could not create SPECS directory in: %s", localRepo) - } - - // Loop through each file/folder and operate accordingly: - files, err := ioutil.ReadDir(localRepo) - if err != nil { - return false, err - } - - for _, file := range files { - - // We don't want to process SOURCES, SPECS, or any of our .git folders - if file.Name() == "SOURCES" || file.Name() == "SPECS" || strings.HasPrefix(file.Name(), ".git") || file.Name() == "." + pkgName + ".metadata" { - continue - } - - // If we have a metadata "sources" file, we need to read it and convert to the old ..metadata format - if file.Name() == "sources" { - convertStatus := convertMetaData(pkgName, localRepo) - - if convertStatus != true { - return false, fmt.Errorf("Error converting sources metadata file to .metadata format") - } - - continue - } - - // Any file that ends in a ".spec" should be put into SPECS/ - if strings.HasSuffix(file.Name(), ".spec") { - err := os.Rename(fmt.Sprintf("%s/%s", localRepo, file.Name()), fmt.Sprintf("%s/SPECS/%s", localRepo, file.Name()) ) - if err != nil { - return false, fmt.Errorf("Error moving .spec file to SPECS/") - } - } - - // if a file isn't skipped in one of the above checks, then it must be a file that belongs in SOURCES/ - os.Rename(fmt.Sprintf("%s/%s", localRepo, file.Name()), fmt.Sprintf("%s/SOURCES/%s", localRepo, file.Name()) ) - } - - return true, nil +func convertLocalRepo(pkgName string, localRepo string) (bool, error) { + + // Make sure we have a SPECS and SOURCES folder made: + if err := os.MkdirAll(fmt.Sprintf("%s/SOURCES", localRepo), 0755); err != nil { + return false, fmt.Errorf("Could not create SOURCES directory in: %s", localRepo) + } + + if err := os.MkdirAll(fmt.Sprintf("%s/SPECS", localRepo), 0755); err != nil { + return false, fmt.Errorf("Could not create SPECS directory in: %s", localRepo) + } + + // Loop through each file/folder and operate accordingly: + files, err := ioutil.ReadDir(localRepo) + if err != nil { + return false, err + } + + for _, file := range files { + + // We don't want to process SOURCES, SPECS, or any of our .git folders + if file.Name() == "SOURCES" || file.Name() == "SPECS" || strings.HasPrefix(file.Name(), ".git") || file.Name() == "."+pkgName+".metadata" { + continue + } + + // If we have a metadata "sources" file, we need to read it and convert to the old ..metadata format + if file.Name() == "sources" { + convertStatus := convertMetaData(pkgName, localRepo) + + if convertStatus != true { + return false, fmt.Errorf("Error converting sources metadata file to .metadata format") + } + + continue + } + + // Any file that ends in a ".spec" should be put into SPECS/ + if strings.HasSuffix(file.Name(), ".spec") { + err := os.Rename(fmt.Sprintf("%s/%s", localRepo, file.Name()), fmt.Sprintf("%s/SPECS/%s", localRepo, file.Name())) + if err != nil { + return false, fmt.Errorf("Error moving .spec file to SPECS/") + } + } + + // if a file isn't skipped in one of the above checks, then it must be a file that belongs in SOURCES/ + os.Rename(fmt.Sprintf("%s/%s", localRepo, file.Name()), fmt.Sprintf("%s/SOURCES/%s", localRepo, file.Name())) + } + + return true, nil } - -// Given a local "sources" metadata file (new CentOS Stream format), convert it into the older +// Given a local "sources" metadata file (new CentOS Stream format), convert it into the older // classic CentOS style: " SOURCES/" -func convertMetaData(pkgName string, localRepo string) (bool) { - - lookAside, err := os.Open(fmt.Sprintf("%s/sources", localRepo)) - if err != nil { - return false - } - - // Split file into lines and start processing: - scanner := bufio.NewScanner(lookAside) - scanner.Split(bufio.ScanLines) - - // convertedLA is our array of new "converted" lookaside lines - var convertedLA []string - - // loop through each line, and: - // - split by whitespace - // - check each line begins with "SHA" or "MD" - validate - // - take the - // Then check - for scanner.Scan() { - - tmpLine := strings.Fields(scanner.Text()) - // make sure line starts with a "SHA" or "MD" before processing - otherwise it might not be a valid format lookaside line! - if !(strings.HasPrefix(tmpLine[0], "SHA") || strings.HasPrefix(tmpLine[0], "MD")) { - continue - } - - // Strip out "( )" characters from file name and prepend SOURCES/ to it - tmpLine[1] = strings.ReplaceAll(tmpLine[1], "(", "" ) - tmpLine[1] = strings.ReplaceAll(tmpLine[1], ")", "" ) - tmpLine[1] = fmt.Sprintf("SOURCES/%s", tmpLine[1]) - - convertedLA = append(convertedLA, fmt.Sprintf("%s %s", tmpLine[3], tmpLine[1]) ) - - } - lookAside.Close() - - // open ..metadata file for writing our old-format lines - lookAside, err = os.OpenFile(fmt.Sprintf("%s/.%s.metadata", localRepo, pkgName), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) - if err != nil { - fmt.Errorf("Error opening new .metadata file for writing.") - return false - } - - writer := bufio.NewWriter(lookAside) - - for _, convertedLine := range convertedLA { - _, _ = writer.WriteString(convertedLine + "\n") - } - - writer.Flush() - lookAside.Close() - - // Remove old "sources" metadata file - we don't need it now that conversion is complete - os.Remove(fmt.Sprintf("%s/sources", localRepo)) - - return true -} +func convertMetaData(pkgName string, localRepo string) bool { + lookAside, err := os.Open(fmt.Sprintf("%s/sources", localRepo)) + if err != nil { + return false + } + + // Split file into lines and start processing: + scanner := bufio.NewScanner(lookAside) + scanner.Split(bufio.ScanLines) + + // convertedLA is our array of new "converted" lookaside lines + var convertedLA []string + + // loop through each line, and: + // - split by whitespace + // - check each line begins with "SHA" or "MD" - validate + // - take the + // Then check + for scanner.Scan() { + + tmpLine := strings.Fields(scanner.Text()) + // make sure line starts with a "SHA" or "MD" before processing - otherwise it might not be a valid format lookaside line! + if !(strings.HasPrefix(tmpLine[0], "SHA") || strings.HasPrefix(tmpLine[0], "MD")) { + continue + } + + // Strip out "( )" characters from file name and prepend SOURCES/ to it + tmpLine[1] = strings.ReplaceAll(tmpLine[1], "(", "") + tmpLine[1] = strings.ReplaceAll(tmpLine[1], ")", "") + tmpLine[1] = fmt.Sprintf("SOURCES/%s", tmpLine[1]) + + convertedLA = append(convertedLA, fmt.Sprintf("%s %s", tmpLine[3], tmpLine[1])) + + } + lookAside.Close() + + // open ..metadata file for writing our old-format lines + lookAside, err = os.OpenFile(fmt.Sprintf("%s/.%s.metadata", localRepo, pkgName), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + fmt.Errorf("Error opening new .metadata file for writing.") + return false + } + + writer := bufio.NewWriter(lookAside) + + for _, convertedLine := range convertedLA { + _, _ = writer.WriteString(convertedLine + "\n") + } + + writer.Flush() + lookAside.Close() + + // Remove old "sources" metadata file - we don't need it now that conversion is complete + os.Remove(fmt.Sprintf("%s/sources", localRepo)) + + return true +} // Given a local checked out folder and package name, including SPECS/ , SOURCES/ , and .package.metadata, this will: // - create a "dummy" SRPM (using dummy sources files we use to populate tarballs from lookaside) // - extract RPM version info from that SRPM, and return it // If we are in tagless mode, we need to get a package version somehow! -func getVersionFromSpec(pkgName string, localRepo string, majorVersion int) (string) { +func getVersionFromSpec(pkgName string, localRepo string, majorVersion int) string { - // Make sure we have "rpm" and "rpmbuild" and "cp" available in our PATH. Otherwise, this won't work: - _, err := exec.LookPath("rpm") + // Make sure we have "rpm" and "rpmbuild" and "cp" available in our PATH. Otherwise, this won't work: + _, err := exec.LookPath("rpm") if err != nil { return "" } - _, err = exec.LookPath("rpmbuild") + _, err = exec.LookPath("rpmbuild") if err != nil { return "" } - + _, err = exec.LookPath("cp") if err != nil { return "" } - - // create separate temp folder space to do our RPM work - we don't want to accidentally contaminate the main Git area: - rpmBuildPath := fmt.Sprintf("%s_rpm", localRepo) - os.Mkdir(rpmBuildPath, 0755 ) - - // Copy SOURCES/ and SPECS/ into the temp rpmbuild directory recursively - // Yes, we could create or import an elaborate Go-native way to do this, but damnit this is easier: - cmdArgs := strings.Fields(fmt.Sprintf("cp -rp %s/SOURCES %s/SPECS %s/", localRepo, localRepo, rpmBuildPath)) - if err := exec.Command(cmdArgs[0], cmdArgs[1:]... ).Run(); err != nil { - log.Println(err) - return "" - } + // create separate temp folder space to do our RPM work - we don't want to accidentally contaminate the main Git area: + rpmBuildPath := fmt.Sprintf("%s_rpm", localRepo) + os.Mkdir(rpmBuildPath, 0755) - - // Loop through ..metadata and get the file names we need to make our SRPM: - lookAside, err := os.Open(fmt.Sprintf("%s/.%s.metadata", localRepo, pkgName) ) - if err != nil { - log.Println(err) - return "" - } - - // Split file into lines and start processing: - scanner := bufio.NewScanner(lookAside) - scanner.Split(bufio.ScanLines) - - // loop through each line, and: - // - isolate the SOURCES/filename entry - // - write out a dummy file of the same name to rpmBuildPath/SOURCES - for scanner.Scan() { - - // lookaside source is always the 2nd part of the line (after the long SHA sum) - srcFile := strings.Fields(scanner.Text())[1] - - // write a dummy file of the same name into the rpmbuild SOURCES/ directory: - dummyFile, err := os.OpenFile(fmt.Sprintf("%s/%s", rpmBuildPath, srcFile), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) - if err != nil { - return "" - } - writer := bufio.NewWriter(dummyFile) - _, _ = writer.WriteString("This is a dummy lookaside file generated by srpmproc. It is only needed to get a working SRPM and extract version information. Please disregard\n") - writer.Flush() - dummyFile.Close() - } - - lookAside.Close() + // Copy SOURCES/ and SPECS/ into the temp rpmbuild directory recursively + // Yes, we could create or import an elaborate Go-native way to do this, but damnit this is easier: + cmdArgs := strings.Fields(fmt.Sprintf("cp -rp %s/SOURCES %s/SPECS %s/", localRepo, localRepo, rpmBuildPath)) + if err := exec.Command(cmdArgs[0], cmdArgs[1:]...).Run(); err != nil { + log.Println(err) + return "" + } + // Loop through ..metadata and get the file names we need to make our SRPM: + lookAside, err := os.Open(fmt.Sprintf("%s/.%s.metadata", localRepo, pkgName)) + if err != nil { + log.Println(err) + return "" + } - // Now, call rpmbuild to produce the dummy src file: - // Example: rpmbuild --define "_topdir /tmp/srpmproctmp_httpd1988142783_rpm" -bs /tmp/srpmproctmp_httpd1988142783_rpm/SPECS/*.spec - cmd := exec.Command("rpmbuild", fmt.Sprintf(`--define=_topdir %s`, rpmBuildPath), fmt.Sprintf(`--define=dist .el%d`, majorVersion), "-bs", fmt.Sprintf("%s/SPECS/%s.spec", rpmBuildPath, pkgName) ) - if err := cmd.Run(); err != nil { - log.Println(err) - return "" - } - - // Read the first file from the SRPMS/ folder in rpmBuildPath. It should be the SRPM that rpmbuild produced above - // (there should only be one file - we check that it ends in ".rpm" just to be sure!) - lsTmp, err := ioutil.ReadDir(fmt.Sprintf("%s/SRPMS/", rpmBuildPath)) - if err != nil { - log.Println(err) - return "" - } - - srpmFile := lsTmp[0].Name() - - if !strings.HasSuffix(srpmFile, ".rpm") { - log.Println("Error, file found in dummy SRPMS directory did not have an .rpm extension! Perhaps rpmbuild didn't produce a proper source RPM?") - return "" - } - - - // Call the rpm binary to extract the version-release info out of it, and tack on ".el" at the end: - cmd = exec.Command("rpm", "-qp", "--qf", `%{NAME}|%{VERSION}|%{RELEASE}\n`, fmt.Sprintf("%s/SRPMS/%s", rpmBuildPath, srpmFile) ) - nvrTmp, err := cmd.CombinedOutput() - if err != nil { - log.Println("Error running rpm command to extract temporary SRPM name-version-release identifiers.") - log.Println("rpmbuild output: ", string(nvrTmp)) - log.Println("rpmbuild command: ", cmd.String()) - return "" - } + // Split file into lines and start processing: + scanner := bufio.NewScanner(lookAside) + scanner.Split(bufio.ScanLines) - // Pull first line of the rpm command's output to get the name-version-release number (there should only be 1 line) - nvr := string(nvrTmp) - nvr = strings.Fields(nvr)[0] - - // Clean up: delete the temporary directory - if err := os.RemoveAll(rpmBuildPath); err != nil { - log.Printf("Error cleaning up temporary RPM directory %s . Non-fatal, continuing anyway...\n", rpmBuildPath) - } + // loop through each line, and: + // - isolate the SOURCES/filename entry + // - write out a dummy file of the same name to rpmBuildPath/SOURCES + for scanner.Scan() { + + // lookaside source is always the 2nd part of the line (after the long SHA sum) + srcFile := strings.Fields(scanner.Text())[1] + + // write a dummy file of the same name into the rpmbuild SOURCES/ directory: + dummyFile, err := os.OpenFile(fmt.Sprintf("%s/%s", rpmBuildPath, srcFile), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + return "" + } + writer := bufio.NewWriter(dummyFile) + _, _ = writer.WriteString("This is a dummy lookaside file generated by srpmproc. It is only needed to get a working SRPM and extract version information. Please disregard\n") + writer.Flush() + dummyFile.Close() + } + + lookAside.Close() + + // Now, call rpmbuild to produce the dummy src file: + // Example: rpmbuild --define "_topdir /tmp/srpmproctmp_httpd1988142783_rpm" -bs /tmp/srpmproctmp_httpd1988142783_rpm/SPECS/*.spec + cmd := exec.Command("rpmbuild", fmt.Sprintf(`--define=_topdir %s`, rpmBuildPath), fmt.Sprintf(`--define=dist .el%d`, majorVersion), "-bs", fmt.Sprintf("%s/SPECS/%s.spec", rpmBuildPath, pkgName)) + if err := cmd.Run(); err != nil { + log.Println(err) + return "" + } + + // Read the first file from the SRPMS/ folder in rpmBuildPath. It should be the SRPM that rpmbuild produced above + // (there should only be one file - we check that it ends in ".rpm" just to be sure!) + lsTmp, err := ioutil.ReadDir(fmt.Sprintf("%s/SRPMS/", rpmBuildPath)) + if err != nil { + log.Println(err) + return "" + } + + srpmFile := lsTmp[0].Name() + + if !strings.HasSuffix(srpmFile, ".rpm") { + log.Println("Error, file found in dummy SRPMS directory did not have an .rpm extension! Perhaps rpmbuild didn't produce a proper source RPM?") + return "" + } + + // Call the rpm binary to extract the version-release info out of it, and tack on ".el" at the end: + cmd = exec.Command("rpm", "-qp", "--qf", `%{NAME}|%{VERSION}|%{RELEASE}\n`, fmt.Sprintf("%s/SRPMS/%s", rpmBuildPath, srpmFile)) + nvrTmp, err := cmd.CombinedOutput() + if err != nil { + log.Println("Error running rpm command to extract temporary SRPM name-version-release identifiers.") + log.Println("rpmbuild output: ", string(nvrTmp)) + log.Println("rpmbuild command: ", cmd.String()) + return "" + } + + // Pull first line of the rpm command's output to get the name-version-release number (there should only be 1 line) + nvr := string(nvrTmp) + nvr = strings.Fields(nvr)[0] + + // Clean up: delete the temporary directory + if err := os.RemoveAll(rpmBuildPath); err != nil { + log.Printf("Error cleaning up temporary RPM directory %s . Non-fatal, continuing anyway...\n", rpmBuildPath) + } + + // return name-version-release string we derived: + log.Printf("Derived NVR %s from tagless repo via temporary SRPM build\n", nvr) + return nvr - // return name-version-release string we derived: - log.Printf("Derived NVR %s from tagless repo via temporary SRPM build\n", nvr) - return nvr - } - - -// We need to loop through the lookaside blob files ("SourcesToIgnore"), +// We need to loop through the lookaside blob files ("SourcesToIgnore"), // and upload them to our target storage (usually an S3 bucket, but could be a local folder) -// +// // We also need to add the source paths to .gitignore in the git repo, so we don't accidentally commit + push them -func processLookasideSources(pd *data.ProcessData, md *data.ModeData, localDir string) (error) { - - w := md.Worktree - metadata, err := w.Filesystem.Create(fmt.Sprintf(".%s.metadata", md.Name) ) - if err != nil { - return fmt.Errorf("could not create metadata file: %v", err) - } - - // Keep track of files we've already uploaded - don't want duplicates! - var alreadyUploadedBlobs []string +func processLookasideSources(pd *data.ProcessData, md *data.ModeData, localDir string) error { + + w := md.Worktree + metadata, err := w.Filesystem.Create(fmt.Sprintf(".%s.metadata", md.Name)) + if err != nil { + return fmt.Errorf("could not create metadata file: %v", err) + } + + // Keep track of files we've already uploaded - don't want duplicates! + var alreadyUploadedBlobs []string + + gitIgnore, err := os.OpenFile(fmt.Sprintf("%s/.gitignore", localDir), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + return err + } + + for _, source := range md.SourcesToIgnore { - - gitIgnore, err := os.OpenFile(fmt.Sprintf("%s/.gitignore", localDir), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) - if err != nil { - return err - } - - - for _, source := range md.SourcesToIgnore { - sourcePath := source.Name _, err := w.Filesystem.Stat(sourcePath) if source.Expired || err != nil { @@ -1312,7 +1272,7 @@ func processLookasideSources(pd *data.ProcessData, md *data.ModeData, localDir s } exists, err := pd.BlobStorage.Exists(checksum) if err != nil { - return err + return err } if !exists && !pd.NoStorageUpload { err := pd.BlobStorage.Write(checksum, sourceFileBts) @@ -1321,31 +1281,24 @@ func processLookasideSources(pd *data.ProcessData, md *data.ModeData, localDir s } pd.Log.Printf("wrote %s to blob storage", checksum) } - alreadyUploadedBlobs = append(alreadyUploadedBlobs, checksum) - - // Add this SOURCES/ lookaside file to be excluded - w.Excludes = append(w.Excludes, gitignore.ParsePattern(sourcePath, nil) ) - - // Append the SOURCES/ path to .gitignore: - _, err = gitIgnore.Write([]byte(fmt.Sprintf("%s\n", sourcePath)) ) - if err != nil { - return err - } - + alreadyUploadedBlobs = append(alreadyUploadedBlobs, checksum) + + // Add this SOURCES/ lookaside file to be excluded + w.Excludes = append(w.Excludes, gitignore.ParsePattern(sourcePath, nil)) + + // Append the SOURCES/ path to .gitignore: + _, err = gitIgnore.Write([]byte(fmt.Sprintf("%s\n", sourcePath))) + if err != nil { + return err + } + } - err = gitIgnore.Close() if err != nil { - return err + return err } - - return nil + + return nil } - - - - - -