feature: "tagless import" feature mode (#4)
In order to build from lookahead sources, srpmproc must support the new gitlab address for centos 9 stream as well as manipulating the sources to expected format. Resolves #2. - Options for tagless import and new stream lookaside format added - Git Mode now supports scanning for branch head in addition to the "normal" pulling of specific version tags - Alternate ProcessRPM added for Tagless mode in process.go (still a WIP) - Tagless mode converts a repo to the "traditional" format (SPECS/ + SOURCES/ + <pkg>.metadata ) - Tagless mode will build a dummy srpm to determine NVR information (rpmbuild + rpm shell commands) (will use this to tag imports in the target git) - Limitation: Tagless imports only pull the latest head from a branch - CentOS-Stream import branches are converted from stream-<MODULE_VERSION>-<RHEL_VERSION> to the more familiar r9s-stream-<VERSION> - stream-style YAML is detected and converted for modules, similar to the older modulemd.src.txt files - This new pattern is for "tagless mode" only, previous tagged imports (from git.centos.org) should not be affected
This commit is contained in:
skip77
GitHub
parent
9b44bc6e50
commit
cff0cc0fbf
|
|
@ -56,6 +56,8 @@ var (
|
|||
basicPassword string
|
||||
packageVersion string
|
||||
packageRelease string
|
||||
taglessMode bool
|
||||
altLookAside bool
|
||||
)
|
||||
|
||||
var root = &cobra.Command{
|
||||
|
|
@ -92,7 +94,10 @@ func mn(_ *cobra.Command, _ []string) {
|
|||
HttpPassword: basicPassword,
|
||||
PackageVersion: packageVersion,
|
||||
PackageRelease: packageRelease,
|
||||
TaglessMode: taglessMode,
|
||||
AltLookAside: altLookAside,
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
|
@ -142,6 +147,8 @@ func main() {
|
|||
root.Flags().StringVar(&basicPassword, "basic-password", "", "Basic auth password")
|
||||
root.Flags().StringVar(&packageVersion, "package-version", "", "Package version to fetch")
|
||||
root.Flags().StringVar(&packageRelease, "package-release", "", "Package release to fetch")
|
||||
root.Flags().BoolVar(&taglessMode, "taglessmode", false, "Tagless mode: If set, pull the latest commit from a branch, and determine version info from spec file (aka upstream versions aren't tagged)")
|
||||
root.Flags().BoolVar(&altLookAside, "altlookaside", false, "If set, uses the new CentOS Stream lookaside pattern (https://<SITE_PREFIX>/<RPM_NAME>/<FILE_NAME>/<SHA_VERSION>/<SHA_SUM>/<FILE_NAME>)")
|
||||
|
||||
if err := root.Execute(); err != nil {
|
||||
log.Fatal(err)
|
||||
|
|
|
|||
|
|
@ -57,4 +57,6 @@ type ProcessData struct {
|
|||
Log *log.Logger
|
||||
PackageVersion string
|
||||
PackageRelease string
|
||||
TaglessMode bool
|
||||
AltLookAside bool
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import (
|
|||
"github.com/rocky-linux/srpmproc/pkg/data"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func GetTagImportRegex(pd *data.ProcessData) *regexp.Regexp {
|
||||
|
|
@ -31,3 +32,22 @@ func GetTagImportRegex(pd *data.ProcessData) *regexp.Regexp {
|
|||
|
||||
return regexp.MustCompile(regex)
|
||||
}
|
||||
|
||||
// Given a git reference in tagless mode (like "refs/heads/c9s", or "refs/heads/stream-httpd-2.4-rhel-9.1.0"), determine
|
||||
// if we are ok with importing that reference. We are looking for the traditional <prefix><version><suffix> pattern, like "c9s", and also the
|
||||
// modular "stream-<NAME>-<VERSION>-rhel-<VERSION> branch pattern as well
|
||||
func TaglessRefOk(tag string, pd *data.ProcessData) bool {
|
||||
|
||||
// First case is very easy: if we are "refs/heads/<prefix><version><suffix>" , then this is def. a branch we should import
|
||||
if strings.HasPrefix(tag, fmt.Sprintf("refs/heads/%s%d%s", pd.ImportBranchPrefix, pd.Version, pd.BranchSuffix)) {
|
||||
return true
|
||||
}
|
||||
|
||||
// Less easy: if a modular branch is present (starts w/ "stream-"), we need to check if it's part of our major version, and return true if it is
|
||||
// (major version means we look for the text "rhel-X." in the branch name, like "rhel-9.1.0")
|
||||
if strings.HasPrefix(tag, "refs/heads/stream-") && strings.Contains(tag, fmt.Sprintf("rhel-%d.", pd.Version)) {
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
|
|
|||
161
pkg/modes/git.go
161
pkg/modes/git.go
|
|
@ -116,7 +116,6 @@ func (g *GitMode) RetrieveSource(pd *data.ProcessData) (*data.ModeData, error) {
|
|||
if exists != nil && exists.when.After(tag.Tagger.When) {
|
||||
return nil
|
||||
}
|
||||
|
||||
latestTags[match[2]] = &remoteTarget{
|
||||
remote: refSpec,
|
||||
when: tag.Tagger.When,
|
||||
|
|
@ -126,11 +125,39 @@ func (g *GitMode) RetrieveSource(pd *data.ProcessData) (*data.ModeData, error) {
|
|||
return nil
|
||||
}
|
||||
|
||||
// In case of "tagless mode", we need to get the head ref of the branch instead
|
||||
// This is a kind of alternative implementation of the above tagAdd assignment
|
||||
refAdd := func(tag *object.Tag) error {
|
||||
if misc.TaglessRefOk(tag.Name, pd) {
|
||||
pd.Log.Printf("Tagless mode: Identified tagless commit for import: %s\n", tag.Name)
|
||||
refSpec := fmt.Sprintf(tag.Name)
|
||||
|
||||
// We split the string by "/", the branch name we're looking for to pass to latestTags is always last
|
||||
// (ex: "refs/heads/c9s" ---> we want latestTags[c9s]
|
||||
tmpRef := strings.Split(refSpec, "/")
|
||||
tmpBranchName := tmpRef[(len(tmpRef) - 1)]
|
||||
|
||||
latestTags[tmpBranchName] = &remoteTarget{
|
||||
remote: refSpec,
|
||||
when: tag.Tagger.When,
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
tagIter, err := repo.TagObjects()
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not get tag objects: %v", err)
|
||||
}
|
||||
_ = tagIter.ForEach(tagAdd)
|
||||
|
||||
// tagless mode means we use "refAdd" (add commit by reference)
|
||||
// normal mode means we can rely on "tagAdd" (the tag should be present for us in the source repo)
|
||||
if pd.TaglessMode {
|
||||
_ = tagIter.ForEach(refAdd)
|
||||
} else {
|
||||
_ = tagIter.ForEach(tagAdd)
|
||||
}
|
||||
|
||||
listOpts := &git.ListOptions{
|
||||
Auth: pd.Authenticator,
|
||||
|
|
@ -157,17 +184,26 @@ func (g *GitMode) RetrieveSource(pd *data.ProcessData) (*data.ModeData, error) {
|
|||
if err != nil {
|
||||
continue
|
||||
}
|
||||
_ = tagAdd(&object.Tag{
|
||||
Name: strings.TrimPrefix(string(ref.Name()), "refs/tags/"),
|
||||
Tagger: commit.Committer,
|
||||
})
|
||||
|
||||
// Call refAdd instead of tagAdd in the case of TaglessMode enabled
|
||||
if pd.TaglessMode {
|
||||
_ = refAdd(&object.Tag{
|
||||
Name: string(ref.Name()),
|
||||
Tagger: commit.Committer,
|
||||
})
|
||||
} else {
|
||||
_ = tagAdd(&object.Tag{
|
||||
Name: strings.TrimPrefix(string(ref.Name()), "refs/tags/"),
|
||||
Tagger: commit.Committer,
|
||||
})
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
for _, branch := range latestTags {
|
||||
pd.Log.Printf("tag: %s", strings.TrimPrefix(branch.remote, "refs/tags/"))
|
||||
branches = append(branches, *branch)
|
||||
}
|
||||
|
||||
sort.Sort(branches)
|
||||
|
||||
var sortedBranches []string
|
||||
|
|
@ -185,54 +221,66 @@ func (g *GitMode) RetrieveSource(pd *data.ProcessData) (*data.ModeData, error) {
|
|||
}
|
||||
|
||||
func (g *GitMode) WriteSource(pd *data.ProcessData, md *data.ModeData) error {
|
||||
|
||||
remote, err := md.Repo.Remote("upstream")
|
||||
if err != nil {
|
||||
|
||||
if err != nil && !pd.TaglessMode {
|
||||
return fmt.Errorf("could not get upstream remote: %v", err)
|
||||
}
|
||||
|
||||
var refspec config.RefSpec
|
||||
var branchName string
|
||||
|
||||
if strings.HasPrefix(md.TagBranch, "refs/heads") {
|
||||
refspec = config.RefSpec(fmt.Sprintf("+%s:%s", md.TagBranch, md.TagBranch))
|
||||
branchName = strings.TrimPrefix(md.TagBranch, "refs/heads/")
|
||||
} else {
|
||||
match := misc.GetTagImportRegex(pd).FindStringSubmatch(md.TagBranch)
|
||||
branchName = match[2]
|
||||
refspec = config.RefSpec(fmt.Sprintf("+refs/heads/%s:%s", branchName, md.TagBranch))
|
||||
}
|
||||
pd.Log.Printf("checking out upstream refspec %s", refspec)
|
||||
fetchOpts := &git.FetchOptions{
|
||||
Auth: pd.Authenticator,
|
||||
RemoteName: "upstream",
|
||||
RefSpecs: []config.RefSpec{refspec},
|
||||
Tags: git.AllTags,
|
||||
Force: true,
|
||||
}
|
||||
err = remote.Fetch(fetchOpts)
|
||||
if err != nil && err != git.NoErrAlreadyUpToDate {
|
||||
if err == transport.ErrInvalidAuthMethod || err == transport.ErrAuthenticationRequired {
|
||||
fetchOpts.Auth = nil
|
||||
err = remote.Fetch(fetchOpts)
|
||||
if err != nil && err != git.NoErrAlreadyUpToDate {
|
||||
// In the case of tagless mode, we already have the transformed repo sitting in the worktree,
|
||||
// and don't need to perform any checkout or fetch operations
|
||||
if !pd.TaglessMode {
|
||||
if strings.HasPrefix(md.TagBranch, "refs/heads") {
|
||||
refspec = config.RefSpec(fmt.Sprintf("+%s:%s", md.TagBranch, md.TagBranch))
|
||||
branchName = strings.TrimPrefix(md.TagBranch, "refs/heads/")
|
||||
} else {
|
||||
match := misc.GetTagImportRegex(pd).FindStringSubmatch(md.TagBranch)
|
||||
branchName = match[2]
|
||||
refspec = config.RefSpec(fmt.Sprintf("+refs/heads/%s:%s", branchName, md.TagBranch))
|
||||
fmt.Println("Found branchname that does not start w/ refs/heads :: ", branchName)
|
||||
}
|
||||
pd.Log.Printf("checking out upstream refspec %s", refspec)
|
||||
|
||||
fetchOpts := &git.FetchOptions{
|
||||
Auth: pd.Authenticator,
|
||||
RemoteName: "upstream",
|
||||
RefSpecs: []config.RefSpec{refspec},
|
||||
Tags: git.AllTags,
|
||||
Force: true,
|
||||
}
|
||||
err = remote.Fetch(fetchOpts)
|
||||
if err != nil && err != git.NoErrAlreadyUpToDate {
|
||||
if err == transport.ErrInvalidAuthMethod || err == transport.ErrAuthenticationRequired {
|
||||
fetchOpts.Auth = nil
|
||||
err = remote.Fetch(fetchOpts)
|
||||
if err != nil && err != git.NoErrAlreadyUpToDate {
|
||||
return fmt.Errorf("could not fetch upstream: %v", err)
|
||||
}
|
||||
} else {
|
||||
return fmt.Errorf("could not fetch upstream: %v", err)
|
||||
}
|
||||
} else {
|
||||
return fmt.Errorf("could not fetch upstream: %v", err)
|
||||
}
|
||||
|
||||
err = md.Worktree.Checkout(&git.CheckoutOptions{
|
||||
Branch: plumbing.ReferenceName(md.TagBranch),
|
||||
Force: true,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not checkout source from git: %v", err)
|
||||
}
|
||||
|
||||
_, err = md.Worktree.Add(".")
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not add Worktree: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
err = md.Worktree.Checkout(&git.CheckoutOptions{
|
||||
Branch: plumbing.ReferenceName(md.TagBranch),
|
||||
Force: true,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not checkout source from git: %v", err)
|
||||
}
|
||||
|
||||
_, err = md.Worktree.Add(".")
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not add Worktree: %v", err)
|
||||
if pd.TaglessMode {
|
||||
branchName = fmt.Sprintf("%s%d%s", pd.ImportBranchPrefix, pd.Version, pd.BranchSuffix)
|
||||
}
|
||||
|
||||
metadataPath := ""
|
||||
|
|
@ -292,7 +340,32 @@ func (g *GitMode) WriteSource(pd *data.ProcessData, md *data.ModeData) error {
|
|||
body = fromBlobStorage
|
||||
pd.Log.Printf("downloading %s from blob storage", hash)
|
||||
} else {
|
||||
url := fmt.Sprintf("%s/%s/%s/%s", pd.CdnUrl, md.Name, branchName, hash)
|
||||
|
||||
url := ""
|
||||
// Alternate lookaside logic: if enabled, we pull from a new URL pattern
|
||||
if !pd.AltLookAside {
|
||||
url = fmt.Sprintf("%s/%s/%s/%s", pd.CdnUrl, md.Name, branchName, hash)
|
||||
} else {
|
||||
// We first need the hash algorithm based on length of hash:
|
||||
hashType := "sha512"
|
||||
switch len(hash) {
|
||||
case 128:
|
||||
hashType = "sha512"
|
||||
case 64:
|
||||
hashType = "sha256"
|
||||
case 40:
|
||||
hashType = "sha1"
|
||||
case 32:
|
||||
hashType = "md5"
|
||||
}
|
||||
|
||||
// need the name of the file without "SOURCES/":
|
||||
fileName := strings.Split(path, "/")[1]
|
||||
|
||||
// Alt. lookaside url is of the form: <cdn> / <name> / <filename> / <hashtype> / <hash> / <filename>
|
||||
url = fmt.Sprintf("%s/%s/%s/%s/%s/%s", pd.CdnUrl, md.Name, fileName, hashType, hash, fileName)
|
||||
}
|
||||
|
||||
pd.Log.Printf("downloading %s", url)
|
||||
|
||||
req, err := http.NewRequest("GET", url, nil)
|
||||
|
|
|
|||
|
|
@ -32,6 +32,7 @@ import (
|
|||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/go-git/go-billy/v5"
|
||||
"github.com/go-git/go-billy/v5/memfs"
|
||||
"github.com/go-git/go-git/v5"
|
||||
"github.com/go-git/go-git/v5/config"
|
||||
|
|
@ -286,13 +287,30 @@ func patchModuleYaml(pd *data.ProcessData, md *data.ModeData) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
mdTxtPath := "SOURCES/modulemd.src.txt"
|
||||
f, err := md.Worktree.Filesystem.Open(mdTxtPath)
|
||||
if err != nil {
|
||||
mdTxtPath = "SOURCES/modulemd.txt"
|
||||
mdTxtPath := ""
|
||||
var f billy.File
|
||||
|
||||
// tagless mode implies we're looking for CentOS Stream modules, which are generally "SOURCES/NAME.yaml" (copied to SOURCES/ from import)
|
||||
// if not tagless mode, proceed as usual with SOURCES/modulemd.*.txt
|
||||
if pd.TaglessMode {
|
||||
mdTxtPath = fmt.Sprintf("SOURCES/%s.yaml", md.Name)
|
||||
f, err = md.Worktree.Filesystem.Open(mdTxtPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not open modulemd file: %v", err)
|
||||
mdTxtPath = fmt.Sprintf("SOURCES/%s.yml", md.Name)
|
||||
f, err = md.Worktree.Filesystem.Open(mdTxtPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not open modulemd file: %v", err)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
mdTxtPath = "SOURCES/modulemd.src.txt"
|
||||
f, err = md.Worktree.Filesystem.Open(mdTxtPath)
|
||||
if err != nil {
|
||||
mdTxtPath = "SOURCES/modulemd.txt"
|
||||
f, err = md.Worktree.Filesystem.Open(mdTxtPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not open modulemd file: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -307,11 +325,13 @@ func patchModuleYaml(pd *data.ProcessData, md *data.ModeData) error {
|
|||
}
|
||||
|
||||
// Get stream branch from tag
|
||||
match := misc.GetTagImportRegex(pd).FindStringSubmatch(md.TagBranch)
|
||||
streamBranch := strings.Split(match[2], "-")
|
||||
// Force stream to be the same as stream name in branch
|
||||
module.Data.Stream = streamBranch[len(streamBranch)-1]
|
||||
|
||||
// (in tagless mode we are trusting the "Stream: <VERSION>" text in the source YAML to be accurate)
|
||||
if !pd.TaglessMode {
|
||||
match := misc.GetTagImportRegex(pd).FindStringSubmatch(md.TagBranch)
|
||||
streamBranch := strings.Split(match[2], "-")
|
||||
// Force stream to be the same as stream name in branch
|
||||
module.Data.Stream = streamBranch[len(streamBranch)-1]
|
||||
}
|
||||
log.Println("This module contains the following rpms:")
|
||||
for name := range module.Data.Components.Rpms {
|
||||
pd.Log.Printf("\t- %s", name)
|
||||
|
|
|
|||
|
|
@ -21,10 +21,12 @@
|
|||
package srpmproc
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"github.com/go-git/go-billy/v5"
|
||||
"github.com/go-git/go-billy/v5/osfs"
|
||||
"github.com/go-git/go-git/v5/plumbing/format/gitignore"
|
||||
"github.com/go-git/go-git/v5/plumbing/transport"
|
||||
"github.com/go-git/go-git/v5/plumbing/transport/http"
|
||||
"github.com/go-git/go-git/v5/plumbing/transport/ssh"
|
||||
|
|
@ -40,6 +42,7 @@ import (
|
|||
"io/ioutil"
|
||||
"log"
|
||||
"os"
|
||||
"os/exec"
|
||||
"os/user"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
|
@ -96,6 +99,9 @@ type ProcessDataRequest struct {
|
|||
|
||||
PackageVersion string
|
||||
PackageRelease string
|
||||
|
||||
TaglessMode bool
|
||||
AltLookAside bool
|
||||
}
|
||||
|
||||
func gitlabify(str string) string {
|
||||
|
|
@ -132,9 +138,14 @@ func NewProcessData(req *ProcessDataRequest) (*data.ProcessData, error) {
|
|||
if req.BranchPrefix == "" {
|
||||
req.BranchPrefix = "r"
|
||||
}
|
||||
if req.CdnUrl == "" {
|
||||
if req.CdnUrl == "" && !req.AltLookAside {
|
||||
req.CdnUrl = "https://git.centos.org/sources"
|
||||
}
|
||||
// If altlookaside is enabled, and the CdnUrl hasn't been changed, then automatically set it to the default
|
||||
// CentOS Stream (the new pattern very much won't work with the old git.centos.org/sources site)
|
||||
if (req.CdnUrl == "https://git.centos.org/sources" || req.CdnUrl == "") && req.AltLookAside {
|
||||
req.CdnUrl = "https://sources.stream.centos.org/sources/rpms"
|
||||
}
|
||||
|
||||
// Validate required
|
||||
if req.Package == "" {
|
||||
|
|
@ -263,6 +274,8 @@ func NewProcessData(req *ProcessDataRequest) (*data.ProcessData, error) {
|
|||
Log: logger,
|
||||
PackageVersion: req.PackageVersion,
|
||||
PackageRelease: req.PackageRelease,
|
||||
TaglessMode: req.TaglessMode,
|
||||
AltLookAside: req.AltLookAside,
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
|
@ -274,6 +287,14 @@ func NewProcessData(req *ProcessDataRequest) (*data.ProcessData, error) {
|
|||
// all files that are remote goes into .gitignore
|
||||
// all ignored files' hash goes into .{Name}.metadata
|
||||
func ProcessRPM(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error) {
|
||||
|
||||
// if we are using "tagless mode", then we need to jump to a completely different import process:
|
||||
// Version info needs to be derived from rpmbuild + spec file, not tags
|
||||
if pd.TaglessMode {
|
||||
result, err := processRPMTagless(pd)
|
||||
return result, err
|
||||
}
|
||||
|
||||
md, err := pd.Importer.RetrieveSource(pd)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
|
@ -372,7 +393,9 @@ func ProcessRPM(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error) {
|
|||
}
|
||||
|
||||
match := misc.GetTagImportRegex(pd).FindStringSubmatch(matchString)
|
||||
|
||||
md.PushBranch = pd.BranchPrefix + strings.TrimPrefix(match[2], pd.ImportBranchPrefix)
|
||||
|
||||
newTag := "imports/" + pd.BranchPrefix + strings.TrimPrefix(match[1], "imports/"+pd.ImportBranchPrefix)
|
||||
newTag = strings.Replace(newTag, "%", "_", -1)
|
||||
|
||||
|
|
@ -656,3 +679,660 @@ func ProcessRPM(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error) {
|
|||
BranchVersions: versionForBranch,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Process for when we want to import a tagless repo (like from CentOS Stream)
|
||||
//
|
||||
func processRPMTagless(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error) {
|
||||
pd.Log.Println("Tagless mode detected, attempting import of latest commit")
|
||||
|
||||
// In tagless mode, we *automatically* set StrictBranchMode to true
|
||||
// Only the exact <PREFIX><VERSION><SUFFIX> branch should be pulled from the source repo
|
||||
pd.StrictBranchMode = true
|
||||
|
||||
// our return values: a mapping of branches -> commits (1:1) that we're bringing in,
|
||||
// and a mapping of branches to: version = X, release = Y
|
||||
latestHashForBranch := map[string]string{}
|
||||
versionForBranch := map[string]*srpmprocpb.VersionRelease{}
|
||||
|
||||
md, err := pd.Importer.RetrieveSource(pd)
|
||||
if err != nil {
|
||||
pd.Log.Println("Error detected in RetrieveSource!")
|
||||
return nil, err
|
||||
}
|
||||
|
||||
md.BlobCache = map[string][]byte{}
|
||||
|
||||
// TODO: add tagless module support
|
||||
remotePrefix := "rpms"
|
||||
if pd.ModuleMode {
|
||||
remotePrefix = "modules"
|
||||
}
|
||||
|
||||
// Set up our remote URL for pushing our repo to
|
||||
var tagIgnoreList []string
|
||||
if pd.NoDupMode {
|
||||
repo, err := git.Init(memory.NewStorage(), memfs.New())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not init git repo: %v", err)
|
||||
}
|
||||
remoteUrl := fmt.Sprintf("%s/%s/%s.git", pd.UpstreamPrefix, remotePrefix, gitlabify(md.Name))
|
||||
refspec := config.RefSpec("+refs/heads/*:refs/remotes/origin/*")
|
||||
|
||||
remote, err := repo.CreateRemote(&config.RemoteConfig{
|
||||
Name: "origin",
|
||||
URLs: []string{remoteUrl},
|
||||
Fetch: []config.RefSpec{refspec},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not create remote: %v", err)
|
||||
}
|
||||
|
||||
list, err := remote.List(&git.ListOptions{
|
||||
Auth: pd.Authenticator,
|
||||
})
|
||||
if err != nil {
|
||||
log.Println("ignoring no-dup-mode")
|
||||
} else {
|
||||
for _, ref := range list {
|
||||
if !strings.HasPrefix(string(ref.Name()), "refs/tags/imports") {
|
||||
continue
|
||||
}
|
||||
tagIgnoreList = append(tagIgnoreList, string(ref.Name()))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
sourceRepo := *md.Repo
|
||||
sourceWorktree := *md.Worktree
|
||||
localPath := ""
|
||||
|
||||
for _, branch := range md.Branches {
|
||||
md.Repo = &sourceRepo
|
||||
md.Worktree = &sourceWorktree
|
||||
md.TagBranch = branch
|
||||
|
||||
for _, source := range md.SourcesToIgnore {
|
||||
source.Expired = true
|
||||
}
|
||||
|
||||
// Create a temporary place to check out our tag/branch : /tmp/srpmproctmp_<PKG_NAME><RANDOMSTRING>/
|
||||
localPath, _ = os.MkdirTemp("/tmp", fmt.Sprintf("srpmproctmp_%s", md.Name))
|
||||
|
||||
if err := os.RemoveAll(localPath); err != nil {
|
||||
return nil, fmt.Errorf("Could not remove previous temporary directory: %s", localPath)
|
||||
}
|
||||
if err := os.Mkdir(localPath, 0755); err != nil {
|
||||
return nil, fmt.Errorf("Could not create temporary directory: %s", localPath)
|
||||
}
|
||||
|
||||
// Clone repo into the temporary path, but only the tag we're interested in:
|
||||
// (TODO: will probably need to assign this a variable or use the md struct gitrepo object to perform a successful tag+push later)
|
||||
_, _ = git.PlainClone(localPath, false, &git.CloneOptions{
|
||||
URL: pd.RpmLocation,
|
||||
SingleBranch: true,
|
||||
ReferenceName: plumbing.ReferenceName(branch),
|
||||
})
|
||||
|
||||
// Now that we're cloned into localPath, we need to "covert" the import into the old format
|
||||
// We want sources to become .PKGNAME.metadata, we want SOURCES and SPECS folders, etc.
|
||||
repoFixed, _ := convertLocalRepo(md.Name, localPath)
|
||||
if !repoFixed {
|
||||
return nil, fmt.Errorf("Error converting repository into SOURCES + SPECS + .package.metadata format")
|
||||
}
|
||||
|
||||
// call extra function to determine the proper way to convert the tagless branch name.
|
||||
// c9s becomes r9s (in the usual case), or in the modular case, stream-httpd-2.4-rhel-9.1.0 becomes r9s-stream-httpd-2.4_r9.1.0
|
||||
md.PushBranch = taglessBranchName(branch, pd)
|
||||
|
||||
rpmVersion := ""
|
||||
|
||||
// get name-version-release of tagless repo, only if we're not a module repo:
|
||||
if !pd.ModuleMode {
|
||||
nvrString := getVersionFromSpec(md.Name, localPath, pd.Version)
|
||||
if nvrString == "" {
|
||||
return nil, fmt.Errorf("Error using rpm or rpmbuild to build SRPM and determine version info! (tagless mode)")
|
||||
}
|
||||
|
||||
// Set version and release fields we extracted (name|version|release are separated by pipes)
|
||||
pd.PackageVersion = strings.Split(nvrString, "|")[1]
|
||||
pd.PackageRelease = strings.Split(nvrString, "|")[2]
|
||||
|
||||
// Set full rpm version: name-version-release (for tagging properly)
|
||||
rpmVersion = fmt.Sprintf("%s-%s-%s", md.Name, pd.PackageVersion, pd.PackageRelease)
|
||||
|
||||
pd.Log.Println("Successfully determined version of tagless checkout: ", rpmVersion)
|
||||
} else {
|
||||
// In case of module mode, we just set rpmVersion to the current date - that's what our tag will end up being
|
||||
rpmVersion = time.Now().Format("2006-01-02")
|
||||
}
|
||||
|
||||
// Make an initial repo we will use to push to our target
|
||||
pushRepo, err := git.PlainInit(localPath+"_gitpush", false)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not create new dist Repo: %v", err)
|
||||
}
|
||||
|
||||
w, err := pushRepo.Worktree()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not get dist Worktree: %v", err)
|
||||
}
|
||||
|
||||
// Create a remote "origin" in our empty git, make the upstream equal to the branch we want to modify
|
||||
pushUrl := fmt.Sprintf("%s/%s/%s.git", pd.UpstreamPrefix, remotePrefix, gitlabify(md.Name))
|
||||
refspec := config.RefSpec(fmt.Sprintf("+refs/heads/%s:refs/remotes/origin/%s", md.PushBranch, md.PushBranch))
|
||||
|
||||
// Make our remote repo the target one - the one we want to push our update to
|
||||
pushRepoRemote, err := pushRepo.CreateRemote(&config.RemoteConfig{
|
||||
Name: "origin",
|
||||
URLs: []string{pushUrl},
|
||||
Fetch: []config.RefSpec{refspec},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not create remote: %v", err)
|
||||
}
|
||||
|
||||
// fetch our branch data (md.PushBranch) into this new repo
|
||||
err = pushRepo.Fetch(&git.FetchOptions{
|
||||
RemoteName: "origin",
|
||||
RefSpecs: []config.RefSpec{refspec},
|
||||
Auth: pd.Authenticator,
|
||||
})
|
||||
|
||||
refName := plumbing.NewBranchReferenceName(md.PushBranch)
|
||||
|
||||
var hash plumbing.Hash
|
||||
h := plumbing.NewSymbolicReference(plumbing.HEAD, refName)
|
||||
if err := pushRepo.Storer.CheckAndSetReference(h, nil); err != nil {
|
||||
return nil, fmt.Errorf("Could not set symbolic reference: %v", err)
|
||||
}
|
||||
|
||||
err = w.Checkout(&git.CheckoutOptions{
|
||||
Branch: plumbing.NewRemoteReferenceName("origin", md.PushBranch),
|
||||
Hash: hash,
|
||||
Force: true,
|
||||
})
|
||||
|
||||
os.Rename(fmt.Sprintf("%s/SPECS", localPath), fmt.Sprintf("%s_gitpush/SPECS", localPath))
|
||||
os.Rename(fmt.Sprintf("%s/SOURCES", localPath), fmt.Sprintf("%s_gitpush/SOURCES", localPath))
|
||||
os.Rename(fmt.Sprintf("%s/.gitignore", localPath), fmt.Sprintf("%s_gitpush/.gitignore", localPath))
|
||||
os.Rename(fmt.Sprintf("%s/.%s.metadata", localPath, md.Name), fmt.Sprintf("%s_gitpush/.%s.metadata", localPath, md.Name))
|
||||
|
||||
md.Repo = pushRepo
|
||||
md.Worktree = w
|
||||
|
||||
// Download lookaside sources (tarballs) into the push git repo:
|
||||
err = pd.Importer.WriteSource(pd, md)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Call function to upload source to target lookaside and
|
||||
// ensure the sources are added to .gitignore
|
||||
err = processLookasideSources(pd, md, localPath+"_gitpush")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Apply patch(es) if needed:
|
||||
if pd.ModuleMode {
|
||||
err := patchModuleYaml(pd, md)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
err := executePatchesRpm(pd, md)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
err = w.AddWithOptions(&git.AddOptions{All: true})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Error adding SOURCES/ , SPECS/ or .metadata file to commit list.")
|
||||
}
|
||||
|
||||
status, err := w.Status()
|
||||
pd.Log.Printf("successfully processed:\n%s", status)
|
||||
|
||||
// assign tag for our new remote we're about to push (derived from the SRPM version)
|
||||
newTag := "refs/tags/imports/" + md.PushBranch + "/" + rpmVersion
|
||||
newTag = strings.Replace(newTag, "%", "_", -1)
|
||||
|
||||
// pushRefspecs is a list of all the references we want to push (tags + heads)
|
||||
// It's an array of colon-separated strings which map local references to their remote counterparts
|
||||
var pushRefspecs []config.RefSpec
|
||||
|
||||
// We need to find out if the remote repo already has this branch
|
||||
// If it doesn't, we want to add *:* to our references for commit. This will allow us to push the new branch
|
||||
// If it does, we can simply push HEAD:refs/heads/<BRANCH>
|
||||
newRepo := true
|
||||
refList, _ := pushRepoRemote.List(&git.ListOptions{Auth: pd.Authenticator})
|
||||
for _, ref := range refList {
|
||||
if strings.HasSuffix(ref.Name().String(), fmt.Sprintf("heads/%s", md.PushBranch)) {
|
||||
newRepo = false
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if newRepo {
|
||||
pushRefspecs = append(pushRefspecs, config.RefSpec("*:*"))
|
||||
pd.Log.Printf("New remote repo detected, creating new remote branch")
|
||||
}
|
||||
|
||||
// Identify specific references we want to push
|
||||
// Should be refs/heads/<target_branch>, and a tag called imports/<target_branch>/<rpm_nvr>
|
||||
pushRefspecs = append(pushRefspecs, config.RefSpec(fmt.Sprintf("HEAD:refs/heads/%s", md.PushBranch)))
|
||||
pushRefspecs = append(pushRefspecs, config.RefSpec(fmt.Sprintf("HEAD:%s", newTag)))
|
||||
|
||||
// Actually do the commit (locally)
|
||||
commit, err := w.Commit("import from tagless source "+pd.Importer.ImportName(pd, md), &git.CommitOptions{
|
||||
Author: &object.Signature{
|
||||
Name: pd.GitCommitterName,
|
||||
Email: pd.GitCommitterEmail,
|
||||
When: time.Now(),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not commit object: %v", err)
|
||||
}
|
||||
|
||||
obj, err := pushRepo.CommitObject(commit)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not get commit object: %v", err)
|
||||
}
|
||||
|
||||
pd.Log.Printf("Committed local repo tagless mode transform:\n%s", obj.String())
|
||||
|
||||
// After commit, we will now tag our local repo on disk:
|
||||
_, err = pushRepo.CreateTag(newTag, commit, &git.CreateTagOptions{
|
||||
Tagger: &object.Signature{
|
||||
Name: pd.GitCommitterName,
|
||||
Email: pd.GitCommitterEmail,
|
||||
When: time.Now(),
|
||||
},
|
||||
Message: "import " + md.TagBranch + " from " + pd.RpmLocation + "(import from tagless source)",
|
||||
SignKey: nil,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not create tag: %v", err)
|
||||
}
|
||||
|
||||
pd.Log.Printf("Pushing these references to the remote: %+v \n", pushRefspecs)
|
||||
|
||||
// Do the actual push to the remote target repository
|
||||
err = pushRepo.Push(&git.PushOptions{
|
||||
RemoteName: "origin",
|
||||
Auth: pd.Authenticator,
|
||||
RefSpecs: pushRefspecs,
|
||||
Force: true,
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not push to remote: %v", err)
|
||||
}
|
||||
|
||||
if err := os.RemoveAll(localPath); err != nil {
|
||||
log.Printf("Error cleaning up temporary git checkout directory %s . Non-fatal, continuing anyway...\n", localPath)
|
||||
}
|
||||
if err := os.RemoveAll(fmt.Sprintf("%s_gitpush", localPath)); err != nil {
|
||||
log.Printf("Error cleaning up temporary git checkout directory %s . Non-fatal, continuing anyway...\n", fmt.Sprintf("%s_gitpush", localPath))
|
||||
}
|
||||
|
||||
// append our processed branch to the return structures:
|
||||
latestHashForBranch[md.PushBranch] = obj.Hash.String()
|
||||
|
||||
versionForBranch[md.PushBranch] = &srpmprocpb.VersionRelease{
|
||||
Version: pd.PackageVersion,
|
||||
Release: pd.PackageRelease,
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// return struct with all our branch:commit and branch:version+release mappings
|
||||
return &srpmprocpb.ProcessResponse{
|
||||
BranchCommits: latestHashForBranch,
|
||||
BranchVersions: versionForBranch,
|
||||
}, nil
|
||||
|
||||
}
|
||||
|
||||
// Given a local repo on disk, ensure it's in the "traditional" format. This means:
|
||||
// - metadata file is named .pkgname.metadata
|
||||
// - metadata file has the old "<SHASUM> SOURCES/<filename>" format
|
||||
// - SPECS/ and SOURCES/ exist and are populated correctly
|
||||
func convertLocalRepo(pkgName string, localRepo string) (bool, error) {
|
||||
|
||||
// Make sure we have a SPECS and SOURCES folder made:
|
||||
if err := os.MkdirAll(fmt.Sprintf("%s/SOURCES", localRepo), 0755); err != nil {
|
||||
return false, fmt.Errorf("Could not create SOURCES directory in: %s", localRepo)
|
||||
}
|
||||
|
||||
if err := os.MkdirAll(fmt.Sprintf("%s/SPECS", localRepo), 0755); err != nil {
|
||||
return false, fmt.Errorf("Could not create SPECS directory in: %s", localRepo)
|
||||
}
|
||||
|
||||
// Loop through each file/folder and operate accordingly:
|
||||
files, err := ioutil.ReadDir(localRepo)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
for _, file := range files {
|
||||
|
||||
// We don't want to process SOURCES, SPECS, or any of our .git folders
|
||||
if file.Name() == "SOURCES" || file.Name() == "SPECS" || strings.HasPrefix(file.Name(), ".git") || file.Name() == "."+pkgName+".metadata" {
|
||||
continue
|
||||
}
|
||||
|
||||
// If we have a metadata "sources" file, we need to read it and convert to the old .<pkgname>.metadata format
|
||||
if file.Name() == "sources" {
|
||||
convertStatus := convertMetaData(pkgName, localRepo)
|
||||
|
||||
if convertStatus != true {
|
||||
return false, fmt.Errorf("Error converting sources metadata file to .metadata format")
|
||||
}
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
// Any file that ends in a ".spec" should be put into SPECS/
|
||||
if strings.HasSuffix(file.Name(), ".spec") {
|
||||
err := os.Rename(fmt.Sprintf("%s/%s", localRepo, file.Name()), fmt.Sprintf("%s/SPECS/%s", localRepo, file.Name()))
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("Error moving .spec file to SPECS/")
|
||||
}
|
||||
}
|
||||
|
||||
// if a file isn't skipped in one of the above checks, then it must be a file that belongs in SOURCES/
|
||||
os.Rename(fmt.Sprintf("%s/%s", localRepo, file.Name()), fmt.Sprintf("%s/SOURCES/%s", localRepo, file.Name()))
|
||||
}
|
||||
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// Given a local "sources" metadata file (new CentOS Stream format), convert it into the older
|
||||
// classic CentOS style: "<HASH> SOURCES/<FILENAME>"
|
||||
func convertMetaData(pkgName string, localRepo string) bool {
|
||||
|
||||
lookAside, err := os.Open(fmt.Sprintf("%s/sources", localRepo))
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
// Split file into lines and start processing:
|
||||
scanner := bufio.NewScanner(lookAside)
|
||||
scanner.Split(bufio.ScanLines)
|
||||
|
||||
// convertedLA is our array of new "converted" lookaside lines
|
||||
var convertedLA []string
|
||||
|
||||
// loop through each line, and:
|
||||
// - split by whitespace
|
||||
// - check each line begins with "SHA" or "MD" - validate
|
||||
// - take the
|
||||
// Then check
|
||||
for scanner.Scan() {
|
||||
|
||||
tmpLine := strings.Fields(scanner.Text())
|
||||
// make sure line starts with a "SHA" or "MD" before processing - otherwise it might not be a valid format lookaside line!
|
||||
if !(strings.HasPrefix(tmpLine[0], "SHA") || strings.HasPrefix(tmpLine[0], "MD")) {
|
||||
continue
|
||||
}
|
||||
|
||||
// Strip out "( )" characters from file name and prepend SOURCES/ to it
|
||||
tmpLine[1] = strings.ReplaceAll(tmpLine[1], "(", "")
|
||||
tmpLine[1] = strings.ReplaceAll(tmpLine[1], ")", "")
|
||||
tmpLine[1] = fmt.Sprintf("SOURCES/%s", tmpLine[1])
|
||||
|
||||
convertedLA = append(convertedLA, fmt.Sprintf("%s %s", tmpLine[3], tmpLine[1]))
|
||||
|
||||
}
|
||||
lookAside.Close()
|
||||
|
||||
// open .<NAME>.metadata file for writing our old-format lines
|
||||
lookAside, err = os.OpenFile(fmt.Sprintf("%s/.%s.metadata", localRepo, pkgName), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
|
||||
if err != nil {
|
||||
fmt.Errorf("Error opening new .metadata file for writing.")
|
||||
return false
|
||||
}
|
||||
|
||||
writer := bufio.NewWriter(lookAside)
|
||||
|
||||
for _, convertedLine := range convertedLA {
|
||||
_, _ = writer.WriteString(convertedLine + "\n")
|
||||
}
|
||||
|
||||
writer.Flush()
|
||||
lookAside.Close()
|
||||
|
||||
// Remove old "sources" metadata file - we don't need it now that conversion is complete
|
||||
os.Remove(fmt.Sprintf("%s/sources", localRepo))
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
// Given a local checked out folder and package name, including SPECS/ , SOURCES/ , and .package.metadata, this will:
|
||||
// - create a "dummy" SRPM (using dummy sources files we use to populate tarballs from lookaside)
|
||||
// - extract RPM version info from that SRPM, and return it
|
||||
// If we are in tagless mode, we need to get a package version somehow!
|
||||
func getVersionFromSpec(pkgName string, localRepo string, majorVersion int) string {
|
||||
|
||||
// Make sure we have "rpm" and "rpmbuild" and "cp" available in our PATH. Otherwise, this won't work:
|
||||
_, err := exec.LookPath("rpm")
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
|
||||
_, err = exec.LookPath("rpmbuild")
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
|
||||
_, err = exec.LookPath("cp")
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
|
||||
// create separate temp folder space to do our RPM work - we don't want to accidentally contaminate the main Git area:
|
||||
rpmBuildPath := fmt.Sprintf("%s_rpm", localRepo)
|
||||
os.Mkdir(rpmBuildPath, 0755)
|
||||
|
||||
// Copy SOURCES/ and SPECS/ into the temp rpmbuild directory recursively
|
||||
// Yes, we could create or import an elaborate Go-native way to do this, but damnit this is easier:
|
||||
cmdArgs := strings.Fields(fmt.Sprintf("cp -rp %s/SOURCES %s/SPECS %s/", localRepo, localRepo, rpmBuildPath))
|
||||
if err := exec.Command(cmdArgs[0], cmdArgs[1:]...).Run(); err != nil {
|
||||
log.Println(err)
|
||||
return ""
|
||||
}
|
||||
|
||||
// Loop through .<package>.metadata and get the file names we need to make our SRPM:
|
||||
lookAside, err := os.Open(fmt.Sprintf("%s/.%s.metadata", localRepo, pkgName))
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
return ""
|
||||
}
|
||||
|
||||
// Split file into lines and start processing:
|
||||
scanner := bufio.NewScanner(lookAside)
|
||||
scanner.Split(bufio.ScanLines)
|
||||
|
||||
// loop through each line, and:
|
||||
// - isolate the SOURCES/filename entry
|
||||
// - write out a dummy file of the same name to rpmBuildPath/SOURCES
|
||||
for scanner.Scan() {
|
||||
|
||||
// lookaside source is always the 2nd part of the line (after the long SHA sum)
|
||||
srcFile := strings.Fields(scanner.Text())[1]
|
||||
|
||||
// write a dummy file of the same name into the rpmbuild SOURCES/ directory:
|
||||
dummyFile, err := os.OpenFile(fmt.Sprintf("%s/%s", rpmBuildPath, srcFile), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
writer := bufio.NewWriter(dummyFile)
|
||||
_, _ = writer.WriteString("This is a dummy lookaside file generated by srpmproc. It is only needed to get a working SRPM and extract version information. Please disregard\n")
|
||||
writer.Flush()
|
||||
dummyFile.Close()
|
||||
}
|
||||
|
||||
lookAside.Close()
|
||||
|
||||
// Now, call rpmbuild to produce the dummy src file:
|
||||
// Example: rpmbuild --define "_topdir /tmp/srpmproctmp_httpd1988142783_rpm" -bs /tmp/srpmproctmp_httpd1988142783_rpm/SPECS/*.spec
|
||||
cmd := exec.Command("rpmbuild", fmt.Sprintf(`--define=_topdir %s`, rpmBuildPath), fmt.Sprintf(`--define=dist .el%d`, majorVersion), "-bs", fmt.Sprintf("%s/SPECS/%s.spec", rpmBuildPath, pkgName))
|
||||
if err := cmd.Run(); err != nil {
|
||||
log.Println(err)
|
||||
return ""
|
||||
}
|
||||
|
||||
// Read the first file from the SRPMS/ folder in rpmBuildPath. It should be the SRPM that rpmbuild produced above
|
||||
// (there should only be one file - we check that it ends in ".rpm" just to be sure!)
|
||||
lsTmp, err := ioutil.ReadDir(fmt.Sprintf("%s/SRPMS/", rpmBuildPath))
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
return ""
|
||||
}
|
||||
|
||||
srpmFile := lsTmp[0].Name()
|
||||
|
||||
if !strings.HasSuffix(srpmFile, ".rpm") {
|
||||
log.Println("Error, file found in dummy SRPMS directory did not have an .rpm extension! Perhaps rpmbuild didn't produce a proper source RPM?")
|
||||
return ""
|
||||
}
|
||||
|
||||
// Call the rpm binary to extract the version-release info out of it, and tack on ".el<VERSION>" at the end:
|
||||
cmd = exec.Command("rpm", "-qp", "--qf", `%{NAME}|%{VERSION}|%{RELEASE}\n`, fmt.Sprintf("%s/SRPMS/%s", rpmBuildPath, srpmFile))
|
||||
nvrTmp, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
log.Println("Error running rpm command to extract temporary SRPM name-version-release identifiers.")
|
||||
log.Println("rpmbuild output: ", string(nvrTmp))
|
||||
log.Println("rpmbuild command: ", cmd.String())
|
||||
return ""
|
||||
}
|
||||
|
||||
// Pull first line of the rpm command's output to get the name-version-release number (there should only be 1 line)
|
||||
nvr := string(nvrTmp)
|
||||
nvr = strings.Fields(nvr)[0]
|
||||
|
||||
// Clean up: delete the temporary directory
|
||||
if err := os.RemoveAll(rpmBuildPath); err != nil {
|
||||
log.Printf("Error cleaning up temporary RPM directory %s . Non-fatal, continuing anyway...\n", rpmBuildPath)
|
||||
}
|
||||
|
||||
// return name-version-release string we derived:
|
||||
log.Printf("Derived NVR %s from tagless repo via temporary SRPM build\n", nvr)
|
||||
return nvr
|
||||
|
||||
}
|
||||
|
||||
// We need to loop through the lookaside blob files ("SourcesToIgnore"),
|
||||
// and upload them to our target storage (usually an S3 bucket, but could be a local folder)
|
||||
//
|
||||
// We also need to add the source paths to .gitignore in the git repo, so we don't accidentally commit + push them
|
||||
func processLookasideSources(pd *data.ProcessData, md *data.ModeData, localDir string) error {
|
||||
|
||||
w := md.Worktree
|
||||
metadata, err := w.Filesystem.Create(fmt.Sprintf(".%s.metadata", md.Name))
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not create metadata file: %v", err)
|
||||
}
|
||||
|
||||
// Keep track of files we've already uploaded - don't want duplicates!
|
||||
var alreadyUploadedBlobs []string
|
||||
|
||||
gitIgnore, err := os.OpenFile(fmt.Sprintf("%s/.gitignore", localDir), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, source := range md.SourcesToIgnore {
|
||||
|
||||
sourcePath := source.Name
|
||||
_, err := w.Filesystem.Stat(sourcePath)
|
||||
if source.Expired || err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
sourceFile, err := w.Filesystem.Open(sourcePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not open ignored source file %s: %v", sourcePath, err)
|
||||
}
|
||||
sourceFileBts, err := ioutil.ReadAll(sourceFile)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not read the whole of ignored source file: %v", err)
|
||||
}
|
||||
|
||||
source.HashFunction.Reset()
|
||||
_, err = source.HashFunction.Write(sourceFileBts)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not write bytes to hash function: %v", err)
|
||||
}
|
||||
checksum := hex.EncodeToString(source.HashFunction.Sum(nil))
|
||||
checksumLine := fmt.Sprintf("%s %s\n", checksum, sourcePath)
|
||||
_, err = metadata.Write([]byte(checksumLine))
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not write to metadata file: %v", err)
|
||||
}
|
||||
|
||||
if data.StrContains(alreadyUploadedBlobs, checksum) {
|
||||
continue
|
||||
}
|
||||
exists, err := pd.BlobStorage.Exists(checksum)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !exists && !pd.NoStorageUpload {
|
||||
err := pd.BlobStorage.Write(checksum, sourceFileBts)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pd.Log.Printf("wrote %s to blob storage", checksum)
|
||||
}
|
||||
alreadyUploadedBlobs = append(alreadyUploadedBlobs, checksum)
|
||||
|
||||
// Add this SOURCES/ lookaside file to be excluded
|
||||
w.Excludes = append(w.Excludes, gitignore.ParsePattern(sourcePath, nil))
|
||||
|
||||
// Append the SOURCES/<file> path to .gitignore:
|
||||
_, err = gitIgnore.Write([]byte(fmt.Sprintf("%s\n", sourcePath)))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
err = gitIgnore.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
|
||||
}
|
||||
|
||||
// Given an input branch name to import from, like "refs/heads/c9s", produce the tagless branch name we want to commit to, like "r9s"
|
||||
// Modular translation of CentOS stream branches i is also done - branch stream-maven-3.8-rhel-9.1.0 ----> r9s-stream-maven-3.8_9.1.0
|
||||
func taglessBranchName(fullBranch string, pd *data.ProcessData) string {
|
||||
|
||||
// Split the full branch name "refs/heads/blah" to only get the short name - last entry
|
||||
tmpBranch := strings.Split(fullBranch, "/")
|
||||
branch := tmpBranch[len(tmpBranch)-1]
|
||||
|
||||
// Simple case: if our branch is not a modular stream branch, just return the normal <prefix><version><suffix> pattern
|
||||
if !strings.HasPrefix(branch, "stream-") {
|
||||
return fmt.Sprintf("%s%d%s", pd.BranchPrefix, pd.Version, pd.BranchSuffix)
|
||||
}
|
||||
|
||||
// index where the "-rhel-" starts near the end of the string
|
||||
rhelSpot := strings.LastIndex(branch, "-rhel-")
|
||||
|
||||
// module name will be everything from the start until that "-rhel-" string (like "stream-httpd-2.4")
|
||||
moduleString := branch[0:rhelSpot]
|
||||
|
||||
// major minor version is everything after the "-rhel-" string
|
||||
majorMinor := branch[rhelSpot+6 : len(branch)]
|
||||
|
||||
// return translated modular branch:
|
||||
return fmt.Sprintf("%s%d%s-%s_%s", pd.BranchPrefix, pd.Version, pd.BranchSuffix, moduleString, majorMinor)
|
||||
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue