Cleaned up formatting w/ gofmt

(no functional changes)

-Skip G.
This commit is contained in:
Skip Grube 2022-09-09 17:19:13 -04:00
parent f339965767
commit f408541841
3 changed files with 551 additions and 611 deletions

View File

@ -150,7 +150,6 @@ func main() {
root.Flags().BoolVar(&taglessMode, "taglessmode", false, "Tagless mode: If set, pull the latest commit from a branch, and determine version info from spec file (aka upstream versions aren't tagged)")
root.Flags().BoolVar(&altLookAside, "altlookaside", false, "If set, uses the new CentOS Stream lookaside pattern (https://<SITE_PREFIX>/<RPM_NAME>/<FILE_NAME>/<SHA_VERSION>/<SHA_SUM>/<FILE_NAME>)")
if err := root.Execute(); err != nil {
log.Fatal(err)
}

View File

@ -125,7 +125,6 @@ func (g *GitMode) RetrieveSource(pd *data.ProcessData) (*data.ModeData, error) {
return nil
}
// In case of "tagless mode", we need to get the head ref of the branch instead
// This is a kind of alternative implementation of the above tagAdd assignment
refAdd := func(tag *object.Tag) error {
@ -133,7 +132,6 @@ func (g *GitMode) RetrieveSource(pd *data.ProcessData) (*data.ModeData, error) {
pd.Log.Printf("Tagless mode: Identified tagless commit for import: %s\n", tag.Name)
refSpec := fmt.Sprintf(tag.Name)
// We split the string by "/", the branch name we're looking for to pass to latestTags is always last
// (ex: "refs/heads/c9s" ---> we want latestTags[c9s]
_tmpRef := strings.Split(refSpec, "/")
@ -145,7 +143,6 @@ func (g *GitMode) RetrieveSource(pd *data.ProcessData) (*data.ModeData, error) {
return nil
}
latestTags[_branchName] = &remoteTarget{
remote: refSpec,
when: tag.Tagger.When,
@ -154,14 +151,12 @@ func (g *GitMode) RetrieveSource(pd *data.ProcessData) (*data.ModeData, error) {
return nil
}
tagIter, err := repo.TagObjects()
if err != nil {
return nil, fmt.Errorf("could not get tag objects: %v", err)
}
// tagless mode means we use "refAdd" (add commit by reference)
// normal mode means we can rely on "tagAdd" (the tag should be present for us in the source repo)
if pd.TaglessMode == true {
@ -170,7 +165,6 @@ func (g *GitMode) RetrieveSource(pd *data.ProcessData) (*data.ModeData, error) {
_ = tagIter.ForEach(tagAdd)
}
listOpts := &git.ListOptions{
Auth: pd.Authenticator,
}
@ -197,7 +191,6 @@ func (g *GitMode) RetrieveSource(pd *data.ProcessData) (*data.ModeData, error) {
continue
}
// Call refAdd instead of tagAdd in the case of TaglessMode enabled
if pd.TaglessMode == true {
_ = refAdd(&object.Tag{
@ -241,13 +234,9 @@ func (g *GitMode) WriteSource(pd *data.ProcessData, md *data.ModeData) error {
return fmt.Errorf("could not get upstream remote: %v", err)
}
var refspec config.RefSpec
var branchName string
fmt.Printf("pd.AltLookaside == %v , pd.CdnUrl == %s \n", pd.AltLookAside, pd.CdnUrl)
// In the case of tagless mode, we already have the transformed repo sitting in the worktree,
// and don't need to perform any checkout or fetch operations
if pd.TaglessMode == false {
@ -262,7 +251,6 @@ func (g *GitMode) WriteSource(pd *data.ProcessData, md *data.ModeData) error {
}
pd.Log.Printf("checking out upstream refspec %s", refspec)
fetchOpts := &git.FetchOptions{
Auth: pd.Authenticator,
RemoteName: "upstream",

View File

@ -21,6 +21,7 @@
package srpmproc
import (
"bufio"
"encoding/hex"
"fmt"
"github.com/go-git/go-billy/v5"
@ -46,7 +47,6 @@ import (
"path/filepath"
"strings"
"time"
"bufio"
"github.com/go-git/go-billy/v5/memfs"
"github.com/go-git/go-git/v5"
@ -367,8 +367,6 @@ func ProcessRPM(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error) {
}
}
for _, branch := range md.Branches {
md.Repo = &sourceRepo
md.Worktree = &sourceWorktree
@ -394,7 +392,6 @@ func ProcessRPM(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error) {
matchString = md.TagBranch
}
match := misc.GetTagImportRegex(pd).FindStringSubmatch(matchString)
md.PushBranch = pd.BranchPrefix + strings.TrimPrefix(match[2], pd.ImportBranchPrefix)
@ -663,7 +660,6 @@ func ProcessRPM(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error) {
pushRefspecs = append(pushRefspecs, config.RefSpec("HEAD:"+plumbing.NewTagReferenceName(newTag)))
err = repo.Push(&git.PushOptions{
RemoteName: "origin",
Auth: pd.Authenticator,
@ -684,7 +680,6 @@ func ProcessRPM(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error) {
}, nil
}
// Process for when we want to import a tagless repo (like from CentOS Stream)
//
func processRPMTagless(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error) {
@ -699,7 +694,6 @@ func processRPMTagless(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error
latestHashForBranch := map[string]string{}
versionForBranch := map[string]*srpmprocpb.VersionRelease{}
md, err := pd.Importer.RetrieveSource(pd)
if err != nil {
pd.Log.Println("Error detected in RetrieveSource!")
@ -714,7 +708,6 @@ func processRPMTagless(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error
remotePrefix = "modules"
}
// Set up our remote URL for pushing our repo to
var tagIgnoreList []string
if pd.NoDupMode {
@ -749,7 +742,6 @@ func processRPMTagless(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error
}
}
sourceRepo := *md.Repo
sourceWorktree := *md.Worktree
localPath := ""
@ -788,7 +780,6 @@ func processRPMTagless(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error
return nil, fmt.Errorf("Error converting repository into SOURCES + SPECS + .package.metadata format")
}
nvrString := getVersionFromSpec(md.Name, localPath, pd.Version)
if nvrString == "" {
return nil, fmt.Errorf("Error using rpm or rpmbuild to build SRPM and determine version info! (tagless mode)")
@ -798,18 +789,15 @@ func processRPMTagless(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error
pd.PackageVersion = strings.Split(nvrString, "|")[1]
pd.PackageRelease = strings.Split(nvrString, "|")[2]
// Set full rpm version: name-version-release (for tagging properly)
rpmVersion := fmt.Sprintf("%s-%s-%s", md.Name, pd.PackageVersion, pd.PackageRelease)
pd.Log.Println("Successfully determined version of tagless checkout: ", rpmVersion)
md.PushBranch = fmt.Sprintf("%s%d%s", pd.BranchPrefix, pd.Version, pd.BranchSuffix)
// Make an initial repo we will use to push to our target
pushRepo, err := git.PlainInit(localPath + "_gitpush", false)
pushRepo, err := git.PlainInit(localPath+"_gitpush", false)
if err != nil {
return nil, fmt.Errorf("could not create new dist Repo: %v", err)
}
@ -819,7 +807,6 @@ func processRPMTagless(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error
return nil, fmt.Errorf("could not get dist Worktree: %v", err)
}
// Create a remote "origin" in our empty git, make the upstream equal to the branch we want to modify
pushUrl := fmt.Sprintf("%s/%s/%s.git", pd.UpstreamPrefix, remotePrefix, gitlabify(md.Name))
refspec := config.RefSpec(fmt.Sprintf("+refs/heads/%s:refs/remotes/origin/%s", md.PushBranch, md.PushBranch))
@ -841,7 +828,6 @@ func processRPMTagless(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error
Auth: pd.Authenticator,
})
refName := plumbing.NewBranchReferenceName(md.PushBranch)
var hash plumbing.Hash
@ -856,13 +842,10 @@ func processRPMTagless(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error
Force: true,
})
os.Rename(fmt.Sprintf("%s/SPECS", localPath), fmt.Sprintf("%s_gitpush/SPECS", localPath) )
os.Rename(fmt.Sprintf("%s/SOURCES", localPath), fmt.Sprintf("%s_gitpush/SOURCES", localPath) )
os.Rename(fmt.Sprintf("%s/.gitignore", localPath), fmt.Sprintf("%s_gitpush/.gitignore", localPath) )
os.Rename(fmt.Sprintf("%s/.%s.metadata", localPath, md.Name ), fmt.Sprintf("%s_gitpush/.%s.metadata", localPath, md.Name) )
os.Rename(fmt.Sprintf("%s/SPECS", localPath), fmt.Sprintf("%s_gitpush/SPECS", localPath))
os.Rename(fmt.Sprintf("%s/SOURCES", localPath), fmt.Sprintf("%s_gitpush/SOURCES", localPath))
os.Rename(fmt.Sprintf("%s/.gitignore", localPath), fmt.Sprintf("%s_gitpush/.gitignore", localPath))
os.Rename(fmt.Sprintf("%s/.%s.metadata", localPath, md.Name), fmt.Sprintf("%s_gitpush/.%s.metadata", localPath, md.Name))
md.Repo = pushRepo
md.Worktree = w
@ -873,16 +856,14 @@ func processRPMTagless(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error
return nil, err
}
// Call function to upload source to target lookaside and
// ensure the sources are added to .gitignore
err = processLookasideSources(pd, md, localPath + "_gitpush")
err = processLookasideSources(pd, md, localPath+"_gitpush")
if err != nil {
return nil, err
}
// Apply patch(es) if needed:
if pd.ModuleMode {
err := patchModuleYaml(pd, md)
@ -896,9 +877,7 @@ func processRPMTagless(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error
}
}
err = w.AddWithOptions(&git.AddOptions{All: true} )
err = w.AddWithOptions(&git.AddOptions{All: true})
if err != nil {
return nil, fmt.Errorf("Error adding SOURCES/ , SPECS/ or .metadata file to commit list.")
}
@ -906,7 +885,6 @@ func processRPMTagless(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error
status, err := w.Status()
fmt.Println("Current repo status == ", status)
// assign tag for our new remote we're about to push (derived from the SRPM version)
newTag := "refs/tags/imports/" + md.PushBranch + "/" + rpmVersion
newTag = strings.Replace(newTag, "%", "_", -1)
@ -915,12 +893,11 @@ func processRPMTagless(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error
// It's an array of colon-separated strings which map local references to their remote counterparts
var pushRefspecs []config.RefSpec
// We need to find out if the remote repo already has this branch
// If it doesn't, we want to add *:* to our references for commit. This will allow us to push the new branch
// If it does, we can simply push HEAD:refs/heads/<BRANCH>
newRepo := true
refList, _ := pushRepoRemote.List(&git.ListOptions{Auth: pd.Authenticator,})
refList, _ := pushRepoRemote.List(&git.ListOptions{Auth: pd.Authenticator})
for _, ref := range refList {
if strings.HasSuffix(ref.Name().String(), fmt.Sprintf("heads/%s", md.PushBranch)) {
newRepo = false
@ -933,13 +910,10 @@ func processRPMTagless(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error
pd.Log.Printf("New remote repo detected, creating new remote branch")
}
// Identify specific references we want to push
// Should be refs/heads/<target_branch>, and a tag called imports/<target_branch>/<rpm_nvr>
pushRefspecs = append(pushRefspecs, config.RefSpec(fmt.Sprintf("HEAD:refs/heads/%s", md.PushBranch)) )
pushRefspecs = append(pushRefspecs, config.RefSpec(fmt.Sprintf("HEAD:%s", newTag)) )
pushRefspecs = append(pushRefspecs, config.RefSpec(fmt.Sprintf("HEAD:refs/heads/%s", md.PushBranch)))
pushRefspecs = append(pushRefspecs, config.RefSpec(fmt.Sprintf("HEAD:%s", newTag)))
// Actually do the commit (locally)
commit, err := w.Commit("import from tagless source "+pd.Importer.ImportName(pd, md), &git.CommitOptions{
@ -960,8 +934,6 @@ func processRPMTagless(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error
pd.Log.Printf("Committed local repo tagless mode transform:\n%s", obj.String())
// After commit, we will now tag our local repo on disk:
_, err = pushRepo.CreateTag(newTag, commit, &git.CreateTagOptions{
Tagger: &object.Signature{
@ -978,7 +950,6 @@ func processRPMTagless(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error
pd.Log.Printf("Pushing these references to the remote: %+v \n", pushRefspecs)
// Do the actual push to the remote target repository
err = pushRepo.Push(&git.PushOptions{
RemoteName: "origin",
@ -1018,7 +989,6 @@ func processRPMTagless(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error
}
// Given a local repo on disk, ensure it's in the "traditional" format. This means:
// - metadata file is named .pkgname.metadata
// - metadata file has the old "<SHASUM> SOURCES/<filename>" format
@ -1043,7 +1013,7 @@ func convertLocalRepo(pkgName string, localRepo string) (bool, error) {
for _, file := range files {
// We don't want to process SOURCES, SPECS, or any of our .git folders
if file.Name() == "SOURCES" || file.Name() == "SPECS" || strings.HasPrefix(file.Name(), ".git") || file.Name() == "." + pkgName + ".metadata" {
if file.Name() == "SOURCES" || file.Name() == "SPECS" || strings.HasPrefix(file.Name(), ".git") || file.Name() == "."+pkgName+".metadata" {
continue
}
@ -1060,23 +1030,22 @@ func convertLocalRepo(pkgName string, localRepo string) (bool, error) {
// Any file that ends in a ".spec" should be put into SPECS/
if strings.HasSuffix(file.Name(), ".spec") {
err := os.Rename(fmt.Sprintf("%s/%s", localRepo, file.Name()), fmt.Sprintf("%s/SPECS/%s", localRepo, file.Name()) )
err := os.Rename(fmt.Sprintf("%s/%s", localRepo, file.Name()), fmt.Sprintf("%s/SPECS/%s", localRepo, file.Name()))
if err != nil {
return false, fmt.Errorf("Error moving .spec file to SPECS/")
}
}
// if a file isn't skipped in one of the above checks, then it must be a file that belongs in SOURCES/
os.Rename(fmt.Sprintf("%s/%s", localRepo, file.Name()), fmt.Sprintf("%s/SOURCES/%s", localRepo, file.Name()) )
os.Rename(fmt.Sprintf("%s/%s", localRepo, file.Name()), fmt.Sprintf("%s/SOURCES/%s", localRepo, file.Name()))
}
return true, nil
}
// Given a local "sources" metadata file (new CentOS Stream format), convert it into the older
// classic CentOS style: "<HASH> SOURCES/<FILENAME>"
func convertMetaData(pkgName string, localRepo string) (bool) {
func convertMetaData(pkgName string, localRepo string) bool {
lookAside, err := os.Open(fmt.Sprintf("%s/sources", localRepo))
if err != nil {
@ -1104,11 +1073,11 @@ func convertMetaData(pkgName string, localRepo string) (bool) {
}
// Strip out "( )" characters from file name and prepend SOURCES/ to it
tmpLine[1] = strings.ReplaceAll(tmpLine[1], "(", "" )
tmpLine[1] = strings.ReplaceAll(tmpLine[1], ")", "" )
tmpLine[1] = strings.ReplaceAll(tmpLine[1], "(", "")
tmpLine[1] = strings.ReplaceAll(tmpLine[1], ")", "")
tmpLine[1] = fmt.Sprintf("SOURCES/%s", tmpLine[1])
convertedLA = append(convertedLA, fmt.Sprintf("%s %s", tmpLine[3], tmpLine[1]) )
convertedLA = append(convertedLA, fmt.Sprintf("%s %s", tmpLine[3], tmpLine[1]))
}
lookAside.Close()
@ -1135,12 +1104,11 @@ func convertMetaData(pkgName string, localRepo string) (bool) {
return true
}
// Given a local checked out folder and package name, including SPECS/ , SOURCES/ , and .package.metadata, this will:
// - create a "dummy" SRPM (using dummy sources files we use to populate tarballs from lookaside)
// - extract RPM version info from that SRPM, and return it
// If we are in tagless mode, we need to get a package version somehow!
func getVersionFromSpec(pkgName string, localRepo string, majorVersion int) (string) {
func getVersionFromSpec(pkgName string, localRepo string, majorVersion int) string {
// Make sure we have "rpm" and "rpmbuild" and "cp" available in our PATH. Otherwise, this won't work:
_, err := exec.LookPath("rpm")
@ -1158,22 +1126,20 @@ func getVersionFromSpec(pkgName string, localRepo string, majorVersion int) (str
return ""
}
// create separate temp folder space to do our RPM work - we don't want to accidentally contaminate the main Git area:
rpmBuildPath := fmt.Sprintf("%s_rpm", localRepo)
os.Mkdir(rpmBuildPath, 0755 )
os.Mkdir(rpmBuildPath, 0755)
// Copy SOURCES/ and SPECS/ into the temp rpmbuild directory recursively
// Yes, we could create or import an elaborate Go-native way to do this, but damnit this is easier:
cmdArgs := strings.Fields(fmt.Sprintf("cp -rp %s/SOURCES %s/SPECS %s/", localRepo, localRepo, rpmBuildPath))
if err := exec.Command(cmdArgs[0], cmdArgs[1:]... ).Run(); err != nil {
if err := exec.Command(cmdArgs[0], cmdArgs[1:]...).Run(); err != nil {
log.Println(err)
return ""
}
// Loop through .<package>.metadata and get the file names we need to make our SRPM:
lookAside, err := os.Open(fmt.Sprintf("%s/.%s.metadata", localRepo, pkgName) )
lookAside, err := os.Open(fmt.Sprintf("%s/.%s.metadata", localRepo, pkgName))
if err != nil {
log.Println(err)
return ""
@ -1204,10 +1170,9 @@ func getVersionFromSpec(pkgName string, localRepo string, majorVersion int) (str
lookAside.Close()
// Now, call rpmbuild to produce the dummy src file:
// Example: rpmbuild --define "_topdir /tmp/srpmproctmp_httpd1988142783_rpm" -bs /tmp/srpmproctmp_httpd1988142783_rpm/SPECS/*.spec
cmd := exec.Command("rpmbuild", fmt.Sprintf(`--define=_topdir %s`, rpmBuildPath), fmt.Sprintf(`--define=dist .el%d`, majorVersion), "-bs", fmt.Sprintf("%s/SPECS/%s.spec", rpmBuildPath, pkgName) )
cmd := exec.Command("rpmbuild", fmt.Sprintf(`--define=_topdir %s`, rpmBuildPath), fmt.Sprintf(`--define=dist .el%d`, majorVersion), "-bs", fmt.Sprintf("%s/SPECS/%s.spec", rpmBuildPath, pkgName))
if err := cmd.Run(); err != nil {
log.Println(err)
return ""
@ -1228,9 +1193,8 @@ func getVersionFromSpec(pkgName string, localRepo string, majorVersion int) (str
return ""
}
// Call the rpm binary to extract the version-release info out of it, and tack on ".el<VERSION>" at the end:
cmd = exec.Command("rpm", "-qp", "--qf", `%{NAME}|%{VERSION}|%{RELEASE}\n`, fmt.Sprintf("%s/SRPMS/%s", rpmBuildPath, srpmFile) )
cmd = exec.Command("rpm", "-qp", "--qf", `%{NAME}|%{VERSION}|%{RELEASE}\n`, fmt.Sprintf("%s/SRPMS/%s", rpmBuildPath, srpmFile))
nvrTmp, err := cmd.CombinedOutput()
if err != nil {
log.Println("Error running rpm command to extract temporary SRPM name-version-release identifiers.")
@ -1254,16 +1218,14 @@ func getVersionFromSpec(pkgName string, localRepo string, majorVersion int) (str
}
// We need to loop through the lookaside blob files ("SourcesToIgnore"),
// and upload them to our target storage (usually an S3 bucket, but could be a local folder)
//
// We also need to add the source paths to .gitignore in the git repo, so we don't accidentally commit + push them
func processLookasideSources(pd *data.ProcessData, md *data.ModeData, localDir string) (error) {
func processLookasideSources(pd *data.ProcessData, md *data.ModeData, localDir string) error {
w := md.Worktree
metadata, err := w.Filesystem.Create(fmt.Sprintf(".%s.metadata", md.Name) )
metadata, err := w.Filesystem.Create(fmt.Sprintf(".%s.metadata", md.Name))
if err != nil {
return fmt.Errorf("could not create metadata file: %v", err)
}
@ -1271,13 +1233,11 @@ func processLookasideSources(pd *data.ProcessData, md *data.ModeData, localDir s
// Keep track of files we've already uploaded - don't want duplicates!
var alreadyUploadedBlobs []string
gitIgnore, err := os.OpenFile(fmt.Sprintf("%s/.gitignore", localDir), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
return err
}
for _, source := range md.SourcesToIgnore {
sourcePath := source.Name
@ -1324,17 +1284,16 @@ func processLookasideSources(pd *data.ProcessData, md *data.ModeData, localDir s
alreadyUploadedBlobs = append(alreadyUploadedBlobs, checksum)
// Add this SOURCES/ lookaside file to be excluded
w.Excludes = append(w.Excludes, gitignore.ParsePattern(sourcePath, nil) )
w.Excludes = append(w.Excludes, gitignore.ParsePattern(sourcePath, nil))
// Append the SOURCES/<file> path to .gitignore:
_, err = gitIgnore.Write([]byte(fmt.Sprintf("%s\n", sourcePath)) )
_, err = gitIgnore.Write([]byte(fmt.Sprintf("%s\n", sourcePath)))
if err != nil {
return err
}
}
err = gitIgnore.Close()
if err != nil {
return err
@ -1343,9 +1302,3 @@ func processLookasideSources(pd *data.ProcessData, md *data.ModeData, localDir s
return nil
}