srpmproc/internal/process.go

390 lines
11 KiB
Go
Raw Normal View History

2021-04-06 19:39:02 +00:00
// Copyright (c) 2021 The Srpmproc Authors
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
2020-12-14 01:29:02 +00:00
package internal
import (
2020-12-14 03:51:03 +00:00
"encoding/hex"
"encoding/json"
2020-12-14 03:51:03 +00:00
"fmt"
"git.rockylinux.org/release-engineering/public/srpmproc/internal/data"
2020-12-14 03:51:03 +00:00
"github.com/go-git/go-billy/v5/memfs"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/config"
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/plumbing/object"
"github.com/go-git/go-git/v5/storage/memory"
"io/ioutil"
"log"
"os"
2020-12-22 16:28:04 +00:00
"path/filepath"
"regexp"
2020-12-14 03:51:03 +00:00
"strings"
"time"
2020-12-14 01:29:02 +00:00
)
var tagImportRegex *regexp.Regexp
2021-03-17 14:54:16 +00:00
func gitlabify(str string) string {
return strings.Replace(str, "+", "plus", -1)
}
2020-12-14 01:29:02 +00:00
// ProcessRPM checks the RPM specs and discards any remote files
// This functions also sorts files into directories
// .spec files goes into -> SPECS
// metadata files goes to root
// source files goes into -> SOURCES
// all files that are remote goes into .gitignore
// all ignored files' hash goes into .{Name}.metadata
func ProcessRPM(pd *data.ProcessData) {
tagImportRegex = regexp.MustCompile(fmt.Sprintf("refs/tags/(imports/(%s.|%s.-.+)/(.*))", pd.ImportBranchPrefix, pd.ImportBranchPrefix))
md := pd.Importer.RetrieveSource(pd)
md.BlobCache = map[string][]byte{}
2020-12-14 03:51:03 +00:00
2021-02-14 20:44:44 +00:00
remotePrefix := "rpms"
2020-12-22 16:28:04 +00:00
if pd.ModuleMode {
remotePrefix = "modules"
}
latestHashForBranch := map[string]string{}
// already uploaded blobs are skipped
var alreadyUploadedBlobs []string
// if no-dup-mode is enabled then skip already imported versions
var tagIgnoreList []string
if pd.NoDupMode {
repo, err := git.Init(memory.NewStorage(), memfs.New())
if err != nil {
2021-03-18 17:56:47 +00:00
log.Fatalf("could not init git repo: %v", err)
}
2021-03-17 14:54:16 +00:00
remoteUrl := fmt.Sprintf("%s/%s/%s.git", pd.UpstreamPrefix, remotePrefix, gitlabify(md.RpmFile.Name()))
refspec := config.RefSpec("+refs/heads/*:refs/remotes/origin/*")
remote, err := repo.CreateRemote(&config.RemoteConfig{
Name: "origin",
URLs: []string{remoteUrl},
Fetch: []config.RefSpec{refspec},
})
if err != nil {
log.Fatalf("could not create remote: %v", err)
}
list, err := remote.List(&git.ListOptions{
Auth: pd.Authenticator,
})
if err != nil {
2020-12-22 16:28:04 +00:00
log.Println("ignoring no-dup-mode")
} else {
for _, ref := range list {
if !strings.HasPrefix(string(ref.Name()), "refs/tags/imports") {
continue
}
tagIgnoreList = append(tagIgnoreList, string(ref.Name()))
}
}
}
sourceRepo := *md.Repo
sourceWorktree := *md.Worktree
2021-04-06 19:35:46 +00:00
commitPin := map[string]string{}
if pd.SingleTag != "" {
md.Branches = []string{fmt.Sprintf("refs/tags/%s", pd.SingleTag)}
2021-04-06 19:35:46 +00:00
} else if len(pd.ManualCommits) > 0 {
md.Branches = []string{}
for _, commit := range pd.ManualCommits {
branchCommit := strings.Split(commit, ":")
if len(branchCommit) != 2 {
log.Fatalln("invalid manual commit list")
}
head := fmt.Sprintf("refs/heads/%s", branchCommit[0])
md.Branches = append(md.Branches, head)
commitPin[head] = branchCommit[1]
}
}
2020-12-14 03:51:03 +00:00
for _, branch := range md.Branches {
md.Repo = &sourceRepo
md.Worktree = &sourceWorktree
md.TagBranch = branch
for _, source := range md.SourcesToIgnore {
source.Expired = true
}
if strings.Contains(md.TagBranch, "-beta") {
continue
}
2020-12-22 16:28:04 +00:00
var matchString string
if !tagImportRegex.MatchString(md.TagBranch) {
2020-12-22 16:28:04 +00:00
if pd.ModuleMode {
prefix := fmt.Sprintf("refs/heads/%s%d", pd.ImportBranchPrefix, pd.Version)
if strings.HasPrefix(md.TagBranch, prefix) {
replace := strings.Replace(md.TagBranch, "refs/heads/", "", 1)
2020-12-22 16:28:04 +00:00
matchString = fmt.Sprintf("refs/tags/imports/%s/%s", replace, filepath.Base(pd.RpmLocation))
log.Printf("using match string: %s", matchString)
}
}
if !tagImportRegex.MatchString(matchString) {
continue
2020-12-22 16:28:04 +00:00
}
} else {
matchString = md.TagBranch
}
2020-12-14 03:51:03 +00:00
2020-12-22 16:28:04 +00:00
match := tagImportRegex.FindStringSubmatch(matchString)
md.PushBranch = pd.BranchPrefix + strings.TrimPrefix(match[2], pd.ImportBranchPrefix)
newTag := "imports/" + pd.BranchPrefix + strings.TrimPrefix(match[1], "imports/"+pd.ImportBranchPrefix)
2021-03-18 17:44:02 +00:00
rpmFile := md.RpmFile
// create new Repo for final dist
repo, err := git.Init(memory.NewStorage(), pd.FsCreator(md.PushBranch))
if err != nil {
log.Fatalf("could not create new dist Repo: %v", err)
}
w, err := repo.Worktree()
if err != nil {
log.Fatalf("could not get dist Worktree: %v", err)
}
shouldContinue := true
for _, ignoredTag := range tagIgnoreList {
if ignoredTag == "refs/tags/"+newTag {
log.Printf("skipping %s", ignoredTag)
shouldContinue = false
}
}
if !shouldContinue {
continue
}
2020-12-14 03:51:03 +00:00
// create a new remote
2021-03-17 14:54:16 +00:00
remoteUrl := fmt.Sprintf("%s/%s/%s.git", pd.UpstreamPrefix, remotePrefix, gitlabify(rpmFile.Name()))
log.Printf("using remote: %s", remoteUrl)
refspec := config.RefSpec(fmt.Sprintf("+refs/heads/%s:refs/remotes/origin/%s", md.PushBranch, md.PushBranch))
log.Printf("using refspec: %s", refspec)
_, err = repo.CreateRemote(&config.RemoteConfig{
Name: "origin",
URLs: []string{remoteUrl},
Fetch: []config.RefSpec{refspec},
2020-12-14 03:51:03 +00:00
})
if err != nil {
log.Fatalf("could not create remote: %v", err)
2020-12-14 03:51:03 +00:00
}
err = repo.Fetch(&git.FetchOptions{
RemoteName: "origin",
RefSpecs: []config.RefSpec{refspec},
Auth: pd.Authenticator,
})
2020-12-14 03:51:03 +00:00
refName := plumbing.NewBranchReferenceName(md.PushBranch)
log.Printf("set reference to ref: %s", refName)
2021-04-06 19:35:46 +00:00
var hash plumbing.Hash
if commitPin[md.PushBranch] != "" {
hash = plumbing.NewHash(commitPin[md.PushBranch])
}
2020-12-14 03:51:03 +00:00
if err != nil {
h := plumbing.NewSymbolicReference(plumbing.HEAD, refName)
if err := repo.Storer.CheckAndSetReference(h, nil); err != nil {
log.Fatalf("could not set reference: %v", err)
}
} else {
err = w.Checkout(&git.CheckoutOptions{
Branch: plumbing.NewRemoteReferenceName("origin", md.PushBranch),
2021-04-06 19:35:46 +00:00
Hash: hash,
Force: true,
})
if err != nil {
log.Fatalf("could not checkout: %v", err)
}
2020-12-14 03:51:03 +00:00
}
pd.Importer.WriteSource(pd, md)
2020-12-14 03:51:03 +00:00
2021-04-08 02:54:44 +00:00
data.CopyFromFs(md.Worktree.Filesystem, w.Filesystem, ".")
md.Repo = repo
md.Worktree = w
2020-12-14 03:51:03 +00:00
2020-12-22 16:28:04 +00:00
if pd.ModuleMode {
patchModuleYaml(pd, md)
} else {
executePatchesRpm(pd, md)
}
2020-12-14 03:51:03 +00:00
// get ignored files hash and add to .{Name}.metadata
metadataFile := fmt.Sprintf(".%s.metadata", rpmFile.Name())
metadata, err := w.Filesystem.Create(metadataFile)
2020-12-14 03:51:03 +00:00
if err != nil {
log.Fatalf("could not create metadata file: %v", err)
2020-12-14 03:51:03 +00:00
}
for _, source := range md.SourcesToIgnore {
sourcePath := source.Name
_, err := w.Filesystem.Stat(sourcePath)
if source.Expired || err != nil {
continue
}
sourceFile, err := w.Filesystem.Open(sourcePath)
if err != nil {
log.Fatalf("could not open ignored source file %s: %v", sourcePath, err)
}
sourceFileBts, err := ioutil.ReadAll(sourceFile)
if err != nil {
log.Fatalf("could not read the whole of ignored source file: %v", err)
}
2020-12-14 03:51:03 +00:00
source.HashFunction.Reset()
_, err = source.HashFunction.Write(sourceFileBts)
2020-12-14 03:51:03 +00:00
if err != nil {
log.Fatalf("could not write bytes to hash function: %v", err)
2020-12-14 03:51:03 +00:00
}
checksum := hex.EncodeToString(source.HashFunction.Sum(nil))
2020-12-20 17:38:25 +00:00
checksumLine := fmt.Sprintf("%s %s\n", checksum, sourcePath)
_, err = metadata.Write([]byte(checksumLine))
if err != nil {
log.Fatalf("could not write to metadata file: %v", err)
2020-12-14 03:51:03 +00:00
}
2020-12-20 17:38:25 +00:00
2021-04-08 02:54:44 +00:00
if data.StrContains(alreadyUploadedBlobs, checksum) {
continue
}
2021-03-18 17:56:47 +00:00
if !pd.BlobStorage.Exists(checksum) && !pd.NoStorageUpload {
pd.BlobStorage.Write(checksum, sourceFileBts)
log.Printf("wrote %s to blob storage", checksum)
}
2021-03-18 17:56:47 +00:00
alreadyUploadedBlobs = append(alreadyUploadedBlobs, checksum)
2020-12-14 03:51:03 +00:00
}
_, err = w.Add(metadataFile)
2020-12-14 03:51:03 +00:00
if err != nil {
log.Fatalf("could not add metadata file: %v", err)
2020-12-14 03:51:03 +00:00
}
lastFilesToAdd := []string{".gitignore", "SPECS"}
for _, f := range lastFilesToAdd {
2020-12-22 16:28:04 +00:00
_, err := w.Filesystem.Stat(f)
if err == nil {
_, err := w.Add(f)
if err != nil {
log.Fatalf("could not add %s: %v", f, err)
}
}
}
2021-03-18 17:56:47 +00:00
if pd.TmpFsMode != "" {
continue
}
pd.Importer.PostProcess(md)
// show status
status, _ := w.Status()
log.Printf("successfully processed:\n%s", status)
statusLines := strings.Split(status.String(), "\n")
for _, line := range statusLines {
trimmed := strings.TrimSpace(line)
if strings.HasPrefix(trimmed, "D") {
path := strings.TrimPrefix(trimmed, "D ")
_, err := w.Remove(path)
if err != nil {
log.Fatalf("could not delete extra file %s: %v", path, err)
}
}
}
var hashes []plumbing.Hash
var pushRefspecs []config.RefSpec
head, err := repo.Head()
if err != nil {
hashes = nil
pushRefspecs = append(pushRefspecs, "*:*")
} else {
log.Printf("tip %s", head.String())
hashes = append(hashes, head.Hash())
refOrigin := "refs/heads/" + md.PushBranch
pushRefspecs = append(pushRefspecs, config.RefSpec(fmt.Sprintf("HEAD:%s", refOrigin)))
}
// we are now finished with the tree and are going to push it to the src Repo
// create import commit
commit, err := w.Commit("import "+pd.Importer.ImportName(pd, md), &git.CommitOptions{
Author: &object.Signature{
Name: pd.GitCommitterName,
Email: pd.GitCommitterEmail,
When: time.Now(),
},
Parents: hashes,
})
2020-12-14 03:51:03 +00:00
if err != nil {
log.Fatalf("could not commit object: %v", err)
2020-12-14 03:51:03 +00:00
}
obj, err := repo.CommitObject(commit)
2020-12-14 03:51:03 +00:00
if err != nil {
log.Fatalf("could not get commit object: %v", err)
2020-12-14 03:51:03 +00:00
}
log.Printf("committed:\n%s", obj.String())
2020-12-14 03:51:03 +00:00
_, err = repo.CreateTag(newTag, commit, &git.CreateTagOptions{
Tagger: &object.Signature{
Name: pd.GitCommitterName,
Email: pd.GitCommitterEmail,
When: time.Now(),
},
Message: "import " + md.TagBranch + " from " + pd.RpmLocation,
SignKey: nil,
})
if err != nil {
log.Fatalf("could not create tag: %v", err)
}
2020-12-14 03:51:03 +00:00
pushRefspecs = append(pushRefspecs, config.RefSpec(fmt.Sprintf("HEAD:%s", plumbing.NewTagReferenceName(newTag))))
2020-12-14 03:51:03 +00:00
err = repo.Push(&git.PushOptions{
RemoteName: "origin",
Auth: pd.Authenticator,
RefSpecs: pushRefspecs,
Force: true,
})
if err != nil {
log.Fatalf("could not push to remote: %v", err)
}
hashString := obj.Hash.String()
latestHashForBranch[md.PushBranch] = hashString
}
err := json.NewEncoder(os.Stdout).Encode(latestHashForBranch)
if err != nil {
log.Fatalf("could not print hashes")
2020-12-14 03:51:03 +00:00
}
2020-12-14 01:29:02 +00:00
}