sig_core/srpmproc
8
0
Fork 0
mirror of https://github.com/rocky-linux/srpmproc.git synced 2024-06-02 08:10:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

update

Browse Source
This commit is contained in:
Mustafa Gezen 2020-12-14 04:51:03 +01:00
parent 18ea4b95fc
commit 18bd2993cc
1 changed files with 350 additions and 349 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

699
internal/process.go
View File

@ -1,84 +1,84 @@
package internal package internal
import ( import (
"bytes" "bytes"
"crypto/sha1" "crypto/sha1"
"encoding/hex" "encoding/hex"
"fmt" "fmt"
"github.com/bluekeyes/go-gitdiff/gitdiff" "github.com/bluekeyes/go-gitdiff/gitdiff"
"github.com/cavaliercoder/go-cpio" "github.com/cavaliercoder/go-cpio"
"github.com/cavaliercoder/go-rpm" "github.com/cavaliercoder/go-rpm"
"github.com/go-git/go-billy/v5/memfs" "github.com/go-git/go-billy/v5/memfs"
"github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/config" "github.com/go-git/go-git/v5/config"
"github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/plumbing/object" "github.com/go-git/go-git/v5/plumbing/object"
"github.com/go-git/go-git/v5/plumbing/transport/ssh" "github.com/go-git/go-git/v5/plumbing/transport/ssh"
"github.com/go-git/go-git/v5/storage/memory" "github.com/go-git/go-git/v5/storage/memory"
"io" "io"
"io/ioutil" "io/ioutil"
"log" "log"
"os" "os"
"os/exec" "os/exec"
"os/user" "os/user"
"path/filepath" "path/filepath"
"strconv" "strconv"
"strings" "strings"
"time" "time"
) )
type ProcessData struct { type ProcessData struct {
RpmLocation string RpmLocation string
UpstreamPrefix string UpstreamPrefix string
SshKeyLocation string SshKeyLocation string
SshUser string SshUser string
Branch string Branch string
} }
type specFileData struct { type specFileData struct {
SourcesToIgnore []string SourcesToIgnore []string
} }
func strContains(a []string, b string) bool { func strContains(a []string, b string) bool {
for _, val := range a { for _, val := range a {
if val == b { if val == b {
return true return true
} }
} }
return false return false
} }
func parseSpecFile(f []byte, orderedSourceFiles []string) *specFileData { func parseSpecFile(f []byte, orderedSourceFiles []string) *specFileData {
spec := specFileData{ spec := specFileData{
SourcesToIgnore: []string{}, SourcesToIgnore: []string{},
} }
for _, line := range strings.Split(string(f), "\n") { for _, line := range strings.Split(string(f), "\n") {
// no need to run through all else ifs // no need to run through all else ifs
if strings.HasPrefix(line, "%changelog") { if strings.HasPrefix(line, "%changelog") {
break break
} }
orderedSrcLen := len(orderedSourceFiles) orderedSrcLen := len(orderedSourceFiles)
if strings.HasPrefix(line, "Source") { if strings.HasPrefix(line, "Source") {
source := strings.SplitN(strings.TrimSpace(line), ":", 2) source := strings.SplitN(strings.TrimSpace(line), ":", 2)
sourceUri := source[1] sourceUri := source[1]
sourceI := strings.Replace(strings.TrimSpace(source[0]), "Source", "", 1) sourceI := strings.Replace(strings.TrimSpace(source[0]), "Source", "", 1)
i, err := strconv.Atoi(sourceI) i, err := strconv.Atoi(sourceI)
if err != nil { if err != nil {
log.Fatalf("could not convert source number: %v", err) log.Fatalf("could not convert source number: %v", err)
} }
// if sourceUri contains :// then it's remote // if sourceUri contains :// then it's remote
// so ignore this file // so ignore this file
if strings.Contains(sourceUri, "://") && strings.Contains(sourceUri, ".tar") { if strings.Contains(sourceUri, "://") && strings.Contains(sourceUri, ".tar") {
spec.SourcesToIgnore = append(spec.SourcesToIgnore, orderedSourceFiles[orderedSrcLen-1-i]) spec.SourcesToIgnore = append(spec.SourcesToIgnore, orderedSourceFiles[orderedSrcLen-1-i])
} }
} }
} }
return &spec return &spec
} }
// ProcessRPM checks the RPM specs and discards any remote files // ProcessRPM checks the RPM specs and discards any remote files
@ -89,333 +89,334 @@ func parseSpecFile(f []byte, orderedSourceFiles []string) *specFileData {
// all files that are remote goes into .gitignore // all files that are remote goes into .gitignore
// all ignored files' hash goes into .{name}.metadata // all ignored files' hash goes into .{name}.metadata
func ProcessRPM(pd *ProcessData) { func ProcessRPM(pd *ProcessData) {
cmd := exec.Command("rpm2cpio", pd.RpmLocation) cmd := exec.Command("rpm2cpio", pd.RpmLocation)
cpioBytes, err := cmd.Output() cpioBytes, err := cmd.Output()
if err != nil { if err != nil {
log.Fatalf("could not convert to cpio (maybe rpm2cpio is missing): %v", err) log.Fatalf("could not convert to cpio (maybe rpm2cpio is missing): %v", err)
} }
// create in memory git repository // create in memory git repository
repo, err := git.Init(memory.NewStorage(), memfs.New()) repo, err := git.Init(memory.NewStorage(), memfs.New())
if err != nil { if err != nil {
log.Fatalf("could not init git repo: %v", err) log.Fatalf("could not init git repo: %v", err)
} }
// this will be set later // this will be set later
specFileName := "" var specBts []byte
var specBts []byte
// read the rpm in cpio format // read the rpm in cpio format
buf := bytes.NewReader(cpioBytes) buf := bytes.NewReader(cpioBytes)
r := cpio.NewReader(buf) r := cpio.NewReader(buf)
fileWrites := map[string][]byte{} fileWrites := map[string][]byte{}
for { for {
hdr, err := r.Next() hdr, err := r.Next()
if err == io.EOF { if err == io.EOF {
// end of cpio archive // end of cpio archive
break break
} }
if err != nil { if err != nil {
log.Fatalln(err) log.Fatalln(err)
} }
// set spec file name // set spec file name
if filepath.Ext(hdr.Name) == ".spec" { if filepath.Ext(hdr.Name) == ".spec" {
specFileName = hdr.Name var err error
var err error specBts, err = ioutil.ReadAll(r)
specBts, err = ioutil.ReadAll(r) if err != nil {
if err != nil { log.Fatalf("could not read spec: %v", err)
log.Fatalf("could not read spec: %v", err) }
} }
continue
}
bts, err := ioutil.ReadAll(r) bts, err := ioutil.ReadAll(r)
if err != nil { if err != nil {
log.Fatalf("could not copy file to virtual filesystem: %v", err) log.Fatalf("could not copy file to virtual filesystem: %v", err)
} }
fileWrites[hdr.Name] = bts fileWrites[hdr.Name] = bts
} }
w, err := repo.Worktree() w, err := repo.Worktree()
if err != nil { if err != nil {
log.Fatalf("could not get worktree: %v", err) log.Fatalf("could not get worktree: %v", err)
} }
// create structure // create structure
err = w.Filesystem.MkdirAll("SPECS", 0755) err = w.Filesystem.MkdirAll("SPECS", 0755)
if err != nil { if err != nil {
log.Fatalf("could not create SPECS dir in vfs: %v", err) log.Fatalf("could not create SPECS dir in vfs: %v", err)
} }
err = w.Filesystem.MkdirAll("SOURCES", 0755) err = w.Filesystem.MkdirAll("SOURCES", 0755)
if err != nil { if err != nil {
log.Fatalf("could not create SOURCES dir in vfs: %v", err) log.Fatalf("could not create SOURCES dir in vfs: %v", err)
} }
f, err := os.Open(pd.RpmLocation) f, err := os.Open(pd.RpmLocation)
if err != nil { if err != nil {
log.Fatalf("could not open the file again: %v", err) log.Fatalf("could not open the file again: %v", err)
} }
rpmFile, err := rpm.ReadPackageFile(f) rpmFile, err := rpm.ReadPackageFile(f)
if err != nil { if err != nil {
log.Fatalf("could not read package, invalid?: %v", err) log.Fatalf("could not read package, invalid?: %v", err)
} }
// read spec and remove remote files // read spec and remove remote files
spec := parseSpecFile(specBts, rpmFile.Source()) spec := parseSpecFile(specBts, rpmFile.Source())
specFile, err := w.Filesystem.Create(filepath.Join("SPECS", specFileName))
if err != nil {
log.Fatalf("could not create spec file: %v", err)
}
_, err = specFile.Write(specBts)
if err != nil {
log.Fatalf("could not write spec file: %v", err)
}
// create a new remote // create a new remote
remoteUrl := fmt.Sprintf("%s/dist/%s.git", pd.UpstreamPrefix, rpmFile.Name()) remoteUrl := fmt.Sprintf("%s/dist/%s.git", pd.UpstreamPrefix, rpmFile.Name())
log.Printf("using remote: %s", remoteUrl) log.Printf("using remote: %s", remoteUrl)
refspec := config.RefSpec(fmt.Sprintf("+refs/heads/%s:refs/remotes/origin/%s", pd.Branch, pd.Branch)) refspec := config.RefSpec(fmt.Sprintf("+refs/heads/%s:refs/remotes/origin/%s", pd.Branch, pd.Branch))
log.Printf("using refspec: %s", refspec) log.Printf("using refspec: %s", refspec)
_, err = repo.CreateRemote(&config.RemoteConfig{ _, err = repo.CreateRemote(&config.RemoteConfig{
Name: "origin", Name: "origin",
URLs: []string{remoteUrl}, URLs: []string{remoteUrl},
Fetch: []config.RefSpec{refspec}, Fetch: []config.RefSpec{refspec},
}) })
if err != nil { if err != nil {
log.Fatalf("could not create remote: %v", err) log.Fatalf("could not create remote: %v", err)
} }
lastKeyLocation := pd.SshKeyLocation lastKeyLocation := pd.SshKeyLocation
if lastKeyLocation == "" { if lastKeyLocation == "" {
usr, err := user.Current() usr, err := user.Current()
if err != nil { if err != nil {
log.Fatalf("could not get user: %v", err) log.Fatalf("could not get user: %v", err)
} }
lastKeyLocation = filepath.Join(usr.HomeDir, ".ssh/id_rsa") lastKeyLocation = filepath.Join(usr.HomeDir, ".ssh/id_rsa")
} }
// create ssh key authenticator // create ssh key authenticator
authenticator, err := ssh.NewPublicKeysFromFile(pd.SshUser, lastKeyLocation, "") authenticator, err := ssh.NewPublicKeysFromFile(pd.SshUser, lastKeyLocation, "")
if err != nil { if err != nil {
log.Fatalf("could not get git authenticator: %v", err) log.Fatalf("could not get git authenticator: %v", err)
} }
err = repo.Fetch(&git.FetchOptions{ err = repo.Fetch(&git.FetchOptions{
RemoteName: "origin", RemoteName: "origin",
RefSpecs: []config.RefSpec{refspec}, RefSpecs: []config.RefSpec{refspec},
Auth: authenticator, Auth: authenticator,
}) })
refName := plumbing.NewBranchReferenceName(pd.Branch) refName := plumbing.NewBranchReferenceName(pd.Branch)
log.Printf("set reference to ref: %s", refName) log.Printf("set reference to ref: %s", refName)
if err != nil { if err != nil {
h := plumbing.NewSymbolicReference(plumbing.HEAD, refName) h := plumbing.NewSymbolicReference(plumbing.HEAD, refName)
if err := repo.Storer.CheckAndSetReference(h, nil); err != nil { if err := repo.Storer.CheckAndSetReference(h, nil); err != nil {
log.Fatalf("could not set reference: %v", err) log.Fatalf("could not set reference: %v", err)
} }
} else { } else {
err = w.Checkout(&git.CheckoutOptions{ err = w.Checkout(&git.CheckoutOptions{
Branch: plumbing.NewRemoteReferenceName("origin", pd.Branch), Branch: plumbing.NewRemoteReferenceName("origin", pd.Branch),
Force: true, Force: true,
}) })
if err != nil { if err != nil {
log.Fatalf("could not checkout: %v", err) log.Fatalf("could not checkout: %v", err)
} }
} }
// TODO: Remove dangling files // TODO: Remove dangling files
for fileName, contents := range fileWrites { for fileName, contents := range fileWrites {
newPath := filepath.Join("SOURCES", fileName) var newPath string
if filepath.Ext(fileName) == ".spec" {
newPath = filepath.Join("SPECS", fileName)
} else {
newPath = filepath.Join("SOURCES", fileName)
}
// add the file to the virtual filesystem // add the file to the virtual filesystem
// we will move it to correct destination later // we will move it to correct destination later
f, err := w.Filesystem.Create(newPath) f, err := w.Filesystem.Create(newPath)
if err != nil { if err != nil {
log.Fatalf("could not create file %s: %v", fileName, err) log.Fatalf("could not create file %s: %v", fileName, err)
} }
_, err = f.Write(contents) _, err = f.Write(contents)
if err != nil { if err != nil {
log.Fatalf("could not write to file %s: %v", fileName, err) log.Fatalf("could not write to file %s: %v", fileName, err)
} }
_ = f.Close() _ = f.Close()
// don't add ignored file to git // don't add ignored file to git
if strContains(spec.SourcesToIgnore, fileName) { if strContains(spec.SourcesToIgnore, fileName) {
continue continue
} }
log.Printf("git add %s", newPath) log.Printf("git add %s", newPath)
_, err = w.Add(newPath) _, err = w.Add(newPath)
if err != nil { if err != nil {
log.Fatalf("could not add source file: %v", err) log.Fatalf("could not add source file: %v", err)
} }
} }
// add sources to ignore (remote sources) // add sources to ignore (remote sources)
gitIgnore, err := w.Filesystem.Create(".gitignore") gitIgnore, err := w.Filesystem.Create(".gitignore")
if err != nil { if err != nil {
log.Fatalf("could not create .gitignore: %v", err) log.Fatalf("could not create .gitignore: %v", err)
} }
for _, ignore := range spec.SourcesToIgnore { for _, ignore := range spec.SourcesToIgnore {
line := fmt.Sprintf("SOURCES/%s\n", ignore) line := fmt.Sprintf("SOURCES/%s\n", ignore)
_, err := gitIgnore.Write([]byte(line)) _, err := gitIgnore.Write([]byte(line))
if err != nil { if err != nil {
log.Fatalf("could not write line to .gitignore: %v", err) log.Fatalf("could not write line to .gitignore: %v", err)
} }
} }
err = gitIgnore.Close() err = gitIgnore.Close()
if err != nil { if err != nil {
log.Fatalf("could not close .gitignore: %v", err) log.Fatalf("could not close .gitignore: %v", err)
} }
// check if patches exist // check if patches exist
_, err = w.Filesystem.Stat("ROCKY") _, err = w.Filesystem.Stat("ROCKY")
if err == nil { if err == nil {
// check SRPM patches // check SRPM patches
_, err = w.Filesystem.Stat("ROCKY/SRPM") _, err = w.Filesystem.Stat("ROCKY/SRPM")
if err == nil { if err == nil {
// iterate through patches // iterate through patches
infos, err := w.Filesystem.ReadDir("ROCKY/SRPM") infos, err := w.Filesystem.ReadDir("ROCKY/SRPM")
if err != nil { if err != nil {
log.Fatalf("could not walk patches: %v", err) log.Fatalf("could not walk patches: %v", err)
} }
for _, info := range infos { for _, info := range infos {
// can only process .patch files // can only process .patch files
if !strings.HasSuffix(info.Name(), ".patch") { if !strings.HasSuffix(info.Name(), ".patch") {
continue continue
} }
log.Printf("applying %s", info.Name()) log.Printf("applying %s", info.Name())
filePath := filepath.Join("ROCKY/SRPM", info.Name()) filePath := filepath.Join("ROCKY/SRPM", info.Name())
patch, err := w.Filesystem.Open(filePath) patch, err := w.Filesystem.Open(filePath)
if err != nil { if err != nil {
log.Fatalf("could not open patch file %s: %v", info.Name(), err) log.Fatalf("could not open patch file %s: %v", info.Name(), err)
} }
files, _, err := gitdiff.Parse(patch) files, _, err := gitdiff.Parse(patch)
if err != nil { if err != nil {
log.Fatalf("could not parse patch file: %v", err) log.Fatalf("could not parse patch file: %v", err)
} }
for _, patchedFile := range files { for _, patchedFile := range files {
srcPath := filepath.Join("SOURCES", patchedFile.NewName) srcPath := filepath.Join("SOURCES", patchedFile.NewName)
patchSubjectFile, err := w.Filesystem.Open(srcPath) patchSubjectFile, err := w.Filesystem.Open(srcPath)
if err != nil { if err != nil {
log.Fatalf("could not open patch subject: %v", err) log.Fatalf("could not open patch subject: %v", err)
} }
var output bytes.Buffer var output bytes.Buffer
err = gitdiff.NewApplier(patchSubjectFile).ApplyFile(&output, patchedFile) err = gitdiff.NewApplier(patchSubjectFile).ApplyFile(&output, patchedFile)
if err != nil { if err != nil {
log.Fatalf("could not apply patch: %v", err) log.Fatalf("could not apply patch: %v", err)
} }
err = w.Filesystem.Remove(srcPath) err = w.Filesystem.Remove(srcPath)
if err != nil { if err != nil {
log.Fatalf("could not remove pre-patch file: %v", err) log.Fatalf("could not remove pre-patch file: %v", err)
} }
newFile, err := w.Filesystem.Create(srcPath) newFile, err := w.Filesystem.Create(srcPath)
if err != nil { if err != nil {
log.Fatalf("could not create post-patch file: %v", err) log.Fatalf("could not create post-patch file: %v", err)
} }
_, err = newFile.Write(output.Bytes()) _, err = newFile.Write(output.Bytes())
if err != nil { if err != nil {
log.Fatalf("could not write post-patch file: %v", err) log.Fatalf("could not write post-patch file: %v", err)
} }
_, err = w.Add(srcPath) _, err = w.Add(srcPath)
if err != nil { if err != nil {
log.Fatalf("could not add file %s to git: %v", srcPath, err) log.Fatalf("could not add file %s to git: %v", srcPath, err)
} }
log.Printf("git add %s", srcPath) log.Printf("git add %s", srcPath)
} }
_, err = w.Add(filePath) _, err = w.Add(filePath)
if err != nil { if err != nil {
log.Fatalf("could not add file %s to git: %v", filePath, err) log.Fatalf("could not add file %s to git: %v", filePath, err)
} }
log.Printf("git add %s", filePath) log.Printf("git add %s", filePath)
} }
} }
} }
// get ignored files sha1 hash and add to .{name}.metadata // get ignored files sha1 hash and add to .{name}.metadata
metadataFile := fmt.Sprintf(".%s.metadata", rpmFile.Name()) metadataFile := fmt.Sprintf(".%s.metadata", rpmFile.Name())
metadata, err := w.Filesystem.Create(metadataFile) metadata, err := w.Filesystem.Create(metadataFile)
if err != nil { if err != nil {
log.Fatalf("could not create metadata file: %v", err) log.Fatalf("could not create metadata file: %v", err)
} }
for _, source := range spec.SourcesToIgnore { for _, source := range spec.SourcesToIgnore {
sourcePath := "SOURCES/" + source sourcePath := "SOURCES/" + source
sourceFile, err := w.Filesystem.Open(sourcePath) sourceFile, err := w.Filesystem.Open(sourcePath)
if err != nil { if err != nil {
log.Fatalf("could not open ignored source file %s: %v", sourcePath, err) log.Fatalf("could not open ignored source file %s: %v", sourcePath, err)
} }
sourceFileBts, err := ioutil.ReadAll(sourceFile) sourceFileBts, err := ioutil.ReadAll(sourceFile)
if err != nil { if err != nil {
log.Fatalf("could not read the whole of ignored source file: %v", err) log.Fatalf("could not read the whole of ignored source file: %v", err)
} }
checksum := sha1.Sum(sourceFileBts) checksum := sha1.Sum(sourceFileBts)
checksumLine := fmt.Sprintf("%s %s\n", hex.EncodeToString(checksum[:]), sourcePath) checksumLine := fmt.Sprintf("%s %s\n", hex.EncodeToString(checksum[:]), sourcePath)
_, err = metadata.Write([]byte(checksumLine)) _, err = metadata.Write([]byte(checksumLine))
if err != nil { if err != nil {
log.Fatalf("could not write to metadata file: %v", err) log.Fatalf("could not write to metadata file: %v", err)
} }
} }
lastFilesToAdd := []string{metadataFile, ".gitignore", "SPECS"} lastFilesToAdd := []string{metadataFile, ".gitignore", "SPECS"}
for _, f := range lastFilesToAdd { for _, f := range lastFilesToAdd {
_, err := w.Add(f) _, err := w.Add(f)
if err != nil { if err != nil {
log.Fatalf("could not add metadata file: %v", err) log.Fatalf("could not add metadata file: %v", err)
} }
} }
// show status // show status
status, _ := w.Status() status, _ := w.Status()
log.Printf("successfully processed:\n%s", status) log.Printf("successfully processed:\n%s", status)
head, err := repo.Head() var hashes []plumbing.Hash
if err != nil { var pushRefspecs []config.RefSpec
log.Fatalf("could not get ref: %v", err)
}
log.Printf("tip %s", head.String())
// we are now finished with the tree and are going to push it to the src repo head, err := repo.Head()
// create import commit if err != nil {
commit, err := w.Commit("import "+filepath.Base(pd.RpmLocation), &git.CommitOptions{ hashes = nil
Author: &object.Signature{ pushRefspecs = nil
Name: "Mustafa Gezen", } else {
Email: "mustafa@bycrates.com", log.Printf("tip %s", head.String())
When: time.Now(), hashes = append(hashes, head.Hash())
}, refOrigin := "refs/heads/" + pd.Branch
Parents: []plumbing.Hash{head.Hash()}, pushRefspecs = append(pushRefspecs, config.RefSpec(fmt.Sprintf("HEAD:%s", refOrigin)))
}) }
if err != nil {
log.Fatalf("could not commit object: %v", err)
}
obj, err := repo.CommitObject(commit) // we are now finished with the tree and are going to push it to the src repo
if err != nil { // create import commit
log.Fatalf("could not get commit object: %v", err) commit, err := w.Commit("import "+filepath.Base(pd.RpmLocation), &git.CommitOptions{
} Author: &object.Signature{
Name: "Mustafa Gezen",
Email: "mustafa@bycrates.com",
When: time.Now(),
},
Parents: hashes,
})
if err != nil {
log.Fatalf("could not commit object: %v", err)
}
log.Printf("committed:\n%s", obj.String()) obj, err := repo.CommitObject(commit)
if err != nil {
log.Fatalf("could not get commit object: %v", err)
}
refOrigin := "refs/heads/" + pd.Branch log.Printf("committed:\n%s", obj.String())
err = repo.Push(&git.PushOptions{
RemoteName: "origin", err = repo.Push(&git.PushOptions{
Auth: authenticator, RemoteName: "origin",
RefSpecs: []config.RefSpec{config.RefSpec(fmt.Sprintf("HEAD:%s", refOrigin))}, Auth: authenticator,
}) RefSpecs: pushRefspecs,
if err != nil { })
log.Fatalf("could not push to remote: %v", err) if err != nil {
} log.Fatalf("could not push to remote: %v", err)
}
} }