peridot/peridot
7
0
Fork 0
mirror of https://github.com/peridotbuild/peridot.git synced 2024-12-07 20:06:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

More srpm_import tests

Browse Source
This commit is contained in:
Mustafa Gezen 2023-08-28 02:10:39 +02:00
parent 12ec42f8d0
commit 9cbeae7b17
Signed by: mustafa
GPG Key ID: DCDF010D946438C1
2 changed files with 567 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
tools/mothership/worker_server/srpm_import/srpm_import.go
View File

@ -39,10 +39,18 @@ import (
var elDistRegex = regexp.MustCompile(`el\d+`)
type State struct {
// tempDir is the temporary directory where the SRPM is extracted to.
tempDir string
// rpm is the SRPM.
rpm *rpmutils.Rpm
// authorName is the name of the author of the commit.
authorName string
// authorEmail is the email of the author of the commit.
authorEmail string
// lookasideBlobs is a map of blob names to their SHA256 hashes.
lookasideBlobs map[string]string
}
@ -102,6 +110,8 @@ func FromFile(path string, keys ...*openpgp.Entity) (*State, error) {
state := &State{
rpm: rpm,
authorName: "Mship Bot",
authorEmail: "no-reply+mshipbot@resf.org",
lookasideBlobs: make(map[string]string),
}
// Create a temporary directory.
@ -127,6 +137,11 @@ func (s *State) GetDir() string {
return s.tempDir
}
func (s *State) SetAuthor(name, email string) {
s.authorName = name
s.authorEmail = email
}
// determineLookasideBlobs determines which blobs need to be uploaded to the
// lookaside cache.
// Currently, the rule is that if a file is larger than 5MB, and is binary,
@ -433,10 +448,11 @@ func (s *State) populateTargetRepo(repo *git.Repository, targetFS billy.Filesyst
if err != nil {
return errors.Wrap(err, "failed to get NEVRA")
}
_, err = wt.Commit("import "+nevra.String(), &git.CommitOptions{
importStr := fmt.Sprintf("import %s-%s-%s", nevra.Name, nevra.Version, nevra.Release)
hash, err := wt.Commit(importStr, &git.CommitOptions{
Author: &object.Signature{
Name: "Mship Bot",
Email: "no-reply+mshipbot@resf.org",
Name: s.authorName,
Email: s.authorEmail,
When: time.Now(),
},
})
@ -452,12 +468,13 @@ func (s *State) populateTargetRepo(repo *git.Repository, targetFS billy.Filesyst
return errors.Wrap(err, "failed to determine dist tag")
}
tag := fmt.Sprintf("imports/%s/%s-%s-%s", dist, nevra.Name, nevra.Version, nevra.Release)
_, err = repo.CreateTag(tag, plumbing.NewHash("HEAD"), &git.CreateTagOptions{
_, err = repo.CreateTag(tag, hash, &git.CreateTagOptions{
Tagger: &object.Signature{
Name: "Mship Bot",
Email: "no-reply+mshipbot@resf.org",
Name: s.authorName,
Email: s.authorEmail,
When: time.Now(),
},
Message: tag,
})
if err != nil {
return errors.Wrap(err, "failed to create tag")
@ -492,7 +509,17 @@ func (s *State) Import(opts *git.CloneOptions, storer storage2.Storer, targetFS
}
// Push the target repository.
err = s.pushTargetRepo(repo, &git.PushOptions{})
nevra, err := s.rpm.Header.GetNEVRA()
if err != nil {
return errors.Wrap(err, "failed to get NEVRA")
}
dist := elDistRegex.FindString(nevra.Release)
err = s.pushTargetRepo(repo, &git.PushOptions{
RefSpecs: []config.RefSpec{
config.RefSpec(fmt.Sprintf("refs/heads/%s:refs/heads/%[1]s", dist)),
config.RefSpec(fmt.Sprintf("refs/tags/imports/%s/*:refs/tags/imports/%[1]s/*", dist)),
},
})
if err != nil {
return errors.Wrap(err, "failed to push target repo")
}

533
tools/mothership/worker_server/srpm_import/srpm_import_test.go
View File

@ -28,6 +28,7 @@ import (
"golang.org/x/crypto/openpgp"
"io"
"os"
"sort"
"testing"
"time"
)
@ -341,3 +342,535 @@ func TestGetRepo_Existing(t *testing.T) {
require.Nil(t, err)
require.Equal(t, "test commit", obj.Message)
}
func TestCleanTargetRepo_Existing(t *testing.T) {
s, err := FromFile("testdata/efi-rpm-macros-3-3.el8.src.rpm")
require.Nil(t, err)
require.NotNil(t, s)
defer func() {
require.Nil(t, s.Close())
}()
tempDir, err := os.MkdirTemp("", "peridot-srpm-import-test-*")
require.Nil(t, err)
defer os.RemoveAll(tempDir)
// Create a bare repo in tempDir
osfsTemp := osfs.New(tempDir)
dot, err := osfsTemp.Chroot(".git")
require.Nil(t, err)
filesystemTemp := filesystem.NewStorage(dot, cache.NewObjectLRUDefault())
require.Nil(t, filesystemTemp.Init())
_, err = git.Init(filesystemTemp, nil)
require.Nil(t, err)
// Push a commit to the bare repo
newTempDir, err := os.MkdirTemp("", "peridot-srpm-import-test-*")
require.Nil(t, err)
defer os.RemoveAll(newTempDir)
osfs2 := osfs.New(newTempDir)
dot2, err := osfs2.Chroot(".git")
require.Nil(t, err)
filesystemTemp2 := filesystem.NewStorage(dot2, cache.NewObjectLRUDefault())
repo, err := git.InitWithOptions(filesystemTemp2, osfs2, git.InitOptions{
DefaultBranch: "refs/heads/el8",
})
require.Nil(t, err)
_, err = repo.CreateRemote(&config.RemoteConfig{
Name: "origin",
URLs: []string{tempDir},
})
require.Nil(t, err)
w, err := repo.Worktree()
require.Nil(t, err)
f, err := w.Filesystem.Create("testfile")
require.Nil(t, err)
_, err = f.Write([]byte("test"))
require.Nil(t, err)
err = f.Close()
require.Nil(t, err)
_, err = w.Add("testfile")
require.Nil(t, err)
_, err = w.Commit("test commit", &git.CommitOptions{
Author: &object.Signature{
Name: "test",
Email: "test@resf.org",
When: time.Now(),
},
})
require.Nil(t, err)
err = repo.Push(&git.PushOptions{
RefSpecs: []config.RefSpec{"refs/heads/el8:refs/heads/el8"},
})
require.Nil(t, err)
storer := memory.NewStorage()
fs := memfs.New()
opts := &git.CloneOptions{
URL: "file://" + tempDir,
}
repo, err = s.getRepo(opts, storer, fs)
require.Nil(t, err)
require.NotNil(t, repo)
wt, err := repo.Worktree()
require.Nil(t, err)
// Verify commit
objIter, err := repo.CommitObjects()
require.Nil(t, err)
obj, err := objIter.Next()
require.Nil(t, err)
require.Equal(t, "test commit", obj.Message)
// Clean repo
require.Nil(t, s.cleanTargetRepo(wt, "."))
// Verify empty
ls, err := wt.Filesystem.ReadDir(".")
require.Nil(t, err)
require.Equal(t, 0, len(ls))
}
func TestPopulateTargetRepo_New(t *testing.T) {
s, err := FromFile("testdata/efi-rpm-macros-3-3.el8.src.rpm")
require.Nil(t, err)
require.NotNil(t, s)
defer func() {
require.Nil(t, s.Close())
}()
tempDir, err := os.MkdirTemp("", "peridot-srpm-import-test-*")
require.Nil(t, err)
defer os.RemoveAll(tempDir)
storer := memory.NewStorage()
fs := memfs.New()
opts := &git.CloneOptions{
URL: "file://" + tempDir,
}
repo, err := s.getRepo(opts, storer, fs)
require.Nil(t, err)
require.NotNil(t, repo)
// Verify empty
objIter, err := repo.CommitObjects()
require.Nil(t, err)
_, err = objIter.Next()
require.Equal(t, io.EOF, err)
// Populate repo
inMemory := storage_memory.New(osfs.New("/"))
require.Nil(t, s.populateTargetRepo(repo, fs, inMemory))
// Verify commit
objIter, err = repo.CommitObjects()
require.Nil(t, err)
obj, err := objIter.Next()
require.Nil(t, err)
require.Equal(t, "import efi-rpm-macros-3-3.el8", obj.Message)
// Verify tag
tagIter, err := repo.Tags()
require.Nil(t, err)
tag, err := tagIter.Next()
require.Nil(t, err)
require.Equal(t, "imports/el8/efi-rpm-macros-3-3.el8", tag.Name().Short())
// Verify metadata
f, err := fs.Open(".efi-rpm-macros.metadata")
require.Nil(t, err)
buf, err := io.ReadAll(f)
require.Nil(t, err)
require.Equal(t, "SOURCES/efi-rpm-macros-3.tar.bz2 f002f60baed7a47ca3e98b8dd7ece2f7352dac9ffab7ae3557eb56b481ce2f86\n", string(buf))
// Verify layout
ls, err := fs.ReadDir(".")
require.Nil(t, err)
require.Equal(t, 4, len(ls))
require.Equal(t, ".efi-rpm-macros.metadata", ls[0].Name())
require.Equal(t, ".gitignore", ls[1].Name())
require.Equal(t, "SOURCES", ls[2].Name())
require.Equal(t, "SPECS", ls[3].Name())
ls, err = fs.ReadDir("SOURCES")
require.Nil(t, err)
require.Equal(t, 1, len(ls))
require.Equal(t, "0001-macros.efi-srpm-make-all-of-our-macros-always-expand.patch", ls[0].Name())
ls, err = fs.ReadDir("SPECS")
require.Nil(t, err)
require.Equal(t, 1, len(ls))
require.Equal(t, "efi-rpm-macros.spec", ls[0].Name())
}
func TestPopulateTargetRepo_Existing(t *testing.T) {
s, err := FromFile("testdata/efi-rpm-macros-3-3.el8.src.rpm")
require.Nil(t, err)
require.NotNil(t, s)
defer func() {
require.Nil(t, s.Close())
}()
tempDir, err := os.MkdirTemp("", "peridot-srpm-import-test-*")
require.Nil(t, err)
defer os.RemoveAll(tempDir)
// Create a bare repo in tempDir
osfsTemp := osfs.New(tempDir)
dot, err := osfsTemp.Chroot(".git")
require.Nil(t, err)
filesystemTemp := filesystem.NewStorage(dot, cache.NewObjectLRUDefault())
require.Nil(t, filesystemTemp.Init())
_, err = git.Init(filesystemTemp, nil)
require.Nil(t, err)
// Push a commit to the bare repo
newTempDir, err := os.MkdirTemp("", "peridot-srpm-import-test-*")
require.Nil(t, err)
defer os.RemoveAll(newTempDir)
osfs2 := osfs.New(newTempDir)
dot2, err := osfs2.Chroot(".git")
require.Nil(t, err)
filesystemTemp2 := filesystem.NewStorage(dot2, cache.NewObjectLRUDefault())
repo, err := git.InitWithOptions(filesystemTemp2, osfs2, git.InitOptions{
DefaultBranch: "refs/heads/el8",
})
require.Nil(t, err)
_, err = repo.CreateRemote(&config.RemoteConfig{
Name: "origin",
URLs: []string{tempDir},
})
require.Nil(t, err)
w, err := repo.Worktree()
require.Nil(t, err)
f, err := w.Filesystem.Create("testfile")
require.Nil(t, err)
_, err = f.Write([]byte("test"))
require.Nil(t, err)
err = f.Close()
require.Nil(t, err)
_, err = w.Add("testfile")
require.Nil(t, err)
_, err = w.Commit("test commit", &git.CommitOptions{
Author: &object.Signature{
Name: "test",
Email: "test@resf.org",
// We're subtracting an hour here because the commit time is
// truncated to the nearest second, and we want to make sure
// that the commit time is different from the one we're going
// to create below.
When: time.Now().Add(-1 * time.Hour),
},
})
require.Nil(t, err)
err = repo.Push(&git.PushOptions{
RefSpecs: []config.RefSpec{"refs/heads/el8:refs/heads/el8"},
})
require.Nil(t, err)
storer := memory.NewStorage()
fs := memfs.New()
opts := &git.CloneOptions{
URL: tempDir,
}
repo, err = s.getRepo(opts, storer, fs)
require.Nil(t, err)
require.NotNil(t, repo)
// Verify commit
objIter, err := repo.CommitObjects()
require.Nil(t, err)
obj, err := objIter.Next()
require.Nil(t, err)
require.Equal(t, "test commit", obj.Message)
// Populate repo
inMemory := storage_memory.New(osfs.New("/"))
require.Nil(t, s.populateTargetRepo(repo, fs, inMemory))
// Verify commit (second one)
var sortedCommits []*object.Commit
objIter, err = repo.CommitObjects()
require.Nil(t, err)
obj, err = objIter.Next()
require.Nil(t, err)
sortedCommits = append(sortedCommits, obj)
obj, err = objIter.Next()
require.Nil(t, err)
sortedCommits = append(sortedCommits, obj)
sort.Slice(sortedCommits, func(i, j int) bool {
return sortedCommits[i].Author.When.After(sortedCommits[j].Author.When)
})
require.Equal(t, "import efi-rpm-macros-3-3.el8", sortedCommits[0].Message)
// Verify tag
tagIter, err := repo.Tags()
require.Nil(t, err)
tag, err := tagIter.Next()
require.Nil(t, err)
require.Equal(t, "imports/el8/efi-rpm-macros-3-3.el8", tag.Name().Short())
// Verify metadata
f, err = fs.Open(".efi-rpm-macros.metadata")
require.Nil(t, err)
buf, err := io.ReadAll(f)
require.Nil(t, err)
require.Equal(t, "SOURCES/efi-rpm-macros-3.tar.bz2 f002f60baed7a47ca3e98b8dd7ece2f7352dac9ffab7ae3557eb56b481ce2f86\n", string(buf))
// Verify layout
ls, err := fs.ReadDir(".")
require.Nil(t, err)
require.Equal(t, 4, len(ls))
require.Equal(t, ".efi-rpm-macros.metadata", ls[0].Name())
require.Equal(t, ".gitignore", ls[1].Name())
require.Equal(t, "SOURCES", ls[2].Name())
require.Equal(t, "SPECS", ls[3].Name())
ls, err = fs.ReadDir("SOURCES")
require.Nil(t, err)
require.Equal(t, 1, len(ls))
require.Equal(t, "0001-macros.efi-srpm-make-all-of-our-macros-always-expand.patch", ls[0].Name())
ls, err = fs.ReadDir("SPECS")
require.Nil(t, err)
require.Equal(t, 1, len(ls))
require.Equal(t, "efi-rpm-macros.spec", ls[0].Name())
}
func TestPushTargetRepo(t *testing.T) {
s, err := FromFile("testdata/efi-rpm-macros-3-3.el8.src.rpm")
require.Nil(t, err)
require.NotNil(t, s)
defer func() {
require.Nil(t, s.Close())
}()
tempDir, err := os.MkdirTemp("", "peridot-srpm-import-test-*")
require.Nil(t, err)
defer os.RemoveAll(tempDir)
// Create a bare repo in tempDir
osfsTemp := osfs.New(tempDir)
dot, err := osfsTemp.Chroot(".git")
require.Nil(t, err)
filesystemTemp := filesystem.NewStorage(dot, cache.NewObjectLRUDefault())
require.Nil(t, filesystemTemp.Init())
_, err = git.Init(filesystemTemp, nil)
require.Nil(t, err)
// Push a commit to the bare repo
newTempDir, err := os.MkdirTemp("", "peridot-srpm-import-test-*")
require.Nil(t, err)
defer os.RemoveAll(newTempDir)
osfs2 := osfs.New(newTempDir)
dot2, err := osfs2.Chroot(".git")
require.Nil(t, err)
filesystemTemp2 := filesystem.NewStorage(dot2, cache.NewObjectLRUDefault())
repo, err := git.InitWithOptions(filesystemTemp2, osfs2, git.InitOptions{
DefaultBranch: "refs/heads/el8",
})
require.Nil(t, err)
_, err = repo.CreateRemote(&config.RemoteConfig{
Name: "origin",
URLs: []string{tempDir},
})
require.Nil(t, err)
w, err := repo.Worktree()
require.Nil(t, err)
f, err := w.Filesystem.Create("testfile")
require.Nil(t, err)
_, err = f.Write([]byte("test"))
require.Nil(t, err)
err = f.Close()
require.Nil(t, err)
_, err = w.Add("testfile")
require.Nil(t, err)
_, err = w.Commit("test commit", &git.CommitOptions{
Author: &object.Signature{
Name: "test",
Email: "test@resf.org",
When: time.Now(),
},
})
require.Nil(t, err)
require.Nil(t, s.pushTargetRepo(repo, &git.PushOptions{
RefSpecs: []config.RefSpec{"refs/heads/el8:refs/heads/el8"},
}))
// Verify testfile is still there
f, err = osfs2.Open("testfile")
require.Nil(t, err)
buf, err := io.ReadAll(f)
require.Nil(t, err)
require.Equal(t, "test", string(buf))
}
func TestImport1_New(t *testing.T) {
s, err := FromFile("testdata/efi-rpm-macros-3-3.el8.src.rpm")
require.Nil(t, err)
tempDir, err := os.MkdirTemp("", "peridot-srpm-import-test-*")
require.Nil(t, err)
defer os.RemoveAll(tempDir)
// Create a bare repo in tempDir
osfsTemp := osfs.New(tempDir)
dot, err := osfsTemp.Chroot(".git")
require.Nil(t, err)
filesystemTemp := filesystem.NewStorage(dot, cache.NewObjectLRUDefault())
require.Nil(t, filesystemTemp.Init())
_, err = git.Init(filesystemTemp, nil)
require.Nil(t, err)
opts := &git.CloneOptions{
URL: tempDir,
}
storer := memory.NewStorage()
fs := memfs.New()
lookaside := storage_memory.New(osfs.New("/"))
require.Nil(t, s.Import(opts, storer, fs, lookaside))
// Open repo
repo, err := git.PlainOpen(tempDir)
require.Nil(t, err)
// Switch to el8 branch
w, err := repo.Worktree()
require.Nil(t, err)
err = w.Checkout(&git.CheckoutOptions{
Branch: "refs/heads/el8",
})
require.Nil(t, err)
// Verify commit
objIter, err := repo.CommitObjects()
require.Nil(t, err)
obj, err := objIter.Next()
require.Nil(t, err)
require.Equal(t, "import efi-rpm-macros-3-3.el8", obj.Message)
// Verify tag
tagIter, err := repo.Tags()
require.Nil(t, err)
tag, err := tagIter.Next()
require.Nil(t, err)
require.Equal(t, "imports/el8/efi-rpm-macros-3-3.el8", tag.Name().Short())
// Verify metadata
f, err := fs.Open(".efi-rpm-macros.metadata")
require.Nil(t, err)
buf, err := io.ReadAll(f)
require.Nil(t, err)
require.Equal(t, "SOURCES/efi-rpm-macros-3.tar.bz2 f002f60baed7a47ca3e98b8dd7ece2f7352dac9ffab7ae3557eb56b481ce2f86\n", string(buf))
// Verify layout
ls, err := fs.ReadDir(".")
require.Nil(t, err)
require.Equal(t, 4, len(ls))
require.Equal(t, ".efi-rpm-macros.metadata", ls[0].Name())
require.Equal(t, ".gitignore", ls[1].Name())
require.Equal(t, "SOURCES", ls[2].Name())
require.Equal(t, "SPECS", ls[3].Name())
ls, err = fs.ReadDir("SOURCES")
require.Nil(t, err)
require.Equal(t, 1, len(ls))
require.Equal(t, "0001-macros.efi-srpm-make-all-of-our-macros-always-expand.patch", ls[0].Name())
ls, err = fs.ReadDir("SPECS")
require.Nil(t, err)
require.Equal(t, 1, len(ls))
require.Equal(t, "efi-rpm-macros.spec", ls[0].Name())
// Verify lookaside
ok, err := lookaside.Exists("f002f60baed7a47ca3e98b8dd7ece2f7352dac9ffab7ae3557eb56b481ce2f86")
require.Nil(t, err)
require.True(t, ok)
}
func TestImport2_New(t *testing.T) {
s, err := FromFile("testdata/bash-4.4.20-4.el8_6.src.rpm")
require.Nil(t, err)
tempDir, err := os.MkdirTemp("", "peridot-srpm-import-test-*")
require.Nil(t, err)
// Create a bare repo in tempDir
osfsTemp := osfs.New(tempDir)
dot, err := osfsTemp.Chroot(".git")
require.Nil(t, err)
filesystemTemp := filesystem.NewStorage(dot, cache.NewObjectLRUDefault())
require.Nil(t, filesystemTemp.Init())
_, err = git.Init(filesystemTemp, nil)
require.Nil(t, err)
opts := &git.CloneOptions{
URL: tempDir,
}
storer := memory.NewStorage()
fs := memfs.New()
lookaside := storage_memory.New(osfs.New("/"))
require.Nil(t, s.Import(opts, storer, fs, lookaside))
// Open repo
repo, err := git.PlainOpen(tempDir)
require.Nil(t, err)
// Switch to el8 branch
w, err := repo.Worktree()
require.Nil(t, err)
err = w.Checkout(&git.CheckoutOptions{
Branch: "refs/heads/el8",
})
require.Nil(t, err)
// Verify commit
objIter, err := repo.CommitObjects()
require.Nil(t, err)
obj, err := objIter.Next()
require.Nil(t, err)
require.Equal(t, "import bash-4.4.20-4.el8_6", obj.Message)
// Verify tag
tagIter, err := repo.Tags()
require.Nil(t, err)
tag, err := tagIter.Next()
require.Nil(t, err)
require.Equal(t, "imports/el8/bash-4.4.20-4.el8_6", tag.Name().Short())
// Verify metadata
f, err := fs.Open(".bash.metadata")
require.Nil(t, err)
buf, err := io.ReadAll(f)
require.Nil(t, err)
require.Equal(t, "SOURCES/bash-4.4.tar.gz d86b3392c1202e8ff5a423b302e6284db7f8f435ea9f39b5b1b20fd3ac36dfcb\n", string(buf))
// Verify layout
ls, err := fs.ReadDir(".")
require.Nil(t, err)
require.Equal(t, 4, len(ls))
require.Equal(t, ".bash.metadata", ls[0].Name())
require.Equal(t, ".gitignore", ls[1].Name())
require.Equal(t, "SOURCES", ls[2].Name())
require.Equal(t, "SPECS", ls[3].Name())
ls, err = fs.ReadDir("SOURCES")
require.Nil(t, err)
require.Equal(t, 61, len(ls))
ls, err = fs.ReadDir("SPECS")
require.Nil(t, err)
require.Equal(t, 1, len(ls))
require.Equal(t, "bash.spec", ls[0].Name())
// Verify lookaside
ok, err := lookaside.Exists("d86b3392c1202e8ff5a423b302e6284db7f8f435ea9f39b5b1b20fd3ac36dfcb")
require.Nil(t, err)
require.True(t, ok)
}