From 12ec42f8d04d3fae16affec58cda1fe40efd56fc Mon Sep 17 00:00:00 2001 From: Mustafa Gezen Date: Sun, 27 Aug 2023 08:31:49 +0200 Subject: [PATCH] Add more srpm_import tests --- .../worker_server/srpm_import/BUILD | 13 +- .../worker_server/srpm_import/srpm_import.go | 541 ++++++++++++++---- .../srpm_import/srpm_import_test.go | 232 +++++++- .../testdata/basesystem-11-5.el8.src.rpm | Bin 0 -> 10758 bytes 4 files changed, 684 insertions(+), 102 deletions(-) create mode 100644 tools/mothership/worker_server/srpm_import/testdata/basesystem-11-5.el8.src.rpm diff --git a/tools/mothership/worker_server/srpm_import/BUILD b/tools/mothership/worker_server/srpm_import/BUILD index c18dd76c..7901ae16 100644 --- a/tools/mothership/worker_server/srpm_import/BUILD +++ b/tools/mothership/worker_server/srpm_import/BUILD @@ -20,8 +20,13 @@ go_library( importpath = "go.resf.org/peridot/tools/mothership/worker_server/srpm_import", visibility = ["//visibility:public"], deps = [ - "//base/go", "//base/go/storage", + "//vendor/github.com/go-git/go-billy/v5:go-billy", + "//vendor/github.com/go-git/go-git/v5:go-git", + "//vendor/github.com/go-git/go-git/v5/config", + "//vendor/github.com/go-git/go-git/v5/plumbing", + "//vendor/github.com/go-git/go-git/v5/plumbing/object", + "//vendor/github.com/go-git/go-git/v5/storage", "//vendor/github.com/pkg/errors", "//vendor/github.com/sassoftware/go-rpmutils", "//vendor/golang.org/x/crypto/openpgp", @@ -38,6 +43,12 @@ go_test( "//base/go/storage/memory", "//vendor/github.com/go-git/go-billy/v5/memfs", "//vendor/github.com/go-git/go-billy/v5/osfs", + "//vendor/github.com/go-git/go-git/v5:go-git", + "//vendor/github.com/go-git/go-git/v5/config", + "//vendor/github.com/go-git/go-git/v5/plumbing/cache", + "//vendor/github.com/go-git/go-git/v5/plumbing/object", + "//vendor/github.com/go-git/go-git/v5/storage/filesystem", + "//vendor/github.com/go-git/go-git/v5/storage/memory", "//vendor/github.com/stretchr/testify/require", "//vendor/golang.org/x/crypto/openpgp", ], diff --git a/tools/mothership/worker_server/srpm_import/srpm_import.go b/tools/mothership/worker_server/srpm_import/srpm_import.go index 5efda4f6..8d670bc1 100644 --- a/tools/mothership/worker_server/srpm_import/srpm_import.go +++ b/tools/mothership/worker_server/srpm_import/srpm_import.go @@ -15,142 +15,487 @@ package srpm_import import ( - "crypto/sha256" - "encoding/hex" - "github.com/pkg/errors" - "github.com/sassoftware/go-rpmutils" - "go.resf.org/peridot/base/go/storage" - "golang.org/x/crypto/openpgp" - "io" - "os" - "path/filepath" - "strings" + "crypto/sha256" + "encoding/hex" + "fmt" + "github.com/go-git/go-billy/v5" + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/config" + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/object" + storage2 "github.com/go-git/go-git/v5/storage" + "github.com/pkg/errors" + "github.com/sassoftware/go-rpmutils" + "go.resf.org/peridot/base/go/storage" + "golang.org/x/crypto/openpgp" + "io" + "os" + "path/filepath" + "regexp" + "strings" + "time" ) -type State struct { - tempDir string +var elDistRegex = regexp.MustCompile(`el\d+`) - // lookasideBlobs is a map of blob names to their SHA256 hashes. - lookasideBlobs map[string]string +type State struct { + tempDir string + + rpm *rpmutils.Rpm + + // lookasideBlobs is a map of blob names to their SHA256 hashes. + lookasideBlobs map[string]string +} + +// copyFromOS copies specified file from OS filesystem to target filesystem. +func copyFromOS(targetFS billy.Filesystem, path string, targetPath string) error { + // Open file from OS filesystem. + f, err := os.Open(path) + if err != nil { + return errors.Wrap(err, "failed to open file") + } + defer f.Close() + + // Create file in target filesystem. + targetFile, err := targetFS.Create(targetPath) + if err != nil { + return errors.Wrap(err, "failed to create file") + } + defer targetFile.Close() + + // Copy contents of file from OS filesystem to target filesystem. + _, err = io.Copy(targetFile, f) + if err != nil { + return errors.Wrap(err, "failed to copy file") + } + + return nil } // FromFile creates a new State from an SRPM file. // The SRPM file is extracted to a temporary directory. func FromFile(path string, keys ...*openpgp.Entity) (*State, error) { - f, err := os.Open(path) - if err != nil { - return nil, errors.Wrap(err, "failed to open file") - } - defer f.Close() + f, err := os.Open(path) + if err != nil { + return nil, errors.Wrap(err, "failed to open file") + } + defer f.Close() - // If keys is not empty, then verify the RPM signature. - if len(keys) > 0 { - _, _, err := rpmutils.Verify(f, keys) - if err != nil { - return nil, errors.Wrap(err, "failed to verify RPM") - } + // If keys is not empty, then verify the RPM signature. + if len(keys) > 0 { + _, _, err := rpmutils.Verify(f, keys) + if err != nil { + return nil, errors.Wrap(err, "failed to verify RPM") + } - // After verifying the RPM, seek back to the start of the file. - _, err = f.Seek(0, io.SeekStart) - if err != nil { - return nil, errors.Wrap(err, "failed to seek to start of file") - } - } + // After verifying the RPM, seek back to the start of the file. + _, err = f.Seek(0, io.SeekStart) + if err != nil { + return nil, errors.Wrap(err, "failed to seek to start of file") + } + } - rpm, err := rpmutils.ReadRpm(f) - if err != nil { - return nil, errors.Wrap(err, "failed to read RPM") - } + rpm, err := rpmutils.ReadRpm(f) + if err != nil { + return nil, errors.Wrap(err, "failed to read RPM") + } - state := &State{ - lookasideBlobs: make(map[string]string), - } - // Create a temporary directory. - state.tempDir, err = os.MkdirTemp("", "srpm_import-*") - if err != nil { - return nil, errors.Wrap(err, "failed to create temporary directory") - } + state := &State{ + rpm: rpm, + lookasideBlobs: make(map[string]string), + } + // Create a temporary directory. + state.tempDir, err = os.MkdirTemp("", "srpm_import-*") + if err != nil { + return nil, errors.Wrap(err, "failed to create temporary directory") + } - // Extract the SRPM. - err = rpm.ExpandPayload(state.tempDir) - if err != nil { - return nil, errors.Wrap(err, "failed to extract SRPM") - } + // Extract the SRPM. + err = rpm.ExpandPayload(state.tempDir) + if err != nil { + return nil, errors.Wrap(err, "failed to extract SRPM") + } - return state, nil + return state, nil } func (s *State) Close() error { - return os.RemoveAll(s.tempDir) + return os.RemoveAll(s.tempDir) } func (s *State) GetDir() string { - return s.tempDir + return s.tempDir } // determineLookasideBlobs determines which blobs need to be uploaded to the // lookaside cache. // Currently, the rule is that if a file is larger than 5MB, and is binary, // then it should be uploaded to the lookaside cache. +// If the file name contains ".tar", then it is assumed to be a tarball, and +// is ALWAYS uploaded to the lookaside cache. func (s *State) determineLookasideBlobs() error { - // Read all files in tempDir, except for the SPEC file - // For each file, if it is larger than 5MB, and is binary, then add it to - // the lookasideBlobs map. - // If the file is not binary, then skip it. - ls, err := os.ReadDir(s.tempDir) - if err != nil { - return errors.Wrap(err, "failed to read directory") - } + // Read all files in tempDir, except for the SPEC file + // For each file, if it is larger than 5MB, and is binary, then add it to + // the lookasideBlobs map. + // If the file is not binary, then skip it. + ls, err := os.ReadDir(s.tempDir) + if err != nil { + return errors.Wrap(err, "failed to read directory") + } - for _, f := range ls { - // If file ends with ".spec", then skip it. - if f.IsDir() || strings.HasSuffix(f.Name(), ".spec") { - continue - } + for _, f := range ls { + // If file ends with ".spec", then skip it. + if f.IsDir() || strings.HasSuffix(f.Name(), ".spec") { + continue + } - // If file is larger than 5MB, then add it to the lookasideBlobs map. - info, err := f.Info() - if err != nil { - return errors.Wrap(err, "failed to get file info") - } + // If file is larger than 5MB, then add it to the lookasideBlobs map. + info, err := f.Info() + if err != nil { + return errors.Wrap(err, "failed to get file info") + } - if info.Size() > 5*1024*1024 { - sum, err := func() (string, error) { - hash := sha256.New() - file, err := os.Open(filepath.Join(s.tempDir, f.Name())) - if err != nil { - return "", errors.Wrap(err, "failed to open file") - } - defer file.Close() + if info.Size() > 5*1024*1024 || strings.Contains(f.Name(), ".tar") { + sum, err := func() (string, error) { + hash := sha256.New() + file, err := os.Open(filepath.Join(s.tempDir, f.Name())) + if err != nil { + return "", errors.Wrap(err, "failed to open file") + } + defer file.Close() - _, err = io.Copy(hash, file) - if err != nil { - return "", errors.Wrap(err, "failed to copy file") - } + _, err = io.Copy(hash, file) + if err != nil { + return "", errors.Wrap(err, "failed to copy file") + } - return hex.EncodeToString(hash.Sum(nil)), nil - }() - if err != nil { - return err - } + return hex.EncodeToString(hash.Sum(nil)), nil + }() + if err != nil { + return err + } - s.lookasideBlobs[f.Name()] = sum - } - } + s.lookasideBlobs[f.Name()] = sum + } + } - return nil + return nil } // uploadLookasideBlobs uploads all blobs in the lookasideBlobs map to the // lookaside cache. func (s *State) uploadLookasideBlobs(lookaside storage.Storage) error { - // The object name is the SHA256 hash of the file. - for path, hash := range s.lookasideBlobs { - _, err := lookaside.Put(hash, filepath.Join(s.tempDir, path)) - if err != nil { - return errors.Wrap(err, "failed to upload file") - } - } + // The object name is the SHA256 hash of the file. + for path, hash := range s.lookasideBlobs { + _, err := lookaside.Put(hash, filepath.Join(s.tempDir, path)) + if err != nil { + return errors.Wrap(err, "failed to upload file") + } + } - return nil + return nil +} + +// writeMetadata file writes the metadata map file. +// The metadata file contains lines of the format: +// +// +// +// For example: +// +// SOURCES/bar 1234567890abcdef +func (s *State) writeMetadataFile(targetFS billy.Filesystem) error { + // Open metadata file for writing. + name, err := s.rpm.Header.GetStrings(rpmutils.NAME) + if err != nil { + return errors.Wrap(err, "failed to get RPM name") + } + + metadataFile := fmt.Sprintf(".%s.metadata", name[0]) + f, err := targetFS.Create(metadataFile) + if err != nil { + return errors.Wrap(err, "failed to open metadata file") + } + defer f.Close() + + // Write each line to the metadata file. + for path, hash := range s.lookasideBlobs { + // RPM sources MUST be in SOURCES/ directory + _, err = f.Write([]byte(filepath.Join("SOURCES", path) + " " + hash + "\n")) + if err != nil { + return errors.Wrap(err, "failed to write line to metadata file") + } + } + + // Each file in metadata needs to be added to gitignore + // Overwrite the gitignore file + gitignoreFile := ".gitignore" + f, err = targetFS.OpenFile(gitignoreFile, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644) + if err != nil { + return errors.Wrap(err, "failed to open gitignore file") + } + + // Write each line to the gitignore file. + for path, _ := range s.lookasideBlobs { + _, err = f.Write([]byte(filepath.Join("SOURCES", path) + "\n")) + if err != nil { + return errors.Wrap(err, "failed to write line to gitignore file") + } + } + + return nil +} + +// expandLayout expands the layout of the SRPM into the target filesystem. +// Moves all sources into SOURCES/ directory. +// Spec file is moved to SPECS/ directory. +func (s *State) expandLayout(targetFS billy.Filesystem) error { + // Create SOURCES/ directory. + err := targetFS.MkdirAll("SOURCES", 0755) + if err != nil { + return errors.Wrap(err, "failed to create SOURCES directory") + } + + // Copy all files from OS filesystem to target filesystem. + ls, err := os.ReadDir(s.tempDir) + if err != nil { + return errors.Wrap(err, "failed to read directory") + } + + for _, f := range ls { + baseName := filepath.Base(f.Name()) + // If file ends with ".spec", then copy to SPECS/ directory. + if strings.HasSuffix(f.Name(), ".spec") { + err := copyFromOS(targetFS, filepath.Join(s.tempDir, f.Name()), filepath.Join("SPECS", baseName)) + if err != nil { + return errors.Wrap(err, "failed to copy spec file") + } + } else { + // Copy all other files to SOURCES/ directory. + // Only if they are not present in lookasideBlobs + _, ok := s.lookasideBlobs[f.Name()] + if ok { + continue + } + err := copyFromOS(targetFS, filepath.Join(s.tempDir, f.Name()), filepath.Join("SOURCES", baseName)) + if err != nil { + return errors.Wrap(err, "failed to copy file") + } + } + } + + return nil +} + +// getRepo returns the target repository for the SRPM. +// This is where the payload is uploaded to. +func (s *State) getRepo(opts *git.CloneOptions, storer storage2.Storer, targetFS billy.Filesystem) (*git.Repository, error) { + // Determine dist tag + nevra, err := s.rpm.Header.GetNEVRA() + if err != nil { + return nil, errors.Wrap(err, "failed to get NEVRA") + } + + // The dist tag will be used as the branch + dist := elDistRegex.FindString(nevra.Release) + if dist == "" { + return nil, errors.Wrap(err, "failed to determine dist tag") + } + + // Set branch to dist tag + opts.ReferenceName = plumbing.NewBranchReferenceName(dist) + opts.SingleBranch = true + + // Clone the repository, to the target filesystem. + // We do an init, then a fetch, then a checkout + // If the repo doesn't exist, then we init only + repo, err := git.Init(storer, targetFS) + if err != nil { + return nil, errors.Wrap(err, "failed to init repo") + } + wt, err := repo.Worktree() + if err != nil { + return nil, errors.Wrap(err, "failed to get worktree") + } + + // Create a new remote + refspec := config.RefSpec(fmt.Sprintf("+refs/heads/%s:refs/remotes/origin/%[1]s", dist)) + _, err = repo.CreateRemote(&config.RemoteConfig{ + Name: "origin", + URLs: []string{opts.URL}, + Fetch: []config.RefSpec{refspec}, + }) + if err != nil { + return nil, errors.Wrap(err, "failed to create remote") + } + + // Fetch the remote + err = repo.Fetch(&git.FetchOptions{ + RemoteName: "origin", + RefSpecs: []config.RefSpec{refspec}, + }) + + // Checkout the branch + refName := plumbing.NewBranchReferenceName(dist) + + if err != nil { + h := plumbing.NewSymbolicReference(plumbing.HEAD, refName) + if err := repo.Storer.CheckAndSetReference(h, nil); err != nil { + return nil, errors.Wrap(err, "failed to checkout branch") + } + } else { + err = wt.Checkout(&git.CheckoutOptions{ + Branch: plumbing.NewRemoteReferenceName("origin", dist), + Force: true, + }) + } + + return repo, nil +} + +// cleanTargetRepo deletes all files in the target repository. +func (s *State) cleanTargetRepo(wt *git.Worktree, root string) error { + // Delete all files in the target repository. + ls, err := wt.Filesystem.ReadDir(root) + if err != nil { + return errors.Wrap(err, "failed to read directory") + } + + for _, f := range ls { + // If it's a directory, then recurse into it. + if f.IsDir() { + err := s.cleanTargetRepo(wt, filepath.Join(root, f.Name())) + if err != nil { + return errors.Wrap(err, "failed to clean target repo") + } + } else { + // Otherwise, delete the file. + _, err := wt.Remove(filepath.Join(root, f.Name())) + if err != nil { + return errors.Wrap(err, "failed to remove file") + } + } + } + + return nil +} + +// populateTargetRepo runs the following steps: +// 1. Clean the target repository. +// 2. Determine which blobs need to be uploaded to the lookaside cache. +// 3. Upload blobs to the lookaside cache. +// 4. Write the metadata file. +// 5. Expand the layout of the SRPM. +// 6. Commit the changes to the target repository. +func (s *State) populateTargetRepo(repo *git.Repository, targetFS billy.Filesystem, lookaside storage.Storage) error { + // Clean the target repository. + wt, err := repo.Worktree() + if err != nil { + return errors.Wrap(err, "failed to get worktree") + } + + err = s.cleanTargetRepo(wt, ".") + if err != nil { + return errors.Wrap(err, "failed to clean target repo") + } + + // Determine which blobs need to be uploaded to the lookaside cache. + err = s.determineLookasideBlobs() + if err != nil { + return errors.Wrap(err, "failed to determine lookaside blobs") + } + + // Upload blobs to the lookaside cache. + err = s.uploadLookasideBlobs(lookaside) + if err != nil { + return errors.Wrap(err, "failed to upload lookaside blobs") + } + + // Write the metadata file. + err = s.writeMetadataFile(targetFS) + if err != nil { + return errors.Wrap(err, "failed to write metadata file") + } + + // Expand the layout of the SRPM. + err = s.expandLayout(targetFS) + if err != nil { + return errors.Wrap(err, "failed to expand layout") + } + + // Commit the changes to the target repository. + _, err = wt.Add(".") + if err != nil { + return errors.Wrap(err, "failed to add files") + } + + nevra, err := s.rpm.Header.GetNEVRA() + if err != nil { + return errors.Wrap(err, "failed to get NEVRA") + } + _, err = wt.Commit("import "+nevra.String(), &git.CommitOptions{ + Author: &object.Signature{ + Name: "Mship Bot", + Email: "no-reply+mshipbot@resf.org", + When: time.Now(), + }, + }) + if err != nil { + return errors.Wrap(err, "failed to commit changes") + } + + // Create a tag + // The tag should follow the following format: + // imports// + dist := elDistRegex.FindString(nevra.Release) + if dist == "" { + return errors.Wrap(err, "failed to determine dist tag") + } + tag := fmt.Sprintf("imports/%s/%s-%s-%s", dist, nevra.Name, nevra.Version, nevra.Release) + _, err = repo.CreateTag(tag, plumbing.NewHash("HEAD"), &git.CreateTagOptions{ + Tagger: &object.Signature{ + Name: "Mship Bot", + Email: "no-reply+mshipbot@resf.org", + When: time.Now(), + }, + }) + if err != nil { + return errors.Wrap(err, "failed to create tag") + } + + return nil +} + +// pushTargetRepo pushes the target repository to the upstream repository. +func (s *State) pushTargetRepo(repo *git.Repository, opts *git.PushOptions) error { + // Push the target repository to the upstream repository. + err := repo.Push(opts) + if err != nil { + return errors.Wrap(err, "failed to push repo") + } + + return nil +} + +// Import imports the SRPM into the target repository. +func (s *State) Import(opts *git.CloneOptions, storer storage2.Storer, targetFS billy.Filesystem, lookaside storage.Storage) error { + // Get the target repository. + repo, err := s.getRepo(opts, storer, targetFS) + if err != nil { + return errors.Wrap(err, "failed to get repo") + } + + // Populate the target repository. + err = s.populateTargetRepo(repo, targetFS, lookaside) + if err != nil { + return errors.Wrap(err, "failed to populate target repo") + } + + // Push the target repository. + err = s.pushTargetRepo(repo, &git.PushOptions{}) + if err != nil { + return errors.Wrap(err, "failed to push target repo") + } + + return nil } diff --git a/tools/mothership/worker_server/srpm_import/srpm_import_test.go b/tools/mothership/worker_server/srpm_import/srpm_import_test.go index 482ee1b4..1190f6f0 100644 --- a/tools/mothership/worker_server/srpm_import/srpm_import_test.go +++ b/tools/mothership/worker_server/srpm_import/srpm_import_test.go @@ -17,11 +17,19 @@ package srpm_import import ( "github.com/go-git/go-billy/v5/memfs" "github.com/go-git/go-billy/v5/osfs" + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/config" + "github.com/go-git/go-git/v5/plumbing/cache" + "github.com/go-git/go-git/v5/plumbing/object" + "github.com/go-git/go-git/v5/storage/filesystem" + "github.com/go-git/go-git/v5/storage/memory" "github.com/stretchr/testify/require" storage_memory "go.resf.org/peridot/base/go/storage/memory" "golang.org/x/crypto/openpgp" + "io" "os" "testing" + "time" ) func TestFromFile(t *testing.T) { @@ -58,7 +66,7 @@ func TestFromFile_SignatureFail(t *testing.T) { } func TestDetermineLookasideBlobs_Empty(t *testing.T) { - s, err := FromFile("testdata/efi-rpm-macros-3-3.el8.src.rpm") + s, err := FromFile("testdata/basesystem-11-5.el8.src.rpm") require.Nil(t, err) require.NotNil(t, s) defer func() { @@ -68,7 +76,7 @@ func TestDetermineLookasideBlobs_Empty(t *testing.T) { require.Equal(t, 0, len(s.lookasideBlobs)) } -func TestDetermineLookasideBlobs_NotEmpty_Over5MB(t *testing.T) { +func TestDetermineLookasideBlobs_NotEmpty_Tarball(t *testing.T) { s, err := FromFile("testdata/bash-4.4.20-4.el8_6.src.rpm") require.Nil(t, err) require.NotNil(t, s) @@ -80,7 +88,7 @@ func TestDetermineLookasideBlobs_NotEmpty_Over5MB(t *testing.T) { } func TestUploadLookaside_Empty(t *testing.T) { - s, err := FromFile("testdata/efi-rpm-macros-3-3.el8.src.rpm") + s, err := FromFile("testdata/basesystem-11-5.el8.src.rpm") require.Nil(t, err) require.NotNil(t, s) defer func() { @@ -115,3 +123,221 @@ func TestUploadLookaside_NotEmpty(t *testing.T) { require.Nil(t, err) require.True(t, ok) } + +func TestWriteMetadataFile(t *testing.T) { + s, err := FromFile("testdata/bash-4.4.20-4.el8_6.src.rpm") + require.Nil(t, err) + require.NotNil(t, s) + defer func() { + require.Nil(t, s.Close()) + }() + require.Nil(t, s.determineLookasideBlobs()) + + fs := memfs.New() + require.Nil(t, s.writeMetadataFile(fs)) + + fi, err := fs.ReadDir(".") + require.Nil(t, err) + require.Equal(t, 2, len(fi)) + require.Equal(t, ".bash.metadata", fi[0].Name()) + require.Equal(t, ".gitignore", fi[1].Name()) + + f, err := fs.Open(".bash.metadata") + require.Nil(t, err) + + buf := make([]byte, 1024) + n, err := f.Read(buf) + require.Nil(t, err) + + require.Equal(t, "SOURCES/bash-4.4.tar.gz d86b3392c1202e8ff5a423b302e6284db7f8f435ea9f39b5b1b20fd3ac36dfcb\n", string(buf[:n])) + + f, err = fs.Open(".gitignore") + require.Nil(t, err) + + buf = make([]byte, 1024) + n, err = f.Read(buf) + require.Nil(t, err) + + require.Equal(t, "SOURCES/bash-4.4.tar.gz\n", string(buf[:n])) +} + +func TestExpandLayout(t *testing.T) { + s, err := FromFile("testdata/efi-rpm-macros-3-3.el8.src.rpm") + require.Nil(t, err) + require.NotNil(t, s) + defer func() { + require.Nil(t, s.Close()) + }() + + fs := memfs.New() + require.Nil(t, s.expandLayout(fs)) + + fi, err := fs.ReadDir(".") + require.Nil(t, err) + require.Equal(t, 2, len(fi)) + require.Equal(t, "SOURCES", fi[0].Name()) + require.Equal(t, "SPECS", fi[1].Name()) + + fi, err = fs.ReadDir("SOURCES") + require.Nil(t, err) + + require.Equal(t, 2, len(fi)) + require.Equal(t, "0001-macros.efi-srpm-make-all-of-our-macros-always-expand.patch", fi[0].Name()) + require.Equal(t, "efi-rpm-macros-3.tar.bz2", fi[1].Name()) + + fi, err = fs.ReadDir("SPECS") + require.Nil(t, err) + + require.Equal(t, 1, len(fi)) + require.Equal(t, "efi-rpm-macros.spec", fi[0].Name()) +} + +func TestWriteMetadataExpandLayout(t *testing.T) { + s, err := FromFile("testdata/efi-rpm-macros-3-3.el8.src.rpm") + require.Nil(t, err) + require.NotNil(t, s) + defer func() { + require.Nil(t, s.Close()) + }() + + fs := memfs.New() + require.Nil(t, s.determineLookasideBlobs()) + require.Nil(t, s.writeMetadataFile(fs)) + require.Nil(t, s.expandLayout(fs)) + + fi, err := fs.ReadDir(".") + require.Nil(t, err) + require.Equal(t, 4, len(fi)) + require.Equal(t, ".efi-rpm-macros.metadata", fi[0].Name()) + require.Equal(t, ".gitignore", fi[1].Name()) + require.Equal(t, "SOURCES", fi[2].Name()) + require.Equal(t, "SPECS", fi[3].Name()) + + fi, err = fs.ReadDir("SOURCES") + require.Nil(t, err) + + require.Equal(t, 1, len(fi)) + require.Equal(t, "0001-macros.efi-srpm-make-all-of-our-macros-always-expand.patch", fi[0].Name()) + + fi, err = fs.ReadDir("SPECS") + require.Nil(t, err) + + require.Equal(t, 1, len(fi)) + require.Equal(t, "efi-rpm-macros.spec", fi[0].Name()) + + f, err := fs.Open(".efi-rpm-macros.metadata") + require.Nil(t, err) + + buf, err := io.ReadAll(f) + require.Nil(t, err) + + require.Equal(t, "SOURCES/efi-rpm-macros-3.tar.bz2 f002f60baed7a47ca3e98b8dd7ece2f7352dac9ffab7ae3557eb56b481ce2f86\n", string(buf)) +} + +func TestGetRepo_New(t *testing.T) { + s, err := FromFile("testdata/efi-rpm-macros-3-3.el8.src.rpm") + require.Nil(t, err) + require.NotNil(t, s) + defer func() { + require.Nil(t, s.Close()) + }() + + tempDir, err := os.MkdirTemp("", "peridot-srpm-import-test-*") + require.Nil(t, err) + defer os.RemoveAll(tempDir) + + storer := memory.NewStorage() + fs := memfs.New() + opts := &git.CloneOptions{ + URL: "file://" + tempDir, + } + repo, err := s.getRepo(opts, storer, fs) + require.Nil(t, err) + require.NotNil(t, repo) + + // Verify empty + objIter, err := repo.CommitObjects() + require.Nil(t, err) + _, err = objIter.Next() + require.Equal(t, io.EOF, err) +} + +func TestGetRepo_Existing(t *testing.T) { + s, err := FromFile("testdata/efi-rpm-macros-3-3.el8.src.rpm") + require.Nil(t, err) + require.NotNil(t, s) + defer func() { + require.Nil(t, s.Close()) + }() + + tempDir, err := os.MkdirTemp("", "peridot-srpm-import-test-*") + require.Nil(t, err) + defer os.RemoveAll(tempDir) + + // Create a bare repo in tempDir + osfsTemp := osfs.New(tempDir) + dot, err := osfsTemp.Chroot(".git") + require.Nil(t, err) + filesystemTemp := filesystem.NewStorage(dot, cache.NewObjectLRUDefault()) + require.Nil(t, filesystemTemp.Init()) + _, err = git.Init(filesystemTemp, nil) + require.Nil(t, err) + + // Push a commit to the bare repo + newTempDir, err := os.MkdirTemp("", "peridot-srpm-import-test-*") + require.Nil(t, err) + defer os.RemoveAll(newTempDir) + + osfs2 := osfs.New(newTempDir) + dot2, err := osfs2.Chroot(".git") + require.Nil(t, err) + filesystemTemp2 := filesystem.NewStorage(dot2, cache.NewObjectLRUDefault()) + + repo, err := git.InitWithOptions(filesystemTemp2, osfs2, git.InitOptions{ + DefaultBranch: "refs/heads/el8", + }) + require.Nil(t, err) + _, err = repo.CreateRemote(&config.RemoteConfig{ + Name: "origin", + URLs: []string{tempDir}, + }) + require.Nil(t, err) + w, err := repo.Worktree() + require.Nil(t, err) + f, err := w.Filesystem.Create("testfile") + require.Nil(t, err) + _, err = f.Write([]byte("test")) + require.Nil(t, err) + err = f.Close() + require.Nil(t, err) + _, err = w.Add("testfile") + require.Nil(t, err) + _, err = w.Commit("test commit", &git.CommitOptions{ + Author: &object.Signature{ + Name: "test", + Email: "test@resf.org", + When: time.Now(), + }, + }) + require.Nil(t, err) + err = repo.Push(&git.PushOptions{ + RefSpecs: []config.RefSpec{"refs/heads/el8:refs/heads/el8"}, + }) + require.Nil(t, err) + + storer := memory.NewStorage() + fs := memfs.New() + opts := &git.CloneOptions{ + URL: "file://" + tempDir, + } + repo, err = s.getRepo(opts, storer, fs) + require.Nil(t, err) + require.NotNil(t, repo) + + // Verify commit + objIter, err := repo.CommitObjects() + require.Nil(t, err) + obj, err := objIter.Next() + require.Nil(t, err) + require.Equal(t, "test commit", obj.Message) +} diff --git a/tools/mothership/worker_server/srpm_import/testdata/basesystem-11-5.el8.src.rpm b/tools/mothership/worker_server/srpm_import/testdata/basesystem-11-5.el8.src.rpm new file mode 100644 index 0000000000000000000000000000000000000000..29463506deb173819be6bab5e85203df35cc7a26 GIT binary patch literal 10758 zcmeHNdss~C_upNRlF)@imn}t6)66t8&1BScRiV2g!k(EuH8stQnW^MflzelkbL1|U zQslT)R7f#M91*7+MdTJmN-nvUzVBO`FX#N8bDVQ}p5H(A^Xysg=Y7{&d%f$uuJugo z{ zbO`wLpk54q5mg4j+Mv+Cw+!q;RE83GE}#IbH3O;++IJE737|lqYId)T+S6XFeiKlj zU+o#7fL|TfL-}^19Z;?HDXO;Wh>8YcWH4G?>Q^q^XZb3y>ZILK{k+fvQHqdx{Z!}q zI6g--2M+mEGi|v}w?f(f#*8cHPb#*~O1qwK-hHWgRenPGE8YD;T$%Rd+(jvqnn%UI ze^^@kX6Yv9ig&lDk*(%?hVSGIe>}kZjlq;bb4GnUiP(e6$`45dly=G+B>ZN`9)ruR&e_Wowe#4W72H=27B0vJxyI(r|JY2o;2>i>WW$2 zmd;eSH!`L0=G8{YjiQ2$o4UsNTLsp-RX$K8z7HN=+Ak>FB

d?x-UNeD!iA7FrjN zKYBO4rR4IOQxPK@9yH;hB{NOlH%2)6uIl>b)yoN$OQun7+AgiHTG6@hh@tq}V3*}t zkB>Cm7?jT(ms@l07LtGHykXssMT@A0u4C^XvkN-1esTW$hYoMr;thvJu8=j|`8Y2$ z|NQyY15N9at2z_A)KKTKHJetaoqJNcMiTS$g`&8ZiB5T8u5B|WO7HGX+OjHRBH5#H z(B)alSjT_*p?-NuhHAC-* z84Nm;&Z6Er@|`7B2Zb3w*WG6|9V1uGJy%#ix%pj-`551u8MJ#(i#f9mhD9z`9jP(V zxX{M=@Uo+CN$A_I)Qgr{SGqaH$-rQm=+l}~TKWym=-@l}K zaB{ubkNRUR%4V)Kp&q|={?^#&=2)_^`L;{-gZ5TS=NQ^eV&07zJl67gxyzuzvLByKMgr!%HT-t~%T~?aFd?ollhByuc`}Ca(rj?i(?!$a;nSLOv zFPup(KK;Y9Ny%~BR@TIvD)!sF`N}>|hntLkS$Cc2Z?r#L6lLY=8pNg6TKL|-VVMG^ z5;6?#wcnrb9{BEo?;iN>f$tvp?t$+f`0j!49{BEo|6dREjMn=4T<&<1V#Y`W1{{8ppf5!z#|C^{Ri7& zJ?MjKIG|8J6Hus63g{S6WC050I|*#xiw9x^cIw4CK(g9as;)?@zQ(B`_RUDDO+)P(UGnAc5x)m``9x;rDR@Lke*OfdvGX^kQ8& z?ohviz!gOM0s=Pz3ct6Mzz+e1{7Q&jkpJZ`5efzA&0hwFIJNjN&o4EDjXDR0j>|HQNfs#C%k!&?IEUkR3|O@F+F>tV0$!A5iP?xzmpE=NhYIi;uxV!B8~#Cvw?j>dh{b= z2`1x(_vl}i4Q=CQG3g*%X2+s41T=;%m5ej&SOQx<3nbQ9OeSVWr*Ltc!J<(FIL@*q z+p<`E8iPxx&{$L^l?gI!J?0_^8U;-4=^6q{2m?j?xWJLvt8*oDuLtMKZoNpxRT?oHeNZ+r$Ol@%;I=vr71l+WlFc^ zp-R`nMvlvt`AR20A%|mC%5fmAR8Hb0C@1OabJ$OBDA{bXk|hdNve->Z#;ZFVM%!d% z&$~)rNxw?v&|T#m+PFN9JI(W8=PBUW zI-u4lO!FiELYVe%gl+$gFct9;^JRDh8U$8hB+8b=K+2Kf{BTS`0!!5ac8>B_pRk6m z5edOSlm2f9k>Qy4ks5NJSi|ZQEzCaAV%H~H7~e3Ka2)<}s{ZL{_KB%~IcNXW1i0d0 zN2rKzxKM;n5esE10CdluYS_0t$)IHGIJpfh2NhM&+>?m{wjcf^S z*L*(82Mb++h5e)PqtN@H&(uE9@rOC!933S^L9if)0SXKvWIgMg-?Pq%+aZMvd_kjA zs7w}|zAoURDUOz+QV?Q%igRRmj1Z3nY*Z?pL1scWLeb}v;x9{FXbaE;7>U(|f41lE z*MUIg_a^X3XmB`=<0L}cCk7Z`!Kh6Py3o65@&AP_s0F-5!>ct7c8f0&k5d4_`O!id zh_9qks0CMsW07*O2S{--9~blDP+m9)kb0f)nGeYJ>V`Wg2q!*c83+LZ070Nw-Ww^( zK@h_i3WT8hkeLsHN`VXnmNLK=h4I1Wg7yYbpZxzRp87lS9ZF$P=yo&~%NC{@V9ek1 zeky+vQIp^~i6A_wP=bJSQi{M_1bj526#{D}!exQKH}IGkj`8W39h1etF)Eczqmfy5 zAZDY}1QfCzm%`;z=s1fGDyUodK9@UtrN*N0vy z9qK%?GCgUcnXkRGc94^JNY-pE)`|lEjp{F~b{~!Cau}t%pPe%AX*SZZ>*NRTjNQ|h zeo#HOhBHGF<6d&*w&&V?C8u+}W5;&z3)gJ758s}W73z`vW1)e)ykv_sGCJqAv+mIs zJ_nziTzx!)o3*FGqE4TRl;t}9)W?*N?tWB>8PQRZ%b^nAx zQK63?Ps#mh%(aj?YpYlx1{ub@NZXn_^|RhBeHJu(b=k1T)aE%mWIR%3x*~I3jpq)| z)tC(Xw0qZXr3Qqps3|a5v14<0xcAytW>M{vefgnh=SMgu2jm;=eWbb9FJpM|%SEXH zT2=h)=b8@wS3}QoZXfH|P+{??*!!n<)yr&En~vw0VQwCW7EgX2>9@DsxQevvLgFpw zWhy~eXFX)`%7S*+9wK$QskW@&?D(t8WlPh{X1k7#C+#gu*A}$Ws%QLqp`Tyu(~BCB z4%!prC&q}ZPjoG|f0W%c^{71!5I)Uu|? z8%xM*+*BnwSCd2PoMJ-Tfxc=aQ%O)@R_BeVSd^4OB%8CP4UH*K_k zICAmuNsZ2Ycb%{H+b^0X~-c#4Md~fYP zRN-~9gcLi$LT_MnP^`!MS$FKq^Q}%Sjnu`oRAQ1_bfb<4z~{;+TR^NdZxlLIriPpNfM<;HaYMppE1Js+?Az^tM&5}j;EjA zQ+~e6`hr!Za=q42N0psL<44@KOQ;m??Gzh0*8j43LMHA07`|2M#g&;s;$wx4PZQ@m zbfp!Zt*y+=Z2q`JW~ESV^{=vd=MuQmhhlnt_29(pWAjW~56{UPkVntacB)-`cAOYB zn!37Ul;h5Vkn-`Dr-?_T>$yC1l(N{)IAfU05&N*V_T148dTDn}&EtxkZX z-rCNazBXGk*Z3yKauU;jjC4w+Z8-E(kh-mM)`_eQE9#D~ot@8E9DD4<=(G6sh}tgJ zfTnkAHgLVA)MdK*Qy%bVug$xzDdpW|?`;d?Z}g6Pzhn2M{)#4>M-zXb-SrAcJ=ou0 zKP5%EF3?r&rEAUA15LM9DXOK_6CHo`xj5!YdssjhTez?xHNN}J=F9Oj-W7$h@2shM Z(;XM9`1th35u=442S~Nil0m}|