Start working on srpm importer in Go

tools/mothership/worker_server/srpm_import/BUILD

@@ -0,0 +1,30 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "srpm_import",
+    srcs = ["srpm_import.go"],
+    importpath = "go.resf.org/peridot/tools/mothership/worker_server/srpm_import",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//base/go",
+        "//base/go/storage",
+        "//vendor/github.com/pkg/errors",
+        "//vendor/github.com/sassoftware/go-rpmutils",
+        "//vendor/golang.org/x/crypto/openpgp",
+    ],
+)
+
+go_test(
+    name = "srpm_import_test",
+    size = "small",
+    srcs = ["srpm_import_test.go"],
+    data = glob(["testdata/**"]),
+    embed = [":srpm_import"],
+    deps = [
+        "//base/go/storage/memory",
+        "//vendor/github.com/go-git/go-billy/v5/memfs",
+        "//vendor/github.com/go-git/go-billy/v5/osfs",
+        "//vendor/github.com/stretchr/testify/require",
+        "//vendor/golang.org/x/crypto/openpgp",
+    ],
+)

tools/mothership/worker_server/srpm_import/srpm_import.go

@@ -0,0 +1,142 @@
+package srpm_import
+
+import (
+    "crypto/sha256"
+    "encoding/hex"
+    "github.com/pkg/errors"
+    "github.com/sassoftware/go-rpmutils"
+    "go.resf.org/peridot/base/go/storage"
+    "golang.org/x/crypto/openpgp"
+    "io"
+    "os"
+    "path/filepath"
+    "strings"
+)
+
+type State struct {
+    tempDir string
+
+    // lookasideBlobs is a map of blob names to their SHA256 hashes.
+    lookasideBlobs map[string]string
+}
+
+// FromFile creates a new State from an SRPM file.
+// The SRPM file is extracted to a temporary directory.
+func FromFile(path string, keys ...*openpgp.Entity) (*State, error) {
+    f, err := os.Open(path)
+    if err != nil {
+        return nil, errors.Wrap(err, "failed to open file")
+    }
+    defer f.Close()
+
+    // If keys is not empty, then verify the RPM signature.
+    if len(keys) > 0 {
+        _, _, err := rpmutils.Verify(f, keys)
+        if err != nil {
+            return nil, errors.Wrap(err, "failed to verify RPM")
+        }
+
+        // After verifying the RPM, seek back to the start of the file.
+        _, err = f.Seek(0, io.SeekStart)
+        if err != nil {
+            return nil, errors.Wrap(err, "failed to seek to start of file")
+        }
+    }
+
+    rpm, err := rpmutils.ReadRpm(f)
+    if err != nil {
+        return nil, errors.Wrap(err, "failed to read RPM")
+    }
+
+    state := &State{
+        lookasideBlobs: make(map[string]string),
+    }
+    // Create a temporary directory.
+    state.tempDir, err = os.MkdirTemp("", "srpm_import-*")
+    if err != nil {
+        return nil, errors.Wrap(err, "failed to create temporary directory")
+    }
+
+    // Extract the SRPM.
+    err = rpm.ExpandPayload(state.tempDir)
+    if err != nil {
+        return nil, errors.Wrap(err, "failed to extract SRPM")
+    }
+
+    return state, nil
+}
+
+func (s *State) Close() error {
+    return os.RemoveAll(s.tempDir)
+}
+
+func (s *State) GetDir() string {
+    return s.tempDir
+}
+
+// determineLookasideBlobs determines which blobs need to be uploaded to the
+// lookaside cache.
+// Currently, the rule is that if a file is larger than 5MB, and is binary,
+// then it should be uploaded to the lookaside cache.
+func (s *State) determineLookasideBlobs() error {
+    // Read all files in tempDir, except for the SPEC file
+    // For each file, if it is larger than 5MB, and is binary, then add it to
+    // the lookasideBlobs map.
+    // If the file is not binary, then skip it.
+    ls, err := os.ReadDir(s.tempDir)
+    if err != nil {
+        return errors.Wrap(err, "failed to read directory")
+    }
+
+    for _, f := range ls {
+        // If file ends with ".spec", then skip it.
+        if f.IsDir() || strings.HasSuffix(f.Name(), ".spec") {
+            continue
+        }
+
+        // If file is larger than 5MB, then add it to the lookasideBlobs map.
+        info, err := f.Info()
+        if err != nil {
+            return errors.Wrap(err, "failed to get file info")
+        }
+
+        if info.Size() > 5*1024*1024 {
+            sum, err := func() (string, error) {
+                hash := sha256.New()
+                file, err := os.Open(filepath.Join(s.tempDir, f.Name()))
+                if err != nil {
+                    return "", errors.Wrap(err, "failed to open file")
+                }
+                defer file.Close()
+
+                _, err = io.Copy(hash, file)
+                if err != nil {
+                    return "", errors.Wrap(err, "failed to copy file")
+                }
+
+                return hex.EncodeToString(hash.Sum(nil)), nil
+            }()
+            if err != nil {
+                return err
+            }
+
+            s.lookasideBlobs[f.Name()] = sum
+        }
+    }
+
+    return nil
+}
+
+// uploadLookasideBlobs uploads all blobs in the lookasideBlobs map to the
+// lookaside cache.
+func (s *State) uploadLookasideBlobs(lookaside storage.Storage) error {
+    // The object name is the SHA256 hash of the file.
+    for path, hash := range s.lookasideBlobs {
+        _, err := lookaside.Put(hash, filepath.Join(s.tempDir, path))
+        if err != nil {
+            return errors.Wrap(err, "failed to upload file")
+        }
+    }
+
+    return nil
+}

tools/mothership/worker_server/srpm_import/srpm_import_test.go

@@ -0,0 +1,103 @@
+package srpm_import
+
+import (
+	"github.com/go-git/go-billy/v5/memfs"
+	"github.com/go-git/go-billy/v5/osfs"
+	"github.com/stretchr/testify/require"
+	storage_memory "go.resf.org/peridot/base/go/storage/memory"
+	"golang.org/x/crypto/openpgp"
+	"os"
+	"testing"
+)
+
+func TestFromFile(t *testing.T) {
+	s, err := FromFile("testdata/efi-rpm-macros-3-3.el8.src.rpm")
+	require.Nil(t, err)
+	require.NotNil(t, s)
+	require.Nil(t, s.Close())
+}
+
+func TestFromFile_SignatureOK(t *testing.T) {
+	keyF, err := os.Open("testdata/RPM-GPG-KEY-Rocky-8")
+	require.Nil(t, err)
+
+	testKey, err := openpgp.ReadArmoredKeyRing(keyF)
+	require.Nil(t, err)
+
+	s, err := FromFile("testdata/efi-rpm-macros-3-3.el8.src.rpm", testKey...)
+	require.Nil(t, err)
+	require.NotNil(t, s)
+	require.Nil(t, s.Close())
+}
+
+func TestFromFile_SignatureFail(t *testing.T) {
+	keyF, err := os.Open("testdata/RPM-GPG-KEY-Rocky-9")
+	require.Nil(t, err)
+
+	testKey, err := openpgp.ReadArmoredKeyRing(keyF)
+	require.Nil(t, err)
+
+	s, err := FromFile("testdata/efi-rpm-macros-3-3.el8.src.rpm", testKey...)
+	require.NotNil(t, err)
+	require.Nil(t, s)
+	require.Equal(t, "failed to verify RPM: keyid 15af5dac6d745a60 not found", err.Error())
+}
+
+func TestDetermineLookasideBlobs_Empty(t *testing.T) {
+	s, err := FromFile("testdata/efi-rpm-macros-3-3.el8.src.rpm")
+	require.Nil(t, err)
+	require.NotNil(t, s)
+	defer func() {
+		require.Nil(t, s.Close())
+	}()
+	require.Nil(t, s.determineLookasideBlobs())
+	require.Equal(t, 0, len(s.lookasideBlobs))
+}
+
+func TestDetermineLookasideBlobs_NotEmpty_Over5MB(t *testing.T) {
+	s, err := FromFile("testdata/bash-4.4.20-4.el8_6.src.rpm")
+	require.Nil(t, err)
+	require.NotNil(t, s)
+	defer func() {
+		require.Nil(t, s.Close())
+	}()
+	require.Nil(t, s.determineLookasideBlobs())
+	require.Equal(t, 1, len(s.lookasideBlobs))
+}
+
+func TestUploadLookaside_Empty(t *testing.T) {
+	s, err := FromFile("testdata/efi-rpm-macros-3-3.el8.src.rpm")
+	require.Nil(t, err)
+	require.NotNil(t, s)
+	defer func() {
+		require.Nil(t, s.Close())
+	}()
+	require.Nil(t, s.determineLookasideBlobs())
+
+	// we can use memfs since we're not actually writing anything
+	fs := memfs.New()
+	lookaside := storage_memory.New(fs)
+	require.Nil(t, s.uploadLookasideBlobs(lookaside))
+
+	fi, err := fs.ReadDir(".")
+	require.Nil(t, err)
+	require.Equal(t, 0, len(fi))
+}
+
+func TestUploadLookaside_NotEmpty(t *testing.T) {
+	s, err := FromFile("testdata/bash-4.4.20-4.el8_6.src.rpm")
+	require.Nil(t, err)
+	require.NotNil(t, s)
+	defer func() {
+		require.Nil(t, s.Close())
+	}()
+	require.Nil(t, s.determineLookasideBlobs())
+
+	fs := osfs.New("/")
+	lookaside := storage_memory.New(fs)
+	require.Nil(t, s.uploadLookasideBlobs(lookaside))
+
+	ok, err := lookaside.Exists("d86b3392c1202e8ff5a423b302e6284db7f8f435ea9f39b5b1b20fd3ac36dfcb")
+	require.Nil(t, err)
+	require.True(t, ok)
+}

tools/mothership/worker_server/srpm_import/testdata/RPM-GPG-KEY-Rocky-8

@@ -0,0 +1,29 @@
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+
+mQINBGAofzYBEAC6yS1azw6f3wmaVd//3aSy6O2c9+jeetulRQvg2LvhRRS1eNqp
+/x9tbBhfohu/tlDkGpYHV7diePgMml9SZDy1sKlI3tDhx6GZ3xwF0fd1vWBZpmNk
+D9gRkUmYBeLotmcXQZ8ZpWLicosFtDpJEYpLUhuIgTKwt4gxJrHvkWsGQiBkJxKD
+u3/RlL4IYA3Ot9iuCBflc91EyAw1Yj0gKcDzbOqjvlGtS3ASXgxPqSfU0uLC9USF
+uKDnP2tcnlKKGfj0u6VkqISliSuRAzjlKho9Meond+mMIFOTT6qp4xyu+9Dj3IjZ
+IC6rBXRU3xi8z0qYptoFZ6hx70NV5u+0XUzDMXdjQ5S859RYJKijiwmfMC7gZQAf
+OkdOcicNzen/TwD/slhiCDssHBNEe86Wwu5kmDoCri7GJlYOlWU42Xi0o1JkVltN
+D8ZId+EBDIms7ugSwGOVSxyZs43q2IAfFYCRtyKHFlgHBRe9/KTWPUrnsfKxGJgC
+Do3Yb63/IYTvfTJptVfhQtL1AhEAeF1I+buVoJRmBEyYKD9BdU4xQN39VrZKziO3
+hDIGng/eK6PaPhUdq6XqvmnsZ2h+KVbyoj4cTo2gKCB2XA7O2HLQsuGduHzYKNjf
+QR9j0djjwTrsvGvzfEzchP19723vYf7GdcLvqtPqzpxSX2FNARpCGXBw9wARAQAB
+tDNSZWxlYXNlIEVuZ2luZWVyaW5nIDxpbmZyYXN0cnVjdHVyZUByb2NreWxpbnV4
+Lm9yZz6JAk4EEwEIADgWIQRwUcRwqSn0VM6+N7cVr12sbXRaYAUCYCh/NgIbDwUL
+CQgHAgYVCgkICwIEFgIDAQIeAQIXgAAKCRAVr12sbXRaYLFmEACSMvoO1FDdyAbu
+1m6xEzDhs7FgnZeQNzLZECv2j+ggFSJXezlNVOZ5I1I8umBan2ywfKQD8M+IjmrW
+k9/7h9i54t8RS/RN7KNo7ECGnKXqXDPzBBTs1Gwo1WzltAoaDKUfXqQ4oJ4aCP/q
+/XPVWEzgpJO1XEezvCq8VXisutyDiXEjjMIeBczxb1hbamQX+jLTIQ1MDJ4Zo1YP
+zlUqrHW434XC2b1/WbSaylq8Wk9cksca5J+g3FqTlgiWozyy0uxygIRjb6iTzKXk
+V7SYxeXp3hNTuoUgiFkjh5/0yKWCwx7aQqlHar9GjpxmBDAO0kzOlgtTw//EqTwR
+KnYZLig9FW0PhwvZJUigr0cvs/XXTTb77z/i/dfHkrjVTTYenNyXogPtTtSyxqca
+61fbPf0B/S3N43PW8URXBRS0sykpX4SxKu+PwKCqf+OJ7hMEVAapqzTt1q9T7zyB
+QwvCVx8s7WWvXbs2d6ZUrArklgjHoHQcdxJKdhuRmD34AuXWCLW+gH8rJWZpuNl3
++WsPZX4PvjKDgMw6YMcV7zhWX6c0SevKtzt7WP3XoKDuPhK1PMGJQqQ7spegGB+5
+DZvsJS48Ip0S45Qfmj82ibXaCBJHTNZE8Zs+rdTjQ9DS5qvzRA1sRA1dBb/7OLYE
+JmeWf4VZyebm+gc50szsg6Ut2yT8hw==
+=AiP8
+-----END PGP PUBLIC KEY BLOCK-----

tools/mothership/worker_server/srpm_import/testdata/RPM-GPG-KEY-Rocky-9

@@ -0,0 +1,31 @@
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+Version: resf.keykeeper.v1
+Comment: Keykeeper
+
+xsFNBGJ5RksBEADF/Lzssm7uryV6+VHAgL36klyCVcHwvx9Bk853LBOuHVEZWsme
+kbJF3fQG7i7gfCKGuV5XW15xINToe4fBThZteGJziboSZRpkEQ2z3lYcbg34X7+d
+co833lkBNgz1v6QO7PmAdY/x76Q6Hx0J9yiJWd+4j+vRi4hbWuh64vUtTd7rPwk8
+0y3g4oK1YT0NR0Xm/QUO9vWmkSTVflQ6y82HhHIUrG+1vQnSOrWaC0O1lqUI3Nuo
+b6jTARCmbaPsi+XVQnBbsnPPq6Tblwc+NYJSqj5d9nT0uEXT7Zovj4Je5oWVFXp9
+P1OWkbo2z5XkKjoeobM/zKDESJR78h+YQAN9IOKFjL/u/Gzrk1oEgByCABXOX+H5
+hfucrq5U3bbcKy4e5tYgnnZxqpELv3fN/2l8iZknHEh5aYNT5WXVHpD/8u2rMmwm
+I9YTEMueEtmVy0ZV3opUzOlC+3ZUwjmvAJtdfJyeVW/VMy3Hw3Ih0Fij91rO613V
+7n72ggVlJiX25jYyT4AXlaGfAOMndJNVgBps0RArOBYsJRPnvfHlLi5cfjVd7vYx
+QhGX9ODYuvyJ/rW70dMVikeSjlBDKS08tvdqOgtiYy4yhtY4ijQC9BmCE9H9gOxU
+FN297iLimAxr0EVsED96fP96TbDGILWsfJuxAvoqmpkElv8J+P1/F7to2QARAQAB
+zU9Sb2NreSBFbnRlcnByaXNlIFNvZnR3YXJlIEZvdW5kYXRpb24gLSBSZWxlYXNl
+IGtleSAyMDIyIDxyZWxlbmdAcm9ja3lsaW51eC5vcmc+wsGKBBMBCAA0BQJieUZL
+FiEEIcslauFvxUxuZSlJcC1CbTUNJ10CGwMCHgECGQEDCwkHAhUIAxYAAgIiAQAK
+CRBwLUJtNQ0nXWQ5D/9472seOyRO6//bQ2ns3w9lE+aTLlJ5CY0GSTb4xNuyv+AD
+IXpgvLSMtTR0fp9GV3vMw6QIWsehDqt7O5xKWi+3tYdaXRpb1cvnh8r/oCcvI4uL
+k8kImNgsx+Cj+drKeQo03vFxBTDi1BTQFkfEt32fA2Aw5gYcGElM717sNMAMQFEH
+P+OW5hYDH4kcLbtUypPXFbcXUbaf6jUjfiEp5lLjqquzAyDPLlkzMr5RVa9n3/rI
+R6OQp5loPVzCRZMgDLALBU2TcFXLVP+6hAW8qM77c+q/rOysP+Yd+N7GAd0fvEvA
+mfeA4Y6dP0mMRu96EEAJ1qSKFWUul6K6nuqy+JTxktpw8F/IBAz44na17Tf02MJH
+GCUWyM0n5vuO5kK+Ykkkwd+v43ZlqDnwG7akDkLwgj6O0QNx2TGkdgt3+C6aHN5S
+MiF0pi0qYbiN9LO0e05Ai2r3zTFC/pCaBWlG1ph2jx1pDy4yUVPfswWFNfe5I+4i
+CMHPRFsZNYxQnIA2Prtgt2YMwz3VIGI6DT/Z56Joqw4eOfaJTTQSXCANts/gD7qW
+D3SZXPc7wQD63TpDEjJdqhmepaTECbxN7x/p+GwIZYWJN+AYhvrfGXfjud3eDu8/
+i+YIbPKH1TAOMwiyxC106mIL705p+ORf5zATZMyB8Y0OvRIz5aKkBDFZM2QN6A==
+=PzIf
+-----END PGP PUBLIC KEY BLOCK-----