Add more tests

tools/mothership/worker_server/BUILD

@@ -20,6 +20,7 @@ go_library(
         "entry.go",
         "process_rpm.go",
         "retract_entry.go",
+        "utils.go",
         "worker.go",
         "workflows.go",
     ],
@@ -58,7 +59,6 @@ go_test(
         "main_test.go",
         "process_rpm_test.go",
         "retract_entry_test.go",
-        "worker_test.go",
         "workflows_test.go",
     ],
     data = glob(["testdata/**"]),
@@ -72,7 +72,10 @@ go_test(
         "//tools/mothership/proto/v1:pb",
         "//tools/mothership/worker_server/forge",
         "//vendor/github.com/go-git/go-billy/v5/osfs",
+        "//vendor/github.com/go-git/go-git/v5:go-git",
+        "//vendor/github.com/go-git/go-git/v5/plumbing/cache",
         "//vendor/github.com/go-git/go-git/v5/plumbing/transport/http",
+        "//vendor/github.com/go-git/go-git/v5/storage/filesystem",
         "//vendor/github.com/stretchr/testify/mock",
         "//vendor/github.com/stretchr/testify/require",
         "//vendor/github.com/stretchr/testify/suite",

tools/mothership/worker_server/entry.go

@@ -26,7 +26,6 @@ import (
 	mothershippb "go.resf.org/peridot/tools/mothership/pb"
 	"go.temporal.io/sdk/temporal"
 	"io"
-	"net/url"
 	"os"
 	"path/filepath"
 	"time"
@@ -78,19 +77,9 @@ func (w *Worker) SetEntryIDFromRPM(entry string, uri string, checksumSha256 stri
 	}
 	defer os.RemoveAll(tempDir)
 
-	// Parse uri
+	object, err := getObjectPath(uri)
-	parsed, err := url.Parse(uri)
 	if err != nil {
-		return nil, errors.Wrap(err, "failed to parse resource URI")
+		return nil, err
-	}
-
-	// Download the resource to the temporary directory
-	// S3 for example must include bucket, while memory:// does not.
-	// So memory://test.rpm would be parsed as host=test.rpm, path="".
-	// While s3://mship/test.rpm would be parsed as host=mship, path=test.rpm.
-	object := parsed.Path
-	if object == "" {
-		object = parsed.Host
 	}
 
 	err = w.storage.Download(object, filepath.Join(tempDir, "resource.rpm"))

tools/mothership/worker_server/entry_test.go

@@ -125,7 +125,7 @@ func TestWorker_SetEntryState(t *testing.T) {
 
 	importRpmRes := &mothershippb.ImportRPMResponse{
 		CommitHash:   "123",
-		CommitUri:    "https://forge.resf.org/peridot/efi-rpm-macros/commit/123",
+		CommitUri:    "https://testforge.resf.org/peridot/efi-rpm-macros/commit/123",
 		CommitBranch: "el-8.8",
 		CommitTag:    "imports/el-8.8/efi-rpm-macros-3-3.el8",
 		Nevra:        "efi-rpm-macros-0:3-3.el8.aarch64",
@@ -136,7 +136,7 @@ func TestWorker_SetEntryState(t *testing.T) {
 	require.NotNil(t, entry)
 	require.Equal(t, mothershippb.Entry_ARCHIVED, entry.State)
 	require.Equal(t, "123", entry.CommitHash)
-	require.Equal(t, "https://forge.resf.org/peridot/efi-rpm-macros/commit/123", entry.CommitUri)
+	require.Equal(t, "https://testforge.resf.org/peridot/efi-rpm-macros/commit/123", entry.CommitUri)
 	require.Equal(t, "el-8.8", entry.CommitBranch)
 	require.Equal(t, "imports/el-8.8/efi-rpm-macros-3-3.el8", entry.CommitTag)
 	require.Equal(t, "efi-rpm-macros", entry.Pkg)

tools/mothership/worker_server/forge_test.go

@@ -15,20 +15,28 @@
 package mothership_worker_server
 
 import (
+	"errors"
 	"fmt"
+	"github.com/go-git/go-billy/v5/osfs"
+	"github.com/go-git/go-git/v5"
+	"github.com/go-git/go-git/v5/plumbing/cache"
 	transport_http "github.com/go-git/go-git/v5/plumbing/transport/http"
+	"github.com/go-git/go-git/v5/storage/filesystem"
 	"go.resf.org/peridot/tools/mothership/worker_server/forge"
+	"path/filepath"
 	"time"
 )
 
 type inMemoryForge struct {
-	localTempDir  string
+	localTempDir        string
-	repos         map[string]bool
+	repos               map[string]bool
-	remoteBaseURL string
+	remoteBaseURL       string
+	invalidUsernamePass bool
+	noAuthMethod        bool
 }
 
 func (f *inMemoryForge) GetAuthenticator() (*forge.Authenticator, error) {
-	return &forge.Authenticator{
+	ret := &forge.Authenticator{
 		AuthMethod: &transport_http.BasicAuth{
 			Username: "user",
 			Password: "pass",
@@ -36,7 +44,18 @@ func (f *inMemoryForge) GetAuthenticator() (*forge.Authenticator, error) {
 		AuthorName:  "Test User",
 		AuthorEmail: "test@resf.org",
 		Expires:     time.Now().Add(time.Hour),
-	}, nil
+	}
+
+	if f.noAuthMethod {
+		ret.AuthMethod = nil
+	} else if f.invalidUsernamePass {
+		ret.AuthMethod = &transport_http.BasicAuth{
+			Username: "invalid",
+			Password: "invalid",
+		}
+	}
+
+	return ret, nil
 }
 
 func (f *inMemoryForge) GetRemote(repo string) string {
@@ -48,6 +67,37 @@ func (f *inMemoryForge) GetCommitViewerURL(repo string, commit string) string {
 }
 
 func (f *inMemoryForge) EnsureRepositoryExists(auth *forge.Authenticator, repo string) error {
+	// Try casting auth.AuthMethod to *transport_http.BasicAuth
+	// If it fails, return an error
+	authx, ok := auth.AuthMethod.(*transport_http.BasicAuth)
+	if !ok {
+		return errors.New("auth failed")
+	}
+	if authx.Username != "user" || authx.Password != "pass" {
+		return errors.New("username or password incorrect")
+	}
+
+	if f.repos[repo] {
+		return nil
+	}
+
+	osfsTemp := osfs.New(filepath.Join(f.localTempDir, repo))
+	dot, err := osfsTemp.Chroot(".git")
+	if err != nil {
+		return err
+	}
+
+	filesystemTemp := filesystem.NewStorage(dot, cache.NewObjectLRUDefault())
+	err = filesystemTemp.Init()
+	if err != nil {
+		return err
+	}
+
+	_, err = git.Init(filesystemTemp, nil)
+	if err != nil {
+		return err
+	}
+
 	f.repos[repo] = true
 	return nil
 }

tools/mothership/worker_server/process_rpm.go

@@ -26,7 +26,6 @@ import (
 	"go.resf.org/peridot/tools/mothership/worker_server/srpm_import"
 	"go.temporal.io/sdk/temporal"
 	"io"
-	"net/url"
 	"os"
 	"path/filepath"
 	"strings"
@@ -48,24 +47,9 @@ func (w *Worker) VerifyResourceExists(uri string) error {
 		)
 	}
 
-	// Get object name from URI.
+	object, err := getObjectPath(uri)
-	// Check if object exists.
-	// If not, return error.
-	parsed, err := url.Parse(uri)
 	if err != nil {
-		return temporal.NewNonRetryableApplicationError(
+		return err
-			"could not parse resource URI",
-			"couldNotParseResourceURI",
-			errors.Wrap(err, "failed to parse resource URI"),
-		)
-	}
-
-	// S3 for example must include bucket, while memory:// does not.
-	// So memory://test.rpm would be parsed as host=test.rpm, path="".
-	// While s3://mship/test.rpm would be parsed as host=mship, path=test.rpm.
-	object := parsed.Path
-	if object == "" {
-		object = parsed.Host
 	}
 
 	exists, err := w.storage.Exists(object)
@@ -94,13 +78,13 @@ func (w *Worker) ImportRPM(uri string, checksumSha256 string, osRelease string)
 	defer os.RemoveAll(tempDir)
 
 	// Parse uri
-	parsed, err := url.Parse(uri)
+	object, err := getObjectPath(uri)
 	if err != nil {
-		return nil, errors.Wrap(err, "failed to parse resource URI")
+		return nil, err
 	}
 
 	// Download the resource to the temporary directory
-	err = w.storage.Download(parsed.Path, filepath.Join(tempDir, "resource.rpm"))
+	err = w.storage.Download(object, filepath.Join(tempDir, "resource.rpm"))
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to download resource")
 	}

tools/mothership/worker_server/process_rpm_test.go

@@ -34,3 +34,100 @@ func TestWorker_VerifyResourceExists_CannotRead(t *testing.T) {
 	require.NotNil(t, err)
 	require.Contains(t, err.Error(), "client submitted a resource URI that cannot be read by server")
 }
+
+func TestWorker_ImportRPM(t *testing.T) {
+	require.False(t, inmf.repos["efi-rpm-macros"])
+
+	res, err := testW.ImportRPM(
+		"memory://efi-rpm-macros-3-3.el8.src.rpm",
+		"518a9418fec1deaeb4c636615d8d81fb60146883c431ea15ab1127893d075d28",
+		"Rocky Linux release 8.8 (Green Obsidian)",
+	)
+	require.Nil(t, err)
+	require.NotNil(t, res)
+	require.Equal(t, "efi-rpm-macros", res.Pkg)
+	require.Equal(t, "efi-rpm-macros-0:3-3.el8.noarch.rpm", res.Nevra)
+
+	require.True(t, inmf.repos["efi-rpm-macros"])
+}
+
+func TestWorker_ImportRPM_Existing(t *testing.T) {
+	require.False(t, inmf.repos["basesystem"])
+
+	res, err := testW.ImportRPM(
+		"memory://basesystem-11-5.el8.src.rpm",
+		"6beff4cbfd5425e2c193312a9a184969a27d6bbd2d4cc29d7ce72dbe3d9f6416",
+		"Rocky Linux release 8.8 (Green Obsidian)",
+	)
+	require.Nil(t, err)
+	require.NotNil(t, res)
+
+	require.True(t, inmf.repos["basesystem"])
+
+	res, err = testW.ImportRPM(
+		"memory://basesystem-11-5.el8.src.rpm",
+		"6beff4cbfd5425e2c193312a9a184969a27d6bbd2d4cc29d7ce72dbe3d9f6416",
+		"Rocky Linux release 8.8 (Green Obsidian)",
+	)
+	require.Nil(t, err)
+	require.NotNil(t, res)
+
+	require.True(t, inmf.repos["basesystem"])
+
+	remote := testW.forge.GetRemote("basesystem")
+
+	repo, err := getRepo(remote)
+	require.Nil(t, err)
+
+	commitIter, err := repo.CommitObjects()
+	require.Nil(t, err)
+	c, err := commitIter.Next()
+	require.Nil(t, err)
+	require.NotNil(t, c)
+	require.Equal(t, "import basesystem-11-5.el8", c.Message)
+	c, err = commitIter.Next()
+	require.Nil(t, err)
+	require.NotNil(t, c)
+	require.Equal(t, "import basesystem-11-5.el8", c.Message)
+}
+
+func TestWorker_ImportRPM_ChecksumDoesntMatch(t *testing.T) {
+	res, err := testW.ImportRPM(
+		"memory://efi-rpm-macros-3-3.el8.src.rpm",
+		"518a9418fec1deaeb4c636615d8d81fb60146883c431ea15ab1127893d075d27",
+		"Rocky Linux release 8.8 (Green Obsidian)",
+	)
+	require.NotNil(t, err)
+	require.Nil(t, res)
+	require.Contains(t, err.Error(), "checksum does not match")
+}
+
+func TestWorker_ImportRPM_AuthError(t *testing.T) {
+	inmf.noAuthMethod = true
+
+	res, err := testW.ImportRPM(
+		"memory://efi-rpm-macros-3-3.el8.src.rpm",
+		"518a9418fec1deaeb4c636615d8d81fb60146883c431ea15ab1127893d075d28",
+		"Rocky Linux release 8.8 (Green Obsidian)",
+	)
+	require.NotNil(t, err)
+	require.Nil(t, res)
+	require.Contains(t, err.Error(), "auth failed")
+
+	inmf.noAuthMethod = false
+}
+
+func TestWorker_ImportRPM_InvalidCredentials(t *testing.T) {
+	inmf.invalidUsernamePass = true
+
+	res, err := testW.ImportRPM(
+		"memory://efi-rpm-macros-3-3.el8.src.rpm",
+		"518a9418fec1deaeb4c636615d8d81fb60146883c431ea15ab1127893d075d28",
+		"Rocky Linux release 8.8 (Green Obsidian)",
+	)
+	require.NotNil(t, err)
+	require.Nil(t, res)
+	require.Contains(t, err.Error(), "username or password incorrect")
+
+	inmf.invalidUsernamePass = false
+}

tools/mothership/worker_server/utils.go

@@ -0,0 +1,31 @@
+package mothership_worker_server
+
+import (
+	"github.com/pkg/errors"
+	"go.temporal.io/sdk/temporal"
+	"net/url"
+)
+
+func getObjectPath(uri string) (string, error) {
+	// Get object name from URI.
+	// Check if object exists.
+	// If not, return error.
+	parsed, err := url.Parse(uri)
+	if err != nil {
+		return "", temporal.NewNonRetryableApplicationError(
+			"could not parse resource URI",
+			"couldNotParseResourceURI",
+			errors.Wrap(err, "failed to parse resource URI"),
+		)
+	}
+
+	// S3 for example must include bucket, while memory:// does not.
+	// So memory://test.rpm would be parsed as host=test.rpm, path="".
+	// While s3://mship/test.rpm would be parsed as host=mship, path=test.rpm.
+	object := parsed.Path
+	if object == "" {
+		object = parsed.Host
+	}
+
+	return object, nil
+}