Add memory:// to detector (storage)

base/go/flags.go

@@ -282,7 +282,15 @@ func ChangeDefaultForEnvVar(envVar EnvVar, newDefault string) {
 
 // RareUseChangeDefault changes the default value of an arbitrary environment variable.
 func RareUseChangeDefault(envVar string, newDefault string) {
-	ChangeDefaultForEnvVar(EnvVar(envVar), newDefault)
+	// Check if the environment variable is set.
+	if _, ok := os.LookupEnv(envVar); ok {
+		return
+	}
+
+	// Change the default value.
+	if err := os.Setenv(envVar, newDefault); err != nil {
+		LogFatalf("failed to set environment variable %s: %v", envVar, err)
+	}
 }
 
 // ChangeDefaultDatabaseURL changes the default value of the database url based on an environment variable.

base/go/storage/detector/BUILD

@@ -22,7 +22,9 @@ go_library(
     deps = [
         "//base/go",
         "//base/go/storage",
+        "//base/go/storage/memory",
         "//base/go/storage/s3",
+        "//vendor/github.com/go-git/go-billy/v5/osfs",
         "//vendor/github.com/pkg/errors",
         "//vendor/github.com/urfave/cli/v2:cli",
     ],

base/go/storage/detector/detector.go

@@ -15,16 +15,17 @@
 package storage_detector
 
 import (
+	"github.com/go-git/go-billy/v5/osfs"
 	"github.com/pkg/errors"
 	"github.com/urfave/cli/v2"
-	base "go.resf.org/peridot/base/go"
 	"go.resf.org/peridot/base/go/storage"
+	storage_memory "go.resf.org/peridot/base/go/storage/memory"
 	storage_s3 "go.resf.org/peridot/base/go/storage/s3"
 	"net/url"
 )
 
 func FromFlags(ctx *cli.Context) (storage.Storage, error) {
-	parsedURI, err := url.Parse(ctx.String(string(base.EnvVarStorageConnectionString)))
+	parsedURI, err := url.Parse(ctx.String("storage-connection-string"))
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to parse storage connection string")
 	}
@@ -32,6 +33,8 @@ func FromFlags(ctx *cli.Context) (storage.Storage, error) {
 	switch parsedURI.Scheme {
 	case "s3":
 		return storage_s3.FromFlags(ctx)
+	case "memory":
+		return storage_memory.New(osfs.New("/")), nil
 	default:
 		return nil, errors.Errorf("unknown storage scheme: %s", parsedURI.Scheme)
 	}

base/go/storage/s3/detector.go

@@ -24,28 +24,30 @@ import (
 
 func FromFlags(ctx *cli.Context) (*S3, error) {
 	// Parse the connection string
-	parsedURI, err := url.Parse(ctx.String(string(base.EnvVarStorageConnectionString)))
+	parsedURI, err := url.Parse(ctx.String("storage-connection-string"))
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to parse storage connection string")
 	}
 
 	// Retrieve the bucket name
-	bucket := parsedURI.Path
+	bucket := parsedURI.Host
 
 	// Remove the leading/trailing slashes
 	bucket = strings.TrimSuffix(strings.TrimPrefix(bucket, "/"), "/")
 
 	// Convert certain flags into environment variables so that they can be used by the AWS SDK
-	base.RareUseChangeDefault("AWS_REGION", ctx.String(string(base.EnvVarStorageRegion)))
-	base.RareUseChangeDefault("AWS_ENDPOINT", ctx.String(string(base.EnvVarStorageEndpoint)))
+	base.RareUseChangeDefault("AWS_REGION", ctx.String("storage-region"))
+	base.RareUseChangeDefault("AWS_ENDPOINT", ctx.String("storage-endpoint"))
 
-	if !ctx.Bool(string(base.EnvVarStorageSecure)) {
+	if !ctx.Bool("storage-secure") {
 		base.RareUseChangeDefault("AWS_DISABLE_SSL", "true")
 	}
 
-	if ctx.Bool(string(base.EnvVarStoragePathStyle)) {
+	if ctx.Bool("storage-path-style") {
 		base.RareUseChangeDefault("AWS_S3_FORCE_PATH_STYLE", "true")
 	}
 
+	base.LogInfof("Using S3 bucket: %s", bucket)
+
 	return New(bucket)
 }

base/go/storage/s3/s3.go

@@ -25,7 +25,6 @@ import (
 	"go.resf.org/peridot/base/go/storage"
 	"net/url"
 	"os"
-	"strings"
 )
 
 // S3 is an implementation of the Storage interface for S3.
@@ -178,22 +177,8 @@ func (s *S3) CanReadURI(uri string) (bool, error) {
 		return false, nil
 	}
 
-	// Split the path into bucket and object.
-	// The first element is the bucket, the rest is the object.
-	split := strings.SplitN(parsed.Path, "/", 2)
-
-	// Verify length.
-	if len(split) < 2 {
-		return false, nil
-	}
-
-	// Verify bucket.
-	if split[0] != s.bucket {
-		return false, nil
-	}
-
-	// Verify object.
-	if len(split[1]) == 0 {
+	// Verify that the host matches the bucket.
+	if parsed.Host != s.bucket {
 		return false, nil
 	}