Make storage_memory fallback to FS and support a rootPath for Download

base/go/storage/memory/memory.go

@@ -20,27 +20,66 @@ import (
 	"go.resf.org/peridot/base/go/storage"
 	"io"
 	"os"
+	"path/filepath"
 	"strings"
 )
 
 type InMemory struct {
 	storage.Storage
 
-	fs    billy.Filesystem
-	blobs map[string][]byte
+	rootPath string
+	fs       billy.Filesystem
+	blobs    map[string][]byte
 }
 
-func New(fs billy.Filesystem) *InMemory {
-	return &InMemory{
+// New creates a new InMemory storage.
+// rootPath is a spread operator because it's optional and it looks nicer.
+func New(fs billy.Filesystem, rootPath ...string) *InMemory {
+	if len(rootPath) > 1 {
+		panic("too many arguments")
+	}
+
+	inm := &InMemory{
 		fs:    fs,
 		blobs: make(map[string][]byte),
 	}
+	if len(rootPath) == 1 {
+		inm.rootPath = rootPath[0]
+	}
+	return inm
+}
+
+func (im *InMemory) getBlob(object string) ([]byte, error) {
+	blob, ok := im.blobs[object]
+	if !ok {
+		// If not in memory, check if it's on disk
+		path := object
+		if im.rootPath != "" {
+			path = filepath.Join(im.rootPath, object)
+		}
+		f, err := im.fs.Open(path)
+		if err != nil {
+			return nil, storage.ErrNotFound
+		}
+
+		// Read file into blob
+		blob, err := io.ReadAll(f)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to read file")
+		}
+
+		// Store blob
+		im.blobs[object] = blob
+
+		return blob, nil
+	}
+	return blob, nil
 }
 
 func (im *InMemory) Download(object string, toPath string) error {
-	blob, ok := im.blobs[object]
-	if !ok {
-		return storage.ErrNotFound
+	blob, err := im.getBlob(object)
+	if err != nil {
+		return err
 	}
 
 	// Open file
@@ -59,9 +98,9 @@ func (im *InMemory) Download(object string, toPath string) error {
 }
 
 func (im *InMemory) Get(object string) ([]byte, error) {
-	blob, ok := im.blobs[object]
-	if !ok {
-		return nil, storage.ErrNotFound
+	blob, err := im.getBlob(object)
+	if err != nil {
+		return nil, err
 	}
 	return blob, nil
 }
@@ -104,8 +143,14 @@ func (im *InMemory) Delete(object string) error {
 }
 
 func (im *InMemory) Exists(object string) (bool, error) {
-	_, ok := im.blobs[object]
-	return ok, nil
+	_, err := im.getBlob(object)
+	if err != nil {
+		if err == storage.ErrNotFound {
+			return false, nil
+		}
+		return false, err
+	}
+	return true, nil
 }
 
 func (im *InMemory) CanReadURI(uri string) (bool, error) {

base/go/storage/memory/memory_test.go

@@ -18,6 +18,7 @@ import (
 	"github.com/go-git/go-billy/v5/memfs"
 	"github.com/stretchr/testify/require"
 	"go.resf.org/peridot/base/go/storage"
+	"os"
 	"testing"
 )
 
@@ -44,6 +45,31 @@ func TestInMemory_Download_Found(t *testing.T) {
 	require.Equal(t, []byte("bar"), buf)
 }
 
+func TestInMemory_Download_Found_OnFS(t *testing.T) {
+	fs := memfs.New()
+	{
+		f, _ := fs.OpenFile("foo", os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
+		_, err := f.Write([]byte("bar"))
+		require.Nil(t, err)
+		require.Nil(t, f.Close())
+	}
+	im := New(fs)
+
+	err := im.Download("foo", "foo")
+	require.Nil(t, err)
+
+	_, err = fs.Stat("foo")
+	require.Nil(t, err)
+
+	f, err := fs.Open("foo")
+	require.Nil(t, err)
+
+	buf := make([]byte, 3)
+	_, err = f.Read(buf)
+	require.Nil(t, err)
+	require.Equal(t, []byte("bar"), buf)
+}
+
 func TestInMemory_Download_NotFound(t *testing.T) {
 	fs := memfs.New()
 	im := New(fs)
@@ -60,6 +86,20 @@ func TestInMemory_Get_Found(t *testing.T) {
 	require.Equal(t, []byte("bar"), blob)
 }
 
+func TestInMemory_Get_Found_OnFS(t *testing.T) {
+	fs := memfs.New()
+	{
+		f, _ := fs.OpenFile("foo", os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
+		_, err := f.Write([]byte("bar"))
+		require.Nil(t, err)
+		require.Nil(t, f.Close())
+	}
+	im := New(fs)
+	blob, err := im.Get("foo")
+	require.Nil(t, err)
+	require.Equal(t, []byte("bar"), blob)
+}
+
 func TestInMemory_Get_NotFound(t *testing.T) {
 	fs := memfs.New()
 	im := New(fs)
@@ -120,6 +160,20 @@ func TestInMemory_Exists_Found(t *testing.T) {
 	require.True(t, ok)
 }
 
+func TestInMemory_Exists_Found_OnFS(t *testing.T) {
+	fs := memfs.New()
+	{
+		f, _ := fs.OpenFile("foo", os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
+		_, err := f.Write([]byte("bar"))
+		require.Nil(t, err)
+		require.Nil(t, f.Close())
+	}
+	im := New(fs)
+	ok, err := im.Exists("foo")
+	require.Nil(t, err)
+	require.True(t, ok)
+}
+
 func TestInMemory_Exists_NotFound(t *testing.T) {
 	fs := memfs.New()
 	im := New(fs)