rocky-linux/peridot
4
0
Fork 0
mirror of https://github.com/rocky-linux/peridot.git synced 2024-11-18 11:21:25 +00:00
Code Issues Projects Releases Activity

Buffering download for S3 (bypasses billyfs)

Browse Source 
Signed-off-by: Mustafa Gezen <mustafa@ctrliq.com>
This commit is contained in:
Mustafa Gezen 2022-08-16 12:52:26 +02:00
parent 0a712673a7
commit 4016569ce1
Signed by untrusted user who does not match committer: mustafa
GPG Key ID: DCDF010D946438C1
1 changed files with 17 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
peridot/lookaside/s3/s3.go
View File

@ -41,16 +41,16 @@ import (
"github.com/aws/aws-sdk-go/service/s3/s3manager" "github.com/aws/aws-sdk-go/service/s3/s3manager"
"github.com/go-git/go-billy/v5" "github.com/go-git/go-billy/v5"
"github.com/spf13/viper" "github.com/spf13/viper"
"io"
"io/ioutil" "io/ioutil"
"os" "os"
"peridot.resf.org/peridot/lookaside" "peridot.resf.org/peridot/lookaside"
) )
type Storage struct { type Storage struct {
bucket string bucket string
uploader *s3manager.Uploader uploader *s3manager.Uploader
fs billy.Filesystem downloader *s3manager.Downloader
fs billy.Filesystem
} }
func New(fs billy.Filesystem) (*Storage, error) { func New(fs billy.Filesystem) (*Storage, error) {
@ -81,32 +81,29 @@ func New(fs billy.Filesystem) (*Storage, error) {
return nil, err return nil, err
} }
uploader := s3manager.NewUploader(sess) uploader := s3manager.NewUploader(sess)
downloader := s3manager.NewDownloader(sess)
return &Storage{ return &Storage{
bucket: viper.GetString("s3-bucket"), bucket: viper.GetString("s3-bucket"),
uploader: uploader, uploader: uploader,
fs: fs, downloader: downloader,
fs: fs,
}, nil }, nil
} }
func (s *Storage) DownloadObject(objectName string, path string) error { func (s *Storage) DownloadObject(objectName string, path string) error {
obj, err := s.uploader.S3.GetObject(&s3.GetObjectInput{ f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
Bucket: aws.String(s.bucket),
Key: aws.String(objectName),
})
if err != nil { if err != nil {
return err return err
} }
f, err := s.fs.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644) _, err = s.downloader.Download(
if err != nil { f,
return err &s3.GetObjectInput{
} Bucket: aws.String(s.bucket),
Key: aws.String(objectName),
_, err = io.Copy(f, obj.Body) },
if err != nil { )
return err
}
return nil return nil
} }