rocky-linux/peridot
4
0
Fork 0
mirror of https://github.com/rocky-linux/peridot.git synced 2024-11-18 03:11:24 +00:00
Code Issues Projects Releases Activity

Buffering download for S3 (bypasses billyfs)

Browse Source 
Signed-off-by: Mustafa Gezen <mustafa@ctrliq.com>
This commit is contained in:
Mustafa Gezen 2022-08-16 12:52:26 +02:00
parent 0a712673a7
commit 4016569ce1
Signed by untrusted user who does not match committer: mustafa
GPG Key ID: DCDF010D946438C1
1 changed files with 17 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
peridot/lookaside/s3/s3.go
View File

@ -41,16 +41,16 @@ import (
"github.com/aws/aws-sdk-go/service/s3/s3manager"
"github.com/go-git/go-billy/v5"
"github.com/spf13/viper"
"io"
"io/ioutil"
"os"
"peridot.resf.org/peridot/lookaside"
)
type Storage struct {
bucket string
uploader *s3manager.Uploader
fs billy.Filesystem
bucket string
uploader *s3manager.Uploader
downloader *s3manager.Downloader
fs billy.Filesystem
}
func New(fs billy.Filesystem) (*Storage, error) {
@ -81,32 +81,29 @@ func New(fs billy.Filesystem) (*Storage, error) {
return nil, err
}
uploader := s3manager.NewUploader(sess)
downloader := s3manager.NewDownloader(sess)
return &Storage{
bucket: viper.GetString("s3-bucket"),
uploader: uploader,
fs: fs,
bucket: viper.GetString("s3-bucket"),
uploader: uploader,
downloader: downloader,
fs: fs,
}, nil
}
func (s *Storage) DownloadObject(objectName string, path string) error {
obj, err := s.uploader.S3.GetObject(&s3.GetObjectInput{
Bucket: aws.String(s.bucket),
Key: aws.String(objectName),
})
f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
if err != nil {
return err
}
f, err := s.fs.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
if err != nil {
return err
}
_, err = io.Copy(f, obj.Body)
if err != nil {
return err
}
_, err = s.downloader.Download(
f,
&s3.GetObjectInput{
Bucket: aws.String(s.bucket),
Key: aws.String(objectName),
},
)
return nil
}