diff --git a/pv2/util/constants.py b/pv2/util/constants.py index bf27166..5fe2996 100644 --- a/pv2/util/constants.py +++ b/pv2/util/constants.py @@ -98,6 +98,9 @@ class ErrorConstants: RPM_ERR_INFO = 9402 RPM_ERR_BUILD = 9403 + # Upload errors + UPLOAD_ERR = 9500 + # pylint: disable=too-few-public-methods class MockConstants: """ diff --git a/pv2/util/error.py b/pv2/util/error.py index 4d2c3b6..44a015b 100644 --- a/pv2/util/error.py +++ b/pv2/util/error.py @@ -32,6 +32,7 @@ __all__ = [ 'RpmSigError', 'RpmInfoError', 'RpmBuildError', + 'UploadError', ] @@ -180,3 +181,10 @@ class RpmBuildError(GenericError): There was an issue building or packing the RPM. """ fault_code = errconst.RPM_ERR_BUILD + +class UploadError(GenericError): + """ + There was an issue for uploading an artifact or the uploader is not + working. + """ + fault_code = errconst.UPLOAD_ERR diff --git a/pv2/util/uploader.py b/pv2/util/uploader.py new file mode 100644 index 0000000..e798403 --- /dev/null +++ b/pv2/util/uploader.py @@ -0,0 +1,75 @@ +# -*- mode:python; coding:utf-8; -*- +# Louis Abel +""" +Utility functions for uploading artifacts +""" +import os +import sys +import threading +import pv2.util.error as err +try: + import boto3 + s3 = boto3.client('s3') +except ImportError: + s3 = None + +__all__ = [ + 'S3ProgressPercentage', + 'upload_to_s3' +] + +class S3ProgressPercentage: + """ + Displays progress of uploads. Loosely borrowed from the aws documentation. + """ + def __init__(self, filename): + self.__filename = filename + self.__size = float(os.path.getsize(filename)) + self.__seen = 0 + self.__lock = threading.Lock() + + def __call__(self, num_of_bytes): + with self.__lock: + self.__seen += num_of_bytes + percentage = (self.__seen / self.__size) * 100 + sys.stdout.write( + "\r%s %s / %s (%.2f%%)" % (self.__filename, + self.__seen, + self.__size, + percentage) + ) + sys.stdout.flush() + +def upload_to_s3( + input_file, + bucket, + access_key_id: str, + access_key: str, + dest_name=None + ): + """ + Uploads an artifact to s3. + """ + if dest_name is None: + dest_name = os.path.basename(input_file) + + if s3 is None: + err.UploadError('s3 module is not available') + + s3_client = boto3.client( + 's3', + aws_access_key_id=access_key_id, + aws_secret_access_key=access_key + ) + + with open(input_file, 'rb') as inner: + s3_client.upload_fileobj( + inner, + bucket, + dest_name, + Callback=S3ProgressPercentage(input_file) + ) + inner.close() + + # Hacky way to get a new line + sys.stdout.write('\n') diff --git a/pyproject.toml b/pyproject.toml index 48e8710..580c327 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,7 +17,8 @@ dependencies = [ "GitPython >= 3.1.30", "lxml >= 4.6.5", "file-magic >= 0.4.0", - "pycurl >= 7.43.0.6" + "pycurl >= 7.43.0.6", + "boto3 >= 1.22.10" ] [project.urls]