From 82c7b60ef2a95147fad771a95c0bed5549fc5e5e Mon Sep 17 00:00:00 2001 From: Louis Abel Date: Thu, 11 May 2023 18:16:59 -0700 Subject: [PATCH] attempt to fix tar extract and http reqs --- iso/empanadas/empanadas/util/shared.py | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/iso/empanadas/empanadas/util/shared.py b/iso/empanadas/empanadas/util/shared.py index 4245c9f..b9fe0fa 100644 --- a/iso/empanadas/empanadas/util/shared.py +++ b/iso/empanadas/empanadas/util/shared.py @@ -637,18 +637,30 @@ class Shared: logger.error('There was an issue downloading from %s' % s3_bucket) @staticmethod - def reqs_determine_latest(s3_bucket_url, release, arches, filetype, name, logger): + def reqs_determine_latest(s3_bucket_url, release, arches, filetype, name, logger, page_size=1000): """ Using requests, determine the latest artifacts and return a list """ temp = [] data = {} + marker = None - try: - bucket_data = requests.get(s3_bucket_url) - except requests.exceptions.RequestException as e: - logger.error('The s3 bucket http endpoint is inaccessible') - raise SystemExit(e) + while True: + params = {} + if marker is not None: + params['marker'] = marker + params['delimiter'] = '/' + params['max-keys'] = str(page_size) + + try: + bucket_data = requests.get( + s3_bucket_url, + params=params, + timeout=100 + ) + except requests.exceptions.RequestException as e: + logger.error('The s3 bucket http endpoint is inaccessible') + raise SystemExit(e) resp = xmltodict.parse(bucket_data.content) @@ -1143,7 +1155,7 @@ class Shared: if not Shared.tar_is_within_directory(path, member_path): raise Exception("Path traversal attempted in tar file") - tar.extractall(path, members, numeric_owner) + tar.extractall(path=path, members=members, numeric_owner=numeric_owner) @staticmethod def dnf_sync(repo, sync_root, work_root, arch, logger):