From cab05e961f63075b9fefcc53295b62db7ea6c143 Mon Sep 17 00:00:00 2001 From: Louis Abel Date: Fri, 18 Nov 2022 21:33:10 -0700 Subject: [PATCH] try to fix s3 pagination --- iso/empanadas/empanadas/util/iso_utils.py | 4 ++++ iso/empanadas/empanadas/util/shared.py | 18 ++++++++++++++---- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/iso/empanadas/empanadas/util/iso_utils.py b/iso/empanadas/empanadas/util/iso_utils.py index 87fd3e5..834d2c3 100644 --- a/iso/empanadas/empanadas/util/iso_utils.py +++ b/iso/empanadas/empanadas/util/iso_utils.py @@ -365,6 +365,10 @@ class IsoBuild: arch ) + if arch not in latest_artifacts: + self.log.error(Color.FAIL + 'No lorax artifacts for ' + arch) + continue + source_path = latest_artifacts[arch] full_drop = '{}/lorax-{}-{}.tar.gz'.format( diff --git a/iso/empanadas/empanadas/util/shared.py b/iso/empanadas/empanadas/util/shared.py index ed2d18c..0e99712 100644 --- a/iso/empanadas/empanadas/util/shared.py +++ b/iso/empanadas/empanadas/util/shared.py @@ -566,7 +566,7 @@ class Shared: s3 = boto3.client('s3') try: - s3.list_objects(Bucket=s3_bucket)['Contents'] + res = s3.list_objects(Bucket=s3_bucket)['Contents'] except: logger.error( '[' + Color.BOLD + Color.RED + 'FAIL' + Color.END + '] ' + @@ -574,9 +574,19 @@ class Shared: ) raise SystemExit() - for y in s3.list_objects(Bucket=s3_bucket)['Contents']: - if filetype in y['Key'] and release in y['Key'] and name in y['Key']: - temp.append(y['Key']) + objs = res['Contents'] + + while True: + for y in objs: + key = y['Key'] + if filetype in key and release in key and name in key: + temp.append(y['Key']) + next_token = res.get("NextContinuationToken", None) + if next_token: + res = s3.list_objects_v2(Bucket=s3_bucket, ContinuationToken=next_token) + objs = res['Contents'] + else: + break for arch in arches: temps = []