kemotaha/toolkit
1
0
Fork 0
forked from sig_core/toolkit
Code Pull Requests Activity

try to fix s3 pagination

Browse Source
This commit is contained in:
Louis Abel 2022-11-18 21:33:10 -07:00
parent d6cc56c7c0
commit cab05e961f
Signed by untrusted user: label
GPG Key ID: B37E62D143879B36
2 changed files with 18 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
iso/empanadas/empanadas/util/iso_utils.py
View File

@ -365,6 +365,10 @@ class IsoBuild:
arch arch
) )
if arch not in latest_artifacts:
self.log.error(Color.FAIL + 'No lorax artifacts for ' + arch)
continue
source_path = latest_artifacts[arch] source_path = latest_artifacts[arch]
full_drop = '{}/lorax-{}-{}.tar.gz'.format( full_drop = '{}/lorax-{}-{}.tar.gz'.format(

16
iso/empanadas/empanadas/util/shared.py
View File

@ -566,7 +566,7 @@ class Shared:
s3 = boto3.client('s3') s3 = boto3.client('s3')
try: try:
s3.list_objects(Bucket=s3_bucket)['Contents'] res = s3.list_objects(Bucket=s3_bucket)['Contents']
except: except:
logger.error( logger.error(
'[' + Color.BOLD + Color.RED + 'FAIL' + Color.END + '] ' + '[' + Color.BOLD + Color.RED + 'FAIL' + Color.END + '] ' +
@ -574,9 +574,19 @@ class Shared:
) )
raise SystemExit() raise SystemExit()
for y in s3.list_objects(Bucket=s3_bucket)['Contents']: objs = res['Contents']
if filetype in y['Key'] and release in y['Key'] and name in y['Key']:
while True:
for y in objs:
key = y['Key']
if filetype in key and release in key and name in key:
temp.append(y['Key']) temp.append(y['Key'])
next_token = res.get("NextContinuationToken", None)
if next_token:
res = s3.list_objects_v2(Bucket=s3_bucket, ContinuationToken=next_token)
objs = res['Contents']
else:
break
for arch in arches: for arch in arches:
temps = [] temps = []