add in neil's fixes for the req func

This commit is contained in:
Louis Abel 2023-05-11 20:47:49 -07:00
parent 82c7b60ef2
commit bd67c4d9fa
Signed by untrusted user: label
GPG Key ID: B37E62D143879B36

View File

@ -636,6 +636,7 @@ class Shared:
except: except:
logger.error('There was an issue downloading from %s' % s3_bucket) logger.error('There was an issue downloading from %s' % s3_bucket)
@staticmethod @staticmethod
def reqs_determine_latest(s3_bucket_url, release, arches, filetype, name, logger, page_size=1000): def reqs_determine_latest(s3_bucket_url, release, arches, filetype, name, logger, page_size=1000):
""" """
@ -645,25 +646,28 @@ class Shared:
data = {} data = {}
marker = None marker = None
# Hardcoding this for now until we can come up with a better solution
if 'lorax' in name:
prefix = "buildiso"
else:
prefix = "buildimage"
while True: while True:
params = {} params = {}
if marker is not None: if marker:
params['marker'] = marker params['marker'] = marker
params['delimiter'] = '/' params['prefix'] = f"{prefix}-{release.split('.')[0]}-"
params['max-keys'] = str(page_size) params['max-keys'] = str(page_size)
try: try:
bucket_data = requests.get( bucket_data = requests.get(s3_bucket_url, params=params, timeout=100)
s3_bucket_url, except requests.exceptions.RequestException as exception:
params=params,
timeout=100
)
except requests.exceptions.RequestException as e:
logger.error('The s3 bucket http endpoint is inaccessible') logger.error('The s3 bucket http endpoint is inaccessible')
raise SystemExit(e) raise SystemExit(exception) from exception
resp = xmltodict.parse(bucket_data.content) resp = xmltodict.parse(bucket_data.content)
if 'Contents' in resp['ListBucketResult'].keys():
for y in resp['ListBucketResult']['Contents']: for y in resp['ListBucketResult']['Contents']:
if y['Key'].endswith(filetype) and release in y['Key'] and name in y['Key']: if y['Key'].endswith(filetype) and release in y['Key'] and name in y['Key']:
temp.append(y['Key']) temp.append(y['Key'])
@ -677,6 +681,18 @@ class Shared:
if len(temps) > 0: if len(temps) > 0:
data[arch] = temps[0] data[arch] = temps[0]
truncated = resp['ListBucketResult'].get('IsTruncated')
# break from loop if there are no more results
if truncated == 'false':
break
# If truncated was true, we must set the marker for the next request to the last key of the current response
# ListObjects does not return NextMarker unless using Delimiter.. which is annoying
next_key = resp['ListBucketResult']['Contents'][-1].get('Key')
logger.info(Color.INFO + 'requesting another page starting with key: %s', next_key)
marker = next_key
return data return data
@staticmethod @staticmethod