sig_core/toolkit
sig_core/toolkit
7
2
Fork 3
Code Issues 6 Pull Requests 2 Actions Projects Releases Wiki Activity

Tarball pull and extract

Browse Source
This commit is contained in:
Louis Abel 2022-06-19 22:52:20 -07:00
parent db55677a88
commit 72f98dcdb6
Signed by: label
GPG Key ID: B37E62D143879B36
5 changed files with 331 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
iso/empanadas/empanadas/scripts/pull_unpack_artifact.py Executable file
View File

@ -0,0 +1,35 @@
# builds ISO's
import argparse
from empanadas.common import *
from empanadas.util import Checks
from empanadas.util import IsoBuild
parser = argparse.ArgumentParser(description="ISO Artifact Builder")
parser.add_argument('--release', type=str, help="Major Release Version", required=True)
parser.add_argument('--s3', action='store_true', help="Release Candidate")
parser.add_argument('--arch', type=str, help="Architecture")
parser.add_argument('--local-compose', action='store_true', help="Compose Directory is Here")
parser.add_argument('--force-unpack', action='store_true', help="Force an unpack")
parser.add_argument('--force-download', action='store_true', help="Force a download")
parser.add_argument('--logger', type=str)
results = parser.parse_args()
rlvars = rldict[results.release]
major = rlvars['major']
a = IsoBuild(
rlvars,
config,
major=major,
s3=results.s3,
arch=results.arch,
force_unpack=results.force_unpack,
force_download=results.force_download,
compose_dir_is_here=results.local_compose,
logger=results.logger,
)
def run():
a.run_pull_lorax_artifacts()

10
iso/empanadas/empanadas/util/dnf_utils.py
View File

@ -452,7 +452,10 @@ class RepoSync:
join_all_pods = ' '.join(entry_name_list)
time.sleep(3)
self.log.info('Syncing %s ...' % r)
self.log.info(
'[' + Color.BOLD + Color.GREEN + 'INFO' + Color.END + '] ' +
'Syncing ' + r + ' ...'
)
pod_watcher = '{} wait {}'.format(
cmd,
join_all_pods
@ -500,7 +503,10 @@ class RepoSync:
)
entry_name_list.clear()
self.log.info('Syncing %s completed' % r)
self.log.info(
'[' + Color.BOLD + Color.GREEN + 'INFO' + Color.END + '] ' +
'Syncing ' + r + ' completed'
)
if len(bad_exit_list) > 0:
self.log.error(

241
iso/empanadas/empanadas/util/iso_utils.py
View File

@ -11,7 +11,14 @@ import os.path
import subprocess
import shlex
import time
import re
import tarfile
# lazy person's s3 parser
import requests
import json
import xmltodict
# if we can access s3
import boto3
# This is for treeinfo
from configparser import ConfigParser
@ -21,13 +28,6 @@ from productmd.extra_files import ExtraFiles
import productmd.treeinfo
# End treeinfo
# lazy person's s3 parser
import urllib
import json
import xmltodict
# if we can access s3
import boto3
from jinja2 import Environment, FileSystemLoader
from empanadas.common import Color, _rootdir
@ -49,6 +49,7 @@ class IsoBuild:
arch=None,
rc: bool = False,
s3: bool = False,
force_download: bool = False,
force_unpack: bool = False,
isolation: str = 'auto',
compose_dir_is_here: bool = False,
@ -77,6 +78,7 @@ class IsoBuild:
self.release_candidate = rc
self.s3 = s3
self.force_unpack = force_unpack
self.force_download = force_download
# Relevant major version items
self.arch = arch
@ -102,6 +104,9 @@ class IsoBuild:
self.s3_bucket = config['bucket']
self.s3_bucket_url = config['bucket_url']
if s3:
self.s3 = boto3.client('s3')
# Templates
file_loader = FileSystemLoader(f"{_rootdir}/templates")
self.tmplenv = Environment(loader=file_loader)
@ -302,21 +307,223 @@ class IsoBuild:
def run_pull_lorax_artifacts(self):
"""
Pulls the required artifacts and unacps it to work/lorax/$arch
Pulls the required artifacts and unpacks it to work/lorax/$arch
"""
self.log.info('Determining the latest pull...')
print()
# Determine if we're only managing one architecture out of all of them.
# It does not hurt to do everything at once. But the option is there.
unpack_single_arch = False
arches_to_unpack = self.arches
if self.arch:
unpack_single_arch = True
arches_to_unpack = [self.arch]
def _download_artifacts(self, force_unpack, arch=None):
"""
Download the requested artifact(s)
"""
print()
self.log.info(
'[' + Color.BOLD + Color.GREEN + 'INFO' + Color.END + '] ' +
'Determining the latest pulls...'
)
if self.s3:
latest_artifacts = self._s3_determine_latest()
else:
latest_artifacts = self._reqs_determine_latest()
def _unpack_artifacts(self, force_unpack, arch=None):
self.log.info(
'[' + Color.BOLD + Color.GREEN + 'INFO' + Color.END + '] ' +
'Downloading requested artifact(s)'
)
for arch in arches_to_unpack:
lorax_arch_dir = os.path.join(
self.lorax_work_dir,
arch
)
source_path = latest_artifacts[arch]
full_drop = '{}/lorax-{}-{}.tar.gz'.format(
lorax_arch_dir,
self.major_version,
arch
)
if not os.path.exists(lorax_arch_dir):
os.makedirs(lorax_arch_dir, exist_ok=True)
self.log.info(
'Downloading artifact for ' + Color.BOLD + arch + Color.END
)
if self.s3:
self._s3_download_artifacts(
self.force_download,
source_path,
full_drop
)
else:
self._reqs_download_artifacts(
self.force_download,
source_path,
full_drop
)
self.log.info(
'[' + Color.BOLD + Color.GREEN + 'INFO' + Color.END + '] ' +
'Download phase completed'
)
self.log.info(
'[' + Color.BOLD + Color.GREEN + 'INFO' + Color.END + '] ' +
'Beginning unpack phase...'
)
for arch in arches_to_unpack:
tarname = 'lorax-{}-{}.tar.gz'.format(
self.major_version,
arch
)
tarball = os.path.join(
self.lorax_work_dir,
arch,
tarname
)
if not os.path.exists(tarball):
self.log.error(
'[' + Color.BOLD + Color.RED + 'FAIL' + Color.END + '] ' +
'Artifact does not exist: ' + tarball
)
continue
self._unpack_artifacts(self.force_unpack, arch, tarball)
self.log.info(
'[' + Color.BOLD + Color.GREEN + 'INFO' + Color.END + '] ' +
'Unpack phase completed'
)
def _s3_determine_latest(self):
"""
Using native s3, determine the latest artifacts and return a list
"""
temp = []
data = {}
try:
self.s3.list_objects(Bucket=self.s3_bucket)['Contents']
except:
self.log.error(
'[' + Color.BOLD + Color.RED + 'FAIL' + Color.END + '] ' +
'Cannot access s3 bucket.'
)
raise SystemExit()
for y in self.s3.list_objects(Bucket=self.s3_bucket)['Contents']:
if 'tar.gz' in y['Key']:
temp.append(y['Key'])
for arch in self.arches:
temps = []
for y in temp:
if arch in y:
temps.append(y)
temps.sort(reverse=True)
data[arch] = temps[0]
return data
def _s3_download_artifacts(self, force_download, source, dest):
"""
Download the requested artifact(s) via s3
"""
if os.path.exists(dest):
if not force_download:
self.log.warn(
'[' + Color.BOLD + Color.YELLOW + 'WARN' + Color.END + '] ' +
'Artifact at ' + dest + ' already exists'
)
return
self.log.info('Downloading to: %s' % dest)
try:
self.s3.download_file(
Bucket=self.s3_bucket,
Key=source,
Filename=dest
)
except:
self.log.error('There was an issue downloading from %s' % self.s3_bucket)
def _reqs_determine_latest(self):
"""
Using requests, determine the latest artifacts and return a list
"""
temp = []
data = {}
try:
bucket_data = requests.get(self.s3_bucket_url)
except requests.exceptions.RequestException as e:
self.log.error('The s3 bucket http endpoint is inaccessible')
raise SystemExit(e)
resp = xmltodict.parse(bucket_data.content)
for y in resp['ListBucketResult']['Contents']:
if 'tar.gz' in y['Key']:
temp.append(y['Key'])
for arch in self.arches:
temps = []
for y in temp:
if arch in y:
temps.append(y)
temps.sort(reverse=True)
data[arch] = temps[0]
return data
def _reqs_download_artifacts(self, force_download, source, dest):
"""
Download the requested artifact(s) via requests only
"""
if os.path.exists(dest):
if not force_download:
self.log.warn(
'[' + Color.BOLD + Color.YELLOW + 'WARN' + Color.END + '] ' +
'Artifact at ' + dest + ' already exists'
)
return
unurl = self.s3_bucket_url + '/' + source
self.log.info('Downloading to: %s' % dest)
try:
with requests.get(unurl, allow_redirects=True) as r:
with open(dest, 'wb') as f:
f.write(r.content)
f.close()
r.close()
except requests.exceptions.RequestException as e:
self.log.error('There was a problem downloading the artifact')
raise SystemExit(e)
def _unpack_artifacts(self, force_unpack, arch, tarball):
"""
Unpack the requested artifacts(s)
"""
unpack_dir = os.path.join(self.lorax_work_dir, arch)
if not force_unpack:
file_check = os.path.join(unpack_dir, 'lorax/.treeinfo')
if os.path.exists(file_check):
self.log.warn(
'[' + Color.BOLD + Color.YELLOW + 'WARN' + Color.END + '] ' +
'Artifact (' + arch + ') already unpacked'
)
return
self.log.info('Unpacking %s' % tarball)
with tarfile.open(tarball) as t:
t.extractall(unpack_dir)
t.close()
def _copy_lorax_to_variant(self, force_unpack, arch):
"""
Copy to variants for easy access of mkiso and copying to compose dirs
"""
print()
def run_boot_sync(self):

63
iso/empanadas/poetry.lock generated
View File

@ -52,6 +52,25 @@ urllib3 = ">=1.25.4,<1.27"
[package.extras]
crt = ["awscrt (==0.13.8)"]
[[package]]
name = "certifi"
version = "2022.6.15"
description = "Python package for providing Mozilla's CA Bundle."
category = "main"
optional = false
python-versions = ">=3.6"
[[package]]
name = "charset-normalizer"
version = "2.0.12"
description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet."
category = "main"
optional = false
python-versions = ">=3.5.0"
[package.extras]
unicode_backport = ["unicodedata2"]
[[package]]
name = "colorama"
version = "0.4.5"
@ -60,6 +79,14 @@ category = "dev"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
[[package]]
name = "idna"
version = "3.3"
description = "Internationalized Domain Names in Applications (IDNA)"
category = "main"
optional = false
python-versions = ">=3.5"
[[package]]
name = "importlib-metadata"
version = "4.8.3"
@ -227,6 +254,24 @@ category = "main"
optional = false
python-versions = ">=3.6"
[[package]]
name = "requests"
version = "2.28.0"
description = "Python HTTP for Humans."
category = "main"
optional = false
python-versions = ">=3.7, <4"
[package.dependencies]
certifi = ">=2017.4.17"
charset-normalizer = ">=2.0.0,<2.1.0"
idna = ">=2.5,<4"
urllib3 = ">=1.21.1,<1.27"
[package.extras]
socks = ["PySocks (>=1.5.6,!=1.5.7)"]
use_chardet_on_py3 = ["chardet (>=3.0.2,<5)"]
[[package]]
name = "rpm-py-installer"
version = "1.1.0"
@ -309,7 +354,7 @@ testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytes
[metadata]
lock-version = "1.1"
python-versions = ">=3.7,<4"
content-hash = "93600aadcd1d588e33fc16d0fd7f505ee10484722c85bdadb612f57b10e9439b"
content-hash = "d011f4622c248f6aa107fd679616eaa19a897147398c6f52dd0dea0ab1d74486"
[metadata.files]
atomicwrites = [
@ -328,10 +373,22 @@ botocore = [
{file = "botocore-1.27.12-py3-none-any.whl", hash = "sha256:b8ac156e55267da6e728ea0b806bfcd97adf882801cffe7849c4b88ce4780326"},
{file = "botocore-1.27.12.tar.gz", hash = "sha256:17d3ec9f684d21e06b64d9cb224934557bcd95031e2ecb551bf16271e8722fec"},
]
certifi = [
{file = "certifi-2022.6.15-py3-none-any.whl", hash = "sha256:fe86415d55e84719d75f8b69414f6438ac3547d2078ab91b67e779ef69378412"},
{file = "certifi-2022.6.15.tar.gz", hash = "sha256:84c85a9078b11105f04f3036a9482ae10e4621616db313fe045dd24743a0820d"},
]
charset-normalizer = [
{file = "charset-normalizer-2.0.12.tar.gz", hash = "sha256:2857e29ff0d34db842cd7ca3230549d1a697f96ee6d3fb071cfa6c7393832597"},
{file = "charset_normalizer-2.0.12-py3-none-any.whl", hash = "sha256:6881edbebdb17b39b4eaaa821b438bf6eddffb4468cf344f09f89def34a8b1df"},
]
colorama = [
{file = "colorama-0.4.5-py2.py3-none-any.whl", hash = "sha256:854bf444933e37f5824ae7bfc1e98d5bce2ebe4160d46b5edf346a89358e99da"},
{file = "colorama-0.4.5.tar.gz", hash = "sha256:e6c6b4334fc50988a639d9b98aa429a0b57da6e17b9a44f0451f930b6967b7a4"},
]
idna = [
{file = "idna-3.3-py3-none-any.whl", hash = "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff"},
{file = "idna-3.3.tar.gz", hash = "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d"},
]
importlib-metadata = [
{file = "importlib_metadata-4.8.3-py3-none-any.whl", hash = "sha256:65a9576a5b2d58ca44d133c42a241905cc45e34d2c06fd5ba2bafa221e5d7b5e"},
{file = "importlib_metadata-4.8.3.tar.gz", hash = "sha256:766abffff765960fcc18003801f7044eb6755ffae4521c8e8ce8e83b9c9b0668"},
@ -486,6 +543,10 @@ pyyaml = [
{file = "PyYAML-6.0-cp39-cp39-win_amd64.whl", hash = "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c"},
{file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"},
]
requests = [
{file = "requests-2.28.0-py3-none-any.whl", hash = "sha256:bc7861137fbce630f17b03d3ad02ad0bf978c844f3536d0edda6499dafce2b6f"},
{file = "requests-2.28.0.tar.gz", hash = "sha256:d568723a7ebd25875d8d1eaf5dfa068cd2fc8194b2e483d7b1f7c81918dbec6b"},
]
rpm-py-installer = [
{file = "rpm-py-installer-1.1.0.tar.gz", hash = "sha256:66e5f4f9247752ed386345642683103afaee50fb16928878a204bc12504b9bbe"},
]

2
iso/empanadas/pyproject.toml
View File

@ -14,6 +14,7 @@ productmd = "~1.33"
importlib-resources = "^5.8.0"
boto3 = "^1.24.12"
xmltodict = "^0.13.0"
requests = "^2.28.0"
[tool.poetry.dev-dependencies]
pytest = "~5"
@ -23,6 +24,7 @@ sync_from_peridot = "empanadas.scripts.sync_from_peridot:run"
sync_from_peridot_test = "empanadas.scripts.sync_from_peridot_test:run"
sync_sig = "empanadas.scripts.sync_sig:run"
build-iso = "empanadas.scripts.build_iso:run"
pull-unpack-artifact = "empanadas.scripts.pull_unpack_artifact:run"
[build-system]
requires = ["poetry-core>=1.0.0"]