sig_core
/
toolkit
7
2
Fork 1
Code Issues 5 Pull Requests Actions Projects Releases Wiki Activity

add in live generation

This commit is contained in:
Louis Abel 2022-07-11 01:06:26 -07:00
parent 1f94680924
commit 9568b0cbcb
Signed by: label
GPG Key ID: B37E62D143879B36
13 changed files with 435 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
iso/empanadas/empanadas/configs/el8.yaml
View File

@ -105,6 +105,10 @@
- 'vim-minimal'
- 'pykickstart'
- 'git'
variantmap:
git_repo: 'https://git.rockylinux.org/rocky/pungi-rocky.git'
branch: 'r8'
git_raw_path: 'https://git.rockylinux.org/rocky/pungi-rocky/-/raw/r8/'
repoclosure_map:
arches:
x86_64: '--forcearch=x86_64 --arch=x86_64 --arch=athlon --arch=i686 --arch=i586 --arch=i486 --arch=i386 --arch=noarch'

4
iso/empanadas/empanadas/configs/el9-beta.yaml
View File

@ -93,6 +93,10 @@
- 'vim-minimal'
- 'pykickstart'
- 'git'
variantmap:
git_repo: 'https://git.rockylinux.org/rocky/pungi-rocky.git'
branch: 'r9-beta'
git_raw_path: 'https://git.rockylinux.org/rocky/pungi-rocky/-/raw/r9-beta/'
repoclosure_map:
arches:
x86_64: '--forcearch=x86_64 --arch=x86_64 --arch=athlon --arch=i686 --arch=i586 --arch=i486 --arch=i386 --arch=noarch'

4
iso/empanadas/empanadas/configs/el9.yaml
View File

@ -93,6 +93,10 @@
- 'vim-minimal'
- 'pykickstart'
- 'git'
variantmap:
git_repo: 'https://git.rockylinux.org/rocky/pungi-rocky.git'
branch: 'r9'
git_raw_path: 'https://git.rockylinux.org/rocky/pungi-rocky/-/raw/r9/'
repoclosure_map:
arches:
x86_64: '--forcearch=x86_64 --arch=x86_64 --arch=athlon --arch=i686 --arch=i586 --arch=i486 --arch=i386 --arch=noarch'

4
iso/empanadas/empanadas/configs/el9lh.yaml
View File

@ -93,6 +93,10 @@
- 'vim-minimal'
- 'pykickstart'
- 'git'
variantmap:
git_repo: 'https://git.rockylinux.org/rocky/pungi-rocky.git'
branch: 'r9lh'
git_raw_path: 'https://git.rockylinux.org/rocky/pungi-rocky/-/raw/r9lh/'
repoclosure_map:
arches:
x86_64: '--forcearch=x86_64 --arch=x86_64 --arch=athlon --arch=i686 --arch=i586 --arch=i486 --arch=i386 --arch=noarch'

2
iso/empanadas/empanadas/scripts/build_iso.py
View File

@ -13,6 +13,7 @@ parser.add_argument('--isolation', type=str, help="mock isolation mode")
parser.add_argument('--rc', action='store_true', help="Release Candidate, Beta, RLN")
parser.add_argument('--local-compose', action='store_true', help="Compose Directory is Here")
parser.add_argument('--logger', type=str)
parser.add_argument('--hashed', action='store_true')
results = parser.parse_args()
rlvars = rldict[results.release]
major = rlvars['major']
@ -24,6 +25,7 @@ a = IsoBuild(
rc=results.rc,
isolation=results.isolation,
compose_dir_is_here=results.local_compose,
hashed=results.hashed,
logger=results.logger,
)

2
iso/empanadas/empanadas/scripts/build_iso_extra.py
View File

@ -16,6 +16,7 @@ parser.add_argument('--local-compose', action='store_true', help="Compose Direct
parser.add_argument('--logger', type=str)
parser.add_argument('--extra-iso', type=str, help="Granular choice in which iso is built")
parser.add_argument('--extra-iso-mode', type=str, default='local')
parser.add_argument('--hashed', action='store_true')
results = parser.parse_args()
rlvars = rldict[results.release]
major = rlvars['major']
@ -30,6 +31,7 @@ a = IsoBuild(
extra_iso=results.extra_iso,
extra_iso_mode=results.extra_iso_mode,
compose_dir_is_here=results.local_compose,
hashed=results.hashed,
logger=results.logger
)

2
iso/empanadas/empanadas/scripts/build_iso_live.py
View File

@ -14,6 +14,7 @@ parser.add_argument('--local-compose', action='store_true', help="Compose Direct
parser.add_argument('--image', type=str, help="Granular choice in which live image is built")
parser.add_argument('--logger', type=str)
parser.add_argument('--live-iso-mode', type=str, default='local')
parser.add_argument('--hashed', action='store_true')
results = parser.parse_args()
rlvars = rldict[results.release]
major = rlvars['major']
@ -26,6 +27,7 @@ a = LiveBuild(
live_iso_mode=results.live_iso_mode,
image=results.image,
compose_dir_is_here=results.local_compose,
hashed=results.hashed,
logger=results.logger
)

2
iso/empanadas/empanadas/scripts/launch_builds.py
View File

@ -31,7 +31,7 @@ def run():
elif results.env == "all":
arches = EKSARCH+EXTARCH
command = ["build-iso", "--release", f"{results.release}", "--isolation", "simple"]
command = ["build-iso", "--release", f"{results.release}", "--isolation", "simple", "--hashed"]
if results.rc:
command += ["--rc"]

1
iso/empanadas/empanadas/scripts/sync_from_peridot_test.py
View File

@ -18,3 +18,4 @@ def run():
print(rlvars.keys())
print(rlvars)
print(empanadas.__version__)
print(a.hashed)

21
iso/empanadas/empanadas/templates/buildLiveImage.tmpl.sh
View File

@ -6,10 +6,31 @@ set -ex
mkdir -p {{ compose_live_work_dir }}/{{ arch }}
cd {{ compose_live_work_dir }}/{{ arch }}
test -f {{ isoname }} && { echo "ERROR: ISO ALREDY EXISTS!"; exit 1; }
major=$(grep loop /proc/devices | cut -c3)
for index in 0 1 2 3 4 5; do
mknod /dev/loop$index $major $index
done
{% else %}
cd /builddir
{% endif %}
{{ git_clone }}
if [ -d "/builddir/ks/live/{{ major }}/peridot" ]; then
pushd /builddir/ks/live/{{ major }}/peridot || { echo "Could not change directory"; exit 1; }
else
pushd /builddir/ks/live/{{ major }}/staging || { echo "Could not change directory"; exit 1; }
fi
ksflatten -c {{ ks_file }} -o /builddir/ks.cfg
if [ $? -ne 0 ]; then
echo "Error flattening kickstart"
exit 1
fi
popd || { echo "Could not leave directory"; exit 1; }
{{ make_image }}
{% if live_iso_mode == "podman" %}
cp /builddir/lmc/{{ isoname }} {{ compose_live_work_dir }}/{{ arch }}/{{ isoname }}
{% endif %}

58
iso/empanadas/empanadas/templates/liveisobuild.tmpl.sh Normal file
View File

@ -0,0 +1,58 @@
#!/bin/bash
# This is a template that is used to build extra ISO's for Rocky Linux. Only
# under extreme circumstances should you be filling this out and running
# manually.
set -o pipefail
# Vars
MOCK_CFG="/var/tmp/live-{{ major }}.cfg"
MOCK_ROOT="/var/lib/mock/{{ shortname|lower }}-{{ major }}-{{ arch }}"
MOCK_RESL="${MOCK_ROOT}/result"
MOCK_CHRO="${MOCK_ROOT}/root"
MOCK_LOG="${MOCK_RESL}/mock-output.log"
IMAGE_SCR="{{ entries_dir }}/buildLiveImage-{{ arch }}-{{ image }}.sh"
IMAGE_ISO="{{ isoname }}"
ISOLATION="{{ isolation }}"
BUILDDIR="{{ builddir }}"
if [ -f "/usr/sbin/setenforce" ]; then
sudo setenforce 0
fi
# Init the container
mock \
-r "${MOCK_CFG}" \
--isolation="${ISOLATION}" \
--enable-network \
--init
init_ret_val=$?
if [ $init_ret_val -ne 0 ]; then
echo "!! MOCK INIT FAILED !!"
exit 1
fi
mkdir -p "${MOCK_RESL}"
cp "${IMAGE_SCR}" "${MOCK_CHRO}${IMAGE_SCR}"
mock \
-r "${MOCK_CFG}" \
--shell \
--isolation="${ISOLATION}" \
--enable-network -- /bin/bash "${IMAGE_SCR}" | tee -a "${MOCK_LOG}"
mock_ret_val=$?
if [ $mock_ret_val -eq 0 ]; then
# Copy resulting data to /var/lib/mock/{{ shortname|lower }}-{{ major }}-{{ arch }}/result
mkdir -p "${MOCK_RESL}"
cp "${MOCK_CHRO}${BUILDDIR}/lmc/${IMAGE_ISO}" "${MOCK_RESL}"
else
echo "!! EXTRA ISO RUN FAILED !!"
exit 1
fi
# Clean up?
if [ -f "/usr/sbin/setenforce" ]; then
sudo setenforce 1
fi

325
iso/empanadas/empanadas/util/iso_utils.py
View File

@ -60,6 +60,7 @@ class IsoBuild:
extra_iso=None,
extra_iso_mode: str = 'local',
compose_dir_is_here: bool = False,
hashed: bool = False,
image=None,
logger=None
):
@ -91,6 +92,7 @@ class IsoBuild:
self.extra_iso_mode = extra_iso_mode
self.checksum = rlvars['checksum']
self.profile = rlvars['profile']
self.hashed = hashed
# Relevant major version items
self.arch = arch
@ -185,7 +187,15 @@ class IsoBuild:
self.log.addHandler(handler)
self.log.info('iso build init')
self.repolist = self.build_repo_list()
self.repolist = Shared.build_repo_list(
self.repo_base_url,
self.repos,
self.project_id,
self.current_arch,
self.compose_latest_sync,
self.compose_dir_is_here,
self.hashed
)
self.log.info(self.revision)
def run(self):
@ -208,36 +218,6 @@ class IsoBuild:
)
self.log.info('ISO Build completed.')
def build_repo_list(self):
"""
Builds the repo dictionary
"""
repolist = []
for name in self.repos:
if not self.compose_dir_is_here:
constructed_url = '{}/{}/repo/hashed-{}/{}'.format(
self.repo_base_url,
self.project_id,
name,
self.current_arch
)
else:
constructed_url = 'file://{}/{}/{}/os'.format(
self.compose_latest_sync,
name,
self.current_arch
)
repodata = {
'name': name,
'url': constructed_url
}
repolist.append(repodata)
return repolist
def iso_build(self):
"""
This does the general ISO building for the current running
@ -761,7 +741,7 @@ class IsoBuild:
elif self.extra_iso_mode == 'podman':
continue
else:
self.log.info(Color.FAIL + 'Mode specified is not valid.')
self.log.error(Color.FAIL + 'Mode specified is not valid.')
raise SystemExit()
if self.extra_iso_mode == 'podman':
@ -1409,6 +1389,22 @@ class IsoBuild:
c.write(checksum)
c.close()
self.log.info('Creating a symlink to latest image...')
latest_name = '{}/{}-{}-{}.latest.{}.{}'.format(
image_arch_dir,
self.shortname,
self.major_version,
imagename,
arch,
formattype
)
# For some reason python doesn't have a "yeah just change this
# link" part of the function
if os.path.exists(latest_name):
os.remove(latest_name)
os.symlink(drop_name, latest_name)
self.log.info(Color.INFO + 'Image download phase completed')
@ -1427,6 +1423,7 @@ class LiveBuild:
isolation: str = 'simple',
live_iso_mode: str = 'local',
compose_dir_is_here: bool = False,
hashed: bool = False,
image=None,
logger=None
):
@ -1440,7 +1437,7 @@ class LiveBuild:
self.major_version = major
self.compose_dir_is_here = compose_dir_is_here
self.date_stamp = config['date_stamp']
self.timestamp = time.strftime("%Y%m%d", time.localtime())
self.date = time.strftime("%Y%m%d", time.localtime())
self.compose_root = config['compose_root']
self.compose_base = config['compose_root'] + "/" + major
self.current_arch = config['arch']
@ -1453,6 +1450,7 @@ class LiveBuild:
self.live_iso_mode = live_iso_mode
self.checksum = rlvars['checksum']
self.profile = rlvars['profile']
self.hashed = hashed
# Relevant major version items
self.arch = config['arch']
@ -1461,6 +1459,7 @@ class LiveBuild:
self.minor_version = rlvars['minor']
self.revision = rlvars['revision'] + "-" + rlvars['rclvl']
self.rclvl = rlvars['rclvl']
self.disttag = config['dist']
self.repos = rlvars['iso_map']['lorax']['repos']
self.repo_base_url = config['repo_base_url']
self.project_id = rlvars['project_id']
@ -1513,6 +1512,15 @@ class LiveBuild:
self.log.addHandler(handler)
self.log.info('live build init')
self.repolist = Shared.build_repo_list(
self.repo_base_url,
self.repos,
self.project_id,
self.current_arch,
self.compose_latest_sync,
self.compose_dir_is_here,
self.hashed
)
self.log.info(self.revision)
def run_build_live_iso(self):
@ -1555,6 +1563,20 @@ class LiveBuild:
', '.join(images_to_build)
)
for i in images_to_build:
self._live_iso_local_config(i, work_root)
if self.live_iso_mode == 'local':
self._live_iso_local_run(self.current_arch, i, work_root)
elif self.live_iso_mode == 'podman':
continue
else:
self.log.error(Color.FAIL + 'Mode specified is not valid.')
raise SystemExit()
if self.live_iso_mode == 'podman':
self._live_iso_podman_run(self.current_arch, images_to_build, work_root)
def _live_iso_local_config(self, image, work_root):
"""
Live ISO build configuration - This generates both mock and podman
@ -1562,6 +1584,128 @@ class LiveBuild:
"""
self.log.info('Generating Live ISO configuration and script')
entries_dir = os.path.join(work_root, "entries")
mock_iso_template = self.tmplenv.get_template('isomock.tmpl.cfg')
mock_sh_template = self.tmplenv.get_template('liveisobuild.tmpl.sh')
iso_template = self.tmplenv.get_template('buildLiveImage.tmpl.sh')
mock_iso_path = '/var/tmp/live-{}.cfg'.format(self.major_version)
mock_sh_path = '{}/liveisobuild-{}-{}.sh'.format(
entries_dir,
self.current_arch,
image
)
iso_template_path = '{}/buildLiveImage-{}-{}.sh'.format(
entries_dir,
self.current_arch,
image
)
log_root = os.path.join(
work_root,
"logs",
self.date_stamp
)
ks_start = self.livemap['ksentry'][image]
if not os.path.exists(log_root):
os.makedirs(log_root, exist_ok=True)
log_path_command = '| tee -a {}/{}-{}.log'.format(
log_root,
self.current_arch,
image
)
required_pkgs = self.livemap['required_pkgs']
volid = '{}-{}-{}'.format(
self.shortname,
image,
self.release
)
isoname = '{}-{}-{}-{}-{}.iso'.format(
self.shortname,
image,
self.release,
self.current_arch,
self.date
)
live_pkg_cmd = '/usr/bin/dnf install {} -y {}'.format(
' '.join(required_pkgs),
log_path_command
)
git_clone_cmd = '/usr/bin/git clone {} -b {} /builddir/ks {}'.format(
self.livemap['git_repo'],
self.livemap['branch'],
log_path_command
)
make_image_cmd = ('/usr/sbin/livemedia-creator --ks {} --no-virt '
'--resultdir /builddir/lmc --project="{}" --make-iso --volid {} '
'--iso-only --iso-name {} --releasever={} --nomacboot {}').format(
'/builddir/ks.cfg',
self.distname,
volid,
isoname,
self.release,
log_path_command
)
mock_iso_template_output = mock_iso_template.render(
arch=self.current_arch,
major=self.major_version,
fullname=self.fullname,
shortname=self.shortname,
required_pkgs=required_pkgs,
dist=self.disttag,
repos=self.repolist,
compose_dir_is_here=True,
user_agent='{{ user_agent }}',
compose_dir=self.compose_root,
)
mock_sh_template_output = mock_sh_template.render(
arch=self.current_arch,
major=self.major_version,
isolation=self.mock_isolation,
builddir=self.mock_work_root,
shortname=self.shortname,
isoname=isoname,
entries_dir=entries_dir,
image=image,
)
iso_template_output = iso_template.render(
live_iso_mode=self.live_iso_mode,
arch=self.current_arch,
compose_live_work_dir=self.live_work_dir,
make_image=make_image_cmd,
live_pkg_cmd=live_pkg_cmd,
isoname=isoname,
major=self.major_version,
git_clone=git_clone_cmd,
ks_file=ks_start,
)
with open(mock_iso_path, "w+") as mip:
mip.write(mock_iso_template_output)
mip.close()
with open(mock_sh_path, "w+") as msp:
msp.write(mock_sh_template_output)
msp.close()
with open(iso_template_path, "w+") as itp:
itp.write(iso_template_output)
itp.close()
os.chmod(mock_sh_path, 0o755)
os.chmod(iso_template_path, 0o755)
def _live_iso_podman_run(self, arch, images, work_root):
"""
Does all the image building in podman containers to parallelize the
@ -1578,9 +1722,112 @@ class LiveBuild:
isos_dir = self.live_work_dir
bad_exit_list = []
checksum_list = []
entry_name_list = []
for i in images:
entry_name_list = []
image_name = i
entry_name = 'buildLiveImage-{}-{}.sh'.format(arch, i)
entry_name_list.append(entry_name)
isoname = '{}/{}-{}-{}-{}-{}.iso'.format(
arch,
self.shortname,
i,
self.major_version,
arch,
self.date
)
checksum_list.append(isoname)
print(entry_name_list, cmd, entries_dir)
for pod in entry_name_list:
podman_cmd_entry = '{} run -d -it -v "{}:{}" -v "{}:{}" --name {} --entrypoint {}/{} {}'.format(
cmd,
self.compose_root,
self.compose_root,
entries_dir,
entries_dir,
pod,
entries_dir,
pod,
self.container
)
process = subprocess.call(
shlex.split(podman_cmd_entry),
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL
)
join_all_pods = ' '.join(entry_name_list)
time.sleep(3)
self.log.info(Color.INFO + 'Building requested live images ...')
pod_watcher = '{} wait {}'.format(
cmd,
join_all_pods
)
watch_man = subprocess.call(
shlex.split(pod_watcher),
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL
)
# After the above is done, we'll check each pod process for an exit
# code.
pattern = "Exited (0)"
for pod in entry_name_list:
checkcmd = '{} ps -f status=exited -f name={}'.format(
cmd,
pod
)
podcheck = subprocess.Popen(
checkcmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True
)
output, errors = podcheck.communicate()
if 'Exited (0)' not in output.decode():
self.log.error(Color.FAIL + pod)
bad_exit_list.append(pod)
rmcmd = '{} rm {}'.format(
cmd,
join_all_pods
)
rmpod = subprocess.Popen(
rmcmd,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
shell=True
)
entry_name_list.clear()
for p in checksum_list:
path = os.path.join(isos_dir, p)
if os.path.exists(path):
self.log.info(Color.INFO + 'Performing checksum for ' + p)
checksum = Shared.get_checksum(path, self.checksum, self.log)
if not checksum:
self.log.error(Color.FAIL + path + ' not found! Are you sure it was built?')
with open(path + '.CHECKSUM', "w+") as c:
c.write(checksum)
c.close()
self.log.info(Color.INFO + 'Building live images completed')
if len(bad_exit_list) == 0:
self.log.info(Color.INFO + 'Copying ISOs over to compose directory...')
else:
self.log.error(
Color.FAIL +
'There were issues with the work done. As a result, ' +
'the ISOs will not be copied.'
)
def _live_iso_local_run(self, arch, image, work_root):
"""
@ -1598,7 +1845,11 @@ class LiveBuild:
raise SystemExit()
self.log.warn(
Color.WARN +
'If you are looping images, your built image may get overwritten.'
Color.WARN + 'This is meant for builds done in peridot or ' +
'locally for an end user.'
)
self.log.warn(
Color.WARN +
'If you are looping images, your built image WILL get ' +
'overwritten.'
)

44
iso/empanadas/empanadas/util/shared.py
View File

@ -10,6 +10,7 @@ import requests
import boto3
import xmltodict
import productmd.treeinfo
import productmd.composeinfo
import empanadas
import kobo.shortcuts
from empanadas.common import Color
@ -871,3 +872,46 @@ class Shared:
shlex.quote(opts['iso_name']),
)
@staticmethod
def build_repo_list(
repo_base_url,
repos,
project_id,
current_arch,
compose_latest_sync,
compose_dir_is_here: bool = False,
hashed: bool = False,
):
"""
Builds the repo dictionary
"""
repolist = []
prehashed = ''
if hashed:
prehashed = 'hashed-'
for name in repos:
if not compose_dir_is_here:
constructed_url = '{}/{}/repo/{}{}/{}'.format(
repo_base_url,
project_id,
prehashed,
name,
current_arch
)
else:
constructed_url = 'file://{}/{}/{}/os'.format(
compose_latest_sync,
name,
current_arch
)
repodata = {
'name': name,
'url': constructed_url
}
repolist.append(repodata)
return repolist