sshtepa/toolkit
1
0
Fork 0
forked from sig_core/toolkit
Code Pull Requests Activity

update iso utils and prep

Browse Source
This commit is contained in:
Louis Abel 2022-06-15 13:53:12 -07:00
parent 602553e878
commit 3d161df451
Signed by untrusted user: label
GPG Key ID: B37E62D143879B36
10 changed files with 219 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
iso/py/common.py
View File

@ -34,7 +34,10 @@ config = {
"category_stub": "mirror/pub/rocky",
"sig_category_stub": "mirror/pub/sig",
"repo_base_url": "https://yumrepofs.build.resf.org/v1/projects",
"container": "centos:stream9"
"mock_work_root": "/builddir",
"container": "centos:stream9",
"distname": "Rocky Linux",
"shortname": "Rocky"
}
# Importing the config from yaml

16
iso/py/configs/el8.yaml
View File

@ -8,10 +8,6 @@
- aarch64
provide_multilib: False
project_id: ''
required_packages:
- 'lorax'
- 'genisoimage'
- 'isomd5sum'
repo_symlinks:
devel: 'Devel'
NFV: 'nfv'
@ -54,6 +50,18 @@
- dvd1
- minimal
- boot
repos:
- 'BaseOS'
- 'AppStream'
variant: 'BaseOS'
lorax_removes:
- 'libreport-rhel-anaconda-bugzilla'
required_packages:
- 'lorax'
- 'genisoimage'
- 'isomd5sum'
- 'lorax-templates-rhel'
- 'lorax-templates-generic'
repoclosure_map:
arches:
x86_64: '--arch=x86_64 --arch=athlon --arch=i686 --arch=i586 --arch=i486 --arch=i386 --arch=noarch'

13
iso/py/configs/el9.yaml
View File

@ -3,6 +3,7 @@
fullname: 'Rocky Linux 9'
revision: '9.0'
rclvl: 'RC1'
bugurl: 'https://bugs.rockylinux.org'
allowed_arches:
- x86_64
- aarch64
@ -10,10 +11,6 @@
- s390x
provide_multilib: True
project_id: '55b17281-bc54-4929-8aca-a8a11d628738'
required_packages:
- 'lorax'
- 'genisoimage'
- 'isomd5sum'
repo_symlinks:
NFV: 'nfv'
renames:
@ -62,6 +59,14 @@
- 'BaseOS'
- 'AppStream'
variant: 'BaseOS'
lorax_removes:
- 'libreport-rhel-anaconda-bugzilla'
required_pkgs:
- 'lorax'
- 'genisoimage'
- 'isomd5sum'
- 'lorax-templates-rhel'
- 'lorax-templates-generic'
repoclosure_map:
arches:
x86_64: '--forcearch=x86_64 --arch=x86_64 --arch=athlon --arch=i686 --arch=i586 --arch=i486 --arch=i386 --arch=noarch'

19
iso/py/templates/buildImage.tmpl.sh Normal file
View File

@ -0,0 +1,19 @@
#!/bin/bash
{% for pkg in lorax %}
sed -i '/{{ pkg }}/ s/^/#/' /usr/share/lorax/templates.d/80-rhel/runtime-install.tmpl
{% endfor %}
lorax --product='{{ distname }}' \
--version='{{ revision }}' \
--release='{{ revision }}' \
{%- for repo in repos %}
--source={{ repo.url }} \
{%- endfor %}
--variant={{ variant }} \
--nomacboot \
--buildarch={{ arch }} \
--volid={{ shortname }}-{{ major }}-{{ minor }}-{{ arch }}-dvd1 \
--logfile={{ mock_work_root }}/lorax.log \
--rootfs-size=3 \
{{ lorax_work_root }}

23
iso/py/templates/isobuild.tmpl.sh
View File

@ -4,12 +4,29 @@
# Init the container
mock \
-r /mnt/compose/9/latest-Rocky-9/work/entries/lorax-{{ major }}-{{ arch }}.cfg \
-r /var/tmp/lorax-{{ major }}.cfg \
--isolation={{ isolation }} \
--enable-network \
--init
cp /var/tmp/buildImage.sh \
/var/lib/mock/{{ shortname|lower }}-{{ major }}-{{ arch }}/root/var/tmp
mock \
-r /mnt/compose/9/latest-Rocky-9/work/entries/lorax-{{ major }}-{{ arch }}.cfg \
-r /var/tmp/lorax-{{ major }}.cfg \
--shell \
--enable-network -- /bin/bash /mnt/compose/9/latest-Rocky-9/work/entries/runLorax-{{ arch }}.sh
--isolation={{ isolation }} \
--enable-network -- /bin/bash /var/tmp/buildImage.sh
ret_val=$?
if [ $ret_val -eq 0 ]; then
# Copy resulting data to /var/lib/mock/{{ shortname|lower }}-{{ major }}-{{ arch }}/result
mkdir /var/lib/mock/{{ shortname|lower }}-{{ major }}-{{ arch }}/result
cp /var/lib/mock/{{ shortname|lower }}-{{ major }}-{{ arch }}/root/{{ builddir }}/lorax-{{ major }}-{{ arch }}.tar.gz \
/var/lib/mock/{{ shortname|lower }}-{{ major }}-{{ arch }}/result
else
echo "!! LORAX RUN FAILED !!"
exit 1
fi
# Clean up?

6
iso/py/templates/isolorax.tmpl.sh
View File

@ -1,2 +1,8 @@
#!/bin/bash
# You should not be running this manually.
# everything should be logged like | tee -a {{ log_path }}/lorax-{{ arch }}-{{ date_stamp }}.log
# for the dvd, we need to rely on pulling from {{ entries_root }}/dvd-{{ arch }}-list
# Run the base lorax steps into a work dir specific to its arch
# copy everything into BaseOS/arch/os

16
iso/py/templates/isomock.tmpl → iso/py/templates/isomock.tmpl.cfg
View File

@ -3,16 +3,18 @@ config_opts['description'] = '{{ fullname }}'
config_opts['target_arch'] = '{{ arch }}'
config_opts['legal_host_arches'] = ('{{ arch }}',)
config_opts['chroot_setup_cmd'] = 'install bash bzip2 coreutils cpio diffutils redhat-release findutils gawk glibc-minimal-langpack grep gzip info patch redhat-rpm-config rpm-build sed shadow-utils tar unzip util-linux which xz genisoimage isomd5sum lorax lorax-templates-rhel lorax-templates-generic'
config_opts['chroot_setup_cmd'] = 'install bash bzip2 coreutils cpio diffutils redhat-release findutils gawk glibc-minimal-langpack grep gzip info patch redhat-rpm-config rpm-build sed shadow-utils tar unzip util-linux which xz {{ required_pkgs|join(' ') }}'
config_opts['dist'] = '{{ dist }}' # only useful for --resultdir variable subst
config_opts['releasever'] = '{{ major }}'
config_opts['package_manager'] = 'dnf'
config_opts['package_manager'] = '{{ pkgmanager|default("dnf") }}'
config_opts['extra_chroot_dirs'] = [ '/run/lock', ]
# config_opts['bootstrap_image'] = 'quay.io/rockylinux/rockylinux:{{ major }}'
# If compose is local, the bind mounts will be here
{% if compose_dir_is_here %}
config_opts['plugin_conf']['bind_mount_enable'] = True
config_opts['plugin_conf']['bind_mount_opts']['dirs'].append(('/mnt/compose', '/mnt/compose'))
config_opts['plugin_conf']['bind_mount_opts']['dirs'].append(('{{ compose_dir }}', '{{ compose_dir }}'))
{% endif %}
config_opts['dnf.conf'] = """
[main]
@ -35,9 +37,9 @@ module_platform_id=platform:{{ dist }}
user_agent={{ user_agent }}
{% for repo in repos %}
[{{ repo }}]
name={{ repo }}
baseurl=file:///mnt/compose/{{ major }}/latest-Rocky-{{ major }}/compose/{{ repo }}/{{ arch }}/os
[{{ repo.name }}]
name={{ repo.name }}
baseurl={{ repo.url }}
enabled=1
gpgcheck=0

7
iso/py/util/dnf_utils.py
View File

@ -12,6 +12,7 @@ import subprocess
import shlex
import time
import re
import json
#import pipes
from common import Color
from jinja2 import Environment, FileSystemLoader
@ -793,7 +794,11 @@ class RepoSync:
def deploy_extra_files(self):
"""
deploys extra files based on info of rlvars
deploys extra files based on info of rlvars including a
extra_files.json
also deploys COMPOSE_ID and maybe in the future a metadata dir with a
bunch of compose-esque stuff.
"""
pass

176
iso/py/util/iso_utils.py
View File

@ -14,30 +14,34 @@ import time
import re
from productmd.common import SortedConfigParser
from common import Color
from jinja2 import Environment, FileSystemLoader
class IsoBuild:
"""
This helps us build the generic ISO's for a Rocky Linux release. In
particular, this is for the boot and dvd images.
particular, this is for the boot images.
Live images are built in another class.
While there is a function for building the DVD and live images, this not
the main design of this class. The other functions can be called on their
own to facilitate those particular builds.
"""
def __init__(
self,
rlvars,
config,
major,
host=None,
isolation: str = 'auto',
compose_dir_is_here: bool = False,
image=None,
arch=None,
logger=None
):
self.arch = arch
self.image = image
self.host = host
self.fullname = rlvars['fullname']
self.distname = config['distname']
self.shortname = config['shortname']
# Relevant config items
self.major_version = major
self.compose_dir_is_here = compose_dir_is_here
self.disttag = config['dist']
self.date_stamp = config['date_stamp']
self.timestamp = time.time()
@ -45,11 +49,21 @@ class IsoBuild:
self.compose_base = config['compose_root'] + "/" + major
self.iso_drop = config['compose_root'] + "/" + major + "/isos"
self.current_arch = config['arch']
self.extra_files = rlvars['extra_files']
self.required_pkgs = rlvars['iso_map']['required_pkgs']
self.mock_work_root = config['mock_work_root']
self.lorax_result_root = config['mock_work_root'] + "/" + "lorax"
self.mock_isolation = isolation
self.iso_map = rlvars['iso_map']
# Relevant major version items
self.release = rlvars['revision']
self.minor_version = rlvars['revision'].split('.')[1]
self.revision = rlvars['revision'] + "-" + rlvars['rclvl']
self.arches = rlvars['allowed_arches']
self.repos = rlvars['iso_map']['repos']
self.repo_base_url = config['repo_base_url']
self.project_id = rlvars['project_id']
self.extra_files = rlvars['extra_files']
self.staging_dir = os.path.join(
config['staging_root'],
@ -57,6 +71,10 @@ class IsoBuild:
self.revision
)
# Templates
file_loader = FileSystemLoader('templates')
self.tmplenv = Environment(loader=file_loader)
self.compose_latest_dir = os.path.join(
config['compose_root'],
major,
@ -73,6 +91,12 @@ class IsoBuild:
"work/logs"
)
self.iso_work_dir = os.path.join(
self.compose_latest_dir,
"work/iso",
config['arch']
)
# This is temporary for now.
if logger is None:
self.log = logging.getLogger("iso")
@ -87,6 +111,7 @@ class IsoBuild:
self.log.addHandler(handler)
self.log.info('iso build init')
self.repolist = self.build_repo_list()
self.log.info(self.revision)
def run(self):
@ -101,66 +126,121 @@ class IsoBuild:
"logs"
)
self.iso_build(
sync_root,
work_root,
log_root,
self.arch,
self.host
)
self.iso_build()
self.log.info('Compose repo directory: %s' % sync_root)
self.log.info('ISO Build Logs: %s' % log_root)
self.log.info('ISO Build completed.')
def iso_build(self, sync_root, work_root, log_root, arch, host):
def build_repo_list(self):
"""
Calls out the ISO builds to the individual hosts listed in the map.
Each architecture is expected to build their own ISOs, similar to
runroot operations of koji and pungi.
Builds the repo dictionary
"""
repolist = []
for name in self.repos:
if not self.compose_dir_is_here:
constructed_url = '{}/{}/repo/hashed-{}/{}'.format(
self.repo_base_url,
self.project_id,
name,
self.current_arch
)
else:
constructed_url = 'file://{}/{}/{}/os'.format(
self.compose_latest_sync,
name,
self.current_arch
)
It IS possible to run locally, but that would mean this only builds
ISOs for the architecture of the running machine. Please keep this in
mind when stating host=local.
repodata = {
'name': name,
'url': constructed_url
}
repolist.append(repodata)
return repolist
def iso_build(self):
"""
This does the general ISO building for the current running
architecture. This generates the mock config and the general script
needed to get this part running.
"""
# Check for local build, build accordingly
# Check for arch specific build, build accordingly
# local AND arch cannot be used together, local supersedes. print
# warning.
local_only = False
if 'local' in self.host:
local_only = True
arch = self.arch.copy()
if local_only and self.arch:
self.log.warn('You cannot set local build AND an architecture.')
self.log.warn('The architecture %s will be set' % self.current_arch)
arch = self.current_arch
def iso_build_local(self, sync_root, work_root, log_root):
"""
Local iso builds only. Architecture is locked.
"""
print()
def iso_build_remote(self, sync_root, work_root, log_root, arch):
"""
Remote ISO builds. Architecture is all or single.
"""
self.log.info('Generating ISO configuration and scripts')
self.generate_iso_scripts()
print()
def generate_iso_scripts(self):
"""
Generates the scripts needed to be ran in the mock roots
"""
mock_iso_template = self.tmplenv.get_template('isomock.tmpl.cfg')
mock_sh_template = self.tmplenv.get_template('isobuild.tmpl.sh')
iso_template = self.tmplenv.get_template('buildImage.tmpl.sh')
mock_iso_path = '/var/tmp/lorax-' + self.major_version + '.cfg'
mock_sh_path = '/var/tmp/isobuild.sh'
iso_template_path = '/var/tmp/buildImage.sh'
mock_iso_template_output = mock_iso_template.render(
arch=self.current_arch,
major=self.major_version,
fullname=self.fullname,
required_pkgs=self.required_pkgs,
dist=self.disttag,
repos=self.repolist,
user_agent='{{ user_agent }}',
)
mock_sh_template_output = mock_sh_template.render(
arch=self.current_arch,
major=self.major_version,
isolation=self.mock_isolation,
builddir=self.mock_work_root,
shortname=self.shortname,
)
iso_template_output = iso_template.render(
arch=self.current_arch,
major=self.major_version,
minor=self.minor_version,
shortname=self.shortname,
repos=self.repolist,
variant=self.iso_map['variant'],
lorax=self.iso_map['lorax_removes'],
distname=self.distname,
revision=self.release,
)
mock_iso_entry = open(mock_iso_path, "w+")
mock_iso_entry.write(mock_iso_template_output)
mock_iso_entry.close()
mock_sh_entry = open(mock_sh_path, "w+")
mock_sh_entry.write(mock_sh_template_output)
mock_sh_entry.close()
iso_template_entry = open(iso_template_path, "w+")
iso_template_entry.write(iso_template_output)
iso_template_entry.close()
print()
# !!! Send help, we would prefer to do this using the productmd python
# !!! library. If you are reading this and you can help us, please do so!
def treeinfo_write(self):
"""
Ensure treeinfo is written correctly
"""
print()
# !!! Send help, we would prefer to do this using the productmd python
# !!! library. If you are reading this and you can help us, please do so!
def discinfo_write(self):
"""
Ensure discinfo is written correctly
@ -187,13 +267,19 @@ class IsoBuild:
"",
]
def generate_graft_points(self):
def build_extra_iso(self):
"""
Get a list of packages for an ISO
Builds DVD images based on the data created from the initial lorax on
each arch. This should NOT be called during the usual run() section.
"""
print()
def generate_graft_points(self):
"""
Get a list of packages for an extras ISO. This should NOT be called
during the usual run() section.
"""
print()
class LiveBuild:
"""

6
sync/common_9
View File

@ -169,6 +169,6 @@ EOF
/bin/cp "${TREEINFO_VAR}" "${PRISTINE_TREE}"
}
export -f treeinfoFixer
export -f treeinfoModder
export -f treeinfoModderKickstart
#export -f treeinfoFixer
#export -f treeinfoModder
#export -f treeinfoModderKickstart