update iso utils and prep

This commit is contained in:
Louis Abel 2022-06-15 13:53:12 -07:00
parent 602553e878
commit 3d161df451
Signed by untrusted user: label
GPG Key ID: B37E62D143879B36
10 changed files with 219 additions and 68 deletions

View File

@ -34,7 +34,10 @@ config = {
"category_stub": "mirror/pub/rocky", "category_stub": "mirror/pub/rocky",
"sig_category_stub": "mirror/pub/sig", "sig_category_stub": "mirror/pub/sig",
"repo_base_url": "https://yumrepofs.build.resf.org/v1/projects", "repo_base_url": "https://yumrepofs.build.resf.org/v1/projects",
"container": "centos:stream9" "mock_work_root": "/builddir",
"container": "centos:stream9",
"distname": "Rocky Linux",
"shortname": "Rocky"
} }
# Importing the config from yaml # Importing the config from yaml

View File

@ -8,10 +8,6 @@
- aarch64 - aarch64
provide_multilib: False provide_multilib: False
project_id: '' project_id: ''
required_packages:
- 'lorax'
- 'genisoimage'
- 'isomd5sum'
repo_symlinks: repo_symlinks:
devel: 'Devel' devel: 'Devel'
NFV: 'nfv' NFV: 'nfv'
@ -54,6 +50,18 @@
- dvd1 - dvd1
- minimal - minimal
- boot - boot
repos:
- 'BaseOS'
- 'AppStream'
variant: 'BaseOS'
lorax_removes:
- 'libreport-rhel-anaconda-bugzilla'
required_packages:
- 'lorax'
- 'genisoimage'
- 'isomd5sum'
- 'lorax-templates-rhel'
- 'lorax-templates-generic'
repoclosure_map: repoclosure_map:
arches: arches:
x86_64: '--arch=x86_64 --arch=athlon --arch=i686 --arch=i586 --arch=i486 --arch=i386 --arch=noarch' x86_64: '--arch=x86_64 --arch=athlon --arch=i686 --arch=i586 --arch=i486 --arch=i386 --arch=noarch'

View File

@ -3,6 +3,7 @@
fullname: 'Rocky Linux 9' fullname: 'Rocky Linux 9'
revision: '9.0' revision: '9.0'
rclvl: 'RC1' rclvl: 'RC1'
bugurl: 'https://bugs.rockylinux.org'
allowed_arches: allowed_arches:
- x86_64 - x86_64
- aarch64 - aarch64
@ -10,10 +11,6 @@
- s390x - s390x
provide_multilib: True provide_multilib: True
project_id: '55b17281-bc54-4929-8aca-a8a11d628738' project_id: '55b17281-bc54-4929-8aca-a8a11d628738'
required_packages:
- 'lorax'
- 'genisoimage'
- 'isomd5sum'
repo_symlinks: repo_symlinks:
NFV: 'nfv' NFV: 'nfv'
renames: renames:
@ -62,6 +59,14 @@
- 'BaseOS' - 'BaseOS'
- 'AppStream' - 'AppStream'
variant: 'BaseOS' variant: 'BaseOS'
lorax_removes:
- 'libreport-rhel-anaconda-bugzilla'
required_pkgs:
- 'lorax'
- 'genisoimage'
- 'isomd5sum'
- 'lorax-templates-rhel'
- 'lorax-templates-generic'
repoclosure_map: repoclosure_map:
arches: arches:
x86_64: '--forcearch=x86_64 --arch=x86_64 --arch=athlon --arch=i686 --arch=i586 --arch=i486 --arch=i386 --arch=noarch' x86_64: '--forcearch=x86_64 --arch=x86_64 --arch=athlon --arch=i686 --arch=i586 --arch=i486 --arch=i386 --arch=noarch'

View File

@ -0,0 +1,19 @@
#!/bin/bash
{% for pkg in lorax %}
sed -i '/{{ pkg }}/ s/^/#/' /usr/share/lorax/templates.d/80-rhel/runtime-install.tmpl
{% endfor %}
lorax --product='{{ distname }}' \
--version='{{ revision }}' \
--release='{{ revision }}' \
{%- for repo in repos %}
--source={{ repo.url }} \
{%- endfor %}
--variant={{ variant }} \
--nomacboot \
--buildarch={{ arch }} \
--volid={{ shortname }}-{{ major }}-{{ minor }}-{{ arch }}-dvd1 \
--logfile={{ mock_work_root }}/lorax.log \
--rootfs-size=3 \
{{ lorax_work_root }}

View File

@ -4,12 +4,29 @@
# Init the container # Init the container
mock \ mock \
-r /mnt/compose/9/latest-Rocky-9/work/entries/lorax-{{ major }}-{{ arch }}.cfg \ -r /var/tmp/lorax-{{ major }}.cfg \
--isolation={{ isolation }} \
--enable-network \
--init --init
cp /var/tmp/buildImage.sh \
/var/lib/mock/{{ shortname|lower }}-{{ major }}-{{ arch }}/root/var/tmp
mock \ mock \
-r /mnt/compose/9/latest-Rocky-9/work/entries/lorax-{{ major }}-{{ arch }}.cfg \ -r /var/tmp/lorax-{{ major }}.cfg \
--shell \ --shell \
--enable-network -- /bin/bash /mnt/compose/9/latest-Rocky-9/work/entries/runLorax-{{ arch }}.sh --isolation={{ isolation }} \
--enable-network -- /bin/bash /var/tmp/buildImage.sh
ret_val=$?
if [ $ret_val -eq 0 ]; then
# Copy resulting data to /var/lib/mock/{{ shortname|lower }}-{{ major }}-{{ arch }}/result
mkdir /var/lib/mock/{{ shortname|lower }}-{{ major }}-{{ arch }}/result
cp /var/lib/mock/{{ shortname|lower }}-{{ major }}-{{ arch }}/root/{{ builddir }}/lorax-{{ major }}-{{ arch }}.tar.gz \
/var/lib/mock/{{ shortname|lower }}-{{ major }}-{{ arch }}/result
else
echo "!! LORAX RUN FAILED !!"
exit 1
fi
# Clean up? # Clean up?

View File

@ -1,2 +1,8 @@
#!/bin/bash #!/bin/bash
# You should not be running this manually. # You should not be running this manually.
# everything should be logged like | tee -a {{ log_path }}/lorax-{{ arch }}-{{ date_stamp }}.log
# for the dvd, we need to rely on pulling from {{ entries_root }}/dvd-{{ arch }}-list
# Run the base lorax steps into a work dir specific to its arch
# copy everything into BaseOS/arch/os

View File

@ -3,16 +3,18 @@ config_opts['description'] = '{{ fullname }}'
config_opts['target_arch'] = '{{ arch }}' config_opts['target_arch'] = '{{ arch }}'
config_opts['legal_host_arches'] = ('{{ arch }}',) config_opts['legal_host_arches'] = ('{{ arch }}',)
config_opts['chroot_setup_cmd'] = 'install bash bzip2 coreutils cpio diffutils redhat-release findutils gawk glibc-minimal-langpack grep gzip info patch redhat-rpm-config rpm-build sed shadow-utils tar unzip util-linux which xz genisoimage isomd5sum lorax lorax-templates-rhel lorax-templates-generic' config_opts['chroot_setup_cmd'] = 'install bash bzip2 coreutils cpio diffutils redhat-release findutils gawk glibc-minimal-langpack grep gzip info patch redhat-rpm-config rpm-build sed shadow-utils tar unzip util-linux which xz {{ required_pkgs|join(' ') }}'
config_opts['dist'] = '{{ dist }}' # only useful for --resultdir variable subst config_opts['dist'] = '{{ dist }}' # only useful for --resultdir variable subst
config_opts['releasever'] = '{{ major }}' config_opts['releasever'] = '{{ major }}'
config_opts['package_manager'] = 'dnf' config_opts['package_manager'] = '{{ pkgmanager|default("dnf") }}'
config_opts['extra_chroot_dirs'] = [ '/run/lock', ] config_opts['extra_chroot_dirs'] = [ '/run/lock', ]
# config_opts['bootstrap_image'] = 'quay.io/rockylinux/rockylinux:{{ major }}' # config_opts['bootstrap_image'] = 'quay.io/rockylinux/rockylinux:{{ major }}'
# If compose is local, the bind mounts will be here
{% if compose_dir_is_here %}
config_opts['plugin_conf']['bind_mount_enable'] = True config_opts['plugin_conf']['bind_mount_enable'] = True
config_opts['plugin_conf']['bind_mount_opts']['dirs'].append(('/mnt/compose', '/mnt/compose')) config_opts['plugin_conf']['bind_mount_opts']['dirs'].append(('{{ compose_dir }}', '{{ compose_dir }}'))
{% endif %}
config_opts['dnf.conf'] = """ config_opts['dnf.conf'] = """
[main] [main]
@ -35,9 +37,9 @@ module_platform_id=platform:{{ dist }}
user_agent={{ user_agent }} user_agent={{ user_agent }}
{% for repo in repos %} {% for repo in repos %}
[{{ repo }}] [{{ repo.name }}]
name={{ repo }} name={{ repo.name }}
baseurl=file:///mnt/compose/{{ major }}/latest-Rocky-{{ major }}/compose/{{ repo }}/{{ arch }}/os baseurl={{ repo.url }}
enabled=1 enabled=1
gpgcheck=0 gpgcheck=0

View File

@ -12,6 +12,7 @@ import subprocess
import shlex import shlex
import time import time
import re import re
import json
#import pipes #import pipes
from common import Color from common import Color
from jinja2 import Environment, FileSystemLoader from jinja2 import Environment, FileSystemLoader
@ -793,7 +794,11 @@ class RepoSync:
def deploy_extra_files(self): def deploy_extra_files(self):
""" """
deploys extra files based on info of rlvars deploys extra files based on info of rlvars including a
extra_files.json
also deploys COMPOSE_ID and maybe in the future a metadata dir with a
bunch of compose-esque stuff.
""" """
pass pass

View File

@ -14,30 +14,34 @@ import time
import re import re
from productmd.common import SortedConfigParser from productmd.common import SortedConfigParser
from common import Color from common import Color
from jinja2 import Environment, FileSystemLoader
class IsoBuild: class IsoBuild:
""" """
This helps us build the generic ISO's for a Rocky Linux release. In This helps us build the generic ISO's for a Rocky Linux release. In
particular, this is for the boot and dvd images. particular, this is for the boot images.
Live images are built in another class. While there is a function for building the DVD and live images, this not
the main design of this class. The other functions can be called on their
own to facilitate those particular builds.
""" """
def __init__( def __init__(
self, self,
rlvars, rlvars,
config, config,
major, major,
host=None, isolation: str = 'auto',
compose_dir_is_here: bool = False,
image=None, image=None,
arch=None,
logger=None logger=None
): ):
self.arch = arch
self.image = image self.image = image
self.host = host
self.fullname = rlvars['fullname'] self.fullname = rlvars['fullname']
self.distname = config['distname']
self.shortname = config['shortname']
# Relevant config items # Relevant config items
self.major_version = major self.major_version = major
self.compose_dir_is_here = compose_dir_is_here
self.disttag = config['dist'] self.disttag = config['dist']
self.date_stamp = config['date_stamp'] self.date_stamp = config['date_stamp']
self.timestamp = time.time() self.timestamp = time.time()
@ -45,11 +49,21 @@ class IsoBuild:
self.compose_base = config['compose_root'] + "/" + major self.compose_base = config['compose_root'] + "/" + major
self.iso_drop = config['compose_root'] + "/" + major + "/isos" self.iso_drop = config['compose_root'] + "/" + major + "/isos"
self.current_arch = config['arch'] self.current_arch = config['arch']
self.extra_files = rlvars['extra_files'] self.required_pkgs = rlvars['iso_map']['required_pkgs']
self.mock_work_root = config['mock_work_root']
self.lorax_result_root = config['mock_work_root'] + "/" + "lorax"
self.mock_isolation = isolation
self.iso_map = rlvars['iso_map']
# Relevant major version items # Relevant major version items
self.release = rlvars['revision']
self.minor_version = rlvars['revision'].split('.')[1]
self.revision = rlvars['revision'] + "-" + rlvars['rclvl'] self.revision = rlvars['revision'] + "-" + rlvars['rclvl']
self.arches = rlvars['allowed_arches'] self.repos = rlvars['iso_map']['repos']
self.repo_base_url = config['repo_base_url']
self.project_id = rlvars['project_id']
self.extra_files = rlvars['extra_files']
self.staging_dir = os.path.join( self.staging_dir = os.path.join(
config['staging_root'], config['staging_root'],
@ -57,6 +71,10 @@ class IsoBuild:
self.revision self.revision
) )
# Templates
file_loader = FileSystemLoader('templates')
self.tmplenv = Environment(loader=file_loader)
self.compose_latest_dir = os.path.join( self.compose_latest_dir = os.path.join(
config['compose_root'], config['compose_root'],
major, major,
@ -73,6 +91,12 @@ class IsoBuild:
"work/logs" "work/logs"
) )
self.iso_work_dir = os.path.join(
self.compose_latest_dir,
"work/iso",
config['arch']
)
# This is temporary for now. # This is temporary for now.
if logger is None: if logger is None:
self.log = logging.getLogger("iso") self.log = logging.getLogger("iso")
@ -87,6 +111,7 @@ class IsoBuild:
self.log.addHandler(handler) self.log.addHandler(handler)
self.log.info('iso build init') self.log.info('iso build init')
self.repolist = self.build_repo_list()
self.log.info(self.revision) self.log.info(self.revision)
def run(self): def run(self):
@ -101,66 +126,121 @@ class IsoBuild:
"logs" "logs"
) )
self.iso_build( self.iso_build()
sync_root,
work_root,
log_root,
self.arch,
self.host
)
self.log.info('Compose repo directory: %s' % sync_root) self.log.info('Compose repo directory: %s' % sync_root)
self.log.info('ISO Build Logs: %s' % log_root) self.log.info('ISO Build Logs: %s' % log_root)
self.log.info('ISO Build completed.') self.log.info('ISO Build completed.')
def iso_build(self, sync_root, work_root, log_root, arch, host): def build_repo_list(self):
""" """
Calls out the ISO builds to the individual hosts listed in the map. Builds the repo dictionary
Each architecture is expected to build their own ISOs, similar to """
runroot operations of koji and pungi. repolist = []
for name in self.repos:
if not self.compose_dir_is_here:
constructed_url = '{}/{}/repo/hashed-{}/{}'.format(
self.repo_base_url,
self.project_id,
name,
self.current_arch
)
else:
constructed_url = 'file://{}/{}/{}/os'.format(
self.compose_latest_sync,
name,
self.current_arch
)
It IS possible to run locally, but that would mean this only builds
ISOs for the architecture of the running machine. Please keep this in repodata = {
mind when stating host=local. 'name': name,
'url': constructed_url
}
repolist.append(repodata)
return repolist
def iso_build(self):
"""
This does the general ISO building for the current running
architecture. This generates the mock config and the general script
needed to get this part running.
""" """
# Check for local build, build accordingly # Check for local build, build accordingly
# Check for arch specific build, build accordingly # Check for arch specific build, build accordingly
# local AND arch cannot be used together, local supersedes. print # local AND arch cannot be used together, local supersedes. print
# warning. # warning.
local_only = False self.log.info('Generating ISO configuration and scripts')
if 'local' in self.host: self.generate_iso_scripts()
local_only = True
arch = self.arch.copy()
if local_only and self.arch:
self.log.warn('You cannot set local build AND an architecture.')
self.log.warn('The architecture %s will be set' % self.current_arch)
arch = self.current_arch
def iso_build_local(self, sync_root, work_root, log_root):
"""
Local iso builds only. Architecture is locked.
"""
print()
def iso_build_remote(self, sync_root, work_root, log_root, arch):
"""
Remote ISO builds. Architecture is all or single.
"""
print() print()
def generate_iso_scripts(self): def generate_iso_scripts(self):
""" """
Generates the scripts needed to be ran in the mock roots Generates the scripts needed to be ran in the mock roots
""" """
mock_iso_template = self.tmplenv.get_template('isomock.tmpl.cfg')
mock_sh_template = self.tmplenv.get_template('isobuild.tmpl.sh')
iso_template = self.tmplenv.get_template('buildImage.tmpl.sh')
mock_iso_path = '/var/tmp/lorax-' + self.major_version + '.cfg'
mock_sh_path = '/var/tmp/isobuild.sh'
iso_template_path = '/var/tmp/buildImage.sh'
mock_iso_template_output = mock_iso_template.render(
arch=self.current_arch,
major=self.major_version,
fullname=self.fullname,
required_pkgs=self.required_pkgs,
dist=self.disttag,
repos=self.repolist,
user_agent='{{ user_agent }}',
)
mock_sh_template_output = mock_sh_template.render(
arch=self.current_arch,
major=self.major_version,
isolation=self.mock_isolation,
builddir=self.mock_work_root,
shortname=self.shortname,
)
iso_template_output = iso_template.render(
arch=self.current_arch,
major=self.major_version,
minor=self.minor_version,
shortname=self.shortname,
repos=self.repolist,
variant=self.iso_map['variant'],
lorax=self.iso_map['lorax_removes'],
distname=self.distname,
revision=self.release,
)
mock_iso_entry = open(mock_iso_path, "w+")
mock_iso_entry.write(mock_iso_template_output)
mock_iso_entry.close()
mock_sh_entry = open(mock_sh_path, "w+")
mock_sh_entry.write(mock_sh_template_output)
mock_sh_entry.close()
iso_template_entry = open(iso_template_path, "w+")
iso_template_entry.write(iso_template_output)
iso_template_entry.close()
print() print()
# !!! Send help, we would prefer to do this using the productmd python
# !!! library. If you are reading this and you can help us, please do so!
def treeinfo_write(self): def treeinfo_write(self):
""" """
Ensure treeinfo is written correctly Ensure treeinfo is written correctly
""" """
print() print()
# !!! Send help, we would prefer to do this using the productmd python
# !!! library. If you are reading this and you can help us, please do so!
def discinfo_write(self): def discinfo_write(self):
""" """
Ensure discinfo is written correctly Ensure discinfo is written correctly
@ -187,13 +267,19 @@ class IsoBuild:
"", "",
] ]
def build_extra_iso(self):
def generate_graft_points(self):
""" """
Get a list of packages for an ISO Builds DVD images based on the data created from the initial lorax on
each arch. This should NOT be called during the usual run() section.
""" """
print() print()
def generate_graft_points(self):
"""
Get a list of packages for an extras ISO. This should NOT be called
during the usual run() section.
"""
print()
class LiveBuild: class LiveBuild:
""" """

View File

@ -169,6 +169,6 @@ EOF
/bin/cp "${TREEINFO_VAR}" "${PRISTINE_TREE}" /bin/cp "${TREEINFO_VAR}" "${PRISTINE_TREE}"
} }
export -f treeinfoFixer #export -f treeinfoFixer
export -f treeinfoModder #export -f treeinfoModder
export -f treeinfoModderKickstart #export -f treeinfoModderKickstart