From bffff511f35b7e92f845a5966ac36406628c738f Mon Sep 17 00:00:00 2001 From: Louis Abel Date: Fri, 5 Aug 2022 01:24:49 -0700 Subject: [PATCH] Add unversioned linking and checksums Add symlinked versions of all images that are unversioned. This is primarily for users who would like a URL path to the latest ISO for a particular release. This is also for cases like osinfo where there should be a generic path per major release. --- .../empanadas/scripts/build_iso_extra.py | 6 +- .../templates/buildExtraImage.tmpl.sh | 6 ++ iso/empanadas/empanadas/util/dnf_utils.py | 5 +- iso/empanadas/empanadas/util/iso_utils.py | 102 +++++++++++------- 4 files changed, 75 insertions(+), 44 deletions(-) diff --git a/iso/empanadas/empanadas/scripts/build_iso_extra.py b/iso/empanadas/empanadas/scripts/build_iso_extra.py index b645747..90b2681 100755 --- a/iso/empanadas/empanadas/scripts/build_iso_extra.py +++ b/iso/empanadas/empanadas/scripts/build_iso_extra.py @@ -17,6 +17,8 @@ parser.add_argument('--logger', type=str) parser.add_argument('--extra-iso', type=str, help="Granular choice in which iso is built") parser.add_argument('--extra-iso-mode', type=str, default='local') parser.add_argument('--hashed', action='store_true') +parser.add_argument('--updated-image', action='store_true') +parser.add_argument('--image-increment',type=str, default='0') results = parser.parse_args() rlvars = rldict[results.release] major = rlvars['major'] @@ -32,7 +34,9 @@ a = IsoBuild( extra_iso_mode=results.extra_iso_mode, compose_dir_is_here=results.local_compose, hashed=results.hashed, - logger=results.logger + logger=results.logger, + updated_image=results.updated_image, + image_increment=results.image_increment ) def run(): diff --git a/iso/empanadas/empanadas/templates/buildExtraImage.tmpl.sh b/iso/empanadas/empanadas/templates/buildExtraImage.tmpl.sh index fbac286..cbb6c5f 100644 --- a/iso/empanadas/empanadas/templates/buildExtraImage.tmpl.sh +++ b/iso/empanadas/empanadas/templates/buildExtraImage.tmpl.sh @@ -23,3 +23,9 @@ fi {{ make_manifest }} +{% if extra_iso_mode == "podman" %} +# symlink to unversioned image name +ln -sf {{ isoname }} {{ generic_isoname }} +ln -sf {{ isoname }}.manifest {{ generic_isoname }}.manifest + +{% endif %} diff --git a/iso/empanadas/empanadas/util/dnf_utils.py b/iso/empanadas/empanadas/util/dnf_utils.py index 43371c2..a10a19c 100644 --- a/iso/empanadas/empanadas/util/dnf_utils.py +++ b/iso/empanadas/empanadas/util/dnf_utils.py @@ -96,7 +96,8 @@ class RepoSync: # Relevant major version items self.shortname = config['shortname'] - self.revision = rlvars['revision'] + "-" + rlvars['rclvl'] + self.revision_level = rlvars['revision'] + "-" + rlvars['rclvl'] + self.revision = rlvars['revision'] self.fullversion = rlvars['revision'] self.arches = rlvars['allowed_arches'] self.project_id = rlvars['project_id'] @@ -178,7 +179,7 @@ class RepoSync: self.log.addHandler(handler) self.log.info('reposync init') - self.log.info(self.revision) + self.log.info(self.revision_level) # The repo name should be valid if self.repo is not None: diff --git a/iso/empanadas/empanadas/util/iso_utils.py b/iso/empanadas/empanadas/util/iso_utils.py index 9689e0a..16cae11 100644 --- a/iso/empanadas/empanadas/util/iso_utils.py +++ b/iso/empanadas/empanadas/util/iso_utils.py @@ -105,7 +105,8 @@ class IsoBuild: self.arches = rlvars['allowed_arches'] self.release = rlvars['revision'] self.minor_version = rlvars['minor'] - self.revision = rlvars['revision'] + "-" + rlvars['rclvl'] + self.revision_level = rlvars['revision'] + "-" + rlvars['rclvl'] + self.revision = rlvars['revision'] self.rclvl = rlvars['rclvl'] self.repos = rlvars['iso_map']['lorax']['repos'] self.repo_base_url = config['repo_base_url'] @@ -119,12 +120,6 @@ class IsoBuild: if 'container' in rlvars and len(rlvars['container']) > 0: self.container = rlvars['container'] - self.staging_dir = os.path.join( - config['staging_root'], - config['category_stub'], - self.revision - ) - # all bucket related info self.s3_region = config['aws_region'] self.s3_bucket = config['bucket'] @@ -202,7 +197,7 @@ class IsoBuild: self.compose_dir_is_here, self.hashed ) - self.log.info(self.revision) + self.log.info(self.revision_level) def run(self): work_root = os.path.join( @@ -516,21 +511,9 @@ class IsoBuild: ) def _copy_boot_to_work(self, force_unpack, arch): - src_to_image = os.path.join( - self.lorax_work_dir, - arch, - 'lorax' - ) - - iso_to_go = os.path.join( - self.iso_work_dir, - arch - ) - - path_to_src_image = '{}/{}'.format( - src_to_image, - '/images/boot.iso' - ) + src_to_image = os.path.join(self.lorax_work_dir, arch, 'lorax') + iso_to_go = os.path.join(self.iso_work_dir, arch) + path_to_src_image = os.path.join(src_to_image, 'images/boot.iso') rclevel = '' if self.release_candidate: @@ -545,15 +528,13 @@ class IsoBuild: 'boot' ) - isobootpath = '{}/{}'.format( - iso_to_go, - discname - ) - - manifest = '{}.{}'.format( - isobootpath, - 'manifest' - ) + isobootpath = os.path.join(iso_to_go, discname) + manifest = '{}.manifest'.format(isobootpath) + link_name = '{}-{}-boot.iso'.format(self.shortname, arch) + link_manifest = link_name + '.manifest' + isobootpath = os.path.join(iso_to_go, discname) + linkbootpath = os.path.join(iso_to_go, link_name) + manifestlink = os.path.join(iso_to_go, link_manifest) if not force_unpack: file_check = isobootpath @@ -563,9 +544,18 @@ class IsoBuild: self.log.info('Copying %s boot iso to work directory...' % arch) os.makedirs(iso_to_go, exist_ok=True) - shutil.copy2(path_to_src_image, isobootpath) + try: + shutil.copy2(path_to_src_image, isobootpath) + if os.path.exists(linkbootpath): + os.remove(linkbootpath) + os.symlink(discname, linkbootpath) + except Exception as e: + self.log.error(Color.FAIL + 'We could not copy the image or create a symlink.') + raise SystemExit(e) + if os.path.exists(path_to_src_image + '.manifest'): shutil.copy2(path_to_src_image + '.manifest', manifest) + os.symlink(manifest.split('/')[-1], manifestlink) self.log.info('Creating checksum for %s boot iso...' % arch) checksum = Shared.get_checksum(isobootpath, self.checksum, self.log) @@ -576,6 +566,14 @@ class IsoBuild: c.write(checksum) c.close() + #linksum = Shared.get_checksum(linkbootpath, self.checksum, self.log) + #if not linksum: + # self.log.error(Color.FAIL + linkbootpath + ' not found! Did we actually make the symlink?') + # return + #with open(linkbootpath + '.CHECKSUM', "w+") as l: + # l.write(linksum) + # l.close() + def _copy_nondisc_to_repo(self, force_unpack, arch, repo): """ Syncs data from a non-disc set of images to the appropriate repo. Repo @@ -818,7 +816,7 @@ class IsoBuild: datestamp = '' if self.updated_image: - datestamp = '-' + self.updated_image_date.copy() + datestamp = '-' + self.updated_image_date volid = '{}-{}-{}{}-{}-{}'.format( self.shortname, @@ -829,15 +827,17 @@ class IsoBuild: volname ) - isoname = '{}-{}.{}{}-{}-{}.iso'.format( + isoname = '{}-{}{}{}-{}-{}.iso'.format( self.shortname, - self.major_version, - self.minor_version, + self.revision, rclevel, + datestamp, arch, image ) + generic_isoname = '{}-{}-{}.iso'.format(self.shortname, arch, image) + lorax_pkg_cmd = '/usr/bin/dnf install {} -y {}'.format( ' '.join(required_pkgs), log_path_command @@ -914,6 +914,7 @@ class IsoBuild: make_manifest=make_manifest, lorax_pkg_cmd=lorax_pkg_cmd, isoname=isoname, + generic_isoname=generic_isoname, ) mock_iso_entry = open(mock_iso_path, "w+") @@ -963,6 +964,11 @@ class IsoBuild: isos_dir = os.path.join(work_root, "isos") bad_exit_list = [] checksum_list = [] + + datestamp = '' + if self.updated_image: + datestamp = '-' + self.updated_image_date + for i in images: entry_name_list = [] image_name = i @@ -976,17 +982,25 @@ class IsoBuild: if self.release_candidate: rclevel = '-' + self.rclvl - isoname = '{}/{}-{}.{}{}-{}-{}.iso'.format( + isoname = '{}/{}-{}{}{}-{}-{}.iso'.format( a, self.shortname, - self.major_version, - self.minor_version, + self.revision, rclevel, + datestamp, + a, + i + ) + + genericname = '{}/{}-{}-{}.iso'.format( + a, + self.shortname, a, i ) checksum_list.append(isoname) + checksum_list.append(genericname) for pod in entry_name_list: podman_cmd_entry = '{} run -d -it -v "{}:{}" -v "{}:{}" --name {} --entrypoint {}/{} {}'.format( @@ -1477,6 +1491,8 @@ class LiveBuild: image=None, justcopyit: bool = False, force_build: bool = False, + updated_image: bool = False, + image_increment: str = '0', logger=None ): @@ -1490,7 +1506,6 @@ class LiveBuild: self.major_version = major self.compose_dir_is_here = compose_dir_is_here self.date_stamp = config['date_stamp'] - self.date = time.strftime("%Y%m%d", time.localtime()) self.compose_root = config['compose_root'] self.compose_base = config['compose_root'] + "/" + major self.current_arch = config['arch'] @@ -1524,6 +1539,11 @@ class LiveBuild: if 'container' in rlvars and len(rlvars['container']) > 0: self.container = rlvars['container'] + self.updated_image = updated_image + self.updated_image_increment = "." + image_increment + self.date = (time.strftime("%Y%m%d", time.localtime()) + + self.updated_image_increment) + # Templates file_loader = FileSystemLoader(f"{_rootdir}/templates") self.tmplenv = Environment(loader=file_loader)