From 520db534be16c2c6befa5f0d66f41f8c1e34e39f Mon Sep 17 00:00:00 2001 From: Louis Abel Date: Mon, 11 Jul 2022 15:12:41 -0700 Subject: [PATCH] add justcopyit for live images --- iso/empanadas/README.md | 2 + iso/empanadas/empanadas/.common.py.swp | Bin 0 -> 16384 bytes .../empanadas/scripts/build_iso_live.py | 2 + .../empanadas/scripts/generate_compose.py | 72 ++++++++++++++++++ iso/empanadas/empanadas/util/iso_utils.py | 56 ++++++++++++-- iso/empanadas/empanadas/util/shared.py | 4 + iso/empanadas/pyproject.toml | 1 + 7 files changed, 130 insertions(+), 7 deletions(-) create mode 100644 iso/empanadas/empanadas/.common.py.swp create mode 100755 iso/empanadas/empanadas/scripts/generate_compose.py diff --git a/iso/empanadas/README.md b/iso/empanadas/README.md index b7ec7ef..5058d5f 100644 --- a/iso/empanadas/README.md +++ b/iso/empanadas/README.md @@ -41,6 +41,8 @@ Changes to the poetry.lock should be commited if dependencies are added or updat * finalize_compose -> Finalizes a compose with metadata and checksums, as well as copies images * launch-builds -> Creates a kube config to run build-iso * build-image -> Runs build-iso +* generate_compose -> Creates a compose directory right away and optionally links it as latest + (You should only use this if you are running into errors with images) ``` ## wrappers diff --git a/iso/empanadas/empanadas/.common.py.swp b/iso/empanadas/empanadas/.common.py.swp new file mode 100644 index 0000000000000000000000000000000000000000..15e89105dbe45c91f269931e98c09cfd2a34137b GIT binary patch literal 16384 zcmeHNO^h5z6>dWK3B-vZQN)d+&1z3BbNosf4#Na^o! zX&wh2kNYCjvge1=-@Pi;t``e4nenUEV67jRnhyFkUn)^EQii)o2(+w;DB&^hay@qE z%P5kumkcLOl}{xDB?Iq=fzB+nnqzmkEHfs%oefs%oe zfs%oefs%oefs%oefs%oef&W1Uc-L|6hF@<{z>o9)BL08vPRIEZ@G|f#;OD^4fFAJj$6u1-k>k-HK4e)E=8Q>{E0$&79 z1NQ<)fwymWoWB4s0WSbQ0Cs_sz^%Y5A9tMJ0xtkJfG2Nvj#UId;4o&_EUwt;(qyMeoae|*GoUIl&xJP*V` z9k>HH0=)iV$N2^D6mS{11l$h1{vmJ!oCDsz)p33T7(fFGI1Ste{2d!FuL4g4-v+(~ zXkZ(-54Z<-3!68u051dI0s6oUKyCVFv5|AA?=f6Kk1egPu3v7fv^TF@XtXaawVF$7 zn;TngGfcz|TU=yoG8T1*l<87D(1JDgeUX?z#?_v?2}KrY7Q}lz47znzy=F6CpJUhd zxC(e|uFtx&4&_F1R>op!kYRe+b*C$O%o?-G?c7S$KGm5vY7Sy+**-10?kt=C9P8ji zlDc!Hc(=~B+|{777bxS-A^D8lcXtld&WTv4z+Vn!+J!DvcuIh`djHEwXE9qGQYJ=P#^pG}_IL z_4eiU)*~y;7DPpAYkhr_-N#(|zNY5ljw^pa3HG7Ojc9Abqr^F;LXf19RfXNaH(M%< zxUb~SG|8S&NN+VRH8+~;Yc>yM-l{0E7icteWd~(c<}3Kk>_X&+dtn9H*ycc>Q(O(g zkaYyp!nUm$h(IwVbRv8c>oE7T&^KWLNlHyiV) zy|N6AwH4wYHMdbOUwWeq?s9M`gORRJ&%BNCl?QQ#YFD zat2^f$%v8Gf!oa%S`g{)KZKgI& zd&C2ELV8oO^2S-n22r<@`n$rkQx#(9atFpFx?ZcH*xvl8C{HhVsh&qiH}fYv9`VOy z%&%$>pY6G}6XaIN!emZ28y{V0qbhy8tE4lsm5Z;BjiP$t)bV{%%5Fw|%uN#iJ~K<9~hWXoycBgbJ; zg>IlH1#C5l)BQpWf=APZ>x3L^EA)=HZ$1GaHzftr(xD^3cj>XNxX~4uaeXS{%blS?Ha3?wk+HHx{^pa#@|C4Enlng7$(R!8 zaR*ZkWjVXlTs@qn)vzignv&!4Qj69K6NOx9(0Y>Wl5|R@=Gxiy!JQ|S+^-_TsfEJK>cs=)Q-$`*QtJ^* zSE0XfGS{C-JyoP$xY)X|I%em=Tsx~+VdT__Jeg=6C6e!c$hFR9>UCz(XZS9I7*6`Wl$e~nPS-0C* zP;qp%cDqVia|~*$EaU6Xqc!%J)5yfL)cJ&JtBrycn~Sr?82lf zKJdk%M7oR`W}2BpSeG4kXxtN>CPGz5UeMbyXN}H!?Kalj1~r+GX6|LSb2-%%HZ!6p zl&jF=X@~`bZ|ovrDW+L76cGk$EJm?Rv+DzzDbMl_A_bNV%>c=mgB4f!E8Sr7|K44lEtH4p5&SsSBhj>^Az_qrWNz>R61`qVEfl$Js`2TUl$Zr7@{}<=`e?}aC1Na8e0zLsejhOvGUACrJ6( zhIMCW%f-g7yehhFQWe_8jyC0`YusUk#MmP8bP~eJQN>&@FOK3yis;(8s0wQ4CQ2_3 zp)zjl6Cy_{=ClnkBZ2r3VHKrVa_WfH4T|jrBJ9%sHcTUQUqos~l6CJrwQ`4Sfg@{~ z0cyI#>3Og_)}D3eG-^jXTf0}=G7g7}n+j2X7N=4OJ|3Y$Z7%aTt4ZEbD2J^X)o%KE zVC_9!>p3A=rt4xn_bk+CHW2>CKjT0o> n^qUS|^J7+K4`8t!NxQr#tMHl832xD)C#yrpwRCKg9qs-byo?E< literal 0 HcmV?d00001 diff --git a/iso/empanadas/empanadas/scripts/build_iso_live.py b/iso/empanadas/empanadas/scripts/build_iso_live.py index b7a7d84..92cc100 100755 --- a/iso/empanadas/empanadas/scripts/build_iso_live.py +++ b/iso/empanadas/empanadas/scripts/build_iso_live.py @@ -15,6 +15,7 @@ parser.add_argument('--image', type=str, help="Granular choice in which live ima parser.add_argument('--logger', type=str) parser.add_argument('--live-iso-mode', type=str, default='local') parser.add_argument('--hashed', action='store_true') +parser.add_argument('--just-copy-it', action='store_true', help="Just copy the images to the compose dir") results = parser.parse_args() rlvars = rldict[results.release] major = rlvars['major'] @@ -28,6 +29,7 @@ a = LiveBuild( image=results.image, compose_dir_is_here=results.local_compose, hashed=results.hashed, + justcopyit=results.just_copy_it, logger=results.logger ) diff --git a/iso/empanadas/empanadas/scripts/generate_compose.py b/iso/empanadas/empanadas/scripts/generate_compose.py new file mode 100755 index 0000000..d2a50fe --- /dev/null +++ b/iso/empanadas/empanadas/scripts/generate_compose.py @@ -0,0 +1,72 @@ +# This script can be called to do single syncs or full on syncs. + +import argparse +import logging +import sys + +from empanadas.common import * +from empanadas.util import Checks +from empanadas.util import RepoSync +from empanadas.util import Shared + +# Start up the parser baby +parser = argparse.ArgumentParser(description="Peridot Sync and Compose") + +# All of our options +parser.add_argument('--release', type=str, help="Major Release Version or major-type (eg 9-beta)", required=True) +parser.add_argument('--logger', type=str) + +# Parse them +results = parser.parse_args() +rlvars = rldict[results.release] +major = rlvars['major'] + +r = Checks(rlvars, config['arch']) +r.check_valid_arch() + +# Send them and do whatever I guess +def run(): + if results.logger is None: + log = logging.getLogger("generate") + log.setLevel(logging.INFO) + handler = logging.StreamHandler(sys.stdout) + handler.setLevel(logging.INFO) + formatter = logging.Formatter( + '%(asctime)s :: %(name)s :: %(message)s', + '%Y-%m-%d %H:%M:%S' + ) + handler.setFormatter(formatter) + log.addHandler(handler) + else: + log = results.logger + + compose_base = config['compose_root'] + "/" + major + shortname = config['shortname'] + version = rlvars['revision'] + date_stamp = config['date_stamp'] + profile = rlvars['profile'] + logger = log + + generated_dir = Shared.generate_compose_dirs( + compose_base, + shortname, + version, + date_stamp, + logger + ) + + if results.symlink: + compose_latest_dir = os.path.join( + config['compose_root'], + major, + "latest-{}-{}".format( + shortname, + profile, + ) + ) + if os.path.exists(compose_latest_dir): + os.remove(compose_latest_dir) + + os.symlink(generated_dir, compose_latest_dir) + + log.info('Generated compose dirs.') diff --git a/iso/empanadas/empanadas/util/iso_utils.py b/iso/empanadas/empanadas/util/iso_utils.py index 6c6bf04..ccba2f9 100644 --- a/iso/empanadas/empanadas/util/iso_utils.py +++ b/iso/empanadas/empanadas/util/iso_utils.py @@ -1039,12 +1039,12 @@ class IsoBuild: self.log.info(Color.INFO + 'Building ' + i + ' completed') if len(bad_exit_list) == 0: - self.log.info(Color.INFO + 'Copying ISOs over to compose directory...') + self.log.info(Color.INFO + 'Images built successfully.') else: self.log.error( Color.FAIL + 'There were issues with the work done. As a result, ' + - 'the ISOs will not be copied.' + 'some/all ISOs may not exist.' ) @@ -1425,10 +1425,12 @@ class LiveBuild: compose_dir_is_here: bool = False, hashed: bool = False, image=None, + justcopyit: bool = False, logger=None ): self.image = image + self.justcopyit = justcopyit self.fullname = rlvars['fullname'] self.distname = config['distname'] self.shortname = config['shortname'] @@ -1523,6 +1525,14 @@ class LiveBuild: ) self.log.info(self.revision) + if not os.path.exists(self.compose_latest_dir): + self.log.warn(Color.WARN + 'A compose directory was not found ' + + 'here. If there is a failure, it may be due to it ' + + 'missing. You may want to generate a fake compose if ' + + 'you are simply making your own live images and you run ' + + 'into any errors beyond this point.' + ) + def run_build_live_iso(self): """ Builds DVD images based on the data created from the initial lorax on @@ -1575,7 +1585,10 @@ class LiveBuild: raise SystemExit() if self.live_iso_mode == 'podman': - self._live_iso_podman_run(self.current_arch, images_to_build, work_root) + #self._live_iso_podman_run(self.current_arch, images_to_build, work_root) + self.log.error(Color.FAIL + 'At this time, live images cannot be ' + + 'built in podman.') + raise SystemExit() def _live_iso_local_config(self, image, work_root): """ @@ -1723,6 +1736,7 @@ class LiveBuild: bad_exit_list = [] checksum_list = [] entry_name_list = [] + self.log.warn(Color.WARN + 'This mode does not work properly. It will fail.') for i in images: image_name = i entry_name = 'buildLiveImage-{}-{}.sh'.format(arch, i) @@ -1821,12 +1835,12 @@ class LiveBuild: self.log.info(Color.INFO + 'Building live images completed') if len(bad_exit_list) == 0: - self.log.info(Color.INFO + 'Copying ISOs over to compose directory...') + self.log.info(Color.INFO + 'Live images completed successfully.') else: self.log.error( Color.FAIL + 'There were issues with the work done. As a result, ' + - 'the ISOs will not be copied.' + 'some or all ISOs may not be copied later.' ) def _live_iso_local_run(self, arch, image, work_root): @@ -1836,6 +1850,19 @@ class LiveBuild: have mock available. """ entries_dir = os.path.join(work_root, "entries") + live_dir_arch = os.path.join(self.live_work_dir, arch) + isoname = '{}-{}-{}-{}-{}.iso'.format( + self.shortname, + image, + self.release, + arch, + self.date + ) + live_res_dir = '/var/lib/mock/{}-{}-{}/result'.format( + self.shortname, + self.major_version, + arch + ) live_iso_cmd = '/bin/bash {}/liveisobuild-{}-{}.sh'.format(entries_dir, arch, image) self.log.info('Starting mock build...') p = subprocess.call(shlex.split(live_iso_cmd)) @@ -1850,6 +1877,21 @@ class LiveBuild: ) self.log.warn( Color.WARN + - 'If you are looping images, your built image WILL get ' + - 'overwritten.' + 'If you are looping images, your built image may get ' + + 'overwritten. Ensure you have justcopyit enabled to avoid this.' ) + + if self.justcopyit: + self.log.info(Color.INFO + 'Copying image to work directory') + source_path = os.path.join(live_res_dir, isoname) + dest_path = os.path.join(live_dir_arch, isoname) + os.makedirs(live_dir_arch, exist_ok=True) + shutil.copy2(source_path, dest_path) + self.log.info(Color.INFO + 'Generating checksum') + checksum = Shared.get_checksum(dest_path, self.checksum, self.log) + if not checksum: + self.log.error(Color.FAIL + dest_path + ' not found. Did we copy it?') + return + with open(dest_path + '.CHECKSUM', "w+") as c: + c.write(checksum) + c.close() diff --git a/iso/empanadas/empanadas/util/shared.py b/iso/empanadas/empanadas/util/shared.py index 50894e8..65c72c6 100644 --- a/iso/empanadas/empanadas/util/shared.py +++ b/iso/empanadas/empanadas/util/shared.py @@ -319,6 +319,10 @@ class Shared: logger.info('Creating compose directory %s' % compose_base_dir) if not os.path.exists(compose_base_dir): os.makedirs(compose_base_dir) + os.makedirs(compose_base_dir + '/work') + os.makedirs(compose_base_dir + '/work/entries') + os.makedirs(compose_base_dir + '/work/logs') + os.makedirs(compose_base_dir + '/compose') return compose_base_dir diff --git a/iso/empanadas/pyproject.toml b/iso/empanadas/pyproject.toml index 0376d8a..96b5288 100644 --- a/iso/empanadas/pyproject.toml +++ b/iso/empanadas/pyproject.toml @@ -33,6 +33,7 @@ launch-builds = "empanadas.scripts.launch_builds:run" build-image = "empanadas.scripts.build_image:run" finalize_compose = "empanadas.scripts.finalize_compose:run" pull-cloud-image = "empanadas.scripts.pull_cloud_image:run" +generate_compose = "empanadas.scripts.generate_compose:run" [build-system] requires = ["poetry-core>=1.0.0"]