diff --git a/iso/empanadas/empanadas/scripts/sync_sig.py b/iso/empanadas/empanadas/scripts/sync_sig.py index 67919c6..8602d4d 100755 --- a/iso/empanadas/empanadas/scripts/sync_sig.py +++ b/iso/empanadas/empanadas/scripts/sync_sig.py @@ -55,6 +55,7 @@ a = SigRepoSync( dryrun=results.dry_run, fullrun=results.full_run, nofail=results.no_fail, + refresh_extra_files=results.refresh_extra_files, logger=results.logger ) diff --git a/iso/empanadas/empanadas/util/dnf_utils.py b/iso/empanadas/empanadas/util/dnf_utils.py index 626b292..d9205fe 100644 --- a/iso/empanadas/empanadas/util/dnf_utils.py +++ b/iso/empanadas/empanadas/util/dnf_utils.py @@ -292,15 +292,7 @@ class RepoSync: if self.parallel: self.podman_sync(repo, sync_root, work_root, log_root, global_work_root, arch) else: - self.dnf_sync(repo, sync_root, work_root, arch) - - def dnf_sync(self, repo, sync_root, work_root, arch): - """ - This is for normal dnf syncs. This is very slow. - """ - self.log.error('DNF syncing has been removed.') - self.log.error('Please install podman and enable parallel') - raise SystemExit() + Shared.dnf_sync(repo, sync_root, work_root, arch, self.log) def podman_sync( self, @@ -1516,7 +1508,7 @@ class SigRepoSync: self.repo_base_url = config['repo_base_url'] self.compose_root = config['compose_root'] self.compose_base = config['compose_root'] + "/" + major - self.profile = rlvars['profile'] + self.profile = sigvars['profile'] self.sigprofile = sigvars['profile'] self.iso_map = rlvars['iso_map'] self.distname = config['distname'] @@ -1536,6 +1528,12 @@ class SigRepoSync: if 'additional_vars' in sigvars: self.additional_dirs = sigvars['additional_dirs'] + self.compose_id = '{}-{}-{}'.format( + self.profile, + self.major_version, + config['date_stamp'] + ) + # Templates file_loader = FileSystemLoader(f"{_rootdir}/templates") self.tmplenv = Environment(loader=file_loader) @@ -1593,7 +1591,7 @@ class SigRepoSync: self.log.addHandler(handler) self.log.info('sig reposync init') - self.log.info(major) + self.log.info(self.profile + ' ' + self.major_version) # The repo name should be valid if self.sigrepo is not None: @@ -1662,9 +1660,23 @@ class SigRepoSync: sig_sync_root = os.path.join( sync_root, + self.major_version, self.sigprofile ) + self.dnf_config = Shared.generate_conf( + self.profile, + self.major_version, + self.sigrepos, + self.repo_base_url, + self.project_id, + self.hashed, + self.extra_files, + self.gpgkey, + self.tmplenv, + self.log + ) + # dnf config here if self.dryrun: self.log.error('Dry Runs are not supported just yet. Sorry!') @@ -1673,15 +1685,395 @@ class SigRepoSync: if self.fullrun and self.refresh_extra_files: self.log.warn(Color.WARN + 'A full run implies extra files are also deployed.') - #self.sync(self.repo, sync_root, work_root, log_root, global_work_root, self.arch) + self.sync(self.sigrepo, sync_root, work_root, log_root, global_work_root, self.arch) if self.fullrun: Shared.deploy_extra_files(self.extra_files, sig_sync_root, global_work_root, self.log) Shared.symlink_to_latest(simplename, self.major_version, generated_dir, self.compose_latest_dir, self.log) - print() if self.refresh_extra_files and not self.fullrun: Shared.deploy_extra_files(self.extra_files, sig_sync_root, global_work_root, self.log) - print() + def sync(self, repo, sync_root, work_root, log_root, global_work_root, arch=None): + """ + Calls out syncing of the repos. We generally sync each component of a + repo: + * each arch + * each arch debug + * each source + + If paralel is true, we will run in podman. + """ + # I think we need to do a bit of leg work here, there is a chance that + # a sig may have repos that have repos that are not applicable to all + # arches... + if self.parallel: + self.podman_sync(repo, sync_root, work_root, log_root, global_work_root, arch) + else: + Shared.dnf_sync(repo, sync_root, work_root, arch, self.log) + + + def podman_sync( + self, + repo, + sync_root, + work_root, + log_root, + global_work_root, + arch + ): + """ + This is for podman syncs + + Create sync/root/work/entries + Generate scripts as needed into dir + Each container runs their own script + wait till all is finished + """ + cmd = Shared.podman_cmd(self.log) + bad_exit_list = [] + self.log.info('Generating container entries') + entries_dir = os.path.join(work_root, "entries") + if not os.path.exists(entries_dir): + os.makedirs(entries_dir, exist_ok=True) + + # yeah, I know. + if not os.path.exists(global_work_root): + os.makedirs(global_work_root, exist_ok=True) + + if not os.path.exists(log_root): + os.makedirs(log_root, exist_ok=True) + + repos_to_sync = self.sigrepos + if repo and not self.fullrun: + repos_to_sync = [repo] + + for r in repos_to_sync: + entry_name_list = [] + repo_name = r + # Each repo can have specific allowed arches, based on the request + # of the SIG. What we also want to make sure is that if an arch was + # asked for but a repo (regardless if we are choosing a repo or not) + # we have to pass it with a warning. + arch_sync = self.sigvars['repo'][r]['allowed_arches'].copy() + if arch: + arch_sync = [arch] + + for a in arch_sync: + entry_name = '{}-{}'.format(r, a) + debug_entry_name = '{}-debug-{}'.format(r, a) + + entry_name_list.append(entry_name) + if not self.ignore_debug and not a == 'source': + entry_name_list.append(debug_entry_name) + + entry_point_sh = os.path.join( + entries_dir, + entry_name + ) + + debug_entry_point_sh = os.path.join( + entries_dir, + debug_entry_name + ) + + os_sync_path = os.path.join( + sync_root, + self.major_version, + self.profile, + a, + r + ) + + debug_sync_path = os.path.join( + sync_root, + self.major_version, + self.profile, + a, + r + '-debug' + ) + + import_gpg_cmd = ("/usr/bin/rpm --import {}{}").format( + self.extra_files['git_raw_path'], + self.extra_files['gpg'][self.gpgkey] + ) + + arch_force_cp = ("/usr/bin/sed 's|$basearch|{}|g' {} > {}.{}".format( + a, + self.dnf_config, + self.dnf_config, + a + )) + + sync_log = ("{}/{}-{}.log").format( + log_root, + repo_name, + a + ) + + debug_sync_log = ("{}/{}-{}-debug.log").format( + log_root, + repo_name, + a + ) + + metadata_cmd = ("/usr/bin/dnf makecache -c {}.{} --repoid={} " + "--forcearch {} --assumeyes 2>&1").format( + self.dnf_config, + a, + r, + a + ) + + sync_cmd = ("/usr/bin/dnf reposync -c {}.{} --download-metadata " + "--repoid={} -p {} --forcearch {} --norepopath --remote-time " + "--gpgcheck --assumeyes 2>&1").format( + self.dnf_config, + a, + r, + os_sync_path, + a + ) + + debug_metadata_cmd = ("/usr/bin/dnf makecache -c {}.{} --repoid={}-debug " + "--forcearch {} --assumeyes 2>&1").format( + self.dnf_config, + a, + r, + a + ) + + + debug_sync_cmd = ("/usr/bin/dnf reposync -c {}.{} " + "--download-metadata --repoid={}-debug -p {} --forcearch {} " + "--gpgcheck --norepopath --remote-time --assumeyes 2>&1").format( + self.dnf_config, + a, + r, + debug_sync_path, + a + ) + + dnf_plugin_cmd = "/usr/bin/dnf install dnf-plugins-core -y" + + sync_template = self.tmplenv.get_template('reposync.tmpl') + sync_output = sync_template.render( + import_gpg_cmd=import_gpg_cmd, + arch_force_cp=arch_force_cp, + dnf_plugin_cmd=dnf_plugin_cmd, + sync_cmd=sync_cmd, + metadata_cmd=metadata_cmd, + sync_log=sync_log, + download_path=os_sync_path + ) + + debug_sync_template = self.tmplenv.get_template('reposync.tmpl') + debug_sync_output = debug_sync_template.render( + import_gpg_cmd=import_gpg_cmd, + arch_force_cp=arch_force_cp, + dnf_plugin_cmd=dnf_plugin_cmd, + sync_cmd=debug_sync_cmd, + metadata_cmd=debug_metadata_cmd, + sync_log=debug_sync_log, + download_path=debug_sync_path + ) + + entry_point_open = open(entry_point_sh, "w+") + debug_entry_point_open = open(debug_entry_point_sh, "w+") + + entry_point_open.write(sync_output) + debug_entry_point_open.write(debug_sync_output) + + entry_point_open.close() + debug_entry_point_open.close() + + os.chmod(entry_point_sh, 0o755) + os.chmod(debug_entry_point_sh, 0o755) + + # We ignoring sources? + if (not self.ignore_source and not arch) or ( + not self.ignore_source and arch == 'source'): + source_entry_name = '{}-source'.format(r) + entry_name_list.append(source_entry_name) + + source_entry_point_sh = os.path.join( + entries_dir, + source_entry_name + ) + + source_sync_path = os.path.join( + sync_root, + self.major_version, + self.profile, + 'source', + r + ) + + source_sync_log = ("{}/{}-source.log").format( + log_root, + repo_name + ) + + source_metadata_cmd = ("/usr/bin/dnf makecache -c {} --repoid={}-source " + "--assumeyes 2>&1").format( + self.dnf_config, + r + ) + + source_sync_cmd = ("/usr/bin/dnf reposync -c {} " + "--download-metadata --repoid={}-source -p {} " + "--gpgcheck --norepopath --remote-time --assumeyes 2>&1").format( + self.dnf_config, + r, + source_sync_path + ) + + source_sync_template = self.tmplenv.get_template('reposync-src.tmpl') + source_sync_output = source_sync_template.render( + import_gpg_cmd=import_gpg_cmd, + dnf_plugin_cmd=dnf_plugin_cmd, + sync_cmd=source_sync_cmd, + metadata_cmd=source_metadata_cmd, + sync_log=source_sync_log + ) + + source_entry_point_open = open(source_entry_point_sh, "w+") + source_entry_point_open.write(source_sync_output) + source_entry_point_open.close() + os.chmod(source_entry_point_sh, 0o755) + + # Spawn up all podman processes for repo + self.log.info(Color.INFO + 'Starting podman processes for %s ...' % r) + + #print(entry_name_list) + for pod in entry_name_list: + podman_cmd_entry = '{} run -d -it -v "{}:{}" -v "{}:{}:z" -v "{}:{}" --name {} --entrypoint {}/{} {}'.format( + cmd, + self.compose_root, + self.compose_root, + self.dnf_config, + self.dnf_config, + entries_dir, + entries_dir, + pod, + entries_dir, + pod, + self.container + ) + #print(podman_cmd_entry) + process = subprocess.call( + shlex.split(podman_cmd_entry), + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL + ) + + join_all_pods = ' '.join(entry_name_list) + time.sleep(3) + self.log.info(Color.INFO + 'Syncing ' + r + ' ...') + self.log.info(Color.INFO + 'Arches: ' + ' '.join(arch_sync)) + pod_watcher = '{} wait {}'.format( + cmd, + join_all_pods + ) + + #print(pod_watcher) + watch_man = subprocess.call( + shlex.split(pod_watcher), + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL + ) + + # After the above is done, we'll check each pod process for an exit + # code. + pattern = "Exited (0)" + for pod in entry_name_list: + checkcmd = '{} ps -f status=exited -f name={}'.format( + cmd, + pod + ) + podcheck = subprocess.Popen( + checkcmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + shell=True + ) + + output, errors = podcheck.communicate() + if 'Exited (0)' not in output.decode(): + self.log.error(Color.FAIL + pod) + bad_exit_list.append(pod) + + rmcmd = '{} rm {}'.format( + cmd, + join_all_pods + ) + + rmpod = subprocess.Popen( + rmcmd, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + shell=True + ) + + entry_name_list.clear() + self.log.info(Color.INFO + 'Syncing ' + r + ' completed') + + if len(bad_exit_list) > 0: + self.log.error( + Color.BOLD + Color.RED + 'There were issues syncing these ' + 'repositories:' + Color.END + ) + for issue in bad_exit_list: + self.log.error(issue) + else: + self.log.info( + '[' + Color.BOLD + Color.GREEN + ' OK ' + Color.END + '] ' + 'No issues detected.' + ) + + def deploy_metadata(self, sync_root): + """ + Deploys metadata that defines information about the compose. Some data + will be close to how pungi produces it, but it won't be exact nor a + perfect replica. + """ + self.log.info(Color.INFO + 'Deploying metadata for this compose') + # Create metadata here + # Create COMPOSE_ID here (this doesn't necessarily match anything, it's + # just an indicator) + metadata_dir = os.path.join( + sync_root, + "metadata" + ) + + # It should already exist from a full run or refresh. This is just in + # case and it doesn't hurt. + if not os.path.exists(metadata_dir): + os.makedirs(metadata_dir, exist_ok=True) + + with open(metadata_dir + '/COMPOSE_ID', "w+") as f: + f.write(self.compose_id) + f.close() + + Shared.write_metadata( + self.timestamp, + self.date_stamp, + self.distname, + self.fullversion, + self.compose_id, + metadata_dir + '/metadata' + ) + + # TODO: Add in each repo and their corresponding arch. + productmd_date = self.date_stamp.split('.')[0] + Shared.composeinfo_write( + metadata_dir + '/composeinfo', + self.distname, + self.shortname, + self.fullversion, + 'updates', + productmd_date + ) + + self.log.info(Color.INFO + 'Metadata files phase completed.') diff --git a/iso/empanadas/empanadas/util/shared.py b/iso/empanadas/empanadas/util/shared.py index be8e94f..f0a4ea4 100644 --- a/iso/empanadas/empanadas/util/shared.py +++ b/iso/empanadas/empanadas/util/shared.py @@ -1058,3 +1058,13 @@ class Shared: logger.error(Color.FAIL + 'Directory ' + tmpclone + ' could not be removed: ' + e.strerror ) + + @staticmethod + def dnf_sync(repo, sync_root, work_root, arch, logger): + """ + This is for normal dnf syncs. This is very slow. + """ + logger.error('DNF syncing has been removed.') + logger.error('Please install podman and enable parallel') + raise SystemExit() +