From b376c168bb5e5609fb486407ac38b6741a007a5a Mon Sep 17 00:00:00 2001 From: nazunalika Date: Sat, 11 Jun 2022 10:49:37 -0700 Subject: [PATCH] start using templates for reposync --- iso/py/sync-from-peridot-test | 4 +- iso/py/templates/repoconfig.tmpl | 20 ++++++ iso/py/templates/reposync-src.tmpl | 4 ++ iso/py/templates/reposync.tmpl | 5 ++ iso/py/util/dnf_utils.py | 97 +++++++++++++++--------------- 5 files changed, 78 insertions(+), 52 deletions(-) create mode 100644 iso/py/templates/repoconfig.tmpl create mode 100644 iso/py/templates/reposync-src.tmpl create mode 100644 iso/py/templates/reposync.tmpl diff --git a/iso/py/sync-from-peridot-test b/iso/py/sync-from-peridot-test index 34cd4b6..b8db670 100755 --- a/iso/py/sync-from-peridot-test +++ b/iso/py/sync-from-peridot-test @@ -12,5 +12,5 @@ r = Checks(rlvars, config['arch']) r.check_valid_arch() #a = RepoSync(rlvars, config, major="9", repo="ResilientStorage", parallel=True, ignore_debug=False, ignore_source=False) -a = RepoSync(rlvars, config, major="9", repo="ResilientStorage", parallel=True, ignore_debug=False, ignore_source=False) -a.run() +a = RepoSync(rlvars, config, major="9", repo="ResilientStorage", parallel=True, ignore_debug=False, ignore_source=False, hashed=True) +#a.run() diff --git a/iso/py/templates/repoconfig.tmpl b/iso/py/templates/repoconfig.tmpl new file mode 100644 index 0000000..04da1d8 --- /dev/null +++ b/iso/py/templates/repoconfig.tmpl @@ -0,0 +1,20 @@ +{%- for repo in repos -%} +[{{ repo.name }}] +name={{repo.name}} +baseurl={{ repo.baseurl }} +enabled=1 +gpgcheck=0 + +[{{ repo.name }}-debug] +name={{repo.name}} +baseurl={{ repo.baseurl }}-debug +enabled=1 +gpgcheck=0 + +[{{ repo.name }}-source] +name={{repo.name}} +baseurl={{ repo.srcbaseurl }} +enabled=1 +gpgcheck=0 + +{% endfor %} diff --git a/iso/py/templates/reposync-src.tmpl b/iso/py/templates/reposync-src.tmpl new file mode 100644 index 0000000..a0b634d --- /dev/null +++ b/iso/py/templates/reposync-src.tmpl @@ -0,0 +1,4 @@ +#!/bin/bash +set -o pipefail +{{ dnf_plugin_cmd }} +{{ sync_cmd }} diff --git a/iso/py/templates/reposync.tmpl b/iso/py/templates/reposync.tmpl new file mode 100644 index 0000000..4b5306f --- /dev/null +++ b/iso/py/templates/reposync.tmpl @@ -0,0 +1,5 @@ +#!/bin/bash +set -o pipefail +{{ arch_force_cp }} +{{ dnf_plugin_cmd }} +{{ sync_cmd }} diff --git a/iso/py/util/dnf_utils.py b/iso/py/util/dnf_utils.py index 062ad6b..c4fce49 100644 --- a/iso/py/util/dnf_utils.py +++ b/iso/py/util/dnf_utils.py @@ -14,6 +14,7 @@ import time import re #import pipes from common import Color +from jinja2 import Environment, FileSystemLoader #HAS_LIBREPO = True #try: @@ -73,6 +74,10 @@ class RepoSync: self.repo = repo self.extra_files = rlvars['extra_files'] + # Templates + file_loader = FileSystemLoader('templates') + self.tmplenv = Environment(loader=file_loader) + # each el can have its own designated container to run stuff in, # otherwise we'll just default to the default config. self.container = config['container'] @@ -300,18 +305,22 @@ class RepoSync: a )) + sync_log = ("{}/{}-{}-{}.log").format( + log_root, + repo_name, + a, + self.date_stamp + ) + sync_cmd = ("/usr/bin/dnf reposync -c {}.{} --download-metadata " "--repoid={} -p {} --forcearch {} --norepopath 2>&1 " - "| tee -a {}/{}-{}-{}.log").format( + "| tee -a {}").format( self.dnf_config, a, r, os_sync_path, a, - log_root, - repo_name, - a, - self.date_stamp + sync_log, ) debug_sync_cmd = ("/usr/bin/dnf reposync -c {}.{} " @@ -345,20 +354,25 @@ class RepoSync: self.date_stamp ) + sync_template = self.tmplenv.get_template('reposync.tmpl') + sync_output = sync_template.render( + arch_force_cp=arch_force_cp, + dnf_plugin_cmd=dnf_plugin_cmd, + sync_cmd=sync_cmd + ) + + debug_sync_template = self.tmplenv.get_template('reposync.tmpl') + debug_sync_output = debug_sync_template.render( + arch_force_cp=arch_force_cp, + dnf_plugin_cmd=debug_dnf_plugin_cmd, + sync_cmd=debug_sync_cmd + ) + entry_point_open = open(entry_point_sh, "w+") debug_entry_point_open = open(debug_entry_point_sh, "w+") - entry_point_open.write('#!/bin/bash\n') - entry_point_open.write('set -o pipefail\n') - entry_point_open.write(arch_force_cp + '\n') - entry_point_open.write(dnf_plugin_cmd + '\n') - entry_point_open.write(sync_cmd + '\n') - - debug_entry_point_open.write('#!/bin/bash\n') - debug_entry_point_open.write('set -o pipefail\n') - debug_entry_point_open.write(arch_force_cp + '\n') - debug_entry_point_open.write(debug_dnf_plugin_cmd + '\n') - debug_entry_point_open.write(debug_sync_cmd + '\n') + entry_point_open.write(sync_output) + debug_entry_point_open.write(debug_sync_output) entry_point_open.close() debug_entry_point_open.close() @@ -400,11 +414,14 @@ class RepoSync: self.date_stamp ) + source_sync_template = self.tmplenv.get_template('reposync-src.tmpl') + source_sync_output = source_sync_template.render( + dnf_plugin_cmd=source_dnf_plugin_cmd, + sync_cmd=source_sync_cmd + ) + source_entry_point_open = open(source_entry_point_sh, "w+") - source_entry_point_open.write('#!/bin/bash\n') - source_entry_point_open.write('set -o pipefail\n') - source_entry_point_open.write(source_dnf_plugin_cmd + '\n') - source_entry_point_open.write(source_sync_cmd + '\n') + source_entry_point_open.write(source_sync_output) source_entry_point_open.close() os.chmod(source_entry_point_sh, 0o755) @@ -545,11 +562,9 @@ class RepoSync: if not os.path.exists(dest_path): os.makedirs(dest_path, exist_ok=True) config_file = open(fname, "w+") + repolist = [] for repo in self.repos: - #if 'all' in repo: - # prehashed = '' - constructed_url = '{}/{}/repo/{}{}/$basearch'.format( self.repo_base_url, self.project_id, @@ -557,13 +572,6 @@ class RepoSync: repo, ) - constructed_url_debug = '{}/{}/repo/{}{}/$basearch-debug'.format( - self.repo_base_url, - self.project_id, - prehashed, - repo, - ) - constructed_url_src = '{}/{}/repo/{}{}/src'.format( self.repo_base_url, self.project_id, @@ -571,27 +579,16 @@ class RepoSync: repo, ) - # normal - config_file.write('[%s]\n' % repo) - config_file.write('name=%s\n' % repo) - config_file.write('baseurl=%s\n' % constructed_url) - config_file.write("enabled=1\n") - config_file.write("gpgcheck=0\n\n") - - # debug - config_file.write('[%s-debug]\n' % repo) - config_file.write('name=%s debug\n' % repo) - config_file.write('baseurl=%s\n' % constructed_url_debug) - config_file.write("enabled=1\n") - config_file.write("gpgcheck=0\n\n") - - # src - config_file.write('[%s-source]\n' % repo) - config_file.write('name=%s source\n' % repo) - config_file.write('baseurl=%s\n' % constructed_url_src) - config_file.write("enabled=1\n") - config_file.write("gpgcheck=0\n\n") + repodata = { + 'name': repo, + 'baseurl': constructed_url, + 'srcbaseurl': constructed_url_src + } + repolist.append(repodata) + template = self.tmplenv.get_template('repoconfig.tmpl') + output = template.render(repos=repolist) + config_file.write(output) config_file.close() return fname