update extra files

This commit is contained in:
Louis Abel 2022-06-21 23:03:08 -07:00
parent 5667a75fa2
commit ab7c36e02c
Signed by untrusted user: label
GPG Key ID: B37E62D143879B36

View File

@ -10,6 +10,7 @@ import os
import os.path
import subprocess
import shlex
import shutil
import time
import re
import json
@ -41,6 +42,7 @@ class RepoSync:
ignore_debug: bool = False,
ignore_source: bool = False,
repoclosure: bool = False,
refresh_extra_files: bool = False,
skip_all: bool = False,
hashed: bool = False,
parallel: bool = False,
@ -60,6 +62,7 @@ class RepoSync:
self.skip_all = skip_all
self.hashed = hashed
self.repoclosure = repoclosure
self.refresh_extra_files = refresh_extra_files
# Enables podman syncing, which should effectively speed up operations
self.parallel = parallel
# Relevant config items
@ -117,6 +120,11 @@ class RepoSync:
"work/logs"
)
self.compose_global_work_dir = os.path.join(
self.compose_latest_dir,
"work/global"
)
# This is temporary for now.
if logger is None:
self.log = logging.getLogger("reposync")
@ -186,6 +194,11 @@ class RepoSync:
self.date_stamp
)
global_work_root = os.path.join(
work_root,
"global",
)
if self.dryrun:
self.log.error('Dry Runs are not supported just yet. Sorry!')
raise SystemExit()
@ -193,12 +206,15 @@ class RepoSync:
self.sync(self.repo, sync_root, work_root, log_root, self.arch)
if self.fullrun:
self.deploy_extra_files()
self.deploy_extra_files(global_work_root)
self.symlink_to_latest(generated_dir)
if self.repoclosure:
self.repoclosure_work(sync_root, work_root, log_root)
if self.refresh_extra_files:
self.deploy_extra_files(global_work_root)
self.log.info('Compose repo directory: %s' % sync_root)
self.log.info('Compose logs: %s' % log_root)
self.log.info('Compose completed.')
@ -240,10 +256,14 @@ class RepoSync:
bad_exit_list = []
self.log.info('Generating container entries')
entries_dir = os.path.join(work_root, "entries")
global_work_root = os.path.join(work_root, "global")
if not os.path.exists(entries_dir):
os.makedirs(entries_dir, exist_ok=True)
# yeah, I know.
if not os.path.exists(global_work_root):
os.makedirs(global_work_root, exist_ok=True)
if not os.path.exists(log_root):
os.makedirs(log_root, exist_ok=True)
@ -648,6 +668,22 @@ class RepoSync:
)
return cmd
def git_cmd(self) -> str:
"""
This generates the git command. This is when we need to pull down extra
files or do work from a git repository.
"""
cmd = None
if os.path.exists("/usr/bin/git"):
cmd = "/usr/bin/git"
else:
self.log.error('/usr/bin/git was not found. Good bye.')
raise SystemExit("\n\n/usr/bin/git was not found.\n\nPlease "
" ensure that you have installed the necessary packages on "
" this system. "
)
return cmd
def repoclosure_work(self, sync_root, work_root, log_root):
"""
This is where we run repoclosures, based on the configuration of each
@ -806,7 +842,7 @@ class RepoSync:
for issue in bad_exit_list:
self.log.error(issue)
def deploy_extra_files(self):
def deploy_extra_files(self, global_work_dir):
"""
deploys extra files based on info of rlvars including a
extra_files.json
@ -814,7 +850,60 @@ class RepoSync:
also deploys COMPOSE_ID and maybe in the future a metadata dir with a
bunch of compose-esque stuff.
"""
self.log.info('Deploying extra files...')
cmd = self.git_cmd()
tmpclone = '/tmp/clone'
extra_files_dir = os.path.join(
global_work_dir,
'extra-files'
)
self.log.info(
'[' + Color.BOLD + Color.GREEN + 'INFO' + Color.END + '] ' +
'Deploying extra files to work directory ...'
)
if not os.path.exists(global_work_dir):
os.makedirs(global_work_dir, exist_ok=True)
clonecmd = '{} clone {} -b {} -q {}'.format(
cmd,
self.extra_files['git_repo'],
self.extra_files['branch'],
tmpclone
)
git_clone = subprocess.Popen(
clonecmd,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
shell=True
)
# Copy files
for extra in self.extra_files['list']:
src = '/tmp/clone/' + extra
try:
shutil.copy2(src, extra_files_dir)
except:
self.log.warn(
'[' + Color.BOLD + Color.YELLOW + 'WARN' + Color.END + '] ' +
'Extra file not copied: ' + src
)
try:
shutil.rmtree(tmpclone)
except OSError as e:
self.log.error(
'[' + Color.BOLD + Color.RED + 'FAIL' + Color.END + '] ' +
'Directory ' + tmpclone + ' could not be removed: ' +
e.strerror
)
# Create metadata here?
self.log.info(
'[' + Color.BOLD + Color.GREEN + 'INFO' + Color.END + '] ' +
'Extra files phase completed.'
)
class SigRepoSync:
"""
@ -831,6 +920,7 @@ class SigRepoSync:
arch=None,
ignore_source: bool = False,
repoclosure: bool = False,
refresh_extra_files: bool = False,
skip_all: bool = False,
hashed: bool = False,
parallel: bool = False,
@ -847,6 +937,7 @@ class SigRepoSync:
self.skip_all = skip_all
self.hashed = hashed
self.repoclosure = repoclosure
self.refresh_extra_files = refresh_extra_files
# Enables podman syncing, which should effectively speed up operations
self.parallel = parallel
# Relevant config items
@ -894,6 +985,11 @@ class SigRepoSync:
"work/logs"
)
self.compose_global_work_dir = os.path.join(
self.compose_latest_dir,
"work/global"
)
# This is temporary for now.
if logger is None:
self.log = logging.getLogger("sigreposync")