forked from sig_core/toolkit
update extra files
This commit is contained in:
parent
5667a75fa2
commit
ab7c36e02c
@ -10,6 +10,7 @@ import os
|
|||||||
import os.path
|
import os.path
|
||||||
import subprocess
|
import subprocess
|
||||||
import shlex
|
import shlex
|
||||||
|
import shutil
|
||||||
import time
|
import time
|
||||||
import re
|
import re
|
||||||
import json
|
import json
|
||||||
@ -41,6 +42,7 @@ class RepoSync:
|
|||||||
ignore_debug: bool = False,
|
ignore_debug: bool = False,
|
||||||
ignore_source: bool = False,
|
ignore_source: bool = False,
|
||||||
repoclosure: bool = False,
|
repoclosure: bool = False,
|
||||||
|
refresh_extra_files: bool = False,
|
||||||
skip_all: bool = False,
|
skip_all: bool = False,
|
||||||
hashed: bool = False,
|
hashed: bool = False,
|
||||||
parallel: bool = False,
|
parallel: bool = False,
|
||||||
@ -60,6 +62,7 @@ class RepoSync:
|
|||||||
self.skip_all = skip_all
|
self.skip_all = skip_all
|
||||||
self.hashed = hashed
|
self.hashed = hashed
|
||||||
self.repoclosure = repoclosure
|
self.repoclosure = repoclosure
|
||||||
|
self.refresh_extra_files = refresh_extra_files
|
||||||
# Enables podman syncing, which should effectively speed up operations
|
# Enables podman syncing, which should effectively speed up operations
|
||||||
self.parallel = parallel
|
self.parallel = parallel
|
||||||
# Relevant config items
|
# Relevant config items
|
||||||
@ -117,6 +120,11 @@ class RepoSync:
|
|||||||
"work/logs"
|
"work/logs"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
self.compose_global_work_dir = os.path.join(
|
||||||
|
self.compose_latest_dir,
|
||||||
|
"work/global"
|
||||||
|
)
|
||||||
|
|
||||||
# This is temporary for now.
|
# This is temporary for now.
|
||||||
if logger is None:
|
if logger is None:
|
||||||
self.log = logging.getLogger("reposync")
|
self.log = logging.getLogger("reposync")
|
||||||
@ -186,6 +194,11 @@ class RepoSync:
|
|||||||
self.date_stamp
|
self.date_stamp
|
||||||
)
|
)
|
||||||
|
|
||||||
|
global_work_root = os.path.join(
|
||||||
|
work_root,
|
||||||
|
"global",
|
||||||
|
)
|
||||||
|
|
||||||
if self.dryrun:
|
if self.dryrun:
|
||||||
self.log.error('Dry Runs are not supported just yet. Sorry!')
|
self.log.error('Dry Runs are not supported just yet. Sorry!')
|
||||||
raise SystemExit()
|
raise SystemExit()
|
||||||
@ -193,12 +206,15 @@ class RepoSync:
|
|||||||
self.sync(self.repo, sync_root, work_root, log_root, self.arch)
|
self.sync(self.repo, sync_root, work_root, log_root, self.arch)
|
||||||
|
|
||||||
if self.fullrun:
|
if self.fullrun:
|
||||||
self.deploy_extra_files()
|
self.deploy_extra_files(global_work_root)
|
||||||
self.symlink_to_latest(generated_dir)
|
self.symlink_to_latest(generated_dir)
|
||||||
|
|
||||||
if self.repoclosure:
|
if self.repoclosure:
|
||||||
self.repoclosure_work(sync_root, work_root, log_root)
|
self.repoclosure_work(sync_root, work_root, log_root)
|
||||||
|
|
||||||
|
if self.refresh_extra_files:
|
||||||
|
self.deploy_extra_files(global_work_root)
|
||||||
|
|
||||||
self.log.info('Compose repo directory: %s' % sync_root)
|
self.log.info('Compose repo directory: %s' % sync_root)
|
||||||
self.log.info('Compose logs: %s' % log_root)
|
self.log.info('Compose logs: %s' % log_root)
|
||||||
self.log.info('Compose completed.')
|
self.log.info('Compose completed.')
|
||||||
@ -240,10 +256,14 @@ class RepoSync:
|
|||||||
bad_exit_list = []
|
bad_exit_list = []
|
||||||
self.log.info('Generating container entries')
|
self.log.info('Generating container entries')
|
||||||
entries_dir = os.path.join(work_root, "entries")
|
entries_dir = os.path.join(work_root, "entries")
|
||||||
|
global_work_root = os.path.join(work_root, "global")
|
||||||
if not os.path.exists(entries_dir):
|
if not os.path.exists(entries_dir):
|
||||||
os.makedirs(entries_dir, exist_ok=True)
|
os.makedirs(entries_dir, exist_ok=True)
|
||||||
|
|
||||||
# yeah, I know.
|
# yeah, I know.
|
||||||
|
if not os.path.exists(global_work_root):
|
||||||
|
os.makedirs(global_work_root, exist_ok=True)
|
||||||
|
|
||||||
if not os.path.exists(log_root):
|
if not os.path.exists(log_root):
|
||||||
os.makedirs(log_root, exist_ok=True)
|
os.makedirs(log_root, exist_ok=True)
|
||||||
|
|
||||||
@ -648,6 +668,22 @@ class RepoSync:
|
|||||||
)
|
)
|
||||||
return cmd
|
return cmd
|
||||||
|
|
||||||
|
def git_cmd(self) -> str:
|
||||||
|
"""
|
||||||
|
This generates the git command. This is when we need to pull down extra
|
||||||
|
files or do work from a git repository.
|
||||||
|
"""
|
||||||
|
cmd = None
|
||||||
|
if os.path.exists("/usr/bin/git"):
|
||||||
|
cmd = "/usr/bin/git"
|
||||||
|
else:
|
||||||
|
self.log.error('/usr/bin/git was not found. Good bye.')
|
||||||
|
raise SystemExit("\n\n/usr/bin/git was not found.\n\nPlease "
|
||||||
|
" ensure that you have installed the necessary packages on "
|
||||||
|
" this system. "
|
||||||
|
)
|
||||||
|
return cmd
|
||||||
|
|
||||||
def repoclosure_work(self, sync_root, work_root, log_root):
|
def repoclosure_work(self, sync_root, work_root, log_root):
|
||||||
"""
|
"""
|
||||||
This is where we run repoclosures, based on the configuration of each
|
This is where we run repoclosures, based on the configuration of each
|
||||||
@ -806,7 +842,7 @@ class RepoSync:
|
|||||||
for issue in bad_exit_list:
|
for issue in bad_exit_list:
|
||||||
self.log.error(issue)
|
self.log.error(issue)
|
||||||
|
|
||||||
def deploy_extra_files(self):
|
def deploy_extra_files(self, global_work_dir):
|
||||||
"""
|
"""
|
||||||
deploys extra files based on info of rlvars including a
|
deploys extra files based on info of rlvars including a
|
||||||
extra_files.json
|
extra_files.json
|
||||||
@ -814,7 +850,60 @@ class RepoSync:
|
|||||||
also deploys COMPOSE_ID and maybe in the future a metadata dir with a
|
also deploys COMPOSE_ID and maybe in the future a metadata dir with a
|
||||||
bunch of compose-esque stuff.
|
bunch of compose-esque stuff.
|
||||||
"""
|
"""
|
||||||
self.log.info('Deploying extra files...')
|
cmd = self.git_cmd()
|
||||||
|
tmpclone = '/tmp/clone'
|
||||||
|
extra_files_dir = os.path.join(
|
||||||
|
global_work_dir,
|
||||||
|
'extra-files'
|
||||||
|
)
|
||||||
|
self.log.info(
|
||||||
|
'[' + Color.BOLD + Color.GREEN + 'INFO' + Color.END + '] ' +
|
||||||
|
'Deploying extra files to work directory ...'
|
||||||
|
)
|
||||||
|
|
||||||
|
if not os.path.exists(global_work_dir):
|
||||||
|
os.makedirs(global_work_dir, exist_ok=True)
|
||||||
|
|
||||||
|
clonecmd = '{} clone {} -b {} -q {}'.format(
|
||||||
|
cmd,
|
||||||
|
self.extra_files['git_repo'],
|
||||||
|
self.extra_files['branch'],
|
||||||
|
tmpclone
|
||||||
|
)
|
||||||
|
|
||||||
|
git_clone = subprocess.Popen(
|
||||||
|
clonecmd,
|
||||||
|
stdout=subprocess.DEVNULL,
|
||||||
|
stderr=subprocess.DEVNULL,
|
||||||
|
shell=True
|
||||||
|
)
|
||||||
|
|
||||||
|
# Copy files
|
||||||
|
for extra in self.extra_files['list']:
|
||||||
|
src = '/tmp/clone/' + extra
|
||||||
|
try:
|
||||||
|
shutil.copy2(src, extra_files_dir)
|
||||||
|
except:
|
||||||
|
self.log.warn(
|
||||||
|
'[' + Color.BOLD + Color.YELLOW + 'WARN' + Color.END + '] ' +
|
||||||
|
'Extra file not copied: ' + src
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
shutil.rmtree(tmpclone)
|
||||||
|
except OSError as e:
|
||||||
|
self.log.error(
|
||||||
|
'[' + Color.BOLD + Color.RED + 'FAIL' + Color.END + '] ' +
|
||||||
|
'Directory ' + tmpclone + ' could not be removed: ' +
|
||||||
|
e.strerror
|
||||||
|
)
|
||||||
|
|
||||||
|
# Create metadata here?
|
||||||
|
|
||||||
|
self.log.info(
|
||||||
|
'[' + Color.BOLD + Color.GREEN + 'INFO' + Color.END + '] ' +
|
||||||
|
'Extra files phase completed.'
|
||||||
|
)
|
||||||
|
|
||||||
class SigRepoSync:
|
class SigRepoSync:
|
||||||
"""
|
"""
|
||||||
@ -831,6 +920,7 @@ class SigRepoSync:
|
|||||||
arch=None,
|
arch=None,
|
||||||
ignore_source: bool = False,
|
ignore_source: bool = False,
|
||||||
repoclosure: bool = False,
|
repoclosure: bool = False,
|
||||||
|
refresh_extra_files: bool = False,
|
||||||
skip_all: bool = False,
|
skip_all: bool = False,
|
||||||
hashed: bool = False,
|
hashed: bool = False,
|
||||||
parallel: bool = False,
|
parallel: bool = False,
|
||||||
@ -847,6 +937,7 @@ class SigRepoSync:
|
|||||||
self.skip_all = skip_all
|
self.skip_all = skip_all
|
||||||
self.hashed = hashed
|
self.hashed = hashed
|
||||||
self.repoclosure = repoclosure
|
self.repoclosure = repoclosure
|
||||||
|
self.refresh_extra_files = refresh_extra_files
|
||||||
# Enables podman syncing, which should effectively speed up operations
|
# Enables podman syncing, which should effectively speed up operations
|
||||||
self.parallel = parallel
|
self.parallel = parallel
|
||||||
# Relevant config items
|
# Relevant config items
|
||||||
@ -894,6 +985,11 @@ class SigRepoSync:
|
|||||||
"work/logs"
|
"work/logs"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
self.compose_global_work_dir = os.path.join(
|
||||||
|
self.compose_latest_dir,
|
||||||
|
"work/global"
|
||||||
|
)
|
||||||
|
|
||||||
# This is temporary for now.
|
# This is temporary for now.
|
||||||
if logger is None:
|
if logger is None:
|
||||||
self.log = logging.getLogger("sigreposync")
|
self.log = logging.getLogger("sigreposync")
|
||||||
|
Loading…
Reference in New Issue
Block a user