peridot
/
pv2
mirror of https://github.com/peridotbuild/pv2.git
7
0
Fork 0
Code Issues 1 Packages Projects Releases Wiki Activity

Add module importer and git example 

* Add ModuleImport class to import module repos
* Add import_git.py example of using GitImport
This commit is contained in:
Louis Abel 2023-07-07 14:37:56 -07:00
parent edd6b41608
commit 60e210aaa6
Signed by: label
GPG Key ID: 3331F061D1D9990E
4 changed files with 307 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
examples/import_git.py Normal file
View File

@ -0,0 +1,32 @@
#!/usr/bin/python3
import argparse
import pv2.importer as importutil
parser = argparse.ArgumentParser(description="Importer")
parser.add_argument('--srpm', type=str, required=True)
parser.add_argument('--source-giturl', type=str, required=True)
parser.add_argument('--source-gitorg', type=str, required=True)
parser.add_argument('--branch', type=str, required=True)
parser.add_argument('--giturl', type=str, required=True)
parser.add_argument('--gitorg', type=str, required=False, default='rpms')
parser.add_argument('--dest-branch', type=str, required=False, default='')
parser.add_argument('--release', type=str, required=False, default='')
parser.add_argument('--distprefix', type=str, required=False, default='el')
parser.add_argument('--upstream-lookaside', type=str, required=True)
results = parser.parse_args()
classy = importutil.GitImport(
results.srpm,
source_git_url_path=results.source_giturl,
source_git_org_path=results.source_gitorg,
git_url_path=results.giturl,
org=results.gitorg,
release=results.release,
branch=results.branch,
dest_branch=results.dest_branch,
upstream_lookaside=results.upstream_lookaside,
distprefix=results.distprefix
)
classy.pkg_import()

2
examples/import_pkg.py
View File

@ -6,7 +6,7 @@ Import a source RPM into a git forge using pv2
import argparse
import pv2.importer as importutil
parser = argparse.ArgumentParser(description="ISO Compose")
parser = argparse.ArgumentParser(description="Importer")
parser.add_argument('--giturl', type=str, required=True)
parser.add_argument('--branch', type=str, required=True)

2
pv2/importer/__init__.py
View File

@ -7,4 +7,4 @@ This assists packagers by taking input as srpm or git location, importing and
tagging it as appropriate.
"""
from .operation import Import, SrpmImport, GitImport
from .operation import Import, SrpmImport, GitImport, ModuleImport

284
pv2/importer/operation.py
View File

@ -8,14 +8,24 @@ import os
import re
import shutil
import string
import datetime
from pv2.util import gitutil, fileutil, rpmutil, processor, generic
from pv2.util import error as err
from pv2.util import constants as const
#try:
# import gi
# gi.require_version('Modulemd', '2.0')
# from gi.repository import Modulemd
# HAS_GI = True
#except ImportError:
# HAS_GI = False
__all__ = [
'Import',
'SrpmImport',
'GitImport'
'GitImport',
'ModuleImport'
]
# todo: add in logging and replace print with log
@ -223,6 +233,35 @@ class Import:
except Exception as exc:
raise err.FileNotFound(f'{directory} could not be deleted. Please check. {exc}')
@staticmethod
def get_module_stream_name(source_branch):
"""
Returns a branch name for modules
"""
regex = r'stream-([a-zA-Z0-9_\.]+)-([a-zA-Z0-9_\.]+)'
regex_search = re.search(regex, source_branch)
return regex_search.group(2)
@staticmethod
def get_module_stream_version(release, source_branch, timestamp):
"""
Returns a code of major, minor, micro version if applicable
"""
if 'rhel' not in source_branch:
return f'{release}'
regex = r'rhel-([0-9]+)\.([0-9]+)\.([0-9]+)'
regex_search = re.search(regex, source_branch)
minor_version = regex_search.group(2)
micro_version = regex_search.group(3)
if len(regex_search.group(2)) == 1:
minor_version = f'0{regex_search.group(2)}'
if len(regex_search.group(3)) == 1:
micro_version = f'0{regex_search.group(3)}'
return f'{release}{minor_version}{micro_version}'
# pylint: disable=too-many-instance-attributes
class SrpmImport(Import):
"""
@ -534,7 +573,8 @@ class GitImport(Import):
# it'll be a module. Since this should always be the case, we'll change
# dest_branch to be: {dest_branch}-stream-{stream_name}
if "stream" in source_branch:
dest_branch = self.__get_module_stream_branch_name(source_branch, dest_branch)
_stream_name = self.get_module_stream_name(source_branch)
dest_branch = f'{dest_branch}-stream-{_stream_name}'
distmarker = self.dist_tag.lstrip('.')
_dist_tag = f'.module+{distmarker}+1010+deadbeef'
@ -676,15 +716,6 @@ class GitImport(Import):
substitute = template.substitute(dict_template)
return substitute
@staticmethod
def __get_module_stream_branch_name(source_branch, dest_branch):
"""
Returns a branch name for modules
"""
regex = r'stream-([a-zA-Z0-9_\.]+)-([a-zA-Z0-9_\.]+)'
regex_search = re.search(regex, source_branch)
return f'{dest_branch}-stream-{regex_search.group(2)}'
@property
def rpm_name(self):
"""
@ -755,3 +786,234 @@ class GitImport(Import):
Returns destination local lookaside
"""
return self.__dest_lookaside
class ModuleImport(Import):
"""
Imports module repos
"""
# This needs to clone whatever is there, find if there's a SOURCES
# directory, if not make it. Make changes to the YAML to point to the
# destination branch, copy it to SOURCES, make a metadata file.
# pylint: disable=too-many-arguments
def __init__(
self,
module: str,
source_git_url_path: str,
source_git_org_path: str,
git_url_path: str,
release: str,
branch: str,
source_git_protocol: str = 'https',
dest_branch: str = '',
distprefix: str = 'el',
git_user: str = 'git',
org: str = 'modules'
):
"""
Init the class
"""
#if not HAS_GI:
# raise err.GenericError('This class cannot be loaded due to missing modules.')
self.__module = module
self.__release = release
# pylint: disable=line-too-long
self.__source_git_url = f'{source_git_protocol}://{source_git_url_path}/{source_git_org_path}/{module}.git'
self.__git_url = f'ssh://{git_user}@{git_url_path}/{org}/{module}.git'
self.__dist_prefix = distprefix
self.__dist_tag = f'.{distprefix}{release}'
self.__branch = branch
self.__dest_branch = branch
self.__current_time = datetime.datetime.utcnow().strftime('%Y%m%d%H%M%S')
if len(dest_branch) > 0:
self.__dest_branch = dest_branch
if "stream" not in self.__branch:
raise err.ConfigurationError('Source branch does not contain stream')
self.__stream_name = self.get_module_stream_name(branch)
def module_import(self):
"""
Actually perform the import.
"""
check_source_repo = gitutil.lsremote(self.source_git_url)
check_dest_repo = gitutil.lsremote(self.dest_git_url)
source_git_repo_path = f'/var/tmp/{self.module_name}-source'
dest_git_repo_path = f'/var/tmp/{self.module_name}'
modulemd_file = f'{source_git_repo_path}/{self.module_name}.yaml'
metadata_file = f'{dest_git_repo_path}/.{self.module_name}.metadata'
source_branch = self.source_branch
dest_branch = self.dest_branch
_dist_tag = self.dist_tag
stream_name = self.stream_name
repo_tags = []
# If the upstream repo doesn't report anything, exit.
if not check_source_repo:
raise err.GitInitError('Upstream git repo does not exist')
dest_branch = f'{dest_branch}-stream-{stream_name}'
module_version = self.get_module_stream_version(self.release,
source_branch,
self.datestamp)
nsvc = f'{self.module_name}-{stream_name}-{module_version}.deadbeef'
import_tag = generic.safe_encoding(
f'imports/{dest_branch}/{nsvc}'
)
commit_msg = f'import {nsvc}'
print(f'Cloning upstream: {self.module_name}')
source_repo = gitutil.clone(
git_url_path=self.source_git_url,
repo_name=self.module_name,
to_path=source_git_repo_path,
branch=source_branch
)
if check_dest_repo:
ref_check = f'refs/heads/{dest_branch}' in check_dest_repo
print(f'Cloning: {self.module_name}')
if ref_check:
dest_repo = gitutil.clone(
git_url_path=self.dest_git_url,
repo_name=self.module_name,
to_path=dest_git_repo_path,
branch=dest_branch
)
else:
dest_repo = gitutil.clone(
git_url_path=self.dest_git_url,
repo_name=self.module_name,
to_path=dest_git_repo_path,
branch=None
)
gitutil.checkout(dest_repo, branch=dest_branch, orphan=True)
self.remove_everything(dest_repo.working_dir)
for tag_name in dest_repo.tags:
repo_tags.append(tag_name.name)
else:
print('Repo may not exist or is private. Try to import anyway.')
dest_repo = gitutil.init(
git_url_path=self.dest_git_url,
repo_name=self.module_name,
to_path=dest_git_repo_path,
branch=dest_branch
)
# We'd normally look for similar tags. But the date time is always
# going to change, so we're skipping that part.
if not os.path.exists(f'{dest_git_repo_path}/SOURCES'):
try:
os.makedirs(f'{dest_git_repo_path}/SOURCES')
except Exception as exc:
raise err.GenericError(f'Directory could not be created: {exc}')
# We eventually want to do it this way.
#if Version(Modulemd.get_version()) < Version("2.11"):
# source_modulemd = Modulemd.ModuleStream.read_file(
# modulemd_file,
# True,
# self.module_name
# )
#else:
# source_modulemd = Modulemd.read_packager_file(modulemd_file,
# self.module_name,
# stream_name)
#components = source_modulemd.get_rpm_component_names()
#for component in components:
# change = source_modulemd.get_rpm_component(component)
# change.set_ref(dest_branch)
with open(modulemd_file, 'r') as module_yaml:
content = module_yaml.read()
content_new = re.sub('ref:\s+(.*)', f'ref: {dest_branch}', content)
module_yaml.close()
# Write to the root
with open(f'{dest_git_repo_path}/{self.module_name}.yaml', 'w') as module_yaml:
module_yaml.write(content_new)
module_yaml.close()
# Write to the sources, should be the same content
with open(f'{dest_git_repo_path}/SOURCES/modulemd.src.txt', 'w') as module_yaml:
module_yaml.write(content_new)
module_yaml.close()
self.generate_metadata(dest_git_repo_path, self.module_name, {})
gitutil.add_all(dest_repo)
verify = dest_repo.is_dirty()
if verify:
gitutil.commit(dest_repo, commit_msg)
ref = gitutil.tag(dest_repo, import_tag, commit_msg)
gitutil.push(dest_repo, ref=ref)
self.perform_cleanup([source_git_repo_path, dest_git_repo_path])
return True
print('Nothing to push')
self.perform_cleanup([source_git_repo_path, dest_git_repo_path])
return False
@property
def module_name(self):
"""
Returns the module name
"""
return self.__module
@property
def source_branch(self):
"""
Returns the starting branch
"""
return self.__branch
@property
def dest_branch(self):
"""
Returns the starting branch
"""
return self.__dest_branch
@property
def source_git_url(self):
"""
Returns the source git url
"""
return self.__source_git_url
@property
def dest_git_url(self):
"""
Returns the destination git url
"""
return self.__git_url
@property
def dist_tag(self):
"""
Returns the dist tag
"""
return self.__dist_tag
@property
def datestamp(self):
"""
Returns a date time stamp
"""
return self.__current_time
@property
def stream_name(self):
"""
Returns the stream name
"""
return self.__stream_name
@property
def release(self):
"""
Returns the release
"""
return self.__release