peridot
/
pv2
mirror of https://github.com/peridotbuild/pv2.git
7
0
Fork 0
Code Issues 1 Packages Projects Releases Wiki Activity

make destination lookaside configurable

This commit is contained in:
Louis Abel 2023-07-03 12:43:36 -07:00
parent c4dc7195bb
commit df30fa4c72
Signed by: label
GPG Key ID: 3331F061D1D9990E
4 changed files with 145 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
README.md
View File

@ -20,3 +20,8 @@ in the RESF (such as Rocky Linux).
* GitPython (python3-GitPython or via pip)
* lxml (python3-lxml or via pip)
* rpm (python3-rpm)
## Example Scripts
Example scripts are found in the `examples` directory, which can utilize
parts of the pv2 module.

18
examples/import_pkg.py Normal file
View File

@ -0,0 +1,18 @@
#!/usr/bin/python3
"""
Import a source RPM into a git forge using pv2
"""
import argparse
import pv2.importer as importutil
parser = argparse.ArgumentParser(description="ISO Compose")
parser.add_argument('--giturl', type=str, required=True)
parser.add_argument('--branch', type=str, required=True)
parser.add_argument('--srpm', type=str, required=True)
parser.add_argument('--release', type=str, required=False, default='')
results = parser.parse_args()
# pylint: disable=line-too-long
classy = importutil.SrpmImport(git_url_path=results.giturl, srpm_path=results.srpm, branch=results.branch, release=results.release)
classy.pkg_import()

116
pv2/importer/operation.py
View File

@ -9,6 +9,7 @@ import re
import shutil
from pv2.util import gitutil, fileutil, rpmutil, processor, generic
from pv2.util import error as err
from pv2.util import constants as const
__all__ = [
'Import',
@ -99,12 +100,18 @@ class Import:
return metadata
@staticmethod
def import_lookaside(repo_path: str, repo_name: str, branch: str, file_dict: dict):
def import_lookaside(
repo_path: str,
repo_name: str,
branch: str,
file_dict: dict,
dest_lookaside: str = '/var/www/html/sources'
):
"""
Attempts to move the lookaside files if they don't exist to their
hashed name.
"""
dest_dir = f'/var/www/html/sources/{repo_name}/{branch}'
dest_dir = f'{dest_lookaside}/{repo_name}/{branch}'
if not os.path.exists(dest_dir):
os.makedirs(dest_dir, 0o755)
for name, sha in file_dict.items():
@ -129,6 +136,7 @@ class Import:
source_path = f'{repo_path}/{name}'
os.remove(source_path)
# pylint: disable=too-many-instance-attributes
class SrpmImport(Import):
"""
Import class for importing rpms to a git service
@ -146,6 +154,7 @@ class SrpmImport(Import):
distprefix: str = 'el',
git_user: str = 'git',
org: str = 'rpms',
dest_lookaside: str = '/var/www/html/sources',
verify_signature: bool = False
):
"""
@ -160,6 +169,7 @@ class SrpmImport(Import):
verify_signature)
self.__release = release
self.__dist_prefix = distprefix
self.__dest_lookaside = dest_lookaside
pkg_name = self.__srpm_metadata['name']
git_url = f'ssh://{git_user}@{git_url_path}/{org}/{pkg_name}.git'
@ -259,7 +269,8 @@ class SrpmImport(Import):
if skip_lookaside:
self.skip_import_lookaside(git_repo_path, sources)
else:
self.import_lookaside(git_repo_path, self.rpm_name, branch, sources)
self.import_lookaside(git_repo_path, self.rpm_name, branch,
sources, self.dest_lookaside)
gitutil.add_all(repo)
@ -347,6 +358,14 @@ class SrpmImport(Import):
"""
return self.__dist_prefix
@property
def dest_lookaside(self):
"""
Returns the destination path for the local lookaside
"""
return self.__dest_lookaside
# pylint: disable=too-many-instance-attributes
class GitImport(Import):
"""
Import class for importing from git (e.g. pagure or gitlab)
@ -365,10 +384,12 @@ class GitImport(Import):
git_url_path: str,
release: str,
branch: str,
upstream_lookaside: str = '',
dest_lookaside: str = '/var/www/html/sources',
dest_branch: str = '',
distprefix: str = 'el',
git_user: str = 'git',
org: str = 'rpms',
verify_signature: bool = False
org: str = 'rpms'
):
"""
Init the class.
@ -378,12 +399,49 @@ class GitImport(Import):
"""
self.__rpm = package
self.__release = release
source_git_url = f'https://{source_git_url_path}/{source_git_org_path}/{package}.git'
git_url = f'ssh://{git_user}@{git_url_path}/{org}/{package}.git'
self.__git_url = git_url
self.__source_git_url = f'https://{source_git_url_path}/{source_git_org_path}/{package}.git'
self.__git_url = f'ssh://{git_user}@{git_url_path}/{org}/{package}.git'
self.__dist_prefix = distprefix
self.__dist_tag = f'.{distprefix}{release}'
self.__branch = branch
self.__dest_branch = branch
self.__dest_lookaside = dest_lookaside
self.__upstream_lookaside = upstream_lookaside
if len(dest_branch) > 0:
self.__dest_branch = dest_branch
def pkg_import(self, skip_lookaside: bool = False):
"""
Actually perform the import
If skip_lookaside is True, source files will just be deleted rather
than uploaded to lookaside.
"""
check_source_repo = gitutil.lsremote(self.source_git_url)
check_dest_repo = gitutil.lsremote(self.source_git_url)
source_git_repo_path = f'/var/tmp/{self.rpm_name}-source'
dest_git_repo_path = f'/var/tmp/{self.rpm_name}'
source_branch = self.source_branch
dest_branch = self.dest_branch
repo_tags = []
@staticmethod
def __get_lookaside_template_path(source):
"""
Attempts to return the lookaside template
"""
# This is an extremely hacky way to return the right value. In python
# 3.10, match-case was introduced. However, we need to assume that
# python 3.9 is the lowest used version for this module, so we need to
# be inefficient until we no longer use EL9 as the base line.
return {
'rocky8': const.GitConstants.ROCKY8_LOOKASIDE_PATH,
'rocky': const.GitConstants.ROCKY_LOOKASIDE_PATH,
'centos': const.GitConstants.CENTOS_LOOKASIDE_PATH,
'stream': const.GitConstants.STREAM_LOOKASIDE_PATH,
'fedora': const.GitConstants.FEDORA_LOOKASIDE_PATH,
}.get(source, None)
@property
def rpm_name(self):
@ -391,3 +449,45 @@ class GitImport(Import):
Returns the name of the RPM we're working with
"""
return self.__rpm
@property
def source_branch(self):
"""
Returns the starting branch
"""
return self.__branch
@property
def dest_branch(self):
"""
Returns the starting branch
"""
return self.__dest_branch
@property
def source_git_url(self):
"""
Returns the source git url
"""
return self.__source_git_url
@property
def dest_git_url(self):
"""
Returns the destination git url
"""
return self.__git_url
@property
def dist_tag(self):
"""
Returns the dist tag
"""
return self.__dist_tag
@property
def dest_lookaside(self):
"""
Returns destination local lookaside
"""
return self.__dest_lookaside

15
pv2/util/constants.py
View File

@ -6,7 +6,8 @@ All constants
__all__ = [
'RpmConstants',
'ErrorConstants',
'MockConstants'
'MockConstants',
'GitConstants'
]
# pylint: disable=too-few-public-methods
@ -177,3 +178,15 @@ class MockConstants:
MOCK_EXIT_SIGHUP_RECEIVED = 129
MOCK_EXIT_SIGPIPE_RECEIVED = 141
MOCK_EXIT_SIGTERM_RECEIVED = 143
class GitConstants:
"""
All git related constants
"""
CENTOS_LOOKASIDE_PATH = 'https://git.centos.org/sources/${PKG_NAME}/${BRANCH}/${HASH}'
# pylint: disable=line-too-long
STREAM_LOOKASIDE_PATH = 'https://sources.stream.centos.org/sources/rpms/${PKG_NAME}/${FILENAME}/${HASH_TYPE}/${FILENAME}'
FEDORA_LOOKASIDE_PATH = 'https://src.fedoraproject.org/repo/pkgs/${PKG_NAME}/${FILENAME}/${HASH_TYPE}/${FILENAME}'
ROCKY8_LOOKASIDE_PATH = 'https://rocky-linux-sources-staging.a1.rockylinux.org/${HASH}'
ROCKY_LOOKASIDE_PATH = 'https://sources.build.resf.org/${HASH}'