mirror of
https://github.com/peridotbuild/pv2.git
synced 2024-06-02 11:00:14 +00:00
Compare commits
17 Commits
4fc3f849e1
...
01a6696adc
Author | SHA1 | Date | |
---|---|---|---|
01a6696adc | |||
99035af3e9 | |||
90aa3e2016 | |||
3e925ca471 | |||
b982646f3a | |||
0854a39c58 | |||
45abe82b40 | |||
f3213dafaf | |||
1255a10ef9 | |||
7001664d9f | |||
236f18412b | |||
0913ae912f | |||
93f5fbe65d | |||
95c701aa38 | |||
f4499ca17d | |||
655a580afd | |||
022d4ed03a |
16
README.md
16
README.md
|
@ -7,19 +7,19 @@
|
||||||
![license](https://img.shields.io/github/license/peridotbuild/pv2)
|
![license](https://img.shields.io/github/license/peridotbuild/pv2)
|
||||||
|
|
||||||
pv2 is a backend module framework for building and development. Initially
|
pv2 is a backend module framework for building and development. Initially
|
||||||
designed as a POC to support peridot's transition to python, it provides
|
designed as a POC to support peridot's potential transition to python, it
|
||||||
utilities that can be used for developers in and outside of the projects
|
provides utilities that can be used for developers in and outside of the
|
||||||
in the RESF (such as Rocky Linux).
|
projects in the RESF (such as Rocky Linux).
|
||||||
|
|
||||||
## Requirements
|
## Requirements
|
||||||
|
|
||||||
* An RPM Distribution
|
* An RPM Distribution
|
||||||
|
|
||||||
* Fedora
|
* Fedora
|
||||||
* Enterprise Linux 8, 9+ recommended
|
* Enterprise Linux 9+ recommended
|
||||||
* CentOS Stream 8, 9+ recommended
|
* CentOS Stream 9+ recommended
|
||||||
|
|
||||||
* Python 3.6 or higher - Python 3.9+ recommended
|
* Python 3.9 or higher
|
||||||
* rpm-build
|
* rpm-build
|
||||||
* A few python modules
|
* A few python modules
|
||||||
|
|
||||||
|
@ -34,6 +34,10 @@ in the RESF (such as Rocky Linux).
|
||||||
* \*-rpm-macros
|
* \*-rpm-macros
|
||||||
* \*-srpm-macros
|
* \*-srpm-macros
|
||||||
|
|
||||||
|
* additional packages either in Fedora Linux or EPEL
|
||||||
|
|
||||||
|
* rpmautospec-rpm-macros
|
||||||
|
|
||||||
## Example Scripts
|
## Example Scripts
|
||||||
|
|
||||||
Example scripts are found in the `examples` directory, which can utilize
|
Example scripts are found in the `examples` directory, which can utilize
|
||||||
|
|
|
@ -22,6 +22,13 @@ from pv2.util import uploader as upload
|
||||||
#except ImportError:
|
#except ImportError:
|
||||||
# HAS_GI = False
|
# HAS_GI = False
|
||||||
|
|
||||||
|
try:
|
||||||
|
from rpmautospec.subcommands import process_distgit as rpmautocl
|
||||||
|
HAS_RPMAUTOSPEC = True
|
||||||
|
except ImportError:
|
||||||
|
HAS_RPMAUTOSPEC = False
|
||||||
|
print('WARNING! rpmautospec was not found on this system and is not loaded.')
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
'Import',
|
'Import',
|
||||||
'SrpmImport',
|
'SrpmImport',
|
||||||
|
@ -152,9 +159,19 @@ class Import:
|
||||||
magic = fileutil.get_magic_file(full_path)
|
magic = fileutil.get_magic_file(full_path)
|
||||||
if magic.name == 'empty':
|
if magic.name == 'empty':
|
||||||
continue
|
continue
|
||||||
|
# PGP public keys have been in the lookaside before. We'll
|
||||||
|
# just do it this way. It gets around weird gitignores and
|
||||||
|
# weird srpmproc behavior.
|
||||||
|
if 'PGP public' in magic.name:
|
||||||
|
source_dict[f'SOURCES/{file.name}'] = fileutil.get_checksum(full_path)
|
||||||
if magic.encoding == 'binary':
|
if magic.encoding == 'binary':
|
||||||
source_dict[f'SOURCES/{file.name}'] = fileutil.get_checksum(full_path)
|
source_dict[f'SOURCES/{file.name}'] = fileutil.get_checksum(full_path)
|
||||||
|
|
||||||
|
# This is a list of possible file names that should be in
|
||||||
|
# lookaside, even if their type ISN'T that.
|
||||||
|
if full_path.endswith('.rpm'):
|
||||||
|
source_dict[f'SOURCES/{file.name}'] = fileutil.get_checksum(full_path)
|
||||||
|
|
||||||
return source_dict
|
return source_dict
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
|
@ -342,6 +359,7 @@ class SrpmImport(Import):
|
||||||
distprefix: str = 'el',
|
distprefix: str = 'el',
|
||||||
git_user: str = 'git',
|
git_user: str = 'git',
|
||||||
org: str = 'rpms',
|
org: str = 'rpms',
|
||||||
|
preconv_names: bool = False,
|
||||||
dest_lookaside: str = '/var/www/html/sources',
|
dest_lookaside: str = '/var/www/html/sources',
|
||||||
verify_signature: bool = False,
|
verify_signature: bool = False,
|
||||||
aws_access_key_id: str = '',
|
aws_access_key_id: str = '',
|
||||||
|
@ -363,7 +381,12 @@ class SrpmImport(Import):
|
||||||
self.__dest_lookaside = dest_lookaside
|
self.__dest_lookaside = dest_lookaside
|
||||||
|
|
||||||
pkg_name = self.__srpm_metadata['name']
|
pkg_name = self.__srpm_metadata['name']
|
||||||
git_url = f'ssh://{git_user}@{git_url_path}/{org}/{pkg_name}.git'
|
|
||||||
|
package_name = pkg_name
|
||||||
|
if preconv_names:
|
||||||
|
package_name = pkg_name.replace('+', 'plus')
|
||||||
|
|
||||||
|
git_url = f'ssh://{git_user}@{git_url_path}/{org}/{package_name}.git'
|
||||||
self.__git_url = git_url
|
self.__git_url = git_url
|
||||||
|
|
||||||
file_name_search_srpm_res = re.search(r'.*?\.src\.rpm$',
|
file_name_search_srpm_res = re.search(r'.*?\.src\.rpm$',
|
||||||
|
@ -408,7 +431,7 @@ class SrpmImport(Import):
|
||||||
than uploaded to lookaside.
|
than uploaded to lookaside.
|
||||||
"""
|
"""
|
||||||
check_repo = gitutil.lsremote(self.git_url)
|
check_repo = gitutil.lsremote(self.git_url)
|
||||||
git_repo_path = f'/var/tmp/{self.rpm_name}'
|
git_repo_path = f'/var/tmp/{self.rpm_name_replace}'
|
||||||
branch = self.__branch
|
branch = self.__branch
|
||||||
repo_tags = []
|
repo_tags = []
|
||||||
|
|
||||||
|
@ -440,7 +463,7 @@ class SrpmImport(Import):
|
||||||
repo_name=self.rpm_name_replace,
|
repo_name=self.rpm_name_replace,
|
||||||
branch=None
|
branch=None
|
||||||
)
|
)
|
||||||
gitutil.checkout(repo, branch=self.__branch, orphan=True)
|
gitutil.checkout(repo, branch=branch, orphan=True)
|
||||||
# Remove everything, plain and simple. Only needed for clone.
|
# Remove everything, plain and simple. Only needed for clone.
|
||||||
self.remove_everything(repo.working_dir)
|
self.remove_everything(repo.working_dir)
|
||||||
for tag_name in repo.tags:
|
for tag_name in repo.tags:
|
||||||
|
@ -536,6 +559,14 @@ class SrpmImport(Import):
|
||||||
"""
|
"""
|
||||||
return self.__srpm_metadata['name']
|
return self.__srpm_metadata['name']
|
||||||
|
|
||||||
|
@property
|
||||||
|
def rpm_name_replace(self):
|
||||||
|
"""
|
||||||
|
Returns name of srpm
|
||||||
|
"""
|
||||||
|
new_name = self.__srpm_metadata['name'].replace('+', 'plus')
|
||||||
|
return new_name
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def rpm_version(self):
|
def rpm_version(self):
|
||||||
"""
|
"""
|
||||||
|
@ -548,7 +579,9 @@ class SrpmImport(Import):
|
||||||
"""
|
"""
|
||||||
Returns release of srpm
|
Returns release of srpm
|
||||||
"""
|
"""
|
||||||
return self.__srpm_metadata['release']
|
# Remove ~bootstrap
|
||||||
|
final_string = self.__srpm_metadata['release'].replace('~bootstrap', '')
|
||||||
|
return final_string
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def part_of_module(self):
|
def part_of_module(self):
|
||||||
|
@ -563,14 +596,6 @@ class SrpmImport(Import):
|
||||||
|
|
||||||
return False
|
return False
|
||||||
|
|
||||||
@property
|
|
||||||
def rpm_name_replace(self):
|
|
||||||
"""
|
|
||||||
Returns a "fixed" version of the RPM name
|
|
||||||
"""
|
|
||||||
new_name = self.__srpm_metadata['name'].replace('+', 'plus')
|
|
||||||
return new_name
|
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def distprefix(self):
|
def distprefix(self):
|
||||||
"""
|
"""
|
||||||
|
@ -608,6 +633,7 @@ class GitImport(Import):
|
||||||
scl_mode: bool = False,
|
scl_mode: bool = False,
|
||||||
scl_package: str = '',
|
scl_package: str = '',
|
||||||
alternate_spec_name: str = '',
|
alternate_spec_name: str = '',
|
||||||
|
preconv_names: bool = False,
|
||||||
dest_lookaside: str = '/var/www/html/sources',
|
dest_lookaside: str = '/var/www/html/sources',
|
||||||
source_git_protocol: str = 'https',
|
source_git_protocol: str = 'https',
|
||||||
dest_branch: str = '',
|
dest_branch: str = '',
|
||||||
|
@ -632,8 +658,12 @@ class GitImport(Import):
|
||||||
if source_git_protocol == 'ssh':
|
if source_git_protocol == 'ssh':
|
||||||
full_source_git_url_path = f'{source_git_user}@{source_git_url_path}'
|
full_source_git_url_path = f'{source_git_user}@{source_git_url_path}'
|
||||||
|
|
||||||
self.__source_git_url = f'{source_git_protocol}://{full_source_git_url_path}/{source_git_org_path}/{package}.git'
|
package_name = package
|
||||||
self.__dest_git_url = f'ssh://{dest_git_user}@{dest_git_url_path}/{dest_org}/{package}.git'
|
if preconv_names:
|
||||||
|
package_name = package.replace('+', 'plus')
|
||||||
|
|
||||||
|
self.__source_git_url = f'{source_git_protocol}://{full_source_git_url_path}/{source_git_org_path}/{package_name}.git'
|
||||||
|
self.__dest_git_url = f'ssh://{dest_git_user}@{dest_git_url_path}/{dest_org}/{package_name}.git'
|
||||||
self.__dist_prefix = distprefix
|
self.__dist_prefix = distprefix
|
||||||
self.__dist_tag = f'.{distprefix}{release}'
|
self.__dist_tag = f'.{distprefix}{release}'
|
||||||
self.__source_branch = source_branch
|
self.__source_branch = source_branch
|
||||||
|
@ -642,6 +672,7 @@ class GitImport(Import):
|
||||||
self.__upstream_lookaside = upstream_lookaside
|
self.__upstream_lookaside = upstream_lookaside
|
||||||
self.__upstream_lookaside_url = self.get_lookaside_template_path(upstream_lookaside)
|
self.__upstream_lookaside_url = self.get_lookaside_template_path(upstream_lookaside)
|
||||||
self.__alternate_spec_name = alternate_spec_name
|
self.__alternate_spec_name = alternate_spec_name
|
||||||
|
self.__preconv_names = preconv_names
|
||||||
self.__aws_access_key_id = aws_access_key_id
|
self.__aws_access_key_id = aws_access_key_id
|
||||||
self.__aws_access_key = aws_access_key
|
self.__aws_access_key = aws_access_key
|
||||||
self.__aws_bucket = aws_bucket
|
self.__aws_bucket = aws_bucket
|
||||||
|
@ -664,6 +695,7 @@ class GitImport(Import):
|
||||||
check_dest_repo = gitutil.lsremote(self.dest_git_url)
|
check_dest_repo = gitutil.lsremote(self.dest_git_url)
|
||||||
source_git_repo_path = f'/var/tmp/{self.rpm_name}-source'
|
source_git_repo_path = f'/var/tmp/{self.rpm_name}-source'
|
||||||
source_git_repo_spec = f'{source_git_repo_path}/{self.rpm_name}.spec'
|
source_git_repo_spec = f'{source_git_repo_path}/{self.rpm_name}.spec'
|
||||||
|
source_git_repo_changelog = f'{source_git_repo_path}/changelog'
|
||||||
dest_git_repo_path = f'/var/tmp/{self.rpm_name}'
|
dest_git_repo_path = f'/var/tmp/{self.rpm_name}'
|
||||||
metadata_file = f'{source_git_repo_path}/.{self.rpm_name}.metadata'
|
metadata_file = f'{source_git_repo_path}/.{self.rpm_name}.metadata'
|
||||||
sources_file = f'{source_git_repo_path}/sources'
|
sources_file = f'{source_git_repo_path}/sources'
|
||||||
|
@ -746,7 +778,10 @@ class GitImport(Import):
|
||||||
raise err.ConfigurationError(f'sources files are not supported with {self.upstream_lookaside}')
|
raise err.ConfigurationError(f'sources files are not supported with {self.upstream_lookaside}')
|
||||||
metafile_to_use = sources_file
|
metafile_to_use = sources_file
|
||||||
else:
|
else:
|
||||||
raise err.GenericError('sources or metadata file NOT found')
|
#raise err.GenericError('sources or metadata file NOT found')
|
||||||
|
print('WARNING: There was no sources or metadata found. Making blank file.')
|
||||||
|
with open(metadata_file, 'w+') as metadata_handle:
|
||||||
|
pass
|
||||||
|
|
||||||
sources_dict = self.parse_metadata_file(metafile_to_use)
|
sources_dict = self.parse_metadata_file(metafile_to_use)
|
||||||
|
|
||||||
|
@ -775,6 +810,31 @@ class GitImport(Import):
|
||||||
if not os.path.exists(source_git_repo_spec) and len(self.alternate_spec_name) == 0:
|
if not os.path.exists(source_git_repo_spec) and len(self.alternate_spec_name) == 0:
|
||||||
source_git_repo_spec = self.find_spec_file(source_git_repo_path)
|
source_git_repo_spec = self.find_spec_file(source_git_repo_path)
|
||||||
|
|
||||||
|
# do rpm autochangelog logic here
|
||||||
|
if HAS_RPMAUTOSPEC and os.path.exists(source_git_repo_changelog):
|
||||||
|
# Check that the spec file really has %autochangelog
|
||||||
|
AUTOCHANGELOG = False
|
||||||
|
with open(source_git_repo_spec, 'r') as spec_file:
|
||||||
|
for line in spec_file:
|
||||||
|
if re.match(r'^%autochangelog', line):
|
||||||
|
print('autochangelog found')
|
||||||
|
AUTOCHANGELOG = True
|
||||||
|
spec_file.close()
|
||||||
|
# It was easier to do this then reimplement logic
|
||||||
|
if AUTOCHANGELOG:
|
||||||
|
try:
|
||||||
|
rpmautocl.process_distgit(
|
||||||
|
source_git_repo_spec,
|
||||||
|
f'/tmp/{self.rpm_name}.spec'
|
||||||
|
)
|
||||||
|
except Exception as exc:
|
||||||
|
raise err.GenericError('There was an error with autospec.') from exc
|
||||||
|
|
||||||
|
shutil.copy(f'/tmp/{self.rpm_name}.spec',
|
||||||
|
f'{source_git_repo_path}/{self.rpm_name}.spec')
|
||||||
|
|
||||||
|
os.remove(f'/tmp/{self.rpm_name}.spec')
|
||||||
|
|
||||||
# attempt to pack up the RPM, get metadata
|
# attempt to pack up the RPM, get metadata
|
||||||
packed_srpm = self.pack_srpm(source_git_repo_path,
|
packed_srpm = self.pack_srpm(source_git_repo_path,
|
||||||
source_git_repo_spec,
|
source_git_repo_spec,
|
||||||
|
@ -843,8 +903,11 @@ class GitImport(Import):
|
||||||
"""
|
"""
|
||||||
Returns the translated URL to obtain sources
|
Returns the translated URL to obtain sources
|
||||||
"""
|
"""
|
||||||
|
rpm_name = self.rpm_name
|
||||||
|
if self.preconv_names:
|
||||||
|
rpm_name = self.rpm_name_replace
|
||||||
dict_template = {
|
dict_template = {
|
||||||
'PKG_NAME': self.rpm_name,
|
'PKG_NAME': rpm_name,
|
||||||
'FILENAME': filename,
|
'FILENAME': filename,
|
||||||
'HASH_TYPE': hashtype.lower(),
|
'HASH_TYPE': hashtype.lower(),
|
||||||
'HASH': checksum
|
'HASH': checksum
|
||||||
|
@ -932,6 +995,13 @@ class GitImport(Import):
|
||||||
"""
|
"""
|
||||||
return self.__dest_lookaside
|
return self.__dest_lookaside
|
||||||
|
|
||||||
|
@property
|
||||||
|
def preconv_names(self):
|
||||||
|
"""
|
||||||
|
Returns if names are being preconverted
|
||||||
|
"""
|
||||||
|
return self.__preconv_names
|
||||||
|
|
||||||
class ModuleImport(Import):
|
class ModuleImport(Import):
|
||||||
"""
|
"""
|
||||||
Imports module repos
|
Imports module repos
|
||||||
|
|
|
@ -20,7 +20,7 @@ rpm_parser.add_argument('--release', type=str, required=False, default='')
|
||||||
rpm_parser.add_argument('--gitorg', type=str, required=False, default='rpms')
|
rpm_parser.add_argument('--gitorg', type=str, required=False, default='rpms')
|
||||||
rpm_parser.add_argument('--distprefix', type=str, required=False, default='el')
|
rpm_parser.add_argument('--distprefix', type=str, required=False, default='el')
|
||||||
rpm_parser.add_argument('--dest-lookaside', type=str, required=False, default='/var/www/html/sources')
|
rpm_parser.add_argument('--dest-lookaside', type=str, required=False, default='/var/www/html/sources')
|
||||||
rpm_parser.add_argument('--verify-signature', action='store_true')
|
rpm_parser.add_argument('--no-verify-signature', action='store_true')
|
||||||
rpm_parser.add_argument('--skip-lookaside-upload',
|
rpm_parser.add_argument('--skip-lookaside-upload',
|
||||||
action='store_true',
|
action='store_true',
|
||||||
help='Set this flag to skip uploading to /var/www/html/sources esque lookaside')
|
help='Set this flag to skip uploading to /var/www/html/sources esque lookaside')
|
||||||
|
@ -39,6 +39,7 @@ git_parser.add_argument('--dest-giturl', type=str, required=True)
|
||||||
git_parser.add_argument('--dest-gitorg', type=str, required=False, default='rpms')
|
git_parser.add_argument('--dest-gitorg', type=str, required=False, default='rpms')
|
||||||
git_parser.add_argument('--dest-branch', type=str, required=False, default='')
|
git_parser.add_argument('--dest-branch', type=str, required=False, default='')
|
||||||
git_parser.add_argument('--release', type=str, required=False, default='')
|
git_parser.add_argument('--release', type=str, required=False, default='')
|
||||||
|
git_parser.add_argument('--preconv-names', action='store_true', help='Convert + to plus first')
|
||||||
git_parser.add_argument('--distprefix', type=str, required=False, default='el')
|
git_parser.add_argument('--distprefix', type=str, required=False, default='el')
|
||||||
git_parser.add_argument('--dest-lookaside', type=str, required=False, default='/var/www/html/sources')
|
git_parser.add_argument('--dest-lookaside', type=str, required=False, default='/var/www/html/sources')
|
||||||
git_parser.add_argument('--upstream-lookaside',
|
git_parser.add_argument('--upstream-lookaside',
|
||||||
|
@ -73,7 +74,7 @@ def main():
|
||||||
git_user=results.gituser,
|
git_user=results.gituser,
|
||||||
org=results.gitorg,
|
org=results.gitorg,
|
||||||
dest_lookaside=results.dest_lookaside,
|
dest_lookaside=results.dest_lookaside,
|
||||||
verify_signature=results.verify_signature,
|
verify_signature=results.no_verify_signature,
|
||||||
aws_access_key_id=results.aws_access_key_id,
|
aws_access_key_id=results.aws_access_key_id,
|
||||||
aws_access_key=results.aws_access_key,
|
aws_access_key=results.aws_access_key,
|
||||||
aws_bucket=results.aws_bucket,
|
aws_bucket=results.aws_bucket,
|
||||||
|
@ -90,6 +91,7 @@ def main():
|
||||||
dest_git_url_path=results.dest_giturl,
|
dest_git_url_path=results.dest_giturl,
|
||||||
dest_org=results.dest_gitorg,
|
dest_org=results.dest_gitorg,
|
||||||
release=results.release,
|
release=results.release,
|
||||||
|
preconv_names=results.preconv_names,
|
||||||
source_branch=results.source_branch,
|
source_branch=results.source_branch,
|
||||||
dest_branch=results.dest_branch,
|
dest_branch=results.dest_branch,
|
||||||
upstream_lookaside=results.upstream_lookaside,
|
upstream_lookaside=results.upstream_lookaside,
|
||||||
|
|
|
@ -46,8 +46,8 @@ def get_checksum(file_path: str, hashtype: str = 'sha256') -> str:
|
||||||
Borrowed from empanadas with some modifications
|
Borrowed from empanadas with some modifications
|
||||||
"""
|
"""
|
||||||
# We shouldn't be using sha1 or md5.
|
# We shouldn't be using sha1 or md5.
|
||||||
if hashtype in ('sha', 'sha1', 'md5'):
|
#if hashtype in ('sha', 'sha1', 'md5'):
|
||||||
raise err.ProvidedValueError(f'{hashtype} is not allowed.')
|
# raise err.ProvidedValueError(f'{hashtype} is not allowed.')
|
||||||
|
|
||||||
try:
|
try:
|
||||||
checksum = hashlib.new(hashtype)
|
checksum = hashlib.new(hashtype)
|
||||||
|
|
Loading…
Reference in New Issue
Block a user