processor fix

README.md

@@ -7,19 +7,19 @@
 ![license](https://img.shields.io/github/license/peridotbuild/pv2)
 
 pv2 is a backend module framework for building and development. Initially
-designed as a POC to support peridot's transition to python, it provides
-utilities that can be used for developers in and outside of the projects
-in the RESF (such as Rocky Linux).
+designed as a POC to support peridot's potential transition to python, it
+provides utilities that can be used for developers in and outside of the
+projects in the RESF (such as Rocky Linux).
 
 ## Requirements
 
 * An RPM Distribution
 
   * Fedora
-  * Enterprise Linux 8, 9+ recommended
-  * CentOS Stream 8, 9+ recommended
+  * Enterprise Linux 9+ recommended
+  * CentOS Stream 9+ recommended
 
-* Python 3.6 or higher - Python 3.9+ recommended
+* Python 3.9 or higher
 * rpm-build
 * A few python modules
 
@@ -34,6 +34,10 @@ in the RESF (such as Rocky Linux).
   * \*-rpm-macros
   * \*-srpm-macros
 
+* additional packages either in Fedora Linux or EPEL
+
+  * rpmautospec-rpm-macros
+
 ## Example Scripts
 
 Example scripts are found in the `examples` directory, which can utilize

pv2/importer/operation.py

@@ -22,6 +22,13 @@ from pv2.util import uploader as upload
 #except ImportError:
 #    HAS_GI = False
 
+try:
+    from rpmautospec.subcommands import process_distgit as rpmautocl
+    HAS_RPMAUTOSPEC = True
+except ImportError:
+    HAS_RPMAUTOSPEC = False
+    print('WARNING! rpmautospec was not found on this system and is not loaded.')
+
 __all__ = [
         'Import',
         'SrpmImport',
@@ -152,9 +159,19 @@ class Import:
                 magic = fileutil.get_magic_file(full_path)
                 if magic.name == 'empty':
                     continue
+                # PGP public keys have been in the lookaside before. We'll
+                # just do it this way. It gets around weird gitignores and
+                # weird srpmproc behavior.
+                if 'PGP public' in magic.name:
+                    source_dict[f'SOURCES/{file.name}'] = fileutil.get_checksum(full_path)
                 if magic.encoding == 'binary':
                     source_dict[f'SOURCES/{file.name}'] = fileutil.get_checksum(full_path)
 
+                # This is a list of possible file names that should be in
+                # lookaside, even if their type ISN'T that.
+                if full_path.endswith('.rpm'):
+                    source_dict[f'SOURCES/{file.name}'] = fileutil.get_checksum(full_path)
+
         return source_dict
 
     @staticmethod
@@ -342,6 +359,7 @@ class SrpmImport(Import):
             distprefix: str = 'el',
             git_user: str = 'git',
             org: str = 'rpms',
+            preconv_names: bool = False,
             dest_lookaside: str = '/var/www/html/sources',
             verify_signature: bool = False,
             aws_access_key_id: str = '',
@@ -363,7 +381,12 @@ class SrpmImport(Import):
         self.__dest_lookaside = dest_lookaside
 
         pkg_name = self.__srpm_metadata['name']
-        git_url = f'ssh://{git_user}@{git_url_path}/{org}/{pkg_name}.git'
+
+        package_name = pkg_name
+        if preconv_names:
+            package_name = pkg_name.replace('+', 'plus')
+
+        git_url = f'ssh://{git_user}@{git_url_path}/{org}/{package_name}.git'
         self.__git_url = git_url
 
         file_name_search_srpm_res = re.search(r'.*?\.src\.rpm$',
@@ -408,7 +431,7 @@ class SrpmImport(Import):
         than uploaded to lookaside.
         """
         check_repo = gitutil.lsremote(self.git_url)
-        git_repo_path = f'/var/tmp/{self.rpm_name}'
+        git_repo_path = f'/var/tmp/{self.rpm_name_replace}'
         branch = self.__branch
         repo_tags = []
 
@@ -440,7 +463,7 @@ class SrpmImport(Import):
                         repo_name=self.rpm_name_replace,
                         branch=None
                 )
-                gitutil.checkout(repo, branch=self.__branch, orphan=True)
+                gitutil.checkout(repo, branch=branch, orphan=True)
             # Remove everything, plain and simple. Only needed for clone.
             self.remove_everything(repo.working_dir)
             for tag_name in repo.tags:
@@ -536,6 +559,14 @@ class SrpmImport(Import):
         """
         return self.__srpm_metadata['name']
 
+    @property
+    def rpm_name_replace(self):
+        """
+        Returns name of srpm
+        """
+        new_name = self.__srpm_metadata['name'].replace('+', 'plus')
+        return new_name
+
     @property
     def rpm_version(self):
         """
@@ -548,7 +579,9 @@ class SrpmImport(Import):
         """
         Returns release of srpm
         """
-        return self.__srpm_metadata['release']
+        # Remove ~bootstrap
+        final_string = self.__srpm_metadata['release'].replace('~bootstrap', '')
+        return final_string
 
     @property
     def part_of_module(self):
@@ -563,14 +596,6 @@ class SrpmImport(Import):
 
         return False
 
-    @property
-    def rpm_name_replace(self):
-        """
-        Returns a "fixed" version of the RPM name
-        """
-        new_name = self.__srpm_metadata['name'].replace('+', 'plus')
-        return new_name
-
     @property
     def distprefix(self):
         """
@@ -608,6 +633,7 @@ class GitImport(Import):
             scl_mode: bool = False,
             scl_package: str = '',
             alternate_spec_name: str = '',
+            preconv_names: bool = False,
             dest_lookaside: str = '/var/www/html/sources',
             source_git_protocol: str = 'https',
             dest_branch: str = '',
@@ -632,8 +658,12 @@ class GitImport(Import):
         if source_git_protocol == 'ssh':
             full_source_git_url_path = f'{source_git_user}@{source_git_url_path}'
 
-        self.__source_git_url = f'{source_git_protocol}://{full_source_git_url_path}/{source_git_org_path}/{package}.git'
-        self.__dest_git_url = f'ssh://{dest_git_user}@{dest_git_url_path}/{dest_org}/{package}.git'
+        package_name = package
+        if preconv_names:
+            package_name = package.replace('+', 'plus')
+
+        self.__source_git_url = f'{source_git_protocol}://{full_source_git_url_path}/{source_git_org_path}/{package_name}.git'
+        self.__dest_git_url = f'ssh://{dest_git_user}@{dest_git_url_path}/{dest_org}/{package_name}.git'
         self.__dist_prefix = distprefix
         self.__dist_tag = f'.{distprefix}{release}'
         self.__source_branch = source_branch
@@ -642,6 +672,7 @@ class GitImport(Import):
         self.__upstream_lookaside = upstream_lookaside
         self.__upstream_lookaside_url = self.get_lookaside_template_path(upstream_lookaside)
         self.__alternate_spec_name = alternate_spec_name
+        self.__preconv_names = preconv_names
         self.__aws_access_key_id = aws_access_key_id
         self.__aws_access_key = aws_access_key
         self.__aws_bucket = aws_bucket
@@ -664,6 +695,7 @@ class GitImport(Import):
         check_dest_repo = gitutil.lsremote(self.dest_git_url)
         source_git_repo_path = f'/var/tmp/{self.rpm_name}-source'
         source_git_repo_spec = f'{source_git_repo_path}/{self.rpm_name}.spec'
+        source_git_repo_changelog = f'{source_git_repo_path}/changelog'
         dest_git_repo_path = f'/var/tmp/{self.rpm_name}'
         metadata_file = f'{source_git_repo_path}/.{self.rpm_name}.metadata'
         sources_file = f'{source_git_repo_path}/sources'
@@ -746,7 +778,10 @@ class GitImport(Import):
                 raise err.ConfigurationError(f'sources files are not supported with {self.upstream_lookaside}')
             metafile_to_use = sources_file
         else:
-            raise err.GenericError('sources or metadata file NOT found')
+            #raise err.GenericError('sources or metadata file NOT found')
+            print('WARNING: There was no sources or metadata found. Making blank file.')
+            with open(metadata_file, 'w+') as metadata_handle:
+                pass
 
         sources_dict = self.parse_metadata_file(metafile_to_use)
 
@@ -775,6 +810,31 @@ class GitImport(Import):
         if not os.path.exists(source_git_repo_spec) and len(self.alternate_spec_name) == 0:
             source_git_repo_spec = self.find_spec_file(source_git_repo_path)
 
+        # do rpm autochangelog logic here
+        if HAS_RPMAUTOSPEC and os.path.exists(source_git_repo_changelog):
+            # Check that the spec file really has %autochangelog
+            AUTOCHANGELOG = False
+            with open(source_git_repo_spec, 'r') as spec_file:
+                for line in spec_file:
+                    if re.match(r'^%autochangelog', line):
+                        print('autochangelog found')
+                        AUTOCHANGELOG = True
+                spec_file.close()
+            # It was easier to do this then reimplement logic
+            if AUTOCHANGELOG:
+                try:
+                    rpmautocl.process_distgit(
+                            source_git_repo_spec,
+                            f'/tmp/{self.rpm_name}.spec'
+                    )
+                except Exception as exc:
+                    raise err.GenericError('There was an error with autospec.') from exc
+
+                shutil.copy(f'/tmp/{self.rpm_name}.spec',
+                            f'{source_git_repo_path}/{self.rpm_name}.spec')
+
+                os.remove(f'/tmp/{self.rpm_name}.spec')
+
         # attempt to pack up the RPM, get metadata
         packed_srpm = self.pack_srpm(source_git_repo_path,
                                      source_git_repo_spec,
@@ -843,8 +903,11 @@ class GitImport(Import):
         """
         Returns the translated URL to obtain sources
         """
+        rpm_name = self.rpm_name
+        if self.preconv_names:
+            rpm_name = self.rpm_name_replace
         dict_template = {
-                'PKG_NAME': self.rpm_name,
+                'PKG_NAME': rpm_name,
                 'FILENAME': filename,
                 'HASH_TYPE': hashtype.lower(),
                 'HASH': checksum
@@ -932,6 +995,13 @@ class GitImport(Import):
         """
         return self.__dest_lookaside
 
+    @property
+    def preconv_names(self):
+        """
+        Returns if names are being preconverted
+        """
+        return self.__preconv_names
+
 class ModuleImport(Import):
     """
     Imports module repos

pv2/scripts/import_pkg.py

@@ -20,7 +20,7 @@ rpm_parser.add_argument('--release', type=str, required=False, default='')
 rpm_parser.add_argument('--gitorg', type=str, required=False, default='rpms')
 rpm_parser.add_argument('--distprefix', type=str, required=False, default='el')
 rpm_parser.add_argument('--dest-lookaside', type=str, required=False, default='/var/www/html/sources')
-rpm_parser.add_argument('--verify-signature', action='store_true')
+rpm_parser.add_argument('--no-verify-signature', action='store_true')
 rpm_parser.add_argument('--skip-lookaside-upload',
                         action='store_true',
                         help='Set this flag to skip uploading to /var/www/html/sources esque lookaside')
@@ -39,6 +39,7 @@ git_parser.add_argument('--dest-giturl', type=str, required=True)
 git_parser.add_argument('--dest-gitorg', type=str, required=False, default='rpms')
 git_parser.add_argument('--dest-branch', type=str, required=False, default='')
 git_parser.add_argument('--release', type=str, required=False, default='')
+git_parser.add_argument('--preconv-names', action='store_true', help='Convert + to plus first')
 git_parser.add_argument('--distprefix', type=str, required=False, default='el')
 git_parser.add_argument('--dest-lookaside', type=str, required=False, default='/var/www/html/sources')
 git_parser.add_argument('--upstream-lookaside',
@@ -73,7 +74,7 @@ def main():
                 git_user=results.gituser,
                 org=results.gitorg,
                 dest_lookaside=results.dest_lookaside,
-                verify_signature=results.verify_signature,
+                verify_signature=results.no_verify_signature,
                 aws_access_key_id=results.aws_access_key_id,
                 aws_access_key=results.aws_access_key,
                 aws_bucket=results.aws_bucket,
@@ -90,6 +91,7 @@ def main():
                 dest_git_url_path=results.dest_giturl,
                 dest_org=results.dest_gitorg,
                 release=results.release,
+                preconv_names=results.preconv_names,
                 source_branch=results.source_branch,
                 dest_branch=results.dest_branch,
                 upstream_lookaside=results.upstream_lookaside,

pv2/util/fileutil.py

@@ -46,8 +46,8 @@ def get_checksum(file_path: str, hashtype: str = 'sha256') -> str:
     Borrowed from empanadas with some modifications
     """
     # We shouldn't be using sha1 or md5.
-    if hashtype in ('sha', 'sha1', 'md5'):
-        raise err.ProvidedValueError(f'{hashtype} is not allowed.')
+    #if hashtype in ('sha', 'sha1', 'md5'):
+    #    raise err.ProvidedValueError(f'{hashtype} is not allowed.')
 
     try:
         checksum = hashlib.new(hashtype)