Add module importer and git example

* Add ModuleImport class to import module repos
* Add import_git.py example of using GitImport

examples/import_git.py

@@ -0,0 +1,32 @@
+#!/usr/bin/python3
+
+import argparse
+import pv2.importer as importutil
+
+parser = argparse.ArgumentParser(description="Importer")
+
+parser.add_argument('--srpm', type=str, required=True)
+parser.add_argument('--source-giturl', type=str, required=True)
+parser.add_argument('--source-gitorg', type=str, required=True)
+parser.add_argument('--branch', type=str, required=True)
+parser.add_argument('--giturl', type=str, required=True)
+parser.add_argument('--gitorg', type=str, required=False, default='rpms')
+parser.add_argument('--dest-branch', type=str, required=False, default='')
+parser.add_argument('--release', type=str, required=False, default='')
+parser.add_argument('--distprefix', type=str, required=False, default='el')
+parser.add_argument('--upstream-lookaside', type=str, required=True)
+results = parser.parse_args()
+classy = importutil.GitImport(
+        results.srpm,
+        source_git_url_path=results.source_giturl,
+        source_git_org_path=results.source_gitorg,
+        git_url_path=results.giturl,
+        org=results.gitorg,
+        release=results.release,
+        branch=results.branch,
+        dest_branch=results.dest_branch,
+        upstream_lookaside=results.upstream_lookaside,
+        distprefix=results.distprefix
+)
+
+classy.pkg_import()

examples/import_pkg.py

@@ -6,7 +6,7 @@ Import a source RPM into a git forge using pv2
 import argparse
 import pv2.importer as importutil
 
-parser = argparse.ArgumentParser(description="ISO Compose")
+parser = argparse.ArgumentParser(description="Importer")
 
 parser.add_argument('--giturl', type=str, required=True)
 parser.add_argument('--branch', type=str, required=True)

pv2/importer/__init__.py

@@ -7,4 +7,4 @@ This assists packagers by taking input as srpm or git location, importing and
 tagging it as appropriate.
 """
 
-from .operation import Import, SrpmImport, GitImport
+from .operation import Import, SrpmImport, GitImport, ModuleImport

pv2/importer/operation.py

@@ -8,14 +8,24 @@ import os
 import re
 import shutil
 import string
+import datetime
 from pv2.util import gitutil, fileutil, rpmutil, processor, generic
 from pv2.util import error as err
 from pv2.util import constants as const
 
+#try:
+#    import gi
+#    gi.require_version('Modulemd', '2.0')
+#    from gi.repository import Modulemd
+#    HAS_GI = True
+#except ImportError:
+#    HAS_GI = False
+
 __all__ = [
         'Import',
         'SrpmImport',
-        'GitImport'
+        'GitImport',
+        'ModuleImport'
 ]
 # todo: add in logging and replace print with log
 
@@ -223,6 +233,35 @@ class Import:
             except Exception as exc:
                 raise err.FileNotFound(f'{directory} could not be deleted. Please check. {exc}')
 
+    @staticmethod
+    def get_module_stream_name(source_branch):
+        """
+        Returns a branch name for modules
+        """
+        regex = r'stream-([a-zA-Z0-9_\.]+)-([a-zA-Z0-9_\.]+)'
+        regex_search = re.search(regex, source_branch)
+        return regex_search.group(2)
+
+    @staticmethod
+    def get_module_stream_version(release, source_branch, timestamp):
+        """
+        Returns a code of major, minor, micro version if applicable
+        """
+        if 'rhel' not in source_branch:
+            return f'{release}'
+
+        regex = r'rhel-([0-9]+)\.([0-9]+)\.([0-9]+)'
+        regex_search = re.search(regex, source_branch)
+        minor_version = regex_search.group(2)
+        micro_version = regex_search.group(3)
+        if len(regex_search.group(2)) == 1:
+            minor_version = f'0{regex_search.group(2)}'
+
+        if len(regex_search.group(3)) == 1:
+            micro_version = f'0{regex_search.group(3)}'
+
+        return f'{release}{minor_version}{micro_version}'
+
 # pylint: disable=too-many-instance-attributes
 class SrpmImport(Import):
     """
@@ -534,7 +573,8 @@ class GitImport(Import):
         # it'll be a module. Since this should always be the case, we'll change
         # dest_branch to be: {dest_branch}-stream-{stream_name}
         if "stream" in source_branch:
-            dest_branch = self.__get_module_stream_branch_name(source_branch, dest_branch)
+            _stream_name = self.get_module_stream_name(source_branch)
+            dest_branch = f'{dest_branch}-stream-{_stream_name}'
             distmarker = self.dist_tag.lstrip('.')
             _dist_tag = f'.module+{distmarker}+1010+deadbeef'
 
@@ -676,15 +716,6 @@ class GitImport(Import):
         substitute = template.substitute(dict_template)
         return substitute
 
-    @staticmethod
-    def __get_module_stream_branch_name(source_branch, dest_branch):
-        """
-        Returns a branch name for modules
-        """
-        regex = r'stream-([a-zA-Z0-9_\.]+)-([a-zA-Z0-9_\.]+)'
-        regex_search = re.search(regex, source_branch)
-        return f'{dest_branch}-stream-{regex_search.group(2)}'
-
     @property
     def rpm_name(self):
         """
@@ -755,3 +786,234 @@ class GitImport(Import):
         Returns destination local lookaside
         """
         return self.__dest_lookaside
+
+class ModuleImport(Import):
+    """
+    Imports module repos
+    """
+    # This needs to clone whatever is there, find if there's a SOURCES
+    # directory, if not make it. Make changes to the YAML to point to the
+    # destination branch, copy it to SOURCES, make a metadata file.
+    # pylint: disable=too-many-arguments
+    def __init__(
+            self,
+            module: str,
+            source_git_url_path: str,
+            source_git_org_path: str,
+            git_url_path: str,
+            release: str,
+            branch: str,
+            source_git_protocol: str = 'https',
+            dest_branch: str = '',
+            distprefix: str = 'el',
+            git_user: str = 'git',
+            org: str = 'modules'
+    ):
+        """
+        Init the class
+        """
+        #if not HAS_GI:
+        #    raise err.GenericError('This class cannot be loaded due to missing modules.')
+
+        self.__module = module
+        self.__release = release
+        # pylint: disable=line-too-long
+        self.__source_git_url = f'{source_git_protocol}://{source_git_url_path}/{source_git_org_path}/{module}.git'
+        self.__git_url = f'ssh://{git_user}@{git_url_path}/{org}/{module}.git'
+        self.__dist_prefix = distprefix
+        self.__dist_tag = f'.{distprefix}{release}'
+        self.__branch = branch
+        self.__dest_branch = branch
+        self.__current_time = datetime.datetime.utcnow().strftime('%Y%m%d%H%M%S')
+
+        if len(dest_branch) > 0:
+            self.__dest_branch = dest_branch
+
+        if "stream" not in self.__branch:
+            raise err.ConfigurationError('Source branch does not contain stream')
+
+        self.__stream_name = self.get_module_stream_name(branch)
+
+    def module_import(self):
+        """
+        Actually perform the import.
+        """
+        check_source_repo = gitutil.lsremote(self.source_git_url)
+        check_dest_repo = gitutil.lsremote(self.dest_git_url)
+        source_git_repo_path = f'/var/tmp/{self.module_name}-source'
+        dest_git_repo_path = f'/var/tmp/{self.module_name}'
+        modulemd_file = f'{source_git_repo_path}/{self.module_name}.yaml'
+        metadata_file = f'{dest_git_repo_path}/.{self.module_name}.metadata'
+        source_branch = self.source_branch
+        dest_branch = self.dest_branch
+        _dist_tag = self.dist_tag
+        stream_name = self.stream_name
+        repo_tags = []
+
+        # If the upstream repo doesn't report anything, exit.
+        if not check_source_repo:
+            raise err.GitInitError('Upstream git repo does not exist')
+
+        dest_branch = f'{dest_branch}-stream-{stream_name}'
+        module_version = self.get_module_stream_version(self.release,
+                                                        source_branch,
+                                                        self.datestamp)
+        nsvc = f'{self.module_name}-{stream_name}-{module_version}.deadbeef'
+        import_tag = generic.safe_encoding(
+                f'imports/{dest_branch}/{nsvc}'
+        )
+        commit_msg = f'import {nsvc}'
+
+        print(f'Cloning upstream: {self.module_name}')
+        source_repo = gitutil.clone(
+                git_url_path=self.source_git_url,
+                repo_name=self.module_name,
+                to_path=source_git_repo_path,
+                branch=source_branch
+        )
+
+        if check_dest_repo:
+            ref_check = f'refs/heads/{dest_branch}' in check_dest_repo
+            print(f'Cloning: {self.module_name}')
+            if ref_check:
+                dest_repo = gitutil.clone(
+                        git_url_path=self.dest_git_url,
+                        repo_name=self.module_name,
+                        to_path=dest_git_repo_path,
+                        branch=dest_branch
+                )
+            else:
+                dest_repo = gitutil.clone(
+                        git_url_path=self.dest_git_url,
+                        repo_name=self.module_name,
+                        to_path=dest_git_repo_path,
+                        branch=None
+                )
+                gitutil.checkout(dest_repo, branch=dest_branch, orphan=True)
+            self.remove_everything(dest_repo.working_dir)
+            for tag_name in dest_repo.tags:
+                repo_tags.append(tag_name.name)
+        else:
+            print('Repo may not exist or is private. Try to import anyway.')
+            dest_repo = gitutil.init(
+                    git_url_path=self.dest_git_url,
+                    repo_name=self.module_name,
+                    to_path=dest_git_repo_path,
+                    branch=dest_branch
+            )
+
+        # We'd normally look for similar tags. But the date time is always
+        # going to change, so we're skipping that part.
+
+        if not os.path.exists(f'{dest_git_repo_path}/SOURCES'):
+            try:
+                os.makedirs(f'{dest_git_repo_path}/SOURCES')
+            except Exception as exc:
+                raise err.GenericError(f'Directory could not be created: {exc}')
+
+        # We eventually want to do it this way.
+        #if Version(Modulemd.get_version()) < Version("2.11"):
+        #    source_modulemd = Modulemd.ModuleStream.read_file(
+        #            modulemd_file,
+        #            True,
+        #            self.module_name
+        #    )
+        #else:
+        #    source_modulemd = Modulemd.read_packager_file(modulemd_file,
+        #                                                  self.module_name,
+        #                                                  stream_name)
+        #components = source_modulemd.get_rpm_component_names()
+        #for component in components:
+        #    change = source_modulemd.get_rpm_component(component)
+        #    change.set_ref(dest_branch)
+
+        with open(modulemd_file, 'r') as module_yaml:
+            content = module_yaml.read()
+            content_new = re.sub('ref:\s+(.*)', f'ref: {dest_branch}', content)
+            module_yaml.close()
+
+        # Write to the root
+        with open(f'{dest_git_repo_path}/{self.module_name}.yaml', 'w') as module_yaml:
+            module_yaml.write(content_new)
+            module_yaml.close()
+
+        # Write to the sources, should be the same content
+        with open(f'{dest_git_repo_path}/SOURCES/modulemd.src.txt', 'w') as module_yaml:
+            module_yaml.write(content_new)
+            module_yaml.close()
+
+        self.generate_metadata(dest_git_repo_path, self.module_name, {})
+        gitutil.add_all(dest_repo)
+        verify = dest_repo.is_dirty()
+        if verify:
+            gitutil.commit(dest_repo, commit_msg)
+            ref = gitutil.tag(dest_repo, import_tag, commit_msg)
+            gitutil.push(dest_repo, ref=ref)
+            self.perform_cleanup([source_git_repo_path, dest_git_repo_path])
+            return True
+        print('Nothing to push')
+        self.perform_cleanup([source_git_repo_path, dest_git_repo_path])
+        return False
+
+    @property
+    def module_name(self):
+        """
+        Returns the module name
+        """
+        return self.__module
+
+    @property
+    def source_branch(self):
+        """
+        Returns the starting branch
+        """
+        return self.__branch
+
+    @property
+    def dest_branch(self):
+        """
+        Returns the starting branch
+        """
+        return self.__dest_branch
+
+    @property
+    def source_git_url(self):
+        """
+        Returns the source git url
+        """
+        return self.__source_git_url
+
+    @property
+    def dest_git_url(self):
+        """
+        Returns the destination git url
+        """
+        return self.__git_url
+
+    @property
+    def dist_tag(self):
+        """
+        Returns the dist tag
+        """
+        return self.__dist_tag
+
+    @property
+    def datestamp(self):
+        """
+        Returns a date time stamp
+        """
+        return self.__current_time
+
+    @property
+    def stream_name(self):
+        """
+        Returns the stream name
+        """
+        return self.__stream_name
+
+    @property
+    def release(self):
+        """
+        Returns the release
+        """
+        return self.__release