Merge pull request #16 from dirkmueller/openela

Support the openELA "PATCHES" directory format as well

.goreleaser.yml

@@ -17,11 +17,13 @@ builds:
       - s390x
       - ppc64le
 archives:
-  - replacements:
-      darwin: Darwin
-      linux: Linux
-      amd64: x86_64
-      arm64: aarch64
+  - name_template: >-
+      {{- .ProjectName }}_
+      {{- .Version }}
+      {{- title .Os }}_
+      {{- if eq .Arch "amd64" }}x86_64
+      {{- else if eq .Arch "arm64" }}aarch64
+      {{- else }}{{ .Arch }}{{ end }}{{ end -}}
 checksum:
   name_template: 'checksums.txt'
 snapshot:

README.md

@@ -1,14 +1,14 @@
 # srpmproc
 Upstream package importer with auto patching. Reference implementation for OpenPatch
 
-# Usage
+## Usage
 ```
 Usage:
   srpmproc [flags]
   srpmproc [command]
 
 Available Commands:
-  fetch
+  fetch       
   help        Help about any command
 
 Flags:
@@ -16,7 +16,8 @@ Flags:
       --basic-username string           Basic auth username
       --branch-prefix string            Branch prefix (replaces import-branch-prefix) (default "r")
       --branch-suffix string            Branch suffix to use for imported branches
-      --cdn-url string                  CDN URL to download blobs from (default "https://git.centos.org/sources")
+      --cdn string                      CDN URL shortcuts for well-known distros, auto-assigns --cdn-url.  Valid values:  rocky8, rocky, fedora, centos, centos-stream.  Setting this overrides --cdn-url
+      --cdn-url string                  CDN URL to download blobs from. Simple URL follows default rocky/centos patterns. Can be customized using macros (see docs) (default "https://git.centos.org/sources")
       --git-committer-email string      Email of committer (default "rockyautomation@rockylinux.org")
       --git-committer-name string       Name of committer (default "rockyautomation")
   -h, --help                            help for srpmproc
@@ -28,6 +29,8 @@ Flags:
       --no-dup-mode                     If enabled, skips already imported tags
       --no-storage-download             If enabled, blobs are always downloaded from upstream
       --no-storage-upload               If enabled, blobs are not uploaded to blob storage
+      --package-release string          Package release to fetch
+      --package-version string          Package version to fetch
       --rpm-prefix string               Where to retrieve SRPM content. Only used when source-rpm is not a local file (default "https://git.centos.org/rpms")
       --single-tag string               If set, only this tag is imported
       --source-rpm string               Location of RPM to process
@@ -35,9 +38,42 @@ Flags:
       --ssh-user string                 SSH User (default "git")
       --storage-addr string             Bucket to use as blob storage
       --strict-branch-mode              If enabled, only branches with the calculated name are imported and not prefix only
+      --taglessmode                     Tagless mode:  If set, pull the latest commit from the branch and determine version numbers from spec file.  This is auto-tried if tags aren't found.
       --tmpfs-mode string               If set, packages are imported to path and patched but not pushed
       --upstream-prefix string          Upstream git repository prefix
       --version int                     Upstream version
 
 Use "srpmproc [command] --help" for more information about a command.
 ```
+
+<br />
+
+## Examples:
+
+1. Import the kernel package from git.centos.org/rpms/, to local folder /opt/gitroot/rpms/kernel.git/ .  Download the lookaside source tarballs from the default CentOS file server location to local folder `/opt/fake_s3/` .  We want to grab branch "c8" (import prefix plus RHEL version), and it will be committed as branch "r8" (branch prefix plus RHEL version).  This assumes that `/opt/fake_s3` exists, and `/opt/gitroot/rpms/kernel.git` exists and is a git repository of some kind (even an empty one).
+
+```
+srpmproc --branch-prefix "r"  --import-branch-prefix "c"  --rpm-prefix "https://git.centos.org/rpms" --version 8 --storage-addr file:///opt/fake_s3  --upstream-prefix file:///opt/gitroot   --cdn centos --strict-branch-mode --source-rpm kernel
+```
+
+<br />
+
+## CDN and --cdn-url
+The --cdn-url option allows for Go-style templates to craft complex URL patterns.  These templates are: `{{.Name}}` (package name), `{{.Hash}}` (hash of lookaside file), `{{.Hashtype}}` (hash type of file, like "sha256" or "sha512"), `{{.Branch}}` (the branch we are importing), and `{{.Filename}}` (the lookaside file's name as it appears in SOURCES/).  You can add these values as part of --cdn-url to craft your lookaside pattern.
+
+
+For example, if I wanted my lookaside downloads to come from CentOS 9 Stream, I would use as part of my command:
+```
+--cdn-url "https://sources.stream.centos.org/sources/rpms/{{.Name}}/{{.Filename}}/{{.Hashtype}}/{{.Hash}}/{{.Filename}}"
+```
+
+
+**Default Behavior:**  If these templates are not used, the default behavior of `--cdn-url` is to fall back on the traditional RHEL import pattern:  `<CDN_URL>/<NAME>/<BRANCH>/<HASH>` .  If that fails, a further fallback is attempted, the simple: `<CDN_URL>/<HASH>`.  These cover the common Rocky Linux and RHEL/CentOS imports if the base lookaside URL is the only thing given.  If no `--cdn-url` is specified, it defaults to "https://git.centos.org/sources" (for RHEL imports into Rocky Linux)
+
+
+**CDN Shorthand:** For convenience, some lookaside patterns for popular distros are provided via the `--cdn` option.  You can specify this without needing to use the longer `--cdn-url`.  For example, when importing from CentOS 9 Stream, you could use `--cdn centos-stream`
+
+
+
+
+

cmd/srpmproc/main.go

@@ -58,7 +58,8 @@ var (
 	packageVersion       string
 	packageRelease       string
 	taglessMode          bool
-	altLookAside         bool
+	cdn                  string
+	moduleBranchNames    bool
 )
 
 var root = &cobra.Command{
@@ -96,7 +97,8 @@ func mn(_ *cobra.Command, _ []string) {
 		PackageVersion:       packageVersion,
 		PackageRelease:       packageRelease,
 		TaglessMode:          taglessMode,
-		AltLookAside:         altLookAside,
+		Cdn:                  cdn,
+		ModuleBranchNames:    moduleBranchNames,
 	})
 	if err != nil {
 		log.Fatal(err)
@@ -131,7 +133,7 @@ func main() {
 	root.Flags().StringVar(&rpmPrefix, "rpm-prefix", "https://git.centos.org/rpms", "Where to retrieve SRPM content. Only used when source-rpm is not a local file")
 	root.Flags().StringVar(&importBranchPrefix, "import-branch-prefix", "c", "Import branch prefix")
 	root.Flags().StringVar(&branchPrefix, "branch-prefix", "r", "Branch prefix (replaces import-branch-prefix)")
-	root.Flags().StringVar(&cdnUrl, "cdn-url", "https://git.centos.org/sources", "CDN URL to download blobs from")
+	root.Flags().StringVar(&cdnUrl, "cdn-url", "https://git.centos.org/sources", "CDN URL to download blobs from. Simple URL follows default rocky/centos patterns. Can be customized using macros (see docs)")
 	root.Flags().StringVar(&singleTag, "single-tag", "", "If set, only this tag is imported")
 	root.Flags().BoolVar(&noDupMode, "no-dup-mode", false, "If enabled, skips already imported tags")
 	root.Flags().BoolVar(&moduleMode, "module-mode", false, "If enabled, imports a module instead of a package")
@@ -146,8 +148,9 @@ func main() {
 	root.Flags().StringVar(&basicPassword, "basic-password", "", "Basic auth password")
 	root.Flags().StringVar(&packageVersion, "package-version", "", "Package version to fetch")
 	root.Flags().StringVar(&packageRelease, "package-release", "", "Package release to fetch")
-	root.Flags().BoolVar(&taglessMode, "taglessmode", false, "Tagless mode:  If set, pull the latest commit from a branch, and determine version info from spec file (aka upstream versions aren't tagged)")
-	root.Flags().BoolVar(&altLookAside, "altlookaside", false, "If set, uses the new CentOS Stream lookaside pattern (https://<SITE_PREFIX>/<RPM_NAME>/<FILE_NAME>/<SHA_VERSION>/<SHA_SUM>/<FILE_NAME>)")
+	root.Flags().BoolVar(&taglessMode, "taglessmode", false, "Tagless mode:  If set, pull the latest commit from the branch and determine version numbers from spec file.  This is auto-tried if tags aren't found.")
+	root.Flags().StringVar(&cdn, "cdn", "", "CDN URL shortcuts for well-known distros, auto-assigns --cdn-url.  Valid values:  rocky8, rocky, fedora, centos, centos-stream.  Setting this overrides --cdn-url")
+	root.Flags().BoolVar(&moduleBranchNames, "module-branch-names-only", false, "If enabled, module imports will use the branch name that is being imported, rather than use the commit hash.")
 
 	if err := root.Execute(); err != nil {
 		log.Fatal(err)

pkg/data/process.go

@@ -59,5 +59,6 @@ type ProcessData struct {
 	PackageVersion       string
 	PackageRelease       string
 	TaglessMode          bool
-	AltLookAside         bool
+	Cdn                  string
+	ModuleBranchNames    bool
 }

pkg/misc/regex.go

@@ -38,8 +38,8 @@ func GetTagImportRegex(pd *data.ProcessData) *regexp.Regexp {
 // if we are ok with importing that reference.  We are looking for the traditional <prefix><version><suffix> pattern, like "c9s", and also the
 // modular "stream-<NAME>-<VERSION>-rhel-<VERSION> branch pattern as well
 func TaglessRefOk(tag string, pd *data.ProcessData) bool {
-	// First case is very easy: if we are "refs/heads/<prefix><version><suffix>" , then this is def. a branch we should import
-	if strings.HasPrefix(tag, fmt.Sprintf("refs/heads/%s%d%s", pd.ImportBranchPrefix, pd.Version, pd.BranchSuffix)) {
+	// First case is very easy: if we are exactly "refs/heads/<prefix><version><suffix>" , then this is def. a branch we should import
+	if tag == fmt.Sprintf("refs/heads/%s%d%s", pd.ImportBranchPrefix, pd.Version, pd.BranchSuffix) {
 		return true
 	}
 

pkg/modes/git.go

@@ -21,12 +21,15 @@
 package modes
 
 import (
+	"bytes"
 	"fmt"
 	"io/ioutil"
+	"log"
 	"net/http"
 	"path/filepath"
 	"sort"
 	"strings"
+	"text/template"
 	"time"
 
 	"github.com/go-git/go-git/v5/plumbing/transport"
@@ -46,6 +49,15 @@ type remoteTarget struct {
 	when   time.Time
 }
 
+// Struct to define the possible template values ( {{.Value}} in CDN URL strings:
+type Lookaside struct {
+	Name     string
+	Branch   string
+	Hash     string
+	Hashtype string
+	Filename string
+}
+
 type remoteTargetSlice []remoteTarget
 
 func (p remoteTargetSlice) Len() int {
@@ -341,44 +353,66 @@ func (g *GitMode) WriteSource(pd *data.ProcessData, md *data.ModeData) error {
 			} else {
 
 				url := ""
-				// Alternate lookaside logic:  if enabled, we pull from a new URL pattern
-				if !pd.AltLookAside {
-					url = fmt.Sprintf("%s/%s/%s/%s", pd.CdnUrl, md.Name, branchName, hash)
-				} else {
-					// We first need the hash algorithm based on length of hash:
-					hashType := "sha512"
-					switch len(hash) {
-					case 128:
-						hashType = "sha512"
-					case 64:
-						hashType = "sha256"
-					case 40:
-						hashType = "sha1"
-					case 32:
-						hashType = "md5"
+
+				// We need to figure out the hashtype for templating purposes:
+				hashType := "sha512"
+				switch len(hash) {
+				case 128:
+					hashType = "sha512"
+				case 64:
+					hashType = "sha256"
+				case 40:
+					hashType = "sha1"
+				case 32:
+					hashType = "md5"
+				}
+
+				// need the name of the file without "SOURCES/":
+				fileName := strings.Split(path, "/")[1]
+
+				// Feed our template info to ProcessUrl and transform to the real values: ( {{.Name}}, {{.Branch}}, {{.Hash}}, {{.Hashtype}}, {{.Filename}} )
+				url, hasTemplate := ProcessUrl(pd.CdnUrl, md.Name, branchName, hash, hashType, fileName)
+
+				var req *http.Request
+				var resp *http.Response
+
+				// Download the --cdn-url given, but *only* if it contains template strings ( {{.Name}} , {{.Hash}} , etc. )
+				// Otherwise we need to fall back to the traditional cdn-url patterns
+				if hasTemplate {
+					pd.Log.Printf("downloading %s", url)
+
+					req, err := http.NewRequest("GET", url, nil)
+					if err != nil {
+						return fmt.Errorf("could not create new http request: %v", err)
 					}
+					req.Header.Set("Accept-Encoding", "*")
 
-					// need the name of the file without "SOURCES/":
-					fileName := strings.Split(path, "/")[1]
-
-					// Alt. lookaside url is of the form: <cdn> / <name> / <filename> / <hashtype> / <hash> / <filename>
-					url = fmt.Sprintf("%s/%s/%s/%s/%s/%s", pd.CdnUrl, md.Name, fileName, hashType, hash, fileName)
+					resp, err = client.Do(req)
+					if err != nil {
+						return fmt.Errorf("could not download dist-git file: %v", err)
+					}
 				}
 
-				pd.Log.Printf("downloading %s", url)
-
-				req, err := http.NewRequest("GET", url, nil)
-				if err != nil {
-					return fmt.Errorf("could not create new http request: %v", err)
+				// Default cdn-url:  If we don't have a templated download string, try the default <SITE>/<PKG>/<BRANCH>/<HASH> pattern:
+				if resp == nil || resp.StatusCode != http.StatusOK {
+					url = fmt.Sprintf("%s/%s/%s/%s", pd.CdnUrl, md.Name, branchName, hash)
+					pd.Log.Printf("Attempting default URL: %s", url)
+					req, err = http.NewRequest("GET", url, nil)
+					if err != nil {
+						return fmt.Errorf("could not create new http request: %v", err)
+					}
+					req.Header.Set("Accept-Encoding", "*")
+					resp, err = client.Do(req)
+					if err != nil {
+						return fmt.Errorf("could not download dist-git file: %v", err)
+					}
 				}
-				req.Header.Set("Accept-Encoding", "*")
 
-				resp, err := client.Do(req)
-				if err != nil {
-					return fmt.Errorf("could not download dist-git file: %v", err)
-				}
-				if resp.StatusCode != http.StatusOK {
+				// If the default URL fails, we have one more pattern to try.  The simple <SITE>/<HASH> pattern
+				// If this one fails, we are truly lost, and have to bail out w/ an error:
+				if resp == nil || resp.StatusCode != http.StatusOK {
 					url = fmt.Sprintf("%s/%s", pd.CdnUrl, hash)
+					pd.Log.Printf("Attempting 2nd fallback URL: %s", url)
 					req, err = http.NewRequest("GET", url, nil)
 					if err != nil {
 						return fmt.Errorf("could not create new http request: %v", err)
@@ -458,3 +492,29 @@ func (g *GitMode) ImportName(pd *data.ProcessData, md *data.ModeData) string {
 
 	return strings.Replace(strings.TrimPrefix(md.TagBranch, "refs/heads/"), "%", "_", -1)
 }
+
+// Given a cdnUrl string as input, return same string, but with substituted
+// template values ( {{.Name}} , {{.Hash}}, {{.Filename}}, etc. )
+func ProcessUrl(cdnUrl string, name string, branch string, hash string, hashtype string, filename string) (string, bool) {
+	tmpUrl := Lookaside{name, branch, hash, hashtype, filename}
+
+	// Return cdnUrl as-is if we don't have any templates ("{{ .Variable }}") to process:
+	if !(strings.Contains(cdnUrl, "{{") && strings.Contains(cdnUrl, "}}")) {
+		return cdnUrl, false
+	}
+
+	// If we run into trouble with our template parsing, we'll just return the cdnUrl, exactly as we found it
+	tmpl, err := template.New("").Parse(cdnUrl)
+	if err != nil {
+		return cdnUrl, false
+	}
+
+	var result bytes.Buffer
+	err = tmpl.Execute(&result, tmpUrl)
+	if err != nil {
+		log.Fatalf("ERROR: Could not process CDN URL template(s) from URL string: %s\n", cdnUrl)
+	}
+
+	return result.String(), true
+
+}

pkg/rpmutils/regex.go

@@ -3,4 +3,4 @@ package rpmutils
 import "regexp"
 
 // Nvr is a regular expression that matches a NVR.
-var Nvr = regexp.MustCompile("^(\\S+)-([\\w~%.+]+)-(\\w+(?:\\.[\\w+]+)+?)(?:\\.rpm)?$")
+var Nvr = regexp.MustCompile("^(\\S+)-([\\w~%.+]+)-(\\w+(?:\\.[\\w~%+]+)+?)(?:\\.rpm)?$")

pkg/srpmproc/patch.go

@@ -23,7 +23,7 @@ package srpmproc
 import (
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"log"
 	"os"
 	"path/filepath"
@@ -48,10 +48,16 @@ import (
 
 func cfgPatches(pd *data.ProcessData, md *data.ModeData, patchTree *git.Worktree, pushTree *git.Worktree) error {
 	// check CFG patches
-	_, err := patchTree.Filesystem.Stat("ROCKY/CFG")
+	// use PATCHES directory if it exists otherwise ROCKY/CFG
+	cfgdir := "PATCHES"
+	_, err := patchTree.Filesystem.Stat(cfgdir)
+	if err != nil {
+		cfgdir = "ROCKY/CFG"
+	}
+	_, err = patchTree.Filesystem.Stat(cfgdir)
 	if err == nil {
 		// iterate through patches
-		infos, err := patchTree.Filesystem.ReadDir("ROCKY/CFG")
+		infos, err := patchTree.Filesystem.ReadDir(cfgdir)
 		if err != nil {
 			return fmt.Errorf("could not walk patches: %v", err)
 		}
@@ -63,12 +69,12 @@ func cfgPatches(pd *data.ProcessData, md *data.ModeData, patchTree *git.Worktree
 			}
 
 			pd.Log.Printf("applying directive %s", info.Name())
-			filePath := filepath.Join("ROCKY/CFG", info.Name())
+			filePath := filepath.Join(cfgdir, info.Name())
 			directive, err := patchTree.Filesystem.Open(filePath)
 			if err != nil {
 				return fmt.Errorf("could not open directive file %s: %v", info.Name(), err)
 			}
-			directiveBytes, err := ioutil.ReadAll(directive)
+			directiveBytes, err := io.ReadAll(directive)
 			if err != nil {
 				return fmt.Errorf("could not read directive file: %v", err)
 			}
@@ -96,7 +102,12 @@ func cfgPatches(pd *data.ProcessData, md *data.ModeData, patchTree *git.Worktree
 
 func applyPatches(pd *data.ProcessData, md *data.ModeData, patchTree *git.Worktree, pushTree *git.Worktree) error {
 	// check if patches exist
-	_, err := patchTree.Filesystem.Stat("ROCKY")
+	cfgdir := "PATCHES"
+	_, err := patchTree.Filesystem.Stat(cfgdir)
+	if err != nil {
+		cfgdir = "ROCKY"
+	}
+	_, err = patchTree.Filesystem.Stat(cfgdir)
 	if err == nil {
 		err := cfgPatches(pd, md, patchTree, pushTree)
 		if err != nil {
@@ -315,7 +326,7 @@ func patchModuleYaml(pd *data.ProcessData, md *data.ModeData) error {
 		}
 	}
 
-	content, err := ioutil.ReadAll(f)
+	content, err := io.ReadAll(f)
 	if err != nil {
 		return fmt.Errorf("could not read modulemd file: %v", err)
 	}
@@ -397,6 +408,12 @@ func patchModuleYaml(pd *data.ProcessData, md *data.ModeData) error {
 			}
 		}
 
+		// If we got this far, we're good. This means the repos, branches, and commits exist.
+		// If this option is on, just use the branch name.
+		if pd.ModuleBranchNames {
+			tipHash = md.PushBranch
+		}
+
 		rpm.Ref = tipHash
 	}
 

pkg/srpmproc/process.go

@@ -101,8 +101,15 @@ type ProcessDataRequest struct {
 	PackageVersion string
 	PackageRelease string
 
-	TaglessMode  bool
-	AltLookAside bool
+	TaglessMode bool
+	Cdn         string
+
+	ModuleBranchNames bool
+}
+
+type LookasidePath struct {
+	Distro string
+	Url    string
 }
 
 func gitlabify(str string) string {
@@ -113,7 +120,57 @@ func gitlabify(str string) string {
 	return strings.Replace(str, "+", "plus", -1)
 }
 
+// List of distros and their lookaside patterns
+// If we find one of these passed as --cdn (ex: "--cdn fedora"), then we override, and assign this URL to be our --cdn-url
+func StaticLookasides() []LookasidePath {
+	centos := LookasidePath{
+		Distro: "centos",
+		Url:    "https://git.centos.org/sources/{{.Name}}/{{.Branch}}/{{.Hash}}",
+	}
+	centosStream := LookasidePath{
+		Distro: "centos-stream",
+		Url:    "https://sources.stream.centos.org/sources/rpms/{{.Name}}/{{.Filename}}/{{.Hashtype}}/{{.Hash}}/{{.Filename}}",
+	}
+	rocky8 := LookasidePath{
+		Distro: "rocky8",
+		Url:    "https://rocky-linux-sources-staging.a1.rockylinux.org/{{.Hash}}",
+	}
+	rocky := LookasidePath{
+		Distro: "rocky",
+		Url:    "https://sources.build.resf.org/{{.Hash}}",
+	}
+	fedora := LookasidePath{
+		Distro: "fedora",
+		Url:    "https://src.fedoraproject.org/repo/pkgs/{{.Name}}/{{.Filename}}/{{.Hashtype}}/{{.Hash}}/{{.Filename}}",
+	}
+
+	return []LookasidePath{centos, centosStream, rocky8, rocky, fedora}
+
+}
+
+// Given a "--cdn" entry like "centos", we can search through our struct list of distros, and return the proper lookaside URL
+// If we can't find it, we return false and the calling function will error out
+func FindDistro(cdn string) (string, bool) {
+	var cdnUrl = ""
+
+	// Loop through each distro in the static list defined, try to find a match with "--cdn":
+	for _, distro := range StaticLookasides() {
+		if distro.Distro == strings.ToLower(cdn) {
+			cdnUrl = distro.Url
+			return cdnUrl, true
+		}
+	}
+	return "", false
+}
+
 func NewProcessData(req *ProcessDataRequest) (*data.ProcessData, error) {
+	// Build the logger to use for the data import
+	var writer io.Writer = os.Stdout
+	if req.LogWriter != nil {
+		writer = req.LogWriter
+	}
+	logger := log.New(writer, "", log.LstdFlags)
+
 	// Set defaults
 	if req.ModulePrefix == "" {
 		req.ModulePrefix = ModulePrefixCentOS
@@ -139,13 +196,22 @@ func NewProcessData(req *ProcessDataRequest) (*data.ProcessData, error) {
 	if req.BranchPrefix == "" {
 		req.BranchPrefix = "r"
 	}
-	if req.CdnUrl == "" && !req.AltLookAside {
+	if req.CdnUrl == "" {
 		req.CdnUrl = "https://git.centos.org/sources"
 	}
-	// If altlookaside is enabled, and the CdnUrl hasn't been changed, then automatically set it to the default
-	// CentOS Stream (the new pattern very much won't work with the old git.centos.org/sources site)
-	if (req.CdnUrl == "https://git.centos.org/sources" || req.CdnUrl == "") && req.AltLookAside {
-		req.CdnUrl = "https://sources.stream.centos.org/sources/rpms"
+
+	// If a Cdn distro is defined, we try to find a match from StaticLookasides() array of structs
+	// see if we have a match to --cdn (matching values are things like fedora, centos, rocky8, etc.)
+	// If we match, then we want to short-circuit the CdnUrl to the assigned distro's one
+	if req.Cdn != "" {
+		newCdn, foundDistro := FindDistro(req.Cdn)
+
+		if !foundDistro {
+			return nil, fmt.Errorf("Error, distro name given as --cdn argument is not valid.")
+		}
+
+		req.CdnUrl = newCdn
+		logger.Printf("Discovered --cdn distro: %s .  Using override CDN URL Pattern: %s", req.Cdn, req.CdnUrl)
 	}
 
 	// Validate required
@@ -214,12 +280,6 @@ func NewProcessData(req *ProcessDataRequest) (*data.ProcessData, error) {
 		reqFsCreator = req.FsCreator
 	}
 
-	var writer io.Writer = os.Stdout
-	if req.LogWriter != nil {
-		writer = req.LogWriter
-	}
-	logger := log.New(writer, "", log.LstdFlags)
-
 	if req.TmpFsMode != "" {
 		logger.Printf("using tmpfs dir: %s", req.TmpFsMode)
 		fsCreator = func(branch string) (billy.Filesystem, error) {
@@ -276,7 +336,8 @@ func NewProcessData(req *ProcessDataRequest) (*data.ProcessData, error) {
 		PackageVersion:       req.PackageVersion,
 		PackageRelease:       req.PackageRelease,
 		TaglessMode:          req.TaglessMode,
-		AltLookAside:         req.AltLookAside,
+		Cdn:                  req.Cdn,
+		ModuleBranchNames:    req.ModuleBranchNames,
 	}, nil
 }
 
@@ -334,6 +395,7 @@ func ProcessRPM(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error) {
 		list, err := remote.List(&git.ListOptions{
 			Auth: pd.Authenticator,
 		})
+
 		if err != nil {
 			log.Println("ignoring no-dup-mode")
 		} else {
@@ -354,17 +416,17 @@ func ProcessRPM(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error) {
 	if pd.SingleTag != "" {
 		md.Branches = []string{fmt.Sprintf("refs/tags/%s", pd.SingleTag)}
 	} else if len(pd.ManualCommits) > 0 {
-		md.Branches = []string{}
-		for _, commit := range pd.ManualCommits {
-			branchCommit := strings.Split(commit, ":")
-			if len(branchCommit) != 2 {
-				return nil, fmt.Errorf("invalid manual commit list")
-			}
+		log.Println("Manual commits were listed for import.  Switching to perform a tagless import of these commit(s).")
+		pd.TaglessMode = true
+		return processRPMTagless(pd)
+	}
 
-			head := fmt.Sprintf("refs/tags/imports/%s/%s-%s", branchCommit[0], md.Name, branchCommit[1])
-			md.Branches = append(md.Branches, head)
-			commitPin[head] = branchCommit[1]
-		}
+	// If we have no valid branches to consider, then we'll automatically switch to attempt a tagless import:
+	if len(md.Branches) == 0 {
+		log.Println("No valid tags (refs/tags/imports/*) found in repository!  Switching to perform a tagless import.")
+		pd.TaglessMode = true
+		result, err := processRPMTagless(pd)
+		return result, err
 	}
 
 	for _, branch := range md.Branches {
@@ -527,7 +589,7 @@ func ProcessRPM(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error) {
 			if err != nil {
 				return nil, fmt.Errorf("could not open ignored source file %s: %v", sourcePath, err)
 			}
-			sourceFileBts, err := ioutil.ReadAll(sourceFile)
+			sourceFileBts, err := io.ReadAll(sourceFile)
 			if err != nil {
 				return nil, fmt.Errorf("could not read the whole of ignored source file: %v", err)
 			}
@@ -596,6 +658,12 @@ func ProcessRPM(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error) {
 
 		// show status
 		status, _ := w.Status()
+		if !pd.ModuleMode {
+			if status.IsClean() {
+				pd.Log.Printf("No changes detected. Our downstream is up to date.")
+				continue
+			}
+		}
 		pd.Log.Printf("successfully processed:\n%s", status)
 
 		statusLines := strings.Split(status.String(), "\n")
@@ -681,7 +749,6 @@ func ProcessRPM(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error) {
 }
 
 // Process for when we want to import a tagless repo (like from CentOS Stream)
-//
 func processRPMTagless(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error) {
 	pd.Log.Println("Tagless mode detected, attempting import of latest commit")
 
@@ -746,6 +813,20 @@ func processRPMTagless(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error
 	sourceWorktree := *md.Worktree
 	localPath := ""
 
+	// if a manual commit list is provided, we want to create our md.Branches[] array in a special format:
+	if len(pd.ManualCommits) > 0 {
+		md.Branches = []string{}
+		for _, commit := range pd.ManualCommits {
+			branchCommit := strings.Split(commit, ":")
+			if len(branchCommit) != 2 {
+				return nil, fmt.Errorf("invalid manual commit list")
+			}
+
+			head := fmt.Sprintf("COMMIT:%s:%s", branchCommit[0], branchCommit[1])
+			md.Branches = append(md.Branches, head)
+		}
+	}
+
 	for _, branch := range md.Branches {
 		md.Repo = &sourceRepo
 		md.Worktree = &sourceWorktree
@@ -765,13 +846,35 @@ func processRPMTagless(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error
 			return nil, fmt.Errorf("Could not create temporary directory: %s", localPath)
 		}
 
+		// we'll make our branch we're processing more presentable if it's in the COMMIT:<branch>:<hash> format:
+		if strings.HasPrefix(branch, "COMMIT:") {
+			branch = fmt.Sprintf("refs/heads/%s", strings.Split(branch, ":")[1])
+		}
+
 		// Clone repo into the temporary path, but only the tag we're interested in:
 		// (TODO: will probably need to assign this a variable or use the md struct gitrepo object to perform a successful tag+push later)
-		_, _ = git.PlainClone(localPath, false, &git.CloneOptions{
+		rTmp, err := git.PlainClone(localPath, false, &git.CloneOptions{
 			URL:           pd.RpmLocation,
 			SingleBranch:  true,
 			ReferenceName: plumbing.ReferenceName(branch),
 		})
+		if err != nil {
+			return nil, err
+		}
+
+		// If we're dealing with a special manual commit to import ("COMMIT:<branch>:<githash>"), then we need to check out that
+		// specific hash from the repo we are importing from:
+		if strings.HasPrefix(md.TagBranch, "COMMIT:") {
+			commitList := strings.Split(md.TagBranch, ":")
+
+			wTmp, _ := rTmp.Worktree()
+			err = wTmp.Checkout(&git.CheckoutOptions{
+				Hash: plumbing.NewHash(commitList[2]),
+			})
+			if err != nil {
+				return nil, fmt.Errorf("Could not find manual commit %s in the repository.  Must be a valid commit hash.", commitList[2])
+			}
+		}
 
 		// Now that we're cloned into localPath, we need to "covert" the import into the old format
 		// We want sources to become .PKGNAME.metadata, we want SOURCES and SPECS folders, etc.
@@ -788,9 +891,9 @@ func processRPMTagless(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error
 
 		// get name-version-release of tagless repo, only if we're not a module repo:
 		if !pd.ModuleMode {
-			nvrString := getVersionFromSpec(md.Name, localPath, pd.Version)
-			if nvrString == "" {
-				return nil, fmt.Errorf("Error using rpm or rpmbuild to build SRPM and determine version info! (tagless mode)")
+			nvrString, err := getVersionFromSpec(localPath, pd.Version)
+			if err != nil {
+				return nil, err
 			}
 
 			// Set version and release fields we extracted (name|version|release are separated by pipes)
@@ -852,6 +955,12 @@ func processRPMTagless(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error
 			Force:  true,
 		})
 
+		// These os commands actually move the data from our cloned import repo to our cloned "push" repo (upstream -> downstream)
+		// First we clobber the target push repo's data, and then move the new data into it from the upstream repo we cloned earlier
+		os.RemoveAll(fmt.Sprintf("%s_gitpush/SPECS", localPath))
+		os.RemoveAll(fmt.Sprintf("%s_gitpush/SOURCES", localPath))
+		os.RemoveAll(fmt.Sprintf("%s_gitpush/.gitignore", localPath))
+		os.RemoveAll(fmt.Sprintf("%s_gitpush/%s.metadata", localPath, md.Name))
 		os.Rename(fmt.Sprintf("%s/SPECS", localPath), fmt.Sprintf("%s_gitpush/SPECS", localPath))
 		os.Rename(fmt.Sprintf("%s/SOURCES", localPath), fmt.Sprintf("%s_gitpush/SOURCES", localPath))
 		os.Rename(fmt.Sprintf("%s/.gitignore", localPath), fmt.Sprintf("%s_gitpush/.gitignore", localPath))
@@ -892,6 +1001,12 @@ func processRPMTagless(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error
 		}
 
 		status, err := w.Status()
+		if !pd.ModuleMode {
+			if status.IsClean() {
+				pd.Log.Printf("No changes detected. Our downstream is up to date.")
+				continue
+			}
+		}
 		pd.Log.Printf("successfully processed:\n%s", status)
 
 		// assign tag for our new remote we're about to push (derived from the SRPM version)
@@ -985,7 +1100,6 @@ func processRPMTagless(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error
 			Version: pd.PackageVersion,
 			Release: pd.PackageRelease,
 		}
-
 	}
 
 	// return struct with all our branch:commit and branch:version+release mappings
@@ -1108,113 +1222,50 @@ func convertMetaData(pkgName string, localRepo string) bool {
 // Given a local checked out folder and package name, including SPECS/ , SOURCES/ , and .package.metadata, this will:
 //   - create a "dummy" SRPM (using dummy sources files we use to populate tarballs from lookaside)
 //   - extract RPM version info from that SRPM, and return it
+//
 // If we are in tagless mode, we need to get a package version somehow!
-func getVersionFromSpec(pkgName string, localRepo string, majorVersion int) string {
+func getVersionFromSpec(localRepo string, majorVersion int) (string, error) {
 	// Make sure we have "rpm" and "rpmbuild" and "cp" available in our PATH.  Otherwise, this won't work:
-	_, err := exec.LookPath("rpm")
+	_, err := exec.LookPath("rpmspec")
 	if err != nil {
-		return ""
+		return "", fmt.Errorf("Could not find rpmspec program in PATH")
 	}
 
-	_, err = exec.LookPath("rpmbuild")
+	// Read the first file from SPECS/ to get our spec file
+	// (there should only be one file - we check that it ends in ".spec" just to be sure!)
+	lsTmp, err := ioutil.ReadDir(fmt.Sprintf("%s/SPECS/", localRepo))
 	if err != nil {
-		return ""
+		return "", err
+	}
+	specFile := lsTmp[0].Name()
+
+	if !strings.HasSuffix(specFile, ".spec") {
+		return "", fmt.Errorf("First file found in SPECS/ is not a .spec file!  Check the SPECS/ directory in the repo?")
 	}
 
-	_, err = exec.LookPath("cp")
+	// Call the rpmspec binary to extract the version-release info out of it, and tack on ".el<VERSION>" at the end:
+	cmdArgs := []string{
+		"--srpm",
+		fmt.Sprintf(`--define=dist  .el%d`, majorVersion),
+		fmt.Sprintf(`--define=_topdir  %s`, localRepo),
+		"-q",
+		"--queryformat",
+		`%{NAME}|%{VERSION}|%{RELEASE}\n`,
+		fmt.Sprintf("%s/SPECS/%s", localRepo, specFile),
+	}
+	cmd := exec.Command("rpmspec", cmdArgs...)
+	nvrTmp, err := cmd.Output()
 	if err != nil {
-		return ""
+		return "", fmt.Errorf("Error running rpmspec command to determine RPM name-version-release identifier. \nCommand attempted: %s \nCommand output: %s", cmd.String(), string(nvrTmp))
 	}
 
-	// create separate temp folder space to do our RPM work - we don't want to accidentally contaminate the main Git area:
-	rpmBuildPath := fmt.Sprintf("%s_rpm", localRepo)
-	os.Mkdir(rpmBuildPath, 0o755)
-
-	// Copy SOURCES/ and SPECS/ into the temp rpmbuild directory recursively
-	// Yes, we could create or import an elaborate Go-native way to do this, but damnit this is easier:
-	cmdArgs := strings.Fields(fmt.Sprintf("cp -rp %s/SOURCES %s/SPECS %s/", localRepo, localRepo, rpmBuildPath))
-	if err := exec.Command(cmdArgs[0], cmdArgs[1:]...).Run(); err != nil {
-		log.Println(err)
-		return ""
-	}
-
-	// Loop through .<package>.metadata and get the file names we need to make our SRPM:
-	lookAside, err := os.Open(fmt.Sprintf("%s/.%s.metadata", localRepo, pkgName))
-	if err != nil {
-		log.Println(err)
-		return ""
-	}
-
-	// Split file into lines and start processing:
-	scanner := bufio.NewScanner(lookAside)
-	scanner.Split(bufio.ScanLines)
-
-	// loop through each line, and:
-	//   - isolate the SOURCES/filename entry
-	//   - write out a dummy file of the same name to rpmBuildPath/SOURCES
-	for scanner.Scan() {
-
-		// lookaside source is always the 2nd part of the line (after the long SHA sum)
-		srcFile := strings.Fields(scanner.Text())[1]
-
-		// write a dummy file of the same name into the rpmbuild SOURCES/ directory:
-		dummyFile, err := os.OpenFile(fmt.Sprintf("%s/%s", rpmBuildPath, srcFile), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644)
-		if err != nil {
-			return ""
-		}
-		writer := bufio.NewWriter(dummyFile)
-		_, _ = writer.WriteString("This is a dummy lookaside file generated by srpmproc.  It is only needed to get a working SRPM and extract version information.  Please disregard\n")
-		writer.Flush()
-		dummyFile.Close()
-	}
-
-	lookAside.Close()
-
-	// Now, call rpmbuild to produce the dummy src file:
-	// Example:  rpmbuild  --define "_topdir  /tmp/srpmproctmp_httpd1988142783_rpm"  -bs /tmp/srpmproctmp_httpd1988142783_rpm/SPECS/*.spec
-	cmd := exec.Command("rpmbuild", fmt.Sprintf(`--define=_topdir  %s`, rpmBuildPath), fmt.Sprintf(`--define=dist  .el%d`, majorVersion), "-bs", fmt.Sprintf("%s/SPECS/%s.spec", rpmBuildPath, pkgName))
-	if err := cmd.Run(); err != nil {
-		log.Println(err)
-		return ""
-	}
-
-	// Read the first file from the SRPMS/ folder in rpmBuildPath.  It should be the SRPM that rpmbuild produced above
-	// (there should only be one file - we check that it ends in ".rpm" just to be sure!)
-	lsTmp, err := ioutil.ReadDir(fmt.Sprintf("%s/SRPMS/", rpmBuildPath))
-	if err != nil {
-		log.Println(err)
-		return ""
-	}
-
-	srpmFile := lsTmp[0].Name()
-
-	if !strings.HasSuffix(srpmFile, ".rpm") {
-		log.Println("Error, file found in dummy SRPMS directory did not have an .rpm extension!  Perhaps rpmbuild didn't produce a proper source RPM?")
-		return ""
-	}
-
-	// Call the rpm binary to extract the version-release info out of it, and tack on ".el<VERSION>" at the end:
-	cmd = exec.Command("rpm", "-qp", "--qf", `%{NAME}|%{VERSION}|%{RELEASE}\n`, fmt.Sprintf("%s/SRPMS/%s", rpmBuildPath, srpmFile))
-	nvrTmp, err := cmd.CombinedOutput()
-	if err != nil {
-		log.Println("Error running rpm command to extract temporary SRPM name-version-release identifiers.")
-		log.Println("rpmbuild output: ", string(nvrTmp))
-		log.Println("rpmbuild command: ", cmd.String())
-		return ""
-	}
-
-	// Pull first line of the rpm command's output to get the name-version-release number (there should only be 1 line)
+	// Pull first line of the version output to get the name-version-release number (there should only be 1 line)
 	nvr := string(nvrTmp)
 	nvr = strings.Fields(nvr)[0]
 
-	// Clean up: delete the temporary directory
-	if err := os.RemoveAll(rpmBuildPath); err != nil {
-		log.Printf("Error cleaning up temporary RPM directory %s .  Non-fatal, continuing anyway...\n", rpmBuildPath)
-	}
-
 	// return name-version-release string we derived:
-	log.Printf("Derived NVR %s from tagless repo via temporary SRPM build\n", nvr)
-	return nvr
+	log.Printf("Derived NVR %s from tagless repo via rpmspec command\n", nvr)
+	return nvr, nil
 }
 
 // We need to loop through the lookaside blob files ("SourcesToIgnore"),
@@ -1248,7 +1299,7 @@ func processLookasideSources(pd *data.ProcessData, md *data.ModeData, localDir s
 		if err != nil {
 			return fmt.Errorf("could not open ignored source file %s: %v", sourcePath, err)
 		}
-		sourceFileBts, err := ioutil.ReadAll(sourceFile)
+		sourceFileBts, err := io.ReadAll(sourceFile)
 		if err != nil {
 			return fmt.Errorf("could not read the whole of ignored source file: %v", err)
 		}