feature: "tagless import" feature mode (#4)

In order to build from lookahead sources, srpmproc must support the new gitlab address for centos 9 stream as well as manipulating the sources to expected format.

Resolves #2.


- Options for tagless import and new stream lookaside format added
- Git Mode now supports scanning for branch head in addition to the "normal" pulling of specific version tags
- Alternate ProcessRPM added for Tagless mode in process.go (still a WIP)
- Tagless mode converts a repo to the "traditional" format (SPECS/ + SOURCES/ + <pkg>.metadata )
- Tagless mode will build a dummy srpm to determine NVR information (rpmbuild + rpm shell commands) (will use this to tag imports in the target git)
- Limitation:  Tagless imports only pull the latest head from a branch
- CentOS-Stream import branches are converted from stream-<MODULE_VERSION>-<RHEL_VERSION> to the more familiar r9s-stream-<VERSION>
- stream-style YAML is detected and converted for modules, similar to the older modulemd.src.txt files
- This new pattern is for "tagless mode" only, previous tagged imports (from git.centos.org) should not be affected

cmd/srpmproc/main.go

@@ -56,6 +56,8 @@ var (
  basicPassword string
  packageVersion string
  packageRelease string
+ taglessMode bool
+ altLookAside bool
 )
 
 var root = &cobra.Command{
@@ -92,7 +94,10 @@ func mn(_ *cobra.Command, _ []string) {
  HttpPassword: basicPassword,
  PackageVersion: packageVersion,
  PackageRelease: packageRelease,
+ TaglessMode: taglessMode,
+ AltLookAside: altLookAside,
  })
+
  if err != nil {
  log.Fatal(err)
  }
@@ -142,6 +147,8 @@ func main() {
  root.Flags().StringVar(&basicPassword, "basic-password", "", "Basic auth password")
  root.Flags().StringVar(&packageVersion, "package-version", "", "Package version to fetch")
  root.Flags().StringVar(&packageRelease, "package-release", "", "Package release to fetch")
+ root.Flags().BoolVar(&taglessMode, "taglessmode", false, "Tagless mode: If set, pull the latest commit from a branch, and determine version info from spec file (aka upstream versions aren't tagged)")
+ root.Flags().BoolVar(&altLookAside, "altlookaside", false, "If set, uses the new CentOS Stream lookaside pattern (https://<SITE_PREFIX>/<RPM_NAME>/<FILE_NAME>/<SHA_VERSION>/<SHA_SUM>/<FILE_NAME>)")
 
  if err := root.Execute(); err != nil {
  log.Fatal(err)

pkg/data/process.go

@@ -57,4 +57,6 @@ type ProcessData struct {
  Log *log.Logger
  PackageVersion string
  PackageRelease string
+ TaglessMode bool
+ AltLookAside bool
 }

pkg/misc/regex.go

@@ -5,6 +5,7 @@ import (
  "github.com/rocky-linux/srpmproc/pkg/data"
  "path/filepath"
  "regexp"
+ "strings"
 )
 
 func GetTagImportRegex(pd *data.ProcessData) *regexp.Regexp {
@@ -31,3 +32,22 @@ func GetTagImportRegex(pd *data.ProcessData) *regexp.Regexp {
 
  return regexp.MustCompile(regex)
 }
+
+// Given a git reference in tagless mode (like "refs/heads/c9s", or "refs/heads/stream-httpd-2.4-rhel-9.1.0"), determine
+// if we are ok with importing that reference. We are looking for the traditional <prefix><version><suffix> pattern, like "c9s", and also the
+// modular "stream-<NAME>-<VERSION>-rhel-<VERSION> branch pattern as well
+func TaglessRefOk(tag string, pd *data.ProcessData) bool {
+
+ // First case is very easy: if we are "refs/heads/<prefix><version><suffix>" , then this is def. a branch we should import
+ if strings.HasPrefix(tag, fmt.Sprintf("refs/heads/%s%d%s", pd.ImportBranchPrefix, pd.Version, pd.BranchSuffix)) {
+ return true
+ }
+
+ // Less easy: if a modular branch is present (starts w/ "stream-"), we need to check if it's part of our major version, and return true if it is
+ // (major version means we look for the text "rhel-X." in the branch name, like "rhel-9.1.0")
+ if strings.HasPrefix(tag, "refs/heads/stream-") && strings.Contains(tag, fmt.Sprintf("rhel-%d.", pd.Version)) {
+ return true
+ }
+
+ return false
+}

pkg/modes/git.go

@@ -116,7 +116,6 @@ func (g *GitMode) RetrieveSource(pd *data.ProcessData) (*data.ModeData, error) {
  if exists != nil && exists.when.After(tag.Tagger.When) {
  return nil
  }
-
  latestTags[match[2]] = &remoteTarget{
  remote: refSpec,
  when: tag.Tagger.When,
@@ -126,11 +125,39 @@ func (g *GitMode) RetrieveSource(pd *data.ProcessData) (*data.ModeData, error) {
  return nil
  }
 
+ // In case of "tagless mode", we need to get the head ref of the branch instead
+ // This is a kind of alternative implementation of the above tagAdd assignment
+ refAdd := func(tag *object.Tag) error {
+ if misc.TaglessRefOk(tag.Name, pd) {
+ pd.Log.Printf("Tagless mode: Identified tagless commit for import: %s\n", tag.Name)
+ refSpec := fmt.Sprintf(tag.Name)
+
+ // We split the string by "/", the branch name we're looking for to pass to latestTags is always last
+ // (ex: "refs/heads/c9s" ---> we want latestTags[c9s]
+ tmpRef := strings.Split(refSpec, "/")
+ tmpBranchName := tmpRef[(len(tmpRef) - 1)]
+
+ latestTags[tmpBranchName] = &remoteTarget{
+ remote: refSpec,
+ when: tag.Tagger.When,
+ }
+ }
+ return nil
+ }
+
  tagIter, err := repo.TagObjects()
+
  if err != nil {
  return nil, fmt.Errorf("could not get tag objects: %v", err)
  }
- _ = tagIter.ForEach(tagAdd)
+
+ // tagless mode means we use "refAdd" (add commit by reference)
+ // normal mode means we can rely on "tagAdd" (the tag should be present for us in the source repo)
+ if pd.TaglessMode {
+ _ = tagIter.ForEach(refAdd)
+ } else {
+ _ = tagIter.ForEach(tagAdd)
+ }
 
  listOpts := &git.ListOptions{
  Auth: pd.Authenticator,
@@ -157,17 +184,26 @@ func (g *GitMode) RetrieveSource(pd *data.ProcessData) (*data.ModeData, error) {
  if err != nil {
  continue
  }
- _ = tagAdd(&object.Tag{
- Name: strings.TrimPrefix(string(ref.Name()), "refs/tags/"),
- Tagger: commit.Committer,
- })
+
+ // Call refAdd instead of tagAdd in the case of TaglessMode enabled
+ if pd.TaglessMode {
+ _ = refAdd(&object.Tag{
+ Name: string(ref.Name()),
+ Tagger: commit.Committer,
+ })
+ } else {
+ _ = tagAdd(&object.Tag{
+ Name: strings.TrimPrefix(string(ref.Name()), "refs/tags/"),
+ Tagger: commit.Committer,
+ })
+ }
+
  }
 
  for _, branch := range latestTags {
  pd.Log.Printf("tag: %s", strings.TrimPrefix(branch.remote, "refs/tags/"))
  branches = append(branches, *branch)
  }
-
  sort.Sort(branches)
 
  var sortedBranches []string
@@ -185,54 +221,66 @@ func (g *GitMode) RetrieveSource(pd *data.ProcessData) (*data.ModeData, error) {
 }
 
 func (g *GitMode) WriteSource(pd *data.ProcessData, md *data.ModeData) error {
+
  remote, err := md.Repo.Remote("upstream")
- if err != nil {
+
+ if err != nil && !pd.TaglessMode {
  return fmt.Errorf("could not get upstream remote: %v", err)
  }
 
  var refspec config.RefSpec
  var branchName string
 
- if strings.HasPrefix(md.TagBranch, "refs/heads") {
- refspec = config.RefSpec(fmt.Sprintf("+%s:%s", md.TagBranch, md.TagBranch))
- branchName = strings.TrimPrefix(md.TagBranch, "refs/heads/")
- } else {
- match := misc.GetTagImportRegex(pd).FindStringSubmatch(md.TagBranch)
- branchName = match[2]
- refspec = config.RefSpec(fmt.Sprintf("+refs/heads/%s:%s", branchName, md.TagBranch))
- }
- pd.Log.Printf("checking out upstream refspec %s", refspec)
- fetchOpts := &git.FetchOptions{
- Auth: pd.Authenticator,
- RemoteName: "upstream",
- RefSpecs: []config.RefSpec{refspec},
- Tags: git.AllTags,
- Force: true,
- }
- err = remote.Fetch(fetchOpts)
- if err != nil && err != git.NoErrAlreadyUpToDate {
- if err == transport.ErrInvalidAuthMethod || err == transport.ErrAuthenticationRequired {
- fetchOpts.Auth = nil
- err = remote.Fetch(fetchOpts)
- if err != nil && err != git.NoErrAlreadyUpToDate {
+ // In the case of tagless mode, we already have the transformed repo sitting in the worktree,
+ // and don't need to perform any checkout or fetch operations
+ if !pd.TaglessMode {
+ if strings.HasPrefix(md.TagBranch, "refs/heads") {
+ refspec = config.RefSpec(fmt.Sprintf("+%s:%s", md.TagBranch, md.TagBranch))
+ branchName = strings.TrimPrefix(md.TagBranch, "refs/heads/")
+ } else {
+ match := misc.GetTagImportRegex(pd).FindStringSubmatch(md.TagBranch)
+ branchName = match[2]
+ refspec = config.RefSpec(fmt.Sprintf("+refs/heads/%s:%s", branchName, md.TagBranch))
+ fmt.Println("Found branchname that does not start w/ refs/heads :: ", branchName)
+ }
+ pd.Log.Printf("checking out upstream refspec %s", refspec)
+
+ fetchOpts := &git.FetchOptions{
+ Auth: pd.Authenticator,
+ RemoteName: "upstream",
+ RefSpecs: []config.RefSpec{refspec},
+ Tags: git.AllTags,
+ Force: true,
+ }
+ err = remote.Fetch(fetchOpts)
+ if err != nil && err != git.NoErrAlreadyUpToDate {
+ if err == transport.ErrInvalidAuthMethod || err == transport.ErrAuthenticationRequired {
+ fetchOpts.Auth = nil
+ err = remote.Fetch(fetchOpts)
+ if err != nil && err != git.NoErrAlreadyUpToDate {
+ return fmt.Errorf("could not fetch upstream: %v", err)
+ }
+ } else {
  return fmt.Errorf("could not fetch upstream: %v", err)
  }
- } else {
- return fmt.Errorf("could not fetch upstream: %v", err)
  }
- }
 
- err = md.Worktree.Checkout(&git.CheckoutOptions{
- Branch: plumbing.ReferenceName(md.TagBranch),
- Force: true,
- })
- if err != nil {
- return fmt.Errorf("could not checkout source from git: %v", err)
+ err = md.Worktree.Checkout(&git.CheckoutOptions{
+ Branch: plumbing.ReferenceName(md.TagBranch),
+ Force: true,
+ })
+ if err != nil {
+ return fmt.Errorf("could not checkout source from git: %v", err)
+ }
+
+ _, err = md.Worktree.Add(".")
+ if err != nil {
+ return fmt.Errorf("could not add Worktree: %v", err)
+ }
  }
 
- _, err = md.Worktree.Add(".")
- if err != nil {
- return fmt.Errorf("could not add Worktree: %v", err)
+ if pd.TaglessMode {
+ branchName = fmt.Sprintf("%s%d%s", pd.ImportBranchPrefix, pd.Version, pd.BranchSuffix)
  }
 
  metadataPath := ""
@@ -292,7 +340,32 @@ func (g *GitMode) WriteSource(pd *data.ProcessData, md *data.ModeData) error {
  body = fromBlobStorage
  pd.Log.Printf("downloading %s from blob storage", hash)
  } else {
- url := fmt.Sprintf("%s/%s/%s/%s", pd.CdnUrl, md.Name, branchName, hash)
+
+ url := ""
+ // Alternate lookaside logic: if enabled, we pull from a new URL pattern
+ if !pd.AltLookAside {
+ url = fmt.Sprintf("%s/%s/%s/%s", pd.CdnUrl, md.Name, branchName, hash)
+ } else {
+ // We first need the hash algorithm based on length of hash:
+ hashType := "sha512"
+ switch len(hash) {
+ case 128:
+ hashType = "sha512"
+ case 64:
+ hashType = "sha256"
+ case 40:
+ hashType = "sha1"
+ case 32:
+ hashType = "md5"
+ }
+
+ // need the name of the file without "SOURCES/":
+ fileName := strings.Split(path, "/")[1]
+
+ // Alt. lookaside url is of the form: <cdn> / <name> / <filename> / <hashtype> / <hash> / <filename>
+ url = fmt.Sprintf("%s/%s/%s/%s/%s/%s", pd.CdnUrl, md.Name, fileName, hashType, hash, fileName)
+ }
+
  pd.Log.Printf("downloading %s", url)
 
  req, err := http.NewRequest("GET", url, nil)

pkg/srpmproc/patch.go

@@ -32,6 +32,7 @@ import (
  "strings"
  "time"
 
+ "github.com/go-git/go-billy/v5"
  "github.com/go-git/go-billy/v5/memfs"
  "github.com/go-git/go-git/v5"
  "github.com/go-git/go-git/v5/config"
@@ -286,13 +287,30 @@ func patchModuleYaml(pd *data.ProcessData, md *data.ModeData) error {
  return nil
  }
 
- mdTxtPath := "SOURCES/modulemd.src.txt"
- f, err := md.Worktree.Filesystem.Open(mdTxtPath)
- if err != nil {
- mdTxtPath = "SOURCES/modulemd.txt"
+ mdTxtPath := ""
+ var f billy.File
+
+ // tagless mode implies we're looking for CentOS Stream modules, which are generally "SOURCES/NAME.yaml" (copied to SOURCES/ from import)
+ // if not tagless mode, proceed as usual with SOURCES/modulemd.*.txt
+ if pd.TaglessMode {
+ mdTxtPath = fmt.Sprintf("SOURCES/%s.yaml", md.Name)
+ f, err = md.Worktree.Filesystem.Open(mdTxtPath)
+ if err != nil {
+ mdTxtPath = fmt.Sprintf("SOURCES/%s.yml", md.Name)
+ f, err = md.Worktree.Filesystem.Open(mdTxtPath)
+ if err != nil {
+ return fmt.Errorf("could not open modulemd file: %v", err)
+ }
+ }
+ } else {
+ mdTxtPath = "SOURCES/modulemd.src.txt"
  f, err = md.Worktree.Filesystem.Open(mdTxtPath)
  if err != nil {
- return fmt.Errorf("could not open modulemd file: %v", err)
+ mdTxtPath = "SOURCES/modulemd.txt"
+ f, err = md.Worktree.Filesystem.Open(mdTxtPath)
+ if err != nil {
+ return fmt.Errorf("could not open modulemd file: %v", err)
+ }
  }
  }
 
@@ -307,11 +325,13 @@ func patchModuleYaml(pd *data.ProcessData, md *data.ModeData) error {
  }
 
  // Get stream branch from tag
- match := misc.GetTagImportRegex(pd).FindStringSubmatch(md.TagBranch)
- streamBranch := strings.Split(match[2], "-")
- // Force stream to be the same as stream name in branch
- module.Data.Stream = streamBranch[len(streamBranch)-1]
-
+ // (in tagless mode we are trusting the "Stream: <VERSION>" text in the source YAML to be accurate)
+ if !pd.TaglessMode {
+ match := misc.GetTagImportRegex(pd).FindStringSubmatch(md.TagBranch)
+ streamBranch := strings.Split(match[2], "-")
+ // Force stream to be the same as stream name in branch
+ module.Data.Stream = streamBranch[len(streamBranch)-1]
+ }
  log.Println("This module contains the following rpms:")
  for name := range module.Data.Components.Rpms {
  pd.Log.Printf("\t- %s", name)