[mirror] Treat versions in filter expression as minimal releases to d…

…ownload instead of specific

Signed-off-by: Maxim Vasilenko <[email protected]>

internal/mirror/cmd/modules/pull/flags.go

@@ -40,7 +40,7 @@ func addFlags(flagSet *pflag.FlagSet) {
  "filter",
  "f",
  "",
- "Filter which modules to pull. Format is \"moduleName:v1.2.3\" or \"moduleName:release-channel\", separated by ';'.",
+ "Filter which modules starting with which version to pull. Format is \"moduleName@v1.2.3\" separated by ';' where version after @ is the earliest pulled version of the module.",
  )
  flagSet.BoolVar(
  &SkipTLSVerify,

internal/mirror/cmd/modules/pull/pull.go

@@ -36,7 +36,6 @@ import (
  "github.com/deckhouse/deckhouse-cli/internal/mirror/layouts"
  "github.com/deckhouse/deckhouse-cli/internal/mirror/modules"
  "github.com/deckhouse/deckhouse-cli/internal/mirror/util/log"
- "github.com/deckhouse/deckhouse-cli/internal/mirror/util/modfilter"
 )
 
 var pullLong = templates.LongDesc(`
@@ -104,21 +103,30 @@ func pullExternalModulesToLocalFS(
  if err != nil {
  return fmt.Errorf("Get external modules from %q: %w", src.Spec.Registry.Repo, err)
  }
-
  if len(modulesFromRepo) == 0 {
  log.WarnLn("No modules found in ModuleSource")
  return nil
  }
 
- tagsResolver := layouts.NewTagsResolver()
- filter := modfilter.ParseModuleFilterString(moduleFilterExpression)
- for i, module := range modulesFromRepo {
- if !filter.Match(module) {
- continue
- }
+ modulesFilter, err := modules.NewFilter(moduleFilterExpression)
+ if err != nil {
+ return fmt.Errorf("Bad modules filter: %w", err)
+ }
+ filteredModules := make([]modules.Module, 0)
+ if len(modulesFilter) > 0 {
+ for _, moduleData := range modulesFromRepo {
+ if !modulesFilter.MatchesFilter(&moduleData) {
+ continue
+ }
 
- filter.FilterModuleReleases(&module)
+ modulesFilter.FilterReleases(&moduleData)
+ filteredModules = append(filteredModules, moduleData)
+ }
+ modulesFromRepo = filteredModules
+ }
 
+ tagsResolver := layouts.NewTagsResolver()
+ for i, module := range modulesFromRepo {
  log.InfoF("[%d / %d] Pulling module %s...\n", i+1, len(modulesFromRepo), module.RegistryPath)
 
  moduleLayout, err := layouts.CreateEmptyImageLayoutAtPath(filepath.Join(mirrorDirectoryPath, module.Name))
@@ -130,7 +138,7 @@ func pullExternalModulesToLocalFS(
  return fmt.Errorf("Create module OCI Layouts: %w", err)
  }
 
- moduleImageSet, releasesImageSet, err := modules.FindExternalModuleImages(&module, authProvider, filter != nil, insecure, skipVerifyTLS)
+ moduleImageSet, releasesImageSet, err := modules.FindExternalModuleImages(&module, modulesFilter, authProvider, insecure, skipVerifyTLS)
  if err != nil {
  return fmt.Errorf("Find external module images`: %w", err)
  }
@@ -147,11 +155,6 @@ func pullExternalModulesToLocalFS(
  SkipTLSVerification: skipVerifyTLS,
  RegistryAuth: authProvider,
  },
- DoGOSTDigests: false,
- SkipModulesPull: false,
- BundleChunkSize: 0,
- MinVersion: nil,
- SpecificVersion: nil,
  }
 
  log.InfoLn("Beginning to pull module contents")

internal/mirror/cmd/modules/pull/validation.go

@@ -36,7 +36,7 @@ func validateModuleFilterFormat() error {
  return nil
  }
 
- if !regexp.MustCompile(`([a-zA-Z0-9-_]+:(v\d+\.\d+\.\d+|[a-zA-Z0-9_\-]+));?`).MatchString(ModulesFilter) {
+ if !regexp.MustCompile(`([a-zA-Z0-9-_]+@(v?\d+\.\d+\.\d+));?`).MatchString(ModulesFilter) {
  return errors.New("Invalid filter pattern")
  }
 

internal/mirror/modules/filter.go

@@ -0,0 +1,96 @@
+/*
+Copyright 2024 Flant JSC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package modules
+
+import (
+ "fmt"
+ "strings"
+
+ "github.com/deckhouse/deckhouse-cli/internal/mirror/util/log"
+
+ "github.com/Masterminds/semver/v3"
+)
+
+// Filter maps module names to minimal versions of these modules to be pulled
+type Filter map[string]*semver.Version
+
+func NewFilter(filterExpression string) (Filter, error) {
+ filter := make(Filter)
+ if filterExpression == "" {
+ return filter, nil
+ }
+
+ filters := strings.Split(filterExpression, ";")
+ for _, filterExpr := range filters {
+ moduleName, moduleMinVersionString, validSplit := strings.Cut(strings.TrimSpace(filterExpr), "@")
+ if !validSplit {
+ log.WarnF("Malformed filter %q is ignored: invalid filter syntax\n", filterExpr)
+ continue
+ }
+
+ moduleName = strings.TrimSpace(moduleName)
+ if moduleName == "" {
+ return nil, fmt.Errorf("Malformed filter expression %q: empty module name", filterExpr)
+ }
+ if _, moduleRedeclared := filter[moduleName]; moduleRedeclared {
+ return nil, fmt.Errorf("Malformed filter expression: module %s is declared multiple times", moduleName)
+ }
+
+ moduleMinVersion, err := semver.NewVersion(strings.TrimSpace(moduleMinVersionString))
+ if err != nil {
+ return nil, fmt.Errorf("Malformed filter expression %q: %w", filterExpr, err)
+ }
+
+ filter[moduleName] = moduleMinVersion
+ }
+
+ return filter, nil
+}
+
+func (f Filter) MatchesFilter(mod *Module) bool {
+ _, hasMinVersion := f[mod.Name]
+ if !hasMinVersion {
+ return false
+ }
+
+ return true
+}
+
+func (f Filter) FilterReleases(mod *Module) {
+ moduleMinVersion, hasMinVersion := f[mod.Name]
+ if !hasMinVersion {
+ return
+ }
+
+ filteredReleases := make([]string, 0)
+ for _, tag := range mod.Releases {
+ v, err := semver.NewVersion(tag)
+ if err != nil {
+ log.DebugLn("Failed to parse module release tag as semver", tag, err.Error())
+ filteredReleases = append(filteredReleases, tag) // This is probably a release channel, so just leave it
+ continue
+ }
+
+ if moduleMinVersion.GreaterThan(v) {
+ continue
+ }
+
+ filteredReleases = append(filteredReleases, tag)
+ }
+
+ mod.Releases = filteredReleases
+}

internal/mirror/modules/filter_test.go

@@ -0,0 +1,168 @@
+/*
+Copyright 2024 Flant JSC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package modules
+
+import (
+ "testing"
+
+ "github.com/Masterminds/semver/v3"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func TestParseFilterString(t *testing.T) {
+ type args struct {
+ str string
+ }
+ tests := []struct {
+ name string
+ args args
+ want Filter
+ wantErr assert.ErrorAssertionFunc
+ }{
+ {
+ name: "Empty filter expression",
+ args: args{str: ""},
+ want: Filter{},
+ wantErr: assert.NoError,
+ },
+ {
+ name: "One filter expression",
+ args: args{str: "[email protected]"},
+ want: Filter{"moduleName": semver.MustParse("v12.34.56")},
+ wantErr: assert.NoError,
+ },
+ {
+ name: "Multiple filter expression for one module",
+ args: args{str: "[email protected];[email protected];"},
+ want: nil,
+ wantErr: func(t assert.TestingT, err error, i ...interface{}) bool {
+ return assert.ErrorContains(t, err, "declared multiple times")
+ },
+ },
+ {
+ name: "Multiple filter expression for different modules",
+ args: args{str: "[email protected];[email protected];"},
+ want: Filter{"module1": semver.MustParse("v12.34.56"), "module2": semver.MustParse("v0.0.1")},
+ wantErr: assert.NoError,
+ },
+ {
+ name: "Multiple filter expression for different modules with bad spacing and sloppy formatting",
+ args: args{str: " ; module1 @1.1.1;module2 @ v2.3.2; "},
+ want: Filter{"module1": semver.MustParse("v1.1.1"), "module2": semver.MustParse("v2.3.2")},
+ wantErr: assert.NoError,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got, err := NewFilter(tt.args.str)
+ tt.wantErr(t, err)
+
+ require.Len(t, got, len(tt.want))
+
+ for moduleName, minVersion := range tt.want {
+ require.Contains(t, got, moduleName)
+ require.Condition(t, func() bool {
+ return minVersion.Equal(got[moduleName])
+ })
+ }
+ })
+ }
+}
+
+func TestFilter_MatchesFilter(t *testing.T) {
+ type args struct {
+ mod *Module
+ }
+ tests := []struct {
+ name string
+ f Filter
+ args args
+ want bool
+ }{
+ {
+ name: "empty filter",
+ f: Filter{},
+ args: args{
+ mod: &Module{Name: "module1"},
+ },
+ want: false,
+ },
+ {
+ name: "match",
+ f: Filter{
+ "module1": semver.MustParse("v12.34.56"),
+ "module2": semver.MustParse("v0.0.1"),
+ },
+ args: args{
+ mod: &Module{Name: "module1"},
+ },
+ want: true,
+ },
+ {
+ name: "no match",
+ f: Filter{
+ "module1": semver.MustParse("v12.34.56"),
+ "module2": semver.MustParse("v0.0.1"),
+ },
+ args: args{
+ mod: &Module{Name: "module3"},
+ },
+ want: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ assert.Equalf(t, tt.want, tt.f.MatchesFilter(tt.args.mod), "MatchesFilter(%v)", tt.args.mod)
+ })
+ }
+}
+
+func TestFilter_FilterReleases(t *testing.T) {
+ tests := []struct {
+ name string
+ filter Filter
+ mod *Module
+ want []string
+ }{
+ {
+ name: "happy path",
+ filter: Filter{"module1": semver.MustParse("v1.3.0"), "module2": semver.MustParse("2.1.47")},
+ mod: &Module{
+ Name: "module1",
+ Releases: []string{"alpha", "beta", "early-access", "stable", "rock-solid", "v1.0.0", "v1.1.0", "v1.2.0", "v1.3.0", "v1.4.1"},
+ },
+ want: []string{"alpha", "beta", "early-access", "stable", "rock-solid", "v1.3.0", "v1.4.1"},
+ },
+ {
+ name: "module not in filter",
+ filter: Filter{"module1": semver.MustParse("v1.3.0"), "module2": semver.MustParse("2.1.47")},
+ mod: &Module{
+ Name: "module",
+ Releases: []string{"alpha", "beta", "early-access", "stable", "rock-solid", "v1.0.0", "v1.1.0", "v1.2.0", "v1.3.0", "v1.4.1"},
+ },
+ want: []string{"alpha", "beta", "early-access", "stable", "rock-solid", "v1.0.0", "v1.1.0", "v1.2.0", "v1.3.0", "v1.4.1"},
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ tt.filter.FilterReleases(tt.mod)
+ require.ElementsMatch(t, tt.want, tt.mod.Releases)
+ require.Len(t, tt.mod.Releases, len(tt.want))
+ })
+ }
+}