From 229eea581a3c7888606bb15c9e22f79ebc7738e2 Mon Sep 17 00:00:00 2001 From: Maxim Vasilenko Date: Fri, 19 Jul 2024 17:53:30 +0300 Subject: [PATCH] [mirror] Treat versions in filter expression as minimal releases to download instead of specific Signed-off-by: Maxim Vasilenko --- internal/mirror/cmd/modules/pull/flags.go | 2 +- internal/mirror/cmd/modules/pull/pull.go | 33 ++-- .../mirror/cmd/modules/pull/validation.go | 2 +- internal/mirror/modules/filter.go | 96 ++++++++++ internal/mirror/modules/filter_test.go | 168 ++++++++++++++++++ internal/mirror/modules/modules.go | 40 +++-- .../mirror/util/modfilter/module_filter.go | 74 -------- .../util/modfilter/module_filter_test.go | 112 ------------ 8 files changed, 306 insertions(+), 221 deletions(-) create mode 100644 internal/mirror/modules/filter.go create mode 100644 internal/mirror/modules/filter_test.go delete mode 100644 internal/mirror/util/modfilter/module_filter.go delete mode 100644 internal/mirror/util/modfilter/module_filter_test.go diff --git a/internal/mirror/cmd/modules/pull/flags.go b/internal/mirror/cmd/modules/pull/flags.go index 7c90c0d..b299a42 100644 --- a/internal/mirror/cmd/modules/pull/flags.go +++ b/internal/mirror/cmd/modules/pull/flags.go @@ -40,7 +40,7 @@ func addFlags(flagSet *pflag.FlagSet) { "filter", "f", "", - "Filter which modules to pull. Format is \"moduleName:v1.2.3\" or \"moduleName:release-channel\", separated by ';'.", + "Filter which modules starting with which version to pull. Format is \"moduleName@v1.2.3\" separated by ';' where version after @ is the earliest pulled version of the module.", ) flagSet.BoolVar( &SkipTLSVerify, diff --git a/internal/mirror/cmd/modules/pull/pull.go b/internal/mirror/cmd/modules/pull/pull.go index 460569e..b5d8479 100644 --- a/internal/mirror/cmd/modules/pull/pull.go +++ b/internal/mirror/cmd/modules/pull/pull.go @@ -36,7 +36,6 @@ import ( "github.com/deckhouse/deckhouse-cli/internal/mirror/layouts" "github.com/deckhouse/deckhouse-cli/internal/mirror/modules" "github.com/deckhouse/deckhouse-cli/internal/mirror/util/log" - "github.com/deckhouse/deckhouse-cli/internal/mirror/util/modfilter" ) var pullLong = templates.LongDesc(` @@ -104,21 +103,30 @@ func pullExternalModulesToLocalFS( if err != nil { return fmt.Errorf("Get external modules from %q: %w", src.Spec.Registry.Repo, err) } - if len(modulesFromRepo) == 0 { log.WarnLn("No modules found in ModuleSource") return nil } - tagsResolver := layouts.NewTagsResolver() - filter := modfilter.ParseModuleFilterString(moduleFilterExpression) - for i, module := range modulesFromRepo { - if !filter.Match(module) { - continue - } + modulesFilter, err := modules.NewFilter(moduleFilterExpression) + if err != nil { + return fmt.Errorf("Bad modules filter: %w", err) + } + filteredModules := make([]modules.Module, 0) + if len(modulesFilter) > 0 { + for _, moduleData := range modulesFromRepo { + if !modulesFilter.MatchesFilter(&moduleData) { + continue + } - filter.FilterModuleReleases(&module) + modulesFilter.FilterReleases(&moduleData) + filteredModules = append(filteredModules, moduleData) + } + modulesFromRepo = filteredModules + } + tagsResolver := layouts.NewTagsResolver() + for i, module := range modulesFromRepo { log.InfoF("[%d / %d] Pulling module %s...\n", i+1, len(modulesFromRepo), module.RegistryPath) moduleLayout, err := layouts.CreateEmptyImageLayoutAtPath(filepath.Join(mirrorDirectoryPath, module.Name)) @@ -130,7 +138,7 @@ func pullExternalModulesToLocalFS( return fmt.Errorf("Create module OCI Layouts: %w", err) } - moduleImageSet, releasesImageSet, err := modules.FindExternalModuleImages(&module, authProvider, filter != nil, insecure, skipVerifyTLS) + moduleImageSet, releasesImageSet, err := modules.FindExternalModuleImages(&module, modulesFilter, authProvider, insecure, skipVerifyTLS) if err != nil { return fmt.Errorf("Find external module images`: %w", err) } @@ -147,11 +155,6 @@ func pullExternalModulesToLocalFS( SkipTLSVerification: skipVerifyTLS, RegistryAuth: authProvider, }, - DoGOSTDigests: false, - SkipModulesPull: false, - BundleChunkSize: 0, - MinVersion: nil, - SpecificVersion: nil, } log.InfoLn("Beginning to pull module contents") diff --git a/internal/mirror/cmd/modules/pull/validation.go b/internal/mirror/cmd/modules/pull/validation.go index b9d0402..0ab4664 100644 --- a/internal/mirror/cmd/modules/pull/validation.go +++ b/internal/mirror/cmd/modules/pull/validation.go @@ -36,7 +36,7 @@ func validateModuleFilterFormat() error { return nil } - if !regexp.MustCompile(`([a-zA-Z0-9-_]+:(v\d+\.\d+\.\d+|[a-zA-Z0-9_\-]+));?`).MatchString(ModulesFilter) { + if !regexp.MustCompile(`([a-zA-Z0-9-_]+@(v?\d+\.\d+\.\d+));?`).MatchString(ModulesFilter) { return errors.New("Invalid filter pattern") } diff --git a/internal/mirror/modules/filter.go b/internal/mirror/modules/filter.go new file mode 100644 index 0000000..3c5a1b0 --- /dev/null +++ b/internal/mirror/modules/filter.go @@ -0,0 +1,96 @@ +/* +Copyright 2024 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package modules + +import ( + "fmt" + "strings" + + "github.com/deckhouse/deckhouse-cli/internal/mirror/util/log" + + "github.com/Masterminds/semver/v3" +) + +// Filter maps module names to minimal versions of these modules to be pulled +type Filter map[string]*semver.Version + +func NewFilter(filterExpression string) (Filter, error) { + filter := make(Filter) + if filterExpression == "" { + return filter, nil + } + + filters := strings.Split(filterExpression, ";") + for _, filterExpr := range filters { + moduleName, moduleMinVersionString, validSplit := strings.Cut(strings.TrimSpace(filterExpr), "@") + if !validSplit { + log.WarnF("Malformed filter %q is ignored: invalid filter syntax\n", filterExpr) + continue + } + + moduleName = strings.TrimSpace(moduleName) + if moduleName == "" { + return nil, fmt.Errorf("Malformed filter expression %q: empty module name", filterExpr) + } + if _, moduleRedeclared := filter[moduleName]; moduleRedeclared { + return nil, fmt.Errorf("Malformed filter expression: module %s is declared multiple times", moduleName) + } + + moduleMinVersion, err := semver.NewVersion(strings.TrimSpace(moduleMinVersionString)) + if err != nil { + return nil, fmt.Errorf("Malformed filter expression %q: %w", filterExpr, err) + } + + filter[moduleName] = moduleMinVersion + } + + return filter, nil +} + +func (f Filter) MatchesFilter(mod *Module) bool { + _, hasMinVersion := f[mod.Name] + if !hasMinVersion { + return false + } + + return true +} + +func (f Filter) FilterReleases(mod *Module) { + moduleMinVersion, hasMinVersion := f[mod.Name] + if !hasMinVersion { + return + } + + filteredReleases := make([]string, 0) + for _, tag := range mod.Releases { + v, err := semver.NewVersion(tag) + if err != nil { + log.DebugLn("Failed to parse module release tag as semver", tag, err.Error()) + filteredReleases = append(filteredReleases, tag) // This is probably a release channel, so just leave it + continue + } + + if moduleMinVersion.GreaterThan(v) { + continue + } + + filteredReleases = append(filteredReleases, tag) + } + + mod.Releases = filteredReleases +} diff --git a/internal/mirror/modules/filter_test.go b/internal/mirror/modules/filter_test.go new file mode 100644 index 0000000..0766dc0 --- /dev/null +++ b/internal/mirror/modules/filter_test.go @@ -0,0 +1,168 @@ +/* +Copyright 2024 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package modules + +import ( + "testing" + + "github.com/Masterminds/semver/v3" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestParseFilterString(t *testing.T) { + type args struct { + str string + } + tests := []struct { + name string + args args + want Filter + wantErr assert.ErrorAssertionFunc + }{ + { + name: "Empty filter expression", + args: args{str: ""}, + want: Filter{}, + wantErr: assert.NoError, + }, + { + name: "One filter expression", + args: args{str: "moduleName@v12.34.56"}, + want: Filter{"moduleName": semver.MustParse("v12.34.56")}, + wantErr: assert.NoError, + }, + { + name: "Multiple filter expression for one module", + args: args{str: "moduleName@v12.34.56;moduleName@v0.0.1;"}, + want: nil, + wantErr: func(t assert.TestingT, err error, i ...interface{}) bool { + return assert.ErrorContains(t, err, "declared multiple times") + }, + }, + { + name: "Multiple filter expression for different modules", + args: args{str: "module1@v12.34.56;module2@v0.0.1;"}, + want: Filter{"module1": semver.MustParse("v12.34.56"), "module2": semver.MustParse("v0.0.1")}, + wantErr: assert.NoError, + }, + { + name: "Multiple filter expression for different modules with bad spacing and sloppy formatting", + args: args{str: " ; module1 @1.1.1;module2 @ v2.3.2; "}, + want: Filter{"module1": semver.MustParse("v1.1.1"), "module2": semver.MustParse("v2.3.2")}, + wantErr: assert.NoError, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := NewFilter(tt.args.str) + tt.wantErr(t, err) + + require.Len(t, got, len(tt.want)) + + for moduleName, minVersion := range tt.want { + require.Contains(t, got, moduleName) + require.Condition(t, func() bool { + return minVersion.Equal(got[moduleName]) + }) + } + }) + } +} + +func TestFilter_MatchesFilter(t *testing.T) { + type args struct { + mod *Module + } + tests := []struct { + name string + f Filter + args args + want bool + }{ + { + name: "empty filter", + f: Filter{}, + args: args{ + mod: &Module{Name: "module1"}, + }, + want: false, + }, + { + name: "match", + f: Filter{ + "module1": semver.MustParse("v12.34.56"), + "module2": semver.MustParse("v0.0.1"), + }, + args: args{ + mod: &Module{Name: "module1"}, + }, + want: true, + }, + { + name: "no match", + f: Filter{ + "module1": semver.MustParse("v12.34.56"), + "module2": semver.MustParse("v0.0.1"), + }, + args: args{ + mod: &Module{Name: "module3"}, + }, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equalf(t, tt.want, tt.f.MatchesFilter(tt.args.mod), "MatchesFilter(%v)", tt.args.mod) + }) + } +} + +func TestFilter_FilterReleases(t *testing.T) { + tests := []struct { + name string + filter Filter + mod *Module + want []string + }{ + { + name: "happy path", + filter: Filter{"module1": semver.MustParse("v1.3.0"), "module2": semver.MustParse("2.1.47")}, + mod: &Module{ + Name: "module1", + Releases: []string{"alpha", "beta", "early-access", "stable", "rock-solid", "v1.0.0", "v1.1.0", "v1.2.0", "v1.3.0", "v1.4.1"}, + }, + want: []string{"alpha", "beta", "early-access", "stable", "rock-solid", "v1.3.0", "v1.4.1"}, + }, + { + name: "module not in filter", + filter: Filter{"module1": semver.MustParse("v1.3.0"), "module2": semver.MustParse("2.1.47")}, + mod: &Module{ + Name: "module", + Releases: []string{"alpha", "beta", "early-access", "stable", "rock-solid", "v1.0.0", "v1.1.0", "v1.2.0", "v1.3.0", "v1.4.1"}, + }, + want: []string{"alpha", "beta", "early-access", "stable", "rock-solid", "v1.0.0", "v1.1.0", "v1.2.0", "v1.3.0", "v1.4.1"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.filter.FilterReleases(tt.mod) + require.ElementsMatch(t, tt.want, tt.mod.Releases) + require.Len(t, tt.mod.Releases, len(tt.want)) + }) + } +} diff --git a/internal/mirror/modules/modules.go b/internal/mirror/modules/modules.go index a8ff977..d281657 100644 --- a/internal/mirror/modules/modules.go +++ b/internal/mirror/modules/modules.go @@ -21,6 +21,7 @@ import ( "fmt" "io/fs" + "github.com/Masterminds/semver/v3" "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/v1/remote" @@ -114,34 +115,37 @@ func getModulesForRepo( func FindExternalModuleImages( mod *Module, + filter Filter, authProvider authn.Authenticator, - skipReleaseChannels, insecure, skipVerifyTLS bool, + insecure, skipVerifyTLS bool, ) (moduleImages, releaseImages map[string]struct{}, err error) { nameOpts, remoteOpts := auth.MakeRemoteRegistryRequestOptions(authProvider, insecure, skipVerifyTLS) moduleImages = map[string]struct{}{} releaseImages = map[string]struct{}{} - if skipReleaseChannels { - for _, tag := range mod.Releases { - moduleImages[mod.RegistryPath+":"+tag] = struct{}{} - releaseImages[mod.RegistryPath+"/release:"+tag] = struct{}{} - } - } else { - releaseImages, err = getAvailableReleaseChannelsImagesForModule(mod, nameOpts, remoteOpts) - if err != nil { - return nil, nil, fmt.Errorf("Get available release channels of module: %w", err) - } + releaseImages, err = getAvailableReleaseChannelsImagesForModule(mod, nameOpts, remoteOpts) + if err != nil { + return nil, nil, fmt.Errorf("Get available release channels of module: %w", err) + } - releaseChannelVersions, err := releases.FetchVersionsFromModuleReleaseChannels(releaseImages, authProvider, insecure, skipVerifyTLS) - if err != nil { - return nil, nil, fmt.Errorf("Fetch versions from %q release channels: %w", mod.Name, err) - } - for _, versionTag := range releaseChannelVersions { - moduleImages[mod.RegistryPath+":"+versionTag] = struct{}{} - releaseImages[mod.RegistryPath+"/release:"+versionTag] = struct{}{} + releaseChannelVersions, err := releases.FetchVersionsFromModuleReleaseChannels(releaseImages, authProvider, insecure, skipVerifyTLS) + if err != nil { + return nil, nil, fmt.Errorf("Fetch versions from %q release channels: %w", mod.Name, err) + } + + minVersion, hasMinVersion := filter[mod.Name] + for _, tag := range mod.Releases { + version, err := semver.NewVersion(tag) + if err == nil && hasMinVersion && minVersion.Compare(version) <= 0 { + releaseImages[mod.RegistryPath+"/release:"+tag] = struct{}{} + moduleImages[mod.RegistryPath+":"+tag] = struct{}{} } } + for _, versionTag := range releaseChannelVersions { + moduleImages[mod.RegistryPath+":"+versionTag] = struct{}{} + releaseImages[mod.RegistryPath+"/release:"+versionTag] = struct{}{} + } for _, imageTag := range maps.Keys(moduleImages) { ref, err := name.ParseReference(imageTag, nameOpts...) diff --git a/internal/mirror/util/modfilter/module_filter.go b/internal/mirror/util/modfilter/module_filter.go deleted file mode 100644 index 65b68c6..0000000 --- a/internal/mirror/util/modfilter/module_filter.go +++ /dev/null @@ -1,74 +0,0 @@ -/* -Copyright 2024 Flant JSC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package modfilter - -import ( - "strings" - - "github.com/deckhouse/deckhouse-cli/internal/mirror/modules" - "github.com/deckhouse/deckhouse-cli/internal/mirror/util/log" -) - -// ModuleFilter maps module names to required tags -type ModuleFilter map[string][]string - -func ParseModuleFilterString(str string) ModuleFilter { - filter := make(ModuleFilter) - if str == "" { - return nil - } - - modules := strings.Split(str, ";") - for _, filterEntry := range modules { - moduleName, moduleTag, validSplit := strings.Cut(strings.TrimSpace(filterEntry), ":") - if !validSplit { - log.WarnF("Malformed filter %q is ignored\n", filterEntry) - continue - } - - moduleName = strings.TrimSpace(moduleName) - moduleTag = strings.TrimSpace(moduleTag) - - if filter[moduleName] == nil { - filter[moduleName] = make([]string, 0) - } - - filter[moduleName] = append(filter[moduleName], moduleTag) - } - - return filter -} - -func (f ModuleFilter) Match(mod modules.Module) bool { - if len(f) == 0 { - return true - } - - for _, tag := range f[mod.Name] { - for _, release := range mod.Releases { - if release == tag { - return true - } - } - } - - return false -} - -func (f ModuleFilter) FilterModuleReleases(mod *modules.Module) { - mod.Releases = f[mod.Name] -} diff --git a/internal/mirror/util/modfilter/module_filter_test.go b/internal/mirror/util/modfilter/module_filter_test.go deleted file mode 100644 index 4a97778..0000000 --- a/internal/mirror/util/modfilter/module_filter_test.go +++ /dev/null @@ -1,112 +0,0 @@ -/* -Copyright 2024 Flant JSC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package modfilter - -import ( - "testing" - - "github.com/stretchr/testify/require" - - "github.com/deckhouse/deckhouse-cli/internal/mirror/modules" -) - -func TestModuleFilter_Match(t *testing.T) { - type args struct { - mod modules.Module - } - tests := []struct { - name string - f ModuleFilter - args args - want bool - }{ - { - name: "Empty filter matches anything", - f: map[string][]string{}, - args: args{mod: modules.Module{Name: "test", RegistryPath: "registry.example.com/dh", Releases: []string{"v1.2.3"}}}, - want: true, - }, - { - name: "Happy path, module doesn't match", - f: map[string][]string{"deckhouse-admin": {"v1.2.0", "v1.2.1", "v1.2.2", "v1.2.3"}}, - args: args{mod: modules.Module{Name: "test", RegistryPath: "registry.example.com/dh", Releases: []string{"v1.2.3"}}}, - want: false, - }, - { - name: "Happy path, module matches", - f: map[string][]string{"deckhouse-admin": {"v1.2.0", "v1.2.1", "v1.2.2", "v1.2.3"}}, - args: args{mod: modules.Module{Name: "deckhouse-admin", RegistryPath: "registry.example.com/dh", Releases: []string{"v1.2.3"}}}, - want: true, - }, - { - name: "Happy path, module unknown to filter", - f: map[string][]string{"deckhouse-admin": {"v1.2.0", "v1.2.1", "v1.2.2", "v1.2.3"}}, - args: args{mod: modules.Module{Name: "op-monitoring", RegistryPath: "registry.example.com/dh", Releases: []string{"v1.2.3"}}}, - want: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := tt.f.Match(tt.args.mod); got != tt.want { - t.Errorf("Match() = %v, want %v", got, tt.want) - } - }) - } -} - -func TestParseModuleFilterString(t *testing.T) { - type args struct { - str string - } - tests := []struct { - name string - args args - want ModuleFilter - }{ - { - name: "Empty filter expression", - args: args{str: ""}, - want: nil, - }, - { - name: "One filter expression", - args: args{str: "moduleName:v12.34.56"}, - want: ModuleFilter{"moduleName": {"v12.34.56"}}, - }, - { - name: "Multiple filter expression for one module", - args: args{str: "moduleName:v12.34.56;moduleName:v0.0.1;"}, - want: ModuleFilter{"moduleName": {"v12.34.56", "v0.0.1"}}, - }, - { - name: "Multiple filter expression for different modules", - args: args{str: "module1:v12.34.56;module2:v0.0.1;"}, - want: ModuleFilter{"module1": {"v12.34.56"}, "module2": {"v0.0.1"}}, - }, - { - name: "Multiple filter expression for different modules with many versions and bad spacing", - args: args{str: " ; module1: v12.34.56; module1 :v1.1.1; module2:v0.0.1; module2 : v2.3.2;"}, - want: ModuleFilter{"module1": {"v12.34.56", "v1.1.1"}, "module2": {"v0.0.1", "v2.3.2"}}, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := ParseModuleFilterString(tt.args.str) - require.Equal(t, tt.want, got) - }) - } -}