From ad9be94130b3c6b06b4827ad5b062f966dcb8dae Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Thu, 13 Jul 2023 11:41:58 +0200 Subject: [PATCH] introduce ResourceResolver to accept remote resources Signed-off-by: Nicolas De Loof --- cli/options.go | 28 ++++- loader/include.go | 17 ++- loader/loader.go | 52 ++++++-- loader/loader_test.go | 117 ++++++++++++++++++ loader/testdata/remote/compose.yaml | 7 ++ .../testdata/remote/cycle/compose-cycle.yaml | 5 + loader/testdata/remote/cycle/compose.yaml | 5 + loader/testdata/remote/env | 1 + .../remote/nested/compose-nested.yaml | 3 + loader/testdata/remote/nested/compose.yaml | 5 + 10 files changed, 223 insertions(+), 17 deletions(-) create mode 100644 loader/testdata/remote/compose.yaml create mode 100644 loader/testdata/remote/cycle/compose-cycle.yaml create mode 100644 loader/testdata/remote/cycle/compose.yaml create mode 100644 loader/testdata/remote/env create mode 100644 loader/testdata/remote/nested/compose-nested.yaml create mode 100644 loader/testdata/remote/nested/compose.yaml diff --git a/cli/options.go b/cli/options.go index e6d49cdf..f20803b0 100644 --- a/cli/options.go +++ b/cli/options.go @@ -17,6 +17,7 @@ package cli import ( + "context" "io" "os" "path/filepath" @@ -35,6 +36,8 @@ import ( // ProjectOptions provides common configuration for loading a project. type ProjectOptions struct { + ctx context.Context + // Name is a valid Compose project name to be used or empty. // // If empty, the project loader will automatically infer a reasonable @@ -301,6 +304,24 @@ func WithResolvedPaths(resolve bool) ProjectOptionsFn { } } +// WithContext sets the context used to load model and resources +func WithContext(ctx context.Context) ProjectOptionsFn { + return func(o *ProjectOptions) error { + o.ctx = ctx + return nil + } +} + +// WithResourceLoader register support for ResourceLoader to manage remote resources +func WithResourceLoader(r loader.ResourceLoader) ProjectOptionsFn { + return func(o *ProjectOptions) error { + o.loadOptions = append(o.loadOptions, func(options *loader.Options) { + options.ResourceLoaders = append(options.ResourceLoaders, r) + }) + return nil + } +} + // DefaultFileNames defines the Compose file names for auto-discovery (in order of preference) var DefaultFileNames = []string{"compose.yaml", "compose.yml", "docker-compose.yml", "docker-compose.yaml"} @@ -367,7 +388,12 @@ func ProjectFromOptions(options *ProjectOptions) (*types.Project, error) { withNamePrecedenceLoad(absWorkingDir, options), withConvertWindowsPaths(options)) - project, err := loader.Load(types.ConfigDetails{ + ctx := options.ctx + if ctx == nil { + ctx = context.Background() + } + + project, err := loader.LoadWithContext(ctx, types.ConfigDetails{ ConfigFiles: configs, WorkingDir: workingDir, Environment: options.Environment, diff --git a/loader/include.go b/loader/include.go index eeddf8b1..5bc86b74 100644 --- a/loader/include.go +++ b/loader/include.go @@ -17,6 +17,7 @@ package loader import ( + "context" "fmt" "path/filepath" @@ -43,12 +44,20 @@ var transformIncludeConfig TransformerFunc = func(data interface{}) (interface{} } } -func loadInclude(configDetails types.ConfigDetails, model *types.Config, options *Options, loaded []string) (*types.Config, error) { +func loadInclude(ctx context.Context, configDetails types.ConfigDetails, model *types.Config, options *Options, loaded []string) (*types.Config, error) { for _, r := range model.Include { for i, p := range r.Path { - if !filepath.IsAbs(p) { - r.Path[i] = filepath.Join(configDetails.WorkingDir, p) + for _, loader := range options.ResourceLoaders { + if loader.Accept(p) { + path, err := loader.Load(ctx, p) + if err != nil { + return nil, err + } + p = path + break + } } + r.Path[i] = absPath(configDetails.WorkingDir, p) } if r.ProjectDirectory == "" { r.ProjectDirectory = filepath.Dir(r.Path[0]) @@ -65,7 +74,7 @@ func loadInclude(configDetails types.ConfigDetails, model *types.Config, options return nil, err } - imported, err := load(types.ConfigDetails{ + imported, err := load(ctx, types.ConfigDetails{ WorkingDir: r.ProjectDirectory, ConfigFiles: types.ToConfigFiles(r.Path), Environment: env, diff --git a/loader/loader.go b/loader/loader.go index 82283469..b6f8fe90 100644 --- a/loader/loader.go +++ b/loader/loader.go @@ -17,6 +17,7 @@ package loader import ( + "context" "fmt" "os" paths "path" @@ -68,6 +69,16 @@ type Options struct { projectNameImperativelySet bool // Profiles set profiles to enable Profiles []string + // ResourceLoaders manages support for remote resources + ResourceLoaders []ResourceLoader +} + +// ResourceLoader is a plugable remote resource resolver +type ResourceLoader interface { + // Accept returns `true` is the resource reference matches ResourceLoader supported protocol(s) + Accept(path string) bool + // Load returns the path to a local copy of remote resource identified by `path`. + Load(ctx context.Context, path string) (string, error) } func (o *Options) clone() *Options { @@ -85,6 +96,7 @@ func (o *Options) clone() *Options { projectName: o.projectName, projectNameImperativelySet: o.projectNameImperativelySet, Profiles: o.Profiles, + ResourceLoaders: o.ResourceLoaders, } } @@ -193,8 +205,14 @@ func parseYAML(source []byte) (map[string]interface{}, PostProcessor, error) { return converted.(map[string]interface{}), &processor, nil } -// Load reads a ConfigDetails and returns a fully loaded configuration +// Load reads a ConfigDetails and returns a fully loaded configuration. +// Deprecated: use LoadWithContext. func Load(configDetails types.ConfigDetails, options ...func(*Options)) (*types.Project, error) { + return LoadWithContext(context.Background(), configDetails, options...) +} + +// LoadWithContext reads a ConfigDetails and returns a fully loaded configuration +func LoadWithContext(ctx context.Context, configDetails types.ConfigDetails, options ...func(*Options)) (*types.Project, error) { if len(configDetails.ConfigFiles) < 1 { return nil, errors.Errorf("No files specified") } @@ -217,10 +235,10 @@ func Load(configDetails types.ConfigDetails, options ...func(*Options)) (*types. return nil, err } opts.projectName = projectName - return load(configDetails, opts, nil) + return load(ctx, configDetails, opts, nil) } -func load(configDetails types.ConfigDetails, opts *Options, loaded []string) (*types.Project, error) { +func load(ctx context.Context, configDetails types.ConfigDetails, opts *Options, loaded []string) (*types.Project, error) { var model *types.Config mainFile := configDetails.ConfigFiles[0].Filename @@ -261,13 +279,13 @@ func load(configDetails types.ConfigDetails, opts *Options, loaded []string) (*t configDict = groupXFieldsIntoExtensions(configDict) - cfg, err := loadSections(file.Filename, configDict, configDetails, opts) + cfg, err := loadSections(ctx, file.Filename, configDict, configDetails, opts) if err != nil { return nil, err } if !opts.SkipInclude { - cfg, err = loadInclude(configDetails, cfg, opts, loaded) + cfg, err = loadInclude(ctx, configDetails, cfg, opts, loaded) if err != nil { return nil, err } @@ -453,7 +471,7 @@ func groupXFieldsIntoExtensions(dict map[string]interface{}) map[string]interfac return dict } -func loadSections(filename string, config map[string]interface{}, configDetails types.ConfigDetails, opts *Options) (*types.Config, error) { +func loadSections(ctx context.Context, filename string, config map[string]interface{}, configDetails types.ConfigDetails, opts *Options) (*types.Config, error) { var err error cfg := types.Config{ Filename: filename, @@ -466,7 +484,7 @@ func loadSections(filename string, config map[string]interface{}, configDetails } } cfg.Name = name - cfg.Services, err = LoadServices(filename, getSection(config, "services"), configDetails.WorkingDir, configDetails.LookupEnv, opts) + cfg.Services, err = LoadServices(ctx, filename, getSection(config, "services"), configDetails.WorkingDir, configDetails.LookupEnv, opts) if err != nil { return nil, err } @@ -659,7 +677,7 @@ func formatInvalidKeyError(keyPrefix string, key interface{}) error { // LoadServices produces a ServiceConfig map from a compose file Dict // the servicesDict is not validated if directly used. Use Load() to enable validation -func LoadServices(filename string, servicesDict map[string]interface{}, workingDir string, lookupEnv template.Mapping, opts *Options) ([]types.ServiceConfig, error) { +func LoadServices(ctx context.Context, filename string, servicesDict map[string]interface{}, workingDir string, lookupEnv template.Mapping, opts *Options) ([]types.ServiceConfig, error) { var services []types.ServiceConfig x, ok := servicesDict[extensions] @@ -672,7 +690,7 @@ func LoadServices(filename string, servicesDict map[string]interface{}, workingD } for name := range servicesDict { - serviceConfig, err := loadServiceWithExtends(filename, name, servicesDict, workingDir, lookupEnv, opts, &cycleTracker{}) + serviceConfig, err := loadServiceWithExtends(ctx, filename, name, servicesDict, workingDir, lookupEnv, opts, &cycleTracker{}) if err != nil { return nil, err } @@ -683,7 +701,7 @@ func LoadServices(filename string, servicesDict map[string]interface{}, workingD return services, nil } -func loadServiceWithExtends(filename, name string, servicesDict map[string]interface{}, workingDir string, lookupEnv template.Mapping, opts *Options, ct *cycleTracker) (*types.ServiceConfig, error) { +func loadServiceWithExtends(ctx context.Context, filename, name string, servicesDict map[string]interface{}, workingDir string, lookupEnv template.Mapping, opts *Options, ct *cycleTracker) (*types.ServiceConfig, error) { if err := ct.Add(filename, name); err != nil { return nil, err } @@ -707,11 +725,21 @@ func loadServiceWithExtends(filename, name string, servicesDict map[string]inter var baseService *types.ServiceConfig file := serviceConfig.Extends.File if file == "" { - baseService, err = loadServiceWithExtends(filename, baseServiceName, servicesDict, workingDir, lookupEnv, opts, ct) + baseService, err = loadServiceWithExtends(ctx, filename, baseServiceName, servicesDict, workingDir, lookupEnv, opts, ct) if err != nil { return nil, err } } else { + for _, loader := range opts.ResourceLoaders { + if loader.Accept(file) { + path, err := loader.Load(ctx, file) + if err != nil { + return nil, err + } + file = path + break + } + } // Resolve the path to the imported file, and load it. baseFilePath := absPath(workingDir, file) @@ -726,7 +754,7 @@ func loadServiceWithExtends(filename, name string, servicesDict map[string]inter } baseFileServices := getSection(baseFile, "services") - baseService, err = loadServiceWithExtends(baseFilePath, baseServiceName, baseFileServices, filepath.Dir(baseFilePath), lookupEnv, opts, ct) + baseService, err = loadServiceWithExtends(ctx, baseFilePath, baseServiceName, baseFileServices, filepath.Dir(baseFilePath), lookupEnv, opts, ct) if err != nil { return nil, err } diff --git a/loader/loader_test.go b/loader/loader_test.go index 234378ca..697ef8d1 100644 --- a/loader/loader_test.go +++ b/loader/loader_test.go @@ -18,6 +18,7 @@ package loader import ( "bytes" + "context" "fmt" "os" "path/filepath" @@ -2592,3 +2593,119 @@ services: }, }) } + +type customLoader struct { + prefix string +} + +func (c customLoader) Accept(s string) bool { + return strings.HasPrefix(s, c.prefix+":") +} + +func (c customLoader) Load(ctx context.Context, s string) (string, error) { + path := filepath.Join("testdata", c.prefix, s[len(c.prefix)+1:]) + _, err := os.Stat(path) + if err != nil { + return "", err + } + return filepath.Abs(path) +} + +func TestLoadWithRemoteResources(t *testing.T) { + config := buildConfigDetails(` +name: test-remote-resources +services: + foo: + extends: + file: remote:compose.yaml + service: foo + +`, nil) + p, err := LoadWithContext(context.Background(), config, func(options *Options) { + options.SkipConsistencyCheck = true + options.SkipNormalization = true + options.ResolvePaths = true + options.ResourceLoaders = []ResourceLoader{ + customLoader{prefix: "remote"}, + } + }) + assert.NilError(t, err) + assert.DeepEqual(t, p.Services, types.Services{ + { + Name: "foo", + Image: "foo", + Environment: types.MappingWithEquals{"FOO": strPtr("BAR")}, + EnvFile: types.StringList{ + filepath.Join(config.WorkingDir, "testdata", "remote", "env"), + }, + Scale: 1, + Volumes: []types.ServiceVolumeConfig{ + { + Type: types.VolumeTypeBind, + Source: filepath.Join(config.WorkingDir, "testdata", "remote"), + Target: "/foo", + Bind: &types.ServiceVolumeBind{CreateHostPath: true}, + }, + }, + }, + }) +} + +func TestLoadWithMissingResources(t *testing.T) { + config := buildConfigDetails(` +name: test-missing-resources +services: + foo: + extends: + file: remote:unavailable.yaml + service: foo + +`, nil) + _, err := LoadWithContext(context.Background(), config, func(options *Options) { + options.SkipConsistencyCheck = true + options.SkipNormalization = true + options.ResolvePaths = true + options.ResourceLoaders = []ResourceLoader{ + customLoader{prefix: "remote"}, + } + }) + assert.Check(t, os.IsNotExist(err)) +} + +func TestLoadWithNestedResources(t *testing.T) { + config := buildConfigDetails(` +name: test-nested-resources +include: + - remote:nested/compose.yaml +`, nil) + _, err := LoadWithContext(context.Background(), config, func(options *Options) { + options.SkipConsistencyCheck = true + options.SkipNormalization = true + options.ResolvePaths = true + options.ResourceLoaders = []ResourceLoader{ + customLoader{prefix: "remote"}, + } + }) + assert.NilError(t, err) +} + +func TestLoadWithResourcesCycle(t *testing.T) { + config := buildConfigDetails(` +name: test-resources-cycle +services: + foo: + extends: + file: remote:cycle/compose.yaml + service: foo + +`, nil) + _, err := LoadWithContext(context.Background(), config, func(options *Options) { + options.SkipConsistencyCheck = true + options.SkipNormalization = true + options.ResolvePaths = true + options.ResourceLoaders = []ResourceLoader{ + customLoader{prefix: "remote"}, + } + }) + assert.ErrorContains(t, err, "Circular reference") +} diff --git a/loader/testdata/remote/compose.yaml b/loader/testdata/remote/compose.yaml new file mode 100644 index 00000000..6d51ebc2 --- /dev/null +++ b/loader/testdata/remote/compose.yaml @@ -0,0 +1,7 @@ +services: + foo: + image: foo + env_file: + - ./env + volumes: + - .:/foo diff --git a/loader/testdata/remote/cycle/compose-cycle.yaml b/loader/testdata/remote/cycle/compose-cycle.yaml new file mode 100644 index 00000000..93f2c535 --- /dev/null +++ b/loader/testdata/remote/cycle/compose-cycle.yaml @@ -0,0 +1,5 @@ +services: + bar: + extends: + file: remote:cycle/compose.yaml + service: foo diff --git a/loader/testdata/remote/cycle/compose.yaml b/loader/testdata/remote/cycle/compose.yaml new file mode 100644 index 00000000..10e2f77d --- /dev/null +++ b/loader/testdata/remote/cycle/compose.yaml @@ -0,0 +1,5 @@ +services: + foo: + extends: + file: remote:cycle/compose-cycle.yaml + service: bar diff --git a/loader/testdata/remote/env b/loader/testdata/remote/env new file mode 100644 index 00000000..6ac867af --- /dev/null +++ b/loader/testdata/remote/env @@ -0,0 +1 @@ +FOO=BAR diff --git a/loader/testdata/remote/nested/compose-nested.yaml b/loader/testdata/remote/nested/compose-nested.yaml new file mode 100644 index 00000000..2b50bfd4 --- /dev/null +++ b/loader/testdata/remote/nested/compose-nested.yaml @@ -0,0 +1,3 @@ +services: + bar: + image: bar diff --git a/loader/testdata/remote/nested/compose.yaml b/loader/testdata/remote/nested/compose.yaml new file mode 100644 index 00000000..ef741154 --- /dev/null +++ b/loader/testdata/remote/nested/compose.yaml @@ -0,0 +1,5 @@ +services: + foo: + extends: + file: remote:nested/compose-nested.yaml + service: bar