diff --git a/README.md b/README.md index b5a91c40..db9c1473 100644 --- a/README.md +++ b/README.md @@ -53,20 +53,16 @@ uor-client-go version ### Registry Config This registry config can be stored to individually configure each registry. It should be named `registry-config.yaml`. -The locations this can be stored in are the current working directory and at `$HOME/.uor/registry-config.yaml`. If mirror -endpoints are configured, a mirrored collection will attempt to be pulled before the user-provided reference. +The locations this can be stored in are the current working directory and at `$HOME/.uor/registry-config.yaml`. +As a special case, the prefix field can be missing; if so, it defaults to the value of the location field. Example: ```bash registries: - - prefix: "localhost*" - endpoint: - location: "localhost:5000" - skipTLS: false - plainHTTP: true - mirrors: - - location: "localhost:5001" - plainHTTP: true + - prefix: "localhost:5001/test" + location: localhost:5001 + skipTLS: false + plainHTTP: true ``` diff --git a/cmd/client/commands/options/remote.go b/cmd/client/commands/options/remote.go index 3b20c8f5..2b13b777 100644 --- a/cmd/client/commands/options/remote.go +++ b/cmd/client/commands/options/remote.go @@ -3,7 +3,6 @@ package options import ( "errors" - "github.com/mitchellh/mapstructure" "github.com/spf13/pflag" "github.com/spf13/viper" @@ -37,11 +36,8 @@ func (o *Remote) LoadRegistryConfig() error { } return err } - option := viper.DecoderConfigOption(func(config *mapstructure.DecoderConfig) { - config.TagName = "json" - }) - return viper.Unmarshal(&o.RegistryConfig, option) + return viper.Unmarshal(&o.RegistryConfig) } // RemoteAuth describes remote authentication configuration options that can be set. diff --git a/cmd/client/commands/push.go b/cmd/client/commands/push.go index 74f9a0e2..3f5f9ca9 100644 --- a/cmd/client/commands/push.go +++ b/cmd/client/commands/push.go @@ -76,14 +76,15 @@ func (o *PushOptions) Run(ctx context.Context) error { return err } - // Not adding a registry configuration when pushing since only - // one reference us being handled at a time. - // QUESTION(jpower432): Could this create a problem? I think it would - // be more unexpected to publish collection to declared mirrors. + if err := o.Remote.LoadRegistryConfig(); err != nil { + return err + } + client, err := orasclient.NewClient( orasclient.SkipTLSVerify(o.Insecure), orasclient.WithAuthConfigs(o.Configs), orasclient.WithPlainHTTP(o.PlainHTTP), + orasclient.WithRegistryConfig(o.RegistryConfig), ) if err != nil { diff --git a/cmd/client/commands/serve.go b/cmd/client/commands/serve.go index 1da45051..e3e25af7 100644 --- a/cmd/client/commands/serve.go +++ b/cmd/client/commands/serve.go @@ -93,7 +93,10 @@ func (o *ServeOptions) Run(ctx context.Context) error { PullCache: cache, RegistryConfig: o.RegistryConfig, } - service := collectionmanager.FromManager(manager, opts) + service, err := collectionmanager.FromManager(manager, opts) + if err != nil { + return err + } // Register the service with the gRPC server managerapi.RegisterCollectionManagerServer(rpc, service) diff --git a/registryclient/orasclient/oras.go b/registryclient/orasclient/oras.go index 9530b4ba..379ce387 100644 --- a/registryclient/orasclient/oras.go +++ b/registryclient/orasclient/oras.go @@ -121,12 +121,12 @@ func (c *orasClient) LoadCollection(ctx context.Context, reference string) (coll return value.(collection.Collection), nil } - repo, updatedRef, err := c.setupRepo(ctx, reference) + repo, err := c.setupRepo(ctx, reference) if err != nil { return collection.Collection{}, fmt.Errorf("could not create registry target: %w", err) } - graph, err := loadCollection(ctx, repo, updatedRef) + graph, err := loadCollection(ctx, repo, reference) if err != nil { return collection.Collection{}, err } @@ -140,7 +140,7 @@ func (c *orasClient) Pull(ctx context.Context, reference string, store content.S var allDescs []ocispec.Descriptor var from oras.Target - repo, updatedRef, err := c.setupRepo(ctx, reference) + repo, err := c.setupRepo(ctx, reference) if err != nil { return ocispec.Descriptor{}, allDescs, fmt.Errorf("could not create registry target: %w", err) } @@ -157,7 +157,7 @@ func (c *orasClient) Pull(ctx context.Context, reference string, store content.S if exists { graph = value.(collection.Collection) } else { - graph, err = loadCollection(ctx, repo, updatedRef) + graph, err = loadCollection(ctx, repo, reference) if err != nil { return ocispec.Descriptor{}, allDescs, err } @@ -235,7 +235,7 @@ func (c *orasClient) Pull(ctx context.Context, reference string, store content.S cCopyOpts := c.copyOpts cCopyOpts.FindSuccessors = successorFn - desc, err := oras.Copy(ctx, from, updatedRef, store, updatedRef, cCopyOpts) + desc, err := oras.Copy(ctx, from, reference, store, reference, cCopyOpts) if err != nil { return ocispec.Descriptor{}, allDescs, err } @@ -245,25 +245,25 @@ func (c *orasClient) Pull(ctx context.Context, reference string, store content.S // Push performs a copy of OCI artifacts to a remote location. func (c *orasClient) Push(ctx context.Context, store content.Store, reference string) (ocispec.Descriptor, error) { - repo, updatedRef, err := c.setupRepo(ctx, reference) + repo, err := c.setupRepo(ctx, reference) if err != nil { return ocispec.Descriptor{}, fmt.Errorf("could not create registry target: %w", err) } - return oras.Copy(ctx, store, updatedRef, repo, updatedRef, c.copyOpts) + return oras.Copy(ctx, store, reference, repo, reference, c.copyOpts) } // GetManifest returns the manifest the reference resolves to. func (c *orasClient) GetManifest(ctx context.Context, reference string) (ocispec.Descriptor, io.ReadCloser, error) { - repo, updatedRef, err := c.setupRepo(ctx, reference) + repo, err := c.setupRepo(ctx, reference) if err != nil { return ocispec.Descriptor{}, nil, fmt.Errorf("could not create registry target: %w", err) } - return repo.FetchReference(ctx, updatedRef) + return repo.FetchReference(ctx, reference) } // GetContent retrieves the content for a specified descriptor at a specified reference. func (c *orasClient) GetContent(ctx context.Context, reference string, desc ocispec.Descriptor) ([]byte, error) { - repo, _, err := c.setupRepo(ctx, reference) + repo, err := c.setupRepo(ctx, reference) if err != nil { return nil, fmt.Errorf("could not create registry target: %w", err) } @@ -306,78 +306,29 @@ func (c *orasClient) authClient(insecure bool) *auth.Client { } // setupRepo configures the client to access the remote repository. -func (c *orasClient) setupRepo(ctx context.Context, reference string) (registry.Repository, string, error) { +func (c *orasClient) setupRepo(ctx context.Context, reference string) (registry.Repository, error) { registryConfig, err := registryclient.FindRegistry(c.registryConf, reference) if err != nil { - return nil, reference, err + return nil, err } repo, err := remote.NewRepository(reference) if err != nil { - return nil, reference, fmt.Errorf("could not create registry target: %w", err) + return nil, fmt.Errorf("could not create registry target: %w", err) } - // If the incoming reference does not match any registry prefixes, - // use the client configuration. If there is a match and the registry - // has mirrors configured, try each one before attempting to contact the - // input reference (the default). switch { case registryConfig == nil: repo.PlainHTTP = c.plainHTTP repo.Client = c.authClient(c.insecure) - return repo, reference, nil - case len(registryConfig.Mirrors) != 0: - mirrorRepo, mirrorReference, err := c.pickMirror(ctx, *registryConfig, reference) - if err == nil { - return mirrorRepo, mirrorReference, nil - } - - var merr *registryclient.ErrNoAvailableMirrors - if err != nil && !errors.As(err, &merr) { - return nil, reference, err - } - - fallthrough + return repo, nil default: repo.PlainHTTP = registryConfig.PlainHTTP repo.Client = c.authClient(registryConfig.SkipTLS) - return repo, reference, nil + return repo, nil } } -// pickMirror is used if the reference is linked to a registry in the registry configuration that has mirrors. It returns -// a configured remote.Repository and rewritten reference per the mirror configuration. -func (c *orasClient) pickMirror(ctx context.Context, reg registryclient.Registry, ref string) (registry.Repository, string, error) { - pullSources, err := reg.PullSourceFromReference(ref) - if err != nil { - return nil, ref, err - } - - for _, ps := range pullSources { - mirror := ps.Endpoint - mirrorReg, err := remote.NewRegistry(mirror.Location) - if err != nil { - return nil, ref, fmt.Errorf("could not create registry target: %w", err) - } - mirrorReg.PlainHTTP = mirror.PlainHTTP - mirrorReg.Client = c.authClient(mirror.SkipTLS) - - if err := mirrorReg.Ping(ctx); err == nil { - mirrorReference, err := registry.ParseReference(ps.Reference) - if err != nil { - return nil, ref, fmt.Errorf("reference %q: %w", mirrorReference, err) - } - mirrorRepo, err := mirrorReg.Repository(ctx, mirrorReference.Repository) - if err != nil { - return nil, ref, err - } - return mirrorRepo, mirrorReference.String(), nil - } - } - - return nil, ref, ®istryclient.ErrNoAvailableMirrors{Registry: reg.Location} -} - // loadFiles stores files in a file store and creates descriptors representing each file in the store. func loadFiles(ctx context.Context, store *file.Store, mediaType string, files ...string) ([]ocispec.Descriptor, error) { var descs []ocispec.Descriptor diff --git a/registryclient/orasclient/oras_test.go b/registryclient/orasclient/oras_test.go index e85c336a..d43346a2 100644 --- a/registryclient/orasclient/oras_test.go +++ b/registryclient/orasclient/oras_test.go @@ -259,34 +259,6 @@ func TestPushPull(t *testing.T) { require.NoError(t, c.Destroy()) }) - t.Run("Success/PullWithRegistryConfigMirror", func(t *testing.T) { - expDigest := "sha256:98f36e12e9dbacfbb10b9d1f32a46641eb42de588e54cfd7e8627d950ae8140a" - config := registryclient.RegistryConfig{ - Registries: []registryclient.Registry{ - { - Prefix: "test.server.com", - Endpoint: registryclient.Endpoint{ - PlainHTTP: false, - Location: "test.server.com", - }, - Mirrors: []registryclient.Endpoint{ - { - PlainHTTP: true, - Location: u.Host, - }, - }, - }, - }, - } - c, err := NewClient(WithRegistryConfig(config)) - require.NoError(t, err) - root, descs, err := c.Pull(context.TODO(), "test.server.com/test:latest", memory.New()) - require.NoError(t, err) - require.Equal(t, expDigest, root.Digest.String()) - require.Len(t, descs, 4) - require.NoError(t, c.Destroy()) - }) - t.Run("Success/PullWithRegistryConfigNoMatch", func(t *testing.T) { expDigest := "sha256:98f36e12e9dbacfbb10b9d1f32a46641eb42de588e54cfd7e8627d950ae8140a" config := registryclient.RegistryConfig{ diff --git a/registryclient/registries.go b/registryclient/registries.go index 8ab49483..689ce3a2 100644 --- a/registryclient/registries.go +++ b/registryclient/registries.go @@ -1,11 +1,8 @@ package registryclient import ( - "fmt" "regexp" "strings" - - "oras.land/oras-go/v2/errdef" ) // This configuration is slightly modified and paired down version of the registries.conf. @@ -19,65 +16,26 @@ import ( // Endpoint describes a remote location of a registry. type Endpoint struct { // The endpoint's remote location. - Location string `json:"location"` + Location string `mapstructure:"location" json:"location"` // If true, certs verification will be skipped. - SkipTLS bool `json:"skipTLS"` + SkipTLS bool `mapstructure:"skipTLS" json:"skipTLS"` // If true, the client will use HTTP to // connect to the registry. - PlainHTTP bool `json:"plainHTTP"` -} - -// RewriteReference returns a reference for the endpoint given the original -// reference and registry prefix. -func (e Endpoint) RewriteReference(reference string) (string, error) { - if e.Location == "" { - return reference, nil - } - - parts := strings.SplitN(reference, "/", 2) - if len(parts) == 1 { - return " ", fmt.Errorf("%w: missing repository", errdef.ErrInvalidReference) - } - path := parts[1] - return fmt.Sprintf("%s/%s", e.Location, path), nil -} - -// PullSource is a reference that is associated with a -// specific endpoint. This is used to generate references -// for registry mirrors and correlate them the mirror endpoint -type PullSource struct { - Reference string - Endpoint + PlainHTTP bool `mapstructure:"plainHTTP" json:"plainHTTP"` } // Registry represents a registry. type Registry struct { // Prefix is used for endpoint matching. - Prefix string `json:"prefix"` + Prefix string `mapstructure:"prefix" json:"prefix"` // A registry is an Endpoint too - Endpoint `json:"endpoint"` - // The registry mirrors - Mirrors []Endpoint `json:"mirrors,omitempty"` -} - -// PullSourceFromReference returns all pull source for the registry mirrors from -// a given reference. -func (r *Registry) PullSourceFromReference(ref string) ([]PullSource, error) { - var sources []PullSource - for _, mirror := range r.Mirrors { - rewritten, err := mirror.RewriteReference(ref) - if err != nil { - return nil, err - } - sources = append(sources, PullSource{Endpoint: mirror, Reference: rewritten}) - } - return sources, nil + Endpoint `mapstructure:",squash" json:",inline"` } // RegistryConfig is a configuration to configure multiple // registry endpoints. type RegistryConfig struct { - Registries []Registry `json:"registries"` + Registries []Registry `mapstructure:"registries" json:"registries"` } // FindRegistry returns the registry from the registry config that @@ -87,14 +45,18 @@ func FindRegistry(registryConfig RegistryConfig, ref string) (*Registry, error) prefixLen := 0 for _, r := range registryConfig.Registries { - prefixExp, err := regexp.Compile(validPrefix(r.Prefix)) + match := r.Prefix + if match == "" { + match = r.Location + } + prefixExp, err := regexp.Compile(validPrefix(match)) if err != nil { return nil, err } if prefixExp.MatchString(ref) { - if len(r.Prefix) > prefixLen { + if len(match) > prefixLen { reg = r - prefixLen = len(r.Prefix) + prefixLen = len(match) } } } diff --git a/registryclient/registries_test.go b/registryclient/registries_test.go index 67c90013..9822e274 100644 --- a/registryclient/registries_test.go +++ b/registryclient/registries_test.go @@ -125,103 +125,3 @@ func TestFindRegistry(t *testing.T) { }) } } - -func TestEndpoint_RewriteReference(t *testing.T) { - type spec struct { - name string - expError string - endpoint Endpoint - inRef string - expRef string - } - - cases := []spec{ - { - name: "Success/MatchingPrefix", - endpoint: Endpoint{ - Location: "alt.example.com", - }, - inRef: "reg.example.com/test:latest", - expRef: "alt.example.com/test:latest", - }, - { - name: "Success/EmptyLocation", - endpoint: Endpoint{ - Location: "", - }, - inRef: "reg.example.com/test:latest", - expRef: "reg.example.com/test:latest", - }, - } - for _, c := range cases { - t.Run(c.name, func(t *testing.T) { - ref, err := c.endpoint.RewriteReference(c.inRef) - if c.expError != "" { - require.EqualError(t, err, c.expError) - } else { - require.NoError(t, err) - require.Equal(t, c.expRef, ref) - } - }) - } -} - -func TestRegistry_PullSourceFromReference(t *testing.T) { - type spec struct { - name string - expError string - registry Registry - inRef string - expSources []PullSource - } - cases := []spec{ - { - name: "Success/NoMirrors", - registry: Registry{ - Prefix: "reg.example.com", - Endpoint: Endpoint{ - SkipTLS: false, - }, - }, - inRef: "reg.example.com/test:latest", - }, - { - name: "Success/OneMirror", - registry: Registry{ - Prefix: "reg.example.com", - Endpoint: Endpoint{ - SkipTLS: false, - }, - Mirrors: []Endpoint{ - { - SkipTLS: true, - Location: "alt.registry.com", - }, - }, - }, - inRef: "reg.example.com/test:latest", - expSources: []PullSource{ - { - Reference: "alt.registry.com/test:latest", - Endpoint: Endpoint{ - SkipTLS: true, - PlainHTTP: false, - Location: "alt.registry.com", - }, - }, - }, - }, - } - - for _, c := range cases { - t.Run(c.name, func(t *testing.T) { - sources, err := c.registry.PullSourceFromReference(c.inRef) - if c.expError != "" { - require.EqualError(t, err, c.expError) - } else { - require.NoError(t, err) - require.Equal(t, c.expSources, sources) - } - }) - } -} diff --git a/services/collectionmanager/service.go b/services/collectionmanager/service.go index dcdd3e76..464e3ca3 100644 --- a/services/collectionmanager/service.go +++ b/services/collectionmanager/service.go @@ -2,7 +2,7 @@ package collectionmanager import ( "context" - "fmt" + "os" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -13,6 +13,7 @@ import ( "github.com/uor-framework/uor-client-go/attributes/matchers" "github.com/uor-framework/uor-client-go/config" "github.com/uor-framework/uor-client-go/content" + "github.com/uor-framework/uor-client-go/log" "github.com/uor-framework/uor-client-go/manager" "github.com/uor-framework/uor-client-go/registryclient" "github.com/uor-framework/uor-client-go/registryclient/orasclient" @@ -33,15 +34,23 @@ type ServiceOptions struct { Insecure bool PlainHTTP bool PullCache content.Store + Logger log.Logger RegistryConfig registryclient.RegistryConfig } // FromManager returns a CollectionManager API server from a Manager type. -func FromManager(mg manager.Manager, serviceOptions ServiceOptions) managerapi.CollectionManagerServer { +func FromManager(mg manager.Manager, serviceOptions ServiceOptions) (managerapi.CollectionManagerServer, error) { + if serviceOptions.Logger == nil { + logger, err := log.NewLogger(os.Stderr, "debug") + if err != nil { + return nil, err + } + serviceOptions.Logger = logger + } return &service{ mg: mg, options: serviceOptions, - } + }, nil } // PublishContent publishes collection content to a storage provide based on client input. @@ -59,7 +68,7 @@ func (s *service) PublishContent(ctx context.Context, message *managerapi.Publis } defer func() { if err := client.Destroy(); err != nil { - fmt.Println(err.Error()) + s.options.Logger.Errorf(err.Error()) } }() @@ -131,7 +140,7 @@ func (s *service) RetrieveContent(ctx context.Context, message *managerapi.Retri } defer func() { if err := client.Destroy(); err != nil { - fmt.Println(err.Error()) + s.options.Logger.Errorf(err.Error()) } }() diff --git a/services/collectionmanager/service_test.go b/services/collectionmanager/service_test.go index 4b698c75..f88ff594 100644 --- a/services/collectionmanager/service_test.go +++ b/services/collectionmanager/service_test.go @@ -110,7 +110,8 @@ func TestCollectionManagerServer_All(t *testing.T) { require.NoError(t, err) manager := defaultmanager.New(testContentStore{Store: memory.New()}, testlogr) - srv := FromManager(manager, ServiceOptions{PlainHTTP: true}) + srv, err := FromManager(manager, ServiceOptions{PlainHTTP: true}) + require.NoError(t, err) conn, err := grpc.DialContext(ctx, "", grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithContextDialer(dialer(srv))) require.NoError(t, err) diff --git a/testme/fish.jpg b/testme/fish.jpg new file mode 100644 index 00000000..0085907a Binary files /dev/null and b/testme/fish.jpg differ diff --git a/testme/level1/fish2.jpg b/testme/level1/fish2.jpg new file mode 100644 index 00000000..54567a29 Binary files /dev/null and b/testme/level1/fish2.jpg differ