From 78926364c03acdc070f59b3ac970eda754c31548 Mon Sep 17 00:00:00 2001 From: Aaron Birkland Date: Tue, 23 Apr 2019 11:53:44 -0400 Subject: [PATCH 1/5] Add pass client --- web/client.go | 80 +++++++++++++++++++++++++ web/client_test.go | 145 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 225 insertions(+) create mode 100644 web/client.go create mode 100644 web/client_test.go diff --git a/web/client.go b/web/client.go new file mode 100644 index 0000000..c71a5d7 --- /dev/null +++ b/web/client.go @@ -0,0 +1,80 @@ +package web + +import ( + "encoding/json" + "fmt" + "net/http" + "strings" + + "github.com/pkg/errors" +) + +const ( + headerUserAgent = "User-Agent" + headerAccept = "Accept" +) + +const ( + mediaJSONTypes = "application/json, application/ld+json" +) + +// InternalPassClient uses "private" backend URIs for interacting with the PASS repository +// It is intended for use on private networks. Public URIs will be +// converted to private URIs when accessing the repository. +type InternalPassClient struct { + Requester + ExternalBaseURI string + InternalBaseURI string + Credentials *Credentials +} + +type Credentials struct { + Username string + Password string +} + +// Requester performs http requests +type Requester interface { + Do(req *http.Request) (*http.Response, error) +} + +// FetchEntity fetches and parses the PASS entity at the given URL to the struct or map +// pointed to by entityPointer +func (c *InternalPassClient) FetchEntity(url string, entityPointer interface{}) error { + url, err := c.translate(url) + if err != nil { + return errors.Wrapf(err, "error translating url") + } + + request, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return errors.Wrapf(err, "could not build http request to %s", url) + } + + if c.Credentials != nil { + request.SetBasicAuth(c.Credentials.Username, c.Credentials.Password) + } + request.Header.Set(headerUserAgent, "pass-policy-service") + request.Header.Set(headerAccept, mediaJSONTypes) + + resp, err := c.Do(request) + if err != nil { + return errors.Wrapf(err, "error connecting to %s", url) + } + defer resp.Body.Close() + + err = json.NewDecoder(resp.Body).Decode(entityPointer) + if err != nil { + return errors.Wrapf(err, "could not decode resource JSON") + } + + return nil +} + +func (c *InternalPassClient) translate(uri string) (string, error) { + if !strings.HasPrefix(uri, c.ExternalBaseURI) && + !strings.HasPrefix(uri, c.InternalBaseURI) { + return uri, fmt.Errorf(`uri "%s" must start with internal or external baseuri"`, uri) + } + return strings.Replace(uri, c.ExternalBaseURI, c.InternalBaseURI, 1), nil +} diff --git a/web/client_test.go b/web/client_test.go new file mode 100644 index 0000000..33b3403 --- /dev/null +++ b/web/client_test.go @@ -0,0 +1,145 @@ +package web_test + +import ( + "fmt" + "io" + "net/http" + "strings" + "testing" + + "github.com/go-test/deep" + "github.com/oa-pass/pass-policy-service/web" +) + +type fakeRequester struct { + f func(req *http.Request) (*http.Response, error) +} + +func (r *fakeRequester) Do(req *http.Request) (*http.Response, error) { + if r.f != nil { + return r.f(req) + } + + return nil, nil +} + +type fakeBody struct { + io.Reader + closeFunc func() error +} + +func (b *fakeBody) Close() error { + if b.closeFunc != nil { + return b.closeFunc() + } + return nil +} + +func TestPrivateFetchEntity(t *testing.T) { + publicBaseURI := "https://example.org/foo/fcrepo/rest" + privateBaseURI := "http://127.0.0.1:8080/rest" + + privateResource := privateBaseURI + "/foo/bar" + publicResource := publicBaseURI + "/foo/bar" + + username := "foo" + password := "bar" + + client := web.InternalPassClient{ + Requester: &fakeRequester{ + f: func(req *http.Request) (*http.Response, error) { + + if req.URL.String() != privateResource { + t.Fatalf("private resource URI is incorrect") + } + + user, pass, ok := req.BasicAuth() + if !ok || user != username || pass != password { + t.Fatalf("basic auth is wrong") + } + + return &http.Response{ + Body: &fakeBody{ + Reader: strings.NewReader(`{ + "foo" : [ + "bar", + "baz" + ] + }`), + }, + }, nil + }, + }, + InternalBaseURI: privateBaseURI, + ExternalBaseURI: publicBaseURI, + Credentials: &web.Credentials{ + Username: username, + Password: password, + }, + } + + ref := make(map[string]interface{}) + err := client.FetchEntity(publicResource, &ref) + if err != nil { + t.Fatalf("Client fetch resulted in error %+v", err) + } + + diffs := deep.Equal(ref["foo"].([]interface{}), []interface{}{"bar", "baz"}) + if len(diffs) > 0 { + t.Fatalf("found difference in deserialized content %s", diffs) + } + + err = client.FetchEntity("http://example.org/bad/resource", &ref) + if err == nil { + t.Fatalf("Should have thrown error on non whitelisted uri") + } +} + +func TestPrivateFetchEntityErrors(t *testing.T) { + cases := []struct { + name string + url string + f func(req *http.Request) (*http.Response, error) + }{ + { + name: "badURI", + url: "0http://bad", + }, + { + name: "httpError", + url: "http://example.org/foo", + f: func(req *http.Request) (*http.Response, error) { + return nil, fmt.Errorf("this is an error") + }, + }, + { + name: "badJSON", + url: "http://example.org/foo", + f: func(req *http.Request) (*http.Response, error) { + return &http.Response{ + Body: &fakeBody{ + Reader: strings.NewReader(`{BAD JSON-,`), + }, + }, nil + }, + }, + } + + for _, c := range cases { + c := c + t.Run(c.name, func(t *testing.T) { + client := web.InternalPassClient{ + Requester: &fakeRequester{ + f: c.f, + }, + } + + ref := make(map[string]interface{}) + + err := client.FetchEntity(c.url, &ref) + if err == nil { + t.Fatalf("Should have terminated with an error") + } + }) + } +} From 977b84cb892e2e9a279060ea9e1216dbbc1624ec Mon Sep 17 00:00:00 2001 From: Aaron Birkland Date: Tue, 23 Apr 2019 15:59:22 -0400 Subject: [PATCH 2/5] Add policy endpoint impl --- web/policy_endpoint.go | 89 ++++++++++++++++++++++++++++++++++++++++++ web/service.go | 40 +++++++++++++++++++ 2 files changed, 129 insertions(+) create mode 100644 web/policy_endpoint.go create mode 100644 web/service.go diff --git a/web/policy_endpoint.go b/web/policy_endpoint.go new file mode 100644 index 0000000..95db129 --- /dev/null +++ b/web/policy_endpoint.go @@ -0,0 +1,89 @@ +package web + +import ( + "encoding/json" + "log" + "net/http" + + "github.com/oa-pass/pass-policy-service/rule" + "github.com/pkg/errors" +) + +const ( + submissionQueryParam = "submission" +) + +type policyEndpoint struct { + *PolicyService +} + +func (p *policyEndpoint) findPolicies(submission string, headers map[string][]string) ([]rule.Policy, error) { + context := &rule.Context{ + SubmissionURI: submission, + Headers: headers, + PassClient: p.fetcher, + } + policies := make([]rule.Policy, 0, len(p.rules.Policies)*2) + for _, policy := range p.rules.Policies { + resolved, err := policy.Resolve(context) + if err != nil { + return nil, errors.Wrapf(err, "could not resolve policies for submission %s", submission) + } + policies = append(policies, resolved...) + } + + return policies, nil +} + +func (p *policyEndpoint) sendPolicies(w http.ResponseWriter, r *http.Request, policies []rule.Policy, err error) { + if err != nil { + log.Printf("Error resolving policies: %+v", err) + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + encoder := json.NewEncoder(w) + encoder.SetIndent("", " ") + err = encoder.Encode(policies) + if err != nil { + log.Printf("error encoding JSON response: %s", err) + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } +} + +func (p *policyEndpoint) handleGet(w http.ResponseWriter, r *http.Request) { + uri, ok := r.URL.Query()[submissionQueryParam] + if !ok { + // It would be nice to provide a pretty html page + http.Error(w, "No submission query param provided", http.StatusBadRequest) + return + } + + policies, err := p.findPolicies(uri[0], r.Header) + p.sendPolicies(w, r, policies, err) +} + +func (p *policyEndpoint) handlePost(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Content-Type") != "application/x-www-form-urlencoded" { + http.Error(w, + "expected media type application/x-www-form-urlencoded, instead got "+ + r.Header.Get("Content-Type"), + http.StatusBadRequest) + return + } + + if err := r.ParseForm(); err != nil { + http.Error(w, "Could not parse form input: "+err.Error(), http.StatusInternalServerError) + return + } + + url := r.Form.Get(submissionQueryParam) + if url == "" { + http.Error(w, "No submission value provided", http.StatusBadRequest) + return + } + + policies, err := p.findPolicies(url, r.Header) + p.sendPolicies(w, r, policies, err) +} diff --git a/web/service.go b/web/service.go new file mode 100644 index 0000000..2283bc9 --- /dev/null +++ b/web/service.go @@ -0,0 +1,40 @@ +package web + +import ( + "net/http" + + "github.com/oa-pass/pass-policy-service/rule" + "github.com/pkg/errors" +) + +type PolicyService struct { + rules *rule.DSL + fetcher rule.PassEntityFetcher +} + +func NewPolicyService(rulesDoc []byte, fetcher rule.PassEntityFetcher) (service PolicyService, err error) { + + service = PolicyService{fetcher: fetcher} + service.rules, err = rule.Validate(rulesDoc) + if err != nil { + return service, errors.Wrapf(err, "could not validate rules dsl") + } + + return service, nil +} + +func (s *PolicyService) RequestPolicies(w http.ResponseWriter, r *http.Request) { + + policyEndpoint := policyEndpoint{s} + + w.Header().Set("Content-Type", "application/json") + + switch r.Method { + case http.MethodGet: + policyEndpoint.handleGet(w, r) + case http.MethodPost: + policyEndpoint.handlePost(w, r) + default: + http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) + } +} From e080ae605a3c344a473a31f9b71714eaebc9262f Mon Sep 17 00:00:00 2001 From: Aaron Birkland Date: Tue, 23 Apr 2019 16:02:05 -0400 Subject: [PATCH 3/5] Use policy service for serve' command --- cmd/pass-policy-service/serve.go | 46 +++++++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/cmd/pass-policy-service/serve.go b/cmd/pass-policy-service/serve.go index f38a897..17bf938 100644 --- a/cmd/pass-policy-service/serve.go +++ b/cmd/pass-policy-service/serve.go @@ -2,7 +2,13 @@ package main import ( "fmt" + "io/ioutil" + "log" + "net" + "net/http" + "github.com/oa-pass/pass-policy-service/web" + "github.com/pkg/errors" "github.com/urfave/cli" ) @@ -63,5 +69,43 @@ func serve() cli.Command { } func serveAction(opts serveOpts, args []string) error { - return fmt.Errorf("not implemented") + + if len(args) != 1 { + return fmt.Errorf("expecting exactly one argument: the rules doc file") + } + + var credentials *web.Credentials + if opts.username != "" { + credentials = &web.Credentials{ + Username: opts.username, + Password: opts.passwd, + } + } + + rules, err := ioutil.ReadFile(args[0]) + if err != nil { + return fmt.Errorf("error reading %s: %s", args[0], err.Error()) + } + + policyService, err := web.NewPolicyService(rules, &web.InternalPassClient{ + Requester: &http.Client{}, + ExternalBaseURI: opts.publicBaseURI, + InternalBaseURI: opts.privateBaseURI, + Credentials: credentials, + }) + if err != nil { + return errors.Wrapf(err, "could not initialize policy service") + } + + http.HandleFunc("/policies", policyService.RequestPolicies) + + listener, err := net.Listen("tcp", fmt.Sprintf(":%d", opts.port)) + if err != nil { + return err + } + + opts.port = listener.Addr().(*net.TCPAddr).Port + log.Printf("Listening on port %d", opts.port) + + return http.Serve(listener, nil) } From 4b60f0a1dda80594783cfa635d55f758edd404f2 Mon Sep 17 00:00:00 2001 From: Aaron Birkland Date: Tue, 23 Apr 2019 16:02:41 -0400 Subject: [PATCH 4/5] Add docker-appropriate policies dsl --- .env | 2 ++ Dockerfile | 3 ++- policies/docker.json | 43 +++++++++++++++++++++++++++++++++++++++++++ scripts/entrypoint.sh | 2 +- 4 files changed, 48 insertions(+), 2 deletions(-) create mode 100644 policies/docker.json diff --git a/.env b/.env index af277c4..f5432ba 100644 --- a/.env +++ b/.env @@ -1,5 +1,7 @@ POLICY_SERVICE_PORT=8088 +POLICY_FILE=docker.json + PASS_EXTERNAL_FEDORA_BASEURL=http://localhost:8080/fcrepo/rest PASS_FEDORA_BASEURL=http://fcrepo:8080/fcrepo/rest diff --git a/Dockerfile b/Dockerfile index 2828a25..2f88ee3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,8 @@ RUN go generate ./... && \ CGO_ENABLED=0 go build ./cmd/pass-policy-service FROM alpine:3.9 -COPY --from=builder /root/pass-policy-service /root/scripts / +ENV POLICY_FILE=docker.json +COPY --from=builder /root/pass-policy-service /root/scripts /root/policies / RUN chmod 700 /entrypoint.sh diff --git a/policies/docker.json b/policies/docker.json new file mode 100644 index 0000000..d53430c --- /dev/null +++ b/policies/docker.json @@ -0,0 +1,43 @@ +{ + "$schema": "https://oa-pass.github.io/pass-policy-service/schemas/policy_config_1.0.json", + "policy-rules": [ + { + "description": "Must deposit to one of the repositories indicated by primary funder", + "policy-id": "${submission.grants.primaryFunder.policy}", + "repositories": [ + { + "repository-id": "${policy.repositories}" + } + ] + }, + { + "description": "Must deposit to one of the repositories indicated by direct funder", + "policy-id": "${submission.grants.directFunder.policy}", + "repositories": [ + { + "repository-id": "${policy.repositories}" + } + ] + }, + { + "description": "Members of the JHU community must deposit into JScholarship, or some other repository.", + "policy-id": "policies/", + "conditions": [ + { + "endsWith": { + "@johnshopkins.edu": "${header.Eppn}" + } + } + ], + "repositories": [ + { + "repository-id": "https://pass.local/fcrepo/rest/repositories/j10p", + "selected": true + }, + { + "repository-id": "*" + } + ] + } + ] +} \ No newline at end of file diff --git a/scripts/entrypoint.sh b/scripts/entrypoint.sh index 92326d1..412433a 100644 --- a/scripts/entrypoint.sh +++ b/scripts/entrypoint.sh @@ -4,4 +4,4 @@ printf "\n**** Begin Environment Variable Dump ****\n\n" printenv | sort printf "\n**** End Environment Variable Dump ****\n\n" -./pass-policy-service serve +./pass-policy-service serve ${POLICY_FILE} From 2bc0a58c14355dda929c6dcf72b0b74d04bd2e84 Mon Sep 17 00:00:00 2001 From: Aaron Birkland Date: Tue, 23 Apr 2019 16:03:55 -0400 Subject: [PATCH 5/5] Add integration test for /policies endpoint --- cmd/pass-policy-service/integration_test.go | 215 ++++++++++++++++++++ 1 file changed, 215 insertions(+) diff --git a/cmd/pass-policy-service/integration_test.go b/cmd/pass-policy-service/integration_test.go index 8fe4290..0e74f68 100644 --- a/cmd/pass-policy-service/integration_test.go +++ b/cmd/pass-policy-service/integration_test.go @@ -3,9 +3,224 @@ package main import ( + "encoding/json" + "fmt" + "io" + "io/ioutil" + "net/http" + "os" + "strings" "testing" + + "github.com/oa-pass/pass-policy-service/rule" ) +const defaultFedoraBaseuri = "http://localhost:8080/fcrepo/rest" + func TestFedoraIntegration(t *testing.T) { + client := &http.Client{} + + fedora := resourceHelper{t, client} + + nihRepo := fedora.repository("repositories/nih") + decRepo := fedora.repository("repositories/dec") + + nihPolicy := fedora.policy("policies/nih", []string{nihRepo}) + decPolcy := fedora.policy("policies/dec", []string{decRepo}) + + nihFunder := fedora.funder("funders/nih", nihPolicy) + decFunder := fedora.funder("funders/dec", decPolcy) + + nihGrant := fedora.grant("grants/nih", nihFunder, "") + decGrant := fedora.grant("grants/dec", "", decFunder) + + submission := fedora.submission("submissions/foo", []string{nihGrant, decGrant}) + + get, _ := http.NewRequest(http.MethodGet, policyServiceURI()+"/policies?submission="+submission, nil) + get.Header.Set("Eppn", "someone@johnshopkins.edu") + + resp, err := client.Do(get) + if err != nil { + t.Fatalf("GET request to %s failed: %s", get.RequestURI, err) + } + + if resp.StatusCode > 299 { + t.Fatalf("Policy service returned an error on GET to %s: %d", get.URL, resp.StatusCode) + } + + // Read in the body + defer resp.Body.Close() + body, _ := ioutil.ReadAll(resp.Body) + + var policies []rule.Policy + + _ = json.Unmarshal(body, &policies) + if len(policies) != 3 { + t.Fatalf("Wrong number of policies, got %d", len(policies)) + } +} + +func authz(r *http.Request) { + username, ok := os.LookupEnv("PASS_FEDORA_USER") + if !ok { + username = "fedoraAdmin" + } + + passwd, ok := os.LookupEnv("PASS_FEDORA_PASSWORD") + if !ok { + passwd = "moo" + } + + r.SetBasicAuth(username, passwd) +} + +func policyServiceURI() string { + port, ok := os.LookupEnv("POLICY_SERVICE_PORT") + if !ok { + port = "8088" + } + + host, ok := os.LookupEnv("POLICY_SERVICE_HOST") + if !ok { + host = "localhost" + } + + return fmt.Sprintf("http://%s:%s", host, port) +} + +func fedoraURI(uripath string) string { + return fmt.Sprintf("%s/%s", fedoraBaseURI(), uripath) +} + +func fedoraBaseURI() string { + baseuri, ok := os.LookupEnv("PASS_EXTERNAL_FEDORA_BASEURL") + if !ok { + return defaultFedoraBaseuri + } + + return strings.Trim(baseuri, "/") +} + +type resourceHelper struct { + t *testing.T + c *http.Client +} + +func (r *resourceHelper) submission(path string, funders []string) string { + uri := fedoraURI(path) + + submission := fmt.Sprintf(`{ + "@context" : "https://oa-pass.github.io/pass-data-model/src/main/resources/context-3.4.jsonld", + "@id" : "%s", + "grants": [ + %s + ], + "@type" : "Submission" + } + `, uri, jsonList(funders)) + + r.putResource(uri, submission) + return uri +} + +func (r *resourceHelper) grant(path, priFunder, dirFunder string) string { + var funders string + if priFunder != "" { + funders = fmt.Sprintf(`"primaryFunder": "%s",`, priFunder) + "\n" + } + if dirFunder != "" { + funders = fmt.Sprintf(`%s"directFunder": "%s",`, funders, dirFunder) + "\n" + } + + uri := fedoraURI(path) + + funder := fmt.Sprintf(`{ + "@context" : "https://oa-pass.github.io/pass-data-model/src/main/resources/context-3.4.jsonld", + "@id" : "%s", + %s + "@type" : "Grant" + } + `, uri, funders) + + r.putResource(uri, funder) + return uri + +} + +func (r *resourceHelper) funder(path, policy string) string { + uri := fedoraURI(path) + + funder := fmt.Sprintf(`{ + "@context" : "https://oa-pass.github.io/pass-data-model/src/main/resources/context-3.4.jsonld", + "@id" : "%s", + "policy": "%s", + "@type" : "Funder" + } + `, uri, policy) + + r.putResource(uri, funder) + return uri +} + +func (r *resourceHelper) policy(path string, repositories []string) string { + uri := fedoraURI(path) + + policy := fmt.Sprintf(`{ + "@context" : "https://oa-pass.github.io/pass-data-model/src/main/resources/context-3.4.jsonld", + "@id" : "%s", + "repositories": [ + %s + ], + "@type" : "Policy" + } + `, uri, jsonList(repositories)) + + r.putResource(uri, policy) + return uri +} + +func jsonList(list []string) string { + var jsonList []string + for _, item := range list { + jsonList = append(jsonList, fmt.Sprintf(`"%s"`, item)) + } + + return strings.Trim(strings.Join(jsonList, ",\n"), ",\n") +} + +func (r *resourceHelper) repository(path string) string { + uri := fedoraURI(path) + + repo := fmt.Sprintf(`{ + "@context" : "https://oa-pass.github.io/pass-data-model/src/main/resources/context-3.4.jsonld", + "@id" : "%s", + "@type" : "Repository" + } + `, uri) + + r.putResource(uri, repo) + return uri +} + +func (r *resourceHelper) putResource(uri, body string) { + request, err := http.NewRequest(http.MethodPut, uri, strings.NewReader(body)) + if err != nil { + r.t.Fatalf("Building request failed: %s", err) + } + + request.Header.Set("Content-Type", "application/ld+json") + request.Header.Set("Prefer", `handling=lenient; received="minimal"`) + authz(request) + + resp, err := r.c.Do(request) + if err != nil { + r.t.Fatalf("PUT request failed: %s", err) + } + + defer resp.Body.Close() + io.Copy(ioutil.Discard, resp.Body) + if resp.StatusCode > 299 { + r.t.Fatalf("Could not add resource: %d, body:\n%s", resp.StatusCode, body) + } }