From 785dc5f84b84b09b0a0f16497ae88728daab7722 Mon Sep 17 00:00:00 2001 From: Ian Rees Date: Fri, 27 Oct 2023 12:03:51 -0700 Subject: [PATCH 1/2] Add GeoJSONL support --- server/rest/rest.go | 24 ++++++++++++++++++++++-- server/rest/route_request.go | 5 +++++ server/rest/trip_request.go | 20 +++++++++----------- server/rest/trip_request_test.go | 4 ++-- 4 files changed, 38 insertions(+), 15 deletions(-) diff --git a/server/rest/rest.go b/server/rest/rest.go index d9f4e5ee..5b9182bd 100644 --- a/server/rest/rest.go +++ b/server/rest/rest.go @@ -331,7 +331,7 @@ func makeRequest(ctx context.Context, cfg restConfig, ent apiHandler, format str } } - if format == "geojson" || format == "png" { + if format == "geojson" || format == "geojsonl" || format == "png" { // TODO: Don't process response in-place. if v, ok := ent.(canProcessGeoJSON); ok { if err := v.ProcessGeoJSON(response); err != nil { @@ -342,7 +342,9 @@ func makeRequest(ctx context.Context, cfg restConfig, ent apiHandler, format str return nil, err } } - if format == "png" { + if format == "geojsonl" { + return renderGeojsonl(response) + } else if format == "png" { b, err := json.Marshal(response) if err != nil { return nil, err @@ -353,6 +355,24 @@ func makeRequest(ctx context.Context, cfg restConfig, ent apiHandler, format str return json.Marshal(response) } +func renderGeojsonl(response map[string]any) ([]byte, error) { + var ret []byte + feats, ok := response["features"].([]map[string]any) + if !ok { + return nil, errors.New("not features") + } + for _, feat := range feats { + j, err := json.Marshal(feat) + if err != nil { + return nil, err + } + ret = append(ret, j...) + ret = append(ret, byte('\n')) + } + + return ret, nil +} + func makeHandlerFunc(cfg restConfig, handlerName string, f func(restConfig, http.ResponseWriter, *http.Request)) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { if apiMeter := meters.ForContext(r.Context()); apiMeter != nil { diff --git a/server/rest/route_request.go b/server/rest/route_request.go index 1a1c592a..90ec0e1f 100644 --- a/server/rest/route_request.go +++ b/server/rest/route_request.go @@ -39,6 +39,11 @@ func (r RouteRequest) ResponseKey() string { return "routes" } // Query returns a GraphQL query string and variables. func (r RouteRequest) Query() (string, map[string]interface{}) { + // These formats will need geometries included + if r.ID > 0 || r.Format == "geojson" || r.Format == "geojsonl" || r.Format == "png" { + r.IncludeGeometry = true + } + // Handle operator key if r.AgencyKey == "" { // pass diff --git a/server/rest/trip_request.go b/server/rest/trip_request.go index 31e58c4f..c35d0746 100644 --- a/server/rest/trip_request.go +++ b/server/rest/trip_request.go @@ -17,9 +17,9 @@ type TripRequest struct { RouteOnestopID string `json:"route_onestop_id"` FeedOnestopID string `json:"feed_onestop_id"` FeedVersionSHA1 string `json:"feed_version_sha1"` - IncludeGeometry string `json:"include_geometry"` - IncludeStopTimes string `json:"include_stop_times"` ServiceDate string `json:"service_date"` + IncludeGeometry bool `json:"include_geometry,string"` + IncludeStopTimes bool `json:"include_stop_times,string"` IncludeAlerts bool `json:"include_alerts,string"` Format string LicenseFilter @@ -62,15 +62,13 @@ func (r TripRequest) Query() (string, map[string]interface{}) { } where["license"] = checkLicenseFilter(r.LicenseFilter) // Include geometry when in geojson format - includeGeometry := false - if r.ID > 0 || r.IncludeGeometry == "true" || r.Format == "geojson" { - includeGeometry = true + if r.ID > 0 || r.Format == "geojson" || r.Format == "geojsonl" { + r.IncludeGeometry = true } // Only include stop times when requesting a specific trip. - includeStopTimes := false - // || r.IncludeStopTimes == "true" - if r.ID > 0 || r.Format == "geojson" { - includeStopTimes = true + r.IncludeStopTimes = false + if r.ID > 0 { + r.IncludeStopTimes = true } includeRoute := false return tripQuery, hw{ @@ -78,8 +76,8 @@ func (r TripRequest) Query() (string, map[string]interface{}) { "after": r.CheckAfter(), "ids": checkIds(r.ID), "where": where, - "include_geometry": includeGeometry, - "include_stop_times": includeStopTimes, + "include_geometry": r.IncludeGeometry, + "include_stop_times": r.IncludeStopTimes, "include_route": includeRoute, "include_alerts": r.IncludeAlerts, } diff --git a/server/rest/trip_request_test.go b/server/rest/trip_request_test.go index faf04aab..8f8f6686 100644 --- a/server/rest/trip_request_test.go +++ b/server/rest/trip_request_test.go @@ -119,14 +119,14 @@ func TestTripRequest(t *testing.T) { }, { name: "include_geometry=true", - h: TripRequest{TripID: "5132248WKDY", IncludeGeometry: "true"}, + h: TripRequest{TripID: "5132248WKDY", IncludeGeometry: true}, selector: "trips.0.shape.geometry.type", expectSelect: []string{"LineString"}, expectLength: 0, }, { name: "include_geometry=false", - h: TripRequest{TripID: "5132248WKDY", IncludeGeometry: "false"}, + h: TripRequest{TripID: "5132248WKDY", IncludeGeometry: false}, selector: "trips.0.shape.geometry.type", expectSelect: []string{}, expectLength: 0, From cbe26ea7ca4eaab629ae3dd81688fa8c1e10cbd9 Mon Sep 17 00:00:00 2001 From: Ian Rees Date: Fri, 27 Oct 2023 14:28:55 -0700 Subject: [PATCH 2/2] Tests --- server/rest/agency_request_test.go | 37 ++++++++++++++++++++++++++++ server/rest/feed_request_test.go | 39 ++++++++++++++++++++++++++++++ server/rest/map.go | 6 +++-- server/rest/operator_request.gql | 1 + server/rest/rest.go | 6 +++-- server/rest/route_request_test.go | 37 ++++++++++++++++++++++++++++ server/rest/stop_request.go | 1 + server/rest/stop_request_test.go | 37 ++++++++++++++++++++++++++++ server/rest/trip_request_test.go | 37 ++++++++++++++++++++++++++++ 9 files changed, 197 insertions(+), 4 deletions(-) diff --git a/server/rest/agency_request_test.go b/server/rest/agency_request_test.go index 508d379d..f1569a73 100644 --- a/server/rest/agency_request_test.go +++ b/server/rest/agency_request_test.go @@ -2,6 +2,7 @@ package rest import ( "context" + "strings" "testing" "github.com/stretchr/testify/assert" @@ -163,6 +164,42 @@ func TestAgencyRequest(t *testing.T) { } } +func TestAgencyRequest_Format(t *testing.T) { + tcs := []testRest{ + { + name: "agency geojson", + format: "geojson", + h: AgencyRequest{FeedOnestopID: "CT", WithCursor: WithCursor{Limit: 1}}, + f: func(t *testing.T, jj string) { + a := gjson.Get(jj, "features").Array() + assert.Equal(t, 1, len(a)) + assert.Equal(t, "Feature", gjson.Get(jj, "features.0.type").String()) + assert.Equal(t, "Polygon", gjson.Get(jj, "features.0.geometry.type").String()) + assert.Equal(t, "CT", gjson.Get(jj, "features.0.properties.feed_version.feed.onestop_id").String()) + assert.Greater(t, gjson.Get(jj, "meta.after").Int(), int64(0)) + }, + }, + { + name: "agency geojsonl", + format: "geojsonl", + h: AgencyRequest{FeedOnestopID: "CT", WithCursor: WithCursor{Limit: 1}}, + f: func(t *testing.T, jj string) { + split := strings.Split(jj, "\n") + assert.Equal(t, 1, len(split)) + assert.Equal(t, "Feature", gjson.Get(split[0], "type").String()) + assert.Equal(t, "Polygon", gjson.Get(split[0], "geometry.type").String()) + assert.Equal(t, "CT", gjson.Get(split[0], "properties.feed_version.feed.onestop_id").String()) + }, + }, + } + srv, te := testRestConfig(t) + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + testquery(t, srv, te, tc) + }) + } +} + func TestAgencyRequest_Pagination(t *testing.T) { srv, te := testRestConfig(t) allEnts, err := te.Finder.FindAgencies(context.Background(), nil, nil, nil, nil) diff --git a/server/rest/feed_request_test.go b/server/rest/feed_request_test.go index 57c603f2..0ea1d5b3 100644 --- a/server/rest/feed_request_test.go +++ b/server/rest/feed_request_test.go @@ -1,9 +1,12 @@ package rest import ( + "strings" "testing" "github.com/interline-io/transitland-server/model" + "github.com/stretchr/testify/assert" + "github.com/tidwall/gjson" ) func TestFeedRequest(t *testing.T) { @@ -161,6 +164,42 @@ func TestFeedRequest(t *testing.T) { } } +func TestFeedRequest_Format(t *testing.T) { + tcs := []testRest{ + { + name: "feed geojson", + format: "geojson", + h: FeedRequest{WithCursor: WithCursor{Limit: 5}}, + f: func(t *testing.T, jj string) { + a := gjson.Get(jj, "features").Array() + assert.Equal(t, 5, len(a)) + assert.Equal(t, "Feature", gjson.Get(jj, "features.0.type").String()) + assert.Equal(t, "Polygon", gjson.Get(jj, "features.0.geometry.type").String()) + assert.Equal(t, "CT", gjson.Get(jj, "features.0.properties.onestop_id").String()) + assert.Greater(t, gjson.Get(jj, "meta.after").Int(), int64(0)) + }, + }, + { + name: "feed geojsonl", + format: "geojsonl", + h: FeedRequest{WithCursor: WithCursor{Limit: 5}}, + f: func(t *testing.T, jj string) { + split := strings.Split(jj, "\n") + assert.Equal(t, 5, len(split)) + assert.Equal(t, "Feature", gjson.Get(split[0], "type").String()) + assert.Equal(t, "Polygon", gjson.Get(split[0], "geometry.type").String()) + assert.Equal(t, "CT", gjson.Get(split[0], "properties.onestop_id").String()) + }, + }, + } + srv, te := testRestConfig(t) + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + testquery(t, srv, te, tc) + }) + } +} + func TestFeedRequest_License(t *testing.T) { testcases := []testRest{ { diff --git a/server/rest/map.go b/server/rest/map.go index 52bbd93e..bd6d855e 100644 --- a/server/rest/map.go +++ b/server/rest/map.go @@ -112,8 +112,10 @@ func processGeoJSON(ent apiHandler, response map[string]interface{}) error { } geometry := f["geometry"] if geometry == nil { - log.Infof("feature has no geometry, skipping") - continue + geometry = map[string]any{ + "type": "Polygon", + "coordinates": []float64{}, + } } delete(f, "geometry") properties := hw{} diff --git a/server/rest/operator_request.gql b/server/rest/operator_request.gql index a05cf755..e517ff12 100644 --- a/server/rest/operator_request.gql +++ b/server/rest/operator_request.gql @@ -46,6 +46,7 @@ query ($limit: Int, $after: Int, $include_alerts: Boolean!, $where: OperatorFilt id agency_id agency_name + geometry alerts @include(if: $include_alerts) { ...alert } diff --git a/server/rest/rest.go b/server/rest/rest.go index 5b9182bd..49624907 100644 --- a/server/rest/rest.go +++ b/server/rest/rest.go @@ -361,13 +361,15 @@ func renderGeojsonl(response map[string]any) ([]byte, error) { if !ok { return nil, errors.New("not features") } - for _, feat := range feats { + for i, feat := range feats { j, err := json.Marshal(feat) if err != nil { return nil, err } ret = append(ret, j...) - ret = append(ret, byte('\n')) + if i < len(feats)-1 { + ret = append(ret, byte('\n')) + } } return ret, nil diff --git a/server/rest/route_request_test.go b/server/rest/route_request_test.go index 86de8888..8b952032 100644 --- a/server/rest/route_request_test.go +++ b/server/rest/route_request_test.go @@ -2,6 +2,7 @@ package rest import ( "context" + "strings" "testing" "github.com/interline-io/transitland-server/model" @@ -123,6 +124,42 @@ func TestRouteRequest(t *testing.T) { } } +func TestRouteRequest_Format(t *testing.T) { + tcs := []testRest{ + { + name: "route geojson", + format: "geojson", + h: RouteRequest{FeedOnestopID: "CT", Format: "geojson", WithCursor: WithCursor{Limit: 5}}, + f: func(t *testing.T, jj string) { + a := gjson.Get(jj, "features").Array() + assert.Equal(t, 5, len(a)) + assert.Equal(t, "Feature", gjson.Get(jj, "features.0.type").String()) + assert.Equal(t, "MultiLineString", gjson.Get(jj, "features.0.geometry.type").String()) + assert.Equal(t, "CT", gjson.Get(jj, "features.0.properties.feed_version.feed.onestop_id").String()) + assert.Greater(t, gjson.Get(jj, "meta.after").Int(), int64(0)) + }, + }, + { + name: "route geojsonl", + format: "geojsonl", + h: RouteRequest{FeedOnestopID: "CT", Format: "geojsonl", WithCursor: WithCursor{Limit: 5}}, + f: func(t *testing.T, jj string) { + split := strings.Split(jj, "\n") + assert.Equal(t, 5, len(split)) + assert.Equal(t, "Feature", gjson.Get(split[0], "type").String()) + assert.Equal(t, "MultiLineString", gjson.Get(split[0], "geometry.type").String()) + assert.Equal(t, "CT", gjson.Get(split[0], "properties.feed_version.feed.onestop_id").String()) + }, + }, + } + srv, te := testRestConfig(t) + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + testquery(t, srv, te, tc) + }) + } +} + func TestRouteRequest_Pagination(t *testing.T) { srv, te := testRestConfig(t) allEnts, err := te.Finder.FindRoutes(context.Background(), nil, nil, nil, nil) diff --git a/server/rest/stop_request.go b/server/rest/stop_request.go index fcb6eff4..076d49d6 100644 --- a/server/rest/stop_request.go +++ b/server/rest/stop_request.go @@ -22,6 +22,7 @@ type StopRequest struct { Lon float64 `json:"lon,string"` Lat float64 `json:"lat,string"` Radius float64 `json:"radius,string"` + Format string `json:"format"` ServedByOnestopIds string `json:"served_by_onestop_ids"` ServedByRouteType *int `json:"served_by_route_type,string"` IncludeAlerts bool `json:"include_alerts,string"` diff --git a/server/rest/stop_request_test.go b/server/rest/stop_request_test.go index a89b613e..ad5a8cd3 100644 --- a/server/rest/stop_request_test.go +++ b/server/rest/stop_request_test.go @@ -2,6 +2,7 @@ package rest import ( "context" + "strings" "testing" "github.com/interline-io/transitland-server/model" @@ -203,6 +204,42 @@ func TestStopRequest_AdminCache(t *testing.T) { testquery(t, srv, te, tc) } +func TestStopRequest_Format(t *testing.T) { + tcs := []testRest{ + { + name: "stop geojson", + format: "geojson", + h: StopRequest{FeedOnestopID: "CT", Format: "geojson", WithCursor: WithCursor{Limit: 20}}, + f: func(t *testing.T, jj string) { + a := gjson.Get(jj, "features").Array() + assert.Equal(t, 20, len(a)) + assert.Equal(t, "Feature", gjson.Get(jj, "features.0.type").String()) + assert.Equal(t, "Point", gjson.Get(jj, "features.0.geometry.type").String()) + assert.Equal(t, "CT", gjson.Get(jj, "features.0.properties.feed_version.feed.onestop_id").String()) + assert.Greater(t, gjson.Get(jj, "meta.after").Int(), int64(0)) + }, + }, + { + name: "stop geojsonl", + format: "geojsonl", + h: StopRequest{FeedOnestopID: "CT", Format: "geojsonl", WithCursor: WithCursor{Limit: 20}}, + f: func(t *testing.T, jj string) { + split := strings.Split(jj, "\n") + assert.Equal(t, 20, len(split)) + assert.Equal(t, "Feature", gjson.Get(split[0], "type").String()) + assert.Equal(t, "Point", gjson.Get(split[0], "geometry.type").String()) + assert.Equal(t, "CT", gjson.Get(split[0], "properties.feed_version.feed.onestop_id").String()) + }, + }, + } + srv, te := testRestConfig(t) + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + testquery(t, srv, te, tc) + }) + } +} + func TestStopRequest_Pagination(t *testing.T) { srv, te := testRestConfig(t) allEnts, err := te.Finder.FindStops(context.Background(), nil, nil, nil, nil) diff --git a/server/rest/trip_request_test.go b/server/rest/trip_request_test.go index 8f8f6686..75f8961e 100644 --- a/server/rest/trip_request_test.go +++ b/server/rest/trip_request_test.go @@ -3,6 +3,7 @@ package rest import ( "context" "strconv" + "strings" "testing" "github.com/stretchr/testify/assert" @@ -183,6 +184,42 @@ func TestTripRequest(t *testing.T) { } } +func TestTripRequest_Format(t *testing.T) { + tcs := []testRest{ + { + name: "trip geojson", + format: "geojson", + h: TripRequest{TripID: "5132248WKDY", Format: "geojson", WithCursor: WithCursor{Limit: 1}}, + f: func(t *testing.T, jj string) { + a := gjson.Get(jj, "features").Array() + assert.Equal(t, 1, len(a)) + assert.Equal(t, "Feature", gjson.Get(jj, "features.0.type").String()) + assert.Equal(t, "LineString", gjson.Get(jj, "features.0.geometry.type").String()) + assert.Equal(t, "BA", gjson.Get(jj, "features.0.properties.feed_version.feed.onestop_id").String()) + assert.Greater(t, gjson.Get(jj, "meta.after").Int(), int64(0)) + }, + }, + { + name: "trip geojsonl", + format: "geojsonl", + h: TripRequest{TripID: "5132248WKDY", Format: "geojsonl"}, + f: func(t *testing.T, jj string) { + split := strings.Split(jj, "\n") + assert.Equal(t, 1, len(split)) + assert.Equal(t, "Feature", gjson.Get(split[0], "type").String()) + assert.Equal(t, "LineString", gjson.Get(split[0], "geometry.type").String()) + assert.Equal(t, "BA", gjson.Get(split[0], "properties.feed_version.feed.onestop_id").String()) + }, + }, + } + srv, te := testRestConfig(t) + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + testquery(t, srv, te, tc) + }) + } +} + func TestTripRequest_Pagination(t *testing.T) { testcases := []testRest{ {