From 84612ff8f8236605e829c7049fc682ba421d3fe4 Mon Sep 17 00:00:00 2001 From: Richard Kettelerij Date: Fri, 29 Nov 2024 16:40:29 +0100 Subject: [PATCH 01/38] feat: add search endpoint - groundwork --- cmd/main.go | 3 ++ internal/ogc/setup.go | 2 -- internal/search/.keep | 0 internal/search/main.go | 68 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 71 insertions(+), 2 deletions(-) delete mode 100644 internal/search/.keep create mode 100644 internal/search/main.go diff --git a/cmd/main.go b/cmd/main.go index b8845fb..af5ca14 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -8,6 +8,7 @@ import ( "strconv" "github.com/PDOK/gomagpie/config" + "github.com/PDOK/gomagpie/internal/search" "github.com/iancoleman/strcase" eng "github.com/PDOK/gomagpie/internal/engine" @@ -172,6 +173,8 @@ func main() { } // Each OGC API building block makes use of said Engine ogc.SetupBuildingBlocks(engine, dbConn) + // Start search logic + search.NewSearch(engine) return engine.Start(address, debugPort, shutdownDelay) }, diff --git a/internal/ogc/setup.go b/internal/ogc/setup.go index d7386e4..f91e92d 100644 --- a/internal/ogc/setup.go +++ b/internal/ogc/setup.go @@ -14,6 +14,4 @@ func SetupBuildingBlocks(engine *engine.Engine, _ string) { if engine.Config.HasCollections() { geospatial.NewCollections(engine) } - - // TODO Something with the dbConnString param in PDOK-17118 } diff --git a/internal/search/.keep b/internal/search/.keep deleted file mode 100644 index e69de29..0000000 diff --git a/internal/search/main.go b/internal/search/main.go new file mode 100644 index 0000000..b4013bf --- /dev/null +++ b/internal/search/main.go @@ -0,0 +1,68 @@ +package search + +import ( + "log" + "net/http" + "net/url" + "strings" + + "github.com/PDOK/gomagpie/internal/engine" +) + +type Search struct { + engine *engine.Engine +} + +func NewSearch(e *engine.Engine) *Search { + s := &Search{ + engine: e, + } + e.Router.Get("/search/suggest", s.Suggest()) + return s +} + +// Suggest autosuggest locations based on user input +func (s *Search) Suggest() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + params, err := parseQueryParams(r.URL.Query()) + if err != nil { + log.Printf("%v", err) + engine.RenderProblem(engine.ProblemBadRequest, w) + return + } + searchQuery := params["q"] + delete(params, "q") + format := params["f"] + delete(params, "f") + crs := params["crs"] + delete(params, "crs") + + log.Printf("crs %s, format %s, query %s, params %v", crs, format, searchQuery, params) + } +} + +func parseQueryParams(query url.Values) (map[string]any, error) { + result := make(map[string]any, len(query)) + + deepObjectParams := make(map[string]map[string]string) + for key, values := range query { + if strings.Contains(key, "[") { + // Extract deepObject parameters + parts := strings.SplitN(key, "[", 2) + mainKey := parts[0] + subKey := strings.TrimSuffix(parts[1], "]") + + if _, exists := deepObjectParams[mainKey]; !exists { + deepObjectParams[mainKey] = make(map[string]string) + } + deepObjectParams[mainKey][subKey] = values[0] + } else { + // Extract regular (flat) parameters + result[key] = values[0] + } + } + for mainKey, subParams := range deepObjectParams { + result[mainKey] = subParams + } + return result, nil +} From 668face00007155f760bc4dd76f5e43f50acd18b Mon Sep 17 00:00:00 2001 From: Richard Kettelerij Date: Mon, 2 Dec 2024 17:16:24 +0100 Subject: [PATCH 02/38] feat: add search endpoint - connect to postgres --- cmd/main.go | 11 +- go.mod | 1 + go.sum | 2 + internal/etl/etl.go | 2 +- internal/etl/load/postgres.go | 12 +- internal/search/datasources/datasource.go | 14 +++ .../search/datasources/postgres/postgres.go | 93 +++++++++++++++ internal/search/json.go | 51 +++++++++ internal/search/main.go | 33 +++++- internal/search/main_test.go | 106 ++++++++++++++++++ 10 files changed, 312 insertions(+), 13 deletions(-) create mode 100644 internal/search/datasources/datasource.go create mode 100644 internal/search/datasources/postgres/postgres.go create mode 100644 internal/search/json.go create mode 100644 internal/search/main_test.go diff --git a/cmd/main.go b/cmd/main.go index af5ca14..8e99737 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -153,6 +153,13 @@ func main() { commonDBFlags[dbUsernameFlag], commonDBFlags[dbPasswordFlag], commonDBFlags[dbSslModeFlag], + &cli.PathFlag{ + Name: searchIndexFlag, + EnvVars: []string{strcase.ToScreamingSnake(searchIndexFlag)}, + Usage: "Name of search index to use", + Required: true, + Value: "search_index", + }, }, Action: func(c *cli.Context) error { log.Println(c.Command.Usage) @@ -173,8 +180,8 @@ func main() { } // Each OGC API building block makes use of said Engine ogc.SetupBuildingBlocks(engine, dbConn) - // Start search logic - search.NewSearch(engine) + // Create search endpoint + search.NewSearch(engine, dbConn, c.String(searchIndexFlag)) return engine.Start(address, debugPort, shutdownDelay) }, diff --git a/go.mod b/go.mod index 426ca2c..e78c797 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,7 @@ require ( github.com/go-playground/validator/v10 v10.22.1 github.com/go-spatial/geom v0.1.0 github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 + github.com/goccy/go-json v0.10.3 github.com/gomarkdown/markdown v0.0.0-20240930133441-72d49d9543d8 github.com/iancoleman/strcase v0.3.0 github.com/jackc/pgx/v5 v5.7.1 diff --git a/go.sum b/go.sum index a34df63..21a6334 100644 --- a/go.sum +++ b/go.sum @@ -87,6 +87,8 @@ github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEe github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM= github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= +github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= +github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/gomarkdown/markdown v0.0.0-20240930133441-72d49d9543d8 h1:4txT5G2kqVAKMjzidIabL/8KqjIK71yj30YOeuxLn10= diff --git a/internal/etl/etl.go b/internal/etl/etl.go index b6c9da5..372abdd 100644 --- a/internal/etl/etl.go +++ b/internal/etl/etl.go @@ -114,7 +114,7 @@ func newSourceToExtract(filePath string) (Extract, error) { func newTargetToLoad(dbConn string) (Load, error) { if strings.HasPrefix(dbConn, "postgres:") { - return load.NewPostgis(dbConn) + return load.NewPostgres(dbConn) } // add new targets here (elasticsearch, solr, etc) return nil, fmt.Errorf("unsupported target database connection: %s", dbConn) diff --git a/internal/etl/load/postgres.go b/internal/etl/load/postgres.go index 65dfd10..453c6b6 100644 --- a/internal/etl/load/postgres.go +++ b/internal/etl/load/postgres.go @@ -9,12 +9,12 @@ import ( pgxgeom "github.com/twpayne/pgx-geom" ) -type Postgis struct { +type Postgres struct { db *pgx.Conn ctx context.Context } -func NewPostgis(dbConn string) (*Postgis, error) { +func NewPostgres(dbConn string) (*Postgres, error) { ctx := context.Background() db, err := pgx.Connect(ctx, dbConn) if err != nil { @@ -24,14 +24,14 @@ func NewPostgis(dbConn string) (*Postgis, error) { if err := pgxgeom.Register(ctx, db); err != nil { return nil, err } - return &Postgis{db: db, ctx: ctx}, nil + return &Postgres{db: db, ctx: ctx}, nil } -func (p *Postgis) Close() { +func (p *Postgres) Close() { _ = p.db.Close(p.ctx) } -func (p *Postgis) Load(records []t.SearchIndexRecord, index string) (int64, error) { +func (p *Postgres) Load(records []t.SearchIndexRecord, index string) (int64, error) { loaded, err := p.db.CopyFrom( p.ctx, pgx.Identifier{index}, @@ -48,7 +48,7 @@ func (p *Postgis) Load(records []t.SearchIndexRecord, index string) (int64, erro } // Init initialize search index -func (p *Postgis) Init(index string) error { +func (p *Postgres) Init(index string) error { geometryType := `create type geometry_type as enum ('POINT', 'MULTIPOINT', 'LINESTRING', 'MULTILINESTRING', 'POLYGON', 'MULTIPOLYGON');` _, err := p.db.Exec(p.ctx, geometryType) if err != nil { diff --git a/internal/search/datasources/datasource.go b/internal/search/datasources/datasource.go new file mode 100644 index 0000000..955d2bb --- /dev/null +++ b/internal/search/datasources/datasource.go @@ -0,0 +1,14 @@ +package datasources + +import ( + "context" +) + +// Datasource knows how make different kinds of queries/actions on the underlying actual datastore. +// This abstraction allows the rest of the system to stay datastore agnostic. +type Datasource interface { + Suggest(ctx context.Context, suggestForThis string) ([]string, error) + + // Close closes (connections to) the datasource gracefully + Close() +} diff --git a/internal/search/datasources/postgres/postgres.go b/internal/search/datasources/postgres/postgres.go new file mode 100644 index 0000000..5830b27 --- /dev/null +++ b/internal/search/datasources/postgres/postgres.go @@ -0,0 +1,93 @@ +package postgres + +import ( + "context" + "fmt" + + "github.com/jackc/pgx/v5" + pgxgeom "github.com/twpayne/pgx-geom" + + "strings" + "time" +) + +type Postgres struct { + db *pgx.Conn + ctx context.Context + + queryTimeout time.Duration + searchIndex string +} + +func NewPostgres(dbConn string, queryTimeout time.Duration, searchIndex string) (*Postgres, error) { + ctx := context.Background() + db, err := pgx.Connect(ctx, dbConn) + if err != nil { + return nil, fmt.Errorf("unable to connect to database: %w", err) + } + // add support for Go <-> PostGIS conversions + if err := pgxgeom.Register(ctx, db); err != nil { + return nil, err + } + return &Postgres{db, ctx, queryTimeout, searchIndex}, nil +} + +func (p *Postgres) Close() { + _ = p.db.Close(p.ctx) +} + +func (p *Postgres) Suggest(ctx context.Context, suggestForThis string) ([]string, error) { + queryCtx, cancel := context.WithTimeout(ctx, p.queryTimeout) + defer cancel() + + // Prepare dynamic full-text search query + // Split terms by spaces and append :* to each term + terms := strings.Fields(suggestForThis) + for i, term := range terms { + terms[i] = term + ":*" + } + searchTerm := strings.Join(terms, " & ") + + sqlQuery := fmt.Sprintf( + `SELECT + r.display_name AS display_name, + max(r.rank) AS rank, + max(r.highlighted_text) AS highlighted_text + FROM ( + SELECT display_name, + ts_rank_cd(ts, to_tsquery('%[1]s'), 1) AS rank, + ts_headline('dutch', suggest, to_tsquery('%[2]s')) AS highlighted_text + FROM + %[3]s + WHERE ts @@ to_tsquery('%[4]s') LIMIT 500 + ) r + GROUP BY display_name + ORDER BY rank DESC, display_name ASC LIMIT 50`, + searchTerm, searchTerm, p.searchIndex, searchTerm) + + // Execute query + rows, err := p.db.Query(queryCtx, sqlQuery) + if err != nil { + return nil, fmt.Errorf("query '%s' failed: %w", sqlQuery, err) + } + defer rows.Close() + + if queryCtx.Err() != nil { + return nil, queryCtx.Err() + } + + var suggestions []string + for rows.Next() { + var displayName, highlightedText string + var rank float64 + + // Scan all selected columns + if err := rows.Scan(&displayName, &rank, &highlightedText); err != nil { + return nil, err + } + + suggestions = append(suggestions, highlightedText) // or displayName, whichever you want to return + } + + return suggestions, nil +} diff --git a/internal/search/json.go b/internal/search/json.go new file mode 100644 index 0000000..3e054ae --- /dev/null +++ b/internal/search/json.go @@ -0,0 +1,51 @@ +package search + +import ( + stdjson "encoding/json" + "io" + "log" + "net/http" + "os" + "strconv" + + "github.com/PDOK/gomagpie/internal/engine" + perfjson "github.com/goccy/go-json" +) + +var ( + disableJSONPerfOptimization, _ = strconv.ParseBool(os.Getenv("DISABLE_JSON_PERF_OPTIMIZATION")) +) + +// serveJSON serves JSON *WITHOUT* OpenAPI validation by writing directly to the response output stream +func serveJSON(input any, contentType string, w http.ResponseWriter) { + w.Header().Set(engine.HeaderContentType, contentType) + + if err := getEncoder(w).Encode(input); err != nil { + handleJSONEncodingFailure(err, w) + return + } +} + +type jsonEncoder interface { + Encode(input any) error +} + +// Create JSONEncoder. Note escaping of '<', '>' and '&' is disabled (HTMLEscape is false). +// Especially the '&' is important since we use this character in the next/prev links. +func getEncoder(w io.Writer) jsonEncoder { + if disableJSONPerfOptimization { + // use Go stdlib JSON encoder + encoder := stdjson.NewEncoder(w) + encoder.SetEscapeHTML(false) + return encoder + } + // use ~7% overall faster 3rd party JSON encoder (in case of issues switch back to stdlib using env variable) + encoder := perfjson.NewEncoder(w) + encoder.SetEscapeHTML(false) + return encoder +} + +func handleJSONEncodingFailure(err error, w http.ResponseWriter) { + log.Printf("JSON encoding failed: %v", err) + engine.RenderProblem(engine.ProblemServerError, w, "Failed to write JSON response") +} diff --git a/internal/search/main.go b/internal/search/main.go index b4013bf..329124a 100644 --- a/internal/search/main.go +++ b/internal/search/main.go @@ -5,17 +5,24 @@ import ( "net/http" "net/url" "strings" + "time" "github.com/PDOK/gomagpie/internal/engine" + ds "github.com/PDOK/gomagpie/internal/search/datasources" + "github.com/PDOK/gomagpie/internal/search/datasources/postgres" ) +const timeout = time.Second * 15 + type Search struct { - engine *engine.Engine + engine *engine.Engine + datasource ds.Datasource } -func NewSearch(e *engine.Engine) *Search { +func NewSearch(e *engine.Engine, dbConn string, searchIndex string) *Search { s := &Search{ - engine: e, + engine: e, + datasource: newDatasource(e, dbConn, searchIndex), } e.Router.Get("/search/suggest", s.Suggest()) return s @@ -36,8 +43,17 @@ func (s *Search) Suggest() http.HandlerFunc { delete(params, "f") crs := params["crs"] delete(params, "crs") + limit := params["limit"] + delete(params, "limit") + + log.Printf("crs %s, limit %d, format %s, query %s, params %v", crs, limit, format, searchQuery, params) - log.Printf("crs %s, format %s, query %s, params %v", crs, format, searchQuery, params) + suggestions, err := s.datasource.Suggest(r.Context(), r.URL.Query().Get("q")) + if err != nil { + engine.RenderProblem(engine.ProblemServerError, w, err.Error()) + return + } + serveJSON(suggestions, engine.MediaTypeGeoJSON, w) } } @@ -66,3 +82,12 @@ func parseQueryParams(query url.Values) (map[string]any, error) { } return result, nil } + +func newDatasource(e *engine.Engine, dbConn string, searchIndex string) ds.Datasource { + datasource, err := postgres.NewPostgres(dbConn, timeout, searchIndex) + if err != nil { + log.Fatalf("failed to create datasource: %v", err) + } + e.RegisterShutdownHook(datasource.Close) + return datasource +} diff --git a/internal/search/main_test.go b/internal/search/main_test.go new file mode 100644 index 0000000..c885e58 --- /dev/null +++ b/internal/search/main_test.go @@ -0,0 +1,106 @@ +package search + +import ( + "context" + "fmt" + "log" + "os" + "path" + "path/filepath" + "runtime" + "testing" + "time" + + "github.com/PDOK/gomagpie/config" + "github.com/PDOK/gomagpie/internal/engine" + "github.com/PDOK/gomagpie/internal/etl" + "github.com/docker/go-connections/nat" + "github.com/stretchr/testify/assert" + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/wait" +) + +func init() { + // change working dir to root + _, filename, _, _ := runtime.Caller(0) + dir := path.Join(path.Dir(filename), "../../") + err := os.Chdir(dir) + if err != nil { + panic(err) + } +} + +func TestSuggest(t *testing.T) { + if testing.Short() { + t.Skip("Skipping integration test in short mode") + } + ctx := context.Background() + + // given + dbPort, postgisContainer, err := setupPostgis(ctx, t) + if err != nil { + t.Error(err) + } + defer terminateContainer(ctx, t, postgisContainer) + + dbConn := fmt.Sprintf("postgres://postgres:postgres@127.0.0.1:%d/%s?sslmode=disable", dbPort.Int(), "test_db") + + err = etl.CreateSearchIndex(dbConn, "search_index") + assert.NoError(t, err) + + cfg, err := config.NewConfig("internal/etl/testdata/config.yaml") + assert.NoError(t, err) + table := config.FeatureTable{Name: "addresses", FID: "fid", Geom: "geom"} + err = etl.ImportFile(cfg, "search_index", "internal/etl/testdata/addresses-crs84.gpkg", table, 1000, dbConn) + assert.NoError(t, err) + + // when/then + e, err := engine.NewEngine("internal/etl/testdata/config.yaml", false, false) + assert.NoError(t, err) + searchEndpoint := NewSearch(e, dbConn, "search_index") + searchEndpoint.Suggest() +} + +func setupPostgis(ctx context.Context, t *testing.T) (nat.Port, testcontainers.Container, error) { + req := testcontainers.ContainerRequest{ + Image: "docker.io/postgis/postgis:16-3.5-alpine", + Env: map[string]string{ + "POSTGRES_USER": "postgres", + "POSTGRES_PASSWORD": "postgres", + "POSTGRES_DB": "postgres", + }, + ExposedPorts: []string{"5432/tcp"}, + Cmd: []string{"postgres", "-c", "fsync=off"}, + WaitingFor: wait.ForLog("PostgreSQL init process complete; ready for start up."), + Files: []testcontainers.ContainerFile{ + { + HostFilePath: "tests/testdata/sql/init-db.sql", + ContainerFilePath: "/docker-entrypoint-initdb.d/" + filepath.Base("testdata/init-db.sql"), + FileMode: 0755, + }, + }, + } + + container, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: req, + Started: true, + }) + if err != nil { + t.Error(err) + } + port, err := container.MappedPort(ctx, "5432/tcp") + if err != nil { + t.Error(err) + } + + log.Println("Giving postgres a few extra seconds to fully start") + time.Sleep(2 * time.Second) + + return port, container, err +} + +func terminateContainer(ctx context.Context, t *testing.T, container testcontainers.Container) { + if err := container.Terminate(ctx); err != nil { + t.Fatalf("Failed to terminate container: %s", err.Error()) + } +} From 85e6f0d62cd9a4bc1e821df9ba815ea29429ab7b Mon Sep 17 00:00:00 2001 From: Richard Kettelerij Date: Tue, 3 Dec 2024 10:51:01 +0100 Subject: [PATCH 03/38] feat: add search endpoint - disable e2e test (already covered by testcontainers, maybe later we'll add more) --- .github/workflows/e2e-test.yml | 37 +++++++++++---------- internal/search/main.go | 13 +++----- internal/search/main_test.go | 60 ++++++++++++++++++++++++++++++++-- 3 files changed, 80 insertions(+), 30 deletions(-) diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index 0f89eae..a203015 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -19,21 +19,22 @@ jobs: push: false tags: gomagpie:local - - name: Start gomagpie test instance - run: | - docker run \ - -v `pwd`/examples:/examples \ - --rm --detach -p 8080:8080 \ - --name gomagpie \ - gomagpie:local start-service --config-file /examples/config.yaml - - # E2E Test - - name: E2E Test => Cypress - uses: cypress-io/github-action@v6 - with: - working-directory: ./tests - browser: chrome - - - name: Stop gomagpie test instance - run: | - docker stop gomagpie +# TODO build end-to-end test +# - name: Start gomagpie test instance +# run: | +# docker run \ +# -v `pwd`/examples:/examples \ +# --rm --detach -p 8080:8080 \ +# --name gomagpie \ +# gomagpie:local start-service some_index --config-file /examples/config.yaml +# +# # E2E Test +# - name: E2E Test => Cypress +# uses: cypress-io/github-action@v6 +# with: +# working-directory: ./tests +# browser: chrome +# +# - name: Stop gomagpie test instance +# run: | +# docker stop gomagpie diff --git a/internal/search/main.go b/internal/search/main.go index 329124a..28b4610 100644 --- a/internal/search/main.go +++ b/internal/search/main.go @@ -31,12 +31,7 @@ func NewSearch(e *engine.Engine, dbConn string, searchIndex string) *Search { // Suggest autosuggest locations based on user input func (s *Search) Suggest() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - params, err := parseQueryParams(r.URL.Query()) - if err != nil { - log.Printf("%v", err) - engine.RenderProblem(engine.ProblemBadRequest, w) - return - } + params := parseQueryParams(r.URL.Query()) searchQuery := params["q"] delete(params, "q") format := params["f"] @@ -48,7 +43,7 @@ func (s *Search) Suggest() http.HandlerFunc { log.Printf("crs %s, limit %d, format %s, query %s, params %v", crs, limit, format, searchQuery, params) - suggestions, err := s.datasource.Suggest(r.Context(), r.URL.Query().Get("q")) + suggestions, err := s.datasource.Suggest(r.Context(), searchQuery.(string)) // TODO check before casting if err != nil { engine.RenderProblem(engine.ProblemServerError, w, err.Error()) return @@ -57,7 +52,7 @@ func (s *Search) Suggest() http.HandlerFunc { } } -func parseQueryParams(query url.Values) (map[string]any, error) { +func parseQueryParams(query url.Values) map[string]any { result := make(map[string]any, len(query)) deepObjectParams := make(map[string]map[string]string) @@ -80,7 +75,7 @@ func parseQueryParams(query url.Values) (map[string]any, error) { for mainKey, subParams := range deepObjectParams { result[mainKey] = subParams } - return result, nil + return result } func newDatasource(e *engine.Engine, dbConn string, searchIndex string) ds.Datasource { diff --git a/internal/search/main_test.go b/internal/search/main_test.go index c885e58..aedf1fa 100644 --- a/internal/search/main_test.go +++ b/internal/search/main_test.go @@ -4,6 +4,9 @@ import ( "context" "fmt" "log" + "net" + "net/http" + "net/http/httptest" "os" "path" "path/filepath" @@ -15,6 +18,7 @@ import ( "github.com/PDOK/gomagpie/internal/engine" "github.com/PDOK/gomagpie/internal/etl" "github.com/docker/go-connections/nat" + "github.com/go-chi/chi/v5" "github.com/stretchr/testify/assert" "github.com/testcontainers/testcontainers-go" "github.com/testcontainers/testcontainers-go/wait" @@ -36,7 +40,7 @@ func TestSuggest(t *testing.T) { } ctx := context.Background() - // given + // given postgres available dbPort, postgisContainer, err := setupPostgis(ctx, t) if err != nil { t.Error(err) @@ -45,20 +49,42 @@ func TestSuggest(t *testing.T) { dbConn := fmt.Sprintf("postgres://postgres:postgres@127.0.0.1:%d/%s?sslmode=disable", dbPort.Int(), "test_db") + // given empty search index err = etl.CreateSearchIndex(dbConn, "search_index") assert.NoError(t, err) + // given imported gpkg cfg, err := config.NewConfig("internal/etl/testdata/config.yaml") assert.NoError(t, err) table := config.FeatureTable{Name: "addresses", FID: "fid", Geom: "geom"} err = etl.ImportFile(cfg, "search_index", "internal/etl/testdata/addresses-crs84.gpkg", table, 1000, dbConn) assert.NoError(t, err) - // when/then + // given engine available e, err := engine.NewEngine("internal/etl/testdata/config.yaml", false, false) assert.NoError(t, err) + + // given server available + rr, ts := createMockServer() + defer ts.Close() + + // when perform autosuggest searchEndpoint := NewSearch(e, dbConn, "search_index") - searchEndpoint.Suggest() + handler := searchEndpoint.Suggest() + req, err := createRequest("http://localhost:8080/search/suggest?q=\"Oudeschild\"") + assert.NoError(t, err) + handler.ServeHTTP(rr, req) + + // then + assert.Equal(t, 200, rr.Code) + assert.JSONEq(t, `[ + "Barentszstraat, 1792AD Oudeschild", + "Bolwerk, 1792AS Oudeschild", + "Commandeurssingel, 1792AV Oudeschild", + "De Houtmanstraat, 1792BC Oudeschild", + "De Ruyterstraat, 1792AP Oudeschild", + "De Wittstraat, 1792BP Oudeschild" + ]`, rr.Body.String()) } func setupPostgis(ctx context.Context, t *testing.T) (nat.Port, testcontainers.Container, error) { @@ -104,3 +130,31 @@ func terminateContainer(ctx context.Context, t *testing.T, container testcontain t.Fatalf("Failed to terminate container: %s", err.Error()) } } + +func createMockServer() (*httptest.ResponseRecorder, *httptest.Server) { + rr := httptest.NewRecorder() + l, err := net.Listen("tcp", "localhost:9095") + if err != nil { + log.Fatal(err) + } + ts := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + engine.SafeWrite(w.Write, []byte(r.URL.String())) + })) + err = ts.Listener.Close() + if err != nil { + log.Fatal(err) + } + ts.Listener = l + ts.Start() + return rr, ts +} + +func createRequest(url string) (*http.Request, error) { + req, err := http.NewRequest(http.MethodGet, url, nil) + if req == nil || err != nil { + return req, err + } + rctx := chi.NewRouteContext() + req = req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, rctx)) + return req, err +} From 927cd70e82ca6d6552a36506d3c1f4d6278abf9e Mon Sep 17 00:00:00 2001 From: Richard Kettelerij Date: Wed, 4 Dec 2024 13:02:42 +0100 Subject: [PATCH 04/38] feat: add search endpoint - expand query parameter parsing --- internal/search/datasources/datasource.go | 5 +- .../search/datasources/postgres/postgres.go | 19 +- internal/search/domain/spatialref.go | 12 ++ internal/search/main.go | 164 +++++++++++++++--- internal/search/main_test.go | 131 ++++++++++---- 5 files changed, 264 insertions(+), 67 deletions(-) create mode 100644 internal/search/domain/spatialref.go diff --git a/internal/search/datasources/datasource.go b/internal/search/datasources/datasource.go index 955d2bb..34fa7a4 100644 --- a/internal/search/datasources/datasource.go +++ b/internal/search/datasources/datasource.go @@ -2,12 +2,15 @@ package datasources import ( "context" + + "github.com/PDOK/gomagpie/internal/search/domain" ) // Datasource knows how make different kinds of queries/actions on the underlying actual datastore. // This abstraction allows the rest of the system to stay datastore agnostic. type Datasource interface { - Suggest(ctx context.Context, suggestForThis string) ([]string, error) + Suggest(ctx context.Context, searchTerm string, collections map[string]map[string]string, + srid domain.SRID, limit int) ([]string, error) // Close closes (connections to) the datasource gracefully Close() diff --git a/internal/search/datasources/postgres/postgres.go b/internal/search/datasources/postgres/postgres.go index 5830b27..4e81662 100644 --- a/internal/search/datasources/postgres/postgres.go +++ b/internal/search/datasources/postgres/postgres.go @@ -4,6 +4,7 @@ import ( "context" "fmt" + "github.com/PDOK/gomagpie/internal/search/domain" "github.com/jackc/pgx/v5" pgxgeom "github.com/twpayne/pgx-geom" @@ -36,17 +37,17 @@ func (p *Postgres) Close() { _ = p.db.Close(p.ctx) } -func (p *Postgres) Suggest(ctx context.Context, suggestForThis string) ([]string, error) { +func (p *Postgres) Suggest(ctx context.Context, searchTerm string, _ map[string]map[string]string, _ domain.SRID, limit int) ([]string, error) { queryCtx, cancel := context.WithTimeout(ctx, p.queryTimeout) defer cancel() // Prepare dynamic full-text search query // Split terms by spaces and append :* to each term - terms := strings.Fields(suggestForThis) + terms := strings.Fields(searchTerm) for i, term := range terms { terms[i] = term + ":*" } - searchTerm := strings.Join(terms, " & ") + searchTermForPostgres := strings.Join(terms, " & ") sqlQuery := fmt.Sprintf( `SELECT @@ -56,17 +57,15 @@ func (p *Postgres) Suggest(ctx context.Context, suggestForThis string) ([]string FROM ( SELECT display_name, ts_rank_cd(ts, to_tsquery('%[1]s'), 1) AS rank, - ts_headline('dutch', suggest, to_tsquery('%[2]s')) AS highlighted_text - FROM - %[3]s - WHERE ts @@ to_tsquery('%[4]s') LIMIT 500 + ts_headline('dutch', suggest, to_tsquery('%[1]s')) AS highlighted_text + FROM %[2]s + WHERE ts @@ to_tsquery('%[1]s') LIMIT 500 ) r GROUP BY display_name - ORDER BY rank DESC, display_name ASC LIMIT 50`, - searchTerm, searchTerm, p.searchIndex, searchTerm) + ORDER BY rank DESC, display_name ASC LIMIT $1`, searchTermForPostgres, p.searchIndex) // Execute query - rows, err := p.db.Query(queryCtx, sqlQuery) + rows, err := p.db.Query(queryCtx, sqlQuery, limit) if err != nil { return nil, fmt.Errorf("query '%s' failed: %w", sqlQuery, err) } diff --git a/internal/search/domain/spatialref.go b/internal/search/domain/spatialref.go new file mode 100644 index 0000000..5a666a6 --- /dev/null +++ b/internal/search/domain/spatialref.go @@ -0,0 +1,12 @@ +package domain + +const ( + CrsURIPrefix = "http://www.opengis.net/def/crs/" + UndefinedSRID = 0 + WGS84SRIDPostgis = 4326 // Use the same SRID as used during ETL + WGS84CodeOGC = "CRS84" +) + +// SRID Spatial Reference System Identifier: a unique value to unambiguously identify a spatial coordinate system. +// For example '28992' in https://www.opengis.net/def/crs/EPSG/0/28992 +type SRID int diff --git a/internal/search/main.go b/internal/search/main.go index 28b4610..ba63fca 100644 --- a/internal/search/main.go +++ b/internal/search/main.go @@ -1,18 +1,37 @@ package search import ( + "context" + "errors" + "fmt" "log" "net/http" "net/url" + "regexp" + "strconv" "strings" "time" "github.com/PDOK/gomagpie/internal/engine" ds "github.com/PDOK/gomagpie/internal/search/datasources" "github.com/PDOK/gomagpie/internal/search/datasources/postgres" + "github.com/PDOK/gomagpie/internal/search/domain" ) -const timeout = time.Second * 15 +const ( + queryParam = "q" + limitParam = "limit" + crsParam = "crs" + + limitDefault = 10 + limitMax = 50 + + timeout = time.Second * 15 +) + +var ( + deepObjectParamRegex = regexp.MustCompile(`\w+\[\w+]`) +) type Search struct { engine *engine.Engine @@ -31,31 +50,42 @@ func NewSearch(e *engine.Engine, dbConn string, searchIndex string) *Search { // Suggest autosuggest locations based on user input func (s *Search) Suggest() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - params := parseQueryParams(r.URL.Query()) - searchQuery := params["q"] - delete(params, "q") - format := params["f"] - delete(params, "f") - crs := params["crs"] - delete(params, "crs") - limit := params["limit"] - delete(params, "limit") - - log.Printf("crs %s, limit %d, format %s, query %s, params %v", crs, limit, format, searchQuery, params) - - suggestions, err := s.datasource.Suggest(r.Context(), searchQuery.(string)) // TODO check before casting + collections, searchTerm, outputSRID, limit, err := parseQueryParams(r.URL.Query()) if err != nil { - engine.RenderProblem(engine.ProblemServerError, w, err.Error()) + engine.RenderProblem(engine.ProblemBadRequest, w, err.Error()) + return + } + suggestions, err := s.datasource.Suggest(r.Context(), searchTerm, collections, outputSRID, limit) + if err != nil { + handleQueryError(w, err) + return + } + format := s.engine.CN.NegotiateFormat(r) + switch format { + case engine.FormatGeoJSON, engine.FormatJSON: + serveJSON(suggestions, engine.MediaTypeGeoJSON, w) + default: + engine.RenderProblem(engine.ProblemNotAcceptable, w, fmt.Sprintf("format '%s' is not supported", format)) return } - serveJSON(suggestions, engine.MediaTypeGeoJSON, w) } } -func parseQueryParams(query url.Values) map[string]any { - result := make(map[string]any, len(query)) +func parseQueryParams(query url.Values) (collectionsWithParams map[string]map[string]string, searchTerm string, outputSRID domain.SRID, limit int, err error) { + err = validateNoUnknownParams(query) + if err != nil { + return + } + searchTerm, searchTermErr := parseSearchTerm(query) + collectionsWithParams = parseCollectionDeepObjectParams(query) + outputSRID, outputSRIDErr := parseCrsToSRID(query, crsParam) + limit, limitErr := parseLimit(query) + err = errors.Join(searchTermErr, limitErr, outputSRIDErr) + return +} - deepObjectParams := make(map[string]map[string]string) +func parseCollectionDeepObjectParams(query url.Values) map[string]map[string]string { + deepObjectParams := make(map[string]map[string]string, len(query)) for key, values := range query { if strings.Contains(key, "[") { // Extract deepObject parameters @@ -67,15 +97,17 @@ func parseQueryParams(query url.Values) map[string]any { deepObjectParams[mainKey] = make(map[string]string) } deepObjectParams[mainKey][subKey] = values[0] - } else { - // Extract regular (flat) parameters - result[key] = values[0] } } - for mainKey, subParams := range deepObjectParams { - result[mainKey] = subParams + return deepObjectParams +} + +func parseSearchTerm(query url.Values) (searchTerm string, err error) { + searchTerm = query.Get(queryParam) + if searchTerm == "" { + err = fmt.Errorf("no search term provided, '%s' query parameter is required", queryParam) } - return result + return } func newDatasource(e *engine.Engine, dbConn string, searchIndex string) ds.Datasource { @@ -86,3 +118,85 @@ func newDatasource(e *engine.Engine, dbConn string, searchIndex string) ds.Datas e.RegisterShutdownHook(datasource.Close) return datasource } + +// log error, but send generic message to client to prevent possible information leakage from datasource +func handleQueryError(w http.ResponseWriter, err error) { + msg := "failed to fulfill search request" + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + // provide more context when user hits the query timeout + msg += ": querying took too long (timeout encountered). Simplify your request and try again, or contact support" + } + log.Printf("%s, error: %v\n", msg, err) + engine.RenderProblem(engine.ProblemServerError, w, msg) // don't include sensitive information in details msg +} + +// implements req 7.6 (https://docs.ogc.org/is/17-069r4/17-069r4.html#query_parameters) +func validateNoUnknownParams(query url.Values) error { + copyParams := clone(query) + copyParams.Del(engine.FormatParam) + copyParams.Del(queryParam) + copyParams.Del(limitParam) + copyParams.Del(crsParam) + for key := range query { + if deepObjectParamRegex.MatchString(key) { + copyParams.Del(key) + } + } + if len(copyParams) > 0 { + return fmt.Errorf("unknown query parameter(s) found: %v", copyParams.Encode()) + } + return nil +} + +func clone(params url.Values) url.Values { + copyParams := url.Values{} + for k, v := range params { + copyParams[k] = v + } + return copyParams +} + +func parseCrsToSRID(params url.Values, paramName string) (domain.SRID, error) { + param := params.Get(paramName) + if param == "" { + return domain.UndefinedSRID, nil + } + param = strings.TrimSpace(param) + if !strings.HasPrefix(param, domain.CrsURIPrefix) { + return domain.UndefinedSRID, fmt.Errorf("%s param should start with %s, got: %s", paramName, domain.CrsURIPrefix, param) + } + var srid domain.SRID + lastIndex := strings.LastIndex(param, "/") + if lastIndex != -1 { + crsCode := param[lastIndex+1:] + if crsCode == domain.WGS84CodeOGC { + return domain.WGS84SRIDPostgis, nil // CRS84 is WGS84, just like EPSG:4326 (only axis order differs but SRID is the same) + } + val, err := strconv.Atoi(crsCode) + if err != nil { + return 0, fmt.Errorf("expected numerical CRS code, received: %s", crsCode) + } + srid = domain.SRID(val) + } + return srid, nil +} + +func parseLimit(params url.Values) (int, error) { + limit := limitDefault + var err error + if params.Get(limitParam) != "" { + limit, err = strconv.Atoi(params.Get(limitParam)) + if err != nil { + err = errors.New("limit must be numeric") + } + // "If the value of the limit parameter is larger than the maximum value, this SHALL NOT result + // in an error (instead use the maximum as the parameter value)." + if limit > limitMax { + limit = limitMax + } + } + if limit < 0 { + err = errors.New("limit can't be negative") + } + return limit, err +} diff --git a/internal/search/main_test.go b/internal/search/main_test.go index aedf1fa..20cb570 100644 --- a/internal/search/main_test.go +++ b/internal/search/main_test.go @@ -24,6 +24,8 @@ import ( "github.com/testcontainers/testcontainers-go/wait" ) +const testSearchIndex = "search_index" + func init() { // change working dir to root _, filename, _, _ := runtime.Caller(0) @@ -40,7 +42,7 @@ func TestSuggest(t *testing.T) { } ctx := context.Background() - // given postgres available + // given available postgres dbPort, postgisContainer, err := setupPostgis(ctx, t) if err != nil { t.Error(err) @@ -49,42 +51,110 @@ func TestSuggest(t *testing.T) { dbConn := fmt.Sprintf("postgres://postgres:postgres@127.0.0.1:%d/%s?sslmode=disable", dbPort.Int(), "test_db") + // given available engine + eng, err := engine.NewEngine("internal/etl/testdata/config.yaml", false, false) + assert.NoError(t, err) + + // given search endpoint + searchEndpoint := NewSearch(eng, dbConn, testSearchIndex) + // given empty search index - err = etl.CreateSearchIndex(dbConn, "search_index") + err = etl.CreateSearchIndex(dbConn, testSearchIndex) assert.NoError(t, err) - // given imported gpkg + // given imported geopackage cfg, err := config.NewConfig("internal/etl/testdata/config.yaml") assert.NoError(t, err) table := config.FeatureTable{Name: "addresses", FID: "fid", Geom: "geom"} - err = etl.ImportFile(cfg, "search_index", "internal/etl/testdata/addresses-crs84.gpkg", table, 1000, dbConn) - assert.NoError(t, err) - - // given engine available - e, err := engine.NewEngine("internal/etl/testdata/config.yaml", false, false) + err = etl.ImportFile(cfg, testSearchIndex, "internal/etl/testdata/addresses-crs84.gpkg", table, 1000, dbConn) assert.NoError(t, err) - // given server available - rr, ts := createMockServer() - defer ts.Close() - - // when perform autosuggest - searchEndpoint := NewSearch(e, dbConn, "search_index") - handler := searchEndpoint.Suggest() - req, err := createRequest("http://localhost:8080/search/suggest?q=\"Oudeschild\"") - assert.NoError(t, err) - handler.ServeHTTP(rr, req) - - // then - assert.Equal(t, 200, rr.Code) - assert.JSONEq(t, `[ - "Barentszstraat, 1792AD Oudeschild", - "Bolwerk, 1792AS Oudeschild", - "Commandeurssingel, 1792AV Oudeschild", - "De Houtmanstraat, 1792BC Oudeschild", - "De Ruyterstraat, 1792AP Oudeschild", - "De Wittstraat, 1792BP Oudeschild" - ]`, rr.Body.String()) + // run test cases + type fields struct { + url string + } + type want struct { + body string + statusCode int + } + tests := []struct { + name string + fields fields + want want + }{ + { + name: "Suggest: Oudeschild", + fields: fields{ + url: "http://localhost:8080/search/suggest?q=\"Oudeschild\"&limit=50", + }, + want: want{ + body: `[ + "Barentszstraat, 1792AD Oudeschild", + "Bolwerk, 1792AS Oudeschild", + "Commandeurssingel, 1792AV Oudeschild", + "De Houtmanstraat, 1792BC Oudeschild", + "De Ruyterstraat, 1792AP Oudeschild", + "De Wittstraat, 1792BP Oudeschild" + ]`, + statusCode: http.StatusOK, + }, + }, + { + name: "Suggest: Den ", + fields: fields{ + url: "http://localhost:8080/search/suggest?q=\"Den\"&limit=50", + }, + want: want{ + body: `[ + "Abbewaal, 1791WZ Den Burg", + "Achterom, 1791AN Den Burg", + "Akenbuurt, 1791PJ Den Burg", + "Amaliaweg, 1797SW Den Hoorn", + "Bakkenweg, 1797RJ Den Hoorn", + "Beatrixlaan, 1791GE Den Burg", + "Ada van Hollandstraat, 1791DH Den Burg", + "Anne Frankstraat, 1791DT Den Burg" + ]`, + statusCode: http.StatusOK, + }, + }, + { + name: "Suggest: Den. With deepCopy params", + fields: fields{ + url: "http://localhost:8080/search/suggest?q=\"Den\"&weg[version]=2&weg[relevance]=0.8&adres[version]=1&adres[relevance]=1&limit=10&f=json&crs=http%3A%2F%2Fwww.opengis.net%2Fdef%2Fcrs%2FEPSG%2F0%2F28992", + }, + want: want{ + body: `[ + "Abbewaal, 1791WZ Den Burg", + "Achterom, 1791AN Den Burg", + "Akenbuurt, 1791PJ Den Burg", + "Amaliaweg, 1797SW Den Hoorn", + "Bakkenweg, 1797RJ Den Hoorn", + "Beatrixlaan, 1791GE Den Burg", + "Ada van Hollandstraat, 1791DH Den Burg", + "Anne Frankstraat, 1791DT Den Burg" + ]`, + statusCode: http.StatusOK, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // given available server + rr, ts := createMockServer() + defer ts.Close() + + // when + handler := searchEndpoint.Suggest() + req, err := createRequest(tt.fields.url) + assert.NoError(t, err) + handler.ServeHTTP(rr, req) + + // then + assert.Equal(t, tt.want.statusCode, rr.Code) + assert.JSONEq(t, tt.want.body, rr.Body.String()) + }) + } } func setupPostgis(ctx context.Context, t *testing.T) (nat.Port, testcontainers.Container, error) { @@ -154,7 +224,6 @@ func createRequest(url string) (*http.Request, error) { if req == nil || err != nil { return req, err } - rctx := chi.NewRouteContext() - req = req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, rctx)) + req = req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, chi.NewRouteContext())) return req, err } From 5598b60da0215f2f01eed9639805859d0f7b3d0b Mon Sep 17 00:00:00 2001 From: Richard Kettelerij Date: Mon, 9 Dec 2024 13:17:25 +0100 Subject: [PATCH 05/38] feat: add search endpoint - make args with defaults optional --- cmd/main.go | 46 +++++++++++++++++++++------------------------- 1 file changed, 21 insertions(+), 25 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 8e99737..e80ec3e 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -109,6 +109,7 @@ var ( dbNameFlag: &cli.StringFlag{ Name: dbNameFlag, Usage: "Connect to this database", + Value: "postgres", EnvVars: []string{strcase.ToScreamingSnake(dbNameFlag)}, }, dbSslModeFlag: &cli.StringFlag{ @@ -154,11 +155,10 @@ func main() { commonDBFlags[dbPasswordFlag], commonDBFlags[dbSslModeFlag], &cli.PathFlag{ - Name: searchIndexFlag, - EnvVars: []string{strcase.ToScreamingSnake(searchIndexFlag)}, - Usage: "Name of search index to use", - Required: true, - Value: "search_index", + Name: searchIndexFlag, + EnvVars: []string{strcase.ToScreamingSnake(searchIndexFlag)}, + Usage: "Name of search index to use", + Value: "search_index", }, }, Action: func(c *cli.Context) error { @@ -223,11 +223,10 @@ func main() { commonDBFlags[dbSslModeFlag], serviceFlags[configFileFlag], &cli.PathFlag{ - Name: searchIndexFlag, - EnvVars: []string{strcase.ToScreamingSnake(searchIndexFlag)}, - Usage: "Name of search index in which to import the given file", - Required: true, - Value: "search_index", + Name: searchIndexFlag, + EnvVars: []string{strcase.ToScreamingSnake(searchIndexFlag)}, + Usage: "Name of search index in which to import the given file", + Value: "search_index", }, &cli.PathFlag{ Name: fileFlag, @@ -236,18 +235,16 @@ func main() { Required: true, }, &cli.StringFlag{ - Name: featureTableFidFlag, - EnvVars: []string{strcase.ToScreamingSnake(featureTableFidFlag)}, - Usage: "Name of feature ID field in file", - Required: true, - Value: "fid", + Name: featureTableFidFlag, + EnvVars: []string{strcase.ToScreamingSnake(featureTableFidFlag)}, + Usage: "Name of feature ID field in file", + Value: "fid", }, &cli.StringFlag{ - Name: featureTableGeomFlag, - EnvVars: []string{strcase.ToScreamingSnake(featureTableGeomFlag)}, - Usage: "Name of geometry field in file", - Required: true, - Value: "geom", + Name: featureTableGeomFlag, + EnvVars: []string{strcase.ToScreamingSnake(featureTableGeomFlag)}, + Usage: "Name of geometry field in file", + Value: "geom", }, &cli.StringFlag{ Name: featureTableFlag, @@ -256,11 +253,10 @@ func main() { Required: true, }, &cli.IntFlag{ - Name: pageSizeFlag, - EnvVars: []string{strcase.ToScreamingSnake(pageSizeFlag)}, - Usage: "Page/batch size to use when extracting records from file", - Required: true, - Value: 10000, + Name: pageSizeFlag, + EnvVars: []string{strcase.ToScreamingSnake(pageSizeFlag)}, + Usage: "Page/batch size to use when extracting records from file", + Value: 10000, }, }, Action: func(c *cli.Context) error { From 1ec613bace1c876280c0fac4b70c371dd6abb415 Mon Sep 17 00:00:00 2001 From: Richard Kettelerij Date: Mon, 9 Dec 2024 13:18:03 +0100 Subject: [PATCH 06/38] feat: add search endpoint - return more columns in select in orde to build up GeoJSON response --- .../search/datasources/postgres/postgres.go | 45 ++++++++++--------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/internal/search/datasources/postgres/postgres.go b/internal/search/datasources/postgres/postgres.go index 4e81662..caa5bbe 100644 --- a/internal/search/datasources/postgres/postgres.go +++ b/internal/search/datasources/postgres/postgres.go @@ -6,6 +6,7 @@ import ( "github.com/PDOK/gomagpie/internal/search/domain" "github.com/jackc/pgx/v5" + pggeom "github.com/twpayne/go-geom" pgxgeom "github.com/twpayne/pgx-geom" "strings" @@ -49,20 +50,25 @@ func (p *Postgres) Suggest(ctx context.Context, searchTerm string, _ map[string] } searchTermForPostgres := strings.Join(terms, " & ") - sqlQuery := fmt.Sprintf( - `SELECT - r.display_name AS display_name, - max(r.rank) AS rank, - max(r.highlighted_text) AS highlighted_text - FROM ( - SELECT display_name, - ts_rank_cd(ts, to_tsquery('%[1]s'), 1) AS rank, - ts_headline('dutch', suggest, to_tsquery('%[1]s')) AS highlighted_text - FROM %[2]s - WHERE ts @@ to_tsquery('%[1]s') LIMIT 500 + sqlQuery := fmt.Sprintf(` + select r.display_name as display_name, + max(r.feature_id) as feature_id, + max(r.collection_id) as collection_id, + max(r.collection_version) as collection_version, + cast(max(r.bbox) as geometry) as bbox, + max(r.rank) as rank, + max(r.highlighted_text) as highlighted_text + from ( + select display_name, feature_id, collection_id, collection_version, bbox, + ts_rank_cd(ts, to_tsquery('%[1]s'), 1) as rank, + ts_headline('dutch', suggest, to_tsquery('%[1]s')) as highlighted_text + from %[2]s + where ts @@ to_tsquery('%[1]s') + limit 500 ) r - GROUP BY display_name - ORDER BY rank DESC, display_name ASC LIMIT $1`, searchTermForPostgres, p.searchIndex) + group by r.display_name + order by rank desc, display_name asc + limit $1`, searchTermForPostgres, p.searchIndex) // Execute query rows, err := p.db.Query(queryCtx, sqlQuery, limit) @@ -71,22 +77,17 @@ func (p *Postgres) Suggest(ctx context.Context, searchTerm string, _ map[string] } defer rows.Close() - if queryCtx.Err() != nil { - return nil, queryCtx.Err() - } - var suggestions []string for rows.Next() { - var displayName, highlightedText string + var displayName, highlightedText, featureID, collectionID, collectionVersion string var rank float64 + var bbox pggeom.Polygon - // Scan all selected columns - if err := rows.Scan(&displayName, &rank, &highlightedText); err != nil { + if err := rows.Scan(&displayName, &featureID, &collectionID, &collectionVersion, &bbox, &rank, &highlightedText); err != nil { return nil, err } - suggestions = append(suggestions, highlightedText) // or displayName, whichever you want to return } - return suggestions, nil + return suggestions, queryCtx.Err() } From 6d3da80abd8ff12fb1d989d533c6563d80775327 Mon Sep 17 00:00:00 2001 From: Richard Kettelerij Date: Mon, 9 Dec 2024 13:36:53 +0100 Subject: [PATCH 07/38] feat: add search endpoint - weird test failure, fixing --- internal/search/main_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/search/main_test.go b/internal/search/main_test.go index 20cb570..9cce78b 100644 --- a/internal/search/main_test.go +++ b/internal/search/main_test.go @@ -63,10 +63,10 @@ func TestSuggest(t *testing.T) { assert.NoError(t, err) // given imported geopackage - cfg, err := config.NewConfig("internal/etl/testdata/config.yaml") + conf, err := config.NewConfig("internal/etl/testdata/config.yaml") assert.NoError(t, err) table := config.FeatureTable{Name: "addresses", FID: "fid", Geom: "geom"} - err = etl.ImportFile(cfg, testSearchIndex, "internal/etl/testdata/addresses-crs84.gpkg", table, 1000, dbConn) + err = etl.ImportFile(conf, testSearchIndex, "internal/etl/testdata/addresses-crs84.gpkg", table, 1000, dbConn) assert.NoError(t, err) // run test cases From 100655f6027f20a474de0b3ab46157f57227963d Mon Sep 17 00:00:00 2001 From: Richard Kettelerij Date: Mon, 9 Dec 2024 21:24:57 +0100 Subject: [PATCH 08/38] feat: add search endpoint - return GeoJSON response --- internal/etl/etl.go | 4 +- internal/search/datasources/datasource.go | 3 +- .../search/datasources/postgres/postgres.go | 73 ++++++++++++------- internal/search/domain/geojson.go | 53 ++++++++++++++ internal/search/json.go | 15 ++++ internal/search/main.go | 4 +- internal/search/main_test.go | 42 ++++------- 7 files changed, 135 insertions(+), 59 deletions(-) create mode 100644 internal/search/domain/geojson.go diff --git a/internal/etl/etl.go b/internal/etl/etl.go index 372abdd..209c860 100644 --- a/internal/etl/etl.go +++ b/internal/etl/etl.go @@ -52,11 +52,11 @@ func CreateSearchIndex(dbConn string, searchIndex string) error { } // ImportFile import source data into target search index using extract-transform-load principle -func ImportFile(cfg *config.Config, searchIndex string, filePath string, table config.FeatureTable, +func ImportFile(conf *config.Config, searchIndex string, filePath string, table config.FeatureTable, pageSize int, dbConn string) error { log.Println("start importing") - collection, err := getCollectionForTable(cfg, table) + collection, err := getCollectionForTable(conf, table) if err != nil { return err } diff --git a/internal/search/datasources/datasource.go b/internal/search/datasources/datasource.go index 34fa7a4..80c7824 100644 --- a/internal/search/datasources/datasource.go +++ b/internal/search/datasources/datasource.go @@ -9,8 +9,7 @@ import ( // Datasource knows how make different kinds of queries/actions on the underlying actual datastore. // This abstraction allows the rest of the system to stay datastore agnostic. type Datasource interface { - Suggest(ctx context.Context, searchTerm string, collections map[string]map[string]string, - srid domain.SRID, limit int) ([]string, error) + Suggest(ctx context.Context, searchTerm string, collections map[string]map[string]string, srid domain.SRID, limit int) (*domain.FeatureCollection, error) // Close closes (connections to) the datasource gracefully Close() diff --git a/internal/search/datasources/postgres/postgres.go b/internal/search/datasources/postgres/postgres.go index caa5bbe..89c1ec9 100644 --- a/internal/search/datasources/postgres/postgres.go +++ b/internal/search/datasources/postgres/postgres.go @@ -3,10 +3,12 @@ package postgres import ( "context" "fmt" + "log" "github.com/PDOK/gomagpie/internal/search/domain" "github.com/jackc/pgx/v5" pggeom "github.com/twpayne/go-geom" + "github.com/twpayne/go-geom/encoding/geojson" pgxgeom "github.com/twpayne/pgx-geom" "strings" @@ -38,7 +40,7 @@ func (p *Postgres) Close() { _ = p.db.Close(p.ctx) } -func (p *Postgres) Suggest(ctx context.Context, searchTerm string, _ map[string]map[string]string, _ domain.SRID, limit int) ([]string, error) { +func (p *Postgres) Suggest(ctx context.Context, searchTerm string, collections map[string]map[string]string, srid domain.SRID, limit int) (*domain.FeatureCollection, error) { queryCtx, cancel := context.WithTimeout(ctx, p.queryTimeout) defer cancel() @@ -48,9 +50,51 @@ func (p *Postgres) Suggest(ctx context.Context, searchTerm string, _ map[string] for i, term := range terms { terms[i] = term + ":*" } - searchTermForPostgres := strings.Join(terms, " & ") + termsConcat := strings.Join(terms, " & ") + searchQuery := makeSearchQuery(termsConcat, p.searchIndex) - sqlQuery := fmt.Sprintf(` + // Execute search query + rows, err := p.db.Query(queryCtx, searchQuery, limit) + if err != nil { + return nil, fmt.Errorf("query '%s' failed: %w", searchQuery, err) + } + defer rows.Close() + + fc := domain.FeatureCollection{Features: make([]*domain.Feature, 0)} + for rows.Next() { + var displayName, highlightedText, featureID, collectionID, collectionVersion string + var rank float64 + var bbox pggeom.T + + if err := rows.Scan(&displayName, &featureID, &collectionID, &collectionVersion, + &bbox, &rank, &highlightedText); err != nil { + return nil, err + } + geojsonGeom, err := geojson.Encode(bbox) + if err != nil { + return nil, err + } + f := domain.Feature{ + ID: featureID, + Geometry: *geojsonGeom, + Properties: map[string]any{ + "collectionId": collectionID, + "collectionVersion": collectionVersion, + "displayName": displayName, + "highlight": highlightedText, + "href": "", // TODO add href + "score": rank, + }, + // TODO add href also to Links + } + log.Printf("collections %s, srid %v", collections, srid) //TODO use params + fc.Features = append(fc.Features, &f) + } + return &fc, queryCtx.Err() +} + +func makeSearchQuery(term string, index string) string { + return fmt.Sprintf(` select r.display_name as display_name, max(r.feature_id) as feature_id, max(r.collection_id) as collection_id, @@ -68,26 +112,5 @@ func (p *Postgres) Suggest(ctx context.Context, searchTerm string, _ map[string] ) r group by r.display_name order by rank desc, display_name asc - limit $1`, searchTermForPostgres, p.searchIndex) - - // Execute query - rows, err := p.db.Query(queryCtx, sqlQuery, limit) - if err != nil { - return nil, fmt.Errorf("query '%s' failed: %w", sqlQuery, err) - } - defer rows.Close() - - var suggestions []string - for rows.Next() { - var displayName, highlightedText, featureID, collectionID, collectionVersion string - var rank float64 - var bbox pggeom.Polygon - - if err := rows.Scan(&displayName, &featureID, &collectionID, &collectionVersion, &bbox, &rank, &highlightedText); err != nil { - return nil, err - } - suggestions = append(suggestions, highlightedText) // or displayName, whichever you want to return - } - - return suggestions, queryCtx.Err() + limit $1`, term, index) } diff --git a/internal/search/domain/geojson.go b/internal/search/domain/geojson.go new file mode 100644 index 0000000..e72d032 --- /dev/null +++ b/internal/search/domain/geojson.go @@ -0,0 +1,53 @@ +package domain + +import ( + "github.com/twpayne/go-geom/encoding/geojson" +) + +// featureCollectionType allows the GeoJSON type to be automatically set during json marshalling +type featureCollectionType struct{} + +func (fc *featureCollectionType) MarshalJSON() ([]byte, error) { + return []byte(`"FeatureCollection"`), nil +} + +// featureType allows the type for Feature to be automatically set during json Marshalling +type featureType struct{} + +func (ft *featureType) MarshalJSON() ([]byte, error) { + return []byte(`"Feature"`), nil +} + +// FeatureCollection is a GeoJSON FeatureCollection with extras such as links +// Note: fields in this struct are sorted for optimal memory usage (field alignment) +type FeatureCollection struct { + Type featureCollectionType `json:"type"` + Timestamp string `json:"timeStamp,omitempty"` + Links []Link `json:"links,omitempty"` + Features []*Feature `json:"features"` + NumberReturned int `json:"numberReturned"` +} + +// Feature is a GeoJSON Feature with extras such as links +// Note: fields in this struct are sorted for optimal memory usage (field alignment) +type Feature struct { + Type featureType `json:"type"` + Properties map[string]any `json:"properties"` + Geometry geojson.Geometry `json:"geometry"` + // We expect feature ids to be auto-incrementing integers (which is the default in geopackages) + // since we use it for cursor-based pagination. + ID string `json:"id"` + Links []Link `json:"links,omitempty"` +} + +// Link according to RFC 8288, https://datatracker.ietf.org/doc/html/rfc8288 +// Note: fields in this struct are sorted for optimal memory usage (field alignment) +type Link struct { + Rel string `json:"rel"` + Title string `json:"title,omitempty"` + Type string `json:"type,omitempty"` + Href string `json:"href"` + Hreflang string `json:"hreflang,omitempty"` + Length int64 `json:"length,omitempty"` + Templated bool `json:"templated,omitempty"` +} diff --git a/internal/search/json.go b/internal/search/json.go index 3e054ae..2eb5e11 100644 --- a/internal/search/json.go +++ b/internal/search/json.go @@ -7,15 +7,30 @@ import ( "net/http" "os" "strconv" + "time" "github.com/PDOK/gomagpie/internal/engine" + "github.com/PDOK/gomagpie/internal/search/domain" perfjson "github.com/goccy/go-json" ) var ( + now = time.Now // allow mocking disableJSONPerfOptimization, _ = strconv.ParseBool(os.Getenv("DISABLE_JSON_PERF_OPTIMIZATION")) ) +func featuresAsGeoJSON(w http.ResponseWriter, fc *domain.FeatureCollection) { + fc.Timestamp = now().Format(time.RFC3339) + // fc.Links = createFeatureCollectionLinks(engine.FormatGeoJSON, collectionID, cursor, featuresURL) // TODO add links + + // TODO add validation + // if jf.validateResponse { + // jf.serveAndValidateJSON(&fc, engine.MediaTypeGeoJSON, r, w) + // } else { + serveJSON(&fc, engine.MediaTypeGeoJSON, w) + // } +} + // serveJSON serves JSON *WITHOUT* OpenAPI validation by writing directly to the response output stream func serveJSON(input any, contentType string, w http.ResponseWriter) { w.Header().Set(engine.HeaderContentType, contentType) diff --git a/internal/search/main.go b/internal/search/main.go index ba63fca..b28ac5c 100644 --- a/internal/search/main.go +++ b/internal/search/main.go @@ -55,7 +55,7 @@ func (s *Search) Suggest() http.HandlerFunc { engine.RenderProblem(engine.ProblemBadRequest, w, err.Error()) return } - suggestions, err := s.datasource.Suggest(r.Context(), searchTerm, collections, outputSRID, limit) + fc, err := s.datasource.Suggest(r.Context(), searchTerm, collections, outputSRID, limit) if err != nil { handleQueryError(w, err) return @@ -63,7 +63,7 @@ func (s *Search) Suggest() http.HandlerFunc { format := s.engine.CN.NegotiateFormat(r) switch format { case engine.FormatGeoJSON, engine.FormatJSON: - serveJSON(suggestions, engine.MediaTypeGeoJSON, w) + featuresAsGeoJSON(w, fc) default: engine.RenderProblem(engine.ProblemNotAcceptable, w, fmt.Sprintf("format '%s' is not supported", format)) return diff --git a/internal/search/main_test.go b/internal/search/main_test.go index 9cce78b..c00a773 100644 --- a/internal/search/main_test.go +++ b/internal/search/main_test.go @@ -88,14 +88,9 @@ func TestSuggest(t *testing.T) { url: "http://localhost:8080/search/suggest?q=\"Oudeschild\"&limit=50", }, want: want{ - body: `[ - "Barentszstraat, 1792AD Oudeschild", - "Bolwerk, 1792AS Oudeschild", - "Commandeurssingel, 1792AV Oudeschild", - "De Houtmanstraat, 1792BC Oudeschild", - "De Ruyterstraat, 1792AP Oudeschild", - "De Wittstraat, 1792BP Oudeschild" - ]`, + body: ` +{"type":"FeatureCollection","timeStamp":"2000-01-01T00:00:00Z","features":[{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"Barentszstraat - Oudeschild","highlight":"Barentszstraat, 1792AD Oudeschild","href":"","score":0.14426951110363007},"geometry":{"type":"Polygon","coordinates":[[[4.748384044242354,52.93901709012591],[4.948384044242354,52.93901709012591],[4.948384044242354,53.13901709012591],[4.748384044242354,53.13901709012591],[4.748384044242354,52.93901709012591]]]},"id":"548"},{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"Bolwerk - Oudeschild","highlight":"Bolwerk, 1792AS Oudeschild","href":"","score":0.14426951110363007},"geometry":{"type":"Polygon","coordinates":[[[4.75002232386939,52.93847294238573],[4.95002232386939,52.93847294238573],[4.95002232386939,53.13847294238573],[4.75002232386939,53.13847294238573],[4.75002232386939,52.93847294238573]]]},"id":"1050"},{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"Commandeurssingel - Oudeschild","highlight":"Commandeurssingel, 1792AV Oudeschild","href":"","score":0.14426951110363007},"geometry":{"type":"Polygon","coordinates":[[[4.7451477245429015,52.93967814281323],[4.945147724542901,52.93967814281323],[4.945147724542901,53.13967814281323],[4.7451477245429015,53.13967814281323],[4.7451477245429015,52.93967814281323]]]},"id":"2725"},{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"De Houtmanstraat - Oudeschild","highlight":"De Houtmanstraat, 1792BC Oudeschild","href":"","score":0.12426698952913284},"geometry":{"type":"Polygon","coordinates":[[[4.748360166368449,52.93815392755542],[4.948360166368448,52.93815392755542],[4.948360166368448,53.13815392755542],[4.748360166368449,53.13815392755542],[4.748360166368449,52.93815392755542]]]},"id":"2921"},{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"De Ruyterstraat - Oudeschild","highlight":"De Ruyterstraat, 1792AP Oudeschild","href":"","score":0.12426698952913284},"geometry":{"type":"Polygon","coordinates":[[[4.747714279539418,52.93617309495475],[4.947714279539417,52.93617309495475],[4.947714279539417,53.136173094954756],[4.747714279539418,53.136173094954756],[4.747714279539418,52.93617309495475]]]},"id":"3049"},{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"De Wittstraat - Oudeschild","highlight":"De Wittstraat, 1792BP Oudeschild","href":"","score":0.12426698952913284},"geometry":{"type":"Polygon","coordinates":[[[4.745616492688666,52.93705261983951],[4.945616492688665,52.93705261983951],[4.945616492688665,53.137052619839515],[4.745616492688666,53.137052619839515],[4.745616492688666,52.93705261983951]]]},"id":"3041"}],"numberReturned":0} +`, statusCode: http.StatusOK, }, }, @@ -105,16 +100,9 @@ func TestSuggest(t *testing.T) { url: "http://localhost:8080/search/suggest?q=\"Den\"&limit=50", }, want: want{ - body: `[ - "Abbewaal, 1791WZ Den Burg", - "Achterom, 1791AN Den Burg", - "Akenbuurt, 1791PJ Den Burg", - "Amaliaweg, 1797SW Den Hoorn", - "Bakkenweg, 1797RJ Den Hoorn", - "Beatrixlaan, 1791GE Den Burg", - "Ada van Hollandstraat, 1791DH Den Burg", - "Anne Frankstraat, 1791DT Den Burg" - ]`, + body: ` +{"type":"FeatureCollection","timeStamp":"2000-01-01T00:00:00Z","features":[{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"Abbewaal - Den Burg","highlight":"Abbewaal, 1791WZ Den Burg","href":"","score":0.11162212491035461},"geometry":{"type":"Polygon","coordinates":[[[4.701721422439945,52.9619223105808],[4.901721422439945,52.9619223105808],[4.901721422439945,53.161922310580806],[4.701721422439945,53.161922310580806],[4.701721422439945,52.9619223105808]]]},"id":"99"},{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"Achterom - Den Burg","highlight":"Achterom, 1791AN Den Burg","href":"","score":0.11162212491035461},"geometry":{"type":"Polygon","coordinates":[[[4.699813158490893,52.95463219709524],[4.899813158490892,52.95463219709524],[4.899813158490892,53.154632197095246],[4.699813158490893,53.154632197095246],[4.699813158490893,52.95463219709524]]]},"id":"114"},{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"Akenbuurt - Den Burg","highlight":"Akenbuurt, 1791PJ Den Burg","href":"","score":0.11162212491035461},"geometry":{"type":"Polygon","coordinates":[[[4.680059099895046,52.95346592050607],[4.880059099895045,52.95346592050607],[4.880059099895045,53.15346592050607],[4.680059099895046,53.15346592050607],[4.680059099895046,52.95346592050607]]]},"id":"46"},{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"Amaliaweg - Den Hoorn","highlight":"Amaliaweg, 1797SW Den Hoorn","href":"","score":0.11162212491035461},"geometry":{"type":"Polygon","coordinates":[[[4.68911630577304,52.92449928128154],[4.889116305773039,52.92449928128154],[4.889116305773039,53.124499281281544],[4.68911630577304,53.124499281281544],[4.68911630577304,52.92449928128154]]]},"id":"50"},{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"Bakkenweg - Den Hoorn","highlight":"Bakkenweg, 1797RJ Den Hoorn","href":"","score":0.11162212491035461},"geometry":{"type":"Polygon","coordinates":[[[4.6548723261037095,52.94811743920973],[4.854872326103709,52.94811743920973],[4.854872326103709,53.148117439209734],[4.6548723261037095,53.148117439209734],[4.6548723261037095,52.94811743920973]]]},"id":"520"},{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"Beatrixlaan - Den Burg","highlight":"Beatrixlaan, 1791GE Den Burg","href":"","score":0.11162212491035461},"geometry":{"type":"Polygon","coordinates":[[[4.690892824472019,52.95558352001795],[4.890892824472019,52.95558352001795],[4.890892824472019,53.155583520017956],[4.690892824472019,53.155583520017956],[4.690892824472019,52.95558352001795]]]},"id":"591"},{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"Ada van Hollandstraat - Den Burg","highlight":"Ada van Hollandstraat, 1791DH Den Burg","href":"","score":0.09617967158555984},"geometry":{"type":"Polygon","coordinates":[[[4.696235388824104,52.95196001510249],[4.8962353888241035,52.95196001510249],[4.8962353888241035,53.151960015102496],[4.696235388824104,53.151960015102496],[4.696235388824104,52.95196001510249]]]},"id":"26"},{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"Anne Frankstraat - Den Burg","highlight":"Anne Frankstraat, 1791DT Den Burg","href":"","score":0.09617967158555984},"geometry":{"type":"Polygon","coordinates":[[[4.692873779103581,52.950932925919574],[4.892873779103581,52.950932925919574],[4.892873779103581,53.15093292591958],[4.692873779103581,53.15093292591958],[4.692873779103581,52.950932925919574]]]},"id":"474"}],"numberReturned":0} +`, statusCode: http.StatusOK, }, }, @@ -124,22 +112,18 @@ func TestSuggest(t *testing.T) { url: "http://localhost:8080/search/suggest?q=\"Den\"&weg[version]=2&weg[relevance]=0.8&adres[version]=1&adres[relevance]=1&limit=10&f=json&crs=http%3A%2F%2Fwww.opengis.net%2Fdef%2Fcrs%2FEPSG%2F0%2F28992", }, want: want{ - body: `[ - "Abbewaal, 1791WZ Den Burg", - "Achterom, 1791AN Den Burg", - "Akenbuurt, 1791PJ Den Burg", - "Amaliaweg, 1797SW Den Hoorn", - "Bakkenweg, 1797RJ Den Hoorn", - "Beatrixlaan, 1791GE Den Burg", - "Ada van Hollandstraat, 1791DH Den Burg", - "Anne Frankstraat, 1791DT Den Burg" - ]`, + body: ` +{"type":"FeatureCollection","timeStamp":"2000-01-01T00:00:00Z","features":[{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"Abbewaal - Den Burg","highlight":"Abbewaal, 1791WZ Den Burg","href":"","score":0.11162212491035461},"geometry":{"type":"Polygon","coordinates":[[[4.701721422439945,52.9619223105808],[4.901721422439945,52.9619223105808],[4.901721422439945,53.161922310580806],[4.701721422439945,53.161922310580806],[4.701721422439945,52.9619223105808]]]},"id":"99"},{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"Achterom - Den Burg","highlight":"Achterom, 1791AN Den Burg","href":"","score":0.11162212491035461},"geometry":{"type":"Polygon","coordinates":[[[4.699813158490893,52.95463219709524],[4.899813158490892,52.95463219709524],[4.899813158490892,53.154632197095246],[4.699813158490893,53.154632197095246],[4.699813158490893,52.95463219709524]]]},"id":"114"},{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"Akenbuurt - Den Burg","highlight":"Akenbuurt, 1791PJ Den Burg","href":"","score":0.11162212491035461},"geometry":{"type":"Polygon","coordinates":[[[4.680059099895046,52.95346592050607],[4.880059099895045,52.95346592050607],[4.880059099895045,53.15346592050607],[4.680059099895046,53.15346592050607],[4.680059099895046,52.95346592050607]]]},"id":"46"},{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"Amaliaweg - Den Hoorn","highlight":"Amaliaweg, 1797SW Den Hoorn","href":"","score":0.11162212491035461},"geometry":{"type":"Polygon","coordinates":[[[4.68911630577304,52.92449928128154],[4.889116305773039,52.92449928128154],[4.889116305773039,53.124499281281544],[4.68911630577304,53.124499281281544],[4.68911630577304,52.92449928128154]]]},"id":"50"},{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"Bakkenweg - Den Hoorn","highlight":"Bakkenweg, 1797RJ Den Hoorn","href":"","score":0.11162212491035461},"geometry":{"type":"Polygon","coordinates":[[[4.6548723261037095,52.94811743920973],[4.854872326103709,52.94811743920973],[4.854872326103709,53.148117439209734],[4.6548723261037095,53.148117439209734],[4.6548723261037095,52.94811743920973]]]},"id":"520"},{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"Beatrixlaan - Den Burg","highlight":"Beatrixlaan, 1791GE Den Burg","href":"","score":0.11162212491035461},"geometry":{"type":"Polygon","coordinates":[[[4.690892824472019,52.95558352001795],[4.890892824472019,52.95558352001795],[4.890892824472019,53.155583520017956],[4.690892824472019,53.155583520017956],[4.690892824472019,52.95558352001795]]]},"id":"591"},{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"Ada van Hollandstraat - Den Burg","highlight":"Ada van Hollandstraat, 1791DH Den Burg","href":"","score":0.09617967158555984},"geometry":{"type":"Polygon","coordinates":[[[4.696235388824104,52.95196001510249],[4.8962353888241035,52.95196001510249],[4.8962353888241035,53.151960015102496],[4.696235388824104,53.151960015102496],[4.696235388824104,52.95196001510249]]]},"id":"26"},{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"Anne Frankstraat - Den Burg","highlight":"Anne Frankstraat, 1791DT Den Burg","href":"","score":0.09617967158555984},"geometry":{"type":"Polygon","coordinates":[[[4.692873779103581,52.950932925919574],[4.892873779103581,52.950932925919574],[4.892873779103581,53.15093292591958],[4.692873779103581,53.15093292591958],[4.692873779103581,52.950932925919574]]]},"id":"474"}],"numberReturned":0} +`, statusCode: http.StatusOK, }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + // mock time + now = func() time.Time { return time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC) } + // given available server rr, ts := createMockServer() defer ts.Close() @@ -152,6 +136,8 @@ func TestSuggest(t *testing.T) { // then assert.Equal(t, tt.want.statusCode, rr.Code) + + log.Printf("============ ACTUAL:\n %s", rr.Body.String()) assert.JSONEq(t, tt.want.body, rr.Body.String()) }) } From a06da94fd4bb4041f0ccf025903b7b0007053987 Mon Sep 17 00:00:00 2001 From: Richard Kettelerij Date: Mon, 9 Dec 2024 21:38:18 +0100 Subject: [PATCH 09/38] feat: add search endpoint - sync with master --- internal/etl/etl.go | 2 +- internal/search/main_test.go | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/internal/etl/etl.go b/internal/etl/etl.go index 08b9403..ae59dcd 100644 --- a/internal/etl/etl.go +++ b/internal/etl/etl.go @@ -110,7 +110,7 @@ func newSourceToExtract(filePath string) (Extract, error) { func newTargetToLoad(dbConn string) (Load, error) { if strings.HasPrefix(dbConn, "postgres:") { - return load.NewPostgis(dbConn) + return load.NewPostgres(dbConn) } // add new targets here (elasticsearch, solr, etc) return nil, fmt.Errorf("unsupported target database connection: %s", dbConn) diff --git a/internal/search/main_test.go b/internal/search/main_test.go index c00a773..3ddf2de 100644 --- a/internal/search/main_test.go +++ b/internal/search/main_test.go @@ -65,8 +65,9 @@ func TestSuggest(t *testing.T) { // given imported geopackage conf, err := config.NewConfig("internal/etl/testdata/config.yaml") assert.NoError(t, err) + collection := config.CollectionByID(conf, "addresses") table := config.FeatureTable{Name: "addresses", FID: "fid", Geom: "geom"} - err = etl.ImportFile(conf, testSearchIndex, "internal/etl/testdata/addresses-crs84.gpkg", table, 1000, dbConn) + err = etl.ImportFile(*collection, testSearchIndex, "internal/etl/testdata/addresses-crs84.gpkg", table, 1000, dbConn) assert.NoError(t, err) // run test cases From 09fd6ac892266275629b5100584c8f0de9ef8b47 Mon Sep 17 00:00:00 2001 From: Richard Kettelerij Date: Mon, 9 Dec 2024 21:44:59 +0100 Subject: [PATCH 10/38] feat: add search endpoint - linting --- internal/search/datasources/postgres/postgres.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/search/datasources/postgres/postgres.go b/internal/search/datasources/postgres/postgres.go index 89c1ec9..d6dbd2f 100644 --- a/internal/search/datasources/postgres/postgres.go +++ b/internal/search/datasources/postgres/postgres.go @@ -87,13 +87,14 @@ func (p *Postgres) Suggest(ctx context.Context, searchTerm string, collections m }, // TODO add href also to Links } - log.Printf("collections %s, srid %v", collections, srid) //TODO use params + log.Printf("collections %s, srid %v", collections, srid) // TODO use params fc.Features = append(fc.Features, &f) } return &fc, queryCtx.Err() } func makeSearchQuery(term string, index string) string { + // language=postgresql return fmt.Sprintf(` select r.display_name as display_name, max(r.feature_id) as feature_id, From 50ec5eefc9f132a9a6e833a6541ab923b4789262 Mon Sep 17 00:00:00 2001 From: Richard Kettelerij Date: Mon, 9 Dec 2024 22:06:02 +0100 Subject: [PATCH 11/38] feat: add search endpoint - add numberReturned --- internal/search/datasources/postgres/postgres.go | 5 ++++- internal/search/main_test.go | 6 +++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/internal/search/datasources/postgres/postgres.go b/internal/search/datasources/postgres/postgres.go index d6dbd2f..bc777d1 100644 --- a/internal/search/datasources/postgres/postgres.go +++ b/internal/search/datasources/postgres/postgres.go @@ -40,7 +40,9 @@ func (p *Postgres) Close() { _ = p.db.Close(p.ctx) } -func (p *Postgres) Suggest(ctx context.Context, searchTerm string, collections map[string]map[string]string, srid domain.SRID, limit int) (*domain.FeatureCollection, error) { +func (p *Postgres) Suggest(ctx context.Context, searchTerm string, collections map[string]map[string]string, + srid domain.SRID, limit int) (*domain.FeatureCollection, error) { + queryCtx, cancel := context.WithTimeout(ctx, p.queryTimeout) defer cancel() @@ -89,6 +91,7 @@ func (p *Postgres) Suggest(ctx context.Context, searchTerm string, collections m } log.Printf("collections %s, srid %v", collections, srid) // TODO use params fc.Features = append(fc.Features, &f) + fc.NumberReturned = len(fc.Features) } return &fc, queryCtx.Err() } diff --git a/internal/search/main_test.go b/internal/search/main_test.go index 3ddf2de..3abf0e2 100644 --- a/internal/search/main_test.go +++ b/internal/search/main_test.go @@ -90,7 +90,7 @@ func TestSuggest(t *testing.T) { }, want: want{ body: ` -{"type":"FeatureCollection","timeStamp":"2000-01-01T00:00:00Z","features":[{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"Barentszstraat - Oudeschild","highlight":"Barentszstraat, 1792AD Oudeschild","href":"","score":0.14426951110363007},"geometry":{"type":"Polygon","coordinates":[[[4.748384044242354,52.93901709012591],[4.948384044242354,52.93901709012591],[4.948384044242354,53.13901709012591],[4.748384044242354,53.13901709012591],[4.748384044242354,52.93901709012591]]]},"id":"548"},{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"Bolwerk - Oudeschild","highlight":"Bolwerk, 1792AS Oudeschild","href":"","score":0.14426951110363007},"geometry":{"type":"Polygon","coordinates":[[[4.75002232386939,52.93847294238573],[4.95002232386939,52.93847294238573],[4.95002232386939,53.13847294238573],[4.75002232386939,53.13847294238573],[4.75002232386939,52.93847294238573]]]},"id":"1050"},{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"Commandeurssingel - Oudeschild","highlight":"Commandeurssingel, 1792AV Oudeschild","href":"","score":0.14426951110363007},"geometry":{"type":"Polygon","coordinates":[[[4.7451477245429015,52.93967814281323],[4.945147724542901,52.93967814281323],[4.945147724542901,53.13967814281323],[4.7451477245429015,53.13967814281323],[4.7451477245429015,52.93967814281323]]]},"id":"2725"},{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"De Houtmanstraat - Oudeschild","highlight":"De Houtmanstraat, 1792BC Oudeschild","href":"","score":0.12426698952913284},"geometry":{"type":"Polygon","coordinates":[[[4.748360166368449,52.93815392755542],[4.948360166368448,52.93815392755542],[4.948360166368448,53.13815392755542],[4.748360166368449,53.13815392755542],[4.748360166368449,52.93815392755542]]]},"id":"2921"},{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"De Ruyterstraat - Oudeschild","highlight":"De Ruyterstraat, 1792AP Oudeschild","href":"","score":0.12426698952913284},"geometry":{"type":"Polygon","coordinates":[[[4.747714279539418,52.93617309495475],[4.947714279539417,52.93617309495475],[4.947714279539417,53.136173094954756],[4.747714279539418,53.136173094954756],[4.747714279539418,52.93617309495475]]]},"id":"3049"},{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"De Wittstraat - Oudeschild","highlight":"De Wittstraat, 1792BP Oudeschild","href":"","score":0.12426698952913284},"geometry":{"type":"Polygon","coordinates":[[[4.745616492688666,52.93705261983951],[4.945616492688665,52.93705261983951],[4.945616492688665,53.137052619839515],[4.745616492688666,53.137052619839515],[4.745616492688666,52.93705261983951]]]},"id":"3041"}],"numberReturned":0} +{"type":"FeatureCollection","timeStamp":"2000-01-01T00:00:00Z","features":[{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"Barentszstraat - Oudeschild","highlight":"Barentszstraat, 1792AD Oudeschild","href":"","score":0.14426951110363007},"geometry":{"type":"Polygon","coordinates":[[[4.748384044242354,52.93901709012591],[4.948384044242354,52.93901709012591],[4.948384044242354,53.13901709012591],[4.748384044242354,53.13901709012591],[4.748384044242354,52.93901709012591]]]},"id":"548"},{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"Bolwerk - Oudeschild","highlight":"Bolwerk, 1792AS Oudeschild","href":"","score":0.14426951110363007},"geometry":{"type":"Polygon","coordinates":[[[4.75002232386939,52.93847294238573],[4.95002232386939,52.93847294238573],[4.95002232386939,53.13847294238573],[4.75002232386939,53.13847294238573],[4.75002232386939,52.93847294238573]]]},"id":"1050"},{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"Commandeurssingel - Oudeschild","highlight":"Commandeurssingel, 1792AV Oudeschild","href":"","score":0.14426951110363007},"geometry":{"type":"Polygon","coordinates":[[[4.7451477245429015,52.93967814281323],[4.945147724542901,52.93967814281323],[4.945147724542901,53.13967814281323],[4.7451477245429015,53.13967814281323],[4.7451477245429015,52.93967814281323]]]},"id":"2725"},{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"De Houtmanstraat - Oudeschild","highlight":"De Houtmanstraat, 1792BC Oudeschild","href":"","score":0.12426698952913284},"geometry":{"type":"Polygon","coordinates":[[[4.748360166368449,52.93815392755542],[4.948360166368448,52.93815392755542],[4.948360166368448,53.13815392755542],[4.748360166368449,53.13815392755542],[4.748360166368449,52.93815392755542]]]},"id":"2921"},{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"De Ruyterstraat - Oudeschild","highlight":"De Ruyterstraat, 1792AP Oudeschild","href":"","score":0.12426698952913284},"geometry":{"type":"Polygon","coordinates":[[[4.747714279539418,52.93617309495475],[4.947714279539417,52.93617309495475],[4.947714279539417,53.136173094954756],[4.747714279539418,53.136173094954756],[4.747714279539418,52.93617309495475]]]},"id":"3049"},{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"De Wittstraat - Oudeschild","highlight":"De Wittstraat, 1792BP Oudeschild","href":"","score":0.12426698952913284},"geometry":{"type":"Polygon","coordinates":[[[4.745616492688666,52.93705261983951],[4.945616492688665,52.93705261983951],[4.945616492688665,53.137052619839515],[4.745616492688666,53.137052619839515],[4.745616492688666,52.93705261983951]]]},"id":"3041"}],"numberReturned":6} `, statusCode: http.StatusOK, }, @@ -102,7 +102,7 @@ func TestSuggest(t *testing.T) { }, want: want{ body: ` -{"type":"FeatureCollection","timeStamp":"2000-01-01T00:00:00Z","features":[{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"Abbewaal - Den Burg","highlight":"Abbewaal, 1791WZ Den Burg","href":"","score":0.11162212491035461},"geometry":{"type":"Polygon","coordinates":[[[4.701721422439945,52.9619223105808],[4.901721422439945,52.9619223105808],[4.901721422439945,53.161922310580806],[4.701721422439945,53.161922310580806],[4.701721422439945,52.9619223105808]]]},"id":"99"},{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"Achterom - Den Burg","highlight":"Achterom, 1791AN Den Burg","href":"","score":0.11162212491035461},"geometry":{"type":"Polygon","coordinates":[[[4.699813158490893,52.95463219709524],[4.899813158490892,52.95463219709524],[4.899813158490892,53.154632197095246],[4.699813158490893,53.154632197095246],[4.699813158490893,52.95463219709524]]]},"id":"114"},{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"Akenbuurt - Den Burg","highlight":"Akenbuurt, 1791PJ Den Burg","href":"","score":0.11162212491035461},"geometry":{"type":"Polygon","coordinates":[[[4.680059099895046,52.95346592050607],[4.880059099895045,52.95346592050607],[4.880059099895045,53.15346592050607],[4.680059099895046,53.15346592050607],[4.680059099895046,52.95346592050607]]]},"id":"46"},{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"Amaliaweg - Den Hoorn","highlight":"Amaliaweg, 1797SW Den Hoorn","href":"","score":0.11162212491035461},"geometry":{"type":"Polygon","coordinates":[[[4.68911630577304,52.92449928128154],[4.889116305773039,52.92449928128154],[4.889116305773039,53.124499281281544],[4.68911630577304,53.124499281281544],[4.68911630577304,52.92449928128154]]]},"id":"50"},{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"Bakkenweg - Den Hoorn","highlight":"Bakkenweg, 1797RJ Den Hoorn","href":"","score":0.11162212491035461},"geometry":{"type":"Polygon","coordinates":[[[4.6548723261037095,52.94811743920973],[4.854872326103709,52.94811743920973],[4.854872326103709,53.148117439209734],[4.6548723261037095,53.148117439209734],[4.6548723261037095,52.94811743920973]]]},"id":"520"},{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"Beatrixlaan - Den Burg","highlight":"Beatrixlaan, 1791GE Den Burg","href":"","score":0.11162212491035461},"geometry":{"type":"Polygon","coordinates":[[[4.690892824472019,52.95558352001795],[4.890892824472019,52.95558352001795],[4.890892824472019,53.155583520017956],[4.690892824472019,53.155583520017956],[4.690892824472019,52.95558352001795]]]},"id":"591"},{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"Ada van Hollandstraat - Den Burg","highlight":"Ada van Hollandstraat, 1791DH Den Burg","href":"","score":0.09617967158555984},"geometry":{"type":"Polygon","coordinates":[[[4.696235388824104,52.95196001510249],[4.8962353888241035,52.95196001510249],[4.8962353888241035,53.151960015102496],[4.696235388824104,53.151960015102496],[4.696235388824104,52.95196001510249]]]},"id":"26"},{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"Anne Frankstraat - Den Burg","highlight":"Anne Frankstraat, 1791DT Den Burg","href":"","score":0.09617967158555984},"geometry":{"type":"Polygon","coordinates":[[[4.692873779103581,52.950932925919574],[4.892873779103581,52.950932925919574],[4.892873779103581,53.15093292591958],[4.692873779103581,53.15093292591958],[4.692873779103581,52.950932925919574]]]},"id":"474"}],"numberReturned":0} +{"type":"FeatureCollection","timeStamp":"2000-01-01T00:00:00Z","features":[{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"Abbewaal - Den Burg","highlight":"Abbewaal, 1791WZ Den Burg","href":"","score":0.11162212491035461},"geometry":{"type":"Polygon","coordinates":[[[4.701721422439945,52.9619223105808],[4.901721422439945,52.9619223105808],[4.901721422439945,53.161922310580806],[4.701721422439945,53.161922310580806],[4.701721422439945,52.9619223105808]]]},"id":"99"},{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"Achterom - Den Burg","highlight":"Achterom, 1791AN Den Burg","href":"","score":0.11162212491035461},"geometry":{"type":"Polygon","coordinates":[[[4.699813158490893,52.95463219709524],[4.899813158490892,52.95463219709524],[4.899813158490892,53.154632197095246],[4.699813158490893,53.154632197095246],[4.699813158490893,52.95463219709524]]]},"id":"114"},{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"Akenbuurt - Den Burg","highlight":"Akenbuurt, 1791PJ Den Burg","href":"","score":0.11162212491035461},"geometry":{"type":"Polygon","coordinates":[[[4.680059099895046,52.95346592050607],[4.880059099895045,52.95346592050607],[4.880059099895045,53.15346592050607],[4.680059099895046,53.15346592050607],[4.680059099895046,52.95346592050607]]]},"id":"46"},{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"Amaliaweg - Den Hoorn","highlight":"Amaliaweg, 1797SW Den Hoorn","href":"","score":0.11162212491035461},"geometry":{"type":"Polygon","coordinates":[[[4.68911630577304,52.92449928128154],[4.889116305773039,52.92449928128154],[4.889116305773039,53.124499281281544],[4.68911630577304,53.124499281281544],[4.68911630577304,52.92449928128154]]]},"id":"50"},{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"Bakkenweg - Den Hoorn","highlight":"Bakkenweg, 1797RJ Den Hoorn","href":"","score":0.11162212491035461},"geometry":{"type":"Polygon","coordinates":[[[4.6548723261037095,52.94811743920973],[4.854872326103709,52.94811743920973],[4.854872326103709,53.148117439209734],[4.6548723261037095,53.148117439209734],[4.6548723261037095,52.94811743920973]]]},"id":"520"},{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"Beatrixlaan - Den Burg","highlight":"Beatrixlaan, 1791GE Den Burg","href":"","score":0.11162212491035461},"geometry":{"type":"Polygon","coordinates":[[[4.690892824472019,52.95558352001795],[4.890892824472019,52.95558352001795],[4.890892824472019,53.155583520017956],[4.690892824472019,53.155583520017956],[4.690892824472019,52.95558352001795]]]},"id":"591"},{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"Ada van Hollandstraat - Den Burg","highlight":"Ada van Hollandstraat, 1791DH Den Burg","href":"","score":0.09617967158555984},"geometry":{"type":"Polygon","coordinates":[[[4.696235388824104,52.95196001510249],[4.8962353888241035,52.95196001510249],[4.8962353888241035,53.151960015102496],[4.696235388824104,53.151960015102496],[4.696235388824104,52.95196001510249]]]},"id":"26"},{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"Anne Frankstraat - Den Burg","highlight":"Anne Frankstraat, 1791DT Den Burg","href":"","score":0.09617967158555984},"geometry":{"type":"Polygon","coordinates":[[[4.692873779103581,52.950932925919574],[4.892873779103581,52.950932925919574],[4.892873779103581,53.15093292591958],[4.692873779103581,53.15093292591958],[4.692873779103581,52.950932925919574]]]},"id":"474"}],"numberReturned":8} `, statusCode: http.StatusOK, }, @@ -114,7 +114,7 @@ func TestSuggest(t *testing.T) { }, want: want{ body: ` -{"type":"FeatureCollection","timeStamp":"2000-01-01T00:00:00Z","features":[{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"Abbewaal - Den Burg","highlight":"Abbewaal, 1791WZ Den Burg","href":"","score":0.11162212491035461},"geometry":{"type":"Polygon","coordinates":[[[4.701721422439945,52.9619223105808],[4.901721422439945,52.9619223105808],[4.901721422439945,53.161922310580806],[4.701721422439945,53.161922310580806],[4.701721422439945,52.9619223105808]]]},"id":"99"},{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"Achterom - Den Burg","highlight":"Achterom, 1791AN Den Burg","href":"","score":0.11162212491035461},"geometry":{"type":"Polygon","coordinates":[[[4.699813158490893,52.95463219709524],[4.899813158490892,52.95463219709524],[4.899813158490892,53.154632197095246],[4.699813158490893,53.154632197095246],[4.699813158490893,52.95463219709524]]]},"id":"114"},{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"Akenbuurt - Den Burg","highlight":"Akenbuurt, 1791PJ Den Burg","href":"","score":0.11162212491035461},"geometry":{"type":"Polygon","coordinates":[[[4.680059099895046,52.95346592050607],[4.880059099895045,52.95346592050607],[4.880059099895045,53.15346592050607],[4.680059099895046,53.15346592050607],[4.680059099895046,52.95346592050607]]]},"id":"46"},{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"Amaliaweg - Den Hoorn","highlight":"Amaliaweg, 1797SW Den Hoorn","href":"","score":0.11162212491035461},"geometry":{"type":"Polygon","coordinates":[[[4.68911630577304,52.92449928128154],[4.889116305773039,52.92449928128154],[4.889116305773039,53.124499281281544],[4.68911630577304,53.124499281281544],[4.68911630577304,52.92449928128154]]]},"id":"50"},{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"Bakkenweg - Den Hoorn","highlight":"Bakkenweg, 1797RJ Den Hoorn","href":"","score":0.11162212491035461},"geometry":{"type":"Polygon","coordinates":[[[4.6548723261037095,52.94811743920973],[4.854872326103709,52.94811743920973],[4.854872326103709,53.148117439209734],[4.6548723261037095,53.148117439209734],[4.6548723261037095,52.94811743920973]]]},"id":"520"},{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"Beatrixlaan - Den Burg","highlight":"Beatrixlaan, 1791GE Den Burg","href":"","score":0.11162212491035461},"geometry":{"type":"Polygon","coordinates":[[[4.690892824472019,52.95558352001795],[4.890892824472019,52.95558352001795],[4.890892824472019,53.155583520017956],[4.690892824472019,53.155583520017956],[4.690892824472019,52.95558352001795]]]},"id":"591"},{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"Ada van Hollandstraat - Den Burg","highlight":"Ada van Hollandstraat, 1791DH Den Burg","href":"","score":0.09617967158555984},"geometry":{"type":"Polygon","coordinates":[[[4.696235388824104,52.95196001510249],[4.8962353888241035,52.95196001510249],[4.8962353888241035,53.151960015102496],[4.696235388824104,53.151960015102496],[4.696235388824104,52.95196001510249]]]},"id":"26"},{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"Anne Frankstraat - Den Burg","highlight":"Anne Frankstraat, 1791DT Den Burg","href":"","score":0.09617967158555984},"geometry":{"type":"Polygon","coordinates":[[[4.692873779103581,52.950932925919574],[4.892873779103581,52.950932925919574],[4.892873779103581,53.15093292591958],[4.692873779103581,53.15093292591958],[4.692873779103581,52.950932925919574]]]},"id":"474"}],"numberReturned":0} +{"type":"FeatureCollection","timeStamp":"2000-01-01T00:00:00Z","features":[{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"Abbewaal - Den Burg","highlight":"Abbewaal, 1791WZ Den Burg","href":"","score":0.11162212491035461},"geometry":{"type":"Polygon","coordinates":[[[4.701721422439945,52.9619223105808],[4.901721422439945,52.9619223105808],[4.901721422439945,53.161922310580806],[4.701721422439945,53.161922310580806],[4.701721422439945,52.9619223105808]]]},"id":"99"},{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"Achterom - Den Burg","highlight":"Achterom, 1791AN Den Burg","href":"","score":0.11162212491035461},"geometry":{"type":"Polygon","coordinates":[[[4.699813158490893,52.95463219709524],[4.899813158490892,52.95463219709524],[4.899813158490892,53.154632197095246],[4.699813158490893,53.154632197095246],[4.699813158490893,52.95463219709524]]]},"id":"114"},{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"Akenbuurt - Den Burg","highlight":"Akenbuurt, 1791PJ Den Burg","href":"","score":0.11162212491035461},"geometry":{"type":"Polygon","coordinates":[[[4.680059099895046,52.95346592050607],[4.880059099895045,52.95346592050607],[4.880059099895045,53.15346592050607],[4.680059099895046,53.15346592050607],[4.680059099895046,52.95346592050607]]]},"id":"46"},{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"Amaliaweg - Den Hoorn","highlight":"Amaliaweg, 1797SW Den Hoorn","href":"","score":0.11162212491035461},"geometry":{"type":"Polygon","coordinates":[[[4.68911630577304,52.92449928128154],[4.889116305773039,52.92449928128154],[4.889116305773039,53.124499281281544],[4.68911630577304,53.124499281281544],[4.68911630577304,52.92449928128154]]]},"id":"50"},{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"Bakkenweg - Den Hoorn","highlight":"Bakkenweg, 1797RJ Den Hoorn","href":"","score":0.11162212491035461},"geometry":{"type":"Polygon","coordinates":[[[4.6548723261037095,52.94811743920973],[4.854872326103709,52.94811743920973],[4.854872326103709,53.148117439209734],[4.6548723261037095,53.148117439209734],[4.6548723261037095,52.94811743920973]]]},"id":"520"},{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"Beatrixlaan - Den Burg","highlight":"Beatrixlaan, 1791GE Den Burg","href":"","score":0.11162212491035461},"geometry":{"type":"Polygon","coordinates":[[[4.690892824472019,52.95558352001795],[4.890892824472019,52.95558352001795],[4.890892824472019,53.155583520017956],[4.690892824472019,53.155583520017956],[4.690892824472019,52.95558352001795]]]},"id":"591"},{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"Ada van Hollandstraat - Den Burg","highlight":"Ada van Hollandstraat, 1791DH Den Burg","href":"","score":0.09617967158555984},"geometry":{"type":"Polygon","coordinates":[[[4.696235388824104,52.95196001510249],[4.8962353888241035,52.95196001510249],[4.8962353888241035,53.151960015102496],[4.696235388824104,53.151960015102496],[4.696235388824104,52.95196001510249]]]},"id":"26"},{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"Anne Frankstraat - Den Burg","highlight":"Anne Frankstraat, 1791DT Den Burg","href":"","score":0.09617967158555984},"geometry":{"type":"Polygon","coordinates":[[[4.692873779103581,52.950932925919574],[4.892873779103581,52.950932925919574],[4.892873779103581,53.15093292591958],[4.692873779103581,53.15093292591958],[4.692873779103581,52.950932925919574]]]},"id":"474"}],"numberReturned":8} `, statusCode: http.StatusOK, }, From b0c3ab4773ac3a1338ab4cbaf8c3c083746dc7a2 Mon Sep 17 00:00:00 2001 From: Richard Kettelerij Date: Tue, 10 Dec 2024 15:41:22 +0100 Subject: [PATCH 12/38] feat: add search endpoint - add href to allow users to get the actual features from the OGC API --- internal/etl/extract/geopackage.go | 22 +- .../search/datasources/postgres/postgres.go | 20 +- internal/search/main.go | 41 ++ internal/search/main_test.go | 18 +- .../expected-search-den-deepcopy.json | 391 ++++++++++++++++++ .../search/testdata/expected-search-den.json | 391 ++++++++++++++++++ .../testdata/expected-search-oudeschild.json | 295 +++++++++++++ 7 files changed, 1153 insertions(+), 25 deletions(-) create mode 100644 internal/search/testdata/expected-search-den-deepcopy.json create mode 100644 internal/search/testdata/expected-search-den.json create mode 100644 internal/search/testdata/expected-search-oudeschild.json diff --git a/internal/etl/extract/geopackage.go b/internal/etl/extract/geopackage.go index 24811b5..51e7f23 100644 --- a/internal/etl/extract/geopackage.go +++ b/internal/etl/extract/geopackage.go @@ -93,23 +93,35 @@ func (g *GeoPackage) Extract(table config.FeatureTable, fields []string, where s if len(row) != len(fields)+nrOfStandardFieldsInQuery { return nil, fmt.Errorf("unexpected row length (%v)", len(row)) } - result = append(result, mapRowToRawRecord(row, fields)) + record, err := mapRowToRawRecord(row, fields) + if err != nil { + return nil, err + } + result = append(result, record) } return result, nil } -func mapRowToRawRecord(row []any, fields []string) t.RawRecord { +func mapRowToRawRecord(row []any, fields []string) (t.RawRecord, error) { bbox := row[1:5] + fid := row[0].(int64) + if fid <= 0 { + return t.RawRecord{}, errors.New("encountered negative fid") + } + geomType := row[5].(string) + if geomType == "" { + return t.RawRecord{}, fmt.Errorf("encountered empty geometry type for fid %d", fid) + } return t.RawRecord{ - FeatureID: row[0].(int64), + FeatureID: fid, Bbox: &geom.Extent{ bbox[0].(float64), bbox[1].(float64), bbox[2].(float64), bbox[3].(float64), }, - GeometryType: row[5].(string), + GeometryType: geomType, FieldValues: row[nrOfStandardFieldsInQuery : nrOfStandardFieldsInQuery+len(fields)], - } + }, nil } diff --git a/internal/search/datasources/postgres/postgres.go b/internal/search/datasources/postgres/postgres.go index bc777d1..14dd4da 100644 --- a/internal/search/datasources/postgres/postgres.go +++ b/internal/search/datasources/postgres/postgres.go @@ -64,11 +64,11 @@ func (p *Postgres) Suggest(ctx context.Context, searchTerm string, collections m fc := domain.FeatureCollection{Features: make([]*domain.Feature, 0)} for rows.Next() { - var displayName, highlightedText, featureID, collectionID, collectionVersion string + var displayName, highlightedText, featureID, collectionID, collectionVersion, geomType string var rank float64 var bbox pggeom.T - if err := rows.Scan(&displayName, &featureID, &collectionID, &collectionVersion, + if err := rows.Scan(&displayName, &featureID, &collectionID, &collectionVersion, &geomType, &bbox, &rank, &highlightedText); err != nil { return nil, err } @@ -80,14 +80,13 @@ func (p *Postgres) Suggest(ctx context.Context, searchTerm string, collections m ID: featureID, Geometry: *geojsonGeom, Properties: map[string]any{ - "collectionId": collectionID, - "collectionVersion": collectionVersion, - "displayName": displayName, - "highlight": highlightedText, - "href": "", // TODO add href - "score": rank, + "collectionId": collectionID, + "collectionVersion": collectionVersion, + "collectionGeometryType": geomType, + "displayName": displayName, + "highlight": highlightedText, + "score": rank, }, - // TODO add href also to Links } log.Printf("collections %s, srid %v", collections, srid) // TODO use params fc.Features = append(fc.Features, &f) @@ -103,11 +102,12 @@ func makeSearchQuery(term string, index string) string { max(r.feature_id) as feature_id, max(r.collection_id) as collection_id, max(r.collection_version) as collection_version, + max(r.geometry_type) as geometry_type, cast(max(r.bbox) as geometry) as bbox, max(r.rank) as rank, max(r.highlighted_text) as highlighted_text from ( - select display_name, feature_id, collection_id, collection_version, bbox, + select display_name, feature_id, collection_id, collection_version, geometry_type, bbox, ts_rank_cd(ts, to_tsquery('%[1]s'), 1) as rank, ts_headline('dutch', suggest, to_tsquery('%[1]s')) as highlighted_text from %[2]s diff --git a/internal/search/main.go b/internal/search/main.go index b28ac5c..e6eb729 100644 --- a/internal/search/main.go +++ b/internal/search/main.go @@ -12,6 +12,7 @@ import ( "strings" "time" + "github.com/PDOK/gomagpie/config" "github.com/PDOK/gomagpie/internal/engine" ds "github.com/PDOK/gomagpie/internal/search/datasources" "github.com/PDOK/gomagpie/internal/search/datasources/postgres" @@ -60,6 +61,46 @@ func (s *Search) Suggest() http.HandlerFunc { handleQueryError(w, err) return } + + for _, feat := range fc.Features { + collectionID, ok := feat.Properties["collectionId"] + if !ok || collectionID == "" { + log.Printf("collection reference not found in feature %s", feat.ID) + engine.RenderProblem(engine.ProblemServerError, w) + return + } + collection := config.CollectionByID(s.engine.Config, collectionID.(string)) + if collection.Search != nil { + for _, ogcColl := range collection.Search.OGCCollections { + geomType, ok := feat.Properties["collectionGeometryType"] + if !ok || geomType == "" { + log.Printf("geometry type not found in feature %s", feat.ID) + engine.RenderProblem(engine.ProblemServerError, w) + return + } + if strings.EqualFold(ogcColl.GeometryType, geomType.(string)) { + href, err := url.JoinPath(ogcColl.APIBaseURL.String(), "collections", ogcColl.CollectionID, "items", feat.ID) + if err != nil { + log.Printf("failed to construct API url %v", err) + engine.RenderProblem(engine.ProblemServerError, w) + } + href += "?f=json" + + // add href to feature both in GeoJSON properties (for broad compatibility and in line with OGC API Features part 5) and as a Link. + feat.Properties["href"] = href + feat.Links = []domain.Link{ + domain.Link{ + Rel: "canonical", + Title: "The actual feature in the corresponding OGC API", + Type: "application/geo+json", + Href: href, + }, + } + } + } + } + } + format := s.engine.CN.NegotiateFormat(r) switch format { case engine.FormatGeoJSON, engine.FormatJSON: diff --git a/internal/search/main_test.go b/internal/search/main_test.go index 3abf0e2..768d69c 100644 --- a/internal/search/main_test.go +++ b/internal/search/main_test.go @@ -89,9 +89,7 @@ func TestSuggest(t *testing.T) { url: "http://localhost:8080/search/suggest?q=\"Oudeschild\"&limit=50", }, want: want{ - body: ` -{"type":"FeatureCollection","timeStamp":"2000-01-01T00:00:00Z","features":[{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"Barentszstraat - Oudeschild","highlight":"Barentszstraat, 1792AD Oudeschild","href":"","score":0.14426951110363007},"geometry":{"type":"Polygon","coordinates":[[[4.748384044242354,52.93901709012591],[4.948384044242354,52.93901709012591],[4.948384044242354,53.13901709012591],[4.748384044242354,53.13901709012591],[4.748384044242354,52.93901709012591]]]},"id":"548"},{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"Bolwerk - Oudeschild","highlight":"Bolwerk, 1792AS Oudeschild","href":"","score":0.14426951110363007},"geometry":{"type":"Polygon","coordinates":[[[4.75002232386939,52.93847294238573],[4.95002232386939,52.93847294238573],[4.95002232386939,53.13847294238573],[4.75002232386939,53.13847294238573],[4.75002232386939,52.93847294238573]]]},"id":"1050"},{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"Commandeurssingel - Oudeschild","highlight":"Commandeurssingel, 1792AV Oudeschild","href":"","score":0.14426951110363007},"geometry":{"type":"Polygon","coordinates":[[[4.7451477245429015,52.93967814281323],[4.945147724542901,52.93967814281323],[4.945147724542901,53.13967814281323],[4.7451477245429015,53.13967814281323],[4.7451477245429015,52.93967814281323]]]},"id":"2725"},{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"De Houtmanstraat - Oudeschild","highlight":"De Houtmanstraat, 1792BC Oudeschild","href":"","score":0.12426698952913284},"geometry":{"type":"Polygon","coordinates":[[[4.748360166368449,52.93815392755542],[4.948360166368448,52.93815392755542],[4.948360166368448,53.13815392755542],[4.748360166368449,53.13815392755542],[4.748360166368449,52.93815392755542]]]},"id":"2921"},{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"De Ruyterstraat - Oudeschild","highlight":"De Ruyterstraat, 1792AP Oudeschild","href":"","score":0.12426698952913284},"geometry":{"type":"Polygon","coordinates":[[[4.747714279539418,52.93617309495475],[4.947714279539417,52.93617309495475],[4.947714279539417,53.136173094954756],[4.747714279539418,53.136173094954756],[4.747714279539418,52.93617309495475]]]},"id":"3049"},{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"De Wittstraat - Oudeschild","highlight":"De Wittstraat, 1792BP Oudeschild","href":"","score":0.12426698952913284},"geometry":{"type":"Polygon","coordinates":[[[4.745616492688666,52.93705261983951],[4.945616492688665,52.93705261983951],[4.945616492688665,53.137052619839515],[4.745616492688666,53.137052619839515],[4.745616492688666,52.93705261983951]]]},"id":"3041"}],"numberReturned":6} -`, + body: "internal/search/testdata/expected-search-oudeschild.json", statusCode: http.StatusOK, }, }, @@ -101,9 +99,7 @@ func TestSuggest(t *testing.T) { url: "http://localhost:8080/search/suggest?q=\"Den\"&limit=50", }, want: want{ - body: ` -{"type":"FeatureCollection","timeStamp":"2000-01-01T00:00:00Z","features":[{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"Abbewaal - Den Burg","highlight":"Abbewaal, 1791WZ Den Burg","href":"","score":0.11162212491035461},"geometry":{"type":"Polygon","coordinates":[[[4.701721422439945,52.9619223105808],[4.901721422439945,52.9619223105808],[4.901721422439945,53.161922310580806],[4.701721422439945,53.161922310580806],[4.701721422439945,52.9619223105808]]]},"id":"99"},{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"Achterom - Den Burg","highlight":"Achterom, 1791AN Den Burg","href":"","score":0.11162212491035461},"geometry":{"type":"Polygon","coordinates":[[[4.699813158490893,52.95463219709524],[4.899813158490892,52.95463219709524],[4.899813158490892,53.154632197095246],[4.699813158490893,53.154632197095246],[4.699813158490893,52.95463219709524]]]},"id":"114"},{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"Akenbuurt - Den Burg","highlight":"Akenbuurt, 1791PJ Den Burg","href":"","score":0.11162212491035461},"geometry":{"type":"Polygon","coordinates":[[[4.680059099895046,52.95346592050607],[4.880059099895045,52.95346592050607],[4.880059099895045,53.15346592050607],[4.680059099895046,53.15346592050607],[4.680059099895046,52.95346592050607]]]},"id":"46"},{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"Amaliaweg - Den Hoorn","highlight":"Amaliaweg, 1797SW Den Hoorn","href":"","score":0.11162212491035461},"geometry":{"type":"Polygon","coordinates":[[[4.68911630577304,52.92449928128154],[4.889116305773039,52.92449928128154],[4.889116305773039,53.124499281281544],[4.68911630577304,53.124499281281544],[4.68911630577304,52.92449928128154]]]},"id":"50"},{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"Bakkenweg - Den Hoorn","highlight":"Bakkenweg, 1797RJ Den Hoorn","href":"","score":0.11162212491035461},"geometry":{"type":"Polygon","coordinates":[[[4.6548723261037095,52.94811743920973],[4.854872326103709,52.94811743920973],[4.854872326103709,53.148117439209734],[4.6548723261037095,53.148117439209734],[4.6548723261037095,52.94811743920973]]]},"id":"520"},{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"Beatrixlaan - Den Burg","highlight":"Beatrixlaan, 1791GE Den Burg","href":"","score":0.11162212491035461},"geometry":{"type":"Polygon","coordinates":[[[4.690892824472019,52.95558352001795],[4.890892824472019,52.95558352001795],[4.890892824472019,53.155583520017956],[4.690892824472019,53.155583520017956],[4.690892824472019,52.95558352001795]]]},"id":"591"},{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"Ada van Hollandstraat - Den Burg","highlight":"Ada van Hollandstraat, 1791DH Den Burg","href":"","score":0.09617967158555984},"geometry":{"type":"Polygon","coordinates":[[[4.696235388824104,52.95196001510249],[4.8962353888241035,52.95196001510249],[4.8962353888241035,53.151960015102496],[4.696235388824104,53.151960015102496],[4.696235388824104,52.95196001510249]]]},"id":"26"},{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"Anne Frankstraat - Den Burg","highlight":"Anne Frankstraat, 1791DT Den Burg","href":"","score":0.09617967158555984},"geometry":{"type":"Polygon","coordinates":[[[4.692873779103581,52.950932925919574],[4.892873779103581,52.950932925919574],[4.892873779103581,53.15093292591958],[4.692873779103581,53.15093292591958],[4.692873779103581,52.950932925919574]]]},"id":"474"}],"numberReturned":8} -`, + body: "internal/search/testdata/expected-search-den.json", statusCode: http.StatusOK, }, }, @@ -113,9 +109,7 @@ func TestSuggest(t *testing.T) { url: "http://localhost:8080/search/suggest?q=\"Den\"&weg[version]=2&weg[relevance]=0.8&adres[version]=1&adres[relevance]=1&limit=10&f=json&crs=http%3A%2F%2Fwww.opengis.net%2Fdef%2Fcrs%2FEPSG%2F0%2F28992", }, want: want{ - body: ` -{"type":"FeatureCollection","timeStamp":"2000-01-01T00:00:00Z","features":[{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"Abbewaal - Den Burg","highlight":"Abbewaal, 1791WZ Den Burg","href":"","score":0.11162212491035461},"geometry":{"type":"Polygon","coordinates":[[[4.701721422439945,52.9619223105808],[4.901721422439945,52.9619223105808],[4.901721422439945,53.161922310580806],[4.701721422439945,53.161922310580806],[4.701721422439945,52.9619223105808]]]},"id":"99"},{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"Achterom - Den Burg","highlight":"Achterom, 1791AN Den Burg","href":"","score":0.11162212491035461},"geometry":{"type":"Polygon","coordinates":[[[4.699813158490893,52.95463219709524],[4.899813158490892,52.95463219709524],[4.899813158490892,53.154632197095246],[4.699813158490893,53.154632197095246],[4.699813158490893,52.95463219709524]]]},"id":"114"},{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"Akenbuurt - Den Burg","highlight":"Akenbuurt, 1791PJ Den Burg","href":"","score":0.11162212491035461},"geometry":{"type":"Polygon","coordinates":[[[4.680059099895046,52.95346592050607],[4.880059099895045,52.95346592050607],[4.880059099895045,53.15346592050607],[4.680059099895046,53.15346592050607],[4.680059099895046,52.95346592050607]]]},"id":"46"},{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"Amaliaweg - Den Hoorn","highlight":"Amaliaweg, 1797SW Den Hoorn","href":"","score":0.11162212491035461},"geometry":{"type":"Polygon","coordinates":[[[4.68911630577304,52.92449928128154],[4.889116305773039,52.92449928128154],[4.889116305773039,53.124499281281544],[4.68911630577304,53.124499281281544],[4.68911630577304,52.92449928128154]]]},"id":"50"},{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"Bakkenweg - Den Hoorn","highlight":"Bakkenweg, 1797RJ Den Hoorn","href":"","score":0.11162212491035461},"geometry":{"type":"Polygon","coordinates":[[[4.6548723261037095,52.94811743920973],[4.854872326103709,52.94811743920973],[4.854872326103709,53.148117439209734],[4.6548723261037095,53.148117439209734],[4.6548723261037095,52.94811743920973]]]},"id":"520"},{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"Beatrixlaan - Den Burg","highlight":"Beatrixlaan, 1791GE Den Burg","href":"","score":0.11162212491035461},"geometry":{"type":"Polygon","coordinates":[[[4.690892824472019,52.95558352001795],[4.890892824472019,52.95558352001795],[4.890892824472019,53.155583520017956],[4.690892824472019,53.155583520017956],[4.690892824472019,52.95558352001795]]]},"id":"591"},{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"Ada van Hollandstraat - Den Burg","highlight":"Ada van Hollandstraat, 1791DH Den Burg","href":"","score":0.09617967158555984},"geometry":{"type":"Polygon","coordinates":[[[4.696235388824104,52.95196001510249],[4.8962353888241035,52.95196001510249],[4.8962353888241035,53.151960015102496],[4.696235388824104,53.151960015102496],[4.696235388824104,52.95196001510249]]]},"id":"26"},{"type":"Feature","properties":{"collectionId":"addresses","collectionVersion":"1","displayName":"Anne Frankstraat - Den Burg","highlight":"Anne Frankstraat, 1791DT Den Burg","href":"","score":0.09617967158555984},"geometry":{"type":"Polygon","coordinates":[[[4.692873779103581,52.950932925919574],[4.892873779103581,52.950932925919574],[4.892873779103581,53.15093292591958],[4.692873779103581,53.15093292591958],[4.692873779103581,52.950932925919574]]]},"id":"474"}],"numberReturned":8} -`, + body: "internal/search/testdata/expected-search-den-deepcopy.json", statusCode: http.StatusOK, }, }, @@ -139,7 +133,11 @@ func TestSuggest(t *testing.T) { assert.Equal(t, tt.want.statusCode, rr.Code) log.Printf("============ ACTUAL:\n %s", rr.Body.String()) - assert.JSONEq(t, tt.want.body, rr.Body.String()) + expectedBody, err := os.ReadFile(tt.want.body) + if err != nil { + assert.NoError(t, err) + } + assert.JSONEq(t, string(expectedBody), rr.Body.String()) }) } } diff --git a/internal/search/testdata/expected-search-den-deepcopy.json b/internal/search/testdata/expected-search-den-deepcopy.json new file mode 100644 index 0000000..77c42a7 --- /dev/null +++ b/internal/search/testdata/expected-search-den-deepcopy.json @@ -0,0 +1,391 @@ +{ + "type": "FeatureCollection", + "timeStamp": "2000-01-01T00:00:00Z", + "features": [ + { + "type": "Feature", + "properties": { + "collectionGeometryType": "POINT", + "collectionId": "addresses", + "collectionVersion": "1", + "displayName": "Abbewaal - Den Burg", + "highlight": "Abbewaal, 1791WZ Den Burg", + "href": "https://example.com/ogc/v1/collections/addresses/items/99?f=json", + "score": 0.11162212491035461 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 4.701721422439945, + 52.9619223105808 + ], + [ + 4.901721422439945, + 52.9619223105808 + ], + [ + 4.901721422439945, + 53.161922310580806 + ], + [ + 4.701721422439945, + 53.161922310580806 + ], + [ + 4.701721422439945, + 52.9619223105808 + ] + ] + ] + }, + "id": "99", + "links": [ + { + "rel": "canonical", + "title": "The actual feature in the corresponding OGC API", + "type": "application/geo+json", + "href": "https://example.com/ogc/v1/collections/addresses/items/99?f=json" + } + ] + }, + { + "type": "Feature", + "properties": { + "collectionGeometryType": "POINT", + "collectionId": "addresses", + "collectionVersion": "1", + "displayName": "Achterom - Den Burg", + "highlight": "Achterom, 1791AN Den Burg", + "href": "https://example.com/ogc/v1/collections/addresses/items/114?f=json", + "score": 0.11162212491035461 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 4.699813158490893, + 52.95463219709524 + ], + [ + 4.899813158490892, + 52.95463219709524 + ], + [ + 4.899813158490892, + 53.154632197095246 + ], + [ + 4.699813158490893, + 53.154632197095246 + ], + [ + 4.699813158490893, + 52.95463219709524 + ] + ] + ] + }, + "id": "114", + "links": [ + { + "rel": "canonical", + "title": "The actual feature in the corresponding OGC API", + "type": "application/geo+json", + "href": "https://example.com/ogc/v1/collections/addresses/items/114?f=json" + } + ] + }, + { + "type": "Feature", + "properties": { + "collectionGeometryType": "POINT", + "collectionId": "addresses", + "collectionVersion": "1", + "displayName": "Akenbuurt - Den Burg", + "highlight": "Akenbuurt, 1791PJ Den Burg", + "href": "https://example.com/ogc/v1/collections/addresses/items/46?f=json", + "score": 0.11162212491035461 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 4.680059099895046, + 52.95346592050607 + ], + [ + 4.880059099895045, + 52.95346592050607 + ], + [ + 4.880059099895045, + 53.15346592050607 + ], + [ + 4.680059099895046, + 53.15346592050607 + ], + [ + 4.680059099895046, + 52.95346592050607 + ] + ] + ] + }, + "id": "46", + "links": [ + { + "rel": "canonical", + "title": "The actual feature in the corresponding OGC API", + "type": "application/geo+json", + "href": "https://example.com/ogc/v1/collections/addresses/items/46?f=json" + } + ] + }, + { + "type": "Feature", + "properties": { + "collectionGeometryType": "POINT", + "collectionId": "addresses", + "collectionVersion": "1", + "displayName": "Amaliaweg - Den Hoorn", + "highlight": "Amaliaweg, 1797SW Den Hoorn", + "href": "https://example.com/ogc/v1/collections/addresses/items/50?f=json", + "score": 0.11162212491035461 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 4.68911630577304, + 52.92449928128154 + ], + [ + 4.889116305773039, + 52.92449928128154 + ], + [ + 4.889116305773039, + 53.124499281281544 + ], + [ + 4.68911630577304, + 53.124499281281544 + ], + [ + 4.68911630577304, + 52.92449928128154 + ] + ] + ] + }, + "id": "50", + "links": [ + { + "rel": "canonical", + "title": "The actual feature in the corresponding OGC API", + "type": "application/geo+json", + "href": "https://example.com/ogc/v1/collections/addresses/items/50?f=json" + } + ] + }, + { + "type": "Feature", + "properties": { + "collectionGeometryType": "POINT", + "collectionId": "addresses", + "collectionVersion": "1", + "displayName": "Bakkenweg - Den Hoorn", + "highlight": "Bakkenweg, 1797RJ Den Hoorn", + "href": "https://example.com/ogc/v1/collections/addresses/items/520?f=json", + "score": 0.11162212491035461 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 4.6548723261037095, + 52.94811743920973 + ], + [ + 4.854872326103709, + 52.94811743920973 + ], + [ + 4.854872326103709, + 53.148117439209734 + ], + [ + 4.6548723261037095, + 53.148117439209734 + ], + [ + 4.6548723261037095, + 52.94811743920973 + ] + ] + ] + }, + "id": "520", + "links": [ + { + "rel": "canonical", + "title": "The actual feature in the corresponding OGC API", + "type": "application/geo+json", + "href": "https://example.com/ogc/v1/collections/addresses/items/520?f=json" + } + ] + }, + { + "type": "Feature", + "properties": { + "collectionGeometryType": "POINT", + "collectionId": "addresses", + "collectionVersion": "1", + "displayName": "Beatrixlaan - Den Burg", + "highlight": "Beatrixlaan, 1791GE Den Burg", + "href": "https://example.com/ogc/v1/collections/addresses/items/591?f=json", + "score": 0.11162212491035461 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 4.690892824472019, + 52.95558352001795 + ], + [ + 4.890892824472019, + 52.95558352001795 + ], + [ + 4.890892824472019, + 53.155583520017956 + ], + [ + 4.690892824472019, + 53.155583520017956 + ], + [ + 4.690892824472019, + 52.95558352001795 + ] + ] + ] + }, + "id": "591", + "links": [ + { + "rel": "canonical", + "title": "The actual feature in the corresponding OGC API", + "type": "application/geo+json", + "href": "https://example.com/ogc/v1/collections/addresses/items/591?f=json" + } + ] + }, + { + "type": "Feature", + "properties": { + "collectionGeometryType": "POINT", + "collectionId": "addresses", + "collectionVersion": "1", + "displayName": "Ada van Hollandstraat - Den Burg", + "highlight": "Ada van Hollandstraat, 1791DH Den Burg", + "href": "https://example.com/ogc/v1/collections/addresses/items/26?f=json", + "score": 0.09617967158555984 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 4.696235388824104, + 52.95196001510249 + ], + [ + 4.8962353888241035, + 52.95196001510249 + ], + [ + 4.8962353888241035, + 53.151960015102496 + ], + [ + 4.696235388824104, + 53.151960015102496 + ], + [ + 4.696235388824104, + 52.95196001510249 + ] + ] + ] + }, + "id": "26", + "links": [ + { + "rel": "canonical", + "title": "The actual feature in the corresponding OGC API", + "type": "application/geo+json", + "href": "https://example.com/ogc/v1/collections/addresses/items/26?f=json" + } + ] + }, + { + "type": "Feature", + "properties": { + "collectionGeometryType": "POINT", + "collectionId": "addresses", + "collectionVersion": "1", + "displayName": "Anne Frankstraat - Den Burg", + "highlight": "Anne Frankstraat, 1791DT Den Burg", + "href": "https://example.com/ogc/v1/collections/addresses/items/474?f=json", + "score": 0.09617967158555984 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 4.692873779103581, + 52.950932925919574 + ], + [ + 4.892873779103581, + 52.950932925919574 + ], + [ + 4.892873779103581, + 53.15093292591958 + ], + [ + 4.692873779103581, + 53.15093292591958 + ], + [ + 4.692873779103581, + 52.950932925919574 + ] + ] + ] + }, + "id": "474", + "links": [ + { + "rel": "canonical", + "title": "The actual feature in the corresponding OGC API", + "type": "application/geo+json", + "href": "https://example.com/ogc/v1/collections/addresses/items/474?f=json" + } + ] + } + ], + "numberReturned": 8 +} diff --git a/internal/search/testdata/expected-search-den.json b/internal/search/testdata/expected-search-den.json new file mode 100644 index 0000000..77c42a7 --- /dev/null +++ b/internal/search/testdata/expected-search-den.json @@ -0,0 +1,391 @@ +{ + "type": "FeatureCollection", + "timeStamp": "2000-01-01T00:00:00Z", + "features": [ + { + "type": "Feature", + "properties": { + "collectionGeometryType": "POINT", + "collectionId": "addresses", + "collectionVersion": "1", + "displayName": "Abbewaal - Den Burg", + "highlight": "Abbewaal, 1791WZ Den Burg", + "href": "https://example.com/ogc/v1/collections/addresses/items/99?f=json", + "score": 0.11162212491035461 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 4.701721422439945, + 52.9619223105808 + ], + [ + 4.901721422439945, + 52.9619223105808 + ], + [ + 4.901721422439945, + 53.161922310580806 + ], + [ + 4.701721422439945, + 53.161922310580806 + ], + [ + 4.701721422439945, + 52.9619223105808 + ] + ] + ] + }, + "id": "99", + "links": [ + { + "rel": "canonical", + "title": "The actual feature in the corresponding OGC API", + "type": "application/geo+json", + "href": "https://example.com/ogc/v1/collections/addresses/items/99?f=json" + } + ] + }, + { + "type": "Feature", + "properties": { + "collectionGeometryType": "POINT", + "collectionId": "addresses", + "collectionVersion": "1", + "displayName": "Achterom - Den Burg", + "highlight": "Achterom, 1791AN Den Burg", + "href": "https://example.com/ogc/v1/collections/addresses/items/114?f=json", + "score": 0.11162212491035461 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 4.699813158490893, + 52.95463219709524 + ], + [ + 4.899813158490892, + 52.95463219709524 + ], + [ + 4.899813158490892, + 53.154632197095246 + ], + [ + 4.699813158490893, + 53.154632197095246 + ], + [ + 4.699813158490893, + 52.95463219709524 + ] + ] + ] + }, + "id": "114", + "links": [ + { + "rel": "canonical", + "title": "The actual feature in the corresponding OGC API", + "type": "application/geo+json", + "href": "https://example.com/ogc/v1/collections/addresses/items/114?f=json" + } + ] + }, + { + "type": "Feature", + "properties": { + "collectionGeometryType": "POINT", + "collectionId": "addresses", + "collectionVersion": "1", + "displayName": "Akenbuurt - Den Burg", + "highlight": "Akenbuurt, 1791PJ Den Burg", + "href": "https://example.com/ogc/v1/collections/addresses/items/46?f=json", + "score": 0.11162212491035461 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 4.680059099895046, + 52.95346592050607 + ], + [ + 4.880059099895045, + 52.95346592050607 + ], + [ + 4.880059099895045, + 53.15346592050607 + ], + [ + 4.680059099895046, + 53.15346592050607 + ], + [ + 4.680059099895046, + 52.95346592050607 + ] + ] + ] + }, + "id": "46", + "links": [ + { + "rel": "canonical", + "title": "The actual feature in the corresponding OGC API", + "type": "application/geo+json", + "href": "https://example.com/ogc/v1/collections/addresses/items/46?f=json" + } + ] + }, + { + "type": "Feature", + "properties": { + "collectionGeometryType": "POINT", + "collectionId": "addresses", + "collectionVersion": "1", + "displayName": "Amaliaweg - Den Hoorn", + "highlight": "Amaliaweg, 1797SW Den Hoorn", + "href": "https://example.com/ogc/v1/collections/addresses/items/50?f=json", + "score": 0.11162212491035461 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 4.68911630577304, + 52.92449928128154 + ], + [ + 4.889116305773039, + 52.92449928128154 + ], + [ + 4.889116305773039, + 53.124499281281544 + ], + [ + 4.68911630577304, + 53.124499281281544 + ], + [ + 4.68911630577304, + 52.92449928128154 + ] + ] + ] + }, + "id": "50", + "links": [ + { + "rel": "canonical", + "title": "The actual feature in the corresponding OGC API", + "type": "application/geo+json", + "href": "https://example.com/ogc/v1/collections/addresses/items/50?f=json" + } + ] + }, + { + "type": "Feature", + "properties": { + "collectionGeometryType": "POINT", + "collectionId": "addresses", + "collectionVersion": "1", + "displayName": "Bakkenweg - Den Hoorn", + "highlight": "Bakkenweg, 1797RJ Den Hoorn", + "href": "https://example.com/ogc/v1/collections/addresses/items/520?f=json", + "score": 0.11162212491035461 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 4.6548723261037095, + 52.94811743920973 + ], + [ + 4.854872326103709, + 52.94811743920973 + ], + [ + 4.854872326103709, + 53.148117439209734 + ], + [ + 4.6548723261037095, + 53.148117439209734 + ], + [ + 4.6548723261037095, + 52.94811743920973 + ] + ] + ] + }, + "id": "520", + "links": [ + { + "rel": "canonical", + "title": "The actual feature in the corresponding OGC API", + "type": "application/geo+json", + "href": "https://example.com/ogc/v1/collections/addresses/items/520?f=json" + } + ] + }, + { + "type": "Feature", + "properties": { + "collectionGeometryType": "POINT", + "collectionId": "addresses", + "collectionVersion": "1", + "displayName": "Beatrixlaan - Den Burg", + "highlight": "Beatrixlaan, 1791GE Den Burg", + "href": "https://example.com/ogc/v1/collections/addresses/items/591?f=json", + "score": 0.11162212491035461 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 4.690892824472019, + 52.95558352001795 + ], + [ + 4.890892824472019, + 52.95558352001795 + ], + [ + 4.890892824472019, + 53.155583520017956 + ], + [ + 4.690892824472019, + 53.155583520017956 + ], + [ + 4.690892824472019, + 52.95558352001795 + ] + ] + ] + }, + "id": "591", + "links": [ + { + "rel": "canonical", + "title": "The actual feature in the corresponding OGC API", + "type": "application/geo+json", + "href": "https://example.com/ogc/v1/collections/addresses/items/591?f=json" + } + ] + }, + { + "type": "Feature", + "properties": { + "collectionGeometryType": "POINT", + "collectionId": "addresses", + "collectionVersion": "1", + "displayName": "Ada van Hollandstraat - Den Burg", + "highlight": "Ada van Hollandstraat, 1791DH Den Burg", + "href": "https://example.com/ogc/v1/collections/addresses/items/26?f=json", + "score": 0.09617967158555984 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 4.696235388824104, + 52.95196001510249 + ], + [ + 4.8962353888241035, + 52.95196001510249 + ], + [ + 4.8962353888241035, + 53.151960015102496 + ], + [ + 4.696235388824104, + 53.151960015102496 + ], + [ + 4.696235388824104, + 52.95196001510249 + ] + ] + ] + }, + "id": "26", + "links": [ + { + "rel": "canonical", + "title": "The actual feature in the corresponding OGC API", + "type": "application/geo+json", + "href": "https://example.com/ogc/v1/collections/addresses/items/26?f=json" + } + ] + }, + { + "type": "Feature", + "properties": { + "collectionGeometryType": "POINT", + "collectionId": "addresses", + "collectionVersion": "1", + "displayName": "Anne Frankstraat - Den Burg", + "highlight": "Anne Frankstraat, 1791DT Den Burg", + "href": "https://example.com/ogc/v1/collections/addresses/items/474?f=json", + "score": 0.09617967158555984 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 4.692873779103581, + 52.950932925919574 + ], + [ + 4.892873779103581, + 52.950932925919574 + ], + [ + 4.892873779103581, + 53.15093292591958 + ], + [ + 4.692873779103581, + 53.15093292591958 + ], + [ + 4.692873779103581, + 52.950932925919574 + ] + ] + ] + }, + "id": "474", + "links": [ + { + "rel": "canonical", + "title": "The actual feature in the corresponding OGC API", + "type": "application/geo+json", + "href": "https://example.com/ogc/v1/collections/addresses/items/474?f=json" + } + ] + } + ], + "numberReturned": 8 +} diff --git a/internal/search/testdata/expected-search-oudeschild.json b/internal/search/testdata/expected-search-oudeschild.json new file mode 100644 index 0000000..e44320e --- /dev/null +++ b/internal/search/testdata/expected-search-oudeschild.json @@ -0,0 +1,295 @@ +{ + "type": "FeatureCollection", + "timeStamp": "2000-01-01T00:00:00Z", + "features": [ + { + "type": "Feature", + "properties": { + "collectionGeometryType": "POINT", + "collectionId": "addresses", + "collectionVersion": "1", + "displayName": "Barentszstraat - Oudeschild", + "highlight": "Barentszstraat, 1792AD Oudeschild", + "href": "https://example.com/ogc/v1/collections/addresses/items/548?f=json", + "score": 0.14426951110363007 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 4.748384044242354, + 52.93901709012591 + ], + [ + 4.948384044242354, + 52.93901709012591 + ], + [ + 4.948384044242354, + 53.13901709012591 + ], + [ + 4.748384044242354, + 53.13901709012591 + ], + [ + 4.748384044242354, + 52.93901709012591 + ] + ] + ] + }, + "id": "548", + "links": [ + { + "rel": "canonical", + "title": "The actual feature in the corresponding OGC API", + "type": "application/geo+json", + "href": "https://example.com/ogc/v1/collections/addresses/items/548?f=json" + } + ] + }, + { + "type": "Feature", + "properties": { + "collectionGeometryType": "POINT", + "collectionId": "addresses", + "collectionVersion": "1", + "displayName": "Bolwerk - Oudeschild", + "highlight": "Bolwerk, 1792AS Oudeschild", + "href": "https://example.com/ogc/v1/collections/addresses/items/1050?f=json", + "score": 0.14426951110363007 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 4.75002232386939, + 52.93847294238573 + ], + [ + 4.95002232386939, + 52.93847294238573 + ], + [ + 4.95002232386939, + 53.13847294238573 + ], + [ + 4.75002232386939, + 53.13847294238573 + ], + [ + 4.75002232386939, + 52.93847294238573 + ] + ] + ] + }, + "id": "1050", + "links": [ + { + "rel": "canonical", + "title": "The actual feature in the corresponding OGC API", + "type": "application/geo+json", + "href": "https://example.com/ogc/v1/collections/addresses/items/1050?f=json" + } + ] + }, + { + "type": "Feature", + "properties": { + "collectionGeometryType": "POINT", + "collectionId": "addresses", + "collectionVersion": "1", + "displayName": "Commandeurssingel - Oudeschild", + "highlight": "Commandeurssingel, 1792AV Oudeschild", + "href": "https://example.com/ogc/v1/collections/addresses/items/2725?f=json", + "score": 0.14426951110363007 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 4.7451477245429015, + 52.93967814281323 + ], + [ + 4.945147724542901, + 52.93967814281323 + ], + [ + 4.945147724542901, + 53.13967814281323 + ], + [ + 4.7451477245429015, + 53.13967814281323 + ], + [ + 4.7451477245429015, + 52.93967814281323 + ] + ] + ] + }, + "id": "2725", + "links": [ + { + "rel": "canonical", + "title": "The actual feature in the corresponding OGC API", + "type": "application/geo+json", + "href": "https://example.com/ogc/v1/collections/addresses/items/2725?f=json" + } + ] + }, + { + "type": "Feature", + "properties": { + "collectionGeometryType": "POINT", + "collectionId": "addresses", + "collectionVersion": "1", + "displayName": "De Houtmanstraat - Oudeschild", + "highlight": "De Houtmanstraat, 1792BC Oudeschild", + "href": "https://example.com/ogc/v1/collections/addresses/items/2921?f=json", + "score": 0.12426698952913284 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 4.748360166368449, + 52.93815392755542 + ], + [ + 4.948360166368448, + 52.93815392755542 + ], + [ + 4.948360166368448, + 53.13815392755542 + ], + [ + 4.748360166368449, + 53.13815392755542 + ], + [ + 4.748360166368449, + 52.93815392755542 + ] + ] + ] + }, + "id": "2921", + "links": [ + { + "rel": "canonical", + "title": "The actual feature in the corresponding OGC API", + "type": "application/geo+json", + "href": "https://example.com/ogc/v1/collections/addresses/items/2921?f=json" + } + ] + }, + { + "type": "Feature", + "properties": { + "collectionGeometryType": "POINT", + "collectionId": "addresses", + "collectionVersion": "1", + "displayName": "De Ruyterstraat - Oudeschild", + "highlight": "De Ruyterstraat, 1792AP Oudeschild", + "href": "https://example.com/ogc/v1/collections/addresses/items/3049?f=json", + "score": 0.12426698952913284 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 4.747714279539418, + 52.93617309495475 + ], + [ + 4.947714279539417, + 52.93617309495475 + ], + [ + 4.947714279539417, + 53.136173094954756 + ], + [ + 4.747714279539418, + 53.136173094954756 + ], + [ + 4.747714279539418, + 52.93617309495475 + ] + ] + ] + }, + "id": "3049", + "links": [ + { + "rel": "canonical", + "title": "The actual feature in the corresponding OGC API", + "type": "application/geo+json", + "href": "https://example.com/ogc/v1/collections/addresses/items/3049?f=json" + } + ] + }, + { + "type": "Feature", + "properties": { + "collectionGeometryType": "POINT", + "collectionId": "addresses", + "collectionVersion": "1", + "displayName": "De Wittstraat - Oudeschild", + "highlight": "De Wittstraat, 1792BP Oudeschild", + "href": "https://example.com/ogc/v1/collections/addresses/items/3041?f=json", + "score": 0.12426698952913284 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 4.745616492688666, + 52.93705261983951 + ], + [ + 4.945616492688665, + 52.93705261983951 + ], + [ + 4.945616492688665, + 53.137052619839515 + ], + [ + 4.745616492688666, + 53.137052619839515 + ], + [ + 4.745616492688666, + 52.93705261983951 + ] + ] + ] + }, + "id": "3041", + "links": [ + { + "rel": "canonical", + "title": "The actual feature in the corresponding OGC API", + "type": "application/geo+json", + "href": "https://example.com/ogc/v1/collections/addresses/items/3041?f=json" + } + ] + } + ], + "numberReturned": 6 +} From 927f96e10bcc101389d03e74b0352ece10d8976a Mon Sep 17 00:00:00 2001 From: Richard Kettelerij Date: Tue, 10 Dec 2024 16:10:35 +0100 Subject: [PATCH 13/38] feat: add search endpoint - add self link (alternate links to HTML/JSON-FG can be added in the future). ALso changed endpoint to just 'search' in line with OGC API Features part 9 Test Search (early draft) --- .../search/datasources/postgres/postgres.go | 1 + internal/search/json.go | 35 +++++++++++++++++-- internal/search/main.go | 8 ++--- internal/search/main_test.go | 16 ++++----- .../expected-search-den-deepcopy.json | 9 +++++ .../search/testdata/expected-search-den.json | 8 +++++ .../testdata/expected-search-oudeschild.json | 8 +++++ 7 files changed, 71 insertions(+), 14 deletions(-) diff --git a/internal/search/datasources/postgres/postgres.go b/internal/search/datasources/postgres/postgres.go index 14dd4da..1479bdb 100644 --- a/internal/search/datasources/postgres/postgres.go +++ b/internal/search/datasources/postgres/postgres.go @@ -62,6 +62,7 @@ func (p *Postgres) Suggest(ctx context.Context, searchTerm string, collections m } defer rows.Close() + // Turn rows into FeatureCollection fc := domain.FeatureCollection{Features: make([]*domain.Feature, 0)} for rows.Next() { var displayName, highlightedText, featureID, collectionID, collectionVersion, geomType string diff --git a/internal/search/json.go b/internal/search/json.go index 2eb5e11..777d8a7 100644 --- a/internal/search/json.go +++ b/internal/search/json.go @@ -5,6 +5,7 @@ import ( "io" "log" "net/http" + "net/url" "os" "strconv" "time" @@ -19,9 +20,9 @@ var ( disableJSONPerfOptimization, _ = strconv.ParseBool(os.Getenv("DISABLE_JSON_PERF_OPTIMIZATION")) ) -func featuresAsGeoJSON(w http.ResponseWriter, fc *domain.FeatureCollection) { +func featuresAsGeoJSON(w http.ResponseWriter, baseURL url.URL, fc *domain.FeatureCollection) { fc.Timestamp = now().Format(time.RFC3339) - // fc.Links = createFeatureCollectionLinks(engine.FormatGeoJSON, collectionID, cursor, featuresURL) // TODO add links + fc.Links = createFeatureCollectionLinks(baseURL) // TODO add links // TODO add validation // if jf.validateResponse { @@ -41,6 +42,36 @@ func serveJSON(input any, contentType string, w http.ResponseWriter) { } } +func createFeatureCollectionLinks(baseURL url.URL) []domain.Link { + links := make([]domain.Link, 0) + + href := baseURL.JoinPath("search") + query := href.Query() + query.Set(engine.FormatParam, engine.FormatJSON) + href.RawQuery = query.Encode() + + links = append(links, domain.Link{ + Rel: "self", + Title: "This document as GeoJSON", + Type: engine.MediaTypeGeoJSON, + Href: href.String(), + }) + // TODO: support HTML and JSON-FG output in location API + // links = append(links, domain.Link{ + // Rel: "alternate", + // Title: "This document as JSON-FG", + // Type: engine.MediaTypeJSONFG, + // Href: featuresURL.toSelfURL(collectionID, engine.FormatJSONFG), + // }) + // links = append(links, domain.Link{ + // Rel: "alternate", + // Title: "This document as HTML", + // Type: engine.MediaTypeHTML, + // Href: featuresURL.toSelfURL(collectionID, engine.FormatHTML), + // }) + return links +} + type jsonEncoder interface { Encode(input any) error } diff --git a/internal/search/main.go b/internal/search/main.go index e6eb729..a9b02be 100644 --- a/internal/search/main.go +++ b/internal/search/main.go @@ -44,12 +44,12 @@ func NewSearch(e *engine.Engine, dbConn string, searchIndex string) *Search { engine: e, datasource: newDatasource(e, dbConn, searchIndex), } - e.Router.Get("/search/suggest", s.Suggest()) + e.Router.Get("/search", s.Search()) return s } -// Suggest autosuggest locations based on user input -func (s *Search) Suggest() http.HandlerFunc { +// Search autosuggest locations based on user input +func (s *Search) Search() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { collections, searchTerm, outputSRID, limit, err := parseQueryParams(r.URL.Query()) if err != nil { @@ -104,7 +104,7 @@ func (s *Search) Suggest() http.HandlerFunc { format := s.engine.CN.NegotiateFormat(r) switch format { case engine.FormatGeoJSON, engine.FormatJSON: - featuresAsGeoJSON(w, fc) + featuresAsGeoJSON(w, *s.engine.Config.BaseURL.URL, fc) default: engine.RenderProblem(engine.ProblemNotAcceptable, w, fmt.Sprintf("format '%s' is not supported", format)) return diff --git a/internal/search/main_test.go b/internal/search/main_test.go index 768d69c..9a86bbc 100644 --- a/internal/search/main_test.go +++ b/internal/search/main_test.go @@ -36,7 +36,7 @@ func init() { } } -func TestSuggest(t *testing.T) { +func TestSearch(t *testing.T) { if testing.Short() { t.Skip("Skipping integration test in short mode") } @@ -84,9 +84,9 @@ func TestSuggest(t *testing.T) { want want }{ { - name: "Suggest: Oudeschild", + name: "Search: Oudeschild", fields: fields{ - url: "http://localhost:8080/search/suggest?q=\"Oudeschild\"&limit=50", + url: "http://localhost:8080/search?q=\"Oudeschild\"&limit=50", }, want: want{ body: "internal/search/testdata/expected-search-oudeschild.json", @@ -94,9 +94,9 @@ func TestSuggest(t *testing.T) { }, }, { - name: "Suggest: Den ", + name: "Search: Den ", fields: fields{ - url: "http://localhost:8080/search/suggest?q=\"Den\"&limit=50", + url: "http://localhost:8080/search?q=\"Den\"&limit=50", }, want: want{ body: "internal/search/testdata/expected-search-den.json", @@ -104,9 +104,9 @@ func TestSuggest(t *testing.T) { }, }, { - name: "Suggest: Den. With deepCopy params", + name: "Search: Den. With deepCopy params", fields: fields{ - url: "http://localhost:8080/search/suggest?q=\"Den\"&weg[version]=2&weg[relevance]=0.8&adres[version]=1&adres[relevance]=1&limit=10&f=json&crs=http%3A%2F%2Fwww.opengis.net%2Fdef%2Fcrs%2FEPSG%2F0%2F28992", + url: "http://localhost:8080/search?q=\"Den\"&weg[version]=2&weg[relevance]=0.8&adres[version]=1&adres[relevance]=1&limit=10&f=json&crs=http%3A%2F%2Fwww.opengis.net%2Fdef%2Fcrs%2FEPSG%2F0%2F28992", }, want: want{ body: "internal/search/testdata/expected-search-den-deepcopy.json", @@ -124,7 +124,7 @@ func TestSuggest(t *testing.T) { defer ts.Close() // when - handler := searchEndpoint.Suggest() + handler := searchEndpoint.Search() req, err := createRequest(tt.fields.url) assert.NoError(t, err) handler.ServeHTTP(rr, req) diff --git a/internal/search/testdata/expected-search-den-deepcopy.json b/internal/search/testdata/expected-search-den-deepcopy.json index 77c42a7..e896b20 100644 --- a/internal/search/testdata/expected-search-den-deepcopy.json +++ b/internal/search/testdata/expected-search-den-deepcopy.json @@ -1,6 +1,14 @@ { "type": "FeatureCollection", "timeStamp": "2000-01-01T00:00:00Z", + "links": [ + { + "rel": "self", + "title": "This document as GeoJSON", + "type": "application/geo+json", + "href": "http://localhost:8080/search?f=json" + } + ], "features": [ { "type": "Feature", @@ -389,3 +397,4 @@ ], "numberReturned": 8 } + \ No newline at end of file diff --git a/internal/search/testdata/expected-search-den.json b/internal/search/testdata/expected-search-den.json index 77c42a7..4c020dd 100644 --- a/internal/search/testdata/expected-search-den.json +++ b/internal/search/testdata/expected-search-den.json @@ -1,6 +1,14 @@ { "type": "FeatureCollection", "timeStamp": "2000-01-01T00:00:00Z", + "links": [ + { + "rel": "self", + "title": "This document as GeoJSON", + "type": "application/geo+json", + "href": "http://localhost:8080/search?f=json" + } + ], "features": [ { "type": "Feature", diff --git a/internal/search/testdata/expected-search-oudeschild.json b/internal/search/testdata/expected-search-oudeschild.json index e44320e..1bf07f5 100644 --- a/internal/search/testdata/expected-search-oudeschild.json +++ b/internal/search/testdata/expected-search-oudeschild.json @@ -1,6 +1,14 @@ { "type": "FeatureCollection", "timeStamp": "2000-01-01T00:00:00Z", + "links": [ + { + "rel": "self", + "title": "This document as GeoJSON", + "type": "application/geo+json", + "href": "http://localhost:8080/search?f=json" + } + ], "features": [ { "type": "Feature", From 4faae30c57e6cf30add5742d0581aabb90765e99 Mon Sep 17 00:00:00 2001 From: Richard Kettelerij Date: Tue, 10 Dec 2024 16:53:39 +0100 Subject: [PATCH 14/38] feat: add search endpoint - highlight display name, not suggest --- .../search/datasources/postgres/postgres.go | 2 +- .../testdata/expected-search-den-deepcopy.json | 17 ++++++++--------- .../search/testdata/expected-search-den.json | 16 ++++++++-------- .../testdata/expected-search-oudeschild.json | 12 ++++++------ 4 files changed, 23 insertions(+), 24 deletions(-) diff --git a/internal/search/datasources/postgres/postgres.go b/internal/search/datasources/postgres/postgres.go index 1479bdb..1080107 100644 --- a/internal/search/datasources/postgres/postgres.go +++ b/internal/search/datasources/postgres/postgres.go @@ -110,7 +110,7 @@ func makeSearchQuery(term string, index string) string { from ( select display_name, feature_id, collection_id, collection_version, geometry_type, bbox, ts_rank_cd(ts, to_tsquery('%[1]s'), 1) as rank, - ts_headline('dutch', suggest, to_tsquery('%[1]s')) as highlighted_text + ts_headline('dutch', display_name, to_tsquery('%[1]s')) as highlighted_text from %[2]s where ts @@ to_tsquery('%[1]s') limit 500 diff --git a/internal/search/testdata/expected-search-den-deepcopy.json b/internal/search/testdata/expected-search-den-deepcopy.json index e896b20..0dbc6f9 100644 --- a/internal/search/testdata/expected-search-den-deepcopy.json +++ b/internal/search/testdata/expected-search-den-deepcopy.json @@ -17,7 +17,7 @@ "collectionId": "addresses", "collectionVersion": "1", "displayName": "Abbewaal - Den Burg", - "highlight": "Abbewaal, 1791WZ Den Burg", + "highlight": "Abbewaal - Den Burg", "href": "https://example.com/ogc/v1/collections/addresses/items/99?f=json", "score": 0.11162212491035461 }, @@ -65,7 +65,7 @@ "collectionId": "addresses", "collectionVersion": "1", "displayName": "Achterom - Den Burg", - "highlight": "Achterom, 1791AN Den Burg", + "highlight": "Achterom - Den Burg", "href": "https://example.com/ogc/v1/collections/addresses/items/114?f=json", "score": 0.11162212491035461 }, @@ -113,7 +113,7 @@ "collectionId": "addresses", "collectionVersion": "1", "displayName": "Akenbuurt - Den Burg", - "highlight": "Akenbuurt, 1791PJ Den Burg", + "highlight": "Akenbuurt - Den Burg", "href": "https://example.com/ogc/v1/collections/addresses/items/46?f=json", "score": 0.11162212491035461 }, @@ -161,7 +161,7 @@ "collectionId": "addresses", "collectionVersion": "1", "displayName": "Amaliaweg - Den Hoorn", - "highlight": "Amaliaweg, 1797SW Den Hoorn", + "highlight": "Amaliaweg - Den Hoorn", "href": "https://example.com/ogc/v1/collections/addresses/items/50?f=json", "score": 0.11162212491035461 }, @@ -209,7 +209,7 @@ "collectionId": "addresses", "collectionVersion": "1", "displayName": "Bakkenweg - Den Hoorn", - "highlight": "Bakkenweg, 1797RJ Den Hoorn", + "highlight": "Bakkenweg - Den Hoorn", "href": "https://example.com/ogc/v1/collections/addresses/items/520?f=json", "score": 0.11162212491035461 }, @@ -257,7 +257,7 @@ "collectionId": "addresses", "collectionVersion": "1", "displayName": "Beatrixlaan - Den Burg", - "highlight": "Beatrixlaan, 1791GE Den Burg", + "highlight": "Beatrixlaan - Den Burg", "href": "https://example.com/ogc/v1/collections/addresses/items/591?f=json", "score": 0.11162212491035461 }, @@ -305,7 +305,7 @@ "collectionId": "addresses", "collectionVersion": "1", "displayName": "Ada van Hollandstraat - Den Burg", - "highlight": "Ada van Hollandstraat, 1791DH Den Burg", + "highlight": "Ada van Hollandstraat - Den Burg", "href": "https://example.com/ogc/v1/collections/addresses/items/26?f=json", "score": 0.09617967158555984 }, @@ -353,7 +353,7 @@ "collectionId": "addresses", "collectionVersion": "1", "displayName": "Anne Frankstraat - Den Burg", - "highlight": "Anne Frankstraat, 1791DT Den Burg", + "highlight": "Anne Frankstraat - Den Burg", "href": "https://example.com/ogc/v1/collections/addresses/items/474?f=json", "score": 0.09617967158555984 }, @@ -397,4 +397,3 @@ ], "numberReturned": 8 } - \ No newline at end of file diff --git a/internal/search/testdata/expected-search-den.json b/internal/search/testdata/expected-search-den.json index 4c020dd..0dbc6f9 100644 --- a/internal/search/testdata/expected-search-den.json +++ b/internal/search/testdata/expected-search-den.json @@ -17,7 +17,7 @@ "collectionId": "addresses", "collectionVersion": "1", "displayName": "Abbewaal - Den Burg", - "highlight": "Abbewaal, 1791WZ Den Burg", + "highlight": "Abbewaal - Den Burg", "href": "https://example.com/ogc/v1/collections/addresses/items/99?f=json", "score": 0.11162212491035461 }, @@ -65,7 +65,7 @@ "collectionId": "addresses", "collectionVersion": "1", "displayName": "Achterom - Den Burg", - "highlight": "Achterom, 1791AN Den Burg", + "highlight": "Achterom - Den Burg", "href": "https://example.com/ogc/v1/collections/addresses/items/114?f=json", "score": 0.11162212491035461 }, @@ -113,7 +113,7 @@ "collectionId": "addresses", "collectionVersion": "1", "displayName": "Akenbuurt - Den Burg", - "highlight": "Akenbuurt, 1791PJ Den Burg", + "highlight": "Akenbuurt - Den Burg", "href": "https://example.com/ogc/v1/collections/addresses/items/46?f=json", "score": 0.11162212491035461 }, @@ -161,7 +161,7 @@ "collectionId": "addresses", "collectionVersion": "1", "displayName": "Amaliaweg - Den Hoorn", - "highlight": "Amaliaweg, 1797SW Den Hoorn", + "highlight": "Amaliaweg - Den Hoorn", "href": "https://example.com/ogc/v1/collections/addresses/items/50?f=json", "score": 0.11162212491035461 }, @@ -209,7 +209,7 @@ "collectionId": "addresses", "collectionVersion": "1", "displayName": "Bakkenweg - Den Hoorn", - "highlight": "Bakkenweg, 1797RJ Den Hoorn", + "highlight": "Bakkenweg - Den Hoorn", "href": "https://example.com/ogc/v1/collections/addresses/items/520?f=json", "score": 0.11162212491035461 }, @@ -257,7 +257,7 @@ "collectionId": "addresses", "collectionVersion": "1", "displayName": "Beatrixlaan - Den Burg", - "highlight": "Beatrixlaan, 1791GE Den Burg", + "highlight": "Beatrixlaan - Den Burg", "href": "https://example.com/ogc/v1/collections/addresses/items/591?f=json", "score": 0.11162212491035461 }, @@ -305,7 +305,7 @@ "collectionId": "addresses", "collectionVersion": "1", "displayName": "Ada van Hollandstraat - Den Burg", - "highlight": "Ada van Hollandstraat, 1791DH Den Burg", + "highlight": "Ada van Hollandstraat - Den Burg", "href": "https://example.com/ogc/v1/collections/addresses/items/26?f=json", "score": 0.09617967158555984 }, @@ -353,7 +353,7 @@ "collectionId": "addresses", "collectionVersion": "1", "displayName": "Anne Frankstraat - Den Burg", - "highlight": "Anne Frankstraat, 1791DT Den Burg", + "highlight": "Anne Frankstraat - Den Burg", "href": "https://example.com/ogc/v1/collections/addresses/items/474?f=json", "score": 0.09617967158555984 }, diff --git a/internal/search/testdata/expected-search-oudeschild.json b/internal/search/testdata/expected-search-oudeschild.json index 1bf07f5..95a38a0 100644 --- a/internal/search/testdata/expected-search-oudeschild.json +++ b/internal/search/testdata/expected-search-oudeschild.json @@ -17,7 +17,7 @@ "collectionId": "addresses", "collectionVersion": "1", "displayName": "Barentszstraat - Oudeschild", - "highlight": "Barentszstraat, 1792AD Oudeschild", + "highlight": "Barentszstraat - Oudeschild", "href": "https://example.com/ogc/v1/collections/addresses/items/548?f=json", "score": 0.14426951110363007 }, @@ -65,7 +65,7 @@ "collectionId": "addresses", "collectionVersion": "1", "displayName": "Bolwerk - Oudeschild", - "highlight": "Bolwerk, 1792AS Oudeschild", + "highlight": "Bolwerk - Oudeschild", "href": "https://example.com/ogc/v1/collections/addresses/items/1050?f=json", "score": 0.14426951110363007 }, @@ -113,7 +113,7 @@ "collectionId": "addresses", "collectionVersion": "1", "displayName": "Commandeurssingel - Oudeschild", - "highlight": "Commandeurssingel, 1792AV Oudeschild", + "highlight": "Commandeurssingel - Oudeschild", "href": "https://example.com/ogc/v1/collections/addresses/items/2725?f=json", "score": 0.14426951110363007 }, @@ -161,7 +161,7 @@ "collectionId": "addresses", "collectionVersion": "1", "displayName": "De Houtmanstraat - Oudeschild", - "highlight": "De Houtmanstraat, 1792BC Oudeschild", + "highlight": "De Houtmanstraat - Oudeschild", "href": "https://example.com/ogc/v1/collections/addresses/items/2921?f=json", "score": 0.12426698952913284 }, @@ -209,7 +209,7 @@ "collectionId": "addresses", "collectionVersion": "1", "displayName": "De Ruyterstraat - Oudeschild", - "highlight": "De Ruyterstraat, 1792AP Oudeschild", + "highlight": "De Ruyterstraat - Oudeschild", "href": "https://example.com/ogc/v1/collections/addresses/items/3049?f=json", "score": 0.12426698952913284 }, @@ -257,7 +257,7 @@ "collectionId": "addresses", "collectionVersion": "1", "displayName": "De Wittstraat - Oudeschild", - "highlight": "De Wittstraat, 1792BP Oudeschild", + "highlight": "De Wittstraat - Oudeschild", "href": "https://example.com/ogc/v1/collections/addresses/items/3041?f=json", "score": 0.12426698952913284 }, From 5c923f447dd404a728dc7740669e235ae839a564 Mon Sep 17 00:00:00 2001 From: Richard Kettelerij Date: Tue, 10 Dec 2024 17:00:20 +0100 Subject: [PATCH 15/38] feat: add search endpoint - merge --- internal/search/main_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/search/main_test.go b/internal/search/main_test.go index 9a86bbc..58bc44d 100644 --- a/internal/search/main_test.go +++ b/internal/search/main_test.go @@ -67,7 +67,8 @@ func TestSearch(t *testing.T) { assert.NoError(t, err) collection := config.CollectionByID(conf, "addresses") table := config.FeatureTable{Name: "addresses", FID: "fid", Geom: "geom"} - err = etl.ImportFile(*collection, testSearchIndex, "internal/etl/testdata/addresses-crs84.gpkg", table, 1000, dbConn) + err = etl.ImportFile(*collection, testSearchIndex, "internal/etl/testdata/addresses-crs84.gpkg", + "internal/etl/testdata/substitution.csv", table, 5000, dbConn) assert.NoError(t, err) // run test cases From f5805c97f3ccd5ad50ad1b67e5c8898a5cf88e3c Mon Sep 17 00:00:00 2001 From: Richard Kettelerij Date: Tue, 10 Dec 2024 17:14:58 +0100 Subject: [PATCH 16/38] feat: add search endpoint - linting --- cmd/main.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 5233eff..90a2ab1 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -117,11 +117,11 @@ var ( EnvVars: []string{strcase.ToScreamingSnake(dbPortFlag)}, }, dbNameFlag: &cli.StringFlag{ - Name: dbNameFlag, - Usage: "Connect to this database", - Value: "postgres", + Name: dbNameFlag, + Usage: "Connect to this database", + Value: "postgres", Required: false, - EnvVars: []string{strcase.ToScreamingSnake(dbNameFlag)}, + EnvVars: []string{strcase.ToScreamingSnake(dbNameFlag)}, }, dbSslModeFlag: &cli.StringFlag{ Name: dbSslModeFlag, From 6806f811e2f83ee674458a71341732c365ac0aee Mon Sep 17 00:00:00 2001 From: Richard Kettelerij Date: Wed, 11 Dec 2024 15:29:04 +0100 Subject: [PATCH 17/38] feat: add search endpoint - refactor magic strings to constants + split method --- internal/search/datasources/datasource.go | 9 +- .../search/datasources/postgres/postgres.go | 23 ++--- internal/search/domain/search_props.go | 11 +++ internal/search/main.go | 89 ++++++++++--------- 4 files changed, 77 insertions(+), 55 deletions(-) create mode 100644 internal/search/domain/search_props.go diff --git a/internal/search/datasources/datasource.go b/internal/search/datasources/datasource.go index 80c7824..9de3972 100644 --- a/internal/search/datasources/datasource.go +++ b/internal/search/datasources/datasource.go @@ -9,8 +9,15 @@ import ( // Datasource knows how make different kinds of queries/actions on the underlying actual datastore. // This abstraction allows the rest of the system to stay datastore agnostic. type Datasource interface { - Suggest(ctx context.Context, searchTerm string, collections map[string]map[string]string, srid domain.SRID, limit int) (*domain.FeatureCollection, error) + Search(ctx context.Context, searchTerm string, collections CollectionsWithParams, srid domain.SRID, limit int) (*domain.FeatureCollection, error) // Close closes (connections to) the datasource gracefully Close() } + +// CollectionsWithParams collection name with associated CollectionParams +// These are provided though a URL query string as "deep object" params, e.g. paramName[prop1]=value1¶mName[prop2]=value2&.... +type CollectionsWithParams map[string]CollectionParams + +// CollectionParams parameter key with associated value +type CollectionParams map[string]string diff --git a/internal/search/datasources/postgres/postgres.go b/internal/search/datasources/postgres/postgres.go index 1080107..af0cbe9 100644 --- a/internal/search/datasources/postgres/postgres.go +++ b/internal/search/datasources/postgres/postgres.go @@ -5,7 +5,8 @@ import ( "fmt" "log" - "github.com/PDOK/gomagpie/internal/search/domain" + "github.com/PDOK/gomagpie/internal/search/datasources" + d "github.com/PDOK/gomagpie/internal/search/domain" "github.com/jackc/pgx/v5" pggeom "github.com/twpayne/go-geom" "github.com/twpayne/go-geom/encoding/geojson" @@ -40,8 +41,8 @@ func (p *Postgres) Close() { _ = p.db.Close(p.ctx) } -func (p *Postgres) Suggest(ctx context.Context, searchTerm string, collections map[string]map[string]string, - srid domain.SRID, limit int) (*domain.FeatureCollection, error) { +func (p *Postgres) Search(ctx context.Context, searchTerm string, collections datasources.CollectionsWithParams, + srid d.SRID, limit int) (*d.FeatureCollection, error) { queryCtx, cancel := context.WithTimeout(ctx, p.queryTimeout) defer cancel() @@ -63,7 +64,7 @@ func (p *Postgres) Suggest(ctx context.Context, searchTerm string, collections m defer rows.Close() // Turn rows into FeatureCollection - fc := domain.FeatureCollection{Features: make([]*domain.Feature, 0)} + fc := d.FeatureCollection{Features: make([]*d.Feature, 0)} for rows.Next() { var displayName, highlightedText, featureID, collectionID, collectionVersion, geomType string var rank float64 @@ -77,16 +78,16 @@ func (p *Postgres) Suggest(ctx context.Context, searchTerm string, collections m if err != nil { return nil, err } - f := domain.Feature{ + f := d.Feature{ ID: featureID, Geometry: *geojsonGeom, Properties: map[string]any{ - "collectionId": collectionID, - "collectionVersion": collectionVersion, - "collectionGeometryType": geomType, - "displayName": displayName, - "highlight": highlightedText, - "score": rank, + d.PropCollectionID: collectionID, + d.PropCollectionVersion: collectionVersion, + d.PropGeomType: geomType, + d.PropDisplayName: displayName, + d.PropHighlight: highlightedText, + d.PropScore: rank, }, } log.Printf("collections %s, srid %v", collections, srid) // TODO use params diff --git a/internal/search/domain/search_props.go b/internal/search/domain/search_props.go new file mode 100644 index 0000000..0c75efd --- /dev/null +++ b/internal/search/domain/search_props.go @@ -0,0 +1,11 @@ +package domain + +const ( + PropCollectionID = "collectionId" + PropCollectionVersion = "collectionVersion" + PropGeomType = "collectionGeometryType" + PropDisplayName = "displayName" + PropHighlight = "highlight" + PropScore = "score" + PropHref = "href" +) diff --git a/internal/search/main.go b/internal/search/main.go index a9b02be..2a3d427 100644 --- a/internal/search/main.go +++ b/internal/search/main.go @@ -56,49 +56,14 @@ func (s *Search) Search() http.HandlerFunc { engine.RenderProblem(engine.ProblemBadRequest, w, err.Error()) return } - fc, err := s.datasource.Suggest(r.Context(), searchTerm, collections, outputSRID, limit) + fc, err := s.datasource.Search(r.Context(), searchTerm, collections, outputSRID, limit) if err != nil { handleQueryError(w, err) return } - - for _, feat := range fc.Features { - collectionID, ok := feat.Properties["collectionId"] - if !ok || collectionID == "" { - log.Printf("collection reference not found in feature %s", feat.ID) - engine.RenderProblem(engine.ProblemServerError, w) - return - } - collection := config.CollectionByID(s.engine.Config, collectionID.(string)) - if collection.Search != nil { - for _, ogcColl := range collection.Search.OGCCollections { - geomType, ok := feat.Properties["collectionGeometryType"] - if !ok || geomType == "" { - log.Printf("geometry type not found in feature %s", feat.ID) - engine.RenderProblem(engine.ProblemServerError, w) - return - } - if strings.EqualFold(ogcColl.GeometryType, geomType.(string)) { - href, err := url.JoinPath(ogcColl.APIBaseURL.String(), "collections", ogcColl.CollectionID, "items", feat.ID) - if err != nil { - log.Printf("failed to construct API url %v", err) - engine.RenderProblem(engine.ProblemServerError, w) - } - href += "?f=json" - - // add href to feature both in GeoJSON properties (for broad compatibility and in line with OGC API Features part 5) and as a Link. - feat.Properties["href"] = href - feat.Links = []domain.Link{ - domain.Link{ - Rel: "canonical", - Title: "The actual feature in the corresponding OGC API", - Type: "application/geo+json", - Href: href, - }, - } - } - } - } + if err = s.enrichFeaturesWithHref(fc); err != nil { + engine.RenderProblem(engine.ProblemServerError, w, err.Error()) + return } format := s.engine.CN.NegotiateFormat(r) @@ -112,21 +77,59 @@ func (s *Search) Search() http.HandlerFunc { } } -func parseQueryParams(query url.Values) (collectionsWithParams map[string]map[string]string, searchTerm string, outputSRID domain.SRID, limit int, err error) { +func (s *Search) enrichFeaturesWithHref(fc *domain.FeatureCollection) error { + for _, feat := range fc.Features { + collectionID, ok := feat.Properties[domain.PropCollectionID] + if !ok || collectionID == "" { + return fmt.Errorf("collection reference not found in feature %s", feat.ID) + } + collection := config.CollectionByID(s.engine.Config, collectionID.(string)) + if collection.Search != nil { + for _, ogcColl := range collection.Search.OGCCollections { + geomType, ok := feat.Properties[domain.PropGeomType] + if !ok || geomType == "" { + return fmt.Errorf("geometry type not found in feature %s", feat.ID) + } + if strings.EqualFold(ogcColl.GeometryType, geomType.(string)) { + href, err := url.JoinPath(ogcColl.APIBaseURL.String(), "collections", ogcColl.CollectionID, "items", feat.ID) + if err != nil { + return fmt.Errorf("failed to construct API url %w", err) + } + href += "?f=json" + + // add href to feature both in GeoJSON properties (for broad compatibility and in line with OGC API Features part 5) and as a Link. + feat.Properties[domain.PropHref] = href + feat.Links = []domain.Link{ + { + Rel: "canonical", + Title: "The actual feature in the corresponding OGC API", + Type: "application/geo+json", + Href: href, + }, + } + } + } + } + } + return nil +} + +func parseQueryParams(query url.Values) (collections ds.CollectionsWithParams, searchTerm string, outputSRID domain.SRID, limit int, err error) { err = validateNoUnknownParams(query) if err != nil { return } searchTerm, searchTermErr := parseSearchTerm(query) - collectionsWithParams = parseCollectionDeepObjectParams(query) + collections = parseDeepObjectParams(query) outputSRID, outputSRIDErr := parseCrsToSRID(query, crsParam) limit, limitErr := parseLimit(query) err = errors.Join(searchTermErr, limitErr, outputSRIDErr) return } -func parseCollectionDeepObjectParams(query url.Values) map[string]map[string]string { - deepObjectParams := make(map[string]map[string]string, len(query)) +// Parse "deep object" params, e.g. paramName[prop1]=value1¶mName[prop2]=value2&.... +func parseDeepObjectParams(query url.Values) ds.CollectionsWithParams { + deepObjectParams := make(ds.CollectionsWithParams, len(query)) for key, values := range query { if strings.Contains(key, "[") { // Extract deepObject parameters From 5b4fefc4ebde45976896089b0c1f3557045c0e9e Mon Sep 17 00:00:00 2001 From: Richard Kettelerij Date: Wed, 11 Dec 2024 15:36:38 +0100 Subject: [PATCH 18/38] feat: add search endpoint - use prepared statement for query term --- .../search/datasources/postgres/postgres.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/internal/search/datasources/postgres/postgres.go b/internal/search/datasources/postgres/postgres.go index af0cbe9..6b1c923 100644 --- a/internal/search/datasources/postgres/postgres.go +++ b/internal/search/datasources/postgres/postgres.go @@ -54,12 +54,12 @@ func (p *Postgres) Search(ctx context.Context, searchTerm string, collections da terms[i] = term + ":*" } termsConcat := strings.Join(terms, " & ") - searchQuery := makeSearchQuery(termsConcat, p.searchIndex) + query := makeSearchQuery(p.searchIndex) // Execute search query - rows, err := p.db.Query(queryCtx, searchQuery, limit) + rows, err := p.db.Query(queryCtx, query, limit, termsConcat) if err != nil { - return nil, fmt.Errorf("query '%s' failed: %w", searchQuery, err) + return nil, fmt.Errorf("query '%s' failed: %w", query, err) } defer rows.Close() @@ -97,7 +97,7 @@ func (p *Postgres) Search(ctx context.Context, searchTerm string, collections da return &fc, queryCtx.Err() } -func makeSearchQuery(term string, index string) string { +func makeSearchQuery(index string) string { // language=postgresql return fmt.Sprintf(` select r.display_name as display_name, @@ -110,13 +110,13 @@ func makeSearchQuery(term string, index string) string { max(r.highlighted_text) as highlighted_text from ( select display_name, feature_id, collection_id, collection_version, geometry_type, bbox, - ts_rank_cd(ts, to_tsquery('%[1]s'), 1) as rank, - ts_headline('dutch', display_name, to_tsquery('%[1]s')) as highlighted_text - from %[2]s - where ts @@ to_tsquery('%[1]s') + ts_rank_cd(ts, to_tsquery($2), 1) as rank, + ts_headline('dutch', display_name, to_tsquery($2)) as highlighted_text + from %[1]s + where ts @@ to_tsquery($2) limit 500 ) r group by r.display_name order by rank desc, display_name asc - limit $1`, term, index) + limit $1`, index) } From 06879d2846df3a7266e95ca21cbab6c872be31e5 Mon Sep 17 00:00:00 2001 From: Richard Kettelerij Date: Wed, 11 Dec 2024 17:11:49 +0100 Subject: [PATCH 19/38] feat: add search endpoint - search in specific collection --- .../search/datasources/postgres/postgres.go | 24 ++++++++++++++----- internal/search/main_test.go | 4 ++-- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/internal/search/datasources/postgres/postgres.go b/internal/search/datasources/postgres/postgres.go index 6b1c923..b2db214 100644 --- a/internal/search/datasources/postgres/postgres.go +++ b/internal/search/datasources/postgres/postgres.go @@ -5,6 +5,7 @@ import ( "fmt" "log" + "github.com/PDOK/gomagpie/internal/engine/util" "github.com/PDOK/gomagpie/internal/search/datasources" d "github.com/PDOK/gomagpie/internal/search/domain" "github.com/jackc/pgx/v5" @@ -54,10 +55,10 @@ func (p *Postgres) Search(ctx context.Context, searchTerm string, collections da terms[i] = term + ":*" } termsConcat := strings.Join(terms, " & ") - query := makeSearchQuery(p.searchIndex) + query, args := makeSearchQuery(p.searchIndex, limit, termsConcat, util.Keys(collections)) // Execute search query - rows, err := p.db.Query(queryCtx, query, limit, termsConcat) + rows, err := p.db.Query(queryCtx, query, args...) if err != nil { return nil, fmt.Errorf("query '%s' failed: %w", query, err) } @@ -97,9 +98,18 @@ func (p *Postgres) Search(ctx context.Context, searchTerm string, collections da return &fc, queryCtx.Err() } -func makeSearchQuery(index string) string { +func makeSearchQuery(index string, limit int, terms string, collections []string) (string, []any) { + args := []any{limit, terms} + + collectionsClause := "" + if len(collections) > 0 { + // language=postgresql + collectionsClause = `and collection_id = any($3)` + args = append(args, collections) + } + // language=postgresql - return fmt.Sprintf(` + query := fmt.Sprintf(` select r.display_name as display_name, max(r.feature_id) as feature_id, max(r.collection_id) as collection_id, @@ -113,10 +123,12 @@ func makeSearchQuery(index string) string { ts_rank_cd(ts, to_tsquery($2), 1) as rank, ts_headline('dutch', display_name, to_tsquery($2)) as highlighted_text from %[1]s - where ts @@ to_tsquery($2) + where ts @@ to_tsquery($2) %[2]s limit 500 ) r group by r.display_name order by rank desc, display_name asc - limit $1`, index) + limit $1`, index, collectionsClause) // don't add user input here, use $X params for user input! + + return query, args } diff --git a/internal/search/main_test.go b/internal/search/main_test.go index 58bc44d..80f9d35 100644 --- a/internal/search/main_test.go +++ b/internal/search/main_test.go @@ -105,9 +105,9 @@ func TestSearch(t *testing.T) { }, }, { - name: "Search: Den. With deepCopy params", + name: "Search: Den. With deepCopy params for a single collection", fields: fields{ - url: "http://localhost:8080/search?q=\"Den\"&weg[version]=2&weg[relevance]=0.8&adres[version]=1&adres[relevance]=1&limit=10&f=json&crs=http%3A%2F%2Fwww.opengis.net%2Fdef%2Fcrs%2FEPSG%2F0%2F28992", + url: "http://localhost:8080/search?q=\"Den\"&addresses[version]=2&addresses[relevance]=0.8&limit=10&f=json&crs=http%3A%2F%2Fwww.opengis.net%2Fdef%2Fcrs%2FEPSG%2F0%2F28992", }, want: want{ body: "internal/search/testdata/expected-search-den-deepcopy.json", From b98f2657982680668e91683b994930b9a181ad64 Mon Sep 17 00:00:00 2001 From: Richard Kettelerij Date: Thu, 12 Dec 2024 16:08:09 +0100 Subject: [PATCH 20/38] feat: add search endpoint - make collections parameter(s) required --- .../search/datasources/postgres/postgres.go | 20 +- internal/search/main.go | 5 + internal/search/main_test.go | 19 +- ...xpected-search-den-single-collection.json} | 160 +++---- .../search/testdata/expected-search-den.json | 399 ------------------ .../expected-search-no-collection.json | 6 + .../testdata/expected-search-oudeschild.json | 303 ------------- 7 files changed, 107 insertions(+), 805 deletions(-) rename internal/search/testdata/{expected-search-den-deepcopy.json => expected-search-den-single-collection.json} (75%) delete mode 100644 internal/search/testdata/expected-search-den.json create mode 100644 internal/search/testdata/expected-search-no-collection.json delete mode 100644 internal/search/testdata/expected-search-oudeschild.json diff --git a/internal/search/datasources/postgres/postgres.go b/internal/search/datasources/postgres/postgres.go index b2db214..4fee71e 100644 --- a/internal/search/datasources/postgres/postgres.go +++ b/internal/search/datasources/postgres/postgres.go @@ -48,14 +48,13 @@ func (p *Postgres) Search(ctx context.Context, searchTerm string, collections da queryCtx, cancel := context.WithTimeout(ctx, p.queryTimeout) defer cancel() - // Prepare dynamic full-text search query // Split terms by spaces and append :* to each term terms := strings.Fields(searchTerm) for i, term := range terms { terms[i] = term + ":*" } termsConcat := strings.Join(terms, " & ") - query, args := makeSearchQuery(p.searchIndex, limit, termsConcat, util.Keys(collections)) + query, args := makeSearchQuery(p.searchIndex, limit, termsConcat, util.Keys(collections), srid) // Execute search query rows, err := p.db.Query(queryCtx, query, args...) @@ -98,15 +97,8 @@ func (p *Postgres) Search(ctx context.Context, searchTerm string, collections da return &fc, queryCtx.Err() } -func makeSearchQuery(index string, limit int, terms string, collections []string) (string, []any) { - args := []any{limit, terms} - - collectionsClause := "" - if len(collections) > 0 { - // language=postgresql - collectionsClause = `and collection_id = any($3)` - args = append(args, collections) - } +func makeSearchQuery(index string, limit int, terms string, collections []string, srid d.SRID) (string, []any) { + args := []any{limit, terms, collections} // language=postgresql query := fmt.Sprintf(` @@ -115,7 +107,7 @@ func makeSearchQuery(index string, limit int, terms string, collections []string max(r.collection_id) as collection_id, max(r.collection_version) as collection_version, max(r.geometry_type) as geometry_type, - cast(max(r.bbox) as geometry) as bbox, + cast(st_transform(max(r.bbox), %[2]d) as geometry) as bbox, max(r.rank) as rank, max(r.highlighted_text) as highlighted_text from ( @@ -123,12 +115,12 @@ func makeSearchQuery(index string, limit int, terms string, collections []string ts_rank_cd(ts, to_tsquery($2), 1) as rank, ts_headline('dutch', display_name, to_tsquery($2)) as highlighted_text from %[1]s - where ts @@ to_tsquery($2) %[2]s + where ts @@ to_tsquery($2) and collection_id = any($3) limit 500 ) r group by r.display_name order by rank desc, display_name asc - limit $1`, index, collectionsClause) // don't add user input here, use $X params for user input! + limit $1`, index, srid) // don't add user input here, use $X params for user input! return query, args } diff --git a/internal/search/main.go b/internal/search/main.go index 2a3d427..7e5aa74 100644 --- a/internal/search/main.go +++ b/internal/search/main.go @@ -121,6 +121,11 @@ func parseQueryParams(query url.Values) (collections ds.CollectionsWithParams, s } searchTerm, searchTermErr := parseSearchTerm(query) collections = parseDeepObjectParams(query) + if len(collections) == 0 { + return nil, "", 0, 0, errors.New( + "no collection(s) specified in request, specify at least one collection and version. " + + "For example: foo[version]=1&bar[version]=2 where 'foo' and 'bar' are collection names") + } outputSRID, outputSRIDErr := parseCrsToSRID(query, crsParam) limit, limitErr := parseLimit(query) err = errors.Join(searchTermErr, limitErr, outputSRIDErr) diff --git a/internal/search/main_test.go b/internal/search/main_test.go index 80f9d35..845c34b 100644 --- a/internal/search/main_test.go +++ b/internal/search/main_test.go @@ -85,32 +85,32 @@ func TestSearch(t *testing.T) { want want }{ { - name: "Search: Oudeschild", + name: "Fail on search without collection parameter(s)", fields: fields{ url: "http://localhost:8080/search?q=\"Oudeschild\"&limit=50", }, want: want{ - body: "internal/search/testdata/expected-search-oudeschild.json", - statusCode: http.StatusOK, + body: "internal/search/testdata/expected-search-no-collection.json", + statusCode: http.StatusBadRequest, }, }, { - name: "Search: Den ", + name: "Search: 'Den' for a single collection", fields: fields{ - url: "http://localhost:8080/search?q=\"Den\"&limit=50", + url: "http://localhost:8080/search?q=\"Den\"&addresses[version]=2&addresses[relevance]=0.8&limit=10&f=json&crs=http%3A%2F%2Fwww.opengis.net%2Fdef%2Fcrs%2FEPSG%2F0%2F28992", }, want: want{ - body: "internal/search/testdata/expected-search-den.json", + body: "internal/search/testdata/expected-search-den-single-collection.json", statusCode: http.StatusOK, }, }, { - name: "Search: Den. With deepCopy params for a single collection", + name: "Search: 'Den' for multiple collections (with one not existing collection, so same output as single collection)", fields: fields{ - url: "http://localhost:8080/search?q=\"Den\"&addresses[version]=2&addresses[relevance]=0.8&limit=10&f=json&crs=http%3A%2F%2Fwww.opengis.net%2Fdef%2Fcrs%2FEPSG%2F0%2F28992", + url: "http://localhost:8080/search?q=\"Den\"&addresses[version]=2&addresses[relevance]=0.8&foo[version]=2&foo[relevance]=0.8&limit=10&f=json&crs=http%3A%2F%2Fwww.opengis.net%2Fdef%2Fcrs%2FEPSG%2F0%2F28992", }, want: want{ - body: "internal/search/testdata/expected-search-den-deepcopy.json", + body: "internal/search/testdata/expected-search-den-single-collection.json", statusCode: http.StatusOK, }, }, @@ -119,6 +119,7 @@ func TestSearch(t *testing.T) { t.Run(tt.name, func(t *testing.T) { // mock time now = func() time.Time { return time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC) } + engine.Now = now // given available server rr, ts := createMockServer() diff --git a/internal/search/testdata/expected-search-den-deepcopy.json b/internal/search/testdata/expected-search-den-single-collection.json similarity index 75% rename from internal/search/testdata/expected-search-den-deepcopy.json rename to internal/search/testdata/expected-search-den-single-collection.json index 0dbc6f9..2b86f9d 100644 --- a/internal/search/testdata/expected-search-den-deepcopy.json +++ b/internal/search/testdata/expected-search-den-single-collection.json @@ -26,24 +26,24 @@ "coordinates": [ [ [ - 4.701721422439945, - 52.9619223105808 + 108940.29073913672, + 552985.5894477183 ], [ - 4.901721422439945, - 52.9619223105808 + 122378.69131049563, + 552876.583266793 ], [ - 4.901721422439945, - 53.161922310580806 + 122528.48208323204, + 575133.1149730522 ], [ - 4.701721422439945, - 53.161922310580806 + 109151.81060375126, + 575241.7668599344 ], [ - 4.701721422439945, - 52.9619223105808 + 108940.29073913672, + 552985.5894477183 ] ] ] @@ -74,24 +74,24 @@ "coordinates": [ [ [ - 4.699813158490893, - 52.95463219709524 + 108804.34590509304, + 552175.5839961797 ], [ - 4.899813158490892, - 52.95463219709524 + 122244.99285675734, + 552066.209512857 ], [ - 4.899813158490892, - 53.154632197095246 + 122395.36487788748, + 574322.6888311192 ], [ - 4.699813158490893, - 53.154632197095246 + 109016.44385035255, + 574431.7079399591 ], [ - 4.699813158490893, - 52.95463219709524 + 108804.34590509304, + 552175.5839961797 ] ] ] @@ -122,24 +122,24 @@ "coordinates": [ [ [ - 4.680059099895046, - 52.95346592050607 + 107475.55019422778, + 552058.6284843497 ], [ - 4.880059099895045, - 52.95346592050607 + 120916.53534647425, + 551945.5725395872 ], [ - 4.880059099895045, - 53.15346592050607 + 121073.00275881319, + 574202.0142312867 ], [ - 4.680059099895046, - 53.15346592050607 + 107693.74304765201, + 574314.7028564449 ], [ - 4.680059099895046, - 52.95346592050607 + 107475.55019422778, + 552058.6284843497 ] ] ] @@ -170,24 +170,24 @@ "coordinates": [ [ [ - 4.68911630577304, - 52.92449928128154 + 108053.06245183083, + 548829.392089723 ], [ - 4.889116305773039, - 52.92449928128154 + 121502.9902353966, + 548717.9709095402 ], [ - 4.889116305773039, - 53.124499281281544 + 121656.63098808826, + 570974.2312531795 ], [ - 4.68911630577304, - 53.124499281281544 + 108268.41604413971, + 571085.2908990474 ], [ - 4.68911630577304, - 52.92449928128154 + 108053.06245183083, + 548829.392089723 ] ] ] @@ -218,24 +218,24 @@ "coordinates": [ [ [ - 4.6548723261037095, - 52.94811743920973 + 105776.85318090196, + 551480.3458531145 ], [ - 4.854872326103709, - 52.94811743920973 + 119219.4593937051, + 551362.588411998 ], [ - 4.854872326103709, - 53.148117439209734 + 119383.69392558266, + 573618.9542783926 ], [ - 4.6548723261037095, - 53.148117439209734 + 106002.81086040766, + 573736.3292134333 ], [ - 4.6548723261037095, - 52.94811743920973 + 105776.85318090196, + 551480.3458531145 ] ] ] @@ -266,24 +266,24 @@ "coordinates": [ [ [ - 4.690892824472019, - 52.95558352001795 + 108205.89714467606, + 552287.1906286391 ], [ - 4.890892824472019, - 52.95558352001795 + 121646.24107836874, + 552176.1563847166 ], [ - 4.890892824472019, - 53.155583520017956 + 121799.36720075001, + 574432.6288915055 ], [ - 4.690892824472019, - 53.155583520017956 + 108420.74961558703, + 574543.3023513828 ], [ - 4.690892824472019, - 52.95558352001795 + 108205.89714467606, + 552287.1906286391 ] ] ] @@ -314,24 +314,24 @@ "coordinates": [ [ [ - 4.696235388824104, - 52.95196001510249 + 108561.06373114887, + 551880.5267859272 ], [ - 4.8962353888241035, - 52.95196001510249 + 122002.53096929599, + 551770.481156097 ], [ - 4.8962353888241035, - 53.151960015102496 + 122154.00434113854, + 574026.9370521428 ], [ - 4.696235388824104, - 53.151960015102496 + 108774.26186957877, + 574136.6251694224 ], [ - 4.696235388824104, - 52.95196001510249 + 108561.06373114887, + 551880.5267859272 ] ] ] @@ -362,24 +362,24 @@ "coordinates": [ [ [ - 4.692873779103581, - 52.950932925919574 + 108334.04142692257, + 551768.4032578692 ], [ - 4.892873779103581, - 52.950932925919574 + 121775.82179598782, + 551657.7296376136 ], [ - 4.892873779103581, - 53.15093292591958 + 121928.33153477598, + 573914.1735634004 ], [ - 4.692873779103581, - 53.15093292591958 + 108548.27548981196, + 574024.4876471001 ], [ - 4.692873779103581, - 52.950932925919574 + 108334.04142692257, + 551768.4032578692 ] ] ] diff --git a/internal/search/testdata/expected-search-den.json b/internal/search/testdata/expected-search-den.json deleted file mode 100644 index 0dbc6f9..0000000 --- a/internal/search/testdata/expected-search-den.json +++ /dev/null @@ -1,399 +0,0 @@ -{ - "type": "FeatureCollection", - "timeStamp": "2000-01-01T00:00:00Z", - "links": [ - { - "rel": "self", - "title": "This document as GeoJSON", - "type": "application/geo+json", - "href": "http://localhost:8080/search?f=json" - } - ], - "features": [ - { - "type": "Feature", - "properties": { - "collectionGeometryType": "POINT", - "collectionId": "addresses", - "collectionVersion": "1", - "displayName": "Abbewaal - Den Burg", - "highlight": "Abbewaal - Den Burg", - "href": "https://example.com/ogc/v1/collections/addresses/items/99?f=json", - "score": 0.11162212491035461 - }, - "geometry": { - "type": "Polygon", - "coordinates": [ - [ - [ - 4.701721422439945, - 52.9619223105808 - ], - [ - 4.901721422439945, - 52.9619223105808 - ], - [ - 4.901721422439945, - 53.161922310580806 - ], - [ - 4.701721422439945, - 53.161922310580806 - ], - [ - 4.701721422439945, - 52.9619223105808 - ] - ] - ] - }, - "id": "99", - "links": [ - { - "rel": "canonical", - "title": "The actual feature in the corresponding OGC API", - "type": "application/geo+json", - "href": "https://example.com/ogc/v1/collections/addresses/items/99?f=json" - } - ] - }, - { - "type": "Feature", - "properties": { - "collectionGeometryType": "POINT", - "collectionId": "addresses", - "collectionVersion": "1", - "displayName": "Achterom - Den Burg", - "highlight": "Achterom - Den Burg", - "href": "https://example.com/ogc/v1/collections/addresses/items/114?f=json", - "score": 0.11162212491035461 - }, - "geometry": { - "type": "Polygon", - "coordinates": [ - [ - [ - 4.699813158490893, - 52.95463219709524 - ], - [ - 4.899813158490892, - 52.95463219709524 - ], - [ - 4.899813158490892, - 53.154632197095246 - ], - [ - 4.699813158490893, - 53.154632197095246 - ], - [ - 4.699813158490893, - 52.95463219709524 - ] - ] - ] - }, - "id": "114", - "links": [ - { - "rel": "canonical", - "title": "The actual feature in the corresponding OGC API", - "type": "application/geo+json", - "href": "https://example.com/ogc/v1/collections/addresses/items/114?f=json" - } - ] - }, - { - "type": "Feature", - "properties": { - "collectionGeometryType": "POINT", - "collectionId": "addresses", - "collectionVersion": "1", - "displayName": "Akenbuurt - Den Burg", - "highlight": "Akenbuurt - Den Burg", - "href": "https://example.com/ogc/v1/collections/addresses/items/46?f=json", - "score": 0.11162212491035461 - }, - "geometry": { - "type": "Polygon", - "coordinates": [ - [ - [ - 4.680059099895046, - 52.95346592050607 - ], - [ - 4.880059099895045, - 52.95346592050607 - ], - [ - 4.880059099895045, - 53.15346592050607 - ], - [ - 4.680059099895046, - 53.15346592050607 - ], - [ - 4.680059099895046, - 52.95346592050607 - ] - ] - ] - }, - "id": "46", - "links": [ - { - "rel": "canonical", - "title": "The actual feature in the corresponding OGC API", - "type": "application/geo+json", - "href": "https://example.com/ogc/v1/collections/addresses/items/46?f=json" - } - ] - }, - { - "type": "Feature", - "properties": { - "collectionGeometryType": "POINT", - "collectionId": "addresses", - "collectionVersion": "1", - "displayName": "Amaliaweg - Den Hoorn", - "highlight": "Amaliaweg - Den Hoorn", - "href": "https://example.com/ogc/v1/collections/addresses/items/50?f=json", - "score": 0.11162212491035461 - }, - "geometry": { - "type": "Polygon", - "coordinates": [ - [ - [ - 4.68911630577304, - 52.92449928128154 - ], - [ - 4.889116305773039, - 52.92449928128154 - ], - [ - 4.889116305773039, - 53.124499281281544 - ], - [ - 4.68911630577304, - 53.124499281281544 - ], - [ - 4.68911630577304, - 52.92449928128154 - ] - ] - ] - }, - "id": "50", - "links": [ - { - "rel": "canonical", - "title": "The actual feature in the corresponding OGC API", - "type": "application/geo+json", - "href": "https://example.com/ogc/v1/collections/addresses/items/50?f=json" - } - ] - }, - { - "type": "Feature", - "properties": { - "collectionGeometryType": "POINT", - "collectionId": "addresses", - "collectionVersion": "1", - "displayName": "Bakkenweg - Den Hoorn", - "highlight": "Bakkenweg - Den Hoorn", - "href": "https://example.com/ogc/v1/collections/addresses/items/520?f=json", - "score": 0.11162212491035461 - }, - "geometry": { - "type": "Polygon", - "coordinates": [ - [ - [ - 4.6548723261037095, - 52.94811743920973 - ], - [ - 4.854872326103709, - 52.94811743920973 - ], - [ - 4.854872326103709, - 53.148117439209734 - ], - [ - 4.6548723261037095, - 53.148117439209734 - ], - [ - 4.6548723261037095, - 52.94811743920973 - ] - ] - ] - }, - "id": "520", - "links": [ - { - "rel": "canonical", - "title": "The actual feature in the corresponding OGC API", - "type": "application/geo+json", - "href": "https://example.com/ogc/v1/collections/addresses/items/520?f=json" - } - ] - }, - { - "type": "Feature", - "properties": { - "collectionGeometryType": "POINT", - "collectionId": "addresses", - "collectionVersion": "1", - "displayName": "Beatrixlaan - Den Burg", - "highlight": "Beatrixlaan - Den Burg", - "href": "https://example.com/ogc/v1/collections/addresses/items/591?f=json", - "score": 0.11162212491035461 - }, - "geometry": { - "type": "Polygon", - "coordinates": [ - [ - [ - 4.690892824472019, - 52.95558352001795 - ], - [ - 4.890892824472019, - 52.95558352001795 - ], - [ - 4.890892824472019, - 53.155583520017956 - ], - [ - 4.690892824472019, - 53.155583520017956 - ], - [ - 4.690892824472019, - 52.95558352001795 - ] - ] - ] - }, - "id": "591", - "links": [ - { - "rel": "canonical", - "title": "The actual feature in the corresponding OGC API", - "type": "application/geo+json", - "href": "https://example.com/ogc/v1/collections/addresses/items/591?f=json" - } - ] - }, - { - "type": "Feature", - "properties": { - "collectionGeometryType": "POINT", - "collectionId": "addresses", - "collectionVersion": "1", - "displayName": "Ada van Hollandstraat - Den Burg", - "highlight": "Ada van Hollandstraat - Den Burg", - "href": "https://example.com/ogc/v1/collections/addresses/items/26?f=json", - "score": 0.09617967158555984 - }, - "geometry": { - "type": "Polygon", - "coordinates": [ - [ - [ - 4.696235388824104, - 52.95196001510249 - ], - [ - 4.8962353888241035, - 52.95196001510249 - ], - [ - 4.8962353888241035, - 53.151960015102496 - ], - [ - 4.696235388824104, - 53.151960015102496 - ], - [ - 4.696235388824104, - 52.95196001510249 - ] - ] - ] - }, - "id": "26", - "links": [ - { - "rel": "canonical", - "title": "The actual feature in the corresponding OGC API", - "type": "application/geo+json", - "href": "https://example.com/ogc/v1/collections/addresses/items/26?f=json" - } - ] - }, - { - "type": "Feature", - "properties": { - "collectionGeometryType": "POINT", - "collectionId": "addresses", - "collectionVersion": "1", - "displayName": "Anne Frankstraat - Den Burg", - "highlight": "Anne Frankstraat - Den Burg", - "href": "https://example.com/ogc/v1/collections/addresses/items/474?f=json", - "score": 0.09617967158555984 - }, - "geometry": { - "type": "Polygon", - "coordinates": [ - [ - [ - 4.692873779103581, - 52.950932925919574 - ], - [ - 4.892873779103581, - 52.950932925919574 - ], - [ - 4.892873779103581, - 53.15093292591958 - ], - [ - 4.692873779103581, - 53.15093292591958 - ], - [ - 4.692873779103581, - 52.950932925919574 - ] - ] - ] - }, - "id": "474", - "links": [ - { - "rel": "canonical", - "title": "The actual feature in the corresponding OGC API", - "type": "application/geo+json", - "href": "https://example.com/ogc/v1/collections/addresses/items/474?f=json" - } - ] - } - ], - "numberReturned": 8 -} diff --git a/internal/search/testdata/expected-search-no-collection.json b/internal/search/testdata/expected-search-no-collection.json new file mode 100644 index 0000000..a23097a --- /dev/null +++ b/internal/search/testdata/expected-search-no-collection.json @@ -0,0 +1,6 @@ +{ + "detail": "no collection(s) specified in request, specify at least one collection and version. For example: foo[version]=1\u0026bar[version]=2 where 'foo' and 'bar' are collection names", + "status": 400, + "timeStamp": "2000-01-01T00:00:00Z", + "title": "Bad Request" +} diff --git a/internal/search/testdata/expected-search-oudeschild.json b/internal/search/testdata/expected-search-oudeschild.json deleted file mode 100644 index 95a38a0..0000000 --- a/internal/search/testdata/expected-search-oudeschild.json +++ /dev/null @@ -1,303 +0,0 @@ -{ - "type": "FeatureCollection", - "timeStamp": "2000-01-01T00:00:00Z", - "links": [ - { - "rel": "self", - "title": "This document as GeoJSON", - "type": "application/geo+json", - "href": "http://localhost:8080/search?f=json" - } - ], - "features": [ - { - "type": "Feature", - "properties": { - "collectionGeometryType": "POINT", - "collectionId": "addresses", - "collectionVersion": "1", - "displayName": "Barentszstraat - Oudeschild", - "highlight": "Barentszstraat - Oudeschild", - "href": "https://example.com/ogc/v1/collections/addresses/items/548?f=json", - "score": 0.14426951110363007 - }, - "geometry": { - "type": "Polygon", - "coordinates": [ - [ - [ - 4.748384044242354, - 52.93901709012591 - ], - [ - 4.948384044242354, - 52.93901709012591 - ], - [ - 4.948384044242354, - 53.13901709012591 - ], - [ - 4.748384044242354, - 53.13901709012591 - ], - [ - 4.748384044242354, - 52.93901709012591 - ] - ] - ] - }, - "id": "548", - "links": [ - { - "rel": "canonical", - "title": "The actual feature in the corresponding OGC API", - "type": "application/geo+json", - "href": "https://example.com/ogc/v1/collections/addresses/items/548?f=json" - } - ] - }, - { - "type": "Feature", - "properties": { - "collectionGeometryType": "POINT", - "collectionId": "addresses", - "collectionVersion": "1", - "displayName": "Bolwerk - Oudeschild", - "highlight": "Bolwerk - Oudeschild", - "href": "https://example.com/ogc/v1/collections/addresses/items/1050?f=json", - "score": 0.14426951110363007 - }, - "geometry": { - "type": "Polygon", - "coordinates": [ - [ - [ - 4.75002232386939, - 52.93847294238573 - ], - [ - 4.95002232386939, - 52.93847294238573 - ], - [ - 4.95002232386939, - 53.13847294238573 - ], - [ - 4.75002232386939, - 53.13847294238573 - ], - [ - 4.75002232386939, - 52.93847294238573 - ] - ] - ] - }, - "id": "1050", - "links": [ - { - "rel": "canonical", - "title": "The actual feature in the corresponding OGC API", - "type": "application/geo+json", - "href": "https://example.com/ogc/v1/collections/addresses/items/1050?f=json" - } - ] - }, - { - "type": "Feature", - "properties": { - "collectionGeometryType": "POINT", - "collectionId": "addresses", - "collectionVersion": "1", - "displayName": "Commandeurssingel - Oudeschild", - "highlight": "Commandeurssingel - Oudeschild", - "href": "https://example.com/ogc/v1/collections/addresses/items/2725?f=json", - "score": 0.14426951110363007 - }, - "geometry": { - "type": "Polygon", - "coordinates": [ - [ - [ - 4.7451477245429015, - 52.93967814281323 - ], - [ - 4.945147724542901, - 52.93967814281323 - ], - [ - 4.945147724542901, - 53.13967814281323 - ], - [ - 4.7451477245429015, - 53.13967814281323 - ], - [ - 4.7451477245429015, - 52.93967814281323 - ] - ] - ] - }, - "id": "2725", - "links": [ - { - "rel": "canonical", - "title": "The actual feature in the corresponding OGC API", - "type": "application/geo+json", - "href": "https://example.com/ogc/v1/collections/addresses/items/2725?f=json" - } - ] - }, - { - "type": "Feature", - "properties": { - "collectionGeometryType": "POINT", - "collectionId": "addresses", - "collectionVersion": "1", - "displayName": "De Houtmanstraat - Oudeschild", - "highlight": "De Houtmanstraat - Oudeschild", - "href": "https://example.com/ogc/v1/collections/addresses/items/2921?f=json", - "score": 0.12426698952913284 - }, - "geometry": { - "type": "Polygon", - "coordinates": [ - [ - [ - 4.748360166368449, - 52.93815392755542 - ], - [ - 4.948360166368448, - 52.93815392755542 - ], - [ - 4.948360166368448, - 53.13815392755542 - ], - [ - 4.748360166368449, - 53.13815392755542 - ], - [ - 4.748360166368449, - 52.93815392755542 - ] - ] - ] - }, - "id": "2921", - "links": [ - { - "rel": "canonical", - "title": "The actual feature in the corresponding OGC API", - "type": "application/geo+json", - "href": "https://example.com/ogc/v1/collections/addresses/items/2921?f=json" - } - ] - }, - { - "type": "Feature", - "properties": { - "collectionGeometryType": "POINT", - "collectionId": "addresses", - "collectionVersion": "1", - "displayName": "De Ruyterstraat - Oudeschild", - "highlight": "De Ruyterstraat - Oudeschild", - "href": "https://example.com/ogc/v1/collections/addresses/items/3049?f=json", - "score": 0.12426698952913284 - }, - "geometry": { - "type": "Polygon", - "coordinates": [ - [ - [ - 4.747714279539418, - 52.93617309495475 - ], - [ - 4.947714279539417, - 52.93617309495475 - ], - [ - 4.947714279539417, - 53.136173094954756 - ], - [ - 4.747714279539418, - 53.136173094954756 - ], - [ - 4.747714279539418, - 52.93617309495475 - ] - ] - ] - }, - "id": "3049", - "links": [ - { - "rel": "canonical", - "title": "The actual feature in the corresponding OGC API", - "type": "application/geo+json", - "href": "https://example.com/ogc/v1/collections/addresses/items/3049?f=json" - } - ] - }, - { - "type": "Feature", - "properties": { - "collectionGeometryType": "POINT", - "collectionId": "addresses", - "collectionVersion": "1", - "displayName": "De Wittstraat - Oudeschild", - "highlight": "De Wittstraat - Oudeschild", - "href": "https://example.com/ogc/v1/collections/addresses/items/3041?f=json", - "score": 0.12426698952913284 - }, - "geometry": { - "type": "Polygon", - "coordinates": [ - [ - [ - 4.745616492688666, - 52.93705261983951 - ], - [ - 4.945616492688665, - 52.93705261983951 - ], - [ - 4.945616492688665, - 53.137052619839515 - ], - [ - 4.745616492688666, - 53.137052619839515 - ], - [ - 4.745616492688666, - 52.93705261983951 - ] - ] - ] - }, - "id": "3041", - "links": [ - { - "rel": "canonical", - "title": "The actual feature in the corresponding OGC API", - "type": "application/geo+json", - "href": "https://example.com/ogc/v1/collections/addresses/items/3041?f=json" - } - ] - } - ], - "numberReturned": 6 -} From 2c48234e83856701f316d3f0e71da3f865d72cb0 Mon Sep 17 00:00:00 2001 From: Richard Kettelerij Date: Thu, 12 Dec 2024 16:09:49 +0100 Subject: [PATCH 21/38] feat: add search endpoint - remove log --- internal/search/datasources/postgres/postgres.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/internal/search/datasources/postgres/postgres.go b/internal/search/datasources/postgres/postgres.go index 4fee71e..47ab39b 100644 --- a/internal/search/datasources/postgres/postgres.go +++ b/internal/search/datasources/postgres/postgres.go @@ -78,7 +78,7 @@ func (p *Postgres) Search(ctx context.Context, searchTerm string, collections da if err != nil { return nil, err } - f := d.Feature{ + fc.Features = append(fc.Features, &d.Feature{ ID: featureID, Geometry: *geojsonGeom, Properties: map[string]any{ @@ -89,9 +89,7 @@ func (p *Postgres) Search(ctx context.Context, searchTerm string, collections da d.PropHighlight: highlightedText, d.PropScore: rank, }, - } - log.Printf("collections %s, srid %v", collections, srid) // TODO use params - fc.Features = append(fc.Features, &f) + }) fc.NumberReturned = len(fc.Features) } return &fc, queryCtx.Err() From 5b3315c8cab6dc7bc4c74caa71fc08de43d16e01 Mon Sep 17 00:00:00 2001 From: Richard Kettelerij Date: Thu, 12 Dec 2024 16:12:02 +0100 Subject: [PATCH 22/38] feat: add search endpoint - remove log --- internal/search/datasources/postgres/postgres.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/search/datasources/postgres/postgres.go b/internal/search/datasources/postgres/postgres.go index 47ab39b..076e34e 100644 --- a/internal/search/datasources/postgres/postgres.go +++ b/internal/search/datasources/postgres/postgres.go @@ -3,7 +3,6 @@ package postgres import ( "context" "fmt" - "log" "github.com/PDOK/gomagpie/internal/engine/util" "github.com/PDOK/gomagpie/internal/search/datasources" From 95ebdd35b3719941df51dc4f4c9acb0d5d593d27 Mon Sep 17 00:00:00 2001 From: Richard Kettelerij Date: Thu, 12 Dec 2024 16:22:47 +0100 Subject: [PATCH 23/38] feat: add search endpoint - move funcs in line with GoKoala --- internal/search/error.go | 21 +++++++ internal/search/main.go | 130 --------------------------------------- internal/search/url.go | 129 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 150 insertions(+), 130 deletions(-) create mode 100644 internal/search/error.go create mode 100644 internal/search/url.go diff --git a/internal/search/error.go b/internal/search/error.go new file mode 100644 index 0000000..1505671 --- /dev/null +++ b/internal/search/error.go @@ -0,0 +1,21 @@ +package search + +import ( + "context" + "errors" + "log" + "net/http" + + "github.com/PDOK/gomagpie/internal/engine" +) + +// log error, but send generic message to client to prevent possible information leakage from datasource +func handleQueryError(w http.ResponseWriter, err error) { + msg := "failed to fulfill search request" + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + // provide more context when user hits the query timeout + msg += ": querying took too long (timeout encountered). Simplify your request and try again, or contact support" + } + log.Printf("%s, error: %v\n", msg, err) + engine.RenderProblem(engine.ProblemServerError, w, msg) // don't include sensitive information in details msg +} diff --git a/internal/search/main.go b/internal/search/main.go index 7e5aa74..8676d62 100644 --- a/internal/search/main.go +++ b/internal/search/main.go @@ -1,14 +1,11 @@ package search import ( - "context" - "errors" "fmt" "log" "net/http" "net/url" "regexp" - "strconv" "strings" "time" @@ -114,51 +111,6 @@ func (s *Search) enrichFeaturesWithHref(fc *domain.FeatureCollection) error { return nil } -func parseQueryParams(query url.Values) (collections ds.CollectionsWithParams, searchTerm string, outputSRID domain.SRID, limit int, err error) { - err = validateNoUnknownParams(query) - if err != nil { - return - } - searchTerm, searchTermErr := parseSearchTerm(query) - collections = parseDeepObjectParams(query) - if len(collections) == 0 { - return nil, "", 0, 0, errors.New( - "no collection(s) specified in request, specify at least one collection and version. " + - "For example: foo[version]=1&bar[version]=2 where 'foo' and 'bar' are collection names") - } - outputSRID, outputSRIDErr := parseCrsToSRID(query, crsParam) - limit, limitErr := parseLimit(query) - err = errors.Join(searchTermErr, limitErr, outputSRIDErr) - return -} - -// Parse "deep object" params, e.g. paramName[prop1]=value1¶mName[prop2]=value2&.... -func parseDeepObjectParams(query url.Values) ds.CollectionsWithParams { - deepObjectParams := make(ds.CollectionsWithParams, len(query)) - for key, values := range query { - if strings.Contains(key, "[") { - // Extract deepObject parameters - parts := strings.SplitN(key, "[", 2) - mainKey := parts[0] - subKey := strings.TrimSuffix(parts[1], "]") - - if _, exists := deepObjectParams[mainKey]; !exists { - deepObjectParams[mainKey] = make(map[string]string) - } - deepObjectParams[mainKey][subKey] = values[0] - } - } - return deepObjectParams -} - -func parseSearchTerm(query url.Values) (searchTerm string, err error) { - searchTerm = query.Get(queryParam) - if searchTerm == "" { - err = fmt.Errorf("no search term provided, '%s' query parameter is required", queryParam) - } - return -} - func newDatasource(e *engine.Engine, dbConn string, searchIndex string) ds.Datasource { datasource, err := postgres.NewPostgres(dbConn, timeout, searchIndex) if err != nil { @@ -167,85 +119,3 @@ func newDatasource(e *engine.Engine, dbConn string, searchIndex string) ds.Datas e.RegisterShutdownHook(datasource.Close) return datasource } - -// log error, but send generic message to client to prevent possible information leakage from datasource -func handleQueryError(w http.ResponseWriter, err error) { - msg := "failed to fulfill search request" - if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { - // provide more context when user hits the query timeout - msg += ": querying took too long (timeout encountered). Simplify your request and try again, or contact support" - } - log.Printf("%s, error: %v\n", msg, err) - engine.RenderProblem(engine.ProblemServerError, w, msg) // don't include sensitive information in details msg -} - -// implements req 7.6 (https://docs.ogc.org/is/17-069r4/17-069r4.html#query_parameters) -func validateNoUnknownParams(query url.Values) error { - copyParams := clone(query) - copyParams.Del(engine.FormatParam) - copyParams.Del(queryParam) - copyParams.Del(limitParam) - copyParams.Del(crsParam) - for key := range query { - if deepObjectParamRegex.MatchString(key) { - copyParams.Del(key) - } - } - if len(copyParams) > 0 { - return fmt.Errorf("unknown query parameter(s) found: %v", copyParams.Encode()) - } - return nil -} - -func clone(params url.Values) url.Values { - copyParams := url.Values{} - for k, v := range params { - copyParams[k] = v - } - return copyParams -} - -func parseCrsToSRID(params url.Values, paramName string) (domain.SRID, error) { - param := params.Get(paramName) - if param == "" { - return domain.UndefinedSRID, nil - } - param = strings.TrimSpace(param) - if !strings.HasPrefix(param, domain.CrsURIPrefix) { - return domain.UndefinedSRID, fmt.Errorf("%s param should start with %s, got: %s", paramName, domain.CrsURIPrefix, param) - } - var srid domain.SRID - lastIndex := strings.LastIndex(param, "/") - if lastIndex != -1 { - crsCode := param[lastIndex+1:] - if crsCode == domain.WGS84CodeOGC { - return domain.WGS84SRIDPostgis, nil // CRS84 is WGS84, just like EPSG:4326 (only axis order differs but SRID is the same) - } - val, err := strconv.Atoi(crsCode) - if err != nil { - return 0, fmt.Errorf("expected numerical CRS code, received: %s", crsCode) - } - srid = domain.SRID(val) - } - return srid, nil -} - -func parseLimit(params url.Values) (int, error) { - limit := limitDefault - var err error - if params.Get(limitParam) != "" { - limit, err = strconv.Atoi(params.Get(limitParam)) - if err != nil { - err = errors.New("limit must be numeric") - } - // "If the value of the limit parameter is larger than the maximum value, this SHALL NOT result - // in an error (instead use the maximum as the parameter value)." - if limit > limitMax { - limit = limitMax - } - } - if limit < 0 { - err = errors.New("limit can't be negative") - } - return limit, err -} diff --git a/internal/search/url.go b/internal/search/url.go new file mode 100644 index 0000000..2686a81 --- /dev/null +++ b/internal/search/url.go @@ -0,0 +1,129 @@ +package search + +import ( + "errors" + "fmt" + "net/url" + "strconv" + "strings" + + "github.com/PDOK/gomagpie/internal/engine" + ds "github.com/PDOK/gomagpie/internal/search/datasources" + "github.com/PDOK/gomagpie/internal/search/domain" +) + +func parseQueryParams(query url.Values) (collections ds.CollectionsWithParams, searchTerm string, outputSRID domain.SRID, limit int, err error) { + err = validateNoUnknownParams(query) + if err != nil { + return + } + searchTerm, searchTermErr := parseSearchTerm(query) + collections = parseDeepObjectParams(query) + if len(collections) == 0 { + return nil, "", 0, 0, errors.New( + "no collection(s) specified in request, specify at least one collection and version. " + + "For example: foo[version]=1&bar[version]=2 where 'foo' and 'bar' are collection names") + } + outputSRID, outputSRIDErr := parseCrsToSRID(query, crsParam) + limit, limitErr := parseLimit(query) + err = errors.Join(searchTermErr, limitErr, outputSRIDErr) + return +} + +// Parse "deep object" params, e.g. paramName[prop1]=value1¶mName[prop2]=value2&.... +func parseDeepObjectParams(query url.Values) ds.CollectionsWithParams { + deepObjectParams := make(ds.CollectionsWithParams, len(query)) + for key, values := range query { + if strings.Contains(key, "[") { + // Extract deepObject parameters + parts := strings.SplitN(key, "[", 2) + mainKey := parts[0] + subKey := strings.TrimSuffix(parts[1], "]") + + if _, exists := deepObjectParams[mainKey]; !exists { + deepObjectParams[mainKey] = make(map[string]string) + } + deepObjectParams[mainKey][subKey] = values[0] + } + } + return deepObjectParams +} + +func parseSearchTerm(query url.Values) (searchTerm string, err error) { + searchTerm = query.Get(queryParam) + if searchTerm == "" { + err = fmt.Errorf("no search term provided, '%s' query parameter is required", queryParam) + } + return +} + +// implements req 7.6 (https://docs.ogc.org/is/17-069r4/17-069r4.html#query_parameters) +func validateNoUnknownParams(query url.Values) error { + copyParams := clone(query) + copyParams.Del(engine.FormatParam) + copyParams.Del(queryParam) + copyParams.Del(limitParam) + copyParams.Del(crsParam) + for key := range query { + if deepObjectParamRegex.MatchString(key) { + copyParams.Del(key) + } + } + if len(copyParams) > 0 { + return fmt.Errorf("unknown query parameter(s) found: %v", copyParams.Encode()) + } + return nil +} + +func clone(params url.Values) url.Values { + copyParams := url.Values{} + for k, v := range params { + copyParams[k] = v + } + return copyParams +} + +func parseCrsToSRID(params url.Values, paramName string) (domain.SRID, error) { + param := params.Get(paramName) + if param == "" { + return domain.UndefinedSRID, nil + } + param = strings.TrimSpace(param) + if !strings.HasPrefix(param, domain.CrsURIPrefix) { + return domain.UndefinedSRID, fmt.Errorf("%s param should start with %s, got: %s", paramName, domain.CrsURIPrefix, param) + } + var srid domain.SRID + lastIndex := strings.LastIndex(param, "/") + if lastIndex != -1 { + crsCode := param[lastIndex+1:] + if crsCode == domain.WGS84CodeOGC { + return domain.WGS84SRIDPostgis, nil // CRS84 is WGS84, just like EPSG:4326 (only axis order differs but SRID is the same) + } + val, err := strconv.Atoi(crsCode) + if err != nil { + return 0, fmt.Errorf("expected numerical CRS code, received: %s", crsCode) + } + srid = domain.SRID(val) + } + return srid, nil +} + +func parseLimit(params url.Values) (int, error) { + limit := limitDefault + var err error + if params.Get(limitParam) != "" { + limit, err = strconv.Atoi(params.Get(limitParam)) + if err != nil { + err = errors.New("limit must be numeric") + } + // "If the value of the limit parameter is larger than the maximum value, this SHALL NOT result + // in an error (instead use the maximum as the parameter value)." + if limit > limitMax { + limit = limitMax + } + } + if limit < 0 { + err = errors.New("limit can't be negative") + } + return limit, err +} From d8311ca1474814394969d7ed87c0ffcc932d60f9 Mon Sep 17 00:00:00 2001 From: Richard Kettelerij Date: Thu, 12 Dec 2024 16:29:11 +0100 Subject: [PATCH 24/38] feat: add search endpoint - move consts to url.go --- internal/search/main.go | 12 ------------ internal/search/url.go | 14 ++++++++++++++ 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/internal/search/main.go b/internal/search/main.go index 8676d62..4292848 100644 --- a/internal/search/main.go +++ b/internal/search/main.go @@ -5,7 +5,6 @@ import ( "log" "net/http" "net/url" - "regexp" "strings" "time" @@ -17,20 +16,9 @@ import ( ) const ( - queryParam = "q" - limitParam = "limit" - crsParam = "crs" - - limitDefault = 10 - limitMax = 50 - timeout = time.Second * 15 ) -var ( - deepObjectParamRegex = regexp.MustCompile(`\w+\[\w+]`) -) - type Search struct { engine *engine.Engine datasource ds.Datasource diff --git a/internal/search/url.go b/internal/search/url.go index 2686a81..1e97795 100644 --- a/internal/search/url.go +++ b/internal/search/url.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "net/url" + "regexp" "strconv" "strings" @@ -12,6 +13,19 @@ import ( "github.com/PDOK/gomagpie/internal/search/domain" ) +const ( + queryParam = "q" + limitParam = "limit" + crsParam = "crs" + + limitDefault = 10 + limitMax = 50 +) + +var ( + deepObjectParamRegex = regexp.MustCompile(`\w+\[\w+]`) +) + func parseQueryParams(query url.Values) (collections ds.CollectionsWithParams, searchTerm string, outputSRID domain.SRID, limit int, err error) { err = validateNoUnknownParams(query) if err != nil { From 876f7789eb043b3bd33fb2ea5a500164fe2a5df8 Mon Sep 17 00:00:00 2001 From: Richard Kettelerij Date: Fri, 13 Dec 2024 16:03:30 +0100 Subject: [PATCH 25/38] feat: add search endpoint - require collection version in requests --- internal/search/datasources/datasource.go | 17 ++++++++++ .../search/datasources/postgres/postgres.go | 15 ++++---- internal/search/main_test.go | 34 +++++++++++++++++-- .../expected-search-no-collection.json | 2 +- .../expected-search-no-version-1.json | 6 ++++ .../expected-search-no-version-2.json | 6 ++++ .../expected-search-no-version-3.json | 6 ++++ internal/search/url.go | 29 +++++++++------- 8 files changed, 92 insertions(+), 23 deletions(-) create mode 100644 internal/search/testdata/expected-search-no-version-1.json create mode 100644 internal/search/testdata/expected-search-no-version-2.json create mode 100644 internal/search/testdata/expected-search-no-version-3.json diff --git a/internal/search/datasources/datasource.go b/internal/search/datasources/datasource.go index 9de3972..3ae1a37 100644 --- a/internal/search/datasources/datasource.go +++ b/internal/search/datasources/datasource.go @@ -2,6 +2,7 @@ package datasources import ( "context" + "strconv" "github.com/PDOK/gomagpie/internal/search/domain" ) @@ -21,3 +22,19 @@ type CollectionsWithParams map[string]CollectionParams // CollectionParams parameter key with associated value type CollectionParams map[string]string + +func (cp CollectionsWithParams) NamesAndVersions() (names []string, versions []int) { + for name := range cp { + version, ok := cp[name]["version"] + if !ok { + continue + } + versionNr, err := strconv.Atoi(version) + if err != nil { + continue + } + versions = append(versions, versionNr) + names = append(names, name) + } + return names, versions +} diff --git a/internal/search/datasources/postgres/postgres.go b/internal/search/datasources/postgres/postgres.go index 076e34e..1fe449e 100644 --- a/internal/search/datasources/postgres/postgres.go +++ b/internal/search/datasources/postgres/postgres.go @@ -4,7 +4,6 @@ import ( "context" "fmt" - "github.com/PDOK/gomagpie/internal/engine/util" "github.com/PDOK/gomagpie/internal/search/datasources" d "github.com/PDOK/gomagpie/internal/search/domain" "github.com/jackc/pgx/v5" @@ -47,16 +46,18 @@ func (p *Postgres) Search(ctx context.Context, searchTerm string, collections da queryCtx, cancel := context.WithTimeout(ctx, p.queryTimeout) defer cancel() + collectionNames, collectionVersions := collections.NamesAndVersions() + // Split terms by spaces and append :* to each term terms := strings.Fields(searchTerm) for i, term := range terms { terms[i] = term + ":*" } termsConcat := strings.Join(terms, " & ") - query, args := makeSearchQuery(p.searchIndex, limit, termsConcat, util.Keys(collections), srid) + query := makeSearchQuery(p.searchIndex, srid) // Execute search query - rows, err := p.db.Query(queryCtx, query, args...) + rows, err := p.db.Query(queryCtx, query, limit, termsConcat, collectionNames, collectionVersions) if err != nil { return nil, fmt.Errorf("query '%s' failed: %w", query, err) } @@ -94,9 +95,7 @@ func (p *Postgres) Search(ctx context.Context, searchTerm string, collections da return &fc, queryCtx.Err() } -func makeSearchQuery(index string, limit int, terms string, collections []string, srid d.SRID) (string, []any) { - args := []any{limit, terms, collections} - +func makeSearchQuery(index string, srid d.SRID) string { // language=postgresql query := fmt.Sprintf(` select r.display_name as display_name, @@ -112,12 +111,12 @@ func makeSearchQuery(index string, limit int, terms string, collections []string ts_rank_cd(ts, to_tsquery($2), 1) as rank, ts_headline('dutch', display_name, to_tsquery($2)) as highlighted_text from %[1]s - where ts @@ to_tsquery($2) and collection_id = any($3) + where ts @@ to_tsquery($2) and collection_id = any($3) and collection_version = any($4) limit 500 ) r group by r.display_name order by rank desc, display_name asc limit $1`, index, srid) // don't add user input here, use $X params for user input! - return query, args + return query } diff --git a/internal/search/main_test.go b/internal/search/main_test.go index 845c34b..61d4d8e 100644 --- a/internal/search/main_test.go +++ b/internal/search/main_test.go @@ -94,10 +94,40 @@ func TestSearch(t *testing.T) { statusCode: http.StatusBadRequest, }, }, + { + name: "Fail on search with collection without version (first variant)", + fields: fields{ + url: "http://localhost:8080/search?q=\"Oudeschild\"&addresses", + }, + want: want{ + body: "internal/search/testdata/expected-search-no-version-1.json", + statusCode: http.StatusBadRequest, + }, + }, + { + name: "Fail on search with collection without version (second variant)", + fields: fields{ + url: "http://localhost:8080/search?q=\"Oudeschild\"&addresses=1", + }, + want: want{ + body: "internal/search/testdata/expected-search-no-version-2.json", + statusCode: http.StatusBadRequest, + }, + }, + { + name: "Fail on search with collection without version (third variant)", + fields: fields{ + url: "http://localhost:8080/search?q=\"Oudeschild\"&addresses[foo]=1", + }, + want: want{ + body: "internal/search/testdata/expected-search-no-version-3.json", + statusCode: http.StatusBadRequest, + }, + }, { name: "Search: 'Den' for a single collection", fields: fields{ - url: "http://localhost:8080/search?q=\"Den\"&addresses[version]=2&addresses[relevance]=0.8&limit=10&f=json&crs=http%3A%2F%2Fwww.opengis.net%2Fdef%2Fcrs%2FEPSG%2F0%2F28992", + url: "http://localhost:8080/search?q=\"Den\"&addresses[version]=1&addresses[relevance]=0.8&limit=10&f=json&crs=http%3A%2F%2Fwww.opengis.net%2Fdef%2Fcrs%2FEPSG%2F0%2F28992", }, want: want{ body: "internal/search/testdata/expected-search-den-single-collection.json", @@ -107,7 +137,7 @@ func TestSearch(t *testing.T) { { name: "Search: 'Den' for multiple collections (with one not existing collection, so same output as single collection)", fields: fields{ - url: "http://localhost:8080/search?q=\"Den\"&addresses[version]=2&addresses[relevance]=0.8&foo[version]=2&foo[relevance]=0.8&limit=10&f=json&crs=http%3A%2F%2Fwww.opengis.net%2Fdef%2Fcrs%2FEPSG%2F0%2F28992", + url: "http://localhost:8080/search?q=\"Den\"&addresses[version]=1&addresses[relevance]=0.8&foo[version]=2&foo[relevance]=0.8&limit=10&f=json&crs=http%3A%2F%2Fwww.opengis.net%2Fdef%2Fcrs%2FEPSG%2F0%2F28992", }, want: want{ body: "internal/search/testdata/expected-search-den-single-collection.json", diff --git a/internal/search/testdata/expected-search-no-collection.json b/internal/search/testdata/expected-search-no-collection.json index a23097a..68cf186 100644 --- a/internal/search/testdata/expected-search-no-collection.json +++ b/internal/search/testdata/expected-search-no-collection.json @@ -1,5 +1,5 @@ { - "detail": "no collection(s) specified in request, specify at least one collection and version. For example: foo[version]=1\u0026bar[version]=2 where 'foo' and 'bar' are collection names", + "detail": "no collection(s) specified in request, specify at least one collection and version. For example: 'foo[version]=1' where 'foo' is the collection and '1' the version", "status": 400, "timeStamp": "2000-01-01T00:00:00Z", "title": "Bad Request" diff --git a/internal/search/testdata/expected-search-no-version-1.json b/internal/search/testdata/expected-search-no-version-1.json new file mode 100644 index 0000000..1c2a224 --- /dev/null +++ b/internal/search/testdata/expected-search-no-version-1.json @@ -0,0 +1,6 @@ +{ + "detail": "unknown query parameter(s) found: addresses=", + "status": 400, + "timeStamp": "2000-01-01T00:00:00Z", + "title": "Bad Request" +} \ No newline at end of file diff --git a/internal/search/testdata/expected-search-no-version-2.json b/internal/search/testdata/expected-search-no-version-2.json new file mode 100644 index 0000000..4306f23 --- /dev/null +++ b/internal/search/testdata/expected-search-no-version-2.json @@ -0,0 +1,6 @@ +{ + "detail": "unknown query parameter(s) found: addresses=1", + "status": 400, + "timeStamp": "2000-01-01T00:00:00Z", + "title": "Bad Request" +} \ No newline at end of file diff --git a/internal/search/testdata/expected-search-no-version-3.json b/internal/search/testdata/expected-search-no-version-3.json new file mode 100644 index 0000000..fbf837a --- /dev/null +++ b/internal/search/testdata/expected-search-no-version-3.json @@ -0,0 +1,6 @@ +{ + "detail": "no version specified in request for collection addresses, specify at least one collection and version. For example: 'foo[version]=1' where 'foo' is the collection and '1' the version", + "status": 400, + "timeStamp": "2000-01-01T00:00:00Z", + "title": "Bad Request" +} diff --git a/internal/search/url.go b/internal/search/url.go index 1e97795..c3aaa54 100644 --- a/internal/search/url.go +++ b/internal/search/url.go @@ -14,9 +14,10 @@ import ( ) const ( - queryParam = "q" - limitParam = "limit" - crsParam = "crs" + queryParam = "q" + limitParam = "limit" + crsParam = "crs" + VersionParam = "version" limitDefault = 10 limitMax = 50 @@ -32,20 +33,15 @@ func parseQueryParams(query url.Values) (collections ds.CollectionsWithParams, s return } searchTerm, searchTermErr := parseSearchTerm(query) - collections = parseDeepObjectParams(query) - if len(collections) == 0 { - return nil, "", 0, 0, errors.New( - "no collection(s) specified in request, specify at least one collection and version. " + - "For example: foo[version]=1&bar[version]=2 where 'foo' and 'bar' are collection names") - } + collections, collErr := parseCollections(query) outputSRID, outputSRIDErr := parseCrsToSRID(query, crsParam) limit, limitErr := parseLimit(query) - err = errors.Join(searchTermErr, limitErr, outputSRIDErr) + err = errors.Join(collErr, searchTermErr, limitErr, outputSRIDErr) return } // Parse "deep object" params, e.g. paramName[prop1]=value1¶mName[prop2]=value2&.... -func parseDeepObjectParams(query url.Values) ds.CollectionsWithParams { +func parseCollections(query url.Values) (ds.CollectionsWithParams, error) { deepObjectParams := make(ds.CollectionsWithParams, len(query)) for key, values := range query { if strings.Contains(key, "[") { @@ -60,7 +56,16 @@ func parseDeepObjectParams(query url.Values) ds.CollectionsWithParams { deepObjectParams[mainKey][subKey] = values[0] } } - return deepObjectParams + errMsg := "specify at least one collection and version. For example: 'foo[version]=1' where 'foo' is the collection and '1' the version" + if len(deepObjectParams) == 0 { + return nil, fmt.Errorf("no collection(s) specified in request, %s", errMsg) + } + for name := range deepObjectParams { + if version, ok := deepObjectParams[name][VersionParam]; !ok || version == "" { + return nil, fmt.Errorf("no version specified in request for collection %s, %s", name, errMsg) + } + } + return deepObjectParams, nil } func parseSearchTerm(query url.Values) (searchTerm string, err error) { From dcf0d5c6b2deeb08d7f6b9921ee35f817d02660f Mon Sep 17 00:00:00 2001 From: Richard Kettelerij Date: Fri, 13 Dec 2024 16:37:25 +0100 Subject: [PATCH 26/38] feat: add search endpoint - refactoring + add extra tst with different CRS --- internal/search/datasources/datasource.go | 27 +- .../search/datasources/postgres/postgres.go | 3 +- internal/search/domain/search_collections.go | 30 ++ internal/search/main_test.go | 18 +- ...cted-search-den-single-collection-rd.json} | 0 ...ed-search-den-single-collection-wgs84.json | 399 ++++++++++++++++++ internal/search/url.go | 38 +- 7 files changed, 464 insertions(+), 51 deletions(-) create mode 100644 internal/search/domain/search_collections.go rename internal/search/testdata/{expected-search-den-single-collection.json => expected-search-den-single-collection-rd.json} (100%) create mode 100644 internal/search/testdata/expected-search-den-single-collection-wgs84.json diff --git a/internal/search/datasources/datasource.go b/internal/search/datasources/datasource.go index 3ae1a37..253a7dd 100644 --- a/internal/search/datasources/datasource.go +++ b/internal/search/datasources/datasource.go @@ -2,7 +2,6 @@ package datasources import ( "context" - "strconv" "github.com/PDOK/gomagpie/internal/search/domain" ) @@ -10,31 +9,9 @@ import ( // Datasource knows how make different kinds of queries/actions on the underlying actual datastore. // This abstraction allows the rest of the system to stay datastore agnostic. type Datasource interface { - Search(ctx context.Context, searchTerm string, collections CollectionsWithParams, srid domain.SRID, limit int) (*domain.FeatureCollection, error) + Search(ctx context.Context, searchTerm string, collections domain.CollectionsWithParams, + srid domain.SRID, limit int) (*domain.FeatureCollection, error) // Close closes (connections to) the datasource gracefully Close() } - -// CollectionsWithParams collection name with associated CollectionParams -// These are provided though a URL query string as "deep object" params, e.g. paramName[prop1]=value1¶mName[prop2]=value2&.... -type CollectionsWithParams map[string]CollectionParams - -// CollectionParams parameter key with associated value -type CollectionParams map[string]string - -func (cp CollectionsWithParams) NamesAndVersions() (names []string, versions []int) { - for name := range cp { - version, ok := cp[name]["version"] - if !ok { - continue - } - versionNr, err := strconv.Atoi(version) - if err != nil { - continue - } - versions = append(versions, versionNr) - names = append(names, name) - } - return names, versions -} diff --git a/internal/search/datasources/postgres/postgres.go b/internal/search/datasources/postgres/postgres.go index 1fe449e..371a818 100644 --- a/internal/search/datasources/postgres/postgres.go +++ b/internal/search/datasources/postgres/postgres.go @@ -4,7 +4,6 @@ import ( "context" "fmt" - "github.com/PDOK/gomagpie/internal/search/datasources" d "github.com/PDOK/gomagpie/internal/search/domain" "github.com/jackc/pgx/v5" pggeom "github.com/twpayne/go-geom" @@ -40,7 +39,7 @@ func (p *Postgres) Close() { _ = p.db.Close(p.ctx) } -func (p *Postgres) Search(ctx context.Context, searchTerm string, collections datasources.CollectionsWithParams, +func (p *Postgres) Search(ctx context.Context, searchTerm string, collections d.CollectionsWithParams, srid d.SRID, limit int) (*d.FeatureCollection, error) { queryCtx, cancel := context.WithTimeout(ctx, p.queryTimeout) diff --git a/internal/search/domain/search_collections.go b/internal/search/domain/search_collections.go new file mode 100644 index 0000000..56bb8b6 --- /dev/null +++ b/internal/search/domain/search_collections.go @@ -0,0 +1,30 @@ +package domain + +import "strconv" + +const ( + VersionParam = "version" +) + +// CollectionsWithParams collection name with associated CollectionParams +// These are provided though a URL query string as "deep object" params, e.g. paramName[prop1]=value1¶mName[prop2]=value2&.... +type CollectionsWithParams map[string]CollectionParams + +// CollectionParams parameter key with associated value +type CollectionParams map[string]string + +func (cp CollectionsWithParams) NamesAndVersions() (names []string, versions []int) { + for name := range cp { + version, ok := cp[name][VersionParam] + if !ok { + continue + } + versionNr, err := strconv.Atoi(version) + if err != nil { + continue + } + versions = append(versions, versionNr) + names = append(names, name) + } + return names, versions +} diff --git a/internal/search/main_test.go b/internal/search/main_test.go index 61d4d8e..e1b3df1 100644 --- a/internal/search/main_test.go +++ b/internal/search/main_test.go @@ -125,22 +125,32 @@ func TestSearch(t *testing.T) { }, }, { - name: "Search: 'Den' for a single collection", + name: "Search: 'Den' for a single collection in WGS84 (default)", + fields: fields{ + url: "http://localhost:8080/search?q=\"Den\"&addresses[version]=1&addresses[relevance]=0.8&limit=10&f=json", + }, + want: want{ + body: "internal/search/testdata/expected-search-den-single-collection-wgs84.json", + statusCode: http.StatusOK, + }, + }, + { + name: "Search: 'Den' for a single collection in RD", fields: fields{ url: "http://localhost:8080/search?q=\"Den\"&addresses[version]=1&addresses[relevance]=0.8&limit=10&f=json&crs=http%3A%2F%2Fwww.opengis.net%2Fdef%2Fcrs%2FEPSG%2F0%2F28992", }, want: want{ - body: "internal/search/testdata/expected-search-den-single-collection.json", + body: "internal/search/testdata/expected-search-den-single-collection-rd.json", statusCode: http.StatusOK, }, }, { - name: "Search: 'Den' for multiple collections (with one not existing collection, so same output as single collection)", + name: "Search: 'Den' for multiple collections (with one not existing collection, so same output as single collection) in RD", fields: fields{ url: "http://localhost:8080/search?q=\"Den\"&addresses[version]=1&addresses[relevance]=0.8&foo[version]=2&foo[relevance]=0.8&limit=10&f=json&crs=http%3A%2F%2Fwww.opengis.net%2Fdef%2Fcrs%2FEPSG%2F0%2F28992", }, want: want{ - body: "internal/search/testdata/expected-search-den-single-collection.json", + body: "internal/search/testdata/expected-search-den-single-collection-rd.json", statusCode: http.StatusOK, }, }, diff --git a/internal/search/testdata/expected-search-den-single-collection.json b/internal/search/testdata/expected-search-den-single-collection-rd.json similarity index 100% rename from internal/search/testdata/expected-search-den-single-collection.json rename to internal/search/testdata/expected-search-den-single-collection-rd.json diff --git a/internal/search/testdata/expected-search-den-single-collection-wgs84.json b/internal/search/testdata/expected-search-den-single-collection-wgs84.json new file mode 100644 index 0000000..0dbc6f9 --- /dev/null +++ b/internal/search/testdata/expected-search-den-single-collection-wgs84.json @@ -0,0 +1,399 @@ +{ + "type": "FeatureCollection", + "timeStamp": "2000-01-01T00:00:00Z", + "links": [ + { + "rel": "self", + "title": "This document as GeoJSON", + "type": "application/geo+json", + "href": "http://localhost:8080/search?f=json" + } + ], + "features": [ + { + "type": "Feature", + "properties": { + "collectionGeometryType": "POINT", + "collectionId": "addresses", + "collectionVersion": "1", + "displayName": "Abbewaal - Den Burg", + "highlight": "Abbewaal - Den Burg", + "href": "https://example.com/ogc/v1/collections/addresses/items/99?f=json", + "score": 0.11162212491035461 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 4.701721422439945, + 52.9619223105808 + ], + [ + 4.901721422439945, + 52.9619223105808 + ], + [ + 4.901721422439945, + 53.161922310580806 + ], + [ + 4.701721422439945, + 53.161922310580806 + ], + [ + 4.701721422439945, + 52.9619223105808 + ] + ] + ] + }, + "id": "99", + "links": [ + { + "rel": "canonical", + "title": "The actual feature in the corresponding OGC API", + "type": "application/geo+json", + "href": "https://example.com/ogc/v1/collections/addresses/items/99?f=json" + } + ] + }, + { + "type": "Feature", + "properties": { + "collectionGeometryType": "POINT", + "collectionId": "addresses", + "collectionVersion": "1", + "displayName": "Achterom - Den Burg", + "highlight": "Achterom - Den Burg", + "href": "https://example.com/ogc/v1/collections/addresses/items/114?f=json", + "score": 0.11162212491035461 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 4.699813158490893, + 52.95463219709524 + ], + [ + 4.899813158490892, + 52.95463219709524 + ], + [ + 4.899813158490892, + 53.154632197095246 + ], + [ + 4.699813158490893, + 53.154632197095246 + ], + [ + 4.699813158490893, + 52.95463219709524 + ] + ] + ] + }, + "id": "114", + "links": [ + { + "rel": "canonical", + "title": "The actual feature in the corresponding OGC API", + "type": "application/geo+json", + "href": "https://example.com/ogc/v1/collections/addresses/items/114?f=json" + } + ] + }, + { + "type": "Feature", + "properties": { + "collectionGeometryType": "POINT", + "collectionId": "addresses", + "collectionVersion": "1", + "displayName": "Akenbuurt - Den Burg", + "highlight": "Akenbuurt - Den Burg", + "href": "https://example.com/ogc/v1/collections/addresses/items/46?f=json", + "score": 0.11162212491035461 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 4.680059099895046, + 52.95346592050607 + ], + [ + 4.880059099895045, + 52.95346592050607 + ], + [ + 4.880059099895045, + 53.15346592050607 + ], + [ + 4.680059099895046, + 53.15346592050607 + ], + [ + 4.680059099895046, + 52.95346592050607 + ] + ] + ] + }, + "id": "46", + "links": [ + { + "rel": "canonical", + "title": "The actual feature in the corresponding OGC API", + "type": "application/geo+json", + "href": "https://example.com/ogc/v1/collections/addresses/items/46?f=json" + } + ] + }, + { + "type": "Feature", + "properties": { + "collectionGeometryType": "POINT", + "collectionId": "addresses", + "collectionVersion": "1", + "displayName": "Amaliaweg - Den Hoorn", + "highlight": "Amaliaweg - Den Hoorn", + "href": "https://example.com/ogc/v1/collections/addresses/items/50?f=json", + "score": 0.11162212491035461 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 4.68911630577304, + 52.92449928128154 + ], + [ + 4.889116305773039, + 52.92449928128154 + ], + [ + 4.889116305773039, + 53.124499281281544 + ], + [ + 4.68911630577304, + 53.124499281281544 + ], + [ + 4.68911630577304, + 52.92449928128154 + ] + ] + ] + }, + "id": "50", + "links": [ + { + "rel": "canonical", + "title": "The actual feature in the corresponding OGC API", + "type": "application/geo+json", + "href": "https://example.com/ogc/v1/collections/addresses/items/50?f=json" + } + ] + }, + { + "type": "Feature", + "properties": { + "collectionGeometryType": "POINT", + "collectionId": "addresses", + "collectionVersion": "1", + "displayName": "Bakkenweg - Den Hoorn", + "highlight": "Bakkenweg - Den Hoorn", + "href": "https://example.com/ogc/v1/collections/addresses/items/520?f=json", + "score": 0.11162212491035461 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 4.6548723261037095, + 52.94811743920973 + ], + [ + 4.854872326103709, + 52.94811743920973 + ], + [ + 4.854872326103709, + 53.148117439209734 + ], + [ + 4.6548723261037095, + 53.148117439209734 + ], + [ + 4.6548723261037095, + 52.94811743920973 + ] + ] + ] + }, + "id": "520", + "links": [ + { + "rel": "canonical", + "title": "The actual feature in the corresponding OGC API", + "type": "application/geo+json", + "href": "https://example.com/ogc/v1/collections/addresses/items/520?f=json" + } + ] + }, + { + "type": "Feature", + "properties": { + "collectionGeometryType": "POINT", + "collectionId": "addresses", + "collectionVersion": "1", + "displayName": "Beatrixlaan - Den Burg", + "highlight": "Beatrixlaan - Den Burg", + "href": "https://example.com/ogc/v1/collections/addresses/items/591?f=json", + "score": 0.11162212491035461 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 4.690892824472019, + 52.95558352001795 + ], + [ + 4.890892824472019, + 52.95558352001795 + ], + [ + 4.890892824472019, + 53.155583520017956 + ], + [ + 4.690892824472019, + 53.155583520017956 + ], + [ + 4.690892824472019, + 52.95558352001795 + ] + ] + ] + }, + "id": "591", + "links": [ + { + "rel": "canonical", + "title": "The actual feature in the corresponding OGC API", + "type": "application/geo+json", + "href": "https://example.com/ogc/v1/collections/addresses/items/591?f=json" + } + ] + }, + { + "type": "Feature", + "properties": { + "collectionGeometryType": "POINT", + "collectionId": "addresses", + "collectionVersion": "1", + "displayName": "Ada van Hollandstraat - Den Burg", + "highlight": "Ada van Hollandstraat - Den Burg", + "href": "https://example.com/ogc/v1/collections/addresses/items/26?f=json", + "score": 0.09617967158555984 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 4.696235388824104, + 52.95196001510249 + ], + [ + 4.8962353888241035, + 52.95196001510249 + ], + [ + 4.8962353888241035, + 53.151960015102496 + ], + [ + 4.696235388824104, + 53.151960015102496 + ], + [ + 4.696235388824104, + 52.95196001510249 + ] + ] + ] + }, + "id": "26", + "links": [ + { + "rel": "canonical", + "title": "The actual feature in the corresponding OGC API", + "type": "application/geo+json", + "href": "https://example.com/ogc/v1/collections/addresses/items/26?f=json" + } + ] + }, + { + "type": "Feature", + "properties": { + "collectionGeometryType": "POINT", + "collectionId": "addresses", + "collectionVersion": "1", + "displayName": "Anne Frankstraat - Den Burg", + "highlight": "Anne Frankstraat - Den Burg", + "href": "https://example.com/ogc/v1/collections/addresses/items/474?f=json", + "score": 0.09617967158555984 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 4.692873779103581, + 52.950932925919574 + ], + [ + 4.892873779103581, + 52.950932925919574 + ], + [ + 4.892873779103581, + 53.15093292591958 + ], + [ + 4.692873779103581, + 53.15093292591958 + ], + [ + 4.692873779103581, + 52.950932925919574 + ] + ] + ] + }, + "id": "474", + "links": [ + { + "rel": "canonical", + "title": "The actual feature in the corresponding OGC API", + "type": "application/geo+json", + "href": "https://example.com/ogc/v1/collections/addresses/items/474?f=json" + } + ] + } + ], + "numberReturned": 8 +} diff --git a/internal/search/url.go b/internal/search/url.go index c3aaa54..daf46ef 100644 --- a/internal/search/url.go +++ b/internal/search/url.go @@ -9,15 +9,13 @@ import ( "strings" "github.com/PDOK/gomagpie/internal/engine" - ds "github.com/PDOK/gomagpie/internal/search/datasources" - "github.com/PDOK/gomagpie/internal/search/domain" + d "github.com/PDOK/gomagpie/internal/search/domain" ) const ( - queryParam = "q" - limitParam = "limit" - crsParam = "crs" - VersionParam = "version" + queryParam = "q" + limitParam = "limit" + crsParam = "crs" limitDefault = 10 limitMax = 50 @@ -27,22 +25,22 @@ var ( deepObjectParamRegex = regexp.MustCompile(`\w+\[\w+]`) ) -func parseQueryParams(query url.Values) (collections ds.CollectionsWithParams, searchTerm string, outputSRID domain.SRID, limit int, err error) { +func parseQueryParams(query url.Values) (collections d.CollectionsWithParams, searchTerm string, outputSRID d.SRID, limit int, err error) { err = validateNoUnknownParams(query) if err != nil { return } searchTerm, searchTermErr := parseSearchTerm(query) collections, collErr := parseCollections(query) - outputSRID, outputSRIDErr := parseCrsToSRID(query, crsParam) + outputSRID, outputSRIDErr := parseCrsToPostgisSRID(query, crsParam) limit, limitErr := parseLimit(query) err = errors.Join(collErr, searchTermErr, limitErr, outputSRIDErr) return } -// Parse "deep object" params, e.g. paramName[prop1]=value1¶mName[prop2]=value2&.... -func parseCollections(query url.Values) (ds.CollectionsWithParams, error) { - deepObjectParams := make(ds.CollectionsWithParams, len(query)) +// Parse collections as "deep object" params, e.g. collectionName[prop1]=value1&collectionName[prop2]=value2&.... +func parseCollections(query url.Values) (d.CollectionsWithParams, error) { + deepObjectParams := make(d.CollectionsWithParams, len(query)) for key, values := range query { if strings.Contains(key, "[") { // Extract deepObject parameters @@ -61,7 +59,7 @@ func parseCollections(query url.Values) (ds.CollectionsWithParams, error) { return nil, fmt.Errorf("no collection(s) specified in request, %s", errMsg) } for name := range deepObjectParams { - if version, ok := deepObjectParams[name][VersionParam]; !ok || version == "" { + if version, ok := deepObjectParams[name][d.VersionParam]; !ok || version == "" { return nil, fmt.Errorf("no version specified in request for collection %s, %s", name, errMsg) } } @@ -102,27 +100,27 @@ func clone(params url.Values) url.Values { return copyParams } -func parseCrsToSRID(params url.Values, paramName string) (domain.SRID, error) { +func parseCrsToPostgisSRID(params url.Values, paramName string) (d.SRID, error) { param := params.Get(paramName) if param == "" { - return domain.UndefinedSRID, nil + return d.WGS84SRIDPostgis, nil // default to WGS84 } param = strings.TrimSpace(param) - if !strings.HasPrefix(param, domain.CrsURIPrefix) { - return domain.UndefinedSRID, fmt.Errorf("%s param should start with %s, got: %s", paramName, domain.CrsURIPrefix, param) + if !strings.HasPrefix(param, d.CrsURIPrefix) { + return d.UndefinedSRID, fmt.Errorf("%s param should start with %s, got: %s", paramName, d.CrsURIPrefix, param) } - var srid domain.SRID + var srid d.SRID lastIndex := strings.LastIndex(param, "/") if lastIndex != -1 { crsCode := param[lastIndex+1:] - if crsCode == domain.WGS84CodeOGC { - return domain.WGS84SRIDPostgis, nil // CRS84 is WGS84, just like EPSG:4326 (only axis order differs but SRID is the same) + if crsCode == d.WGS84CodeOGC { + return d.WGS84SRIDPostgis, nil // CRS84 is WGS84, we use EPSG:4326 for Postgres TODO: check if correct since axis order differs between CRS84 and EPSG:4326 } val, err := strconv.Atoi(crsCode) if err != nil { return 0, fmt.Errorf("expected numerical CRS code, received: %s", crsCode) } - srid = domain.SRID(val) + srid = d.SRID(val) } return srid, nil } From dbe21fee94a566e6ef05835c7cf58c049f66f85b Mon Sep 17 00:00:00 2001 From: Richard Kettelerij Date: Mon, 16 Dec 2024 12:10:53 +0100 Subject: [PATCH 27/38] feat: add search endpoint - add order by in subquery + naming --- internal/search/datasources/datasource.go | 4 +- .../search/datasources/postgres/postgres.go | 62 +++--- .../{search_collections.go => search.go} | 11 + internal/search/domain/search_props.go | 11 - internal/search/main.go | 2 +- ...ected-search-den-single-collection-rd.json | 190 +++++------------- ...ed-search-den-single-collection-wgs84.json | 190 +++++------------- 7 files changed, 142 insertions(+), 328 deletions(-) rename internal/search/domain/{search_collections.go => search.go} (70%) delete mode 100644 internal/search/domain/search_props.go diff --git a/internal/search/datasources/datasource.go b/internal/search/datasources/datasource.go index 253a7dd..7aeb918 100644 --- a/internal/search/datasources/datasource.go +++ b/internal/search/datasources/datasource.go @@ -9,7 +9,9 @@ import ( // Datasource knows how make different kinds of queries/actions on the underlying actual datastore. // This abstraction allows the rest of the system to stay datastore agnostic. type Datasource interface { - Search(ctx context.Context, searchTerm string, collections domain.CollectionsWithParams, + // SearchFeaturesAcrossCollections search features in one or more collections. Collections can be located + // in this dataset or in other datasets. + SearchFeaturesAcrossCollections(ctx context.Context, searchTerm string, collections domain.CollectionsWithParams, srid domain.SRID, limit int) (*domain.FeatureCollection, error) // Close closes (connections to) the datasource gracefully diff --git a/internal/search/datasources/postgres/postgres.go b/internal/search/datasources/postgres/postgres.go index 371a818..b73b510 100644 --- a/internal/search/datasources/postgres/postgres.go +++ b/internal/search/datasources/postgres/postgres.go @@ -39,14 +39,12 @@ func (p *Postgres) Close() { _ = p.db.Close(p.ctx) } -func (p *Postgres) Search(ctx context.Context, searchTerm string, collections d.CollectionsWithParams, +func (p *Postgres) SearchFeaturesAcrossCollections(ctx context.Context, searchTerm string, collections d.CollectionsWithParams, srid d.SRID, limit int) (*d.FeatureCollection, error) { queryCtx, cancel := context.WithTimeout(ctx, p.queryTimeout) defer cancel() - collectionNames, collectionVersions := collections.NamesAndVersions() - // Split terms by spaces and append :* to each term terms := strings.Fields(searchTerm) for i, term := range terms { @@ -56,6 +54,7 @@ func (p *Postgres) Search(ctx context.Context, searchTerm string, collections d. query := makeSearchQuery(p.searchIndex, srid) // Execute search query + collectionNames, collectionVersions := collections.NamesAndVersions() rows, err := p.db.Query(queryCtx, query, limit, termsConcat, collectionNames, collectionVersions) if err != nil { return nil, fmt.Errorf("query '%s' failed: %w", query, err) @@ -63,6 +62,37 @@ func (p *Postgres) Search(ctx context.Context, searchTerm string, collections d. defer rows.Close() // Turn rows into FeatureCollection + return mapRowsToFeatures(queryCtx, rows) +} + +func makeSearchQuery(index string, srid d.SRID) string { + // language=postgresql + query := fmt.Sprintf(` + select r.display_name as display_name, + max(r.feature_id) as feature_id, + max(r.collection_id) as collection_id, + max(r.collection_version) as collection_version, + max(r.geometry_type) as geometry_type, + cast(st_transform(max(r.bbox), %[2]d) as geometry) as bbox, + max(r.rank) as rank, + max(r.highlighted_text) as highlighted_text + from ( + select display_name, feature_id, collection_id, collection_version, geometry_type, bbox, + ts_rank_cd(ts, to_tsquery($2), 1) as rank, + ts_headline('dutch', display_name, to_tsquery($2)) as highlighted_text + from %[1]s + where ts @@ to_tsquery($2) and collection_id = any($3) and collection_version = any($4) + order by rank desc, display_name asc + limit 500 + ) r + group by r.display_name + order by rank desc, display_name asc + limit $1`, index, srid) // don't add user input here, use $X params for user input! + + return query +} + +func mapRowsToFeatures(queryCtx context.Context, rows pgx.Rows) (*d.FeatureCollection, error) { fc := d.FeatureCollection{Features: make([]*d.Feature, 0)} for rows.Next() { var displayName, highlightedText, featureID, collectionID, collectionVersion, geomType string @@ -93,29 +123,3 @@ func (p *Postgres) Search(ctx context.Context, searchTerm string, collections d. } return &fc, queryCtx.Err() } - -func makeSearchQuery(index string, srid d.SRID) string { - // language=postgresql - query := fmt.Sprintf(` - select r.display_name as display_name, - max(r.feature_id) as feature_id, - max(r.collection_id) as collection_id, - max(r.collection_version) as collection_version, - max(r.geometry_type) as geometry_type, - cast(st_transform(max(r.bbox), %[2]d) as geometry) as bbox, - max(r.rank) as rank, - max(r.highlighted_text) as highlighted_text - from ( - select display_name, feature_id, collection_id, collection_version, geometry_type, bbox, - ts_rank_cd(ts, to_tsquery($2), 1) as rank, - ts_headline('dutch', display_name, to_tsquery($2)) as highlighted_text - from %[1]s - where ts @@ to_tsquery($2) and collection_id = any($3) and collection_version = any($4) - limit 500 - ) r - group by r.display_name - order by rank desc, display_name asc - limit $1`, index, srid) // don't add user input here, use $X params for user input! - - return query -} diff --git a/internal/search/domain/search_collections.go b/internal/search/domain/search.go similarity index 70% rename from internal/search/domain/search_collections.go rename to internal/search/domain/search.go index 56bb8b6..f99b01f 100644 --- a/internal/search/domain/search_collections.go +++ b/internal/search/domain/search.go @@ -6,6 +6,17 @@ const ( VersionParam = "version" ) +// GeoJSON properties in search response +const ( + PropCollectionID = "collectionId" + PropCollectionVersion = "collectionVersion" + PropGeomType = "collectionGeometryType" + PropDisplayName = "displayName" + PropHighlight = "highlight" + PropScore = "score" + PropHref = "href" +) + // CollectionsWithParams collection name with associated CollectionParams // These are provided though a URL query string as "deep object" params, e.g. paramName[prop1]=value1¶mName[prop2]=value2&.... type CollectionsWithParams map[string]CollectionParams diff --git a/internal/search/domain/search_props.go b/internal/search/domain/search_props.go deleted file mode 100644 index 0c75efd..0000000 --- a/internal/search/domain/search_props.go +++ /dev/null @@ -1,11 +0,0 @@ -package domain - -const ( - PropCollectionID = "collectionId" - PropCollectionVersion = "collectionVersion" - PropGeomType = "collectionGeometryType" - PropDisplayName = "displayName" - PropHighlight = "highlight" - PropScore = "score" - PropHref = "href" -) diff --git a/internal/search/main.go b/internal/search/main.go index 4292848..387b4f8 100644 --- a/internal/search/main.go +++ b/internal/search/main.go @@ -41,7 +41,7 @@ func (s *Search) Search() http.HandlerFunc { engine.RenderProblem(engine.ProblemBadRequest, w, err.Error()) return } - fc, err := s.datasource.Search(r.Context(), searchTerm, collections, outputSRID, limit) + fc, err := s.datasource.SearchFeaturesAcrossCollections(r.Context(), searchTerm, collections, outputSRID, limit) if err != nil { handleQueryError(w, err) return diff --git a/internal/search/testdata/expected-search-den-single-collection-rd.json b/internal/search/testdata/expected-search-den-single-collection-rd.json index 2b86f9d..8dd7211 100644 --- a/internal/search/testdata/expected-search-den-single-collection-rd.json +++ b/internal/search/testdata/expected-search-den-single-collection-rd.json @@ -26,24 +26,24 @@ "coordinates": [ [ [ - 108940.29073913672, - 552985.5894477183 + 109043.61280645092, + 553071.2827189546 ], [ - 122378.69131049563, - 552876.583266793 + 122481.77479068137, + 552962.5620117659 ], [ - 122528.48208323204, - 575133.1149730522 + 122631.09554431966, + 575219.1012624607 ], [ - 109151.81060375126, - 575241.7668599344 + 109254.66298884692, + 575327.4685912606 ], [ - 108940.29073913672, - 552985.5894477183 + 109043.61280645092, + 553071.2827189546 ] ] ] @@ -66,7 +66,7 @@ "collectionVersion": "1", "displayName": "Achterom - Den Burg", "highlight": "Achterom - Den Burg", - "href": "https://example.com/ogc/v1/collections/addresses/items/114?f=json", + "href": "https://example.com/ogc/v1/collections/addresses/items/22215?f=json", "score": 0.11162212491035461 }, "geometry": { @@ -74,35 +74,35 @@ "coordinates": [ [ [ - 108804.34590509304, - 552175.5839961797 + 108759.77100749934, + 552248.1945484902 ], [ - 122244.99285675734, - 552066.209512857 + 122200.21718796142, + 552138.6957663981 ], [ - 122395.36487788748, - 574322.6888311192 + 122350.79776828067, + 574395.1784952144 ], [ - 109016.44385035255, - 574431.7079399591 + 108972.07779478905, + 574504.3214884505 ], [ - 108804.34590509304, - 552175.5839961797 + 108759.77100749934, + 552248.1945484902 ] ] ] }, - "id": "114", + "id": "22215", "links": [ { "rel": "canonical", "title": "The actual feature in the corresponding OGC API", "type": "application/geo+json", - "href": "https://example.com/ogc/v1/collections/addresses/items/114?f=json" + "href": "https://example.com/ogc/v1/collections/addresses/items/22215?f=json" } ] }, @@ -122,24 +122,24 @@ "coordinates": [ [ [ - 107475.55019422778, - 552058.6284843497 + 107382.89003167403, + 552466.0160286687 ], [ - 120916.53534647425, - 551945.5725395872 + 120822.74710915676, + 552352.698917785 ], [ - 121073.00275881319, - 574202.0142312867 + 120979.66241412575, + 574609.1632009319 ], [ - 107693.74304765201, - 574314.7028564449 + 107601.53236763355, + 574722.1120828141 ], [ - 107475.55019422778, - 552058.6284843497 + 107382.89003167403, + 552466.0160286687 ] ] ] @@ -258,7 +258,7 @@ "collectionVersion": "1", "displayName": "Beatrixlaan - Den Burg", "highlight": "Beatrixlaan - Den Burg", - "href": "https://example.com/ogc/v1/collections/addresses/items/591?f=json", + "href": "https://example.com/ogc/v1/collections/addresses/items/827?f=json", "score": 0.11162212491035461 }, "geometry": { @@ -266,134 +266,38 @@ "coordinates": [ [ [ - 108205.89714467606, - 552287.1906286391 + 108218.13045314618, + 552388.953964499 ], [ - 121646.24107836874, - 552176.1563847166 + 121658.19222109031, + 552277.9525492993 ], [ - 121799.36720075001, - 574432.6288915055 + 121811.26766043308, + 574534.4315263145 ], [ - 108420.74961558703, - 574543.3023513828 + 108432.93263887867, + 574645.0722492639 ], [ - 108205.89714467606, - 552287.1906286391 + 108218.13045314618, + 552388.953964499 ] ] ] }, - "id": "591", + "id": "827", "links": [ { "rel": "canonical", "title": "The actual feature in the corresponding OGC API", "type": "application/geo+json", - "href": "https://example.com/ogc/v1/collections/addresses/items/591?f=json" - } - ] - }, - { - "type": "Feature", - "properties": { - "collectionGeometryType": "POINT", - "collectionId": "addresses", - "collectionVersion": "1", - "displayName": "Ada van Hollandstraat - Den Burg", - "highlight": "Ada van Hollandstraat - Den Burg", - "href": "https://example.com/ogc/v1/collections/addresses/items/26?f=json", - "score": 0.09617967158555984 - }, - "geometry": { - "type": "Polygon", - "coordinates": [ - [ - [ - 108561.06373114887, - 551880.5267859272 - ], - [ - 122002.53096929599, - 551770.481156097 - ], - [ - 122154.00434113854, - 574026.9370521428 - ], - [ - 108774.26186957877, - 574136.6251694224 - ], - [ - 108561.06373114887, - 551880.5267859272 - ] - ] - ] - }, - "id": "26", - "links": [ - { - "rel": "canonical", - "title": "The actual feature in the corresponding OGC API", - "type": "application/geo+json", - "href": "https://example.com/ogc/v1/collections/addresses/items/26?f=json" - } - ] - }, - { - "type": "Feature", - "properties": { - "collectionGeometryType": "POINT", - "collectionId": "addresses", - "collectionVersion": "1", - "displayName": "Anne Frankstraat - Den Burg", - "highlight": "Anne Frankstraat - Den Burg", - "href": "https://example.com/ogc/v1/collections/addresses/items/474?f=json", - "score": 0.09617967158555984 - }, - "geometry": { - "type": "Polygon", - "coordinates": [ - [ - [ - 108334.04142692257, - 551768.4032578692 - ], - [ - 121775.82179598782, - 551657.7296376136 - ], - [ - 121928.33153477598, - 573914.1735634004 - ], - [ - 108548.27548981196, - 574024.4876471001 - ], - [ - 108334.04142692257, - 551768.4032578692 - ] - ] - ] - }, - "id": "474", - "links": [ - { - "rel": "canonical", - "title": "The actual feature in the corresponding OGC API", - "type": "application/geo+json", - "href": "https://example.com/ogc/v1/collections/addresses/items/474?f=json" + "href": "https://example.com/ogc/v1/collections/addresses/items/827?f=json" } ] } ], - "numberReturned": 8 + "numberReturned": 6 } diff --git a/internal/search/testdata/expected-search-den-single-collection-wgs84.json b/internal/search/testdata/expected-search-den-single-collection-wgs84.json index 0dbc6f9..b73eede 100644 --- a/internal/search/testdata/expected-search-den-single-collection-wgs84.json +++ b/internal/search/testdata/expected-search-den-single-collection-wgs84.json @@ -26,24 +26,24 @@ "coordinates": [ [ [ - 4.701721422439945, - 52.9619223105808 + 4.703246926093469, + 52.9627011349704 ], [ - 4.901721422439945, - 52.9619223105808 + 4.903246926093468, + 52.9627011349704 ], [ - 4.901721422439945, - 53.161922310580806 + 4.903246926093468, + 53.162701134970405 ], [ - 4.701721422439945, - 53.161922310580806 + 4.703246926093469, + 53.162701134970405 ], [ - 4.701721422439945, - 52.9619223105808 + 4.703246926093469, + 52.9627011349704 ] ] ] @@ -66,7 +66,7 @@ "collectionVersion": "1", "displayName": "Achterom - Den Burg", "highlight": "Achterom - Den Burg", - "href": "https://example.com/ogc/v1/collections/addresses/items/114?f=json", + "href": "https://example.com/ogc/v1/collections/addresses/items/22215?f=json", "score": 0.11162212491035461 }, "geometry": { @@ -74,35 +74,35 @@ "coordinates": [ [ [ - 4.699813158490893, - 52.95463219709524 + 4.699139629150976, + 52.95528084051514 ], [ - 4.899813158490892, - 52.95463219709524 + 4.899139629150976, + 52.95528084051514 ], [ - 4.899813158490892, - 53.154632197095246 + 4.899139629150976, + 53.15528084051514 ], [ - 4.699813158490893, - 53.154632197095246 + 4.699139629150976, + 53.15528084051514 ], [ - 4.699813158490893, - 52.95463219709524 + 4.699139629150976, + 52.95528084051514 ] ] ] }, - "id": "114", + "id": "22215", "links": [ { "rel": "canonical", "title": "The actual feature in the corresponding OGC API", "type": "application/geo+json", - "href": "https://example.com/ogc/v1/collections/addresses/items/114?f=json" + "href": "https://example.com/ogc/v1/collections/addresses/items/22215?f=json" } ] }, @@ -122,24 +122,24 @@ "coordinates": [ [ [ - 4.680059099895046, - 52.95346592050607 + 4.678620944865994, + 52.957118421718 ], [ - 4.880059099895045, - 52.95346592050607 + 4.878620944865993, + 52.957118421718 ], [ - 4.880059099895045, - 53.15346592050607 + 4.878620944865993, + 53.157118421718 ], [ - 4.680059099895046, - 53.15346592050607 + 4.678620944865994, + 53.157118421718 ], [ - 4.680059099895046, - 52.95346592050607 + 4.678620944865994, + 52.957118421718 ] ] ] @@ -258,7 +258,7 @@ "collectionVersion": "1", "displayName": "Beatrixlaan - Den Burg", "highlight": "Beatrixlaan - Den Burg", - "href": "https://example.com/ogc/v1/collections/addresses/items/591?f=json", + "href": "https://example.com/ogc/v1/collections/addresses/items/827?f=json", "score": 0.11162212491035461 }, "geometry": { @@ -266,134 +266,38 @@ "coordinates": [ [ [ - 4.690892824472019, - 52.95558352001795 + 4.691060243798572, + 52.956498997973284 ], [ - 4.890892824472019, - 52.95558352001795 + 4.891060243798571, + 52.956498997973284 ], [ - 4.890892824472019, - 53.155583520017956 + 4.891060243798571, + 53.15649899797329 ], [ - 4.690892824472019, - 53.155583520017956 + 4.691060243798572, + 53.15649899797329 ], [ - 4.690892824472019, - 52.95558352001795 + 4.691060243798572, + 52.956498997973284 ] ] ] }, - "id": "591", + "id": "827", "links": [ { "rel": "canonical", "title": "The actual feature in the corresponding OGC API", "type": "application/geo+json", - "href": "https://example.com/ogc/v1/collections/addresses/items/591?f=json" - } - ] - }, - { - "type": "Feature", - "properties": { - "collectionGeometryType": "POINT", - "collectionId": "addresses", - "collectionVersion": "1", - "displayName": "Ada van Hollandstraat - Den Burg", - "highlight": "Ada van Hollandstraat - Den Burg", - "href": "https://example.com/ogc/v1/collections/addresses/items/26?f=json", - "score": 0.09617967158555984 - }, - "geometry": { - "type": "Polygon", - "coordinates": [ - [ - [ - 4.696235388824104, - 52.95196001510249 - ], - [ - 4.8962353888241035, - 52.95196001510249 - ], - [ - 4.8962353888241035, - 53.151960015102496 - ], - [ - 4.696235388824104, - 53.151960015102496 - ], - [ - 4.696235388824104, - 52.95196001510249 - ] - ] - ] - }, - "id": "26", - "links": [ - { - "rel": "canonical", - "title": "The actual feature in the corresponding OGC API", - "type": "application/geo+json", - "href": "https://example.com/ogc/v1/collections/addresses/items/26?f=json" - } - ] - }, - { - "type": "Feature", - "properties": { - "collectionGeometryType": "POINT", - "collectionId": "addresses", - "collectionVersion": "1", - "displayName": "Anne Frankstraat - Den Burg", - "highlight": "Anne Frankstraat - Den Burg", - "href": "https://example.com/ogc/v1/collections/addresses/items/474?f=json", - "score": 0.09617967158555984 - }, - "geometry": { - "type": "Polygon", - "coordinates": [ - [ - [ - 4.692873779103581, - 52.950932925919574 - ], - [ - 4.892873779103581, - 52.950932925919574 - ], - [ - 4.892873779103581, - 53.15093292591958 - ], - [ - 4.692873779103581, - 53.15093292591958 - ], - [ - 4.692873779103581, - 52.950932925919574 - ] - ] - ] - }, - "id": "474", - "links": [ - { - "rel": "canonical", - "title": "The actual feature in the corresponding OGC API", - "type": "application/geo+json", - "href": "https://example.com/ogc/v1/collections/addresses/items/474?f=json" + "href": "https://example.com/ogc/v1/collections/addresses/items/827?f=json" } ] } ], - "numberReturned": 8 + "numberReturned": 6 } From c2572e0747b73ad8fe585d736aed0a40bd99e31b Mon Sep 17 00:00:00 2001 From: Richard Kettelerij Date: Mon, 16 Dec 2024 12:36:33 +0100 Subject: [PATCH 28/38] feat: add search endpoint - merge master --- internal/search/main_test.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/internal/search/main_test.go b/internal/search/main_test.go index e1b3df1..6668f47 100644 --- a/internal/search/main_test.go +++ b/internal/search/main_test.go @@ -67,8 +67,10 @@ func TestSearch(t *testing.T) { assert.NoError(t, err) collection := config.CollectionByID(conf, "addresses") table := config.FeatureTable{Name: "addresses", FID: "fid", Geom: "geom"} - err = etl.ImportFile(*collection, testSearchIndex, "internal/etl/testdata/addresses-crs84.gpkg", - "internal/etl/testdata/substitution.csv", table, 5000, dbConn) + err = etl.ImportFile(*collection, testSearchIndex, + "internal/etl/testdata/addresses-crs84.gpkg", + "internal/etl/testdata/substitution.csv", + "internal/etl/testdata/synonyms.csv", table, 5000, dbConn) assert.NoError(t, err) // run test cases From 1cdb9eae113a167d7aa76d1f05bcfb647a43b65a Mon Sep 17 00:00:00 2001 From: Richard Kettelerij Date: Mon, 16 Dec 2024 12:43:27 +0100 Subject: [PATCH 29/38] feat: add search endpoint - fix test --- internal/search/main_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/search/main_test.go b/internal/search/main_test.go index 6668f47..a3e065c 100644 --- a/internal/search/main_test.go +++ b/internal/search/main_test.go @@ -69,7 +69,7 @@ func TestSearch(t *testing.T) { table := config.FeatureTable{Name: "addresses", FID: "fid", Geom: "geom"} err = etl.ImportFile(*collection, testSearchIndex, "internal/etl/testdata/addresses-crs84.gpkg", - "internal/etl/testdata/substitution.csv", + "internal/etl/testdata/substitutions.csv", "internal/etl/testdata/synonyms.csv", table, 5000, dbConn) assert.NoError(t, err) From 1ebd5c8505af0aaa983ccd501e4f64cc4ac27f0b Mon Sep 17 00:00:00 2001 From: Richard Kettelerij Date: Wed, 18 Dec 2024 16:40:29 +0100 Subject: [PATCH 30/38] feat: add search endpoint - use collection_id + collection_version as tuples in where clause, instead of individually + add extra tests. --- internal/etl/load/postgres.go | 2 +- internal/etl/transform/transform.go | 5 +- .../search/datasources/postgres/postgres.go | 9 +- internal/search/main_test.go | 51 ++- internal/search/testdata/config.yaml | 56 ++++ ...-search-den-building-collection-wgs84.json | 303 ++++++++++++++++++ 6 files changed, 410 insertions(+), 16 deletions(-) create mode 100644 internal/search/testdata/config.yaml create mode 100644 internal/search/testdata/expected-search-den-building-collection-wgs84.json diff --git a/internal/etl/load/postgres.go b/internal/etl/load/postgres.go index 453c6b6..ac29070 100644 --- a/internal/etl/load/postgres.go +++ b/internal/etl/load/postgres.go @@ -58,7 +58,7 @@ func (p *Postgres) Init(index string) error { searchIndexTable := fmt.Sprintf(` create table if not exists %[1]s ( id serial, - feature_id varchar (8) not null , + feature_id text not null , collection_id text not null, collection_version int not null, display_name text not null, diff --git a/internal/etl/transform/transform.go b/internal/etl/transform/transform.go index 8fdded0..2d811c8 100644 --- a/internal/etl/transform/transform.go +++ b/internal/etl/transform/transform.go @@ -83,7 +83,10 @@ func (t Transformer) Transform(records []RawRecord, collection config.GeoSpatial } func (t Transformer) renderTemplate(templateFromConfig string, fieldValuesByName map[string]any) (string, error) { - parsedTemplate, err := template.New("").Funcs(engine.GlobalTemplateFuncs).Parse(templateFromConfig) + parsedTemplate, err := template.New(""). + Funcs(engine.GlobalTemplateFuncs). + Option("missingkey=zero"). + Parse(templateFromConfig) if err != nil { return "", err } diff --git a/internal/search/datasources/postgres/postgres.go b/internal/search/datasources/postgres/postgres.go index b73b510..e5225dc 100644 --- a/internal/search/datasources/postgres/postgres.go +++ b/internal/search/datasources/postgres/postgres.go @@ -54,8 +54,8 @@ func (p *Postgres) SearchFeaturesAcrossCollections(ctx context.Context, searchTe query := makeSearchQuery(p.searchIndex, srid) // Execute search query - collectionNames, collectionVersions := collections.NamesAndVersions() - rows, err := p.db.Query(queryCtx, query, limit, termsConcat, collectionNames, collectionVersions) + names, ints := collections.NamesAndVersions() + rows, err := p.db.Query(queryCtx, query, limit, termsConcat, names, ints) if err != nil { return nil, fmt.Errorf("query '%s' failed: %w", query, err) } @@ -81,7 +81,10 @@ func makeSearchQuery(index string, srid d.SRID) string { ts_rank_cd(ts, to_tsquery($2), 1) as rank, ts_headline('dutch', display_name, to_tsquery($2)) as highlighted_text from %[1]s - where ts @@ to_tsquery($2) and collection_id = any($3) and collection_version = any($4) + where ts @@ to_tsquery($2) and (collection_id, collection_version) in ( + -- make a virtual table by creating tuples from the provided arrays. + select * from unnest($3::text[], $4::int[]) + ) order by rank desc, display_name asc limit 500 ) r diff --git a/internal/search/main_test.go b/internal/search/main_test.go index a3e065c..f4c8b30 100644 --- a/internal/search/main_test.go +++ b/internal/search/main_test.go @@ -25,6 +25,7 @@ import ( ) const testSearchIndex = "search_index" +const configFile = "internal/search/testdata/config.yaml" func init() { // change working dir to root @@ -52,7 +53,7 @@ func TestSearch(t *testing.T) { dbConn := fmt.Sprintf("postgres://postgres:postgres@127.0.0.1:%d/%s?sslmode=disable", dbPort.Int(), "test_db") // given available engine - eng, err := engine.NewEngine("internal/etl/testdata/config.yaml", false, false) + eng, err := engine.NewEngine(configFile, false, false) assert.NoError(t, err) // given search endpoint @@ -62,15 +63,10 @@ func TestSearch(t *testing.T) { err = etl.CreateSearchIndex(dbConn, testSearchIndex) assert.NoError(t, err) - // given imported geopackage - conf, err := config.NewConfig("internal/etl/testdata/config.yaml") + // given imported geopackage (creates two collections in search_index with identical data) + err = importAddressesGpkg("addresses", dbConn) assert.NoError(t, err) - collection := config.CollectionByID(conf, "addresses") - table := config.FeatureTable{Name: "addresses", FID: "fid", Geom: "geom"} - err = etl.ImportFile(*collection, testSearchIndex, - "internal/etl/testdata/addresses-crs84.gpkg", - "internal/etl/testdata/substitutions.csv", - "internal/etl/testdata/synonyms.csv", table, 5000, dbConn) + err = importAddressesGpkg("buildings", dbConn) assert.NoError(t, err) // run test cases @@ -147,7 +143,17 @@ func TestSearch(t *testing.T) { }, }, { - name: "Search: 'Den' for multiple collections (with one not existing collection, so same output as single collection) in RD", + name: "Search: 'Den' in another collection in RD", + fields: fields{ + url: "http://localhost:8080/search?q=\"Den\"&buildings[version]=1&limit=10&f=json", + }, + want: want{ + body: "internal/search/testdata/expected-search-den-building-collection-wgs84.json", + statusCode: http.StatusOK, + }, + }, + { + name: "Search: 'Den' in multiple collections: with one non-existing collection, so same output as single collection) in RD", fields: fields{ url: "http://localhost:8080/search?q=\"Den\"&addresses[version]=1&addresses[relevance]=0.8&foo[version]=2&foo[relevance]=0.8&limit=10&f=json&crs=http%3A%2F%2Fwww.opengis.net%2Fdef%2Fcrs%2FEPSG%2F0%2F28992", }, @@ -156,10 +162,20 @@ func TestSearch(t *testing.T) { statusCode: http.StatusOK, }, }, + { + name: "Search: 'Den' in multiple collections: collection addresses + collection buildings, but addresses with non-existing version", + fields: fields{ + url: "http://localhost:8080/search?q=\"Den\"&addresses[version]=2&buildings[version]=1&limit=20&f=json", + }, + want: want{ + body: "internal/search/testdata/expected-search-den-building-collection-wgs84.json", // only expect building results since addresses version doesn't exist. + statusCode: http.StatusOK, + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - // mock time + // given mock time now = func() time.Time { return time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC) } engine.Now = now @@ -186,6 +202,19 @@ func TestSearch(t *testing.T) { } } +func importAddressesGpkg(collectionName string, dbConn string) error { + conf, err := config.NewConfig(configFile) + if err != nil { + return err + } + collection := config.CollectionByID(conf, collectionName) + table := config.FeatureTable{Name: "addresses", FID: "fid", Geom: "geom"} + return etl.ImportFile(*collection, testSearchIndex, + "internal/etl/testdata/addresses-crs84.gpkg", + "internal/etl/testdata/substitutions.csv", + "internal/etl/testdata/synonyms.csv", table, 5000, dbConn) +} + func setupPostgis(ctx context.Context, t *testing.T) (nat.Port, testcontainers.Container, error) { req := testcontainers.ContainerRequest{ Image: "docker.io/postgis/postgis:16-3.5-alpine", diff --git a/internal/search/testdata/config.yaml b/internal/search/testdata/config.yaml new file mode 100644 index 0000000..8d650ff --- /dev/null +++ b/internal/search/testdata/config.yaml @@ -0,0 +1,56 @@ +--- +version: 1.0.0 +lastUpdated: "2024-10-22T12:00:00Z" +baseUrl: http://localhost:8080 +availableLanguages: + - nl + - en +collections: + - id: addresses + metadata: + title: Addresses + description: These are example addresses + extent: + bbox: + - 50.2129 + - 2.52713 + - 55.7212 + - 7.37403 + search: + fields: + - component_thoroughfarename + - component_postaldescriptor + - component_addressareaname + displayNameTemplate: "{{ .component_thoroughfarename }} - {{ .component_addressareaname | firstupper }}" + etl: + suggestTemplates: + - "{{ .component_thoroughfarename }} {{ .component_addressareaname }}" + - "{{ .component_thoroughfarename }}, {{ .component_postaldescriptor }} {{ .component_addressareaname }}" + ogcCollections: + - api: https://example.com/ogc/v1 + collection: addresses + geometryType: point + - id: buildings + metadata: + title: Buildings + description: These are example buildings + extent: + bbox: + - 50.2129 + - 2.52713 + - 55.7212 + - 7.37403 + search: + fields: + - component_thoroughfarename + - component_postaldescriptor + - component_addressareaname + displayNameTemplate: "Building {{ .component_thoroughfarename }} - {{ .component_addressareaname | firstupper }}" + etl: + suggestTemplates: + - "{{ .component_thoroughfarename }} {{ .component_addressareaname }}" + - "{{ .component_thoroughfarename }}, {{ .component_postaldescriptor }} {{ .component_addressareaname }}" + ogcCollections: + - api: https://example.com/ogc/v1 + collection: buildings + geometryType: point \ No newline at end of file diff --git a/internal/search/testdata/expected-search-den-building-collection-wgs84.json b/internal/search/testdata/expected-search-den-building-collection-wgs84.json new file mode 100644 index 0000000..4f440cd --- /dev/null +++ b/internal/search/testdata/expected-search-den-building-collection-wgs84.json @@ -0,0 +1,303 @@ +{ + "type": "FeatureCollection", + "timeStamp": "2000-01-01T00:00:00Z", + "links": [ + { + "rel": "self", + "title": "This document as GeoJSON", + "type": "application/geo+json", + "href": "http://localhost:8080/search?f=json" + } + ], + "features": [ + { + "type": "Feature", + "properties": { + "collectionGeometryType": "POINT", + "collectionId": "buildings", + "collectionVersion": "1", + "displayName": "Building Abbewaal - Den Burg", + "highlight": "Building Abbewaal - Den Burg", + "href": "https://example.com/ogc/v1/collections/buildings/items/99?f=json", + "score": 0.10277967154979706 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 4.703246926093469, + 52.9627011349704 + ], + [ + 4.903246926093468, + 52.9627011349704 + ], + [ + 4.903246926093468, + 53.162701134970405 + ], + [ + 4.703246926093469, + 53.162701134970405 + ], + [ + 4.703246926093469, + 52.9627011349704 + ] + ] + ] + }, + "id": "99", + "links": [ + { + "rel": "canonical", + "title": "The actual feature in the corresponding OGC API", + "type": "application/geo+json", + "href": "https://example.com/ogc/v1/collections/buildings/items/99?f=json" + } + ] + }, + { + "type": "Feature", + "properties": { + "collectionGeometryType": "POINT", + "collectionId": "buildings", + "collectionVersion": "1", + "displayName": "Building Achterom - Den Burg", + "highlight": "Building Achterom - Den Burg", + "href": "https://example.com/ogc/v1/collections/buildings/items/22215?f=json", + "score": 0.10277967154979706 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 4.699139629150976, + 52.95528084051514 + ], + [ + 4.899139629150976, + 52.95528084051514 + ], + [ + 4.899139629150976, + 53.15528084051514 + ], + [ + 4.699139629150976, + 53.15528084051514 + ], + [ + 4.699139629150976, + 52.95528084051514 + ] + ] + ] + }, + "id": "22215", + "links": [ + { + "rel": "canonical", + "title": "The actual feature in the corresponding OGC API", + "type": "application/geo+json", + "href": "https://example.com/ogc/v1/collections/buildings/items/22215?f=json" + } + ] + }, + { + "type": "Feature", + "properties": { + "collectionGeometryType": "POINT", + "collectionId": "buildings", + "collectionVersion": "1", + "displayName": "Building Akenbuurt - Den Burg", + "highlight": "Building Akenbuurt - Den Burg", + "href": "https://example.com/ogc/v1/collections/buildings/items/46?f=json", + "score": 0.10277967154979706 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 4.678620944865994, + 52.957118421718 + ], + [ + 4.878620944865993, + 52.957118421718 + ], + [ + 4.878620944865993, + 53.157118421718 + ], + [ + 4.678620944865994, + 53.157118421718 + ], + [ + 4.678620944865994, + 52.957118421718 + ] + ] + ] + }, + "id": "46", + "links": [ + { + "rel": "canonical", + "title": "The actual feature in the corresponding OGC API", + "type": "application/geo+json", + "href": "https://example.com/ogc/v1/collections/buildings/items/46?f=json" + } + ] + }, + { + "type": "Feature", + "properties": { + "collectionGeometryType": "POINT", + "collectionId": "buildings", + "collectionVersion": "1", + "displayName": "Building Amaliaweg - Den Hoorn", + "highlight": "Building Amaliaweg - Den Hoorn", + "href": "https://example.com/ogc/v1/collections/buildings/items/50?f=json", + "score": 0.10277967154979706 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 4.68911630577304, + 52.92449928128154 + ], + [ + 4.889116305773039, + 52.92449928128154 + ], + [ + 4.889116305773039, + 53.124499281281544 + ], + [ + 4.68911630577304, + 53.124499281281544 + ], + [ + 4.68911630577304, + 52.92449928128154 + ] + ] + ] + }, + "id": "50", + "links": [ + { + "rel": "canonical", + "title": "The actual feature in the corresponding OGC API", + "type": "application/geo+json", + "href": "https://example.com/ogc/v1/collections/buildings/items/50?f=json" + } + ] + }, + { + "type": "Feature", + "properties": { + "collectionGeometryType": "POINT", + "collectionId": "buildings", + "collectionVersion": "1", + "displayName": "Building Bakkenweg - Den Hoorn", + "highlight": "Building Bakkenweg - Den Hoorn", + "href": "https://example.com/ogc/v1/collections/buildings/items/520?f=json", + "score": 0.10277967154979706 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 4.6548723261037095, + 52.94811743920973 + ], + [ + 4.854872326103709, + 52.94811743920973 + ], + [ + 4.854872326103709, + 53.148117439209734 + ], + [ + 4.6548723261037095, + 53.148117439209734 + ], + [ + 4.6548723261037095, + 52.94811743920973 + ] + ] + ] + }, + "id": "520", + "links": [ + { + "rel": "canonical", + "title": "The actual feature in the corresponding OGC API", + "type": "application/geo+json", + "href": "https://example.com/ogc/v1/collections/buildings/items/520?f=json" + } + ] + }, + { + "type": "Feature", + "properties": { + "collectionGeometryType": "POINT", + "collectionId": "buildings", + "collectionVersion": "1", + "displayName": "Building Beatrixlaan - Den Burg", + "highlight": "Building Beatrixlaan - Den Burg", + "href": "https://example.com/ogc/v1/collections/buildings/items/827?f=json", + "score": 0.10277967154979706 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 4.691060243798572, + 52.956498997973284 + ], + [ + 4.891060243798571, + 52.956498997973284 + ], + [ + 4.891060243798571, + 53.15649899797329 + ], + [ + 4.691060243798572, + 53.15649899797329 + ], + [ + 4.691060243798572, + 52.956498997973284 + ] + ] + ] + }, + "id": "827", + "links": [ + { + "rel": "canonical", + "title": "The actual feature in the corresponding OGC API", + "type": "application/geo+json", + "href": "https://example.com/ogc/v1/collections/buildings/items/827?f=json" + } + ] + } + ], + "numberReturned": 6 +} From 00e0399d785f5ce30b119cb378d1386891b86715 Mon Sep 17 00:00:00 2001 From: Richard Kettelerij Date: Wed, 18 Dec 2024 17:01:14 +0100 Subject: [PATCH 31/38] feat: add search endpoint - fix test --- internal/search/main_test.go | 2 +- ...ltiple-collection-single-output-wgs84.json | 303 ++++++++++++++++++ 2 files changed, 304 insertions(+), 1 deletion(-) create mode 100644 internal/search/testdata/expected-search-den-multiple-collection-single-output-wgs84.json diff --git a/internal/search/main_test.go b/internal/search/main_test.go index f4c8b30..ac50973 100644 --- a/internal/search/main_test.go +++ b/internal/search/main_test.go @@ -168,7 +168,7 @@ func TestSearch(t *testing.T) { url: "http://localhost:8080/search?q=\"Den\"&addresses[version]=2&buildings[version]=1&limit=20&f=json", }, want: want{ - body: "internal/search/testdata/expected-search-den-building-collection-wgs84.json", // only expect building results since addresses version doesn't exist. + body: "internal/search/testdata/expected-search-den-multiple-collection-single-output-wgs84.json", // only expect building results since addresses version doesn't exist. statusCode: http.StatusOK, }, }, diff --git a/internal/search/testdata/expected-search-den-multiple-collection-single-output-wgs84.json b/internal/search/testdata/expected-search-den-multiple-collection-single-output-wgs84.json new file mode 100644 index 0000000..01357eb --- /dev/null +++ b/internal/search/testdata/expected-search-den-multiple-collection-single-output-wgs84.json @@ -0,0 +1,303 @@ +{ + "type": "FeatureCollection", + "timeStamp": "2000-01-01T00:00:00Z", + "links": [ + { + "rel": "self", + "title": "This document as GeoJSON", + "type": "application/geo+json", + "href": "http://localhost:8080/search?f=json" + } + ], + "features": [ + { + "type": "Feature", + "properties": { + "collectionGeometryType": "POINT", + "collectionId": "buildings", + "collectionVersion": "1", + "displayName": "Building Abbewaal - Den Burg", + "highlight": "Building Abbewaal - Den Burg", + "href": "https://example.com/ogc/v1/collections/buildings/items/99?f=json", + "score": 0.10277967154979706 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 4.703246926093469, + 52.9627011349704 + ], + [ + 4.903246926093468, + 52.9627011349704 + ], + [ + 4.903246926093468, + 53.162701134970405 + ], + [ + 4.703246926093469, + 53.162701134970405 + ], + [ + 4.703246926093469, + 52.9627011349704 + ] + ] + ] + }, + "id": "99", + "links": [ + { + "rel": "canonical", + "title": "The actual feature in the corresponding OGC API", + "type": "application/geo+json", + "href": "https://example.com/ogc/v1/collections/buildings/items/99?f=json" + } + ] + }, + { + "type": "Feature", + "properties": { + "collectionGeometryType": "POINT", + "collectionId": "buildings", + "collectionVersion": "1", + "displayName": "Building Achterom - Den Burg", + "highlight": "Building Achterom - Den Burg", + "href": "https://example.com/ogc/v1/collections/buildings/items/22215?f=json", + "score": 0.10277967154979706 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 4.699139629150976, + 52.95528084051514 + ], + [ + 4.899139629150976, + 52.95528084051514 + ], + [ + 4.899139629150976, + 53.15528084051514 + ], + [ + 4.699139629150976, + 53.15528084051514 + ], + [ + 4.699139629150976, + 52.95528084051514 + ] + ] + ] + }, + "id": "22215", + "links": [ + { + "rel": "canonical", + "title": "The actual feature in the corresponding OGC API", + "type": "application/geo+json", + "href": "https://example.com/ogc/v1/collections/buildings/items/22215?f=json" + } + ] + }, + { + "type": "Feature", + "properties": { + "collectionGeometryType": "POINT", + "collectionId": "buildings", + "collectionVersion": "1", + "displayName": "Building Akenbuurt - Den Burg", + "highlight": "Building Akenbuurt - Den Burg", + "href": "https://example.com/ogc/v1/collections/buildings/items/46?f=json", + "score": 0.10277967154979706 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 4.678620944865994, + 52.957118421718 + ], + [ + 4.878620944865993, + 52.957118421718 + ], + [ + 4.878620944865993, + 53.157118421718 + ], + [ + 4.678620944865994, + 53.157118421718 + ], + [ + 4.678620944865994, + 52.957118421718 + ] + ] + ] + }, + "id": "46", + "links": [ + { + "rel": "canonical", + "title": "The actual feature in the corresponding OGC API", + "type": "application/geo+json", + "href": "https://example.com/ogc/v1/collections/buildings/items/46?f=json" + } + ] + }, + { + "type": "Feature", + "properties": { + "collectionGeometryType": "POINT", + "collectionId": "buildings", + "collectionVersion": "1", + "displayName": "Building Amaliaweg - Den Hoorn", + "highlight": "Building Amaliaweg - Den Hoorn", + "href": "https://example.com/ogc/v1/collections/buildings/items/50?f=json", + "score": 0.10277967154979706 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 4.68911630577304, + 52.92449928128154 + ], + [ + 4.889116305773039, + 52.92449928128154 + ], + [ + 4.889116305773039, + 53.124499281281544 + ], + [ + 4.68911630577304, + 53.124499281281544 + ], + [ + 4.68911630577304, + 52.92449928128154 + ] + ] + ] + }, + "id": "50", + "links": [ + { + "rel": "canonical", + "title": "The actual feature in the corresponding OGC API", + "type": "application/geo+json", + "href": "https://example.com/ogc/v1/collections/buildings/items/50?f=json" + } + ] + }, + { + "type": "Feature", + "properties": { + "collectionGeometryType": "POINT", + "collectionId": "buildings", + "collectionVersion": "1", + "displayName": "Building Bakkenweg - Den Hoorn", + "highlight": "Building Bakkenweg - Den Hoorn", + "href": "https://example.com/ogc/v1/collections/buildings/items/520?f=json", + "score": 0.10277967154979706 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 4.6548723261037095, + 52.94811743920973 + ], + [ + 4.854872326103709, + 52.94811743920973 + ], + [ + 4.854872326103709, + 53.148117439209734 + ], + [ + 4.6548723261037095, + 53.148117439209734 + ], + [ + 4.6548723261037095, + 52.94811743920973 + ] + ] + ] + }, + "id": "520", + "links": [ + { + "rel": "canonical", + "title": "The actual feature in the corresponding OGC API", + "type": "application/geo+json", + "href": "https://example.com/ogc/v1/collections/buildings/items/520?f=json" + } + ] + }, + { + "type": "Feature", + "properties": { + "collectionGeometryType": "POINT", + "collectionId": "buildings", + "collectionVersion": "1", + "displayName": "Building Beatrixlaan - Den Burg", + "highlight": "Building Beatrixlaan - Den Burg", + "href": "https://example.com/ogc/v1/collections/buildings/items/826?f=json", + "score": 0.10277967154979706 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 4.691060243798572, + 52.956498997973284 + ], + [ + 4.891060243798571, + 52.956498997973284 + ], + [ + 4.891060243798571, + 53.15649899797329 + ], + [ + 4.691060243798572, + 53.15649899797329 + ], + [ + 4.691060243798572, + 52.956498997973284 + ] + ] + ] + }, + "id": "826", + "links": [ + { + "rel": "canonical", + "title": "The actual feature in the corresponding OGC API", + "type": "application/geo+json", + "href": "https://example.com/ogc/v1/collections/buildings/items/826?f=json" + } + ] + } + ], + "numberReturned": 6 +} From f36887e55e35a16725464fcc79d1f7ff224f686a Mon Sep 17 00:00:00 2001 From: Richard Kettelerij Date: Thu, 19 Dec 2024 09:25:50 +0100 Subject: [PATCH 32/38] feat: add search endpoint - fix test --- internal/search/datasources/postgres/postgres.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/search/datasources/postgres/postgres.go b/internal/search/datasources/postgres/postgres.go index e5225dc..f2c4e14 100644 --- a/internal/search/datasources/postgres/postgres.go +++ b/internal/search/datasources/postgres/postgres.go @@ -88,7 +88,7 @@ func makeSearchQuery(index string, srid d.SRID) string { order by rank desc, display_name asc limit 500 ) r - group by r.display_name + group by r.display_name, r.collection_id, r.collection_version order by rank desc, display_name asc limit $1`, index, srid) // don't add user input here, use $X params for user input! From 8be5610bf69b704a99e720839009f1ef2d7f179d Mon Sep 17 00:00:00 2001 From: Richard Kettelerij Date: Thu, 19 Dec 2024 15:10:05 +0100 Subject: [PATCH 33/38] feat: add search endpoint - fix tests, multiple changes: - group by more fields to prevent dups - optimize performance by executing ts_query once - include housenr in testdata - covert all fields used in suggestions to strings since the suggestion field in the index is always text/string. --- internal/etl/transform/extend_values.go | 14 +- internal/etl/transform/extend_values_test.go | 24 +- internal/etl/transform/transform.go | 16 +- .../search/datasources/postgres/postgres.go | 15 +- internal/search/testdata/config.yaml | 10 +- ...-search-den-building-collection-wgs84.json | 354 ++++++-- ...ltiple-collection-single-output-wgs84.json | 834 ++++++++++++++++-- ...ected-search-den-single-collection-rd.json | 366 ++++++-- ...ed-search-den-single-collection-wgs84.json | 366 ++++++-- 9 files changed, 1630 insertions(+), 369 deletions(-) diff --git a/internal/etl/transform/extend_values.go b/internal/etl/transform/extend_values.go index 05e0d0a..e9c3355 100644 --- a/internal/etl/transform/extend_values.go +++ b/internal/etl/transform/extend_values.go @@ -9,7 +9,7 @@ import ( ) // Return slice of fieldValuesByName -func extendFieldValues(fieldValuesByName map[string]any, substitutionsFile, synonymsFile string) ([]map[string]any, error) { +func extendFieldValues(fieldValuesByName map[string]string, substitutionsFile, synonymsFile string) ([]map[string]string, error) { substitutions, err := readCsvFile(substitutionsFile) if err != nil { return nil, err @@ -21,7 +21,7 @@ func extendFieldValues(fieldValuesByName map[string]any, substitutionsFile, syno var fieldValuesByNameWithAllValues = make(map[string][]string) for key, value := range fieldValuesByName { - valueLower := strings.ToLower(value.(string)) + valueLower := strings.ToLower(value) // Get all substitutions substitutedValues, err := extendValues([]string{valueLower}, substitutions) @@ -49,7 +49,7 @@ func extendFieldValues(fieldValuesByName map[string]any, substitutionsFile, syno // Transform a map[string][]string into a []map[string]string using the cartesian product, i.e. // - both maps have the same keys // - values exist for all possible combinations -func generateAllFieldValuesByName(input map[string][]string) []map[string]any { +func generateAllFieldValuesByName(input map[string][]string) []map[string]string { keys := []string{} values := [][]string{} @@ -61,16 +61,16 @@ func generateAllFieldValuesByName(input map[string][]string) []map[string]any { return generateCombinations(keys, values) } -func generateCombinations(keys []string, values [][]string) []map[string]any { +func generateCombinations(keys []string, values [][]string) []map[string]string { if len(keys) == 0 || len(values) == 0 { return nil } - result := []map[string]any{{}} // contains empty map so the first iteration works + result := []map[string]string{{}} // contains empty map so the first iteration works for keyDepth := 0; keyDepth < len(keys); keyDepth++ { - var newResult []map[string]any + var newResult []map[string]string for _, entry := range result { for _, val := range values[keyDepth] { - newEntry := make(map[string]any) + newEntry := make(map[string]string) for k, v := range entry { newEntry[k] = v } diff --git a/internal/etl/transform/extend_values_test.go b/internal/etl/transform/extend_values_test.go index 043d1aa..1f2d2e9 100644 --- a/internal/etl/transform/extend_values_test.go +++ b/internal/etl/transform/extend_values_test.go @@ -9,20 +9,20 @@ import ( func Test_generateAllFieldValues(t *testing.T) { type args struct { - fieldValuesByName map[string]any + fieldValuesByName map[string]string substitutionsFile string synonymsFile string } tests := []struct { name string args args - want []map[string]any + want []map[string]string wantErr assert.ErrorAssertionFunc }{ - {"simple record", args{map[string]any{"component_thoroughfarename": "foo", "component_postaldescriptor": "1234AB", "component_addressareaname": "bar"}, "../testdata/substitutions.csv", "../testdata/synonyms.csv"}, []map[string]any{{"component_thoroughfarename": "foo", "component_postaldescriptor": "1234ab", "component_addressareaname": "bar"}}, assert.NoError}, - {"single synonym record", args{map[string]any{"component_thoroughfarename": "eerste", "component_postaldescriptor": "1234AB", "component_addressareaname": "bar"}, "../testdata/substitutions.csv", "../testdata/synonyms.csv"}, []map[string]any{{"component_thoroughfarename": "eerste", "component_postaldescriptor": "1234ab", "component_addressareaname": "bar"}, {"component_thoroughfarename": "1ste", "component_postaldescriptor": "1234ab", "component_addressareaname": "bar"}}, assert.NoError}, - {"single synonym with capital", args{map[string]any{"component_thoroughfarename": "Eerste", "component_postaldescriptor": "1234AB", "component_addressareaname": "bar"}, "../testdata/substitutions.csv", "../testdata/synonyms.csv"}, []map[string]any{{"component_thoroughfarename": "eerste", "component_postaldescriptor": "1234ab", "component_addressareaname": "bar"}, {"component_thoroughfarename": "1ste", "component_postaldescriptor": "1234ab", "component_addressareaname": "bar"}}, assert.NoError}, - {"two-way synonym record", args{map[string]any{"component_thoroughfarename": "eerste 2de", "component_postaldescriptor": "1234AB", "component_addressareaname": "bar"}, "../testdata/substitutions.csv", "../testdata/synonyms.csv"}, []map[string]any{{"component_thoroughfarename": "eerste 2de", "component_postaldescriptor": "1234ab", "component_addressareaname": "bar"}, {"component_thoroughfarename": "1ste 2de", "component_postaldescriptor": "1234ab", "component_addressareaname": "bar"}, {"component_thoroughfarename": "eerste tweede", "component_postaldescriptor": "1234ab", "component_addressareaname": "bar"}, {"component_thoroughfarename": "1ste tweede", "component_postaldescriptor": "1234ab", "component_addressareaname": "bar"}}, assert.NoError}, + {"simple record", args{map[string]string{"component_thoroughfarename": "foo", "component_postaldescriptor": "1234AB", "component_addressareaname": "bar"}, "../testdata/substitutions.csv", "../testdata/synonyms.csv"}, []map[string]string{{"component_thoroughfarename": "foo", "component_postaldescriptor": "1234ab", "component_addressareaname": "bar"}}, assert.NoError}, + {"single synonym record", args{map[string]string{"component_thoroughfarename": "eerste", "component_postaldescriptor": "1234AB", "component_addressareaname": "bar"}, "../testdata/substitutions.csv", "../testdata/synonyms.csv"}, []map[string]string{{"component_thoroughfarename": "eerste", "component_postaldescriptor": "1234ab", "component_addressareaname": "bar"}, {"component_thoroughfarename": "1ste", "component_postaldescriptor": "1234ab", "component_addressareaname": "bar"}}, assert.NoError}, + {"single synonym with capital", args{map[string]string{"component_thoroughfarename": "Eerste", "component_postaldescriptor": "1234AB", "component_addressareaname": "bar"}, "../testdata/substitutions.csv", "../testdata/synonyms.csv"}, []map[string]string{{"component_thoroughfarename": "eerste", "component_postaldescriptor": "1234ab", "component_addressareaname": "bar"}, {"component_thoroughfarename": "1ste", "component_postaldescriptor": "1234ab", "component_addressareaname": "bar"}}, assert.NoError}, + {"two-way synonym record", args{map[string]string{"component_thoroughfarename": "eerste 2de", "component_postaldescriptor": "1234AB", "component_addressareaname": "bar"}, "../testdata/substitutions.csv", "../testdata/synonyms.csv"}, []map[string]string{{"component_thoroughfarename": "eerste 2de", "component_postaldescriptor": "1234ab", "component_addressareaname": "bar"}, {"component_thoroughfarename": "1ste 2de", "component_postaldescriptor": "1234ab", "component_addressareaname": "bar"}, {"component_thoroughfarename": "eerste tweede", "component_postaldescriptor": "1234ab", "component_addressareaname": "bar"}, {"component_thoroughfarename": "1ste tweede", "component_postaldescriptor": "1234ab", "component_addressareaname": "bar"}}, assert.NoError}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -43,13 +43,13 @@ func Test_generateCombinations(t *testing.T) { tests := []struct { name string args args - want []map[string]any + want []map[string]string }{ - {"Single key, single value", args{[]string{"key1"}, [][]string{{"value1"}}}, []map[string]any{{"key1": "value1"}}}, - {"Single key, slice of values", args{[]string{"key1"}, [][]string{{"value1", "value2"}}}, []map[string]any{{"key1": "value1"}, {"key1": "value2"}}}, - {"Two keys, two single values", args{[]string{"key1", "key2"}, [][]string{{"value1"}, {"value2"}}}, []map[string]any{{"key1": "value1", "key2": "value2"}}}, - {"Two keys, slice + single value", args{[]string{"key1", "key2"}, [][]string{{"value1", "value2"}, {"value3"}}}, []map[string]any{{"key1": "value1", "key2": "value3"}, {"key1": "value2", "key2": "value3"}}}, - {"Two keys, two slices values", args{[]string{"key1", "key2"}, [][]string{{"value1", "value2"}, {"value3", "value4"}}}, []map[string]any{{"key1": "value1", "key2": "value3"}, {"key1": "value1", "key2": "value4"}, {"key1": "value2", "key2": "value3"}, {"key1": "value2", "key2": "value4"}}}, + {"Single key, single value", args{[]string{"key1"}, [][]string{{"value1"}}}, []map[string]string{{"key1": "value1"}}}, + {"Single key, slice of values", args{[]string{"key1"}, [][]string{{"value1", "value2"}}}, []map[string]string{{"key1": "value1"}, {"key1": "value2"}}}, + {"Two keys, two single values", args{[]string{"key1", "key2"}, [][]string{{"value1"}, {"value2"}}}, []map[string]string{{"key1": "value1", "key2": "value2"}}}, + {"Two keys, slice + single value", args{[]string{"key1", "key2"}, [][]string{{"value1", "value2"}, {"value3"}}}, []map[string]string{{"key1": "value1", "key2": "value3"}, {"key1": "value2", "key2": "value3"}}}, + {"Two keys, two slices values", args{[]string{"key1", "key2"}, [][]string{{"value1", "value2"}, {"value3", "value4"}}}, []map[string]string{{"key1": "value1", "key2": "value3"}, {"key1": "value1", "key2": "value4"}, {"key1": "value2", "key2": "value3"}, {"key1": "value2", "key2": "value4"}}}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/internal/etl/transform/transform.go b/internal/etl/transform/transform.go index 2d811c8..3013f05 100644 --- a/internal/etl/transform/transform.go +++ b/internal/etl/transform/transform.go @@ -38,7 +38,7 @@ type Transformer struct{} func (t Transformer) Transform(records []RawRecord, collection config.GeoSpatialCollection, substitutionsFile string, synonymsFile string) ([]SearchIndexRecord, error) { result := make([]SearchIndexRecord, 0, len(records)) for _, r := range records { - fieldValuesByName, err := slicesToMap(collection.Search.Fields, r.FieldValues) + fieldValuesByName, err := slicesToStringMap(collection.Search.Fields, r.FieldValues) if err != nil { return nil, err } @@ -82,7 +82,7 @@ func (t Transformer) Transform(records []RawRecord, collection config.GeoSpatial return result, nil } -func (t Transformer) renderTemplate(templateFromConfig string, fieldValuesByName map[string]any) (string, error) { +func (t Transformer) renderTemplate(templateFromConfig string, fieldValuesByName map[string]string) (string, error) { parsedTemplate, err := template.New(""). Funcs(engine.GlobalTemplateFuncs). Option("missingkey=zero"). @@ -94,7 +94,7 @@ func (t Transformer) renderTemplate(templateFromConfig string, fieldValuesByName if err = parsedTemplate.Execute(&b, fieldValuesByName); err != nil { return "", err } - return b.String(), err + return strings.TrimSpace(b.String()), err } func (r RawRecord) transformBbox() (*pggeom.Polygon, error) { @@ -122,13 +122,17 @@ func (r RawRecord) transformBbox() (*pggeom.Polygon, error) { return polygon, nil } -func slicesToMap(keys []string, values []any) (map[string]any, error) { +func slicesToStringMap(keys []string, values []any) (map[string]string, error) { if len(keys) != len(values) { return nil, fmt.Errorf("slices must be of the same length, got %d keys and %d values", len(keys), len(values)) } - result := make(map[string]any, len(keys)) + result := make(map[string]string, len(keys)) for i := range keys { - result[keys[i]] = values[i] + value := values[i] + if value != nil { + stringValue := fmt.Sprintf("%v", value) + result[keys[i]] = stringValue + } } return result, nil } diff --git a/internal/search/datasources/postgres/postgres.go b/internal/search/datasources/postgres/postgres.go index f2c4e14..698bc37 100644 --- a/internal/search/datasources/postgres/postgres.go +++ b/internal/search/datasources/postgres/postgres.go @@ -68,27 +68,30 @@ func (p *Postgres) SearchFeaturesAcrossCollections(ctx context.Context, searchTe func makeSearchQuery(index string, srid d.SRID) string { // language=postgresql query := fmt.Sprintf(` + with query as ( + select to_tsquery('dutch', $2) query + ) select r.display_name as display_name, max(r.feature_id) as feature_id, max(r.collection_id) as collection_id, max(r.collection_version) as collection_version, max(r.geometry_type) as geometry_type, - cast(st_transform(max(r.bbox), %[2]d) as geometry) as bbox, + st_transform(max(r.bbox), %[2]d)::geometry as bbox, max(r.rank) as rank, max(r.highlighted_text) as highlighted_text from ( select display_name, feature_id, collection_id, collection_version, geometry_type, bbox, - ts_rank_cd(ts, to_tsquery($2), 1) as rank, - ts_headline('dutch', display_name, to_tsquery($2)) as highlighted_text + ts_rank_cd(ts, (select query from query), 1) as rank, + ts_headline('dutch', display_name, (select query from query)) as highlighted_text from %[1]s - where ts @@ to_tsquery($2) and (collection_id, collection_version) in ( + where ts @@ (select query from query) and (collection_id, collection_version) in ( -- make a virtual table by creating tuples from the provided arrays. select * from unnest($3::text[], $4::int[]) ) - order by rank desc, display_name asc + order by rank desc, display_name asc -- keep the same as outer 'order by' clause limit 500 ) r - group by r.display_name, r.collection_id, r.collection_version + group by r.display_name, r.collection_id, r.collection_version, r.feature_id order by rank desc, display_name asc limit $1`, index, srid) // don't add user input here, use $X params for user input! diff --git a/internal/search/testdata/config.yaml b/internal/search/testdata/config.yaml index 8d650ff..996e439 100644 --- a/internal/search/testdata/config.yaml +++ b/internal/search/testdata/config.yaml @@ -21,7 +21,10 @@ collections: - component_thoroughfarename - component_postaldescriptor - component_addressareaname - displayNameTemplate: "{{ .component_thoroughfarename }} - {{ .component_addressareaname | firstupper }}" + - locator_designator_addressnumber + - locator_designator_addressnumberextension + - locator_designator_addressnumber2ndextension + displayNameTemplate: "{{ .component_thoroughfarename }} - {{ .component_addressareaname | firstupper }} {{ .locator_designator_addressnumber }} {{ .locator_designator_addressnumberextension }} {{ .locator_designator_addressnumber2ndextension }}" etl: suggestTemplates: - "{{ .component_thoroughfarename }} {{ .component_addressareaname }}" @@ -45,7 +48,10 @@ collections: - component_thoroughfarename - component_postaldescriptor - component_addressareaname - displayNameTemplate: "Building {{ .component_thoroughfarename }} - {{ .component_addressareaname | firstupper }}" + - locator_designator_addressnumber + - locator_designator_addressnumberextension + - locator_designator_addressnumber2ndextension + displayNameTemplate: "{{ .component_thoroughfarename }} - {{ .component_addressareaname | firstupper }} {{ .locator_designator_addressnumber }} {{ .locator_designator_addressnumberextension }} {{ .locator_designator_addressnumber2ndextension }}" etl: suggestTemplates: - "{{ .component_thoroughfarename }} {{ .component_addressareaname }}" diff --git a/internal/search/testdata/expected-search-den-building-collection-wgs84.json b/internal/search/testdata/expected-search-den-building-collection-wgs84.json index 4f440cd..7092af7 100644 --- a/internal/search/testdata/expected-search-den-building-collection-wgs84.json +++ b/internal/search/testdata/expected-search-den-building-collection-wgs84.json @@ -16,9 +16,201 @@ "collectionGeometryType": "POINT", "collectionId": "buildings", "collectionVersion": "1", - "displayName": "Building Abbewaal - Den Burg", - "highlight": "Building Abbewaal - Den Burg", - "href": "https://example.com/ogc/v1/collections/buildings/items/99?f=json", + "displayName": "Abbewaal - Den Burg 1", + "highlight": "Abbewaal - Den Burg 1", + "href": "https://example.com/ogc/v1/collections/buildings/items/51?f=json", + "score": 0.10277967154979706 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 4.701583684040178, + 52.96172161329086 + ], + [ + 4.901583684040177, + 52.96172161329086 + ], + [ + 4.901583684040177, + 53.161721613290865 + ], + [ + 4.701583684040178, + 53.161721613290865 + ], + [ + 4.701583684040178, + 52.96172161329086 + ] + ] + ] + }, + "id": "51", + "links": [ + { + "rel": "canonical", + "title": "The actual feature in the corresponding OGC API", + "type": "application/geo+json", + "href": "https://example.com/ogc/v1/collections/buildings/items/51?f=json" + } + ] + }, + { + "type": "Feature", + "properties": { + "collectionGeometryType": "POINT", + "collectionId": "buildings", + "collectionVersion": "1", + "displayName": "Abbewaal - Den Burg 1", + "highlight": "Abbewaal - Den Burg 1", + "href": "https://example.com/ogc/v1/collections/buildings/items/52?f=json", + "score": 0.10277967154979706 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 4.701583684040178, + 52.96172161329086 + ], + [ + 4.901583684040177, + 52.96172161329086 + ], + [ + 4.901583684040177, + 53.161721613290865 + ], + [ + 4.701583684040178, + 53.161721613290865 + ], + [ + 4.701583684040178, + 52.96172161329086 + ] + ] + ] + }, + "id": "52", + "links": [ + { + "rel": "canonical", + "title": "The actual feature in the corresponding OGC API", + "type": "application/geo+json", + "href": "https://example.com/ogc/v1/collections/buildings/items/52?f=json" + } + ] + }, + { + "type": "Feature", + "properties": { + "collectionGeometryType": "POINT", + "collectionId": "buildings", + "collectionVersion": "1", + "displayName": "Abbewaal - Den Burg 10", + "highlight": "Abbewaal - Den Burg 10", + "href": "https://example.com/ogc/v1/collections/buildings/items/32183?f=json", + "score": 0.10277967154979706 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 4.703248382838079, + 52.96279808509741 + ], + [ + 4.903248382838078, + 52.96279808509741 + ], + [ + 4.903248382838078, + 53.162798085097414 + ], + [ + 4.703248382838079, + 53.162798085097414 + ], + [ + 4.703248382838079, + 52.96279808509741 + ] + ] + ] + }, + "id": "32183", + "links": [ + { + "rel": "canonical", + "title": "The actual feature in the corresponding OGC API", + "type": "application/geo+json", + "href": "https://example.com/ogc/v1/collections/buildings/items/32183?f=json" + } + ] + }, + { + "type": "Feature", + "properties": { + "collectionGeometryType": "POINT", + "collectionId": "buildings", + "collectionVersion": "1", + "displayName": "Abbewaal - Den Burg 10", + "highlight": "Abbewaal - Den Burg 10", + "href": "https://example.com/ogc/v1/collections/buildings/items/53?f=json", + "score": 0.10277967154979706 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 4.703248382838079, + 52.96279808509741 + ], + [ + 4.903248382838078, + 52.96279808509741 + ], + [ + 4.903248382838078, + 53.162798085097414 + ], + [ + 4.703248382838079, + 53.162798085097414 + ], + [ + 4.703248382838079, + 52.96279808509741 + ] + ] + ] + }, + "id": "53", + "links": [ + { + "rel": "canonical", + "title": "The actual feature in the corresponding OGC API", + "type": "application/geo+json", + "href": "https://example.com/ogc/v1/collections/buildings/items/53?f=json" + } + ] + }, + { + "type": "Feature", + "properties": { + "collectionGeometryType": "POINT", + "collectionId": "buildings", + "collectionVersion": "1", + "displayName": "Abbewaal - Den Burg 10", + "highlight": "Abbewaal - Den Burg 10", + "href": "https://example.com/ogc/v1/collections/buildings/items/32184?f=json", "score": 0.10277967154979706 }, "geometry": { @@ -48,13 +240,13 @@ ] ] }, - "id": "99", + "id": "32184", "links": [ { "rel": "canonical", "title": "The actual feature in the corresponding OGC API", "type": "application/geo+json", - "href": "https://example.com/ogc/v1/collections/buildings/items/99?f=json" + "href": "https://example.com/ogc/v1/collections/buildings/items/32184?f=json" } ] }, @@ -64,9 +256,9 @@ "collectionGeometryType": "POINT", "collectionId": "buildings", "collectionVersion": "1", - "displayName": "Building Achterom - Den Burg", - "highlight": "Building Achterom - Den Burg", - "href": "https://example.com/ogc/v1/collections/buildings/items/22215?f=json", + "displayName": "Abbewaal - Den Burg 10", + "highlight": "Abbewaal - Den Burg 10", + "href": "https://example.com/ogc/v1/collections/buildings/items/54?f=json", "score": 0.10277967154979706 }, "geometry": { @@ -74,35 +266,35 @@ "coordinates": [ [ [ - 4.699139629150976, - 52.95528084051514 + 4.703248382838079, + 52.96279808509741 ], [ - 4.899139629150976, - 52.95528084051514 + 4.903248382838078, + 52.96279808509741 ], [ - 4.899139629150976, - 53.15528084051514 + 4.903248382838078, + 53.162798085097414 ], [ - 4.699139629150976, - 53.15528084051514 + 4.703248382838079, + 53.162798085097414 ], [ - 4.699139629150976, - 52.95528084051514 + 4.703248382838079, + 52.96279808509741 ] ] ] }, - "id": "22215", + "id": "54", "links": [ { "rel": "canonical", "title": "The actual feature in the corresponding OGC API", "type": "application/geo+json", - "href": "https://example.com/ogc/v1/collections/buildings/items/22215?f=json" + "href": "https://example.com/ogc/v1/collections/buildings/items/54?f=json" } ] }, @@ -112,9 +304,9 @@ "collectionGeometryType": "POINT", "collectionId": "buildings", "collectionVersion": "1", - "displayName": "Building Akenbuurt - Den Burg", - "highlight": "Building Akenbuurt - Den Burg", - "href": "https://example.com/ogc/v1/collections/buildings/items/46?f=json", + "displayName": "Abbewaal - Den Burg 11", + "highlight": "Abbewaal - Den Burg 11", + "href": "https://example.com/ogc/v1/collections/buildings/items/22549?f=json", "score": 0.10277967154979706 }, "geometry": { @@ -122,35 +314,35 @@ "coordinates": [ [ [ - 4.678620944865994, - 52.957118421718 + 4.701889188504978, + 52.962632527818215 ], [ - 4.878620944865993, - 52.957118421718 + 4.901889188504978, + 52.962632527818215 ], [ - 4.878620944865993, - 53.157118421718 + 4.901889188504978, + 53.16263252781822 ], [ - 4.678620944865994, - 53.157118421718 + 4.701889188504978, + 53.16263252781822 ], [ - 4.678620944865994, - 52.957118421718 + 4.701889188504978, + 52.962632527818215 ] ] ] }, - "id": "46", + "id": "22549", "links": [ { "rel": "canonical", "title": "The actual feature in the corresponding OGC API", "type": "application/geo+json", - "href": "https://example.com/ogc/v1/collections/buildings/items/46?f=json" + "href": "https://example.com/ogc/v1/collections/buildings/items/22549?f=json" } ] }, @@ -160,9 +352,9 @@ "collectionGeometryType": "POINT", "collectionId": "buildings", "collectionVersion": "1", - "displayName": "Building Amaliaweg - Den Hoorn", - "highlight": "Building Amaliaweg - Den Hoorn", - "href": "https://example.com/ogc/v1/collections/buildings/items/50?f=json", + "displayName": "Abbewaal - Den Burg 13", + "highlight": "Abbewaal - Den Burg 13", + "href": "https://example.com/ogc/v1/collections/buildings/items/56?f=json", "score": 0.10277967154979706 }, "geometry": { @@ -170,35 +362,35 @@ "coordinates": [ [ [ - 4.68911630577304, - 52.92449928128154 + 4.702154512142386, + 52.96275394293894 ], [ - 4.889116305773039, - 52.92449928128154 + 4.902154512142385, + 52.96275394293894 ], [ - 4.889116305773039, - 53.124499281281544 + 4.902154512142385, + 53.16275394293894 ], [ - 4.68911630577304, - 53.124499281281544 + 4.702154512142386, + 53.16275394293894 ], [ - 4.68911630577304, - 52.92449928128154 + 4.702154512142386, + 52.96275394293894 ] ] ] }, - "id": "50", + "id": "56", "links": [ { "rel": "canonical", "title": "The actual feature in the corresponding OGC API", "type": "application/geo+json", - "href": "https://example.com/ogc/v1/collections/buildings/items/50?f=json" + "href": "https://example.com/ogc/v1/collections/buildings/items/56?f=json" } ] }, @@ -208,9 +400,9 @@ "collectionGeometryType": "POINT", "collectionId": "buildings", "collectionVersion": "1", - "displayName": "Building Bakkenweg - Den Hoorn", - "highlight": "Building Bakkenweg - Den Hoorn", - "href": "https://example.com/ogc/v1/collections/buildings/items/520?f=json", + "displayName": "Abbewaal - Den Burg 13", + "highlight": "Abbewaal - Den Burg 13", + "href": "https://example.com/ogc/v1/collections/buildings/items/55?f=json", "score": 0.10277967154979706 }, "geometry": { @@ -218,35 +410,35 @@ "coordinates": [ [ [ - 4.6548723261037095, - 52.94811743920973 + 4.702154512142386, + 52.96275394293894 ], [ - 4.854872326103709, - 52.94811743920973 + 4.902154512142385, + 52.96275394293894 ], [ - 4.854872326103709, - 53.148117439209734 + 4.902154512142385, + 53.16275394293894 ], [ - 4.6548723261037095, - 53.148117439209734 + 4.702154512142386, + 53.16275394293894 ], [ - 4.6548723261037095, - 52.94811743920973 + 4.702154512142386, + 52.96275394293894 ] ] ] }, - "id": "520", + "id": "55", "links": [ { "rel": "canonical", "title": "The actual feature in the corresponding OGC API", "type": "application/geo+json", - "href": "https://example.com/ogc/v1/collections/buildings/items/520?f=json" + "href": "https://example.com/ogc/v1/collections/buildings/items/55?f=json" } ] }, @@ -256,9 +448,9 @@ "collectionGeometryType": "POINT", "collectionId": "buildings", "collectionVersion": "1", - "displayName": "Building Beatrixlaan - Den Burg", - "highlight": "Building Beatrixlaan - Den Burg", - "href": "https://example.com/ogc/v1/collections/buildings/items/827?f=json", + "displayName": "Abbewaal - Den Burg 15", + "highlight": "Abbewaal - Den Burg 15", + "href": "https://example.com/ogc/v1/collections/buildings/items/57?f=json", "score": 0.10277967154979706 }, "geometry": { @@ -266,38 +458,38 @@ "coordinates": [ [ [ - 4.691060243798572, - 52.956498997973284 + 4.702695564526969, + 52.962965478349716 ], [ - 4.891060243798571, - 52.956498997973284 + 4.902695564526968, + 52.962965478349716 ], [ - 4.891060243798571, - 53.15649899797329 + 4.902695564526968, + 53.16296547834972 ], [ - 4.691060243798572, - 53.15649899797329 + 4.702695564526969, + 53.16296547834972 ], [ - 4.691060243798572, - 52.956498997973284 + 4.702695564526969, + 52.962965478349716 ] ] ] }, - "id": "827", + "id": "57", "links": [ { "rel": "canonical", "title": "The actual feature in the corresponding OGC API", "type": "application/geo+json", - "href": "https://example.com/ogc/v1/collections/buildings/items/827?f=json" + "href": "https://example.com/ogc/v1/collections/buildings/items/57?f=json" } ] } ], - "numberReturned": 6 + "numberReturned": 10 } diff --git a/internal/search/testdata/expected-search-den-multiple-collection-single-output-wgs84.json b/internal/search/testdata/expected-search-den-multiple-collection-single-output-wgs84.json index 01357eb..adfbbe0 100644 --- a/internal/search/testdata/expected-search-den-multiple-collection-single-output-wgs84.json +++ b/internal/search/testdata/expected-search-den-multiple-collection-single-output-wgs84.json @@ -16,9 +16,201 @@ "collectionGeometryType": "POINT", "collectionId": "buildings", "collectionVersion": "1", - "displayName": "Building Abbewaal - Den Burg", - "highlight": "Building Abbewaal - Den Burg", - "href": "https://example.com/ogc/v1/collections/buildings/items/99?f=json", + "displayName": "Abbewaal - Den Burg 1", + "highlight": "Abbewaal - Den Burg 1", + "href": "https://example.com/ogc/v1/collections/buildings/items/51?f=json", + "score": 0.10277967154979706 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 4.701583684040178, + 52.96172161329086 + ], + [ + 4.901583684040177, + 52.96172161329086 + ], + [ + 4.901583684040177, + 53.161721613290865 + ], + [ + 4.701583684040178, + 53.161721613290865 + ], + [ + 4.701583684040178, + 52.96172161329086 + ] + ] + ] + }, + "id": "51", + "links": [ + { + "rel": "canonical", + "title": "The actual feature in the corresponding OGC API", + "type": "application/geo+json", + "href": "https://example.com/ogc/v1/collections/buildings/items/51?f=json" + } + ] + }, + { + "type": "Feature", + "properties": { + "collectionGeometryType": "POINT", + "collectionId": "buildings", + "collectionVersion": "1", + "displayName": "Abbewaal - Den Burg 1", + "highlight": "Abbewaal - Den Burg 1", + "href": "https://example.com/ogc/v1/collections/buildings/items/52?f=json", + "score": 0.10277967154979706 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 4.701583684040178, + 52.96172161329086 + ], + [ + 4.901583684040177, + 52.96172161329086 + ], + [ + 4.901583684040177, + 53.161721613290865 + ], + [ + 4.701583684040178, + 53.161721613290865 + ], + [ + 4.701583684040178, + 52.96172161329086 + ] + ] + ] + }, + "id": "52", + "links": [ + { + "rel": "canonical", + "title": "The actual feature in the corresponding OGC API", + "type": "application/geo+json", + "href": "https://example.com/ogc/v1/collections/buildings/items/52?f=json" + } + ] + }, + { + "type": "Feature", + "properties": { + "collectionGeometryType": "POINT", + "collectionId": "buildings", + "collectionVersion": "1", + "displayName": "Abbewaal - Den Burg 10", + "highlight": "Abbewaal - Den Burg 10", + "href": "https://example.com/ogc/v1/collections/buildings/items/32183?f=json", + "score": 0.10277967154979706 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 4.703248382838079, + 52.96279808509741 + ], + [ + 4.903248382838078, + 52.96279808509741 + ], + [ + 4.903248382838078, + 53.162798085097414 + ], + [ + 4.703248382838079, + 53.162798085097414 + ], + [ + 4.703248382838079, + 52.96279808509741 + ] + ] + ] + }, + "id": "32183", + "links": [ + { + "rel": "canonical", + "title": "The actual feature in the corresponding OGC API", + "type": "application/geo+json", + "href": "https://example.com/ogc/v1/collections/buildings/items/32183?f=json" + } + ] + }, + { + "type": "Feature", + "properties": { + "collectionGeometryType": "POINT", + "collectionId": "buildings", + "collectionVersion": "1", + "displayName": "Abbewaal - Den Burg 10", + "highlight": "Abbewaal - Den Burg 10", + "href": "https://example.com/ogc/v1/collections/buildings/items/53?f=json", + "score": 0.10277967154979706 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 4.703248382838079, + 52.96279808509741 + ], + [ + 4.903248382838078, + 52.96279808509741 + ], + [ + 4.903248382838078, + 53.162798085097414 + ], + [ + 4.703248382838079, + 53.162798085097414 + ], + [ + 4.703248382838079, + 52.96279808509741 + ] + ] + ] + }, + "id": "53", + "links": [ + { + "rel": "canonical", + "title": "The actual feature in the corresponding OGC API", + "type": "application/geo+json", + "href": "https://example.com/ogc/v1/collections/buildings/items/53?f=json" + } + ] + }, + { + "type": "Feature", + "properties": { + "collectionGeometryType": "POINT", + "collectionId": "buildings", + "collectionVersion": "1", + "displayName": "Abbewaal - Den Burg 10", + "highlight": "Abbewaal - Den Burg 10", + "href": "https://example.com/ogc/v1/collections/buildings/items/32184?f=json", "score": 0.10277967154979706 }, "geometry": { @@ -48,13 +240,493 @@ ] ] }, - "id": "99", + "id": "32184", + "links": [ + { + "rel": "canonical", + "title": "The actual feature in the corresponding OGC API", + "type": "application/geo+json", + "href": "https://example.com/ogc/v1/collections/buildings/items/32184?f=json" + } + ] + }, + { + "type": "Feature", + "properties": { + "collectionGeometryType": "POINT", + "collectionId": "buildings", + "collectionVersion": "1", + "displayName": "Abbewaal - Den Burg 10", + "highlight": "Abbewaal - Den Burg 10", + "href": "https://example.com/ogc/v1/collections/buildings/items/54?f=json", + "score": 0.10277967154979706 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 4.703248382838079, + 52.96279808509741 + ], + [ + 4.903248382838078, + 52.96279808509741 + ], + [ + 4.903248382838078, + 53.162798085097414 + ], + [ + 4.703248382838079, + 53.162798085097414 + ], + [ + 4.703248382838079, + 52.96279808509741 + ] + ] + ] + }, + "id": "54", + "links": [ + { + "rel": "canonical", + "title": "The actual feature in the corresponding OGC API", + "type": "application/geo+json", + "href": "https://example.com/ogc/v1/collections/buildings/items/54?f=json" + } + ] + }, + { + "type": "Feature", + "properties": { + "collectionGeometryType": "POINT", + "collectionId": "buildings", + "collectionVersion": "1", + "displayName": "Abbewaal - Den Burg 11", + "highlight": "Abbewaal - Den Burg 11", + "href": "https://example.com/ogc/v1/collections/buildings/items/22549?f=json", + "score": 0.10277967154979706 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 4.701889188504978, + 52.962632527818215 + ], + [ + 4.901889188504978, + 52.962632527818215 + ], + [ + 4.901889188504978, + 53.16263252781822 + ], + [ + 4.701889188504978, + 53.16263252781822 + ], + [ + 4.701889188504978, + 52.962632527818215 + ] + ] + ] + }, + "id": "22549", + "links": [ + { + "rel": "canonical", + "title": "The actual feature in the corresponding OGC API", + "type": "application/geo+json", + "href": "https://example.com/ogc/v1/collections/buildings/items/22549?f=json" + } + ] + }, + { + "type": "Feature", + "properties": { + "collectionGeometryType": "POINT", + "collectionId": "buildings", + "collectionVersion": "1", + "displayName": "Abbewaal - Den Burg 13", + "highlight": "Abbewaal - Den Burg 13", + "href": "https://example.com/ogc/v1/collections/buildings/items/56?f=json", + "score": 0.10277967154979706 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 4.702154512142386, + 52.96275394293894 + ], + [ + 4.902154512142385, + 52.96275394293894 + ], + [ + 4.902154512142385, + 53.16275394293894 + ], + [ + 4.702154512142386, + 53.16275394293894 + ], + [ + 4.702154512142386, + 52.96275394293894 + ] + ] + ] + }, + "id": "56", + "links": [ + { + "rel": "canonical", + "title": "The actual feature in the corresponding OGC API", + "type": "application/geo+json", + "href": "https://example.com/ogc/v1/collections/buildings/items/56?f=json" + } + ] + }, + { + "type": "Feature", + "properties": { + "collectionGeometryType": "POINT", + "collectionId": "buildings", + "collectionVersion": "1", + "displayName": "Abbewaal - Den Burg 13", + "highlight": "Abbewaal - Den Burg 13", + "href": "https://example.com/ogc/v1/collections/buildings/items/55?f=json", + "score": 0.10277967154979706 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 4.702154512142386, + 52.96275394293894 + ], + [ + 4.902154512142385, + 52.96275394293894 + ], + [ + 4.902154512142385, + 53.16275394293894 + ], + [ + 4.702154512142386, + 53.16275394293894 + ], + [ + 4.702154512142386, + 52.96275394293894 + ] + ] + ] + }, + "id": "55", + "links": [ + { + "rel": "canonical", + "title": "The actual feature in the corresponding OGC API", + "type": "application/geo+json", + "href": "https://example.com/ogc/v1/collections/buildings/items/55?f=json" + } + ] + }, + { + "type": "Feature", + "properties": { + "collectionGeometryType": "POINT", + "collectionId": "buildings", + "collectionVersion": "1", + "displayName": "Abbewaal - Den Burg 15", + "highlight": "Abbewaal - Den Burg 15", + "href": "https://example.com/ogc/v1/collections/buildings/items/57?f=json", + "score": 0.10277967154979706 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 4.702695564526969, + 52.962965478349716 + ], + [ + 4.902695564526968, + 52.962965478349716 + ], + [ + 4.902695564526968, + 53.16296547834972 + ], + [ + 4.702695564526969, + 53.16296547834972 + ], + [ + 4.702695564526969, + 52.962965478349716 + ] + ] + ] + }, + "id": "57", + "links": [ + { + "rel": "canonical", + "title": "The actual feature in the corresponding OGC API", + "type": "application/geo+json", + "href": "https://example.com/ogc/v1/collections/buildings/items/57?f=json" + } + ] + }, + { + "type": "Feature", + "properties": { + "collectionGeometryType": "POINT", + "collectionId": "buildings", + "collectionVersion": "1", + "displayName": "Abbewaal - Den Burg 15", + "highlight": "Abbewaal - Den Burg 15", + "href": "https://example.com/ogc/v1/collections/buildings/items/58?f=json", + "score": 0.10277967154979706 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 4.702695564526969, + 52.962965478349716 + ], + [ + 4.902695564526968, + 52.962965478349716 + ], + [ + 4.902695564526968, + 53.16296547834972 + ], + [ + 4.702695564526969, + 53.16296547834972 + ], + [ + 4.702695564526969, + 52.962965478349716 + ] + ] + ] + }, + "id": "58", + "links": [ + { + "rel": "canonical", + "title": "The actual feature in the corresponding OGC API", + "type": "application/geo+json", + "href": "https://example.com/ogc/v1/collections/buildings/items/58?f=json" + } + ] + }, + { + "type": "Feature", + "properties": { + "collectionGeometryType": "POINT", + "collectionId": "buildings", + "collectionVersion": "1", + "displayName": "Abbewaal - Den Burg 17", + "highlight": "Abbewaal - Den Burg 17", + "href": "https://example.com/ogc/v1/collections/buildings/items/16128?f=json", + "score": 0.10277967154979706 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 4.702643767605495, + 52.96301123478128 + ], + [ + 4.902643767605494, + 52.96301123478128 + ], + [ + 4.902643767605494, + 53.16301123478128 + ], + [ + 4.702643767605495, + 53.16301123478128 + ], + [ + 4.702643767605495, + 52.96301123478128 + ] + ] + ] + }, + "id": "16128", + "links": [ + { + "rel": "canonical", + "title": "The actual feature in the corresponding OGC API", + "type": "application/geo+json", + "href": "https://example.com/ogc/v1/collections/buildings/items/16128?f=json" + } + ] + }, + { + "type": "Feature", + "properties": { + "collectionGeometryType": "POINT", + "collectionId": "buildings", + "collectionVersion": "1", + "displayName": "Abbewaal - Den Burg 19", + "highlight": "Abbewaal - Den Burg 19", + "href": "https://example.com/ogc/v1/collections/buildings/items/59?f=json", + "score": 0.10277967154979706 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 4.702569893786273, + 52.963164466064796 + ], + [ + 4.902569893786272, + 52.963164466064796 + ], + [ + 4.902569893786272, + 53.1631644660648 + ], + [ + 4.702569893786273, + 53.1631644660648 + ], + [ + 4.702569893786273, + 52.963164466064796 + ] + ] + ] + }, + "id": "59", + "links": [ + { + "rel": "canonical", + "title": "The actual feature in the corresponding OGC API", + "type": "application/geo+json", + "href": "https://example.com/ogc/v1/collections/buildings/items/59?f=json" + } + ] + }, + { + "type": "Feature", + "properties": { + "collectionGeometryType": "POINT", + "collectionId": "buildings", + "collectionVersion": "1", + "displayName": "Abbewaal - Den Burg 2", + "highlight": "Abbewaal - Den Burg 2", + "href": "https://example.com/ogc/v1/collections/buildings/items/16130?f=json", + "score": 0.10277967154979706 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 4.702320309835308, + 52.96208847172587 + ], + [ + 4.902320309835307, + 52.96208847172587 + ], + [ + 4.902320309835307, + 53.16208847172587 + ], + [ + 4.702320309835308, + 53.16208847172587 + ], + [ + 4.702320309835308, + 52.96208847172587 + ] + ] + ] + }, + "id": "16130", + "links": [ + { + "rel": "canonical", + "title": "The actual feature in the corresponding OGC API", + "type": "application/geo+json", + "href": "https://example.com/ogc/v1/collections/buildings/items/16130?f=json" + } + ] + }, + { + "type": "Feature", + "properties": { + "collectionGeometryType": "POINT", + "collectionId": "buildings", + "collectionVersion": "1", + "displayName": "Abbewaal - Den Burg 2", + "highlight": "Abbewaal - Den Burg 2", + "href": "https://example.com/ogc/v1/collections/buildings/items/16129?f=json", + "score": 0.10277967154979706 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 4.702320309835308, + 52.96208847172587 + ], + [ + 4.902320309835307, + 52.96208847172587 + ], + [ + 4.902320309835307, + 53.16208847172587 + ], + [ + 4.702320309835308, + 53.16208847172587 + ], + [ + 4.702320309835308, + 52.96208847172587 + ] + ] + ] + }, + "id": "16129", "links": [ { "rel": "canonical", "title": "The actual feature in the corresponding OGC API", "type": "application/geo+json", - "href": "https://example.com/ogc/v1/collections/buildings/items/99?f=json" + "href": "https://example.com/ogc/v1/collections/buildings/items/16129?f=json" } ] }, @@ -64,9 +736,9 @@ "collectionGeometryType": "POINT", "collectionId": "buildings", "collectionVersion": "1", - "displayName": "Building Achterom - Den Burg", - "highlight": "Building Achterom - Den Burg", - "href": "https://example.com/ogc/v1/collections/buildings/items/22215?f=json", + "displayName": "Abbewaal - Den Burg 21", + "highlight": "Abbewaal - Den Burg 21", + "href": "https://example.com/ogc/v1/collections/buildings/items/60?f=json", "score": 0.10277967154979706 }, "geometry": { @@ -74,35 +746,35 @@ "coordinates": [ [ [ - 4.699139629150976, - 52.95528084051514 + 4.702700429994286, + 52.963319045764216 ], [ - 4.899139629150976, - 52.95528084051514 + 4.902700429994285, + 52.963319045764216 ], [ - 4.899139629150976, - 53.15528084051514 + 4.902700429994285, + 53.16331904576422 ], [ - 4.699139629150976, - 53.15528084051514 + 4.702700429994286, + 53.16331904576422 ], [ - 4.699139629150976, - 52.95528084051514 + 4.702700429994286, + 52.963319045764216 ] ] ] }, - "id": "22215", + "id": "60", "links": [ { "rel": "canonical", "title": "The actual feature in the corresponding OGC API", "type": "application/geo+json", - "href": "https://example.com/ogc/v1/collections/buildings/items/22215?f=json" + "href": "https://example.com/ogc/v1/collections/buildings/items/60?f=json" } ] }, @@ -112,9 +784,9 @@ "collectionGeometryType": "POINT", "collectionId": "buildings", "collectionVersion": "1", - "displayName": "Building Akenbuurt - Den Burg", - "highlight": "Building Akenbuurt - Den Burg", - "href": "https://example.com/ogc/v1/collections/buildings/items/46?f=json", + "displayName": "Abbewaal - Den Burg 23", + "highlight": "Abbewaal - Den Burg 23", + "href": "https://example.com/ogc/v1/collections/buildings/items/16131?f=json", "score": 0.10277967154979706 }, "geometry": { @@ -122,35 +794,35 @@ "coordinates": [ [ [ - 4.678620944865994, - 52.957118421718 + 4.702907537974755, + 52.963443724518335 ], [ - 4.878620944865993, - 52.957118421718 + 4.902907537974754, + 52.963443724518335 ], [ - 4.878620944865993, - 53.157118421718 + 4.902907537974754, + 53.16344372451834 ], [ - 4.678620944865994, - 53.157118421718 + 4.702907537974755, + 53.16344372451834 ], [ - 4.678620944865994, - 52.957118421718 + 4.702907537974755, + 52.963443724518335 ] ] ] }, - "id": "46", + "id": "16131", "links": [ { "rel": "canonical", "title": "The actual feature in the corresponding OGC API", "type": "application/geo+json", - "href": "https://example.com/ogc/v1/collections/buildings/items/46?f=json" + "href": "https://example.com/ogc/v1/collections/buildings/items/16131?f=json" } ] }, @@ -160,9 +832,9 @@ "collectionGeometryType": "POINT", "collectionId": "buildings", "collectionVersion": "1", - "displayName": "Building Amaliaweg - Den Hoorn", - "highlight": "Building Amaliaweg - Den Hoorn", - "href": "https://example.com/ogc/v1/collections/buildings/items/50?f=json", + "displayName": "Abbewaal - Den Burg 23", + "highlight": "Abbewaal - Den Burg 23", + "href": "https://example.com/ogc/v1/collections/buildings/items/16132?f=json", "score": 0.10277967154979706 }, "geometry": { @@ -170,35 +842,35 @@ "coordinates": [ [ [ - 4.68911630577304, - 52.92449928128154 + 4.702907537974755, + 52.963443724518335 ], [ - 4.889116305773039, - 52.92449928128154 + 4.902907537974754, + 52.963443724518335 ], [ - 4.889116305773039, - 53.124499281281544 + 4.902907537974754, + 53.16344372451834 ], [ - 4.68911630577304, - 53.124499281281544 + 4.702907537974755, + 53.16344372451834 ], [ - 4.68911630577304, - 52.92449928128154 + 4.702907537974755, + 52.963443724518335 ] ] ] }, - "id": "50", + "id": "16132", "links": [ { "rel": "canonical", "title": "The actual feature in the corresponding OGC API", "type": "application/geo+json", - "href": "https://example.com/ogc/v1/collections/buildings/items/50?f=json" + "href": "https://example.com/ogc/v1/collections/buildings/items/16132?f=json" } ] }, @@ -208,9 +880,9 @@ "collectionGeometryType": "POINT", "collectionId": "buildings", "collectionVersion": "1", - "displayName": "Building Bakkenweg - Den Hoorn", - "highlight": "Building Bakkenweg - Den Hoorn", - "href": "https://example.com/ogc/v1/collections/buildings/items/520?f=json", + "displayName": "Abbewaal - Den Burg 25", + "highlight": "Abbewaal - Den Burg 25", + "href": "https://example.com/ogc/v1/collections/buildings/items/63?f=json", "score": 0.10277967154979706 }, "geometry": { @@ -218,35 +890,35 @@ "coordinates": [ [ [ - 4.6548723261037095, - 52.94811743920973 + 4.703028657055511, + 52.963494467409724 ], [ - 4.854872326103709, - 52.94811743920973 + 4.90302865705551, + 52.963494467409724 ], [ - 4.854872326103709, - 53.148117439209734 + 4.90302865705551, + 53.16349446740973 ], [ - 4.6548723261037095, - 53.148117439209734 + 4.703028657055511, + 53.16349446740973 ], [ - 4.6548723261037095, - 52.94811743920973 + 4.703028657055511, + 52.963494467409724 ] ] ] }, - "id": "520", + "id": "63", "links": [ { "rel": "canonical", "title": "The actual feature in the corresponding OGC API", "type": "application/geo+json", - "href": "https://example.com/ogc/v1/collections/buildings/items/520?f=json" + "href": "https://example.com/ogc/v1/collections/buildings/items/63?f=json" } ] }, @@ -256,9 +928,9 @@ "collectionGeometryType": "POINT", "collectionId": "buildings", "collectionVersion": "1", - "displayName": "Building Beatrixlaan - Den Burg", - "highlight": "Building Beatrixlaan - Den Burg", - "href": "https://example.com/ogc/v1/collections/buildings/items/826?f=json", + "displayName": "Abbewaal - Den Burg 25", + "highlight": "Abbewaal - Den Burg 25", + "href": "https://example.com/ogc/v1/collections/buildings/items/62?f=json", "score": 0.10277967154979706 }, "geometry": { @@ -266,38 +938,38 @@ "coordinates": [ [ [ - 4.691060243798572, - 52.956498997973284 + 4.703028657055511, + 52.963494467409724 ], [ - 4.891060243798571, - 52.956498997973284 + 4.90302865705551, + 52.963494467409724 ], [ - 4.891060243798571, - 53.15649899797329 + 4.90302865705551, + 53.16349446740973 ], [ - 4.691060243798572, - 53.15649899797329 + 4.703028657055511, + 53.16349446740973 ], [ - 4.691060243798572, - 52.956498997973284 + 4.703028657055511, + 52.963494467409724 ] ] ] }, - "id": "826", + "id": "62", "links": [ { "rel": "canonical", "title": "The actual feature in the corresponding OGC API", "type": "application/geo+json", - "href": "https://example.com/ogc/v1/collections/buildings/items/826?f=json" + "href": "https://example.com/ogc/v1/collections/buildings/items/62?f=json" } ] } ], - "numberReturned": 6 + "numberReturned": 20 } diff --git a/internal/search/testdata/expected-search-den-single-collection-rd.json b/internal/search/testdata/expected-search-den-single-collection-rd.json index 8dd7211..0461836 100644 --- a/internal/search/testdata/expected-search-den-single-collection-rd.json +++ b/internal/search/testdata/expected-search-den-single-collection-rd.json @@ -16,10 +16,202 @@ "collectionGeometryType": "POINT", "collectionId": "addresses", "collectionVersion": "1", - "displayName": "Abbewaal - Den Burg", - "highlight": "Abbewaal - Den Burg", - "href": "https://example.com/ogc/v1/collections/addresses/items/99?f=json", - "score": 0.11162212491035461 + "displayName": "Abbewaal - Den Burg 1", + "highlight": "Abbewaal - Den Burg 1", + "href": "https://example.com/ogc/v1/collections/addresses/items/51?f=json", + "score": 0.10277967154979706 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 108930.82374757381, + 552963.3442540341 + ], + [ + 122369.28607266696, + 552854.3120668632 + ], + [ + 122519.11914567353, + 575110.8422034585 + ], + [ + 109142.385825345, + 575219.5200152525 + ], + [ + 108930.82374757381, + 552963.3442540341 + ] + ] + ] + }, + "id": "51", + "links": [ + { + "rel": "canonical", + "title": "The actual feature in the corresponding OGC API", + "type": "application/geo+json", + "href": "https://example.com/ogc/v1/collections/addresses/items/51?f=json" + } + ] + }, + { + "type": "Feature", + "properties": { + "collectionGeometryType": "POINT", + "collectionId": "addresses", + "collectionVersion": "1", + "displayName": "Abbewaal - Den Burg 1", + "highlight": "Abbewaal - Den Burg 1", + "href": "https://example.com/ogc/v1/collections/addresses/items/52?f=json", + "score": 0.10277967154979706 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 108930.82374757381, + 552963.3442540341 + ], + [ + 122369.28607266696, + 552854.3120668632 + ], + [ + 122519.11914567353, + 575110.8422034585 + ], + [ + 109142.385825345, + 575219.5200152525 + ], + [ + 108930.82374757381, + 552963.3442540341 + ] + ] + ] + }, + "id": "52", + "links": [ + { + "rel": "canonical", + "title": "The actual feature in the corresponding OGC API", + "type": "application/geo+json", + "href": "https://example.com/ogc/v1/collections/addresses/items/52?f=json" + } + ] + }, + { + "type": "Feature", + "properties": { + "collectionGeometryType": "POINT", + "collectionId": "addresses", + "collectionVersion": "1", + "displayName": "Abbewaal - Den Burg 10", + "highlight": "Abbewaal - Den Burg 10", + "href": "https://example.com/ogc/v1/collections/addresses/items/32183?f=json", + "score": 0.10277967154979706 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 109043.81291995465, + 553082.0701673088 + ], + [ + 122481.94500341009, + 552973.3499022151 + ], + [ + 122631.26540948273, + 575229.8898158654 + ], + [ + 109254.86279694513, + 575338.2567024586 + ], + [ + 109043.81291995465, + 553082.0701673088 + ] + ] + ] + }, + "id": "32183", + "links": [ + { + "rel": "canonical", + "title": "The actual feature in the corresponding OGC API", + "type": "application/geo+json", + "href": "https://example.com/ogc/v1/collections/addresses/items/32183?f=json" + } + ] + }, + { + "type": "Feature", + "properties": { + "collectionGeometryType": "POINT", + "collectionId": "addresses", + "collectionVersion": "1", + "displayName": "Abbewaal - Den Burg 10", + "highlight": "Abbewaal - Den Burg 10", + "href": "https://example.com/ogc/v1/collections/addresses/items/53?f=json", + "score": 0.10277967154979706 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 109043.81291995465, + 553082.0701673088 + ], + [ + 122481.94500341009, + 552973.3499022151 + ], + [ + 122631.26540948273, + 575229.8898158654 + ], + [ + 109254.86279694513, + 575338.2567024586 + ], + [ + 109043.81291995465, + 553082.0701673088 + ] + ] + ] + }, + "id": "53", + "links": [ + { + "rel": "canonical", + "title": "The actual feature in the corresponding OGC API", + "type": "application/geo+json", + "href": "https://example.com/ogc/v1/collections/addresses/items/53?f=json" + } + ] + }, + { + "type": "Feature", + "properties": { + "collectionGeometryType": "POINT", + "collectionId": "addresses", + "collectionVersion": "1", + "displayName": "Abbewaal - Den Burg 10", + "highlight": "Abbewaal - Den Burg 10", + "href": "https://example.com/ogc/v1/collections/addresses/items/32184?f=json", + "score": 0.10277967154979706 }, "geometry": { "type": "Polygon", @@ -48,13 +240,13 @@ ] ] }, - "id": "99", + "id": "32184", "links": [ { "rel": "canonical", "title": "The actual feature in the corresponding OGC API", "type": "application/geo+json", - "href": "https://example.com/ogc/v1/collections/addresses/items/99?f=json" + "href": "https://example.com/ogc/v1/collections/addresses/items/32184?f=json" } ] }, @@ -64,45 +256,45 @@ "collectionGeometryType": "POINT", "collectionId": "addresses", "collectionVersion": "1", - "displayName": "Achterom - Den Burg", - "highlight": "Achterom - Den Burg", - "href": "https://example.com/ogc/v1/collections/addresses/items/22215?f=json", - "score": 0.11162212491035461 + "displayName": "Abbewaal - Den Burg 10", + "highlight": "Abbewaal - Den Burg 10", + "href": "https://example.com/ogc/v1/collections/addresses/items/54?f=json", + "score": 0.10277967154979706 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ - 108759.77100749934, - 552248.1945484902 + 109043.81291995465, + 553082.0701673088 ], [ - 122200.21718796142, - 552138.6957663981 + 122481.94500341009, + 552973.3499022151 ], [ - 122350.79776828067, - 574395.1784952144 + 122631.26540948273, + 575229.8898158654 ], [ - 108972.07779478905, - 574504.3214884505 + 109254.86279694513, + 575338.2567024586 ], [ - 108759.77100749934, - 552248.1945484902 + 109043.81291995465, + 553082.0701673088 ] ] ] }, - "id": "22215", + "id": "54", "links": [ { "rel": "canonical", "title": "The actual feature in the corresponding OGC API", "type": "application/geo+json", - "href": "https://example.com/ogc/v1/collections/addresses/items/22215?f=json" + "href": "https://example.com/ogc/v1/collections/addresses/items/54?f=json" } ] }, @@ -112,45 +304,45 @@ "collectionGeometryType": "POINT", "collectionId": "addresses", "collectionVersion": "1", - "displayName": "Akenbuurt - Den Burg", - "highlight": "Akenbuurt - Den Burg", - "href": "https://example.com/ogc/v1/collections/addresses/items/46?f=json", - "score": 0.11162212491035461 + "displayName": "Abbewaal - Den Burg 11", + "highlight": "Abbewaal - Den Burg 11", + "href": "https://example.com/ogc/v1/collections/addresses/items/22549?f=json", + "score": 0.10277967154979706 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ - 107382.89003167403, - 552466.0160286687 + 108952.31359697209, + 553064.5136385753 ], [ - 120822.74710915676, - 552352.698917785 + 122390.49529567861, + 552955.5399556488 ], [ - 120979.66241412575, - 574609.1632009319 + 122540.2350379685, + 575212.0767479953 ], [ - 107601.53236763355, - 574722.1120828141 + 109163.78273979851, + 575320.6962311822 ], [ - 107382.89003167403, - 552466.0160286687 + 108952.31359697209, + 553064.5136385753 ] ] ] }, - "id": "46", + "id": "22549", "links": [ { "rel": "canonical", "title": "The actual feature in the corresponding OGC API", "type": "application/geo+json", - "href": "https://example.com/ogc/v1/collections/addresses/items/46?f=json" + "href": "https://example.com/ogc/v1/collections/addresses/items/22549?f=json" } ] }, @@ -160,45 +352,45 @@ "collectionGeometryType": "POINT", "collectionId": "addresses", "collectionVersion": "1", - "displayName": "Amaliaweg - Den Hoorn", - "highlight": "Amaliaweg - Den Hoorn", - "href": "https://example.com/ogc/v1/collections/addresses/items/50?f=json", - "score": 0.11162212491035461 + "displayName": "Abbewaal - Den Burg 13", + "highlight": "Abbewaal - Den Burg 13", + "href": "https://example.com/ogc/v1/collections/addresses/items/56?f=json", + "score": 0.10277967154979706 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ - 108053.06245183083, - 548829.392089723 + 108970.26902163994, + 553077.8551786089 ], [ - 121502.9902353966, - 548717.9709095402 + 122408.41355513701, + 552968.9311218729 ], [ - 121656.63098808826, - 570974.2312531795 + 122558.07153432549, + 575225.4691312271 ], [ - 108268.41604413971, - 571085.2908990474 + 109181.65645385033, + 575334.0391476178 ], [ - 108053.06245183083, - 548829.392089723 + 108970.26902163994, + 553077.8551786089 ] ] ] }, - "id": "50", + "id": "56", "links": [ { "rel": "canonical", "title": "The actual feature in the corresponding OGC API", "type": "application/geo+json", - "href": "https://example.com/ogc/v1/collections/addresses/items/50?f=json" + "href": "https://example.com/ogc/v1/collections/addresses/items/56?f=json" } ] }, @@ -208,45 +400,45 @@ "collectionGeometryType": "POINT", "collectionId": "addresses", "collectionVersion": "1", - "displayName": "Bakkenweg - Den Hoorn", - "highlight": "Bakkenweg - Den Hoorn", - "href": "https://example.com/ogc/v1/collections/addresses/items/520?f=json", - "score": 0.11162212491035461 + "displayName": "Abbewaal - Den Burg 13", + "highlight": "Abbewaal - Den Burg 13", + "href": "https://example.com/ogc/v1/collections/addresses/items/55?f=json", + "score": 0.10277967154979706 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ - 105776.85318090196, - 551480.3458531145 + 108970.26902163994, + 553077.8551786089 ], [ - 119219.4593937051, - 551362.588411998 + 122408.41355513701, + 552968.9311218729 ], [ - 119383.69392558266, - 573618.9542783926 + 122558.07153432549, + 575225.4691312271 ], [ - 106002.81086040766, - 573736.3292134333 + 109181.65645385033, + 575334.0391476178 ], [ - 105776.85318090196, - 551480.3458531145 + 108970.26902163994, + 553077.8551786089 ] ] ] }, - "id": "520", + "id": "55", "links": [ { "rel": "canonical", "title": "The actual feature in the corresponding OGC API", "type": "application/geo+json", - "href": "https://example.com/ogc/v1/collections/addresses/items/520?f=json" + "href": "https://example.com/ogc/v1/collections/addresses/items/55?f=json" } ] }, @@ -256,48 +448,48 @@ "collectionGeometryType": "POINT", "collectionId": "addresses", "collectionVersion": "1", - "displayName": "Beatrixlaan - Den Burg", - "highlight": "Beatrixlaan - Den Burg", - "href": "https://example.com/ogc/v1/collections/addresses/items/827?f=json", - "score": 0.11162212491035461 + "displayName": "Abbewaal - Den Burg 15", + "highlight": "Abbewaal - Den Burg 15", + "href": "https://example.com/ogc/v1/collections/addresses/items/57?f=json", + "score": 0.10277967154979706 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ - 108218.13045314618, - 552388.953964499 + 109006.84566435352, + 553101.0494150533 ], [ - 121658.19222109031, - 552277.9525492993 + 122444.9255303106, + 552992.226492696 ], [ - 121811.26766043308, - 574534.4315263145 + 122594.41673838987, + 575248.7667374964 ], [ - 108432.93263887867, - 574645.0722492639 + 109218.0664168045, + 575357.2359449365 ], [ - 108218.13045314618, - 552388.953964499 + 109006.84566435352, + 553101.0494150533 ] ] ] }, - "id": "827", + "id": "57", "links": [ { "rel": "canonical", "title": "The actual feature in the corresponding OGC API", "type": "application/geo+json", - "href": "https://example.com/ogc/v1/collections/addresses/items/827?f=json" + "href": "https://example.com/ogc/v1/collections/addresses/items/57?f=json" } ] } ], - "numberReturned": 6 + "numberReturned": 10 } diff --git a/internal/search/testdata/expected-search-den-single-collection-wgs84.json b/internal/search/testdata/expected-search-den-single-collection-wgs84.json index b73eede..04ae83c 100644 --- a/internal/search/testdata/expected-search-den-single-collection-wgs84.json +++ b/internal/search/testdata/expected-search-den-single-collection-wgs84.json @@ -16,10 +16,202 @@ "collectionGeometryType": "POINT", "collectionId": "addresses", "collectionVersion": "1", - "displayName": "Abbewaal - Den Burg", - "highlight": "Abbewaal - Den Burg", - "href": "https://example.com/ogc/v1/collections/addresses/items/99?f=json", - "score": 0.11162212491035461 + "displayName": "Abbewaal - Den Burg 1", + "highlight": "Abbewaal - Den Burg 1", + "href": "https://example.com/ogc/v1/collections/addresses/items/51?f=json", + "score": 0.10277967154979706 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 4.701583684040178, + 52.96172161329086 + ], + [ + 4.901583684040177, + 52.96172161329086 + ], + [ + 4.901583684040177, + 53.161721613290865 + ], + [ + 4.701583684040178, + 53.161721613290865 + ], + [ + 4.701583684040178, + 52.96172161329086 + ] + ] + ] + }, + "id": "51", + "links": [ + { + "rel": "canonical", + "title": "The actual feature in the corresponding OGC API", + "type": "application/geo+json", + "href": "https://example.com/ogc/v1/collections/addresses/items/51?f=json" + } + ] + }, + { + "type": "Feature", + "properties": { + "collectionGeometryType": "POINT", + "collectionId": "addresses", + "collectionVersion": "1", + "displayName": "Abbewaal - Den Burg 1", + "highlight": "Abbewaal - Den Burg 1", + "href": "https://example.com/ogc/v1/collections/addresses/items/52?f=json", + "score": 0.10277967154979706 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 4.701583684040178, + 52.96172161329086 + ], + [ + 4.901583684040177, + 52.96172161329086 + ], + [ + 4.901583684040177, + 53.161721613290865 + ], + [ + 4.701583684040178, + 53.161721613290865 + ], + [ + 4.701583684040178, + 52.96172161329086 + ] + ] + ] + }, + "id": "52", + "links": [ + { + "rel": "canonical", + "title": "The actual feature in the corresponding OGC API", + "type": "application/geo+json", + "href": "https://example.com/ogc/v1/collections/addresses/items/52?f=json" + } + ] + }, + { + "type": "Feature", + "properties": { + "collectionGeometryType": "POINT", + "collectionId": "addresses", + "collectionVersion": "1", + "displayName": "Abbewaal - Den Burg 10", + "highlight": "Abbewaal - Den Burg 10", + "href": "https://example.com/ogc/v1/collections/addresses/items/32183?f=json", + "score": 0.10277967154979706 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 4.703248382838079, + 52.96279808509741 + ], + [ + 4.903248382838078, + 52.96279808509741 + ], + [ + 4.903248382838078, + 53.162798085097414 + ], + [ + 4.703248382838079, + 53.162798085097414 + ], + [ + 4.703248382838079, + 52.96279808509741 + ] + ] + ] + }, + "id": "32183", + "links": [ + { + "rel": "canonical", + "title": "The actual feature in the corresponding OGC API", + "type": "application/geo+json", + "href": "https://example.com/ogc/v1/collections/addresses/items/32183?f=json" + } + ] + }, + { + "type": "Feature", + "properties": { + "collectionGeometryType": "POINT", + "collectionId": "addresses", + "collectionVersion": "1", + "displayName": "Abbewaal - Den Burg 10", + "highlight": "Abbewaal - Den Burg 10", + "href": "https://example.com/ogc/v1/collections/addresses/items/53?f=json", + "score": 0.10277967154979706 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 4.703248382838079, + 52.96279808509741 + ], + [ + 4.903248382838078, + 52.96279808509741 + ], + [ + 4.903248382838078, + 53.162798085097414 + ], + [ + 4.703248382838079, + 53.162798085097414 + ], + [ + 4.703248382838079, + 52.96279808509741 + ] + ] + ] + }, + "id": "53", + "links": [ + { + "rel": "canonical", + "title": "The actual feature in the corresponding OGC API", + "type": "application/geo+json", + "href": "https://example.com/ogc/v1/collections/addresses/items/53?f=json" + } + ] + }, + { + "type": "Feature", + "properties": { + "collectionGeometryType": "POINT", + "collectionId": "addresses", + "collectionVersion": "1", + "displayName": "Abbewaal - Den Burg 10", + "highlight": "Abbewaal - Den Burg 10", + "href": "https://example.com/ogc/v1/collections/addresses/items/32184?f=json", + "score": 0.10277967154979706 }, "geometry": { "type": "Polygon", @@ -48,13 +240,13 @@ ] ] }, - "id": "99", + "id": "32184", "links": [ { "rel": "canonical", "title": "The actual feature in the corresponding OGC API", "type": "application/geo+json", - "href": "https://example.com/ogc/v1/collections/addresses/items/99?f=json" + "href": "https://example.com/ogc/v1/collections/addresses/items/32184?f=json" } ] }, @@ -64,45 +256,45 @@ "collectionGeometryType": "POINT", "collectionId": "addresses", "collectionVersion": "1", - "displayName": "Achterom - Den Burg", - "highlight": "Achterom - Den Burg", - "href": "https://example.com/ogc/v1/collections/addresses/items/22215?f=json", - "score": 0.11162212491035461 + "displayName": "Abbewaal - Den Burg 10", + "highlight": "Abbewaal - Den Burg 10", + "href": "https://example.com/ogc/v1/collections/addresses/items/54?f=json", + "score": 0.10277967154979706 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ - 4.699139629150976, - 52.95528084051514 + 4.703248382838079, + 52.96279808509741 ], [ - 4.899139629150976, - 52.95528084051514 + 4.903248382838078, + 52.96279808509741 ], [ - 4.899139629150976, - 53.15528084051514 + 4.903248382838078, + 53.162798085097414 ], [ - 4.699139629150976, - 53.15528084051514 + 4.703248382838079, + 53.162798085097414 ], [ - 4.699139629150976, - 52.95528084051514 + 4.703248382838079, + 52.96279808509741 ] ] ] }, - "id": "22215", + "id": "54", "links": [ { "rel": "canonical", "title": "The actual feature in the corresponding OGC API", "type": "application/geo+json", - "href": "https://example.com/ogc/v1/collections/addresses/items/22215?f=json" + "href": "https://example.com/ogc/v1/collections/addresses/items/54?f=json" } ] }, @@ -112,45 +304,45 @@ "collectionGeometryType": "POINT", "collectionId": "addresses", "collectionVersion": "1", - "displayName": "Akenbuurt - Den Burg", - "highlight": "Akenbuurt - Den Burg", - "href": "https://example.com/ogc/v1/collections/addresses/items/46?f=json", - "score": 0.11162212491035461 + "displayName": "Abbewaal - Den Burg 11", + "highlight": "Abbewaal - Den Burg 11", + "href": "https://example.com/ogc/v1/collections/addresses/items/22549?f=json", + "score": 0.10277967154979706 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ - 4.678620944865994, - 52.957118421718 + 4.701889188504978, + 52.962632527818215 ], [ - 4.878620944865993, - 52.957118421718 + 4.901889188504978, + 52.962632527818215 ], [ - 4.878620944865993, - 53.157118421718 + 4.901889188504978, + 53.16263252781822 ], [ - 4.678620944865994, - 53.157118421718 + 4.701889188504978, + 53.16263252781822 ], [ - 4.678620944865994, - 52.957118421718 + 4.701889188504978, + 52.962632527818215 ] ] ] }, - "id": "46", + "id": "22549", "links": [ { "rel": "canonical", "title": "The actual feature in the corresponding OGC API", "type": "application/geo+json", - "href": "https://example.com/ogc/v1/collections/addresses/items/46?f=json" + "href": "https://example.com/ogc/v1/collections/addresses/items/22549?f=json" } ] }, @@ -160,45 +352,45 @@ "collectionGeometryType": "POINT", "collectionId": "addresses", "collectionVersion": "1", - "displayName": "Amaliaweg - Den Hoorn", - "highlight": "Amaliaweg - Den Hoorn", - "href": "https://example.com/ogc/v1/collections/addresses/items/50?f=json", - "score": 0.11162212491035461 + "displayName": "Abbewaal - Den Burg 13", + "highlight": "Abbewaal - Den Burg 13", + "href": "https://example.com/ogc/v1/collections/addresses/items/56?f=json", + "score": 0.10277967154979706 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ - 4.68911630577304, - 52.92449928128154 + 4.702154512142386, + 52.96275394293894 ], [ - 4.889116305773039, - 52.92449928128154 + 4.902154512142385, + 52.96275394293894 ], [ - 4.889116305773039, - 53.124499281281544 + 4.902154512142385, + 53.16275394293894 ], [ - 4.68911630577304, - 53.124499281281544 + 4.702154512142386, + 53.16275394293894 ], [ - 4.68911630577304, - 52.92449928128154 + 4.702154512142386, + 52.96275394293894 ] ] ] }, - "id": "50", + "id": "56", "links": [ { "rel": "canonical", "title": "The actual feature in the corresponding OGC API", "type": "application/geo+json", - "href": "https://example.com/ogc/v1/collections/addresses/items/50?f=json" + "href": "https://example.com/ogc/v1/collections/addresses/items/56?f=json" } ] }, @@ -208,45 +400,45 @@ "collectionGeometryType": "POINT", "collectionId": "addresses", "collectionVersion": "1", - "displayName": "Bakkenweg - Den Hoorn", - "highlight": "Bakkenweg - Den Hoorn", - "href": "https://example.com/ogc/v1/collections/addresses/items/520?f=json", - "score": 0.11162212491035461 + "displayName": "Abbewaal - Den Burg 13", + "highlight": "Abbewaal - Den Burg 13", + "href": "https://example.com/ogc/v1/collections/addresses/items/55?f=json", + "score": 0.10277967154979706 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ - 4.6548723261037095, - 52.94811743920973 + 4.702154512142386, + 52.96275394293894 ], [ - 4.854872326103709, - 52.94811743920973 + 4.902154512142385, + 52.96275394293894 ], [ - 4.854872326103709, - 53.148117439209734 + 4.902154512142385, + 53.16275394293894 ], [ - 4.6548723261037095, - 53.148117439209734 + 4.702154512142386, + 53.16275394293894 ], [ - 4.6548723261037095, - 52.94811743920973 + 4.702154512142386, + 52.96275394293894 ] ] ] }, - "id": "520", + "id": "55", "links": [ { "rel": "canonical", "title": "The actual feature in the corresponding OGC API", "type": "application/geo+json", - "href": "https://example.com/ogc/v1/collections/addresses/items/520?f=json" + "href": "https://example.com/ogc/v1/collections/addresses/items/55?f=json" } ] }, @@ -256,48 +448,48 @@ "collectionGeometryType": "POINT", "collectionId": "addresses", "collectionVersion": "1", - "displayName": "Beatrixlaan - Den Burg", - "highlight": "Beatrixlaan - Den Burg", - "href": "https://example.com/ogc/v1/collections/addresses/items/827?f=json", - "score": 0.11162212491035461 + "displayName": "Abbewaal - Den Burg 15", + "highlight": "Abbewaal - Den Burg 15", + "href": "https://example.com/ogc/v1/collections/addresses/items/57?f=json", + "score": 0.10277967154979706 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ - 4.691060243798572, - 52.956498997973284 + 4.702695564526969, + 52.962965478349716 ], [ - 4.891060243798571, - 52.956498997973284 + 4.902695564526968, + 52.962965478349716 ], [ - 4.891060243798571, - 53.15649899797329 + 4.902695564526968, + 53.16296547834972 ], [ - 4.691060243798572, - 53.15649899797329 + 4.702695564526969, + 53.16296547834972 ], [ - 4.691060243798572, - 52.956498997973284 + 4.702695564526969, + 52.962965478349716 ] ] ] }, - "id": "827", + "id": "57", "links": [ { "rel": "canonical", "title": "The actual feature in the corresponding OGC API", "type": "application/geo+json", - "href": "https://example.com/ogc/v1/collections/addresses/items/827?f=json" + "href": "https://example.com/ogc/v1/collections/addresses/items/57?f=json" } ] } ], - "numberReturned": 6 + "numberReturned": 10 } From d6d2f0145bc8dcc54c3b622bdde398930799c70c Mon Sep 17 00:00:00 2001 From: Richard Kettelerij Date: Fri, 20 Dec 2024 16:03:48 +0100 Subject: [PATCH 34/38] feat: add search endpoint - add OpenAPI spec --- config/collections.go | 9 + internal/engine/openapi.go | 16 +- .../templates/openapi/features-search.go.json | 665 ++++++++++++++++++ 3 files changed, 684 insertions(+), 6 deletions(-) create mode 100644 internal/engine/templates/openapi/features-search.go.json diff --git a/config/collections.go b/config/collections.go index f025d94..80fdbb7 100644 --- a/config/collections.go +++ b/config/collections.go @@ -134,6 +134,15 @@ func (c *Config) AllCollections() GeoSpatialCollections { return c.Collections } +func (g GeoSpatialCollections) SupportsSearch() bool { + for _, collection := range g { + if collection.Search != nil { + return true + } + } + return false +} + // Unique lists all unique GeoSpatialCollections (no duplicate IDs). // Don't use in hot path (creates a map on every invocation). func (g GeoSpatialCollections) Unique() []GeoSpatialCollection { diff --git a/internal/engine/openapi.go b/internal/engine/openapi.go index 6a4d3fb..b67dc94 100644 --- a/internal/engine/openapi.go +++ b/internal/engine/openapi.go @@ -26,12 +26,13 @@ import ( ) const ( - specPath = templatesDir + "openapi/" - preamble = specPath + "preamble.go.json" - problems = specPath + "problems.go.json" - commonCollections = specPath + "common-collections.go.json" - commonSpec = specPath + "common.go.json" - HTMLRegex = `<[/]?([a-zA-Z]+).*?>` + specPath = templatesDir + "openapi/" + preamble = specPath + "preamble.go.json" + problems = specPath + "problems.go.json" + commonCollections = specPath + "common-collections.go.json" + commonSpec = specPath + "common.go.json" + featuresSearchSpec = specPath + "features-search.go.json" + HTMLRegex = `<[/]?([a-zA-Z]+).*?>` ) type OpenAPI struct { @@ -51,6 +52,9 @@ func newOpenAPI(config *gomagpieconfig.Config) *OpenAPI { if config.AllCollections() != nil { defaultOpenAPIFiles = append(defaultOpenAPIFiles, commonCollections) } + if config.Collections.SupportsSearch() { + defaultOpenAPIFiles = append(defaultOpenAPIFiles, featuresSearchSpec) + } // add preamble first openAPIFiles := []string{preamble} diff --git a/internal/engine/templates/openapi/features-search.go.json b/internal/engine/templates/openapi/features-search.go.json new file mode 100644 index 0000000..abac32a --- /dev/null +++ b/internal/engine/templates/openapi/features-search.go.json @@ -0,0 +1,665 @@ +{{- /*gotype: github.com/PDOK/gokoala/internal/engine.TemplateData*/ -}} +{{ $cfg := .Config }} +{ + "openapi": "3.0.0", + "info": { + "title": "", + "description": "", + "version": "1.0.0" + }, + "paths": { + "/search": { + "get": { + "tags" : [ "Features" ], + "summary": "search features in one or more collections across datasets.", + "description": "This endpoint allows one to implement autocomplete functionality for location search. The `q` parameter accepts a partial location name and will return all matching locations up to the specified `limit`. The list of search results are offered as features (in GeoJSON, JSON-FG) but contain only minimal information; like a feature ID, highlighted text and a bounding box. When you want to get the full feature you must follow the included link (`href`) in the search result. This allows one to retrieve all properties of the feature and the full geometry from the corresponding OGC API.", + "operationId": "search", + "parameters": [ + { + "$ref": "#/components/parameters/limit-search" + }, + { + "$ref": "#/components/parameters/crs" + } + ], + "responses": { + "200": { + "description": "The response is a document consisting of features in the collection.\nThe features contain only minimal information but include a link (href) to the actual feature in another OGC API. Follow that link to get the full feature data.", + "headers": { + "Content-Crs": { + "description": "a URI, in angular brackets, identifying the coordinate reference system used in the content / payload", + "schema": { + "type": "string" + }, + "example": "" + } + }, + "content": { + "application/geo+json": { + "schema": { + "$ref": "#/components/schemas/featureCollectionGeoJSON" + }, + "example": { + "type": "FeatureCollection", + "links": [ + { + "href": "http://data.example.com/collections/buildings/items.json", + "rel": "self", + "type": "application/geo+json", + "title": "this document" + }, + { + "href": "http://data.example.com/collections/buildings/items?f=html", + "rel": "alternate", + "type": "text/html", + "title": "this document as HTML" + } + ], + "timeStamp": "2018-04-03T14:52:23Z", +{{/* "numberMatched": 123,*/}} + "numberReturned": 2, + "features": [ + { + "type": "Feature", + "id": "123", + "geometry": { + "type": "Polygon", + "coordinates": [ + "..." + ] + }, + "properties": { + "function": "residential", + "floors": "2", + "lastUpdate": "2015-08-01T12:34:56Z" + } + }, + { + "type": "Feature", + "id": "132", + "geometry": { + "type": "Polygon", + "coordinates": [ + "..." + ] + }, + "properties": { + "function": "public use", + "floors": "10", + "lastUpdate": "2013-12-03T10:15:37Z" + } + } + ] + } + }, + "application/vnd.ogc.fg+json": { + "schema": { + "$ref": "#/components/schemas/featureCollectionJSONFG" + }, + "example": { + "conformsTo": [ + "http://www.opengis.net/spec/json-fg-1/0.2/conf/core" + ], + "type": "FeatureCollection", + "links": [ + { + "href": "http://data.example.com/collections/buildings/items.json", + "rel": "self", + "type": "application/geo+json", + "title": "this document" + }, + { + "href": "http://data.example.com/collections/buildings/items?f=html", + "rel": "alternate", + "type": "text/html", + "title": "this document as HTML" + } + ], + "timeStamp": "2018-04-03T14:52:23Z", + {{/* "numberMatched": 123,*/}} + "numberReturned": 2, + "features": [ + { + "type": "Feature", + "id": "123", + "place": { + "type": "Polygon", + "coordinates": [ + "..." + ] + }, + "geometry": null, + "time": null, + "properties": { + "function": "residential", + "floors": "2", + "lastUpdate": "2015-08-01T12:34:56Z" + } + }, + { + "type": "Feature", + "id": "132", + "place": { + "type": "Polygon", + "coordinates": [ + "..." + ] + }, + "geometry": null, + "time": null, + "properties": { + "function": "public use", + "floors": "10", + "lastUpdate": "2013-12-03T10:15:37Z" + } + } + ] + } + }, + "text/html": { + "schema": { + "type": "string" + } + } + } + }, + {{block "problems" . }}{{end}} + } + } + } + }, + "components": { + "schemas": { + "featureCollectionJSONFG": { + "required": [ + "features", + "type" + ], + "type": "object", + "properties": { + "conformsTo": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "format": "uri" + } + }, + "coordRefSys": { + "type": "string", + "format": "uri" + }, + "features": { + "type": "array", + "items": { + "$ref": "#/components/schemas/featureJSONFG" + } + }, + "links": { + "type": "array", + "items": { + "$ref": "#/components/schemas/link" + } + }, + "timeStamp": { + "$ref": "#/components/schemas/timeStamp" + }, +{{/* "numberMatched": {*/}} +{{/* "$ref": "#/components/schemas/numberMatched"*/}} +{{/* },*/}} + "numberReturned": { + "$ref": "#/components/schemas/numberReturned" + } + } + }, + "featureCollectionGeoJSON": { + "required": [ + "features", + "type" + ], + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "FeatureCollection" + ] + }, + "features": { + "type": "array", + "items": { + "$ref": "#/components/schemas/featureGeoJSON" + } + }, + "links": { + "type": "array", + "items": { + "$ref": "#/components/schemas/link" + } + }, + "timeStamp": { + "$ref": "#/components/schemas/timeStamp" + }, +{{/* "numberMatched": {*/}} +{{/* "$ref": "#/components/schemas/numberMatched"*/}} +{{/* },*/}} + "numberReturned": { + "$ref": "#/components/schemas/numberReturned" + } + } + }, + "featureJSONFG": { + "required": [ + "time", + "place", + "geometry", + "properties", + "type" + ], + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "Feature" + ] + }, + "conformsTo": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "format": "uri" + } + }, + "coordRefSys": { + "type": "string", + "format": "uri" + }, + "time": { + {{/* not implemented yet, since we don't yet support temporal data */}} + "nullable": true + }, + "place": { + "nullable": true, + "allOf": [ + {{/* 3D conformance class not implemented, so just delegate to GeoJSON compatible geometries */}} + { + "$ref": "#/components/schemas/geometryGeoJSON" + } + ] + }, + "geometry": { + "nullable": true, + "allOf": [ + {{/* 3D conformance class not implemented, so just delegate to GeoJSON compatible geometries */}} + { + "$ref": "#/components/schemas/geometryGeoJSON" + } + ] + }, + "properties": { + "type": "object", + "nullable": true + }, + "id": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "links": { + "type": "array", + "items": { + "$ref": "#/components/schemas/link" + } + } + } + }, + "featureGeoJSON": { + "required": [ + "geometry", + "properties", + "type" + ], + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "Feature" + ] + }, + "geometry": { + "$ref": "#/components/schemas/geometryGeoJSON" + }, + "properties": { + "type": "object", + "nullable": true + }, + "id": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "links": { + "type": "array", + "items": { + "$ref": "#/components/schemas/link" + } + } + } + }, + "geometryGeoJSON": { + "oneOf": [ + { + "$ref": "#/components/schemas/pointGeoJSON" + }, + { + "$ref": "#/components/schemas/multipointGeoJSON" + }, + { + "$ref": "#/components/schemas/linestringGeoJSON" + }, + { + "$ref": "#/components/schemas/multilinestringGeoJSON" + }, + { + "$ref": "#/components/schemas/polygonGeoJSON" + }, + { + "$ref": "#/components/schemas/multipolygonGeoJSON" + }, + { + "$ref": "#/components/schemas/geometrycollectionGeoJSON" + } + ] + }, + "geometrycollectionGeoJSON": { + "required": [ + "geometries", + "type" + ], + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "GeometryCollection" + ] + }, + "geometries": { + "type": "array", + "items": { + "$ref": "#/components/schemas/geometryGeoJSON" + } + } + } + }, + "linestringGeoJSON": { + "required": [ + "coordinates", + "type" + ], + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "LineString" + ] + }, + "coordinates": { + "minItems": 2, + "type": "array", + "items": { + "minItems": 2, + "type": "array", + "items": { + "type": "number" + } + } + } + } + }, + "link": { + "required": [ + "href" + ], + "type": "object", + "properties": { + "href": { + "type": "string", + "example": "http://data.example.com/buildings/123" + }, + "rel": { + "type": "string", + "example": "alternate" + }, + "type": { + "type": "string", + "example": "application/geo+json" + }, + "hreflang": { + "type": "string", + "example": "en" + }, + "title": { + "type": "string", + "example": "Trierer Strasse 70, 53115 Bonn" + }, + "length": { + "type": "integer" + } + } + }, + "multilinestringGeoJSON": { + "required": [ + "coordinates", + "type" + ], + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "MultiLineString" + ] + }, + "coordinates": { + "type": "array", + "items": { + "minItems": 2, + "type": "array", + "items": { + "minItems": 2, + "type": "array", + "items": { + "type": "number" + } + } + } + } + } + }, + "multipointGeoJSON": { + "required": [ + "coordinates", + "type" + ], + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "MultiPoint" + ] + }, + "coordinates": { + "type": "array", + "items": { + "minItems": 2, + "type": "array", + "items": { + "type": "number" + } + } + } + } + }, + "multipolygonGeoJSON": { + "required": [ + "coordinates", + "type" + ], + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "MultiPolygon" + ] + }, + "coordinates": { + "type": "array", + "items": { + "type": "array", + "items": { + "minItems": 4, + "type": "array", + "items": { + "minItems": 2, + "type": "array", + "items": { + "type": "number" + } + } + } + } + } + } + }, +{{/* "numberMatched": {*/}} +{{/* "minimum": 0,*/}} +{{/* "type": "integer",*/}} +{{/* "description": "The number of features of the feature type that match the selection\nparameters like `bbox`.",*/}} +{{/* "example": 127*/}} +{{/* },*/}} + "numberReturned": { + "minimum": 0, + "type": "integer", + "description": "The number of features in the feature collection.\n\nA server may omit this information in a response, if the information\nabout the number of features is not known or difficult to compute.\n\nIf the value is provided, the value shall be identical to the number\nof items in the \"features\" array.", + "example": 10 + }, + "pointGeoJSON": { + "required": [ + "coordinates", + "type" + ], + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "Point" + ] + }, + "coordinates": { + "minItems": 2, + "type": "array", + "items": { + "type": "number" + } + } + } + }, + "polygonGeoJSON": { + "required": [ + "coordinates", + "type" + ], + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "Polygon" + ] + }, + "coordinates": { + "type": "array", + "items": { + "minItems": 4, + "type": "array", + "items": { + "minItems": 2, + "type": "array", + "items": { + "type": "number" + } + } + } + } + } + }, + "timeStamp": { + "type": "string", + "description": "This property indicates the time and date when the response was generated.", + "format": "date-time", + "example": "2017-08-17T08:05:32Z" + } + }, + "parameters": { + "crs": { + "name": "crs", + "in": "query", + "description": "The coordinate reference system of the geometries in the response. Default is WGS84 longitude/latitude", + "required": false, + "schema": { + "type": "string", + "format": "uri", + "default": "http://www.opengis.net/def/crs/OGC/1.3/CRS84", + "enum": [ + "http://www.opengis.net/def/crs/OGC/1.3/CRS84" + ] + }, + "style": "form", + "explode": false + }, + "collectionId": { + "name": "collectionId", + "in": "path", + "description": "local identifier of a collection", + "required": true, + "style": "simple", + "explode": false, + "schema": { + "type": "string" + } + }, + "limit-search": { + "name": "limit", + "in": "query", + "description": "The optional limit parameter limits the number of items that are presented in the response document.\n\nOnly items are counted that are on the first level of the collection in the response document.\nNested objects contained within the explicitly requested items shall not be counted.\n\nMinimum = 1. Maximum = 50. Default = 10.", + "required": false, + "style": "form", + "explode": false, + "schema": { + "maximum": 50, + "minimum": 1, + "type": "integer", + "default": 10 + } + } + } + } +} From 71fa65e9392c5d445bf2d0cde70467dbaa61f74b Mon Sep 17 00:00:00 2001 From: Richard Kettelerij Date: Mon, 23 Dec 2024 11:54:56 +0100 Subject: [PATCH 35/38] feat: add search endpoint - add OpenAPI spec, with all params --- config/collections.go | 9 +-- internal/engine/openapi.go | 2 +- .../templates/openapi/features-search.go.json | 65 +++++++++++++++---- internal/search/json.go | 38 ++++++++--- internal/search/main.go | 5 +- 5 files changed, 94 insertions(+), 25 deletions(-) diff --git a/config/collections.go b/config/collections.go index 80fdbb7..396b8e4 100644 --- a/config/collections.go +++ b/config/collections.go @@ -123,7 +123,7 @@ func (c *Config) HasCollections() bool { return c.AllCollections() != nil } -// AllCollections get all collections - with for example features, tiles, 3d tiles - offered through this OGC API. +// AllCollections get all collections - with for example features, tiles, 3d tiles - offered through this OGC API. // Results are returned in alphabetic or literal order. func (c *Config) AllCollections() GeoSpatialCollections { if len(c.CollectionOrder) > 0 { @@ -134,13 +134,14 @@ func (c *Config) AllCollections() GeoSpatialCollections { return c.Collections } -func (g GeoSpatialCollections) SupportsSearch() bool { +func (g GeoSpatialCollections) WithSearch() GeoSpatialCollections { + result := make([]GeoSpatialCollection, 0, len(g)) for _, collection := range g { if collection.Search != nil { - return true + result = append(result, collection) } } - return false + return result } // Unique lists all unique GeoSpatialCollections (no duplicate IDs). diff --git a/internal/engine/openapi.go b/internal/engine/openapi.go index b67dc94..faacf6e 100644 --- a/internal/engine/openapi.go +++ b/internal/engine/openapi.go @@ -52,7 +52,7 @@ func newOpenAPI(config *gomagpieconfig.Config) *OpenAPI { if config.AllCollections() != nil { defaultOpenAPIFiles = append(defaultOpenAPIFiles, commonCollections) } - if config.Collections.SupportsSearch() { + if len(config.Collections.WithSearch()) > 0 { defaultOpenAPIFiles = append(defaultOpenAPIFiles, featuresSearchSpec) } diff --git a/internal/engine/templates/openapi/features-search.go.json b/internal/engine/templates/openapi/features-search.go.json index abac32a..7998c40 100644 --- a/internal/engine/templates/openapi/features-search.go.json +++ b/internal/engine/templates/openapi/features-search.go.json @@ -15,6 +15,16 @@ "description": "This endpoint allows one to implement autocomplete functionality for location search. The `q` parameter accepts a partial location name and will return all matching locations up to the specified `limit`. The list of search results are offered as features (in GeoJSON, JSON-FG) but contain only minimal information; like a feature ID, highlighted text and a bounding box. When you want to get the full feature you must follow the included link (`href`) in the search result. This allows one to retrieve all properties of the feature and the full geometry from the corresponding OGC API.", "operationId": "search", "parameters": [ + { + "$ref": "#/components/parameters/q" + }, + {{- range $index, $coll := .Config.Collections.WithSearch -}} + {{- if $index -}},{{- end -}} + { + "$ref": "#/components/parameters/{{ $coll.ID }}-collection-search" + } + {{- end -}} + , { "$ref": "#/components/parameters/limit-search" }, @@ -619,6 +629,50 @@ } }, "parameters": { + "q": { + "name": "q", + "in": "query", + "description": "The search term(s)", + "required": true, + "style": "form", + "explode": false, + "schema": { + "type": "string", + "minLength": 2, + "maxLength": 200 + } + }, + {{- range $index, $coll := .Config.Collections.WithSearch -}} + {{- if $index -}},{{- end -}} + "{{ $coll.ID }}-collection-search": { + "name": "{{ $coll.ID }}", + "in": "query", + "description": "When provided the {{ $coll.ID }} collection is included in the search. This parameter should be provided as a [deep object](https://swagger.io/docs/specification/v3_0/serialization/#query-parameters) containing the version and relevance of the {{ $coll.ID }} collection, for example `q=foo&{{ $coll.ID }}[version]=1&{{ $coll.ID }}[relevance]=0.5`", + "required": false, + "style": "deepObject", + "explode": true, + "schema": { + "type": "object", + "required": [ + "version" + ], + "properties": { + "version": { + "type": "number", + "description": "The version of the {{ $coll.ID }} collection.", + "example": "1" + }, + "relevance": { + "type": "number", + "format": "float", + "description": "The relevance score of the {{ $coll.ID }} collection.", + "example": 0.50 + } + } + } + } + {{- end -}} + , "crs": { "name": "crs", "in": "query", @@ -635,17 +689,6 @@ "style": "form", "explode": false }, - "collectionId": { - "name": "collectionId", - "in": "path", - "description": "local identifier of a collection", - "required": true, - "style": "simple", - "explode": false, - "schema": { - "type": "string" - } - }, "limit-search": { "name": "limit", "in": "query", diff --git a/internal/search/json.go b/internal/search/json.go index 777d8a7..c6a1d90 100644 --- a/internal/search/json.go +++ b/internal/search/json.go @@ -1,6 +1,7 @@ package search import ( + "bytes" stdjson "encoding/json" "io" "log" @@ -20,20 +21,41 @@ var ( disableJSONPerfOptimization, _ = strconv.ParseBool(os.Getenv("DISABLE_JSON_PERF_OPTIMIZATION")) ) -func featuresAsGeoJSON(w http.ResponseWriter, baseURL url.URL, fc *domain.FeatureCollection) { +type jsonFeatures struct { + engine *engine.Engine + validateResponse bool +} + +func newJSONFeatures(e *engine.Engine) *jsonFeatures { + return &jsonFeatures{ + engine: e, + validateResponse: true, // TODO make configurable + } +} + +func (jf *jsonFeatures) featuresAsGeoJSON(w http.ResponseWriter, r *http.Request, baseURL url.URL, fc *domain.FeatureCollection) { fc.Timestamp = now().Format(time.RFC3339) fc.Links = createFeatureCollectionLinks(baseURL) // TODO add links - // TODO add validation - // if jf.validateResponse { - // jf.serveAndValidateJSON(&fc, engine.MediaTypeGeoJSON, r, w) - // } else { - serveJSON(&fc, engine.MediaTypeGeoJSON, w) - // } + if jf.validateResponse { + jf.serveAndValidateJSON(&fc, engine.MediaTypeGeoJSON, r, w) + } else { + jf.serveJSON(&fc, engine.MediaTypeGeoJSON, w) + } +} + +// serveAndValidateJSON serves JSON after performing OpenAPI response validation. +func (jf *jsonFeatures) serveAndValidateJSON(input any, contentType string, r *http.Request, w http.ResponseWriter) { + json := &bytes.Buffer{} + if err := getEncoder(json).Encode(input); err != nil { + handleJSONEncodingFailure(err, w) + return + } + jf.engine.Serve(w, r, false /* performed earlier */, jf.validateResponse, contentType, json.Bytes()) } // serveJSON serves JSON *WITHOUT* OpenAPI validation by writing directly to the response output stream -func serveJSON(input any, contentType string, w http.ResponseWriter) { +func (jf *jsonFeatures) serveJSON(input any, contentType string, w http.ResponseWriter) { w.Header().Set(engine.HeaderContentType, contentType) if err := getEncoder(w).Encode(input); err != nil { diff --git a/internal/search/main.go b/internal/search/main.go index 387b4f8..5ddd120 100644 --- a/internal/search/main.go +++ b/internal/search/main.go @@ -22,12 +22,15 @@ const ( type Search struct { engine *engine.Engine datasource ds.Datasource + + json *jsonFeatures } func NewSearch(e *engine.Engine, dbConn string, searchIndex string) *Search { s := &Search{ engine: e, datasource: newDatasource(e, dbConn, searchIndex), + json: newJSONFeatures(e), } e.Router.Get("/search", s.Search()) return s @@ -54,7 +57,7 @@ func (s *Search) Search() http.HandlerFunc { format := s.engine.CN.NegotiateFormat(r) switch format { case engine.FormatGeoJSON, engine.FormatJSON: - featuresAsGeoJSON(w, *s.engine.Config.BaseURL.URL, fc) + s.json.featuresAsGeoJSON(w, r, *s.engine.Config.BaseURL.URL, fc) default: engine.RenderProblem(engine.ProblemNotAcceptable, w, fmt.Sprintf("format '%s' is not supported", format)) return From 5724870af85db5384718b7b4d077e0b4b41c106d Mon Sep 17 00:00:00 2001 From: Richard Kettelerij Date: Mon, 23 Dec 2024 11:58:07 +0100 Subject: [PATCH 36/38] feat: add search endpoint - remove e2e test for search, already covered in integration test --- .github/workflows/e2e-test.yml | 40 ---------------------------------- 1 file changed, 40 deletions(-) delete mode 100644 .github/workflows/e2e-test.yml diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml deleted file mode 100644 index a203015..0000000 --- a/.github/workflows/e2e-test.yml +++ /dev/null @@ -1,40 +0,0 @@ ---- -name: e2e-test -on: - pull_request: -jobs: - end-to-end-test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - # Build a local test image for (potential) re-use across end-to-end tests - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - with: - driver: docker - - name: Build test image - uses: docker/build-push-action@v5 - with: - push: false - tags: gomagpie:local - -# TODO build end-to-end test -# - name: Start gomagpie test instance -# run: | -# docker run \ -# -v `pwd`/examples:/examples \ -# --rm --detach -p 8080:8080 \ -# --name gomagpie \ -# gomagpie:local start-service some_index --config-file /examples/config.yaml -# -# # E2E Test -# - name: E2E Test => Cypress -# uses: cypress-io/github-action@v6 -# with: -# working-directory: ./tests -# browser: chrome -# -# - name: Stop gomagpie test instance -# run: | -# docker stop gomagpie From 941c4c38628b45025bbbef7c5cfa842db696f407 Mon Sep 17 00:00:00 2001 From: Richard Kettelerij Date: Mon, 23 Dec 2024 12:10:30 +0100 Subject: [PATCH 37/38] feat: add search endpoint - add OpenAPI request validation --- internal/engine/templates/openapi/features-search.go.json | 4 +++- internal/search/main.go | 4 ++++ internal/search/testdata/expected-search-no-version-3.json | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/internal/engine/templates/openapi/features-search.go.json b/internal/engine/templates/openapi/features-search.go.json index 7998c40..a86e85c 100644 --- a/internal/engine/templates/openapi/features-search.go.json +++ b/internal/engine/templates/openapi/features-search.go.json @@ -683,7 +683,9 @@ "format": "uri", "default": "http://www.opengis.net/def/crs/OGC/1.3/CRS84", "enum": [ - "http://www.opengis.net/def/crs/OGC/1.3/CRS84" + {{/* TODO make configurable */}} + "http://www.opengis.net/def/crs/OGC/1.3/CRS84", + "http://www.opengis.net/def/crs/EPSG/0/28992" ] }, "style": "form", diff --git a/internal/search/main.go b/internal/search/main.go index 5ddd120..a61124a 100644 --- a/internal/search/main.go +++ b/internal/search/main.go @@ -39,6 +39,10 @@ func NewSearch(e *engine.Engine, dbConn string, searchIndex string) *Search { // Search autosuggest locations based on user input func (s *Search) Search() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { + if err := s.engine.OpenAPI.ValidateRequest(r); err != nil { + engine.RenderProblem(engine.ProblemBadRequest, w, err.Error()) + return + } collections, searchTerm, outputSRID, limit, err := parseQueryParams(r.URL.Query()) if err != nil { engine.RenderProblem(engine.ProblemBadRequest, w, err.Error()) diff --git a/internal/search/testdata/expected-search-no-version-3.json b/internal/search/testdata/expected-search-no-version-3.json index fbf837a..2afd740 100644 --- a/internal/search/testdata/expected-search-no-version-3.json +++ b/internal/search/testdata/expected-search-no-version-3.json @@ -1,5 +1,5 @@ { - "detail": "no version specified in request for collection addresses, specify at least one collection and version. For example: 'foo[version]=1' where 'foo' is the collection and '1' the version", + "detail": "request doesn't conform to OpenAPI spec: parameter \"addresses\" in query has an error: property \"version\" is missing", "status": 400, "timeStamp": "2000-01-01T00:00:00Z", "title": "Bad Request" From 449e581562173193a2215b1bfe6ccc3d356c6973 Mon Sep 17 00:00:00 2001 From: Richard Kettelerij Date: Mon, 23 Dec 2024 12:16:34 +0100 Subject: [PATCH 38/38] feat: add search endpoint - formatting --- .../search/datasources/postgres/postgres.go | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/internal/search/datasources/postgres/postgres.go b/internal/search/datasources/postgres/postgres.go index 698bc37..f68b38a 100644 --- a/internal/search/datasources/postgres/postgres.go +++ b/internal/search/datasources/postgres/postgres.go @@ -71,18 +71,18 @@ func makeSearchQuery(index string, srid d.SRID) string { with query as ( select to_tsquery('dutch', $2) query ) - select r.display_name as display_name, - max(r.feature_id) as feature_id, - max(r.collection_id) as collection_id, - max(r.collection_version) as collection_version, - max(r.geometry_type) as geometry_type, - st_transform(max(r.bbox), %[2]d)::geometry as bbox, - max(r.rank) as rank, - max(r.highlighted_text) as highlighted_text + select r.display_name as display_name, + max(r.feature_id) as feature_id, + max(r.collection_id) as collection_id, + max(r.collection_version) as collection_version, + max(r.geometry_type) as geometry_type, + st_transform(max(r.bbox), %[2]d)::geometry as bbox, + max(r.rank) as rank, + max(r.highlighted_text) as highlighted_text from ( select display_name, feature_id, collection_id, collection_version, geometry_type, bbox, - ts_rank_cd(ts, (select query from query), 1) as rank, - ts_headline('dutch', display_name, (select query from query)) as highlighted_text + ts_rank_cd(ts, (select query from query), 1) as rank, + ts_headline('dutch', display_name, (select query from query)) as highlighted_text from %[1]s where ts @@ (select query from query) and (collection_id, collection_version) in ( -- make a virtual table by creating tuples from the provided arrays.