✨ feat(search): Add search functionalities

backend/.example.env

@@ -22,3 +22,7 @@ S3_HOST=https://s3.yourhost.com
 S3_ACCESS_KEY=accesskeyaccesskeyaccesskeyaccesskey
 S3_SECRET_KEY=secretkeysecretkeysecretkeysecretkey
 S3_BUCKET=yourbucket
+
+MEILISEARCH_HOST=https://search.yourhost.com
+MEILISEARCH_KEY=tokentokentokentokentokentokentokentokentoken
+MEILISEARCH_INDEX=map

backend/package.json

@@ -46,6 +46,7 @@
     "jsonwebtoken": "^8.5.1",
     "keycloak-connect": "^17.0.0",
     "log4js": "^6.3.0",
+    "meilisearch": "^0.30.0",
     "minio": "^7.0.32",
     "reflect-metadata": "^0.1.13",
     "rfdc": "^1.3.0",

backend/prisma/schema.prisma

@@ -26,6 +26,7 @@ model Region {
   isEventRegion     Boolean             @default(false)
   isPlotRegion      Boolean             @default(false)
   buildings         Int                 @default(0)
+  osmDisplayName    String              @default("")
 }
 
 model User {

backend/src/Core.ts

@@ -14,6 +14,8 @@ import KeycloakAdmin from "./util/KeycloakAdmin";
 import {PrismaClient} from "@prisma/client";
 import DiscordIntegration from "./util/DiscordIntegration";
 import S3Controller from "./util/S3Controller";
+import SearchController from "./util/SearchController";
+import {MeiliSearch} from "meilisearch";
 
 class Core {
     web: Web;
@@ -24,6 +26,8 @@ class Core {
     discord: DiscordIntegration;
     s3: S3Controller;
 
+    search: SearchController;
+
 
     constructor() {
         this.setUpLogger();
@@ -40,6 +44,7 @@ class Core {
         })
         this.discord = new DiscordIntegration(this);
         this.s3 = new S3Controller(this);
+        this.search = new SearchController(this);
 
     }
 
@@ -54,7 +59,8 @@ class Core {
     public getPrisma = (): PrismaClient => this.prisma;
     public getDiscord = (): DiscordIntegration => this.discord;
     public getWeb = (): Web => this.web;
-    public getS3 = (): S3Controller => this.s3;
+    public getS3 = (): S3Controller => this.s3.getMinioInstance();
+    public getSearch = (): MeiliSearch => this.search.getMeiliInstance();
 
 }
 

backend/src/controllers/AdminController.ts

@@ -9,15 +9,18 @@
 import Core from "../Core";
 import axios from "axios";
 import {response} from "express";
+import {centerOfMass, polygon} from "@turf/turf";
 
 class AdminController {
     private core: Core;
 
     private reCalcProgress: number;
+    private osmDisplayNameProgress: number;
 
     constructor(core: Core) {
         this.core = core;
         this.reCalcProgress = 0;
+        this.osmDisplayNameProgress = 0;
     }
 
     public async getAllUsers(req, res) {
@@ -130,9 +133,101 @@ class AdminController {
         this.reCalcProgress = 0;
     }
 
-    public async getCalculationProgess(req, res) {
+    public async getCalculationProgress(req, res) {
         res.send(this.reCalcProgress.toString());
     }
+
+    public async getOsmDisplayNames(req, res) {
+        if (this.osmDisplayNameProgress > 0) {
+            response.send("Already started.")
+            return;
+        }
+        let regions = await this.core.getPrisma().region.findMany();
+        res.send({status: "ok", count: regions.length})
+
+        for (const [i, region] of regions.entries()) {
+            if (req.query?.skipOld === "true" && region.osmDisplayName !== "") {
+                continue;
+            }
+            let coords = JSON.parse(region.data);
+            coords.push(coords[0]);
+            let poly = polygon([coords]);
+            let centerMass = centerOfMass(poly);
+            let center = centerMass.geometry.coordinates;
+            try {
+                const {data} = await axios.get(`https://nominatim.openstreetmap.org/reverse?lat=${center[0]}&lon=${center[1]}&format=json&accept-language=de`, {headers: {'User-Agent': 'BTEMAP/1.0'}});
+                this.core.getLogger().debug(`Got data for ${region.id}`)
+                if (data?.display_name) {
+                    await this.core.getPrisma().region.update({
+                        where: {
+                            id: region.id
+                        },
+                        data: {
+                            osmDisplayName: data.display_name
+                        }
+                    })
+                }
+            } catch (e) {
+                this.core.getLogger().error(e);
+            }
+
+            this.osmDisplayNameProgress = i;
+
+
+        }
+
+        this.osmDisplayNameProgress = 0;
+
+    }
+
+
+    public async getOsmDisplayNameProgress(req, res) {
+        res.send(this.osmDisplayNameProgress.toString());
+    }
+
+    public async syncWithSearchDB(req, res) {
+
+        // reset index
+        try {
+            let oldIndex = await this.core.getSearch().getIndex(process.env.MEILISEARCH_INDEX);
+            if (oldIndex) {
+                await this.core.getSearch().deleteIndex(process.env.MEILISEARCH_INDEX)
+            }
+        } catch (e) {
+
+        }
+
+
+        await this.core.getSearch().createIndex(process.env.MEILISEARCH_INDEX);
+        await this.core.getSearch().index(process.env.MEILISEARCH_INDEX).updateFilterableAttributes(['_geo']);
+        await this.core.getSearch().index(process.env.MEILISEARCH_INDEX).updateSortableAttributes(['_geo']);
+
+        const regions = await this.core.getPrisma().region.findMany();
+
+        let formattedRegions = regions.map((region) => {
+            let coords = JSON.parse(region.data);
+            coords.push(coords[0]);
+            let poly = polygon([coords]);
+            let centerMass = centerOfMass(poly);
+            let center = centerMass.geometry.coordinates;
+            return {
+                id: region.id,
+                city: region.city,
+                "_geo": {
+                    "lat": center[0],
+                    "lng": center[1]
+                },
+                osmDisplayName: region.osmDisplayName,
+                username: region.username
+            }
+        })
+
+        await this.core.getSearch().index(process.env.MEILISEARCH_INDEX).addDocuments(formattedRegions)
+
+        res.send("ok")
+
+
+    }
 }
 
 

backend/src/util/SearchController.ts

@@ -0,0 +1,32 @@
+/******************************************************************************
+ * SearchController.ts                                                        *
+ *                                                                            *
+ * Copyright (c) 2022-2023 Robin Ferch                                        *
+ * https://robinferch.me                                                      *
+ * This project is released under the MIT license.                            *
+ ******************************************************************************/
+
+import Core from "../Core";
+import {MeiliSearch} from "meilisearch";
+
+class S3Controller {
+
+    private core: Core;
+
+    private readonly meiliInstance: MeiliSearch;
+
+
+    constructor(core: Core) {
+        this.core = core;
+        this.meiliInstance = new MeiliSearch({host: process.env.MEILISEARCH_HOST, apiKey: process.env.MEILISEARCH_KEY});
+        this.core.getLogger().debug("Started Search Controller.")
+    }
+
+
+    public getMeiliInstance(): any {
+        return this.meiliInstance;
+    }
+}
+
+
+export default S3Controller;

backend/src/web/routes/index.ts

@@ -127,7 +127,20 @@ class Routes {
         }, this.keycloak.protect("realm:mapadmin"), checkNewUser(this.web.getCore().getPrisma(), this.web.getCore()), body('userId').isString())
 
         router.addRoute(RequestMethods.GET, "/admin/calculateProgress", async (request, response) => {
-            await adminController.getCalculationProgess(request, response);
+            await adminController.getCalculationProgress(request, response);
+        }, this.keycloak.protect("realm:mapadmin"), checkNewUser(this.web.getCore().getPrisma(), this.web.getCore()), body('userId').isString())
+
+        router.addRoute(RequestMethods.GET, "/admin/getOsmDisplayNames", async (request, response) => {
+            await adminController.getOsmDisplayNames(request, response);
+        }, this.keycloak.protect("realm:mapadmin"), checkNewUser(this.web.getCore().getPrisma(), this.web.getCore()), body('userId').isString())
+
+        router.addRoute(RequestMethods.GET, "/admin/osmDisplayNameProgress", async (request, response) => {
+            await adminController.getOsmDisplayNameProgress(request, response);
+        }, this.keycloak.protect("realm:mapadmin"), checkNewUser(this.web.getCore().getPrisma(), this.web.getCore()), body('userId').isString())
+
+
+        router.addRoute(RequestMethods.GET, "/admin/syncWithSearchDB", async (request, response) => {
+            await adminController.syncWithSearchDB(request, response);
         }, this.keycloak.protect("realm:mapadmin"), checkNewUser(this.web.getCore().getPrisma(), this.web.getCore()), body('userId').isString())
 
 

frontend/.env

@@ -1 +1,4 @@
 VITE_WS_HOST=https://map.bte-germany.de
+VITE_SEARCH_URL=https://search.bte-germany.de
+VITE_SEARCH_KEY=ac0f027dadb439dbf0d4fc7f3e00da433aa872dae2f75417ba01ad0d0e6b0035
+VITE_SEARCH_INDEX=map

frontend/package.json

@@ -29,6 +29,7 @@
     "mantine-datatable": "^1.7.34",
     "mapbox-gl": "^2.12.0",
     "mapbox-gl-style-switcher": "^1.0.11",
+    "meilisearch": "^0.30.0",
     "react": "^18.0.0",
     "react-dom": "^18.0.0",
     "react-icons": "^4.7.1",

frontend/src/components/AdminGeneral.jsx

@@ -7,31 +7,38 @@
  +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/
 
 import React, {useEffect, useState} from 'react';
-import {Button, Checkbox, Progress, Title} from "@mantine/core";
+import {Button, Checkbox, Paper, Progress, Title, Badge, Group, Alert} from "@mantine/core";
 import axios from "axios";
 import {useKeycloak} from "@react-keycloak-fork/web";
+import {AiFillWarning, AiOutlineWarning, IoWarningOutline} from "react-icons/all";
+import {showNotification} from "@mantine/notifications";
 
 const AdminGeneral = props => {
 
-    const [progress, setProgess] = useState(0);
+    const [progress, setProgress] = useState(0);
+    const [osmProgress, setOsmProgress] = useState(0);
     const [allCount, setAllCount] = useState(0);
     const [allBuildingsCount, setAllBuildingsCount] = useState(0);
     const [skipOld, setSkipOld] = useState(false);
+    const [skipOldOsm, setSkipOldOsm] = useState(false);
 
     const {keycloak} = useKeycloak();
 
     useEffect(() => {
         let interval;
         axios.get("/api/v1/stats/general").then(({data: statsData}) => {
             setAllCount(statsData.regionCount)
+            setAllBuildingsCount(statsData.totalBuildings)
             interval = setInterval(async () => {
-                console.log("test123")
                 const {data: progress} = await axios.get(`/api/v1/admin/calculateProgress`,
                     {headers: {authorization: "Bearer " + keycloak.token}})
+                const {data: progressOsm} = await axios.get(`/api/v1/admin/osmDisplayNameProgress`,
+                    {headers: {authorization: "Bearer " + keycloak.token}})
 
                 const {data: stats} = await axios.get(`/api/v1/stats/general`)
                 setAllBuildingsCount(stats.totalBuildings)
-                setProgess(progress / statsData.regionCount * 100)
+                setProgress(progress / statsData.regionCount * 100)
+                setOsmProgress(progressOsm / statsData.regionCount * 100)
             }, 2000)
         })
 
@@ -40,7 +47,7 @@ const AdminGeneral = props => {
     }, []);
 
     const start = () => {
-        setProgess(0.0000000001);
+        setProgress(0.0000000001);
         axios.get(`/api/v1/admin/recalculateBuildings${skipOld ? "?skipOld=true" : ""}`, {headers: {authorization: "Bearer " + keycloak.token}}).then(({data}) => {
             showNotification({
                 title: "Ok",
@@ -49,19 +56,79 @@ const AdminGeneral = props => {
         })
     }
 
+    const startOsm = () => {
+        setOsmProgress(0.0000000001);
+        axios.get(`/api/v1/admin/getOsmDisplayNames${skipOld ? "?skipOld=true" : ""}`, {headers: {authorization: "Bearer " + keycloak.token}}).then(({data}) => {
+            showNotification({
+                title: "Ok",
+                message: `Von ${data.count} Regionen werden die OSM Namen geholt.`,
+                color: "hreen"
+            })
+        })
+    }
+
+    const syncSearch = async () => {
+        showNotification({
+            title: 'Ok',
+            message: 'Synchronisiere Search-DB',
+            color: "green"
+        })
+        await axios.get(`/api/v1/admin/syncWithSearchDB`, {headers: {authorization: "Bearer " + keycloak.token}})
+        showNotification({
+            title: 'Fertig',
+            message: 'Synchronisierung abgeschlossen',
+            color: "green"
+        })
+    }
+
 
     return (
         <div>
-            <Button mt={"xl"} loading={progress > 0} onClick={() => start()}>Anzahl der Buildings berechnen</Button>
-            <Checkbox label={"Nur neue Regionen (Regionen mit Anzahl > 0 werden übersprungen)"} mt={"md"}
-                      value={skipOld} onChange={(event) => setSkipOld(event.currentTarget.checked)}/>
-            {
-                progress > 0 &&
-                <Progress value={progress} label={`${Math.round(progress)}%`} animate mt={"xl"} radius="xl" size="xl"/>
-            }
-            {
-                <Title mt={"xl"}>Aktuell {allBuildingsCount} Gebäude</Title>
-            }
+
+            <Paper withBorder shadow={"md"} radius={"md"} p={"xl"} mt={"md"}>
+                <Group>
+                    <Title>Buildings</Title>
+                    <Badge>Aktuell {allBuildingsCount} Gebäude</Badge>
+                </Group>
+
+                <Button mt={"xl"} loading={progress > 0} onClick={() => start()}>Anzahl der Buildings berechnen</Button>
+                <Checkbox label={"Nur neue Regionen (Regionen mit Anzahl > 0 werden übersprungen)"} mt={"md"}
+                          value={skipOld} onChange={(event) => setSkipOld(event.currentTarget.checked)}/>
+                {
+                    progress > 0 &&
+                    <Progress value={progress} label={`${Math.round(progress)}%`} animate mt={"xl"} radius="xl"
+                              size="xl"/>
+                }
+            </Paper>
+
+            <Paper withBorder shadow={"md"} radius={"md"} p={"xl"} mt={"md"}>
+                <Group>
+                    <Title>Search</Title>
+
+                </Group>
+
+                <Alert color={"red"} icon={<IoWarningOutline size={18}/>} mt={"sm"}>
+                    Der gesamte Index wird gelöscht und danach neu erstellt!
+                </Alert>
+                <Button color={"red"} mt={"md"} onClick={() => syncSearch()}>Daten neu synchronisieren</Button>
+            </Paper>
+
+            <Paper withBorder shadow={"md"} radius={"md"} p={"xl"} mt={"md"}>
+                <Group>
+                    <Title>OSM Display Name</Title>
+                </Group>
+
+                <Button mt={"xl"} loading={osmProgress > 0} onClick={() => startOsm()}>OSM Display Name neu
+                    holen</Button>
+                <Checkbox label={"Nur neue Regionen (Regionen mit Display Name werden übersprungen)"} mt={"md"}
+                          value={skipOldOsm} onChange={(event) => setSkipOldOsm(event.currentTarget.checked)}/>
+                {
+                    osmProgress > 0 &&
+                    <Progress value={osmProgress} label={`${Math.round(osmProgress)}%`} animate mt={"xl"} radius="xl"
+                              size="xl"/>
+                }
+            </Paper>
+
 
         </div>
     );