Suche und SearchIndex-Refresh, Tag-Service

doc/db2/01_schema/10_search.sql

@@ -1,14 +1,10 @@
----
---- Suchindex
----
-
 CREATE MATERIALIZED VIEW IF NOT EXISTS mv_searchindex AS (
 
 SELECT
-        ar.id AS pid, -- für referenz
+        ar.nodeid AS nodeid, -- für referenz
         ar.title1 AS ptitle, -- für ergebnisanzeiuge
         concat_ws(';', ar.title2, ar.title3, ar.title4, ar.title5, ar.title6, ar.title7) AS psubtitles, -- für ergebnisanzeige
-        nsl.name as purl, -- für schnelles verlinken
+        '/wiki/' || nsl.name as purl, -- für schnelles verlinken
         ar.content AS content,
         att.attachment_data AS image,
         string_agg(tags.name, ';') AS tags,				  
@@ -18,12 +14,13 @@ SELECT
         setweight(to_tsvector(ar.title4_lang::regconfig, coalesce(unaccent(ar.title4), '')), 'A') ||
         setweight(to_tsvector(ar.title5_lang::regconfig, coalesce(unaccent(ar.title5), '')), 'A') ||
         setweight(to_tsvector(ar.title6_lang::regconfig, coalesce(unaccent(ar.title6), '')), 'A') ||        
-        setweight(to_tsvector(ar.title7_lang::regconfig, coalesce(unaccent(ar.title7), '')), 'A') ||        setweight(to_tsvector('german', coalesce(ar.content, '')), 'B') -- content will be mostly german
---        setweight(to_tsvector(name_lang::regconfig, tags.name), 'C')
+        setweight(to_tsvector(ar.title7_lang::regconfig, coalesce(unaccent(ar.title7), '')), 'A') ||        
+        setweight(to_tsvector('german', coalesce(ar.content, '')), 'B') || -- content will be mostly german
+        setweight(to_tsvector('simple', string_agg(tags.name, ';')), 'C') -- tags should be matched exactly
 	AS document,
         'article' :: node_type AS doctype
     FROM article_revisions ar
-    INNER JOIN articles a on a.versionid = ar.id  
+    INNER JOIN articles a on a.revisionid = ar.id  
     LEFT JOIN node_tags ON node_tags.nodeid = ar.nodeid
     LEFT JOIN tags ON tags.id = node_tags.tagid
     LEFT JOIN attachments att ON att.id = (
@@ -43,15 +40,16 @@ SELECT
 union -- ab hier news
 
 SELECT
-        news_revisions.id AS pid, -- für referenz
+        news_revisions.nodeid AS nodeid, -- für referenz
         news_revisions.title AS ptitle, -- für ergebnisanzeiuge
         '' as psubtitle, -- für spaltenkompatibilität zu articles
         nsl.name as purl, -- für schnelles verlinken
         news_revisions.content AS content,
         att.attachment_data AS image,
         string_agg(tags.name, ';') AS tags,				  
-        setweight(to_tsvector(news_revisions.title_lang::regconfig, unaccent(news_revisions.title)), 'A') ||  setweight(to_tsvector('german', coalesce(news_revisions.content, '')), 'B') -- content will be mostly german
---        setweight(to_tsvector(name_lang::regconfig, tags.name), 'C')
+        setweight(to_tsvector(news_revisions.title_lang::regconfig, unaccent(news_revisions.title)), 'A') ||  
+        setweight(to_tsvector('german', coalesce(news_revisions.content, '')), 'B') || -- content will be mostly german
+        setweight(to_tsvector('simple', string_agg(tags.name, ';')), 'C') -- tags should be matched exactly
 	AS document,
         'news' :: node_type AS doctype
     FROM news_revisions
@@ -75,14 +73,15 @@ SELECT
 union -- ab hier forumthreads   
 
     select 
-    ft.id as pid, -- für referenz
+    ft.nodeid as nodeid, -- für referenz
     ft.title as ptitle, --für ergebnisanzeige
     '' as psubtitle, -- für spaltenkompatibilität zu articles
     nsl.name as purl, -- für schnelles verlinken
     ft.content as content,
     '' as image,
     '' as tags,
-    setweight(to_tsvector(ft.title_lang::regconfig, unaccent(ft.title)), 'A') ||  setweight(to_tsvector('german', coalesce(ft.content, '')), 'B') -- content will be mostly german
+    setweight(to_tsvector(ft.title_lang::regconfig, unaccent(ft.title)), 'A') ||  
+    setweight(to_tsvector('german', coalesce(ft.content, '')), 'B') -- content will be mostly german
     as document,
     'thread' :: node_type as doctype
     from forum_threads ft

src/main/java/de/holarse/api/imports/Drückblick.java

@@ -1,5 +1,6 @@
 package de.holarse.api.imports;
 
+import static de.holarse.config.JmsQueueTypes.QUEUE_DRÜCKBLICK;
 import static de.holarse.config.RoleApiTypes.*;
 import jakarta.transaction.Transactional;
 import jakarta.validation.Valid;
@@ -31,7 +32,7 @@ public class Drückblick {
     @PostMapping(consumes = MediaType.APPLICATION_JSON_VALUE)
     public ResponseEntity<String> upload(@Valid @RequestBody final de.holarse.backend.api.drückblick.DrückblickEntry importItem) throws Exception {
         try {
-            jmsTemplate.convertAndSend("drueckblick", importItem);
+            jmsTemplate.convertAndSend(QUEUE_DRÜCKBLICK, importItem);
         } catch (JmsException je) {
             throw new RuntimeException("error while jms send", je);
         }

src/main/java/de/holarse/backend/db/SearchIndex.java

@@ -0,0 +1,28 @@
+package de.holarse.backend.db;
+
+import jakarta.persistence.Column;
+import jakarta.persistence.Entity;
+import jakarta.persistence.Id;
+import jakarta.persistence.Table;
+
+/**
+ * Als Abbildung über Spring Data Repository SearchRepository
+ * @author comrad
+ */
+@Table(name = "mv_searchindex")
+@Entity
+public class SearchIndex {
+
+    @Id
+    @Column(name = "nodeid")
+    private Integer nodeId;
+
+    public Integer getNodeId() {
+        return nodeId;
+    }
+
+    public void setNodeId(Integer nodeId) {
+        this.nodeId = nodeId;
+    }
+
+}

src/main/java/de/holarse/backend/db/datasets/SearchResultView.java

@@ -0,0 +1,9 @@
+package de.holarse.backend.db.datasets;
+
+public interface SearchResultView {
+
+    Integer getNodeId();
+    String getTitle();
+    String getUrl();
+
+}

src/main/java/de/holarse/backend/db/repositories/SearchRepository.java

@@ -0,0 +1,25 @@
+package de.holarse.backend.db.repositories;
+
+import de.holarse.backend.db.SearchIndex;
+import de.holarse.backend.db.datasets.SearchResultView;
+import java.util.List;
+import org.springframework.data.jpa.repository.JpaRepository;
+import org.springframework.data.jpa.repository.Modifying;
+import org.springframework.data.jpa.repository.Query;
+import org.springframework.data.repository.query.Param;
+import org.springframework.stereotype.Repository;
+import org.springframework.transaction.annotation.Transactional;
+
+@Repository
+public interface SearchRepository extends JpaRepository<SearchIndex, Integer> {
+
+    @Query(value = "select nodeid as nodeid, ptitle as title, purl as url from mv_searchindex " +
+                   "where document @@ websearch_to_tsquery('german', :query) " + 
+                   "order by ts_rank(document, websearch_to_tsquery('german', :query)) desc", nativeQuery = true)
+    List<SearchResultView> search(@Param("query") final String query);
+
+    @Modifying
+    @Query(value = "refresh materialized view mv_searchindex", nativeQuery = true)
+    void refreshIndex();
+
+}

src/main/java/de/holarse/config/JmsConfig.java

@@ -50,7 +50,7 @@ public ConnectionFactory jmsConnectionFactory() {
         connectionFactory.setUserName(amqUsername);
         connectionFactory.setPassword(amqPassword);
 
-        connectionFactory.setTrustedPackages(Arrays.asList("de.holarse.backend.api.drückblick", "de.holarse.backend.types"));
+        connectionFactory.setTrustedPackages(Arrays.asList("de.holarse.backend.api.drückblick", "de.holarse.backend.types", "de.holarse.queues.commands"));
 
         return connectionFactory;        
     }    

src/main/java/de/holarse/queues/commands/SearchRefresh.java

@@ -0,0 +1,13 @@
+package de.holarse.queues.commands;
+
+import java.io.Serializable;
+
+/**
+ * Marker-Command, um das Materialized Refresh anzustossen
+ * @author comrad
+ */
+public class SearchRefresh implements Serializable {
+
+    private static final long serialVersionUID = 1L;
+
+}

src/main/java/de/holarse/queues/consumers/SearchWorker.java

@@ -0,0 +1,25 @@
+package de.holarse.queues.consumers;
+
+import de.holarse.backend.db.repositories.SearchRepository;
+import de.holarse.queues.commands.SearchRefresh;
+import static de.holarse.config.JmsQueueTypes.QUEUE_SEARCH;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.jms.annotation.JmsListener;
+import org.springframework.stereotype.Component;
+
+@Component
+public class SearchWorker {
+
+    private final static transient Logger log = LoggerFactory.getLogger(SearchWorker.class);
+
+    @Autowired
+    private SearchRepository searchRepository;
+
+    @JmsListener(destination = QUEUE_SEARCH)
+    public void importDrückblickEntries(final SearchRefresh cmd) {
+        log.debug("Incoming Command to Refresh Materialized View");
+        searchRepository.refreshIndex();
+    }    
+}

src/main/java/de/holarse/web/controller/SearchController.java

@@ -1,19 +1,27 @@
 package de.holarse.web.controller;
 
+import de.holarse.backend.db.repositories.SearchRepository;
 import de.holarse.web.controller.commands.SearchForm;
 import de.holarse.web.defines.WebDefines;
-import jakarta.servlet.http.HttpServletRequest;
+import jakarta.transaction.Transactional;
 import jakarta.validation.Valid;
 import java.io.UnsupportedEncodingException;
 import java.net.URLDecoder;
 import java.net.URLEncoder;
 import java.nio.charset.StandardCharsets;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.MediaType;
+import org.springframework.http.ResponseEntity;
+import org.springframework.jms.JmsException;
+import org.springframework.jms.core.JmsTemplate;
 import org.springframework.stereotype.Controller;
 import org.springframework.web.bind.annotation.GetMapping;
 import org.springframework.web.bind.annotation.ModelAttribute;
 import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.servlet.ModelAndView;
@@ -25,6 +33,9 @@ public class SearchController {
 
     private final static transient Logger logger = LoggerFactory.getLogger(SearchController.class);
 
+    @Autowired
+    SearchRepository searchRepository;
+
     @ModelAttribute
     public SearchForm setupSearchForm() {
         return new SearchForm();
@@ -43,6 +54,10 @@ public ModelAndView search(@RequestParam("q") final String q, final ModelAndView
 
         mv.addObject("q", URLDecoder.decode(q, StandardCharsets.UTF_8));
 
+        var results = searchRepository.search(String.join(" | ", q.trim().split(" ")));
+        mv.addObject("results", results);
+        mv.addObject("count", results.size());
+
         return mv;
     }
 

src/main/java/de/holarse/web/controller/WikiController.java

@@ -3,31 +3,33 @@
 import de.holarse.backend.db.Article;
 import de.holarse.backend.db.ArticleRevision;
 import de.holarse.backend.db.NodeSlug;
+import de.holarse.backend.db.NodeStatus;
 import de.holarse.backend.db.Tag;
 import de.holarse.backend.db.repositories.ArticleRepository;
 import de.holarse.backend.db.repositories.ArticleRevisionRepository;
 import de.holarse.backend.db.repositories.NodeSlugRepository;
 import de.holarse.backend.db.repositories.TagRepository;
 import de.holarse.backend.view.ArticleView;
 import de.holarse.backend.view.TagView;
+import de.holarse.config.JmsQueueTypes;
+import static de.holarse.config.JmsQueueTypes.QUEUE_SEARCH;
+import de.holarse.queues.commands.SearchRefresh;
 import de.holarse.web.controller.commands.ArticleForm;
 import de.holarse.web.defines.WebDefines;
 import static de.holarse.web.defines.WebDefines.WIKI_ARTICLES_DEFAULT_PAGE_SIZE;
 import de.holarse.web.renderer.Renderer;
 import de.holarse.web.services.SlugService;
+import de.holarse.web.services.TagService;
 import jakarta.persistence.EntityNotFoundException;
 import java.security.Principal;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.data.domain.Pageable;
 import org.springframework.data.web.PageableDefault;
+import org.springframework.jms.core.JmsTemplate;
 import org.springframework.stereotype.Controller;
 import org.springframework.transaction.annotation.Transactional;
 import org.springframework.web.bind.annotation.GetMapping;
@@ -58,9 +60,15 @@ public class WikiController {
     @Autowired
     private SlugService slugService;
 
+    @Autowired
+    private TagService tagService;
+
     @Autowired
     private Renderer renderer;
 
+    @Autowired
+    private JmsTemplate jmsTemplate;
+
     @GetMapping
     public ModelAndView index(@PageableDefault(sort={"title1"}, value=WIKI_ARTICLES_DEFAULT_PAGE_SIZE) final Pageable pageable, final ModelAndView mv) {
         mv.setViewName("layouts/bare");
@@ -162,24 +170,25 @@ public ModelAndView update(@PathVariable("nodeId") final int nodeId, @ModelAttri
         articleRevision.setContent(form.getContent());
         articleRevisionRepository.save(articleRevision);
 
-        final Set<Tag> articleTags = new HashSet<>();
-
-        // Tags auslesen, in Entities umwandeln und ggf. erzeugen
-        final String[] tagNames = form.getTags().split(",");
-        for (final String tagName : tagNames) {
-            final Tag tag = tagRepository.findByName(tagName.trim()).orElseGet(() -> new Tag(tagName.trim()));
-            articleTags.add(tag);
-        }
-        // Alle Tags mit Slugs versehen, bestehende werden nicht angefasst
-        articleTags.stream().forEach(t -> slugService.slugify(t));
+        final Set<Tag> articleTags = tagService.extract(form);
 
-        // TODO Bilder, Anhänge        
+        // TODO Bilder, Anhänge 
+
+        // Status
+        final NodeStatus nodeStatus = article.getNodeStatus();
+        nodeStatus.setPublished(form.isPublished());
 
         // Artikel auf neue Revision setzen
         article.setArticleRevision(articleRevision);
         article.setTags(articleTags);        
+        article.setNodeStatus(nodeStatus);
         articleRepository.saveAndFlush(article);
 
+        // Suche aktualisieren
+        if (nodeStatus.isPublished()) { 
+            jmsTemplate.convertAndSend(QUEUE_SEARCH, new SearchRefresh());
+        }
+
         final NodeSlug nodeSlug = nodeSlugRepository.findMainSlug(nodeId).orElseThrow(EntityNotFoundException::new);
         return new ModelAndView(String.format("redirect:{}", nodeSlug.getName()));
     }