Merge pull request #3485 from magda-io/issue/3480

Issue/3480 Full text search support to registry record APIs & admin UI record management feature improvement

CHANGES.md

@@ -1,5 +1,10 @@
 # CHANGELOG
 
+## v2.3.2
+
+- #3485: New feature for registry records manager: able to search record by keywords & added records list view
+- Display organizational unit ID info in the user general info panel
+
 ## v2.3.1
 
 - Use Magda forked version minio to support auth to GCS bucket using GCS [Workload Identity](https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity)

lerna.json

@@ -2,12 +2,7 @@
   "lerna": "2.5.1",
   "npmClient": "yarn",
   "useWorkspaces": true,
-  "packages": [
-    "deploy/",
-    "magda-*",
-    "scripts/",
-    "packages/*"
-  ],
+  "packages": ["deploy/", "magda-*", "scripts/", "packages/*"],
   "version": "2.3.1",
   "hoist": [
     "tsmonad",
@@ -29,10 +24,7 @@
   ],
   "command": {
     "docker-build-prod": {
-      "ignore": [
-        "magda-authorization-db",
-        "magda-authorization-api"
-      ]
+      "ignore": ["magda-authorization-db", "magda-authorization-api"]
     }
   }
 }

magda-migrator-registry-db/sql/V2_7__record_full_text_search.sql

@@ -0,0 +1,3 @@
+-- create index for record full text search 
+CREATE INDEX idx_data_full_text ON recordaspects 
+USING GIN (jsonb_to_tsvector('english'::regconfig, data, '["string"]'))

magda-registry-api/src/main/resources/application.conf

@@ -83,12 +83,19 @@ db-query {
     # Similar to `default-timeout`. But this setting allow us to use different (often longer) timeout settings for long queries.
     # e.g. Trim operations.
     long-query-timeout = "10m"
+
+    # Full text search configuration. psql's \dF command shows all available configurations
+    # Once the configuration changed, the index on recordaspects table should be recreated.
+    # You can use the following SQL (with `new-config-name` replaced with the new text search config set here):
+    # CREATE INDEX idx_data_full_text ON recordaspects
+    # USING GIN (jsonb_to_tsvector('new-config-name'::regconfig, data, '["string"]'));
+    text-search-config = "english"
 }
 
 authorization {
   # Skip asking authorization decisions from policy engine.
   # `UnconditionalTrueDecision` will be always returned for this case
-  # Useful when running locally - DO NOT TURN ON IN PRODUCTION 
+  # Useful when running locally - DO NOT TURN ON IN PRODUCTION
   skipOpaQuery = false
 }
 

magda-registry-api/src/main/scala/au/csiro/data61/magda/registry/RecordPersistence.scala

@@ -45,7 +45,8 @@ trait RecordPersistence {
       pageToken: Option[String],
       start: Option[Int],
       limit: Option[Int],
-      reversePageTokenOrder: Option[Boolean] = None
+      reversePageTokenOrder: Option[Boolean] = None,
+      fullTextSearchText: Option[String] = None
   )(implicit session: DBSession): RecordsPage[RecordSummary]
 
   def getAllWithAspects(
@@ -60,15 +61,17 @@ trait RecordPersistence {
       aspectQueries: Iterable[AspectQuery] = Nil,
       aspectOrQueries: Iterable[AspectQuery] = Nil,
       orderBy: Option[OrderByDef] = None,
-      reversePageTokenOrder: Option[Boolean] = None
+      reversePageTokenOrder: Option[Boolean] = None,
+      fullTextSearchText: Option[String] = None
   )(implicit session: DBSession): RecordsPage[Record]
 
   def getCount(
       tenantId: TenantId,
       authDecision: AuthDecision,
       aspectIds: Iterable[String],
       aspectQueries: Iterable[AspectQuery] = Nil,
-      aspectOrQueries: Iterable[AspectQuery] = Nil
+      aspectOrQueries: Iterable[AspectQuery] = Nil,
+      fullTextSearchText: Option[String] = None
   )(implicit session: DBSession): Long
 
   def getById(
@@ -331,15 +334,17 @@ class DefaultRecordPersistence(config: Config)
       pageToken: Option[String],
       start: Option[Int],
       limit: Option[Int],
-      reversePageTokenOrder: Option[Boolean] = None
+      reversePageTokenOrder: Option[Boolean] = None,
+      fullTextSearchText: Option[String] = None
   )(implicit session: DBSession): RecordsPage[RecordSummary] = {
     this.getSummaries(
       tenantId,
       authDecision,
       pageToken,
       start,
       limit,
-      reversePageTokenOrder = reversePageTokenOrder
+      reversePageTokenOrder = reversePageTokenOrder,
+      fullTextSearchText = fullTextSearchText
     )
   }
 
@@ -377,7 +382,8 @@ class DefaultRecordPersistence(config: Config)
       aspectQueries: Iterable[AspectQuery] = Nil,
       aspectOrQueries: Iterable[AspectQuery] = Nil,
       orderBy: Option[OrderByDef] = None,
-      reversePageTokenOrder: Option[Boolean] = None
+      reversePageTokenOrder: Option[Boolean] = None,
+      fullTextSearchText: Option[String] = None
   )(implicit session: DBSession): RecordsPage[Record] = {
 
     // --- make sure if orderBy is used, the involved aspectId is, at least, included in optionalAspectIds
@@ -402,7 +408,8 @@ class DefaultRecordPersistence(config: Config)
       selectors,
       orderBy,
       None,
-      reversePageTokenOrder = reversePageTokenOrder
+      reversePageTokenOrder = reversePageTokenOrder,
+      fullTextSearchText = fullTextSearchText
     )
   }
 
@@ -411,15 +418,17 @@ class DefaultRecordPersistence(config: Config)
       authDecision: AuthDecision,
       aspectIds: Iterable[String],
       aspectQueries: Iterable[AspectQuery] = Nil,
-      aspectOrQueries: Iterable[AspectQuery] = Nil
+      aspectOrQueries: Iterable[AspectQuery] = Nil,
+      fullTextSearchText: Option[String] = None
   )(implicit session: DBSession): Long = {
 
     this.getCountInner(
       tenantId,
       authDecision,
       aspectIds,
       aspectQueries,
-      aspectOrQueries
+      aspectOrQueries,
+      fullTextSearchText
     )
   }
 
@@ -1734,22 +1743,21 @@ class DefaultRecordPersistence(config: Config)
     }
   }
 
-  // The current implementation assumes this API is not used by the system tenant.
-  // Is this what we want?
-  // See ticket https://github.com/magda-io/magda/issues/2359
   private def getSummaries(
       tenantId: TenantId,
       authDecision: AuthDecision,
       pageToken: Option[String],
       start: Option[Int],
       rawLimit: Option[Int],
       recordId: Option[String] = None,
-      reversePageTokenOrder: Option[Boolean] = None
+      reversePageTokenOrder: Option[Boolean] = None,
+      fullTextSearchText: Option[String] = None
   )(implicit session: DBSession): RecordsPage[RecordSummary] = {
     val idWhereClause = recordId.map(id => sqls"Records.recordId=$id")
     val authDecisionCondition = authDecision.toSql()
+    val fullTextSearchClause = createFullTextSearchQuery(fullTextSearchText)
 
-    val whereClauseParts = Seq(idWhereClause) :+ authDecisionCondition :+ SQLUtils
+    val whereClauseParts = Seq(idWhereClause) :+ authDecisionCondition :+ fullTextSearchClause :+ SQLUtils
       .tenantIdToWhereClause(tenantId) :+ pageToken
       .map(
         token =>
@@ -1800,6 +1808,53 @@ class DefaultRecordPersistence(config: Config)
     )
   }
 
+  private val textSearchConfigName =
+    (if (config.hasPath("db-query.text-search-config")) {
+       val settingVal = config.getString("db-query.text-search-config").trim
+       if (settingVal.isEmpty) None else Some(settingVal)
+     } else {
+       None
+     }).getOrElse("english")
+
+  private def createFullTextSearchQuery(
+      fullTextSearchText: Option[String],
+      recordIdSqlRef: String = "records.recordid",
+      tenantIdSqlRef: String = "records.tenantid"
+  ): Option[SQLSyntax] =
+    fullTextSearchText.flatMap { queryText =>
+      if (queryText.trim.isEmpty) {
+        None
+      } else {
+        // the text search config is required to be inline for PG to generate correct query plan
+        // the value is from internal config file should be safe
+        val textSearchConfigSql =
+          SQLSyntax.createUnsafely(s"'${textSearchConfigName}'")
+        val aspectLookupCondition =
+          sqls"(recordid, tenantid)=(${SQLSyntax.createUnsafely(recordIdSqlRef)}, ${SQLSyntax
+            .createUnsafely(tenantIdSqlRef)})"
+        val trimedQueryText = queryText.trim
+        val fullTextSearchCondition =
+          sqls"""
+            (
+              jsonb_to_tsvector(${textSearchConfigSql}::regconfig, data, '["string"]') @@ websearch_to_tsquery(${textSearchConfigSql}::regconfig, ${trimedQueryText})
+              OR recordid = ${trimedQueryText}
+              OR aspectid = ${trimedQueryText}
+            )
+              """
+        val whereConditions = SQLUtils.toAndConditionOpt(
+          Some(aspectLookupCondition),
+          Some(fullTextSearchCondition)
+        )
+        Some(
+          SQLSyntax
+            .exists(
+              sqls"SELECT 1 FROM recordaspects"
+                .where(whereConditions)
+            )
+        )
+      }
+    }
+
   private def getRecords(
       tenantId: TenantId,
       authDecision: AuthDecision,
@@ -1812,7 +1867,8 @@ class DefaultRecordPersistence(config: Config)
       recordSelector: Iterable[Option[SQLSyntax]] = Iterable(),
       orderBy: Option[OrderByDef] = None,
       maxLimit: Option[Int] = None,
-      reversePageTokenOrder: Option[Boolean] = None
+      reversePageTokenOrder: Option[Boolean] = None,
+      fullTextSearchText: Option[String] = None
   )(implicit session: DBSession): RecordsPage[Record] = {
 
     if (orderBy.isDefined && pageToken.isDefined) {
@@ -1835,8 +1891,11 @@ class DefaultRecordPersistence(config: Config)
       aspectIds
     )
 
+    val fullTextSearchClauses =
+      createFullTextSearchQuery(fullTextSearchText)
+
     val whereClauseParts: Seq[Option[SQLSyntax]] =
-      theRecordSelector.toSeq :+ aspectWhereClauses :+ authQueryConditions :+ pageToken
+      theRecordSelector.toSeq :+ aspectWhereClauses :+ authQueryConditions :+ fullTextSearchClauses :+ pageToken
         .map(
           token =>
             if (reversePageTokenOrder.getOrElse(false))
@@ -1927,7 +1986,8 @@ class DefaultRecordPersistence(config: Config)
       authDecision: AuthDecision,
       aspectIds: Iterable[String],
       aspectQueries: Iterable[AspectQuery] = Nil,
-      aspectOrQueries: Iterable[AspectQuery] = Nil
+      aspectOrQueries: Iterable[AspectQuery] = Nil,
+      fullTextSearchText: Option[String] = None
   )(implicit session: DBSession): Long = {
 
     val statement = if (aspectIds.size == 1) {
@@ -1950,12 +2010,20 @@ class DefaultRecordPersistence(config: Config)
         recordIdSqlRef = "ras_tbl.recordid",
         tenantIdSqlRef = "ras_tbl.tenantid"
       )
+
+      val fullTextSearchClause = createFullTextSearchQuery(
+        fullTextSearchText,
+        recordIdSqlRef = "ras_tbl.recordid",
+        tenantIdSqlRef = "ras_tbl.tenantid"
+      )
+
       val whereClauses = SQLSyntax.where(
         SQLUtils.toAndConditionOpt(
           aspectIdsWhereClause,
           SQLUtils.tenantIdToWhereClause(tenantId, "ras_tbl.tenantid"),
           authDecision.toSql(config),
-          aspectQueriesClause
+          aspectQueriesClause,
+          fullTextSearchClause
         )
       )
       sql"select count(*) from RecordAspects as ras_tbl ${whereClauses}"
@@ -1966,12 +2034,17 @@ class DefaultRecordPersistence(config: Config)
       val aspectQueriesClause =
         getSqlFromAspectQueries(aspectQueries.toSeq, aspectOrQueries.toSeq)
 
+      val fullTextSearchClause = createFullTextSearchQuery(
+        fullTextSearchText
+      )
+
       val whereClauses = SQLSyntax.where(
         SQLUtils.toAndConditionOpt(
           aspectIdsWhereClause,
           SQLUtils.tenantIdToWhereClause(tenantId),
           authDecision.toSql(),
-          aspectQueriesClause
+          aspectQueriesClause,
+          fullTextSearchClause
         )
       )
       sql"select count(*) from Records ${whereClauses}"