From 698e93e7e0c39c2bd63d8d5852b8b9df5cad2f76 Mon Sep 17 00:00:00 2001 From: takahashim Date: Sat, 20 Jul 2024 02:02:04 +0900 Subject: [PATCH 1/6] fix: allow to use search with pg_bigm --- app/models/decidim/searchable_resource.rb | 38 ++++++++++ app/packs/stylesheets/decidim/cfj/search.scss | 4 - .../decidim/decidim_application.scss | 1 - config/initializers/pg_search_override.rb | 74 +++++++++++++++++++ config/routes.rb | 1 - ...20240713180919_enable_pg_bigm_extension.rb | 9 +++ 6 files changed, 121 insertions(+), 6 deletions(-) create mode 100644 app/models/decidim/searchable_resource.rb delete mode 100644 app/packs/stylesheets/decidim/cfj/search.scss create mode 100644 config/initializers/pg_search_override.rb create mode 100644 db/migrate/20240713180919_enable_pg_bigm_extension.rb diff --git a/app/models/decidim/searchable_resource.rb b/app/models/decidim/searchable_resource.rb new file mode 100644 index 0000000000..e8e23368e8 --- /dev/null +++ b/app/models/decidim/searchable_resource.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +require "pg_search" + +module Decidim + # A Searchable Resource. + # This is a model to a PgSearch table that indexes all searchable resources. + # This table is used to perform textual searches. + # + # Main attributes are: + # - locale: One entry per locale is required, so each resource will be indexed once per locale. + # - content_a: The most relevant textual content. + # - content_b: The second most relevant textual content. + # - content_c: The third most relevant textual content. + # - content_d: The less relevant textual content. + # - datetime: The timestamp that places this resource in the line of time. Used as second criteria (first is text relevance) for sorting. + # + class SearchableResource < ApplicationRecord + include PgSearch::Model + + belongs_to :organization, + foreign_key: "decidim_organization_id", + class_name: "Decidim::Organization" + belongs_to :scope, + foreign_key: "decidim_scope_id", + class_name: "Decidim::Scope", + optional: true + belongs_to :resource, polymorphic: true + belongs_to :decidim_participatory_space, polymorphic: true, optional: true + + validates :locale, uniqueness: { scope: [:decidim_organization_id, :resource_type, :resource_id] } + + pg_search_scope :global_search, + against: { content_a: "A", content_b: "B", content_c: "C", content_d: "D" }, + ranked_by: ':bigram', + using: :bigram + end +end diff --git a/app/packs/stylesheets/decidim/cfj/search.scss b/app/packs/stylesheets/decidim/cfj/search.scss deleted file mode 100644 index 8d853590a8..0000000000 --- a/app/packs/stylesheets/decidim/cfj/search.scss +++ /dev/null @@ -1,4 +0,0 @@ -/* fix decidim-core/app/assets/stylesheets/decidim/modules/_navbar.scss */ -.topbar__search { - display: none; -} diff --git a/app/packs/stylesheets/decidim/decidim_application.scss b/app/packs/stylesheets/decidim/decidim_application.scss index 9b6e96153e..b9cbaeedcd 100644 --- a/app/packs/stylesheets/decidim/decidim_application.scss +++ b/app/packs/stylesheets/decidim/decidim_application.scss @@ -9,6 +9,5 @@ @import "./cfj/buttons"; @import "./cfj/comment_content"; @import "./cfj/forms"; -@import "./cfj/search"; @import "./cfj/media_print"; @import "./cfj/ql_html_editor"; diff --git a/config/initializers/pg_search_override.rb b/config/initializers/pg_search_override.rb new file mode 100644 index 0000000000..7788911c45 --- /dev/null +++ b/config/initializers/pg_search_override.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true + +require "pg_search" + +Rails.application.config.to_prepare do + ## define `PgSearch::Features::Bigram` to use pg_bigm extension + module PgSearch + module Features + class Bigram < Feature + def conditions + if options[:threshold] + Arel::Nodes::Grouping.new( + similarity.gteq(options[:threshold]) + ) + else + Arel::Nodes::Grouping.new( + Arel::Nodes::InfixOperation.new( + infix_operator, + normalized_document, + normalized_query_for_like + ) + ) + end + end + + def rank + Arel::Nodes::Grouping.new(similarity) + end + + private + + def similarity_function + "bigm_similarity" + end + + def infix_operator + "like" + end + + def similarity + Arel::Nodes::NamedFunction.new( + similarity_function, + [ + normalized_query, + normalized_document + ] + ) + end + + def normalized_document + Arel::Nodes::Grouping.new(Arel.sql(normalize(document))) + end + + def normalized_query_for_like + sanitized_query = connection.quote(query) + Arel.sql("likequery(#{normalize(sanitized_query)})") + end + + def normalized_query + sanitized_query = connection.quote(query) + Arel.sql(normalize(sanitized_query)) + end + end + end + end + + ## override `PgSearch::ScopeOptions::FEATURE_CLASSES` + PgSearch::ScopeOptions::FEATURE_CLASSES = { + dmetaphone: PgSearch::Features::DMetaphone, + tsearch: PgSearch::Features::TSearch, + trigram: PgSearch::Features::Trigram, + bigram: PgSearch::Features::Bigram + }.freeze +end diff --git a/config/routes.rb b/config/routes.rb index defea30985..6880a1689d 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -4,7 +4,6 @@ Rails.application.routes.draw do # Redirect to Metadecidim Japan get "/", to: redirect("https://meta.diycities.jp/"), constraints: { host: "www.diycities.jp" } - get "/search", to: redirect("/") mount Decidim::Core::Engine => "/" # For details on the DSL available within this file, see http://guides.rubyonrails.org/routing.html diff --git a/db/migrate/20240713180919_enable_pg_bigm_extension.rb b/db/migrate/20240713180919_enable_pg_bigm_extension.rb new file mode 100644 index 0000000000..aa4b06bca8 --- /dev/null +++ b/db/migrate/20240713180919_enable_pg_bigm_extension.rb @@ -0,0 +1,9 @@ +class EnablePgBigmExtension < ActiveRecord::Migration[6.1] + def up + enable_extension 'pg_bigm' + end + + def down + disable_extension 'pg_bigm' + end +end From 46fa913a3b6c20c11035e435f845a2ac1ec0d7de Mon Sep 17 00:00:00 2001 From: takahashim Date: Sat, 20 Jul 2024 02:02:11 +0900 Subject: [PATCH 2/6] rubocop --- app/models/decidim/searchable_resource.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/models/decidim/searchable_resource.rb b/app/models/decidim/searchable_resource.rb index e8e23368e8..cd71479406 100644 --- a/app/models/decidim/searchable_resource.rb +++ b/app/models/decidim/searchable_resource.rb @@ -28,11 +28,11 @@ class SearchableResource < ApplicationRecord belongs_to :resource, polymorphic: true belongs_to :decidim_participatory_space, polymorphic: true, optional: true - validates :locale, uniqueness: { scope: [:decidim_organization_id, :resource_type, :resource_id] } + validates :locale, uniqueness: { scope: [:decidim_organization_id, :resource_type, :resource_id] } # rubocop:disable Rails/UniqueValidationWithoutIndex pg_search_scope :global_search, against: { content_a: "A", content_b: "B", content_c: "C", content_d: "D" }, - ranked_by: ':bigram', + ranked_by: ":bigram", using: :bigram end end From 2da3e19c3c33b9ce0ed0c1a95771e7e14fab5c83 Mon Sep 17 00:00:00 2001 From: takahashim Date: Mon, 22 Jul 2024 20:03:03 +0900 Subject: [PATCH 3/6] update UPGRADE.md --- docs/UPGRADE.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/UPGRADE.md b/docs/UPGRADE.md index 2e519f176d..39ed356322 100644 --- a/docs/UPGRADE.md +++ b/docs/UPGRADE.md @@ -71,6 +71,10 @@ Decidim本体のバージョンを更新する際、特に注意が必要な内 https://github.com/codeforjapan/decidim-cfj/pull/415 で追加されたファイル。ディベートでconclusionsに空文字列を許すための修正。 +* `app/models/decidim/searchable_resource.rb` + + https://github.com/codeforjapan/decidim-cfj/pull/615 で追加したファイル。pg_searchのfeatureとしてbigram(`pg_bigm`)に対応させるためのもの。 + * `app/uploaders/decidim/cw/application_uploader.rb` https://github.com/decidim/decidim/issues/6720 や https://github.com/codeforjapan/decidim-cfj/issues/101 などの対応のために導入。 From 20ab9692b15b0ae688c2445f00626f7a23ca8b75 Mon Sep 17 00:00:00 2001 From: takahashim Date: Mon, 22 Jul 2024 20:14:34 +0900 Subject: [PATCH 4/6] WIP: use temporary image --- .github/workflows/_check.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/_check.yaml b/.github/workflows/_check.yaml index af7184282d..b92767ec93 100644 --- a/.github/workflows/_check.yaml +++ b/.github/workflows/_check.yaml @@ -43,7 +43,7 @@ jobs: SLACK_MESSAGE_CHANNEL: '#test' services: db: - image: postgres:12.14 + image: ghcr.io/takahashim/postgresql_bigm:12-latest ports: - 5432:5432 env: From 4344fa8c8d5c2f87b28e6cabf2fddd8848c955e8 Mon Sep 17 00:00:00 2001 From: takahashim Date: Tue, 23 Jul 2024 23:50:56 +0900 Subject: [PATCH 5/6] fix to use an image of codeforjapan --- .github/workflows/_check.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/_check.yaml b/.github/workflows/_check.yaml index b92767ec93..ac85da5468 100644 --- a/.github/workflows/_check.yaml +++ b/.github/workflows/_check.yaml @@ -43,7 +43,7 @@ jobs: SLACK_MESSAGE_CHANNEL: '#test' services: db: - image: ghcr.io/takahashim/postgresql_bigm:12-latest + image: ghcr.io/codeforjapan/postgresql_bigm:12-latest ports: - 5432:5432 env: From 691508e1bfcf23e3e52f9c89a69df16fa9e15159 Mon Sep 17 00:00:00 2001 From: takahashim Date: Wed, 24 Jul 2024 00:01:50 +0900 Subject: [PATCH 6/6] fix permissions to allow to read packages --- .github/workflows/_check.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/_check.yaml b/.github/workflows/_check.yaml index ac85da5468..33d787334e 100644 --- a/.github/workflows/_check.yaml +++ b/.github/workflows/_check.yaml @@ -41,6 +41,8 @@ jobs: IMAGEMAGICK_SRC: 7.1.0-50.tar.gz SLACK_API_TOKEN: xoxb-dummy SLACK_MESSAGE_CHANNEL: '#test' + permissions: + packages: read services: db: image: ghcr.io/codeforjapan/postgresql_bigm:12-latest