From 049223271f3eb20e6b78bf5f02109a91deacc6b3 Mon Sep 17 00:00:00 2001 From: Florent Gravin Date: Wed, 18 Oct 2023 13:37:52 +0200 Subject: [PATCH 1/5] conf(lang): add the 'current' option for metadata_language if set to current, the search is requested with the current language of the application (UI language) --- conf/default.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/conf/default.toml b/conf/default.toml index 661eacf206..9cdacd9a3f 100644 --- a/conf/default.toml +++ b/conf/default.toml @@ -12,8 +12,9 @@ geonetwork4_api_url = "/geonetwork/srv/api" proxy_path = "" # This optional parameter defines, in which language metadata should be queried in elasticsearch. # Use ISO 639-2/B (https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes) format to indicate the language of the metadata. +# Setting to "current" will use the current language of the Datahub, the UI langage. # If not indicated, a wildcard is used and no language preference is applied for the search. -# metadata_language = "fre" +# metadata_language = "fre" or "current" # This optional URL should point to the login page that allows authentication to the datahub. # If not indicated, the default geonetwork login page is used. # The following three placeholders can be part of this URL: From 6f73c9d9061606c1d20c1a32326cd339a741d99e Mon Sep 17 00:00:00 2001 From: Florent Gravin Date: Wed, 18 Oct 2023 13:39:20 +0200 Subject: [PATCH 2/5] feat(search): handle 'current' lang option for ES request if current is set and UI lang is 'eng', then the requested fields are .langeng ones --- .../elasticsearch.service.spec.ts | 50 ++++++++++++++++++- .../elasticsearch/elasticsearch.service.ts | 34 ++++++------- 2 files changed, 64 insertions(+), 20 deletions(-) diff --git a/libs/api/repository/src/lib/gn4/elasticsearch/elasticsearch.service.spec.ts b/libs/api/repository/src/lib/gn4/elasticsearch/elasticsearch.service.spec.ts index 4dbe744ef7..9bea27fac6 100644 --- a/libs/api/repository/src/lib/gn4/elasticsearch/elasticsearch.service.spec.ts +++ b/libs/api/repository/src/lib/gn4/elasticsearch/elasticsearch.service.spec.ts @@ -1,13 +1,18 @@ import { ElasticsearchService } from './elasticsearch.service' import { ES_FIXTURE_AGGS_RESPONSE } from '@geonetwork-ui/common/fixtures' -import { EsSearchParams } from '../types/elasticsearch.model' +import { LangService } from '@geonetwork-ui/util/i18n' +import { EsSearchParams } from '@geonetwork-ui/api/metadata-converter' + +const langServiceMock = { + iso3: 'eng', +} as LangService describe('ElasticsearchService', () => { let service: ElasticsearchService let searchFilters beforeEach(() => { - service = new ElasticsearchService('fre') + service = new ElasticsearchService(langServiceMock, 'fre') }) it('should be created', () => { @@ -356,6 +361,47 @@ describe('ElasticsearchService', () => { }) }) + describe('#injectLangInQueryStringFields - Search language', () => { + const queryStringFields = ['resourceTitleObject.${searchLang}'] + describe('When no lang from config', () => { + beforeEach(() => { + service['metadataLang'] = undefined + }) + it('use * wildcard', () => { + expect( + service['injectLangInQueryStringFields'](queryStringFields)[0].split( + '.' + )[1] + ).toEqual('*') + }) + }) + describe('When one lang in config', () => { + beforeEach(() => { + service['metadataLang'] = 'fre' + }) + it('search in the config language', () => { + expect( + service['injectLangInQueryStringFields'](queryStringFields)[0].split( + '.' + )[1] + ).toEqual('langfre') + }) + }) + describe('When "current" language from config"', () => { + beforeEach(() => { + service['metadataLang'] = 'current' + service['lang3'] = 'eng' + }) + it('search in the UI language', () => { + expect( + service['injectLangInQueryStringFields'](queryStringFields)[0].split( + '.' + )[1] + ).toEqual('langeng') + }) + }) + }) + describe('#buildAutocompletePayload', () => { describe('given an autocomplete config', () => { it('returns the search payload', () => { diff --git a/libs/api/repository/src/lib/gn4/elasticsearch/elasticsearch.service.ts b/libs/api/repository/src/lib/gn4/elasticsearch/elasticsearch.service.ts index 8a9b92af1d..14a35832d2 100644 --- a/libs/api/repository/src/lib/gn4/elasticsearch/elasticsearch.service.ts +++ b/libs/api/repository/src/lib/gn4/elasticsearch/elasticsearch.service.ts @@ -19,6 +19,7 @@ import { SortParams, TermsAggregationResult, } from '@geonetwork-ui/api/metadata-converter' +import { LangService } from '@geonetwork-ui/util/i18n' @Injectable({ providedIn: 'root', @@ -27,8 +28,10 @@ export class ElasticsearchService { // runtime fields are computed using a Painless script // see: https://www.elastic.co/guide/en/elasticsearch/reference/current/runtime-mapping-fields.html private runtimeFields: Record = {} + private lang3 = this.langService.iso3 constructor( + private langService: LangService, @Optional() @Inject(METADATA_LANGUAGE) private metadataLang: string ) {} @@ -167,11 +170,12 @@ export class ElasticsearchService { return fields.map((field) => ({ [field[1]]: field[0] })) } - private injectLangInQueryStringFields( - queryStringFields: string[], - lang: string - ) { - const queryLang = lang ? `lang${lang}` : `*` + private injectLangInQueryStringFields(queryStringFields: string[]) { + const queryLang = this.metadataLang + ? this.metadataLang === 'current' + ? `lang${this.lang3}` + : `lang${this.metadataLang}` + : `*` return queryStringFields.map((field) => { return field.replace(/\$\{searchLang\}/g, queryLang) }) @@ -203,10 +207,7 @@ export class ElasticsearchService { query_string: { query: this.escapeSpecialCharacters(any), default_operator: 'AND', - fields: this.injectLangInQueryStringFields( - ES_QUERY_STRING_FIELDS, - this.metadataLang - ), + fields: this.injectLangInQueryStringFields(ES_QUERY_STRING_FIELDS), }, }) } @@ -290,15 +291,12 @@ export class ElasticsearchService { multi_match: { query, type: 'bool_prefix', - fields: this.injectLangInQueryStringFields( - [ - 'resourceTitleObject.${searchLang}', - 'resourceAbstractObject.${searchLang}', - 'tag', - 'resourceIdentifier', - ], - this.metadataLang - ), + fields: this.injectLangInQueryStringFields([ + 'resourceTitleObject.${searchLang}', + 'resourceAbstractObject.${searchLang}', + 'tag', + 'resourceIdentifier', + ]), }, }, ], From 474116c62f4d6be4c622d5a80e0c00d79a40bff5 Mon Sep 17 00:00:00 2001 From: Florent Gravin Date: Thu, 19 Oct 2023 16:33:22 +0200 Subject: [PATCH 3/5] feat(search): add * fallback on searched fields if lang=current --- conf/default.toml | 4 +-- .../elasticsearch.service.spec.ts | 27 ++++++++++++++++++- .../elasticsearch/elasticsearch.service.ts | 26 ++++++++++++++---- 3 files changed, 49 insertions(+), 8 deletions(-) diff --git a/conf/default.toml b/conf/default.toml index 9cdacd9a3f..fee3f97505 100644 --- a/conf/default.toml +++ b/conf/default.toml @@ -12,9 +12,9 @@ geonetwork4_api_url = "/geonetwork/srv/api" proxy_path = "" # This optional parameter defines, in which language metadata should be queried in elasticsearch. # Use ISO 639-2/B (https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes) format to indicate the language of the metadata. -# Setting to "current" will use the current language of the Datahub, the UI langage. +# Setting to "current" will use the current language of the User Interface. # If not indicated, a wildcard is used and no language preference is applied for the search. -# metadata_language = "fre" or "current" +# metadata_language = "current" # This optional URL should point to the login page that allows authentication to the datahub. # If not indicated, the default geonetwork login page is used. # The following three placeholders can be part of this URL: diff --git a/libs/api/repository/src/lib/gn4/elasticsearch/elasticsearch.service.spec.ts b/libs/api/repository/src/lib/gn4/elasticsearch/elasticsearch.service.spec.ts index 9bea27fac6..1e9165077e 100644 --- a/libs/api/repository/src/lib/gn4/elasticsearch/elasticsearch.service.spec.ts +++ b/libs/api/repository/src/lib/gn4/elasticsearch/elasticsearch.service.spec.ts @@ -362,7 +362,7 @@ describe('ElasticsearchService', () => { }) describe('#injectLangInQueryStringFields - Search language', () => { - const queryStringFields = ['resourceTitleObject.${searchLang}'] + let queryStringFields = ['resourceTitleObject.${searchLang}'] describe('When no lang from config', () => { beforeEach(() => { service['metadataLang'] = undefined @@ -399,6 +399,31 @@ describe('ElasticsearchService', () => { )[1] ).toEqual('langeng') }) + it('add * fallback with low priority', () => { + queryStringFields = [ + 'resourceTitleObject.${searchLang}^5', + 'tag.${searchLang}^4', + 'resourceAbstractObject.${searchLang}^3', + 'lineageObject.${searchLang}^2', + 'any.${searchLang}', + 'uuid', + ] + expect( + service['injectLangInQueryStringFields'](queryStringFields) + ).toEqual([ + 'resourceTitleObject.langeng^5', + 'tag.langeng^4', + 'resourceAbstractObject.langeng^3', + 'lineageObject.langeng^2', + 'any.langeng', + 'uuid', + 'resourceTitleObject.*', + 'tag.*', + 'resourceAbstractObject.*', + 'lineageObject.*', + 'any.*', + ]) + }) }) }) diff --git a/libs/api/repository/src/lib/gn4/elasticsearch/elasticsearch.service.ts b/libs/api/repository/src/lib/gn4/elasticsearch/elasticsearch.service.ts index 14a35832d2..a4a82a76ec 100644 --- a/libs/api/repository/src/lib/gn4/elasticsearch/elasticsearch.service.ts +++ b/libs/api/repository/src/lib/gn4/elasticsearch/elasticsearch.service.ts @@ -171,14 +171,30 @@ export class ElasticsearchService { } private injectLangInQueryStringFields(queryStringFields: string[]) { - const queryLang = this.metadataLang - ? this.metadataLang === 'current' + const queryLang = this.getQueryLang() + return [ + ...queryStringFields.map((field) => { + return field.replace(/\$\{searchLang\}/g, queryLang) + }), + ...(this.isCurrentSearchLang() + ? queryStringFields + .filter((field) => /\$\{searchLang\}/.test(field)) + .map((field) => { + return field.replace(/\.\$\{searchLang\}(\^\d+)?/, '.*') + }) + : []), + ] + } + + private getQueryLang(): string { + return this.metadataLang + ? this.isCurrentSearchLang() ? `lang${this.lang3}` : `lang${this.metadataLang}` : `*` - return queryStringFields.map((field) => { - return field.replace(/\$\{searchLang\}/g, queryLang) - }) + } + private isCurrentSearchLang() { + return this.metadataLang === 'current' } private buildPayloadQuery( From e70070dc71cea5e047ae8928a79c4f59fc34cb49 Mon Sep 17 00:00:00 2001 From: Florent Gravin Date: Tue, 31 Oct 2023 20:21:37 +0100 Subject: [PATCH 4/5] refactor(es): use an object for search field priorities if the lang is 'current', then multilingual fields priority are increased by 10, with a fallback on the * wildcard --- .../src/lib/gn4/elasticsearch/constant.ts | 17 +++--- .../elasticsearch.service.spec.ts | 46 +++++++-------- .../elasticsearch/elasticsearch.service.ts | 56 ++++++++++++------- 3 files changed, 67 insertions(+), 52 deletions(-) diff --git a/libs/api/repository/src/lib/gn4/elasticsearch/constant.ts b/libs/api/repository/src/lib/gn4/elasticsearch/constant.ts index bff735b7d4..676a59ae64 100644 --- a/libs/api/repository/src/lib/gn4/elasticsearch/constant.ts +++ b/libs/api/repository/src/lib/gn4/elasticsearch/constant.ts @@ -14,11 +14,12 @@ export const ES_SOURCE_SUMMARY = [ 'userSavedCount', ] -export const ES_QUERY_STRING_FIELDS = [ - 'resourceTitleObject.${searchLang}^5', - 'tag.${searchLang}^4', - 'resourceAbstractObject.${searchLang}^3', - 'lineageObject.${searchLang}^2', - 'any.${searchLang}', - 'uuid', -] +export type EsQueryFieldsPriorityType = Record +export const ES_QUERY_FIELDS_PRIORITY = { + 'resourceTitleObject.${searchLang}': 5, + 'tag.${searchLang}': 4, + 'resourceAbstractObject.${searchLang}': 3, + 'lineageObject.${searchLang}': 2, + 'any.${searchLang}': 1, + uuid: 1, +} diff --git a/libs/api/repository/src/lib/gn4/elasticsearch/elasticsearch.service.spec.ts b/libs/api/repository/src/lib/gn4/elasticsearch/elasticsearch.service.spec.ts index 1e9165077e..8d6ceeb0d1 100644 --- a/libs/api/repository/src/lib/gn4/elasticsearch/elasticsearch.service.spec.ts +++ b/libs/api/repository/src/lib/gn4/elasticsearch/elasticsearch.service.spec.ts @@ -362,7 +362,7 @@ describe('ElasticsearchService', () => { }) describe('#injectLangInQueryStringFields - Search language', () => { - let queryStringFields = ['resourceTitleObject.${searchLang}'] + let queryStringFields = { 'resourceTitleObject.${searchLang}': 1 } describe('When no lang from config', () => { beforeEach(() => { service['metadataLang'] = undefined @@ -397,31 +397,31 @@ describe('ElasticsearchService', () => { service['injectLangInQueryStringFields'](queryStringFields)[0].split( '.' )[1] - ).toEqual('langeng') + ).toEqual('langeng^11') }) it('add * fallback with low priority', () => { - queryStringFields = [ - 'resourceTitleObject.${searchLang}^5', - 'tag.${searchLang}^4', - 'resourceAbstractObject.${searchLang}^3', - 'lineageObject.${searchLang}^2', - 'any.${searchLang}', - 'uuid', - ] + queryStringFields = { + 'resourceTitleObject.${searchLang}': 5, + 'tag.${searchLang}': 4, + 'resourceAbstractObject.${searchLang}': 3, + 'lineageObject.${searchLang}': 2, + 'any.${searchLang}': 1, + uuid: 1, + } expect( service['injectLangInQueryStringFields'](queryStringFields) ).toEqual([ - 'resourceTitleObject.langeng^5', - 'tag.langeng^4', - 'resourceAbstractObject.langeng^3', - 'lineageObject.langeng^2', - 'any.langeng', - 'uuid', - 'resourceTitleObject.*', - 'tag.*', - 'resourceAbstractObject.*', - 'lineageObject.*', + 'resourceTitleObject.langeng^15', + 'resourceTitleObject.*^5', + 'tag.langeng^14', + 'tag.*^4', + 'resourceAbstractObject.langeng^13', + 'resourceAbstractObject.*^3', + 'lineageObject.langeng^12', + 'lineageObject.*^2', + 'any.langeng^11', 'any.*', + 'uuid', ]) }) }) @@ -444,9 +444,9 @@ describe('ElasticsearchService', () => { { multi_match: { fields: [ - 'resourceTitleObject.langfre', - 'resourceAbstractObject.langfre', - 'tag', + 'resourceTitleObject.langfre^4', + 'resourceAbstractObject.langfre^3', + 'tag^2', 'resourceIdentifier', ], query: 'blarg', diff --git a/libs/api/repository/src/lib/gn4/elasticsearch/elasticsearch.service.ts b/libs/api/repository/src/lib/gn4/elasticsearch/elasticsearch.service.ts index a4a82a76ec..2390989b24 100644 --- a/libs/api/repository/src/lib/gn4/elasticsearch/elasticsearch.service.ts +++ b/libs/api/repository/src/lib/gn4/elasticsearch/elasticsearch.service.ts @@ -1,6 +1,10 @@ import { Inject, Injectable, Optional } from '@angular/core' import { Geometry } from 'geojson' -import { ES_QUERY_STRING_FIELDS, ES_SOURCE_SUMMARY } from './constant' +import { + ES_QUERY_FIELDS_PRIORITY, + ES_SOURCE_SUMMARY, + EsQueryFieldsPriorityType, +} from './constant' import { Aggregation, AggregationParams, @@ -170,20 +174,30 @@ export class ElasticsearchService { return fields.map((field) => ({ [field[1]]: field[0] })) } - private injectLangInQueryStringFields(queryStringFields: string[]) { + private injectLangInQueryStringFields( + queryFieldsPriority: EsQueryFieldsPriorityType + ) { const queryLang = this.getQueryLang() - return [ - ...queryStringFields.map((field) => { - return field.replace(/\$\{searchLang\}/g, queryLang) - }), - ...(this.isCurrentSearchLang() - ? queryStringFields - .filter((field) => /\$\{searchLang\}/.test(field)) - .map((field) => { - return field.replace(/\.\$\{searchLang\}(\^\d+)?/, '.*') - }) - : []), - ] + return Object.keys(queryFieldsPriority).reduce((query, field) => { + const multiLangRegExp = /\$\{searchLang\}/g + const isMultilangField = multiLangRegExp.test(field) + const fieldPriority = queryFieldsPriority[field] + return [ + ...query, + ...(this.isCurrentSearchLang() && isMultilangField + ? [ + `${field.replace(multiLangRegExp, queryLang)}^${ + fieldPriority + 10 + }`, + field.replace(multiLangRegExp, '*') + + (fieldPriority > 1 ? `^${fieldPriority}` : ''), + ] + : [ + field.replace(multiLangRegExp, queryLang) + + (fieldPriority > 1 ? `^${fieldPriority}` : ''), + ]), + ] + }, []) } private getQueryLang(): string { @@ -223,7 +237,7 @@ export class ElasticsearchService { query_string: { query: this.escapeSpecialCharacters(any), default_operator: 'AND', - fields: this.injectLangInQueryStringFields(ES_QUERY_STRING_FIELDS), + fields: this.injectLangInQueryStringFields(ES_QUERY_FIELDS_PRIORITY), }, }) } @@ -307,12 +321,12 @@ export class ElasticsearchService { multi_match: { query, type: 'bool_prefix', - fields: this.injectLangInQueryStringFields([ - 'resourceTitleObject.${searchLang}', - 'resourceAbstractObject.${searchLang}', - 'tag', - 'resourceIdentifier', - ]), + fields: this.injectLangInQueryStringFields({ + 'resourceTitleObject.${searchLang}': 4, + 'resourceAbstractObject.${searchLang}': 3, + tag: 2, + resourceIdentifier: 1, + }), }, }, ], From 020fa079a15d6d9a2f420ec18ef191118ca6d24e Mon Sep 17 00:00:00 2001 From: Florent Gravin Date: Fri, 3 Nov 2023 10:53:14 +0000 Subject: [PATCH 5/5] test(elasticsearch): use TestBed for service creation --- .../elasticsearch.service.spec.ts | 22 +++++++++++++++---- .../elasticsearch/elasticsearch.service.ts | 6 ++--- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/libs/api/repository/src/lib/gn4/elasticsearch/elasticsearch.service.spec.ts b/libs/api/repository/src/lib/gn4/elasticsearch/elasticsearch.service.spec.ts index 8d6ceeb0d1..c3e1b5688d 100644 --- a/libs/api/repository/src/lib/gn4/elasticsearch/elasticsearch.service.spec.ts +++ b/libs/api/repository/src/lib/gn4/elasticsearch/elasticsearch.service.spec.ts @@ -2,17 +2,31 @@ import { ElasticsearchService } from './elasticsearch.service' import { ES_FIXTURE_AGGS_RESPONSE } from '@geonetwork-ui/common/fixtures' import { LangService } from '@geonetwork-ui/util/i18n' import { EsSearchParams } from '@geonetwork-ui/api/metadata-converter' +import { TestBed } from '@angular/core/testing' +import { METADATA_LANGUAGE } from '../../metadata-language' -const langServiceMock = { - iso3: 'eng', -} as LangService +class LangServiceMock { + iso3 = 'eng' +} describe('ElasticsearchService', () => { let service: ElasticsearchService let searchFilters beforeEach(() => { - service = new ElasticsearchService(langServiceMock, 'fre') + TestBed.configureTestingModule({ + providers: [ + { + provide: LangService, + useClass: LangServiceMock, + }, + { + provide: METADATA_LANGUAGE, + useValue: 'fre', + }, + ], + }) + service = TestBed.inject(ElasticsearchService) }) it('should be created', () => { diff --git a/libs/api/repository/src/lib/gn4/elasticsearch/elasticsearch.service.ts b/libs/api/repository/src/lib/gn4/elasticsearch/elasticsearch.service.ts index 2390989b24..3bdd0b7c0a 100644 --- a/libs/api/repository/src/lib/gn4/elasticsearch/elasticsearch.service.ts +++ b/libs/api/repository/src/lib/gn4/elasticsearch/elasticsearch.service.ts @@ -201,11 +201,11 @@ export class ElasticsearchService { } private getQueryLang(): string { - return this.metadataLang - ? this.isCurrentSearchLang() + if (this.metadataLang) { + return this.isCurrentSearchLang() ? `lang${this.lang3}` : `lang${this.metadataLang}` - : `*` + } else return '*' } private isCurrentSearchLang() { return this.metadataLang === 'current'