From 33a4c96b06573a75ae3cc0df7b82e6f95d22e95d Mon Sep 17 00:00:00 2001 From: milanmajchrak Date: Thu, 31 Aug 2023 13:49:01 +0200 Subject: [PATCH 1/5] Initial commit From 8271efd07364501db0bd0dd159fe4ea400908d06 Mon Sep 17 00:00:00 2001 From: milanmajchrak Date: Fri, 1 Sep 2023 10:58:45 +0200 Subject: [PATCH 2/5] Working --- .../content/InstallItemServiceImpl.java | 25 +++++++++++++++++++ .../org/dspace/discovery}/IsoLangCodes.java | 3 ++- .../org/dspace/discovery/SolrServiceImpl.java | 6 +++++ .../DiscoveryConfigurationParameters.java | 2 ++ .../indexobject/ItemIndexFactoryImpl.java | 16 ++++++++++++ .../src/main/resources/lang_codes.txt | 0 .../resources/functions/GetLangForCodeFn.java | 2 +- dspace/config/registries/local-types.xml | 7 ++++++ dspace/config/spring/api/discovery.xml | 2 +- .../fast-build/dspace-api-package-update.bat | 1 + 10 files changed, 61 insertions(+), 3 deletions(-) rename {dspace-oai/src/main/java/org/dspace/utils => dspace-api/src/main/java/org/dspace/discovery}/IsoLangCodes.java (99%) rename {dspace-server-webapp => dspace-api}/src/main/resources/lang_codes.txt (100%) diff --git a/dspace-api/src/main/java/org/dspace/content/InstallItemServiceImpl.java b/dspace-api/src/main/java/org/dspace/content/InstallItemServiceImpl.java index 32c5b92c605..b0a7adfec5c 100644 --- a/dspace-api/src/main/java/org/dspace/content/InstallItemServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/content/InstallItemServiceImpl.java @@ -12,6 +12,7 @@ import java.util.List; import java.util.Map; +import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.dspace.authorize.AuthorizeException; @@ -23,6 +24,7 @@ import org.dspace.content.service.ItemService; import org.dspace.core.Constants; import org.dspace.core.Context; +import org.dspace.discovery.IsoLangCodes; import org.dspace.embargo.service.EmbargoService; import org.dspace.event.Event; import org.dspace.identifier.Identifier; @@ -200,6 +202,9 @@ protected void populateMetadata(Context c, Item item) // Add provenance description itemService.addMetadata(c, item, MetadataSchemaEnum.DC.getName(), "description", "provenance", "en", provDescription); + + // Add language name into metadata. The lang name is fetched from the `lang_codes.txt`. + addLanguageNameToMetadata(c, item); } /** @@ -271,4 +276,24 @@ public String getBitstreamProvenanceMessage(Context context, Item myitem) return myMessage.toString(); } + + private void addLanguageNameToMetadata(Context c, Item item) throws SQLException { + itemService.clearMetadata(c, item, "local", "language", "name", null); + List languageMetadata = itemService.getMetadataByMetadataString(item, "dc.language.iso"); + for (MetadataValue mv: languageMetadata) { + if (StringUtils.isBlank(mv.getValue())) { + log.error("Cannot get name of the iso language (`dc.language.iso`) because the value is blank."); + return; + } + String langName = IsoLangCodes + .getLangForCode(mv.getValue()); + if (StringUtils.isBlank(langName)) { + log.error(String + .format("No language found for iso code %s", + langName)); + return; + } + itemService.addMetadata(c, item, "local", "language", "name", null, langName); + } + } } diff --git a/dspace-oai/src/main/java/org/dspace/utils/IsoLangCodes.java b/dspace-api/src/main/java/org/dspace/discovery/IsoLangCodes.java similarity index 99% rename from dspace-oai/src/main/java/org/dspace/utils/IsoLangCodes.java rename to dspace-api/src/main/java/org/dspace/discovery/IsoLangCodes.java index a994cf1f912..6a6d49ded91 100644 --- a/dspace-oai/src/main/java/org/dspace/utils/IsoLangCodes.java +++ b/dspace-api/src/main/java/org/dspace/discovery/IsoLangCodes.java @@ -7,7 +7,7 @@ */ /* Created for LINDAT/CLARIAH-CZ (UFAL) */ -package org.dspace.utils; +package org.dspace.discovery; import java.io.BufferedReader; import java.io.IOException; @@ -21,6 +21,7 @@ /** * Class is copied from the LINDAT/CLARIAH-CZ (https://github.com/ufal/clarin-dspace/blob * /si-master-origin/dspace-api/src/main/java/cz/cuni/mff/ufal/IsoLangCodes.java) and modified by + * * @author Marian Berger (marian.berger at dataquest.sk) */ public class IsoLangCodes { diff --git a/dspace-api/src/main/java/org/dspace/discovery/SolrServiceImpl.java b/dspace-api/src/main/java/org/dspace/discovery/SolrServiceImpl.java index 68d3b48ec09..f3b8d2c56a9 100644 --- a/dspace-api/src/main/java/org/dspace/discovery/SolrServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/discovery/SolrServiceImpl.java @@ -1409,6 +1409,12 @@ protected String transformFacetField(DiscoverFacetField facetFieldConfig, String } } else if (facetFieldConfig.getType().equals(DiscoveryConfigurationParameters.TYPE_STANDARD)) { return field; + } else if (facetFieldConfig.getType().equals(DiscoveryConfigurationParameters.TYPE_ISO_LANG)){ + if(removePostfix) { + return field.substring(0, field.lastIndexOf("_filter")); + } else { + return field + "_filter"; + } } else { return field; } diff --git a/dspace-api/src/main/java/org/dspace/discovery/configuration/DiscoveryConfigurationParameters.java b/dspace-api/src/main/java/org/dspace/discovery/configuration/DiscoveryConfigurationParameters.java index ed51fa8ef82..59955e9b991 100644 --- a/dspace-api/src/main/java/org/dspace/discovery/configuration/DiscoveryConfigurationParameters.java +++ b/dspace-api/src/main/java/org/dspace/discovery/configuration/DiscoveryConfigurationParameters.java @@ -21,6 +21,8 @@ public class DiscoveryConfigurationParameters { public static final String TYPE_AUTHORITY = "authority"; public static final String TYPE_STANDARD = "standard"; + public static final String TYPE_ISO_LANG = "iso_language"; + public static enum SORT { VALUE, COUNT } /** diff --git a/dspace-api/src/main/java/org/dspace/discovery/indexobject/ItemIndexFactoryImpl.java b/dspace-api/src/main/java/org/dspace/discovery/indexobject/ItemIndexFactoryImpl.java index 61827747b72..a191ae4a589 100644 --- a/dspace-api/src/main/java/org/dspace/discovery/indexobject/ItemIndexFactoryImpl.java +++ b/dspace-api/src/main/java/org/dspace/discovery/indexobject/ItemIndexFactoryImpl.java @@ -46,7 +46,9 @@ import org.dspace.core.Context; import org.dspace.core.LogHelper; import org.dspace.discovery.FullTextContentStreams; +import org.dspace.discovery.IsoLangCodes; import org.dspace.discovery.SearchUtils; +import org.dspace.discovery.SolrServiceImpl; import org.dspace.discovery.configuration.DiscoveryConfiguration; import org.dspace.discovery.configuration.DiscoveryConfigurationParameters; import org.dspace.discovery.configuration.DiscoveryHitHighlightFieldConfiguration; @@ -819,6 +821,20 @@ private void indexIfFilterTypeFacet(SolrInputDocument doc, DiscoverySearchFilter } //Also add prefix field with all parts of value saveFacetPrefixParts(doc, searchFilter, value, separator, authority, preferedLabel); + } else if (searchFilter.getType().equals(DiscoveryConfigurationParameters.TYPE_ISO_LANG)) { + String langName = IsoLangCodes + .getLangForCode(value); + if (StringUtils.isBlank(langName)) { + log.error(String + .format("No language found for iso code %s", + value)); + return; + } + String convertedValue = langName.toLowerCase() + SearchUtils.FILTER_SEPARATOR + langName; + doc.addField(searchFilter.getIndexFieldName() + "_filter", convertedValue); + doc.addField(searchFilter.getIndexFieldName(), langName); + doc.addField(searchFilter.getIndexFieldName() + "_keyword", langName); + doc.addField(searchFilter.getIndexFieldName() + "_ac", langName); } } diff --git a/dspace-server-webapp/src/main/resources/lang_codes.txt b/dspace-api/src/main/resources/lang_codes.txt similarity index 100% rename from dspace-server-webapp/src/main/resources/lang_codes.txt rename to dspace-api/src/main/resources/lang_codes.txt diff --git a/dspace-oai/src/main/java/org/dspace/xoai/services/impl/resources/functions/GetLangForCodeFn.java b/dspace-oai/src/main/java/org/dspace/xoai/services/impl/resources/functions/GetLangForCodeFn.java index 79b41281524..a84de181b87 100644 --- a/dspace-oai/src/main/java/org/dspace/xoai/services/impl/resources/functions/GetLangForCodeFn.java +++ b/dspace-oai/src/main/java/org/dspace/xoai/services/impl/resources/functions/GetLangForCodeFn.java @@ -8,7 +8,7 @@ package org.dspace.xoai.services.impl.resources.functions; -import org.dspace.utils.IsoLangCodes; +import org.dspace.discovery.IsoLangCodes; /** * Serves as proxy for call from XSL engine. Calls SpecialItemService. diff --git a/dspace/config/registries/local-types.xml b/dspace/config/registries/local-types.xml index 8e78d2a30ab..ad1aba08803 100644 --- a/dspace/config/registries/local-types.xml +++ b/dspace/config/registries/local-types.xml @@ -138,4 +138,11 @@ . Check/extend html.xsl for available variables. + + local + language + name + ISO language name + + diff --git a/dspace/config/spring/api/discovery.xml b/dspace/config/spring/api/discovery.xml index 3a40ca0bb45..9b8b8351fe5 100644 --- a/dspace/config/spring/api/discovery.xml +++ b/dspace/config/spring/api/discovery.xml @@ -2463,7 +2463,7 @@ - + diff --git a/scripts/fast-build/dspace-api-package-update.bat b/scripts/fast-build/dspace-api-package-update.bat index e0f4de3d787..842938c0165 100644 --- a/scripts/fast-build/dspace-api-package-update.bat +++ b/scripts/fast-build/dspace-api-package-update.bat @@ -9,6 +9,7 @@ call mvn clean package rem copy created jar into tomcat/webapps/server xcopy /e /h /i /q /y %dspace_source%\dspace-api\target\dspace-api-7.5.jar %tomcat%\webapps\server\WEB-INF\lib\ +xcopy /e /h /i /q /y %dspace_source%\dspace-api\target\dspace-api-7.5.jar %dspace_application%\lib\ cd %dspace_source%\scripts\fast-build\ From db56bddf2371b44593e18e3f12cc3d341853aeaf Mon Sep 17 00:00:00 2001 From: milanmajchrak Date: Fri, 1 Sep 2023 11:23:26 +0200 Subject: [PATCH 3/5] Fixed checkstyle issue --- .../src/main/java/org/dspace/discovery/SolrServiceImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/discovery/SolrServiceImpl.java b/dspace-api/src/main/java/org/dspace/discovery/SolrServiceImpl.java index f3b8d2c56a9..3f59ba4f481 100644 --- a/dspace-api/src/main/java/org/dspace/discovery/SolrServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/discovery/SolrServiceImpl.java @@ -1409,8 +1409,8 @@ protected String transformFacetField(DiscoverFacetField facetFieldConfig, String } } else if (facetFieldConfig.getType().equals(DiscoveryConfigurationParameters.TYPE_STANDARD)) { return field; - } else if (facetFieldConfig.getType().equals(DiscoveryConfigurationParameters.TYPE_ISO_LANG)){ - if(removePostfix) { + } else if (facetFieldConfig.getType().equals(DiscoveryConfigurationParameters.TYPE_ISO_LANG)) { + if (removePostfix) { return field.substring(0, field.lastIndexOf("_filter")); } else { return field + "_filter"; From 1349d7ce3a60bca28d86f158a591f6e943807909 Mon Sep 17 00:00:00 2001 From: milanmajchrak Date: Fri, 1 Sep 2023 12:23:05 +0200 Subject: [PATCH 4/5] Fixed failing integration tests --- .../java/org/dspace/app/rest/matcher/FacetEntryMatcher.java | 2 +- .../java/org/dspace/app/rest/matcher/SearchFilterMatcher.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/matcher/FacetEntryMatcher.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/matcher/FacetEntryMatcher.java index 984d43bbb9e..b850d388729 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/matcher/FacetEntryMatcher.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/matcher/FacetEntryMatcher.java @@ -123,7 +123,7 @@ public static Matcher clarinLicenseRightsFacet(boolean hasNext) public static Matcher clarinItemsLanguageFacet(boolean hasNext) { return allOf( hasJsonPath("$.name", is("language")), - hasJsonPath("$.facetType", is("standard")), + hasJsonPath("$.facetType", is("iso_language")), hasJsonPath("$.facetLimit", any(Integer.class)), hasJsonPath("$._links.self.href", containsString("api/discover/facets/language")), hasJsonPath("$._links", matchNextLink(hasNext, "api/discover/facets/language")) diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/matcher/SearchFilterMatcher.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/matcher/SearchFilterMatcher.java index adf7be06bab..2d38aeb1296 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/matcher/SearchFilterMatcher.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/matcher/SearchFilterMatcher.java @@ -170,7 +170,7 @@ public static Matcher clarinItemsLanguageFilter() { return allOf( hasJsonPath("$.filter", is("language")), hasJsonPath("$.hasFacets", is(true)), - hasJsonPath("$.type", is("standard")), + hasJsonPath("$.type", is("iso_language")), hasJsonPath("$.openByDefault", is(false)), checkOperators() ); From 530d9d34282d8516dc97e96783d6557bc1f77538 Mon Sep 17 00:00:00 2001 From: milanmajchrak Date: Fri, 1 Sep 2023 12:30:33 +0200 Subject: [PATCH 5/5] Added doc --- .../java/org/dspace/content/InstallItemServiceImpl.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/dspace-api/src/main/java/org/dspace/content/InstallItemServiceImpl.java b/dspace-api/src/main/java/org/dspace/content/InstallItemServiceImpl.java index b0a7adfec5c..0a824c98e08 100644 --- a/dspace-api/src/main/java/org/dspace/content/InstallItemServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/content/InstallItemServiceImpl.java @@ -277,6 +277,14 @@ public String getBitstreamProvenanceMessage(Context context, Item myitem) return myMessage.toString(); } + /** + * Language is stored in the metadatavalue in the ISO format e.g., `fra, cse,..` and not in the human satisfying + * format e.g., `France, Czech`. This method converts ISO format into human satisfying format e.g., `cse -> Czech` + * and stores it into `local.language.name` metadata field. + * @param c + * @param item + * @throws SQLException + */ private void addLanguageNameToMetadata(Context c, Item item) throws SQLException { itemService.clearMetadata(c, item, "local", "language", "name", null); List languageMetadata = itemService.getMetadataByMetadataString(item, "dc.language.iso");