Prepare PR for merging TUL into CLARIN-DSpace7.5.

.github/workflows/docker.yml

@@ -6,7 +6,7 @@ name: Docker images
 on:
   push:
     branches:
-      - dtq-dev
+      - customer/TUL
   pull_request:
   workflow_dispatch:
 
@@ -170,11 +170,11 @@ jobs:
           labels: ${{ steps.meta_build_cli.outputs.labels }}
 
       - name: redeploy
-        if: 'false'
+        if: '!cancelled()'
         run: |
           curl -H "Accept: application/vnd.github.everest-preview+json" \
           -H "Authorization: token ${{ secrets.DEPLOY_DEV5_GH_ACTION_DISPATCH }}" \
           --request POST \
           https://api.github.com/repos/dataquest-dev/\
           dspace-angular/actions/workflows/deploy.yml/dispatches \
-          --data "{\"ref\":\"refs/heads/dtq-dev-7.5\"}"
+          --data "{\"ref\":\"refs/heads/customer/TUL\"}"

dspace-api/src/main/java/org/dspace/app/statistics/clarin/ClarinMatomoBitstreamTracker.java

@@ -111,8 +111,10 @@ private String getItemIdentifier(Item item) {
     public void trackBitstreamDownload(Context context, HttpServletRequest request, Bitstream bit) throws SQLException {
         // We only track a download request when serving a request without Range header. Do not track the
         // download if the downloading continues or the tracking is not allowed by the configuration.
-        if (StringUtils.isNotBlank(request.getHeader("Range")) &&
-                BooleanUtils.isFalse(configurationService.getBooleanProperty("matomo.track.enabled"))) {
+        if (StringUtils.isNotBlank(request.getHeader("Range"))) {
+            return;
+        }
+        if (BooleanUtils.isFalse(configurationService.getBooleanProperty("matomo.track.enabled"))) {
             return;
         }
 

dspace-api/src/main/java/org/dspace/authenticate/clarin/Headers.java

@@ -8,20 +8,25 @@
 /* Created for LINDAT/CLARIN */
 package org.dspace.authenticate.clarin;
 
+import java.io.UnsupportedEncodingException;
 import java.util.ArrayList;
 import java.util.Enumeration;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import javax.servlet.http.HttpServletRequest;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
 /**
  * Helper class for request headers.
  * Class is copied from UFAL/CLARIN-DSPACE (https://github.com/ufal/clarin-dspace) and modified by
  * @author Milan Majchrak (milan.majchrak at dataquest.sk)
  */
 public class Headers {
 
+    private static final Logger log = LogManager.getLogger(org.dspace.authenticate.clarin.Headers.class);
     // variables
     //
 
@@ -56,7 +61,7 @@ public void initialise(HttpServletRequest request, String header_separator, List
             List<String> vals = new ArrayList<String>();
             Enumeration e_vals = request.getHeaders(key);
             while (e_vals.hasMoreElements()) {
-                String values = (String)e_vals.nextElement();
+                String values = updateValueByCharset((String) e_vals.nextElement());
                 vals.addAll( header2values(values) );
             }
 
@@ -149,4 +154,20 @@ private List<String> header2values(String header) {
 
         return values;
     }
+
+
+    /**
+     * Convert ISO header value to UTF-8
+     * @param value ISO header value String
+     * @return
+     */
+    private String updateValueByCharset(String value) {
+        try {
+            return new String(value.getBytes("ISO-8859-1"), "UTF-8");
+        } catch (UnsupportedEncodingException ex) {
+            log.warn("Failed to reconvert shibboleth attribute with value ("
+                    + value + ").", ex);
+        }
+        return value;
+    }
 }

dspace-api/src/test/data/dspaceFolder/config/local.cfg

@@ -261,3 +261,6 @@ authority.controlled.dspace.object.owner = true
 # Configuration required for thorough testing of browse links
 webui.browse.link.1 = author:dc.contributor.*
 webui.browse.link.2 = subject:dc.subject.*
+
+# If the versioning is disabled Versioning Integration Tests will fail - allow it for the tests
+versioning.enabled=true

dspace-server-webapp/src/main/java/org/dspace/app/rest/Application.java

@@ -12,6 +12,7 @@
 import java.io.IOException;
 import java.sql.SQLException;
 import java.util.List;
+import java.util.function.Predicate;
 import javax.servlet.Filter;
 
 import org.dspace.app.rest.filter.DSpaceRequestContextFilter;
@@ -39,6 +40,8 @@
 import org.springframework.lang.NonNull;
 import org.springframework.scheduling.annotation.EnableScheduling;
 import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.security.web.firewall.HttpFirewall;
+import org.springframework.security.web.firewall.StrictHttpFirewall;
 import org.springframework.web.context.request.RequestContextListener;
 import org.springframework.web.cors.CorsConfiguration;
 import org.springframework.web.method.support.HandlerMethodArgumentResolver;
@@ -167,6 +170,23 @@ protected LinkRelationProvider dspaceLinkRelationProvider() {
         return new DSpaceLinkRelationProvider();
     }
 
+    /**
+     * StrictHttpFirewall doesn't allow ISO header values by default. It could throw an error during Shibboleth
+     * authentication if the user has UTF-8 characters in the name.
+     * Updated allowedHeaderValues without any regex - it allows every character.
+     * @return
+     */
+    @Bean
+    public HttpFirewall allowUrlEncodedSlashHttpFirewall() {
+        StrictHttpFirewall firewall = new StrictHttpFirewall();
+        Predicate<String> test = (s) -> {
+            return true;
+        };
+
+        firewall.setAllowedHeaderValues(test);
+        return firewall;
+    }
+
     @Bean
     public WebMvcConfigurer webMvcConfigurer() {
 

dspace-server-webapp/src/main/java/org/dspace/app/rest/ClarinDiscoJuiceFeedsDownloadService.java

@@ -78,11 +78,11 @@ public void afterPropertiesSet() throws Exception {
                 service = new DatabaseReader.Builder(
                         new BufferedInputStream(Files.newInputStream(Paths.get(dbfile)))).build();
             } catch (IOException e) {
-                log.debug("ERROR: Unable to load GeoLite Database file (" + dbfile + ")! " +
+                log.error("Unable to load GeoLite Database file (" + dbfile + ")! " +
                         "You may need to reinstall it. See the DSpace installation instructions for more details.", e);
             }
         } else {
-            log.debug("ERROR: The required 'dbfile' configuration is missing in solr-statistics.cfg!");
+            log.error("The required 'dbfile' configuration is missing in solr-statistics.cfg!");
         }
         locationService = service;
 
@@ -100,7 +100,7 @@ public void afterPropertiesSet() throws Exception {
     }
 
     public String createFeedsContent() {
-        log.debug("Going to create feeds content.");
+        log.info("Going to create feeds content.");
         String[] feedsConfig = configurationService.getArrayProperty("discojuice.feeds");
         String shibbolethDiscoFeedUrl = configurationService.getProperty("shibboleth.discofeed.url");
 

dspace-server-webapp/src/main/java/org/dspace/app/rest/ClarinDiscoJuiceFeedsUpdateScheduler.java

@@ -54,7 +54,7 @@ public void cronJobSch() {
             return;
         }
 
-        log.debug("CRON Job - going to download the discojuice feeds.");
+        log.info("CRON Job - going to download the discojuice feeds.");
         String newFeedsContent = clarinDiscoJuiceFeedsDownloadService.createFeedsContent();
         if (isNotBlank(newFeedsContent)) {
             feedsContent = newFeedsContent;

dspace-server-webapp/src/main/java/org/dspace/app/rest/security/clarin/ClarinShibbolethLoginFilter.java

@@ -78,7 +78,8 @@ public class ClarinShibbolethLoginFilter extends StatelessLoginFilter {
 
     public static final String VERIFICATION_TOKEN_HEADER = "Verification-Token";
 
-    private static final Logger log = LogManager.getLogger(org.dspace.app.rest.security.ShibbolethLoginFilter.class);
+    private static final Logger log = LogManager.getLogger(org.dspace.app.rest.security.clarin.
+            ClarinShibbolethLoginFilter.class);
 
     /**
      * Property which handles information if the IdP send required information.
@@ -176,6 +177,11 @@ public Authentication attemptAuthentication(HttpServletRequest req,
             }
         }
 
+        // logging
+        log.info("Shib-Identity-Provider: " + idp);
+        log.info("authentication-shibboleth.netid-header: " + netidHeader + " with value: " + netid);
+        log.info("authentication-shibboleth.email-header: " + emailHeader + " with value: " + email);
+
         try {
             if (StringUtils.isEmpty(netid) || StringUtils.isEmpty(idp)) {
                 log.error("Cannot load the netid or idp from the request headers.");

dspace-server-webapp/src/test/java/org/dspace/app/rest/ClarinDiscoveryRestControllerIT.java

@@ -76,6 +76,8 @@
  *
  * @author Milan Majchrak (milan.majchrak at dataquest.sk)
  */
+// Ignore for the TUL customer
+@Ignore
 public class ClarinDiscoveryRestControllerIT extends AbstractControllerIntegrationTest {
     @Autowired
     ConfigurationService configurationService;

dspace-server-webapp/src/test/java/org/dspace/app/rest/ClarinWorkspaceItemRestRepositoryIT.java

@@ -49,6 +49,7 @@
 import org.dspace.content.service.clarin.ClarinLicenseService;
 import org.dspace.services.ConfigurationService;
 import org.junit.Assert;
+import org.junit.Ignore;
 import org.junit.Test;
 import org.springframework.beans.factory.annotation.Autowired;
 
@@ -57,6 +58,8 @@
  *
  * @author Milan Majchrak (milan.majchrak at dataquest.sk)
  */
+// Ignore for TUL customer
+@Ignore
 public class ClarinWorkspaceItemRestRepositoryIT extends AbstractControllerIntegrationTest {
 
     public static final String REST_SERVER_URL = "http://localhost/api/";

dspace-server-webapp/src/test/java/org/dspace/app/rest/DiscoveryRestControllerIT.java

@@ -75,7 +75,6 @@
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.http.MediaType;
 
-@Ignore
 public class DiscoveryRestControllerIT extends AbstractControllerIntegrationTest {
     @Autowired
     ConfigurationService configurationService;

dspace-server-webapp/src/test/java/org/dspace/app/rest/security/ClarinShibbolethLoginFilterIT.java

@@ -458,4 +458,35 @@ public void testRedirectToGivenUntrustedUrl() throws Exception {
                         .header("SHIB-NETID", NET_ID_TEST_EPERSON))
                 .andExpect(status().isBadRequest());
     }
+
+    @Test
+    public void testUTF8ShibHeaders() throws Exception {
+        // NOTE: The initial call to /shibboleth comes *from* an external Shibboleth site. So, it is always
+        // unauthenticated, but it must include some expected SHIB attributes.
+        // SHIB-MAIL attribute is the default email header sent from Shibboleth after a successful login.
+        // In this test we are simply mocking that behavior by setting it to an existing EPerson.
+        String token = getClient().perform(get("/api/authn/shibboleth")
+                        .header("SHIB-MAIL", clarinEperson.getEmail())
+                        .header("Shib-Identity-Provider", IDP_TEST_EPERSON)
+                        .header("SHIB-NETID", NET_ID_TEST_EPERSON)
+                        .header("SHIB-GIVENNAME", "knihovna KůÅ\u0088 test ŽluÅ¥ouÄ\u008Dký"))
+                .andExpect(status().is3xxRedirection())
+                .andExpect(redirectedUrl("http://localhost:4000"))
+                .andReturn().getResponse().getHeader("Authorization");
+
+
+        getClient(token).perform(get("/api/authn/status"))
+                .andExpect(status().isOk())
+                .andExpect(jsonPath("$.authenticated", is(true)))
+                .andExpect(jsonPath("$.authenticationMethod", is("shibboleth")));
+
+        getClient(token).perform(
+                        get("/api/authz/authorizations/search/object")
+                                .param("embed", "feature")
+                                .param("feature", feature)
+                                .param("uri", utils.linkToSingleResource(ePersonRest, "self").getHref()))
+                .andExpect(status().isOk())
+                .andExpect(jsonPath("$.page.totalElements", is(0)))
+                .andExpect(jsonPath("$._embedded").doesNotExist());
+    }
 }

dspace/config/clarin-dspace.cfg

@@ -125,7 +125,7 @@ featured.service.teitok.description = A web-based platform for viewing, creating
 
 ##### Shibboleth #####
 # Turn off the discofeed, it is allowed by default
-# shibboleth.discofeed.allowed = false
+shibboleth.discofeed.allowed = false
 # File where is DiscoJuiceFeed response
 shibboleth.discofeed.url = https://lindat.mff.cuni.cz/Shibboleth.sso/DiscoFeed
 
@@ -139,7 +139,7 @@ discojuice.rewriteCountries = https://idp.scc.kit.edu/idp/shibboleth, https://fe
 
 ##### Matomo statistics #####
 # Auth token
-matomo.track.enabled = true
+matomo.track.enabled = false
 matomo.auth.token = 26388b4164695d69e6ee6e2dd527b723
 matomo.site.id = 1
 matomo.tracker.bitstream.site_id = 1
@@ -148,29 +148,74 @@ statistics.cache-server.uri = http://cache-server.none
 
 
 ##### Citacepro config #####
-# citace.pro.url = https://www.citacepro.com/api/dspace/citace/oai
-# citace.pro.university = dspace.tul.cz
+citace.pro.url = https://www.citacepro.com/api/dspace/citace/oai
+citace.pro.university = dspace.tul.cz
 
 # only use true or false for the value of citace pro allowed
-# citace.pro.allowed = true
+citace.pro.allowed = true
 
 #google config
-# google.analytics.key =
+google.analytics.key=UA-134956309-1
 
 # The max number of events held in the GA buffer (default: 256)
-# google.analytics.buffer.limit = 256
+google.analytics.buffer.limit=256
 
 # Define cron for how frequently events tracked in the DSpace backend will be sent to Google Analytics
 # This MUST be enabled if you wish to use `google.analytics.api-secret` to track bitstream download statistics (and similar)
 # Cron syntax is defined at https://www.quartz-scheduler.org/api/2.3.0/org/quartz/CronTrigger.html
 # Keep in mind, changing the schedule requires rebooting your servlet container, e.g. Tomcat.
 # The below example will run this task daily, every 5 minutes
-# google.analytics.cron = 0 0/1 * * * ?
+google.analytics.cron = 0 0/1 * * * ?
 
 # Defines a Measurement Protocol API Secret to be used to track interactions which occur outside of the user's browser.
 # For example , this is required to track downloads of bitstreams. This setting is only used by Google Analytics 4.
 # For more details see https://developers.google.com/analytics/devguides/collection/protocol/ga4
-# google.analytics.api-secret =
+# google.analytics.api-secret = EvBfmLs_QPaKei9XRibauw
 
 ##### Importing #####
 import.metadata.field.not.update = dc.description.provenance, dc.date.available, dc.date.accessioned, dc.identifier.uri
+
+#------------------------------------------------------------------#
+#-------------------------Configure DOI----------------------------#
+#------------------------------------------------------------------#
+
+# Credentials used to authenticate against the registration agency:
+identifier.doi.user = username
+identifier.doi.password = password
+# DOI prefix used to mint DOIs. All DOIs minted by DSpace will use this prefix.
+# The Prefix will be assigned by the registration agency.
+identifier.doi.prefix = 10.5072
+# If you want to, you can further separate your namespace. Should all the
+# suffixes of all DOIs minted by DSpace start with a special string to separate
+# it from other services also minting DOIs under your prefix?
+identifier.doi.namespaceseparator = dspace/
+
+##
+## Configure XSLT-driven submission crosswalk for DataCite
+##
+crosswalk.dissemination.DataCite.stylesheet = crosswalks/DIM2DataCite.xsl
+crosswalk.dissemination.DataCite.schemaLocation = \
+    http://datacite.org/schema/kernel-3 \
+    http://schema.datacite.org/meta/kernel-3/metadata.xsd
+crosswalk.dissemination.DataCite.preferList = false
+crosswalk.dissemination.DataCite.publisher = My University
+#crosswalk.dissemination.DataCite.dataManager = # defaults to publisher
+#crosswalk.dissemination.DataCite.hostingInstitution = # defaults to publisher
+crosswalk.dissemination.DataCite.namespace = http://datacite.org/schema/kernel-3
+
+# consumer to update metadata of DOIs
+event.consumer.doi.class = org.dspace.identifier.doi.DOIConsumer
+event.consumer.doi.filters = Item+Modify_Metadata
+
+# Add doi here if you are using org.dspace.identifier.DOIIdentifierProvider to generate DOIs.
+# Adding doi here makes DSpace send metadata updates to your doi registration agency.
+# Add rdf here, if you are using dspace-rdf to export your repository content as RDF.
+# Add iiif here, if you are using dspace-iiif.
+event.dispatcher.default.consumers = versioning, discovery, eperson, doi
+
+# Edit Item - Status option
+identifiers.item-status.register-doi = true
+
+##### Dataquest URL - sing in the footer #####
+themed.by.url = https://www.dataquest.sk/dspace
+themed.by.company.name = dataquest s.r.o.