Skip to content

Commit

Permalink
Merge branch 'dspace-cris-2023_02_x' into main-cris
Browse files Browse the repository at this point in the history
  • Loading branch information
abollini committed Apr 19, 2024
2 parents f768bbe + 78756f3 commit 00aa90c
Show file tree
Hide file tree
Showing 91 changed files with 5,237 additions and 471 deletions.
68 changes: 64 additions & 4 deletions bitbucket-pipelines.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ image: maven:3.6.1

definitions:
docker:
memory: 2048 # increase memory for docker-in-docker from 1GB to 2GB
memory: 4096 # increase memory for docker-in-docker from 1GB to 4GB
caches:
gradlewrapper: ~/.gradle/wrapper
steps:
Expand Down Expand Up @@ -46,9 +46,40 @@ definitions:
script:
- export MAVEN_OPTS="-Xmx4096M"
- ./dspace-api/src/test/data/dspaceFolder/bin/install_grobid.sh
- mvn clean install license:check -DskipUnitTests=true -Pdspace-rest -DskipITs=false -Pdspace-rest -DskipIntegrationTests=false -P !assembly -B -V -Dsurefire.rerunFailingTestsCount=2 -Dtest.argLine=-Xmx4096M
- mvn clean install -DskipUnitTests=true -Pdspace-rest -DskipITs=true -DskipIntegrationTests=true -P !assembly -B -V
- mvn test -pl dspace-iiif,dspace-oai,dspace-rdf,dspace-rest,dspace-server-webapp,dspace-services,dspace-sword,dspace-swordv2 -Dmaven.test.skip=false -DskipUnitTests=false -Pdspace-rest -DskipITs=false -Pdspace-rest -DskipIntegrationTests=false -B -V -Dsurefire.rerunFailingTestsCount=2 -Dtest.argLine=-Xmx4096M -DfailIfNoTests=false -Dtest=*IT,!Abstract*,!ItemImportIT,!GenericAuthorizationFeatureIT,!ItemRestRepositoryIT,!LeftTiltedRelationshipRestRepositoryIT,!RelationshipRestRepositoryIT,!StatisticsRestRepositoryIT,!WorkspaceItemRestRepositoryIT,!DiscoveryRestControllerIT,!PatchMetadataIT,!RightTiltedRelationshipRestRepositoryIT
artifacts:
- dspace-iiif/target/surefire-reports/*-output.txt
- dspace-oai/target/surefire-reports/*-output.txt
- dspace-rdf/target/surefire-reports/*-output.txt
- dspace-rest/target/surefire-reports/*-output.txt
- dspace-server-webapp/target/surefire-reports/*-output.txt
- dspace-services/target/surefire-reports/*-output.txt
- dspace-sword/target/surefire-reports/*-output.txt
- dspace-swordv2/target/surefire-reports/*-output.txt
- dspace-iiif/target/failsafe-reports/*-output.txt
- dspace-oai/target/failsafe-reports/*-output.txt
- dspace-rdf/target/failsafe-reports/*-output.txt
- dspace-rest/target/failsafe-reports/*-output.txt
- dspace-server-webapp/target/failsafe-reports/*-output.txt
- dspace-services/target/failsafe-reports/*-output.txt
- dspace-sword/target/failsafe-reports/*-output.txt
- dspace-swordv2/target/failsafe-reports/*-output.txt

- step: &integration-tests-slow
runs-on: self.hosted
name: integration-tests-slow
size: 4x
caches:
- gradle
- gradlewrapper
- maven
script:
- export MAVEN_OPTS="-Xmx4096M"
- ./dspace-api/src/test/data/dspaceFolder/bin/install_grobid.sh
- mvn clean install -DskipUnitTests=true -Pdspace-rest -DskipITs=true -DskipIntegrationTests=true -P !assembly -B -V
- mvn test -Dmaven.test.skip=false -DskipUnitTests=false -Pdspace-rest -DskipITs=false -Pdspace-rest -DskipIntegrationTests=false -B -V -Dsurefire.rerunFailingTestsCount=2 -Dtest.argLine=-Xmx4096M -DfailIfNoTests=false -Dtest=ItemImportIT,GenericAuthorizationFeatureIT,ItemRestRepositoryIT,LeftTiltedRelationshipRestRepositoryIT,RelationshipRestRepositoryIT,StatisticsRestRepositoryIT,WorkspaceItemRestRepositoryIT,DiscoveryRestControllerIT,PatchMetadataIT,RightTiltedRelationshipRestRepositoryIT
artifacts:
- dspace-api/target/surefire-reports/*-output.txt
- dspace-iiif/target/surefire-reports/*-output.txt
- dspace-oai/target/surefire-reports/*-output.txt
- dspace-rdf/target/surefire-reports/*-output.txt
Expand All @@ -57,7 +88,6 @@ definitions:
- dspace-services/target/surefire-reports/*-output.txt
- dspace-sword/target/surefire-reports/*-output.txt
- dspace-swordv2/target/surefire-reports/*-output.txt
- dspace-api/target/failsafe-reports/*-output.txt
- dspace-iiif/target/failsafe-reports/*-output.txt
- dspace-oai/target/failsafe-reports/*-output.txt
- dspace-rdf/target/failsafe-reports/*-output.txt
Expand All @@ -67,21 +97,51 @@ definitions:
- dspace-sword/target/failsafe-reports/*-output.txt
- dspace-swordv2/target/failsafe-reports/*-output.txt

- step: &integration-tests-dspace-api
runs-on: self.hosted
name: integration-tests-dspace-api
size: 4x
caches:
- gradle
- gradlewrapper
- maven
script:
- export MAVEN_OPTS="-Xmx4096M"
- mvn clean install license:check -DskipUnitTests=true -Pdspace-rest -DskipITs=true -DskipIntegrationTests=true -P !assembly -B -V
- mvn verify -Dmaven.test.skip=false -pl dspace-api -DskipUnitTests=true -Pdspace-rest -DskipITs=false -Pdspace-rest -DskipIntegrationTests=false -P !assembly -B -V -Dsurefire.rerunFailingTestsCount=2 -Dtest.argLine=-Xmx4096M
artifacts:
- dspace-api/target/surefire-reports/*-output.txt
- dspace-api/target/failsafe-reports/*-output.txt

pipelines:
branches:
'main-cris':
- parallel:
- step: *unit-test-code-checks
- step: *integration-tests
- step: *integration-tests-slow
- step: *integration-tests-dspace-api
'dspace-cris-2023_02_x':
- parallel:
- step: *unit-test-code-checks
- step: *integration-tests
- step: *integration-tests-slow
- step: *integration-tests-dspace-api

pull-requests:
'**':
- parallel:
- step: *unit-test-code-checks
- step: *integration-tests
- step: *integration-tests-slow
- step: *integration-tests-dspace-api
custom:
integration:
- step: *integration-tests
integration-slow:
- step: *integration-tests-slow
integration-dspace-api:
- step: *integration-tests-dspace-api
unit:
- step: *unit-test-code-checks

Original file line number Diff line number Diff line change
Expand Up @@ -1981,7 +1981,9 @@ public List<MetadataValue> getMetadata(Item item, String schema, String element,
List<MetadataValue> dbMetadataValues = item.getMetadata();

List<MetadataValue> fullMetadataValueList = new LinkedList<>();
fullMetadataValueList.addAll(relationshipMetadataService.getRelationshipMetadata(item, true));
if (configurationService.getBooleanProperty("item.enable-virtual-metadata", false)) {
fullMetadataValueList.addAll(relationshipMetadataService.getRelationshipMetadata(item, true));
}
fullMetadataValueList.addAll(dbMetadataValues);

item.setCachedMetadata(MetadataValueComparators.sort(fullMetadataValueList));
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
/**
* The contents of this file are subject to the license and copyright
* detailed in the LICENSE and NOTICE files at the root of the source
* tree and available online at
*
* http://www.dspace.org/license/
*/
package org.dspace.content.dao;

import java.sql.SQLException;
import java.util.UUID;

import org.dspace.core.Context;

public interface ItemForMetadataEnhancementUpdateDAO {

/**
* Add to the metadata_enhancement_update table queue an entry of each items
* that is potentially affected by the update of the Item with the specified
* uuid. The items potentially affected are the one that have the provided uuid
* as value of a cris.virtualsource.* metadata
*
* @param context the DSpace Context object
* @param uuid the uuid of the updated item
* @return the number of affected items scheduled for update
* @throws SQLException if a problem with the database occurs
*/
int saveAffectedItemsForUpdate(Context context, UUID uuid);

/**
* Remove from the metadata_enhancement_update table queue the entry if any
* related to the specified id in older than the current date
*
* @param context the DSpace Context object
* @param itemToRemove the uuid of the processed item
* @throws SQLException if a problem with the database occurs
*/
void removeItemForUpdate(Context context, UUID itemToRemove);

/**
* Extract and remove from the table the first uuid to process from the
* itemupdate_metadata_enhancement table ordered by date queued asc (older
* first)
*
* @param context
* @return
*/
UUID pollItemToUpdate(Context context);
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,152 @@
/**
* The contents of this file are subject to the license and copyright
* detailed in the LICENSE and NOTICE files at the root of the source
* tree and available online at
*
* http://www.dspace.org/license/
*/
package org.dspace.content.dao.impl;

import java.sql.SQLException;
import java.util.UUID;

import org.dspace.content.MetadataSchema;
import org.dspace.content.dao.ItemForMetadataEnhancementUpdateDAO;
import org.dspace.content.factory.ContentServiceFactory;
import org.dspace.content.service.MetadataSchemaService;
import org.dspace.core.Context;
import org.dspace.core.DBConnection;
import org.dspace.services.ConfigurationService;
import org.dspace.utils.DSpace;
import org.hibernate.Session;
import org.hibernate.query.NativeQuery;
import org.springframework.beans.factory.annotation.Autowired;

/**
* Hibernate implementation of the Database Access Object interface class for
* the ItemForMetadataEnhancementUpdate object. This class is responsible for
* all database calls for the ItemForMetadataEnhancementUpdate object and is
* autowired by spring This class should never be accessed directly.
*/
public class ItemForMetadataEnhancementUpdateDAOImpl implements ItemForMetadataEnhancementUpdateDAO {
@Autowired
ConfigurationService configurationService;

@Override
public void removeItemForUpdate(Context context, UUID itemToRemove) {
try {
Session session = getHibernateSession();
String sql = "DELETE FROM itemupdate_metadata_enhancement WHERE uuid = :uuid";
NativeQuery<?> query = session.createNativeQuery(sql);
query.setParameter("uuid", itemToRemove);
query.executeUpdate();
} catch (Exception e) {
throw new RuntimeException(e);
}

}

@Override
public UUID pollItemToUpdate(Context context) {
try {
Session session = getHibernateSession();
String sql = "SELECT cast(uuid as varchar) FROM itemupdate_metadata_enhancement"
+ " ORDER BY date_queued ASC LIMIT 1";
NativeQuery<?> query = session.createNativeQuery(sql);
Object uuidObj = query.uniqueResult();
if (uuidObj != null) {
UUID uuid;
if (uuidObj instanceof String) {
uuid = (UUID) UUID.fromString((String) uuidObj);
} else {
throw new RuntimeException("Unexpected result type from the database " + uuidObj);
}
removeItemForUpdate(context, uuid);
return uuid;
} else {
return null;
}
} catch (Exception e) {
throw new RuntimeException(e);
}

}

@Override
public int saveAffectedItemsForUpdate(Context context, UUID uuid) {
try {
Session session = getHibernateSession();
MetadataSchemaService schemaService = ContentServiceFactory.getInstance().getMetadataSchemaService();
MetadataSchema schema = schemaService.find(context, "cris");
String sqlInsertOrUpdate;
if ("org.h2.Driver".equals(configurationService.getProperty("db.driver"))) {
// H2 doesn't support the INSERT OR UPDATE statement so let's do in two steps
// update queued date for records already in the queue
String sqlUpdate = "UPDATE itemupdate_metadata_enhancement"
+ " SET date_queued = CURRENT_TIMESTAMP"
+ " WHERE uuid IN ("
+ " SELECT dspace_object_id FROM metadatavalue " + " WHERE metadata_field_id IN"
+ " ( SELECT metadata_field_id FROM metadatafieldregistry "
+ " WHERE metadata_schema_id = :schema AND element = 'virtualsource')"
+ " AND SUBSTRING(text_value,1,36) = :uuid)";
String sqlInsert =
"INSERT INTO itemupdate_metadata_enhancement (uuid, date_queued)"
+ " SELECT DISTINCT dspace_object_id, CURRENT_TIMESTAMP FROM metadatavalue "
+ " WHERE metadata_field_id IN"
+ " ( SELECT metadata_field_id FROM metadatafieldregistry "
+ " WHERE metadata_schema_id = :schema AND element = 'virtualsource')"
+ " AND SUBSTRING(text_value,1,36) = :uuid "
+ " AND dspace_object_id NOT IN ("
+ " SELECT uuid"
+ " FROM itemupdate_metadata_enhancement"
+ " )";
NativeQuery<?> queryUpdate = session.createNativeQuery(sqlUpdate);
queryUpdate.setParameter("uuid", uuid.toString());
queryUpdate.setParameter("schema", schema.getID());
queryUpdate.executeUpdate();
NativeQuery<?> queryInsert = session.createNativeQuery(sqlInsert);
queryInsert.setParameter("uuid", uuid.toString());
queryInsert.setParameter("schema", schema.getID());
return queryInsert.executeUpdate();
} else {
sqlInsertOrUpdate = "INSERT INTO itemupdate_metadata_enhancement (uuid, date_queued) "
+ " SELECT DISTINCT dspace_object_id, CURRENT_TIMESTAMP FROM metadatavalue "
+ " WHERE metadata_field_id IN"
+ " ( SELECT metadata_field_id FROM metadatafieldregistry "
+ " WHERE metadata_schema_id = :schema AND element = 'virtualsource')"
+ " AND substring(text_value,1,36) = :uuid "
+ " ON CONFLICT (uuid) DO UPDATE"
+ " SET date_queued = EXCLUDED.date_queued";
NativeQuery<?> queryInsertOrUpdate = session.createNativeQuery(sqlInsertOrUpdate);
queryInsertOrUpdate.setParameter("uuid", uuid.toString());
queryInsertOrUpdate.setParameter("schema", schema.getID());
return queryInsertOrUpdate.executeUpdate();
}
} catch (Exception e) {
throw new RuntimeException(e);
}
}

/**
* The Hibernate Session used in the current thread
*
* @return the current Session.
* @throws SQLException
*/
private Session getHibernateSession() throws SQLException {
DBConnection dbConnection = new DSpace().getServiceManager().getServiceByName(null, DBConnection.class);
return ((Session) dbConnection.getSession());
}

public UUID ConvertByteArrayToUUID(byte[] bytea) {
long mostSigBits = 0;
long leastSigBits = 0;
for (int i = 0; i < 8; i++) {
mostSigBits = (mostSigBits << 8) | (bytea[i] & 0xff);
leastSigBits = (leastSigBits << 8) | (bytea[i + 8] & 0xff);
}

UUID uuid = new UUID(mostSigBits, leastSigBits);
return uuid;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,12 @@

import java.util.HashSet;
import java.util.Set;
import java.util.UUID;

import org.dspace.content.Item;
import org.dspace.content.enhancer.service.ItemEnhancerService;
import org.dspace.content.factory.ContentServiceFactory;
import org.dspace.content.service.ItemService;
import org.dspace.core.Context;
import org.dspace.event.Consumer;
import org.dspace.event.Event;
Expand All @@ -29,10 +32,12 @@
public class ItemEnhancerConsumer implements Consumer {

public static final String ITEMENHANCER_ENABLED = "itemenhancer.enabled";
private Set<Item> itemsAlreadyProcessed = new HashSet<Item>();
private Set<UUID> itemsToProcess = new HashSet<UUID>();

private ItemEnhancerService itemEnhancerService;

private ItemService itemService = ContentServiceFactory.getInstance().getItemService();

private ConfigurationService configurationService = DSpaceServicesFactory.getInstance().getConfigurationService();

@Override
Expand All @@ -53,19 +58,11 @@ public void consume(Context context, Event event) throws Exception {
}

Item item = (Item) event.getSubject(context);
if (item == null || itemsAlreadyProcessed.contains(item) || !item.isArchived()) {
if (item == null || !item.isArchived()) {
return;
}

itemsAlreadyProcessed.add(item);

context.turnOffAuthorisationSystem();
try {
itemEnhancerService.enhance(context, item, false);
} finally {
context.restoreAuthSystemState();
}

itemsToProcess.add(item.getID());
}

protected boolean isConsumerEnabled() {
Expand All @@ -74,7 +71,19 @@ protected boolean isConsumerEnabled() {

@Override
public void end(Context ctx) throws Exception {
itemsAlreadyProcessed.clear();
ctx.turnOffAuthorisationSystem();
try {
for (UUID uuid : itemsToProcess) {
Item item = itemService.find(ctx, uuid);
if (item != null) {
itemEnhancerService.enhance(ctx, item, false);
itemEnhancerService.saveAffectedItemsForUpdate(ctx, item.getID());
}
}
} finally {
ctx.restoreAuthSystemState();
}
itemsToProcess.clear();
}

}
Loading

0 comments on commit 00aa90c

Please sign in to comment.