Adds Search across all databases

deploys/docker-compose-dev.yml

@@ -18,6 +18,7 @@ services:
  environment:
  ZK_HOST: zoo:2181
  SOLR_HOST: localhost
+ SOLR_HEAP: 4G
  SOLR_OPTS: "-Dsolr.environment=dev,label=DEV+ENV"
  depends_on:
  - zoo

src/main/java/com/databasepreservation/DBVTK.java

@@ -8,6 +8,7 @@
 package com.databasepreservation;
 
 import java.io.IOException;
+import java.net.http.HttpClient;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
@@ -261,6 +262,8 @@ public void addResourceHandlers(ResourceHandlerRegistry registry) {
  }
  }
 
+
+
  // @Bean
  // MultipartConfigElement multipartConfigElement() {
  // MultipartConfigFactory factory = new MultipartConfigFactory();

src/main/java/com/databasepreservation/common/api/v1/CollectionResource.java

@@ -195,7 +195,9 @@ public StringResponse createCollection(String databaseUUID) {
 
  try {
  final ViewerDatabase database = ViewerFactory.getSolrManager().retrieve(ViewerDatabase.class, databaseUUID);
- return new StringResponse(SIARDController.loadFromLocal(database.getPath(), databaseUUID));
+ StringResponse collection = new StringResponse(SIARDController.loadFromLocal(database.getPath(), databaseUUID));
+
+ return collection;
  } catch (GenericException | NotFoundException e) {
  state = LogEntryState.FAILURE;
  throw new RESTException(e);

src/main/java/com/databasepreservation/common/api/v1/DatabaseResource.java

@@ -7,10 +7,14 @@
  */
 package com.databasepreservation.common.api.v1;
 
+import java.io.IOException;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 
+import org.apache.solr.client.solrj.SolrServerException;
 import org.roda.core.data.exceptions.GenericException;
 import org.roda.core.data.exceptions.NotFoundException;
 import org.roda.core.data.exceptions.RequestNotValidException;
@@ -24,11 +28,17 @@
 import com.databasepreservation.common.client.exceptions.RESTException;
 import com.databasepreservation.common.client.index.FindRequest;
 import com.databasepreservation.common.client.index.IndexResult;
+import com.databasepreservation.common.client.index.facets.FacetFieldResult;
+import com.databasepreservation.common.client.index.facets.FacetParameter;
+import com.databasepreservation.common.client.index.facets.FacetValue;
+import com.databasepreservation.common.client.index.facets.Facets;
+import com.databasepreservation.common.client.index.facets.SimpleFacetParameter;
 import com.databasepreservation.common.client.index.filter.AndFiltersParameters;
 import com.databasepreservation.common.client.index.filter.Filter;
 import com.databasepreservation.common.client.index.filter.FilterParameter;
 import com.databasepreservation.common.client.index.filter.OrFiltersParameters;
 import com.databasepreservation.common.client.index.filter.SimpleFilterParameter;
+import com.databasepreservation.common.client.index.sort.Sorter;
 import com.databasepreservation.common.client.models.activity.logs.LogEntryState;
 import com.databasepreservation.common.client.models.status.database.DatabaseStatus;
 import com.databasepreservation.common.client.models.structure.ViewerDatabase;
@@ -39,6 +49,8 @@
 import com.databasepreservation.common.server.ViewerConfiguration;
 import com.databasepreservation.common.server.ViewerFactory;
 import com.databasepreservation.common.server.controller.SIARDController;
+import com.databasepreservation.common.server.index.utils.IterableDatabaseResult;
+import com.databasepreservation.common.server.index.utils.SolrUtils;
 import com.databasepreservation.common.utils.ControllerAssistant;
 import com.databasepreservation.common.utils.UserUtility;
 
@@ -69,15 +81,43 @@ public IndexResult<ViewerDatabase> find(FindRequest findRequest, String localeSt
  fieldsToReturn.add(ViewerConstants.SOLR_DATABASES_METADATA);
  fieldsToReturn.add(ViewerConstants.SOLR_DATABASES_PERMISSIONS);
 
- FindRequest request = new FindRequest(findRequest.classToReturn, getDatabaseFilterForUser(user),
+ FindRequest userFindRequest = new FindRequest(findRequest.classToReturn, getDatabaseFilterForUser(user),
  findRequest.sorter, findRequest.sublist, findRequest.facets, findRequest.exportFacets, fieldsToReturn);
- return getViewerDatabaseIndexResult(request, fieldsToReturn, controllerAssistant, user, state);
+ return getViewerDatabaseIndexResult(userFindRequest, fieldsToReturn, controllerAssistant, user, state);
  }
  } else {
  return getViewerDatabaseIndexResult(findRequest, controllerAssistant, user, state);
  }
  }
 
+ @Override
+ public IndexResult<ViewerDatabase> findAll(FindRequest findRequest, String localeString) {
+ ControllerAssistant controllerAssistant = new ControllerAssistant() {};
+
+ LogEntryState state = LogEntryState.SUCCESS;
+ User user = controllerAssistant.checkRoles(request);
+
+ if (ViewerConfiguration.getInstance().getApplicationEnvironment().equals(ViewerConstants.APPLICATION_ENV_SERVER)) {
+ if (user.isAdmin() || user.isWhiteList()) {
+
+ return getCrossViewerDatabaseIndexResult(findRequest, controllerAssistant, user, state);
+ } else {
+ List<String> fieldsToReturn = new ArrayList<>();
+ fieldsToReturn.add(ViewerConstants.INDEX_ID);
+ fieldsToReturn.add(ViewerConstants.SOLR_DATABASES_STATUS);
+ fieldsToReturn.add(ViewerConstants.SOLR_DATABASES_METADATA);
+ fieldsToReturn.add(ViewerConstants.SOLR_DATABASES_PERMISSIONS);
+
+ FindRequest userFindRequest = new FindRequest(findRequest.classToReturn, getDatabaseFilterForUser(user),
+ findRequest.sorter, findRequest.sublist, findRequest.facets, findRequest.exportFacets, fieldsToReturn);
+ return getCrossViewerDatabaseIndexResult(userFindRequest, controllerAssistant, user, state,
+ getDatabaseFilterForUser(user));
+ }
+ } else {
+ return getCrossViewerDatabaseIndexResult(findRequest, controllerAssistant, user, state);
+ }
+ }
+
  @Override
  public StringResponse create(String path) {
  final ControllerAssistant controllerAssistant = new ControllerAssistant() {};
@@ -134,6 +174,82 @@ private IndexResult<ViewerDatabase> getViewerDatabaseIndexResult(FindRequest fin
  }
  }
 
+ private IndexResult<ViewerDatabase> getCrossViewerDatabaseIndexResult(FindRequest findRequest,
+ ControllerAssistant controllerAssistant, User user, LogEntryState state) {
+ return getCrossViewerDatabaseIndexResult(findRequest, controllerAssistant, user, state, new Filter());
+ }
+
+ private IndexResult<ViewerDatabase> getCrossViewerDatabaseIndexResult(FindRequest findRequest,
+ ControllerAssistant controllerAssistant, User user, LogEntryState state, Filter userFilter) {
+ long count = 0;
+ try {
+ IterableDatabaseResult<ViewerDatabase> databases = ViewerFactory.getSolrManager().findAll(ViewerDatabase.class,
+ userFilter, Sorter.NONE, findRequest.fieldsToReturn);
+
+ if (databases.getTotalCount() == 0) {
+ return new IndexResult<>();
+ }
+
+ // Search on all collections
+ List<String> collections = new ArrayList<>();
+ Map<String, ViewerDatabase> databaseMap = new HashMap<>();
+ for (ViewerDatabase database : databases) {
+ databaseMap.put(database.getUuid(), database);
+ // only add the available collections
+ if(database.getStatus().equals(ViewerDatabaseStatus.AVAILABLE)){
+ String collectionName = ViewerConstants.SOLR_INDEX_ROW_COLLECTION_NAME_PREFIX + database.getUuid();
+ collections.add(collectionName);
+ }
+ }
+
+ String collectionAlias = SolrUtils.crateSearchAllAlias(ViewerFactory.getSolrClient(), "alias-" + user.getUUID(),
+ collections);
+
+ SimpleFacetParameter simpleFacetParameter = new SimpleFacetParameter(ViewerConstants.SOLR_ROWS_DATABASE_UUID,
+ FacetParameter.SORT.COUNT);
+ simpleFacetParameter.setMinCount(1);
+ simpleFacetParameter.setLimit(findRequest.sublist.getMaximumElementCount());
+ simpleFacetParameter.setOffset(findRequest.sublist.getFirstElementIndex());
+
+ final IndexResult<ViewerDatabase> facetsSearch = ViewerFactory.getSolrManager().findHits(ViewerDatabase.class,
+ collectionAlias, findRequest.filter, findRequest.sorter, findRequest.sublist, new Facets(simpleFacetParameter));
+ count = facetsSearch.getTotalCount();
+ FacetFieldResult facetResults = facetsSearch.getFacetResults().get(0);
+
+ IndexResult<ViewerDatabase> searchHitsResult = new IndexResult<>();
+ if(facetResults.getValues().size() < findRequest.sublist.getMaximumElementCount() && findRequest.sublist.getFirstElementIndex() == 0) {
+ searchHitsResult.setTotalCount(facetResults.getValues().size());
+ } else {
+ searchHitsResult.setTotalCount(-1);
+ }
+ searchHitsResult.setLimit(findRequest.sublist.getMaximumElementCount());
+ searchHitsResult.setOffset(findRequest.sublist.getFirstElementIndex());
+ List<ViewerDatabase> resultsFromFacet = new ArrayList<>();
+
+ // Retrieve the databases HITs
+ for (FacetValue value : facetResults.getValues()) {
+ String databaseUUID = value.getValue();
+ long searchHits = value.getCount();
+
+ ViewerDatabase vd = databaseMap.get(databaseUUID);
+ vd.setSearchHits(searchHits);
+ resultsFromFacet.add(vd);
+ }
+
+ searchHitsResult.setResults(resultsFromFacet);
+ return searchHitsResult;
+
+ } catch (GenericException | RequestNotValidException | SolrServerException | IOException e) {
+ state = LogEntryState.FAILURE;
+ throw new RESTException(e);
+ } finally {
+ // register action
+ controllerAssistant.registerAction(user, state, ViewerConstants.CONTROLLER_FILTER_PARAM,
+ JsonUtils.getJsonFromObject(findRequest.filter), ViewerConstants.CONTROLLER_SUBLIST_PARAM,
+ JsonUtils.getJsonFromObject(findRequest.sublist), ViewerConstants.CONTROLLER_RETRIEVE_COUNT, count);
+ }
+ }
+
  private Filter getDatabaseFilterForUser(User user) {
  ArrayList<FilterParameter> permissionFilterParameters = new ArrayList<>();
  // Only retrieve databases with AVAILABLE status

src/main/java/com/databasepreservation/common/client/ViewerConstants.java

@@ -177,6 +177,9 @@ public class ViewerConstants {
 
  public static final String ACTIVITY_LOG_PROPERTY = "activityLogEntry";
 
+ public static final String UI_LISTS_PROPERTY = "ui.lists";
+ public static final String UI_LISTS_SEARCH_SELECTEDINFO_LABEL_DEFAULT_I18N_PROPERTY = "search.selectedInfo.label.default.i18n";
+
  public static final String LISTS_PROPERTY = "lists";
 
  public static final String LISTS_FACETS_QUERY_PROPERTY = "facets.query";
@@ -210,6 +213,7 @@ public class ViewerConstants {
  /*
  * ROW FIELDS
  */
+ public static final String SOLR_ROWS_DATABASE_UUID = "databaseUUID";
  public static final String SOLR_ROWS_TABLE_ID = "tableId";
  public static final String SOLR_ROWS_TABLE_UUID = "tableUUID";
  public static final String SOLR_ROWS_NESTED_UUID = "nestedUUID";