wip - search on all db

deploys/docker-compose-dev.yml

@@ -18,6 +18,7 @@ services:
  environment:
  ZK_HOST: zoo:2181
  SOLR_HOST: localhost
+ SOLR_HEAP: 4G
  SOLR_OPTS: "-Dsolr.environment=dev,label=DEV+ENV"
  depends_on:
  - zoo

src/main/java/com/databasepreservation/DBVTK.java

@@ -8,6 +8,7 @@
 package com.databasepreservation;
 
 import java.io.IOException;
+import java.net.http.HttpClient;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
@@ -261,6 +262,8 @@ public void addResourceHandlers(ResourceHandlerRegistry registry) {
  }
  }
 
+
+
  // @Bean
  // MultipartConfigElement multipartConfigElement() {
  // MultipartConfigFactory factory = new MultipartConfigFactory();

src/main/java/com/databasepreservation/common/api/v1/CollectionResource.java

@@ -195,7 +195,9 @@ public StringResponse createCollection(String databaseUUID) {
 
  try {
  final ViewerDatabase database = ViewerFactory.getSolrManager().retrieve(ViewerDatabase.class, databaseUUID);
- return new StringResponse(SIARDController.loadFromLocal(database.getPath(), databaseUUID));
+ StringResponse collection = new StringResponse(SIARDController.loadFromLocal(database.getPath(), databaseUUID));
+
+ return collection;
  } catch (GenericException | NotFoundException e) {
  state = LogEntryState.FAILURE;
  throw new RESTException(e);

src/main/java/com/databasepreservation/common/api/v1/DatabaseResource.java

@@ -8,7 +8,9 @@
 package com.databasepreservation.common.api.v1;
 
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 
 import org.roda.core.data.exceptions.GenericException;
@@ -24,11 +26,17 @@
 import com.databasepreservation.common.client.exceptions.RESTException;
 import com.databasepreservation.common.client.index.FindRequest;
 import com.databasepreservation.common.client.index.IndexResult;
+import com.databasepreservation.common.client.index.facets.FacetFieldResult;
+import com.databasepreservation.common.client.index.facets.FacetParameter;
+import com.databasepreservation.common.client.index.facets.FacetValue;
+import com.databasepreservation.common.client.index.facets.Facets;
+import com.databasepreservation.common.client.index.facets.SimpleFacetParameter;
 import com.databasepreservation.common.client.index.filter.AndFiltersParameters;
 import com.databasepreservation.common.client.index.filter.Filter;
 import com.databasepreservation.common.client.index.filter.FilterParameter;
 import com.databasepreservation.common.client.index.filter.OrFiltersParameters;
 import com.databasepreservation.common.client.index.filter.SimpleFilterParameter;
+import com.databasepreservation.common.client.index.sort.Sorter;
 import com.databasepreservation.common.client.models.activity.logs.LogEntryState;
 import com.databasepreservation.common.client.models.status.database.DatabaseStatus;
 import com.databasepreservation.common.client.models.structure.ViewerDatabase;
@@ -39,6 +47,7 @@
 import com.databasepreservation.common.server.ViewerConfiguration;
 import com.databasepreservation.common.server.ViewerFactory;
 import com.databasepreservation.common.server.controller.SIARDController;
+import com.databasepreservation.common.server.index.utils.IterableDatabaseResult;
 import com.databasepreservation.common.utils.ControllerAssistant;
 import com.databasepreservation.common.utils.UserUtility;
 
@@ -62,7 +71,10 @@ public IndexResult<ViewerDatabase> find(FindRequest findRequest, String localeSt
 
  if (ViewerConfiguration.getInstance().getApplicationEnvironment().equals(ViewerConstants.APPLICATION_ENV_SERVER)) {
  if (user.isAdmin() || user.isWhiteList()) {
- return getViewerDatabaseIndexResult(findRequest, controllerAssistant, user, state);
+
+ return getAllViewerDatabaseIndexResult(findRequest, controllerAssistant, user, state);
+ // return getViewerDatabaseIndexResult(findRequest, controllerAssistant, user,
+ // state);
  } else {
  List<String> fieldsToReturn = new ArrayList<>();
  fieldsToReturn.add(ViewerConstants.INDEX_ID);
@@ -115,6 +127,69 @@ private IndexResult<ViewerDatabase> getViewerDatabaseIndexResult(FindRequest fin
  }
  }
 
+ private IndexResult<ViewerDatabase> getAllViewerDatabaseIndexResult(FindRequest findRequest,
+ ControllerAssistant controllerAssistant, User user, LogEntryState state) {
+ long count = 0;
+ try {
+
+ // List databases
+ Filter availableFilter = new Filter(
+ new SimpleFilterParameter(ViewerConstants.SOLR_DATABASES_STATUS, ViewerDatabaseStatus.AVAILABLE.name()));
+
+ IterableDatabaseResult<ViewerDatabase> databases = ViewerFactory.getSolrManager().findAll(ViewerDatabase.class,
+ availableFilter, Sorter.NONE, findRequest.fieldsToReturn);
+
+ // Search on all collections
+ List<String> collections = new ArrayList<>();
+ Map<String, ViewerDatabase> databaseMap = new HashMap<>();
+ for (ViewerDatabase database : databases) {
+ String collectionName = ViewerConstants.SOLR_INDEX_ROW_COLLECTION_NAME_PREFIX + database.getUuid();
+ collections.add(collectionName);
+ databaseMap.put(database.getUuid(), database);
+ }
+
+ SimpleFacetParameter simpleFacetParameter = new SimpleFacetParameter(ViewerConstants.SOLR_ROWS_DATABASE_UUID,
+ FacetParameter.SORT.COUNT);
+ simpleFacetParameter.setMinCount(1);
+ simpleFacetParameter.setLimit(15);
+ // set offset 0
+ Facets facets = new Facets(simpleFacetParameter);
+
+ final IndexResult<ViewerDatabase> result = ViewerFactory.getSolrManager().find(ViewerDatabase.class, collections,
+ findRequest.filter, findRequest.sorter, findRequest.sublist, facets, findRequest.fieldsToReturn);
+ count = result.getTotalCount();
+
+ IndexResult<ViewerDatabase> result2 = new IndexResult<>();
+ result2.setTotalCount(-1);
+ result2.setLimit(100);
+ result2.setOffset(findRequest.sublist.getFirstElementIndex());
+ List<ViewerDatabase> resultsFromFacet = new ArrayList<>();
+ result2.setResults(resultsFromFacet);
+ // Retrieve the databases HITs
+
+ for (FacetFieldResult facetResult : result.getFacetResults()) {
+ for (FacetValue value : facetResult.getValues()) {
+ String databaseUUID = value.getValue();
+ long searchHits = value.getCount();
+
+ ViewerDatabase vd = databaseMap.get(databaseUUID);
+ vd.setSearchHits(searchHits);
+ resultsFromFacet.add(vd);
+ }
+ }
+ return result2;
+
+ } catch (GenericException | RequestNotValidException e) {
+ state = LogEntryState.FAILURE;
+ throw new RESTException(e);
+ } finally {
+ // register action
+ controllerAssistant.registerAction(user, state, ViewerConstants.CONTROLLER_FILTER_PARAM,
+ JsonUtils.getJsonFromObject(findRequest.filter), ViewerConstants.CONTROLLER_SUBLIST_PARAM,
+ JsonUtils.getJsonFromObject(findRequest.sublist), ViewerConstants.CONTROLLER_RETRIEVE_COUNT, count);
+ }
+ }
+
  private IndexResult<ViewerDatabase> getViewerDatabaseIndexResult(FindRequest findRequest, List<String> fieldsToReturn,
  ControllerAssistant controllerAssistant, User user, LogEntryState state) {
  long count = 0;

src/main/java/com/databasepreservation/common/client/ViewerConstants.java

@@ -210,6 +210,7 @@ public class ViewerConstants {
  /*
  * ROW FIELDS
  */
+ public static final String SOLR_ROWS_DATABASE_UUID = "databaseUUID";
  public static final String SOLR_ROWS_TABLE_ID = "tableId";
  public static final String SOLR_ROWS_TABLE_UUID = "tableUUID";
  public static final String SOLR_ROWS_NESTED_UUID = "nestedUUID";

src/main/java/com/databasepreservation/common/client/models/structure/ViewerDatabase.java

@@ -39,6 +39,8 @@ public class ViewerDatabase extends IsIndexed {
  private String validationSkipped;
  private Set<String> permissions;
 
+ private long searchHits = -1;
+
  public ViewerDatabase() {
  }
 
@@ -216,4 +218,13 @@ public Set<String> getPermissions() {
  public void setPermissions(Set<String> permissions) {
  this.permissions = permissions;
  }
+
+
+ public long getSearchHits() {
+ return searchHits;
+ }
+
+ public void setSearchHits(long searchHits) {
+ this.searchHits = searchHits;
+ }
 }

src/main/java/com/databasepreservation/common/client/models/structure/ViewerRow.java

@@ -24,6 +24,7 @@ public class ViewerRow extends IsIndexed {
  @Serial
  private static final long serialVersionUID = 5337660979887460340L;
  private String UUID;
+ private String databaseUUID;
  private String tableUUID;
  private String tableId;
  private Map<String, ViewerCell> cells;
@@ -51,6 +52,14 @@ public void setUuid(String UUID) {
  this.UUID = UUID;
  }
 
+ public String getDatabaseUUID() {
+ return databaseUUID;
+ }
+
+ public void setDatabaseUUID(String databaseUUID) {
+ this.databaseUUID = databaseUUID;
+ }
+
  /**
  * @return Map of solrColumnName to cell value as string
  */

src/main/java/com/databasepreservation/common/server/index/DatabaseRowsSolrManager.java

@@ -17,14 +17,12 @@
 import java.util.Map;
 import java.util.Set;
 
-import com.databasepreservation.common.server.index.utils.IterableDatabaseResult;
 import org.apache.solr.client.solrj.SolrClient;
 import org.apache.solr.client.solrj.SolrQuery;
 import org.apache.solr.client.solrj.SolrRequest;
 import org.apache.solr.client.solrj.SolrServerException;
 import org.apache.solr.client.solrj.request.CollectionAdminRequest;
 import org.apache.solr.client.solrj.response.UpdateResponse;
-import org.apache.solr.common.SolrDocument;
 import org.apache.solr.common.SolrException;
 import org.apache.solr.common.SolrInputDocument;
 import org.apache.solr.common.SolrInputField;
@@ -61,6 +59,7 @@
 import com.databasepreservation.common.server.index.schema.SolrDefaultCollectionRegistry;
 import com.databasepreservation.common.server.index.schema.SolrRowsCollectionRegistry;
 import com.databasepreservation.common.server.index.schema.collections.RowsCollection;
+import com.databasepreservation.common.server.index.utils.IterableDatabaseResult;
 import com.databasepreservation.common.server.index.utils.IterableIndexResult;
 import com.databasepreservation.common.server.index.utils.IterableNestedIndexResult;
 import com.databasepreservation.common.server.index.utils.JsonTransformer;
@@ -182,6 +181,13 @@ public <T extends IsIndexed> IndexResult<T> find(Class<T> classToReturn, Filter
  fieldsToReturn, new HashMap<>());
  }
 
+ public <T extends IsIndexed> IndexResult<T> find(Class<T> classToReturn, List<String> collections, Filter filter,
+ Sorter sorter, Sublist sublist, Facets facets, List<String> fieldsToReturn)
+ throws GenericException, RequestNotValidException {
+ return SolrUtils.find(client, SolrDefaultCollectionRegistry.get(classToReturn), collections, filter, sorter,
+ sublist, facets, fieldsToReturn, new HashMap<>());
+ }
+
  public <T extends IsIndexed> IndexResult<T> find(Class<T> classToReturn, Filter filter, Sorter sorter,
  Sublist sublist, Facets facets) throws GenericException, RequestNotValidException {
  return find(classToReturn, filter, sorter, sublist, facets, new ArrayList<>());

src/main/java/com/databasepreservation/common/server/index/schema/collections/RowsCollection.java

@@ -8,6 +8,7 @@
 package com.databasepreservation.common.server.index.schema.collections;
 
 import static com.databasepreservation.common.client.ViewerConstants.SOLR_INDEX_ROW_COLLECTION_NAME_PREFIX;
+import static com.databasepreservation.common.client.ViewerConstants.SOLR_ROWS_DATABASE_UUID;
 import static com.databasepreservation.common.client.ViewerConstants.SOLR_ROWS_NESTED_TABLE_ID;
 import static com.databasepreservation.common.client.ViewerConstants.SOLR_ROWS_TABLE_ID;
 import static com.databasepreservation.common.client.ViewerConstants.SOLR_ROWS_TABLE_UUID;
@@ -72,6 +73,7 @@ public List<CopyField> getCopyFields() {
  public List<Field> getFields() {
  List<Field> fields = new ArrayList<>(super.getFields());
 
+ fields.add(new Field(SOLR_ROWS_DATABASE_UUID, Field.TYPE_STRING).setIndexed(true).setStored(true));
  fields.add(new Field(SOLR_ROWS_TABLE_ID, Field.TYPE_STRING).setIndexed(true).setStored(true));
  fields.add(new Field(SOLR_ROWS_TABLE_UUID, Field.TYPE_STRING).setIndexed(true).setStored(true));
 
@@ -88,6 +90,7 @@ public SolrInputDocument toSolrDocument(ViewerRow row) throws ViewerException, R
 
  SolrInputDocument doc = super.toSolrDocument(row);
 
+ doc.setField(ViewerConstants.SOLR_ROWS_DATABASE_UUID, row.getDatabaseUUID());
  doc.setField(ViewerConstants.SOLR_ROWS_TABLE_ID, row.getTableId());
  doc.setField(SOLR_ROWS_TABLE_UUID, row.getTableUUID());
  for (Map.Entry<String, ViewerCell> cellEntry : row.getCells().entrySet()) {
@@ -120,6 +123,7 @@ public SolrInputDocument toSolrDocument(ViewerRow row) throws ViewerException, R
  public ViewerRow fromSolrDocument(SolrDocument doc) throws ViewerException {
  ViewerRow viewerRow = super.fromSolrDocument(doc);
 
+ viewerRow.setDatabaseUUID(SolrUtils.objectToString(doc.get(ViewerConstants.SOLR_ROWS_DATABASE_UUID), null));
  viewerRow.setTableId(SolrUtils.objectToString(doc.get(ViewerConstants.SOLR_ROWS_TABLE_ID), null));
  viewerRow.setTableUUID(SolrUtils.objectToString(doc.get(ViewerConstants.SOLR_ROWS_TABLE_UUID), null));
  viewerRow.setNestedUUID(SolrUtils.objectToString(doc.get(ViewerConstants.SOLR_ROWS_NESTED_UUID), null));
@@ -159,6 +163,7 @@ public ViewerRow fromSolrDocument(SolrDocument doc) throws ViewerException {
  private ViewerRow populateNestedRow(SolrDocument doc) throws ViewerException {
  ViewerRow nestedRow = super.fromSolrDocument(doc);
 
+ nestedRow.setDatabaseUUID(SolrUtils.objectToString(doc.get(SOLR_ROWS_DATABASE_UUID), null));
  nestedRow.setTableId(SolrUtils.objectToString(doc.get(SOLR_ROWS_TABLE_ID), null));
  nestedRow.setTableUUID(SolrUtils.objectToString(doc.get(SOLR_ROWS_TABLE_UUID), null));
  nestedRow.setTableId(SolrUtils.objectToString(doc.get(SOLR_ROWS_NESTED_TABLE_ID), null));

src/main/java/com/databasepreservation/common/server/index/utils/SolrAlias.java

@@ -0,0 +1,60 @@
+package com.databasepreservation.common.server.index.utils;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import java.io.Serial;
+import java.io.Serializable;
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * @author Gabriel Barros <[email protected]>
+ */
+public class SolrAlias implements Serializable {
+ @Serial
+ private static final long serialVersionUID = -5247919236401053565L;
+
+ @JsonProperty("name")
+ private String name;
+ @JsonProperty("collections")
+ private List<String> collections;
+
+ public SolrAlias() {
+ }
+
+ public SolrAlias(String aliasName, List<String> collections) {
+ this.name = aliasName;
+ this.collections = collections;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public List<String> getCollections() {
+ return collections;
+ }
+
+ public void setCollections(List<String> collections) {
+ this.collections = collections;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o)
+ return true;
+ if (o == null || getClass() != o.getClass())
+ return false;
+ SolrAlias solrAlias = (SolrAlias) o;
+ return Objects.equals(name, solrAlias.name) && Objects.equals(collections, solrAlias.collections);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(name, collections);
+ }
+}