From c7ef2abe88612ef168a9def9ef8b87684324b91f Mon Sep 17 00:00:00 2001 From: Adam Collins Date: Thu, 11 Apr 2024 14:17:35 +1000 Subject: [PATCH 1/2] #894 fix inclusion of conservation list values in facet download --- .../ala/biocache/service/ListsService.java | 2 +- .../biocache/web/OccurrenceController.java | 236 ++++++++++-------- 2 files changed, 133 insertions(+), 105 deletions(-) diff --git a/src/main/java/au/org/ala/biocache/service/ListsService.java b/src/main/java/au/org/ala/biocache/service/ListsService.java index afa7742f6..d8201326f 100644 --- a/src/main/java/au/org/ala/biocache/service/ListsService.java +++ b/src/main/java/au/org/ala/biocache/service/ListsService.java @@ -123,7 +123,7 @@ private Map> getItemsMap(Map speciesLists, boolean conservat String status = ""; if (item.kvpValues != null) { for (KvpDTO m : item.kvpValues) { - if (m.key != null) { + if (m.key != null && "status".equalsIgnoreCase(m.key)) { status = ": " + m.value; } } diff --git a/src/main/java/au/org/ala/biocache/web/OccurrenceController.java b/src/main/java/au/org/ala/biocache/web/OccurrenceController.java index 680139ec5..7a92a31e7 100644 --- a/src/main/java/au/org/ala/biocache/web/OccurrenceController.java +++ b/src/main/java/au/org/ala/biocache/web/OccurrenceController.java @@ -28,7 +28,6 @@ import au.org.ala.biocache.util.SearchUtils; import au.org.ala.biocache.util.converter.FqField; import au.org.ala.ws.security.profile.AlaUserProfile; -import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.util.concurrent.ThreadFactoryBuilder; import com.nimbusds.jose.util.ArrayUtils; @@ -86,7 +85,9 @@ import java.util.function.Function; import java.util.regex.Pattern; import java.util.stream.Collectors; + import com.fasterxml.jackson.annotation.JsonInclude; + import static au.org.ala.biocache.dto.DuplicateRecordDetails.ASSOCIATED; import static au.org.ala.biocache.dto.DuplicateRecordDetails.REPRESENTATIVE; import static au.org.ala.biocache.dto.OccurrenceIndex.*; @@ -99,10 +100,12 @@ */ @Controller(value = "Occurrence") @JsonInclude(JsonInclude.Include.NON_NULL) -@SecurityScheme(name = "JWT", type = SecuritySchemeType.HTTP, scheme = "bearer", bearerFormat= "JWT") +@SecurityScheme(name = "JWT", type = SecuritySchemeType.HTTP, scheme = "bearer", bearerFormat = "JWT") public class OccurrenceController extends AbstractSecureController { - /** Logger initialisation*/ + /** + * Logger initialisation + */ private static final Logger logger = Logger.getLogger(OccurrenceController.class); public static final String LEGACY_REPRESENTATIVE_RECORD_VALUE = "R"; @@ -129,6 +132,10 @@ public class OccurrenceController extends AbstractSecureController { @Inject protected SpeciesLookupService speciesLookupService; @Inject + protected SpeciesCountsService speciesCountsService; + @Inject + protected SpeciesImageService speciesImageService; + @Inject protected AuthService authService; @Inject protected OccurrenceUtils occurrenceUtils; @@ -270,10 +277,10 @@ Map emptyJson() { } @Secured({"ROLE_ADMIN"}) - @SecurityRequirement(name="JWT") + @SecurityRequirement(name = "JWT") @Operation(summary = "Get list of current downloads", tags = "Monitoring") - @Tag(name="Monitoring", description = "Admin services for monitoring the application, download stats, and index. Protected APIs require administrative role for access.") - @RequestMapping(value = { "/active/download/stats" }, method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE) + @Tag(name = "Monitoring", description = "Admin services for monitoring the application, download stats, and index. Protected APIs require administrative role for access.") + @RequestMapping(value = {"/active/download/stats"}, method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE) public @ResponseBody List getCurrentDownloads() { return downloadService.getCurrentDownloads(); @@ -282,6 +289,7 @@ List getCurrentDownloads() { /** * Returns the default facets that are applied to a search * No longer in use + * * @return */ @Deprecated @@ -300,7 +308,7 @@ String[] listAllFacets() { * @return */ @Operation(summary = "List available facets with grouping", tags = "Search") - @Tag(name="Search", description = "Services for the retrieval of search facets") + @Tag(name = "Search", description = "Services for the retrieval of search facets") @RequestMapping(value = { "/search/grouped/facets" }, method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE) @@ -436,7 +444,7 @@ public int compare(IndexFieldDTO o1, IndexFieldDTO o2) { * @throws Exception */ @Operation(summary = "Download a list of indexed fields", tags = "Download") - @RequestMapping(value = "index/fields.csv", method = RequestMethod.GET, produces = {"text/csv", "text/plain"}) + @RequestMapping(value = "index/fields.csv", method = RequestMethod.GET, produces = {"text/csv", "text/plain"}) public void getIndexedFields( @RequestParam(value = "fl", required = false) String fields, @RequestParam(value = "indexed", required = false) Boolean indexed, @@ -518,7 +526,7 @@ public void getIndexedFields( * @throws Exception */ @Secured({"ROLE_ADMIN"}) - @SecurityRequirement(name="JWT") + @SecurityRequirement(name = "JWT") @Operation(summary = "Show index version information", tags = "Monitoring") @RequestMapping(value = {"index/version"}, method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE) public @ResponseBody @@ -542,22 +550,38 @@ Map getIndexedFields(@RequestParam(value = "force", required = false, defaultVal * @throws Exception */ @Secured({"ROLE_ADMIN"}) - @SecurityRequirement(name="JWT") + @SecurityRequirement(name = "JWT") @Operation(summary = "Show configured max boolean clauses", tags = "Monitoring") @RequestMapping(value = { "index/maxBooleanClauses" }, method = RequestMethod.GET) - public @ResponseBody Map getIndexedFields(){ + public @ResponseBody Map getIndexedFields() { int m = searchDAO.getMaxBooleanClauses(); Map map = new HashMap(); map.put("maxBooleanClauses", m); return map; } + + @Secured({"ala/internal"}) + @SecurityRequirement(name = "JWT") + @Operation(summary = "Export of lft,image", tags = "Monitoring") + @RequestMapping(value = {"index/speciesImages"}, method = RequestMethod.GET) + public @ResponseBody SpeciesImagesDTO getSpeciesImages() { + return speciesImageService.getSpeciesImages(); + } + + @SecurityRequirement(name = "JWT") + @Operation(summary = "Export of lft,count", tags = "Monitoring") + @RequestMapping(value = {"index/speciesOccurrences"}, method = RequestMethod.GET) + public @ResponseBody SpeciesCountDTO getSpeciesCounts() { + return speciesCountsService.getCounts(null); + } + @Secured({"ROLE_ADMIN"}) - @SecurityRequirement(name="JWT", scopes = {"ROLE_ADMIN"}) + @SecurityRequirement(name = "JWT", scopes = {"ROLE_ADMIN"}) @Operation(summary = "Show configuration", tags = "Monitoring", - description = " Public service that reports limits and other useful config for clients." + description = " Public service that reports limits and other useful config for clients." ) @RequestMapping(value = { "config", @@ -585,10 +609,10 @@ Map getIndexedFields(@RequestParam(value = "force", required = false, defaultVal } @Operation(summary = "Get distinct facet counts", tags = "Occurrence", - description="Can be used to retrieve distinct counts in a query. e.g. the distinct number of " + + description = "Can be used to retrieve distinct counts in a query. e.g. the distinct number of " + "scientificName values where stateProvince:Queensland" ) - @Tag(name="Occurrence", description = "Specimen & observation data searching") + @Tag(name = "Occurrence", description = "Specimen & observation data searching") @RequestMapping(value = { "occurrences/facets" }, method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE) @@ -609,10 +633,10 @@ List getOccurrenceFacetDetailsDeprecated(@Valid @ParameterObject } @Operation(summary = "Show a list of images associated with records for a taxon", tags = "Images", - description="Returns a list of image urls for the supplied taxon uuid." + - "An empty list is returned when no images are available." + description = "Returns a list of image urls for the supplied taxon uuid." + + "An empty list is returned when no images are available." ) - @Tag(name="Images", description = "Services for the retrieval of taxon image data") + @Tag(name = "Images", description = "Services for the retrieval of taxon image data") @RequestMapping(value = "/images/taxon/**", method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE) public @ResponseBody List getImages(HttpServletRequest request) throws Exception { @@ -633,7 +657,7 @@ List getImages(HttpServletRequest request) throws Exception { } @Operation(summary = "Checks to see if the supplied GUID represents an native species", tags = "Taxonomy", - description="Checks to see if the supplied GUID represents an native species." + description = "Checks to see if the supplied GUID represents an native species." ) @RequestMapping(value = {"/native/taxon/**"}, method = RequestMethod.GET, @@ -747,7 +771,7 @@ private NativeDTO getIsNativeForGuid(String taxonConceptID) throws Exception { method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE) public @ResponseBody SearchResultDTO occurrenceSearchByTaxon( - @NotNull @PathVariable(name="taxonConceptID") String taxonConceptID, + @NotNull @PathVariable(name = "taxonConceptID") String taxonConceptID, @Valid @ParameterObject SpatialSearchRequestParams requestParams) throws Exception { requestParams.setQ("taxonConceptID:" + taxonConceptID); @@ -795,7 +819,7 @@ List sourceByTaxon(SpatialSearchRequestParams requestParams "/occurrences/dataProviders/{uid}", "/occurrences/dataProviders/{uid}.json", "/occurrences/dataHubs/{uid}", - "/occurrences/dataHubs/{uid}.json" }, method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE) + "/occurrences/dataHubs/{uid}.json"}, method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE) @Deprecated @ApiParam(value = "uid", required = true) public @ResponseBody @@ -823,7 +847,7 @@ private SearchResultDTO occurrenceSearch(SpatialSearchRequestParams requestParam @Deprecated @Operation(summary = "Deprecated - use /occurrences/search", - tags="Deprecated" + tags = "Deprecated" ) @RequestMapping(value = { "/occurrences/search.json*", @@ -846,8 +870,8 @@ SearchResultDTO occurrenceSearchDeprecated( */ @SecurityRequirement(name = "JWT") @Operation(summary = "Occurrence search", - description = "Occurrence search service that supports facets", - tags="Occurrence" + description = "Occurrence search service that supports facets", + tags = "Occurrence" ) @RequestMapping(value = { // "/occurrences/search.json*", @@ -856,10 +880,10 @@ SearchResultDTO occurrenceSearchDeprecated( }, method = {RequestMethod.GET, RequestMethod.POST}, produces = MediaType.APPLICATION_JSON_VALUE) public @ResponseBody SearchResultDTO occurrenceSearch( - @Valid @ParameterObject SpatialSearchRequestParams requestParams, - @Parameter(description = "Include image metadata") - @RequestParam(value = "im", required = false, defaultValue = "false") Boolean lookupImageMetadata, - HttpServletRequest request) throws Exception { + @Valid @ParameterObject SpatialSearchRequestParams requestParams, + @Parameter(description = "Include image metadata") + @RequestParam(value = "im", required = false, defaultValue = "false") Boolean lookupImageMetadata, + HttpServletRequest request) throws Exception { SpatialSearchRequestDTO dto = SpatialSearchRequestDTO.create(requestParams); @@ -874,51 +898,51 @@ SearchResultDTO occurrenceSearch( dto.setPageSize(pageDepthMax - dto.getStart()); } - // handle empty param values, e.g. &sort=&dir= - SearchUtils.setDefaultParams(dto); - Map map = request != null ? SearchUtils.getExtraParams(request.getParameterMap()) : null; - - if (logger.isDebugEnabled()) { - logger.debug("occurrence search params = " + requestParams + " extra params = " + map); - } - - SearchResultDTO srtdto = null; - //FIXME - under what circumstances do we allow sensitive search results.... - if (request.getUserPrincipal() != null && request.isUserInRole("ROLE_ADMIN")) { - // what do we do here - srtdto = searchDAO.findByFulltextSpatialQuery(dto, true, map); - } else { - srtdto = searchDAO.findByFulltextSpatialQuery(dto, false, map); - } - - if (srtdto.getTotalRecords() > 0 && lookupImageMetadata) { - //use the image service API & grab the list of IDs - List occurrenceIDs = new ArrayList(); - for (OccurrenceIndex oi : srtdto.getOccurrences()) { - if (oi.getImages() != null) { - occurrenceIDs.addAll(Arrays.asList(oi.getImages())); - } - } - - Map> imageMap = imageMetadataService.getImageMetadataForOccurrences(occurrenceIDs); - - for (OccurrenceIndex oi : srtdto.getOccurrences()) { - if (oi.getImages() != null) { - for (int i = 0; i < oi.getImages().length; i++) { - Map imageMetadata = imageMap.get(oi.getImages()[i]); - if (oi != null) { - List> md = oi.getImageMetadata(); - if (md == null) { - md = new ArrayList<>(); - } - md.add(imageMetadata); - oi.setImageMetadata(md); - } - } - } - } - } - return srtdto; + // handle empty param values, e.g. &sort=&dir= + SearchUtils.setDefaultParams(dto); + Map map = request != null ? SearchUtils.getExtraParams(request.getParameterMap()) : null; + + if (logger.isDebugEnabled()) { + logger.debug("occurrence search params = " + requestParams + " extra params = " + map); + } + + SearchResultDTO srtdto = null; + //FIXME - under what circumstances do we allow sensitive search results.... + if (request.getUserPrincipal() != null && request.isUserInRole("ROLE_ADMIN")) { + // what do we do here + srtdto = searchDAO.findByFulltextSpatialQuery(dto, true, map); + } else { + srtdto = searchDAO.findByFulltextSpatialQuery(dto, false, map); + } + + if (srtdto.getTotalRecords() > 0 && lookupImageMetadata) { + //use the image service API & grab the list of IDs + List occurrenceIDs = new ArrayList(); + for (OccurrenceIndex oi : srtdto.getOccurrences()) { + if (oi.getImages() != null) { + occurrenceIDs.addAll(Arrays.asList(oi.getImages())); + } + } + + Map> imageMap = imageMetadataService.getImageMetadataForOccurrences(occurrenceIDs); + + for (OccurrenceIndex oi : srtdto.getOccurrences()) { + if (oi.getImages() != null) { + for (int i = 0; i < oi.getImages().length; i++) { + Map imageMetadata = imageMap.get(oi.getImages()[i]); + if (oi != null) { + List> md = oi.getImageMetadata(); + if (md == null) { + md = new ArrayList<>(); + } + md.add(imageMetadata); + oi.setImageMetadata(md); + } + } + } + } + } + return srtdto; } /** @@ -928,8 +952,8 @@ SearchResultDTO occurrenceSearch( * @throws Exception */ @Secured({"ROLE_ADMIN"}) - @SecurityRequirement(name="JWT") - @Operation(summary = "Refresh caches", tags="Monitoring") + @SecurityRequirement(name = "JWT") + @Operation(summary = "Refresh caches", tags = "Monitoring") @RequestMapping(value = {"/cache/refresh"}, method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE) public @ResponseBody String refreshCache() throws Exception { @@ -955,8 +979,8 @@ private void regenerateETag() { } @Operation(summary = "Downloads the complete list of values in the supplied facet", - tags={"Download", "Occurrence"}, - description ="Downloads the complete list of values in the supplied e.g. complete list" + + tags = {"Download", "Occurrence"}, + description = "Downloads the complete list of values in the supplied e.g. complete list" + " of distinct scientificNames matching a query" ) @RequestMapping(value = "/occurrences/facets/download", method = {RequestMethod.GET, RequestMethod.POST}, produces = {"text/csv", "text/plain"}) @@ -1011,26 +1035,26 @@ private void writeExpandedListColumns(File tmp, HttpServletResponse response) th reader.close(); if (all.size() > 1) { - String [] header = all.get(0); + String[] header = all.get(0); int conservationColumn = header.length - 2; int invasiveColumn = header.length - 1; Set conservationFields = new HashSet<>(); Set invasiveFields = new HashSet<>(); - for (int i=1;i downloadUser = authService.getDownloadUser(dto, request); - if (!downloadUser.isPresent()){ + if (!downloadUser.isPresent()) { response.sendError(HttpServletResponse.SC_UNAUTHORIZED, "No authentication"); return null; } @@ -1228,7 +1255,7 @@ public void run() { * @return * @throws Exception */ - @Operation(summary = "Given a list of queries for a single field, return an AJAX response with the qid (cached query id).", tags="Occurrence") + @Operation(summary = "Given a list of queries for a single field, return an AJAX response with the qid (cached query id).", tags = "Occurrence") @RequestMapping(value = "/occurrences/batchSearch", method = RequestMethod.POST, params = "action=Search") public void batchSearch( HttpServletResponse response, @@ -1302,11 +1329,11 @@ private Long getQidForBatchSearch(String listOfNames, String field, String separ /** * Webservice to report the occurrence counts for the supplied list of taxa */ - @Operation(summary = "Report the occurrence counts for the supplied list of taxa", tags="Occurrence") - @RequestMapping(value = { "/occurrences/taxaCount"}, + @Operation(summary = "Report the occurrence counts for the supplied list of taxa", tags = "Occurrence") + @RequestMapping(value = {"/occurrences/taxaCount"}, method = {RequestMethod.POST, RequestMethod.GET}, produces = MediaType.APPLICATION_JSON_VALUE) public @ResponseBody Map occurrenceSpeciesCounts( - @Parameter(description = "taxonConceptIDs, newline separated (by default)") @RequestParam(name="guids") String listOfGuids, + @Parameter(description = "taxonConceptIDs, newline separated (by default)") @RequestParam(name = "guids") String listOfGuids, @FqField @RequestParam(value = "fq", required = false) String[] filterQueries, @RequestParam(defaultValue = "\n") String separator, HttpServletResponse response @@ -1339,7 +1366,7 @@ private Long getQidForBatchSearch(String listOfNames, String field, String separ * 2) API Key and X-Auth-Id - email address retrieved from CAS/Userdetails - email address is ignored... * 3) Email address supplied (Galah) - email address is verified - no sensitive access * 4) Email address supplied and emailOnlyEnabled == false - email is not verified - no sensitive access - * + *

* TODO: implement DownloadController.isAuthorisedSystem method before removing this deprecated service */ @Deprecated @@ -1347,7 +1374,7 @@ private Long getQidForBatchSearch(String listOfNames, String field, String separ @Operation( summary = "Download occurrence service - Synchronous", tags = "Deprecated", - security = @SecurityRequirement(name = "JWT") + security = @SecurityRequirement(name = "JWT") ) @GetMapping(value = "/occurrences/download") public void occurrenceDownload(@Valid @ParameterObject DownloadRequestParams downloadParams, @@ -1419,7 +1446,7 @@ private Double distInMetres(Double lat1, Double lon1, Double lat2, Double lon2) */ @Hidden @Operation(summary = "Utility method for retrieving a list of occurrences", tags = "Deprecated") - @RequestMapping(value = {"/occurrences/nearest", "/occurrences/nearest.json" }, method = RequestMethod.GET) + @RequestMapping(value = {"/occurrences/nearest", "/occurrences/nearest.json"}, method = RequestMethod.GET) @Deprecated public @ResponseBody Map nearestOccurrence(SpatialSearchRequestParams requestParams) throws Exception { @@ -1519,7 +1546,7 @@ Object showOccurrenceNonRestDeprecated(@RequestParam("uuid") String uuid, HttpSe * @throws Exception */ @Operation(summary = "Returns a data structure allowing comparison of verbatim vs interpreted values", tags = "Occurrence") - @RequestMapping( value = {"/occurrences/compare/{recordUuid}"}, method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE) + @RequestMapping(value = {"/occurrences/compare/{recordUuid}"}, method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE) @ApiParam(value = "recordUuid", required = true) public @ResponseBody Object showOccurrence(@PathVariable("recordUuid") String recordUuid, HttpServletResponse response) throws Exception { @@ -1660,7 +1687,7 @@ Object showOccurrence(@PathVariable("recordUuid") String recordUuid, Object showOccurrenceDeprecated(@PathVariable("recordUuid") String recordUuid, @Parameter(description = "Include image metadata") @RequestParam(value = "im", required = false, defaultValue = "false") Boolean im, - HttpServletRequest request, HttpServletResponse response) throws Exception { + HttpServletRequest request, HttpServletResponse response) throws Exception { return showOccurrence(recordUuid, im, request, response); } @@ -1674,19 +1701,19 @@ private Object getOccurrenceInformation(String uuid, Boolean includeImageMetadat SolrDocumentList sdl = null; Boolean includeSensitive = false; - if (!authenticatedUser.isPresent() || authenticatedUser.get().getRoles().isEmpty()){ + if (!authenticatedUser.isPresent() || authenticatedUser.get().getRoles().isEmpty()) { // no authentication SpatialSearchRequestDTO idRequest = createRecirdQuery(uuid); sdl = searchDAO.findByFulltext(idRequest); } else { // do queries with sensitive filters....if no records returned, do without sensitive filters String sensitiveFq = downloadService.getSensitiveFq(authenticatedUser.get().getRoles()); - if (StringUtils.isNotEmpty(sensitiveFq)){ + if (StringUtils.isNotEmpty(sensitiveFq)) { SpatialSearchRequestDTO idRequest = createRecirdQuery(uuid); idRequest.setFq(new String[]{sensitiveFq}); sdl = searchDAO.findByFulltext(idRequest); } - if (sdl == null || sdl.isEmpty()){ + if (sdl == null || sdl.isEmpty()) { SpatialSearchRequestDTO idRequest = createRecirdQuery(uuid); // do query without filter, user doesnt have access idRequest.setFq(new String[]{}); @@ -1774,6 +1801,7 @@ private boolean isSensitive(SolrDocument doc) { /** * Convert a SOLR Document to a simple nested map. + * * @param sd * @return */ From 1f57b85657da7b2b8499afa8c24de405545b6e27 Mon Sep 17 00:00:00 2001 From: Adam Collins Date: Thu, 11 Apr 2024 15:36:20 +1000 Subject: [PATCH 2/2] hopefully a fix DownloadServiceTest concurrency issues --- .../biocache/service/DownloadServiceTest.java | 37 ++++++++++++++----- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/src/test/java/au/org/ala/biocache/service/DownloadServiceTest.java b/src/test/java/au/org/ala/biocache/service/DownloadServiceTest.java index 498fd825e..9bdbb7b91 100644 --- a/src/test/java/au/org/ala/biocache/service/DownloadServiceTest.java +++ b/src/test/java/au/org/ala/biocache/service/DownloadServiceTest.java @@ -24,6 +24,7 @@ import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.context.event.ContextClosedEvent; import org.springframework.context.support.AbstractMessageSource; import org.springframework.context.support.GenericApplicationContext; @@ -449,6 +450,8 @@ public void tearDown() throws Exception { // Ensure we are not stuck on the countdown latch if it failed to be // called in test as expected testLatch.countDown(); + + FileUtils.deleteDirectory(testCacheDir.toFile()); } /** @@ -458,6 +461,7 @@ public void tearDown() throws Exception { @Test public final void testInit() throws Exception { testService.init(); + Thread.sleep(500); } /** @@ -467,6 +471,7 @@ public final void testInit() throws Exception { @Test public final void testOnApplicationEvent() throws Exception { testService.init(); + Thread.sleep(500); // Check that this method completes reliably testService.onApplicationEvent(new ContextClosedEvent(new GenericApplicationContext())); } @@ -478,6 +483,7 @@ public final void testOnApplicationEvent() throws Exception { @Test public final void testAdd() throws Exception { testService.init(); + Thread.sleep(500); testService.add(new DownloadDetailsDTO(new DownloadRequestDTO(), TEST_USER, "::1", "", DownloadType.RECORDS_INDEX)); assertEquals(persistentQueueDAO.getAllDownloads().size(), 1); assertEquals(testService.userExecutors.size(), 1); @@ -490,6 +496,7 @@ public final void testAdd() throws Exception { @Test public final void testCancel() throws Exception { testService.init(); + Thread.sleep(500); DownloadDetailsDTO dd = new DownloadDetailsDTO(new DownloadRequestDTO(), TEST_USER, "::1", "", DownloadType.RECORDS_INDEX); testService.add(dd); @@ -519,6 +526,7 @@ private DownloadRequestDTO getParams(String query){ @Test public final void testCancelSingleUser() throws Exception { testService.init(); + Thread.sleep(500); DownloadDetailsDTO dd = new DownloadDetailsDTO(new DownloadRequestDTO(), TEST_USER, "::1", "", DownloadType.RECORDS_INDEX); testService.add(dd); DownloadDetailsDTO dd2 = new DownloadDetailsDTO(getParams("test2"), TEST_USER,"127.0.0.1", "", DownloadType.FACET); @@ -542,6 +550,7 @@ public final void testCancelSingleUser() throws Exception { @Test public final void testCancelMultipleUser() throws Exception { testService.init(); + Thread.sleep(500); DownloadDetailsDTO dd = new DownloadDetailsDTO(new DownloadRequestDTO(), TEST_USER, "::1", "", DownloadType.RECORDS_INDEX); testService.add(dd); DownloadDetailsDTO dd2 = new DownloadDetailsDTO(getParams("test2"), TEST_USER2,"127.0.0.1", "", DownloadType.FACET); @@ -569,6 +578,7 @@ public final void testDoiApplicationMetadataIsPassedToTheDoiService() throws Exc // Initialisation - if we don't do this the tests will not run. testLatch.countDown(); testService.init(); + Thread.sleep(500); // Setup mocks and stubs - could be in setup but I don't want to interfere with the other tests. DoiService doiService = mock(DoiService.class); @@ -621,6 +631,7 @@ public final void testDoiDTOContainsProfileFullNameWhenDataProfileIsProvded() th // Initialisation - if we don't do this the tests will not run. testLatch.countDown(); testService.init(); + Thread.sleep(500); // Setup mocks and stubs - could be in setup but I don't want to interfere with the other tests. DoiService doiService = mock(DoiService.class); @@ -673,6 +684,7 @@ public final void testDoiDTOContainsNoProfileNameWhenNoneProvided() throws Excep // Initialisation - if we don't do this the tests will not run. testLatch.countDown(); testService.init(); + Thread.sleep(500); // Setup mocks and stubs - could be in setup but I don't want to interfere with the other tests. DoiService doiService = mock(DoiService.class); @@ -719,6 +731,7 @@ public final void testUserAgentPassedToLoggerService() throws Exception { // Initialisation - if we don't do this the tests will not run. testLatch.countDown(); testService.init(); + Thread.sleep(500); // Setup mocks and stubs - could be in setup but I don't want to interfere with the other tests. DoiService doiService = mock(DoiService.class); @@ -758,7 +771,6 @@ public final void testUserAgentPassedToLoggerService() throws Exception { } @Test - @Ignore // One of the other tests puts something into the download.cache.dir so it ends up doing 2 downloads. Unsure how to fix. public final void testOfflineDownload() throws Exception { testService = createDownloadServiceForOfflineTest(); @@ -777,13 +789,8 @@ public final void testOfflineDownload() throws Exception { testService.biocacheDownloadEmailTemplate = "/tmp/download-email.html"; testService.biocacheDownloadReadmeTemplate = "/tmp/readme.txt"; - // delete download cache - File cache = new File("/tmp/cache"); - if (cache.exists()) { - FileUtils.deleteDirectory(cache); - } - testService.init(); + Thread.sleep(500); List emptyDownloads = testService.getCurrentDownloads(); assertEquals(0, emptyDownloads.size()); @@ -798,7 +805,6 @@ public final void testOfflineDownload() throws Exception { } @Test - @Ignore // Fails in travis, works locally public final void testOfflineDownloadWithQualityFiltersAndDoi() throws Exception { testService = createDownloadServiceForOfflineTest(); @@ -818,6 +824,7 @@ public final void testOfflineDownloadWithQualityFiltersAndDoi() throws Exception testService.init(); + Thread.sleep(500); List emptyDownloads = testService.getCurrentDownloads(); assertEquals(0, emptyDownloads.size()); @@ -902,6 +909,7 @@ public final void testOfflineDownloadWithQualityFiltersAndDoiAndProvidedSearchUr testService.biocacheDownloadDoiReadmeTemplate = "/tmp/readme.txt"; testService.init(); + Thread.sleep(500); List emptyDownloads = testService.getCurrentDownloads(); assertEquals(0, emptyDownloads.size()); @@ -972,7 +980,6 @@ public final void testOfflineDownloadWithQualityFiltersAndDoiAndProvidedSearchUr } @Test - @Ignore // One of the other tests puts something into the download.cache.dir so it ends up doing 2 downloads. Unsure how to fix. public final void testOfflineDownloadNoEmailNotify() throws Exception { testService = createDownloadServiceForOfflineTest(); @@ -993,6 +1000,7 @@ public final void testOfflineDownloadNoEmailNotify() throws Exception { } testService.init(); + Thread.sleep(500); List emptyDownloads = testService.getCurrentDownloads(); assertEquals(0, emptyDownloads.size()); @@ -1024,6 +1032,17 @@ public final void testDataQualityResourceTemplate() throws Exception { } private DownloadService createDownloadServiceForOfflineTest() { + // empty cached download directory, in case it is messing up tests + try { + File dir = new File(testCacheDir.toAbsolutePath().toString()); + if (dir.exists() && dir.isDirectory()) { + for (File f : dir.listFiles()) { + f.delete(); + } + } + } catch (Exception ignored) { + } + DownloadService testService = new DownloadService() { { sensitiveAccessRoles20 = "{}";