From 721b6642fb56a86be62831dc2ce7485f8dbf9722 Mon Sep 17 00:00:00 2001 From: Guillaume Poirier-Morency Date: Mon, 6 Jun 2022 14:26:01 -0700 Subject: [PATCH 1/4] Update versions for hotfix --- gemma-core/pom.xml | 2 +- gemma-web/pom.xml | 2 +- pom.xml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/gemma-core/pom.xml b/gemma-core/pom.xml index a82b774fa1..3eb9f1ee2c 100644 --- a/gemma-core/pom.xml +++ b/gemma-core/pom.xml @@ -3,7 +3,7 @@ gemma gemma - 1.27.14 + 1.27.15 4.0.0 gemma-core diff --git a/gemma-web/pom.xml b/gemma-web/pom.xml index dd2d751b11..af0e1624fd 100644 --- a/gemma-web/pom.xml +++ b/gemma-web/pom.xml @@ -3,7 +3,7 @@ gemma gemma - 1.27.14 + 1.27.15 4.0.0 gemma-web diff --git a/pom.xml b/pom.xml index c8346eb57b..fccc38ccbb 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ Gemma gemma gemma - 1.27.14 + 1.27.15 2005 The Gemma Project for meta-analysis of genomics data https://gemma.msl.ubc.ca From 264ce1cc94cdcd593bdd776870092b9ee865d8f1 Mon Sep 17 00:00:00 2001 From: Guillaume Poirier-Morency Date: Mon, 6 Jun 2022 10:22:18 -0700 Subject: [PATCH 2/4] Add getDatasetRawExpression endpoint --- .../service/ExpressionDataFileService.java | 15 +++++++ .../ExpressionDataFileServiceImpl.java | 18 ++++++++ .../service/ExpressionDataMatrixService.java | 8 ++++ .../ExpressionDataMatrixServiceImpl.java | 41 +++++++++++++++++-- .../experiment/ExpressionExperimentDao.java | 2 + .../ExpressionExperimentDaoImpl.java | 8 ++++ .../ExpressionExperimentService.java | 3 ++ .../ExpressionExperimentServiceImpl.java | 6 +++ .../web/services/rest/DatasetsWebService.java | 36 +++++++++++++--- .../web/services/rest/DatasetsRestTest.java | 20 +++++++++ 10 files changed, 148 insertions(+), 9 deletions(-) diff --git a/gemma-core/src/main/java/ubic/gemma/core/analysis/service/ExpressionDataFileService.java b/gemma-core/src/main/java/ubic/gemma/core/analysis/service/ExpressionDataFileService.java index 29cf489328..e33d24faa5 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/analysis/service/ExpressionDataFileService.java +++ b/gemma-core/src/main/java/ubic/gemma/core/analysis/service/ExpressionDataFileService.java @@ -25,6 +25,7 @@ import java.io.File; import java.io.IOException; +import java.io.Writer; import java.util.Collection; import java.util.List; import java.util.Map; @@ -120,6 +121,20 @@ List analysisResultSetToString( Expression File writeDataFile( ExpressionExperiment ee, boolean filtered, String fileName, boolean compress ) throws IOException; + /** + * Write raw expression data to a given writer. + * + * Note: the preferred quantitations are used. + * + * Note: For compression, wrap a {@link java.util.zip.GZIPOutputStream} with a {@link java.io.OutputStreamWriter}. + * To write to a string, consider using {@link java.io.StringWriter}. + * + * @param ee the expression experiment + * @param writer the destination for the raw expression data + * @throws IOException if operations with the writer fails + */ + void writeRawExpressionData( ExpressionExperiment ee, Writer writer ) throws IOException; + /** * Write or located the coexpression data file for a given experiment * diff --git a/gemma-core/src/main/java/ubic/gemma/core/analysis/service/ExpressionDataFileServiceImpl.java b/gemma-core/src/main/java/ubic/gemma/core/analysis/service/ExpressionDataFileServiceImpl.java index 8e95ea83e5..38c296df6a 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/analysis/service/ExpressionDataFileServiceImpl.java +++ b/gemma-core/src/main/java/ubic/gemma/core/analysis/service/ExpressionDataFileServiceImpl.java @@ -25,6 +25,7 @@ import org.apache.commons.logging.LogFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Transactional; import ubic.basecode.util.FileTools; import ubic.basecode.util.StringUtil; import ubic.gemma.core.analysis.expression.diff.DifferentialExpressionAnalysisConfig; @@ -42,6 +43,7 @@ import ubic.gemma.model.common.quantitationtype.QuantitationType; import ubic.gemma.model.expression.arrayDesign.ArrayDesign; import ubic.gemma.model.expression.bioAssayData.DesignElementDataVector; +import ubic.gemma.model.expression.bioAssayData.RawExpressionDataVector; import ubic.gemma.model.expression.designElement.CompositeSequence; import ubic.gemma.model.expression.experiment.*; import ubic.gemma.model.genome.Taxon; @@ -57,6 +59,7 @@ import java.io.*; import java.util.*; import java.util.Map.Entry; +import java.util.stream.Collectors; import java.util.zip.GZIPOutputStream; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; @@ -347,6 +350,21 @@ public void writeDiffExArchiveFile( BioAssaySet experimentAnalyzed, Differential } } + @Override + @Transactional(readOnly = true) + public void writeRawExpressionData( ExpressionExperiment ee, Writer writer ) throws IOException { + ee = expressionExperimentService.findWithRawExpressionDataVectors( ee ); + if ( ee == null ) { + throw new IllegalArgumentException( "ExpressionExperiment has been removed." ); + } + ExpressionDataDoubleMatrix matrix = expressionDataMatrixService.getRawExpressionDataMatrix( ee ); + Set ads = ee.getRawExpressionDataVectors().stream() + .map( RawExpressionDataVector::getDesignElement ) + .map( CompositeSequence::getArrayDesign ) + .collect( Collectors.toSet() ); + new MatrixWriter().writeWithStringifiedGeneAnnotations( writer, matrix, getGeneAnnotationsAsStringsByProbe( ads ), true ); + } + @Override public File writeOrLocateCoexpressionDataFile( ExpressionExperiment ee, boolean forceWrite ) { diff --git a/gemma-core/src/main/java/ubic/gemma/core/analysis/service/ExpressionDataMatrixService.java b/gemma-core/src/main/java/ubic/gemma/core/analysis/service/ExpressionDataMatrixService.java index 0cb7c5b99e..c6c2f10104 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/analysis/service/ExpressionDataMatrixService.java +++ b/gemma-core/src/main/java/ubic/gemma/core/analysis/service/ExpressionDataMatrixService.java @@ -14,6 +14,7 @@ */ package ubic.gemma.core.analysis.service; +import org.springframework.transaction.annotation.Transactional; import ubic.basecode.dataStructure.matrix.DoubleMatrix; import ubic.gemma.core.analysis.preprocess.filter.FilterConfig; import ubic.gemma.core.datastructure.matrix.ExpressionDataDoubleMatrix; @@ -61,6 +62,13 @@ ExpressionDataDoubleMatrix getFilteredMatrix( String arrayDesignName, FilterConf */ ExpressionDataDoubleMatrix getProcessedExpressionDataMatrix( ExpressionExperiment ee ); + /** + * @throws IllegalArgumentException if the expression experiment has no preferred raw quantitation types + * @param ee + * @return + */ + ExpressionDataDoubleMatrix getRawExpressionDataMatrix( ExpressionExperiment ee ); + DoubleMatrix getRankMatrix( Collection genes, Collection ees, ProcessedExpressionDataVectorDao.RankMethod method ); diff --git a/gemma-core/src/main/java/ubic/gemma/core/analysis/service/ExpressionDataMatrixServiceImpl.java b/gemma-core/src/main/java/ubic/gemma/core/analysis/service/ExpressionDataMatrixServiceImpl.java index 4d575dd2a5..5fa6b74a3f 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/analysis/service/ExpressionDataMatrixServiceImpl.java +++ b/gemma-core/src/main/java/ubic/gemma/core/analysis/service/ExpressionDataMatrixServiceImpl.java @@ -19,28 +19,31 @@ package ubic.gemma.core.analysis.service; import cern.colt.list.DoubleArrayList; +import lombok.extern.apachecommons.CommonsLog; import org.apache.commons.lang3.ArrayUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Transactional; import ubic.basecode.dataStructure.matrix.DenseDoubleMatrix; import ubic.basecode.dataStructure.matrix.DoubleMatrix; import ubic.basecode.math.DescriptiveWithMissing; import ubic.gemma.core.analysis.preprocess.filter.ExpressionExperimentFilter; import ubic.gemma.core.analysis.preprocess.filter.FilterConfig; import ubic.gemma.core.datastructure.matrix.ExpressionDataDoubleMatrix; +import ubic.gemma.model.common.quantitationtype.QuantitationType; import ubic.gemma.model.expression.arrayDesign.ArrayDesign; import ubic.gemma.model.expression.bioAssayData.ProcessedExpressionDataVector; +import ubic.gemma.model.expression.bioAssayData.RawExpressionDataVector; import ubic.gemma.model.expression.experiment.ExpressionExperiment; import ubic.gemma.model.genome.Gene; import ubic.gemma.persistence.service.expression.arrayDesign.ArrayDesignService; import ubic.gemma.persistence.service.expression.bioAssayData.ProcessedExpressionDataVectorDao; import ubic.gemma.persistence.service.expression.bioAssayData.ProcessedExpressionDataVectorService; +import ubic.gemma.persistence.service.expression.bioAssayData.RawExpressionDataVectorService; import ubic.gemma.persistence.service.expression.experiment.ExpressionExperimentService; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashSet; -import java.util.Map; +import java.util.*; +import java.util.stream.Collectors; /** * Tools for easily getting data matrices for analysis in a consistent way. @@ -48,6 +51,7 @@ * @author keshav */ @Component +@CommonsLog public class ExpressionDataMatrixServiceImpl implements ExpressionDataMatrixService { @Autowired @@ -56,6 +60,9 @@ public class ExpressionDataMatrixServiceImpl implements ExpressionDataMatrixServ @Autowired private ProcessedExpressionDataVectorService processedExpressionDataVectorService; + @Autowired + private RawExpressionDataVectorService rawExpressionDataVectorService; + @Autowired private ArrayDesignService arrayDesignService; @@ -97,6 +104,32 @@ public ExpressionDataDoubleMatrix getProcessedExpressionDataMatrix( ExpressionEx return new ExpressionDataDoubleMatrix( dataVectors ); } + @Override + @Transactional(readOnly = true) + public ExpressionDataDoubleMatrix getRawExpressionDataMatrix( ExpressionExperiment ee ) { + Map> rawVectorsByQt = ee.getRawExpressionDataVectors().stream() + .collect( Collectors.groupingBy( RawExpressionDataVector::getQuantitationType, Collectors.toList() ) ); + + Set preferredQuantitationTypes = rawVectorsByQt.keySet().stream() + .filter( QuantitationType::getIsPreferred ) + .collect( Collectors.toSet() ); + + if ( preferredQuantitationTypes.isEmpty() ) { + throw new IllegalArgumentException( "There are no RawExpressionDataVectors for " + ee + ", they must be created first." ); + } + + if ( preferredQuantitationTypes.size() > 1 ) { + log.warn( "There are more than one preferred quantitation type for " + ee + " raw expression vectors." ); + } + + // pick the QT with the maximum ID, which should be the latest one created + QuantitationType pickedQuantitationType = preferredQuantitationTypes.stream() + .max( Comparator.comparing( QuantitationType::getId ) ) + .orElse( null ); + + return new ExpressionDataDoubleMatrix( rawVectorsByQt.get( pickedQuantitationType ) ); + } + @Override public DoubleMatrix getRankMatrix( Collection genes, Collection ees, ProcessedExpressionDataVectorDao.RankMethod method ) { diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/experiment/ExpressionExperimentDao.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/experiment/ExpressionExperimentDao.java index 37ce22051e..8c587fc02c 100644 --- a/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/experiment/ExpressionExperimentDao.java +++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/experiment/ExpressionExperimentDao.java @@ -42,6 +42,8 @@ public interface ExpressionExperimentDao Collection filterByTaxon( Collection ids, Taxon taxon ); + ExpressionExperiment findWithRawExpressionDataVectors( ExpressionExperiment ee ); + Collection findByAccession( DatabaseEntry accession ); Collection findByAccession( String accession ); diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/experiment/ExpressionExperimentDaoImpl.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/experiment/ExpressionExperimentDaoImpl.java index fbdf9ace56..dfe0dbaa16 100644 --- a/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/experiment/ExpressionExperimentDaoImpl.java +++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/experiment/ExpressionExperimentDaoImpl.java @@ -129,6 +129,14 @@ public ExpressionExperiment find( ExpressionExperiment entity ) { return ( ExpressionExperiment ) criteria.uniqueResult(); } + @Override + public ExpressionExperiment findWithRawExpressionDataVectors( ExpressionExperiment ee ) { + return ( ExpressionExperiment ) this.getSessionFactory().getCurrentSession() + .createQuery( "select ee from ExpressionExperiment as ee join fetch ee.rawExpressionDataVectors where ee = :ee" ) + .setParameter( "ee", ee ) + .uniqueResult(); + } + @Override public Collection findByAccession( DatabaseEntry accession ) { Criteria criteria = this.getSessionFactory().getCurrentSession().createCriteria( ExpressionExperiment.class ); diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/experiment/ExpressionExperimentService.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/experiment/ExpressionExperimentService.java index d488a01fba..e71740e9d1 100644 --- a/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/experiment/ExpressionExperimentService.java +++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/experiment/ExpressionExperimentService.java @@ -108,6 +108,9 @@ ExpressionExperiment addRawVectors( ExpressionExperiment eeToUpdate, @Secured({ "GROUP_USER", "AFTER_ACL_READ" }) ExpressionExperiment findOrCreate( ExpressionExperiment expressionExperiment ); + @Secured({ "IS_AUTHENTICATED_ANONYMOUSLY", "AFTER_ACL_READ" }) + ExpressionExperiment findWithRawExpressionDataVectors( ExpressionExperiment ee ); + @Override @Secured({ "GROUP_USER" }) ExpressionExperiment create( ExpressionExperiment expressionExperiment ); diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/experiment/ExpressionExperimentServiceImpl.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/experiment/ExpressionExperimentServiceImpl.java index 91a00f362d..ab46a35ba6 100755 --- a/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/experiment/ExpressionExperimentServiceImpl.java +++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/experiment/ExpressionExperimentServiceImpl.java @@ -317,6 +317,12 @@ public Collection filterByTaxon( Collection ids, Taxon taxon ) { return this.expressionExperimentDao.filterByTaxon( ids, taxon ); } + @Override + @Transactional(readOnly = true) + public ExpressionExperiment findWithRawExpressionDataVectors( ExpressionExperiment ee ) { + return this.expressionExperimentDao.findWithRawExpressionDataVectors( ee ); + } + /** * @see ExpressionExperimentService#findByAccession(DatabaseEntry) */ diff --git a/gemma-web/src/main/java/ubic/gemma/web/services/rest/DatasetsWebService.java b/gemma-web/src/main/java/ubic/gemma/web/services/rest/DatasetsWebService.java index ab3353bfcf..ccac3216f9 100644 --- a/gemma-web/src/main/java/ubic/gemma/web/services/rest/DatasetsWebService.java +++ b/gemma-web/src/main/java/ubic/gemma/web/services/rest/DatasetsWebService.java @@ -15,6 +15,9 @@ package ubic.gemma.web.services.rest; import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.media.Content; +import io.swagger.v3.oas.annotations.media.Schema; +import io.swagger.v3.oas.annotations.responses.ApiResponse; import lombok.extern.apachecommons.CommonsLog; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -40,10 +43,8 @@ import ubic.gemma.persistence.service.expression.bioAssayData.ProcessedExpressionDataVectorService; import ubic.gemma.persistence.service.expression.experiment.ExpressionExperimentService; import ubic.gemma.persistence.util.Filters; -import ubic.gemma.web.services.rest.util.ArgUtils; -import ubic.gemma.web.services.rest.util.PaginatedResponseDataObject; -import ubic.gemma.web.services.rest.util.Responder; -import ubic.gemma.web.services.rest.util.ResponseDataObject; +import ubic.gemma.web.services.rest.annotations.GZIP; +import ubic.gemma.web.services.rest.util.*; import ubic.gemma.web.services.rest.util.args.*; import javax.servlet.http.HttpServletResponse; @@ -51,8 +52,13 @@ import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; +import javax.ws.rs.core.StreamingOutput; import java.io.File; -import java.util.*; +import java.io.OutputStreamWriter; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; /** * RESTful interface for datasets. @@ -249,6 +255,26 @@ public Response datasetData( // Params: return this.outputDataFile( ee, filterData.getValue() ); } + /** + * Retrieve raw expression data. + * + * The payload is transparently compressed via a Content-Encoding header and streamed to avoid dumping + * the whole payload in memory. + */ + @GZIP + @GET + @Path("/{dataset}/data/raw") + @Produces("text/tab-separated-values; charset=UTF-8") + @Operation(summary = "Retrieve raw expression data of a dataset", responses = { + @ApiResponse(responseCode = "200", content = @Content(mediaType = "text/tab-separated-values; charset=UTF-8", + schema = @Schema(type = "string", format = "binary"))), + @ApiResponse(responseCode = "404", description = "The dataset does not exist.", + content = @Content(mediaType = MediaType.APPLICATION_JSON, schema = @Schema(implementation = ResponseErrorObject.class))) }) + public StreamingOutput getDatasetRawExpression( @PathParam("dataset") DatasetArg datasetArg ) { + ExpressionExperiment ee = datasetArg.getEntity( expressionExperimentService ); + return ( output ) -> expressionDataFileService.writeRawExpressionData( ee, new OutputStreamWriter( output ) ); + } + /** * Retrieves the design for the given dataset. * diff --git a/gemma-web/src/test/java/ubic/gemma/web/services/rest/DatasetsRestTest.java b/gemma-web/src/test/java/ubic/gemma/web/services/rest/DatasetsRestTest.java index 435f18ade2..e362441b9b 100644 --- a/gemma-web/src/test/java/ubic/gemma/web/services/rest/DatasetsRestTest.java +++ b/gemma-web/src/test/java/ubic/gemma/web/services/rest/DatasetsRestTest.java @@ -17,6 +17,9 @@ import ubic.gemma.web.services.rest.util.args.*; import ubic.gemma.web.util.BaseSpringWebTest; +import javax.ws.rs.core.StreamingOutput; +import java.io.*; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -239,4 +242,21 @@ public void testFilterByGeeqPublicationScore() { SortArg.valueOf( "+id" ), new MockHttpServletResponse() ); } + + @Test + public void testGetDatasetRawExpression() throws IOException { + ExpressionExperiment ee = ees.get( 0 ); + StreamingOutput streamingOutput = datasetsWebService.getDatasetRawExpression( DatasetArg.valueOf( String.valueOf( ee.getId() ) ) ); + byte[] payload; + try ( ByteArrayOutputStream os = new ByteArrayOutputStream() ) { + streamingOutput.write( os ); + payload = os.toByteArray(); + } + String decodedPayload = new String( payload, StandardCharsets.UTF_8 ); + // there's 7 comment lines, 1 header and then one line per raw EV (there are two platforms the default collection size in the fixture) + assertThat( decodedPayload ) + .isNotEmpty() + .contains( ee.getShortName() ) + .hasLineCount( 8 + 2 * testHelper.getTestElementCollectionSize() ); + } } From 3f6b368a7539047ca719ea9ab9f6d95cab3fbaa2 Mon Sep 17 00:00:00 2001 From: Guillaume Poirier-Morency Date: Mon, 6 Jun 2022 13:59:32 -0700 Subject: [PATCH 3/4] Add Content-Disposition header to getDatasetRawExpression endpoint --- .../ubic/gemma/web/services/rest/DatasetsWebService.java | 7 +++++-- .../ubic/gemma/web/services/rest/DatasetsRestTest.java | 5 +++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/gemma-web/src/main/java/ubic/gemma/web/services/rest/DatasetsWebService.java b/gemma-web/src/main/java/ubic/gemma/web/services/rest/DatasetsWebService.java index ccac3216f9..89ed349969 100644 --- a/gemma-web/src/main/java/ubic/gemma/web/services/rest/DatasetsWebService.java +++ b/gemma-web/src/main/java/ubic/gemma/web/services/rest/DatasetsWebService.java @@ -270,9 +270,12 @@ public Response datasetData( // Params: schema = @Schema(type = "string", format = "binary"))), @ApiResponse(responseCode = "404", description = "The dataset does not exist.", content = @Content(mediaType = MediaType.APPLICATION_JSON, schema = @Schema(implementation = ResponseErrorObject.class))) }) - public StreamingOutput getDatasetRawExpression( @PathParam("dataset") DatasetArg datasetArg ) { + public Response getDatasetRawExpression( @PathParam("dataset") DatasetArg datasetArg ) { ExpressionExperiment ee = datasetArg.getEntity( expressionExperimentService ); - return ( output ) -> expressionDataFileService.writeRawExpressionData( ee, new OutputStreamWriter( output ) ); + StreamingOutput stream = ( output ) -> expressionDataFileService.writeRawExpressionData( ee, new OutputStreamWriter( output ) ); + return Response.ok( stream ) + .header( "Content-Disposition", String.format( "attachment; filename=%d_%s_expmat.unfilt.raw.data.txt", ee.getId(), ee.getShortName() ) ) + .build(); } /** diff --git a/gemma-web/src/test/java/ubic/gemma/web/services/rest/DatasetsRestTest.java b/gemma-web/src/test/java/ubic/gemma/web/services/rest/DatasetsRestTest.java index e362441b9b..2d61a880db 100644 --- a/gemma-web/src/test/java/ubic/gemma/web/services/rest/DatasetsRestTest.java +++ b/gemma-web/src/test/java/ubic/gemma/web/services/rest/DatasetsRestTest.java @@ -17,6 +17,7 @@ import ubic.gemma.web.services.rest.util.args.*; import ubic.gemma.web.util.BaseSpringWebTest; +import javax.ws.rs.core.Response; import javax.ws.rs.core.StreamingOutput; import java.io.*; import java.nio.charset.StandardCharsets; @@ -246,10 +247,10 @@ public void testFilterByGeeqPublicationScore() { @Test public void testGetDatasetRawExpression() throws IOException { ExpressionExperiment ee = ees.get( 0 ); - StreamingOutput streamingOutput = datasetsWebService.getDatasetRawExpression( DatasetArg.valueOf( String.valueOf( ee.getId() ) ) ); + Response response = datasetsWebService.getDatasetRawExpression( DatasetArg.valueOf( String.valueOf( ee.getId() ) ) ); byte[] payload; try ( ByteArrayOutputStream os = new ByteArrayOutputStream() ) { - streamingOutput.write( os ); + ( ( StreamingOutput ) response.getEntity() ).write( os ); payload = os.toByteArray(); } String decodedPayload = new String( payload, StandardCharsets.UTF_8 ); From 7b6ea0cef69e6d05be9e6e973d74d900296335fe Mon Sep 17 00:00:00 2001 From: Guillaume Poirier-Morency Date: Mon, 6 Jun 2022 14:25:19 -0700 Subject: [PATCH 4/4] Use Hibernate.initialize() to initialise raw EVs efficiently --- .../analysis/service/ExpressionDataFileServiceImpl.java | 5 ++++- .../expression/experiment/ExpressionExperimentDao.java | 2 -- .../experiment/ExpressionExperimentDaoImpl.java | 8 -------- .../experiment/ExpressionExperimentService.java | 3 --- .../experiment/ExpressionExperimentServiceImpl.java | 6 ------ 5 files changed, 4 insertions(+), 20 deletions(-) diff --git a/gemma-core/src/main/java/ubic/gemma/core/analysis/service/ExpressionDataFileServiceImpl.java b/gemma-core/src/main/java/ubic/gemma/core/analysis/service/ExpressionDataFileServiceImpl.java index 38c296df6a..1e51093aa7 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/analysis/service/ExpressionDataFileServiceImpl.java +++ b/gemma-core/src/main/java/ubic/gemma/core/analysis/service/ExpressionDataFileServiceImpl.java @@ -23,6 +23,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.hibernate.Hibernate; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.springframework.transaction.annotation.Transactional; @@ -353,10 +354,12 @@ public void writeDiffExArchiveFile( BioAssaySet experimentAnalyzed, Differential @Override @Transactional(readOnly = true) public void writeRawExpressionData( ExpressionExperiment ee, Writer writer ) throws IOException { - ee = expressionExperimentService.findWithRawExpressionDataVectors( ee ); + ee = expressionExperimentService.find( ee ); if ( ee == null ) { throw new IllegalArgumentException( "ExpressionExperiment has been removed." ); } + // pre-initialize it so that it get fetched in a single query without a jointure with the EE + Hibernate.initialize( ee.getRawExpressionDataVectors() ); ExpressionDataDoubleMatrix matrix = expressionDataMatrixService.getRawExpressionDataMatrix( ee ); Set ads = ee.getRawExpressionDataVectors().stream() .map( RawExpressionDataVector::getDesignElement ) diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/experiment/ExpressionExperimentDao.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/experiment/ExpressionExperimentDao.java index 8c587fc02c..37ce22051e 100644 --- a/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/experiment/ExpressionExperimentDao.java +++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/experiment/ExpressionExperimentDao.java @@ -42,8 +42,6 @@ public interface ExpressionExperimentDao Collection filterByTaxon( Collection ids, Taxon taxon ); - ExpressionExperiment findWithRawExpressionDataVectors( ExpressionExperiment ee ); - Collection findByAccession( DatabaseEntry accession ); Collection findByAccession( String accession ); diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/experiment/ExpressionExperimentDaoImpl.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/experiment/ExpressionExperimentDaoImpl.java index dfe0dbaa16..fbdf9ace56 100644 --- a/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/experiment/ExpressionExperimentDaoImpl.java +++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/experiment/ExpressionExperimentDaoImpl.java @@ -129,14 +129,6 @@ public ExpressionExperiment find( ExpressionExperiment entity ) { return ( ExpressionExperiment ) criteria.uniqueResult(); } - @Override - public ExpressionExperiment findWithRawExpressionDataVectors( ExpressionExperiment ee ) { - return ( ExpressionExperiment ) this.getSessionFactory().getCurrentSession() - .createQuery( "select ee from ExpressionExperiment as ee join fetch ee.rawExpressionDataVectors where ee = :ee" ) - .setParameter( "ee", ee ) - .uniqueResult(); - } - @Override public Collection findByAccession( DatabaseEntry accession ) { Criteria criteria = this.getSessionFactory().getCurrentSession().createCriteria( ExpressionExperiment.class ); diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/experiment/ExpressionExperimentService.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/experiment/ExpressionExperimentService.java index e71740e9d1..d488a01fba 100644 --- a/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/experiment/ExpressionExperimentService.java +++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/experiment/ExpressionExperimentService.java @@ -108,9 +108,6 @@ ExpressionExperiment addRawVectors( ExpressionExperiment eeToUpdate, @Secured({ "GROUP_USER", "AFTER_ACL_READ" }) ExpressionExperiment findOrCreate( ExpressionExperiment expressionExperiment ); - @Secured({ "IS_AUTHENTICATED_ANONYMOUSLY", "AFTER_ACL_READ" }) - ExpressionExperiment findWithRawExpressionDataVectors( ExpressionExperiment ee ); - @Override @Secured({ "GROUP_USER" }) ExpressionExperiment create( ExpressionExperiment expressionExperiment ); diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/experiment/ExpressionExperimentServiceImpl.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/experiment/ExpressionExperimentServiceImpl.java index ab46a35ba6..91a00f362d 100755 --- a/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/experiment/ExpressionExperimentServiceImpl.java +++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/experiment/ExpressionExperimentServiceImpl.java @@ -317,12 +317,6 @@ public Collection filterByTaxon( Collection ids, Taxon taxon ) { return this.expressionExperimentDao.filterByTaxon( ids, taxon ); } - @Override - @Transactional(readOnly = true) - public ExpressionExperiment findWithRawExpressionDataVectors( ExpressionExperiment ee ) { - return this.expressionExperimentDao.findWithRawExpressionDataVectors( ee ); - } - /** * @see ExpressionExperimentService#findByAccession(DatabaseEntry) */