Merge branch 'hotfix-1.28.2'

gemma-core/pom.xml

@@ -3,7 +3,7 @@
  <parent>
  <artifactId>gemma</artifactId>
  <groupId>gemma</groupId>
- <version>1.28.1</version>
+ <version>1.28.2</version>
  </parent>
  <modelVersion>4.0.0</modelVersion>
  <artifactId>gemma-core</artifactId>
@@ -87,7 +87,7 @@
  </program>
  </programs>
  <!-- this allows Log4j 2 to lookup for Log4j 1.2 configuration -->
- <extraJvmArguments>-Dlog4j1.compatibility=true --add-opens java.base/java.lang=ALL-UNNAMED</extraJvmArguments>
+ <extraJvmArguments>-Dlog4j1.compatibility=true</extraJvmArguments>
  <!-- Gemma CLI extensions can be dropped in that folder -->
  <endorsedDir>contrib</endorsedDir>
  </configuration>
@@ -326,7 +326,7 @@
  <dependency>
  <groupId>ubc.chibi.compass-fork</groupId>
  <artifactId>compass-fork</artifactId>
- <version>1.1.1</version>
+ <version>1.1.3</version>
  </dependency>
 
  <!-- Apache Commons -->

gemma-core/src/main/config/log4j.properties

@@ -126,11 +126,6 @@ log4j.logger.org.directwebremoting=WARN
 # Apache Commons
 log4j.logger.org.apache.commons=WARN
 
-# Ehcache
-log4j.logger.net.sf.ehcache=ERROR
-#filter out "A soft-locked cache entry was removed already. Out of balance lock/unlock sequences"
-log4j.logger.net.sf.ehcache.hibernate.strategy.AbstractReadWriteEhcacheAccessStrategy=FATAL
-
 # Compass
 log4j.logger.org.compass=WARN
 # annoying message we can't fix from Compass.

gemma-core/src/main/java/ubic/gemma/core/analysis/report/ArrayDesignReportServiceImpl.java

@@ -176,7 +176,7 @@ public void generateArrayDesignReport( ArrayDesignValueObject adVo ) {
  ObjectOutputStream oos = new ObjectOutputStream( fos )) {
  oos.writeObject( adVo );
  } catch ( Throwable e ) {
- ArrayDesignReportServiceImpl.log.error( "Cannot write to file: " + reportFileName );
+ ArrayDesignReportServiceImpl.log.error( "Cannot write to file: " + reportFileName, e );
  return;
  }
  ArrayDesignReportServiceImpl.log.info( "Generated report for " + ad );

gemma-core/src/main/java/ubic/gemma/core/analysis/service/ExpressionDataFileService.java

@@ -124,18 +124,29 @@ File writeDataFile( ExpressionExperiment ee, boolean filtered, String fileName,
  throws IOException, FilteringException;
 
  /**
- * Write raw expression data to a given writer.
- *
- * Note: the preferred quantitations are used.
+ * Write raw expression data to a given writer for a given quantitation type.
+ * <p>
+ * Note: For compression, wrap a {@link java.util.zip.GZIPOutputStream} with a {@link java.io.OutputStreamWriter}.
+ * To write to a string, consider using {@link java.io.StringWriter}.
  *
+ * @param ee the expression experiment
+ * @param qt a quantitation type to use, otherwise the preferred one is picked
+ * @param writer the destination for the raw expression data
+ * @throws IOException if operations with the writer fails
+ */
+ void writeRawExpressionData( ExpressionExperiment ee, QuantitationType qt, Writer writer ) throws IOException;
+
+ /**
+ * Write processed expression data to a given writer for a given quantitation type.
+ * <p>
  * Note: For compression, wrap a {@link java.util.zip.GZIPOutputStream} with a {@link java.io.OutputStreamWriter}.
  * To write to a string, consider using {@link java.io.StringWriter}.
  *
  * @param ee the expression experiment
  * @param writer the destination for the raw expression data
  * @throws IOException if operations with the writer fails
  */
- void writeRawExpressionData( ExpressionExperiment ee, Writer writer ) throws IOException;
+ void writeProcessedExpressionData( ExpressionExperiment ee, QuantitationType qt, Writer writer ) throws IOException;
 
  /**
  * Write or located the coexpression data file for a given experiment

gemma-core/src/main/java/ubic/gemma/core/analysis/service/ExpressionDataFileServiceImpl.java

@@ -23,7 +23,6 @@
 import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.hibernate.Hibernate;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 import org.springframework.transaction.annotation.Transactional;
@@ -33,7 +32,6 @@
 import ubic.gemma.core.analysis.preprocess.ExpressionDataMatrixBuilder;
 import ubic.gemma.core.analysis.preprocess.filter.FilterConfig;
 import ubic.gemma.core.analysis.preprocess.filter.FilteringException;
-import ubic.gemma.core.analysis.preprocess.filter.NoRowsLeftAfterFilteringException;
 import ubic.gemma.core.datastructure.matrix.ExperimentalDesignWriter;
 import ubic.gemma.core.datastructure.matrix.ExpressionDataDoubleMatrix;
 import ubic.gemma.core.datastructure.matrix.ExpressionDataMatrix;
@@ -46,7 +44,6 @@
 import ubic.gemma.model.common.quantitationtype.QuantitationType;
 import ubic.gemma.model.expression.arrayDesign.ArrayDesign;
 import ubic.gemma.model.expression.bioAssayData.DesignElementDataVector;
-import ubic.gemma.model.expression.bioAssayData.RawExpressionDataVector;
 import ubic.gemma.model.expression.designElement.CompositeSequence;
 import ubic.gemma.model.expression.experiment.*;
 import ubic.gemma.model.genome.Taxon;
@@ -355,16 +352,27 @@ public void writeDiffExArchiveFile( BioAssaySet experimentAnalyzed, Differential
 
  @Override
  @Transactional(readOnly = true)
- public void writeRawExpressionData( ExpressionExperiment ee, Writer writer ) throws IOException {
+ public void writeRawExpressionData( ExpressionExperiment ee, QuantitationType qt, Writer writer ) throws IOException {
  ee = expressionExperimentService.find( ee );
  if ( ee == null ) {
  throw new IllegalArgumentException( "ExpressionExperiment has been removed." );
  }
- // pre-initialize it so that it get fetched in a single query without a jointure with the EE
- Hibernate.initialize( ee.getRawExpressionDataVectors() );
- ExpressionDataDoubleMatrix matrix = expressionDataMatrixService.getRawExpressionDataMatrix( ee );
- Set<ArrayDesign> ads = ee.getRawExpressionDataVectors().stream()
- .map( RawExpressionDataVector::getDesignElement )
+ ExpressionDataDoubleMatrix matrix = expressionDataMatrixService.getRawExpressionDataMatrix( ee, qt );
+ Set<ArrayDesign> ads = matrix.getDesignElements().stream()
+ .map( CompositeSequence::getArrayDesign )
+ .collect( Collectors.toSet() );
+ new MatrixWriter().writeWithStringifiedGeneAnnotations( writer, matrix, getGeneAnnotationsAsStringsByProbe( ads ), true );
+ }
+
+ @Override
+ @Transactional(readOnly = true)
+ public void writeProcessedExpressionData( ExpressionExperiment ee, QuantitationType qt, Writer writer ) throws IOException {
+ ee = expressionExperimentService.find( ee );
+ if ( ee == null ) {
+ throw new IllegalArgumentException( "ExpressionExperiment has been removed." );
+ }
+ ExpressionDataDoubleMatrix matrix = expressionDataMatrixService.getProcessedExpressionDataMatrix( ee, qt );
+ Set<ArrayDesign> ads = matrix.getDesignElements().stream()
  .map( CompositeSequence::getArrayDesign )
  .collect( Collectors.toSet() );
  new MatrixWriter().writeWithStringifiedGeneAnnotations( writer, matrix, getGeneAnnotationsAsStringsByProbe( ads ), true );

gemma-core/src/main/java/ubic/gemma/core/analysis/service/ExpressionDataMatrixService.java

@@ -20,11 +20,14 @@
 import ubic.gemma.core.analysis.preprocess.filter.FilteringException;
 import ubic.gemma.core.analysis.preprocess.filter.NoRowsLeftAfterFilteringException;
 import ubic.gemma.core.datastructure.matrix.ExpressionDataDoubleMatrix;
+import ubic.gemma.model.common.quantitationtype.QuantitationType;
 import ubic.gemma.model.expression.bioAssayData.ProcessedExpressionDataVector;
+import ubic.gemma.model.expression.bioAssayData.RawExpressionDataVector;
 import ubic.gemma.model.expression.experiment.ExpressionExperiment;
 import ubic.gemma.model.genome.Gene;
 import ubic.gemma.persistence.service.expression.bioAssayData.ProcessedExpressionDataVectorDao;
 
+import javax.annotation.Nullable;
 import java.util.Collection;
 
 /**
@@ -65,11 +68,16 @@ ExpressionDataDoubleMatrix getFilteredMatrix( String arrayDesignName, FilterConf
  ExpressionDataDoubleMatrix getProcessedExpressionDataMatrix( ExpressionExperiment ee );
 
  /**
- * @throws IllegalArgumentException if the expression experiment has no preferred raw quantitation types
- * @param ee
- * @return
+ * Obtain a processed expression data matrix for a given quantitation type.
+ * @param quantitationType the quantitation type, or null to pick the preferred one
  */
- ExpressionDataDoubleMatrix getRawExpressionDataMatrix( ExpressionExperiment ee );
+ ExpressionDataDoubleMatrix getProcessedExpressionDataMatrix( ExpressionExperiment ee, QuantitationType quantitationType );
+
+ /**
+ * Obtain a raw expression data matrix for a given quantitation type
+ * @param quantitationType the quantitation type, or null to pick the preferred one
+ */
+ ExpressionDataDoubleMatrix getRawExpressionDataMatrix( ExpressionExperiment ee, QuantitationType quantitationType );
 
  DoubleMatrix<Gene, ExpressionExperiment> getRankMatrix( Collection<Gene> genes,
  Collection<ExpressionExperiment> ees, ProcessedExpressionDataVectorDao.RankMethod method );

gemma-core/src/main/java/ubic/gemma/core/analysis/service/ExpressionDataMatrixServiceImpl.java

@@ -34,7 +34,6 @@
 import ubic.gemma.model.common.quantitationtype.QuantitationType;
 import ubic.gemma.model.expression.arrayDesign.ArrayDesign;
 import ubic.gemma.model.expression.bioAssayData.ProcessedExpressionDataVector;
-import ubic.gemma.model.expression.bioAssayData.RawExpressionDataVector;
 import ubic.gemma.model.expression.experiment.ExpressionExperiment;
 import ubic.gemma.model.genome.Gene;
 import ubic.gemma.persistence.service.expression.arrayDesign.ArrayDesignService;
@@ -44,7 +43,6 @@
 import ubic.gemma.persistence.service.expression.experiment.ExpressionExperimentService;
 
 import java.util.*;
-import java.util.stream.Collectors;
 
 /**
  * Tools for easily getting data matrices for analysis in a consistent way.
@@ -107,28 +105,14 @@ public ExpressionDataDoubleMatrix getProcessedExpressionDataMatrix( ExpressionEx
 
  @Override
  @Transactional(readOnly = true)
- public ExpressionDataDoubleMatrix getRawExpressionDataMatrix( ExpressionExperiment ee ) {
- Map<QuantitationType, List<RawExpressionDataVector>> rawVectorsByQt = ee.getRawExpressionDataVectors().stream()
- .collect( Collectors.groupingBy( RawExpressionDataVector::getQuantitationType, Collectors.toList() ) );
-
- Set<QuantitationType> preferredQuantitationTypes = rawVectorsByQt.keySet().stream()
- .filter( QuantitationType::getIsPreferred )
- .collect( Collectors.toSet() );
-
- if ( preferredQuantitationTypes.isEmpty() ) {
- throw new IllegalArgumentException( "There are no RawExpressionDataVectors for " + ee + ", they must be created first." );
- }
-
- if ( preferredQuantitationTypes.size() > 1 ) {
- log.warn( "There are more than one preferred quantitation type for " + ee + " raw expression vectors." );
- }
-
- // pick the QT with the maximum ID, which should be the latest one created
- QuantitationType pickedQuantitationType = preferredQuantitationTypes.stream()
- .max( Comparator.comparing( QuantitationType::getId ) )
- .orElse( null );
+ public ExpressionDataDoubleMatrix getProcessedExpressionDataMatrix( ExpressionExperiment ee, QuantitationType quantitationType ) {
+ return new ExpressionDataDoubleMatrix( processedExpressionDataVectorService.findByExpressionExperiment( ee, quantitationType ) );
+ }
 
- return new ExpressionDataDoubleMatrix( rawVectorsByQt.get( pickedQuantitationType ) );
+ @Override
+ @Transactional(readOnly = true)
+ public ExpressionDataDoubleMatrix getRawExpressionDataMatrix( ExpressionExperiment ee, QuantitationType quantitationType ) {
+ return new ExpressionDataDoubleMatrix( rawExpressionDataVectorService.findByExpressionExperiment( ee, quantitationType ) );
  }
 
  @Override

gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/BaseExpressionDataMatrix.java

@@ -148,6 +148,16 @@ public int getColumnIndex( BioMaterial bioMaterial ) {
  return columnBioMaterialMap.get( bioMaterial );
  }
 
+ @Override
+ public List<CompositeSequence> getDesignElements() {
+ Vector<CompositeSequence> compositeSequences = new Vector<>();
+ compositeSequences.setSize( rows() );
+ for ( int i = 0; i < rows(); i++ ) {
+ compositeSequences.set( i, this.rowDesignElementMapByInteger.get( i ) );
+ }
+ return compositeSequences;
+ }
+
  @Override
  public CompositeSequence getDesignElementForRow( int index ) {
  return this.rowDesignElementMapByInteger.get( index );

gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/ExpressionDataMatrix.java

@@ -157,6 +157,11 @@ public interface ExpressionDataMatrix<T> {
  */
  T[][] getColumns( List<BioAssay> bioAssays );
 
+ /**
+ * Obtain all the design elements in this data matrix.
+ */
+ List<CompositeSequence> getDesignElements();
+
  /**
  * @param index i
  * @return cs

gemma-core/src/main/java/ubic/gemma/model/IdentifiableValueObject.java

@@ -3,6 +3,7 @@
 import lombok.EqualsAndHashCode;
 import ubic.gemma.model.common.Identifiable;
 
+import java.io.Serializable;
 import java.util.Objects;
 
 /**