Merge branch 'hotfix-1.27.17'

gemma-core/pom.xml

@@ -3,7 +3,7 @@
  <parent>
  <artifactId>gemma</artifactId>
  <groupId>gemma</groupId>
- <version>1.27.16</version>
+ <version>1.27.17</version>
  </parent>
  <modelVersion>4.0.0</modelVersion>
  <artifactId>gemma-core</artifactId>
@@ -88,6 +88,8 @@
  </programs>
  <!-- this allows Log4j 2 to lookup for Log4j 1.2 configuration -->
  <extraJvmArguments>-Dlog4j1.compatibility=true</extraJvmArguments>
+ <!-- Gemma CLI extensions can be dropped in that folder -->
+ <endorsedDir>contrib</endorsedDir>
  </configuration>
  <executions>
  <execution>

gemma-core/src/main/config/log4j.properties

@@ -101,6 +101,9 @@ log4j.logger.ubic.gemma.persistence.util.monitor.MonitorAdvice=INFO,metricFile
 log4j.logger.ubic.gemma.web.controller.common.CharacteristicBrowserController=INFO,annotationsFile
 log4j.logger.ubic.gemma.web.controller.expression.experiment.AnnotationController=INFO,annotationsFile
 
+# Gemma (externally contributed components)
+log4j.logger.ubic.gemma.contrib=INFO
+
 ############# THIRD PARTY CLASSES #####################
 # Control logging for other packages
 log4j.logger.org=WARN

gemma-core/src/main/java/ubic/gemma/core/analysis/expression/diff/DifferentialExpressionAnalysisConfig.java

@@ -40,7 +40,7 @@ public class DifferentialExpressionAnalysisConfig implements Serializable {
  /**
  * Default value for whether empirical Bayes moderation of test statistics should be used.
  */
- public static final boolean DEFAULT_EBAYES = false;
+ public static final boolean DEFAULT_EBAYES = true;
 
  private static final long serialVersionUID = 622877438067070041L;
 
@@ -65,6 +65,7 @@ public class DifferentialExpressionAnalysisConfig implements Serializable {
  private FactorValue subsetFactorValue = null;
 
  private boolean useWeights = false;
+ private boolean makeArchiveFile = true;
 
  public void addInteractionToInclude( Collection<ExperimentalFactor> factors ) {
  interactionsToInclude.add( factors );
@@ -232,4 +233,11 @@ public String toString() {
 
  }
 
+ public boolean getMakeArchiveFile() {
+ return makeArchiveFile;
+ }
+
+ public void setMakeArchiveFile( boolean makeArchiveFile ) {
+ this.makeArchiveFile = makeArchiveFile;
+ }
 }

gemma-core/src/main/java/ubic/gemma/core/analysis/expression/diff/DifferentialExpressionAnalyzerServiceImpl.java

@@ -244,11 +244,13 @@ public DifferentialExpressionAnalysis persistAnalysis( ExpressionExperiment expr
  // get a clean copy of the analysis object from the DB.
  analysis = differentialExpressionAnalysisService.load( analysis.getId() );
  // we do this here because now we have IDs for everything.
- try {
- expressionDataFileService.writeDiffExArchiveFile( expressionExperiment, analysis, config );
- } catch ( IOException e ) {
- DifferentialExpressionAnalyzerServiceImpl.log
- .error( "Unable to save the data to a file: " + e.getMessage() );
+ if (config.getMakeArchiveFile() ) {
+ try {
+ expressionDataFileService.writeDiffExArchiveFile( expressionExperiment, analysis, config );
+ } catch ( IOException e ) {
+ DifferentialExpressionAnalyzerServiceImpl.log
+ .error( "Unable to save the data to a file: " + e.getMessage() );
+ }
  }
 
  // final transaction: audit.

gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/SplitExperimentServiceImpl.java

@@ -171,9 +171,13 @@ public Collection<ExpressionExperiment> split( ExpressionExperiment toSplit, Exp
  if ( StringUtils.isBlank( factorValueString ) ) {
  factorValueString = splitValue.getDescriptiveString();
  }
- split.setName( "Split part " + splitNumber + " of: " + toSplit.getName() + " ["
- + splitValue.getExperimentalFactor().getCategory().getValue() + " = "
- + factorValueString + "]" );
+ split.setName( String.format( "Split part %d of: %s [%s = %s]",
+ splitNumber,
+ toSplit.getName(),
+ splitValue.getExperimentalFactor().getCategory() != null ?
+ splitValue.getExperimentalFactor().getCategory().getValue() :
+ splitValue.getExperimentalFactor().getName(),
+ factorValueString ) );
  split.setDescription( "This experiment was created by Gemma splitting another: \n" + toSplit + toSplit.getDescription() );
 
  split.setCharacteristics( this.cloneCharacteristics( toSplit.getCharacteristics() ) );
@@ -403,7 +407,9 @@ private Collection<ExperimentalFactor> cloneExperimentalFactors( Collection<Expe
  for ( ExperimentalFactor ef : experimentalFactors ) {
  ExperimentalFactor clone = ExperimentalFactor.Factory.newInstance();
  clone.setAnnotations( this.cloneCharacteristics( ef.getAnnotations() ) );
- clone.setCategory( this.cloneCharacteristic( ef.getCategory() ) );
+ if ( ef.getCategory() != null ) {
+ clone.setCategory( this.cloneCharacteristic( ef.getCategory() ) );
+ }
  clone.setName( ef.getName() );
  clone.setDescription( ef.getDescription() );
  clone.setType( ef.getType() );

gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/batcheffects/BatchConfound.java

@@ -122,7 +122,7 @@ private static Collection<BatchConfoundValueObject> factorBatchConfoundTest( Bio
  continue;
 
  // ignore factors that we add with the aim of resolving confounds.
- if ( ef.getCategory().getValue().equalsIgnoreCase( "collection of material" ) )
+ if ( ef.getCategory() != null && ef.getCategory().getValue().equalsIgnoreCase( "collection of material" ) )
  continue;
 
  Map<Long, Double> bmToFv = bioMaterialFactorMap.get( ef );

gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/batcheffects/BatchConfoundValueObject.java

@@ -76,7 +76,7 @@ public String toString() {
  } else {
  name = " Subset " + ee.getName() + " of " + ( ( ExpressionExperiment ) ee ).getShortName();
  }
- return ee.getId() + "\t" + name + "\t" + ef.getId() + "\t" + ef.getCategory().getCategory() + "\t"
+ return ee.getId() + "\t" + name + "\t" + ef.getId() + "\t" + ( ef.getCategory() != null ? ef.getCategory().getCategory() : ef.getName() ) + "\t"
  + String.format( "%.2f", chiSquare ) + "\t" + df + "\t" + String.format( "%.2g", p ) + "\t"
  + numBatches;
  }

gemma-core/src/main/java/ubic/gemma/core/analysis/report/DatabaseViewGeneratorImpl.java

@@ -291,7 +291,7 @@ private void generateDifferentialExpressionView( Integer limit, Collection<Expre
 
  for ( ExperimentalFactor ef : ears.getExperimentalFactors() ) {
  factorName.append( ef.getName() ).append( "," );
- if ( StringUtils.isNotBlank( ef.getCategory().getCategoryUri() ) ) {
+ if ( ef.getCategory() != null && StringUtils.isNotBlank( ef.getCategory().getCategoryUri() ) ) {
  factorURI.append( ef.getCategory().getCategoryUri() ).append( "," );
  }
  }

gemma-core/src/main/java/ubic/gemma/core/analysis/report/ExpressionExperimentReportServiceImpl.java

@@ -355,9 +355,12 @@ public void recalculateBatchInfo() {
  try {
  List<AuditEvent> events = auditEventService.getEvents( ee );
  // Don't update if the only recent event was another BatchProblemsUpdateEvent
- if ( events != null && !events.isEmpty() && BatchProblemsUpdateEvent.class
- .isAssignableFrom( events.get( events.size() - 1 ).getEventType().getClass() ) ) {
- continue;
+ if ( events != null && !events.isEmpty() ) {
+ AuditEvent ae = events.get( events.size() - 1 );
+ if ( ae.getEventType() != null && BatchProblemsUpdateEvent.class
+ .isAssignableFrom( ae.getEventType().getClass() ) ) {
+ continue;
+ }
  }
  recalculateExperimentBatchInfo( ee );
  } catch ( Exception e ) {

gemma-core/src/main/java/ubic/gemma/core/analysis/sequence/ProbeMapperConfig.java

@@ -84,7 +84,7 @@ public class ProbeMapperConfig {
  private double nonSpecificSiteCountThreshold = ProbeMapperConfig.NON_SPECIFIC_SITE_THRESHOLD;
  private boolean useEnsembl = false;
  private boolean useEsts = false;
- private boolean useKnownGene = true; // as of rn6, cannot be used for rat genome
+ private boolean useKnownGene = true; // as of rn6 and rn7, cannot be used for rat genome
  private boolean useMiRNA = false; // just confuses things.
  private boolean useMrnas = false; // doesn't add much.
  private boolean useRefGene = true;

gemma-core/src/main/java/ubic/gemma/core/analysis/sequence/ProbeMapperImpl.java

@@ -48,7 +48,7 @@
 @Component
 public class ProbeMapperImpl implements ProbeMapper {
 
- private static final int MAX_WARNINGS = 100;
+ private static final int MAX_WARNINGS = 10;
  private final Log log = LogFactory.getLog( ProbeMapperImpl.class.getName() );
  private final ThreePrimeDistanceMethod threePrimeMethod = ThreePrimeDistanceMethod.RIGHT;
 

gemma-core/src/main/java/ubic/gemma/core/analysis/service/ExpressionAnalysisResultSetFileServiceImpl.java

@@ -1,6 +1,8 @@
 package ubic.gemma.core.analysis.service;
 
+import lombok.extern.apachecommons.CommonsLog;
 import org.apache.commons.csv.CSVPrinter;
+import org.apache.commons.lang3.tuple.Pair;
 import org.springframework.stereotype.Service;
 import ubic.gemma.model.analysis.expression.diff.ContrastResult;
 import ubic.gemma.model.analysis.expression.diff.DifferentialExpressionAnalysisResult;
@@ -15,7 +17,10 @@
 import java.util.*;
 import java.util.stream.Collectors;
 
+import static java.util.function.Function.identity;
+
 @Service
+@CommonsLog
 public class ExpressionAnalysisResultSetFileServiceImpl extends AbstractTsvFileService<ExpressionAnalysisResultSet> implements ExpressionAnalysisResultSetFileService {
 
  @Override
@@ -27,52 +32,72 @@ public void writeTsvToAppendable( ExpressionAnalysisResultSet analysisResultSet,
  // add the basic columns
  List<String> header = new ArrayList<>( Arrays.asList( "id", "probe_id", "probe_name", "gene_id", "gene_name", "gene_ncbi_id", "gene_official_symbol", "gene_official_name", "pvalue", "corrected_pvalue", "rank" ) );
 
+ // this is the order the factor values are displayed
+ Comparator<ContrastResult> contrastResultComparator = Comparator
+ .comparing( ContrastResult::getFactorValue, Comparator.nullsLast( Comparator.comparing( FactorValue::getId ) ) )
+ .thenComparing( ContrastResult::getSecondFactorValue, Comparator.nullsLast( Comparator.comparing( FactorValue::getId ) ) );
+
  // we need to peek in the contrast result to understand factor value interactions
  // i.e. interaction between genotype and time point might result in a contrast_male_3h column, although we would
  // use factor value IDs in the actual column name which might result in something like contrast_1292_2938
- final Collection<ContrastResult> firstContrastResults = analysisResultSet.getResults().stream()
+ final List<ContrastResult> firstContrastResults = analysisResultSet.getResults().stream()
  .findFirst()
  .map( DifferentialExpressionAnalysisResult::getContrasts )
- .orElse( Collections.emptyList() );
-
- // this is the order the factor values are displayed
- Comparator<ContrastResult> contrastResultComparator = Comparator
- .comparing( ContrastResult::getFactorValue, Comparator.nullsLast( Comparator.comparing( FactorValue::getId ) ) )
- .thenComparing( ContrastResult::getSecondFactorValue, Comparator.nullsLast( Comparator.comparing( FactorValue::getId ) ) );
+ .orElse( Collections.emptyList() )
+ .stream().sorted( contrastResultComparator )
+ .collect( Collectors.toList() );
 
- firstContrastResults.stream().sorted( contrastResultComparator ).forEachOrdered( contrastResult -> {
+ for ( ContrastResult contrastResult : firstContrastResults ) {
  String contrastResultPrefix = "contrast"
  + ( contrastResult.getFactorValue() != null ? "_" + contrastResult.getFactorValue().getId() : "" )
  + ( contrastResult.getSecondFactorValue() != null ? "_" + contrastResult.getSecondFactorValue().getId() : "" );
  header.addAll( Arrays.asList(
  contrastResultPrefix + "_log2fc",
  contrastResultPrefix + "_tstat",
  contrastResultPrefix + "_pvalue" ) );
- } );
+ }
+
+ List<Pair<FactorValue, FactorValue>> factors = firstContrastResults.stream().sorted( contrastResultComparator )
+ .map( cr -> Pair.of( cr.getFactorValue(), cr.getSecondFactorValue() ) )
+ .collect( Collectors.toList() );
 
- CSVPrinter printer = getTsvFormatBuilder( "Experimental factors: " + experimentalFactorsMetadata )
+ try ( CSVPrinter printer = getTsvFormatBuilder( "Experimental factors: " + experimentalFactorsMetadata )
  .setHeader( header.toArray( new String[0] ) )
  .build()
- .print( appendable );
- for ( DifferentialExpressionAnalysisResult analysisResult : analysisResultSet.getResults() ) {
- final List<Gene> genes = result2Genes.getOrDefault( analysisResult, Collections.emptyList() );
- final List<Object> record = new ArrayList<>( Arrays.asList( analysisResult.getId(),
- analysisResult.getProbe().getId(),
- analysisResult.getProbe().getName(),
- genes.stream().map( Gene::getId ).map( String::valueOf ).collect( Collectors.joining( getSubDelimiter() ) ),
- genes.stream().map( Gene::getName ).collect( Collectors.joining( getSubDelimiter() ) ),
- genes.stream().map( Gene::getNcbiGeneId ).map( String::valueOf ).collect( Collectors.joining( getSubDelimiter() ) ),
- genes.stream().map( Gene::getOfficialSymbol ).collect( Collectors.joining( getSubDelimiter() ) ),
- genes.stream().map( Gene::getOfficialName ).collect( Collectors.joining( getSubDelimiter() ) ),
- format( analysisResult.getPvalue() ),
- format( analysisResult.getCorrectedPvalue() ),
- format( analysisResult.getRank() ) ) );
- analysisResult.getContrasts().stream()
- .sorted( contrastResultComparator )
- .forEachOrdered( contrastResult -> record.addAll( Arrays.asList( format( contrastResult.getLogFoldChange() ), format( contrastResult.getTstat() ), format( contrastResult.getPvalue() ) ) ) );
- printer.printRecord( record );
+ .print( appendable ) ) {
+ for ( DifferentialExpressionAnalysisResult analysisResult : analysisResultSet.getResults() ) {
+ final List<Gene> genes = result2Genes.getOrDefault( analysisResult, Collections.emptyList() );
+ final List<Object> record = new ArrayList<>( Arrays.asList( analysisResult.getId(),
+ analysisResult.getProbe().getId(),
+ analysisResult.getProbe().getName(),
+ genes.stream().map( Gene::getId ).map( String::valueOf ).collect( Collectors.joining( getSubDelimiter() ) ),
+ genes.stream().map( Gene::getName ).collect( Collectors.joining( getSubDelimiter() ) ),
+ genes.stream().map( Gene::getNcbiGeneId ).map( String::valueOf ).collect( Collectors.joining( getSubDelimiter() ) ),
+ genes.stream().map( Gene::getOfficialSymbol ).collect( Collectors.joining( getSubDelimiter() ) ),
+ genes.stream().map( Gene::getOfficialName ).collect( Collectors.joining( getSubDelimiter() ) ),
+ format( analysisResult.getPvalue() ),
+ format( analysisResult.getCorrectedPvalue() ),
+ format( analysisResult.getRank() ) ) );
+ Map<Pair<FactorValue, FactorValue>, ContrastResult> contrastsByFirstAndSecondFactorValue = analysisResult.getContrasts().stream()
+ .collect( Collectors.toMap( fv -> Pair.of( fv.getFactorValue(), fv.getSecondFactorValue() ), identity() ) );
+ // render contrast results in the same order than the first row and handle possibly missing columns
+ for ( ContrastResult contrastResult : firstContrastResults ) {
+ ContrastResult cr = contrastsByFirstAndSecondFactorValue.get( Pair.of( contrastResult.getFactorValue(), contrastResult.getSecondFactorValue() ) );
+ if ( cr != null ) {
+ record.add( format( cr.getLogFoldChange() ) );
+ record.add( format( cr.getTstat() ) );
+ record.add( format( cr.getPvalue() ) );
+ } else {
+ log.warn( String.format( "%s is missing contrast result for [%s, %s]. The corresponding column in the TSV will be treated as NaN.",
+ analysisResult, contrastResult.getFactorValue(), contrastResult.getSecondFactorValue() ) );
+ record.add( "" );
+ record.add( "" );
+ record.add( "" );
+ }
+ }
+ printer.printRecord( record );
+ }
  }
-
  }
 
  private String formatExperimentalFactor( ExperimentalFactor experimentalFactor ) {

gemma-core/src/main/java/ubic/gemma/core/analysis/util/ExperimentalDesignUtils.java

@@ -178,8 +178,8 @@ public static boolean isBatch( ExperimentalFactor ef ) {
  return false;
  }
  Characteristic category = ef.getCategory();
- return ef.getName().equals( ExperimentalDesignUtils.BATCH_FACTOR_NAME ) && category.getCategory()
- .equals( ExperimentalDesignUtils.BATCH_FACTOR_CATEGORY_NAME );
+ return ef.getName().equals( ExperimentalDesignUtils.BATCH_FACTOR_NAME ) &&
+ ( category != null && category.getCategory().equals( ExperimentalDesignUtils.BATCH_FACTOR_CATEGORY_NAME ) );
  }
 
  /**

gemma-core/src/main/java/ubic/gemma/core/apps/ArrayDesignBlatCli.java

@@ -143,7 +143,7 @@ protected void doWork() throws Exception {
 
  for ( ArrayDesign arrayDesign : this.getArrayDesignsToProcess() ) {
  if ( !this.shouldRun( skipIfLastRunLaterThan, arrayDesign, ArrayDesignSequenceAnalysisEvent.class ) ) {
- AbstractCLI.log.warn( arrayDesign + " was last run more recently than " + skipIfLastRunLaterThan );
+ AbstractCLI.log.warn( arrayDesign + " does not meet criteria to be processed");
  return;
  }