Merge branch 'hotfix-1.31.8'

gemma-cli/pom.xml

@@ -3,7 +3,7 @@
  <parent>
  <artifactId>gemma</artifactId>
  <groupId>gemma</groupId>
- <version>1.31.7</version>
+ <version>1.31.8</version>
  </parent>
  <modelVersion>4.0.0</modelVersion>
  <artifactId>gemma-cli</artifactId>

gemma-cli/src/main/java/ubic/gemma/core/apps/AffyDataFromCelCli.java

@@ -165,10 +165,20 @@ protected void doWork() throws Exception {
  addSuccessObject( thawedEe );
  } else if ( asd.isMerged( Collections.singleton( ad.getId() ) ).get( ad.getId() ) ) {
  ad = asd.thawLite( ad );
- if ( GeoPlatform.isAffyPlatform( ad.getMergees().iterator().next().getShortName() ) ) {
+ ArrayDesign mergee = ad.getMergees().iterator().next();
+
+ // handle one level of sub-mergees....
+ if ( asd.isMerged( Collections.singleton( mergee.getId() ) ).get( mergee.getId() ) ) {
+ mergee = asd.thawLite( mergee );
+ mergee = asd.thawLite( mergee.getMergees().iterator().next() );
+ }
+
+ if ( GeoPlatform.isAffyPlatform( mergee.getShortName() ) ) {
  AbstractCLI.log.info( ad + " looks like Affy array made from merger of other platforms" );
  serv.reprocessAffyDataFromCel( thawedEe );
  addSuccessObject( thawedEe );
+ } else {
+ addErrorObject( ee, ad + " is not recognized as an Affymetrix platform. If this is a mistake, the Gemma configuration needs to be updated." );
  }
  } else {
 

gemma-cli/src/main/java/ubic/gemma/core/apps/ArrayDesignSequenceManipulatingCli.java

@@ -265,7 +265,7 @@ private boolean needToAutoRun( ArrayDesign arrayDesign, Class<? extends ArrayDes
  AuditEvent lastEventOfCurrentType = eventsOfCurrentType.get( eventsOfCurrentType.size() - 1 );
  assert lastEventOfCurrentType != null;
 
- if ( lastEventOfCurrentType.getEventType().getClass().isAssignableFrom( eventClass ) ) {
+ if ( lastEventOfCurrentType.getEventType() != null && eventClass.isAssignableFrom( lastEventOfCurrentType.getEventType().getClass() ) ) {
  // then definitely don't run it. The last event was the same as the one we're trying to renew.
  AbstractCLI.log.debug( "Last event on " + arrayDesign + " was also a " + eventClass + ", skipping." );
  return false;

gemma-cli/src/main/java/ubic/gemma/core/apps/BlacklistCli.java

@@ -28,14 +28,14 @@
 import ubic.gemma.core.loader.expression.geo.service.GeoBrowser;
 import ubic.gemma.core.util.AbstractAuthenticatedCLI;
 import ubic.gemma.core.util.AbstractCLI;
-import ubic.gemma.model.common.description.DatabaseEntry;
-import ubic.gemma.model.common.description.ExternalDatabase;
 import ubic.gemma.model.blacklist.BlacklistedEntity;
-import ubic.gemma.model.blacklist.BlacklistedPlatform;
 import ubic.gemma.model.blacklist.BlacklistedExperiment;
+import ubic.gemma.model.blacklist.BlacklistedPlatform;
+import ubic.gemma.model.common.description.DatabaseEntry;
+import ubic.gemma.model.common.description.ExternalDatabase;
 import ubic.gemma.model.expression.experiment.ExpressionExperiment;
-import ubic.gemma.persistence.service.common.description.ExternalDatabaseService;
 import ubic.gemma.persistence.service.blacklist.BlacklistedEntityService;
+import ubic.gemma.persistence.service.common.description.ExternalDatabaseService;
 import ubic.gemma.persistence.service.expression.experiment.ExpressionExperimentService;
 
 import java.io.BufferedReader;
@@ -54,12 +54,20 @@
 public class BlacklistCli extends AbstractAuthenticatedCLI {
 
  private static final int MAX_RETRIES = 3;
- String fileName = null;
+
+ // accessions of file containing accessions to blacklist
+ private String accession = null;
+ private String fileName = null;
+
+ // reason (if adding to blacklist)
+ private String reason = null;
+
+ // remove from blacklist
  private boolean remove = false;
+
+ // proactive mode
  private boolean proactive = false;
  private Collection<String> platformsToScreen;
- private String reason = null;
- private String accession = null;
 
  @Override
  public CommandGroup getCommandGroup() {
@@ -124,7 +132,7 @@ protected void doWork() throws Exception {
  return;
  }
 
- if (reason.isEmpty()) {
+ if ( reason.isEmpty() ) {
  throw new IllegalArgumentException( "A reason for blacklisting must be provided for " + accession );
  }
 
@@ -356,46 +364,38 @@ private int fetchAndBlacklist( ExternalDatabase geo, GeoBrowser gbs, Blacklisted
 
  @Override
  protected void processOptions( CommandLine commandLine ) {
-
  if ( commandLine.hasOption( "accession" ) ) {
- if ( !commandLine.hasOption( "reason" ) && !commandLine.hasOption( "undo" ) ) {
- throw new IllegalArgumentException( "Must provide a reason for blacklisting (unless using -sundo)" );
+ if ( commandLine.hasOption( "pp" ) || commandLine.hasOption( "file" ) ) {
+ throw new IllegalArgumentException( "The -accession option cannot be combined with -pp or -file" );
  }
-
- if ( commandLine.hasOption( "file" ) ) {
- throw new IllegalArgumentException( "The accession option cannot be combined with the file option" );
-
+ if ( !commandLine.hasOption( "undo" ) && StringUtils.isBlank( commandLine.getOptionValue( "reason" ) ) ) {
+ throw new IllegalArgumentException( "Must provide a reason for blacklisting (unless using -undo)" );
  }
-
  this.accession = commandLine.getOptionValue( "accession" );
  this.reason = commandLine.getOptionValue( "reason" );
- return;
- }
-
-
- if ( commandLine.hasOption( "pp" ) ) {
- if ( this.remove || this.fileName != null ) {
+ this.remove = commandLine.hasOption( "undo" );
+ } else if ( commandLine.hasOption( "pp" ) ) {
+ if ( commandLine.hasOption( "accession" ) || commandLine.hasOption( "file" ) ) {
  throw new IllegalArgumentException( "The pp option cannot be combined with others" );
  }
+ if ( commandLine.hasOption( "undo" ) ) {
+ throw new IllegalArgumentException( "The -pp option cannot be compiled with -undo." );
+ }
  this.proactive = true;
-
  if ( commandLine.hasOption( "a" ) ) {
  this.platformsToScreen = Arrays.asList( StringUtils.split( commandLine.getOptionValue( "a" ) ) );
  }
-
- return;
- }
-
- if ( commandLine.hasOption( "file" ) ) {
+ } else if ( commandLine.hasOption( "file" ) ) {
+ if ( commandLine.hasOption( "accession" ) || commandLine.hasOption( "pp" ) ) {
+ throw new IllegalArgumentException( "The -file option cannot be combined with -pp or -accession." );
+ }
+ if ( commandLine.hasOption( "reason" ) ) {
+ throw new IllegalArgumentException( "The -file option cannot be combined with -reason, use the second column to specify blacklisting reason." );
+ }
  this.fileName = commandLine.getOptionValue( "file" );
+ this.remove = commandLine.hasOption( "undo" );
  } else {
- throw new IllegalArgumentException( "Must provide an input file" );
+ throw new IllegalArgumentException( "Must provide one of -accession, -pp or -file option." );
  }
-
- if ( commandLine.hasOption( "undo" ) ) {
- this.remove = true;
- }
-
  }
-
 }

gemma-cli/src/main/java/ubic/gemma/core/apps/ExpressionExperimentPlatformSwitchCli.java

@@ -77,7 +77,7 @@ public String getShortDesc() {
  protected void buildOptions( Options options ) {
  super.buildOptions( options );
  Option arrayDesignOption = Option.builder( "a" ).hasArg().argName( "Array design" ).desc(
- "Array design name (or short name) - no need to specifiy if the platforms used by the EE are merged" )
+ "Array design short name to be switched to - no need to specify if the platforms used by the EE are merged" )
  .longOpt( "array" ).build();
  options.addOption( arrayDesignOption );
  this.addForceOption( options );
@@ -115,7 +115,6 @@ private void processExperiment( ExpressionExperiment ee ) {
 
  /**
  * @param name of the array design to find.
- * @param arrayDesignService the arrayDesignService to use for the AD retrieval
  * @return an array design, if found. Bails otherwise with an error exit code
  */
  private ArrayDesign locateArrayDesign( String name ) {

gemma-cli/src/main/java/ubic/gemma/core/apps/ExternalDatabaseOverviewCli.java

@@ -69,7 +69,10 @@ private String summarize( AuditTrail auditTrail ) {
  }
 
  private String summarize( AuditEvent auditEvent ) {
- String s = String.format( "%s: %s by %s", auditEvent.getDate(), auditEvent.getAction(), auditEvent.getPerformer().getUserName() );
+ String s = String.format( "%s: %s", auditEvent.getDate(), auditEvent.getAction() );
+ if ( auditEvent.getPerformer() != null ) {
+ s += " by " + auditEvent.getPerformer().getUserName();
+ }
  if ( auditEvent.getNote() != null ) {
  s += "\n\t" + auditEvent.getNote();
  }

gemma-cli/src/main/java/ubic/gemma/core/apps/GemmaCLI.java

@@ -19,7 +19,6 @@
 package ubic.gemma.core.apps;
 
 import lombok.Value;
-import lombok.extern.apachecommons.CommonsLog;
 import org.apache.commons.cli.*;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.exception.ExceptionUtils;
@@ -50,10 +49,16 @@
  *
  * @author paul
  */
-@SuppressWarnings({ "unused", "WeakerAccess" }) // Possible external use
-@CommonsLog
 public class GemmaCLI {
 
+ static {
+ // set this as early as possible, otherwise logging might be broken
+ if ( System.getProperty( "gemma.log.dir" ) == null ) {
+ System.err.println( "The 'gemma.log.dir' system property is not set, will default to the current folder." );
+ System.setProperty( "gemma.log.dir", "." );
+ }
+ }
+
  private static final String
  HELP_OPTION = "h",
  HELP_ALL_OPTION = "ha",
@@ -184,7 +189,8 @@ public static void main( String[] args ) {
  try {
  cliInstance = ( CLI ) beanClass.newInstance();
  } catch ( InstantiationException | IllegalAccessException e2 ) {
- log.warn( String.format( "Failed to create %s using reflection, will have to create a complete bean to extract its metadata.", beanClass.getName() ), e2 );
+ System.err.printf( "Failed to create %s using reflection, will have to create a complete bean to extract its metadata.%n", beanClass.getName() );
+ e2.printStackTrace( System.err );
  cliInstance = ctx.getBean( beanName, CLI.class );
  }
  String commandName = cliInstance.getCommandName();
@@ -291,7 +297,7 @@ public static void main( String[] args ) {
  static String getOptStringForLogging( Object[] argsToPass ) {
  Matcher matcher = PASSWORD_IN_CLI_MATCHER.matcher( StringUtils.join( argsToPass, " " ) );
  if ( matcher.matches() ) {
- log.warn( "It seems that you still supply the -p/--password argument through the CLI. This feature has been removed for security purposes in Gemma 1.29." );
+ System.err.println( "It seems that you still supply the -p/--password argument through the CLI. This feature has been removed for security purposes in Gemma 1.29." );
  }
  return matcher.replaceAll( "$1 XXXXXX" );
  }

gemma-core/pom.xml

@@ -3,7 +3,7 @@
  <parent>
  <artifactId>gemma</artifactId>
  <groupId>gemma</groupId>
- <version>1.31.7</version>
+ <version>1.31.8</version>
  </parent>
  <modelVersion>4.0.0</modelVersion>
  <artifactId>gemma-core</artifactId>

gemma-core/src/main/java/ubic/gemma/core/analysis/expression/coexpression/links/LinkAnalysisServiceImpl.java

@@ -59,6 +59,7 @@
 import ubic.gemma.persistence.service.common.auditAndSecurity.AuditTrailService;
 import ubic.gemma.persistence.service.expression.bioAssayData.ProcessedExpressionDataVectorService;
 import ubic.gemma.persistence.service.expression.designElement.CompositeSequenceService;
+import ubic.gemma.core.analysis.preprocess.batcheffects.ExpressionExperimentBatchInformationService;
 import ubic.gemma.persistence.service.expression.experiment.ExpressionExperimentService;
 
 import java.io.IOException;
@@ -102,6 +103,9 @@ public class LinkAnalysisServiceImpl implements LinkAnalysisService {
  @Autowired
  private ProcessedExpressionDataVectorService processedExpressionDataVectorService;
 
+ @Autowired
+ private ExpressionExperimentBatchInformationService expressionExperimentBatchInformationService;
+
  @Override
  public LinkAnalysis process( ExpressionExperiment ee, FilterConfig filterConfig,
  LinkAnalysisConfig linkAnalysisConfig ) {
@@ -441,9 +445,9 @@ private void qcCheck( LinkAnalysisConfig config, ExpressionExperiment ee ) throw
  }
 
  if ( config.isCheckForBatchEffect() ) {
- BatchEffectDetails batchEffect = eeService.getBatchEffectDetails( ee );
+ BatchEffectDetails batchEffect = expressionExperimentBatchInformationService.getBatchEffectDetails( ee );
 
- if ( batchEffect.getDataWasBatchCorrected() ) {
+ if ( batchEffect.dataWasBatchCorrected() ) {
  LinkAnalysisServiceImpl.log.info( "Data are batch-corrected" );
  return;
  }

gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/PreprocessingException.java

@@ -19,7 +19,7 @@
 
 /**
  * Allows us to catch preprocessing errors and handle them correctly.
- *
+ * <p>
  * The main kind of preprocessing exceptions are {@link ubic.gemma.core.analysis.preprocess.filter.FilteringException}
  * and {@link ubic.gemma.core.analysis.preprocess.batcheffects.BatchInfoPopulationException}.
  *
@@ -33,6 +33,10 @@ public PreprocessingException( ExpressionExperiment ee, String message ) {
  super( String.format( "Failed to pre-process %s: %s", ee.getShortName(), message ) );
  }
 
+ public PreprocessingException( ExpressionExperiment ee, String message, Throwable cause ) {
+ super( String.format( "Failed to pre-process %s: %s", ee.getShortName(), message ), cause );
+ }
+
  public PreprocessingException( ExpressionExperiment ee, Throwable cause ) {
  super( String.format( "Failed to pre-process %s: %s", ee.getShortName(), ExceptionUtils.getRootCauseMessage( cause ) ), cause );
  }

gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/VectorMergingServiceImpl.java

@@ -231,6 +231,7 @@ public void mergeVectors( ExpressionExperiment ee ) {
  // replace raw vectors with
  expressionExperimentService.removeAllRawDataVectors( ee );
  ee.getRawExpressionDataVectors().addAll( newVectors );
+ ee.getQuantitationTypes().addAll( qt2Vec.keySet() );
 
  // remove processed vectors
  expressionExperimentService.removeProcessedDataVectors( ee );