Merge branch 'hotfix-1.30.4'

gemma-cli/pom.xml

@@ -3,7 +3,7 @@
  <parent>
  <artifactId>gemma</artifactId>
  <groupId>gemma</groupId>
- <version>1.30.3</version>
+ <version>1.30.4</version>
  </parent>
  <modelVersion>4.0.0</modelVersion>
  <artifactId>gemma-cli</artifactId>

gemma-cli/src/main/config/log4j-dev.properties

@@ -0,0 +1,62 @@
+# This is the configuration for logging displayed in the CLIs
+# If you are developing Gemma from an IDE, consider setting -Dlog4j.configuration=file:gemma-core/src/config/log4j.properties
+# to pickup this file, or even better: define your own.
+
+#################################################################################################
+# APPENDERS
+#################################################################################################
+
+# Basic appender to log to the console.
+log4j.appender.stderr=org.apache.log4j.ConsoleAppender
+log4j.appender.stderr.target=System.err
+log4j.appender.stderr.layout=org.apache.log4j.PatternLayout
+log4j.appender.stderr.layout.ConversionPattern=%d %highlight{%p} %style{%pid}{magenta} [%t] %style{%C{1.}.%M(%L)}{cyan} | %m%n
+
+#####################################################################################################
+# LOGGING LEVELS
+#####################################################################################################
+
+# root
+log4j.rootLogger=INFO,stderr
+
+# baseCode
+log4j.logger.ubic.basecode=INFO
+log4j.logger.ubic.basecode.ontology.model.PropertyFactory=ERROR
+log4j.logger.ubic.basecode.ontology.providers=WARN
+
+# Gemma
+log4j.logger.ubic.gemma=INFO
+
+# Gemma (externally contributed components)
+log4j.logger.ubic.gemma.contrib=INFO
+
+############# THIRD PARTY CLASSES #####################
+# Control logging for other packages
+log4j.logger.org=WARN
+log4j.logger.net=WARN
+log4j.logger.com=WARN
+
+# suppress inspection "SpellCheckingInspection"
+log4j.logger.com.opensymphony.oscache=WARN
+
+# Spring
+log4j.logger.org.springframework=WARN
+log4j.logger.org.springframework.beans.GenericTypeAwarePropertyDescriptor=FATAL
+log4j.logger.org.springframework.security.access.event.LoggerListener=INFO
+log4j.logger.org.springframework.security.authentication.event.LoggerListener=ERROR
+
+# Apache Commons
+log4j.logger.org.apache.commons=WARN
+
+# Hibernate
+log4j.logger.org.hibernate=WARN
+# Avoid warnings about expired objects.
+log4j.logger.org.hibernate.cache.ReadWriteCache=ERROR
+# Avoid warnings about collection fail-safe cleanup.
+log4j.logger.org.hibernate.engine.loading.LoadContexts=ERROR
+log4j.logger.net.sf.ehcache.hibernate.strategy.AbstractReadWriteEhcacheAccessStrategy=ERROR
+log4j.logger.org.hibernate.cache.ehcache.internal.strategy.AbstractReadWriteEhcacheAccessStrategy=ERROR
+
+# Hibernate Search
+# This is necessary to see progress in IndexGemmaCLI command
+log4j.logger.org.hibernate.search.impl.SimpleIndexingProgressMonitor=INFO

gemma-cli/src/main/config/log4j.properties

@@ -10,7 +10,7 @@
 log4j.appender.stderr=org.apache.log4j.ConsoleAppender
 log4j.appender.stderr.target=System.err
 log4j.appender.stderr.layout=org.apache.log4j.PatternLayout
-log4j.appender.stderr.layout.ConversionPattern=%d %highlight{%p} %style{%pid}{magenta} [%t] %style{%C.%M(%L)}{cyan} | %m%n
+log4j.appender.stderr.layout.ConversionPattern=%d %highlight{%p} %style{%pid}{magenta} [%t] %style{%C{1.}.%M(%L)}{cyan} | %m%n
 
 # Log to a file. Note that if File is a relative path, the output file goes wherever the application JVM was started from.
 # Define gemma.log.dir as a parameter to your JAVA_OPTS and make sure this is passed to java when you start it.

gemma-cli/src/main/java/ubic/gemma/core/apps/DeleteExperimentsCli.java

@@ -47,7 +47,6 @@ protected void buildOptions( Options options ) {
  Option.builder( "a" ).longOpt( "array" )
  .desc( "Delete platform(s) instead; you must delete associated experiments first; other options are ignored" )
  .argName( "comma-delimited list of platform short names" ).hasArg().build() );
- super.suppressAllOption();
  }
 
  @Override

gemma-cli/src/main/java/ubic/gemma/core/apps/ExpressionExperimentManipulatingCLI.java

@@ -22,6 +22,7 @@
 import org.apache.commons.cli.Option;
 import org.apache.commons.cli.Options;
 import org.apache.commons.lang3.StringUtils;
+import org.springframework.beans.factory.annotation.Autowired;
 import ubic.gemma.core.apps.GemmaCLI.CommandGroup;
 import ubic.gemma.core.genome.gene.service.GeneService;
 import ubic.gemma.core.search.SearchException;
@@ -36,77 +37,69 @@
 import ubic.gemma.model.expression.experiment.ExpressionExperiment;
 import ubic.gemma.model.genome.Gene;
 import ubic.gemma.model.genome.Taxon;
-import ubic.gemma.persistence.service.common.auditAndSecurity.AuditEventService;
 import ubic.gemma.persistence.service.expression.arrayDesign.ArrayDesignService;
 import ubic.gemma.persistence.service.expression.experiment.ExpressionExperimentService;
 import ubic.gemma.persistence.service.expression.experiment.ExpressionExperimentSetService;
 import ubic.gemma.persistence.service.genome.taxon.TaxonService;
 
 import java.io.IOException;
-import java.util.*;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
 import java.util.stream.Collectors;
 
 /**
  * Base class for CLIs that needs one or more expression experiment as an input. It offers the following ways of reading
  * them in:
  * <ul>
- * <li>All EEs
- * <li>All EEs for a particular taxon.
+ * <li>All EEs (if -all is supplied)</li>
+ * <li>All EEs for a particular taxon.</li>
  * <li>A specific ExpressionExperimentSet, identified by name</li>
- * <li>A comma-delimited list of one or more EEs identified by short name given on the command line
- * <li>From a file, with one short name per line.
- * <li>EEs matching a query string (e.g., 'brain')
+ * <li>A comma-delimited list of one or more EEs identified by short name given on the command line</li>
+ * <li>From a file, with one short name per line.</li>
+ * <li>EEs matching a query string (e.g., 'brain')</li>
  * <li>(Optional) 'Auto' mode, in which experiments to analyze are selected automatically based on their workflow state.
- * This can be enabled and modified by subclasses who override the "needToRun" method.
- * <li>All EEs that were last processed after a given date, similar to 'auto' otherwise.
+ * This can be enabled and modified by subclasses who override the "needToRun" method.</li>
+ * <li>All EEs that were last processed after a given date, similar to 'auto' otherwise.</li>
  * </ul>
  * Some of these options can be (or should be) combined, and modified by a (optional) "force" option, and will have
  * customized behavior.
+ * <p>
  * In addition, EEs can be excluded based on a list given in a separate file.
  *
  * @author Paul
  */
 public abstract class ExpressionExperimentManipulatingCLI extends AbstractCLIContextCLI {
- ExpressionExperimentService eeService;
- Set<BioAssaySet> expressionExperiments = new HashSet<>();
- private boolean allowProcessingAll = true;
 
- public Set<BioAssaySet> getExpressionExperiments() {
- return expressionExperiments;
- }
-
- protected ExpressionExperimentService getEeService() {
- return eeService;
- }
-
- protected Taxon getTaxon() {
- return taxon;
- }
-
- protected void setTaxon( Taxon taxon ) {
- this.taxon = taxon;
- }
-
- protected TaxonService getTaxonService() {
- return taxonService;
- }
-
- protected GeneService getGeneService() {
- return geneService;
- }
-
- boolean force = false;
- private Taxon taxon = null;
+ @Autowired
+ protected ExpressionExperimentService eeService;
+ @Autowired
+ private ExpressionExperimentSetService expressionExperimentSetService;
+ @Autowired
  private TaxonService taxonService;
+ @Autowired
  private GeneService geneService;
+ @Autowired
  private SearchService searchService;
+ @Autowired
+ private ArrayDesignService arrayDesignService;
+
+ protected final Set<BioAssaySet> expressionExperiments = new HashSet<>();
+ /**
+ * Taxon used for filtering EEs.
+ */
+ private Taxon taxon = null;
+ protected boolean force = false;
+
+ protected ExpressionExperimentManipulatingCLI() {
+ }
 
  @Override
  public CommandGroup getCommandGroup() {
  return CommandGroup.EXPERIMENT;
  }
 
- @SuppressWarnings("AccessStaticViaInstance") // Cleaner like this
  @Override
  protected void buildOptions( Options options ) {
  Option expOption = Option.builder( "e" ).hasArg().argName( "shortname" ).desc(
@@ -116,6 +109,8 @@ protected void buildOptions( Options options ) {
 
  options.addOption( expOption );
 
+ options.addOption( "all", false, "Process all expression experiments" );
+
  Option eeFileListOption = Option.builder( "f" ).hasArg().argName( "file" ).desc(
  "File with list of short names or IDs of expression experiments (one per line; use instead of '-e')" )
  .longOpt( "eeListfile" ).build();
@@ -155,11 +150,6 @@ protected Gene findGeneByOfficialSymbol( String symbol, Taxon t ) {
 
  @Override
  protected void processOptions( CommandLine commandLine ) {
- eeService = this.getBean( ExpressionExperimentService.class );
- geneService = this.getBean( GeneService.class );
- taxonService = this.getBean( TaxonService.class );
- searchService = this.getBean( SearchService.class );
- this.auditEventService = this.getBean( AuditEventService.class );
  if ( commandLine.hasOption( 't' ) ) {
  this.taxon = this.setTaxonByName( commandLine, taxonService );
  }
@@ -168,42 +158,37 @@ protected void processOptions( CommandLine commandLine ) {
  this.force = true;
  }
 
- if ( commandLine.hasOption( "eeset" ) ) {
+ if ( commandLine.hasOption( "all" ) ) {
+ log.warn( "Loading all expression experiments, this might take a while..." );
+ this.expressionExperiments.addAll( eeService.loadAll() );
+ } else if ( commandLine.hasOption( "eeset" ) ) {
  this.experimentsFromEeSet( commandLine.getOptionValue( "eeset" ) );
  } else if ( commandLine.hasOption( 'e' ) ) {
  this.experimentsFromCliList( commandLine );
  } else if ( commandLine.hasOption( 'f' ) ) {
  String experimentListFile = commandLine.getOptionValue( 'f' );
  AbstractCLI.log.info( "Reading experiment list from " + experimentListFile );
  try {
- this.expressionExperiments = this.readExpressionExperimentListFile( experimentListFile );
+ this.expressionExperiments.addAll( this.readExpressionExperimentListFile( experimentListFile ) );
  } catch ( IOException e ) {
  throw new RuntimeException( e );
  }
  } else if ( commandLine.hasOption( 'q' ) ) {
  AbstractCLI.log.info( "Processing all experiments that match query " + commandLine.getOptionValue( 'q' ) );
  try {
- this.expressionExperiments = this.findExpressionExperimentsByQuery( commandLine.getOptionValue( 'q' ) );
+ this.expressionExperiments.addAll( this.findExpressionExperimentsByQuery( commandLine.getOptionValue( 'q' ) ) );
  } catch ( SearchException e ) {
  log.error( "Failed to retrieve EEs for the passed query via -q.", e );
  }
- } else if ( taxon != null ) {
- if ( !commandLine.hasOption( "dataFile" ) ) {
- AbstractCLI.log.info( "Processing all experiments for " + taxon.getCommonName() );
- this.expressionExperiments = new HashSet<BioAssaySet>( eeService.findByTaxon( taxon ) );
- }
  } else {
- if ( !commandLine.hasOption( "dataFile" ) && allowProcessingAll ) {
- AbstractCLI.log.info( "Processing all experiments (further filtering may modify)" );
- this.expressionExperiments = new HashSet<BioAssaySet>( eeService.loadAll() );
- }
+ throw new IllegalArgumentException( "At least one of -all, -e, -eeset, -f, or -q must be provided." );
  }
 
- if ( commandLine.hasOption( 'x' ) ) {
+ if ( commandLine.hasOption( 'x' ) && !expressionExperiments.isEmpty() ) {
  this.excludeFromFile( commandLine );
  }
 
- if ( expressionExperiments != null && expressionExperiments.size() > 0 && !force ) {
+ if ( !force && !expressionExperiments.isEmpty() ) {
 
  if ( commandLine.hasOption( AbstractCLI.AUTO_OPTION_NAME ) ) {
  this.autoSeek = true;
@@ -228,22 +213,17 @@ protected void processOptions( CommandLine commandLine ) {
  }
  }
 
- if ( expressionExperiments != null && expressionExperiments.size() > 1 ) {
- AbstractCLI.log.info( "Final list: " + this.expressionExperiments.size()
- + " expressionExperiments (futher filtering may modify)" );
- } else if ( expressionExperiments == null || expressionExperiments.size() == 0 ) {
- if ( commandLine.hasOption( "dataFile" ) ) {
- // AbstractCLI.log.info( "Expression matrix from data file selected" );
- } else {
- // AbstractCLI.log.info( "No experiments selected" );
- }
+ if ( expressionExperiments.isEmpty() ) {
+ log.warn( "No expression experiments matched the given options." );
+ } else if ( expressionExperiments.size() == 1 ) {
+ AbstractCLI.log.info( "Final dataset: " + expressionExperiments.iterator().next() );
+ } else {
+ AbstractCLI.log.info( String.format( "Final list: %d expression experiments", this.expressionExperiments.size() ) );
  }
-
  }
 
  void addForceOption( Options options ) {
  String desc = "Ignore other reasons for skipping experiments (e.g., trouble) and overwrite existing data (see documentation for this tool to see exact behavior if not clear)";
- @SuppressWarnings("static-access")
  Option forceOption = Option.builder( "force" ).longOpt( "force" ).desc( desc ).build();
  options.addOption( forceOption );
  }
@@ -256,7 +236,7 @@ private void excludeFromFile( CommandLine commandLine ) {
  } catch ( IOException e ) {
  throw new RuntimeException( e );
  }
- assert expressionExperiments.size() > 0;
+ assert !expressionExperiments.isEmpty();
 
  int before = expressionExperiments.size();
 
@@ -277,10 +257,9 @@ private void experimentsFromCliList( CommandLine commandLine ) {
  AbstractCLI.log.warn( shortName + " not found" );
  continue;
  }
- eeService.thawLite( expressionExperiment );
- expressionExperiments.add( expressionExperiment );
+ expressionExperiments.add( eeService.thawLite( expressionExperiment ) );
  }
- if ( expressionExperiments.size() == 0 ) {
+ if ( expressionExperiments.isEmpty() ) {
  throw new RuntimeException( "There were no valid experimnents specified" );
  }
  }
@@ -291,16 +270,14 @@ private void experimentsFromEeSet( String optionValue ) {
  throw new IllegalArgumentException( "Please provide an eeset name" );
  }
 
- ExpressionExperimentSetService expressionExperimentSetService = this
- .getBean( ExpressionExperimentSetService.class );
  Collection<ExpressionExperimentSet> sets = expressionExperimentSetService.findByName( optionValue );
  if ( sets.size() > 1 ) {
  throw new IllegalArgumentException( "More than on EE set has name '" + optionValue + "'" );
- } else if ( sets.size() == 0 ) {
+ } else if ( sets.isEmpty() ) {
  throw new IllegalArgumentException( "No EE set has name '" + optionValue + "'" );
  }
  ExpressionExperimentSet set = sets.iterator().next();
- this.expressionExperiments = new HashSet<>( set.getExperiments() );
+ this.expressionExperiments.addAll( set.getExperiments() );
 
  }
 
@@ -312,9 +289,9 @@ private Set<BioAssaySet> findExpressionExperimentsByQuery( String query ) throws
 
  // explicitly support one case
  if ( query.matches( "GPL[0-9]+" ) ) {
- ArrayDesign ad = this.getBean( ArrayDesignService.class ).findByShortName( query );
+ ArrayDesign ad = arrayDesignService.findByShortName( query );
  if ( ad != null ) {
- Collection<ExpressionExperiment> ees2 = this.getBean( ArrayDesignService.class ).getExpressionExperiments( ad );
+ Collection<ExpressionExperiment> ees2 = arrayDesignService.getExpressionExperiments( ad );
  ees.addAll( ees2 );
  log.info( ees.size() + " experiments matched to platform " + ad );
  }
@@ -391,7 +368,7 @@ private Set<BioAssaySet> readExpressionExperimentListFile( String fileName ) thr
  * @return a collection of troubled experiemnt, or an empty set of non are
  */
  private Set<BioAssaySet> getTroubledExpressionExperiments() {
- if ( expressionExperiments == null || expressionExperiments.size() == 0 ) {
+ if ( expressionExperiments.isEmpty() ) {
  AbstractCLI.log.warn( "No experiments to remove troubled from" );
  return Collections.emptySet();
  }
@@ -401,12 +378,4 @@ private Set<BioAssaySet> getTroubledExpressionExperiments() {
  .filter( ee -> ee.getCurationDetails().getTroubled() )
  .collect( Collectors.toSet() );
  }
-
- /**
- * Disable the ability for this CLI to process all experiments when no other specification is given.
- * The user must explicitly define the experiments to be processed.
- */
- protected void suppressAllOption() {
- this.allowProcessingAll = false;
- }
 }