Merge branch 'release-1.30.0'

.gitignore

@@ -116,8 +116,9 @@ gemma-core/src/main/uml/*.bak
 
 # IntelliJ
 /out/
-.idea/
+.idea/*
 !.idea/codeStyles/
+!.idea/runConfigurations/
 *.iml
 
 # Eclipse
@@ -129,3 +130,8 @@ gemma-core/src/main/uml/*.bak
 # JIRA plugin
 atlassian-ide-plugin.xml
 !/gemma-core/src/main/
+
+# a user-defined Gemma.properties file
+/Gemma.properties
+/gemma-data
+/gemma-logs

gemma-cli/pom.xml

@@ -3,7 +3,7 @@
  <parent>
  <artifactId>gemma</artifactId>
  <groupId>gemma</groupId>
- <version>1.29.16</version>
+ <version>1.30.0</version>
  </parent>
  <modelVersion>4.0.0</modelVersion>
  <artifactId>gemma-cli</artifactId>

gemma-cli/src/main/config/log4j.properties

@@ -37,14 +37,6 @@ log4j.appender.errorFile.layout.ConversionPattern=[Gemma - %t (%d)] %p %C.%M(%L)
 log4j.appender.errorFile.DatePattern='.'yyyy-MM-dd
 log4j.appender.errorFile.Threshold=ERROR
 
-# Special appender for method execution metrics.
-log4j.appender.metricFile=org.apache.log4j.RollingFileAppender
-log4j.appender.metricFile.File=${gemma.log.dir}/gemma-metrics.log
-log4j.appender.metricFile.MaxFileSize=10000KB
-log4j.appender.metricFile.MaxBackupIndex=5
-log4j.appender.metricFile.layout=org.apache.log4j.PatternLayout
-log4j.appender.metricFile.layout.ConversionPattern=[Metrics - %d] %p | %m%n
-
 # Special appender for messages about annotation modifications.
 log4j.appender.annotationsFile=org.apache.log4j.RollingFileAppender
 log4j.appender.annotationsFile.File=${gemma.log.dir}/gemma-annotations.log
@@ -95,8 +87,6 @@ log4j.logger.ubic.gemma.core.association.phenotype=INFO,phenocartaFile
 log4j.logger.ubic.gemma.core.ontology.OntologyService=INFO,annotationsFile
 log4j.logger.ubic.gemma.core.security.audit.AclAuditLogger=INFO,securityAuditFile
 log4j.logger.ubic.gemma.core.security.audit.AuditAdvice=INFO,auditFile
-log4j.logger.ubic.gemma.persistence.util.monitor.MonitorAdvice=INFO,metricFile
-log4j.logger.ubic.gemma.persistence.util.CommonsConfigurationPropertyPlaceholderConfigurer=WARN
 # Add this logger for AuditAdvice: Set to Info on production.
 log4j.logger.ubic.gemma.web.controller.common.CharacteristicBrowserController=INFO,annotationsFile
 log4j.logger.ubic.gemma.web.controller.expression.experiment.AnnotationController=INFO,annotationsFile

gemma-cli/src/main/java/ubic/gemma/core/apps/ArrayDesignAlternativePopulateCli.java

@@ -22,6 +22,7 @@
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.Options;
 import org.apache.commons.lang3.StringUtils;
+import org.springframework.core.io.ClassPathResource;
 import ubic.gemma.core.apps.GemmaCLI.CommandGroup;
 import ubic.gemma.core.util.AbstractCLIContextCLI;
 import ubic.gemma.model.expression.arrayDesign.ArrayDesign;
@@ -69,7 +70,7 @@ protected void doWork() throws Exception {
  ArrayDesignService arrayDesignService = this.getBean( ArrayDesignService.class );
 
  // Read in the mapping file, which is in the classpath. This is also used by GeoPlatform (and in the DataUpdater)
- InputStream r = this.getClass().getResourceAsStream( "/ubic/gemma/core/loader/affy.altmappings.txt" );
+ InputStream r = new ClassPathResource( "/ubic/gemma/core/loader/affy.altmappings.txt" ).getInputStream();
  try ( BufferedReader in = new BufferedReader( new InputStreamReader( r ) ) ) {
  while ( in.ready() ) {
  String line = in.readLine().trim();

gemma-cli/src/main/java/ubic/gemma/core/apps/ArrayDesignAnnotationFileCli.java

@@ -36,6 +36,7 @@
 import ubic.gemma.core.analysis.service.ArrayDesignAnnotationService;
 import ubic.gemma.core.apps.GemmaCLI.CommandGroup;
 import ubic.gemma.core.genome.gene.service.GeneService;
+import ubic.gemma.core.ontology.OntologyUtils;
 import ubic.gemma.core.ontology.providers.GeneOntologyService;
 import ubic.gemma.core.util.AbstractCLI;
 import ubic.gemma.model.expression.arrayDesign.ArrayDesign;
@@ -165,9 +166,7 @@ public String getCommandName() {
  protected void doWork() throws Exception {
  this.taxonService = this.getBean( TaxonService.class );
 
- this.waitForGeneOntologyReady();
-
- this.goService.initialize( true, false );
+ OntologyUtils.ensureInitialized( goService );
 
  log.info( "***** Annotation file(s) will be written to " + ArrayDesignAnnotationService.ANNOT_DATA_DIR + " ******" );
 
@@ -400,21 +399,4 @@ private void processOneAD( ArrayDesign inputAd ) throws IOException {
 
  }
 
- private void waitForGeneOntologyReady() {
-
- goService.initialize( true, false );
-
- while ( !goService.isOntologyLoaded() ) {
- try {
- Thread.sleep( 10000 );
- AbstractCLI.log.info( "Waiting for Gene Ontology to load ..." );
- } catch ( InterruptedException e ) {
- throw new RuntimeException( "Failure while waiting for GO to load", e );
- }
- }
- if ( !this.notifiedAboutGOState ) {
- AbstractCLI.log.info( "GO is ready." );
- this.notifiedAboutGOState = true;
- }
- }
 }

gemma-cli/src/main/java/ubic/gemma/core/apps/DifferentialExpressionAnalysisCli.java

@@ -477,7 +477,7 @@ private Collection<ExperimentalFactor> guessFactors( ExpressionExperiment ee ) {
  if ( this.factorNames.size() > 0 ) {
  throw new IllegalArgumentException( "Please provide factor names or ids, not a mixture of each" );
  }
- ExperimentalFactor factor = efs.load( factorId );
+ ExperimentalFactor factor = efs.loadOrFail( factorId );
  factor = efs.thaw( factor );
  if ( factor == null ) {
  throw new IllegalArgumentException( "No factor for id=" + factorId );

gemma-cli/src/main/java/ubic/gemma/core/apps/ExperimentalDesignImportCli.java

@@ -24,8 +24,7 @@
 import ubic.basecode.ontology.providers.ExperimentalFactorOntologyService;
 import ubic.gemma.core.apps.GemmaCLI.CommandGroup;
 import ubic.gemma.core.loader.expression.simple.ExperimentalDesignImporter;
-import ubic.gemma.core.ontology.OntologyService;
-import ubic.gemma.core.util.AbstractCLI;
+import ubic.gemma.core.ontology.OntologyUtils;
 import ubic.gemma.core.util.AbstractCLIContextCLI;
 import ubic.gemma.model.expression.experiment.ExpressionExperiment;
 import ubic.gemma.persistence.service.expression.experiment.ExpressionExperimentService;
@@ -69,18 +68,8 @@ protected void buildOptions( Options options ) {
 
  @Override
  protected void doWork() throws Exception {
- ExperimentalFactorOntologyService mos = this.getBean( OntologyService.class )
- .getExperimentalFactorOntologyService();
- mos.startInitializationThread( true, false ); // note will *not* re-index
- while ( !mos.isOntologyLoaded() ) {
- try {
- Thread.sleep( 5000 );
- } catch ( InterruptedException e1 ) {
- //
- }
- AbstractCLI.log.info( "Waiting for EFO to load" );
- }
-
+ ExperimentalFactorOntologyService mos = this.getBean( ExperimentalFactorOntologyService.class );
+ OntologyUtils.ensureInitialized( mos );
  ExperimentalDesignImporter edImp = this.getBean( ExperimentalDesignImporter.class );
  ExpressionExperimentService ees = this.getBean( ExpressionExperimentService.class );
  expressionExperiment = ees.thawBioAssays( expressionExperiment );

gemma-cli/src/main/java/ubic/gemma/core/apps/ExpressionExperimentManipulatingCLI.java

@@ -322,7 +322,8 @@ private Set<BioAssaySet> findExpressionExperimentsByQuery( String query ) throws
  }
 
  Collection<SearchResult<ExpressionExperiment>> eeSearchResults = searchService
- .search( SearchSettings.expressionExperimentSearch( query ), ExpressionExperiment.class );
+ .search( SearchSettings.expressionExperimentSearch( query ) )
+ .getByResultObjectType( ExpressionExperiment.class );
 
  // Filter out all the ee that are not of correct taxon
  for ( SearchResult<ExpressionExperiment> sr : eeSearchResults ) {

gemma-cli/src/main/java/ubic/gemma/core/apps/ExternalDatabaseUpdaterCli.java

@@ -60,7 +60,8 @@ protected void buildOptions( Options options ) {
  .longOpt( "name" )
  .hasArg()
  .optionalArg( false )
- .desc( "External database name" ).build() );
+ .desc( "External database name" )
+ .required( true ).build() );
  options.addOption( DESCRIPTION_OPTION, "description", true, "New description" );
  options.addOption( RELEASE_OPTION, "release", false, "Update the release (only affects last modified moment))" );
  options.addOption( RELEASE_VERSION_OPTION, "release-version", true, "Release version" );
@@ -131,7 +132,7 @@ protected void doWork() throws Exception {
  if ( parentDatabase != null ) {
  AbstractCLI.log.info( String.format( "Updating the parent database of %s to %s.", name, parentDatabase ) );
  parentDatabase.getExternalDatabases().add( ed );
- externalDatabaseService.update( ed );
+ externalDatabaseService.update( parentDatabase );
  }
  }
 }

gemma-cli/src/main/java/ubic/gemma/core/apps/GemmaCLI.java

@@ -26,8 +26,11 @@
 import org.springframework.security.core.context.SecurityContextHolder;
 import ubic.gemma.core.util.AbstractCLI;
 import ubic.gemma.core.util.CLI;
+import ubic.gemma.core.logging.log4j.Log4jConfigurer;
+import ubic.gemma.core.logging.LoggingConfigurer;
 import ubic.gemma.persistence.util.Settings;
 import ubic.gemma.persistence.util.SpringContextUtil;
+import ubic.gemma.persistence.util.SpringProfiles;
 
 import javax.annotation.Nullable;
 import java.io.PrintWriter;
@@ -50,7 +53,9 @@ public class GemmaCLI {
  private static final String
  HELP_OPTION = "h",
  HELP_ALL_OPTION = "ha",
- VERSION_OPTION = "v",
+ VERSION_OPTION = "version",
+ LOGGER_OPTION = "logger",
+ VERBOSITY_OPTION = "v",
  TESTING_OPTION = "testing"; // historically named '-testing', but now '--testing' is also accepted
 
  /**
@@ -60,11 +65,24 @@ public class GemmaCLI {
  */
  private static final Pattern PASSWORD_IN_CLI_MATCHER = Pattern.compile( "(-{1,2}p(?:assword)?)\\s+(.+?)\\b" );
 
+ private static final LoggingConfigurer loggingConfigurer = new Log4jConfigurer();
+
  public static void main( String[] args ) {
+ Option logOpt = Option.builder( VERBOSITY_OPTION )
+ .longOpt( "verbosity" ).hasArg()
+ .desc( "Set verbosity level for all loggers (0=silent, 5=very verbose; default is custom, see log4j.properties)" )
+ .type( Number.class )
+ .build();
+ Option otherLogOpt = Option.builder( LOGGER_OPTION )
+ .longOpt( "logger" ).hasArg()
+ .desc( "Configure a specific logger verbosity. For example, '--logger ubic.gemma=5' or --logger log4j.logger.org.hibernate.SQL=5" )
+ .build();
  Options options = new Options()
  .addOption( HELP_OPTION, "help", false, "Show help" )
  .addOption( HELP_ALL_OPTION, "help-all", false, "Show complete help with all available CLI commands" )
  .addOption( VERSION_OPTION, "version", false, "Show Gemma version" )
+ .addOption( otherLogOpt )
+ .addOption( logOpt )
  .addOption( TESTING_OPTION, "testing", false, "Use the test environment" );
  CommandLine commandLine;
  try {
@@ -87,15 +105,53 @@ public static void main( String[] args ) {
  return;
  }
 
+ if ( commandLine.hasOption( VERBOSITY_OPTION ) ) {
+ try {
+ loggingConfigurer.configureAllLoggers( ( ( Number ) commandLine.getParsedOptionValue( VERBOSITY_OPTION ) ).intValue() );
+ } catch ( ParseException | IllegalArgumentException e ) {
+ System.err.printf( "Failed to parse the %s option: %s.%n", VERBOSITY_OPTION,
+ ExceptionUtils.getRootCauseMessage( e ) );
+ System.exit( 1 );
+ return;
+ }
+ }
+
+ if ( commandLine.hasOption( LOGGER_OPTION ) ) {
+ for ( String value : commandLine.getOptionValues( LOGGER_OPTION ) ) {
+ String[] vals = value.split( "=" );
+ if ( vals.length != 2 ) {
+ System.err.println( "Logging value must in format [loggerName]=[value]." );
+ System.exit( 1 );
+ }
+ String loggerName = vals[0];
+ try {
+ loggingConfigurer.configureLogger( loggerName, Integer.parseInt( vals[1] ) );
+ } catch ( IllegalArgumentException e ) {
+ System.err.printf( "Failed to parse the %s option for %s: %s.%n", VERBOSITY_OPTION,
+ loggerName,
+ ExceptionUtils.getRootCauseMessage( e ) );
+ System.exit( 1 );
+ return;
+ }
+ }
+ }
+
+ loggingConfigurer.apply();
+
  /*
  * Guarantee that the security settings are uniform throughout the application (all threads).
  */
  SecurityContextHolder.setStrategyName( SecurityContextHolder.MODE_INHERITABLETHREADLOCAL );
 
+ List<String> profiles = new ArrayList<>();
+ profiles.add( "cli" );
+ if ( commandLine.hasOption( TESTING_OPTION ) ) {
+ profiles.add( SpringProfiles.TEST );
+ }
+
  // check for the -testing flag to load the appropriate application context
  /* webapp */
- ApplicationContext ctx = SpringContextUtil.getApplicationContext( commandLine.hasOption( TESTING_OPTION ),
- "classpath*:ubic/gemma/cliContext-component-scan.xml" );
+ ApplicationContext ctx = SpringContextUtil.getApplicationContext( profiles.toArray( new String[0] ) );
 
  /*
  * Build a map from command names to classes.