Merge branch 'hotfix-1.31.3'

.editorconfig

@@ -4,9 +4,6 @@ root = true
 indent_style=space
 indent_size=4
 
-[init-indices.sql]
-indent_size=2
-
 [/pom.xml]
 indent_style=tab
 

gemma-cli/pom.xml

@@ -3,7 +3,7 @@
  <parent>
  <artifactId>gemma</artifactId>
  <groupId>gemma</groupId>
- <version>1.31.2</version>
+ <version>1.31.3</version>
  </parent>
  <modelVersion>4.0.0</modelVersion>
  <artifactId>gemma-cli</artifactId>

gemma-cli/src/main/java/ubic/gemma/core/apps/ArrayDesignBlatCli.java

@@ -73,7 +73,7 @@ protected void buildOptions( Options options ) {
  .desc(
  "Threshold (0-1.0) for acceptance of BLAT alignments [Default = " + this.blatScoreThreshold + "]" )
  .longOpt( "scoreThresh" )
- .type( Double.class )
+ .type( Number.class )
  .build();
 
  options.addOption( Option.builder( "sensitive" ).desc( "Run on more sensitive server, if available" ).build() );
@@ -107,7 +107,7 @@ protected void processOptions( CommandLine commandLine ) throws ParseException {
 // }
 
  if ( commandLine.hasOption( 's' ) ) {
- this.blatScoreThreshold = ( Double ) commandLine.getParsedOptionValue( 's' );
+ this.blatScoreThreshold = ( ( Number ) commandLine.getParsedOptionValue( 's' ) ).doubleValue();
  }
 
  TaxonService taxonService = this.getBean( TaxonService.class );

gemma-cli/src/main/java/ubic/gemma/core/apps/ArrayDesignProbeMapperCli.java

@@ -87,17 +87,17 @@ protected void buildOptions( Options options ) {
  super.buildOptions( options );
 
  options.addOption( Option.builder( "i" ).hasArg().argName( "value" )
- .type( Double.class )
+ .type( Number.class )
  .desc( "Sequence identity threshold, default = " + ProbeMapperConfig.DEFAULT_IDENTITY_THRESHOLD )
  .longOpt( "identityThreshold" ).build() );
 
  options.addOption( Option.builder( "s" ).hasArg().argName( "value" )
- .type( Double.class )
+ .type( Number.class )
  .desc( "Blat score threshold, default = " + ProbeMapperConfig.DEFAULT_SCORE_THRESHOLD )
  .longOpt( "scoreThreshold" ).build() );
 
  options.addOption( Option.builder( "o" ).hasArg().argName( "value" )
- .type( Double.class )
+ .type( Number.class )
  .desc( "Minimum fraction of probe overlap with exons, default = " + ProbeMapperConfig.DEFAULT_MINIMUM_EXON_OVERLAP_FRACTION )
  .longOpt( "overlapThreshold" )
  .build() );
@@ -234,7 +234,7 @@ protected void processOptions( CommandLine commandLine ) throws ParseException {
  }
 
  if ( commandLine.hasOption( 's' ) ) {
- blatScoreThreshold = ( Double ) commandLine.getParsedOptionValue( 's' );
+ blatScoreThreshold = ( ( Number ) commandLine.getParsedOptionValue( 's' ) ).doubleValue();
  if ( blatScoreThreshold < 0 || blatScoreThreshold > 1 ) {
  throw new IllegalArgumentException( "BLAT score threshold must be between 0 and 1" );
  }
@@ -249,14 +249,14 @@ protected void processOptions( CommandLine commandLine ) throws ParseException {
  this.mirnaOnlyModeOption = commandLine.hasOption( ArrayDesignProbeMapperCli.MIRNA_ONLY_MODE_OPTION );
 
  if ( commandLine.hasOption( 'i' ) ) {
- identityThreshold = ( Double ) commandLine.getParsedOptionValue( 'i' );
+ identityThreshold = ( ( Number ) commandLine.getParsedOptionValue( 'i' ) ).doubleValue();
  if ( identityThreshold < 0 || identityThreshold > 1 ) {
  throw new IllegalArgumentException( "Identity threshold must be between 0 and 1" );
  }
  }
 
  if ( commandLine.hasOption( 'o' ) ) {
- overlapThreshold = ( Double ) commandLine.getParsedOptionValue( 'o' );
+ overlapThreshold = ( ( Number ) commandLine.getParsedOptionValue( 'o' ) ).doubleValue();
  if ( overlapThreshold < 0 || overlapThreshold > 1 ) {
  throw new IllegalArgumentException( "Overlap threshold must be between 0 and 1" );
  }

gemma-cli/src/main/java/ubic/gemma/core/apps/FindObsoleteTermsCli.java

@@ -8,7 +8,7 @@
 import org.springframework.core.task.AsyncTaskExecutor;
 import ubic.gemma.core.ontology.OntologyService;
 import ubic.gemma.core.util.AbstractCLI;
-import ubic.gemma.model.common.description.CharacteristicValueObject;
+import ubic.gemma.model.common.description.Characteristic;
 
 import java.util.LinkedHashMap;
 import java.util.List;
@@ -90,14 +90,13 @@ protected void doWork() throws Exception {
 
  log.info( "Ontologies warmed up, starting check..." );
 
- Map<String, CharacteristicValueObject> vos = ontologyService.findObsoleteTermUsage();
+ Map<Characteristic, Long> vos = ontologyService.findObsoleteTermUsage();
 
  AbstractCLI.log.info( "Obsolete term check finished, printing ..." );
 
  System.out.println( "Value\tValueUri\tCount" );
- for ( CharacteristicValueObject vo : vos.values() ) {
- System.out.println( vo.getValue() + "\t" + vo.getValueUri() + "\t" + vo.getNumTimesUsed() );
+ for ( Map.Entry<Characteristic, Long> vo : vos.entrySet() ) {
+ System.out.println( vo.getKey().getValue() + "\t" + vo.getKey().getValueUri() + "\t" + vo.getValue() );
  }
-
  }
 }

gemma-cli/src/main/java/ubic/gemma/core/apps/IndexGemmaCLI.java

@@ -4,6 +4,7 @@
 import org.apache.commons.cli.Options;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
 import ubic.gemma.core.search.IndexerService;
 import ubic.gemma.core.util.AbstractCLI;
 import ubic.gemma.model.analysis.expression.ExpressionExperimentSet;
@@ -17,33 +18,34 @@
 import ubic.gemma.model.genome.gene.GeneSet;
 
 import java.io.File;
+import java.util.Arrays;
 import java.util.HashSet;
 import java.util.Set;
 import java.util.stream.Collectors;
 
+@Component
 public class IndexGemmaCLI extends AbstractCLI {
 
- private static final String THREADS_OPTION = "threads";
-
  /**
  * A list of all searchable entities this CLI supports.
  */
  private static final IndexableEntity[] indexableEntities = {
- new IndexableEntity( "g", "genes", Gene.class ),
- new IndexableEntity( "e", "datasets", ExpressionExperiment.class ),
- new IndexableEntity( "a", "platforms", ArrayDesign.class ),
- new IndexableEntity( "b", "bibliographic references", BibliographicReference.class ),
- new IndexableEntity( "s", "probes", CompositeSequence.class ),
- new IndexableEntity( "q", "sequences", BioSequence.class ),
- new IndexableEntity( "x", "datasets groups", ExpressionExperimentSet.class ),
- new IndexableEntity( "y", "gene sets", GeneSet.class )
+ new IndexableEntity( "g", "genes", Gene.class, 1000 ),
+ new IndexableEntity( "e", "datasets", ExpressionExperiment.class, 1000 ),
+ new IndexableEntity( "a", "platforms", ArrayDesign.class, 100 ),
+ new IndexableEntity( "b", "bibliographic references", BibliographicReference.class, 1000 ),
+ new IndexableEntity( "s", "probes", CompositeSequence.class, 100000 ),
+ new IndexableEntity( "q", "sequences", BioSequence.class, 100000 ),
+ new IndexableEntity( "x", "datasets groups", ExpressionExperimentSet.class, 100 ),
+ new IndexableEntity( "y", "gene sets", GeneSet.class, 10 )
  };
 
  @lombok.Value
  private static class IndexableEntity {
  String option;
  String description;
  Class<? extends Identifiable> clazz;
+ int loggingFrequency;
  }
 
  @Autowired
@@ -52,8 +54,7 @@ private static class IndexableEntity {
  @Value("${gemma.search.dir}")
  private File searchDir;
 
- private final Set<Class<? extends Identifiable>> classesToIndex = new HashSet<>();
- private int numThreads;
+ private final Set<IndexableEntity> classesToIndex = new HashSet<>();
 
  @Override
  public String getCommandName() {
@@ -82,21 +83,28 @@ protected void buildOptions( Options options ) {
  protected void processOptions( CommandLine commandLine ) {
  for ( IndexableEntity ie : indexableEntities ) {
  if ( commandLine.hasOption( ie.option ) ) {
- classesToIndex.add( ie.clazz );
+ classesToIndex.add( ie );
  }
  }
+ if ( classesToIndex.isEmpty() ) {
+ classesToIndex.addAll( Arrays.asList( indexableEntities ) );
+ }
+ indexerService.setNumThreads( getNumThreads() );
  }
 
  @Override
  protected void doWork() throws Exception {
- if ( classesToIndex.isEmpty() ) {
- log.info( String.format( "All entities will be indexed under %s.", searchDir.getAbsolutePath() ) );
- indexerService.index( getNumThreads() );
- } else {
+ if ( classesToIndex.size() < indexableEntities.length ) {
  log.info( String.format( "The following entities will be indexed under %s:\n\t%s",
  searchDir.getAbsolutePath(),
- classesToIndex.stream().map( Class::getName ).collect( Collectors.joining( "\n\t" ) ) ) );
- indexerService.index( classesToIndex, getNumThreads() );
+ classesToIndex.stream().map( IndexableEntity::getClazz ).map( Class::getName ).collect( Collectors.joining( "\n\t" ) ) ) );
+ } else {
+ log.info( String.format( "All entities will be indexed under %s.", searchDir.getAbsolutePath() ) );
+ }
+ for ( IndexableEntity classToIndex : classesToIndex ) {
+ log.info( "Indexing " + classToIndex.getClazz().getName() + "..." );
+ indexerService.setLoggingFrequency( classToIndex.loggingFrequency );
+ indexerService.index( classToIndex.clazz );
  }
  }
 }

gemma-cli/src/main/java/ubic/gemma/core/apps/LinkAnalysisCli.java

@@ -291,7 +291,7 @@ protected void buildOptions( Options options ) {
  .build();
  options.addOption( chooseCutOption );
 
- options.addOption( Option.builder( "probeDegreeLim" ).hasArg().type( Integer.class ).build() );
+ options.addOption( Option.builder( "probeDegreeLim" ).hasArg().type( Number.class ).build() );
 
  // finer-grained control is possible, of course.
  Option skipQC = Option.builder( "noqc" )
@@ -427,7 +427,7 @@ protected void processOptions( CommandLine commandLine ) throws ParseException {
  }
 
  if ( commandLine.hasOption( "probeDegreeLim" ) ) {
- this.linkAnalysisConfig.setProbeDegreeThreshold( ( Integer ) commandLine.getParsedOptionValue( "probeDegreeLim" ) );
+ this.linkAnalysisConfig.setProbeDegreeThreshold( ( ( Number ) commandLine.getParsedOptionValue( "probeDegreeLim" ) ).intValue() );
  }
 
  }

gemma-cli/src/main/java/ubic/gemma/core/apps/UpdateEE2CCli.java

@@ -0,0 +1,52 @@
+package ubic.gemma.core.apps;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.springframework.beans.factory.annotation.Autowired;
+import ubic.gemma.core.util.AbstractAuthenticatedCLI;
+import ubic.gemma.persistence.service.TableMaintenanceUtil;
+
+import javax.annotation.Nullable;
+
+public class UpdateEE2CCli extends AbstractAuthenticatedCLI {
+
+ private static final String TRUNCATE_OPTION = "truncate";
+
+ @Autowired
+ private TableMaintenanceUtil tableMaintenanceUtil;
+
+ private boolean truncate;
+
+ @Override
+ protected void buildOptions( Options options ) {
+ options.addOption( TRUNCATE_OPTION, "truncate", false, "Truncate the table before updating it" );
+ }
+
+ @Override
+ protected void processOptions( CommandLine commandLine ) throws ParseException {
+ truncate = commandLine.hasOption( TRUNCATE_OPTION );
+ }
+
+ @Override
+ protected void doWork() throws Exception {
+ tableMaintenanceUtil.updateExpressionExperiment2CharacteristicEntries( truncate );
+ }
+
+ @Nullable
+ @Override
+ public String getCommandName() {
+ return "updateEe2c";
+ }
+
+ @Nullable
+ @Override
+ public String getShortDesc() {
+ return "Update the EXPRESSION_EXPERIMENT2CHARACTERISTIC table";
+ }
+
+ @Override
+ public GemmaCLI.CommandGroup getCommandGroup() {
+ return GemmaCLI.CommandGroup.EXPERIMENT;
+ }
+}

gemma-cli/src/main/java/ubic/gemma/core/util/AbstractCLI.java

@@ -273,7 +273,7 @@ protected void addDateOption( Options options ) {
  protected void addThreadsOption( Options options ) {
  options.addOption( Option.builder( THREADS_OPTION ).argName( "numThreads" ).hasArg()
  .desc( "Number of threads to use for batch processing." )
- .type( Integer.class )
+ .type( Number.class )
  .build() );
  }
 
@@ -349,7 +349,7 @@ protected void processStandardOptions( CommandLine commandLine ) throws ParseExc
  this.autoSeek = commandLine.hasOption( AbstractCLI.AUTO_OPTION_NAME );
 
  if ( commandLine.hasOption( THREADS_OPTION ) ) {
- this.numThreads = ( Integer ) commandLine.getParsedOptionValue( THREADS_OPTION );
+ this.numThreads = ( ( Number ) commandLine.getParsedOptionValue( THREADS_OPTION ) ).intValue();
  if ( this.numThreads < 1 ) {
  throw new IllegalArgumentException( "Number of threads must be greater than 1." );
  }

gemma-core/pom.xml

@@ -3,7 +3,7 @@
  <parent>
  <artifactId>gemma</artifactId>
  <groupId>gemma</groupId>
- <version>1.31.2</version>
+ <version>1.31.3</version>
  </parent>
  <modelVersion>4.0.0</modelVersion>
  <artifactId>gemma-core</artifactId>
@@ -216,8 +216,9 @@
  <srcFile>${project.build.directory}/schema/gemma/gsec/sql/gsec-acl-ddl.sql</srcFile>
  <scrFile>${project.build.directory}/schema/gemma/gsec/sql/init-acl-indices.sql</scrFile>
  <srcFile>${project.basedir}/src/main/resources/sql/init-acls.sql</srcFile>
- <scrFile>${project.basedir}/src/main/resources/sql/init-indices.sql</scrFile>
  <srcFile>${project.basedir}/src/main/resources/sql/init-entities.sql</srcFile>
+ <srcFile>${project.basedir}/src/main/resources/sql/mysql/init-entities.sql</srcFile>
+ <srcFile>${project.basedir}/src/main/resources/sql/init-data.sql</srcFile>
  </srcFiles>
  <!--suppress MavenModelInspection -->
  <skip>${skipIntegrationTests}</skip>