Merge branch 'release-1.27.0'

gemma-core/pom.xml

@@ -3,20 +3,13 @@
  <parent>
  <artifactId>gemma</artifactId>
  <groupId>gemma</groupId>
- <version>1.26.9</version>
+ <version>1.27.0</version>
  </parent>
  <modelVersion>4.0.0</modelVersion>
  <artifactId>gemma-core</artifactId>
  <name>Gemma Core</name>
  <description>Module containing Gemma backend logic (everything but web-related)</description>
- <properties>
- <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
- </properties>
  <build>
- <sourceDirectory>src/main/java</sourceDirectory>
- <testSourceDirectory>src/test/java</testSourceDirectory>
- <outputDirectory>target/classes</outputDirectory>
- <testOutputDirectory>target/test-classes</testOutputDirectory>
  <resources>
  <resource>
  <directory>src/main/resources</directory>
@@ -65,8 +58,6 @@
  </includes>
  </testResource>
  </testResources>
- <directory>target</directory>
- <finalName>${project.artifactId}-${project.version}</finalName>
  <plugins>
  <plugin>
  <groupId>org.apache.maven.plugins</groupId>
@@ -117,13 +108,26 @@
  </dependency>
  </dependencies>
  </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <version>3.2.0</version>
+ <executions>
+ <execution>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
  <plugin>
  <groupId>org.codehaus.mojo</groupId>
  <artifactId>appassembler-maven-plugin</artifactId>
  <version>2.1.0</version>
  <configuration>
  <repositoryName>lib</repositoryName>
  <repositoryLayout>flat</repositoryLayout>
+ <extraJvmArguments>-Dlog4j.configurationFile=@BASEDIR@/etc/log4j.properties</extraJvmArguments>
  <platforms>
  <platform>unix</platform>
  </platforms>
@@ -236,12 +240,12 @@
  <dependency>
  <groupId>mysql</groupId>
  <artifactId>mysql-connector-java</artifactId>
- <version>5.1.48</version>
+ <version>${mysql.version}</version>
  </dependency>
  </dependencies>
  <!-- common configuration shared by all executions -->
  <configuration>
- <driver>com.mysql.jdbc.Driver</driver>
+ <driver>com.mysql.cj.jdbc.Driver</driver>
  <username>${gemma.db.build.user}</username>
  <password>${gemma.db.build.password}</password>
  <onError>continue</onError>
@@ -260,7 +264,7 @@
  <sqlCommand>
  <![CDATA[
  DROP DATABASE IF EXISTS ${gemma.testdb.name};
- CREATE DATABASE ${gemma.testdb.name}
+ CREATE DATABASE ${gemma.testdb.name} CHARACTER SET utf8mb4;
  ]]>
  </sqlCommand>
  </configuration>
@@ -341,25 +345,20 @@
  <artifactId>aopalliance</artifactId>
  <version>1.0</version>
  </dependency>
- <dependency>
- <groupId>commons-net</groupId>
- <artifactId>commons-net</artifactId>
- <version>1.4.1</version>
- </dependency>
  <dependency>
  <groupId>commons-cli</groupId>
  <artifactId>commons-cli</artifactId>
- <version>1.3</version>
+ <version>1.4</version>
  </dependency>
  <dependency>
- <groupId>commons-pool</groupId>
- <artifactId>commons-pool</artifactId>
- <version>1.5.4</version>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-pool2</artifactId>
+ <version>2.11.1</version>
  </dependency>
  <dependency>
  <groupId>org.apache.commons</groupId>
  <artifactId>commons-csv</artifactId>
- <version>1.8</version>
+ <version>1.9.0</version>
  </dependency>
  <dependency>
  <groupId>org.apache.commons</groupId>
@@ -435,7 +434,7 @@
  <dependency>
  <groupId>org.apache.ant</groupId>
  <artifactId>ant</artifactId>
- <version>1.9.15</version>
+ <version>1.10.11</version>
  </dependency>
  <dependency>
  <groupId>RoSuDA</groupId>

gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/batcheffects/BatchInfoPopulationHelperServiceImpl.java

@@ -728,7 +728,7 @@ protected boolean hadUseableHeader() {
  /**
  * Only for use when we don't have a useable device:run etc.
  * 
- * @param device e.g. GPLXXXX
+ * @param platform e.g. GPLXXXX
  */
  FastqHeaderData( String platform ) {
  this.device = platform;

gemma-core/src/main/java/ubic/gemma/core/analysis/report/ExpressionExperimentReportServiceImpl.java

@@ -126,7 +126,7 @@ public ExpressionExperimentDetailsValueObject generateSummary( Long id ) {
  public void generateSummaryObjects() {
  Collection<Long> ids = EntityUtils.getIds( expressionExperimentService.loadAll() );
  Collection<ExpressionExperimentDetailsValueObject> vos = expressionExperimentService
- .loadDetailsValueObjects( null, false, ids, null, 0, 0 );
+ .loadDetailsValueObjects( null, ids, null, 0, 0 );
  this.getStats( vos );
  }
 
@@ -405,7 +405,7 @@ public void recalculateExperimentBatchInfo( ExpressionExperiment ee ) {
  private Collection<ExpressionExperimentDetailsValueObject> generateSummaryObjects( Collection<Long> ids ) {
 
  Collection<ExpressionExperimentDetailsValueObject> vos = expressionExperimentService
- .loadDetailsValueObjects( null, false, ids, null, 0, 0 );
+ .loadDetailsValueObjects( null, ids, null, 0, 0 );
  this.getStats( vos );
 
  for ( ExpressionExperimentValueObject vo : vos ) {

gemma-core/src/main/java/ubic/gemma/core/apps/ArrayDesignSequenceAssociationCli.java

@@ -96,22 +96,34 @@ protected void doWork() throws Exception {
  }
 
  if ( this.sequenceFile != null ) {
- try (InputStream sequenceFileIs = FileTools
- .getInputStreamFromPlainOrCompressedFile( sequenceFile )) {
+ try ( InputStream sequenceFileIs = FileTools
+ .getInputStreamFromPlainOrCompressedFile( sequenceFile ) ) {
 
  if ( sequenceFileIs == null ) {
  throw new IllegalArgumentException( "No file " + sequenceFile + " was readable" );
  }
 
  AbstractCLI.log.info( "Processing ArrayDesign..." );
 
- arrayDesignSequenceProcessingService
- .processArrayDesign( arrayDesign, sequenceFileIs, sequenceTypeEn, taxon );
+ if ( idFile != null ) {
+ try ( InputStream idFileIs = FileTools
+ .getInputStreamFromPlainOrCompressedFile( idFile ) ) {
+
+ arrayDesignSequenceProcessingService
+ .processArrayDesign( arrayDesign, sequenceFileIs, idFileIs, sequenceTypeEn, taxon );
+
+
+ }
+ } else {
+ // sequence file has to have a way to identify the probes they go with
+ arrayDesignSequenceProcessingService
+ .processArrayDesign( arrayDesign, sequenceFileIs, sequenceTypeEn, taxon );
+ }
 
  this.audit( arrayDesign, "Sequences read from file: " + sequenceFile );
  }
  } else if ( this.idFile != null ) {
- try (InputStream idFileIs = FileTools.getInputStreamFromPlainOrCompressedFile( idFile )) {
+ try ( InputStream idFileIs = FileTools.getInputStreamFromPlainOrCompressedFile( idFile ) ) {
 
  if ( idFileIs == null ) {
  throw new IllegalArgumentException( "No file " + idFile + " was readable" );
@@ -165,12 +177,13 @@ protected void buildOptions( Options options ) {
  super.buildOptions( options );
 
  Option fileOption = Option.builder( "f" ).argName( "Input sequence file" ).hasArg()
- .desc( "Path to file (FASTA)" ).longOpt( "file" ).build();
+ .desc( "Path to file (FASTA). If the FASTA file doesn't have " +
+ "probe identifiers included, provide identifiers via the -i option." ).longOpt( "file" ).build();
 
  options.addOption( fileOption );
 
  Option sequenceIdentifierOption = Option.builder( "i" ).argName( "Input identifier file" ).hasArg()
- .desc( "Path to file (two columns with probe ids and sequence accessions)" )
+ .desc( "Path to file (two columns with probe ids and sequence accessions); can use in combination with -file" )
  .longOpt( "ids" ).build();
 
  options.addOption( sequenceIdentifierOption );
@@ -201,7 +214,7 @@ protected void buildOptions( Options options ) {
  options.addOption( forceOption );
 
  Option taxonOption = Option.builder( "t" ).hasArg().argName( "taxon" ).desc(
- "Taxon common name (e.g., human) for sequences (only required if array design is 'naive')" )
+  "Taxon common name (e.g., human) for sequences (only required if array design is 'naive')" )
  .build();
 
  options.addOption( taxonOption );

gemma-core/src/main/java/ubic/gemma/core/apps/ArrayDesignSubsumptionTesterCli.java

@@ -27,9 +27,8 @@
 import ubic.gemma.model.expression.arrayDesign.ArrayDesign;
 import ubic.gemma.model.expression.arrayDesign.TechnologyType;
 
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.HashSet;
+import java.util.*;
+import java.util.concurrent.ArrayBlockingQueue;
 
 /**
  * Test two array designs to see if one subsumes the other, and if so update their information.
@@ -39,6 +38,7 @@
 public class ArrayDesignSubsumptionTesterCli extends ArrayDesignSequenceManipulatingCli {
 
  private Collection<String> otherArrayDesignNames;
+ private boolean allWays = false;
 
  @Override
  public String getCommandName() {
@@ -59,16 +59,20 @@ protected void doWork() throws Exception {
  "This tool is only for microarray platforms; " + arrayDesign.getShortName() + " is a sequencing platform" );
  }
 
+ List<ArrayDesign> allToCompare = new ArrayList<>();
+ allToCompare.add( arrayDesign );
+
  for ( String otherArrayDesignName : otherArrayDesignNames ) {
  ArrayDesign otherArrayDesign = this.locateArrayDesign( otherArrayDesignName, getArrayDesignService() );
 
+ if ( otherArrayDesign == null ) {
+ throw new Exception( "No arrayDesign " + otherArrayDesignName + " found" );
+ }
+
  if ( arrayDesign.equals( otherArrayDesign ) ) {
  continue;
  }
 
- if ( otherArrayDesign == null ) {
- throw new Exception( "No arrayDesign " + otherArrayDesignName + " found" );
- }
 
  otherArrayDesign = this.thaw( otherArrayDesign );
 
@@ -77,16 +81,43 @@ protected void doWork() throws Exception {
  "This tool is only for microarray platforms; " + otherArrayDesign.getShortName() + " is a sequencing platform" );
  }
 
- Boolean aSubsumes = this.getArrayDesignService().updateSubsumingStatus( arrayDesign, otherArrayDesign );
+ if ( allWays ) {
+ allToCompare.add( otherArrayDesign );
+ } else {
+ Boolean aSubsumes = this.getArrayDesignService().updateSubsumingStatus( arrayDesign, otherArrayDesign );
+ }
+// if ( !aSubsumes ) {
+// // test other way around, but only if first way failed (to avoid cycles)
+// this.getArrayDesignService().updateSubsumingStatus( otherArrayDesign, arrayDesign );
+// }
+// this.audit( otherArrayDesign, "Tested to see if it is subsumed by " + arrayDesign );
+ }
+
+
+ Collection<ArrayDesign> done = new HashSet<>();
+ if ( allWays ) {
+ for ( int i = 0; i < allToCompare.size(); i++ ) {
+ ArrayDesign a = allToCompare.get( i );
+ for ( int j = 1; j < allToCompare.size(); j++ ) {
+ ArrayDesign b = allToCompare.get( j );
+
+ if ( a.equals( b ) ) continue;
+
+ if ( done.contains( a ) || done.contains( b ) ) continue;
+
+ log.info( "--- comparing " + a.getShortName() + " to " + b.getShortName() );
+
+ boolean subsumes = this.getArrayDesignService().updateSubsumingStatus( a, b );
+
+ if ( subsumes ) {
+ done.add( b );
+ }
 
- if ( !aSubsumes ) {
- // test other way around, but only if first way failed (to avoid cycles)
- this.getArrayDesignService().updateSubsumingStatus( otherArrayDesign, arrayDesign );
+ }
  }
- this.audit( otherArrayDesign, "Tested to see if it is subsumed by " + arrayDesign );
  }
 
- this.audit( arrayDesign, "Tested to see if it subsumes: " + StringUtils.join( otherArrayDesignNames, ',' ) );
+// this.audit( arrayDesign, "Tested to see if it subsumes: " + StringUtils.join( otherArrayDesignNames, ',' ) );
  }
 
  @Override
@@ -102,8 +133,10 @@ protected void buildOptions( Options options ) {
  Option otherArrayDesignOption = Option.builder( "o" ).required().hasArg().argName( "Other platform" )
  .desc( "Short name(s) of platforms to compare to the first one, comma-delimited" )
  .longOpt( "other" ).build();
+ Option allways = Option.builder( "all" ).desc( "Test all platforms listed against all (not just to the first one)" ).build();
 
  options.addOption( otherArrayDesignOption );
+ options.addOption( allways );
  }
 
  @Override
@@ -115,6 +148,7 @@ protected void processOptions( CommandLine commandLine ) {
  this.otherArrayDesignNames = new HashSet<>();
  this.otherArrayDesignNames.addAll( Arrays.asList( names ) );
  }
+ this.allWays = commandLine.hasOption( "all" );
  }
 
  private void audit( ArrayDesign arrayDesign, String note ) {

gemma-core/src/main/java/ubic/gemma/core/apps/DeleteExperimentsCli.java

@@ -48,13 +48,18 @@ protected void buildOptions( Options options ) {
  Option.builder( "a" ).longOpt( "array" )
  .desc( "Delete platform(s) instead; you must delete associated experiments first; other options are ignored" )
  .argName( "comma-delimited list of platform short names" ).hasArg().build() );
+ super.suppressAllOption();
  }
 
  @Override
  protected void processOptions( CommandLine commandLine ) {
  super.processOptions( commandLine );
  if ( commandLine.hasOption( 'a' ) ) {
  this.platformAccs = Arrays.asList( StringUtils.split( commandLine.getOptionValue( 'a' ), "," ) );
+
+ if ( platformAccs.isEmpty() ) {
+ throw new IllegalArgumentException( "No platform accessions were obtained from the -a option" );
+ }
  }
  }
 
@@ -63,8 +68,11 @@ protected void doWork() throws Exception {
  this.force = true;
 
  if ( platformAccs != null ) {
+
  ArrayDesignService ads = this.getBean( ArrayDesignService.class );
 
+ log.info( "Deleting " + platformAccs.size() + " platform(s)" );
+
  for ( String p : platformAccs ) {
 
  ArrayDesign a = ads.findByShortName( p );