+ * The input file describes transition from a bag of {@link ubic.gemma.model.common.description.Characteristic} to more
+ * structured {@link ubic.gemma.model.expression.experiment.Statement} which are allowed to have up to two related
+ * objects.
+ *
+ * All unmentioned characteristics will be migrated to subject-only statements.
+ * @deprecated this will be removed as soon as all the old-style characteristics are migrated
+ * @author poirigui
+ */
+@Deprecated
+public class FactorValueMigratorCLI extends AbstractAuthenticatedCLI {
+
+ private static final String
+ MIGRATION_FILE_OPTION = "migrationFile",
+ MIGRATE_REMAINING_CHARACTERISTICS_OPTION = "migrateRemainingCharacteristics",
+ MIGRATE_REMAINING_FACTOR_VALUES_OPTION = "migrateRemainingFactorValues",
+ MIGRATE_NON_TRIVIAL_CASES_OPTION = "migrateNonTrivialCases",
+ NOOP_OPTION = "noop";
+
+ @Autowired
+ private FactorValueMigratorService factorValueMigratorService;
+
+ /**
+ * A list of migrations to perform.
+ */
+ private List migrations;
+ private boolean migrateRemainingCharacteristics;
+ private boolean migrateRemainingFactorValues;
+ private boolean migrateNonTrivialCases;
+ private boolean noop;
+
+ @Override
+ public String getCommandName() {
+ return "migrateFactorValues";
+ }
+
+ @Override
+ public String getShortDesc() {
+ return "Perform the migration of old-style characteristics to statements";
+ }
+
+ @Override
+ public GemmaCLI.CommandGroup getCommandGroup() {
+ return GemmaCLI.CommandGroup.MISC;
+ }
+
+ @Override
+ protected void buildOptions( Options options ) {
+ options.addOption( Option.builder( MIGRATION_FILE_OPTION ).hasArg().type( File.class ).desc( "File containing factor value migrations" ).build() );
+ options.addOption( MIGRATE_REMAINING_CHARACTERISTICS_OPTION, false, "Migrate remaining characteristics of factor values that were mentioned in the migration file. The affected factor values will be marked as needs attention." );
+ options.addOption( MIGRATE_REMAINING_FACTOR_VALUES_OPTION, false, "Migrate remaining factor values that weren't mentioned in the migration file." );
+ options.addOption( MIGRATE_NON_TRIVIAL_CASES_OPTION, false, "Migrate non-trivial cases (i.e. 2 or more old-style characteristics) to subject-only statements. The affected factor values will be marked as needs attention." );
+ options.addOption( NOOP_OPTION, false, "Only validate migrations; no statements will be saved" );
+ }
+
+ @Value
+ static class MigrationWithLineNumber {
+ long lineNumber;
+ FactorValueMigratorService.Migration migration;
+
+ @Override
+ public String toString() {
+ return String.format( "[%d] %s", lineNumber, migration );
+ }
+ }
+
+ @Override
+ protected void processOptions( CommandLine commandLine ) throws ParseException {
+ if ( !commandLine.hasOption( MIGRATION_FILE_OPTION ) && !commandLine.hasOption( MIGRATE_REMAINING_FACTOR_VALUES_OPTION ) ) {
+ throw new MissingOptionException( String.format( "At least one of -%s or -%s must be specified.",
+ MIGRATION_FILE_OPTION, MIGRATE_REMAINING_FACTOR_VALUES_OPTION ) );
+ }
+ if ( !commandLine.hasOption( MIGRATION_FILE_OPTION ) && commandLine.hasOption( MIGRATE_REMAINING_CHARACTERISTICS_OPTION ) ) {
+ throw new MissingOptionException( String.format( "-%s must be specified if -%s is used.",
+ MIGRATION_FILE_OPTION, MIGRATE_REMAINING_CHARACTERISTICS_OPTION ) );
+ }
+ if ( !commandLine.hasOption( MIGRATE_REMAINING_FACTOR_VALUES_OPTION ) && commandLine.hasOption( MIGRATE_NON_TRIVIAL_CASES_OPTION ) ) {
+ throw new MissingOptionException( String.format( "-%s must be specified if -%s is used.",
+ MIGRATE_REMAINING_FACTOR_VALUES_OPTION, MIGRATE_NON_TRIVIAL_CASES_OPTION ) );
+ }
+ migrations = new ArrayList<>();
+ if ( commandLine.hasOption( MIGRATION_FILE_OPTION ) ) {
+ File migrationFile = ( File ) commandLine.getParsedOptionValue( MIGRATION_FILE_OPTION );
+ try ( CSVParser parser = CSVParser.parse( migrationFile, StandardCharsets.UTF_8, CSVFormat.TDF.withFirstRecordAsHeader() ) ) {
+ boolean hasSecondObjectColumns = CollectionUtils.containsAny( parser.getHeaderNames(),
+ "SecondObjectID", "SecondObjectURI", "SecondObject" );
+ boolean hasThirdObjectColumns = CollectionUtils.containsAny( parser.getHeaderNames(),
+ "ThirdObjectID", "ThirdObjectURI", "ThirdObject" );
+ for ( CSVRecord row : parser ) {
+ FactorValueMigratorService.Migration migration;
+ try {
+ FactorValueMigratorService.Migration.MigrationBuilder migrationBuilder = FactorValueMigratorService.Migration.builder()
+ .factorValueId( parseLongIfNonBlank( row.get( "FactorValueID" ) ) )
+ .category( stripToNull( row.get( "Category" ) ) ).categoryUri( stripToNull( row.get( "CategoryURI" ) ) )
+ .oldStyleCharacteristicIdUsedAsSubject( parseLongIfNonBlank( row.get( "SubjectID" ) ) )
+ .subject( stripToNull( row.get( "Subject" ) ) ).subjectUri( stripToNull( row.get( "SubjectURI" ) ) )
+ .predicate( stripToNull( row.get( "Predicate" ) ) ).predicateUri( stripToNull( row.get( "PredicateURI" ) ) )
+ .oldStyleCharacteristicIdUsedAsObject( parseLongIfNonBlank( row.get( "ObjectID" ) ) )
+ .object( stripToNull( row.get( "Object" ) ) ).objectUri( stripToNull( row.get( "ObjectURI" ) ) );
+ if ( hasSecondObjectColumns ) {
+ // optionally...
+ migrationBuilder
+ .secondPredicate( stripToNull( row.get( "SecondPredicate" ) ) ).secondPredicateUri( stripToNull( row.get( "SecondPredicateURI" ) ) )
+ .oldStyleCharacteristicIdUsedAsSecondObject( parseLongIfNonBlank( row.get( "SecondObjectID" ) ) )
+ .secondObject( stripToNull( row.get( "SecondObject" ) ) ).secondObjectUri( stripToNull( row.get( "SecondObjectURI" ) ) );
+ }
+ if ( hasThirdObjectColumns && ( StringUtils.isNotBlank( row.get( "ThirdObjectID" ) ) || StringUtils.isNotBlank( row.get( "ThirdObjectURI" ) ) || StringUtils.isNotBlank( row.get( "ThirdObject" ) ) ) ) {
+ throw new IllegalArgumentException( "Statements do not support a third object, make sure that any of the columns related to a third object are left blank." );
+ }
+ migration = migrationBuilder.build();
+ } catch ( Exception e ) {
+ throw new RuntimeException( String.format( "The following migration is invalid:\n\t[%d] %s\n\t%s",
+ row.getRecordNumber(), Arrays.toString( row.values() ), ExceptionUtils.getRootCauseMessage( e ) ), e );
+ }
+ migrations.add( new MigrationWithLineNumber( row.getRecordNumber(), migration ) );
+ }
+ } catch ( IOException e ) {
+ throw new RuntimeException( e );
+ }
+ if ( migrations.isEmpty() ) {
+ log.warn( String.format( "The migration file %s is empty.", migrationFile.getAbsolutePath() ) );
+ }
+ }
+ migrateRemainingCharacteristics = commandLine.hasOption( MIGRATE_REMAINING_CHARACTERISTICS_OPTION );
+ migrateRemainingFactorValues = commandLine.hasOption( MIGRATE_REMAINING_FACTOR_VALUES_OPTION );
+ migrateNonTrivialCases = commandLine.hasOption( MIGRATE_NON_TRIVIAL_CASES_OPTION );
+ noop = commandLine.hasOption( NOOP_OPTION );
+ }
+
+ private Long parseLongIfNonBlank( String s ) {
+ return StringUtils.isBlank( s ) ? null : Long.parseLong( StringUtils.strip( s ) );
+ }
+
+ @Override
+ protected void doWork() throws Exception {
+ if ( noop ) {
+ log.info( "Noop mode enabled, no statements will be saved." );
+ }
+ if ( migrateRemainingFactorValues ) {
+ promptConfirmationOrAbort( "Migrating remaining factor values will create a lot of statements." );
+ }
+ if ( migrateNonTrivialCases ) {
+ promptConfirmationOrAbort( "Migrating non-trivial cases will create a lot of statements and needs attention events on the affected datasets." );
+ }
+ Map> migrationsByFactorValue = migrations.stream()
+ .collect( Collectors.groupingBy( migration -> migration.getMigration().getFactorValueId(), Collectors.toList() ) );
+ for ( Map.Entry> entry : migrationsByFactorValue.entrySet() ) {
+ Long fvId = entry.getKey();
+ List ms = entry.getValue();
+ // all the mentioned old-style characteristics will be marked for removal and the remaining ones will be
+ // ported to simple subject-only statements
+ Set mentionedOldStyleCharacteristicIds = ms.stream()
+ .map( MigrationWithLineNumber::getMigration )
+ .flatMap( m -> Stream.of( m.getOldStyleCharacteristicIdUsedAsSubject(), m.getOldStyleCharacteristicIdUsedAsObject(), m.getOldStyleCharacteristicIdUsedAsSecondObject() ) )
+ .filter( Objects::nonNull )
+ .collect( Collectors.toSet() );
+ if ( ms.size() == 1 ) {
+ MigrationWithLineNumber m = ms.iterator().next();
+ try {
+ List results = new ArrayList<>();
+ results.add( factorValueMigratorService.performMigration( m.getMigration(), noop ) );
+ if ( migrateRemainingCharacteristics ) {
+ results.addAll( factorValueMigratorService.performMigrationOfRemainingOldStyleCharacteristics( fvId, mentionedOldStyleCharacteristicIds, noop ) );
+ }
+ addSuccessObject( "FactorValue #" + fvId, summarizeMigrationResults( results ) );
+ } catch ( IllegalArgumentException e ) {
+ String summary = String.format( "[%d] %s", m.getLineNumber(), m.getMigration() );
+ addErrorObject( "FactorValue #" + fvId,
+ "The following migration failed:\n\t" + summary + "\n\t" + ExceptionUtils.getRootCauseMessage( e ) );
+ } catch ( Exception e ) {
+ throw interruptMigrationProcess( m, e );
+ }
+ } else {
+ try {
+ List fvm = ms.stream()
+ .map( MigrationWithLineNumber::getMigration )
+ .collect( Collectors.toList() );
+ List results = new ArrayList<>( fvm.size() );
+ results.addAll( factorValueMigratorService.performMultipleMigrations( fvm, noop ) );
+ if ( migrateRemainingCharacteristics ) {
+ results.addAll( factorValueMigratorService.performMigrationOfRemainingOldStyleCharacteristics( fvId, mentionedOldStyleCharacteristicIds, noop ) );
+ }
+ addSuccessObject( "FactorValue #" + fvId, summarizeMigrationResults( results ) );
+ } catch ( FactorValueMigratorService.MigrationFailedException e ) {
+ // skip the migrations that succeeded from appearing in the error message
+ int successfulMigrationsToSkip = 0;
+ for ( MigrationWithLineNumber m : ms ) {
+ if ( m.getMigration() == e.getMigration() ) {
+ break;
+ }
+ successfulMigrationsToSkip++;
+ }
+ // match it with the migration with line number
+ String summary = ms.stream()
+ .skip( successfulMigrationsToSkip )
+ .map( m -> String.format( "[%d] %s", m.getLineNumber(), m.getMigration() ) )
+ .collect( Collectors.joining( "\n\t" ) );
+ if ( e.getCause() instanceof IllegalArgumentException ) {
+ addErrorObject( "FactorValue #" + fvId,
+ "One or more of the following migrations failed:\n\t" + summary + "\n\t" + ExceptionUtils.getRootCauseMessage( e ) );
+ } else {
+ throw interruptMigrationProcess( ms.get( successfulMigrationsToSkip ), e );
+ }
+ }
+ }
+ }
+ if ( migrateRemainingFactorValues ) {
+ try {
+ factorValueMigratorService.performMigrationOfRemainingFactorValues( migrationsByFactorValue.keySet(), migrateNonTrivialCases, noop )
+ .forEach( ( fvId, stmts ) -> {
+ addSuccessObject( "FactorValue #" + fvId,
+ summarizeMigrationResults( stmts ) );
+ } );
+ } catch ( IllegalArgumentException e ) {
+ addErrorObject( "Remaining FactorValues", "Failed to migrate the remaining factor values.", e );
+ }
+ }
+ }
+
+ private String summarizeMigrationResults( List results ) {
+ if ( results.isEmpty() ) {
+ return "No statements were created or updated.";
+ } else if ( results.size() <= 5 ) {
+ return results.stream()
+ .map( r -> String.format( "%s %s", r.isCreated() ? "Created" : "Updated", r.getStatement() ) )
+ .collect( Collectors.joining( "\n" ) );
+ } else {
+ long created = results.stream()
+ .filter( FactorValueMigratorService.MigrationResult::isCreated )
+ .count();
+ long updated = ( results.size() - created );
+ if ( created > 0 && updated > 0 ) {
+ return "Created " + created + " and updated " + updated + " statements";
+ } else if ( created > 0 ) {
+ return "Created " + created + " statements";
+ } else if ( updated > 0 ) {
+ return "Updated " + updated + " statements";
+ } else {
+ return "No statements were created or updated.";
+ }
+ }
+ }
+
+ @CheckReturnValue
+ private Exception interruptMigrationProcess( MigrationWithLineNumber m, Exception cause ) {
+ log.fatal( "A " + cause.getClass().getName() + " exception occurred, the migration process will not continue." );
+ String summary = String.format( "[%d] %s", m.getLineNumber(), m.getMigration() );
+ return new Exception( "The following migration failed:\n\t" + summary, cause );
+ }
+}
diff --git a/gemma-cli/src/main/java/ubic/gemma/core/apps/FindObsoleteTermsCli.java b/gemma-cli/src/main/java/ubic/gemma/core/apps/FindObsoleteTermsCli.java
new file mode 100644
index 0000000000..effbe261f3
--- /dev/null
+++ b/gemma-cli/src/main/java/ubic/gemma/core/apps/FindObsoleteTermsCli.java
@@ -0,0 +1,103 @@
+package ubic.gemma.core.apps;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Options;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.core.task.AsyncTaskExecutor;
+import ubic.gemma.core.ontology.OntologyService;
+import ubic.gemma.core.util.AbstractCLI;
+import ubic.gemma.model.common.description.CharacteristicValueObject;
+
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CompletionService;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.Future;
+import java.util.stream.Collectors;
+
+public class FindObsoleteTermsCli extends AbstractCLI {
+
+ @Autowired
+ private OntologyService ontologyService;
+
+ @Autowired
+ @Qualifier("ontologyTaskExecutor")
+ private AsyncTaskExecutor ontologyTaskExecutor;
+
+ @Value("${load.ontologies}")
+ private boolean autoLoadOntologies;
+
+ @Autowired
+ private List ontologies;
+
+ @Override
+ public GemmaCLI.CommandGroup getCommandGroup() {
+ return GemmaCLI.CommandGroup.METADATA;
+ }
+
+ @Override
+ public String getShortDesc() {
+ return "Check for characteristics using obsolete terms as values (excluding GO), prints to sdout";
+ }
+
+ @Override
+ protected void processOptions( CommandLine commandLine ) {
+ // no extra options.
+ }
+
+ @Override
+ public String getCommandName() {
+ return "findObsoleteTerms";
+ }
+
+ @Override
+ protected void buildOptions( Options options ) {
+ }
+
+ @Override
+ protected void doWork() throws Exception {
+ if ( autoLoadOntologies ) {
+ throw new IllegalArgumentException( "Auto-loading of ontologies is enabled, disable it by setting load.ontologies=false in Gemma.properties." );
+ }
+
+ log.info( String.format( "Warming up %d ontologies ...", ontologies.size() ) );
+ CompletionService completionService = new ExecutorCompletionService<>( ontologyTaskExecutor );
+ Map> futures = new LinkedHashMap<>();
+ for ( ubic.basecode.ontology.providers.OntologyService ontology : ontologies ) {
+ futures.put( ontology, completionService.submit( () -> {
+ // we don't need all those features for detecting obsolete terms
+ ontology.setSearchEnabled( false );
+ ontology.setInferenceMode( ubic.basecode.ontology.providers.OntologyService.InferenceMode.NONE );
+ ontology.initialize( true, false );
+ return ontology;
+ } ) );
+ }
+
+ for ( int i = 0; i < ontologies.size(); i++ ) {
+ ubic.basecode.ontology.providers.OntologyService os = completionService.take().get();
+ log.info( String.format( " === Ontology (%d/%d) warmed up: %s", i + 1, ontologies.size(), os ) );
+ int remainingToLoad = ontologies.size() - ( i + 1 );
+ if ( remainingToLoad > 0 && remainingToLoad <= 5 ) {
+ log.info( "Still loading:\n\t" + futures.entrySet().stream().filter( e -> !e.getValue().isDone() )
+ .map( Map.Entry::getKey )
+ .map( ubic.basecode.ontology.providers.OntologyService::toString )
+ .collect( Collectors.joining( "\n\t" ) ) );
+ }
+ }
+
+ log.info( "Ontologies warmed up, starting check..." );
+
+ Map vos = ontologyService.findObsoleteTermUsage();
+
+ AbstractCLI.log.info( "Obsolete term check finished, printing ..." );
+
+ System.out.println( "Value\tValueUri\tCount" );
+ for ( CharacteristicValueObject vo : vos.values() ) {
+ System.out.println( vo.getValue() + "\t" + vo.getValueUri() + "\t" + vo.getNumTimesUsed() );
+ }
+
+ }
+}
diff --git a/gemma-cli/src/main/java/ubic/gemma/core/apps/GeeqCli.java b/gemma-cli/src/main/java/ubic/gemma/core/apps/GeeqCli.java
index a8053839b0..d17c00f293 100644
--- a/gemma-cli/src/main/java/ubic/gemma/core/apps/GeeqCli.java
+++ b/gemma-cli/src/main/java/ubic/gemma/core/apps/GeeqCli.java
@@ -21,8 +21,8 @@
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Component;
import ubic.gemma.core.util.AbstractCLI;
import ubic.gemma.model.common.auditAndSecurity.eventType.GeeqEvent;
import ubic.gemma.model.expression.experiment.BioAssaySet;
@@ -34,7 +34,6 @@
*
* @author tesar
*/
-@Component
public class GeeqCli extends ExpressionExperimentManipulatingCLI {
@Autowired
@@ -48,7 +47,7 @@ public String getShortDesc() {
}
@Override
- protected void processOptions( CommandLine commandLine ) {
+ protected void processOptions( CommandLine commandLine ) throws ParseException {
super.processOptions( commandLine );
if ( commandLine.hasOption( 'm' ) ) {
this.mode = GeeqService.ScoreMode.valueOf( commandLine.getOptionValue( 'm' ) );
@@ -64,9 +63,8 @@ public String getCommandName() {
protected void buildOptions( Options options ) {
super.buildOptions( options );
- super.addAutoOption( options );
+ super.addAutoOption( options, GeeqEvent.class );
super.addDateOption( options );
- this.autoSeekEventType = GeeqEvent.class;
super.addForceOption( options );
Option modeOption = Option.builder( "m" ).longOpt( "mode" )
diff --git a/gemma-cli/src/main/java/ubic/gemma/core/apps/GemmaCLI.java b/gemma-cli/src/main/java/ubic/gemma/core/apps/GemmaCLI.java
index 2ba7511c78..4a2b428c98 100644
--- a/gemma-cli/src/main/java/ubic/gemma/core/apps/GemmaCLI.java
+++ b/gemma-cli/src/main/java/ubic/gemma/core/apps/GemmaCLI.java
@@ -29,7 +29,6 @@
import ubic.gemma.core.util.AbstractCLI;
import ubic.gemma.core.util.BuildInfo;
import ubic.gemma.core.util.CLI;
-import ubic.gemma.persistence.util.Settings;
import ubic.gemma.persistence.util.SpringContextUtil;
import ubic.gemma.persistence.util.SpringProfiles;
@@ -93,14 +92,14 @@ public static void main( String[] args ) {
// quick help without loading the context
if ( commandLine.hasOption( HELP_OPTION ) ) {
- GemmaCLI.printHelp( options, null );
+ GemmaCLI.printHelp( options, null, new PrintWriter( System.out, true ) );
System.exit( 0 );
return;
}
if ( commandLine.hasOption( VERSION_OPTION ) ) {
BuildInfo buildInfo = BuildInfo.fromSettings();
- System.err.printf( "Gemma version %s%n", buildInfo );
+ System.out.printf( "Gemma version %s%n", buildInfo );
System.exit( 0 );
return;
}
@@ -111,6 +110,7 @@ public static void main( String[] args ) {
} catch ( ParseException | IllegalArgumentException e ) {
System.err.printf( "Failed to parse the %s option: %s.%n", VERBOSITY_OPTION,
ExceptionUtils.getRootCauseMessage( e ) );
+ GemmaCLI.printHelp( options, null, new PrintWriter( System.err, true ) );
System.exit( 1 );
return;
}
@@ -127,9 +127,10 @@ public static void main( String[] args ) {
try {
loggingConfigurer.configureLogger( loggerName, Integer.parseInt( vals[1] ) );
} catch ( IllegalArgumentException e ) {
- System.err.printf( "Failed to parse the %s option for %s: %s.%n", VERBOSITY_OPTION,
+ System.err.printf( "Failed to parse the %s option for %s: %s.%n", LOGGER_OPTION,
loggerName,
ExceptionUtils.getRootCauseMessage( e ) );
+ GemmaCLI.printHelp( options, null, new PrintWriter( System.err, true ) );
System.exit( 1 );
return;
}
@@ -151,8 +152,6 @@ public static void main( String[] args ) {
profiles.add( SpringProfiles.TEST );
}
- lintConfiguration();
-
ApplicationContext ctx = SpringContextUtil.getApplicationContext( profiles.toArray( new String[0] ) );
/*
@@ -178,7 +177,7 @@ public static void main( String[] args ) {
// no command is passed
if ( commandLine.getArgList().isEmpty() ) {
System.err.println( "No command was supplied." );
- GemmaCLI.printHelp( options, commandGroups );
+ GemmaCLI.printHelp( options, commandGroups, new PrintWriter( System.err, true ) );
System.exit( 1 );
}
@@ -190,7 +189,7 @@ public static void main( String[] args ) {
int statusCode;
if ( !commandsByName.containsKey( commandRequested ) ) {
System.err.println( "Unrecognized command: " + commandRequested );
- GemmaCLI.printHelp( options, commandGroups );
+ GemmaCLI.printHelp( options, commandGroups, new PrintWriter( System.err, true ) );
statusCode = 1;
} else {
try {
@@ -221,23 +220,8 @@ static String getOptStringForLogging( Object[] argsToPass ) {
return matcher.replaceAll( "$1 XXXXXX" );
}
- private static void lintConfiguration() {
- // check some common settings that might affect initialization time
- if ( Settings.getBoolean( "load.ontologies" ) ) {
- log.warn( "Auto-loading of ontologies is enabled, this is not recommended for the CLI. Disable it by setting load.ontologies=false in Gemma.properties." );
- }
-
- if ( Settings.getBoolean( "load.homologene" ) ) {
- log.warn( "Homologene is enabled, this is not recommended for the CLI. Disable it by setting load.homologene=false in Gemma.properties." );
- }
-
- if ( Settings.getString( "gemma.hibernate.hbm2ddl.auto" ).equals( "validate" ) ) {
- log.warn( "Hibernate is configured to validate the database schema, this is not recommended for the CLI. Disable it by setting gemma.hibernate.hbm2ddl.auto= in Gemma.properties." );
- }
- }
-
- private static void printHelp( Options options, @Nullable SortedMap> commands ) {
- System.err.println( "============ Gemma CLI tools ============" );
+ private static void printHelp( Options options, @Nullable SortedMap> commands, PrintWriter writer ) {
+ writer.println( "============ Gemma CLI tools ============" );
StringBuilder footer = new StringBuilder();
if ( commands != null ) {
@@ -259,7 +243,7 @@ private static void printHelp( Options options, @Nullable SortedMap [options]",
+ new HelpFormatter().printHelp( writer, 150, "gemma-cli [options]",
AbstractCLI.HEADER, options, HelpFormatter.DEFAULT_LEFT_PAD, HelpFormatter.DEFAULT_DESC_PAD, footer.toString() );
}
diff --git a/gemma-cli/src/main/java/ubic/gemma/core/apps/GenericGenelistDesignGenerator.java b/gemma-cli/src/main/java/ubic/gemma/core/apps/GenericGenelistDesignGenerator.java
index 50fc65f0be..f92fffda37 100644
--- a/gemma-cli/src/main/java/ubic/gemma/core/apps/GenericGenelistDesignGenerator.java
+++ b/gemma-cli/src/main/java/ubic/gemma/core/apps/GenericGenelistDesignGenerator.java
@@ -17,18 +17,16 @@
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
-import org.apache.commons.lang3.StringUtils;
+import org.springframework.beans.factory.annotation.Autowired;
import ubic.gemma.core.analysis.report.ArrayDesignReportService;
import ubic.gemma.core.analysis.service.ArrayDesignAnnotationService;
import ubic.gemma.core.apps.GemmaCLI.CommandGroup;
import ubic.gemma.core.genome.gene.service.GeneService;
+import ubic.gemma.core.util.AbstractAuthenticatedCLI;
import ubic.gemma.core.util.AbstractCLI;
-import ubic.gemma.core.util.AbstractCLIContextCLI;
+import ubic.gemma.core.util.FileUtils;
import ubic.gemma.model.common.auditAndSecurity.eventType.AnnotationBasedGeneMappingEvent;
-import ubic.gemma.model.common.description.DatabaseEntry;
-import ubic.gemma.model.common.description.ExternalDatabase;
import ubic.gemma.model.expression.arrayDesign.ArrayDesign;
-import ubic.gemma.model.expression.arrayDesign.TechnologyType;
import ubic.gemma.model.expression.designElement.CompositeSequence;
import ubic.gemma.model.genome.Gene;
import ubic.gemma.model.genome.Taxon;
@@ -37,40 +35,54 @@
import ubic.gemma.model.genome.biosequence.SequenceType;
import ubic.gemma.model.genome.gene.GeneProduct;
import ubic.gemma.model.genome.sequenceAnalysis.AnnotationAssociation;
-import ubic.gemma.persistence.service.common.description.ExternalDatabaseService;
+import ubic.gemma.persistence.service.common.auditAndSecurity.AuditTrailService;
import ubic.gemma.persistence.service.expression.arrayDesign.ArrayDesignService;
import ubic.gemma.persistence.service.expression.designElement.CompositeSequenceService;
import ubic.gemma.persistence.service.genome.biosequence.BioSequenceService;
+import ubic.gemma.persistence.service.genome.gene.GeneProductService;
import ubic.gemma.persistence.service.genome.sequenceAnalysis.AnnotationAssociationService;
import ubic.gemma.persistence.service.genome.taxon.TaxonService;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Map;
+import java.io.File;
+import java.util.*;
/**
- * Create (or update) an array design based on the current set of transcripts for a taxon.
- * This is used to create a 'platform' for linking non-array based data to the system, or data for which we have only
- * gene or transcript-level information.
- * See also: To generate annotation files for all genes in a taxon, this can also accomplished by
- * ArrayDesignAnnotationFileCli. The difference here is that an array design is actually created.
+ * Create (or update) an array design based on a list of NCBI gene IDs desired to be on the platform.
*
* @author paul
*/
-public class GenericGenelistDesignGenerator extends AbstractCLIContextCLI {
+public class GenericGenelistDesignGenerator extends AbstractAuthenticatedCLI {
+ @Autowired
private AnnotationAssociationService annotationAssociationService;
+ @Autowired
private ArrayDesignAnnotationService arrayDesignAnnotationService;
+ @Autowired
private ArrayDesignReportService arrayDesignReportService;
+ @Autowired
private ArrayDesignService arrayDesignService;
+ @Autowired
private BioSequenceService bioSequenceService;
+ @Autowired
private CompositeSequenceService compositeSequenceService;
- private ExternalDatabaseService externalDatabaseService;
+ @Autowired
private GeneService geneService;
+ @Autowired
+ private GeneProductService geneProductService;
+
+ @Autowired
+ private TaxonService taxonService;
+
+ private String platformShortName = null;
+ private String geneListFileName = null;
private Taxon taxon = null;
- private boolean useEnsemblIds = false;
- private boolean useNCBIIds = false;
+
+ private boolean noDB = false;
+
+
+ @Autowired
+ private AuditTrailService auditTrailService;
@Override
public CommandGroup getCommandGroup() {
@@ -82,377 +94,267 @@ public String getCommandName() {
return "genericPlatform";
}
- @SuppressWarnings("static-access")
@Override
protected void buildOptions( Options options ) {
- options.addOption( Option.builder( "t" ).longOpt( "taxon" ).desc( "Taxon of the genes" ).argName( "taxon" ).hasArg().build() );
- options.addOption( "ncbiids", "use NCBI numeric IDs as the identifiers instead of gene symbols" );
- options.addOption( "ensembl", "use Ensembl identifiers instead of gene symbols" );
- }
-
- @Override
- protected void doWork() throws Exception {
- ExternalDatabase genbank = externalDatabaseService.findByName( "Genbank" );
- ExternalDatabase ensembl = externalDatabaseService.findByName( "Ensembl" );
- if ( genbank == null || ensembl == null ) {
- throw new IllegalStateException( "A record for Genbank and/or Ensembl couldn't be found" );
- }
+ options.addOption( Option.builder( "t" ).longOpt( "taxon" ).desc( "Taxon of the genes" ).argName( "taxon" ).required().hasArg().build() );
- /*
- * Create the stub array design for the organism. The name and etc. are generated automatically. If the design
- * exists, we update it.
- */
+ Option arrayDesignOption = Option.builder( "a" ).hasArg().argName( "shortName" )
+ .desc( "Platform short name (existing or new to add)" ).required().longOpt( "platform" ).build();
+ options.addOption( arrayDesignOption );
- String shortName = this.generateShortName();
+ Option geneListOption = Option.builder( "f" ).hasArg().argName( "file" ).desc(
+ "File with list of NCBI IDs of genes to add to platform (one per line)" )
+ .longOpt( "geneListFile" ).required().build();
+ options.addOption( geneListOption );
- ArrayDesign arrayDesign = ArrayDesign.Factory.newInstance();
- arrayDesign.setShortName( shortName );
+ options.addOption( Option.builder( "nodb" ).desc( "Dry run: Do not update the database nor delete any flat files" ).build() );
+ }
- // common name
- arrayDesign.setPrimaryTaxon( taxon );
- String nameExt = useNCBIIds ? ", indexed by NCBI IDs" : useEnsemblIds ? ", indexed by Ensembl IDs" : "";
- arrayDesign.setName( "Generic platform for " + taxon.getScientificName() + nameExt );
- arrayDesign.setDescription( "Created by Gemma" );
- arrayDesign.setTechnologyType( TechnologyType.GENELIST ); // this is key
+ @Override
+ protected void doWork() throws Exception {
- if ( arrayDesignService.find( arrayDesign ) != null ) {
- AbstractCLI.log.info( "Platform for " + taxon + " already exists, will update" );
- arrayDesign = arrayDesignService.findOrFail( arrayDesign );
- arrayDesignService.deleteGeneProductAnnotationAssociations( arrayDesign );
- arrayDesign = arrayDesignService.loadOrFail( arrayDesign.getId() );
+ ArrayDesign platform = arrayDesignService.findByShortName( this.platformShortName );
+ platform = arrayDesignService.thaw( platform );
- } else {
- AbstractCLI.log.info( "Creating new 'generic' platform" );
- arrayDesign = arrayDesignService.create( arrayDesign );
+ // test whether the geneListFileName file exists and is readable
+ File geneListFile = new File( this.geneListFileName );
+ if ( !geneListFile.exists() || !geneListFile.canRead() ) {
+ throw new IllegalArgumentException( "File " + this.geneListFileName + " does not exist or cannot be read" );
}
- arrayDesign = arrayDesignService.thaw( arrayDesign );
-
- // temporary: making sure we set it, as it is new.
- arrayDesign.setTechnologyType( TechnologyType.GENELIST );
- /*
- * Load up the genes for the organism.
- */
- Collection knownGenes = geneService.loadAll( taxon );
- AbstractCLI.log.info( "Taxon has " + knownGenes.size() + " genes" );
-
- // this would be good for cases where the identifier we are using has changed.
- Map existingGeneMap = new HashMap<>();
-
- if ( !useNCBIIds && !useEnsemblIds ) {
- // only using this for symbol changes.
- existingGeneMap = this.getExistingGeneMap( arrayDesign );
- }
+ Set ncbiIds = new HashSet( FileUtils.readListFileToStrings( this.geneListFileName ) );
+ AbstractCLI.log.info( "File had " + ncbiIds.size() + " gene ids" );
- Map existingSymbolMap = this.getExistingProbeNameMap( arrayDesign );
+ Map existingSymbolMap = this.nameMap( platform );
int count = 0;
int numWithNoTranscript = 0;
- // int hasGeneAlready = 0;
+ int hasGeneAlready = 0;
// int numNewGenes = 0;
+ int geneNotFound = 0;
int numNewElements = 0;
int numUpdatedElements = 0;
- for ( Gene gene : knownGenes ) {
- gene = geneService.thaw( gene );
+ int needsDummyElement = 0;
+ for ( String ncbiId : ncbiIds ) {
- Collection products = gene.getProducts();
-
- log.debug( "> Processing: " + gene.getOfficialSymbol() );
-
- if ( products.isEmpty() ) {
- numWithNoTranscript++;
- AbstractCLI.log.info( "No transcript for " + gene );
- continue;
- }
-
- count++;
+ log.debug( "> Processing element for NCBI ID = " + ncbiId );
CompositeSequence csForGene = null;
-
- if ( useNCBIIds ) {
- if ( gene.getNcbiGeneId() == null ) {
- AbstractCLI.log.debug( "No NCBI ID for " + gene + ", skipping" );
- continue;
- }
- if ( existingSymbolMap.containsKey( gene.getNcbiGeneId().toString() ) ) {
- csForGene = existingSymbolMap.get( gene.getNcbiGeneId().toString() );
- log.debug( " ... gene exists on platform, will update if necessary [" + csForGene + "]" );
- }
- } else if ( useEnsemblIds ) {
- if ( gene.getEnsemblId() == null ) {
- AbstractCLI.log.debug( "No Ensembl ID for " + gene + ", skipping" );
+ if ( existingSymbolMap.containsKey( ncbiId ) ) {
+ /*
+ Work out if the existing association is to a dummy sequence or not; if not, we have to make a new one.
+ */
+ csForGene = existingSymbolMap.get( ncbiId );
+
+ if ( csForGene.getBiologicalCharacteristic().getType() == null ) {
+// log.info( "Gene NCBI ID=" + ncbiId + " already has an element [" + csForGene + "], but sequence type of " + csForGene.getBiologicalCharacteristic()
+// + " is null, will replace with dummy" ); // rare case
+ needsDummyElement++;
+ } else if ( csForGene.getBiologicalCharacteristic().getType().equals( SequenceType.DUMMY ) ) {
+ log.debug( "Gene NCBI ID=" + ncbiId + " already has a usable element, nothing to be done" ); // FOR NOW. This could be a dangling sequence if the gene didn't exist.
continue;
- }
- if ( existingSymbolMap.containsKey( gene.getEnsemblId() ) ) {
- csForGene = existingSymbolMap.get( gene.getEnsemblId() );
- log.debug( " ... gene exists on platform , will update if necessary [" + csForGene + "]" );
+ } else {
+ needsDummyElement++;
+ // AbstractCLI.log.info( "Gene NCBI ID=" + ncbiId + " already has an element [" + csForGene + "], but it is not a dummy, will update" );
}
} else {
- // the "by symbols" platform.
+ AbstractCLI.log.info( "Gene NCBI ID=" + ncbiId + " not on platform, may add" );
+ }
- /*
- * detect when the symbol has changed
- */
- if ( existingSymbolMap.containsKey( gene.getOfficialSymbol() ) ) {
- csForGene = existingSymbolMap.get( gene.getOfficialSymbol() );
- } else if ( existingGeneMap.containsKey( gene ) ) {
- csForGene = existingGeneMap.get( gene );
- AbstractCLI.log
- .debug( "Gene symbol has changed for: " + gene + "? Current element has name=" + csForGene
- .getName() );
- csForGene.setName( gene.getOfficialSymbol() );
- }
+ Gene gene = null;
+ try {
+ gene = geneService.findByNCBIId( Integer.parseInt( ncbiId ) );
+ } catch ( NumberFormatException e ) {
+ // shouldn't happen but just in case
+ AbstractCLI.log.error( "Could not parse NCBI ID = " + ncbiId + " as an integer" );
}
- assert csForGene == null || csForGene.getId() != null : "Null id for " + csForGene;
+ boolean geneExists = gene != null;
+ if ( !geneExists ) {
+ AbstractCLI.log.warn( "No gene for NCBI ID = " + ncbiId + " but adding dummy sequence anyway (no gene product association wil be made)" );
+ geneNotFound++;
+ // continue;
+ // but still make sure there is an element on the platform for it.
+ } else {
+ gene = geneService.thawLite( gene );
+ }
+ if ( gene != null && gene.getProducts().isEmpty() ) {
+ numWithNoTranscript++;
+ AbstractCLI.log.info( "No transcripts for " + gene + ", adding element anyway" );
+ }
+
+ AnnotationAssociation aa = null;
+ Collection associationsToRemove = new HashSet<>();
/*
- * We arbitrarily link the "probe" to one of the gene's RNA transcripts. We could consider other strategies
- * to pick the representative, but it generally doesn't matter.
+ This block is to try to re-use existing usable dummy elements for the gene, but for the first time run it mostly just finds one that we want to remove.
+ Such re-use makes sense if we have multiple "generations" of the same platform but if we have just one, this really isn't necessary (and it's going to be slow because of the thaws)
*/
- for ( GeneProduct geneProduct : products ) {
-
- /*
- * Name is usually the genbank or ensembl accession
- */
- String name = geneProduct.getName();
- BioSequence bioSequence = BioSequence.Factory.newInstance();
- Collection accessions = geneProduct.getAccessions();
- bioSequence.setName( name );
- bioSequence.setTaxon( taxon );
- bioSequence.setPolymerType( PolymerType.RNA );
- bioSequence.setType( SequenceType.mRNA );
- BioSequence existing = null;
-
- if ( accessions.isEmpty() ) {
- // this should not be hit.
- AbstractCLI.log.warn( "No accession for " + name );
- DatabaseEntry de = DatabaseEntry.Factory.newInstance();
- de.setAccession( name );
- if ( name.startsWith( "ENS" ) && name.length() > 10 ) {
- de.setExternalDatabase( ensembl );
- } else {
- if ( name.matches( "^[A-Z]{1,2}(_?)[0-9]+(\\.[0-9]+)?$" ) ) {
- de.setExternalDatabase( genbank );
- } else {
- AbstractCLI.log.info( "Name doesn't look like genbank or ensembl, skipping: " + name );
- continue;
+ if ( gene != null && !gene.getProducts().isEmpty() ) {
+ Collection aas = annotationAssociationService.find( gene ); // making fetching eager would help avoid thaws below, but not a big deal.
+ for ( AnnotationAssociation aae : aas ) {
+ GeneProduct gp = geneProductService.thaw( aae.getGeneProduct() );
+ BioSequence bp = bioSequenceService.thaw( aae.getBioSequence() );
+ if ( gp == null || bp == null ) {
+ log.warn( "Invalid association of gp=" + gp + " and bp=" + bp + " for " + gene + ", marking for removal" );
+ associationsToRemove.add( aae );
+ } else if ( bp.getType() != null && bp.getType().equals( SequenceType.DUMMY ) && gp.isDummy() ) {
+ if ( aa != null ) { // this is a sanity check, if we are sure this isn't an issue we can just break here.
+ throw new IllegalStateException( "More than one dummy annotation association for " + gene );
}
+ log.info( "Re-using dummy association for " + gene );
+ aa = aae;
+ } else {
+ // otherwise, we're going to want to delete these old AnnotationAssociations assuming they aren't used for anything.
+ associationsToRemove.add( aae );
}
- bioSequence.setSequenceDatabaseEntry( de );
- } else {
- // FIXME It is possible that this sequence will have been aligned to the genome, which is a bit
- // confusing. So it will map to a gene. Worse case: it maps to more than one gene ...
- existing = bioSequenceService.findByAccession( accessions.iterator().next() );
- if ( existing == null ) {
- // create a copy, each biosequence must own their database entry
- DatabaseEntry databaseEntry = accessions.iterator().next();
- DatabaseEntry clone = DatabaseEntry.Factory.newInstance();
- clone.setAccession( databaseEntry.getAccession() );
- clone.setAccessionVersion( databaseEntry.getAccessionVersion() );
- clone.setUri( databaseEntry.getUri() );
- clone.setExternalDatabase( databaseEntry.getExternalDatabase() );
- bioSequence.setSequenceDatabaseEntry( clone );
- }
- }
-
- if ( existing == null ) {
- bioSequence = ( BioSequence ) this.getPersisterHelper().persist( bioSequence );
- } else {
- bioSequence = existing;
}
+ }
- assert bioSequence != null && bioSequence.getId() != null;
+ BioSequence bioSequence = null;
+ if ( aa == null ) {
+ /* create a dummy gene Product and sequence for the gene. */
- if ( bioSequence.getSequenceDatabaseEntry() == null ) {
- AbstractCLI.log.info( "No DB entry for " + bioSequence + "(" + gene
- + "), will look for a better sequence to use ..." );
- continue;
- }
+ bioSequence = csForGene == null ? null : csForGene.getBiologicalCharacteristic();
+ if ( bioSequence != null && bioSequence.getType() != null && bioSequence.getType().equals( SequenceType.DUMMY ) ) {
+ log.debug( "Existing dummy sequence for element, will reuse" );
+ } else {
+ bioSequence = BioSequence.Factory.newInstance();
- if ( csForGene == null ) { // i.e. it is new
- log.info( "New element " + " with sequence used:" + bioSequence.getName() + " for " + gene.getOfficialSymbol() );
- csForGene = CompositeSequence.Factory.newInstance();
- if ( useNCBIIds ) {
- csForGene.setName( gene.getNcbiGeneId().toString() );
- } else if ( useEnsemblIds ) {
- csForGene.setName( gene.getEnsemblId() );
+ if ( gene == null ) {
+ bioSequence.setName( "NCBI ID=" + ncbiId + " generic sequence placeholder" );
} else {
- csForGene.setName( gene.getOfficialSymbol() );
- }
-
- csForGene.setArrayDesign( arrayDesign );
- csForGene.setBiologicalCharacteristic( bioSequence );
- csForGene.setDescription( "Generic expression element for " + gene );
- csForGene = compositeSequenceService.create( csForGene );
- assert csForGene.getId() != null : "No id for " + csForGene + " for " + gene;
- arrayDesign.getCompositeSequences().add( csForGene );
- numNewElements++;
- } else { // i.e. it is already in the system; just updating
- boolean changed = false;
- assert csForGene.getId() != null : "No id for " + csForGene + " for " + gene;
-
- if ( !csForGene.getArrayDesign().equals( arrayDesign ) ) {
- // does this happen?
- log.debug( "Platform changed? " + csForGene + " on " + csForGene.getArrayDesign() );
- csForGene.setArrayDesign( arrayDesign );
- csForGene.setDescription( "Generic expression element for " + gene );
- changed = true;
+ bioSequence.setName( gene.getOfficialSymbol() + " [NCBI ID=" + gene.getNcbiGeneId() + "] generic sequence placeholder" );
}
+ bioSequence.setTaxon( this.taxon );
+ bioSequence.setPolymerType( PolymerType.RNA );
+ bioSequence.setType( SequenceType.DUMMY );
+ if ( !noDB ) bioSequence = bioSequenceService.create( bioSequence );
+ }
- if ( csForGene.getBiologicalCharacteristic() == null ) {
- log.warn( csForGene + " had no sequence, setting to " + bioSequence );
- csForGene.setBiologicalCharacteristic( bioSequence );
- changed = true;
- }
+ if ( geneExists ) { // we only create the Biosequence side if the gene doesn't exist. But we make this dummy gene product even if the gene has no transcripts in Gemma.
+ GeneProduct geneProduct = GeneProduct.Factory.newInstance();
+ geneProduct.setGene( gene );
+ geneProduct.setDummy( true );
+ geneProduct.setName( gene.getOfficialSymbol() + " [NCBI ID=" + gene.getNcbiGeneId() + "] generic element placeholder" );
+ if ( !noDB ) geneProduct = geneProductService.create( geneProduct );
- if ( !csForGene.getBiologicalCharacteristic().equals( bioSequence ) ) {
- // does this happen?
- csForGene.setBiologicalCharacteristic( bioSequence );
- changed = true;
- }
+ aa = AnnotationAssociation.Factory.newInstance();
+ aa.setGeneProduct( geneProduct );
+ aa.setBioSequence( bioSequence );
+ if ( !noDB ) aa = annotationAssociationService.create( aa );
- if ( changed ) {
- compositeSequenceService.update( csForGene );
- }
-
- // making sure ...
- csForGene = compositeSequenceService.loadOrFail( csForGene.getId() );
- assert csForGene.getId() != null;
- arrayDesign.getCompositeSequences().add( csForGene );
-
- if ( changed ) {
- if ( AbstractCLI.log.isDebugEnabled() )
- AbstractCLI.log
- .debug( "Updating existing element: " + csForGene + " with " + bioSequence + " for "
- + gene );
- numUpdatedElements++;
- }
+ assert noDB || bioSequence.getId() != null;
+ } else {
+ log.info( "Gene does not exist, will not create dummy gene product or annotation association" );
}
+ }
- assert bioSequence.getId() != null;
- assert geneProduct.getId() != null;
- assert csForGene.getBiologicalCharacteristic() != null
- && csForGene.getBiologicalCharacteristic().getId() != null;
+ if ( csForGene == null ) {
+ if ( gene == null ) {
+ log.info( "New platform element for NCBI=" + ncbiId + " (" + this.taxon.getCommonName() + ") - but no corresponding gene exists in Gemma" );
+ } else {
+ log.info( "New platform element for " + gene.getOfficialSymbol() + " NCBI=" + gene.getNcbiGeneId() + " (" + gene.getTaxon().getCommonName() + ")" );
+ }
+ csForGene = CompositeSequence.Factory.newInstance();
+ csForGene.setName( ncbiId ); // IMPORTANT that this be just the NCBI ID.
+ csForGene.setArrayDesign( platform );
+ csForGene.setBiologicalCharacteristic( bioSequence );
+ if ( gene == null ) {
+ csForGene.setDescription( "Generic expression element for NCBI ID = " + ncbiId );
+ } else {
+ csForGene.setDescription( "Generic expression element for " + gene );
+ }
+ if ( !noDB ) csForGene = compositeSequenceService.create( csForGene );
- AnnotationAssociation aa = AnnotationAssociation.Factory.newInstance();
- aa.setGeneProduct( geneProduct );
- aa.setBioSequence( bioSequence );
- annotationAssociationService.create( aa );
+ platform.getCompositeSequences().add( csForGene );
+ numNewElements++;
+ } else {
+ if ( gene == null ) { // this shouldn't happen.
+ log.info( "Updating element to use dummy for NCBI=" + ncbiId + " (" + taxon.getCommonName() + ")" );
+ } else {
+ log.info( "Updating element to use dummy for " + gene.getOfficialSymbol() + ": NCBI=" + gene.getNcbiGeneId() + " (" + gene.getTaxon().getCommonName() + ")" );
+ }
+ csForGene.setBiologicalCharacteristic( aa.getBioSequence() );
+ if ( !noDB ) compositeSequenceService.update( csForGene );
+ numUpdatedElements++;
+ }
- break;
+ if ( !associationsToRemove.isEmpty() ) {
+ log.info( associationsToRemove.size() + " old 'non-dummy' annotation associations to remove" );
+ if ( !noDB ) {
+ try {
+ annotationAssociationService.remove( associationsToRemove ); // may fail if there are other associations.
+ } catch ( Exception e ) {
+ log.warn( "Could not delete old annotation associations for " + gene + ": " + e.getMessage() );
+ }
+ }
}
- if ( count % 100 == 0 )
- AbstractCLI.log
- .info( count + " genes processed; " + numNewElements + " new elements; " + numUpdatedElements
- + " updated elements; " + numWithNoTranscript
- + " genes had no transcript and were skipped." );
+ assert noDB || ( csForGene.getBiologicalCharacteristic() != null
+ && csForGene.getBiologicalCharacteristic().getId() != null );
+
+ count++;
+ if ( count % 200 == 0 )
+ log.info( " >>>>>>>>> " + count + " genes processed; " + numNewElements + " new elements; " + numUpdatedElements
+ + " updated elements; " + numWithNoTranscript
+ + " genes had no transcript." );
}
- AbstractCLI.log.info( "Platform has " + arrayDesignService.numCompositeSequenceWithGenes( arrayDesign )
+ AbstractCLI.log.info( "Platform has " + arrayDesignService.numCompositeSequenceWithGenes( platform )
+ " 'elements' associated with genes." );
- arrayDesignReportService.generateArrayDesignReport( arrayDesign.getId() );
+ if ( !noDB ) arrayDesignReportService.generateArrayDesignReport( platform.getId() );
+
+ String auditMessage = count + " genes processed; " + numNewElements + " new elements; " + numUpdatedElements
+ + " updated elements; " + numWithNoTranscript + " genes had no transcript; " + geneNotFound + " genes from the file could not be found";
+ log.info( auditMessage );
- log.info( count + " genes processed; " + numNewElements + " new elements; " + numUpdatedElements
- + " updated elements; " + numWithNoTranscript + " genes had no transcript and were skipped." );
+ if ( !noDB ) auditTrailService.addUpdateEvent( platform, AnnotationBasedGeneMappingEvent.class, auditMessage );
- auditTrailService.addUpdateEvent( arrayDesign, AnnotationBasedGeneMappingEvent.class,
- count + " genes processed; " + numNewElements + " new elements; " + numUpdatedElements
- + " updated elements; " + numWithNoTranscript + " genes had no transcript and were skipped." );
- arrayDesignAnnotationService.deleteExistingFiles( arrayDesign );
+ AbstractCLI.log.info( "Don't forget to update the annotation files, any old ones will be deleted (unless dry run)" );
+ if ( !noDB ) arrayDesignAnnotationService.deleteExistingFiles( platform );
- AbstractCLI.log.info( "Don't forget to update the annotation files" );
+ /*
+ Delete elements for the platform that are not on the input list. This should probably not be kept here; we'll do it offline.
+ */
+// for ( String geneID : existingSymbolMap.keySet() ) {
+// if ( !ncbiIds.contains( geneID ) ) {
+// log.info( "Gene " + geneID + " is not in the input list, will remove element from platform if possible (" + existingSymbolMap.get( geneID ) + ")" );
+// if ( !noDB ) {
+// try {
+// // compositeSequenceService.remove( existingSymbolMap.get( geneID ) );
+// } catch ( Exception e ) {
+// // if there is data associated with it, this will fail. We would need to delete DEA results at least.
+// log.warn( "Could not remove unneeded platform element for geneID=" + geneID + ": " + existingSymbolMap.get( geneID ) + ": " + e.getMessage() );
+// }
+// }
+// }
+// }
}
@Override
public String getShortDesc() {
- return "Update or create a 'platform' based on the genes for the organism";
+ return "Update a 'platform' based on a list of NCBI IDs";
}
@Override
protected void processOptions( CommandLine commandLine ) {
- geneService = this.getBean( GeneService.class );
- arrayDesignAnnotationService = this.getBean( ArrayDesignAnnotationService.class );
- TaxonService taxonService = this.getBean( TaxonService.class );
- bioSequenceService = this.getBean( BioSequenceService.class );
- arrayDesignService = this.getBean( ArrayDesignService.class );
- compositeSequenceService = this.getBean( CompositeSequenceService.class );
- annotationAssociationService = this.getBean( AnnotationAssociationService.class );
- externalDatabaseService = this.getBean( ExternalDatabaseService.class );
- arrayDesignReportService = this.getBean( ArrayDesignReportService.class );
-
- if ( commandLine.hasOption( 't' ) ) {
- this.taxon = this.setTaxonByName( commandLine, taxonService );
- }
- if ( commandLine.hasOption( "ncbiids" ) ) {
- this.useNCBIIds = true;
- } else if ( commandLine.hasOption( "ensembl" ) ) {
- this.useEnsemblIds = true;
- }
+ this.platformShortName = commandLine.getOptionValue( "a" );
+ this.taxon = this.taxonService.findByCommonName( commandLine.getOptionValue( "t" ) );
+ this.geneListFileName = commandLine.getOptionValue( "f" );
- if ( useNCBIIds && useEnsemblIds ) {
- throw new IllegalArgumentException( "Choose one of ensembl or ncbi ids or gene symbols" );
- }
- }
+ this.noDB = commandLine.hasOption( "nodb" );
- private String generateShortName() {
- String ncbiIdSuffix = useNCBIIds ? "_ncbiIds" : "";
- String ensemblIdSuffix = useEnsemblIds ? "_ensemblIds" : "";
- String shortName;
- if ( StringUtils.isBlank( taxon.getCommonName() ) ) {
- shortName = "Generic_" + StringUtils.strip( taxon.getScientificName() ).replaceAll( " ", "_" ) + ncbiIdSuffix;
- } else {
- shortName = "Generic_" + StringUtils.strip( taxon.getCommonName() ).replaceAll( " ", "_" ) + ncbiIdSuffix
- + ensemblIdSuffix;
+ if ( noDB ) {
+ log.warn( "***** DRY RUN - no changes will be saved (you may still see relevant logging messages) *****" );
}
- return shortName;
- }
-
- /**
- * For gene symbols.
- */
- private Map getExistingGeneMap( ArrayDesign arrayDesign ) {
-
- Map existingElements = new HashMap<>();
-
- if ( arrayDesign.getCompositeSequences().isEmpty() )
- return existingElements;
- AbstractCLI.log.info( "Loading genes for existing platform ..." );
- Map> geneMap = compositeSequenceService
- .getGenes( arrayDesign.getCompositeSequences() );
-
- AbstractCLI.log
- .info( "Platform has genes already for " + geneMap.size() + "/" + arrayDesign.getCompositeSequences()
- .size() + " elements." );
-
- for ( CompositeSequence cs : geneMap.keySet() ) {
- Collection genes = geneMap.get( cs );
-
- /*
- * Two genes with the same symbol, but might be a mistake from an earlier run.
- */
- Gene g = null;
- if ( genes.size() > 1 ) {
- AbstractCLI.log.warn( "More than one gene for: " + cs + ": " + StringUtils.join( genes, ";" ) );
- for ( Gene cg : genes ) {
- if ( cg.getOfficialSymbol().equals( cs.getName() ) ) {
- g = cg;
- }
- }
- } else {
- g = genes.iterator().next();
- }
- existingElements.put( g, cs );
- }
-
- return existingElements;
}
- private Map getExistingProbeNameMap( ArrayDesign arrayDesign ) {
+
+ private Map nameMap( ArrayDesign arrayDesign ) {
Map existingElements = new HashMap<>();
diff --git a/gemma-cli/src/main/java/ubic/gemma/core/apps/GeoGrabberCli.java b/gemma-cli/src/main/java/ubic/gemma/core/apps/GeoGrabberCli.java
index aaa9999f92..8b76bf2399 100755
--- a/gemma-cli/src/main/java/ubic/gemma/core/apps/GeoGrabberCli.java
+++ b/gemma-cli/src/main/java/ubic/gemma/core/apps/GeoGrabberCli.java
@@ -22,8 +22,8 @@
import ubic.gemma.core.apps.GemmaCLI.CommandGroup;
import ubic.gemma.core.loader.expression.geo.model.GeoRecord;
import ubic.gemma.core.loader.expression.geo.service.GeoBrowser;
+import ubic.gemma.core.util.AbstractAuthenticatedCLI;
import ubic.gemma.core.util.AbstractCLI;
-import ubic.gemma.core.util.AbstractCLIContextCLI;
import ubic.gemma.model.expression.arrayDesign.ArrayDesign;
import ubic.gemma.model.genome.Taxon;
import ubic.gemma.persistence.service.expression.arrayDesign.ArrayDesignService;
@@ -45,10 +45,10 @@
*
* @author paul
*/
-public class GeoGrabberCli extends AbstractCLIContextCLI {
+public class GeoGrabberCli extends AbstractAuthenticatedCLI {
private static final int NCBI_CHUNK_SIZE = 100;
- private static final int MAX_RETRIES = 3; // on failures
+ private static final int MAX_RETRIES = 5; // on failures
private static final int MAX_EMPTY_CHUNKS_IN_A_ROW = 20; // stop condition when we stop seeing useful records
private Date dateLimit;
private String gseLimit;
@@ -98,7 +98,7 @@ protected void buildOptions( Options options ) {
}
@Override
- protected void processOptions( CommandLine commandLine ) throws Exception {
+ protected void processOptions( CommandLine commandLine ) {
if ( !commandLine.hasOption( "output" ) ) {
throw new IllegalArgumentException( "You must provide an output file name" );
@@ -253,7 +253,7 @@ protected void doWork() throws Exception {
retries++;
if ( retries <= MAX_RETRIES ) {
log.warn( "Failure while fetching records, retrying " + e.getMessage() );
- Thread.sleep( 500 );
+ Thread.sleep( 500 * retries );
continue;
}
throw new IOException( "Too many failures: " + e.getMessage() );
diff --git a/gemma-cli/src/main/java/ubic/gemma/core/apps/IndexGemmaCLI.java b/gemma-cli/src/main/java/ubic/gemma/core/apps/IndexGemmaCLI.java
index fb921d1e0d..94a9e68c4c 100644
--- a/gemma-cli/src/main/java/ubic/gemma/core/apps/IndexGemmaCLI.java
+++ b/gemma-cli/src/main/java/ubic/gemma/core/apps/IndexGemmaCLI.java
@@ -4,7 +4,6 @@
import org.apache.commons.cli.Options;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
-import org.springframework.stereotype.Component;
import ubic.gemma.core.search.IndexerService;
import ubic.gemma.core.util.AbstractCLI;
import ubic.gemma.model.analysis.expression.ExpressionExperimentSet;
@@ -22,9 +21,10 @@
import java.util.Set;
import java.util.stream.Collectors;
-@Component
public class IndexGemmaCLI extends AbstractCLI {
+ private static final String THREADS_OPTION = "threads";
+
/**
* A list of all searchable entities this CLI supports.
*/
@@ -53,6 +53,7 @@ private static class IndexableEntity {
private File searchDir;
private final Set> classesToIndex = new HashSet<>();
+ private int numThreads;
@Override
public String getCommandName() {
@@ -78,7 +79,7 @@ protected void buildOptions( Options options ) {
}
@Override
- protected void processOptions( CommandLine commandLine ) throws Exception {
+ protected void processOptions( CommandLine commandLine ) {
for ( IndexableEntity ie : indexableEntities ) {
if ( commandLine.hasOption( ie.option ) ) {
classesToIndex.add( ie.clazz );
@@ -90,12 +91,12 @@ protected void processOptions( CommandLine commandLine ) throws Exception {
protected void doWork() throws Exception {
if ( classesToIndex.isEmpty() ) {
log.info( String.format( "All entities will be indexed under %s.", searchDir.getAbsolutePath() ) );
- indexerService.index( numThreads );
+ indexerService.index( getNumThreads() );
} else {
log.info( String.format( "The following entities will be indexed under %s:\n\t%s",
searchDir.getAbsolutePath(),
classesToIndex.stream().map( Class::getName ).collect( Collectors.joining( "\n\t" ) ) ) );
- indexerService.index( classesToIndex, numThreads );
+ indexerService.index( classesToIndex, getNumThreads() );
}
}
}
diff --git a/gemma-cli/src/main/java/ubic/gemma/core/apps/LinkAnalysisCli.java b/gemma-cli/src/main/java/ubic/gemma/core/apps/LinkAnalysisCli.java
index a183b58b9c..7f965191b3 100644
--- a/gemma-cli/src/main/java/ubic/gemma/core/apps/LinkAnalysisCli.java
+++ b/gemma-cli/src/main/java/ubic/gemma/core/apps/LinkAnalysisCli.java
@@ -21,9 +21,9 @@
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
import org.apache.commons.lang3.time.StopWatch;
import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Component;
import ubic.basecode.dataStructure.matrix.DoubleMatrix;
import ubic.basecode.io.ByteArrayConverter;
import ubic.basecode.util.FileTools;
@@ -64,7 +64,6 @@
* @author paul (refactoring)
* @author vaneet
*/
-@Component
public class LinkAnalysisCli extends ExpressionExperimentManipulatingCLI {
@Autowired
@@ -292,6 +291,8 @@ protected void buildOptions( Options options ) {
.build();
options.addOption( chooseCutOption );
+ options.addOption( Option.builder( "probeDegreeLim" ).hasArg().type( Integer.class ).build() );
+
// finer-grained control is possible, of course.
Option skipQC = Option.builder( "noqc" )
.desc( "Skip strict QC for outliers, batch effects and correlation distribution" )
@@ -304,12 +305,11 @@ protected void buildOptions( Options options ) {
options.addOption( deleteOption );
this.addForceOption( options );
- this.addAutoOption( options );
+ this.addAutoOption( options, LinkAnalysisEvent.class );
}
@Override
- protected void processOptions( CommandLine commandLine ) {
- this.autoSeekEventType = LinkAnalysisEvent.class;
+ protected void processOptions( CommandLine commandLine ) throws ParseException {
super.processOptions( commandLine );
if ( commandLine.hasOption( "delete" ) ) {
@@ -427,7 +427,7 @@ protected void processOptions( CommandLine commandLine ) {
}
if ( commandLine.hasOption( "probeDegreeLim" ) ) {
- this.linkAnalysisConfig.setProbeDegreeThreshold( this.getIntegerOptionValue( commandLine, "probeDegreeLim" ) );
+ this.linkAnalysisConfig.setProbeDegreeThreshold( ( Integer ) commandLine.getParsedOptionValue( "probeDegreeLim" ) );
}
}
diff --git a/gemma-cli/src/main/java/ubic/gemma/core/apps/LoadExpressionDataCli.java b/gemma-cli/src/main/java/ubic/gemma/core/apps/LoadExpressionDataCli.java
index 771fd50945..1a52693022 100644
--- a/gemma-cli/src/main/java/ubic/gemma/core/apps/LoadExpressionDataCli.java
+++ b/gemma-cli/src/main/java/ubic/gemma/core/apps/LoadExpressionDataCli.java
@@ -29,9 +29,8 @@
import ubic.gemma.core.apps.GemmaCLI.CommandGroup;
import ubic.gemma.core.loader.expression.geo.GeoDomainObjectGenerator;
import ubic.gemma.core.loader.expression.geo.service.GeoService;
+import ubic.gemma.core.util.AbstractAuthenticatedCLI;
import ubic.gemma.core.util.AbstractCLI;
-import ubic.gemma.core.util.AbstractCLIContextCLI;
-import ubic.gemma.model.common.Describable;
import ubic.gemma.model.common.description.DatabaseEntry;
import ubic.gemma.model.common.description.ExternalDatabase;
import ubic.gemma.model.expression.arrayDesign.ArrayDesign;
@@ -51,7 +50,7 @@
*
* @author pavlidis
*/
-public class LoadExpressionDataCli extends AbstractCLIContextCLI {
+public class LoadExpressionDataCli extends AbstractAuthenticatedCLI {
// Command line Options
private String accessionFile = null;
@@ -228,6 +227,7 @@ protected void processOptions( CommandLine commandLine ) {
private void processAccession( GeoService geoService, String accession ) {
try {
+ log.info(" ***** Starting processing of " + accession + " *****");
if ( updateOnly ) {
geoService.updateFromGEO( accession );
addSuccessObject( accession, "Updated" );
diff --git a/gemma-cli/src/main/java/ubic/gemma/core/apps/LoadSimpleExpressionDataCli.java b/gemma-cli/src/main/java/ubic/gemma/core/apps/LoadSimpleExpressionDataCli.java
index eb6e6e6451..153572ded0 100644
--- a/gemma-cli/src/main/java/ubic/gemma/core/apps/LoadSimpleExpressionDataCli.java
+++ b/gemma-cli/src/main/java/ubic/gemma/core/apps/LoadSimpleExpressionDataCli.java
@@ -26,8 +26,8 @@
import ubic.gemma.core.apps.GemmaCLI.CommandGroup;
import ubic.gemma.core.loader.expression.simple.SimpleExpressionDataLoaderService;
import ubic.gemma.core.loader.expression.simple.model.SimpleExpressionExperimentMetaData;
+import ubic.gemma.core.util.AbstractAuthenticatedCLI;
import ubic.gemma.core.util.AbstractCLI;
-import ubic.gemma.core.util.AbstractCLIContextCLI;
import ubic.gemma.model.common.quantitationtype.GeneralType;
import ubic.gemma.model.common.quantitationtype.ScaleType;
import ubic.gemma.model.common.quantitationtype.StandardQuantitationType;
@@ -48,7 +48,7 @@
*
* @author xiangwan
*/
-public class LoadSimpleExpressionDataCli extends AbstractCLIContextCLI {
+public class LoadSimpleExpressionDataCli extends AbstractAuthenticatedCLI {
private final static String SPLIT_CHAR = "\t";
private final static int NAME_I = 0;
private final static int SHORT_NAME_I = LoadSimpleExpressionDataCli.NAME_I + 1;
diff --git a/gemma-cli/src/main/java/ubic/gemma/core/apps/MakeExperimentPrivateCli.java b/gemma-cli/src/main/java/ubic/gemma/core/apps/MakeExperimentPrivateCli.java
index 8b1de43530..aa68546aa7 100644
--- a/gemma-cli/src/main/java/ubic/gemma/core/apps/MakeExperimentPrivateCli.java
+++ b/gemma-cli/src/main/java/ubic/gemma/core/apps/MakeExperimentPrivateCli.java
@@ -2,11 +2,9 @@
import gemma.gsec.SecurityService;
import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Component;
import ubic.gemma.model.common.auditAndSecurity.eventType.MakePrivateEvent;
import ubic.gemma.model.expression.experiment.BioAssaySet;
-@Component
public class MakeExperimentPrivateCli extends ExpressionExperimentManipulatingCLI {
@Autowired
diff --git a/gemma-cli/src/main/java/ubic/gemma/core/apps/MultifunctionalityCli.java b/gemma-cli/src/main/java/ubic/gemma/core/apps/MultifunctionalityCli.java
index e28c85d3f0..3b53c46cda 100644
--- a/gemma-cli/src/main/java/ubic/gemma/core/apps/MultifunctionalityCli.java
+++ b/gemma-cli/src/main/java/ubic/gemma/core/apps/MultifunctionalityCli.java
@@ -18,15 +18,15 @@
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import ubic.gemma.core.analysis.service.GeneMultifunctionalityPopulationService;
+import ubic.gemma.core.util.AbstractAuthenticatedCLI;
import ubic.gemma.core.util.AbstractCLI;
-import ubic.gemma.core.util.AbstractCLIContextCLI;
import ubic.gemma.model.genome.Taxon;
import ubic.gemma.persistence.service.genome.taxon.TaxonService;
/**
* @author paul
*/
-public class MultifunctionalityCli extends AbstractCLIContextCLI {
+public class MultifunctionalityCli extends AbstractAuthenticatedCLI {
private Taxon taxon;
diff --git a/gemma-cli/src/main/java/ubic/gemma/core/apps/NCBIGene2GOAssociationLoaderCLI.java b/gemma-cli/src/main/java/ubic/gemma/core/apps/NCBIGene2GOAssociationLoaderCLI.java
index 553c09305f..62fa71004f 100755
--- a/gemma-cli/src/main/java/ubic/gemma/core/apps/NCBIGene2GOAssociationLoaderCLI.java
+++ b/gemma-cli/src/main/java/ubic/gemma/core/apps/NCBIGene2GOAssociationLoaderCLI.java
@@ -22,13 +22,12 @@
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Component;
import ubic.gemma.core.apps.GemmaCLI.CommandGroup;
import ubic.gemma.core.loader.association.NCBIGene2GOAssociationLoader;
import ubic.gemma.core.loader.association.NCBIGene2GOAssociationParser;
import ubic.gemma.core.loader.util.fetcher.HttpFetcher;
+import ubic.gemma.core.util.AbstractAuthenticatedCLI;
import ubic.gemma.core.util.AbstractCLI;
-import ubic.gemma.core.util.AbstractCLIContextCLI;
import ubic.gemma.model.common.description.ExternalDatabase;
import ubic.gemma.model.common.description.ExternalDatabases;
import ubic.gemma.model.common.description.LocalFile;
@@ -49,8 +48,7 @@
*
* @author pavlidis
*/
-@Component
-public class NCBIGene2GOAssociationLoaderCLI extends AbstractCLIContextCLI {
+public class NCBIGene2GOAssociationLoaderCLI extends AbstractAuthenticatedCLI {
private static final String GENE2GO_FILE = "gene2go.gz";
diff --git a/gemma-cli/src/main/java/ubic/gemma/core/apps/NcbiGeneLoaderCLI.java b/gemma-cli/src/main/java/ubic/gemma/core/apps/NcbiGeneLoaderCLI.java
index 3e16fff744..aac0a0274f 100755
--- a/gemma-cli/src/main/java/ubic/gemma/core/apps/NcbiGeneLoaderCLI.java
+++ b/gemma-cli/src/main/java/ubic/gemma/core/apps/NcbiGeneLoaderCLI.java
@@ -23,11 +23,10 @@
import org.apache.commons.cli.Options;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Component;
import ubic.gemma.core.apps.GemmaCLI.CommandGroup;
import ubic.gemma.core.loader.genome.gene.ncbi.NcbiGeneLoader;
+import ubic.gemma.core.util.AbstractAuthenticatedCLI;
import ubic.gemma.core.util.AbstractCLI;
-import ubic.gemma.core.util.AbstractCLIContextCLI;
import ubic.gemma.model.common.description.ExternalDatabase;
import ubic.gemma.model.common.description.ExternalDatabases;
import ubic.gemma.model.genome.Taxon;
@@ -43,8 +42,7 @@
*
* @author joseph
*/
-@Component
-public class NcbiGeneLoaderCLI extends AbstractCLIContextCLI {
+public class NcbiGeneLoaderCLI extends AbstractAuthenticatedCLI {
private static final String GENE_INFO_FILE = "gene_info.gz";
private static final String GENE2ACCESSION_FILE = "gene2accession.gz";
private static final String GENE_HISTORY_FILE = "gene_history.gz";
diff --git a/gemma-cli/src/main/java/ubic/gemma/core/apps/ProcessedDataComputeCLI.java b/gemma-cli/src/main/java/ubic/gemma/core/apps/ProcessedDataComputeCLI.java
index 5a4705b50d..5d918c2efc 100644
--- a/gemma-cli/src/main/java/ubic/gemma/core/apps/ProcessedDataComputeCLI.java
+++ b/gemma-cli/src/main/java/ubic/gemma/core/apps/ProcessedDataComputeCLI.java
@@ -20,8 +20,8 @@
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Component;
import ubic.gemma.core.analysis.preprocess.PreprocessorService;
import ubic.gemma.core.analysis.preprocess.QuantitationMismatchPreprocessingException;
import ubic.gemma.core.datastructure.matrix.SuspiciousValuesForQuantitationException;
@@ -41,7 +41,6 @@
* @author xwan, paul
* @see ProcessedExpressionDataVectorServiceImpl
*/
-@Component
public class ProcessedDataComputeCLI extends ExpressionExperimentManipulatingCLI {
private static final String
@@ -79,7 +78,7 @@ protected void buildOptions( Options options ) {
}
@Override
- protected void processOptions( CommandLine commandLine ) {
+ protected void processOptions( CommandLine commandLine ) throws ParseException {
super.processOptions( commandLine );
this.updateDiagnostics = commandLine.hasOption( UPDATE_DIAGNOSTICS_OPTION );
this.updateRanks = commandLine.hasOption( UPDATE_RANKS_OPTION );
diff --git a/gemma-cli/src/main/java/ubic/gemma/core/apps/PubMedLoaderCli.java b/gemma-cli/src/main/java/ubic/gemma/core/apps/PubMedLoaderCli.java
index 0b67b1f6fc..a3db2363d4 100644
--- a/gemma-cli/src/main/java/ubic/gemma/core/apps/PubMedLoaderCli.java
+++ b/gemma-cli/src/main/java/ubic/gemma/core/apps/PubMedLoaderCli.java
@@ -23,7 +23,7 @@
import org.apache.commons.cli.Options;
import ubic.gemma.core.apps.GemmaCLI.CommandGroup;
import ubic.gemma.core.loader.entrez.pubmed.PubMedService;
-import ubic.gemma.core.util.AbstractCLIContextCLI;
+import ubic.gemma.core.util.AbstractAuthenticatedCLI;
import java.io.File;
@@ -32,7 +32,7 @@
*
* @author pavlidis
*/
-public class PubMedLoaderCli extends AbstractCLIContextCLI {
+public class PubMedLoaderCli extends AbstractAuthenticatedCLI {
private String directory;
diff --git a/gemma-cli/src/main/java/ubic/gemma/core/apps/RNASeqBatchInfoCli.java b/gemma-cli/src/main/java/ubic/gemma/core/apps/RNASeqBatchInfoCli.java
index e391c056a5..fa4ad23b41 100644
--- a/gemma-cli/src/main/java/ubic/gemma/core/apps/RNASeqBatchInfoCli.java
+++ b/gemma-cli/src/main/java/ubic/gemma/core/apps/RNASeqBatchInfoCli.java
@@ -16,9 +16,8 @@
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Component;
-import ubic.gemma.core.analysis.preprocess.batcheffects.BatchInfoPopulationException;
import ubic.gemma.core.analysis.preprocess.batcheffects.BatchInfoPopulationService;
import ubic.gemma.model.expression.experiment.BioAssaySet;
import ubic.gemma.model.expression.experiment.ExpressionExperiment;
@@ -30,7 +29,6 @@
* @author tesar
* @deprecated this should not be necessary and the regular batch population tool can be used instead.
*/
-@Component
public class RNASeqBatchInfoCli extends ExpressionExperimentManipulatingCLI {
@Autowired
@@ -50,7 +48,7 @@ protected void buildOptions( Options options ) {
}
@Override
- protected void processOptions( CommandLine commandLine ) {
+ protected void processOptions( CommandLine commandLine ) throws ParseException {
super.processOptions( commandLine );
}
diff --git a/gemma-cli/src/main/java/ubic/gemma/core/apps/RNASeqDataAddCli.java b/gemma-cli/src/main/java/ubic/gemma/core/apps/RNASeqDataAddCli.java
index ac8104a9e0..74c0d17699 100644
--- a/gemma-cli/src/main/java/ubic/gemma/core/apps/RNASeqDataAddCli.java
+++ b/gemma-cli/src/main/java/ubic/gemma/core/apps/RNASeqDataAddCli.java
@@ -17,9 +17,9 @@
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
import org.apache.commons.io.FileUtils;
import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Component;
import ubic.basecode.dataStructure.matrix.DoubleMatrix;
import ubic.basecode.io.reader.DoubleMatrixReader;
import ubic.gemma.core.analysis.service.ExpressionDataFileService;
@@ -45,7 +45,6 @@
*
* @author Paul
*/
-@Component
public class RNASeqDataAddCli extends ExpressionExperimentManipulatingCLI {
private static final String ALLOW_MISSING = "allowMissing";
@@ -93,7 +92,7 @@ protected void buildOptions( Options options ) {
}
@Override
- protected void processOptions( CommandLine commandLine ) {
+ protected void processOptions( CommandLine commandLine ) throws ParseException {
super.processOptions( commandLine );
if ( commandLine.hasOption( "log2cpm" ) ) {
diff --git a/gemma-cli/src/main/java/ubic/gemma/core/apps/ReplaceDataCli.java b/gemma-cli/src/main/java/ubic/gemma/core/apps/ReplaceDataCli.java
index e325a708f2..346278bc8b 100644
--- a/gemma-cli/src/main/java/ubic/gemma/core/apps/ReplaceDataCli.java
+++ b/gemma-cli/src/main/java/ubic/gemma/core/apps/ReplaceDataCli.java
@@ -17,6 +17,7 @@
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
import ubic.basecode.dataStructure.matrix.DoubleMatrix;
import ubic.basecode.io.reader.DoubleMatrixReader;
import ubic.gemma.core.apps.GemmaCLI.CommandGroup;
@@ -55,7 +56,7 @@ protected void buildOptions( Options options ) {
}
@Override
- protected void processOptions( CommandLine commandLine ) {
+ protected void processOptions( CommandLine commandLine ) throws ParseException {
super.processOptions( commandLine );
this.file = commandLine.getOptionValue( "file" );
diff --git a/gemma-cli/src/main/java/ubic/gemma/core/apps/SplitExperimentCli.java b/gemma-cli/src/main/java/ubic/gemma/core/apps/SplitExperimentCli.java
index 3cf06ac2b0..e08c318ad1 100644
--- a/gemma-cli/src/main/java/ubic/gemma/core/apps/SplitExperimentCli.java
+++ b/gemma-cli/src/main/java/ubic/gemma/core/apps/SplitExperimentCli.java
@@ -25,6 +25,7 @@
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
import ubic.gemma.core.analysis.preprocess.SplitExperimentService;
import ubic.gemma.core.analysis.preprocess.batcheffects.BatchInfoPopulationServiceImpl;
import ubic.gemma.core.apps.GemmaCLI.CommandGroup;
@@ -107,7 +108,7 @@ protected void doWork() throws Exception {
}
@Override
- protected void processOptions( CommandLine commandLine ) {
+ protected void processOptions( CommandLine commandLine ) throws ParseException {
super.processOptions( commandLine );
if ( !commandLine.hasOption( FACTOR_OPTION ) ) {
throw new IllegalArgumentException( "Please specify the factor" );
diff --git a/gemma-cli/src/main/java/ubic/gemma/core/apps/TaxonLoaderCli.java b/gemma-cli/src/main/java/ubic/gemma/core/apps/TaxonLoaderCli.java
index 3b91248d0c..6710645dfc 100644
--- a/gemma-cli/src/main/java/ubic/gemma/core/apps/TaxonLoaderCli.java
+++ b/gemma-cli/src/main/java/ubic/gemma/core/apps/TaxonLoaderCli.java
@@ -23,7 +23,7 @@
import ubic.gemma.core.apps.GemmaCLI.CommandGroup;
import ubic.gemma.core.loader.genome.taxon.TaxonFetcher;
import ubic.gemma.core.loader.genome.taxon.TaxonLoader;
-import ubic.gemma.core.util.AbstractCLIContextCLI;
+import ubic.gemma.core.util.AbstractAuthenticatedCLI;
import ubic.gemma.model.common.description.LocalFile;
import ubic.gemma.persistence.persister.PersisterHelper;
@@ -32,7 +32,7 @@
/**
* @author pavlidis
*/
-public class TaxonLoaderCli extends AbstractCLIContextCLI {
+public class TaxonLoaderCli extends AbstractAuthenticatedCLI {
@Override
public String getCommandName() {
@@ -55,7 +55,7 @@ protected void buildOptions( Options options ) {
}
@Override
- protected void processOptions( CommandLine commandLine ) throws Exception {
+ protected void processOptions( CommandLine commandLine ) {
}
diff --git a/gemma-cli/src/main/java/ubic/gemma/core/apps/UpdatePubMedCli.java b/gemma-cli/src/main/java/ubic/gemma/core/apps/UpdatePubMedCli.java
index f5998acf67..176fe7630a 100644
--- a/gemma-cli/src/main/java/ubic/gemma/core/apps/UpdatePubMedCli.java
+++ b/gemma-cli/src/main/java/ubic/gemma/core/apps/UpdatePubMedCli.java
@@ -17,16 +17,17 @@
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.Options;
import org.apache.commons.lang3.StringUtils;
+import org.springframework.beans.factory.annotation.Autowired;
import ubic.gemma.core.annotation.reference.BibliographicReferenceService;
import ubic.gemma.core.loader.entrez.pubmed.PubMedSearch;
import ubic.gemma.core.loader.expression.geo.model.GeoRecord;
import ubic.gemma.core.loader.expression.geo.service.GeoBrowser;
-import ubic.gemma.core.util.AbstractCLIContextCLI;
+import ubic.gemma.core.util.AbstractAuthenticatedCLI;
import ubic.gemma.model.common.description.BibliographicReference;
import ubic.gemma.model.common.description.DatabaseEntry;
import ubic.gemma.model.common.description.ExternalDatabase;
import ubic.gemma.model.expression.experiment.ExpressionExperiment;
-import ubic.gemma.persistence.persister.Persister;
+import ubic.gemma.persistence.persister.PersisterHelper;
import ubic.gemma.persistence.service.expression.experiment.ExpressionExperimentService;
import java.io.IOException;
@@ -40,7 +41,11 @@
* Fetch their GEO records and check for pubmed IDs
* Add the publications where we find them.
*/
-public class UpdatePubMedCli extends AbstractCLIContextCLI {
+public class UpdatePubMedCli extends AbstractAuthenticatedCLI {
+
+ @Autowired
+ private PersisterHelper persisterHelper;
+
@Override
public String getCommandName() {
return "findDatasetPubs";
@@ -124,7 +129,7 @@ protected void doWork() throws Exception {
}
@Override
- protected void processOptions( CommandLine commandLine ) throws Exception {
+ protected void processOptions( CommandLine commandLine ) {
}
@@ -142,7 +147,6 @@ public GemmaCLI.CommandGroup getCommandGroup() {
private BibliographicReference getBibliographicReference( String pubmedId ) {
// check if it already in the system
BibliographicReferenceService bibliographicReferenceService = this.getBean( BibliographicReferenceService.class );
- Persister persisterHelper = this.getPersisterHelper();
BibliographicReference publication = bibliographicReferenceService.findByExternalId( pubmedId );
if ( publication == null ) {
PubMedSearch pms = new PubMedSearch();
diff --git a/gemma-cli/src/main/java/ubic/gemma/core/loader/association/phenotype/CtdDatabaseImporterCli.java b/gemma-cli/src/main/java/ubic/gemma/core/loader/association/phenotype/CtdDatabaseImporterCli.java
index 0b23b66989..28c017f5fe 100644
--- a/gemma-cli/src/main/java/ubic/gemma/core/loader/association/phenotype/CtdDatabaseImporterCli.java
+++ b/gemma-cli/src/main/java/ubic/gemma/core/loader/association/phenotype/CtdDatabaseImporterCli.java
@@ -36,11 +36,6 @@ public class CtdDatabaseImporterCli extends ExternalDatabaseEvidenceImporterAbst
// location of the ctd file
private String ctdFile = "";
- @SuppressWarnings({ "unused", "WeakerAccess" }) // Possible external use
- public CtdDatabaseImporterCli() throws Exception {
- super();
- }
-
@Override
public String getCommandName() {
return "ctdDownload";
diff --git a/gemma-cli/src/main/java/ubic/gemma/core/loader/association/phenotype/DeleteEvidenceCLI.java b/gemma-cli/src/main/java/ubic/gemma/core/loader/association/phenotype/DeleteEvidenceCLI.java
index 7a50984b04..8d8e89d91f 100644
--- a/gemma-cli/src/main/java/ubic/gemma/core/loader/association/phenotype/DeleteEvidenceCLI.java
+++ b/gemma-cli/src/main/java/ubic/gemma/core/loader/association/phenotype/DeleteEvidenceCLI.java
@@ -19,8 +19,8 @@
import org.apache.commons.cli.Options;
import ubic.gemma.core.apps.GemmaCLI.CommandGroup;
import ubic.gemma.core.association.phenotype.PhenotypeAssociationManagerService;
+import ubic.gemma.core.util.AbstractAuthenticatedCLI;
import ubic.gemma.core.util.AbstractCLI;
-import ubic.gemma.core.util.AbstractCLIContextCLI;
import ubic.gemma.model.association.phenotype.PhenotypeAssociation;
import ubic.gemma.model.genome.gene.phenotype.valueObject.EvidenceValueObject;
@@ -31,7 +31,7 @@
*
* @author nicolas
*/
-public class DeleteEvidenceCLI extends AbstractCLIContextCLI {
+public class DeleteEvidenceCLI extends AbstractAuthenticatedCLI {
private String externalDatabaseName = "";
private PhenotypeAssociationManagerService phenotypeAssociationService = null;
diff --git a/gemma-cli/src/main/java/ubic/gemma/core/loader/association/phenotype/EvidenceImporterAbstractCLI.java b/gemma-cli/src/main/java/ubic/gemma/core/loader/association/phenotype/EvidenceImporterAbstractCLI.java
index ba75c82e58..ee135291e5 100755
--- a/gemma-cli/src/main/java/ubic/gemma/core/loader/association/phenotype/EvidenceImporterAbstractCLI.java
+++ b/gemma-cli/src/main/java/ubic/gemma/core/loader/association/phenotype/EvidenceImporterAbstractCLI.java
@@ -23,8 +23,8 @@
import ubic.gemma.core.association.phenotype.PhenotypeAssociationManagerService;
import ubic.gemma.core.genome.gene.service.GeneService;
import ubic.gemma.core.ontology.providers.MondoOntologyService;
+import ubic.gemma.core.util.AbstractAuthenticatedCLI;
import ubic.gemma.core.util.AbstractCLI;
-import ubic.gemma.core.util.AbstractCLIContextCLI;
import ubic.gemma.model.common.description.ExternalDatabaseValueObject;
import ubic.gemma.model.genome.gene.phenotype.valueObject.EvidenceSourceValueObject;
import ubic.gemma.persistence.service.genome.taxon.TaxonService;
@@ -37,7 +37,7 @@
import java.util.*;
@Deprecated
-public abstract class EvidenceImporterAbstractCLI extends AbstractCLIContextCLI {
+public abstract class EvidenceImporterAbstractCLI extends AbstractAuthenticatedCLI {
static final String WRITE_FOLDER = Settings.getString( "gemma.appdata.home" ) + File.separator + "EvidenceImporterNeurocarta";
diff --git a/gemma-cli/src/main/java/ubic/gemma/core/loader/association/phenotype/EvidenceImporterCLI.java b/gemma-cli/src/main/java/ubic/gemma/core/loader/association/phenotype/EvidenceImporterCLI.java
index 480ff1b63a..7b49d5303d 100755
--- a/gemma-cli/src/main/java/ubic/gemma/core/loader/association/phenotype/EvidenceImporterCLI.java
+++ b/gemma-cli/src/main/java/ubic/gemma/core/loader/association/phenotype/EvidenceImporterCLI.java
@@ -21,6 +21,7 @@
import ubic.gemma.core.apps.GemmaCLI.CommandGroup;
import ubic.gemma.core.association.phenotype.EntityNotFoundException;
import ubic.gemma.core.util.AbstractCLI;
+import ubic.gemma.model.common.description.CharacteristicValueObject;
import ubic.gemma.model.common.description.CitationValueObject;
import ubic.gemma.model.genome.Gene;
import ubic.gemma.model.genome.gene.phenotype.valueObject.*;
@@ -38,7 +39,7 @@
*
* @author nicolas
*/
-@SuppressWarnings({"unused", "WeakerAccess"}) // Possible external use
+@SuppressWarnings({ "unused", "WeakerAccess" }) // Possible external use
public class EvidenceImporterCLI extends EvidenceImporterAbstractCLI {
@Override
@@ -254,7 +255,7 @@ private void createImportLog( EvidenceValueObject evidenceValueObject ) {
}
private Set experimentTags2Ontology( Set values, String category,
- String categoryUri, OntologyService ontologyUsed ) throws OntologySearchException {
+ String categoryUri, OntologyService ontologyUsed ) throws OntologySearchException {
Set experimentTags = new HashSet<>();
@@ -271,7 +272,7 @@ private Set experimentTags2Ontology( Set valu
}
}
- CharacteristicValueObject c = new CharacteristicValueObject( -1L, term, category, valueUri, categoryUri );
+ CharacteristicValueObject c = new CharacteristicValueObject( term, valueUri, category, categoryUri );
experimentTags.add( c );
}
return experimentTags;
@@ -472,7 +473,7 @@ private void populateCommonFields( EvidenceValueObject evidence, String[] tok
*/
@SuppressWarnings("StatementWithEmptyBody") // Better readability
private void setScoreDependingOnExternalSource( String externalDatabaseName, EvidenceValueObject evidence,
- String evidenceTaxon ) {
+ String evidenceTaxon ) {
// OMIM got special character in description to find score
if ( externalDatabaseName.equalsIgnoreCase( "OMIM" ) ) {
@@ -552,7 +553,7 @@ private SortedSet toValuesUri( Set phenotypes
String valueUri = this.phenotype2Ontology( phenotype );
if ( valueUri != null ) {
- CharacteristicValueObject c = new CharacteristicValueObject( -1L, valueUri );
+ CharacteristicValueObject c = new CharacteristicValueObject( "", valueUri );
characteristicPhenotypes.add( c );
}
}
diff --git a/gemma-cli/src/main/java/ubic/gemma/core/loader/association/phenotype/ExternalDatabaseEvidenceImporterAbstractCLI.java b/gemma-cli/src/main/java/ubic/gemma/core/loader/association/phenotype/ExternalDatabaseEvidenceImporterAbstractCLI.java
index 07a0ab78c0..b9d1b52075 100644
--- a/gemma-cli/src/main/java/ubic/gemma/core/loader/association/phenotype/ExternalDatabaseEvidenceImporterAbstractCLI.java
+++ b/gemma-cli/src/main/java/ubic/gemma/core/loader/association/phenotype/ExternalDatabaseEvidenceImporterAbstractCLI.java
@@ -14,20 +14,19 @@
*/
package ubic.gemma.core.loader.association.phenotype;
-import ubic.basecode.ontology.providers.DiseaseOntologyService;
import ubic.basecode.ontology.providers.HumanPhenotypeOntologyService;
import ubic.basecode.ontology.providers.MedicOntologyService;
import ubic.gemma.core.apps.GemmaCLI.CommandGroup;
import ubic.gemma.core.genome.gene.service.GeneService;
import ubic.gemma.core.ontology.providers.MondoOntologyService;
-import ubic.gemma.core.util.AbstractCLIContextCLI;
+import ubic.gemma.core.util.AbstractAuthenticatedCLI;
import ubic.gemma.persistence.service.genome.taxon.TaxonService;
/**
* @author nicolas
*/
@Deprecated
-public abstract class ExternalDatabaseEvidenceImporterAbstractCLI extends AbstractCLIContextCLI {
+public abstract class ExternalDatabaseEvidenceImporterAbstractCLI extends AbstractAuthenticatedCLI {
protected String writeFolder = null;
diff --git a/gemma-cli/src/main/java/ubic/gemma/core/loader/association/phenotype/LoadEvidenceForClassifier.java b/gemma-cli/src/main/java/ubic/gemma/core/loader/association/phenotype/LoadEvidenceForClassifier.java
index c146e70fd7..f1bc663678 100644
--- a/gemma-cli/src/main/java/ubic/gemma/core/loader/association/phenotype/LoadEvidenceForClassifier.java
+++ b/gemma-cli/src/main/java/ubic/gemma/core/loader/association/phenotype/LoadEvidenceForClassifier.java
@@ -19,7 +19,7 @@
import org.springframework.core.io.ClassPathResource;
import ubic.gemma.core.annotation.reference.BibliographicReferenceService;
import ubic.gemma.core.apps.GemmaCLI.CommandGroup;
-import ubic.gemma.core.util.AbstractCLIContextCLI;
+import ubic.gemma.core.util.AbstractAuthenticatedCLI;
import ubic.gemma.model.common.description.BibliographicReference;
import ubic.gemma.model.common.description.DatabaseEntry;
import ubic.gemma.model.common.description.MedicalSubjectHeading;
@@ -34,7 +34,7 @@
*
* @author nicolas
*/
-public class LoadEvidenceForClassifier extends AbstractCLIContextCLI {
+public class LoadEvidenceForClassifier extends AbstractAuthenticatedCLI {
// a monthly dump of all evidence, takes too long to all, use files auto-generated
private final String evidenceDumpPath =
@@ -64,7 +64,7 @@ protected void buildOptions( Options options ) {
}
@Override
- protected void processOptions( CommandLine commandLine ) throws Exception {
+ protected void processOptions( CommandLine commandLine ) {
}
diff --git a/gemma-cli/src/main/java/ubic/gemma/core/loader/association/phenotype/OmimDatabaseImporterCli.java b/gemma-cli/src/main/java/ubic/gemma/core/loader/association/phenotype/OmimDatabaseImporterCli.java
index f2d72581cf..847bc4c1c1 100644
--- a/gemma-cli/src/main/java/ubic/gemma/core/loader/association/phenotype/OmimDatabaseImporterCli.java
+++ b/gemma-cli/src/main/java/ubic/gemma/core/loader/association/phenotype/OmimDatabaseImporterCli.java
@@ -70,7 +70,7 @@ protected void buildOptions( Options options ) {
}
@Override
- protected void processOptions( CommandLine commandLine ) throws Exception {
+ protected void processOptions( CommandLine commandLine ) {
}
diff --git a/gemma-cli/src/main/java/ubic/gemma/core/loader/entrez/pubmed/PubMedSearcher.java b/gemma-cli/src/main/java/ubic/gemma/core/loader/entrez/pubmed/PubMedSearcher.java
index 0a5bab4af1..c1c67df11f 100644
--- a/gemma-cli/src/main/java/ubic/gemma/core/loader/entrez/pubmed/PubMedSearcher.java
+++ b/gemma-cli/src/main/java/ubic/gemma/core/loader/entrez/pubmed/PubMedSearcher.java
@@ -20,9 +20,11 @@
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.Options;
+import org.springframework.beans.factory.annotation.Autowired;
import ubic.gemma.core.apps.GemmaCLI.CommandGroup;
-import ubic.gemma.core.util.AbstractCLIContextCLI;
+import ubic.gemma.core.util.AbstractAuthenticatedCLI;
import ubic.gemma.model.common.description.BibliographicReference;
+import ubic.gemma.persistence.persister.PersisterHelper;
import java.util.Collection;
@@ -31,8 +33,10 @@
*
* @author pavlidis
*/
-public class PubMedSearcher extends AbstractCLIContextCLI {
+public class PubMedSearcher extends AbstractAuthenticatedCLI {
private static final PubMedSearch pms = new PubMedSearch();
+ @Autowired
+ private PersisterHelper persisterHelper;
private Collection args;
private boolean persist = false;
@@ -70,7 +74,7 @@ protected void doWork() throws Exception {
System.out.println( refs.size() + " references found" );
if ( this.persist ) {
- this.getPersisterHelper().persist( refs );
+ persisterHelper.persist( refs );
}
}
diff --git a/gemma-cli/src/main/java/ubic/gemma/core/util/AbstractSpringAwareCLI.java b/gemma-cli/src/main/java/ubic/gemma/core/util/AbstractAuthenticatedCLI.java
similarity index 57%
rename from gemma-cli/src/main/java/ubic/gemma/core/util/AbstractSpringAwareCLI.java
rename to gemma-cli/src/main/java/ubic/gemma/core/util/AbstractAuthenticatedCLI.java
index 54c0439fa4..2caf470887 100644
--- a/gemma-cli/src/main/java/ubic/gemma/core/util/AbstractSpringAwareCLI.java
+++ b/gemma-cli/src/main/java/ubic/gemma/core/util/AbstractAuthenticatedCLI.java
@@ -20,41 +20,33 @@
import gemma.gsec.authentication.ManualAuthenticationService;
import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.ParseException;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
-import org.springframework.beans.factory.BeanFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.security.concurrent.DelegatingSecurityContextCallable;
import org.springframework.security.core.Authentication;
import org.springframework.security.core.context.SecurityContextHolder;
-import ubic.gemma.model.common.Auditable;
-import ubic.gemma.model.common.auditAndSecurity.AuditEvent;
-import ubic.gemma.model.common.auditAndSecurity.curation.Curatable;
-import ubic.gemma.model.common.auditAndSecurity.eventType.AuditEventType;
-import ubic.gemma.model.expression.arrayDesign.ArrayDesign;
-import ubic.gemma.model.expression.experiment.ExpressionExperiment;
-import ubic.gemma.persistence.persister.Persister;
-import ubic.gemma.persistence.service.common.auditAndSecurity.AuditEventService;
-import ubic.gemma.persistence.service.common.auditAndSecurity.AuditTrailService;
-import ubic.gemma.persistence.service.expression.experiment.ExpressionExperimentService;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.util.Collection;
-import java.util.Date;
import java.util.List;
import java.util.concurrent.Callable;
import java.util.stream.Collectors;
/**
- * Subclass this to create command line interface (CLI) tools that need a Spring context. A standard set of CLI options
- * are provided to manage authentication.
- *
+ * Subclass this to create command line interface (CLI) tools that need authentication.
+ *
+ * Credentials may be supplied via the environment using the {@code $GEMMMA_USERNAME} and {@code $GEMMA_PASSWORD}
+ * variables. A more secure {@code $GEMMA_PASSWORD_CMD} variable can be used to specify a command that produces the
+ * password. If no environment variables are supplied, they will be prompted if the standard input is attached to a
+ * console (i.e tty).
* @author pavlidis
*/
-public abstract class AbstractSpringAwareCLI extends AbstractCLI {
+public abstract class AbstractAuthenticatedCLI extends AbstractCLI {
/**
* Environment variable used to store the username (if not passed directly to the CLI).
@@ -71,24 +63,8 @@ public abstract class AbstractSpringAwareCLI extends AbstractCLI {
*/
private static final String PASSWORD_CMD_ENV = "GEMMA_PASSWORD_CMD";
- @Autowired
- private BeanFactory ctx;
@Autowired
private ManualAuthenticationService manAuthentication;
- @Autowired
- protected AuditTrailService auditTrailService;
- @Autowired
- protected AuditEventService auditEventService;
- @Autowired
- private ExpressionExperimentService ees;
- @Autowired
- private Persister persisterHelper;
-
- @SuppressWarnings({ "unused", "WeakerAccess" }) // Possible external use
- @Autowired
- public AbstractSpringAwareCLI() {
- super();
- }
@Override
public String getShortDesc() {
@@ -97,7 +73,7 @@ public String getShortDesc() {
/**
* Indicate if the command requires authentication.
- *
+ *
* Override this to return true to make authentication required.
*
* @return true if login is required, otherwise false
@@ -110,99 +86,11 @@ protected boolean requireLogin() {
* You must override this method to process any options you added.
*/
@Override
- protected void processStandardOptions( CommandLine commandLine ) {
+ protected void processStandardOptions( CommandLine commandLine ) throws ParseException {
super.processStandardOptions( commandLine );
this.authenticate();
}
- /**
- * Convenience method to obtain instance of any bean by name.
- *
- * @deprecated Use {@link Autowired} to specify your dependencies, this is just a wrapper around the current
- * {@link BeanFactory}.
- *
- * @param the bean class type
- * @param clz class
- * @param name name
- * @return bean
- */
- @SuppressWarnings("SameParameterValue") // Better for general use
- @Deprecated
- protected T getBean( String name, Class clz ) {
- assert ctx != null : "Spring context was not initialized";
- return ctx.getBean( name, clz );
- }
-
- @Deprecated
- protected T getBean( Class clz ) {
- assert ctx != null : "Spring context was not initialized";
- return ctx.getBean( clz );
- }
-
- @Deprecated
- protected Persister getPersisterHelper() {
- return persisterHelper;
- }
-
- /**
- * @param auditable auditable
- * @param eventClass can be null
- * @return boolean
- */
- protected boolean noNeedToRun( Auditable auditable, Class eventClass ) {
- boolean needToRun = true;
- Date skipIfLastRunLaterThan = this.getLimitingDate();
- List events = this.auditEventService.getEvents( auditable );
-
- boolean okToRun = true; // assume okay unless indicated otherwise
-
- // figure out if we need to run it by date; or if there is no event of the given class; "Fail" type events don't
- // count.
- for ( int j = events.size() - 1; j >= 0; j-- ) {
- AuditEvent event = events.get( j );
- if ( event == null ) {
- continue; // legacy of ordered-list which could end up with gaps; should not be needed any more
- }
- AuditEventType eventType = event.getEventType();
- if ( eventType != null && eventClass != null && eventClass.isAssignableFrom( eventType.getClass() )
- && !eventType.getClass().getSimpleName().startsWith( "Fail" ) ) {
- if ( skipIfLastRunLaterThan != null ) {
- if ( event.getDate().after( skipIfLastRunLaterThan ) ) {
- AbstractCLI.log.info( auditable + ": " + " run more recently than " + skipIfLastRunLaterThan );
- addErrorObject( auditable, "Run more recently than " + skipIfLastRunLaterThan );
- needToRun = false;
- }
- } else {
- needToRun = false; // it has been run already at some point
- }
- }
- }
-
- /*
- * Always skip if the object is curatable and troubled
- */
- if ( auditable instanceof Curatable ) {
- Curatable curatable = ( Curatable ) auditable;
- okToRun = !curatable.getCurationDetails().getTroubled(); //not ok if troubled
-
- // special case for expression experiments - check associated ADs.
- if ( okToRun && curatable instanceof ExpressionExperiment ) {
- for ( ArrayDesign ad : ees.getArrayDesignsUsed( ( ExpressionExperiment ) auditable ) ) {
- if ( ad.getCurationDetails().getTroubled() ) {
- okToRun = false; // not ok if even one parent AD is troubled, no need to check the remaining ones.
- break;
- }
- }
- }
-
- if ( !okToRun ) {
- addErrorObject( auditable, "Has an active 'trouble' flag" );
- }
- }
-
- return !needToRun || !okToRun;
- }
-
/**
* check username and password.
*/
diff --git a/gemma-cli/src/main/java/ubic/gemma/core/util/AbstractCLI.java b/gemma-cli/src/main/java/ubic/gemma/core/util/AbstractCLI.java
index f7b5899139..c503b22d2d 100644
--- a/gemma-cli/src/main/java/ubic/gemma/core/util/AbstractCLI.java
+++ b/gemma-cli/src/main/java/ubic/gemma/core/util/AbstractCLI.java
@@ -20,298 +20,345 @@
import lombok.Value;
import org.apache.commons.cli.*;
+import org.apache.commons.csv.CSVFormat;
+import org.apache.commons.csv.CSVPrinter;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.apache.commons.lang3.time.StopWatch;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.BeanFactory;
+import org.springframework.beans.factory.annotation.Autowired;
import ubic.basecode.util.DateUtil;
import ubic.gemma.model.common.auditAndSecurity.eventType.AuditEventType;
import javax.annotation.Nullable;
-import java.io.File;
-import java.io.PrintWriter;
+import java.io.*;
+import java.nio.file.Files;
import java.util.*;
import java.util.concurrent.*;
+import java.util.stream.Collectors;
/**
- * Base Command Line Interface. Provides some default functionality.
+ * Basic implementation of the {@link CLI} interface.
*
- * To use this, in your concrete subclass, implement a main method. You must implement buildOptions and processOptions
- * to handle any application-specific options (they can be no-ops).
+ * To use this, in your concrete subclass, implement {@link #buildOptions} and {@link #processOptions} to handle any
+ * application-specific options (they can be no-ops) and {@link #doWork()} to perform the actual work of the CLI.
*
- * To facilitate testing of your subclass, your main method must call a non-static 'doWork' method, that will be exposed
- * for testing. In that method call processCommandline. You should return any non-null return value from
- * processCommandLine.
- *
+ * Use {@link AbstractAuthenticatedCLI} if you need to authenticate the user.
* @author pavlidis
*/
-@SuppressWarnings({ "unused", "WeakerAccess" }) // Possible external use
public abstract class AbstractCLI implements CLI {
+ protected static final Log log = LogFactory.getLog( AbstractCLI.class );
+
/**
- * Exit code used for a successful doWork execution.
+ * Exit code used for a successful {@link #doWork} execution.
*/
- public static final int SUCCESS = 0;
+ protected static final int SUCCESS = 0;
/**
- * Exit code used for a failed doWork execution.
+ * Exit code used for a failed {@link #doWork} execution.
*/
- public static final int FAILURE = 1;
+ protected static final int FAILURE = 1;
/**
* Exit code used for a successful doWork execution that resulted in failed error objects.
*/
- public static final int FAILURE_FROM_ERROR_OBJECTS = 1;
+ protected static final int FAILURE_FROM_ERROR_OBJECTS = 1;
+ /**
+ * Exit code used when a {@link #doWork()} is aborted.
+ */
+ protected static final int ABORTED = 2;
- public static final String FOOTER = "The Gemma project, Copyright (c) 2007-2023 University of British Columbia.";
- protected static final String AUTO_OPTION_NAME = "auto";
- protected static final String THREADS_OPTION = "threads";
- protected static final Log log = LogFactory.getLog( AbstractCLI.class );
- private static final int DEFAULT_PORT = 3306;
public static final String HEADER = "Options:";
- private static final String HOST_OPTION = "H";
- private static final String PORT_OPTION = "P";
+ public static final String FOOTER = "The Gemma project, Copyright (c) 2007-2021 University of British Columbia.";
+
+ private static final String AUTO_OPTION_NAME = "auto";
+ private static final String THREADS_OPTION = "threads";
private static final String HELP_OPTION = "h";
private static final String TESTING_OPTION = "testing";
private static final String DATE_OPTION = "mdate";
+ private static final String BATCH_FORMAT_OPTION = "batchFormat";
+ private static final String BATCH_OUTPUT_FILE_OPTION = "batchOutputFile";
+
+ @Autowired
+ private BeanFactory ctx;
+
+ /**
+ * Indicate if this CLI allows positional arguments.
+ */
+ private boolean allowPositionalArguments = false;
/* support for convenience options */
- private final String DEFAULT_HOST = "localhost";
/**
* Automatically identify which entities to run the tool on. To enable call addAutoOption.
*/
- protected boolean autoSeek = false;
+ private boolean autoSeek;
/**
* The event type to look for the lack of, when using auto-seek.
*/
- protected Class autoSeekEventType = null;
+ private Class autoSeekEventType;
/**
* Date used to identify which entities to run the tool on (e.g., those which were run less recently than mDate). To
* enable call addDateOption.
*/
- protected String mDate = null;
- protected int numThreads = 1;
- protected String host = DEFAULT_HOST;
- protected int port = AbstractCLI.DEFAULT_PORT;
+ private String mDate;
+ /**
+ * Number of threads to use for batch processing.
+ */
+ private int numThreads;
+ /**
+ * Format to use to summarize batch processing.
+ */
+ private BatchFormat batchFormat;
+ /**
+ * Destination for batch processing summary.
+ */
+ @Nullable
+ private File batchOutputFile;
private ExecutorService executorService;
- // hold the results of the command execution
- // needs to be concurrently modifiable and kept in-order
- private final List errorObjects = Collections.synchronizedList( new ArrayList<>() );
- private final List successObjects = Collections.synchronizedList( new ArrayList<>() );
+ /**
+ * Hold the results of the command execution
+ * needs to be concurrently modifiable and kept in-order
+ */
+ private final List batchProcessingResults = Collections.synchronizedList( new ArrayList<>() );
/**
- * Run the command.
- *
- * Parse and process CLI arguments, invoke the command doWork implementation, and print basic statistics about time
- * usage.
+ * Convenience method to obtain instance of any bean by name.
*
- * @param args Arguments to pass to {@link #processCommandLine(Options, String[])}
- * @return Exit code intended to be used with {@link System#exit(int)} to indicate a success or failure to the
- * end-user. Any exception raised by doWork results in a value of {@link #FAILURE}, and any error set in the
- * internal error objects will result in a value of {@link #FAILURE_FROM_ERROR_OBJECTS}.
+ * @deprecated Use {@link Autowired} to specify your dependencies, this is just a wrapper around the current
+ * {@link BeanFactory}.
+ *
+ * @param the bean class type
+ * @param clz class
+ * @param name name
+ * @return bean
+ */
+ @SuppressWarnings("SameParameterValue") // Better for general use
+ @Deprecated
+ protected T getBean( String name, Class clz ) {
+ assert ctx != null : "Spring context was not initialized";
+ return ctx.getBean( name, clz );
+ }
+
+ @Deprecated
+ protected T getBean( Class clz ) {
+ assert ctx != null : "Spring context was not initialized";
+ return ctx.getBean( clz );
+ }
+
+ /**
+ * {@inheritDoc}
+ *
+ * Parse and process CLI arguments, invoke the command {@link #doWork()} implementation, and print basic statistics
+ * about time usage.
+ *
+ * Any exception raised by doWork results in a value of {@link #FAILURE}, and any error set in the internal error
+ * objects will result in a value of {@link #FAILURE_FROM_ERROR_OBJECTS}.
*/
@Override
- public int executeCommand( String[] args ) {
- StopWatch watch = new StopWatch();
- watch.start();
- if ( args == null ) {
- System.err.println( "No arguments" );
- return FAILURE;
- }
+ public int executeCommand( String... args ) {
+ Options options = new Options();
+ buildStandardOptions( options );
+ buildOptions( options );
try {
- Options options = new Options();
- buildStandardOptions( options );
- buildOptions( options );
DefaultParser parser = new DefaultParser();
- CommandLine commandLine;
- try {
- commandLine = parser.parse( options, args );
- } catch ( ParseException e ) {
- if ( e instanceof MissingOptionException ) {
- // check if -h/--help was passed alongside a required argument
- if ( ArrayUtils.contains( args, "-h" ) || ArrayUtils.contains( args, "--help" ) ) {
- printHelp( options );
- return SUCCESS;
- }
- System.err.println( "Required option(s) were not supplied: " + e.getMessage() );
- } else if ( e instanceof AlreadySelectedException ) {
- System.err.println( "The option(s) " + e.getMessage() + " were already selected" );
- } else if ( e instanceof MissingArgumentException ) {
- System.err.println( "Missing argument: " + e.getMessage() );
- } else if ( e instanceof UnrecognizedOptionException ) {
- System.err.println( "Unrecognized option: " + e.getMessage() );
- }
- printHelp( options );
- return FAILURE;
- }
+ CommandLine commandLine = parser.parse( options, args );
// check if -h/--help is provided before pursuing option processing
if ( commandLine.hasOption( HELP_OPTION ) ) {
- printHelp( options );
+ printHelp( options, new PrintWriter( System.out, true ) );
return SUCCESS;
}
if ( commandLine.hasOption( TESTING_OPTION ) ) {
- AbstractCLI.log.error( String.format( "The -testing/--testing option must be passed before the %s command.", getCommandName() ) );
- return FAILURE;
+ throw new UnrecognizedOptionException( String.format( "The -testing/--testing option must be passed before the %s command.", getCommandName() ) );
+ }
+ if ( !allowPositionalArguments && !commandLine.getArgList().isEmpty() ) {
+ throw new UnrecognizedOptionException( "The command line does not allow positional arguments." );
}
processStandardOptions( commandLine );
processOptions( commandLine );
+ } catch ( ParseException e ) {
+ if ( e instanceof MissingOptionException ) {
+ // check if -h/--help was passed alongside a required argument
+ if ( ArrayUtils.contains( args, "-h" ) || ArrayUtils.contains( args, "--help" ) ) {
+ printHelp( options, new PrintWriter( System.out, true ) );
+ return SUCCESS;
+ }
+ System.err.println( "Required option(s) were not supplied: " + e.getMessage() );
+ } else if ( e instanceof AlreadySelectedException ) {
+ System.err.println( "The option(s) " + e.getMessage() + " were already selected" );
+ } else if ( e instanceof MissingArgumentException ) {
+ System.err.println( "Missing argument: " + e.getMessage() );
+ } else if ( e instanceof UnrecognizedOptionException ) {
+ System.err.println( "Unrecognized option: " + e.getMessage() );
+ } else {
+ System.err.println( e.getMessage() );
+ }
+ printHelp( options, new PrintWriter( System.err, true ) );
+ return FAILURE;
+ }
+ StopWatch watch = StopWatch.createStarted();
+ try {
doWork();
- return errorObjects.isEmpty() ? SUCCESS : FAILURE_FROM_ERROR_OBJECTS;
+ return batchProcessingResults.stream().noneMatch( BatchProcessingResult::isError ) ? SUCCESS : FAILURE_FROM_ERROR_OBJECTS;
+ } catch ( WorkAbortedException e ) {
+ log.warn( "Operation was aborted by the current user." );
+ return ABORTED;
} catch ( Exception e ) {
log.error( String.format( "%s failed: %s", getCommandName(), ExceptionUtils.getRootCauseMessage( e ) ), e );
return FAILURE;
} finally {
// always summarize processing, even if an error is thrown
- summarizeProcessing();
+ summarizeBatchProcessing();
log.info( String.format( "Elapsed time: %d seconds.", watch.getTime( TimeUnit.SECONDS ) ) );
}
}
+ private void printHelp( Options options, PrintWriter writer ) {
+ new HelpFormatter().printHelp( writer, 150,
+ this.getCommandName() + " [options]",
+ this.getShortDesc() + "\n" + AbstractCLI.HEADER,
+ options, HelpFormatter.DEFAULT_LEFT_PAD, HelpFormatter.DEFAULT_DESC_PAD, AbstractCLI.FOOTER );
+ }
+
/**
- * You must implement the handling for this option.
+ * Add the {@code -auto} option.
+ *
+ * The auto option value can be retrieved with {@link #isAutoSeek()}.
*/
protected void addAutoOption( Options options ) {
- Option autoSeekOption = Option.builder( AUTO_OPTION_NAME )
+ options.addOption( Option.builder( AUTO_OPTION_NAME )
.desc( "Attempt to process entities that need processing based on workflow criteria." )
- .build();
-
- options.addOption( autoSeekOption );
+ .build() );
}
- protected void addDateOption( Options options ) {
- Option dateOption = Option.builder( DATE_OPTION ).hasArg().desc(
- "Constrain to run only on entities with analyses older than the given date. "
- + "For example, to run only on entities that have not been analyzed in the last 10 days, use '-10d'. "
- + "If there is no record of when the analysis was last run, it will be run." )
- .build();
-
- options.addOption( dateOption );
+ /**
+ * Add the {@code -auto} option for a specific {@link AuditEventType}.
+ *
+ * The event type can be retrieved with {@link #getAutoSeekEventType()}.
+ */
+ protected void addAutoOption( Options options, Class autoSeekEventType ) {
+ addAutoOption( options );
+ this.autoSeekEventType = autoSeekEventType;
}
/**
- * Convenience method to add a standard pair of options to intake a host name and port number. *
- *
- * @param hostRequired Whether the host name is required
- * @param portRequired Whether the port is required
+ * Add the {@code -mdate} option.
+ *
+ * The limiting date can be retrieved with {@link #getLimitingDate()}.
*/
- protected void addHostAndPortOptions( Options options, boolean hostRequired, boolean portRequired ) {
- Option hostOpt = Option.builder( HOST_OPTION ).argName( "host name" ).longOpt( "host" ).hasArg()
- .desc( "Hostname to use (Default = " + DEFAULT_HOST + ")" )
- .build();
-
- hostOpt.setRequired( hostRequired );
-
- Option portOpt = Option.builder( PORT_OPTION ).argName( "port" ).longOpt( "port" ).hasArg()
- .desc( "Port to use on host (Default = " + AbstractCLI.DEFAULT_PORT + ")" )
- .build();
-
- portOpt.setRequired( portRequired );
-
- options.addOption( hostOpt );
- options.addOption( portOpt );
+ protected void addDateOption( Options options ) {
+ options.addOption( Option.builder( DATE_OPTION ).hasArg()
+ .desc( "Constrain to run only on entities with analyses older than the given date. "
+ + "For example, to run only on entities that have not been analyzed in the last 10 days, use '-10d'. "
+ + "If there is no record of when the analysis was last run, it will be run." )
+ .build() );
}
/**
- * Convenience method to add an option for parallel processing option.
+ * Add the {@code -threads} option.
+ *
+ * This is used to configure the internal batch processing thread pool which can be used with
+ * {@link #executeBatchTasks(Collection)}. You may also use {@link #getNumThreads()} to retrieve the number of
+ * threads to use.
*/
protected void addThreadsOption( Options options ) {
- Option threadsOpt = Option.builder( THREADS_OPTION ).argName( "numThreads" ).hasArg()
+ options.addOption( Option.builder( THREADS_OPTION ).argName( "numThreads" ).hasArg()
.desc( "Number of threads to use for batch processing." )
- .build();
- options.addOption( threadsOpt );
+ .type( Integer.class )
+ .build() );
}
/**
- * Build option implementation.
+ * Allow positional arguments to be specified. The default is false and an error will be produced if positional
+ * arguments are supplied by the user.
*
- * Implement this method to add options to your command line, using the OptionBuilder.
- *
- * This is called right after {@link #buildStandardOptions(Options)} so the options will be added after standard options.
+ * Those arguments can be retrieved in {@link #processOptions(CommandLine)} by using {@link CommandLine#getArgList()}.
*/
- protected abstract void buildOptions( Options options );
+ @SuppressWarnings("unused")
+ protected void setAllowPositionalArguments( boolean allowPositionalArguments ) {
+ this.allowPositionalArguments = allowPositionalArguments;
+ }
+
+ protected boolean isAutoSeek() {
+ return autoSeek;
+ }
+
+ protected Class getAutoSeekEventType() {
+ return autoSeekEventType;
+ }
+
+ protected int getNumThreads() {
+ return numThreads;
+ }
+
+ protected Date getLimitingDate() {
+ Date skipIfLastRunLaterThan = null;
+ if ( StringUtils.isNotBlank( mDate ) ) {
+ skipIfLastRunLaterThan = DateUtil.getRelativeDate( new Date(), mDate );
+ AbstractCLI.log.info( "Analyses will be run only if last was older than " + skipIfLastRunLaterThan );
+ }
+ return skipIfLastRunLaterThan;
+ }
protected void buildStandardOptions( Options options ) {
AbstractCLI.log.debug( "Creating standard options" );
options.addOption( HELP_OPTION, "help", false, "Print this message" );
options.addOption( TESTING_OPTION, "testing", false, "Use the test environment. This option must be passed before the command." );
+ options.addOption( BATCH_FORMAT_OPTION, true, "Format to use to the batch summary" );
+ options.addOption( Option.builder( BATCH_OUTPUT_FILE_OPTION ).hasArg().type( File.class ).desc( "Output file to use for the batch summary (default is standard output)" ).build() );
}
/**
- * Command line implementation.
+ * Build option implementation.
*
- * This is called after {@link #buildOptions(Options)} and {@link #processOptions(CommandLine)}, so the implementation can assume that
- * all its arguments have already been initialized.
- *
- * @throws Exception in case of unrecoverable failure, an exception is thrown and will result in a {@link #FAILURE}
- * exit code, otherwise use {@link #addErrorObject}
+ * Implement this method to add options to your command line, using the OptionBuilder.
+ *
+ * This is called right after {@link #buildStandardOptions(Options)} so the options will be added after standard options.
*/
- protected abstract void doWork() throws Exception;
-
- protected final double getDoubleOptionValue( CommandLine commandLine, char option ) {
- try {
- return Double.parseDouble( commandLine.getOptionValue( option ) );
- } catch ( NumberFormatException e ) {
- throw new RuntimeException( this.invalidOptionString( commandLine, String.valueOf( option ) ) + ", not a valid double", e );
- }
- }
+ protected abstract void buildOptions( Options options );
- protected final double getDoubleOptionValue( CommandLine commandLine, String option ) {
- try {
- return Double.parseDouble( commandLine.getOptionValue( option ) );
- } catch ( NumberFormatException e ) {
- throw new RuntimeException( this.invalidOptionString( commandLine, option ) + ", not a valid double", e );
+ /**
+ * Somewhat annoying: This causes subclasses to be unable to safely use 'h', 'p', 'u' and 'P' etc. for their own
+ * purposes.
+ */
+ protected void processStandardOptions( CommandLine commandLine ) throws ParseException {
+ if ( commandLine.hasOption( DATE_OPTION ) ^ commandLine.hasOption( AbstractCLI.AUTO_OPTION_NAME ) ) {
+ throw new IllegalArgumentException( String.format( "Please only select one of -%s or -%s", DATE_OPTION, AUTO_OPTION_NAME ) );
}
- }
- protected final String getFileNameOptionValue( CommandLine commandLine, char c ) {
- String fileName = commandLine.getOptionValue( c );
- File f = new File( fileName );
- if ( !f.canRead() ) {
- throw new RuntimeException( this.invalidOptionString( commandLine, String.valueOf( c ) ) + ", cannot read from file" );
+ if ( commandLine.hasOption( DATE_OPTION ) ) {
+ this.mDate = commandLine.getOptionValue( DATE_OPTION );
}
- return fileName;
- }
- protected final String getFileNameOptionValue( CommandLine commandLine, String c ) {
- String fileName = commandLine.getOptionValue( c );
- File f = new File( fileName );
- if ( !f.canRead() ) {
- throw new RuntimeException( this.invalidOptionString( commandLine, c ) + ", cannot read from file" );
- }
- return fileName;
- }
+ this.autoSeek = commandLine.hasOption( AbstractCLI.AUTO_OPTION_NAME );
- protected final int getIntegerOptionValue( CommandLine commandLine, char option ) {
- try {
- return Integer.parseInt( commandLine.getOptionValue( option ) );
- } catch ( NumberFormatException e ) {
- throw new RuntimeException( this.invalidOptionString( commandLine, String.valueOf( option ) ) + ", not a valid integer", e );
+ if ( commandLine.hasOption( THREADS_OPTION ) ) {
+ this.numThreads = ( Integer ) commandLine.getParsedOptionValue( THREADS_OPTION );
+ if ( this.numThreads < 1 ) {
+ throw new IllegalArgumentException( "Number of threads must be greater than 1." );
+ }
+ } else {
+ this.numThreads = 1;
}
- }
- protected final int getIntegerOptionValue( CommandLine commandLine, String option ) {
- try {
- return Integer.parseInt( commandLine.getOptionValue( option ) );
- } catch ( NumberFormatException e ) {
- throw new RuntimeException( this.invalidOptionString( commandLine, option ) + ", not a valid integer", e );
+ if ( this.numThreads > 1 ) {
+ this.executorService = Executors.newFixedThreadPool( this.numThreads );
+ } else {
+ this.executorService = Executors.newSingleThreadExecutor();
}
- }
- protected Date getLimitingDate() {
- Date skipIfLastRunLaterThan = null;
- if ( StringUtils.isNotBlank( mDate ) ) {
- skipIfLastRunLaterThan = DateUtil.getRelativeDate( new Date(), mDate );
- AbstractCLI.log.info( "Analyses will be run only if last was older than " + skipIfLastRunLaterThan );
+ if ( commandLine.hasOption( BATCH_FORMAT_OPTION ) ) {
+ try {
+ this.batchFormat = BatchFormat.valueOf( commandLine.getOptionValue( BATCH_FORMAT_OPTION ).toUpperCase() );
+ } catch ( IllegalArgumentException e ) {
+ throw new ParseException( String.format( "Unsupported batch format: %s.", commandLine.getOptionValue( BATCH_FORMAT_OPTION ) ) );
+ }
+ } else {
+ this.batchFormat = commandLine.hasOption( BATCH_OUTPUT_FILE_OPTION ) ? BatchFormat.TSV : BatchFormat.TEXT;
}
- return skipIfLastRunLaterThan;
- }
-
- protected void printHelp( Options options ) {
- new HelpFormatter().printHelp( new PrintWriter( System.err, true ), 150,
- this.getCommandName() + " [options]",
- this.getShortDesc() + "\n" + AbstractCLI.HEADER,
- options, HelpFormatter.DEFAULT_LEFT_PAD, HelpFormatter.DEFAULT_DESC_PAD, AbstractCLI.FOOTER );
+ this.batchOutputFile = ( File ) commandLine.getParsedOptionValue( BATCH_OUTPUT_FILE_OPTION );
}
/**
@@ -319,35 +366,61 @@ protected void printHelp( Options options ) {
*
* Implement this to provide processing of options. It is called after {@link #buildOptions(Options)} and right before
* {@link #doWork()}.
+ * @throws ParseException in case of unrecoverable failure (i.e. missing option or invalid value), an exception can
+ * be raised and will result in an exit code of {@link #FAILURE}.
+ */
+ protected abstract void processOptions( CommandLine commandLine ) throws ParseException;
+
+ /**
+ * Command line implementation.
+ *
+ * This is called after {@link #buildOptions(Options)} and {@link #processOptions(CommandLine)}, so the
+ * implementation can assume that all its arguments have already been initialized.
*
- * @throws Exception in case of unrecoverable failure (i.e. missing option or invalid value), an exception can be
- * raised and will result in an exit code of {@link #FAILURE}.
+ * @throws Exception in case of unrecoverable failure, an exception is thrown and will result in a
+ * {@link #FAILURE} exit code, otherwise use {@link #addErrorObject} to indicate an
+ * error and resume processing
*/
- protected abstract void processOptions( CommandLine commandLine ) throws Exception;
+ protected abstract void doWork() throws Exception;
+
+ /**
+ * Prompt the user for a confirmation or raise an exception to abort the {@link #doWork()} method.
+ */
+ protected void promptConfirmationOrAbort( String message ) throws Exception {
+ if ( System.console() == null ) {
+ throw new IllegalStateException( "A console must be available for prompting confirmation." );
+ }
+ String line = System.console().readLine( "WARNING: %s\nWARNING: Enter YES to continue: ",
+ message.replaceAll( "\n", "\nWARNING: " ) );
+ if ( "YES".equals( line.trim() ) ) {
+ return;
+ }
+ throw new WorkAbortedException( "Confirmation failed, the command cannot proceed." );
+ }
/**
* Add a success object to indicate success in a batch processing.
*
- * This is further used in {@link #summarizeProcessing()} to summarize the execution of the command.
+ * This is further used in {@link #summarizeBatchProcessing()} to summarize the execution of the command.
*
* @param successObject object that was processed
* @param message success message
*/
protected void addSuccessObject( Object successObject, String message ) {
- successObjects.add( new BatchProcessingResult( successObject, message, null ) );
+ batchProcessingResults.add( new BatchProcessingResult( false, successObject, message, null ) );
}
/**
* @see #addSuccessObject(Object, String)
*/
protected void addSuccessObject( Object successObject ) {
- successObjects.add( new BatchProcessingResult( successObject, null, null ) );
+ batchProcessingResults.add( new BatchProcessingResult( false, successObject, null, null ) );
}
/**
* Add an error object with a stacktrace to indicate failure in a batch processing.
*
- * This is further used in {@link #summarizeProcessing()} to summarize the execution of the command.
+ * This is further used in {@link #summarizeBatchProcessing()} to summarize the execution of the command.
*
* This is intended to be used when an {@link Exception} is caught.
*
@@ -356,7 +429,7 @@ protected void addSuccessObject( Object successObject ) {
* @param throwable throwable to produce a stacktrace
*/
protected void addErrorObject( @Nullable Object errorObject, String message, Throwable throwable ) {
- errorObjects.add( new BatchProcessingResult( errorObject, message, throwable ) );
+ batchProcessingResults.add( new BatchProcessingResult( true, errorObject, message, throwable ) );
log.error( "Error while processing " + ( errorObject != null ? errorObject : "unknown object" ) + ":\n\t" + message, throwable );
}
@@ -365,7 +438,7 @@ protected void addErrorObject( @Nullable Object errorObject, String message, Thr
* @see #addErrorObject(Object, String)
*/
protected void addErrorObject( @Nullable Object errorObject, String message ) {
- errorObjects.add( new BatchProcessingResult( errorObject, message, null ) );
+ batchProcessingResults.add( new BatchProcessingResult( true, errorObject, message, null ) );
log.error( "Error while processing " + ( errorObject != null ? errorObject : "unknown object" ) + ":\n\t" + message );
}
@@ -374,7 +447,7 @@ protected void addErrorObject( @Nullable Object errorObject, String message ) {
* @see #addErrorObject(Object, String, Throwable)
*/
protected void addErrorObject( @Nullable Object errorObject, Exception exception ) {
- errorObjects.add( new BatchProcessingResult( errorObject, exception.getMessage(), exception ) );
+ batchProcessingResults.add( new BatchProcessingResult( true, errorObject, exception.getMessage(), exception ) );
log.error( "Error while processing " + ( errorObject != null ? errorObject : "unknown object" ), exception );
}
@@ -382,8 +455,32 @@ protected void addErrorObject( @Nullable Object errorObject, Exception exception
* Print out a summary of what the program did. Useful when analyzing lists of experiments etc. Use the
* 'successObjects' and 'errorObjects'
*/
- private void summarizeProcessing() {
- if ( successObjects.size() > 0 ) {
+ private void summarizeBatchProcessing() {
+ if ( batchProcessingResults.isEmpty() ) {
+ return;
+ }
+ if ( batchFormat != BatchFormat.SUPPRESS && batchOutputFile != null ) {
+ log.info( String.format( "Batch processing summary will be written to %s", batchOutputFile.getAbsolutePath() ) );
+ }
+ try ( Writer dest = batchOutputFile != null ? new OutputStreamWriter( Files.newOutputStream( batchOutputFile.toPath() ) ) : null ) {
+ switch ( batchFormat ) {
+ case TEXT:
+ summarizeBatchProcessingToText( dest != null ? dest : System.out );
+ break;
+ case TSV:
+ summarizeBatchProcessingToTsv( dest != null ? dest : System.out );
+ break;
+ case SUPPRESS:
+ break;
+ }
+ } catch ( IOException e ) {
+ log.error( "Failed to summarize batch processing.", e );
+ }
+ }
+
+ private void summarizeBatchProcessingToText( Appendable dest ) throws IOException {
+ List successObjects = batchProcessingResults.stream().filter( bp -> !bp.isError() ).collect( Collectors.toList() );
+ if ( !successObjects.isEmpty() ) {
StringBuilder buf = new StringBuilder();
buf.append( "\n---------------------\nSuccessfully processed " ).append( successObjects.size() )
.append( " objects:\n" );
@@ -391,11 +488,11 @@ private void summarizeProcessing() {
buf.append( result ).append( "\n" );
}
buf.append( "---------------------\n" );
-
- AbstractCLI.log.info( buf );
+ dest.append( buf );
}
- if ( errorObjects.size() > 0 ) {
+ List errorObjects = batchProcessingResults.stream().filter( BatchProcessingResult::isError ).collect( Collectors.toList() );
+ if ( !errorObjects.isEmpty() ) {
StringBuilder buf = new StringBuilder();
buf.append( "\n---------------------\nErrors occurred during the processing of " )
.append( errorObjects.size() ).append( " objects:\n" );
@@ -403,7 +500,19 @@ private void summarizeProcessing() {
buf.append( result ).append( "\n" );
}
buf.append( "---------------------\n" );
- AbstractCLI.log.error( buf );
+ dest.append( buf );
+ }
+ }
+
+ private void summarizeBatchProcessingToTsv( Appendable dest ) throws IOException {
+ try ( CSVPrinter printer = new CSVPrinter( dest, CSVFormat.TDF ) ) {
+ for ( BatchProcessingResult result : batchProcessingResults ) {
+ printer.printRecord(
+ result.getSource(),
+ result.isError() ? "ERROR" : "SUCCESS",
+ result.getMessage(),
+ result.throwable != null ? ExceptionUtils.getRootCauseMessage( result.throwable ) : null );
+ }
}
}
@@ -414,47 +523,23 @@ private void summarizeProcessing() {
protected List executeBatchTasks( Collection> tasks ) throws InterruptedException {
List> futures = executorService.invokeAll( tasks );
List futureResults = new ArrayList<>( futures.size() );
- int i = 0;
+ int i = 1;
for ( Future future : futures ) {
try {
futureResults.add( future.get() );
} catch ( ExecutionException e ) {
- addErrorObject( null, String.format( "Batch task #%d failed", ++i ), e.getCause() );
+ addErrorObject( null, String.format( "Batch task #%d failed", i ), e.getCause() );
+ } finally {
+ i++;
}
}
return futureResults;
}
- private String invalidOptionString( CommandLine commandLine, String option ) {
- return "Invalid value '" + commandLine.getOptionValue( option ) + " for option " + option;
- }
-
- /**
- * Somewhat annoying: This causes subclasses to be unable to safely use 'h', 'p', 'u' and 'P' etc. for their own
- * purposes.
- */
- protected void processStandardOptions( CommandLine commandLine ) {
-
- if ( commandLine.hasOption( AbstractCLI.HOST_OPTION ) ) {
- this.host = commandLine.getOptionValue( AbstractCLI.HOST_OPTION );
- } else {
- this.host = DEFAULT_HOST;
- }
-
- if ( commandLine.hasOption( AbstractCLI.PORT_OPTION ) ) {
- this.port = this.getIntegerOptionValue( commandLine, AbstractCLI.PORT_OPTION );
- } else {
- this.port = AbstractCLI.DEFAULT_PORT;
- }
-
- if ( commandLine.hasOption( DATE_OPTION ) ) {
- this.mDate = commandLine.getOptionValue( DATE_OPTION );
- }
-
- if ( this.numThreads < 1 ) {
- throw new IllegalArgumentException( "Number of threads must be greater than 1." );
- }
- this.executorService = new ForkJoinPool( this.numThreads );
+ private enum BatchFormat {
+ TEXT,
+ TSV,
+ SUPPRESS
}
/**
@@ -462,6 +547,7 @@ protected void processStandardOptions( CommandLine commandLine ) {
*/
@Value
private static class BatchProcessingResult {
+ boolean isError;
@Nullable
Object source;
@Nullable
@@ -469,7 +555,8 @@ private static class BatchProcessingResult {
@Nullable
Throwable throwable;
- public BatchProcessingResult( @Nullable Object source, @Nullable String message, @Nullable Throwable throwable ) {
+ public BatchProcessingResult( boolean isError, @Nullable Object source, @Nullable String message, @Nullable Throwable throwable ) {
+ this.isError = isError;
this.source = source;
this.message = message;
this.throwable = throwable;
@@ -480,10 +567,26 @@ public String toString() {
StringBuilder buf = new StringBuilder();
buf.append( source != null ? source : "Unknown object" );
if ( message != null ) {
- buf.append( ":\n\t" )
- .append( message.replace( "\n", "\n\t" ) );
+ buf.append( "\t" )
+ .append( message.replace( "\n", "\n\t" ) ); // FIXME We don't want newlines here at all, but I'm not sure what condition this is meant to fix exactly.
+ }
+ if ( throwable != null ) {
+ buf.append( "\t" )
+ .append( "Reason: " )
+ .append( ExceptionUtils.getRootCauseMessage( throwable ) );
}
return buf.toString();
}
}
+
+ /**
+ * Exception raised when a {@link #doWork()} aborted by the user.
+ * @author poirigui
+ */
+ private static class WorkAbortedException extends Exception {
+
+ private WorkAbortedException( String message ) {
+ super( message );
+ }
+ }
}
diff --git a/gemma-cli/src/main/java/ubic/gemma/core/util/AbstractCLIContextCLI.java b/gemma-cli/src/main/java/ubic/gemma/core/util/AbstractCLIContextCLI.java
deleted file mode 100644
index 220e2b953d..0000000000
--- a/gemma-cli/src/main/java/ubic/gemma/core/util/AbstractCLIContextCLI.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * The gemma project
- *
- * Copyright (c) 2013 University of British Columbia
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-package ubic.gemma.core.util;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.lang3.StringUtils;
-import org.springframework.beans.factory.BeanFactory;
-import ubic.gemma.core.apps.GemmaCLI.CommandGroup;
-import ubic.gemma.model.expression.arrayDesign.ArrayDesign;
-import ubic.gemma.model.genome.Taxon;
-import ubic.gemma.persistence.service.common.auditAndSecurity.AuditEventService;
-import ubic.gemma.persistence.service.common.auditAndSecurity.AuditTrailService;
-import ubic.gemma.persistence.service.expression.arrayDesign.ArrayDesignService;
-import ubic.gemma.persistence.service.genome.taxon.TaxonService;
-
-import java.io.BufferedReader;
-import java.io.FileReader;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-
-/**
- * Spring configuration for CLI.
- *
- * @author anton date: 18/02/13
- */
-public abstract class AbstractCLIContextCLI extends AbstractSpringAwareCLI {
-
- /**
- * may be tab-delimited, only first column used, commented (#) lines are ignored.
- *
- * @param fileName the file name
- * @return list of ee identifiers
- * @throws IOException in case there is an IO error while reading the file
- */
- protected static List readListFileToStrings( String fileName ) throws IOException {
- List eeNames = new ArrayList<>();
- try ( BufferedReader in = new BufferedReader( new FileReader( fileName ) ) ) {
- while ( in.ready() ) {
- String line = in.readLine().trim();
- if ( line.startsWith( "#" ) ) {
- continue;
- }
- if ( line.isEmpty() )
- continue;
- String[] split = StringUtils.split( line, "\t" );
- eeNames.add( split[0] );
- }
- return eeNames;
- }
- }
-
- protected Taxon setTaxonByName( CommandLine commandLine, TaxonService taxonService ) {
- String taxonName = commandLine.getOptionValue( 't' );
- ubic.gemma.model.genome.Taxon taxon = taxonService.findByCommonName( taxonName );
- if ( taxon == null ) {
- AbstractCLI.log.error( "ERROR: Cannot find taxon " + taxonName );
- }
- return taxon;
- }
-
- /**
- * @param name of the array design to find.
- * @param arrayDesignService the arrayDesignService to use for the AD retrieval
- * @return an array design, if found. Bails otherwise with an error exit code
- */
- protected ArrayDesign locateArrayDesign( String name, ArrayDesignService arrayDesignService ) {
-
- ArrayDesign arrayDesign = null;
-
- Collection byname = arrayDesignService.findByName( name.trim().toUpperCase() );
- if ( byname.size() > 1 ) {
- throw new IllegalArgumentException( "Ambiguous name: " + name );
- } else if ( byname.size() == 1 ) {
- arrayDesign = byname.iterator().next();
- }
-
- if ( arrayDesign == null ) {
- arrayDesign = arrayDesignService.findByShortName( name );
- }
-
- if ( arrayDesign == null ) {
- AbstractCLI.log.error( "No arrayDesign " + name + " found" );
- }
- return arrayDesign;
- }
-
-}
diff --git a/gemma-cli/src/main/java/ubic/gemma/core/util/CLI.java b/gemma-cli/src/main/java/ubic/gemma/core/util/CLI.java
index 9936dd9fa1..e012a00413 100644
--- a/gemma-cli/src/main/java/ubic/gemma/core/util/CLI.java
+++ b/gemma-cli/src/main/java/ubic/gemma/core/util/CLI.java
@@ -36,5 +36,5 @@ public interface CLI {
* Execute the given command given CLI arguments.
* @return an exit code
*/
- int executeCommand( String[] args );
+ int executeCommand( String... args );
}
diff --git a/gemma-cli/src/main/java/ubic/gemma/core/util/ConfigurationLinter.java b/gemma-cli/src/main/java/ubic/gemma/core/util/ConfigurationLinter.java
new file mode 100644
index 0000000000..0c6491c155
--- /dev/null
+++ b/gemma-cli/src/main/java/ubic/gemma/core/util/ConfigurationLinter.java
@@ -0,0 +1,40 @@
+package ubic.gemma.core.util;
+
+import lombok.extern.apachecommons.CommonsLog;
+import org.springframework.beans.factory.InitializingBean;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Profile;
+import org.springframework.stereotype.Component;
+
+/**
+ * Lint various aspects of the configuration for the CLI profile.
+ */
+@CommonsLog
+@Profile("cli")
+@Component
+public class ConfigurationLinter implements InitializingBean {
+
+ @Value("${load.ontologies}")
+ private boolean autoLoadOntologies;
+
+ @Value("${load.homologene}")
+ private boolean loadHomologene;
+
+ @Value("${gemma.hibernate.hbm2ddl.auto}")
+ private String hbm2ddl;
+
+ @Override
+ public void afterPropertiesSet() {
+ if ( autoLoadOntologies ) {
+ log.warn( "Auto-loading of ontologies is enabled, this is not recommended for the CLI. Disable it by setting load.ontologies=false in Gemma.properties." );
+ }
+
+ if ( loadHomologene ) {
+ log.warn( "Homologene is enabled, this is not recommended for the CLI. Disable it by setting load.homologene=false in Gemma.properties." );
+ }
+
+ if ( "validate".equals( hbm2ddl ) ) {
+ log.warn( "Hibernate is configured to validate the database schema, this is not recommended for the CLI. Disable it by setting gemma.hibernate.hbm2ddl.auto= in Gemma.properties." );
+ }
+ }
+}
diff --git a/gemma-cli/src/main/java/ubic/gemma/core/util/FileUtils.java b/gemma-cli/src/main/java/ubic/gemma/core/util/FileUtils.java
new file mode 100644
index 0000000000..03d26de2a7
--- /dev/null
+++ b/gemma-cli/src/main/java/ubic/gemma/core/util/FileUtils.java
@@ -0,0 +1,35 @@
+package ubic.gemma.core.util;
+
+import org.apache.commons.lang3.StringUtils;
+
+import java.io.BufferedReader;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+public class FileUtils {
+ /**
+ * may be tab-delimited, only first column used, commented (#) lines are ignored.
+ *
+ * @param fileName the file name
+ * @return list of ee identifiers
+ * @throws IOException in case there is an IO error while reading the file
+ */
+ public static List readListFileToStrings( String fileName ) throws IOException {
+ List eeNames = new ArrayList<>();
+ try ( BufferedReader in = new BufferedReader( new FileReader( fileName ) ) ) {
+ while ( in.ready() ) {
+ String line = in.readLine().trim();
+ if ( line.startsWith( "#" ) ) {
+ continue;
+ }
+ if ( line.isEmpty() )
+ continue;
+ String[] split = StringUtils.split( line, "\t" );
+ eeNames.add( split[0] );
+ }
+ return eeNames;
+ }
+ }
+}
diff --git a/gemma-cli/src/test/java/ubic/gemma/core/apps/ArrayDesignMergeCliTest.java b/gemma-cli/src/test/java/ubic/gemma/core/apps/ArrayDesignMergeCliTest.java
index ba2c761e1c..9c61a30f57 100644
--- a/gemma-cli/src/test/java/ubic/gemma/core/apps/ArrayDesignMergeCliTest.java
+++ b/gemma-cli/src/test/java/ubic/gemma/core/apps/ArrayDesignMergeCliTest.java
@@ -1,17 +1,20 @@
package ubic.gemma.core.apps;
+import gemma.gsec.authentication.ManualAuthenticationService;
import org.junit.After;
import org.junit.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.security.test.context.support.WithMockUser;
+import org.springframework.security.test.context.support.WithSecurityContextTestExecutionListener;
import org.springframework.test.context.ContextConfiguration;
+import org.springframework.test.context.TestExecutionListeners;
import ubic.gemma.core.analysis.report.ArrayDesignReportService;
import ubic.gemma.core.loader.expression.arrayDesign.ArrayDesignMergeService;
-import ubic.gemma.core.util.AbstractCLI;
import ubic.gemma.core.util.test.BaseCliTest;
import ubic.gemma.model.expression.arrayDesign.ArrayDesign;
+import ubic.gemma.persistence.service.common.auditAndSecurity.AuditTrailService;
import ubic.gemma.persistence.service.expression.arrayDesign.ArrayDesignService;
import ubic.gemma.persistence.util.TestComponent;
@@ -23,17 +26,28 @@
import static org.mockito.Mockito.*;
@ContextConfiguration
+@TestExecutionListeners(WithSecurityContextTestExecutionListener.class)
public class ArrayDesignMergeCliTest extends BaseCliTest {
@Configuration
@TestComponent
- public static class ArrayDesignMergeCliTestContextConfiguration extends BaseCliTestContextConfiguration {
+ public static class ArrayDesignMergeCliTestContextConfiguration {
@Bean
public ArrayDesignMergeCli arrayDesignMergeCli() {
return new ArrayDesignMergeCli();
}
+ @Bean
+ public ManualAuthenticationService manualAuthenticationService() {
+ return mock();
+ }
+
+ @Bean
+ public AuditTrailService auditTrailService() {
+ return mock();
+ }
+
@Bean
public ArrayDesignMergeService arrayDesignMergeService() {
return mock( ArrayDesignMergeService.class );
@@ -76,8 +90,8 @@ public void test() {
when( arrayDesignService.thaw( any( ArrayDesign.class ) ) ).thenAnswer( args -> args.getArgument( 0 ) );
when( arrayDesignService.thaw( anyCollection() ) ).thenAnswer( args -> args.getArgument( 0 ) );
Collection otherPlatforms = new HashSet<>( Arrays.asList( b, c ) );
- assertThat( arrayDesignMergeCli.executeCommand( new String[] { "-a", "1", "-o", "2,3", "-s", "4", "-n", "four is better than one" } ) )
- .isEqualTo( AbstractCLI.SUCCESS );
+ assertThat( arrayDesignMergeCli.executeCommand( "-a", "1", "-o", "2,3", "-s", "4", "-n", "four is better than one" ) )
+ .isEqualTo( 0 );
verify( arrayDesignMergeService ).merge( a, otherPlatforms, "four is better than one", "4", false );
}
}
diff --git a/gemma-cli/src/test/java/ubic/gemma/core/apps/ExternalDatabaseUpdaterCliTest.java b/gemma-cli/src/test/java/ubic/gemma/core/apps/ExternalDatabaseUpdaterCliTest.java
index 0b689e8d92..f5935cac36 100644
--- a/gemma-cli/src/test/java/ubic/gemma/core/apps/ExternalDatabaseUpdaterCliTest.java
+++ b/gemma-cli/src/test/java/ubic/gemma/core/apps/ExternalDatabaseUpdaterCliTest.java
@@ -1,5 +1,6 @@
package ubic.gemma.core.apps;
+import gemma.gsec.authentication.ManualAuthenticationService;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -7,12 +8,15 @@
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.security.test.context.support.WithMockUser;
+import org.springframework.security.test.context.support.WithSecurityContextTestExecutionListener;
import org.springframework.test.context.ContextConfiguration;
+import org.springframework.test.context.TestExecutionListeners;
import ubic.gemma.core.security.authentication.UserManager;
import ubic.gemma.core.util.test.BaseCliTest;
import ubic.gemma.model.common.auditAndSecurity.User;
import ubic.gemma.model.common.description.DatabaseType;
import ubic.gemma.model.common.description.ExternalDatabase;
+import ubic.gemma.persistence.service.common.auditAndSecurity.AuditTrailService;
import ubic.gemma.persistence.service.common.description.ExternalDatabaseService;
import ubic.gemma.persistence.util.TestComponent;
@@ -23,11 +27,12 @@
import static org.mockito.Mockito.*;
@ContextConfiguration
+@TestExecutionListeners(WithSecurityContextTestExecutionListener.class)
public class ExternalDatabaseUpdaterCliTest extends BaseCliTest {
@Configuration
@TestComponent
- static class ExternalDatabaseUpdaterCliTestContextConfiguration extends BaseCliTestContextConfiguration {
+ static class ExternalDatabaseUpdaterCliTestContextConfiguration {
@Bean
public ExternalDatabaseUpdaterCli externalDatabaseUpdaterCli() {
@@ -43,6 +48,16 @@ public ExternalDatabaseService externalDatabaseService() {
public UserManager userManager() {
return mock( UserManager.class );
}
+
+ @Bean
+ public ManualAuthenticationService manualAuthenticationService() {
+ return mock();
+ }
+
+ @Bean
+ public AuditTrailService auditTrailService() {
+ return mock();
+ }
}
@Autowired
diff --git a/gemma-cli/src/test/java/ubic/gemma/core/apps/FactorValueMigratorCLITest.java b/gemma-cli/src/test/java/ubic/gemma/core/apps/FactorValueMigratorCLITest.java
new file mode 100644
index 0000000000..93fc53ad7e
--- /dev/null
+++ b/gemma-cli/src/test/java/ubic/gemma/core/apps/FactorValueMigratorCLITest.java
@@ -0,0 +1,165 @@
+package ubic.gemma.core.apps;
+
+import gemma.gsec.authentication.ManualAuthenticationService;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.junit.Before;
+import org.junit.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.core.io.ClassPathResource;
+import org.springframework.security.test.context.support.WithMockUser;
+import org.springframework.security.test.context.support.WithSecurityContextTestExecutionListener;
+import org.springframework.test.context.ContextConfiguration;
+import org.springframework.test.context.TestExecutionListeners;
+import org.springframework.transaction.PlatformTransactionManager;
+import ubic.gemma.core.util.test.BaseCliTest;
+import ubic.gemma.model.common.description.Characteristic;
+import ubic.gemma.model.expression.experiment.FactorValue;
+import ubic.gemma.model.expression.experiment.Statement;
+import ubic.gemma.persistence.service.expression.experiment.FactorValueMigratorService;
+import ubic.gemma.persistence.service.expression.experiment.FactorValueMigratorServiceImpl;
+import ubic.gemma.persistence.service.expression.experiment.FactorValueService;
+import ubic.gemma.persistence.util.TestComponent;
+
+import java.io.IOException;
+import java.util.concurrent.atomic.AtomicLong;
+
+import static org.mockito.Mockito.*;
+
+@Deprecated
+@ContextConfiguration
+@TestExecutionListeners(WithSecurityContextTestExecutionListener.class)
+public class FactorValueMigratorCLITest extends BaseCliTest {
+
+ @Configuration
+ @TestComponent
+ static class FactorValueMigratorCLITestContextConfiguration {
+
+ @Bean
+ public FactorValueMigratorCLI factorValueMigratorCLI() {
+ return new FactorValueMigratorCLI();
+ }
+
+ @Bean
+ public FactorValueMigratorService factorValueMigratorService() {
+ return new FactorValueMigratorServiceImpl();
+ }
+
+ @Bean
+ public FactorValueService factorValueService() {
+ return mock();
+ }
+
+ @Bean
+ public PlatformTransactionManager platformTransactionManager() {
+ return mock();
+ }
+
+ @Bean
+ public ManualAuthenticationService manualAuthenticationService() {
+ return mock();
+ }
+ }
+
+ @Autowired
+ private FactorValueMigratorCLI cli;
+
+ @Autowired
+ private FactorValueService factorValueService;
+
+ private final FactorValue[] fvs = {
+ createFactorValue( 1L, 1L, 2L, 3L ),
+ createFactorValue( 2L ),
+ createFactorValue( 3L, 4L ),
+ createFactorValue( 4L, 5L, 6L ),
+ createFactorValue( 5L, 7L, 8L, 9L ),
+ createFactorValue( 6L, 10L, 11L ),
+ createFactorValue( 7L, 12L, 13L ),
+ createFactorValue( 8L, 14L, 15L )
+ };
+
+ private Characteristic getObjectById( long fvId, long id ) {
+ return fvs[( int ) ( fvId - 1 )].getOldStyleCharacteristics().stream()
+ .filter( c -> c.getId().equals( id ) )
+ .findAny()
+ .orElse( null );
+ }
+
+ private String getCategory( long fvId, long id ) {
+ return getObjectById( fvId, id ).getCategory();
+ }
+
+ private String getObject( long fvId, long id ) {
+ return getObjectById( fvId, id ).getValue();
+ }
+
+ @Before
+ public void setUp() {
+ when( factorValueService.loadWithOldStyleCharacteristics( any(), anyBoolean() ) )
+ .thenAnswer( a -> fvs[a.getArgument( 0, Long.class ).intValue() - 1] );
+ when( factorValueService.countAll() ).thenReturn( ( long ) fvs.length );
+ AtomicLong id = new AtomicLong( 0L );
+ when( factorValueService.saveStatementIgnoreAcl( any(), any() ) ).thenAnswer( a -> {
+ Statement s = a.getArgument( 1, Statement.class );
+ if ( s.getId() == null ) {
+ s.setId( id.incrementAndGet() );
+ }
+ return s;
+ } );
+ }
+
+ @Test
+ @WithMockUser
+ public void testMigrateFactorValues() throws IOException {
+ cli.executeCommand(
+ "-migrationFile", new ClassPathResource( "ubic/gemma/core/apps/factor-value-migration.tsv" ).getFile().getAbsolutePath(),
+ "-batchFormat", "suppress" );
+ verify( factorValueService, times( 8 ) ).loadWithOldStyleCharacteristics( any(), eq( false ) );
+ verify( factorValueService ).saveStatementIgnoreAcl( any(), eq( createStatement( getCategory( 1L, 1L ), "Pax6", "has_modifier", getObject( 1L, 2L ), "has_modifier", getObject( 1L, 3L ) ) ) );
+ verify( factorValueService ).saveStatementIgnoreAcl( any(), eq( createStatement( "Gene", "Pax6" ) ) );
+ verify( factorValueService ).saveStatementIgnoreAcl( any(), eq( createStatement( getCategory( 3L, 4L ), getObject( 3L, 4L ) ) ) );
+ verify( factorValueService ).saveStatementIgnoreAcl( any(), eq( createStatement( getCategory( 4L, 5L ), getObject( 4L, 5L ), "has_modifier", getObject( 4L, 6L ) ) ) );
+ verify( factorValueService ).saveStatementIgnoreAcl( any(), eq( createStatement( getCategory( 5L, 7L ), getObject( 5L, 7L ), "has_modifier", getObject( 5L, 8L ), "has_modifier", getObject( 5L, 9L ) ) ) );
+ verify( factorValueService ).saveStatementIgnoreAcl( any(), eq( createStatement( getCategory( 6L, 10L ), getObject( 6L, 10L ), "has_dose", "5mg", "has_modifier", getObject( 6L, 11L ) ) ) );
+ verify( factorValueService ).saveStatementIgnoreAcl( any(), eq( createStatement( "genotype", "Pax7", "has_modifier", getObject( 7L, 12L ), "has_modifier", getObject( 7L, 13L ) ) ) );
+ verify( factorValueService ).saveStatementIgnoreAcl( any(), eq( createStatement( getCategory( 8L, 14L ), getObject( 8L, 14L ), "has_modifier", getObject( 8L, 15L ), "has_modifier", getObject( 8L, 15L ) ) ) );
+ verifyNoMoreInteractions( factorValueService );
+ }
+
+ private FactorValue createFactorValue( Long id, Long... osIds ) {
+ FactorValue fv = new FactorValue();
+ fv.setId( id );
+ for ( Long osId : osIds ) {
+ fv.getOldStyleCharacteristics().add( createCharacteristic( osId ) );
+ }
+ return fv;
+ }
+
+ private Characteristic createCharacteristic( Long id ) {
+ Characteristic c = Characteristic.Factory.newInstance();
+ c.setCategory( RandomStringUtils.randomAlphanumeric( 10 ) );
+ c.setValue( RandomStringUtils.randomAlphanumeric( 10 ) );
+ c.setId( id );
+ return c;
+ }
+
+ private Statement createStatement( String category, String value ) {
+ return createStatement( category, value, null, null, null, null );
+ }
+
+ private Statement createStatement( String category, String value, String predicate, String object ) {
+ return createStatement( category, value, predicate, object, null, null );
+ }
+
+ private Statement createStatement( String category, String value, String predicate, String object, String secondPredicate, String secondObject ) {
+ Statement s = new Statement();
+ s.setCategory( category );
+ s.setSubject( value );
+ s.setPredicate( predicate );
+ s.setObject( object );
+ s.setSecondPredicate( secondPredicate );
+ s.setSecondObject( secondObject );
+ return s;
+ }
+}
\ No newline at end of file
diff --git a/gemma-cli/src/test/java/ubic/gemma/core/apps/FactorValueMigratorServiceTest.java b/gemma-cli/src/test/java/ubic/gemma/core/apps/FactorValueMigratorServiceTest.java
new file mode 100644
index 0000000000..7388a7e3f9
--- /dev/null
+++ b/gemma-cli/src/test/java/ubic/gemma/core/apps/FactorValueMigratorServiceTest.java
@@ -0,0 +1,134 @@
+package ubic.gemma.core.apps;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.test.context.ContextConfiguration;
+import org.springframework.test.context.junit4.AbstractJUnit4SpringContextTests;
+import org.springframework.transaction.PlatformTransactionManager;
+import ubic.gemma.model.common.description.Characteristic;
+import ubic.gemma.model.expression.experiment.FactorValue;
+import ubic.gemma.model.expression.experiment.Statement;
+import ubic.gemma.persistence.service.expression.experiment.FactorValueMigratorService;
+import ubic.gemma.persistence.service.expression.experiment.FactorValueMigratorServiceImpl;
+import ubic.gemma.persistence.service.expression.experiment.FactorValueService;
+import ubic.gemma.persistence.util.TestComponent;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.*;
+
+@Deprecated
+@ContextConfiguration
+public class FactorValueMigratorServiceTest extends AbstractJUnit4SpringContextTests {
+
+ @Configuration
+ @TestComponent
+ static class FVMSTCC {
+
+ @Bean
+ public FactorValueMigratorService factorValueMigratorService() {
+ return new FactorValueMigratorServiceImpl();
+ }
+
+ @Bean
+ public FactorValueService factorValueService() {
+ return mock();
+ }
+
+ @Bean
+ public PlatformTransactionManager platformTransactionManager() {
+ return mock();
+ }
+ }
+
+ @Autowired
+ private FactorValueMigratorService factorValueMigratorService;
+
+ @Autowired
+ private FactorValueService factorValueService;
+
+ @Before
+ public void setUp() {
+ when( factorValueService.saveStatementIgnoreAcl( any(), any() ) ).thenAnswer( a -> a.getArgument( 1 ) );
+ }
+
+ @After
+ public void tearDown() {
+ reset( factorValueService );
+ }
+
+ @Test
+ public void testMigrationThatReusesExistingStatement() {
+ FactorValue fv = new FactorValue();
+ Statement stmt = new Statement();
+ stmt.setCategory( "genotype" );
+ stmt.setSubject( "VPLL1" );
+ fv.getCharacteristics().add( stmt );
+ FactorValueMigratorService.Migration migration = FactorValueMigratorService.Migration.builder()
+ .factorValueId( 1L )
+ .category( "genotype" )
+ .subject( "VPLL1" )
+ .build();
+ when( factorValueService.loadWithOldStyleCharacteristics( 1L, false ) ).thenReturn( fv );
+ factorValueMigratorService.performMigration( migration, false );
+ verify( factorValueService ).loadWithOldStyleCharacteristics( 1L, false );
+ verify( factorValueService ).saveStatementIgnoreAcl( same( fv ), same( stmt ) );
+ }
+
+ @Test
+ public void testMigrationThatReuseObject() {
+ FactorValue fv = new FactorValue();
+ Characteristic c = new Characteristic();
+ c.setId( 1L );
+ c.setValue( "bob" );
+ fv.getOldStyleCharacteristics().add( c );
+ FactorValueMigratorService.Migration migration = FactorValueMigratorService.Migration.builder()
+ .factorValueId( 1L )
+ .category( "genotype" )
+ .subject( "VPLL1" )
+ .predicate( "has" )
+ .oldStyleCharacteristicIdUsedAsObject( 1L )
+ .secondPredicate( "also has" )
+ .oldStyleCharacteristicIdUsedAsSecondObject( 1L )
+ .build();
+ Statement stmt = new Statement();
+ stmt.setCategory( "genotype" );
+ stmt.setSubject( "VPLL1" );
+ stmt.setPredicate( "has" );
+ stmt.setObject( "bob" );
+ stmt.setSecondPredicate( "also has" );
+ stmt.setSecondObject( "bob" );
+ when( factorValueService.loadWithOldStyleCharacteristics( 1L, false ) ).thenReturn( fv );
+ factorValueMigratorService.performMigration( migration, false );
+ verify( factorValueService ).loadWithOldStyleCharacteristics( 1L, false );
+ verify( factorValueService ).saveStatementIgnoreAcl( same( fv ), eq( stmt ) );
+ }
+
+ @Test
+ public void testMigrationWithFreeTextSubject() {
+ FactorValue fv = new FactorValue();
+ Characteristic c = new Characteristic();
+ c.setId( 2L );
+ c.setCategory( "bar" );
+ c.setCategoryUri( "http://bar" );
+ c.setValueUri( "http://foo/" );
+ fv.getOldStyleCharacteristics().add( c );
+ FactorValueMigratorService.Migration migration = FactorValueMigratorService.Migration.builder()
+ .factorValueId( 1L )
+ .oldStyleCharacteristicIdUsedAsSubject( 2L )
+ .subject( "foo" )
+ .subjectUri( null )
+ .build();
+ when( factorValueService.loadWithOldStyleCharacteristics( 1L, false ) ).thenReturn( fv );
+ FactorValueMigratorService.MigrationResult result = factorValueMigratorService.performMigration( migration, false );
+ assertThat( result.getStatement() )
+ .hasFieldOrPropertyWithValue( "category", "bar" )
+ .hasFieldOrPropertyWithValue( "categoryUri", "http://bar" )
+ .hasFieldOrPropertyWithValue( "subject", "foo" )
+ .hasFieldOrPropertyWithValue( "subjectUri", null );
+ verify( factorValueService ).saveStatementIgnoreAcl( same( fv ), any() );
+ }
+}
\ No newline at end of file
diff --git a/gemma-cli/src/test/java/ubic/gemma/core/apps/FindObsoleteTermsCliTest.java b/gemma-cli/src/test/java/ubic/gemma/core/apps/FindObsoleteTermsCliTest.java
new file mode 100644
index 0000000000..168c84dc73
--- /dev/null
+++ b/gemma-cli/src/test/java/ubic/gemma/core/apps/FindObsoleteTermsCliTest.java
@@ -0,0 +1,69 @@
+package ubic.gemma.core.apps;
+
+import org.junit.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.core.task.AsyncTaskExecutor;
+import org.springframework.core.task.SimpleAsyncTaskExecutor;
+import org.springframework.test.context.ContextConfiguration;
+import ubic.gemma.core.ontology.OntologyService;
+import ubic.gemma.core.util.test.BaseCliTest;
+import ubic.gemma.core.util.test.TestPropertyPlaceholderConfigurer;
+import ubic.gemma.persistence.util.TestComponent;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+
+@ContextConfiguration
+public class FindObsoleteTermsCliTest extends BaseCliTest {
+
+ @Configuration
+ @TestComponent
+ static class FindObsoleteTermsCliTestContextConfiguration {
+
+ @Bean
+ public static TestPropertyPlaceholderConfigurer testPlaceholderConfigurer() {
+ return new TestPropertyPlaceholderConfigurer( "load.ontologies=false" );
+ }
+
+ @Bean
+ public FindObsoleteTermsCli findObsoleteTermsCli() {
+ return new FindObsoleteTermsCli();
+ }
+
+ @Bean
+ public OntologyService ontologyService() {
+ return mock();
+ }
+
+ @Bean
+ public AsyncTaskExecutor ontologyTaskExecutor() {
+ return new SimpleAsyncTaskExecutor();
+ }
+
+ @Bean
+ public ubic.basecode.ontology.providers.OntologyService ontology1() {
+ return mock();
+ }
+ }
+
+ @Autowired
+ private FindObsoleteTermsCli findObsoleteTermsCli;
+
+ @Autowired
+ private OntologyService ontologyService;
+
+ @Autowired
+ private ubic.basecode.ontology.providers.OntologyService ontology1;
+
+ @Test
+ public void test() {
+ assertEquals( 0, findObsoleteTermsCli.executeCommand() );
+ verify( ontology1 ).setSearchEnabled( false );
+ verify( ontology1 ).setInferenceMode( ubic.basecode.ontology.providers.OntologyService.InferenceMode.NONE );
+ verify( ontology1 ).initialize( true, false );
+ verify( ontologyService ).findObsoleteTermUsage();
+ }
+}
\ No newline at end of file
diff --git a/gemma-cli/src/test/java/ubic/gemma/core/apps/NCBIGene2GOAssociationLoaderCLITest.java b/gemma-cli/src/test/java/ubic/gemma/core/apps/NCBIGene2GOAssociationLoaderCLITest.java
index abee3f0023..3afdd45593 100644
--- a/gemma-cli/src/test/java/ubic/gemma/core/apps/NCBIGene2GOAssociationLoaderCLITest.java
+++ b/gemma-cli/src/test/java/ubic/gemma/core/apps/NCBIGene2GOAssociationLoaderCLITest.java
@@ -1,17 +1,22 @@
package ubic.gemma.core.apps;
+import gemma.gsec.authentication.ManualAuthenticationService;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.security.test.context.support.WithMockUser;
+import org.springframework.security.test.context.support.WithSecurityContextTestExecutionListener;
import org.springframework.test.context.ContextConfiguration;
+import org.springframework.test.context.TestExecutionListeners;
import ubic.gemma.core.util.test.BaseCliTest;
import ubic.gemma.core.util.test.category.SlowTest;
import ubic.gemma.model.common.description.DatabaseType;
import ubic.gemma.model.common.description.ExternalDatabase;
+import ubic.gemma.persistence.persister.PersisterHelper;
import ubic.gemma.persistence.service.association.Gene2GOAssociationService;
+import ubic.gemma.persistence.service.common.auditAndSecurity.AuditTrailService;
import ubic.gemma.persistence.service.common.description.ExternalDatabaseService;
import ubic.gemma.persistence.service.genome.taxon.TaxonService;
import ubic.gemma.persistence.util.TestComponent;
@@ -21,11 +26,12 @@
@Category(SlowTest.class)
@ContextConfiguration
+@TestExecutionListeners(WithSecurityContextTestExecutionListener.class)
public class NCBIGene2GOAssociationLoaderCLITest extends BaseCliTest {
@Configuration
@TestComponent
- static class NCBIGene2GOAssociationLoaderCLITestContextConfiguration extends BaseCliTestContextConfiguration {
+ static class NCBIGene2GOAssociationLoaderCLITestContextConfiguration {
@Bean
public NCBIGene2GOAssociationLoaderCLI ncbiGene2GOAssociationLoaderCLI() {
@@ -46,6 +52,21 @@ public Gene2GOAssociationService gene2GOAssociationService() {
public ExternalDatabaseService externalDatabaseService() {
return mock( ExternalDatabaseService.class );
}
+
+ @Bean
+ public ManualAuthenticationService manualAuthenticationService() {
+ return mock();
+ }
+
+ @Bean
+ public AuditTrailService auditTrailService() {
+ return mock();
+ }
+
+ @Bean
+ public PersisterHelper persisterHelper() {
+ return mock();
+ }
}
@Autowired
diff --git a/gemma-cli/src/test/java/ubic/gemma/core/apps/RNASeqDataAddCliTest.java b/gemma-cli/src/test/java/ubic/gemma/core/apps/RNASeqDataAddCliTest.java
index 641ce7bca9..3628e475c5 100644
--- a/gemma-cli/src/test/java/ubic/gemma/core/apps/RNASeqDataAddCliTest.java
+++ b/gemma-cli/src/test/java/ubic/gemma/core/apps/RNASeqDataAddCliTest.java
@@ -1,5 +1,6 @@
package ubic.gemma.core.apps;
+import gemma.gsec.authentication.ManualAuthenticationService;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -10,7 +11,9 @@
import org.springframework.context.annotation.Scope;
import org.springframework.core.io.ClassPathResource;
import org.springframework.security.test.context.support.WithMockUser;
+import org.springframework.security.test.context.support.WithSecurityContextTestExecutionListener;
import org.springframework.test.context.ContextConfiguration;
+import org.springframework.test.context.TestExecutionListeners;
import ubic.gemma.core.analysis.service.ExpressionDataFileService;
import ubic.gemma.core.genome.gene.service.GeneService;
import ubic.gemma.core.loader.expression.DataUpdater;
@@ -18,6 +21,9 @@
import ubic.gemma.core.util.test.BaseCliTest;
import ubic.gemma.model.expression.arrayDesign.ArrayDesign;
import ubic.gemma.model.expression.experiment.ExpressionExperiment;
+import ubic.gemma.persistence.persister.PersisterHelper;
+import ubic.gemma.persistence.service.common.auditAndSecurity.AuditEventService;
+import ubic.gemma.persistence.service.common.auditAndSecurity.AuditTrailService;
import ubic.gemma.persistence.service.expression.arrayDesign.ArrayDesignService;
import ubic.gemma.persistence.service.expression.experiment.ExpressionExperimentService;
import ubic.gemma.persistence.service.expression.experiment.ExpressionExperimentSetService;
@@ -30,11 +36,12 @@
import static org.mockito.Mockito.*;
@ContextConfiguration
+@TestExecutionListeners(WithSecurityContextTestExecutionListener.class)
public class RNASeqDataAddCliTest extends BaseCliTest {
@Configuration
@TestComponent
- static class RNASeqDataAddCliTestContextConfiguration extends BaseCliTestContextConfiguration {
+ static class RNASeqDataAddCliTestContextConfiguration {
@Bean
@Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE)
@@ -72,10 +79,30 @@ public ArrayDesignService arrayDesignService() {
return mock();
}
+ @Bean
+ public ExpressionExperimentService expressionExperimentService() {
+ return mock();
+ }
+
@Bean
public ExpressionExperimentSetService expressionExperimentSetService() {
return mock();
}
+
+ @Bean
+ public ManualAuthenticationService manualAuthenticationService() {
+ return mock();
+ }
+
+ @Bean
+ public AuditTrailService auditTrailService() {
+ return mock();
+ }
+
+ @Bean
+ public AuditEventService auditEventService() {
+ return mock();
+ }
}
@Autowired
@@ -98,7 +125,7 @@ public ExpressionExperimentSetService expressionExperimentSetService() {
public void setUp() throws IOException {
ad = new ArrayDesign();
ee = new ExpressionExperiment();
- rpkmFile = new ClassPathResource( "test.rpkm.txt" ).getFile().getAbsolutePath();
+ rpkmFile = new ClassPathResource( "ubic/gemma/core/apps/test.rpkm.txt" ).getFile().getAbsolutePath();
when( expressionExperimentService.findByShortName( "GSE000001" ) ).thenReturn( ee );
when( expressionExperimentService.thawLite( any() ) ).thenAnswer( a -> a.getArgument( 0 ) );
when( arrayDesignService.findByShortName( "test" ) ).thenReturn( ad );
diff --git a/gemma-cli/src/test/java/ubic/gemma/core/util/test/BaseCliTest.java b/gemma-cli/src/test/java/ubic/gemma/core/util/test/BaseCliTest.java
index e05c57ed15..4413c44cd1 100644
--- a/gemma-cli/src/test/java/ubic/gemma/core/util/test/BaseCliTest.java
+++ b/gemma-cli/src/test/java/ubic/gemma/core/util/test/BaseCliTest.java
@@ -1,58 +1,13 @@
package ubic.gemma.core.util.test;
-import gemma.gsec.authentication.ManualAuthenticationService;
-import org.junit.Before;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.context.annotation.Bean;
-import org.springframework.security.authentication.AuthenticationManager;
-import org.springframework.security.test.context.support.WithSecurityContextTestExecutionListener;
import org.springframework.test.context.ActiveProfiles;
-import org.springframework.test.context.TestExecutionListeners;
import org.springframework.test.context.junit4.AbstractJUnit4SpringContextTests;
-import ubic.gemma.core.security.authentication.UserManager;
-import ubic.gemma.persistence.persister.Persister;
-import ubic.gemma.persistence.service.common.auditAndSecurity.AuditEventService;
-import ubic.gemma.persistence.service.common.auditAndSecurity.AuditTrailService;
-import ubic.gemma.persistence.service.expression.experiment.ExpressionExperimentService;
import ubic.gemma.persistence.util.SpringProfiles;
-import static org.mockito.Mockito.mock;
-
/**
* Minimal setup
*/
@ActiveProfiles({ "cli", SpringProfiles.TEST })
-@TestExecutionListeners(WithSecurityContextTestExecutionListener.class)
public abstract class BaseCliTest extends AbstractJUnit4SpringContextTests {
- /**
- * Basic context configuration
- */
- public static class BaseCliTestContextConfiguration {
-
- @Bean
- public ManualAuthenticationService manualAuthenticationService() {
- return mock( ManualAuthenticationService.class );
- }
-
- @Bean
- public AuditTrailService auditTrailService() {
- return mock( AuditTrailService.class );
- }
-
- @Bean
- public AuditEventService auditEventService() {
- return mock( AuditEventService.class );
- }
-
- @Bean
- public ExpressionExperimentService expressionExperimentService() {
- return mock( ExpressionExperimentService.class );
- }
-
- @Bean
- public Persister persisterHelper() {
- return mock( Persister.class );
- }
- }
}
diff --git a/gemma-cli/src/test/resources/ubic/gemma/core/apps/factor-value-migration.tsv b/gemma-cli/src/test/resources/ubic/gemma/core/apps/factor-value-migration.tsv
new file mode 100644
index 0000000000..1e6870c771
--- /dev/null
+++ b/gemma-cli/src/test/resources/ubic/gemma/core/apps/factor-value-migration.tsv
@@ -0,0 +1,9 @@
+FactorValueID SubjectID ObjectID SecondObjectID SubjectURI Subject Predicate PredicateURI Object ObjectURI SecondPredicate SecondPredicateURI SecondObject SecondObjectURI Category CategoryURI
+1 1 2 3 Pax6 has_modifier has_modifier "" "" ""
+2 "" "" "" Pax6 "" "" "" Gene ""
+3 4 "" "" "" "" "" "" ""
+4 5 6 "" has_modifier "" "" "" ""
+5 7 8 9 has_modifier has_modifier "" "" ""
+6 10 "" 11 has_dose 5mg has_modifier "" "" ""
+7 "" 12 13 Pax7 has_modifier has_modifier "" genotype ""
+8 14 15 15 has_modifier has_modifier ""
diff --git a/gemma-cli/src/test/resources/test.rpkm.txt b/gemma-cli/src/test/resources/ubic/gemma/core/apps/test.rpkm.txt
similarity index 100%
rename from gemma-cli/src/test/resources/test.rpkm.txt
rename to gemma-cli/src/test/resources/ubic/gemma/core/apps/test.rpkm.txt
diff --git a/gemma-core/pom.xml b/gemma-core/pom.xml
index c0465c3ab8..e79a07adda 100644
--- a/gemma-core/pom.xml
+++ b/gemma-core/pom.xml
@@ -3,7 +3,7 @@
gemmagemma
- 1.30.6
+ 1.31.04.0.0gemma-core
diff --git a/gemma-core/src/main/java/ubic/gemma/core/analysis/expression/coexpression/links/LinkAnalysisServiceImpl.java b/gemma-core/src/main/java/ubic/gemma/core/analysis/expression/coexpression/links/LinkAnalysisServiceImpl.java
index 94b4dd4f09..ac6d01ec82 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/analysis/expression/coexpression/links/LinkAnalysisServiceImpl.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/analysis/expression/coexpression/links/LinkAnalysisServiceImpl.java
@@ -454,16 +454,19 @@ private void qcCheck( LinkAnalysisConfig config, ExpressionExperiment ee ) throw
"No batch information available, out of an abundance of caution we are skipping" );
}
- if ( batchEffect.getPvalue() < 0.001 ) {
+ if ( batchEffect.getBatchEffectStatistics() == null ) {
+ throw new UnsuitableForAnalysisException( ee,
+ "Batch effect statistics are missing, it's not possible to detect a batch effect." );
+ }
- double componentVarianceProportion = batchEffect.getComponentVarianceProportion();
- Integer component = batchEffect.getComponent();
+ if ( batchEffect.getBatchEffectStatistics().getPvalue() < 0.001 ) {
+ double componentVarianceProportion = batchEffect.getBatchEffectStatistics().getComponentVarianceProportion();
+ int component = batchEffect.getBatchEffectStatistics().getComponent();
// don't worry if it is a "minor" component. remember that is must be one of the first few to make it
// this far.
if ( component > 2 && componentVarianceProportion < 0.1 ) {
return;
}
-
throw new UnsuitableForAnalysisException( ee,
String.format( "Strong batch effect detected (%s)", batchEffect ) );
}
diff --git a/gemma-core/src/main/java/ubic/gemma/core/analysis/expression/diff/BaselineSelection.java b/gemma-core/src/main/java/ubic/gemma/core/analysis/expression/diff/BaselineSelection.java
index 0a5789f012..77b9e7e443 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/analysis/expression/diff/BaselineSelection.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/analysis/expression/diff/BaselineSelection.java
@@ -19,12 +19,13 @@
package ubic.gemma.core.analysis.expression.diff;
-import org.apache.commons.lang3.StringUtils;
import ubic.gemma.model.common.description.Characteristic;
import ubic.gemma.model.expression.experiment.FactorValue;
+import ubic.gemma.model.expression.experiment.Statement;
-import java.util.HashSet;
-import java.util.Set;
+import java.util.*;
+
+import static org.apache.commons.lang3.StringUtils.normalizeSpace;
/**
* Utilities for deciding if a factor value is a baseline condition.
@@ -32,129 +33,124 @@
* @author paul
*/
public class BaselineSelection {
- private static final Set controlGroupTerms = new HashSet<>();
- // see bug 4316. This term is "control"
- private static final String FORCED_BASELINE_VALUE_URI = "http://www.ebi.ac.uk/efo/EFO_0001461".toLowerCase();
-
- static {
- /*
- * Values or ontology terms we treat as 'baseline'.
- */
- BaselineSelection.controlGroupTerms
- .add( "http://purl.obolibrary.org/obo/OBI_0000025".toLowerCase() ); // - reference substance
- // role
-
- BaselineSelection.controlGroupTerms
- .add( "http://purl.obolibrary.org/obo/OBI_0000220".toLowerCase() );// - reference subject role
- BaselineSelection.controlGroupTerms
- .add( "http://purl.obolibrary.org/obo/OBI_0000143".toLowerCase() );// - baseline participant
- // role
-
- BaselineSelection.controlGroupTerms.add( "reference_substance_role" );
- BaselineSelection.controlGroupTerms.add( "reference_subject_role" );
- BaselineSelection.controlGroupTerms.add( "baseline_participant_role" );
-
- BaselineSelection.controlGroupTerms.add( "control group" );
- BaselineSelection.controlGroupTerms.add( "control" );
- BaselineSelection.controlGroupTerms.add( "normal" );
- BaselineSelection.controlGroupTerms.add( "untreated" );
- BaselineSelection.controlGroupTerms.add( "baseline" );
- BaselineSelection.controlGroupTerms.add( "control_group" );
- BaselineSelection.controlGroupTerms.add( "wild_type" );
- BaselineSelection.controlGroupTerms.add( "wild type" );
- BaselineSelection.controlGroupTerms.add( "wild type genotype" );
- BaselineSelection.controlGroupTerms.add( "initial time point" );
-
- BaselineSelection.controlGroupTerms.add( "to_be_treated_with_placebo_role" );
-
- BaselineSelection.controlGroupTerms
- .add( "http://purl.obolibrary.org/obo/OBI_0100046".toLowerCase() ); // phosphate buffered
- // saline.
- BaselineSelection.controlGroupTerms
- .add( "http://mged.sourceforge.net/ontologies/MGEDOntology.owl#wild_type".toLowerCase() );
-
- BaselineSelection.controlGroupTerms
- .add( "http://purl.org/nbirn/birnlex/ontology/BIRNLex-Investigation.owl#birnlex_2201"
- .toLowerCase() ); // control_group, old.
- BaselineSelection.controlGroupTerms
- .add( "http://ontology.neuinfo.org/NIF/DigitalEntities/NIF-Investigation.owl#birnlex_2201"
- .toLowerCase() ); // control_group, new version.(retired)
-
- BaselineSelection.controlGroupTerms
- .add( "http://ontology.neuinfo.org/NIF/DigitalEntities/NIF-Investigation.owl#birnlex_2001"
- .toLowerCase() ); // " normal control_group", (retired)
-
- BaselineSelection.controlGroupTerms
- .add( "http://purl.obolibrary.org/obo/OBI_0000825".toLowerCase() ); // - to be treated with
- // placebo
+ // see bug 4316. This term is "control"
+ private static final String FORCED_BASELINE_VALUE_URI = "http://www.ebi.ac.uk/efo/EFO_0001461";
- BaselineSelection.controlGroupTerms
- .add( "http://www.ebi.ac.uk/efo/EFO_0005168".toLowerCase() ); // wild type genotype
+ /**
+ * Values we treat as baseline.
+ */
+ private static final Set controlGroupTerms = createTermSet(
+ "baseline participant role",
+ "baseline",
+ "control diet",
+ "control group",
+ "control",
+ "initial time point",
+ "normal",
+ "placebo",
+ "reference subject role",
+ "reference substance role",
+ "to be treated with placebo role",
+ "untreated",
+ "wild type control",
+ "wild type genotype",
+ "wild type"
+ );
+ /**
+ * Ontology terms we treat as baseline.
+ */
+ private static final Set controlGroupUris = createTermSet(
+ "http://mged.sourceforge.net/ontologies/MGEDOntology.owl#wild_type",
+ "http://ontology.neuinfo.org/NIF/DigitalEntities/NIF-Investigation.owl#birnlex_2001", // normal control_group (retired)
+ "http://ontology.neuinfo.org/NIF/DigitalEntities/NIF-Investigation.owl#birnlex_2201", // control_group, new version (retired)
+ "http://purl.obolibrary.org/obo/OBI_0000025", // reference substance
+ "http://purl.obolibrary.org/obo/OBI_0000143", // baseline participant role
+ "http://purl.obolibrary.org/obo/OBI_0000220", // reference subject role
+ "http://purl.obolibrary.org/obo/OBI_0000825", // to be treated with placebo
+ "http://purl.obolibrary.org/obo/OBI_0100046", // phosphate buffered saline
+ "http://purl.org/nbirn/birnlex/ontology/BIRNLex-Investigation.owl#birnlex_2201", // control group, old
+ "http://www.ebi.ac.uk/efo/EFO_0001461", // control
+ "http://www.ebi.ac.uk/efo/EFO_0001674", // placebo
+ "http://www.ebi.ac.uk/efo/EFO_0004425",// initial time point
+ "http://www.ebi.ac.uk/efo/EFO_0005168" // wild type genotype
+ );
- BaselineSelection.controlGroupTerms
- .add( "http://www.ebi.ac.uk/efo/EFO_0004425".toLowerCase() ); // initial time point
+ /**
+ * Create an immutable, case-insensitive set.
+ */
+ private static Set createTermSet( String... terms ) {
+ Set c = new TreeSet<>( Comparator.nullsLast( String.CASE_INSENSITIVE_ORDER ) );
+ c.addAll( Arrays.asList( terms ) );
+ return Collections.unmodifiableSet( c );
}
+ /**
+ * Check if a given factor value indicates a baseline condition.
+ */
public static boolean isBaselineCondition( FactorValue factorValue ) {
-
- if ( factorValue.getIsBaseline() != null )
- return factorValue.getIsBaseline();
-
- // for backwards compatibility we check anyway
-
if ( factorValue.getMeasurement() != null ) {
return false;
- } else if ( factorValue.getCharacteristics().isEmpty() ) {
- /*
- * Just use the value.
- */
- return StringUtils.isNotBlank( factorValue.getValue() ) && BaselineSelection.controlGroupTerms
- .contains( factorValue.getValue().toLowerCase() );
- } else {
- for ( Characteristic c : factorValue.getCharacteristics() ) {
- if ( isBaselineCondition( c ) )
- return true;
- }
}
- return false;
+ if ( factorValue.getIsBaseline() != null ) {
+ return factorValue.getIsBaseline();
+ }
+ //noinspection deprecation
+ return factorValue.getCharacteristics().stream().anyMatch( BaselineSelection::isBaselineCondition )
+ // for backwards compatibility we check anyway
+ || BaselineSelection.controlGroupTerms.contains( normalizeTerm( factorValue.getValue() ) );
}
/**
- * @param c characteristic
- * @return true if this looks like a baseline condition
+ * Check if a given statement indicates a baseline condition.
+ */
+ public static boolean isBaselineCondition( Statement c ) {
+ return BaselineSelection.controlGroupUris.contains( c.getSubjectUri() )
+ || BaselineSelection.controlGroupUris.contains( c.getObjectUri() )
+ || BaselineSelection.controlGroupUris.contains( c.getSecondObjectUri() )
+ // free text checks
+ || ( c.getSubjectUri() == null && BaselineSelection.controlGroupTerms.contains( normalizeTerm( c.getSubject() ) ) )
+ || ( c.getObjectUri() == null && BaselineSelection.controlGroupTerms.contains( normalizeTerm( c.getObject() ) ) )
+ || ( c.getSecondObjectUri() == null && BaselineSelection.controlGroupTerms.contains( normalizeTerm( c.getSecondObject() ) ) );
+ }
+
+ /**
+ * Check if a given characteristic indicate a baseline condition.
*/
public static boolean isBaselineCondition( Characteristic c ) {
- String valueUri = c.getValueUri();
+ return BaselineSelection.controlGroupUris.contains( c.getValueUri() )
+ || ( c.getValueUri() == null && BaselineSelection.controlGroupTerms.contains( normalizeTerm( c.getValue() ) ) );
+ }
- if ( StringUtils.isNotBlank( valueUri ) && BaselineSelection.controlGroupTerms
- .contains( valueUri.toLowerCase() ) ) {
- return true;
+ private static String normalizeTerm( String term ) {
+ if ( term == null ) {
+ return null;
}
-
- return StringUtils.isNotBlank( c.getValue() ) && BaselineSelection.controlGroupTerms
- .contains( c.getValue().toLowerCase() );
+ return normalizeSpace( term.replace( '_', ' ' ) );
}
/**
* Check if this factor value is the baseline, overriding other possible baselines.
- *
- * @param fv factor value
- * @return true if given fv is forced baseline
+ *
+ * A baseline can be *forced* in two ways: either by setting {@link FactorValue#setIsBaseline(Boolean)} to true or
+ * by adding a characteristic with the {@code FORCED_BASELINE_VALUE_URI} URI. In practice, this is not much
+ * different from {@link #isBaselineCondition(Statement)}, but there might be cases where you would want to indicate
+ * that the baseline was explicitly forced.
*/
public static boolean isForcedBaseline( FactorValue fv ) {
- if ( fv.getMeasurement() != null || fv.getCharacteristics().isEmpty() ) {
+ if ( fv.getMeasurement() != null ) {
return false;
}
- for ( Characteristic c : fv.getCharacteristics() ) {
- String valueUri = c.getValueUri();
- if ( StringUtils.isNotBlank( valueUri ) && valueUri.toLowerCase()
- .equals( BaselineSelection.FORCED_BASELINE_VALUE_URI ) ) {
- return true;
- }
-
+ if ( fv.getIsBaseline() != null ) {
+ return fv.getIsBaseline();
}
- return false;
+ return fv.getCharacteristics().stream().anyMatch( BaselineSelection::isForcedBaseline );
+ }
+
+ private static boolean isForcedBaseline( Statement stmt ) {
+ return BaselineSelection.FORCED_BASELINE_VALUE_URI.equalsIgnoreCase( stmt.getSubjectUri() )
+ || BaselineSelection.FORCED_BASELINE_VALUE_URI.equalsIgnoreCase( stmt.getObjectUri() )
+ || BaselineSelection.FORCED_BASELINE_VALUE_URI.equalsIgnoreCase( stmt.getSecondObjectUri() );
}
}
diff --git a/gemma-core/src/main/java/ubic/gemma/core/analysis/expression/diff/LinearModelAnalyzer.java b/gemma-core/src/main/java/ubic/gemma/core/analysis/expression/diff/LinearModelAnalyzer.java
index ff3d7bce1b..7ccf21a933 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/analysis/expression/diff/LinearModelAnalyzer.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/analysis/expression/diff/LinearModelAnalyzer.java
@@ -471,7 +471,7 @@ public Collection run( ExpressionExperiment expr
*/
ExpressionExperimentSubSet eeSubSet = ExpressionExperimentSubSet.Factory.newInstance();
eeSubSet.setSourceExperiment( expressionExperiment );
- eeSubSet.setName( "Subset for " + subsetFactorValue );
+ eeSubSet.setName( "Subset for " + FactorValueUtils.getSummaryString( subsetFactorValue ) );
Collection bioAssays = new HashSet<>();
for ( BioMaterial bm : bioMaterials ) {
bioAssays.addAll( bm.getBioAssaysUsedIn() );
diff --git a/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/SplitExperimentServiceImpl.java b/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/SplitExperimentServiceImpl.java
index c6b5da277b..9c5916759d 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/SplitExperimentServiceImpl.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/SplitExperimentServiceImpl.java
@@ -40,14 +40,12 @@
import ubic.gemma.model.expression.bioAssayData.RawExpressionDataVector;
import ubic.gemma.model.expression.biomaterial.BioMaterial;
import ubic.gemma.model.expression.biomaterial.Treatment;
-import ubic.gemma.model.expression.experiment.ExperimentalDesign;
-import ubic.gemma.model.expression.experiment.ExperimentalFactor;
-import ubic.gemma.model.expression.experiment.ExpressionExperiment;
-import ubic.gemma.model.expression.experiment.FactorValue;
+import ubic.gemma.model.expression.experiment.*;
import ubic.gemma.persistence.persister.Persister;
import ubic.gemma.persistence.service.expression.bioAssayData.RawExpressionDataVectorService;
import ubic.gemma.persistence.service.expression.experiment.ExpressionExperimentService;
import ubic.gemma.persistence.service.expression.experiment.ExpressionExperimentSetService;
+import ubic.gemma.persistence.service.expression.experiment.FactorValueService;
import java.util.*;
@@ -87,6 +85,9 @@ public class SplitExperimentServiceImpl implements SplitExperimentService {
@Autowired
private ExpressionExperimentSetService expressionExperimentSetService;
+ @Autowired
+ private FactorValueService factorValueService;
+
/*
* (non-Javadoc)
*
@@ -324,7 +325,7 @@ public ExpressionExperimentSet split( ExpressionExperiment toSplit, Experimental
ExpressionExperimentSet g = ExpressionExperimentSet.Factory.newInstance();
g.setDescription( "Parts of " + toSplit.getShortName() + " that were split on " + splitOn.getName() );
g.setName( toSplit.getShortName() + " splits" );
- g.setTaxon( toSplit.getBioAssays().iterator().next().getSampleUsed().getSourceTaxon() );
+ g.setTaxon( toSplit.getTaxon() );
g.getExperiments().addAll( result );
g = this.expressionExperimentSetService.create( g );
@@ -344,7 +345,7 @@ public ExpressionExperimentSet split( ExpressionExperiment toSplit, Experimental
static String generateNameForSplit( ExpressionExperiment toSplit, int splitNumber, FactorValue splitValue ) {
String template = "Split part %d of: %s [%s = %s]";
String originalName = StringUtils.strip( toSplit.getName() );
- String factorValueString = splitValue.getDescriptiveString();
+ String factorValueString = FactorValueUtils.getSummaryString( splitValue );
String newFullName = String.format( template, splitNumber, originalName,
StringUtils.strip( splitValue.getExperimentalFactor().getCategory() != null ?
splitValue.getExperimentalFactor().getCategory().getValue() :
@@ -397,6 +398,7 @@ private Collection cloneExperimentalFactors( Collection result = new HashSet<>();
for ( ExperimentalFactor ef : experimentalFactors ) {
ExperimentalFactor clone = ExperimentalFactor.Factory.newInstance();
+ //noinspection deprecation
clone.setAnnotations( this.cloneCharacteristics( ef.getAnnotations() ) );
if ( ef.getCategory() != null ) {
clone.setCategory( this.cloneCharacteristic( ef.getCategory() ) );
@@ -418,8 +420,9 @@ private Collection cloneFactorValues( Collection facto
Collection result = new HashSet<>();
for ( FactorValue fv : factorValues ) {
FactorValue clone = FactorValue.Factory.newInstance( ef );
- clone.setCharacteristics( this.cloneCharacteristics( fv.getCharacteristics() ) );
+ clone.setCharacteristics( cloneStatements( fv ) );
clone.setIsBaseline( fv.getIsBaseline() );
+ //noinspection deprecation
clone.setValue( fv.getValue() );
clone.setMeasurement( this.cloneMeasurement( fv.getMeasurement() ) );
result.add( clone );
@@ -430,6 +433,37 @@ private Collection cloneFactorValues( Collection facto
return result;
}
+ private Set cloneStatements( FactorValue fv ) {
+ Collection ch = fv.getCharacteristics();
+ // pair of original -> clone
+ List result = new ArrayList<>( ch.size() );
+ for ( Statement s : ch ) {
+ result.add( cloneStatement( s ) );
+ }
+ return new HashSet<>( result );
+ }
+
+ private Statement cloneStatement( Statement s ) {
+ Statement clone = Statement.Factory.newInstance();
+ clone.setName( s.getName() );
+ clone.setDescription( s.getDescription() );
+ clone.setOriginalValue( s.getOriginalValue() );
+ clone.setSubject( s.getSubject() );
+ clone.setSubjectUri( s.getSubjectUri() );
+ clone.setCategory( s.getCategory() );
+ clone.setCategoryUri( s.getCategoryUri() );
+ clone.setEvidenceCode( s.getEvidenceCode() );
+ clone.setPredicate( s.getPredicate() );
+ clone.setPredicateUri( s.getPredicateUri() );
+ clone.setObject( s.getObject() );
+ clone.setObjectUri( s.getObjectUri() );
+ clone.setSecondPredicate( s.getSecondPredicate() );
+ clone.setSecondPredicateUri( s.getSecondPredicateUri() );
+ clone.setSecondObject( s.getSecondObject() );
+ clone.setSecondObjectUri( s.getSecondObjectUri() );
+ return clone;
+ }
+
private Measurement cloneMeasurement( Measurement measurement ) {
if ( measurement == null ) return null;
@@ -451,6 +485,7 @@ private QuantitationType cloneQt( QuantitationType qt, ExpressionExperiment spli
clone.setIsBackground( qt.getIsBackground() );
clone.setIsBackgroundSubtracted( qt.getIsBackgroundSubtracted() );
clone.setIsBatchCorrected( qt.getIsBatchCorrected() );
+ //noinspection deprecation
clone.setIsMaskedPreferred( qt.getIsMaskedPreferred() );
clone.setIsNormalized( qt.getIsNormalized() );
clone.setIsPreferred( qt.getIsPreferred() );
@@ -473,8 +508,16 @@ private Set cloneCharacteristics( Collection ch
}
private Characteristic cloneCharacteristic( Characteristic c ) {
- return Characteristic.Factory.newInstance( c.getName(), c.getDescription(), c.getValue(), c.getValueUri(),
- c.getCategory(), c.getCategoryUri(), c.getEvidenceCode() );
+ Characteristic clone = Characteristic.Factory.newInstance();
+ clone.setName( c.getName() );
+ clone.setDescription( c.getDescription() );
+ clone.setCategory( c.getCategory() );
+ clone.setCategoryUri( c.getCategoryUri() );
+ clone.setValue( c.getValue() );
+ clone.setValueUri( c.getValueUri() );
+ clone.setOriginalValue( c.getOriginalValue() );
+ clone.setEvidenceCode( c.getEvidenceCode() );
+ return clone;
}
private BioAssay cloneBioAssay( BioAssay ba ) {
@@ -500,8 +543,14 @@ private BioAssay cloneBioAssay( BioAssay ba ) {
private DatabaseEntry cloneAccession( DatabaseEntry de ) {
if ( de == null ) return null;
- return DatabaseEntry.Factory.newInstance( de.getAccession(), de.getAccessionVersion(), de.getUri(),
- de.getExternalDatabase() );
+ DatabaseEntry clone = DatabaseEntry.Factory.newInstance();
+ clone.setAccession( de.getAccession() );
+ clone.setAccessionVersion( de.getAccessionVersion() );
+ //noinspection deprecation
+ clone.setUri( de.getUri() );
+ clone.setExternalDatabase( de.getExternalDatabase() );
+ ;
+ return clone;
}
private BioMaterial cloneBioMaterial( BioMaterial bm, BioAssay ba ) {
diff --git a/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/batcheffects/BatchEffectDetails.java b/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/batcheffects/BatchEffectDetails.java
index ea28370c11..f7db994f4a 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/batcheffects/BatchEffectDetails.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/batcheffects/BatchEffectDetails.java
@@ -14,7 +14,11 @@
*/
package ubic.gemma.core.analysis.preprocess.batcheffects;
-import ubic.gemma.model.common.auditAndSecurity.eventType.*;
+import org.springframework.util.Assert;
+import ubic.gemma.model.common.auditAndSecurity.eventType.BatchInformationFetchingEvent;
+import ubic.gemma.model.common.auditAndSecurity.eventType.FailedBatchInformationFetchingEvent;
+import ubic.gemma.model.common.auditAndSecurity.eventType.SingletonBatchInvalidEvent;
+import ubic.gemma.model.common.auditAndSecurity.eventType.UninformativeFASTQHeadersForBatchingEvent;
import javax.annotation.Nullable;
@@ -25,75 +29,97 @@
*/
public class BatchEffectDetails {
- private Integer component = null;
+ public class BatchEffectStatistics {
- private double componentVarianceProportion;
- private final boolean dataWasBatchCorrected;
- private boolean failedToGetBatchInformation = false;
- private Boolean hadSingletonBatches = false;
- private Boolean hadUninformativeHeaders = false;
+ private BatchEffectStatistics() {
+
+ }
+
+ /**
+ * A PCA component that is explained by the batch factor. It is 1-based.
+ */
+ public int getComponent() {
+ return component;
+ }
+
+ /**
+ * The variance explained by the component.
+ */
+ public double getComponentVarianceProportion() {
+ return componentVarianceProportion;
+ }
+
+ /**
+ * A P-value statistic for that component.
+ */
+ public double getPvalue() {
+ return pvalue;
+ }
+ }
+
+ /**
+ * Indicate if the batch information is present.
+ */
private final boolean hasBatchInformation;
- private double pvalue;
+ /**
+ * Indicate if the batch information is uninformative.
+ */
+ private final boolean hasUninformativeBatchInformation;
+ /**
+ * Indicate if the batch information is problematic.
+ */
+ private final boolean hasProblematicBatchInformation;
+ /**
+ * Indicate if the dataset has singleton batches (i.e. a batch only one sample).
+ */
+ private final boolean hasSingletonBatches;
+ /**
+ * Indicate if batch correction was performed on the expression data.
+ */
+ private final boolean dataWasBatchCorrected;
private final boolean singleBatch;
- public BatchEffectDetails( @Nullable BatchInformationFetchingEvent infoEvent, boolean dataWasBatchCorrected, boolean singleBatch ) {
+ /* if present and suitable, those are filled */
+ private boolean hasBatchEffectStatistics = false;
+ private double pvalue;
+ private int component;
+ private double componentVarianceProportion;
- if ( infoEvent == null ) {
- this.hasBatchInformation = false;
+ public BatchEffectDetails( @Nullable BatchInformationFetchingEvent infoEvent, boolean dataWasBatchCorrected, boolean singleBatch ) {
+ this.hasBatchInformation = infoEvent != null;
+ if ( infoEvent != null ) {
+ this.hasProblematicBatchInformation = FailedBatchInformationFetchingEvent.class.isAssignableFrom( ( infoEvent.getClass() ) );
+ this.hasSingletonBatches = SingletonBatchInvalidEvent.class.isAssignableFrom( infoEvent.getClass() );
+ this.hasUninformativeBatchInformation = UninformativeFASTQHeadersForBatchingEvent.class.isAssignableFrom( infoEvent.getClass() );
} else {
- if ( SingletonBatchInvalidEvent.class.isAssignableFrom( infoEvent.getClass() ) ) {
- this.hasBatchInformation = false;
- this.hadSingletonBatches = true;
- } else if ( UninformativeFASTQHeadersForBatchingEvent.class.isAssignableFrom( infoEvent.getClass() ) ) {
- this.hasBatchInformation = false;
- this.hadUninformativeHeaders = true;
- } else if ( FailedBatchInformationMissingEvent.class.isAssignableFrom( infoEvent.getClass() ) ) {
- this.hasBatchInformation = false;
- this.failedToGetBatchInformation = true;
- } else if ( FailedBatchInformationFetchingEvent.class.isAssignableFrom( ( infoEvent.getClass() ) ) ) {
- this.hasBatchInformation = false;
- this.failedToGetBatchInformation = true;
- } else {
- this.hasBatchInformation = true;
- }
+ this.hasProblematicBatchInformation = false;
+ this.hasSingletonBatches = false;
+ this.hasUninformativeBatchInformation = false;
}
-
this.dataWasBatchCorrected = dataWasBatchCorrected;
this.singleBatch = singleBatch;
this.pvalue = 1.0;
}
- public Integer getComponent() {
- return component;
- }
-
- public double getComponentVarianceProportion() {
- return componentVarianceProportion;
- }
-
public boolean getDataWasBatchCorrected() {
return this.dataWasBatchCorrected;
}
- public Boolean getHadSingletonBatches() {
- return hadSingletonBatches;
- }
-
- public Boolean getHadUninformativeHeaders() {
- return hadUninformativeHeaders;
+ public boolean getHasSingletonBatches() {
+ return hasSingletonBatches;
}
- public double getPvalue() {
- return pvalue;
+ public boolean getHasUninformativeBatchInformation() {
+ return hasUninformativeBatchInformation;
}
public boolean hasBatchInformation() {
return hasBatchInformation;
}
- public boolean isFailedToGetBatchInformation() {
- return failedToGetBatchInformation;
+ public boolean hasProblematicBatchInformation() {
+ return hasProblematicBatchInformation;
}
/**
@@ -105,22 +131,33 @@ public boolean isSingleBatch() {
return singleBatch;
}
- public void setComponent( Integer component ) {
- this.component = component;
- }
-
- public void setComponentVarianceProportion( double componentVarianceProportion ) {
- this.componentVarianceProportion = componentVarianceProportion;
+ @Nullable
+ public BatchEffectStatistics getBatchEffectStatistics() {
+ if ( hasBatchEffectStatistics ) {
+ return new BatchEffectStatistics();
+ } else {
+ return null;
+ }
}
- public void setPvalue( double pvalue ) {
- this.pvalue = pvalue;
+ public void setBatchEffectStatistics( double pVal, int i, double variance ) {
+ Assert.isTrue( pVal >= 0 );
+ Assert.isTrue( pVal <= 1 );
+ Assert.isTrue( i >= 1 );
+ Assert.isTrue( variance >= 0 );
+ this.hasBatchEffectStatistics = true;
+ this.pvalue = pVal;
+ this.component = i;
+ this.componentVarianceProportion = variance;
}
@Override
public String toString() {
- return String.format( "BatchEffectDetails [pvalue=%.2g, component=%d, varFraction=%.2f]", pvalue, component,
- componentVarianceProportion );
+ if ( hasBatchEffectStatistics ) {
+ return String.format( "BatchEffectDetails [pvalue=%.2g, component=%d, varFraction=%.2f]", pvalue, component,
+ componentVarianceProportion );
+ } else {
+ return "BatchEffectDetails";
+ }
}
-
}
diff --git a/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/batcheffects/BatchInfoPopulationHelperServiceImpl.java b/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/batcheffects/BatchInfoPopulationHelperServiceImpl.java
index 4593f6c53d..a2ed01beb9 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/batcheffects/BatchInfoPopulationHelperServiceImpl.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/batcheffects/BatchInfoPopulationHelperServiceImpl.java
@@ -907,10 +907,10 @@ private ExperimentalFactor createExperimentalFactor( ExpressionExperiment ee
FactorValue fv = FactorValue.Factory.newInstance();
fv.setIsBaseline( false ); /* we could set true for the first batch, but nobody cares. */
fv.setValue( batchId );
- Set chars = new HashSet<>();
- Characteristic c = Characteristic.Factory.newInstance();
+ Set chars = new HashSet<>();
+ Statement c = Statement.Factory.newInstance();
c.setCategory( ExperimentalDesignUtils.BATCH_FACTOR_CATEGORY_NAME );
- c.setValue( batchId );
+ c.setSubject( batchId );
c.setCategoryUri( ExperimentalDesignUtils.BATCH_FACTOR_CATEGORY_URI );
c.setEvidenceCode( GOEvidenceCode.IIA );
diff --git a/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/batcheffects/BatchInfoPopulationServiceImpl.java b/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/batcheffects/BatchInfoPopulationServiceImpl.java
index da61835bf1..fb16919694 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/batcheffects/BatchInfoPopulationServiceImpl.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/batcheffects/BatchInfoPopulationServiceImpl.java
@@ -82,17 +82,8 @@ public static boolean isBatchFactor( ExperimentalFactor ef ) {
if ( c == null )
return false;
- boolean isBatchFactor = false;
-
- boolean looksLikeBatch = ef.getName().equals( ExperimentalDesignUtils.BATCH_FACTOR_NAME );
-
- if ( c.getCategory() != null && c.getCategory().equals( ExperimentalDesignUtils.BATCH_FACTOR_CATEGORY_NAME ) ) {
- isBatchFactor = true;
- } else if ( looksLikeBatch ) {
- isBatchFactor = true;
- }
-
- return isBatchFactor;
+ return ExperimentalDesignUtils.BATCH_FACTOR_CATEGORY_NAME.equals( c.getCategory() )
+ || ExperimentalDesignUtils.BATCH_FACTOR_NAME.equals( ef.getName() );
}
@Autowired
diff --git a/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/svd/SVDServiceHelperImpl.java b/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/svd/SVDServiceHelperImpl.java
index 30cf04807d..b81392d5b7 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/svd/SVDServiceHelperImpl.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/svd/SVDServiceHelperImpl.java
@@ -20,7 +20,8 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Component;
+import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
import ubic.basecode.dataStructure.matrix.DoubleMatrix;
import ubic.basecode.math.CorrelationStats;
import ubic.basecode.math.Distance;
@@ -54,7 +55,7 @@
* @author paul
* @see PrincipalComponentAnalysisService
*/
-@Component
+@Service
public class SVDServiceHelperImpl implements SVDServiceHelper {
/**
@@ -133,6 +134,7 @@ public static void populateBMFMap( Map> bi
* @return value or null if there isn't one.
*/
@Override
+ @Transactional(readOnly = true)
public SVDValueObject retrieveSvd( ExpressionExperiment ee ) {
PrincipalComponentAnalysis pca = this.principalComponentAnalysisService.loadForExperiment( ee );
if ( pca == null )
@@ -147,6 +149,7 @@ public SVDValueObject retrieveSvd( ExpressionExperiment ee ) {
}
@Override
+ @Transactional
public SVDValueObject svd( ExpressionExperiment ee ) throws SVDException {
assert ee != null;
@@ -177,6 +180,7 @@ public SVDValueObject svd( ExpressionExperiment ee ) throws SVDException {
}
@Override
+ @Transactional(readOnly = true)
public Map getTopLoadedVectors( ExpressionExperiment ee, int component,
int count ) {
PrincipalComponentAnalysis pca = principalComponentAnalysisService.loadForExperiment( ee );
@@ -253,11 +257,13 @@ public Map getTopLoadedVectors( Expressio
}
@Override
+ @Transactional(readOnly = true)
public boolean hasPca( ExpressionExperiment ee ) {
return this.retrieveSvd( ee ) != null;
}
@Override
+ @Transactional(readOnly = true)
public Set getImportantFactors( ExpressionExperiment ee,
Collection experimentalFactors, Double importanceThreshold ) {
Set importantFactors = new HashSet<>();
@@ -295,6 +301,7 @@ public Set getImportantFactors( ExpressionExperiment ee,
}
@Override
+ @Transactional(readOnly = true)
public SVDValueObject svdFactorAnalysis( PrincipalComponentAnalysis pca ) {
BioAssayDimension bad = pca.getBioAssayDimension();
@@ -335,6 +342,7 @@ public SVDValueObject svdFactorAnalysis( PrincipalComponentAnalysis pca ) {
}
@Override
+ @Transactional(readOnly = true)
public SVDValueObject svdFactorAnalysis( ExpressionExperiment ee ) {
PrincipalComponentAnalysis pca = principalComponentAnalysisService.loadForExperiment( ee );
if ( pca == null ) {
@@ -426,7 +434,7 @@ private void analyzeComponent( SVDValueObject svo, int componentNumber, DoubleMa
boolean initializing = false;
if ( !svo.getFactors().containsKey( ef.getId() ) ) {
- svo.getFactors().put( ef.getId(), new ArrayList() );
+ svo.getFactors().put( ef.getId(), new ArrayList<>() );
initializing = true;
}
diff --git a/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/svd/SVDServiceImpl.java b/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/svd/SVDServiceImpl.java
index 913dbf35e2..0716b83bf2 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/svd/SVDServiceImpl.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/svd/SVDServiceImpl.java
@@ -1,13 +1,13 @@
/*
* The Gemma project
- *
+ *
* Copyright (c) 2011 University of British Columbia
- *
+ *
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
@@ -16,9 +16,9 @@
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
import ubic.gemma.model.analysis.expression.pca.ProbeLoading;
import ubic.gemma.model.expression.bioAssayData.DoubleVectorValueObject;
-import ubic.gemma.model.expression.experiment.ExpressionExperiment;
import ubic.gemma.persistence.service.expression.experiment.ExpressionExperimentService;
import java.util.Map;
@@ -41,44 +41,33 @@ public class SVDServiceImpl implements SVDService {
* @return value or null if there isn't one.
*/
@Override
+ @Transactional(readOnly = true)
public SVDValueObject getSvd( Long eeId ) {
- ExpressionExperiment ee = expressionExperimentService.load( eeId );
- return svdServiceHelper.retrieveSvd( ee );
+ return svdServiceHelper.retrieveSvd( expressionExperimentService.loadOrFail( eeId ) );
}
@Override
+ @Transactional(readOnly = true)
public SVDValueObject getSvdFactorAnalysis( Long eeId ) {
-
- ExpressionExperiment ee = expressionExperimentService.load( eeId );
-
- return svdServiceHelper.svdFactorAnalysis( ee );
+ return svdServiceHelper.svdFactorAnalysis( expressionExperimentService.loadOrFail( eeId ) );
}
@Override
+ @Transactional(readOnly = true)
public Map getTopLoadedVectors( Long eeId, int component, int count ) {
-
- ExpressionExperiment ee = expressionExperimentService.load( eeId );
-
- if ( ee == null ) return null;
-
- return svdServiceHelper.getTopLoadedVectors( expressionExperimentService.thawBioAssays( ee ), component, count );
+ return svdServiceHelper.getTopLoadedVectors( expressionExperimentService.thawBioAssays( expressionExperimentService.loadOrFail( eeId ) ), component, count );
}
@Override
+ @Transactional(readOnly = true)
public boolean hasPca( Long eeId ) {
- ExpressionExperiment ee = expressionExperimentService.load( eeId );
-
- return svdServiceHelper.hasPca( ee );
-
+ return svdServiceHelper.hasPca( expressionExperimentService.loadOrFail( eeId ) );
}
@Override
+ @Transactional
public SVDValueObject svd( Long eeId ) throws SVDException {
-
- ExpressionExperiment ee = expressionExperimentService.load( eeId );
-
- return svdServiceHelper.svd( ee );
+ return svdServiceHelper.svd( expressionExperimentService.loadOrFail( eeId ) );
}
-
}
diff --git a/gemma-core/src/main/java/ubic/gemma/core/analysis/report/DatabaseViewGeneratorImpl.java b/gemma-core/src/main/java/ubic/gemma/core/analysis/report/DatabaseViewGeneratorImpl.java
index aa6de7750e..f98c5769f5 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/analysis/report/DatabaseViewGeneratorImpl.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/analysis/report/DatabaseViewGeneratorImpl.java
@@ -35,6 +35,7 @@
import ubic.gemma.model.expression.experiment.ExperimentalFactor;
import ubic.gemma.model.expression.experiment.ExpressionExperiment;
import ubic.gemma.model.expression.experiment.FactorValue;
+import ubic.gemma.model.expression.experiment.FactorValueUtils;
import ubic.gemma.model.genome.Gene;
import ubic.gemma.model.genome.Taxon;
import ubic.gemma.persistence.service.analysis.expression.diff.DifferentialExpressionAnalysisService;
@@ -119,7 +120,7 @@ private void generateDatasetTissueView( Integer limit, Collection";
if ( !Objects.equals( confound, ee.getBatchConfound() ) ) {
ee.setBatchConfound( confound );
auditTrailService.addUpdateEvent( ee, BatchProblemsUpdateEvent.class,
- ExpressionExperimentReportServiceImpl.NOTE_UPDATED_CONFOUND, confound != null ? confound : "" );
- log.info( "New batch confound for " + ee + ": " + ( confound != null ? confound : "" ) );
+ ExpressionExperimentReportServiceImpl.NOTE_UPDATED_CONFOUND, confoundSummary );
+ log.info( "New batch confound for " + ee + ": " + confoundSummary );
}
- if ( !Objects.equals( effect, ee.getBatchEffect() ) ) {
- auditTrailService.addUpdateEvent( ee, BatchProblemsUpdateEvent.class,
- ExpressionExperimentReportServiceImpl.NOTE_UPDATED_EFFECT, effect );
+ if ( !Objects.equals( effect, ee.getBatchEffect() ) || !Objects.equals( effectStatistics, ee.getBatchEffectStatistics() ) ) {
ee.setBatchEffect( effect );
- log.info( "New batch effect for " + ee + ": " + effect );
+ ee.setBatchEffectStatistics( effectStatistics );
+ auditTrailService.addUpdateEvent( ee, BatchProblemsUpdateEvent.class,
+ ExpressionExperimentReportServiceImpl.NOTE_UPDATED_EFFECT, effectSummary );
+ log.info( "New batch effect for " + ee + ": " + effectSummary );
}
}
diff --git a/gemma-core/src/main/java/ubic/gemma/core/analysis/service/ExpressionAnalysisResultSetFileServiceImpl.java b/gemma-core/src/main/java/ubic/gemma/core/analysis/service/ExpressionAnalysisResultSetFileServiceImpl.java
index 5a4cff238d..48e6c79c49 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/analysis/service/ExpressionAnalysisResultSetFileServiceImpl.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/analysis/service/ExpressionAnalysisResultSetFileServiceImpl.java
@@ -118,7 +118,7 @@ private String formatMeasurement( Measurement measurement ) {
+ ( measurement.getUnit() != null ? measurement.getUnit().getUnitNameCV() : "" );
}
- private String formatCharacteristics( Collection characteristics ) {
+ private String formatCharacteristics( Collection characteristics ) {
return characteristics.stream().map( Characteristic::getValue ).collect( Collectors.joining( ", " ) );
}
diff --git a/gemma-core/src/main/java/ubic/gemma/core/analysis/util/ExperimentalDesignUtils.java b/gemma-core/src/main/java/ubic/gemma/core/analysis/util/ExperimentalDesignUtils.java
index ed5fe0296a..c533fc5e75 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/analysis/util/ExperimentalDesignUtils.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/analysis/util/ExperimentalDesignUtils.java
@@ -22,10 +22,7 @@
import ubic.gemma.model.common.description.Characteristic;
import ubic.gemma.model.common.measurement.Measurement;
import ubic.gemma.model.expression.biomaterial.BioMaterial;
-import ubic.gemma.model.expression.experiment.ExperimentalFactor;
-import ubic.gemma.model.expression.experiment.ExperimentalFactorValueObject;
-import ubic.gemma.model.expression.experiment.FactorType;
-import ubic.gemma.model.expression.experiment.FactorValue;
+import ubic.gemma.model.expression.experiment.*;
import ubic.gemma.persistence.service.expression.experiment.ExperimentalFactorService;
import java.util.*;
@@ -233,23 +230,6 @@ public static String nameForR( FactorValue fv, boolean isBaseline ) {
return ExperimentalDesignUtils.FACTOR_VALUE_RNAME_PREFIX + fv.getId() + ( isBaseline ? "_base" : "" );
}
- public static String prettyString( FactorValue fv ) {
-
- if ( fv.getMeasurement() != null ) {
- return fv.getMeasurement().getValue();
- } else if ( fv.getCharacteristics().isEmpty() ) {
- return fv.getValue();
- }
- StringBuilder buf = new StringBuilder();
- for ( Characteristic c : fv.getCharacteristics() ) {
- buf.append( c.getValue() );
- if ( fv.getCharacteristics().size() > 1 )
- buf.append( " | " );
- }
- return buf.toString();
-
- }
-
/**
* @param factors factors
* @param baselines baselines
diff --git a/gemma-core/src/main/java/ubic/gemma/core/association/phenotype/PhenotypeAssoOntologyHelper.java b/gemma-core/src/main/java/ubic/gemma/core/association/phenotype/PhenotypeAssoOntologyHelper.java
index ae8c8077c0..88e47eb82b 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/association/phenotype/PhenotypeAssoOntologyHelper.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/association/phenotype/PhenotypeAssoOntologyHelper.java
@@ -18,7 +18,7 @@
import ubic.basecode.ontology.providers.OntologyService;
import ubic.gemma.core.search.SearchException;
import ubic.gemma.model.common.description.Characteristic;
-import ubic.gemma.model.genome.gene.phenotype.valueObject.CharacteristicValueObject;
+import ubic.gemma.model.common.description.CharacteristicValueObject;
import javax.annotation.Nullable;
import java.util.Collection;
diff --git a/gemma-core/src/main/java/ubic/gemma/core/association/phenotype/PhenotypeAssoOntologyHelperImpl.java b/gemma-core/src/main/java/ubic/gemma/core/association/phenotype/PhenotypeAssoOntologyHelperImpl.java
index a2e138754f..4f7409ac43 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/association/phenotype/PhenotypeAssoOntologyHelperImpl.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/association/phenotype/PhenotypeAssoOntologyHelperImpl.java
@@ -28,7 +28,7 @@
import ubic.gemma.core.search.BaseCodeOntologySearchException;
import ubic.gemma.core.search.SearchException;
import ubic.gemma.model.common.description.Characteristic;
-import ubic.gemma.model.genome.gene.phenotype.valueObject.CharacteristicValueObject;
+import ubic.gemma.model.common.description.CharacteristicValueObject;
import java.util.*;
@@ -169,13 +169,9 @@ public Characteristic valueUri2Characteristic( String valueUri ) {
*/
private Set ontology2CharacteristicValueObject(
Collection ontologyTerms ) {
-
Set characteristicsVO = new HashSet<>();
-
for ( OntologyTerm ontologyTerm : ontologyTerms ) {
- CharacteristicValueObject phenotype = new CharacteristicValueObject( -1L,
- ontologyTerm.getLabel().toLowerCase(), ontologyTerm.getUri() );
- characteristicsVO.add( phenotype );
+ characteristicsVO.add( new CharacteristicValueObject( ontologyTerm.getLabel().toLowerCase(), ontologyTerm.getUri() ) );
}
return characteristicsVO;
}
diff --git a/gemma-core/src/main/java/ubic/gemma/core/association/phenotype/PhenotypeAssociationManagerService.java b/gemma-core/src/main/java/ubic/gemma/core/association/phenotype/PhenotypeAssociationManagerService.java
index 492228129e..046abd61c2 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/association/phenotype/PhenotypeAssociationManagerService.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/association/phenotype/PhenotypeAssociationManagerService.java
@@ -19,6 +19,7 @@
import ubic.gemma.core.search.SearchException;
import ubic.gemma.model.association.phenotype.PhenotypeAssociation;
import ubic.gemma.model.common.description.BibliographicReferenceValueObject;
+import ubic.gemma.model.common.description.CharacteristicValueObject;
import ubic.gemma.model.common.description.ExternalDatabaseValueObject;
import ubic.gemma.model.genome.Taxon;
import ubic.gemma.model.genome.gene.GeneValueObject;
diff --git a/gemma-core/src/main/java/ubic/gemma/core/association/phenotype/PhenotypeAssociationManagerServiceImpl.java b/gemma-core/src/main/java/ubic/gemma/core/association/phenotype/PhenotypeAssociationManagerServiceImpl.java
index 2aeb9676f2..3bcec1822a 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/association/phenotype/PhenotypeAssociationManagerServiceImpl.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/association/phenotype/PhenotypeAssociationManagerServiceImpl.java
@@ -769,7 +769,7 @@ public Collection searchInDatabaseForPhenotype( Strin
}
return ontologyTermsFound.stream()
- .map( t -> new CharacteristicValueObject( -1L, t.getLabel().toLowerCase(), t.getUri() ) )
+ .map( t -> new CharacteristicValueObject( t.getLabel().toLowerCase(), t.getUri() ) )
.limit( maxResults > 0 ? maxResults : Long.MAX_VALUE )
.collect( Collectors.toCollection( TreeSet::new ) );
}
@@ -824,7 +824,7 @@ public Collection searchOntologyForPhenotypes( String
.loadAllNeurocartaPhenotypes();
for ( PhenotypeValueObject pvo : allNeurocartaPhenotypes ) {
- CharacteristicValueObject cha = new CharacteristicValueObject( -1L, pvo.getValue(), pvo.getValueUri() );
+ CharacteristicValueObject cha = new CharacteristicValueObject( pvo.getValue(), pvo.getValueUri() );
// set flag for UI, flag if the phenotype is on the Gene or if in the database
cha.setAlreadyPresentOnGene( true );
cha.setAlreadyPresentInDatabase( true );
@@ -1752,7 +1752,7 @@ private TreeCharacteristicValueObject ontology2TreeCharacteristicValueObjects( O
}
}
- return new TreeCharacteristicValueObject( -1L, ontologyTerm.getLabel(), ontologyTerm.getUri(), children );
+ return new TreeCharacteristicValueObject( ontologyTerm.getLabel(), ontologyTerm.getUri(), children );
}
private void countPrivateGeneForEachNode( TreeCharacteristicValueObject tc, Map> phenotypesGenesAssociations, Set visited ) {
@@ -2021,7 +2021,7 @@ private void findParentRoot( TreeCharacteristicValueObject tc,
if ( alreadyOnTree != null ) {
alreadyOnTree.getChildren().add( tc );
} else {
- TreeCharacteristicValueObject tree = new TreeCharacteristicValueObject( -1L, onTerm.getLabel(),
+ TreeCharacteristicValueObject tree = new TreeCharacteristicValueObject( onTerm.getLabel(),
onTerm.getUri() );
// add children to the parent
diff --git a/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/BaseExpressionDataMatrix.java b/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/BaseExpressionDataMatrix.java
index 053080ec94..d189998f2e 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/BaseExpressionDataMatrix.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/BaseExpressionDataMatrix.java
@@ -258,7 +258,7 @@ void addToRowMaps( Integer row, CompositeSequence designElement ) {
* For example, in the following diagram "-" indicates a biomaterial, while "*" indicates a bioassay. Each row of
* "*" indicates samples run on a different microarray design (a different bio assay material). In the examples we
* assume there is just a single biomaterial dimension.
- *
+ *
*
* ---------------
* ***** -- only a few samples run on this platform
@@ -268,7 +268,7 @@ void addToRowMaps( Integer row, CompositeSequence designElement ) {
*
* Every sample was run on a different array design:
- *
+ *
*
* -----------------------
* ******
@@ -384,7 +384,9 @@ void selectVectors( Collection vectors ) {
this.getQuantitationTypes().add( vectorQuantitationType );
} else {
if ( quantitationType != vectorQuantitationType ) {
- throw new IllegalArgumentException( "Cannot pass vectors from more than one quantitation type" );
+ throw new IllegalArgumentException( "Cannot pass vectors from more than one quantitation type: " +
+ vectorQuantitationType + " vs "
+ + quantitationType );
}
}
diff --git a/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/ExperimentalDesignWriter.java b/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/ExperimentalDesignWriter.java
index 55f9f1bdb6..80b3d8afa1 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/ExperimentalDesignWriter.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/ExperimentalDesignWriter.java
@@ -141,7 +141,7 @@ private void writeHeader( ExpressionExperiment expressionExperiment, Collection<
for ( ExperimentalFactor ef : factors ) {
buf.append( ExperimentalDesignImporterImpl.EXPERIMENTAL_FACTOR_DESCRIPTION_LINE_INDICATOR );
- buf.append( ef.getName().replaceAll("\\s", ".") ).append( " :" );
+ buf.append( ef.getName().replaceAll( "\\s", "." ) ).append( " :" );
if ( ef.getCategory() != null ) {
buf.append( " Category=" ).append( ef.getCategory().getValue().replaceAll( "\\s", "_" ) );
}
diff --git a/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/ExpressionDataMatrixColumnSort.java b/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/ExpressionDataMatrixColumnSort.java
index c919ac3642..2333fad2f5 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/ExpressionDataMatrixColumnSort.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/ExpressionDataMatrixColumnSort.java
@@ -118,7 +118,7 @@ public static Map getBaselineLevels( List getBaselineLevels( List getFactors( Collection matrix, int
}
/**
+ * Produce a value for representing a factor value.
+ *
+ * In the context of the design file, this is focusing on the value (i.e. subjects or measurement value) itself and
+ * not its metadata which are instead exposed in the file header.
+ *
* Replaces spaces and hyphens with underscores.
- *
* @param factorValue FV
* @return replaced string
*/
public static String constructFactorValueName( FactorValue factorValue ) {
-
- StringBuilder buf = new StringBuilder();
-
- if ( factorValue.getCharacteristics().size() > 0 ) {
- for ( Characteristic c : factorValue.getCharacteristics() ) {
- buf.append( StringUtils.strip( c.getValue() ) );
- if ( factorValue.getCharacteristics().size() > 1 )
- buf.append( " | " );
+ String v;
+ if ( factorValue.getMeasurement() != null ) {
+ v = factorValue.getMeasurement().getValue();
+ } else {
+ String valueFromStatements = factorValue.getCharacteristics().stream()
+ .map( Statement::getSubject )
+ .collect( Collectors.joining( " | " ) );
+ if ( StringUtils.isNotBlank( valueFromStatements ) ) {
+ v = valueFromStatements;
+ } else if ( StringUtils.isNotBlank( factorValue.getValue() ) ) {
+ v = factorValue.getValue();
+ } else {
+ v = ""; // this is treated as NaN in most scenarios
}
- } else if ( factorValue.getMeasurement() != null ) {
- buf.append( factorValue.getMeasurement().getValue() );
- } else if ( StringUtils.isNotBlank( factorValue.getValue() ) ) {
- buf.append( StringUtils.strip( factorValue.getValue() ) );
}
-
- String matchedFactorValue = buf.toString();
-
- matchedFactorValue = matchedFactorValue.trim();
- matchedFactorValue = matchedFactorValue.replaceAll( "-", "_" );
- matchedFactorValue = matchedFactorValue.replaceAll( "\\s", "_" );
- return matchedFactorValue;
+ return v.replace( '-', '_' )
+ .replaceAll( "\\s+", "_" );
}
/**
diff --git a/gemma-core/src/main/java/ubic/gemma/core/expression/experiment/service/ExpressionExperimentSearchServiceImpl.java b/gemma-core/src/main/java/ubic/gemma/core/expression/experiment/service/ExpressionExperimentSearchServiceImpl.java
index 2cff9f473b..6866a04e8e 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/expression/experiment/service/ExpressionExperimentSearchServiceImpl.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/expression/experiment/service/ExpressionExperimentSearchServiceImpl.java
@@ -18,8 +18,8 @@
*/
package ubic.gemma.core.expression.experiment.service;
-import com.google.common.collect.Sets;
import gemma.gsec.SecurityService;
+import org.apache.commons.collections4.SetUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -116,7 +116,7 @@ public Collection searchExpressionExperiments(
}
current = new HashSet<>( this.searchExpressionExperiments( s ) );
- all = Sets.intersection( all, current );
+ all = SetUtils.intersection( all, current );
}
return all;
}
diff --git a/gemma-core/src/main/java/ubic/gemma/core/genome/gene/service/GeneServiceImpl.java b/gemma-core/src/main/java/ubic/gemma/core/genome/gene/service/GeneServiceImpl.java
index 0b21f1338d..6d02a65461 100755
--- a/gemma-core/src/main/java/ubic/gemma/core/genome/gene/service/GeneServiceImpl.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/genome/gene/service/GeneServiceImpl.java
@@ -44,7 +44,7 @@
import ubic.gemma.model.genome.PhysicalLocationValueObject;
import ubic.gemma.model.genome.Taxon;
import ubic.gemma.model.genome.gene.*;
-import ubic.gemma.model.genome.gene.phenotype.valueObject.CharacteristicValueObject;
+import ubic.gemma.model.common.description.CharacteristicValueObject;
import ubic.gemma.persistence.service.AbstractFilteringVoEnabledService;
import ubic.gemma.persistence.service.AbstractService;
import ubic.gemma.persistence.service.association.Gene2GOAssociationService;
@@ -307,6 +307,7 @@ public GeneValueObject loadFullyPopulatedValueObject( Long id ) {
GeneValueObject gvo = GeneValueObject.convert2ValueObject( gene );
+ // FIXME: this is redundant as aliases are setup by the converter
Collection aliasObjects = gene.getAliases();
SortedSet aliasStrings = new TreeSet<>();
for ( GeneAlias ga : aliasObjects ) {
diff --git a/gemma-core/src/main/java/ubic/gemma/core/job/executor/common/TaskPostProcessing.java b/gemma-core/src/main/java/ubic/gemma/core/job/executor/common/TaskPostProcessing.java
index ffa0301dea..8bf906a887 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/job/executor/common/TaskPostProcessing.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/job/executor/common/TaskPostProcessing.java
@@ -18,10 +18,10 @@
*/
package ubic.gemma.core.job.executor.common;
-import com.google.common.util.concurrent.ListenableFuture;
import ubic.gemma.core.job.EmailNotificationContext;
import ubic.gemma.core.job.TaskResult;
+import java.util.concurrent.CompletableFuture;
import java.util.concurrent.Executor;
/**
@@ -29,5 +29,5 @@
* date: 10/02/13
*/
public interface TaskPostProcessing {
- void addEmailNotification( ListenableFuture future, EmailNotificationContext context, Executor executor );
+ void addEmailNotification( CompletableFuture future, EmailNotificationContext context, Executor executor );
}
diff --git a/gemma-core/src/main/java/ubic/gemma/core/job/executor/common/TaskPostProcessingImpl.java b/gemma-core/src/main/java/ubic/gemma/core/job/executor/common/TaskPostProcessingImpl.java
index 152ede054d..f7d399c9c6 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/job/executor/common/TaskPostProcessingImpl.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/job/executor/common/TaskPostProcessingImpl.java
@@ -18,9 +18,6 @@
*/
package ubic.gemma.core.job.executor.common;
-import com.google.common.util.concurrent.FutureCallback;
-import com.google.common.util.concurrent.Futures;
-import com.google.common.util.concurrent.ListenableFuture;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.annotation.Autowired;
@@ -29,6 +26,7 @@
import ubic.gemma.core.job.TaskResult;
import ubic.gemma.core.util.MailUtils;
+import java.util.concurrent.CompletableFuture;
import java.util.concurrent.Executor;
/**
@@ -42,29 +40,13 @@ public class TaskPostProcessingImpl implements TaskPostProcessing {
private MailUtils mailUtils;
@Override
- public void addEmailNotification( ListenableFuture future, EmailNotificationContext context, Executor executor ) {
- FutureCallback emailNotificationCallback = this.createEmailNotificationFutureCallback( context );
- // This will be called when future with our running task is done.
- Futures.addCallback( future, emailNotificationCallback, executor );
+ public void addEmailNotification( CompletableFuture future, EmailNotificationContext context, Executor executor ) {
+ future.thenAcceptAsync( taskResult -> {
+ // This will be called when future with our running task is done.
+ mailUtils.sendTaskCompletedNotificationEmail( context, taskResult );
+ }, executor ).exceptionally( throwable -> {
+ TaskPostProcessingImpl.log.error( "Shouldn't happen since we take care of exceptions inside ExecutingTask. ", throwable );
+ return null;
+ } );
}
-
- private FutureCallback createEmailNotificationFutureCallback( final EmailNotificationContext context ) {
-
- return new FutureCallback() {
- private final EmailNotificationContext emailNotificationContext = context;
-
- @Override
- public void onSuccess( TaskResult taskResult ) {
- mailUtils.sendTaskCompletedNotificationEmail( emailNotificationContext, taskResult );
- }
-
- @Override
- public void onFailure( Throwable throwable ) {
- TaskPostProcessingImpl.log
- .error( "Shouldn't happen since we take care of exceptions inside ExecutingTask. " + throwable
- .getMessage() );
- }
- };
- }
-
}
diff --git a/gemma-core/src/main/java/ubic/gemma/core/job/executor/webapp/SubmittedTaskLocal.java b/gemma-core/src/main/java/ubic/gemma/core/job/executor/webapp/SubmittedTaskLocal.java
index 48e446de21..5648815c7a 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/job/executor/webapp/SubmittedTaskLocal.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/job/executor/webapp/SubmittedTaskLocal.java
@@ -18,7 +18,6 @@
*/
package ubic.gemma.core.job.executor.webapp;
-import com.google.common.util.concurrent.ListenableFuture;
import ubic.gemma.core.job.EmailNotificationContext;
import ubic.gemma.core.job.TaskCommand;
import ubic.gemma.core.job.TaskResult;
@@ -27,9 +26,7 @@
import java.util.Date;
import java.util.Deque;
import java.util.Queue;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Executor;
-import java.util.concurrent.LinkedBlockingDeque;
+import java.util.concurrent.*;
/**
* SubmittedTask implementation representing the task running on local TaskRunningService.
@@ -39,7 +36,7 @@ public class SubmittedTaskLocal extends SubmittedTaskAbstract {
private final TaskPostProcessing taskPostProcessing;
private final Deque progressUpdates = new LinkedBlockingDeque<>();
private final Executor executor;
- private ListenableFuture future;
+ private CompletableFuture future;
public SubmittedTaskLocal( TaskCommand taskCommand, TaskPostProcessing taskPostProcessing, Executor executor ) {
super( taskCommand );
@@ -91,14 +88,13 @@ public synchronized void requestCancellation() {
}
}
- @SuppressWarnings("unchecked")
@Override
public synchronized void addEmailAlert() {
if ( emailAlert )
return;
emailAlert = true;
assert taskPostProcessing != null : "Task postprocessing was null";
- taskPostProcessing.addEmailNotification( ( ListenableFuture ) future,
+ taskPostProcessing.addEmailNotification( future,
new EmailNotificationContext( taskCommand.getTaskId(), taskCommand.getSubmitter(),
taskCommand.getTaskClass().getSimpleName() ), executor );
}
@@ -123,14 +119,14 @@ public synchronized boolean isDone() {
@SuppressWarnings("unused")
// Possible external use
- ListenableFuture getFuture() {
+ CompletableFuture getFuture() {
return future;
}
/*
* Package-private methods, used by TaskRunningService
*/
- void setFuture( ListenableFuture future ) {
+ void setFuture( CompletableFuture future ) {
this.future = future;
}
diff --git a/gemma-core/src/main/java/ubic/gemma/core/job/executor/webapp/TaskRunningServiceImpl.java b/gemma-core/src/main/java/ubic/gemma/core/job/executor/webapp/TaskRunningServiceImpl.java
index db4153de5c..c865b22f7f 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/job/executor/webapp/TaskRunningServiceImpl.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/job/executor/webapp/TaskRunningServiceImpl.java
@@ -18,14 +18,12 @@
*/
package ubic.gemma.core.job.executor.webapp;
-import com.google.common.util.concurrent.ListenableFuture;
-import com.google.common.util.concurrent.ListeningExecutorService;
-import com.google.common.util.concurrent.MoreExecutors;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.security.concurrent.DelegatingSecurityContextCallable;
import org.springframework.stereotype.Component;
+import org.springframework.util.Assert;
import ubic.gemma.core.job.SubmittedTask;
import ubic.gemma.core.job.TaskCommand;
import ubic.gemma.core.job.TaskResult;
@@ -37,10 +35,7 @@
import java.util.Collection;
import java.util.Date;
import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.Executors;
-
-import static com.google.common.base.Preconditions.checkNotNull;
+import java.util.concurrent.*;
/**
* Handles the execution of tasks in threads that can be checked by clients later.
@@ -51,8 +46,7 @@
@Component
public class TaskRunningServiceImpl implements TaskRunningService {
private static final Log log = LogFactory.getLog( TaskRunningServiceImpl.class );
- private final ListeningExecutorService executorService = MoreExecutors
- .listeningDecorator( Executors.newFixedThreadPool( 20 ) );
+ private final ExecutorService executorService = Executors.newFixedThreadPool( 20 );
private final Map submittedTasks = new ConcurrentHashMap<>();
@Autowired
@@ -116,7 +110,14 @@ public void onProgress( String message ) {
}
} );
- ListenableFuture future = executorService.submit( new DelegatingSecurityContextCallable<>( executingTask, taskCommand.getSecurityContext() ) );
+ Callable callable = new DelegatingSecurityContextCallable<>( executingTask, taskCommand.getSecurityContext() );
+ CompletableFuture future = CompletableFuture.supplyAsync( () -> {
+ try {
+ return callable.call();
+ } catch ( Exception e ) {
+ throw new RuntimeException( e );
+ }
+ }, executorService );
submittedTask.setFuture( future );
// Adding post-processing steps, they will run on future completion.
@@ -140,10 +141,10 @@ public String submitTaskCommand( final C taskCommand ) {
}
private void checkTask( Task task ) {
- checkNotNull( task, "Must provide a task." );
+ Assert.notNull( task, "Must provide a task." );
}
private void checkTaskCommand( TaskCommand taskCommand ) {
- checkNotNull( taskCommand.getTaskId(), "Must have taskId." );
+ Assert.notNull( taskCommand.getTaskId(), "Must have taskId." );
}
}
diff --git a/gemma-core/src/main/java/ubic/gemma/core/loader/expression/geo/GeoConverter.java b/gemma-core/src/main/java/ubic/gemma/core/loader/expression/geo/GeoConverter.java
index fffaef2d93..3da296754c 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/loader/expression/geo/GeoConverter.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/loader/expression/geo/GeoConverter.java
@@ -38,9 +38,13 @@ public interface GeoConverter extends Converter {
@Override
Collection