diff --git a/.idea/runConfigurations/Gemma_CLI.xml b/.idea/runConfigurations/Gemma_CLI.xml
index cea41c8a33..c326b99331 100644
--- a/.idea/runConfigurations/Gemma_CLI.xml
+++ b/.idea/runConfigurations/Gemma_CLI.xml
@@ -2,7 +2,7 @@
   <configuration default="false" name="Gemma CLI" type="Application" factoryName="Application">
     <option name="MAIN_CLASS_NAME" value="ubic.gemma.core.apps.GemmaCLI" />
     <module name="gemma-cli" />
-    <option name="VM_PARAMETERS" value="-Dspring.profiles.active=dev -Dlog4j.configuration=$PROJECT_DIR$/gemma-cli/src/main/config/log4j.properties" />
+    <option name="VM_PARAMETERS" value="-Dspring.profiles.active=dev -Dlog4j.configuration=$PROJECT_DIR$/gemma-cli/src/main/config/log4j-dev.properties" />
     <method v="2">
       <option name="Make" enabled="true" />
     </method>
diff --git a/.idea/runConfigurations/Gemma_Web.xml b/.idea/runConfigurations/Gemma_Web.xml
index 7231e4b6d5..0b2f9f6d18 100644
--- a/.idea/runConfigurations/Gemma_Web.xml
+++ b/.idea/runConfigurations/Gemma_Web.xml
@@ -1,5 +1,5 @@
 <component name="ProjectRunConfigurationManager">
-  <configuration default="false" name="Gemma Web" type="#com.intellij.j2ee.web.tomcat.TomcatRunConfigurationFactory" factoryName="Local" APPLICATION_SERVER_NAME="Tomcat 8.5.83" ALTERNATIVE_JRE_ENABLED="false">
+  <configuration default="false" name="Gemma Web" type="#com.intellij.j2ee.web.tomcat.TomcatRunConfigurationFactory" factoryName="Local" APPLICATION_SERVER_NAME="Tomcat 9.0.80" ALTERNATIVE_JRE_ENABLED="false">
     <option name="COMMON_VM_ARGUMENTS" value="-Dspring.profiles.active=dev -Dlog4j.configuration=$PROJECT_DIR$/gemma-web/src/main/config/log4j-dev.properties -Dcom.mysql.cj.disableAbandonedConnectionCleanup=true" />
     <option name="UPDATE_ON_FRAME_DEACTIVATION" value="true" />
     <option name="UPDATING_POLICY" value="redeploy-artifacts" />
diff --git a/gemma-cli/pom.xml b/gemma-cli/pom.xml
index 816ba5e589..74bb6e5747 100644
--- a/gemma-cli/pom.xml
+++ b/gemma-cli/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <artifactId>gemma</artifactId>
         <groupId>gemma</groupId>
-        <version>1.30.3</version>
+        <version>1.30.4</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
     <artifactId>gemma-cli</artifactId>
diff --git a/gemma-cli/src/main/config/log4j-dev.properties b/gemma-cli/src/main/config/log4j-dev.properties
new file mode 100644
index 0000000000..9277ae3932
--- /dev/null
+++ b/gemma-cli/src/main/config/log4j-dev.properties
@@ -0,0 +1,62 @@
+# This is the configuration for logging displayed in the CLIs
+# If you are developing Gemma from an IDE, consider setting -Dlog4j.configuration=file:gemma-core/src/config/log4j.properties
+# to pickup this file, or even better: define your own.
+
+#################################################################################################
+# APPENDERS
+#################################################################################################
+
+# Basic appender to log to the console.
+log4j.appender.stderr=org.apache.log4j.ConsoleAppender
+log4j.appender.stderr.target=System.err
+log4j.appender.stderr.layout=org.apache.log4j.PatternLayout
+log4j.appender.stderr.layout.ConversionPattern=%d %highlight{%p} %style{%pid}{magenta} [%t] %style{%C{1.}.%M(%L)}{cyan} | %m%n
+
+#####################################################################################################
+# LOGGING LEVELS
+#####################################################################################################
+
+# root
+log4j.rootLogger=INFO,stderr
+
+# baseCode
+log4j.logger.ubic.basecode=INFO
+log4j.logger.ubic.basecode.ontology.model.PropertyFactory=ERROR
+log4j.logger.ubic.basecode.ontology.providers=WARN
+
+# Gemma
+log4j.logger.ubic.gemma=INFO
+
+# Gemma (externally contributed components)
+log4j.logger.ubic.gemma.contrib=INFO
+
+############# THIRD PARTY CLASSES #####################
+# Control logging for other packages
+log4j.logger.org=WARN
+log4j.logger.net=WARN
+log4j.logger.com=WARN
+
+# suppress inspection "SpellCheckingInspection"
+log4j.logger.com.opensymphony.oscache=WARN
+
+# Spring
+log4j.logger.org.springframework=WARN
+log4j.logger.org.springframework.beans.GenericTypeAwarePropertyDescriptor=FATAL
+log4j.logger.org.springframework.security.access.event.LoggerListener=INFO
+log4j.logger.org.springframework.security.authentication.event.LoggerListener=ERROR
+
+# Apache Commons
+log4j.logger.org.apache.commons=WARN
+
+# Hibernate
+log4j.logger.org.hibernate=WARN
+# Avoid warnings about expired objects.
+log4j.logger.org.hibernate.cache.ReadWriteCache=ERROR
+# Avoid warnings about collection fail-safe cleanup.
+log4j.logger.org.hibernate.engine.loading.LoadContexts=ERROR
+log4j.logger.net.sf.ehcache.hibernate.strategy.AbstractReadWriteEhcacheAccessStrategy=ERROR
+log4j.logger.org.hibernate.cache.ehcache.internal.strategy.AbstractReadWriteEhcacheAccessStrategy=ERROR
+
+# Hibernate Search
+# This is necessary to see progress in IndexGemmaCLI command
+log4j.logger.org.hibernate.search.impl.SimpleIndexingProgressMonitor=INFO
\ No newline at end of file
diff --git a/gemma-cli/src/main/config/log4j.properties b/gemma-cli/src/main/config/log4j.properties
index 3ea5767ecf..08306b788f 100644
--- a/gemma-cli/src/main/config/log4j.properties
+++ b/gemma-cli/src/main/config/log4j.properties
@@ -10,7 +10,7 @@
 log4j.appender.stderr=org.apache.log4j.ConsoleAppender
 log4j.appender.stderr.target=System.err
 log4j.appender.stderr.layout=org.apache.log4j.PatternLayout
-log4j.appender.stderr.layout.ConversionPattern=%d %highlight{%p} %style{%pid}{magenta} [%t] %style{%C.%M(%L)}{cyan} | %m%n
+log4j.appender.stderr.layout.ConversionPattern=%d %highlight{%p} %style{%pid}{magenta} [%t] %style{%C{1.}.%M(%L)}{cyan} | %m%n
 
 # Log to a file. Note that if File is a relative path, the output file goes wherever the application JVM was started from.
 # Define gemma.log.dir as a parameter to your JAVA_OPTS and make sure this is passed to java when you start it.
diff --git a/gemma-cli/src/main/java/ubic/gemma/core/apps/DeleteExperimentsCli.java b/gemma-cli/src/main/java/ubic/gemma/core/apps/DeleteExperimentsCli.java
index 6797a8ae5c..07ca29d56a 100644
--- a/gemma-cli/src/main/java/ubic/gemma/core/apps/DeleteExperimentsCli.java
+++ b/gemma-cli/src/main/java/ubic/gemma/core/apps/DeleteExperimentsCli.java
@@ -47,7 +47,6 @@ protected void buildOptions( Options options ) {
                 Option.builder( "a" ).longOpt( "array" )
                         .desc( "Delete platform(s) instead; you must delete associated experiments first; other options are ignored" )
                         .argName( "comma-delimited list of platform short names" ).hasArg().build() );
-        super.suppressAllOption();
     }
 
     @Override
diff --git a/gemma-cli/src/main/java/ubic/gemma/core/apps/ExpressionExperimentManipulatingCLI.java b/gemma-cli/src/main/java/ubic/gemma/core/apps/ExpressionExperimentManipulatingCLI.java
index c5cd292787..148318e8f9 100644
--- a/gemma-cli/src/main/java/ubic/gemma/core/apps/ExpressionExperimentManipulatingCLI.java
+++ b/gemma-cli/src/main/java/ubic/gemma/core/apps/ExpressionExperimentManipulatingCLI.java
@@ -22,6 +22,7 @@
 import org.apache.commons.cli.Option;
 import org.apache.commons.cli.Options;
 import org.apache.commons.lang3.StringUtils;
+import org.springframework.beans.factory.annotation.Autowired;
 import ubic.gemma.core.apps.GemmaCLI.CommandGroup;
 import ubic.gemma.core.genome.gene.service.GeneService;
 import ubic.gemma.core.search.SearchException;
@@ -36,77 +37,69 @@
 import ubic.gemma.model.expression.experiment.ExpressionExperiment;
 import ubic.gemma.model.genome.Gene;
 import ubic.gemma.model.genome.Taxon;
-import ubic.gemma.persistence.service.common.auditAndSecurity.AuditEventService;
 import ubic.gemma.persistence.service.expression.arrayDesign.ArrayDesignService;
 import ubic.gemma.persistence.service.expression.experiment.ExpressionExperimentService;
 import ubic.gemma.persistence.service.expression.experiment.ExpressionExperimentSetService;
 import ubic.gemma.persistence.service.genome.taxon.TaxonService;
 
 import java.io.IOException;
-import java.util.*;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
 import java.util.stream.Collectors;
 
 /**
  * Base class for CLIs that needs one or more expression experiment as an input. It offers the following ways of reading
  * them in:
  * <ul>
- * <li>All EEs
- * <li>All EEs for a particular taxon.
+ * <li>All EEs (if -all is supplied)</li>
+ * <li>All EEs for a particular taxon.</li>
  * <li>A specific ExpressionExperimentSet, identified by name</li>
- * <li>A comma-delimited list of one or more EEs identified by short name given on the command line
- * <li>From a file, with one short name per line.
- * <li>EEs matching a query string (e.g., 'brain')
+ * <li>A comma-delimited list of one or more EEs identified by short name given on the command line</li>
+ * <li>From a file, with one short name per line.</li>
+ * <li>EEs matching a query string (e.g., 'brain')</li>
  * <li>(Optional) 'Auto' mode, in which experiments to analyze are selected automatically based on their workflow state.
- * This can be enabled and modified by subclasses who override the "needToRun" method.
- * <li>All EEs that were last processed after a given date, similar to 'auto' otherwise.
+ * This can be enabled and modified by subclasses who override the "needToRun" method.</li>
+ * <li>All EEs that were last processed after a given date, similar to 'auto' otherwise.</li>
  * </ul>
  * Some of these options can be (or should be) combined, and modified by a (optional) "force" option, and will have
  * customized behavior.
+ * <p>
  * In addition, EEs can be excluded based on a list given in a separate file.
  *
  * @author Paul
  */
 public abstract class ExpressionExperimentManipulatingCLI extends AbstractCLIContextCLI {
-    ExpressionExperimentService eeService;
-    Set<BioAssaySet> expressionExperiments = new HashSet<>();
-    private boolean allowProcessingAll = true;
 
-    public Set<BioAssaySet> getExpressionExperiments() {
-        return expressionExperiments;
-    }
-
-    protected ExpressionExperimentService getEeService() {
-        return eeService;
-    }
-
-    protected Taxon getTaxon() {
-        return taxon;
-    }
-
-    protected void setTaxon( Taxon taxon ) {
-        this.taxon = taxon;
-    }
-
-    protected TaxonService getTaxonService() {
-        return taxonService;
-    }
-
-    protected GeneService getGeneService() {
-        return geneService;
-    }
-
-    boolean force = false;
-    private Taxon taxon = null;
+    @Autowired
+    protected ExpressionExperimentService eeService;
+    @Autowired
+    private ExpressionExperimentSetService expressionExperimentSetService;
+    @Autowired
     private TaxonService taxonService;
+    @Autowired
     private GeneService geneService;
+    @Autowired
     private SearchService searchService;
+    @Autowired
+    private ArrayDesignService arrayDesignService;
+
+    protected final Set<BioAssaySet> expressionExperiments = new HashSet<>();
+    /**
+     * Taxon used for filtering EEs.
+     */
+    private Taxon taxon = null;
+    protected boolean force = false;
+
+    protected ExpressionExperimentManipulatingCLI() {
+    }
 
     @Override
     public CommandGroup getCommandGroup() {
         return CommandGroup.EXPERIMENT;
     }
 
-    @SuppressWarnings("AccessStaticViaInstance") // Cleaner like this
     @Override
     protected void buildOptions( Options options ) {
         Option expOption = Option.builder( "e" ).hasArg().argName( "shortname" ).desc(
@@ -116,6 +109,8 @@ protected void buildOptions( Options options ) {
 
         options.addOption( expOption );
 
+        options.addOption( "all", false, "Process all expression experiments" );
+
         Option eeFileListOption = Option.builder( "f" ).hasArg().argName( "file" ).desc(
                         "File with list of short names or IDs of expression experiments (one per line; use instead of '-e')" )
                 .longOpt( "eeListfile" ).build();
@@ -155,11 +150,6 @@ protected Gene findGeneByOfficialSymbol( String symbol, Taxon t ) {
 
     @Override
     protected void processOptions( CommandLine commandLine ) {
-        eeService = this.getBean( ExpressionExperimentService.class );
-        geneService = this.getBean( GeneService.class );
-        taxonService = this.getBean( TaxonService.class );
-        searchService = this.getBean( SearchService.class );
-        this.auditEventService = this.getBean( AuditEventService.class );
         if ( commandLine.hasOption( 't' ) ) {
             this.taxon = this.setTaxonByName( commandLine, taxonService );
         }
@@ -168,7 +158,10 @@ protected void processOptions( CommandLine commandLine ) {
             this.force = true;
         }
 
-        if ( commandLine.hasOption( "eeset" ) ) {
+        if ( commandLine.hasOption( "all" ) ) {
+            log.warn( "Loading all expression experiments, this might take a while..." );
+            this.expressionExperiments.addAll( eeService.loadAll() );
+        } else if ( commandLine.hasOption( "eeset" ) ) {
             this.experimentsFromEeSet( commandLine.getOptionValue( "eeset" ) );
         } else if ( commandLine.hasOption( 'e' ) ) {
             this.experimentsFromCliList( commandLine );
@@ -176,34 +169,26 @@ protected void processOptions( CommandLine commandLine ) {
             String experimentListFile = commandLine.getOptionValue( 'f' );
             AbstractCLI.log.info( "Reading experiment list from " + experimentListFile );
             try {
-                this.expressionExperiments = this.readExpressionExperimentListFile( experimentListFile );
+                this.expressionExperiments.addAll( this.readExpressionExperimentListFile( experimentListFile ) );
             } catch ( IOException e ) {
                 throw new RuntimeException( e );
             }
         } else if ( commandLine.hasOption( 'q' ) ) {
             AbstractCLI.log.info( "Processing all experiments that match query " + commandLine.getOptionValue( 'q' ) );
             try {
-                this.expressionExperiments = this.findExpressionExperimentsByQuery( commandLine.getOptionValue( 'q' ) );
+                this.expressionExperiments.addAll( this.findExpressionExperimentsByQuery( commandLine.getOptionValue( 'q' ) ) );
             } catch ( SearchException e ) {
                 log.error( "Failed to retrieve EEs for the passed query via -q.", e );
             }
-        } else if ( taxon != null ) {
-            if ( !commandLine.hasOption( "dataFile" ) ) {
-                AbstractCLI.log.info( "Processing all experiments for " + taxon.getCommonName() );
-                this.expressionExperiments = new HashSet<BioAssaySet>( eeService.findByTaxon( taxon ) );
-            }
         } else {
-            if ( !commandLine.hasOption( "dataFile" ) && allowProcessingAll ) {
-                AbstractCLI.log.info( "Processing all experiments (further filtering may modify)" );
-                this.expressionExperiments = new HashSet<BioAssaySet>( eeService.loadAll() );
-            }
+            throw new IllegalArgumentException( "At least one of -all, -e, -eeset, -f, or -q must be provided." );
         }
 
-        if ( commandLine.hasOption( 'x' ) ) {
+        if ( commandLine.hasOption( 'x' ) && !expressionExperiments.isEmpty() ) {
             this.excludeFromFile( commandLine );
         }
 
-        if ( expressionExperiments != null && expressionExperiments.size() > 0 && !force ) {
+        if ( !force && !expressionExperiments.isEmpty() ) {
 
             if ( commandLine.hasOption( AbstractCLI.AUTO_OPTION_NAME ) ) {
                 this.autoSeek = true;
@@ -228,22 +213,17 @@ protected void processOptions( CommandLine commandLine ) {
             }
         }
 
-        if ( expressionExperiments != null && expressionExperiments.size() > 1 ) {
-            AbstractCLI.log.info( "Final list: " + this.expressionExperiments.size()
-                    + " expressionExperiments (futher filtering may modify)" );
-        } else if ( expressionExperiments == null || expressionExperiments.size() == 0 ) {
-            if ( commandLine.hasOption( "dataFile" ) ) {
-                //    AbstractCLI.log.info( "Expression matrix from data file selected" );
-            } else {
-                //   AbstractCLI.log.info( "No experiments selected" );
-            }
+        if ( expressionExperiments.isEmpty() ) {
+            log.warn( "No expression experiments matched the given options." );
+        } else if ( expressionExperiments.size() == 1 ) {
+            AbstractCLI.log.info( "Final dataset: " + expressionExperiments.iterator().next() );
+        } else {
+            AbstractCLI.log.info( String.format( "Final list: %d expression experiments", this.expressionExperiments.size() ) );
         }
-
     }
 
     void addForceOption( Options options ) {
         String desc = "Ignore other reasons for skipping experiments (e.g., trouble) and overwrite existing data (see documentation for this tool to see exact behavior if not clear)";
-        @SuppressWarnings("static-access")
         Option forceOption = Option.builder( "force" ).longOpt( "force" ).desc( desc ).build();
         options.addOption( forceOption );
     }
@@ -256,7 +236,7 @@ private void excludeFromFile( CommandLine commandLine ) {
         } catch ( IOException e ) {
             throw new RuntimeException( e );
         }
-        assert expressionExperiments.size() > 0;
+        assert !expressionExperiments.isEmpty();
 
         int before = expressionExperiments.size();
 
@@ -277,10 +257,9 @@ private void experimentsFromCliList( CommandLine commandLine ) {
                 AbstractCLI.log.warn( shortName + " not found" );
                 continue;
             }
-            eeService.thawLite( expressionExperiment );
-            expressionExperiments.add( expressionExperiment );
+            expressionExperiments.add( eeService.thawLite( expressionExperiment ) );
         }
-        if ( expressionExperiments.size() == 0 ) {
+        if ( expressionExperiments.isEmpty() ) {
             throw new RuntimeException( "There were no valid experimnents specified" );
         }
     }
@@ -291,16 +270,14 @@ private void experimentsFromEeSet( String optionValue ) {
             throw new IllegalArgumentException( "Please provide an eeset name" );
         }
 
-        ExpressionExperimentSetService expressionExperimentSetService = this
-                .getBean( ExpressionExperimentSetService.class );
         Collection<ExpressionExperimentSet> sets = expressionExperimentSetService.findByName( optionValue );
         if ( sets.size() > 1 ) {
             throw new IllegalArgumentException( "More than on EE set has name '" + optionValue + "'" );
-        } else if ( sets.size() == 0 ) {
+        } else if ( sets.isEmpty() ) {
             throw new IllegalArgumentException( "No EE set has name '" + optionValue + "'" );
         }
         ExpressionExperimentSet set = sets.iterator().next();
-        this.expressionExperiments = new HashSet<>( set.getExperiments() );
+        this.expressionExperiments.addAll( set.getExperiments() );
 
     }
 
@@ -312,9 +289,9 @@ private Set<BioAssaySet> findExpressionExperimentsByQuery( String query ) throws
 
         // explicitly support one case
         if ( query.matches( "GPL[0-9]+" ) ) {
-            ArrayDesign ad = this.getBean( ArrayDesignService.class ).findByShortName( query );
+            ArrayDesign ad = arrayDesignService.findByShortName( query );
             if ( ad != null ) {
-                Collection<ExpressionExperiment> ees2 = this.getBean( ArrayDesignService.class ).getExpressionExperiments( ad );
+                Collection<ExpressionExperiment> ees2 = arrayDesignService.getExpressionExperiments( ad );
                 ees.addAll( ees2 );
                 log.info( ees.size() + " experiments matched to platform " + ad );
             }
@@ -391,7 +368,7 @@ private Set<BioAssaySet> readExpressionExperimentListFile( String fileName ) thr
      * @return a collection of troubled experiemnt, or an empty set of non are
      */
     private Set<BioAssaySet> getTroubledExpressionExperiments() {
-        if ( expressionExperiments == null || expressionExperiments.size() == 0 ) {
+        if ( expressionExperiments.isEmpty() ) {
             AbstractCLI.log.warn( "No experiments to remove troubled from" );
             return Collections.emptySet();
         }
@@ -401,12 +378,4 @@ private Set<BioAssaySet> getTroubledExpressionExperiments() {
                 .filter( ee -> ee.getCurationDetails().getTroubled() )
                 .collect( Collectors.toSet() );
     }
-
-    /**
-     * Disable the ability for this CLI to process all experiments when no other specification is given.
-     * The user must explicitly define the experiments to be processed.
-     */
-    protected void suppressAllOption() {
-        this.allowProcessingAll = false;
-    }
 }
diff --git a/gemma-cli/src/main/java/ubic/gemma/core/apps/ExternalDatabaseAdderCli.java b/gemma-cli/src/main/java/ubic/gemma/core/apps/ExternalDatabaseAdderCli.java
index 009d19e226..a6de5af4d3 100644
--- a/gemma-cli/src/main/java/ubic/gemma/core/apps/ExternalDatabaseAdderCli.java
+++ b/gemma-cli/src/main/java/ubic/gemma/core/apps/ExternalDatabaseAdderCli.java
@@ -20,8 +20,11 @@
 package ubic.gemma.core.apps;
 
 import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
 import org.apache.commons.cli.Options;
+import org.springframework.beans.factory.annotation.Autowired;
 import ubic.gemma.core.util.AbstractCLIContextCLI;
+import ubic.gemma.model.common.description.DatabaseType;
 import ubic.gemma.model.common.description.ExternalDatabase;
 import ubic.gemma.persistence.service.common.description.ExternalDatabaseService;
 
@@ -32,6 +35,12 @@
  */
 public class ExternalDatabaseAdderCli extends AbstractCLIContextCLI {
 
+    @Autowired
+    private ExternalDatabaseService externalDatabaseService;
+
+    private String name;
+    private DatabaseType type;
+
     @Override
     public GemmaCLI.CommandGroup getCommandGroup() {
         return GemmaCLI.CommandGroup.SYSTEM;
@@ -44,49 +53,24 @@ public String getCommandName() {
 
     @Override
     public String getShortDesc() {
-        return "Add a new external database, but requires editing the code to do so. It can be done by SQL manually as well.";
+        return "Add a new external database.";
     }
 
     @Override
     protected void buildOptions( Options options ) {
+        options.addOption( Option.builder( "n" ).longOpt( "name" ).hasArg().required().build() );
+        options.addOption( Option.builder( "t" ).longOpt( "type" ).hasArg().required().build() );
     }
 
     @Override
     protected void processOptions( CommandLine commandLine ) throws Exception {
-
+        this.name = commandLine.getOptionValue( "n" );
+        this.type = DatabaseType.valueOf( commandLine.getOptionValue( "t" ).toUpperCase() );
     }
 
     @Override
     protected void doWork() throws Exception {
-        // ContactService contactService = this.getBean( ContactService.class );
-
-        ExternalDatabase toAdd = ExternalDatabase.Factory.newInstance();
-
-        // Contact c = contactService.findByName( "Affymetrix" ).iterator().next();
-        // toAdd.setDatabaseSupplier( c );
-        // toAdd.setDescription( "The NetAffx Analysis Center enables researchers to correlate their "
-        // + "GeneChip array results with array design and annotation information." );
-        // toAdd.setName( "NetAFFX" );
-        // toAdd.setType( DatabaseType.SEQUENCE );
-        // toAdd.setWebUri( "http://www.affymetrix.com/analysis/index.affx" );
-
-        // Contact c = Contact.Factory.newInstance();
-        // c.setName( "McKusick-Nathans Institute of Genetic Medicine" );
-        // c = contactService.findOrCreate( c );
-        // toAdd.setDatabaseSupplier( c );
-        // toAdd.setDescription(
-        // "Online Mendelian Inheritance in Man is a comprehensive, authoritative, and timely compendium of human
-        // genes and genetic phenotypes. "
-        // +
-        // "OMIM and Online Mendelian Inheritance in Man are registered trademarks of the Johns Hopkins University."
-        // );
-        // toAdd.setName( "OMIM" );
-        // toAdd.setType( DatabaseType.OTHER );
-        // toAdd.setWebUri( "http://omim.org/" );
-
-        ExternalDatabaseService eds = this.getBean( ExternalDatabaseService.class );
-
-        eds.findOrCreate( toAdd );
+        ExternalDatabase created = externalDatabaseService.create( ExternalDatabase.Factory.newInstance( name, type ) );
+        log.info( "Created " + created );
     }
-
 }
diff --git a/gemma-cli/src/main/java/ubic/gemma/core/apps/ExternalDatabaseOverviewCli.java b/gemma-cli/src/main/java/ubic/gemma/core/apps/ExternalDatabaseOverviewCli.java
new file mode 100644
index 0000000000..4082e709dd
--- /dev/null
+++ b/gemma-cli/src/main/java/ubic/gemma/core/apps/ExternalDatabaseOverviewCli.java
@@ -0,0 +1,81 @@
+package ubic.gemma.core.apps;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Options;
+import org.springframework.beans.factory.annotation.Autowired;
+import ubic.gemma.core.util.AbstractSpringAwareCLI;
+import ubic.gemma.model.common.auditAndSecurity.AuditEvent;
+import ubic.gemma.model.common.auditAndSecurity.AuditTrail;
+import ubic.gemma.model.common.description.ExternalDatabase;
+import ubic.gemma.persistence.service.common.description.ExternalDatabaseService;
+
+import javax.annotation.Nullable;
+import java.util.Comparator;
+import java.util.stream.Collectors;
+
+public class ExternalDatabaseOverviewCli extends AbstractSpringAwareCLI {
+
+    @Autowired
+    private ExternalDatabaseService externalDatabaseService;
+
+    @Nullable
+    @Override
+    public String getCommandName() {
+        return "listExternalDatabases";
+    }
+
+    @Nullable
+    @Override
+    public String getShortDesc() {
+        return "Print an overview of all external databases used by Gemma";
+    }
+
+    @Override
+    public GemmaCLI.CommandGroup getCommandGroup() {
+        return GemmaCLI.CommandGroup.MISC;
+    }
+
+    @Override
+    protected void buildOptions( Options options ) {
+
+    }
+
+    @Override
+    protected void processOptions( CommandLine commandLine ) throws Exception {
+
+    }
+
+    @Override
+    protected void doWork() throws Exception {
+        externalDatabaseService.loadAllWithAuditTrail().stream()
+                .sorted( Comparator.comparing( ExternalDatabase::getLastUpdated, Comparator.nullsLast( Comparator.reverseOrder() ) ) )
+                .forEachOrdered( ed -> addSuccessObject( ed, summarize( ed ) ) );
+    }
+
+    private String summarize( ExternalDatabase ed ) {
+        return String.format( "Description: %s\nRelease Version: %s\nRelease URL: %s\nLast Updated: %s\nEvents:\n%s",
+                ed.getDescription(),
+                ed.getReleaseVersion(),
+                ed.getReleaseUrl(),
+                ed.getLastUpdated(),
+                summarize( ed.getAuditTrail() ) );
+    }
+
+    private String summarize( AuditTrail auditTrail ) {
+        return auditTrail.getEvents().stream()
+                .sorted( Comparator.comparing( AuditEvent::getDate, Comparator.reverseOrder() ) ) // most recent events first
+                .map( this::summarize )
+                .collect( Collectors.joining( "\n" ) );
+    }
+
+    private String summarize( AuditEvent auditEvent ) {
+        String s = String.format( "%s: %s by %s", auditEvent.getDate(), auditEvent.getAction(), auditEvent.getPerformer().getUserName() );
+        if ( auditEvent.getNote() != null ) {
+            s += "\n\t" + auditEvent.getNote();
+        }
+        if ( auditEvent.getDetail() != null ) {
+            s += "\n\t" + auditEvent.getDetail();
+        }
+        return s;
+    }
+}
diff --git a/gemma-cli/src/main/java/ubic/gemma/core/apps/ExternalDatabaseUpdaterCli.java b/gemma-cli/src/main/java/ubic/gemma/core/apps/ExternalDatabaseUpdaterCli.java
index 404d8fbb5b..10f0b7f882 100644
--- a/gemma-cli/src/main/java/ubic/gemma/core/apps/ExternalDatabaseUpdaterCli.java
+++ b/gemma-cli/src/main/java/ubic/gemma/core/apps/ExternalDatabaseUpdaterCli.java
@@ -20,30 +20,35 @@ public class ExternalDatabaseUpdaterCli extends AbstractSpringAwareCLI {
 
     private static final String NAME_OPTION = "n",
             DESCRIPTION_OPTION = "d",
+            PARENT_DATABASE_OPTION = "parentDatabase",
             RELEASE_OPTION = "release",
             RELEASE_NOTE_OPTION = "releaseNote",
             RELEASE_VERSION_OPTION = "releaseVersion",
             RELEASE_URL_OPTION = "releaseUrl",
-            LAST_UPDATED_OPTION = "lastUpdated",
-            PARENT_DATABASE_OPTION = "parentDatabase";
+            LAST_UPDATED_OPTION = "lastUpdated";
 
     @Autowired
     private ExternalDatabaseService externalDatabaseService;
 
     private String name;
     private String description;
+    private ExternalDatabase parentDatabase;
     private boolean release;
     private String releaseNote;
     private String releaseVersion;
     private URL releaseUrl;
     private Date lastUpdated;
-    private ExternalDatabase parentDatabase;
 
     @Override
     public String getCommandName() {
         return "updateExternalDatabase";
     }
 
+    @Override
+    public String getShortDesc() {
+        return "Update an external database and optionally perform a release.";
+    }
+
     @Override
     public GemmaCLI.CommandGroup getCommandGroup() {
         return GemmaCLI.CommandGroup.SYSTEM;
@@ -62,9 +67,11 @@ protected void buildOptions( Options options ) {
                 .optionalArg( false )
                 .desc( "External database name" )
                 .required( true ).build() );
-        options.addOption( DESCRIPTION_OPTION, "description", true, "New description" );
-        options.addOption( RELEASE_OPTION, "release", false, "Update the release (only affects last modified moment))" );
-        options.addOption( RELEASE_VERSION_OPTION, "release-version", true, "Release version" );
+        options.addOption( DESCRIPTION_OPTION, "description", true, "Update the description" );
+        options.addOption( PARENT_DATABASE_OPTION, "parent-database", true, "Update the parent database either by ID or name" );
+        // release-specific options
+        options.addOption( RELEASE_OPTION, "release", false, "Update the release details" );
+        options.addOption( RELEASE_VERSION_OPTION, "release-version", true, "Release version (optional)" );
         options.addOption( Option.builder( RELEASE_URL_OPTION )
                 .longOpt( "release-url" )
                 .hasArg()
@@ -74,13 +81,8 @@ protected void buildOptions( Options options ) {
         options.addOption( Option.builder( LAST_UPDATED_OPTION )
                 .longOpt( "last-updated" )
                 .hasArg()
-                .desc( "Moment the release was performed if known, otherwise the current time will be used." )
+                .desc( "Moment the release was performed if known, otherwise the current time will be used" )
                 .type( Date.class ).build() );
-        options.addOption( Option.builder( PARENT_DATABASE_OPTION )
-                .longOpt( "parent-database" )
-                .hasArg()
-                .desc( "Identifier or name of the parent database." )
-                .build() );
 
     }
 
@@ -93,6 +95,9 @@ protected void processOptions( CommandLine commandLine ) throws Exception {
         releaseVersion = commandLine.getOptionValue( RELEASE_VERSION_OPTION );
         releaseUrl = ( URL ) commandLine.getParsedOptionValue( RELEASE_URL_OPTION );
         lastUpdated = ( Date ) commandLine.getParsedOptionValue( LAST_UPDATED_OPTION );
+        if ( !release && ( releaseNote != null || releaseVersion != null || releaseUrl != null || lastUpdated != null ) ) {
+            throw new IllegalArgumentException( "The -release,--release flag must be used to update release details." );
+        }
         if ( lastUpdated == null ) {
             lastUpdated = new Date();
         }
@@ -117,7 +122,7 @@ protected void doWork() throws Exception {
         if ( description != null ) {
             ed.setDescription( description );
         }
-        if ( release || releaseVersion != null ) {
+        if ( release ) {
             if ( releaseVersion != null ) {
                 AbstractCLI.log.info( String.format( "Updating %s release version to %s.", name, releaseVersion ) );
                 externalDatabaseService.updateReleaseDetails( ed, releaseVersion, releaseUrl, releaseNote, lastUpdated );
diff --git a/gemma-cli/src/main/java/ubic/gemma/core/apps/GemmaCLI.java b/gemma-cli/src/main/java/ubic/gemma/core/apps/GemmaCLI.java
index de7f546db0..2edb25d8a2 100644
--- a/gemma-cli/src/main/java/ubic/gemma/core/apps/GemmaCLI.java
+++ b/gemma-cli/src/main/java/ubic/gemma/core/apps/GemmaCLI.java
@@ -187,24 +187,27 @@ public static void main( String[] args ) {
         String commandRequested = commandArgs.remove( 0 );
         String[] argsToPass = commandArgs.toArray( new String[] {} );
 
+        int statusCode;
         if ( !commandsByName.containsKey( commandRequested ) ) {
             System.err.println( "Unrecognized command: " + commandRequested );
             GemmaCLI.printHelp( options, commandGroups );
-            System.exit( 1 );
+            statusCode = 1;
         } else {
             try {
                 CLI cli = commandsByName.get( commandRequested );
                 System.err.println( "========= Gemma CLI invocation of " + commandRequested + " ============" );
                 System.err.println( "Options: " + GemmaCLI.getOptStringForLogging( argsToPass ) );
-                System.exit( cli.executeCommand( argsToPass ) );
+                statusCode = cli.executeCommand( argsToPass );
             } catch ( Exception e ) {
                 System.err.println( "Gemma CLI error: " + e.getClass().getName() + " - " + e.getMessage() );
                 System.err.println( ExceptionUtils.getStackTrace( e ) );
-                throw new RuntimeException( e );
+                statusCode = 1;
             } finally {
                 System.err.println( "========= Gemma CLI run of " + commandRequested + " complete ============" );
             }
         }
+
+        System.exit( statusCode );
     }
 
     /**
diff --git a/gemma-cli/src/main/java/ubic/gemma/core/apps/LinkAnalysisCli.java b/gemma-cli/src/main/java/ubic/gemma/core/apps/LinkAnalysisCli.java
index 940443dd8f..a183b58b9c 100644
--- a/gemma-cli/src/main/java/ubic/gemma/core/apps/LinkAnalysisCli.java
+++ b/gemma-cli/src/main/java/ubic/gemma/core/apps/LinkAnalysisCli.java
@@ -22,6 +22,8 @@
 import org.apache.commons.cli.Option;
 import org.apache.commons.cli.Options;
 import org.apache.commons.lang3.time.StopWatch;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
 import ubic.basecode.dataStructure.matrix.DoubleMatrix;
 import ubic.basecode.io.ByteArrayConverter;
 import ubic.basecode.util.FileTools;
@@ -44,8 +46,10 @@
 import ubic.gemma.model.expression.experiment.BioAssaySet;
 import ubic.gemma.model.expression.experiment.ExpressionExperiment;
 import ubic.gemma.model.expression.experiment.ExpressionExperimentValueObject;
+import ubic.gemma.model.genome.Taxon;
 import ubic.gemma.persistence.service.association.coexpression.CoexpressionService;
 import ubic.gemma.persistence.service.expression.arrayDesign.ArrayDesignService;
+import ubic.gemma.persistence.service.genome.taxon.TaxonService;
 import ubic.gemma.persistence.util.EntityUtils;
 
 import java.io.File;
@@ -60,13 +64,27 @@
  * @author paul (refactoring)
  * @author vaneet
  */
+@Component
 public class LinkAnalysisCli extends ExpressionExperimentManipulatingCLI {
 
+    @Autowired
+    private LinkAnalysisService linkAnalysisService;
+    @Autowired
+    private LinkAnalysisPersister linkAnalysisPersister;
+    @Autowired
+    private CoexpressionService coexpressionService;
+    @Autowired
+    private TaxonService taxonService;
+    @Autowired
+    private SimpleExpressionDataLoaderService simpleExpressionDataLoaderService;
+    @Autowired
+    private ArrayDesignService arrayDesignService;
+
     private final FilterConfig filterConfig = new FilterConfig();
     private final LinkAnalysisConfig linkAnalysisConfig = new LinkAnalysisConfig();
     private String analysisTaxon = null;
     private String dataFileName = null;
-    private LinkAnalysisService linkAnalysisService;
+    private Taxon taxon;
     private boolean initializeFromOldData = false;
     private boolean updateNodeDegree = false;
     private boolean deleteAnalyses = false;
@@ -79,27 +97,20 @@ public String getCommandName() {
     @Override
     protected void doWork() throws Exception {
         if ( initializeFromOldData ) {
-            AbstractCLI.log.info( "Initializing links from old data for " + this.getTaxon() );
-            LinkAnalysisPersister s = this.getBean( LinkAnalysisPersister.class );
-            s.initializeLinksFromOldData( this.getTaxon() );
+            AbstractCLI.log.info( "Initializing links from old data for " + taxon );
+            linkAnalysisPersister.initializeLinksFromOldData( taxon );
             return;
         } else if ( updateNodeDegree ) {
-
             // we waste some time here getting the experiments.
             this.loadTaxon();
-
-            this.getBean( CoexpressionService.class ).updateNodeDegrees( this.getTaxon() );
+            coexpressionService.updateNodeDegrees( taxon );
         }
 
-        this.linkAnalysisService = this.getBean( LinkAnalysisService.class );
-
         if ( this.dataFileName != null ) {
             /*
              * Read vectors from file. Could provide as a matrix, but it's easier to provide vectors (less mess in later
              * code)
              */
-            ArrayDesignService arrayDesignService = this.getBean( ArrayDesignService.class );
-
             ArrayDesign arrayDesign = arrayDesignService.findByShortName( this.linkAnalysisConfig.getArrayName() );
 
             if ( arrayDesign == null ) {
@@ -118,10 +129,8 @@ protected void doWork() throws Exception {
 
             QuantitationType qtype = this.makeQuantitationType();
 
-            SimpleExpressionDataLoaderService simpleExpressionDataLoaderService = this
-                    .getBean( SimpleExpressionDataLoaderService.class );
             ByteArrayConverter bArrayConverter = new ByteArrayConverter();
-            try ( InputStream data = new FileInputStream( new File( this.dataFileName ) ) ) {
+            try ( InputStream data = new FileInputStream( this.dataFileName ) ) {
 
                 DoubleMatrix<String, String> matrix = simpleExpressionDataLoaderService.parse( data );
 
@@ -147,11 +156,9 @@ protected void doWork() throws Exception {
                 }
                 AbstractCLI.log.info( "Read " + dataVectors.size() + " data vectors" );
 
-            } catch ( Exception e ) {
-                throw e;
             }
 
-            this.linkAnalysisService.processVectors( this.getTaxon(), dataVectors, filterConfig, linkAnalysisConfig );
+            this.linkAnalysisService.processVectors( taxon, dataVectors, filterConfig, linkAnalysisConfig );
         } else {
 
             /*
@@ -166,19 +173,16 @@ protected void doWork() throws Exception {
                         .loadValueObjectsByIds( EntityUtils.getIds( expressionExperiments ), true );
                 final Map<Long, ExpressionExperimentValueObject> idMap = EntityUtils.getIdMap( vos );
 
-                Collections.sort( sees, new Comparator<BioAssaySet>() {
-                    @Override
-                    public int compare( BioAssaySet o1, BioAssaySet o2 ) {
+                sees.sort( ( o1, o2 ) -> {
 
-                        ExpressionExperimentValueObject e1 = idMap.get( o1.getId() );
-                        ExpressionExperimentValueObject e2 = idMap.get( o2.getId() );
+                    ExpressionExperimentValueObject e1 = idMap.get( o1.getId() );
+                    ExpressionExperimentValueObject e2 = idMap.get( o2.getId() );
 
-                        assert e1 != null : "No valueobject: " + e2;
-                        assert e2 != null : "No valueobject: " + e1;
+                    assert e1 != null : "No valueobject: " + e2;
+                    assert e2 != null : "No valueobject: " + e1;
 
-                        return -e1.getBioMaterialCount().compareTo( e2.getBioMaterialCount() );
+                    return -e1.getBioMaterialCount().compareTo( e2.getBioMaterialCount() );
 
-                    }
                 } );
             }
 
@@ -197,7 +201,6 @@ public String getShortDesc() {
         return "Analyze expression data sets for coexpressed genes";
     }
 
-    @SuppressWarnings("static-access")
     @Override
     protected void buildOptions( Options options ) {
         super.buildOptions( options );
@@ -334,7 +337,7 @@ protected void processOptions( CommandLine commandLine ) {
         }
 
         if ( commandLine.hasOption( "dataFile" ) ) {
-            if ( this.expressionExperiments.size() > 0 ) {
+            if ( !this.expressionExperiments.isEmpty() ) {
                 throw new RuntimeException( "The 'dataFile' option is incompatible with other data set selection options" );
             }
 
@@ -429,7 +432,6 @@ protected void processOptions( CommandLine commandLine ) {
 
     }
 
-    @SuppressWarnings("static-access")
     private void buildFilterConfigOptions( Options options ) {
         Option minPresentFraction = Option.builder( "m" ).hasArg().argName( "Missing Value Threshold" ).desc(
                         "Fraction of data points that must be present in a profile to be retained , default="
@@ -473,11 +475,11 @@ private void getFilterConfigOptions( CommandLine commandLine ) {
     }
 
     private void loadTaxon() {
-        super.setTaxon( this.getTaxonService().findByCommonName( analysisTaxon ) );
-        if ( this.getTaxon() == null || !this.getTaxon().getIsGenesUsable() ) {
-            throw new IllegalArgumentException( "No such taxon or, does not have usable gene information: " + getTaxon() );
+        this.taxon = this.taxonService.findByCommonName( analysisTaxon );
+        if ( taxon == null || !taxon.getIsGenesUsable() ) {
+            throw new IllegalArgumentException( "No such taxon or, does not have usable gene information: " + taxon );
         }
-        AbstractCLI.log.debug( getTaxon() + "is used" );
+        AbstractCLI.log.debug( taxon + "is used" );
     }
 
     private BioAssayDimension makeBioAssayDimension( ArrayDesign arrayDesign, DoubleMatrix<String, String> matrix ) {
@@ -489,7 +491,7 @@ private BioAssayDimension makeBioAssayDimension( ArrayDesign arrayDesign, Double
 
             BioMaterial bioMaterial = BioMaterial.Factory.newInstance();
             bioMaterial.setName( columnName.toString() );
-            bioMaterial.setSourceTaxon( getTaxon() );
+            bioMaterial.setSourceTaxon( taxon );
 
             BioAssay assay = BioAssay.Factory.newInstance();
             assay.setName( columnName.toString() );
@@ -524,7 +526,7 @@ private void processExperiment( ExpressionExperiment ee ) {
 
         if ( this.deleteAnalyses ) {
             AbstractCLI.log.info( "======= Deleting coexpression analysis (if any) for: " + ee );
-            if ( this.getBean( LinkAnalysisPersister.class ).deleteAnalyses( ee ) ) {
+            if ( linkAnalysisPersister.deleteAnalyses( ee ) ) {
                 addSuccessObject( ee );
             } else {
                 addErrorObject( ee, "Seems to not have any eligible link analysis to remove" );
diff --git a/gemma-cli/src/main/java/ubic/gemma/core/util/AbstractCLI.java b/gemma-cli/src/main/java/ubic/gemma/core/util/AbstractCLI.java
index bb05780421..9178bf195e 100644
--- a/gemma-cli/src/main/java/ubic/gemma/core/util/AbstractCLI.java
+++ b/gemma-cli/src/main/java/ubic/gemma/core/util/AbstractCLI.java
@@ -20,13 +20,14 @@
 
 import lombok.Value;
 import org.apache.commons.cli.*;
+import org.apache.commons.lang3.ArrayUtils;
 import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.exception.ExceptionUtils;
 import org.apache.commons.lang3.time.StopWatch;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import ubic.basecode.util.DateUtil;
 import ubic.gemma.model.common.auditAndSecurity.eventType.AuditEventType;
-import ubic.gemma.persistence.util.Settings;
 
 import javax.annotation.Nullable;
 import java.io.File;
@@ -114,18 +115,43 @@ public abstract class AbstractCLI implements CLI {
     public int executeCommand( String[] args ) {
         StopWatch watch = new StopWatch();
         watch.start();
+        if ( args == null ) {
+            System.err.println( "No arguments" );
+            return FAILURE;
+        }
         try {
             Options options = new Options();
             buildStandardOptions( options );
             buildOptions( options );
-            CommandLine commandLine = processCommandLine( options, args );
+            DefaultParser parser = new DefaultParser();
+            CommandLine commandLine;
+            try {
+                commandLine = parser.parse( options, args );
+            } catch ( ParseException e ) {
+                if ( e instanceof MissingOptionException ) {
+                    // check if -h/--help was passed alongside a required argument
+                    if ( ArrayUtils.contains( args, "-h" ) || ArrayUtils.contains( args, "--help" ) ) {
+                        printHelp( options );
+                        return SUCCESS;
+                    }
+                    System.err.println( "Required option(s) were not supplied: " + e.getMessage() );
+                } else if ( e instanceof AlreadySelectedException ) {
+                    System.err.println( "The option(s) " + e.getMessage() + " were already selected" );
+                } else if ( e instanceof MissingArgumentException ) {
+                    System.err.println( "Missing argument: " + e.getMessage() );
+                } else if ( e instanceof UnrecognizedOptionException ) {
+                    System.err.println( "Unrecognized option: " + e.getMessage() );
+                }
+                printHelp( options );
+                return FAILURE;
+            }
             // check if -h/--help is provided before pursuing option processing
-            if ( commandLine.hasOption( 'h' ) ) {
+            if ( commandLine.hasOption( HELP_OPTION ) ) {
                 printHelp( options );
                 return SUCCESS;
             }
             if ( commandLine.hasOption( TESTING_OPTION ) ) {
-                System.err.printf( "The -testing/--testing option must be passed before the %s command.%n", getCommandName() );
+                AbstractCLI.log.error( String.format( "The -testing/--testing option must be passed before the %s command.", getCommandName() ) );
                 return FAILURE;
             }
             processStandardOptions( commandLine );
@@ -133,12 +159,12 @@ public int executeCommand( String[] args ) {
             doWork();
             return errorObjects.isEmpty() ? SUCCESS : FAILURE_FROM_ERROR_OBJECTS;
         } catch ( Exception e ) {
-            log.error( getCommandName() + " failed.", e );
+            log.error( String.format( "%s failed: %s", getCommandName(), ExceptionUtils.getRootCauseMessage( e ) ), e );
             return FAILURE;
         } finally {
             // always summarize processing, even if an error is thrown
             summarizeProcessing();
-            AbstractCLI.log.info( "Elapsed time: " + watch.getTime() / 1000 + " seconds." );
+            log.info( String.format( "Elapsed time: %d seconds.", watch.getTime( TimeUnit.SECONDS ) ) );
         }
     }
 
@@ -207,10 +233,8 @@ protected void addThreadsOption( Options options ) {
 
     protected void buildStandardOptions( Options options ) {
         AbstractCLI.log.debug( "Creating standard options" );
-        Option helpOpt = new Option( HELP_OPTION, "help", false, "Print this message" );
-        Option testOpt = new Option( TESTING_OPTION, "testing", false, "Use the test environment. This option must be passed before the command." );
-        options.addOption( helpOpt );
-        options.addOption( testOpt );
+        options.addOption( HELP_OPTION, "help", false, "Print this message" );
+        options.addOption( TESTING_OPTION, "testing", false, "Use the test environment. This option must be passed before the command." );
     }
 
     /**
@@ -290,51 +314,6 @@ protected void printHelp( Options options ) {
                 options, HelpFormatter.DEFAULT_LEFT_PAD, HelpFormatter.DEFAULT_DESC_PAD, AbstractCLI.FOOTER );
     }
 
-    /**
-     * This must be called in your main method. It triggers parsing of the command line and processing of the options.
-     * Check the error code to decide whether execution of your program should proceed.
-     *
-     * @param args args
-     * @return Exception; null if nothing went wrong.
-     */
-    private CommandLine processCommandLine( Options options, String[] args ) throws Exception {
-        /* COMMAND LINE PARSER STAGE */
-        DefaultParser parser = new DefaultParser();
-        String appVersion = Settings.getAppVersion();
-        if ( appVersion == null )
-            appVersion = "?";
-        System.err.println( "Gemma version " + appVersion );
-
-        if ( args == null ) {
-            this.printHelp( options );
-            throw new Exception( "No arguments" );
-        }
-
-        try {
-            return parser.parse( options, args );
-        } catch ( ParseException e ) {
-            if ( e instanceof MissingOptionException ) {
-                System.err.println( "Required option(s) were not supplied: " + e.getMessage() );
-            } else if ( e instanceof AlreadySelectedException ) {
-                System.err.println( "The option(s) " + e.getMessage() + " were already selected" );
-            } else if ( e instanceof MissingArgumentException ) {
-                System.err.println( "Missing argument: " + e.getMessage() );
-            } else if ( e instanceof UnrecognizedOptionException ) {
-                System.err.println( "Unrecognized option: " + e.getMessage() );
-            } else {
-                e.printStackTrace();
-            }
-
-            this.printHelp( options );
-
-            if ( AbstractCLI.log.isDebugEnabled() ) {
-                AbstractCLI.log.debug( e );
-            }
-
-            throw e;
-        }
-    }
-
     /**
      * Process command line options.
      * <p>
@@ -378,7 +357,7 @@ protected void addSuccessObject( Object successObject ) {
      */
     protected void addErrorObject( @Nullable Object errorObject, String message, Throwable throwable ) {
         errorObjects.add( new BatchProcessingResult( errorObject, message, throwable ) );
-        log.error( message, throwable );
+        log.error( "Error while processing " + ( errorObject != null ? errorObject : "unknown object" ) + ":\n\t" + message, throwable );
     }
 
     /**
@@ -387,7 +366,7 @@ protected void addErrorObject( @Nullable Object errorObject, String message, Thr
      */
     protected void addErrorObject( @Nullable Object errorObject, String message ) {
         errorObjects.add( new BatchProcessingResult( errorObject, message, null ) );
-        log.error( message );
+        log.error( "Error while processing " + ( errorObject != null ? errorObject : "unknown object" ) + ":\n\t" + message );
     }
 
     /**
@@ -396,7 +375,7 @@ protected void addErrorObject( @Nullable Object errorObject, String message ) {
      */
     protected void addErrorObject( @Nullable Object errorObject, Exception exception ) {
         errorObjects.add( new BatchProcessingResult( errorObject, exception.getMessage(), exception ) );
-        log.error( exception.getMessage(), exception );
+        log.error( "Error while processing " + ( errorObject != null ? errorObject : "unknown object" ), exception );
     }
 
     /**
diff --git a/gemma-cli/src/main/java/ubic/gemma/core/util/AbstractSpringAwareCLI.java b/gemma-cli/src/main/java/ubic/gemma/core/util/AbstractSpringAwareCLI.java
index 164e90462b..54c0439fa4 100644
--- a/gemma-cli/src/main/java/ubic/gemma/core/util/AbstractSpringAwareCLI.java
+++ b/gemma-cli/src/main/java/ubic/gemma/core/util/AbstractSpringAwareCLI.java
@@ -259,12 +259,10 @@ private String getPassword() {
                         return reader.readLine();
                     }
                 } else {
-                    log.error( "Could not read the password from '" + passwordCommand + "':\n"
+                    throw new IllegalArgumentException( "Could not read the password from '" + passwordCommand + "':\n"
                             + String.join( "\n", IOUtils.readLines( proc.getErrorStream(), StandardCharsets.UTF_8 ) ) );
-                    throw new IllegalArgumentException( "Could not read the password from '" + passwordCommand + "'." );
                 }
             } catch ( IOException | InterruptedException e ) {
-                log.error( "Could not read the password from '" + passwordCommand + "'.", e );
                 throw new IllegalArgumentException( "Could not read the password from '" + passwordCommand + "'.", e );
             }
         }
diff --git a/gemma-cli/src/main/java/ubic/gemma/core/util/CLI.java b/gemma-cli/src/main/java/ubic/gemma/core/util/CLI.java
index 0a9c1995af..9936dd9fa1 100644
--- a/gemma-cli/src/main/java/ubic/gemma/core/util/CLI.java
+++ b/gemma-cli/src/main/java/ubic/gemma/core/util/CLI.java
@@ -2,9 +2,11 @@
 
 import ubic.gemma.core.apps.GemmaCLI;
 
+import javax.annotation.Nullable;
+
 /**
  * Interface for CLI tools.
- *
+ * <p>
  * Implementing this interface will make the tool detectable by {@link GemmaCLI}.
  *
  * @author poirigui
@@ -14,24 +16,25 @@ public interface CLI {
     /**
      * A short memorable name for the command that can be used to locate this class.
      *
-     * @return name; if null, this will not be available as a shortcut command.
+     * @return name; if null or blank, this will not be available as a shortcut command.
      */
+    @Nullable
     String getCommandName();
 
     /**
      * Obtain a short description for this command explaining what it does.
      */
+    @Nullable
     String getShortDesc();
 
     /**
-     * Obtain the command grup.
-     *
-     * @return the command group for this CLI
+     * Obtain the command group for this CLI.
      */
     GemmaCLI.CommandGroup getCommandGroup();
 
     /**
      * Execute the given command given CLI arguments.
+     * @return an exit code
      */
     int executeCommand( String[] args );
 }
diff --git a/gemma-cli/src/test/java/ubic/gemma/core/apps/ExternalDatabaseUpdaterCliTest.java b/gemma-cli/src/test/java/ubic/gemma/core/apps/ExternalDatabaseUpdaterCliTest.java
index 3e7dc33b95..0b689e8d92 100644
--- a/gemma-cli/src/test/java/ubic/gemma/core/apps/ExternalDatabaseUpdaterCliTest.java
+++ b/gemma-cli/src/test/java/ubic/gemma/core/apps/ExternalDatabaseUpdaterCliTest.java
@@ -74,7 +74,7 @@ public void test() throws MalformedURLException {
         when( userManager.getCurrentUser() ).thenReturn( user );
         when( externalDatabaseService.findByNameWithAuditTrail( "test" ) ).thenReturn( ed );
         when( externalDatabaseService.findByNameWithExternalDatabases( "test2" ) ).thenReturn( ed2 );
-        externalDatabaseUpdaterCli.executeCommand( new String[] { "--name", "test", "--description", "Youpi!", "--release-note", "Yep", "--release-version", "123", "--release-url", "http://example.com/test", "--parent-database", "test2" } );
+        externalDatabaseUpdaterCli.executeCommand( new String[] { "--name", "test", "--description", "Youpi!", "--release", "--release-note", "Yep", "--release-version", "123", "--release-url", "http://example.com/test", "--parent-database", "test2" } );
         verify( externalDatabaseService ).findByNameWithExternalDatabases( "test2" );
         verify( externalDatabaseService ).findByNameWithAuditTrail( "test" );
         assertThat( ed.getDescription() ).isEqualTo( "Youpi!" );
diff --git a/gemma-cli/src/test/java/ubic/gemma/core/apps/RNASeqDataAddCliTest.java b/gemma-cli/src/test/java/ubic/gemma/core/apps/RNASeqDataAddCliTest.java
index 76f6d56794..641ce7bca9 100644
--- a/gemma-cli/src/test/java/ubic/gemma/core/apps/RNASeqDataAddCliTest.java
+++ b/gemma-cli/src/test/java/ubic/gemma/core/apps/RNASeqDataAddCliTest.java
@@ -20,6 +20,7 @@
 import ubic.gemma.model.expression.experiment.ExpressionExperiment;
 import ubic.gemma.persistence.service.expression.arrayDesign.ArrayDesignService;
 import ubic.gemma.persistence.service.expression.experiment.ExpressionExperimentService;
+import ubic.gemma.persistence.service.expression.experiment.ExpressionExperimentSetService;
 import ubic.gemma.persistence.service.genome.taxon.TaxonService;
 import ubic.gemma.persistence.util.TestComponent;
 
@@ -70,6 +71,11 @@ public ExpressionDataFileService expressionDataFileService() {
         public ArrayDesignService arrayDesignService() {
             return mock();
         }
+
+        @Bean
+        public ExpressionExperimentSetService expressionExperimentSetService() {
+            return mock();
+        }
     }
 
     @Autowired
@@ -94,6 +100,7 @@ public void setUp() throws IOException {
         ee = new ExpressionExperiment();
         rpkmFile = new ClassPathResource( "test.rpkm.txt" ).getFile().getAbsolutePath();
         when( expressionExperimentService.findByShortName( "GSE000001" ) ).thenReturn( ee );
+        when( expressionExperimentService.thawLite( any() ) ).thenAnswer( a -> a.getArgument( 0 ) );
         when( arrayDesignService.findByShortName( "test" ) ).thenReturn( ad );
     }
 
diff --git a/gemma-core/1.30.4.sql b/gemma-core/1.30.4.sql
new file mode 100644
index 0000000000..744f11787a
--- /dev/null
+++ b/gemma-core/1.30.4.sql
@@ -0,0 +1,4 @@
+alter table CHARACTERISTIC
+    add column class VARCHAR(255) after ID;
+alter table CHARACTERISTIC
+    add index class (class);
\ No newline at end of file
diff --git a/gemma-core/pom.xml b/gemma-core/pom.xml
index cc7fde0ea7..8ca2b25ad2 100644
--- a/gemma-core/pom.xml
+++ b/gemma-core/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <artifactId>gemma</artifactId>
         <groupId>gemma</groupId>
-        <version>1.30.3</version>
+        <version>1.30.4</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
     <artifactId>gemma-core</artifactId>
diff --git a/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/batcheffects/BatchConfoundUtils.java b/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/batcheffects/BatchConfoundUtils.java
index 28c0d8dfe7..4c8568933b 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/batcheffects/BatchConfoundUtils.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/batcheffects/BatchConfoundUtils.java
@@ -191,13 +191,17 @@ private static Collection<BatchConfound> factorBatchConfoundTest( BioAssaySet ee
                 }
 
                 for ( Long bm : bmToFv.keySet() ) {
-                    long fv = factorValueMembership.get( bm );
                     Long batch = batchMembership.get( bm );
-                    if ( batch == null ) {
+                    if ( batch == null || !batchIndexes.containsKey( batch ) ) {
                         log.warn( "No batch membership for : " + bm );
                         continue;
                     }
                     int batchIndex = batchIndexes.get( batch );
+                    Long fv = factorValueMembership.get( bm );
+                    if ( fv == null || !factorValueIndexes.containsKey( fv ) ) {
+                        log.warn( "No factor value for : " + bm );
+                        continue;
+                    }
                     int factorIndex = factorValueIndexes.get( fv );
                     counts[batchIndex][factorIndex]++;
                 }
diff --git a/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/batcheffects/BatchInfoPopulationServiceImpl.java b/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/batcheffects/BatchInfoPopulationServiceImpl.java
index 09c7a9e198..da61835bf1 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/batcheffects/BatchInfoPopulationServiceImpl.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/batcheffects/BatchInfoPopulationServiceImpl.java
@@ -23,10 +23,7 @@
 import ubic.gemma.core.analysis.util.ExperimentalDesignUtils;
 import ubic.gemma.core.loader.expression.geo.fetcher.RawDataFetcher;
 import ubic.gemma.model.common.auditAndSecurity.AuditEvent;
-import ubic.gemma.model.common.auditAndSecurity.eventType.BatchInformationFetchingEvent;
-import ubic.gemma.model.common.auditAndSecurity.eventType.FailedBatchInformationFetchingEvent;
-import ubic.gemma.model.common.auditAndSecurity.eventType.FailedBatchInformationMissingEvent;
-import ubic.gemma.model.common.auditAndSecurity.eventType.SingleBatchDeterminationEvent;
+import ubic.gemma.model.common.auditAndSecurity.eventType.*;
 import ubic.gemma.model.common.description.Characteristic;
 import ubic.gemma.model.common.description.DatabaseEntry;
 import ubic.gemma.model.common.description.LocalFile;
diff --git a/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/batcheffects/ExpressionExperimentBatchCorrectionServiceImpl.java b/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/batcheffects/ExpressionExperimentBatchCorrectionServiceImpl.java
index 25cc5addde..1f7236b9da 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/batcheffects/ExpressionExperimentBatchCorrectionServiceImpl.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/batcheffects/ExpressionExperimentBatchCorrectionServiceImpl.java
@@ -234,7 +234,13 @@ private ExpressionDataDoubleMatrix doComBat( ExpressionExperiment ee, Expression
          */
         ComBat<CompositeSequence, BioMaterial> comBat = new ComBat<>( matrix, designU );
 
-        DoubleMatrix2D results = comBat.run( true ); // false: NONPARAMETRIC
+        DoubleMatrix2D results = null;
+
+        try {
+            results = comBat.run( true ); // false: NONPARAMETRIC
+        } catch (ComBatException e ) {
+            throw new IllegalStateException("ComBat failed likely due to confounds between batching and other factors: " + e.getMessage(), e );
+        }
 
         // note these plots always reflect the parametric setup.
         comBat.plot( ee.getId() + "." + FileTools.cleanForFileName( ee.getShortName() ) ); // TEMPORARY?
diff --git a/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/svd/SVDServiceHelperImpl.java b/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/svd/SVDServiceHelperImpl.java
index 736e60251d..30cf04807d 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/svd/SVDServiceHelperImpl.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/svd/SVDServiceHelperImpl.java
@@ -86,31 +86,34 @@ public class SVDServiceHelperImpl implements SVDServiceHelper {
     private ExpressionExperimentService expressionExperimentService;
 
     /**
-     * Retrieve relationships between factors, biomaterials and factorvalues.
-     *
-     * @param bioMaterialFactorMap to be populated, of experimental factor -&gt; biomaterial ID -&gt; ID of the factor value
-     *                             (just an indicator)
+     * Retrieve relationships between factors, biomaterials and factor values.
+     * <p>
+     * Continuous factor values are converted to a {@link Double} if possible, otherwise the {@link FactorValue#getId()}
+     * is used. This is the case for categorical measurement.
+     * @param bioMaterialFactorMap to be populated, of experimental factor -&gt; biomaterial ID -&gt; factor value
+     *                             (double value if possible otherwise ID)
      * @param bm                   to populate for
      */
     public static void populateBMFMap( Map<ExperimentalFactor, Map<Long, Double>> bioMaterialFactorMap,
             BioMaterial bm ) {
         for ( FactorValue fv : bm.getFactorValues() ) {
-
             ExperimentalFactor experimentalFactor = fv.getExperimentalFactor();
-
-            if ( !bioMaterialFactorMap.containsKey( experimentalFactor ) ) {
-                bioMaterialFactorMap.put( experimentalFactor, new HashMap<Long, Double>() );
-            }
-
             double valueToStore;
             if ( fv.getMeasurement() != null ) {
-                try {
-                    valueToStore = Double.parseDouble( fv.getMeasurement().getValue() );
-                } catch ( NumberFormatException e ) {
-                    SVDServiceHelperImpl.log.warn( "Measurement wasn't a number for " + fv );
-                    valueToStore = Double.NaN;
+                switch ( fv.getMeasurement().getRepresentation() ) {
+                    case INT:
+                        valueToStore = Integer.parseInt( fv.getMeasurement().getValue() );
+                        break;
+                    case LONG:
+                        valueToStore = Long.parseLong( fv.getMeasurement().getValue() );
+                        break;
+                    case DOUBLE:
+                        valueToStore = Double.parseDouble( fv.getMeasurement().getValue() );
+                        break;
+                    default:
+                        // non-numerical measurement can be treated as categorical
+                        valueToStore = fv.getId().doubleValue();
                 }
-
             } else {
                 /*
                  * This is a hack so we don't need special datastructures for
@@ -118,7 +121,9 @@ public static void populateBMFMap( Map<ExperimentalFactor, Map<Long, Double>> bi
                  */
                 valueToStore = fv.getId().doubleValue();
             }
-            bioMaterialFactorMap.get( experimentalFactor ).put( bm.getId(), valueToStore );
+            bioMaterialFactorMap
+                    .computeIfAbsent( experimentalFactor, k -> new HashMap<>() )
+                    .put( bm.getId(), valueToStore );
         }
     }
 
diff --git a/gemma-core/src/main/java/ubic/gemma/core/analysis/service/AbstractFileService.java b/gemma-core/src/main/java/ubic/gemma/core/analysis/service/AbstractFileService.java
index 79139a04dc..99a25da64d 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/analysis/service/AbstractFileService.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/analysis/service/AbstractFileService.java
@@ -69,9 +69,11 @@ protected String getSubDelimiter() {
     protected String format( @Nullable Double d ) {
         if ( d == null ) {
             return "";
-        } else if ( d < 1e-4 ) {
+        } else if ( d == 0.0 ) {
+            return "0.0";
+        } else if ( Math.abs( d ) < 1e-4 ) {
             return smallNumberFormat.format( d );
-        } else if ( d < 1e3 ) {
+        } else if ( Math.abs( d ) < 1e3 ) {
             return midNumberFormat.format( d );
         } else {
             return largeNumberFormat.format( d );
diff --git a/gemma-core/src/main/java/ubic/gemma/core/analysis/service/ArrayDesignAnnotationServiceImpl.java b/gemma-core/src/main/java/ubic/gemma/core/analysis/service/ArrayDesignAnnotationServiceImpl.java
index e6e678945b..7e564a24ae 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/analysis/service/ArrayDesignAnnotationServiceImpl.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/analysis/service/ArrayDesignAnnotationServiceImpl.java
@@ -648,7 +648,7 @@ private Writer initOutputFile( ArrayDesign arrayDesign, String fileBaseName, boo
         buf.append( "# " ).append( arrayDesign.getPrimaryTaxon().getScientificName() ).append( "\n" );
 
         writer.write( buf.toString() );
-        writer.write( "ProbeName\tGeneSymbols\tGeneNames\tGOTerms\tGemmaIDs\tNCBIids\n" );
+        writer.write( "ElementName\tGeneSymbols\tGeneNames\tGOTerms\tGemmaIDs\tNCBIids\n" );
 
         return writer;
     }
diff --git a/gemma-core/src/main/java/ubic/gemma/core/loader/expression/arrayDesign/ArrayDesignProbeMapperServiceImpl.java b/gemma-core/src/main/java/ubic/gemma/core/loader/expression/arrayDesign/ArrayDesignProbeMapperServiceImpl.java
index 862a53939a..20967aa521 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/loader/expression/arrayDesign/ArrayDesignProbeMapperServiceImpl.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/loader/expression/arrayDesign/ArrayDesignProbeMapperServiceImpl.java
@@ -25,6 +25,7 @@
 import org.springframework.core.task.TaskExecutor;
 import org.springframework.security.concurrent.DelegatingSecurityContextRunnable;
 import org.springframework.stereotype.Component;
+import org.springframework.transaction.annotation.Propagation;
 import org.springframework.transaction.annotation.Transactional;
 import ubic.gemma.core.analysis.report.ArrayDesignReportService;
 import ubic.gemma.core.analysis.sequence.ProbeMapUtils;
@@ -123,13 +124,13 @@ public void printResult( CompositeSequence compositeSequence, Collection<BlatAss
     }
 
     @Override
-    @Transactional
+    @Transactional(propagation = Propagation.NEVER)
     public void processArrayDesign( ArrayDesign arrayDesign ) {
         this.processArrayDesign( arrayDesign, new ProbeMapperConfig(), true );
     }
 
     @Override
-    @Transactional
+    @Transactional(propagation = Propagation.NEVER)
     public void processArrayDesign( ArrayDesign arrayDesign, ProbeMapperConfig config, boolean useDB ) {
 
         assert config != null;
@@ -217,7 +218,7 @@ public void processArrayDesign( ArrayDesign arrayDesign, ProbeMapperConfig confi
     }
 
     @Override
-    @Transactional
+    @Transactional(propagation = Propagation.NEVER)
     public void processArrayDesign( ArrayDesign arrayDesign, Taxon taxon, File source, ExternalDatabase sourceDB,
             boolean ncbiIds ) throws IOException {
 
@@ -380,7 +381,7 @@ public void processArrayDesign( ArrayDesign arrayDesign, Taxon taxon, File sourc
     }
 
     @Override
-    @Transactional
+    @Transactional(propagation = Propagation.NEVER)
     public Map<String, Collection<BlatAssociation>> processCompositeSequence( ProbeMapperConfig config, Taxon taxon,
             GoldenPathSequenceAnalysis goldenPathDb, CompositeSequence compositeSequence ) {
         BioSequence bs = compositeSequence.getBiologicalCharacteristic();
diff --git a/gemma-core/src/main/java/ubic/gemma/core/loader/expression/geo/GeoConverterImpl.java b/gemma-core/src/main/java/ubic/gemma/core/loader/expression/geo/GeoConverterImpl.java
index a046dae0f9..00a8752f5c 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/loader/expression/geo/GeoConverterImpl.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/loader/expression/geo/GeoConverterImpl.java
@@ -272,7 +272,7 @@ public void convertSubsetToExperimentalFactor( ExpressionExperiment expExp, GeoS
      *
      * @param  platformTaxa Collection of taxa that were given on the GEO array submission as platform taxa
      * @param  probeTaxa    Collection of taxa strings defining the taxon of each probe on the array.
-     * @return              Primary taxon of array as determined by this method
+     * @return Primary taxon of array as determined by this method
      */
     @Override
     public Taxon getPrimaryArrayTaxon( Collection<Taxon> platformTaxa, Collection<String> probeTaxa )
@@ -467,12 +467,12 @@ public String toString() {
 
         return buf.toString();
     }
-    
+
     /**
      * Form title (will be experiment name) and ensure is valid length
-     * 
+     *
      * @param  appendix can be null; e.g. species or platform name added when we are splitting up a record.
-     * @return          title
+     * @return title
      */
     protected String makeTitle( String title, String appendix ) {
 
@@ -523,7 +523,7 @@ private void addFactorValueToBioMaterial( ExpressionExperiment expExp, GeoSubset
     /**
      * @param  bioMaterial        BA
      * @param  experimentalFactor exp factor
-     * @return                    true if the biomaterial already has a factorvalue for the given experimentalFactor;
+     * @return true if the biomaterial already has a factorvalue for the given experimentalFactor;
      *                            false otherwise.
      */
     private boolean alreadyHasFactorValueForFactor( BioMaterial bioMaterial, ExperimentalFactor experimentalFactor ) {
@@ -552,9 +552,9 @@ private void checkForDataToSkip( GeoSeries series, Collection<String> dataSetsTo
             if ( dataset.getExperimentType().equals( ExperimentType.arrayCGH ) || dataset.getExperimentType()
                     .equals( ExperimentType.ChIPChip )
                     || dataset.getExperimentType()
-                            .equals( ExperimentType.geneExpressionSAGEbased )
+                    .equals( ExperimentType.geneExpressionSAGEbased )
                     || dataset.getExperimentType()
-                            .equals( ExperimentType.Other ) ) {
+                    .equals( ExperimentType.Other ) ) {
                 GeoConverterImpl.log
                         .warn( "Gemma does not know how to handle experiment type=" + dataset.getExperimentType() );
 
@@ -582,7 +582,7 @@ private void checkForDataToSkip( GeoSeries series, Collection<String> dataSetsTo
                     // have to drill down.
                     if ( sample.getLibStrategy().equals( "RNA-Seq" ) || sample.getLibStrategy().equals( "ncRNA-Seq" )
                             || sample.getLibStrategy().equals( "miRNA-Seq" ) || sample.getLibStrategy()
-                                    .equals( "ssRNA-seq" ) ) {
+                            .equals( "ssRNA-seq" ) ) {
                         continue;
                     }
                 }
@@ -772,7 +772,7 @@ private void convertChannel( GeoSample sample, GeoChannel channel, BioMaterial b
     /**
      * GEO gives sample descriptors (provided by submitters) that we try to parse into Characteristics associated with
      * the BioMaterial.
-     * 
+     *
      * The format for these strings varies idiosyncratically. This parser is designed to handle things like:
      * <ul>
      * <li>Sex=M or Sex:M
@@ -780,16 +780,16 @@ private void convertChannel( GeoSample sample, GeoChannel channel, BioMaterial b
      * <li>Sex:M,Tissue:brain
      * <li>and variants thereof.
      * </ul>
-     * 
+     *
      * <p>
      * Terms are not mapped unless we recognized the category, acceptable choices being
      * a hard-coded and limited list of terms (See GeoVariable).
-     * 
+     *
      * <p>
      * This method does not do anything too sophisticated (e.g., stemming),
      * and the mappings are created manually, so many strings will not be matched.
      * Because characteristics on biomaterials are not mission-critical, it's not worth too much effort.
-     * 
+     *
      * @param characteristic string to be parsed
      * @param bioMaterial    to which characteristics will be added
      */
@@ -797,6 +797,7 @@ void parseGEOSampleCharacteristicString( String characteristic, BioMaterial bioM
         /*
          * Sometimes strings are like Age :8 weeks; Sex: M so we should first split on ";" - sometimes "," is used.
          */
+        //log.info( characteristic );
         String[] topFields = characteristic.split( "[;,]" );
 
         for ( String field : topFields ) {
@@ -804,7 +805,10 @@ void parseGEOSampleCharacteristicString( String characteristic, BioMaterial bioM
             /*
              * Sometimes values are like Age:8 weeks, so we can try to convert them.
              */
-            String[] fields = field.split( "[:=]" ); // sometimes it is '='
+            String[] fields = field.split( ":", 2 ); // sometimes it is '=' ,but not allowed any more see https://www.ncbi.nlm.nih.gov/geo/info/soft.html#guidelines_tabs
+            if ( fields.length != 2 ) {
+                fields = field.split( "=", 2 ); // this shouldn't occur, but is present in some old GEO records apparently
+            }
             String defaultDescription = "GEO Sample characteristic";
             if ( fields.length == 2 ) {
 
@@ -813,17 +817,21 @@ void parseGEOSampleCharacteristicString( String characteristic, BioMaterial bioM
                 value = value.replaceFirst( "^(human|mouse|rat|murine|mus musculus|homo sapiens)\\s", "" );
 
                 Characteristic gemmaChar = Characteristic.Factory.newInstance();
-                gemmaChar.setOriginalValue( value );
+                gemmaChar.setOriginalValue( field ); // always retain the original thing, unsplit.
                 gemmaChar.setEvidenceCode( GOEvidenceCode.IIA );
                 gemmaChar.setDescription( defaultDescription );
 
                 VariableType vartype = GeoVariable.convertStringToType( category );
-                if ( vartype == null ) {
+                if ( vartype == null || vartype.equals( VariableType.other ) ) {
                     log.debug( "Could not parse into VariableType: " + category + " (in: " + characteristic + ")" );
-                    this.doFallback( bioMaterial, value, defaultDescription );
+                    gemmaChar.setCategory( category ); // This is not one of our "standard" categories, but it's okay
+                    gemmaChar.setValue( value );
+                    gemmaChar.setDescription( defaultDescription );
+                    bioMaterial.getCharacteristics().add( gemmaChar );
                     continue;
                 }
 
+                assert !vartype.equals( VariableType.other );
                 this.convertVariableType( gemmaChar, vartype );
 
                 CharacteristicBasicValueObject mappedValueTerm = ontologyLookupSampleCharacteristic( value, gemmaChar.getCategory() );
@@ -840,12 +848,13 @@ void parseGEOSampleCharacteristicString( String characteristic, BioMaterial bioM
                     }
                     bioMaterial.getCharacteristics().add( gemmaChar );
                 } catch ( Exception e ) {
-                    // conversion didn't work, fall back.
+                    // conversion didn't work, fall back. (not sure why this would happen so adding logging)
+                    log.warn( "Could not convert " + field + " to characteristic ", e );
                     this.doFallback( bioMaterial, value, defaultDescription );
                 }
 
             } else {
-                // no colon, just use raw (same as fallback above)
+                // no colon or equals sign, just use raw (same as fallback above)
                 this.doFallback( bioMaterial, field, defaultDescription );
             }
         }
@@ -1122,7 +1131,7 @@ private RawExpressionDataVector convertDesignElementDataVector( GeoPlatform geoP
     /**
      * @param  datasetSamples List of GeoSamples to be matched up with BioAssays.
      * @param  expExp         ExpresssionExperiment
-     * @return                BioAssayDimension representing the samples.
+     * @return BioAssayDimension representing the samples.
      */
     private BioAssayDimension convertGeoSampleList( List<GeoSample> datasetSamples, ExpressionExperiment expExp ) {
         BioAssayDimension resultBioAssayDimension = BioAssayDimension.Factory.newInstance();
@@ -1151,7 +1160,7 @@ private BioAssayDimension convertGeoSampleList( List<GeoSample> datasetSamples,
      * Given an organisms name from GEO, create or find the taxon in the DB.
      *
      * @param  taxonScientificName name as provided by GEO presumed to be a scientific name
-     * @return                     Taxon details
+     * @return Taxon details
      */
     private Taxon convertOrganismToTaxon( String taxonScientificName ) {
         assert taxonScientificName != null;
@@ -1271,7 +1280,7 @@ private ArrayDesign convertPlatform( GeoPlatform platform ) {
      * @param  sequences           sequences
      * @param  probeOrganism       probe organism
      * @param  primaryTaxon        primary taxon
-     * @return                     true if we expect this platform to be fully usable or whether some or all elements
+     * @return true if we expect this platform to be fully usable or whether some or all elements
      *                             may have been omitted
      *                             in our parse (so how freaked out later in processing should we be if an element in
      *                             the data doesn't
@@ -1308,8 +1317,8 @@ private boolean convertPlatformElements( String identifier, GeoPlatform platform
 
         assert externalRefs == null
                 || externalRefs.iterator().next().size() == identifiers.size() : "Unequal numbers of identifiers and external references! "
-                        + externalRefs.iterator().next().size()
-                        + " != " + identifiers.size();
+                + externalRefs.iterator().next().size()
+                + " != " + identifiers.size();
 
         if ( GeoConverterImpl.log.isDebugEnabled() ) {
             GeoConverterImpl.log
@@ -1530,7 +1539,7 @@ private void setBsProps( GeoPlatform platform, ExternalDatabase externalDb, List
      *
      * @param  platform             GEO platform details
      * @param  probeTaxonColumnName Column name of probe taxa
-     * @return                      List of taxa on platform
+     * @return List of taxa on platform
      */
     private Collection<Taxon> convertPlatformOrganisms( GeoPlatform platform, String probeTaxonColumnName ) {
         Collection<String> organisms = platform.getOrganisms();
@@ -1569,7 +1578,7 @@ private Collection<Taxon> convertPlatformOrganisms( GeoPlatform platform, String
      *
      * @param  probeOrganism            scientific name or common name of organism associated to a
      *                                  biosequence.
-     * @return                          Taxon of biosequence.
+     * @return Taxon of biosequence.
      * @throws IllegalArgumentException taxon supplied has not been processed before, it does not match the scientific
      *                                  names used in platform definition and does not match a known common name in the
      *                                  database.
@@ -1667,7 +1676,7 @@ private Characteristic convertReplicatationType( ReplicationType repType ) {
 
     /**
      * @param  replication Convert a variable into a ExperimentalFactor
-     * @return             exp factor
+     * @return exp factor
      */
     private ExperimentalFactor convertReplicationToFactor( GeoReplication replication ) {
         GeoConverterImpl.log.debug( "Converting replication " + replication.getType() );
@@ -1701,7 +1710,7 @@ private void convertReplicationToFactorValue( GeoReplication replication, Experi
      * @param  sample             sample
      * @param  bioMaterial        BA
      * @param  experimentalDesign experimental design
-     * @return                    BA
+     * @return BA
      */
     private BioAssay convertSample( GeoSample sample, BioMaterial bioMaterial, ExperimentalDesign experimentalDesign ) {
         if ( sample == null ) {
@@ -1792,7 +1801,7 @@ private BioAssay convertSample( GeoSample sample, BioMaterial bioMaterial, Exper
      * GSE2939_biomaterial_1|GSM12393|GSN12394.
      *
      * @param  series series
-     * @return        ees
+     * @return ees
      */
     private Collection<ExpressionExperiment> convertSeries( GeoSeries series ) {
 
@@ -1862,7 +1871,7 @@ private void convertSeriesDataVectors( GeoSeries geoSeries, ExpressionExperiment
     /**
      * Main method that converts a single (mono-species) GEO series to an ExpressionExperiment.
      *
-     * @return        ExpressionExperiment, or null if the series cannot be converted (wrong sample type, etc.)
+     * @return ExpressionExperiment, or null if the series cannot be converted (wrong sample type, etc.)
      */
     private ExpressionExperiment convertSeriesSingle( GeoSeries series ) {
         if ( series == null )
@@ -2177,7 +2186,7 @@ private void convertSubsetAssociations( ExpressionExperiment result, GeoDataset
      *
      * @param  experimentalFactor ef
      * @param  geoSubSet          geo subset
-     * @return                    FV
+     * @return FV
      */
     private FactorValue convertSubsetDescriptionToFactorValue( GeoSubset geoSubSet,
             ExperimentalFactor experimentalFactor ) {
@@ -2225,7 +2234,7 @@ private FactorValue convertTypeToFactorValue( VariableType type, String value )
 
     /**
      * @param  variable Convert a variable into a ExperimentalFactor
-     * @return          ef
+     * @return ef
      */
     private ExperimentalFactor convertVariableToFactor( GeoVariable variable ) {
         GeoConverterImpl.log.debug( "Converting variable " + variable.getType() );
@@ -2243,7 +2252,7 @@ private ExperimentalFactor convertVariableToFactor( GeoVariable variable ) {
 
     /**
      * @param  variable variable
-     * @return          Category will be filled in with a URI but value will just be plain text.
+     * @return Category will be filled in with a URI but value will just be plain text.
      */
     private FactorValue convertVariableToFactorValue( GeoVariable variable ) {
         GeoConverterImpl.log.info( "Converting variable " + variable );
@@ -2313,7 +2322,7 @@ private void convertVariableType( Characteristic c, VariableType varType ) {
             uri = "http://www.ebi.ac.uk/efo/EFO_0000651";
             term = "phenotype";
         } else if ( varType.equals( VariableType.other ) ) {
-            // NO-OP
+            // NO-OP. We'll fill this in with something later.
         } else if ( varType.equals( VariableType.protocol ) ) {
             uri = "http://purl.obolibrary.org/obo/OBI_0000272";
             term = "protocol";
@@ -2639,7 +2648,7 @@ private Collection<String> determinePlatformExternalReferenceIdentifier( GeoPlat
      * the species/organism used to create probe.
      *
      * @param  platform Parsed GEO platform details.
-     * @return          Column name in GEO used to identify column containing species/organism used to create probe
+     * @return Column name in GEO used to identify column containing species/organism used to create probe
      */
     private String determinePlatformProbeOrganismColumn( GeoPlatform platform ) {
         Collection<String> columnNames = platform.getColumnNames();
@@ -2689,7 +2698,8 @@ private FactorValue findMatchingExperimentalFactorValue( Collection<Experimental
         Characteristic c = characteristics.iterator().next();
 
         FactorValue matchingFactorValue = null;
-        factors: for ( ExperimentalFactor factor : experimentalFactors ) {
+        factors:
+        for ( ExperimentalFactor factor : experimentalFactors ) {
             for ( FactorValue fv : factor.getFactorValues() ) {
                 for ( Characteristic m : fv.getCharacteristics() ) {
                     if ( m.getCategory().equals( c.getCategory() ) && m.getValue().equals( c.getValue() ) ) {
@@ -2953,7 +2963,7 @@ private boolean isPopulated( Map<String, List<Object>> dataVectors ) {
      * use, we keep it.
      *
      * @param  series series
-     * @return        is usable
+     * @return is usable
      */
     private boolean isUsable( GeoSeries series ) {
 
@@ -2968,7 +2978,7 @@ private boolean isUsable( GeoSeries series ) {
      * @param  datasetSamples           The samples we want to get data for. These should all have been run on the same
      *                                  platform.
      * @param  quantitationTypeIndex    - first index is 0
-     * @return                          A map of Strings (design element names) to Lists of Strings containing the data.
+     * @return A map of Strings (design element names) to Lists of Strings containing the data.
      * @throws IllegalArgumentException if the columnNumber is not valid
      */
     private Map<String, List<Object>> makeDataVectors( GeoValues values, List<GeoSample> datasetSamples,
diff --git a/gemma-core/src/main/java/ubic/gemma/core/loader/expression/geo/GeoFamilyParser.java b/gemma-core/src/main/java/ubic/gemma/core/loader/expression/geo/GeoFamilyParser.java
index b0b0a6b218..94695f8807 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/loader/expression/geo/GeoFamilyParser.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/loader/expression/geo/GeoFamilyParser.java
@@ -1344,7 +1344,7 @@ private void parseSampleLine( String line, String value ) {
         } else if ( this.startsWithIgnoreCase( line, "!Sample_characteristics" ) ) {
             int channel = this.extractChannelNumber( line );
             GeoSample sample = results.getSampleMap().get( currentSampleAccession );
-            sample.getChannel( channel ).addCharacteristic( value );
+            sample.getChannel( channel ).addCharacteristic( value ); // might be something like "rin: 7.2"
         } else if ( this.startsWithIgnoreCase( line, "!Sample_platform_id" ) ) {
             this.sampleSet( currentSampleAccession, GeoSample::setId, value );
             if ( results.getPlatformMap().containsKey( value ) ) {
diff --git a/gemma-core/src/main/java/ubic/gemma/model/common/auditAndSecurity/eventType/BatchInformationEvent.java b/gemma-core/src/main/java/ubic/gemma/model/common/auditAndSecurity/eventType/BatchInformationEvent.java
new file mode 100644
index 0000000000..6ebe72d4cc
--- /dev/null
+++ b/gemma-core/src/main/java/ubic/gemma/model/common/auditAndSecurity/eventType/BatchInformationEvent.java
@@ -0,0 +1,10 @@
+package ubic.gemma.model.common.auditAndSecurity.eventType;
+
+/**
+ * Abstract class for events related to batch information.
+ * <p>
+ * These events indicate whether batch information is available, missing or problematic.
+ * @author poirigui
+ */
+public abstract class BatchInformationEvent extends ExpressionExperimentAnalysisEvent {
+}
diff --git a/gemma-core/src/main/java/ubic/gemma/model/common/auditAndSecurity/eventType/BatchInformationFetchingEvent.java b/gemma-core/src/main/java/ubic/gemma/model/common/auditAndSecurity/eventType/BatchInformationFetchingEvent.java
index 34a853086c..c66294757c 100644
--- a/gemma-core/src/main/java/ubic/gemma/model/common/auditAndSecurity/eventType/BatchInformationFetchingEvent.java
+++ b/gemma-core/src/main/java/ubic/gemma/model/common/auditAndSecurity/eventType/BatchInformationFetchingEvent.java
@@ -19,16 +19,18 @@
 package ubic.gemma.model.common.auditAndSecurity.eventType;
 
 /**
+ * Indicates that batch information was successfully obtained.
  * <p>
- * Indicates that batch information was obtained
+ * Successful obtention is not indicative of problematic batch information and the {@link FailedBatchInformationFetchingEvent}
+ * for representing such cases.
  * </p>
+ * Use {@link BatchInformationMissingEvent} for missing batch information.
+ * @author poirigui
  */
-public class BatchInformationFetchingEvent
-        extends ubic.gemma.model.common.auditAndSecurity.eventType.ExpressionExperimentAnalysisEvent {
+public class BatchInformationFetchingEvent extends BatchInformationEvent {
 
     /**
      * The serial version UID of this class. Needed for serialization.
      */
     private static final long serialVersionUID = 4635587632131568136L;
-
 }
\ No newline at end of file
diff --git a/gemma-core/src/main/java/ubic/gemma/model/common/auditAndSecurity/eventType/BatchInformationMissingEvent.java b/gemma-core/src/main/java/ubic/gemma/model/common/auditAndSecurity/eventType/BatchInformationMissingEvent.java
new file mode 100644
index 0000000000..d89bf99497
--- /dev/null
+++ b/gemma-core/src/main/java/ubic/gemma/model/common/auditAndSecurity/eventType/BatchInformationMissingEvent.java
@@ -0,0 +1,10 @@
+package ubic.gemma.model.common.auditAndSecurity.eventType;
+
+/**
+ * Indicate that batch information has been looked for and was missing.
+ * <p>
+ * This does not indicate that the batch information is problematic unlike {@link FailedBatchInformationFetchingEvent}
+ * @author poirigui
+ */
+public class BatchInformationMissingEvent extends BatchInformationEvent {
+}
diff --git a/gemma-core/src/main/java/ubic/gemma/model/common/auditAndSecurity/eventType/FailedBatchInformationFetchingEvent.java b/gemma-core/src/main/java/ubic/gemma/model/common/auditAndSecurity/eventType/FailedBatchInformationFetchingEvent.java
index 5a9dd5ec66..ba0c3cb850 100644
--- a/gemma-core/src/main/java/ubic/gemma/model/common/auditAndSecurity/eventType/FailedBatchInformationFetchingEvent.java
+++ b/gemma-core/src/main/java/ubic/gemma/model/common/auditAndSecurity/eventType/FailedBatchInformationFetchingEvent.java
@@ -19,17 +19,9 @@
 package ubic.gemma.model.common.auditAndSecurity.eventType;
 
 /**
+ * Indicates that the attempt to get batch information failed due to an error.
  * <p>
- * Indicates that the attempt to get batch information failed due to an error. If the problem is that the information
- * just isn't available, use the specific subtype of this.
- * </p>
+ * If the problem is that the information just isn't available, use {@link BatchInformationMissingEvent} instead.
  */
-public class FailedBatchInformationFetchingEvent
-        extends ubic.gemma.model.common.auditAndSecurity.eventType.BatchInformationFetchingEvent {
-
-    /**
-     * The serial version UID of this class. Needed for serialization.
-     */
-    private static final long serialVersionUID = -3776787605548724345L;
-
+public class FailedBatchInformationFetchingEvent extends BatchInformationFetchingEvent {
 }
\ No newline at end of file
diff --git a/gemma-core/src/main/java/ubic/gemma/model/common/auditAndSecurity/eventType/FailedBatchInformationMissingEvent.java b/gemma-core/src/main/java/ubic/gemma/model/common/auditAndSecurity/eventType/FailedBatchInformationMissingEvent.java
index 5f238d12e8..e02bb95e6d 100644
--- a/gemma-core/src/main/java/ubic/gemma/model/common/auditAndSecurity/eventType/FailedBatchInformationMissingEvent.java
+++ b/gemma-core/src/main/java/ubic/gemma/model/common/auditAndSecurity/eventType/FailedBatchInformationMissingEvent.java
@@ -19,13 +19,13 @@
 package ubic.gemma.model.common.auditAndSecurity.eventType;
 
 /**
- * <p>
  * Use to indicate that the batch information has been successfully looked for, but is not available, so we shouldn't
  * look again. Do not use to indicate other types of failure such as an unsupported raw data type.
- * </p>
+ * <p>
+ * @deprecated use {@link BatchInformationMissingEvent}
  */
-public class FailedBatchInformationMissingEvent
-        extends ubic.gemma.model.common.auditAndSecurity.eventType.FailedBatchInformationFetchingEvent {
+@Deprecated
+public class FailedBatchInformationMissingEvent extends BatchInformationMissingEvent {
 
     /**
      * The serial version UID of this class. Needed for serialization.
diff --git a/gemma-core/src/main/java/ubic/gemma/model/expression/experiment/ExpressionExperimentDetailsValueObject.java b/gemma-core/src/main/java/ubic/gemma/model/expression/experiment/ExpressionExperimentDetailsValueObject.java
index da6632e9e2..a8bf35a534 100644
--- a/gemma-core/src/main/java/ubic/gemma/model/expression/experiment/ExpressionExperimentDetailsValueObject.java
+++ b/gemma-core/src/main/java/ubic/gemma/model/expression/experiment/ExpressionExperimentDetailsValueObject.java
@@ -277,6 +277,10 @@ public Collection<ExpressionExperimentValueObject> getOtherParts() {
         return otherParts;
     }
 
+    public void setOtherParts( Collection<ExpressionExperimentValueObject> otherParts ) {
+        this.otherParts = otherParts;
+    }
+
     public String getPcaAnalysisEventType() {
         return pcaAnalysisEventType;
     }
diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/TableMaintenanceUtilImpl.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/TableMaintenanceUtilImpl.java
index 6c3d304f40..6c6c3a57d7 100644
--- a/gemma-core/src/main/java/ubic/gemma/persistence/service/TableMaintenanceUtilImpl.java
+++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/TableMaintenanceUtilImpl.java
@@ -97,7 +97,8 @@ public class TableMaintenanceUtilImpl implements TableMaintenanceUtil {
                     + "join EXPERIMENTAL_FACTOR EF on EXPERIMENTAL_DESIGN.ID = EF.EXPERIMENTAL_DESIGN_FK "
                     + "join FACTOR_VALUE FV on FV.EXPERIMENTAL_FACTOR_FK = EF.ID "
                     + "join CHARACTERISTIC C on FV.ID = C.FACTOR_VALUE_FK "
-                    + "where I.class = 'ExpressionExperiment'";
+                    // exclude statements from being added (see https://github.com/PavlidisLab/Gemma/issues/909 for details)
+                    + "where I.class = 'ExpressionExperiment' and C.class is null";
     private static final Path DEFAULT_GENE2CS_INFO_PATH = Paths.get( Settings.getString( "gemma.appdata.home" ), "DbReports", "gene2cs.info" );
     private static final Log log = LogFactory.getLog( TableMaintenanceUtil.class.getName() );
     @Autowired
diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/diff/DifferentialExpressionAnalysisDaoImpl.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/diff/DifferentialExpressionAnalysisDaoImpl.java
index 82bb4e86a4..a4ee6d20c2 100644
--- a/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/diff/DifferentialExpressionAnalysisDaoImpl.java
+++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/diff/DifferentialExpressionAnalysisDaoImpl.java
@@ -198,6 +198,8 @@ public DifferentialExpressionAnalysis create( DifferentialExpressionAnalysis ent
             insertRowsAndAssignGeneratedKeys( INSERT_CONTRAST_SQL, insertContrastStmt, contrasts, contrastPersister, ( SessionImplementor ) session );
         } );
 
+        getSessionFactory().getCurrentSession().flush();
+
         return finalEntity;
     }
 
diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/common/description/ExternalDatabaseService.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/common/description/ExternalDatabaseService.java
index 9f0a508ec0..fbde07aa34 100644
--- a/gemma-core/src/main/java/ubic/gemma/persistence/service/common/description/ExternalDatabaseService.java
+++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/common/description/ExternalDatabaseService.java
@@ -33,6 +33,9 @@
  */
 public interface ExternalDatabaseService extends BaseService<ExternalDatabase> {
 
+    @Secured({ "GROUP_ADMIN" })
+    Collection<ExternalDatabase> loadAllWithAuditTrail();
+
     @Override
     @Secured({ "GROUP_ADMIN" })
     Collection<ExternalDatabase> create( Collection<ExternalDatabase> entities );
diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/common/description/ExternalDatabaseServiceImpl.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/common/description/ExternalDatabaseServiceImpl.java
index 122f151ff3..594b1479b5 100644
--- a/gemma-core/src/main/java/ubic/gemma/persistence/service/common/description/ExternalDatabaseServiceImpl.java
+++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/common/description/ExternalDatabaseServiceImpl.java
@@ -30,10 +30,7 @@
 
 import javax.annotation.Nullable;
 import java.net.URL;
-import java.util.Comparator;
-import java.util.Date;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 import java.util.stream.Collectors;
 
 /**
@@ -51,6 +48,14 @@ public ExternalDatabaseServiceImpl( ExternalDatabaseDao mainDao ) {
         externalDatabaseDao = mainDao;
     }
 
+    @Override
+    @Transactional(readOnly = true)
+    public Collection<ExternalDatabase> loadAllWithAuditTrail() {
+        Collection<ExternalDatabase> eds = externalDatabaseDao.loadAll();
+        eds.forEach( ed -> Hibernate.initialize( ed.getAuditTrail() ) );
+        return eds;
+    }
+
     @Override
     @Transactional(readOnly = true)
     public ExternalDatabase loadWithExternalDatabases( Long id ) {
diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/experiment/ExpressionExperimentDaoImpl.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/experiment/ExpressionExperimentDaoImpl.java
index b7df0b34a5..4ea295fabd 100644
--- a/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/experiment/ExpressionExperimentDaoImpl.java
+++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/experiment/ExpressionExperimentDaoImpl.java
@@ -649,21 +649,39 @@ public Map<Characteristic, Long> getCategoriesUsageFrequency( @Nullable Collecti
             excludedTermUris = new HashSet<>( excludedTermUris );
             excludedTermUris.removeAll( retainedTermUris );
         }
-        Query q = getSessionFactory().getCurrentSession().createSQLQuery(
-                        "select T.CATEGORY, T.CATEGORY_URI, count(distinct T.EXPRESSION_EXPERIMENT_FK) as EE_COUNT from EXPRESSION_EXPERIMENT2CHARACTERISTIC T "
-                                + AclQueryUtils.formNativeAclJoinClause( "T.EXPRESSION_EXPERIMENT_FK" ) + " "
-                                + "where T.ID is not null "
-                                + ( eeIds != null ? " and T.EXPRESSION_EXPERIMENT_FK in :eeIds" : "" )
-                                // FIXME: there's a bug in Hibernate that that prevents it from producing proper tuples
-                                //        for the excludedCategoryUris, retainedTermUris parameters, so we add explicit
-                                //        parentheses
-                                + ( excludedCategoryUris != null && !excludedCategoryUris.isEmpty() ? " and (T.CATEGORY_URI not in (:excludedCategoryUris)" + ( retainedTermUris != null && !retainedTermUris.isEmpty() ? " or T.VALUE_URI in (:retainedTermUris)" : "" ) + ")" : "" )
-                                + ( excludedTermUris != null && !excludedTermUris.isEmpty() ? " and T.VALUE_URI not in :excludedTermUris" : "" )
-                                + AclQueryUtils.formNativeAclRestrictionClause( ( SessionFactoryImplementor ) getSessionFactory() ) + " "
-                                + "group by COALESCE(T.CATEGORY_URI, T.CATEGORY) "
-                                + "order by EE_COUNT desc" )
-                .addScalar( "T.CATEGORY", StandardBasicTypes.STRING )
-                .addScalar( "T.CATEGORY_URI", StandardBasicTypes.STRING )
+        boolean excludeFreeTextCategories = false;
+        boolean excludeUncategorized = false;
+        if ( excludedCategoryUris != null ) {
+            if ( excludedCategoryUris.contains( null ) ) {
+                excludeFreeTextCategories = true;
+                excludedCategoryUris = excludedCategoryUris.stream().filter( Objects::nonNull ).collect( Collectors.toList() );
+            }
+            if ( excludedCategoryUris.contains( UNCATEGORIZED ) ) {
+                excludeUncategorized = true;
+                excludedCategoryUris = excludedCategoryUris.stream().filter( c -> !c.equals( UNCATEGORIZED ) ).collect( Collectors.toList() );
+            }
+        }
+        boolean excludeFreeTextTerms = false;
+        if ( excludedTermUris != null ) {
+            if ( excludedTermUris.contains( null ) ) {
+                excludeFreeTextTerms = true;
+                excludedTermUris = excludedTermUris.stream().filter( Objects::nonNull ).collect( Collectors.toList() );
+            }
+        }
+        String query = "select max(T.CATEGORY) as CATEGORY, max(T.CATEGORY_URI) as CATEGORY_URI, count(distinct T.EXPRESSION_EXPERIMENT_FK) as EE_COUNT from EXPRESSION_EXPERIMENT2CHARACTERISTIC T "
+                + AclQueryUtils.formNativeAclJoinClause( "T.EXPRESSION_EXPERIMENT_FK" ) + " "
+                + "where T.ID is not null ";
+        if ( eeIds != null ) {
+            query += " and T.EXPRESSION_EXPERIMENT_FK in :eeIds";
+        }
+        query += getExcludeUrisClause( "T.CATEGORY_URI", "T.CATEGORY", "excludedCategoryUris", excludedCategoryUris, excludeFreeTextCategories, excludeUncategorized, retainedTermUris );
+        query += getExcludeUrisClause( "T.VALUE_URI", "T.`VALUE`", "excludedTermUris", excludedTermUris, excludeFreeTextTerms, false, retainedTermUris );
+        query += AclQueryUtils.formNativeAclRestrictionClause( ( SessionFactoryImplementor ) getSessionFactory() ) + " "
+                + "group by COALESCE(T.CATEGORY_URI, T.CATEGORY) "
+                + "order by EE_COUNT desc";
+        Query q = getSessionFactory().getCurrentSession().createSQLQuery( query )
+                .addScalar( "CATEGORY", StandardBasicTypes.STRING )
+                .addScalar( "CATEGORY_URI", StandardBasicTypes.STRING )
                 .addScalar( "EE_COUNT", StandardBasicTypes.LONG )
                 .addSynchronizedQuerySpace( "EXPRESSION_EXPERIMENT2CHARACTERISTIC" )
                 .addSynchronizedEntityClass( Characteristic.class )
@@ -705,29 +723,57 @@ public Map<Characteristic, Long> getAnnotationsUsageFrequency( @Nullable Collect
             excludedTermUris = new HashSet<>( excludedTermUris );
             excludedTermUris.removeAll( retainedTermUris );
         }
-        Query q = getSessionFactory().getCurrentSession().createSQLQuery(
-                        "select T.`VALUE`, T.VALUE_URI, T.CATEGORY, T.CATEGORY_URI, T.EVIDENCE_CODE, count(distinct T.EXPRESSION_EXPERIMENT_FK) as EE_COUNT from EXPRESSION_EXPERIMENT2CHARACTERISTIC T "
-                                + AclQueryUtils.formNativeAclJoinClause( "T.EXPRESSION_EXPERIMENT_FK" ) + " "
-                                + "where T.ID is not null " // this is necessary for the clause building since there might be no clause
-                                + ( eeIds != null ? " and T.EXPRESSION_EXPERIMENT_FK in :eeIds" : "" )
-                                + ( level != null ? " and T.LEVEL = :level" : "" )
-                                + ( category != null ? category.equals( UNCATEGORIZED ) ? " and COALESCE(T.CATEGORY_URI, T.CATEGORY) is NULL" : " and COALESCE(T.CATEGORY_URI, T.CATEGORY) = :category" : "" )
-                                // FIXME: there's a bug in Hibernate that that prevents it from producing proper tuples
-                                //        for the excludedCategoryUris, retainedTermUris parameters, so we add explicit
-                                //        parentheses
-                                + ( excludedCategoryUris != null && !excludedCategoryUris.isEmpty() ? " and (T.CATEGORY_URI not in (:excludedCategoryUris)" + ( retainedTermUris != null && !retainedTermUris.isEmpty() ? " or T.VALUE_URI in (:retainedTermUris)" : "" ) + ")" : "" )
-                                + ( excludedTermUris != null && !excludedTermUris.isEmpty() ? " and T.VALUE_URI not in :excludedTermUris" : "" )
-                                + AclQueryUtils.formNativeAclRestrictionClause( ( SessionFactoryImplementor ) getSessionFactory() ) + " "
-                                + "group by COALESCE(T.CATEGORY_URI, T.CATEGORY), COALESCE(T.VALUE_URI, T.`VALUE`) "
-                                + "having EE_COUNT >= :minFrequency "
-                                + ( retainedTermUris != null && !retainedTermUris.isEmpty() ? "or T.VALUE_URI in (:retainedTermUris) " : "" )
-                                + "order by EE_COUNT desc" )
-                .addScalar( "T.VALUE", StandardBasicTypes.STRING )
-                .addScalar( "T.VALUE_URI", StandardBasicTypes.STRING )
-                .addScalar( "T.CATEGORY", StandardBasicTypes.STRING )
-                .addScalar( "T.CATEGORY_URI", StandardBasicTypes.STRING )
+        boolean excludeFreeTextCategories = false;
+        boolean excludeUncategorized = false;
+        if ( excludedCategoryUris != null ) {
+            if ( excludedCategoryUris.contains( null ) ) {
+                excludeFreeTextCategories = true;
+                excludedCategoryUris = excludedCategoryUris.stream().filter( Objects::nonNull ).collect( Collectors.toList() );
+            }
+            if ( excludedCategoryUris.contains( UNCATEGORIZED ) ) {
+                excludeUncategorized = true;
+                excludedCategoryUris = excludedCategoryUris.stream().filter( c -> !c.equals( UNCATEGORIZED ) ).collect( Collectors.toList() );
+            }
+        }
+        boolean excludeFreeTextTerms = false;
+        if ( excludedTermUris != null ) {
+            if ( excludedTermUris.contains( null ) ) {
+                excludeFreeTextTerms = true;
+                excludedTermUris = excludedTermUris.stream().filter( Objects::nonNull ).collect( Collectors.toList() );
+            }
+        }
+        String query = "select max(T.`VALUE`) as `VALUE`, max(T.VALUE_URI) as VALUE_URI, max(T.CATEGORY) as CATEGORY, max(T.CATEGORY_URI) as CATEGORY_URI, max(T.EVIDENCE_CODE) as EVIDENCE_CODE, count(distinct T.EXPRESSION_EXPERIMENT_FK) as EE_COUNT from EXPRESSION_EXPERIMENT2CHARACTERISTIC T "
+                + AclQueryUtils.formNativeAclJoinClause( "T.EXPRESSION_EXPERIMENT_FK" ) + " "
+                + "where T.ID is not null"; // this is necessary for the clause building since there might be no clause
+        if ( eeIds != null ) {
+            query += " and T.EXPRESSION_EXPERIMENT_FK in :eeIds";
+        }
+        if ( level != null ) {
+            query += " and T.LEVEL = :level";
+        }
+        if ( category != null ) {
+            if ( category.equals( UNCATEGORIZED ) ) {
+                query += " and COALESCE(T.CATEGORY_URI, T.CATEGORY) is NULL";
+            } else {
+                query += " and COALESCE(T.CATEGORY_URI, T.CATEGORY) = :category";
+            }
+        }
+        query += getExcludeUrisClause( "T.CATEGORY_URI", "T.CATEGORY", "excludedCategoryUris", excludedCategoryUris, excludeFreeTextCategories, excludeUncategorized, retainedTermUris );
+        query += getExcludeUrisClause( "T.VALUE_URI", "T.`VALUE`", "excludedTermUris", excludedTermUris, excludeFreeTextTerms, false, retainedTermUris );
+        query += AclQueryUtils.formNativeAclRestrictionClause( ( SessionFactoryImplementor ) getSessionFactory() ) + " "
+                + "group by COALESCE(T.CATEGORY_URI, T.CATEGORY), COALESCE(T.VALUE_URI, T.`VALUE`) "
+                + "having EE_COUNT >= :minFrequency ";
+        if ( retainedTermUris != null && !retainedTermUris.isEmpty() ) {
+            query += " or VALUE_URI in (:retainedTermUris)";
+        }
+        query += "order by EE_COUNT desc";
+        Query q = getSessionFactory().getCurrentSession().createSQLQuery( query )
+                .addScalar( "VALUE", StandardBasicTypes.STRING )
+                .addScalar( "VALUE_URI", StandardBasicTypes.STRING )
+                .addScalar( "CATEGORY", StandardBasicTypes.STRING )
+                .addScalar( "CATEGORY_URI", StandardBasicTypes.STRING )
                 // FIXME: use an EnumType for converting
-                .addScalar( "T.EVIDENCE_CODE", StandardBasicTypes.STRING )
+                .addScalar( "EVIDENCE_CODE", StandardBasicTypes.STRING )
                 .addScalar( "EE_COUNT", StandardBasicTypes.LONG )
                 .addSynchronizedQuerySpace( "EXPRESSION_EXPERIMENT2CHARACTERISTIC" )
                 .addSynchronizedEntityClass( Characteristic.class ) // ensures that the cache is invalidated if characteristics are added or removed
@@ -769,6 +815,45 @@ public Map<Characteristic, Long> getAnnotationsUsageFrequency( @Nullable Collect
         return byC;
     }
 
+    /**
+     * Produce a SQL clause for excluding URIs and free-text (i.e. null) URIs.
+     * <p>
+     * FIXME: There's a bug in Hibernate that that prevents it from producing proper tuples the excluded URIs and
+     *        retained term URIs
+     * @param column            column holding the URI to be excluded
+     * @param labelColumn       column holding the label (only used if excludeFreeText or excludeUncategorized is true,
+     *                          then we will check if the label is non-null to cover some edge cases)
+     * @param excludedUrisParam name of the binding parameter for the excluded URIs
+     * @param excludedUris      list of URIs to exclude
+     * @param excludeFreeText   whether to exclude free-text URIs
+     * @param retainedTermUris  list of terms that should bypass the exclusion
+     */
+    private String getExcludeUrisClause( String column, String labelColumn, String excludedUrisParam, @Nullable Collection<String> excludedUris, boolean excludeFreeText, boolean excludeUncategorized, @Nullable Collection<String> retainedTermUris ) {
+        String query = "";
+        if ( excludedUris != null && !excludedUris.isEmpty() ) {
+            query += " and ((" + column + " not in (:" + excludedUrisParam + ")";
+            if ( excludeUncategorized ) {
+                query += " and COALESCE(" + column + ", " + labelColumn + ") is not null";
+            }
+            if ( excludeFreeText ) {
+                query += " and (" + column + " is not null or " + labelColumn + " is null)";
+            }
+            query += ")";
+            if ( retainedTermUris != null && !retainedTermUris.isEmpty() ) {
+                query += " or T.VALUE_URI in (:retainedTermUris)";
+            }
+            query += ")";
+        } else {
+            if ( excludeUncategorized ) {
+                query += " and COALESCE(" + column + ", " + labelColumn + ") is not null";
+            }
+            if ( excludeFreeText ) {
+                query += " and (" + column + " is not null or " + labelColumn + " is null)";
+            }
+        }
+        return query;
+    }
+
     @Override
     public Collection<ExpressionExperiment> getExperimentsLackingPublications() {
         //noinspection unchecked
@@ -1347,15 +1432,15 @@ public Taxon getTaxon( BioAssaySet ee ) {
     private static class ExpressionExperimentDetail {
 
         public static ExpressionExperimentDetail fromRow( Object[] row ) {
-            return new ExpressionExperimentDetail( ( ArrayDesign ) row[1], ( ArrayDesign ) row[2], ( ExpressionExperiment ) row[3], ( Integer ) row[4] );
+            return new ExpressionExperimentDetail( ( Long ) row[1], ( Long ) row[2], ( Long ) row[3], ( Integer ) row[4] );
         }
 
         @Nullable
-        ArrayDesign arrayDesignUsed;
+        Long arrayDesignUsedId;
         @Nullable
-        ArrayDesign originalPlatform;
+        Long originalPlatformId;
         @Nullable
-        ExpressionExperiment otherPart;
+        Long otherPartId;
         Integer bioAssaysCount;
     }
 
@@ -1368,13 +1453,14 @@ private Map<Long, List<ExpressionExperimentDetail>> getExpressionExperimentDetai
         }
         //noinspection unchecked
         List<Object[]> results = getSessionFactory().getCurrentSession()
-                .createQuery( "select ee.id, ad, op, oe, ee.bioAssays.size from ExpressionExperiment as ee "
+                .createQuery( "select ee.id, ad.id, op.id, oe.id, ee.bioAssays.size from ExpressionExperiment as ee "
                         + "left join ee.bioAssays ba "
                         + "left join ba.arrayDesignUsed ad "
                         + "left join ba.originalPlatform op " // not all bioAssays have an original platform
                         + "left join ee.otherParts as oe "    // not all experiments are splitted
                         + "where ee.id in :eeIds "
-                        + "group by ee, ad, op" )
+                        // FIXME: apply ACLs, other parts or platform might be private
+                        + "group by ee, ad, op, oe" )
                 .setParameterList( "eeIds", expressionExperimentIds )
                 .setCacheable( cacheable )
                 .list();
@@ -1540,22 +1626,25 @@ public List<ExpressionExperimentDetailsValueObject> transformListTyped( List<Exp
                 for ( ExpressionExperimentDetailsValueObject vo : vos ) {
                     List<ExpressionExperimentDetail> details = detailsByEE.get( vo.getId() );
 
-                    // we need those later for computing original platforms
-                    Collection<ArrayDesign> arrayDesignsUsed = details.stream()
-                            .map( ExpressionExperimentDetail::getArrayDesignUsed )
+                    Set<Long> arrayDesignsUsedIds = details.stream()
+                            .map( ExpressionExperimentDetail::getArrayDesignUsedId )
                             .filter( Objects::nonNull )
                             .collect( Collectors.toSet() );
 
-                    Collection<ArrayDesignValueObject> adVos = arrayDesignsUsed.stream()
+                    // we need those later for computing original platforms
+                    Collection<ArrayDesignValueObject> adVos = arrayDesignsUsedIds.stream()
+                            .map( id -> ( ArrayDesign ) getSessionFactory().getCurrentSession().get( ArrayDesign.class, id ) )
                             .map( ArrayDesignValueObject::new )
                             .collect( Collectors.toSet() );
                     vo.setArrayDesigns( adVos ); // also sets taxon name, technology type, and number of ADs.
 
                     // original platforms
                     Collection<ArrayDesignValueObject> originalPlatformsVos = details.stream()
-                            .map( ExpressionExperimentDetail::getOriginalPlatform )
+                            .map( ExpressionExperimentDetail::getOriginalPlatformId )
                             .filter( Objects::nonNull ) // on original platform for the bioAssay
-                            .filter( op -> !arrayDesignsUsed.contains( op ) )
+                            .distinct()
+                            .filter( op -> !arrayDesignsUsedIds.contains( op ) ) // omit noop switches
+                            .map( id -> ( ArrayDesign ) getSessionFactory().getCurrentSession().get( ArrayDesign.class, id ) )
                             .map( ArrayDesignValueObject::new )
                             .collect( Collectors.toSet() );
                     vo.setOriginalPlatforms( originalPlatformsVos );
@@ -1566,13 +1655,16 @@ public List<ExpressionExperimentDetailsValueObject> transformListTyped( List<Exp
                             .orElse( 0 );
                     vo.setNumberOfBioAssays( bioAssayCount );
 
-                    Set<ExpressionExperiment> otherParts = details.stream()
-                            .map( ExpressionExperimentDetail::getOtherPart )
+                    Set<Long> otherPartsIds = details.stream()
+                            .map( ExpressionExperimentDetail::getOtherPartId )
                             .filter( Objects::nonNull )
                             .collect( Collectors.toSet() );
 
+                    List<ExpressionExperimentValueObject> otherPartsVos = loadValueObjectsByIds( otherPartsIds ).stream()
+                            .sorted( Comparator.comparing( ExpressionExperimentValueObject::getShortName ) ).collect( Collectors.toList() );
+
                     // other parts (maybe fetch in details query?)
-                    vo.getOtherParts().addAll( otherParts.stream().map( ee2 -> doLoadValueObject( ee2 ) ).collect( Collectors.toList() ) );
+                    vo.setOtherParts( otherPartsVos );
                 }
 
                 try ( StopWatchUtils.StopWatchRegion ignored = StopWatchUtils.measuredRegion( analysisInformationTimer ) ) {
diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/experiment/FactorValueDaoImpl.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/experiment/FactorValueDaoImpl.java
index 337b322ee2..1e2759d1aa 100644
--- a/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/experiment/FactorValueDaoImpl.java
+++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/experiment/FactorValueDaoImpl.java
@@ -23,7 +23,6 @@
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Repository;
 import ubic.gemma.model.expression.biomaterial.BioMaterial;
-import ubic.gemma.model.expression.experiment.ExperimentalDesign;
 import ubic.gemma.model.expression.experiment.ExperimentalFactor;
 import ubic.gemma.model.expression.experiment.FactorValue;
 import ubic.gemma.model.expression.experiment.FactorValueValueObject;
@@ -89,6 +88,14 @@ public void remove( @Nullable final FactorValue factorValue ) {
             this.getSessionFactory().getCurrentSession().update( ef );
         }
 
+        // remove any attached statements since those are not mapped in the collection
+        // remove this in the 1.31 since it will become unnecessary (see https://github.com/PavlidisLab/Gemma/issues/909)
+        int removedStatements = getSessionFactory().getCurrentSession()
+                .createSQLQuery( "delete from CHARACTERISTIC where class = 'Statement' and FACTOR_VALUE_FK = :fvId" )
+                .setParameter( "fvId", factorValue.getId() )
+                .executeUpdate();
+        log.info( String.format( "Removed %d statements from %s", removedStatements, factorValue ) );
+
         super.remove( factorValue );
     }
 
diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/genome/GeneDaoImpl.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/genome/GeneDaoImpl.java
index dab99bac9b..7f5135a0cf 100644
--- a/gemma-core/src/main/java/ubic/gemma/persistence/service/genome/GeneDaoImpl.java
+++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/genome/GeneDaoImpl.java
@@ -482,6 +482,7 @@ public int removeAll() {
         if ( !plIds.isEmpty() ) {
             removedPhysicalLocations = getSessionFactory().getCurrentSession()
                     .createQuery( "delete from PhysicalLocation pl where pl.id in :plIds" )
+                    .setParameterList( "plIds", plIds )
                     .executeUpdate();
         } else {
             removedPhysicalLocations = 0;
diff --git a/gemma-core/src/main/resources/default.properties b/gemma-core/src/main/resources/default.properties
index 8f57484d88..9087685d5e 100755
--- a/gemma-core/src/main/resources/default.properties
+++ b/gemma-core/src/main/resources/default.properties
@@ -190,6 +190,6 @@ gemma.testdb.sqlMode=${gemma.db.sqlMode}
 #the external database id to exclude by default in phenocarta
 gemma.neurocarta.exluded_database_id=85
 # Featured external databases in Gemma Web About page and Gemma REST main endpoint
-gemma.externalDatabases.featured=hg38,mm10,rn7,gene,go
+gemma.externalDatabases.featured=hg38,mm39,rn7,gene,go
 # Enable cache statistics on startup
 gemma.cacheMonitor.enableStatistics=false
\ No newline at end of file
diff --git a/gemma-core/src/main/resources/sql/init-entities.sql b/gemma-core/src/main/resources/sql/init-entities.sql
index 4506d8f36a..bf37c5ba16 100644
--- a/gemma-core/src/main/resources/sql/init-entities.sql
+++ b/gemma-core/src/main/resources/sql/init-entities.sql
@@ -95,6 +95,7 @@ call add_external_database('hg38', NULL, '', NULL, 'SEQUENCE');
 call add_external_database('mm8', NULL, '', NULL, 'SEQUENCE');
 call add_external_database('mm9', NULL, '', NULL, 'SEQUENCE');
 call add_external_database('mm10', NULL, '', NULL, 'SEQUENCE');
+call add_external_database('mm39', NULL, '', NULL, 'SEQUENCE');
 call add_external_database('rn4', NULL, '', NULL, 'SEQUENCE');
 call add_external_database('rn6', NULL, '', NULL, 'SEQUENCE');
 call add_external_database('rn7', NULL, '', NULL, 'SEQUENCE');
@@ -104,13 +105,13 @@ call add_external_database('hg38 annotations', NULL, 'https://hgdownload.cse.ucs
 call add_external_database('mm8 annotations', NULL, 'https://hgdownload.cse.ucsc.edu/goldenpath/mm8/database/', NULL, 'OTHER');
 call add_external_database('mm9 annotations', NULL, 'https://hgdownload.cse.ucsc.edu/goldenpath/mm9/database/', NULL, 'OTHER');
 call add_external_database('mm10 annotations', NULL, 'https://hgdownload.cse.ucsc.edu/goldenpath/mm10/database/', NULL, 'OTHER');
-call add_external_database('mm11 annotations', NULL, 'https://hgdownload.cse.ucsc.edu/goldenpath/mm11/database/', NULL, 'OTHER');
 call add_external_database('mm39 annotations', NULL, 'https://hgdownload.cse.ucsc.edu/goldenpath/mm39/database/', NULL, 'OTHER');
 call add_external_database('rn4 annotations', NULL, 'https://hgdownload.cse.ucsc.edu/goldenpath/rn4/database/', NULL, 'OTHER');
 call add_external_database('rn6 annotations', NULL, 'https://hgdownload.cse.ucsc.edu/goldenpath/rn6/database/', NULL, 'OTHER');
 call add_external_database('rn7 annotations', NULL, 'https://hgdownload.cse.ucsc.edu/goldenpath/rn7/database/', NULL, 'OTHER');
 call add_external_database('hg38 RNA-Seq annotations', NULL, NULL, NULL, 'OTHER');
 call add_external_database('mm10 RNA-Seq annotations', NULL, NULL, NULL, 'OTHER');
+call add_external_database('mm39 RNA-Seq annotations', NULL, NULL, NULL, 'OTHER');
 call add_external_database('rn7 RNA-Seq annotations', NULL, NULL, NULL, 'OTHER');
 call add_external_database('gene', NULL, NULL, 'https://ftp.ncbi.nih.gov/gene/DATA/gene_info.gz', 'OTHER');
 call add_external_database('go', NULL, NULL, 'https://ftp.ncbi.nih.gov/gene/DATA/gene2go.gz', 'ONTOLOGY');
@@ -124,6 +125,7 @@ create procedure add_external_database_relation(in parent_name varchar(255), in
 call add_external_database_relation('hg38', 'hg38 annotations');
 call add_external_database_relation('hg19', 'hg19 annotations');
 call add_external_database_relation('hg18', 'hg18 annotations');
+call add_external_database_relation('mm39', 'mm39 annotations');
 call add_external_database_relation('mm10', 'mm10 annotations');
 call add_external_database_relation('mm9', 'mm9 annotations');
 call add_external_database_relation('mm8', 'mm8 annotations');
@@ -132,6 +134,7 @@ call add_external_database_relation('rn6', 'rn4 annotations');
 call add_external_database_relation('rn4', 'rn6 annotations');
 
 call add_external_database_relation('hg38', 'hg38 RNA-Seq annotations');
+call add_external_database_relation('mm39', 'mm39 RNA-Seq annotations');
 call add_external_database_relation('mm10', 'mm10 RNA-Seq annotations');
 call add_external_database_relation('rn7', 'rn7 RNA-Seq annotations');
 
diff --git a/gemma-core/src/main/resources/ubic/gemma/model/common/auditAndSecurity/eventType/AuditEventType.hbm.xml b/gemma-core/src/main/resources/ubic/gemma/model/common/auditAndSecurity/eventType/AuditEventType.hbm.xml
index 98d570183b..99844d4474 100644
--- a/gemma-core/src/main/resources/ubic/gemma/model/common/auditAndSecurity/eventType/AuditEventType.hbm.xml
+++ b/gemma-core/src/main/resources/ubic/gemma/model/common/auditAndSecurity/eventType/AuditEventType.hbm.xml
@@ -67,19 +67,24 @@
             </subclass>
             <subclass abstract="false" discriminator-value="ExpressionExperimentVectorMergeEvent"
                       name="ubic.gemma.model.common.auditAndSecurity.eventType.ExpressionExperimentVectorMergeEvent"/>
-            <subclass abstract="false" discriminator-value="BatchInformationFetchingEvent"
-                      name="ubic.gemma.model.common.auditAndSecurity.eventType.BatchInformationFetchingEvent">
-                <subclass abstract="false" discriminator-value="FailedBatchInformationFetchingEvent"
-                          name="ubic.gemma.model.common.auditAndSecurity.eventType.FailedBatchInformationFetchingEvent">
+            <subclass abstract="true" name="ubic.gemma.model.common.auditAndSecurity.eventType.BatchInformationEvent">
+                <subclass abstract="false" discriminator-value="BatchInformationFetchingEvent"
+                          name="ubic.gemma.model.common.auditAndSecurity.eventType.BatchInformationFetchingEvent">
+                    <subclass abstract="false" name="ubic.gemma.model.common.auditAndSecurity.eventType.FailedBatchInformationFetchingEvent"
+                              discriminator-value="FailedBatchInformationFetchingEvent">
+                        <subclass abstract="false" discriminator-value="UninformativeFASTQHeadersForBatchingEvent"
+                                  name="ubic.gemma.model.common.auditAndSecurity.eventType.UninformativeFASTQHeadersForBatchingEvent"/>
+                        <subclass abstract="false" discriminator-value="SingletonBatchInvalidEvent"
+                                  name="ubic.gemma.model.common.auditAndSecurity.eventType.SingletonBatchInvalidEvent"/>
+                    </subclass>
+                    <subclass abstract="false" discriminator-value="SingleBatchDeterminationEvent"
+                              name="ubic.gemma.model.common.auditAndSecurity.eventType.SingleBatchDeterminationEvent">
+                    </subclass>
+                </subclass>
+                <subclass abstract="false" discriminator-value="BatchInformationMissingEvent"
+                          name="ubic.gemma.model.common.auditAndSecurity.eventType.BatchInformationMissingEvent">
                     <subclass abstract="false" discriminator-value="FailedBatchInformationMissingEvent"
                               name="ubic.gemma.model.common.auditAndSecurity.eventType.FailedBatchInformationMissingEvent"/>
-                    <subclass abstract="false" discriminator-value="UninformativeFASTQHeadersForBatchingEvent"
-                              name="ubic.gemma.model.common.auditAndSecurity.eventType.UninformativeFASTQHeadersForBatchingEvent"/>
-                    <subclass abstract="false" discriminator-value="SingletonBatchInvalidEvent"
-                              name="ubic.gemma.model.common.auditAndSecurity.eventType.SingletonBatchInvalidEvent"/>
-                </subclass>
-                <subclass abstract="false" discriminator-value="SingleBatchDeterminationEvent"
-                          name="ubic.gemma.model.common.auditAndSecurity.eventType.SingleBatchDeterminationEvent">
                 </subclass>
             </subclass>
             <subclass abstract="false" discriminator-value="PCAAnalysisEvent"
diff --git a/gemma-core/src/main/resources/ubic/gemma/model/common/description/Characteristic.hbm.xml b/gemma-core/src/main/resources/ubic/gemma/model/common/description/Characteristic.hbm.xml
index abc7260f81..e73d053f8e 100644
--- a/gemma-core/src/main/resources/ubic/gemma/model/common/description/Characteristic.hbm.xml
+++ b/gemma-core/src/main/resources/ubic/gemma/model/common/description/Characteristic.hbm.xml
@@ -5,13 +5,14 @@
 
 <hibernate-mapping>
     <class name="ubic.gemma.model.common.description.Characteristic" table="CHARACTERISTIC"
-           discriminator-value="Characteristic">
+           discriminator-value="null">
         <cache usage="nonstrict-read-write"/>
         <id name="id" type="java.lang.Long" unsaved-value="null">
             <column name="ID" sql-type="BIGINT"/>
             <generator class="native">
             </generator>
         </id>
+        <discriminator column="class" not-null="false"/>
         <property name="name" type="java.lang.String">
             <column name="NAME" not-null="false" unique="false"
                     sql-type="VARCHAR(255)"/>
diff --git a/gemma-core/src/main/resources/ubic/gemma/model/expression/experiment/FactorValue.hbm.xml b/gemma-core/src/main/resources/ubic/gemma/model/expression/experiment/FactorValue.hbm.xml
index 4b4f5526a8..9f50f88fa6 100644
--- a/gemma-core/src/main/resources/ubic/gemma/model/expression/experiment/FactorValue.hbm.xml
+++ b/gemma-core/src/main/resources/ubic/gemma/model/expression/experiment/FactorValue.hbm.xml
@@ -28,7 +28,8 @@
          <column name="MEASUREMENT_FK" not-null="false" sql-type="BIGINT" unique="true"/>
       </many-to-one>
       <!-- assumed readily available in FactorValueValueObject -->
-      <set name="characteristics" lazy="false" fetch="select" cascade="all">
+      <!-- revert the 'class is null' in the 1.31 (see https://github.com/PavlidisLab/Gemma/issues/909) -->
+      <set name="characteristics" lazy="false" fetch="select" cascade="all" where="class is null">
          <cache usage="read-write"/>
          <key foreign-key="CHARACTERISTIC_FACTOR_VALUE_FKC">
             <column name="FACTOR_VALUE_FK" sql-type="BIGINT"/>
diff --git a/gemma-core/src/test/java/ubic/gemma/core/analysis/service/AbstractFileServiceTest.java b/gemma-core/src/test/java/ubic/gemma/core/analysis/service/AbstractFileServiceTest.java
index 45eba6ae66..78d02f9741 100644
--- a/gemma-core/src/test/java/ubic/gemma/core/analysis/service/AbstractFileServiceTest.java
+++ b/gemma-core/src/test/java/ubic/gemma/core/analysis/service/AbstractFileServiceTest.java
@@ -23,7 +23,10 @@ public void writeTsv( Object entity, Writer writer ) throws IOException {
 
     @Test
     public void testFormatNumber() {
+        assertEquals( "0.0", tsvFileService.format( 0.0 ) );
+        assertEquals( "0.0", tsvFileService.format( -0.0 ) );
         assertEquals( "1E-14", tsvFileService.format( 1e-14 ) );
+        assertEquals( "-1E-14", tsvFileService.format( -1e-14 ) );
         assertEquals( "0.1111", tsvFileService.format( 0.1111 ) );
         assertEquals( "0.0001", tsvFileService.format( 0.0001 ) );
         assertEquals( "1E-5", tsvFileService.format( 1e-5 ) );
@@ -31,7 +34,9 @@ public void testFormatNumber() {
         assertEquals( "1000.0", tsvFileService.format( 1000.0 ) );
         assertEquals( "1234.5", tsvFileService.format( 1234.5 ) );
         assertEquals( "100000.0", tsvFileService.format( 1e5 ) );
+        assertEquals( "-100000.0", tsvFileService.format( -1e5 ) );
         assertEquals( "100.1234", tsvFileService.format( 100.1234 ) );
+        assertEquals( "-100.1234", tsvFileService.format( -100.1234 ) );
         assertEquals( "1000.1", tsvFileService.format( 1000.1234 ) );
         assertEquals( "100000.1", tsvFileService.format( 100000.1234 ) );
         assertEquals( "", tsvFileService.format( null ) );
diff --git a/gemma-core/src/test/java/ubic/gemma/core/loader/expression/geo/GeoCharacteristicParseTest.java b/gemma-core/src/test/java/ubic/gemma/core/loader/expression/geo/GeoCharacteristicParseTest.java
index 45b6b99761..8353704075 100644
--- a/gemma-core/src/test/java/ubic/gemma/core/loader/expression/geo/GeoCharacteristicParseTest.java
+++ b/gemma-core/src/test/java/ubic/gemma/core/loader/expression/geo/GeoCharacteristicParseTest.java
@@ -1,8 +1,8 @@
 /*
  * The gemma-core project
- * 
+ *
  * Copyright (c) 2018 University of British Columbia
- * 
+ *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
@@ -19,17 +19,15 @@
 
 package ubic.gemma.core.loader.expression.geo;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
 import org.junit.Test;
-
 import ubic.gemma.model.common.description.Characteristic;
 import ubic.gemma.model.expression.biomaterial.BioMaterial;
 
+import static org.junit.Assert.*;
+
 /**
- * 
- * 
+ *
+ *
  * @author paul
  */
 public class GeoCharacteristicParseTest {
@@ -81,8 +79,11 @@ public final void testParseGEOSampleCharacteristic() throws Exception {
         t = BioMaterial.Factory.newInstance();
         g.parseGEOSampleCharacteristicString( "Clinical info: Submitting diagnosis: DLBCL", t );
         c = t.getCharacteristics().iterator().next();
-        assertEquals( "Clinical info: Submitting diagnosis: DLBCL", c.getValue() );
-        
+        assertEquals( "Clinical info", c.getCategory() );
+        assertNull( c.getCategoryUri() );
+        assertEquals( "Submitting diagnosis: DLBCL", c.getValue() );
+        assertNull( c.getValueUri() );
+
         // test trimming of species names
         t = BioMaterial.Factory.newInstance();
         g.parseGEOSampleCharacteristicString( "cell type: human fibroblast", t );
diff --git a/gemma-core/src/test/java/ubic/gemma/core/loader/expression/geo/GeoConverterTest.java b/gemma-core/src/test/java/ubic/gemma/core/loader/expression/geo/GeoConverterTest.java
index 71fbcfd4f7..0b3ae5a0ba 100644
--- a/gemma-core/src/test/java/ubic/gemma/core/loader/expression/geo/GeoConverterTest.java
+++ b/gemma-core/src/test/java/ubic/gemma/core/loader/expression/geo/GeoConverterTest.java
@@ -43,6 +43,7 @@
 import ubic.gemma.core.loader.expression.geo.model.GeoSeries;
 import ubic.gemma.core.util.test.BaseSpringContextTest;
 import ubic.gemma.core.util.test.category.SlowTest;
+import ubic.gemma.model.common.description.Characteristic;
 import ubic.gemma.model.common.quantitationtype.PrimitiveType;
 import ubic.gemma.model.common.quantitationtype.QuantitationType;
 import ubic.gemma.model.common.quantitationtype.StandardQuantitationType;
@@ -50,6 +51,7 @@
 import ubic.gemma.model.expression.bioAssay.BioAssay;
 import ubic.gemma.model.expression.bioAssayData.DesignElementDataVector;
 import ubic.gemma.model.expression.bioAssayData.RawExpressionDataVector;
+import ubic.gemma.model.expression.biomaterial.BioMaterial;
 import ubic.gemma.model.expression.designElement.CompositeSequence;
 import ubic.gemma.model.expression.experiment.ExpressionExperiment;
 import ubic.gemma.model.genome.Taxon;
@@ -738,6 +740,7 @@ public void testParseGSE18707() throws Exception {
         assertEquals( 100, e.getRawExpressionDataVectors().size() );
         assertEquals( 1, e.getQuantitationTypes().size() ); // this is normal, before any processing.
 
+
         QuantitationType qt = e.getQuantitationTypes().iterator().next();
         assertEquals( "Processed Affymetrix Rosetta intensity values", qt.getDescription() );
 
@@ -770,6 +773,28 @@ public void testParseGSE44625() throws Exception {
         assertEquals( 2, ee.getQuantitationTypes().size() );
 
         assertNotNull( ee.getTaxon() );
+
+        // Confirmation for #908 that we take in the right data from sample characteristics.
+        boolean found1 = false;
+        for ( BioAssay ba : ee.getBioAssays() ) {
+            for ( Characteristic c : ba.getSampleUsed().getCharacteristics() ) {
+               // log.info( c );
+                String category = c.getCategory();
+                if ( !category.equals( "molecular entity" ) && !category.equals( "labelling" ) && !category.equals( "BioSource" ) ) { // we lose these original strings, or they have diff format; not important.
+                    assertNotNull( c.getOriginalValue() );
+                    assertTrue( c.getOriginalValue().contains( ":" ) ); // for this particular dataset; just a check that we are storing the actual original, not just the second half.
+                }
+                assertNotNull( category );
+                if ( !category.equals( "strain" ) )
+                    assertTrue( !c.getValue().contains( ":" ) ); // this is actually allowed if there's : in the value itself...Crl:CD1(ICR)
+                if ( category.equals( "weight" ) ) { // some crazy non-standard category.
+                    found1 = true;
+                }
+            }
+        }
+
+        assertTrue( found1 );
+
     }
 
     /*
diff --git a/gemma-core/src/test/java/ubic/gemma/persistence/service/expression/experiment/ExpressionExperimentDaoTest.java b/gemma-core/src/test/java/ubic/gemma/persistence/service/expression/experiment/ExpressionExperimentDaoTest.java
index 1a990300e0..50393011a5 100644
--- a/gemma-core/src/test/java/ubic/gemma/persistence/service/expression/experiment/ExpressionExperimentDaoTest.java
+++ b/gemma-core/src/test/java/ubic/gemma/persistence/service/expression/experiment/ExpressionExperimentDaoTest.java
@@ -11,6 +11,7 @@
 import org.springframework.security.test.context.support.WithMockUser;
 import org.springframework.test.context.ContextConfiguration;
 import ubic.gemma.core.util.test.BaseDatabaseTest;
+import ubic.gemma.model.common.description.Characteristic;
 import ubic.gemma.model.expression.arrayDesign.ArrayDesign;
 import ubic.gemma.model.expression.bioAssay.BioAssay;
 import ubic.gemma.model.expression.bioAssayData.ProcessedExpressionDataVector;
@@ -20,6 +21,7 @@
 import ubic.gemma.model.genome.Taxon;
 import ubic.gemma.persistence.util.*;
 
+import javax.annotation.Nullable;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
@@ -142,15 +144,97 @@ public void testGetOriginalPlatformUsageFrequency() {
     }
 
     @Test
-    @WithMockUser
+    @WithMockUser(authorities = "GROUP_ADMIN")
     public void testGetCategoriesWithUsageFrequency() {
-        expressionExperimentDao.getCategoriesUsageFrequency( null, null, null, null );
+        Characteristic c = createCharacteristic( "foo", "foo", "bar", "bar" );
+        Assertions.assertThat( expressionExperimentDao.getCategoriesUsageFrequency( null, null, null, null ) )
+                .containsEntry( c, 1L );
     }
 
     @Test
-    @WithMockUser
+    @WithMockUser(authorities = "GROUP_ADMIN")
     public void testGetAnnotationUsageFrequency() {
-        expressionExperimentDao.getAnnotationsUsageFrequency( null, null, 10, 1, null, null, null, null );
+        Characteristic c = createCharacteristic( "foo", "foo", "bar", "bar" );
+        Assertions.assertThat( expressionExperimentDao.getAnnotationsUsageFrequency( null, null, 10, 1, null, null, null, null ) )
+                .containsEntry( c, 1L );
+    }
+
+    @Test
+    @WithMockUser(authorities = "GROUP_ADMIN")
+    public void testGetAnnotationUsageFrequencyRetainMentionedTerm() {
+        Characteristic c = createCharacteristic( "foo", "foo", "bar", "http://bar" );
+        Characteristic c1 = createCharacteristic( "foo", "foo", "bar", "bar" );
+        Assertions.assertThat( expressionExperimentDao.getAnnotationsUsageFrequency( null, null, 10, 2, null, null, null, Collections.singleton( "http://bar" ) ) )
+                .containsEntry( c, 1L ) // bypasses the minimum frequency requirement
+                .doesNotContainKey( c1 );
+    }
+
+    @Test
+    @WithMockUser(authorities = "GROUP_ADMIN")
+    public void testGetAnnotationUsageFrequencyExcludingFreeTextTerms() {
+        Characteristic c = createCharacteristic( "foo", "foo", "bar", "bar" );
+        Characteristic c1 = createCharacteristic( "foo", "foo", "bar", null );
+        Assertions.assertThat( expressionExperimentDao.getAnnotationsUsageFrequency( null, null, 10, 1, null, null, Collections.singleton( null ), null ) )
+                .containsEntry( c, 1L )
+                .doesNotContainKey( c1 );
+    }
+
+    @Test
+    @WithMockUser(authorities = "GROUP_ADMIN")
+    public void testGetAnnotationUsageFrequencyExcludingFreeTextCategories() {
+        Characteristic c = createCharacteristic( "foo", "foo", "bar", "http://bar" );
+        Characteristic c1 = createCharacteristic( "foo", null, "bar", null );
+        Characteristic c2 = createCharacteristic( null, null, "bar", null );
+        Assertions.assertThat( expressionExperimentDao.getAnnotationsUsageFrequency( null, null, 10, 1, null, Collections.singleton( null ), null, null ) )
+                .containsEntry( c, 1L )
+                .doesNotContainKey( c1 )
+                .containsEntry( c2, 1L ); // uncategorized is not a free-text category
+    }
+
+    @Test
+    @WithMockUser(authorities = "GROUP_ADMIN")
+    public void testGetAnnotationUsageFrequencyExcludingUncategorizedTerms() {
+        Characteristic c = createCharacteristic( "foo", "foo", "bar", "http://bar" );
+        Characteristic c1 = createCharacteristic( "bar", null, "bar", "http://bar" );
+        Characteristic c2 = createCharacteristic( null, null, "bar", "http://bar" );
+        Assertions.assertThat( expressionExperimentDao.getAnnotationsUsageFrequency( null, null, 10, 1, null, Collections.singleton( ExpressionExperimentDao.UNCATEGORIZED ), null, null ) )
+                .containsEntry( c, 1L )
+                .containsEntry( c1, 1L ) // free-text category is not uncategorized
+                .doesNotContainKey( c2 );
+    }
+
+    @Test
+    @WithMockUser(authorities = "GROUP_ADMIN")
+    public void testGetAnnotationUsageFrequencyWithUncategorizedCategory() {
+        Characteristic c = createCharacteristic( null, null, "bar", "bar" );
+        Characteristic c1 = createCharacteristic( "foo", "foo", "bar", null );
+        Characteristic c2 = createCharacteristic( "foo", null, "bar", null );
+        Assertions.assertThat( expressionExperimentDao.getAnnotationsUsageFrequency( null, null, 10, 1, ExpressionExperimentDao.UNCATEGORIZED, null, null, null ) )
+                .containsEntry( c, 1L )
+                .doesNotContainKey( c1 )
+                .doesNotContainKey( c2 );
+    }
+
+    private Characteristic createCharacteristic( @Nullable String category, @Nullable String categoryUri, String value, @Nullable String valueUri ) {
+        ExpressionExperiment ee = new ExpressionExperiment();
+        sessionFactory.getCurrentSession().persist( ee );
+        Characteristic c = new Characteristic();
+        c.setCategory( category );
+        c.setCategoryUri( categoryUri );
+        c.setValue( value );
+        c.setValueUri( valueUri );
+        sessionFactory.getCurrentSession().persist( c );
+        sessionFactory.getCurrentSession()
+                .createSQLQuery( "insert into EXPRESSION_EXPERIMENT2CHARACTERISTIC (ID, CATEGORY, CATEGORY_URI, `VALUE`, VALUE_URI, EXPRESSION_EXPERIMENT_FK, LEVEL) values (:id, :category, :categoryUri, :value, :valueUri, :eeId, :level)" )
+                .setParameter( "id", c.getId() )
+                .setParameter( "category", c.getCategory() )
+                .setParameter( "categoryUri", c.getCategoryUri() )
+                .setParameter( "value", c.getValue() )
+                .setParameter( "valueUri", c.getValueUri() )
+                .setParameter( "eeId", ee.getId() )
+                .setParameter( "level", ExpressionExperiment.class )
+                .executeUpdate();
+        return c;
     }
 
     @Test
diff --git a/gemma-core/src/test/java/ubic/gemma/persistence/service/expression/experiment/FactorValueDaoTest.java b/gemma-core/src/test/java/ubic/gemma/persistence/service/expression/experiment/FactorValueDaoTest.java
new file mode 100644
index 0000000000..d4ef53b3a3
--- /dev/null
+++ b/gemma-core/src/test/java/ubic/gemma/persistence/service/expression/experiment/FactorValueDaoTest.java
@@ -0,0 +1,70 @@
+package ubic.gemma.persistence.service.expression.experiment;
+
+import org.hibernate.SessionFactory;
+import org.junit.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.test.context.ContextConfiguration;
+import ubic.gemma.core.util.test.BaseDatabaseTest;
+import ubic.gemma.model.common.description.Characteristic;
+import ubic.gemma.model.expression.experiment.ExperimentalDesign;
+import ubic.gemma.model.expression.experiment.ExperimentalFactor;
+import ubic.gemma.model.expression.experiment.FactorType;
+import ubic.gemma.model.expression.experiment.FactorValue;
+import ubic.gemma.persistence.util.TestComponent;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+@ContextConfiguration
+public class FactorValueDaoTest extends BaseDatabaseTest {
+
+    @Configuration
+    @TestComponent
+    static class FactorValueDaoTestContextConfiguration extends BaseDatabaseTestContextConfiguration {
+
+        @Bean
+        public FactorValueDao factorValueDao( SessionFactory sessionFactory ) {
+            return new FactorValueDaoImpl( sessionFactory );
+        }
+    }
+
+    @Autowired
+    private FactorValueDao factorValueDao;
+
+    @Test
+    public void testFactorValueStatementsNotListedInCharacteristics() {
+        ExperimentalDesign ed = new ExperimentalDesign();
+        sessionFactory.getCurrentSession().persist( ed );
+        ExperimentalFactor ef = new ExperimentalFactor();
+        ef.setExperimentalDesign( ed );
+        ef.setType( FactorType.CATEGORICAL );
+        sessionFactory.getCurrentSession().persist( ef );
+        FactorValue fv = new FactorValue();
+        fv.setExperimentalFactor( ef );
+        Characteristic c1 = new Characteristic();
+        c1.setValue( "test" );
+        Characteristic c2 = new Characteristic();
+        c1.setValue( "test2" );
+        fv.getCharacteristics().add( c1 );
+        fv.getCharacteristics().add( c2 );
+        fv = factorValueDao.create( fv );
+        assertThat( fv.getId() ).isNotNull();
+        assertThat( c1.getId() ).isNotNull();
+        assertThat( c2.getId() ).isNotNull();
+        // make c2 a statement
+        sessionFactory.getCurrentSession()
+                .createSQLQuery( "update CHARACTERISTIC set class = 'Statement' where ID = :id" )
+                .setParameter( "id", c2.getId() )
+                .executeUpdate();
+        sessionFactory.getCurrentSession().flush();
+        sessionFactory.getCurrentSession().evict( fv );
+        // reload
+        fv = factorValueDao.load( fv.getId() );
+        assertThat( fv ).isNotNull();
+        assertThat( fv.getCharacteristics() )
+                .contains( c1 )
+                .doesNotContain( c2 );
+        factorValueDao.remove( fv );
+    }
+}
\ No newline at end of file
diff --git a/gemma-core/src/test/java/ubic/gemma/persistence/util/SettingsConfigTest.java b/gemma-core/src/test/java/ubic/gemma/persistence/util/SettingsConfigTest.java
index 6bdd02d9b0..159b618bd4 100644
--- a/gemma-core/src/test/java/ubic/gemma/persistence/util/SettingsConfigTest.java
+++ b/gemma-core/src/test/java/ubic/gemma/persistence/util/SettingsConfigTest.java
@@ -73,7 +73,7 @@ public void test() {
         assertTrue( allowedOrigins.contains( hostUrl ) );
         assertThat( featuredExternalDatabases )
                 .isNotNull()
-                .contains( "hg38", "mm10" );
+                .contains( "hg38", "mm39" );
         assertEquals( "/var/tmp/gemmaData", appDataHome );
         assertEquals( appDataHome + "/download", downloadPath );
         assertEquals( appDataHome + "/searchIndices", searchDir );
diff --git a/gemma-core/src/test/resources/log4j-test.properties b/gemma-core/src/test/resources/log4j-test.properties
index 4eda872347..855937ec79 100644
--- a/gemma-core/src/test/resources/log4j-test.properties
+++ b/gemma-core/src/test/resources/log4j-test.properties
@@ -4,7 +4,7 @@
 log4j.appender.stderr=org.apache.log4j.ConsoleAppender
 log4j.appender.stderr.target=System.out
 log4j.appender.stderr.layout=org.apache.log4j.PatternLayout
-log4j.appender.stderr.layout.ConversionPattern=%d %highlight{%p} %style{%pid}{magenta} [%t] %style{%C.%M(%L)}{cyan} | %m%n
+log4j.appender.stderr.layout.ConversionPattern=%d %highlight{%p} %style{%pid}{magenta} [%t] %style{%C{1.}.%M(%L)}{cyan} | %m%n
 
 # this is used for the ProgressUpdateAppenderTest and ExecutingTaskTest
 log4j.appender.progressUpdate=ubic.gemma.core.job.executor.common.ProgressUpdateAppender
diff --git a/gemma-groovy-support/pom.xml b/gemma-groovy-support/pom.xml
index 4ec7c34125..5795de29cf 100644
--- a/gemma-groovy-support/pom.xml
+++ b/gemma-groovy-support/pom.xml
@@ -6,7 +6,7 @@
     <parent>
         <groupId>gemma</groupId>
         <artifactId>gemma</artifactId>
-        <version>1.30.3</version>
+        <version>1.30.4</version>
     </parent>
 
     <artifactId>gemma-groovy-support</artifactId>
diff --git a/gemma-rest/pom.xml b/gemma-rest/pom.xml
index 907726be81..bd55c2d3c5 100644
--- a/gemma-rest/pom.xml
+++ b/gemma-rest/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <artifactId>gemma</artifactId>
         <groupId>gemma</groupId>
-        <version>1.30.3</version>
+        <version>1.30.4</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 
diff --git a/gemma-rest/src/main/java/ubic/gemma/rest/DatasetsWebService.java b/gemma-rest/src/main/java/ubic/gemma/rest/DatasetsWebService.java
index fc177a5c4b..5ccf907947 100644
--- a/gemma-rest/src/main/java/ubic/gemma/rest/DatasetsWebService.java
+++ b/gemma-rest/src/main/java/ubic/gemma/rest/DatasetsWebService.java
@@ -81,10 +81,13 @@
 import javax.ws.rs.core.Response;
 import javax.ws.rs.core.StreamingOutput;
 import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
 import java.io.OutputStreamWriter;
 import java.net.URI;
 import java.util.*;
 import java.util.stream.Collectors;
+import java.util.zip.GZIPInputStream;
 
 /**
  * RESTful interface for datasets.
@@ -296,7 +299,10 @@ public QueriedAndFilteredResponseDataObject<CategoryWithUsageStatisticsValueObje
             @QueryParam("query") String query,
             @QueryParam("filter") @DefaultValue("") FilterArg<ExpressionExperiment> filter,
             @Parameter(description = "Excluded category URIs.", hidden = true) @QueryParam("excludedCategories") StringArrayArg excludedCategoryUris,
+            @Parameter(description = "Exclude free-text categories (i.e. those with null URIs).", hidden = true) @QueryParam("excludeFreeTextCategories") @DefaultValue("false") Boolean excludeFreeTextCategories,
             @Parameter(description = "Excluded term URIs; this list is expanded with subClassOf inference.", hidden = true) @QueryParam("excludedTerms") StringArrayArg excludedTermUris,
+            @Parameter(description = "Exclude free-text terms (i.e. those with null URIs).", hidden = true) @QueryParam("excludeFreeTextTerms") @DefaultValue("false") Boolean excludeFreeTextTerms,
+            @Parameter(description = "Exclude uncategorized terms.", hidden = true) @QueryParam("excludeUncategorizedTerms") @DefaultValue("false") Boolean excludeUncategorizedTerms,
             @Parameter(description = "Retain the categories applicable to terms mentioned in the `filter` parameter even if they are excluded by `excludedCategories` or `excludedTerms`.", hidden = true) @QueryParam("retainMentionedTerms") @DefaultValue("false") Boolean retainMentionedTerms
     ) {
         // ensure that implied terms are retained in the usage frequency
@@ -310,8 +316,8 @@ public QueriedAndFilteredResponseDataObject<CategoryWithUsageStatisticsValueObje
         }
         List<CategoryWithUsageStatisticsValueObject> results = expressionExperimentService.getCategoriesUsageFrequency(
                         filtersWithQuery,
-                        excludedCategoryUris != null ? excludedCategoryUris.getValue() : null,
-                        excludedTermUris != null ? excludedTermUris.getValue() : null,
+                        datasetArgService.getExcludedUris( excludedCategoryUris, excludeFreeTextCategories, excludeUncategorizedTerms ),
+                        datasetArgService.getExcludedUris( excludedTermUris, excludeFreeTextTerms, excludeUncategorizedTerms ),
                         mentionedTerms != null ? mentionedTerms.stream().map( OntologyTerm::getUri ).collect( Collectors.toSet() ) : null )
                 .entrySet()
                 .stream()
@@ -353,7 +359,10 @@ public LimitedResponseDataObject<AnnotationWithUsageStatisticsValueObject> getDa
             @Parameter(description = "Minimum number of associated datasets to report an annotation. If used, the limit will default to " + MAX_DATASETS_ANNOTATIONS + ".") @QueryParam("minFrequency") Integer minFrequency,
             @Parameter(description = "A category URI to restrict reported annotations. If unspecified, annotations from all categories are reported. If empty, uncategorized terms are reported.") @QueryParam("category") String category,
             @Parameter(description = "Excluded category URIs.", hidden = true) @QueryParam("excludedCategories") StringArrayArg excludedCategoryUris,
+            @Parameter(description = "Exclude free-text categories (i.e. those with null URIs).", hidden = true) @QueryParam("excludeFreeTextCategories") @DefaultValue("false") Boolean excludeFreeTextCategories,
             @Parameter(description = "Excluded term URIs; this list is expanded with subClassOf inference.", hidden = true) @QueryParam("excludedTerms") StringArrayArg excludedTermUris,
+            @Parameter(description = "Exclude free-text terms (i.e. those with null URIs).", hidden = true) @QueryParam("excludeFreeTextTerms") @DefaultValue("false") Boolean excludeFreeTextTerms,
+            @Parameter(description = "Exclude uncategorized terms.", hidden = true) @QueryParam("excludeUncategorizedTerms") @DefaultValue("false") Boolean excludeUncategorizedTerms,
             @Parameter(description = "Retain terms mentioned in the `filter` parameter even if they don't meet the `minFrequency` threshold or are excluded via `excludedCategories` or `excludedTerms`.", hidden = true) @QueryParam("retainMentionedTerms") @DefaultValue("false") Boolean retainMentionedTerms ) {
         boolean excludeParentTerms = getExcludedFields( exclude ).contains( "parentTerms" );
         // if a minFrequency is requested, use the hard cap, otherwise use 100 as a reasonable default
@@ -380,8 +389,8 @@ public LimitedResponseDataObject<AnnotationWithUsageStatisticsValueObject> getDa
                 limit,
                 minFrequency != null ? minFrequency : 0,
                 category,
-                excludedCategoryUris != null ? excludedCategoryUris.getValue() : null,
-                excludedTermUris != null ? excludedTermUris.getValue() : null,
+                datasetArgService.getExcludedUris( excludedCategoryUris, excludeFreeTextCategories, excludeUncategorizedTerms ),
+                datasetArgService.getExcludedUris( excludedTermUris, excludeFreeTextTerms, excludeUncategorizedTerms ),
                 mentionedTerms != null ? mentionedTerms.stream().map( OntologyTerm::getUri ).collect( Collectors.toSet() ) : null );
         List<AnnotationWithUsageStatisticsValueObject> results = initialResults
                 .stream()
@@ -719,6 +728,8 @@ public Response getDatasetExpression( // Params:
             return Response.noContent().build();
         } catch ( FilteringException e ) {
             throw new InternalServerErrorException( String.format( "Filtering of dataset %s failed.", ee.getShortName() ), e );
+        } catch ( IOException e ) {
+            throw new InternalServerErrorException( e );
         }
     }
 
@@ -803,7 +814,11 @@ public Response getDatasetDesign( // Params:
             @PathParam("dataset") DatasetArg<?> datasetArg // Required
     ) {
         ExpressionExperiment ee = datasetArgService.getEntity( datasetArg );
-        return this.outputDesignFile( ee );
+        try {
+            return this.outputDesignFile( ee );
+        } catch ( IOException e ) {
+            throw new InternalServerErrorException( e );
+        }
     }
 
     /**
@@ -817,7 +832,7 @@ public Response getDatasetDesign( // Params:
     @Path("/{dataset}/hasbatch")
     @Produces(MediaType.APPLICATION_JSON)
     @Operation(summary = "Indicate of a dataset has batch information", hidden = true)
-    public ResponseDataObject<Boolean> getDatasetHasBatch( // Params:
+    public ResponseDataObject<Boolean> getDatasetHasBatchInformation( // Params:
             @PathParam("dataset") DatasetArg<?> datasetArg // Required
     ) {
         ExpressionExperiment ee = datasetArgService.getEntity( datasetArg );
@@ -981,24 +996,24 @@ public ResponseDataObject<List<ExperimentExpressionLevelsValueObject>> getDatase
         );
     }
 
-    private Response outputDataFile( ExpressionExperiment ee, boolean filter ) throws FilteringException {
+    private Response outputDataFile( ExpressionExperiment ee, boolean filter ) throws FilteringException, IOException {
         ee = expressionExperimentService.thawLite( ee );
         File file = expressionDataFileService.writeOrLocateDataFile( ee, false, filter );
         return this.outputFile( file, DatasetsWebService.ERROR_DATA_FILE_NOT_AVAILABLE, ee.getShortName() );
     }
 
-    private Response outputDesignFile( ExpressionExperiment ee ) {
+    private Response outputDesignFile( ExpressionExperiment ee ) throws IOException {
         ee = expressionExperimentService.thawLite( ee );
         File file = expressionDataFileService.writeOrLocateDesignFile( ee, false );
         return this.outputFile( file, DatasetsWebService.ERROR_DESIGN_FILE_NOT_AVAILABLE, ee.getShortName() );
     }
 
-    private Response outputFile( File file, String error, String shortName ) {
+    private Response outputFile( File file, String error, String shortName ) throws IOException {
         if ( file == null || !file.exists() ) {
             throw new NotFoundException( String.format( error, shortName ) );
         }
         // we remove the .gz extension because we use HTTP Content-Encoding
-        return Response.ok( file )
+        return Response.ok( new GZIPInputStream( new FileInputStream( file ) ) )
                 .header( "Content-Encoding", "gzip" )
                 .header( "Content-Disposition", "attachment; filename=" + FilenameUtils.removeExtension( file.getName() ) )
                 .build();
diff --git a/gemma-rest/src/main/java/ubic/gemma/rest/util/args/DatasetArgService.java b/gemma-rest/src/main/java/ubic/gemma/rest/util/args/DatasetArgService.java
index a74bed7d1d..c6b39e50cf 100644
--- a/gemma-rest/src/main/java/ubic/gemma/rest/util/args/DatasetArgService.java
+++ b/gemma-rest/src/main/java/ubic/gemma/rest/util/args/DatasetArgService.java
@@ -47,6 +47,35 @@ public DatasetArgService( ExpressionExperimentService service, SearchService sea
         this.outlierDetectionService = outlierDetectionService;
     }
 
+    /**
+     * Obtain a list of exclude URIs from an argument containing excluded URIs.
+     * @param excludedUrisArg    argument containing excluded URIs or null if unspecified
+     * @param excludeFreeText if true, null will be included in the returned list which will result in the exclusion of
+     *                        free-text categories or terms
+     * @return null if excludedUrisArg is null and excludeFreeText is false, otherwise a list of excluded URIs
+     */
+    @Nullable
+    public List<String> getExcludedUris( @Nullable StringArrayArg excludedUrisArg, boolean excludeFreeText, boolean excludeUncategorizedTerms ) {
+        List<String> result = null;
+        if ( excludedUrisArg != null ) {
+            result = excludedUrisArg.getValue();
+        }
+        if ( excludeFreeText || excludeUncategorizedTerms ) {
+            if ( result == null ) {
+                result = new ArrayList<>();
+            } else {
+                result = new ArrayList<>( result );
+            }
+        }
+        if ( excludeFreeText ) {
+            result.add( null );
+        }
+        if ( excludeUncategorizedTerms ) {
+            result.add( ExpressionExperimentService.UNCATEGORIZED );
+        }
+        return result;
+    }
+
     @Override
     public Filters getFilters( FilterArg<ExpressionExperiment> filterArg ) throws BadRequestException {
         return getFilters( filterArg, null );
diff --git a/gemma-rest/src/test/resources/log4j-test.properties b/gemma-rest/src/test/resources/log4j-test.properties
index 9dbccf6dbc..694a71249b 100644
--- a/gemma-rest/src/test/resources/log4j-test.properties
+++ b/gemma-rest/src/test/resources/log4j-test.properties
@@ -3,7 +3,7 @@
 log4j.appender.stderr=org.apache.log4j.ConsoleAppender
 log4j.appender.stderr.target=System.out
 log4j.appender.stderr.layout=org.apache.log4j.PatternLayout
-log4j.appender.stderr.layout.ConversionPattern=%d %highlight{%p} %style{%pid}{magenta} [%t] %style{%C.%M(%L)}{cyan} | %m%n
+log4j.appender.stderr.layout.ConversionPattern=%d %highlight{%p} %style{%pid}{magenta} [%t] %style{%C{1.}.%M(%L)}{cyan} | %m%n
 
 log4j.rootLogger=WARN,stderr
 log4j.logger.net.sf.ehcache.hibernate.strategy.AbstractReadWriteEhcacheAccessStrategy=ERROR
diff --git a/gemma-web/pom.xml b/gemma-web/pom.xml
index b9d682efc5..0e5a68b27d 100644
--- a/gemma-web/pom.xml
+++ b/gemma-web/pom.xml
@@ -3,7 +3,7 @@
 	<parent>
 		<artifactId>gemma</artifactId>
 		<groupId>gemma</groupId>
-		<version>1.30.3</version>
+		<version>1.30.4</version>
 	</parent>
 	<modelVersion>4.0.0</modelVersion>
 	<artifactId>gemma-web</artifactId>
diff --git a/gemma-web/src/main/config/log4j-dev.properties b/gemma-web/src/main/config/log4j-dev.properties
index aa3e8fdd78..7e6caf3581 100644
--- a/gemma-web/src/main/config/log4j-dev.properties
+++ b/gemma-web/src/main/config/log4j-dev.properties
@@ -10,7 +10,7 @@
 log4j.appender.stderr=org.apache.log4j.ConsoleAppender
 log4j.appender.stderr.target=System.out
 log4j.appender.stderr.layout=org.apache.log4j.PatternLayout
-log4j.appender.stderr.layout.ConversionPattern=%d %highlight{%p} %style{%pid}{magenta} [%t] %style{%C.%M(%L)}{cyan} | %m%n
+log4j.appender.stderr.layout.ConversionPattern=%d %highlight{%p} %style{%pid}{magenta} [%t] %style{%C{1.}.%M(%L)}{cyan} | %m%n
 
 # Special appender for task progress updates
 log4j.appender.progressUpdate=ubic.gemma.core.job.executor.common.ProgressUpdateAppender
diff --git a/gemma-web/src/main/java/ubic/gemma/web/controller/OntologyController.java b/gemma-web/src/main/java/ubic/gemma/web/controller/OntologyController.java
index 202c8e035a..c154d4aaa1 100644
--- a/gemma-web/src/main/java/ubic/gemma/web/controller/OntologyController.java
+++ b/gemma-web/src/main/java/ubic/gemma/web/controller/OntologyController.java
@@ -11,6 +11,7 @@
 import ubic.basecode.ontology.model.OntologyTerm;
 import ubic.gemma.core.ontology.providers.GemmaOntologyService;
 import ubic.gemma.web.util.EntityNotFoundException;
+import ubic.gemma.web.util.ServiceUnavailableException;
 
 /**
  * Provide minimal support for exposing Gemma ontology.
@@ -32,13 +33,17 @@ public RedirectView getOntology() {
 
     @RequestMapping(value = "/ont/{termId}", method = RequestMethod.GET)
     public RedirectView getTerm( @PathVariable("termId") String termId ) {
-        OntologyTerm term = gemmaOntologyService.getTerm( "http://gemma.msl.ubc.ca/ont/" + termId );
+        if ( !gemmaOntologyService.isOntologyLoaded() ) {
+            throw new ServiceUnavailableException( "TGEMO is not loaded." );
+        }
+        String iri = "http://gemma.msl.ubc.ca/ont/" + termId;
+        OntologyTerm term = gemmaOntologyService.getTerm( iri );
         if ( term == null ) {
-            throw new EntityNotFoundException( String.format( "No term with ID %s in TGEMO.", termId ) );
+            throw new EntityNotFoundException( String.format( "No term with IRI %s in TGEMO.", iri ) );
         }
         String gemmaOntologyUrl = gemmaOntologyService.getOntologyUrl();
         String urlWithFragment = UriComponentsBuilder.fromHttpUrl( gemmaOntologyUrl )
-                .fragment( term.getLocalName() )
+                .fragment( iri )
                 .build()
                 .toUriString();
         RedirectView redirectView = new RedirectView( urlWithFragment );
diff --git a/gemma-web/src/main/java/ubic/gemma/web/controller/expression/experiment/ExperimentalDesignControllerImpl.java b/gemma-web/src/main/java/ubic/gemma/web/controller/expression/experiment/ExperimentalDesignControllerImpl.java
index 2b8a08788a..ef78f8954e 100644
--- a/gemma-web/src/main/java/ubic/gemma/web/controller/expression/experiment/ExperimentalDesignControllerImpl.java
+++ b/gemma-web/src/main/java/ubic/gemma/web/controller/expression/experiment/ExperimentalDesignControllerImpl.java
@@ -20,6 +20,7 @@
 
 import gemma.gsec.SecurityService;
 import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.math.NumberUtils;
 import org.directwebremoting.extend.AccessDeniedException;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Controller;
@@ -303,13 +304,13 @@ private void filterCharacteristics( Collection<BioMaterialValueObject> result )
             for ( CharacteristicValueObject ch : bmo.getCharacteristics() ) {
 
                 String category = ch.getCategory();
+                String value = ch.getValue();
                 if ( StringUtils.isBlank( category ) ) {
-
                     /*
-                    Experimental: split on ":", use first part as the category.
+                    Experimental: split on ":" or "=", use first part as the category.
                      */
-                    if ( StringUtils.isNotBlank( ch.getValue() ) && ch.getValue().contains( ":" ) ) {
-                        String[] split = ch.getValue().split( ":" );
+                    if ( StringUtils.isNotBlank( value ) && value.matches( ".+[:=].+" ) ) { // note: GEO only allows ":" now but we have "=" in the db for older entries.
+                        String[] split = value.split( "[:=]", 2 );
                         category = StringUtils.strip( split[0] );
                     } else {
                         continue;
@@ -328,7 +329,7 @@ private void filterCharacteristics( Collection<BioMaterialValueObject> result )
          */
         Collection<String> toremove = new HashSet<>();
         for ( String category : map.keySet() ) {
-            // log.info( ">>>>>>>>>> " + category + ", " + map.get( category ).size() + " items" );
+            //log.info( ">>>>>>>>>> " + category + ", " + map.get( category ).size() + " items" );
             if ( map.get( category ).size() != result.size() ) {
                 toremove.add( category );
                 continue;
@@ -352,9 +353,10 @@ private void filterCharacteristics( Collection<BioMaterialValueObject> result )
 
                     if ( StringUtils.isBlank( mappedCategory ) ) {
                         // redo split (will refactor later)
-                        if ( StringUtils.isNotBlank( mappedValue ) && mappedValue.contains( ":" ) ) {
-                            String[] split = mappedValue.split( ":" );
+                        if ( StringUtils.isNotBlank( mappedValue ) && mappedValue.matches( ".+[:=].+" ) ) {
+                            String[] split = mappedValue.split( "[:=]", 2 );
                             mappedCategory = StringUtils.strip( split[0] );
+                            mappedValue = StringUtils.strip( split[1] ); // to show the trimmed up value.
                         } else {
                             continue bms;
                         }
@@ -381,7 +383,7 @@ private void filterCharacteristics( Collection<BioMaterialValueObject> result )
                 //                }
             }
 
-            if ( vals.size() < 2 ) {
+            if ( vals.size() < 2 ) { // constant
                 toremove.add( category );
             }
         }
diff --git a/gemma-web/src/main/java/ubic/gemma/web/util/ServiceUnavailableException.java b/gemma-web/src/main/java/ubic/gemma/web/util/ServiceUnavailableException.java
new file mode 100644
index 0000000000..dd9a61fe7d
--- /dev/null
+++ b/gemma-web/src/main/java/ubic/gemma/web/util/ServiceUnavailableException.java
@@ -0,0 +1,8 @@
+package ubic.gemma.web.util;
+
+public class ServiceUnavailableException extends RuntimeException {
+
+    public ServiceUnavailableException( String message ) {
+        super( message );
+    }
+}
diff --git a/gemma-web/src/main/resources/messages.properties b/gemma-web/src/main/resources/messages.properties
index bcaf3c4c28..90f12558fe 100644
--- a/gemma-web/src/main/resources/messages.properties
+++ b/gemma-web/src/main/resources/messages.properties
@@ -16,6 +16,8 @@ errorPage.info.missing=The cause of the error was not recovered
 404.message=The page you requested was not found. You might try returning to the <a href="{0}">home page</a>.
 403.title=Access Denied
 403.message=You are not authorized to access this page. If you believe this is an error, please contact the webmaster. You might try returning to the <a href="{0}">home page</a>.
+503.title=Service Unavailable
+503.message=The page you requested is not available. You might try returning to the <a href="{0}">home page</a>.
 
 error.noCriteria=You must enter some search criteria.
 
diff --git a/gemma-web/src/main/webapp/503.jsp b/gemma-web/src/main/webapp/503.jsp
new file mode 100644
index 0000000000..3ab99a478c
--- /dev/null
+++ b/gemma-web/src/main/webapp/503.jsp
@@ -0,0 +1,7 @@
+<%@ include file="/common/taglibs.jsp" %>
+
+<%@ page isErrorPage="true" %>
+
+<page:applyDecorator name="default">
+    <%@ include file="/pages/error/503.jsp" %>
+</page:applyDecorator>
\ No newline at end of file
diff --git a/gemma-web/src/main/webapp/WEB-INF/gemma-servlet.xml b/gemma-web/src/main/webapp/WEB-INF/gemma-servlet.xml
index 4c82364d5e..a44f4d4d9c 100644
--- a/gemma-web/src/main/webapp/WEB-INF/gemma-servlet.xml
+++ b/gemma-web/src/main/webapp/WEB-INF/gemma-servlet.xml
@@ -108,6 +108,7 @@
             <props>
                 <prop key="org.springframework.security.access.AccessDeniedException">error/403</prop>
                 <prop key="ubic.gemma.web.util.EntityNotFoundException">error/404</prop>
+                <prop key="ubic.gemma.web.util.ServiceUnavailableException">error/503</prop>
                 <prop key="java.lang.IllegalArgumentException">error/400</prop>
             </props>
         </property>
diff --git a/gemma-web/src/main/webapp/WEB-INF/web.xml b/gemma-web/src/main/webapp/WEB-INF/web.xml
index f1bd545340..95c1a4eacc 100644
--- a/gemma-web/src/main/webapp/WEB-INF/web.xml
+++ b/gemma-web/src/main/webapp/WEB-INF/web.xml
@@ -325,9 +325,15 @@
     </error-page>
     <error-page>
         <error-code>503</error-code>
-        <location>/500.jsp</location>
+        <location>/503.jsp</location>
     </error-page>
 
+    <!-- https://github.com/PavlidisLab/Gemma/issues/923 -->
+    <mime-mapping>
+        <extension>js</extension>
+        <mime-type>application/javascript</mime-type>
+    </mime-mapping>
+
     <!--  for web fonts -->
     <mime-mapping>
         <extension>eot</extension>
diff --git a/gemma-web/src/main/webapp/scripts/api/entities/gene/GeneDetailsTab.js b/gemma-web/src/main/webapp/scripts/api/entities/gene/GeneDetailsTab.js
index 2013905b06..462437ba14 100755
--- a/gemma-web/src/main/webapp/scripts/api/entities/gene/GeneDetailsTab.js
+++ b/gemma-web/src/main/webapp/scripts/api/entities/gene/GeneDetailsTab.js
@@ -92,8 +92,8 @@ Gemma.GeneDetails = Ext.extend(Ext.Panel, {
         if (geneDetails.multifunctionalityRank) {
             text = geneDetails.numGoTerms + " GO Terms; Overall multifunctionality "
                 + geneDetails.multifunctionalityRank.toFixed(2);
-            text += "&nbsp;<img style='cursor:pointer' src='" + ctxBasePath + "/images/magnifier.png' ext:qtip='View the GO term tab'"
-                + "onClick='Ext.getCmp(&#39;" + this.id + "&#39;).changeTab(&#39;goGrid&#39;)'>";
+            // text += "&nbsp;<img style='cursor:pointer' src='" + ctxBasePath + "/images/magnifier.png' ext:qtip='View the GO term tab'"
+            //     + "onClick='Ext.getCmp(&#39;" + this.id + "&#39;).changeTab(&#39;goGrid&#39;)'>";
         } else {
             text = "[ Not available ]";
         }
diff --git a/gemma-web/src/test/java/ubic/gemma/web/controller/OntologyControllerTest.java b/gemma-web/src/test/java/ubic/gemma/web/controller/OntologyControllerTest.java
index ab4aee2931..923ee36504 100644
--- a/gemma-web/src/test/java/ubic/gemma/web/controller/OntologyControllerTest.java
+++ b/gemma-web/src/test/java/ubic/gemma/web/controller/OntologyControllerTest.java
@@ -37,7 +37,7 @@ public void testGetObo() throws Exception {
     public void testGetTerm() throws Exception {
         mvc.perform( get( "/ont/TGEMO_00001" ) )
                 .andExpect( status().isFound() )
-                .andExpect( redirectedUrl( gemmaOntologyUrl + "#TGEMO_00001" ) );
+                .andExpect( redirectedUrl( gemmaOntologyUrl + "#http://gemma.msl.ubc.ca/ont/TGEMO_00001" ) );
     }
 
     @Test
diff --git a/gemma-web/src/test/resources/log4j-test.properties b/gemma-web/src/test/resources/log4j-test.properties
index a4ece3bf91..be96cd079b 100644
--- a/gemma-web/src/test/resources/log4j-test.properties
+++ b/gemma-web/src/test/resources/log4j-test.properties
@@ -4,7 +4,7 @@
 log4j.appender.stderr=org.apache.log4j.ConsoleAppender
 log4j.appender.stderr.target=System.out
 log4j.appender.stderr.layout=org.apache.log4j.PatternLayout
-log4j.appender.stderr.layout.ConversionPattern=%d %highlight{%p} %style{%pid}{magenta} [%t] %style{%C.%M(%L)}{cyan} | %m%n
+log4j.appender.stderr.layout.ConversionPattern=%d %highlight{%p} %style{%pid}{magenta} [%t] %style{%C{1.}.%M(%L)}{cyan} | %m%n
 
 # this is used for SlackAppenderTest and will be ignored if -Dgemma.slack.token or -Dgemma.slack.channel are not set
 log4j.appender.slack=ubic.gemma.web.logging.SlackAppender
diff --git a/pom.xml b/pom.xml
index 29dc92c92e..794bdc7aeb 100644
--- a/pom.xml
+++ b/pom.xml
@@ -5,7 +5,7 @@
 	<name>Gemma</name>
 	<groupId>gemma</groupId>
 	<artifactId>gemma</artifactId>
-	<version>1.30.3</version>
+	<version>1.30.4</version>
 	<inceptionYear>2005</inceptionYear>
 	<description>The Gemma Project for meta-analysis of genomics data</description>
 	<url>https://gemma.msl.ubc.ca</url>
@@ -638,6 +638,7 @@
 		<gsec.version>0.0.15</gsec.version>
 		<spring.version>3.2.18.RELEASE</spring.version>
 		<spring.security.version>3.2.10.RELEASE</spring.security.version>
+		<tomcat.version>9.0.80</tomcat.version>
 		<jersey.version>2.25.1</jersey.version>
 		<jackson.version>2.15.2</jackson.version>
 		<swagger.version>2.2.11</swagger.version>