From 8b9f9343970ed53466cb65721ce25e6e9e4f6b6c Mon Sep 17 00:00:00 2001 From: Paul Pavlidis Date: Tue, 18 Apr 2023 12:43:26 -0700 Subject: [PATCH 001/370] updates for finding obsolete ontology terms --- .../core/apps/CountObsoleteTermsCli.java | 94 ------------------- .../gemma/core/apps/FindObsoleteTermsCli.java | 71 ++++++++++++++ .../gemma/core/ontology/OntologyService.java | 25 ++++- .../core/ontology/OntologyServiceImpl.java | 90 ++++++++++-------- .../experiment/AnnotationController.java | 2 +- 5 files changed, 145 insertions(+), 137 deletions(-) delete mode 100644 gemma-cli/src/main/java/ubic/gemma/core/apps/CountObsoleteTermsCli.java create mode 100644 gemma-cli/src/main/java/ubic/gemma/core/apps/FindObsoleteTermsCli.java diff --git a/gemma-cli/src/main/java/ubic/gemma/core/apps/CountObsoleteTermsCli.java b/gemma-cli/src/main/java/ubic/gemma/core/apps/CountObsoleteTermsCli.java deleted file mode 100644 index 6e23bcdf91..0000000000 --- a/gemma-cli/src/main/java/ubic/gemma/core/apps/CountObsoleteTermsCli.java +++ /dev/null @@ -1,94 +0,0 @@ -package ubic.gemma.core.apps; - -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.Option; -import org.apache.commons.cli.Options; -import ubic.gemma.core.ontology.OntologyService; -import ubic.gemma.core.util.AbstractCLI; -import ubic.gemma.core.util.AbstractCLIContextCLI; -import ubic.gemma.model.genome.gene.phenotype.valueObject.CharacteristicValueObject; - -import java.util.Map; - -public class CountObsoleteTermsCli extends AbstractCLIContextCLI { - private OntologyService ontologyService; - - private String startArg; - private String stepArg = "100000"; - private String stopArg; - - @Override - public GemmaCLI.CommandGroup getCommandGroup() { - return GemmaCLI.CommandGroup.EXPERIMENT; - } - - @Override - public String getShortDesc() { - return "Check for characteristics using obsolete terms in the given ID range."; - } - - @Override - protected void processOptions( CommandLine commandLine ) { - ontologyService = this.getBean( OntologyService.class ); - - if ( commandLine.hasOption( "start" ) ) { - this.startArg = commandLine.getOptionValue( "start" ); - } - - if ( commandLine.hasOption( "step" ) ) { - this.stepArg = commandLine.getOptionValue( "step" ); - } - - if ( commandLine.hasOption( "stop" ) ) { - this.stopArg = commandLine.getOptionValue( "stop" ); - } - } - - @Override - public String getCommandName() { - return "countObsoleteTerms"; - } - - @SuppressWarnings("AccessStaticViaInstance") - @Override - protected void buildOptions( Options options ) { - - Option startOption = Option.builder( "start" ) - .desc( "The ID of the first characteristic to check, i.e the end of the ID range." ).hasArg().required() - .build(); - options.addOption( startOption ); - - Option stepOption = Option.builder( "step" ) - .desc( "The amount of characteristics to load in one batch. Default value is 100000." ).hasArg() - .build(); - options.addOption( stepOption ); - - Option stopOption = Option.builder( "stop" ) - .desc( "The ID of the last characteristic to check, i.e the end of the ID range." ).hasArg().required() - .build(); - options.addOption( stopOption ); - } - - @Override - protected void doWork() throws Exception { - int start = Integer.parseInt( startArg ) + 1; - int step = Integer.parseInt( stepArg ) + 1; - int stop = Integer.parseInt( stopArg ) + 1; - - Map vos = ontologyService.countObsoleteOccurrences( start, stop, step ); - - // Output results - - AbstractCLI.log.info( "======================================================" ); - AbstractCLI.log.info( "Obsolete term check finished." ); - AbstractCLI.log.info( "Below is a tab-separated output of all found terms:" ); - AbstractCLI.log.info( "======================================================" ); - AbstractCLI.log.info( "======================================================" ); - AbstractCLI.log.info( "Value\tValueUri\tCount" ); - for ( CharacteristicValueObject vo : vos.values() ) { - AbstractCLI.log.info( vo.getValue() + "\t" + vo.getValueUri() + "\t" + vo.getNumTimesUsed() ); - } - AbstractCLI.log.info( "======================================================" ); - AbstractCLI.log.info( "======================================================" ); - } -} diff --git a/gemma-cli/src/main/java/ubic/gemma/core/apps/FindObsoleteTermsCli.java b/gemma-cli/src/main/java/ubic/gemma/core/apps/FindObsoleteTermsCli.java new file mode 100644 index 0000000000..4e256b64a8 --- /dev/null +++ b/gemma-cli/src/main/java/ubic/gemma/core/apps/FindObsoleteTermsCli.java @@ -0,0 +1,71 @@ +package ubic.gemma.core.apps; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.springframework.beans.factory.annotation.Autowired; +import ubic.gemma.core.ontology.OntologyService; +import ubic.gemma.core.util.AbstractCLI; +import ubic.gemma.core.util.AbstractCLIContextCLI; +import ubic.gemma.model.genome.gene.phenotype.valueObject.CharacteristicValueObject; + +import java.util.Map; + +public class FindObsoleteTermsCli extends AbstractCLIContextCLI { + + @Autowired + private OntologyService ontologyService; + + private int start = 1; + private int step = 100000; + private int stop = -1; + + @Override + public GemmaCLI.CommandGroup getCommandGroup() { + return GemmaCLI.CommandGroup.METADATA; + } + + @Override + public String getShortDesc() { + return "Check for characteristics using obsolete terms as values (excluding GO)"; + } + + @Override + protected void processOptions( CommandLine commandLine ) { + // no extra options. + } + + @Override + public String getCommandName() { + return "findObsoleteTerms"; + } + + @SuppressWarnings("AccessStaticViaInstance") + @Override + protected void buildOptions( Options options ) { + } + + @Override + protected void doWork() throws Exception { + log.info( "Warming up ontologies ..." ); + ontologyService.initializeAllOntologies(); + + while ( ontologyService.isInitializing() ) { + log.info( "Waiting for ontologies to warm up ..." ); + Thread.sleep( 5000 ); + } + + log.info( "Ontologies warmed up, starting check..." ); + + Map vos = ontologyService.findObsoleteTermUsage( ); + + AbstractCLI.log.info( "Obsolete term check finished, printing ..." ); + + + System.out.println( "Value\tValueUri\tCount" ); + for ( CharacteristicValueObject vo : vos.values() ) { + System.out.println( vo.getValue() + "\t" + vo.getValueUri() + "\t" + vo.getNumTimesUsed() ); + } + AbstractCLI.log.info( "========= done ========" ); + } +} diff --git a/gemma-core/src/main/java/ubic/gemma/core/ontology/OntologyService.java b/gemma-core/src/main/java/ubic/gemma/core/ontology/OntologyService.java index 032b289dbf..e1abe1465a 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/ontology/OntologyService.java +++ b/gemma-core/src/main/java/ubic/gemma/core/ontology/OntologyService.java @@ -29,7 +29,6 @@ import javax.annotation.Nullable; import java.util.Collection; -import java.util.List; import java.util.Map; import java.util.Set; @@ -48,7 +47,13 @@ public interface OntologyService { */ void addExpressionExperimentStatement( Characteristic vc, ExpressionExperiment ee ); - Map countObsoleteOccurrences( int start, int stop, int step ); + /** + * Locates usages of obsolete terms in Characteristics, ignoring Gene Ontology annotations. + * + * @return map of value URI to a representative characteristic using the term. The latter will contain a count + * of how many ocurrences there were. + */ + Map findObsoleteTermUsage( ); /** * Using the ontology and values in the database, for a search searchQuery given by the client give an ordered list @@ -185,10 +190,10 @@ Collection findExperimentsCharacteristicTags( String void reindexAllOntologies(); /** - * Reinitialize all the ontologies "from scratch". This is necessary if indices are old etc. This should be + * Reinitialize (and reindex) all the ontologies "from scratch". This is necessary if indices are old etc. This should be * admin-only. */ - void reinitializeAllOntologies(); + void reinitializeAndReindexAllOntologies(); void removeBioMaterialStatement( Long characterId, BioMaterial bm ); @@ -201,4 +206,16 @@ Collection findExperimentsCharacteristicTags( String void saveBioMaterialStatement( Characteristic vc, BioMaterial bm ); Collection termsToCharacteristics( Collection terms ); + + /** + * Force all ontologies to be initialized (enabled). Useful if a CLI needs access to all ontologies. + */ + void initializeAllOntologies(); + + /** + * + * @return true if any ontologies are still inthe process of being initialized. False if no ontologies are being initialized. + * This means they either were initialized or failed to initialize, or weren't configured to be initialized. + */ + boolean isInitializing(); } \ No newline at end of file diff --git a/gemma-core/src/main/java/ubic/gemma/core/ontology/OntologyServiceImpl.java b/gemma-core/src/main/java/ubic/gemma/core/ontology/OntologyServiceImpl.java index 8a6069fa38..03581f18d3 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/ontology/OntologyServiceImpl.java +++ b/gemma-core/src/main/java/ubic/gemma/core/ontology/OntologyServiceImpl.java @@ -108,9 +108,9 @@ public class OntologyServiceImpl implements OntologyService, InitializingBean { private DiseaseOntologyService diseaseOntologyService; @Autowired private ExperimentalFactorOntologyService experimentalFactorOntologyService; - @Deprecated - @Autowired - private FMAOntologyService fmaOntologyService; +// @Deprecated +// @Autowired +// private FMAOntologyService fmaOntologyService; @Autowired private GemmaOntologyService gemmaOntologyService; @Autowired @@ -121,9 +121,9 @@ public class OntologyServiceImpl implements OntologyService, InitializingBean { private MammalianPhenotypeOntologyService mammalianPhenotypeOntologyService; @Autowired private MouseDevelopmentOntologyService mouseDevelopmentOntologyService; - @Deprecated - @Autowired - private NIFSTDOntologyService nifstdOntologyService; +// @Deprecated +// @Autowired +// private NIFSTDOntologyService nifstdOntologyService; @Autowired private ObiService obiService; @Autowired @@ -153,7 +153,7 @@ public void afterPropertiesSet() throws Exception { .filter( ubic.basecode.ontology.providers.OntologyService::isEnabled ) .collect( Collectors.toList() ); if ( enabledOntologyServices.isEmpty() ) { - log.warn( "No ontology services are enabled, consider enabling them by setting 'load.{name}Ontology' options in Gemma.properties." ); + log.info( "No ontology services are enabled, consider enabling them by setting 'load.{name}Ontology' options in Gemma.properties." ); } else { log.info( "The following ontology services are enabled:\n\t" + enabledOntologyServices.stream() .map( ubic.basecode.ontology.providers.OntologyService::toString ) @@ -537,7 +537,7 @@ public void reindexAllOntologies() { } @Override - public void reinitializeAllOntologies() { + public void reinitializeAndReindexAllOntologies() { for ( ubic.basecode.ontology.providers.OntologyService serv : this.ontologyServices ) { serv.startInitializationThread( true, true ); } @@ -635,58 +635,72 @@ public Collection termsToCharacteristics( final Collection countObsoleteOccurrences( int start, int stop, int step ) { - Map vos = new HashMap<>(); - - int minId = start; - int maxId = step; + public void initializeAllOntologies() { + for ( ubic.basecode.ontology.providers.OntologyService serv : this.ontologyServices ) { + if ( serv.isOntologyLoaded() ) { + log.info( "Already loaded: " + serv ); + continue; + } + log.info( "Initializing " + serv ); + serv.startInitializationThread( true, false ); + } + } - int nullCnt = 0; - int obsoleteCnt = 0; + @Override + public boolean isInitializing() { + return ontologyServices.stream().allMatch( o -> !o.isInitializationThreadAlive() || o.isOntologyLoaded() ); + } - // Loading all characteristics in steps - while ( maxId < stop ) { + @Override + public Map findObsoleteTermUsage() { + Map vos = new HashMap<>(); - OntologyServiceImpl.log.info( "Checking characteristics with IDs between " + minId + " and " + maxId ); + int start = 0; + int step = 5000; - List ids = new ArrayList<>( step ); - for ( int i = minId; i < maxId + 1; i++ ) { - ids.add( ( long ) i ); - } + int obsoleteCnt = 0; + int prevObsoleteCnt = 0; - minId = maxId + 1; - maxId += step; + while ( true ) { - Collection chars = characteristicService.load( ids ); + Collection chars = characteristicService.browse( start, step ); + start += step; if ( chars == null || chars.isEmpty() ) { - OntologyServiceImpl.log.info( "No characteristics in the current ID range, moving on." ); - continue; + break; } - OntologyServiceImpl.log.info( "Found " + chars.size() + + OntologyServiceImpl.log.debug( "Found " + chars.size() + " characteristics in the current ID range, checking for obsoletes." ); - // Detect obsoletes + CharacteristicValueObject lastObsolete = null; for ( Characteristic ch : chars ) { if ( StringUtils.isBlank( ch.getValueUri() ) ) { - nullCnt++; - } else if ( this.isObsolete( ch.getValueUri() ) ) { + continue; + } + + if ( this.isObsolete( ch.getValueUri() ) ) { String key = this.foundValueKey( ch ); if ( !vos.containsKey( key ) ) { vos.put( key, new CharacteristicValueObject( ch ) ); } vos.get( key ).incrementOccurrenceCount(); obsoleteCnt++; - OntologyServiceImpl.log.info( "Found obsolete term: " + ch.getValue() + " / " + ch.getValueUri() ); + if ( log.isDebugEnabled() ) + OntologyServiceImpl.log.debug( "Found obsolete term: " + ch.getValue() + " -" + ch.getValueUri() ); + lastObsolete = vos.get( key ); } } - ids.clear(); - chars.clear(); + if ( obsoleteCnt > prevObsoleteCnt ) { + OntologyServiceImpl.log.info( "Found " + obsoleteCnt + " obsolete terms so far." ); + OntologyServiceImpl.log.info( "Last obsolete term seen: " + lastObsolete.getValue() + " -" + lastObsolete.getValueUri() ); + } + + prevObsoleteCnt = obsoleteCnt; } - OntologyServiceImpl.log.info( "Terms with empty uri: " + nullCnt ); - OntologyServiceImpl.log.info( "Obsolete terms found: " + obsoleteCnt ); + OntologyServiceImpl.log.info( "Done, obsolete terms found: " + obsoleteCnt ); return vos; } @@ -803,8 +817,8 @@ private Collection findCharacteristicsFromOntology( S List ontologyServicesToUse; if ( useNeuroCartaOntology ) { ontologyServicesToUse = Arrays.asList( - nifstdOntologyService, - fmaOntologyService, +// nifstdOntologyService, +// fmaOntologyService, obiService ); } else { ontologyServicesToUse = this.ontologyServices; diff --git a/gemma-web/src/main/java/ubic/gemma/web/controller/expression/experiment/AnnotationController.java b/gemma-web/src/main/java/ubic/gemma/web/controller/expression/experiment/AnnotationController.java index d5246386db..8ed3aad3b4 100644 --- a/gemma-web/src/main/java/ubic/gemma/web/controller/expression/experiment/AnnotationController.java +++ b/gemma-web/src/main/java/ubic/gemma/web/controller/expression/experiment/AnnotationController.java @@ -132,7 +132,7 @@ public void reinitializeOntologyIndices() { log.warn( "Attempt to run ontology re-indexing as non-admin." ); return; } - ontologyService.reinitializeAllOntologies(); + ontologyService.reinitializeAndReindexAllOntologies(); } public void removeBiomaterialTag( Characteristic vc, Long id ) { From d5cfacdc82a6cc01ee2e6470a6dfc03e6bd9fb00 Mon Sep 17 00:00:00 2001 From: Paul Pavlidis Date: Tue, 18 Apr 2023 13:59:42 -0700 Subject: [PATCH 002/370] progress --- .../gemma/core/apps/FindObsoleteTermsCli.java | 18 +++++++++-- .../gemma/core/ontology/OntologyService.java | 9 ++---- .../core/ontology/OntologyServiceImpl.java | 31 ++++++------------- .../gemma/applicationContext-ontology.xml | 4 +-- 4 files changed, 30 insertions(+), 32 deletions(-) diff --git a/gemma-cli/src/main/java/ubic/gemma/core/apps/FindObsoleteTermsCli.java b/gemma-cli/src/main/java/ubic/gemma/core/apps/FindObsoleteTermsCli.java index 4e256b64a8..a4027b2fab 100644 --- a/gemma-cli/src/main/java/ubic/gemma/core/apps/FindObsoleteTermsCli.java +++ b/gemma-cli/src/main/java/ubic/gemma/core/apps/FindObsoleteTermsCli.java @@ -5,10 +5,12 @@ import org.apache.commons.cli.Options; import org.springframework.beans.factory.annotation.Autowired; import ubic.gemma.core.ontology.OntologyService; +import ubic.gemma.core.ontology.providers.OntologyServiceFactory; import ubic.gemma.core.util.AbstractCLI; import ubic.gemma.core.util.AbstractCLIContextCLI; import ubic.gemma.model.genome.gene.phenotype.valueObject.CharacteristicValueObject; +import java.util.List; import java.util.Map; public class FindObsoleteTermsCli extends AbstractCLIContextCLI { @@ -16,6 +18,9 @@ public class FindObsoleteTermsCli extends AbstractCLIContextCLI { @Autowired private OntologyService ontologyService; + @Autowired + private List ontologies; + private int start = 1; private int step = 100000; private int stop = -1; @@ -48,10 +53,19 @@ protected void buildOptions( Options options ) { @Override protected void doWork() throws Exception { log.info( "Warming up ontologies ..." ); - ontologyService.initializeAllOntologies(); + for ( ubic.basecode.ontology.providers.OntologyService ontology : ontologies ) { + ontology.startInitializationThread( true, false ); + } + + for ( ubic.basecode.ontology.providers.OntologyService ontology : ontologies ) { + ontology.waitForInitializationThread(); + } + + log.info( "Waiting for ontologies to warm up ..." ); + ontologyService.waitForAllOntologiesToLoad(); while ( ontologyService.isInitializing() ) { - log.info( "Waiting for ontologies to warm up ..." ); + Thread.sleep( 5000 ); } diff --git a/gemma-core/src/main/java/ubic/gemma/core/ontology/OntologyService.java b/gemma-core/src/main/java/ubic/gemma/core/ontology/OntologyService.java index e1abe1465a..647d4ac9ef 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/ontology/OntologyService.java +++ b/gemma-core/src/main/java/ubic/gemma/core/ontology/OntologyService.java @@ -48,12 +48,12 @@ public interface OntologyService { void addExpressionExperimentStatement( Characteristic vc, ExpressionExperiment ee ); /** - * Locates usages of obsolete terms in Characteristics, ignoring Gene Ontology annotations. + * Locates usages of obsolete terms in Characteristics, ignoring Gene Ontology annotations. Requires the ontologies are loaded into memory. * * @return map of value URI to a representative characteristic using the term. The latter will contain a count * of how many ocurrences there were. */ - Map findObsoleteTermUsage( ); + Map findObsoleteTermUsage(); /** * Using the ontology and values in the database, for a search searchQuery given by the client give an ordered list @@ -207,11 +207,6 @@ Collection findExperimentsCharacteristicTags( String Collection termsToCharacteristics( Collection terms ); - /** - * Force all ontologies to be initialized (enabled). Useful if a CLI needs access to all ontologies. - */ - void initializeAllOntologies(); - /** * * @return true if any ontologies are still inthe process of being initialized. False if no ontologies are being initialized. diff --git a/gemma-core/src/main/java/ubic/gemma/core/ontology/OntologyServiceImpl.java b/gemma-core/src/main/java/ubic/gemma/core/ontology/OntologyServiceImpl.java index 03581f18d3..a1f5fa479e 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/ontology/OntologyServiceImpl.java +++ b/gemma-core/src/main/java/ubic/gemma/core/ontology/OntologyServiceImpl.java @@ -22,9 +22,9 @@ import org.apache.commons.lang3.time.StopWatch; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.compass.core.util.concurrent.ConcurrentHashSet; import org.springframework.beans.factory.InitializingBean; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.core.io.ClassPathResource; import org.springframework.core.io.Resource; import org.springframework.core.task.AsyncTaskExecutor; @@ -108,7 +108,7 @@ public class OntologyServiceImpl implements OntologyService, InitializingBean { private DiseaseOntologyService diseaseOntologyService; @Autowired private ExperimentalFactorOntologyService experimentalFactorOntologyService; -// @Deprecated + // @Deprecated // @Autowired // private FMAOntologyService fmaOntologyService; @Autowired @@ -121,7 +121,7 @@ public class OntologyServiceImpl implements OntologyService, InitializingBean { private MammalianPhenotypeOntologyService mammalianPhenotypeOntologyService; @Autowired private MouseDevelopmentOntologyService mouseDevelopmentOntologyService; -// @Deprecated + // @Deprecated // @Autowired // private NIFSTDOntologyService nifstdOntologyService; @Autowired @@ -634,21 +634,9 @@ public Collection termsToCharacteristics( final Collection !o.isInitializationThreadAlive() || o.isOntologyLoaded() ); + return ontologyServices.stream().allMatch( o -> o.isInitializationThreadAlive() ); } @Override @@ -660,7 +648,7 @@ public Map findObsoleteTermUsage() { int obsoleteCnt = 0; int prevObsoleteCnt = 0; - + int checked = 0; while ( true ) { Collection chars = characteristicService.browse( start, step ); @@ -679,28 +667,29 @@ public Map findObsoleteTermUsage() { continue; } + checked++; + if ( this.isObsolete( ch.getValueUri() ) ) { String key = this.foundValueKey( ch ); if ( !vos.containsKey( key ) ) { vos.put( key, new CharacteristicValueObject( ch ) ); } vos.get( key ).incrementOccurrenceCount(); - obsoleteCnt++; if ( log.isDebugEnabled() ) - OntologyServiceImpl.log.debug( "Found obsolete term: " + ch.getValue() + " -" + ch.getValueUri() ); + OntologyServiceImpl.log.debug( "Found obsolete term: " + ch.getValue() + " - " + ch.getValueUri() ); lastObsolete = vos.get( key ); } } if ( obsoleteCnt > prevObsoleteCnt ) { - OntologyServiceImpl.log.info( "Found " + obsoleteCnt + " obsolete terms so far." ); + OntologyServiceImpl.log.info( "Found " + vos.size() + " obsolete terms so far, tested " + checked + " characteristics" ); OntologyServiceImpl.log.info( "Last obsolete term seen: " + lastObsolete.getValue() + " -" + lastObsolete.getValueUri() ); } prevObsoleteCnt = obsoleteCnt; } - OntologyServiceImpl.log.info( "Done, obsolete terms found: " + obsoleteCnt ); + OntologyServiceImpl.log.info( "Done, obsolete terms found: " + vos.size() ); return vos; } diff --git a/gemma-core/src/main/resources/ubic/gemma/applicationContext-ontology.xml b/gemma-core/src/main/resources/ubic/gemma/applicationContext-ontology.xml index 8a6beca6e9..8516dea4c7 100644 --- a/gemma-core/src/main/resources/ubic/gemma/applicationContext-ontology.xml +++ b/gemma-core/src/main/resources/ubic/gemma/applicationContext-ontology.xml @@ -12,12 +12,12 @@ - + From 968ec77f66329b997a1b40a4b9f7ca24f6c8e836 Mon Sep 17 00:00:00 2001 From: Paul Pavlidis Date: Tue, 18 Apr 2023 14:02:00 -0700 Subject: [PATCH 003/370] oops --- .../java/ubic/gemma/core/apps/FindObsoleteTermsCli.java | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/gemma-cli/src/main/java/ubic/gemma/core/apps/FindObsoleteTermsCli.java b/gemma-cli/src/main/java/ubic/gemma/core/apps/FindObsoleteTermsCli.java index a4027b2fab..07d1ddcdca 100644 --- a/gemma-cli/src/main/java/ubic/gemma/core/apps/FindObsoleteTermsCli.java +++ b/gemma-cli/src/main/java/ubic/gemma/core/apps/FindObsoleteTermsCli.java @@ -57,18 +57,11 @@ protected void doWork() throws Exception { ontology.startInitializationThread( true, false ); } + log.info( "Waiting for ontologies to warm up ..." ); for ( ubic.basecode.ontology.providers.OntologyService ontology : ontologies ) { ontology.waitForInitializationThread(); } - log.info( "Waiting for ontologies to warm up ..." ); - ontologyService.waitForAllOntologiesToLoad(); - - while ( ontologyService.isInitializing() ) { - - Thread.sleep( 5000 ); - } - log.info( "Ontologies warmed up, starting check..." ); Map vos = ontologyService.findObsoleteTermUsage( ); From ad1f63b6af1978896f4ab6fbca63c754f85c2276 Mon Sep 17 00:00:00 2001 From: Paul Pavlidis Date: Tue, 18 Apr 2023 15:03:34 -0700 Subject: [PATCH 004/370] try again --- .../gemma/core/apps/FindObsoleteTermsCli.java | 33 ++++++++++++++----- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/gemma-cli/src/main/java/ubic/gemma/core/apps/FindObsoleteTermsCli.java b/gemma-cli/src/main/java/ubic/gemma/core/apps/FindObsoleteTermsCli.java index 07d1ddcdca..7edda97d66 100644 --- a/gemma-cli/src/main/java/ubic/gemma/core/apps/FindObsoleteTermsCli.java +++ b/gemma-cli/src/main/java/ubic/gemma/core/apps/FindObsoleteTermsCli.java @@ -10,8 +10,13 @@ import ubic.gemma.core.util.AbstractCLIContextCLI; import ubic.gemma.model.genome.gene.phenotype.valueObject.CharacteristicValueObject; +import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; public class FindObsoleteTermsCli extends AbstractCLIContextCLI { @@ -37,7 +42,7 @@ public String getShortDesc() { @Override protected void processOptions( CommandLine commandLine ) { - // no extra options. + // no extra options. } @Override @@ -52,27 +57,39 @@ protected void buildOptions( Options options ) { @Override protected void doWork() throws Exception { + + log.info( "Warming up ontologies ..." ); + ExecutorService executorService = Executors.newFixedThreadPool( 5 ); + List> futures = new ArrayList<>(); + for ( ubic.basecode.ontology.providers.OntologyService ontology : ontologies ) { - ontology.startInitializationThread( true, false ); + Future future = executorService.submit( () -> { + ontology.startInitializationThread( true, false ); + return null; + } ); + futures.add( future ); } - log.info( "Waiting for ontologies to warm up ..." ); - for ( ubic.basecode.ontology.providers.OntologyService ontology : ontologies ) { - ontology.waitForInitializationThread(); + for ( Future future : futures ) { + try { + future.get(); + } catch ( InterruptedException | ExecutionException e ) { + e.printStackTrace(); + } } + executorService.shutdown(); log.info( "Ontologies warmed up, starting check..." ); - Map vos = ontologyService.findObsoleteTermUsage( ); + Map vos = ontologyService.findObsoleteTermUsage(); AbstractCLI.log.info( "Obsolete term check finished, printing ..." ); - System.out.println( "Value\tValueUri\tCount" ); for ( CharacteristicValueObject vo : vos.values() ) { System.out.println( vo.getValue() + "\t" + vo.getValueUri() + "\t" + vo.getNumTimesUsed() ); } - AbstractCLI.log.info( "========= done ========" ); + } } From a3945a4d42a73a2d7f18026c8b10da8515d256f4 Mon Sep 17 00:00:00 2001 From: Paul Pavlidis Date: Tue, 18 Apr 2023 16:58:08 -0700 Subject: [PATCH 005/370] bug --- .../java/ubic/gemma/core/ontology/OntologyServiceImpl.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/gemma-core/src/main/java/ubic/gemma/core/ontology/OntologyServiceImpl.java b/gemma-core/src/main/java/ubic/gemma/core/ontology/OntologyServiceImpl.java index a1f5fa479e..737f22b943 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/ontology/OntologyServiceImpl.java +++ b/gemma-core/src/main/java/ubic/gemma/core/ontology/OntologyServiceImpl.java @@ -646,7 +646,6 @@ public Map findObsoleteTermUsage() { int start = 0; int step = 5000; - int obsoleteCnt = 0; int prevObsoleteCnt = 0; int checked = 0; while ( true ) { @@ -681,12 +680,12 @@ public Map findObsoleteTermUsage() { } } - if ( obsoleteCnt > prevObsoleteCnt ) { + if ( vos.size() > prevObsoleteCnt ) { OntologyServiceImpl.log.info( "Found " + vos.size() + " obsolete terms so far, tested " + checked + " characteristics" ); OntologyServiceImpl.log.info( "Last obsolete term seen: " + lastObsolete.getValue() + " -" + lastObsolete.getValueUri() ); } - prevObsoleteCnt = obsoleteCnt; + prevObsoleteCnt = vos.size(); } OntologyServiceImpl.log.info( "Done, obsolete terms found: " + vos.size() ); From 0ae04d1b2692bca6edceca30be427755f10cbf97 Mon Sep 17 00:00:00 2001 From: Paul Pavlidis Date: Tue, 18 Apr 2023 16:59:58 -0700 Subject: [PATCH 006/370] another bug --- .../main/java/ubic/gemma/core/apps/FindObsoleteTermsCli.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gemma-cli/src/main/java/ubic/gemma/core/apps/FindObsoleteTermsCli.java b/gemma-cli/src/main/java/ubic/gemma/core/apps/FindObsoleteTermsCli.java index 7edda97d66..810d91d229 100644 --- a/gemma-cli/src/main/java/ubic/gemma/core/apps/FindObsoleteTermsCli.java +++ b/gemma-cli/src/main/java/ubic/gemma/core/apps/FindObsoleteTermsCli.java @@ -65,7 +65,7 @@ protected void doWork() throws Exception { for ( ubic.basecode.ontology.providers.OntologyService ontology : ontologies ) { Future future = executorService.submit( () -> { - ontology.startInitializationThread( true, false ); + ontology.initialize( true, false ); return null; } ); futures.add( future ); From 0d865daf6f65e039f29d528fc87b8580abaa468a Mon Sep 17 00:00:00 2001 From: Paul Pavlidis Date: Tue, 18 Apr 2023 17:09:39 -0700 Subject: [PATCH 007/370] tidying --- .../gemma/core/apps/FindObsoleteTermsCli.java | 8 ++--- .../core/ontology/OntologyServiceImpl.java | 29 ++++++++----------- 2 files changed, 15 insertions(+), 22 deletions(-) diff --git a/gemma-cli/src/main/java/ubic/gemma/core/apps/FindObsoleteTermsCli.java b/gemma-cli/src/main/java/ubic/gemma/core/apps/FindObsoleteTermsCli.java index 810d91d229..9afeb052fe 100644 --- a/gemma-cli/src/main/java/ubic/gemma/core/apps/FindObsoleteTermsCli.java +++ b/gemma-cli/src/main/java/ubic/gemma/core/apps/FindObsoleteTermsCli.java @@ -61,24 +61,22 @@ protected void doWork() throws Exception { log.info( "Warming up ontologies ..." ); ExecutorService executorService = Executors.newFixedThreadPool( 5 ); - List> futures = new ArrayList<>(); + List> futures = new ArrayList<>(); for ( ubic.basecode.ontology.providers.OntologyService ontology : ontologies ) { - Future future = executorService.submit( () -> { + Future future = executorService.submit( () -> { ontology.initialize( true, false ); - return null; } ); futures.add( future ); } - for ( Future future : futures ) { + for ( Future future : futures ) { try { future.get(); } catch ( InterruptedException | ExecutionException e ) { e.printStackTrace(); } } - executorService.shutdown(); log.info( "Ontologies warmed up, starting check..." ); diff --git a/gemma-core/src/main/java/ubic/gemma/core/ontology/OntologyServiceImpl.java b/gemma-core/src/main/java/ubic/gemma/core/ontology/OntologyServiceImpl.java index 737f22b943..d77969e1aa 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/ontology/OntologyServiceImpl.java +++ b/gemma-core/src/main/java/ubic/gemma/core/ontology/OntologyServiceImpl.java @@ -648,6 +648,8 @@ public Map findObsoleteTermUsage() { int prevObsoleteCnt = 0; int checked = 0; + CharacteristicValueObject lastObsolete = null; + while ( true ) { Collection chars = characteristicService.browse( start, step ); @@ -660,29 +662,29 @@ public Map findObsoleteTermUsage() { OntologyServiceImpl.log.debug( "Found " + chars.size() + " characteristics in the current ID range, checking for obsoletes." ); - CharacteristicValueObject lastObsolete = null; for ( Characteristic ch : chars ) { - if ( StringUtils.isBlank( ch.getValueUri() ) ) { + String valueUri = ch.getValueUri(); + if ( StringUtils.isBlank( valueUri ) ) { continue; } checked++; - if ( this.isObsolete( ch.getValueUri() ) ) { - String key = this.foundValueKey( ch ); - if ( !vos.containsKey( key ) ) { - vos.put( key, new CharacteristicValueObject( ch ) ); + if ( this.isObsolete( valueUri ) ) { + + if ( !vos.containsKey( valueUri ) ) { + vos.put( valueUri, new CharacteristicValueObject( ch ) ); } - vos.get( key ).incrementOccurrenceCount(); + vos.get( valueUri ).incrementOccurrenceCount(); if ( log.isDebugEnabled() ) - OntologyServiceImpl.log.debug( "Found obsolete term: " + ch.getValue() + " - " + ch.getValueUri() ); - lastObsolete = vos.get( key ); + OntologyServiceImpl.log.debug( "Found obsolete term: " + ch.getValue() + " - " + valueUri ); + lastObsolete = vos.get( valueUri ); } } if ( vos.size() > prevObsoleteCnt ) { OntologyServiceImpl.log.info( "Found " + vos.size() + " obsolete terms so far, tested " + checked + " characteristics" ); - OntologyServiceImpl.log.info( "Last obsolete term seen: " + lastObsolete.getValue() + " -" + lastObsolete.getValueUri() ); + OntologyServiceImpl.log.info( "Last obsolete term seen: " + lastObsolete.getValue() + " - " + lastObsolete.getValueUri() ); } prevObsoleteCnt = vos.size(); @@ -827,13 +829,6 @@ private Collection findCharacteristicsFromOntology( S }, ontologyServicesToUse ); } - private String foundValueKey( Characteristic c ) { - if ( StringUtils.isNotBlank( c.getValueUri() ) ) { - return c.getValueUri().toLowerCase(); - } - return c.getValue().toLowerCase(); - } - private String foundValueKey( CharacteristicValueObject c ) { if ( c.getValueUri() != null && StringUtils.isNotBlank( c.getValueUri() ) ) { return c.getValueUri().toLowerCase(); From a48fc06181822204288fbf1f7f83b5ae113ca943 Mon Sep 17 00:00:00 2001 From: Paul Pavlidis Date: Wed, 19 Apr 2023 09:57:50 -0700 Subject: [PATCH 008/370] make easier to debug --- .../gemma/core/apps/FindObsoleteTermsCli.java | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/gemma-cli/src/main/java/ubic/gemma/core/apps/FindObsoleteTermsCli.java b/gemma-cli/src/main/java/ubic/gemma/core/apps/FindObsoleteTermsCli.java index 9afeb052fe..0e902f36fd 100644 --- a/gemma-cli/src/main/java/ubic/gemma/core/apps/FindObsoleteTermsCli.java +++ b/gemma-cli/src/main/java/ubic/gemma/core/apps/FindObsoleteTermsCli.java @@ -13,10 +13,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; +import java.util.concurrent.*; public class FindObsoleteTermsCli extends AbstractCLIContextCLI { @@ -61,18 +58,20 @@ protected void doWork() throws Exception { log.info( "Warming up ontologies ..." ); ExecutorService executorService = Executors.newFixedThreadPool( 5 ); - List> futures = new ArrayList<>(); + List> futures = new ArrayList<>(); for ( ubic.basecode.ontology.providers.OntologyService ontology : ontologies ) { - Future future = executorService.submit( () -> { + Future future = executorService.submit( () -> { ontology.initialize( true, false ); + return ontology; } ); futures.add( future ); } - for ( Future future : futures ) { + for ( Future future : futures ) { try { - future.get(); + ubic.basecode.ontology.providers.OntologyService os = future.get(); + log.info( " === Ontology warmed up: " + os ); } catch ( InterruptedException | ExecutionException e ) { e.printStackTrace(); } From d22b8d21cee7723e1a0f32e63e08a1c3c30224c7 Mon Sep 17 00:00:00 2001 From: Paul Pavlidis Date: Wed, 19 Apr 2023 10:14:30 -0700 Subject: [PATCH 009/370] cleanup --- .../java/ubic/gemma/core/apps/FindObsoleteTermsCli.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/gemma-cli/src/main/java/ubic/gemma/core/apps/FindObsoleteTermsCli.java b/gemma-cli/src/main/java/ubic/gemma/core/apps/FindObsoleteTermsCli.java index 0e902f36fd..d709e625b8 100644 --- a/gemma-cli/src/main/java/ubic/gemma/core/apps/FindObsoleteTermsCli.java +++ b/gemma-cli/src/main/java/ubic/gemma/core/apps/FindObsoleteTermsCli.java @@ -23,10 +23,6 @@ public class FindObsoleteTermsCli extends AbstractCLIContextCLI { @Autowired private List ontologies; - private int start = 1; - private int step = 100000; - private int stop = -1; - @Override public GemmaCLI.CommandGroup getCommandGroup() { return GemmaCLI.CommandGroup.METADATA; @@ -34,7 +30,7 @@ public GemmaCLI.CommandGroup getCommandGroup() { @Override public String getShortDesc() { - return "Check for characteristics using obsolete terms as values (excluding GO)"; + return "Check for characteristics using obsolete terms as values (excluding GO), prints to sdout"; } @Override From 9d03d1f5bcf46916e48efa0e2acb8bf48cf8798e Mon Sep 17 00:00:00 2001 From: Paul Pavlidis Date: Wed, 19 Apr 2023 15:38:46 -0700 Subject: [PATCH 010/370] also report missing terms --- .../gemma/core/ontology/OntologyService.java | 12 ++++++------ .../gemma/core/ontology/OntologyServiceImpl.java | 16 ++++------------ 2 files changed, 10 insertions(+), 18 deletions(-) diff --git a/gemma-core/src/main/java/ubic/gemma/core/ontology/OntologyService.java b/gemma-core/src/main/java/ubic/gemma/core/ontology/OntologyService.java index 647d4ac9ef..4a6f5fdb78 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/ontology/OntologyService.java +++ b/gemma-core/src/main/java/ubic/gemma/core/ontology/OntologyService.java @@ -48,7 +48,12 @@ public interface OntologyService { void addExpressionExperimentStatement( Characteristic vc, ExpressionExperiment ee ); /** + *

* Locates usages of obsolete terms in Characteristics, ignoring Gene Ontology annotations. Requires the ontologies are loaded into memory. + *

+ *

+ * Will also find terms that are no longer in an ontology we use. + *

* * @return map of value URI to a representative characteristic using the term. The latter will contain a count * of how many ocurrences there were. @@ -207,10 +212,5 @@ Collection findExperimentsCharacteristicTags( String Collection termsToCharacteristics( Collection terms ); - /** - * - * @return true if any ontologies are still inthe process of being initialized. False if no ontologies are being initialized. - * This means they either were initialized or failed to initialize, or weren't configured to be initialized. - */ - boolean isInitializing(); + } \ No newline at end of file diff --git a/gemma-core/src/main/java/ubic/gemma/core/ontology/OntologyServiceImpl.java b/gemma-core/src/main/java/ubic/gemma/core/ontology/OntologyServiceImpl.java index d77969e1aa..c5736c0304 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/ontology/OntologyServiceImpl.java +++ b/gemma-core/src/main/java/ubic/gemma/core/ontology/OntologyServiceImpl.java @@ -634,11 +634,6 @@ public Collection termsToCharacteristics( final Collection o.isInitializationThreadAlive() ); - } - @Override public Map findObsoleteTermUsage() { Map vos = new HashMap<>(); @@ -659,9 +654,6 @@ public Map findObsoleteTermUsage() { break; } - OntologyServiceImpl.log.debug( "Found " + chars.size() - + " characteristics in the current ID range, checking for obsoletes." ); - for ( Characteristic ch : chars ) { String valueUri = ch.getValueUri(); if ( StringUtils.isBlank( valueUri ) ) { @@ -670,27 +662,27 @@ public Map findObsoleteTermUsage() { checked++; - if ( this.isObsolete( valueUri ) ) { + if ( this.getTerm( valueUri ) == null || this.isObsolete( valueUri ) ) { if ( !vos.containsKey( valueUri ) ) { vos.put( valueUri, new CharacteristicValueObject( ch ) ); } vos.get( valueUri ).incrementOccurrenceCount(); if ( log.isDebugEnabled() ) - OntologyServiceImpl.log.debug( "Found obsolete term: " + ch.getValue() + " - " + valueUri ); + OntologyServiceImpl.log.debug( "Found obsolete or missing term: " + ch.getValue() + " - " + valueUri ); lastObsolete = vos.get( valueUri ); } } if ( vos.size() > prevObsoleteCnt ) { - OntologyServiceImpl.log.info( "Found " + vos.size() + " obsolete terms so far, tested " + checked + " characteristics" ); + OntologyServiceImpl.log.info( "Found " + vos.size() + " obsolete or missing terms so far, tested " + checked + " characteristics" ); OntologyServiceImpl.log.info( "Last obsolete term seen: " + lastObsolete.getValue() + " - " + lastObsolete.getValueUri() ); } prevObsoleteCnt = vos.size(); } - OntologyServiceImpl.log.info( "Done, obsolete terms found: " + vos.size() ); + OntologyServiceImpl.log.info( "Done, obsolete or missing terms found: " + vos.size() ); return vos; } From eed654845085951c428e8e0f0a52797d93c4840b Mon Sep 17 00:00:00 2001 From: Paul Pavlidis Date: Wed, 19 Apr 2023 16:21:58 -0700 Subject: [PATCH 011/370] handle special (non-ontology) case of NCBI genes. --- .../java/ubic/gemma/core/ontology/OntologyServiceImpl.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/gemma-core/src/main/java/ubic/gemma/core/ontology/OntologyServiceImpl.java b/gemma-core/src/main/java/ubic/gemma/core/ontology/OntologyServiceImpl.java index c5736c0304..26c6550f57 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/ontology/OntologyServiceImpl.java +++ b/gemma-core/src/main/java/ubic/gemma/core/ontology/OntologyServiceImpl.java @@ -664,6 +664,12 @@ public Map findObsoleteTermUsage() { if ( this.getTerm( valueUri ) == null || this.isObsolete( valueUri ) ) { + if (valueUri.contains( "http://purl.org/commons/record/ncbi_gene" )) { + // these are false positives, they aren't in an ontology. + continue; + } + + if ( !vos.containsKey( valueUri ) ) { vos.put( valueUri, new CharacteristicValueObject( ch ) ); } From 4594f0d2b695e7d0039d8b0f34d004d37a1cd0b7 Mon Sep 17 00:00:00 2001 From: Paul Pavlidis Date: Wed, 19 Apr 2023 16:27:58 -0700 Subject: [PATCH 012/370] another special case --- .../java/ubic/gemma/core/ontology/OntologyServiceImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gemma-core/src/main/java/ubic/gemma/core/ontology/OntologyServiceImpl.java b/gemma-core/src/main/java/ubic/gemma/core/ontology/OntologyServiceImpl.java index 26c6550f57..519d292353 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/ontology/OntologyServiceImpl.java +++ b/gemma-core/src/main/java/ubic/gemma/core/ontology/OntologyServiceImpl.java @@ -664,8 +664,8 @@ public Map findObsoleteTermUsage() { if ( this.getTerm( valueUri ) == null || this.isObsolete( valueUri ) ) { - if (valueUri.contains( "http://purl.org/commons/record/ncbi_gene" )) { - // these are false positives, they aren't in an ontology. + if ( valueUri.startsWith( "http://purl.org/commons/record/ncbi_gene" ) || valueUri.startsWith( "http://purl.obolibrary.org/obo/GO_" ) ) { + // these are false positives, they aren't in an ontology, and we aren't looking at GO Terms. continue; } From 487435a4c77a5ab10e57501834953b58eff24227 Mon Sep 17 00:00:00 2001 From: Paul Pavlidis Date: Tue, 20 Jun 2023 16:35:27 -0700 Subject: [PATCH 013/370] eliminate last few DO terms --- .../core/ontology/valueStringToOntologyTermMappings.txt | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/gemma-core/src/main/resources/ubic/gemma/core/ontology/valueStringToOntologyTermMappings.txt b/gemma-core/src/main/resources/ubic/gemma/core/ontology/valueStringToOntologyTermMappings.txt index fe5d4972a0..76e2f912f9 100644 --- a/gemma-core/src/main/resources/ubic/gemma/core/ontology/valueStringToOntologyTermMappings.txt +++ b/gemma-core/src/main/resources/ubic/gemma/core/ontology/valueStringToOntologyTermMappings.txt @@ -73,7 +73,7 @@ brain amygdala amygdala http://purl.obolibrary.org/obo/UBERON_0001876 organism p brain anterior cingulate cortex anterior cingulate cortex http://purl.obolibrary.org/obo/UBERON_0009835 organism part http://www.ebi.ac.uk/efo/EFO_0000635 Brain microglia microglial cell http://purl.obolibrary.org/obo/CL_0000129 cell type http://www.ebi.ac.uk/efo/EFO_0000324 brain striatum striatum http://purl.obolibrary.org/obo/UBERON_0002435 organism part http://www.ebi.ac.uk/efo/EFO_0000635 -brain tumor (ependymoma) brain ependymoma http://purl.obolibrary.org/obo/DOID_7497 disease http://www.ebi.ac.uk/efo/EFO_0000408 +brain tumor (ependymoma) ependymoma http://purl.obolibrary.org/obo/MONDO_0016698 disease http://www.ebi.ac.uk/efo/EFO_0000408 brain, cerebellum cerebellum http://purl.obolibrary.org/obo/UBERON_0002037 organism part http://www.ebi.ac.uk/efo/EFO_0000635 brain, frontal cortex frontal cortex http://purl.obolibrary.org/obo/UBERON_0001870 organism part http://www.ebi.ac.uk/efo/EFO_0000635 brainstem brainstem http://purl.obolibrary.org/obo/UBERON_0002298 organism part http://www.ebi.ac.uk/efo/EFO_0000635 @@ -205,13 +205,13 @@ Embryonic stem cells embyronic stem cell http://purl.obolibrary.org/obo/CL_00023 endometrium endometrium http://purl.obolibrary.org/obo/UBERON_0001295 organism part http://www.ebi.ac.uk/efo/EFO_0000635 endothelial cell endothelial cell http://purl.obolibrary.org/obo/CL_0000115 cell type http://www.ebi.ac.uk/efo/EFO_0000324 Entorhinal Cortex entorhinal cortex http://purl.obolibrary.org/obo/UBERON_0002728 organism part http://www.ebi.ac.uk/efo/EFO_0000635 -Ependymoma brain tumor brain ependymoma http://purl.obolibrary.org/obo/DOID_7497 disease http://www.ebi.ac.uk/efo/EFO_0000408 +Ependymoma brain tumor ependymoma http://purl.obolibrary.org/obo/MONDO_0016698 disease http://www.ebi.ac.uk/efo/EFO_0000408 epidermis skin epidermis http://purl.obolibrary.org/obo/UBERON_0001003 organism part http://www.ebi.ac.uk/efo/EFO_0000635 epididymal adipose tissue epididymal fat pad http://purl.obolibrary.org/obo/UBERON_0010412 organism part http://www.ebi.ac.uk/efo/EFO_0000635 Epididymal white adipose tissue epididymal fat pad http://purl.obolibrary.org/obo/UBERON_0010412 organism part http://www.ebi.ac.uk/efo/EFO_0000635 epilepsy epilepsy http://purl.obolibrary.org/obo/MONDO_0005027 disease http://www.ebi.ac.uk/efo/EFO_0000408 epithelial cells from the inferior nasal turbinate inferior nasal concha http://purl.obolibrary.org/obo/UBERON_0005922 organism part http://www.ebi.ac.uk/efo/EFO_0000635 -epithelian ovarian tumor ovary epithelial cancer http://purl.obolibrary.org/obo/DOID_2152 disease http://www.ebi.ac.uk/efo/EFO_0000408 +epithelian ovarian tumor malignant epithelial tumor of ovary http://purl.obolibrary.org/obo/MONDO_0018364 disease http://www.ebi.ac.uk/efo/EFO_0000408 ER+ estrogen-receptor positive breast cancer http://purl.obolibrary.org/obo/MONDO_0006512 disease http://www.ebi.ac.uk/efo/EFO_0000408 ES cell embryonic stem cell http://purl.obolibrary.org/obo/CL_0002322 cell type http://www.ebi.ac.uk/efo/EFO_0000324 ES cell culture embryonic stem cell http://purl.obolibrary.org/obo/CL_0002322 cell type http://www.ebi.ac.uk/efo/EFO_0000324 @@ -562,7 +562,7 @@ schizophrenia (SCZ) schizophrenia http://purl.obolibrary.org/obo/MONDO_0005090 d sciatic nerve sciatic nerve http://purl.obolibrary.org/obo/UBERON_0001322 organism part http://www.ebi.ac.uk/efo/EFO_0000635 SD Sprague Dawley http://gemma.msl.ubc.ca/ont/TGEMO_00099 strain http://www.ebi.ac.uk/efo/EFO_0005135 sentinel lymph node lymph node http://purl.obolibrary.org/obo/UBERON_0000029 organism part http://www.ebi.ac.uk/efo/EFO_0000635 -Sepsis bacterial sepsis http://purl.obolibrary.org/obo/DOID_0040085 disease http://www.ebi.ac.uk/efo/EFO_0000408 +Sepsis sepsis http://purl.obolibrary.org/obo/MP_0005044 disease http://www.ebi.ac.uk/efo/EFO_0000408 Septum septum http://purl.obolibrary.org/obo/UBERON_0003037 organism part http://www.ebi.ac.uk/efo/EFO_0000635 SH-SY5Y SH-SY5Y cell http://purl.obolibrary.org/obo/CLO_0009015 cell line http://purl.obolibrary.org/obo/CLO_0000031 Sjögrens syndrome Sjogren syndrome http://purl.obolibrary.org/obo/MONDO_0010030 disease http://www.ebi.ac.uk/efo/EFO_0000408 @@ -625,6 +625,7 @@ total RNA total RNA http://www.ebi.ac.uk/efo/EFO_0004964 collection of material trabecular bone trabecular bone http://purl.obolibrary.org/obo/UBERON_0002483 organism part http://www.ebi.ac.uk/efo/EFO_0000635 trachea trachea http://purl.obolibrary.org/obo/UBERON_0003126 organism part http://www.ebi.ac.uk/efo/EFO_0000635 triple-negative breast cancer triple-negative breast carcinoma http://purl.obolibrary.org/obo/MONDO_0005494 disease http://www.ebi.ac.uk/efo/EFO_0000408 +triple-negative breast cancer triple-negative breast carcinoma http://purl.obolibrary.org/obo/MONDO_0005494 disease http://www.ebi.ac.uk/efo/EFO_0000408 Tuberculosis tuberculosis http://purl.obolibrary.org/obo/MONDO_0018076 disease http://www.ebi.ac.uk/efo/EFO_0000408 U2OS U-2 OS cell http://purl.obolibrary.org/obo/CLO_0009454 cell line http://purl.obolibrary.org/obo/CLO_0000031 U937 U937(CD59+) cell http://purl.obolibrary.org/obo/CLO_0009466 cell line http://purl.obolibrary.org/obo/CLO_0000031 From 7d54bae4be027d2545cc6027d581fc436a7fb718 Mon Sep 17 00:00:00 2001 From: Paul Pavlidis Date: Tue, 20 Jun 2023 16:41:01 -0700 Subject: [PATCH 014/370] use EFO instead of TGEMO terms for strains where possible --- .../valueStringToOntologyTermMappings.txt | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/gemma-core/src/main/resources/ubic/gemma/core/ontology/valueStringToOntologyTermMappings.txt b/gemma-core/src/main/resources/ubic/gemma/core/ontology/valueStringToOntologyTermMappings.txt index 76e2f912f9..372cad76e9 100644 --- a/gemma-core/src/main/resources/ubic/gemma/core/ontology/valueStringToOntologyTermMappings.txt +++ b/gemma-core/src/main/resources/ubic/gemma/core/ontology/valueStringToOntologyTermMappings.txt @@ -6,7 +6,7 @@ 126S6/SvEvTac 129S6/SvEvTac http://gemma.msl.ubc.ca/ont/TGEMO_00048 strain http://www.ebi.ac.uk/efo/EFO_0005135 129 129 http://gemma.msl.ubc.ca/ont/TGEMO_00034 strain http://www.ebi.ac.uk/efo/EFO_0005135 129/Sv 129S/SvEv http://gemma.msl.ubc.ca/ont/TGEMO_00036 strain http://www.ebi.ac.uk/efo/EFO_0005135 -129SV 129/Sv http://gemma.msl.ubc.ca/ont/TGEMO_00035 strain http://www.ebi.ac.uk/efo/EFO_0005135 +129SV 129/Sv http://www.ebi.ac.uk/efo/EFO_0000599 strain http://www.ebi.ac.uk/efo/EFO_0005135 17beta-estradiol 17beta-estradiol http://purl.obolibrary.org/obo/CHEBI_16469 treatment http://www.ebi.ac.uk/efo/EFO_0000727 A/J A/J http://gemma.msl.ubc.ca/ont/TGEMO_00093 strain http://www.ebi.ac.uk/efo/EFO_0005135 A549 A549 cell http://purl.obolibrary.org/obo/CLO_0001601 cell line http://purl.obolibrary.org/obo/CLO_0000031 @@ -92,7 +92,7 @@ C Caucasian http://purl.obolibrary.org/obo/HANCESTRO_0005 population http://purl C2C12 RCB0987 cell http://purl.obolibrary.org/obo/CLO_0050871 cell line http://purl.obolibrary.org/obo/CLO_0000031 C3H Sprague Dawley http://gemma.msl.ubc.ca/ont/TGEMO_00053 strain http://www.ebi.ac.uk/efo/EFO_0005135 C3H/HeJ C3H/HeJ http://gemma.msl.ubc.ca/ont/TGEMO_00055 strain http://www.ebi.ac.uk/efo/EFO_0005135 -C56BL/6J C57BL/6J http://gemma.msl.ubc.ca/ont/TGEMO_00037 strain http://www.ebi.ac.uk/efo/EFO_0005135 +C56BL/6J C57BL/6J http://www.ebi.ac.uk/efo/EFO_0000606 strain http://www.ebi.ac.uk/efo/EFO_0005135 C57/B6 C57BL/6 http://gemma.msl.ubc.ca/ont/TGEMO_00016 strain http://www.ebi.ac.uk/efo/EFO_0005135 C57/BL6 C57BL/6 http://gemma.msl.ubc.ca/ont/TGEMO_00016 strain http://www.ebi.ac.uk/efo/EFO_0005138 C57B6 C57BL/6 http://gemma.msl.ubc.ca/ont/TGEMO_00016 strain http://www.ebi.ac.uk/efo/EFO_0005135 @@ -560,7 +560,7 @@ Sarin sarin http://purl.obolibrary.org/obo/CHEBI_75701 treatment http://www.ebi. schizophrenia schizophrenia http://purl.obolibrary.org/obo/MONDO_0005090 disease http://www.ebi.ac.uk/efo/EFO_0000408 schizophrenia (SCZ) schizophrenia http://purl.obolibrary.org/obo/MONDO_0005090 disease http://www.ebi.ac.uk/efo/EFO_0000408 sciatic nerve sciatic nerve http://purl.obolibrary.org/obo/UBERON_0001322 organism part http://www.ebi.ac.uk/efo/EFO_0000635 -SD Sprague Dawley http://gemma.msl.ubc.ca/ont/TGEMO_00099 strain http://www.ebi.ac.uk/efo/EFO_0005135 +SD Sprague Dawley http://www.ebi.ac.uk/efo/EFO_0001352 strain http://www.ebi.ac.uk/efo/EFO_0005135 sentinel lymph node lymph node http://purl.obolibrary.org/obo/UBERON_0000029 organism part http://www.ebi.ac.uk/efo/EFO_0000635 Sepsis sepsis http://purl.obolibrary.org/obo/MP_0005044 disease http://www.ebi.ac.uk/efo/EFO_0000408 Septum septum http://purl.obolibrary.org/obo/UBERON_0003037 organism part http://www.ebi.ac.uk/efo/EFO_0000635 @@ -583,11 +583,11 @@ spinal cord spinal cord http://purl.obolibrary.org/obo/UBERON_0002240 organism p spiny projection neurons pyramidal neuron http://purl.obolibrary.org/obo/CL_0000598 cell type http://www.ebi.ac.uk/efo/EFO_0000324 spleen spleen http://purl.obolibrary.org/obo/UBERON_0002106 organism part http://www.ebi.ac.uk/efo/EFO_0000635 Spleen Cell Cultures spleen http://purl.obolibrary.org/obo/UBERON_0002106 organism part http://www.ebi.ac.uk/efo/EFO_0000635 -sprague dawley Sprague Dawley http://gemma.msl.ubc.ca/ont/TGEMO_00099 strain http://www.ebi.ac.uk/efo/EFO_0005138 -Sprague-Dawley Sprague Dawley http://gemma.msl.ubc.ca/ont/TGEMO_00099 strain http://www.ebi.ac.uk/efo/EFO_0005135 -Sprague-Dawley (SD) Sprague Dawley http://gemma.msl.ubc.ca/ont/TGEMO_00099 strain http://www.ebi.ac.uk/efo/EFO_0005135 -Sprague-Dawley rat Sprague Dawley http://gemma.msl.ubc.ca/ont/TGEMO_00099 strain http://www.ebi.ac.uk/efo/EFO_0005135 -Sprague-Dawley rats Sprague Dawley http://gemma.msl.ubc.ca/ont/TGEMO_00099 strain http://www.ebi.ac.uk/efo/EFO_0005135 +sprague dawley Sprague Dawley http://www.ebi.ac.uk/efo/EFO_0001352 strain http://www.ebi.ac.uk/efo/EFO_0005138 +Sprague-Dawley Sprague Dawley http://www.ebi.ac.uk/efo/EFO_0001352 strain http://www.ebi.ac.uk/efo/EFO_0005135 +Sprague-Dawley (SD) Sprague Dawley http://www.ebi.ac.uk/efo/EFO_0001352 strain http://www.ebi.ac.uk/efo/EFO_0005135 +Sprague-Dawley rat Sprague Dawley http://www.ebi.ac.uk/efo/EFO_0001352 strain http://www.ebi.ac.uk/efo/EFO_0005135 +Sprague-Dawley rats Sprague Dawley http://www.ebi.ac.uk/efo/EFO_0001352 strain http://www.ebi.ac.uk/efo/EFO_0005135 sputum sputum http://purl.obolibrary.org/obo/UBERON_0007311 organism part http://www.ebi.ac.uk/efo/EFO_0000635 Squamous cell carcinoma squamous cell carcinoma http://purl.obolibrary.org/obo/MONDO_0005096 disease http://www.ebi.ac.uk/efo/EFO_0000408 sscortex secondary somatosensory cortex http://purl.obolibrary.org/obo/UBERON_0008934 organism part http://www.ebi.ac.uk/efo/EFO_0000635 @@ -674,7 +674,7 @@ Whole visceral adipose tissue visceral abdominal adipose tissue http://purl.obol whole-blood blood http://purl.obolibrary.org/obo/UBERON_0000178 organism part http://www.ebi.ac.uk/efo/EFO_0000635 wild type wild type genotype http://www.ebi.ac.uk/efo/EFO_0005168 genotype http://www.ebi.ac.uk/efo/EFO_0000513 wildtype wild type genotype http://www.ebi.ac.uk/efo/EFO_0005168 genotype http://www.ebi.ac.uk/efo/EFO_0000513 -wistar Wistar http://gemma.msl.ubc.ca/ont/TGEMO_00096 strain http://www.ebi.ac.uk/efo/EFO_0005139 -Wistar rat Wistar http://gemma.msl.ubc.ca/ont/TGEMO_00096 strain http://www.ebi.ac.uk/efo/EFO_0005135 +wistar Wistar http://www.ebi.ac.uk/efo/EFO_0001342 strain http://www.ebi.ac.uk/efo/EFO_0005139 +Wistar rat Wistar http://www.ebi.ac.uk/efo/EFO_0001342 strain http://www.ebi.ac.uk/efo/EFO_0005135 wt wild type genotype http://www.ebi.ac.uk/efo/EFO_0005168 genotype http://www.ebi.ac.uk/efo/EFO_0000513 zebularine zebularine http://purl.obolibrary.org/obo/CHEBI_46938 treatment http://www.ebi.ac.uk/efo/EFO_0000727 From 320487b2c12a58db7d16c0f3b06b7e5683e0878c Mon Sep 17 00:00:00 2001 From: Paul Pavlidis Date: Tue, 20 Jun 2023 16:48:37 -0700 Subject: [PATCH 015/370] fix for broken call --- gemma-web/src/main/java/ubic/gemma/web/util/AnchorTagUtil.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gemma-web/src/main/java/ubic/gemma/web/util/AnchorTagUtil.java b/gemma-web/src/main/java/ubic/gemma/web/util/AnchorTagUtil.java index 1efda9fa24..7705933e2c 100644 --- a/gemma-web/src/main/java/ubic/gemma/web/util/AnchorTagUtil.java +++ b/gemma-web/src/main/java/ubic/gemma/web/util/AnchorTagUtil.java @@ -24,7 +24,7 @@ public static String getArrayDesignLink( Long adId, String link, String hover, S } public static String getBioMaterialLink( Long bmId, String link, ServletContext servletContext ) { - return AnchorTagUtil.getBioMaterialLink( bmId, link, null ); + return AnchorTagUtil.getBioMaterialLink( bmId, link, null,null ); } public static String getBioMaterialLink( Long bmId, String link, String hover, ServletContext servletContext ) { From eb387802afb9c003f04991ad05792cd769bb4e20 Mon Sep 17 00:00:00 2001 From: Paul Pavlidis Date: Tue, 20 Jun 2023 16:57:07 -0700 Subject: [PATCH 016/370] last harmonization --- .../valueStringToOntologyTermMappings.txt | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/gemma-core/src/main/resources/ubic/gemma/core/ontology/valueStringToOntologyTermMappings.txt b/gemma-core/src/main/resources/ubic/gemma/core/ontology/valueStringToOntologyTermMappings.txt index 372cad76e9..1c6d9e2005 100644 --- a/gemma-core/src/main/resources/ubic/gemma/core/ontology/valueStringToOntologyTermMappings.txt +++ b/gemma-core/src/main/resources/ubic/gemma/core/ontology/valueStringToOntologyTermMappings.txt @@ -47,8 +47,8 @@ B cells B cell http://purl.obolibrary.org/obo/CL_0000236 cell type http://www.eb B-cell lymphoma B-cell neoplasm http://purl.obolibrary.org/obo/MONDO_0004095 disease http://www.ebi.ac.uk/efo/EFO_0000408 B6 C57BL/6 http://gemma.msl.ubc.ca/ont/TGEMO_00016 strain http://www.ebi.ac.uk/efo/EFO_0005135 B6C3F1 B6C3F1 http://www.ebi.ac.uk/efo/EFO_0001340 strain http://www.ebi.ac.uk/efo/EFO_0005135 -BALB BALB/c http://gemma.msl.ubc.ca/ont/TGEMO_00026 strain http://www.ebi.ac.uk/efo/EFO_0005135 -BALB/c BALB/c http://gemma.msl.ubc.ca/ont/TGEMO_00026 strain http://www.ebi.ac.uk/efo/EFO_0005138 +BALB BALB/c http://www.ebi.ac.uk/efo/EFO_0000602 strain http://www.ebi.ac.uk/efo/EFO_0005135 +BALB/c BALB/c http://www.ebi.ac.uk/efo/EFO_0000602 strain http://www.ebi.ac.uk/efo/EFO_0005138 BALB/cJ BALB/cJ http://gemma.msl.ubc.ca/ont/TGEMO_00028 strain http://www.ebi.ac.uk/efo/EFO_0005135 BEAS-2B BEAS-2B cell http://purl.obolibrary.org/obo/CLO_0001925 cell line http://purl.obolibrary.org/obo/CLO_0000031 Biceps biceps brachii http://purl.obolibrary.org/obo/UBERON_0001507 organism part http://www.ebi.ac.uk/efo/EFO_0000635 @@ -99,12 +99,12 @@ C57B6 C57BL/6 http://gemma.msl.ubc.ca/ont/TGEMO_00016 strain http://www.ebi.ac.u C57BL C57BL/6 http://gemma.msl.ubc.ca/ont/TGEMO_00016 strain http://www.ebi.ac.uk/efo/EFO_0005135 c57bl/6 C57BL/6 http://gemma.msl.ubc.ca/ont/TGEMO_00016 strain http://www.ebi.ac.uk/efo/EFO_0005138 C57BL/6 C57BL/6 http://gemma.msl.ubc.ca/ont/TGEMO_00016 strain http://www.ebi.ac.uk/efo/EFO_0005135 -C57BL/6J C57BL/6J http://gemma.msl.ubc.ca/ont/TGEMO_00037 strain http://www.ebi.ac.uk/efo/EFO_0005138 -C57BL/6J mouse C57BL/6J http://gemma.msl.ubc.ca/ont/TGEMO_00037 strain http://www.ebi.ac.uk/efo/EFO_0005135 +C57BL/6J C57BL/6J http://www.ebi.ac.uk/efo/EFO_0005138 strain http://www.ebi.ac.uk/efo/EFO_0005138 +C57BL/6J mouse C57BL/6J http://www.ebi.ac.uk/efo/EFO_0005138 strain http://www.ebi.ac.uk/efo/EFO_0005135 C57BL/6N C57BL/6 http://gemma.msl.ubc.ca/ont/TGEMO_00016 strain http://www.ebi.ac.uk/efo/EFO_0005135 C57BL6 C57BL/6 http://gemma.msl.ubc.ca/ont/TGEMO_00016 strain http://www.ebi.ac.uk/efo/EFO_0005135 -C57BL6/J C57BL/6J http://gemma.msl.ubc.ca/ont/TGEMO_00037 strain http://www.ebi.ac.uk/efo/EFO_0005135 -C57BL6J C57BL/6J http://gemma.msl.ubc.ca/ont/TGEMO_00037 strain http://www.ebi.ac.uk/efo/EFO_0005135 +C57BL6/J C57BL/6J http://www.ebi.ac.uk/efo/EFO_0005138 strain http://www.ebi.ac.uk/efo/EFO_0005135 +C57BL6J C57BL/6J http://www.ebi.ac.uk/efo/EFO_0005138 strain http://www.ebi.ac.uk/efo/EFO_0005135 CA1 hippocampal subregion CA1 field of hippocampus http://purl.obolibrary.org/obo/UBERON_0003881 organism part http://www.ebi.ac.uk/efo/EFO_0000635 CA1 hippocampus CA1 field of hippocampus http://purl.obolibrary.org/obo/UBERON_0003881 organism part http://www.ebi.ac.uk/efo/EFO_0000635 Caffeine caffeine http://purl.obolibrary.org/obo/CHEBI_27732 treatment http://www.ebi.ac.uk/efo/EFO_0000727 @@ -364,9 +364,9 @@ Major Depression major depressive disorder http://purl.obolibrary.org/obo/MONDO_ Major Depressive Disorder major depressive disorder http://purl.obolibrary.org/obo/MONDO_0002009 disease http://www.ebi.ac.uk/efo/EFO_0000408 major depressive disorder (MDD) major depressive disorder http://purl.obolibrary.org/obo/MONDO_0002009 disease http://www.ebi.ac.uk/efo/EFO_0000408 male male http://purl.obolibrary.org/obo/PATO_0000384 biological sex http://purl.obolibrary.org/obo/PATO_0000047 -Male C57BL/6J C57BL/6J http://gemma.msl.ubc.ca/ont/TGEMO_00037 strain http://www.ebi.ac.uk/efo/EFO_0005135 +Male C57BL/6J C57BL/6J http://www.ebi.ac.uk/efo/EFO_0005138 strain http://www.ebi.ac.uk/efo/EFO_0005135 Males male http://purl.obolibrary.org/obo/PATO_0000384 biological sex http://purl.obolibrary.org/obo/PATO_0000047 -malignant malignant http://www.ebi.ac.uk/efo/EFO_0006851 disease http://www.ebi.ac.uk/efo/EFO_0000408 +malignant malignant http://purl.obolibrary.org/obo/PATO_0002097 disease http://www.ebi.ac.uk/efo/EFO_0000408 Malignant ovarian ovarian cancer http://purl.obolibrary.org/obo/MONDO_0008170 disease http://www.ebi.ac.uk/efo/EFO_0000408 Mammary fat pad mammary fat pad http://purl.obolibrary.org/obo/UBERON_0012282 organism part http://www.ebi.ac.uk/efo/EFO_0000635 mammary gland mammary gland http://purl.obolibrary.org/obo/UBERON_0001911 organism part http://www.ebi.ac.uk/efo/EFO_0000635 From 11c6c9873e71a59146903f00eff0501a0b4316eb Mon Sep 17 00:00:00 2001 From: Paul Pavlidis Date: Wed, 21 Jun 2023 10:03:22 -0700 Subject: [PATCH 017/370] increase retry limit (probably futile) --- .../src/main/java/ubic/gemma/core/apps/GeoGrabberCli.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gemma-cli/src/main/java/ubic/gemma/core/apps/GeoGrabberCli.java b/gemma-cli/src/main/java/ubic/gemma/core/apps/GeoGrabberCli.java index aaa9999f92..2cad3e565f 100755 --- a/gemma-cli/src/main/java/ubic/gemma/core/apps/GeoGrabberCli.java +++ b/gemma-cli/src/main/java/ubic/gemma/core/apps/GeoGrabberCli.java @@ -48,7 +48,7 @@ public class GeoGrabberCli extends AbstractCLIContextCLI { private static final int NCBI_CHUNK_SIZE = 100; - private static final int MAX_RETRIES = 3; // on failures + private static final int MAX_RETRIES = 5; // on failures private static final int MAX_EMPTY_CHUNKS_IN_A_ROW = 20; // stop condition when we stop seeing useful records private Date dateLimit; private String gseLimit; @@ -253,7 +253,7 @@ protected void doWork() throws Exception { retries++; if ( retries <= MAX_RETRIES ) { log.warn( "Failure while fetching records, retrying " + e.getMessage() ); - Thread.sleep( 500 ); + Thread.sleep( 500 * retries ); continue; } throw new IOException( "Too many failures: " + e.getMessage() ); From 8d6bf312c39529a88ee815887d3e56d583182ff6 Mon Sep 17 00:00:00 2001 From: Paul Pavlidis Date: Wed, 21 Jun 2023 13:45:26 -0700 Subject: [PATCH 018/370] adding a few other terms we noted --- .../gemma/core/ontology/valueStringToOntologyTermMappings.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/gemma-core/src/main/resources/ubic/gemma/core/ontology/valueStringToOntologyTermMappings.txt b/gemma-core/src/main/resources/ubic/gemma/core/ontology/valueStringToOntologyTermMappings.txt index 1c6d9e2005..ea66ff7d77 100644 --- a/gemma-core/src/main/resources/ubic/gemma/core/ontology/valueStringToOntologyTermMappings.txt +++ b/gemma-core/src/main/resources/ubic/gemma/core/ontology/valueStringToOntologyTermMappings.txt @@ -145,6 +145,7 @@ Circulating mononuclear cells peripheral blood mononuclear cell http://purl.obol clear cell renal cell carcinoma clear cell renal carcinoma http://purl.obolibrary.org/obo/MONDO_0005005 disease http://www.ebi.ac.uk/efo/EFO_0000408 Clear-cell renal cell carcinoma clear cell renal carcinoma http://purl.obolibrary.org/obo/MONDO_0005005 disease http://www.ebi.ac.uk/efo/EFO_0000408 Clinical Breast Cancer Specimen breast cancer http://purl.obolibrary.org/obo/MONDO_0007254 disease http://www.ebi.ac.uk/efo/EFO_0000408 +colitis colitis http://purl.obolibrary.org/obo/MONDO_0005292 disease http://www.ebi.ac.uk/efo/EFO_0000408 Colon colon http://purl.obolibrary.org/obo/UBERON_0001155 organism part http://www.ebi.ac.uk/efo/EFO_0000635 colon adenocarcinoma cells colon adenocarcinoma http://purl.obolibrary.org/obo/MONDO_0002271 disease http://www.ebi.ac.uk/efo/EFO_0000408 colon cancer malignant colon neoplasm http://purl.obolibrary.org/obo/MONDO_0021063 disease http://www.ebi.ac.uk/efo/EFO_0000408 @@ -177,6 +178,7 @@ dermal fibroblasts fibroblast http://purl.obolibrary.org/obo/CL_0000057 cell typ dexamethasone dexamethasone http://purl.obolibrary.org/obo/CHEBI_41879 treatment http://www.ebi.ac.uk/efo/EFO_0000727 diabetic diabetes mellitus http://purl.obolibrary.org/obo/MONDO_0005015 disease http://www.ebi.ac.uk/efo/EFO_0000408 differentiated adipocytes fat cell http://purl.obolibrary.org/obo/CL_0000136 cell type http://www.ebi.ac.uk/efo/EFO_0000324 +diffuse intrinsic pontine glioma diffuse intrinsic pontine glioma http://purl.obolibrary.org/obo/MONDO_0006033 disease http://www.ebi.ac.uk/efo/EFO_0000408 Distal colon left colon http://purl.obolibrary.org/obo/UBERON_0008971 organism part http://www.ebi.ac.uk/efo/EFO_0000635 distal lung epithelium lung epithelium http://purl.obolibrary.org/obo/UBERON_0000115 organism part http://www.ebi.ac.uk/efo/EFO_0000635 DLBCL diffuse large B-cell lymphoma http://purl.obolibrary.org/obo/MONDO_0018905 disease http://www.ebi.ac.uk/efo/EFO_0000408 @@ -594,6 +596,7 @@ sscortex secondary somatosensory cortex http://purl.obolibrary.org/obo/UBERON_00 stem cells stem cell http://purl.obolibrary.org/obo/CL_0000034 cell type http://www.ebi.ac.uk/efo/EFO_0000324 stress environmental stress http://www.ebi.ac.uk/efo/EFO_0000470 environmental history http://www.ebi.ac.uk/efo/EFO_0004444 striatum striatum http://purl.obolibrary.org/obo/UBERON_0002435 organism part http://www.ebi.ac.uk/efo/EFO_0000635 +stroke stroke http://purl.obolibrary.org/obo/MONDO_0005098 disease http://www.ebi.ac.uk/efo/EFO_0000408 subcortical white matter white matter http://purl.obolibrary.org/obo/UBERON_0002316 organism part http://www.ebi.ac.uk/efo/EFO_0000635 subcutaneous adipose subcutaneous adipose http://purl.obolibrary.org/obo/UBERON_0002190 organism part http://www.ebi.ac.uk/efo/EFO_0000635 Subcutaneous Fat subcutaneous adipose tissue http://purl.obolibrary.org/obo/UBERON_0002190 organism part http://www.ebi.ac.uk/efo/EFO_0000635 From 81213799d9d4ac53339466e2075886bdec8b755d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 7 Jul 2023 22:01:05 +0000 Subject: [PATCH 019/370] Bump h2 from 2.1.214 to 2.2.220 Bumps [h2](https://github.com/h2database/h2database) from 2.1.214 to 2.2.220. - [Release notes](https://github.com/h2database/h2database/releases) - [Commits](https://github.com/h2database/h2database/compare/version-2.1.214...version-2.2.220) --- updated-dependencies: - dependency-name: com.h2database:h2 dependency-type: direct:development ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index dc9f923fa1..6637933964 100644 --- a/pom.xml +++ b/pom.xml @@ -454,7 +454,7 @@ com.h2database h2 - 2.1.214 + 2.2.220 test From fdc9c97c76dc0ee65cb8cf2a95aa0e9bedb2ff92 Mon Sep 17 00:00:00 2001 From: Paul Pavlidis Date: Fri, 14 Jul 2023 14:52:18 -0700 Subject: [PATCH 020/370] recognize another sequence column name --- .../ubic/gemma/core/loader/expression/geo/util/GeoConstants.java | 1 + 1 file changed, 1 insertion(+) diff --git a/gemma-core/src/main/java/ubic/gemma/core/loader/expression/geo/util/GeoConstants.java b/gemma-core/src/main/java/ubic/gemma/core/loader/expression/geo/util/GeoConstants.java index 0cc5306e98..becb5a8e79 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/loader/expression/geo/util/GeoConstants.java +++ b/gemma-core/src/main/java/ubic/gemma/core/loader/expression/geo/util/GeoConstants.java @@ -63,6 +63,7 @@ public class GeoConstants { sequenceColumnNames = new HashSet<>(); GeoConstants.sequenceColumnNames.add( "SEQUENCE" ); // agilent. + GeoConstants.sequenceColumnNames.add( "PROBE_SEQUENCE" ); // e.g. GPL7182 //LMD 24/07/09 Bug 1647 probeOrganismColumnNames = new HashSet<>(); From 2ce41d9de42367268e97de7a7a975298e4a1c489 Mon Sep 17 00:00:00 2001 From: Paul Pavlidis Date: Fri, 4 Aug 2023 14:24:29 -0700 Subject: [PATCH 021/370] help with debugging --- .../matrix/BaseExpressionDataMatrix.java | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/BaseExpressionDataMatrix.java b/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/BaseExpressionDataMatrix.java index 053080ec94..d189998f2e 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/BaseExpressionDataMatrix.java +++ b/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/BaseExpressionDataMatrix.java @@ -258,7 +258,7 @@ void addToRowMaps( Integer row, CompositeSequence designElement ) { * For example, in the following diagram "-" indicates a biomaterial, while "*" indicates a bioassay. Each row of * "*" indicates samples run on a different microarray design (a different bio assay material). In the examples we * assume there is just a single biomaterial dimension. - * + * * 
      * ---------------
      * *****              -- only a few samples run on this platform
@@ -268,7 +268,7 @@ void addToRowMaps( Integer row, CompositeSequence designElement ) {
      * 

      * A simpler case:
      * 

-     * 
+     *
      * 
      * ---------------
      * ***************
@@ -278,7 +278,7 @@ void addToRowMaps( Integer row, CompositeSequence designElement ) {
      * 

      * A more typical and easy case (one microarray design used):
      * 

-     * 
+     *
      * 
      * ----------------
      * ****************
@@ -286,7 +286,7 @@ void addToRowMaps( Integer row, CompositeSequence designElement ) {
      * 

      * If every sample was run on two different array designs:
      * 

-     * 
+     *
      * 
      * ----------------
      * ****************
@@ -294,7 +294,7 @@ void addToRowMaps( Integer row, CompositeSequence designElement ) {
      * 

      * 

      * Every sample was run on a different array design:
-     * 
+     *
      * 
      * -----------------------
      * ******
@@ -384,7 +384,9 @@ void selectVectors( Collection vectors ) {
                 this.getQuantitationTypes().add( vectorQuantitationType );
             } else {
                 if ( quantitationType != vectorQuantitationType ) {
-                    throw new IllegalArgumentException( "Cannot pass vectors from more than one quantitation type" );
+                    throw new IllegalArgumentException( "Cannot pass vectors from more than one quantitation type: " +
+                            vectorQuantitationType + " vs "
+                            + quantitationType );
                 }
 
             }

From 42f5a45f438fc7c1ebbf5dfa0e29e2f6d4e53870 Mon Sep 17 00:00:00 2001
From: Paul Pavlidis 
Date: Fri, 4 Aug 2023 14:25:04 -0700
Subject: [PATCH 022/370] add the category to factorvalue toString

---
 .../ubic/gemma/model/expression/experiment/FactorValue.java    | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/gemma-core/src/main/java/ubic/gemma/model/expression/experiment/FactorValue.java b/gemma-core/src/main/java/ubic/gemma/model/expression/experiment/FactorValue.java
index 358ba4eb19..9c5ef911a9 100644
--- a/gemma-core/src/main/java/ubic/gemma/model/expression/experiment/FactorValue.java
+++ b/gemma-core/src/main/java/ubic/gemma/model/expression/experiment/FactorValue.java
@@ -109,9 +109,10 @@ public String toString() {
         buf.append( "FactorValue " ).append( this.getId() ).append( ": " );
 
         if ( this.getExperimentalFactor() != null )
-            buf.append( this.getExperimentalFactor().getName() ).append( ":" );
+            buf.append( this.getExperimentalFactor().getName() ).append( ": " );
         if ( this.getCharacteristics().size() > 0 ) {
             for ( Characteristic c : this.getCharacteristics() ) {
+                buf.append(c.getCategory() + " - ");
                 buf.append( c.getValue() );
                 if ( this.getCharacteristics().size() > 1 )
                     buf.append( " | " );

From 7b86b2ba98eb2ee00e1b5a0dd5c9e13e297199ee Mon Sep 17 00:00:00 2001
From: Paul Pavlidis 
Date: Tue, 8 Aug 2023 16:12:07 -0700
Subject: [PATCH 023/370] for work on
 https://github.com/PavlidisLab/Gemma/issues/705

directly support two complete triplet-like statements within the GUI

Backend support for this is not in the model - TBD
---
 .../expression/geo/GeoConverterImpl.java      |    1 +
 .../gemma/core/ontology/OntologyService.java  |    6 +
 .../core/ontology/OntologyServiceImpl.java    |   33 +
 .../experiment/FactorValueValueObject.java    |    6 +
 .../gemma/core/ontology/Relation.terms.txt    |   20 +
 .../experiment/AnnotationController.java      |    4 +
 .../ExperimentalDesignControllerImpl.java     |    4 +
 .../src/main/webapp/WEB-INF/gemma-servlet.xml |    1 +
 .../api/annotation/FactorValueEditor.js       | 1498 +++++++++--------
 .../scripts/api/annotation/RelationCombo.js   |   72 +
 .../main/webapp/scripts/api/dwrServices.js    |    3 +
 11 files changed, 976 insertions(+), 672 deletions(-)
 create mode 100644 gemma-core/src/main/resources/ubic/gemma/core/ontology/Relation.terms.txt
 create mode 100644 gemma-web/src/main/webapp/scripts/api/annotation/RelationCombo.js

diff --git a/gemma-core/src/main/java/ubic/gemma/core/loader/expression/geo/GeoConverterImpl.java b/gemma-core/src/main/java/ubic/gemma/core/loader/expression/geo/GeoConverterImpl.java
index ad9b69ecec..bd775f170f 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/loader/expression/geo/GeoConverterImpl.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/loader/expression/geo/GeoConverterImpl.java
@@ -910,6 +910,7 @@ private void initializeTerm2OntologyMappings() {
                     continue;
                 }
 
+                // NOTE: extensions via modifiers is not to be supported here, as GEO only has key-value pairs.
                 CharacteristicBasicValueObject c = new CharacteristicBasicValueObject( null, value, valueUri, category, categoryUri );
                 term2OntologyMappings.get( category ).put( inputValue, c );
             }
diff --git a/gemma-core/src/main/java/ubic/gemma/core/ontology/OntologyService.java b/gemma-core/src/main/java/ubic/gemma/core/ontology/OntologyService.java
index ed9f66bb24..32c2a33db2 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/ontology/OntologyService.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/ontology/OntologyService.java
@@ -101,6 +101,12 @@ Collection findExperimentsCharacteristicTags( String
      */
     Collection getCategoryTerms();
 
+    /**
+     *
+     * @return terms allowed for the predicate (relationship) in a Characteristic
+     */
+    Collection getRelationTerms();
+
     /**
      * Obtain the parents of a collection of terms.
      * @see OntologyTerm#getParents(boolean, boolean)
diff --git a/gemma-core/src/main/java/ubic/gemma/core/ontology/OntologyServiceImpl.java b/gemma-core/src/main/java/ubic/gemma/core/ontology/OntologyServiceImpl.java
index 49afe9608d..a0bb602781 100644
--- a/gemma-core/src/main/java/ubic/gemma/core/ontology/OntologyServiceImpl.java
+++ b/gemma-core/src/main/java/ubic/gemma/core/ontology/OntologyServiceImpl.java
@@ -117,6 +117,8 @@ public class OntologyServiceImpl implements OntologyService, InitializingBean {
 
     private Set categoryTerms = null;
 
+    private Set relationTerms = null;
+
     @Override
     public void afterPropertiesSet() throws Exception {
         List enabledOntologyServices = ontologyServiceFactories.stream()
@@ -140,6 +142,7 @@ public void afterPropertiesSet() throws Exception {
         // remove GeneOntologyService, it was originally not included in the list before bean injection was used
         ontologyServices.remove( geneOntologyService );
         initializeCategoryTerms();
+        initializeRelationTerms();
     }
 
     private void countOccurrences( Map results ) {
@@ -419,6 +422,17 @@ public Collection getCategoryTerms() {
                 .collect( Collectors.toSet() );
     }
 
+
+    @Override
+    public Collection getRelationTerms() {
+        // FIXME: it's not quite like categoryTerms so this map operation is probably not needed at all, the relations don't come from any particular ontology
+        return relationTerms.stream()
+                .map( term -> {
+                    return term;
+                } )
+                .collect( Collectors.toSet() );
+    }
+
     @Override
     public OntologyResource getResource( String uri ) {
         return findFirst( ontology -> ontology.getResource( uri ) );
@@ -822,6 +836,25 @@ private void initializeCategoryTerms() throws IOException {
         }
     }
 
+
+    private void initializeRelationTerms() throws IOException {
+        Set relationTerms= new HashSet<>();
+        Resource resource = new ClassPathResource( "/ubic/gemma/core/ontology/Relation.terms.txt" );
+        try ( BufferedReader reader = new BufferedReader( new InputStreamReader( resource.getInputStream() ) ) ) {
+            String line;
+            while ( ( line = reader.readLine() ) != null ) {
+                if ( line.startsWith( "#" ) || StringUtils.isEmpty( line ) )
+                    continue;
+                String[] f = StringUtils.split( line, '\t' );
+                if ( f.length < 2 ) {
+                    continue;
+                }
+                relationTerms.add( new OntologyTermSimple( f[0], f[1] ) );
+            }
+            this.relationTerms = Collections.unmodifiableSet( relationTerms );
+        }
+    }
+
     /**
      * given a collection of characteristics add them to the correct List
      */
diff --git a/gemma-core/src/main/java/ubic/gemma/model/expression/experiment/FactorValueValueObject.java b/gemma-core/src/main/java/ubic/gemma/model/expression/experiment/FactorValueValueObject.java
index d47665c9f2..25ad9d64b6 100644
--- a/gemma-core/src/main/java/ubic/gemma/model/expression/experiment/FactorValueValueObject.java
+++ b/gemma-core/src/main/java/ubic/gemma/model/expression/experiment/FactorValueValueObject.java
@@ -42,6 +42,12 @@ public class FactorValueValueObject extends IdentifiableValueObject
     private String factorValue;
     private String value;
     private String valueUri;
+
+    /*
+     * TODO: support objects and predicates for extension of the characteristics
+     */
+
+
     /**
      * It could be the id of the measurement if there is no characteristic.
      */
diff --git a/gemma-core/src/main/resources/ubic/gemma/core/ontology/Relation.terms.txt b/gemma-core/src/main/resources/ubic/gemma/core/ontology/Relation.terms.txt
new file mode 100644
index 0000000000..826ea0da32
--- /dev/null
+++ b/gemma-core/src/main/resources/ubic/gemma/core/ontology/Relation.terms.txt
@@ -0,0 +1,20 @@
+# Terms usable for relations among concepts.
+http://purl.obolibrary.org/obo/RO_0000087	has role
+http://purl.obolibrary.org/obo/RO_0001000	derives from
+http://purl.obolibrary.org/obo/RO_0000053	has characteristic
+http://purl.obolibrary.org/obo/ENVO_01003004	 derives from part of
+http://purl.obolibrary.org/obo/RO_0002260	has biological role
+http://purl.obolibrary.org/obo/RO_0000086	has quality
+http://purl.obolibrary.org/obo/RO_0002573	has modifier
+http://purl.obolibrary.org/obo/RO_0016002	has disease
+http://purl.obolibrary.org/obo/RO_0002200	has phenotype
+http://purl.obolibrary.org/obo/RO_0001025	located in
+# missing:
+# has_delivered or delivered_by
+# has_timepoint
+# from_cell_line (derived from?)
+# has_genotype
+# has_dose
+# has_stage (developmental stage)
+# has_allele
+# fusion_gene
diff --git a/gemma-web/src/main/java/ubic/gemma/web/controller/expression/experiment/AnnotationController.java b/gemma-web/src/main/java/ubic/gemma/web/controller/expression/experiment/AnnotationController.java
index 8a66ffaff4..71f0eb4000 100644
--- a/gemma-web/src/main/java/ubic/gemma/web/controller/expression/experiment/AnnotationController.java
+++ b/gemma-web/src/main/java/ubic/gemma/web/controller/expression/experiment/AnnotationController.java
@@ -79,6 +79,10 @@ public Collection getCategoryTerms() {
         return ontologyService.getCategoryTerms();
     }
 
+    public Collection getRelationTerms() {
+        return ontologyService.getRelationTerms();
+    }
+
     public void createBiomaterialTag( Characteristic vc, Long id ) {
         BioMaterial bm = bioMaterialService.load( id );
         if ( bm == null ) {
diff --git a/gemma-web/src/main/java/ubic/gemma/web/controller/expression/experiment/ExperimentalDesignControllerImpl.java b/gemma-web/src/main/java/ubic/gemma/web/controller/expression/experiment/ExperimentalDesignControllerImpl.java
index 0dc019df65..4a2402d4cd 100644
--- a/gemma-web/src/main/java/ubic/gemma/web/controller/expression/experiment/ExperimentalDesignControllerImpl.java
+++ b/gemma-web/src/main/java/ubic/gemma/web/controller/expression/experiment/ExperimentalDesignControllerImpl.java
@@ -656,6 +656,10 @@ public void updateExperimentalFactors( ExperimentalFactorValueObject[] efvos ) {
     @Override
     public void updateFactorValueCharacteristics( FactorValueValueObject[] fvvos ) {
 
+        /*
+         * TODO: support Characteristic extensions (predicate-object)
+         */
+
         if ( fvvos == null || fvvos.length == 0 )
             return;
 
diff --git a/gemma-web/src/main/webapp/WEB-INF/gemma-servlet.xml b/gemma-web/src/main/webapp/WEB-INF/gemma-servlet.xml
index 4ce09c059d..b6ad4a6f63 100644
--- a/gemma-web/src/main/webapp/WEB-INF/gemma-servlet.xml
+++ b/gemma-web/src/main/webapp/WEB-INF/gemma-servlet.xml
@@ -472,6 +472,7 @@
         
             
             
+            
             
             
             
diff --git a/gemma-web/src/main/webapp/scripts/api/annotation/FactorValueEditor.js b/gemma-web/src/main/webapp/scripts/api/annotation/FactorValueEditor.js
index 9c67f88baf..49f5ca805c 100755
--- a/gemma-web/src/main/webapp/scripts/api/annotation/FactorValueEditor.js
+++ b/gemma-web/src/main/webapp/scripts/api/annotation/FactorValueEditor.js
@@ -1,695 +1,849 @@
-Ext.namespace('Gemma');
+Ext.namespace( 'Gemma' );
 
 /**
  * Forms the bottom panel of the experimental design editor.
  *
  * @author Luke, Paul
  */
-Gemma.FactorValueRecord = Ext.data.Record.create([{
-    name: "charId"
+Gemma.FactorValueRecord = Ext.data.Record.create( [ {
+   name : "charId"
 }, {
-    name: "id"
+   name : "id"
 }, {
-    name: "category",
-    type: "string"
+   name : "category",
+   type : "string"
 }, {
-    name: "categoryUri",
-    type: "string"
+   name : "categoryUri",
+   type : "string"
 }, {
-    name: "value",
-    type: "string"
+   name : "value",
+   type : "string"
 }, {
-    name: "measurement",
-    type: "bool"
+   name : "valueUri",
+   type : "string"
 }, {
-    name: "valueUri",
-    type: "string"
+   name : "measurement",
+   type : "bool"
 }, {
-    name: "factorValue",
-    type: "string"
-}]);
-
-Gemma.FactorValueGrid = Ext.extend(Gemma.GemmaGridPanel, {
-
-    loadMask: true,
-
-    viewConfig: {
-        forceFit: false
-    },
-
-    taxonId: null,
-
-    disabledClass: '.x-factor-grid-disabled',
-
-    record: Gemma.FactorValueRecord,
-
-    /**
-     * @memberOf Gemma.FactorValueGrid
-     */
-    categoryStyler: function (value, metadata, record, row, col, ds) {
-        return Gemma.GemmaGridPanel.formatTermWithStyle(value, record.data.categoryUri);
-    },
-
-    valueStyler: function (value, metadata, record, row, col, ds) {
-        if (value) {
-            return Gemma.GemmaGridPanel.formatTermWithStyle(value, record.data.valueUri);
-        } else {
-            return ''
-                + ((this.editable) ? 'Double-click to define the value' : '(no value)') + '';
-        }
-    },
-
-    createNew: function () {
-        var ef = this.experimentalFactor;
-        var oldmsg = this.loadMask.msg;
-        this.loadMask.msg = "Creating new factor value";
-        this.loadMask.show();
-
-        var callback = function () {
-            this.loadMask.hide();
-            this.loadMask.msg = oldmsg;
-            this.factorValueCreated(ef);
-            this.getTopToolbar().characteristicToolbar.setExperimentalFactor(ef.id);
-        }.createDelegate(this);
-
-        var errorHandler = function (er) {
-            this.loadMask.hide();
-            this.loadMask.msg = oldmsg;
-            Ext.Msg.alert("Error", er);
-        }.createDelegate(this);
-
-        ExperimentalDesignController.createFactorValue(this.experimentalFactor, {
-            callback: callback,
-            errorHandler: errorHandler
-        });
-    },
-
-    initComponent: function () {
-
-        this.columns = [{
-            header: "FactorValue",
-            dataIndex: "id"
-        }, {
-            header: "Category",
-            dataIndex: "category",
-            renderer: this.categoryStyler,
-            width: 120
-        }, {
-            header: "Value",
-            dataIndex: "value",
-            renderer: this.valueStyler
-        }, {
-            header: "Summary",
-            dataIndex: "factorValue",
-            hidden: true
-        }];
-
-        this.experimentalDesign = {
-            id: this.edId,
-            classDelegatingFor: "ExperimentalDesign"
-        };
-        this.experimentalFactor = {
-            id: this.efId,
-            classDelegatingFor: "ExperimentalFactor"
-        };
-
-        this.categoryCombo = new Gemma.CategoryCombo({
-            lazyRender: true,
-            termKey: "factorvalue"
-        });
-        var categoryEditor = new Ext.grid.GridEditor(this.categoryCombo);
-        // when user selects a category from the combo.
-        this.categoryCombo.on("select", function (combo, record, index) {
-            categoryEditor.completeEdit();
-        });
-
-        this.valueCombo = new Gemma.CharacteristicCombo({
-            lazyRender: true,
-            taxonId: this.taxonId
-        });
-        var valueEditor = new Ext.grid.GridEditor(this.valueCombo);
-        // when user selects an item from the search results.
-        this.valueCombo.on("select", function (combo, record, index) {
-            valueEditor.completeEdit();
-        });
-
-        this.store = new Ext.data.GroupingStore({
-            proxy: new Ext.data.DWRProxy(ExperimentalDesignController.getFactorValuesWithCharacteristics),
-            reader: new Ext.data.ListRangeReader({
-                id: "charId"
-            }, this.record),
-            groupField: "id"
-        });
-
-        /*
-         * The checkboxes defined here require that this.form be set: a 
 element wrapping the div that this
-         * goes in. Clumsy but it works.
-         */
-        var groupTextTpl = this.editable ? '   ' : '';
-        groupTextTpl = groupTextTpl + '{[ values.rs[0].data.factorValue ]}';
-
-        this.view = new Ext.grid.GroupingView({
-            enableGroupingMenu: false,
-            enableNoGroups: false,
-            groupTextTpl: groupTextTpl,
-            hideGroupedColumn: true,
-            showGroupName: true,
-            startCollapsed: true
-        });
-
-        this.tbar = new Gemma.FactorValueToolbar({
-            editable: this.editable,
-            experimentalDesign: this.experimentalDesign
-        });
-
-        this.tbar.on("save", function(e){
-            this.valueCombo.clearCharacteristic(); // might not be necessary but can't hurt
-        }, this);
-
-        var FACTOR_VALUE_COLUMN = 0; // The id.
-        var CATEGORY_COLUMN = 1;
-        var VALUE_COLUMN = 2;
-        this.autoExpandColumn = VALUE_COLUMN;
-
-        // ///////////////////////////////////////
-        Gemma.FactorValueGrid.superclass.initComponent.call(this);
-
-        this.addEvents('factorvaluecreate', 'factorvaluechange', 'factorvaluedelete');
-
-        if (this.editable) {
-
-            this.getColumnModel().setEditor(CATEGORY_COLUMN, categoryEditor);
-            this.getColumnModel().setEditor(VALUE_COLUMN, valueEditor);
-
-            this.on("afteredit", function (e) {
-                var col = this.getColumnModel().getColumnId(e.column);
-                if (col == CATEGORY_COLUMN) {
-                    var term = this.categoryCombo.getTerm.call(this.categoryCombo);
-                    e.record.set("category", term.term);
-                    e.record.set("categoryUri", term.uri);
-                } else if (col == VALUE_COLUMN) {
-                    var c = this.valueCombo.getCharacteristic.call(this.valueCombo);
-                    e.record.set("value", c.value);
-                    e.record.set("valueUri", c.valueUri);
-                    this.valueCombo.clearCharacteristic.call(this.valueCombo); // avoid carryover
-                }
-                this.getView().refresh();
-            });
-
-            this.getSelectionModel().on("selectionchange", function (model) {
-                var selected = model.getSelections();
-                this.revertButton.disable();
-                for (var i = 0; i < selected.length; ++i) {
-                    if (selected[i].dirty) {
-                        this.revertButton.enable();
-                        break;
-                    }
-                }
-                if (selected.length > 0) {
-                    this.characteristicToolbar.deleteButton.enable();
-                } else {
-                    this.characteristicToolbar.deleteButton.disable();
-                }
-            }, this.getTopToolbar());
-
-            this.on('groupclick', function (grid, groupField, groupValue, e) {
-                var el = Ext.get(grid.getView().getGroupId(groupValue));
-                var cb = el.dom.getElementsByTagName('input')[0];
-                if (cb.checked) {
-                    this.deleteFactorValueButton.enable();
-                } else {
-                    this.deleteFactorValueButton.disable();
-                }
-            }, this.getTopToolbar());
-
-            this.on("afteredit", function (model) {
-                this.saveButton.enable();
-                this.revertButton.enable();
-            }, this.getTopToolbar());
-
-            /*
-             * Create a new factorvalue
-             */
-            this.getTopToolbar().on(
-                "create",
-                function () {
-                    /*
-                     * Avoid accidents...
-                     */
-                    if (this.store.getModifiedRecords().length > 0) {
-                        Ext.Msg
-                            .confirm(
-                                'Unsaved changes!',
-                                'You should save your changes before creating new values. Are you sure you want to erase them?',
-                                function (but) {
-                                    if (but == 'yes') {
-                                        this.store.rejectChanges();
-
-                                        this.createNew();
-                                    }
-                                }.createDelegate(this));
-                    } else {
-                        this.createNew();
-                    }
-
-                }.createDelegate(this));
-
-            /*
-             * delete a factor value
-             */
-            this.getTopToolbar().on(
-                "delete",
-                function () {
-                    var selectedIds = this.getSelectedFactorValues();
-
-                    if (selectedIds && selectedIds.length > 0) {
-
-                        Ext.Msg.confirm(
-                            Gemma.HelpText.WidgetDefaults.ExperimentalFactorToolbar.deleteFactorWarningTitle,
-                            Gemma.HelpText.WidgetDefaults.ExperimentalFactorToolbar.deleteFactorWarningText, function (but) {
-                                if (but == 'yes') {
-                                    Ext.getCmp('factor-value-delete-button').disable();
-                                    var ef = this.experimentalFactor;
-                                    this.getEl().mask();
-                                    var callback = function () {
-                                        this.factorValuesDeleted(selectedIds);
-                                    }.createDelegate(this);
-                                    ExperimentalDesignController.deleteFactorValues(ef, selectedIds, callback, this);
-                                }
-                            }, this);
-                    } else {
-                        Ext.Msg.alert("Nothing selected", "You have not checked any factor values for deletion");
-                    }
-                }.createDelegate(this), this);
-
-            /*
-             * Commit changes to factor values (added characteristics or updated characteristics)
-             */
-            this.getTopToolbar().on("save", function () {
-                var edited = this.getEditedRecords();
-                var callback = function () {
-                    this.factorValuesChanged(edited);
-                }.createDelegate(this);
-                ExperimentalDesignController.updateFactorValueCharacteristics(edited, callback);
-            }, this);
-
-            this.getTopToolbar().on("undo", this.revertSelected.createDelegate(this), this);
-
-            this.getTopToolbar().on("refresh", function () {
-                this.getStore().reload();
-            }, this);
-
-        }
-
-        this.getTopToolbar().on("toggleExpand", function () {
-            this.getView().toggleAllGroups(true);
-        }.createDelegate(this), this);
-
-        this.getTopToolbar().on("toggleCollapse", function () {
-            this.getView().toggleAllGroups(false);
-        }.createDelegate(this), this);
-
-        if (this.experimentalFactor.id) {
-            this.store.load({
-                params: [this.experimentalFactor]
-            });
-        }
-
-    }, // init component
-
-    onRender: function (c, p) {
-        Gemma.FactorValueGrid.superclass.onRender.call(this, c, p);
-        // have to do this here, because the toolbar isn't created until
-        // rendering.
-        var ct = this.getTopToolbar().characteristicToolbar;
-        if (ct) {
-            ct.on("create", function (f, c) {
-                var callback = function () {
-                    // console.log("reload");
-                    ct.factorValueCombo.store.reload();
-                    this.factorValuesChanged.call(this, []);
-                }.createDelegate(this);
-                ExperimentalDesignController.createFactorValueCharacteristic(f, c, callback);
-            }.createDelegate(this), this);
-
-            ct.on("delete", function () {
-                // console.log("deleting factorvalue(s)");
-                var selected = this.getSelectedRecords();
-                this.store.reload();
-                var callback = function () {
-                    this.factorValuesChanged.call(this, selected);
-                }.createDelegate(this);
-                ExperimentalDesignController.deleteFactorValueCharacteristics(selected, callback);
-            }.createDelegate(this), this);
-        }
-    },
-
-    factorValueCreated: function (ef) {
-        this.refresh();
-        var fvs = [];
-        var ct = this.getTopToolbar().characteristicToolbar;
-        ct.factorValueCombo.store.reload();
-        this.fireEvent('factorvaluecreate', this, fvs);
-    },
-
-    factorValuesChanged: function (fvs) {
-        this.refresh();
-        this.store.rejectChanges();
-        var ct = this.getTopToolbar().characteristicToolbar;
-        ct.factorValueCombo.store.reload();
-        this.fireEvent('factorvaluechange', this, fvs);
-    },
-
-    factorValuesDeleted: function (fvIds) {
-        // don't reload store, caching issues can cause an error (bug 2553)
-        // remove deleted records from store instead
-        var ct = this.getTopToolbar().characteristicToolbar;
-        var i;
-        var matchIds = function (record, id) {
-            return (fvIds.indexOf(record.data.id + "") > -1);
-        };
-        for (i = 0; i < fvIds.length; i++) {
-            var fvId = fvIds[i];
-            // combo and grid are not using the same store
-            // store.findExact doesn't work
-            // will be >1 record per factor value if a factor value has >1 category
-            var indexToRemove;
-            while (ct.factorValueCombo.getStore().findBy(matchIds) > -1) {
-                indexToRemove = ct.factorValueCombo.getStore().findBy(matchIds);
-                ct.factorValueCombo.getStore().removeAt(indexToRemove);
+   name : "object",
+   type : "string"
+}, {
+   name : "objectUri",
+   type : "string"
+}, {
+   name : "predicate",
+   type : "string"
+}, {
+   name : "predicateUri",
+   type : "string"
+}, {
+   name : "object2",
+   type : "string"
+}, {
+   name : "objectUri2",
+   type : "string"
+}, {
+   name : "predicate2",
+   type : "string"
+}, {
+   name : "predicateUri2",
+   type : "string"
+}, {
+   name : "factorValue",
+   type : "string"
+} ] );
+
+Gemma.FactorValueGrid = Ext.extend( Gemma.GemmaGridPanel, {
+
+   loadMask : true,
+
+   viewConfig : {
+      forceFit : false
+   },
+
+   taxonId : null,
+
+   disabledClass : '.x-factor-grid-disabled',
+
+   record : Gemma.FactorValueRecord,
+
+   /**
+    * @memberOf Gemma.FactorValueGrid
+    */
+   categoryStyler : function( value, metadata, record, row, col, ds ) {
+      return Gemma.GemmaGridPanel.formatTermWithStyle( value, record.data.categoryUri );
+   },
+
+   valueStyler : function( value, metadata, record, row, col, ds ) {
+      if ( value ) {
+         return Gemma.GemmaGridPanel.formatTermWithStyle( value, record.data.valueUri );
+      } else {
+         return ''
+            + ((this.editable) ? 'Double-click to define the value' : '(no value)') + '';
+      }
+   },
+
+
+   predicateStyler : function( value, metadata, record, row, col, ds ) {
+      if ( value ) {
+         return Gemma.GemmaGridPanel.formatTermWithStyle( value, record.data.predicateUri );
+      } else {
+         return ''
+            + ((this.editable) ? 'Double-click to define the predicate' : '(no value)') + '';
+      }
+   },
+
+
+   objectStyler : function( value, metadata, record, row, col, ds ) {
+      if ( value ) {
+         return Gemma.GemmaGridPanel.formatTermWithStyle( value, record.data.objectUri );
+      } else {
+         return ''
+            + ((this.editable) ? 'Double-click to define the object term' : '(no value)') + '';
+      }
+   },
+
+
+   // FIXME repeat stylers seems unnecessary
+   predicateStyler2 : function( value, metadata, record, row, col, ds ) {
+      if ( value ) {
+         return Gemma.GemmaGridPanel.formatTermWithStyle( value, record.data.predicateUri2 );
+      } else {
+         return ''
+            + ((this.editable) ? 'Double-click to define the predicate' : '(no value)') + '';
+      }
+   },
+
+
+   objectStyler2 : function( value, metadata, record, row, col, ds ) {
+      if ( value ) {
+         return Gemma.GemmaGridPanel.formatTermWithStyle( value, record.data.objectUri2 );
+      } else {
+         return ''
+            + ((this.editable) ? 'Double-click to define the object term' : '(no value)') + '';
+      }
+   },
+
+   createNew : function() {
+      var ef = this.experimentalFactor;
+      var oldmsg = this.loadMask.msg;
+      this.loadMask.msg = "Creating new factor value";
+      this.loadMask.show();
+
+      var callback = function() {
+         this.loadMask.hide();
+         this.loadMask.msg = oldmsg;
+         this.factorValueCreated( ef );
+         this.getTopToolbar().characteristicToolbar.setExperimentalFactor( ef.id );
+      }.createDelegate( this );
+
+      var errorHandler = function( er ) {
+         this.loadMask.hide();
+         this.loadMask.msg = oldmsg;
+         Ext.Msg.alert( "Error", er );
+      }.createDelegate( this );
+
+      ExperimentalDesignController.createFactorValue( this.experimentalFactor, {
+         callback : callback,
+         errorHandler : errorHandler
+      } );
+   },
+
+   initComponent : function() {
+
+      this.columns = [ {
+         header : "FactorValue",
+         dataIndex : "id"
+      }, {
+         header : "Category",
+         dataIndex : "category",
+         renderer : this.categoryStyler,
+         width : 120
+      }, {
+         header : "Value",
+         dataIndex : "value",
+         renderer : this.valueStyler
+      }, {
+         header : "Predicate",
+         dataIndex : "predicate",
+         renderer : this.predicateStyler,
+         width : 100
+      }, {
+         header : "Object",
+         dataIndex : "object",
+         renderer : this.objectStyler,
+         width : 200
+      }, {
+         header : "Predicate2",
+         dataIndex : "predicate2",
+         renderer : this.predicateStyler2,
+         width : 100
+      }, {
+         header : "Object2",
+         dataIndex : "object2",
+         renderer : this.objectStyler2,
+         width : 200
+      }, {
+            header : "Summary",
+            dataIndex : "factorValue",
+            hidden : true
+         } ];
+
+      this.experimentalDesign = {
+         id : this.edId,
+         classDelegatingFor : "ExperimentalDesign"
+      };
+      this.experimentalFactor = {
+         id : this.efId,
+         classDelegatingFor : "ExperimentalFactor"
+      };
+
+      this.categoryCombo = new Gemma.CategoryCombo( {
+         lazyRender : true,
+         termKey : "factorvalue" // not sure this does anything
+      } );
+      var categoryEditor = new Ext.grid.GridEditor( this.categoryCombo );
+      // when user selects a category from the combo.
+      this.categoryCombo.on( "select", function( combo, record, index ) {
+         categoryEditor.completeEdit();
+      } );
+
+      this.valueCombo = new Gemma.CharacteristicCombo( {
+         lazyRender : true,
+         taxonId : this.taxonId
+      } );
+      var valueEditor = new Ext.grid.GridEditor( this.valueCombo );
+      // when user selects an item from the search results.
+      this.valueCombo.on( "select", function( combo, record, index ) {
+         valueEditor.completeEdit();
+      } );
+
+
+      this.predicateCombo = new Gemma.RelationCombo( {
+         lazyRender : true,
+         termKey : "predicate" // not sure this does anything
+      } );
+      var predicateEditor = new Ext.grid.GridEditor( this.predicateCombo );
+      // when user selects a predicate from the combo.
+      this.predicateCombo.on( "select", function( combo, record, index ) {
+         predicateEditor.completeEdit();
+      } );
+
+      this.objectCombo = new Gemma.CharacteristicCombo( {
+         lazyRender : true,
+         taxonId : this.taxonId
+      } );
+      var objectEditor = new Ext.grid.GridEditor( this.objectCombo );
+      this.objectCombo.on( "select", function( combo, record, index ) {
+         objectEditor.completeEdit();
+      } );
+
+
+      this.predicateCombo2 = new Gemma.RelationCombo( {
+         lazyRender : true,
+         termKey : "predicate2" // not sure this does anything
+      } );
+      var predicateEditor2 = new Ext.grid.GridEditor( this.predicateCombo2 );
+      // when user selects a predicate from the combo.
+      this.predicateCombo2.on( "select", function( combo, record, index ) {
+         predicateEditor2.completeEdit();
+      } );
+
+      this.objectCombo2 = new Gemma.CharacteristicCombo( {
+         lazyRender : true,
+         taxonId : this.taxonId
+      } );
+      var objectEditor2 = new Ext.grid.GridEditor( this.objectCombo2 );
+      this.objectCombo2.on( "select", function( combo, record, index ) {
+         objectEditor2.completeEdit();
+      } );
+
+      this.store = new Ext.data.GroupingStore( {
+         proxy : new Ext.data.DWRProxy( ExperimentalDesignController.getFactorValuesWithCharacteristics ),
+         reader : new Ext.data.ListRangeReader( {
+            id : "charId"
+         }, this.record ),
+         groupField : "id"
+      } );
+
+      /*
+       * The checkboxes defined here require that this.form be set: a  element wrapping the div that this
+       * goes in. Clumsy but it works.
+       */
+      var groupTextTpl = this.editable ? '   ' : '';
+      groupTextTpl = groupTextTpl + '{[ values.rs[0].data.factorValue ]}';
+
+      this.view = new Ext.grid.GroupingView( {
+         enableGroupingMenu : false,
+         enableNoGroups : false,
+         groupTextTpl : groupTextTpl,
+         hideGroupedColumn : true,
+         showGroupName : true,
+         startCollapsed : true
+      } );
+
+      this.tbar = new Gemma.FactorValueToolbar( {
+         editable : this.editable,
+         experimentalDesign : this.experimentalDesign
+      } );
+
+      this.tbar.on( "save", function( e ) {
+         this.valueCombo.clearCharacteristic(); // might not be necessary but can't hurt
+      }, this );
+
+      var FACTOR_VALUE_COLUMN = 0; // The id.
+      var CATEGORY_COLUMN = 1;
+      var VALUE_COLUMN = 2;
+      var PREDICATE_COLUMN = 3;
+      var OBJECT_COLUMN = 4;
+      var PREDICATE_COLUMN2 = 5;
+      var OBJECT_COLUMN2 = 6;
+      this.autoExpandColumn = VALUE_COLUMN;
+
+      // ///////////////////////////////////////
+      Gemma.FactorValueGrid.superclass.initComponent.call( this );
+
+      this.addEvents( 'factorvaluecreate', 'factorvaluechange', 'factorvaluedelete' );
+
+      if ( this.editable ) {
+
+         this.getColumnModel().setEditor( CATEGORY_COLUMN, categoryEditor );
+         this.getColumnModel().setEditor( VALUE_COLUMN, valueEditor );
+         this.getColumnModel().setEditor( PREDICATE_COLUMN, predicateEditor );
+         this.getColumnModel().setEditor( OBJECT_COLUMN, objectEditor );
+         this.getColumnModel().setEditor( PREDICATE_COLUMN2, predicateEditor2 );
+         this.getColumnModel().setEditor( OBJECT_COLUMN2, objectEditor2 );
+
+         this.on( "afteredit", function( e ) {
+            var col = this.getColumnModel().getColumnId( e.column );
+            if ( col == CATEGORY_COLUMN ) {
+               var term = this.categoryCombo.getTerm.call( this.categoryCombo );
+               e.record.set( "category", term.term );
+               e.record.set( "categoryUri", term.uri );
+            } else if ( col == VALUE_COLUMN ) {
+               var c = this.valueCombo.getCharacteristic.call( this.valueCombo );
+               e.record.set( "value", c.value );
+               e.record.set( "valueUri", c.valueUri );
+               this.valueCombo.clearCharacteristic.call( this.valueCombo ); // avoid carryover
+            } else if ( col == PREDICATE_COLUMN ) {
+               var c = this.predicateCombo.getTerm.call( this.predicateCombo );
+               e.record.set( "predicate", c.term );
+               e.record.set( "predicateUri", c.uri );
+            } else if ( col == OBJECT_COLUMN ) {
+               var c = this.objectCombo.getCharacteristic.call( this.objectCombo );
+               e.record.set( "object", c.value );
+               e.record.set( "objectUri", c.valueUri );
+               this.objectCombo.clearCharacteristic.call( this.objectCombo ); // avoid carryover
+            } else if ( col == PREDICATE_COLUMN2 ) {
+               var c = this.predicateCombo2.getTerm.call( this.predicateCombo2 );
+               e.record.set( "predicate2", c.term );
+               e.record.set( "predicateUri2", c.uri );
+            } else if ( col == OBJECT_COLUMN2 ) {
+               var c = this.objectCombo2.getCharacteristic.call( this.objectCombo2 );
+               e.record.set( "object2", c.value );
+               e.record.set( "objectUri2", c.valueUri );
+               this.objectCombo.clearCharacteristic.call( this.objectCombo2 ); // avoid carryover
             }
-
-            while (this.store.findBy(matchIds) > -1) {
-                indexToRemove = this.store.findBy(matchIds);
-                this.store.removeAt(indexToRemove);
+            this.getView().refresh();
+         } );
+
+         this.getSelectionModel().on( "selectionchange", function( model ) {
+            var selected = model.getSelections();
+            this.revertButton.disable();
+            for ( var i = 0; i < selected.length; ++i ) {
+               if ( selected[i].dirty ) {
+                  this.revertButton.enable();
+                  break;
+               }
             }
-
-        }
-        this.getEl().unmask();
-        this.fireEvent('factorvaluedelete', this, fvIds);
-    },
-
-    changeExperimentalFactor: function (efId) {
-        this.experimentalFactor.id = efId;
-        this.store.rejectChanges(); // reset.
-
-        if (efId === null) {
-            this.store.removeAll();
-        } else {
-            this.refresh([this.experimentalFactor]); // causes a load.
-        }
-        this.getTopToolbar().setExperimentalFactor(efId);
-
-    },
-
-    setExperimentalFactor: function (efId) {
-
-        if (this.store.getModifiedRecords().length > 0) {
-            Ext.Msg
-                .confirm(
-                    'Unsaved changes!',
-                    'You should save your changes before switching to another factor. Are you sure you want to lose your changes?',
-                    function (but) {
-                        if (but == 'yes') {
-                            this.changeExperimentalFactor(efId);
-                        }
-                    }.createDelegate(this));
-        } else {
-            this.changeExperimentalFactor(efId);
-        }
-    },
-
-    /**
-     * Processing of the checkboxes defined in the tmpl for the groups. Requires that this.form be set: a 
-     * element wrapping the div that this goes in. Clumsy but it works.
-     */
-    getSelectedFactorValues: function () {
-        if (this.form) {
-            var f = document.forms[this.form];
-            var checkboxes = f.selectedFactorValues;
-            if (!checkboxes.length) {
-                checkboxes = [checkboxes];
+            if ( selected.length > 0 ) {
+               this.characteristicToolbar.deleteButton.enable();
+            } else {
+               this.characteristicToolbar.deleteButton.disable();
             }
-            var values = [];
-            for (var i = 0; i < checkboxes.length; ++i) {
-                if (checkboxes[i].checked) {
-                    values.push(checkboxes[i].value);
-                }
+         }, this.getTopToolbar() );
+
+         this.on( 'groupclick', function( grid, groupField, groupValue, e ) {
+            var el = Ext.get( grid.getView().getGroupId( groupValue ) );
+            var cb = el.dom.getElementsByTagName( 'input' )[0];
+            if ( cb.checked ) {
+               this.deleteFactorValueButton.enable();
+            } else {
+               this.deleteFactorValueButton.disable();
             }
-            return values;
-        }
-    }
-
-});
-
-
-Gemma.FactorValueToolbar = Ext.extend(Ext.Toolbar, {
-
-    /**
-     * @memberOf Gemma.FactorValueToolbar
-     */
-    initComponent: function () {
-        Gemma.FactorValueToolbar.superclass.initComponent.call(this);
-        this.addEvents("create", "save", "delete", "undo", "refresh", "toggleExpand", "toggleCollapse");
-    },
-
-    onRender: function (c, p) {
-
-        Gemma.FactorValueToolbar.superclass.onRender.call(this, c, p);
-
-        this.createFactorValueButton = new Ext.Toolbar.Button({
-            text: "Create",
-            id: 'factor-value-create-button',
-            tooltip: "Create a new factor value for the current factor",
-            disabled: true,
-            handler: function () {
-                this.fireEvent("create");
-                this.deleteFactorValueButton.enable();
-            },
-            scope: this
-        });
-
-        this.deleteFactorValueButton = new Ext.Toolbar.Button({
-            text: "Delete",
-            id: 'factor-value-delete-button',
-            tooltip: "Delete checked factor values",
-            disabled: true,
-            handler: function () {
-                this.fireEvent("delete");
-            }.createDelegate(this)
-        });
-
-        this.saveButton = new Ext.Toolbar.Button({
-            text: "Save",
-            id: 'factor-value-save-button',
-            tooltip: "Commit changes to factor values",
-            disabled: true,
-            handler: function () {
-                this.saveButton.disable();
-                this.fireEvent("save");
-            }.createDelegate(this)
-        });
-
-        this.revertButton = new Ext.Toolbar.Button({
-            text: "Undo",
-            id: 'factor-value-undo-button',
-            tooltip: "Undo changes to selected factor values",
-            disabled: true,
-            handler: function () {
-                this.fireEvent("undo");
-            },
-            scope: this
-        });
-
-        this.refreshButton = new Ext.Toolbar.Button({
-            icon: ctxBasePath + '/images/icons/arrow_refresh_small.png',
-            tooltip: 'Refresh the factor values',
-            handler: function () {
-                this.fireEvent("refresh");
-            },
-            scope: this
-        });
-
-        if (this.editable) {
-            this.addButton(this.createFactorValueButton);
-            this.addSeparator();
-            this.addButton(this.deleteFactorValueButton);
-            this.addSpacer();
-            this.addButton(this.saveButton);
-            this.addSpacer();
-            this.addButton(this.revertButton);
-            this.addSpacer();
-            this.addButton(this.refreshButton);
-        }
-
-        this.collapseButton = new Ext.Toolbar.Button({
-            text: "Collapse all",
-            tooltip: "Hide all factor value details",
-            handler: function () {
-                this.fireEvent("toggleCollapse");
-            },
-            scope: this
-        });
-
-        this.addFill();
-        this.addButton(this.collapseButton);
-
-        this.expandButton = new Ext.Toolbar.Button({
-            text: "Expand all",
-            tooltip: "Show all factor value details",
-            handler: function () {
-                this.fireEvent("toggleExpand");
-            },
-            scope: this
-        });
-
-        this.addFill();
-        this.addButton(this.expandButton);
-
-        if (this.editable) {
-            this.characteristicToolbar = new Gemma.FactorValueCharacteristicToolbar({
-                id: 'fv-char-toolbar',
-                renderTo: this.ownerCt.tbar
-            });
-        }
-
-    },
-
-    setExperimentalFactor: function (efId) {
-        this.efId = efId;
-        if (efId != null) {
-            this.createFactorValueButton.enable();
-        }
-        if (this.characteristicToolbar) {
-            this.characteristicToolbar.setExperimentalFactor(efId);
-        }
-
-    }
-
-});
+         }, this.getTopToolbar() );
+
+         this.on( "afteredit", function( model ) {
+            this.saveButton.enable();
+            this.revertButton.enable();
+         }, this.getTopToolbar() );
+
+         /*
+          * Create a new factorvalue
+          */
+         this.getTopToolbar().on(
+            "create",
+            function() {
+               /*
+                * Avoid accidents...
+                */
+               if ( this.store.getModifiedRecords().length > 0 ) {
+                  Ext.Msg
+                     .confirm(
+                        'Unsaved changes!',
+                        'You should save your changes before creating new values. Are you sure you want to erase them?',
+                        function( but ) {
+                           if ( but == 'yes' ) {
+                              this.store.rejectChanges();
+
+                              this.createNew();
+                           }
+                        }.createDelegate( this ) );
+               } else {
+                  this.createNew();
+               }
+
+            }.createDelegate( this ) );
+
+         /*
+          * delete a factor value
+          */
+         this.getTopToolbar().on(
+            "delete",
+            function() {
+               var selectedIds = this.getSelectedFactorValues();
+
+               if ( selectedIds && selectedIds.length > 0 ) {
+
+                  Ext.Msg.confirm(
+                     Gemma.HelpText.WidgetDefaults.ExperimentalFactorToolbar.deleteFactorWarningTitle,
+                     Gemma.HelpText.WidgetDefaults.ExperimentalFactorToolbar.deleteFactorWarningText, function( but ) {
+                        if ( but == 'yes' ) {
+                           Ext.getCmp( 'factor-value-delete-button' ).disable();
+                           var ef = this.experimentalFactor;
+                           this.getEl().mask();
+                           var callback = function() {
+                              this.factorValuesDeleted( selectedIds );
+                           }.createDelegate( this );
+                           ExperimentalDesignController.deleteFactorValues( ef, selectedIds, callback, this );
+                        }
+                     }, this );
+               } else {
+                  Ext.Msg.alert( "Nothing selected", "You have not checked any factor values for deletion" );
+               }
+            }.createDelegate( this ), this );
+
+         /*
+          * Commit changes to factor values (added characteristics or updated characteristics)
+          */
+         this.getTopToolbar().on( "save", function() {
+            var edited = this.getEditedRecords();
+            var callback = function() {
+               this.factorValuesChanged( edited );
+            }.createDelegate( this );
+            ExperimentalDesignController.updateFactorValueCharacteristics( edited, callback );
+         }, this );
+
+         this.getTopToolbar().on( "undo", this.revertSelected.createDelegate( this ), this );
+
+         this.getTopToolbar().on( "refresh", function() {
+            this.getStore().reload();
+         }, this );
+
+      }
+
+      this.getTopToolbar().on( "toggleExpand", function() {
+         this.getView().toggleAllGroups( true );
+      }.createDelegate( this ), this );
+
+      this.getTopToolbar().on( "toggleCollapse", function() {
+         this.getView().toggleAllGroups( false );
+      }.createDelegate( this ), this );
+
+      if ( this.experimentalFactor.id ) {
+         this.store.load( {
+            params : [ this.experimentalFactor ]
+         } );
+      }
+
+   }, // init component
+
+   onRender : function( c, p ) {
+      Gemma.FactorValueGrid.superclass.onRender.call( this, c, p );
+      // have to do this here, because the toolbar isn't created until
+      // rendering.
+      var ct = this.getTopToolbar().characteristicToolbar;
+      if ( ct ) {
+         ct.on( "create", function( f, c ) {
+            var callback = function() {
+               // console.log("reload");
+               ct.factorValueCombo.store.reload();
+               this.factorValuesChanged.call( this, [] );
+            }.createDelegate( this );
+            ExperimentalDesignController.createFactorValueCharacteristic( f, c, callback );
+         }.createDelegate( this ), this );
+
+         ct.on( "delete", function() {
+            // console.log("deleting factorvalue(s)");
+            var selected = this.getSelectedRecords();
+            this.store.reload();
+            var callback = function() {
+               this.factorValuesChanged.call( this, selected );
+            }.createDelegate( this );
+            ExperimentalDesignController.deleteFactorValueCharacteristics( selected, callback );
+         }.createDelegate( this ), this );
+      }
+   },
+
+   factorValueCreated : function( ef ) {
+      this.refresh();
+      var fvs = [];
+      var ct = this.getTopToolbar().characteristicToolbar;
+      ct.factorValueCombo.store.reload();
+      this.fireEvent( 'factorvaluecreate', this, fvs );
+   },
+
+   factorValuesChanged : function( fvs ) {
+      this.refresh();
+      this.store.rejectChanges();
+      var ct = this.getTopToolbar().characteristicToolbar;
+      ct.factorValueCombo.store.reload();
+      this.fireEvent( 'factorvaluechange', this, fvs );
+   },
+
+   factorValuesDeleted : function( fvIds ) {
+      // don't reload store, caching issues can cause an error (bug 2553)
+      // remove deleted records from store instead
+      var ct = this.getTopToolbar().characteristicToolbar;
+      var i;
+      var matchIds = function( record, id ) {
+         return (fvIds.indexOf( record.data.id + "" ) > -1);
+      };
+      for ( i = 0; i < fvIds.length; i++ ) {
+         var fvId = fvIds[i];
+         // combo and grid are not using the same store
+         // store.findExact doesn't work
+         // will be >1 record per factor value if a factor value has >1 category
+         var indexToRemove;
+         while ( ct.factorValueCombo.getStore().findBy( matchIds ) > -1 ) {
+            indexToRemove = ct.factorValueCombo.getStore().findBy( matchIds );
+            ct.factorValueCombo.getStore().removeAt( indexToRemove );
+         }
+
+         while ( this.store.findBy( matchIds ) > -1 ) {
+            indexToRemove = this.store.findBy( matchIds );
+            this.store.removeAt( indexToRemove );
+         }
+
+      }
+      this.getEl().unmask();
+      this.fireEvent( 'factorvaluedelete', this, fvIds );
+   },
+
+   changeExperimentalFactor : function( efId ) {
+      this.experimentalFactor.id = efId;
+      this.store.rejectChanges(); // reset.
+
+      if ( efId === null ) {
+         this.store.removeAll();
+      } else {
+         this.refresh( [ this.experimentalFactor ] ); // causes a load.
+      }
+      this.getTopToolbar().setExperimentalFactor( efId );
+
+   },
+
+   setExperimentalFactor : function( efId ) {
+
+      if ( this.store.getModifiedRecords().length > 0 ) {
+         Ext.Msg
+            .confirm(
+               'Unsaved changes!',
+               'You should save your changes before switching to another factor. Are you sure you want to lose your changes?',
+               function( but ) {
+                  if ( but == 'yes' ) {
+                     this.changeExperimentalFactor( efId );
+                  }
+               }.createDelegate( this ) );
+      } else {
+         this.changeExperimentalFactor( efId );
+      }
+   },
+
+   /**
+    * Processing of the checkboxes defined in the tmpl for the groups. Requires that this.form be set: a 
+    * element wrapping the div that this goes in. Clumsy but it works.
+    */
+   getSelectedFactorValues : function() {
+      if ( this.form ) {
+         var f = document.forms[this.form];
+         var checkboxes = f.selectedFactorValues;
+         if ( !checkboxes.length ) {
+            checkboxes = [ checkboxes ];
+         }
+         var values = [];
+         for ( var i = 0; i < checkboxes.length; ++i ) {
+            if ( checkboxes[i].checked ) {
+               values.push( checkboxes[i].value );
+            }
+         }
+         return values;
+      }
+   }
+
+} );
+
+
+/*
+"Upper" toolbar for factorvalue pane.
+ */
+Gemma.FactorValueToolbar = Ext.extend( Ext.Toolbar, {
+
+   /**
+    * @memberOf Gemma.FactorValueToolbar
+    */
+   initComponent : function() {
+      Gemma.FactorValueToolbar.superclass.initComponent.call( this );
+      this.addEvents( "create", "save", "delete", "undo", "refresh", "toggleExpand", "toggleCollapse" );
+   },
+
+   onRender : function( c, p ) {
+
+      Gemma.FactorValueToolbar.superclass.onRender.call( this, c, p );
+
+      this.createFactorValueButton = new Ext.Toolbar.Button( {
+         text : "Create",
+         id : 'factor-value-create-button',
+         tooltip : "Create a new factor value for the current factor",
+         disabled : true,
+         handler : function() {
+            this.fireEvent( "create" );
+            this.deleteFactorValueButton.enable();
+         },
+         scope : this
+      } );
+
+      this.deleteFactorValueButton = new Ext.Toolbar.Button( {
+         text : "Delete",
+         id : 'factor-value-delete-button',
+         tooltip : "Delete checked factor values",
+         disabled : true,
+         handler : function() {
+            this.fireEvent( "delete" );
+         }.createDelegate( this )
+      } );
+
+      this.saveButton = new Ext.Toolbar.Button( {
+         text : "Save",
+         id : 'factor-value-save-button',
+         tooltip : "Commit changes to factor values",
+         disabled : true,
+         handler : function() {
+            this.saveButton.disable();
+            this.fireEvent( "save" );
+         }.createDelegate( this )
+      } );
+
+      this.revertButton = new Ext.Toolbar.Button( {
+         text : "Undo",
+         id : 'factor-value-undo-button',
+         tooltip : "Undo changes to selected factor values",
+         disabled : true,
+         handler : function() {
+            this.fireEvent( "undo" );
+         },
+         scope : this
+      } );
+
+      this.refreshButton = new Ext.Toolbar.Button( {
+         icon : ctxBasePath + '/images/icons/arrow_refresh_small.png',
+         tooltip : 'Refresh the factor values',
+         handler : function() {
+            this.fireEvent( "refresh" );
+         },
+         scope : this
+      } );
+
+      if ( this.editable ) {
+         this.addButton( this.createFactorValueButton );
+         this.addSeparator();
+         this.addButton( this.deleteFactorValueButton );
+         this.addSpacer();
+         this.addButton( this.saveButton );
+         this.addSpacer();
+         this.addButton( this.revertButton );
+         this.addSpacer();
+         this.addButton( this.refreshButton );
+      }
+
+      this.collapseButton = new Ext.Toolbar.Button( {
+         text : "Collapse all",
+         tooltip : "Hide all factor value details",
+         handler : function() {
+            this.fireEvent( "toggleCollapse" );
+         },
+         scope : this
+      } );
+
+      this.addFill();
+      this.addButton( this.collapseButton );
+
+      this.expandButton = new Ext.Toolbar.Button( {
+         text : "Expand all",
+         tooltip : "Show all factor value details",
+         handler : function() {
+            this.fireEvent( "toggleExpand" );
+         },
+         scope : this
+      } );
+
+      this.addFill();
+      this.addButton( this.expandButton );
+
+      if ( this.editable ) {
+         this.characteristicToolbar = new Gemma.FactorValueCharacteristicToolbar( {
+            id : 'fv-char-toolbar',
+            renderTo : this.ownerCt.tbar
+         } );
+      }
+
+   },
+
+   setExperimentalFactor : function( efId ) {
+      this.efId = efId;
+      if ( efId != null ) {
+         this.createFactorValueButton.enable();
+      }
+      if ( this.characteristicToolbar ) {
+         this.characteristicToolbar.setExperimentalFactor( efId );
+      }
+
+   }
+
+} );
 
 /**
- *
+ * The toolbar that allows the user to add characteristics to factor values, it has its own combo boxes for value and category.
+ * Once the user inializes the characteristic they can edit it further in the grid.
  * @class Gemma.FactorValueCharacteristicToolbar
  * @extends Ext.Toolbar
  */
-Gemma.FactorValueCharacteristicToolbar = Ext.extend(Ext.Toolbar, {
-
-    /**
-     * @memberOf Gemma.FactorValueCharacteristicToolbar
-     */
-    initComponent: function () {
-        Gemma.FactorValueCharacteristicToolbar.superclass.initComponent.call(this);
-
-        Ext.apply(this, {
-            factorValue: {
-                id: -1,
-                classDelegatingFor: "FactorValue"
-            },
-
-            experimentalFactor: {
-                id: -1,
-                classDelegatingFor: "ExperimentalFactor"
+Gemma.FactorValueCharacteristicToolbar = Ext.extend( Ext.Toolbar, {
+
+   /**
+    * @memberOf Gemma.FactorValueCharacteristicToolbar
+    */
+   initComponent : function() {
+      Gemma.FactorValueCharacteristicToolbar.superclass.initComponent.call( this );
+
+      Ext.apply( this, {
+         factorValue : {
+            id : -1,
+            classDelegatingFor : "FactorValue"
+         },
+
+         experimentalFactor : {
+            id : -1,
+            classDelegatingFor : "ExperimentalFactor"
+         }
+      } );
+
+      this.addEvents( "create", "delete" );
+   },
+
+   onRender : function( c, l ) {
+      Gemma.FactorValueCharacteristicToolbar.superclass.onRender.call( this, c, l );
+
+      this.factorValueCombo = new Gemma.FactorValueCombo( {
+         id : 'fv-char-toolbar-fvcombo',
+         taxonId : this.taxonId,
+         disabled : this.experimentalFactor.id >= 0 ? false : true,
+         efId : this.experimentalFactor.id >= 0 ? this.experimentalFactor.id : null
+      } );
+
+      this.factorValueCombo.on( "select", function( combo, record, index ) {
+         this.factorValue.id = record.get( "id" );
+         this.categoryCombo.enable();
+      }.createDelegate( this ) );
+
+      this.categoryCombo = new Gemma.CategoryCombo( {
+         disabled : true,
+         emptyText : "Select a category",
+         termKey : "factorvalue"
+      } );
+
+      this.categoryCombo.on( "select", function( combo, record, index ) {
+         this.charCombo.setCategory( record.get( "term" ), record.get( "uri" ) );
+         this.charCombo.enable();
+         this.createButton.enable();
+      }.createDelegate( this ) );
+
+      this.charCombo = new Gemma.CharacteristicCombo( {
+         disabled : true,
+         taxonId : this.taxonId
+      } );
+
+      this.createButton = new Ext.Toolbar.Button( {
+         text : "Add",
+         tooltip : "Add the new characteristic to the selected factor value",
+         disabled : true,
+         handler : function() {
+            var c = this.charCombo.getCharacteristic();
+
+            if ( !this.factorValue || !c ) {
+               Ext.Msg.alert( "You must select a factor value and set a characteristic." );
+            } else {
+               this.createButton.disable();
+               // removed in response to bug 1016
+               // categoryCombo.reset();
+               this.charCombo.reset();
+               this.fireEvent( "create", this.factorValue, c );
+               this.charCombo.clearCharacteristic(); // avoid carryover to next edit
             }
-        });
-
-        this.addEvents("create", "delete");
-    },
-
-    onRender: function (c, l) {
-        Gemma.FactorValueCharacteristicToolbar.superclass.onRender.call(this, c, l);
-
-        this.factorValueCombo = new Gemma.FactorValueCombo({
-            id: 'fv-char-toolbar-fvcombo',
-            taxonId: this.taxonId,
-            disabled: this.experimentalFactor.id >= 0 ? false : true,
-            efId: this.experimentalFactor.id >= 0 ? this.experimentalFactor.id : null
-        });
-
-        this.factorValueCombo.on("select", function (combo, record, index) {
-            this.factorValue.id = record.get("id");
-            this.categoryCombo.enable();
-        }.createDelegate(this));
-
-        this.categoryCombo = new Gemma.CategoryCombo({
-            disabled: true,
-            emptyText: "Select a category",
-            termKey: "factorvalue"
-        });
-
-        this.categoryCombo.on("select", function (combo, record, index) {
-            this.charCombo.setCategory(record.get("term"), record.get("uri"));
-            this.charCombo.enable();
-            this.createButton.enable();
-        }.createDelegate(this));
-
-        this.charCombo = new Gemma.CharacteristicCombo({
-            disabled: true,
-            taxonId: this.taxonId
-        });
-
-        this.createButton = new Ext.Toolbar.Button({
-            text: "Add",
-            tooltip: "Add the new characteristic to the selected factor value",
-            disabled: true,
-            handler: function () {
-                var c = this.charCombo.getCharacteristic();
-
-                if (!this.factorValue || !c) {
-                    Ext.Msg.alert("You must select a factor value and set a characteristic.");
-                } else {
-                    this.createButton.disable();
-                    // removed in response to bug 1016
-                    // categoryCombo.reset();
-                    this.charCombo.reset();
-                    this.fireEvent("create", this.factorValue, c);
-                    this.charCombo.clearCharacteristic(); // avoid carryover to next edit
-                }
-            },
-            scope: this
-        });
-
-        this.charCombo.on("select", function () {
-            this.createButton.enable();
-        }.createDelegate(this));
-
-        this.charCombo.on("change", function () {
-            this.createButton.enable();
-        }.createDelegate(this));
-
-        this.deleteButton = new Ext.Toolbar.Button({
-            text: "Remove Characteristic",
-            tooltip: "Delete the selected characteristic(s) from factor values",
-            disabled: true,
-            handler: function () {
-                this.deleteButton.disable();
-                this.fireEvent("delete");
-            },
-            scope: this
-        });
-
-        this.addText("Append a characteristic to:");
-        this.add(this.factorValueCombo);
-        this.addSpacer();
-        this.addText(" with category:");
-        this.add(this.categoryCombo);
-        this.addSpacer();
-        this.addText(" and value:");
-        this.add(this.charCombo);
-        this.addSpacer();
-        this.addButton(this.createButton);
-        this.addSpacer();
-        this.addButton(this.deleteButton);
-
-    },
-
-    setExperimentalFactor: function (efId) {
-        this.experimentalFactor.id = efId;
-        this.factorValueCombo.setExperimentalFactor(efId, function () {
-            this.factorValueCombo.enable();
-            this.categoryCombo.enable();
-            this.charCombo.enable();
-        }.createDelegate(this));
-    }
-});
\ No newline at end of file
+         },
+         scope : this
+      } );
+
+      this.charCombo.on( "select", function() {
+         this.createButton.enable();
+      }.createDelegate( this ) );
+
+      this.charCombo.on( "change", function() {
+         this.createButton.enable();
+      }.createDelegate( this ) );
+
+      this.deleteButton = new Ext.Toolbar.Button( {
+         text : "Remove Characteristic",
+         tooltip : "Remove/delete the selected characteristic(s) from the factor value",
+         disabled : true,
+         handler : function() {
+            this.deleteButton.disable();
+            this.fireEvent( "delete" );
+         },
+         scope : this
+      } );
+
+      this.addText( "Append a characteristic to:" );
+      this.add( this.factorValueCombo );
+      this.addSpacer();
+      this.addText( " with category:" );
+      this.add( this.categoryCombo );
+      this.addSpacer();
+      this.addText( " and value:" );
+      this.add( this.charCombo );
+      this.addSpacer();
+      this.addButton( this.createButton );
+      this.addSpacer();
+      this.addButton( this.deleteButton );
+   },
+
+   setExperimentalFactor : function( efId ) {
+      this.experimentalFactor.id = efId;
+      this.factorValueCombo.setExperimentalFactor( efId, function() {
+         this.factorValueCombo.enable();
+         this.categoryCombo.enable();
+         this.charCombo.enable();
+      }.createDelegate( this ) );
+   }
+} );
\ No newline at end of file
diff --git a/gemma-web/src/main/webapp/scripts/api/annotation/RelationCombo.js b/gemma-web/src/main/webapp/scripts/api/annotation/RelationCombo.js
new file mode 100644
index 0000000000..49a55ee8d8
--- /dev/null
+++ b/gemma-web/src/main/webapp/scripts/api/annotation/RelationCombo.js
@@ -0,0 +1,72 @@
+Ext.namespace("Gemma");
+
+/**
+ * Dropdown menu of relations to use in annotations such as "has_role"
+ *
+ * @class Gemma.RelationCombo - similar to CategoryCombo, but for relations
+ * @extends Ext.form.ComboBox
+ */
+Gemma.RelationCombo = Ext.extend(Ext.form.ComboBox, {
+
+   editable: true,
+   mode: 'local',
+   selectOnFocus: true,
+   triggerAction: 'all',
+   typeAhead: true,
+   forceSelection: true,
+   displayField: 'term',
+
+   record: Ext.data.Record.create([{
+      name: "uri"
+   }, {
+      name: "term"
+   }, {
+      name: "comment"
+   }, {
+      name: "label"
+   },{
+      name: "obsolete" // boolean
+   }
+   ]),
+
+   getTerm: function () {
+      return this.selectedTerm;
+   },
+
+   initComponent: function () {
+
+      this.dwrMethod = AnnotationController.getRelationTerms;
+      this.dwrParams = [];
+
+      this.store = new Ext.data.Store({
+         proxy: new Ext.data.DWRProxy(this.dwrMethod),
+         reader: new Ext.data.ListRangeReader({
+            id: "uri"
+         }, this.record),
+         remoteSort: false,
+         sortInfo: {
+            field: "term"
+         }
+      });
+
+      Gemma.RelationCombo.superclass.initComponent.call(this);
+
+      this.on("change", function (combo) {
+         if(combo.value){
+            this.selectedTerm = combo.store.data.items[combo.selectedIndex].data;
+            combo.setValue(this.selectedTerm.term);
+         }else{
+            this.selectedTerm = undefined;
+         }
+      });
+
+      // Otherwise the combo is only firing this event after losing focus
+      this.on("select", function (combo, record, index) {
+         this.fireEvent("change", combo);
+      });
+
+      this.store.load({
+         params: this.dwrParams
+      });
+   }
+});
diff --git a/gemma-web/src/main/webapp/scripts/api/dwrServices.js b/gemma-web/src/main/webapp/scripts/api/dwrServices.js
index 15afd899f9..56a111bd53 100755
--- a/gemma-web/src/main/webapp/scripts/api/dwrServices.js
+++ b/gemma-web/src/main/webapp/scripts/api/dwrServices.js
@@ -33,6 +33,9 @@ AnnotationController.createBioMaterialTag = function (p0, p1, callback) {
 AnnotationController.getCategoryTerms = function (callback) {
     dwr.engine._execute(AnnotationController._path, 'AnnotationController', 'getCategoryTerms', callback);
 };
+AnnotationController.getRelationTerms = function (callback) {
+    dwr.engine._execute(AnnotationController._path, 'AnnotationController', 'getRelationTerms', callback);
+};
 AnnotationController.removeBioMaterialTag = function (p0, p1, callback) {
     dwr.engine._execute(AnnotationController._path, 'AnnotationController', 'removeBioMaterialTag', p0, p1, callback);
 };

From f785b833b3eb9c0b50e7e4188206e04ecd112139 Mon Sep 17 00:00:00 2001
From: Paul Pavlidis 
Date: Wed, 9 Aug 2023 09:52:07 -0700
Subject: [PATCH 024/370] found some addition potential terms and sources for
 more possibilities

---
 .../ubic/gemma/core/ontology/Relation.terms.txt | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/gemma-core/src/main/resources/ubic/gemma/core/ontology/Relation.terms.txt b/gemma-core/src/main/resources/ubic/gemma/core/ontology/Relation.terms.txt
index 826ea0da32..de32747be9 100644
--- a/gemma-core/src/main/resources/ubic/gemma/core/ontology/Relation.terms.txt
+++ b/gemma-core/src/main/resources/ubic/gemma/core/ontology/Relation.terms.txt
@@ -9,12 +9,19 @@ http://purl.obolibrary.org/obo/RO_0002573	has modifier
 http://purl.obolibrary.org/obo/RO_0016002	has disease
 http://purl.obolibrary.org/obo/RO_0002200	has phenotype
 http://purl.obolibrary.org/obo/RO_0001025	located in
+http://purl.obolibrary.org/obo/GENO_0000222	has_genotype
+http://purl.obolibrary.org/obo/GENO_0000413	has_allele
+http://purl.obolibrary.org/obo/GENO_0000447	is_gene_target_of
+http://purl.obolibrary.org/obo/CLO_0037210	derived from cell line
+http://purl.obolibrary.org/obo/CLO_0000015	derives from patient having disease
+http://purl.obolibrary.org/obo/BCGO_0000182	has duration # perhaps not ideal
 # missing:
 # has_delivered or delivered_by
 # has_timepoint
-# from_cell_line (derived from?)
-# has_genotype
-# has_dose
+# has_dose ;there are terms for 'dose' but not for 'has dose'?
 # has_stage (developmental stage)
-# has_allele
-# fusion_gene
+# fusion_gene - not an object property
+# see https://ontobee.org/ontology/catalog/GENO?iri=http://www.w3.org/2002/07/owl%23ObjectProperty&max=200
+# https://ontobee.org/ontology/catalog/CLO?iri=http://www.w3.org/2002/07/owl%23ObjectProperty&max=500
+# https://ontobee.org/ontology/catalog/CL?iri=http://www.w3.org/2002/07/owl%23ObjectProperty&max=500 (mostly RO terms)
+# warning: CLO has its own "has disease" http://purl.obolibrary.org/obo/CLO_0000167

From 20061acfe43b913644ca24b4c104e4e52a31490a Mon Sep 17 00:00:00 2001
From: Guillaume Poirier-Morency 
Date: Mon, 14 Aug 2023 15:28:32 -0700
Subject: [PATCH 025/370] Bump version for development

---
 gemma-cli/pom.xml            | 2 +-
 gemma-core/pom.xml           | 2 +-
 gemma-groovy-support/pom.xml | 2 +-
 gemma-rest/pom.xml           | 2 +-
 gemma-web/pom.xml            | 2 +-
 pom.xml                      | 2 +-
 6 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/gemma-cli/pom.xml b/gemma-cli/pom.xml
index a7e6a3b503..2fae5f2bbd 100644
--- a/gemma-cli/pom.xml
+++ b/gemma-cli/pom.xml
@@ -3,7 +3,7 @@
     
         gemma
         gemma
-        1.30.0
+        1.31.0-SNAPSHOT
     
     4.0.0
     gemma-cli
diff --git a/gemma-core/pom.xml b/gemma-core/pom.xml
index 4062deeb5c..aa68548416 100644
--- a/gemma-core/pom.xml
+++ b/gemma-core/pom.xml
@@ -3,7 +3,7 @@
     
         gemma
         gemma
-        1.30.0
+        1.31.0-SNAPSHOT
     
     4.0.0
     gemma-core
diff --git a/gemma-groovy-support/pom.xml b/gemma-groovy-support/pom.xml
index 236e931ce9..9457894581 100644
--- a/gemma-groovy-support/pom.xml
+++ b/gemma-groovy-support/pom.xml
@@ -6,7 +6,7 @@
     
         gemma
         gemma
-        1.30.0
+        1.31.0-SNAPSHOT
     
 
     gemma-groovy-support
diff --git a/gemma-rest/pom.xml b/gemma-rest/pom.xml
index 2b8c5a9eca..6e5bfadd05 100644
--- a/gemma-rest/pom.xml
+++ b/gemma-rest/pom.xml
@@ -5,7 +5,7 @@
     
         gemma
         gemma
-        1.30.0
+        1.31.0-SNAPSHOT
     
     4.0.0
 
diff --git a/gemma-web/pom.xml b/gemma-web/pom.xml
index e197374c95..47e118b47a 100644
--- a/gemma-web/pom.xml
+++ b/gemma-web/pom.xml
@@ -3,7 +3,7 @@
 	
 		gemma
 		gemma
-		1.30.0
+		1.31.0-SNAPSHOT
 	
 	4.0.0
 	gemma-web
diff --git a/pom.xml b/pom.xml
index 75e54d81ad..e9b0ee323c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -5,7 +5,7 @@
 	Gemma
 	gemma
 	gemma
-	1.30.0
+	1.31.0-SNAPSHOT
 	2005
 	The Gemma Project for meta-analysis of genomics data
 	https://gemma.msl.ubc.ca

From 345f7265e683b0bdca961fed0668650a23ec36af Mon Sep 17 00:00:00 2001
From: Guillaume Poirier-Morency 
Date: Mon, 14 Aug 2023 16:23:22 -0700
Subject: [PATCH 026/370] Remove unused includeOriginalPlatforms argument

---
 .../experiment/ExpressionExperimentService.java    |  8 +++-----
 .../ExpressionExperimentServiceImpl.java           | 14 +++++---------
 .../java/ubic/gemma/rest/DatasetsWebService.java   |  2 +-
 .../ubic/gemma/rest/DatasetsWebServiceTest.java    |  2 +-
 4 files changed, 10 insertions(+), 16 deletions(-)

diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/experiment/ExpressionExperimentService.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/experiment/ExpressionExperimentService.java
index 9550bd23e5..0fef677093 100644
--- a/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/experiment/ExpressionExperimentService.java
+++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/experiment/ExpressionExperimentService.java
@@ -295,12 +295,10 @@ class CharacteristicWithUsageStatisticsAndOntologyTerm {
     /**
      * Calculate the usage frequency of platforms by the datasets matching the provided filters.
      *
-     * @param filters                  a set of filters to be applied as per {@link #load(Filters, Sort, int, int)}
-     * @param includeOriginalPlatforms if true, original platforms as per {@link BioAssay#getOriginalPlatform()} are
-     *                                 also included.
-     * @param maxResults               the maximum of results, or unlimited if less than 1
+     * @param filters    a set of filters to be applied as per {@link #load(Filters, Sort, int, int)}
+     * @param maxResults the maximum of results, or unlimited if less than 1
      */
-    Map getArrayDesignUsedOrOriginalPlatformUsageFrequency( @Nullable Filters filters, boolean includeOriginalPlatforms, int maxResults );
+    Map getArrayDesignUsedOrOriginalPlatformUsageFrequency( @Nullable Filters filters, int maxResults );
 
     /**
      * Calculate the usage frequency of taxa by the datasets matching the provided filters.
diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/experiment/ExpressionExperimentServiceImpl.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/experiment/ExpressionExperimentServiceImpl.java
index 11fcdd2a68..24ade824e1 100755
--- a/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/experiment/ExpressionExperimentServiceImpl.java
+++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/experiment/ExpressionExperimentServiceImpl.java
@@ -776,22 +776,18 @@ public Map getTechnologyTypeUsageFrequency( @Nullable Filt
 
     @Override
     @Transactional(readOnly = true)
-    public Map getArrayDesignUsedOrOriginalPlatformUsageFrequency( @Nullable Filters filters, boolean includeOriginalPlatforms, int maxResults ) {
+    public Map getArrayDesignUsedOrOriginalPlatformUsageFrequency( @Nullable Filters filters, int maxResults ) {
         Map result;
         if ( filters == null || filters.isEmpty() ) {
             result = new HashMap<>( expressionExperimentDao.getArrayDesignsUsageFrequency( maxResults ) );
-            if ( includeOriginalPlatforms ) {
-                for ( Map.Entry e : expressionExperimentDao.getOriginalPlatformsUsageFrequency( maxResults ).entrySet() ) {
-                    result.compute( e.getKey(), ( k, v ) -> ( v != null ? v : 0L ) + e.getValue() );
-                }
+            for ( Map.Entry e : expressionExperimentDao.getOriginalPlatformsUsageFrequency( maxResults ).entrySet() ) {
+                result.compute( e.getKey(), ( k, v ) -> ( v != null ? v : 0L ) + e.getValue() );
             }
         } else {
             List ids = this.expressionExperimentDao.loadIdsWithCache( filters, null );
             result = new HashMap<>( expressionExperimentDao.getArrayDesignsUsageFrequency( ids, maxResults ) );
-            if ( includeOriginalPlatforms ) {
-                for ( Map.Entry e : expressionExperimentDao.getOriginalPlatformsUsageFrequency( ids, maxResults ).entrySet() ) {
-                    result.compute( e.getKey(), ( k, v ) -> ( v != null ? v : 0L ) + e.getValue() );
-                }
+            for ( Map.Entry e : expressionExperimentDao.getOriginalPlatformsUsageFrequency( ids, maxResults ).entrySet() ) {
+                result.compute( e.getKey(), ( k, v ) -> ( v != null ? v : 0L ) + e.getValue() );
             }
         }
         // retain top results
diff --git a/gemma-rest/src/main/java/ubic/gemma/rest/DatasetsWebService.java b/gemma-rest/src/main/java/ubic/gemma/rest/DatasetsWebService.java
index fa8344666c..87d30f66fb 100644
--- a/gemma-rest/src/main/java/ubic/gemma/rest/DatasetsWebService.java
+++ b/gemma-rest/src/main/java/ubic/gemma/rest/DatasetsWebService.java
@@ -254,7 +254,7 @@ public LimitedResponseDataObject getD
         }
         Integer l = limit.getValueNoMaximum();
         Map tts = expressionExperimentService.getTechnologyTypeUsageFrequency( filters );
-        Map ads = expressionExperimentService.getArrayDesignUsedOrOriginalPlatformUsageFrequency( filters, true, l );
+        Map ads = expressionExperimentService.getArrayDesignUsedOrOriginalPlatformUsageFrequency( filters, l );
         List adsVos = arrayDesignService.loadValueObjects( ads.keySet() );
         Map countsById = ads.entrySet().stream().collect( Collectors.toMap( e -> e.getKey().getId(), Map.Entry::getValue ) );
         List results =
diff --git a/gemma-rest/src/test/java/ubic/gemma/rest/DatasetsWebServiceTest.java b/gemma-rest/src/test/java/ubic/gemma/rest/DatasetsWebServiceTest.java
index 9931d6a9fd..4c507cafb1 100644
--- a/gemma-rest/src/test/java/ubic/gemma/rest/DatasetsWebServiceTest.java
+++ b/gemma-rest/src/test/java/ubic/gemma/rest/DatasetsWebServiceTest.java
@@ -241,7 +241,7 @@ public void testGetDatasetsPlatformsUsageStatistics() {
                 .hasEncoding( "gzip" );
         verify( expressionExperimentService ).getFilter( "id", Filter.Operator.lessThan, "10" );
         verify( expressionExperimentService ).getFiltersWithInferredAnnotations( Filters.by( f ), null );
-        verify( expressionExperimentService ).getArrayDesignUsedOrOriginalPlatformUsageFrequency( Filters.by( f ), true, 50 );
+        verify( expressionExperimentService ).getArrayDesignUsedOrOriginalPlatformUsageFrequency( Filters.by( f ), 50 );
     }
 
     @Test

From 5d8a4c2263b458a6886df12d542970a31404b25c Mon Sep 17 00:00:00 2001
From: Guillaume Poirier-Morency 
Date: Tue, 15 Aug 2023 11:35:19 -0700
Subject: [PATCH 027/370] Set -Dgemma.log.dir in Gemma CLI run configuration

---
 .idea/runConfigurations/Gemma_CLI.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.idea/runConfigurations/Gemma_CLI.xml b/.idea/runConfigurations/Gemma_CLI.xml
index cea41c8a33..535e961318 100644
--- a/.idea/runConfigurations/Gemma_CLI.xml
+++ b/.idea/runConfigurations/Gemma_CLI.xml
@@ -2,7 +2,7 @@