From 40e47834977606bc5c2c6c4239905e073f540179 Mon Sep 17 00:00:00 2001 From: Lum Date: Tue, 8 Aug 2023 12:12:51 -0700 Subject: [PATCH 01/15] check point --- .../assay/AbstractAssayTsvDataHandler.java | 16 ++- .../plate/AssayPlateMetadataService.java | 7 + .../api/assay/AssayResultDomainKind.java | 2 + .../org/labkey/api/assay/plate/Plate.java | 2 + .../org/labkey/api/assay/plate/Position.java | 2 + .../labkey/api/assay/plate/PositionImpl.java | 1 + .../src/org/labkey/assay/PlateController.java | 47 +++++- .../org/labkey/assay/TsvAssayProvider.java | 21 ++- .../plate/AssayPlateMetadataServiceImpl.java | 74 ++++++++++ .../src/org/labkey/assay/plate/PlateImpl.java | 17 +++ .../org/labkey/assay/plate/PlateManager.java | 134 ++++++++++++------ ...{PlateField.java => PlateCustomField.java} | 9 +- .../org/labkey/assay/plate/model/Well.java | 77 ++++++++++ .../assay/plate/model/WellCustomField.java | 30 ++++ 14 files changed, 385 insertions(+), 54 deletions(-) rename assay/src/org/labkey/assay/plate/model/{PlateField.java => PlateCustomField.java} (87%) create mode 100644 assay/src/org/labkey/assay/plate/model/Well.java create mode 100644 assay/src/org/labkey/assay/plate/model/WellCustomField.java diff --git a/api/src/org/labkey/api/assay/AbstractAssayTsvDataHandler.java b/api/src/org/labkey/api/assay/AbstractAssayTsvDataHandler.java index 82019f4f243..b08d253129a 100644 --- a/api/src/org/labkey/api/assay/AbstractAssayTsvDataHandler.java +++ b/api/src/org/labkey/api/assay/AbstractAssayTsvDataHandler.java @@ -659,17 +659,27 @@ protected ParticipantVisitResolver createResolver(User user, ExpRun run, ExpProt /** Insert the data into the database. Transaction is active. */ protected List> insertRowData(ExpData data, User user, Container container, ExpRun run, ExpProtocol protocol, AssayProvider provider,Domain dataDomain, List> fileData, TableInfo tableInfo, boolean autoFillDefaultColumns) - throws SQLException, ValidationException + throws SQLException, ValidationException, ExperimentException { + OntologyManager.UpdateableTableImportHelper importHelper = new SimpleAssayDataImportHelper(data); + if (provider.isPlateMetadataEnabled(protocol)) + { + AssayPlateMetadataService svc = AssayPlateMetadataService.getService(PlateMetadataDataHandler.DATA_TYPE); + if (svc != null) + { + importHelper = svc.getImportHelper(container, user, run, data, protocol, provider); + } + } + if (tableInfo instanceof UpdateableTableInfo) { - return OntologyManager.insertTabDelimited(tableInfo, container, user, new SimpleAssayDataImportHelper(data), fileData, autoFillDefaultColumns, LOG); + return OntologyManager.insertTabDelimited(tableInfo, container, user, importHelper, fileData, autoFillDefaultColumns, LOG); } else { Integer id = OntologyManager.ensureObject(container, data.getLSID()); List lsids = OntologyManager.insertTabDelimited(container, user, id, - new SimpleAssayDataImportHelper(data), dataDomain, fileData, false); + importHelper, dataDomain, fileData, false); // TODO: Add LSID values into return value rows return fileData; } diff --git a/api/src/org/labkey/api/assay/plate/AssayPlateMetadataService.java b/api/src/org/labkey/api/assay/plate/AssayPlateMetadataService.java index 314b5758583..14f480a279b 100644 --- a/api/src/org/labkey/api/assay/plate/AssayPlateMetadataService.java +++ b/api/src/org/labkey/api/assay/plate/AssayPlateMetadataService.java @@ -7,6 +7,7 @@ import org.labkey.api.data.Container; import org.labkey.api.exp.ExperimentException; import org.labkey.api.exp.Lsid; +import org.labkey.api.exp.OntologyManager; import org.labkey.api.exp.api.ExpData; import org.labkey.api.exp.api.ExpProtocol; import org.labkey.api.exp.api.ExpRun; @@ -72,6 +73,12 @@ List> mergePlateMetadata(Lsid plateLsid, List parsePlateMetadata(JSONObject json) throws ExperimentException; Map parsePlateMetadata(File jsonData) throws ExperimentException; + /** + * Returns an import helper to help join assay results data to well data and metadata that is associated + * with the plate used in the assay run import + */ + OntologyManager.UpdateableTableImportHelper getImportHelper(Container container, User user, ExpRun run, ExpData data, ExpProtocol protocol, AssayProvider provider) throws ExperimentException; + interface MetadataLayer { // the name of this layer diff --git a/assay/api-src/org/labkey/api/assay/AssayResultDomainKind.java b/assay/api-src/org/labkey/api/assay/AssayResultDomainKind.java index c96c3fc725b..11e93296d2d 100644 --- a/assay/api-src/org/labkey/api/assay/AssayResultDomainKind.java +++ b/assay/api-src/org/labkey/api/assay/AssayResultDomainKind.java @@ -45,6 +45,7 @@ public class AssayResultDomainKind extends AssayDomainKind { public static final String WELL_LOCATION_COLUMN_NAME = "WellLocation"; + public static final String WELL_LSID_COLUMN_NAME = "WellLsid"; public AssayResultDomainKind() { @@ -137,6 +138,7 @@ public Set getMandatoryPropertyNames(Domain domain) if (provider.isPlateMetadataEnabled(protocol)) { mandatoryNames.add(WELL_LOCATION_COLUMN_NAME); + mandatoryNames.add(WELL_LSID_COLUMN_NAME); } } } diff --git a/assay/api-src/org/labkey/api/assay/plate/Plate.java b/assay/api-src/org/labkey/api/assay/plate/Plate.java index a05eb51221b..cf0cb83302d 100644 --- a/assay/api-src/org/labkey/api/assay/plate/Plate.java +++ b/assay/api-src/org/labkey/api/assay/plate/Plate.java @@ -41,6 +41,8 @@ public interface Plate extends PropertySet, Identifiable Well getWell(int row, int col); + Well getWell(int rowId); + WellGroup getWellGroup(WellGroup.Type type, String wellGroupName); @Nullable WellGroup getWellGroup(int rowId); diff --git a/assay/api-src/org/labkey/api/assay/plate/Position.java b/assay/api-src/org/labkey/api/assay/plate/Position.java index 9123a8f7e01..57df15047c2 100644 --- a/assay/api-src/org/labkey/api/assay/plate/Position.java +++ b/assay/api-src/org/labkey/api/assay/plate/Position.java @@ -25,6 +25,8 @@ public interface Position { Integer getRowId(); + String getLsid(); + int getColumn(); int getRow(); diff --git a/assay/api-src/org/labkey/api/assay/plate/PositionImpl.java b/assay/api-src/org/labkey/api/assay/plate/PositionImpl.java index 277f48dedb5..9734ec43b53 100644 --- a/assay/api-src/org/labkey/api/assay/plate/PositionImpl.java +++ b/assay/api-src/org/labkey/api/assay/plate/PositionImpl.java @@ -125,6 +125,7 @@ public void setRow(int row) _row = row; } + @Override public String getLsid() { return _lsid; diff --git a/assay/src/org/labkey/assay/PlateController.java b/assay/src/org/labkey/assay/PlateController.java index 6d9dbd946c3..6ba0652c80b 100644 --- a/assay/src/org/labkey/assay/PlateController.java +++ b/assay/src/org/labkey/assay/PlateController.java @@ -57,8 +57,9 @@ import org.labkey.assay.plate.PlateDataServiceImpl; import org.labkey.assay.plate.PlateManager; import org.labkey.assay.plate.PlateUrls; -import org.labkey.assay.plate.model.PlateField; +import org.labkey.assay.plate.model.PlateCustomField; import org.labkey.assay.plate.model.PlateType; +import org.labkey.assay.plate.model.WellCustomField; import org.labkey.assay.view.AssayGWTView; import org.springframework.validation.BindException; import org.springframework.validation.Errors; @@ -660,7 +661,7 @@ public void validateForm(CreatePlateMetadataFieldsForm form, Errors errors) @Override public Object execute(CreatePlateMetadataFieldsForm form, BindException errors) throws Exception { - List newFields = PlateManager.get().createPlateMetadataFields(getContainer(), getUser(), form.getFields()); + List newFields = PlateManager.get().createPlateMetadataFields(getContainer(), getUser(), form.getFields()); return success(newFields); } } @@ -677,14 +678,14 @@ public Object execute(Object o, BindException errors) throws Exception public static class DeletePlateMetadataFieldsForm { - private List _fields; + private List _fields; - public List getFields() + public List getFields() { return _fields; } - public void setFields(List fields) + public void setFields(List fields) { _fields = fields; } @@ -751,4 +752,40 @@ public Object execute(CustomFieldsForm form, BindException errors) throws Except return success(PlateManager.get().removeFields(getContainer(), getUser(), form.getPlateId(), form.getFields())); } } + + public static class SetFieldsForm extends CustomFieldsForm + { + private Integer _wellId; + private List _wellFields; + + public Integer getWellId() + { + return _wellId; + } + + public void setWellId(Integer wellId) + { + _wellId = wellId; + } + + public List getWellFields() + { + return _wellFields; + } + + public void setWellFields(List wellFields) + { + _wellFields = wellFields; + } + } + + @RequiresPermission(UpdatePermission.class) + public class SetFieldsAction extends MutatingApiAction + { + @Override + public Object execute(SetFieldsForm form, BindException errors) throws Exception + { + return success(PlateManager.get().setFields(getContainer(), getUser(), form.getPlateId(), form.getWellId(), form.getWellFields())); + } + } } diff --git a/assay/src/org/labkey/assay/TsvAssayProvider.java b/assay/src/org/labkey/assay/TsvAssayProvider.java index 31f21e43822..0b0acb69121 100644 --- a/assay/src/org/labkey/assay/TsvAssayProvider.java +++ b/assay/src/org/labkey/assay/TsvAssayProvider.java @@ -85,6 +85,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; /** * User: brittp @@ -364,15 +365,29 @@ public void changeDomain(User user, ExpProtocol protocol, GWTDomain field.getName().equals(AssayResultDomainKind.WELL_LOCATION_COLUMN_NAME))) + ArrayList newFields = new ArrayList<>(); + Set existingFields = update.getFields().stream().map(GWTPropertyDescriptor::getName).collect(Collectors.toSet()); + + if (!existingFields.contains(AssayResultDomainKind.WELL_LOCATION_COLUMN_NAME)) { GWTPropertyDescriptor wellLocation = new GWTPropertyDescriptor(AssayResultDomainKind.WELL_LOCATION_COLUMN_NAME, PropertyType.STRING.getTypeUri()); wellLocation.setShownInUpdateView(false); - ArrayList newFields = new ArrayList<>(); newFields.add(wellLocation); - newFields.addAll(update.getFields()); + } + + if (!existingFields.contains(AssayResultDomainKind.WELL_LSID_COLUMN_NAME)) + { + GWTPropertyDescriptor wellLsid = new GWTPropertyDescriptor(AssayResultDomainKind.WELL_LSID_COLUMN_NAME, PropertyType.STRING.getTypeUri()); + wellLsid.setShownInUpdateView(false); + wellLsid.setHidden(true); + + newFields.add(wellLsid); + } + if (!newFields.isEmpty()) + { + newFields.addAll(update.getFields()); update.setFields(newFields); } } diff --git a/assay/src/org/labkey/assay/plate/AssayPlateMetadataServiceImpl.java b/assay/src/org/labkey/assay/plate/AssayPlateMetadataServiceImpl.java index 2d659d22792..14b75eb8786 100644 --- a/assay/src/org/labkey/assay/plate/AssayPlateMetadataServiceImpl.java +++ b/assay/src/org/labkey/assay/plate/AssayPlateMetadataServiceImpl.java @@ -8,6 +8,7 @@ import org.labkey.api.assay.AssayProvider; import org.labkey.api.assay.AssayResultDomainKind; import org.labkey.api.assay.AssaySchema; +import org.labkey.api.assay.SimpleAssayDataImportHelper; import org.labkey.api.assay.plate.AssayPlateMetadataService; import org.labkey.api.assay.plate.Plate; import org.labkey.api.assay.plate.PlateService; @@ -19,10 +20,14 @@ import org.labkey.api.collections.CaseInsensitiveLinkedHashMap; import org.labkey.api.data.Container; import org.labkey.api.data.JdbcType; +import org.labkey.api.data.ParameterMapStatement; import org.labkey.api.data.PropertyStorageSpec; +import org.labkey.api.data.SimpleFilter; import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; import org.labkey.api.exp.ExperimentException; import org.labkey.api.exp.Lsid; +import org.labkey.api.exp.OntologyManager; import org.labkey.api.exp.PropertyDescriptor; import org.labkey.api.exp.api.ExpData; import org.labkey.api.exp.api.ExpProtocol; @@ -32,18 +37,25 @@ import org.labkey.api.exp.property.DomainProperty; import org.labkey.api.exp.property.PropertyService; import org.labkey.api.query.BatchValidationException; +import org.labkey.api.query.FieldKey; import org.labkey.api.query.QueryUpdateService; +import org.labkey.api.query.ValidationException; import org.labkey.api.security.User; import org.labkey.assay.TSVProtocolSchema; +import org.labkey.assay.plate.model.Well; +import org.labkey.assay.query.AssayDbSchema; import java.io.File; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import static org.labkey.api.assay.AssayResultDomainKind.WELL_LSID_COLUMN_NAME; + public class AssayPlateMetadataServiceImpl implements AssayPlateMetadataService { private boolean _domainDirty; @@ -369,6 +381,68 @@ else if (propEntry.getValue().isBoolean()) return layers; } + @Override + public OntologyManager.UpdateableTableImportHelper getImportHelper( + Container container, + User user, + ExpRun run, + ExpData data, + ExpProtocol protocol, + AssayProvider provider) throws ExperimentException + { + // get the plate associated with this run + Domain runDomain = provider.getRunDomain(protocol); + DomainProperty property = runDomain.getPropertyByName(AssayPlateMetadataService.PLATE_TEMPLATE_COLUMN_NAME); + Lsid plateLsid = null; + + if (property != null) + plateLsid = Lsid.parse(String.valueOf(run.getProperty(property))); + + Plate plate = PlateService.get().getPlate(protocol.getContainer(), plateLsid); + if (plate == null) + throw new ExperimentException(String.format("Unable to resolve the plate : %s for the run", plateLsid)); + + // create the map of well locations to the well table lsid + Map positionToWellLsid = new HashMap<>(); + + SimpleFilter filter = SimpleFilter.createContainerFilter(plate.getContainer()); + filter.addCondition(FieldKey.fromParts("PlateId"), plate.getRowId()); + for (Well well : new TableSelector(AssayDbSchema.getInstance().getTableInfoWell(), filter, null).getArrayList(Well.class)) + positionToWellLsid.put(new PositionImpl(plate.getContainer(), well.getRow(), well.getCol()), Lsid.parse(well.getLsid())); + + return new PlateMetadataImportHelper(plate.getContainer(), data, Collections.unmodifiableMap(positionToWellLsid)); + } + + private static class PlateMetadataImportHelper extends SimpleAssayDataImportHelper + { + private Map _wellPositionMap; + private Container _container; + + public PlateMetadataImportHelper(Container container, ExpData data, Map wellPositionMap) + { + super(data); + _wellPositionMap = wellPositionMap; + _container = container; + } + + @Override + public void bindAdditionalParameters(Map map, ParameterMapStatement target) throws ValidationException + { + super.bindAdditionalParameters(map, target); + + // to join plate based metadata to assay results we need to line up the incoming assay results with the + // corresponding well on the plate used in the import + if (map.containsKey(AssayResultDomainKind.WELL_LOCATION_COLUMN_NAME)) + { + PositionImpl pos = new PositionImpl(_container, String.valueOf(map.get(AssayResultDomainKind.WELL_LOCATION_COLUMN_NAME))); + // need to adjust the column value to be 0 based to match the template locations + pos.setCol(pos.getColumn() - 1); + if (_wellPositionMap.containsKey(pos)) + target.put(WELL_LSID_COLUMN_NAME, _wellPositionMap.get(pos)); + } + } + } + private static class MetadataLayerImpl implements MetadataLayer { private String _name; diff --git a/assay/src/org/labkey/assay/plate/PlateImpl.java b/assay/src/org/labkey/assay/plate/PlateImpl.java index 50654cb6d2c..378635d51c6 100644 --- a/assay/src/org/labkey/assay/plate/PlateImpl.java +++ b/assay/src/org/labkey/assay/plate/PlateImpl.java @@ -24,6 +24,7 @@ import org.labkey.api.assay.plate.PlateService; import org.labkey.api.assay.plate.Position; import org.labkey.api.assay.plate.PositionImpl; +import org.labkey.api.assay.plate.Well; import org.labkey.api.assay.plate.WellGroup; import org.labkey.api.data.Container; import org.labkey.api.data.Transient; @@ -33,6 +34,7 @@ import org.labkey.assay.PlateController; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.Date; import java.util.HashMap; @@ -59,6 +61,8 @@ public class PlateImpl extends PropertySetImpl implements Plate private List _deletedGroups; private WellImpl[][] _wells; + private Map _wellMap; + private int _runId; // NO_RUNID means no run yet, well data comes from file, dilution data must be calculated private int _plateNumber; @@ -480,6 +484,13 @@ public WellImpl getWell(int row, int col) } } + @Override + @Nullable + public Well getWell(int rowId) + { + return _wellMap != null ? _wellMap.get(rowId) : null; + } + @JsonIgnore @Override public WellGroup getWellGroup(WellGroup.Type type, String wellGroupName) @@ -500,6 +511,12 @@ protected WellGroupImpl createWellGroup(String name, WellGroup.Type type, List

(); + Arrays.stream(_wells).toList().forEach(w -> { + Arrays.stream(w).toList().forEach(well -> _wellMap.put(well.getRowId(), well)); + }); } @JsonIgnore diff --git a/assay/src/org/labkey/assay/plate/PlateManager.java b/assay/src/org/labkey/assay/plate/PlateManager.java index 81539ee3019..0fde7dfebb0 100644 --- a/assay/src/org/labkey/assay/plate/PlateManager.java +++ b/assay/src/org/labkey/assay/plate/PlateManager.java @@ -82,8 +82,9 @@ import org.labkey.api.view.ActionURL; import org.labkey.api.webdav.WebdavResource; import org.labkey.assay.TsvAssayProvider; -import org.labkey.assay.plate.model.PlateField; +import org.labkey.assay.plate.model.PlateCustomField; import org.labkey.assay.plate.model.PlateType; +import org.labkey.assay.plate.model.WellCustomField; import org.labkey.assay.plate.model.WellGroupBean; import org.labkey.assay.plate.query.PlateSchema; import org.labkey.assay.plate.query.PlateTable; @@ -416,15 +417,6 @@ protected void populatePlate(PlateImpl plate) // set plate properties: setProperties(plate.getContainer(), plate); - Position[][] positionArray; - if (plate.isTemplate()) - positionArray = new Position[plate.getRows()][plate.getColumns()]; - else - positionArray = new WellImpl[plate.getRows()][plate.getColumns()]; - - // get position objects - PositionImpl[] positions = getPositions(plate); - // query for all well to well group mappings on the plate SQLFragment sql = new SQLFragment(); sql.append("SELECT wgp.wellId, wgp.wellGroupId FROM ") @@ -449,24 +441,24 @@ protected void populatePlate(PlateImpl plate) // construct groupIdToPositions: map of wellGroupId -> List of PositionImpl Map> groupIdToPositions = new HashMap<>(); - for (PositionImpl position : positions) + WellImpl[] wells = getWells(plate); + WellImpl[][] wellArray = new WellImpl[plate.getRows()][plate.getColumns()]; + for (WellImpl well : wells) { - positionArray[position.getRow()][position.getColumn()] = position; + wellArray[well.getRow()][well.getColumn()] = well; - Set wellGroupIds = wellToWellGroups.get(position.getRowId()); + Set wellGroupIds = wellToWellGroups.get(well.getRowId()); if (wellGroupIds != null) { for (Integer wellGroupId : wellGroupIds) { List groupPositions = groupIdToPositions.computeIfAbsent(wellGroupId, k -> new ArrayList<>()); - groupPositions.add(position); + groupPositions.add(well); } } } - - // not sure if this would ever be true - if (positionArray instanceof WellImpl[][] wells) - plate.setWells(wells); + // add the wells to the plate + plate.setWells(wellArray); // populate well groups: assign all positions to the well group object WellGroupImpl[] wellgroups = getWellGroups(plate); @@ -486,12 +478,11 @@ protected void populatePlate(PlateImpl plate) plate.addWellGroup(group); } - private PositionImpl[] getPositions(Plate plate) + private WellImpl[] getWells(Plate plate) { SimpleFilter plateFilter = new SimpleFilter(FieldKey.fromParts("PlateId"), plate.getRowId()); Sort sort = new Sort("Col,Row"); - Class clazz = plate.isTemplate() ? PositionImpl.class : WellImpl.class; - return new TableSelector(AssayDbSchema.getInstance().getTableInfoWell(), plateFilter, sort).getArray(clazz); + return new TableSelector(AssayDbSchema.getInstance().getTableInfoWell(), plateFilter, sort).getArray(WellImpl.class); } @@ -620,7 +611,7 @@ private int savePlateImpl(Container container, User user, PlateImpl plate) throw Map, PositionImpl> existingPositionMap = new HashMap<>(); if (updateExisting) { - for (PositionImpl existingPosition : getPositions(plate)) + for (PositionImpl existingPosition : getWells(plate)) { existingPositionMap.put(Pair.of(existingPosition.getRow(), existingPosition.getCol()), existingPosition); } @@ -780,14 +771,14 @@ public void beforePlateDelete(Container container, Integer plateId) PlateImpl plate = new TableSelector(schema.getTableInfoPlate(), plateFilter, null).getObject(PlateImpl.class); WellGroupImpl[] wellgroups = getWellGroups(plate); - PositionImpl[] positions = getPositions(plate); + WellImpl[] wells = getWells(plate); List lsids = new ArrayList<>(); lsids.add(plate.getLSID()); for (WellGroupImpl wellgroup : wellgroups) lsids.add(wellgroup.getLSID()); - for (PositionImpl position : positions) - lsids.add(position.getLsid()); + for (WellImpl well : wells) + lsids.add(well.getLsid()); SimpleFilter plateIdFilter = SimpleFilter.createContainerFilter(container); plateIdFilter.addCondition(FieldKey.fromParts("PlateId"), plate.getRowId()); @@ -1243,7 +1234,7 @@ private Container getPlateMetadataDomainContainer(Container container) /** * Adds custom fields to the well domain */ - public @NotNull List createPlateMetadataFields(Container container, User user, List fields) throws Exception + public @NotNull List createPlateMetadataFields(Container container, User user, List fields) throws Exception { Domain vocabDomain = ensurePlateMetadataDomain(container, user); DomainKind domainKind = vocabDomain.getDomainKind(); @@ -1273,7 +1264,7 @@ private Container getPlateMetadataDomainContainer(Container container) return getPlateMetadataFields(container, user); } - public @NotNull List deletePlateMetadataFields(Container container, User user, List fields) throws Exception + public @NotNull List deletePlateMetadataFields(Container container, User user, List fields) throws Exception { Domain vocabDomain = getPlateMetadataDomain(container, user); @@ -1286,7 +1277,7 @@ private Container getPlateMetadataDomainContainer(Container container) if (!fields.isEmpty()) { List propertyURIs = new ArrayList<>(); - for (PlateField field : fields) + for (PlateCustomField field : fields) { if (field.getPropertyURI() == null) throw new IllegalStateException("Unable to remove fields, the property URI must be specified."); @@ -1304,7 +1295,7 @@ private Container getPlateMetadataDomainContainer(Container container) try (DbScope.Transaction tx = ExperimentService.get().ensureTransaction()) { Set existingProperties = vocabDomain.getProperties().stream().map(ImportAliasable::getPropertyURI).collect(Collectors.toSet()); - for (PlateField field : fields) + for (PlateCustomField field : fields) { if (!existingProperties.contains(field.getPropertyURI())) throw new IllegalStateException(String.format("Unable to remove field: %s on domain: %s. The field does not exist.", field.getName(), vocabDomain.getTypeURI())); @@ -1319,20 +1310,20 @@ private Container getPlateMetadataDomainContainer(Container container) return getPlateMetadataFields(container, user); } - public @NotNull List getPlateMetadataFields(Container container, User user) + public @NotNull List getPlateMetadataFields(Container container, User user) { Domain vocabDomain = getPlateMetadataDomain(container, user); if (vocabDomain != null) { - List fields = vocabDomain.getProperties().stream().map(PlateField::new).toList(); + List fields = vocabDomain.getProperties().stream().map(PlateCustomField::new).toList(); return fields.stream() - .sorted(Comparator.comparing(PlateField::getName)) + .sorted(Comparator.comparing(PlateCustomField::getName)) .collect(Collectors.toList()); } return Collections.emptyList(); } - public @NotNull List addFields(Container container, User user, Integer plateId, List fields) throws SQLException + public @NotNull List addFields(Container container, User user, Integer plateId, List fields) throws SQLException { if (plateId == null) throw new IllegalArgumentException("Failed to add plate custom fields. Invalid plateId provided."); @@ -1350,7 +1341,7 @@ private Container getPlateMetadataDomainContainer(Container container) List fieldsToAdd = new ArrayList<>(); // validate fields - for (PlateField field : fields) + for (PlateCustomField field : fields) { DomainProperty dp = domain.getPropertyByURI(field.getPropertyURI()); if (dp == null) @@ -1389,7 +1380,18 @@ private Container getPlateMetadataDomainContainer(Container container) return getFields(container, user, plateId); } - public List getFields(Container container, User user, Integer plateId) + public List getFields(Container container, User user, Integer plateId) + { + List fields = _getFields(container, user, plateId).stream().map(PlateCustomField::new).toList(); + return fields.stream() + .sorted(Comparator.comparing(PlateCustomField::getName)) + .collect(Collectors.toList()); + } + + /** + * Returns the list of custom properties associated with a plate + */ + private List _getFields(Container container, User user, Integer plateId) { Plate plate = getPlate(container, plateId); if (plate == null) @@ -1402,22 +1404,35 @@ public List getFields(Container container, User user, Integer plateI SQLFragment sql = new SQLFragment("SELECT PropertyURI FROM ").append(AssayDbSchema.getInstance().getTableInfoPlateProperty(), "PP") .append(" WHERE PlateId = ?").add(plateId); - List fields = new ArrayList<>(); + List fields = new ArrayList<>(); for (String uri : new SqlSelector(AssayDbSchema.getInstance().getSchema(), sql).getArrayList(String.class)) { DomainProperty dp = domain.getPropertyByURI(uri); if (dp == null) throw new IllegalArgumentException("Failed to get plate custom field. \"" + uri + "\" does not exist on domain."); - fields.add(new PlateField(dp)); + fields.add(dp); } + return fields; + } + + private List getWellCustomFields(Container container, User user, Integer plateId, Integer wellId) + { + List fields = _getFields(container, user, plateId).stream().map(WellCustomField::new).toList(); + // need to get the well values associated with each custom field + Plate plate = getPlate(container, plateId); + Well well = plate.getWell(wellId); + if (well == null) + throw new IllegalArgumentException("Failed to get well custom fields. Well id \"" + wellId + "\" not found."); + + Map properties = OntologyManager.getProperties(container, well.getLsid()); return fields.stream() - .sorted(Comparator.comparing(PlateField::getName)) + .sorted(Comparator.comparing(PlateCustomField::getName)) .collect(Collectors.toList()); } - public List removeFields(Container container, User user, Integer plateId, List fields) + public List removeFields(Container container, User user, Integer plateId, List fields) { Plate plate = getPlate(container, plateId); if (plate == null) @@ -1429,7 +1444,7 @@ public List removeFields(Container container, User user, Integer pla List fieldsToRemove = new ArrayList<>(); // validate fields - for (PlateField field : fields) + for (PlateCustomField field : fields) { DomainProperty dp = domain.getPropertyByURI(field.getPropertyURI()); if (dp == null) @@ -1451,6 +1466,45 @@ public List removeFields(Container container, User user, Integer pla return getFields(container, user, plateId); } + public List setFields(Container container, User user, Integer plateId, Integer wellId, List fields) throws ValidationException + { + Plate plate = getPlate(container, plateId); + if (plate == null) + throw new IllegalArgumentException("Failed to set plate custom field values. Plate id \"" + plateId + "\" not found."); + + WellImpl well = (WellImpl) plate.getWell(wellId); + if (well == null) + throw new IllegalArgumentException("Failed to set plate custom field values. Well id \"" + wellId + "\" not found."); + + Domain domain = getPlateMetadataDomain(container, user); + if (domain == null) + throw new IllegalArgumentException("Failed to set plate custom field values. Custom fields domain does not exist. Try creating fields first."); + + // verify the fields have been configured for the plate + Set configuredProps = new HashSet<>(); + SQLFragment sql = new SQLFragment("SELECT PropertyURI FROM ").append(AssayDbSchema.getInstance().getTableInfoPlateProperty(), "") + .append(" WHERE PlateId = ?").add(plateId); + new SqlSelector(AssayDbSchema.getInstance().getSchema(), sql).forEach(String.class, configuredProps::add); + + try (DbScope.Transaction transaction = ExperimentService.get().ensureTransaction()) + { + for (WellCustomField field : fields) + { + if (!configuredProps.contains(field.getPropertyURI())) + throw new IllegalArgumentException("Failed to set plate custom field value. Custom field \"" + field.getPropertyURI() + "\" is not configured for this plate."); + + DomainProperty dp = domain.getPropertyByURI(field.getPropertyURI()); + if (dp == null) + throw new IllegalArgumentException("Failed to set plate custom field values. Custom field \"" + field.getPropertyURI() + "\" not found."); + + OntologyManager.updateObjectProperty(user, container, dp.getPropertyDescriptor(), well.getLsid(), field.getValue(), null, true); + } + transaction.commit(); + } + + return getWellCustomFields(container, user, plateId, wellId); + } + public static final class TestCase { @Test diff --git a/assay/src/org/labkey/assay/plate/model/PlateField.java b/assay/src/org/labkey/assay/plate/model/PlateCustomField.java similarity index 87% rename from assay/src/org/labkey/assay/plate/model/PlateField.java rename to assay/src/org/labkey/assay/plate/model/PlateCustomField.java index 84e71f45619..2bf207e2b8f 100644 --- a/assay/src/org/labkey/assay/plate/model/PlateField.java +++ b/assay/src/org/labkey/assay/plate/model/PlateCustomField.java @@ -3,8 +3,11 @@ import com.fasterxml.jackson.annotation.JsonInclude; import org.labkey.api.exp.property.DomainProperty; +/** + * Represents a custom field that is configured for a specific plate + */ @JsonInclude(JsonInclude.Include.NON_NULL) -public class PlateField +public class PlateCustomField { private String _label; private String _name; @@ -12,11 +15,11 @@ public class PlateField private String _rangeURI; private String _container; - public PlateField() + public PlateCustomField() { } - public PlateField(DomainProperty prop) + public PlateCustomField(DomainProperty prop) { _name = prop.getName(); _label = prop.getLabel(); diff --git a/assay/src/org/labkey/assay/plate/model/Well.java b/assay/src/org/labkey/assay/plate/model/Well.java new file mode 100644 index 00000000000..59639d2bac9 --- /dev/null +++ b/assay/src/org/labkey/assay/plate/model/Well.java @@ -0,0 +1,77 @@ +package org.labkey.assay.plate.model; + +import com.fasterxml.jackson.annotation.JsonInclude; + +/** + * Represents a row in the plate.well table + */ +@JsonInclude(JsonInclude.Include.NON_NULL) +public class Well +{ + private Integer _rowId; + private String _lsid; + private Integer _plateId; + private Integer _row; + private Integer _col; + private String _container; + + public Integer getRowId() + { + return _rowId; + } + + public void setRowId(Integer rowId) + { + _rowId = rowId; + } + + public String getLsid() + { + return _lsid; + } + + public void setLsid(String lsid) + { + _lsid = lsid; + } + + public Integer getPlateId() + { + return _plateId; + } + + public void setPlateId(Integer plateId) + { + _plateId = plateId; + } + + public Integer getRow() + { + return _row; + } + + public void setRow(Integer row) + { + _row = row; + } + + public Integer getCol() + { + return _col; + } + + public void setCol(Integer col) + { + _col = col; + } + + public String getContainer() + { + return _container; + } + + public void setContainer(String container) + { + _container = container; + } +} diff --git a/assay/src/org/labkey/assay/plate/model/WellCustomField.java b/assay/src/org/labkey/assay/plate/model/WellCustomField.java new file mode 100644 index 00000000000..224d5e5f322 --- /dev/null +++ b/assay/src/org/labkey/assay/plate/model/WellCustomField.java @@ -0,0 +1,30 @@ +package org.labkey.assay.plate.model; + +import org.labkey.api.exp.property.DomainProperty; + +/** + * Represents a custom field, including value, that is assigned to a well location + */ +public class WellCustomField extends PlateCustomField +{ + private Object _value; + + public WellCustomField() + { + } + + public WellCustomField(DomainProperty dp) + { + super(dp); + } + + public Object getValue() + { + return _value; + } + + public void setValue(Object value) + { + _value = value; + } +} From eb6aa19590592e7b81edd2afb19ca6703e24232d Mon Sep 17 00:00:00 2001 From: Lum Date: Wed, 9 Aug 2023 16:31:54 -0700 Subject: [PATCH 02/15] support import of plate with pre-defined metadata --- .../org/labkey/assay/TSVProtocolSchema.java | 67 ++++++++++++------- .../src/org/labkey/assay/plate/PlateImpl.java | 5 +- .../org/labkey/assay/plate/PlateManager.java | 3 + 3 files changed, 50 insertions(+), 25 deletions(-) diff --git a/assay/src/org/labkey/assay/TSVProtocolSchema.java b/assay/src/org/labkey/assay/TSVProtocolSchema.java index 141a45f8638..1f465b2de98 100644 --- a/assay/src/org/labkey/assay/TSVProtocolSchema.java +++ b/assay/src/org/labkey/assay/TSVProtocolSchema.java @@ -18,6 +18,7 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.labkey.api.assay.AssayProtocolSchema; +import org.labkey.api.assay.AssayResultDomainKind; import org.labkey.api.assay.AssayResultTable; import org.labkey.api.assay.AssayUrls; import org.labkey.api.assay.AssayWellExclusionService; @@ -155,40 +156,58 @@ private class _AssayResultTable extends AssayResultTable } List defaultColumns = new ArrayList<>(getDefaultVisibleColumns()); - Domain plateDataDomain = AssayPlateMetadataService.getService(PlateMetadataDataHandler.DATA_TYPE).getPlateDataDomain(getProtocol()); - if (plateDataDomain != null) + if (getProvider().isPlateMetadataEnabled(getProtocol())) { - List plateDefaultColumns = new ArrayList<>(); - ColumnInfo lsidCol = getColumn("Lsid"); - if (lsidCol != null) + // legacy standard assay plate metadata support + Domain plateDataDomain = AssayPlateMetadataService.getService(PlateMetadataDataHandler.DATA_TYPE).getPlateDataDomain(getProtocol()); + if (plateDataDomain != null) { - BaseColumnInfo col = new AliasedColumn("PlateData", lsidCol); - col.setFk(QueryForeignKey + List plateDefaultColumns = new ArrayList<>(); + ColumnInfo lsidCol = getColumn("Lsid"); + if (lsidCol != null) + { + BaseColumnInfo col = new AliasedColumn("PlateData", lsidCol); + col.setFk(QueryForeignKey .from(getUserSchema(), getContainerFilter()) .to(PLATE_DATA_TABLE, "Lsid", null) + ); + col.setUserEditable(false); + col.setCalculated(true); + addColumn(col); + + for (DomainProperty prop : plateDataDomain.getProperties()) + { + plateDefaultColumns.add(FieldKey.fromParts("PlateData", prop.getName())); + } + + // show the layer columns first + plateDefaultColumns.sort((o1, o2) -> { + if (o1.toString().toLowerCase().endsWith(PLATE_DATA_LAYER_SUFFIX)) + return -1; + else if (o2.toString().toLowerCase().endsWith(PLATE_DATA_LAYER_SUFFIX)) + return 1; + else + return o1.toString().compareTo(o2.toString()); + }); + defaultColumns.addAll(plateDefaultColumns); + } + } + setDefaultVisibleColumns(defaultColumns); + + // join to the well table which may have plate metadata + ColumnInfo wellLsidCol = getColumn(AssayResultDomainKind.WELL_LSID_COLUMN_NAME); + if (wellLsidCol != null) + { + BaseColumnInfo col = new AliasedColumn("Well", wellLsidCol); + col.setFk(QueryForeignKey + .from(getUserSchema(), getContainerFilter()) + .schema("plate").table("well").key("Lsid") ); col.setUserEditable(false); col.setCalculated(true); addColumn(col); - - for (DomainProperty prop : plateDataDomain.getProperties()) - { - plateDefaultColumns.add(FieldKey.fromParts("PlateData", prop.getName())); - } - - // show the layer columns first - plateDefaultColumns.sort((o1, o2) -> { - if (o1.toString().toLowerCase().endsWith(PLATE_DATA_LAYER_SUFFIX)) - return -1; - else if (o2.toString().toLowerCase().endsWith(PLATE_DATA_LAYER_SUFFIX)) - return 1; - else - return o1.toString().compareTo(o2.toString()); - }); - defaultColumns.addAll(plateDefaultColumns); } } - setDefaultVisibleColumns(defaultColumns); } } diff --git a/assay/src/org/labkey/assay/plate/PlateImpl.java b/assay/src/org/labkey/assay/plate/PlateImpl.java index 378635d51c6..d7841e76304 100644 --- a/assay/src/org/labkey/assay/plate/PlateImpl.java +++ b/assay/src/org/labkey/assay/plate/PlateImpl.java @@ -515,7 +515,10 @@ public void setWells(WellImpl[][] wells) // create a rowId to well map _wellMap = new HashMap<>(); Arrays.stream(_wells).toList().forEach(w -> { - Arrays.stream(w).toList().forEach(well -> _wellMap.put(well.getRowId(), well)); + Arrays.stream(w).toList().forEach(well -> { + if (well != null) + _wellMap.put(well.getRowId(), well); + }); }); } diff --git a/assay/src/org/labkey/assay/plate/PlateManager.java b/assay/src/org/labkey/assay/plate/PlateManager.java index 0fde7dfebb0..35fa068b3b4 100644 --- a/assay/src/org/labkey/assay/plate/PlateManager.java +++ b/assay/src/org/labkey/assay/plate/PlateManager.java @@ -1427,6 +1427,9 @@ private List getWellCustomFields(Container container, User user throw new IllegalArgumentException("Failed to get well custom fields. Well id \"" + wellId + "\" not found."); Map properties = OntologyManager.getProperties(container, well.getLsid()); + for (WellCustomField field : fields) + field.setValue(properties.get(field.getPropertyURI())); + return fields.stream() .sorted(Comparator.comparing(PlateCustomField::getName)) .collect(Collectors.toList()); From 8227358b5e53944de8dfb5270396e6d6b8dab797 Mon Sep 17 00:00:00 2001 From: Lum Date: Wed, 9 Aug 2023 17:21:34 -0700 Subject: [PATCH 03/15] plate metadata optional (in the app), add metrics for assays with plate support --- .../org/labkey/api/assay/AbstractAssayTsvDataHandler.java | 7 ++++++- experiment/src/org/labkey/experiment/ExperimentModule.java | 1 + 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/api/src/org/labkey/api/assay/AbstractAssayTsvDataHandler.java b/api/src/org/labkey/api/assay/AbstractAssayTsvDataHandler.java index b08d253129a..96de697ad08 100644 --- a/api/src/org/labkey/api/assay/AbstractAssayTsvDataHandler.java +++ b/api/src/org/labkey/api/assay/AbstractAssayTsvDataHandler.java @@ -63,6 +63,7 @@ import org.labkey.api.reader.DataLoader; import org.labkey.api.reader.TabLoader; import org.labkey.api.security.User; +import org.labkey.api.settings.ExperimentalFeatureService; import org.labkey.api.study.ParticipantVisit; import org.labkey.api.study.Study; import org.labkey.api.study.StudyService; @@ -648,7 +649,11 @@ else if (getRawPlateMetadata() != null) throw new ExperimentException("No PlateMetadataService registered for data type : " + plateData.getDataType().toString()); } else - throw new ExperimentException("Unable to locate the ExpData with the plate metadata"); + { + // plate metadata is optional if the experimental plate flag is enabled + if (!ExperimentalFeatureService.get().isFeatureEnabled("experimental-app-plate-support")) + throw new ExperimentException("Unable to locate the ExpData with the plate metadata"); + } } protected ParticipantVisitResolver createResolver(User user, ExpRun run, ExpProtocol protocol, AssayProvider provider, Container container) diff --git a/experiment/src/org/labkey/experiment/ExperimentModule.java b/experiment/src/org/labkey/experiment/ExperimentModule.java index 0b862aa64bc..7e2bb234742 100644 --- a/experiment/src/org/labkey/experiment/ExperimentModule.java +++ b/experiment/src/org/labkey/experiment/ExperimentModule.java @@ -583,6 +583,7 @@ public void containerDeleted(Container c, User user) assayMetrics.put(assayProvider.getName(), protocolMetrics); } assayMetrics.put("autoLinkedAssayCount", new SqlSelector(ExperimentService.get().getSchema(), "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.propertyuri = 'terms.labkey.org#AutoCopyTargetContainer'").getObject(Long.class)); + assayMetrics.put("standardAssayWithPlateSupportCount", new SqlSelector(ExperimentService.get().getSchema(), "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.name = 'PlateMetadata' AND floatValue = 1").getObject(Long.class)); Map sampleLookupCountMetrics = new HashMap<>(); SQLFragment baseAssaySampleLookupSQL = new SQLFragment("SELECT COUNT(*) FROM exp.propertydescriptor WHERE (lookupschema = 'samples' OR (lookupschema = 'exp' AND lookupquery = 'Materials')) AND propertyuri LIKE ?"); From 661ac6495e76891a62f4584bc2127d764a2f8511 Mon Sep 17 00:00:00 2001 From: Lum Date: Thu, 10 Aug 2023 16:41:52 -0700 Subject: [PATCH 04/15] support for transform scripts --- .../assay/AbstractAssayTsvDataHandler.java | 24 +++++--- .../plate/AssayPlateMetadataService.java | 8 ++- .../labkey/api/qc/TsvDataExchangeHandler.java | 23 ++++---- .../plate/AssayPlateMetadataServiceImpl.java | 56 ++++++++++++++++--- .../org/labkey/assay/plate/PlateManager.java | 3 +- 5 files changed, 83 insertions(+), 31 deletions(-) diff --git a/api/src/org/labkey/api/assay/AbstractAssayTsvDataHandler.java b/api/src/org/labkey/api/assay/AbstractAssayTsvDataHandler.java index 96de697ad08..ae8b6ef9030 100644 --- a/api/src/org/labkey/api/assay/AbstractAssayTsvDataHandler.java +++ b/api/src/org/labkey/api/assay/AbstractAssayTsvDataHandler.java @@ -63,7 +63,6 @@ import org.labkey.api.reader.DataLoader; import org.labkey.api.reader.TabLoader; import org.labkey.api.security.User; -import org.labkey.api.settings.ExperimentalFeatureService; import org.labkey.api.study.ParticipantVisit; import org.labkey.api.study.Study; import org.labkey.api.study.StudyService; @@ -179,11 +178,15 @@ public Map>> getValidationDataMap(ExpData dat AssayProvider provider = AssayService.get().getProvider(protocol); boolean plateMetadataEnabled = provider.isPlateMetadataEnabled(protocol); File plateMetadataFile = null; + Map rawPlateMetadata = null; if (plateMetadataEnabled) { if (context instanceof AssayUploadXarContext assayContext) { + // the plate metadata will either be uploaded as a file or in a raw, already parsed form depending on + // how the run is imported. + rawPlateMetadata = assayContext.getContext().getRawPlateMetadata(); plateMetadataFile = (File)assayContext.getContext().getUploadedData().get(AssayDataCollector.PLATE_METADATA_FILE); if (plateMetadataFile != null) { @@ -208,21 +211,28 @@ public Map>> getValidationDataMap(ExpData dat // assays with plate metadata support will merge the plate metadata with the data rows to make it easier for // transform scripts to perform metadata related calculations - if (plateMetadataEnabled && plateMetadataFile != null) + AssayPlateMetadataService svc = AssayPlateMetadataService.getService(PlateMetadataDataHandler.DATA_TYPE); + if (svc != null) { - AssayPlateMetadataService svc = AssayPlateMetadataService.getService(PlateMetadataDataHandler.DATA_TYPE); - if (svc != null) + if (plateMetadataEnabled) { - Map plateMetadata = svc.parsePlateMetadata(plateMetadataFile); + Map plateMetadata = null; + if (plateMetadataFile != null || rawPlateMetadata != null) + { + plateMetadata = plateMetadataFile != null + ? svc.parsePlateMetadata(plateMetadataFile) + : rawPlateMetadata; + } Domain runDomain = provider.getRunDomain(protocol); DomainProperty property = runDomain.getPropertyByName(AssayPlateMetadataService.PLATE_TEMPLATE_COLUMN_NAME); if (property != null) { Object lsid = ((AssayUploadXarContext)context).getContext().getRunProperties().get(property); - dataRows = svc.mergePlateMetadata(Lsid.parse(String.valueOf(lsid)), dataRows, plateMetadata, protocol); + dataRows = svc.mergePlateMetadata(context.getContainer(), context.getUser(), Lsid.parse(String.valueOf(lsid)), dataRows, plateMetadata, protocol); } } } + datas.put(getDataType(), dataRows); return datas; } @@ -651,7 +661,7 @@ else if (getRawPlateMetadata() != null) else { // plate metadata is optional if the experimental plate flag is enabled - if (!ExperimentalFeatureService.get().isFeatureEnabled("experimental-app-plate-support")) + if (!AssayPlateMetadataService.isExperimentalAppPlateEnabled()) throw new ExperimentException("Unable to locate the ExpData with the plate metadata"); } } diff --git a/api/src/org/labkey/api/assay/plate/AssayPlateMetadataService.java b/api/src/org/labkey/api/assay/plate/AssayPlateMetadataService.java index 14f480a279b..cdf1d2b1472 100644 --- a/api/src/org/labkey/api/assay/plate/AssayPlateMetadataService.java +++ b/api/src/org/labkey/api/assay/plate/AssayPlateMetadataService.java @@ -13,6 +13,7 @@ import org.labkey.api.exp.api.ExpRun; import org.labkey.api.exp.property.Domain; import org.labkey.api.security.User; +import org.labkey.api.settings.ExperimentalFeatureService; import java.io.File; import java.util.HashMap; @@ -34,6 +35,11 @@ static void registerService(AssayDataType dataType, AssayPlateMetadataService ha throw new RuntimeException("The specified assay data type is null"); } + static boolean isExperimentalAppPlateEnabled() + { + return ExperimentalFeatureService.get().isFeatureEnabled("experimental-app-plate-support"); + } + @Nullable static AssayPlateMetadataService getService(AssayDataType dataType) { @@ -64,7 +70,7 @@ void addAssayPlateMetadata(ExpData resultData, Map plateM * * @return the merged rows */ - List> mergePlateMetadata(Lsid plateLsid, List> rows, Map plateMetadata, + List> mergePlateMetadata(Container container, User user, Lsid plateLsid, List> rows, @Nullable Map plateMetadata, ExpProtocol protocol) throws ExperimentException; /** diff --git a/api/src/org/labkey/api/qc/TsvDataExchangeHandler.java b/api/src/org/labkey/api/qc/TsvDataExchangeHandler.java index d8fbd466406..0f912cdfad3 100644 --- a/api/src/org/labkey/api/qc/TsvDataExchangeHandler.java +++ b/api/src/org/labkey/api/qc/TsvDataExchangeHandler.java @@ -306,21 +306,18 @@ protected Set _writeRunData(AssayRunUploadContext context, ExpRun run, Fil File runData = new File(scriptDir, RUN_DATA_FILE); result.add(runData); - if (rawPlateMetadata != null) + AssayPlateMetadataService svc = AssayPlateMetadataService.getService(PlateMetadataDataHandler.DATA_TYPE); + if (svc != null) { - AssayPlateMetadataService svc = AssayPlateMetadataService.getService(PlateMetadataDataHandler.DATA_TYPE); - if (svc != null) - { - ExpProtocol protocol = run.getProtocol(); - AssayProvider provider = AssayService.get().getProvider(protocol); + ExpProtocol protocol = run.getProtocol(); + AssayProvider provider = AssayService.get().getProvider(protocol); - Domain runDomain = provider.getRunDomain(protocol); - DomainProperty property = runDomain.getPropertyByName(AssayPlateMetadataService.PLATE_TEMPLATE_COLUMN_NAME); - if (property != null) - { - Object lsid = context.getRunProperties().get(property); - rawData = svc.mergePlateMetadata(Lsid.parse(String.valueOf(lsid)), rawData, rawPlateMetadata, protocol); - } + Domain runDomain = provider.getRunDomain(protocol); + DomainProperty property = runDomain.getPropertyByName(AssayPlateMetadataService.PLATE_TEMPLATE_COLUMN_NAME); + if (property != null) + { + Object lsid = context.getRunProperties().get(property); + rawData = svc.mergePlateMetadata(context.getContainer(), context.getUser(), Lsid.parse(String.valueOf(lsid)), rawData, rawPlateMetadata, protocol); } } addToMergedMap(mergedDataMap, Map.of(dataType, rawData)); diff --git a/assay/src/org/labkey/assay/plate/AssayPlateMetadataServiceImpl.java b/assay/src/org/labkey/assay/plate/AssayPlateMetadataServiceImpl.java index 14b75eb8786..c5c14c5dbd3 100644 --- a/assay/src/org/labkey/assay/plate/AssayPlateMetadataServiceImpl.java +++ b/assay/src/org/labkey/assay/plate/AssayPlateMetadataServiceImpl.java @@ -43,6 +43,7 @@ import org.labkey.api.security.User; import org.labkey.assay.TSVProtocolSchema; import org.labkey.assay.plate.model.Well; +import org.labkey.assay.plate.model.WellCustomField; import org.labkey.assay.query.AssayDbSchema; import java.io.File; @@ -242,31 +243,70 @@ private Map> prepareMergedPlateData( @Override public List> mergePlateMetadata( + Container container, + User user, Lsid plateLsid, List> rows, - Map plateMetadata, + @Nullable Map plateMetadata, ExpProtocol protocol) throws ExperimentException { - Map> plateData = prepareMergedPlateData(plateLsid, plateMetadata, null, protocol, false); List> mergedRows = new ArrayList<>(); for (Map row : rows) { Map newRow = new CaseInsensitiveLinkedHashMap<>(row); // ensure the result data includes a wellLocation field with values like : A1, F12, etc - if (newRow.containsKey(AssayResultDomainKind.WELL_LOCATION_COLUMN_NAME)) + if (!newRow.containsKey(AssayResultDomainKind.WELL_LOCATION_COLUMN_NAME)) + throw new ExperimentException("Imported data must contain a WellLocation column to support plate metadata integration"); + mergedRows.add(newRow); + } + + if (plateMetadata != null) + { + Map> plateData = prepareMergedPlateData(plateLsid, plateMetadata, null, protocol, false); + for (Map row : mergedRows) { - PositionImpl well = new PositionImpl(null, String.valueOf(newRow.get(AssayResultDomainKind.WELL_LOCATION_COLUMN_NAME))); + PositionImpl well = new PositionImpl(null, String.valueOf(row.get(AssayResultDomainKind.WELL_LOCATION_COLUMN_NAME))); // need to adjust the column value to be 0 based to match the template locations well.setColumn(well.getColumn()-1); if (plateData.containsKey(well)) + row.putAll(plateData.get(well)); + } + } + + if (AssayPlateMetadataService.isExperimentalAppPlateEnabled()) + { + // include metadata that may have been applied directly to the plate + Plate plate = PlateService.get().getPlate(protocol.getContainer(), plateLsid); + if (plate == null) + throw new ExperimentException("Unable to resolve the plate for the run"); + + // if there are metadata fields configured for this plate + if (!PlateManager.get().getFields(container, user, plate.getRowId()).isEmpty()) + { + // create the map of well locations to the well + Map positionToWell = new HashMap<>(); + + SimpleFilter filter = SimpleFilter.createContainerFilter(plate.getContainer()); + filter.addCondition(FieldKey.fromParts("PlateId"), plate.getRowId()); + for (Well well : new TableSelector(AssayDbSchema.getInstance().getTableInfoWell(), filter, null).getArrayList(Well.class)) + positionToWell.put(new PositionImpl(plate.getContainer(), well.getRow(), well.getCol()), well); + + for (Map row : mergedRows) { - newRow.putAll(plateData.get(well)); + PositionImpl well = new PositionImpl(null, String.valueOf(row.get(AssayResultDomainKind.WELL_LOCATION_COLUMN_NAME))); + // need to adjust the column value to be 0 based to match the template locations + well.setColumn(well.getColumn()-1); + + if (positionToWell.containsKey(well)) + { + for (WellCustomField customField : PlateManager.get().getWellCustomFields(container, user, plate.getRowId(), positionToWell.get(well).getRowId())) + { + row.put(customField.getName(), customField.getValue()); + } + } } - mergedRows.add(newRow); } - else - throw new ExperimentException("Imported data must contain a WellLocation column to support plate metadata integration"); } return mergedRows; } diff --git a/assay/src/org/labkey/assay/plate/PlateManager.java b/assay/src/org/labkey/assay/plate/PlateManager.java index 5138f8e05bc..883c44889d6 100644 --- a/assay/src/org/labkey/assay/plate/PlateManager.java +++ b/assay/src/org/labkey/assay/plate/PlateManager.java @@ -483,7 +483,6 @@ private WellImpl[] getWells(Plate plate) SimpleFilter plateFilter = new SimpleFilter(FieldKey.fromParts("PlateId"), plate.getRowId()); Sort sort = new Sort("Col,Row"); return new TableSelector(AssayDbSchema.getInstance().getTableInfoWell(), plateFilter, sort).getArray(WellImpl.class); - } private WellGroupImpl[] getWellGroups(Plate plate) @@ -1417,7 +1416,7 @@ private List _getFields(Container container, User user, Integer return fields; } - private List getWellCustomFields(Container container, User user, Integer plateId, Integer wellId) + public List getWellCustomFields(Container container, User user, Integer plateId, Integer wellId) { List fields = _getFields(container, user, plateId).stream().map(WellCustomField::new).toList(); From a35a6fd0b973336e0746a89cd2c4814c26b011e6 Mon Sep 17 00:00:00 2001 From: Lum Date: Mon, 14 Aug 2023 16:53:44 -0700 Subject: [PATCH 05/15] change the plate cache granularity to per-plate --- .../org/labkey/assay/plate/PlateCache.java | 228 +++++++++++++----- .../org/labkey/assay/plate/PlateManager.java | 16 +- .../labkey/assay/plate/query/PlateTable.java | 6 +- 3 files changed, 186 insertions(+), 64 deletions(-) diff --git a/assay/src/org/labkey/assay/plate/PlateCache.java b/assay/src/org/labkey/assay/plate/PlateCache.java index d96b69464b4..38d78480a20 100644 --- a/assay/src/org/labkey/assay/plate/PlateCache.java +++ b/assay/src/org/labkey/assay/plate/PlateCache.java @@ -4,121 +4,237 @@ import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import org.json.JSONObject; import org.labkey.api.assay.plate.Plate; import org.labkey.api.cache.Cache; +import org.labkey.api.cache.CacheLoader; import org.labkey.api.cache.CacheManager; import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerManager; import org.labkey.api.data.SimpleFilter; import org.labkey.api.data.TableSelector; import org.labkey.api.exp.Lsid; +import org.labkey.api.query.FieldKey; import org.labkey.assay.query.AssayDbSchema; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; public class PlateCache { - private static final Cache PLATE_COLLECTIONS_CACHE = CacheManager.getBlockingCache(CacheManager.UNLIMITED, CacheManager.DAY, "Plate Cache", (c, argument) -> new PlateCollections(c)); + private static final PlateLoader _loader = new PlateLoader(); + private static final Cache PLATE_CACHE = CacheManager.getBlockingStringKeyCache(CacheManager.UNLIMITED, CacheManager.DAY, "Plate Cache", _loader); private static final Logger LOG = LogManager.getLogger(PlateCache.class); - private static class PlateCollections + private static class PlateLoader implements CacheLoader { - private final Map _rowIdMap; - private final Map _nameMap; - private final Map _lsidMap; - private final List _templates; + private Map> _rowIdMap = new HashMap<>(); + private Map> _nameMap = new HashMap<>(); + private Map> _lsidMap = new HashMap<>(); - private PlateCollections(Container c) + @Override + public Plate load(@NotNull String key, @Nullable Object argument) { - Map rowIdMap = new HashMap<>(); - Map nameMap = new HashMap<>(); - Map lsidMap = new HashMap<>(); - List templates = new ArrayList<>(); + // parse the cache key + PlateCacheKey cacheKey = new PlateCacheKey(key); - new TableSelector(AssayDbSchema.getInstance().getTableInfoPlate(), SimpleFilter.createContainerFilter(c), null).forEach(PlateImpl.class, plate -> { - PlateManager.get().populatePlate(plate); + // check our internal caches + Plate cachedPlate = getFromCollections(cacheKey); + if (cachedPlate != null) + return cachedPlate; - rowIdMap.put(plate.getRowId(), plate); - if (nameMap.containsKey(plate.getName())) - { - LOG.error(String.format("A duplicate Plate name : %s was found in the same folder : %s. We recommend that the duplicate plate(s) are deleted.", plate.getName(), c.getPath())); - } - nameMap.put(plate.getName(), plate); - lsidMap.put(Lsid.parse(plate.getLSID()), plate); - if (plate.isTemplate()) - templates.add(plate); - }); - - templates.sort(Comparator.comparing(Plate::getName)); - - _templates = templates; - _rowIdMap = rowIdMap; - _nameMap = nameMap; - _lsidMap = lsidMap; - } + SimpleFilter filter = SimpleFilter.createContainerFilter(cacheKey._container); + filter.addCondition(FieldKey.fromParts(cacheKey._type.name()), cacheKey._identifier); - private @Nullable Plate getForRowId(int rowId) - { - return _rowIdMap.get(rowId); - } + List plates = new TableSelector(AssayDbSchema.getInstance().getTableInfoPlate(), filter, null).getArrayList(PlateImpl.class); + assert plates.size() <= 1; - private @Nullable Plate getForName(String name) - { - return _nameMap.get(name); + if (plates.size() == 1) + { + PlateImpl plate = plates.get(0); + PlateManager.get().populatePlate(plate); + + // add to internal collections + addToCollections(cacheKey, plate); + return plate; + } + return null; } - private @Nullable Plate getForLsid(Lsid lsid) + @Nullable + private Plate getFromCollections(PlateCacheKey cacheKey) { - return _lsidMap.get(lsid); + Map> map = switch (cacheKey._type) + { + case lsid -> _lsidMap; + case name -> _nameMap; + case rowId -> _rowIdMap; + }; + + if (map.containsKey(cacheKey._container)) + return map.get(cacheKey._container).get(cacheKey._identifier); + return null; } - private @NotNull Collection getPlates() + private void addToCollections(PlateCacheKey cacheKey, Plate plate) { - return _rowIdMap.values(); + if (plate != null) + { + if (plate.getName() == null) + throw new IllegalArgumentException("Plate cannot be cached, name is null"); + if (plate.getRowId() == null) + throw new IllegalArgumentException("Plate cannot be cached, rowId is null"); + if (plate.getLSID() == null) + throw new IllegalArgumentException("Plate cannot be cached, LSID is null"); + + _lsidMap.computeIfAbsent(cacheKey._container, k -> new HashMap<>()).put(plate.getLSID(), plate); + _nameMap.computeIfAbsent(cacheKey._container, k -> new HashMap<>()).put(plate.getName(), plate); + _rowIdMap.computeIfAbsent(cacheKey._container, k -> new HashMap<>()).put(plate.getRowId(), plate); + } } - private @NotNull List getPlateTemplates() + private void removeFromCollections(Container c, Plate plate) { - return _templates; + if (plate != null) + { + if (plate.getName() == null) + throw new IllegalArgumentException("Plate cannot be uncached, name is null"); + if (plate.getRowId() == null) + throw new IllegalArgumentException("Plate cannot be uncached, rowId is null"); + if (plate.getLSID() == null) + throw new IllegalArgumentException("Plate cannot be uncached, LSID is null"); + + if (_lsidMap.containsKey(c)) _lsidMap.get(c).remove(plate.getLSID()); + if (_nameMap.containsKey(c)) _nameMap.get(c).remove(plate.getName()); + if (_rowIdMap.containsKey(c)) _rowIdMap.get(c).remove(plate.getRowId()); + } } } - static @Nullable Plate getPlate(Container c, int rowId) + public static @Nullable Plate getPlate(Container c, int rowId) { - return PLATE_COLLECTIONS_CACHE.get(c).getForRowId(rowId); + return PLATE_CACHE.get(PlateCacheKey.getCacheKey(c, rowId)); } - static @Nullable Plate getPlate(Container c, String name) + public static @Nullable Plate getPlate(Container c, String name) { - return PLATE_COLLECTIONS_CACHE.get(c).getForName(name); + return PLATE_CACHE.get(PlateCacheKey.getCacheKey(c, name)); } - static @Nullable Plate getPlate(Container c, Lsid lsid) + public static @Nullable Plate getPlate(Container c, Lsid lsid) { - return PLATE_COLLECTIONS_CACHE.get(c).getForLsid(lsid); + return PLATE_CACHE.get(PlateCacheKey.getCacheKey(c, lsid)); } - static @NotNull Collection getPlates(Container c) + public static @NotNull Collection getPlates(Container c) { - return PLATE_COLLECTIONS_CACHE.get(c).getPlates(); + List plates = new ArrayList<>(); + List ids = new TableSelector(AssayDbSchema.getInstance().getTableInfoPlate(), + Collections.singleton("RowId"), + SimpleFilter.createContainerFilter(c), null).getArrayList(Integer.class); + for (Integer id : ids) + { + plates.add(PLATE_CACHE.get(PlateCacheKey.getCacheKey(c, id))); + } + return plates; } - static @NotNull List getPlateTemplates(Container c) + public static @NotNull List getPlateTemplates(Container c) { - return PLATE_COLLECTIONS_CACHE.get(c).getPlateTemplates(); + List templates = new ArrayList<>(); + for (Plate plate : getPlates(c)) + { + if (plate.isTemplate()) + templates.add(plate); + } + return templates.stream() + .sorted(Comparator.comparing(Plate::getName)) + .collect(Collectors.toList()); } public static void uncache(Container c) { - PLATE_COLLECTIONS_CACHE.remove(c); + // uncache all plates for this container + if (_loader._rowIdMap.containsKey(c)) + { + for (Plate plate : _loader._rowIdMap.get(c).values()) + { + uncache(c, plate); + } + } + } + + public static void uncache(Container c, Plate plate) + { + if (plate.getName() == null) + throw new IllegalArgumentException("Plate cannot be uncached, name is null"); + if (plate.getRowId() == null) + throw new IllegalArgumentException("Plate cannot be uncached, rowId is null"); + if (plate.getLSID() == null) + throw new IllegalArgumentException("Plate cannot be uncached, LSID is null"); + + PLATE_CACHE.remove(PlateCacheKey.getCacheKey(c, plate.getName())); + PLATE_CACHE.remove(PlateCacheKey.getCacheKey(c, plate.getRowId())); + PLATE_CACHE.remove(PlateCacheKey.getCacheKey(c, Lsid.parse(plate.getLSID()))); + + _loader.removeFromCollections(c, plate); } public static void clearCache() { - PLATE_COLLECTIONS_CACHE.clear(); + PLATE_CACHE.clear(); + } + + private static class PlateCacheKey + { + enum Type + { + rowId, + name, + lsid, + } + private Type _type; + private Container _container; + private Object _identifier; + + PlateCacheKey(String key) + { + JSONObject json = new JSONObject(key); + + _type = json.getEnum(Type.class, "type"); + _container = ContainerManager.getForId(json.getString("container")); + _identifier = json.get("identifier"); + } + + public static String getCacheKey(Container c, String name) + { + return _getCacheKey(c, Type.name, name); + } + + public static String getCacheKey(Container c, Integer plateId) + { + return _getCacheKey(c, Type.rowId, plateId); + } + + public static String getCacheKey(Container c, Lsid lsid) + { + return _getCacheKey(c, Type.lsid, lsid.toString()); + } + + private static String _getCacheKey(Container c, Type type, Object identifier) + { + JSONObject json = new JSONObject(); + json.put("container", c.getId()); + json.put("type", type.name()); + json.put("identifier", identifier); + + return json.toString(); + } } } diff --git a/assay/src/org/labkey/assay/plate/PlateManager.java b/assay/src/org/labkey/assay/plate/PlateManager.java index 883c44889d6..d9d9fe0c7b7 100644 --- a/assay/src/org/labkey/assay/plate/PlateManager.java +++ b/assay/src/org/labkey/assay/plate/PlateManager.java @@ -672,7 +672,10 @@ private int savePlateImpl(Container container, User user, PlateImpl plate) throw final Integer plateRowId = plateId; transaction.addCommitTask(() -> { - clearCache(container); + if (updateExisting) + { + clearCache(container, plate); + } indexPlate(container, plateRowId); }, DbScope.CommitTaskOption.POSTCOMMIT); transaction.commit(); @@ -753,10 +756,10 @@ public void deletePlate(Container container, User user, int rowid) throws Except } // Called by the Plate Query Update Service after deleting a plate - public void afterPlateDelete(Container container, Lsid plateLsid) + public void afterPlateDelete(Container container, Plate plate) { - clearCache(container); - deindexPlates(List.of(plateLsid)); + clearCache(container, plate); + deindexPlates(List.of(Lsid.parse(plate.getLSID()))); } // Called by the Plate Query Update Service prior to deleting a plate @@ -1065,6 +1068,11 @@ public void registerPlateTypeHandler(PlateTypeHandler handler) _plateTypeHandlers.put(handler.getAssayType(), handler); } + public void clearCache(Container c, Plate plate) + { + PlateCache.uncache(c, plate); + } + public void clearCache(Container c) { PlateCache.uncache(c); diff --git a/assay/src/org/labkey/assay/plate/query/PlateTable.java b/assay/src/org/labkey/assay/plate/query/PlateTable.java index 59c59cadefd..96bba362da0 100644 --- a/assay/src/org/labkey/assay/plate/query/PlateTable.java +++ b/assay/src/org/labkey/assay/plate/query/PlateTable.java @@ -223,7 +223,6 @@ public DataIteratorBuilder createImportDIB(User user, Container container, DataI public List> insertRows(User user, Container container, List> rows, BatchValidationException errors, @Nullable Map configParameters, Map extraScriptContext) { List> results = super._insertRowsUsingDIB(user, container, rows, getDataIteratorContext(errors, InsertOption.INSERT, configParameters), extraScriptContext); - PlateManager.get().clearCache(container); return results; } @@ -249,7 +248,7 @@ protected Map updateRow(User user, Container container, Map newRow = super.updateRow(user, container, row, oldRow); - PlateManager.get().clearCache(container); + PlateManager.get().clearCache(container, plate); return newRow; } @@ -267,11 +266,10 @@ protected Map deleteRow(User user, Container container, Map returnMap = super.deleteRow(user, container, oldRowMap); - transaction.addCommitTask(() -> PlateManager.get().afterPlateDelete(container, plateLsid), DbScope.CommitTaskOption.POSTCOMMIT); + transaction.addCommitTask(() -> PlateManager.get().afterPlateDelete(container, plate), DbScope.CommitTaskOption.POSTCOMMIT); transaction.commit(); return returnMap; From 2b4cee898254399f2dde51612c95f4e71f36979e Mon Sep 17 00:00:00 2001 From: Lum Date: Wed, 16 Aug 2023 11:09:53 -0700 Subject: [PATCH 06/15] use the plate cache more often --- .../org/labkey/api/assay/plate/Plate.java | 2 + .../labkey/api/assay/plate/PositionImpl.java | 5 -- .../assay/plate/PlateDocumentProvider.java | 2 +- .../src/org/labkey/assay/plate/PlateImpl.java | 7 +-- .../org/labkey/assay/plate/PlateManager.java | 48 ++++++++----------- .../org/labkey/assay/plate/WellGroupImpl.java | 5 +- .../assay/plate/query/WellGroupTable.java | 20 ++++++-- 7 files changed, 46 insertions(+), 43 deletions(-) diff --git a/assay/api-src/org/labkey/api/assay/plate/Plate.java b/assay/api-src/org/labkey/api/assay/plate/Plate.java index cf0cb83302d..c9da75ebdca 100644 --- a/assay/api-src/org/labkey/api/assay/plate/Plate.java +++ b/assay/api-src/org/labkey/api/assay/plate/Plate.java @@ -43,6 +43,8 @@ public interface Plate extends PropertySet, Identifiable Well getWell(int rowId); + List getWells(); + WellGroup getWellGroup(WellGroup.Type type, String wellGroupName); @Nullable WellGroup getWellGroup(int rowId); diff --git a/assay/api-src/org/labkey/api/assay/plate/PositionImpl.java b/assay/api-src/org/labkey/api/assay/plate/PositionImpl.java index 9734ec43b53..92ee8cc982a 100644 --- a/assay/api-src/org/labkey/api/assay/plate/PositionImpl.java +++ b/assay/api-src/org/labkey/api/assay/plate/PositionImpl.java @@ -143,11 +143,6 @@ public void setCol(int column) setColumn(column); } - public int getCol() - { - return getColumn(); - } - public Integer getPlateId() { return _plateId; diff --git a/assay/src/org/labkey/assay/plate/PlateDocumentProvider.java b/assay/src/org/labkey/assay/plate/PlateDocumentProvider.java index e3cc55513d6..8dfa7381403 100644 --- a/assay/src/org/labkey/assay/plate/PlateDocumentProvider.java +++ b/assay/src/org/labkey/assay/plate/PlateDocumentProvider.java @@ -109,7 +109,7 @@ private Lsid fromDocumentId(@NotNull String resourceIdentifier) private @Nullable Plate getPlate(@NotNull String resourceIdentifier) { Lsid lsid = fromDocumentId(resourceIdentifier); - return PlateManager.get().getPlate(lsid.toString()); + return PlateManager.get().getPlate(lsid); } @Override diff --git a/assay/src/org/labkey/assay/plate/PlateImpl.java b/assay/src/org/labkey/assay/plate/PlateImpl.java index b99287a6c65..2a1ade7d9d3 100644 --- a/assay/src/org/labkey/assay/plate/PlateImpl.java +++ b/assay/src/org/labkey/assay/plate/PlateImpl.java @@ -61,7 +61,7 @@ public class PlateImpl extends PropertySetImpl implements Plate private List _deletedGroups; private WellImpl[][] _wells; - private Map _wellMap; + private Map _wellMap; private int _runId; // NO_RUNID means no run yet, well data comes from file, dilution data must be calculated private int _plateNumber; @@ -523,9 +523,10 @@ public void setWells(WellImpl[][] wells) } @JsonIgnore - public WellImpl[][] getWells() + @Override + public List getWells() { - return _wells; + return _wellMap.values().stream().toList(); } @Override diff --git a/assay/src/org/labkey/assay/plate/PlateManager.java b/assay/src/org/labkey/assay/plate/PlateManager.java index e257991071d..f940d6cc6c5 100644 --- a/assay/src/org/labkey/assay/plate/PlateManager.java +++ b/assay/src/org/labkey/assay/plate/PlateManager.java @@ -35,6 +35,7 @@ import org.labkey.api.assay.plate.WellGroup; import org.labkey.api.collections.ArrayListMap; import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerManager; import org.labkey.api.data.DbScope; import org.labkey.api.data.ImportAliasable; import org.labkey.api.data.ObjectFactory; @@ -336,16 +337,17 @@ public Plate createPlateTemplate(Container container, String templateType, int r return PlateCache.getPlate(container, lsid); } - /** - * Note that this does not use the cache. - */ - public @Nullable Plate getPlate(String lsid) + public @Nullable Plate getPlate(Lsid lsid) { SimpleFilter filter = new SimpleFilter(FieldKey.fromParts("lsid"), lsid); - PlateImpl plate = new TableSelector(AssayDbSchema.getInstance().getTableInfoPlate(), filter, null).getObject(PlateImpl.class); - populatePlate(plate); - - return plate; + String container = new TableSelector(AssayDbSchema.getInstance().getTableInfoPlate(), Collections.singleton("Container"), filter, null).getObject(String.class); + if (container != null) + { + Container c = ContainerManager.getForId(container); + if (c != null) + return PlateCache.getPlate(c, lsid); + } + return null; } /** @@ -353,10 +355,7 @@ public Plate createPlateTemplate(Container container, String templateType, int r */ public boolean plateExists(Container c, String name) { - SimpleFilter filter = SimpleFilter.createContainerFilter(c); - filter.addCondition(FieldKey.fromParts("Name"), name); - - return new TableSelector(AssayDbSchema.getInstance().getTableInfoPlate(), filter, null).getRowCount() > 0; + return PlateCache.getPlate(c, name) != null; } private Collection getPlates(Container c) @@ -396,7 +395,6 @@ public WellGroup getWellGroup(String lsid) return null; } - private void setProperties(Container container, PropertySetImpl propertySet) { Map props = OntologyManager.getPropertyObjects(container, propertySet.getLSID()); @@ -609,9 +607,9 @@ private int savePlateImpl(Container container, User user, PlateImpl plate) throw Map, PositionImpl> existingPositionMap = new HashMap<>(); if (updateExisting) { - for (PositionImpl existingPosition : getWells(plate)) + for (Well existingPosition : plate.getWells()) { - existingPositionMap.put(Pair.of(existingPosition.getRow(), existingPosition.getCol()), existingPosition); + existingPositionMap.put(Pair.of(existingPosition.getRow(), existingPosition.getColumn()), (PositionImpl) existingPosition); } } else @@ -767,18 +765,12 @@ public void beforePlateDelete(Container container, Integer plateId) { final AssayDbSchema schema = AssayDbSchema.getInstance(); - SimpleFilter plateFilter = SimpleFilter.createContainerFilter(container); - plateFilter.addCondition(FieldKey.fromParts("RowId"), plateId); - PlateImpl plate = new TableSelector(schema.getTableInfoPlate(), - plateFilter, null).getObject(PlateImpl.class); - WellGroupImpl[] wellgroups = getWellGroups(plate); - WellImpl[] wells = getWells(plate); - + Plate plate = PlateCache.getPlate(container, plateId); List lsids = new ArrayList<>(); lsids.add(plate.getLSID()); - for (WellGroupImpl wellgroup : wellgroups) + for (WellGroup wellgroup : plate.getWellGroups()) lsids.add(wellgroup.getLSID()); - for (WellImpl well : wells) + for (Well well : plate.getWells()) lsids.add(well.getLsid()); SimpleFilter plateIdFilter = SimpleFilter.createContainerFilter(container); @@ -960,7 +952,7 @@ public Plate getObject(Lsid lsid) if (lsid == null) return null; - return PlateManager.get().getPlate(lsid.toString()); + return PlateManager.get().getPlate(lsid); } @Override @@ -1038,10 +1030,10 @@ public void registerLsidHandlers() public Plate copyPlate(Plate source, User user, Container destContainer) throws Exception { - Plate destination = PlateService.get().getPlate(destContainer, source.getName()); + Plate destination = getPlate(destContainer, source.getName()); if (destination != null) throw new PlateService.NameConflictException(source.getName()); - destination = PlateService.get().createPlateTemplate(destContainer, source.getType(), source.getRows(), source.getColumns()); + destination = createPlateTemplate(destContainer, source.getType(), source.getRows(), source.getColumns()); destination.setName(source.getName()); for (String property : source.getPropertyNames()) destination.setProperty(property, source.getProperty(property)); @@ -1055,7 +1047,7 @@ public Plate copyPlate(Plate source, User user, Container destContainer) copyGroup.setProperty(property, originalGroup.getProperty(property)); } save(destContainer, user, destination); - return this.getPlate(destContainer, destination.getName()); + return getPlate(destContainer, destination.getName()); } @Override diff --git a/assay/src/org/labkey/assay/plate/WellGroupImpl.java b/assay/src/org/labkey/assay/plate/WellGroupImpl.java index 0a48dd6656f..d4b195cf4a3 100644 --- a/assay/src/org/labkey/assay/plate/WellGroupImpl.java +++ b/assay/src/org/labkey/assay/plate/WellGroupImpl.java @@ -97,7 +97,6 @@ public WellGroupImpl(PlateImpl plate, WellGroupImpl template) return template.detailsURL(); } - private static List sortPositions(List positions) { List sortedPositions = new ArrayList<>(positions); @@ -312,6 +311,10 @@ public double getMean() @Override public Plate getPlate() { + if (_plate == null && _plateId != null) + { + _plate = (PlateImpl) PlateCache.getPlate(getContainer(), _plateId); + } return _plate; } diff --git a/assay/src/org/labkey/assay/plate/query/WellGroupTable.java b/assay/src/org/labkey/assay/plate/query/WellGroupTable.java index ad7cc853035..478865efd8c 100644 --- a/assay/src/org/labkey/assay/plate/query/WellGroupTable.java +++ b/assay/src/org/labkey/assay/plate/query/WellGroupTable.java @@ -17,6 +17,7 @@ package org.labkey.assay.plate.query; import org.jetbrains.annotations.Nullable; +import org.labkey.api.assay.plate.Plate; import org.labkey.api.assay.plate.WellGroup; import org.labkey.api.collections.CaseInsensitiveHashSet; import org.labkey.api.data.ColumnInfo; @@ -50,6 +51,7 @@ import org.labkey.api.query.SimpleUserSchema; import org.labkey.api.query.UserSchema; import org.labkey.api.security.User; +import org.labkey.assay.plate.PlateCache; import org.labkey.assay.plate.PlateManager; import org.labkey.assay.query.AssayDbSchema; @@ -181,11 +183,19 @@ protected Map deleteRow(User user, Container container, Map returnMap = super.deleteRow(user, container, oldRowMap); - transaction.commit(); - - return returnMap; + WellGroup wellGroup = PlateManager.get().getWellGroup(container, wellGroupId); + if (wellGroup != null) + { + Plate plate = wellGroup.getPlate(); + PlateManager.get().beforeDeleteWellGroup(container, wellGroupId); + Map returnMap = super.deleteRow(user, container, oldRowMap); + + if (plate != null) + transaction.addCommitTask(() -> PlateCache.uncache(container, plate), DbScope.CommitTaskOption.POSTCOMMIT); + transaction.commit(); + + return returnMap; + } } return Collections.emptyMap(); } From 9c6aa8e36ec46434ed22f58bf36a79354fa1b971 Mon Sep 17 00:00:00 2001 From: Lum Date: Thu, 17 Aug 2023 13:12:33 -0700 Subject: [PATCH 07/15] Update vocab properties via query, add validation to the well table for in use plates --- .../labkey/assay/plate/query/WellTable.java | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/assay/src/org/labkey/assay/plate/query/WellTable.java b/assay/src/org/labkey/assay/plate/query/WellTable.java index 9d07383d654..9b14c7789e7 100644 --- a/assay/src/org/labkey/assay/plate/query/WellTable.java +++ b/assay/src/org/labkey/assay/plate/query/WellTable.java @@ -2,6 +2,7 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import org.labkey.api.assay.plate.Plate; import org.labkey.api.assay.plate.Well; import org.labkey.api.collections.CaseInsensitiveHashSet; import org.labkey.api.data.ColumnInfo; @@ -17,21 +18,28 @@ import org.labkey.api.dataiterator.StandardDataIteratorBuilder; import org.labkey.api.dataiterator.TableInsertDataIteratorBuilder; import org.labkey.api.exp.Lsid; +import org.labkey.api.exp.OntologyManager; +import org.labkey.api.exp.PropertyColumn; +import org.labkey.api.exp.PropertyDescriptor; import org.labkey.api.exp.property.Domain; import org.labkey.api.exp.property.PropertyService; import org.labkey.api.query.BatchValidationException; import org.labkey.api.query.DefaultQueryUpdateService; import org.labkey.api.query.FieldKey; +import org.labkey.api.query.InvalidKeyException; import org.labkey.api.query.PropertyForeignKey; import org.labkey.api.query.QueryForeignKey; import org.labkey.api.query.QueryUpdateService; +import org.labkey.api.query.QueryUpdateServiceException; import org.labkey.api.query.SimpleUserSchema; import org.labkey.api.query.UserSchema; import org.labkey.api.query.ValidationException; import org.labkey.api.security.User; +import org.labkey.api.util.URIUtil; import org.labkey.assay.plate.PlateManager; import org.labkey.assay.query.AssayDbSchema; +import java.sql.SQLException; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -107,6 +115,36 @@ protected boolean acceptColumn(ColumnInfo col) return super.acceptColumn(col) && !ignoredColumns.contains(col.getName()); } + /** + * Override to resolve Property URIs for vocabulary columns during update. Consider adding the + * capability to resolve vocabulary columns by field key or name. + */ + @Override + protected ColumnInfo resolveColumn(String name) + { + ColumnInfo lsidCol = getColumn("LSID", false); + if (lsidCol != null) + { + // Attempt to resolve the column name as a property URI if it looks like a URI + if (URIUtil.hasURICharacters(name)) + { + // mark vocab propURI col as Voc column + PropertyDescriptor pd = OntologyManager.getPropertyDescriptor(name /* uri */, getContainer()); + if (pd != null) + { + PropertyColumn pc = new PropertyColumn(pd, lsidCol, getContainer(), getUserSchema().getUser(), false); + // use the property URI as the column's FieldKey name + String label = pc.getLabel(); + pc.setFieldKey(FieldKey.fromParts(name)); + pc.setLabel(label); + + return pc; + } + } + } + return super.resolveColumn(name); + } + @Override public MutableColumnInfo wrapColumn(ColumnInfo col) { @@ -203,5 +241,22 @@ public List> insertRows(User user, Container container, List { return super._insertRowsUsingDIB(user, container, rows, getDataIteratorContext(errors, InsertOption.INSERT, configParameters), extraScriptContext); } + + @Override + protected Map updateRow(User user, Container container, Map row, @NotNull Map oldRow) throws InvalidKeyException, ValidationException, QueryUpdateServiceException, SQLException + { + // enforce no updates if the plate has been imported in an assay run + if (oldRow.containsKey("plateId")) + { + Plate plate = PlateManager.get().getPlate(container, (Integer)oldRow.get("plateId")); + if (plate != null) + { + int runsInUse = PlateManager.get().getRunCountUsingPlate(container, plate); + if (runsInUse > 0) + throw new QueryUpdateServiceException(String.format("This %s is used by %d runs and its wells cannot be modified.", plate.isTemplate() ? "Plate template" : "Plate", runsInUse)); + } + } + return super.updateRow(user, container, row, oldRow); + } } } From d8b3269e954b15af1383073c3a22a88212f32ac2 Mon Sep 17 00:00:00 2001 From: lum Date: Fri, 18 Aug 2023 17:00:31 -0700 Subject: [PATCH 08/15] cacheing fixes, unit tests for plate metadata add plate custom fields to plate cache --- .../org/labkey/api/assay/plate/Plate.java | 6 + .../api/assay/plate}/PlateCustomField.java | 2 +- .../labkey/api/assay/plate/PositionImpl.java | 6 + .../api/assay/plate}/WellCustomField.java | 2 +- .../src/org/labkey/assay/PlateController.java | 6 +- .../plate/AssayPlateMetadataServiceImpl.java | 4 +- .../org/labkey/assay/plate/PlateCache.java | 3 +- .../src/org/labkey/assay/plate/PlateImpl.java | 19 +- .../org/labkey/assay/plate/PlateManager.java | 226 ++++++++++++++---- 9 files changed, 225 insertions(+), 49 deletions(-) rename assay/{src/org/labkey/assay/plate/model => api-src/org/labkey/api/assay/plate}/PlateCustomField.java (97%) rename assay/{src/org/labkey/assay/plate/model => api-src/org/labkey/api/assay/plate}/WellCustomField.java (92%) diff --git a/assay/api-src/org/labkey/api/assay/plate/Plate.java b/assay/api-src/org/labkey/api/assay/plate/Plate.java index c9da75ebdca..172b3b34ee2 100644 --- a/assay/api-src/org/labkey/api/assay/plate/Plate.java +++ b/assay/api-src/org/labkey/api/assay/plate/Plate.java @@ -16,6 +16,7 @@ package org.labkey.api.assay.plate; +import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.labkey.api.exp.Identifiable; import org.labkey.api.study.PropertySet; @@ -73,4 +74,9 @@ public interface Plate extends PropertySet, Identifiable @Override @Nullable ActionURL detailsURL(); + + /** + * The list of metadata fields that are configured for this plate + */ + @NotNull List getCustomFields(); } diff --git a/assay/src/org/labkey/assay/plate/model/PlateCustomField.java b/assay/api-src/org/labkey/api/assay/plate/PlateCustomField.java similarity index 97% rename from assay/src/org/labkey/assay/plate/model/PlateCustomField.java rename to assay/api-src/org/labkey/api/assay/plate/PlateCustomField.java index 2bf207e2b8f..b5aa8b3a022 100644 --- a/assay/src/org/labkey/assay/plate/model/PlateCustomField.java +++ b/assay/api-src/org/labkey/api/assay/plate/PlateCustomField.java @@ -1,4 +1,4 @@ -package org.labkey.assay.plate.model; +package org.labkey.api.assay.plate; import com.fasterxml.jackson.annotation.JsonInclude; import org.labkey.api.exp.property.DomainProperty; diff --git a/assay/api-src/org/labkey/api/assay/plate/PositionImpl.java b/assay/api-src/org/labkey/api/assay/plate/PositionImpl.java index 92ee8cc982a..fb2943ea206 100644 --- a/assay/api-src/org/labkey/api/assay/plate/PositionImpl.java +++ b/assay/api-src/org/labkey/api/assay/plate/PositionImpl.java @@ -78,6 +78,12 @@ public int getColumn() return _column; } + // needed for DB serialization + public int getCol() + { + return getColumn(); + } + @Override public int getRow() { diff --git a/assay/src/org/labkey/assay/plate/model/WellCustomField.java b/assay/api-src/org/labkey/api/assay/plate/WellCustomField.java similarity index 92% rename from assay/src/org/labkey/assay/plate/model/WellCustomField.java rename to assay/api-src/org/labkey/api/assay/plate/WellCustomField.java index 224d5e5f322..f13ea45aa42 100644 --- a/assay/src/org/labkey/assay/plate/model/WellCustomField.java +++ b/assay/api-src/org/labkey/api/assay/plate/WellCustomField.java @@ -1,4 +1,4 @@ -package org.labkey.assay.plate.model; +package org.labkey.api.assay.plate; import org.labkey.api.exp.property.DomainProperty; diff --git a/assay/src/org/labkey/assay/PlateController.java b/assay/src/org/labkey/assay/PlateController.java index 6ba0652c80b..869081d8bca 100644 --- a/assay/src/org/labkey/assay/PlateController.java +++ b/assay/src/org/labkey/assay/PlateController.java @@ -57,9 +57,9 @@ import org.labkey.assay.plate.PlateDataServiceImpl; import org.labkey.assay.plate.PlateManager; import org.labkey.assay.plate.PlateUrls; -import org.labkey.assay.plate.model.PlateCustomField; +import org.labkey.api.assay.plate.PlateCustomField; import org.labkey.assay.plate.model.PlateType; -import org.labkey.assay.plate.model.WellCustomField; +import org.labkey.api.assay.plate.WellCustomField; import org.labkey.assay.view.AssayGWTView; import org.springframework.validation.BindException; import org.springframework.validation.Errors; @@ -739,7 +739,7 @@ public class GetFieldsAction extends MutatingApiAction @Override public Object execute(CustomFieldsForm form, BindException errors) throws Exception { - return success(PlateManager.get().getFields(getContainer(), getUser(), form.getPlateId())); + return success(PlateManager.get().getFields(getContainer(), form.getPlateId())); } } diff --git a/assay/src/org/labkey/assay/plate/AssayPlateMetadataServiceImpl.java b/assay/src/org/labkey/assay/plate/AssayPlateMetadataServiceImpl.java index db4caaddd44..957df93cc75 100644 --- a/assay/src/org/labkey/assay/plate/AssayPlateMetadataServiceImpl.java +++ b/assay/src/org/labkey/assay/plate/AssayPlateMetadataServiceImpl.java @@ -44,7 +44,7 @@ import org.labkey.api.security.User; import org.labkey.assay.TSVProtocolSchema; import org.labkey.assay.plate.model.Well; -import org.labkey.assay.plate.model.WellCustomField; +import org.labkey.api.assay.plate.WellCustomField; import org.labkey.assay.query.AssayDbSchema; import java.io.File; @@ -283,7 +283,7 @@ public List> mergePlateMetadata( throw new ExperimentException("Unable to resolve the plate for the run"); // if there are metadata fields configured for this plate - if (!PlateManager.get().getFields(container, user, plate.getRowId()).isEmpty()) + if (!PlateManager.get().getFields(container, plate.getRowId()).isEmpty()) { // create the map of well locations to the well Map positionToWell = new HashMap<>(); diff --git a/assay/src/org/labkey/assay/plate/PlateCache.java b/assay/src/org/labkey/assay/plate/PlateCache.java index 38d78480a20..e70d7db845a 100644 --- a/assay/src/org/labkey/assay/plate/PlateCache.java +++ b/assay/src/org/labkey/assay/plate/PlateCache.java @@ -163,7 +163,8 @@ public static void uncache(Container c) // uncache all plates for this container if (_loader._rowIdMap.containsKey(c)) { - for (Plate plate : _loader._rowIdMap.get(c).values()) + List plates = new ArrayList<>(_loader._rowIdMap.get(c).values()); + for (Plate plate : plates) { uncache(c, plate); } diff --git a/assay/src/org/labkey/assay/plate/PlateImpl.java b/assay/src/org/labkey/assay/plate/PlateImpl.java index 2a1ade7d9d3..7762227140e 100644 --- a/assay/src/org/labkey/assay/plate/PlateImpl.java +++ b/assay/src/org/labkey/assay/plate/PlateImpl.java @@ -21,6 +21,7 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.labkey.api.assay.plate.Plate; +import org.labkey.api.assay.plate.PlateCustomField; import org.labkey.api.assay.plate.PlateService; import org.labkey.api.assay.plate.Position; import org.labkey.api.assay.plate.PositionImpl; @@ -65,6 +66,7 @@ public class PlateImpl extends PropertySetImpl implements Plate private int _runId; // NO_RUNID means no run yet, well data comes from file, dilution data must be calculated private int _plateNumber; + private List _customFields = Collections.emptyList(); public PlateImpl() { @@ -526,7 +528,10 @@ public void setWells(WellImpl[][] wells) @Override public List getWells() { - return _wellMap.values().stream().toList(); + if (_wellMap != null) + return _wellMap.values().stream().toList(); + else + return Collections.emptyList(); } @Override @@ -557,4 +562,16 @@ public int getPlateNumber() { return _plateNumber; } + + @JsonIgnore + @Override + public @NotNull List getCustomFields() + { + return _customFields; + } + + public void setCustomFields(List customFields) + { + _customFields = customFields; + } } diff --git a/assay/src/org/labkey/assay/plate/PlateManager.java b/assay/src/org/labkey/assay/plate/PlateManager.java index f940d6cc6c5..83975d04dd2 100644 --- a/assay/src/org/labkey/assay/plate/PlateManager.java +++ b/assay/src/org/labkey/assay/plate/PlateManager.java @@ -27,18 +27,23 @@ import org.labkey.api.assay.dilution.DilutionCurve; import org.labkey.api.assay.plate.AbstractPlateTypeHandler; import org.labkey.api.assay.plate.Plate; +import org.labkey.api.assay.plate.PlateCustomField; import org.labkey.api.assay.plate.PlateService; import org.labkey.api.assay.plate.PlateTypeHandler; import org.labkey.api.assay.plate.Position; import org.labkey.api.assay.plate.PositionImpl; import org.labkey.api.assay.plate.Well; +import org.labkey.api.assay.plate.WellCustomField; import org.labkey.api.assay.plate.WellGroup; import org.labkey.api.collections.ArrayListMap; +import org.labkey.api.collections.CaseInsensitiveHashMap; +import org.labkey.api.data.ColumnInfo; import org.labkey.api.data.Container; import org.labkey.api.data.ContainerManager; import org.labkey.api.data.DbScope; import org.labkey.api.data.ImportAliasable; import org.labkey.api.data.ObjectFactory; +import org.labkey.api.data.Results; import org.labkey.api.data.SQLFragment; import org.labkey.api.data.SimpleFilter; import org.labkey.api.data.Sort; @@ -83,9 +88,7 @@ import org.labkey.api.view.ActionURL; import org.labkey.api.webdav.WebdavResource; import org.labkey.assay.TsvAssayProvider; -import org.labkey.assay.plate.model.PlateCustomField; import org.labkey.assay.plate.model.PlateType; -import org.labkey.assay.plate.model.WellCustomField; import org.labkey.assay.plate.model.WellGroupBean; import org.labkey.assay.plate.query.PlateSchema; import org.labkey.assay.plate.query.PlateTable; @@ -115,6 +118,7 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; public class PlateManager implements PlateService { @@ -474,6 +478,31 @@ protected void populatePlate(PlateImpl plate) for (WellGroupImpl group : sortedGroups) plate.addWellGroup(group); + + // custom plate properties + Domain domain = getPlateMetadataDomain(plate.getContainer(), null); + if (domain != null) + { + SQLFragment sqlPlateProps = new SQLFragment("SELECT PropertyURI FROM ").append(AssayDbSchema.getInstance().getTableInfoPlateProperty(), "PP") + .append(" WHERE PlateId = ?").add(plate.getRowId()); + + List fields = new ArrayList<>(); + for (String uri : new SqlSelector(AssayDbSchema.getInstance().getSchema(), sqlPlateProps).getArrayList(String.class)) + { + DomainProperty dp = domain.getPropertyByURI(uri); + if (dp == null) + throw new IllegalArgumentException("Failed to get plate custom field. \"" + uri + "\" does not exist on domain."); + + fields.add(dp); + } + + if (!fields.isEmpty()) + { + plate.setCustomFields(fields.stream() + .sorted(Comparator.comparing(DomainProperty::getName)) + .map(PlateCustomField::new).toList()); + } + } } private WellImpl[] getWells(Plate plate) @@ -538,6 +567,11 @@ private int savePlateImpl(Container container, User user, PlateImpl plate) throw Lsid lsid = Lsid.parse(plateInstanceLsid); plateObjectLsid = lsid.edit().setObjectId(LSID_CLASS_OBJECT_ID).toString(); + // special case if the plate name changes, we want to remove the cache key with the old name + Plate oldPlate = getPlate(container, plateId); + if (!oldPlate.getName().equals(plate.getName())) + clearCache(container, oldPlate); + qus.updateRows(user, container, Collections.singletonList(plateRow), null, errors, null, null); if (errors.hasErrors()) throw errors; @@ -550,6 +584,8 @@ private int savePlateImpl(Container container, User user, PlateImpl plate) throw throw errors; plateId = (Integer)insertedRows.get(0).get("RowId"); plateInstanceLsid = (String)insertedRows.get(0).get("Lsid"); + plate.setRowId(plateId); + plate.setLsid(plateInstanceLsid); } savePropertyBag(container, plateInstanceLsid, plateObjectLsid, plate.getProperties(), updateExisting); @@ -670,10 +706,7 @@ private int savePlateImpl(Container container, User user, PlateImpl plate) throw final Integer plateRowId = plateId; transaction.addCommitTask(() -> { - if (updateExisting) - { - clearCache(container, plate); - } + clearCache(container, plate); indexPlate(container, plateRowId); }, DbScope.CommitTaskOption.POSTCOMMIT); transaction.commit(); @@ -1346,46 +1379,46 @@ private Container getPlateMetadataDomainContainer(Container container) DomainProperty dp = domain.getPropertyByURI(field.getPropertyURI()); if (dp == null) throw new IllegalArgumentException("Failed to add plate custom field. \"" + field.getPropertyURI() + "\" does not exist on domain."); - fieldsToAdd.add(dp); } if (!fieldsToAdd.isEmpty()) { - List propertyURIs = fieldsToAdd.stream().map(DomainProperty::getPropertyURI).collect(Collectors.toList()); - - // verify fields aren't already associated with the plate - SQLFragment sql = new SQLFragment("SELECT PropertyURI FROM ").append(AssayDbSchema.getInstance().getTableInfoPlateProperty(), "PP") - .append(" WHERE PlateId = ? ").add(plateId) - .append(" AND PropertyURI ").appendInClause(propertyURIs, AssayDbSchema.getInstance().getSchema().getSqlDialect()); - - List existingProps = new SqlSelector(AssayDbSchema.getInstance().getSchema(), sql).getArrayList(String.class); - if (!existingProps.isEmpty()) + try (DbScope.Transaction transaction = ExperimentService.get().ensureTransaction()) { - throw new IllegalArgumentException("Failed to add plate custom fields. Custom fields \"" + String.join(",", existingProps) + "\" already are associated with this plate."); - } + Set existingProps = plate.getCustomFields().stream().map(PlateCustomField::getPropertyURI).collect(Collectors.toSet()); + for (DomainProperty dp : fieldsToAdd) + { + if (existingProps.contains(dp.getPropertyURI())) + throw new IllegalArgumentException(String.format("Failed to add plate custom fields. Custom field \"%s\" already is associated with this plate.", dp.getName())); + } - List> insertedValues = new LinkedList<>(); - for (DomainProperty dp : fieldsToAdd) - { - insertedValues.add(List.of(plateId, - dp.getPropertyId(), - dp.getPropertyURI())); + List> insertedValues = new LinkedList<>(); + for (DomainProperty dp : fieldsToAdd) + { + insertedValues.add(List.of(plateId, + dp.getPropertyId(), + dp.getPropertyURI())); + } + String insertSql = "INSERT INTO " + AssayDbSchema.getInstance().getTableInfoPlateProperty() + + " (plateId, propertyId, propertyURI)" + + " VALUES (?, ?, ?)"; + Table.batchExecute(AssayDbSchema.getInstance().getSchema(), insertSql, insertedValues); + + transaction.addCommitTask(() -> clearCache(container, plate), DbScope.CommitTaskOption.POSTCOMMIT); + transaction.commit(); } - String insertSql = "INSERT INTO " + AssayDbSchema.getInstance().getTableInfoPlateProperty() + - " (plateId, propertyId, propertyURI)" + - " VALUES (?, ?, ?)"; - Table.batchExecute(AssayDbSchema.getInstance().getSchema(), insertSql, insertedValues); } - return getFields(container, user, plateId); + return getFields(container, plateId); } - public List getFields(Container container, User user, Integer plateId) + public List getFields(Container container, Integer plateId) { - List fields = _getFields(container, user, plateId).stream().map(PlateCustomField::new).toList(); - return fields.stream() - .sorted(Comparator.comparing(PlateCustomField::getName)) - .collect(Collectors.toList()); + Plate plate = getPlate(container, plateId); + if (plate == null) + throw new IllegalArgumentException("Failed to get plate custom fields. Plate id \"" + plateId + "\" not found."); + + return plate.getCustomFields(); } /** @@ -1458,15 +1491,27 @@ public List removeFields(Container container, User user, Integ if (!fieldsToRemove.isEmpty()) { - List propertyURIs = fieldsToRemove.stream().map(DomainProperty::getPropertyURI).collect(Collectors.toList()); + try (DbScope.Transaction transaction = ExperimentService.get().ensureTransaction()) + { + List propertyURIs = fieldsToRemove.stream().map(DomainProperty::getPropertyURI).collect(Collectors.toList()); + Set existingProps = plate.getCustomFields().stream().map(PlateCustomField::getPropertyURI).collect(Collectors.toSet()); + for (DomainProperty dp : fieldsToRemove) + { + if (!existingProps.contains(dp.getPropertyURI())) + throw new IllegalArgumentException(String.format("Failed to remove plate custom fields. Custom field \"%s\" is not currently associated with this plate.", dp.getName())); + } + + SQLFragment sql = new SQLFragment("DELETE FROM ").append(AssayDbSchema.getInstance().getTableInfoPlateProperty(), "PP") + .append(" WHERE PlateId = ? ").add(plateId) + .append(" AND PropertyURI ").appendInClause(propertyURIs, AssayDbSchema.getInstance().getSchema().getSqlDialect()); - SQLFragment sql = new SQLFragment("DELETE FROM ").append(AssayDbSchema.getInstance().getTableInfoPlateProperty(), "PP") - .append(" WHERE PlateId = ? ").add(plateId) - .append(" AND PropertyURI ").appendInClause(propertyURIs, AssayDbSchema.getInstance().getSchema().getSqlDialect()); + new SqlExecutor(AssayDbSchema.getInstance().getSchema()).execute(sql); - new SqlExecutor(AssayDbSchema.getInstance().getSchema()).execute(sql); + transaction.addCommitTask(() -> clearCache(container, plate), DbScope.CommitTaskOption.POSTCOMMIT); + transaction.commit(); + } } - return getFields(container, user, plateId); + return getFields(container, plateId); } public List setFields(Container container, User user, Integer plateId, Integer wellId, List fields) throws ValidationException @@ -1669,5 +1714,106 @@ public void testCreatePlateTemplates() throws Exception assertTrue("Expected saved plate to have the template field set to true", template.isTemplate()); } } + + @Test + public void testCreatePlateMetadata() throws Exception + { + final Container c = JunitUtil.getTestContainer(); + final User user = TestContext.get().getUser(); + + PlateService.get().deleteAllPlateData(c); + Domain domain = PlateManager.get().getPlateMetadataDomain(c ,user); + if (domain != null) + domain.delete(user); + + Plate plate = PlateService.get().createPlateTemplate(c, TsvPlateTypeHandler.TYPE, 16, 24); + plate.setName("new plate with metadata"); + int plateId = PlateService.get().save(c, user, plate); + + // Assert + assertTrue("Expected saved plateId to be returned", plateId != 0); + + // create custom properties + List customFields = List.of( + new GWTPropertyDescriptor("barcode", "http://www.w3.org/2001/XMLSchema#string"), + new GWTPropertyDescriptor("concentration", "http://www.w3.org/2001/XMLSchema#double"), + new GWTPropertyDescriptor("negativeControl", "http://www.w3.org/2001/XMLSchema#double")); + + List fields = PlateManager.get().createPlateMetadataFields(c, user, customFields); + + // Verify returned sorted by name + assertTrue("Expected plate custom fields", fields.size() == 3); + assertTrue("Expected barcode custom field", fields.get(0).getName().equals("barcode")); + assertTrue("Expected concentration custom field", fields.get(1).getName().equals("concentration")); + assertTrue("Expected negativeControl custom field", fields.get(2).getName().equals("negativeControl")); + + // assign custom fields to the plate + assertTrue("Expected custom fields to be added to the plate", PlateManager.get().addFields(c, user, plateId, fields).size() == 3); + + // verification when adding custom fields t the plate + try + { + PlateManager.get().addFields(c, user, plateId, fields); + fail("Expected a validation error when adding existing fields"); + } + catch (IllegalArgumentException e) + { + assertTrue("Expected validation exception", e.getMessage().equals("Failed to add plate custom fields. Custom field \"barcode\" already is associated with this plate.")); + } + + // remove a plate custom field + fields = PlateManager.get().removeFields(c, user, plateId, List.of(fields.get(0))); + assertTrue("Expected 2 plate custom fields", fields.size() == 2); + assertTrue("Expected concentration custom field", fields.get(0).getName().equals("concentration")); + assertTrue("Expected negativeControl custom field", fields.get(1).getName().equals("negativeControl")); + + // select wells + SimpleFilter filter = SimpleFilter.createContainerFilter(c); + filter.addCondition(FieldKey.fromParts("PlateId"), plateId); + filter.addCondition(FieldKey.fromParts("Row"), 0); + List< org.labkey.assay.plate.model.Well> wells = new TableSelector(AssayDbSchema.getInstance().getTableInfoWell(), filter, new Sort("Col")).getArrayList(org.labkey.assay.plate.model.Well.class); + + assertTrue("Expected 24 wells to be returned", wells.size() == 24); + + // update + TableInfo wellTable = QueryService.get().getUserSchema(user, c, PlateSchema.SCHEMA_NAME).getTable(WellTable.NAME); + QueryUpdateService qus = wellTable.getUpdateService(); + BatchValidationException errors = new BatchValidationException(); + + org.labkey.assay.plate.model.Well well = wells.get(0); + List> rows = List.of(CaseInsensitiveHashMap.of( + "rowid", well.getRowId(), + fields.get(0).getPropertyURI(), 1.25, // concentration + fields.get(1).getPropertyURI(), 5.25 // negativeControl + )); + + qus.updateRows(user, c, rows, null, errors, null, null); + if (errors.hasErrors()) + fail(errors.getMessage()); + + ColumnInfo colConcentration =wellTable.getColumn(fields.get(0).getPropertyURI()); + ColumnInfo colNegControl =wellTable.getColumn(fields.get(1).getPropertyURI()); + + // verify vocab property updates + try (Results r = QueryService.get().select(wellTable, List.of(colConcentration, colNegControl), filter, new Sort("Col"))) + { + int row = 0; + while (r.next()) + { + if (row == 0) + { + assertEquals(1.25, r.getDouble(FieldKey.fromParts(colConcentration.getName())), 0); + assertEquals(5.25, r.getDouble(FieldKey.fromParts(colNegControl.getName())), 0); + } + else + { + // the remainder should be null + assertEquals(0, r.getDouble(FieldKey.fromParts(colConcentration.getName())), 0); + assertEquals(0, r.getDouble(FieldKey.fromParts(colNegControl.getName())), 0); + } + row++; + } + } + } } } From 0994976e4a928fd72a92efa27d72f4ebd6a28a10 Mon Sep 17 00:00:00 2001 From: Lum Date: Mon, 21 Aug 2023 12:42:43 -0700 Subject: [PATCH 09/15] unit test fixes, ensure plate metadata domain for LKB folders return metadata domain ID from Plate --- .../plate/AssayPlateMetadataService.java | 3 +- .../org/labkey/api/assay/plate/Plate.java | 8 ++++++ .../labkey/api/assay/plate/PlateService.java | 7 +++++ assay/src/org/labkey/assay/AssayModule.java | 5 ++++ .../org/labkey/assay/plate/PlateCache.java | 11 ++++++-- .../src/org/labkey/assay/plate/PlateImpl.java | 28 ++++++++++++++++++- .../org/labkey/assay/plate/PlateManager.java | 4 ++- 7 files changed, 60 insertions(+), 6 deletions(-) diff --git a/api/src/org/labkey/api/assay/plate/AssayPlateMetadataService.java b/api/src/org/labkey/api/assay/plate/AssayPlateMetadataService.java index 51418b27794..afd647ac38a 100644 --- a/api/src/org/labkey/api/assay/plate/AssayPlateMetadataService.java +++ b/api/src/org/labkey/api/assay/plate/AssayPlateMetadataService.java @@ -25,6 +25,7 @@ public interface AssayPlateMetadataService { String PLATE_TEMPLATE_COLUMN_NAME = "PlateTemplate"; Map _handlers = new HashMap<>(); + String EXPERIMENTAL_APP_PLATE_SUPPORT = "experimental-app-plate-support"; static void registerService(AssayDataType dataType, AssayPlateMetadataService handler) { @@ -38,7 +39,7 @@ static void registerService(AssayDataType dataType, AssayPlateMetadataService ha static boolean isExperimentalAppPlateEnabled() { - return ExperimentalFeatureService.get().isFeatureEnabled("experimental-app-plate-support"); + return ExperimentalFeatureService.get().isFeatureEnabled(EXPERIMENTAL_APP_PLATE_SUPPORT); } @Nullable diff --git a/assay/api-src/org/labkey/api/assay/plate/Plate.java b/assay/api-src/org/labkey/api/assay/plate/Plate.java index 172b3b34ee2..6225d1379ae 100644 --- a/assay/api-src/org/labkey/api/assay/plate/Plate.java +++ b/assay/api-src/org/labkey/api/assay/plate/Plate.java @@ -79,4 +79,12 @@ public interface Plate extends PropertySet, Identifiable * The list of metadata fields that are configured for this plate */ @NotNull List getCustomFields(); + + Plate copy(); + + /** + * Returns the domain ID for the plate metadata domain. + */ + @Nullable + Integer getMetadataDomainId(); } diff --git a/assay/api-src/org/labkey/api/assay/plate/PlateService.java b/assay/api-src/org/labkey/api/assay/plate/PlateService.java index fbc1967347a..154f227b2b3 100644 --- a/assay/api-src/org/labkey/api/assay/plate/PlateService.java +++ b/assay/api-src/org/labkey/api/assay/plate/PlateService.java @@ -25,6 +25,8 @@ import org.labkey.api.data.statistics.StatsService; import org.labkey.api.exp.Lsid; import org.labkey.api.exp.api.ExpRun; +import org.labkey.api.exp.property.Domain; +import org.labkey.api.query.ValidationException; import org.labkey.api.security.User; import org.labkey.api.services.ServiceRegistry; import org.labkey.api.view.ActionURL; @@ -236,6 +238,11 @@ static PlateService get() */ TableInfo getPlateTableInfo(); + /** + * Create the plate metadata domain for this container. + */ + @NotNull Domain ensurePlateMetadataDomain(Container container, User user) throws ValidationException; + /** * A PlateDetailsResolver implementation provides a URL where a detailed, plate-type specific * UI can be found. diff --git a/assay/src/org/labkey/assay/AssayModule.java b/assay/src/org/labkey/assay/AssayModule.java index 93c85497696..02cdfbbea8b 100644 --- a/assay/src/org/labkey/assay/AssayModule.java +++ b/assay/src/org/labkey/assay/AssayModule.java @@ -54,6 +54,7 @@ import org.labkey.api.security.User; import org.labkey.api.security.permissions.ReadPermission; import org.labkey.api.security.roles.RoleManager; +import org.labkey.api.settings.AdminConsole; import org.labkey.api.util.ContextListener; import org.labkey.api.util.JspTestCase; import org.labkey.api.util.PageFlowUtil; @@ -89,6 +90,7 @@ import static org.labkey.api.assay.DefaultDataTransformer.LEGACY_SESSION_COOKIE_NAME_REPLACEMENT; import static org.labkey.api.assay.DefaultDataTransformer.LEGACY_SESSION_ID_REPLACEMENT; +import static org.labkey.api.assay.plate.AssayPlateMetadataService.EXPERIMENTAL_APP_PLATE_SUPPORT; public class AssayModule extends SpringModule { @@ -246,6 +248,9 @@ public void moduleStartupComplete(ServletContext servletContext) }); ExperimentService.get().addExperimentListener(new AssayExperimentListener()); + + AdminConsole.addExperimentalFeatureFlag(new AdminConsole.ExperimentalFeatureFlag(EXPERIMENTAL_APP_PLATE_SUPPORT, + "Plate samples in Biologics", "Plate samples in Biologics for import and analysis.", false, true)); } @Override diff --git a/assay/src/org/labkey/assay/plate/PlateCache.java b/assay/src/org/labkey/assay/plate/PlateCache.java index e70d7db845a..abad03364c2 100644 --- a/assay/src/org/labkey/assay/plate/PlateCache.java +++ b/assay/src/org/labkey/assay/plate/PlateCache.java @@ -119,17 +119,22 @@ private void removeFromCollections(Container c, Plate plate) public static @Nullable Plate getPlate(Container c, int rowId) { - return PLATE_CACHE.get(PlateCacheKey.getCacheKey(c, rowId)); + Plate plate = PLATE_CACHE.get(PlateCacheKey.getCacheKey(c, rowId)); + // We allow plates to be mutated, return a copy of the cached object which still references the + // original wells and well groups + return plate != null ? plate.copy() : null; } public static @Nullable Plate getPlate(Container c, String name) { - return PLATE_CACHE.get(PlateCacheKey.getCacheKey(c, name)); + Plate plate = PLATE_CACHE.get(PlateCacheKey.getCacheKey(c, name)); + return plate != null ? plate.copy() : null; } public static @Nullable Plate getPlate(Container c, Lsid lsid) { - return PLATE_CACHE.get(PlateCacheKey.getCacheKey(c, lsid)); + Plate plate = PLATE_CACHE.get(PlateCacheKey.getCacheKey(c, lsid)); + return plate != null ? plate.copy() : null; } public static @NotNull Collection getPlates(Container c) diff --git a/assay/src/org/labkey/assay/plate/PlateImpl.java b/assay/src/org/labkey/assay/plate/PlateImpl.java index 7762227140e..871cec55b5e 100644 --- a/assay/src/org/labkey/assay/plate/PlateImpl.java +++ b/assay/src/org/labkey/assay/plate/PlateImpl.java @@ -31,6 +31,7 @@ import org.labkey.api.data.Transient; import org.labkey.api.query.QueryRowReference; import org.labkey.api.util.GUID; +import org.labkey.api.util.UnexpectedException; import org.labkey.api.view.ActionURL; import org.labkey.assay.PlateController; @@ -44,7 +45,7 @@ import java.util.Map; @JsonInclude(JsonInclude.Include.NON_NULL) -public class PlateImpl extends PropertySetImpl implements Plate +public class PlateImpl extends PropertySetImpl implements Plate, Cloneable { private String _name; private Integer _rowId; @@ -67,6 +68,7 @@ public class PlateImpl extends PropertySetImpl implements Plate private int _runId; // NO_RUNID means no run yet, well data comes from file, dilution data must be calculated private int _plateNumber; private List _customFields = Collections.emptyList(); + private Integer _metadataDomainId; public PlateImpl() { @@ -574,4 +576,28 @@ public void setCustomFields(List customFields) { _customFields = customFields; } + + public PlateImpl copy() + { + try + { + return (PlateImpl)super.clone(); + } + catch (CloneNotSupportedException e) + { + throw UnexpectedException.wrap(e); + } + } + + @Nullable + @Override + public Integer getMetadataDomainId() + { + return _metadataDomainId; + } + + public void setMetadataDomainId(Integer metadataDomainId) + { + _metadataDomainId = metadataDomainId; + } } diff --git a/assay/src/org/labkey/assay/plate/PlateManager.java b/assay/src/org/labkey/assay/plate/PlateManager.java index 83975d04dd2..02da316039d 100644 --- a/assay/src/org/labkey/assay/plate/PlateManager.java +++ b/assay/src/org/labkey/assay/plate/PlateManager.java @@ -483,6 +483,7 @@ protected void populatePlate(PlateImpl plate) Domain domain = getPlateMetadataDomain(plate.getContainer(), null); if (domain != null) { + plate.setMetadataDomainId(domain.getTypeId()); SQLFragment sqlPlateProps = new SQLFragment("SELECT PropertyURI FROM ").append(AssayDbSchema.getInstance().getTableInfoPlateProperty(), "PP") .append(" WHERE PlateId = ?").add(plate.getRowId()); @@ -1247,7 +1248,8 @@ private Container getPlateMetadataDomainContainer(Container container) return container.isProject() ? container : container.getProject(); } - private @NotNull Domain ensurePlateMetadataDomain(Container container, User user) throws ValidationException + @Override + public @NotNull Domain ensurePlateMetadataDomain(Container container, User user) throws ValidationException { Domain vocabDomain = getPlateMetadataDomain(container, user); From c0d5f67cd34878ce95f8747042d24e035a94d677 Mon Sep 17 00:00:00 2001 From: Lum Date: Mon, 21 Aug 2023 15:34:05 -0700 Subject: [PATCH 10/15] new plate endpoint for metadata domain --- .../src/org/labkey/assay/PlateController.java | 45 +++++-------------- .../org/labkey/assay/plate/PlateManager.java | 41 +---------------- 2 files changed, 13 insertions(+), 73 deletions(-) diff --git a/assay/src/org/labkey/assay/PlateController.java b/assay/src/org/labkey/assay/PlateController.java index 869081d8bca..6e7390f86ca 100644 --- a/assay/src/org/labkey/assay/PlateController.java +++ b/assay/src/org/labkey/assay/PlateController.java @@ -30,10 +30,12 @@ import org.labkey.api.action.SimpleViewAction; import org.labkey.api.action.SpringActionController; import org.labkey.api.assay.plate.Plate; +import org.labkey.api.assay.plate.PlateCustomField; import org.labkey.api.assay.plate.PlateService; import org.labkey.api.assay.security.DesignAssayPermission; import org.labkey.api.data.Container; import org.labkey.api.data.ContainerManager; +import org.labkey.api.exp.property.Domain; import org.labkey.api.gwt.client.model.GWTPropertyDescriptor; import org.labkey.api.gwt.server.BaseRemoteService; import org.labkey.api.security.RequiresAnyOf; @@ -57,9 +59,7 @@ import org.labkey.assay.plate.PlateDataServiceImpl; import org.labkey.assay.plate.PlateManager; import org.labkey.assay.plate.PlateUrls; -import org.labkey.api.assay.plate.PlateCustomField; import org.labkey.assay.plate.model.PlateType; -import org.labkey.api.assay.plate.WellCustomField; import org.labkey.assay.view.AssayGWTView; import org.springframework.validation.BindException; import org.springframework.validation.Errors; @@ -734,7 +734,7 @@ public Object execute(CustomFieldsForm form, BindException errors) throws Except } @RequiresPermission(ReadPermission.class) - public class GetFieldsAction extends MutatingApiAction + public class GetFieldsAction extends ReadOnlyApiAction { @Override public Object execute(CustomFieldsForm form, BindException errors) throws Exception @@ -753,39 +753,18 @@ public Object execute(CustomFieldsForm form, BindException errors) throws Except } } - public static class SetFieldsForm extends CustomFieldsForm - { - private Integer _wellId; - private List _wellFields; - - public Integer getWellId() - { - return _wellId; - } - - public void setWellId(Integer wellId) - { - _wellId = wellId; - } - - public List getWellFields() - { - return _wellFields; - } - - public void setWellFields(List wellFields) - { - _wellFields = wellFields; - } - } - - @RequiresPermission(UpdatePermission.class) - public class SetFieldsAction extends MutatingApiAction + /** + * Returns the Domain ID for the plate metadata configured for this container. If the domain has + * not been created then null will be returned. + */ + @RequiresPermission(ReadPermission.class) + public class GetPlateMetadataDomainAction extends ReadOnlyApiAction { @Override - public Object execute(SetFieldsForm form, BindException errors) throws Exception + public Object execute(Object form, BindException errors) throws Exception { - return success(PlateManager.get().setFields(getContainer(), getUser(), form.getPlateId(), form.getWellId(), form.getWellFields())); + Domain domain = PlateManager.get().getPlateMetadataDomain(getContainer(), getUser()); + return success(domain != null ? domain.getTypeId() : null); } } } diff --git a/assay/src/org/labkey/assay/plate/PlateManager.java b/assay/src/org/labkey/assay/plate/PlateManager.java index 02da316039d..586cdcc7790 100644 --- a/assay/src/org/labkey/assay/plate/PlateManager.java +++ b/assay/src/org/labkey/assay/plate/PlateManager.java @@ -1516,45 +1516,6 @@ public List removeFields(Container container, User user, Integ return getFields(container, plateId); } - public List setFields(Container container, User user, Integer plateId, Integer wellId, List fields) throws ValidationException - { - Plate plate = getPlate(container, plateId); - if (plate == null) - throw new IllegalArgumentException("Failed to set plate custom field values. Plate id \"" + plateId + "\" not found."); - - WellImpl well = (WellImpl) plate.getWell(wellId); - if (well == null) - throw new IllegalArgumentException("Failed to set plate custom field values. Well id \"" + wellId + "\" not found."); - - Domain domain = getPlateMetadataDomain(container, user); - if (domain == null) - throw new IllegalArgumentException("Failed to set plate custom field values. Custom fields domain does not exist. Try creating fields first."); - - try (DbScope.Transaction transaction = ExperimentService.get().ensureTransaction()) - { - // verify the fields have been configured for the plate - Set configuredProps = new HashSet<>(); - SQLFragment sql = new SQLFragment("SELECT PropertyURI FROM ").append(AssayDbSchema.getInstance().getTableInfoPlateProperty(), "") - .append(" WHERE PlateId = ?").add(plateId); - new SqlSelector(AssayDbSchema.getInstance().getSchema(), sql).forEach(String.class, configuredProps::add); - - for (WellCustomField field : fields) - { - if (!configuredProps.contains(field.getPropertyURI())) - throw new IllegalArgumentException("Failed to set plate custom field value. Custom field \"" + field.getPropertyURI() + "\" is not configured for this plate."); - - DomainProperty dp = domain.getPropertyByURI(field.getPropertyURI()); - if (dp == null) - throw new IllegalArgumentException("Failed to set plate custom field values. Custom field \"" + field.getPropertyURI() + "\" not found."); - - OntologyManager.updateObjectProperty(user, container, dp.getPropertyDescriptor(), well.getLsid(), field.getValue(), null, true); - } - transaction.commit(); - } - - return getWellCustomFields(container, user, plateId, wellId); - } - public static final class TestCase { @Test @@ -1752,7 +1713,7 @@ public void testCreatePlateMetadata() throws Exception // assign custom fields to the plate assertTrue("Expected custom fields to be added to the plate", PlateManager.get().addFields(c, user, plateId, fields).size() == 3); - // verification when adding custom fields t the plate + // verification when adding custom fields to the plate try { PlateManager.get().addFields(c, user, plateId, fields); From f75c4d3fdf3bf2eb18962b4021cfed117c3d7060 Mon Sep 17 00:00:00 2001 From: Lum Date: Tue, 22 Aug 2023 11:25:03 -0700 Subject: [PATCH 11/15] Add GetPlate endpoint --- .../src/org/labkey/assay/PlateController.java | 32 +++++++++++++++++++ .../src/org/labkey/assay/plate/PlateImpl.java | 1 - 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/assay/src/org/labkey/assay/PlateController.java b/assay/src/org/labkey/assay/PlateController.java index 6e7390f86ca..eb6b935dd36 100644 --- a/assay/src/org/labkey/assay/PlateController.java +++ b/assay/src/org/labkey/assay/PlateController.java @@ -767,4 +767,36 @@ public Object execute(Object form, BindException errors) throws Exception return success(domain != null ? domain.getTypeId() : null); } } + + public static class GetPlateForm + { + private Integer _rowId; + + public Integer getRowId() + { + return _rowId; + } + + public void setRowId(Integer rowId) + { + _rowId = rowId; + } + } + + @RequiresPermission(ReadPermission.class) + public static class GetPlateAction extends ReadOnlyApiAction + { + @Override + public void validateForm(GetPlateForm form, Errors errors) + { + if (form.getRowId() == null) + errors.reject(ERROR_GENERIC, "Plate \"rowId\" is required."); + } + + @Override + public Object execute(GetPlateForm form, BindException errors) throws Exception + { + return PlateManager.get().getPlate(getContainer(), form.getRowId()); + } + } } diff --git a/assay/src/org/labkey/assay/plate/PlateImpl.java b/assay/src/org/labkey/assay/plate/PlateImpl.java index 871cec55b5e..246c3e2f511 100644 --- a/assay/src/org/labkey/assay/plate/PlateImpl.java +++ b/assay/src/org/labkey/assay/plate/PlateImpl.java @@ -565,7 +565,6 @@ public int getPlateNumber() return _plateNumber; } - @JsonIgnore @Override public @NotNull List getCustomFields() { From 9d30a8e2aabfb80bb64f75fb2cd6d84193b5c8a1 Mon Sep 17 00:00:00 2001 From: Lum Date: Tue, 22 Aug 2023 14:43:54 -0700 Subject: [PATCH 12/15] plate cache logging --- assay/src/org/labkey/assay/plate/PlateCache.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/assay/src/org/labkey/assay/plate/PlateCache.java b/assay/src/org/labkey/assay/plate/PlateCache.java index abad03364c2..b828cd55c23 100644 --- a/assay/src/org/labkey/assay/plate/PlateCache.java +++ b/assay/src/org/labkey/assay/plate/PlateCache.java @@ -59,6 +59,7 @@ public Plate load(@NotNull String key, @Nullable Object argument) { PlateImpl plate = plates.get(0); PlateManager.get().populatePlate(plate); + LOG.info(String.format("Caching plate %s for folder %s", plate.getName(), cacheKey._container.getPath())); // add to internal collections addToCollections(cacheKey, plate); @@ -165,6 +166,8 @@ private void removeFromCollections(Container c, Plate plate) public static void uncache(Container c) { + LOG.info(String.format("Clearing cache for folder %s", c.getPath())); + // uncache all plates for this container if (_loader._rowIdMap.containsKey(c)) { @@ -178,6 +181,8 @@ public static void uncache(Container c) public static void uncache(Container c, Plate plate) { + LOG.info(String.format("Un-caching plate %s for folder %s", plate.getName(), c.getPath())); + if (plate.getName() == null) throw new IllegalArgumentException("Plate cannot be uncached, name is null"); if (plate.getRowId() == null) From c174b28f53f299fe58e4ec1b2fe8757828d305aa Mon Sep 17 00:00:00 2001 From: Lum Date: Tue, 22 Aug 2023 14:57:10 -0700 Subject: [PATCH 13/15] plate cache logging --- assay/src/org/labkey/assay/plate/PlateCache.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/assay/src/org/labkey/assay/plate/PlateCache.java b/assay/src/org/labkey/assay/plate/PlateCache.java index b828cd55c23..90864b815cd 100644 --- a/assay/src/org/labkey/assay/plate/PlateCache.java +++ b/assay/src/org/labkey/assay/plate/PlateCache.java @@ -59,7 +59,7 @@ public Plate load(@NotNull String key, @Nullable Object argument) { PlateImpl plate = plates.get(0); PlateManager.get().populatePlate(plate); - LOG.info(String.format("Caching plate %s for folder %s", plate.getName(), cacheKey._container.getPath())); + LOG.info(String.format("Caching plate \"%s\" for folder %s", plate.getName(), cacheKey._container.getPath())); // add to internal collections addToCollections(cacheKey, plate); @@ -181,7 +181,7 @@ public static void uncache(Container c) public static void uncache(Container c, Plate plate) { - LOG.info(String.format("Un-caching plate %s for folder %s", plate.getName(), c.getPath())); + LOG.info(String.format("Un-caching plate \"%s\" for folder %s", plate.getName(), c.getPath())); if (plate.getName() == null) throw new IllegalArgumentException("Plate cannot be uncached, name is null"); From f1ec5e68b0e47fe16eea3a5ecb4ca21ad8e8dda9 Mon Sep 17 00:00:00 2001 From: Lum Date: Tue, 22 Aug 2023 16:39:33 -0700 Subject: [PATCH 14/15] simplify plate cache --- .../org/labkey/assay/plate/PlateCache.java | 65 +++++-------------- 1 file changed, 17 insertions(+), 48 deletions(-) diff --git a/assay/src/org/labkey/assay/plate/PlateCache.java b/assay/src/org/labkey/assay/plate/PlateCache.java index 90864b815cd..d827a61bae4 100644 --- a/assay/src/org/labkey/assay/plate/PlateCache.java +++ b/assay/src/org/labkey/assay/plate/PlateCache.java @@ -34,9 +34,7 @@ public class PlateCache private static class PlateLoader implements CacheLoader { - private Map> _rowIdMap = new HashMap<>(); - private Map> _nameMap = new HashMap<>(); - private Map> _lsidMap = new HashMap<>(); + private Map> _containerPlateMap = new HashMap<>(); // internal collection to help un-cache all plates for a container @Override public Plate load(@NotNull String key, @Nullable Object argument) @@ -44,11 +42,6 @@ public Plate load(@NotNull String key, @Nullable Object argument) // parse the cache key PlateCacheKey cacheKey = new PlateCacheKey(key); - // check our internal caches - Plate cachedPlate = getFromCollections(cacheKey); - if (cachedPlate != null) - return cachedPlate; - SimpleFilter filter = SimpleFilter.createContainerFilter(cacheKey._container); filter.addCondition(FieldKey.fromParts(cacheKey._type.name()), cacheKey._identifier); @@ -61,29 +54,14 @@ public Plate load(@NotNull String key, @Nullable Object argument) PlateManager.get().populatePlate(plate); LOG.info(String.format("Caching plate \"%s\" for folder %s", plate.getName(), cacheKey._container.getPath())); - // add to internal collections - addToCollections(cacheKey, plate); + // add all cache keys for this plate + addCacheKeys(cacheKey, plate); return plate; } return null; } - @Nullable - private Plate getFromCollections(PlateCacheKey cacheKey) - { - Map> map = switch (cacheKey._type) - { - case lsid -> _lsidMap; - case name -> _nameMap; - case rowId -> _rowIdMap; - }; - - if (map.containsKey(cacheKey._container)) - return map.get(cacheKey._container).get(cacheKey._identifier); - return null; - } - - private void addToCollections(PlateCacheKey cacheKey, Plate plate) + private void addCacheKeys(PlateCacheKey cacheKey, Plate plate) { if (plate != null) { @@ -94,26 +72,15 @@ private void addToCollections(PlateCacheKey cacheKey, Plate plate) if (plate.getLSID() == null) throw new IllegalArgumentException("Plate cannot be cached, LSID is null"); - _lsidMap.computeIfAbsent(cacheKey._container, k -> new HashMap<>()).put(plate.getLSID(), plate); - _nameMap.computeIfAbsent(cacheKey._container, k -> new HashMap<>()).put(plate.getName(), plate); - _rowIdMap.computeIfAbsent(cacheKey._container, k -> new HashMap<>()).put(plate.getRowId(), plate); - } - } - - private void removeFromCollections(Container c, Plate plate) - { - if (plate != null) - { - if (plate.getName() == null) - throw new IllegalArgumentException("Plate cannot be uncached, name is null"); - if (plate.getRowId() == null) - throw new IllegalArgumentException("Plate cannot be uncached, rowId is null"); - if (plate.getLSID() == null) - throw new IllegalArgumentException("Plate cannot be uncached, LSID is null"); + // add the plate for the other key types + if (cacheKey._type != PlateCacheKey.Type.rowId) + PLATE_CACHE.put(PlateCacheKey.getCacheKey(plate.getContainer(), plate.getRowId()), plate); + if (cacheKey._type != PlateCacheKey.Type.lsid) + PLATE_CACHE.put(PlateCacheKey.getCacheKey(plate.getContainer(), Lsid.parse(plate.getLSID())), plate); + if (cacheKey._type != PlateCacheKey.Type.name) + PLATE_CACHE.put(PlateCacheKey.getCacheKey(plate.getContainer(), plate.getName()), plate); - if (_lsidMap.containsKey(c)) _lsidMap.get(c).remove(plate.getLSID()); - if (_nameMap.containsKey(c)) _nameMap.get(c).remove(plate.getName()); - if (_rowIdMap.containsKey(c)) _rowIdMap.get(c).remove(plate.getRowId()); + _containerPlateMap.computeIfAbsent(cacheKey._container, k -> new ArrayList<>()).add(plate); } } } @@ -169,13 +136,14 @@ public static void uncache(Container c) LOG.info(String.format("Clearing cache for folder %s", c.getPath())); // uncache all plates for this container - if (_loader._rowIdMap.containsKey(c)) + if (_loader._containerPlateMap.containsKey(c)) { - List plates = new ArrayList<>(_loader._rowIdMap.get(c).values()); + List plates = new ArrayList<>(_loader._containerPlateMap.get(c)); for (Plate plate : plates) { uncache(c, plate); } + _loader._containerPlateMap.remove(c); } } @@ -194,7 +162,8 @@ public static void uncache(Container c, Plate plate) PLATE_CACHE.remove(PlateCacheKey.getCacheKey(c, plate.getRowId())); PLATE_CACHE.remove(PlateCacheKey.getCacheKey(c, Lsid.parse(plate.getLSID()))); - _loader.removeFromCollections(c, plate); + if (_loader._containerPlateMap.containsKey(c)) + _loader._containerPlateMap.get(c).remove(plate); } public static void clearCache() From 333151fc0ef85ebeb83476325e3b77ab159e7ee6 Mon Sep 17 00:00:00 2001 From: Lum Date: Wed, 23 Aug 2023 11:37:53 -0700 Subject: [PATCH 15/15] resolve well metadata columns by field key (and PropertyURI) --- .../org/labkey/assay/plate/PlateManager.java | 31 ++++++++++++++----- .../labkey/assay/plate/query/WellTable.java | 21 ++++++++++--- 2 files changed, 40 insertions(+), 12 deletions(-) diff --git a/assay/src/org/labkey/assay/plate/PlateManager.java b/assay/src/org/labkey/assay/plate/PlateManager.java index 586cdcc7790..2f81105ad86 100644 --- a/assay/src/org/labkey/assay/plate/PlateManager.java +++ b/assay/src/org/labkey/assay/plate/PlateManager.java @@ -1503,7 +1503,7 @@ public List removeFields(Container container, User user, Integ throw new IllegalArgumentException(String.format("Failed to remove plate custom fields. Custom field \"%s\" is not currently associated with this plate.", dp.getName())); } - SQLFragment sql = new SQLFragment("DELETE FROM ").append(AssayDbSchema.getInstance().getTableInfoPlateProperty(), "PP") + SQLFragment sql = new SQLFragment("DELETE FROM ").append(AssayDbSchema.getInstance().getTableInfoPlateProperty(), "") .append(" WHERE PlateId = ? ").add(plateId) .append(" AND PropertyURI ").appendInClause(propertyURIs, AssayDbSchema.getInstance().getSchema().getSqlDialect()); @@ -1743,6 +1743,7 @@ public void testCreatePlateMetadata() throws Exception QueryUpdateService qus = wellTable.getUpdateService(); BatchValidationException errors = new BatchValidationException(); + // verify metadata update works for Property URI as well as field key org.labkey.assay.plate.model.Well well = wells.get(0); List> rows = List.of(CaseInsensitiveHashMap.of( "rowid", well.getRowId(), @@ -1754,8 +1755,19 @@ public void testCreatePlateMetadata() throws Exception if (errors.hasErrors()) fail(errors.getMessage()); - ColumnInfo colConcentration =wellTable.getColumn(fields.get(0).getPropertyURI()); - ColumnInfo colNegControl =wellTable.getColumn(fields.get(1).getPropertyURI()); + well = wells.get(1); + rows = List.of(CaseInsensitiveHashMap.of( + "rowid", well.getRowId(), + "properties/concentration", 2.25, + "properties/negativeControl", 6.25 + )); + + qus.updateRows(user, c, rows, null, errors, null, null); + if (errors.hasErrors()) + fail(errors.getMessage()); + + ColumnInfo colConcentration = wellTable.getColumn("properties/concentration"); + ColumnInfo colNegControl = wellTable.getColumn("properties/negativeControl"); // verify vocab property updates try (Results r = QueryService.get().select(wellTable, List.of(colConcentration, colNegControl), filter, new Sort("Col"))) @@ -1765,14 +1777,19 @@ public void testCreatePlateMetadata() throws Exception { if (row == 0) { - assertEquals(1.25, r.getDouble(FieldKey.fromParts(colConcentration.getName())), 0); - assertEquals(5.25, r.getDouble(FieldKey.fromParts(colNegControl.getName())), 0); + assertEquals(1.25, r.getDouble(colConcentration.getFieldKey()), 0); + assertEquals(5.25, r.getDouble(colNegControl.getFieldKey()), 0); + } + else if (row == 1) + { + assertEquals(2.25, r.getDouble(colConcentration.getFieldKey()), 0); + assertEquals(6.25, r.getDouble(colNegControl.getFieldKey()), 0); } else { // the remainder should be null - assertEquals(0, r.getDouble(FieldKey.fromParts(colConcentration.getName())), 0); - assertEquals(0, r.getDouble(FieldKey.fromParts(colNegControl.getName())), 0); + assertEquals(0, r.getDouble(colConcentration.getFieldKey()), 0); + assertEquals(0, r.getDouble(colNegControl.getFieldKey()), 0); } row++; } diff --git a/assay/src/org/labkey/assay/plate/query/WellTable.java b/assay/src/org/labkey/assay/plate/query/WellTable.java index 9b14c7789e7..541adaa4699 100644 --- a/assay/src/org/labkey/assay/plate/query/WellTable.java +++ b/assay/src/org/labkey/assay/plate/query/WellTable.java @@ -22,6 +22,7 @@ import org.labkey.api.exp.PropertyColumn; import org.labkey.api.exp.PropertyDescriptor; import org.labkey.api.exp.property.Domain; +import org.labkey.api.exp.property.DomainProperty; import org.labkey.api.exp.property.PropertyService; import org.labkey.api.query.BatchValidationException; import org.labkey.api.query.DefaultQueryUpdateService; @@ -35,12 +36,12 @@ import org.labkey.api.query.UserSchema; import org.labkey.api.query.ValidationException; import org.labkey.api.security.User; -import org.labkey.api.util.URIUtil; import org.labkey.assay.plate.PlateManager; import org.labkey.assay.query.AssayDbSchema; import java.sql.SQLException; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -53,6 +54,7 @@ public class WellTable extends SimpleUserSchema.SimpleTable public static final String NAME = "Well"; private static final List defaultVisibleColumns = new ArrayList<>(); private static final Set ignoredColumns = new CaseInsensitiveHashSet(); + private Map _vocabularyFieldMap = new HashMap<>(); static { @@ -91,6 +93,13 @@ private void addVocabularyDomains() col.setLabel("Plate Metadata"); col.setDescription("Custom properties associated with the plate well"); } + + for (DomainProperty field : domain.getProperties()) + { + // resolve vocabulary fields by property URI and field key + _vocabularyFieldMap.put(FieldKey.fromParts("properties", field.getName()), field); + _vocabularyFieldMap.put(FieldKey.fromParts(field.getPropertyURI()), field); + } } } @@ -126,16 +135,18 @@ protected ColumnInfo resolveColumn(String name) if (lsidCol != null) { // Attempt to resolve the column name as a property URI if it looks like a URI - if (URIUtil.hasURICharacters(name)) + FieldKey fieldKey = FieldKey.decode(name); + if (_vocabularyFieldMap.containsKey(fieldKey)) { + DomainProperty field = _vocabularyFieldMap.get(fieldKey); + // mark vocab propURI col as Voc column - PropertyDescriptor pd = OntologyManager.getPropertyDescriptor(name /* uri */, getContainer()); + PropertyDescriptor pd = OntologyManager.getPropertyDescriptor(field.getPropertyURI(), getContainer()); if (pd != null) { PropertyColumn pc = new PropertyColumn(pd, lsidCol, getContainer(), getUserSchema().getUser(), false); - // use the property URI as the column's FieldKey name String label = pc.getLabel(); - pc.setFieldKey(FieldKey.fromParts(name)); + pc.setFieldKey(fieldKey); pc.setLabel(label); return pc;