diff --git a/pom.xml b/pom.xml index 805e87bfc3..90c4822f4a 100644 --- a/pom.xml +++ b/pom.xml @@ -3,14 +3,14 @@ 4.0.0 org.snomed.otf - 7.2.1 + 7.3.0-SNAPSHOT reporting-engine pom org.snomed snomed-parent-bom - 3.2.2 + 3.3.0-SNAPSHOT diff --git a/reporting-engine-worker/pom.xml b/reporting-engine-worker/pom.xml index a5527ae107..3baa1a5b5a 100644 --- a/reporting-engine-worker/pom.xml +++ b/reporting-engine-worker/pom.xml @@ -6,7 +6,7 @@ org.snomed.otf - 7.2.1 + 7.3.0-SNAPSHOT reporting-engine diff --git a/reporting-engine-worker/src/main/java/org/ihtsdo/termserver/scripting/fixes/drugs/CreateMissingDrugConcepts.java b/reporting-engine-worker/src/main/java/org/ihtsdo/termserver/scripting/fixes/drugs/CreateMissingDrugConcepts.java index 5a19ed3644..507ff6b086 100644 --- a/reporting-engine-worker/src/main/java/org/ihtsdo/termserver/scripting/fixes/drugs/CreateMissingDrugConcepts.java +++ b/reporting-engine-worker/src/main/java/org/ihtsdo/termserver/scripting/fixes/drugs/CreateMissingDrugConcepts.java @@ -77,9 +77,6 @@ protected CreateMissingDrugConcepts(BatchFix clone) { public static void main(String[] args) throws TermServerScriptException, IOException, InterruptedException { Map params = new HashMap<>(); - params.put(NEW_CONCEPTS_ONLY, "false"); - params.put(CONCEPTS_PER_TASK, "5"); - params.put(DRY_RUN, "true"); TermServerReport.run(CreateMissingDrugConcepts.class, args, params); } @@ -115,8 +112,8 @@ protected void preInit() throws TermServerScriptException { JobRun jobRun = getJobRun(); newConceptsOnly = jobRun.getParamBoolean(NEW_CONCEPTS_ONLY); - //dryRun = jobRun.getParamBoolean(DRY_RUN); - //taskSize = Integer.parseInt(jobRun.getMandatoryParamValue(CONCEPTS_PER_TASK)); + dryRun = jobRun.getParamBoolean(DRY_RUN); + taskSize = Integer.parseInt(jobRun.getMandatoryParamValue(CONCEPTS_PER_TASK)); } public void postInit() throws TermServerScriptException { @@ -156,13 +153,6 @@ public void postInit() throws TermServerScriptException { suppress.add("Product containing pituitary follicle stimulating hormone (medicinal product)"); suppress.add("Product containing recombinant antihemophilic factor (medicinal product)"); suppress.add("Product containing only adenosine deaminase in parenteral dose form (medicinal product form)"); - suppress.add("Product containing only pibrentasvir (medicinal product)"); - suppress.add("Product containing only bictegravir (medicinal product)"); - suppress.add("Product containing only elbasvir (medicinal product)"); - suppress.add("Product containing only glecaprevir (medicinal product)"); - suppress.add("Product containing only grazoprevir (medicinal product)"); - suppress.add("Product containing only velpatasvir (medicinal product)"); - suppress.add("Product containing only voxilaprevir (medicinal product)"); suppress.add("Product containing only antigen of bacteria and antigen of virus (medicinal product)"); suppress.add("Product containing only antigen of bacteria (medicinal product)"); suppress.add("Product containing only antigen of virus (medicinal product)"); @@ -226,7 +216,7 @@ public int doFix(Task task, Concept concept, String info) throws TermServerScrip for (Concept required : conceptsRequired) { termGenerator.ensureTermsConform(task, required, CharacteristicType.STATED_RELATIONSHIP); required.setDefinitionStatus(DefinitionStatus.FULLY_DEFINED); - report (task, concept, Severity.NONE, ReportActionType.INFO, "Concepts suggests need for :" + required); + report (task, concept, Severity.NONE, ReportActionType.INFO, "Concept suggests need for : " + required.getFsn()); String expression = required.toExpression(CharacteristicType.STATED_RELATIONSHIP); required = createConcept(task, required, info); @@ -238,7 +228,10 @@ public int doFix(Task task, Concept concept, String info) throws TermServerScrip .filter(parent -> parent.getConceptType().equals(invalidParentType)) .map(parent -> parent.toString()) .collect(Collectors.joining(",\n")); - report (task, concept, Severity.LOW, ReportActionType.INFO, "Existing parents considered insufficient: " + (currentParents.isEmpty() ? "None detected" : currentParents)); + //If we don't detect problems with the parents, then don't report anything + if (!currentParents.isEmpty()) { + report(task, concept, Severity.MEDIUM, ReportActionType.VALIDATION_CHECK, "Existing parents problematic due to same drug class level as this concept : " + currentParents); + } } task.addAfter(required, concept); //With the CD reported, we don't actually need to load it in the edit panel @@ -410,7 +403,7 @@ protected List identifyComponentsToProcess() throws TermServerScriptE allAffected.add(c); } } else { - report (SECONDARY_REPORT, (Task)null, c, Severity.LOW, ReportActionType.VALIDATION_CHECK, "No underlying CD detected for MP", mp.toExpression(CharacteristicType.STATED_RELATIONSHIP)); + //report (SECONDARY_REPORT, (Task)null, c, Severity.LOW, ReportActionType.VALIDATION_CHECK, "No underlying CD detected for MP", mp.toExpression(CharacteristicType.STATED_RELATIONSHIP)); } } @@ -430,7 +423,7 @@ protected List identifyComponentsToProcess() throws TermServerScriptE } } } else { - report ((Task)null, c, Severity.LOW, ReportActionType.VALIDATION_CHECK, "No underlying CD detected for MPFO", mpfo.toExpression(CharacteristicType.STATED_RELATIONSHIP)); + //report ((Task)null, c, Severity.LOW, ReportActionType.VALIDATION_CHECK, "No underlying CD detected for MPFO", mpfo.toExpression(CharacteristicType.STATED_RELATIONSHIP)); } } } else if (c.getConceptType().equals(ConceptType.MEDICINAL_PRODUCT)) { diff --git a/reporting-engine-worker/src/main/java/org/ihtsdo/termserver/scripting/reports/InactivatedConcepts.java b/reporting-engine-worker/src/main/java/org/ihtsdo/termserver/scripting/reports/InactivatedConcepts.java index e58bebe892..3fc43980f0 100644 --- a/reporting-engine-worker/src/main/java/org/ihtsdo/termserver/scripting/reports/InactivatedConcepts.java +++ b/reporting-engine-worker/src/main/java/org/ihtsdo/termserver/scripting/reports/InactivatedConcepts.java @@ -71,7 +71,7 @@ public void runJob() throws TermServerScriptException { for (Concept c : scopeAndSort(gl.getAllConcepts())) { boolean reported = false; InactivationIndicator i = c.getInactivationIndicator(); - for (AssociationEntry a : c.getAssociations(ActiveState.ACTIVE, true)) { + for (AssociationEntry a : c.getAssociationEntries(ActiveState.ACTIVE, true)) { String assocType = SnomedUtils.getAssociationType(a); Concept assocValue = gl.getConcept(a.getTargetComponentId()); report (c, i, assocType, assocValue); diff --git a/reporting-engine-worker/src/main/java/org/ihtsdo/termserver/scripting/reports/managedService/ConceptsTranslatedInactivated.java b/reporting-engine-worker/src/main/java/org/ihtsdo/termserver/scripting/reports/managedService/ConceptsTranslatedInactivated.java index 276fab721c..77ab180421 100644 --- a/reporting-engine-worker/src/main/java/org/ihtsdo/termserver/scripting/reports/managedService/ConceptsTranslatedInactivated.java +++ b/reporting-engine-worker/src/main/java/org/ihtsdo/termserver/scripting/reports/managedService/ConceptsTranslatedInactivated.java @@ -86,7 +86,7 @@ public void runJob() throws TermServerScriptException { boolean reported = false; InactivationIndicator i = c.getInactivationIndicator(); String translations = getTranslations(c); - for (AssociationEntry a : c.getAssociations(ActiveState.ACTIVE, true)) { + for (AssociationEntry a : c.getAssociationEntries(ActiveState.ACTIVE, true)) { String assocType = SnomedUtils.getAssociationType(a); Concept assocValue = gl.getConcept(a.getTargetComponentId()); String assocTranslations = getTranslations(assocValue); diff --git a/reporting-engine-worker/src/main/java/org/ihtsdo/termserver/scripting/reports/release/HistoricDataUser.java b/reporting-engine-worker/src/main/java/org/ihtsdo/termserver/scripting/reports/release/HistoricDataUser.java index 8f1c062c7f..5c4933edc4 100644 --- a/reporting-engine-worker/src/main/java/org/ihtsdo/termserver/scripting/reports/release/HistoricDataUser.java +++ b/reporting-engine-worker/src/main/java/org/ihtsdo/termserver/scripting/reports/release/HistoricDataUser.java @@ -67,6 +67,10 @@ protected void loadProjectSnapshot(boolean fsnOnly) throws TermServerScriptExcep if (!StringUtils.isEmpty(getJobRun().getParamValue(THIS_RELEASE))) { compareTwoSnapshots = true; projectKey = getJobRun().getParamValue(THIS_RELEASE); + //Have we got what looks like a zip file but someone left the .zip off? + if (projectKey.contains("T120000") && !projectKey.endsWith(".zip")) { + throw new TermServerScriptException("Suspect release '" + projectKey + "' should end with .zip"); + } //If this release has been specified, the previous must also be, explicitly if (StringUtils.isEmpty(getJobRun().getParamValue(PREV_RELEASE))) { throw new TermServerScriptException("Previous release must be specified if current release is."); diff --git a/reporting-engine-worker/src/main/java/org/ihtsdo/termserver/scripting/reports/release/PreReleaseContentValidation.java b/reporting-engine-worker/src/main/java/org/ihtsdo/termserver/scripting/reports/release/PreReleaseContentValidation.java index f188ac7208..1451f7e5d6 100644 --- a/reporting-engine-worker/src/main/java/org/ihtsdo/termserver/scripting/reports/release/PreReleaseContentValidation.java +++ b/reporting-engine-worker/src/main/java/org/ihtsdo/termserver/scripting/reports/release/PreReleaseContentValidation.java @@ -296,7 +296,7 @@ private void compileSummaryCounts() { String msg = "Active concepts with active historical associations"; initialiseSummaryInformation(msg); for (Concept c : allActiveConceptsSorted) { - if (c.getAssociations(ActiveState.ACTIVE, true).size() > 0) { + if (c.getAssociationEntries(ActiveState.ACTIVE, true).size() > 0) { incrementSummaryInformation(projectKey); } } diff --git a/reporting-engine-worker/src/main/java/org/ihtsdo/termserver/scripting/reports/release/SummaryComponentStats.java b/reporting-engine-worker/src/main/java/org/ihtsdo/termserver/scripting/reports/release/SummaryComponentStats.java index a9a7201adf..c44900a8c3 100644 --- a/reporting-engine-worker/src/main/java/org/ihtsdo/termserver/scripting/reports/release/SummaryComponentStats.java +++ b/reporting-engine-worker/src/main/java/org/ihtsdo/termserver/scripting/reports/release/SummaryComponentStats.java @@ -132,10 +132,10 @@ public static void main(String[] args) throws TermServerScriptException, IOExcep Map params = new HashMap<>(); //params.put(THIS_RELEASE, "SnomedCT_USEditionRF2_PRODUCTION_20220301T120000Z.zip"); //params.put(PREV_RELEASE, "SnomedCT_USEditionRF2_PRODUCTION_20210901T120000Z.zip"); - params.put(PREV_RELEASE, "SnomedCT_InternationalRF2_PRODUCTION_20230430T120000Z.zip"); - params.put(THIS_RELEASE, "SnomedCT_InternationalRF2_PRODUCTION_20230531T120000Z.zip"); - //params.put(REPORT_OUTPUT_TYPES, "S3"); - //params.put(REPORT_FORMAT_TYPE, "JSON"); + params.put(PREV_RELEASE, "SnomedCT_InternationalRF2_PRODUCTION_20230901T120000Z.zip"); + params.put(THIS_RELEASE, "SnomedCT_InternationalRF2_PRODUCTION_20231001T120000Z.zip"); + params.put(REPORT_OUTPUT_TYPES, "S3"); + params.put(REPORT_FORMAT_TYPE, "JSON"); //params.put(MODULES, "731000124108"); TermServerReport.run(SummaryComponentStats.class, args, params); } @@ -302,7 +302,7 @@ private void analyzeConcepts() throws TermServerScriptException { analyzeComponents(isNewConcept, (datum==null?null:datum.relIds), (datum==null?null:datum.relIdsInact), summaryData[TAB_CD], concreteRels); analyzeComponents(isNewConcept, (datum==null?null:datum.axiomIds), (datum==null?null:datum.axiomIdsInact), summaryData[TAB_AXIOMS], c.getAxiomEntries()); analyzeComponents(isNewConcept, (datum==null?null:datum.inactivationIds), (datum==null?null:datum.inactivationIdsInact), summaryData[TAB_INACT_IND], c.getInactivationIndicatorEntries()); - analyzeComponents(isNewConcept, (datum==null?null:datum.histAssocIds), (datum==null?null:datum.histAssocIdsInact), summaryData[TAB_HIST], c.getAssociations(ActiveState.BOTH, true)); + analyzeComponents(isNewConcept, (datum==null?null:datum.histAssocIds), (datum==null?null:datum.histAssocIdsInact), summaryData[TAB_HIST], c.getAssociationEntries(ActiveState.BOTH, true)); List langRefsetEntries = c.getDescriptions().stream() .flatMap(d -> d.getLangRefsetEntries().stream()) .collect(Collectors.toList()); diff --git a/reporting-engine-worker/src/main/java/org/ihtsdo/termserver/scripting/reports/release/SummaryComponentStatsExtensions.java b/reporting-engine-worker/src/main/java/org/ihtsdo/termserver/scripting/reports/release/SummaryComponentStatsExtensions.java index d2503340c7..2451284288 100644 --- a/reporting-engine-worker/src/main/java/org/ihtsdo/termserver/scripting/reports/release/SummaryComponentStatsExtensions.java +++ b/reporting-engine-worker/src/main/java/org/ihtsdo/termserver/scripting/reports/release/SummaryComponentStatsExtensions.java @@ -30,6 +30,9 @@ public static void main(String[] args) throws TermServerScriptException, IOExcep params.put(PREV_RELEASE, "SnomedCT_ManagedServiceNZ_PRODUCTION_NZ1000210_20211001T121212Z.zip"); params.put(PREV_DEPENDENCY, "SnomedCT_InternationalRF2_PRODUCTION_20210731T120000Z.zip"); + + //params.put(MODULES, "21000210109"); //NZ Module + */ params.put(THIS_RELEASE, "SnomedCT_ManagedServiceBE_PRODUCTION_BE1000172_20221115T120000Z.zip"); params.put(THIS_DEPENDENCY, "SnomedCT_InternationalRF2_PRODUCTION_20220930T120000Z.zip"); @@ -38,10 +41,8 @@ public static void main(String[] args) throws TermServerScriptException, IOExcep params.put(PREV_DEPENDENCY, "SnomedCT_InternationalRF2_PRODUCTION_20220131T120000Z.zip"); params.put(MODULES, "11000172109"); // Belgium Module - - params.put(MODULES, "21000210109"); //NZ Module params.put(REPORT_OUTPUT_TYPES, "S3"); - params.put(REPORT_FORMAT_TYPE, "JSON");*/ + params.put(REPORT_FORMAT_TYPE, "JSON"); TermServerReport.run(SummaryComponentStatsExtensions.class, args, params); } diff --git a/reporting-engine-worker/src/main/java/org/ihtsdo/termserver/scripting/reports/release/ValidateInactivationsWithAssociations.java b/reporting-engine-worker/src/main/java/org/ihtsdo/termserver/scripting/reports/release/ValidateInactivationsWithAssociations.java index 6779803108..cf9633cc0b 100644 --- a/reporting-engine-worker/src/main/java/org/ihtsdo/termserver/scripting/reports/release/ValidateInactivationsWithAssociations.java +++ b/reporting-engine-worker/src/main/java/org/ihtsdo/termserver/scripting/reports/release/ValidateInactivationsWithAssociations.java @@ -127,7 +127,7 @@ public void runJob() throws TermServerScriptException { //Ensure that association target is active. To ignore legacy issues, both the //concept AND the target must be legacy - for (AssociationEntry a : c.getAssociations(ActiveState.ACTIVE, true)) { + for (AssociationEntry a : c.getAssociationEntries(ActiveState.ACTIVE, true)) { Concept target = gl.getConcept(a.getTargetComponentId(), false, false); if (target == null) { incrementSummaryInformation("Inactive concept association target not a concept"); @@ -297,7 +297,7 @@ public void runJob() throws TermServerScriptException { } private void validate(Concept c, InactivationIndicatorEntry i, List associationsWithCardinality, Boolean legacy) throws TermServerScriptException { - String data = c.getAssociations(ActiveState.ACTIVE).stream() + String data = c.getAssociationEntries(ActiveState.ACTIVE).stream() .map(h->h.toString()) .collect(Collectors.joining(",\n")); @@ -310,7 +310,7 @@ private void validate(Concept c, InactivationIndicatorEntry i, List SnomedUtils.translateAssociation(h.getRefsetId()).toString() + " " + gl.getConceptSafely(h.getTargetComponentId()).toString()) .collect(Collectors.joining(",\n")); @@ -349,7 +349,7 @@ private String validate(Concept c, InactivationIndicatorEntry i, String cardinal //Special case, we're getting limited inactivations with both SameAs and Was A attributes. Record stats, but skip if (i.getInactivationReasonId().equals(SCTID_INACT_LIMITED) || i.getInactivationReasonId().equals(SCTID_INACT_MOVED_ELSEWHERE)) { - String typesOfAssoc = c.getAssociations(ActiveState.ACTIVE).stream() + String typesOfAssoc = c.getAssociationEntries(ActiveState.ACTIVE).stream() .map(h->SnomedUtils.translateAssociation(h.getRefsetId()).toString()) .collect(Collectors.joining(", ")); typesOfAssoc = typesOfAssoc.isEmpty()? "No associations" : typesOfAssoc; @@ -358,8 +358,8 @@ private String validate(Concept c, InactivationIndicatorEntry i, String cardinal } //First check cardinality - int assocCount = c.getAssociations(ActiveState.ACTIVE, assocId).size(); - int allAssocCount = c.getAssociations(ActiveState.ACTIVE, true).size(); + int assocCount = c.getAssociationEntries(ActiveState.ACTIVE, assocId).size(); + int allAssocCount = c.getAssociationEntries(ActiveState.ACTIVE, true).size(); if (assocCount > 0 && assocCount != allAssocCount && mutuallyExclusive) { return inactStr + " inactivation's " + reqAssocStr + " association is mutually exclusive with other associations types."; @@ -373,7 +373,7 @@ private String validate(Concept c, InactivationIndicatorEntry i, String cardinal return CARDINALITY_ISSUE; } else { //Now check association is appropriate for the inactivation indicator used - for (AssociationEntry h : c.getAssociations(ActiveState.ACTIVE, true)) { + for (AssociationEntry h : c.getAssociationEntries(ActiveState.ACTIVE, true)) { String assocStr = SnomedUtils.translateAssociation(h.getRefsetId()).toString(); Concept target = gl.getConcept(h.getTargetComponentId()); //Only the "MovedTo" historical association should point to a Namespace Concept diff --git a/reporting-engine-worker/src/main/resources/application.properties b/reporting-engine-worker/src/main/resources/application.properties index 7c1bde834a..e31aa0f03f 100644 --- a/reporting-engine-worker/src/main/resources/application.properties +++ b/reporting-engine-worker/src/main/resources/application.properties @@ -33,7 +33,7 @@ resources.cloud.path=prod/international #AWS Credentials should be blank when running in an EC2 instance aws.key = -aws.privateKey = +aws.secretKey = logging.level.org.ihtsdo=DEBUG logging.level.ch.qos.logback.classic.joran=WARN diff --git a/schedule-manager/pom.xml b/schedule-manager/pom.xml index f9d11bc724..79f627c30f 100644 --- a/schedule-manager/pom.xml +++ b/schedule-manager/pom.xml @@ -6,7 +6,7 @@ org.snomed.otf - 7.2.1 + 7.3.0-SNAPSHOT reporting-engine diff --git a/script-engine/pom.xml b/script-engine/pom.xml index 1816fe3e87..b54cac5022 100644 --- a/script-engine/pom.xml +++ b/script-engine/pom.xml @@ -6,7 +6,7 @@ org.snomed.otf - 7.2.1 + 7.3.0-SNAPSHOT reporting-engine diff --git a/script-engine/src/main/java/org/ihtsdo/termserver/scripting/GraphLoader.java b/script-engine/src/main/java/org/ihtsdo/termserver/scripting/GraphLoader.java index b7a179e8ee..e3c0b1d357 100644 --- a/script-engine/src/main/java/org/ihtsdo/termserver/scripting/GraphLoader.java +++ b/script-engine/src/main/java/org/ihtsdo/termserver/scripting/GraphLoader.java @@ -600,8 +600,8 @@ public void addRelationshipToConcept(CharacteristicType charType, Relationship r } } - public Concept getConcept(String identifier) throws TermServerScriptException { - return getConcept(identifier.trim(), true, true); + public Concept getConcept(String sctid) throws TermServerScriptException { + return getConcept(sctid.trim(), true, true); } public Concept getConceptSafely (String identifier) { diff --git a/script-engine/src/main/java/org/ihtsdo/termserver/scripting/TermServerScript.java b/script-engine/src/main/java/org/ihtsdo/termserver/scripting/TermServerScript.java index 9e5a1a5091..b369fed56e 100644 --- a/script-engine/src/main/java/org/ihtsdo/termserver/scripting/TermServerScript.java +++ b/script-engine/src/main/java/org/ihtsdo/termserver/scripting/TermServerScript.java @@ -1409,6 +1409,7 @@ private synchronized void recordSummaryText(String msg) { try { //Split the colon into it's own column (unless it's a time stamp!) if (msg.contains(":") + && !msg.contains("http") && !msg.contains("at: ") && !msg.contains("\"") && !msg.contains("Completed processing in")) { @@ -1502,9 +1503,9 @@ public String getReportName() { protected String getPrettyHistoricalAssociation (Concept c) throws TermServerScriptException { String prettyString = "No association specified."; - if (c.getAssociations(ActiveState.ACTIVE).size() > 0) { + if (c.getAssociationEntries(ActiveState.ACTIVE).size() > 0) { prettyString = " "; - for (AssociationEntry assoc : c.getAssociations(ActiveState.ACTIVE)) { + for (AssociationEntry assoc : c.getAssociationEntries(ActiveState.ACTIVE)) { prettyString += SnomedUtils.deconstructFSN(gl.getConcept(assoc.getRefsetId()).getFsn())[0].replace(" association reference set", ""); prettyString += " -> "; prettyString += gl.getConcept(assoc.getTargetComponentId()); diff --git a/script-engine/src/main/java/org/ihtsdo/termserver/scripting/client/TermServerClient.java b/script-engine/src/main/java/org/ihtsdo/termserver/scripting/client/TermServerClient.java index 9d16e7f9f9..9528b8882f 100644 --- a/script-engine/src/main/java/org/ihtsdo/termserver/scripting/client/TermServerClient.java +++ b/script-engine/src/main/java/org/ihtsdo/termserver/scripting/client/TermServerClient.java @@ -39,8 +39,8 @@ import com.google.gson.GsonBuilder; public class TermServerClient { - - public enum ExtractType { + + public enum ExtractType { DELTA, SNAPSHOT, FULL } @@ -199,8 +199,11 @@ public Concept updateConcept(Concept c, String branchPath) throws TermServerScri } public Concept getConcept(String sctid, String branchPath) throws TermServerScriptException { - String url = getConceptBrowserPath(branchPath) + "/" + sctid; - return restTemplate.getForObject(url, Concept.class); + String url = getConceptBrowserPath(branchPath) + "/" + sctid; + Concept concept = restTemplate.getForObject(url, Concept.class); + concept.setId(concept.getConceptId()); //RestTemplate is not calling the setter so this is cheaper + //than writing a custom deserializer + return concept; } public Description getDescription(String descriptionId, String branchPath) { @@ -880,4 +883,15 @@ public static String getParentBranchPath(String branchPath) { return branchPath.substring(0, endIndex); } + public boolean adminRollbackCommit(Branch b) throws TermServerScriptException { + String url = this.url + "/admin/" + b.getPath() + "/actions/rollback-commit?commitHeadTime=" + b.getHeadTimestamp(); + HttpEntity> entity = new HttpEntity<>(null); + try { + ResponseEntity response = restTemplate.exchange(url, HttpMethod.POST, entity, Object.class); + return response.getStatusCode().is2xxSuccessful(); + } catch (RestClientException e) { + throw new TermServerScriptException(translateRestClientException(e)); + } + } + } diff --git a/script-engine/src/main/java/org/ihtsdo/termserver/scripting/dao/ReportDataBroker.java b/script-engine/src/main/java/org/ihtsdo/termserver/scripting/dao/ReportDataBroker.java index 1fa9c57534..b32190b1b6 100644 --- a/script-engine/src/main/java/org/ihtsdo/termserver/scripting/dao/ReportDataBroker.java +++ b/script-engine/src/main/java/org/ihtsdo/termserver/scripting/dao/ReportDataBroker.java @@ -67,7 +67,8 @@ public void upload(File outputFile, String data) throws TermServerScriptExceptio //In case we're running on a PC we need to convert backslashes to forward String filePath = outputFile.getPath().replaceAll("\\\\", "/"); InputStream is = IOUtils.toInputStream(data, StandardCharsets.UTF_8); - LOGGER.info("Uploading to S3: " + outputFile); + s3Manager.getResourceManager().getBucketNamePath(); + LOGGER.info("Uploading to S3 ({}): {}", s3Manager.getResourceManager().getBucketNamePath(), outputFile); s3Manager.getResourceManager().writeResource(filePath, is); } catch (Exception e) { throw new TermServerScriptException(e); diff --git a/script-engine/src/main/java/org/ihtsdo/termserver/scripting/delta/DeltaGenerator.java b/script-engine/src/main/java/org/ihtsdo/termserver/scripting/delta/DeltaGenerator.java index fd31db68cf..89c8b0a761 100644 --- a/script-engine/src/main/java/org/ihtsdo/termserver/scripting/delta/DeltaGenerator.java +++ b/script-engine/src/main/java/org/ihtsdo/termserver/scripting/delta/DeltaGenerator.java @@ -147,10 +147,15 @@ protected void checkSettingsWithUser(JobRun jobRun) throws TermServerScriptExcep if (!projectName.endsWith(".zip")){ try{ scaClient = new AuthoringServicesClient(url, authenticatedCookie); - project = scaClient.getProject(projectName); - if (project.getBranchPath().contains("SNOMEDCT-") - && project.getMetadata() != null) { - moduleId = project.getMetadata().getDefaultModuleId(); + //MAIN is not a project, but we know what the default module is + if (projectName.toUpperCase().equals("MAIN")) { + moduleId = SCTID_CORE_MODULE; + } else { + project = scaClient.getProject(projectName); + if (project.getBranchPath().contains("SNOMEDCT-") + && project.getMetadata() != null) { + moduleId = project.getMetadata().getDefaultModuleId(); + } } } catch (Exception e) { LOGGER.error("Failed to retrieve project metadata for " + projectName, e); @@ -470,14 +475,17 @@ protected int createOutputArchive() throws TermServerScriptException { } protected int createOutputArchive(boolean outputModifiedComponents) throws TermServerScriptException { + return createOutputArchive(outputModifiedComponents, 0); + } + + protected int createOutputArchive(boolean outputModifiedComponents, int conceptsOutput) throws TermServerScriptException { if (dryRun) { String msg = "Dry run, skipping archive creation"; LOGGER.info(msg); report((Concept) null, Severity.NONE, ReportActionType.INFO, msg); } else { - int conceptsOutput = NOT_SET; if (outputModifiedComponents) { - conceptsOutput = outputModifiedComponents(true); + conceptsOutput += outputModifiedComponents(true); } getRF2Manager().flushFiles(true); //Just flush the RF2, we might want to keep the report going File archive = SnomedUtils.createArchive(new File(outputDirName)); diff --git a/script-engine/src/main/java/org/ihtsdo/termserver/scripting/delta/ExtractExtensionComponents.java b/script-engine/src/main/java/org/ihtsdo/termserver/scripting/delta/ExtractExtensionComponents.java index 85c89e7918..62a38fe11a 100644 --- a/script-engine/src/main/java/org/ihtsdo/termserver/scripting/delta/ExtractExtensionComponents.java +++ b/script-engine/src/main/java/org/ihtsdo/termserver/scripting/delta/ExtractExtensionComponents.java @@ -56,8 +56,8 @@ public class ExtractExtensionComponents extends DeltaGenerator { protected boolean copyInferredRelationshipsToStatedWhereMissing = true; public static void main(String[] args) throws TermServerScriptException, IOException, InterruptedException { - //ExtractExtensionComponents delta = new ExtractExtensionComponents(); - ExtractExtensionComponents delta = new ExtractExtensionComponentsAndLateralize(); + ExtractExtensionComponents delta = new ExtractExtensionComponents(); + // ExtractExtensionComponents delta = new ExtractExtensionComponentsAndLateralize(); try { ReportSheetManager.targetFolderId = "12ZyVGxnFVXZfsKIHxr3Ft2Z95Kdb7wPl"; //Extract and Promote delta.runStandAlone = false; @@ -1027,14 +1027,19 @@ private boolean moveDescriptionToTargetModule(Description d, Concept conceptOnTS if (doShiftDescription || StringUtils.isEmpty(usEntry.getEffectiveTime())) { usEntry.setModuleId(targetModuleId); usEntry.setDirty(); - if (!usGbVariance && d.getLangRefsetEntries(ActiveState.ACTIVE, GB_ENG_LANG_REFSET).size() ==0) { + if (!usGbVariance && d.getLangRefsetEntries(ActiveState.ACTIVE, GB_ENG_LANG_REFSET).size() == 0) { //Might be an inactive GB refset entry that we can reuse + //In which case, bring it into line with the US value List gbInactiveLangRefs = d.getLangRefsetEntries(ActiveState.INACTIVE, GB_ENG_LANG_REFSET); if (gbInactiveLangRefs.size() > 0) { + LOGGER.debug("Reactivating GB langrefset entry " + d.getDescriptionId()); LangRefsetEntry gbEntry = gbInactiveLangRefs.get(0); gbEntry.setActive(true); gbEntry.setEffectiveTime(null); + gbEntry.setModuleId(targetModuleId); + gbEntry.setAcceptabilityId(usEntry.getAcceptabilityId()); gbEntry.setDirty(); + d.addLangRefsetEntry(gbEntry); } else { //Otherwise, clone the US entry to use as GB LOGGER.debug("Adding gb entry cloned from US " + d.getDescriptionId()); @@ -1054,7 +1059,6 @@ private boolean moveDescriptionToTargetModule(Description d, Concept conceptOnTS // This ensures existing langrefset entries are re-used if present, or a new one is created. d.setAcceptability(US_ENG_LANG_REFSET, SnomedUtils.translateAcceptability(gbEntry.getAcceptabilityId())); } - } } diff --git a/script-engine/src/main/java/org/ihtsdo/termserver/scripting/delta/GroupSelfGroupedAttributes.java b/script-engine/src/main/java/org/ihtsdo/termserver/scripting/delta/GroupSelfGroupedAttributes.java index 0d77c1974a..075b8f669b 100644 --- a/script-engine/src/main/java/org/ihtsdo/termserver/scripting/delta/GroupSelfGroupedAttributes.java +++ b/script-engine/src/main/java/org/ihtsdo/termserver/scripting/delta/GroupSelfGroupedAttributes.java @@ -28,8 +28,8 @@ public class GroupSelfGroupedAttributes extends DeltaGenerator implements Script Concept COMPONENT; - //private final int BatchSize = 25; - private final int BatchSize = 99999; + private final int BatchSize = 25; + //private final int BatchSize = 99999; public static void main(String[] args) throws TermServerScriptException, IOException, InterruptedException { GroupSelfGroupedAttributes delta = new GroupSelfGroupedAttributes(); @@ -47,7 +47,7 @@ public static void main(String[] args) throws TermServerScriptException, IOExcep } public void postInit() throws TermServerScriptException { - hierarchies.add(OBSERVABLE_ENTITY); + //hierarchies.add(OBSERVABLE_ENTITY); hierarchies.add(gl.getConcept("386053000 |Evaluation procedure|")); skipAttributeTypes.add(gl.getConcept("363702006 |Has focus (attribute)|")); @@ -95,8 +95,12 @@ private void process() throws ValidationFailure, TermServerScriptException, IOEx outputRF2(c, true); conceptsInThisBatch++; if (conceptsInThisBatch >= BatchSize) { + createOutputArchive(false, conceptsInThisBatch); + gl.setAllComponentsClean(); + outputDirName = "output"; //Reset so we don't end up with _1_1_1 + initialiseOutputDirectory(); + initialiseFileHeaders(); conceptsInThisBatch = 0; - createOutputArchive(false); } } } diff --git a/script-engine/src/main/java/org/ihtsdo/termserver/scripting/delta/UpdateHistoricalAssociationsDriven.java b/script-engine/src/main/java/org/ihtsdo/termserver/scripting/delta/UpdateHistoricalAssociationsDriven.java index 0e7de8d1a8..42ffde4cf0 100644 --- a/script-engine/src/main/java/org/ihtsdo/termserver/scripting/delta/UpdateHistoricalAssociationsDriven.java +++ b/script-engine/src/main/java/org/ihtsdo/termserver/scripting/delta/UpdateHistoricalAssociationsDriven.java @@ -1,13 +1,12 @@ package org.ihtsdo.termserver.scripting.delta; -import java.io.File; import java.io.IOException; import java.nio.charset.Charset; import java.nio.file.Files; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; +import java.util.stream.Collectors; +import org.ihtsdo.otf.RF2Constants; import org.ihtsdo.otf.exception.TermServerScriptException; import org.ihtsdo.termserver.scripting.ValidationFailure; import org.ihtsdo.termserver.scripting.domain.*; @@ -24,29 +23,62 @@ public class UpdateHistoricalAssociationsDriven extends DeltaGenerator implement private static final Logger LOGGER = LoggerFactory.getLogger(UpdateHistoricalAssociationsDriven.class); - private Map replacementMap = new HashMap<>(); + private Map replacementMap = new HashMap<>(); + private Map cathyNotes = new HashMap<>(); + private Set reportedAsAdditional = new HashSet<>(); + + private List targetPrefixes = Arrays.asList(new String[] {"OE ", "CO ", "O/E", "C/O", "Complaining of", "On examination"}); + + private List manageManually = Arrays.asList(new String[]{ + "163092009","163094005","163096007","164018003","275284008", + "271879001","164075007","162952003","164398007","164346005", + "141331007","141369002","141751009","140614001","140514005", + "140605006","140761009","141666001","141492001","141849001", + "141853004","141870007","141874003","141857003","141864001", + "141819003","140962005","163432001" + }); public static void main(String[] args) throws TermServerScriptException, IOException, InterruptedException { UpdateHistoricalAssociationsDriven delta = new UpdateHistoricalAssociationsDriven(); try { - ReportSheetManager.targetFolderId = "1fIHGIgbsdSfh5euzO3YKOSeHw4QHCM-m"; //Ad-Hoc Batch Updates + ReportSheetManager.targetFolderId = "13XiH3KVll3v0vipVxKwWjjf-wmjzgdDe"; //Technical Specialist delta.newIdsRequired = false; delta.init(args); delta.loadProjectSnapshot(false); delta.postInit(); delta.process(); - delta.flushFiles(false); //Need to flush files before zipping - SnomedUtils.createArchive(new File(delta.outputDirName)); + delta.createOutputArchive(); } finally { delta.finish(); } } + public void postInit() throws TermServerScriptException { + String[] columnHeadings = new String[]{ + "SCTID, FSN, SemTag, Severity, Action, Details, Details, Cathy Notes, , ", + "Issue, Detail", + "SCTID, FSN, SemTag, Notes, Replacement Mismatch, Existing Inact / HistAssoc, Sibling Lexical Match, Sibling Active, Sibling Already in Delta, Sibling HistAssoc, Cousin Lexical Match, Cousin Active, Cousin HistAssoc, Cathy Notes, " + }; + + String[] tabNames = new String[]{ + "Delta Records Created", + "Other processing issues", + "Additional Inactive Concepts" + }; + postInit(tabNames, columnHeadings, false); + } + private void process() throws ValidationFailure, TermServerScriptException { - + + populateCathyNotes(); populateReplacementMap(); - + populateUpdatedReplacementMap(); + checkForRecentInactivations(); + for (Concept c : SnomedUtils.sort(gl.getAllConcepts())) { + /*if (!c.getId().equals("164427005")) { + continue; + }*/ //Is this a concept we've been told to replace the associations on? if (replacementMap.containsKey(c)) { if (!c.isActive()) { @@ -64,13 +96,20 @@ private void process() throws ValidationFailure, TermServerScriptException { report(c, Severity.MEDIUM, ReportActionType.NO_CHANGE, assocStr, "Supplied: " + replacementMap.get(c)); continue; } - - if (!prevInactValue.equals(InactivationIndicator.NONCONFORMANCE_TO_EDITORIAL_POLICY)) { - i.setInactivationReasonId(SCTID_INACT_NON_CONFORMANCE); + + //Have we got some specific inactivation indicator we're expecting to use? + InactivationIndicator newII = InactivationIndicator.NONCONFORMANCE_TO_EDITORIAL_POLICY; + UpdateAction action = replacementMap.get(c); + if (action != null && action.inactivationIndicator != null) { + newII = action.inactivationIndicator; + } + + if (!action.inactivationIndicator.equals(prevInactValue)) { + i.setInactivationReasonId(SnomedUtils.translateInactivationIndicator(newII)); i.setEffectiveTime(null); //Will mark as dirty - report(c, Severity.LOW, ReportActionType.INACT_IND_MODIFIED, prevInactValue + " --> NCEP"); + report(c, Severity.LOW, ReportActionType.INACT_IND_MODIFIED, prevInactValue + " --> " + newII); } - + replaceHistoricalAssociations(c); } else { @@ -81,42 +120,370 @@ private void process() throws ValidationFailure, TermServerScriptException { } private void replaceHistoricalAssociations(Concept c) throws TermServerScriptException { + + UpdateAction action = replacementMap.get(c); + List updatedReplacements = new ArrayList<>(action.replacements); + for (Concept replacement : action.replacements) { + //Check that our replacement is still active + if (!replacement.isActive()) { + report(c, Severity.HIGH, ReportActionType.VALIDATION_CHECK, "Originally suggested replacement is now inactive", replacement); + Concept updatedReplacement = null; + try { + updatedReplacement = getReplacement(PRIMARY_REPORT, c, replacement, false); + } catch (Exception e) { + report(c, Severity.HIGH, ReportActionType.VALIDATION_ERROR, "Failed to find replacement for inactive replacement" + replacement, e.getMessage()); + } + if (updatedReplacement == null || !updatedReplacement.isActive()) { + return; + } + updatedReplacements.remove(replacement); + updatedReplacements.add(updatedReplacement); + } + } + action.replacements = updatedReplacements; + String prevAssocStr = SnomedUtils.prettyPrintHistoricalAssociations(c, gl); for (AssociationEntry a : c.getAssociationEntries()) { if (a.getRefsetId().equals(SCTID_ASSOC_REPLACED_BY_REFSETID)) { report(c, Severity.HIGH, ReportActionType.VALIDATION_ERROR, "Concept already using a ReplacedBy association"); return; } - a.setActive(false); - a.setEffectiveTime(null); + a.setActive(false); //This will reset effective time and mark as dirty + } + + String associationTypeSCTID = SnomedUtils.translateAssociation(action.type); + for (Concept replacement : action.replacements) { + AssociationEntry assoc = AssociationEntry.create(c, associationTypeSCTID, replacement); + assoc.setDirty(); + c.getAssociationEntries().add(assoc); } - Concept replacement = replacementMap.get(c); - AssociationEntry assoc = AssociationEntry.create(c, SCTID_ASSOC_REPLACED_BY_REFSETID, replacement); - c.getAssociationEntries().add(assoc); String newAssocStr = SnomedUtils.prettyPrintHistoricalAssociations(c, gl); report(c, Severity.LOW, ReportActionType.ASSOCIATION_CHANGED, prevAssocStr, newAssocStr); } private void populateReplacementMap() throws TermServerScriptException { + int lineNo = 0; try { for (String line : Files.readAllLines(getInputFile().toPath(), Charset.defaultCharset())) { try { + lineNo++; String[] items = line.split(TAB); Concept inactive = gl.getConcept(items[0]); Concept replacement = gl.getConcept(items[2]); - Concept existing = replacementMap.get(inactive); - if (existing != null && !existing.equals(replacement)) { - LOGGER.warn("Map replacement for inactive " + inactive + " was " + existing + " now " + replacement); + UpdateAction alreadyMapped = replacementMap.get(inactive); + UpdateAction thisMapping = createReplacementAssociation(replacement); + if (alreadyMapped != null && !alreadyMapped.equals(replacement)) { + report(SECONDARY_REPORT,"Map replacement for inactive " + inactive + " already seen. Was " + alreadyMapped + " now " + thisMapping); } - replacementMap.put(inactive, replacement); + replacementMap.put(inactive, thisMapping); } catch (Exception e) { - LOGGER.warn("Failed to parse line: " + line); + report(SECONDARY_REPORT,"Failed to parse line (1st file) at line " + lineNo + ": " + line + " due to " + e.getMessage()); } } } catch (Exception e) { throw new TermServerScriptException(e); } - + } + + private void populateUpdatedReplacementMap() throws TermServerScriptException { + int lineNo = 0; + try { + for (String line : Files.readAllLines(getInputFile(1).toPath(), Charset.defaultCharset())) { + try { + lineNo++; + String[] items = line.split(TAB); + Concept inactive = gl.getConcept(items[0]); + String inactivationIndicatorStr = items[2]; + String replacementsStr = items[3]; + InactivationIndicator inactivationIndicator = InactivationIndicator.NONCONFORMANCE_TO_EDITORIAL_POLICY; + Association association = Association.REPLACED_BY; + if (inactivationIndicatorStr.contains("Ambiguous")) { + inactivationIndicator = InactivationIndicator.AMBIGUOUS; + association = Association.POSS_EQUIV_TO; + } + List replacements = splitConcepts(replacementsStr); + UpdateAction action = new UpdateAction(); + action.inactivationIndicator = inactivationIndicator; + action.replacements = replacements; + action.type = association; + replacementMap.put(inactive, action); + } catch (Exception e) { + report(SECONDARY_REPORT,"Failed to parse line (2nd file) at line " + lineNo + ": " + line + " due to " + e.getMessage()); + } + } + } catch (Exception e) { + throw new TermServerScriptException(e); + } + } + + private void populateCathyNotes() throws TermServerScriptException { + int lineNo = 0; + try { + for (String line : Files.readAllLines(getInputFile(2).toPath(), Charset.defaultCharset())) { + lineNo++; + String[] items = line.split(TAB); + Concept c = gl.getConcept(items[0]); + String notes = items[1]; + if (items.length == 3) { + notes += " " + items[2]; + } + if (cathyNotes.containsKey(c)) { + //If we're saying the same thing, it's cool + if (cathyNotes.get(c).equals(notes)) { + continue; + } + notes = cathyNotes.get(c) + " + " + notes; + } + cathyNotes.put(c, notes); + } + } catch (Exception e) { + throw new TermServerScriptException("Failed to read " + getInputFile(2) + " at line " + lineNo,e); + } + } + + private List splitConcepts(String replacementsStr) { + String[] conceptIds = replacementsStr.split(","); + return Arrays.stream(conceptIds).map(s -> gl.getConceptSafely(s)).collect(Collectors.toList()); + } + + private void checkForRecentInactivations() throws TermServerScriptException { + for (Concept c : SnomedUtils.sort(gl.getAllConcepts())) { + if (c.getFSNDescription() == null) { + LOGGER.warn("Unpopulated " + c.getId()); + continue; + } + if (c.isActive() || + !inScope(c) || + manageManually.contains(c.getId()) || + replacementMap.containsKey(c) || + reportedAsAdditional.contains(c)) { + continue; + } + String assocStr = SnomedUtils.prettyPrintHistoricalAssociations(c, gl, true); + String notes = ""; + //For 'NOS' concepts, we may pull them into our main processing set + if (c.getFsn().contains("NOS")) { + notes = checkForProcessInclusionNOS(c); + } else if (c.FSN.contains("context-dependent")) { + notes = checkForProcessInclusion(c, true); + } else if (c.FSN.contains("[")) { + notes = checkForProcessInclusion(c, false); + } + + //Can we find a sibling for this concept - same text with different prefix? + Concept sibling = findSibling(c); + String[] siblingData = new String[] {"", "", "", ""}; + if (sibling != null) { + reportedAsAdditional.add(sibling); + siblingData[0] = sibling.toString(); + siblingData[1] = sibling.isActive()?"Y":"N"; + siblingData[2] = replacementMap.containsKey(sibling)?"Y":"N"; + siblingData[3] = SnomedUtils.prettyPrintHistoricalAssociations(sibling, gl, true); + } + + Concept cousin = findCousin(c); + String[] cousinData = new String[] {"", "", ""}; + if (cousin != null) { + cousinData[0] = cousin.toString(); + cousinData[1] = cousin.isActive()?"Y":"N"; + cousinData[2] = SnomedUtils.prettyPrintHistoricalAssociations(cousin, gl, true); + } + String mismatchFlag = hasMismatchedAssociations(c, sibling, cousin)?"Y":"N"; + report(TERTIARY_REPORT, c, notes, mismatchFlag, assocStr, siblingData, cousinData); + } + } + + private boolean hasMismatchedAssociations(Concept... concepts) throws TermServerScriptException { + //Do these concepts all have the same historical associations? + Set targets = new HashSet<>(); + for (Concept c : concepts) { + if (c == null) { + continue; + } + if (targets.isEmpty()) { + targets = SnomedUtils.getHistoricalAssocationTargets(c, gl); + } else { + Set theseTargets = SnomedUtils.getHistoricalAssocationTargets(c, gl); + if (!targets.equals(theseTargets)) { + return true; + } + } + } + return false; + } + + private String checkForProcessInclusionNOS(Concept c) { + //Are we already processing this concept? + if (replacementMap.containsKey(c)) { + return "Already included for processing"; + } + //Do we have one association target or more? + List assocTargets = c.getAssociationEntries(ActiveState.ACTIVE, RF2Constants.SCTID_ASSOC_POSS_EQUIV_REFSETID).stream() + .map(a -> gl.getConceptSafely(a.getTargetComponentId())) + .collect(Collectors.toList()); + + UpdateAction action = null; + if (assocTargets.size() == 1) { + action = new UpdateAction() + .withAssociationType(Association.REPLACED_BY) + .withInactivationIndicator(InactivationIndicator.CLASSIFICATION_DERIVED_COMPONENT) + .withReplacements(assocTargets); + } else if (assocTargets.size() > 1) { + action = new UpdateAction() + .withAssociationType(Association.PARTIALLY_EQUIV_TO) + .withInactivationIndicator(InactivationIndicator.CLASSIFICATION_DERIVED_COMPONENT) + .withReplacements(assocTargets); + } else { + return "Not included for processing due to lack of existing associations"; + } + replacementMap.put(c, action); + return "Included for processing - CDC"; + } + + private String checkForProcessInclusion(Concept c, boolean allowMultipleTargets) { + //The CDC here stands for Context Dependent Category + //Are we already processing this concept? + if (replacementMap.containsKey(c)) { + return "Already included for processing"; + } + + if (!c.getInactivationIndicator().equals(InactivationIndicator.AMBIGUOUS)) { + return "Not included for processing - inactivation indicator is not Ambiguous"; + } + //Do we have one association target or more? + List assocTargets = c.getAssociationEntries(ActiveState.ACTIVE, RF2Constants.SCTID_ASSOC_POSS_EQUIV_REFSETID).stream() + .map(a -> gl.getConceptSafely(a.getTargetComponentId())) + .collect(Collectors.toList()); + + UpdateAction action = null; + if (assocTargets.size() == 1) { + action = new UpdateAction() + .withAssociationType(Association.REPLACED_BY) + .withInactivationIndicator(InactivationIndicator.NONCONFORMANCE_TO_EDITORIAL_POLICY) + .withReplacements(assocTargets); + } else if (assocTargets.size() > 1) { + if (allowMultipleTargets) { + action = new UpdateAction() + .withAssociationType(Association.PARTIALLY_EQUIV_TO) + .withInactivationIndicator(InactivationIndicator.NONCONFORMANCE_TO_EDITORIAL_POLICY) + .withReplacements(assocTargets); + } else { + return "Not included for processing due to multiple existing associations"; + } + } else { + return "Not included for processing due to lack of existing associations"; + } + replacementMap.put(c, action); + return "Included for processing - NCEP"; + } + + //A cousin is the same basic FSN, but with a _different_ prefix + private Concept findSibling(Concept c) throws TermServerScriptException { + String rootFsn = "Unknown"; + String thisPrefix = "Unknown"; + for (String targetPrefix : targetPrefixes) { + if (c.getFsn().startsWith(targetPrefix)) { + rootFsn = c.getFsn().replace(targetPrefix, "").replace("- ", "").trim(); + rootFsn = SnomedUtils.deconstructFSN(rootFsn)[0]; + thisPrefix = targetPrefix; + break; + } + } + + for (Concept sibling : gl.getAllConcepts()) { + for (Description d : sibling.getDescriptions()) { + String thisTerm = d.getTerm(); + if (d.getType().equals(DescriptionType.FSN)) { + thisTerm = SnomedUtils.deconstructFSN(thisTerm, true)[0]; + } + thisTerm = thisTerm.replace("-", "").replaceAll(" ", " "); + //Check for all alternative prefixes + for (String siblingPrefix : targetPrefixes) { + if (siblingPrefix.equals(thisPrefix)) { + continue; + } + + String targetFsn = siblingPrefix + " " + rootFsn; + targetFsn = targetFsn.replace(" ", " ").trim(); + + if (targetFsn.equalsIgnoreCase(thisTerm)) { + return sibling; + } + } + } + } + return null; + } + + //A cousin is the same basic FSN, but without any prefix + private Concept findCousin(Concept c) throws TermServerScriptException { + String targetFsn = "Unknown"; + for (String targetPrefix : targetPrefixes) { + if (c.getFsn().startsWith(targetPrefix)) { + targetFsn = c.getFsn().replace(targetPrefix, "").replace("- ", "").trim(); + targetFsn = SnomedUtils.deconstructFSN(targetFsn)[0]; + break; + } + } + + for (Concept cousin : gl.getAllConcepts()) { + for (Description d : cousin.getDescriptions()) { + if (targetFsn.equalsIgnoreCase(d.getTerm())) { + return cousin; + } + } + } + return null; + } + + private boolean inScope(Concept c) { + for (String targetPrefix : targetPrefixes) { + if (c.getFsn().startsWith(targetPrefix)) { + return true; + } + } + return false; + } + + private UpdateAction createReplacementAssociation(Concept replacement) { + UpdateAction action = new UpdateAction(); + action.type = Association.REPLACED_BY; + action.inactivationIndicator = InactivationIndicator.NONCONFORMANCE_TO_EDITORIAL_POLICY; + action.replacements = Arrays.asList(replacement); + return action; + } + + protected boolean report (Concept c, Object...details) throws TermServerScriptException { + if (cathyNotes.containsKey(c)) { + return report(PRIMARY_REPORT, c, details, cathyNotes.get(c)); + } else { + return report(PRIMARY_REPORT, c, details); + } + } + + class UpdateAction { + Association type; + InactivationIndicator inactivationIndicator; + List replacements; + + public String toString() { + return inactivationIndicator + ": " + type + " " + replacements.stream().map(c -> c.toString()).collect(Collectors.joining(", ")); + } + + public UpdateAction withInactivationIndicator(InactivationIndicator inactivationIndicator) { + this.inactivationIndicator = inactivationIndicator; + return this; + } + + public UpdateAction withReplacements(List replacements) { + this.replacements = replacements; + return this; + } + + public UpdateAction withAssociationType(Association type) { + this.type = type; + return this; + } } } diff --git a/script-engine/src/main/java/org/ihtsdo/termserver/scripting/domain/AssociationEntry.java b/script-engine/src/main/java/org/ihtsdo/termserver/scripting/domain/AssociationEntry.java index 934d672dab..916d5e50f9 100644 --- a/script-engine/src/main/java/org/ihtsdo/termserver/scripting/domain/AssociationEntry.java +++ b/script-engine/src/main/java/org/ihtsdo/termserver/scripting/domain/AssociationEntry.java @@ -143,6 +143,7 @@ public String[] getAdditionalFieldNames() { public static AssociationEntry create(Concept c, String refsetId, Concept replacement) { AssociationEntry a = new AssociationEntry(); + a.setId(UUID.randomUUID().toString()); a.setModuleId(c.getModuleId()); a.setRefsetId(refsetId); a.setReferencedComponentId(c.getId()); diff --git a/script-engine/src/main/java/org/ihtsdo/termserver/scripting/domain/Batch.java b/script-engine/src/main/java/org/ihtsdo/termserver/scripting/domain/Batch.java index 8686eb6830..2366796a36 100644 --- a/script-engine/src/main/java/org/ihtsdo/termserver/scripting/domain/Batch.java +++ b/script-engine/src/main/java/org/ihtsdo/termserver/scripting/domain/Batch.java @@ -30,15 +30,19 @@ public Batch(String fileName, GraphLoader gl) { this(fileName); this.gl = gl; } + + public Task addNewTask(String author) { + return addNewTask(author, null); + } - public Task addNewTask(String[] author_reviewer) { - Task task = new Task(this, author_reviewer); + public Task addNewTask(String author, String reviewer) { + Task task = new Task(this, author, reviewer); tasks.add(task); return task; } - public Task insertNewTask(Task after, String[] author_reviewer) { - Task task = new Task(this, author_reviewer); + public Task insertNewTask(Task after, String author, String reviewer) { + Task task = new Task(this, author, reviewer); tasks.add(tasks.indexOf(after), task); return task; } diff --git a/script-engine/src/main/java/org/ihtsdo/termserver/scripting/domain/Concept.java b/script-engine/src/main/java/org/ihtsdo/termserver/scripting/domain/Concept.java index d4205cd0e0..b21463acd7 100644 --- a/script-engine/src/main/java/org/ihtsdo/termserver/scripting/domain/Concept.java +++ b/script-engine/src/main/java/org/ihtsdo/termserver/scripting/domain/Concept.java @@ -1048,18 +1048,18 @@ public List getAssociationEntries() { return associationEntries; } - public List getAssociations(ActiveState activeState) { - return getAssociations(activeState, false); //All associations by default + public List getAssociationEntries(ActiveState activeState) { + return getAssociationEntries(activeState, false); //All associations by default } - public List getAssociations(ActiveState activeState, String refsetId) { + public List getAssociationEntries(ActiveState activeState, String refsetId) { return getAssociationEntries().stream() .filter(a -> a.hasActiveState(activeState)) .filter(a -> a.getRefsetId().contentEquals(refsetId)) .collect(Collectors.toList()); } - public List getAssociations(ActiveState activeState, boolean historicalAssociationsOnly) { + public List getAssociationEntries(ActiveState activeState, boolean historicalAssociationsOnly) { return getAssociationEntries().stream() .filter(a -> a.hasActiveState(activeState)) .filter(a -> !historicalAssociationsOnly || Script.HISTORICAL_REFSETS.contains(a.getRefsetId())) diff --git a/script-engine/src/main/java/org/ihtsdo/termserver/scripting/fixes/AddAttributeIfRequired.java b/script-engine/src/main/java/org/ihtsdo/termserver/scripting/fixes/AddAttributeIfRequired.java index ce8473a317..31d50e09a5 100644 --- a/script-engine/src/main/java/org/ihtsdo/termserver/scripting/fixes/AddAttributeIfRequired.java +++ b/script-engine/src/main/java/org/ihtsdo/termserver/scripting/fixes/AddAttributeIfRequired.java @@ -11,13 +11,6 @@ import org.ihtsdo.termserver.scripting.fixes.BatchFix; import org.snomed.otf.script.dao.ReportSheetManager; -/** - * INFRA-5176 Add an attribute to a given ECL substrate where required - * INFRA-5236 Add DueTo to Abrasions - * QI-731 Add DueTo = Traumatic Event for all open fractures - * QI-802 Add 42752001 |Due to (attribute)| = 1148742001 |Intentional event (event)| - * to all << 59274003 |Intentional drug overdose (disorder)| - */ public class AddAttributeIfRequired extends BatchFix { private Set exclusions; @@ -35,6 +28,7 @@ public static void main(String[] args) throws TermServerScriptException, IOExcep fix.populateTaskDescription = false; fix.selfDetermining = true; fix.reportNoChange = true; + fix.runStandAlone = true; fix.additionalReportColumns = "Action Detail"; fix.init(args); fix.loadProjectSnapshot(true); @@ -70,11 +64,16 @@ private void postLoadInit() throws TermServerScriptException { //QI-801 subsetECL = "<< 59369008 |Accidental drug overdose (disorder)|"; relTemplate = new RelationshipTemplate(DUE_TO, gl.getConcept("418019003 |Accidental event (event)|")); - */ + //QI-802 subsetECL = "<< 59274003 |Intentional drug overdose (disorder)|"; relTemplate = new RelationshipTemplate(DUE_TO, gl.getConcept("1148742001 |Intentional event (event)|")); - + */ + + //INFRA-12357 + subsetECL = "<<276239002 |Therapy (regime/therapy)|"; + relTemplate = new RelationshipTemplate(METHOD, gl.getConcept("360270004 |Therapy - action (qualifier value)|")); + exclusions = new HashSet<>(); super.postInit(); } diff --git a/script-engine/src/main/java/org/ihtsdo/termserver/scripting/fixes/BatchFix.java b/script-engine/src/main/java/org/ihtsdo/termserver/scripting/fixes/BatchFix.java index c22d218729..0a132a8f6e 100644 --- a/script-engine/src/main/java/org/ihtsdo/termserver/scripting/fixes/BatchFix.java +++ b/script-engine/src/main/java/org/ihtsdo/termserver/scripting/fixes/BatchFix.java @@ -42,9 +42,8 @@ public abstract class BatchFix extends TermServerScript implements ScriptConstan protected int taskThrottle = 5; protected int restartFromTask = NOT_SET; protected int conceptThrottle = 2; - protected String targetAuthor; - protected String targetReviewer; - protected String[] author_reviewer; + protected List authors; + protected List reviewers; protected String[] emailDetails; protected boolean selfDetermining = false; //Set to true if the batch fix calculates its own data to process protected boolean populateEditPanel = true; @@ -106,7 +105,7 @@ protected List identifyComponentsToProcess() throws TermServerScriptE protected Batch formIntoBatch (List allComponents) throws TermServerScriptException { Batch batch = new Batch(getScriptName()); - Task task = batch.addNewTask(author_reviewer); + Task task = batch.addNewTask(getNextAuthor(), getNextReviewer()); //Do we need to prioritize some components? if (priorityComponents.size() > 0) { List unprioritized = new ArrayList<> (allComponents); @@ -134,11 +133,11 @@ protected Batch formIntoBatch (List allComponents) throws TermServerS //If we're on the same issue, don't break the task if (!lastIssue.equals(thisComponent.getIssues()) && !peekAheadFits(asConcepts(allComponents), currentPosition,remainingSpace)) { - task = batch.addNewTask(author_reviewer); + task = batch.addNewTask(getNextAuthor(), getNextReviewer()); } } else if (task.size() >= thisTaskMaxSize || (groupByIssue && !lastIssue.equals(thisComponent.getIssues()))) { - task = batch.addNewTask(author_reviewer); + task = batch.addNewTask(getNextAuthor(), getNextReviewer()); } task.add(thisComponent); lastIssue = thisComponent.getIssues(); @@ -150,6 +149,19 @@ protected Batch formIntoBatch (List allComponents) throws TermServerS return batch; } + protected String getNextAuthor() { + int nextAuthorIdx = Task.getNextTaskSequence()%authors.size(); + return authors.get(nextAuthorIdx); + } + + protected String getNextReviewer() { + if (reviewers == null || reviewers.size() == 0) { + return null; + } + int nextReviewerIdx = Task.getNextTaskSequence()%reviewers.size(); + return reviewers.get(nextReviewerIdx); + } + private boolean peekAheadFits(List concepts, int pos, int remainingSpace) { //Given the issue of the current position, can we fit all successive concepts with the same issue into the //same task, or do we need to start a new one? @@ -167,13 +179,13 @@ protected Batch formIntoGroupedBatch (List> allComponentList) th Batch batch = new Batch(getScriptName()); int componentsToProcess = 0; for (List thisSet : allComponentList) { - Task task = batch.addNewTask(author_reviewer); + Task task = batch.addNewTask(getNextAuthor(), getNextReviewer()); if (thisSet.size() > 0) { String lastIssue = thisSet.get(0).getIssues(); for (Component thisComponent : thisSet) { if (task.size() >= taskSize || (groupByIssue && !lastIssue.equals(thisComponent.getIssues()))) { - task = batch.addNewTask(author_reviewer); + task = batch.addNewTask(getNextAuthor(), getNextReviewer()); } task.add(thisComponent); componentsToProcess++; @@ -417,17 +429,9 @@ protected void updateTask(Task task, String reportName, String reportURL) throws } //Reassign the task to the intended author. Set at task or processing level - String taskAuthor = task.getAssignedAuthor(); - if (taskAuthor == null) { - taskAuthor = targetAuthor; - } - - if (taskAuthor != null && !taskAuthor.isEmpty()) { - String reviewMsg = task.getReviewer() == null? "" : " into review for " + task.getReviewer(); - LOGGER.debug("Assigning " + task + " to " + taskAuthor + reviewMsg); - } - - scaClient.updateTask(project.getKey(), task.getKey(), null, taskDescription, taskAuthor, task.getReviewer()); + String reviewMsg = task.getReviewer() == null? "" : " into review for " + task.getReviewer(); + LOGGER.debug("Assigning " + task + " to " + task.getAssignedAuthor() + reviewMsg); + scaClient.updateTask(project.getKey(), task.getKey(), null, taskDescription, task.getAssignedAuthor(), task.getReviewer()); } //Override if working with Refsets or Descriptions directly @@ -456,7 +460,7 @@ protected void init (JobRun jobRun) throws TermServerScriptException { if (jobRun.getParamValue(RESTART_FROM_TASK) != null) { restartFromTask = Integer.parseInt(jobRun.getParamValue(RESTART_FROM_TASK)); } - author_reviewer = new String[] { jobRun.getUser() }; + authors = List.of(jobRun.getUser()); } public void inflightInit (String[] args) throws TermServerScriptException { @@ -502,10 +506,10 @@ protected void init (String[] args) throws TermServerScriptException { restartFromTask = Integer.parseInt(thisArg); isRestartFromTask = false; } else if (isAuthor) { - targetAuthor = thisArg.toLowerCase(); + authors = Arrays.asList(thisArg.toLowerCase().split(",")); isAuthor = false; } else if (isReviewer) { - targetReviewer = thisArg.toLowerCase(); + reviewers = Arrays.asList(thisArg.toLowerCase().split(",")); isReviewer = false; } else if (isTaskSize) { taskSize = Integer.parseInt(thisArg); @@ -517,13 +521,6 @@ protected void init (String[] args) throws TermServerScriptException { } } - - if (targetReviewer != null) { - author_reviewer = new String[] { targetAuthor, targetReviewer }; - } else { - author_reviewer = new String[] { targetAuthor }; - } - //For batch fixes we generally need to know if the components we're modifying have been //released or not, so set this by default getArchiveManager(true).setPopulateReleasedFlag(true); @@ -578,18 +575,12 @@ protected void checkSettingsWithUser(JobRun jobRun) throws TermServerScriptExcep } } - if (targetAuthor == null) { + if (authors == null) { if (jobRun != null && jobRun.getParamValue(AUTHOR) != null) { - targetAuthor = jobRun.getParamValue(AUTHOR); + authors = Arrays.asList(jobRun.getParamValue(AUTHOR).split(",")); } else { throw new TermServerScriptException("No target author detected in command line arguments"); } - } else { - if (targetReviewer != null) { - author_reviewer = new String[] { targetAuthor, targetReviewer }; - } else { - author_reviewer = new String[] { targetAuthor }; - } } LOGGER.info ("\nBatching " + taskSize + " concepts per task"); diff --git a/script-engine/src/main/java/org/ihtsdo/termserver/scripting/fixes/DeleteComponents.java b/script-engine/src/main/java/org/ihtsdo/termserver/scripting/fixes/DeleteComponents.java index f017b32432..8ed4320747 100644 --- a/script-engine/src/main/java/org/ihtsdo/termserver/scripting/fixes/DeleteComponents.java +++ b/script-engine/src/main/java/org/ihtsdo/termserver/scripting/fixes/DeleteComponents.java @@ -17,11 +17,21 @@ public class DeleteComponents extends BatchFix implements ScriptConstants { private static final Logger LOGGER = LoggerFactory.getLogger(DeleteComponents.class); private String[] componentsToDelete = new String[] { - "a0de0643-9861-47b5-b01d-0b4bdfdc49e3", - "54234842-1f8a-4827-aa21-c829665c3085", - "f8a11edc-a9b5-49bf-9bc9-92d4794e0df1", - "aa0f2ada-dca3-4fca-a38b-3c3487b44b9a", - "9c00fe2f-26ef-4ed8-afe0-821bfdcedaaf" + "bd14e5fb-478f-4de8-9977-5d37f5575785","9001d8e1-463a-445e-95cc-2bcaf37db18e","94dc0c97-c45b-42f1-9003-993ff2353904", + "992ec59e-ce60-4ac6-ab85-be1cbfafdbc8","ea288158-8198-4097-af02-25f4af5954de","bcb02d40-f9f8-4393-a96d-bced68729095", + "59fd684c-544c-4fd6-a960-69b8f878c7d9","85c0787e-7746-4e6a-ad28-cc7d2d8b763f","b02c7dee-1e17-4aed-a226-6efa575d8164", + "bd1121bf-1ff2-46ab-ab1e-dd91a51ac987","951a3276-9945-4a89-b578-57de0c0ca534","5926923a-91f7-474b-9e86-0be6b14f65be", + "adcf6572-fe11-4049-8b18-5c016918c79e","44dff1ce-431e-4548-8e64-b08968d9e3eb","05f45084-ba3b-405d-83cb-9f7b9f14edd6", + "ab26dadd-e426-425e-b8d8-b83fabb3de75","9e4eaa44-0aae-4c3a-8da5-8c4e6b75577e","284e98d2-47fe-473a-bd73-316d92404e78", + "3636e8c7-94d4-4c53-8341-757e953ea17d","8c6c1455-473e-42fa-bd36-02a42440f4c4","0c2dac1c-de31-4286-b403-945947355584", + "ccd6b6f8-e457-4032-b215-f7fe33ab7c0a","658b5485-2a0a-499c-900e-93984b1197d9","68f44366-50d5-41e5-8aad-3080dfbd610b", + "17e23b7c-47cc-4d54-b463-abdd0dc63b5f","e8c5cc16-a923-4a78-ab11-1a681104adc1","fc0f8fa1-bad5-4ffc-b3e1-8d8f4f3a123a", + "7a867650-c54e-4bc1-99f4-a905f2129c11","9239a886-13ff-41dd-89af-785bcc3836bf","29f0241f-30dc-4dde-8cc9-883aa0634f1d", + "1c118fc8-7e2b-4f28-b7c8-0c48953040dc","20a30ed1-5bc6-4714-a106-937227619491","62a18088-b378-45e9-8296-49d4598b96a8", + "1c51b3f8-ff8d-4f01-bc12-8b6c59da1b5f","433d3a25-34e2-45b0-86be-beb08a5a7580","6c95892a-bab9-43a4-9dd1-88639a1d4995", + "328a9b96-721e-462b-8eeb-d220e7de40e8","f8f5e465-c7de-4972-a56a-7e6d10d16b7e","34b08a77-3580-4206-a5a8-2b3359d2ef1a", + "daa29599-7910-4f15-85c2-9138580b3679","1449f983-4bb7-448b-a2d1-60988313b6b6","783977f8-2562-4602-95f7-b7f326ae109b", + "9c0a668b-dee8-46f4-b60c-5d18e428b72b","157e61d4-7dc7-40ea-96c4-4ec5bb08a03b","1a9c15ed-20a8-4b80-9e7c-a09685a878ed" }; protected DeleteComponents(BatchFix clone) { diff --git a/script-engine/src/main/java/org/ihtsdo/termserver/scripting/fixes/HistoricalHistoricalIssues.java b/script-engine/src/main/java/org/ihtsdo/termserver/scripting/fixes/HistoricalHistoricalIssues.java index 46fa72d43e..d2dda776c6 100644 --- a/script-engine/src/main/java/org/ihtsdo/termserver/scripting/fixes/HistoricalHistoricalIssues.java +++ b/script-engine/src/main/java/org/ihtsdo/termserver/scripting/fixes/HistoricalHistoricalIssues.java @@ -85,7 +85,7 @@ private int fixHistoricalIssues(Task t, Concept c, Concept loadedConcept) throws } else { String refsetId = ""; boolean wasaComboDetected = loadedConcept.getAssociationTargets().isWasACombo(); - List assocs = c.getAssociations(ActiveState.ACTIVE, true); //Historical associations only! + List assocs = c.getAssociationEntries(ActiveState.ACTIVE, true); //Historical associations only! for (AssociationEntry h : assocs) { //If this is a WAS A as part of a combination with other indicators, remove it if (h.getRefsetId().equals(SCTID_ASSOC_WAS_A_REFSETID) && wasaComboDetected) { diff --git a/script-engine/src/main/java/org/ihtsdo/termserver/scripting/fixes/InactivateConcepts.java b/script-engine/src/main/java/org/ihtsdo/termserver/scripting/fixes/InactivateConcepts.java index 75669c922a..b395bcb943 100644 --- a/script-engine/src/main/java/org/ihtsdo/termserver/scripting/fixes/InactivateConcepts.java +++ b/script-engine/src/main/java/org/ihtsdo/termserver/scripting/fixes/InactivateConcepts.java @@ -42,10 +42,11 @@ public static void main(String[] args) throws TermServerScriptException, IOExcep ReportSheetManager.targetFolderId = "1fIHGIgbsdSfh5euzO3YKOSeHw4QHCM-m"; //Ad-hoc batch updates fix.inputFileHasHeaderRow = false; fix.expectNullConcepts = false; - fix.groupByIssue = true; + fix.groupByIssue = false; fix.reportNoChange = false; - fix.selfDetermining = true; - fix.subsetECL = "< 168123008 |Sample sent for examination (situation)|."; + fix.selfDetermining = false; + fix.runStandAlone = true; + //fix.subsetECL = "< 168123008 |Sample sent for examination (situation)|."; fix.getArchiveManager().setPopulateReleasedFlag(true); fix.init(args); fix.loadProjectSnapshot(true); @@ -251,6 +252,9 @@ private void inactivateHistoricalAssociation(Task task, AssociationEntry assoc, //Do we need to align the incoming concept's inactivation indicator? InactivationIndicator origII = incomingConcept.getInactivationIndicator(); + if (origII == null) { + throw new ValidationFailure(incomingConcept, "Incoming association from " + incomingConcept + " has no inactivation indicator"); + } boolean iiChanged = false; if (!origII.equals(reason)) { incomingConcept.setInactivationIndicator(reason); @@ -296,7 +300,8 @@ private void inactivateHistoricalAssociation(Task task, AssociationEntry assoc, @Override protected List loadLine(String[] lineItems) throws TermServerScriptException { Concept c; - + Concept replacement = null; + if (StringUtils.isNumeric(lineItems[0])) { c = gl.getConcept(lineItems[0]); } else { @@ -310,8 +315,7 @@ protected List loadLine(String[] lineItems) throws TermServerScriptEx } /* - Concept replacement = null; - int idxReplacement = 1; + int idxReplacement = 1; if (lineItems.length > 2) { idxReplacement = 2; //In this case, column 1 will be in inactivation reason @@ -344,17 +348,19 @@ protected List loadLine(String[] lineItems) throws TermServerScriptEx //Specific to IHTSDO-175 //inactivationIndicators.put(c, InactivationIndicator.AMBIGUOUS); //replacement = gl.getConcept(" 782902008 |Implantation procedure (procedure)|"); - inactivationIndicators.put(c, InactivationIndicator.NONCONFORMANCE_TO_EDITORIAL_POLICY); - + //inactivationIndicators.put(c, InactivationIndicator.NONCONFORMANCE_TO_EDITORIAL_POLICY); + inactivationIndicators.put(c, InactivationIndicator.OUTDATED); + replacement = gl.getConcept("399097000 |Administration of anesthesia (procedure)|"); + //Give C an issue of one of it's parents to try to batch sibling concepts together c.setIssue(c.getParents(CharacteristicType.STATED_RELATIONSHIP).iterator().next().getId()); - /*if (replacement != null) { + if (replacement != null) { replacements.put(c, replacement); return Collections.singletonList(c); } else if (!expectReplacements){ return Collections.singletonList(c); - }*/ + } return null; } } diff --git a/script-engine/src/main/java/org/ihtsdo/termserver/scripting/fixes/MissingInactivationIndicatorsFix.java b/script-engine/src/main/java/org/ihtsdo/termserver/scripting/fixes/MissingInactivationIndicatorsFix.java index 68a343d6a1..e3f0f41337 100644 --- a/script-engine/src/main/java/org/ihtsdo/termserver/scripting/fixes/MissingInactivationIndicatorsFix.java +++ b/script-engine/src/main/java/org/ihtsdo/termserver/scripting/fixes/MissingInactivationIndicatorsFix.java @@ -9,7 +9,6 @@ import org.ihtsdo.termserver.scripting.ValidationFailure; import org.ihtsdo.termserver.scripting.domain.*; -import org.ihtsdo.termserver.scripting.fixes.BatchFix; import org.ihtsdo.termserver.scripting.util.SnomedUtils; @@ -107,7 +106,7 @@ private boolean hasIssue(Concept c) { //or having some indicator other than NCEP with no historical associations if (c.getInactivationIndicator() != null && !c.getInactivationIndicator().equals(InactivationIndicator.NONCONFORMANCE_TO_EDITORIAL_POLICY) && - c.getAssociations(ActiveState.ACTIVE, true).size() == 0) { + c.getAssociationEntries(ActiveState.ACTIVE, true).size() == 0) { return true; } diff --git a/script-engine/src/main/java/org/ihtsdo/termserver/scripting/fixes/RemoveWAS_A.java b/script-engine/src/main/java/org/ihtsdo/termserver/scripting/fixes/RemoveWAS_A.java index ec9abf3800..2d61847c48 100644 --- a/script-engine/src/main/java/org/ihtsdo/termserver/scripting/fixes/RemoveWAS_A.java +++ b/script-engine/src/main/java/org/ihtsdo/termserver/scripting/fixes/RemoveWAS_A.java @@ -158,7 +158,7 @@ protected List identifyComponentsToProcess() throws TermServerScriptE Set getWAS_A(Concept c) throws TermServerScriptException { Set was_a = new HashSet<>(); - for (AssociationEntry assoc: c.getAssociations(ActiveState.ACTIVE)) { + for (AssociationEntry assoc: c.getAssociationEntries(ActiveState.ACTIVE)) { if (assoc.getRefsetId().equals(SCTID_ASSOC_WAS_A_REFSETID)) { was_a.add(gl.getConcept(assoc.getTargetComponentId())); } diff --git a/script-engine/src/main/java/org/ihtsdo/termserver/scripting/fixes/TouchConceptsInTask.java b/script-engine/src/main/java/org/ihtsdo/termserver/scripting/fixes/TouchConceptsInTask.java index 958536524e..4167da283c 100644 --- a/script-engine/src/main/java/org/ihtsdo/termserver/scripting/fixes/TouchConceptsInTask.java +++ b/script-engine/src/main/java/org/ihtsdo/termserver/scripting/fixes/TouchConceptsInTask.java @@ -44,7 +44,7 @@ public static void main(String[] args) throws TermServerScriptException, IOExcep public int touchConcepts() throws TermServerScriptException { Batch b = new Batch ("foo"); - Task t = b.addNewTask(new String[] { "bar" }); + Task t = b.addNewTask("bar", null); t.setBranchPath(targetTaskPath); allComponentsToProcess = processFile(getInputFile()); for (Concept c : asConcepts(allComponentsToProcess)) { diff --git a/script-engine/src/main/java/org/ihtsdo/termserver/scripting/fixes/batchImport/BatchImport.java b/script-engine/src/main/java/org/ihtsdo/termserver/scripting/fixes/batchImport/BatchImport.java index ab83bee2d5..123fc17830 100644 --- a/script-engine/src/main/java/org/ihtsdo/termserver/scripting/fixes/batchImport/BatchImport.java +++ b/script-engine/src/main/java/org/ihtsdo/termserver/scripting/fixes/batchImport/BatchImport.java @@ -181,7 +181,7 @@ protected Batch formIntoBatch (List allComponents) { for (Concept thisChild : getRootConcept().getChildren(CharacteristicType.STATED_RELATIONSHIP)) { BatchImportConcept child = (BatchImportConcept) thisChild; if (thisTask == null || thisTask.size() >= taskSize) { - thisTask = batch.addNewTask(author_reviewer); + thisTask = batch.addNewTask(getNextAuthor(), null); } //We can be sure that all descendants will not exceed our batch limit, having already validated thisTask.add(child); diff --git a/script-engine/src/main/java/org/ihtsdo/termserver/scripting/fixes/drugs/CDRemodelling.java b/script-engine/src/main/java/org/ihtsdo/termserver/scripting/fixes/drugs/CDRemodelling.java index 1242f78976..824729a63a 100644 --- a/script-engine/src/main/java/org/ihtsdo/termserver/scripting/fixes/drugs/CDRemodelling.java +++ b/script-engine/src/main/java/org/ihtsdo/termserver/scripting/fixes/drugs/CDRemodelling.java @@ -420,7 +420,7 @@ private Concept getUnitOfPresentation(String unitPresStr) throws TermServerScrip private Concept getAlternative(Task t, Concept c) throws TermServerScriptException { //Work through the active historical associations and find an active alternative - List assocs = c.getAssociations(ActiveState.ACTIVE); + List assocs = c.getAssociationEntries(ActiveState.ACTIVE); if (assocs.size() > 1 || assocs.size() == 0) { String msg = c + " is inactive with " + assocs.size() + " historical associations. Cannot determine alternative concept."; report(t, c, Severity.HIGH, ReportActionType.VALIDATION_ERROR, msg); diff --git a/script-engine/src/main/java/org/ihtsdo/termserver/scripting/fixes/drugs/DrugsReTerming.java b/script-engine/src/main/java/org/ihtsdo/termserver/scripting/fixes/drugs/DrugsReTerming.java index be77bafa86..b90579e0c6 100644 --- a/script-engine/src/main/java/org/ihtsdo/termserver/scripting/fixes/drugs/DrugsReTerming.java +++ b/script-engine/src/main/java/org/ihtsdo/termserver/scripting/fixes/drugs/DrugsReTerming.java @@ -25,10 +25,6 @@ public class DrugsReTerming extends DrugBatchFix implements ScriptConstants{ - private static final Logger LOGGER = LoggerFactory.getLogger(DrugsReTerming.class); - - String[] author_reviewer = new String[] {targetAuthor}; - protected DrugsReTerming(BatchFix clone) { super(clone); } @@ -210,14 +206,14 @@ private Description findDescription(Concept remodelledConcept, String term) { protected Batch formIntoBatch (String fileName, List allConcepts, String branchPath) throws TermServerScriptException { Batch batch = new Batch(getReportName()); - Task task = batch.addNewTask(author_reviewer); + Task task = batch.addNewTask(getNextAuthor()); for (Concept thisConcept : allConcepts) { if (((ConceptChange) thisConcept).getSkipReason() != null) { report(task, thisConcept, Severity.LOW, ReportActionType.VALIDATION_CHECK, "Concept marked as excluded"); } else { if (task.size() >= taskSize) { - task = batch.addNewTask(author_reviewer); + task = batch.addNewTask(getNextAuthor()); } task.add(thisConcept); } diff --git a/script-engine/src/main/java/org/ihtsdo/termserver/scripting/fixes/drugs/FlattenHierarchy.java b/script-engine/src/main/java/org/ihtsdo/termserver/scripting/fixes/drugs/FlattenHierarchy.java index 63824621e6..e61d16bf8c 100644 --- a/script-engine/src/main/java/org/ihtsdo/termserver/scripting/fixes/drugs/FlattenHierarchy.java +++ b/script-engine/src/main/java/org/ihtsdo/termserver/scripting/fixes/drugs/FlattenHierarchy.java @@ -255,7 +255,7 @@ private Set determineNonRedundantAncestors(Task task, Concept load @Override protected Batch formIntoBatch (List allConcepts) throws TermServerScriptException { Batch batch = new Batch(getScriptName()); - Task task = batch.addNewTask(author_reviewer); + Task task = batch.addNewTask(getNextAuthor()); //Include siblings of the specified concepts includeSiblings(allConcepts); @@ -277,7 +277,7 @@ protected Batch formIntoBatch (List allConcepts) throws TermServerScr unallocated.remove(thisConcept); } LOGGER.debug (task + " (" + task.size() + ")"); - task = batch.addNewTask(author_reviewer); + task = batch.addNewTask(getNextAuthor()); } } batch.consolidateIntoLargeTasks(taskSize, 0); //Tasks are large enough already, no wiggle room! diff --git a/script-engine/src/main/java/org/ihtsdo/termserver/scripting/fixes/drugs/ReplaceConcepts.java b/script-engine/src/main/java/org/ihtsdo/termserver/scripting/fixes/drugs/ReplaceConcepts.java index cb2a5821bc..e2fb8381f9 100644 --- a/script-engine/src/main/java/org/ihtsdo/termserver/scripting/fixes/drugs/ReplaceConcepts.java +++ b/script-engine/src/main/java/org/ihtsdo/termserver/scripting/fixes/drugs/ReplaceConcepts.java @@ -165,7 +165,7 @@ private void findReturnedParent(Concept c, ParentFind parentFind) throws TermSer parentFind.parents.add(parent); } else { //Can we find a historical association to replace it? - List assocs = parent.getAssociations(ActiveState.ACTIVE); + List assocs = parent.getAssociationEntries(ActiveState.ACTIVE); if (assocs!= null && assocs.size() > 1) { String assocStr = assocs .stream() diff --git a/script-engine/src/main/java/org/ihtsdo/termserver/scripting/fixes/metadata/UpsertMetadata.java b/script-engine/src/main/java/org/ihtsdo/termserver/scripting/fixes/metadata/UpsertMetadata.java index 303643e900..1adb56567e 100644 --- a/script-engine/src/main/java/org/ihtsdo/termserver/scripting/fixes/metadata/UpsertMetadata.java +++ b/script-engine/src/main/java/org/ihtsdo/termserver/scripting/fixes/metadata/UpsertMetadata.java @@ -32,7 +32,6 @@ public static void main(String[] args) throws TermServerScriptException, IOExcep fix.headers = "CodeSystem, Severity, Action, Detail"; fix.additionalReportColumns=""; fix.selfDetermining = true; - fix.targetAuthor = "empty"; fix.init(args); fix.postInit(); fix.upsertMetadata(); diff --git a/script-engine/src/main/java/org/ihtsdo/termserver/scripting/fixes/rf2Player/DispositionsArchive.java b/script-engine/src/main/java/org/ihtsdo/termserver/scripting/fixes/rf2Player/DispositionsArchive.java index cced464ff0..d9eb4ae2f8 100644 --- a/script-engine/src/main/java/org/ihtsdo/termserver/scripting/fixes/rf2Player/DispositionsArchive.java +++ b/script-engine/src/main/java/org/ihtsdo/termserver/scripting/fixes/rf2Player/DispositionsArchive.java @@ -65,7 +65,7 @@ protected Batch formIntoBatch() throws TermServerScriptException { for (String bucketId : dispositionBuckets.keySet()) { if (dispositionBuckets.get(bucketId).size() <= taskSize ) { - Task task = batch.addNewTask(author_reviewer); + Task task = batch.addNewTask(getNextAuthor()); task.setTaskInfo(bucketId); for (Concept concept : dispositionBuckets.get(bucketId)) { @@ -128,7 +128,7 @@ private void filterBatch(Batch batch) { private void splitBucketIntoNeighbourhoods(Batch batch, String bucketId, Collection remainingConceptsToGroup) throws TermServerScriptException { int initialSize = remainingConceptsToGroup.size(); while (remainingConceptsToGroup.size() > 0) { - Task task = batch.addNewTask(author_reviewer); + Task task = batch.addNewTask(getNextAuthor()); task.setTaskInfo(bucketId); //First identify the lowest concept which will initially be a leaf node Concept currentFocus = findLowestConcept(remainingConceptsToGroup); diff --git a/script-engine/src/main/java/org/ihtsdo/termserver/scripting/reports/DuplicateInactivationAssocationReport.java b/script-engine/src/main/java/org/ihtsdo/termserver/scripting/reports/DuplicateInactivationAssocationReport.java index e0e8114473..6e20cca0c3 100644 --- a/script-engine/src/main/java/org/ihtsdo/termserver/scripting/reports/DuplicateInactivationAssocationReport.java +++ b/script-engine/src/main/java/org/ihtsdo/termserver/scripting/reports/DuplicateInactivationAssocationReport.java @@ -60,7 +60,7 @@ private void reportMatchingInactivations() throws TermServerScriptException { for (InactivationIndicatorEntry i : c.getInactivationIndicatorEntries(ActiveState.BOTH)) { report (c, i.getEffectiveTime(), i); } - for (AssociationEntry h : c.getAssociations(ActiveState.BOTH)) { + for (AssociationEntry h : c.getAssociationEntries(ActiveState.BOTH)) { report (c, h.getEffectiveTime(), h); } incrementSummaryInformation("Concepts reported"); diff --git a/script-engine/src/main/java/org/ihtsdo/termserver/scripting/reports/InactivationAssocationReport.java b/script-engine/src/main/java/org/ihtsdo/termserver/scripting/reports/InactivationAssocationReport.java index 7511ff9e63..3b08f3126f 100644 --- a/script-engine/src/main/java/org/ihtsdo/termserver/scripting/reports/InactivationAssocationReport.java +++ b/script-engine/src/main/java/org/ihtsdo/termserver/scripting/reports/InactivationAssocationReport.java @@ -75,7 +75,7 @@ public void runJob() throws TermServerScriptException { for (InactivationIndicatorEntry inactivationIndicator : c.getInactivationIndicatorEntries(ActiveState.ACTIVE)) { if (inactivationIndicator.getInactivationReasonId().equals(inactivationReasonSctId)) { //Now does the concept have one of our target historical associations? - for (AssociationEntry histAssoc : c.getAssociations(ActiveState.ACTIVE)) { + for (AssociationEntry histAssoc : c.getAssociationEntries(ActiveState.ACTIVE)) { for (String targetAssocationRefsetId : targetAssocationRefsetIds) { if (histAssoc.getRefsetId().equals(targetAssocationRefsetId)) { report(c,inactivationIndicator, histAssoc); diff --git a/script-engine/src/main/java/org/ihtsdo/termserver/scripting/reports/loinc/LOINCStats.java b/script-engine/src/main/java/org/ihtsdo/termserver/scripting/reports/loinc/LOINCStats.java index cb22d17a0b..1e6eb5537a 100644 --- a/script-engine/src/main/java/org/ihtsdo/termserver/scripting/reports/loinc/LOINCStats.java +++ b/script-engine/src/main/java/org/ihtsdo/termserver/scripting/reports/loinc/LOINCStats.java @@ -124,7 +124,7 @@ public void doReport() throws TermServerScriptException, ConversionException { increment("Inactivated Expression"); //Does the inactive concept have a historical association - for (AssociationEntry e : c.getAssociations(ActiveState.ACTIVE)) { + for (AssociationEntry e : c.getAssociationEntries(ActiveState.ACTIVE)) { Concept replacement = gl.getConcept(e.getTargetComponentId()); if (currentContentMap.containsValue(replacement)) { increment("Inactivated concept has replacement value"); diff --git a/script-engine/src/main/java/org/ihtsdo/termserver/scripting/reports/qi/AllKnownTemplates.java b/script-engine/src/main/java/org/ihtsdo/termserver/scripting/reports/qi/AllKnownTemplates.java index 9db7276465..0a4ced2883 100644 --- a/script-engine/src/main/java/org/ihtsdo/termserver/scripting/reports/qi/AllKnownTemplates.java +++ b/script-engine/src/main/java/org/ihtsdo/termserver/scripting/reports/qi/AllKnownTemplates.java @@ -40,8 +40,11 @@ protected void commonInit(JobRun run, boolean singleTemplateMode) { public void init (JobRun run) throws TermServerScriptException { commonInit(run, singleTemplateMode); if (!singleTemplateMode) { - String subsetECL = "<< 125605004"; // QI-5 |Fracture of bone (disorder)| - String[] templateNames = new String[] { "templates/fracture/Fracture of Bone Structure.json", + String subsetECL = ""; + String[] templateNames = new String[] {}; + + subsetECL = "<< 125605004"; // QI-5 |Fracture of bone (disorder)| + templateNames = new String[] { "templates/fracture/Fracture of Bone Structure.json", "templates/fracture/Fracture Dislocation of Bone Structure.json"}; populateTemplates(subsetECL, templateNames); @@ -235,12 +238,7 @@ public void init (JobRun run) throws TermServerScriptException { subsetECL = "<<362975008 |Degenerative disorder (disorder)|: 116676008 |Associated morphology (attribute)| = << 46595003 |Deposition (morphologic abnormality)| "; templateNames = new String[] { "templates/Degenerative disorder.json"}; populateTemplates(subsetECL, templateNames); - - //Replaced with production template Congenital [morphology] of [body structure] - /*subsetECL = "<< 276654001 |Congenital malformation (disorder)|"; //QI-287 - templateNames = new String[] { "templates/Congenital Malformation.json"}; - populateTemplates(subsetECL, templateNames);*/ - + subsetECL = "<< 131148009|Bleeding|"; //QI-319 templateNames = new String[] { "templates/Bleeding - disorder.json"}; populateTemplates(subsetECL, templateNames); @@ -413,24 +411,6 @@ public void init (JobRun run) throws TermServerScriptException { templateNames = new String[] { "templates/morphologies/Retention.json" }; populateTemplates(null, templateNames); - /*templateNames = new String[] { "templates/anatomy/Anatomical Parts.json" }; - populateTemplates(null, templateNames); - - templateNames = new String[] { "templates/anatomy/Entity in region.json" }; - populateTemplates(null, templateNames); - - templateNames = new String[] { "templates/anatomy/Hair follicle.json" }; - populateTemplates(null, templateNames); - - templateNames = new String[] { "templates/anatomy/Region of region.json" }; - populateTemplates(null, templateNames); - - templateNames = new String[] { "templates/anatomy/Skin of body.json" }; - populateTemplates(null, templateNames); - - templateNames = new String[] { "templates/anatomy/Skin of part of body.json" }; - populateTemplates(null, templateNames);*/ - templateNames = new String[] { "templates/procedures/MRI.json" }; populateTemplates(null, templateNames); @@ -529,7 +509,7 @@ public void init (JobRun run) throws TermServerScriptException { templateNames = new String[] { "templates/morphologies/Dislocation.json" }; populateTemplates(null, templateNames); - templateNames = new String[] { "templates/procedures/PlacementOfStent.json" }; + templateNames = new String[] { "templates/procedures/InsertionOfStent.json" }; populateTemplates(null, templateNames); templateNames = new String[] { "templates/Toxic due to plant.json" }; @@ -546,7 +526,7 @@ public void init (JobRun run) throws TermServerScriptException { //Do this one last to pick up whatever is left under Disease subsetECL = "<< 64572001 |Disease (disorder)|"; - templateNames = new String[] { "templates/Disease.json" }; + templateNames = new String[] { "templates/Disease.json" }; populateTemplates(subsetECL, templateNames); populateTemplatesFromTS(); diff --git a/script-engine/src/main/java/org/ihtsdo/termserver/scripting/reports/qi/GnarlyFactorCalculation.java b/script-engine/src/main/java/org/ihtsdo/termserver/scripting/reports/qi/GnarlyFactorCalculation.java index afa11b1d5c..d1fcf465f5 100644 --- a/script-engine/src/main/java/org/ihtsdo/termserver/scripting/reports/qi/GnarlyFactorCalculation.java +++ b/script-engine/src/main/java/org/ihtsdo/termserver/scripting/reports/qi/GnarlyFactorCalculation.java @@ -321,7 +321,7 @@ private String calculateTotalFDsUnderIPs(Concept subHierarchy, Set inte private Concept getAlternative(Concept c) throws TermServerScriptException { //Work through the active historical associations and find an active alternative - List assocs = c.getAssociations(ActiveState.ACTIVE); + List assocs = c.getAssociationEntries(ActiveState.ACTIVE); if (assocs.size() > 1 || assocs.size() == 0) { LOGGER.warn ( c + " is inactive with " + assocs.size() + " historical associations. Cannot determine alternative concept." ); return null; diff --git a/script-engine/src/main/java/org/ihtsdo/termserver/scripting/reports/release/HistoricStatsGenerator.java b/script-engine/src/main/java/org/ihtsdo/termserver/scripting/reports/release/HistoricStatsGenerator.java index 5f769b8409..4733d6749b 100644 --- a/script-engine/src/main/java/org/ihtsdo/termserver/scripting/reports/release/HistoricStatsGenerator.java +++ b/script-engine/src/main/java/org/ihtsdo/termserver/scripting/reports/release/HistoricStatsGenerator.java @@ -147,7 +147,7 @@ public void runJob() throws TermServerScriptException { } private String getHistAssocTargets(Concept c) { - return c.getAssociations(ActiveState.ACTIVE, true) + return c.getAssociationEntries(ActiveState.ACTIVE, true) .stream() .map(a -> a.getTargetComponentId()) .collect(Collectors.joining(",")); @@ -322,13 +322,13 @@ private Collection getDescriptionInactivationIndicat private String[] getHistAssocIds(Concept c) { String[] results = new String[2]; - results[ACTIVE] = c.getAssociations(ActiveState.ACTIVE) + results[ACTIVE] = c.getAssociationEntries(ActiveState.ACTIVE) .stream() .filter(h -> inScope(h)) .map(h -> h.getId()) .collect(Collectors.joining(",")); - results[INACTIVE] = c.getAssociations(ActiveState.INACTIVE) + results[INACTIVE] = c.getAssociationEntries(ActiveState.INACTIVE) .stream() .filter(h -> inScope(h)) .map(h -> h.getId()) diff --git a/script-engine/src/main/java/org/ihtsdo/termserver/scripting/util/HistAssocUtils.java b/script-engine/src/main/java/org/ihtsdo/termserver/scripting/util/HistAssocUtils.java index 55b46a20e4..c2e8f3de49 100644 --- a/script-engine/src/main/java/org/ihtsdo/termserver/scripting/util/HistAssocUtils.java +++ b/script-engine/src/main/java/org/ihtsdo/termserver/scripting/util/HistAssocUtils.java @@ -145,7 +145,7 @@ public Set getReplacements(Concept c) throws TermServerScriptException Set replacements = new HashSet<>(); //We'll recover concept from local store again, in case we're working with one loaded from TS c = gl.getConcept(c.getId()); - for (AssociationEntry entry : c.getAssociations(ActiveState.ACTIVE, true)) { + for (AssociationEntry entry : c.getAssociationEntries(ActiveState.ACTIVE, true)) { replacements.add(gl.getConcept(entry.getTargetComponentId())); } return replacements; diff --git a/script-engine/src/main/java/org/ihtsdo/termserver/scripting/util/MultiArchiveImporter.java b/script-engine/src/main/java/org/ihtsdo/termserver/scripting/util/MultiArchiveImporter.java index 10d9881d77..d530d683de 100644 --- a/script-engine/src/main/java/org/ihtsdo/termserver/scripting/util/MultiArchiveImporter.java +++ b/script-engine/src/main/java/org/ihtsdo/termserver/scripting/util/MultiArchiveImporter.java @@ -41,7 +41,7 @@ public static void main(String[] args) throws TermServerScriptException, IOExcep importer.classifyTasks = true; importer.allowDirectoryInputFile = true; importer.init(args); - importer.postInit(null, new String[] {"Task, Archive, Result"}, false); + importer.postInit(null, new String[] {"Task, Archive, User, Result"}, false); importer.importArchives(); } finally { importer.finish(); @@ -49,14 +49,21 @@ public static void main(String[] args) throws TermServerScriptException, IOExcep } private void importArchives() throws TermServerScriptException { - LOGGER.info("Processing all archives in " + getInputFile()); + String limitStr = processingLimit == NOT_SET ? "all" : processingLimit + ""; + LOGGER.info("Processing " + limitStr + " archives in " + getInputFile()); String[] dirListing = getInputFile().list(); Arrays.sort(dirListing, NumberAwareStringComparator.INSTANCE); + int archivesProcessed = 0; for (String archiveStr : dirListing){ File thisArchive = new File(getInputFile() + File.separator + archiveStr); if (thisArchive.getPath().endsWith(".zip")) { LOGGER.info("Processing: " + thisArchive); importArchive(thisArchive); + archivesProcessed++; + if (processingLimit != NOT_SET && archivesProcessed >= processingLimit) { + LOGGER.info("Processing limit of " + processingLimit + " reached. Exiting."); + break; + } } else { LOGGER.info("Skipping non archive: " + thisArchive); } @@ -67,7 +74,7 @@ private void importArchive(File thisArchive) throws TermServerScriptException { String result = "OK"; Task task = null; try { - task = new Task(null, author_reviewer); + task = new Task(null, getNextAuthor(), getNextReviewer()); task.setSummary(taskPrefix + "Import " + thisArchive.getName()); createTask(task); if (!dryRun) { @@ -84,7 +91,7 @@ private void importArchive(File thisArchive) throws TermServerScriptException { LOGGER.error("Failure to import " + thisArchive, e); result = e.toString(); } - report(PRIMARY_REPORT, task.getKey(), thisArchive.getName(), result); + report(PRIMARY_REPORT, task.getKey(), thisArchive.getName(), task.getAssignedAuthor(), result); } @Override diff --git a/script-engine/src/main/java/org/ihtsdo/termserver/scripting/util/RollbackBranch.java b/script-engine/src/main/java/org/ihtsdo/termserver/scripting/util/RollbackBranch.java new file mode 100644 index 0000000000..16de626b0e --- /dev/null +++ b/script-engine/src/main/java/org/ihtsdo/termserver/scripting/util/RollbackBranch.java @@ -0,0 +1,71 @@ +package org.ihtsdo.termserver.scripting.util; + +import org.ihtsdo.otf.exception.TermServerScriptException; +import org.ihtsdo.termserver.scripting.ReportClass; +import org.ihtsdo.termserver.scripting.domain.Branch; +import org.ihtsdo.termserver.scripting.fixes.BatchFix; +import org.ihtsdo.termserver.scripting.reports.TermServerReport; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.snomed.otf.script.dao.ReportSheetManager; + +import java.io.IOException; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneOffset; + +public class RollbackBranch extends TermServerReport { + + private static final Logger LOGGER = LoggerFactory.getLogger(RollbackBranch.class); + + public static void main(String[] args) throws TermServerScriptException, IOException, InterruptedException { + RollbackBranch importer = new RollbackBranch(); + try { + ReportSheetManager.targetFolderId = "13XiH3KVll3v0vipVxKwWjjf-wmjzgdDe"; //Technical Specialist Kung Foo + importer.init(args); + importer.postInit(new String[] {"Actions"}, new String[] {"Branch, HeadTimestamp, BaseTimestamp, Status, Action"}, false); + importer.rollbackBranch(); + } finally { + importer.finish(); + } + } + + private void rollbackBranch() throws TermServerScriptException { + String branchPath = getProject().getBranchPath(); + String msg = "Rolling back " + branchPath; + LOGGER.info(msg); + report(PRIMARY_REPORT, msg); + boolean userQuit = false; + boolean rollBackToBaseline = false; + while (!userQuit) { + Branch b = tsClient.getBranch(branchPath); + LocalDateTime head = LocalDateTime.ofInstant(Instant.ofEpochMilli(b.getHeadTimestamp()), ZoneOffset.UTC); + LocalDateTime base = LocalDateTime.ofInstant(Instant.ofEpochMilli(b.getBaseTimestamp()), ZoneOffset.UTC); + println( "\n" + b.getName() + " Head: " + head + " Base: " + base + " state: " + b.getState()); + msg = "Rolled-Back"; + + if (b.getHeadTimestamp() <= b.getBaseTimestamp() || b.getState().equals("BEHIND")) { + userQuit = true; + msg = "Final state - base timestamp reached / branch is behind parent"; + } else if (!rollBackToBaseline) { + print("Rollback (R), Rollback to Baseline (B) or Quit (Q): "); + String choice = STDIN.nextLine().trim().toUpperCase(); + if (choice.equals("B")) { + rollBackToBaseline = true; + } else if (choice.equals("Q")) { + userQuit = true; + msg = "Final state"; + } else if (choice.equals("R")) { + tsClient.adminRollbackCommit(b); + } + } + + if (!userQuit && rollBackToBaseline) { + tsClient.adminRollbackCommit(b); + } + report(PRIMARY_REPORT, "", head, base, b.getState(), msg); + } + } + + +} diff --git a/script-engine/src/main/java/org/ihtsdo/termserver/scripting/util/SnomedUtils.java b/script-engine/src/main/java/org/ihtsdo/termserver/scripting/util/SnomedUtils.java index ee6930aa97..1988f3dfd8 100644 --- a/script-engine/src/main/java/org/ihtsdo/termserver/scripting/util/SnomedUtils.java +++ b/script-engine/src/main/java/org/ihtsdo/termserver/scripting/util/SnomedUtils.java @@ -660,6 +660,26 @@ public static InactivationIndicator translateInactivationIndicator(String indica default: throw new IllegalArgumentException("Unrecognised inactivation indicator value " + indicatorSctId); } } + + public static String translateInactivationIndicator(InactivationIndicator ii) { + switch (ii) { + case AMBIGUOUS : return SCTID_INACT_AMBIGUOUS; + case MOVED_ELSEWHERE : return SCTID_INACT_MOVED_ELSEWHERE; + case CONCEPT_NON_CURRENT : return SCTID_INACT_CONCEPT_NON_CURRENT; + case DUPLICATE : return SCTID_INACT_DUPLICATE; + case ERRONEOUS : return SCTID_INACT_ERRONEOUS; + case INAPPROPRIATE : return SCTID_INACT_INAPPROPRIATE; + case LIMITED : return SCTID_INACT_LIMITED; + case OUTDATED : return SCTID_INACT_OUTDATED; + case PENDING_MOVE : return SCTID_INACT_PENDING_MOVE; + case NONCONFORMANCE_TO_EDITORIAL_POLICY : return SCTID_INACT_NON_CONFORMANCE; + case NOT_SEMANTICALLY_EQUIVALENT : return SCTID_INACT_NOT_SEMANTICALLY_EQUIVALENT; + case COMPONENT_MEANING_UNKNOWN : return SCTID_INACT_COMPONENT_MEANING_UNKNOWN; + case CLASSIFICATION_DERIVED_COMPONENT : return SCTID_INACT_CLASS_DERIVED_COMPONENT; + case GRAMMATICAL_DESCRIPTION_ERROR : return SCTID_INACT_GRAMMATICAL_DESCRIPTION_ERROR; + default: throw new IllegalArgumentException("Unrecognised inactivation indicator " + ii.toString()); + } + } public static Association translateAssociation(String assocSctId) { switch (assocSctId) { @@ -677,22 +697,41 @@ public static Association translateAssociation(String assocSctId) { default: throw new IllegalArgumentException("Unrecognised historical association indicator value " + assocSctId); } } - + + public static String translateAssociation(Association assoc) { + switch (assoc) { + case WAS_A : return SCTID_ASSOC_WAS_A_REFSETID; + case POSS_REPLACED_BY : return SCTID_ASSOC_POSS_REPLACED_BY_REFSETID ; + case REPLACED_BY : return SCTID_ASSOC_REPLACED_BY_REFSETID ; + case SAME_AS : return SCTID_ASSOC_SAME_AS_REFSETID ; + case MOVED_TO : return SCTID_ASSOC_MOVED_TO_REFSETID ; + case POSS_EQUIV_TO : return SCTID_ASSOC_POSS_EQUIV_REFSETID ; + case PARTIALLY_EQUIV_TO : return SCTID_ASSOC_PART_EQUIV_REFSETID ; + case ALTERNATIVE : return SCTID_ASSOC_ALTERNATIVE_REFSETID ; + case REFERS_TO : return SCTID_ASSOC_REFERS_TO_REFSETID ; + case ANATOMY_STRUC_ENTIRE : return SCTID_ASSOC_ANATOMY_STRUC_ENTIRE_REFSETID ; + case ANATOMY_STRUC_PART : return SCTID_ASSOC_ANATOMY_STRUC_PART_REFSETID ; + default: throw new IllegalArgumentException("Unrecognised historical association indicator value " + assoc); + } + } + public static String prettyPrintHistoricalAssociations(Concept c, GraphLoader gl) throws TermServerScriptException { - String associations = ""; + return prettyPrintHistoricalAssociations(c, gl, false); + } + + public static String prettyPrintHistoricalAssociations(Concept c, GraphLoader gl, boolean includeInactivationIndicator) throws TermServerScriptException { + String associations = includeInactivationIndicator? c.getInactivationIndicator() + "\n" : ""; boolean isFirst = true; - for (AssociationEntry assoc : c.getAssociationEntries()) { - if (assoc.isActive()) { - if (!isFirst) { - associations += "\n"; - } else { - isFirst = false; - } - if (assoc.getRefsetId() == null || assoc.getTargetComponentId() == null) { - throw new TermServerScriptException("Malformed historical association encountered : " + assoc); - } - associations += translateAssociation(assoc.getRefsetId()) + ": " + gl.getConcept(assoc.getTargetComponentId()); + for (AssociationEntry assoc : c.getAssociationEntries(ActiveState.ACTIVE, true)) { + if (!isFirst) { + associations += "\n"; + } else { + isFirst = false; + } + if (assoc.getRefsetId() == null || assoc.getTargetComponentId() == null) { + throw new TermServerScriptException("Malformed historical association encountered : " + assoc); } + associations += translateAssociation(assoc.getRefsetId()) + ": " + gl.getConcept(assoc.getTargetComponentId()); } if (c.getAssociationEntries().isEmpty()) { @@ -2404,7 +2443,7 @@ public static Set getRecentlyTouchedConcepts(Collection concep } } - for (AssociationEntry a : c.getAssociations(ActiveState.ACTIVE)) { + for (AssociationEntry a : c.getAssociationEntries(ActiveState.ACTIVE)) { if (StringUtils.isEmpty(a.getEffectiveTime())) { recentlyTouched.add(c); continue nextConcept; @@ -2568,4 +2607,12 @@ public static RelationshipGroup createRelationshipGroup(GraphLoader gl, String[] public static RelationshipTemplate createRelationshipTemplate(GraphLoader gl, String c1, String c2) throws TermServerScriptException { return new RelationshipTemplate(gl.getConcept(c1), gl.getConcept(c2)); } + + public static Set getHistoricalAssocationTargets(Concept c, GraphLoader gl) throws TermServerScriptException { + Set targets = new HashSet<>(); + for (AssociationEntry h : c.getAssociationEntries()) { + targets.add(gl.getConcept(h.getTargetComponentId())); + } + return targets; + } } diff --git a/script-engine/src/main/resources/templates/procedures/InsertionOfStent.json b/script-engine/src/main/resources/templates/procedures/InsertionOfStent.json new file mode 100644 index 0000000000..e7309033e0 --- /dev/null +++ b/script-engine/src/main/resources/templates/procedures/InsertionOfStent.json @@ -0,0 +1,8 @@ +{ + "name": "Placement of stent", + "documentation": "https://confluence.ihtsdotools.org/display/SCTEMPLATES/Insertion+of+stent+%28procedure%29+-+v1.0", + "version": 1.0, + "domain": "<< 1296925008 |Insertion of stent (procedure)| MINUS ( << 77343006 |Angiography (procedure)| OR << 418285008 |Angioplasty of blood vessel (procedure)| OR << 363687006 |Endoscopic procedure (procedure)| OR (<< 1296925008 |Insertion of stent (procedure)| : << 405816004 |Procedure morphology (attribute)| = *))", + "logicalTemplate": "71388002 |Procedure (procedure)| : [[~0..1]]{[[~1..1]] 260686004 |Method (attribute)| = [[+id(<< 360037004 |Imaging - action (qualifier value)| ) @imaging]],[[~0..1]] 405813007 |Procedure site - Direct (ttribute)| = [[+id(< 442083009 |Anatomical or acquired body structure (body structure)| ) @bodyStructure]],[[~1..1]] 363703001 |Has intent (attribute)| = 429892002 |Guidance intent (qualifier value)| ,[[~0..1]] 424361007 |Using substance (attribute)| = [[+id(<< 385420005 |Contrast media (substance)| ) @contrast]]}, [[~1..1]]{ [[~1..1]] 260686004 |Method (attribute)| = [[+id (<< 257867005 |Insertion - action (qualifier alue)| )]], [[~0..1]] 260507000 |Access (attribute)| = [[+id (<< 309795001 |Surgical access values (qualifier value)| ) @access ]], [[~0..1]] 405814001 |Procedure site - Indirect (attribute)| = [[+id (< 442083009 | Anatomical or acquired body structure (body structure)| ) @bodyStructure]], [[~1..1]] 363699004 |Direct device (attribute)| = [[+id (<< 65818007 |Stent (physical object)| )]],[[~0..1]] 425391005 |Using access evice attribute)| = [[+id (<< 37270008 |Endoscope, device (physical object)| )]], [[~0..1]] 424876005 |Surgical approach (attribute)| = [[+id(< 103379005 |Procedural approach (qualifier value)| ) @approach]]}" +} + diff --git a/script-engine/src/main/resources/templates/procedures/PlacementOfStent.json b/script-engine/src/main/resources/templates/procedures/PlacementOfStent.json deleted file mode 100644 index c27c6339a1..0000000000 --- a/script-engine/src/main/resources/templates/procedures/PlacementOfStent.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "name": "Placement of stent", - "documentation": "https://confluence.ihtsdotools.org/display/SCTEMPLATES/Insertion+of+stent+%28procedure%29+-+v1.0", - "version": 1.0, - "domain": "(<< 103716009 |Placement of stent (procedure)| MINUS (<< 103716009 |Placement of stent (procedure)| : << 363700003 |Direct morphology (attribute)| = *)) AND (<< 103716009 |Placement of stent (procedure)| MINUS (<< 77343006 |Angiography (procedure)| )) AND (<< 103716009 |Placement of stent (procedure)| MINUS (<< 418285008 |Angioplasty of blood vessel (procedure)| ))", - "logicalTemplate": "71388002 |Procedure (procedure)| : [[~0..1]]{[[~1..1]] 260686004 |Method (attribute)| = [[+id(<< 360037004 |Imaging - action (qualifier value)| ) @imaging]],[[~0..1]] 405813007 |Procedure site - Direct (attribute)| = [[+id(< 442083009 |Anatomical or acquired body structure (body structure)| ) @bodyStructure]],[[~1..1]] 363703001 |Has intent (attribute)| = 429892002 |Guidance intent (qualifier value)| ,[[~0..1]] 424361007 |Using substance (attribute)| = [[+id(<< 385420005 |Contrast media (substance)| ) @contrast]]}, [[~1..1]]{ [[~1..1]] 260686004 |Method (attribute)| = [[+id (<< 257867005 |Insertion - action (qualifier value)| )]], [[~0..1]] 260507000 |Access (attribute)| = 103388001 |Percutaneous approach - access (qualifier value)| , [[~0..1]] 405814001 |Procedure site - Indirect (attribute)| = [[+id (< 442083009 |Anatomical or acquired body structure (body structure)| ) @bodyStructure]], [[~1..1]] 363699004 |Direct device (attribute)| = [[+id (<< 65818007 |Stent (physical object)| )]],[[~0..1]] 425391005 |Using access device (attribute)| = [[+id (<< 37270008 |Endoscope, device (physical object)| )]], [[~0..1]] 424876005 |Surgical approach (attribute)| = [[+id(< 103379005 |Procedural approach (qualifier value)| ) @approach]]}" -} - diff --git a/techstack.md b/techstack.md new file mode 100644 index 0000000000..d369918a5e --- /dev/null +++ b/techstack.md @@ -0,0 +1,195 @@ + +
+ +# Tech Stack File +![](https://img.stackshare.io/repo.svg "repo") [IHTSDO/reporting-engine](https://github.com/IHTSDO/reporting-engine)![](https://img.stackshare.io/public_badge.svg "public") +

+|28
Tools used|03/21/24
Report generated| +|------|------| +
+ +## Languages (2) + + + + + + +
+ Groovy +
+ Groovy +
+ +		 + Java +
+ Java +
+ +
+ +## Frameworks (1) + + + + +
+ Spring Framework +
+ Spring Framework +
+ +
+ +## Data (1) + + + + +
+ ActiveMQ +
+ ActiveMQ +
+ +
+ +## DevOps (5) + + + + + + + + + + + + +
+ Git +
+ Git +
+ +		 + JUnit +
+ JUnit +
+ +		 + Logback +
+ Logback +
+ +		 + SLF4J +
+ SLF4J +
+ +		 + Spock Framework +
+ Spock Framework +
+ +
+ +## Hosting (1) + + + + +
+ Apache Camel +
+ Apache Camel +
+ +
+ +## Other (1) + + + + +
+ Shell +
+ Shell +
+ +
+ + +## Open source packages (17) + +## Apache Maven (17) + +|NAME|VERSION|LAST UPDATED|LAST UPDATED BY|LICENSE|VULNERABILITIES| +|:------|:------|:------|:------|:------|:------| +|[com.google.gdata:core](http://code.google.com/p/gdata-java-client/)|N/A|10/30/23|pgwilliams |Apache-2.0|N/A| +|[commons-lang:commons-lang](http://commons.apache.org/lang/)|N/A|04/01/20|Peter Groves Williams |Apache-2.0|N/A| +|[commons-validator:commons-validator](http://commons.apache.org/proper/commons-validator/)|N/A|04/01/20|Peter Groves Williams |Apache-2.0|N/A| +|[junit:junit](http://junit.org)|N/A|05/14/20|Kal Parmar |EPL-1.0|N/A| +|[net.bytebuddy:byte-buddy](http://bytebuddy.net)|v1.14.10|12/01/23|Jim Cornmell |Apache-2.0|N/A| +|[org.antlr:antlr4-runtime](http://antlr.org)|v4.13.1|02/26/24|Quyen Ly |BSD-3-Clause|N/A| +|[org.apache.commons:commons-collections4](https://commons.apache.org/proper/commons-collections/)|N/A|12/01/23|Jim Cornmell |Apache-2.0|N/A| +|[org.apache.commons:commons-csv](http://commons.apache.org/proper/commons-csv/)|N/A|02/26/24|Quyen Ly |Apache-2.0|N/A| +|[org.objenesis:objenesis](http://objenesis.org)|v3.3|12/01/23|Jim Cornmell |Apache-2.0|N/A| +|[org.reflections:reflections](http://github.com/ronmamo/reflections)|N/A|09/04/23|Jim Cornmell |WTFPL|N/A| +|[org.slf4j:slf4j-api](http://www.slf4j.org)|N/A|10/13/23|Elena Ilyukhina |MIT|N/A| +|[org.spockframework:spock-core](http://spockframework.org)|N/A|09/04/23|Jim Cornmell |Apache-2.0|N/A| +|[org.springframework.boot:spring-boot-starter-data-jpa](https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter-data-jpa)|N/A|02/26/24|Quyen Ly |Apache-2.0|N/A| +|[org.springframework.boot:spring-boot-starter-security](https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter-security)|N/A|02/26/24|Quyen Ly |Apache-2.0|N/A| +|[org.springframework.boot:spring-boot-starter-test](https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter-test)|N/A|09/04/23|Jim Cornmell |Apache-2.0|N/A| +|[org.springframework.boot:spring-boot-starter-web](https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter-web)|N/A|02/26/24|Quyen Ly |Apache-2.0|N/A| +|[org.springframework:spring-web](https://github.com/spring-projects/spring-framework)|N/A|09/04/23|Jim Cornmell |Apache-2.0|N/A| + +
+
+ +Generated via [Stack File](https://github.com/marketplace/stack-file) diff --git a/techstack.yml b/techstack.yml new file mode 100644 index 0000000000..2e6d9d2d7b --- /dev/null +++ b/techstack.yml @@ -0,0 +1,356 @@ +repo_name: IHTSDO/reporting-engine +report_id: 7879009fde4e7ef246e8c5ebedbdb7e8 +version: 0.1 +repo_type: Public +timestamp: '2024-03-21T12:57:30+00:00' +requested_by: pgwilliams +provider: github +branch: develop +detected_tools_count: 28 +tools: +- name: Groovy + description: A multi-faceted language for the Java platform + website_url: https://groovy-lang.org/ + license: Apache-2.0 + open_source: true + hosted_saas: false + category: Languages & Frameworks + sub_category: Languages + image_url: https://img.stackshare.io/service/997/default_7ff5fcd857f42ad25149f659693d8930bffddf14.png + detection_source_url: https://github.com/IHTSDO/reporting-engine + detection_source: Repo Metadata +- name: Java + description: A concurrent, class-based, object-oriented, language specifically designed + to have as few implementation dependencies as possible + website_url: https://www.java.com + open_source: true + hosted_saas: false + category: Languages & Frameworks + sub_category: Languages + image_url: https://img.stackshare.io/service/995/K85ZWV2F.png + detection_source_url: https://github.com/IHTSDO/reporting-engine + detection_source: Repo Metadata +- name: Spring Framework + description: An application framework and inversion of control container for the + Java platform + website_url: https://spring.io/projects/spring-framework + license: Apache-2.0 + open_source: true + hosted_saas: false + category: Languages & Frameworks + sub_category: Frameworks (Full Stack) + image_url: https://img.stackshare.io/service/2006/spring-framework-project-logo.png + detection_source_url: https://github.com/IHTSDO/reporting-engine/blob/develop/reporting-engine-worker/pom.xml + detection_source: reporting-engine-worker/pom.xml + last_updated_by: Jim Cornmell + last_updated_on: 2023-09-04 15:42:07.000000000 Z +- name: ActiveMQ + description: A message broker written in Java together with a full JMS client + website_url: http://activemq.apache.org/ + license: Apache-2.0 + open_source: true + hosted_saas: false + category: Data Stores + sub_category: Message Queue + image_url: https://img.stackshare.io/service/1062/default_08edb6f82e2c79424efc1e297ab096e50acd8e0b.jpg + detection_source_url: https://github.com/IHTSDO/reporting-engine/blob/develop/reporting-engine-worker/pom.xml + detection_source: reporting-engine-worker/pom.xml + last_updated_by: Peter Groves Williams + last_updated_on: 2018-09-14 12:27:11.000000000 Z +- name: Git + description: Fast, scalable, distributed revision control system + website_url: http://git-scm.com/ + open_source: true + hosted_saas: false + category: Build, Test, Deploy + sub_category: Version Control System + image_url: https://img.stackshare.io/service/1046/git.png + detection_source_url: https://github.com/IHTSDO/reporting-engine + detection_source: Repo Metadata +- name: JUnit + description: A programmer-oriented testing framework for Java + website_url: http://junit.org/ + license: EPL-1.0 + open_source: true + hosted_saas: false + category: Build, Test, Deploy + sub_category: Testing Frameworks + image_url: https://img.stackshare.io/service/2020/874086.png + detection_source_url: https://github.com/IHTSDO/reporting-engine/blob/develop/script-engine/pom.xml + detection_source: script-engine/pom.xml + last_updated_by: Peter Groves Williams + last_updated_on: 2020-04-01 13:50:56.000000000 Z +- name: Logback + description: A logging framework for Java applications + website_url: https://logback.qos.ch/ + open_source: false + hosted_saas: false + category: Monitoring + sub_category: Log Management + image_url: https://img.stackshare.io/service/2923/05518ecaa42841e834421e9d6987b04f_400x400.png + detection_source_url: https://github.com/IHTSDO/reporting-engine/blob/develop/reporting-engine-worker/src/main/resources/logback.xml + detection_source: reporting-engine-worker/src/main/resources/logback.xml + last_updated_by: Darryn McGaw + last_updated_on: 2023-06-07 10:22:55.000000000 Z +- name: SLF4J + description: Simple logging facade for Java + website_url: http://slf4j.org/ + open_source: false + hosted_saas: false + category: Monitoring + sub_category: Log Management + image_url: https://img.stackshare.io/service/2805/05518ecaa42841e834421e9d6987b04f_400x400.png + detection_source_url: https://github.com/IHTSDO/reporting-engine/blob/develop/reporting-engine-worker/pom.xml + detection_source: reporting-engine-worker/pom.xml + last_updated_by: Peter Groves Williams + last_updated_on: 2018-08-29 13:38:25.000000000 Z +- name: Spock Framework + description: The Enterprise-ready testing and specification framework + website_url: http://spockframework.org/ + license: Apache-2.0 + open_source: true + hosted_saas: false + category: Build, Test, Deploy + sub_category: Testing Frameworks + image_url: https://img.stackshare.io/service/5548/UB74SoK2_400x400.png + detection_source_url: https://github.com/IHTSDO/reporting-engine/blob/develop/reporting-engine-worker/pom.xml + detection_source: reporting-engine-worker/pom.xml + last_updated_by: Jim Cornmell + last_updated_on: 2023-09-04 15:42:07.000000000 Z +- name: Apache Camel + description: A versatile open source integration framework + website_url: https://camel.apache.org/ + license: Apache-2.0 + open_source: true + hosted_saas: false + category: Application Hosting + sub_category: Platform as a Service + image_url: https://img.stackshare.io/service/3276/xWt1RFo6_400x400.jpg + detection_source_url: https://github.com/IHTSDO/reporting-engine/blob/develop/reporting-engine-worker/src/main/resources/application.properties + detection_source: reporting-engine-worker/src/main/resources/application.properties + last_updated_by: Peter Groves Williams + last_updated_on: 2018-09-14 12:27:11.000000000 Z +- name: Shell + description: A shell is a text-based terminal, used for manipulating programs and + files. Shell scripts typically manage program execution. + website_url: https://en.wikipedia.org/wiki/Shell_script + open_source: false + hosted_saas: false + category: Languages & Frameworks + sub_category: Languages + image_url: https://img.stackshare.io/service/4631/default_c2062d40130562bdc836c13dbca02d318205a962.png + detection_source_url: https://github.com/IHTSDO/reporting-engine + detection_source: Repo Metadata +- name: com.google.gdata:core + description: The Google Data Java client library is written by Google + license: Apache-2.0 + open_source: true + hosted_saas: false + category: Libraries + sub_category: Maven Packages + image_url: https://img.stackshare.io/package/maven/image.png + detection_source_url: https://github.com/IHTSDO/reporting-engine/blob/develop/schedule-manager/pom.xml + detection_source: schedule-manager/pom.xml + last_updated_by: pgwilliams + last_updated_on: 2023-10-30 16:08:06.000000000 Z +- name: commons-lang:commons-lang + description: Commons Lang + license: Apache-2.0 + open_source: true + hosted_saas: false + category: Libraries + sub_category: Maven Packages + image_url: https://img.stackshare.io/package/maven/image.png + detection_source_url: https://github.com/IHTSDO/reporting-engine/blob/develop/script-engine/pom.xml + detection_source: script-engine/pom.xml + last_updated_by: Peter Groves Williams + last_updated_on: 2020-04-01 13:50:56.000000000 Z +- name: commons-validator:commons-validator + description: Apache Commons Validator provides the building blocks for both client + side validation and server side data validation + license: Apache-2.0 + open_source: true + hosted_saas: false + category: Libraries + sub_category: Maven Packages + image_url: https://img.stackshare.io/package/maven/image.png + detection_source_url: https://github.com/IHTSDO/reporting-engine/blob/develop/script-engine/pom.xml + detection_source: script-engine/pom.xml + last_updated_by: Peter Groves Williams + last_updated_on: 2020-04-01 13:50:56.000000000 Z +- name: junit:junit + description: JUnit is a unit testing framework for Java + license: EPL-1.0 + open_source: true + hosted_saas: false + category: Libraries + sub_category: Maven Packages + image_url: https://img.stackshare.io/package/maven/image.png + detection_source_url: https://github.com/IHTSDO/reporting-engine/blob/develop/script-engine/pom.xml + detection_source: script-engine/pom.xml + last_updated_by: Kal Parmar + last_updated_on: 2020-05-14 16:49:33.000000000 Z +- name: net.bytebuddy:byte-buddy + description: Byte Buddy is a Java library for creating Java classes at run time + version: 1.14.10 + license: Apache-2.0 + open_source: true + hosted_saas: false + category: Libraries + sub_category: Maven Packages + image_url: https://img.stackshare.io/package/maven/image.png + detection_source_url: https://github.com/IHTSDO/reporting-engine/blob/develop/script-engine/pom.xml + detection_source: script-engine/pom.xml + last_updated_by: Jim Cornmell + last_updated_on: 2023-12-01 13:00:22.000000000 Z +- name: org.antlr:antlr4-runtime + description: The ANTLR 4 Runtime + version: 4.13.1 + license: BSD-3-Clause + open_source: true + hosted_saas: false + category: Libraries + sub_category: Maven Packages + image_url: https://img.stackshare.io/package/maven/image.png + detection_source_url: https://github.com/IHTSDO/reporting-engine/blob/develop/schedule-manager/pom.xml + detection_source: schedule-manager/pom.xml + last_updated_by: Quyen Ly + last_updated_on: 2024-02-26 11:44:41.000000000 Z +- name: org.apache.commons:commons-collections4 + description: The Apache Commons Collections package contains types that extend and + augment the Java Collections Framework + license: Apache-2.0 + open_source: true + hosted_saas: false + category: Libraries + sub_category: Maven Packages + image_url: https://img.stackshare.io/package/maven/image.png + detection_source_url: https://github.com/IHTSDO/reporting-engine/blob/develop/script-engine/pom.xml + detection_source: script-engine/pom.xml + last_updated_by: Jim Cornmell + last_updated_on: 2023-12-01 13:00:22.000000000 Z +- name: org.apache.commons:commons-csv + description: The Apache Commons CSV library provides a simple interface for reading + and writing CSV files of various types + license: Apache-2.0 + open_source: true + hosted_saas: false + category: Libraries + sub_category: Maven Packages + image_url: https://img.stackshare.io/package/maven/image.png + detection_source_url: https://github.com/IHTSDO/reporting-engine/blob/develop/schedule-manager/pom.xml + detection_source: schedule-manager/pom.xml + last_updated_by: Quyen Ly + last_updated_on: 2024-02-26 11:44:41.000000000 Z +- name: org.objenesis:objenesis + description: A library for instantiating Java objects + version: '3.3' + license: Apache-2.0 + open_source: true + hosted_saas: false + category: Libraries + sub_category: Maven Packages + image_url: https://img.stackshare.io/package/maven/image.png + detection_source_url: https://github.com/IHTSDO/reporting-engine/blob/develop/script-engine/pom.xml + detection_source: script-engine/pom.xml + last_updated_by: Jim Cornmell + last_updated_on: 2023-12-01 13:00:22.000000000 Z +- name: org.reflections:reflections + description: Reflections - a Java runtime metadata analysis + license: WTFPL + open_source: true + hosted_saas: false + category: Libraries + sub_category: Maven Packages + image_url: https://img.stackshare.io/package/maven/image.png + detection_source_url: https://github.com/IHTSDO/reporting-engine/blob/develop/reporting-engine-worker/pom.xml + detection_source: reporting-engine-worker/pom.xml + last_updated_by: Jim Cornmell + last_updated_on: 2023-09-04 15:42:07.000000000 Z +- name: org.slf4j:slf4j-api + description: The slf4j API + license: MIT + open_source: true + hosted_saas: false + category: Libraries + sub_category: Maven Packages + image_url: https://img.stackshare.io/package/maven/image.png + detection_source_url: https://github.com/IHTSDO/reporting-engine/blob/develop/reporting-engine-worker/pom.xml + detection_source: reporting-engine-worker/pom.xml + last_updated_by: Elena Ilyukhina + last_updated_on: 2023-10-13 17:10:22.000000000 Z +- name: org.spockframework:spock-core + description: Spock is a testing and specification framework for Java and Groovy + applications + license: Apache-2.0 + open_source: true + hosted_saas: false + category: Libraries + sub_category: Maven Packages + image_url: https://img.stackshare.io/package/maven/image.png + detection_source_url: https://github.com/IHTSDO/reporting-engine/blob/develop/reporting-engine-worker/pom.xml + detection_source: reporting-engine-worker/pom.xml + last_updated_by: Jim Cornmell + last_updated_on: 2023-09-04 15:42:07.000000000 Z +- name: org.springframework.boot:spring-boot-starter-data-jpa + description: Starter for using Spring Data JPA with Hibernate + license: Apache-2.0 + open_source: true + hosted_saas: false + category: Libraries + sub_category: Maven Packages + image_url: https://img.stackshare.io/package/maven/image.png + detection_source_url: https://github.com/IHTSDO/reporting-engine/blob/develop/schedule-manager/pom.xml + detection_source: schedule-manager/pom.xml + last_updated_by: Quyen Ly + last_updated_on: 2024-02-26 11:44:41.000000000 Z +- name: org.springframework.boot:spring-boot-starter-security + description: Starter for using Spring Security + license: Apache-2.0 + open_source: true + hosted_saas: false + category: Libraries + sub_category: Maven Packages + image_url: https://img.stackshare.io/package/maven/image.png + detection_source_url: https://github.com/IHTSDO/reporting-engine/blob/develop/schedule-manager/pom.xml + detection_source: schedule-manager/pom.xml + last_updated_by: Quyen Ly + last_updated_on: 2024-02-26 11:44:41.000000000 Z +- name: org.springframework.boot:spring-boot-starter-test + description: Starter for testing Spring Boot applications with libraries including + JUnit + license: Apache-2.0 + open_source: true + hosted_saas: false + category: Libraries + sub_category: Maven Packages + image_url: https://img.stackshare.io/package/maven/image.png + detection_source_url: https://github.com/IHTSDO/reporting-engine/blob/develop/reporting-engine-worker/pom.xml + detection_source: reporting-engine-worker/pom.xml + last_updated_by: Jim Cornmell + last_updated_on: 2023-09-04 15:42:07.000000000 Z +- name: org.springframework.boot:spring-boot-starter-web + description: Starter for building web, including RESTful, applications using Spring + MVC + license: Apache-2.0 + open_source: true + hosted_saas: false + category: Libraries + sub_category: Maven Packages + image_url: https://img.stackshare.io/package/maven/image.png + detection_source_url: https://github.com/IHTSDO/reporting-engine/blob/develop/schedule-manager/pom.xml + detection_source: schedule-manager/pom.xml + last_updated_by: Quyen Ly + last_updated_on: 2024-02-26 11:44:41.000000000 Z +- name: org.springframework:spring-web + description: Spring Web + license: Apache-2.0 + open_source: true + hosted_saas: false + category: Libraries + sub_category: Maven Packages + image_url: https://img.stackshare.io/package/maven/image.png + detection_source_url: https://github.com/IHTSDO/reporting-engine/blob/develop/reporting-engine-worker/pom.xml + detection_source: reporting-engine-worker/pom.xml + last_updated_by: Jim Cornmell + last_updated_on: 2023-09-04 15:42:07.000000000 Z