Create tech stack docs (techstack.yml and techstack.md)

pom.xml

@@ -3,14 +3,14 @@
  <modelVersion>4.0.0</modelVersion>
 
  <groupId>org.snomed.otf</groupId>
- <version>7.2.1</version>
+ <version>7.3.0-SNAPSHOT</version>
  <artifactId>reporting-engine</artifactId>
  <packaging>pom</packaging>
 
  <parent>
  <groupId>org.snomed</groupId>
  <artifactId>snomed-parent-bom</artifactId>
- <version>3.2.2</version>
+ <version>3.3.0-SNAPSHOT</version>
  </parent>
 
  <modules>

reporting-engine-worker/pom.xml

@@ -6,7 +6,7 @@
 
  <parent>
  <groupId>org.snomed.otf</groupId>
- <version>7.2.1</version>
+ <version>7.3.0-SNAPSHOT</version>
  <artifactId>reporting-engine</artifactId>
  </parent>
 

reporting-engine-worker/src/main/java/org/ihtsdo/termserver/scripting/fixes/drugs/CreateMissingDrugConcepts.java

@@ -77,9 +77,6 @@ protected CreateMissingDrugConcepts(BatchFix clone) {
 
  public static void main(String[] args) throws TermServerScriptException, IOException, InterruptedException {
  Map<String, String> params = new HashMap<>();
- params.put(NEW_CONCEPTS_ONLY, "false");
- params.put(CONCEPTS_PER_TASK, "5");
- params.put(DRY_RUN, "true");
  TermServerReport.run(CreateMissingDrugConcepts.class, args, params);
  }
 
@@ -115,8 +112,8 @@ protected void preInit() throws TermServerScriptException {
 
  JobRun jobRun = getJobRun();
  newConceptsOnly = jobRun.getParamBoolean(NEW_CONCEPTS_ONLY);
- //dryRun = jobRun.getParamBoolean(DRY_RUN);
- //taskSize = Integer.parseInt(jobRun.getMandatoryParamValue(CONCEPTS_PER_TASK));
+ dryRun = jobRun.getParamBoolean(DRY_RUN);
+ taskSize = Integer.parseInt(jobRun.getMandatoryParamValue(CONCEPTS_PER_TASK));
  }
 
  public void postInit() throws TermServerScriptException {
@@ -156,13 +153,6 @@ public void postInit() throws TermServerScriptException {
  suppress.add("Product containing pituitary follicle stimulating hormone (medicinal product)");
  suppress.add("Product containing recombinant antihemophilic factor (medicinal product)");
  suppress.add("Product containing only adenosine deaminase in parenteral dose form (medicinal product form)");
- suppress.add("Product containing only pibrentasvir (medicinal product)");
- suppress.add("Product containing only bictegravir (medicinal product)");
- suppress.add("Product containing only elbasvir (medicinal product)");
- suppress.add("Product containing only glecaprevir (medicinal product)");
- suppress.add("Product containing only grazoprevir (medicinal product)");
- suppress.add("Product containing only velpatasvir (medicinal product)");
- suppress.add("Product containing only voxilaprevir (medicinal product)");
  suppress.add("Product containing only antigen of bacteria and antigen of virus (medicinal product)");
  suppress.add("Product containing only antigen of bacteria (medicinal product)");
  suppress.add("Product containing only antigen of virus (medicinal product)");
@@ -226,7 +216,7 @@ public int doFix(Task task, Concept concept, String info) throws TermServerScrip
  for (Concept required : conceptsRequired) {
  termGenerator.ensureTermsConform(task, required, CharacteristicType.STATED_RELATIONSHIP);
  required.setDefinitionStatus(DefinitionStatus.FULLY_DEFINED);
- report (task, concept, Severity.NONE, ReportActionType.INFO, "Concepts suggests need for :" + required);
+ report (task, concept, Severity.NONE, ReportActionType.INFO, "Concept suggests need for : " + required.getFsn());
 
  String expression = required.toExpression(CharacteristicType.STATED_RELATIONSHIP);
  required = createConcept(task, required, info);
@@ -238,7 +228,10 @@ public int doFix(Task task, Concept concept, String info) throws TermServerScrip
  .filter(parent -> parent.getConceptType().equals(invalidParentType))
  .map(parent -> parent.toString())
  .collect(Collectors.joining(",\n"));
- report (task, concept, Severity.LOW, ReportActionType.INFO, "Existing parents considered insufficient: " + (currentParents.isEmpty() ? "None detected" : currentParents));
+ //If we don't detect problems with the parents, then don't report anything
+ if (!currentParents.isEmpty()) {
+ report(task, concept, Severity.MEDIUM, ReportActionType.VALIDATION_CHECK, "Existing parents problematic due to same drug class level as this concept : " + currentParents);
+ }
  }
  task.addAfter(required, concept);
  //With the CD reported, we don't actually need to load it in the edit panel
@@ -410,7 +403,7 @@ protected List<Component> identifyComponentsToProcess() throws TermServerScriptE
  allAffected.add(c);
  }
  } else {
- report (SECONDARY_REPORT, (Task)null, c, Severity.LOW, ReportActionType.VALIDATION_CHECK, "No underlying CD detected for MP", mp.toExpression(CharacteristicType.STATED_RELATIONSHIP));
+ //report (SECONDARY_REPORT, (Task)null, c, Severity.LOW, ReportActionType.VALIDATION_CHECK, "No underlying CD detected for MP", mp.toExpression(CharacteristicType.STATED_RELATIONSHIP));
  }
  }
 
@@ -430,7 +423,7 @@ protected List<Component> identifyComponentsToProcess() throws TermServerScriptE
  }
  }
  } else {
- report ((Task)null, c, Severity.LOW, ReportActionType.VALIDATION_CHECK, "No underlying CD detected for MPFO", mpfo.toExpression(CharacteristicType.STATED_RELATIONSHIP));
+ //report ((Task)null, c, Severity.LOW, ReportActionType.VALIDATION_CHECK, "No underlying CD detected for MPFO", mpfo.toExpression(CharacteristicType.STATED_RELATIONSHIP));
  }
  }
  } else if (c.getConceptType().equals(ConceptType.MEDICINAL_PRODUCT)) {

reporting-engine-worker/src/main/java/org/ihtsdo/termserver/scripting/reports/InactivatedConcepts.java

@@ -71,7 +71,7 @@ public void runJob() throws TermServerScriptException {
  for (Concept c : scopeAndSort(gl.getAllConcepts())) {
  boolean reported = false;
  InactivationIndicator i = c.getInactivationIndicator();
- for (AssociationEntry a : c.getAssociations(ActiveState.ACTIVE, true)) {
+ for (AssociationEntry a : c.getAssociationEntries(ActiveState.ACTIVE, true)) {
  String assocType = SnomedUtils.getAssociationType(a);
  Concept assocValue = gl.getConcept(a.getTargetComponentId());
  report (c, i, assocType, assocValue);

reporting-engine-worker/src/main/java/org/ihtsdo/termserver/scripting/reports/managedService/ConceptsTranslatedInactivated.java

@@ -86,7 +86,7 @@ public void runJob() throws TermServerScriptException {
  boolean reported = false;
  InactivationIndicator i = c.getInactivationIndicator();
  String translations = getTranslations(c);
- for (AssociationEntry a : c.getAssociations(ActiveState.ACTIVE, true)) {
+ for (AssociationEntry a : c.getAssociationEntries(ActiveState.ACTIVE, true)) {
  String assocType = SnomedUtils.getAssociationType(a);
  Concept assocValue = gl.getConcept(a.getTargetComponentId());
  String assocTranslations = getTranslations(assocValue);

reporting-engine-worker/src/main/java/org/ihtsdo/termserver/scripting/reports/release/HistoricDataUser.java

@@ -67,6 +67,10 @@ protected void loadProjectSnapshot(boolean fsnOnly) throws TermServerScriptExcep
  if (!StringUtils.isEmpty(getJobRun().getParamValue(THIS_RELEASE))) {
  compareTwoSnapshots = true;
  projectKey = getJobRun().getParamValue(THIS_RELEASE);
+ //Have we got what looks like a zip file but someone left the .zip off?
+ if (projectKey.contains("T120000") && !projectKey.endsWith(".zip")) {
+ throw new TermServerScriptException("Suspect release '" + projectKey + "' should end with .zip");
+ }
  //If this release has been specified, the previous must also be, explicitly
  if (StringUtils.isEmpty(getJobRun().getParamValue(PREV_RELEASE))) {
  throw new TermServerScriptException("Previous release must be specified if current release is.");

reporting-engine-worker/src/main/java/org/ihtsdo/termserver/scripting/reports/release/PreReleaseContentValidation.java

@@ -296,7 +296,7 @@ private void compileSummaryCounts() {
  String msg = "Active concepts with active historical associations";
  initialiseSummaryInformation(msg);
  for (Concept c : allActiveConceptsSorted) {
- if (c.getAssociations(ActiveState.ACTIVE, true).size() > 0) {
+ if (c.getAssociationEntries(ActiveState.ACTIVE, true).size() > 0) {
  incrementSummaryInformation(projectKey);
  }
  }

reporting-engine-worker/src/main/java/org/ihtsdo/termserver/scripting/reports/release/SummaryComponentStats.java

@@ -132,10 +132,10 @@ public static void main(String[] args) throws TermServerScriptException, IOExcep
  Map<String, String> params = new HashMap<>();
  //params.put(THIS_RELEASE, "SnomedCT_USEditionRF2_PRODUCTION_20220301T120000Z.zip");
  //params.put(PREV_RELEASE, "SnomedCT_USEditionRF2_PRODUCTION_20210901T120000Z.zip");
- params.put(PREV_RELEASE, "SnomedCT_InternationalRF2_PRODUCTION_20230430T120000Z.zip");
- params.put(THIS_RELEASE, "SnomedCT_InternationalRF2_PRODUCTION_20230531T120000Z.zip");
- //params.put(REPORT_OUTPUT_TYPES, "S3");
- //params.put(REPORT_FORMAT_TYPE, "JSON");
+ params.put(PREV_RELEASE, "SnomedCT_InternationalRF2_PRODUCTION_20230901T120000Z.zip");
+ params.put(THIS_RELEASE, "SnomedCT_InternationalRF2_PRODUCTION_20231001T120000Z.zip");
+ params.put(REPORT_OUTPUT_TYPES, "S3");
+ params.put(REPORT_FORMAT_TYPE, "JSON");
  //params.put(MODULES, "731000124108");
  TermServerReport.run(SummaryComponentStats.class, args, params);
  }
@@ -302,7 +302,7 @@ private void analyzeConcepts() throws TermServerScriptException {
  analyzeComponents(isNewConcept, (datum==null?null:datum.relIds), (datum==null?null:datum.relIdsInact), summaryData[TAB_CD], concreteRels);
  analyzeComponents(isNewConcept, (datum==null?null:datum.axiomIds), (datum==null?null:datum.axiomIdsInact), summaryData[TAB_AXIOMS], c.getAxiomEntries());
  analyzeComponents(isNewConcept, (datum==null?null:datum.inactivationIds), (datum==null?null:datum.inactivationIdsInact), summaryData[TAB_INACT_IND], c.getInactivationIndicatorEntries());
- analyzeComponents(isNewConcept, (datum==null?null:datum.histAssocIds), (datum==null?null:datum.histAssocIdsInact), summaryData[TAB_HIST], c.getAssociations(ActiveState.BOTH, true));
+ analyzeComponents(isNewConcept, (datum==null?null:datum.histAssocIds), (datum==null?null:datum.histAssocIdsInact), summaryData[TAB_HIST], c.getAssociationEntries(ActiveState.BOTH, true));
  List<LangRefsetEntry> langRefsetEntries = c.getDescriptions().stream()
  .flatMap(d -> d.getLangRefsetEntries().stream())
  .collect(Collectors.toList());

reporting-engine-worker/src/main/java/org/ihtsdo/termserver/scripting/reports/release/SummaryComponentStatsExtensions.java

@@ -30,6 +30,9 @@ public static void main(String[] args) throws TermServerScriptException, IOExcep
 
  params.put(PREV_RELEASE, "SnomedCT_ManagedServiceNZ_PRODUCTION_NZ1000210_20211001T121212Z.zip");
  params.put(PREV_DEPENDENCY, "SnomedCT_InternationalRF2_PRODUCTION_20210731T120000Z.zip");
+
+ //params.put(MODULES, "21000210109"); //NZ Module
+ */
 
  params.put(THIS_RELEASE, "SnomedCT_ManagedServiceBE_PRODUCTION_BE1000172_20221115T120000Z.zip");
  params.put(THIS_DEPENDENCY, "SnomedCT_InternationalRF2_PRODUCTION_20220930T120000Z.zip");
@@ -38,10 +41,8 @@ public static void main(String[] args) throws TermServerScriptException, IOExcep
  params.put(PREV_DEPENDENCY, "SnomedCT_InternationalRF2_PRODUCTION_20220131T120000Z.zip");
 
  params.put(MODULES, "11000172109"); // Belgium Module
-
- params.put(MODULES, "21000210109"); //NZ Module
  params.put(REPORT_OUTPUT_TYPES, "S3");
- params.put(REPORT_FORMAT_TYPE, "JSON");*/
+ params.put(REPORT_FORMAT_TYPE, "JSON");
 
  TermServerReport.run(SummaryComponentStatsExtensions.class, args, params);
  }

reporting-engine-worker/src/main/java/org/ihtsdo/termserver/scripting/reports/release/ValidateInactivationsWithAssociations.java

@@ -127,7 +127,7 @@ public void runJob() throws TermServerScriptException {
 
  //Ensure that association target is active. To ignore legacy issues, both the 
  //concept AND the target must be legacy
- for (AssociationEntry a : c.getAssociations(ActiveState.ACTIVE, true)) {
+ for (AssociationEntry a : c.getAssociationEntries(ActiveState.ACTIVE, true)) {
  Concept target = gl.getConcept(a.getTargetComponentId(), false, false);
  if (target == null) {
  incrementSummaryInformation("Inactive concept association target not a concept");
@@ -297,7 +297,7 @@ public void runJob() throws TermServerScriptException {
  }
 
  private void validate(Concept c, InactivationIndicatorEntry i, List<AssociationCardinality> associationsWithCardinality, Boolean legacy) throws TermServerScriptException {
- String data = c.getAssociations(ActiveState.ACTIVE).stream()
+ String data = c.getAssociationEntries(ActiveState.ACTIVE).stream()
  .map(h->h.toString())
  .collect(Collectors.joining(",\n"));
 
@@ -310,7 +310,7 @@ private void validate(Concept c, InactivationIndicatorEntry i, List<AssociationC
  data = SnomedUtils.translateInactivationIndicator(i.getInactivationReasonId()) + "\n" + data;
  }
 
- String targets = c.getAssociations(ActiveState.ACTIVE).stream()
+ String targets = c.getAssociationEntries(ActiveState.ACTIVE).stream()
  .map(h-> SnomedUtils.translateAssociation(h.getRefsetId()).toString() + " " + gl.getConceptSafely(h.getTargetComponentId()).toString())
  .collect(Collectors.joining(",\n"));
 
@@ -349,7 +349,7 @@ private String validate(Concept c, InactivationIndicatorEntry i, String cardinal
 
  //Special case, we're getting limited inactivations with both SameAs and Was A attributes. Record stats, but skip
  if (i.getInactivationReasonId().equals(SCTID_INACT_LIMITED) || i.getInactivationReasonId().equals(SCTID_INACT_MOVED_ELSEWHERE)) {
- String typesOfAssoc = c.getAssociations(ActiveState.ACTIVE).stream()
+ String typesOfAssoc = c.getAssociationEntries(ActiveState.ACTIVE).stream()
  .map(h->SnomedUtils.translateAssociation(h.getRefsetId()).toString())
  .collect(Collectors.joining(", "));
  typesOfAssoc = typesOfAssoc.isEmpty()? "No associations" : typesOfAssoc;
@@ -358,8 +358,8 @@ private String validate(Concept c, InactivationIndicatorEntry i, String cardinal
  }
 
  //First check cardinality
- int assocCount = c.getAssociations(ActiveState.ACTIVE, assocId).size();
- int allAssocCount = c.getAssociations(ActiveState.ACTIVE, true).size();
+ int assocCount = c.getAssociationEntries(ActiveState.ACTIVE, assocId).size();
+ int allAssocCount = c.getAssociationEntries(ActiveState.ACTIVE, true).size();
 
  if (assocCount > 0 && assocCount != allAssocCount && mutuallyExclusive) {
  return inactStr + " inactivation's " + reqAssocStr + " association is mutually exclusive with other associations types.";
@@ -373,7 +373,7 @@ private String validate(Concept c, InactivationIndicatorEntry i, String cardinal
  return CARDINALITY_ISSUE;
  } else {
  //Now check association is appropriate for the inactivation indicator used
- for (AssociationEntry h : c.getAssociations(ActiveState.ACTIVE, true)) {
+ for (AssociationEntry h : c.getAssociationEntries(ActiveState.ACTIVE, true)) {
  String assocStr = SnomedUtils.translateAssociation(h.getRefsetId()).toString();
  Concept target = gl.getConcept(h.getTargetComponentId());
  //Only the "MovedTo" historical association should point to a Namespace Concept

reporting-engine-worker/src/main/resources/application.properties

@@ -33,7 +33,7 @@ resources.cloud.path=prod/international
 
 #AWS Credentials should be blank when running in an EC2 instance
 aws.key =
-aws.privateKey =
+aws.secretKey =
 
 logging.level.org.ihtsdo=DEBUG
 logging.level.ch.qos.logback.classic.joran=WARN

schedule-manager/pom.xml

@@ -6,7 +6,7 @@
 
  <parent>
  <groupId>org.snomed.otf</groupId>
- <version>7.2.1</version>
+ <version>7.3.0-SNAPSHOT</version>
  <artifactId>reporting-engine</artifactId>
  </parent>
 

script-engine/pom.xml

@@ -6,7 +6,7 @@
 
  <parent>
  <groupId>org.snomed.otf</groupId>
- <version>7.2.1</version>
+ <version>7.3.0-SNAPSHOT</version>
  <artifactId>reporting-engine</artifactId>
  </parent>
 

script-engine/src/main/java/org/ihtsdo/termserver/scripting/GraphLoader.java

@@ -600,8 +600,8 @@ public void addRelationshipToConcept(CharacteristicType charType, Relationship r
  } 
  }
 
- public Concept getConcept(String identifier) throws TermServerScriptException {
- return getConcept(identifier.trim(), true, true);
+ public Concept getConcept(String sctid) throws TermServerScriptException {
+ return getConcept(sctid.trim(), true, true);
  }
 
  public Concept getConceptSafely (String identifier) {

script-engine/src/main/java/org/ihtsdo/termserver/scripting/TermServerScript.java

@@ -1409,6 +1409,7 @@ private synchronized void recordSummaryText(String msg) {
  try {
  //Split the colon into it's own column (unless it's a time stamp!)
  if (msg.contains(":") 
+ && !msg.contains("http")
  && !msg.contains("at: ")
  && !msg.contains("\"")
  && !msg.contains("Completed processing in")) {
@@ -1502,9 +1503,9 @@ public String getReportName() {
 
  protected String getPrettyHistoricalAssociation (Concept c) throws TermServerScriptException {
  String prettyString = "No association specified.";
- if (c.getAssociations(ActiveState.ACTIVE).size() > 0) {
+ if (c.getAssociationEntries(ActiveState.ACTIVE).size() > 0) {
  prettyString = " ";
- for (AssociationEntry assoc : c.getAssociations(ActiveState.ACTIVE)) {
+ for (AssociationEntry assoc : c.getAssociationEntries(ActiveState.ACTIVE)) {
  prettyString += SnomedUtils.deconstructFSN(gl.getConcept(assoc.getRefsetId()).getFsn())[0].replace(" association reference set", "");
  prettyString += " -> ";
  prettyString += gl.getConcept(assoc.getTargetComponentId());

script-engine/src/main/java/org/ihtsdo/termserver/scripting/client/TermServerClient.java

@@ -39,8 +39,8 @@
 import com.google.gson.GsonBuilder;
 
 public class TermServerClient {
- 
- public enum ExtractType {
+
+ public enum ExtractType {
  DELTA, SNAPSHOT, FULL
  }
 
@@ -199,8 +199,11 @@ public Concept updateConcept(Concept c, String branchPath) throws TermServerScri
  }
 
  public Concept getConcept(String sctid, String branchPath) throws TermServerScriptException {
- String url = getConceptBrowserPath(branchPath) + "/" + sctid;
- return restTemplate.getForObject(url, Concept.class);
+ String url = getConceptBrowserPath(branchPath) + "/" + sctid;
+ Concept concept = restTemplate.getForObject(url, Concept.class);
+ concept.setId(concept.getConceptId()); //RestTemplate is not calling the setter so this is cheaper
+ //than writing a custom deserializer
+ return concept;
  }
 
  public Description getDescription(String descriptionId, String branchPath) {
@@ -880,4 +883,15 @@ public static String getParentBranchPath(String branchPath) {
  return branchPath.substring(0, endIndex);
  }
 
+ public boolean adminRollbackCommit(Branch b) throws TermServerScriptException {
+ String url = this.url + "/admin/" + b.getPath() + "/actions/rollback-commit?commitHeadTime=" + b.getHeadTimestamp();
+ HttpEntity<Map<String,Object>> entity = new HttpEntity<>(null);
+ try {
+ ResponseEntity<Object> response = restTemplate.exchange(url, HttpMethod.POST, entity, Object.class);
+ return response.getStatusCode().is2xxSuccessful();
+ } catch (RestClientException e) {
+ throw new TermServerScriptException(translateRestClientException(e));
+ }
+ }
+
 }

script-engine/src/main/java/org/ihtsdo/termserver/scripting/dao/ReportDataBroker.java

@@ -67,7 +67,8 @@ public void upload(File outputFile, String data) throws TermServerScriptExceptio
  //In case we're running on a PC we need to convert backslashes to forward
  String filePath = outputFile.getPath().replaceAll("\\\\", "/");
  InputStream is = IOUtils.toInputStream(data, StandardCharsets.UTF_8);
- LOGGER.info("Uploading to S3: " + outputFile);
+ s3Manager.getResourceManager().getBucketNamePath();
+ LOGGER.info("Uploading to S3 ({}): {}", s3Manager.getResourceManager().getBucketNamePath(), outputFile);
  s3Manager.getResourceManager().writeResource(filePath, is);
  } catch (Exception e) {
  throw new TermServerScriptException(e);