2023 09 gg misc fixes

RELEASE_NOTES.md

@@ -0,0 +1,26 @@
+
+* Loader: Significant Performance improvements parsing JSON resources + other performance improvements
+* Loader: New Parameters apply-wg and default-wg
+* NpmPackages: Start generating .index.db as well as .index.json in packages for faster package reading
+* Validator: Refactor Type handling for faster performance
+* Validator: Validate the stated publisher/WG/contacts for HL7 published resources
+* Validator: Better error message when diff contains bad paths
+* Validator: Pass dependent resources to server and make sure cache-id is filled out properly in all contexts
+* Validator: Fix error in FML parser parsing parameters
+* Validator: Fix issue with dom-6 and contained elements (Internal ChildMap synchro issues)
+* Validator: Better handling of errors from tx.fhir.org
+* Validator: Fix bug checking for implicit value sets
+* Validator: Fix bug checking of mixing snomed display types
+* Validator: Reduce size of validatable concept map to 500 - for now + better handling of errors on server batches
+* Validator: Improve UCUM validation BP rule
+* Validator: set example status when valdiating
+* Validator: don't check examples for deprecated / withdrawn content
+* Renderer: Fix up handling of includes in liquid templates
+* Renderer: Fix up rendering of profile names for abstract profile instantiations
+* Renderer: Improved rendering of codes in include when rendering valuesets
+* Renderer: Fix problem caching look up of implied value sets
+* Renderer: Display workgroup in extension summary table
+* QA: Report max memory used
+* Publication Process: fix issues generating and releasing templates
+* Publication Process: Fix typo in version default logic
+* XIG: Start rewrite (phase 1)

org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/publisher/IGPublisherLiquidTemplateServices.java

@@ -49,7 +49,10 @@ public void load(String path) throws FileNotFoundException, IOException {
  String fn = f.getName();
  fn = fn.substring(0, fn.indexOf("."));
  templates.put(fn.toLowerCase(), TextFile.fileToString(f));
- }
+ } else if (f.getName().endsWith(".html")) {
+ String fn = f.getName();
+ templates.put(fn, TextFile.fileToString(f));
+ } 
  }
  }
 

org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/publisher/Publisher.java

@@ -772,9 +772,11 @@ public enum GenerationTool {
  private SPDXLicense licenseInfo;
  private String publisher;
  private String businessVersion;
+ private String wgm;
  private List<ContactDetail> defaultContacts;
  private List<UsageContext> defaultContexts;
  private String defaultCopyright;
+ private String defaultWgm;
  private List<CodeableConcept> defaultJurisdictions;
  private SPDXLicense defaultLicenseInfo;
  private String defaultPublisher;
@@ -919,6 +921,7 @@ public void setNoSushi(boolean noSushi) {
  private List<String> translationLangs = new ArrayList<>();
  private List<String> translationSupplements = new ArrayList<>();
  private int validationLogTime = 0;
+ private long maxMemory = 0;
 
  private class PreProcessInfo {
  private String xsltName;
@@ -1011,7 +1014,7 @@ private void packageTemplate() throws IOException {
  File od = new File(outputDir);
  FileUtils.cleanDirectory(od);
  npm = new NPMPackageGenerator(Utilities.path(outputDir, "package.tgz"), templateInfo, execTime.getTime(), !publishing);
- npm.loadFiles(rootDir, new File(rootDir), ".git", "output", "package");
+ npm.loadFiles(rootDir, new File(rootDir), ".git", "output", "package", "temp");
  npm.finish();
 
  TextFile.stringToFile(makeTemplateIndexPage(), Utilities.path(outputDir, "index.html"), false);
@@ -1038,6 +1041,7 @@ private void packageTemplate() throws IOException {
  zip.addFileSource("index.html", REDIRECT_SOURCE, false);
  zip.close();
  Utilities.copyFile(Utilities.path(tempDir, "full-ig.zip"), Utilities.path(outputDir, "full-ig.zip"));
+ new File(Utilities.path(tempDir, "full-ig.zip")).delete();
 
  // registering the package locally
  log("Finished. "+DurationUtil.presentDuration(endTime - startTime)+". Output in "+outputDir);
@@ -1060,18 +1064,18 @@ private String makeTemplateIndexPage() {
  }
 
  private String makeTemplateJekyllIndexPage() {
- String page = "---"+
- "layout: page"+
- "title: {{npm}}"+
- "---"+
+ String page = "---\r\n"+
+ "layout: page\r\n"+
+ "title: {{npm}}\r\n"+
+ "---\r\n"+
  " <p><b>Template {{npm}}</b></p>\r\n"+
  " <p>You can <a href=\"package.tgz\">download the template</a>, though you should not need to; just refer to the template as {{npm}} in your IG configuration.</p>\r\n"+
  " <p>A <a href=\"{{canonical}}/history.html\">full version history is published</a></p>\r\n";
  return page.replace("{{npm}}", templateInfo.asString("name")).replace("{{canonical}}", templateInfo.asString("canonical"));
  }
 
  private String makeTemplateQAPage() {
- String page = "<!DOCTYPE HTML><html xmlns=\"http://www.w3.org/1999/xhtml\" xml:lang=\"en\" lang=\"en\"><head><title>Template QA Page</title></head><body><p><b>Template {{npm}}</b></p><p>You can <a href=\"package.tgz\">download the template</a>, though you should not need to; just refer to the template as {{npm}} in your IG configuration.</p></body></html>";
+ String page = "<!DOCTYPE HTML><html xmlns=\"http://www.w3.org/1999/xhtml\" xml:lang=\"en\" lang=\"en\"><head><title>Template QA Page</title></head><body><p><b>Template {{npm}} QA</b></p><p>No useful QA on templates - if you see this page, the template buitl ok.</p></body></html>";
  return page.replace("{{npm}}", templateInfo.asString("name"));
  }
 
@@ -1141,7 +1145,7 @@ public void createIg() throws Exception, IOException, EOperationOutcome, FHIRExc
  log("Validation output in "+val.generate(sourceIg.getName(), errors, fileList, Utilities.path(destDir != null ? destDir : outputDir, "qa.html"), suppressedMessages));
  }
  recordOutcome(null, val);
- log("Finished");
+ log("Finished. Max Memory Used = "+Utilities.describeSize(maxMemory));
  } catch (Exception e) {
  try {
  recordOutcome(e, null);
@@ -1396,6 +1400,7 @@ private void recordOutcome(Exception ex, ValidationPresenter val) {
  j.add("template", templatePck);
  }
  j.add("tool", Constants.VERSION+" ("+ToolsVersion.TOOLS_VERSION+")");
+ j.add("maxMemory", maxMemory);
  String json = org.hl7.fhir.utilities.json.parser.JsonParser.compose(j, true);
  TextFile.stringToFile(json, Utilities.path(destDir != null ? destDir : outputDir, "qa.json"), false);
 
@@ -2705,6 +2710,11 @@ private void initializeFromIg(IniFile ini) throws Exception {
  businessVersion = sourceIg.getVersion();
  }
  break;
+ case "apply-wg":
+ if (p.getValue().equals("true")) {
+ wgm = ToolingExtensions.readStringExtension(sourceIg, ToolingExtensions.EXT_WORKGROUP);
+ }
+ break;
  case "default-contact":
  if (p.getValue().equals("true")) {
  defaultContacts = sourceIg.getContact();
@@ -2740,6 +2750,11 @@ private void initializeFromIg(IniFile ini) throws Exception {
  defaultBusinessVersion = sourceIg.getVersion();
  }
  break;
+ case "default-wg":
+ if (p.getValue().equals("true")) {
+ defaultWgm = ToolingExtensions.readStringExtension(sourceIg, ToolingExtensions.EXT_WORKGROUP);
+ }
+ break;
  case "generate-version": 
  generateVersions.add(p.getValue());
  break;
@@ -4431,6 +4446,9 @@ private boolean load() throws Exception {
  publishedIg.addFhirVersion(FHIRVersion.fromCode(version));
  if (!publishedIg.hasVersion() && businessVersion != null)
  publishedIg.setVersion(businessVersion);
+ if (!publishedIg.hasExtension(ToolingExtensions.EXT_WORKGROUP) && wgm != null) {
+ publishedIg.addExtension(ToolingExtensions.EXT_WORKGROUP, new CodeType(wgm));
+ }
 
  String id = npmName;
  if (npmName.startsWith("hl7.")) {
@@ -5559,8 +5577,8 @@ private void loadConformance() throws Exception {
  loadDepInfo();
  log("Load Info");
  loadInfo();
+ log("Load Resources");
  for (String s : metadataResourceNames()) { 
- log("Load "+s);
  load(s);
  }
  log("Load Paths");
@@ -6342,20 +6360,31 @@ private void load(String type) throws Exception {
  }
  CommaSeparatedStringBuilder b = new CommaSeparatedStringBuilder();
  if (businessVersion != null) {
- if (!bc.hasVersion()) {
+ altered = true;
+ b.append("version="+businessVersion);
+ bc.setVersion(businessVersion);
+ } else if (defaultBusinessVersion != null && !bc.hasVersion()) {
+ altered = true;
+ b.append("version="+defaultBusinessVersion);
+ bc.setVersion(defaultBusinessVersion);
+ }
+
+ if (wgm != null) {
+ if (!bc.hasExtension(ToolingExtensions.EXT_WORKGROUP)) {
  altered = true;
- b.append("version="+businessVersion);
- bc.setVersion(businessVersion);
- } else if (!bc.getVersion().equals(businessVersion)) {
+ b.append("wg="+wgm);
+ bc.addExtension(ToolingExtensions.EXT_WORKGROUP, new CodeType(wgm));
+ } else if (!wgm.equals(ToolingExtensions.readStringExtension(bc, ToolingExtensions.EXT_WORKGROUP))) {
  altered = true;
- b.append("version="+businessVersion);
- bc.setVersion(businessVersion);
+ b.append("wg="+wgm);
+ bc.getExtensionByUrl(ToolingExtensions.EXT_WORKGROUP).setValue(new CodeType(wgm));
  }
- } else if (defaultBusinessVersion != null && !bc.getVersionElement().isEmpty()) {
+ } else if (defaultWgm != null && !bc.hasExtension(ToolingExtensions.EXT_WORKGROUP)) {
  altered = true;
- b.append("version="+defaultBusinessVersion);
- bc.setVersion(defaultBusinessVersion);
+ b.append("wg="+defaultWgm);
+ bc.addExtension(ToolingExtensions.EXT_WORKGROUP, new CodeType(defaultWgm));
  }
+
  if (contacts != null && !contacts.isEmpty()) {
  altered = true;
  b.append("contact");
@@ -7054,7 +7083,7 @@ private FetchedFile findFileForResource(FetchedResource r) {
 
  public void validateSD(FetchedFile f, FetchedResource r) {
  StructureDefinition sd = (StructureDefinition) r.getResource();
- if (!sd.getAbstract()) {
+ if (!sd.getAbstract() && !isClosing(sd)) {
  if (sd.getKind() == StructureDefinitionKind.RESOURCE) {
  int cE = countStatedExamples(sd.getUrl());
  int cI = countFoundExamples(sd.getUrl());
@@ -7085,6 +7114,18 @@ public void validateSD(FetchedFile f, FetchedResource r) {
  }
  }
 
+ private boolean isClosing(StructureDefinition sd) {
+ StandardsStatus ss = ToolingExtensions.getStandardsStatus(sd);
+ if (ss == StandardsStatus.DEPRECATED || ss == StandardsStatus.WITHDRAWN) {
+ return true;
+ }
+ if (sd.getStatus() == PublicationStatus.RETIRED) {
+ return true; 
+ }
+ return false;
+ }
+
+
  private int countUsages(String fixedUrl) {
  int res = 0;
  for (FetchedFile f : fileList) {
@@ -7162,6 +7203,7 @@ private void validate(FetchedFile file, FetchedResource r) throws Exception {
  List<ValidationMessage> errs = new ArrayList<ValidationMessage>();
  r.getElement().setUserData("igpub.context.file", file);
  r.getElement().setUserData("igpub.context.resource", r);
+ validator.setExample(r.isExample());
  if (r.isValidateAsResource()) { 
  Resource res = r.getResource();
  if (res instanceof Bundle) {
@@ -11530,8 +11572,6 @@ public void setLogger(ILoggingService logger) {
  public String getQAFile() throws IOException {
  return Utilities.path(outputDir, "qa.html");
  }
-
- private String lastMsg;
 
  @Override
  public void logMessage(String msg) {
@@ -11544,12 +11584,15 @@ public void logMessage(String msg) {
  long totalMemory = runtime.totalMemory();
  long freeMemory = runtime.freeMemory();
  long usedMemory = totalMemory - freeMemory;
+ if (usedMemory > maxMemory) {
+ maxMemory = usedMemory;
+ }
 
  String s = msg;
  if (tt == null) {
- System.out.println(Utilities.padRight(s, ' ', 190)); 
+ System.out.println(Utilities.padRight(s, ' ', 100)); 
  } else { // if (tt.longerThan(4)) {
- System.out.println(Utilities.padRight(s, ' ', 190)+" ("+tt.milestone()+" / "+tt.clock()+", "+Utilities.describeSize(usedMemory)+")");
+ System.out.println(Utilities.padRight(s, ' ', 100)+" ("+tt.milestone()+" / "+tt.clock()+", "+Utilities.describeSize(usedMemory)+")");
  }
  if (killFile != null && killFile.exists()) {
  killFile.delete();

org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/publisher/R4ToR4BAnalyser.java

@@ -431,7 +431,7 @@ private void genSameVersionPackage(String pid, String source, String dest, boole
  private void processFileSame(NPMPackageGenerator gen, String folder, String filename, byte[] content, Map<String, ResPointer> exemptions, String ver, String pver) {
  try {
  if (Utilities.existsInList(folder, "package", "example")) {
- if (!Utilities.existsInList(filename, "package.json", ".index.json")) {
+ if (!Utilities.existsInList(filename, "package.json", ".index.json", ".index.db")) {
  org.hl7.fhir.r4b.model.Resource res = new org.hl7.fhir.r4b.formats.JsonParser().parse(content);
  boolean exempt = (exemptions.containsKey(res.fhirType()+"/"+res.getIdBase()) ||
  ((res instanceof org.hl7.fhir.r4b.model.CanonicalResource) && exemptions.containsKey(((org.hl7.fhir.r4b.model.CanonicalResource) res).getUrl())));
@@ -501,7 +501,7 @@ private void genOtherVersionPackage(String pid, String source, String dest, Stri
  // we use R4B here, whether it's r4 or r4b - if the content is in the differences, we won't get to the this point
  private void processFileOther(NPMPackageGenerator gen, String folder, String filename, byte[] content, String ver, String pver, String nver, Map<String, ResPointer> exemptions, String pathS, String pathT) throws IOException {
  if (Utilities.existsInList(folder, "package", "example")) {
- if (!Utilities.existsInList(filename, "package.json", ".index.json")) {
+ if (!Utilities.existsInList(filename, "package.json", ".index.json", ".index.db")) {
  org.hl7.fhir.r4b.model.Resource res = new org.hl7.fhir.r4b.formats.JsonParser().parse(content);
  boolean exempt = (exemptions.containsKey(res.fhirType()+"/"+res.getId()) ||
  ((res instanceof org.hl7.fhir.r4b.model.CanonicalResource) && exemptions.containsKey(((org.hl7.fhir.r4b.model.CanonicalResource) res).getUrl())));

org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/publisher/SimpleFetcher.java

@@ -34,7 +34,7 @@
 import org.hl7.fhir.r5.context.IWorkerContext.ILoggingService;
 import org.hl7.fhir.r5.context.IWorkerContext.ILoggingService.LogCategory;
 import org.hl7.fhir.r5.elementmodel.FmlParser;
-import org.hl7.fhir.r5.elementmodel.ParserBase.NamedElement;
+import org.hl7.fhir.r5.elementmodel.ValidatedFragment;
 import org.hl7.fhir.r5.formats.FormatUtilities;
 import org.hl7.fhir.r5.model.CanonicalType;
 import org.hl7.fhir.r5.model.DataType;
@@ -342,7 +342,7 @@ public List<FetchedFile> scan(String sourceDir, IWorkerContext context, boolean
  }
  if (!ok && !Utilities.existsInList(ext, "xml", "ttl", "html", "txt", "fml")) {
  try {
- List<NamedElement> el = new org.hl7.fhir.r5.elementmodel.JsonParser(context).parse(new FileInputStream(fn));
+ List<ValidatedFragment> el = new org.hl7.fhir.r5.elementmodel.JsonParser(context).parse(new FileInputStream(fn));
  if (el.size() == 1) {
  addFile(res, f, el.get(0).getElement(), "application/fhir+json");
  count++;

org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/publisher/SpecMapManager.java

@@ -70,6 +70,7 @@ public enum SpecialPackageType {
  private String version;
  private NpmPackage pi;
  private SpecialPackageType special;
+ private int key;
 
  private SpecMapManager() {
 
@@ -428,4 +429,13 @@ public String toString() {
  return "SpecMapManager [base=" + base + ", name=" + name + ", pi=" + pi + "]";
  }
 
+ public int getKey() {
+ return key;
+ }
+
+ public void setKey(int key) {
+ this.key = key;
+ }
+
+
 }

org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/publisher/loaders/AdjunctFileLoader.java

@@ -18,6 +18,7 @@
 import org.hl7.fhir.r5.model.Property;
 import org.hl7.fhir.r5.model.RelatedArtifact.RelatedArtifactType;
 import org.hl7.fhir.r5.model.Resource;
+import org.hl7.fhir.utilities.NamedItemList;
 import org.hl7.fhir.utilities.TextFile;
 import org.hl7.fhir.utilities.Utilities;
 import org.hl7.fhir.utilities.validation.ValidationMessage;
@@ -253,7 +254,7 @@ private List<ElementWithPath> makeListOfAttachments(Element element) {
  return res;
  }
 
- private void listAttachments(List<ElementWithPath> res, List<Element> children, String path) {
+ private void listAttachments(List<ElementWithPath> res, NamedItemList<Element> children, String path) {
  String lastName = "";
  int c = -1;
  for (int i = 0; i < children.size(); i++) {

org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/publisher/realm/USRealmBusinessRules.java

@@ -227,7 +227,7 @@ public void finishChecks() throws DefinitionException, FHIRFormatError, IOExcept
  ComparisonSession session = new ComparisonSession(context, context, "Comparison of "+name+" with US-Core", pkp, pkp);
  // session.setDebug(true);
  for (ProfilePair c : comparisons) {
- System.out.println("US Core Comparison: compare "+c.local+" to "+c.uscore);
+// System.out.println("US Core Comparison: compare "+c.local+" to "+c.uscore);
  session.compare(c.uscore, c.local); 
  }
  Utilities.createDirectory(Utilities.path(dstDir, "us-core-comparisons"));

org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/publisher/utils/TemplateReleaser.java

@@ -232,6 +232,7 @@ private void build(String source, VersionDecision vd, List<VersionDecision> vers
  }
  }
  Publisher.publishDirect(Utilities.path(source, vd.getId()));
+ vd.built = true;
  }
 
  private void updateVersions(String source, VersionDecision vd, List<VersionDecision> versionsList) throws FileNotFoundException, IOException {
@@ -509,6 +510,13 @@ private void release(String dest, String source, VersionDecision vd, SimpleDateF
  Utilities.copyDirectory(Utilities.path(source, vd.getId(), "output"), Utilities.path(dest, npm.name(), npm.version()), null);
  Utilities.copyDirectory(Utilities.path(source, vd.getId(), "output"), Utilities.path(dest, npm.name()), null);
  Utilities.copyFile(Utilities.path(source, vd.getId(), "package-list.json"), Utilities.path(dest, npm.name(), "package-list.json"));
+
+ // create history page 
+ JsonObject jh = JsonParser.parseObjectFromFile(Utilities.path(source, vd.getId(), "package-list.json"));
+ String history = TextFile.fileToString(Utilities.path(dest, "history.template"));
+ history = history.replace("{{id}}", vd.getId());
+ history = history.replace("{{pl}}", JsonParser.compose(jh, false));
+ TextFile.stringToFile(history, Utilities.path(dest, vd.getId(), "history.html"));
 
  // update rss feed 
  Element item = rss.createElement("item");