From a5794f577ccdfb5a53c8a779e7cae1ecb9dddc18 Mon Sep 17 00:00:00 2001 From: Grahame Grieve Date: Wed, 27 Sep 2023 11:40:58 +1000 Subject: [PATCH 1/6] Don't fail in link checking for illegal file references --- .../fhir/igtools/publisher/HTMLInspector.java | 28 ++++++++++++------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/publisher/HTMLInspector.java b/org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/publisher/HTMLInspector.java index 1e45edd1..cb6f4942 100644 --- a/org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/publisher/HTMLInspector.java +++ b/org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/publisher/HTMLInspector.java @@ -704,27 +704,35 @@ private boolean checkResolveLink(String filename, Location loc, String path, Str page = filename; } else if (page.contains("#")) { name = page.substring(page.indexOf("#")+1); + try { if (altRootFolder != null && filename.startsWith(altRootFolder)) page = Utilities.path(altRootFolder, page.substring(0, page.indexOf("#")).replace("/", File.separator)); else page = Utilities.path(rootFolder, page.substring(0, page.indexOf("#")).replace("/", File.separator)); + } catch (Exception e) { + page = null; + } } else { String folder = Utilities.getDirectoryForFile(filename); page = Utilities.path(folder == null ? (altRootFolder != null && filename.startsWith(altRootFolder) ? altRootFolder : rootFolder) : folder, page.replace("/", File.separator)); } - LoadedFile f = cache.get(page); - if (f != null) { - if (Utilities.noString(name)) - resolved = true; - else { - resolved = f.targets.contains(name); - tgtList = " (valid targets: "+(f.targets.size() > 40 ? Integer.toString(f.targets.size())+" targets" : f.targets.toString())+")"; - for (String s : f.targets) { - if (s.equalsIgnoreCase(name)) { - tgtList = (" - case is wrong ('"+s+"')"); + if (page != null) { + LoadedFile f = cache.get(page); + if (f != null) { + if (Utilities.noString(name)) + resolved = true; + else { + resolved = f.targets.contains(name); + tgtList = " (valid targets: "+(f.targets.size() > 40 ? Integer.toString(f.targets.size())+" targets" : f.targets.toString())+")"; + for (String s : f.targets) { + if (s.equalsIgnoreCase(name)) { + tgtList = (" - case is wrong ('"+s+"')"); + } } } } + } else { + resolved = false; } } } From 572aff7b27029ff13644758d6481544011dd2db5 Mon Sep 17 00:00:00 2001 From: Grahame Grieve Date: Wed, 27 Sep 2023 11:41:10 +1000 Subject: [PATCH 2/6] Don't create wrong phinvads references --- .../igtools/publisher/SpecMapManager.java | 21 +++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/publisher/SpecMapManager.java b/org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/publisher/SpecMapManager.java index 37c4670c..4536d5e1 100644 --- a/org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/publisher/SpecMapManager.java +++ b/org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/publisher/SpecMapManager.java @@ -181,10 +181,23 @@ public String getPath(String url, String def, String rt, String id) throws FHIRE if (special != null) { switch (special) { case Simplifier: return "https://simplifier.net/resolve?scope="+pi.name()+"@"+pi.version()+"&canonical="+url; - case PhinVads: if (url.startsWith(base)) - return "??phinvads??"; - else - break; + case PhinVads: + try { + if (url.startsWith(base)) { + final String fileName = Utilities.urlTail(url) + "json"; + + if ((isValidFilename(fileName) && pi.hasFile("package", fileName)) + || pi.hasCanonical(url)) { + return "phinvads_broken_link.html"; + } else { + return null; + } + + } else + break; + } catch (IOException e) { + return null; + } case Vsac: if (url.contains("cts.nlm.nih.gov")) { return url.replace("http://cts.nlm.nih.gov/fhir/ValueSet/", "https://vsac.nlm.nih.gov/valueset/")+"/expansion"; } From 398ca5d1f0e0ac6808b54af170dcb0892d59edc2 Mon Sep 17 00:00:00 2001 From: Grahame Grieve Date: Wed, 27 Sep 2023 11:41:33 +1000 Subject: [PATCH 3/6] fix wrong generation of logical models (for CDA) --- .../igtools/renderers/StructureDefinitionRenderer.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/renderers/StructureDefinitionRenderer.java b/org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/renderers/StructureDefinitionRenderer.java index df612681..857b44c7 100644 --- a/org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/renderers/StructureDefinitionRenderer.java +++ b/org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/renderers/StructureDefinitionRenderer.java @@ -453,7 +453,7 @@ public String snapshot(String defnFile, Set outputTracker, boolean toTab return ""; else { sdr.getContext().setStructureMode(mode); - return new XhtmlComposer(XhtmlComposer.HTML).compose(sdr.generateTable(defnFile, sd, false, destDir, false, sd.getId(), true, corePath, "", false, false, outputTracker, false, gen, toTabs ? ANCHOR_PREFIX_SNAP : ANCHOR_PREFIX_SNAP)); + return new XhtmlComposer(XhtmlComposer.HTML).compose(sdr.generateTable(defnFile, sd, false, destDir, false, sd.getId(), true, corePath, "", sd.getKind() == StructureDefinitionKind.LOGICAL, false, outputTracker, false, gen, toTabs ? ANCHOR_PREFIX_SNAP : ANCHOR_PREFIX_SNAP)); } } @@ -466,7 +466,7 @@ public String byKey(String defnFile, Set outputTracker, boolean toTabs, sdCopy.getSnapshot().setElement(getKeyElements()); sdr.getContext().setStructureMode(mode); - org.hl7.fhir.utilities.xhtml.XhtmlNode table = sdr.generateTable(defnFile, sdCopy, false, destDir, false, sdCopy.getId(), true, corePath, "", false, false, outputTracker, true, gen, toTabs ? ANCHOR_PREFIX_KEY : ANCHOR_PREFIX_SNAP); + org.hl7.fhir.utilities.xhtml.XhtmlNode table = sdr.generateTable(defnFile, sdCopy, false, destDir, false, sdCopy.getId(), true, corePath, "", sd.getKind() == StructureDefinitionKind.LOGICAL, false, outputTracker, true, gen, toTabs ? ANCHOR_PREFIX_KEY : ANCHOR_PREFIX_SNAP); return composer.compose(table); } @@ -481,7 +481,7 @@ public String byMustSupport(String defnFile, Set outputTracker, boolean sdr.getContext().setStructureMode(mode); sdCopy.getSnapshot().setElement(getMustSupportElements()); - org.hl7.fhir.utilities.xhtml.XhtmlNode table = sdr.generateTable(defnFile, sdCopy, false, destDir, false, sdCopy.getId(), true, corePath, "", false, false, outputTracker, true, gen, toTabs ? ANCHOR_PREFIX_MS : ANCHOR_PREFIX_SNAP); + org.hl7.fhir.utilities.xhtml.XhtmlNode table = sdr.generateTable(defnFile, sdCopy, false, destDir, false, sdCopy.getId(), true, corePath, "", sd.getKind() == StructureDefinitionKind.LOGICAL, false, outputTracker, true, gen, toTabs ? ANCHOR_PREFIX_MS : ANCHOR_PREFIX_SNAP); return composer.compose(table); } @@ -524,7 +524,7 @@ public String byKeyElements(String defnFile, Set outputTracker) throws I List keyElements = getKeyElements(); sdCopy.getSnapshot().setElement(keyElements); - org.hl7.fhir.utilities.xhtml.XhtmlNode table = sdr.generateTable(defnFile, sdCopy, false, destDir, false, sdCopy.getId(), true, corePath, "", false, false, outputTracker, true, gen, ANCHOR_PREFIX_KEY); + org.hl7.fhir.utilities.xhtml.XhtmlNode table = sdr.generateTable(defnFile, sdCopy, false, destDir, false, sdCopy.getId(), true, corePath, "", sd.getKind() == StructureDefinitionKind.LOGICAL, false, outputTracker, true, gen, ANCHOR_PREFIX_KEY); return composer.compose(table); } From dcab5558ab3e737c153c512e85ae696f303e9339 Mon Sep 17 00:00:00 2001 From: Grahame Grieve Date: Wed, 27 Sep 2023 11:41:52 +1000 Subject: [PATCH 4/6] exit code = 1 unless publication run is succesful --- .../java/org/hl7/fhir/igtools/web/PublicationProcess.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/web/PublicationProcess.java b/org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/web/PublicationProcess.java index f459c478..f0ccf93d 100644 --- a/org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/web/PublicationProcess.java +++ b/org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/web/PublicationProcess.java @@ -48,6 +48,8 @@ public class PublicationProcess { + private int exitCode = 1; + /* * checks that must be run prior to runing this: * @@ -131,6 +133,7 @@ public void publish(String source, String web, String date, String registrySourc e.printStackTrace(); } System.out.println("Full log in "+logger.getFilename()); + System.exit(exitCode); } private static String removePassword(String[] args, int i) { @@ -578,6 +581,7 @@ private void doPublish(File fSource, File fOutput, JsonObject qa, String destina System.out.println("No!"); System.out.print("Changes not applied. Finished"); } + exitCode = 0; } private List loadSubPackageList(String path) throws JsonException, IOException { From d9cac51eb053ac8d8a1668158a16715eeb081244 Mon Sep 17 00:00:00 2001 From: Grahame Grieve Date: Wed, 27 Sep 2023 11:42:44 +1000 Subject: [PATCH 5/6] Fix handling of CDA example fragments --- .../igtools/publisher/FetchedResource.java | 4 + .../hl7/fhir/igtools/publisher/Publisher.java | 130 +++++++++++------- 2 files changed, 85 insertions(+), 49 deletions(-) diff --git a/org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/publisher/FetchedResource.java b/org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/publisher/FetchedResource.java index 6a560f92..77e9d427 100644 --- a/org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/publisher/FetchedResource.java +++ b/org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/publisher/FetchedResource.java @@ -82,6 +82,10 @@ public FetchedResource setElement(Element element) { return this; } + public void setType(String type) { + this.type = type; + } + public String getId() { return id; } diff --git a/org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/publisher/Publisher.java b/org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/publisher/Publisher.java index 02beb407..e32b34f7 100644 --- a/org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/publisher/Publisher.java +++ b/org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/publisher/Publisher.java @@ -1075,7 +1075,7 @@ private String makeTemplateJekyllIndexPage() { } private String makeTemplateQAPage() { - String page = "Template QA Page

Template {{npm}} QA

No useful QA on templates - if you see this page, the template buitl ok.

"; + String page = "Template QA Page

Template {{npm}} QA

No useful QA on templates - if you see this page, the template built ok.

"; return page.replace("{{npm}}", templateInfo.asString("name")); } @@ -6029,34 +6029,46 @@ private void loadAsElementModel(FetchedFile file, FetchedResource r, Implementat if (e != null) { try { - if (!Utilities.noString(e.getIdBase())) { - checkResourceUnique(e.fhirType()+"/"+e.getIdBase()); - } + String id; boolean altered = false; - - String id = e.getChildValue("id"); - if (Utilities.noString(id)) { - if (e.hasChild("url")) { - String url = e.getChildValue("url"); - String prefix = Utilities.pathURL(igpkp.getCanonical(), e.fhirType())+"/"; - if (url.startsWith(prefix)) { - id = e.getChildValue("url").substring(prefix.length()); - e.setChildValue("id", id); - altered = true; - } - prefix = Utilities.pathURL(altCanonical, e.fhirType())+"/"; - if (url.startsWith(prefix)) { - id = e.getChildValue("url").substring(prefix.length()); - e.setChildValue("id", id); - altered = true; - } - if (Utilities.noString(id)) { - throw new Exception("Resource has no id in "+file.getPath()+" and canonical URL ("+url+") does not start with the IG canonical URL ("+prefix+")"); + boolean binary = false; + if (!context.getResourceNamesAsSet().contains(e.fhirType())) { + File f = new File(file.getPath()); + id = f.getName(); + id = id.substring(0, id.indexOf(".")); + checkResourceUnique("Binary/"+id); + r.setElement(e).setId(id).setType("Binary"); + igpkp.findConfiguration(file, r); + binary = true; + } else { + id = e.getChildValue("id"); + if (!Utilities.noString(e.getIdBase())) { + checkResourceUnique(e.fhirType()+"/"+e.getIdBase()); + } + + if (Utilities.noString(id)) { + if (e.hasChild("url")) { + String url = e.getChildValue("url"); + String prefix = Utilities.pathURL(igpkp.getCanonical(), e.fhirType())+"/"; + if (url.startsWith(prefix)) { + id = e.getChildValue("url").substring(prefix.length()); + e.setChildValue("id", id); + altered = true; + } + prefix = Utilities.pathURL(altCanonical, e.fhirType())+"/"; + if (url.startsWith(prefix)) { + id = e.getChildValue("url").substring(prefix.length()); + e.setChildValue("id", id); + altered = true; + } + if (Utilities.noString(id)) { + throw new Exception("Resource has no id in "+file.getPath()+" and canonical URL ("+url+") does not start with the IG canonical URL ("+prefix+")"); + } } } + r.setElement(e).setId(id); + igpkp.findConfiguration(file, r); } - r.setElement(e).setId(id); - igpkp.findConfiguration(file, r); if (!suppressLoading) { if (srcForLoad == null) srcForLoad = findIGReference(r.fhirType(), r.getId()); @@ -6116,8 +6128,9 @@ else if (file.getContentType().contains("xml")) if (new AdjunctFileLoader(binaryPaths, cql).replaceAttachments1(file, r, metadataResourceNames())) { altered = true; } - if ((altered && r.getResource() != null) || (ver.equals(Constants.VERSION) && r.getResource() == null)) + if (!binary && ((altered && r.getResource() != null) || (ver.equals(Constants.VERSION) && r.getResource() == null && context.getResourceNamesAsSet().contains(r.fhirType())))) { r.setResource(new ObjectConverter(context).convert(r.getElement())); + } if ((altered && r.getResource() == null)) { if (file.getContentType().contains("json")) { saveToJson(file, e); @@ -6696,7 +6709,6 @@ private void generateSnapshot(FetchedFile f, FetchedResource r, StructureDefinit } if (changed || (!r.getElement().hasChild("snapshot") && sd.hasSnapshot())) { - // for big snapshots, this can take considerable time, and there's really no reason to do it r.setElement(convertToElement(r, sd)); } r.getElement().setUserData("profileutils.snapshot.errors", messages); @@ -6925,28 +6937,32 @@ private void validate() throws Exception { try { logDebugMessage(LogCategory.PROGRESS, " .. validate "+f.getName()); logDebugMessage(LogCategory.PROGRESS, " .. "+f.getName()); - for (FetchedResource r : f.getResources()) { - if (!r.isValidated()) { - logDebugMessage(LogCategory.PROGRESS, " validating "+r.getTitle()); - validate(f, r); - } - } - FetchedResource r = f.getResources().get(0); - if (f.getLogical() != null && f.getResources().size() == 1 && r.fhirType().equals("Binary")) { - Binary bin = (Binary) r.getResource(); - StructureDefinition profile = context.fetchResource(StructureDefinition.class, f.getLogical()); - List errs = new ArrayList(); - if (profile == null) { - errs.add(new ValidationMessage(Source.InstanceValidator, IssueType.NOTFOUND, "file", context.formatMessage(I18nConstants.Bundle_BUNDLE_Entry_NO_LOGICAL_EXPL, r.getId(), f.getLogical()), IssueSeverity.ERROR)); - } else { - FhirFormat fmt = FhirFormat.readFromMimeType(bin.getContentType()); - Session tts = tt.start("validation"); - List profiles = new ArrayList<>(); - profiles.add(profile); - validate(f, r, bin, errs, fmt, profiles); - tts.end(); + FetchedResource r0 = f.getResources().get(0); + if (f.getLogical() != null && f.getResources().size() == 1 && !r0.fhirType().equals("Binary")) { + throw new Error("Not done yet"); + } else { + for (FetchedResource r : f.getResources()) { + if (!r.isValidated()) { + logDebugMessage(LogCategory.PROGRESS, " validating "+r.getTitle()); + validate(f, r); + } + } + if (f.getLogical() != null && f.getResources().size() == 1 && r0.fhirType().equals("Binary")) { + Binary bin = (Binary) r0.getResource(); + StructureDefinition profile = context.fetchResource(StructureDefinition.class, f.getLogical()); + List errs = new ArrayList(); + if (profile == null) { + errs.add(new ValidationMessage(Source.InstanceValidator, IssueType.NOTFOUND, "file", context.formatMessage(I18nConstants.Bundle_BUNDLE_Entry_NO_LOGICAL_EXPL, r0.getId(), f.getLogical()), IssueSeverity.ERROR)); + } else { + FhirFormat fmt = FhirFormat.readFromMimeType(bin.getContentType()); + Session tts = tt.start("validation"); + List profiles = new ArrayList<>(); + profiles.add(profile); + validate(f, r0, bin, errs, fmt, profiles); + tts.end(); + } + processValidationOutcomes(f, r0, errs); } - processValidationOutcomes(f, r, errs); } } finally { f.finish("validate"); @@ -6990,6 +7006,17 @@ private void validate(FetchedFile f, FetchedResource r, List } } + private void validate(FetchedFile f, FetchedResource r, List errs, Binary bin, StructureDefinition sd) { + long ts = System.currentTimeMillis(); + List profiles = new ArrayList(); + profiles.add(sd); + validator.validate(r.getElement(), errs, new ByteArrayInputStream(bin.getContent()), FhirFormat.readFromMimeType(bin.getContentType()), profiles); + long tf = System.currentTimeMillis(); + if (tf-ts > validationLogTime && validationLogTime > 0) { + reportLongValidation(f, r, tf-ts); + } + } + private void validate(FetchedFile f, FetchedResource r, List errs, Resource ber) { long ts = System.currentTimeMillis(); validator.validate(r.getElement(), errs, ber, ber.getUserString("profile")); @@ -7218,6 +7245,10 @@ private void validate(FetchedFile file, FetchedResource r) throws Exception { } else if (res.hasUserData("profile")) { validate(file, r, errs, res); } + } else if (r.getResource() != null && r.getResource() instanceof Binary && file.getLogical() != null && context.hasResource(StructureDefinition.class, file.getLogical())) { + StructureDefinition sd = context.fetchResource(StructureDefinition.class, file.getLogical()); + Binary bin = (Binary) r.getResource(); + validate(file, r, errs, bin, sd); } else if (r.getResource() != null && r.getResource() instanceof Binary && r.getExampleUri() != null) { Binary bin = (Binary) r.getResource(); validate(file, r, errs, bin); @@ -7747,7 +7778,8 @@ private Element convertToElement(FetchedResource r, Resource res) throws Excepti org.hl7.fhir.r5.formats.JsonParser jp = new org.hl7.fhir.r5.formats.JsonParser(); jp.compose(bs, res); } - ByteArrayInputStream bi = new ByteArrayInputStream(bs.toByteArray()); + byte[] cnt = bs.toByteArray(); + ByteArrayInputStream bi = new ByteArrayInputStream(cnt); Element e = new org.hl7.fhir.r5.elementmodel.JsonParser(context).parseSingle(bi, null); if (xhtml != null) { Element div = e.getNamedChild("text").getNamedChild("div"); From e8da3b65ca6b73ec923846989e132b21f55852e6 Mon Sep 17 00:00:00 2001 From: Grahame Grieve Date: Wed, 27 Sep 2023 22:33:05 +1000 Subject: [PATCH 6/6] release notes --- RELEASE_NOTES.md | 17 +++++++++++++++++ pom.xml | 2 +- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index e69de29b..516352d2 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -0,0 +1,17 @@ +* Loader: Fix up parsing of logical models +* Loader: Fix handling of CDA example fragments +* Loader: Fix bug parsing extension with no value in JSON for the validator +* Validator: Fix for issue parsing SHC and not recording line/col correctly +* Validator: Fix issue validating CDA FHIR Path constraints +* Validator: Better error handling validating codes in concept maps +* Validator: Validate special resource rules on contained resources and Bundle entries +* Validator: Improved error messages of observation bp +* Validator: Fix up WG internal model for changes to workgroups +* Validator: fix misleading error message inferring system when filters in play +* Validator: Fix type handling for logical models (CDA fixes) +* Version convertor: Fix version conversion issue between r4 and r5 charge definition issue +* Renderer: Fix rendering extension and missed profile on Reference() +* Renderer: fix wrong generation of logical models (for CDA) +* Renderer: Don't create wrong phinvads references +* Link Checker: Don't fail in link checking for illegal file references +* Publication Process: exit code = 1 unless publication run is succesful diff --git a/pom.xml b/pom.xml index c5cbfa4c..0d16c9c7 100644 --- a/pom.xml +++ b/pom.xml @@ -24,7 +24,7 @@ 1.4.9-SNAPSHOT - 6.1.8 + 6.1.9-SNAPSHOT 3.0.0-M5 5.2.1 4.11.0