Skip to content

Commit

Permalink
Merge pull request #944 from HL7/2024-08-gg-jekyll-performance
Browse files Browse the repository at this point in the history
2024 08 gg jekyll performance
  • Loading branch information
grahamegrieve authored Aug 27, 2024
2 parents 29167c0 + 2377dc1 commit bc9db47
Show file tree
Hide file tree
Showing 7 changed files with 97 additions and 22 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -11170,8 +11170,17 @@ private byte[] saveNativeResourceOutputs(FetchedFile f, FetchedResource r) throw
ByteArrayOutputStream bsj = new ByteArrayOutputStream();
org.hl7.fhir.r5.elementmodel.JsonParser jp = new org.hl7.fhir.r5.elementmodel.JsonParser(context);
Element element = r.getElement();
Element eNN = element;
jp.compose(element, bsj, OutputStyle.NORMAL, igpkp.getCanonical());
npm.addFile(isExample(f,r ) ? Category.EXAMPLE : Category.RESOURCE, element.fhirType()+"-"+r.getId()+".json", bsj.toByteArray());

if (module.isNoNarrative()) {
// we don't use the narrative in these resources in _includes, so we strip it - it slows Jekyll down greatly
eNN = (Element) element.copy();
eNN.removeChild("text");
bsj = new ByteArrayOutputStream();
jp.compose(eNN, bsj, OutputStyle.PRETTY, igpkp.getCanonical());
}
String path = Utilities.path(tempDir, "_includes", r.fhirType()+"-"+r.getId()+".json");
TextFile.bytesToFile(bsj.toByteArray(), path);
String pathEsc = Utilities.path(tempDir, "_includes", r.fhirType()+"-"+r.getId()+".escaped.json");
Expand Down Expand Up @@ -11358,10 +11367,13 @@ private void saveDirectResourceOutputs(FetchedFile f, FetchedResource r, Resourc
xp.compose(r.getElement(), bs, OutputStyle.NORMAL, null);
int size = bs.size();

Element e = r.getElement();
Element e = r.getElement();
if (SpecialTypeHandler.handlesType(r.fhirType(), context.getVersion())) {
e = new ObjectConverter(context).convert(r.getResource());
}
} else if (module.isNoNarrative() && e.hasChild("text")) {
e = (Element) e.copy();
e.removeChild("text");
}

if (igpkp.wantGen(r, "xml-html")) {
long start = System.currentTimeMillis();
Expand Down Expand Up @@ -11402,14 +11414,16 @@ private void saveDirectResourceOutputs(FetchedFile f, FetchedResource r, Resourc
fragment(r.fhirType()+"-"+r.getId()+"-ttl-html", rdf.asHtml(size < PRISM_SIZE_LIMIT), f.getOutputNames(), r, vars, "ttl", start, "ttl-html", "Resource");
}

e = r.getElement();
XhtmlComposer xc = new XhtmlComposer(XhtmlComposer.XML, module.isNoNarrative());
if (igpkp.wantGen(r, "html")) {
long start = System.currentTimeMillis();
XhtmlNode xhtml = getXhtml(f, r);
if (xhtml == null && HistoryGenerator.allEntriesAreHistoryProvenance(r.getElement())) {
RenderingContext ctxt = rc.copy(false).setParser(getTypeLoader(f, r));
List<ProvenanceDetails> entries = loadProvenanceForBundle(igpkp.getLinkFor(r, true), r.getElement(), f);
xhtml = new HistoryGenerator(ctxt).generateForBundle(entries);
fragment(r.fhirType()+"-"+r.getId()+"-html", new XhtmlComposer(XhtmlComposer.XML).compose(xhtml), f.getOutputNames(), r, vars, null, start, "html", "Resource");
fragment(r.fhirType()+"-"+r.getId()+"-html", xc.compose(xhtml), f.getOutputNames(), r, vars, null, start, "html", "Resource");
} else if (r.fhirType().equals("Binary")) {
String pfx = "";
if (r.getExampleUri() != null) {
Expand Down Expand Up @@ -11448,15 +11462,15 @@ private void saveDirectResourceOutputs(FetchedFile f, FetchedResource r, Resourc
}
}
}
String html = xhtml == null ? "" : new XhtmlComposer(XhtmlComposer.XML).compose(xhtml);
String html = xhtml == null ? "" : xc.compose(xhtml);
fragment(r.fhirType()+"-"+r.getId()+"-html", html, f.getOutputNames(), r, vars, null, start, "html", "Resource");
}
}

if (igpkp.wantGen(r, "history")) {
long start = System.currentTimeMillis();
XhtmlNode xhtml = new HistoryGenerator(rc).generate(r);
String html = xhtml == null ? "" : new XhtmlComposer(XhtmlComposer.XML).compose(xhtml);
String html = xhtml == null ? "" : xc.compose(xhtml);
fragment(r.fhirType()+"-"+r.getId()+"-history", html, f.getOutputNames(), r, vars, null, start, "history", "Resource");
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -230,7 +230,7 @@ private void genExtensionPage(String path, StructureDefinition sd) throws IOExce
body.hr();

body.pre().tx(new JsonParser().setOutputStyle(OutputStyle.PRETTY).composeString(sd));
TextFile.stringToFile(new XhtmlComposer(false, false).compose(wrapPage(body, sd.getName())), fn);
TextFile.stringToFile(new XhtmlComposer(false, true).compose(wrapPage(body, sd.getName())), fn);
}

private void renderVersionRange(XhtmlNode x, Extension ext) {
Expand Down Expand Up @@ -280,8 +280,8 @@ private void genChainsHtml(String path, String filename, boolean validOnly, bool
}
}
}
TextFile.stringToFile(new XhtmlComposer(false, false).compose(body), Utilities.path(path, "input", "includes", filename+".xhtml"));
TextFile.stringToFile(new XhtmlComposer(false, false).compose(wrapPage(body, "FHIR Cross Version Extensions")), Utilities.path(path, "temp", "xver-qa", filename+".html"));
TextFile.stringToFile(new XhtmlComposer(false, true).compose(body), Utilities.path(path, "input", "includes", filename+".xhtml"));
TextFile.stringToFile(new XhtmlComposer(false, true).compose(wrapPage(body, "FHIR Cross Version Extensions")), Utilities.path(path, "temp", "xver-qa", filename+".html"));
}


Expand Down Expand Up @@ -450,15 +450,15 @@ private void genVersionType(String path, StructureDefinition sd, ImplementationG
body.add(ConceptMapRenderer.renderMultipleMaps(name, maps, engine, new MultiRowRenderingContext(MultiConceptMapType.CODED, RenderMultiRowSortPolicy.UNSORTED, links)));
}

TextFile.stringToFile(new XhtmlComposer(false, false).compose(body), Utilities.path(path, "input", "includes", "cross-version-"+sd.getName()+".xhtml"));
TextFile.stringToFile(new XhtmlComposer(false, true).compose(body), Utilities.path(path, "input", "includes", "cross-version-"+sd.getName()+".xhtml"));
TextFile.stringToFile("{% include cross-version-"+sd.getName()+".xhtml %}\r\n", Utilities.path(path, "input", "pagecontent", "cross-version-"+sd.getName()+".md"));
ImplementationGuideDefinitionPageComponent p = page.addPage();
p.setSource(new UrlType("cross-version-"+sd.getName()+".md"));
p.setName("cross-version-"+sd.getName()+".html");
p.setTitle("Cross-Version summary for "+sd.getName());
p.setGeneration(GuidePageGeneration.MARKDOWN);
ToolingExtensions.setStandardsStatus(p, StandardsStatus.INFORMATIVE, null);
TextFile.stringToFile(new XhtmlComposer(false, false).compose(wrapPage(body, sd.getName())), Utilities.path(path, "temp", "xver-qa", "cross-version-"+sd.getName()+".html"));
TextFile.stringToFile(new XhtmlComposer(false, true).compose(wrapPage(body, sd.getName())), Utilities.path(path, "temp", "xver-qa", "cross-version-"+sd.getName()+".html"));
}


Expand Down Expand Up @@ -621,8 +621,8 @@ private void genSummaryPages(String path) throws IOException {
maps.add(engine.cm("resources-4bto5"));
XhtmlNode page = ConceptMapRenderer.renderMultipleMaps("R2 Resources", maps, engine, new MultiRowRenderingContext(MultiConceptMapType.SUMMARY, RenderMultiRowSortPolicy.FIRST_COL, "resources"));

TextFile.stringToFile(new XhtmlComposer(false, false).compose(page), Utilities.path(path, "input", "includes", "cross-version-resources.xhtml"));
TextFile.stringToFile(new XhtmlComposer(false, false).compose(wrapPage(page, "Resource Map")), Utilities.path(path, "temp", "xver-qa", "cross-version-resources.html"));
TextFile.stringToFile(new XhtmlComposer(false, true).compose(page), Utilities.path(path, "input", "includes", "cross-version-resources.xhtml"));
TextFile.stringToFile(new XhtmlComposer(false, true).compose(wrapPage(page, "Resource Map")), Utilities.path(path, "temp", "xver-qa", "cross-version-resources.html"));

maps = new ArrayList<>();
maps.add(engine.cm("types-2to3"));
Expand All @@ -631,8 +631,8 @@ private void genSummaryPages(String path) throws IOException {
maps.add(engine.cm("types-4bto5"));
page = ConceptMapRenderer.renderMultipleMaps("R2 DataTypes", maps, engine, new MultiRowRenderingContext(MultiConceptMapType.SUMMARY, RenderMultiRowSortPolicy.FIRST_COL, "types"));

TextFile.stringToFile(new XhtmlComposer(false, false).compose(page), Utilities.path(path, "input", "includes", "cross-version-types.xhtml"));
TextFile.stringToFile(new XhtmlComposer(false, false).compose(wrapPage(page, "Type Map")), Utilities.path(path, "temp", "xver-qa", "cross-version-types.html"));
TextFile.stringToFile(new XhtmlComposer(false, true).compose(page), Utilities.path(path, "input", "includes", "cross-version-types.xhtml"));
TextFile.stringToFile(new XhtmlComposer(false, true).compose(wrapPage(page, "Type Map")), Utilities.path(path, "temp", "xver-qa", "cross-version-types.html"));

}

Expand Down Expand Up @@ -924,6 +924,11 @@ public String getCanonicalForDefaultContext() {
// TODO Auto-generated method stub
return null;
}

@Override
public boolean isNoNarrative() {
return true;
}


}
Original file line number Diff line number Diff line change
Expand Up @@ -31,4 +31,6 @@ public interface IPublisherModule {

// approve the generation of a fragment - mainly for performance reasons
boolean approveFragment(boolean value, String code);

boolean isNoNarrative();
}
Original file line number Diff line number Diff line change
Expand Up @@ -49,4 +49,9 @@ public boolean approveFragment(boolean value, String code) {
return value;
}

@Override
public boolean isNoNarrative() {
return false;
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -182,7 +182,10 @@ public static void main(String[] args) throws Exception {
System.out.println("Done");
}

private String processingPath;

public boolean process(String path) throws FHIRException, IOException {
processingPath = path;
return execute(path);
}

Expand Down Expand Up @@ -1373,9 +1376,9 @@ private Map<String, ElementWithType> processRuleTarget(StructureMap map, String
VSPair vsl = isCoded(source.toSED());
VSPair vsr = isCoded(target.toSED());
if (vsl == null) {
qaMsg("Rule "+r.getName()+" in group "+grp.getName()+" has a translate operation, but the source element "+source.toSED().toString()+" is not coded in "+map.getUrl(), true);
qaMsg("Rule "+r.getName()+" in group "+grp.getName()+" has a translate operation, but the source element "+source.toSED().toString()+" is not coded in "+map.getUrl(), false);
} else if (vsr == null) {
qaMsg("Rule "+r.getName()+" in group "+grp.getName()+" has a translate operation, but the target element "+target.toSED().toString()+" is not coded in "+map.getUrl(), true);
qaMsg("Rule "+r.getName()+" in group "+grp.getName()+" has a translate operation, but the target element "+target.toSED().toString()+" is not coded in "+map.getUrl(), false);
} else {
if (!(isResourceType(vsl.getCodes()) || isResourceType(vsr.getCodes()))) {
ConceptMap cm = conceptMapsByUrl.get(url);
Expand Down Expand Up @@ -1642,8 +1645,8 @@ private void checkMapsReciprocal(ConceptMap left, ConceptMap right, String folde
List<String> issues = new ArrayList<String>();
if (ConceptMapUtilities.checkReciprocal(left, right, issues, save)) {
// wipes formatting in files
new org.hl7.fhir.r5.formats.JsonParser().setOutputStyle(OutputStyle.PRETTY).compose(new FileOutputStream("/Users/grahamegrieve/work/fhir-cross-version/input/"+folder+"/ConceptMap-"+left.getId()+".json"), left);
new org.hl7.fhir.r5.formats.JsonParser().setOutputStyle(OutputStyle.PRETTY).compose(new FileOutputStream("/Users/grahamegrieve/work/fhir-cross-version/input/"+folder+"/ConceptMap-"+right.getId()+".json"), right);
new org.hl7.fhir.r5.formats.JsonParser().setOutputStyle(OutputStyle.PRETTY).compose(new FileOutputStream(Utilities.path(processingPath, "input", folder, "ConceptMap-"+left.getId()+".json")), left);
new org.hl7.fhir.r5.formats.JsonParser().setOutputStyle(OutputStyle.PRETTY).compose(new FileOutputStream(Utilities.path(processingPath, "input", folder, "ConceptMap-"+right.getId()+".json")), right);
}
if (!issues.isEmpty()) {
qaMsg("Found issues checking reciprocity of "+left.getId()+" and "+right.getId(), true);
Expand Down Expand Up @@ -2002,8 +2005,8 @@ private void scanChainElements(SourcedElementDefinition origin, List<ElementDefi
qaMsg("Error between "+cmF.getId()+" and "+cmR.getId()+" maps: "+s, true);
}
if (altered) {
new org.hl7.fhir.r5.formats.JsonParser().setOutputStyle(OutputStyle.PRETTY).compose(new FileOutputStream("/Users/grahamegrieve/work/fhir-cross-version/input/codes/ConceptMap-"+cmR.getId()+".json"), cmR);
new org.hl7.fhir.r5.formats.JsonParser().setOutputStyle(OutputStyle.PRETTY).compose(new FileOutputStream("/Users/grahamegrieve/work/fhir-cross-version/input/codes/ConceptMap-"+cmF.getId()+".json"), cmF);
new org.hl7.fhir.r5.formats.JsonParser().setOutputStyle(OutputStyle.PRETTY).compose(new FileOutputStream(Utilities.path(processingPath, "input", "codes", "ConceptMap-"+cmR.getId()+".json")), cmR);
new org.hl7.fhir.r5.formats.JsonParser().setOutputStyle(OutputStyle.PRETTY).compose(new FileOutputStream(Utilities.path(processingPath, "input", "codes", "ConceptMap-"+cmF.getId()+".json")), cmF);
}
}
link.setNextCM(cmF);
Expand Down Expand Up @@ -2210,7 +2213,7 @@ private ConceptMap makeCM(String id, String rawId, SourcedElementDefinition se,
Collections.sort(g.getElement(), new ConceptMapUtilities.ConceptMapElementSorter());
}
cm.getGroup().removeIf(g -> g.getElement().isEmpty());
new org.hl7.fhir.r5.formats.JsonParser().setOutputStyle(OutputStyle.PRETTY).compose(new FileOutputStream("/Users/grahamegrieve/work/fhir-cross-version/input/codes/ConceptMap-"+cm.getId()+".json"), cm);
new org.hl7.fhir.r5.formats.JsonParser().setOutputStyle(OutputStyle.PRETTY).compose(new FileOutputStream(Utilities.path(processingPath, "input", "codes", "ConceptMap-"+cm.getId()+".json")), cm);
return cm;
}

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
package org.hl7.fhir.igtools.renderers.utils;

import java.io.File;
import java.io.IOException;
import java.nio.file.Files;

import org.hl7.fhir.utilities.TextFile;

public class LineLengthChecker {

public static void main(String[] args) throws IOException {
new LineLengthChecker().check(args[0], 8192);
}

private int total;

private void check(String path, int limit) throws IOException {
System.out.println("Checking line lengths in "+path);
check(new File(path), limit);
System.out.println("Finished - checked "+total+" files");
}

private void check(File file, int limit) throws IOException {
for (File f : file.listFiles()) {
if (f.isDirectory()) {
check(f, limit);
} else if (f.length() > 0) {
total++;
int length = 0;
String source = TextFile.fileToString(f);
for (String line : source.split("\\R")) {
int l = line.length();
if (l > length) {
length = l;
}
}
if (length > limit) {

System.out.println("File "+f.getAbsolutePath()+": longest line = "+length);
}
}
}

}

}
2 changes: 1 addition & 1 deletion pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
<version>1.6.21-SNAPSHOT</version> <!-- See the note above -->

<properties>
<core_version>6.3.20</core_version>
<core_version>6.3.21-SNAPSHOT</core_version>
<maven_surefire_version>3.0.0-M5</maven_surefire_version>
<apache_poi_version>5.2.1</apache_poi_version>
<okhttp.version>4.11.0</okhttp.version>
Expand Down

0 comments on commit bc9db47

Please sign in to comment.