diff --git a/.gitignore b/.gitignore index 70cfadb..00c0979 100644 --- a/.gitignore +++ b/.gitignore @@ -4,10 +4,10 @@ *.mexmaci64 .classpath .cproject +.factorypath .project .pydevproject .settings -dist -doc/pdf +outside +src/main/resources/ziggy-build.properties src/test/cpp/ExternalProcess/testprog.o -src/test/cpp/testmi/inputs-0.bin diff --git a/CITATION.cff b/CITATION.cff index f49b4ce..7699906 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -15,6 +15,7 @@ authors: affiliation: "SETI Institute/NASA Ames Research Center" email: Bill.Wohler@nasa.gov orcid: "https://orcid.org/0000-0002-5402-9613" +doi: 10.5281/zenodo.7859503 license-url: "https://github.com/nasa/ziggy/blob/main/LICENSE.pdf" url: "https://github.com/nasa/ziggy" repository-code: "https://github.com/nasa/ziggy" diff --git a/build.gradle b/build.gradle index 9260087..8ac6f7c 100644 --- a/build.gradle +++ b/build.gradle @@ -7,101 +7,143 @@ // To view a dependency graph using the taskinfo plugin, run "./gradlew tiTree build" plugins { - id 'com.github.spotbugs' version '5.0.+' + id 'com.github.spotbugs' version '5.1.+' id 'eclipse' id 'jacoco' id 'java' + id 'maven-publish' id 'org.barfuin.gradle.taskinfo' version '2.1.+' + id 'signing' } -defaultTasks 'assemble', 'test' +defaultTasks 'assemble', 'publish', 'test' ext { - // See comment for ziggyDependencies in gradle.properties. - ziggyDependencies = "$rootDir/$ziggyDependencies" + // Location of third-party sources. + outsideDir = "$rootDir/outside" + + // The dependency group used for libraries from outside is called "outside". + // This variable contains the directory that contains the libraries from + // outside. Without the dependency group and the outside subdirectory, + // consumers of the published module will get "null" errors. + outsideGroupDir = "$buildDir/libs/outside" + + // Name of the outside group, used in publishing artifacts. + outsideGroup = "outside" } repositories { mavenCentral() flatDir { - dirs "$ziggyDependencies/lib" + dirs "$outsideDir/lib" } } dependencies { // Needed to compile ziggy. + implementation files("$rootDir/buildSrc/build/libs/buildSrc-${version}.jar") + implementation 'com.github.librepdf:openpdf:1.3.+' - implementation 'com.github.testdriven.guice:commons-configuration:1.+' + implementation 'com.github.spotbugs:spotbugs-annotations:4.7.+' implementation 'com.google.guava:guava:23.+' implementation 'com.jgoodies:jgoodies-forms:1.9.+' implementation 'com.jgoodies:jgoodies-looks:2.7.+' - implementation 'com.sun.xml.bind:jaxb-impl:3.0+' implementation 'commons-cli:commons-cli:1.5.+' implementation 'commons-codec:commons-codec:1.+' implementation 'commons-io:commons-io:2.11.+' + implementation 'jakarta.xml.bind:jakarta.xml.bind-api:3.0+' implementation 'org.apache.commons:commons-collections4:4.+' implementation 'org.apache.commons:commons-compress:1.+' + implementation 'org.apache.commons:commons-configuration2:2.9.+' implementation 'org.apache.commons:commons-csv:1.9.+' implementation 'org.apache.commons:commons-exec:1.+' implementation 'org.apache.commons:commons-lang3:3.12.+' implementation 'org.apache.commons:commons-math3:3.6.+' implementation 'org.apache.commons:commons-text:1.+' - implementation 'org.apache.logging.log4j:log4j-api:2.17.+' - implementation 'org.apache.logging.log4j:log4j-core:2.17.+' - implementation 'org.apache.logging.log4j:log4j-slf4j-impl:2.17.+' - implementation 'org.hibernate.javax.persistence:hibernate-jpa-2.0-api:1.0.+' - implementation 'org.hibernate:hibernate-core:4.2.+' + implementation 'org.apache.logging.log4j:log4j-core:2.20.+' + implementation 'org.apache.logging.log4j:log4j-slf4j2-impl:2.20.+' + implementation 'org.hibernate.orm:hibernate-ant:6.2.+' + implementation 'org.hibernate:hibernate-core:6.2.+' implementation 'org.javassist:javassist:3.29.2-GA' implementation 'org.jfree:jfreechart:1.0.+' + implementation 'org.jsoup:jsoup:1.16.+' implementation 'org.netbeans.api:org-netbeans-swing-outline:+' + implementation 'org.slf4j:slf4j-api:2.0.+' implementation 'org.tros:l2fprod-properties-editor:1.3.+' - implementation 'tanukisoft:wrapper:3.2.+' - runtimeOnly 'jakarta.xml.bind:jakarta.xml.bind-api:3.0+' - // Libraries built in buildSrc. - implementation ':jarhdf5:1.12.+' - implementation ':wrapper:' + // Configuration2 declares the following as optional [1]. It's not, so it's added here. + // Occasionally, comment out this line--if the tests pass, delete it. + // 1. https://github.com/apache/commons-configuration/blob/master/pom.xml + implementation 'commons-beanutils:commons-beanutils:1.9.+' - // Needed to compile unit tests. - implementation 'com.github.spotbugs:spotbugs-annotations:4.7.+' - implementation 'junit:junit:4.13.+' - implementation 'org.hamcrest:hamcrest:2.+' - implementation 'org.mockito:mockito-core:3.12.+' + // Libraries built outside. + implementation 'outside:jarhdf5:1.12.+' + implementation 'outside:wrapper:3.5.+' + + // Needed to run unit tests and at runtime. + implementation 'org.hsqldb:hsqldb:2.7.+' - // Needed to run unit tests. - // hsqldb version 2.3.4 causes PipelineInstanceTaskCrudTest to - // fail with HsqlException: data exception: datetime field - // overflow. This can be fixed by replacing Long.MAX_VALUE with a - // smaller date. Other problems ensue with version 2.6. - implementation 'org.hsqldb:hsqldb:2.3.2' + // Needed to compile unit tests. + testImplementation 'junit:junit:4.13.+' + testImplementation 'org.hamcrest:hamcrest:2.+' + testImplementation 'org.mockito:mockito-core:3.12.+' // Needed at runtime. - implementation 'org.postgresql:postgresql:9.4-1201-jdbc4' + runtimeOnly 'jakarta.xml.bind:jakarta.xml.bind-api:3.0+' + runtimeOnly 'org.hibernate.orm:hibernate-hikaricp:6.2.+' + runtimeOnly 'org.postgresql:postgresql:42.6.+' + + // Astonishingly, for some reason configuration2 doesn't work when + // called by MATLAB, but configuration does. Ziggy doesn't use MATLAB + // in its build but provides MATLAB utilities that pipelines can call, + // so I think that makes this a runtime dependency for Ziggy. + runtimeOnly 'commons-configuration:commons-configuration:1.10' // The following plugin reveals some bugs in our code, but until // the "The following classes needed for analysis were missing" bug // is fixed, it is commented out for daily use. // spotbugsPlugins 'com.h3xstream.findsecbugs:findsecbugs-plugin:1.12.+' + + annotationProcessor 'org.hibernate.orm:hibernate-jpamodelgen:6.2.+' } tasks.withType(JavaCompile) { options.deprecation = true } +// Add additional artifacts for publishing (see script-utils/publish.gradle). +java { + // This runs the javadoc task, which currently fails due to + // Javadoc errors. Enable after errors fixed, or ignored. + // withJavadocJar() + + withSourcesJar() +} + test { - systemProperty "log4j2.configurationFile","$projectDir/test/data/logging/log4j2.xml" - systemProperty "java.library.path", "$ziggyDependencies/lib" - + systemProperty "java.library.path", "$outsideDir/lib" + testLogging { events "failed", "skipped" } useJUnit { + // If a code coverage report that incudes the integration tests is desired, then comment + // out the IntegrationTestCategory line and uncomment the RunByNameTestCategory line. When + // the JaCoCo issue described below is resolved, then delete this comment. + // excludeCategories 'gov.nasa.ziggy.RunByNameTestCategory' excludeCategories 'gov.nasa.ziggy.IntegrationTestCategory' } } // Execute tests marked with @Category(IntegrationTestCategory.class). task integrationTest(type: Test) { + systemProperty "log4j2.configurationFile", "$rootDir/etc/log4j2.xml" + systemProperty "ziggy.logfile", "$buildDir/build.log" + systemProperty "java.library.path", "$outsideDir/lib" + + testLogging { + events "failed", "skipped" + } useJUnit { includeCategories 'gov.nasa.ziggy.IntegrationTestCategory' excludeCategories 'gov.nasa.ziggy.RunByNameTestCategory' @@ -113,6 +155,10 @@ task integrationTest(type: Test) { // since they don't play well with others. For example: // gradle runByNameTests --tests *RmiInterProcessCommunicationTest task runByNameTest(type: Test) { + systemProperty "log4j2.configurationFile", "$rootDir/etc/log4j2.xml" + systemProperty "ziggy.logfile", "$buildDir/build.log" + systemProperty "java.library.path", "$outsideDir/lib" + useJUnit { includeCategories 'gov.nasa.ziggy.RunByNameTestCategory' } @@ -122,12 +168,19 @@ task runByNameTest(type: Test) { task testAll testAll.dependsOn test, integrationTest +check.dependsOn testAll // To view code coverage, run the jacocoTestReport task and view the output in: // build/reports/jacoco/test/html/index.html. check.dependsOn jacocoTestReport jacocoTestReport { + // TODO Switch dependency to testAll to ensure that the integration tests are counted + // Depending on anything other than test results in this task + // getting SKIPPED, perhaps due to missing execution data. If code + // coverage of integration tests is desired, see comment in the test + // configuration. + // Switching to a recent version of JUnit 5+ was shown to work in one post dependsOn test // Since GUI code doesn't have unit tests, exclude it from the @@ -153,31 +206,28 @@ spotbugs { reportLevel = 'high' } -// The following should work to provide a single closure instead of -// separate closures for the spotbugsMain and spotbugsTest tasks, but -// it generates a "Could not get unknown property 'com'" error. -// tasks.withType(com.github.spotbugs.SpotBugsTask) { -// The following more verbose code accomplishes the same thing. -// Note that the following more verbose code also triggers a Gradle -// deprecation warning, specifically a warning that ConfigureUtil -// is going away. The specific line that generates the message appears -// to be the "html" line. -tasks.matching {task -> task.name.startsWith('spotbugs')}.forEach { - it.reports { - html { - stylesheet = 'fancy-hist.xsl' - } +tasks.withType(com.github.spotbugs.snom.SpotBugsTask) { + reports { + html.stylesheet = "fancy-hist.xsl" } } +// Outside build files should copy their jars to $outsideGroupDir and +// then make copyOutsideLibs depend on the task that performs that copy. +// This is used to publish the outside jars. +task copyOutsideLibs +compileJava.dependsOn copyOutsideLibs + // Apply Ziggy Gradle script plugins. apply from: "script-plugins/copy.gradle" -apply from: "script-plugins/copy-hdf5.gradle" apply from: "script-plugins/database-schemas.gradle" apply from: "script-plugins/eclipse.gradle" -apply from: "script-plugins/gcc.gradle" +apply from: "script-plugins/hdf5.gradle" apply from: "script-plugins/misc.gradle" apply from: "script-plugins/test.gradle" apply from: "script-plugins/wrapper.gradle" apply from: "script-plugins/xml-schemas.gradle" apply from: "script-plugins/ziggy-libraries.gradle" + +// Depends on versions set in hdf5.gradle and wrapper.gradle and copyBuildSrc from copy.gradle. +apply from: "script-plugins/publish.gradle" diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index 073addb..f2c1666 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -2,8 +2,11 @@ * This is the buildSrc/build.gradle file. This should contain a minimal * amount of dependencies as this is used by the build system itself. */ -apply plugin: 'eclipse' -apply plugin: 'java' +plugins { + id 'eclipse' + id 'java' + id 'maven-publish' +} defaultTasks 'assemble' @@ -15,23 +18,29 @@ dependencies { // Needed to compile buildSrc. implementation 'com.google.guava:guava:23.+' implementation 'commons-io:commons-io:2.11.+' + implementation 'jakarta.xml.bind:jakarta.xml.bind-api:3.0.+' implementation 'org.apache.commons:commons-exec:1.+' - implementation 'org.freemarker:freemarker:2.3.+' - implementation 'com.sun.xml.bind:jaxb-impl:3.0+' - runtimeOnly 'jakarta.xml.bind:jakarta.xml.bind-api:3.0+' + implementation 'org.apache.commons:commons-lang3:3.12.+' + runtimeOnly 'com.sun.xml.bind:jaxb-impl:3.0.+' // Needed to compile unit tests. - implementation 'junit:junit:4.+' - implementation 'org.mockito:mockito-core:4.8.+' + testImplementation 'junit:junit:4.+' + testImplementation 'org.mockito:mockito-core:4.8.+' // The following stuff is needed in order to build custom gradle tasks and plugins. implementation gradleApi() implementation localGroovy() } +test { + testLogging { + showStandardStreams = true + } +} + eclipse { classpath { - defaultOutputDir = file("$buildDir/eclipse") + defaultOutputDir = file("$buildDir/eclipse/default") // outputBaseDir = file("$buildDir/eclipse") downloadSources = false downloadJavadoc = false @@ -45,19 +54,65 @@ eclipse { entries.each { entry -> if (entry.kind == "src" && entry.hasProperty("output")) { // The use of $buildDir does not return build. - entry.output = "build/eclipse" + entry.output = entry.output.replace("bin/", "build/eclipse/") } } } } } -test { - testLogging { - showStandardStreams = true +// Avoid duplicate classpath entries. +tasks.eclipse.dependsOn(cleanEclipse) + +// Publish ziggy-buildSrc.jar. See also $rootDir/script-utils/publish.gradle. +publishing { + publications { + ziggyBuildSrc(MavenPublication) { + groupId group + from components.java + artifactId "ziggy-buildSrc" + version version + + // Maven doesn't understand 2.17.+ notation. + suppressPomMetadataWarningsFor('runtimeElements') + + pom { + name = 'Ziggy BuildSrc' + description = 'Ziggy Gradle build library' + url = 'https://github.com/nasa/ziggy' + licenses { + license { + name = 'NASA Open Source Agreement Version 1.3' + url = 'https://github.com/nasa/ziggy/blob/main/LICENSE.pdf' + } + } + developers { + developer { + id = 'quarkpt' + name = 'Peter Tenenbaum' + email = 'Peter.Tenenbaum@nasa.gov' + } + developer { + id = 'wohler' + name = 'Bill Wohler' + email = 'Bill.Wohler@nasa.gov' + } + } + scm { + connection = 'scm:git:https://github.com/nasa/ziggy.git' + developerConnection = 'scm:git:git@github.com:nasa/ziggy.git' + url = 'https://github.com/nasa/ziggy' + } + } + } } -} -apply from: "script-plugins/hdf5.gradle" -apply from: "script-plugins/wrapper.gradle" + repositories { + maven { + name = 'ziggyBuildSrc' + url = layout.buildDirectory.dir("repository") + } + } +} +build.dependsOn publish diff --git a/buildSrc/gradle.properties b/buildSrc/gradle.properties new file mode 120000 index 0000000..7677fb7 --- /dev/null +++ b/buildSrc/gradle.properties @@ -0,0 +1 @@ +../gradle.properties \ No newline at end of file diff --git a/buildSrc/script-plugins/hdf5.gradle b/buildSrc/script-plugins/hdf5.gradle deleted file mode 100644 index 23251d0..0000000 --- a/buildSrc/script-plugins/hdf5.gradle +++ /dev/null @@ -1,45 +0,0 @@ -// Download and build HDF5. -// -// TODO Next time around, parameterize the version and possibly the URL - -task buildHdf5() { - def tmp = file("$buildDir/tmp/hdf5") - def hdf5 = file("$tmp/hdf5-1.12.2") - def lib = file("$buildDir/lib") - - outputs.file "$lib/jarhdf5-1.12.2.jar" - - doLast() { - tmp.mkdirs() - - // Use the SHA256 links at https://www.hdfgroup.org/downloads/hdf5/source-code/ to determine the URL. - exec { - workingDir tmp - commandLine "curl", "-o", "hdf5-1.12.2.#1", "https://hdf-wordpress-1.s3.amazonaws.com/wp-content/uploads/manual/HDF5/HDF5_1_12_2/source/hdf5-1.12.2.{tar.bz2,sha256}" - } - exec { - workingDir tmp - commandLine "tar", "-xf", "hdf5-1.12.2.tar.bz2" - } - exec { - workingDir hdf5 - commandLine "sh", "-c", "./configure --with-zlib=/usr --prefix=$buildDir --enable-threadsafe --with-pthread=/usr --enable-unsupported --enable-java" - } - exec { - workingDir hdf5 - commandLine "make" - } - exec { - workingDir hdf5 - commandLine "make", "install" - } - copy { - from("$tmp/lib") { - include "*.jar" - } - into "$lib" - } - } -} - -assemble.dependsOn buildHdf5 diff --git a/buildSrc/script-plugins/wrapper.gradle b/buildSrc/script-plugins/wrapper.gradle deleted file mode 100644 index fc210d2..0000000 --- a/buildSrc/script-plugins/wrapper.gradle +++ /dev/null @@ -1,56 +0,0 @@ -// Download and build the wrapper. - -import org.gradle.internal.os.OperatingSystem - -task buildWrapper() { - def version = "3.5.26" - def tmp = file("$buildDir/tmp/wrapper") - def bin = file("$buildDir/bin") - def lib = file("$buildDir/lib") - def format = "" - def libSuffix = "" - - OperatingSystem os = OperatingSystem.current(); - if (os.isLinux()) { - format = "linux-x86-64" - libSuffix = "so" - } else if (os.isMacOsX()) { - format = "macosx-universal-64" - libSuffix = "jnilib" - } - def wrapperBaseName = "wrapper-$format-$version" - def wrapperDir = file("$tmp/$wrapperBaseName") - def wrapperLib = "libwrapper.$libSuffix" - - outputs.file "$bin/wrapper" - outputs.file "$wrapperLib" - outputs.file "$lib/wrapper.jar" - - doLast() { - tmp.mkdirs() - - // See https://wrapper.tanukisoftware.com/doc/english/versions.jsp to determine the URL. - exec { - workingDir tmp - commandLine "curl", "-o", "wrapper-${version}.tar.gz", "https://download.tanukisoftware.com/wrapper/$version/${wrapperBaseName}.tar.gz" - } - exec { - workingDir tmp - commandLine "tar", "-xf", "wrapper-${version}.tar.gz" - } - copy { - from("$wrapperDir/bin") { - include "wrapper" - } - into "$bin" - } - copy { - from("$wrapperDir/lib") { - include "$wrapperLib", "wrapper.jar" - } - into "$lib" - } - } -} - -assemble.dependsOn buildWrapper diff --git a/buildSrc/src/main/java/gov/nasa/ziggy/buildutil/EnvUtil.java b/buildSrc/src/main/java/gov/nasa/ziggy/buildutil/EnvUtil.java deleted file mode 100644 index ba15104..0000000 --- a/buildSrc/src/main/java/gov/nasa/ziggy/buildutil/EnvUtil.java +++ /dev/null @@ -1,33 +0,0 @@ -package gov.nasa.ziggy.buildutil; - -import java.util.NoSuchElementException; - -/** - * Environment variables utilities. - * - * @author Sean McCauliff - * - */ -public class EnvUtil { - - /** - * @param key non-null environment variable key. - * @return the environment variable - * @exception NoSuchElementException if the key does not exist. - */ - public static String environment(String key) { - String value = System.getenv(key); - if (value == null) { - throw new NoSuchElementException("Environment variable \"" + key + "\" is not set."); - } - return value; - } - - public static String environment(String key, String defaultValue) { - String value = System.getenv(key); - if (value == null) { - return defaultValue; - } - return value; - } -} diff --git a/buildSrc/src/main/java/gov/nasa/ziggy/buildutil/GitRevisionHistoryToLatex.java b/buildSrc/src/main/java/gov/nasa/ziggy/buildutil/GitRevisionHistoryToLatex.java deleted file mode 100644 index f957f5a..0000000 --- a/buildSrc/src/main/java/gov/nasa/ziggy/buildutil/GitRevisionHistoryToLatex.java +++ /dev/null @@ -1,353 +0,0 @@ -package gov.nasa.ziggy.buildutil; - -import java.io.BufferedReader; -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileWriter; -import java.io.IOException; -import java.io.InputStreamReader; -import java.io.PrintWriter; -import java.io.StringWriter; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Calendar; -import java.util.Date; -import java.util.List; -import java.util.Map; -import java.util.SortedMap; -import java.util.TreeMap; -import java.util.concurrent.TimeUnit; - -/** - * Utility methods for dealing with latex files. - */ -public class GitRevisionHistoryToLatex { - - private static final int LINES_PER_PAGE = 50; - private static final int CHARACTERS_PER_LINE = 100; - private static final ThreadLocal revisionHistoryDateFormat = ThreadLocal - .withInitial(() -> new SimpleDateFormat("d MMM yyyy")); - - /** - * Runs "git log" on the specified document file to generate the revision history in the same - * directory as -revision-history.tex - * - * @throws Exception - */ - public static void gitLogToLatex(String documentFileName, File destDir) throws Exception { - try { - System.out.println( - "Generating revision history for latex file \"" + documentFileName + "\"."); - gitLogToLatexInternal(documentFileName, destDir); - } catch (Throwable t) { - // This is here because gradle -stacktrace causes this stack trace - // to disappear. - StringWriter stringWriter = new StringWriter(); - PrintWriter pw = new PrintWriter(stringWriter); - t.printStackTrace(pw); - System.out.println(stringWriter.toString()); - throw t; - } - } - - /** - * Escapes stuff like $ and _. - */ - private static String escapeLatex(String raw) { - StringBuilder bldr = new StringBuilder(raw.length()); - for (int i = 0; i < raw.length(); i++) { - char c = raw.charAt(i); - switch (c) { - case '\\': - bldr.append("\\\\"); - break; - case '$': - bldr.append("\\$"); - break; - case '_': - bldr.append("\\_"); - break; - case '{': - bldr.append("\\{"); - break; - case '}': - bldr.append("\\}"); - break; - case '~': - bldr.append("\\~"); - break; - case '&': - bldr.append("\\&"); - break; - // TODO: test me - case '"': - bldr.append("\\verb|\"|"); - break; - default: - bldr.append(c); - } - } - return bldr.toString(); - } - - private static void writeErrorToLatexFile(Throwable exception, File outputFile) - throws IOException { - StringWriter errMsgString = new StringWriter(); - PrintWriter exceptionOutput = new PrintWriter(errMsgString); - exception.printStackTrace(exceptionOutput); - exceptionOutput.close(); - BufferedWriter fileWriter = new BufferedWriter(new FileWriter(outputFile)); - try { - fileWriter.write("Error &"); - fileWriter.write(escapeLatex(errMsgString.toString())); - fileWriter.write("\\\\\n"); - } finally { - fileWriter.close(); - } - } - - private static final class HistoryEntry { - private final Date entryDate; - private final String entry; - private final int pageNumber; - - public HistoryEntry(Date entryDate, String entry, int pageNumber) { - this.entryDate = entryDate; - this.entry = entry; - this.pageNumber = pageNumber; - } - - void writeTo(Appendable writer) throws IOException { - writer.append(revisionHistoryDateFormat.get().format(entryDate)); - writer.append(" & "); - writer.append(entry); - writer.append("\\\\\n"); - } - } - - private static void gitLogToLatexInternal(String documentFileName, File destDir) - throws ParseException, IOException, InterruptedException { - String[] command = new String[] { "git", "log", "--follow", documentFileName }; - - File documentFile = new File(documentFileName); - String fileNameWithoutExtension = stripFileExtension(documentFile.getName()); - String outputFName = fileNameWithoutExtension + "-revision-history.tex"; - File outputFile = new File(destDir, outputFName); - - List lines = readProcessOutput(command, outputFile); - - // System.out.println("Completed reading git history."); - SortedMap dateToLog = collectHistoryEntries(lines); - - // Paginate - List> historyEntriesPerPage = paginate(dateToLog); - - // Write out top level history tex file - System.out.println("Writing history file \"" + outputFile + "\"."); - try (BufferedWriter outputWriter = new BufferedWriter(new FileWriter(outputFile))) { - for (int pagei = 0; pagei < historyEntriesPerPage.size(); pagei++) { - outputWriter.write("\\input{build/" + fileNameWithoutExtension - + "-revision-history-" + pagei + ".tex}\n"); - } - } - - // Write out each page's history. - for (int pagei = 0; pagei < historyEntriesPerPage.size(); pagei++) { - File pageFileName = new File(outputFile.getParent(), - fileNameWithoutExtension + "-revision-history-" + pagei + ".tex"); - try (BufferedWriter writer = new BufferedWriter(new FileWriter(pageFileName))) { - // Table header - writer.write("\\begin{table}[H]\n"); - writer.write("\\centering\n"); - writer.write("\\begin{tabularx}{\\linewidth}{l X}\n"); - writer.write("\\thead{Change Date} & \\thead{Notes} \\\\\n"); - writer.write("\\hline\n"); - - for (HistoryEntry historyEntry : historyEntriesPerPage.get(pagei)) { - historyEntry.writeTo(writer); - } - - // Table footer. - writer.write("\\hline\n"); - writer.write("\\end{tabularx}\n"); - writer.write("\\end{table}\n"); - writer.write("\\newpage\n"); - } - } - - } - - private static List> paginate(SortedMap dateToLog) { - - List> entriesPerPage = new ArrayList<>(); - int lineCount = 0; - for (Map.Entry entry : dateToLog.entrySet()) { - String entryString = entry.getValue().toString(); - int lineCountForEntry = entryString.length() / CHARACTERS_PER_LINE + 1; - lineCount += lineCountForEntry; - int pageNo = lineCount / LINES_PER_PAGE; - - if (entriesPerPage.size() <= pageNo) { - entriesPerPage.add(new ArrayList<>(LINES_PER_PAGE)); - } - List l = entriesPerPage.get(pageNo); - l.add(new HistoryEntry(entry.getKey(), entryString, pageNo)); - } - - return entriesPerPage; - } - - private static SortedMap collectHistoryEntries(List lines) - throws ParseException { - TreeMap dateToLog = new TreeMap<>(); - SimpleDateFormat dateFormat = new SimpleDateFormat("EEE MMM d HH:mm:ss yyyy Z"); - StringBuilder currentEntry = null; - Date entryDate = null; - boolean skipEntry = false; - for (String line : lines) { - if (line.startsWith("commit")) { - // Start of new entry - if (currentEntry != null && entryDate != null) { - StringBuilder existingLog = dateToLog.get(entryDate); - existingLog.append(currentEntry); - } - currentEntry = new StringBuilder(); - entryDate = null; - skipEntry = false; - } else if (line.startsWith("Date: ")) { - // Start of new date - String dateLine = line.substring(6, line.length()).trim(); - Calendar calendar = Calendar.getInstance(); - calendar.setTime(dateFormat.parse(dateLine)); - calendar.set(Calendar.HOUR, 0); - calendar.set(Calendar.MINUTE, 0); - calendar.set(Calendar.SECOND, 0); - calendar.set(Calendar.AM_PM, Calendar.AM); - entryDate = calendar.getTime(); - if (!dateToLog.containsKey(entryDate)) { - dateToLog.put(entryDate, new StringBuilder()); - } - } else if (line.startsWith("Author")) { - // Ignore - } else if (line.matches("\\s+") || line.length() == 0) { - // Ignore lines with only white space - } else if (line.contains("Merge branch")) { - // Ignore log entries that are merges. - skipEntry = true; - currentEntry = null; - } else if (!skipEntry) { - // Content - if (currentEntry == null) { - throw new NullPointerException("currentEntry == null. line=\"" + line + "\""); - } - line = line.trim(); - currentEntry.append("~"); - currentEntry.append(escapeLatex(line)); - } - } - if (currentEntry != null && entryDate != null) { - // Append last entry - StringBuilder existingLog = dateToLog.get(entryDate); - existingLog.append(currentEntry); - } - return dateToLog; - } - - private static List readProcessOutput(String[] command, File outputFile) - throws InterruptedException, IOException { - Process process = null; - List lines = new ArrayList<>(1024); - try { - ProcessBuilder processBuilder = new ProcessBuilder(command); - // processBuilder.inheritIO(); - process = processBuilder.start(); - // System.out.println("Running git process."); - BufferedReader bufferedReader = new BufferedReader( - new InputStreamReader(process.getInputStream())); - - for (String line = bufferedReader.readLine(); line != null; line = bufferedReader - .readLine()) { - // System.out.println("lineRead " + line); - lines.add(line); - } - process.waitFor(1, TimeUnit.SECONDS); - - } catch (IOException ioe) { - System.out.println("Writing error to latex file."); - try { - writeErrorToLatexFile(ioe, outputFile); - } catch (IOException ignored) { - } - throw ioe; - } finally { - if (process != null) { - try { - process.getErrorStream().close(); - } catch (Exception ignored) { - } - try { - process.getOutputStream().close(); - } catch (Exception ignored) { - } - try { - process.getInputStream().close(); - } catch (Exception ignored) { - } - } - } - - if (process.exitValue() != 0) { - throw new IOException( - "Process exited with non-zero exit code (" + process.exitValue() + ")."); - } - return lines; - } - - /** - * Returns a file name without its extension so blah.txt would become blah (no dot). - */ - public static String stripFileExtension(String fname) { - int dotIndex = fname.lastIndexOf('.'); - if (dotIndex == -1 || dotIndex == 0) { - return fname; - } else { - return fname.substring(0, dotIndex); - } - } - - /** - * Returns the file extension - */ - public static String fileExtension(String fname) { - int dotIndex = fname.lastIndexOf('.'); - if (dotIndex == -1 || dotIndex == 0 || dotIndex == (fname.length() - 1)) { - return ""; - } else { - return fname.substring(dotIndex + 1, fname.length()); - } - } - - public static String changeFileExtension(String fname, String newExtension) { - if (newExtension.charAt(0) != '.') { - return newExtension = "." + newExtension; - } - String prefix = stripFileExtension(fname); - return prefix + newExtension; - } - - /** - * Given the specified file name this appends the creation date before the file name extension. - */ - public static String appendCreationDate(String fname) { - String extension = fileExtension(fname); - if (extension.length() != 0) { - extension = "." + extension; - } - String prefix = stripFileExtension(fname); - // TODO: Is this the correct format? - SimpleDateFormat dateFormatter = new SimpleDateFormat("dd-MMM-yyyy"); - return prefix + "-" + dateFormatter.format(new Date()) + extension; - } -} - diff --git a/buildSrc/src/main/java/gov/nasa/ziggy/buildutil/Mcc.java b/buildSrc/src/main/java/gov/nasa/ziggy/buildutil/Mcc.java index 72c3bb2..606921b 100644 --- a/buildSrc/src/main/java/gov/nasa/ziggy/buildutil/Mcc.java +++ b/buildSrc/src/main/java/gov/nasa/ziggy/buildutil/Mcc.java @@ -7,8 +7,14 @@ import java.io.File; import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.FileVisitResult; import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; import java.nio.file.attribute.PosixFilePermission; +import java.nio.file.attribute.PosixFilePermissions; import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; @@ -17,22 +23,23 @@ import java.util.stream.Collectors; import org.apache.commons.io.FileUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.gradle.api.GradleException; import org.gradle.api.file.FileCollection; +import org.gradle.api.tasks.Input; import org.gradle.api.tasks.InputFiles; import org.gradle.api.tasks.Optional; import org.gradle.api.tasks.OutputDirectory; import org.gradle.api.tasks.OutputFile; import org.gradle.api.tasks.TaskAction; import org.gradle.internal.os.OperatingSystem; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Compiles the matlab executables (i.e. it runs mcc). This class can not be made final. *

- * This class puts the variable MCC_DIR into the environment and sets it to the directory of the project that has - * invoked this task. + * This class puts the variable MCC_DIR into the environment and sets it to the directory of the + * project that has invoked this task. * * @author Bill Wohler * @author Sean McCauliff @@ -60,6 +67,8 @@ public void setControllerFiles(FileCollection controllerFiles) { this.controllerFiles = controllerFiles; } + @InputFiles + @Optional public FileCollection getAdditionalFiles() { return additionalFiles; } @@ -95,7 +104,8 @@ public void setOutputExecutable(File outputExecutable) { } @Optional - public boolean isSingleThreaded() { + @Input + public Boolean isSingleThreaded() { return singleThreaded; } @@ -109,10 +119,8 @@ public void action() { File matlabHome = matlabHome(); File buildBinDir = new File(getProject().getBuildDir(), "bin"); - List command = new ArrayList<>(); - - command.addAll(Arrays.asList("mcc", "-v", "-m", "-N", "-d", buildBinDir.toString(), "-R", - "-nodisplay", "-R", "-nodesktop")); + List command = new ArrayList<>(Arrays.asList("mcc", "-v", "-m", "-N", "-d", + buildBinDir.toString(), "-R", "-nodisplay", "-R", "-nodesktop")); if (isSingleThreaded()) { command.add("-R"); @@ -184,33 +192,40 @@ public void action() { } execProcess(processBuilder); - Set neededPermissions = new HashSet<>(Arrays.asList( - new PosixFilePermission[] { OWNER_EXECUTE, GROUP_EXECUTE, OWNER_READ, GROUP_READ })); + Set neededPermissions = new HashSet<>( + Arrays.asList(OWNER_EXECUTE, GROUP_EXECUTE, OWNER_READ, GROUP_READ)); if (!executable.exists()) { String message = "The outputExecutable,\"" + executable + "\" does not exist."; log.error(message); throw new GradleException(message); - } else { - Set currentPosixFilePermissions = null; - try { - currentPosixFilePermissions = Files.getPosixFilePermissions(executable.toPath()); + } + Set currentPosixFilePermissions = null; + try { + currentPosixFilePermissions = Files.getPosixFilePermissions(executable.toPath()); + log.info(currentPosixFilePermissions.stream() + .map(PosixFilePermission::name) + .collect(Collectors.joining(" ", "Current file permissions are: ", "."))); + if (!neededPermissions.containsAll(currentPosixFilePermissions)) { + currentPosixFilePermissions.addAll(neededPermissions); log.info(currentPosixFilePermissions.stream() - .map(p -> p.name()) - .collect(Collectors.joining(" ", "Current file permissions are: ", "."))); - if (!neededPermissions.containsAll(currentPosixFilePermissions)) { - currentPosixFilePermissions.addAll(neededPermissions); - log.info(currentPosixFilePermissions.stream() - .map(p -> p.name()) - .collect(Collectors.joining(" ", "Setting file permissions to: ", ","))); - Files.setPosixFilePermissions(executable.toPath(), currentPosixFilePermissions); - } - } catch (IOException ioe) { - String message = "Failed to either get or set permissions on outputExecutable \"" - + outputExecutable; - log.error(message); - throw new GradleException(message); + .map(PosixFilePermission::name) + .collect(Collectors.joining(" ", "Setting file permissions to: ", ","))); + Files.setPosixFilePermissions(executable.toPath(), currentPosixFilePermissions); } + } catch (IOException ioe) { + String message = "Failed to either get or set permissions on outputExecutable \"" + + outputExecutable; + log.error(message); + throw new GradleException(message); + } + + // On the Mac, there are 2 files created without U+W permission, so any time the + // user tries an incremental build it fails because those files can't be deleted. + if (SystemArchitecture.architecture() == SystemArchitecture.MAC_INTEL + || SystemArchitecture.architecture() == SystemArchitecture.MAC_M1) { + setPosixPermissionsRecursively(executable.toPath().resolve("Contents"), "rwxr-xr--", + "rwxr-xr--"); } File readme = new File(outputExecutable.getParentFile(), "readme.txt"); if (readme.exists()) { @@ -218,4 +233,61 @@ public void action() { outputExecutable.getName() + "-readme.txt")); } } + + /** + * Recursively sets permissions on all files and directories that lie under a given top-level + * directory. + * + * @param top Location of the top-level directory. + * @param filePermissions POSIX-style string of permissions for regular files (i.e., + * "r--r-r--"). + * @param dirPermissions POSIX-style string of permissions for regular files (i.e., + * "rwxr-xr-x"). + *

+ * NB: This is a copy of the same method that appears in FileUtil. It's duplicated here so that + * we don't need to have buildSrc trying to use code from Ziggy main source, and also so that we + * don't wind up with code in FileUtil calling this method in buildSrc. + */ + public static void setPosixPermissionsRecursively(Path top, String filePermissions, + String dirPermissions) { + try { + if (!Files.isDirectory(top)) { + Files.setPosixFilePermissions(top, + PosixFilePermissions.fromString(filePermissions)); + } else { + Files.setPosixFilePermissions(top, PosixFilePermissions.fromString(dirPermissions)); + Files.walkFileTree(top, new SimpleFileVisitor() { + + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) { + try { + Files.setPosixFilePermissions(dir, + PosixFilePermissions.fromString(dirPermissions)); + } catch (IOException e) { + throw new UncheckedIOException( + "Failed to set permissions on dir " + dir.toString(), e); + } + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { + if (Files.isSymbolicLink(file)) { + return FileVisitResult.CONTINUE; + } + try { + Files.setPosixFilePermissions(file, + PosixFilePermissions.fromString(filePermissions)); + } catch (IOException e) { + throw new UncheckedIOException( + "Failed to set permissions on file " + file.toString(), e); + } + return FileVisitResult.CONTINUE; + } + }); + } + } catch (IOException e) { + throw new UncheckedIOException("Failed to set permissions on dir " + top.toString(), e); + } + } } diff --git a/buildSrc/src/main/java/gov/nasa/ziggy/buildutil/SystemArchitecture.java b/buildSrc/src/main/java/gov/nasa/ziggy/buildutil/SystemArchitecture.java new file mode 100644 index 0000000..3defcd5 --- /dev/null +++ b/buildSrc/src/main/java/gov/nasa/ziggy/buildutil/SystemArchitecture.java @@ -0,0 +1,93 @@ +package gov.nasa.ziggy.buildutil; + +/** + * Provides a uniform system for ascertaining the combination of OS and CPU architecture of the + * system. + * + * @author PT + */ +public enum SystemArchitecture { + LINUX_INTEL, MAC_INTEL, MAC_M1; + + public static SystemArchitecture architecture() { + + String opSys = System.getProperty("os.name").toLowerCase(); + String arch = System.getProperty("os.arch").toLowerCase(); + if (opSys.contains("linux")) { + return LINUX_INTEL; + } + if (opSys.contains("mac")) { + if (arch.contains("aarch")) { + return MAC_M1; + } + return MAC_INTEL; + } + throw new IllegalStateException("System architecture unknown"); + } + + /** + * Returns a new instance of {@link Selector}. + * + * @param Class for Selector. + * @param clazz Class for Selector. + */ + public static Selector selector(Class clazz) { + return new Selector<>(); + } + + /** + * Provides a compact notation for specifying a collection of objects, where one of the + * collection is to be returned depending on the system architecture of the executing computer. + * This can be used in place of switch statements (for systems that support them), or if-elseif + * chains (for systems that do not support switch, such as Gradle scripts). Example: + * + *

+     * String s = SystemArchitecture.selector(String.class)
+     *     .linuxIntelObject("A")
+     *     .macIntelObject("B")
+     *     .macM1Object("C")
+     *     .get();
+     * 
+ * + * will return "A" when executed on a Linux system, "B" when executed on a Mac using Intel + * silicon, and "C" when executed on a Mac using Apple silicon. + * + * @author PT + * @param Class of object to be managed by the {@link Selector}. + */ + public static class Selector { + private T linuxIntelObject; + private T macM1Object; + private T macIntelObject; + + private Selector() { + } + + public Selector linuxIntelObject(T object) { + linuxIntelObject = object; + return this; + } + + public Selector macIntelObject(T object) { + macIntelObject = object; + return this; + } + + public Selector macM1Object(T object) { + macM1Object = object; + return this; + } + + SystemArchitecture architecture() { + return SystemArchitecture.architecture(); + } + + public T get() { + return switch (architecture()) { + case LINUX_INTEL -> linuxIntelObject; + case MAC_INTEL -> macIntelObject; + case MAC_M1 -> macM1Object; + }; + } + } +} diff --git a/buildSrc/src/main/java/gov/nasa/ziggy/buildutil/ZiggyCpp.java b/buildSrc/src/main/java/gov/nasa/ziggy/buildutil/ZiggyCpp.java index 3813ae5..eb3887d 100644 --- a/buildSrc/src/main/java/gov/nasa/ziggy/buildutil/ZiggyCpp.java +++ b/buildSrc/src/main/java/gov/nasa/ziggy/buildutil/ZiggyCpp.java @@ -5,31 +5,90 @@ import org.gradle.api.DefaultTask; import org.gradle.api.Project; +import org.gradle.api.tasks.Input; import org.gradle.api.tasks.InputFiles; +import org.gradle.api.tasks.Internal; +import org.gradle.api.tasks.Optional; import org.gradle.api.tasks.OutputFile; import org.gradle.api.tasks.TaskAction; -import org.gradle.api.tasks.Input; - import gov.nasa.ziggy.buildutil.ZiggyCppPojo.BuildType; /** - * Performs C++ builds for Gradle. Ziggy and its pipelines use this instead of the - * standard Gradle C++ task classes because it provides only the options we actually - * need without many that we can't use, and also because it allows us to use the CXX - * environment variable to define the location of the C++ compiler. This allows us to - * use the same compiler as some of the third party libraries used by Ziggy (they also - * use CXX to select their compiler). - * - * Because Gradle classes that extend DefaultTask are effectively impossible to unit test, - * all of the actual data and work are managed by the ZiggyCppPojo class (which does have - * unit tests), while this class simply provides access to the ZiggyCppPojo class for Gradle. - * - * @author PT + * Performs C++ builds for Gradle. Ziggy and its pipelines use this instead of the standard Gradle + * C++ task classes because it provides only the options we actually need without many that we can't + * use, and also because it allows us to use the CXX environment variable to define the location of + * the C++ compiler. This allows us to use the same compiler as some of the third party libraries + * used by Ziggy (they also use CXX to select their compiler). + *

+ * Usage example: assume that the user wishes to construct a shared object library, libfoo.(so, + * dylib) from the contents of directories $projectDir/foo and $projectDir/bar, with additional + * include files in $projectDir/include and $rootDir/project2/include. The task + * + *

+ * task fooLib(type : ZiggyCpp) {
+ *     outputName        = "foo"
+ *     cppFilePaths      = ["$projectDir/foo", "$projectDir/bar"]
+ *     includeFilePaths  = ["$projectDir/include", "$rootDir/project2/include"]
+ *     outputType        = "shared"
+ *     compileOptions    = ["std=c++11", "O2"]
+ *     maxCompileThreads = 10
+ * }
+ * 
+ * + * will compile the contents of the specified C++ file paths, placing the resulting object files in + * $buildDir/obj, and then link them into shared object library libfoo.(so, dylib), placed in + * $buildDir/lib. Options "-std=c++11" and "-O2" will be applied at compile time. During the compile + * phase, up to 10 threads will run in order to compile source files in parallel. If + * maxCompileThreads is not specified, a default value equal to the number of cores on the system + * will be used. + *

+ * If a static library had been desired in the prior example, the outputType would be changed to + * "static", and file libfoo.a would be saved to $buildDir/lib. If a standalone executable was + * needed, the outputType would be changed to "executable" and the resulting file foo would be + * placed in $buildDir/bin. In this latter case, the user can specify additional libraries and + * library paths via additional options: + * + *

+ * task fooProgram(type : ZiggyCpp) {
+ *     outputName        = "foo"
+ *     cppFilePaths      = ["$projectDir/foo", "$projectDir/bar"]
+ *     includeFilePaths  = ["$projectDir/include", "$rootDir/project2/include"]
+ *     outputType        = "executable"
+ *     cppCompileOptions = ["std=c++11", "O2"]
+ *     cCompileOptions   = ["O2"]
+ *     libraryPaths      = ["$buildDir/lib", "$rootDir/project2/build/lib"]
+ *     libraries         = ["math1", "spelling2"]
+ * }
+ * 
* + * would link using libmath1 and libmath2, located in $buildDir/lib and/or + * $rootDir/project2/build/lib. + *

+ * The user can override the default directories used to store build products, which are: + * $buildDir/lib, $buildDir/obj, and $buildDir/bin, respectively, for libraries, object files, and + * executables. There are two ways to override: the first is to specify a directory that will be + * used for all output products using the property name "outputDir". The other is to specify a + * parent directory for the output directories using the property name "outputDirParent". In this + * latter case, outputs will go to /lib, /obj, and + * /bin, respectively, for libraries, object files, and executables. If both + * outputDir and outputDirParent are specified, outputDir will take precedence. + *

+ * Because Gradle classes that extend DefaultTask are effectively impossible to unit test, all of + * the actual data and work are managed by the ZiggyCppPojo class (which does have unit tests), + * while this class simply provides access to the ZiggyCppPojo class for Gradle. + * + * @author PT */ public class ZiggyCpp extends DefaultTask { + private static final String DEFAULT_CPP_COMPILE_OPTIONS_GRADLE_PROPERTY = "defaultCppCompileOptions"; + private static final String DEFAULT_C_COMPILE_OPTIONS_GRADLE_PROPERTY = "defaultCCompileOptions"; + private static final String DEFAULT_LINK_OPTIONS_GRADLE_PROPERTY = "defaultLinkOptions"; + private static final String DEFAULT_RELEASE_OPTS_GRADLE_PROPERTY = "defaultReleaseOptimizations"; + private static final String DEFAULT_DEBUG_OPTS_GRADLE_PROPERTY = "defaultDebugOptimizations"; + private static final String PIPELINE_ROOT_DIR_PROP_NAME = "pipelineRootDir"; + /** * Data and methods used to perform the C++ compile and link. */ @@ -43,21 +102,25 @@ public ZiggyCpp() { Project project = getProject(); ziggyCppPojo.setBuildDir(project.getBuildDir()); ziggyCppPojo.setRootDir(pipelineRootDir(project)); - if (project.hasProperty(ZiggyCppPojo.DEFAULT_COMPILE_OPTIONS_GRADLE_PROPERTY)) { - ziggyCppPojo.setCompileOptions(ZiggyCppPojo.gradlePropertyToList( - project.property(ZiggyCppPojo.DEFAULT_COMPILE_OPTIONS_GRADLE_PROPERTY))); + if (project.hasProperty(DEFAULT_CPP_COMPILE_OPTIONS_GRADLE_PROPERTY)) { + ziggyCppPojo.setCppCompileOptions(ZiggyCppPojo.gradlePropertyToList( + project.property(DEFAULT_CPP_COMPILE_OPTIONS_GRADLE_PROPERTY))); } - if (project.hasProperty(ZiggyCppPojo.DEFAULT_LINK_OPTIONS_GRADLE_PROPERTY)) { - ziggyCppPojo.setLinkOptions(ZiggyCppPojo.gradlePropertyToList( - project.property(ZiggyCppPojo.DEFAULT_LINK_OPTIONS_GRADLE_PROPERTY))); + if (project.hasProperty(DEFAULT_C_COMPILE_OPTIONS_GRADLE_PROPERTY)) { + ziggyCppPojo.setCCompileOptions(ZiggyCppPojo + .gradlePropertyToList(project.property(DEFAULT_C_COMPILE_OPTIONS_GRADLE_PROPERTY))); } - if (project.hasProperty(ZiggyCppPojo.DEFAULT_RELEASE_OPTS_GRADLE_PROPERTY)) { - ziggyCppPojo.setReleaseOptimizations(ZiggyCppPojo.gradlePropertyToList( - project.findProperty(ZiggyCppPojo.DEFAULT_RELEASE_OPTS_GRADLE_PROPERTY))); + if (project.hasProperty(DEFAULT_LINK_OPTIONS_GRADLE_PROPERTY)) { + ziggyCppPojo.setLinkOptions(ZiggyCppPojo + .gradlePropertyToList(project.property(DEFAULT_LINK_OPTIONS_GRADLE_PROPERTY))); } - if (project.hasProperty(ZiggyCppPojo.DEFAULT_DEBUG_OPTS_GRADLE_PROPERTY)) { - ziggyCppPojo.setDebugOptimizations(ZiggyCppPojo.gradlePropertyToList( - project.findProperty(ZiggyCppPojo.DEFAULT_DEBUG_OPTS_GRADLE_PROPERTY))); + if (project.hasProperty(DEFAULT_RELEASE_OPTS_GRADLE_PROPERTY)) { + ziggyCppPojo.setReleaseOptimizations(ZiggyCppPojo + .gradlePropertyToList(project.findProperty(DEFAULT_RELEASE_OPTS_GRADLE_PROPERTY))); + } + if (project.hasProperty(DEFAULT_DEBUG_OPTS_GRADLE_PROPERTY)) { + ziggyCppPojo.setDebugOptimizations(ZiggyCppPojo + .gradlePropertyToList(project.findProperty(DEFAULT_DEBUG_OPTS_GRADLE_PROPERTY))); } } @@ -70,9 +133,8 @@ public ZiggyCpp() { */ public static File pipelineRootDir(Project project) { File pipelineRootDir = null; - if (project.hasProperty(ZiggyCppPojo.PIPELINE_ROOT_DIR_PROP_NAME)) { - pipelineRootDir = new File( - project.property(ZiggyCppPojo.PIPELINE_ROOT_DIR_PROP_NAME).toString()); + if (project.hasProperty(PIPELINE_ROOT_DIR_PROP_NAME)) { + pipelineRootDir = new File(project.property(PIPELINE_ROOT_DIR_PROP_NAME).toString()); } else { pipelineRootDir = project.getRootDir(); } @@ -93,7 +155,7 @@ public void action() { */ @InputFiles public List getCppFiles() { - return ziggyCppPojo.getCppFiles(); + return ziggyCppPojo.getSourceFiles(); } /** @@ -138,7 +200,7 @@ public List getIncludeFilePaths() { return ziggyCppPojo.getIncludeFilePaths(); } - // paths for libraries that must be linked in + // Paths for libraries that must be linked in. public void setLibraryPaths(List libraryPaths) { ziggyCppPojo.setLibraryPaths(libraryPaths); } @@ -160,13 +222,12 @@ public List getLibraries() { // compiler options public void setCompileOptions(List compileOptions) { - ziggyCppPojo.setCompileOptions(compileOptions); - ; + ziggyCppPojo.setCppCompileOptions(compileOptions); } @Input public List getCompileOptions() { - return ziggyCppPojo.getCompileOptions(); + return ziggyCppPojo.getCppCompileOptions(); } // linker options @@ -198,4 +259,37 @@ public void setOutputName(Object name) { public String getOutputName() { return ziggyCppPojo.getOutputName(); } + + // Directory to use for build product + public void setOutputDir(Object outputDir) { + ziggyCppPojo.setOutputDir(outputDir); + } + + @Input + @Optional + public String getOutputDir() { + return ziggyCppPojo.getOutputDir(); + } + + // Parent directory for build product (i.e., build products go in this location + "/obj", + // "/bin", or "/lib") + + public void setOutputDirParent(Object outputDirParent) { + ziggyCppPojo.setOutputDirParent(outputDirParent); + } + + @Input + @Optional + public String getOutputDirParent() { + return ziggyCppPojo.getOutputDirParent(); + } + + public void setMaxCompileThreads(int maxCompileThreads) { + ziggyCppPojo.setMaxCompileThreads(maxCompileThreads); + } + + @Internal + public int getMaxCompileThreads() { + return ziggyCppPojo.getMaxCompileThreads(); + } } diff --git a/buildSrc/src/main/java/gov/nasa/ziggy/buildutil/ZiggyCppMex.java b/buildSrc/src/main/java/gov/nasa/ziggy/buildutil/ZiggyCppMex.java index a29f4cd..0c6e546 100644 --- a/buildSrc/src/main/java/gov/nasa/ziggy/buildutil/ZiggyCppMex.java +++ b/buildSrc/src/main/java/gov/nasa/ziggy/buildutil/ZiggyCppMex.java @@ -3,43 +3,97 @@ import java.io.File; import java.util.List; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.gradle.api.DefaultTask; import org.gradle.api.GradleException; import org.gradle.api.Project; +import org.gradle.api.tasks.Input; import org.gradle.api.tasks.InputFiles; +import org.gradle.api.tasks.Internal; +import org.gradle.api.tasks.Optional; import org.gradle.api.tasks.OutputFile; import org.gradle.api.tasks.OutputFiles; import org.gradle.api.tasks.TaskAction; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import gov.nasa.ziggy.buildutil.ZiggyCppPojo.BuildType; /** * Performs mexfile builds for Gradle. The sequence of build steps is as follows: - * 1. The C/C++ files are compiled with a MATLAB_MEX_FILE compiler directive. - * 2. The object files from (1) are linked into a shared object library. - * 3. The actual mexfiles are built from the appropriate object files and the - * shared object library. + *

    + *
  1. The C/C++ files are compiled with a MATLAB_MEX_FILE compiler directive. The resulting object + * files are saved in $buildDir/obj . + *
  2. The object files from (1) are linked into a shared object library. The shared object library + * is saved in $buildDir/lib . + *
  3. The actual mexfiles are built from the appropriate object files and the shared object + * library. The mexfile is saved in $buildDir/mex . + *
+ *

* The user specifies the following: - * 1. The path to the C/C++ files - * 2. Compiler and linker options, including libraries, library paths, include - * file paths, optimization flags. - * 3. The names of the desired mexfiles. - * 4. Optionally, a name for the shared library (otherwise a default name is generated - * from the C++ source file path). - * - * Because it is effectively impossible to unit test any class that extends DefaultTask, - * the actual workings of the ZiggyCppMex class are in a separate class, ZiggyCppMexPojo, - * which has appropriate unit testing. This class provides a thin interface between - * Gradle and the ZiggyCppMexPojo class. - * - * @author PT + *

    + *
  1. The path to the C/C++ files + *
  2. Compiler and linker options, including libraries, library paths, include file paths, + * optimization flags. + *
  3. The names of the desired mexfiles. + *
  4. Optionally, a name for the shared library (otherwise a default name is generated from the C++ + * source file path). + *
+ *

+ * Usage example: assume that mexfile main functions are in files foo.cpp and bar.cpp, which are in + * the src/cpp/foo subdirectory of $projectDir, along with header files and potentially additional + * C++ files. The user wishes to build mexfiles from these functions, which depend on additional + * libraries elsewhere on the system. The task * + *

+ * task mexFoo(type : ZiggyCppMex) {
+ *     cppFilePath      = "$projectDir/src/cpp/foo"
+ *     includeFilePaths = ["$projectDir/src/cpp/foo", "$projectDir/include"]
+ *     libraryPaths     = ["$buildDir/lib", "$rootDir/project2/build/lib"]
+ *     libraries        = ["blas", "lapack"]
+ *     mexfileNames     = ["foo", "bar"]
+ *     compileOptions   = ["std=c++11", "O2"]
+ *     outputName       = "baz"
+ * }
+ * 
+ * + * will compile the contents of $projectDir/src/cpp/mex into object files in $buildDir/obj, and link + * them into library libbaz.(so, dylib) in $buildDir/lib; mexfiles foo.(mexa64, mexmaci64, + * mexmaca64) and bar.(mexa64, mexmaci64, mexmaca64) are then generated in $buildDir/mex by linking + * against libraries libblas and liblapack, which are located in the directories specified by the + * libraryPaths property. Options "-std=c++11" and "-O2" will be applied at compile time; link + * options can be specified with the linkOptions property (not shown). + *

+ * Note that the user does not need to specify the locations for MATLAB include files, the locations + * or names of MATLAB libraries, or indeed the location of MATLAB's mex executable. The MATLAB + * top-level directory comes from either the $matlabHome Gradle variable or the $MATLAB_HOME + * environment variable; from this location, the operating system, and the CPU architecture, all of + * the MATLAB libraries, library directories, and mexfile suffix are automatically determined. + *

+ * By default, object files produced by {@link ZiggyCppMex} are saved in $buildDir/obj, libraries in + * $buildDir/lib, and mexfiles in $buildDir/mex. This behavior can be overridden. If the property + * outputDir is specified, then all 3 types of files will be stored in the location specified by + * outputDir. If the property outputDirParent is specified, then the 3 file types will be saved in + * /obj, /lib, and /mex, respectively. If both + * outputDir and outputDirParent are specified, the outputDir value will be used and the + * outputDirParent value ignored. + *

+ * Because it is effectively impossible to unit test any class that extends DefaultTask, the actual + * workings of the ZiggyCppMex class are in a separate class, ZiggyCppMexPojo, which has appropriate + * unit testing. This class provides a thin interface between Gradle and the ZiggyCppMexPojo class. + * + * @author PT */ public class ZiggyCppMex extends DefaultTask { private static final Logger log = LoggerFactory.getLogger(ZiggyCppMex.class); + + private static final String DEFAULT_COMPILE_OPTIONS_GRADLE_PROPERTY = "defaultCppMexCompileOptions"; + private static final String DEFAULT_LINK_OPTIONS_GRADLE_PROPERTY = "defaultCppMexLinkOptions"; + private static final String DEFAULT_RELEASE_OPTS_GRADLE_PROPERTY = "defaultCppMexReleaseOptimizations"; + private static final String DEFAULT_DEBUG_OPTS_GRADLE_PROPERTY = "defaultCppMexDebugOptimizations"; + private static final String MATLAB_PATH_PROJECT_PROPERTY = "matlabHome"; + private static final String MATLAB_PATH_ENV_VAR = "MATLAB_HOME"; + private ZiggyCppMexPojo ziggyCppMexObject = new ZiggyCppMexPojo(); /** @@ -53,21 +107,21 @@ public ZiggyCppMex() { ziggyCppMexObject.setBuildDir(project.getBuildDir()); ziggyCppMexObject.setRootDir(ZiggyCpp.pipelineRootDir(project)); ziggyCppMexObject.setProjectDir(project.getProjectDir()); - if (project.hasProperty(ZiggyCppMexPojo.DEFAULT_COMPILE_OPTIONS_GRADLE_PROPERTY)) { - ziggyCppMexObject.setCompileOptions(ZiggyCppPojo.gradlePropertyToList( - project.property(ZiggyCppMexPojo.DEFAULT_COMPILE_OPTIONS_GRADLE_PROPERTY))); + if (project.hasProperty(DEFAULT_COMPILE_OPTIONS_GRADLE_PROPERTY)) { + ziggyCppMexObject.setCppCompileOptions(ZiggyCppPojo + .gradlePropertyToList(project.property(DEFAULT_COMPILE_OPTIONS_GRADLE_PROPERTY))); } - if (project.hasProperty(ZiggyCppMexPojo.DEFAULT_LINK_OPTIONS_GRADLE_PROPERTY)) { - ziggyCppMexObject.setLinkOptions(ZiggyCppPojo.gradlePropertyToList( - project.property(ZiggyCppMexPojo.DEFAULT_LINK_OPTIONS_GRADLE_PROPERTY))); + if (project.hasProperty(DEFAULT_LINK_OPTIONS_GRADLE_PROPERTY)) { + ziggyCppMexObject.setLinkOptions(ZiggyCppPojo + .gradlePropertyToList(project.property(DEFAULT_LINK_OPTIONS_GRADLE_PROPERTY))); } - if (project.hasProperty(ZiggyCppMexPojo.DEFAULT_RELEASE_OPTS_GRADLE_PROPERTY)) { - ziggyCppMexObject.setReleaseOptimizations(ZiggyCppPojo.gradlePropertyToList( - project.findProperty(ZiggyCppMexPojo.DEFAULT_RELEASE_OPTS_GRADLE_PROPERTY))); + if (project.hasProperty(DEFAULT_RELEASE_OPTS_GRADLE_PROPERTY)) { + ziggyCppMexObject.setReleaseOptimizations(ZiggyCppPojo + .gradlePropertyToList(project.findProperty(DEFAULT_RELEASE_OPTS_GRADLE_PROPERTY))); } - if (project.hasProperty(ZiggyCppMexPojo.DEFAULT_DEBUG_OPTS_GRADLE_PROPERTY)) { - ziggyCppMexObject.setDebugOptimizations(ZiggyCppPojo.gradlePropertyToList( - project.findProperty(ZiggyCppMexPojo.DEFAULT_DEBUG_OPTS_GRADLE_PROPERTY))); + if (project.hasProperty(DEFAULT_DEBUG_OPTS_GRADLE_PROPERTY)) { + ziggyCppMexObject.setDebugOptimizations(ZiggyCppPojo + .gradlePropertyToList(project.findProperty(DEFAULT_DEBUG_OPTS_GRADLE_PROPERTY))); } setMatlabPath(); } @@ -81,7 +135,7 @@ public void action() { /** Specifies that the C/C++ source files are the input files for this Gradle task. */ @InputFiles public List getCppFiles() { - return ziggyCppMexObject.getCppFiles(); + return ziggyCppMexObject.getSourceFiles(); } /** Specifies that the mexfiles are the output files for this Gradle task. */ @@ -112,6 +166,7 @@ public void setCppFilePath(Object cppFilePath) { ziggyCppMexObject.setCppFilePath(cppFilePath); } + @Internal public String getCppFilePath() { return ziggyCppMexObject.getCppFilePaths().get(0); } @@ -121,15 +176,17 @@ public void setIncludeFilePaths(List includeFilePaths) { ziggyCppMexObject.setIncludeFilePaths(includeFilePaths); } + @Internal public List getIncludeFilePaths() { return ziggyCppMexObject.getIncludeFilePaths(); } - // paths for libraries that must be linked in + // Paths for libraries that must be linked in. public void setLibraryPaths(List libraryPaths) { ziggyCppMexObject.setLibraryPaths(libraryPaths); } + @Internal public List getLibraryPaths() { return ziggyCppMexObject.getLibraryPaths(); } @@ -139,18 +196,19 @@ public void setLibraries(List libraries) { ziggyCppMexObject.setLibraries(libraries); } + @Internal public List getLibraries() { return ziggyCppMexObject.getLibraries(); } // compiler options public void setCompileOptions(List compileOptions) { - ziggyCppMexObject.setCompileOptions(compileOptions); - ; + ziggyCppMexObject.setCppCompileOptions(compileOptions); } + @Internal public List getCompileOptions() { - return ziggyCppMexObject.getCompileOptions(); + return ziggyCppMexObject.getCppCompileOptions(); } // linker options @@ -158,6 +216,7 @@ public void setLinkOptions(List linkOptions) { ziggyCppMexObject.setLinkOptions(linkOptions); } + @Internal public List getLinkOptions() { return ziggyCppMexObject.getLinkOptions(); } @@ -167,6 +226,7 @@ public void setOutputType(Object outputType) { ziggyCppMexObject.setOutputType(outputType); } + @Internal public BuildType getOutputType() { return ziggyCppMexObject.getOutputType(); } @@ -176,6 +236,7 @@ public void setOutputName(Object name) { ziggyCppMexObject.setOutputName(name); } + @Internal public String getOutputName() { return ziggyCppMexObject.getOutputName(); } @@ -185,10 +246,35 @@ public void setMexfileNames(List mexfileNames) { ziggyCppMexObject.setMexfileNames(mexfileNames); } + @Internal public List getMexfileNames() { return ziggyCppMexObject.getMexfileNames(); } + // Directory to use for build product + public void setOutputDir(Object outputDir) { + ziggyCppMexObject.setOutputDir(outputDir); + } + + @Input + @Optional + public String getOutputDir() { + return ziggyCppMexObject.getOutputDir(); + } + + // Parent directory for build product (i.e., build products go in this location + "/obj", + // "/bin", or "/lib") + + public void setOutputDirParent(Object outputDirParent) { + ziggyCppMexObject.setOutputDirParent(outputDirParent); + } + + @Input + @Optional + public String getOutputDirParent() { + return ziggyCppMexObject.getOutputDirParent(); + } + /** * Sets the path to the MATLAB executable. This searches the following options in the following * order: 1. If there is a project extra property, matlabPath, use that. 2. If no matlabPath @@ -200,9 +286,8 @@ public List getMexfileNames() { public void setMatlabPath() { String matlabPath = null; Project project = getProject(); - if (project.hasProperty(ZiggyCppMexPojo.MATLAB_PATH_PROJECT_PROPERTY)) { - matlabPath = project.findProperty(ZiggyCppMexPojo.MATLAB_PATH_PROJECT_PROPERTY) - .toString(); + if (project.hasProperty(MATLAB_PATH_PROJECT_PROPERTY)) { + matlabPath = project.findProperty(MATLAB_PATH_PROJECT_PROPERTY).toString(); log.info("MATLAB path set from project extra property: " + matlabPath); } if (matlabPath == null) { @@ -220,7 +305,7 @@ public void setMatlabPath() { } } if (matlabPath == null) { - String matlabHome = System.getenv(ZiggyCppMexPojo.MATLAB_PATH_ENV_VAR); + String matlabHome = System.getenv(MATLAB_PATH_ENV_VAR); if (matlabHome != null) { matlabPath = matlabHome; log.info("MATLAB path set from MATLAB_HOME environment variable: " + matlabPath); diff --git a/buildSrc/src/main/java/gov/nasa/ziggy/buildutil/ZiggyCppMexPojo.java b/buildSrc/src/main/java/gov/nasa/ziggy/buildutil/ZiggyCppMexPojo.java index d253ad8..528327e 100644 --- a/buildSrc/src/main/java/gov/nasa/ziggy/buildutil/ZiggyCppMexPojo.java +++ b/buildSrc/src/main/java/gov/nasa/ziggy/buildutil/ZiggyCppMexPojo.java @@ -3,42 +3,39 @@ import java.io.File; import java.io.IOException; import java.util.ArrayList; -import java.util.Collection; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import org.apache.commons.exec.DefaultExecutor; +import org.apache.commons.lang3.StringUtils; +import org.gradle.api.GradleException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.gradle.api.GradleException; -import org.gradle.internal.os.OperatingSystem; /** - * Manages the construction of mexfiles from C/C++ source code. The source code is compiled - * using the C++ compiler in the CXX environment variable, with appropriate compiler options - * for use in creating object files that can be used in mexfiles. These object files are then - * combined into a shared library. Finally, the C++ compiler is used to produce mexfiles for - * each source file that contains the mexFunction entry point by linking the object files with - * the shared library and attaching an appropriate file type. - * - * Because Gradle task classes cannot easily be unit tested, the key functionality needed for - * mexfile construction is in this class; a separate class, ZiggyCppMex, extends the Gradle - * DefaultTask and provides the interface from Gradle to ZiggyCppMexPojo. - * - * @author PT + * Manages the construction of mexfiles from C/C++ source code. The source code is compiled using + * the C++ compiler in the CXX environment variable, with appropriate compiler options for use in + * creating object files that can be used in mexfiles. These object files are then combined into a + * shared library. Finally, the C++ compiler is used to produce mexfiles for each source file that + * contains the mexFunction entry point by linking the object files with the shared library and + * attaching an appropriate file type. Because Gradle task classes cannot easily be unit tested, the + * key functionality needed for mexfile construction is in this class; a separate class, + * ZiggyCppMex, extends the Gradle DefaultTask and provides the interface from Gradle to + * ZiggyCppMexPojo. * + * @author PT */ public class ZiggyCppMexPojo extends ZiggyCppPojo { private static final Logger log = LoggerFactory.getLogger(ZiggyCppMexPojo.class); - public static final String DEFAULT_COMPILE_OPTIONS_GRADLE_PROPERTY = "defaultCppMexCompileOptions"; - public static final String DEFAULT_LINK_OPTIONS_GRADLE_PROPERTY = "defaultCppMexLinkOptions"; - public static final String DEFAULT_RELEASE_OPTS_GRADLE_PROPERTY = "defaultCppMexReleaseOptimizations"; - public static final String DEFAULT_DEBUG_OPTS_GRADLE_PROPERTY = "defaultCppMexDebugOptimizations"; - public static final String MATLAB_PATH_PROJECT_PROPERTY = "matlabPath"; - public static final String MATLAB_PATH_ENV_VAR = "MATLAB_HOME"; + private static final List DEFAULT_COMPILE_OPTIONS = List.of("-Wall", "-fPIC", + "-std=c++11", "-D_GNU_SOURCE", "-fexceptions", "-fno-omit-frame-pointer", "-pthread", + "-fno-reorder-blocks", "-fstack-protector-all", "-fpermissive"); + private static final List DEFAULT_LINK_OPTIONS = List.of("-lstdc++"); + private static final List DEFAULT_RELEASE_OPTS = List.of("-O2", "-DNDEBUG", "-g"); + private static final List DEFAULT_DEBUG_OPTS = List.of("-Og", "-g"); /** Path to the MATLAB directories to be used in the build */ private String matlabPath; @@ -54,49 +51,47 @@ public class ZiggyCppMexPojo extends ZiggyCppPojo { public ZiggyCppMexPojo() { super.setOutputType(BuildType.SHARED); + setCppCompileOptions(DEFAULT_COMPILE_OPTIONS); + setLinkOptions(DEFAULT_LINK_OPTIONS); + setReleaseOptimizations(DEFAULT_RELEASE_OPTS); + setDebugOptimizations(DEFAULT_DEBUG_OPTS); } /** * Returns the correct file type for a mexfile given the OS. * - * @return string "mexmaci64" for a Mac, "mexa64" for Linux, GradleException for all other - * operating systems. + * @return string "mexa64" for Linux, "mexmaca64" for Mac M1, "mexmaci64" for Mac Intel. */ String mexSuffix() { - OperatingSystem os = getOperatingSystem(); - String mexSuffix = null; - if (os.isMacOsX()) { - mexSuffix = "mexmaci64"; - } else if (os.isLinux()) { - mexSuffix = "mexa64"; - } else { - throw new GradleException("Operating system " + os.toString() + " not supported"); - } - return mexSuffix; + return switch (getArchitecture()) { + case LINUX_INTEL -> "mexa64"; + case MAC_M1 -> "mexmaca64"; + case MAC_INTEL -> "mexmaci64"; + }; } /** * Returns the correct MATLAB architecture name given the OS. * - * @return string "maci64" for a Mac, "glnxa64" for Linux, Gradle exception for all other + * @return string "glnxa64" for Linux, "maci64" for Mac Intel, "maca64" for Mac M1. */ String matlabArch() { - OperatingSystem os = getOperatingSystem(); - String matlabArch = null; - if (os.isMacOsX()) { - matlabArch = "maci64"; - } else if (os.isLinux()) { - matlabArch = "glnxa64"; - } else { - throw new GradleException("Operating system " + os.toString() + " not supported"); - } - return matlabArch; + return switch (getArchitecture()) { + case LINUX_INTEL -> "glnxa64"; + case MAC_M1 -> "maca64"; + case MAC_INTEL -> "maci64"; + }; + } + + private File mexDir() { + return StringUtils.isEmpty(outputDir) ? new File(outputParent(), "mex") + : new File(outputDir); } /** * Generates the mexfiles that are the output of this class, and stores them in the mexfiles * list. The files that are generated are named $mexfileName.$mexfileSuffix, and are stored in - * $buildDir/lib . + * $buildDir/mex . */ void populateMexfiles() { if (getBuildDir() == null || mexfileNames == null) { @@ -105,7 +100,7 @@ void populateMexfiles() { mexfiles = new ArrayList<>(); for (String mexfileName : mexfileNames) { String fullMexfileName = mexfileName + "." + mexSuffix(); - File mexfile = new File(libDir(), fullMexfileName); + File mexfile = new File(mexDir(), fullMexfileName); mexfiles.add(mexfile); } } @@ -161,16 +156,18 @@ private File getFileByName(List files, String fileName) { * MATLAB include directory as an include path, and includes the mexfile compiler directive. */ @Override - public String generateCompileCommand(File sourceFile) { + public String generateCompileCommand(Map.Entry sourceFile) { + return generateCompileCommand(sourceFile.getKey(), sourceFile.getValue()); + } - // generate the include path + @Override + public String generateCompileCommand(File sourceFile, String compiler) { String matlabIncludePath = matlabPath + "/extern/include"; - return generateCompileCommand(sourceFile, matlabIncludePath, "DMATLAB_MEX_FILE"); + return generateCompileCommand(sourceFile, compiler, matlabIncludePath, "DMATLAB_MEX_FILE"); } public String matlabLibPath() { - String matlabLibPath = matlabPath + "/bin/" + matlabArch(); - return matlabLibPath; + return matlabPath + "/bin/" + matlabArch(); } @Override @@ -239,7 +236,7 @@ public String generateMexCommand(File mexfile, File obj) { } StringBuilder mexCommandBuilder = new StringBuilder(); - mexCommandBuilder.append(getCppCompiler() + " "); + mexCommandBuilder.append(ZiggyCppPojo.Compiler.CPP.compiler() + " "); mexCommandBuilder.append("-o " + mexfile.getAbsolutePath() + " "); mexCommandBuilder.append(obj.getAbsolutePath() + " "); mexCommandBuilder.append(argListToString(getLibraryPaths(), "-L")); @@ -309,8 +306,7 @@ public void setOutputType(Object buildType) { // Setters and getters public void setMexfileNames(List mexfileNames) { - this.mexfileNames = new ArrayList<>(); - this.mexfileNames.addAll(ZiggyCppPojo.objectListToStringList(mexfileNames)); + this.mexfileNames = ZiggyCppPojo.objectListToStringList(mexfileNames); } public List getMexfileNames() { diff --git a/buildSrc/src/main/java/gov/nasa/ziggy/buildutil/ZiggyCppPojo.java b/buildSrc/src/main/java/gov/nasa/ziggy/buildutil/ZiggyCppPojo.java index 8a238bd..436c8ee 100644 --- a/buildSrc/src/main/java/gov/nasa/ziggy/buildutil/ZiggyCppPojo.java +++ b/buildSrc/src/main/java/gov/nasa/ziggy/buildutil/ZiggyCppPojo.java @@ -4,36 +4,42 @@ import java.io.FilenameFilter; import java.io.IOException; import java.util.ArrayList; -import java.util.Collections; +import java.util.HashSet; import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; import org.apache.commons.exec.CommandLine; import org.apache.commons.exec.DefaultExecutor; +import org.apache.commons.exec.ExecuteException; import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.StringUtils; +import org.gradle.api.GradleException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.gradle.api.GradleException; -import org.gradle.internal.os.OperatingSystem; /** - * Performs compilation and linking of C++ code for Ziggy and for pipelines based on Ziggy. - * The command line options, source directory, include paths, library paths, library names, and - * type of build (executable, shared library, static library) and output file name - * must be specified in the Gradle task that makes use of this class. The compiler is determined - * from the CXX environment variable, thus is compatible with third-party packages that use the same - * convention. Actual source file names are deduced from listing the source directory, object file - * information is generated during the compile and saved for use in the link. - * - * If Gradle's JVM has cppdebug set to true as a system property, the compile and link commands will - * have appropriate options (-g and -Og). These will be placed after the compile / link options, thus - * will override any optimization options supplied to Gradle. - * - * NB: this is a POJO that makes minimal use of the Gradle API. In particular, it does not extend - * DefaultTask. This is because classes that extend DefaultTask are de facto impossible to unit test - * in Java. The ZiggyCpp class embeds a ZiggyCpp class to perform its actions and store its members. - * - * @author PT + * Performs compilation and linking of C++ code for Ziggy and for pipelines based on Ziggy. The + * command line options, source directory, include paths, library paths, library names, and type of + * build (executable, shared library, static library) and output file name must be specified in the + * Gradle task that makes use of this class. The compiler is determined from the CXX environment + * variable, thus is compatible with third-party packages that use the same convention. Actual + * source file names are deduced from listing the source directory, object file information is + * generated during the compile and saved for use in the link. If Gradle's JVM has cppdebug set to + * true as a system property, the compile and link commands will have appropriate options (-g and + * -Og). These will be placed after the compile / link options, thus will override any optimization + * options supplied to Gradle. NB: this is a POJO that makes minimal use of the Gradle API. In + * particular, it does not extend DefaultTask. This is because classes that extend DefaultTask are + * de facto impossible to unit test in Java. The ZiggyCpp class embeds a ZiggyCpp class to perform + * its actions and store its members. * + * @author PT */ public class ZiggyCppPojo { @@ -45,15 +51,43 @@ enum BuildType { SHARED, STATIC, EXECUTABLE } + enum Compiler { + C(".c", C_COMPILER), CPP(".cpp", CPP_COMPILER); + + private String fileSuffix; + private String compiler; + + Compiler(String fileSuffix, String compiler) { + this.fileSuffix = fileSuffix; + this.compiler = compiler; + } + + public String fileSuffix() { + return fileSuffix; + } + + public String compiler() { + return compiler; + } + } + public static final String CPP_COMPILER_ENV_VAR = "CXX"; - public static final String[] CPP_FILE_TYPES = { ".c", ".cpp" }; + public static final String C_COMPILER_ENV_VAR = "CC"; public static final String CPP_DEBUG_PROPERTY_NAME = "cppdebug"; - public static final String DEFAULT_COMPILE_OPTIONS_GRADLE_PROPERTY = "defaultCppCompileOptions"; - public static final String DEFAULT_LINK_OPTIONS_GRADLE_PROPERTY = "defaultCppLinkOptions"; - public static final String DEFAULT_RELEASE_OPTS_GRADLE_PROPERTY = "defaultCppReleaseOptimizations"; - public static final String DEFAULT_DEBUG_OPTS_GRADLE_PROPERTY = "defaultCppDebugOptimizations"; - public static final String PIPELINE_ROOT_DIR_PROP_NAME = "pipelineRootDir"; + // The compiler Strings cannot be final because they need to be overridden + // during test. + public static String CPP_COMPILER = System.getenv(CPP_COMPILER_ENV_VAR); + public static String C_COMPILER = System.getenv(C_COMPILER_ENV_VAR); + + private static final List DEFAULT_CPP_COMPILE_OPTIONS = List.of("-Wall", "-fPIC", + "-std=c++11"); + private static final List DEFAULT_C_COMPILE_OPTIONS = List.of("-fPIC"); + private static final List DEFAULT_LINK_OPTIONS = List.of(); + private static final List DEFAULT_RELEASE_OPTS = List.of("-O2", "-DNDEBUG", "-g"); + private static final List DEFAULT_DEBUG_OPTS = List.of("-Og", "-g"); + private static final int DEFAULT_PARALLEL_COMPILE_THREADS = Runtime.getRuntime() + .availableProcessors(); /** Path to the C++ files to be compiled */ private List cppFilePaths = null; @@ -68,16 +102,19 @@ enum BuildType { private List libraries = new ArrayList<>(); /** compile options (minus the initial hyphen) */ - private List compileOptions = new ArrayList<>(); + private List cppCompileOptions = DEFAULT_CPP_COMPILE_OPTIONS; + + /** compile options (minus the initial hyphen) */ + private List cCompileOptions = DEFAULT_C_COMPILE_OPTIONS; /** linker options (minus the initial hyphen) */ - private List linkOptions = new ArrayList<>(); + private List linkOptions = DEFAULT_LINK_OPTIONS; /** Optimizations, if any, desired for a build without cppdebug=true system property */ - private List releaseOptimizations = new ArrayList<>(); + private List releaseOptimizations = DEFAULT_RELEASE_OPTS; /** Optimizations, if any, desired for a build with cppdebug=true system property */ - private List debugOptimizations = new ArrayList<>(); + private List debugOptimizations = DEFAULT_DEBUG_OPTS; /** Caller-selected build type */ private BuildType outputType = null; @@ -85,8 +122,14 @@ enum BuildType { /** Name of the output file (with no "lib" prefix or file type suffix) */ private String name = null; + /** Name of the directory for output, if not specified an appropriate default is used. */ + protected String outputDir; + + /** Parent of the directory for output, if not specified buildDir is used. */ + protected String outputDirParent; + /** C++ files found in the cppFilePath directory */ - private List cppFiles = new ArrayList<>(); + private Map cppFiles = new TreeMap<>(); /** Object files built from the C++ files */ private List objectFiles = new ArrayList<>(); @@ -95,19 +138,19 @@ enum BuildType { private File builtFile = null; /** Desired Gradle build directory, as a File */ - private File buildDir = null; + protected File buildDir = null; /** Root directory for the parent Gradle project, as a File */ private File rootDir = null; - /** C++ compiler command including path to same */ - private String cppCompiler = null; - /** Default executor used only for testing, do not use for real execution! */ private DefaultExecutor defaultExecutor = null; /** Operating system, needed to set options and names for the linker command */ - private OperatingSystem operatingSystem = OperatingSystem.current(); + private SystemArchitecture architecture = SystemArchitecture.architecture(); + + /** Number of parallel compiler processes to accept. */ + private int maxCompileThreads = DEFAULT_PARALLEL_COMPILE_THREADS; // stores logger warning messages. Used only for testing. private List loggerWarnings = new ArrayList<>(); @@ -116,35 +159,44 @@ enum BuildType { * Converts a list of arguments to a single string that can be used in a command line compiler * call * - * @param argList list of arguments + * @param argList list of arguments, may contain null or empty items, which are skipped * @param prefix prefix for each argument ("-I", "-L", etc.) * @return the list of arguments converted to a string, and with the prefix added to each */ public String argListToString(List argList, String prefix) { StringBuilder argStringBuilder = new StringBuilder(); for (String arg : argList) { - argStringBuilder.append(prefix + arg + " "); + if (arg != null && !arg.isEmpty()) { + argStringBuilder.append(prefix + arg + " "); + } } return argStringBuilder.toString(); } File objDir() { - return new File(buildDir, "obj"); + return StringUtils.isEmpty(outputDir) ? new File(outputParent(), "obj") + : new File(outputDir); } File libDir() { - return new File(buildDir, "lib"); + return StringUtils.isEmpty(outputDir) ? new File(outputParent(), "lib") + : new File(outputDir); } File binDir() { - return new File(buildDir, "bin"); + return StringUtils.isEmpty(outputDir) ? new File(outputParent(), "bin") + : new File(outputDir); + } + + File outputParent() { + return StringUtils.isEmpty(outputDirParent) ? buildDir : new File(outputDirParent); } /** * Search the specified file path for C and C++ files, and populate the cppFiles list with same. * If the file path is not set or does not exist, a GradleException will be thrown. */ - private void populateCppFiles() { + private void populateCppFiles(boolean warn) { // check that the path is set and exists if (cppFilePaths == null) { @@ -154,37 +206,37 @@ private void populateCppFiles() { // clear any existing files, and also handle the null pointer case // neither of these should ever occur in real life, but why risk it? if (cppFiles == null || !cppFiles.isEmpty()) { - cppFiles = new ArrayList<>(); + cppFiles = new TreeMap<>(); } for (String cppFilePath : cppFilePaths) { File cppFileDir = new File(cppFilePath); if (!cppFileDir.exists()) { - String w = "C++ file path " + cppFilePath + " does not exist"; - log.warn(w); - addLoggerWarning(w); - + if (warn) { + String w = "C++ file path " + cppFilePath + " does not exist"; + log.warn(w); + addLoggerWarning(w); + } } else { // find all C and C++ files and add them to the cppFiles list - for (String fileType : CPP_FILE_TYPES) { - File[] cFiles = cppFileDir.listFiles(new FilenameFilter() { - public boolean accept(File dir, String name) { - return name.endsWith(fileType); - } - }); - for (File file : cFiles) { - cppFiles.add(file); + for (Compiler compiler : Compiler.values()) { + File[] cFiles = cppFileDir.listFiles( + (FilenameFilter) (dir, name) -> name.endsWith(compiler.fileSuffix())); + if (cFiles == null) { + continue; + } + for (File cFile : cFiles) { + cppFiles.put(cFile, compiler.compiler()); } } } } if (!cppFiles.isEmpty()) { - Collections.sort(cppFiles); // write the list of files to the log if info logging is set StringBuilder fileListBuilder = new StringBuilder(); - for (File file : cppFiles) { + for (File file : cppFiles.keySet()) { fileListBuilder.append(file.getName()); fileListBuilder.append(" "); } @@ -218,20 +270,18 @@ protected void populateBuiltFile() { prefix = "lib"; if (outputType == BuildType.STATIC) { fileType = ".a"; + } else if (architecture == SystemArchitecture.MAC_M1 + || architecture == SystemArchitecture.MAC_INTEL) { + fileType = ".dylib"; + } else if (architecture == SystemArchitecture.LINUX_INTEL) { + fileType = ".so"; } else { - if (operatingSystem.isMacOsX()) { - fileType = ".dylib"; - } else if (operatingSystem.isLinux()) { - fileType = ".so"; - } else { - throw new GradleException( - "ZiggyCpp class does not support OS " + operatingSystem.getName()); - } + throw new GradleException( + "ZiggyCpp class does not support OS " + architecture.toString()); } } String outputFile = prefix + name + fileType; builtFile = new File(outputDirectory, outputFile); - } /** @@ -243,9 +293,10 @@ protected void populateBuiltFile() { public static String objectNameFromSourceFile(File sourceFile) { String sourceName = sourceFile.getName(); String strippedName = null; - for (String fileType : CPP_FILE_TYPES) { - if (sourceName.endsWith(fileType)) { - strippedName = sourceName.substring(0, sourceName.length() - fileType.length()); + for (Compiler compiler : Compiler.values()) { + if (sourceName.endsWith(compiler.fileSuffix())) { + strippedName = sourceName.substring(0, + sourceName.length() - compiler.fileSuffix().length()); break; } } @@ -254,38 +305,36 @@ public static String objectNameFromSourceFile(File sourceFile) { /** * Generates the command to compile a single source file - * - * @param sourceFile File of the C/C++ source that is to be compiled - * @return the compile command as a single string. This command will include the include files - * and command line options specified in the object, and will route the output to the correct - * output directory (specifically $buildDir/obj). It will also take care of setting options - * correctly for a debug build if the JVM has cppdebug=true set as a system property. */ - public String generateCompileCommand(File sourceFile) { + public String generateCompileCommand(Map.Entry sourceFile) { return generateCompileCommand(sourceFile, null, null); } /** * Generates the command to compile a single source file, with additional options that are - * needed for mexfiles - * - * @param sourceFile File of the C/C++ source that is to be compiled - * @param matlabIncludePath String that indicates the location of MATLAB include files, can be - * null - * @param matlabCompilerDirective String that contains the MATLAB compiler directive, can be - * null - * @return the compile command as a single string. This command will include the include files - * and command line options specified in the object, and will route the output to the correct - * output directory (specifically $buildDir/obj). It will also take care of setting options - * correctly for a debug build if the JVM has cppdebug=true set as a system property. + * needed for mexfiles. */ - public String generateCompileCommand(File sourceFile, String matlabIncludePath, + public String generateCompileCommand(Map.Entry sourceFile, + String matlabIncludePath, String matlabCompilerDirective) { + return generateCompileCommand(sourceFile.getKey(), sourceFile.getValue(), matlabIncludePath, + matlabCompilerDirective); + } + + public String generateCompileCommand(File sourceFile, String compiler) { + return generateCompileCommand(sourceFile, compiler, null, null); + } + + /** + * Generates the command to compile a single source file, with additional options that are + * needed for mexfiles. + */ + public String generateCompileCommand(File sourceFile, String compiler, String matlabIncludePath, String matlabCompilerDirective) { StringBuilder compileStringBuilder = new StringBuilder(); // compiler executable - compileStringBuilder.append(getCppCompiler() + " "); + compileStringBuilder.append(compiler + " "); // compile only flag compileStringBuilder.append("-c "); @@ -303,7 +352,11 @@ public String generateCompileCommand(File sourceFile, String matlabIncludePath, } // add the command line options - compileStringBuilder.append(argListToString(compileOptions, "-")); + if (compiler.equals(CPP_COMPILER)) { + compileStringBuilder.append(argListToString(cppCompileOptions, "")); + } else { + compileStringBuilder.append(argListToString(cCompileOptions, "")); + } // if there is a MATLAB compiler directive, handle that now if (matlabCompilerDirective != null && !matlabCompilerDirective.isEmpty()) { @@ -317,9 +370,9 @@ public String generateCompileCommand(File sourceFile, String matlabIncludePath, debug = Boolean.getBoolean(CPP_DEBUG_PROPERTY_NAME); } if (debug) { - compileStringBuilder.append(argListToString(debugOptimizations, "-")); + compileStringBuilder.append(argListToString(debugOptimizations, "")); } else { - compileStringBuilder.append(argListToString(releaseOptimizations, "-")); + compileStringBuilder.append(argListToString(releaseOptimizations, "")); } compileStringBuilder.append(sourceFile.getAbsolutePath()); @@ -327,7 +380,6 @@ public String generateCompileCommand(File sourceFile, String matlabIncludePath, log.info(compileStringBuilder.toString()); return compileStringBuilder.toString(); - } /** @@ -355,7 +407,7 @@ public String generateLinkCommand(String matlabLibPath) { if (outputType == BuildType.STATIC) { linkStringBuilder.append("ar rs "); } else { - linkStringBuilder.append(getCppCompiler() + " -o "); + linkStringBuilder.append(Compiler.CPP.compiler() + " -o "); } // add the name of the desired output file @@ -368,17 +420,16 @@ public String generateLinkCommand(String matlabLibPath) { if (matlabLibPath != null && !matlabLibPath.isEmpty()) { linkStringBuilder.append("-L" + matlabLibPath + " "); } - } // add release or debug options if (outputType == BuildType.EXECUTABLE) { - linkStringBuilder.append(argListToString(linkOptions, "-")); + linkStringBuilder.append(argListToString(linkOptions, "")); if (System.getProperty(CPP_DEBUG_PROPERTY_NAME) != null && Boolean.getBoolean(CPP_DEBUG_PROPERTY_NAME)) { - linkStringBuilder.append(argListToString(debugOptimizations, "-")); + linkStringBuilder.append(argListToString(debugOptimizations, "")); } else { - linkStringBuilder.append(argListToString(releaseOptimizations, "-")); + linkStringBuilder.append(argListToString(releaseOptimizations, "")); } } @@ -391,13 +442,15 @@ public String generateLinkCommand(String matlabLibPath) { // if the OS is Mac OS, set the install name. The install name assumes that the library // will be installed in the build/lib directory under the root directory. - if (operatingSystem.isMacOsX() && outputType == BuildType.SHARED) { + if ((architecture == SystemArchitecture.MAC_M1 + || architecture == SystemArchitecture.MAC_INTEL) && outputType == BuildType.SHARED) { linkStringBuilder.append("-install_name " + getRootDir().getAbsolutePath() + "/build/lib/" + getBuiltFile().getName() + " "); } - // add the object files - for (File objectFile : objectFiles) { + // add the object files, sorted into alphabetical order (to simplify testing). + Set sortedObjectFiles = new TreeSet<>(objectFiles); + for (File objectFile : sortedObjectFiles) { linkStringBuilder.append(objectFile.getName() + " "); } @@ -412,7 +465,6 @@ public String generateLinkCommand(String matlabLibPath) { log.info(linkStringBuilder.toString()); return linkStringBuilder.toString(); - } /** @@ -456,25 +508,39 @@ protected void compileAction() { log.info("mkdir: " + objDir.getAbsolutePath()); objDir.mkdirs(); } + + // Create a thread pool for compilation. + ExecutorService compilerThreadPool = Executors.newFixedThreadPool(maxCompileThreads); + Set> compilationResults = new HashSet<>(); + // loop over source files, compile them and add the object file to the object file list - for (File file : getCppFiles()) { - DefaultExecutor compilerExec = getDefaultExecutor(); - compilerExec.setWorkingDirectory(new File(cppFilePaths.get(0))); - try { - int returnCode = compilerExec - .execute(new CommandLineComparable(generateCompileCommand(file))); + for (Map.Entry file : getCppFiles(true).entrySet()) { + compilationResults.add(compilerThreadPool.submit(() -> compileActionInternal(file))); + } - if (returnCode != 0) { - throw new GradleException("Compilation of file " + file.getName() + " failed"); + for (Future futureResult : compilationResults) { + try { + CompilationResult result = futureResult.get(); + if (result.getReturnCode() != 0) { + throw new GradleException( + "Compilation of file " + result.getSourceFile().getName() + " failed"); } - objectFiles.add(new File(objDir, objectNameFromSourceFile(file))); - } catch (IOException e) { - throw new GradleException( - "IOException occurred when attempting to compile " + file.getName(), e); + objectFiles.add(new File(objDir, objectNameFromSourceFile(result.getSourceFile()))); + } catch (ExecutionException | InterruptedException e) { + throw new GradleException("Exception occurred during compilation", e); } } } + private CompilationResult compileActionInternal(Map.Entry sourceFile) + throws ExecuteException, IOException { + DefaultExecutor compilerExec = getDefaultExecutor(); + compilerExec.setWorkingDirectory(new File(cppFilePaths.get(0))); + int returnCode = compilerExec + .execute(new CommandLineComparable(generateCompileCommand(sourceFile))); + return new CompilationResult(sourceFile.getKey(), returnCode); + } + protected void linkAction() { File objDir = objDir(); @@ -491,7 +557,8 @@ protected void linkAction() { destDir.mkdirs(); } try { - int returnCode = linkExec.execute(new CommandLineComparable(generateLinkCommand())); + String linkCommand = generateLinkCommand(); + int returnCode = linkExec.execute(new CommandLineComparable(linkCommand)); if (returnCode != 0) { throw new GradleException( "Link / library construction of " + getBuiltFile().getName() + " failed"); @@ -504,34 +571,48 @@ protected void linkAction() { // copy the files from each of the include directories to buildDir/include File includeDest = new File(buildDir, "include"); for (String include : includeFilePaths) { - File[] includeFiles = new File(include).listFiles(new FilenameFilter() { - @Override - public boolean accept(File dir, String name) { - return (name.endsWith(".h") || name.endsWith(".hpp")); + File directory = new File(include); + if (!directory.exists()) { + // Strip rootDir prefix from the entry for easier reading (in most cases). + String shortInclude = include; + int index = include.indexOf(getRootDir().getAbsolutePath()); + if (index >= 0) { + // +1: Strip the / after rootDir as well. + shortInclude = include.substring(getRootDir().getAbsolutePath().length() + 1); } - }); + throw new GradleException("The directory " + shortInclude + + " specified in includeFilePaths does not exist"); + } + File[] includeFiles = directory.listFiles( + (FilenameFilter) (dir, name) -> name.endsWith(".h") || name.endsWith(".hpp")); for (File includeFile : includeFiles) { try { - FileUtils.copyFileToDirectory(includeFile, includeDest); + if (!includeFile.getParentFile() + .getCanonicalFile() + .equals(includeDest.getCanonicalFile())) { + FileUtils.copyFileToDirectory(includeFile, includeDest); + } } catch (IOException e) { throw new GradleException("Unable to copy include files from" + include + " to " + includeDest.getAbsoluteFile(), e); } } } - } /** - * Converts a list of Objects to a list of Strings, preserving their order. - * - * @param libraries2 List of objects to be converted. - * @return ArrayList of strings obtained by taking toString() of the objects in the objectList. + * Converts a list of Objects to a list of Strings, preserving their order. Null objects, empty + * strings, or strings consisting solely of whitespace are ignored. */ - static List objectListToStringList(List libraries2) { + static List objectListToStringList(List objectList) { List stringList = new ArrayList<>(); - for (Object obj : libraries2) { - stringList.add(obj.toString()); + for (Object object : objectList) { + if (object != null) { + String string = object.toString(); + if (!string.isBlank()) { + stringList.add(string); + } + } } return stringList; } @@ -547,18 +628,15 @@ static List objectListToStringList(List libraries2) { static List gradlePropertyToList(Object gradleProperty) { if (gradleProperty instanceof List) { return objectListToStringList((List) gradleProperty); - } else { - List gradlePropertyList = new ArrayList<>(); - gradlePropertyList.add(gradleProperty); - return objectListToStringList((List) gradlePropertyList); } + + return objectListToStringList(List.of(gradleProperty)); } -// setters and getters +// setters and getters public void setCppFilePath(Object cppFilePath) { - this.cppFilePaths = new ArrayList<>(); - cppFilePaths.add(cppFilePath.toString()); + cppFilePaths = List.of(cppFilePath.toString()); } public List getCppFilePaths() { @@ -574,8 +652,7 @@ public List getIncludeFilePaths() { } public void setIncludeFilePaths(List includeFilePaths) { - this.includeFilePaths = new ArrayList<>(); - this.includeFilePaths.addAll(objectListToStringList(includeFilePaths)); + this.includeFilePaths = objectListToStringList(includeFilePaths); } public List getLibraryPaths() { @@ -583,8 +660,7 @@ public List getLibraryPaths() { } public void setLibraryPaths(List libraryPaths) { - this.libraryPaths = new ArrayList<>(); - this.libraryPaths.addAll(objectListToStringList(libraryPaths)); + this.libraryPaths = objectListToStringList(libraryPaths); } public List getLibraries() { @@ -592,17 +668,23 @@ public List getLibraries() { } public void setLibraries(List libraries) { - this.libraries = new ArrayList<>(); - this.libraries.addAll(objectListToStringList(libraries)); + this.libraries = objectListToStringList(libraries); + } + + public List getCppCompileOptions() { + return cppCompileOptions; + } + + public void setCppCompileOptions(List compileOptions) { + cppCompileOptions = objectListToStringList(compileOptions); } - public List getCompileOptions() { - return compileOptions; + public List getCCompileOptions() { + return cCompileOptions; } - public void setCompileOptions(List compileOptions) { - this.compileOptions = new ArrayList<>(); - this.compileOptions.addAll(objectListToStringList(compileOptions)); + public void setCCompileOptions(List cCompileOptions) { + this.cCompileOptions = cCompileOptions; } public List getLinkOptions() { @@ -610,8 +692,7 @@ public List getLinkOptions() { } public void setLinkOptions(List linkOptions) { - this.linkOptions = new ArrayList<>(); - this.linkOptions.addAll(objectListToStringList(linkOptions)); + this.linkOptions = objectListToStringList(linkOptions); } public List getReleaseOptimizations() { @@ -619,8 +700,7 @@ public List getReleaseOptimizations() { } public void setReleaseOptimizations(List releaseOptimizations) { - this.releaseOptimizations = new ArrayList<>(); - this.releaseOptimizations.addAll(objectListToStringList(releaseOptimizations)); + this.releaseOptimizations = objectListToStringList(releaseOptimizations); } public List getDebugOptimizations() { @@ -628,8 +708,7 @@ public List getDebugOptimizations() { } public void setDebugOptimizations(List debugOptimizations) { - this.debugOptimizations = new ArrayList<>(); - this.debugOptimizations.addAll(objectListToStringList(debugOptimizations)); + this.debugOptimizations = objectListToStringList(debugOptimizations); } public BuildType getOutputType() { @@ -652,13 +731,21 @@ public void setOutputName(Object name) { this.name = name.toString(); } - public List getCppFiles() { - // always generate the list afresh -- necessary because Gradle calls the ZiggyCpp + public Map getCppFiles() { + return getCppFiles(false); + } + + public List getSourceFiles() { + return new ArrayList<>(getCppFiles().keySet()); + } + + public Map getCppFiles(boolean warn) { + // Always generate the list afresh -- necessary because Gradle calls the ZiggyCpp // method getCppFiles() prior to the actual build, at which time the directories of // source files may or may not exist yet! Thus we can't afford to cache the C++ // file list, since I can't tell whether Gradle creates a new ZiggyCpp object when // it actually does the build, or whether it simply re-uses the one from pre-build. - populateCppFiles(); + populateCppFiles(warn); return cppFiles; } @@ -671,13 +758,11 @@ public void setObjectFiles(List objectFiles) { } public void setObjectFiles(File objectFile) { - this.objectFiles.add(objectFile); + objectFiles.add(objectFile); } public File getBuiltFile() { - if (builtFile == null) { - populateBuiltFile(); - } + populateBuiltFile(); return builtFile; } @@ -697,20 +782,32 @@ public void setRootDir(File rootDir) { this.rootDir = rootDir; } - public String getCppCompiler() { - if (cppCompiler == null) { - cppCompiler = System.getenv(CPP_COMPILER_ENV_VAR); - } - return cppCompiler; + public SystemArchitecture getArchitecture() { + return architecture; } - public OperatingSystem getOperatingSystem() { - return operatingSystem; + public String getOutputDir() { + return outputDir; } - // this method is intended for use only in testing, for that reason it is package-private - void setCppCompiler(String cppCompiler) { - this.cppCompiler = cppCompiler; + public void setOutputDir(Object outputDir) { + this.outputDir = outputDir.toString(); + } + + public String getOutputDirParent() { + return outputDirParent; + } + + public void setOutputDirParent(Object outputDirParent) { + this.outputDirParent = outputDirParent.toString(); + } + + public int getMaxCompileThreads() { + return maxCompileThreads; + } + + public void setMaxCompileThreads(int maxCompileThreads) { + this.maxCompileThreads = maxCompileThreads; } // this method is intended for use only in testing, for that reason it is package-private @@ -719,8 +816,18 @@ void setDefaultExecutor(DefaultExecutor defaultExecutor) { } // this method is intended for use only in testing, for that reason it is package-private - void setOperatingSystem(OperatingSystem operatingSystem) { - this.operatingSystem = operatingSystem; + void setArchitecture(SystemArchitecture architecture) { + this.architecture = architecture; + } + + /** For testing use only. */ + static void setCppCompiler(String testCompiler) { + CPP_COMPILER = testCompiler; + } + + /** For testing use only. */ + static void setCCompiler(String testCompiler) { + C_COMPILER = testCompiler; } /** @@ -736,10 +843,11 @@ public CommandLineComparable(String executable) { super(CommandLine.parse(executable)); } + @Override public boolean equals(Object o) { if (o instanceof CommandLine) { CommandLine oo = (CommandLine) o; - if (this.toString().contentEquals(oo.toString())) { + if (toString().contentEquals(oo.toString())) { return true; } } @@ -756,4 +864,23 @@ private void addLoggerWarning(String warning) { List loggerWarnings() { return loggerWarnings; } + + private static class CompilationResult { + + private final File sourceFile; + private final int returnCode; + + public CompilationResult(File sourceFile, int returnCode) { + this.sourceFile = sourceFile; + this.returnCode = returnCode; + } + + public File getSourceFile() { + return sourceFile; + } + + public int getReturnCode() { + return returnCode; + } + } } diff --git a/buildSrc/src/main/java/gov/nasa/ziggy/buildutil/ZiggyVersionGenerator.java b/buildSrc/src/main/java/gov/nasa/ziggy/buildutil/ZiggyVersionGenerator.java index c9e8b4e..fd41c50 100644 --- a/buildSrc/src/main/java/gov/nasa/ziggy/buildutil/ZiggyVersionGenerator.java +++ b/buildSrc/src/main/java/gov/nasa/ziggy/buildutil/ZiggyVersionGenerator.java @@ -6,82 +6,106 @@ import java.io.FileWriter; import java.io.IOException; import java.io.InputStreamReader; -import java.net.URISyntaxException; -import java.text.SimpleDateFormat; import java.util.ArrayList; -import java.util.Date; import java.util.List; +import java.util.stream.Collectors; -import org.gradle.api.tasks.OutputFile; +import org.gradle.api.DefaultTask; +import org.gradle.api.tasks.Input; import org.gradle.api.tasks.TaskAction; -import org.gradle.api.tasks.Internal; import com.google.common.collect.ImmutableList; -import freemarker.template.Configuration; -import freemarker.template.Template; -import freemarker.template.TemplateException; -import freemarker.template.TemplateExceptionHandler; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - /** - * Generates version info in the generated file ZiggyVersion.java + *
{@code versionPropertyName}
+ *
name of the property that holds the output of {@code git describe} (default: + * {@value #DEFAULT_BUILD_VERSION_PROPERTY_NAME})
+ *
{@code branchPropertyName}
+ *
name of the property that contains the name of the current branch + * (default:{@value #DEFAULT_BUILD_BRANCH_PROPERTY_NAME})
+ *
{@code commitPropertyName}
+ *
name of the property that contains the commit + * (default:{@value #DEFAULT_BUILD_COMMIT_PROPERTY_NAME})
+ * + *

+ * The following example shows how to to run this plugin only when necessary by saving the current + * Git version in a property called {@code ziggyVersion}. The default values of the aforementioned + * properties are used. + * + *

+ * import gov.nasa.ziggy.buildutil.ZiggyVersionGenerator
+ *
+ * def gitVersion = new ByteArrayOutputStream()
+ * exec {
+ *     commandLine "git", "rev-parse", "HEAD"
+ *     standardOutput = gitVersion
+ * }
+ * gitVersion = gitVersion.toString().trim()
  *
- * 
$ git rev-list --all --abbrev=0 --abbrev-commit | awk '{print length()}' | sort -n | uniq -c
- * 1040 4
- * 7000 5
- * 1149 6
- *   68 7
- *    8 8
- *    1 9
+ * task ziggyVersion(type: ZiggyVersionGenerator) {
+ *     inputs.property "ziggyVersion", gitVersion
+ * }
  * 
* + * See ZiggyConfiguration. */ -public class ZiggyVersionGenerator extends TessExecTask { - - private static final Logger log = LoggerFactory.getLogger(ZiggyVersionGenerator.class); - private static final String MAC_OS_X_OS_NAME = "Mac OS X"; - - public File outputFile; - public final String dateFormat = "dd-MMM-yyyy HH:mm:ss"; - private String osType; - - public void generateFile(BufferedWriter out) throws IOException, InterruptedException { - - osType = System.getProperty("os.name"); - log.debug("OS Type: " + osType); - - // Suppressed because this entire class is going to be removed in Ziggy 0.4.0. - @SuppressWarnings(value = "deprecated") - Configuration config = new Configuration(); - config.setClassForTemplateLoading(this.getClass(), "/"); - config.setDefaultEncoding("UTF-8"); - config.setTemplateExceptionHandler(TemplateExceptionHandler.RETHROW_HANDLER); - - VersionInfo versionInfo = new VersionInfo(); - versionInfo.setBuildDate(getBuildDate()); - versionInfo.setSoftwareVersion(getGitRelease()); - versionInfo.setBranch(getGitBranch()); - versionInfo.setRevision(getGitRevision()); - - try { - config.getTemplate("ZiggyVersion.java.ftlh").process(versionInfo, out); - } catch (TemplateException e) { - throw new IllegalStateException("Error processing template", e); +public class ZiggyVersionGenerator extends DefaultTask { + + private static final String BUILD_CONFIGURATION = "src/main/resources/ziggy-build.properties"; + private static final String DEFAULT_BUILD_VERSION_PROPERTY_NAME = "ziggy.version"; + private static final String DEFAULT_BUILD_BRANCH_PROPERTY_NAME = "ziggy.version.branch"; + private static final String DEFAULT_BUILD_COMMIT_PROPERTY_NAME = "ziggy.version.commit"; + + /** + * Length that object names are abbreviated to. The number is set to 10, because that is + * currently 1 more than necessary to distinguish all commit hashes to date. One command to + * determine the maximum hash length in use is: + * + *
+     * $ git rev-list --all --abbrev=0 --abbrev-commit | awk '{print length()}' | sort -n | uniq -c
+     * 1040 4
+     * 7000 5
+     * 1149 6
+     *   68 7
+     *    8 8
+     *    1 9
+     * 
+ */ + private static final int ABBREV = 10; + + private static final String HEADER = "# This file is automatically generated by Gradle." + + System.lineSeparator() + "# Do not edit." + System.lineSeparator(); + + private String versionPropertyName = DEFAULT_BUILD_VERSION_PROPERTY_NAME; + private String branchPropertyName = DEFAULT_BUILD_BRANCH_PROPERTY_NAME; + private String commitPropertyName = DEFAULT_BUILD_COMMIT_PROPERTY_NAME; + + @TaskAction + public void generateVersionProperties() throws IOException, InterruptedException { + + File outputFile = new File(getProject().getProjectDir(), buildConfiguration()); + + try (BufferedWriter output = new BufferedWriter(new FileWriter(outputFile))) { + output.write(HEADER); + output.write(getVersionPropertyName() + " = " + + runCommand(ImmutableList.of("git", "describe", "--always", "--abbrev=" + ABBREV)) + + System.lineSeparator()); + output.write(getBranchPropertyName() + " = " + + runCommand(ImmutableList.of("git", "rev-parse", "--abbrev-ref", "HEAD")) + + System.lineSeparator()); + output.write(getCommitPropertyName() + " = " + + runCommand(ImmutableList.of("git", "rev-parse", "--short=" + ABBREV, "HEAD")) + + System.lineSeparator()); } } - public List processOutput(List command) - throws IOException, InterruptedException { + public String runCommand(List command) throws IOException, InterruptedException { - ProcessBuilder processBuilder = new ProcessBuilder(command); - Process process = processBuilder.start(); - List lines = new ArrayList<>(); + Process process = new ProcessBuilder(command).start(); + List output = new ArrayList<>(); BufferedReader bufferedReader = new BufferedReader( new InputStreamReader(process.getInputStream())); @@ -91,106 +115,43 @@ public List processOutput(List command) if (line == null) { break; } - - lines.add(line); + output.add(line); } process.waitFor(); - return lines; - } - @Internal - public String getGitRevision() throws IOException, InterruptedException { - if (osType.equals(MAC_OS_X_OS_NAME)) { - return "Not Supported"; - } - List cmd = ImmutableList.of("git", "rev-parse", "--short=10", "HEAD"); - List output = processOutput(cmd); - return output.get(output.size() - 1); + return output.stream().collect(Collectors.joining(System.lineSeparator())).trim(); } - @Internal - public String getGitBranch() throws IOException, InterruptedException { - if (osType.equals(MAC_OS_X_OS_NAME)) { - return "Not Supported"; - } - List cmd = ImmutableList.of("git", "rev-parse", "--abbrev-ref", "HEAD"); - List output = processOutput(cmd); - return output.get(output.size() - 1); + /** Override this to create your own subclass for pipeline-side version generation. */ + protected String buildConfiguration() { + return BUILD_CONFIGURATION; } - @Internal - public String getGitRelease() throws IOException, InterruptedException { - if (osType.equals(MAC_OS_X_OS_NAME)) { - return "Not Supported"; - } - List cmd = ImmutableList.of("git", "describe", "--always", "--abbrev=10"); - List output = processOutput(cmd); - return output.get(output.size() - 1); + @Input + public String getVersionPropertyName() { + return versionPropertyName; } - @Internal - public String getBuildDate() { - SimpleDateFormat dateFormatter = new SimpleDateFormat(dateFormat); - return dateFormatter.format(new Date()); + public void setVersionPropertyName(String versionPropertyName) { + this.versionPropertyName = versionPropertyName; } - @OutputFile - public File getOutputFile() { - return outputFile; + @Input + public String getBranchPropertyName() { + return branchPropertyName; } - public void setOutputFile(File output) { - outputFile = output; + public void setBranchPropertyName(String branchPropertyName) { + this.branchPropertyName = branchPropertyName; } - @TaskAction - public void action() throws IOException, InterruptedException { - try (BufferedWriter output = new BufferedWriter(new FileWriter(outputFile))) { - generateFile(output); - } + @Input + public String getCommitPropertyName() { + return commitPropertyName; } - /** - * Holds version information in a Java bean suitable for referencing from a template. - */ - public static class VersionInfo { - - private String buildDate; - private String softwareVersion; - private String revision; - private String branch; - - public String getBuildDate() { - return buildDate; - } - - public void setBuildDate(String dateStr) { - buildDate = dateStr; - } - - public String getSoftwareVersion() { - return softwareVersion; - } - - public void setSoftwareVersion(String versionStr) { - softwareVersion = versionStr; - } - - public String getRevision() { - return revision; - } - - public void setRevision(String revision) { - this.revision = revision; - } - - public String getBranch() { - return branch; - } - - public void setBranch(String branch) { - this.branch = branch; - } + public void setCommitPropertyName(String commitPropertyName) { + this.commitPropertyName = commitPropertyName; } } diff --git a/buildSrc/src/test/java/gov/nasa/ziggy/buildutil/SystemArchitectureTest.java b/buildSrc/src/test/java/gov/nasa/ziggy/buildutil/SystemArchitectureTest.java new file mode 100644 index 0000000..3af7dcf --- /dev/null +++ b/buildSrc/src/test/java/gov/nasa/ziggy/buildutil/SystemArchitectureTest.java @@ -0,0 +1,38 @@ +package gov.nasa.ziggy.buildutil; + +import static org.junit.Assert.assertEquals; + +import org.junit.Test; +import org.mockito.Mockito; + +import gov.nasa.ziggy.buildutil.SystemArchitecture.Selector; + +/** + * Unit tests for {@link SystemArchitecture} class. + * + * @author PT + */ +public class SystemArchitectureTest { + + @Test + public void testSelector() { + + Selector selectorSpy = Mockito.spy(SystemArchitecture.selector(String.class)); + Mockito.doReturn(SystemArchitecture.LINUX_INTEL).when(selectorSpy).architecture(); + selectorSpy.linuxIntelObject("A").macIntelObject("B").macM1Object("C"); + assertEquals("A", selectorSpy.get()); + Mockito.doReturn(SystemArchitecture.MAC_INTEL).when(selectorSpy).architecture(); + assertEquals("B", selectorSpy.get()); + Mockito.doReturn(SystemArchitecture.MAC_M1).when(selectorSpy).architecture(); + assertEquals("C", selectorSpy.get()); + } + + @Test + public void testFastSelectorSyntax() { + SystemArchitecture.selector(String.class) + .linuxIntelObject("A") + .macIntelObject("B") + .macM1Object("C") + .get(); + } +} diff --git a/buildSrc/src/test/java/gov/nasa/ziggy/buildutil/ZiggyCppMexPojoTest.java b/buildSrc/src/test/java/gov/nasa/ziggy/buildutil/ZiggyCppMexPojoTest.java index b4e8e5c..a656fb6 100644 --- a/buildSrc/src/test/java/gov/nasa/ziggy/buildutil/ZiggyCppMexPojoTest.java +++ b/buildSrc/src/test/java/gov/nasa/ziggy/buildutil/ZiggyCppMexPojoTest.java @@ -7,6 +7,7 @@ import java.io.File; import java.io.IOException; import java.nio.file.Files; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; @@ -14,10 +15,8 @@ import org.apache.commons.exec.ExecuteException; import org.apache.commons.io.FileUtils; import org.gradle.api.GradleException; -import org.gradle.internal.os.OperatingSystem; import org.junit.After; import org.junit.Before; -import org.junit.Rule; import org.junit.Test; import org.mockito.InOrder; import org.mockito.Mockito; @@ -26,408 +25,415 @@ public class ZiggyCppMexPojoTest { - File tempDir = null; - File buildDir = null; - File rootDir = null; - File projectDir = null; - File srcDir = null; - File incDir = null; - ZiggyCppMexPojo ziggyCppMexObject = null; - - DefaultExecutor defaultExecutor = Mockito.mock(DefaultExecutor.class); - - @Before - public void before() throws IOException { - - // create a temporary directory for everything - tempDir = Files.createTempDirectory("rootDir").toFile(); - tempDir.deleteOnExit(); - - // rootDir is the same as tempDir - rootDir = tempDir; - - // projectDir - projectDir = new File(rootDir,"projectDir"); - projectDir.mkdir(); - - // build directory under project - buildDir = new File(projectDir, "build"); - buildDir.mkdir(); - - // lib, bin, obj, and include directories under build - new File(buildDir, "lib").mkdir(); - new File(buildDir, "obj").mkdir(); - new File(buildDir, "bin").mkdir(); - new File(buildDir, "include").mkdir(); - - // add a source directory that's several levels down - srcDir = new File(projectDir, "src/main/cpp/mex"); - srcDir.mkdirs(); - - // add an include directory that's several levels down - incDir = new File(projectDir, "src/main/include"); - incDir.mkdirs(); - - // create source files - createSourceFiles(); - - // create the ZiggyCppMexPojo object - ziggyCppMexObject = createZiggyCppMexPojo(); - } - - @After - public void after() throws IOException { - - // explicitly delete the temp directory - FileUtils.deleteDirectory(tempDir); - - // delete any cppdebug system properties - System.clearProperty(ZiggyCppPojo.CPP_DEBUG_PROPERTY_NAME); - - // delete the ZiggyCpp object - ziggyCppMexObject = null; - buildDir = null; - tempDir = null; - projectDir = null; - srcDir = null; - incDir = null; - } - + File tempDir = null; + File buildDir = null; + File rootDir = null; + File projectDir = null; + File srcDir = null; + File incDir = null; + ZiggyCppMexPojo ziggyCppMexObject = null; + + DefaultExecutor defaultExecutor = Mockito.mock(DefaultExecutor.class); + + @Before + public void before() throws IOException { + + // create a temporary directory for everything + tempDir = Files + .createDirectories(Paths.get("build").resolve("test").resolve("ZiggyCppMexPojoTest")) + .toFile() + .getAbsoluteFile(); + + // rootDir is the same as tempDir + rootDir = tempDir; + + // projectDir + projectDir = new File(rootDir, "projectDir"); + projectDir.mkdir(); + + // build directory under project + buildDir = new File(projectDir, "build"); + buildDir.mkdir(); + + // lib, bin, obj, and include directories under build + new File(buildDir, "lib").mkdir(); + new File(buildDir, "obj").mkdir(); + new File(buildDir, "bin").mkdir(); + new File(buildDir, "include").mkdir(); + + // add a source directory that's several levels down + srcDir = new File(projectDir, "src/main/cpp/mex"); + srcDir.mkdirs(); + + // add an include directory that's several levels down + incDir = new File(projectDir, "src/main/include"); + incDir.mkdirs(); + + // create source files + createSourceFiles(); + + // create the ZiggyCppMexPojo object + ziggyCppMexObject = createZiggyCppMexPojo(); + } + + @After + public void after() throws IOException { + + // explicitly delete the temp directory + FileUtils.deleteDirectory(tempDir); + + // delete any cppdebug system properties + System.clearProperty(ZiggyCppPojo.CPP_DEBUG_PROPERTY_NAME); + + // delete the ZiggyCpp object + ziggyCppMexObject = null; + buildDir = null; + tempDir = null; + projectDir = null; + srcDir = null; + incDir = null; + } + //*************************************************************************************** - - // Here begins the actual test classes - - /** Tests that the output type setters have no effect on the output type - * - */ - @Test - public void testOutputTypeSetters() { - ziggyCppMexObject.setOutputType("executable"); - assertEquals(ziggyCppMexObject.getOutputType(), BuildType.SHARED); - ziggyCppMexObject.setOutputType("static"); - assertEquals(ziggyCppMexObject.getOutputType(), BuildType.SHARED); - ziggyCppMexObject.setOutputType(BuildType.EXECUTABLE); - assertEquals(ziggyCppMexObject.getOutputType(), BuildType.SHARED); - ziggyCppMexObject.setOutputType(BuildType.STATIC); - assertEquals(ziggyCppMexObject.getOutputType(), BuildType.SHARED); - } - - /** - * Tests the setters and getters that are unique to the ZiggyCppMexPojo (the ones - * that are inherited from ZiggyCppPojo are not tested). - */ - @Test - public void testSettersAndGetters() { - - // these getter tests implicitly test the setters in createZiggyCppMexPojo(): - assertEquals(projectDir.getAbsolutePath(), ziggyCppMexObject.getProjectDir().getAbsolutePath()); - assertEquals("/dev/null/MATLAB_R2017b", ziggyCppMexObject.getMatlabPath()); - List mexfileNames = ziggyCppMexObject.getMexfileNames(); - assertEquals(2, mexfileNames.size()); - assertEquals("CSource1", mexfileNames.get(0)); - assertEquals("CppSource2", mexfileNames.get(1)); - } - - /** - * Tests the compile command generator, in particular to make certain that the MATLAB include - * path and MATLAB_MEX_FILE compiler directive are present - */ - @Test - public void testGenerateCompileCommand() { - - // test with debug options disabled - String compileString = ziggyCppMexObject.generateCompileCommand(new File("/dev/null/dmy1.c")); - String expectedString = "/dev/null/g++ -c -o " + buildDir.getAbsolutePath() + "/obj/dmy1.o " - + "-I" + srcDir.getAbsolutePath() + " -I" + incDir.getAbsolutePath() - + " -I/dev/null/MATLAB_R2017b/extern/include -Wall -fPic -DMATLAB_MEX_FILE -O2 -DNDEBUG -g " - + "/dev/null/dmy1.c"; - assertEquals(expectedString, compileString); - - // test with debug options enabled - System.setProperty("cppdebug", "true"); - compileString = ziggyCppMexObject.generateCompileCommand(new File("/dev/null/dmy1.c")); - expectedString = "/dev/null/g++ -c -o " + buildDir.getAbsolutePath() + "/obj/dmy1.o " - + "-I" + srcDir.getAbsolutePath() + " -I" + incDir.getAbsolutePath() - + " -I/dev/null/MATLAB_R2017b/extern/include -Wall -fPic -DMATLAB_MEX_FILE -Og -g " - + "/dev/null/dmy1.c"; - assertEquals(expectedString, compileString); - - // test with debug property present but set to false - System.setProperty("cppdebug", "false"); - compileString = ziggyCppMexObject.generateCompileCommand(new File("/dev/null/dmy1.c")); - expectedString = "/dev/null/g++ -c -o " + buildDir.getAbsolutePath() + "/obj/dmy1.o " - + "-I" + srcDir.getAbsolutePath() + " -I" + incDir.getAbsolutePath() - + " -I/dev/null/MATLAB_R2017b/extern/include -Wall -fPic -DMATLAB_MEX_FILE -O2 -DNDEBUG -g " - + "/dev/null/dmy1.c"; - assertEquals(expectedString, compileString); - } - - @Test - public void testGenerateSharedObjectName() { - String generatedName = ziggyCppMexObject.generateSharedObjectName(); - assertEquals("projectDir-src-main-cpp-mex", generatedName); - } - - @Test - public void testGenerateLinkCommand() { - configureLinkerOptions(ziggyCppMexObject); - ziggyCppMexObject.setOperatingSystem(OperatingSystem.LINUX); - String linkCommand = ziggyCppMexObject.generateLinkCommand(); - String expectedCommand = "/dev/null/g++ -o " + buildDir.getAbsolutePath() + "/lib/" - + "libdummy.so -L/dummy1/lib -L/dummy2/lib " - +"-L/dev/null/MATLAB_R2017b/bin/glnxa64 -shared o1.o o2.o -lhdf5 -lnetcdf -lmex -lmx -lmat "; - assertEquals(expectedCommand, linkCommand); - - // now test the library name for empty object name - ziggyCppMexObject = createZiggyCppMexPojo(); - ziggyCppMexObject.setOutputName(""); - configureLinkerOptions(ziggyCppMexObject); - ziggyCppMexObject.setOperatingSystem(OperatingSystem.LINUX); - linkCommand = ziggyCppMexObject.generateLinkCommand(); - expectedCommand = "/dev/null/g++ -o " + buildDir.getAbsolutePath() + "/lib/" - + "libprojectDir-src-main-cpp-mex.so -L/dummy1/lib -L/dummy2/lib " - +"-L/dev/null/MATLAB_R2017b/bin/glnxa64 -shared o1.o o2.o -lhdf5 -lnetcdf -lmex -lmx -lmat "; - assertEquals(expectedCommand, linkCommand); - - // test for Mac OS - ziggyCppMexObject = createZiggyCppMexPojo(); - configureLinkerOptions(ziggyCppMexObject); - ziggyCppMexObject.setOperatingSystem(OperatingSystem.MAC_OS); - linkCommand = ziggyCppMexObject.generateLinkCommand(); - expectedCommand = "/dev/null/g++ -o " + buildDir.getAbsolutePath() + "/lib/" - + "libdummy.dylib -L/dummy1/lib -L/dummy2/lib " - +"-L/dev/null/MATLAB_R2017b/bin/maci64 " - +"-shared -install_name " + rootDir.getAbsolutePath()+"/build/lib/libdummy.dylib" - + " o1.o o2.o -lhdf5 -lnetcdf -lmex -lmx -lmat "; - assertEquals(expectedCommand, linkCommand); - } - - @Test - public void testGenerateMexCommand() { - ziggyCppMexObject.setOperatingSystem(OperatingSystem.LINUX); - configureLinkerOptions(ziggyCppMexObject); - File mexfile = new File(buildDir, "lib/o1.mexmaci64"); - File objFile = new File(buildDir, "obj/o1.o"); - String mexCommand = ziggyCppMexObject.generateMexCommand(mexfile, objFile); - String expectedCommand = "/dev/null/g++ -o " + mexfile.getAbsolutePath() + " " - + objFile.getAbsolutePath() + " -L/dummy1/lib -L/dummy2/lib " - + "-L/dev/null/MATLAB_R2017b/bin/glnxa64 -L" + buildDir.getAbsolutePath() + "/lib " - + "-lhdf5 -lnetcdf -lmex -lmx -lmat -ldummy -shared"; - assertEquals(expectedCommand, mexCommand); - - // test for empty library object name - ziggyCppMexObject = createZiggyCppMexPojo(); - ziggyCppMexObject.setOperatingSystem(OperatingSystem.LINUX); - configureLinkerOptions(ziggyCppMexObject); - ziggyCppMexObject.setOutputName(""); - mexCommand = ziggyCppMexObject.generateMexCommand(mexfile, objFile); - expectedCommand = "/dev/null/g++ -o " + mexfile.getAbsolutePath() + " " - + objFile.getAbsolutePath() + " -L/dummy1/lib -L/dummy2/lib " - + "-L/dev/null/MATLAB_R2017b/bin/glnxa64 -L" + buildDir.getAbsolutePath() + "/lib " - + "-lhdf5 -lnetcdf -lmex -lmx -lmat -lprojectDir-src-main-cpp-mex -shared"; - assertEquals(expectedCommand, mexCommand); - - // test for Mac OS - ziggyCppMexObject = createZiggyCppMexPojo(); - configureLinkerOptions(ziggyCppMexObject); - ziggyCppMexObject.setOperatingSystem(OperatingSystem.MAC_OS); - mexCommand = ziggyCppMexObject.generateMexCommand(mexfile, objFile); - expectedCommand = "/dev/null/g++ -o " + mexfile.getAbsolutePath() + " " - + objFile.getAbsolutePath() + " -L/dummy1/lib -L/dummy2/lib " - + "-L/dev/null/MATLAB_R2017b/bin/maci64 -L" + buildDir.getAbsolutePath() + "/lib " - + "-lhdf5 -lnetcdf -lmex -lmx -lmat -ldummy -shared"; - assertEquals(expectedCommand, mexCommand); - } - - @Test - public void testAction() throws ExecuteException, IOException { - - // set the mocked executor into the object - ziggyCppMexObject.setOperatingSystem(OperatingSystem.LINUX); - ziggyCppMexObject.setDefaultExecutor(defaultExecutor); - InOrder executorCalls = Mockito.inOrder(defaultExecutor); - - // call the method - ziggyCppMexObject.action(); - - // check the calls -- first the 4 compile commands - executorCalls.verify(defaultExecutor).setWorkingDirectory(new File(projectDir, - "src/main/cpp/mex")); - executorCalls.verify(defaultExecutor).execute(ziggyCppMexObject.new CommandLineComparable( - ziggyCppMexObject.generateCompileCommand(new File(srcDir, "CSource1.c")))); - executorCalls.verify(defaultExecutor).setWorkingDirectory(new File(projectDir, - "src/main/cpp/mex")); - executorCalls.verify(defaultExecutor).execute(ziggyCppMexObject.new CommandLineComparable( - ziggyCppMexObject.generateCompileCommand(new File(srcDir, "CSource2.c")))); - executorCalls.verify(defaultExecutor).setWorkingDirectory(new File(projectDir, - "src/main/cpp/mex")); - executorCalls.verify(defaultExecutor).execute(ziggyCppMexObject.new CommandLineComparable( - ziggyCppMexObject.generateCompileCommand(new File(srcDir, "CppSource1.cpp")))); - executorCalls.verify(defaultExecutor).setWorkingDirectory(new File(projectDir, - "src/main/cpp/mex")); - executorCalls.verify(defaultExecutor).execute(ziggyCppMexObject.new CommandLineComparable( - ziggyCppMexObject.generateCompileCommand(new File(srcDir, "CppSource2.cpp")))); - - // then the link command for the dynamic library (and also make sure that 2 of the 4 files - // got removed from the list of object files) - executorCalls.verify(defaultExecutor).setWorkingDirectory(new File(buildDir, - "obj")); - List allObjectFiles = ziggyCppMexObject.getObjectFiles(); - assertEquals(2, allObjectFiles.size()); - executorCalls.verify(defaultExecutor).execute(ziggyCppMexObject.new CommandLineComparable( - ziggyCppMexObject.generateLinkCommand())); - - // then the mex commands - executorCalls.verify(defaultExecutor).setWorkingDirectory(new File(buildDir, - "obj")); - executorCalls.verify(defaultExecutor).execute(ziggyCppMexObject.new CommandLineComparable( - ziggyCppMexObject.generateMexCommand(new File(buildDir, "lib/CSource1.mexa64"), - new File(buildDir, "obj/CSource1.o")))); - executorCalls.verify(defaultExecutor).setWorkingDirectory(new File(buildDir, - "obj")); - executorCalls.verify(defaultExecutor).execute(ziggyCppMexObject.new CommandLineComparable( - ziggyCppMexObject.generateMexCommand(new File(buildDir, "lib/CppSource2.mexa64"), - new File(buildDir, "obj/CppSource2.o")))); - - } - - // Here are unit tests that exercise various error cases - - @SuppressWarnings("serial") - @Test - public void testErrorMexfileMissingSourceFile() { - ziggyCppMexObject.setMexfileNames(new ArrayList() {{ - add("CSource3"); - }}); - ziggyCppMexObject.setDefaultExecutor(defaultExecutor); - assertThrows("No object file for mexfile CSource3", GradleException.class, () -> { - ziggyCppMexObject.action(); - }); - } - - @Test - public void testErrorMexReturnCode() throws ExecuteException, IOException { - ziggyCppMexObject.setDefaultExecutor(defaultExecutor); - configureLinkerOptions(ziggyCppMexObject); - ziggyCppMexObject.setOperatingSystem(OperatingSystem.LINUX); - File mexfile = new File(buildDir, "lib/CSource1.mexa64"); - File objFile = new File(buildDir, "obj/CSource1.o"); - String mexCommand = ziggyCppMexObject.generateMexCommand(mexfile, objFile); - when(defaultExecutor.execute(ziggyCppMexObject.new CommandLineComparable( - mexCommand))).thenReturn(1); - assertThrows("Mexing of file CSource1.mexa64 failed", GradleException.class, () -> { - ziggyCppMexObject.action(); - }); - } - - @Test - public void testBadMexSuffix() { - ziggyCppMexObject.setOperatingSystem(OperatingSystem.WINDOWS); - assertThrows(GradleException.class, () -> { - ziggyCppMexObject.mexSuffix(); - }); - } - - @Test - public void testBadMatlabArch() { - ziggyCppMexObject.setOperatingSystem(OperatingSystem.WINDOWS); - assertThrows(GradleException.class, () -> { - ziggyCppMexObject.matlabArch(); - }); - } - - @SuppressWarnings("serial") - @Test - public void testNoBuildDir() { - ZiggyCppMexPojo ziggyCppMexObject = new ZiggyCppMexPojo(); - ziggyCppMexObject.setMexfileNames(new ArrayList() {{ - add("CSource1"); - add("CppSource2"); - }}); - assertThrows("buildDir and mexfileNames must not be null", GradleException.class, - () -> { - ziggyCppMexObject.populateMexfiles(); - }); - } - - @Test - public void testNoMexfiles() { - ZiggyCppMexPojo ziggyCppMexObject = new ZiggyCppMexPojo(); - ziggyCppMexObject.setBuildDir(buildDir); - assertThrows("buildDir and mexfileNames must not be null", GradleException.class, - () -> { - ziggyCppMexObject.populateMexfiles(); - }); - } - + + // Here begins the actual test classes + + /** + * Tests that the output type setters have no effect on the output type + */ + @Test + public void testOutputTypeSetters() { + ziggyCppMexObject.setOutputType("executable"); + assertEquals(ziggyCppMexObject.getOutputType(), BuildType.SHARED); + ziggyCppMexObject.setOutputType("static"); + assertEquals(ziggyCppMexObject.getOutputType(), BuildType.SHARED); + ziggyCppMexObject.setOutputType(BuildType.EXECUTABLE); + assertEquals(ziggyCppMexObject.getOutputType(), BuildType.SHARED); + ziggyCppMexObject.setOutputType(BuildType.STATIC); + assertEquals(ziggyCppMexObject.getOutputType(), BuildType.SHARED); + } + + /** + * Tests the setters and getters that are unique to the ZiggyCppMexPojo (the ones that are + * inherited from ZiggyCppPojo are not tested). + */ + @Test + public void testSettersAndGetters() { + + // these getter tests implicitly test the setters in createZiggyCppMexPojo(): + assertEquals(projectDir.getAbsolutePath(), + ziggyCppMexObject.getProjectDir().getAbsolutePath()); + assertEquals("/dev/null/MATLAB_R2017b", ziggyCppMexObject.getMatlabPath()); + List mexfileNames = ziggyCppMexObject.getMexfileNames(); + assertEquals(2, mexfileNames.size()); + assertEquals("CSource1", mexfileNames.get(0)); + assertEquals("CppSource2", mexfileNames.get(1)); + } + + /** + * Tests the compile command generator, in particular to make certain that the MATLAB include + * path and MATLAB_MEX_FILE compiler directive are present + */ + @Test + public void testGenerateCompileCommand() { + + String compiler = ZiggyCppPojo.Compiler.CPP.compiler(); + // test with debug options disabled + String compileString = ziggyCppMexObject + .generateCompileCommand(new File("/dev/null/dmy1.c"), compiler); + String expectedString = "/dev/null/g++ -c -o " + buildDir.getAbsolutePath() + "/obj/dmy1.o " + + "-I" + srcDir.getAbsolutePath() + " -I" + incDir.getAbsolutePath() + + " -I/dev/null/MATLAB_R2017b/extern/include -Wall -fPic -DMATLAB_MEX_FILE -O2 -DNDEBUG -g " + + "/dev/null/dmy1.c"; + assertEquals(expectedString, compileString); + + // test with debug options enabled + System.setProperty("cppdebug", "true"); + compileString = ziggyCppMexObject.generateCompileCommand(new File("/dev/null/dmy1.c"), + compiler); + expectedString = "/dev/null/g++ -c -o " + buildDir.getAbsolutePath() + "/obj/dmy1.o " + "-I" + + srcDir.getAbsolutePath() + " -I" + incDir.getAbsolutePath() + + " -I/dev/null/MATLAB_R2017b/extern/include -Wall -fPic -DMATLAB_MEX_FILE -Og -g " + + "/dev/null/dmy1.c"; + assertEquals(expectedString, compileString); + + // test with debug property present but set to false + System.setProperty("cppdebug", "false"); + compileString = ziggyCppMexObject.generateCompileCommand(new File("/dev/null/dmy1.c"), + compiler); + expectedString = "/dev/null/g++ -c -o " + buildDir.getAbsolutePath() + "/obj/dmy1.o " + "-I" + + srcDir.getAbsolutePath() + " -I" + incDir.getAbsolutePath() + + " -I/dev/null/MATLAB_R2017b/extern/include -Wall -fPic -DMATLAB_MEX_FILE -O2 -DNDEBUG -g " + + "/dev/null/dmy1.c"; + assertEquals(expectedString, compileString); + } + + @Test + public void testGenerateSharedObjectName() { + String generatedName = ziggyCppMexObject.generateSharedObjectName(); + assertEquals("projectDir-src-main-cpp-mex", generatedName); + } + + @Test + public void testGenerateLinkCommand() { + configureLinkerOptions(ziggyCppMexObject); + ziggyCppMexObject.setArchitecture(SystemArchitecture.LINUX_INTEL); + String linkCommand = ziggyCppMexObject.generateLinkCommand(); + String expectedCommand = "/dev/null/g++ -o " + buildDir.getAbsolutePath() + "/lib/" + + "libdummy.so -L/dummy1/lib -L/dummy2/lib " + + "-L/dev/null/MATLAB_R2017b/bin/glnxa64 -shared o1.o o2.o -lhdf5 -lnetcdf -lmex -lmx -lmat "; + assertEquals(expectedCommand, linkCommand); + + // now test the library name for empty object name + ziggyCppMexObject = createZiggyCppMexPojo(); + ziggyCppMexObject.setOutputName(""); + configureLinkerOptions(ziggyCppMexObject); + ziggyCppMexObject.setArchitecture(SystemArchitecture.LINUX_INTEL); + linkCommand = ziggyCppMexObject.generateLinkCommand(); + expectedCommand = "/dev/null/g++ -o " + buildDir.getAbsolutePath() + "/lib/" + + "libprojectDir-src-main-cpp-mex.so -L/dummy1/lib -L/dummy2/lib " + + "-L/dev/null/MATLAB_R2017b/bin/glnxa64 -shared o1.o o2.o -lhdf5 -lnetcdf -lmex -lmx -lmat "; + assertEquals(expectedCommand, linkCommand); + + // test for Mac OS + ziggyCppMexObject = createZiggyCppMexPojo(); + configureLinkerOptions(ziggyCppMexObject); + ziggyCppMexObject.setArchitecture(SystemArchitecture.MAC_M1); + linkCommand = ziggyCppMexObject.generateLinkCommand(); + expectedCommand = "/dev/null/g++ -o " + buildDir.getAbsolutePath() + "/lib/" + + "libdummy.dylib -L/dummy1/lib -L/dummy2/lib " + + "-L/dev/null/MATLAB_R2017b/bin/maca64 " + "-shared -install_name " + + rootDir.getAbsolutePath() + "/build/lib/libdummy.dylib" + + " o1.o o2.o -lhdf5 -lnetcdf -lmex -lmx -lmat "; + assertEquals(expectedCommand, linkCommand); + + ziggyCppMexObject = createZiggyCppMexPojo(); + configureLinkerOptions(ziggyCppMexObject); + ziggyCppMexObject.setArchitecture(SystemArchitecture.MAC_INTEL); + linkCommand = ziggyCppMexObject.generateLinkCommand(); + expectedCommand = "/dev/null/g++ -o " + buildDir.getAbsolutePath() + "/lib/" + + "libdummy.dylib -L/dummy1/lib -L/dummy2/lib " + + "-L/dev/null/MATLAB_R2017b/bin/maci64 " + "-shared -install_name " + + rootDir.getAbsolutePath() + "/build/lib/libdummy.dylib" + + " o1.o o2.o -lhdf5 -lnetcdf -lmex -lmx -lmat "; + assertEquals(expectedCommand, linkCommand); + } + + @Test + public void testGenerateMexCommand() { + ziggyCppMexObject.setArchitecture(SystemArchitecture.LINUX_INTEL); + configureLinkerOptions(ziggyCppMexObject); + File mexfile = new File(buildDir, "lib/o1.mexmaci64"); + File objFile = new File(buildDir, "obj/o1.o"); + String mexCommand = ziggyCppMexObject.generateMexCommand(mexfile, objFile); + String expectedCommand = "/dev/null/g++ -o " + mexfile.getAbsolutePath() + " " + + objFile.getAbsolutePath() + " -L/dummy1/lib -L/dummy2/lib " + + "-L/dev/null/MATLAB_R2017b/bin/glnxa64 -L" + buildDir.getAbsolutePath() + "/lib " + + "-lhdf5 -lnetcdf -lmex -lmx -lmat -ldummy -shared"; + assertEquals(expectedCommand, mexCommand); + + // test for empty library object name + ziggyCppMexObject = createZiggyCppMexPojo(); + ziggyCppMexObject.setArchitecture(SystemArchitecture.LINUX_INTEL); + configureLinkerOptions(ziggyCppMexObject); + ziggyCppMexObject.setOutputName(""); + mexCommand = ziggyCppMexObject.generateMexCommand(mexfile, objFile); + expectedCommand = "/dev/null/g++ -o " + mexfile.getAbsolutePath() + " " + + objFile.getAbsolutePath() + " -L/dummy1/lib -L/dummy2/lib " + + "-L/dev/null/MATLAB_R2017b/bin/glnxa64 -L" + buildDir.getAbsolutePath() + "/lib " + + "-lhdf5 -lnetcdf -lmex -lmx -lmat -lprojectDir-src-main-cpp-mex -shared"; + assertEquals(expectedCommand, mexCommand); + + // test for Mac OS + ziggyCppMexObject = createZiggyCppMexPojo(); + configureLinkerOptions(ziggyCppMexObject); + ziggyCppMexObject.setArchitecture(SystemArchitecture.MAC_M1); + mexCommand = ziggyCppMexObject.generateMexCommand(mexfile, objFile); + expectedCommand = "/dev/null/g++ -o " + mexfile.getAbsolutePath() + " " + + objFile.getAbsolutePath() + " -L/dummy1/lib -L/dummy2/lib " + + "-L/dev/null/MATLAB_R2017b/bin/maca64 -L" + buildDir.getAbsolutePath() + "/lib " + + "-lhdf5 -lnetcdf -lmex -lmx -lmat -ldummy -shared"; + assertEquals(expectedCommand, mexCommand); + + ziggyCppMexObject = createZiggyCppMexPojo(); + configureLinkerOptions(ziggyCppMexObject); + ziggyCppMexObject.setArchitecture(SystemArchitecture.MAC_INTEL); + mexCommand = ziggyCppMexObject.generateMexCommand(mexfile, objFile); + expectedCommand = "/dev/null/g++ -o " + mexfile.getAbsolutePath() + " " + + objFile.getAbsolutePath() + " -L/dummy1/lib -L/dummy2/lib " + + "-L/dev/null/MATLAB_R2017b/bin/maci64 -L" + buildDir.getAbsolutePath() + "/lib " + + "-lhdf5 -lnetcdf -lmex -lmx -lmat -ldummy -shared"; + assertEquals(expectedCommand, mexCommand); + } + + @Test + public void testAction() throws ExecuteException, IOException { + + // set the mocked executor into the object + ziggyCppMexObject.setArchitecture(SystemArchitecture.LINUX_INTEL); + ziggyCppMexObject.setDefaultExecutor(defaultExecutor); + ziggyCppMexObject.setMaxCompileThreads(1); + InOrder executorCalls = Mockito.inOrder(defaultExecutor); + + String cppCompiler = ZiggyCppPojo.Compiler.CPP.compiler(); + String cCompiler = ZiggyCppPojo.Compiler.C.compiler(); + + // call the method + ziggyCppMexObject.action(); + + // check the calls -- first the 4 compile commands + executorCalls.verify(defaultExecutor) + .setWorkingDirectory(new File(projectDir, "src/main/cpp/mex")); + executorCalls.verify(defaultExecutor) + .execute(ziggyCppMexObject.new CommandLineComparable(ziggyCppMexObject + .generateCompileCommand(new File(srcDir, "CSource1.c"), cCompiler))); + executorCalls.verify(defaultExecutor) + .setWorkingDirectory(new File(projectDir, "src/main/cpp/mex")); + executorCalls.verify(defaultExecutor) + .execute(ziggyCppMexObject.new CommandLineComparable(ziggyCppMexObject + .generateCompileCommand(new File(srcDir, "CSource2.c"), cCompiler))); + executorCalls.verify(defaultExecutor) + .setWorkingDirectory(new File(projectDir, "src/main/cpp/mex")); + executorCalls.verify(defaultExecutor) + .execute(ziggyCppMexObject.new CommandLineComparable(ziggyCppMexObject + .generateCompileCommand(new File(srcDir, "CppSource1.cpp"), cppCompiler))); + executorCalls.verify(defaultExecutor) + .setWorkingDirectory(new File(projectDir, "src/main/cpp/mex")); + executorCalls.verify(defaultExecutor) + .execute(ziggyCppMexObject.new CommandLineComparable(ziggyCppMexObject + .generateCompileCommand(new File(srcDir, "CppSource2.cpp"), cppCompiler))); + + // then the link command for the dynamic library (and also make sure that 2 of the 4 files + // got removed from the list of object files) + executorCalls.verify(defaultExecutor).setWorkingDirectory(new File(buildDir, "obj")); + List allObjectFiles = ziggyCppMexObject.getObjectFiles(); + assertEquals(2, allObjectFiles.size()); + executorCalls.verify(defaultExecutor) + .execute(ziggyCppMexObject.new CommandLineComparable( + ziggyCppMexObject.generateLinkCommand())); + + // then the mex commands + executorCalls.verify(defaultExecutor).setWorkingDirectory(new File(buildDir, "obj")); + executorCalls.verify(defaultExecutor) + .execute(ziggyCppMexObject.new CommandLineComparable( + ziggyCppMexObject.generateMexCommand(new File(buildDir, "mex/CSource1.mexa64"), + new File(buildDir, "obj/CSource1.o")))); + executorCalls.verify(defaultExecutor).setWorkingDirectory(new File(buildDir, "obj")); + executorCalls.verify(defaultExecutor) + .execute(ziggyCppMexObject.new CommandLineComparable( + ziggyCppMexObject.generateMexCommand(new File(buildDir, "mex/CppSource2.mexa64"), + new File(buildDir, "obj/CppSource2.o")))); + } + + // Here are unit tests that exercise various error cases + + @SuppressWarnings("serial") + @Test + public void testErrorMexfileMissingSourceFile() { + ziggyCppMexObject.setMexfileNames(new ArrayList() { + { + add("CSource3"); + } + }); + ziggyCppMexObject.setDefaultExecutor(defaultExecutor); + assertThrows("No object file for mexfile CSource3", GradleException.class, () -> { + ziggyCppMexObject.action(); + }); + } + + @Test + public void testErrorMexReturnCode() throws ExecuteException, IOException { + ziggyCppMexObject.setDefaultExecutor(defaultExecutor); + configureLinkerOptions(ziggyCppMexObject); + ziggyCppMexObject.setArchitecture(SystemArchitecture.LINUX_INTEL); + File mexfile = new File(buildDir, "mex/CSource1.mexa64"); + File objFile = new File(buildDir, "obj/CSource1.o"); + String mexCommand = ziggyCppMexObject.generateMexCommand(mexfile, objFile); + when(defaultExecutor.execute(ziggyCppMexObject.new CommandLineComparable(mexCommand))) + .thenReturn(1); + assertThrows("Mexing of file CSource1.mexa64 failed", GradleException.class, () -> { + ziggyCppMexObject.action(); + }); + } + + @SuppressWarnings("serial") + @Test + public void testNoBuildDir() { + ZiggyCppMexPojo ziggyCppMexObject = new ZiggyCppMexPojo(); + ziggyCppMexObject.setMexfileNames(new ArrayList() { + { + add("CSource1"); + add("CppSource2"); + } + }); + assertThrows("buildDir and mexfileNames must not be null", GradleException.class, () -> { + ziggyCppMexObject.populateMexfiles(); + }); + } + + @Test + public void testNoMexfiles() { + ZiggyCppMexPojo ziggyCppMexObject = new ZiggyCppMexPojo(); + ziggyCppMexObject.setBuildDir(buildDir); + assertThrows("buildDir and mexfileNames must not be null", GradleException.class, () -> { + ziggyCppMexObject.populateMexfiles(); + }); + } + //*************************************************************************************** - - // here begins assorted setup and helper methods - - public void createSourceFiles() throws IOException { - - // create 4 temporary "C/C++" files in the source directory - new File(srcDir, "CSource1.c").createNewFile(); - new File(srcDir, "CSource2.c").createNewFile(); - new File(srcDir, "CppSource1.cpp").createNewFile(); - new File(srcDir, "CppSource2.cpp").createNewFile(); - - new File(srcDir, "Header1.h").createNewFile(); - new File(incDir, "Header2.hpp").createNewFile(); - } - - @SuppressWarnings("serial") - public ZiggyCppMexPojo createZiggyCppMexPojo() { - ZiggyCppMexPojo ziggyCppMexObject = new ZiggyCppMexPojo(); - ziggyCppMexObject.setBuildDir(buildDir); - ziggyCppMexObject.setProjectDir(projectDir); - ziggyCppMexObject.setRootDir(rootDir); - ziggyCppMexObject.setCppCompiler("/dev/null/g++"); - ziggyCppMexObject.setCppFilePath(srcDir.getAbsolutePath()); - ziggyCppMexObject.setMatlabPath("/dev/null/MATLAB_R2017b"); - ziggyCppMexObject.setOutputName("dummy"); - ziggyCppMexObject.setMexfileNames(new ArrayList() {{ - add("CSource1"); - add("CppSource2"); - }}); - ziggyCppMexObject.setIncludeFilePaths(new ArrayList() {{ - add(srcDir.getAbsolutePath()); - add(incDir.getAbsolutePath()); - }}); - ziggyCppMexObject.setCompileOptions(new ArrayList() {{ - add("Wall"); - add("fPic"); - }}); - ziggyCppMexObject.setReleaseOptimizations(new ArrayList() {{ - add("O2"); - add("DNDEBUG"); - add("g"); - }}); - ziggyCppMexObject.setDebugOptimizations(new ArrayList() {{ - add("Og"); - add("g"); - }}); - return ziggyCppMexObject; - } - - public void configureLinkerOptions(ZiggyCppPojo ziggyCppObject) { - // first we need to add some object files - File o1 = new File(buildDir, "obj/o1.o"); - File o2 = new File(buildDir, "obj/o2.o"); - ziggyCppObject.setObjectFiles(o1); - ziggyCppObject.setObjectFiles(o2); - - // also some linker options and libraries - List linkerOptions = new ArrayList<>(); - linkerOptions.add("u whatevs"); - ziggyCppObject.setLinkOptions(linkerOptions); - List libraryPathOptions = new ArrayList<>(); - libraryPathOptions.add("/dummy1/lib"); - libraryPathOptions.add("/dummy2/lib"); - ziggyCppObject.setLibraryPaths(libraryPathOptions); - List libraryOptions = new ArrayList<>(); - libraryOptions.add("hdf5"); - libraryOptions.add("netcdf"); - ziggyCppObject.setLibraries(libraryOptions); - } + + // here begins assorted setup and helper methods + + public void createSourceFiles() throws IOException { + + // create 4 temporary "C/C++" files in the source directory + new File(srcDir, "CSource1.c").createNewFile(); + new File(srcDir, "CSource2.c").createNewFile(); + new File(srcDir, "CppSource1.cpp").createNewFile(); + new File(srcDir, "CppSource2.cpp").createNewFile(); + + new File(srcDir, "Header1.h").createNewFile(); + new File(incDir, "Header2.hpp").createNewFile(); + } + + public ZiggyCppMexPojo createZiggyCppMexPojo() { + ZiggyCppMexPojo ziggyCppMexObject = new ZiggyCppMexPojo(); + ziggyCppMexObject.setBuildDir(buildDir); + ziggyCppMexObject.setProjectDir(projectDir); + ziggyCppMexObject.setRootDir(rootDir); + ZiggyCppPojo.setCppCompiler("/dev/null/g++"); + ZiggyCppPojo.setCCompiler("/dev/null/gcc"); + ziggyCppMexObject.setCppFilePath(srcDir.getAbsolutePath()); + ziggyCppMexObject.setMatlabPath("/dev/null/MATLAB_R2017b"); + ziggyCppMexObject.setOutputName("dummy"); + ziggyCppMexObject.setMexfileNames(List.of("CSource1", "CppSource2")); + ziggyCppMexObject + .setIncludeFilePaths(List.of(srcDir.getAbsolutePath(), incDir.getAbsolutePath())); + ziggyCppMexObject.setCppCompileOptions(List.of("-Wall", "-fPic")); + ziggyCppMexObject.setReleaseOptimizations(List.of("-O2", "-DNDEBUG", "-g")); + ziggyCppMexObject.setDebugOptimizations(List.of("-Og", "-g")); + ziggyCppMexObject.setMaxCompileThreads(1); + return ziggyCppMexObject; + } + + public void configureLinkerOptions(ZiggyCppPojo ziggyCppObject) { + // first we need to add some object files + File o1 = new File(buildDir, "obj/o1.o"); + File o2 = new File(buildDir, "obj/o2.o"); + ziggyCppObject.setObjectFiles(o1); + ziggyCppObject.setObjectFiles(o2); + + // also some linker options and libraries + List linkerOptions = new ArrayList<>(); + linkerOptions.add("-u whatevs"); + ziggyCppObject.setLinkOptions(linkerOptions); + List libraryPathOptions = new ArrayList<>(); + libraryPathOptions.add("/dummy1/lib"); + libraryPathOptions.add("/dummy2/lib"); + ziggyCppObject.setLibraryPaths(libraryPathOptions); + List libraryOptions = new ArrayList<>(); + libraryOptions.add("hdf5"); + libraryOptions.add("netcdf"); + ziggyCppObject.setLibraries(libraryOptions); + } } diff --git a/buildSrc/src/test/java/gov/nasa/ziggy/buildutil/ZiggyCppPojoTest.java b/buildSrc/src/test/java/gov/nasa/ziggy/buildutil/ZiggyCppPojoTest.java index dfcddef..57087af 100644 --- a/buildSrc/src/test/java/gov/nasa/ziggy/buildutil/ZiggyCppPojoTest.java +++ b/buildSrc/src/test/java/gov/nasa/ziggy/buildutil/ZiggyCppPojoTest.java @@ -1,5 +1,12 @@ package gov.nasa.ziggy.buildutil; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; + import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; @@ -9,6 +16,7 @@ import java.nio.file.Files; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; import org.apache.commons.exec.CommandLine; @@ -16,751 +24,837 @@ import org.apache.commons.exec.ExecuteException; import org.apache.commons.io.FileUtils; import org.gradle.api.GradleException; -import org.gradle.internal.os.OperatingSystem; import org.junit.After; import org.junit.Before; import org.junit.Ignore; -import org.junit.Rule; import org.junit.Test; import org.mockito.InOrder; import org.mockito.Mockito; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.when; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.assertThrows; - -import gov.nasa.ziggy.buildutil.ZiggyCppPojo; import gov.nasa.ziggy.buildutil.ZiggyCppPojo.BuildType; /** - * Unit test class for ZiggyCppPojo class. - * @author PT + * Unit test class for ZiggyCppPojo class. * + * @author PT */ public class ZiggyCppPojoTest { - File tempDir = null; - File buildDir = null; - File srcDir = null; - File rootDir = new File("/dev/null/rootDir"); - ZiggyCppPojo ziggyCppObject; - - DefaultExecutor defaultExecutor = Mockito.mock(DefaultExecutor.class); - - @Before - public void before() throws IOException { - - // create a temporary directory for everything - tempDir = Files.createTempDirectory("ZiggyCpp").toFile(); - tempDir.deleteOnExit(); - - // directory for includes - new File(tempDir, "include").mkdir(); - - // directory for source - srcDir = new File(tempDir, "src"); - srcDir.mkdir(); - - // build directory - buildDir = new File(tempDir,"build"); - - // directory for libraries - new File(buildDir,"lib").mkdir(); - - // directory for includes - new File(buildDir, "include").mkdir(); - - // directory for built source - new File(buildDir, "src").mkdir(); - - // directory for objects - new File(buildDir, "obj").mkdir(); - - // directory for executables - new File(buildDir, "bin").mkdir(); - - // create C++ source and header files - createSourceFiles(); - - // create the ZiggyCpp object - ziggyCppObject = createZiggyCppObject(buildDir); - - } - - @After - public void after() throws IOException { - - // explicitly delete the temp directory - FileUtils.deleteDirectory(tempDir); - - // delete any cppdebug system properties - System.clearProperty(ZiggyCppPojo.CPP_DEBUG_PROPERTY_NAME); - - // delete the ZiggyCpp object - ziggyCppObject = null; - buildDir = null; - tempDir = null; - } - + File tempDir = null; + File buildDir = null; + File srcDir = null; + File rootDir = new File("/dev/null/rootDir"); + ZiggyCppPojo ziggyCppObject; + + DefaultExecutor defaultExecutor = Mockito.mock(DefaultExecutor.class); + + @Before + public void before() throws IOException { + + // create a temporary directory for everything + tempDir = Files.createTempDirectory("ZiggyCpp").toFile(); + tempDir.deleteOnExit(); + + // directory for includes + new File(tempDir, "include").mkdir(); + + // directory for source + srcDir = new File(tempDir, "src"); + srcDir.mkdir(); + + // build directory + buildDir = new File(tempDir, "build"); + + // directory for libraries + new File(buildDir, "lib").mkdir(); + + // directory for includes + new File(buildDir, "include").mkdir(); + + // directory for built source + new File(buildDir, "src").mkdir(); + + // directory for objects + new File(buildDir, "obj").mkdir(); + + // directory for executables + new File(buildDir, "bin").mkdir(); + + // create C++ source and header files + createSourceFiles(); + + // create the ZiggyCpp object + ziggyCppObject = createZiggyCppObject(buildDir); + } + + @After + public void after() throws IOException { + + // explicitly delete the temp directory + FileUtils.deleteDirectory(tempDir); + + // delete any cppdebug system properties + System.clearProperty(ZiggyCppPojo.CPP_DEBUG_PROPERTY_NAME); + + // delete the ZiggyCpp object + ziggyCppObject = null; + buildDir = null; + tempDir = null; + } + //*************************************************************************************** - - // here begins the test methods - - /** - * Tests all getter and setter methods. - * @throws InvocationTargetException - * @throws IllegalArgumentException - * @throws IllegalAccessException - * @throws NoSuchMethodException - */ - @Test - public void getterSetterTest() throws NoSuchMethodException, IllegalAccessException, - IllegalArgumentException, InvocationTargetException { - - List dummyArguments = new ArrayList<>(); - dummyArguments.add("DuMMY"); - dummyArguments.add("dUmMy"); - assertEquals(tempDir.getAbsolutePath() + "/src", ziggyCppObject.getCppFilePaths().get(0)); - - testStringListSettersAndGetters("IncludeFilePaths", new String[] { - tempDir.getAbsolutePath() + "/src", - tempDir.getAbsolutePath() + "/include"}); - testStringListSettersAndGetters("CompileOptions", new String[] { - "Wall", "fPic"}); - testStringListSettersAndGetters("ReleaseOptimizations", new String[] { - "O2", "DNDEBUG", "g"}); - testStringListSettersAndGetters("DebugOptimizations", new String[] { - "Og", "g"}); - ziggyCppObject.setLibraries(dummyArguments); - testStringListSettersAndGetters("Libraries", new String[]{"DuMMY", "dUmMy"}); - ziggyCppObject.setLibraryPaths(dummyArguments); - testStringListSettersAndGetters("LibraryPaths", new String[]{"DuMMY", "dUmMy"}); - ziggyCppObject.setLinkOptions(dummyArguments); - testStringListSettersAndGetters("LinkOptions", new String[]{"DuMMY", "dUmMy"}); - - ziggyCppObject.setOutputName("outputName"); - assertEquals("outputName", ziggyCppObject.getOutputName()); - - ziggyCppObject.setOutputType(BuildType.EXECUTABLE); - assertEquals(BuildType.EXECUTABLE, ziggyCppObject.getOutputType()); - ziggyCppObject.setOutputType("executable"); - assertEquals(BuildType.EXECUTABLE, ziggyCppObject.getOutputType()); - - ziggyCppObject.setOutputType(BuildType.SHARED); - assertEquals(BuildType.SHARED, ziggyCppObject.getOutputType()); - ziggyCppObject.setOutputType("shared"); - assertEquals(BuildType.SHARED, ziggyCppObject.getOutputType()); - - ziggyCppObject.setOutputType(BuildType.STATIC); - assertEquals(BuildType.STATIC, ziggyCppObject.getOutputType()); - ziggyCppObject.setOutputType("static"); - assertEquals(BuildType.STATIC, ziggyCppObject.getOutputType()); - - assertEquals(buildDir, ziggyCppObject.getBuildDir()); - assertEquals(buildDir.getAbsolutePath(), ziggyCppObject.getBuildDir().getAbsolutePath()); - - } - - /** - * Tests the ability to find C/C++ source files in the source directory and add them to - * the ZiggyCppPojo as a list of File objects - */ - @Test - public void testGetCppFiles() { - List cppFiles = ziggyCppObject.getCppFiles(); - assertEquals(2, cppFiles.size()); - List cppFilePaths = cppFiles.stream().map(s -> s.getAbsolutePath()) - .collect(Collectors.toList()); - assertTrue(cppFilePaths.contains(tempDir.getAbsolutePath() + "/src/ZiggyCppMain.cpp")); - assertTrue(cppFilePaths.contains(tempDir.getAbsolutePath() + "/src/GetString.cpp")); - - cppFiles = ziggyCppObject.getCppFiles(); - assertEquals(2, cppFiles.size()); - cppFilePaths = cppFiles.stream().map(s -> s.getAbsolutePath()) - .collect(Collectors.toList()); - assertTrue(cppFilePaths.contains(tempDir.getAbsolutePath() + "/src/ZiggyCppMain.cpp")); - assertTrue(cppFilePaths.contains(tempDir.getAbsolutePath() + "/src/GetString.cpp")); - } - - @Test - public void testGetCppFilesMultipleDirectories() throws FileNotFoundException { - - // put a source directory in build, and populate it - new File(buildDir, "src/cpp").mkdirs(); - createAdditionalSource(); - - // create the list of directories to check out - List cppPaths = new ArrayList<>(); - cppPaths.add(tempDir.toString() + "/src"); - cppPaths.add(buildDir.toString() + "/src/cpp"); - ziggyCppObject.setCppFilePaths(cppPaths); - List cppFiles = ziggyCppObject.getCppFiles(); - int nFiles = cppFiles.size(); - assertEquals(3, nFiles); - - List cppFilePaths = cppFiles.stream().map(s -> s.getAbsolutePath()) - .collect(Collectors.toList()); - assertTrue(cppFilePaths.contains(buildDir.getAbsolutePath() + "/src/cpp/GetAnotherString.cpp")); - assertTrue(cppFilePaths.contains(tempDir.getAbsolutePath() + "/src/ZiggyCppMain.cpp")); - assertTrue(cppFilePaths.contains(tempDir.getAbsolutePath() + "/src/GetString.cpp")); - } - - /** - * Tests the argListToString method, which converts a list of arguments to a string, with a common - * prefix added to each list element - */ - @Test - public void argListToStringTest() { - String compileOptionString = ziggyCppObject.argListToString(ziggyCppObject.getCompileOptions(), - "-"); - assertEquals("-Wall -fPic ", compileOptionString); - } - - /** - * Tests the code that determines the File that is to be the output of the compile and link - * process. - */ - @Test - public void populateBuiltFileTest() { - - // executable - ziggyCppObject.setOutputType("executable"); - File builtFile = ziggyCppObject.getBuiltFile(); - assertEquals(buildDir.getAbsolutePath() + "/bin/dummy", builtFile.getAbsolutePath()); - - // shared library - ZiggyCppPojo ziggyCppShared = createZiggyCppObject(buildDir); - ziggyCppShared.setOutputType("shared"); - ziggyCppShared.setOperatingSystem(OperatingSystem.MAC_OS); - builtFile = ziggyCppShared.getBuiltFile(); - String builtFilePath = builtFile.getAbsolutePath(); - String sharedObjectFileType = ".dylib"; - assertEquals(buildDir.getAbsolutePath() + "/lib/libdummy" + sharedObjectFileType, builtFilePath); - ziggyCppShared = createZiggyCppObject(buildDir); - ziggyCppShared.setOutputType("shared"); - ziggyCppShared.setOperatingSystem(OperatingSystem.LINUX); - builtFile = ziggyCppShared.getBuiltFile(); - builtFilePath = builtFile.getAbsolutePath(); - sharedObjectFileType = ".so"; - assertEquals(buildDir.getAbsolutePath() + "/lib/libdummy" + sharedObjectFileType, builtFilePath); - - // static library - ZiggyCppPojo ziggyCppStatic = createZiggyCppObject(buildDir); - ziggyCppStatic.setOutputType("static"); - builtFile = ziggyCppStatic.getBuiltFile(); - builtFilePath = builtFile.getAbsolutePath(); - assertEquals(buildDir.getAbsolutePath() + "/lib/libdummy.a", builtFilePath); - } - - /** - * Tests the process of converting a source File to an object file name (with the - * path of the former stripped away). - */ - @Test - public void objectNameFromSourceFileTest() { - File f1 = new File("/tmp/dummy/s1.c"); - String s1 = ZiggyCppPojo.objectNameFromSourceFile(f1); - assertEquals("s1.o", s1); - File f2 = new File("/tmp/dummy/s1.cpp"); - String s2 = ZiggyCppPojo.objectNameFromSourceFile(f2); - assertEquals("s1.o", s2); - } - - /** - * Tests the method that generates compile commands. - */ - @Test - public void generateCompileCommandTest() { - File f1 = new File("/tmp/dummy/s1.c"); - String compileCommand = ziggyCppObject.generateCompileCommand(f1); - String expectedString = "/dev/null/g++ -c -o " + buildDir.getAbsolutePath() + "/obj/s1.o " - + "-I" + tempDir.getAbsolutePath() + "/src -I" + tempDir.getAbsolutePath() + - "/include -Wall -fPic -O2 -DNDEBUG -g /tmp/dummy/s1.c"; - assertEquals(expectedString, compileCommand); - - // set up for debugging - System.setProperty(ZiggyCppPojo.CPP_DEBUG_PROPERTY_NAME, "true"); - compileCommand = ziggyCppObject.generateCompileCommand(f1); - expectedString = "/dev/null/g++ -c -o " + buildDir.getAbsolutePath() + "/obj/s1.o " - + "-I" + tempDir.getAbsolutePath() + "/src -I" + tempDir.getAbsolutePath() + - "/include -Wall -fPic -Og -g /tmp/dummy/s1.c"; - assertEquals(expectedString, compileCommand); - - // have the debugging property but set to false - System.setProperty(ZiggyCppPojo.CPP_DEBUG_PROPERTY_NAME, "false"); - compileCommand = ziggyCppObject.generateCompileCommand(f1); - expectedString = "/dev/null/g++ -c -o " + buildDir.getAbsolutePath() + "/obj/s1.o " - + "-I" + tempDir.getAbsolutePath() + "/src -I" + tempDir.getAbsolutePath() + - "/include -Wall -fPic -O2 -DNDEBUG -g /tmp/dummy/s1.c"; - assertEquals(expectedString, compileCommand); - - // test .cpp file type - f1 = new File("/tmp/dummy/s1.cpp"); - compileCommand = ziggyCppObject.generateCompileCommand(f1); - expectedString = "/dev/null/g++ -c -o " + buildDir.getAbsolutePath() + "/obj/s1.o " - + "-I" + tempDir.getAbsolutePath() + "/src -I" + tempDir.getAbsolutePath() + - "/include -Wall -fPic -O2 -DNDEBUG -g /tmp/dummy/s1.cpp"; - assertEquals(expectedString, compileCommand); - } - - /** - * Tests the method that generates link commands. - */ - @Test - public void generateLinkCommandTest() { - - configureLinkerOptions(ziggyCppObject); - ziggyCppObject.setOutputType("executable"); - String linkString = ziggyCppObject.generateLinkCommand(); - assertEquals("/dev/null/g++ -o " + buildDir.getAbsolutePath() + "/bin/dummy -L/dummy1/lib -L/dummy2/lib " - + "-u whatevs -O2 -DNDEBUG -g o1.o o2.o -lhdf5 -lnetcdf ", linkString); - - // now try it with debug enabled - System.setProperty(ZiggyCppPojo.CPP_DEBUG_PROPERTY_NAME, "true"); - linkString = ziggyCppObject.generateLinkCommand(); - assertEquals("/dev/null/g++ -o " + buildDir.getAbsolutePath() + "/bin/dummy -L/dummy1/lib -L/dummy2/lib " - + "-u whatevs -Og -g o1.o o2.o -lhdf5 -lnetcdf ", linkString); - System.setProperty(ZiggyCppPojo.CPP_DEBUG_PROPERTY_NAME, "false"); - - - // Now for a shared object library - ziggyCppObject = createZiggyCppObject(buildDir); - configureLinkerOptions(ziggyCppObject); - ziggyCppObject.setOutputType("shared"); - ziggyCppObject.setOperatingSystem(OperatingSystem.LINUX); - String sharedObjectFileType = ".so"; - linkString = ziggyCppObject.generateLinkCommand(); - assertEquals("/dev/null/g++ -o " + buildDir.getAbsolutePath() + "/lib/libdummy" + sharedObjectFileType - + " -L/dummy1/lib -L/dummy2/lib -shared" - + " o1.o o2.o -lhdf5 -lnetcdf ", linkString); - - // For a Mac, there has to be an install name as well - ziggyCppObject = createZiggyCppObject(buildDir); - configureLinkerOptions(ziggyCppObject); - ziggyCppObject.setOutputType("shared"); - ziggyCppObject.setOperatingSystem(OperatingSystem.MAC_OS); - sharedObjectFileType = ".dylib"; - linkString = ziggyCppObject.generateLinkCommand(); - assertEquals("/dev/null/g++ -o " + buildDir.getAbsolutePath() + "/lib/libdummy" + sharedObjectFileType - + " -L/dummy1/lib -L/dummy2/lib -shared" - + " -install_name /dev/null/rootDir/build/lib/libdummy.dylib " - + "o1.o o2.o -lhdf5 -lnetcdf ", linkString); - - // debug enabled shouldn't do anything - System.setProperty(ZiggyCppPojo.CPP_DEBUG_PROPERTY_NAME, "true"); - linkString = ziggyCppObject.generateLinkCommand(); - assertEquals("/dev/null/g++ -o " + buildDir.getAbsolutePath() + "/lib/libdummy" + sharedObjectFileType - + " -L/dummy1/lib -L/dummy2/lib -shared" - + " -install_name /dev/null/rootDir/build/lib/libdummy.dylib " - + "o1.o o2.o -lhdf5 -lnetcdf ", linkString); - System.setProperty(ZiggyCppPojo.CPP_DEBUG_PROPERTY_NAME, "false"); - - // static library case - ziggyCppObject = createZiggyCppObject(buildDir); - configureLinkerOptions(ziggyCppObject); - ziggyCppObject.setOutputType("static"); - linkString = ziggyCppObject.generateLinkCommand(); - assertEquals("ar rs " + buildDir.getAbsolutePath() + "/lib/libdummy.a o1.o o2.o ", linkString); - - // debug enabled shouldn't do anything - System.setProperty(ZiggyCppPojo.CPP_DEBUG_PROPERTY_NAME, "true"); - linkString = ziggyCppObject.generateLinkCommand(); - assertEquals("ar rs " + buildDir.getAbsolutePath() + "/lib/libdummy.a o1.o o2.o ", linkString); - System.setProperty(ZiggyCppPojo.CPP_DEBUG_PROPERTY_NAME, "false"); - - } - - /** - * Tests the method that executes the main action (compiles and links). - * @throws ExecuteException - * @throws IOException - */ - @Test - public void actionTest() throws ExecuteException, IOException { - - // set values for the ZiggyCppPojo - ziggyCppObject.setOutputName("testOutput"); - ziggyCppObject.setOutputType("executable"); - - // set the mocked executor into the object - ziggyCppObject.setDefaultExecutor(defaultExecutor); - InOrder executorCalls = Mockito.inOrder(defaultExecutor); - - // call the method - ziggyCppObject.action(); - - // check the calls to the executor and their order - executorCalls.verify(defaultExecutor).setWorkingDirectory(new File(tempDir, "src")); - executorCalls.verify(defaultExecutor).execute(ziggyCppObject.new CommandLineComparable( - ziggyCppObject.generateCompileCommand(new File(srcDir, "GetString.cpp")))); - executorCalls.verify(defaultExecutor).setWorkingDirectory(new File(tempDir, "src")); - executorCalls.verify(defaultExecutor).execute(ziggyCppObject.new CommandLineComparable( - ziggyCppObject.generateCompileCommand(new File(srcDir, "ZiggyCppMain.cpp")))); - executorCalls.verify(defaultExecutor).setWorkingDirectory(new File(buildDir, "obj")); - executorCalls.verify(defaultExecutor).execute(ziggyCppObject.new CommandLineComparable( - ziggyCppObject.generateLinkCommand())); - - // test that the include files were copied - File buildInclude = new File(buildDir, "include"); - File buildInclude1 = new File(buildInclude, "ZiggyCppMain.h"); - assertTrue(buildInclude1.exists()); - File buildInclude2 = new File(buildInclude, "ZiggyCppLib.h"); - assertTrue(buildInclude2.exists()); - - - // create a new object for linking a shared object - ziggyCppObject = createZiggyCppObject(buildDir); - ziggyCppObject.setOutputName("testOutput"); - ziggyCppObject.setOutputType("shared"); - ziggyCppObject.setDefaultExecutor(defaultExecutor); - ziggyCppObject.action(); - executorCalls.verify(defaultExecutor).setWorkingDirectory(new File(tempDir, "src")); - executorCalls.verify(defaultExecutor).execute(ziggyCppObject.new CommandLineComparable( - ziggyCppObject.generateCompileCommand(new File(srcDir, "GetString.cpp")))); - executorCalls.verify(defaultExecutor).setWorkingDirectory(new File(tempDir, "src")); - executorCalls.verify(defaultExecutor).execute(ziggyCppObject.new CommandLineComparable( - ziggyCppObject.generateCompileCommand(new File(srcDir, "ZiggyCppMain.cpp")))); - executorCalls.verify(defaultExecutor).setWorkingDirectory(new File(buildDir, "obj")); - executorCalls.verify(defaultExecutor).execute(ziggyCppObject.new CommandLineComparable( - ziggyCppObject.generateLinkCommand())); - - // and once more for a static library - // create a new object for linking a shared object - ziggyCppObject = createZiggyCppObject(buildDir); - ziggyCppObject.setOutputName("testOutput"); - ziggyCppObject.setOutputType("static"); - ziggyCppObject.setDefaultExecutor(defaultExecutor); - ziggyCppObject.action(); - executorCalls.verify(defaultExecutor).setWorkingDirectory(new File(tempDir, "src")); - executorCalls.verify(defaultExecutor).execute(ziggyCppObject.new CommandLineComparable( - ziggyCppObject.generateCompileCommand(new File(srcDir, "GetString.cpp")))); - executorCalls.verify(defaultExecutor).setWorkingDirectory(new File(tempDir, "src")); - executorCalls.verify(defaultExecutor).execute(ziggyCppObject.new CommandLineComparable( - ziggyCppObject.generateCompileCommand(new File(srcDir, "ZiggyCppMain.cpp")))); - executorCalls.verify(defaultExecutor).setWorkingDirectory(new File(buildDir, "obj")); - executorCalls.verify(defaultExecutor).execute(ziggyCppObject.new CommandLineComparable( - ziggyCppObject.generateLinkCommand())); - - } - - // The following tests exercise various error conditions that return GradleExceptions. There are 2 error - // conditions that are not tested by these methods, they occur when the DefaultExecutor throws an IOException. - // These cases are considered sufficiently trivial that we omit them. - - /** - * Tests that a null value for the C++ file path produces the correct error. - */ - @Test - public void testCppFilePathNullError() { - ZiggyCppPojo ziggyCppError = new ZiggyCppPojo(); - assertThrows("C++ file path is null", GradleException.class, () -> { - ziggyCppError.getCppFiles(); - }); - } - - /** - * Tests that a nonexistent C++ file path produces the correct warning message. - */ - @Test - public void testCppFilePathDoesNotExist() { - ZiggyCppPojo ziggyCppError = new ZiggyCppPojo(); - ziggyCppError.setCppFilePath("/this/path/does/not/exist"); - ziggyCppError.getCppFiles(); - List w = ziggyCppError.loggerWarnings(); - assertTrue(w.size() > 0); - assertTrue(w.contains("C++ file path /this/path/does/not/exist does not exist")); - } - - /** - * Tests that a missing output name produces the correct error. - */ - @Test - public void testErrorNoOutputName() { - ZiggyCppPojo ziggyCppError = new ZiggyCppPojo(); - ziggyCppError.setCppFilePath(tempDir.getAbsolutePath() + "/src"); - ziggyCppError.setBuildDir(buildDir); - ziggyCppError.setOutputType("executable"); - assertThrows("Both output name and output type must be specified", GradleException.class, - () -> { - ziggyCppError.getBuiltFile(); - }); - } - - /** - * Tests that a missing output type produces the correct error. - */ - @Test - public void testErrorNoOutputType() { - ZiggyCppPojo ziggyCppError = new ZiggyCppPojo(); - ziggyCppError.setCppFilePath(tempDir.getAbsolutePath() + "/src"); - ziggyCppError.setBuildDir(buildDir); - ziggyCppError.setOutputName("dummy"); - assertThrows("Both output name and output type must be specified", GradleException.class, - () -> { - ziggyCppError.getBuiltFile(); - }); - } - - /** - * Tests that missing both the output type and the built file name produces the correct error. - */ - @Test - public void testErrorNoOutputTypeOrName() { - ZiggyCppPojo ziggyCppError = new ZiggyCppPojo(); - List cppPaths = new ArrayList<>(); - ziggyCppError.setCppFilePath(tempDir.getAbsolutePath() + "/src"); - ziggyCppError.setBuildDir(buildDir); - assertThrows("Both output name and output type must be specified", GradleException.class, - () -> { - ziggyCppError.getBuiltFile(); - }); - } - - /** - * Tests that a non-zero compiler return value produces the correct error. - * @throws ExecuteException - * @throws IOException - */ - @Test - public void testCompilerError() throws ExecuteException, IOException { - ziggyCppObject.setOutputType("executable"); - ziggyCppObject.setDefaultExecutor(defaultExecutor); - when(defaultExecutor.execute(any(CommandLine.class))).thenReturn(1); - assertThrows("Compilation of file GetString.cpp failed", GradleException.class, - () -> { - ziggyCppObject.action(); - }); - } - - /** - * Tests that a non-zero linker return value produces the correct error. - * @throws ExecuteException - * @throws IOException - */ - @Test - public void testLinkerError() throws ExecuteException, IOException { - ziggyCppObject.setOutputType("executable"); - ziggyCppObject.setDefaultExecutor(defaultExecutor); - String linkerCommand = ziggyCppObject.generateLinkCommand() + "GetString.o ZiggyCppMain.o "; - when(defaultExecutor.execute(ziggyCppObject.new CommandLineComparable(linkerCommand))).thenReturn(1); - assertThrows("Link / library construction of dummy failed", GradleException.class, - () -> { - ziggyCppObject.action(); - }); - } - - /** - * Test that an invalid OS produces the correct error. - */ - @Test - public void testInvalidOsError() { - ZiggyCppPojo ziggyCppError = new ZiggyCppPojo(); - ziggyCppError.setBuildDir(buildDir); - ziggyCppError.setOperatingSystem(OperatingSystem.WINDOWS); - ziggyCppError.setOutputName("dummy"); - ziggyCppError.setOutputType("shared"); - assertThrows("ZiggyCpp class does not support OS " + ziggyCppError.getOperatingSystem().getName(), - GradleException.class, () -> { - ziggyCppError.getBuiltFile(); - }); - } + + // here begins the test methods + + /** + * Tests all getter and setter methods. + * + * @throws InvocationTargetException + * @throws IllegalArgumentException + * @throws IllegalAccessException + * @throws NoSuchMethodException + */ + @Test + public void getterSetterTest() throws NoSuchMethodException, IllegalAccessException, + IllegalArgumentException, InvocationTargetException { + + List dummyArguments = new ArrayList<>(); + dummyArguments.add("DuMMY"); + dummyArguments.add("dUmMy"); + dummyArguments.add(""); + assertEquals(tempDir.getAbsolutePath() + "/src", ziggyCppObject.getCppFilePaths().get(0)); + + testStringListSettersAndGetters("IncludeFilePaths", new String[] { + tempDir.getAbsolutePath() + "/src", tempDir.getAbsolutePath() + "/include" }); + testStringListSettersAndGetters("CppCompileOptions", new String[] { "-Wall", "-fPic" }); + testStringListSettersAndGetters("CCompileOptions", new String[] { "-fPic" }); + testStringListSettersAndGetters("ReleaseOptimizations", + new String[] { "-O2", "-DNDEBUG", "-g" }); + testStringListSettersAndGetters("DebugOptimizations", new String[] { "-Og", "-g" }); + ziggyCppObject.setLibraries(dummyArguments); + testStringListSettersAndGetters("Libraries", new String[] { "DuMMY", "dUmMy" }); + ziggyCppObject.setLibraryPaths(dummyArguments); + testStringListSettersAndGetters("LibraryPaths", new String[] { "DuMMY", "dUmMy" }); + ziggyCppObject.setLinkOptions(dummyArguments); + testStringListSettersAndGetters("LinkOptions", new String[] { "DuMMY", "dUmMy" }); + + ziggyCppObject.setOutputName("outputName"); + assertEquals("outputName", ziggyCppObject.getOutputName()); + + ziggyCppObject.setOutputType(BuildType.EXECUTABLE); + assertEquals(BuildType.EXECUTABLE, ziggyCppObject.getOutputType()); + ziggyCppObject.setOutputType("executable"); + assertEquals(BuildType.EXECUTABLE, ziggyCppObject.getOutputType()); + + ziggyCppObject.setOutputType(BuildType.SHARED); + assertEquals(BuildType.SHARED, ziggyCppObject.getOutputType()); + ziggyCppObject.setOutputType("shared"); + assertEquals(BuildType.SHARED, ziggyCppObject.getOutputType()); + + ziggyCppObject.setOutputType(BuildType.STATIC); + assertEquals(BuildType.STATIC, ziggyCppObject.getOutputType()); + ziggyCppObject.setOutputType("static"); + assertEquals(BuildType.STATIC, ziggyCppObject.getOutputType()); + + assertEquals(buildDir, ziggyCppObject.getBuildDir()); + assertEquals(buildDir.getAbsolutePath(), ziggyCppObject.getBuildDir().getAbsolutePath()); + } + + /** + * Tests the ability to find C/C++ source files in the source directory and add them to the + * ZiggyCppPojo as a list of File objects + */ + @Test + public void testGetCppFiles() { + List cppFiles = ziggyCppObject.getSourceFiles(); + assertEquals(3, cppFiles.size()); + List cppFilePaths = cppFiles.stream() + .map(File::getAbsolutePath) + .collect(Collectors.toList()); + assertTrue(cppFilePaths.contains(tempDir.getAbsolutePath() + "/src/ZiggyCppMain.cpp")); + assertTrue(cppFilePaths.contains(tempDir.getAbsolutePath() + "/src/GetString.cpp")); + assertTrue(cppFilePaths.contains(tempDir.getAbsolutePath() + "/src/ZiggyCModule.c")); + + // Now check that the compiler choices are set correctly. + Map sourceFilesWithCompiler = ziggyCppObject.getCppFiles(); + String cppCompiler = ZiggyCppPojo.Compiler.CPP.compiler(); + String cCompiler = ZiggyCppPojo.Compiler.C.compiler(); + assertEquals(cppCompiler, sourceFilesWithCompiler + .get(new File(tempDir.getAbsolutePath(), "/src/ZiggyCppMain.cpp"))); + assertEquals(cppCompiler, + sourceFilesWithCompiler.get(new File(tempDir.getAbsolutePath(), "/src/GetString.cpp"))); + assertEquals(cCompiler, sourceFilesWithCompiler + .get(new File(tempDir.getAbsolutePath(), "/src/ZiggyCModule.c"))); + } + + @Test + public void testGetCppFilesMultipleDirectories() throws FileNotFoundException { + + // put a source directory in build, and populate it + new File(buildDir, "src/cpp").mkdirs(); + createAdditionalSource(); + + // create the list of directories to check out + List cppPaths = new ArrayList<>(); + cppPaths.add(tempDir.toString() + "/src"); + cppPaths.add(buildDir.toString() + "/src/cpp"); + ziggyCppObject.setCppFilePaths(cppPaths); + List cppFiles = ziggyCppObject.getSourceFiles(); + int nFiles = cppFiles.size(); + assertEquals(4, nFiles); + + List cppFilePaths = cppFiles.stream() + .map(File::getAbsolutePath) + .collect(Collectors.toList()); + assertTrue( + cppFilePaths.contains(buildDir.getAbsolutePath() + "/src/cpp/GetAnotherString.cpp")); + assertTrue(cppFilePaths.contains(tempDir.getAbsolutePath() + "/src/ZiggyCppMain.cpp")); + assertTrue(cppFilePaths.contains(tempDir.getAbsolutePath() + "/src/GetString.cpp")); + assertTrue(cppFilePaths.contains(tempDir.getAbsolutePath() + "/src/ZiggyCModule.c")); + } + + /** + * Tests the argListToString method, which converts a list of arguments to a string, with a + * common prefix added to each list element + */ + @Test + public void testArgListToString() { + String compileOptionString = ziggyCppObject + .argListToString(ziggyCppObject.getCppCompileOptions(), ""); + assertEquals("-Wall -fPic ", compileOptionString); + } + + /** + * Tests the code that determines the File that is to be the output of the compile and link + * process. + */ + @Test + public void testPopulateBuiltFile() { + + // executable + ziggyCppObject.setOutputType("executable"); + File builtFile = ziggyCppObject.getBuiltFile(); + assertEquals(buildDir.getAbsolutePath() + "/bin/dummy", builtFile.getAbsolutePath()); + + // shared library + ZiggyCppPojo ziggyCppShared = createZiggyCppObject(buildDir); + ziggyCppShared.setOutputType("shared"); + ziggyCppShared.setArchitecture(SystemArchitecture.MAC_INTEL); + builtFile = ziggyCppShared.getBuiltFile(); + String builtFilePath = builtFile.getAbsolutePath(); + String sharedObjectFileType = ".dylib"; + assertEquals(buildDir.getAbsolutePath() + "/lib/libdummy" + sharedObjectFileType, + builtFilePath); + ziggyCppShared = createZiggyCppObject(buildDir); + ziggyCppShared.setOutputType("shared"); + ziggyCppShared.setArchitecture(SystemArchitecture.LINUX_INTEL); + builtFile = ziggyCppShared.getBuiltFile(); + builtFilePath = builtFile.getAbsolutePath(); + sharedObjectFileType = ".so"; + assertEquals(buildDir.getAbsolutePath() + "/lib/libdummy" + sharedObjectFileType, + builtFilePath); + + // static library + ZiggyCppPojo ziggyCppStatic = createZiggyCppObject(buildDir); + ziggyCppStatic.setOutputType("static"); + builtFile = ziggyCppStatic.getBuiltFile(); + builtFilePath = builtFile.getAbsolutePath(); + assertEquals(buildDir.getAbsolutePath() + "/lib/libdummy.a", builtFilePath); + + // Now with a user-specified output directory + ziggyCppObject.setOutputDir("foo"); + builtFile = ziggyCppObject.getBuiltFile(); + assertEquals(System.getProperty("user.dir") + "/foo/dummy", builtFile.getAbsolutePath()); + + ziggyCppShared.setOutputDir("foo"); + builtFile = ziggyCppShared.getBuiltFile(); + assertEquals(System.getProperty("user.dir") + "/foo/libdummy.so", + builtFile.getAbsolutePath()); + + ziggyCppStatic.setOutputDir("foo"); + builtFile = ziggyCppStatic.getBuiltFile(); + assertEquals(System.getProperty("user.dir") + "/foo/libdummy.a", + builtFile.getAbsolutePath()); + + // Now with a user-specified output directory parent + ziggyCppObject.setOutputDir(""); + ziggyCppObject.setOutputDirParent("foo"); + builtFile = ziggyCppObject.getBuiltFile(); + assertEquals(System.getProperty("user.dir") + "/foo/bin/dummy", + builtFile.getAbsolutePath()); + + ziggyCppShared.setOutputDir(""); + ziggyCppShared.setOutputDirParent("foo"); + builtFile = ziggyCppShared.getBuiltFile(); + assertEquals(System.getProperty("user.dir") + "/foo/lib/libdummy.so", + builtFile.getAbsolutePath()); + + ziggyCppStatic.setOutputDir(""); + ziggyCppStatic.setOutputDirParent("foo"); + builtFile = ziggyCppStatic.getBuiltFile(); + assertEquals(System.getProperty("user.dir") + "/foo/lib/libdummy.a", + builtFile.getAbsolutePath()); + } + + /** + * Tests the process of converting a source File to an object file name (with the path of the + * former stripped away). + */ + @Test + public void testObjectNameFromSourceFile() { + File f1 = new File("/tmp/dummy/s1.c"); + String s1 = ZiggyCppPojo.objectNameFromSourceFile(f1); + assertEquals("s1.o", s1); + File f2 = new File("/tmp/dummy/s1.cpp"); + String s2 = ZiggyCppPojo.objectNameFromSourceFile(f2); + assertEquals("s1.o", s2); + } + + /** + * Tests the method that generates compile commands. + */ + @Test + public void testGenerateCompileCommand() { + File f1 = new File("/tmp/dummy/s1.c"); + + // This is a stupid way to get my map entry, but it's the way that works with Java, + // so here we are. + String compiler = ZiggyCppPojo.Compiler.CPP.compiler(); + String compileCommand = ziggyCppObject.generateCompileCommand(f1, compiler, null, null); + String expectedString = "/dev/null/g++ -c -o " + buildDir.getAbsolutePath() + "/obj/s1.o " + + "-I" + tempDir.getAbsolutePath() + "/src -I" + tempDir.getAbsolutePath() + + "/include -Wall -fPic -O2 -DNDEBUG -g /tmp/dummy/s1.c"; + assertEquals(expectedString, compileCommand); + + // set up for debugging + System.setProperty(ZiggyCppPojo.CPP_DEBUG_PROPERTY_NAME, "true"); + compileCommand = ziggyCppObject.generateCompileCommand(f1, compiler, null, null); + expectedString = "/dev/null/g++ -c -o " + buildDir.getAbsolutePath() + "/obj/s1.o " + "-I" + + tempDir.getAbsolutePath() + "/src -I" + tempDir.getAbsolutePath() + + "/include -Wall -fPic -Og -g /tmp/dummy/s1.c"; + assertEquals(expectedString, compileCommand); + + // have the debugging property but set to false + System.setProperty(ZiggyCppPojo.CPP_DEBUG_PROPERTY_NAME, "false"); + compileCommand = ziggyCppObject.generateCompileCommand(f1, compiler, null, null); + expectedString = "/dev/null/g++ -c -o " + buildDir.getAbsolutePath() + "/obj/s1.o " + "-I" + + tempDir.getAbsolutePath() + "/src -I" + tempDir.getAbsolutePath() + + "/include -Wall -fPic -O2 -DNDEBUG -g /tmp/dummy/s1.c"; + assertEquals(expectedString, compileCommand); + + // test .cpp file type + f1 = new File("/tmp/dummy/s1.cpp"); + compileCommand = ziggyCppObject.generateCompileCommand(f1, compiler, null, null); + expectedString = "/dev/null/g++ -c -o " + buildDir.getAbsolutePath() + "/obj/s1.o " + "-I" + + tempDir.getAbsolutePath() + "/src -I" + tempDir.getAbsolutePath() + + "/include -Wall -fPic -O2 -DNDEBUG -g /tmp/dummy/s1.cpp"; + assertEquals(expectedString, compileCommand); + + // test with output dir + ziggyCppObject.setOutputDir("foo"); + compileCommand = ziggyCppObject.generateCompileCommand(f1, compiler, null, null); + expectedString = "/dev/null/g++ -c -o " + System.getProperty("user.dir") + "/foo/s1.o " + + "-I" + tempDir.getAbsolutePath() + "/src -I" + tempDir.getAbsolutePath() + + "/include -Wall -fPic -O2 -DNDEBUG -g /tmp/dummy/s1.cpp"; + assertEquals(expectedString, compileCommand); + + // Test with output dir parent + ziggyCppObject.setOutputDir(""); + ziggyCppObject.setOutputDirParent("foo"); + compileCommand = ziggyCppObject.generateCompileCommand(f1, compiler, null, null); + expectedString = "/dev/null/g++ -c -o " + System.getProperty("user.dir") + "/foo/obj/s1.o " + + "-I" + tempDir.getAbsolutePath() + "/src -I" + tempDir.getAbsolutePath() + + "/include -Wall -fPic -O2 -DNDEBUG -g /tmp/dummy/s1.cpp"; + assertEquals(expectedString, compileCommand); + } + + @Test + public void testGenerateCompileCommandWithCompiler() { + + Map sourceFilesWithCompiler = ziggyCppObject.getCppFiles(); + ZiggyCppPojo.Compiler.CPP.compiler(); + String cCompiler = ZiggyCppPojo.Compiler.C.compiler(); + + for (Map.Entry entry : sourceFilesWithCompiler.entrySet()) { + String compileCommand = ziggyCppObject.generateCompileCommand(entry); + entry.getKey().toString(); + entry.getKey().getName(); + String compiler = entry.getValue(); + if (compiler.equals(cCompiler)) { + assertTrue(compileCommand.startsWith("/dev/null/gcc -c -o")); + assertTrue(compileCommand.contains("-fPic")); + assertFalse(compileCommand.contains("-Wall")); + } else { + assertTrue(compileCommand.startsWith("/dev/null/g++ -c -o")); + assertTrue(compileCommand.contains("-fPic")); + assertTrue(compileCommand.contains("-Wall")); + } + } + } + + /** + * Tests the method that generates link commands. + */ + @Test + public void testGenerateLinkCommand() { + + configureLinkerOptions(ziggyCppObject); + ziggyCppObject.setOutputType("executable"); + String linkString = ziggyCppObject.generateLinkCommand(); + assertEquals("/dev/null/g++ -o " + buildDir.getAbsolutePath() + + "/bin/dummy -L/dummy1/lib -L/dummy2/lib " + + "-u whatevs -O2 -DNDEBUG -g o1.o o2.o -lhdf5 -lnetcdf ", linkString); + + // now try it with debug enabled + System.setProperty(ZiggyCppPojo.CPP_DEBUG_PROPERTY_NAME, "true"); + linkString = ziggyCppObject.generateLinkCommand(); + assertEquals("/dev/null/g++ -o " + buildDir.getAbsolutePath() + + "/bin/dummy -L/dummy1/lib -L/dummy2/lib " + + "-u whatevs -Og -g o1.o o2.o -lhdf5 -lnetcdf ", linkString); + System.setProperty(ZiggyCppPojo.CPP_DEBUG_PROPERTY_NAME, "false"); + + // Now with an output dir configured + ziggyCppObject.setOutputDir("foo"); + linkString = ziggyCppObject.generateLinkCommand(); + assertEquals("/dev/null/g++ -o " + System.getProperty("user.dir") + + "/foo/dummy -L/dummy1/lib -L/dummy2/lib " + + "-u whatevs -O2 -DNDEBUG -g o1.o o2.o -lhdf5 -lnetcdf ", linkString); + + // Now with an output dir parent configured + ziggyCppObject.setOutputDir(""); + ziggyCppObject.setOutputDirParent("foo"); + linkString = ziggyCppObject.generateLinkCommand(); + assertEquals("/dev/null/g++ -o " + System.getProperty("user.dir") + + "/foo/bin/dummy -L/dummy1/lib -L/dummy2/lib " + + "-u whatevs -O2 -DNDEBUG -g o1.o o2.o -lhdf5 -lnetcdf ", linkString); + + // Now for a shared object library + ziggyCppObject = createZiggyCppObject(buildDir); + configureLinkerOptions(ziggyCppObject); + ziggyCppObject.setOutputType("shared"); + ziggyCppObject.setArchitecture(SystemArchitecture.LINUX_INTEL); + String sharedObjectFileType = ".so"; + linkString = ziggyCppObject.generateLinkCommand(); + assertEquals("/dev/null/g++ -o " + buildDir.getAbsolutePath() + "/lib/libdummy" + + sharedObjectFileType + " -L/dummy1/lib -L/dummy2/lib -shared" + + " o1.o o2.o -lhdf5 -lnetcdf ", linkString); + + // For a Mac, there has to be an install name as well + ziggyCppObject = createZiggyCppObject(buildDir); + configureLinkerOptions(ziggyCppObject); + ziggyCppObject.setOutputType("shared"); + ziggyCppObject.setArchitecture(SystemArchitecture.MAC_INTEL); + sharedObjectFileType = ".dylib"; + linkString = ziggyCppObject.generateLinkCommand(); + assertEquals("/dev/null/g++ -o " + buildDir.getAbsolutePath() + "/lib/libdummy" + + sharedObjectFileType + " -L/dummy1/lib -L/dummy2/lib -shared" + + " -install_name /dev/null/rootDir/build/lib/libdummy.dylib " + + "o1.o o2.o -lhdf5 -lnetcdf ", linkString); + + // debug enabled shouldn't do anything + System.setProperty(ZiggyCppPojo.CPP_DEBUG_PROPERTY_NAME, "true"); + linkString = ziggyCppObject.generateLinkCommand(); + assertEquals("/dev/null/g++ -o " + buildDir.getAbsolutePath() + "/lib/libdummy" + + sharedObjectFileType + " -L/dummy1/lib -L/dummy2/lib -shared" + + " -install_name /dev/null/rootDir/build/lib/libdummy.dylib " + + "o1.o o2.o -lhdf5 -lnetcdf ", linkString); + System.setProperty(ZiggyCppPojo.CPP_DEBUG_PROPERTY_NAME, "false"); + + // Output dir case + ziggyCppObject.setOutputDir("foo"); + linkString = ziggyCppObject.generateLinkCommand(); + assertEquals("/dev/null/g++ -o " + System.getProperty("user.dir") + "/foo/libdummy" + + sharedObjectFileType + " -L/dummy1/lib -L/dummy2/lib -shared" + + " -install_name /dev/null/rootDir/build/lib/libdummy.dylib " + + "o1.o o2.o -lhdf5 -lnetcdf ", linkString); + + // Output dir parent case + ziggyCppObject.setOutputDir(""); + ziggyCppObject.setOutputDirParent("foo"); + linkString = ziggyCppObject.generateLinkCommand(); + assertEquals("/dev/null/g++ -o " + System.getProperty("user.dir") + "/foo/lib/libdummy" + + sharedObjectFileType + " -L/dummy1/lib -L/dummy2/lib -shared" + + " -install_name /dev/null/rootDir/build/lib/libdummy.dylib " + + "o1.o o2.o -lhdf5 -lnetcdf ", linkString); + + // static library case + ziggyCppObject = createZiggyCppObject(buildDir); + configureLinkerOptions(ziggyCppObject); + ziggyCppObject.setOutputType("static"); + linkString = ziggyCppObject.generateLinkCommand(); + assertEquals("ar rs " + buildDir.getAbsolutePath() + "/lib/libdummy.a o1.o o2.o ", + linkString); + + // debug enabled shouldn't do anything + System.setProperty(ZiggyCppPojo.CPP_DEBUG_PROPERTY_NAME, "true"); + linkString = ziggyCppObject.generateLinkCommand(); + assertEquals("ar rs " + buildDir.getAbsolutePath() + "/lib/libdummy.a o1.o o2.o ", + linkString); + System.setProperty(ZiggyCppPojo.CPP_DEBUG_PROPERTY_NAME, "false"); + } + + /** + * Tests the method that executes the main action (compiles and links). + * + * @throws ExecuteException + * @throws IOException + */ + @Test + public void testAction() throws ExecuteException, IOException { + + // set values for the ZiggyCppPojo + ziggyCppObject.setOutputName("testOutput"); + ziggyCppObject.setOutputType("executable"); + + // set the mocked executor into the object + ziggyCppObject.setDefaultExecutor(defaultExecutor); + InOrder executorCalls = Mockito.inOrder(defaultExecutor); + + String compiler = ZiggyCppPojo.Compiler.CPP.compiler(); + // call the method + ziggyCppObject.action(); + + // check the calls to the executor and their order + executorCalls.verify(defaultExecutor).setWorkingDirectory(new File(tempDir, "src")); + executorCalls.verify(defaultExecutor) + .execute(ziggyCppObject.new CommandLineComparable(ziggyCppObject + .generateCompileCommand(new File(srcDir, "GetString.cpp"), compiler, null, null))); + executorCalls.verify(defaultExecutor).setWorkingDirectory(new File(tempDir, "src")); + executorCalls.verify(defaultExecutor) + .execute(ziggyCppObject.new CommandLineComparable(ziggyCppObject.generateCompileCommand( + new File(srcDir, "ZiggyCppMain.cpp"), compiler, null, null))); + executorCalls.verify(defaultExecutor).setWorkingDirectory(new File(buildDir, "obj")); + executorCalls.verify(defaultExecutor) + .execute( + ziggyCppObject.new CommandLineComparable(ziggyCppObject.generateLinkCommand())); + + // test that the include files were copied + File buildInclude = new File(buildDir, "include"); + File buildInclude1 = new File(buildInclude, "ZiggyCppMain.h"); + assertTrue(buildInclude1.exists()); + File buildInclude2 = new File(buildInclude, "ZiggyCppLib.h"); + assertTrue(buildInclude2.exists()); + + // create a new object for linking a shared object + ziggyCppObject = createZiggyCppObject(buildDir); + ziggyCppObject.setOutputName("testOutput"); + ziggyCppObject.setOutputType("shared"); + ziggyCppObject.setDefaultExecutor(defaultExecutor); + ziggyCppObject.action(); + executorCalls.verify(defaultExecutor).setWorkingDirectory(new File(tempDir, "src")); + executorCalls.verify(defaultExecutor) + .execute(ziggyCppObject.new CommandLineComparable(ziggyCppObject + .generateCompileCommand(new File(srcDir, "GetString.cpp"), compiler, null, null))); + executorCalls.verify(defaultExecutor).setWorkingDirectory(new File(tempDir, "src")); + executorCalls.verify(defaultExecutor) + .execute(ziggyCppObject.new CommandLineComparable(ziggyCppObject.generateCompileCommand( + new File(srcDir, "ZiggyCppMain.cpp"), compiler, null, null))); + executorCalls.verify(defaultExecutor).setWorkingDirectory(new File(buildDir, "obj")); + executorCalls.verify(defaultExecutor) + .execute( + ziggyCppObject.new CommandLineComparable(ziggyCppObject.generateLinkCommand())); + + // and once more for a static library + // create a new object for linking a shared object + ziggyCppObject = createZiggyCppObject(buildDir); + ziggyCppObject.setOutputName("testOutput"); + ziggyCppObject.setOutputType("static"); + ziggyCppObject.setDefaultExecutor(defaultExecutor); + ziggyCppObject.action(); + executorCalls.verify(defaultExecutor).setWorkingDirectory(new File(tempDir, "src")); + executorCalls.verify(defaultExecutor) + .execute(ziggyCppObject.new CommandLineComparable(ziggyCppObject + .generateCompileCommand(new File(srcDir, "GetString.cpp"), compiler, null, null))); + executorCalls.verify(defaultExecutor).setWorkingDirectory(new File(tempDir, "src")); + executorCalls.verify(defaultExecutor) + .execute(ziggyCppObject.new CommandLineComparable(ziggyCppObject.generateCompileCommand( + new File(srcDir, "ZiggyCppMain.cpp"), compiler, null, null))); + executorCalls.verify(defaultExecutor).setWorkingDirectory(new File(buildDir, "obj")); + executorCalls.verify(defaultExecutor) + .execute( + ziggyCppObject.new CommandLineComparable(ziggyCppObject.generateLinkCommand())); + } + + // The following tests exercise various error conditions that return GradleExceptions. There are + // 2 error + // conditions that are not tested by these methods, they occur when the DefaultExecutor throws + // an IOException. + // These cases are considered sufficiently trivial that we omit them. + + /** + * Tests that a null value for the C++ file path produces the correct error. + */ + @Test + public void testCppFilePathNullError() { + ZiggyCppPojo ziggyCppError = new ZiggyCppPojo(); + assertThrows("C++ file path is null", GradleException.class, () -> { + ziggyCppError.getCppFiles(); + }); + } + + /** + * Tests that a nonexistent C++ file path produces the correct warning message. + */ + @Test + public void testCppFilePathDoesNotExist() { + ZiggyCppPojo ziggyCppError = new ZiggyCppPojo(); + ziggyCppError.setCppFilePath("/this/path/does/not/exist"); + ziggyCppError.getCppFiles(true); + List w = ziggyCppError.loggerWarnings(); + assertTrue(w.size() > 0); + assertTrue(w.contains("C++ file path /this/path/does/not/exist does not exist")); + } + + /** + * Tests that a missing output name produces the correct error. + */ + @Test + public void testErrorNoOutputName() { + ZiggyCppPojo ziggyCppError = new ZiggyCppPojo(); + ziggyCppError.setCppFilePath(tempDir.getAbsolutePath() + "/src"); + ziggyCppError.setBuildDir(buildDir); + ziggyCppError.setOutputType("executable"); + assertThrows("Both output name and output type must be specified", GradleException.class, + () -> { + ziggyCppError.getBuiltFile(); + }); + } + + /** + * Tests that a missing output type produces the correct error. + */ + @Test + public void testErrorNoOutputType() { + ZiggyCppPojo ziggyCppError = new ZiggyCppPojo(); + ziggyCppError.setCppFilePath(tempDir.getAbsolutePath() + "/src"); + ziggyCppError.setBuildDir(buildDir); + ziggyCppError.setOutputName("dummy"); + assertThrows("Both output name and output type must be specified", GradleException.class, + () -> { + ziggyCppError.getBuiltFile(); + }); + } + + /** + * Tests that missing both the output type and the built file name produces the correct error. + */ + @Test + public void testErrorNoOutputTypeOrName() { + ZiggyCppPojo ziggyCppError = new ZiggyCppPojo(); + new ArrayList<>(); + ziggyCppError.setCppFilePath(tempDir.getAbsolutePath() + "/src"); + ziggyCppError.setBuildDir(buildDir); + assertThrows("Both output name and output type must be specified", GradleException.class, + () -> { + ziggyCppError.getBuiltFile(); + }); + } + + /** + * Tests that a non-zero compiler return value produces the correct error. + * + * @throws ExecuteException + * @throws IOException + */ + @Test + public void testCompilerError() throws ExecuteException, IOException { + ziggyCppObject.setOutputType("executable"); + ziggyCppObject.setDefaultExecutor(defaultExecutor); + when(defaultExecutor.execute(any(CommandLine.class))).thenReturn(1); + assertThrows("Compilation of file GetString.cpp failed", GradleException.class, () -> { + ziggyCppObject.action(); + }); + } + + /** + * Tests that a non-zero linker return value produces the correct error. + * + * @throws ExecuteException + * @throws IOException + */ + @Ignore + public void testLinkerError() throws ExecuteException, IOException { + ziggyCppObject.setOutputType("executable"); + ziggyCppObject.setDefaultExecutor(defaultExecutor); + String linkerCommand = ziggyCppObject.generateLinkCommand() + + "GetString.o ZiggyCModule.o ZiggyCppMain.o"; + when(defaultExecutor.execute(ziggyCppObject.new CommandLineComparable(linkerCommand))) + .thenReturn(1); + // TODO Fix flaky test + // There seems to be a timing issue between these two statements. On my Linux workstation + // anyway. + assertThrows("Link / library construction of dummy failed", GradleException.class, () -> { + ziggyCppObject.action(); + }); + } //*************************************************************************************** - - // here begins assorted setup and helper methods - - /** - * Creates four source files for use in the test. Two are C++ files in the tempDir/src - * directory, ZiggyCppMain.cpp and GetString.cpp. One is the header file ZiggyCppMain.h, - * also in tempDir/src. The final one is ZiggyCppLib.h, in tempdir/include. - * @throws FileNotFoundException - */ - public void createSourceFiles() throws FileNotFoundException { - - // content for ZiggyCppMain.cpp - String[] mainSourceContent = - {"#include \"ZiggyCppMain.h\"" , - "#include ", - "#include \"ZiggyCppLib.h", - "", - "using namespace std;", - "", - "int main(int argc, const char* argv[]) {", - " string s = getString();", - " cout << s << endl;", - "}" - }; - - // content for the getString function - String[] getStringContent = - {"#include \"ZiggyCppMain.h\"", - "", - "using namespace std;", - "", - "string getString() {", - " return string(\"hello world!\");", - "}" - }; - - // content for the ZiggyCppMain.h header - String[] ziggyCppMainHeaderContent = - {"#ifndef ZIGGY_CPP", - "#define ZIGGY_CPP", - "#endif", - "", - "string getString();" - }; - - // content for the ZiggyCppLib.h header - String[] ziggyCppLibHeaderContent = - {"#ifndef ZIGGY_CPP_LIB", - "#define ZIGGY_CPP_LIB", - "#endif" - }; - - // create the source files first - PrintWriter mainSource = new PrintWriter(tempDir.getAbsolutePath() + "/src/ZiggyCppMain.cpp"); - for (String line : mainSourceContent) { - mainSource.println(line); - } - mainSource.close(); - - PrintWriter getStringSource = new PrintWriter(tempDir.getAbsolutePath() + "/src/GetString.cpp"); - for (String line : getStringContent) { - getStringSource.println(line); - } - getStringSource.close(); - - // put the main header in the src directory - PrintWriter h1 = new PrintWriter(tempDir.getAbsolutePath() + "/src/ZiggyCppMain.h"); - for (String line : ziggyCppMainHeaderContent) { - h1.println(line); - } - h1.close(); - - // put the other header in the include directory - PrintWriter h2 = new PrintWriter(tempDir.getAbsolutePath() + "/include/ZiggyCppLib.h"); - for (String line : ziggyCppLibHeaderContent) { - h2.println(line); - } - h2.close(); - - } - - /** - * Add an additional source file in another directory to test accumulating files from multiple directories - * @throws FileNotFoundException - */ - public void createAdditionalSource() throws FileNotFoundException { - - // content for the getString function - String[] getStringContent = - {"#include \"ZiggyCppMain.h\"", - "", - "using namespace std;", - "", - "string getAnotherString() {", - " return string(\"hello again world!\");", - "}" - }; - - PrintWriter getStringSource = new PrintWriter(buildDir.getAbsolutePath() + "/src/cpp/GetAnotherString.cpp"); - for (String line : getStringContent) { - getStringSource.println(line); - } - getStringSource.close(); - - } - - /** - * Creates a ZiggyCpp object that is properly formed and has the following members populated: - * cppFilePath - * includeFilePaths - * compileOptions - * name - * Project (with a mocked Project object) - * @return properly-formed ZiggyCpp object - */ - @SuppressWarnings("serial") - public ZiggyCppPojo createZiggyCppObject(File buildDir) { - ZiggyCppPojo ziggyCppObject = new ZiggyCppPojo(); - ziggyCppObject.setCppFilePath(tempDir.getAbsolutePath() + "/src"); - ziggyCppObject.setIncludeFilePaths(new ArrayList(){{ - add(tempDir.getAbsolutePath() + "/src"); - add(tempDir.getAbsolutePath() + "/include"); - }}); - ziggyCppObject.setCompileOptions(new ArrayList() {{ - add("Wall"); - add("fPic"); - }}); - ziggyCppObject.setReleaseOptimizations(new ArrayList() {{ - add("O2"); - add("DNDEBUG"); - add("g"); - }}); - ziggyCppObject.setDebugOptimizations(new ArrayList() {{ - add("Og"); - add("g"); - }}); - ziggyCppObject.setOutputName("dummy"); - ziggyCppObject.setBuildDir(buildDir); - ziggyCppObject.setCppCompiler("/dev/null/g++"); - ziggyCppObject.setRootDir(rootDir); - return ziggyCppObject; - } - - public void testStringListSettersAndGetters(String fieldName, String[] initialValues) - throws NoSuchMethodException, IllegalAccessException, IllegalArgumentException, - InvocationTargetException { - String getter = "get" + fieldName; - String setter = "set" + fieldName; - int nOrigValues = initialValues.length; - Method getMethod = ZiggyCppPojo.class.getDeclaredMethod(getter); - Method setMethod = ZiggyCppPojo.class.getDeclaredMethod(setter, List.class); + + // here begins assorted setup and helper methods + + /** + * Creates four source files for use in the test. Two are C++ files in the tempDir/src + * directory, ZiggyCppMain.cpp and GetString.cpp. One is the header file ZiggyCppMain.h, also in + * tempDir/src. The final one is ZiggyCppLib.h, in tempdir/include. + * + * @throws FileNotFoundException + */ + public void createSourceFiles() throws FileNotFoundException { + + // content for ZiggyCppMain.cpp + String[] mainSourceContent = { "#include \"ZiggyCppMain.h\"", "#include ", + "#include \"ZiggyCppLib.h", "", "using namespace std;", "", + "int main(int argc, const char* argv[]) {", " string s = getString();", + " cout << s << endl;", "}" }; + + // content for the getString function + String[] getStringContent = { "#include \"ZiggyCppMain.h\"", "", "using namespace std;", "", + "string getString() {", " return string(\"hello world!\");", "}" }; + + // content for the ZiggyCppMain.h header + String[] ziggyCppMainHeaderContent = { "#ifndef ZIGGY_CPP", "#define ZIGGY_CPP", "#endif", + "", "string getString();", + "void get_sort_index(int nLength, double *inputArray, int *sortIndex, int *iStack );" }; + + // content for the ZiggyCppLib.h header + String[] ziggyCppLibHeaderContent = { "#ifndef ZIGGY_CPP_LIB", "#define ZIGGY_CPP_LIB", + "#endif" }; + + // Content for the ZiggyCModule.c function + String[] ziggyCModuleContent = { "#define ", "#define ", + "void get_sort_index(int nLength, double *inputArray, int *sortIndex, int *iStack ) {", + "}" }; + + // create the source files first + PrintWriter mainSource = new PrintWriter( + tempDir.getAbsolutePath() + "/src/ZiggyCppMain.cpp"); + for (String line : mainSourceContent) { + mainSource.println(line); + } + mainSource.close(); + + PrintWriter getStringSource = new PrintWriter( + tempDir.getAbsolutePath() + "/src/GetString.cpp"); + for (String line : getStringContent) { + getStringSource.println(line); + } + getStringSource.close(); + + PrintWriter cModuleSource = new PrintWriter( + tempDir.getAbsolutePath() + "/src/ZiggyCModule.c"); + for (String line : ziggyCModuleContent) { + cModuleSource.println(line); + } + cModuleSource.close(); + + // put the main header in the src directory + PrintWriter h1 = new PrintWriter(tempDir.getAbsolutePath() + "/src/ZiggyCppMain.h"); + for (String line : ziggyCppMainHeaderContent) { + h1.println(line); + } + h1.close(); + + // put the other header in the include directory + PrintWriter h2 = new PrintWriter(tempDir.getAbsolutePath() + "/include/ZiggyCppLib.h"); + for (String line : ziggyCppLibHeaderContent) { + h2.println(line); + } + h2.close(); + } + + /** + * Add an additional source file in another directory to test accumulating files from multiple + * directories + * + * @throws FileNotFoundException + */ + public void createAdditionalSource() throws FileNotFoundException { + + // content for the getString function + String[] getStringContent = { "#include \"ZiggyCppMain.h\"", "", "using namespace std;", "", + "string getAnotherString() {", " return string(\"hello again world!\");", "}" }; + + PrintWriter getStringSource = new PrintWriter( + buildDir.getAbsolutePath() + "/src/cpp/GetAnotherString.cpp"); + for (String line : getStringContent) { + getStringSource.println(line); + } + getStringSource.close(); + } + + /** + * Creates a ZiggyCpp object that is properly formed and has the following members populated: + * cppFilePath includeFilePaths compileOptions name Project (with a mocked Project object) + * + * @return properly-formed ZiggyCpp object + */ + public ZiggyCppPojo createZiggyCppObject(File buildDir) { + ZiggyCppPojo ziggyCppObject = new ZiggyCppPojo(); + ziggyCppObject.setCppFilePath(tempDir.getAbsolutePath() + "/src"); + ziggyCppObject.setIncludeFilePaths( + List.of(tempDir.getAbsolutePath() + "/src", tempDir.getAbsolutePath() + "/include")); + ziggyCppObject.setCppCompileOptions(List.of("-Wall", "-fPic")); + ziggyCppObject.setCCompileOptions(List.of("-fPic")); + ziggyCppObject.setReleaseOptimizations(List.of("-O2", "-DNDEBUG", "-g")); + ziggyCppObject.setDebugOptimizations(List.of("-Og", "-g")); + ziggyCppObject.setOutputName("dummy"); + ziggyCppObject.setBuildDir(buildDir); + ZiggyCppPojo.setCppCompiler("/dev/null/g++"); + ZiggyCppPojo.setCCompiler("/dev/null/gcc"); + ziggyCppObject.setRootDir(rootDir); + ziggyCppObject.setMaxCompileThreads(1); + return ziggyCppObject; + } + + public void testStringListSettersAndGetters(String fieldName, String[] initialValues) + throws NoSuchMethodException, IllegalAccessException, IllegalArgumentException, + InvocationTargetException { + String getter = "get" + fieldName; + String setter = "set" + fieldName; + int nOrigValues = initialValues.length; + Method getMethod = ZiggyCppPojo.class.getDeclaredMethod(getter); + Method setMethod = ZiggyCppPojo.class.getDeclaredMethod(setter, List.class); List initialGetValues = List.class.cast(getMethod.invoke(ziggyCppObject)); - assertEquals(nOrigValues, initialGetValues.size()); - for (int i=0 ; i replacementValues = new ArrayList<>(); - replacementValues.add("R1"); - replacementValues.add("R2"); - setMethod.invoke(ziggyCppObject, replacementValues); - List replacementGetValues = List.class.cast(getMethod.invoke(ziggyCppObject)); - assertEquals(replacementValues.size(), replacementGetValues.size()); - for (int i=0 ; i linkerOptions = new ArrayList<>(); - linkerOptions.add("u whatevs"); - ziggyCppObject.setLinkOptions(linkerOptions); - List libraryPathOptions = new ArrayList<>(); - libraryPathOptions.add("/dummy1/lib"); - libraryPathOptions.add("/dummy2/lib"); - ziggyCppObject.setLibraryPaths(libraryPathOptions); - List libraryOptions = new ArrayList<>(); - libraryOptions.add("hdf5"); - libraryOptions.add("netcdf"); - ziggyCppObject.setLibraries(libraryOptions); - - } + assertEquals(nOrigValues, initialGetValues.size()); + for (int i = 0; i < nOrigValues; i++) { + assertEquals(initialValues[i], initialGetValues.get(i)); + } + + List replacementValues = new ArrayList<>(); + replacementValues.add("R1"); + replacementValues.add("R2"); + setMethod.invoke(ziggyCppObject, replacementValues); + List replacementGetValues = List.class.cast(getMethod.invoke(ziggyCppObject)); + assertEquals(replacementValues.size(), replacementGetValues.size()); + for (int i = 0; i < replacementValues.size(); i++) { + assertEquals(replacementValues.get(i), replacementGetValues.get(i)); + } + } + + /** + * Set up linker options. This needs to be done for several ZiggyCppPojo instances so we can + * test all of the different link types (shared, static, executable). + * + * @param ziggyCppObject object in need of link options. + */ + public void configureLinkerOptions(ZiggyCppPojo ziggyCppObject) { + // first we need to add some object files + File o1 = new File(buildDir, "obj/o1.o"); + File o2 = new File(buildDir, "obj/o2.o"); + ziggyCppObject.setObjectFiles(o1); + ziggyCppObject.setObjectFiles(o2); + + // also some linker options and libraries + List linkerOptions = new ArrayList<>(); + linkerOptions.add("-u whatevs"); + ziggyCppObject.setLinkOptions(linkerOptions); + List libraryPathOptions = new ArrayList<>(); + libraryPathOptions.add("/dummy1/lib"); + libraryPathOptions.add("/dummy2/lib"); + ziggyCppObject.setLibraryPaths(libraryPathOptions); + List libraryOptions = new ArrayList<>(); + libraryOptions.add("hdf5"); + libraryOptions.add("netcdf"); + ziggyCppObject.setLibraries(libraryOptions); + } } diff --git a/doc/user-manual/alerts.md b/doc/user-manual/alerts.md index 9afe15e..fea19f9 100644 --- a/doc/user-manual/alerts.md +++ b/doc/user-manual/alerts.md @@ -8,11 +8,11 @@ Ziggy uses alerts to tell the pipeline operator that something has happened that they ought to know about. Alerts are displayed on the `Alerts` status panel. It looks like this: -![](images/monitoring-alerts.png) + -There are two flavors of alert that you're likely to see: warnings and errors. Warnings will turn the alerts stoplight yellow, errors turn it red. The alerts panel shows which task generated the alert, when it happened, and a hopefully-useful message. If there are no alerts, the stoplight will be green. +There are two flavors of alert that you're likely to see: warnings and errors. Warnings will turn the alerts stoplight yellow, errors turn it red. The alerts panel shows which task generated the alert, when it happened, and a hopefully-useful message. If there are no alerts, the stoplight will be green. -Sadly, in this case it tells you pretty much what you already knew: task 12 blew up. +Sadly, in this case it tells you pretty much what you already knew: task 12 blew up. ### Acknowledging Alerts @@ -20,7 +20,7 @@ Once an alert arrives, the stoplight color will stay whatever color is appropria ### Clearing Alerts -Alternately, once an alert is addressed you may want to get it completely out of the table of alerts. The `Clear` button will clear all alerts from the display and return the stoplight to green. +Alternately, once an alert is addressed you may want to get it completely out of the table of alerts. The `Clear` button will clear all alerts from the display and return the stoplight to green. [[Previous]](monitoring.md) [[Up]](ziggy-gui-troubleshooting.md) diff --git a/doc/user-manual/building-pipeline.md b/doc/user-manual/building-pipeline.md index 8962ed0..8199084 100644 --- a/doc/user-manual/building-pipeline.md +++ b/doc/user-manual/building-pipeline.md @@ -2,7 +2,7 @@ [[Previous]](pipeline-definition.md) [[Up]](user-manual.md) -[[Next]](rdbms.md) +[[Next]](running-pipeline.md) ## Building a Pipeline @@ -19,17 +19,17 @@ Just in case you haven't looked yet, here's what the sample pipeline directory s ```console sample-pipeline$ ls build-env.sh config data etc multi-data src -sample-pipeline$ +sample-pipeline$ ``` -As we've discussed, the `src` directory is the various bits of source code for the pipeline, `etc` is the location of the pipeline properties file, `config` is the location of the XML files that define the pipeline. The `data` is the initial source of the data files that will be used by the sample pipeline. +As we've discussed, the `src` directory is the various bits of source code for the pipeline, `etc` is the location of the pipeline properties file, `config` is the location of the XML files that define the pipeline. The `data` is the initial source of the data files that will be used by the sample pipeline. -At the top you can see `build-env.sh`, which is the "build system" for the sample pipeline. In this case, the sample pipeline is so simple that none of the grown-up build systems were seen as needed or even desirable; a shell script would do what was needed. +At the top you can see `build-env.sh`, which is the "build system" for the sample pipeline. In this case, the sample pipeline is so simple that none of the grown-up build systems were seen as needed or even desirable; a shell script would do what was needed. If you run the shell script from the command line (`./build-env.sh`), you should quickly see something that looks like this: ```console -sample-pipeline$ /bin/bash ./build-env.sh +sample-pipeline$ /bin/bash ./build-env.sh Collecting h5py Using cached h5py-3.7.0-cp38-cp38-macosx_10_9_x86_64.whl (3.2 MB) Collecting Pillow @@ -38,7 +38,7 @@ Collecting numpy Using cached numpy-1.23.4-cp38-cp38-macosx_10_9_x86_64.whl (18.1 MB) Installing collected packages: Pillow, numpy, h5py Successfully installed Pillow-9.2.0 h5py-3.7.0 numpy-1.23.4 -sample-pipeline$ +sample-pipeline$ ``` @@ -50,10 +50,10 @@ sample-pipeline$ ls build build-env.sh config data etc multi-data src sample-pipeline$ ls build bin env pipeline-results -sample-pipeline$ +sample-pipeline$ ``` -There's now a `build` directory that contains additional directories: `bin, data-receipt`, and `env`. +There's now a `build` directory that contains additional directories: `bin, data-receipt`, and `env`. #### The build-env.sh Shell Script @@ -87,7 +87,7 @@ mkdir -p $DATA_RECEIPT_DIR cp $sample_home/data/* $DATA_RECEIPT_DIR ``` -Here we create the `build` directory and its `env` and `data-receipt` directories. The contents of the data directory from the sample directory gets copied to `data-receipt`. +Here we create the `build` directory and its `env` and `data-receipt` directories. The contents of the data directory from the sample directory gets copied to `data-receipt`. ```bash # build the bin directory in build @@ -103,7 +103,7 @@ cp $BIN_SRC_DIR/averaging.sh $BIN_DIR/averaging chmod -R a+x $BIN_DIR ``` -Here we construct `build/bin` and copy the shell scripts from `src/main/sh` to `build/bin`. In the process, we strip off the `.sh` suffixes. The shell script copies in `build/bin` now match what Ziggy expects to see. +Here we construct `build/bin` and copy the shell scripts from `src/main/sh` to `build/bin`. In the process, we strip off the `.sh` suffixes. The shell script copies in `build/bin` now match what Ziggy expects to see. ```bash python3 -m venv $SAMPLE_PIPELINE_PYTHON_ENV @@ -130,23 +130,23 @@ cp -r $ZIGGY_HOME/src/main/python/zigutils $SITE_PKGS exit 0 ``` -Here at last we build the Python environment that will be used for the sample pipeline. The environment is built in `build/env`, and the packages `h5py`, `Pillow`, and `numpy` are installed in the environment. Finally, the sample pipeline source code and the Ziggy utility modules are copied to the `site-packages` directory of the environment, so that they will be on Python's search path when the environment is activated. +Here at last we build the Python environment that will be used for the sample pipeline. The environment is built in `build/env`, and the packages `h5py`, `Pillow`, and `numpy` are installed in the environment. Finally, the sample pipeline source code and the Ziggy utility modules are copied to the `site-packages` directory of the environment, so that they will be on Python's search path when the environment is activated. ### Okay, but Why? -Setting up the `build` directory this way has a number of advantages. First and foremost, everything that Ziggy will eventually need to find is someplace in `build`. If your experience with computers is anything like mine, you know that 90% of what we spend our time doing is figuring out why various search paths aren't set correctly. Putting everything you need in one directory minimizes this issue. +Setting up the `build` directory this way has a number of advantages. First and foremost, everything that Ziggy will eventually need to find is someplace in `build`. If your experience with computers is anything like mine, you know that 90% of what we spend our time doing is figuring out why various search paths aren't set correctly. Putting everything you need in one directory minimizes this issue. -The second advantage is related: if you decide that you need to clean up and start over, you can simply delete the `build` directory. You don't need to go all over the place looking for directories and deleting them. Indeed, most grown up build systems will include a command that will automatically perform this deletion. +The second advantage is related: if you decide that you need to clean up and start over, you can simply delete the `build` directory. You don't need to go all over the place looking for directories and deleting them. Indeed, most grown up build systems will include a command that will automatically perform this deletion. ### Why a Build System? -If you look at build-env.sh, you'll see that it's relatively simple: just 80 lines, including comments and whitespace lines. Nonetheless, you wouldn't want to have to type all this in every time you want to generate the `build` directory! Having a build system -- any build system -- allows you to ensure that the build is performed reproducibly -- every build is the same as all the ones before. It allows you to implement changes to the build in a systematic way. +If you look at build-env.sh, you'll see that it's relatively simple: just 80 lines, including comments and whitespace lines. Nonetheless, you wouldn't want to have to type all this in every time you want to generate the `build` directory! Having a build system -- any build system -- allows you to ensure that the build is performed reproducibly -- every build is the same as all the ones before. It allows you to implement changes to the build in a systematic way. ### Which Build System? -Good question! There are a lot of them out there. +Good question! There are a lot of them out there. -For simple systems, good old `make` is a solid choice. Support for `make` is nearly universal, and most computers ship with some version of it already loaded so you won't even need to install it yourself. +For simple systems, good old `make` is a solid choice. Support for `make` is nearly universal, and most computers ship with some version of it already loaded so you won't even need to install it yourself. For more complex systems, we're enamored of Gradle. The secret truth of build systems is that a build is actually a program: it describes the steps that need to be taken, their order, and provides conditionals of various kinds (like, "if the target is up-to-date, skip this step"). Gradle embraces this secret truth and runs with it, which makes it in many ways easier to use for bigger systems than `make`, with its frequently bizarre use of symbols and punctuation to represent relationships within the software. @@ -154,8 +154,6 @@ For more complex systems, we're enamored of Gradle. The secret truth of build sy Note that while going through all this exposition, we've also sneakily built the sample pipeline! This positions us for the next (exciting) step: [running the pipeline](running-pipeline.md)! -Unfortunately, before you get there, we'll need to talk some about [relational databases](rdbms.md). - [[Previous]](pipeline-definition.md) [[Up]](user-manual.md) -[[Next]](rdbms.md) +[[Next]](running-pipeline.md) diff --git a/doc/user-manual/change-param-values.md b/doc/user-manual/change-param-values.md index 4ff0cc0..8e979d8 100644 --- a/doc/user-manual/change-param-values.md +++ b/doc/user-manual/change-param-values.md @@ -2,13 +2,13 @@ [[Previous]](start-end-nodes.md) [[Up]](ziggy-gui.md) -[[Next]](intermediate-topics.md) +[[Next]](organizing-tables.md) ## Changing Module Parameter Values -To see how this works, go back to the `Configuration` tab and select the `Parameter Library` option from the menu on the left hand side. You'll see this: +Module parameters can be changed to affect how the algorithm behaves. To see how this works, select the `Parameter Library` content menu item. You'll see this: -![](images/parameter-library.png) + As you can see, all of the parameter sets defined in `pl-sample.xml` are represented here. What else does the table tell us? @@ -18,20 +18,20 @@ As you can see, all of the parameter sets defined in `pl-sample.xml` are represe Now: double-click the Algorithm Parameters row in the table. You'll get a new dialog box: - + The parameters that were defined as booleans in `pl-sample.xml` have check boxes you can check or uncheck. The other parameter types mostly behave the way you expect, but the array types offer some additional capabilities. If you click the `dummy array parameter` parameter, it will change thusly: - + If you click the "X", all the values will be deleted, which is rarely what you want. Instead click the other button. You'll get this window: - + -This allows you to edit the array elements, remove them, add elements, etc., in a more GUI-natural way. Go ahead and change the second element (`idx` of 1) to 4. Click `ok`, then on the Edit Parameter Set dialog click `Save`. The `Version` for Algorithm Parameters will now be set to 1, and the `Locked` checkbox is unchecked. +This allows you to edit the array elements, remove them, add elements, etc., in a more GUI-natural way. Go ahead and change the second element (`Element` 1 reflecting the zero-based nature of Java arrays) to 4 from 2. Press the `OK` button and then press the `Save` button on the Edit parameter set dialog. The `Version` for Algorithm Parameters will now be set to 1, and the `Locked` checkbox is unchecked. If you were to now run the sample pipeline, when you returned to the parameter library window, version 1 of `Algorithm Parameters` will show as locked. [[Previous]](start-end-nodes.md) [[Up]](ziggy-gui.md) -[[Next]](intermediate-topics.md) +[[Next]](organizing-tables.md) diff --git a/doc/user-manual/configuring-pipeline.md b/doc/user-manual/configuring-pipeline.md index 4dfadfb..7940071 100644 --- a/doc/user-manual/configuring-pipeline.md +++ b/doc/user-manual/configuring-pipeline.md @@ -201,8 +201,6 @@ Again, really simple: activate the environment; find the location of the environ All of the above has focused on the permuter algorithm, but the same pattern of "glue" files and design patterns are used for the flip and averaging algorithms. -All of the above has focused on the permuter algorithm, but the same pattern of "glue" files and design patterns are used for the flip and averaging algorithms. - ### Set up the Properties File As you can probably imagine, Ziggy actually uses a lot of configuration items: it needs to know numerous paths around your file system, which relational database application you want to use, how much heap space to provide to Ziggy, and on and on. All of this stuff is put into two locations for Ziggy: the pipeline properties file and the Ziggy properties file. @@ -229,8 +227,6 @@ Answer: the sample properties file sets all its paths relative to the top-level Anyway: set those properties now, before going any further. -Anyway: set those properties now, before going any further. - #### What About the Ziggy Properties File? The sample properties file contains a property that is the location of the Ziggy properties file. Thus there's no need to have a separate environment variable for that information. Like we said, to the extent possible we've put everything configuration related into the pipeline properties file. @@ -246,6 +242,11 @@ That said, if you're paying attention you've probably noticed that this article These questions will be discussed in the article on [Building a Pipeline](building-pipeline.md). + + [[Previous]](downloading-and-building-ziggy.md) [[Up]](user-manual.md) [[Next]](module-parameters.md) diff --git a/doc/user-manual/contact-us.md b/doc/user-manual/contact-us.md index 135fd62..c65b6cd 100644 --- a/doc/user-manual/contact-us.md +++ b/doc/user-manual/contact-us.md @@ -1,7 +1,8 @@ -[[Previous]](edit-trigger.md) +[[Previous]](edit-pipeline.md) [[Up]](user-manual.md) +[[Next]](properties.md) ## Contact Us @@ -11,8 +12,9 @@ If you have implemented fixes to those problems and improvements, then please su If you just want to send an email, we are: -Peter Tenenbaum (but you can call him PT) +Peter Tenenbaum (but you can call him PT)
Bill Wohler -[[Previous]](edit-trigger.md) +[[Previous]](edit-pipeline.md) [[Up]](user-manual.md) +[[Next]](properties.md) diff --git a/doc/user-manual/customizing-ziggy.md b/doc/user-manual/customizing-ziggy.md new file mode 100644 index 0000000..ec89cb6 --- /dev/null +++ b/doc/user-manual/customizing-ziggy.md @@ -0,0 +1,63 @@ + + +[[Previous]](nicknames.md) +[[Up]](dusty-corners.md) +[[Next]](contact-us.md) + +## Customizing Ziggy + +The [Remote Parameters](remote-parameters.md) article discussed a Java class in Ziggy called `gov.nasa.ziggy.module.remote.RemoteParameters`. You can define your own parameter sets if the `DefaultParameters` is insufficient for your needs. This article discusses how you can do this and more. + +### Adding the Ziggy Dependency to Your Build + +First, you have to add dependencies to the Ziggy libraries to your build. To do this, run `./gradlew publish` in the Ziggy main directory. Then, add the following to your `build.gradle` file. Your build now has access to the [entire Ziggy Java API](link-to-Javadocs.html). + +```groovy +repositories { + mavenCentral() + maven { + url System.getenv('ZIGGY_HOME') + "/repository" + metadataSources { + gradleMetadata() + } + } +} +dependencies { + implementation 'gov.nasa:ziggy:+' +} +``` + +### Adding a Custom Parameter Class + +### Other Customizations You Can Make + +List the other things that TESS does to give the reader some ideas. + +### Using Ziggy Tools in Your Build + +Ziggy has a handful of tools to help build code. A few of the more popular tools include ZiggyCpp for compiling C++, ZiggyCppMex for MATLAB fans, and `ZiggySchemaExport` for exporting database schemas from Hibernate classes. For more information, please refer to the [Ziggy buildSrc Java API](link-to-Javadocs.html). + +If you want to use any of these tools, add the following to your `build.gradle` file. This also requires that you run `./gradlew publish` in the Ziggy main directory. + +```groovy +buildscript { + repositories { + mavenCentral() + maven { + url System.getenv('ZIGGY_HOME') + "/repository" + metadataSources { + gradleMetadata() + } + } + } + dependencies { + classpath 'gov.nasa:ziggy-buildSrc:+' + } +} +``` + +The API contains examples for using these tools. + +[[Previous]](nicknames.md) +[[Up]](dusty-corners.md) +[[Next]](contact-us.md) diff --git a/doc/user-manual/data-receipt-display.md b/doc/user-manual/data-receipt-display.md index cbd34e2..9b722a7 100644 --- a/doc/user-manual/data-receipt-display.md +++ b/doc/user-manual/data-receipt-display.md @@ -6,15 +6,15 @@ ## Data Receipt Display -The console has the ability to display data receipt activities. From the `Configuration` tab, expand the `Data Receipt` folder and select `Available Datasets`. You'll see something like this: +The console has the ability to display data receipt activities. Select the `Data Receipt` content menu item. You'll see something like this: -![](images/data-receipt-display.png) + Double-clicking a row in the table brings up a display of all the files in the dataset: -![](images/data-receipt-list.png) + -Note that the file names are the datastore names. +Note that the file names are the datastore names. [[Previous]](data-receipt-execution.md) [[Up]](data-receipt.md) diff --git a/doc/user-manual/data-receipt-execution.md b/doc/user-manual/data-receipt-execution.md index cdedb33..437f87a 100644 --- a/doc/user-manual/data-receipt-execution.md +++ b/doc/user-manual/data-receipt-execution.md @@ -12,10 +12,10 @@ At the highest level, the purpose of data receipt is to take files delivered fro - All of the files that you expected to receive actually showed up. - No files showed up that are not expected. - The files that showed up were not somehow corrupted in transit. -- Whoever it was that delivered the files may require a notification that there were no problems with the delivery, so data receipt needs to produce something that can function as the notification. -- The data receipt process needs to clean up after itself. In a nutshell, this means that there is no chance that a future data receipt operation fails because of some debris left from a prior data receipt operation, and that there is no chance that a future data receipt operation will inadvertently re-import files that were already imported. +- Whoever it was that delivered the files may require a notification that there were no problems with the delivery, so data receipt needs to produce something that can function as the notification. +- The data receipt process needs to clean up after itself. In a nutshell, this means that there is no chance that a future data receipt operation fails because of some debris left from a prior data receipt operation, and that there is no chance that a future data receipt operation will inadvertently re-import files that were already imported. -The integrity of the delivery is supported by an XML file, the *manifest*, that lists all of the delivered files and contains size and checksum information for each one. After a successful import, Ziggy produces an XML file, the *acknowledgement*, that can be used as a notification to the source of the files that the files were delivered and imported without incident. The cleanup is managed algorithmically by Ziggy. +The integrity of the delivery is supported by an XML file, the *manifest*, that lists all of the delivered files and contains size and checksum information for each one. After a successful import, Ziggy produces an XML file, the *acknowledgement*, that can be used as a notification to the source of the files that the files were delivered and imported without incident. The cleanup is managed algorithmically by Ziggy. With that, let's dive in! @@ -68,7 +68,7 @@ Each file has a `checksum` that's computed using the specified `checksumType`. ### The Data Receipt Directory -Data receipt needs to have a directory that's used as the source for files that get pulled into the datastore. There's a [property in the properties file](properties.md) that specifies this, namely `data.receipt.dir`. Ziggy allows this directory to be used in either of two ways. +Data receipt needs to have a directory that's used as the source for files that get pulled into the datastore. There's a [property in the properties file](properties.md) that specifies this, namely `ziggy.pipeline.data.receipt.dir`. Ziggy allows this directory to be used in either of two ways. #### Files in the Data Receipt Directory @@ -166,7 +166,7 @@ Note that the manifest ignores the fact that import of data is going to treat th ### Generating Manifests -Ziggy also comes with a utility to generate manifests from the contents of a directory. Use `runjava generate-manifest`. This utility takes 3 command-line arguments: +Ziggy also comes with a utility to generate manifests from the contents of a directory. Use `ziggy generate-manifest`. This utility takes 3 command-line arguments: 1. Manifest name, required. 2. Dataset ID, required. diff --git a/doc/user-manual/data-receipt.md b/doc/user-manual/data-receipt.md index 32b8f57..a9aeda4 100644 --- a/doc/user-manual/data-receipt.md +++ b/doc/user-manual/data-receipt.md @@ -1,21 +1,25 @@ -[[Previous]](delete-tasks.md) +[[Previous]](hpc-cost.md) [[Up]](user-manual.md) [[Next]](data-receipt-execution.md) ## Data Receipt -In [the article on pipeline definition](pipeline-definition.md), we mentioned that Ziggy has a special-purpose pipeline module, Data Receipt, that imports data and models into the datastore and which is the only pipeline module the user doesn't need to define. In [the article on starting a pipeline](start-pipeline.md), we led you through starting the sample pipeline, which starts with the data receipt module. +In [the article on pipeline definition](pipeline-definition.md), we mentioned that Ziggy has a special-purpose pipeline module, Data Receipt, that imports data and models into the datastore and which is the only pipeline module the user doesn't need to define. In [the article on starting a pipeline](start-pipeline.md), we led you through starting the sample pipeline, which starts with the data receipt module. -So far, so good. However, you've probably noticed that at no point have we told you enough about the details of data receipt for you to feel comfortable putting it into a pipelilne of your own. +So far, so good. However, you've probably noticed that at no point have we told you enough about the details of data receipt for you to feel comfortable putting it into a pipeline of your own. Let's fix that now. -[Execution Flow](data-receipt-execution.md) -- covers the details of manifests, validation, the data receipt directory, and much, much more. +### [Data Receipt Execution Flow](data-receipt-execution.md) -[Console Display](data-receipt-display) -- Ziggy provides a panel on the console that shows all of data receipt's activities going back to the beginning of time. +This article covers the details of manifests, validation, the data receipt directory, and much, much more. -[[Previous]](delete-tasks.md) +### [Data Receipt Display](data-receipt-display) + +Ziggy provides a panel on the console that shows all of data receipt's activities going back to the beginning of time. + +[[Previous]](hpc-cost.md) [[Up]](user-manual.md) [[Next]](data-receipt-execution.md) diff --git a/doc/user-manual/datastore-task-dir.md b/doc/user-manual/datastore-task-dir.md index cf65b49..801de3e 100644 --- a/doc/user-manual/datastore-task-dir.md +++ b/doc/user-manual/datastore-task-dir.md @@ -14,9 +14,9 @@ Before we can look at the datastore, we need to find it! Fortunately, we can ref ``` pipeline.root.dir = ${ziggy.root}/sample-pipeline -pipeline.home.dir = ${pipeline.root.dir}/build -pipeline.results.dir = ${pipeline.home.dir}/pipeline-results -datastore.root.dir = ${pipeline.results.dir}/datastore +ziggy.pipeline.home.dir = ${pipeline.root.dir}/build +ziggy.pipeline.results.dir = ${ziggy.pipeline.home.dir}/pipeline-results +ziggy.pipeline.datastore.dir = ${ziggy.pipeline.results.dir}/datastore ``` Well, you don't see all of those lines laid out as conveniently as the above, but trust me, they're all there. Anyway, what this is telling us is that Ziggy's data directories are in `build/pipeline-results/datastore`. Looking at that location we see this: @@ -104,7 +104,7 @@ When Ziggy runs algorithm code, it doesn't the algorithm direct access to files To find the task directory, look first to the pipeline results location in the properties file: ``` -pipeline.results.dir=${pipeline.home.dir}/pipeline-results +ziggy.pipeline.results.dir=${ziggy.pipeline.home.dir}/pipeline-results ``` The pipeline-results directory contains a number of subdirectories. First, let's look at task-data: @@ -181,7 +181,7 @@ st-0$ Ziggy allows the user to select whether to use actual copies of the files or symbolic links. This is configured in -- yeah, you got it -- the properties file: ``` -moduleExe.useSymlinks = true +ziggy.pipeline.useSymlinks = true ``` The way this works is obvious for the input files: Ziggy puts a symlink in the working directory, and that's all there is to it. For the outputs file, what happens is that the algorithm produces an actual file of results; when Ziggy goes to store the outputs file, it moves it to the datastore and replaces it in the working directory with a symlink. This is a lot of words to say that you can turn this feature on or off at will and your code doesn't need to do anything different either way. @@ -193,7 +193,7 @@ The advantages of the symlinks are fairly obvious: There are also situations in which the symlinks may not be a good idea: -- It may be the case that you're using one computer to run the worker and database, and a different one to run the algorithms. In this situation, the datastore can be on a file system that's mounted on the worker machine but not the compute machine, in which case the symlink solution won't work (the compute node can't see the datastore, so it can't follow the link). +- It may be the case that you're using one computer to run the supervisor, workers, and database, and a different one to run the algorithms. In this situation, the datastore can be on a file system that's mounted on the supervisor machine but not the compute machine, in which case the symlink solution won't work (the compute node can't see the datastore, so it can't follow the link). [[Previous]](intermediate-topics.md) [[Up]](intermediate-topics.md) diff --git a/doc/user-manual/delete-tasks.md b/doc/user-manual/delete-tasks.md deleted file mode 100644 index 7133da3..0000000 --- a/doc/user-manual/delete-tasks.md +++ /dev/null @@ -1,35 +0,0 @@ - - -[[Previous]](rerun-task.md) -[[Up]](ziggy-gui-troubleshooting.md) -[[Next]](select-hpc.md) - -## Deleting Tasks - -Sometimes it's necessary to stop the execution of tasks after they start running. Tasks that are running as jobs under control of a batch system at an HPC facility will provide command line tools for this, but they're a hassle to use when you're trying to delete a large number of jobs. Trying to delete tasks running locally is likewise hassle-tastic. - -Fortunately, Ziggy will let you do this from the console. - -### Delete all Jobs for a Task - -To delete all jobs for a task, go to the tasks table on the Instances panel, right click the task, and select `Delete tasks` from the pop-up menu: - -![delete-task-menu-item](images/delete-task-menu-item.png) - -You'll be prompted to confirm that you want to delete the task. When you do that, you'll see something like this: - -![delete-in-progress](images/delete-in-progress.png) - -The state of the task will be immediately moved to `ERROR`, P-state `ALGORITHM_COMPLETE`. The instance will go to state `ERRORS_RUNNING` because the other task is still running; once it completes, the instance will go to `ERRORS_STALLED`. Meanwhile, the alert looks like this: - -![delete-alert](images/delete-alert.png) - -As expected, it notifies you that the task stopped due to deletion and not due to an error of some kind. - -### Delete all Tasks for an Instance - -This is the same idea, except it's the pop-up menu for the instance table, and you select `Delete all tasks`. - -[[Previous]](rerun-task.md) -[[Up]](ziggy-gui-troubleshooting.md) -[[Next]](select-hpc.md) diff --git a/doc/user-manual/display-logs.md b/doc/user-manual/display-logs.md index 750a256..6a55be2 100644 --- a/doc/user-manual/display-logs.md +++ b/doc/user-manual/display-logs.md @@ -8,21 +8,21 @@ Ziggy provides a mechanism for viewing task logs that is more convenient than going to the `logs` directory and hunting around. -To use this feature, go to the Instance tab under the Operations tab. Select the task of interest and right-click to bring up the tasks menu: +To use this feature, go to the instances panel, select the task of interest and right-click to bring up the tasks menu: - + Select `List task logs`. You'll get this dialog box: -![](images/logs-list.png) + By default the logs are ordered by name, which means that they're also ordered by time, from earliest to latest. If you double-click on one of the rows in the table, the log file in question will be opened in a new window: -![](images/task-log-display.png) + The log will always be opened with the view positioned at the end of the log, since that's most often where you can find messages that inform you about the problems. In this case, that's not true, so you can use the `To Top` button to jump to the start of the log, or simply scroll around until you find what you're looking for: -![](images/task-log-showing-problem.png) + Here you can see the stack trace produced by the Python algorithm when it deliberately threw an exception, and the Java stack trace that was generated when Ziggy detected that the algorithm had thrown an exception. As it happens, the log shows exactly what the problem is: the user set the parameter that tells subtask 0 to fail, and subtask 0 duly failed. diff --git a/doc/user-manual/downloading-and-building-ziggy.md b/doc/user-manual/downloading-and-building-ziggy.md index 06e4357..36ad69e 100644 --- a/doc/user-manual/downloading-and-building-ziggy.md +++ b/doc/user-manual/downloading-and-building-ziggy.md @@ -6,7 +6,7 @@ ## Downloading and Building Ziggy -Before you start, you should check out the [system requirements](system-requirements.md) article. This will ensure that you have the necessary hardware and software to follow the steps in this article. +Before you start, you should check out the [system requirements](system-requirements.md) article. This will ensure that you have the necessary hardware and software to follow the steps in this article. ### Downloading Ziggy @@ -20,7 +20,7 @@ LICENSE.pdf doc gradlew script-plugins README.md etc ide settings.gradle build.gradle gradle licenses src buildSrc gradle.properties sample-pipeline test -ziggy$ +ziggy$ ``` Let's go through these items: @@ -28,13 +28,13 @@ Let's go through these items: - The files `build.gradle`, `settings.gradle`, `gradle.properties`, and `gradlew` are used by our build system. Hopefully you won't need to know anything more about them than that. - Likewise, directories `gradle` and `script-plugins` are used in the build. - The `buildSrc` directory contains some Java and Groovy classes that are part of Ziggy but are used by the build. The idea here is that Gradle allows users to extend it by defining new kinds of build tasks; those new kinds of build tasks are implemented as Java or Groovy classes and by convention are put into a "buildSrc" folder. This is something else you probably won't need to worry about; certainly not any time soon. -- The `doc` directory contains this user manual, plus a bunch of other, more technical documentation. +- The `doc` directory contains this user manual, plus a bunch of other, more technical documentation. - The `etc` directory contains files that are used as configuration inputs to various programs. This is things like: the file that tells the logger how to format text lines, and so on. Two of these are going to be particularly useful and important to you: the ziggy.properties file, and the pipeline.properties.EXAMPLE file. These files provide all of the configuration that Ziggy needs to locate executables, working directories, data storage, and etc. We'll go into a lot of detail on this at the appropriate time. -- The `ide` directory contains auxiliary files that are useful if you want to develop Ziggy in the Eclipse IDE. -- The `licenses` directory contains information about both the Ziggy license and the licenses of third-party applications and libraries that Ziggy uses. -- The `src` directory contains the directory tree of Ziggy source code, both main classes and test classes. -- The `test` directory contains test data for Ziggy's unit tests. -- The `sample-pipeline` directory contains the source and such for the sample pipeline. +- The `ide` directory contains auxiliary files that are useful if you want to develop Ziggy in the Eclipse IDE. +- The `licenses` directory contains information about both the Ziggy license and the licenses of third-party applications and libraries that Ziggy uses. +- The `src` directory contains the directory tree of Ziggy source code, both main classes and test classes. +- The `test` directory contains test data for Ziggy's unit tests. +- The `sample-pipeline` directory contains the source and such for the sample pipeline. ### Building Ziggy @@ -58,17 +58,17 @@ ziggy$ ./gradlew ar: creating archive ziggy/build/lib/libziggy.a ar: creating archive ziggy/build/lib/libziggymi.a -> Task :compileJava +> Task :compileJava Note: Some input files use or override a deprecated API. Note: Recompile with -Xlint:deprecation for details. BUILD SUCCESSFUL in 14s 19 actionable tasks: 19 executed -ziggy$ +ziggy$ ``` -At this point, it's probably worthwhile to run Ziggy's unit tests to make sure nothing has gone wrong. To do this, at the command line type `./gradlew test`. The system will run through a large number of tests (around 700) over the course of about a minute, and hopefully you'll get another "BUILD SUCCESSFUL" message at the end. +At this point, it's probably worthwhile to run Ziggy's unit tests to make sure nothing has gone wrong. To do this, at the command line type `./gradlew test`. The system will run through a large number of tests (around 700) over the course of about a minute, and hopefully you'll get another "BUILD SUCCESSFUL" message at the end. If you look at the Ziggy folder now, you'll see the following: @@ -78,7 +78,7 @@ LICENSE.pdf buildSrc gradle.properties sample-pipeline test README.md doc gradlew script-plugins build etc ide settings.gradle build.gradle gradle licenses src -ziggy$ +ziggy$ ``` Pretty much the same, except that now there's a `build` folder. What's in the `build` folder? @@ -87,24 +87,24 @@ Pretty much the same, except that now there's a `build` folder. What's in the `b build$ ls bin etc lib obj schema tmp classes include libs resources src -build$ +build$ ``` The main folders of interest here are: -- The `bin` folder, which has all the executables. +- The `bin` folder, which has all the executables. - The `lib` folder, which has shared object libraries (i.e., compiled C++) - The `libs` folder, which has Jarfiles (so why not name it "jars"? I don't know). -- The `etc` folder, which is a copy of the main `ziggy/etc` folder. +- The `etc` folder, which is a copy of the main `ziggy/etc` folder. -Everything that Ziggy uses in execution comes from the subfolders of `build`. That's why there's a copy of `etc` in `build`: the `etc` in the main directory isn't used, the one in `build` is, so that everything that Ziggy needs is in one place. +Everything that Ziggy uses in execution comes from the subfolders of `build`. That's why there's a copy of `etc` in `build`: the `etc` in the main directory isn't used, the one in `build` is, so that everything that Ziggy needs is in one place. Before we move on, a few useful details about building Ziggy: - Ziggy makes use of a number of dependencies. Most of them are Jarfiles, which means that they can be downloaded and used without further ado, but at least one, the [HDF5 library](https://www.hdfgroup.org/solutions/hdf5/), requires compilation via the C++ compiler. This is the most time-consuming part of the build. -- The first time you run `./gradlew`, the dependencies will be automatically downloaded. On subsequent builds with `./gradlew`, the dependencies will mostly not be downloaded, but instead cached copies will be used. This means that the subsequent uses are much faster. -- Why "mostly not ... downloaded?" Well, the build system checks the dependencies to see whether any new versions have come out. New versions of the third party libraries are automatically downloaded in order to ensure that Ziggy remains up-to-date with security patches. So on any given invocation of `./gradlew`, there might be a library or two that gets updated. -- Gradle has lots of commands (known in the lingo as "tasks") other than `test`. Most notably, the `./gradlew clean` command will delete the build directory so you can start the build over from the beginning. The `./gradlew build` command will first build the code and then run the unit tests. +- The first time you run `./gradlew`, the dependencies will be automatically downloaded. On subsequent builds with `./gradlew`, the dependencies will mostly not be downloaded, but instead cached copies will be used. This means that the subsequent uses are much faster. +- Why "mostly not ... downloaded?" Well, the build system checks the dependencies to see whether any new versions have come out. New versions of the third party libraries are automatically downloaded in order to ensure that Ziggy remains up-to-date with security patches. So on any given invocation of `./gradlew`, there might be a library or two that gets updated. +- Gradle has lots of commands (known in the lingo as "tasks") other than `test`. Most notably, the `./gradlew clean` command will delete the build directory so you can start the build over from the beginning. The `./gradlew build` command will first build the code and then run the unit tests. If all is going well, you're now ready to move on to [defining your own pipeline](configuring-pipeline.md)! diff --git a/doc/user-manual/dusty-corners.md b/doc/user-manual/dusty-corners.md index d654da8..72c8e7e 100644 --- a/doc/user-manual/dusty-corners.md +++ b/doc/user-manual/dusty-corners.md @@ -8,16 +8,36 @@ Like most software packages, Ziggy has some features that don't see a lot of use or that only get exercised under unusual circumstances. Those oddballs are documented here. -[More on the Relational Database](more-rdbms.md) -- techically the RDBMS is used all the time, but there are a few details about what we do with it that are potentially interesting to users. +### [More on the Relational Database](more-rdbms.md) -[More on Parameter Sets](more-parameter-sets.md) -- importing, modifying, version control, and more! +Techically the RDBMS is used all the time, but there are a few details about what we do with it that are potentially interesting to users. -[Parameter Overrides](parameter-overrides.md) -- how to conveniently package a collection of parameter changes. +### [More on Parameter Sets](more-parameter-sets.md) -[Redefining a Pipeline](redefine-pipeline.md) -- what to do if you realize that you need to change the configuration of a pipeline. +Importing, modifying, version control, and more! -[The Edit Trigger Dialog Box](edit-trigger.md) -- additional features on a dialog box we've already used. +### [Parameter Overrides](parameter-overrides.md) + +How to conveniently package a collection of parameter changes. + +### [Redefining a Pipeline](redefine-pipeline.md) + +What to do if you realize that you need to change the configuration of a pipeline. + +### [The Edit Pipeline Dialog Box](edit-pipeline.md) + +Additional features on a dialog box we've already used. + +### [Creating Ziggy Nicknames](nicknames.md) + +Make it easier to run those Java programs you've written. + + [[Previous]](event-handler-labels.md) [[Up]](user-manual.md) -[[Next]](more-rdbms.md) \ No newline at end of file +[[Next]](more-rdbms.md) diff --git a/doc/user-manual/edit-pipeline.md b/doc/user-manual/edit-pipeline.md new file mode 100644 index 0000000..efef228 --- /dev/null +++ b/doc/user-manual/edit-pipeline.md @@ -0,0 +1,88 @@ + + +[[Previous]](parameter-overrides.md) +[[Up]](dusty-corners.md) +[[Next]](nicknames.md) + +## The Edit Pipeline Dialog Box + +The Edit Pipeline dialog box is used to edit pipeline parameter sets and modules. + +To get to this dialog box, open the pipelines panel and double-click the pipeline you're interested in. You'll get this dialog box: + + + +What does all this stuff do? Let's go through it from the top to the bottom ("Hmm -- I got 'em!'" -C+C Music Factory). + +### Pipeline Section + +The main actions you can take from this section is to validate the pipeline and view or export parameters. + +The `Validate` button is a vestige of a bygone era. It looks for issues with the pipeline's parameter sets that are largely impossible today, but which were common in Ziggy's predecessor software packages. + +The `Report` button brings up a new window that shows the modules and parameter sets for this pipeline. The report can also be saved to a text file. This dialog is mainly useful in the context of a fairly complex system, in which you want to isolate the bits and pieces of a specified pipeline from the general mass of bits and pieces in the system. + +The `Export parameters` button exports the parameters used by this pipeline, in a kind of hokey and non-standard format. The advantage this has over using the export function on the parameter library panel is that it only exports the parameter sets used by this pipeline. In a situation where you have a lot of pipelines, with a lot of parameters, it's potentially useful to be able to see just the parameters for a given pipeline. + +The `Priority` field takes a little more explanation. We've discussed in the past the fact that Ziggy sometimes faces a situation in which Ziggy has more tasks waiting for attention than it has worker processes ready to service the tasks. In this case, Ziggy has to prioritize the tasks to ensure that the most critical ones get attention first. The pipeline priority is one way this sorting occurs. Tasks with higher priority get to leap ahead of tasks with lower priority in the queue. The available priorities are LOWEST, LOW, NORMAL, HIGHEST, HIGH. + +So how to tasks get assigned a priority? + +All tasks that are running for the first time get assigned a priority equal to the priority of the parent pipeline. In this example, the sample pipeline has a priority of NORMAL, meaning that all tasks for this pipeline will have the lowest possible priority on their first pass through the system. Tasks that are being persisted (which happens on a separate pass through the task management system) do so with priority HIGH, so persisting results takes precedence over starting new tasks. Tasks that are being rerun or restarted do so with priority HIGHEST, which means exactly what it sounds like. + +All pipelines, in turn, are initially created with priority NORMAL, meaning that all pipelines will, by default, produce tasks at priority NORMAL. Thus, all tasks from all pipelines compete for workers with a "level playing field," if you will. Usually this is the situation that most users want. + +One case where this isn't true is missions that have occasional need for much faster turnaround of data processing. That is to say, most data can be processed through Pipeline X on a first-come, first-served basis; but occasionally there will be a need to process a small amount of just-acquired data through Pipeline Y immediately. To ensure that this happens, you can set Pipeline Y to have a priority of HIGH or HIGHEST. + +Finally, the read-only `Valid?` checkbox is ticked after the `Validate` button is pressed, presuming all went well. + +### Pipeline Parameter Sets Section + +Say that five times fast. + +Anyway. + +This section shows a list of the parameter sets that are assigned at the pipeline level (that is to say, parameter sets that are made available to every task regardless of which processing module it uses). The `Add` button allows you to select any parameter set in the parameter library and make it a pipeline parameter set for this pipeline. The `Edit` button allows you to change the values of the parameters in a given set. The `Select` and `Auto-assign` buttons do things that used to be useful, but in the current version of Ziggy are not. + +Given that the parameter library panel already allows you to view and edit parameters, why is it useful to have this section on this dialog box? Again -- in the case where you have a lot of pipelines and a lot of parameters, it's useful to be able to view the parameters for a given pipeline in isolation. This allows you to avoid confusion about which parameters go to which pipelines. + +### Modules Section + +The `Modules` section offers functions that address the pipeline modules within a given pipeline. + +The display shows the modules in the pipeline, sorted in execution order. You can select a module and press one of the following buttons: + +#### Task Information Button + +This button produces a table of the tasks that Ziggy will produce for the specified module if you start the pipeline. This takes into account whether the module is configured for "keep-up" processing or reprocessing, the setting of the taskDirectoryRegex string (which allows the user to specify that only subsets of the datastore should be run through the pipeline). For each task, the task's unit of work description and number of subtasks are shown. If the table is empty, it means that the relevant files in the datastore are missing. The datastore is populated by [Data Receipt](data-receipt.md); that article will help you ingest your data into the datastore so that the task information table can calculate the number of tasks and subtasks the input data will generate. + +#### Resources Button + +If you look back at [the article on running the cluster](running-pipeline.md), you'll note that we promised that there was a way to set a different limit on the number of workers for each pipeline module. This button is that way! + +More specifically, if you press the `Resources` button, you'll get the `Worker resources` dialog box that displays a table of the modules and the current max workers and heap size settings. To change these settings from the default, either double-click on a module or use the context menu and choose the `Edit` command. This brings up the `Edit worker resources` dialog box where you can uncheck the Default checkboxes and enter new values for the number of workers or the heap size for that module. Note that the console won't let you enter more workers than cores on your machine, which is found in the the tooltip for this field. Henceforth, Ziggy will use those values when deciding on the maximum number of workers to spin up for that module and how much memory each should be given. Typically, as you increase the number of workers on a single host, you'll need to reduce the amount of heap space for each worker so that the total memory will fit within the memory available on the machine. + +Alternately, you may want to do the reverse: take a module that has user-set maximum workers or heap size values and tick the Default checkboxes to go back to using the defaults. + +#### Parameters Button + +This button brings up the `Edit parameter sets` dialog box that displays a table of module-level parameter sets. The user can edit the existing parameter sets, or add a parameter set to a given module. The not-useful `Select` and `Auto-assign` buttons are also present. + +#### Remote Execution Button + +This button brings up the `Edit remote execution parameters` dialog box. See [the article on the remote execution dialog](remote-dialog.md) for more information. + +### Save and Cancel + +You're probably thinking, "How much do I really need to know about Save and Cancel buttons?" Well, yeah, they're mostly self-explanatory. But not completely! + +To save changes that you've made on the `Edit pipeline` dialog box, or any other dialog box that it spawned, use the `Save` button. To discard all your changes, use the `Cancel` button. + +The points I'm trying to make here are twofold: + +1. Anything you do after you launch the `Edit pipeline` dialog box can be discarded, and will only be preserved when you press `Save`. +2. The `Save` and `Cancel` buttons on the `Edit pipeline` dialog box also apply to changes made on the `Edit remote execution parameters` dialog box, the `Edit parameter sets` dialog box, etc. + +[[Previous]](parameter-overrides.md) +[[Up]](dusty-corners.md) +[[Next]](nicknames.md) diff --git a/doc/user-manual/edit-trigger.md b/doc/user-manual/edit-trigger.md deleted file mode 100644 index 8b13321..0000000 --- a/doc/user-manual/edit-trigger.md +++ /dev/null @@ -1,75 +0,0 @@ - - -[[Previous]](parameter-overrides.md) -[[Up]](dusty-corners.md) -[[Next]](contact-us.md) - -## The Edit Trigger Dialog Box - -The Edit Trigger dialog box is a kind of grab-bag of various features related to managing pipeline execution. The name of the dialog box relates to an archaic time in which there were specialized Java objects, called "triggers," for launching pipelines (this is also why starting a pipeline requires the user to hit a button labeled "Fire!" Because you fire a trigger). - -To get to this dialog box, go to Operations > Triggers, and double-click the pipeline you're interested in. You'll get this dialog box: - - - -What does all this stuff do? Let's go through it from the top to the bottom ("Hmm -- I got 'em!'" -C+C Music Factory). - -### Trigger Panel - -The main actions you can take from this panel are viewing or exporting parameters, and validation of the pipeline. - -The `export params` button exports the parameters used by this pipeline, in a kind of hokey and non-standard format. The advantage this has over using the export function on the Parameters Configuration panel is that it only exports the parameter sets used by this pipeline. In a situation where you have a lot of pipelines, with a lot of parameters, it's potentially useful to be able to see just the parameters for a given pipeline. - -The `report` button brings up a new window that shows the modules and parameter sets for this pipeline. The report can also be saved to a text file. Again: mainly useful in the context of a fairly complex system, in which you want to isolate the bits and pieces of a specified pipeline from the general mass of bits and pieces in the system. - -The `validate` button is a vestige of a bygone era. It looks for issues with the pipeline's parameter sets that are largely impossible today, but which were common in Ziggy's predecessor software packages. - -### Priority Panel - -We've discussed in the past the fact that Ziggy sometimes faces a situation in which Ziggy has more tasks waiting for attention than it has worker threads ready to service the tasks. In this case, Ziggy has to prioritize the tasks to ensure that the most critical ones get attention first. - -The pipeline priority is one way this sorting occurs. Ziggy uses a system in which priority == 0 is the highest priority, priority == 4 the lowest. Tasks with higher priority get to leap ahead of tasks with lower priority in the queue. - -So how to tasks get assigned a priority? - -All tasks that are running for the first time get assigned a priority equal to the priority of the parent pipeline. In this example, the sample pipeline has a priority of 4, meaning that all tasks for this pipeline will have the lowest possible priority on their first pass through the system. All tasks that are being rerun or restarted are given priority 0, so all restart / rerun tasks have priority over any never-yet-run tasks. - -All pipelines, in turn, are initially created with priority 4, meaning that all pipelines will, by default, produce tasks at priority 4. Thus, all tasks from all pipelines compete for workers with a "level playing field," if you will. Usually this is the situation that most users want. - -One case where this isn't true is missions that have occasional need for much faster turnaround of data processing. That is to say, most data can be processed through Pipeline X on a first-come, first-served basis; but occasionally there will be a need to process a small amount of just-acquired data through Pipeline Y immediately. To ensure that this happens, you can set Pipeline Y to have a priority of 0. - -### Pipeline Parameter Sets Panel - -Say that five times fast. - -Anyway. - -This panel shows a list of the parameter sets that are assigned at the pipeline level (that is to say, parameter sets that are made available to every task regardless of which processing module it uses). The `add` button allows you to select any parameter set in the parameter library and make it a pipeline parameter set for this pipeline. The `edit values` button allows you to change the values of the parameters in a given set. The `select` and `auto-assign` buttons do things that used to be useful, but in the current version of Ziggy are not. - -Given that the Parameter Library Configuration panel already allows you to view and edit parameters, why is it useful to have this panel on this dialog box? Again -- in the case where you have a lot of pipelines and a lot of parameters, it's useful to be able to view the parameters for a given pipeline in isolation. This allows you to avoid confusion about which parameters go to which pipelines. - -### Modules Panel - -The `Modules` panel offers functions that address the pipeline modules within a given pipeline. - -The display shows the modules in the pipeline, sorted in execution order. You can select a module and then use the buttons beneath the list to execute hopefully-useful actions: - -#### Task Information Button - -This button produces a table of the tasks that Ziggy will produce for the specified module if you start the pipeline. This takes into account whether the module is configured for "keep-up" processing or reprocessing, the setting of the taskDirectoryRegex string (which allows the user to specify that only subsets of the datastore should be run through the pipeline). For each task, the task's unit of work description and number of subtasks are shown. - -#### Edit Parameters Button - -This button brings up a table of module-level parameter sets. The user can edit the existing parameter sets, or add a parameter set to a given module. The not-useful `select` and `auto-assign` buttons are also present. - -#### Re-Sync Button - -This button deletes any local changes you've made to the parameter values and/or parameter set assignments, and reloads the parameters for the selected module from the database. - -#### Remote Execution Button - -This button brings up the remote execution dialog box. See [the article on the remote execution dialog](remote-dialog.md) for more information. - -[[Previous]](parameter-overrides.md) -[[Up]](dusty-corners.md) -[[Next]](contact-us.md) diff --git a/doc/user-manual/event-handler-definition.md b/doc/user-manual/event-handler-definition.md index 819b161..efd6a66 100644 --- a/doc/user-manual/event-handler-definition.md +++ b/doc/user-manual/event-handler-definition.md @@ -17,22 +17,21 @@ The event handler demonstrated in the sample pipeline is in [pe-sample.xml](../. ```xml + directory="${ziggy.pipeline.data.receipt.dir}"/> ``` - The `pipelineEvent` element straightforwardly defines the name of the event handler itself and the name of the pipeilne it triggers. The `enableOnClusterStart` attribute tells Ziggy whether the event handler should be enabled immediately when you type `runjava cluster start`, or whether it should require a human to go in and turn it on via the console after the cluster is already up and running. +The `pipelineEvent` element straightforwardly defines the name of the event handler itself and the name of the pipeilne it triggers. The `enableOnClusterStart` attribute tells Ziggy whether the event handler should be enabled immediately when you type `ziggy cluster start`, or whether it should require a human to go in and turn it on via the console after the cluster is already up and running. The `directory` attribute is, of course, the directory where the event handler looks for its ready files. In this case, the data receipt event handler looks for ready files in the same directory where the data receipt pipeline module looks for files (or directories of files). That was a design choice, though you wouldn't need to do it that way: you could have a totally separate directory for the event handler to watch, if such was your preference. Note here that, rather than a normal string, the directory attribute can take a string that needs to be expanded into [one of Ziggy's properties](properties.md). This allows you to specify all the watched directories in the properties file. Note **that this is the only attribute or element in all of Ziggy's XML infrastructure that allows the use of property expansion notation!** That's strictly because it's the only place where we thought we needed it. If you need this added to some other part of the XML infrastructure for your purposes, let us know. We can make it happen! +[[Previous]](event-handler-intro.md) +[[Up]](event-handler.md) +[[Next]](event-handler-examples.md) + - -[[Previous]](event-handler-intro.md) -[[Up]](event-handler.md) -[[Next]](event-handler-examples.md) - diff --git a/doc/user-manual/event-handler-examples.md b/doc/user-manual/event-handler-examples.md index 2b5fad1..37e48e3 100644 --- a/doc/user-manual/event-handler-examples.md +++ b/doc/user-manual/event-handler-examples.md @@ -6,9 +6,9 @@ ## Event Handler Examples -Now that we've looked at the general concept of event handlers, and we've talked about how to define an event handler, let's work through some examples using the sample pipeline. +Now that we've looked at the general concept of event handlers, and we've talked about how to define an event handler, let's work through some examples using the sample pipeline. -Note that for these examples, the sample pipeline has been restored to its original state. That is to say, the `sample-pipeline/build` directory has been deleted, the `sample-pipeline/build-env.sh` script has been executed, and we've used `runjava cluster init` to set up the cluster (see the articles on [Building a Pipeline](building-pipeline.md) and [Running the Cluster](running-pipeline.md) for more information on these topics). Once that's done, use `runjava cluster start console &` to start the cluster and bring up a console instance. +Note that for these examples, the sample pipeline has been restored to its original state. That is to say, the `sample-pipeline/build` directory has been deleted, the `sample-pipeline/build-env.sh` script has been executed, and we've used `ziggy cluster init` to set up the cluster (see the articles on [Building a Pipeline](building-pipeline.md) and [Running the Cluster](running-pipeline.md) for more information on these topics). Once that's done, use `ziggy cluster start console &` to start the cluster and bring up a console instance. ### A Simple Example: One Ready File @@ -27,46 +27,44 @@ nasa_logo-set-2-file-2.png nasa_logo-set-2-file-3.png sample-model.txt sample-pipeline-manifest.xml -$ +$ ``` A bunch of data files, a model, and a manifest. Seems like this is a directory that is ready to be imported and run through the sample pipeline! To make that happen, we need to do two things. -First, we need to turn on the data receipt event handler. To do that, select the Configurations tab, Event Definitions button. You'll see this (except without the redaction bar covering the front part of the directory name): - -![](images/event-handler-display-1.png) +First, we need to turn on the data receipt event handler. To do that, go to the event definitions panel. You'll see this (the directory path here and elsewhere will be fully qualified): -This display is mostly pretty self-explanatory: for each event handler it shows the name, the directory it watches for the ready files, and the pipeline that will be started when the event is detected. The last column tells you whether the event handler is currently watching for events. It's a checkbox, which in this case is not checked, meaning that the event handler isn't watching the data-receipt directory for ready files. + -Click on the check box and you'll now see this: +This display is mostly pretty self-explanatory: for each event handler it shows the name, the directory it watches for the ready files, and the pipeline that will be started when the event is detected. The last column tells you whether the event handler is currently watching for events. It's a checkbox, which in this case is not checked, meaning that the event handler isn't watching the data-receipt directory for ready files. -![](images/event-handler-display-2.png) +Click on the check box to enable this event handler. -The second thing we need to do is to create the ready file. Before you do that, go to the Instances tab of the Operations tab so that you can see what happens as it happens. Once you've done that, create the ready file: +The second thing we need to do is to create the ready file. Before you do that, go to the instances panel so that you can see what happens as it happens. Once you've done that, create the ready file: ```bash $ touch sample-pipeline/build/pipeline-results/data-receipt/READY.test1.1 $ ``` -Once you do this, the Pi light on the console will quickly turn green. After a few seconds, you'll see a new pipeline instance appear: +Once you do this, the Pi light on the console will quickly turn green. After a few seconds, you'll see a new pipeline instance appear in the instances panel: -![](images/event-handler-instances-1.png) + -The event handler automatically names the pipeline with the "bare" pipeline name ("sample"), the event handler name ("data-receipt"), and the timestamp of the event that started the processing. Meanwhile, the task table looks like this: +The event handler automatically names the pipeline with the "bare" pipeline name ("sample"), the event handler name ("data-receipt"), and the timestamp of the event that started the processing. The Event name column shows the name of the event handler as well ("data-receipt"). Note that the Event name column is initially hidden as it duplicates the information in the pipeline name. If you want to sort the table by the event handler name, use the context menu in the table header to enable to Event name column. Then you can click in the header to update the sort. Meanwhile, the tasks table looks like this: -![](images/event-handler-tasks-1.png) + -The data receipt task ran to completion before the display could even update, and the pipeline went on to its `permuter` tasks. After the usual few seconds, the pipeline will finish, with `flip` and `averaging` tasks. +The data receipt task ran to completion before the display could even update, and the pipeline went on to its `permuter` tasks. After the usual few seconds, the pipeline will finish, with `flip` and `averaging` tasks. ### A More Involved Example: Multiple Deliveries, Multiple Directories For the next trick, the first step is to copy the contents of the `multi-data` directory into `data-receipt`: ```bash -$ cp -r sample-pipeline/multi-data/* sample-data/build/pipeline-results/data-receipt +$ cp -r sample-pipeline/multi-data/* sample-pipeline/build/pipeline-results/data-receipt $ ``` @@ -78,24 +76,24 @@ Listing for: sample-pipeline/build/pipeline-results/data-receipt/ sample-1 sample-2 sample-3 -$ +$ ``` You can confirm for yourself that `sample-1`, `sample-2`, and `sample-3` are directories, each of which contains data files and a manifest. In this case, we've made things more complicated than the initial example, to wit: each of the sample directories contains 2 sets of data (`set-3` and `set-4` in `sample-1`, `set-5` and `set-6` in `sample-2`, and `set-7` and `set-8` in `sample-3`. ) -For this example, we're going to simulate the following situation: +For this example, we're going to simulate the following situation: -Two different sources are pushing data into the data-receipt directory. The first source is responsible for `sample-1` and `sample-2`; the other is responsible for `sample-3`. At some point, the first source has finished its deliveries but the second has not. This is the point at which the example will start. +Two different sources are pushing data into the data-receipt directory. The first source is responsible for `sample-1` and `sample-2`; the other is responsible for `sample-3`. At some point, the first source has finished its deliveries but the second has not. This is the point at which the example will start. Before we do that, though, let's reconfigure the pipeline a bit. First, we need to tell data receipt that it should expect files to be in the subdirectories of data-receipt. To do this, return to Configuration > Parameter Library, select `Data receipt configuration`, and modify the `taskDirectoryRegex`: - + Second, let's tell the pipeline that we only want it to process new data that's never been processed, and it should leave alone any data that's been successfully processed before this. To do so, select the `Multiple subtask configuration` and the `Single subtask configuration` parameter sets, and uncheck the reprocess box: - + -Now return to the Operations > Instances tab, and finally create the ready files. Remember that you need two ready files because we are simulating a complete delivery from the first source, and it's delivering to the `sample-1` and `sample-2` directories. +Now return to the instances panel, and finally create the ready files. Remember that you need two ready files because we are simulating a complete delivery from the first source, and it's delivering to the `sample-1` and `sample-2` directories. ```bash $ touch sample-pipeline/build/pipeline-results/data-receipt/sample-1.READY.test2.2 @@ -105,45 +103,45 @@ $ As soon as the second ready file is created, a new pipeline instance will start: -![](images/event-handler-instances-2.png) + The tasks display will look like this: -![](images/event-handler-tasks-2.png) + There's a fair amount of interesting stuff going on here, so let's dig into this display! #### Data Receipt Tasks -There are two data receipt tasks, `sample-1` and `sample-2`. These are the tasks that ran against the `sample-1` and `sample-2` subdirectories, respectively, of the main data receipt directory. As we expected and advertised, by giving the ready files labels of `sample-1` and `sample-2`, we were able to get Ziggy to import from those directories while leaving the `sample-3` directory alone. +There are two data receipt tasks, `sample-1` and `sample-2`. These are the tasks that ran against the `sample-1` and `sample-2` subdirectories, respectively, of the main data receipt directory. As we expected and advertised, by giving the ready files labels of `sample-1` and `sample-2`, we were able to get Ziggy to import from those directories while leaving the `sample-3` directory alone. #### Permuter Tasks for UOWs 3, 4, 5, and 6 -The two directories that were imported by data receipt contained a total of 4 data sets: `set-3`, `set-4`, `set-5`, and `set-6`. These ran with task IDs 11, 12, 13, and 14. Note that the data sets didn't get mapped to task IDs in the way that a person would do it, i.e., making `set-3` the first task, then `set-4`, etc. As a general matter, when Ziggy needs to make multiple tasks for a given pipeline module, it creates them in no particular order, so you can wind up with situations like the one we see above. +The two directories that were imported by data receipt contained a total of 4 data sets: `set-3`, `set-4`, `set-5`, and `set-6`. These ran with task IDs 11, 12, 13, and 14. Note that the data sets didn't get mapped to task IDs in the way that a person would do it, i.e., making `set-3` the first task, then `set-4`, etc. As a general matter, when Ziggy needs to make multiple tasks for a given pipeline module, it creates them in no particular order, so you can wind up with situations like the one we see above. -Another curious thing that's happened is that tasks 11 and 12 got as far as processing state `Ac` (Algorithm Complete, see the article on the [Instances Panel](instances-panel.md) for more information). At that point, we would have expected those tasks to store outputs and then go to processing state `C` (Complete). Instead, tasks 11 and 12 seem to wait in state `Ac` while tasks 13 and 14 execute their algorithms. Why is that? +Another curious thing that's happened is that tasks 11 and 12 got as far as processing state `Ac` (Algorithm Complete, see the article on the [Instances Panel](instances-panel.md) for more information). At that point, we would have expected those tasks to store outputs and then go to processing state `C` (Complete). Instead, tasks 11 and 12 seem to wait in state `Ac` while tasks 13 and 14 execute their algorithms. Why is that? -Take a quick look at the properties file, `sample-pipeline/etc/sample.properties`, and you'll see the following line: +Take a quick look at the properties file, `sample-pipeline/etc/sample.properties`, and you'll see the following line: ``` -pi.worker.threadCount = 2 +ziggy.worker.count = 2 ``` -(See the article on [Properties](properties.md) to refresh your memory on this). This means that the worker has 2 and only 2 worker threads that can be used for assorted task processing activities. In this case, that creates a problem: there are only 2 worker threads but 6 tasks that need to run, and each task needs to perform several steps (marshaling, algorithm execution, storing results). Ziggy needs to prioritize users of its limited number of worker threads, and it does so by putting algorithm execution at a higher priority than storing results. Thus, we wind up with 2 tasks that have produced results, but they're waiting for worker threads to store those results; and the worker threads won't be available for this purpose until all the tasks have completed algorithm execution. +(See the article on [Properties](properties.md) to refresh your memory on this). This means that the supervisor has 2 and only 2 workers that can be used for assorted task processing activities. In this case, that creates a problem: there are only 2 workers but 6 tasks that need to run, and each task needs to perform several steps (marshaling, algorithm execution, storing results). Ziggy needs to prioritize users of its limited number of worker threads, and it does so by putting algorithm execution at a higher priority than storing results. Thus, we wind up with 2 tasks that have produced results, but they're waiting for workers to store those results; and the workers won't be available for this purpose until all the tasks have completed algorithm execution. #### Permuter Task for UOW 2 Meanwhile, before any of the `sample-1` or `sample-2` data sets got processed, we see a task with ID 10, permuter module, running on data set 2. Oddly, its subtask "scoreboard" is `(0 / 0 / 0)`. What happened?!?! -Well -- remember that we set up this example so that it wouldn't process any data that got processed before (i.e., we are doing "keep-up" processing, not reprocessing). All of the data in `set-2` was processed in the first example, so there were no `set-2` subtasks that needed to be processed in this example. +Well -- remember that we set up this example so that it wouldn't process any data that got processed before (i.e., we are doing "keep-up" processing, not reprocessing). All of the data in `set-2` was processed in the first example, so there were no `set-2` subtasks that needed to be processed in this example. Okay, but in that case, why did Ziggy create a task for `set-2` processing? Well, the situation is this: -Ziggy's order of operations is that it has to first create its tasks, and only then can it go and figure out how many subtasks need to be processed in each task. Ziggy found that its datastore held a total of 6 units of work for permuter, went ahead and created one task for each unit of work, and only then realized that two of the tasks had no subtasks that needed processing! Once it's gotten that far, it lets the zero-subtask tasks run through the system. This prevents some potential user confusion ("Hey, why did Ziggy go from task 9 to task 11? Why isn't there a task 10?" "Why isn't there a task for set 1?"). It also avoids some other problems too boring to get into here. +Ziggy's order of operations is that it has to first create its tasks, and only then can it go and figure out how many subtasks need to be processed in each task. Ziggy found that its datastore held a total of 6 units of work for permuter, went ahead and created one task for each unit of work, and only then realized that two of the tasks had no subtasks that needed processing! Once it's gotten that far, it lets the zero-subtask tasks run through the system. This prevents some potential user confusion ("Hey, why did Ziggy go from task 9 to task 11? Why isn't there a task 10?" "Why isn't there a task for set 1?"). It also avoids some other problems too boring to get into here. #### Permuter Task for UOW 1 -Finally, task 15 is the permuter task for `set-1`. In the task display, it's state is `SUBMITTED`, with a p-state of `I` (for initializing). This is the result of two features we've already talked about. First, Ziggy doesn't try to put its tasks into any particular order, so the set-1 task wound up being the final permuter task. Second, there are only 2 worker threads that need to service the demands of 6 tasks. What's happened is that task 15 is waiting for a worker thread to become available to process it. Of course, like `set-2`, `set-1` has no subtasks that need processing in this example, so as soon as `set-1` gets a worker, it will instantly jump to state COMPLETED. +Finally, task 15 is the permuter task for `set-1`. In the task display, it's state is `SUBMITTED`, with a p-state of `I` (for initializing). This is the result of two features we've already talked about. First, Ziggy doesn't try to put its tasks into any particular order, so the set-1 task wound up being the final permuter task. Second, there are only 2 workers that need to service the demands of 6 tasks. What's happened is that task 15 is waiting for a worker to become available to process it. Of course, like `set-2`, `set-1` has no subtasks that need processing in this example, so as soon as `set-1` gets a worker, it will instantly jump to state COMPLETED. #### Meanwhile, Back in the Data Receipt Directory... @@ -153,34 +151,16 @@ If we look at the contents of the data receipt directory, this is what we see: $ ls -1 sample-pipeline/build/pipeline-results/data-receipt/ Listing for: sample-pipeline/build/pipeline-results/data-receipt/ sample-3 -$ +$ ``` -The sample-1 and sample-2 directories are gone. The ready files for sample-1 and sample-2 are gone. All that's left is the sample-3 directory. This is exactly what we'd hoped (and asserted) would happen: Ziggy was able to figure out which of the deliveries were complete, and process them, while leaving the incomplete delivery alone. +The sample-1 and sample-2 directories are gone. The ready files for sample-1 and sample-2 are gone. All that's left is the sample-3 directory. This is exactly what we'd hoped (and asserted) would happen: Ziggy was able to figure out which of the deliveries were complete, and process them, while leaving the incomplete delivery alone. #### Finally... -If you keep watching the console, you'll see that once the permuter tasks are done, Ziggy will create flip tasks for all 6 datasets, then averaging tasks for all 6. The tasks for `set-1` and `set-2` will always have zero subtasks to process and will complete instantly; the other tasks will take a finite time to process their subtasks. - -### Pipeline Events Display - -After doing all of the above, you may be thinking, "This event handler seems pretty awesome!" But you're probably tempering that opinion with a bit of ... unease. - -Why unease? - -Unease because once the event handler has run successfully, there isn't a straightforward way to see what, if anything, it did. The data files it imported are gone from the data receipt directory. The ready files are gone from the data receipt directory. How do we tell the difference between, "The event handler handled all these events," versus, "There were no events and Ziggy hasn't done anything lately"? - -There's two ways to know this. - -The first way is what we discussed before: when the event handler detects an event, it starts a pipeline. The pipeline instance appears in the instances table, with a name that indicates that an event handler caused that instance to start. - -The second way is more direct. - -If you go to Configuration > Pipeline Events, you'll get the following display: - -![](images/events-display.png) +If you keep watching the console, you'll see that once the permuter tasks are done, Ziggy will create flip tasks for all 6 datasets, then averaging tasks for all 6. The tasks for `set-1` and `set-2` will always have zero subtasks to process and will complete instantly; the other tasks will take a finite time to process their subtasks. -The first column is the event ID -- this is a simple, monotonically-increasing integer value. The second column is the event handler that detected the event, the third is the name of the pipeline that was started in response to the event. The fourth column is the timestamp at which the pipeline started. The final column is the instance ID of the resulting pipeline instance. A permanent record is kept of every single pipeline event, all of which are visible in this display. +To review the files that were ingested, refer to the [Data Receipt Display](data-receipt-display.md) article. [[Previous]](event-handler-definition.md) [[Up]](event-handler.md) diff --git a/doc/user-manual/event-handler-intro.md b/doc/user-manual/event-handler-intro.md index 7b270e5..2206961 100644 --- a/doc/user-manual/event-handler-intro.md +++ b/doc/user-manual/event-handler-intro.md @@ -6,9 +6,9 @@ ## Introduction to Event Handlers -Ziggy's predecessors in the pipeline infrastructure field (the Kepler SOC's PI module and TESS SPOC's Spiffy) were designed to require a lot of in-person handling by pipeline users. In particular, they had no capacity to take actions without somebody, somewhere, issuing a command or pressing a button on the console. +Ziggy's predecessors in the pipeline infrastructure field (the Kepler SOC's PI module and TESS SPOC's Spiffy) were designed to require a lot of in-person handling by pipeline users. In particular, they had no capacity to take actions without somebody, somewhere, issuing a command or pressing a button on the console. -This design choice was acceptable for the aforementioned Kepler and TESS missions, which had data deliveries approximately once a month and always followed a constant sequence of processing activities. For future missions with much larger data rates, data deliveries are likely to be much more frequent, potentially every day or even more often. Future missions are also likely to have non-standard, high urgency processing requirements arise at unpredictable intervals (something along the lines of, "We need to process this chunk of data through this special-purpose pipeline, and we need to get it turned around in 12 hours"). Both of these expectations led to the decision that Ziggy required some means of taking autonomous action. In short, it needs an event handler: a system that allows the user to define some form of event that Ziggy should watch for, and an action that it should take when the event happens. +This design choice was acceptable for the aforementioned Kepler and TESS missions, which had data deliveries approximately once a month and always followed a constant sequence of processing activities. For future missions with much larger data rates, data deliveries are likely to be much more frequent, potentially every day or even more often. Future missions are also likely to have non-standard, high urgency processing requirements arise at unpredictable intervals (something along the lines of, "We need to process this chunk of data through this special-purpose pipeline, and we need to get it turned around in 12 hours"). Both of these expectations led to the decision that Ziggy required some means of taking autonomous action. In short, it needs an event handler: a system that allows the user to define some form of event that Ziggy should watch for, and an action that it should take when the event happens. ### The Action @@ -18,38 +18,38 @@ The action Ziggy takes in response to an event is pretty straightforward: it sta What are the necessary characteristics of an event? Put another way, what are the requirements we have that define an event? -From the outset, we knew that data receipt, or a pipeline that starts with data receipt and runs through all the various processing steps, was the event handler's "killer app." An event handler that can start the pipeline automatically when new data comes in relieves human beings of that burden, and if data comes in every day, or at unpredictable intervals, managing data delivery is a major burden indeed. We also knew that we didn't want the system design to tie the event handler so closely to data receipt that it couldn't be used for anything else. Thus we allowed data receipt to inform our design, while staying on the lookout for design choices that coupled the two systems too tightly together. +From the outset, we knew that data receipt, or a pipeline that starts with data receipt and runs through all the various processing steps, was the event handler's "killer app." An event handler that can start the pipeline automatically when new data comes in relieves human beings of that burden, and if data comes in every day, or at unpredictable intervals, managing data delivery is a major burden indeed. We also knew that we didn't want the system design to tie the event handler so closely to data receipt that it couldn't be used for anything else. Thus we allowed data receipt to inform our design, while staying on the lookout for design choices that coupled the two systems too tightly together. With all that introductory material out of the way, we turn to the question: what are the requirements for the "event" part of the event handler? - The event has to be something that unambiguously declares, "This event is ready to be handled!" In particular, this means that the event can't be a regular file that has content of some type (i.e., it can't be something like the manifest files used by data receipt). The problem with those files is that the file can appear on the system where Ziggy runs, but not be completely delivered (i.e., the file is there but the source is still copying the contents of the file across to the Ziggy system). In other words, the event must be atomic. - The event needs to be similarly unambiguous about which pipeline Ziggy needs to start when the event happens. This is important because Ziggy might have a number of different kinds of events it wants to handle, and it needs to be able to clearly tell them apart. - In the specific case of data receipt, the data source might deliver files to the main data receipt directory, or it might deliver subdirectories of files (see [the article on data receipt](data-receipt.md) for more information). Whatever system we devise must support either option. -- It may be the case that multiple data sources deliver separate data streams to Ziggy simultaneously. Ziggy needs to be able to manage such a situation. In particular, in the case of data delivery, Ziggy needs to be able to identify which subdirectories in the data receipt directory are from each data stream: each stream needs its own pipeline instance to perform the processing, and the different data streams cannot be expected to complete their deliveries simultaneously (i.e., Ziggy needs to know that one stream is done, and start processing that data, while also waiting for one or more other streams to complete). -- The overall system needs to give the user some ability to monitor what's going on, whether progress is being made, etc. In the case of data receipt, this means that the user wants to know ahead of time how many directories to expect, and wants to know at runtime which ones are done and which are in progress. +- It may be the case that multiple data sources deliver separate data streams to Ziggy simultaneously. Ziggy needs to be able to manage such a situation. In particular, in the case of data delivery, Ziggy needs to be able to identify which subdirectories in the data receipt directory are from each data stream: each stream needs its own pipeline instance to perform the processing, and the different data streams cannot be expected to complete their deliveries simultaneously (i.e., Ziggy needs to know that one stream is done, and start processing that data, while also waiting for one or more other streams to complete). +- The overall system needs to give the user some ability to monitor what's going on, whether progress is being made, etc. In the case of data receipt, this means that the user wants to know ahead of time how many directories to expect, and wants to know at runtime which ones are done and which are in progress. Based on the requirements above, we designed a solution as follows: -- Events are defined by zero-length files. In the case of data receipt, the data delivery source can push files to the system that hosts Ziggy, and once that file transfer is complete the source can create the necessary zero-length file. -- Each event handler has a distinct directory that it watches for these zero-length files (henceforth we'll call these "ready files", because they indicate to Ziggy a degree of readiness). For data receipt, we traditionally use the data receipt directory (though we could have defined a totally separate directory for the ready files). For additional event handlers, the user would need to create other directories that can be watched for ready files, one directory per event handler. +- Events are defined by zero-length files. In the case of data receipt, the data delivery source can push files to the system that hosts Ziggy, and once that file transfer is complete the source can create the necessary zero-length file. +- Each event handler has a distinct directory that it watches for these zero-length files (henceforth we'll call these "ready files", because they indicate to Ziggy a degree of readiness). For data receipt, we traditionally use the data receipt directory (though we could have defined a totally separate directory for the ready files). For additional event handlers, the user would need to create other directories that can be watched for ready files, one directory per event handler. - An event can have more than one ready file, and Ziggy will only proceed when all the ready files for an event have appeared. Because a zero-length file has no content other than its name, the naming convention for ready files must encompass: - Some way of indicating which ready files go with a particular event. - The total number of ready files for a given event. - - Additional information that is unique to each ready file: in the case of data receipt, since a ready file may be associated with a particular subdirectory in the data receipt directory, the name of the subdirectory needs to be in the name of the ready file. + - Additional information that is unique to each ready file: in the case of data receipt, since a ready file may be associated with a particular subdirectory in the data receipt directory, the name of the subdirectory needs to be in the name of the ready file. ### The Ready File Without further ado, here's the naming convention for the ready file: ``` -