Ziggy release 0.2.0 (2022-11-30)

.cproject

@@ -0,0 +1,68 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<?fileVersion 4.0.0?><cproject storage_type_id="org.eclipse.cdt.core.XmlProjectDescriptionStorage">
+	<storageModule moduleId="org.eclipse.cdt.core.settings">
+		<cconfiguration id="cdt.managedbuild.toolchain.gnu.cross.base.1485726495">
+			<storageModule buildSystemId="org.eclipse.cdt.managedbuilder.core.configurationDataProvider" id="cdt.managedbuild.toolchain.gnu.cross.base.1485726495" moduleId="org.eclipse.cdt.core.settings" name="Default">
+				<externalSettings/>
+				<extensions>
+					<extension id="org.eclipse.cdt.core.ELF" point="org.eclipse.cdt.core.BinaryParser"/>
+					<extension id="org.eclipse.cdt.core.GASErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
+					<extension id="org.eclipse.cdt.core.GmakeErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
+					<extension id="org.eclipse.cdt.core.GLDErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
+					<extension id="org.eclipse.cdt.core.CWDLocator" point="org.eclipse.cdt.core.ErrorParser"/>
+					<extension id="org.eclipse.cdt.core.GCCErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
+				</extensions>
+			</storageModule>
+			<storageModule moduleId="cdtBuildSystem" version="4.0.0">
+				<configuration buildProperties="" id="cdt.managedbuild.toolchain.gnu.cross.base.1485726495" name="Default" parent="org.eclipse.cdt.build.core.emptycfg">
+					<folderInfo id="cdt.managedbuild.toolchain.gnu.cross.base.1485726495.483857408" name="/" resourcePath="">
+						<toolChain id="cdt.managedbuild.toolchain.gnu.cross.base.183873464" name="Cross GCC" superClass="cdt.managedbuild.toolchain.gnu.cross.base">
+							<option id="cdt.managedbuild.option.gnu.cross.prefix.861079249" name="Prefix" superClass="cdt.managedbuild.option.gnu.cross.prefix"/>
+							<option id="cdt.managedbuild.option.gnu.cross.path.654420002" name="Path" superClass="cdt.managedbuild.option.gnu.cross.path"/>
+							<targetPlatform archList="all" binaryParser="org.eclipse.cdt.core.ELF" id="cdt.managedbuild.targetPlatform.gnu.cross.1836139642" isAbstract="false" osList="all" superClass="cdt.managedbuild.targetPlatform.gnu.cross"/>
+							<builder id="cdt.managedbuild.builder.gnu.cross.449290212" managedBuildOn="false" name="Gnu Make Builder.Default" superClass="cdt.managedbuild.builder.gnu.cross"/>
+							<tool id="cdt.managedbuild.tool.gnu.cross.c.compiler.1828101191" name="Cross GCC Compiler" superClass="cdt.managedbuild.tool.gnu.cross.c.compiler">
+								<inputType id="cdt.managedbuild.tool.gnu.c.compiler.input.495596777" superClass="cdt.managedbuild.tool.gnu.c.compiler.input"/>
+							</tool>
+							<tool id="cdt.managedbuild.tool.gnu.cross.cpp.compiler.356948423" name="Cross G++ Compiler" superClass="cdt.managedbuild.tool.gnu.cross.cpp.compiler">
+								<inputType id="cdt.managedbuild.tool.gnu.cpp.compiler.input.616981487" superClass="cdt.managedbuild.tool.gnu.cpp.compiler.input"/>
+							</tool>
+							<tool id="cdt.managedbuild.tool.gnu.cross.c.linker.1647311537" name="Cross GCC Linker" superClass="cdt.managedbuild.tool.gnu.cross.c.linker"/>
+							<tool id="cdt.managedbuild.tool.gnu.cross.cpp.linker.1397342134" name="Cross G++ Linker" superClass="cdt.managedbuild.tool.gnu.cross.cpp.linker">
+								<inputType id="cdt.managedbuild.tool.gnu.cpp.linker.input.1188547588" superClass="cdt.managedbuild.tool.gnu.cpp.linker.input">
+									<additionalInput kind="additionalinputdependency" paths="$(USER_OBJS)"/>
+									<additionalInput kind="additionalinput" paths="$(LIBS)"/>
+								</inputType>
+							</tool>
+							<tool id="cdt.managedbuild.tool.gnu.cross.archiver.1740198382" name="Cross GCC Archiver" superClass="cdt.managedbuild.tool.gnu.cross.archiver"/>
+							<tool id="cdt.managedbuild.tool.gnu.cross.assembler.1121043895" name="Cross GCC Assembler" superClass="cdt.managedbuild.tool.gnu.cross.assembler">
+								<inputType id="cdt.managedbuild.tool.gnu.assembler.input.1805920933" superClass="cdt.managedbuild.tool.gnu.assembler.input"/>
+							</tool>
+						</toolChain>
+					</folderInfo>
+				</configuration>
+			</storageModule>
+			<storageModule moduleId="org.eclipse.cdt.core.externalSettings"/>
+		</cconfiguration>
+	</storageModule>
+	<storageModule moduleId="scannerConfiguration">
+		<autodiscovery enabled="true" problemReportingEnabled="true" selectedProfileId=""/>
+	</storageModule>
+	<storageModule moduleId="cdtBuildSystem" version="4.0.0">
+		<project id="ziggy.null.733126594" name="ziggy"/>
+	</storageModule>
+	<storageModule moduleId="org.eclipse.cdt.core.LanguageSettingsProviders"/>
+	<storageModule moduleId="refreshScope"/>
+	<storageModule moduleId="org.eclipse.cdt.make.core.buildtargets">
+		<buildTargets>
+			<target name="clean" path="" targetID="org.eclipse.cdt.build.MakeTargetBuilder">
+				<buildCommand>make</buildCommand>
+				<buildArguments/>
+				<buildTarget>clean</buildTarget>
+				<stopOnError>true</stopOnError>
+				<useDefaultCommand>false</useDefaultCommand>
+				<runAllBuilders>true</runAllBuilders>
+			</target>
+		</buildTargets>
+	</storageModule>
+</cproject>

.gitignore

@@ -0,0 +1,13 @@
+**/.gradle*
+**/__pycache__
+**/build
+*.mexmaci64
+.classpath
+.cproject
+.project
+.pydevproject
+.settings
+dist
+doc/pdf
+src/test/cpp/ExternalProcess/testprog.o
+src/test/cpp/testmi/inputs-0.bin

README.md

@@ -0,0 +1,72 @@
+<!-- -*-visual-line-*- -->
+
+# Super-Basic Introduction
+
+## What is Ziggy?
+
+Ziggy is "A Pipeline management system for Data Analysis Pipelines."
+
+What does that mean?
+
+A data analysis pipeline is any data analysis software that proceeds in a step-by-step fashion, in which the  inputs to the later steps include (but are not limited to) the outputs from the earlier steps. The vast majority of science data analysis falls into this category.
+
+A pipeline management system is all of the functionality that makes such a pipeline work. It's everything other than the actual software packages that do the processing. That includes execution of the algorithms on the data, but also such activities as: routing logging messages to the correct destinations; automatically executing the next step when the current step has completed; ensuring that the pipeline does the right thing when an exception occurs (either in Ziggy or in one of the algorithm packages); providing a user interface so that operators can control and monitor activities; managing a datastore of inputs and results; providing persistence for all of the records that need to be preserved across time; and much more.
+
+## Why does Ziggy exist?
+
+So -- why should anyone use Ziggy, or for that matter any other "pipeline management system" for their data analysis needs? Here's why:
+
+Any data analysis activity that handles more than a trivial amount of data will require some sort of pipeline management system. At a minimum, it's going to be essential to ensure that all the data gets processed and that the processing is uniform, because otherwise any results from the processing become suspect: the user has to wonder, "If I missed processing some subset of the data, would that affect my results?" and, "If I didn't process all of the data the same way -- if I changed how I did the processing midway through my dataset -- will that affect my results?" Because of these issues, data analysis inevitably winds up applying some degree of automation to the process, even if it's just a handful of shell scripts that the user runs manually.
+
+As data volumes get larger, the issue of managing the pipeline becomes more and more onerous, and more and more crucial. At the same time, development and maintenance of the pipeline manager becomes more and more of a distraction to the subject matter experts who just want to perform their data analysis, get their results, publish their papers, etc. At some point, rather than taking on the job of writing all this software that's not in their area of interest, the subject matter experts should look around for some existing software that will do all this management for them -- something that allows them to plug their processing application software into the management system and, presto! Complete system.
+
+Ziggy is that "something."
+
+## Where can I run Ziggy?
+
+Ziggy is actually pretty lightweight. During development, we run Ziggy on some fairly standard laptops without any problems, so you shouldn't have any trouble downloading, building, and trying out Ziggy.
+
+In terms of where you run Ziggy, what's more important than Ziggy itself is the data volume you need to process and the requirements you place on things like keeping the data in a location where mulitple users have access to it. Depending on the answers to those questions, you might be able to run your analysis on a laptop; a workstation; a server; or a cloud or high-performance computing (HPC) environment. Ziggy has been used in all of these different locations, based on the task at hand.
+
+## When did Ziggy get its start?
+
+Ziggy was originally written as the pipeline infrastructure (PI) component for the pipeline that processed data from NASA's Kepler mission, which used transit photometry to look for signals of planets circling distant stars. It was run on server clusters and the NASA Advanced Supercomputer (NAS) at NASA's Ames Research Center, and the original pipeline team was housed at Ames as well.
+
+A few years later, Ames provided the data analysis pipeline for another transit photometry mission, the Transiting Exoplanet Survey Satellite (TESS). A more advanced version of the Kepler PI component was developed for TESS, and was named Spiffy ("Science pipeline infrastructure for you").
+
+In the fullness of time, members of the Kepler and TESS team realized that there was an opportunity to take Spiffy and make it into a software package that was suitable for truly huge data volumes (terabytes per day, as compared to the terabytes per month rate of TESS) and easier to use than Spiffy or PI had been. This resulted in the development process that culminated in Ziggy.
+
+## How do I put Ziggy to use?
+
+Glad you asked that! The intent of the [user manual](doc/user-manual/user-manual.md) is to take you through the process, step by step. Thus, we recommend starting at the first link and clicking your way down as you make progress.
+
+Additionally, Ziggy ships with a sample pipeline. This pipeline uses an extremely simple set of algorithms to demonstrate as much of Ziggy's prowess and features as possible. As we go through the steps, we'll show highlights from the sample pipeline so that you have a sense of what it is you should see. Anyway, it's always easier to explain something by example than to explain in a totally abstract way...
+
+## Who maintains Ziggy?
+
+Ziggy is still maintained by members of the Kepler and TESS team who are based at NASA Ames Research Center.
+
+## License
+
+Copyright © 2022 United States Government as represented by the Administrator of the National Aeronautics and Space Administration. All Rights Reserved.
+
+NASA acknowledges the SETI Institute’s primary role in authoring and producing Ziggy, a Pipeline Management System for Data Analysis Pipelines, under Cooperative Agreement Nos. NNX14AH97A, 80NSSC18M0068 & 80NSSC21M0079.
+
+This file is available under the terms of the NASA Open Source Agreement (NOSA). You should have received a copy of this agreement with the Ziggy source code; see the file [LICENSE.pdf](LICENSE.pdf).
+
+Disclaimers
+
+No Warranty: THE SUBJECT SOFTWARE IS PROVIDED "AS IS" WITHOUT ANY WARRANTY OF ANY KIND, EITHER EXPRESSED, IMPLIED, OR STATUTORY, INCLUDING, BUT NOT LIMITED TO, ANY WARRANTY THAT THE SUBJECT SOFTWARE WILL CONFORM TO SPECIFICATIONS, ANY IMPLIED WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, OR FREEDOM FROM INFRINGEMENT, ANY WARRANTY THAT THE SUBJECT SOFTWARE WILL BE ERROR FREE, OR ANY WARRANTY THAT DOCUMENTATION, IF PROVIDED, WILL CONFORM TO THE SUBJECT SOFTWARE. THIS AGREEMENT DOES NOT, IN ANY MANNER, CONSTITUTE AN ENDORSEMENT BY GOVERNMENT AGENCY OR ANY PRIOR RECIPIENT OF ANY RESULTS, RESULTING DESIGNS, HARDWARE, SOFTWARE PRODUCTS OR ANY OTHER APPLICATIONS RESULTING FROM USE OF THE SUBJECT SOFTWARE. FURTHER, GOVERNMENT AGENCY DISCLAIMS ALL WARRANTIES AND LIABILITIES REGARDING THIRD-PARTY SOFTWARE, IF PRESENT IN THE ORIGINAL SOFTWARE, AND DISTRIBUTES IT "AS IS."
+
+Waiver and Indemnity: RECIPIENT AGREES TO WAIVE ANY AND ALL CLAIMS AGAINST THE UNITED STATES GOVERNMENT, ITS CONTRACTORS AND SUBCONTRACTORS, AS WELL AS ANY PRIOR RECIPIENT. IF RECIPIENT'S USE OF THE SUBJECT SOFTWARE RESULTS IN ANY LIABILITIES, DEMANDS, DAMAGES, EXPENSES OR LOSSES ARISING FROM SUCH USE, INCLUDING ANY DAMAGES FROM PRODUCTS BASED ON, OR RESULTING FROM, RECIPIENT'S USE OF THE SUBJECT SOFTWARE, RECIPIENT SHALL INDEMNIFY AND HOLD HARMLESS THE UNITED STATES GOVERNMENT, ITS CONTRACTORS AND SUBCONTRACTORS, AS WELL AS ANY PRIOR RECIPIENT, TO THE EXTENT PERMITTED BY LAW. RECIPIENT'S SOLE REMEDY FOR ANY SUCH MATTER SHALL BE THE IMMEDIATE, UNILATERAL TERMINATION OF THIS AGREEMENT.
+
+## Other licenses
+
+Ziggy makes use of third-party software. Their licenses appear in the [licenses](licenses/licenses.md) directory.
+
+## Contributing
+
+If you are interested in contributing to Ziggy, please complete the NASA license agreement that applies to you:
+
+* [doc/legal/NASA-Corporate-CLA.pdf](doc/legal/NASA-Corporate-CLA.pdf)
+* [doc/legal/NASA-Individual-CLA.pdf](doc/legal/NASA-Individual-CLA.pdf)

RELEASE-NOTES.md

@@ -0,0 +1,17 @@
+<!-- -*-visual-line-*- -->
+
+# Ziggy Release Notes
+
+These are the release notes for Ziggy. While we're able to provide links to GitHub issues, we are not able to provide links to our internal Jira issues.
+
+## 0.2.0
+
+This is the first Ziggy release to appear on GitHub. The major number of 0 indicates that we're still refactoring the Kepler and TESS codebases and reserve the right to make breaking changes from time to time as we make the pipeline more widely useful. However, the general pipeline infrastructure has been in production use since Kepler's launch in 2009.
+
+### Resolved issues
+
+In future releases, this section will contain a list of GitHub/Jira issues that were resolved and incorporated into the release. If the resolution for an issue introduced a breaking change, it will be described so that you can update your properties files or pipeline configurations in advance.
+
+## 0.1.0
+
+This release was overtaken by events. This release came and went before we got authorization to push our code to GitHub.

build.gradle

@@ -0,0 +1,105 @@
+// Ziggy Gradle Build File
+//
+// This file contains the declarative portion of the build only.
+// Imperative tasks are found in script-plugins and are applied at the
+// bottom of this file.
+
+// See comment for ziggyDependencies in gradle.properties.
+ext.ziggyDependencies = "$rootDir/$ziggyDependencies"
+
+apply plugin: 'java'
+apply plugin: 'eclipse'
+
+defaultTasks "assemble"
+
+repositories {
+    mavenCentral()
+    flatDir {
+        dirs "$ziggyDependencies/lib"
+    }
+}
+
+dependencies {
+    // Needed to compile ziggy.
+    compile 'com.github.librepdf:openpdf:1.3.+'
+    compile 'com.github.testdriven.guice:commons-configuration:1.+'
+    compile 'com.google.guava:guava:23.+'
+    compile 'com.jgoodies:jgoodies-forms:1.9.+'
+    compile 'com.jgoodies:jgoodies-looks:2.7.+'
+    compile 'commons-cli:commons-cli:1.5.+'
+    compile 'commons-codec:commons-codec:1.+'
+    compile 'commons-io:commons-io:2.11.+'
+    compile 'org.apache.commons:commons-collections4:4.+'
+    compile 'org.apache.commons:commons-compress:1.+'
+    compile 'org.apache.commons:commons-csv:1.9.+'
+    compile 'org.apache.commons:commons-exec:1.+'
+    compile 'org.apache.commons:commons-lang3:3.12.+'
+    compile 'org.apache.commons:commons-math3:3.6.+'
+    compile 'org.apache.commons:commons-text:1.+'
+    compile 'org.apache.logging.log4j:log4j-api:2.17.+'
+    compile 'org.apache.logging.log4j:log4j-core:2.17.+'
+    compile 'org.apache.logging.log4j:log4j-slf4j-impl:2.17.+'
+    compile 'org.hibernate.javax.persistence:hibernate-jpa-2.0-api:1.0.+'
+    compile 'org.hibernate:hibernate-core:4.2.+'
+    compile 'org.javassist:javassist:3.18.1-GA'
+    compile 'org.jfree:jfreechart:1.0.+'
+    compile 'org.netbeans.api:org-netbeans-swing-outline:+'
+    compile 'org.tros:l2fprod-properties-editor:1.3.+'
+    compile 'tanukisoft:wrapper:3.2.+'
+
+    // Libraries built in buildSrc.
+    compile ':jarhdf5:1.12.+'
+
+    // Needed to compile unit tests.
+    compile 'junit:junit:4.+'
+    compile 'org.hamcrest:hamcrest:2.+'
+    compile 'org.mockito:mockito-core:1.10.+'
+
+    // Needed to run unit tests.
+    // hsqldb version 2.3.4 causes PipelineInstanceTaskCrudTest to
+    // fail with HsqlException: data exception: datetime field
+    // overflow. This can be fixed by replacing Long.MAX_VALUE with a
+    // smaller date. Other problems ensue with version 2.6.
+    compile 'org.hsqldb:hsqldb:2.3.2'
+
+    // Needed at runtime.
+    compile 'org.postgresql:postgresql:9.4-1201-jdbc4'
+}
+
+test {
+    systemProperty "log4j2.configurationFile","$projectDir/test/data/logging/log4j2.xml"
+    systemProperty "java.library.path", "$ziggyDependencies/lib"
+
+    forkEvery = 1
+
+    testLogging {
+        events "failed", "skipped"
+    }
+
+    // The RMI tests appear to have an ordering sensitivity. To wit,
+    // If Intra- runs before Inter-, they all run successfully.
+    // If Inter- runs before Intra-, Intra- tests all fail.
+    // Nonetheless, these tests are good to have for interactive use,
+    // as they allow to troubleshoot RMI problems more simply. Thus,
+    // we leave that class out of gradle test but keep it around in case
+    // we need it.
+    exclude "**/RmiInt*"
+}
+
+javadoc {
+    title = "Ziggy API"
+    options.overview = "src/main/java/overview.html"
+}
+
+// Apply Ziggy Gradle script plugins.
+apply from: "script-plugins/copy.gradle"
+apply from: "script-plugins/database-schemas.gradle"
+apply from: "script-plugins/eclipse.gradle"
+apply from: "script-plugins/gcc.gradle"
+apply from: "script-plugins/hdf5.gradle"
+// apply from: "script-plugins/matlab.gradle"
+apply from: "script-plugins/misc.gradle"
+apply from: "script-plugins/test.gradle"
+apply from: "script-plugins/wrapper.gradle"
+apply from: "script-plugins/xml-schemas.gradle"
+apply from: "script-plugins/ziggy-libraries.gradle"

buildSrc/build.gradle

@@ -0,0 +1,61 @@
+/**
+ * This is the buildSrc/build.gradle file.  This should contain a minimal
+ * amount of dependencies as this is used by the build system itself.
+ */
+apply plugin: 'eclipse'
+apply plugin: 'java'
+
+defaultTasks 'assemble'
+
+repositories {
+    mavenCentral()
+}
+
+dependencies {
+    // Needed to compile buildSrc.
+    compile 'com.google.guava:guava:23.+'
+    compile 'commons-io:commons-io:2.11.+'
+    compile 'org.apache.commons:commons-exec:1.+'
+    compile 'org.freemarker:freemarker:2.3.+'
+
+    // Needed to compile unit tests.
+    compile 'junit:junit:4.+'
+    compile 'org.mockito:mockito-core:1.10.+'
+
+    // The following stuff is needed in order to build custom gradle tasks and plugins.
+    compile gradleApi()
+    compile localGroovy()
+}
+
+eclipse {
+    classpath {
+        defaultOutputDir = file("$buildDir/eclipse")
+        // outputBaseDir = file("$buildDir/eclipse")
+        downloadSources = false
+        downloadJavadoc = false
+
+        // Gradle 4.4 now specifies all of the output directories, but puts
+        // them in the Eclipse default of "bin". There is a feature request
+        // to add classpath.outputBaseDir that has the same syntax and effect
+        // as the now-useless defaultOutputDir. In the meantime, update the
+        // path manually.
+        file.whenMerged {
+            entries.each { entry ->
+                if (entry.kind == "src" && entry.hasProperty("output")) {
+                    // The use of $buildDir does not return build.
+                    entry.output = "build/eclipse"
+                }
+            }
+        }
+    }
+}
+
+test {
+    testLogging  {
+        showStandardStreams = true
+    }
+}
+
+apply from: "script-plugins/hdf5.gradle"
+apply from: "script-plugins/wrapper.gradle"
+