V2.1

CONTRIBUTING.md

@@ -0,0 +1,43 @@
+# Contributing to Vizier
+
+## General Guidelines
+
+We want everyone to be able to participate here, so when contributing (issues, documentation, code, etc...), please keep the following two guidelines in mind: (i) Think about how your words will be perceived by others before writing them, and (ii) give others the benefit of the doubt when interpreting their words.
+
+## Style 
+
+#### Branches
+
+- The main Vizier branch represents the most recent release candidate.
+  - No one pushes directly to the main branch.
+- Development branches are labeled `v[major].[minor]` (e.g., v2.1).
+  - Pull requests for new features will only be accepted for the most recent development branch.
+  - Only pull requests for bugfixes will be accepted for earlier branches.
+  - Pull requests must pass the default test case suite: 
+    - `mill vizier.test` for backend changes
+    - `mill vizier.ui.test` for frontend changes
+
+#### Commits
+
+- A single commit should address a single issue.  Spread distinct changes across multiple commits.
+- The first line should be an at-most 100 character summary of the commit, containing:
+    - The prefix WIP: if the commit is broken.
+    - A brief description of the goal of the commit's changes.
+    - An issue reference:
+        - (re: #[issue number]) to mark work towards a specific ticket
+        - (closes: #[issue number]) to signify that work towards a ticket is done
+- A brief list of behaviors of the system that were changed: One line per behavior.  For example:
+    - [system component] no longer hangs when passed a [type of input]
+    - Added [field] to the schema of [category] to store [summary]
+    - UI can now display [state].
+- If the reason for the behavioral change is not clear from the summary, add a short discussion providing any necessary context.  For example "[external component] has a bug where if you ask it to [action] then it deletes your hard drive.  Obviously, this would be bad, so we need to sanitize our inputs."
+
+#### Pull Requests
+
+- Ideally, a pull request focuses on one issue, or a linked set of issues.
+    - Pull requests for features must be made to the latest development branch.  
+    - Only pull requests for bug fixes will be accepted to an earlier branch.
+- Pull requests must pass all test cases.
+    - ... excepting any test cases that are broken on a development branch.
+- The text of the pull request should briefly summarize the key changes made in terms of behaviors of the system that are different (see commit messages above).
+    - The text should clearly identify which tickets the pull request is responding to.

LICENSE.txt

@@ -1,4 +1,4 @@
-Copyright (C) 2017-2021 University at Buffalo,
+Copyright (C) 2017-2024 University at Buffalo,
                         New York University,
                         Illinois Institute of Technology.
 Licensed under the Apache License, Version 2.0 (the "License");

build.sc

@@ -15,7 +15,7 @@ import java.util.Calendar
  *** The Vizier Backend 
  *************************************************/
 object vizier extends ScalaModule with PublishModule {
-  val VERSION       = "2.0.0-rc2"
+  val VERSION       = "2.1.0-SNAPSHOT"
   val PLAY_JS       = ivy"com.typesafe.play::play-json::2.9.2"
 
   val MIMIR_CAVEATS = ivy"info.vizierdb::mimir-caveats::0.3.6"
@@ -25,7 +25,7 @@ object vizier extends ScalaModule with PublishModule {
                             "log4j" -> "*",
                           )
 
-  def scalaVersion = "2.12.15"
+  def scalaVersion = "2.12.20"
 
   def repositoriesTask = T.task { super.repositoriesTask() ++ Seq(
     MavenRepository("https://maven.mimirdb.org/"),
@@ -75,6 +75,9 @@ object vizier extends ScalaModule with PublishModule {
 
     f"$VERSION (revision $gitVersion-$gitRevision; built ${date.get(Calendar.YEAR)}%04d-${date.get(Calendar.MONTH)}%02d-${date.get(Calendar.DAY_OF_MONTH)}%02d)"
   }
+  def version = T {
+    VERSION
+  }
 
   def internalJavaVersion = T {
     try {
@@ -165,7 +168,7 @@ object vizier extends ScalaModule with PublishModule {
     ivy"info.vizierdb::vega:1.0.0",
 
     // Scala Cell
-    ivy"org.scala-lang:scala-compiler:2.12.15",
+    ivy"org.scala-lang:scala-compiler:${scalaVersion}",
 
     // Python
     ivy"me.shadaj::scalapy-core:0.5.2",
@@ -212,7 +215,7 @@ object vizier extends ScalaModule with PublishModule {
  *** Backend Resources
  *************************************************/
   def buildRoutesScript = T.sources { os.pwd / "scripts" / "build_routes.sc" }
-  def routesFile        = T.sources { millSourcePath / "shared" / "resources" / "vizier-routes.txt" }
+  def routesFile        = T.sources { millSourcePath / "resources" / "vizier-routes.txt" }
 
   def routes = T { 
     println("Recompiling routes from "+routesFile().head.path); 
@@ -245,7 +248,7 @@ object vizier extends ScalaModule with PublishModule {
   object ui extends ScalaJSModule { 
 
     def scalaVersion = vizier.scalaVersion
-    def scalaJSVersion = "1.7.1"
+    def scalaJSVersion = "1.16.0"
 
 /*************************************************
  *** Frontend Dependencies
@@ -261,6 +264,10 @@ object vizier extends ScalaModule with PublishModule {
       millSourcePath / "src",
       vizier.millSourcePath / "shared" / "src"
     )
+
+    override def scalacOptions: Target[Seq[String]] = T { Seq(
+      "-P:scalajs:nowarnGlobalExecutionContext"
+    )}
 
     override def compile = T {
       routes()

scripts/bootstrap.sh

@@ -1,8 +1,16 @@
-ARTIFACT=info.vizierdb:vizier_2.12:2.0.0-rc2 
+MAVEN_BASE=info.vizierdb:vizier_2.12
+VERSION=$(
+  mill show vizier.versionString 2> /dev/null | 
+    jq -r |
+    sed 's/ .*//'
+)
+ARTIFACT=$MAVEN_BASE:$VERSION
+
+echo $ARTIFACT
 
 coursier bootstrap $ARTIFACT\
   -r https://s01.oss.sonatype.org/content/groups/public/ \
   -r https://repo.osgeo.org/repository/release/ \
   -r https://maven.mimirdb.org/ \
   -M info.vizierdb.Vizier \
-  -o bin/vizier -f
+  -o bin/vizier -f

scripts/build_routes.sc

@@ -506,6 +506,13 @@ def websocketAPICall(route: Route): String =
       "additionalProperties" -> extra,
     )
 
+  def mkAnyOf(options: JsValue*) = 
+  {
+    Json.obj(
+      "anyOf" -> options
+    )
+  }
+
   def mkArray(elem: JsValue) = Json.obj("type" -> "array", "items" -> elem)
 
   define { "Identifier" -> Json.obj(
@@ -562,6 +569,17 @@ def websocketAPICall(route: Route): String =
     "version" -> Json.obj("type" -> "string"),
     "packages" -> mkArray(typeRef("PythonPackage")),
   )}
+  define { "VizierScriptModule" -> mkAnyOf(
+    mkObject()(
+      "type" -> Json.obj("const" -> "inline"),
+      "spec" -> typeRef("JsValue"),
+    ),
+    mkObject()(
+      "type" -> Json.obj("const" -> "in_out"),
+      "imports" -> typeRef("JsValue"),
+      "exports" -> typeRef("JsValue"),
+    )
+  )}
 
   val aliases = Map(
     "CommandArgumentList.T" -> "Seq[CommandArgument]",

scripts/fix_copyrights.py

@@ -16,7 +16,7 @@
 from glob import glob
 import re
 
-LICENSE_VERSION = 2
+LICENSE_VERSION = 3
 
 with open("LICENSE.txt") as f:
   LICENSE = [

scripts/lib/akkagen.sc

@@ -94,6 +94,7 @@ def decodePath(route: Route, handler: AkkaDirective): AkkaDirective =
       case PathVariable(_, "long") => "LongNumber"
       case PathVariable(_, "subpath") => "Remaining"
       case PathVariable(_, "string") => "Segment"
+      case PathVariable(_, pvt) => throw new RuntimeException(s"Invalid path variable type '$pvt' (expecting 'int', 'long', 'subpath', or 'string')")
     }
 
   val (pathParamInputs, pathParamOutputs) = 

scripts/lib/routes.sc

@@ -129,13 +129,15 @@ case class Route(
 
 def readRoutes(path: os.Path): Seq[Route] = 
   os.read(path).split("\n")
+    .toIndexedSeq
     .map { description => 
       val components = description.split("\\s+")
       val pathAndArguments = components(0).split("\\?")
       val path = pathAndArguments(0).split("/").toSeq
       val pathQuery:Seq[PathVariable] = 
         if(pathAndArguments.size > 1){ 
           pathAndArguments(1).split("&")
+                             .toIndexedSeq
                              .map { _.split(":").toSeq }
                              .map {
                               case Seq(identifier, dataType) => PathVariable(identifier, dataType)
@@ -147,6 +149,7 @@ def readRoutes(path: os.Path): Seq[Route] =
         components(6) match {
           case "-" | "_" => Seq()
           case x => x.split(";")
+            .toIndexedSeq
             .map { _.split(":").toSeq }
             .map { 
               case Seq(identifier, dataType) => Param(identifier, dataType) 

scripts/publish.sh

@@ -1,19 +1,36 @@
 #!/bin/bash
 
+# This script expects credentials stored in `pass` 
+# (https://www.passwordstore.org/).  Create a `pass` archive for
+# sonatype.org and make sure it includes two lines: 
+# ```
+# token_user: ...
+# token_pass: ...
+# ```
+# The values from these lines can be populated from the user token
+# which can be obtained from your user profile (instructions here):
+# https://central.sonatype.org/publish/generate-token/
+
+
 cd `dirname $0`/..
 
+# Fix Copyrights
+python3 scripts/fix_copyrights.py
+
 export SONATYPE_USERNAME="$(
   pass sonatype.org |
-    grep '^login:' |
-    sed 's/login: //'
+    grep '^token_user:' |
+    sed 's/token_user: //' |
+    head -n 1
 )"
 
 export SONATYPE_PASSWORD="$(
   pass sonatype.org |
+    grep '^token_pass:' |
+    sed 's/token_pass: //' |
     head -n 1
 )"
 
-
 mill mill.scalalib.PublishModule/publishAll \
       vizier.publishArtifacts \
       --release true\

vizier/backend/src/info/vizierdb/Config.scala

@@ -137,6 +137,17 @@ class Config(arguments: Seq[String])
     descr = "Set the SparkSQL warehouse directory (default: {cache-dir}/spark-warehouse)"
   )
 
+  val extraPlugins = opt[List[File]]("plugin",
+    short = 'P',
+    descr = "Enable a plugin for this session",
+    default = Some(List())
+  )
+
+  lazy val plugins:Seq[File] = 
+    Option(defaults.getProperty("vizier-plugins"))
+        .map { _.split(":").map { new File(_) }.toSeq }
+        .getOrElse { Seq() } ++ extraPlugins()
+
   def workingDirectoryFile: File = 
     new File(
       workingDirectory
@@ -168,7 +179,9 @@ class Config(arguments: Seq[String])
   lazy val pythonVenvDirFile = new File(cacheDirFile, "python")
 
   def resolveToDataDir(path: String) = { new File(dataDirFile, path).getAbsoluteFile }
-
+  def resolveToWorkingDir(path: File): File =
+    if(path.isAbsolute()) { path }
+    else { workingDirectoryFile.toPath.resolve(path.toPath).toFile }
 
   ////////////////////////// Ingest //////////////////////////
 

vizier/backend/src/info/vizierdb/Plugin.scala

@@ -0,0 +1,107 @@
+/* -- copyright-header:v2 --
+ * Copyright (C) 2017-2021 University at Buffalo,
+ *                         New York University,
+ *                         Illinois Institute of Technology.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * -- copyright-header:end -- */
+package info.vizierdb
+
+import play.api.libs.json._
+import java.net.URLClassLoader
+import java.net.URL
+import java.io.File
+import java.io.FileNotFoundException
+import com.typesafe.scalalogging.LazyLogging
+import scala.collection.mutable
+import info.vizierdb.util.ClassLoaderUtils
+
+case class Plugin(
+  name: String,
+  schema_version: Int,
+  plugin_class: String,
+  description: String,
+  documentation: Option[String],
+)
+{
+} 
+
+object Plugin
+  extends LazyLogging
+{
+
+  val loaded = mutable.Map[String, Plugin]()
+  val jars = mutable.Buffer[URL]()
+
+
+  implicit val pluginFormat: Format[Plugin] = Json.format
+
+  def load(file: File): Plugin =
+  {
+    val jar = 
+      if(file.isAbsolute){ file }
+      else { 
+        Vizier.config.workingDirectoryFile.toPath
+              .resolve(file.toPath)
+              .toFile 
+      }
+
+
+    if(!jar.exists()){ throw new FileNotFoundException(jar.getAbsoluteFile.toString) }
+    val url = jar.getAbsoluteFile().toURI().toURL()
+
+
+    // This classloader is used only to load the vizier-plugin.json file.  
+    // and then discarded    
+    val preloader = new URLClassLoader(
+      Array(jar.toURI.toURL),
+      Vizier.mainClassLoader
+    )
+
+    val plugin = 
+      Json.parse(
+        preloader.getResourceAsStream("vizier-plugin.json")
+      ).asOpt[Plugin]
+       .getOrElse { 
+          throw new RuntimeException(s"$jar is not a valid Vizier Plugin")
+       }
+
+    Vizier.sparkSession.sparkContext.addJar(jar.getAbsoluteFile().toString)
+    val loader = Vizier.sparkSession.sharedState.jarClassLoader
+    loader.addURL(url)
+
+    val detail = s"${plugin.name} [$jar]"
+
+    assert(
+      plugin.schema_version > 0 && plugin.schema_version <= 1,
+      s"Unsupported version '${plugin.schema_version} for plugin $detail"
+    )
+
+    assert(
+      !loaded.contains(plugin.name),
+      s"Plugin ${plugin.name} is already loaded."
+    )
+
+    loaded.put(plugin.name, plugin)
+    jars.append(jar.toURI.toURL)
+
+    val clazz = Class.forName(plugin.plugin_class, true, loader)
+    val singleton = clazz.getDeclaredField("MODULE$").get()
+    val initMethod = clazz.getMethod("init", Vizier.sparkSession.getClass)
+
+    ClassLoaderUtils.withContextClassLoader(loader) {
+      initMethod.invoke(singleton, Vizier.sparkSession)
+    }
+
+    return plugin
+  }
+
+  def loadedJars = jars.toSeq
+}