Merge pull request #134 from VizierDB/v1.1

Version 1.1.1

build.sbt

@@ -1,8 +1,8 @@
 scalaVersion := "2.12.12"
 
-val VIZIER_VERSION = "1.1.0"
-val MIMIR_VERSION = "1.1.0"
-val CAVEATS_VERSION = "0.3.3"
+val VIZIER_VERSION = "1.1.1"
+val MIMIR_VERSION = "1.1.1"
+val CAVEATS_VERSION = "0.3.4"
 
 // Project and subprojects
 lazy val vizier = (project in file("."))

scripts/build_routes.py

@@ -42,6 +42,9 @@
       if fieldType == "int":
         regexp += ["([0-9]+)"]
         fields += [ (fieldName, "JsNumber({}.toLong)".format(fieldName)) ]
+      elif fieldType == "string":
+        regexp += ["([^/]+)"]
+        fields += [ (fieldName, "JsString(URLDecoder.decode({}, \"UTF-8\"))".format(fieldName)) ]
       elif fieldType == "subpath":
         regexp += ["(.+)"]
         fields += [ (fieldName, "JsString({})".format(fieldName)) ]
@@ -92,6 +95,7 @@
   print("/* DO NOT EDIT THIS FILE DIRECTLY */")
   print("")
   print("import play.api.libs.json._")
+  print("import java.net.URLDecoder")
   print("import javax.servlet.http.{HttpServlet, HttpServletRequest, HttpServletResponse}")
   print("import org.mimirdb.api.Response")
   print("import info.vizierdb.types._")

src/main/resources/logback-test.xml

@@ -20,6 +20,7 @@
   <logger name="info.vizierdb.Config$"                           level="WARN" />
   <logger name="info.vizierdb.api.WorkflowSQLRequest"            level="WARN" />
   <logger name="info.vizierdb.api.AppendModule"                  level="WARN" />
+  <logger name="info.vizierdb.catalog.Schema$"                   level="INFO" />
   <logger name="info.vizierdb.catalog.Module"                    level="DEBUG" />
   <logger name="info.vizierdb.viztrails.Scheduler$"              level="TRACE" />
   <logger name="info.vizierdb.viztrails.Provenance$"             level="WARN" />

src/main/resources/pycell/dataset.py

@@ -26,7 +26,7 @@
 (format yyyy-MM-dd hh:mm:ss:zzzz).
 """
 DATATYPE_DATE = 'date'
-DATATYPE_DATETIME = 'datetime'
+DATATYPE_DATETIME = 'timestamp'
 DATATYPE_INT = 'int'
 DATATYPE_SHORT = 'short'
 DATATYPE_LONG = 'long'
@@ -420,8 +420,10 @@ def insert_row(self,
     if values is not None:
       if len(values) != len(self.columns):
         raise ValueError('invalid number of values for dataset schema')
+      for v, col in zip(values, self.columns):
+        assert_type(v, col.data_type, col.name)
       row = MutableDatasetRow(
-        values=[str(v) for v in values],
+        values=[v for v in values],
         dataset=self
       )
     else:
@@ -621,7 +623,7 @@ def to_json(self, limit: Optional[int] = None):
       ],
       "data": [
         [
-          export_from_native_type(v, c.data_type)
+          export_from_native_type(v, c.data_type, c.name)
           for (v, c) in zip(row.values, self.columns)
         ]
         for row in rows
@@ -652,38 +654,62 @@ def collabel_2_index(label):
 
 
 def import_to_native_type(value: Any, data_type: str) -> Any:
-  if data_type == "geometry":
+  if value is None:
+    return None
+  elif data_type == DATATYPE_GEOMETRY:
     from shapely import wkt  # type: ignore[import]
     return wkt.loads(value)
+  elif data_type == DATATYPE_DATETIME:
+    from datetime import datetime
+    return datetime.fromisoformat(value)
+  elif data_type == DATATYPE_DATE:
+    from datetime import date
+    return date.fromisoformat(value)
   else:
     return value
 
 
-def export_from_native_type(value: Any, data_type: str) -> Any:
-  if data_type == "geometry":
-    return value.wkt
+def export_from_native_type(value: Any, data_type: str, context = "the value") -> Any:
+  assert_type(value, data_type, context)
+  if value is None:
+    return None
+  elif data_type == DATATYPE_GEOMETRY:
+    from shapely.geometry import asShape
+    return asShape(value).wkt
+  elif data_type == DATATYPE_DATETIME or data_type == DATATYPE_DATE:
+    return value.isoformat()
   else:
     return value
 
 
-def assert_type(value: Any, data_type: str) -> Any:
-  import datetime
-  if data_type == DATATYPE_DATE:
-    assert(isinstance(value, datetime.date))
+def assert_type(value: Any, data_type: str, context = "the value") -> Any:
+  if value is None:
+    return value
+  elif data_type == DATATYPE_DATE:
+    import datetime
+    if not isinstance(value, datetime.date):
+      raise ValueError(f"{context} ({value}) is a {type(value)} but should be a date")
     return value
   elif data_type == DATATYPE_DATETIME:
-    assert(isinstance(value, datetime.datetime))
+    import datetime
+    if not isinstance(value, datetime.datetime):
+      raise ValueError(f"{context} ({value}) is a {type(value)} but should be a datetime")
     return value
   elif data_type == DATATYPE_INT or data_type == DATATYPE_SHORT or data_type == DATATYPE_LONG:
-    assert(isinstance(value, int))
+    if not isinstance(value, int):
+      raise ValueError(f"{context} ({value}) is a {type(value)} but should be an int")
     return value
   elif data_type == DATATYPE_REAL:
-    assert(isinstance(value, float))
+    if not isinstance(value, float):
+      raise ValueError(f"{context} ({value}) is a {type(value)} but should be a float")
     return value
   elif data_type == DATATYPE_VARCHAR:
-    assert(isinstance(value, str))
+    if not isinstance(value, str):
+      raise ValueError(f"{context} ({value}) is a {type(value)} but should be a str")
     return value
   elif data_type == DATATYPE_GEOMETRY:
+    if not hasattr(value, "__geo_interface__"):
+      raise ValueError(f"{context} ({value}) is a {type(value)}, and not a type that supports the geometry interface")
     # Not sure how to validate this...
     return value
   else:

src/main/resources/ui/asset-manifest.json

@@ -1,8 +1,8 @@
 {
   "files": {
     "main.css": "/static/css/main.b41ff8bb.chunk.css",
-    "main.js": "/static/js/main.993fa3b4.chunk.js",
-    "main.js.map": "/static/js/main.993fa3b4.chunk.js.map",
+    "main.js": "/static/js/main.6e8d5bbf.chunk.js",
+    "main.js.map": "/static/js/main.6e8d5bbf.chunk.js.map",
     "runtime-main.js": "/static/js/runtime-main.fa46e1cc.js",
     "runtime-main.js.map": "/static/js/runtime-main.fa46e1cc.js.map",
     "static/css/2.5fff324c.chunk.css": "/static/css/2.5fff324c.chunk.css",
@@ -21,6 +21,6 @@
     "static/css/2.5fff324c.chunk.css",
     "static/js/2.680dae7d.chunk.js",
     "static/css/main.b41ff8bb.chunk.css",
-    "static/js/main.993fa3b4.chunk.js"
+    "static/js/main.6e8d5bbf.chunk.js"
   ]
 }

src/main/resources/ui/index.html

@@ -1 +1 @@
-<!doctype html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name="theme-color" content="#000000"><link rel="manifest" href="/manifest.json"><link rel="shortcut icon" href="/favicon.svg"><title>Vizier</title><link href="/static/css/2.5fff324c.chunk.css" rel="stylesheet"><link href="/static/css/main.b41ff8bb.chunk.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div><script src="/env.js"></script><script>!function(e){function r(r){for(var n,l,a=r[0],p=r[1],f=r[2],c=0,s=[];c<a.length;c++)l=a[c],Object.prototype.hasOwnProperty.call(o,l)&&o[l]&&s.push(o[l][0]),o[l]=0;for(n in p)Object.prototype.hasOwnProperty.call(p,n)&&(e[n]=p[n]);for(i&&i(r);s.length;)s.shift()();return u.push.apply(u,f||[]),t()}function t(){for(var e,r=0;r<u.length;r++){for(var t=u[r],n=!0,a=1;a<t.length;a++){var p=t[a];0!==o[p]&&(n=!1)}n&&(u.splice(r--,1),e=l(l.s=t[0]))}return e}var n={},o={1:0},u=[];function l(r){if(n[r])return n[r].exports;var t=n[r]={i:r,l:!1,exports:{}};return e[r].call(t.exports,t,t.exports,l),t.l=!0,t.exports}l.m=e,l.c=n,l.d=function(e,r,t){l.o(e,r)||Object.defineProperty(e,r,{enumerable:!0,get:t})},l.r=function(e){"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})},l.t=function(e,r){if(1&r&&(e=l(e)),8&r)return e;if(4&r&&"object"==typeof e&&e&&e.__esModule)return e;var t=Object.create(null);if(l.r(t),Object.defineProperty(t,"default",{enumerable:!0,value:e}),2&r&&"string"!=typeof e)for(var n in e)l.d(t,n,function(r){return e[r]}.bind(null,n));return t},l.n=function(e){var r=e&&e.__esModule?function(){return e.default}:function(){return e};return l.d(r,"a",r),r},l.o=function(e,r){return Object.prototype.hasOwnProperty.call(e,r)},l.p="/";var a=this.webpackJsonpplayground=this.webpackJsonpplayground||[],p=a.push.bind(a);a.push=r,a=a.slice();for(var f=0;f<a.length;f++)r(a[f]);var i=p;t()}([])</script><script src="/static/js/2.680dae7d.chunk.js"></script><script src="/static/js/main.993fa3b4.chunk.js"></script></body></html>
+<!doctype html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name="theme-color" content="#000000"><link rel="manifest" href="/manifest.json"><link rel="shortcut icon" href="/favicon.svg"><title>Vizier</title><link href="/static/css/2.5fff324c.chunk.css" rel="stylesheet"><link href="/static/css/main.b41ff8bb.chunk.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div><script src="/env.js"></script><script>!function(e){function r(r){for(var n,l,a=r[0],p=r[1],f=r[2],c=0,s=[];c<a.length;c++)l=a[c],Object.prototype.hasOwnProperty.call(o,l)&&o[l]&&s.push(o[l][0]),o[l]=0;for(n in p)Object.prototype.hasOwnProperty.call(p,n)&&(e[n]=p[n]);for(i&&i(r);s.length;)s.shift()();return u.push.apply(u,f||[]),t()}function t(){for(var e,r=0;r<u.length;r++){for(var t=u[r],n=!0,a=1;a<t.length;a++){var p=t[a];0!==o[p]&&(n=!1)}n&&(u.splice(r--,1),e=l(l.s=t[0]))}return e}var n={},o={1:0},u=[];function l(r){if(n[r])return n[r].exports;var t=n[r]={i:r,l:!1,exports:{}};return e[r].call(t.exports,t,t.exports,l),t.l=!0,t.exports}l.m=e,l.c=n,l.d=function(e,r,t){l.o(e,r)||Object.defineProperty(e,r,{enumerable:!0,get:t})},l.r=function(e){"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})},l.t=function(e,r){if(1&r&&(e=l(e)),8&r)return e;if(4&r&&"object"==typeof e&&e&&e.__esModule)return e;var t=Object.create(null);if(l.r(t),Object.defineProperty(t,"default",{enumerable:!0,value:e}),2&r&&"string"!=typeof e)for(var n in e)l.d(t,n,function(r){return e[r]}.bind(null,n));return t},l.n=function(e){var r=e&&e.__esModule?function(){return e.default}:function(){return e};return l.d(r,"a",r),r},l.o=function(e,r){return Object.prototype.hasOwnProperty.call(e,r)},l.p="/";var a=this.webpackJsonpplayground=this.webpackJsonpplayground||[],p=a.push.bind(a);a.push=r,a=a.slice();for(var f=0;f<a.length;f++)r(a[f]);var i=p;t()}([])</script><script src="/static/js/2.680dae7d.chunk.js"></script><script src="/static/js/main.6e8d5bbf.chunk.js"></script></body></html>

src/main/resources/vizier-routes.txt

@@ -57,5 +57,6 @@
 /projects/{projectId:int}/files                                                                                                   POST    file      CreateFileHandler
 /projects/{projectId:int}/files/{fileId:int}                                                                                      GET     file      GetArtifactHandler.File
 /projects/{projectId:int}/files/{fileId:int}/{tail:subpath}                                                                       GET     file      GetArtifactHandler.File
+/published/{name:string}                                                                                                          GET     file      GetPublishedArtifactHandler
 /tasks                                                                                                                            GET     task      ListTasksHandler
 /reload                                                                                                                           POST    service   ReloadHandler

src/main/scala/info/vizierdb/Config.scala

@@ -24,7 +24,7 @@ class Config(arguments: Seq[String])
   extends ScallopConf(arguments)
   with LazyLogging
 {
-  version("Vizier-Scala 1.1.0 (c) 2021 U. Buffalo, NYU, Ill. Inst. Tech., and Breadcrumb Analytics")
+  version("Vizier-Scala 1.1.1 (c) 2021 U. Buffalo, NYU, Ill. Inst. Tech., and Breadcrumb Analytics")
   banner("""Docs: https://github.com/VizierDB/vizier-scala/wiki
            |Usage: vizier [OPTIONS]
            |    or vizier import [OPTIONS] export

src/main/scala/info/vizierdb/Vizier.scala

@@ -158,6 +158,9 @@ object Vizier
     Schema.initialize()
     initORMLogging()
     bringDatabaseToSaneState()
+    if(config.workingDirectory.isDefined){
+      System.setProperty("user.dir", config.workingDirectory())
+    }
 
     // Set up Mimir
     println("Starting Mimir...")

src/main/scala/info/vizierdb/VizierURLs.scala

@@ -106,6 +106,8 @@ class VizierURLs(
     compose(getBranchHeadModule(projectId, branchId, modulePosition), "/thaw_one")
   def thawOneWorkflowModule(projectId: Identifier, branchId: Identifier, workflowId: Identifier, modulePosition: Int) =
     compose(getWorkflowModule(projectId, branchId, workflowId, modulePosition), "/thaw_one")
+  def publishedArtifact(name: String) = 
+    url(s"published/$name")
 
 
   def getDataset(

src/main/scala/info/vizierdb/api/GetPublishedArtifact.scala

@@ -0,0 +1,44 @@
+/* -- copyright-header:v2 --
+ * Copyright (C) 2017-2021 University at Buffalo,
+ *                         New York University,
+ *                         Illinois Institute of Technology.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * -- copyright-header:end -- */
+package info.vizierdb.api
+
+import scalikejdbc.DB
+import play.api.libs.json._
+import org.mimirdb.api.{ Request, Response }
+import info.vizierdb.api.handler.{ Handler, ClientConnection }
+import info.vizierdb.api.response._
+import info.vizierdb.catalog.PublishedArtifact
+
+object GetPublishedArtifactHandler
+  extends Handler
+{
+  def handle(
+    pathParameters: Map[String, JsValue], 
+    connection: ClientConnection
+  ): Response =
+  {
+    val published:PublishedArtifact = DB.readOnly { implicit s =>
+      PublishedArtifact.getOption(pathParameters("name").as[String])
+                       .getOrElse { 
+                         return NoSuchEntityResponse() 
+                       }
+    }
+
+    return GetArtifactHandler.handle(Map(
+      "projectId"  -> JsNumber(published.projectId),
+      "artifactId" -> JsNumber(published.artifactId),
+    ), connection)
+  }
+}

src/main/scala/info/vizierdb/api/servlet/VizierAPIServletRoutes.scala

@@ -18,6 +18,7 @@ package info.vizierdb.api.servlet
 /* DO NOT EDIT THIS FILE DIRECTLY */
 
 import play.api.libs.json._
+import java.net.URLDecoder
 import javax.servlet.http.{HttpServlet, HttpServletRequest, HttpServletResponse}
 import org.mimirdb.api.Response
 import info.vizierdb.types._
@@ -85,6 +86,7 @@ trait VizierAPIServletRoutes extends HttpServlet {
   val ROUTE_PATTERN_56 = "/projects/([0-9]+)/files".r
   val ROUTE_PATTERN_57 = "/projects/([0-9]+)/files/([0-9]+)".r
   val ROUTE_PATTERN_58 = "/projects/([0-9]+)/files/([0-9]+)/(.+)".r
+  val ROUTE_PATTERN_59 = "/published/([^/]+)".r
 
   override def doGet(request: HttpServletRequest, response: HttpServletResponse) = 
   {
@@ -121,6 +123,7 @@ trait VizierAPIServletRoutes extends HttpServlet {
         case ROUTE_PATTERN_55(projectId, artifactId) => GetArtifactHandler.File.handle(Map("projectId" -> JsNumber(projectId.toLong), "artifactId" -> JsNumber(artifactId.toLong)), connection)
         case ROUTE_PATTERN_57(projectId, fileId) => GetArtifactHandler.File.handle(Map("projectId" -> JsNumber(projectId.toLong), "fileId" -> JsNumber(fileId.toLong)), connection)
         case ROUTE_PATTERN_58(projectId, fileId, tail) => GetArtifactHandler.File.handle(Map("projectId" -> JsNumber(projectId.toLong), "fileId" -> JsNumber(fileId.toLong), "tail" -> JsString(tail)), connection)
+        case ROUTE_PATTERN_59(name) => GetPublishedArtifactHandler.handle(Map("name" -> JsString(URLDecoder.decode(name, "UTF-8"))), connection)
         case "/tasks" => ListTasksHandler.handle(Map(), connection)
         case _ => fourOhFour(request)
       }
@@ -232,6 +235,7 @@ trait VizierAPIServletRoutes extends HttpServlet {
         case ROUTE_PATTERN_56(projectId) => CORSPreflightResponse("POST")
         case ROUTE_PATTERN_57(projectId, fileId) => CORSPreflightResponse("GET")
         case ROUTE_PATTERN_58(projectId, fileId, tail) => CORSPreflightResponse("GET")
+        case ROUTE_PATTERN_59(name) => CORSPreflightResponse("GET")
         case "/tasks" => CORSPreflightResponse("GET")
         case "/reload" => CORSPreflightResponse("POST")
         case _ => fourOhFour(request)

src/main/scala/info/vizierdb/catalog/PublishedArtifact.scala

@@ -0,0 +1,97 @@
+/* -- copyright-header:v2 --
+ * Copyright (C) 2017-2021 University at Buffalo,
+ *                         New York University,
+ *                         Illinois Institute of Technology.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * -- copyright-header:end -- */
+package info.vizierdb.catalog
+
+import scalikejdbc._
+import java.net.URL
+import play.api.libs.json._
+import info.vizierdb.VizierAPI
+import info.vizierdb.types._
+import info.vizierdb.catalog.binders._
+
+case class PublishedArtifact(
+  name: String,
+  artifactId: Identifier,
+  projectId: Identifier,
+  properties: JsObject,
+)
+{
+  def url = VizierAPI.urls.publishedArtifact(name)
+  def artifact(implicit session: DBSession) = Artifact.get(target = artifactId, projectId = Some(projectId))
+}
+
+object PublishedArtifact
+  extends SQLSyntaxSupport[PublishedArtifact]
+{
+  def apply(rs: WrappedResultSet): PublishedArtifact = autoConstruct(rs, (PublishedArtifact.syntax).resultName)
+  override def columns = Schema.columns(table)
+
+  def get(name: String)(implicit session: DBSession): PublishedArtifact =
+    getOption(name).get
+  def getOption(name: String)(implicit session: DBSession): Option[PublishedArtifact] =
+    withSQL { 
+      val b = PublishedArtifact.syntax 
+      select
+        .from(PublishedArtifact as b)
+        .where.eq(b.name, name)
+    }.map { apply(_) }.single.apply()
+  def make(
+    artifact: Artifact, 
+    name: Option[String] = None, 
+    properties: JsObject = Json.obj(),
+    overwrite: Boolean = false
+  )(implicit session: DBSession): PublishedArtifact =
+  {
+    val actualName = name.getOrElse { s"${artifact.t}_${artifact.id}"}
+    if(overwrite){
+      withSQL {
+        val a = PublishedArtifact.column
+        deleteFrom(PublishedArtifact)
+          .where.eq(a.name, actualName)
+      }.update.apply()
+    }
+    withSQL {
+      val a = PublishedArtifact.column
+      insertInto(PublishedArtifact)
+        .namedValues(
+          a.name -> actualName,
+          a.artifactId -> artifact.id,
+          a.projectId -> artifact.projectId,
+          a.properties -> properties
+        )
+    }.update.apply()
+    PublishedArtifact(
+      name = actualName,
+      artifactId = artifact.id,
+      projectId = artifact.projectId,
+      properties = properties
+    )
+  }
+
+  def nameFromURL(url: String): Option[String] = 
+  {
+    val base = 
+      VizierAPI.urls.publishedArtifact("").toString
+    if(url.startsWith(base)){
+      Some(url.drop(base.length).split("/").head)
+    } else { 
+      return None
+    }
+  }
+
+
+  {
+  }
+}