tsparser validation improvements (#1645)

- Reapply "tsparser: improved app validation" (#1633)
- tsparser: support validation in query/header/path

proto/encore/parser/meta/v1/meta.proto

@@ -314,6 +314,8 @@ message PathSegment {
   SegmentType type = 1;
   string value = 2;
   ParamType value_type = 3;
+
+  optional schema.v1.ValidationExpr validation = 4;
 }
 
 message Gateway {

runtimes/core/src/api/endpoint.rs

@@ -340,6 +340,7 @@ pub fn endpoints_from_meta(
                     r#type: meta::path_segment::SegmentType::Literal as i32,
                     value_type: meta::path_segment::ParamType::String as i32,
                     value: format!("/{}.{}", ep.ep.service_name, ep.ep.name),
+                    validation: None,
                 }],
             }),
             handshake,

runtimes/core/src/api/jsonschema/meta.rs

@@ -238,9 +238,6 @@ impl BuilderCtx<'_, '_> {
 
     #[inline]
     fn struct_field<'c>(&mut self, f: &'c schema::Field) -> Result<(&'c String, Field)> {
-        // Note: Our JS/TS support don't include the ability to change
-        // the JSON name from the field name, so we use the field name unconditionally.
-
         let typ = self.typ(&f.typ)?;
         let value = match typ {
             Value::Basic(basic) => BasicOrValue::Basic(basic),

runtimes/core/src/api/jsonschema/mod.rs

@@ -12,7 +12,7 @@ mod de;
 mod meta;
 mod parse;
 mod ser;
-mod validation;
+pub mod validation;
 
 use crate::api::jsonschema::parse::ParseWithSchema;
 use crate::api::APIResult;

runtimes/core/src/api/paths.rs

@@ -176,6 +176,7 @@ mod tests {
             r#type: typ as i32,
             value: value.to_string(),
             value_type: meta::path_segment::ParamType::String as i32,
+            validation: None,
         }
     }
 

runtimes/core/src/api/schema/encoding.rs

@@ -398,6 +398,7 @@ pub fn handshake_encoding(
             value: format!("{}.{}", rpc.service_name, rpc.name),
             r#type: SegmentType::Literal as i32,
             value_type: meta::path_segment::ParamType::String as i32,
+            validation: None,
         }],
         r#type: meta::path::Type::Url as i32,
     };
@@ -489,6 +490,7 @@ pub fn request_encoding(
             value: format!("{}.{}", rpc.service_name, rpc.name),
             r#type: SegmentType::Literal as i32,
             value_type: meta::path_segment::ParamType::String as i32,
+            validation: None,
         }],
         r#type: meta::path::Type::Url as i32,
     };

runtimes/core/src/api/schema/path.rs

@@ -21,7 +21,7 @@ use crate::encore::parser::meta::v1::path_segment::ParamType;
 pub struct Path {
     /// The path segments.
     segments: Vec<Segment>,
-    dynamic_segments: Vec<Basic>,
+    dynamic_segments: Vec<(Basic, Option<jsonschema::validation::Expr>)>,
 
     /// The capacity to use for generating requests.
     capacity: usize,
@@ -32,6 +32,15 @@ impl Path {
         let mut segments = Vec::with_capacity(path.segments.len());
         for seg in &path.segments {
             use meta::path_segment::SegmentType;
+            let validation = seg
+                .validation
+                .as_ref()
+                .map(|v| {
+                    jsonschema::validation::Expr::try_from(v)
+                        .context("invalid path segment validation")
+                })
+                .transpose()?;
+
             match SegmentType::try_from(seg.r#type).context("invalid path segment type")? {
                 SegmentType::Literal => {
                     segments.push(Segment::Literal(seg.value.clone().into_boxed_str()))
@@ -59,14 +68,17 @@ impl Path {
                     segments.push(Segment::Param {
                         name: name.clone().into_boxed_str(),
                         typ,
+                        validation,
                     });
                 }
 
                 SegmentType::Wildcard => segments.push(Segment::Wildcard {
                     name: seg.clone().value.into_boxed_str(),
+                    validation,
                 }),
                 SegmentType::Fallback => segments.push(Segment::Fallback {
                     name: seg.clone().value.into_boxed_str(),
+                    validation,
                 }),
             }
         }
@@ -82,14 +94,16 @@ impl Path {
             capacity += 1; // slash
             match seg {
                 Literal(lit) => capacity += lit.len(),
-                Param { typ, .. } => {
+                Param {
+                    typ, validation, ..
+                } => {
                     capacity += 10; // assume path parameters on average are 10 characters long
-                    dynamic_segments.push(*typ);
+                    dynamic_segments.push((*typ, validation.clone()));
                 }
-                Wildcard { .. } | Fallback { .. } => {
+                Wildcard { validation, .. } | Fallback { validation, .. } => {
                     // Assume path parameters on average are 10 characters long.
                     capacity += 10;
-                    dynamic_segments.push(jsonschema::Basic::String);
+                    dynamic_segments.push((jsonschema::Basic::String, validation.clone()));
                 }
             }
         }
@@ -110,12 +124,15 @@ pub enum Segment {
         name: Box<str>,
         /// The type of the path parameter.
         typ: jsonschema::Basic,
+        validation: Option<jsonschema::validation::Expr>,
     },
     Wildcard {
         name: Box<str>,
+        validation: Option<jsonschema::validation::Expr>,
     },
     Fallback {
         name: Box<str>,
+        validation: Option<jsonschema::validation::Expr>,
     },
 }
 
@@ -129,7 +146,7 @@ impl Path {
             use Segment::*;
             match seg {
                 Literal(lit) => path.push_str(lit),
-                Param { name, .. } | Wildcard { name } | Fallback { name } => {
+                Param { name, .. } | Wildcard { name, .. } | Fallback { name, .. } => {
                     let Some(payload) = payload else {
                         return Err(api::Error {
                             code: api::ErrCode::InvalidArgument,
@@ -220,7 +237,7 @@ impl Path {
 
                 // For each param, find the corresponding segment and deserialize it.
                 for (idx, (name, val)) in params.into_iter().enumerate() {
-                    if let Some(typ) = self.dynamic_segments.get(idx) {
+                    if let Some((typ, validation)) = self.dynamic_segments.get(idx) {
                         // Decode it into the correct type based on the type.
                         let val = match &typ {
                             // For strings and any, use the value directly.
@@ -270,6 +287,19 @@ impl Path {
                             Basic::Null => PValue::Null,
                         };
 
+                        // Validate the value, if we have a validation expression.
+                        if let Some(validation) = validation.as_ref() {
+                            if let Err(err) = validation.validate_pval(&val) {
+                                return Err(api::Error {
+                                    code: api::ErrCode::InvalidArgument,
+                                    message: format!("invalid path parameter {}: {}", name, err),
+                                    internal_message: None,
+                                    stack: None,
+                                    details: None,
+                                });
+                            }
+                        }
+
                         map.insert(name, val);
                     }
                 }

runtimes/go/storage/sqldb/db.go

@@ -326,6 +326,7 @@ func (db *Database) Begin(ctx context.Context) (*Tx, error) {
 // this will be made with backwards compatibility in mind, providing ample notice and
 // time to migrate in an opt-in fashion.
 func Driver[T SupportedDrivers](db *Database) T {
+	db.init()
 	if db.noopDB {
 		var zero T
 		return zero

tsparser/src/app/mod.rs

@@ -1,16 +1,20 @@
-use std::collections::HashMap;
+use std::collections::{HashMap, HashSet};
 
 use matchit::InsertError;
 use swc_common::errors::HANDLER;
+use swc_common::Span;
 
 use crate::encore::parser::meta::v1;
 use crate::legacymeta::compute_meta;
 use crate::parser::parser::{ParseContext, ParseResult};
-use crate::parser::resources::apis::api::{Method, Methods};
+use crate::parser::resources::apis::api::{Endpoint, Method, Methods};
 use crate::parser::resources::Resource;
 use crate::parser::respath::Path;
+use crate::parser::types::visitor::VisitWith;
+use crate::parser::types::{validation, visitor, ObjectId, ResolveState, Type, Validated};
 use crate::parser::Range;
-use litparser::ParseResult as PResult;
+use crate::span_err::ErrReporter;
+use litparser::Sp;
 
 #[derive(Debug)]
 pub struct AppDesc {
@@ -60,9 +64,31 @@ impl Router {
     }
 }
 
-impl AppDesc {
+pub fn validate_and_describe(pc: &ParseContext, parse: ParseResult) -> Option<AppDesc> {
+    AppValidator { pc, parse: &parse }.validate();
+
+    if pc.errs.has_errors() {
+        return None;
+    }
+
+    match compute_meta(pc, &parse) {
+        Ok(meta) => Some(AppDesc { parse, meta }),
+        Err(err) => {
+            err.report();
+            None
+        }
+    }
+}
+
+struct AppValidator<'a> {
+    pc: &'a ParseContext,
+    parse: &'a ParseResult,
+}
+
+impl AppValidator<'_> {
     fn validate(&self) {
-        self.validate_apis()
+        self.validate_apis();
+        self.validate_pubsub()
     }
 
     fn validate_apis(&self) {
@@ -72,17 +98,74 @@ impl AppDesc {
                 if let Resource::APIEndpoint(endpoint) = &bind.resource {
                     let encoding = &endpoint.encoding;
                     router.try_add(&encoding.methods, &encoding.path, endpoint.range);
+
+                    self.validate_endpoint(endpoint);
                 }
             }
         }
     }
-}
 
-pub fn validate_and_describe(pc: &ParseContext, parse: ParseResult) -> PResult<AppDesc> {
-    let meta = compute_meta(pc, &parse)?;
-    let desc = AppDesc { parse, meta };
+    fn validate_endpoint(&self, ep: &Endpoint) {
+        if let Some(schema) = &ep.encoding.raw_req_schema {
+            self.validate_validations(schema);
+        }
+        if let Some(schema) = &ep.encoding.raw_resp_schema {
+            self.validate_validations(schema);
+        }
+    }
+
+    fn validate_validations(&self, schema: &Sp<Type>) {
+        struct Visitor<'a> {
+            state: &'a ResolveState,
+            span: Span,
+            seen_decls: HashSet<ObjectId>,
+        }
+
+        impl visitor::Visit for Visitor<'_> {
+            fn resolve_state(&self) -> &ResolveState {
+                self.state
+            }
+            fn seen_decls(&mut self) -> &mut HashSet<ObjectId> {
+                &mut self.seen_decls
+            }
 
-    desc.validate();
+            fn visit_validated(&mut self, node: &Validated) {
+                if let Err(err) = node.expr.supports_type(&node.typ) {
+                    let s = err.to_string();
+                    self.span.err(&s);
+                } else {
+                    // Don't recurse into the validation expression, as it would report an error
+                    // below as if the expression was standalone.
+                    node.typ.visit_with(self);
+                }
+            }
 
-    Ok(desc)
+            fn visit_validation(&mut self, node: &validation::Expr) {
+                HANDLER.with(|h| {
+                    h.struct_span_err(
+                        self.span,
+                        &format!("unsupported standalone validation expression: {}", node),
+                    )
+                    .note("validation expressions must be attached to a regular type using '&'")
+                    .emit()
+                });
+            }
+        }
+
+        let state = self.pc.type_checker.state();
+        let mut visitor = Visitor {
+            state,
+            span: schema.span(),
+            seen_decls: HashSet::new(),
+        };
+        schema.visit_with(&mut visitor);
+    }
+
+    fn validate_pubsub(&self) {
+        for res in self.parse.resources.iter() {
+            if let Resource::PubSubTopic(topic) = &res {
+                self.validate_validations(&topic.message_type);
+            }
+        }
+    }
 }

tsparser/src/builder/parse.rs

@@ -35,13 +35,7 @@ impl Builder<'_> {
         let parser = Parser::new(pc, pass1);
 
         let result = parser.parse();
-        let desc = match validate_and_describe(pc, result) {
-            Ok(desc) => desc,
-            Err(err) => {
-                err.report();
-                return None;
-            }
-        };
+        let desc = validate_and_describe(pc, result)?;
 
         if pc.errs.has_errors() {
             None

tsparser/src/legacymeta/mod.rs

@@ -12,6 +12,7 @@ use crate::parser::resources::apis::{authhandler, gateway};
 use crate::parser::resources::infra::cron::CronJobSchedule;
 use crate::parser::resources::infra::{cron, objects, pubsub_subscription, pubsub_topic, sqldb};
 use crate::parser::resources::Resource;
+use crate::parser::types::validation;
 use crate::parser::types::{Object, ObjectId};
 use crate::parser::usageparser::Usage;
 use crate::parser::{respath, FilePath, Range};
@@ -696,25 +697,33 @@ impl respath::Path {
                         r#type: SegmentType::Literal as i32,
                         value_type: ParamType::String as i32,
                         value: lit.clone(),
+                        validation: None,
                     },
-                    Segment::Param { name, value_type } => v1::PathSegment {
+                    Segment::Param {
+                        name,
+                        value_type,
+                        validation,
+                    } => v1::PathSegment {
                         r#type: SegmentType::Param as i32,
                         value_type: match value_type {
                             ValueType::String => ParamType::String as i32,
                             ValueType::Int => ParamType::Int as i32,
                             ValueType::Bool => ParamType::Bool as i32,
                         },
                         value: name.clone(),
+                        validation: validation.as_ref().map(validation::Expr::to_pb),
                     },
-                    Segment::Wildcard { name } => v1::PathSegment {
+                    Segment::Wildcard { name, validation } => v1::PathSegment {
                         r#type: SegmentType::Wildcard as i32,
                         value_type: ParamType::String as i32,
                         value: name.clone(),
+                        validation: validation.as_ref().map(validation::Expr::to_pb),
                     },
-                    Segment::Fallback { name } => v1::PathSegment {
+                    Segment::Fallback { name, validation } => v1::PathSegment {
                         r#type: SegmentType::Fallback as i32,
                         value_type: ParamType::String as i32,
                         value: name.clone(),
+                        validation: validation.as_ref().map(validation::Expr::to_pb),
                     },
                 })
                 .collect(),