Merge pull request kubewarden#415 from flavio/extend-e2e-tests

fix: make e2e tests more scrupulous

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "policy-server"
-version = "1.5.2"
+version = "1.5.3"
 authors = [
   "Kubewarden Developers <[email protected]>",
   "Flavio Castelli <[email protected]>",
@@ -20,7 +20,7 @@ lazy_static = "1.4.0"
 num_cpus = "1.15.0"
 opentelemetry-otlp = { version = "0.10.0", features = ["metrics", "tonic"] }
 opentelemetry = { version = "0.17", default-features = false, features = ["metrics", "trace", "rt-tokio", "serialize"] }
-policy-evaluator = { git = "https://github.com/kubewarden/policy-evaluator", tag = "v0.6.2" }
+policy-evaluator = { git = "https://github.com/kubewarden/policy-evaluator", tag = "v0.6.3" }
 rayon = "1.6"
 serde_json = "1.0"
 serde = { version = "1.0", features = ["derive"] }

e2e-tests/02-monitor-and-protect-modes.yml

@@ -23,6 +23,7 @@ testcases:
     assertions:
     - result.statuscode ShouldEqual 200
     - result.bodyjson.response.allowed ShouldEqual false
+    - result.bodyjson.response.status.code ShouldNotEqual 500
 
 - name: The `/validate` endpoint accepts the request because it's in `monitor` mode
   steps:

e2e-tests/03-audit-endpoint.yml

@@ -35,3 +35,4 @@ testcases:
       assertions:
       - result.statuscode ShouldEqual 200
       - result.bodyjson.response.allowed ShouldEqual false
+      - result.bodyjson.response.status.code ShouldNotEqual 500

e2e-tests/04-timeout-protection.yml

@@ -45,6 +45,8 @@ testcases:
       assertions:
       - result.statuscode ShouldEqual 200
       - result.bodyjson.response.allowed ShouldEqual false
+      - result.bodyjson.response.status.code ShouldEqual 500
+      - result.bodyjson.response.status.message ShouldContainSubstring "execution deadline exceeded"
       - result.timeseconds ShouldBeLessThan 3
 
 - name: Accept AGAIN a request that has a short evaluation time

e2e-tests/05-rego-policy.yml

@@ -21,7 +21,8 @@ testcases:
 
 - name: Rego-based policies work as expected
   steps:
-  - type: http
+  - name: Send violating request
+    type: http
     method: POST
     url: http://localhost:3000/validate/disallow-service-loadbalancer
     headers:
@@ -30,7 +31,9 @@ testcases:
     assertions:
     - result.statuscode ShouldEqual 200
     - result.bodyjson.response.allowed ShouldEqual false
-  - type: http
+    - result.bodyjson.response.status.code ShouldNotEqual 500
+  - name: Send non violating request
+    type: http
     method: POST
     url: http://localhost:3000/validate/disallow-service-loadbalancer
     headers: