More improvements + some docs

README.md

@@ -1,20 +1,76 @@
 # Restate SDK test suite
 
-TODO, more coming soon
+This tool is a test suite/conformance suite runner for Restate SDKs.
+
+## Architecture
+
+This tool requires the SDK developer to implement a well-defined set of services, specified [here](./src/main/kotlin/dev/restate/sdktesting/contracts), and package the application as docker container. 
+For an example implementation, look [here](https://github.com/restatedev/sdk-java/tree/main/test-services/src/main/kotlin/dev/restate/sdk/testservices).
+
+The tool then starts for each test class Restate alongside with the service deployment container, and sends some requests to it.
+
+The tool is split into test suite, test classes and test methods. 
+A test suite is composed by multiple test classes, which itself is composed by multiple test methods. 
+Test methods usually run in parallel wrt the other methods in the same class, and each individual class deploys an isolated docker network with its own runtime and its own service deployment container. 
+Test suites run the same set of test classes, with different parameters of the runtime tuned.
 
 ## CI usage
 
-* Download the test-suite jar already built
-* Prepare the service docker image with the services required by the tests
-* Run `java -jar restate-sdk-test-suite.jar run restatedev/e2e-java-services`
-* package the reports to publish on the github ui
+Check as example: https://github.com/restatedev/sdk-python/blob/main/.github/workflows/integration.yaml
+
+## Running locally
+
+You need a JVM >= 17, and the test tool. You can get the test tool from the [releases page](https://github.com/restatedev/sdk-test-suite/releases). If you're investigating a CI failure, you might need the exact tool version the CI is currently using (which is usually printed in the title of the CI task).
+
+To run all the tests:
+
+```shell
+java -jar restate-sdk-test-suite.jar run <TEST_SERVICES_IMAGE>
+```
+
+To run an individual test:
+
+```shell
+java -jar restate-sdk-test-suite.jar run --test-suite=<SUITE> --test-name=<TEST_NAME> <TEST_SERVICES_IMAGE>
+```
+
+To change the runtime container image, use `--restate-container-image=<RUNTIME_CONTAINER_IMAGE>`.
 
-## Local debugging usage
+By default, the tool will always pull images, unless they have the repository prefix `restate.local` or `localhost`. To always try to use the local cache, use `--image-pull-policy=CACHED`.
+
+### Exclusions
+
+Some SDKs might not implement all the features. For this purpose, the tool allows to configure the excluded test classes in an ad-hoc file. To run with the exclusions file:
+
+```shell
+java -jar restate-sdk-test-suite.jar run --exclusions-file exclusions.yaml <TEST_SERVICES_IMAGE>
+```
+
+After every run a new exclusions file is generated in the test report, with the failed/skipped tests.
+
+## Local debug
+
+You can debug the service deployment by exposing it to a local port, and ask the tool to use a local port instead of deploying a container. To do so:
 
 * Run the service with your IDE and the debugger
-* Run `java -jar restate-sdk-test-suite.jar debug --test-suite=<TEST_SUITE> --test-name=<TEST_NAME> default-service=9080`
+* Run `java -jar restate-sdk-test-suite.jar debug --test-suite=<TEST_SUITE> --test-name=<TEST_NAME> default-service=<PORT>`
+
+When the test starts, it will also print to console the environment variables the service deployment should be started with.
+
+Using the `debug` command, you can also:
+
+* Expose the environment to your localhost (e.g. to use the CLI to introspect the state) and keep it up and running after the end of the test using:
+
+```shell
+java -jar restate-sdk-test-suite.jar debug \
+  --local-ingress-port=8080 --local-admin-port=9070 --retain-after-end \
+  --test-suite=<TEST_SUITE> --test-name=<TEST_NAME> \
+  default-service=<PORT>
+```
+
+Please note, some tests requiring to kill/stop the service deployment won't work with the `debug` command.
 
-## Releasing
+## Releasing this tool
 
 Just push a new git tag:
 

src/main/kotlin/dev/restate/sdktesting/infra/RestateContainer.kt

@@ -66,9 +66,11 @@ class RestateContainer(
     }
 
     if (config.localAdminPort != null) {
+      LOG.info("Going to expose Admin port on localhost:{}", config.localAdminPort)
       super.addFixedExposedPort(config.localAdminPort, RUNTIME_META_ENDPOINT_PORT)
     }
     if (config.localIngressPort != null) {
+      LOG.info("Going to expose Admin port on localhost:{}", config.localIngressPort)
       super.addFixedExposedPort(config.localIngressPort, RUNTIME_INGRESS_ENDPOINT_PORT)
     }
   }

src/main/kotlin/dev/restate/sdktesting/infra/RestateDeployer.kt

@@ -19,6 +19,7 @@ import java.io.File
 import java.net.URI
 import java.net.http.HttpClient
 import java.nio.file.Path
+import java.util.*
 import java.util.concurrent.TimeUnit
 import org.apache.logging.log4j.LogManager
 import org.junit.jupiter.api.extension.ExtensionContext
@@ -104,7 +105,6 @@ private constructor(
           mapOf(
               "restate_invoker" to "trace",
               "restate_ingress_kafka" to "trace",
-              "restate_worker::partition::services::non_deterministic::remote_context" to "trace",
               "restate" to "debug")
       val defaultLog =
           (listOf("info") + defaultLogFilters.map { "${it.key}=${it.value}" }).joinToString(
@@ -142,7 +142,7 @@ private constructor(
           .port
   private val restateUri = "http://$RESTATE_RUNTIME:$ingressPort/"
 
-  private val network = Network.newNetwork()
+  private val network = Network.newNetwork()!!
   private val serviceContainers =
       serviceSpecs
           .mapNotNull {
@@ -315,7 +315,14 @@ private constructor(
           }
         }
 
-    LOG.debug("Successfully executed discovery for endpoint {}. Result: {}", url, response)
+    LOG.debug(
+        """
+      Successfully executed discovery for endpoint {}, registered with id {}. Discovered services: {}
+    """
+            .trimIndent(),
+        url,
+        response.id,
+        response.services.map { it.name })
   }
 
   private fun writeEnvironmentReport(testReportDir: String) {
@@ -359,11 +366,16 @@ private constructor(
   }
 
   private fun teardownAll() {
+    if (config.retainAfterEnd) {
+      LOG.info("Press a button to cleanup the test environment. Deployed network: {}", network.id)
+      System.`in`.read()
+      return
+    }
     teardownRuntime()
     teardownAdditionalContainers()
     teardownServices()
     teardownProxy()
-    network!!.close()
+    network.close()
   }
 
   internal fun getContainerPort(hostName: String, port: Int): Int {

src/main/kotlin/dev/restate/sdktesting/infra/RestateDeployerConfig.kt

@@ -27,7 +27,8 @@ data class RestateDeployerConfig(
     val additionalRuntimeEnvs: Map<String, String> = mapOf(),
     val stateDirectoryMount: String? = null,
     val localIngressPort: Int? = null,
-    val localAdminPort: Int? = null
+    val localAdminPort: Int? = null,
+    val retainAfterEnd: Boolean = false
 ) {
   init {
     check(serviceDeploymentConfig.containsKey(ServiceSpec.DEFAULT_SERVICE_NAME)) {

src/main/kotlin/dev/restate/sdktesting/infra/RestateDeployerExtension.kt

@@ -17,8 +17,8 @@ class RestateDeployerExtension(private val deployerFactory: RestateDeployer.Buil
     val builder = RestateDeployer.builder()
     deployerFactory.invoke(builder)
     val deployer = builder.build()
+    context.getStore(NAMESPACE).put(DEPLOYER_KEY, deployer)
     deployer.deployAll(
         RestateDeployer.reportDirectory(getReportPath(context), context.requiredTestClass))
-    context.getStore(NAMESPACE).put(DEPLOYER_KEY, deployer)
   }
 }

src/main/kotlin/dev/restate/sdktesting/junit/TestSuite.kt

@@ -51,10 +51,11 @@ class TestSuite(
     Configurator.reconfigure(log4j2Configuration)
 
     // Apply additional runtime envs
-    registerGlobalConfig(getGlobalConfig().copy(additionalRuntimeEnvs = additionalEnvs))
+    val restateDeployerConfig = getGlobalConfig().copy(additionalRuntimeEnvs = additionalEnvs)
+    registerGlobalConfig(restateDeployerConfig)
 
     // Prepare launch request
-    val request =
+    var builder =
         LauncherDiscoveryRequestBuilder.request()
             .selectors(DiscoverySelectors.selectPackage("dev.restate.sdktesting.tests"))
             .filters(TagFilter.includeTags(junitIncludeTags))
@@ -68,7 +69,15 @@ class TestSuite(
             .configurationParameter(
                 "junit.jupiter.execution.parallel.mode.classes.default",
                 if (parallel) "concurrent" else "same_thread")
-            .build()
+
+    // Disable lifecycle timeout
+    if (restateDeployerConfig.retainAfterEnd) {
+      builder =
+          builder.configurationParameter(
+              "junit.jupiter.execution.timeout.lifecycle.method.default", "360m")
+    }
+
+    val request = builder.build()
 
     // Configure listeners
     val errWriter = PrintWriter(System.err)
@@ -136,7 +145,6 @@ class TestSuite(
 
       rootLogger.add(builder.newAppenderRef("stdout"))
       restateLogger.add(builder.newAppenderRef("stdout"))
-      testContainersLogger.add(builder.newAppenderRef("stdout"))
     }
 
     builder.add(fileAppender)

src/main/kotlin/dev/restate/sdktesting/main.kt

@@ -19,6 +19,7 @@ import com.github.ajalt.clikt.parameters.groups.cooccurring
 import com.github.ajalt.clikt.parameters.groups.provideDelegate
 import com.github.ajalt.clikt.parameters.options.*
 import com.github.ajalt.clikt.parameters.types.enum
+import com.github.ajalt.clikt.parameters.types.int
 import com.github.ajalt.clikt.parameters.types.path
 import com.github.ajalt.mordant.rendering.TextColors.green
 import com.github.ajalt.mordant.rendering.TextColors.red
@@ -119,7 +120,8 @@ Run test suite, executing the service as container.
     val restateDeployerConfig =
         RestateDeployerConfig(
             mapOf(ServiceSpec.DEFAULT_SERVICE_NAME to ContainerServiceDeploymentConfig(imageName)),
-            deployInParallel = parallel)
+            deployInParallel = parallel,
+        )
 
     // Register global config of the deployer
     registerGlobalConfig(testRunnerOptions.applyToDeployerConfig(restateDeployerConfig))
@@ -243,6 +245,17 @@ Run test suite, without executing the service inside a container.
           .multiple(required = true)
           .help(
               "Local containers name=ports. Example: '9080' (for default-service container), 'otherContainer=9081'")
+  val retainAfterEnd by
+      option()
+          .flag("--dont-retain-after-end", default = false)
+          .help(
+              "Retain the created docker network after the end of the test. You MUST manually clean it up afterwards!")
+  val mountStateDirectory by
+      option()
+          .help(
+              "Mount the given state directory as restate data when starting the runtime container")
+  val localIngressPort by option().int().help("Ingress port to bind the runtime container")
+  val localAdminPort by option().int().help("Ingress port to bind the admin container")
 
   override fun run() {
     val terminal = Terminal()
@@ -252,9 +265,19 @@ Run test suite, without executing the service inside a container.
         RestateDeployerConfig(
             localContainers.associate {
               it.first to LocalForwardServiceDeploymentConfig(it.second)
-            })
+            },
+            localAdminPort = this.localAdminPort,
+            localIngressPort = this.localIngressPort,
+            stateDirectoryMount = this.mountStateDirectory,
+            retainAfterEnd = this.retainAfterEnd)
     registerGlobalConfig(testRunnerOptions.applyToDeployerConfig(restateDeployerConfig))
 
+    if (restateDeployerConfig.retainAfterEnd) {
+      // Disable ryuk, as it will otherwise cleanup the network after the JVM goes away.
+      //      System.getenv().put("TESTCONTAINERS_RYUK_DISABLED", "true")
+
+    }
+
     // Resolve test configurations
     val testSuite = TestSuites.resolveSuites(testSuite)[0]
     val testFilters =