Merge pull request #485 from TNO/474-the-ke_runtime_exposed_url-shoul…

…d-include-the-url-schema-https

474 the ke runtime exposed url should include the url schema https

examples/runtime-tests/.gitignore

@@ -0,0 +1 @@
+.python-version

examples/runtime-tests/README.md

@@ -0,0 +1,5 @@
+## Runtime tests
+This docker compose example is used to test several scenario's that are impossible (or difficult) to test within Java Unit Tests.
+
+### Incorrect `KE_RUNTIME_EXPOSED_URL`
+The `docker-compose.yml` contains the exact same scenario as the `multiple-runtimes` example, but additionally it contains several broken runtimes where the exposed URL is incorrect. With this docker project we can test whether these different types of exposed urls are breaking their own KER, other KERs or the Knowledge Directory.

examples/runtime-tests/docker-compose.yml

@@ -0,0 +1,179 @@
+services:
+  # This is the knowledge directory, facilitating discovery between different
+  # runtimes. It exposes its service over port 8282.
+  knowledge-directory:
+    image: ghcr.io/tno/knowledge-engine/knowledge-directory:1.2.3
+
+  # These services are seperate Knowledge Engine runtime, which can host
+  # multiple smart connectors. Note that the REST API port is a DIFFERENT port
+  # number than the ones configured below. It is still the default 8280.
+  runtime-1:
+    image: ghcr.io/tno/knowledge-engine/smart-connector:1.2.4-SNAPSHOT
+    environment: 
+      KE_RUNTIME_PORT: 8081 # The port that the KE uses to listen for inter-KE-runtime communication.
+      KE_RUNTIME_EXPOSED_URL: http://runtime-1:8081 # The URL where the runtime is available for inter-runtime communication from the outside.
+      KD_URL: http://knowledge-directory:8282
+  runtime-2:
+    image: ghcr.io/tno/knowledge-engine/smart-connector:1.2.4-SNAPSHOT
+    environment: 
+      KE_RUNTIME_PORT: 8081
+      KE_RUNTIME_EXPOSED_URL: http://runtime-2:8081
+      KD_URL: http://knowledge-directory:8282
+  runtime-3:
+    image: ghcr.io/tno/knowledge-engine/smart-connector:1.2.4-SNAPSHOT
+    environment: 
+      KE_RUNTIME_PORT: 8081
+      KE_RUNTIME_EXPOSED_URL: http://runtime-3:8081
+      KD_URL: http://knowledge-directory:8282
+
+  # These Knowledge Bases use the different runtimes, and exchange data with eachother.
+  kb1:
+    build: ../common/asking_kb
+    environment:
+      KE_URL: http://runtime-1:8280/rest
+      KB_ID: http://example.org/kb1
+      PREFIXES: |
+        {
+          "ex": "http://example.org/"
+        }
+      GRAPH_PATTERN: |
+        ?a ex:relatedTo ?b .
+  kb2:
+    build: ../common/answering_kb
+    environment:
+      KE_URL: http://runtime-2:8280/rest
+      KB_ID: http://example.org/kb2
+      PREFIXES: |
+        {
+          "ex": "http://example.org/"
+        }
+      GRAPH_PATTERN: |
+        ?a ex:relatedTo ?b .
+      KB_DATA: |
+        [
+          {
+            "a": "<http://example.org/Math>",
+            "b": "<http://example.org/Science>"
+          },
+          {
+            "a": "<http://example.org/Books>",
+            "b": "<http://example.org/Magazines>"
+          }
+        ]
+  kb3:
+    build: ../common/answering_kb
+    environment:
+      KE_URL: http://runtime-3:8280/rest
+      KB_ID: http://example.org/kb3
+      PREFIXES: |
+        {
+          "ex": "http://example.org/"
+        }
+      GRAPH_PATTERN: |
+        ?a ex:relatedTo ?b .
+      KB_DATA: |
+        [
+          {
+            "a": "<http://example.org/Music>",
+            "b": "<http://example.org/Songs>"
+          },
+          {
+            "a": "<http://example.org/Red>",
+            "b": "<http://example.org/Blue>"
+          }
+        ]
+  broken-1:
+    image: ghcr.io/tno/knowledge-engine/smart-connector:1.2.4-SNAPSHOT
+    environment: 
+      KE_RUNTIME_PORT: 8081 # The port that the KE uses to listen for inter-KE-runtime communication.
+      KE_RUNTIME_EXPOSED_URL: runtime-3:8081 # The 'http://' part of the URL with a port is missing.
+      KD_URL: http://knowledge-directory:8282
+  brokenkb1:
+    build: ../common/answering_kb
+    environment:
+      KE_URL: http://broken-1:8280/rest
+      KB_ID: http://example.org/brokenkb1
+      PREFIXES: |
+        {
+          "ex": "http://example.org/"
+        }
+      GRAPH_PATTERN: |
+        ?a ex:relatedTo ?b .
+      KB_DATA: |
+        []
+  broken-2:
+    image: ghcr.io/tno/knowledge-engine/smart-connector:1.2.4-SNAPSHOT
+    environment: 
+      KE_RUNTIME_PORT: 8081 # The port that the KE uses to listen for inter-KE-runtime communication.
+      KE_RUNTIME_EXPOSED_URL: 3test.runtime.nl:8081 # The 'http://' part of the URL with a port is missing and the first character is numeric.
+      KD_URL: http://knowledge-directory:8282
+  brokenkb2:
+    build: ../common/answering_kb
+    environment:
+      KE_URL: http://broken-2:8280/rest
+      KB_ID: http://example.org/brokenkb2
+      PREFIXES: |
+        {
+          "ex": "http://example.org/"
+        }
+      GRAPH_PATTERN: |
+        ?a ex:relatedTo ?b .
+      KB_DATA: |
+        []
+  broken-3:
+    image: ghcr.io/tno/knowledge-engine/smart-connector:1.2.4-SNAPSHOT
+    environment: 
+      KE_RUNTIME_PORT: 8081 # The port that the KE uses to listen for inter-KE-runtime communication.
+      KE_RUNTIME_EXPOSED_URL: 3test.runtime.nl # The 'http://' part of the URL without a port is missing and the first character is numeric.
+      KD_URL: http://knowledge-directory:8282
+  brokenkb3:
+    build: ../common/answering_kb
+    environment:
+      KE_URL: http://broken-3:8280/rest
+      KB_ID: http://example.org/brokenkb3
+      PREFIXES: |
+        {
+          "ex": "http://example.org/"
+        }
+      GRAPH_PATTERN: |
+        ?a ex:relatedTo ?b .
+      KB_DATA: |
+        []
+  broken-4:
+    image: ghcr.io/tno/knowledge-engine/smart-connector:1.2.4-SNAPSHOT
+    environment: 
+      KE_RUNTIME_PORT: 8081 # The port that the KE uses to listen for inter-KE-runtime communication.
+      KE_RUNTIME_EXPOSED_URL: http://runtime-3:8081/ # The URL ends with a '/'
+      KD_URL: http://knowledge-directory:8282
+  brokenkb4:
+    build: ../common/answering_kb
+    environment:
+      KE_URL: http://broken-4:8280/rest
+      KB_ID: http://example.org/brokenkb4
+      PREFIXES: |
+        {
+          "ex": "http://example.org/"
+        }
+      GRAPH_PATTERN: |
+        ?a ex:relatedTo ?b .
+      KB_DATA: |
+        []
+  broken-5:
+    image: ghcr.io/tno/knowledge-engine/smart-connector:1.2.4-SNAPSHOT
+    environment: 
+      KE_RUNTIME_PORT: 8081 # The port that the KE uses to listen for inter-KE-runtime communication.
+      KE_RUNTIME_EXPOSED_URL: runtime-3 # The 'http://' part of the URL without a port is missing.
+      KD_URL: http://knowledge-directory:8282
+  brokenkb5:
+    build: ../common/answering_kb
+    environment:
+      KE_URL: http://broken-5:8280/rest
+      KB_ID: http://example.org/brokenkb5
+      PREFIXES: |
+        {
+          "ex": "http://example.org/"
+        }
+      GRAPH_PATTERN: |
+        ?a ex:relatedTo ?b .
+      KB_DATA: |
+        []

smart-connector-rest-dist/pom.xml

@@ -43,6 +43,13 @@
 	</dependencies>
 	<build>
 		<plugins>
+			<plugin>
+				<groupId>org.apache.maven.plugins</groupId>
+				<artifactId>maven-surefire-plugin</artifactId>
+				<configuration>
+					<reuseForks>false</reuseForks> <!-- This will make sure to spawn a fresh jvm for each test -->
+				</configuration>
+			</plugin>
 			<plugin>
 				<groupId>org.apache.maven.plugins</groupId>
 				<artifactId>maven-dependency-plugin</artifactId>

smart-connector/src/main/java/eu/knowledge/engine/smartconnector/runtime/KeRuntime.java

@@ -1,7 +1,9 @@
 package eu.knowledge.engine.smartconnector.runtime;
 
+import java.net.MalformedURLException;
 import java.net.URI;
 import java.net.URISyntaxException;
+import java.net.URL;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.ThreadFactory;
@@ -41,6 +43,27 @@ public class KeRuntime {
 			System.exit(1);
 		}
 
+		// execute some validation on the EXPOSED URL, because it can have severe
+		// consequences
+		String url = getConfigProperty(CONF_KEY_MY_EXPOSED_URL, null);
+		if (url != null) {
+			if (url.endsWith("/")) {
+				LOG.error(
+						"The '{}' environment variable's value '{}' should be a valid URL without a slash ('/') as the last character.",
+						CONF_KEY_MY_EXPOSED_URL, url);
+				System.exit(1);
+			}
+			try {
+				URL exposedUrl = new URL(url);
+
+			} catch (MalformedURLException e) {
+				LOG.error(
+						"The '{}' environment variable with value '{}' contains a malformed URL '{}'.",
+						CONF_KEY_MY_EXPOSED_URL, url, e.getMessage());
+				System.exit(1);
+			}
+		}
+
 		// we want to make sure that this threadpool does not keep the JVM alive. So we
 		// set the daemon to true.
 		executorService = Executors.newScheduledThreadPool(12, new ThreadFactory() {

smart-connector/src/main/java/eu/knowledge/engine/smartconnector/runtime/messaging/RemoteKerConnection.java

@@ -146,11 +146,11 @@ private void updateRemoteKerDataFromPeer() {
 						"Failed to receive runtimedetails from {}, got status code {}. Trying KER again in {} minutes.",
 						this.remoteKerUri, response.statusCode(), waitTime);
 			}
-		} catch (IOException | URISyntaxException | InterruptedException e) {
+		} catch (IOException | URISyntaxException | InterruptedException | IllegalArgumentException e) {
 			this.remoteKerDetails = null;
 			int waitTime = errorOccurred();
-			LOG.warn("Failed to receive runtimedetails from " + this.remoteKerConnectionDetails.getId()
-					+ ". Trying KER again in " + waitTime + " minutes.");
+			LOG.warn("Failed to receive runtimedetails from {}, got error '{}'. Trying KER again in {} minutes.",
+					this.remoteKerConnectionDetails.getId(), e.getMessage(), waitTime);
 			LOG.debug("", e);
 		}
 		dispatcher.notifySmartConnectorsChanged();
@@ -245,9 +245,10 @@ public void stop() {
 					LOG.warn("Failed to say goodbye to {}, got response {}: {}", this.remoteKerUri,
 							response.statusCode(), response.body());
 				}
-			} catch (IOException | URISyntaxException | InterruptedException e) {
+			} catch (IOException | URISyntaxException | InterruptedException | IllegalArgumentException e) {
 				this.remoteKerDetails = null;
-				LOG.warn("Failed to say goodbye to " + remoteKerConnectionDetails.getId());
+				LOG.warn("Failed to say goodbye to {}, get error '{}'", remoteKerConnectionDetails.getId(),
+						e.getMessage());
 				LOG.debug("", e);
 			}
 		} else
@@ -297,10 +298,11 @@ public void sendToRemoteSmartConnector(KnowledgeMessage message) throws IOExcept
 					throw new IOException("Message not accepted by remote host, status code " + response.statusCode()
 							+ ", body " + response.body());
 				}
-			} catch (URISyntaxException | InterruptedException | IOException e) {
+			} catch (URISyntaxException | InterruptedException | IOException | IllegalArgumentException e) {
 				this.remoteKerDetails = null;
 				int time = this.errorOccurred();
-				LOG.warn("Ignoring KER {} for {} minutes.", this.remoteKerUri, time);
+				LOG.warn("Ignoring KER {} for {} minutes. Error '{}' occurred.", this.remoteKerUri, time,
+						e.getMessage());
 				this.dispatcher.notifySmartConnectorsChanged();
 				throw new IOException(e);
 			}
@@ -329,7 +331,7 @@ public void sendMyKerDetailsToPeer(KnowledgeEngineRuntimeDetails details) {
 							"Ignoring KER {} for {} minutes. Failed to send updated KnowledgeEngineRuntimeDetails, got response {}: {}",
 							this.remoteKerUri, time, response.statusCode(), response.body());
 				}
-			} catch (IOException | URISyntaxException | InterruptedException e) {
+			} catch (IOException | URISyntaxException | InterruptedException | IllegalArgumentException e) {
 				this.remoteKerDetails = null;
 				int time = this.errorOccurred();
 				this.dispatcher.notifySmartConnectorsChanged();