diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml new file mode 100644 index 00000000..170a9c74 --- /dev/null +++ b/.github/workflows/maven.yml @@ -0,0 +1,31 @@ +# This workflow will build a Java project with Maven, and cache/restore any dependencies to improve the workflow execution time +# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-java-with-maven + +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +name: Java CI with Maven + +on: + push: + branches: [ "master" ] + pull_request: + branches: [ "master" ] + +env: + MAVEN_OPTS: -Dhttps.protocols=TLSv1.2 -Dmaven.repo.local=$CI_PROJECT_DIR/.m2/repository -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=WARN -Dorg.slf4j.simpleLogger.showDateTime=true -Djava.awt.headless=true + MAVEN_CLI_OPTS: --batch-mode --errors --fail-at-end --show-version + +jobs: + build: + runs-on: ubuntu-latest + container: maven:3.8.7-eclipse-temurin-17-alpine + + steps: + - uses: actions/checkout@v4 + - name: Compile test source code with Maven + run: mvn $MAVEN_CLI_OPTS test-compile + - name: Run the Maven verify phase + run: mvn $MAVEN_CLI_OPTS verify diff --git a/README.md b/README.md index 1071dfcb..5247f22c 100644 --- a/README.md +++ b/README.md @@ -71,7 +71,7 @@ The easiest way to start a Knowledge Engine runtime is with Docker: ```bash docker run \ -p 8280:8280 \ - ghcr.io/tno/knowledge-engine/smart-connector:1.2.1 + ghcr.io/tno/knowledge-engine/smart-connector:1.2.3 ``` The Knowledge Engine runtime is now available to use via the REST API at base URL `http://localhost:8280/rest` on your host machine. @@ -93,7 +93,7 @@ docker run \ -p 8081:8081 \ -e KD_URL=https://knowledge-directory.example.org \ -e KE_RUNTIME_EXPOSED_URL=https://your-domain.example.org:8081 \ - ghcr.io/tno/knowledge-engine/smart-connector:1.2.1 + ghcr.io/tno/knowledge-engine/smart-connector:1.2.3 ``` ### Running with Java @@ -111,7 +111,7 @@ export KE_RUNTIME_EXPOSED_URL=https://your-domain.example.org:8081 # Start it. The argument (8280) denotes the port number at which it # will listen for connections to the Knowledge Engine REST API. java -jar -Dorg.slf4j.simpleLogger.logFile=smart-connector.log \ - smart-connector-rest-dist-1.2.1-with-dependencies.jar 8280 + smart-connector-rest-dist-1.2.3-with-dependencies.jar 8280 ``` The JAR can be retrieved by compiling the project: @@ -187,7 +187,7 @@ The Knowledge Engine project consists of the following Maven modules: - `smart-connector-rest-server` - This module contains the REST API layer that is built on top of the Java Developer API. - `smart-connector-rest-dist` - - A distribution of the server that provides the REST API layer for your smart connector(s), and uses the smart connector implementation from the `smart-connector` module. For instructions on how to use it, refer to [the section below](#how-to-use-the-rest-api). For instructions on how to set it up, refer to [this section](#how-to-administer-the-rest-api). + - A distribution of the server that provides the REST API layer for your smart connector(s), and uses the smart connector implementation from the `smart-connector` module. For instructions on how to use it, refer to [the section below](#using-the-rest-api). For instructions on how to set it up, refer to [this section](#advanced-administering-a-knowledge-engine-runtime). - `admin-ui` - A REST API which provides meta-data about smart connectors in a knowledge network. Can be used in an administration inferface for a knowledge network. It is implemented as a knowledge base that uses metadata of other knowledge bases. @@ -205,17 +205,17 @@ These are instructions on what to do when we release a new version of the knowle 5. Build and push the new Docker images to GitLab: ```bash -docker buildx build ./smart-connector-rest-dist --platform linux/arm64,linux/amd64 --tag docker-registry.inesctec.pt/interconnect/knowledge-engine/smart-connector-rest-dist:1.2.1 --push -docker buildx build ./knowledge-directory --platform linux/arm64,linux/amd64 --tag docker-registry.inesctec.pt/interconnect/knowledge-engine/knowledge-directory:1.2.1 --push -docker buildx build ./admin-ui --platform linux/arm64,linux/amd64 --tag docker-registry.inesctec.pt/interconnect/knowledge-engine/admin-ui:1.2.1 --push +docker buildx build ./smart-connector-rest-dist --platform linux/arm64,linux/amd64 --tag docker-registry.inesctec.pt/interconnect/knowledge-engine/smart-connector-rest-dist:1.2.3 --push +docker buildx build ./knowledge-directory --platform linux/arm64,linux/amd64 --tag docker-registry.inesctec.pt/interconnect/knowledge-engine/knowledge-directory:1.2.3 --push +docker buildx build ./admin-ui --platform linux/arm64,linux/amd64 --tag docker-registry.inesctec.pt/interconnect/knowledge-engine/admin-ui:1.2.3 --push ``` 6. Build and push the new Docker images to GitHub: ```bash -docker buildx build ./smart-connector-rest-dist --platform linux/arm64,linux/amd64 --tag ghcr.io/tno/knowledge-engine/smart-connector:1.2.1 --push -docker buildx build ./knowledge-directory --platform linux/arm64,linux/amd64 --tag ghcr.io/tno/knowledge-engine/knowledge-directory:1.2.1 --push -docker buildx build ./admin-ui --platform linux/arm64,linux/amd64 --tag ghcr.io/tno/knowledge-engine/admin-ui:1.2.1 --push +docker buildx build ./smart-connector-rest-dist --platform linux/arm64,linux/amd64 --tag ghcr.io/tno/knowledge-engine/smart-connector:1.2.3 --push +docker buildx build ./knowledge-directory --platform linux/arm64,linux/amd64 --tag ghcr.io/tno/knowledge-engine/knowledge-directory:1.2.3 --push +docker buildx build ./admin-ui --platform linux/arm64,linux/amd64 --tag ghcr.io/tno/knowledge-engine/admin-ui:1.2.3 --push ``` 7. Prepare the next SNAPSHOT version and make a commit for that too. @@ -232,7 +232,7 @@ docker buildx build ./admin-ui --platform linux/arm64,linux/amd64 --tag ghcr.io/ 10. Inform mailing list(s) (and [the blog](https://www.knowledge-engine.eu/blog/)) about the new release. ## (advanced) Administering a Knowledge Engine runtime -To start a new instance of the REST API knowledge engine version 1.2.1, make sure you have `git checkout 1.2.1` the tag `1.2.1`. Now make sure you run the `mvn clean install` command successfully from the root of the repository. +To start a new instance of the REST API knowledge engine version 1.2.3, make sure you have `git checkout 1.2.3` the tag `1.2.3`. Now make sure you run the `mvn clean install` command successfully from the root of the repository. ### Starting the Knowledge Engine in local mode When no additional configuration parameters are provided, the Knowledge Engine will by default run in local mode. This means you can create multiple smart connectors that can communicate with each other through the REST API, but the Knowledge Engine will not connect to a knowledge directory and will not be able to connect with smart connectors running in other runtimes. @@ -246,13 +246,13 @@ cd smart-connector-rest-dist/target Finally, start the server (note that you can configure a log file by including the `-Dorg.slf4j.simpleLogger.logFile=ke.log` system property to the JVM): ```bash -java -Dorg.slf4j.simpleLogger.logFile=ke.log -cp "smart-connector-rest-dist-1.2.1.jar:dependency/*" eu.knowledge.engine.rest.Main 8280 +java -Dorg.slf4j.simpleLogger.logFile=ke.log -cp "smart-connector-rest-dist-1.2.3.jar:dependency/*" eu.knowledge.engine.rest.Main 8280 ``` If you want to run in it in the background, you can use the `nohup` linux command (which does not use the simpleLogger configuration system property, but redirects the standard err/out): ```bash -nohup java -cp "smart-connector-rest-dist-1.2.1.jar:dependency/*" eu.knowledge.engine.rest.Main 8280 > ke.log +nohup java -cp "smart-connector-rest-dist-1.2.3.jar:dependency/*" eu.knowledge.engine.rest.Main 8280 > ke.log ``` ### Starting the Knowledge Engine in distributed mode diff --git a/admin-ui/src/main/resources/openapi-admin-ui.yaml b/admin-ui/src/main/resources/openapi-admin-ui.yaml index 93baa872..f5d6a6b2 100644 --- a/admin-ui/src/main/resources/openapi-admin-ui.yaml +++ b/admin-ui/src/main/resources/openapi-admin-ui.yaml @@ -5,7 +5,7 @@ info: description: This API provides information on Knowledge Engine Runtimes (todo), Smart Connectors, Knowledge Bases, and Knowledge Interactions in a Knowledge Engine Network. - version: 1.0.1-SNAPSHOT + version: 1.2.3 paths: /rest/admin/sc/all/{include-meta}: diff --git a/docs/04_distributed_mode.md b/docs/04_distributed_mode.md index 601bbfb2..410b61f0 100644 --- a/docs/04_distributed_mode.md +++ b/docs/04_distributed_mode.md @@ -16,7 +16,7 @@ First of all, you need to start a knowledge directory. The desired port number f ```bash cd knowledge-directory/target/ -java -Dorg.slf4j.simpleLogger.logFile=kd.log -cp "knowledge-directory-1.2.1.jar:dependency/*" eu.knowledge.engine.knowledgedirectory.Main 8080 +java -Dorg.slf4j.simpleLogger.logFile=kd.log -cp "knowledge-directory-1.2.3.jar:dependency/*" eu.knowledge.engine.knowledgedirectory.Main 8080 ``` The `nohup` command can be used to run the process in the background. On overview of the registered Knowledge Engine runtimes can be found on `http://localhost:8080/ker/` (or another host or port if you desire). @@ -26,7 +26,7 @@ Once the knowledge directory is up and running, the REST server can be started. | Key | Descrption | |--------|------------------------------------------------| | KD_URL | URL where the knowledge directory can be found. This url should not end with a slash (`/`).| -| KE_RUNTIME_EXPOSED_URL | URL where other smart connectors (peers) can contact this Knowledge Engine runtime. This allows your Knowledge Engine to be behind a reverse proxy and use TLS. Note that the URL should include the scheme like `http://...` or `https://...` and should not end with a slash (`/`). +| KE_RUNTIME_EXPOSED_URL | URL where other smart connectors (peers) can contact this Knowledge Engine runtime. This allows your Knowledge Engine to be behind a reverse proxy and use TLS. Note that the URL should include the scheme like `http://...` or `https://...` and should not end with a slash (`/`).| | KE_RUNTIME_PORT | Port where where this Knowledge Engine instance will listen for new peer connections | | KE_RUNTIME_HOSTNAME (deprecated) | Hostname where other smart connectors (peers) can contact this Knowledge Engine instance. This variable is superseded by (and conflicts with) KE_RUNTIME_EXPOSED_URL| @@ -39,7 +39,7 @@ export KD_URL=http://localhost:8080 export KE_RUNTIME_EXPOSED_URL=http://localhost:8081 export KE_RUNTIME_PORT=8081 -java -Dorg.slf4j.simpleLogger.logFile=ke.log -cp "smart-connector-rest-dist-1.2.1.jar:dependency/*" eu.knowledge.engine.rest.Main 8280 +java -Dorg.slf4j.simpleLogger.logFile=ke.log -cp "smart-connector-rest-dist-1.2.3.jar:dependency/*" eu.knowledge.engine.rest.Main 8280 ``` ### Using Basic Authentication to secure data exchange diff --git a/examples/authentication/docker-compose.yml b/examples/authentication/docker-compose.yml index fd49c4d7..0212f006 100644 --- a/examples/authentication/docker-compose.yml +++ b/examples/authentication/docker-compose.yml @@ -9,15 +9,15 @@ services: depends_on: - knowledge-directory knowledge-directory: - image: ghcr.io/tno/knowledge-engine/knowledge-directory:1.2.1 + image: ghcr.io/tno/knowledge-engine/knowledge-directory:1.2.3 runtime-1: - image: ghcr.io/tno/knowledge-engine/smart-connector:1.2.1 + image: ghcr.io/tno/knowledge-engine/smart-connector:1.2.3 environment: KE_RUNTIME_PORT: 8081 # The port that the KE uses to listen for inter-KE-runtime communication. KD_URL: http://thisisausername:thisisapassword@nginx/kd KE_RUNTIME_EXPOSED_URL: http://thisisausername:thisisapassword@nginx/ker1 runtime-2: - image: ghcr.io/tno/knowledge-engine/smart-connector:1.2.1 + image: ghcr.io/tno/knowledge-engine/smart-connector:1.2.3 environment: KE_RUNTIME_PORT: 8081 # The port that the KE uses to listen for inter-KE-runtime communication. KD_URL: http://thisisausername:thisisapassword@nginx/kd diff --git a/examples/multiple-runtimes/docker-compose.yml b/examples/multiple-runtimes/docker-compose.yml index dfe7818f..65f64eac 100644 --- a/examples/multiple-runtimes/docker-compose.yml +++ b/examples/multiple-runtimes/docker-compose.yml @@ -2,25 +2,25 @@ services: # This is the knowledge directory, facilitating discovery between different # runtimes. It exposes its service over port 8282. knowledge-directory: - image: ghcr.io/tno/knowledge-engine/knowledge-directory:1.2.1 + image: ghcr.io/tno/knowledge-engine/knowledge-directory:1.2.3 # These services are seperate Knowledge Engine runtime, which can host # multiple smart connectors. Note that the REST API port is a DIFFERENT port # number than the ones configured below. It is still the default 8280. runtime-1: - image: ghcr.io/tno/knowledge-engine/smart-connector:1.2.1 + image: ghcr.io/tno/knowledge-engine/smart-connector:1.2.3 environment: KE_RUNTIME_PORT: 8081 # The port that the KE uses to listen for inter-KE-runtime communication. KE_RUNTIME_EXPOSED_URL: http://runtime-1:8081 # The URL where the runtime is available for inter-runtime communication from the outside. KD_URL: http://knowledge-directory:8282 runtime-2: - image: ghcr.io/tno/knowledge-engine/smart-connector:1.2.1 + image: ghcr.io/tno/knowledge-engine/smart-connector:1.2.3 environment: KE_RUNTIME_PORT: 8081 KE_RUNTIME_EXPOSED_URL: http://runtime-2:8081 KD_URL: http://knowledge-directory:8282 runtime-3: - image: ghcr.io/tno/knowledge-engine/smart-connector:1.2.1 + image: ghcr.io/tno/knowledge-engine/smart-connector:1.2.3 environment: KE_RUNTIME_PORT: 8081 KE_RUNTIME_EXPOSED_URL: http://runtime-3:8081 diff --git a/examples/reasoner/docker-compose.yml b/examples/reasoner/docker-compose.yml index c5a23a29..107bc2e5 100644 --- a/examples/reasoner/docker-compose.yml +++ b/examples/reasoner/docker-compose.yml @@ -1,6 +1,6 @@ services: knowledge-engine: - image: ghcr.io/tno/knowledge-engine/smart-connector:1.2.1 + image: ghcr.io/tno/knowledge-engine/smart-connector:1.2.3 kb1: build: ../common/asking_kb environment: diff --git a/examples/rest-api/docker-compose.yml b/examples/rest-api/docker-compose.yml index 54743264..10731bff 100644 --- a/examples/rest-api/docker-compose.yml +++ b/examples/rest-api/docker-compose.yml @@ -1,6 +1,6 @@ services: knowledge-engine: - image: ghcr.io/tno/knowledge-engine/smart-connector:1.2.1 + image: ghcr.io/tno/knowledge-engine/smart-connector:1.2.3 healthcheck: test: curl -f http://localhost:8280/rest/sc interval: 1s diff --git a/examples/unreachable-runtimes/.gitignore b/examples/unreachable-runtimes/.gitignore new file mode 100644 index 00000000..d6e830be --- /dev/null +++ b/examples/unreachable-runtimes/.gitignore @@ -0,0 +1 @@ +.python-version \ No newline at end of file diff --git a/examples/unreachable-runtimes/Dockerfile b/examples/unreachable-runtimes/Dockerfile new file mode 100644 index 00000000..5114f37f --- /dev/null +++ b/examples/unreachable-runtimes/Dockerfile @@ -0,0 +1,4 @@ +FROM ghcr.io/tno/knowledge-engine/smart-connector:1.2.3 + +RUN apt update -y +RUN apt-get install iptables sudo -y diff --git a/examples/unreachable-runtimes/docker-compose.yml b/examples/unreachable-runtimes/docker-compose.yml new file mode 100644 index 00000000..95573d9a --- /dev/null +++ b/examples/unreachable-runtimes/docker-compose.yml @@ -0,0 +1,98 @@ +services: + # This is the knowledge directory, facilitating discovery between different + # runtimes. It exposes its service over port 8282. + knowledge-directory: + image: ghcr.io/tno/knowledge-engine/knowledge-directory:1.2.3 + + # These services are seperate Knowledge Engine runtime, which can host + # multiple smart connectors. Note that the REST API port is a DIFFERENT port + # number than the ones configured below. It is still the default 8280. + runtime-1: + build: . + #to allow configuring iptables + cap_add: + - NET_ADMIN + environment: + KE_RUNTIME_PORT: 8081 # The port that the KE uses to listen for inter-KE-runtime communication. + KE_RUNTIME_EXPOSED_URL: http://runtime-1:8081 # The URL where the runtime is available for inter-runtime communication from the outside. + KD_URL: http://knowledge-directory:8282 + ENABLE_REASONER: false + JAVA_TOOL_OPTIONS: "-Djdk.httpclient.keepalive.timeout=1" + runtime-2: + build: . + cap_add: + - NET_ADMIN + environment: + KE_RUNTIME_PORT: 8081 + KE_RUNTIME_EXPOSED_URL: http://runtime-2:8081 + KD_URL: http://knowledge-directory:8282 + ENABLE_REASONER: false + JAVA_TOOL_OPTIONS: "-Djdk.httpclient.keepalive.timeout=1" + runtime-3: + build: . + cap_add: + - NET_ADMIN + environment: + KE_RUNTIME_PORT: 8081 + KE_RUNTIME_EXPOSED_URL: http://runtime-3:8081 + KD_URL: http://knowledge-directory:8282 + ENABLE_REASONER: false + JAVA_TOOL_OPTIONS: "-Djdk.httpclient.keepalive.timeout=1" + + + # These Knowledge Bases use the different runtimes, and exchange data with eachother. + kb1: + build: ../common/asking_kb + environment: + KE_URL: http://runtime-1:8280/rest + KB_ID: http://example.org/kb1 + PREFIXES: | + { + "ex": "http://example.org/" + } + GRAPH_PATTERN: | + ?a ex:relatedTo ?b . + kb2: + build: ../common/answering_kb + environment: + KE_URL: http://runtime-2:8280/rest + KB_ID: http://example.org/kb2 + PREFIXES: | + { + "ex": "http://example.org/" + } + GRAPH_PATTERN: | + ?a ex:relatedTo ?b . + KB_DATA: | + [ + { + "a": "", + "b": "" + }, + { + "a": "", + "b": "" + } + ] + kb3: + build: ../common/answering_kb + environment: + KE_URL: http://runtime-3:8280/rest + KB_ID: http://example.org/kb3 + PREFIXES: | + { + "ex": "http://example.org/" + } + GRAPH_PATTERN: | + ?a ex:relatedTo ?b . + KB_DATA: | + [ + { + "a": "", + "b": "" + }, + { + "a": "", + "b": "" + } + ] diff --git a/examples/unreachable-runtimes/readme.md b/examples/unreachable-runtimes/readme.md new file mode 100644 index 00000000..e5d1fc0f --- /dev/null +++ b/examples/unreachable-runtimes/readme.md @@ -0,0 +1,47 @@ +## Knowledge Engine's distributed mode test +This docker compose project is used to test the Knowledge Engine's behavior in distributed mode when something exceptional happens (i.e. divergence from the happy flow). For example, one participant in the Knowledge Network configured its KER incorrectly and therefore it can reach out, but no one can contact the KER from the outside (via the Inter-KER protocol). Under such circumstances, we want the Knowledge Engine to keep functioning and behave as normal as possible. + +To test this, we setup a distributed KER environment with 3 KER+KB combis that exchange data. We have `runtime-1+kb1`, `runtime-2+kb2` and `runtime-3+kb3`. By using the `iptables` tool for `runtime-3` we can simulate a misconfigured KER and test how the other Knowledge Engines behave. Use the following instructions to simulate the misconfigured KER. + +Start the docker compose project: `docker compose up -d` + +Retrieve the internal IP address of the KB3 (because it needs to always be able to contact it `runtime-3` we need its IP to make an exception in `iptables`). This is not really necessary if we use the hostname `kb3` of knowledge base 3 like we do below, but if you use an IP address there you should use the commands below to retrieve this IP. It changes everytime you restart the docker compose project. + +``` +> docker compose exec kb3 sh +> hostname -i +``` + +Make sure runtime-3 is configured to switch between being reachable to being unreachable. First open a shell for runtime-3. + +``` +docker compose exec runtime-3 bash +``` + +Configure `iptables-legacy` to allow the following packets to go through when we block incoming traffic: + +``` +iptables-legacy -A INPUT -i lo -j ACCEPT +iptables-legacy -A INPUT -p tcp -s kb3 -j ACCEPT +iptables-legacy -A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT +``` + +You can quickly test from inside a container whether another container that contains a KER is reachable from there using the following command: `wget -qO- http://runtime-3:8081/runtimedetails` + +For example, if `iptables-legacy` is active and blocking all input traffic, you should no longer be able to receive JSON if you go to `runtime-2` and execute a `wget` to `runtime-3`, but you should be able to do the same from `runtime-3` to `runtime-2`. + +You can quickly test from inside a container whether another container that contains a KER is reachable from there using the following command: `wget -qO- http://runtime-3:8081/runtimedetails`. You should receive some JSON that looks like: + +``` + {"runtimeId":"http://runtime-3:8081","smartConnectorIds":["http://example.org/kb3"]} +``` + +Now, keep an eye on the log file with `docker compose logs -f` and use the following `iptables-legacy` commands to switch between unreachable and reachable. + +``` +iptables-legacy -P INPUT DROP +#runtime-3 is now unreachable for other KERs, but can still reach the Knowledge Directory (KD) and other KERs. + +iptables-legacy -P INPUT ACCEPT +#runtime-3 is now reachable again for other KERs and can also reach the KD and other KERs. +``` \ No newline at end of file diff --git a/pom.xml b/pom.xml index 174bebd5..16e7c9b0 100644 --- a/pom.xml +++ b/pom.xml @@ -74,7 +74,7 @@ 17 17 17 - 1.2.2-SNAPSHOT + 1.2.4-SNAPSHOT diff --git a/reasoner/src/main/java/eu/knowledge/engine/reasoner/BaseRule.java b/reasoner/src/main/java/eu/knowledge/engine/reasoner/BaseRule.java index 7ccf4fe5..0655be69 100644 --- a/reasoner/src/main/java/eu/knowledge/engine/reasoner/BaseRule.java +++ b/reasoner/src/main/java/eu/knowledge/engine/reasoner/BaseRule.java @@ -14,6 +14,8 @@ import java.util.concurrent.CompletableFuture; import org.apache.jena.sparql.core.Var; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import eu.knowledge.engine.reasoner.api.Binding; import eu.knowledge.engine.reasoner.api.BindingSet; @@ -23,6 +25,8 @@ public class BaseRule { + private static final Logger LOG = LoggerFactory.getLogger(BaseRule.class); + public static final String EMPTY = ""; public static final String ARROW = "->"; @@ -178,7 +182,11 @@ public static Set matches(Set aFirstPattern, Set . + ?operation ?output . + ?operation ?parcel . + ?parcel ?crop . + ?crop ?cropType . + ?cropType