Elasticsearch 8

.ci/scripts/run-monkey-tests.sh

@@ -1,32 +1,30 @@
-#!/bin/bash
-
-set -ex
+echo "Testing Kuzzle against node v$NODE_VERSION"
 
-if [ -z "$NODE_VERSION" ];
-then
- echo "Missing NODE_VERSION, use default NODE_20_VERSION"
- export NODE_VERSION=$NODE_20_VERSION
+if [ "$ES_VERSION" == "7" ]; then
+ YML_FILE='./.ci/test-cluster-7.yml'
+elif [ "$ES_VERSION" == "8" ]; then
+ YML_FILE='./.ci/test-cluster-8.yml'
+else
+ echo "Invalid ES_VERSION. It should be either '7' or '8'."
+ exit 1 
 fi
 
-echo "Testing Kuzzle against node v$NODE_VERSION"
-
-docker compose -f ./.ci/test-cluster.yml down -v
+docker compose -f $YML_FILE down -v
 
 echo "Installing dependencies..."
-docker compose -f ./.ci/test-cluster.yml run --rm kuzzle_node_1 npm ci
+docker compose -f $YML_FILE run --rm kuzzle_node_1 npm ci
 
-if [ "$REBUILD" == "true" ];
-then
- docker compose -f ./.ci/test-cluster.yml run --rm kuzzle_node_1 npm rebuild
+if [ "$REBUILD" == "true" ]; then
+ docker compose -f $YML_FILE run --rm kuzzle_node_1 npm rebuild
 fi
 
-docker compose -f ./.ci/test-cluster.yml run --rm kuzzle_node_1 npm run build
+docker compose -f $YML_FILE run --rm kuzzle_node_1 npm run build
 
 echo "[$(date)] - Starting Kuzzle Cluster..."
 
-trap 'docker compose -f ./.ci/test-cluster.yml logs' err
+trap 'docker compose -f $YML_FILE logs' err
 
-docker compose -f ./.ci/test-cluster.yml up -d
+docker compose -f $YML_FILE up -d
 
 # don't wait on 7512: nginx will accept connections far before Kuzzle does
 KUZZLE_PORT=17510 ./bin/wait-kuzzle
@@ -39,4 +37,4 @@ echo "Installing Kuzzle Monkey Tester..."
 
 cd kuzzle-monkey-tests
 npm ci
-node index.js
+node index.js

.ci/scripts/run-test-cluster.sh

@@ -2,7 +2,6 @@
 
 set -ex
 
-
 if [ -z "$NODE_VERSION" ];
 then
  echo "Missing NODE_VERSION, use default NODE_20_VERSION"
@@ -11,23 +10,32 @@ fi
 
 echo "Testing Kuzzle against node v$NODE_VERSION"
 
-docker compose -f ./.ci/test-cluster.yml down -v
+if [ "$ES_VERSION" == "7" ]; then
+ YML_FILE='./.ci/test-cluster-7.yml'
+elif [ "$ES_VERSION" == "8" ]; then
+ YML_FILE='./.ci/test-cluster-8.yml'
+else
+ echo "Invalid ES_VERSION. It should be either '7' or '8'."
+ exit 1 
+fi
+
+docker compose -f $YML_FILE down -v
 
 echo "Installing dependencies..."
-docker compose -f ./.ci/test-cluster.yml run --rm kuzzle_node_1 npm ci
+docker compose -f $YML_FILE run --rm kuzzle_node_1 npm ci
 
 if [ "$REBUILD" == "true" ];
 then
- docker compose -f ./.ci/test-cluster.yml run --rm kuzzle_node_1 npm rebuild
+ docker compose -f $YML_FILE run --rm kuzzle_node_1 npm rebuild
 fi
 
-docker compose -f ./.ci/test-cluster.yml run --rm kuzzle_node_1 npm run build
+docker compose -f $YML_FILE run --rm kuzzle_node_1 npm run build
 
 echo "[$(date)] - Starting Kuzzle Cluster..."
 
-trap 'docker compose -f ./.ci/test-cluster.yml logs' err
+trap 'docker compose -f $YML_FILE logs' err
 
-docker compose -f ./.ci/test-cluster.yml up -d
+docker compose -f $YML_FILE up -d
 
 # don't wait on 7512: nginx will accept connections far before Kuzzle does
 KUZZLE_PORT=17510 ./bin/wait-kuzzle
@@ -36,4 +44,4 @@ KUZZLE_PORT=17512 ./bin/wait-kuzzle
 
 trap - err
 
-npm run $KUZZLE_FUNCTIONAL_TESTS
+npm run $KUZZLE_FUNCTIONAL_TESTS

.ci/services.yml → .ci/services-7.yml

@@ -13,6 +13,7 @@ services:
  sysctls:
  - net.core.somaxconn=8192
  environment:
+ - kuzzle_services__storageEngine__majorVersion=7
  - kuzzle_services__storageEngine__client__node=http://elasticsearch:9200
  - kuzzle_services__internalCache__node__host=redis
  - kuzzle_services__memoryStorage__node__host=redis

.ci/services-8.yml

@@ -0,0 +1,40 @@
+version: '3'
+
+services:
+ kuzzle:
+ build:
+ dockerfile: ./docker/images/${KUZZLE_IMAGE}/Dockerfile
+ context: ..
+ command: ["kuzzle", "start"]
+ cap_add:
+ - SYS_PTRACE
+ ulimits:
+ nofile: 65536
+ sysctls:
+ - net.core.somaxconn=8192
+ environment:
+ - kuzzle_services__storageEngine__majorVersion=8
+ - kuzzle_services__storageEngine__client__node=http://elasticsearch:9200
+ - kuzzle_services__internalCache__node__host=redis
+ - kuzzle_services__memoryStorage__node__host=redis
+ depends_on:
+ - redis
+ - elasticsearch
+ ports:
+ - "7512:7512"
+
+ redis:
+ image: redis:6
+
+ elasticsearch:
+ image: elasticsearch:8.11.3
+ environment:
+ - xpack.security.enabled=false
+ - action.destructive_requires_name=false
+ - cluster.name=kuzzle
+ - node.name=alyx
+ - discovery.type=single-node
+ - ingest.geoip.downloader.enabled=false
+ - indices.id_field_data.enabled=true
+ ports:
+ - '9200:9200'

.ci/test-cluster.yml → .ci/test-cluster-7.yml

@@ -18,6 +18,7 @@ x-kuzzle-config: &kuzzle-config
  - redis
  - elasticsearch
  environment:
+ - kuzzle_services__storageEngine__majorVersion=7
  - kuzzle_services__storageEngine__client__node=http://elasticsearch:9200
  - kuzzle_services__storageEngine__commonMapping__dynamic=true
  - kuzzle_services__internalCache__node__host=redis

.ci/test-cluster-8.yml

@@ -0,0 +1,81 @@
+version: '3.4'
+
+x-kuzzle-config: &kuzzle-config
+ image: kuzzleio/kuzzle-runner:${NODE_VERSION:-20}
+ command: > 
+ bash -c "
+ node ./docker/scripts/start-kuzzle-test.js --enable-plugins kuzzle-plugin-cluster,functional-test-plugin
+ "
+ volumes:
+ - "..:/var/app"
+ cap_add:
+ - SYS_PTRACE
+ ulimits:
+ nofile: 65536
+ sysctls:
+ - net.core.somaxconn=8192
+ depends_on:
+ - redis
+ - elasticsearch
+ environment:
+ - kuzzle_services__storageEngine__majorVersion=8
+ - kuzzle_services__storageEngine__client__node=http://elasticsearch:9200
+ - kuzzle_services__storageEngine__commonMapping__dynamic=true
+ - kuzzle_services__internalCache__node__host=redis
+ - kuzzle_services__memoryStorage__node__host=redis
+ - kuzzle_server__protocols__mqtt__enabled=true
+ - kuzzle_server__protocols__mqtt__developmentMode=false
+ - kuzzle_http__accessControlAllowOrigin=localhost
+ - kuzzle_limits__loginsPerSecond=50
+ - kuzzle_server__protocols__http__additionalContentTypes=*json:["application/x-yaml"]
+ - NODE_ENV=development
+ - NODE_VERSION=${NODE_VERSION:-20}
+ - DEBUG=none
+ - SECRETS_FILE_PREFIX=/var/app/
+ # cluster
+ - kuzzle_plugins__cluster__privileged=true
+ - kuzzle_plugins__cluster__minimumNodes=3
+
+services:
+ nginx:
+ image: nginx:1.19-alpine
+ depends_on:
+ - kuzzle_node_1
+ - kuzzle_node_2
+ - kuzzle_node_3
+ ports:
+ - "7512:7512"
+ volumes:
+ - ../docker/nginx-dev:/etc/nginx/conf.d
+
+ kuzzle_node_1:
+ <<: *kuzzle-config
+ ports:
+ - "17510:7512"
+ - "1883:1883"
+
+ kuzzle_node_2:
+ <<: *kuzzle-config
+ ports:
+ - "17511:7512"
+
+ kuzzle_node_3:
+ <<: *kuzzle-config
+ ports:
+ - "17512:7512"
+
+ redis:
+ image: redis:6
+
+ elasticsearch:
+ image: elasticsearch:8.11.3
+ environment:
+ - xpack.security.enabled=false
+ - action.destructive_requires_name=false
+ - cluster.name=kuzzle
+ - node.name=alyx
+ - discovery.type=single-node
+ - ingest.geoip.downloader.enabled=false
+ - indices.id_field_data.enabled=true
+ ports:
+ - '9200:9200'

.eslintignore

@@ -58,7 +58,8 @@ lib/types/Token.js
 lib/types/User.js
 lib/types/Global.d.ts
 lib/types/Global.js
-lib/types/storage/Elasticsearch.js
+lib/types/storage/7/Elasticsearch.js
+lib/types/storage/8/Elasticsearch.js
 lib/util/interfaces.js
 lib/util/mutex.js
 lib/util/Inflector.js
@@ -126,7 +127,10 @@ lib/util/dump-collection.js
 lib/api/controllers/debugController.js
 lib/api/openapi/components/security/index.js
 lib/config/documentEventAliases.js
-lib/service/storage/elasticsearch.js
+lib/service/storage/Elasticsearch.js
+lib/service/storage/7/elasticsearch.js
+lib/service/storage/8/elasticsearch.js
+lib/service/storage/commons/queryTranslator.js
 lib/types/DebugModule.js
 lib/util/time.js
 lib/util/async.js

.github/actions/build-and-run-kuzzle/action.yml

@@ -2,9 +2,12 @@ name: Build and Run Kuzzle image
 description: Build and Run Kuzzle image
 
 inputs:
- KUZZLE_IMAGE:
+ kuzzle-image:
  description: Kuzzle image target
  required: true
+ es-version:
+ description: Elasticsearch version to use
+ required: true
 
 runs:
  using: "composite"
@@ -19,18 +22,54 @@ runs:
  shell: bash
 
  - run: |
- export KUZZLE_IMAGE=${{ inputs.KUZZLE_IMAGE }}
- docker compose -f ./.ci/services.yml up -d
+ export KUZZLE_IMAGE=${{ inputs.kuzzle-image }}
+ export ES_VERSION=${{ inputs.es-version }}
+
+ if [ "$ES_VERSION" == "7" ]; then
+ YML_FILE='./.ci/services-7.yml'
+ elif [ "$ES_VERSION" == "8" ]; then
+ YML_FILE='./.ci/services-8.yml'
+ else
+ echo "Invalid ES_VERSION. It should be either '7' or '8'."
+ exit 1 
+ fi
+
+ docker compose -f $YML_FILE up -d
  shell: bash
 
  - run: |
- trap 'docker compose -f ./.ci/services.yml logs' err
+ export KUZZLE_IMAGE=${{ inputs.kuzzle-image }}
+ export ES_VERSION=${{ inputs.es-version }}
+
+ if [ "$ES_VERSION" == "7" ]; then
+ YML_FILE='./.ci/services-7.yml'
+ elif [ "$ES_VERSION" == "8" ]; then
+ YML_FILE='./.ci/services-8.yml'
+ else
+ echo "Invalid ES_VERSION. It should be either '7' or '8'."
+ exit 1 
+ fi
+
+ trap 'docker compose -f $YML_FILE logs' err
 
  export MAX_TRIES=60
  ./bin/wait-kuzzle
 
  trap - err
  shell: bash
 
- - run: docker compose -f ./.ci/services.yml down
+ - run: |
+ export KUZZLE_IMAGE=${{ inputs.kuzzle-image }}
+ export ES_VERSION=${{ inputs.es-version }}
+
+ if [ "$ES_VERSION" == "7" ]; then
+ YML_FILE='./.ci/services-7.yml'
+ elif [ "$ES_VERSION" == "8" ]; then
+ YML_FILE='./.ci/services-8.yml'
+ else
+ echo "Invalid ES_VERSION. It should be either '7' or '8'."
+ exit 1 
+ fi
+
+ docker compose -f $YML_FILE down -v
  shell: bash

.github/actions/functional-tests/action.yml

@@ -10,6 +10,9 @@ inputs:
  node-version:
  description: Node version to use in the Kuzzle Docker image
  required: true
+ es-version:
+ description: Elasticsearch version to use
+ required: true
 
 runs:
  using: "composite"
@@ -18,4 +21,5 @@ runs:
  env:
  KUZZLE_FUNCTIONAL_TESTS: ${{ inputs.test-set }}
  NODE_VERSION: ${{ inputs.node-version }}
+ ES_VERSION: ${{ inputs.es-version }}
  shell: bash

.github/actions/monkey-tests/action.yml

@@ -11,4 +11,5 @@ runs:
  - run: ./.ci/scripts/run-monkey-tests.sh
  env:
  NODE_VERSION: ${{ inputs.node-version }}
+ ES_VERSION: ${{ inputs.es-version }}
  shell: bash

.github/workflows/codeql.yml

@@ -24,7 +24,7 @@ jobs:
 
  steps:
  - name: Checkout
- uses: actions/checkout@v3
+ uses: actions/checkout@v4
 
  - name: Initialize CodeQL
  uses: github/codeql-action/init@v2

.github/workflows/core-dev.yml

@@ -13,7 +13,7 @@ jobs:
  name: core-dev image
  runs-on: ubuntu-22.04
  steps:
- - uses: actions/checkout@v3
+ - uses: actions/checkout@v4
 
  - name: Set up QEMU
  uses: docker/setup-qemu-action@v3

.github/workflows/elasticsearch.yml

@@ -14,7 +14,7 @@ jobs:
  name: Elasticsearch image
  runs-on: ubuntu-22.04
  steps:
- - uses: actions/checkout@v3
+ - uses: actions/checkout@v4
 
  - name: Set up QEMU
  uses: docker/setup-qemu-action@v3