diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 4efd11f18..eceb7b9e6 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -56,7 +56,7 @@ body: attributes: label: Docker Selenium version (image tag) description: What version of Docker Selenium are you using? - placeholder: 4.17.0-20240123? Please use the full tag, avoid "latest" + placeholder: 4.18.0-20240220? Please use the full tag, avoid "latest" validations: required: true - type: input diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index d0b68d0f5..1c7727499 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -1,6 +1,9 @@ name: Build & test +concurrency: + group: ${{ github.workflow }} on: + workflow_dispatch: push: paths-ignore: - '**.md' @@ -19,11 +22,11 @@ jobs: matrix: use-random-user: [false, true] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@main - name: Output Docker info run: docker info - name: Set up Python - uses: actions/setup-python@v5.0.0 + uses: actions/setup-python@main with: python-version: '3.11' check-latest: true @@ -44,7 +47,7 @@ jobs: - name: Build Docker images run: VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make build - name: Test Docker images - uses: nick-invision/retry@v2.9.0 + uses: nick-invision/retry@master with: timeout_minutes: 20 max_attempts: 3 diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index a50ea0a30..22bb63151 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -2,9 +2,9 @@ name: Deploys on: workflow_dispatch: -# push: -# branches: -# - trunk + push: + branches: + - trunk jobs: deploy: @@ -15,7 +15,7 @@ jobs: permissions: write-all steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@main with: persist-credentials: false fetch-depth: 0 @@ -49,6 +49,8 @@ jobs: run: echo ${LATEST_TAG} - name: Update tag in docs and files run: ./update_tag_in_docs_and_files.sh ${LATEST_TAG} ${NEXT_TAG} + - name: Update chart CHANGELOG + run: ./generate_chart_changelog.sh HEAD - name: Build images run: VERSION="${GRID_VERSION}" BUILD_DATE=${BUILD_DATE} make build - name: Login Docker Hub @@ -57,7 +59,7 @@ jobs: DOCKER_USERNAME: ${{secrets.DOCKER_USERNAME}} DOCKER_PASSWORD: ${{secrets.DOCKER_PASSWORD}} - name: Deploy new images - uses: nick-invision/retry@14672906e672a08bd6eeb15720e9ed3ce869cdd4 # v2 + uses: nick-invision/retry@master with: timeout_minutes: 20 max_attempts: 3 diff --git a/.github/workflows/helm-chart-release.yml b/.github/workflows/helm-chart-release.yml index 1f2b1f7c4..0139deaf1 100644 --- a/.github/workflows/helm-chart-release.yml +++ b/.github/workflows/helm-chart-release.yml @@ -1,4 +1,6 @@ name: Release Charts +concurrency: + group: ${{ github.workflow }} on: push: @@ -14,7 +16,7 @@ jobs: permissions: write-all steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@main with: fetch-depth: 0 @@ -23,10 +25,14 @@ jobs: git config user.name "$GITHUB_ACTOR" git config user.email "$GITHUB_ACTOR@users.noreply.github.com" + - name: Get chart release notes (chart_release_notes.md) + run: ./generate_chart_changelog.sh HEAD + - name: Run chart-releaser - uses: helm/chart-releaser-action@v1.6.0 + uses: helm/chart-releaser-action@main with: mark_as_latest: false skip_existing: true env: CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + CR_RELEASE_NOTES_FILE: RELEASE_NOTES.md diff --git a/.github/workflows/helm-chart-test.yml b/.github/workflows/helm-chart-test.yml index 8ecb77509..7daf81013 100644 --- a/.github/workflows/helm-chart-test.yml +++ b/.github/workflows/helm-chart-test.yml @@ -1,4 +1,6 @@ -name: Lint and Test Helm Charts +name: Test Helm Charts +concurrency: + group: ${{ github.workflow }} on: push: @@ -8,32 +10,53 @@ on: paths-ignore: - '**.md' workflow_dispatch: + inputs: + request-timeout: + description: 'Test parameter for different request timeout' + required: false + default: '370' + schedule: + - cron: '0 0 * * *' permissions: contents: read jobs: build-and-test: - name: Test Helm charts + name: Test K8s runs-on: ubuntu-latest strategy: fail-fast: false matrix: - test-strategy: [chart_test, chart_test_parallel_autoscaling, chart_test_https, chart_test_parallel_autoscaling_https] + include: + - k8s-version: 'v1.25.16' + test-strategy: job + cluster: 'kind' + - k8s-version: 'v1.26.14' + test-strategy: deployment + cluster: 'kind' + - k8s-version: 'v1.27.11' + test-strategy: job_https + cluster: 'kind' + - k8s-version: 'v1.28.7' + test-strategy: job_hostname + cluster: 'minikube' + - k8s-version: 'v1.29.2' + test-strategy: deployment_https + cluster: 'minikube' + env: + CLUSTER: ${{ matrix.cluster }} + KUBERNETES_VERSION: ${{ matrix.k8s-version }} + ARTIFACT_NAME: ${{ matrix.k8s-version }}-${{ matrix.test-strategy }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@main - name: Output Docker info run: docker info - name: Set up Python - uses: actions/setup-python@v5.0.0 + uses: actions/setup-python@main with: python-version: '3.11' check-latest: true - - name: Install CA certificates - run: | - sudo apt install openssl -y - sudo apt install ca-certificates -y - sudo update-ca-certificates --fresh - name: Get branch name (only for push to branch) if: github.event_name == 'push' run: echo "BRANCH=$(echo ${PUSH_BRANCH##*/})" >> $GITHUB_ENV @@ -51,36 +74,54 @@ jobs: echo "BUILD_DATE=$(date '+%Y%m%d')" >> $GITHUB_ENV echo "IMAGE_REGISTRY=artifactory/selenium" >> $GITHUB_ENV - name: Setup Kubernetes environment - run: make chart_setup_env - - name: Build Docker images - run: BUILD_ARGS="--build-arg TARGETARCH=amd64" NAME=${IMAGE_REGISTRY} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make build - - name: Build and lint charts + uses: nick-invision/retry@master + with: + timeout_minutes: 10 + max_attempts: 3 + command: CLUSTER=${CLUSTER} make chart_setup_env + - name: Build Helm charts run: | BUILD_DATE=${BUILD_DATE} make chart_build echo "CHART_PACKAGE_PATH=$(cat /tmp/selenium_chart_version)" >> $GITHUB_ENV echo "CHART_FILE_NAME=$(basename $(cat /tmp/selenium_chart_version))" >> $GITHUB_ENV + - name: Build Docker images + run: NAME=${IMAGE_REGISTRY} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make build - name: Setup Kubernetes cluster - run: NAME=${IMAGE_REGISTRY} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make chart_cluster_setup - - name: Test Selenium Grid on Kubernetes - uses: nick-invision/retry@v2.9.0 + uses: nick-invision/retry@master + with: + timeout_minutes: 10 + max_attempts: 3 + command: CLUSTER=${CLUSTER} KUBERNETES_VERSION=${KUBERNETES_VERSION} NAME=${IMAGE_REGISTRY} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make chart_cluster_setup + - name: Test chart template + run: NAME=${IMAGE_REGISTRY} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make chart_test_template + - name: Test set custom CA certificate + run: NAME=${IMAGE_REGISTRY} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make test_custom_ca_cert + - name: Set test parameters + if: (matrix.test-strategy == 'job' || matrix.test-strategy == 'deployment') && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') + run: | + echo "AUTOSCALING_POLL_INTERVAL=${AUTOSCALING_POLL_INTERVAL}" >> $GITHUB_ENV + env: + AUTOSCALING_POLL_INTERVAL: ${{ github.event.inputs.request-timeout || '370' }} + - name: Test Selenium Grid on Kubernetes ${{ matrix.k8s-version }} with Autoscaling ${{ matrix.test-strategy }} + uses: nick-invision/retry@master with: timeout_minutes: 30 max_attempts: 3 command: | - NAME=${IMAGE_REGISTRY} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make ${{ matrix.test-strategy }} + NAME=${IMAGE_REGISTRY} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make chart_test_autoscaling_${{ matrix.test-strategy }} - name: Cleanup Kubernetes cluster if: always() - run: make chart_cluster_cleanup + run: CLUSTER=${CLUSTER} make chart_cluster_cleanup - name: Upload Helm chart package if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@main with: - name: ${{ matrix.test-strategy }}_${{ env.CHART_FILE_NAME }} + name: ${{ env.ARTIFACT_NAME }}_${{ env.CHART_FILE_NAME }} path: ${{ env.CHART_PACKAGE_PATH }} - name: Upload chart test artifacts if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@main with: - name: ${{ matrix.test-strategy }}-artifacts + name: ${{ env.ARTIFACT_NAME }}-artifacts path: ./tests/tests/ if-no-files-found: ignore diff --git a/.github/workflows/label-commenter.yml b/.github/workflows/label-commenter.yml index bf2e02b25..9df715b63 100644 --- a/.github/workflows/label-commenter.yml +++ b/.github/workflows/label-commenter.yml @@ -13,6 +13,6 @@ jobs: comment: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@main - name: Label Commenter uses: peaceiris/actions-label-commenter@v1 diff --git a/.github/workflows/nightly.yaml b/.github/workflows/nightly.yaml index 9c2319bba..fb5fdd1f9 100644 --- a/.github/workflows/nightly.yaml +++ b/.github/workflows/nightly.yaml @@ -11,7 +11,7 @@ jobs: permissions: write-all steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@main with: persist-credentials: false fetch-depth: 0 @@ -74,7 +74,7 @@ jobs: - name: Create release notes (release_notes.md) run: ./generate_release_notes.sh ${LATEST_TAG} origin/trunk ${GRID_VERSION} ${BUILD_DATE} - name: Set up Python - uses: actions/setup-python@v5.0.0 + uses: actions/setup-python@main with: python-version: '3.11' check-latest: true diff --git a/.github/workflows/scan-dockerfile.yml b/.github/workflows/scan-dockerfile.yml index 4e34e767c..0d65fb6dc 100644 --- a/.github/workflows/scan-dockerfile.yml +++ b/.github/workflows/scan-dockerfile.yml @@ -1,4 +1,6 @@ name: Scan Dockerfile vulnerabilities +concurrency: + group: ${{ github.workflow }} on: push: @@ -17,7 +19,7 @@ jobs: permissions: write-all runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@main - name: Set severity for PRs if: github.event_name == 'pull_request' || github.event_name == 'push' run: | @@ -36,7 +38,7 @@ jobs: format: 'sarif' output: 'source-results.sarif' scanners: 'vuln,secret,misconfig' - skip-dirs: 'tests,Video,NodeChromium,NodeFirefox' + skip-dirs: 'tests,Video' exit-code: '${{ env.EXIT_CODE }}' severity: '${{ env.SEVERITY }}' limit-severities-for-sarif: true diff --git a/.github/workflows/test-video.yml b/.github/workflows/test-video.yml index 04bf156d0..2ea1b5c7d 100644 --- a/.github/workflows/test-video.yml +++ b/.github/workflows/test-video.yml @@ -1,6 +1,9 @@ name: Test video files +concurrency: + group: ${{ github.workflow }} on: + workflow_dispatch: push: paths-ignore: - '**.md' @@ -16,11 +19,11 @@ jobs: name: Test video recorded through Docker Selenium runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@main - name: Output Docker info run: docker info - name: Set up Python - uses: actions/setup-python@v5.0.0 + uses: actions/setup-python@main with: python-version: '3.11' check-latest: true @@ -41,17 +44,17 @@ jobs: - name: Run Docker Compose to record video run: USE_RANDOM_USER_ID=${USE_RANDOM_USER} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make test_video - name: Upload recorded Chrome video - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@main with: name: chrome_video path: ./tests/videos/chrome_video.mp4 - name: Upload recorded Edge video - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@main with: name: edge_video path: ./tests/videos/edge_video.mp4 - name: Upload recorded Firefox video - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@main with: name: firefox_video path: ./tests/videos/firefox_video.mp4 diff --git a/.github/workflows/update-chart-changelog.yaml b/.github/workflows/update-chart-changelog.yaml deleted file mode 100644 index 3ac45a214..000000000 --- a/.github/workflows/update-chart-changelog.yaml +++ /dev/null @@ -1,33 +0,0 @@ -name: Update Chart CHANGELOG - -on: - release: - types: [published] - workflow_dispatch: - -jobs: - release: - runs-on: ubuntu-latest - permissions: write-all - if: ${{ contains(github.event.release.tag_name, 'selenium-grid') || github.event_name == 'workflow_dispatch' }} - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - persist-credentials: false - fetch-depth: 0 - - - name: Update chart CHANGELOG - run: ./generate_chart_changelog.sh - - - name: Commit files - run: | - git config --local user.email "selenium-ci@users.noreply.github.com" - git config --local user.name "Selenium CI Bot" - git commit -m "Update chart CHANGELOG [skip ci]" -a - - - name: Push changes - uses: ad-m/github-push-action@master - with: - github_token: ${{ secrets.SELENIUM_CI_TOKEN }} - branch: trunk diff --git a/.github/workflows/update-dev-beta-browser-images.yml b/.github/workflows/update-dev-beta-browser-images.yml index 1a0036574..b640b51a2 100644 --- a/.github/workflows/update-dev-beta-browser-images.yml +++ b/.github/workflows/update-dev-beta-browser-images.yml @@ -22,7 +22,7 @@ jobs: CHANNEL: ${{ matrix.channel }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@main - name: Setup environment variables run: | export SELENIUM_VERSION=$(grep BASE_VERSION Makefile | sed 's/.*,\([^)]*\))/\1/p' | head -n 1) diff --git a/.gitignore b/.gitignore index f77615fd4..b0fa8b902 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ tmp/ *_image/ node_modules/ .idea/ +*.iml # Sed backup files. *-e # Vim swap files. @@ -150,3 +151,4 @@ ENV/ /charts/*/charts /charts/*/**.lock /charts/*.tgz +/charts/*/RELEASE_NOTES.md diff --git a/Base/Dockerfile b/Base/Dockerfile index 4ffc33921..7b94cc107 100644 --- a/Base/Dockerfile +++ b/Base/Dockerfile @@ -1,10 +1,4 @@ -# FROM ubuntu:focal-20210416 -#FROM ubuntu:focal-20210827 -#FROM ubuntu:focal-20211006 -#FROM ubuntu:focal-20220302 -#FROM ubuntu:focal-20220415 -#FROM ubuntu:focal-20220531 -#FROM ubuntu:focal-20230301 +#FROM ubuntu:jammy-20240125 FROM debian:bookworm LABEL authors="Selenium " @@ -12,8 +6,8 @@ LABEL authors="Selenium " ARG VERSION ARG RELEASE=selenium-${VERSION} # Default value should be aligned with upstream Selenium (https://github.com/SeleniumHQ/selenium/blob/trunk/java/maven_deps.bzl) -ARG OPENTELEMETRY_VERSION=1.33.0 -ARG GRPC_VERSION=1.60.1 +ARG OPENTELEMETRY_VERSION=1.34.1 +ARG GRPC_VERSION=1.61.0 #Arguments to define the user running Selenium ARG SEL_USER=seluser @@ -118,7 +112,8 @@ COPY supervisord.conf /etc #========== # Selenium & relaxing permissions for OpenShift and other non-sudo environments #========== -RUN mkdir -p /opt/selenium /opt/selenium/assets /var/run/supervisor /var/log/supervisor ${SEL_DOWNLOAD_DIR} ${HOME}/.mozilla ${HOME}/.vnc \ +RUN mkdir -p /opt/selenium /opt/selenium/assets /var/run/supervisor /var/log/supervisor ${SEL_DOWNLOAD_DIR} \ + ${HOME}/.mozilla ${HOME}/.vnc $HOME/.pki/nssdb \ && touch /opt/selenium/config.toml \ && chown -R ${SEL_USER}:${SEL_GROUP} /opt/selenium /var/run/supervisor /var/log/supervisor /etc/passwd ${HOME} \ && chmod -R 775 /opt/selenium /var/run/supervisor /var/log/supervisor /etc/passwd ${HOME} \ @@ -145,8 +140,7 @@ RUN if [ `arch` = "aarch64" ] || [ `arch` = "x86_64" ]; then \ RUN if [ -f "/tmp/cs" ]; then \ /tmp/cs fetch --classpath --cache /external_jars \ io.opentelemetry:opentelemetry-exporter-otlp:${OPENTELEMETRY_VERSION} \ - io.opentelemetry:opentelemetry-exporter-jaeger:${OPENTELEMETRY_VERSION} \ - io.grpc:grpc-netty:${GRPC_VERSION} > /external_jars/.classpath.txt ; \ + io.grpc:grpc-netty:${GRPC_VERSION} > /external_jars/.classpath.txt; \ fi RUN if [ -f "/tmp/cs" ]; then chmod 664 /external_jars/.classpath.txt ; fi @@ -156,10 +150,19 @@ RUN rm -fr /root/.cache/* # Run the following commands as non-privileged user #=================================================== USER ${SEL_UID}:${SEL_GID} -VOLUME ${SEL_DOWNLOAD_DIR} + +#========== +# NSSDB initialization with an empty password +#========== +RUN certutil -d sql:$HOME/.pki/nssdb -N --empty-password # Boolean value, maps "--bind-host" ENV SE_BIND_HOST false +# Boolean value, maps "--reject-unsupported-caps" +ENV SE_REJECT_UNSUPPORTED_CAPS false + +ENV SE_OTEL_JAVA_GLOBAL_AUTOCONFIGURE_ENABLED true +ENV SE_OTEL_TRACES_EXPORTER "otlp" # A too high maximum number of file descriptors (with the default value # inherited from the docker host) can cause issues with some of our tools: diff --git a/Distributor/Dockerfile b/Distributor/Dockerfile index ce901ceeb..e4045804a 100644 --- a/Distributor/Dockerfile +++ b/Distributor/Dockerfile @@ -16,3 +16,12 @@ COPY --chown="${SEL_UID}:${SEL_GID}" start-selenium-grid-distributor.sh \ /opt/bin/ COPY selenium-grid-distributor.conf /etc/supervisor/conf.d/ + +# In seconds, maps to "--session-request-timeout" +ENV SE_SESSION_REQUEST_TIMEOUT 300 +# In seconds, maps to "--session-retry-interval" +ENV SE_SESSION_RETRY_INTERVAL 15 +# In seconds, maps to "--healthcheck-interval" +ENV SE_HEALTHCHECK_INTERVAL 120 + +ENV SE_OTEL_SERVICE_NAME "selenium-distributor" diff --git a/Distributor/start-selenium-grid-distributor.sh b/Distributor/start-selenium-grid-distributor.sh index c9af77f71..75c4b0cac 100755 --- a/Distributor/start-selenium-grid-distributor.sh +++ b/Distributor/start-selenium-grid-distributor.sh @@ -88,14 +88,35 @@ if [ ! -z "$SE_REGISTRATION_SECRET" ]; then SE_OPTS="$SE_OPTS --registration-secret ${SE_REGISTRATION_SECRET}" fi +if [ ! -z "$SE_REJECT_UNSUPPORTED_CAPS" ]; then + echo "Appending Selenium options: --reject-unsupported-caps ${SE_REJECT_UNSUPPORTED_CAPS}" + SE_OPTS="$SE_OPTS --reject-unsupported-caps ${SE_REJECT_UNSUPPORTED_CAPS}" +fi + EXTRA_LIBS="" -if [ ! -z "$SE_ENABLE_TRACING" ]; then +if [ "$SE_ENABLE_TRACING" = "true" ]; then EXTERNAL_JARS=$( .env ; \ echo TAG=$(TAG_VERSION) >> .env ; \ echo NODE=$$node >> .env ; \ + echo UID=$$(id -u) >> .env ; \ if [ $$node = "NodeChrome" ] ; then \ echo BROWSER=chrome >> .env ; \ echo VIDEO_FILE_NAME=chrome_video.mp4 >> .env ; \ @@ -567,15 +554,18 @@ test_video: video hub chrome firefox edge done # Using ffmpeg to verify file integrity # https://superuser.com/questions/100288/how-can-i-check-the-integrity-of-a-video-file-avi-mpeg-mp4 - docker run -v $$(pwd):$$(pwd) -w $$(pwd) $(FFMPEG_BASED_NAME)/ffmpeg:$(FFMPEG_BASED_TAG) -v error -i ./tests/videos/chrome_video.mp4 -f null - 2>error.log - docker run -v $$(pwd):$$(pwd) -w $$(pwd) $(FFMPEG_BASED_NAME)/ffmpeg:$(FFMPEG_BASED_TAG) -v error -i ./tests/videos/firefox_video.mp4 -f null - 2>error.log - docker run -v $$(pwd):$$(pwd) -w $$(pwd) $(FFMPEG_BASED_NAME)/ffmpeg:$(FFMPEG_BASED_TAG) -v error -i ./tests/videos/edge_video.mp4 -f null - 2>error.log + docker run -u $$(id -u) -v $$(pwd):$$(pwd) -w $$(pwd) $(FFMPEG_BASED_NAME)/ffmpeg:$(FFMPEG_BASED_TAG) -v error -i ./tests/videos/chrome_video.mp4 -f null - 2>error.log + docker run -u $$(id -u) -v $$(pwd):$$(pwd) -w $$(pwd) $(FFMPEG_BASED_NAME)/ffmpeg:$(FFMPEG_BASED_TAG) -v error -i ./tests/videos/firefox_video.mp4 -f null - 2>error.log + docker run -u $$(id -u) -v $$(pwd):$$(pwd) -w $$(pwd) $(FFMPEG_BASED_NAME)/ffmpeg:$(FFMPEG_BASED_TAG) -v error -i ./tests/videos/edge_video.mp4 -f null - 2>error.log + +test_custom_ca_cert: + VERSION=$(TAG_VERSION) NAMESPACE=$(NAMESPACE) ./tests/customCACert/bootstrap.sh chart_setup_env: ./tests/charts/make/chart_setup_env.sh chart_cluster_setup: - VERSION=$(TAG_VERSION) NAMESPACE=$(NAMESPACE) ./tests/charts/make/chart_cluster_setup.sh + VERSION=$(TAG_VERSION) NAMESPACE=$(NAMESPACE) BUILD_DATE=$(BUILD_DATE) ./tests/charts/make/chart_cluster_setup.sh chart_cluster_cleanup: ./tests/charts/make/chart_cluster_cleanup.sh @@ -586,34 +576,44 @@ chart_build_nightly: chart_build: VERSION=$(TAG_VERSION) ./tests/charts/make/chart_build.sh -chart_test_https: - SELENIUM_GRID_PROTOCOL=https SELENIUM_GRID_PORT=443 make chart_test - -chart_test: chart_test_template \ - chart_test_chrome \ - chart_test_firefox \ - chart_test_edge - chart_test_template: ./tests/charts/bootstrap.sh chart_test_chrome: - VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) UPLOADER_TAG=$(RCLONE_TAG_VERSION)-$(BUILD_DATE) NAMESPACE=$(NAMESPACE) \ + VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) NAMESPACE=$(NAMESPACE) \ ./tests/charts/make/chart_test.sh NodeChrome chart_test_firefox: - VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) UPLOADER_TAG=$(RCLONE_TAG_VERSION)-$(BUILD_DATE) NAMESPACE=$(NAMESPACE) \ + VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) NAMESPACE=$(NAMESPACE) \ ./tests/charts/make/chart_test.sh NodeFirefox chart_test_edge: - VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) UPLOADER_TAG=$(RCLONE_TAG_VERSION)-$(BUILD_DATE) NAMESPACE=$(NAMESPACE) \ + VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) NAMESPACE=$(NAMESPACE) \ ./tests/charts/make/chart_test.sh NodeEdge -chart_test_parallel_autoscaling_https: - SELENIUM_GRID_PROTOCOL=https SELENIUM_GRID_PORT=443 make chart_test_parallel_autoscaling +chart_test_autoscaling_deployment_https: + CHART_FULL_DISTRIBUTED_MODE=true CHART_ENABLE_INGRESS_HOSTNAME=true CHART_ENABLE_BASIC_AUTH=true SELENIUM_GRID_PROTOCOL=https SELENIUM_GRID_PORT=443 \ + VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) NAMESPACE=$(NAMESPACE) \ + ./tests/charts/make/chart_test.sh DeploymentAutoscaling + +chart_test_autoscaling_deployment: + CHART_ENABLE_TRACING=true SELENIUM_GRID_TEST_HEADLESS=true SELENIUM_GRID_HOST=$$(hostname -i) \ + VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) NAMESPACE=$(NAMESPACE) \ + ./tests/charts/make/chart_test.sh DeploymentAutoscaling + +chart_test_autoscaling_job_https: + SELENIUM_GRID_TEST_HEADLESS=true SELENIUM_GRID_PROTOCOL=https CHART_ENABLE_BASIC_AUTH=true SELENIUM_GRID_PORT=443 \ + VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) NAMESPACE=$(NAMESPACE) \ + ./tests/charts/make/chart_test.sh JobAutoscaling + +chart_test_autoscaling_job_hostname: + CHART_ENABLE_TRACING=true CHART_ENABLE_INGRESS_HOSTNAME=true CHART_ENABLE_BASIC_AUTH=true \ + VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) NAMESPACE=$(NAMESPACE) \ + ./tests/charts/make/chart_test.sh JobAutoscaling -chart_test_parallel_autoscaling: - VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) UPLOADER_TAG=$(RCLONE_TAG_VERSION)-$(BUILD_DATE) NAMESPACE=$(NAMESPACE) \ +chart_test_autoscaling_job: + CHART_ENABLE_TRACING=true CHART_FULL_DISTRIBUTED_MODE=true CHART_ENABLE_INGRESS_HOSTNAME=true \ + VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) NAMESPACE=$(NAMESPACE) \ ./tests/charts/make/chart_test.sh JobAutoscaling .PHONY: \ diff --git a/NodeBase/Dockerfile b/NodeBase/Dockerfile index f305fc612..8ac1f4598 100644 --- a/NodeBase/Dockerfile +++ b/NodeBase/Dockerfile @@ -45,7 +45,7 @@ RUN apt-get -qqy update \ #===== RUN apt-get update -qqy \ && apt-get -qqy --no-install-recommends install \ - x11vnc x11-utils fluxbox \ + x11vnc x11-utils \ && rm -rf /var/lib/apt/lists/* /var/cache/apt/* #========= @@ -54,7 +54,7 @@ RUN apt-get update -qqy \ #========= RUN apt-get update -qqy \ && apt-get -qqy --no-install-recommends install \ - fluxbox \ + fluxbox eterm hsetroot feh \ && rm -rf /var/lib/apt/lists/* /var/cache/apt/* #================ @@ -88,8 +88,7 @@ RUN apt-get -qqy update \ fonts-ipafont-gothic \ fonts-wqy-zenhei \ fonts-tlwg-loma-otf \ - # ttf-ubuntu-font-family \ - # fonts-ubuntu \ + #fonts-ubuntu \ fonts-noto-color-emoji \ && rm -rf /var/lib/apt/lists/* \ && apt-get -qyy clean @@ -196,9 +195,12 @@ ENV SE_NODE_MAX_SESSIONS 1 ENV SE_NODE_SESSION_TIMEOUT 300 # As boolean, maps to "override-max-sessions" ENV SE_NODE_OVERRIDE_MAX_SESSIONS false +# As integer, maps to "--heartbeat-period" in seconds +ENV SE_NODE_HEARTBEAT_PERIOD 30 # Following line fixes https://github.com/SeleniumHQ/docker-selenium/issues/87 ENV DBUS_SESSION_BUS_ADDRESS=/dev/null +ENV SE_OTEL_SERVICE_NAME "selenium-node" # Copying configuration script generator COPY --chown="${SEL_UID}:${SEL_GID}" generate_config /opt/bin/generate_config diff --git a/NodeBase/start-selenium-node.sh b/NodeBase/start-selenium-node.sh index 65402cbec..cf622f752 100755 --- a/NodeBase/start-selenium-node.sh +++ b/NodeBase/start-selenium-node.sh @@ -36,6 +36,31 @@ if [ ! -z "$SE_NODE_SESSION_TIMEOUT" ]; then SE_OPTS="$SE_OPTS --session-timeout ${SE_NODE_SESSION_TIMEOUT}" fi +if [ ! -z "$SE_NODE_ENABLE_MANAGED_DOWNLOADS" ]; then + echo "Appending Selenium options: --enable-managed-downloads ${SE_NODE_ENABLE_MANAGED_DOWNLOADS}" + SE_OPTS="$SE_OPTS --enable-managed-downloads ${SE_NODE_ENABLE_MANAGED_DOWNLOADS}" +fi + +if [ ! -z "$SE_NODE_ENABLE_CDP" ]; then + echo "Appending Selenium options: --enable-cdp ${SE_NODE_ENABLE_CDP}" + SE_OPTS="$SE_OPTS --enable-cdp ${SE_NODE_ENABLE_CDP}" +fi + +if [ ! -z "$SE_NODE_REGISTER_PERIOD" ]; then + echo "Appending Selenium options: --register-period ${SE_NODE_REGISTER_PERIOD}" + SE_OPTS="$SE_OPTS --register-period ${SE_NODE_REGISTER_PERIOD}" +fi + +if [ ! -z "$SE_NODE_REGISTER_CYCLE" ]; then + echo "Appending Selenium options: --register-cycle ${SE_NODE_REGISTER_CYCLE}" + SE_OPTS="$SE_OPTS --register-cycle ${SE_NODE_REGISTER_CYCLE}" +fi + +if [ ! -z "$SE_NODE_HEARTBEAT_PERIOD" ]; then + echo "Appending Selenium options: --heartbeat-period ${SE_NODE_HEARTBEAT_PERIOD}" + SE_OPTS="$SE_OPTS --heartbeat-period ${SE_NODE_HEARTBEAT_PERIOD}" +fi + if [ ! -z "$SE_LOG_LEVEL" ]; then echo "Appending Selenium options: --log-level ${SE_LOG_LEVEL}" SE_OPTS="$SE_OPTS --log-level ${SE_LOG_LEVEL}" @@ -77,12 +102,28 @@ fi EXTRA_LIBS="" -if [ ! -z "$SE_ENABLE_TRACING" ]; then +if [ "$SE_ENABLE_TRACING" = "true" ]; then EXTERNAL_JARS=$( /opt/selenium/browser_name RUN google-chrome --version | awk '{print $3}' > /opt/selenium/browser_version RUN echo "\"goog:chromeOptions\": {\"binary\": \"/usr/bin/google-chrome\"}" > /opt/selenium/browser_binary_location + +ENV SE_OTEL_SERVICE_NAME "selenium-node-chrome" diff --git a/NodeDocker/Dockerfile b/NodeDocker/Dockerfile index b65820cec..495dc9c78 100644 --- a/NodeDocker/Dockerfile +++ b/NodeDocker/Dockerfile @@ -28,3 +28,4 @@ COPY --chown="${SEL_UID}:${SEL_GID}" start-selenium-grid-docker.sh \ COPY selenium-grid-docker.conf /etc/supervisor/conf.d/ +ENV SE_OTEL_SERVICE_NAME "selenium-node-docker" diff --git a/NodeDocker/config.toml b/NodeDocker/config.toml index ec3e0a1a0..cc2693726 100644 --- a/NodeDocker/config.toml +++ b/NodeDocker/config.toml @@ -2,9 +2,9 @@ # Configs have a mapping between the Docker image to use and the capabilities that need to be matched to # start a container with the given image. configs = [ - "selenium/standalone-firefox:4.17.0-20240123", '{"browserName": "firefox", "platformName": "linux"}', - "selenium/standalone-chrome:4.17.0-20240123", '{"browserName": "chrome", "platformName": "linux"}', - "selenium/standalone-edge:4.17.0-20240123", '{"browserName": "MicrosoftEdge", "platformName": "linux"}' + "selenium/standalone-firefox:4.18.0-20240220", '{"browserName": "firefox", "platformName": "linux"}', + "selenium/standalone-chrome:4.18.0-20240220", '{"browserName": "chrome", "platformName": "linux"}', + "selenium/standalone-edge:4.18.0-20240220", '{"browserName": "MicrosoftEdge", "platformName": "linux"}' ] # URL for connecting to the docker daemon @@ -14,7 +14,7 @@ configs = [ # socat -4 TCP-LISTEN:2375,fork UNIX-CONNECT:/var/run/docker.sock url = "http://127.0.0.1:2375" # Docker image used for video recording -video-image = "selenium/video:ffmpeg-6.1-20240123" +video-image = "selenium/video:ffmpeg-6.1-20240220" # Uncomment the following section if you are running the node on a separate VM # Fill out the placeholders with appropriate values diff --git a/NodeDocker/start-selenium-grid-docker.sh b/NodeDocker/start-selenium-grid-docker.sh index 935fbee46..c5e91d597 100755 --- a/NodeDocker/start-selenium-grid-docker.sh +++ b/NodeDocker/start-selenium-grid-docker.sh @@ -60,12 +60,28 @@ fi EXTRA_LIBS="" -if [ ! -z "$SE_ENABLE_TRACING" ]; then +if [ "$SE_ENABLE_TRACING" = "true" ]; then EXTERNAL_JARS=$( /opt/selenium/browser_name RUN microsoft-edge --version | awk '{print $3}' > /opt/selenium/browser_version RUN echo "\"ms:edgeOptions\": {\"binary\": \"/usr/bin/microsoft-edge\"}" > /opt/selenium/browser_binary_location + +ENV SE_OTEL_SERVICE_NAME "selenium-node-edge" diff --git a/NodeFirefox/Dockerfile b/NodeFirefox/Dockerfile index 251be9b29..92d2684c5 100644 --- a/NodeFirefox/Dockerfile +++ b/NodeFirefox/Dockerfile @@ -44,3 +44,5 @@ USER ${SEL_UID} RUN echo "firefox" > /opt/selenium/browser_name RUN firefox --version | awk '{print $3}' > /opt/selenium/browser_version RUN echo "\"moz:firefoxOptions\": {\"binary\": \"/usr/bin/firefox\"}" > /opt/selenium/browser_binary_location + +ENV SE_OTEL_SERVICE_NAME "selenium-node-firefox" diff --git a/README.md b/README.md index a56ed791f..98f81d157 100644 --- a/README.md +++ b/README.md @@ -96,7 +96,7 @@ The project is made possible by volunteer contributors who have put in thousands and made the source code freely available under the [Apache License 2.0](LICENSE.md). These Docker images come with a handful of tags to simplify its usage, have a look at them in one of -our [releases](https://github.com/SeleniumHQ/docker-selenium/releases/tag/4.17.0-20240123). +our [releases](https://github.com/SeleniumHQ/docker-selenium/releases/tag/4.18.0-20240220). To get notifications of new releases, add yourself as a "Releases only" watcher. @@ -109,52 +109,66 @@ Talk to us at https://www.selenium.dev/support/ ## Contents + +* [Community](#community) +* [Contents](#contents) * [Quick start](#quick-start) + * [Try them out in a ready-to-use GitPod environment!](#try-them-out-in-a-ready-to-use-gitpod-environment) * [Experimental Mult-Arch aarch64/armhf/amd64 Images](#experimental-mult-arch-aarch64armhfamd64-images) * [Nightly Images](#nightly-images) * [Dev and Beta Channel Browser Images](#dev-and-beta-channel-browser-images) - + [Dev and Beta Standalone Mode](#dev-and-beta-standalone-mode) - + [Dev and Beta on the Grid](#dev-and-beta-on-the-grid) + * [Dev and Beta Standalone Mode](#dev-and-beta-standalone-mode) + * [Dev and Beta on the Grid](#dev-and-beta-on-the-grid) * [Execution modes](#execution-modes) - + [Standalone](#standalone) - + [Hub and Nodes](#hub-and-nodes) - - [Docker networking](#docker-networking) - - [Using different machines/VMs](#using-different-machinesvms) - - [Docker Compose](#docker-compose) - + [Fully distributed mode - Router, Queue, Distributor, EventBus, SessionMap and Nodes](#fully-distributed-mode---router-queue-distributor-eventbus-sessionmap-and-nodes) + * [Standalone](#standalone) + * [Hub and Nodes](#hub-and-nodes) + * [Fully distributed mode - Router, Queue, Distributor, EventBus, SessionMap and Nodes](#fully-distributed-mode---router-queue-distributor-eventbus-sessionmap-and-nodes) * [Video recording](#video-recording) +* [Video recording and uploading](#video-recording-and-uploading) * [Dynamic Grid](#dynamic-grid) - + [Configuration example](#configuration-example) - + [Execution with Hub & Node roles](#execution-with-hub--node-roles) - + [Execution with Standalone roles](#execution-with-standalone-roles) - + [Using Dynamic Grid in different machines/VMs](#using-dynamic-grid-in-different-machinesvms) - + [Execution with Docker Compose](#execution-with-docker-compose) - + [Video recording, screen resolution, and time zones in a Dynamic Grid](#video-recording-screen-resolution-and-time-zones-in-a-dynamic-grid) -* [Kubernetes](#deploying-to-kubernetes) + * [Configuration example](#configuration-example) + * [Execution with Hub & Node roles](#execution-with-hub--node-roles) + * [Execution with Standalone roles](#execution-with-standalone-roles) + * [Using Dynamic Grid in different machines/VMs](#using-dynamic-grid-in-different-machinesvms) + * [Execution with Docker Compose](#execution-with-docker-compose) + * [Configuring the child containers](#configuring-the-child-containers) + * [Video recording, screen resolution, and time zones in a Dynamic Grid](#video-recording-screen-resolution-and-time-zones-in-a-dynamic-grid) +* [Deploying to Kubernetes](#deploying-to-kubernetes) * [Configuring the containers](#configuring-the-containers) - + [SE_OPTS Selenium Configuration Options](#se_opts-selenium-configuration-options) - + [SE_JAVA_OPTS Java Environment Options](#se_java_opts-java-environment-options) - + [Node configuration options](#node-configuration-options) - + [Setting Sub Path](#setting-sub-path) - + [Setting Screen Resolution](#setting-screen-resolution) - + [Grid Url and Session Timeout](#grid-url-and-session-timeout) - + [Session request timeout](#session-request-timeout) - + [Increasing session concurrency per container](#increasing-session-concurrency-per-container) - + [Running in Headless mode](#running-in-headless-mode) + * [SE_OPTS Selenium Configuration Options](#se_opts-selenium-configuration-options) + * [SE_JAVA_OPTS Java Environment Options](#se_java_opts-java-environment-options) + * [Node configuration options](#node-configuration-options) + * [Setting Sub Path](#setting-sub-path) + * [Setting Screen Resolution](#setting-screen-resolution) + * [Grid Url and Session Timeout](#grid-url-and-session-timeout) + * [Session request timeout](#session-request-timeout) + * [Increasing session concurrency per container](#increasing-session-concurrency-per-container) + * [Running in Headless mode](#running-in-headless-mode) + * [Stopping the Node/Standalone after N sessions have been executed](#stopping-the-nodestandalone-after-n-sessions-have-been-executed) * [Building the images](#building-the-images) * [Waiting for the Grid to be ready](#waiting-for-the-grid-to-be-ready) + * [Adding a HEALTHCHECK to the Grid](#adding-a-healthcheck-to-the-grid) + * [Using a bash script to wait for the Grid](#using-a-bash-script-to-wait-for-the-grid) +* [Install certificates for Chromium-based browsers](#install-certificates-for-chromium-based-browsers) +* [Alternative method: Add certificates to existing Selenium based images for browsers](#alternative-method-add-certificates-to-existing-selenium-based-images-for-browsers) * [Debugging](#debugging) -* [Install certificates for Chromium based browsers](#install-certificates-for-Chromium-based-browsers) + * [Using a VNC client](#using-a-vnc-client) + * [Using your browser (no VNC client is needed)](#using-your-browser-no-vnc-client-is-needed) + * [Disabling VNC](#disabling-vnc) * [Tracing in Grid](#tracing-in-grid) * [Troubleshooting](#troubleshooting) - + * [`--shm-size="2g"`](#--shm-size2g) + * [Headless](#headless) + * [Mounting volumes to retrieve downloaded files](#mounting-volumes-to-retrieve-downloaded-files) + * [Mounting volumes to retrieve video files](#mounting-volumes-to-retrieve-video-files) + ## Quick start 1. Start a Docker container with Firefox ```bash -docker run -d -p 4444:4444 -p 7900:7900 --shm-size="2g" selenium/standalone-firefox:4.17.0-20240123 +docker run -d -p 4444:4444 -p 7900:7900 --shm-size="2g" selenium/standalone-firefox:4.18.0-20240220 ``` 2. Point your WebDriver tests to http://localhost:4444 @@ -349,17 +363,17 @@ For more information on the Dev and Beta channel container images, see the blog ![Firefox](https://raw.githubusercontent.com/alrra/browser-logos/main/src/firefox/firefox_24x24.png) Firefox ```bash -docker run -d -p 4444:4444 --shm-size="2g" selenium/standalone-firefox:4.17.0-20240123 +docker run -d -p 4444:4444 --shm-size="2g" selenium/standalone-firefox:4.18.0-20240220 ``` ![Chrome](https://raw.githubusercontent.com/alrra/browser-logos/main/src/chrome/chrome_24x24.png) Chrome ```bash -docker run -d -p 4444:4444 --shm-size="2g" selenium/standalone-chrome:4.17.0-20240123 +docker run -d -p 4444:4444 --shm-size="2g" selenium/standalone-chrome:4.18.0-20240220 ``` ![Edge](https://raw.githubusercontent.com/alrra/browser-logos/main/src/edge/edge_24x24.png) Edge ```bash -docker run -d -p 4444:4444 --shm-size="2g" selenium/standalone-edge:4.17.0-20240123 +docker run -d -p 4444:4444 --shm-size="2g" selenium/standalone-edge:4.18.0-20240220 ``` _Note: Only one Standalone container can run on port_ `4444` _at the same time._ @@ -378,44 +392,44 @@ A Docker [network](https://docs.docker.com/engine/reference/commandline/network_ ```bash $ docker network create grid -$ docker run -d -p 4442-4444:4442-4444 --net grid --name selenium-hub selenium/hub:4.17.0-20240123 +$ docker run -d -p 4442-4444:4442-4444 --net grid --name selenium-hub selenium/hub:4.18.0-20240220 $ docker run -d --net grid -e SE_EVENT_BUS_HOST=selenium-hub \ --shm-size="2g" \ -e SE_EVENT_BUS_PUBLISH_PORT=4442 \ -e SE_EVENT_BUS_SUBSCRIBE_PORT=4443 \ - selenium/node-chrome:4.17.0-20240123 + selenium/node-chrome:4.18.0-20240220 $ docker run -d --net grid -e SE_EVENT_BUS_HOST=selenium-hub \ --shm-size="2g" \ -e SE_EVENT_BUS_PUBLISH_PORT=4442 \ -e SE_EVENT_BUS_SUBSCRIBE_PORT=4443 \ - selenium/node-edge:4.17.0-20240123 + selenium/node-edge:4.18.0-20240220 $ docker run -d --net grid -e SE_EVENT_BUS_HOST=selenium-hub \ --shm-size="2g" \ -e SE_EVENT_BUS_PUBLISH_PORT=4442 \ -e SE_EVENT_BUS_SUBSCRIBE_PORT=4443 \ - selenium/node-firefox:4.17.0-20240123 + selenium/node-firefox:4.18.0-20240220 ``` ##### Windows PowerShell ```powershell $ docker network create grid -$ docker run -d -p 4442-4444:4442-4444 --net grid --name selenium-hub selenium/hub:4.17.0-20240123 +$ docker run -d -p 4442-4444:4442-4444 --net grid --name selenium-hub selenium/hub:4.18.0-20240220 $ docker run -d --net grid -e SE_EVENT_BUS_HOST=selenium-hub ` --shm-size="2g" ` -e SE_EVENT_BUS_PUBLISH_PORT=4442 ` -e SE_EVENT_BUS_SUBSCRIBE_PORT=4443 ` - selenium/node-chrome:4.17.0-20240123 + selenium/node-chrome:4.18.0-20240220 $ docker run -d --net grid -e SE_EVENT_BUS_HOST=selenium-hub ` --shm-size="2g" ` -e SE_EVENT_BUS_PUBLISH_PORT=4442 ` -e SE_EVENT_BUS_SUBSCRIBE_PORT=4443 ` - selenium/node-edge:4.17.0-20240123 + selenium/node-edge:4.18.0-20240220 $ docker run -d --net grid -e SE_EVENT_BUS_HOST=selenium-hub ` --shm-size="2g" ` -e SE_EVENT_BUS_PUBLISH_PORT=4442 ` -e SE_EVENT_BUS_SUBSCRIBE_PORT=4443 ` - selenium/node-firefox:4.17.0-20240123 + selenium/node-firefox:4.18.0-20240220 ``` When you are done using the Grid, and the containers have exited, the network can be removed with the following command: @@ -432,7 +446,7 @@ configured to expose different ports. ##### Hub - Machine/VM 1 ```bash -$ docker run -d -p 4442-4444:4442-4444 --name selenium-hub selenium/hub:4.17.0-20240123 +$ docker run -d -p 4442-4444:4442-4444 --name selenium-hub selenium/hub:4.18.0-20240220 ``` ##### Node Chrome - Machine/VM 2 @@ -446,7 +460,7 @@ $ docker run -d -p 5555:5555 \ -e SE_EVENT_BUS_PUBLISH_PORT=4442 \ -e SE_EVENT_BUS_SUBSCRIBE_PORT=4443 \ -e SE_NODE_HOST= \ - selenium/node-chrome:4.17.0-20240123 + selenium/node-chrome:4.18.0-20240220 ``` ###### Windows PowerShell @@ -458,7 +472,7 @@ $ docker run -d -p 5555:5555 ` -e SE_EVENT_BUS_PUBLISH_PORT=4442 ` -e SE_EVENT_BUS_SUBSCRIBE_PORT=4443 ` -e SE_NODE_HOST= ` - selenium/node-chrome:4.17.0-20240123 + selenium/node-chrome:4.18.0-20240220 ``` @@ -473,7 +487,7 @@ $ docker run -d -p 5555:5555 \ -e SE_EVENT_BUS_PUBLISH_PORT=4442 \ -e SE_EVENT_BUS_SUBSCRIBE_PORT=4443 \ -e SE_NODE_HOST= \ - selenium/node-edge:4.17.0-20240123 + selenium/node-edge:4.18.0-20240220 ``` ###### Windows PowerShell @@ -485,7 +499,7 @@ $ docker run -d -p 5555:5555 ` -e SE_EVENT_BUS_PUBLISH_PORT=4442 ` -e SE_EVENT_BUS_SUBSCRIBE_PORT=4443 ` -e SE_NODE_HOST= ` - selenium/node-edge:4.17.0-20240123 + selenium/node-edge:4.18.0-20240220 ``` ##### Node Firefox - Machine/VM 4 @@ -499,7 +513,7 @@ $ docker run -d -p 5555:5555 \ -e SE_EVENT_BUS_PUBLISH_PORT=4442 \ -e SE_EVENT_BUS_SUBSCRIBE_PORT=4443 \ -e SE_NODE_HOST= \ - selenium/node-firefox:4.17.0-20240123 + selenium/node-firefox:4.18.0-20240220 ``` ###### Windows PowerShell @@ -511,7 +525,7 @@ $ docker run -d -p 5555:5555 ` -e SE_EVENT_BUS_PUBLISH_PORT=4442 ` -e SE_EVENT_BUS_SUBSCRIBE_PORT=4443 ` -e SE_NODE_HOST= ` - selenium/node-firefox:4.17.0-20240123 + selenium/node-firefox:4.18.0-20240220 ``` ##### Node Chrome - Machine/VM 4 @@ -526,7 +540,7 @@ $ docker run -d -p 5556:5556 \ -e SE_EVENT_BUS_SUBSCRIBE_PORT=4443 \ -e SE_NODE_HOST= \ -e SE_NODE_PORT=5556 \ - selenium/node-chrome:4.17.0-20240123 + selenium/node-chrome:4.18.0-20240220 ``` ###### Windows PowerShell @@ -539,7 +553,7 @@ $ docker run -d -p 5556:5556 ` -e SE_EVENT_BUS_SUBSCRIBE_PORT=4443 ` -e SE_NODE_HOST= ` -e SE_NODE_PORT=5556 ` - selenium/node-chrome:4.17.0-20240123 + selenium/node-chrome:4.18.0-20240220 ``` #### Docker Compose @@ -571,7 +585,7 @@ ___ ## Video recording -Tests execution can be recorded by using the `selenium/video:ffmpeg-6.1-20240123` +Tests execution can be recorded by using the `selenium/video:ffmpeg-6.1-20240220` Docker image. One container is needed per each container where a browser is running. This means if you are running 5 Nodes/Standalone containers, you will need 5 video containers, the mapping is 1-1. @@ -597,8 +611,8 @@ This example shows how to start the containers manually: ``` bash $ docker network create grid -$ docker run -d -p 4444:4444 -p 6900:5900 --net grid --name selenium --shm-size="2g" selenium/standalone-chrome:4.17.0-20240123 -$ docker run -d --net grid --name video -v /tmp/videos:/videos selenium/video:ffmpeg-6.1-20240123 +$ docker run -d -p 4444:4444 -p 6900:5900 --net grid --name selenium --shm-size="2g" selenium/standalone-chrome:4.18.0-20240220 +$ docker run -d --net grid --name video -v /tmp/videos:/videos selenium/video:ffmpeg-6.1-20240220 # Run your tests $ docker stop video && docker rm video $ docker stop selenium && docker rm selenium @@ -609,6 +623,47 @@ Here is an example using a Hub and a few Nodes: [`docker-compose-v3-video.yml`](docker-compose-v3-video.yml) +## Video recording and uploading + +[RCLONE](https://rclone.org/) is installed in the video recorder image. You can use it to upload the videos to a cloud storage service. +Besides the video recording mentioned above, you can enable the upload functionality by setting the following environment variables: + +```yaml +version: "3" +services: + chrome_video: + image: selenium/video:nightly + depends_on: + - chrome + environment: + - DISPLAY_CONTAINER_NAME=chrome + - SE_VIDEO_FILE_NAME=auto + - SE_VIDEO_UPLOAD_ENABLED=true + - SE_VIDEO_INTERNAL_UPLOAD=true + - SE_UPLOAD_DESTINATION_PREFIX=s3://mybucket/path + - RCLONE_CONFIG_S3_TYPE=s3 + - RCLONE_CONFIG_S3_PROVIDER=GCS + - RCLONE_CONFIG_S3_ENV_AUTH=true + - RCLONE_CONFIG_S3_REGION=asia-southeast1 + - RCLONE_CONFIG_S3_LOCATION_CONSTRAINT=asia-southeast1 + - RCLONE_CONFIG_S3_ACL=private + - RCLONE_CONFIG_S3_ACCESS_KEY_ID=xxx + - RCLONE_CONFIG_S3_SECRET_ACCESS_KEY=xxx + - RCLONE_CONFIG_S3_ENDPOINT=https://storage.googleapis.com +``` + +`SE_VIDEO_FILE_NAME=auto` will use the session id as the video file name. This ensures that the video file name is unique to upload. + +`SE_VIDEO_UPLOAD_ENABLED=true` will enable the video upload feature. In the background, it will create a pipefile with file and destination for uploader to consume and proceed. + +`SE_VIDEO_INTERNAL_UPLOAD=true` will use RCLONE installed in the container for upload. If you want to use another container for upload, set it to `false`. + +For environment variables with prefix `RCLONE_` is used to pass remote configuration to RCLONE. You can find more information about RCLONE configuration [here](https://rclone.org/docs/). + +[`docker-compose-v3-video-upload.yml`](docker-compose-v3-video-upload.yml) + +Note that upload function is not supported for Dynamic Grid. If you want it, please create a feature request. + ___ ## Dynamic Grid @@ -630,9 +685,9 @@ You can save this file locally and name it, for example, `config.toml`. # Configs have a mapping between the Docker image to use and the capabilities that need to be matched to # start a container with the given image. configs = [ - "selenium/standalone-firefox:4.17.0-20240123", '{"browserName": "firefox"}', - "selenium/standalone-chrome:4.17.0-20240123", '{"browserName": "chrome"}', - "selenium/standalone-edge:4.17.0-20240123", '{"browserName": "MicrosoftEdge"}' + "selenium/standalone-firefox:4.18.0-20240220", '{"browserName": "firefox"}', + "selenium/standalone-chrome:4.18.0-20240220", '{"browserName": "chrome"}', + "selenium/standalone-edge:4.18.0-20240220", '{"browserName": "MicrosoftEdge"}' ] # URL for connecting to the docker daemon @@ -645,7 +700,7 @@ configs = [ # Linux: varies from machine to machine, please mount /var/run/docker.sock. If this does not work, please create an issue. url = "http://127.0.0.1:2375" # Docker image used for video recording -video-image = "selenium/video:ffmpeg-6.1-20240123" +video-image = "selenium/video:ffmpeg-6.1-20240220" # Uncomment the following section if you are running the node on a separate VM # Fill out the placeholders with appropriate values @@ -664,28 +719,28 @@ virtual machines. ```bash $ docker network create grid -$ docker run -d -p 4442-4444:4442-4444 --net grid --name selenium-hub selenium/hub:4.17.0-20240123 +$ docker run -d -p 4442-4444:4442-4444 --net grid --name selenium-hub selenium/hub:4.18.0-20240220 $ docker run -d --net grid -e SE_EVENT_BUS_HOST=selenium-hub \ -e SE_EVENT_BUS_PUBLISH_PORT=4442 \ -e SE_EVENT_BUS_SUBSCRIBE_PORT=4443 \ -v ${PWD}/config.toml:/opt/bin/config.toml \ -v ${PWD}/assets:/opt/selenium/assets \ -v /var/run/docker.sock:/var/run/docker.sock \ - selenium/node-docker:4.17.0-20240123 + selenium/node-docker:4.18.0-20240220 ``` #### Windows PowerShell ```powershell $ docker network create grid -$ docker run -d -p 4442-4444:4442-4444 --net grid --name selenium-hub selenium/hub:4.17.0-20240123 +$ docker run -d -p 4442-4444:4442-4444 --net grid --name selenium-hub selenium/hub:4.18.0-20240220 $ docker run -d --net grid -e SE_EVENT_BUS_HOST=selenium-hub ` -e SE_EVENT_BUS_PUBLISH_PORT=4442 ` -e SE_EVENT_BUS_SUBSCRIBE_PORT=4443 ` -v ${PWD}/config.toml:/opt/bin/config.toml ` -v ${PWD}/assets:/opt/selenium/assets ` -v /var/run/docker.sock:/var/run/docker.sock ` - selenium/node-docker:4.17.0-20240123 + selenium/node-docker:4.18.0-20240220 ``` To have the assets saved on your host, please mount your host path to `/opt/selenium/assets`. @@ -706,7 +761,7 @@ docker run --rm --name selenium-docker -p 4444:4444 \ -v ${PWD}/config.toml:/opt/bin/config.toml \ -v ${PWD}/assets:/opt/selenium/assets \ -v /var/run/docker.sock:/var/run/docker.sock \ - selenium/standalone-docker:4.17.0-20240123 + selenium/standalone-docker:4.18.0-20240220 ``` #### Windows PowerShell @@ -716,7 +771,7 @@ docker run --rm --name selenium-docker -p 4444:4444 ` -v ${PWD}/config.toml:/opt/bin/config.toml ` -v ${PWD}/assets:/opt/selenium/assets ` -v /var/run/docker.sock:/var/run/docker.sock ` - selenium/standalone-docker:4.17.0-20240123 + selenium/standalone-docker:4.18.0-20240220 ``` ### Using Dynamic Grid in different machines/VMs @@ -724,7 +779,7 @@ docker run --rm --name selenium-docker -p 4444:4444 ` #### Hub - Machine/VM 1 ```bash -$ docker run -d -p 4442-4444:4442-4444 --name selenium-hub selenium/hub:4.17.0-20240123 +$ docker run -d -p 4442-4444:4442-4444 --name selenium-hub selenium/hub:4.18.0-20240220 ``` #### Node Chrome - Machine/VM 2 @@ -739,7 +794,7 @@ $ docker run -d -p 5555:5555 \ -v ${PWD}/config.toml:/opt/bin/config.toml \ -v ${PWD}/assets:/opt/selenium/assets \ -v /var/run/docker.sock:/var/run/docker.sock \ - selenium/node-docker:4.17.0-20240123 + selenium/node-docker:4.18.0-20240220 ``` #### Windows PowerShell @@ -752,7 +807,7 @@ $ docker run -d -p 5555:5555 ` -v ${PWD}/config.toml:/opt/bin/config.toml ` -v ${PWD}/assets:/opt/selenium/assets ` -v /var/run/docker.sock:/var/run/docker.sock ` - selenium/node-docker:4.17.0-20240123 + selenium/node-docker:4.18.0-20240220 ``` Complete the `[server]` section in the `config.toml` file. @@ -761,9 +816,9 @@ Complete the `[server]` section in the `config.toml` file. # Configs have a mapping between the Docker image to use and the capabilities that need to be matched to # start a container with the given image. configs = [ - "selenium/standalone-firefox:4.17.0-20240123", "{\"browserName\": \"firefox\"}", - "selenium/standalone-chrome:4.17.0-20240123", "{\"browserName\": \"chrome\"}", - "selenium/standalone-edge:4.17.0-20240123", "{\"browserName\": \"MicrosoftEdge\"}" + "selenium/standalone-firefox:4.18.0-20240220", "{\"browserName\": \"firefox\"}", + "selenium/standalone-chrome:4.18.0-20240220", "{\"browserName\": \"chrome\"}", + "selenium/standalone-edge:4.18.0-20240220", "{\"browserName\": \"MicrosoftEdge\"}" ] # URL for connecting to the docker daemon @@ -776,7 +831,7 @@ configs = [ # Linux: varies from machine to machine, please mount /var/run/docker.sock. If this does not work, please create an issue. url = "http://127.0.0.1:2375" # Docker image used for video recording -video-image = "selenium/video:ffmpeg-6.1-20240123" +video-image = "selenium/video:ffmpeg-6.1-20240220" # Uncomment the following section if you are running the node on a separate VM # Fill out the placeholders with appropriate values @@ -810,7 +865,7 @@ docker run --rm --name selenium-docker -p 4444:4444 \ -v ${PWD}/config.toml:/opt/bin/config.toml \ -v ${PWD}/assets:/opt/selenium/assets \ -v /var/run/docker.sock:/var/run/docker.sock \ - selenium/standalone-docker:4.17.0-20240123 + selenium/standalone-docker:4.18.0-20240220 ``` #### Windows PowerShell @@ -821,7 +876,7 @@ docker run --rm --name selenium-docker -p 4444:4444 ` -v ${PWD}/config.toml:/opt/bin/config.toml ` -v ${PWD}/assets:/opt/selenium/assets ` -v /var/run/docker.sock:/var/run/docker.sock ` - selenium/standalone-docker:4.17.0-20240123 + selenium/standalone-docker:4.18.0-20240220 ``` @@ -859,7 +914,7 @@ ___ You can pass `SE_OPTS` variable with additional command line parameters for starting a hub or a node. ``` bash -$ docker run -d -p 4444:4444 -e SE_OPTS="--log-level FINE" --name selenium-hub selenium/hub:4.17.0-20240123 +$ docker run -d -p 4444:4444 -e SE_OPTS="--log-level FINE" --name selenium-hub selenium/hub:4.18.0-20240220 ``` ### SE_JAVA_OPTS Java Environment Options @@ -867,7 +922,7 @@ $ docker run -d -p 4444:4444 -e SE_OPTS="--log-level FINE" --name selenium-hub s You can pass `SE_JAVA_OPTS` environment variable to the Java process. ``` bash -$ docker run -d -p 4444:4444 -e SE_JAVA_OPTS=-Xmx512m --name selenium-hub selenium/hub:4.17.0-20240123 +$ docker run -d -p 4444:4444 -e SE_JAVA_OPTS=-Xmx512m --name selenium-hub selenium/hub:4.18.0-20240220 ``` ### Node configuration options @@ -889,7 +944,7 @@ $ docker run -d \ -e SE_EVENT_BUS_HOST= \ -e SE_EVENT_BUS_PUBLISH_PORT=4442 \ -e SE_EVENT_BUS_SUBSCRIBE_PORT=4443 -e SE_NODE_STEREOTYPE="{\"browserName\":\"${SE_NODE_BROWSER_NAME}\",\"browserVersion\":\"${SE_NODE_BROWSER_VERSION}\",\"platformName\": \"Linux\"}" \ - --shm-size="2g" selenium/node-chrome:4.17.0-20240123 + --shm-size="2g" selenium/node-chrome:4.18.0-20240220 ``` ### Setting Sub Path @@ -908,7 +963,7 @@ These settings can be adjusted by specifying `SE_SCREEN_WIDTH`, `SE_SCREEN_HEIGH environmental variables when starting the container. ``` bash -docker run -d -e SE_SCREEN_WIDTH=1366 -e SE_SCREEN_HEIGHT=768 -e SE_SCREEN_DEPTH=24 -e SE_SCREEN_DPI=74 selenium/standalone-firefox:4.17.0-20240123 +docker run -d -e SE_SCREEN_WIDTH=1366 -e SE_SCREEN_HEIGHT=768 -e SE_SCREEN_DEPTH=24 -e SE_SCREEN_DPI=74 selenium/standalone-firefox:4.18.0-20240220 ``` ### Grid Url and Session Timeout @@ -958,7 +1013,7 @@ To avoid starting the server you can set the `SE_START_XVFB` environment variabl ``` bash $ docker run -d --net grid -e SE_EVENT_BUS_HOST=selenium-hub -e SE_EVENT_BUS_PUBLISH_PORT=4442 \ - -e SE_EVENT_BUS_SUBSCRIBE_PORT=4443 -e SE_START_XVFB=false --shm-size="2g" selenium/node-chrome:4.17.0-20240123 + -e SE_EVENT_BUS_SUBSCRIBE_PORT=4443 -e SE_START_XVFB=false --shm-size="2g" selenium/node-chrome:4.18.0-20240220 ``` For more information, see this GitHub [issue](https://github.com/SeleniumHQ/docker-selenium/issues/567). @@ -971,7 +1026,7 @@ pod and then scale a new one after N sessions. Set the environment variable `SE_ a value higher than zero to enable this behaviour. ``` bash -$ docker run -e SE_DRAIN_AFTER_SESSION_COUNT=5 --shm-size="2g" selenium/standalone-firefox:4.17.0-20240123 +$ docker run -e SE_DRAIN_AFTER_SESSION_COUNT=5 --shm-size="2g" selenium/standalone-firefox:4.18.0-20240220 ``` With the previous command, the Standalone container will shut down after 5 sessions have been executed. @@ -1074,22 +1129,22 @@ $ docker network create grid $ docker run -d -p 4442-4444:4442-4444 --net grid --name selenium-hub \ --health-cmd='/opt/bin/check-grid.sh --host 0.0.0.0 --port 4444' \ --health-interval=15s --health-timeout=30s --health-retries=5 \ - selenium/hub:4.17.0-20240123 + selenium/hub:4.18.0-20240220 $ docker run -d --net grid -e SE_EVENT_BUS_HOST=selenium-hub \ --shm-size="2g" \ -e SE_EVENT_BUS_PUBLISH_PORT=4442 \ -e SE_EVENT_BUS_SUBSCRIBE_PORT=4443 \ - selenium/node-chrome:4.17.0-20240123 + selenium/node-chrome:4.18.0-20240220 $ docker run -d --net grid -e SE_EVENT_BUS_HOST=selenium-hub \ --shm-size="2g" \ -e SE_EVENT_BUS_PUBLISH_PORT=4442 \ -e SE_EVENT_BUS_SUBSCRIBE_PORT=4443 \ - selenium/node-edge:4.17.0-20240123 + selenium/node-edge:4.18.0-20240220 $ docker run -d --net grid -e SE_EVENT_BUS_HOST=selenium-hub \ --shm-size="2g" \ -e SE_EVENT_BUS_PUBLISH_PORT=4442 \ -e SE_EVENT_BUS_SUBSCRIBE_PORT=4443 \ - selenium/node-firefox:4.17.0-20240123 + selenium/node-firefox:4.18.0-20240220 ``` **Note:** The `\` line delimiter won't work on Windows-based terminals, try either `^` or a backtick. @@ -1155,45 +1210,29 @@ ___ ## Install certificates for Chromium-based browsers -If you need to install custom certificates, CA, intermediate CA, or client certificates (for example enterprise internal CA) -you can use the different utils that come from libnss3-tools. -The chromium-based browser uses nssdb as a certificate store. -Create a new nssdb: -```bash -certutil -d sql:$HOME/.pki/nssdb -N -``` -For example, to trust a root CA certificate for issuing SSL server certificates, use -```bash -certutil -d sql:$HOME/.pki/nssdb -A -t "C,," -n -i -``` -To import an intermediate CA certificate, use -```bash -certutil -d sql:$HOME/.pki/nssdb -A -t ",," -n -i -``` -You can find more information [here](https://chromium.googlesource.com/chromium/src/+/master/docs/linux/cert_management.md) -Usage example: -If your company has an internal CA you can create your own dockerimage from selenium node image. +By default, the based image is installed `libnss3-tools` and initializes `/home/seluser/.pki/nssdb`, +so you are able to add your certs with rootless. +If you need to install custom certificates, CA, intermediate CA, +or client certificates (for example, enterprise internal CA) +you can create your own docker image from selenium node image. +The Chromium-based browser uses `nssdb` as a certificate store. You can then install all required internal certificates in your Dockerfile like this: + +- Create a script for installing the certificates. For example, [cert-script.sh](tests/customCACert/cert-script.sh) +- Create a Dockerfile that uses the selenium node image as a base and copies the script to the container, and executes it. +For example, [Dockerfile](tests/customCACert/Dockerfile) +- If you have to create a set of different certificates and node images. You can create a bootstrap script to do that in one-shot. +For example, [bootstrap.sh](tests/customCACert/bootstrap.sh) + +The above example can be tested with the following command: + ```bash -FROM selenium/node-edge:latest -USER root -COPY certs/ /etc/certs # copy over the certificate file - -#========= -# libnss3-tools -# Network Security Service tools -# Manage certificates in nssdb (certutil, pk12util, modutil, shlibsign, signtool, ssltap) -#========= -RUN apt-get update -qqy \ - && apt-get -qqy install \ - libnss3-tools \ - && rm -rf /var/lib/apt/lists/* /var/cache/apt/* - -RUN mkdir -p -m755 /home/seluser/.pki/nssdb \ #create nssdb folder - && certutil -d sql:/home/seluser/.pki/nssdb -N --empty-password \ # create new db without password - && certutil -d sql:/home/seluser/.pki/nssdb -A -t "C,," -n companyca -i /etc/certs/companeca.pem \ #trust company CA - && pk12util -d sql:/home/seluser/.pki/nssdb -i client_cert.p12 -W password_of_clent_cert # client certificate install +make test_custom_ca_cert +# ./tests/customCACert/bootstrap.sh ``` + +You can find more information [here](https://chromium.googlesource.com/chromium/src/+/master/docs/linux/cert_management.md) + This way the certificates will be installed and the node will start automatically as before. ___ @@ -1222,7 +1261,7 @@ RUN mkdir -p -m755 /seluser/.pki/nssdb \ && chown -R 1200:1201 /seluser # Start from Selenium image and add relevant files from build image -FROM selenium/node-chrome:4.17.0-20240123 +FROM selenium/node-chrome:4.18.0-20240220 USER root COPY --from=build /seluser/ /home/seluser/ USER seluser @@ -1237,7 +1276,7 @@ RUN mkdir -p "/distribution" "/certs" && \ echo '{ "policies": { "Certificates": { "Install": ["/opt/firefox-latest/YOUR_CA.pem"] }} }' >"/distribution/policies.json" # Start from Selenium image and add relevant files from build image -FROM selenium/node-firefox:4.17.0-20240123 +FROM selenium/node-firefox:4.18.0-20240220 USER root COPY --from=build /certs /opt/firefox-latest COPY --from=build /distribution /opt/firefox-latest/distribution @@ -1260,9 +1299,9 @@ You can override it with the `SE_VNC_PORT` environment variable in case you want Here is an example with the standalone images, the same concept applies to the node images. ``` bash -$ docker run -d -p 4444:4444 -p 5900:5900 --shm-size="2g" selenium/standalone-chrome:4.17.0-20240123 -$ docker run -d -p 4445:4444 -p 5901:5900 --shm-size="2g" selenium/standalone-edge:4.17.0-20240123 -$ docker run -d -p 4446:4444 -p 5902:5900 --shm-size="2g" selenium/standalone-firefox:4.17.0-20240123 +$ docker run -d -p 4444:4444 -p 5900:5900 --shm-size="2g" selenium/standalone-chrome:4.18.0-20240220 +$ docker run -d -p 4445:4444 -p 5901:5900 --shm-size="2g" selenium/standalone-edge:4.18.0-20240220 +$ docker run -d -p 4446:4444 -p 5902:5900 --shm-size="2g" selenium/standalone-firefox:4.18.0-20240220 ``` Then, you would use in your VNC client: @@ -1290,9 +1329,9 @@ You can also override it with the `SE_NO_VNC_PORT` environment variable in case Here is an example with the standalone images, the same concept applies to the node images. ``` bash -$ docker run -d -p 4444:4444 -p 7900:7900 --shm-size="2g" selenium/standalone-chrome:4.17.0-20240123 -$ docker run -d -p 4445:4444 -p 7901:7900 --shm-size="2g" selenium/standalone-edge:4.17.0-20240123 -$ docker run -d -p 4446:4444 -p 7902:7900 --shm-size="2g" selenium/standalone-firefox:4.17.0-20240123 +$ docker run -d -p 4444:4444 -p 7900:7900 --shm-size="2g" selenium/standalone-chrome:4.18.0-20240220 +$ docker run -d -p 4445:4444 -p 7901:7900 --shm-size="2g" selenium/standalone-edge:4.18.0-20240220 +$ docker run -d -p 4446:4444 -p 7902:7900 --shm-size="2g" selenium/standalone-firefox:4.18.0-20240220 ``` Then, you would use in your browser: @@ -1316,29 +1355,32 @@ In order to enable tracing in the Selenium Grid container, the following command ```bash docker network create grid -docker run -d -p 16686:16686 -p 14250:14250 --net grid --name jaeger jaegertracing/all-in-one:1.17 -docker run -d -p 4442-4444:4442-4444 --net grid --name selenium-hub selenium/hub:4.17.0-20240123 +docker run -d -p 16686:16686 -p 4317:4317 --net grid --name jaeger jaegertracing/all-in-one:1.54 +docker run -d -p 4442-4444:4442-4444 --net grid --name selenium-hub selenium/hub:4.18.0-20240220 docker run -d --net grid -e SE_EVENT_BUS_HOST=selenium-hub \ --shm-size="2g" \ -e SE_ENABLE_TRACING=true \ - -e JAVA_OPTS="-Dotel.traces.exporter=jaeger -Dotel.exporter.jaeger.endpoint=http://jaegar:14250 -Dotel.resource.attributes=service.name=selenium-node-chrome" \ + -e SE_OTEL_TRACES_EXPORTER=otlp \ + -e SE_OTEL_EXPORTER_ENDPOINT=http://jaeger:4317 \ -e SE_EVENT_BUS_PUBLISH_PORT=4442 \ -e SE_EVENT_BUS_SUBSCRIBE_PORT=4443 \ - selenium/node-chrome:4.17.0-20240123 + selenium/node-chrome:4.18.0-20240220 docker run -d --net grid -e SE_EVENT_BUS_HOST=selenium-hub \ --shm-size="2g" \ -e SE_ENABLE_TRACING=true \ - -e JAVA_OPTS="-Dotel.traces.exporter=jaeger -Dotel.exporter.jaeger.endpoint=http://jaegar:14250 -Dotel.resource.attributes=service.name=selenium-node-edge" \ + -e SE_OTEL_TRACES_EXPORTER=otlp \ + -e SE_OTEL_EXPORTER_ENDPOINT=http://jaeger:4317 \ -e SE_EVENT_BUS_PUBLISH_PORT=4442 \ -e SE_EVENT_BUS_SUBSCRIBE_PORT=4443 \ - selenium/node-edge:4.17.0-20240123 + selenium/node-edge:4.18.0-20240220 docker run -d --net grid -e SE_EVENT_BUS_HOST=selenium-hub \ --shm-size="2g" \ -e SE_ENABLE_TRACING=true \ - -e JAVA_OPTS="-Dotel.traces.exporter=jaeger -Dotel.exporter.jaeger.endpoint=http://jaegar:14250 -Dotel.resource.attributes=service.name=selenium-node-firefox" \ + -e SE_OTEL_TRACES_EXPORTER=otlp \ + -e SE_OTEL_EXPORTER_ENDPOINT=http://jaeger:4317 \ -e SE_EVENT_BUS_PUBLISH_PORT=4442 \ -e SE_EVENT_BUS_SUBSCRIBE_PORT=4443 \ - selenium/node-firefox:4.17.0-20240123 + selenium/node-firefox:4.18.0-20240220 ``` You can also refer to the below docker-compose yaml files to be able to start a simple grid (or) a dynamic grid. @@ -1347,7 +1389,7 @@ You can also refer to the below docker-compose yaml files to be able to start a * Simple Grid [v2 yaml file](docker-compose-v2-tracing.yml) * Dynamic Grid [v3 yaml file](docker-compose-v3-full-grid-tracing.yml) -You can view the [Jaegar UI](http://localhost:16686/) and trace your request. +You can view the [Jaeger UI](http://localhost:16686/) and trace your request. ___ ## Troubleshooting @@ -1398,7 +1440,7 @@ container in the following way: ```bash docker run -d -p 4444:4444 --shm-size="2g" \ -v /home/ubuntu/files:/home/seluser/Downloads \ - selenium/standalone-chrome:4.17.0-20240123 + selenium/standalone-chrome:4.18.0-20240220 ``` That will mount the host `/home/ubuntu/files` directory @@ -1424,4 +1466,11 @@ After doing this, you should be able to download files to the mounted directory. If you have a better workaround, please send us a pull request! +### Mounting volumes to retrieve video files + +Similar to mount volumes to retrieve downloaded files. For video files, you might need to do the same +```bash +mkdir /tmp/videos +chown 1200:1201 /tmp/videos +``` diff --git a/Router/Dockerfile b/Router/Dockerfile index 3410b932f..cf1d7029b 100644 --- a/Router/Dockerfile +++ b/Router/Dockerfile @@ -13,7 +13,7 @@ USER ${SEL_UID} # In seconds, maps to "--session-request-timeout" ENV SE_SESSION_REQUEST_TIMEOUT 300 # In seconds, maps to "--session-retry-interval" -ENV SE_SESSION_RETRY_INTERVAL 0 +ENV SE_SESSION_RETRY_INTERVAL 5 EXPOSE 4444 @@ -21,3 +21,5 @@ COPY --chown="${SEL_UID}:${SEL_GID}" start-selenium-grid-router.sh \ /opt/bin/ COPY selenium-grid-router.conf /etc/supervisor/conf.d/ + +ENV SE_OTEL_SERVICE_NAME "selenium-router" diff --git a/Router/start-selenium-grid-router.sh b/Router/start-selenium-grid-router.sh index 3288abb15..fa1946f77 100755 --- a/Router/start-selenium-grid-router.sh +++ b/Router/start-selenium-grid-router.sh @@ -95,12 +95,28 @@ fi EXTRA_LIBS="" -if [ ! -z "$SE_ENABLE_TRACING" ]; then +if [ "$SE_ENABLE_TRACING" = "true" ]; then EXTERNAL_JARS=$( /etc/ap # No interactive frontend during docker build ENV DEBIAN_FRONTEND=noninteractive \ - DEBCONF_NONINTERACTIVE_SEEN=true + DEBCONF_NONINTERACTIVE_SEEN=true \ + PIP_ROOT_USER_ACTION=ignore #======================== # Supervisor @@ -20,21 +30,72 @@ ENV DEBIAN_FRONTEND=noninteractive \ RUN apt-get -qqy update \ && apt-get upgrade -yq \ && apt-get -qqy --no-install-recommends install \ - supervisor x11-xserver-utils x11-utils curl jq python3-pip \ + supervisor x11-xserver-utils x11-utils curl jq python3-pip tzdata acl unzip \ && python3 -m pip install --upgrade pip \ && python3 -m pip install --upgrade setuptools \ && python3 -m pip install --upgrade wheel \ + && python3 -m pip install psutil \ && rm -rf /var/lib/apt/lists/* /var/cache/apt/* +#=================== +# Timezone settings +# Possible alternative: https://github.com/docker/docker/issues/3359#issuecomment-32150214 +#=================== +ENV TZ "UTC" +RUN ln -fs /usr/share/zoneinfo/${TZ} /etc/localtime && \ + dpkg-reconfigure -f noninteractive tzdata && \ + cat /etc/timezone + +#====================================== +# Configure environement +#====================================== +ENV SEL_USER=${SEL_USER} +ENV SEL_UID=${UID} +ENV SEL_GID=${GID} +ENV HOME=/home/${SEL_USER} + +#======================================== +# Add normal user and group with passwordless sudo +#======================================== +RUN groupadd ${SEL_GROUP} \ + --gid ${SEL_GID} \ + && useradd ${SEL_USER} \ + --create-home \ + --gid ${SEL_GID} \ + --shell /bin/bash \ + --uid ${SEL_UID} \ + && usermod -a -G sudo ${SEL_USER} \ + && echo 'ALL ALL = (ALL) NOPASSWD: ALL' >> /etc/sudoers \ + && echo "${SEL_USER}:${SEL_PASSWD}" | chpasswd + #====================================== # Add Supervisor configuration files #====================================== COPY supervisord.conf /etc -COPY entry_point.sh video.sh video_ready.py /opt/bin/ -RUN cd /opt/bin && pip install psutil +COPY --chown="${SEL_UID}:${SEL_GID}" entry_point.sh video.sh video_ready.py video_graphQLQuery.sh /opt/bin/ + +#====================================== +# Add RCLONE for uploading videos +#====================================== +RUN curl -fLo /tmp/rclone.zip https://downloads.rclone.org/rclone-${RCLONE_VERSION}-linux-amd64.zip \ + && unzip -a /tmp/rclone.zip -d /tmp \ + && mv /tmp/rclone-*-linux-amd64/rclone /usr/local/bin/ \ + && rm -rf /tmp/rclone-* +COPY --chown="${SEL_UID}:${SEL_GID}" upload.sh upload.conf /opt/bin/ +ENV SE_VIDEO_UPLOAD_ENABLED false +ENV SE_UPLOAD_DESTINATION_PREFIX "" +ENV SE_VIDEO_INTERNAL_UPLOAD false +ENV UPLOAD_OPTS "-P" ENV SE_VIDEO_FOLDER /videos -RUN mkdir -p /var/run/supervisor /var/log/supervisor $SE_VIDEO_FOLDER +RUN mkdir -p /var/run/supervisor /var/log/supervisor $SE_VIDEO_FOLDER \ + && chown -R ${SEL_USER}:${SEL_GROUP} /var/run/supervisor /var/log/supervisor $SE_VIDEO_FOLDER $HOME \ + && chmod -R 775 /var/run/supervisor /var/log/supervisor $SE_VIDEO_FOLDER $HOME \ + && chgrp -R 0 /var/run/supervisor /var/log/supervisor $SE_VIDEO_FOLDER $HOME \ + && chmod -R g=u /var/run/supervisor /var/log/supervisor $SE_VIDEO_FOLDER $HOME \ + && setfacl -Rdm u:${SEL_USER}:rwx,g:${SEL_GROUP}:rwx /var/run/supervisor /var/log/supervisor $SE_VIDEO_FOLDER $HOME + +USER ${SEL_UID} ENTRYPOINT ["/opt/bin/entry_point.sh"] CMD ["/opt/bin/entry_point.sh"] @@ -46,6 +107,6 @@ ENV SE_SCREEN_HEIGHT 1020 ENV SE_FRAME_RATE 15 ENV SE_CODEC libx264 ENV SE_PRESET "-preset ultrafast" -ENV FILE_NAME video.mp4 +ENV SE_VIDEO_FILE_NAME video.mp4 EXPOSE 9000 diff --git a/Video/entry_point.sh b/Video/entry_point.sh index ae951a318..e9c01eea8 100755 --- a/Video/entry_point.sh +++ b/Video/entry_point.sh @@ -1,4 +1,14 @@ #!/usr/bin/env bash +#============================================== +# OpenShift or non-sudo environments support +# https://docs.openshift.com/container-platform/3.11/creating_images/guidelines.html#openshift-specific-guidelines +#============================================== + +if ! whoami &> /dev/null; then + if [ -w /etc/passwd ]; then + echo "${USER_NAME:-default}:x:$(id -u):0:${USER_NAME:-default} user:${HOME}:/sbin/nologin" >> /etc/passwd + fi +fi /usr/bin/supervisord --configuration /etc/supervisord.conf & diff --git a/Video/supervisord.conf b/Video/supervisord.conf old mode 100644 new mode 100755 index 84b7d6ae2..87a194cf9 --- a/Video/supervisord.conf +++ b/Video/supervisord.conf @@ -10,7 +10,6 @@ pidfile=/var/run/supervisor/supervisord.pid ; (supervisord pidfile;default sup nodaemon=true ; (start in foreground if true;default false) minfds=1024 ; (min. avail startup file descriptors;default 1024) minprocs=200 ; (min. avail process descriptors;default 200) -user=root ; (default is current user, required if root) [program:video-recording] priority=0 @@ -35,3 +34,15 @@ stopsignal=INT redirect_stderr=true stdout_logfile=/dev/stdout stdout_logfile_maxbytes=0 + +[program:video-upload] +priority=10 +command=bash -c "if [ ${SE_VIDEO_INTERNAL_UPLOAD} = "true" ]; then /opt/bin/upload.sh; fi" +autostart=%(ENV_SE_VIDEO_INTERNAL_UPLOAD)s +autorestart=%(ENV_SE_VIDEO_INTERNAL_UPLOAD)s +stopsignal=INT + +;Logs (all activity redirected to stdout so it can be seen through "docker logs" +redirect_stderr=true +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 diff --git a/Video/upload.conf b/Video/upload.conf new file mode 100755 index 000000000..e69de29bb diff --git a/Video/upload.sh b/Video/upload.sh new file mode 100755 index 000000000..15f914691 --- /dev/null +++ b/Video/upload.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash + +SE_VIDEO_FOLDER=${SE_VIDEO_FOLDER:-"/videos"} +UPLOAD_CONFIG_DIRECTORY=${UPLOAD_CONFIG_DIRECTORY:-"/opt/bin"} +UPLOAD_CONFIG_FILE_NAME=${UPLOAD_CONFIG_FILE_NAME:-"upload.conf"} +UPLOAD_COMMAND=${UPLOAD_COMMAND:-"copy"} +UPLOAD_RETAIN_LOCAL_FILE=${SE_UPLOAD_RETAIN_LOCAL_FILE:-"false"} +UPLOAD_PIPE_FILE_NAME=${UPLOAD_PIPE_FILE_NAME:-"uploadpipe"} +SE_VIDEO_INTERNAL_UPLOAD=${SE_VIDEO_INTERNAL_UPLOAD:-"false"} +VIDEO_UPLOAD_ENABLED=${VIDEO_UPLOAD_ENABLED:-SE_VIDEO_UPLOAD_ENABLED} +SE_VIDEO_UPLOAD_BATCH_CHECK=${SE_VIDEO_UPLOAD_BATCH_CHECK:-"10"} + +if [ "${SE_VIDEO_INTERNAL_UPLOAD}" = "true" ]; +then + # If using RCLONE in the same container, write signal to /tmp internally + UPLOAD_PIPE_FILE="/tmp/${UPLOAD_PIPE_FILE_NAME}" + FORCE_EXIT_FILE="/tmp/force_exit" +else + # If using external container for uploading, write signal to the video folder + UPLOAD_PIPE_FILE="${SE_VIDEO_FOLDER}/${UPLOAD_PIPE_FILE_NAME}" + FORCE_EXIT_FILE="${SE_VIDEO_FOLDER}/force_exit" +fi + +if [ "${UPLOAD_RETAIN_LOCAL_FILE}" = "false" ]; +then + echo "UPLOAD_RETAIN_LOCAL_FILE is set to false, force to use RCLONE command: move" + UPLOAD_COMMAND="move" +fi + +function rename_rclone_env() { + # This script is used to support passing environment variables for RCLONE configuration in Dynamic Grid + # Dynamic Grid accepts environment variables with the prefix SE_* + # RCLONE accepts environment variables with the prefix RCLONE_* + # To pass the ENV vars to Dynamic Grid then to RCLONE, we need to rename the ENV vars from SE_RCLONE_* to RCLONE_* + for var in $(env | cut -d= -f1); do + if [[ "$var" == SE_RCLONE_* ]]; + then + suffix="${var#SE_RCLONE_}" + new_var="RCLONE_$suffix" + export "$new_var=${!var}" + fi + done +} + +function graceful_exit() { + for pid in "${list_rclone_pid[@]}"; + do + wait ${pid} + done + rm -rf ${FORCE_EXIT_FILE} + echo "Uploader is ready to shutdown" +} +trap graceful_exit EXIT + +while [ ! -p ${UPLOAD_PIPE_FILE} ]; +do + echo "Waiting for ${UPLOAD_PIPE_FILE} to be created" + sleep 1 +done + +echo "Waiting for video files put into pipe for proceeding to upload" + +rename_rclone_env + +list_rclone_pid=() +while read FILE DESTINATION < ${UPLOAD_PIPE_FILE} +do + if [ "${FILE}" = "exit" ]; + then + exit + elif [ "$FILE" != "" ] && [ "$DESTINATION" != "" ]; + then + echo "Uploading ${FILE} to ${DESTINATION}" + rclone --config ${UPLOAD_CONFIG_DIRECTORY}/${UPLOAD_CONFIG_FILE_NAME} ${UPLOAD_COMMAND} ${UPLOAD_OPTS} "${FILE}" "${DESTINATION}" & + list_rclone_pid+=($!) + else + # Wait for a batch rclone processes to finish + if [ ${#list_rclone_pid[@]} -eq ${SE_VIDEO_UPLOAD_BATCH_CHECK} ] + then + for pid in "${list_rclone_pid[@]}"; + do + wait ${pid} + done + list_rclone_pid=() + fi + fi +done + +graceful_exit diff --git a/Video/video.sh b/Video/video.sh index 7bf59a015..ca06910a3 100755 --- a/Video/video.sh +++ b/Video/video.sh @@ -1,28 +1,213 @@ -#!/bin/sh +#!/usr/bin/env bash VIDEO_SIZE="${SE_SCREEN_WIDTH}""x""${SE_SCREEN_HEIGHT}" DISPLAY_CONTAINER_NAME=${DISPLAY_CONTAINER_NAME} DISPLAY_NUM=${DISPLAY_NUM} -FILE_NAME=${FILE_NAME} +VIDEO_FILE_NAME=${FILE_NAME:-$SE_VIDEO_FILE_NAME} FRAME_RATE=${FRAME_RATE:-$SE_FRAME_RATE} CODEC=${CODEC:-$SE_CODEC} PRESET=${PRESET:-$SE_PRESET} VIDEO_FOLDER=${SE_VIDEO_FOLDER} +VIDEO_UPLOAD_ENABLED=${VIDEO_UPLOAD_ENABLED:-$SE_VIDEO_UPLOAD_ENABLED} +VIDEO_CONFIG_DIRECTORY=${VIDEO_CONFIG_DIRECTORY:-"/opt/bin"} +UPLOAD_DESTINATION_PREFIX=${UPLOAD_DESTINATION_PREFIX:-$SE_UPLOAD_DESTINATION_PREFIX} +UPLOAD_PIPE_FILE_NAME=${UPLOAD_PIPE_FILE_NAME:-"uploadpipe"} +SE_VIDEO_INTERNAL_UPLOAD=${SE_VIDEO_INTERNAL_UPLOAD:-"false"} +SE_SERVER_PROTOCOL=${SE_SERVER_PROTOCOL:-"http"} +SE_NODE_PORT=${SE_NODE_PORT:-"5555"} -return_code=1 -max_attempts=50 -attempts=0 -echo 'Checking if the display is open...' -until [ $return_code -eq 0 -o $attempts -eq $max_attempts ]; do - xset -display ${DISPLAY_CONTAINER_NAME}:${DISPLAY_NUM} b off > /dev/null 2>&1 - return_code=$? - if [ $return_code -ne 0 ]; then - echo 'Waiting before next display check...' - sleep 0.5 - fi - attempts=$((attempts+1)) -done - -# exec replaces the video.sh process with ffmpeg, this makes easier to pass the process termination signal -exec ffmpeg -y -f x11grab -video_size ${VIDEO_SIZE} -r ${FRAME_RATE} -i ${DISPLAY_CONTAINER_NAME}:${DISPLAY_NUM}.0 -codec:v ${CODEC} ${PRESET} -pix_fmt yuv420p "$VIDEO_FOLDER/$FILE_NAME" +if [ "${SE_VIDEO_INTERNAL_UPLOAD}" = "true" ]; +then + # If using RCLONE in the same container, write signal to /tmp internally + UPLOAD_PIPE_FILE="/tmp/${UPLOAD_PIPE_FILE_NAME}" + FORCE_EXIT_FILE="/tmp/force_exit" +else + # If using external container for uploading, write signal to the video folder + UPLOAD_PIPE_FILE="${SE_VIDEO_FOLDER}/${UPLOAD_PIPE_FILE_NAME}" + FORCE_EXIT_FILE="${SE_VIDEO_FOLDER}/force_exit" +fi +function create_pipe() { + if [[ "${VIDEO_UPLOAD_ENABLED}" != "false" ]] && [[ -n "${UPLOAD_DESTINATION_PREFIX}" ]]; + then + echo "Create pipe if not exists for video upload stream" + if [[ ! -p ${UPLOAD_PIPE_FILE} ]]; + then + mkfifo ${UPLOAD_PIPE_FILE} + fi + fi +} + +function wait_util_uploader_shutdown() { + max_wait=5 + wait=0 + if [[ "${VIDEO_UPLOAD_ENABLED}" != "false" ]] && [[ -n "${UPLOAD_DESTINATION_PREFIX}" ]]; + then + while [[ -f ${FORCE_EXIT_FILE} ]] && [[ ${wait} -lt ${max_wait} ]]; + do + echo "Waiting for force exit file to be consumed by external upload container" + sleep 1 + wait=$((wait+1)) + done + fi + if [[ "${VIDEO_UPLOAD_ENABLED}" != "false" ]] && [[ -n "${UPLOAD_DESTINATION_PREFIX}" ]] && [[ "${SE_VIDEO_INTERNAL_UPLOAD}" = "true" ]]; + then + while [[ $(pgrep rclone | wc -l) -gt 0 ]] + do + echo "Recorder is waiting for RCLONE to finish" + sleep 1 + done + fi + echo "Ready to shutdown the recorder" +} + +function send_exit_signal_to_uploader() { + if [[ "${VIDEO_UPLOAD_ENABLED}" != "false" ]] && [[ -n "${UPLOAD_DESTINATION_PREFIX}" ]]; + then + echo "exit" >> ${UPLOAD_PIPE_FILE} & + echo "exit" > ${FORCE_EXIT_FILE} + fi +} + +function exit_on_max_session_reach() { + if [[ $max_recorded_count -gt 0 ]] && [[ $recorded_count -ge $max_recorded_count ]]; + then + echo "Node will be drained since max sessions reached count number ($max_recorded_count)" + exit + fi +} + +function stop_recording() { + echo "Stopping to record video" + pkill -INT ffmpeg + recorded_count=$((recorded_count+1)) + recording_started="false" + if [[ "${VIDEO_UPLOAD_ENABLED}" != "false" ]] && [[ -n "${UPLOAD_DESTINATION_PREFIX}" ]]; + then + upload_destination=${UPLOAD_DESTINATION_PREFIX}/${video_file_name} + echo "Add to pipe a signal Uploading video to $upload_destination" + echo $video_file ${UPLOAD_DESTINATION_PREFIX} >> ${UPLOAD_PIPE_FILE} & + elif [[ "${VIDEO_UPLOAD_ENABLED}" != "false" ]] && [[ -z "${UPLOAD_DESTINATION_PREFIX}" ]]; + then + echo Upload destination not known since UPLOAD_DESTINATION_PREFIX is not set. Continue without uploading. + fi +} + +function check_if_recording_inprogress() { + if [[ "$recording_started" = "true" ]] + then + stop_recording + fi +} + +function graceful_exit() { + check_if_recording_inprogress + send_exit_signal_to_uploader + wait_util_uploader_shutdown + kill -INT "$(cat /var/run/supervisor/supervisord.pid)" +} + +if [[ "${VIDEO_UPLOAD_ENABLED}" != "true" ]] && [[ "${VIDEO_FILE_NAME}" != "auto" ]] && [[ -n "${VIDEO_FILE_NAME}" ]]; then + return_code=1 + max_attempts=50 + attempts=0 + echo 'Checking if the display is open...' + until [[ $return_code -eq 0 ]] || [[ $attempts -eq $max_attempts ]]; do + xset -display ${DISPLAY_CONTAINER_NAME}:${DISPLAY_NUM} b off > /dev/null 2>&1 + return_code=$? + if [[ $return_code -ne 0 ]]; then + echo 'Waiting before next display check...' + sleep 0.5 + fi + attempts=$((attempts+1)) + done + + # exec replaces the video.sh process with ffmpeg, this makes easier to pass the process termination signal + exec ffmpeg -y -f x11grab -video_size ${VIDEO_SIZE} -r ${FRAME_RATE} -i ${DISPLAY_CONTAINER_NAME}:${DISPLAY_NUM}.0 -codec:v ${CODEC} ${PRESET} -pix_fmt yuv420p "$VIDEO_FOLDER/$VIDEO_FILE_NAME" + +else + create_pipe + trap graceful_exit SIGTERM SIGINT EXIT + export DISPLAY=${DISPLAY_CONTAINER_NAME}:${DISPLAY_NUM}.0 + + max_attempts=600 + attempts=0 + + echo Checking if the display is open + until xset b off || [[ $attempts = "$max_attempts" ]] + do + echo Waiting before next display check + sleep 0.5 + attempts=$((attempts+1)) + done + if [[ $attempts = "$max_attempts" ]] + then + echo Can not open display, exiting. + exit + fi + + VIDEO_SIZE=$(xdpyinfo | grep 'dimensions:' | awk '{print $2}') + + recording_started="false" + video_file_name="" + video_file="" + prev_session_id="" + attempts=0 + max_recorded_count=${SE_DRAIN_AFTER_SESSION_COUNT:-0} + recorded_count=0 + + echo Checking if node API responds + until curl -sk --request GET ${SE_SERVER_PROTOCOL}://${DISPLAY_CONTAINER_NAME}:${SE_NODE_PORT}/status || [[ $attempts = "$max_attempts" ]] + do + if [ $(($attempts % 60)) -eq 0 ]; + then + echo Waiting before next API check + fi + sleep 0.5 + attempts=$((attempts+1)) + done + if [[ $attempts = "$max_attempts" ]] + then + echo Can not reach node API, exiting. + exit + fi + while curl -sk --request GET ${SE_SERVER_PROTOCOL}://${DISPLAY_CONTAINER_NAME}:${SE_NODE_PORT}/status > /tmp/status.json + do + session_id=$(jq -r '.[]?.node?.slots | .[0]?.session?.sessionId' /tmp/status.json) + if [[ "$session_id" != "null" && "$session_id" != "" && "$recording_started" = "false" ]] + then + echo "Session: $session_id is created" + caps_se_video_record=$(bash ${VIDEO_CONFIG_DIRECTORY}/video_graphQLQuery.sh "$session_id") + if [[ -f "/tmp/graphQL_$session_id.json" ]]; then + cat "/tmp/graphQL_$session_id.json"; echo + fi + echo "se:recordVideo value is: $caps_se_video_record" + fi + if [[ "$session_id" != "null" && "$session_id" != "" && "$recording_started" = "false" && "$caps_se_video_record" = "true" ]] + then + video_file_name="$session_id.mp4" + video_file="${SE_VIDEO_FOLDER}/$video_file_name" + echo "Starting to record video" + ffmpeg -nostdin -y -f x11grab -video_size ${VIDEO_SIZE} -r ${FRAME_RATE} -i ${DISPLAY} -codec:v ${CODEC} ${PRESET} -pix_fmt yuv420p $video_file & + recording_started="true" + echo "Video recording started" + elif [[ "$session_id" != "$prev_session_id" && "$recording_started" = "true" ]] + then + stop_recording + if [[ $max_recorded_count -gt 0 ]] && [[ $recorded_count -ge $max_recorded_count ]]; + then + echo "Node will be drained since max sessions reached count number ($max_recorded_count)" + exit + fi + elif [[ $recording_started = "true" ]] + then + echo "Video recording in progress " + sleep 1 + else + sleep 1 + fi + prev_session_id=$session_id + done + echo "Node API is not responding, exiting." + exit +fi diff --git a/Video/video_graphQLQuery.sh b/Video/video_graphQLQuery.sh new file mode 100755 index 000000000..54ed011d1 --- /dev/null +++ b/Video/video_graphQLQuery.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +# Define parameters +SESSION_ID=$1 +GRAPHQL_ENDPOINT=${2:-$SE_NODE_GRID_GRAPHQL_URL} +VIDEO_CAP_NAME=${VIDEO_CAP_NAME:-"se:recordVideo"} + +if [ -z "${GRAPHQL_ENDPOINT}" ] && [ -n "${SE_NODE_GRID_URL}" ]; then + GRAPHQL_ENDPOINT="${SE_NODE_GRID_URL}/graphql" +fi + +if [ -n "${GRAPHQL_ENDPOINT}" ]; then + # Send GraphQL query + curl --retry 3 -k -X POST \ + -H "Content-Type: application/json" \ + --data '{"query":"{ session (id: \"'${SESSION_ID}'\") { id, capabilities, startTime, uri, nodeId, nodeUri, sessionDurationMillis, slot { id, stereotype, lastStarted } } } "}' \ + -s "${GRAPHQL_ENDPOINT}" > /tmp/graphQL_${SESSION_ID}.json + + RECORD_VIDEO=$(jq -r '.data.session.capabilities | fromjson | ."'${VIDEO_CAP_NAME}'"' /tmp/graphQL_${SESSION_ID}.json) +fi + +if [ "${RECORD_VIDEO,,}" = "false" ]; then + echo "${RECORD_VIDEO,,}" +else + echo true +fi diff --git a/Video/video_ready.py b/Video/video_ready.py old mode 100644 new mode 100755 diff --git a/charts/selenium-grid/CHANGELOG.md b/charts/selenium-grid/CHANGELOG.md index 01678d5ca..9f5041fc0 100644 --- a/charts/selenium-grid/CHANGELOG.md +++ b/charts/selenium-grid/CHANGELOG.md @@ -1,9 +1,35 @@ +## :heavy_check_mark: selenium-grid-0.28.0 + +- Chart is using image tag 4.18.0-20240220 +- Chart is tested on Kubernetes versions: v1.25.16 v1.26.14 v1.27.11 v1.28.7 v1.29.2 + +### Added +- feat: enable tracing observability in docker-compose and helm chart (#2137) :: Viet Nguyen Duc +- feat: video upload supports both docker-compose and helm chart (#2131) :: Viet Nguyen Duc +- feat(chart): set components host & port point to its service :: Viet Nguyen Duc +- feat: non-root user for video recorder (#2122) :: Viet Nguyen Duc +- feat(chart): Log Node preStop exec to console :: Viet Nguyen Duc +- feat(chart): delete file after upload (#2117) :: Doofus100500 + +### Changed +- Update tag in docs and files :: Selenium CI Bot +- Release 4.18.0 :: Viet Nguyen Duc +- test(chart): CI tests run against different Kubernetes version :: Viet Nguyen Duc +- update(tracing): Use OTLP exporter instead of Jaeger specific :: Viet Nguyen Duc +- update(chart): Node preStop and startupProbe in autoscaling Deployment (#2139) :: Viet Nguyen Duc +- update(chart): objects name convention with prefix is chart RELEASENAME (#2134) :: Viet Nguyen Duc +- [🚀 Feature]: Update objects name convention with prefix is Chart RELEASENAME #2109 (#2120) :: Bas M +- update(chart): Make var RECORD_VIDEO lowercase before comparison (#2128) :: Doofus100500 +- test(chart): parallel with autoscalingType deployment & job :: Viet Nguyen Duc +- docs(chart): point shielding in README (#2116) :: Doofus100500 +- Update chart CHANGELOG [skip ci] :: Selenium CI Bot + ## :heavy_check_mark: selenium-grid-0.27.0 - Chart is using image tag 4.17.0-20240123 ### Added -- feat(chart): templating in name(Override) for new object [deploy] Signed-off-by: Viet Nguyen Duc :: Viet Nguyen Duc +- feat(chart): templating in name(Override) for new object [deploy] :: Viet Nguyen Duc - feat(chart): templating in name(Override) (#2107) :: Bas M - feat(chart): Add node registration secret to exec preStop container Update default scalingStrategy.strategy: accurate [skip ci] :: Viet Nguyen Duc - feat(chart): Configuration extra scripts mount to container (#2105) :: Viet Nguyen Duc diff --git a/charts/selenium-grid/Chart.yaml b/charts/selenium-grid/Chart.yaml index a6cfa81bf..7cb33bf5d 100644 --- a/charts/selenium-grid/Chart.yaml +++ b/charts/selenium-grid/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: selenium-grid description: A Helm chart for creating a Selenium Grid Server in Kubernetes type: application -version: 0.27.0 -appVersion: 4.17.0-20240123 +version: 0.28.1 +appVersion: 4.18.0-20240220 icon: https://github.com/SeleniumHQ/docker-selenium/raw/trunk/logo.png dependencies: - repository: https://kedacore.github.io/charts @@ -14,6 +14,10 @@ dependencies: version: 4.9.0 name: ingress-nginx condition: ingress-nginx.enabled +- repository: https://jaegertracing.github.io/helm-charts + version: 1.0.0 + name: jaeger + condition: tracing.enabled maintainers: - name: SeleniumHQ email: selenium-developers@googlegroups.com diff --git a/charts/selenium-grid/README.md b/charts/selenium-grid/README.md index 0902e7107..03de22cef 100644 --- a/charts/selenium-grid/README.md +++ b/charts/selenium-grid/README.md @@ -18,7 +18,7 @@ This chart enables the creation of a Selenium Grid Server in Kubernetes. * [Ingress Configuration](#ingress-configuration) * [Configuration](#configuration) * [Configuration global](#configuration-global) - * [Configuration `global.K8S_PUBLIC_IP`](#configuration-globalk8spublicip) + * [Configuration `global.K8S_PUBLIC_IP`](#configuration-globalk8s_public_ip) * [Configuration of Nodes](#configuration-of-nodes) * [Container ports and Service ports](#container-ports-and-service-ports) * [Probes](#probes) @@ -29,9 +29,11 @@ This chart enables the creation of a Selenium Grid Server in Kubernetes. * [Configuration of Secure Communication (HTTPS)](#configuration-of-secure-communication-https) * [Secure Communication](#secure-communication) * [Node Registration](#node-registration) + * [Configuration of tracing observability](#configuration-of-tracing-observability) * [Configuration of Selenium Grid chart](#configuration-of-selenium-grid-chart) * [Configuration of KEDA](#configuration-of-keda) * [Configuration of Ingress NGINX Controller](#configuration-of-ingress-nginx-controller) + * [Configuration of Jaeger](#configuration-of-jaeger) * [Configuration for Selenium-Hub](#configuration-for-selenium-hub) * [Configuration for isolated components](#configuration-for-isolated-components) @@ -69,7 +71,9 @@ helm install selenium-grid docker-selenium/selenium-grid --version # In both cases grid exposed by default using ingress. You may want to set hostname for the grid. Default hostname is selenium-grid.local. helm install selenium-grid --set ingress.hostname=selenium-grid.k8s.local docker-selenium/chart/selenium-grid/. # Verify ingress configuration via kubectl get ingress + # Notes: In case you want to set hostname is selenium-grid.local. You need to add the IP and hostname to the local host file in `/etc/hosts` +sudo -- sh -c -e "echo \"$(hostname -i) selenium-grid.local\" >> /etc/hosts" ``` ### Installing the Nightly chart @@ -204,7 +208,7 @@ helm uninstall selenium-grid By default, ingress is enabled without annotations set. If NGINX ingress controller is used, you need to set few annotations to override the default timeout values to avoid 504 errors (see [#1808](https://github.com/SeleniumHQ/docker-selenium/issues/1808)). Since in Selenium Grid the default of `SE_NODE_SESSION_TIMEOUT` and `SE_SESSION_REQUEST_TIMEOUT` is `300` seconds. -In order to make user experience better, there are few annotations will be set by default if NGINX ingress controller is used. Mostly relates to timeouts and buffer sizes. +To make the user experience better, there are few annotations will be set by default if NGINX ingress controller is used. Mostly relates to timeouts and buffer sizes. If you are not using NGINX ingress controller, you can disable these default annotations by setting `ingress.nginx` to `nil` (aka null) via Helm CLI `--set ingress.nginx=null`) or via an override-values.yaml as below: @@ -248,7 +252,7 @@ nginx.ingress.kubernetes.io/client-body-buffer-size nginx.ingress.kubernetes.io/proxy-buffers-number ``` -You can generate a dummy self-signed certificate specify for your `hostname`, assign it to spec `ingress.tls` and NGINX ingress controller default certificate (if it is enabled inline). For example: +You can generate a test double self-signed certificate specify for your `hostname`, assign it to spec `ingress.tls` and NGINX ingress controller default certificate (if it is enabled inline). For example: ```yaml tls: @@ -270,17 +274,18 @@ ingress-nginx: ### Configuration global For now, global configuration supported is: -| Parameter | Default | Description | -|---------------------------------------|-----------------------|----------------------------------------| -| `global.K8S_PUBLIC_IP` | `""` | Public IP of the host running K8s | -| `global.seleniumGrid.imageRegistry` | `selenium` | Distribution registry to pull images | -| `global.seleniumGrid.imageTag` | `4.17.0-20240123` | Image tag for all selenium components | -| `global.seleniumGrid.nodesImageTag` | `4.17.0-20240123` | Image tag for browser's nodes | -| `global.seleniumGrid.videoImageTag` | `ffmpeg-6.1-20240123` | Image tag for browser's video recorder | -| `global.seleniumGrid.imagePullSecret` | `""` | Pull secret to be used for all images | -| `global.seleniumGrid.imagePullSecret` | `""` | Pull secret to be used for all images | -| `global.seleniumGrid.affinity` | `{}` | Affinity assigned globally | -| `global.seleniumGrid.logLevel` | `INFO` | Set log level for all components | +| Parameter | Default | Description | +|-----------------------------------------------|-----------------------|----------------------------------------| +| `global.K8S_PUBLIC_IP` | `""` | Public IP of the host running K8s | +| `global.seleniumGrid.imageRegistry` | `selenium` | Distribution registry to pull images | +| `global.seleniumGrid.imageTag` | `4.18.0-20240220` | Image tag for all selenium components | +| `global.seleniumGrid.nodesImageTag` | `4.18.0-20240220` | Image tag for browser's nodes | +| `global.seleniumGrid.videoImageTag` | `ffmpeg-6.1-20240220` | Image tag for browser's video recorder | +| `global.seleniumGrid.imagePullSecret` | `""` | Pull secret to be used for all images | +| `global.seleniumGrid.imagePullSecret` | `""` | Pull secret to be used for all images | +| `global.seleniumGrid.affinity` | `{}` | Affinity assigned globally | +| `global.seleniumGrid.logLevel` | `INFO` | Set log level for all components | +| `global.seleniumGrid.defaultNodeStartupProbe` | `exec` | Default startup probe method in Nodes | #### Configuration `global.K8S_PUBLIC_IP` @@ -338,7 +343,7 @@ chromeNode: targetPort: 7900 # NodePort will be assigned randomly if not set edgeNode: - ports: # You also can give object following manifest of container ports + ports: # You also can give objects following manifest of container ports - containerPort: 5900 name: vnc protocol: TCP @@ -366,7 +371,7 @@ Other settings of probe support to override under `.startupProbe` `.readinessPro successThreshold ``` -You can easily configure the probes (as Kubernetes [supports](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/)) to override the default settings. For example: +You can configure the probes (as Kubernetes [supports](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/)) to override the default settings. For example: ```yaml edgeNode: @@ -392,23 +397,23 @@ nodeConfigMap: echo "Your custom script" recorderConfigMap: - extraScriptsDirectory: "/opt/selenium" + extraScriptsDirectory: "/opt/bin" extraScripts: video.sh: | #!/bin/bash echo "Your custom script" - videoPreStop.sh: | + video_graphQLQuery.sh: | #!/bin/bash echo "My new script" uploaderConfigMap: extraScriptsDirectory: "/opt/bin" extraScripts: - entry_point.sh: | + upload.sh: | #!/bin/bash echo "Your custom entry point" secretFiles: - config.conf: | + upload.conf: | [myremote] type = s3 ``` @@ -429,7 +434,7 @@ Files in `.extraScripts` will be mounted to the container with the same name wit #### Video recorder -The video recorder is a sidecar that is deployed with the browser nodes. It is responsible for recording the video of the browser session. The video recorder is disabled by default. To enable it, you need to set the following values: +The video recorder is a sidecar deployed with the browser nodes. It is responsible for recording the video of the browser session. The video recorder is disabled by default. To enable it, you need to set the following values: ```yaml videoRecorder: @@ -445,14 +450,13 @@ from selenium import webdriver options = ChromeOptions() options.set_capability('se:recordVideo', False) driver = webdriver.Remote(options=options, command_executor="http://localhost:4444") -) ``` -In Node will perform query GraphQL and extract the value of `se:recordVideo` in capabilities before deciding to start video recording process or not. By default, the script is loaded from file [configs/recorder/graphQLRecordVideo.sh](configs/recorder/graphQLRecordVideo.sh) to the ConfigMap. You can customize by reading on section [Configuration extra scripts mount to container](#configuration-extra-scripts-mount-to-container). +In Node will perform query GraphQL in Hub based on Node SessionId and extract the value of `se:recordVideo` in capabilities before deciding to start video recording process or not. You can customize by reading on section [Configuration extra scripts mount to container](#configuration-extra-scripts-mount-to-container). #### Video uploader -The uploader is a sidecar that is deployed with the browser nodes. It is responsible for uploading the video to a remote location. The uploader is disabled by default. To enable it, you need to set the following values: +The uploader is extra utility in the video container. It is responsible for uploading the video to a remote location. The uploader is disabled by default. To enable it, you need to set the following values: ```yaml videoRecorder: @@ -464,14 +468,14 @@ By default, the uploader uses [RCLONE](https://rclone.org/) to upload the video The uploader requires `destinationPrefix` to be set. It is used to instruct the uploader where to upload the video. The format of destinationPrefix is `remote-name://bucket-name/path`. The `remote-name` is configured in RCLONE. The `bucket-name` is the name of the bucket in the remote location. The `path` is the path to the folder in the bucket. -By default, the config file is loaded from file [configs/uploader/rclone/config.conf](configs/uploader/rclone/config.conf) to the Secret. You can override the config file via `--set-file uploaderConfigMap.secretFiles.config\.conf=/path/to/your_config.conf` or set via YAML values. +By default, the config file is empty. You can override the config file via `--set-file uploaderConfigMap.secretFiles.upload\.conf=/path/to/your_config.conf` or set via YAML values. For example, to configure an S3 remote hosted on AWS with named `mys3` and the bucket name is `mybucket`, you can set the following values: ```bash uploaderConfigMap: secretFiles: - config.conf: | + upload.conf: | [mys3] type = s3 provider = AWS @@ -488,7 +492,6 @@ videoRecorder: ``` You can prepare a config file with multiple remotes are defined. Ensure that `[remoteName]` is unique for each remote. -You also can replace your config to default file `configs/uploader/rclone/config.conf` in chart. Instead of using config file, another way that RCLONE also supports to pass the information via environment variables. ENV variable with format: `RCLONE_CONFIG_ + name of remote + _ + name of config file option` (make it all uppercase). In this case the remote name it can only contain letters, digits, or the _ (underscore) character. All those ENV variables can be set via `videoRecorder.uploader.secrets`, it will be stored in Secret. @@ -507,21 +510,46 @@ videoRecorder: RCLONE_CONFIG_MYS3_ACL: "private" RCLONE_CONFIG_MYS3_ACCESS_KEY_ID: "xxx" RCLONE_CONFIG_MYS3_SECRET_ACCESS_KEY: "xxx" + RCLONE_CONFIG_MYS3_ENDPOINT: "https://storage.googleapis.com" ``` -Those 2 ways are equivalent. You can choose one of them or combine them together. When both config file and ENV vars are set, value in `config.conf` will take precedence. +Those two ways are equivalent. You can choose one of them or combine them. When both config file and ENV vars are set, value in `upload.conf` will take precedence. -Beside the configuration, the script for entry point of uploader container also needed. By default, it is loaded from file [configs/uploader/rclone/entry_point.sh](configs/uploader/rclone/entry_point.sh) to the ConfigMap. You can override the script via `--set-file uploaderConfigMap.extraScripts.entry_point\.sh=/path/to/your_script.sh` or set via YAML values. For example: +Besides the configuration, the script for entry point of uploader container also needed. You can override the script via `--set-file uploaderConfigMap.extraScripts.upload\.sh=/path/to/your_script.sh` or set via YAML values. For example: ```yaml uploaderConfigMap: extraScripts: - entry_point.sh: | + upload.sh: | #!/bin/bash echo "Your custom entry point" ``` -You also can replace your script to default file `configs/uploader/rclone/entry_point.sh` in chart. +In case you want to configure another sidecar container for uploader, you can set a name for `videoRecorder.uploader.name` and create a config key with the same name under `videoRecorder` with all the settings for your container. Set name of `videoRecorder.uploader.entryPointFileName` if your container start by a different entry point. For example: + +```yaml +uploaderConfigMap: + extraScripts: + upload.sh: | + #!/bin/bash + echo "Script control the uploader process" + +videoRecorder: + enabled: true + uploader: + enabled: true + name: "s3" + entryPointFileName: "upload.sh" + destinationPrefix: "s3://mybucket" + secrets: + AWS_REGION: "ap-southeast-1" + AWS_ACCESS_KEY_ID: "xxxx" + AWS_SECRET_ACCESS_KEY: "xxxx" + s3: + imageRegistry: public.ecr.aws + imageName: bitnami/aws-cli + imageTag: latest +``` ### Configuration of Secure Communication (HTTPS) @@ -531,7 +559,7 @@ Selenium Grid supports secure communication between components. Refer to the [in In the chart, there is directory [certs](./certs) contains the default certificate, private key (as PKCS8 format), and Java Keystore (JKS) to teach Java about secure connection (since we are using a non-standard CA) for your trial, local testing purpose. You can generate your own self-signed certificate put them in that default directory by using script [cert.sh](./certs/cert.sh) with adjust needed information. The certificate, private key, truststore are mounted to the components via `Secret`. -There are multiple ways to configure your certificate, private key, truststore to the components. You can choose one of them or combine them together. +There are multiple ways to configure your certificate, private key, truststore to the components. You can choose one of them or combine them. - Use the default directory [certs](./certs). Rename your own files to be same as the default files and replace them. Give `--set tls.enabled=true` to enable secure communication. @@ -584,7 +612,7 @@ ingress-nginx: #### Node Registration -In order to enable secure in the node registration to make sure that the node is one you control and not a rouge node, you can enable and provide a registration secret string to Distributor, Router and +To enable secure in the node registration to make sure that the node is one you control and not a rouge node, you can enable and provide a registration secret string to Distributor, Router and Node servers in config `tls.registrationSecret`. For example: ```yaml @@ -595,6 +623,28 @@ tls: value: "matchThisSecret" ``` +### Configuration of tracing observability + +The chart supports tracing observability via Jaeger. To enable it, you need to set the following values: + +```yaml +tracing: + enabled: true +``` + +With this configuration, by default, Jaeger (all-in-one) will be deployed in the same namespace as Selenium Grid. +The Jaeger UI can be accessed via same ingress with prefix `/jaeger`, for example: `http://your.host.name/jaeger`. +The traces will be collected from all the components of Selenium Grid and can be viewed in the Jaeger UI. + +In case you want to use your own existing Jaeger instance, you can set the following values: + +```yaml +tracing: + enabledWithExistingEndpoint: true + exporter: otlp + exporterEndpoint: 'http://jaeger.domain.com:4317' +``` + ### Configuration of Selenium Grid chart This table contains the configuration parameters of the chart and their default values: @@ -604,12 +654,12 @@ This table contains the configuration parameters of the chart and their default | `basicAuth.username` | `admin` | Username of basic auth for Selenium Grid | | `basicAuth.password` | `admin` | Password of basic auth for Selenium Grid | | `isolateComponents` | `false` | Deploy Router, Distributor, EventBus, SessionMap and Nodes separately | -| `serviceAccount.create` | `true` | Enable or disable creation of service account (if `false`, `serviceAccount.name` MUST be specified | -| `serviceAccount.name` | `""` | Name of the service account to be made or existing service account to use for all deployments and jobs | +| `serviceAccount.create` | `true` | Enable or disable creation of service account (if `false`, `serviceAccount.nameOverride` MUST be specified | +| `serviceAccount.nameOverride` | `""` | Name of another service account to be made or existing service account to use for all deployments and jobs | | `serviceAccount.annotations` | `{}` | Custom annotations for service account | -| `busConfigMap.name` | `selenium-event-bus-config` | Name of the configmap that contains SE_EVENT_BUS_HOST, SE_EVENT_BUS_PUBLISH_PORT and SE_EVENT_BUS_SUBSCRIBE_PORT variables | +| `busConfigMap.nameOverride` | `` | Override another configmap that contains SE_EVENT_BUS_HOST, SE_EVENT_BUS_PUBLISH_PORT and SE_EVENT_BUS_SUBSCRIBE_PORT vars | | `busConfigMap.annotations` | `{}` | Custom annotations for configmap | -| `nodeConfigMap.name` | `selenium-node-config` | Name of the configmap that contains common environment variables for browser nodes | +| `nodeConfigMap.nameOverride` | `` | Name of the configmap that contains common environment variables for browser nodes | | `nodeConfigMap.annotations` | `{}` | Custom annotations for configmap | | `ingress.enabled` | `true` | Enable or disable ingress resource | | `ingress.className` | `""` | Name of ingress class to select which controller will implement ingress resource | @@ -639,7 +689,7 @@ This table contains the configuration parameters of the chart and their default | `chromeNode.replicas` | `1` | Number of chrome nodes. Disabled if autoscaling is enabled. | | `chromeNode.imageRegistry` | `nil` | Distribution registry to pull the image (this overwrites `.global.seleniumGrid.imageRegistry` value) | | `chromeNode.imageName` | `node-chrome` | Image of chrome nodes | -| `chromeNode.imageTag` | `4.17.0-20240123` | Image of chrome nodes | +| `chromeNode.imageTag` | `4.18.0-20240220` | Image of chrome nodes | | `chromeNode.imagePullPolicy` | `IfNotPresent` | Image pull policy (see https://kubernetes.io/docs/concepts/containers/images/#updating-images) | | `chromeNode.imagePullSecret` | `""` | Image pull secret (see https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry) | | `chromeNode.ports` | `[]` | Extra ports list to enable on container (e.g VNC, NoVNC, SSH if any) | @@ -682,7 +732,7 @@ This table contains the configuration parameters of the chart and their default | `firefoxNode.replicas` | `1` | Number of firefox nodes. Disabled if autoscaling is enabled. | | `firefoxNode.imageRegistry` | `nil` | Distribution registry to pull the image (this overwrites `.global.seleniumGrid.imageRegistry` value) | | `firefoxNode.imageName` | `node-firefox` | Image of firefox nodes | -| `firefoxNode.imageTag` | `4.17.0-20240123` | Image of firefox nodes | +| `firefoxNode.imageTag` | `4.18.0-20240220` | Image of firefox nodes | | `firefoxNode.imagePullPolicy` | `IfNotPresent` | Image pull policy (see https://kubernetes.io/docs/concepts/containers/images/#updating-images) | | `firefoxNode.imagePullSecret` | `""` | Image pull secret (see https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry) | | `firefoxNode.ports` | `[]` | Extra ports list to enable on container (e.g VNC, NoVNC, SSH if any) | @@ -725,7 +775,7 @@ This table contains the configuration parameters of the chart and their default | `edgeNode.replicas` | `1` | Number of edge nodes. Disabled if autoscaling is enabled. | | `edgeNode.imageRegistry` | `nil` | Distribution registry to pull the image (this overwrites `.global.seleniumGrid.imageRegistry` value) | | `edgeNode.imageName` | `node-edge` | Image of edge nodes | -| `edgeNode.imageTag` | `4.17.0-20240123` | Image of edge nodes | +| `edgeNode.imageTag` | `4.18.0-20240220` | Image of edge nodes | | `edgeNode.imagePullPolicy` | `IfNotPresent` | Image pull policy (see https://kubernetes.io/docs/concepts/containers/images/#updating-images) | | `edgeNode.imagePullSecret` | `""` | Image pull secret (see https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry) | | `edgeNode.ports` | `[]` | Extra ports list to enable on container (e.g VNC, NoVNC, SSH if any) | @@ -766,11 +816,11 @@ This table contains the configuration parameters of the chart and their default | `videoRecorder.enabled` | `false` | Enable video recorder for node | | `videoRecorder.imageRegistry` | `nil` | Distribution registry to pull the image (this overwrites `.global.seleniumGrid.imageRegistry` value) | | `videoRecorder.imageName` | `video` | Selenium video recorder image name | -| `videoRecorder.imageTag` | `ffmpeg-6.1-20240123` | Image tag of video recorder | +| `videoRecorder.imageTag` | `ffmpeg-6.1-20240220` | Image tag of video recorder | | `videoRecorder.imagePullPolicy` | `IfNotPresent` | Image pull policy (see https://kubernetes.io/docs/concepts/containers/images/#updating-images) | | `videoRecorder.uploader.enabled` | `false` | Enable the uploader for videos | | `videoRecorder.uploader.destinationPrefix` | `` | Destination for uploading video file. It is following `rclone` config | -| `videoRecorder.uploader.name` | `rclone` | Name of the uploader to use. Supported default `rclone` | +| `videoRecorder.uploader.name` | `` | Name of the pluggable uploader container to add and configure | | `videoRecorder.uploader.configFileName` | `config.conf` | Config file name for `rclone` in uploader container | | `videoRecorder.uploader.entryPointFileName` | `entry_point.sh` | Script file name for uploader container entry point | | `videoRecorder.uploader.config` | `` | Set value to uploader config file via YAML or `--set-file` | @@ -784,14 +834,8 @@ This table contains the configuration parameters of the chart and their default | `videoRecorder.startupProbe` | `{}` | Probe to check pod is started successfully | | `videoRecorder.livenessProbe` | `{}` | Liveness probe settings | | `videoRecorder.lifecycle` | `{}` | Define lifecycle events for video recorder | -| `videoRecorder.volume.name.folder` | `video` | Name is used to set for the volume to persist and share output video folder in container | -| `videoRecorder.volume.name.scripts` | `video-scripts` | Name is used to set for the volume to persist and share video recorder scripts in container | | `videoRecorder.extraVolumeMounts` | `[]` | Extra mounts of declared ExtraVolumes into pod | | `videoRecorder.extraVolumes` | `[]` | Extra Volumes declarations to be used in the pod (can be any supported volume type: ConfigMap, Secret, PVC, NFS, etc.) | -| `videoRecorder.rclone` | `See values.yaml` | Container spec for the uploader if `videoRecorder.uploader` is `s3`. Similarly, create for your new uploader | -| `videoRecorder.rclone.resources | `See values.yaml` | Resources for video uploader | -| `videoRecorder.rclone.extraEnvFrom` | `` | Custom environment taken from `configMap` or `secret` variables for video uploader | -| `videoRecorder.rclone.extraVolumeMounts` | `[]` | Extra mounts of declared ExtraVolumes into pod of video uploader | | `customLabels` | `{}` | Custom labels for k8s resources | | `ingress-nginx.enabled` | `false` | Enable the dependency chart Ingress controller for Kubernetes (https://github.com/kubernetes/ingress-nginx) | @@ -808,6 +852,11 @@ https://github.com/kedacore/charts/blob/main/keda/README.md for more details. If you are setting `ingress-nginx.enabled` to `true`, chart Ingress NGINX Controller is installed and can be configured with values with the prefix `ingress-nginx`. See https://github.com/kubernetes/ingress-nginx for more details. +### Configuration of Jaeger + +If you are setting `tracing.enabled` to `true`, chart Jaeger is installed and can be configured with +values with the prefix `jaeger`. See https://github.com/jaegertracing/helm-charts for more details. + ### Configuration for Selenium-Hub You can configure the Selenium Hub with these values: diff --git a/charts/selenium-grid/TESTING.md b/charts/selenium-grid/TESTING.md index fd29f09c1..0d4f2a5ea 100644 --- a/charts/selenium-grid/TESTING.md +++ b/charts/selenium-grid/TESTING.md @@ -7,21 +7,21 @@ All related testing to this helm chart will be documented in this file. | Features | TC Description | Coverage | Test via | |------------------------|----------------------------------------------------------------------|----------|----------| | Basic Auth | Basic Auth is disabled | ✓ | Cluster | -| | Basic Auth is enabled | ✗ | | +| | Basic Auth is enabled | ✓ | Cluster | | Auto scaling | Auto scaling with `enableWithExistingKEDA` is `true` | ✓ | Cluster | | | Auto scaling with `scalingType` is `job` | ✓ | Cluster | | | Auto scaling with `scalingType` is `deployment` | ✓ | Cluster | | | Auto scaling with `autoscaling.scaledOptions.minReplicaCount` is `0` | ✓ | Cluster | | | Parallel tests execution against node autoscaling | ✓ | Cluster | | Ingress | Ingress is enabled without `hostname` | ✓ | Cluster | -| | Ingress is enabled with `hostname` is set | ✗ | | +| | Ingress is enabled with `hostname` is set | ✓ | Cluster | | | Hub `sub-path` is set with Ingress `ImplementationSpecific` paths | ✓ | Cluster | | | `ingress.nginx` configs for NGINX ingress controller annotations | ✓ | Template | | Distributed components | `isolateComponents` is enabled | ✓ | Cluster | | | `isolateComponents` is disabled | ✓ | Cluster | | Browser Nodes | Node `nameOverride` is set | ✓ | Cluster | | | Sanity tests in node | ✓ | Cluster | -| | Video recorder is enabled in node | ✗ | | +| | Video recorder is enabled in node | ✓ | Cluster | | | Node `extraEnvironmentVariables` is set value | ✓ | Cluster | | General | Set new image registry via `global.seleniumGrid.imageRegistry` | ✓ | Cluster | | | Components are able to set `.affinity` | ✓ | Template | @@ -32,6 +32,26 @@ All related testing to this helm chart will be documented in this file. | | Extra ports can be exposed on Service via `.service.ports` | ✓ | Cluster | | | Service type change to `NodePort`, specific NodePort can be set | ✓ | Cluster | +## CI Test Traceability Matrix + +We have a CI pipeline to test the Helm chart with the following test cases. + +| Matrix | job | deployment | job_https | job_hostname | deployment_https | +|---------------------------|----------|------------|-----------|--------------|------------------| +| Cluster | Kind | Kind | Kind | Minikube | Minikube | +| Kubernetes version | v1.25.16 | v1.26.14 | v1.27.11 | v1.28.7 | v1.29.2 | +| Autoscaling | x | x | x | x | x | +| Scaling type `deployment` | | x | | | x | +| Scaling type `job` | x | | x | x | | +| Full distributed mode | x | | | | x | +| Enable basic auth | | x | | x | x | +| Ingress `hostname` | x | | | x | x | +| Ingress `K8S_PUBLIC_IP` | | x | x | | | +| HTTPS `tls.enabled` | | | x | | x | +| Enable tracing | x | x | | x | | +| Enable video recorder | x | x | x | x | x | +| Test headless | | x | x | | | + ## Test Chart Template - By using `helm template` command, the chart template is tested without installing it to Kubernetes cluster. - Templates are rendered and the output as a YAML manifest file. The manifest file is then asserted with [pyyaml](https://pyyaml.org/wiki/PyYAMLDocumentation). @@ -70,9 +90,9 @@ make chart_build make chart_cluster_setup # Test Selenium Grid on Kubernetes -make chart_test +make chart_test_autoscaling_deployment -# make chart_test_parallel_autoscaling +make chart_test_autoscaling_job # Cleanup Kubernetes cluster make chart_cluster_cleanup diff --git a/charts/selenium-grid/configs/node/nodePreStop.sh b/charts/selenium-grid/configs/node/nodePreStop.sh index 7b6397b6d..7f0e2285f 100644 --- a/charts/selenium-grid/configs/node/nodePreStop.sh +++ b/charts/selenium-grid/configs/node/nodePreStop.sh @@ -1,4 +1,11 @@ #!/bin/bash + +function on_exit() { + rm -rf /tmp/preStopOutput +} +trap on_exit EXIT + +# Set headers if Node Registration Secret is set if [ ! -z "${SE_REGISTRATION_SECRET}" ]; then HEADERS="X-REGISTRATION-SECRET: ${SE_REGISTRATION_SECRET}" @@ -6,9 +13,69 @@ else HEADERS="X-REGISTRATION-SECRET;" fi -if curl -sfk ${SE_SERVER_PROTOCOL}://127.0.0.1:${SE_NODE_PORT}/status; then +function is_full_distributed_mode() { + if [ -n "${SE_DISTRIBUTOR_HOST}" ] && [ -n "${SE_DISTRIBUTOR_PORT}" ]; then + DISTRIBUTED_MODE=true + echo "Detected full distributed mode: ${DISTRIBUTED_MODE}. Since SE_DISTRIBUTOR_HOST and SE_DISTRIBUTOR_PORT are set in Node ConfigMap" + else + DISTRIBUTED_MODE=false + echo "Detected full distributed mode: ${DISTRIBUTED_MODE}" + fi +} +is_full_distributed_mode + +function signal_distributor_to_drain_node() { + if [ "${DISTRIBUTED_MODE}" = true ]; then + echo "Signaling Distributor to drain node" + set -x + curl -k -X POST ${SE_SERVER_PROTOCOL}://${SE_DISTRIBUTOR_HOST}:${SE_DISTRIBUTOR_PORT}/se/grid/distributor/node/${NODE_ID}/drain --header "${HEADERS}" + set +x + fi +} + +function signal_hub_to_drain_node() { + if [ "${DISTRIBUTED_MODE}" = false ]; then + echo "Signaling Hub to drain node" + curl -k -X POST ${SE_GRID_URL}/se/grid/distributor/node/${NODE_ID}/drain --header "${HEADERS}" + fi +} + +function signal_node_to_drain() { + echo "Signaling Node to drain itself" curl -k -X POST ${SE_SERVER_PROTOCOL}://127.0.0.1:${SE_NODE_PORT}/se/grid/node/drain --header "${HEADERS}" - while curl -sfk ${SE_SERVER_PROTOCOL}://127.0.0.1:${SE_NODE_PORT}/status; do sleep 1; done +} + +function replace_localhost_by_service_name() { + internal="${SE_HUB_HOST:-$SE_ROUTER_HOST}:${SE_HUB_PORT:-$SE_ROUTER_PORT}" + echo "SE_NODE_GRID_URL: ${SE_NODE_GRID_URL}" + if [[ "${SE_NODE_GRID_URL}" == *"/localhost"* ]]; then + SE_GRID_URL=${SE_NODE_GRID_URL//localhost/${internal}} + elif [[ "${SE_NODE_GRID_URL}" == *"/127.0.0.1"* ]]; then + SE_GRID_URL=${SE_NODE_GRID_URL//127.0.0.1/${internal}} + elif [[ "${SE_NODE_GRID_URL}" == *"/0.0.0.0"* ]]; then + SE_GRID_URL=${SE_NODE_GRID_URL//0.0.0.0/${internal}} + else + SE_GRID_URL=${SE_NODE_GRID_URL} + fi + echo "Set SE_GRID_URL internally: ${SE_GRID_URL}" +} +replace_localhost_by_service_name + +if curl -sfk ${SE_SERVER_PROTOCOL}://127.0.0.1:${SE_NODE_PORT}/status > /tmp/preStopOutput; then + NODE_ID=$(jq -r '.value.node.nodeId' /tmp/preStopOutput) + if [ -n "${NODE_ID}" ]; then + echo "Current Node ID is: ${NODE_ID}" + signal_hub_to_drain_node + signal_distributor_to_drain_node + echo + fi + signal_node_to_drain + # Wait for the current session to be finished if any + while curl -sfk ${SE_SERVER_PROTOCOL}://127.0.0.1:${SE_NODE_PORT}/status -o /tmp/preStopOutput; + do + echo "Node preStop is waiting for current session to be finished if any. Node details: message: $(jq -r '.value.message' /tmp/preStopOutput || "unknown"), availability: $(jq -r '.value.node.availability' /tmp/preStopOutput || "unknown")" + sleep 1; + done else echo "Node is already drained. Shutting down gracefully!" fi diff --git a/charts/selenium-grid/configs/node/nodeProbe.sh b/charts/selenium-grid/configs/node/nodeProbe.sh new file mode 100644 index 000000000..13e1538de --- /dev/null +++ b/charts/selenium-grid/configs/node/nodeProbe.sh @@ -0,0 +1,53 @@ +#!/bin/bash + +function on_exit() { + rm -rf /tmp/nodeProbe${ID} + rm -rf /tmp/gridProbe${ID} +} +trap on_exit EXIT + +ID=$(echo $RANDOM) + +function replace_localhost_by_service_name() { + internal="${SE_HUB_HOST:-$SE_ROUTER_HOST}:${SE_HUB_PORT:-$SE_ROUTER_PORT}" + echo "SE_NODE_GRID_URL: ${SE_NODE_GRID_URL}" + if [[ "${SE_NODE_GRID_URL}" == *"/localhost"* ]]; then + SE_GRID_URL=${SE_NODE_GRID_URL//localhost/${internal}} + elif [[ "${SE_NODE_GRID_URL}" == *"/127.0.0.1"* ]]; then + SE_GRID_URL=${SE_NODE_GRID_URL//127.0.0.1/${internal}} + elif [[ "${SE_NODE_GRID_URL}" == *"/0.0.0.0"* ]]; then + SE_GRID_URL=${SE_NODE_GRID_URL//0.0.0.0/${internal}} + else + SE_GRID_URL=${SE_NODE_GRID_URL} + fi + echo "Set SE_GRID_URL internally: ${SE_GRID_URL}" +} +replace_localhost_by_service_name + +if curl -sfk ${SE_SERVER_PROTOCOL}://127.0.0.1:${SE_NODE_PORT}/status -o /tmp/nodeProbe${ID}; then + NODE_ID=$(jq -r '.value.node.nodeId' /tmp/nodeProbe${ID}) + NODE_STATUS=$(jq -r '.value.node.availability' /tmp/nodeProbe${ID}) + if [ -n "${NODE_ID}" ]; then + echo "Node responds the ID: ${NODE_ID} with status: ${NODE_STATUS}" + else + echo "Wait for the Node to report its status" + exit 1 + fi + + curl -sfk "${SE_GRID_URL}/status" -o /tmp/gridProbe${ID} + GRID_NODE_ID=$(jq -e ".value.nodes[].id|select(. == \"${NODE_ID}\")" /tmp/gridProbe${ID} | tr -d '"' || true) + if [ -n "${GRID_NODE_ID}" ]; then + echo "Grid responds a matched Node ID: ${GRID_NODE_ID}" + fi + + if [ "${NODE_STATUS}" = "UP" ] && [ -n "${NODE_ID}" ] && [ -n "${GRID_NODE_ID}" ] && [ "${NODE_ID}" = "${GRID_NODE_ID}" ]; then + echo "Node ID: ${NODE_ID} is found in the Grid. The registration is successful." + exit 0 + else + echo "Node ID: ${NODE_ID} is not found in the Grid. The registration could be in progress." + exit 1 + fi +else + echo "Wait for the Node to report its status" + exit 1 +fi diff --git a/charts/selenium-grid/configs/recorder/graphQLRecordVideo.sh b/charts/selenium-grid/configs/recorder/graphQLRecordVideo.sh deleted file mode 100755 index 8061f9ae3..000000000 --- a/charts/selenium-grid/configs/recorder/graphQLRecordVideo.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -# Define parameters -SESSION_ID=$1 -GRAPHQL_ENDPOINT=${2:-$SE_NODE_GRID_GRAPHQL_URL} - -# Send GraphQL query -curl --retry 3 -k -X POST \ - -H "Content-Type: application/json" \ - --data '{"query":"{ session (id: \"'${SESSION_ID}'\") { id, capabilities, startTime, uri, nodeId, nodeUri, sessionDurationMillis, slot { id, stereotype, lastStarted } } } "}' \ - -s "${GRAPHQL_ENDPOINT}" > /tmp/graphQL_$SESSION_ID.json - -RECORD_VIDEO=$(jq -r '.data.session.capabilities | fromjson | ."se:recordVideo"' /tmp/graphQL_$SESSION_ID.json) - -if [ "${RECORD_VIDEO}" = "false" ]; then - echo "${RECORD_VIDEO}" -else - echo true -fi diff --git a/charts/selenium-grid/configs/recorder/video.sh b/charts/selenium-grid/configs/recorder/video.sh deleted file mode 100755 index e4648cada..000000000 --- a/charts/selenium-grid/configs/recorder/video.sh +++ /dev/null @@ -1,143 +0,0 @@ -#!/usr/bin/env bash - -VIDEO_UPLOAD_ENABLED=${VIDEO_UPLOAD_ENABLED:-"false"} -VIDEO_CONFIG_DIRECTORY=${VIDEO_CONFIG_DIRECTORY:-"/opt/bin"} - -function create_pipe() { - if [[ "${VIDEO_UPLOAD_ENABLED}" != "false" ]] && [[ ! -z "${UPLOAD_DESTINATION_PREFIX}" ]]; - then - echo "Create pipe if not exists for video upload stream" - if [ ! -p ${SE_VIDEO_FOLDER}/uploadpipe ]; - then - mkfifo ${SE_VIDEO_FOLDER}/uploadpipe - fi - fi -} -create_pipe - -function wait_util_force_exit_consume() { - if [[ "${VIDEO_UPLOAD_ENABLED}" != "false" ]] && [[ ! -z "${UPLOAD_DESTINATION_PREFIX}" ]]; - then - while [[ -f ${SE_VIDEO_FOLDER}/force_exit ]] - do - echo "Waiting for force exit file to be consumed by uploader" - sleep 1 - done - echo "Ready to shutdown the recorder" - fi -} - -function add_exit_signal() { - if [[ "${VIDEO_UPLOAD_ENABLED}" != "false" ]] && [[ ! -z "${UPLOAD_DESTINATION_PREFIX}" ]]; - then - echo "exit" >> ${SE_VIDEO_FOLDER}/uploadpipe & - echo "exit" > ${SE_VIDEO_FOLDER}/force_exit - fi -} - -function exit_on_max_session_reach() { - if [ $max_recorded_count -gt 0 ] && [ $recorded_count -ge $max_recorded_count ]; - then - echo "Node will be drained since max sessions reached count number ($max_recorded_count)" - exit - fi -} - -function finish { - add_exit_signal - wait_util_force_exit_consume - kill -INT `cat /var/run/supervisor/supervisord.pid` -} -trap finish EXIT - -FRAME_RATE=${FRAME_RATE:-$SE_FRAME_RATE} -CODEC=${CODEC:-$SE_CODEC} -PRESET=${PRESET:-$SE_PRESET} -DISPLAY_CONTAINER_NAME=${DISPLAY_CONTAINER_NAME:-"localhost"} -export DISPLAY=${DISPLAY_CONTAINER_NAME}:${DISPLAY_NUM}.0 - -max_attempts=600 -attempts=0 -echo Checking if the display is open -until xset b off || [[ $attempts = $max_attempts ]] -do - echo Waiting before next display check - sleep 0.5 - attempts=$((attempts+1)) -done -if [[ $attempts = $max_attempts ]] -then - echo Can not open display, exiting. - exit -fi -VIDEO_SIZE=$(xdpyinfo | grep 'dimensions:' | awk '{print $2}') - -recording_started="false" -video_file_name="" -video_file="" -prev_session_id="" -attempts=0 -max_recorded_count=${SE_DRAIN_AFTER_SESSION_COUNT:-0} -recorded_count=0 -echo Checking if node API responds -until curl -sk --request GET ${SE_SERVER_PROTOCOL}://${DISPLAY_CONTAINER_NAME}:${SE_NODE_PORT}/status || [[ $attempts = $max_attempts ]] -do - echo Waiting before next API check - sleep 0.5 - attempts=$((attempts+1)) -done -if [[ $attempts = $max_attempts ]] -then - echo Can not reach node API, exiting. - exit -fi -while curl -sk --request GET ${SE_SERVER_PROTOCOL}://${DISPLAY_CONTAINER_NAME}:${SE_NODE_PORT}/status > /tmp/status.json -do - session_id=$(jq -r '.[]?.node?.slots | .[0]?.session?.sessionId' /tmp/status.json) - echo "Session: $session_id is created" - if [[ "$session_id" != "null" && "$session_id" != "" && "$recording_started" = "false" ]] - then - caps_se_video_record=$(bash ${VIDEO_CONFIG_DIRECTORY}/graphQLRecordVideo.sh "$session_id") - cat "/tmp/graphQL_$session_id.json"; echo - echo "se:recordVideo value is: $caps_se_video_record" - fi - if [[ "$session_id" != "null" && "$session_id" != "" && "$recording_started" = "false" && "$caps_se_video_record" = "true" ]] - then - video_file_name="$session_id.mp4" - video_file="${SE_VIDEO_FOLDER}/$video_file_name" - echo "Starting to record video" - ffmpeg -nostdin -y -f x11grab -video_size ${VIDEO_SIZE} -r ${FRAME_RATE} -i ${DISPLAY} -codec:v ${CODEC} ${PRESET} -pix_fmt yuv420p $video_file & - recording_started="true" - echo "Video recording started" - elif [[ "$session_id" != "$prev_session_id" && "$recording_started" = "true" ]] - then - echo "Stopping to record video" - pkill -INT ffmpeg - recorded_count=$((recorded_count+1)) - recording_started="false" - if [[ "${VIDEO_UPLOAD_ENABLED}" != "false" ]] && [[ ! -z "${UPLOAD_DESTINATION_PREFIX}" ]]; - then - upload_destination=${UPLOAD_DESTINATION_PREFIX}/${video_file_name} - echo "Uploading video to $upload_destination" - echo $video_file ${UPLOAD_DESTINATION_PREFIX} >> ${SE_VIDEO_FOLDER}/uploadpipe & - elif [[ "${VIDEO_UPLOAD_ENABLED}" != "false" ]] && [[ -z "${UPLOAD_DESTINATION_PREFIX}" ]]; - then - echo Upload destination not known since UPLOAD_DESTINATION_PREFIX is not set. Continue without uploading. - fi - if [ $max_recorded_count -gt 0 ] && [ $recorded_count -ge $max_recorded_count ]; - then - echo "Node will be drained since max sessions reached count number ($max_recorded_count)" - exit - fi - - elif [[ $recording_started = "true" ]] - then - echo "Video recording in progress" - sleep 1 - else - echo "No session in progress" - sleep 1 - fi - prev_session_id=$session_id -done -echo "Node API is not responding, exiting." diff --git a/charts/selenium-grid/configs/uploader/rclone/config.conf b/charts/selenium-grid/configs/uploader/rclone/config.conf deleted file mode 100644 index db0337c13..000000000 --- a/charts/selenium-grid/configs/uploader/rclone/config.conf +++ /dev/null @@ -1,9 +0,0 @@ -[mys3] -type = s3 -provider = AWS -env_auth = true -region = -location_constraint = -acl = private -access_key_id = -secret_access_key = diff --git a/charts/selenium-grid/configs/uploader/rclone/entry_point.sh b/charts/selenium-grid/configs/uploader/rclone/entry_point.sh deleted file mode 100644 index 211601353..000000000 --- a/charts/selenium-grid/configs/uploader/rclone/entry_point.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env bash - -SE_VIDEO_FOLDER=${SE_VIDEO_FOLDER:-"/videos"} -UPLOAD_CONFIG_DIRECTORY=${UPLOAD_CONFIG_DIRECTORY:-"/opt/bin"} -UPLOAD_CONFIG_FILE_NAME=${UPLOAD_CONFIG_FILE_NAME:-"config.conf"} -UPLOAD_COMMAND=${UPLOAD_COMMAND:-"copy"} - -function consume_force_exit() { - rm -f ${SE_VIDEO_FOLDER}/force_exit - echo "Force exit signal consumed" -} -trap consume_force_exit EXIT - -while [ ! -p ${SE_VIDEO_FOLDER}/uploadpipe ]; -do - echo "Waiting for ${SE_VIDEO_FOLDER}/uploadpipe to be created" - sleep 1 -done - -echo "Waiting for video files put into pipe for proceeding to upload" - -while read FILE DESTINATION < ${SE_VIDEO_FOLDER}/uploadpipe -do - if [ "${FILE}" = "exit" ]; - then - exit - else [ "$FILE" != "" ] && [ "$DESTINATION" != "" ]; - echo "Uploading ${FILE} to ${DESTINATION}" - rclone --config ${UPLOAD_CONFIG_DIRECTORY}/${UPLOAD_CONFIG_FILE_NAME} ${UPLOAD_COMMAND} ${UPLOAD_OPTS} "${FILE}" "${DESTINATION}" - if [ $? -eq 0 ]; - then - rm -rf $FILE - fi - fi - if [ -f ${SE_VIDEO_FOLDER}/force_exit ] && [ ! -s ${SE_VIDEO_FOLDER}/uploadpipe ]; - then - exit - fi -done - -consume_force_exit diff --git a/charts/selenium-grid/configs/uploader/s3/entry_point.sh b/charts/selenium-grid/configs/uploader/s3/upload.sh similarity index 55% rename from charts/selenium-grid/configs/uploader/s3/entry_point.sh rename to charts/selenium-grid/configs/uploader/s3/upload.sh index e2d3d8f5f..4cb72546f 100644 --- a/charts/selenium-grid/configs/uploader/s3/entry_point.sh +++ b/charts/selenium-grid/configs/uploader/s3/upload.sh @@ -1,6 +1,9 @@ #!/usr/bin/env bash SE_VIDEO_FOLDER=${SE_VIDEO_FOLDER:-"/videos"} +UPLOAD_COMMAND=${UPLOAD_COMMAND:-"cp"} +UPLOAD_RETAIN_LOCAL_FILE=${SE_UPLOAD_RETAIN_LOCAL_FILE:-"false"} +SE_VIDEO_UPLOAD_BATCH_CHECK=${SE_VIDEO_UPLOAD_BATCH_CHECK:-"10"} if [[ -z "${AWS_REGION}" ]] || [[ -z "${AWS_ACCESS_KEY_ID}" ]] || [[ -z "${AWS_SECRET_ACCESS_KEY}" ]]; then @@ -13,11 +16,21 @@ aws configure set aws_secret_access_key ${AWS_SECRET_ACCESS_KEY} --profile s3-pr aws configure --profile s3-profile function consume_force_exit() { - rm -f ${SE_VIDEO_FOLDER}/force_exit + for pid in "${list_upload_pid[@]}"; + do + wait ${pid} + done + rm -rf ${SE_VIDEO_FOLDER}/force_exit echo "Force exit signal consumed" } trap consume_force_exit EXIT +if [ "${UPLOAD_RETAIN_LOCAL_FILE}" = "false" ]; +then + echo "UPLOAD_RETAIN_LOCAL_FILE is set to false, force to use command: move" + UPLOAD_COMMAND="mv" +fi + while [ ! -p ${SE_VIDEO_FOLDER}/uploadpipe ]; do echo "Waiting for ${SE_VIDEO_FOLDER}/uploadpipe to be created" @@ -26,15 +39,29 @@ done echo "Waiting for video files put into pipe for proceeding to upload" +list_upload_pid=() while read FILE DESTINATION < ${SE_VIDEO_FOLDER}/uploadpipe do if [ "${FILE}" = "exit" ]; then exit - else [ "$FILE" != "" ] && [ "$DESTINATION" != "" ]; + elif [ "$FILE" != "" ] && [ "$DESTINATION" != "" ]; + then echo "Uploading ${FILE} to ${DESTINATION}" - aws s3 cp "${FILE}" "${DESTINATION}" + aws s3 ${UPLOAD_COMMAND} "${FILE}" "${DESTINATION}" & + list_upload_pid+=($!) + else + # Wait for a batch of processes to finish + if [ ${#list_upload_pid[@]} -eq ${SE_VIDEO_UPLOAD_BATCH_CHECK} ] + then + for pid in "${list_upload_pid[@]}"; + do + wait ${pid} + done + list_upload_pid=() + fi fi + if [ -f ${SE_VIDEO_FOLDER}/force_exit ] && [ ! -s ${SE_VIDEO_FOLDER}/uploadpipe ]; then exit diff --git a/charts/selenium-grid/templates/_helpers.tpl b/charts/selenium-grid/templates/_helpers.tpl index ee293c1eb..ff5363afe 100644 --- a/charts/selenium-grid/templates/_helpers.tpl +++ b/charts/selenium-grid/templates/_helpers.tpl @@ -1,113 +1,3 @@ -{{/* -Expand the name of the chart. -*/}} -{{- define "seleniumGrid.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -Create a default fully qualified app name. -*/}} -{{- define "seleniumGrid.fullname" -}} -{{- if .Values.fullnameOverride -}} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- $name := default .Chart.Name .Values.nameOverride -}} -{{- if contains $name .Release.Name -}} -{{- .Release.Name | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} -{{- end -}} -{{- end -}} -{{- end -}} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "seleniumGrid.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -Common labels -*/}} -{{- define "seleniumGrid.commonLabels" -}} -app.kubernetes.io/managed-by: {{ .Release.Service | lower }} -app.kubernetes.io/instance: {{ .Release.Name }} -app.kubernetes.io/version: {{ .Chart.AppVersion }} -app.kubernetes.io/component: {{ printf "selenium-grid-%s" .Chart.AppVersion }} -helm.sh/chart: {{ include "seleniumGrid.chart" . }} -{{- end -}} - -{{/* -Selenium Hub fullname -*/}} -{{- define "seleniumGrid.hub.fullname" -}} -{{- tpl (default "selenium-hub" .Values.hub.nameOverride) $ | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -Event bus fullname -*/}} -{{- define "seleniumGrid.eventBus.fullname" -}} -{{- tpl (default "selenium-event-bus" .Values.components.eventBus.nameOverride) $ | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -Router fullname -*/}} -{{- define "seleniumGrid.router.fullname" -}} -{{- tpl (default "selenium-router" .Values.components.router.nameOverride) $ | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -Distributor fullname -*/}} -{{- define "seleniumGrid.distributor.fullname" -}} -{{- tpl (default "selenium-distributor" .Values.components.distributor.nameOverride) $ | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -SessionMap fullname -*/}} -{{- define "seleniumGrid.sessionMap.fullname" -}} -{{- tpl (default "selenium-session-map" .Values.components.sessionMap.nameOverride) $ | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -SessionQueue fullname -*/}} -{{- define "seleniumGrid.sessionQueue.fullname" -}} -{{- tpl (default "selenium-session-queue" .Values.components.sessionQueue.nameOverride) $ | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -Chrome node fullname -*/}} -{{- define "seleniumGrid.chromeNode.fullname" -}} -{{- tpl (default "selenium-chrome-node" .Values.chromeNode.nameOverride) $ | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -Firefox node fullname -*/}} -{{- define "seleniumGrid.firefoxNode.fullname" -}} -{{- tpl (default "selenium-firefox-node" .Values.firefoxNode.nameOverride) $ | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -Edge node fullname -*/}} -{{- define "seleniumGrid.edgeNode.fullname" -}} -{{- tpl (default "selenium-edge-node" .Values.edgeNode.nameOverride) $ | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -Ingress fullname -*/}} -{{- define "seleniumGrid.ingress.fullname" -}} -{{- tpl (default "selenium-ingress" .Values.ingress.nameOverride) $ | trunc 63 | trimSuffix "-" -}} -{{- end -}} {{/* Protocol of server components @@ -127,18 +17,20 @@ Probe httpGet schema Check user define custom probe method */}} {{- define "seleniumGrid.probe.fromUserDefine" -}} +{{- $values := index . "values" -}} +{{- $root := index . "root" -}} {{- $overrideProbe := dict -}} -{{- with .exec -}} -{{- $overrideProbe = dict "exec" . -}} +{{- with $values.exec -}} +{{- $overrideProbe = dict "exec" (tpl (toYaml .) $root | fromYaml) -}} {{- end }} -{{- with .httpGet -}} -{{- $overrideProbe = dict "httpGet" . -}} +{{- with $values.httpGet -}} +{{- $overrideProbe = dict "httpGet" (tpl (toYaml .) $root | fromYaml) -}} {{- end }} -{{- with .tcpSocket -}} -{{- $overrideProbe = dict "tcpSocket" . -}} +{{- with $values.tcpSocket -}} +{{- $overrideProbe = dict "tcpSocket" (tpl (toYaml .) $root | fromYaml) -}} {{- end }} -{{- with .grpc -}} -{{- $overrideProbe = dict "grpc" . -}} +{{- with $values.grpc -}} +{{- $overrideProbe = dict "grpc" (tpl (toYaml .) $root | fromYaml) -}} {{- end -}} {{- $overrideProbe | toYaml -}} {{- end -}} @@ -166,13 +58,6 @@ Get probe settings {{- $settings | toYaml -}} {{- end -}} -{{/* -Secret TLS fullname -*/}} -{{- define "seleniumGrid.tls.fullname" -}} -{{- ( tpl (default "selenium-tls-secret" .Values.tls.nameOverride) $ )| trunc 63 | trimSuffix "-" -}} -{{- end -}} - {{/* Is registration secret enabled */}} @@ -193,13 +78,6 @@ Get default certificate file name in chart {{- $content -}} {{- end -}} -{{/* -Common secrets cross components -*/}} -{{- define "seleniumGrid.common.secrets" -}} -{{- tpl (default "selenium-secrets" .Values.secrets.nameOverride) $ | trunc 63 | trimSuffix "-" -}} -{{- end -}} - {{- define "seleniumGrid.ingress.nginx.annotations.default" -}} {{- with .Values.ingress.nginx }} {{- with .proxyTimeout }} @@ -228,31 +106,17 @@ nginx.ingress.kubernetes.io/backend-protocol: "HTTPS" {{- end -}} {{/* -Service Account fullname -*/}} -{{- define "seleniumGrid.serviceAccount.fullname" -}} -{{- tpl (.Values.serviceAccount.name | default "selenium-serviceaccount") $ | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -Recorder ConfigMap fullname -*/}} -{{- define "seleniumGrid.recorder.fullname" -}} -{{- tpl (default "selenium-recorder-config" .Values.recorderConfigMap.name) $ | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -Uploader ConfigMap fullname +Is autoscaling using KEDA enabled */}} -{{- define "seleniumGrid.uploader.fullname" -}} -{{- tpl (default "selenium-uploader-config" .Values.uploaderConfigMap.name) $ | trunc 63 | trimSuffix "-" -}} +{{- define "seleniumGrid.useKEDA" -}} +{{- or .Values.autoscaling.enabled .Values.autoscaling.enableWithExistingKEDA | ternary "true" "" -}} {{- end -}} {{/* -Is autoscaling using KEDA enabled +Is tracing enabled */}} -{{- define "seleniumGrid.useKEDA" -}} -{{- or .Values.autoscaling.enabled .Values.autoscaling.enableWithExistingKEDA | ternary "true" "" -}} +{{- define "seleniumGrid.enableTracing" -}} +{{- or .Values.tracing.enabled .Values.tracing.enabledWithExistingEndpoint | ternary "true" "" -}} {{- end -}} {{/* @@ -274,7 +138,7 @@ Common autoscaling spec template {{- with .node.scaledObjectOptions -}} {{- $spec = mergeOverwrite ($spec | fromYaml) . | toYaml -}} {{- end -}} - {{- $spec = mergeOverwrite ($spec | fromYaml) (dict "scaleTargetRef" (dict "name" .name)) | toYaml -}} + {{- $spec = mergeOverwrite ($spec | fromYaml) (dict "scaleTargetRef" (dict "name" .name) "advanced" (dict "horizontalPodAutoscalerConfig" (dict "name" .name))) | toYaml -}} {{- else if eq .Values.autoscaling.scalingType "job" -}} {{- with .Values.autoscaling.scaledJobOptions -}} {{- $spec = mergeOverwrite ($spec | fromYaml) . | toYaml -}} @@ -320,14 +184,14 @@ template: spec: serviceAccountName: {{ template "seleniumGrid.serviceAccount.fullname" . }} serviceAccount: {{ template "seleniumGrid.serviceAccount.fullname" . }} - restartPolicy: {{ and (eq (include "seleniumGrid.useKEDA" .) "true") (eq .Values.autoscaling.scalingType "job") | ternary "Never" "Always" }} + restartPolicy: {{ template "seleniumGrid.node.restartPolicy" . }} {{- with .node.hostAliases }} hostAliases: {{ toYaml . | nindent 6 }} {{- end }} + {{- with .node.initContainers }} initContainers: - {{- if .node.initContainers }} - {{- toYaml .node.initContainers | nindent 6 }} - {{- end }} + {{- toYaml . | nindent 6 }} + {{- end }} containers: - name: {{.name}} {{- $imageTag := default .Values.global.seleniumGrid.nodesImageTag .node.imageTag }} @@ -335,6 +199,8 @@ template: image: {{ printf "%s/%s:%s" $imageRegistry .node.imageName $imageTag }} imagePullPolicy: {{ .node.imagePullPolicy }} env: + - name: SE_OTEL_SERVICE_NAME + value: {{ .name | quote }} - name: SE_NODE_PORT value: {{ .node.port | quote }} {{- with .node.extraEnvironmentVariables }} @@ -342,15 +208,15 @@ template: {{- end }} envFrom: - configMapRef: - name: {{ tpl (toYaml .Values.busConfigMap.name) $ }} + name: {{ template "seleniumGrid.eventBus.configmap.fullname" $ }} - configMapRef: - name: {{ tpl (toYaml .Values.nodeConfigMap.name) $ }} + name: {{ template "seleniumGrid.node.configmap.fullname" $ }} - configMapRef: - name: {{ tpl (toYaml .Values.loggingConfigMap.name) $ }} + name: {{ template "seleniumGrid.logging.configmap.fullname" $ }} - configMapRef: - name: {{ tpl (toYaml .Values.serverConfigMap.name) $ }} + name: {{ template "seleniumGrid.server.configmap.fullname" $ }} - secretRef: - name: {{ include "seleniumGrid.common.secrets" $ }} + name: {{ include "seleniumGrid.common.secrets.fullname" $ }} {{- with .node.extraEnvFrom }} {{- tpl (toYaml .) $ | nindent 10 }} {{- end }} @@ -372,7 +238,7 @@ template: - name: dshm mountPath: /dev/shm {{- range $fileName, $value := .Values.nodeConfigMap.extraScripts }} - - name: {{ $.Values.nodeConfigMap.scriptVolumeMountName }} + - name: {{ tpl (default (include "seleniumGrid.node.configmap.fullname" $) $.Values.nodeConfigMap.scriptVolumeMountName) $ }} mountPath: {{ $.Values.nodeConfigMap.extraScriptsDirectory }}/{{ $fileName }} subPath: {{ $fileName }} {{- end }} @@ -394,8 +260,11 @@ template: {{- if .node.startupProbe.enabled }} {{- with .node.startupProbe }} startupProbe: - {{- if (ne (include "seleniumGrid.probe.fromUserDefine" .) "{}") }} - {{- include "seleniumGrid.probe.fromUserDefine" . | nindent 10 }} + {{- if (ne (include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $)) "{}") }} + {{- include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $) | nindent 10 }} + {{- else if eq $.Values.global.seleniumGrid.defaultNodeStartupProbe "exec" }} + exec: + command: ["bash", "-c", "{{ $.Values.nodeConfigMap.extraScriptsDirectory }}/nodeProbe.sh >> /proc/1/fd/1"] {{- else }} httpGet: scheme: {{ default (include "seleniumGrid.probe.httpGet.schema" $) .schema }} @@ -410,8 +279,8 @@ template: {{- if .node.readinessProbe.enabled }} {{- with .node.readinessProbe }} readinessProbe: - {{- if (ne (include "seleniumGrid.probe.fromUserDefine" .) "{}") }} - {{- include "seleniumGrid.probe.fromUserDefine" . | nindent 12 }} + {{- if (ne (include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $)) "{}") }} + {{- include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $) | nindent 12 }} {{- else }} httpGet: scheme: {{ default (include "seleniumGrid.probe.httpGet.schema" $) .schema }} @@ -426,8 +295,8 @@ template: {{- if .node.livenessProbe.enabled }} {{- with .node.livenessProbe }} livenessProbe: - {{- if (ne (include "seleniumGrid.probe.fromUserDefine" .) "{}") }} - {{- include "seleniumGrid.probe.fromUserDefine" . | nindent 10 }} + {{- if (ne (include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $)) "{}") }} + {{- include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $) | nindent 10 }} {{- else }} httpGet: scheme: {{ default (include "seleniumGrid.probe.httpGet.schema" $) .schema }} @@ -460,13 +329,17 @@ template: {{- end }} envFrom: - configMapRef: - name: {{ tpl (toYaml .Values.busConfigMap.name) $ }} + name: {{ template "seleniumGrid.eventBus.configmap.fullname" $ }} - configMapRef: - name: {{ tpl (toYaml .Values.nodeConfigMap.name) $ }} + name: {{ template "seleniumGrid.node.configmap.fullname" $ }} - configMapRef: - name: {{ tpl (toYaml .Values.recorderConfigMap.name) $ }} + name: {{ template "seleniumGrid.recorder.configmap.fullname" $ }} - configMapRef: - name: {{ tpl (toYaml .Values.serverConfigMap.name) $ }} + name: {{ template "seleniumGrid.server.configmap.fullname" $ }} + {{- if and .Values.videoRecorder.uploader.enabled (not .Values.videoRecorder.uploader.name) }} + - secretRef: + name: {{ tpl (default (include "seleniumGrid.common.secrets.fullname" $) .Values.uploaderConfigMap.secretVolumeMountName) $ }} + {{- end }} {{- with .Values.videoRecorder.extraEnvFrom }} {{- tpl (toYaml .) $ | nindent 8 }} {{- end }} @@ -496,10 +369,10 @@ template: {{- with .Values.videoRecorder.lifecycle }} lifecycle: {{- toYaml . | nindent 10 }} {{- end }} - {{- if .Values.videoRecorder.uploader.enabled }} + {{- if and .Values.videoRecorder.uploader.enabled (not (empty .Values.videoRecorder.uploader.name)) }} - name: uploader - {{- $imageTag := default .Values.global.seleniumGrid.uploaderImageTag .uploader.imageTag }} - {{- $imageRegistry := default .Values.global.seleniumGrid.imageRegistry .uploader.imageRegistry }} + {{- $imageTag := .uploader.imageTag }} + {{- $imageRegistry := .uploader.imageRegistry }} image: {{ printf "%s/%s:%s" $imageRegistry .uploader.imageName $imageTag }} imagePullPolicy: {{ .uploader.imagePullPolicy }} {{- if .uploader.command }} @@ -517,9 +390,9 @@ template: {{- end }} envFrom: - configMapRef: - name: {{ tpl (toYaml .Values.uploaderConfigMap.name) $ }} + name: {{ template "seleniumGrid.uploader.configmap.fullname" $ }} - secretRef: - name: {{ tpl (toYaml .Values.uploaderConfigMap.secretVolumeMountName) $ }} + name: {{ tpl (default (include "seleniumGrid.common.secrets.fullname" $) .Values.uploaderConfigMap.secretVolumeMountName) $ }} {{- with .uploader.extraEnvFrom }} {{- tpl (toYaml .) $ | nindent 10 }} {{- end }} @@ -552,9 +425,9 @@ template: {{- end }} terminationGracePeriodSeconds: {{ .node.terminationGracePeriodSeconds }} volumes: - - name: {{ .Values.nodeConfigMap.scriptVolumeMountName }} + - name: {{ tpl (default (include "seleniumGrid.node.configmap.fullname" $) $.Values.nodeConfigMap.scriptVolumeMountName) $ }} configMap: - name: {{ tpl .Values.nodeConfigMap.name $ }} + name: {{ template "seleniumGrid.node.configmap.fullname" $ }} defaultMode: {{ .Values.nodeConfigMap.defaultMode }} - name: dshm emptyDir: @@ -620,6 +493,8 @@ Get the url of the grid. If the external url can be figured out from the ingress {{- if .Values.ingress.enabled -}} {{- if or (ne (.Values.ingress.ports.http | toString) "80") (ne (.Values.ingress.ports.https | toString) "443") -}} {{- $port = printf ":%s" (ternary (.Values.ingress.ports.http | toString) (.Values.ingress.ports.https | toString) (eq (include "seleniumGrid.url.schema" .) "http")) -}} + {{- else if and .Values.ingress.hostname (eq (tpl .Values.ingress.hostname $) "selenium-grid.local") }} + {{- $port = $port -}} {{- else -}} {{- $port = "" -}} {{- end -}} @@ -668,7 +543,7 @@ Define preStop hook for the node pod. Node preStop script is stored in a ConfigM {{- define "seleniumGrid.node.deregisterLifecycle" -}} preStop: exec: - command: ["bash", "-c", "{{ $.Values.nodeConfigMap.extraScriptsDirectory }}/nodePreStop.sh"] + command: ["bash", "-c", "{{ $.Values.nodeConfigMap.extraScriptsDirectory }}/nodePreStop.sh >> /proc/1/fd/1"] {{- end -}} {{/* @@ -712,12 +587,27 @@ Define terminationGracePeriodSeconds of the node pod. {{- $period -}} {{- end -}} +{{- define "seleniumGrid.node.restartPolicy" -}} +{{- $restartPolicy := "Always" -}} +{{- if and (eq (include "seleniumGrid.useKEDA" .) "true") (eq .Values.autoscaling.scalingType "job") -}} + {{- $restartPolicy = "Never" -}} +{{- end -}} +{{- $restartPolicy -}} +{{- end -}} + {{- define "seleniumGrid.video.volumeMounts.default" -}} {{- range $fileName, $value := .Values.recorderConfigMap.extraScripts }} -- name: {{ tpl (toYaml $.Values.recorderConfigMap.scriptVolumeMountName) $ }} +- name: {{ tpl (default (include "seleniumGrid.recorder.configmap.fullname" $) $.Values.recorderConfigMap.scriptVolumeMountName) $ }} mountPath: {{ $.Values.recorderConfigMap.extraScriptsDirectory }}/{{ $fileName }} subPath: {{ $fileName }} {{- end }} +{{- if and .Values.videoRecorder.uploader.enabled (not .Values.videoRecorder.uploader.name) }} +{{- range $fileName, $value := .Values.uploaderConfigMap.secretFiles }} +- name: {{ tpl (default (include "seleniumGrid.common.secrets.fullname" $) $.Values.uploaderConfigMap.secretVolumeMountName) $ }} + mountPath: {{ $.Values.uploaderConfigMap.extraScriptsDirectory }}/{{ $fileName }} + subPath: {{ $fileName }} +{{- end }} +{{- end }} - name: {{ tpl (toYaml $.Values.recorderConfigMap.videoVolumeMountName) $ }} mountPath: {{ $.Values.videoRecorder.targetFolder }} {{- end -}} @@ -725,27 +615,27 @@ Define terminationGracePeriodSeconds of the node pod. {{- define "seleniumGrid.video.volumes.default" -}} - name: {{ tpl (toYaml $.Values.recorderConfigMap.videoVolumeMountName) $ }} emptyDir: {} -- name: {{ tpl (toYaml $.Values.recorderConfigMap.scriptVolumeMountName) $ }} +- name: {{ tpl (default (include "seleniumGrid.recorder.configmap.fullname" $) $.Values.recorderConfigMap.scriptVolumeMountName) $ }} configMap: - name: {{ tpl (toYaml $.Values.recorderConfigMap.name) $ }} + name: {{ template "seleniumGrid.recorder.configmap.fullname" $ }} defaultMode: {{ $.Values.recorderConfigMap.defaultMode }} -- name: {{ tpl (toYaml $.Values.uploaderConfigMap.scriptVolumeMountName) $ }} +- name: {{ tpl (default (include "seleniumGrid.uploader.configmap.fullname" $) $.Values.uploaderConfigMap.scriptVolumeMountName) $ }} configMap: - name: {{ tpl (toYaml $.Values.uploaderConfigMap.name) $ }} + name: {{ template "seleniumGrid.uploader.configmap.fullname" $ }} defaultMode: {{ $.Values.uploaderConfigMap.defaultMode }} -- name: {{ tpl (toYaml $.Values.uploaderConfigMap.secretVolumeMountName) $ }} +- name: {{ tpl (default (include "seleniumGrid.common.secrets.fullname" $) $.Values.uploaderConfigMap.secretVolumeMountName) $ }} secret: - secretName: {{ tpl (toYaml $.Values.uploaderConfigMap.secretVolumeMountName) $ }} + secretName: {{ tpl (default (include "seleniumGrid.common.secrets.fullname" $) $.Values.uploaderConfigMap.secretVolumeMountName) $ }} {{- end -}} {{- define "seleniumGrid.video.uploader.volumeMounts.default" -}} {{- range $fileName, $value := .Values.uploaderConfigMap.extraScripts }} -- name: {{ tpl (toYaml $.Values.uploaderConfigMap.scriptVolumeMountName) $ }} +- name: {{ tpl (default (include "seleniumGrid.uploader.configmap.fullname" $) $.Values.uploaderConfigMap.scriptVolumeMountName) $ }} mountPath: {{ $.Values.uploaderConfigMap.extraScriptsDirectory }}/{{ $fileName }} subPath: {{ $fileName }} {{- end }} {{- range $fileName, $value := .Values.uploaderConfigMap.secretFiles }} -- name: {{ tpl (toYaml $.Values.uploaderConfigMap.secretVolumeMountName) $ }} +- name: {{ tpl (default (include "seleniumGrid.common.secrets.fullname" $) $.Values.uploaderConfigMap.secretVolumeMountName) $ }} mountPath: {{ $.Values.uploaderConfigMap.extraScriptsDirectory }}/{{ $fileName }} subPath: {{ $fileName }} {{- end }} diff --git a/charts/selenium-grid/templates/_nameHelpers.tpl b/charts/selenium-grid/templates/_nameHelpers.tpl new file mode 100644 index 000000000..b7d5abc2f --- /dev/null +++ b/charts/selenium-grid/templates/_nameHelpers.tpl @@ -0,0 +1,173 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "seleniumGrid.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +*/}} +{{- define "seleniumGrid.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "seleniumGrid.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Common labels +*/}} +{{- define "seleniumGrid.commonLabels" -}} +app.kubernetes.io/managed-by: {{ .Release.Service | lower }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/version: {{ .Chart.AppVersion }} +app.kubernetes.io/component: {{ printf "selenium-grid-%s" .Chart.AppVersion }} +helm.sh/chart: {{ include "seleniumGrid.chart" . }} +{{- end -}} + +{{/* +Selenium Hub fullname +*/}} +{{- define "seleniumGrid.hub.fullname" -}} +{{- tpl (default (printf "%s-selenium-hub" .Release.Name) .Values.hub.nameOverride) $ | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Event bus fullname +*/}} +{{- define "seleniumGrid.eventBus.fullname" -}} +{{- tpl (default (printf "%s-selenium-event-bus" .Release.Name) .Values.components.eventBus.nameOverride) $ | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Event bus ConfigMap fullname +*/}} +{{- define "seleniumGrid.eventBus.configmap.fullname" -}} +{{- tpl (default (include "seleniumGrid.eventBus.fullname" $) .Values.busConfigMap.nameOverride) $ | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Router fullname +*/}} +{{- define "seleniumGrid.router.fullname" -}} +{{- tpl (default (printf "%s-selenium-router" .Release.Name) .Values.components.router.nameOverride) $ | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Distributor fullname +*/}} +{{- define "seleniumGrid.distributor.fullname" -}} +{{- tpl (default (printf "%s-selenium-distributor" .Release.Name) .Values.components.distributor.nameOverride) $ | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +SessionMap fullname +*/}} +{{- define "seleniumGrid.sessionMap.fullname" -}} +{{- tpl (default (printf "%s-selenium-session-map" .Release.Name) .Values.components.sessionMap.nameOverride) $ | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +SessionQueue fullname +*/}} +{{- define "seleniumGrid.sessionQueue.fullname" -}} +{{- tpl (default (printf "%s-selenium-session-queue" .Release.Name) .Values.components.sessionQueue.nameOverride) $ | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Chrome node fullname +*/}} +{{- define "seleniumGrid.chromeNode.fullname" -}} +{{- tpl (default (printf "%s-selenium-chrome-node" .Release.Name) .Values.chromeNode.nameOverride) $ | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Firefox node fullname +*/}} +{{- define "seleniumGrid.firefoxNode.fullname" -}} +{{- tpl (default (printf "%s-selenium-firefox-node" .Release.Name) .Values.firefoxNode.nameOverride) $ | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Edge node fullname +*/}} +{{- define "seleniumGrid.edgeNode.fullname" -}} +{{- tpl (default (printf "%s-selenium-edge-node" .Release.Name) .Values.edgeNode.nameOverride) $ | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Ingress fullname +*/}} +{{- define "seleniumGrid.ingress.fullname" -}} +{{- tpl (default (printf "%s-selenium-ingress" .Release.Name) .Values.ingress.nameOverride) $ | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Common secrets cross components +*/}} +{{- define "seleniumGrid.common.secrets.fullname" -}} +{{- tpl (default (printf "%s-selenium-secrets" .Release.Name) .Values.secrets.nameOverride) $ | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Secret TLS fullname +*/}} +{{- define "seleniumGrid.tls.fullname" -}} +{{- tpl (default (printf "%s-selenium-tls-secret" .Release.Name) .Values.tls.nameOverride) $ | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Service Account fullname +*/}} +{{- define "seleniumGrid.serviceAccount.fullname" -}} +{{- tpl (default (printf "%s-selenium-serviceaccount" .Release.Name) .Values.serviceAccount.nameOverride) $ | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Recorder ConfigMap fullname +*/}} +{{- define "seleniumGrid.recorder.configmap.fullname" -}} +{{- tpl (default (printf "%s-selenium-recorder-config" .Release.Name) .Values.recorderConfigMap.nameOverride) $ | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Uploader ConfigMap fullname +*/}} +{{- define "seleniumGrid.uploader.configmap.fullname" -}} +{{- tpl (default (printf "%s-selenium-uploader-config" .Release.Name) .Values.uploaderConfigMap.nameOverride) $ | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Logging ConfigMap fullname +*/}} +{{- define "seleniumGrid.logging.configmap.fullname" -}} +{{- tpl (default (printf "%s-selenium-logging-config" .Release.Name) .Values.loggingConfigMap.nameOverride) $ | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Node ConfigMap fullname +*/}} +{{- define "seleniumGrid.node.configmap.fullname" -}} +{{- tpl (default (printf "%s-selenium-node-config" .Release.Name) .Values.nodeConfigMap.nameOverride) $ | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Server ConfigMap fullname +*/}} +{{- define "seleniumGrid.server.configmap.fullname" -}} +{{- tpl (default (printf "%s-selenium-server-config" .Release.Name) .Values.serverConfigMap.nameOverride) $ | trunc 63 | trimSuffix "-" -}} +{{- end -}} diff --git a/charts/selenium-grid/templates/chrome-node-hpa.yaml b/charts/selenium-grid/templates/chrome-node-hpa.yaml index 107e87b8b..36d6c3f43 100644 --- a/charts/selenium-grid/templates/chrome-node-hpa.yaml +++ b/charts/selenium-grid/templates/chrome-node-hpa.yaml @@ -2,7 +2,7 @@ apiVersion: keda.sh/v1alpha1 kind: ScaledObject metadata: - name: selenium-grid-chrome-scaledobject + name: {{ template "seleniumGrid.chromeNode.fullname" . }} namespace: {{ .Release.Namespace }} annotations: helm.sh/resource-policy: delete diff --git a/charts/selenium-grid/templates/distributor-deployment.yaml b/charts/selenium-grid/templates/distributor-deployment.yaml index c1848faa4..7b605bf9c 100644 --- a/charts/selenium-grid/templates/distributor-deployment.yaml +++ b/charts/selenium-grid/templates/distributor-deployment.yaml @@ -33,12 +33,18 @@ spec: image: {{ printf "%s/%s:%s" $imageRegistry .Values.components.distributor.imageName $imageTag }} imagePullPolicy: {{ .Values.components.distributor.imagePullPolicy }} env: + - name: SE_OTEL_SERVICE_NAME + value: '{{ template "seleniumGrid.distributor.fullname" . }}' + - name: SE_DISTRIBUTOR_HOST + value: '{{ template "seleniumGrid.distributor.fullname" . }}.{{ .Release.Namespace }}' + - name: SE_DISTRIBUTOR_PORT + value: {{ .Values.components.distributor.port | quote }} - name: SE_SESSIONS_MAP_HOST - value: {{ template "seleniumGrid.sessionMap.fullname" . }} + value: '{{ template "seleniumGrid.sessionMap.fullname" . }}.{{ .Release.Namespace }}' - name: SE_SESSIONS_MAP_PORT value: {{ .Values.components.sessionMap.port | quote }} - name: SE_SESSION_QUEUE_HOST - value: {{ template "seleniumGrid.sessionQueue.fullname" . }} + value: '{{ template "seleniumGrid.sessionQueue.fullname" . }}.{{ .Release.Namespace }}' - name: SE_SESSION_QUEUE_PORT value: {{ .Values.components.sessionQueue.port | quote }} {{- with .Values.components.extraEnvironmentVariables }} @@ -46,13 +52,13 @@ spec: {{- end }} envFrom: - configMapRef: - name: {{ tpl .Values.busConfigMap.name $ }} + name: {{ template "seleniumGrid.eventBus.configmap.fullname" . }} - configMapRef: - name: {{ tpl .Values.loggingConfigMap.name $ }} + name: {{ template "seleniumGrid.logging.configmap.fullname" . }} - configMapRef: - name: {{ tpl .Values.serverConfigMap.name $ }} + name: {{ template "seleniumGrid.server.configmap.fullname" . }} - secretRef: - name: {{ include "seleniumGrid.common.secrets" $ | quote }} + name: {{ include "seleniumGrid.common.secrets.fullname" $ | quote }} {{- with .Values.components.extraEnvFrom }} {{- toYaml . | nindent 12 }} {{- end }} diff --git a/charts/selenium-grid/templates/edge-node-hpa.yaml b/charts/selenium-grid/templates/edge-node-hpa.yaml index 32bd3968e..604b89156 100644 --- a/charts/selenium-grid/templates/edge-node-hpa.yaml +++ b/charts/selenium-grid/templates/edge-node-hpa.yaml @@ -2,7 +2,7 @@ apiVersion: keda.sh/v1alpha1 kind: ScaledObject metadata: - name: selenium-grid-edge-scaledobject + name: {{ template "seleniumGrid.edgeNode.fullname" . }} namespace: {{ .Release.Namespace }} annotations: helm.sh/resource-policy: delete diff --git a/charts/selenium-grid/templates/event-bus-configmap.yaml b/charts/selenium-grid/templates/event-bus-configmap.yaml index 0515aff22..7bbfd952d 100644 --- a/charts/selenium-grid/templates/event-bus-configmap.yaml +++ b/charts/selenium-grid/templates/event-bus-configmap.yaml @@ -4,7 +4,7 @@ apiVersion: v1 kind: ConfigMap metadata: - name: {{ tpl .Values.busConfigMap.name $ }} + name: {{ template "seleniumGrid.eventBus.configmap.fullname" . }} namespace: {{ .Release.Namespace }} {{- with .Values.busConfigMap.annotations }} annotations: {{- toYaml . | nindent 4 }} diff --git a/charts/selenium-grid/templates/event-bus-deployment.yaml b/charts/selenium-grid/templates/event-bus-deployment.yaml index e783625a9..a9bb72e47 100644 --- a/charts/selenium-grid/templates/event-bus-deployment.yaml +++ b/charts/selenium-grid/templates/event-bus-deployment.yaml @@ -39,16 +39,23 @@ spec: protocol: TCP - containerPort: {{ .Values.components.eventBus.subscribePort }} protocol: TCP + env: + - name: SE_OTEL_SERVICE_NAME + value: '{{ template "seleniumGrid.eventBus.fullname" . }}' + - name: SE_EVENT_BUS_HOST + value: '{{ template "seleniumGrid.eventBus.fullname" . }}.{{ .Release.Namespace }}' + - name: SE_EVENT_BUS_PORT + value: {{ .Values.components.eventBus.port | quote }} {{- with .Values.components.extraEnvironmentVariables }} - env: {{- tpl (toYaml .) $ | nindent 12 }} + {{- tpl (toYaml .) $ | nindent 12 }} {{- end }} envFrom: - configMapRef: - name: {{ tpl .Values.loggingConfigMap.name $ }} + name: {{ template "seleniumGrid.logging.configmap.fullname" . }} - configMapRef: - name: {{ tpl .Values.serverConfigMap.name $ }} + name: {{ template "seleniumGrid.server.configmap.fullname" . }} - secretRef: - name: {{ include "seleniumGrid.common.secrets" $ | quote }} + name: {{ template "seleniumGrid.common.secrets.fullname" . }} {{- with .Values.components.extraEnvFrom }} {{- toYaml . | nindent 12 }} {{- end }} diff --git a/charts/selenium-grid/templates/firefox-node-hpa.yaml b/charts/selenium-grid/templates/firefox-node-hpa.yaml index e331a3475..490e16949 100644 --- a/charts/selenium-grid/templates/firefox-node-hpa.yaml +++ b/charts/selenium-grid/templates/firefox-node-hpa.yaml @@ -2,7 +2,7 @@ apiVersion: keda.sh/v1alpha1 kind: ScaledObject metadata: - name: selenium-grid-firefox-scaledobject + name: {{ template "seleniumGrid.firefoxNode.fullname" . }} namespace: {{ .Release.Namespace }} annotations: helm.sh/resource-policy: delete diff --git a/charts/selenium-grid/templates/hub-deployment.yaml b/charts/selenium-grid/templates/hub-deployment.yaml index 60d10d8ed..77ffed0be 100644 --- a/charts/selenium-grid/templates/hub-deployment.yaml +++ b/charts/selenium-grid/templates/hub-deployment.yaml @@ -45,8 +45,8 @@ spec: {{- if .Values.hub.startupProbe.enabled }} {{- with .Values.hub.startupProbe }} startupProbe: - {{- if (ne (include "seleniumGrid.probe.fromUserDefine" .) "{}") }} - {{- include "seleniumGrid.probe.fromUserDefine" . | nindent 10 }} + {{- if (ne (include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $)) "{}") }} + {{- include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $) | nindent 10 }} {{- else }} httpGet: scheme: {{ default (include "seleniumGrid.probe.httpGet.schema" $) .schema }} @@ -61,8 +61,8 @@ spec: {{- if .Values.hub.readinessProbe.enabled }} {{- with .Values.hub.readinessProbe }} readinessProbe: - {{- if (ne (include "seleniumGrid.probe.fromUserDefine" .) "{}") }} - {{- include "seleniumGrid.probe.fromUserDefine" . | nindent 10 }} + {{- if (ne (include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $)) "{}") }} + {{- include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $) | nindent 10 }} {{- else }} httpGet: scheme: {{ default (include "seleniumGrid.probe.httpGet.schema" $) .schema }} @@ -77,8 +77,8 @@ spec: {{- if .Values.hub.livenessProbe.enabled }} {{- with .Values.hub.livenessProbe }} livenessProbe: - {{- if (ne (include "seleniumGrid.probe.fromUserDefine" .) "{}") }} - {{- include "seleniumGrid.probe.fromUserDefine" . | nindent 10 }} + {{- if (ne (include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $)) "{}") }} + {{- include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $) | nindent 10 }} {{- else }} httpGet: scheme: {{ default (include "seleniumGrid.probe.httpGet.schema" $) .schema }} @@ -91,6 +91,12 @@ spec: {{- end }} {{- end }} env: + - name: SE_OTEL_SERVICE_NAME + value: '{{ template "seleniumGrid.hub.fullname" . }}' + - name: SE_HUB_HOST + value: '{{ template "seleniumGrid.hub.fullname" . }}.{{ .Release.Namespace }}' + - name: SE_HUB_PORT + value: {{ .Values.hub.port | quote }} {{- with .Values.hub.subPath }} - name: SE_SUB_PATH value: {{ . | quote }} @@ -104,11 +110,11 @@ spec: {{- end }} envFrom: - configMapRef: - name: {{ tpl .Values.loggingConfigMap.name $ }} + name: {{ template "seleniumGrid.logging.configmap.fullname" . }} - configMapRef: - name: {{ tpl .Values.serverConfigMap.name $ }} + name: {{ template "seleniumGrid.server.configmap.fullname" . }} - secretRef: - name: {{ include "seleniumGrid.common.secrets" $ | quote }} + name: {{ template "seleniumGrid.common.secrets.fullname" . }} {{- with .Values.hub.extraEnvFrom }} {{- toYaml . | nindent 12 }} {{- end }} diff --git a/charts/selenium-grid/templates/jaeger-ingress.yaml b/charts/selenium-grid/templates/jaeger-ingress.yaml new file mode 100644 index 000000000..db086138b --- /dev/null +++ b/charts/selenium-grid/templates/jaeger-ingress.yaml @@ -0,0 +1,51 @@ +{{- if and (and .Values.tracing.enabled (not .Values.tracing.enabledWithExistingEndpoint)) .Values.tracing.ingress.enabled }} +{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} + {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} + {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} + {{- end }} +{{- end }} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ .Release.Name }}-jaeger-ingress + namespace: {{ .Release.Namespace }} + labels: + {{- include "seleniumGrid.commonLabels" . | nindent 4 }} + {{- with .Values.customLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.tracing.ingress.annotations }} + annotations: + {{- . | toYaml | nindent 4 }} + {{- end }} +spec: + {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} + ingressClassName: {{ .Values.ingress.className }} + {{- end }} + {{- if and (or .Values.tls.enabled .Values.tls.ingress.generateTLS) (tpl .Values.ingress.hostname $) (not .Values.ingress.tls) }} + tls: + - hosts: + - {{ tpl .Values.ingress.hostname $ | quote }} + secretName: {{ include "seleniumGrid.tls.fullname" . | quote }} + {{- else if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ tpl . $ | quote }} + {{- end }} + secretName: {{ tpl (.secretName) $ | quote }} + {{- end }} + {{- end }} + rules: + {{- if $.Values.ingress.hostname }} + - host: {{ tpl $.Values.ingress.hostname $ }} + http: + {{- else }} + - http: + {{- end }} + paths: + {{- with .Values.tracing.ingress.paths }} + {{- tpl (toYaml . | nindent 10) $ }} + {{- end }} +{{- end }} diff --git a/charts/selenium-grid/templates/logging-configmap.yaml b/charts/selenium-grid/templates/logging-configmap.yaml index a8c8de1f2..12ea1570d 100644 --- a/charts/selenium-grid/templates/logging-configmap.yaml +++ b/charts/selenium-grid/templates/logging-configmap.yaml @@ -1,7 +1,7 @@ apiVersion: v1 kind: ConfigMap metadata: - name: {{ tpl .Values.loggingConfigMap.name $ }} + name: {{ template "seleniumGrid.logging.configmap.fullname" . }} namespace: {{ .Release.Namespace }} {{- with .Values.loggingConfigMap.annotations }} annotations: {{- toYaml . | nindent 4 }} @@ -13,3 +13,9 @@ metadata: {{- end }} data: SE_LOG_LEVEL: "{{ default "INFO" .Values.global.seleniumGrid.logLevel }}" +{{- if (eq (include "seleniumGrid.enableTracing" .) "true") }} + SE_ENABLE_TRACING: "true" + SE_OTEL_TRACES_EXPORTER: {{ .Values.tracing.exporter | quote }} + SE_OTEL_JAVA_GLOBAL_AUTOCONFIGURE_ENABLED: {{ .Values.tracing.globalAutoConfigure | quote }} + SE_OTEL_EXPORTER_ENDPOINT: {{ tpl .Values.tracing.exporterEndpoint $ | quote }} +{{- end }} diff --git a/charts/selenium-grid/templates/node-configmap.yaml b/charts/selenium-grid/templates/node-configmap.yaml index bdd72461a..c6b1e6c99 100644 --- a/charts/selenium-grid/templates/node-configmap.yaml +++ b/charts/selenium-grid/templates/node-configmap.yaml @@ -1,7 +1,7 @@ apiVersion: v1 kind: ConfigMap metadata: - name: {{ tpl .Values.nodeConfigMap.name $ }} + name: {{ template "seleniumGrid.node.configmap.fullname" $ }} namespace: {{ .Release.Namespace }} {{- with .Values.nodeConfigMap.annotations }} annotations: {{- toYaml . | nindent 4 }} @@ -12,9 +12,18 @@ metadata: {{- toYaml . | nindent 4 }} {{- end }} data: +{{- if .Values.isolateComponents }} + SE_DISTRIBUTOR_HOST: '{{ include "seleniumGrid.distributor.fullname" . }}.{{ .Release.Namespace }}' + SE_DISTRIBUTOR_PORT: '{{ .Values.components.distributor.port }}' + SE_ROUTER_HOST: '{{ include "seleniumGrid.router.fullname" . }}.{{ .Release.Namespace }}' + SE_ROUTER_PORT: '{{ .Values.components.router.port }}' +{{- else }} + SE_HUB_HOST: '{{ include "seleniumGrid.hub.fullname" . }}.{{ .Release.Namespace }}' + SE_HUB_PORT: '{{ .Values.hub.port }}' +{{- end }} SE_DRAIN_AFTER_SESSION_COUNT: '{{- and (eq (include "seleniumGrid.useKEDA" .) "true") (eq .Values.autoscaling.scalingType "job") | ternary "1" "0" -}}' SE_NODE_GRID_URL: '{{ include "seleniumGrid.url" .}}' - SE_NODE_GRID_GRAPHQL_URL: '{{ include "seleniumGrid.graphqlURL" .}}' + SE_NODE_GRID_GRAPHQL_URL: '{{ include "seleniumGrid.graphqlURL" . }}' {{- range $fileName, $value := .Values.nodeConfigMap.extraScripts }} {{- if not (empty $value) }} {{ $fileName | indent 2 -}}: | diff --git a/charts/selenium-grid/templates/recorder-configmap.yaml b/charts/selenium-grid/templates/recorder-configmap.yaml index d11dd3fd5..9a6cf3b8f 100644 --- a/charts/selenium-grid/templates/recorder-configmap.yaml +++ b/charts/selenium-grid/templates/recorder-configmap.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: ConfigMap metadata: - name: {{ template "seleniumGrid.recorder.fullname" $ }} + name: {{ template "seleniumGrid.recorder.configmap.fullname" $ }} namespace: {{ .Release.Namespace }} {{- with .Values.recorderConfigMap.annotations }} annotations: {{- toYaml . | nindent 4 }} @@ -13,9 +13,13 @@ metadata: {{- toYaml . | nindent 4 }} {{- end }} data: - VIDEO_UPLOAD_ENABLED: '{{ .Values.videoRecorder.uploader.enabled }}' + SE_VIDEO_UPLOAD_ENABLED: '{{ .Values.videoRecorder.uploader.enabled }}' + SE_VIDEO_FILE_NAME: 'auto' +{{- if and .Values.videoRecorder.uploader.enabled (empty .Values.videoRecorder.uploader.name) }} + SE_VIDEO_INTERNAL_UPLOAD: 'true' +{{- end }} VIDEO_CONFIG_DIRECTORY: '{{ .Values.recorderConfigMap.extraScriptsDirectory }}' - UPLOAD_DESTINATION_PREFIX: '{{ .Values.videoRecorder.uploader.destinationPrefix }}' + SE_UPLOAD_DESTINATION_PREFIX: '{{ .Values.videoRecorder.uploader.destinationPrefix }}' {{- range $fileName, $value := .Values.recorderConfigMap.extraScripts }} {{- if not (empty $value) }} {{ $fileName | indent 2 -}}: | diff --git a/charts/selenium-grid/templates/router-deployment.yaml b/charts/selenium-grid/templates/router-deployment.yaml index 65a980403..088e29477 100644 --- a/charts/selenium-grid/templates/router-deployment.yaml +++ b/charts/selenium-grid/templates/router-deployment.yaml @@ -33,16 +33,22 @@ spec: image: {{ printf "%s/%s:%s" $imageRegistry .Values.components.router.imageName $imageTag }} imagePullPolicy: {{ .Values.components.router.imagePullPolicy }} env: + - name: SE_OTEL_SERVICE_NAME + value: '{{ template "seleniumGrid.router.fullname" . }}' + - name: SE_ROUTER_HOST + value: '{{ template "seleniumGrid.router.fullname" . }}.{{ .Release.Namespace }}' + - name: SE_ROUTER_PORT + value: {{ .Values.components.router.port | quote }} - name: SE_DISTRIBUTOR_HOST - value: {{ template "seleniumGrid.distributor.fullname" . }} + value: '{{ template "seleniumGrid.distributor.fullname" . }}.{{ .Release.Namespace }}' - name: SE_DISTRIBUTOR_PORT value: {{ .Values.components.distributor.port | quote }} - name: SE_SESSIONS_MAP_HOST - value: {{ template "seleniumGrid.sessionMap.fullname" . }} + value: '{{ template "seleniumGrid.sessionMap.fullname" . }}.{{ .Release.Namespace }}' - name: SE_SESSIONS_MAP_PORT value: {{ .Values.components.sessionMap.port | quote }} - name: SE_SESSION_QUEUE_HOST - value: {{ template "seleniumGrid.sessionQueue.fullname" . }} + value: '{{ template "seleniumGrid.sessionQueue.fullname" . }}.{{ .Release.Namespace }}' - name: SE_SESSION_QUEUE_PORT value: {{ .Values.components.sessionQueue.port | quote }} {{- with .Values.components.subPath }} @@ -58,11 +64,11 @@ spec: {{- end }} envFrom: - configMapRef: - name: {{ tpl .Values.loggingConfigMap.name $ }} + name: {{ template "seleniumGrid.logging.configmap.fullname" . }} - configMapRef: - name: {{ tpl .Values.serverConfigMap.name $ }} + name: {{ template "seleniumGrid.server.configmap.fullname" . }} - secretRef: - name: {{ include "seleniumGrid.common.secrets" $ | quote }} + name: {{ template "seleniumGrid.common.secrets.fullname" . }} {{- with .Values.components.extraEnvFrom }} {{- toYaml . | nindent 12 }} {{- end }} @@ -78,8 +84,8 @@ spec: {{- if .Values.components.router.startupProbe.enabled }} {{- with .Values.components.router.startupProbe }} startupProbe: - {{- if (ne (include "seleniumGrid.probe.fromUserDefine" .) "{}") }} - {{- include "seleniumGrid.probe.fromUserDefine" . | nindent 10 }} + {{- if (ne (include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $)) "{}") }} + {{- include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $) | nindent 10 }} {{- else }} httpGet: scheme: {{ default (include "seleniumGrid.probe.httpGet.schema" $) .schema }} @@ -94,8 +100,8 @@ spec: {{- if .Values.components.router.readinessProbe.enabled }} {{- with .Values.components.router.readinessProbe }} readinessProbe: - {{- if (ne (include "seleniumGrid.probe.fromUserDefine" .) "{}") }} - {{- include "seleniumGrid.probe.fromUserDefine" . | nindent 10 }} + {{- if (ne (include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $)) "{}") }} + {{- include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $) | nindent 10 }} {{- else }} httpGet: scheme: {{ default (include "seleniumGrid.probe.httpGet.schema" $) .schema }} @@ -110,8 +116,8 @@ spec: {{- if .Values.components.router.livenessProbe.enabled }} livenessProbe: {{- with .Values.components.router.livenessProbe }} - {{- if (ne (include "seleniumGrid.probe.fromUserDefine" .) "{}") }} - {{- include "seleniumGrid.probe.fromUserDefine" . | nindent 10 }} + {{- if (ne (include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $)) "{}") }} + {{- include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $) | nindent 10 }} {{- else }} httpGet: scheme: {{ default (include "seleniumGrid.probe.httpGet.schema" $) .schema }} diff --git a/charts/selenium-grid/templates/secrets.yaml b/charts/selenium-grid/templates/secrets.yaml index 5d76c6930..87c57d8ff 100644 --- a/charts/selenium-grid/templates/secrets.yaml +++ b/charts/selenium-grid/templates/secrets.yaml @@ -5,7 +5,7 @@ metadata: {{- with .Values.secrets.annotations }} annotations: {{- toYaml . | nindent 4 }} {{- end }} - name: {{ include "seleniumGrid.common.secrets" . }} + name: {{ include "seleniumGrid.common.secrets.fullname" . }} labels: {{- include "seleniumGrid.commonLabels" . | nindent 4 }} {{- with .Values.customLabels }} diff --git a/charts/selenium-grid/templates/server-configmap.yaml b/charts/selenium-grid/templates/server-configmap.yaml index 33b573aa5..fbf161195 100644 --- a/charts/selenium-grid/templates/server-configmap.yaml +++ b/charts/selenium-grid/templates/server-configmap.yaml @@ -1,7 +1,7 @@ apiVersion: v1 kind: ConfigMap metadata: - name: {{ tpl .Values.serverConfigMap.name $ }} + name: {{ template "seleniumGrid.server.configmap.fullname" . }} namespace: {{ .Release.Namespace }} {{- with .Values.busConfigMap.annotations }} annotations: {{- toYaml . | nindent 4 }} diff --git a/charts/selenium-grid/templates/session-map-deployment.yaml b/charts/selenium-grid/templates/session-map-deployment.yaml index 52c5ca42e..1057d00cc 100644 --- a/charts/selenium-grid/templates/session-map-deployment.yaml +++ b/charts/selenium-grid/templates/session-map-deployment.yaml @@ -32,18 +32,25 @@ spec: {{- $imageRegistry := default .Values.global.seleniumGrid.imageRegistry .Values.components.sessionMap.imageRegistry }} image: {{ printf "%s/%s:%s" $imageRegistry .Values.components.sessionMap.imageName $imageTag }} imagePullPolicy: {{ .Values.components.sessionMap.imagePullPolicy }} + env: + - name: SE_OTEL_SERVICE_NAME + value: '{{ template "seleniumGrid.sessionMap.fullname" . }}' + - name: SE_SESSIONS_HOST + value: '{{ template "seleniumGrid.sessionMap.fullname" . }}.{{ .Release.Namespace }}' + - name: SE_SESSIONS_PORT + value: {{ .Values.components.sessionMap.port | quote }} {{- with .Values.components.extraEnvironmentVariables }} - env: {{- tpl (toYaml .) $ | nindent 12 }} + {{- tpl (toYaml .) $ | nindent 12 }} {{- end }} envFrom: - configMapRef: - name: {{ tpl .Values.loggingConfigMap.name $ }} + name: {{ template "seleniumGrid.logging.configmap.fullname" . }} - configMapRef: - name: {{ tpl .Values.serverConfigMap.name $ }} + name: {{ template "seleniumGrid.server.configmap.fullname" . }} - secretRef: - name: {{ include "seleniumGrid.common.secrets" $ | quote }} + name: {{ template "seleniumGrid.common.secrets.fullname" . }} - configMapRef: - name: {{ tpl .Values.busConfigMap.name $ }} + name: {{ template "seleniumGrid.eventBus.configmap.fullname" . }} {{- with .Values.components.extraEnvFrom }} {{- toYaml . | nindent 12 }} {{- end }} diff --git a/charts/selenium-grid/templates/session-queuer-deployment.yaml b/charts/selenium-grid/templates/session-queuer-deployment.yaml index 332db749e..9765a315a 100644 --- a/charts/selenium-grid/templates/session-queuer-deployment.yaml +++ b/charts/selenium-grid/templates/session-queuer-deployment.yaml @@ -32,16 +32,23 @@ spec: {{- $imageRegistry := default .Values.global.seleniumGrid.imageRegistry .Values.components.sessionQueue.imageRegistry }} image: {{ printf "%s/%s:%s" $imageRegistry .Values.components.sessionQueue.imageName $imageTag }} imagePullPolicy: {{ .Values.components.sessionQueue.imagePullPolicy }} + env: + - name: SE_OTEL_SERVICE_NAME + value: '{{ template "seleniumGrid.sessionQueue.fullname" . }}' + - name: SE_SESSION_QUEUE_HOST + value: '{{ template "seleniumGrid.sessionQueue.fullname" . }}.{{ .Release.Namespace }}' + - name: SE_SESSION_QUEUE_PORT + value: {{ .Values.components.sessionQueue.port | quote }} {{- with .Values.components.extraEnvironmentVariables }} - env: {{- tpl (toYaml .) $ | nindent 12 }} + {{- tpl (toYaml .) $ | nindent 12 }} {{- end }} envFrom: - configMapRef: - name: {{ tpl .Values.loggingConfigMap.name $ }} + name: {{ template "seleniumGrid.logging.configmap.fullname" . }} - configMapRef: - name: {{ tpl .Values.serverConfigMap.name $ }} + name: {{ template "seleniumGrid.server.configmap.fullname" . }} - secretRef: - name: {{ include "seleniumGrid.common.secrets" $ | quote }} + name: {{ template "seleniumGrid.common.secrets.fullname" . }} {{- with .Values.components.extraEnvFrom }} {{- toYaml . | nindent 12 }} {{- end }} diff --git a/charts/selenium-grid/templates/uploader-configmap.yaml b/charts/selenium-grid/templates/uploader-configmap.yaml index bf36f9600..22e5a8ba3 100644 --- a/charts/selenium-grid/templates/uploader-configmap.yaml +++ b/charts/selenium-grid/templates/uploader-configmap.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: ConfigMap metadata: - name: {{ template "seleniumGrid.uploader.fullname" $ }} + name: {{ template "seleniumGrid.uploader.configmap.fullname" $ }} namespace: {{ .Release.Namespace }} {{- with .Values.uploaderConfigMap.annotations }} annotations: {{- toYaml . | nindent 4 }} @@ -13,8 +13,12 @@ metadata: {{- toYaml . | nindent 4 }} {{- end }} data: - UPLOAD_CONFIG_DIRECTORY: '{{ .Values.uploaderConfigMap.extraScriptsDirectory }}' - UPLOAD_CONFIG_FILE_NAME: '{{ .Values.videoRecorder.uploader.configFileName }}' +{{- with .Values.uploaderConfigMap.extraScriptsDirectory }} + UPLOAD_CONFIG_DIRECTORY: '{{ . }}' +{{- end }} +{{- with .Values.videoRecorder.uploader.configFileName }} + UPLOAD_CONFIG_FILE_NAME: '{{ . }}' +{{- end }} {{- range $fileName, $value := .Values.uploaderConfigMap.extraScripts }} {{- if not (empty $value) }} {{ $fileName | indent 2 -}}: | diff --git a/charts/selenium-grid/values.yaml b/charts/selenium-grid/values.yaml index 33139674f..57473f1b4 100644 --- a/charts/selenium-grid/values.yaml +++ b/charts/selenium-grid/values.yaml @@ -7,17 +7,18 @@ global: # Image registry for all selenium components imageRegistry: selenium # Image tag for all selenium components - imageTag: 4.17.0-20240123 + imageTag: 4.18.0-20240220 # Image tag for browser's nodes - nodesImageTag: 4.17.0-20240123 + nodesImageTag: 4.18.0-20240220 # Image tag for browser's video recorder - videoImageTag: ffmpeg-6.1-20240123 - # Image tag for browser's uploader - uploaderImageTag: rclone-1.65-20240123 + videoImageTag: ffmpeg-6.1-20240220 # Pull secret for all components, can be overridden individually imagePullSecret: "" # Log level for all components. Possible values describe here: https://www.selenium.dev/documentation/grid/configuration/cli_options/#logging logLevel: INFO + # Set default startup probe for all nodes (supplied values: httpGet, exec). If not set, the default is httpGet + # startup probe method `exec.command` is using a script is mounted from `nodeConfigMap.extraScripts.nodeProbe.sh` + defaultNodeStartupProbe: exec tls: enabled: false @@ -59,7 +60,7 @@ isolateComponents: false # Service Account for all components serviceAccount: create: true - name: "" + # nameOverride: annotations: {} # eks.amazonaws.com/role-arn: "arn:aws:iam::12345678:role/video-bucket-permissions" @@ -94,69 +95,70 @@ ingress: # ConfigMap that contains SE_EVENT_BUS_HOST, SE_EVENT_BUS_PUBLISH_PORT and SE_EVENT_BUS_SUBSCRIBE_PORT variables busConfigMap: # Name of the configmap - name: selenium-event-bus-config + # nameOverride: # Custom annotations for configmap annotations: {} # ConfigMap that contains common environment variables for browser nodes nodeConfigMap: - name: selenium-node-config + # nameOverride: # Default mode for ConfigMap is mounted as file defaultMode: 0755 # File name of preStop script in ConfigMap extraScriptsDirectory: "/opt/selenium" extraScripts: nodePreStop.sh: + nodeProbe.sh: # Name of volume mount is used to mount scripts in the ConfigMap - scriptVolumeMountName: node-helper-scripts + scriptVolumeMountName: # Custom annotations for configmap annotations: {} recorderConfigMap: - name: selenium-recorder-config + # nameOverride: # Default mode for ConfigMap is mounted as file defaultMode: 0755 # Directory where the extra scripts are mounted to extraScriptsDirectory: "/opt/bin" # List of extra scripts to be mounted to the container. Format as `filename: content` extraScripts: - video.sh: - graphQLRecordVideo.sh: + # video.sh: + # video_graphQLQuery.sh: # Name of volume mount is used to mount scripts in the ConfigMap - scriptVolumeMountName: recorder-helper-scripts + scriptVolumeMountName: videoVolumeMountName: videos # Custom annotations for configmap annotations: {} uploaderConfigMap: - name: selenium-uploader-config + # nameOverride: # Default mode for ConfigMap is mounted as file defaultMode: 0755 # Directory where the extra scripts are mounted to extraScriptsDirectory: "/opt/bin" # List of extra scripts to be mounted to the container. Format as `filename: content` extraScripts: - entry_point.sh: + upload.sh: # Extra files stored in Secret to be mounted to the container. secretFiles: - config.conf: + upload.conf: "[sample]" # Name of volume mount is used to mount scripts in the ConfigMap - scriptVolumeMountName: uploader-helper-scripts + scriptVolumeMountName: # Name of Secret is used to store the `secretFiles` - secretVolumeMountName: '{{ include "seleniumGrid.common.secrets" $ }}' + secretVolumeMountName: # Custom annotations for configmap annotations: {} # ConfigMap that contains common environment variables for Logging (https://www.selenium.dev/documentation/grid/configuration/cli_options/#logging) loggingConfigMap: - name: selenium-logging-config + # nameOverride: # Custom annotations for configmap annotations: {} # ConfigMap that contains common environment variables for Server (https://www.selenium.dev/documentation/grid/configuration/cli_options/#server) serverConfigMap: - name: selenium-server-config + # nameOverride: certVolumeMountPath: /etc/ssl/certs/selenium certificateFile: selenium.pem privateKeyFile: selenium.pkcs8 @@ -166,10 +168,10 @@ serverConfigMap: # Custom annotations for configmap annotations: {} -# Secrets for all components. Components environment variables contain sensitive data should be stored in secrets. +# Secrets for all components. Component environment variables contain sensitive data should be stored in secrets. secrets: create: true - name: selenium-secrets + # nameOverride: env: SE_VNC_PASSWORD: "secret" annotations: {} @@ -183,7 +185,7 @@ components: # Router image name imageName: router # Router image tag (this overwrites global.seleniumGrid.imageTag parameter) - # imageTag: 4.17.0-20240123 + # imageTag: 4.18.0-20240220 # Image pull policy (see https://kubernetes.io/docs/concepts/containers/images/#updating-images) imagePullPolicy: IfNotPresent @@ -245,7 +247,7 @@ components: # Distributor image name imageName: distributor # Distributor image tag (this overwrites global.seleniumGrid.imageTag parameter) - # imageTag: 4.17.0-20240123 + # imageTag: 4.18.0-20240220 # Image pull policy (see https://kubernetes.io/docs/concepts/containers/images/#updating-images) imagePullPolicy: IfNotPresent @@ -278,7 +280,7 @@ components: # Event Bus image name imageName: event-bus # Event Bus image tag (this overwrites global.seleniumGrid.imageTag parameter) - # imageTag: 4.17.0-20240123 + # imageTag: 4.18.0-20240220 # Image pull policy (see https://kubernetes.io/docs/concepts/containers/images/#updating-images) imagePullPolicy: IfNotPresent @@ -317,7 +319,7 @@ components: # Session Map image name imageName: sessions # Session Map image tag (this overwrites global.seleniumGrid.imageTag parameter) - # imageTag: 4.17.0-20240123 + # imageTag: 4.18.0-20240220 # Image pull policy (see https://kubernetes.io/docs/concepts/containers/images/#updating-images) imagePullPolicy: IfNotPresent @@ -348,7 +350,7 @@ components: # Session Queue image name imageName: session-queue # Session Queue image tag (this overwrites global.seleniumGrid.imageTag parameter) - # imageTag: 4.17.0-20240123 + # imageTag: 4.18.0-20240220 # Image pull policy (see https://kubernetes.io/docs/concepts/containers/images/#updating-images) imagePullPolicy: IfNotPresent @@ -402,7 +404,7 @@ hub: # Selenium Hub image name imageName: hub # Selenium Hub image tag (this overwrites global.seleniumGrid.imageTag parameter) - # imageTag: 4.17.0-20240123 + # imageTag: 4.18.0-20240220 # Image pull policy (see https://kubernetes.io/docs/concepts/containers/images/#updating-images) imagePullPolicy: IfNotPresent # Image pull secret (see https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/) @@ -494,6 +496,24 @@ hub: # Priority class name for selenium-hub pods priorityClassName: "" +tracing: + enabled: false + enabledWithExistingEndpoint: false + exporter: otlp + exporterEndpoint: 'http://{{ .Release.Name }}-jaeger-collector.{{ .Release.Namespace }}:4317' + globalAutoConfigure: true + ingress: + enabled: true + annotations: + paths: + - backend: + service: + name: '{{ .Release.Name }}-jaeger-query' + port: + number: 16686 + path: &jaegerBasePath "/jaeger" + pathType: Prefix + # Keda scaled object configuration autoscaling: # Enable autoscaling. Implies installing KEDA @@ -531,9 +551,9 @@ autoscaling: scaledObjectOptions: scaleTargetRef: kind: Deployment - # Define terminationGracePeriodSeconds for scalingType "deployment". Period for `deregisterLifecycle` to gracefully shut down the node before force killing it + # Define terminationGracePeriodSeconds for scalingType "deployment". Period for `deregisterLifecycle` to gracefully shut down the node before force terminating it terminationGracePeriodSeconds: 3600 - # Define preStop command to shuts down the node gracefully when scalingType is set to "deployment" + # Define preStop command to shut down the node gracefully when scalingType is set to "deployment" deregisterLifecycle: # preStop: # exec: @@ -546,7 +566,7 @@ chromeNode: # NOTE: Only used when autoscaling.enabled is false # Enable creation of Deployment - # true (default) - if you want long living pods + # true (default) - if you want long-living pods # false - for provisioning your own custom type such as Jobs deploymentEnabled: true @@ -556,7 +576,7 @@ chromeNode: # Image of chrome nodes imageName: node-chrome # Image of chrome nodes (this overwrites global.seleniumGrid.nodesImageTag) - # imageTag: 4.17.0-20240123 + # imageTag: 4.18.0-20240220 # Image pull policy (see https://kubernetes.io/docs/concepts/containers/images/#updating-images) imagePullPolicy: IfNotPresent # Image pull secret (see https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/) @@ -636,9 +656,9 @@ chromeNode: enabled: true path: /status initialDelaySeconds: 0 - failureThreshold: 10 - timeoutSeconds: 5 periodSeconds: 5 + timeoutSeconds: 125 + failureThreshold: 25 successThreshold: 1 # Readiness probe settings @@ -663,7 +683,7 @@ chromeNode: # Time to wait for pod termination terminationGracePeriodSeconds: 30 - # Define preStop command to shuts down the chrome node gracefully. This overwrites autoscaling.deregisterLifecycle + # Define preStop command to shut down the chrome node gracefully. This overwrites autoscaling.deregisterLifecycle deregisterLifecycle: # Define postStart and preStop events. This overwrites the defined preStop in deregisterLifecycle if any lifecycle: {} @@ -683,17 +703,17 @@ chromeNode: # scaledJobOptions: # scaledObjectOptions: hpa: - url: '{{ include "seleniumGrid.graphqlURL" . }}' + url: '{{ template "seleniumGrid.graphqlURL" . }}' browserName: chrome sessionBrowserName: 'chrome' # browserVersion: '91.0' # Optional. Only required when supporting multiple versions of browser in your Selenium Grid. - unsafeSsl: '{{ include "seleniumGrid.graphqlURL.unsafeSsl" . }}' # Optional + unsafeSsl: '{{ template "seleniumGrid.graphqlURL.unsafeSsl" . }}' # Optional # It is used to add initContainers in the same pod of the browser node. # It should be set using the --set-json option initContainers: [] - # It is used to add a sidecars proxy in the same pod of the browser node. + # It is used to add sidecars proxy in the same pod of the browser node. # It means it will add a new container to the deployment itself. # It should be set using the --set-json option sidecars: [] @@ -716,7 +736,7 @@ firefoxNode: # Image of firefox nodes imageName: node-firefox # Image of firefox nodes (this overwrites global.seleniumGrid.nodesImageTag) - # imageTag: 4.17.0-20240123 + # imageTag: 4.18.0-20240220 # Image pull policy (see https://kubernetes.io/docs/concepts/containers/images/#updating-images) imagePullPolicy: IfNotPresent # Image pull secret (see https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/) @@ -796,9 +816,9 @@ firefoxNode: enabled: true path: /status initialDelaySeconds: 0 - failureThreshold: 10 - timeoutSeconds: 5 periodSeconds: 5 + timeoutSeconds: 125 + failureThreshold: 25 successThreshold: 1 # Readiness probe settings @@ -843,16 +863,16 @@ firefoxNode: # scaledJobOptions: # scaledObjectOptions: hpa: - url: '{{ include "seleniumGrid.graphqlURL" . }}' + url: '{{ template "seleniumGrid.graphqlURL" . }}' browserName: firefox sessionBrowserName: 'firefox' - unsafeSsl: '{{ include "seleniumGrid.graphqlURL.unsafeSsl" . }}' # Optional + unsafeSsl: '{{ template "seleniumGrid.graphqlURL.unsafeSsl" . }}' # Optional # It is used to add initContainers in the same pod of the browser node. # It should be set using the --set-json option initContainers: [] - # It is used to add a sidecars proxy in the same pod of the browser node. + # It is used to add sidecars proxy in the same pod of the browser node. # It means it will add a new container to the deployment itself. # It should be set using the --set-json option sidecars: [] @@ -874,7 +894,7 @@ edgeNode: # Image of edge nodes imageName: node-edge # Image of edge nodes (this overwrites global.seleniumGrid.nodesImageTag) - # imageTag: 4.17.0-20240123 + # imageTag: 4.18.0-20240220 # Image pull policy (see https://kubernetes.io/docs/concepts/containers/images/#updating-images) imagePullPolicy: IfNotPresent # Image pull secret (see https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/) @@ -954,9 +974,9 @@ edgeNode: enabled: true path: /status initialDelaySeconds: 0 - failureThreshold: 10 - timeoutSeconds: 5 periodSeconds: 5 + timeoutSeconds: 125 + failureThreshold: 25 successThreshold: 1 # Readiness probe settings @@ -1001,16 +1021,16 @@ edgeNode: # scaledJobOptions: # scaledObjectOptions: hpa: - url: '{{ include "seleniumGrid.graphqlURL" . }}' + url: '{{ template "seleniumGrid.graphqlURL" . }}' browserName: MicrosoftEdge sessionBrowserName: 'msedge' - unsafeSsl: '{{ include "seleniumGrid.graphqlURL.unsafeSsl" . }}' # Optional + unsafeSsl: '{{ template "seleniumGrid.graphqlURL.unsafeSsl" . }}' # Optional # It is used to add initContainers in the same pod of the browser node. # It should be set using the --set-json option initContainers: [] - # It is used to add a sidecars proxy in the same pod of the browser node. + # It is used to add sidecars proxy in the same pod of the browser node. # It means it will add a new container to the deployment itself. # It should be set using the --set-json option sidecars: [] @@ -1021,7 +1041,7 @@ videoRecorder: # Image of video recorder imageName: video # Image of video recorder - # imageTag: ffmpeg-6.1-20240123 + # imageTag: ffmpeg-6.1-20240220 # Image pull policy (see https://kubernetes.io/docs/concepts/containers/images/#updating-images) imagePullPolicy: IfNotPresent # Image pull secret (see https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/) @@ -1031,10 +1051,9 @@ videoRecorder: # Where to upload the video file e.g. remoteName://bucketName/path. Refer to destination syntax of rclone https://rclone.org/docs/ destinationPrefix: # What uploader to use. See .videRecorder.rclone for how to create a new one. - name: "rclone" - configFileName: "config.conf" - entryPointFileName: "entry_point.sh" - scriptMountPath: "/opt/bin" + name: + configFileName: upload.conf + entryPointFileName: upload.sh # For environment variables used in uploader which contains sensitive information, store in secret and refer envFrom # Set config for rclone via ENV var with format: RCLONE_CONFIG_ + name of remote + _ + name of config file option (make it all uppercase) secrets: @@ -1066,7 +1085,6 @@ videoRecorder: cpu: "1" # SecurityContext for recorder container securityContext: - runAsUser: 0 extraEnvironmentVariables: # - name: SE_VIDEO_FOLDER # value: /videos @@ -1092,10 +1110,6 @@ videoRecorder: # Define lifecycle events for video recorder lifecycle: {} - volume: - # name: - # folder: video - # scripts: video-scripts # Custom video recorder back-end scripts (video.sh, video_ready.py, etc.) further by ConfigMap. # NOTE: For the mount point with the name "video", or "video-scripts", it will override the default. For other names, it will be appended. extraVolumeMounts: [] @@ -1115,24 +1129,6 @@ videoRecorder: # persistentVolumeClaim: # claimName: video-pv-claim - # Container spec for the uploader if above it is defined as "uploader.name: rclone" - rclone: - # imageRegistry: selenium - imageName: uploader - # imageTag: "rclone-1.65-20240119" - imagePullPolicy: IfNotPresent - # SecurityContext for uploader container - securityContext: - runAsUser: 0 - command: [] - args: [] - extraEnvironmentVariables: - # Extra options for rclone. Refer to https://rclone.org/flags - - name: UPLOAD_OPTS - value: "--progress" - # - name: UPLOAD_COMMAND - # value: "copy" - # Container spec for the uploader if above it is defined as "uploader.name: s3" s3: imageRegistry: public.ecr.aws @@ -1172,3 +1168,21 @@ ingress-nginx: controller: admissionWebhooks: enabled: false + +# Configuration for dependency chart jaeger +jaeger: + provisionDataStore: + cassandra: false + allInOne: + enabled: true + extraEnv: + - name: QUERY_BASE_PATH + value: *jaegerBasePath + storage: + type: none + agent: + enabled: false + collector: + enabled: false + query: + enabled: false diff --git a/docker-compose-v2-tracing.yml b/docker-compose-v2-tracing.yml index 5cdb207b6..1db71fb5f 100644 --- a/docker-compose-v2-tracing.yml +++ b/docker-compose-v2-tracing.yml @@ -3,13 +3,13 @@ # To stop the execution, hit Ctrl+C, and then `docker-compose -f docker-compose-v2-tracing.yml down` version: '2' services: - jaegar: - image: jaegertracing/all-in-one:1.17 + jaeger: + image: jaegertracing/all-in-one:1.54 ports: - "16686:16686" - - "14250:14250" + - "4317:4317" chrome: - image: selenium/node-chrome:4.17.0-20240123 + image: selenium/node-chrome:4.18.0-20240220 shm_size: 2gb depends_on: - selenium-hub @@ -18,12 +18,13 @@ services: - SE_EVENT_BUS_PUBLISH_PORT=4442 - SE_EVENT_BUS_SUBSCRIBE_PORT=4443 - SE_ENABLE_TRACING=true - - JAVA_OPTS=-Dotel.traces.exporter=jaeger -Dotel.exporter.jaeger.endpoint=http://jaegar:14250 -Dotel.resource.attributes=service.name=selenium-node-chrome + - SE_OTEL_TRACES_EXPORTER=otlp + - SE_OTEL_EXPORTER_ENDPOINT=http://jaeger:4317 ports: - "6900:5900" edge: - image: selenium/node-edge:4.17.0-20240123 + image: selenium/node-edge:4.18.0-20240220 shm_size: 2gb depends_on: - selenium-hub @@ -32,12 +33,13 @@ services: - SE_EVENT_BUS_PUBLISH_PORT=4442 - SE_EVENT_BUS_SUBSCRIBE_PORT=4443 - SE_ENABLE_TRACING=true - - JAVA_OPTS=-Dotel.traces.exporter=jaeger -Dotel.exporter.jaeger.endpoint=http://jaegar:14250 -Dotel.resource.attributes=service.name=selenium-node-edge + - SE_OTEL_TRACES_EXPORTER=otlp + - SE_OTEL_EXPORTER_ENDPOINT=http://jaeger:4317 ports: - "6901:5900" firefox: - image: selenium/node-firefox:4.17.0-20240123 + image: selenium/node-firefox:4.18.0-20240220 shm_size: 2gb depends_on: - selenium-hub @@ -46,18 +48,20 @@ services: - SE_EVENT_BUS_PUBLISH_PORT=4442 - SE_EVENT_BUS_SUBSCRIBE_PORT=4443 - SE_ENABLE_TRACING=true - - JAVA_OPTS=-Dotel.traces.exporter=jaeger -Dotel.exporter.jaeger.endpoint=http://jaegar:14250 -Dotel.resource.attributes=service.name=selenium-node-firefox + - SE_OTEL_TRACES_EXPORTER=otlp + - SE_OTEL_EXPORTER_ENDPOINT=http://jaeger:4317 ports: - "6902:5900" selenium-hub: - image: selenium/hub:4.17.0-20240123 + image: selenium/hub:4.18.0-20240220 ports: - "4442:4442" - "4443:4443" - "4444:4444" depends_on: - - jaegar + - jaeger environment: - SE_ENABLE_TRACING=true - - JAVA_OPTS=-Dotel.traces.exporter=jaeger -Dotel.exporter.jaeger.endpoint=http://jaegar:14250 -Dotel.resource.attributes=service.name=selenium-hub + - SE_OTEL_TRACES_EXPORTER=otlp + - SE_OTEL_EXPORTER_ENDPOINT=http://jaeger:4317 diff --git a/docker-compose-v2.yml b/docker-compose-v2.yml index 69b904718..540c8fb08 100644 --- a/docker-compose-v2.yml +++ b/docker-compose-v2.yml @@ -4,7 +4,7 @@ version: '2' services: chrome: - image: selenium/node-chrome:4.17.0-20240123 + image: selenium/node-chrome:4.18.0-20240220 shm_size: 2gb depends_on: - selenium-hub @@ -16,7 +16,7 @@ services: - "6900:5900" edge: - image: selenium/node-edge:4.17.0-20240123 + image: selenium/node-edge:4.18.0-20240220 shm_size: 2gb depends_on: - selenium-hub @@ -28,7 +28,7 @@ services: - "6901:5900" firefox: - image: selenium/node-firefox:4.17.0-20240123 + image: selenium/node-firefox:4.18.0-20240220 shm_size: 2gb depends_on: - selenium-hub @@ -40,7 +40,7 @@ services: - "6902:5900" selenium-hub: - image: selenium/hub:4.17.0-20240123 + image: selenium/hub:4.18.0-20240220 ports: - "4442:4442" - "4443:4443" diff --git a/docker-compose-v3-basicauth.yml b/docker-compose-v3-basicauth.yml index 6d24ba9ae..fda6e5039 100644 --- a/docker-compose-v3-basicauth.yml +++ b/docker-compose-v3-basicauth.yml @@ -4,7 +4,7 @@ version: "3" services: chrome: - image: selenium/node-chrome:4.17.0-20240123 + image: selenium/node-chrome:4.18.0-20240220 shm_size: 2gb depends_on: - selenium-hub @@ -14,7 +14,7 @@ services: - SE_EVENT_BUS_SUBSCRIBE_PORT=4443 edge: - image: selenium/node-edge:4.17.0-20240123 + image: selenium/node-edge:4.18.0-20240220 shm_size: 2gb depends_on: - selenium-hub @@ -24,7 +24,7 @@ services: - SE_EVENT_BUS_SUBSCRIBE_PORT=4443 firefox: - image: selenium/node-firefox:4.17.0-20240123 + image: selenium/node-firefox:4.18.0-20240220 shm_size: 2gb depends_on: - selenium-hub @@ -34,7 +34,7 @@ services: - SE_EVENT_BUS_SUBSCRIBE_PORT=4443 selenium-hub: - image: selenium/hub:4.17.0-20240123 + image: selenium/hub:4.18.0-20240220 container_name: selenium-hub ports: - "4442:4442" diff --git a/docker-compose-v3-dev.yml b/docker-compose-v3-dev.yml index fa40c2b2d..0e4d61a20 100644 --- a/docker-compose-v3-dev.yml +++ b/docker-compose-v3-dev.yml @@ -4,7 +4,7 @@ version: "3" services: chrome: - image: selenium/node-chrome:4.17.0-20240123 + image: selenium/node-chrome:4.18.0-20240220 shm_size: 2gb volumes: - ./selenium_server_deploy.jar:/opt/selenium/selenium-server.jar @@ -16,7 +16,7 @@ services: - SE_EVENT_BUS_SUBSCRIBE_PORT=4443 edge: - image: selenium/node-edge:4.17.0-20240123 + image: selenium/node-edge:4.18.0-20240220 shm_size: 2gb volumes: - ./selenium_server_deploy.jar:/opt/selenium/selenium-server.jar @@ -28,7 +28,7 @@ services: - SE_EVENT_BUS_SUBSCRIBE_PORT=4443 firefox: - image: selenium/node-firefox:4.17.0-20240123 + image: selenium/node-firefox:4.18.0-20240220 shm_size: 2gb volumes: - ./selenium_server_deploy.jar:/opt/selenium/selenium-server.jar @@ -40,7 +40,7 @@ services: - SE_EVENT_BUS_SUBSCRIBE_PORT=4443 selenium-hub: - image: selenium/hub:4.17.0-20240123 + image: selenium/hub:4.18.0-20240220 container_name: selenium-hub volumes: - ./selenium_server_deploy.jar:/opt/selenium/selenium-server.jar diff --git a/docker-compose-v3-dynamic-grid.yml b/docker-compose-v3-dynamic-grid.yml index 8edc27303..a949338dd 100644 --- a/docker-compose-v3-dynamic-grid.yml +++ b/docker-compose-v3-dynamic-grid.yml @@ -4,7 +4,7 @@ version: "3" services: node-docker: - image: selenium/node-docker:4.17.0-20240123 + image: selenium/node-docker:4.18.0-20240220 volumes: - ./assets:/opt/selenium/assets - ./NodeDocker/config.toml:/opt/bin/config.toml @@ -17,7 +17,7 @@ services: - SE_EVENT_BUS_SUBSCRIBE_PORT=4443 selenium-hub: - image: selenium/hub:4.17.0-20240123 + image: selenium/hub:4.18.0-20240220 container_name: selenium-hub ports: - "4442:4442" diff --git a/docker-compose-v3-full-grid-dev.yml b/docker-compose-v3-full-grid-dev.yml index c79948cc8..0ba2cf250 100644 --- a/docker-compose-v3-full-grid-dev.yml +++ b/docker-compose-v3-full-grid-dev.yml @@ -4,7 +4,7 @@ version: "3" services: selenium-event-bus: - image: selenium/event-bus:4.17.0-20240123 + image: selenium/event-bus:4.18.0-20240220 volumes: - ./selenium_server_deploy.jar:/opt/selenium/selenium-server.jar container_name: selenium-event-bus @@ -14,7 +14,7 @@ services: - "5557:5557" selenium-sessions: - image: selenium/sessions:4.17.0-20240123 + image: selenium/sessions:4.18.0-20240220 volumes: - ./selenium_server_deploy.jar:/opt/selenium/selenium-server.jar container_name: selenium-sessions @@ -28,7 +28,7 @@ services: - SE_EVENT_BUS_SUBSCRIBE_PORT=4443 selenium-session-queue: - image: selenium/session-queue:4.17.0-20240123 + image: selenium/session-queue:4.18.0-20240220 volumes: - ./selenium_server_deploy.jar:/opt/selenium/selenium-server.jar container_name: selenium-session-queue @@ -36,7 +36,7 @@ services: - "5559:5559" selenium-distributor: - image: selenium/distributor:4.17.0-20240123 + image: selenium/distributor:4.18.0-20240220 volumes: - ./selenium_server_deploy.jar:/opt/selenium/selenium-server.jar container_name: selenium-distributor @@ -56,7 +56,7 @@ services: - SE_SESSION_QUEUE_PORT=5559 selenium-router: - image: selenium/router:4.17.0-20240123 + image: selenium/router:4.18.0-20240220 volumes: - ./selenium_server_deploy.jar:/opt/selenium/selenium-server.jar container_name: selenium-router @@ -75,7 +75,7 @@ services: - SE_SESSION_QUEUE_PORT=5559 chrome: - image: selenium/node-chrome:4.17.0-20240123 + image: selenium/node-chrome:4.18.0-20240220 shm_size: 2gb volumes: - ./selenium_server_deploy.jar:/opt/selenium/selenium-server.jar @@ -87,7 +87,7 @@ services: - SE_EVENT_BUS_SUBSCRIBE_PORT=4443 edge: - image: selenium/node-edge:4.17.0-20240123 + image: selenium/node-edge:4.18.0-20240220 shm_size: 2gb volumes: - ./selenium_server_deploy.jar:/opt/selenium/selenium-server.jar @@ -99,7 +99,7 @@ services: - SE_EVENT_BUS_SUBSCRIBE_PORT=4443 firefox: - image: selenium/node-firefox:4.17.0-20240123 + image: selenium/node-firefox:4.18.0-20240220 shm_size: 2gb volumes: - ./selenium_server_deploy.jar:/opt/selenium/selenium-server.jar diff --git a/docker-compose-v3-full-grid-swarm.yml b/docker-compose-v3-full-grid-swarm.yml index e7df09dba..453961e80 100644 --- a/docker-compose-v3-full-grid-swarm.yml +++ b/docker-compose-v3-full-grid-swarm.yml @@ -7,7 +7,7 @@ version: '3.7' services: chrome: - image: selenium/node-chrome:4.17.0-20240123 + image: selenium/node-chrome:4.18.0-20240220 shm_size: 2gb environment: - SE_EVENT_BUS_HOST=selenium-hub @@ -18,7 +18,7 @@ services: entrypoint: bash -c 'SE_OPTS="--host $$HOSTNAME" /opt/bin/entry_point.sh' edge: - image: selenium/node-edge:4.17.0-20240123 + image: selenium/node-edge:4.18.0-20240220 shm_size: 2gb environment: - SE_EVENT_BUS_HOST=selenium-hub @@ -29,7 +29,7 @@ services: entrypoint: bash -c 'SE_OPTS="--host $$HOSTNAME" /opt/bin/entry_point.sh' firefox: - image: selenium/node-firefox:4.17.0-20240123 + image: selenium/node-firefox:4.18.0-20240220 shm_size: 2gb environment: - SE_EVENT_BUS_HOST=selenium-hub @@ -40,7 +40,7 @@ services: entrypoint: bash -c 'SE_OPTS="--host $$HOSTNAME" /opt/bin/entry_point.sh' selenium-hub: - image: selenium/hub:4.17.0-20240123 + image: selenium/hub:4.18.0-20240220 ports: - "4442:4442" - "4443:4443" diff --git a/docker-compose-v3-full-grid-tracing.yml b/docker-compose-v3-full-grid-tracing.yml index 5ddfd0cee..bbf9f4433 100644 --- a/docker-compose-v3-full-grid-tracing.yml +++ b/docker-compose-v3-full-grid-tracing.yml @@ -3,25 +3,26 @@ # To stop the execution, hit Ctrl+C, and then `docker-compose -f docker-compose-v3-full-grid-tracing.yml down` version: "3" services: - jaegar: - image: jaegertracing/all-in-one:1.17 + jaeger: + image: jaegertracing/all-in-one:1.54 ports: - "16686:16686" - - "14250:14250" + - "4317:4317" selenium-event-bus: - image: selenium/event-bus:4.17.0-20240123 + image: selenium/event-bus:4.18.0-20240220 container_name: selenium-event-bus ports: - "4442:4442" - "4443:4443" - "5557:5557" depends_on: - - jaegar + - jaeger environment: - SE_ENABLE_TRACING=true - - JAVA_OPTS=-Dotel.traces.exporter=jaeger -Dotel.exporter.jaeger.endpoint=http://jaegar:14250 -Dotel.resource.attributes=service.name=selenium-event-bus + - SE_OTEL_TRACES_EXPORTER=otlp + - SE_OTEL_EXPORTER_ENDPOINT=http://jaeger:4317 selenium-sessions: - image: selenium/sessions:4.17.0-20240123 + image: selenium/sessions:4.18.0-20240220 container_name: selenium-sessions ports: - "5556:5556" @@ -32,17 +33,19 @@ services: - SE_EVENT_BUS_PUBLISH_PORT=4442 - SE_EVENT_BUS_SUBSCRIBE_PORT=4443 - SE_ENABLE_TRACING=true - - JAVA_OPTS=-Dotel.traces.exporter=jaeger -Dotel.exporter.jaeger.endpoint=http://jaegar:14250 -Dotel.resource.attributes=service.name=selenium-sessions + - SE_OTEL_TRACES_EXPORTER=otlp + - SE_OTEL_EXPORTER_ENDPOINT=http://jaeger:4317 selenium-session-queue: - image: selenium/session-queue:4.17.0-20240123 + image: selenium/session-queue:4.18.0-20240220 container_name: selenium-session-queue ports: - "5559:5559" environment: - SE_ENABLE_TRACING=true - - JAVA_OPTS=-Dotel.traces.exporter=jaeger -Dotel.exporter.jaeger.endpoint=http://jaegar:14250 -Dotel.resource.attributes=service.name=selenium-session-queue + - SE_OTEL_TRACES_EXPORTER=otlp + - SE_OTEL_EXPORTER_ENDPOINT=http://jaeger:4317 selenium-distributor: - image: selenium/distributor:4.17.0-20240123 + image: selenium/distributor:4.18.0-20240220 container_name: selenium-distributor ports: - "5553:5553" @@ -59,9 +62,10 @@ services: - SE_SESSION_QUEUE_HOST=selenium-session-queue - SE_SESSION_QUEUE_PORT=5559 - SE_ENABLE_TRACING=true - - JAVA_OPTS=-Dotel.traces.exporter=jaeger -Dotel.exporter.jaeger.endpoint=http://jaegar:14250 -Dotel.resource.attributes=service.name=selenium-distributor + - SE_OTEL_TRACES_EXPORTER=otlp + - SE_OTEL_EXPORTER_ENDPOINT=http://jaeger:4317 selenium-router: - image: selenium/router:4.17.0-20240123 + image: selenium/router:4.18.0-20240220 container_name: selenium-router ports: - "4444:4444" @@ -76,10 +80,11 @@ services: - SE_SESSIONS_MAP_PORT=5556 - SE_SESSION_QUEUE_HOST=selenium-session-queue - SE_SESSION_QUEUE_PORT=5559 - - SE_ENABLE_TRACING=true - - JAVA_OPTS=-Dotel.traces.exporter=jaeger -Dotel.exporter.jaeger.endpoint=http://jaegar:14250 -Dotel.resource.attributes=service.name=selenium-router + - SE_ENABLE_TRACING=true + - SE_OTEL_TRACES_EXPORTER=otlp + - SE_OTEL_EXPORTER_ENDPOINT=http://jaeger:4317 chrome: - image: selenium/node-chrome:4.17.0-20240123 + image: selenium/node-chrome:4.18.0-20240220 shm_size: 2gb depends_on: - selenium-event-bus @@ -88,9 +93,10 @@ services: - SE_EVENT_BUS_PUBLISH_PORT=4442 - SE_EVENT_BUS_SUBSCRIBE_PORT=4443 - SE_ENABLE_TRACING=true - - JAVA_OPTS=-Dotel.traces.exporter=jaeger -Dotel.exporter.jaeger.endpoint=http://jaegar:14250 -Dotel.resource.attributes=service.name=selenium-node-chrome + - SE_OTEL_TRACES_EXPORTER=otlp + - SE_OTEL_EXPORTER_ENDPOINT=http://jaeger:4317 edge: - image: selenium/node-edge:4.17.0-20240123 + image: selenium/node-edge:4.18.0-20240220 shm_size: 2gb depends_on: - selenium-event-bus @@ -98,10 +104,11 @@ services: - SE_EVENT_BUS_HOST=selenium-event-bus - SE_EVENT_BUS_PUBLISH_PORT=4442 - SE_EVENT_BUS_SUBSCRIBE_PORT=4443 - - SE_ENABLE_TRACING=true - - JAVA_OPTS=-Dotel.traces.exporter=jaeger -Dotel.exporter.jaeger.endpoint=http://jaegar:14250 -Dotel.resource.attributes=service.name=selenium-node-edge + - SE_ENABLE_TRACING=true + - SE_OTEL_TRACES_EXPORTER=otlp + - SE_OTEL_EXPORTER_ENDPOINT=http://jaeger:4317 firefox: - image: selenium/node-firefox:4.17.0-20240123 + image: selenium/node-firefox:4.18.0-20240220 shm_size: 2gb depends_on: - selenium-event-bus @@ -110,4 +117,5 @@ services: - SE_EVENT_BUS_PUBLISH_PORT=4442 - SE_EVENT_BUS_SUBSCRIBE_PORT=4443 - SE_ENABLE_TRACING=true - - JAVA_OPTS=-Dotel.traces.exporter=jaeger -Dotel.exporter.jaeger.endpoint=http://jaegar:14250 -Dotel.resource.attributes=service.name=selenium-node-firefox \ No newline at end of file + - SE_OTEL_TRACES_EXPORTER=otlp + - SE_OTEL_EXPORTER_ENDPOINT=http://jaeger:4317 diff --git a/docker-compose-v3-full-grid.yml b/docker-compose-v3-full-grid.yml index b97282e4f..db818c348 100644 --- a/docker-compose-v3-full-grid.yml +++ b/docker-compose-v3-full-grid.yml @@ -4,7 +4,7 @@ version: "3" services: selenium-event-bus: - image: selenium/event-bus:4.17.0-20240123 + image: selenium/event-bus:4.18.0-20240220 container_name: selenium-event-bus ports: - "4442:4442" @@ -12,7 +12,7 @@ services: - "5557:5557" selenium-sessions: - image: selenium/sessions:4.17.0-20240123 + image: selenium/sessions:4.18.0-20240220 container_name: selenium-sessions ports: - "5556:5556" @@ -24,13 +24,13 @@ services: - SE_EVENT_BUS_SUBSCRIBE_PORT=4443 selenium-session-queue: - image: selenium/session-queue:4.17.0-20240123 + image: selenium/session-queue:4.18.0-20240220 container_name: selenium-session-queue ports: - "5559:5559" selenium-distributor: - image: selenium/distributor:4.17.0-20240123 + image: selenium/distributor:4.18.0-20240220 container_name: selenium-distributor ports: - "5553:5553" @@ -48,7 +48,7 @@ services: - SE_SESSION_QUEUE_PORT=5559 selenium-router: - image: selenium/router:4.17.0-20240123 + image: selenium/router:4.18.0-20240220 container_name: selenium-router ports: - "4444:4444" @@ -65,7 +65,7 @@ services: - SE_SESSION_QUEUE_PORT=5559 chrome: - image: selenium/node-chrome:4.17.0-20240123 + image: selenium/node-chrome:4.18.0-20240220 shm_size: 2gb depends_on: - selenium-event-bus @@ -75,7 +75,7 @@ services: - SE_EVENT_BUS_SUBSCRIBE_PORT=4443 edge: - image: selenium/node-edge:4.17.0-20240123 + image: selenium/node-edge:4.18.0-20240220 shm_size: 2gb depends_on: - selenium-event-bus @@ -85,7 +85,7 @@ services: - SE_EVENT_BUS_SUBSCRIBE_PORT=4443 firefox: - image: selenium/node-firefox:4.17.0-20240123 + image: selenium/node-firefox:4.18.0-20240220 shm_size: 2gb depends_on: - selenium-event-bus diff --git a/docker-compose-v3-swarm.yml b/docker-compose-v3-swarm.yml index e7df09dba..453961e80 100644 --- a/docker-compose-v3-swarm.yml +++ b/docker-compose-v3-swarm.yml @@ -7,7 +7,7 @@ version: '3.7' services: chrome: - image: selenium/node-chrome:4.17.0-20240123 + image: selenium/node-chrome:4.18.0-20240220 shm_size: 2gb environment: - SE_EVENT_BUS_HOST=selenium-hub @@ -18,7 +18,7 @@ services: entrypoint: bash -c 'SE_OPTS="--host $$HOSTNAME" /opt/bin/entry_point.sh' edge: - image: selenium/node-edge:4.17.0-20240123 + image: selenium/node-edge:4.18.0-20240220 shm_size: 2gb environment: - SE_EVENT_BUS_HOST=selenium-hub @@ -29,7 +29,7 @@ services: entrypoint: bash -c 'SE_OPTS="--host $$HOSTNAME" /opt/bin/entry_point.sh' firefox: - image: selenium/node-firefox:4.17.0-20240123 + image: selenium/node-firefox:4.18.0-20240220 shm_size: 2gb environment: - SE_EVENT_BUS_HOST=selenium-hub @@ -40,7 +40,7 @@ services: entrypoint: bash -c 'SE_OPTS="--host $$HOSTNAME" /opt/bin/entry_point.sh' selenium-hub: - image: selenium/hub:4.17.0-20240123 + image: selenium/hub:4.18.0-20240220 ports: - "4442:4442" - "4443:4443" diff --git a/docker-compose-v3-tracing.yml b/docker-compose-v3-tracing.yml index 7b099880f..dd1cd917a 100644 --- a/docker-compose-v3-tracing.yml +++ b/docker-compose-v3-tracing.yml @@ -3,13 +3,13 @@ # To stop the execution, hit Ctrl+C, and then `docker-compose -f docker-compose-v3-tracing.yml down` version: "3" services: - jaegar: - image: jaegertracing/all-in-one:1.17 + jaeger: + image: jaegertracing/all-in-one:1.54 ports: - "16686:16686" - - "14250:14250" + - "4317:4317" chrome: - image: selenium/node-chrome:4.17.0-20240123 + image: selenium/node-chrome:4.18.0-20240220 shm_size: 2gb depends_on: - selenium-hub @@ -18,10 +18,11 @@ services: - SE_EVENT_BUS_PUBLISH_PORT=4442 - SE_EVENT_BUS_SUBSCRIBE_PORT=4443 - SE_ENABLE_TRACING=true - - JAVA_OPTS=-Dotel.traces.exporter=jaeger -Dotel.exporter.jaeger.endpoint=http://jaegar:14250 -Dotel.resource.attributes=service.name=selenium-node-chrome + - SE_OTEL_TRACES_EXPORTER=otlp + - SE_OTEL_EXPORTER_ENDPOINT=http://jaeger:4317 edge: - image: selenium/node-edge:4.17.0-20240123 + image: selenium/node-edge:4.18.0-20240220 shm_size: 2gb depends_on: - selenium-hub @@ -30,10 +31,11 @@ services: - SE_EVENT_BUS_PUBLISH_PORT=4442 - SE_EVENT_BUS_SUBSCRIBE_PORT=4443 - SE_ENABLE_TRACING=true - - JAVA_OPTS=-Dotel.traces.exporter=jaeger -Dotel.exporter.jaeger.endpoint=http://jaegar:14250 -Dotel.resource.attributes=service.name=selenium-node-edge + - SE_OTEL_TRACES_EXPORTER=otlp + - SE_OTEL_EXPORTER_ENDPOINT=http://jaeger:4317 firefox: - image: selenium/node-firefox:4.17.0-20240123 + image: selenium/node-firefox:4.18.0-20240220 shm_size: 2gb depends_on: - selenium-hub @@ -42,17 +44,19 @@ services: - SE_EVENT_BUS_PUBLISH_PORT=4442 - SE_EVENT_BUS_SUBSCRIBE_PORT=4443 - SE_ENABLE_TRACING=true - - JAVA_OPTS=-Dotel.traces.exporter=jaeger -Dotel.exporter.jaeger.endpoint=http://jaegar:14250 -Dotel.resource.attributes=service.name=selenium-node-firefox + - SE_OTEL_TRACES_EXPORTER=otlp + - SE_OTEL_EXPORTER_ENDPOINT=http://jaeger:4317 selenium-hub: - image: selenium/hub:4.17.0-20240123 + image: selenium/hub:4.18.0-20240220 container_name: selenium-hub ports: - "4442:4442" - "4443:4443" - "4444:4444" depends_on: - - jaegar + - jaeger environment: - SE_ENABLE_TRACING=true - - JAVA_OPTS=-Dotel.traces.exporter=jaeger -Dotel.exporter.jaeger.endpoint=http://jaegar:14250 -Dotel.resource.attributes=service.name=selenium-hub + - SE_OTEL_TRACES_EXPORTER=otlp + - SE_OTEL_EXPORTER_ENDPOINT=http://jaeger:4317 diff --git a/docker-compose-v3-video-upload.yml b/docker-compose-v3-video-upload.yml new file mode 100644 index 000000000..10c096318 --- /dev/null +++ b/docker-compose-v3-video-upload.yml @@ -0,0 +1,103 @@ +# To execute this docker-compose yml file use `docker-compose -f docker-compose-v3-video-upload.yml up` +# Add the `-d` flag at the end for detached execution +# To stop the execution, hit Ctrl+C, and then `docker-compose -f docker-compose-v3-video-upload.yml down` +# ${variable_pattern} get value from .env in the same directory +version: "3" +services: + chrome: + image: selenium/node-chrome:4.18.0-20240220 + shm_size: 2gb + depends_on: + - selenium-hub + environment: + - SE_EVENT_BUS_HOST=selenium-hub + - SE_EVENT_BUS_PUBLISH_PORT=4442 + - SE_EVENT_BUS_SUBSCRIBE_PORT=4443 + + edge: + image: selenium/node-edge:4.18.0-20240220 + shm_size: 2gb + depends_on: + - selenium-hub + environment: + - SE_EVENT_BUS_HOST=selenium-hub + - SE_EVENT_BUS_PUBLISH_PORT=4442 + - SE_EVENT_BUS_SUBSCRIBE_PORT=4443 + + firefox: + image: selenium/node-firefox:4.18.0-20240220 + shm_size: 2gb + depends_on: + - selenium-hub + environment: + - SE_EVENT_BUS_HOST=selenium-hub + - SE_EVENT_BUS_PUBLISH_PORT=4442 + - SE_EVENT_BUS_SUBSCRIBE_PORT=4443 + + chrome_video: + image: selenium/video:ffmpeg-6.1-20240220 + depends_on: + - chrome + environment: + - DISPLAY_CONTAINER_NAME=chrome + - SE_VIDEO_FILE_NAME=auto + - SE_VIDEO_UPLOAD_ENABLED=true + - SE_VIDEO_INTERNAL_UPLOAD=true + - SE_UPLOAD_DESTINATION_PREFIX=s3://${BUCKET_NAME} + - RCLONE_CONFIG_S3_TYPE=s3 + - RCLONE_CONFIG_S3_PROVIDER=GCS + - RCLONE_CONFIG_S3_ENV_AUTH=true + - RCLONE_CONFIG_S3_REGION=asia-southeast1 + - RCLONE_CONFIG_S3_LOCATION_CONSTRAINT=asia-southeast1 + - RCLONE_CONFIG_S3_ACL=private + - RCLONE_CONFIG_S3_ACCESS_KEY_ID=${GS_ACCESS_KEY_ID} + - RCLONE_CONFIG_S3_SECRET_ACCESS_KEY=${GS_SECRET_ACCESS_KEY} + - RCLONE_CONFIG_S3_ENDPOINT=https://storage.googleapis.com + + edge_video: + image: selenium/video:ffmpeg-6.1-20240220 + depends_on: + - edge + environment: + - DISPLAY_CONTAINER_NAME=edge + - SE_VIDEO_FILE_NAME=auto + - SE_VIDEO_UPLOAD_ENABLED=true + - SE_VIDEO_INTERNAL_UPLOAD=true + - SE_UPLOAD_DESTINATION_PREFIX=s3://${BUCKET_NAME} + - RCLONE_CONFIG_S3_TYPE=s3 + - RCLONE_CONFIG_S3_PROVIDER=GCS + - RCLONE_CONFIG_S3_ENV_AUTH=true + - RCLONE_CONFIG_S3_REGION=asia-southeast1 + - RCLONE_CONFIG_S3_LOCATION_CONSTRAINT=asia-southeast1 + - RCLONE_CONFIG_S3_ACL=private + - RCLONE_CONFIG_S3_ACCESS_KEY_ID=${GS_ACCESS_KEY_ID} + - RCLONE_CONFIG_S3_SECRET_ACCESS_KEY=${GS_SECRET_ACCESS_KEY} + - RCLONE_CONFIG_S3_ENDPOINT=https://storage.googleapis.com + + firefox_video: + image: selenium/video:ffmpeg-6.1-20240220 + depends_on: + - firefox + environment: + - DISPLAY_CONTAINER_NAME=firefox + - SE_VIDEO_FILE_NAME=auto + - SE_VIDEO_UPLOAD_ENABLED=true + - SE_VIDEO_INTERNAL_UPLOAD=true + - SE_UPLOAD_DESTINATION_PREFIX=s3://${BUCKET_NAME} + - RCLONE_CONFIG_S3_TYPE=s3 + - RCLONE_CONFIG_S3_PROVIDER=GCS + - RCLONE_CONFIG_S3_ENV_AUTH=true + - RCLONE_CONFIG_S3_REGION=asia-southeast1 + - RCLONE_CONFIG_S3_LOCATION_CONSTRAINT=asia-southeast1 + - RCLONE_CONFIG_S3_ACL=private + - RCLONE_CONFIG_S3_ACCESS_KEY_ID=${GS_ACCESS_KEY_ID} + - RCLONE_CONFIG_S3_SECRET_ACCESS_KEY=${GS_SECRET_ACCESS_KEY} + - RCLONE_CONFIG_S3_ENDPOINT=https://storage.googleapis.com + + selenium-hub: + image: selenium/hub:4.18.0-20240220 + container_name: selenium-hub + ports: + - "4442:4442" + - "4443:4443" + - "4444:4444" diff --git a/docker-compose-v3-video.yml b/docker-compose-v3-video.yml index dd6199369..60b04298c 100644 --- a/docker-compose-v3-video.yml +++ b/docker-compose-v3-video.yml @@ -4,7 +4,7 @@ version: "3" services: chrome: - image: selenium/node-chrome:4.17.0-20240123 + image: selenium/node-chrome:4.18.0-20240220 shm_size: 2gb depends_on: - selenium-hub @@ -14,7 +14,7 @@ services: - SE_EVENT_BUS_SUBSCRIBE_PORT=4443 edge: - image: selenium/node-edge:4.17.0-20240123 + image: selenium/node-edge:4.18.0-20240220 shm_size: 2gb depends_on: - selenium-hub @@ -24,7 +24,7 @@ services: - SE_EVENT_BUS_SUBSCRIBE_PORT=4443 firefox: - image: selenium/node-firefox:4.17.0-20240123 + image: selenium/node-firefox:4.18.0-20240220 shm_size: 2gb depends_on: - selenium-hub @@ -34,7 +34,7 @@ services: - SE_EVENT_BUS_SUBSCRIBE_PORT=4443 chrome_video: - image: selenium/video:ffmpeg-6.1-20240123 + image: selenium/video:ffmpeg-6.1-20240220 volumes: - /tmp/videos:/videos depends_on: @@ -44,7 +44,7 @@ services: - FILE_NAME=chrome_video.mp4 edge_video: - image: selenium/video:ffmpeg-6.1-20240123 + image: selenium/video:ffmpeg-6.1-20240220 volumes: - /tmp/videos:/videos depends_on: @@ -54,7 +54,7 @@ services: - FILE_NAME=edge_video.mp4 firefox_video: - image: selenium/video:ffmpeg-6.1-20240123 + image: selenium/video:ffmpeg-6.1-20240220 volumes: - /tmp/videos:/videos depends_on: @@ -64,7 +64,7 @@ services: - FILE_NAME=firefox_video.mp4 selenium-hub: - image: selenium/hub:4.17.0-20240123 + image: selenium/hub:4.18.0-20240220 container_name: selenium-hub ports: - "4442:4442" diff --git a/docker-compose-v3.yml b/docker-compose-v3.yml index 97d111701..90a119da1 100644 --- a/docker-compose-v3.yml +++ b/docker-compose-v3.yml @@ -4,7 +4,7 @@ version: "3" services: chrome: - image: selenium/node-chrome:4.17.0-20240123 + image: selenium/node-chrome:4.18.0-20240220 shm_size: 2gb depends_on: - selenium-hub @@ -14,7 +14,7 @@ services: - SE_EVENT_BUS_SUBSCRIBE_PORT=4443 edge: - image: selenium/node-edge:4.17.0-20240123 + image: selenium/node-edge:4.18.0-20240220 shm_size: 2gb depends_on: - selenium-hub @@ -24,7 +24,7 @@ services: - SE_EVENT_BUS_SUBSCRIBE_PORT=4443 firefox: - image: selenium/node-firefox:4.17.0-20240123 + image: selenium/node-firefox:4.18.0-20240220 shm_size: 2gb depends_on: - selenium-hub @@ -34,7 +34,7 @@ services: - SE_EVENT_BUS_SUBSCRIBE_PORT=4443 selenium-hub: - image: selenium/hub:4.17.0-20240123 + image: selenium/hub:4.18.0-20240220 container_name: selenium-hub ports: - "4442:4442" diff --git a/generate_chart_changelog.sh b/generate_chart_changelog.sh index 4afaedfb6..90d4675c9 100755 --- a/generate_chart_changelog.sh +++ b/generate_chart_changelog.sh @@ -4,6 +4,7 @@ CHART_DIR="./charts/selenium-grid" CHANGELOG_FILE="./charts/selenium-grid/CHANGELOG.md" TAG_PATTERN="selenium-grid" +SET_TAG=${1:-""} # Get current chart app version CHART_APP_VERSION=$(find . \( -type d -name .git -prune \) -o -type f -name 'Chart.yaml' -print0 | xargs -0 cat | grep ^appVersion | cut -d ':' -f 2 | tr -d '[:space:]') @@ -14,38 +15,42 @@ generate_changelog() { tags=($(git tag --sort=committerdate | grep "^$TAG_PATTERN")) tags_size=${#tags[@]} + CURRENT_CHART_VERSION=$(find . \( -type d -name .git -prune \) -o -type f -name 'Chart.yaml' -print0 | xargs -0 cat | grep ^version | cut -d ':' -f 2 | tr -d '[:space:]') + # Check if there are tags if [ ${#tags[@]} -eq 0 ]; then commit_range="HEAD" - elif [ ${#tags[@]} -eq 1 ]; then + change_title="${TAG_PATTERN}-${CURRENT_CHART_VERSION}" + elif [ ${#tags[@]} -eq 1 ] || [ "$SET_TAG" = "HEAD" ]; then previous_tag="${tags[$tags_size-1]}" current_tag="HEAD" commit_range="${previous_tag}..${current_tag}" + change_title="${TAG_PATTERN}-${CURRENT_CHART_VERSION}" else previous_tag="${tags[$tags_size-2]}" current_tag="${tags[$tags_size-1]}" commit_range="${previous_tag}..${current_tag}" + change_title="$current_tag" fi + echo "Generating changelog for ${change_title}" + # Get the changes for each section (Added, Removed, Fixed, Changed) image_tag_changes=$(echo "Chart is using image tag $CHART_APP_VERSION" | sed -e 's/^/- /') + k8s_versions_tested=$(echo "Chart is tested on Kubernetes versions: $(cat .github/workflows/helm-chart-test.yml | grep -oP "k8s-version: '\Kv.*(?=')" | tr '\n' ' ')" | sed -e 's/^/- /') added_changes=$(git log --pretty=format:"%s :: %an" "$commit_range" -- "$CHART_DIR" | grep -iE "^feat|^add" | sed -e 's/^/- /') removed_changes=$(git log --pretty=format:"%s :: %an" "$commit_range" -- "$CHART_DIR" | grep -iE "^remove|^deprecate|^delete" | sed -e 's/^/- /') fixed_changes=$(git log --pretty=format:"%s :: %an" "$commit_range" -- "$CHART_DIR" | grep -iE "^fix|^bug" | sed -e 's/^/- /') changed_changes=$(git log --pretty=format:"%s :: %an" "$commit_range" -- "$CHART_DIR" | grep -iEv "^feat|^add|^remove|^deprecate|^delete|^fix|^bug" | sed -e 's/^/- /') - if [[ $(cat $CHANGELOG_FILE) == *"${current_tag}"* ]]; then - echo "Changelog already generated for ${current_tag}" - exit 0 - fi - # Create a temporary file temp_file=$(mktemp) # Write to the temporary file - echo "## :heavy_check_mark: ${current_tag}" >> "$temp_file" + echo "## :heavy_check_mark: ${change_title}" >> "$temp_file" echo "" >> "$temp_file" echo "$image_tag_changes" >> "$temp_file" + echo "$k8s_versions_tested" >> "$temp_file" echo "" >> "$temp_file" if [ -n "$added_changes" ]; then @@ -72,11 +77,26 @@ generate_changelog() { echo "" >> "$temp_file" fi + # Create chart_release_notes.md + release_notes_file="$CHART_DIR/RELEASE_NOTES.md" + chart_description=$(find . \( -type d -name .git -prune \) -o -type f -name 'Chart.yaml' -print0 | xargs -0 cat | grep ^description | cut -d ':' -f 2) + echo "$chart_description" > "$release_notes_file" + echo "" >> "$release_notes_file" + cat $temp_file >> "$release_notes_file" + echo "Generated release notes at $release_notes_file" + # Append the existing content of CHANGELOG to the temporary file cat "$CHANGELOG_FILE" >> "$temp_file" - # Overwrite CHANGELOG with the content of the temporary file - mv "$temp_file" "$CHANGELOG_FILE" + if [[ $(cat $CHANGELOG_FILE) == *"${change_title}"* ]]; then + echo "Changelog already generated for ${change_title}" + rm -rf "$temp_file" + exit 0 + else + # Overwrite CHANGELOG with the content of the temporary file + mv "$temp_file" "$CHANGELOG_FILE" + fi + } # Run the function to generate the changelog diff --git a/generate_release_notes.sh b/generate_release_notes.sh index f3fe9b954..e64dfcdda 100755 --- a/generate_release_notes.sh +++ b/generate_release_notes.sh @@ -21,7 +21,7 @@ EDGEDRIVER_VERSION=$(docker run --rm ${NAMESPACE}/node-edge:${TAG_VERSION} msedg FIREFOX_VERSION=$(docker run --rm ${NAMESPACE}/node-firefox:${TAG_VERSION} firefox --version | awk '{print $3}') GECKODRIVER_VERSION=$(docker run --rm ${NAMESPACE}/node-firefox:${TAG_VERSION} geckodriver --version | awk 'NR==1{print $2}') FFMPEG_VERSION=$(docker run --entrypoint="" --rm ${NAMESPACE}/video:${FFMPEG_TAG_VERSION}-${BUILD_DATE} ffmpeg -version | awk '{print $3}' | head -n 1) -RCLONE_VERSION=$(docker run --entrypoint="" --rm ${NAMESPACE}/uploader:${RCLONE_TAG_VERSION}-${BUILD_DATE} rclone version | head -n 1 | awk '{print $2}') +RCLONE_VERSION=$(docker run --entrypoint="" --rm ${NAMESPACE}/video:${FFMPEG_TAG_VERSION}-${BUILD_DATE} rclone version | head -n 1 | awk '{print $2}') echo "" >> release_notes.md diff --git a/tests/Dockerfile b/tests/Dockerfile index cf2d16926..ba837fa97 100644 --- a/tests/Dockerfile +++ b/tests/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.8.6-buster +FROM python:3.11-slim WORKDIR /usr/src/app diff --git a/tests/SeleniumTests/__init__.py b/tests/SeleniumTests/__init__.py index 756c26931..c5cd0eab7 100644 --- a/tests/SeleniumTests/__init__.py +++ b/tests/SeleniumTests/__init__.py @@ -2,6 +2,7 @@ import concurrent.futures import os import traceback +import time from selenium import webdriver from selenium.webdriver.common.by import By from selenium.webdriver.support.ui import WebDriverWait @@ -13,8 +14,14 @@ SELENIUM_GRID_PROTOCOL = os.environ.get('SELENIUM_GRID_PROTOCOL', 'http') SELENIUM_GRID_HOST = os.environ.get('SELENIUM_GRID_HOST', 'localhost') SELENIUM_GRID_PORT = os.environ.get('SELENIUM_GRID_PORT', '4444') +SELENIUM_GRID_USERNAME = os.environ.get('SELENIUM_GRID_USERNAME', '') +SELENIUM_GRID_PASSWORD = os.environ.get('SELENIUM_GRID_PASSWORD', '') +SELENIUM_GRID_TEST_HEADLESS = os.environ.get('SELENIUM_GRID_TEST_HEADLESS', 'false').lower() == 'true' WEB_DRIVER_WAIT_TIMEOUT = int(os.environ.get('WEB_DRIVER_WAIT_TIMEOUT', 60)) +if SELENIUM_GRID_USERNAME and SELENIUM_GRID_PASSWORD: + SELENIUM_GRID_HOST = f"{SELENIUM_GRID_USERNAME}:{SELENIUM_GRID_PASSWORD}@{SELENIUM_GRID_HOST}" + class SeleniumGenericTests(unittest.TestCase): def test_title(self): @@ -76,85 +83,141 @@ def test_download_file(self): driver.get('https://the-internet.herokuapp.com/download') file_name = 'some-file.txt' is_continue = True - try: - wait = WebDriverWait(driver, 30) - file_link = wait.until( - EC.element_to_be_clickable((By.LINK_TEXT, file_name)) - ) - except: - is_continue = False - if is_continue: - file_link.click() - wait.until( - lambda d: str(d.get_downloadable_files()[0]).endswith(file_name) - ) - self.assertTrue(str(driver.get_downloadable_files()[0]).endswith(file_name)) + wait = WebDriverWait(driver, 30) + file_link = wait.until( + EC.element_to_be_clickable((By.LINK_TEXT, file_name)) + ) + file_link.click() + wait.until( + lambda d: str(d.get_downloadable_files()[0]).endswith(file_name) + ) + self.assertTrue(str(driver.get_downloadable_files()[0]).endswith(file_name)) def tearDown(self): - self.driver.quit() + try: + self.driver.quit() + except Exception as e: + print(f"::error::Exception: {str(e)}") + print(traceback.format_exc()) + raise e class ChromeTests(SeleniumGenericTests): def setUp(self): - options = ChromeOptions() - options.enable_downloads = True - options.add_argument('disable-features=DownloadBubble,DownloadBubbleV2') - options.set_capability('se:recordVideo', True) - self.driver = webdriver.Remote( - options=options, - command_executor="%s://%s:%s" % (SELENIUM_GRID_PROTOCOL,SELENIUM_GRID_HOST,SELENIUM_GRID_PORT) - ) + try: + options = ChromeOptions() + options.enable_downloads = True + options.add_argument('disable-features=DownloadBubble,DownloadBubbleV2') + options.set_capability('se:recordVideo', True) + if SELENIUM_GRID_TEST_HEADLESS: + options.add_argument('--headless=new') + start_time = time.time() + self.driver = webdriver.Remote( + options=options, + command_executor="%s://%s:%s" % (SELENIUM_GRID_PROTOCOL,SELENIUM_GRID_HOST,SELENIUM_GRID_PORT) + ) + end_time = time.time() + print(f"<< {self._testMethodName} ({self.__class__.__name__}) WebDriver initialization completed in {end_time - start_time} (s)") + except Exception as e: + print(f"::error::Exception: {str(e)}") + print(traceback.format_exc()) + raise e class EdgeTests(SeleniumGenericTests): def setUp(self): - options = EdgeOptions() - options.enable_downloads = True - options.add_argument('disable-features=DownloadBubble,DownloadBubbleV2') - options.set_capability('se:recordVideo', True) - self.driver = webdriver.Remote( - options=options, - command_executor="%s://%s:%s" % (SELENIUM_GRID_PROTOCOL,SELENIUM_GRID_HOST,SELENIUM_GRID_PORT) - ) - + try: + options = EdgeOptions() + options.enable_downloads = True + options.add_argument('disable-features=DownloadBubble,DownloadBubbleV2') + options.set_capability('se:recordVideo', True) + if SELENIUM_GRID_TEST_HEADLESS: + options.add_argument('--headless=new') + start_time = time.time() + self.driver = webdriver.Remote( + options=options, + command_executor="%s://%s:%s" % (SELENIUM_GRID_PROTOCOL,SELENIUM_GRID_HOST,SELENIUM_GRID_PORT) + ) + end_time = time.time() + print(f"<< {self._testMethodName} ({self.__class__.__name__}) WebDriver initialization completed in {end_time - start_time} (s)") + except Exception as e: + print(f"::error::Exception: {str(e)}") + print(traceback.format_exc()) + raise e class FirefoxTests(SeleniumGenericTests): def setUp(self): - profile = webdriver.FirefoxProfile() - profile.set_preference("browser.download.manager.showWhenStarting", False) - profile.set_preference("browser.helperApps.neverAsk.saveToDisk", "*/*") - options = FirefoxOptions() - options.profile = profile - options.enable_downloads = True - options.set_capability('se:recordVideo', True) - self.driver = webdriver.Remote( - options=options, - command_executor="%s://%s:%s" % (SELENIUM_GRID_PROTOCOL,SELENIUM_GRID_HOST,SELENIUM_GRID_PORT) - ) + try: + profile = webdriver.FirefoxProfile() + profile.set_preference("browser.download.manager.showWhenStarting", False) + profile.set_preference("browser.helperApps.neverAsk.saveToDisk", "*/*") + options = FirefoxOptions() + options.profile = profile + options.enable_downloads = True + options.set_capability('se:recordVideo', True) + if SELENIUM_GRID_TEST_HEADLESS: + options.add_argument('-headless') + start_time = time.time() + self.driver = webdriver.Remote( + options=options, + command_executor="%s://%s:%s" % (SELENIUM_GRID_PROTOCOL,SELENIUM_GRID_HOST,SELENIUM_GRID_PORT) + ) + end_time = time.time() + print(f"<< {self._testMethodName} ({self.__class__.__name__}) WebDriver initialization completed in {end_time - start_time} (s)") + except Exception as e: + print(f"::error::Exception: {str(e)}") + print(traceback.format_exc()) + raise e def test_title_and_maximize_window(self): self.driver.get('https://the-internet.herokuapp.com') self.driver.maximize_window() self.assertTrue(self.driver.title == 'The Internet') -class JobAutoscaling(): +class Autoscaling(): def run(self, test_classes): with concurrent.futures.ThreadPoolExecutor() as executor: futures = [] + tests = [] + start_times = {} for test_class in test_classes: suite = unittest.TestLoader().loadTestsFromTestCase(test_class) for test in suite: + start_times[test] = time.time() futures.append(executor.submit(test)) - for future in concurrent.futures.as_completed(futures): + tests.append(test) + failed_tests = [] + for future, test in zip(concurrent.futures.as_completed(futures), tests): try: - result = future.result() - if not result.wasSuccessful(): - raise Exception(f"Test {str(test)} failed") + completion_time = time.time() - start_times[test] + print(f">> {str(test)} completed in {str(completion_time)} (s)") + if not future.result().wasSuccessful(): + raise Exception except Exception as e: - print(f"{str(test)} failed with exception: {str(e)}") + failed_tests.append(test) print(traceback.format_exc()) - raise Exception(f"Parallel tests failed: {str(test)} failed with exception: {str(e)}") + print(f"{str(test)} failed with exception: {str(e)}") + print(f"Original exception: {e.__cause__}") + if len(failed_tests) > 0: + print(f"Number of failed tests: {len(failed_tests)}. Going to rerun!") + for test in failed_tests: + try: + print(f"Rerunning test: {str(test)}") + rerun_result = test.run() + if not rerun_result.wasSuccessful(): + raise Exception + except Exception as e: + print(traceback.format_exc()) + print(f"Test {str(test)} failed again with exception: {str(e)}") + print(f"Original exception: {e.__cause__}") + raise Exception(f"Rerun test failed: {str(test)} failed with exception: {str(e)}") + print(f"::warning:: Number of failed tests: {len(failed_tests)}. All tests passed in rerun!") + +class DeploymentAutoscalingTests(unittest.TestCase): + def test_parallel_autoscaling(self): + runner = Autoscaling() + runner.run([FirefoxTests, EdgeTests, ChromeTests]) class JobAutoscalingTests(unittest.TestCase): def test_parallel_autoscaling(self): - runner = JobAutoscaling() - runner.run([ChromeTests, EdgeTests, FirefoxTests]) + runner = Autoscaling() + runner.run([FirefoxTests, EdgeTests, ChromeTests]) diff --git a/tests/SmokeTests/__init__.py b/tests/SmokeTests/__init__.py index 2cd737017..8652eea6d 100644 --- a/tests/SmokeTests/__init__.py +++ b/tests/SmokeTests/__init__.py @@ -3,14 +3,14 @@ import time import json from ssl import _create_unverified_context -try: - from urllib2 import urlopen -except ImportError: - from urllib.request import urlopen +import requests +from requests.auth import HTTPBasicAuth SELENIUM_GRID_PROTOCOL = os.environ.get('SELENIUM_GRID_PROTOCOL', 'http') SELENIUM_GRID_HOST = os.environ.get('SELENIUM_GRID_HOST', 'localhost') SELENIUM_GRID_PORT = os.environ.get('SELENIUM_GRID_PORT', '4444') +SELENIUM_GRID_USERNAME = os.environ.get('SELENIUM_GRID_USERNAME', '') +SELENIUM_GRID_PASSWORD = os.environ.get('SELENIUM_GRID_PASSWORD', '') SELENIUM_GRID_AUTOSCALING = os.environ.get('SELENIUM_GRID_AUTOSCALING', 'false') SELENIUM_GRID_AUTOSCALING_MIN_REPLICA = os.environ.get('SELENIUM_GRID_AUTOSCALING_MIN_REPLICA', 0) HUB_CHECKS_MAX_ATTEMPTS = os.environ.get('HUB_CHECKS_MAX_ATTEMPTS', 3) @@ -29,8 +29,12 @@ def smoke_test_container(self, port): while current_attempts < max_attempts: current_attempts = current_attempts + 1 try: - response = urlopen('%s://%s:%s/status' % (SELENIUM_GRID_PROTOCOL, SELENIUM_GRID_HOST, port), context=_create_unverified_context()) - status_json = json.loads(response.read()) + grid_url_status = '%s://%s:%s/status' % (SELENIUM_GRID_PROTOCOL, SELENIUM_GRID_HOST, port) + if SELENIUM_GRID_USERNAME and SELENIUM_GRID_PASSWORD: + response = requests.get(grid_url_status, auth=HTTPBasicAuth(SELENIUM_GRID_USERNAME, SELENIUM_GRID_PASSWORD)) + else: + response = requests.get(grid_url_status) + status_json = response.json() if not auto_scaling or (auto_scaling and auto_scaling_min_replica > 0): self.assertTrue(status_json['value']['ready'], "Container is not ready on port %s" % port) else: diff --git a/tests/bootstrap.sh b/tests/bootstrap.sh index 24d3c9980..df744cea8 100755 --- a/tests/bootstrap.sh +++ b/tests/bootstrap.sh @@ -7,7 +7,7 @@ if [ "${CI:-false}" = "false" ]; then source docker-selenium-tests/bin/activate fi -python -m pip install selenium==4.17.2 \ +python -m pip install selenium==4.18.0 \ docker===6.1.3 \ | grep -v 'Requirement already satisfied' diff --git a/tests/charts/bootstrap.sh b/tests/charts/bootstrap.sh index 33721c44b..e2999fe30 100755 --- a/tests/charts/bootstrap.sh +++ b/tests/charts/bootstrap.sh @@ -15,13 +15,13 @@ cd .. helm template dummy --values tests/charts/templates/render/dummy.yaml \ --set-file 'nodeConfigMap.extraScripts.nodePreStop\.sh=tests/charts/templates/render/dummy_external.sh' \ --set-file 'recorderConfigMap.extraScripts.video\.sh=tests/charts/templates/render/dummy_external.sh' \ - --set-file 'recorderConfigMap.extraScripts.graphQLRecordVideo\.sh=tests/charts/templates/render/dummy_external.sh' \ + --set-file 'recorderConfigMap.extraScripts.video_graphQLQuery\.sh=tests/charts/templates/render/dummy_external.sh' \ --set-file 'recorderConfigMap.extraScripts.newInsertScript\.sh=tests/charts/templates/render/dummy_external.sh' \ - --set-file 'uploaderConfigMap.extraScripts.entry_point\.sh=tests/charts/templates/render/dummy_external.sh' \ - --set-file 'uploaderConfigMap.secretFiles.config\.conf=tests/charts/templates/render/dummy_external.sh' \ + --set-file 'uploaderConfigMap.extraScripts.upload\.sh=tests/charts/templates/render/dummy_external.sh' \ + --set-file 'uploaderConfigMap.secretFiles.upload\.conf=tests/charts/templates/render/dummy_external.sh' \ charts/selenium-grid > ./tests/tests/dummy_template_manifests.yaml -python tests/charts/templates/test.py "./tests/tests/dummy_template_manifests.yaml" +python tests/charts/templates/test.py "./tests/tests/dummy_template_manifests.yaml" dummy ret_code=$? if [ "${CI:-false}" = "false" ]; then diff --git a/tests/charts/ci/DeploymentAutoScaling-values.yaml b/tests/charts/ci/DeploymentAutoscaling-values.yaml similarity index 80% rename from tests/charts/ci/DeploymentAutoScaling-values.yaml rename to tests/charts/ci/DeploymentAutoscaling-values.yaml index 08dcf1cb0..230eb95cf 100644 --- a/tests/charts/ci/DeploymentAutoScaling-values.yaml +++ b/tests/charts/ci/DeploymentAutoscaling-values.yaml @@ -3,10 +3,12 @@ autoscaling: scalingType: deployment scaledOptions: minReplicaCount: 0 - maxReplicaCount: 3 + maxReplicaCount: 2 + pollingInterval: 20 scaledObjectOptions: cooldownPeriod: 30 terminationGracePeriodSeconds: 360 + # Configuration for chrome nodes chromeNode: # (test): user is able to change `SE_NODE_PORT` @@ -33,12 +35,25 @@ chromeNode: # NodePort will be assigned randomly if not set nameOverride: my-chrome-name extraEnvironmentVariables: &extraEnvironmentVariables - - name: SE_OPTS - value: "--enable-managed-downloads true" + - name: SE_NODE_ENABLE_MANAGED_DOWNLOADS + value: "true" + - name: SE_VNC_NO_PASSWORD + value: "true" + - name: SE_SCREEN_WIDTH + value: "1920" + - name: SE_SCREEN_HEIGHT + value: "1080" + - name: TZ + value: "Asia/Saigon" + - name: SE_NODE_REGISTER_PERIOD + value: "800" + - name: SE_NODE_REGISTER_CYCLE + value: "3" readinessProbe: enabled: &readinessProbe true livenessProbe: enabled: &livenessProbe true + # Configuration for edge nodes edgeNode: port: 8888 @@ -61,8 +76,10 @@ edgeNode: enabled: *readinessProbe livenessProbe: enabled: *livenessProbe + # Configuration for firefox nodes firefoxNode: + port: 7777 nameOverride: my-firefox-name extraEnvironmentVariables: *extraEnvironmentVariables readinessProbe: diff --git a/tests/charts/ci/JobAutoscaling-values.yaml b/tests/charts/ci/JobAutoscaling-values.yaml index 2e689b57c..c6eabfd5d 100644 --- a/tests/charts/ci/JobAutoscaling-values.yaml +++ b/tests/charts/ci/JobAutoscaling-values.yaml @@ -1,21 +1,32 @@ -isolateComponents: false - autoscaling: enableWithExistingKEDA: true scalingType: job scaledJobOptions: + successfulJobsHistoryLimit: 100 scalingStrategy: strategy: default scaledOptions: minReplicaCount: 0 - maxReplicaCount: 5 + maxReplicaCount: 6 pollingInterval: 20 # Configuration for chrome nodes chromeNode: nameOverride: my-chrome-name extraEnvironmentVariables: &extraEnvironmentVariables - - name: SE_OPTS - value: "--enable-managed-downloads true" + - name: SE_NODE_ENABLE_MANAGED_DOWNLOADS + value: "true" + - name: SE_VNC_NO_PASSWORD + value: "true" + - name: SE_SCREEN_WIDTH + value: "1920" + - name: SE_SCREEN_HEIGHT + value: "1080" + - name: TZ + value: "Asia/Saigon" + - name: SE_NODE_REGISTER_PERIOD + value: "800" + - name: SE_NODE_REGISTER_CYCLE + value: "3" readinessProbe: enabled: &readinessProbe false livenessProbe: diff --git a/tests/charts/ci/NodeChrome-values.yaml b/tests/charts/ci/NodeChrome-values.yaml index d7c9df098..5a612e0e6 100644 --- a/tests/charts/ci/NodeChrome-values.yaml +++ b/tests/charts/ci/NodeChrome-values.yaml @@ -4,8 +4,8 @@ chromeNode: port: 6666 nameOverride: my-chrome-name extraEnvironmentVariables: - - name: SE_OPTS - value: "--enable-managed-downloads true" + - name: SE_NODE_ENABLE_MANAGED_DOWNLOADS + value: "true" # Configuration for edge nodes edgeNode: enabled: false diff --git a/tests/charts/ci/NodeEdge-values.yaml b/tests/charts/ci/NodeEdge-values.yaml index 7c78ccbe7..3fdc12a6d 100644 --- a/tests/charts/ci/NodeEdge-values.yaml +++ b/tests/charts/ci/NodeEdge-values.yaml @@ -7,8 +7,8 @@ edgeNode: port: 8888 nameOverride: my-edge-name extraEnvironmentVariables: - - name: SE_OPTS - value: "--enable-managed-downloads true" + - name: SE_NODE_ENABLE_MANAGED_DOWNLOADS + value: "true" # Configuration for firefox nodes firefoxNode: enabled: false diff --git a/tests/charts/ci/NodeFirefox-values.yaml b/tests/charts/ci/NodeFirefox-values.yaml index 72cce27c5..574ce19e3 100644 --- a/tests/charts/ci/NodeFirefox-values.yaml +++ b/tests/charts/ci/NodeFirefox-values.yaml @@ -10,5 +10,5 @@ firefoxNode: port: 7777 nameOverride: my-firefox-name extraEnvironmentVariables: - - name: SE_OPTS - value: "--enable-managed-downloads true" + - name: SE_NODE_ENABLE_MANAGED_DOWNLOADS + value: "true" diff --git a/tests/charts/ci/auth-ingress-values.yaml b/tests/charts/ci/auth-ingress-values.yaml deleted file mode 100644 index c597ebc2a..000000000 --- a/tests/charts/ci/auth-ingress-values.yaml +++ /dev/null @@ -1,72 +0,0 @@ -global: - K8S_PUBLIC_IP: localhost - seleniumGrid: - logLevel: INFO - -ingress: - annotations: - kubernetes.io/ingress.class: nginx - nginx.ingress.kubernetes.io/use-regex: "true" - nginx.ingress.kubernetes.io/rewrite-target: /$2 - nginx.ingress.kubernetes.io/app-root: &gridAppRoot "/selenium" - nginx.ingress.kubernetes.io/proxy-connect-timeout: "360" - nginx.ingress.kubernetes.io/proxy-read-timeout: "360" - nginx.ingress.kubernetes.io/proxy-send-timeout: "360" - ingressClassName: nginx - hostname: "" - paths: - - path: /selenium(/|$)(.*) - pathType: ImplementationSpecific - backend: - service: - name: '{{ ternary (include "seleniumGrid.router.fullname" $ ) (include "seleniumGrid.hub.fullname" $ ) $.Values.isolateComponents }}' - port: - number: 4444 - -basicAuth: - enabled: false - -isolateComponents: true - -hub: - subPath: *gridAppRoot - -components: - subPath: *gridAppRoot - -ingress-nginx: - enabled: true - controller: - hostPort: - enabled: true - kind: DaemonSet - service: - type: ClusterIP - -videoRecorder: - enabled: true - uploader: - enabled: false - name: rclone - destinationPrefix: "gs://ndviet" - secrets: - RCLONE_CONFIG_S3_TYPE: "s3" - RCLONE_CONFIG_S3_PROVIDER: "AWS" - RCLONE_CONFIG_S3_ENV_AUTH: "true" - RCLONE_CONFIG_S3_REGION: "ap-southeast-1" - RCLONE_CONFIG_S3_LOCATION_CONSTRAINT: "ap-southeast-1" - RCLONE_CONFIG_S3_ACL: "private" - RCLONE_CONFIG_S3_ACCESS_KEY_ID: "****" - RCLONE_CONFIG_S3_SECRET_ACCESS_KEY: "****" - RCLONE_CONFIG_GS_TYPE: "s3" - RCLONE_CONFIG_GS_PROVIDER: "GCS" - RCLONE_CONFIG_GS_ENV_AUTH: "true" - RCLONE_CONFIG_GS_REGION: "asia-southeast1" - RCLONE_CONFIG_GS_LOCATION_CONSTRAINT: "asia-southeast1" - RCLONE_CONFIG_GS_ACL: "private" - RCLONE_CONFIG_GS_ACCESS_KEY_ID: "****" - RCLONE_CONFIG_GS_SECRET_ACCESS_KEY: "****" - RCLONE_CONFIG_GS_ENDPOINT: "https://storage.googleapis.com" - AWS_REGION: "ap-southeast-1" - AWS_ACCESS_KEY_ID: "****" - AWS_SECRET_ACCESS_KEY: "****" diff --git a/tests/charts/ci/base-auth-ingress-values.yaml b/tests/charts/ci/base-auth-ingress-values.yaml new file mode 100644 index 000000000..9e02237c1 --- /dev/null +++ b/tests/charts/ci/base-auth-ingress-values.yaml @@ -0,0 +1,49 @@ +global: + seleniumGrid: + logLevel: INFO + +ingress: + annotations: + nginx.ingress.kubernetes.io/use-regex: "true" + nginx.ingress.kubernetes.io/rewrite-target: /$2 + nginx.ingress.kubernetes.io/app-root: &gridAppRoot "/selenium" + className: nginx + hostname: "" + paths: + - path: /selenium(/|$)(.*) + pathType: ImplementationSpecific + backend: + service: + name: '{{ ternary (include "seleniumGrid.router.fullname" $ ) (include "seleniumGrid.hub.fullname" $ ) $.Values.isolateComponents }}' + port: + number: 4444 + +basicAuth: + enabled: false + +isolateComponents: true + +hub: + subPath: *gridAppRoot + extraEnvironmentVariables: &extraEnvironmentVariables + - name: SE_SESSION_REQUEST_TIMEOUT + value: "800" + - name: SE_SESSION_RETRY_INTERVAL + value: "5" + - name: SE_HEALTHCHECK_INTERVAL + value: "15" + - name: SE_REJECT_UNSUPPORTED_CAPS + value: "false" + +components: + subPath: *gridAppRoot + extraEnvironmentVariables: *extraEnvironmentVariables + +ingress-nginx: + enabled: true + controller: + hostPort: + enabled: true + kind: DaemonSet + service: + type: ClusterIP diff --git a/tests/charts/ci/base-recorder-values.yaml b/tests/charts/ci/base-recorder-values.yaml new file mode 100644 index 000000000..2b7e7447c --- /dev/null +++ b/tests/charts/ci/base-recorder-values.yaml @@ -0,0 +1,40 @@ +# Use different container sidecar for uploader +#videoRecorder: +# enabled: true +# uploader: +# enabled: "${UPLOAD_ENABLED}" +# name: s3 +# destinationPrefix: "s3://${BUCKET_NAME}" +# secrets: +# AWS_REGION: "ap-southeast-1" +# AWS_ACCESS_KEY_ID: "${AWS_ACCESS_KEY_ID}" +# AWS_SECRET_ACCESS_KEY: "${AWS_SECRET_ACCESS_KEY}" + +videoRecorder: + enabled: true + uploader: + enabled: "${UPLOAD_ENABLED}" + destinationPrefix: "gs://${BUCKET_NAME}" + secrets: + RCLONE_CONFIG_S3_TYPE: "s3" + RCLONE_CONFIG_S3_PROVIDER: "AWS" + RCLONE_CONFIG_S3_ENV_AUTH: "true" + RCLONE_CONFIG_S3_REGION: "ap-southeast-1" + RCLONE_CONFIG_S3_LOCATION_CONSTRAINT: "ap-southeast-1" + RCLONE_CONFIG_S3_ACL: "private" + RCLONE_CONFIG_S3_ACCESS_KEY_ID: "${AWS_ACCESS_KEY_ID}" + RCLONE_CONFIG_S3_SECRET_ACCESS_KEY: "${AWS_SECRET_ACCESS_KEY}" + RCLONE_CONFIG_GS_TYPE: "s3" + RCLONE_CONFIG_GS_PROVIDER: "GCS" + RCLONE_CONFIG_GS_ENV_AUTH: "true" + RCLONE_CONFIG_GS_REGION: "asia-southeast1" + RCLONE_CONFIG_GS_LOCATION_CONSTRAINT: "asia-southeast1" + RCLONE_CONFIG_GS_ACL: "private" + RCLONE_CONFIG_GS_ACCESS_KEY_ID: "${GS_ACCESS_KEY_ID}" + RCLONE_CONFIG_GS_SECRET_ACCESS_KEY: "${GS_SECRET_ACCESS_KEY}" + RCLONE_CONFIG_GS_ENDPOINT: "https://storage.googleapis.com" + +ingress-nginx: + controller: + extraArgs: + default-ssl-certificate: '$(POD_NAMESPACE)/${RELEASE_NAME}-selenium-tls-secret' diff --git a/tests/charts/ci/base-resources-values.yaml b/tests/charts/ci/base-resources-values.yaml new file mode 100644 index 000000000..5679f5246 --- /dev/null +++ b/tests/charts/ci/base-resources-values.yaml @@ -0,0 +1,104 @@ +components: + router: + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 2500Mi + distributor: + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 200m + memory: 2500Mi + eventBus: + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 200m + memory: 2500Mi + sessionMap: + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 200m + memory: 512Mi + sessionQueue: + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 200m + memory: 512Mi + +hub: + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 2500Mi + +chromeNode: + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 250m + memory: 2500Mi + startupProbe: + initialDelaySeconds: 0 + periodSeconds: 5 + timeoutSeconds: 400 + failureThreshold: 80 + successThreshold: 1 + +firefoxNode: + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 250m + memory: 2500Mi + startupProbe: + initialDelaySeconds: 0 + periodSeconds: 5 + timeoutSeconds: 400 + failureThreshold: 80 + successThreshold: 1 + +edgeNode: + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 250m + memory: 2500Mi + startupProbe: + initialDelaySeconds: 0 + periodSeconds: 5 + timeoutSeconds: 400 + failureThreshold: 80 + successThreshold: 1 + +videoRecorder: + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 200m + memory: 512Mi diff --git a/tests/charts/ci/base-tls-values.yaml b/tests/charts/ci/base-tls-values.yaml new file mode 100644 index 000000000..5cdd2427e --- /dev/null +++ b/tests/charts/ci/base-tls-values.yaml @@ -0,0 +1,6 @@ +tls: + enabled: true + generateTLS: false + registrationSecret: + enabled: true + value: "HappyTestOps" diff --git a/tests/charts/ci/tls-values.yaml b/tests/charts/ci/tls-values.yaml deleted file mode 100644 index 0f7439251..000000000 --- a/tests/charts/ci/tls-values.yaml +++ /dev/null @@ -1,11 +0,0 @@ -tls: - enabled: true - generateTLS: false - registrationSecret: - enabled: true - value: "HappyTestOps" - -ingress-nginx: - controller: - extraArgs: - default-ssl-certificate: '$(POD_NAMESPACE)/selenium-tls-secret' diff --git a/tests/charts/ci/tracing-values.yaml b/tests/charts/ci/tracing-values.yaml deleted file mode 100644 index 065b2c081..000000000 --- a/tests/charts/ci/tracing-values.yaml +++ /dev/null @@ -1,13 +0,0 @@ -hub: - extraEnvironmentVariables: - - name: SE_ENABLE_TRACING - value: "true" - - name: SE_JAVA_OPTS - value: "-Dotel.traces.exporter=jaeger -Dotel.exporter.jaeger.endpoint=http://{{ .Release.Name }}-jaeger-all-in-one-headless:14250 -Dotel.resource.attributes=service.name=selenium -Dotel.java.global-autoconfigure.enabled=true" - -components: - extraEnvironmentVariables: - - name: SE_ENABLE_TRACING - value: "true" - - name: SE_JAVA_OPTS - value: "-Dotel.traces.exporter=jaeger -Dotel.exporter.jaeger.endpoint=http://{{ .Release.Name }}-jaeger-all-in-one-headless:14250 -Dotel.resource.attributes=service.name=selenium -Dotel.java.global-autoconfigure.enabled=true" diff --git a/tests/charts/config/ct.yaml b/tests/charts/config/ct.yaml index 2b2446823..bad585538 100755 --- a/tests/charts/config/ct.yaml +++ b/tests/charts/config/ct.yaml @@ -6,6 +6,7 @@ chart-dirs: chart-repos: - kedacore=https://kedacore.github.io/charts - ingressNginx=https://kubernetes.github.io/ingress-nginx + - jaegertracing=https://jaegertracing.github.io/helm-charts upgrade: false helm-extra-args: --timeout 600s check-version-increment: false diff --git a/tests/charts/make/chart_cluster_cleanup.sh b/tests/charts/make/chart_cluster_cleanup.sh index 972a9e8b6..b0210a7c1 100755 --- a/tests/charts/make/chart_cluster_cleanup.sh +++ b/tests/charts/make/chart_cluster_cleanup.sh @@ -1,11 +1,17 @@ #!/bin/bash echo "Set ENV variables" +CLUSTER=${CLUSTER:-"minikube"} CLUSTER_NAME=${CLUSTER_NAME:-"chart-testing"} cleanup() { - echo "Clean up kind cluster" + if [ "${CLUSTER}" = "kind" ]; then + echo "Clean up Kind cluster" kind delete clusters ${CLUSTER_NAME} + elif [ "${CLUSTER}" = "minikube" ]; then + echo "Clean up Minikube cluster" + sudo -SE minikube delete + fi } cleanup diff --git a/tests/charts/make/chart_cluster_setup.sh b/tests/charts/make/chart_cluster_setup.sh index e413bd984..dc37ac267 100755 --- a/tests/charts/make/chart_cluster_setup.sh +++ b/tests/charts/make/chart_cluster_setup.sh @@ -1,6 +1,8 @@ #!/bin/bash +set -o xtrace echo "Set ENV variables" +CLUSTER=${CLUSTER:-"minikube"} CLUSTER_NAME=${CLUSTER_NAME:-"chart-testing"} RELEASE_NAME=${RELEASE_NAME:-"test"} SELENIUM_NAMESPACE=${SELENIUM_NAMESPACE:-"selenium"} @@ -13,6 +15,9 @@ SELENIUM_GRID_HOST=${SELENIUM_GRID_HOST:-"localhost"} SELENIUM_GRID_PORT=${SELENIUM_GRID_PORT:-"80"} WAIT_TIMEOUT=${WAIT_TIMEOUT:-"90s"} SKIP_CLEANUP=${SKIP_CLEANUP:-"false"} # For debugging purposes, retain the cluster after the test run +KUBERNETES_VERSION=${KUBERNETES_VERSION:-$(curl -L -s https://dl.k8s.io/release/stable.txt)} +CNI=${CNI:-"calico"} # auto, calico, cilium +CONTAINER_RUNTIME=${CONTAINER_RUNTIME:-"docker"} # docker, containerd, cri-o # Function to clean up for retry step on workflow cleanup() { @@ -32,14 +37,39 @@ on_failure() { # Trap ERR signal and call on_failure function trap 'on_failure' ERR -echo "Create Kind cluster" -kind create cluster --wait ${WAIT_TIMEOUT} --name ${CLUSTER_NAME} --config tests/charts/config/kind-cluster.yaml +# Limit the number of resources to avoid host OOM +CPUs=$(grep -c ^processor /proc/cpuinfo) +if [ "${CPUs}" -gt 1 ]; then + CPUs=$((CPUs-1)) +fi + +MEMORY=$(free -m | awk '/^Mem:/{print $7}') +if [ "${MEMORY}" = "" ]; then + MEMORY=$(free -m | awk '/^Mem:/{print $2}') +fi + +if [ "${CLUSTER}" = "kind" ]; then + echo "Start Kind cluster" + kind create cluster --image kindest/node:${KUBERNETES_VERSION} --wait ${WAIT_TIMEOUT} --name ${CLUSTER_NAME} --config tests/charts/config/kind-cluster.yaml +elif [ "${CLUSTER}" = "minikube" ]; then + echo "Start Minikube cluster" + sudo chmod 777 /tmp + export CHANGE_MINIKUBE_NONE_USER=true + sudo -SE minikube start --vm-driver=none --cpus ${CPUs} --memory ${MEMORY} \ + --kubernetes-version=${KUBERNETES_VERSION} --network-plugin=cni --cni=${CNI} --container-runtime=${CONTAINER_RUNTIME} --wait=all + sudo chown -R $USER $HOME/.kube $HOME/.minikube +fi echo "Install KEDA core on kind kubernetes cluster" -kubectl apply --server-side -f https://github.com/kedacore/keda/releases/download/v2.12.1/keda-2.12.1-core.yaml +helm upgrade -i ${KEDA_NAMESPACE} -n ${KEDA_NAMESPACE} --create-namespace --set webhooks.enabled=false kedacore/keda + +if [ "${CLUSTER}" = "kind" ]; then + echo "Load built local Docker Images into Kind Cluster" + image_list=$(docker images --format "{{.Repository}}:{{.Tag}}" | grep ${NAMESPACE} | grep ${BUILD_DATE:-$VERSION}) + for image in $image_list; do + kind load docker-image --name ${CLUSTER_NAME} "$image" + done +fi -echo "Load built local Docker Images into Kind Cluster" -image_list=$(docker images --format "{{.Repository}}:{{.Tag}}" | grep ${NAMESPACE} | grep ${BUILD_DATE:-$VERSION}) -for image in $image_list; do - kind load docker-image --name ${CLUSTER_NAME} "$image" -done +echo "Wait for KEDA core to be ready" +kubectl -n ${KEDA_NAMESPACE} wait --for=condition=ready pod -l app.kubernetes.io/instance=${KEDA_NAMESPACE} --timeout 180s diff --git a/tests/charts/make/chart_setup_env.sh b/tests/charts/make/chart_setup_env.sh index 8c77101bf..4043516a4 100755 --- a/tests/charts/make/chart_setup_env.sh +++ b/tests/charts/make/chart_setup_env.sh @@ -1,4 +1,8 @@ #!/bin/bash + +echo "Set ENV variables" +CLUSTER=${CLUSTER:-"minikube"} + # Function to be executed on command failure on_failure() { local exit_status=$? @@ -10,14 +14,71 @@ on_failure() { trap 'on_failure' ERR if [ "$(uname -m)" = "x86_64" ]; then - echo "Installing kind for AMD64 / x86_64" - curl -fsSL -o ./kind https://kind.sigs.k8s.io/dl/latest/kind-linux-amd64 - chmod +x ./kind - sudo cp -frp ./kind /usr/local/bin/kind - sudo ln -sf /usr/local/bin/kind /usr/bin/kind - rm -rf kind - kind version - echo "===============================" + if [ "${CLUSTER}" = "kind" ]; then + echo "Installing kind for AMD64 / x86_64" + curl -fsSL -o ./kind https://kind.sigs.k8s.io/dl/latest/kind-linux-amd64 + chmod +x ./kind + sudo cp -frp ./kind /usr/local/bin/kind + sudo ln -sf /usr/local/bin/kind /usr/bin/kind + rm -rf kind + kind version + echo "===============================" + elif [ "${CLUSTER}" = "minikube" ]; then + echo "Installing additional dependencies for running Minikube on none driver CRI-dockerd" + echo "===============================" + echo "Installing conntrack" + sudo apt-get install -y conntrack + echo "===============================" + echo "Installing Minikube" + curl -sLO https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64 + sudo install minikube-linux-amd64 /usr/local/bin/minikube + minikube version + rm -rf minikube-linux-amd64 + echo "===============================" + echo "Installing Go" + GO_VERSION="1.21.6" + curl -sLO https://go.dev/dl/go$GO_VERSION.linux-amd64.tar.gz + sudo tar -xf go$GO_VERSION.linux-amd64.tar.gz -C /usr/local + rm -rf go$GO_VERSION.linux-amd64.tar.gz* + echo 'export PATH=$PATH:/usr/local/go/bin' >> ~/.bashrc + source ~/.bashrc || true + go version + echo "===============================" + echo "Installing CRI-CTL (CLI for CRI-compatible container runtimes)" + CRICTL_VERSION="v1.26.0" + curl -sLO https://github.com/kubernetes-sigs/cri-tools/releases/download/$CRICTL_VERSION/crictl-$CRICTL_VERSION-linux-amd64.tar.gz + sudo tar -xf crictl-$CRICTL_VERSION-linux-amd64.tar.gz -C /usr/local/bin + rm -rf crictl-$CRICTL_VERSION-linux-amd64.tar.gz + crictl --version || true + echo "===============================" + echo "Installing CRI-Dockerd" + CRI_DOCKERD_VERSION="v0.3.9" + rm -rf cri-dockerd + git clone -q https://github.com/Mirantis/cri-dockerd.git --branch $CRI_DOCKERD_VERSION --single-branch -c advice.detachedHead=false + cd cri-dockerd || true + sudo go get -v + sudo go build -v -o /usr/local/bin/cri-dockerd + sudo mkdir -p /etc/systemd/system + sudo cp -a -f packaging/systemd/* /etc/systemd/system + sudo sed -i -e 's,/usr/bin/cri-dockerd,/usr/local/bin/cri-dockerd,' /etc/systemd/system/cri-docker.service + sudo systemctl daemon-reload + sudo systemctl enable cri-docker.service + sudo systemctl enable cri-docker.socket + sudo systemctl status --no-pager cri-docker.socket || true + cd .. || true + rm -rf cri-dockerd + cri-dockerd --version + echo "===============================" + echo "Installing CNI-Plugins (Container Network Interface)" + CNI_PLUGIN_VERSION="v1.4.0" + CNI_PLUGIN_TAR="cni-plugins-linux-amd64-$CNI_PLUGIN_VERSION.tgz" + CNI_PLUGIN_INSTALL_DIR="/opt/cni/bin" + curl -sLO "https://github.com/containernetworking/plugins/releases/download/$CNI_PLUGIN_VERSION/$CNI_PLUGIN_TAR" + sudo mkdir -p "$CNI_PLUGIN_INSTALL_DIR" + sudo tar -xf "$CNI_PLUGIN_TAR" -C "$CNI_PLUGIN_INSTALL_DIR" + rm -rf "$CNI_PLUGIN_TAR" + echo "===============================" + fi echo "Installing kubectl for AMD64 / x86_64" curl -fsSL -o ./kubectl "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" @@ -47,4 +108,10 @@ if [ "$(uname -m)" = "x86_64" ]; then rm -rf ct.tar.gz ct version echo "===============================" + echo "Installing envsubst for AMD64 / x86_64" + curl -L https://github.com/a8m/envsubst/releases/download/v1.4.2/envsubst-`uname -s`-`uname -m` -o envsubst + chmod +x envsubst + sudo mv envsubst /usr/local/bin + sudo ln -sf /usr/local/bin/envsubst /usr/bin/envsubst + echo "===============================" fi diff --git a/tests/charts/make/chart_test.sh b/tests/charts/make/chart_test.sh index dd95d81f7..1b5747fe8 100755 --- a/tests/charts/make/chart_test.sh +++ b/tests/charts/make/chart_test.sh @@ -21,11 +21,18 @@ WAIT_TIMEOUT=${WAIT_TIMEOUT:-"90s"} HUB_CHECKS_INTERVAL=${HUB_CHECKS_INTERVAL:-45} HUB_CHECKS_MAX_ATTEMPTS=${HUB_CHECKS_MAX_ATTEMPTS:-6} WEB_DRIVER_WAIT_TIMEOUT=${WEB_DRIVER_WAIT_TIMEOUT:-120} +AUTOSCALING_POLL_INTERVAL=${AUTOSCALING_POLL_INTERVAL:-20} SKIP_CLEANUP=${SKIP_CLEANUP:-"false"} # For debugging purposes, retain the cluster after the test run CHART_CERT_PATH=${CHART_CERT_PATH:-"${CHART_PATH}/certs/selenium.pem"} SSL_CERT_DIR=${SSL_CERT_DIR:-"/etc/ssl/certs"} VIDEO_TAG=${VIDEO_TAG:-"latest"} -UPLOADER_TAG=${UPLOADER_TAG:-"latest"} +CHART_ENABLE_TRACING=${CHART_ENABLE_TRACING:-"false"} +CHART_FULL_DISTRIBUTED_MODE=${CHART_FULL_DISTRIBUTED_MODE:-"false"} +HOSTNAME_ADDRESS=${HOSTNAME_ADDRESS:-"selenium-grid.local"} +CHART_ENABLE_INGRESS_HOSTNAME=${CHART_ENABLE_INGRESS_HOSTNAME:-"false"} +CHART_ENABLE_BASIC_AUTH=${CHART_ENABLE_BASIC_AUTH:-"false"} +BASIC_AUTH_USERNAME=${BASIC_AUTH_USERNAME:-"sysAdminUser"} +BASIC_AUTH_PASSWORD=${BASIC_AUTH_PASSWORD:-"myStrongPassword"} cleanup() { if [ "${SKIP_CLEANUP}" = "false" ]; then @@ -39,38 +46,91 @@ cleanup() { on_failure() { local exit_status=$? echo "Describe all resources in the ${SELENIUM_NAMESPACE} namespace for debugging purposes" - kubectl describe all -n ${SELENIUM_NAMESPACE} > tests/tests/describe_all_resources_${MATRIX_BROWSER}.txt + kubectl describe all -n ${SELENIUM_NAMESPACE} >> tests/tests/describe_all_resources_${MATRIX_BROWSER}.txt + kubectl describe pod -n ${SELENIUM_NAMESPACE} >> tests/tests/describe_all_resources_${MATRIX_BROWSER}.txt echo "There is step failed with exit status $exit_status" cleanup exit $exit_status } # Trap ERR signal and call on_failure function -trap 'on_failure' ERR +trap 'on_failure' ERR EXIT + +touch tests/tests/describe_all_resources_${MATRIX_BROWSER}.txt + +if [ -f .env ] +then + export "$(cat .env | xargs)" +else + export UPLOAD_ENABLED=false +fi +export RELEASE_NAME=${RELEASE_NAME} +RECORDER_VALUES_FILE=${TEST_VALUES_PATH}/base-recorder-values.yaml +envsubst < ${RECORDER_VALUES_FILE} > ./tests/tests/base-recorder-values.yaml +RECORDER_VALUES_FILE=./tests/tests/base-recorder-values.yaml + +HELM_COMMAND_SET_IMAGES=" \ +--set global.seleniumGrid.imageRegistry=${NAMESPACE} \ +--set global.seleniumGrid.imageTag=${VERSION} \ +--set global.seleniumGrid.nodesImageTag=${VERSION} \ +--set global.seleniumGrid.videoImageTag=${VIDEO_TAG} \ +--set autoscaling.scaledOptions.pollingInterval=${AUTOSCALING_POLL_INTERVAL} \ +--set tracing.enabled=${CHART_ENABLE_TRACING} \ +--set isolateComponents=${CHART_FULL_DISTRIBUTED_MODE} \ +" + +if [ "${CHART_ENABLE_INGRESS_HOSTNAME}" = "true" ]; then + if [[ ! $(cat /etc/hosts) == *"${HOSTNAME_ADDRESS}"* ]]; then + sudo -- sh -c -e "echo \"$(hostname -i) ${HOSTNAME_ADDRESS}\" >> /etc/hosts" + fi + ping -c 2 ${HOSTNAME_ADDRESS} + HELM_COMMAND_SET_IMAGES="${HELM_COMMAND_SET_IMAGES} \ + --set ingress.hostname=${HOSTNAME_ADDRESS} \ + " + SELENIUM_GRID_HOST=${HOSTNAME_ADDRESS} +else + HELM_COMMAND_SET_IMAGES="${HELM_COMMAND_SET_IMAGES} \ + --set global.K8S_PUBLIC_IP=${SELENIUM_GRID_HOST} \ + " +fi + +if [ "${CHART_ENABLE_BASIC_AUTH}" = "true" ]; then + HELM_COMMAND_SET_IMAGES="${HELM_COMMAND_SET_IMAGES} \ + --set basicAuth.enabled=${CHART_ENABLE_BASIC_AUTH} \ + --set basicAuth.username=${BASIC_AUTH_USERNAME} \ + --set basicAuth.password=${BASIC_AUTH_PASSWORD} \ + " + export SELENIUM_GRID_USERNAME=${BASIC_AUTH_USERNAME} + export SELENIUM_GRID_PASSWORD=${BASIC_AUTH_PASSWORD} +fi -HELM_COMMAND_SET_AUTOSCALING="" if [ "${SELENIUM_GRID_AUTOSCALING}" = "true" ]; then - HELM_COMMAND_SET_AUTOSCALING="--values ${TEST_VALUES_PATH}/DeploymentAutoScaling-values.yaml \ + HELM_COMMAND_SET_AUTOSCALING=" \ --set autoscaling.enableWithExistingKEDA=${SELENIUM_GRID_AUTOSCALING} \ - --set autoscaling.scaledOptions.minReplicaCount=${SELENIUM_GRID_AUTOSCALING_MIN_REPLICA}" + --set autoscaling.scaledOptions.minReplicaCount=${SELENIUM_GRID_AUTOSCALING_MIN_REPLICA} \ + " fi -HELM_COMMAND_SET_TLS="" +HELM_COMMAND_SET_BASE_VALUES=" \ +--values ${TEST_VALUES_PATH}/base-auth-ingress-values.yaml \ +--values ${RECORDER_VALUES_FILE} \ +--values ${TEST_VALUES_PATH}/base-resources-values.yaml \ +" + if [ "${SELENIUM_GRID_PROTOCOL}" = "https" ]; then - HELM_COMMAND_SET_TLS="--values ${TEST_VALUES_PATH}/tls-values.yaml" + HELM_COMMAND_SET_BASE_VALUES="${HELM_COMMAND_SET_BASE_VALUES} \ + --values ${TEST_VALUES_PATH}/base-tls-values.yaml \ + " fi +HELM_COMMAND_SET_BASE_VALUES="${HELM_COMMAND_SET_BASE_VALUES} \ +--values ${TEST_VALUES_PATH}/${MATRIX_BROWSER}-values.yaml \ +" + HELM_COMMAND_ARGS="${RELEASE_NAME} \ ---values ${TEST_VALUES_PATH}/auth-ingress-values.yaml \ ---values ${TEST_VALUES_PATH}/tracing-values.yaml \ +${HELM_COMMAND_SET_BASE_VALUES} \ ${HELM_COMMAND_SET_AUTOSCALING} \ -${HELM_COMMAND_SET_TLS} \ ---values ${TEST_VALUES_PATH}/${MATRIX_BROWSER}-values.yaml \ ---set global.seleniumGrid.imageRegistry=${NAMESPACE} \ ---set global.seleniumGrid.imageTag=${VERSION} \ ---set global.seleniumGrid.nodesImageTag=${VERSION} \ ---set global.seleniumGrid.videoImageTag=${VIDEO_TAG} \ ---set global.seleniumGrid.uploaderImageTag=${UPLOADER_TAG} \ +${HELM_COMMAND_SET_IMAGES} \ ${CHART_PATH} --namespace ${SELENIUM_NAMESPACE} --create-namespace" echo "Render manifests YAML for this deployment" @@ -79,6 +139,8 @@ helm template --debug ${HELM_COMMAND_ARGS} > tests/tests/cluster_deployment_mani echo "Deploy Selenium Grid Chart" helm upgrade --install ${HELM_COMMAND_ARGS} +kubectl get pods -A + echo "Run Tests" export CHART_CERT_PATH=$(readlink -f ${CHART_CERT_PATH}) export SELENIUM_GRID_PROTOCOL=${SELENIUM_GRID_PROTOCOL} @@ -90,12 +152,13 @@ export RUN_IN_DOCKER_COMPOSE=true export HUB_CHECKS_INTERVAL=${HUB_CHECKS_INTERVAL} export HUB_CHECKS_MAX_ATTEMPTS=${HUB_CHECKS_MAX_ATTEMPTS} export WEB_DRIVER_WAIT_TIMEOUT=${WEB_DRIVER_WAIT_TIMEOUT} +export SELENIUM_GRID_TEST_HEADLESS=${SELENIUM_GRID_TEST_HEADLESS:-"false"} ./tests/bootstrap.sh ${MATRIX_BROWSER} echo "Get pods status" kubectl get pods -n ${SELENIUM_NAMESPACE} -echo "Get all resources in the ${SELENIUM_NAMESPACE} namespace" -kubectl get all -n ${SELENIUM_NAMESPACE} > tests/tests/describe_all_resources_${MATRIX_BROWSER}.txt +echo "Get all resources in all namespaces" +kubectl get all -A >> tests/tests/describe_all_resources_${MATRIX_BROWSER}.txt cleanup diff --git a/tests/charts/refValues/sample-aws.yaml b/tests/charts/refValues/sample-aws.yaml index 097eb4277..dc7b7e61d 100644 --- a/tests/charts/refValues/sample-aws.yaml +++ b/tests/charts/refValues/sample-aws.yaml @@ -11,11 +11,10 @@ global: ingress: enabled: true annotations: - kubernetes.io/ingress.class: nginx nginx.ingress.kubernetes.io/use-regex: "true" nginx.ingress.kubernetes.io/rewrite-target: /$2 nginx.ingress.kubernetes.io/app-root: &gridAppRoot "/selenium" - ingressClassName: nginx + className: nginx hostname: "aws.ndviet.org" # Replace with your hostname paths: - path: /selenium(/|$)(.*) @@ -58,8 +57,8 @@ chromeNode: value: "300" - name: SE_VNC_NO_PASSWORD value: "true" - - name: SE_OPTS - value: "--enable-managed-downloads true" + - name: SE_NODE_ENABLE_MANAGED_DOWNLOADS + value: "true" firefoxNode: extraEnvironmentVariables: *extraEnvironmentVariablesNodes diff --git a/tests/charts/refValues/simplex-minikube.yaml b/tests/charts/refValues/simplex-minikube.yaml index 3541ac2db..9fc5162e8 100644 --- a/tests/charts/refValues/simplex-minikube.yaml +++ b/tests/charts/refValues/simplex-minikube.yaml @@ -22,11 +22,10 @@ tls: ingress: enabled: true annotations: - kubernetes.io/ingress.class: nginx nginx.ingress.kubernetes.io/use-regex: "true" nginx.ingress.kubernetes.io/rewrite-target: /$2 nginx.ingress.kubernetes.io/app-root: &gridAppRoot "/selenium" - ingressClassName: nginx + className: nginx hostname: "" paths: - path: /selenium(/|$)(.*) @@ -73,8 +72,8 @@ chromeNode: value: "300" - name: SE_VNC_NO_PASSWORD value: "true" - - name: SE_OPTS - value: "--enable-managed-downloads true" + - name: SE_NODE_ENABLE_MANAGED_DOWNLOADS + value: "true" firefoxNode: extraEnvironmentVariables: *extraEnvironmentVariablesNodes diff --git a/tests/charts/templates/render/dummy.yaml b/tests/charts/templates/render/dummy.yaml index 1e8411c6a..acf584ec9 100644 --- a/tests/charts/templates/render/dummy.yaml +++ b/tests/charts/templates/render/dummy.yaml @@ -14,6 +14,10 @@ global: - selenium topologyKey: "kubernetes.io/hostname" +autoscaling: + enableWithExistingKEDA: true + scalingType: deployment + basicAuth: username: sysadmin password: strongPassword @@ -83,7 +87,7 @@ videoRecorder: enabled: true uploader: enabled: true - destinationPrefix: "s3://ndviet" + destinationPrefix: "s3://bucket-name" secrets: RCLONE_CONFIG_S3_TYPE: "s3" RCLONE_CONFIG_S3_PROVIDER: "AWS" diff --git a/tests/charts/templates/test.py b/tests/charts/templates/test.py index 1d0196584..5ac422594 100644 --- a/tests/charts/templates/test.py +++ b/tests/charts/templates/test.py @@ -17,8 +17,14 @@ def load_template(yaml_file): class ChartTemplateTests(unittest.TestCase): def test_set_affinity(self): - resources_name = ['selenium-chrome-node', 'selenium-distributor', 'selenium-edge-node', 'selenium-firefox-node', - 'selenium-event-bus', 'selenium-router', 'selenium-session-map', 'selenium-session-queue'] + resources_name = ['{0}-selenium-chrome-node'.format(RELEASE_NAME), + '{0}-selenium-distributor'.format(RELEASE_NAME), + '{0}-selenium-edge-node'.format(RELEASE_NAME), + '{0}-selenium-firefox-node'.format(RELEASE_NAME), + '{0}-selenium-event-bus'.format(RELEASE_NAME), + '{0}-selenium-router'.format(RELEASE_NAME), + '{0}-selenium-session-map'.format(RELEASE_NAME), + '{0}-selenium-session-queue'.format(RELEASE_NAME)] count = 0 logger.info(f"Assert affinity is set in global and nodes") for doc in LIST_OF_DOCUMENTS: @@ -29,7 +35,7 @@ def test_set_affinity(self): self.assertEqual(count, len(resources_name), "Not all resources have affinity set") def test_ingress_nginx_annotations(self): - resources_name = ['selenium-ingress'] + resources_name = ['{0}-selenium-ingress'.format(RELEASE_NAME)] count = 0 for doc in LIST_OF_DOCUMENTS: if doc['metadata']['name'] in resources_name and doc['kind'] == 'Ingress': @@ -46,7 +52,7 @@ def test_ingress_nginx_annotations(self): self.assertEqual(count, len(resources_name), "No ingress resources found") def test_sub_path_append_to_node_grid_url(self): - resources_name = ['selenium-node-config'] + resources_name = ['{0}-selenium-node-config'.format(RELEASE_NAME)] count = 0 for doc in LIST_OF_DOCUMENTS: if doc['metadata']['name'] in resources_name and doc['kind'] == 'ConfigMap': @@ -56,7 +62,7 @@ def test_sub_path_append_to_node_grid_url(self): self.assertEqual(count, len(resources_name), "No node config resources found") def test_sub_path_set_to_grid_env_var(self): - resources_name = ['selenium-router'] + resources_name = ['{0}-selenium-router'.format(RELEASE_NAME)] is_present = False for doc in LIST_OF_DOCUMENTS: if doc['metadata']['name'] in resources_name and doc['kind'] == 'Deployment': @@ -68,7 +74,7 @@ def test_sub_path_set_to_grid_env_var(self): self.assertTrue(is_present, "ENV variable SE_SUB_PATH is not populated") def test_disable_ui_set_to_grid_env_var(self): - resources_name = ['selenium-router'] + resources_name = ['{0}-selenium-router'.format(RELEASE_NAME)] is_present = False for doc in LIST_OF_DOCUMENTS: if doc['metadata']['name'] in resources_name and doc['kind'] == 'Deployment': @@ -80,12 +86,18 @@ def test_disable_ui_set_to_grid_env_var(self): self.assertTrue(is_present, "ENV variable SE_DISABLE_UI is not populated") def test_log_level_set_to_logging_config_map(self): - resources_name = ['selenium-chrome-node', 'selenium-distributor', 'selenium-edge-node', 'selenium-firefox-node', - 'selenium-event-bus', 'selenium-router', 'selenium-session-map', 'selenium-session-queue'] + resources_name = ['{0}-selenium-chrome-node'.format(RELEASE_NAME), + '{0}-selenium-distributor'.format(RELEASE_NAME), + '{0}-selenium-edge-node'.format(RELEASE_NAME), + '{0}-selenium-firefox-node'.format(RELEASE_NAME), + '{0}-selenium-event-bus'.format(RELEASE_NAME), + '{0}-selenium-router'.format(RELEASE_NAME), + '{0}-selenium-session-map'.format(RELEASE_NAME), + '{0}-selenium-session-queue'.format(RELEASE_NAME)] logger.info(f"Assert log level value is set to logging ConfigMap") count_config = 0 for doc in LIST_OF_DOCUMENTS: - if doc['metadata']['name'] == 'selenium-logging-config' and doc['kind'] == 'ConfigMap': + if doc['metadata']['name'] == '{0}-selenium-logging-config'.format(RELEASE_NAME) and doc['kind'] == 'ConfigMap': self.assertTrue(doc['data']['SE_LOG_LEVEL'] == 'FINE') count_config += 1 self.assertEqual(count_config, 1, "No logging ConfigMap found") @@ -97,14 +109,16 @@ def test_log_level_set_to_logging_config_map(self): list_env_from = doc['spec']['template']['spec']['containers'][0]['envFrom'] for env in list_env_from: if env.get('configMapRef') is not None: - if env['configMapRef']['name'] == 'selenium-logging-config': + if env['configMapRef']['name'] == '{0}-selenium-logging-config'.format(RELEASE_NAME): is_present = True self.assertTrue(is_present, "envFrom doesn't contain logging ConfigMap") count += 1 self.assertEqual(count, len(resources_name), "Logging ConfigMap is not present in expected resources") def test_node_port_set_when_service_type_is_node_port(self): - single_node_port = {'selenium-distributor': 30553, 'selenium-router': 30444, 'selenium-session-queue': 30559} + single_node_port = {'{0}-selenium-distributor'.format(RELEASE_NAME): 30553, + '{0}-selenium-router'.format(RELEASE_NAME): 30444, + '{0}-selenium-session-queue'.format(RELEASE_NAME): 30559} count = 0 logger.info(f"Assert NodePort is set to components service") for doc in LIST_OF_DOCUMENTS: @@ -114,10 +128,18 @@ def test_node_port_set_when_service_type_is_node_port(self): count += 1 self.assertEqual(count, len(single_node_port.keys()), "Number of services with NodePort is not correct") + def test_all_metadata_name_is_prefixed_with_release_name(self): + logger.info(f"Assert all metadata name is prefixed with RELEASE NAME") + for doc in LIST_OF_DOCUMENTS: + logger.info(f"Assert metadata name: {doc['metadata']['name']}") + self.assertTrue(doc['metadata']['name'].startswith(RELEASE_NAME), + f"Metadata name {doc['metadata']['name']} is not prefixed with RELEASE NAME: {RELEASE_NAME}") + if __name__ == '__main__': failed = False try: FILE_NAME = sys.argv[1] + RELEASE_NAME = sys.argv[2] LIST_OF_DOCUMENTS = load_template(FILE_NAME) suite = unittest.TestLoader().loadTestsFromTestCase(ChartTemplateTests) test_runner = unittest.TextTestRunner(verbosity=3) diff --git a/tests/customCACert/Dockerfile b/tests/customCACert/Dockerfile new file mode 100644 index 000000000..38750c2af --- /dev/null +++ b/tests/customCACert/Dockerfile @@ -0,0 +1,21 @@ +ARG NAMESPACE=selenium +ARG BASE=node-chrome +ARG VERSION=latest +FROM ${NAMESPACE}/${BASE}:${VERSION} + +USER root + +ARG CERT_FILE +ARG PERM=577 +RUN mkdir -p /usr/share/ca-certificates/extra/ +COPY ${CERT_FILE} /usr/share/ca-certificates/extra/ +RUN chmod -R ${PERM} /usr/share/ca-certificates/extra/ +RUN update-ca-certificates + +ARG CERT_SCRIPT +COPY --chown="${SEL_UID}:${SEL_GID}" ${CERT_SCRIPT} /usr/share/ca-certificates/cert-script.sh +RUN chmod +x /usr/share/ca-certificates/cert-script.sh + +USER ${SEL_UID} +ARG TRUST_ATTR=TCu,Cu,Tu +RUN bash /usr/share/ca-certificates/cert-script.sh /usr/share/ca-certificates/extra/ ${TRUST_ATTR} diff --git a/tests/customCACert/bootstrap.sh b/tests/customCACert/bootstrap.sh new file mode 100755 index 000000000..cd83a37de --- /dev/null +++ b/tests/customCACert/bootstrap.sh @@ -0,0 +1,32 @@ +#!/bin/bash +set -x + +# Function to be executed on command failure +on_failure() { + local exit_status=$? + echo "There is step failed with exit status $exit_status" + exit $exit_status +} + +# Trap ERR signal and call on_failure function +trap 'on_failure' ERR + +NAMESPACE=${NAME:-"selenium"} +VERSION=${VERSION:-$TAG_VERSION} +CERT_FILE=${CERT_FILE:-"./charts/selenium-grid/certs/*.pem"} +CERT_SCRIPT=${CERT_SCRIPT:-"./tests/customCACert/cert-script.sh"} + +COMMON_BUILD_ARGS="--build-arg NAMESPACE=${NAMESPACE} --build-arg VERSION=${VERSION} --build-arg CERT_FILE=${CERT_FILE} --build-arg CERT_SCRIPT=${CERT_SCRIPT}" + +docker build ${COMMON_BUILD_ARGS} --build-arg BASE=node-chrome -t ${NAMESPACE}/node-chrome:${VERSION} -f ./tests/customCACert/Dockerfile . +docker build ${COMMON_BUILD_ARGS} --build-arg BASE=node-firefox -t ${NAMESPACE}/node-firefox:${VERSION} -f ./tests/customCACert/Dockerfile . +docker build ${COMMON_BUILD_ARGS} --build-arg BASE=node-edge -t ${NAMESPACE}/node-edge:${VERSION} -f ./tests/customCACert/Dockerfile . + +list_cert_files=($(find ./charts/selenium-grid/certs/ -name "*.pem")) +for cert_file_path in "${list_cert_files[@]}"; do + cert_file_name="$(basename ${cert_file_path})" + cert_name="${cert_file_name%.*}" + docker run --entrypoint="" --rm ${NAMESPACE}/node-chrome:${VERSION} bash -c "certutil -L -d sql:/home/seluser/.pki/nssdb -n ${cert_name}" | grep -q "SeleniumHQ" + docker run --entrypoint="" --rm ${NAMESPACE}/node-firefox:${VERSION} bash -c "certutil -L -d sql:/home/seluser/.pki/nssdb -n ${cert_name}" | grep -q "SeleniumHQ" + docker run --entrypoint="" --rm ${NAMESPACE}/node-edge:${VERSION} bash -c "certutil -L -d sql:/home/seluser/.pki/nssdb -n ${cert_name}" | grep -q "SeleniumHQ" +done diff --git a/tests/customCACert/cert-script.sh b/tests/customCACert/cert-script.sh new file mode 100644 index 000000000..24ab4389c --- /dev/null +++ b/tests/customCACert/cert-script.sh @@ -0,0 +1,24 @@ +#!/bin/bash +set -x + +echo "$(whoami) is running cert script!" + +CERT_DIR=${1:-"/usr/share/ca-certificates/extra/"} +TRUST_ATTR=${2:-"TCu,Cu,Tu"} + +# Get the list of certs copied +cert_files=($(ls $CERT_DIR)) + +# Find all "cert9.db" files +cert_db_files=($(find ${HOME}/ -name "cert9.db")) + +for cert_file in "${cert_files[@]}"; do + cert_file="${CERT_DIR}/${cert_file}" + file_name=$(basename $cert_file) + cert_name="${file_name%.*}" + for cert_db_file in "${cert_db_files[@]}"; do + cert_db_path=$(dirname $cert_db_file) + certutil -d "sql:${cert_db_path}" -A -t "${TRUST_ATTR}" -n "${cert_name}" -i "${cert_file}" + certutil -L -d "sql:${cert_db_path}" -n "${cert_name}" + done +done diff --git a/tests/docker-compose-v3-test-video.yml b/tests/docker-compose-v3-test-video.yml index bbfb6f6ab..308bbcfce 100644 --- a/tests/docker-compose-v3-test-video.yml +++ b/tests/docker-compose-v3-test-video.yml @@ -5,6 +5,7 @@ version: "3" services: browser: image: selenium/node-${BROWSER}:${TAG} + user: ${UID} shm_size: 2gb depends_on: - selenium-hub @@ -12,12 +13,13 @@ services: - SE_EVENT_BUS_HOST=selenium-hub - SE_EVENT_BUS_PUBLISH_PORT=4442 - SE_EVENT_BUS_SUBSCRIBE_PORT=4443 - - SE_OPTS=--enable-managed-downloads true + - SE_NODE_ENABLE_MANAGED_DOWNLOADS=true ports: - "6900:5900" browser_video: image: selenium/video:${VIDEO_TAG} + user: ${UID} volumes: - ./videos:/videos depends_on: @@ -28,6 +30,7 @@ services: selenium-hub: image: selenium/hub:${TAG} + user: ${UID} container_name: selenium-hub ports: - "4442:4442" diff --git a/tests/test.py b/tests/test.py index 975772103..5ef93f4f7 100644 --- a/tests/test.py +++ b/tests/test.py @@ -65,7 +65,8 @@ 'StandaloneChromium': 'ChromeTests', # Chart Parallel Test - 'JobAutoscaling': 'JobAutoscalingTests' + 'JobAutoscaling': 'JobAutoscalingTests', + 'DeploymentAutoscaling': 'DeploymentAutoscalingTests', } FROM_IMAGE_ARGS = { @@ -148,7 +149,7 @@ def launch_container(container, **kwargs): 'SE_EVENT_BUS_SUBSCRIBE_PORT': 4443 } if container != 'Hub': - environment['SE_OPTS'] = "--enable-managed-downloads true" + environment['SE_NODE_ENABLE_MANAGED_DOWNLOADS'] = "true" container_id = client.containers.run("%s/%s:%s" % (NAMESPACE, IMAGE_NAME_MAP[container], VERSION), detach=True, environment=environment,