Merge remote-tracking branch 'upstream/main'

zelinh · Dec 5, 2023 · 63f4dfa · 63f4dfa
2 parents d655fe8 + 6a5d0cd
commit 63f4dfa
Show file tree

Hide file tree

Showing 70 changed files with 5,885 additions and 476 deletions.
diff --git a/.github/workflows/automatic-merges.yml b/.github/workflows/automatic-merges.yml
@@ -15,6 +15,7 @@ on:
 
 jobs:
   automatic-merge-manifests:
+    if: github.repository == 'opensearch-project/opensearch-build'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -47,6 +48,7 @@ jobs:
             manifests/*/*.yml
 
   automatic-merge-new-manifests:
+    if: github.repository == 'opensearch-project/opensearch-build'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4

diff --git a/.github/workflows/manifests.yml b/.github/workflows/manifests.yml
@@ -3,9 +3,6 @@ name: manifests
 
 on:
   pull_request:
-    paths:
-      - 'manifests/**/*.yml'
-      - '!manifests/templates/**/'
 
 jobs:
   list-changed-manifests:
@@ -28,45 +25,26 @@ jobs:
         id: set-matrix
         run: echo "matrix={\"manifest\":${{ steps.list-changed-manifests.outputs.all_changed_files }}}" >> "$GITHUB_OUTPUT"
 
-  manifest-checks-jdk11:
+  manifest-checks:
     needs: [list-changed-manifests]
     runs-on: ubuntu-latest
     env:
       PYTHON_VERSION: 3.9
-      JDK_VERSION: 11
     strategy:
+      fail-fast: false
       matrix: ${{ fromJson(needs.list-changed-manifests.outputs.matrix) }}
     steps:
       - uses: actions/checkout@v3
-      - name: Set Up JDK ${{ env.JDK_VERSION }}
-        uses: actions/setup-java@v1
-        with:
-          java-version: ${{ env.JDK_VERSION }}
-      - name: Set up Python ${{ env.PYTHON_VERSION }}
-        uses: actions/setup-python@v3
-        with:
-          python-version: ${{ env.PYTHON_VERSION }}
-      - name: Install Pipenv and Dependencies
+      - name: Check JDK Version
         run: |
-          python -m pip install --upgrade pipenv wheel
-      - name: OpenSearch Manifests
-        run: |-
-          ./ci.sh ${{ matrix.manifest }} --snapshot
-
-  manifest-checks-jdk17:
-    needs: [list-changed-manifests]
-    runs-on: ubuntu-latest
-    env:
-      PYTHON_VERSION: 3.9
-      JDK_VERSION: 17
-    strategy:
-      matrix: ${{ fromJson(needs.list-changed-manifests.outputs.matrix) }}
-    steps:
-      - uses: actions/checkout@v3
-      - name: Set Up JDK ${{ env.JDK_VERSION }}
+          java_version=`cat ${{ matrix.manifest }} | yq -r .ci.image.args | grep -Eo '[0-9]+' || echo ''`
+          echo $java_version
+          echo "JAVA_VERSION=$java_version" >> "$GITHUB_ENV"
+      - name: Set Up JDK ${{ env.JAVA_VERSION }}
+        if: ${{ env.JAVA_VERSION }}
         uses: actions/setup-java@v1
         with:
-          java-version: ${{ env.JDK_VERSION }}
+          java-version: ${{ env.JAVA_VERSION }}
       - name: Set up Python ${{ env.PYTHON_VERSION }}
         uses: actions/setup-python@v3
         with:

diff --git a/.github/workflows/publish-wiki.yml b/.github/workflows/publish-wiki.yml
@@ -0,0 +1,23 @@
+---
+name: Publish wiki
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - docs/**
+      - .github/workflows/publish-wiki.yml
+
+concurrency:
+  group: publish-wiki
+  cancel-in-progress: true
+
+permissions:
+  contents: write
+
+jobs:
+  publish-wiki:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: Andrew-Chen-Wang/github-wiki-action@v4
diff --git a/docker/ci/config/windows-servercore-setup.ps1 b/docker/ci/config/windows-servercore-setup.ps1
@@ -84,7 +84,7 @@ regedit /s $zlibRegFilePath
 # Temurin jdk does not have all the versions supported on scoop, especially version 14, 20, and above
 # As of now we will mix temurin and openjdk as temurin for production has support policies for fixes and patches
 # We need to make sure we do not mis-install temurin and openjdk with the same version or the distribution build code will have issues
-$jdkVersionList = "temurin8-jdk JAVA8_HOME", "temurin11-jdk JAVA11_HOME", "openjdk14 JAVA14_HOME", "temurin17-jdk JAVA17_HOME", "temurin19-jdk JAVA19_HOME", "openjdk20 JAVA20_HOME", "openjdk21 JAVA21_HOME"
+$jdkVersionList = "temurin8-jdk JAVA8_HOME", "temurin11-jdk JAVA11_HOME", "openjdk14 JAVA14_HOME", "temurin17-jdk JAVA17_HOME", "temurin19-jdk JAVA19_HOME", "openjdk20 JAVA20_HOME", "temurin21-jdk JAVA21_HOME"
 Foreach ($jdkVersion in $jdkVersionList)
 {
     $jdkVersion

diff --git a/docker/ci/dockerfiles/current/build.centos7.opensearch.x64.arm64.dockerfile b/docker/ci/dockerfiles/current/build.centos7.opensearch.x64.arm64.dockerfile
@@ -13,6 +13,8 @@
 FROM centos:7
 
 ARG MAVEN_DIR=/usr/local/apache-maven
+ARG CONTAINER_USER=ci-runner
+ARG CONTAINER_USER_HOME=/home/ci-runner
 
 # Ensure localedef running correct with root permission
 USER 0
@@ -24,10 +26,10 @@ RUN yum clean all && yum-config-manager --add-repo https://cli.github.com/packag
     yum install -y which curl git gnupg2 tar net-tools procps-ng python3 python3-devel python3-pip zip unzip jq gh pigz
 
 # Create user group
-RUN groupadd -g 1000 opensearch && \
-    useradd -u 1000 -g 1000 -d /usr/share/opensearch opensearch && \
-    mkdir -p /usr/share/opensearch && \
-    chown -R 1000:1000 /usr/share/opensearch
+RUN groupadd -g 1000 $CONTAINER_USER && \
+    useradd -u 1000 -g 1000 -d $CONTAINER_USER_HOME $CONTAINER_USER && \
+    mkdir -p $CONTAINER_USER_HOME && \
+    chown -R 1000:1000 $CONTAINER_USER_HOME
 
 # Add Python dependencies
 RUN yum install -y @development zlib-devel bzip2 bzip2-devel readline-devel sqlite sqlite-devel openssl-devel xz xz-devel libffi-devel findutils
@@ -70,7 +72,7 @@ RUN . /etc/profile.d/rvm.sh && rvm install 2.6.0 && rvm --default use 2.6.0 && y
 
 ENV RUBY_HOME=/usr/local/rvm/rubies/ruby-2.6.0/bin
 ENV RVM_HOME=/usr/local/rvm/bin
-ENV GEM_HOME=/usr/share/opensearch/.gem
+ENV GEM_HOME=$CONTAINER_USER_HOME/.gem
 ENV GEM_PATH=$GEM_HOME
 ENV PATH=$RUBY_HOME:$RVM_HOME:$PATH
 
@@ -101,10 +103,10 @@ ENV ENV="/usr/local/bin/scl_setup"
 ENV PROMPT_COMMAND=". /usr/local/bin/scl_setup"
 
 # Change User
-USER 1000
-WORKDIR /usr/share/opensearch
+USER $CONTAINER_USER
+WORKDIR $CONTAINER_USER_HOME
 
 # Install fpm for opensearch dashboards core
 RUN gem install fpm -v 1.14.2
-ENV PATH=/usr/share/opensearch/.gem/gems/fpm-1.14.2/bin:$PATH
+ENV PATH=$CONTAINER_USER_HOME/.gem/gems/fpm-1.14.2/bin:$PATH
 RUN fpm -v
diff --git a/docker/ci/dockerfiles/current/build.rockylinux8.opensearch-dashboards.x64.arm64.dockerfile b/docker/ci/dockerfiles/current/build.rockylinux8.opensearch-dashboards.x64.arm64.dockerfile
@@ -11,6 +11,9 @@
 
 FROM rockylinux:8
 
+ARG CONTAINER_USER=ci-runner
+ARG CONTAINER_USER_HOME=/home/ci-runner
+
 # Ensure localedef running correct with root permission
 USER 0
 
@@ -26,10 +29,10 @@ COPY --chown=0:0 config/yq-setup.sh /tmp
 RUN /tmp/yq-setup.sh
 
 # Create user group
-RUN groupadd -g 1000 opensearch && \
-    useradd -u 1000 -g 1000 -d /usr/share/opensearch opensearch && \
-    mkdir -p /usr/share/opensearch && \
-    chown -R 1000:1000 /usr/share/opensearch
+RUN groupadd -g 1000 $CONTAINER_USER && \
+    useradd -u 1000 -g 1000 -d $CONTAINER_USER_HOME $CONTAINER_USER && \
+    mkdir -p $CONTAINER_USER_HOME && \
+    chown -R 1000:1000 $CONTAINER_USER_HOME
 
 # Add Python dependencies
 RUN dnf install -y @development zlib-devel bzip2 bzip2-devel readline-devel sqlite sqlite-devel openssl-devel xz xz-devel libffi-devel findutils
@@ -59,7 +62,7 @@ RUN . /etc/profile.d/rvm.sh && rvm install 2.6.0 && rvm --default use 2.6.0 && d
 
 ENV RUBY_HOME=/usr/local/rvm/rubies/ruby-2.6.0/bin
 ENV RVM_HOME=/usr/local/rvm/bin
-ENV GEM_HOME=/usr/share/opensearch/.gem
+ENV GEM_HOME=$CONTAINER_USER_HOME/.gem
 ENV GEM_PATH=$GEM_HOME
 ENV PATH=$RUBY_HOME:$RVM_HOME:$PATH
 
@@ -73,23 +76,23 @@ RUN update-alternatives --set python /usr/bin/python3.9 && \
 #ENV AWS_CLI_FILE_ENCODING=UTF-8
 
 # Change User
-USER 1000
-WORKDIR /usr/share/opensearch
+USER $CONTAINER_USER
+WORKDIR $CONTAINER_USER_HOME
 
 # Install fpm for opensearch dashboards core
 RUN gem install fpm -v 1.14.2
-ENV PATH=/usr/share/opensearch/.gem/gems/fpm-1.14.2/bin:$PATH
+ENV PATH=$CONTAINER_USER_HOME/.gem/gems/fpm-1.14.2/bin:$PATH
 
 # Hard code node version and yarn version for now
 # nvm environment variables
-ENV NVM_DIR /usr/share/opensearch/.nvm
+ENV NVM_DIR $CONTAINER_USER_HOME/.nvm
 ENV NODE_VERSION 10.24.1
 ARG NODE_VERSION_LIST="10.24.1 14.19.1 14.20.0 14.20.1 14.21.3 16.20.0 18.16.0"
 # install nvm
 # https://github.com/creationix/nvm#install-script
 RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.38.0/install.sh | bash
 # install node and npm
-COPY --chown=1000:1000 config/yarn-version.sh /tmp
+COPY --chown=$CONTAINER_USER:$CONTAINER_USER config/yarn-version.sh /tmp
 RUN source $NVM_DIR/nvm.sh && \
     for node_version in $NODE_VERSION_LIST; do nvm install $node_version; npm install -g yarn@`/tmp/yarn-version.sh main`; done
 # add node and npm to path so the commands are available

diff --git a/docker/ci/dockerfiles/current/build.rockylinux8.opensearch.x64.arm64.dockerfile b/docker/ci/dockerfiles/current/build.rockylinux8.opensearch.x64.arm64.dockerfile
@@ -13,6 +13,8 @@
 FROM rockylinux:8
 
 ARG MAVEN_DIR=/usr/local/apache-maven
+ARG CONTAINER_USER=ci-runner
+ARG CONTAINER_USER_HOME=/home/ci-runner
 
 # Ensure localedef running correct with root permission
 USER 0
@@ -23,10 +25,10 @@ RUN dnf clean all && dnf install -y 'dnf-command(config-manager)' && dnf config-
     dnf install -y which curl git gnupg2 tar net-tools procps-ng python39 python39-devel python39-pip zip unzip jq gh
 
 # Create user group
-RUN groupadd -g 1000 opensearch && \
-    useradd -u 1000 -g 1000 -d /usr/share/opensearch opensearch && \
-    mkdir -p /usr/share/opensearch && \
-    chown -R 1000:1000 /usr/share/opensearch
+RUN groupadd -g 1000 $CONTAINER_USER && \
+    useradd -u 1000 -g 1000 -d $CONTAINER_USER_HOME $CONTAINER_USER && \
+    mkdir -p $CONTAINER_USER_HOME && \
+    chown -R 1000:1000 $CONTAINER_USER_HOME
 
 # Add Python dependencies
 RUN dnf install -y @development zlib-devel bzip2 bzip2-devel readline-devel sqlite sqlite-devel openssl-devel xz xz-devel libffi-devel findutils
@@ -69,7 +71,7 @@ RUN . /etc/profile.d/rvm.sh && rvm install 2.6.0 && rvm --default use 2.6.0 && d
 
 ENV RUBY_HOME=/usr/local/rvm/rubies/ruby-2.6.0/bin
 ENV RVM_HOME=/usr/local/rvm/bin
-ENV GEM_HOME=/usr/share/opensearch/.gem
+ENV GEM_HOME=$CONTAINER_USER_HOME/.gem
 ENV GEM_PATH=$GEM_HOME
 ENV PATH=$RUBY_HOME:$RVM_HOME:$PATH
 
@@ -87,10 +89,10 @@ RUN dnf install -y 'dnf-command(config-manager)' && \
 RUN pip3 install cmake==3.23.3
 
 # Change User
-USER 1000
-WORKDIR /usr/share/opensearch
+USER $CONTAINER_USER
+WORKDIR $CONTAINER_USER_HOME
 
 # Install fpm for opensearch dashboards core
 RUN gem install fpm -v 1.14.2
-ENV PATH=/usr/share/opensearch/.gem/gems/fpm-1.14.2/bin:$PATH
+ENV PATH=$CONTAINER_USER_HOME/.gem/gems/fpm-1.14.2/bin:$PATH
 RUN fpm -v
diff --git a/docker/ci/dockerfiles/current/build.ubuntu2004.opensearch.x64.arm64.dockerfile b/docker/ci/dockerfiles/current/build.ubuntu2004.opensearch.x64.arm64.dockerfile
@@ -11,6 +11,8 @@
 FROM ubuntu:20.04
 
 ARG DEBIAN_FRONTEND=noninteractive
+ARG CONTAINER_USER=ci-runner
+ARG CONTAINER_USER_HOME=/home/ci-runner
 
 # Install python dependencies
 RUN apt-get update -y && apt-get install -y software-properties-common
@@ -42,13 +44,13 @@ RUN curl -o- https://www.aptly.info/pubkey.txt | apt-key add - && \
 
 # Tools setup
 COPY --chown=0:0 config/jdk-setup.sh config/yq-setup.sh /tmp/
-RUN /tmp/jdk-setup.sh && /tmp/yq-setup.sh
+RUN /tmp/jdk-setup.sh && /tmp/yq-setup.sh # Ubuntu has a bug where entrypoint=bash does not actually run .bashrc correctly
 
 # Create user group
-RUN groupadd -g 1000 opensearch && \
-    useradd -u 1000 -g 1000 -d /usr/share/opensearch -m opensearch && \
-    mkdir -p /usr/share/opensearch && \
-    chown -R 1000:1000 /usr/share/opensearch 
+RUN groupadd -g 1000 $CONTAINER_USER && \
+    useradd -u 1000 -g 1000 -s /bin/bash -d $CONTAINER_USER_HOME -m $CONTAINER_USER && \
+    mkdir -p $CONTAINER_USER_HOME && \
+    chown -R 1000:1000 $CONTAINER_USER_HOME
 
 # Install gh cli
 RUN curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | dd of=/usr/share/keyrings/githubcli-archive-keyring.gpg && \
@@ -57,5 +59,5 @@ RUN curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | d
     apt-get update && apt-get install -y gh && apt-get clean
 
 # Change User
-USER 1000
-WORKDIR /usr/share/opensearch
+USER $CONTAINER_USER
+WORKDIR $CONTAINER_USER_HOME
diff --git a/docker/ci/dockerfiles/current/docker-builder.ubuntu2004.x64.dockerfile b/docker/ci/dockerfiles/current/docker-builder.ubuntu2004.x64.dockerfile
@@ -17,6 +17,8 @@
 FROM ubuntu:20.04
 
 ARG DEBIAN_FRONTEND=noninteractive
+ARG CONTAINER_USER=ci-runner
+ARG CONTAINER_USER_HOME=/home/ci-runner
 
 # Import necessary repository for installing qemu 5.0
 RUN apt-get update -y && apt-get install -y software-properties-common && add-apt-repository ppa:jacob/virtualisation -y
@@ -45,10 +47,10 @@ RUN curl -SL https://github.com/adoptium/temurin11-binaries/releases/download/jd
     rm /opt/jdk11.tar.gz
 
 # Create user group
-RUN groupadd -g 1000 opensearch && \
-    useradd -u 1000 -g 1000 -d /usr/share/opensearch opensearch && \
-    mkdir -p /usr/share/opensearch && \
-    chown -R 1000:1000 /usr/share/opensearch 
+RUN groupadd -g 1000 $CONTAINER_USER && \
+    useradd -u 1000 -g 1000 -s /bin/bash -d $CONTAINER_USER_HOME -m $CONTAINER_USER && \
+    mkdir -p $CONTAINER_USER_HOME && \
+    chown -R 1000:1000 $CONTAINER_USER_HOME
 
 # ENV JDK
 ENV JAVA_HOME=/opt/java/openjdk-11