v2.3.4

Signed-off-by: Ji Bin <[email protected]>

.devcontainer/Dockerfile

@@ -0,0 +1,34 @@
+FROM milvusdb/milvus-env:amd64-centos7-20230606-c9d9940
+
+# Ignore tool missing warnings, and wo also not need clang-tools
+RUN rm -fr /etc/profile.d/llvm-toolset*.sh
+
+# python38 for dev
+RUN yum -y install rh-python38
+RUN echo 'source scl_source enable rh-python38' > /etc/profile.d/rh-python38.sh
+
+# git new for devel
+RUN yum -y install rh-git227-git-all
+RUN echo 'source scl_source enable rh-git227' > /etc/profile.d/rh-git227.sh
+
+# Add local user in container for dev
+RUN yum -y install sudo
+ADD setup/env /tmp/env
+RUN . /tmp/env && \
+    if getent group $DEV_GROUP 2>/dev/null 1>/dev/null ; then \
+        echo group $DEV_GROUP already exist ; \
+    elif getent group $DEV_GID 2>/dev/null 1>/dev/null ; then \
+        echo group $DEV_GID already exist ; \
+    else \
+        groupadd -g $DEV_GID $DEV_GROUP ; \
+    fi ; \
+    if id $DEV_USER 2>/dev/null 1>/dev/null ; then \
+        echo user $DEV_USER already exist ; \
+        exit 1 ; \
+    elif id $DEV_UID 2>/dev/null 1>/dev/null ; then \
+        echo user $DEV_UID already exist ; \
+        exit 1 ; \
+    else \
+        useradd -g $DEV_GID -u $DEV_UID -m -d $DEV_HOME -s /bin/bash $DEV_USER ; \
+    fi && \
+    echo "${DEV_USER} ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/user

.devcontainer/devcontainer.json

@@ -0,0 +1,20 @@
+{
+    "name": "Embd Milvus Development",
+    "build": {
+        "dockerfile": "Dockerfile"
+    },
+    "initializeCommand": "bash .devcontainer/setup/gen_env.sh",
+    "onCreateCommand": "bash .devcontainer/setup/setup_dev.sh",
+    "containerUser": "${localEnv:USER}",
+    "customizations": {
+        "vscode": {
+            "extensions": [
+                "ms-python.python",
+                "bungcip.better-toml"
+            ]
+        }
+    },
+    "mounts": [
+        "source=${localWorkspaceFolder}/.conan,target=/home/${localEnv:USER}/.conan,type=bind,consistency=cached"
+    ]
+}

.devcontainer/setup/.gitignore

@@ -0,0 +1 @@
+env

.devcontainer/setup/gen_env.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+this_dir=$(dirname $0)
+
+echo -n > ${this_dir}/env
+
+echo DEV_USER=$(id -un) >> ${this_dir}/env
+echo DEV_UID=$(id -u) >> ${this_dir}/env
+echo DEV_GROUP=$(id -gn) >> ${this_dir}/env
+echo DEV_GID=$(id -g) >> ${this_dir}/env
+echo DEV_HOME=$HOME >> ${this_dir}/env
+echo DEV_SHELL=$SHELL >> ${this_dir}/env
+
+mkdir -p ${this_dir}/../../.conan

.devcontainer/setup/setup_dev.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -e
+
+project_dir=$(dirname $(dirname $(cd $(dirname $0); pwd)))
+
+python3 -m pip install --user -U pip
+python3 -m pip install --user -r ${project_dir}/requirements.txt

.github/workflows/main.yml

@@ -0,0 +1,282 @@
+on:
+  push:
+    branches:
+      - '*'
+  pull_request:
+
+jobs:
+  build_linux:
+    name: Build Wheel - Linux
+    runs-on: ubuntu-latest
+    container:
+      # note: keep same as .devcontainer/Dockerfile
+      image: milvusdb/milvus-env:amd64-centos7-20230606-c9d9940
+    env:
+      CCACHE_DIR: ${{ github.workspace }}/.ccache
+      CCACHE_COMPILERCHECK: content
+      CCACHE_COMPRESS: 1
+      CCACHE_COMPRESSLEVEL: 5
+      CCACHE_MAXSIZE: 2G
+    timeout-minutes: 360
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Cache go
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: linux-go-${{ hashFiles('milvus_binary/env.sh') }}
+          restore-keys: linux-go-
+      - name: Cache conan
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/.conan/data
+          key: linux-conan-${{ hashFiles('milvus_binary/env.sh') }}
+          restore-keys: linux-conan-
+      - name: Cache ccache
+        uses: actions/cache@v3
+        with:
+          path: |
+            ${{ github.workspace }}/.ccache
+          key: linux-ccache-${{ hashFiles('milvus_binary/env.sh') }}
+          restore-keys: linux-ccache-
+      - name: Install build requires
+        run: |
+          yum -y install rh-python38 rh-git227-git-all patch
+      - name: Build Wheel
+        run: |
+          # devtoolset 11
+          export PATH=/opt/rh/devtoolset-11/root/usr/bin${PATH:+:${PATH}}
+          export LD_LIBRARY_PATH=/opt/rh/devtoolset-11/root$rpmlibdir$rpmlibdir32${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}
+          # python 3.8
+          export PATH=/opt/rh/rh-python38/root/usr/local/bin:/opt/rh/rh-python38/root/usr/bin${PATH:+:${PATH}}
+          export LD_LIBRARY_PATH=/opt/rh/rh-python38/root/usr/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}
+          # git
+          export PATH=/opt/rh/rh-git227/root/usr/bin${PATH:+:${PATH}}
+          export LD_LIBRARY_PATH=/opt/rh/httpd24/root/usr/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}
+          python3 -m pip install --user -U pip
+          python3 -m pip install --user build wheel 'setuptools>64.0'
+          # install cmake
+          python3 -m pip install cmake==3.27.7
+          # install patchelf
+          yum -y install patchelf
+          yum -y install perl-IPC-Cmd
+          python3 -m build -w -n
+      - uses: actions/upload-artifact@v3
+        with:
+          name: wheel_linux
+          path: dist/*.whl
+          retention-days: 5
+  build_macos:
+    name: Build Wheel - macOS
+    runs-on: macos-11
+    env:
+      CCACHE_DIR: ${{ github.workspace }}/.ccache
+      CCACHE_COMPILERCHECK: content
+      CCACHE_COMPRESS: 1
+      CCACHE_COMPRESSLEVEL: 5
+      CCACHE_MAXSIZE: 2G
+    timeout-minutes: 600
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Cache go
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: macos-go-${{ hashFiles('milvus_binary/env.sh') }}
+          restore-keys: macos-go-
+      - name: Cache conan
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/.conan/data
+          key: macos-conan-${{ hashFiles('milvus_binary/env.sh') }}
+          restore-keys: macos-conan-
+      - name: Cache ccache
+        uses: actions/cache@v3
+        with:
+          path: |
+            ${{ github.workspace }}/.ccache
+          key: macos-ccache-${{ hashFiles('milvus_binary/env.sh') }}
+          restore-keys: macos-ccache-
+      - name: Setup Go environment
+        uses: actions/setup-go@v4
+        with:
+          go-version: '~1.18.10'
+          cache: false
+      - name: Build Wheel
+        run: |
+          python3 -m pip install --user build wheel 'setuptools>64.0'
+          python3 -m build -w -n
+      - uses: actions/upload-artifact@v3
+        with:
+          name: wheel_macos
+          path: dist/*.whl
+          retention-days: 5
+  # build_windows:
+  #   name: Build Wheel - windows
+  #   runs-on: windows-latest
+  #   env:
+  #     CCACHE_DIR: ${{ github.workspace }}/.ccache
+  #     CCACHE_COMPILERCHECK: content
+  #     CCACHE_COMPRESS: 1
+  #     CCACHE_COMPRESSLEVEL: 5
+  #     CCACHE_MAXSIZE: 2G
+  #   timeout-minutes: 180
+  #   steps:
+  #     - name: Set git to use LF
+  #       run: |
+  #         git config --global core.autocrlf false
+  #         git config --global core.eol lf
+  #     - name: Checkout
+  #       uses: actions/checkout@v3
+  #     - name: Cache go
+  #       uses: actions/cache@v3
+  #       with:
+  #         path: |
+  #           ~\AppData\Local\go-build
+  #           ~\go\pkg\mod
+  #         key: windows-go-${{ hashFiles('milvus_binary/env.sh') }}
+  #         restore-keys: windows-go-
+  #     - name: Cache conan
+  #       uses: actions/cache@v3
+  #       with:
+  #         path: |
+  #           ~/.conan/data
+  #         key: windows-conan-${{ hashFiles('milvus_binary/env.sh') }}
+  #         restore-keys: windows-conan-
+  #     - name: Cache ccache
+  #       uses: actions/cache@v3
+  #       with:
+  #         path: |
+  #           ${{ github.workspace }}/.ccache
+  #         key: windows-ccache-${{ hashFiles('milvus_binary/env.sh') }}
+  #         restore-keys: windows-ccache-
+  #     - name: Configure Toolchain
+  #       uses: msys2/[email protected]
+  #       with:
+  #         msystem: mingw64
+  #     - name: Prepare
+  #       shell: msys2 {0}
+  #       run: |
+  #         # workaround for keyring error
+  #         if ! pacman -S --noconfirm --needed git patch ; then
+  #           pacman-key --refresh-keys
+  #           pacman -S --noconfirm --needed git patch
+  #         fi
+  #         pacman -S --noconfirm --needed mingw-w64-x86_64-python mingw-w64-x86_64-python-wheel mingw-w64-x86_64-python-pip
+  #     - name: Build Wheel
+  #       shell: msys2 {0}
+  #       run: |
+  #         python3 -m pip install --user build wheel 'setuptools>64.0'
+  #         python3 -m build -w -n
+  #     - uses: actions/upload-artifact@v3
+  #       with:
+  #         name: wheel_windows
+  #         path: dist/*.whl
+  #         retention-days: 5
+
+  acceptance_test_many_linux:
+    needs:
+      - build_linux
+    name: Acceptance Test ${{ matrix.os }}
+    runs-on: ubuntu-latest
+    container:
+      image: ${{ matrix.os }}
+    timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      matrix:
+        os: ["ubuntu:18.04", "ubuntu:22.04", "centos:7", "fedora:36"]
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/download-artifact@v3
+        with:
+          name: wheel_linux
+          path: dist
+      - name: Install python3 for os
+        shell: bash
+        run: |
+          if [[ "${{ matrix.os }}" == "centos:7" ]] ; then
+            yum -y install epel-release
+            yum -y install python36 python36-pip python36-wheel
+          elif [[ "${{ matrix.os }}" == "fedora:36" ]] ; then
+            dnf -y install python3 python3-pip python3-wheel python3-devel gcc gcc-c++
+          elif [[ "${{ matrix.os }}" =~ "ubuntu" ]] ; then
+            apt update
+            apt -y install python3 python3-pip python3-wheel
+          fi
+      - name: Run hello milvus
+        run: |
+          python3 -m pip install --user -U pip 
+          python3 -m pip install --user "$(echo dist/*.whl)[client]"
+          cd examples
+          python3 example.py
+      - name: Upload runtime log
+        uses: actions/upload-artifact@v3
+        if: ${{ failure() }}
+        with:
+          name: milvus-log
+          path: test_milvus/logs/*.log
+  acceptance_test_macos:
+    needs:
+      - build_macos
+    name: Acceptance Test ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      matrix:
+        os: ["macos-11", "macos-12"]
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.8'
+      - uses: actions/download-artifact@v3
+        with:
+          name: wheel_macos
+          path: dist
+      - name: Run hello milvus
+        run: |
+          python3 -m pip install --user wheel setuptools
+          python3 -m pip install --user "$(echo dist/*.whl)[client]"
+          cd examples
+          python3 example.py
+      - name: Upload runtime log
+        uses: actions/upload-artifact@v3
+        if: ${{ failure() }}
+        with:
+          name: milvus-log
+          path: test_milvus/logs/*.log
+  # acceptance_test_windows:
+  #   needs:
+  #     - build_windows
+  #   name: Acceptance Test Windows (py${{ matrix.python_version }})
+  #   runs-on: windows-latest
+  #   timeout-minutes: 60
+  #   strategy:
+  #     fail-fast: false
+  #     matrix:
+  #       python_version: ["3.8", "3.10"]
+  #   steps:
+  #     - uses: actions/checkout@v3
+  #     - uses: actions/download-artifact@v3
+  #       with:
+  #         name: wheel_windows
+  #         path: dist
+  #     - uses: actions/setup-python@v4
+  #       with:
+  #         python-version: ${{ matrix.python_version }}
+  #     - name: Run hello milvus
+  #       run: |
+  #         $x = Get-ChildItem dist\*.whl
+  #         pip install "$x[client]"
+  #         cd examples
+  #         python example.py