From 9d79747da1af6f6c0343aa3ec5fbf11ec310b708 Mon Sep 17 00:00:00 2001 From: Govind Kamat Date: Fri, 13 Sep 2024 09:22:49 -0700 Subject: [PATCH] More updates to the OSB build (#635) Signed-off-by: Govind Kamat --- .ci/build.sh | 26 ++++++-------- .github/workflows/integ-test.yml | 4 +-- ...elease-drafter.yml => publish-release.yml} | 19 +++++------ .github/workflows/unit-test.yml | 4 +-- DEVELOPER_GUIDE.md | 21 +++++++++--- Makefile | 34 +++++++++++-------- docker/Dockerfile | 14 ++++---- 7 files changed, 67 insertions(+), 55 deletions(-) rename .github/workflows/{release-drafter.yml => publish-release.yml} (79%) diff --git a/.ci/build.sh b/.ci/build.sh index e3c4d5e22..aa5eb54e0 100644 --- a/.ci/build.sh +++ b/.ci/build.sh @@ -1,35 +1,31 @@ #!/usr/bin/env bash -pyenv_init() { - PATH=$HOME/.pyenv/shims:$PATH:$HOME/.pyenv/bin -} - function setup { - export THESPLOG_FILE="${THESPLOG_FILE:-${BENCHMARK_HOME}/.benchmark/logs/actor-system-internal.log}" - # this value is in bytes, the default is 50kB. We increase it to 200kiB. - export THESPLOG_FILE_MAXSIZE=${THESPLOG_FILE_MAXSIZE:-204800} - # adjust the default log level from WARNING - export THESPLOG_THRESHOLD="INFO" + export BENCHMARK_HOME=$GITHUB_WORKSPACE + + export THESPLOG_FILE=$BENCHMARK_HOME/.benchmark/logs/actor-system-internal.log + export THESPLOG_FILE_MAXSIZE=204800 # default is 50 KiB + export THESPLOG_THRESHOLD=INFO # default log level is WARNING - pyenv_init export TERM=dumb export LC_ALL=en_US.UTF-8 + + # Init pyenv. + PATH=$HOME/.pyenv/shims:$PATH:$HOME/.pyenv/bin } -function build { +function build_and_unit_test { setup set -e - make install-devel + make develop make lint make test } -function build_it { +function run_it { setup - export BENCHMARK_HOME="$GITHUB_WORKSPACE" - docker pull ubuntu/squid:latest # make it38, it39, etc. so they run as concurrent GHA jobs diff --git a/.github/workflows/integ-test.yml b/.github/workflows/integ-test.yml index a8c3cf444..9195f0eaa 100644 --- a/.github/workflows/integ-test.yml +++ b/.github/workflows/integ-test.yml @@ -42,6 +42,4 @@ jobs: - run: echo "JAVA17_HOME=$JAVA_HOME" >> $GITHUB_ENV - name: Run the CI build script - run: bash .ci/build.sh build_it ${{ matrix.python-version }} - env: - BENCHMARK_HOME: env.GITHUB_WORKSPACE + run: bash .ci/build.sh run_it ${{ matrix.python-version }} diff --git a/.github/workflows/release-drafter.yml b/.github/workflows/publish-release.yml similarity index 79% rename from .github/workflows/release-drafter.yml rename to .github/workflows/publish-release.yml index 54e12cf82..ab07aadd5 100644 --- a/.github/workflows/release-drafter.yml +++ b/.github/workflows/publish-release.yml @@ -1,4 +1,4 @@ -name: Release drafter +name: Publish Release to GitHub on: push: @@ -6,7 +6,7 @@ on: - "*" jobs: - draft-a-release: + publish-release: runs-on: ubuntu-latest steps: - name: Checkout Repository @@ -22,21 +22,20 @@ jobs: issue-title: 'Release opensearch-benchmark' issue-body: "Please approve or deny the release of opensearch-benchmark. **Tag**: ${{ github.ref_name }} **Commit**: ${{ github.sha }}" exclude-workflow-initiator-as-approver: true + - name: Set up Python 3 uses: actions/setup-python@v3 with: python-version: '3.x' - - name: Install build tools - run: | - python -m pip install --upgrade build + - name: Build project for distribution run: | - python -m build - tar -zvcf artifacts.tar.gz dist - - name: Release + make build + tar zcvf artifacts.tar.gz dist + + - name: Publish release uses: softprops/action-gh-release@v1 with: draft: true generate_release_notes: true - files: | - artifacts.tar.gz \ No newline at end of file + files: artifacts.tar.gz diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index 44393e995..5ae43cbf7 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -17,6 +17,4 @@ jobs: run: git clone https://github.com/pyenv/pyenv.git ~/.pyenv - name: Run the CI build script - run: bash .ci/build.sh build - env: - BENCHMARK_HOME: env.GITHUB_WORKSPACE + run: bash .ci/build.sh build_and_unit_test diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index b895bb6fe..e9088d9a2 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -23,16 +23,29 @@ This document will walk you through on what's needed to start contributing code ### Prerequisites - - **Pyenv**: Install `pyenv` and follow the instructions in the output of `pyenv init` to set up your shell and restart it before proceeding. - For more details please refer to the [PyEnv installation instructions](https://github.com/pyenv/pyenv#installation). + - **pyenv**: Install `pyenv` and follow the instructions in the output of `pyenv init` to set up your shell and restart it before proceeding. + For more details please refer to the [pyenv installation instructions](https://github.com/pyenv/pyenv#installation). - **Optional Step:** For Debian-based systems, install the following modules to continue with the next steps: + `pyenv` requires that the C compiler and development libraries be installed, so that the specified Python versions can be build from source. The installation instructions vary from platform to platform. + + For Debian-based systems, install the following modules to continue with the next steps: ``` sudo apt-get install -y make build-essential libssl-dev zlib1g-dev libbz2-dev \ libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev \ xz-utils tk-dev libffi-dev liblzma-dev git ``` + For Amazon Linux 2023, run the following command: + ``` + sudo yum -y install gcc openssl-devel bzip2-devel libffi-devel ncurses-devel sqlite-devel readline-devel zlib-devel xz-devel + ``` + + On the Mac platform, XCode needs to be installed as well as some additional required libraries: + ``` + xcode-select --install + brew install pyenv jq zlib xz + ``` + - **JDK**: Although OSB is a Python application, it optionally builds and provisions OpenSearch clusters. JDK version 17 is used to build the current version of OpenSearch. Please refer to the [build setup requirements](https://github.com/opensearch-project/OpenSearch/blob/ca564fd04f5059cf9e3ce8aba442575afb3d99f1/DEVELOPER_GUIDE.md#install-prerequisites). Note that the `javadoc` executable should be available in the JDK installation. An earlier version of the JDK can be used, but not all the integration tests will pass. @@ -54,7 +67,7 @@ For those working on WSL2, it is recommended to clone the repository and set up After you git cloned the forked copy of OpenSearch Benchmark, use the following command-line instructions to set up OpenSearch Benchmark for development: ``` cd opensearch-benchmark -make install-devel +make develop ``` Depending on the platform and shell you have, use the following command to activate the virtual environment: diff --git a/Makefile b/Makefile index 55f4a0248..a1d1ae7c6 100644 --- a/Makefile +++ b/Makefile @@ -16,11 +16,14 @@ # under the License. SHELL = /bin/bash +PYTHON = python3 PIP = pip3 VERSIONS = $(shell jq -r '.python_versions | .[]' .ci/variables.json | sed '$$d') VERSION38 = $(shell jq -r '.python_versions | .[]' .ci/variables.json | sed '$$d' | grep 3\.8) PYENV_ERROR = "\033[0;31mIMPORTANT\033[0m: Please install pyenv and run \033[0;31meval \"\$$(pyenv init -)\"\033[0m.\n" +all: develop + pyinst: @which pyenv > /dev/null 2>&1 || { printf $(PYENV_ERROR); exit 1; } @for i in $(VERSIONS); do pyenv install --skip-existing $$i; done @@ -31,6 +34,10 @@ pyinst38: pyenv install --skip-existing $(VERSION38) pyenv local $(VERSION38) +check-pip: + # Install pyenv if the Python environment is externally managed. + @if ! $(PIP) > /dev/null 2>&1 || ! $(PIP) install pip > /dev/null 2>&1; then make pyinst38; fi + check-java: @if ! test "$(JAVA_HOME)" || ! java --version > /dev/null 2>&1 || ! javadoc --help > /dev/null 2>&1; then \ echo "Java installation issues for running integration tests" >&2; \ @@ -40,23 +47,22 @@ check-java: echo "NOTE: Java version 17 required to have all integration tests pass" >&2; \ fi -install-deps: pyinst38 - # @if test `uname` = Darwin -o `python3 --version | sed 's/.* 3.\([0-9]*\).*/3\1/'` -lt 38; then make pyinst38; fi +install-deps: check-pip $(PIP) install --upgrade pip setuptools wheel -install-user: install-deps - PIP_ONLY_BINARY=h5py $(PIP) install -e . +# pylint does not work with Python versions >3.8: +# Value 'Optional' is unsubscriptable (unsubscriptable-object) +develop: pyinst38 install-deps + PIP_ONLY_BINARY=h5py $(PIP) install -e .[develop] -install-devel: install-deps - $(PIP) install -e .[develop] - -wheel: - $(PIP) install --upgrade pip setuptools wheel - PIP_ONLY_BINARY=h5py $(PIP) wheel . +build: install-deps + $(PIP) install --upgrade build + $(PYTHON) -m build -install: wheel - PIP_ONLY_BINARY=h5py $(PIP) install opensearch_benchmark-*.whl - rm -r *.whl *.egg-info +# Builds a wheel from source, then installs it. +install: build + PIP_ONLY_BINARY=h5py $(PIP) install dist/opensearch_benchmark-*.whl + rm -rf dist clean: rm -rf .benchmarks .eggs .tox .benchmark_it .cache build dist *.egg-info logs junit-py*.xml *.whl NOTICE.txt @@ -75,7 +81,7 @@ tox-env-clean: lint: @find osbenchmark benchmarks scripts tests it -name "*.py" -exec pylint -j0 -rn --load-plugins pylint_quotes --rcfile=$(CURDIR)/.pylintrc \{\} + -test: +test: develop pytest tests/ it: pyinst check-java python-caches-clean tox-env-clean diff --git a/docker/Dockerfile b/docker/Dockerfile index 104a10972..0a777952d 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -6,7 +6,6 @@ # Stage 1: build packages and compile where needed # FROM python:3.11.2-slim AS build-stage -WORKDIR /opensearch-benchmark RUN apt-get -y update && \ apt-get install -y curl git gcc pbzip2 pigz make jq && \ @@ -14,31 +13,34 @@ RUN apt-get -y update && \ COPY . opensearch-benchmark -RUN cd opensearch-benchmark; make wheel +# There is no binary package currently available for yappi on ARM. +RUN cd opensearch-benchmark; \ + make build; \ + if test "`uname -m`" = aarch64; then pip wheel yappi && cp yappi-*.whl dist; fi # # Stage 2: create image # FROM python:3.11.2-slim AS image-stage -WORKDIR /opensearch-benchmark ENV BENCHMARK_RUNNING_IN_DOCKER=True RUN groupadd --gid 1000 opensearch-benchmark && \ useradd -d /opensearch-benchmark -m -k /dev/null -g 1000 -N -u 1000 -l -s /bin/bash benchmark +WORKDIR /opensearch-benchmark + RUN mkdir -p /opensearch-benchmark/.benchmark && \ chown -R 1000:0 /opensearch-benchmark/.benchmark -COPY --from=build-stage /opensearch-benchmark/opensearch-benchmark/yappi-*.whl /opensearch-benchmark/opensearch-benchmark/opensearch_benchmark-*.whl ./ +COPY --from=build-stage /opensearch-benchmark/dist/*.whl ./ -# There is no binary package currently available for yappi on ARM. RUN set -ex; \ apt-get -y update; \ apt-get install -y git pbzip2; \ apt-get -y upgrade; \ rm -rf /var/lib/apt/lists/*; \ - PIP_ONLY_BINARY=h5py pip install yappi-*.whl opensearch_benchmark-*.whl; \ + PIP_ONLY_BINARY=h5py pip install *.whl; \ rm *.whl USER 1000