diff --git a/.github/actions/build/action.yml.j2 b/.github/actions/build/action.yml.j2 new file mode 100644 index 00000000..59a6ba29 --- /dev/null +++ b/.github/actions/build/action.yml.j2 @@ -0,0 +1,90 @@ +name: build + +description: Build specified project + +inputs: + build-type: + required: true + type: string + description: One of ci / release + target-device: + required: true + type: string + host-platform: + required: true + type: string + use-container: + required: true + type: boolean + docker-image: + type: string + required: true + upload-enabled: + required: true + type: boolean + +runs: + using: composite + steps: + +<% for package_id, package_info in packages.items() %> + - name: Download <> (artifacts) + uses: ./.github/actions/download-artifacts + with: + artifact-repo: "<>" + artifact-name: "<>" + target-device: "${{ inputs.target-device }}" + git_sha: "<>" + host-platform: ${{ inputs.host-platform }} + dest-dir: ${{ env.ARTIFACTS_DIR }} + dependencies-workflow: <> +<% endfor %> + +<% if packages %> + + - name: Display structure of downloaded artifacts + shell: bash --noprofile --norc -xeuo pipefail {0} + run: | + pwd + ls -lahR ${{ env.ARTIFACTS_DIR }} +<% endif %> + + - if: ${{ inputs.use-container }} + name: Build (in container) + shell: bash --noprofile --norc -xeuo pipefail {0} + run: | + + docker run \ + -e AWS_REGION \ + -e AWS_SESSION_TOKEN \ + -e AWS_ACCESS_KEY_ID \ + -e AWS_SECRET_ACCESS_KEY \ + -e GITHUB_TOKEN \ + -e ARTIFACTS_DIR="$ARTIFACTS_DIR" \ + -e UPLOAD_ENABLED="$UPLOAD_ENABLED" \ + -e USE_CUDA="$USE_CUDA" \ + -e REPO_DIR="$REPO_DIR" \ + -e LEGATE_CORE_BUILD_MODE="$LEGATE_CORE_BUILD_MODE" \ + -e PYTHON_VERSION="$PYTHON_VERSION" \ + -v "${{ env.REPO_DIR }}:${{ env.REPO_DIR }}" \ + -v "${{ env.ARTIFACTS_DIR }}:${{ env.ARTIFACTS_DIR }}" \ + --rm "${{ inputs.docker-image }}" \ + /bin/bash -c "${{ env.REPO_DIR }}/continuous_integration/scripts/entrypoint ${{ env.REPO_DIR }}/continuous_integration/scripts/build ${{ inputs.build-type}} ${{ inputs.target-device }}" + + - if: ${{ !inputs.use-container }} + name: Build (without container) + shell: bash --noprofile --norc -xeuo pipefail {0} + run: | + "${{ env.REPO_DIR }}/continuous_integration/scripts/entrypoint" "${{ env.REPO_DIR }}/continuous_integration/scripts/build" "${{ inputs.build-type}}" "${{ inputs.target-device }}" + + - name: Display structure of the artifacts folder (post build) + shell: bash --noprofile --norc -xeuo pipefail {0} + run: | + sudo chown -R $(whoami) ${{ env.ARTIFACTS_DIR }} + ls -lahR ${{ env.ARTIFACTS_DIR }} + + - name: Upload build artifacts + uses: actions/upload-artifact@v4 + with: + name: ${{ env.ARTIFACT_NAME }} + path: ${{ env.ARTIFACTS_DIR }} diff --git a/.github/actions/download-artifacts/action.yml b/.github/actions/download-artifacts/action.yml new file mode 100644 index 00000000..c3dffa02 --- /dev/null +++ b/.github/actions/download-artifacts/action.yml @@ -0,0 +1,59 @@ +name: download-artifacts + +description: Download dependencies (artifacts) + +inputs: + artifact-repo: + type: string + require: true + artifact-name: + type: string + require: true + target-device: + type: string + required: true + git_sha: + type: string + required: true + host-platform: + type: string + required: true + dest-dir: + type: string + required: true + dependencies-workflow: + required: true + type: string + +runs: + using: composite + steps: + + - id: cache + name: Cache conda artifacts + uses: actions/cache@v4 + with: + key: "nvidia/{ inputs.artifact-repo }}@${{ inputs.host-platform }}-${{ inputs.git_sha }}-${{ inputs.target-device }}" + path: ${{ inputs.dest-dir }} + + - if: steps.cache.outputs.cache-hit != 'true' + name: Download ${{ inputs.artifact-repo }} artifacts + uses: dawidd6/action-download-artifact@v3 + with: + path: ${{ inputs.dest-dir }} + repo: nvidia/${{ inputs.artifact-repo }} + check_artifacts: true + commit: ${{ inputs.git_sha }} + workflow_conclusion: "" + workflow: ${{ inputs.dependencies-workflow }} + name: ${{ inputs.artifact-name }} + skip_unpack: true + if_no_artifact_found: fail + allow_forks: false + + - if: steps.cache.outputs.cache-hit != 'true' + name: Unpack artifact + shell: bash --noprofile --norc -xeuo pipefail {0} + run: | + cd ${{ inputs.dest-dir }} + unzip *.zip diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml new file mode 100644 index 00000000..adeb48df --- /dev/null +++ b/.github/actions/setup/action.yml @@ -0,0 +1,68 @@ +name: Common setup + +inputs: + client-repo: + required: true + type: string + build-type: + required: true + type: string + target-device: + required: true + type: string + host-platform: + required: true + type: string + build-mode: + required: true + type: string + upload-enabled: + required: true + type: boolean + python-version: + required: false + type: string + +runs: + using: composite + steps: + - name: Set REPO_DIR and Dump environment + shell: bash --noprofile --norc -xeuo pipefail {0} + run: | + echo "REPO_DIR=$(pwd)" >> $GITHUB_ENV + env + + - name: Set environment variables + shell: bash --noprofile --norc -xeuo pipefail {0} + run: | + + WITH_TESTS_STR='' + if [[ ("${{ inputs.upload-enabled }}" == "false") && ("${{ inputs.build-type }}" != "ci") ]]; then + WITH_TESTS_STR='-with_tests' + fi + + TARGET_PLATFORM='linux-64' + if [[ "${{ inputs.host-platform }}" == "linux-aarch64" ]]; then + TARGET_PLATFORM='linux-aarch64' + fi + + BUILD_MODE="${{ inputs.build-mode }}" + BUILD_MODE_STR="" + [ -n "${BUILD_MODE}" ] && BUILD_MODE_STR="-${BUILD_MODE}" + + if [[ ("${BUILD_MODE}" == "") || ("${BUILD_MODE}" == "release") ]]; then + # We upload release versions in the default folder. + PKG_DIR="${TARGET_PLATFORM}" + else + PKG_DIR="${BUILD_MODE}/${TARGET_PLATFORM}" + fi + + echo "ARTIFACT_NAME=${{ inputs.host-platform }}-${{ inputs.build-type }}-${{ inputs.client-repo }}-python${{ inputs.python-version }}-${{ inputs.target-device }}${BUILD_MODE_STR}${WITH_TESTS_STR}-${{ github.sha }}" >> $GITHUB_ENV + echo "ARTIFACTS_DIR=$(realpath "$(pwd)/dist")" >> $GITHUB_ENV + echo "USE_CUDA=${{ (inputs.target-device == 'cpu' && 'OFF') || 'ON' }}" >> $GITHUB_ENV + echo "UPLOAD_ENABLED=${{ (inputs.upload-enabled == 'true' && 'ON') || 'OFF' }}" >> $GITHUB_ENV + echo "LEGATE_CORE_BUILD_MODE=${BUILD_MODE}" >> $GITHUB_ENV + echo "BUILD_DATE=$(date +%Y%m%d)" >> $GITHUB_ENV + echo "TARGET_PLATFORM=${TARGET_PLATFORM}" >> $GITHUB_ENV + echo "PKG_DIR=${PKG_DIR}" >> $GITHUB_ENV + echo "PYTHON_VERSION=${{ inputs.python-version }}" >> $GITHUB_ENV \ No newline at end of file diff --git a/.github/workflows/ci-gh.yml b/.github/workflows/ci-gh.yml new file mode 100644 index 00000000..2c43d03f --- /dev/null +++ b/.github/workflows/ci-gh.yml @@ -0,0 +1,35 @@ +name: Build and test + +concurrency: + group: ${{ startsWith(github.ref_name, 'main') && format('unique-{0}', github.run_id) || format('ci-build-and-test-on-{0}-from-{1}', github.event_name, github.ref_name) }} + cancel-in-progress: true + +on: + push: + branches: + - "pull-request/[0-9]+" + - "main" + +jobs: + build-and-test: + name: Build and test (${{ matrix.host-platform }}, ${{ matrix.target-device }}, ${{ matrix.build-mode }}) + strategy: + fail-fast: false + matrix: + host-platform: + - linux-x64 + target-device: + - gpu + build-mode: + - release + upload-enabled: + - false + uses: + ./.github/workflows/gh-build-and-test.yml + with: + host-platform: ${{ matrix.host-platform }} + target-device: ${{ matrix.target-device }} + build-mode: ${{ matrix.build-mode }} + build-type: ci + upload-enabled: ${{ matrix.upload-enabled }} + secrets: inherit diff --git a/.github/workflows/gh-build-and-test.yml b/.github/workflows/gh-build-and-test.yml new file mode 100644 index 00000000..4376776d --- /dev/null +++ b/.github/workflows/gh-build-and-test.yml @@ -0,0 +1,34 @@ +on: + workflow_call: + inputs: + host-platform: + type: string + required: true + target-device: + type: string + required: true + build-mode: + type: string + required: true + build-type: + type: string + required: true + upload-enabled: + type: boolean + required: true +jobs: + build: + if: ${{ github.repository_owner == 'nvidia' }} + uses: + ./.github/workflows/gh-build.yml + with: + client-repo: ${{ github.event.repository.name }} + target-device: ${{ inputs.target-device }} + runs-on: ${{ (inputs.host-platform == 'linux-x64' && 'linux-amd64-cpu16') || (inputs.host-platform == 'linux-aarch64' && 'linux-arm64-cpu16') || (inputs.host-platform == 'mac' && 'macos-latest') }} + build-type: ${{ inputs.build-type }} + use-container: ${{ inputs.host-platform == 'linux-x64' || inputs.host-platform == 'linux-aarch64'}} + host-platform: ${{ inputs.host-platform }} + dependencies-file: "" + build-mode: ${{ inputs.build-mode }} + upload-enabled: ${{ inputs.upload-enabled }} + secrets: inherit diff --git a/.github/workflows/gh-build.yml b/.github/workflows/gh-build.yml new file mode 100644 index 00000000..922e04ec --- /dev/null +++ b/.github/workflows/gh-build.yml @@ -0,0 +1,95 @@ +name: Build + +on: + workflow_call: + inputs: + client-repo: + required: true + type: string + target-device: + required: true + type: string + runs-on: + required: true + type: string + build-type: + required: true + type: string + description: One of ci / release + use-container: + required: true + type: boolean + host-platform: + required: true + type: string + dependencies-file: + required: true + type: string + description: path to versions.json relative to the target repo dir + build-mode: + required: true + type: string + upload-enabled: + required: true + type: boolean + python-version: + required: false + type: string + +jobs: + build: + name: Build (${{ inputs.host-platform }}, ${{ inputs.target-device }}, ${{ inputs.build-type }}, CMake build-mode=${{ inputs.build-mode }}, Python "${{ inputs.python-version }}", Use container=${{ inputs.use-container }} ) + + permissions: + id-token: write # This is required for configure-aws-credentials + contents: read # This is required for actions/checkout + + runs-on: ${{ inputs.runs-on }} + + steps: + - name: Checkout ${{ inputs.client-repo }} + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Setup + uses: ./.github/actions/setup + with: + client-repo: ${{ inputs.client-repo }} + build-type: ${{ inputs.build-type }} + target-device: "${{ inputs.target-device }}" + host-platform: ${{ inputs.host-platform }} + build-mode: ${{ inputs.build-mode }} + upload-enabled: ${{ inputs.upload-enabled }} + python-version: ${{ inputs.python-version }} + + - name: Render templates + shell: bash --noprofile --norc -xeuo pipefail {0} + run: | + pip -q install jinja2 + + DEPENDENCIES_FILE="" + + if [ -z "${{ inputs.dependencies-file }}" ]; then + DEPENDENCIES_FILE="${REPO_DIR}/continuous_integration/no_dependencies.json" + else + DEPENDENCIES_FILE="${REPO_DIR}/${{ inputs.dependencies-file }}" + fi + + ${REPO_DIR}/continuous_integration/scripts/render-template.py .github/actions/build/action.yml.j2 "${DEPENDENCIES_FILE}" .github/actions/build/action.yml + + - name: Dump templates + shell: bash --noprofile --norc -xeuo pipefail {0} + run: | + echo ${REPO_DIR}/.github/actions/build/action.yml + cat ${REPO_DIR}/.github/actions/build/action.yml + + - name: Call build action + uses: ./.github/actions/build + with: + build-type: ${{ inputs.build-type }} + target-device: "${{ inputs.target-device }}" + host-platform: ${{ inputs.host-platform }} + use-container: ${{ inputs.use-container }} + docker-image: "condaforge/miniforge3:latest" + upload-enabled: ${{ inputs.upload-enabled }} diff --git a/.gitignore b/.gitignore index c3441050..cb11a1bb 100644 --- a/.gitignore +++ b/.gitignore @@ -168,3 +168,6 @@ dmypy.json # Cython debug symbols cython_debug/ + +# Dont ignore +!.github/actions/build/ \ No newline at end of file diff --git a/continuous_integration/environment.yml b/continuous_integration/environment.yml new file mode 100644 index 00000000..6d922d43 --- /dev/null +++ b/continuous_integration/environment.yml @@ -0,0 +1,24 @@ +name: cuda_python +channels: + - defaults +dependencies: + - python>=3.10 + - cython>=3.0.0 + - pytest>=6.2.4 + - numpy>=1.21.1 + - setuptools + - wheel + - pip + - cuda-version=12.6 + - cuda-cudart-static + - cuda-driver-dev + - cuda-cudart-dev + - cuda-profiler-api + - cuda-nvrtc-dev + - cuda-nvcc + - pip: + - pytest-benchmark>=3.4.1 + - pyclibrary>=0.1.7 + - versioneer==0.29 + - tomli; python_version < "3.11" + - pywin32; sys_platform == 'win32' diff --git a/continuous_integration/no_dependencies.json b/continuous_integration/no_dependencies.json new file mode 100644 index 00000000..e2d7bd79 --- /dev/null +++ b/continuous_integration/no_dependencies.json @@ -0,0 +1 @@ +{ "packages" : {} } diff --git a/continuous_integration/scripts/build b/continuous_integration/scripts/build new file mode 100755 index 00000000..5db25e67 --- /dev/null +++ b/continuous_integration/scripts/build @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +build_ci() { + set -xeou pipefail + + cd "${REPO_DIR}" + + export CUDA_HOME="${CONDA_PREFIX}/targets/x86_64-linux" + export PARALLEL_LEVEL=$(nproc --ignore 1) + + python setup.py bdist_wheel +} + +build_project() { + set -xeou pipefail + + export PYTHONUNBUFFERED=1 + + . setup-utils; + init_build_env "$@"; + + git config --global --add safe.directory "$REPO_DIR/.git" + + case "${BUILD_TYPE}" in + ci) build_ci;; + *) return 1;; + esac +} + +(build_project "$@"); diff --git a/continuous_integration/scripts/conda-utils b/continuous_integration/scripts/conda-utils new file mode 100755 index 00000000..e0dd32ca --- /dev/null +++ b/continuous_integration/scripts/conda-utils @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +activate_conda_env() { + set +xu + eval "$(conda shell.bash hook)" + conda activate "${CONDA_ENV}"; + set -xu + : ${PYTHON_VERSION:=$(python -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')")} + export PYTHON_VERSION +} + +conda_info() { + set +x + conda info + set -x +} diff --git a/continuous_integration/scripts/entrypoint b/continuous_integration/scripts/entrypoint new file mode 100755 index 00000000..fe4f5cea --- /dev/null +++ b/continuous_integration/scripts/entrypoint @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +set_initial_env() { + set -xeuo pipefail + + export PATH="${PATH}:${REPO_DIR}/continuous_integration/scripts" +} + +entrypoint() { + set -xeuo pipefail + set_initial_env; + + git config --global --add safe.directory "$REPO_DIR/.git" + + cd "${REPO_DIR}" + + exec "$@"; +} + +entrypoint "$@"; diff --git a/continuous_integration/scripts/make-conda-env b/continuous_integration/scripts/make-conda-env new file mode 100755 index 00000000..1294f038 --- /dev/null +++ b/continuous_integration/scripts/make-conda-env @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +set -x + +make_ci_env() { + mamba env create -n "${CONDA_ENV}" -f "${REPO_DIR}/continuous_integration/environment.yml" +} + +make_test_env() { + . conda-utils + + mamba env create -n "${CONDA_ENV}" -f "${REPO_DIR}/continuous_integration/environment.yml" + + activate_conda_env + + pip install "${ARTIFACTS_DIR}"/*.whl + +} + +make_conda_env() { + set -xeuo pipefail + + . setup-utils; + set_base_defs; + + case "$1" in + ci) make_ci_env;; + test) make_test_env;; + *) return 1;; + esac + + return 0; +} + +(make_conda_env "$@"); diff --git a/continuous_integration/scripts/render-template.py b/continuous_integration/scripts/render-template.py new file mode 100755 index 00000000..b887e361 --- /dev/null +++ b/continuous_integration/scripts/render-template.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 + +import argparse +import json +from jinja2 import Environment, FileSystemLoader +import os +import re + +# TODO: make this work for arbitrary context. ie. implement replace_using_context() +def replace_placeholder(source_str, variable_name, variable_value): + # Escaping any regex special characters in variable_name + variable_name_escaped = re.escape(variable_name) + + # Using regular expression to replace ${variable_name} with actual variable_value + # \s* means any amount of whitespace (including none) + # pattern = rf'\$\{{\s*\{{\s*{variable_name_escaped}\s*\}}\s*\}}' + pattern = rf'<<\s*{variable_name_escaped}\s*>>' + return re.sub(pattern, variable_value.strip(), source_str) + +# Setup command-line argument parsing +parser = argparse.ArgumentParser(description='Render a Jinja2 template using a JSON context.') +parser.add_argument('template_file', type=str, help='Path to the Jinja2 template file (with .j2 extension).') +parser.add_argument('json_file', type=str, help='Path to the JSON file to use as the rendering context.') +parser.add_argument('output_file', type=str, help='Path to the output file.') + +args = parser.parse_args() + +# Load JSON file as the rendering context +with open(args.json_file, 'r') as file: + context = json.load(file) + +# Setup Jinja2 environment and load the template +env = Environment( + loader=FileSystemLoader(searchpath='./'), + variable_start_string='<<', + variable_end_string='>>', + block_start_string='<%', + block_end_string='%>', + comment_start_string='<#', + comment_end_string='#>') +env.filters['replace_placeholder'] = replace_placeholder + +template = env.get_template(args.template_file) + +# Render the template with the context +rendered_content = template.render(context) +# print(rendered_content) + +with open(args.output_file, 'w') as file: + file.write(rendered_content) + +print(f'Template rendered successfully. Output saved to {args.output_file}') diff --git a/continuous_integration/scripts/setup-utils b/continuous_integration/scripts/setup-utils new file mode 100755 index 00000000..62579e63 --- /dev/null +++ b/continuous_integration/scripts/setup-utils @@ -0,0 +1,156 @@ +#!/usr/bin/env bash + +install_from_apt() { + set -xeuo pipefail + + export DEBIAN_FRONTEND=non-interactive + + # Run package updates and install packages + apt-get -q update + apt-get -q install -y wget curl jq sudo ninja-build vim numactl rsync +} + +install_cmake() { + set -xeuo pipefail + + wget -q https://github.com/Kitware/CMake/releases/download/v3.26.5/cmake-3.26.5-linux-x86_64.tar.gz + + tar -xzf cmake-3.26.5-linux-x86_64.tar.gz +} + +setup_linux_build_env() { + set -xeuo pipefail + export OS_SHORT_NAME=linux + export PATH="${PATH}:${PREBUILD_DIR}/cmake-3.26.5-linux-x86_64/bin" + + mkdir -p /tmp/out /tmp/env_yaml +} + +install_linux_tools() { + set -xeuo pipefail + + export SED=sed + export READLINK=readlink + + install_from_apt; + install_cmake; + + mkdir -p /tmp/out /tmp/env_yaml +} + +install_linux_test_tools() { + set -xeuo pipefail + + export SED=sed + export READLINK=readlink + + # Run package updates and install packages + apt-get -q update + apt-get -q install -y numactl +} + +set_base_defs() { + set -xeuo pipefail + + export CONDA_ENV=cuda_python + + CONDA_PLATFORM=$(conda info | grep 'platform' | awk -F ' : ' '{print $2}') + export CONDA_PLATFORM + + export PREBUILD_DIR=/tmp/prebuild + mkdir -p "$PREBUILD_DIR" + + export BUILD_DIR="$REPO_DIR/build" + + # Get the machine architecture + ARCH=$(uname -m) + + if [ "$ARCH" == "aarch64" ]; then + # Use the gcc march value used by aarch64 Ubuntu. + BUILD_MARCH=armv8-a + else + # Use uname -m otherwise + BUILD_MARCH=$(uname -m | tr '_' '-') + fi + + export BUILD_MARCH + + export CUDA_VERSION=12.2.2 + + export MAX_LIBSANITIZER_VERSION=11.4 + + export USE_OPENMP=ON +} + +# ----------------------------------------------------------------------------- + +prep_git() { + # Temporarily disable exit on error + set +e + git config --global user.email > /dev/null + local email_exit_status=$? + git config --global user.name > /dev/null + local name_exit_status=$? + # Re-enable exit on error + set -e + + if [ $email_exit_status -ne 0 ]; then + git config --global --add user.email "users.noreply.github.com" + echo "git user.email was not set. It's now set to users.noreply.github.com" + else + echo "Note: git user.email is already set." + fi + + if [ $name_exit_status -ne 0 ]; then + git config --global --add user.name "anon" + echo "git user.name was not set. It's now set to anon" + else + echo "Note: git user.name is already set." + fi + + # Fix "fatal: detected dubious ownership in repository at '/tmp/legate.core'" + # during local builds. + git config --global --add safe.directory "$REPO_DIR" +} + + +setup_build_env() { + set -xeuo pipefail + + install_linux_tools; + + setup_linux_build_env; + + rm -rf "$PREBUILD_DIR" + mkdir -p "$PREBUILD_DIR" + cd $PREBUILD_DIR + + prep_git; +} + +init_build_env() { + set -x; + + . conda-utils; + + export BUILD_TYPE=$1 + + set -xeuo pipefail; + + set_base_defs; + + cd "$PREBUILD_DIR" + + setup_build_env; + + cd "$REPO_DIR"; + + if [[ -d "${BUILD_DIR}" ]]; then + rm -rf "${BUILD_DIR}" + fi + + make-conda-env "$BUILD_TYPE"; + + activate_conda_env; + conda_info; +} \ No newline at end of file