diff --git a/build_tools/integrate/README.md b/build_tools/integrate/README.md new file mode 100644 index 0000000000..a578713497 --- /dev/null +++ b/build_tools/integrate/README.md @@ -0,0 +1,87 @@ +# Integrate Scripts + +A collection of scripts used to integrate the StableHLO repository into the +rest of the OpenXLA repos. These scripts are run ~2x/wk by a StableHLO oncall +rotation at Google to ensure new changes can propagate to the rest of the +ecosystem in a reasonable amount of time. + +## Integrate Process + +### Bump LLVM Revision + +The XLA repo has constant integrates of LLVM and carries a patch file for +StableHLO in [temporary.patch](https://github.com/openxla/xla/blob/main/third_party/stablehlo/temporary.patch), +which contains any changes needed to build StableHLO at a new LLVM revision. +To sync to XLA's LLVM revision and apply necessary patches, use: + +```sh +$ ./build_tools/integrate/llvm_bump_revision.sh +Using LLVM commit: b3134fa2338388adf8cfb2d77339d0b042eab9f6 +Updating LLVM Commit & SHA256 +Bumping commit to: b3134fa2338388adf8cfb2d77339d0b042eab9f6 +Bumping sha256 to: b6024606092290b0838735c26ad1c5c239b3e931136b420af8680e3a1156e759 +Patch file openxla/xla/third_party/stablehlo/temporary.patch is empty +Skipping patch apply +Commit changes with message: +git add . +git commit -m "Integrate LLVM at llvm/llvm-project@b3134fa23383" +``` + +### Integarte into OpenXLA Repositories + +The StableHLO oncall then integrates the change into the google monorepo, which +propagates the new StableHLO features to XLA, Shardy, JAX, TF, etc, including +any changes or patches that were needed to build these projects with the new +feature. + +_Note: this is the only step that must be carried out by a Google team member._ + +### Tag the integrated StableHLO commit and bump StableHLO version numbers + +This step takes care of a few things: + +1. Add a tag for the integrated StableHLO version +2. Bump the patch version in [Version.h](https://github.com/openxla/stablehlo/tree/main/stablehlo/dialect/Version.h#L41) +3. Bump the 4w and 12w forward compatibility requirement versions in + [Version.cpp](https://github.com/openxla/stablehlo/blob/main/stablehlo/dialect/Version.cpp#L75) + +```sh +Usage: ./build_tools/integrate/stablehlo_tag_and_bump_version.sh [-t ] + -t Specify a commit to tag, must be an integrated StableHLO commit + available on https://github.com/search?q=repo%3Aopenxla%2Fxla+integrate+stablehlo&type=commits + If not specifed, will only bump the 4w and 12w versions. + +$ ./build_tools/integrate/stablehlo_tag_and_bump_version.sh -t 37487a8e +Bumping 4w and 12w compatibility window values +New WEEK_4 Version: 1, 7, 8 +New WEEK_12 Version: 1, 6, 0 + return Version(1, 7, 8); // WEEK_4 ANCHOR: DO NOT MODIFY + return Version(1, 6, 0); // WEEK_12 ANCHOR: DO NOT MODIFY +Bumping version Version(1, 8, 4) -> Version(1, 8, 5) +Using commit: +Integrate LLVM at llvm/llvm-project@246b57cb2086 (#2620) + +Is this the correct commit? [y] y + +Creating tagged release v1.8.4 at 37487a8e +$ git tag -a v1.8.4 37487a8e -m "StableHLO v1.8.4" + +Most recent tags: +v1.8.4 +v1.8.3 +v1.8.2 + +If this is incorrect, can undo using: +$ git tag -d v1.8.4 + +Bumping revision to: Version(1, 8, 5) +$ sed -i "s/Version(1, 8, 4)/Version(1, 8, 5)/" ./stablehlo/dialect/Version.h + +NEXT STEPS + Push tag to upstream using: + $ git push upstream v1.8.4 + + Commit and patch bump changes: + $ git add . + $ git commit -m "Bump patch version after integrate 1.8.4 -> 1.8.5" +``` diff --git a/build_tools/integrate/llvm_bump_revision.sh b/build_tools/integrate/llvm_bump_revision.sh new file mode 100755 index 0000000000..99e64073b8 --- /dev/null +++ b/build_tools/integrate/llvm_bump_revision.sh @@ -0,0 +1,55 @@ +#!/bin/bash +# Copyright 2024 The StableHLO Authors. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +SCRIPT_DIR="$(dirname "$(realpath "$0")")" +GH_ACTIONS="$SCRIPT_DIR/../github_actions" +REPO_ROOT="$SCRIPT_DIR/../.." + +# Update build files +bump_to_xla_llvm_version() { + LLVM_URL="https://raw.githubusercontent.com/openxla/xla/refs/heads/main/third_party/llvm/workspace.bzl" + LLVM_REV=$(curl -s $LLVM_URL | grep 'LLVM_COMMIT =' | cut -d '"' -f 2) + echo "Using LLVM commit: $LLVM_REV" + echo "$LLVM_REV" > ./build_tools/llvm_version.txt + "$GH_ACTIONS/lint_llvm_commit.sh" -f . +} + +apply_xla_patch() { + PATCH_URL="https://raw.githubusercontent.com/openxla/xla/refs/heads/main/third_party/stablehlo/temporary.patch" + PATCH=$(curl -s "$PATCH_URL") + if (( $(echo "$PATCH" | wc -l) < 2 )); then + echo "Patch file openxla/xla/third_party/stablehlo/temporary.patch is empty" + echo "Skipping patch apply" + return 0 + fi + + TMP_DIR=$(mktemp -d) + TMP_PATCH="$TMP_DIR/temporary.patch" + echo "Cloning patch into $TMP_PATCH" + echo "$PATCH" > "$TMP_PATCH" + cd "$REPO_ROOT" || exit 1 + patch -p1 < "$TMP_PATCH" +} + +set -o errexit # Exit immediately if any command returns a non-zero exit status +set -o nounset # Using uninitialized variables raises error and exits +set -o pipefail # Ensures the script detects errors in any part of a pipeline. + +bump_to_xla_llvm_version +apply_xla_patch + +# Print the commit message +echo "Commit changes with message:" +echo "git add ." +echo "git commit -m \"Integrate LLVM at llvm/llvm-project@${LLVM_REV:0:12}\"" diff --git a/build_tools/integrate/stablehlo_tag_and_bump_version.sh b/build_tools/integrate/stablehlo_tag_and_bump_version.sh new file mode 100755 index 0000000000..b03aa107e3 --- /dev/null +++ b/build_tools/integrate/stablehlo_tag_and_bump_version.sh @@ -0,0 +1,216 @@ +#!/bin/bash +# Copyright 2024 The StableHLO Authors. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +exit_with_usage() { + echo "Usage: $0 [-t ]" + echo " -t Specify a commit to tag, must be an integrated StableHLO commit" + echo " available on https://github.com/search?q=repo%3Aopenxla%2Fxla+integrate+stablehlo&type=commits" + echo " If not specifed, will only bump the 4w and 12w versions." + exit 1 +} + +## Setup global variables: +SCRIPT_DIR="$(dirname "$(realpath "$0")")" +VERSION="" +VERSION_H="$SCRIPT_DIR/../../stablehlo/dialect/Version.h" +VERSION_CPP="$SCRIPT_DIR/../../stablehlo/dialect/Version.cpp" +setup_version_vars() { + # getCurrentVersion() { Version(0, X, Y); } + local VERSION_STR + VERSION_STR=$(grep getCurrentVersion -A1 "$VERSION_H" | grep -o 'Version([0-9], .*)') + local REGEX="Version\(([0-9]+), ([0-9]+), ([0-9]+)\)" + if [[ $VERSION_STR =~ $REGEX ]] + then + VERSION=("${BASH_REMATCH[1]}" "${BASH_REMATCH[2]}" "${BASH_REMATCH[3]}") + else + echo "Error: Could not find current version string in Version.h" >&2 + exit 1 + fi +} +setup_version_vars + +## Tag old version +tag_integrated_version() { + local COMMIT="$1" + local RELEASE_TAG="$2" + + # Must be valid commit + if ! git rev-parse "$COMMIT" > /dev/null; then + echo "Could not find commit $COMMIT." + echo "Sync with upstream and pull changes to local repo." + exit 1 + fi + + # Ensure proper commit + # NOTE: THIS COMMIT MUST BE INTEGRATED INTO OPENXLA/XLA BY STABLEHLO ONCALL + # Valid commit hashes here: + # https://github.com/search?q=repo%3Aopenxla%2Fxla+integrate+stablehlo&type=commits + echo "Using commit:" + git log --format=%B -n 1 "$COMMIT" + echo + read -p "Is this the correct commit? [y] " -n 1 -r + echo; echo + if [[ "$REPLY" =~ ^[Nn]$ ]]; then + echo "Exiting..." + exit 1 + fi + + echo "Creating tagged release $RELEASE_TAG at $COMMIT" + echo "$ git tag -a $RELEASE_TAG $COMMIT -m \"StableHLO $RELEASE_TAG\"" + git tag -a "$RELEASE_TAG" "$COMMIT" -m "StableHLO $RELEASE_TAG" + echo + echo "Most recent tags:" + git tag --sort=-version:refname | head -3 + echo + echo "If this is incorrect, can undo using:" + echo "$ git tag -d $RELEASE_TAG" + echo +} + +bump_patch_version() { + ## Bump patch version in Version.h + local VERSION_STR="Version(${VERSION[0]}, ${VERSION[1]}, ${VERSION[2]})" + local NEW_VERSION_STR="Version(${VERSION[0]}, ${VERSION[1]}, $((VERSION[2]+1)))" + echo "Bumping revision to: $NEW_VERSION_STR" + echo "$ sed -i \"s/$VERSION_STR/$NEW_VERSION_STR/\" $VERSION_H" + sed -i "s/$VERSION_STR/$NEW_VERSION_STR/" "$VERSION_H" + echo +} + +tag_and_bump() { + local COMMIT="$1" + local RELEASE="${VERSION[0]}.${VERSION[1]}.${VERSION[2]}" + local RELEASE_TAG="v$RELEASE" + local NEXT_RELEASE="${VERSION[0]}.${VERSION[1]}.$((VERSION[2]+1))" + tag_integrated_version "$COMMIT" "$RELEASE_TAG" + bump_patch_version + + ## Next steps + echo "NEXT STEPS" + echo " Push tag to upstream using:" + echo " $ git push upstream $RELEASE_TAG" + echo + echo " Commit and patch bump changes:" + echo " $ git add ." + echo " $ git commit -m \"Bump patch version after integrate $RELEASE -> $NEXT_RELEASE\"" +} + +## Update the 4w and 12w forward compatiblility versions +# This function: +# Replaces WEEK_4 and WEEK12 versions in stablehlo/dialect/Version.cpp +# return Version(a, b, c); // WEEK_4 ANCHOR: DO NOT MODIFY +# return Version(m, n, p); // WEEK_12 ANCHOR: DO NOT MODIFY +# WEEK_4 version - The most recent git tag that was created at least 28 days ago. +# WEEK_12 version - The most recent git tag that was created at least 84 days ago. + +update_forward_compat_versions() { + echo "Bumping 4w and 12w compatibility window values" + + local UTC_TIME + UTC_TIME=$(date -u +%s) + + local WEEK_4_TAG="" + local WEEK_12_TAG="" + + git fetch --tags upstream + + # Iterate over tags finding the proper tag + while IFS= read -r line; do + # split line CSV + IFS=',' read -r tag_ts tag_v <<< "$line" + ts_diff=$(( (UTC_TIME - tag_ts) / 86400 )) + + if [ -z "$WEEK_4_TAG" ] && [ "$ts_diff" -ge 28 ]; then + WEEK_4_TAG=$tag_v + fi + + if [ -z "$WEEK_12_TAG" ] && [ "$ts_diff" -ge 84 ]; then + WEEK_12_TAG=$tag_v + break + fi + done < <(git for-each-ref --sort=taggerdate --format '%(taggerdate:unix),%(refname:short)' refs/tags | tail -40 | tac) + + if [ -z "$WEEK_4_TAG" ] || [ -z "$WEEK_12_TAG" ]; then + echo "Error: WEEK_4 or WEEK_12 tag not found." >&2 + exit 1 + fi + + WEEK_4_TAG=$(echo "$WEEK_4_TAG" | sed -n 's/.*v\([0-9]\+\)\.\([0-9]\+\)\.\([0-9]\+\).*/\1, \2, \3/p') + WEEK_12_TAG=$(echo "$WEEK_12_TAG" | sed -n 's/.*v\([0-9]\+\)\.\([0-9]\+\)\.\([0-9]\+\).*/\1, \2, \3/p') + + if [ -z "$WEEK_4_TAG" ] || [ -z "$WEEK_12_TAG" ]; then + echo "Error: Unable to parse the WEEK_4 or WEEK_12 tag" >&2 + exit 1 + fi + + echo "New WEEK_4 Version: $WEEK_4_TAG" >&2 + echo "New WEEK_12 Version: $WEEK_12_TAG" >&2 + + sed -i -E \ + -e "s/(return Version\()([0-9]+), ([0-9]+), ([0-9]+)(\);\s+\/\/ WEEK_4 ANCHOR: DO NOT MODIFY)/\1$WEEK_4_TAG\5/" \ + -e "s/(return Version\()([0-9]+), ([0-9]+), ([0-9]+)(\);\s+\/\/ WEEK_12 ANCHOR: DO NOT MODIFY)/\1$WEEK_12_TAG\5/" \ + "$VERSION_CPP" + + + # Checking Version.cpp values + grep "WEEK_4 ANCHOR: DO NOT MODIFY" "$VERSION_CPP" >&2 + grep "WEEK_12 ANCHOR: DO NOT MODIFY" "$VERSION_CPP" >&2 + + if [ "$(grep -c -E "return Version\($WEEK_4_TAG\);\s+\/\/ WEEK_4 ANCHOR: DO NOT MODIFY" "$VERSION_CPP")" -ne 1 ]; then + echo "ERROR: WEEK_4 version is not correct" >&2 + exit 1 + fi + + if [ "$(grep -c -E "return Version\($WEEK_12_TAG\);\s+\/\/ WEEK_12 ANCHOR: DO NOT MODIFY" "$VERSION_CPP")" -ne 1 ]; then + echo "ERROR: WEEK_12 version is not correct" >&2 + exit 1 + fi +} + +# ------ +# MAIN +# ------ + +set -o errexit # Exit immediately if any command returns a non-zero exit status +set -o nounset # Using uninitialized variables raises error and exits +set -o pipefail # Ensures the script detects errors in any part of a pipeline. + +COMMIT_TO_TAG="" +while getopts 't:' flag; do + case "${flag}" in + t) COMMIT_TO_TAG="$OPTARG" ;; + *) exit_with_usage ;; + esac +done +shift $(( OPTIND - 1 )) + +## Validation +if [ $# -ne 0 ]; then + exit_with_usage +fi + +# Must have upstream remote - This is needed to fetch and add tags +if ! git remote | grep upstream > /dev/null; then + echo "Missing upstream remote. Use:" + echo "$ git remote add upstream https://github.com/openxla/stablehlo.git" + exit 1 +fi + +update_forward_compat_versions + +if [ -n "$COMMIT_TO_TAG" ]; then + tag_and_bump "$COMMIT_TO_TAG" +else + echo "No commit to tag specified, only bumping 4w and 12w values." +fi diff --git a/build_tools/update_version_h_cpp.sh b/build_tools/update_version_h_cpp.sh deleted file mode 100755 index 752cc87932..0000000000 --- a/build_tools/update_version_h_cpp.sh +++ /dev/null @@ -1,124 +0,0 @@ -#!/bin/bash -# Copyright 2024 The StableHLO Authors. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This script: -# 1. Replaces current version in stablehlo/dialect/Version.h with the next -# patch version -# static Version getCurrentVersion() { return Version(x, y, z); } -# new version will be (x, y, z + 1) -# 2. Replaces WEEK_4 and WEEK12 versions in stablehlo/dialect/Version.cpp -# return Version(a, b, c); // WEEK_4 ANCHOR: DO NOT MODIFY -# return Version(m, n, p); // WEEK_12 ANCHOR: DO NOT MODIFY -# WEEK_4 version - The most recent git tag that was created at least 28 days ago. -# WEEK_12 version - The most recent git tag that was created at least 84 days ago. - -set -o errexit -set -o nounset -set -o pipefail - -script_dir="$(dirname "$(realpath "$0")")" -version_h="$script_dir/../stablehlo/dialect/Version.h" -version_cpp="$script_dir/../stablehlo/dialect/Version.cpp" - -fetch_current_version() { - # getCurrentVersion() { Version(X, Y, Z); } - ver_str=$(grep -A1 getCurrentVersion "$version_h" | grep -o 'Version(.*[0-9])') - REGEX="Version\(([0-9]+), ([0-9]+), ([0-9]+)\)" - if [[ $ver_str =~ $REGEX ]]; then - curr_ver=("${BASH_REMATCH[1]}" "${BASH_REMATCH[2]}" "${BASH_REMATCH[3]}") - else - echo "Error: Could not find current version string in $version_h" >&2 - exit 1 - fi -} -fetch_current_version - -# calculate next current ver (patch ver + 1) -next_curr_z=$((curr_ver[2] + 1)) - -utc_time=$(date -u +%s) - -week_4_tag="" -week_12_tag="" - -while IFS= read -r line; do - # split line CSV - IFS=',' read -r tag_ts tag_v <<< "$line" - ts_diff=$(( (utc_time - tag_ts) / 86400 )) - - if [ -z "$week_4_tag" ] && [ "$ts_diff" -ge 28 ]; then - week_4_tag=$tag_v - fi - - if [ -z "$week_12_tag" ] && [ "$ts_diff" -ge 84 ]; then - week_12_tag=$tag_v - break - fi -done < <(git for-each-ref --sort=taggerdate --format '%(taggerdate:unix),%(refname:short)' refs/tags | tail -40 | tac) - -if [ -z "$week_4_tag" ] || [ -z "$week_12_tag" ]; then - echo "Error: WEEK_4 or WEEK_12 tag not found." >&2 - exit 1 -fi - -week_4_tag=$(echo "$week_4_tag" | sed -n 's/.*v\([0-9]\+\)\.\([0-9]\+\)\.\([0-9]\+\).*/\1, \2, \3/p') -week_12_tag=$(echo "$week_12_tag" | sed -n 's/.*v\([0-9]\+\)\.\([0-9]\+\)\.\([0-9]\+\).*/\1, \2, \3/p') - -if [ -z "$week_4_tag" ] || [ -z "$week_12_tag" ]; then - echo "Error: Unable to parse the WEEK_4 or WEEK_12 tag" >&2 - exit 1 -fi - -echo "Next Current Version: ${curr_ver[0]}, ${curr_ver[1]}, $next_curr_z" >&2 -echo "WEEK_4 Version: $week_4_tag" >&2 -echo "WEEK_12 Version: $week_12_tag" >&2 - -# Saving Version.h values -echo "Saving $version_h..." >&2 - -sed -i -E \ --e "s/(static Version getCurrentVersion\(\) \{ return Version\()([0-9]+), ([0-9]+), ([0-9]+)(\); \})/\1\2, \3, $next_curr_z\5/" \ -"$version_h" - -# Checking Version.h values -grep "static Version getCurrentVersion()" "$version_h" >&2 - -if [ "$(grep -c "static Version getCurrentVersion() { return Version(${curr_ver[0]}, ${curr_ver[1]}, $next_curr_z); }" "$version_h")" -ne 1 ]; then - echo "ERROR: getCurrentVersion() version is not correct" >&2 - exit 1 -fi - -# Saving Version.cpp values -echo "Saving $version_cpp..." >&2 - -sed -i -E \ --e "s/(return Version\()([0-9]+), ([0-9]+), ([0-9]+)(\);\s+\/\/ WEEK_4 ANCHOR: DO NOT MODIFY)/\1$week_4_tag\5/" \ --e "s/(return Version\()([0-9]+), ([0-9]+), ([0-9]+)(\);\s+\/\/ WEEK_12 ANCHOR: DO NOT MODIFY)/\1$week_12_tag\5/" \ -"$version_cpp" - -# Checking Version.cpp values -grep "WEEK_4 ANCHOR: DO NOT MODIFY" "$version_cpp" >&2 -grep "WEEK_12 ANCHOR: DO NOT MODIFY" "$version_cpp" >&2 - -if [ "$(grep -c -E "return Version\($week_4_tag\);\s+\/\/ WEEK_4 ANCHOR: DO NOT MODIFY" "$version_cpp")" -ne 1 ]; then - echo "ERROR: WEEK_4 version is not correct" >&2 - exit 1 -fi - -if [ "$(grep -c -E "return Version\($week_12_tag\);\s+\/\/ WEEK_12 ANCHOR: DO NOT MODIFY" "$version_cpp")" -ne 1 ]; then - echo "ERROR: WEEK_12 version is not correct" >&2 - exit 1 -fi - -echo "Done" >&2