Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[bazel] MVP for building with bazel #6994

Merged
merged 38 commits into from
Oct 19, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
38 commits
Select commit Hold shift + click to select a range
3636e48
[bazel] MVP for building wpiutil with bazel
pjreiniger Aug 23, 2024
ccfe170
Add wpinet
pjreiniger Aug 23, 2024
7e3cf21
Add ntcore
pjreiniger Aug 23, 2024
8ddbdb9
Add hal
pjreiniger Aug 23, 2024
80e6627
Add hal
pjreiniger Aug 23, 2024
2a63894
wpiunits
pjreiniger Aug 23, 2024
3799ff2
Opencv
pjreiniger Aug 23, 2024
99c7b85
wpimath
pjreiniger Aug 23, 2024
d155569
cameraserver
pjreiniger Aug 23, 2024
ea8c009
wpilibj
pjreiniger Aug 23, 2024
da393c1
lint
pjreiniger Aug 23, 2024
64e180b
Suppress bazel changes in upstream utils
pjreiniger Aug 23, 2024
e167510
Fixup flags
pjreiniger Aug 24, 2024
856eb29
Share compiler flags across similar arch
pjreiniger Aug 24, 2024
e488f76
Amalgamate build files
pjreiniger Aug 25, 2024
5ba87d5
Fixup amalgamation
pjreiniger Aug 25, 2024
3205b9c
Code review
pjreiniger Aug 25, 2024
c62253f
Fixup
pjreiniger Aug 25, 2024
27a07a9
Code review
pjreiniger Aug 27, 2024
fbef3bd
Add readme, run debug builds
pjreiniger Aug 29, 2024
341e7fa
format
pjreiniger Aug 30, 2024
951ce90
Remove old language from readme
pjreiniger Aug 31, 2024
20afb6e
Apply suggestions from code review
pjreiniger Aug 31, 2024
30324d8
Fixup review suggestions
pjreiniger Aug 31, 2024
90967d4
Buildifier
pjreiniger Aug 31, 2024
efbef29
Fix build after units update; cherry-pick from maven experimental branch
pjreiniger Sep 8, 2024
0faf27e
Merge and fix new third party library
pjreiniger Sep 12, 2024
1fafe61
Blindly try artifactory endpoint
pjreiniger Sep 12, 2024
e21e104
Fixup
pjreiniger Sep 12, 2024
b9e7b82
Forgot to add bazel auth include
pjreiniger Sep 12, 2024
424f06e
Try without auth to see what happens
pjreiniger Sep 12, 2024
ce489b8
Was using the wrong parameter...
pjreiniger Sep 12, 2024
c5db8e6
Change back to buildbuddy
pjreiniger Sep 19, 2024
3499bda
Whoops
pjreiniger Sep 19, 2024
58aa6e0
(finally) transition to wpilibsuite owned libraries
pjreiniger Oct 5, 2024
ec5692b
Update ni libraries
pjreiniger Oct 12, 2024
30c3248
Simplify ntcore
pjreiniger Oct 12, 2024
e066cda
Update repo url
pjreiniger Oct 13, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 20 additions & 0 deletions .bazelignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
build_cmake
build-cmake

pjreiniger marked this conversation as resolved.
Show resolved Hide resolved
# Auto generated by vscode
apriltag/bin
cameraserver/bin
cameraserver/multiCameraServer/bin
cscore/bin
fieldImages/bin
hal/bin
developerRobot/bin
ntcore/bin
romiVendordep/bin
wpilibNewCommands/bin
wpilibj/bin
wpimath/bin
wpinet/bin
wpiutil/bin
wpiunits/bin
xrpVendordep/bin
51 changes: 51 additions & 0 deletions .bazelrc
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
try-import %workspace%/bazel_auth.rc
try-import %workspace%/user.bazelrc
auscompgeek marked this conversation as resolved.
Show resolved Hide resolved

common --noenable_bzlmod

build --java_language_version=17
build --java_runtime_version=roboriojdk_17
build --tool_java_language_version=17
build --tool_java_runtime_version=remotejdk_17

test --test_output=errors
test --test_verbose_timeout_warnings

import shared/bazel/compiler_flags/sanitizers.rc
import shared/bazel/compiler_flags/base_linux_flags.rc
import shared/bazel/compiler_flags/linux_flags.rc
import shared/bazel/compiler_flags/osx_flags.rc
import shared/bazel/compiler_flags/roborio_flags.rc
import shared/bazel/compiler_flags/windows_flags.rc
import shared/bazel/compiler_flags/coverage_flags.rc

build:build_java --test_tag_filters=allwpilib-build-java --build_tag_filters=allwpilib-build-java
build:build_cpp --test_tag_filters=+allwpilib-build-cpp --build_tag_filters=+allwpilib-build-cpp
build:no_example --test_tag_filters=-wpi-example --build_tag_filters=-wpi-example
test:no_example --test_tag_filters=-wpi-example --build_tag_filters=-wpi-example

# Build Buddy Cache Setup
build:build_buddy --bes_results_url=https://app.buildbuddy.io/invocation/
build:build_buddy --bes_backend=grpcs://remote.buildbuddy.io
build:build_buddy --remote_cache=grpcs://remote.buildbuddy.io
build:build_buddy --remote_timeout=3600

# Additional suggestions from buildbuddy for speed
build:build_buddy --experimental_remote_cache_compression
build:build_buddy --experimental_remote_cache_compression_threshold=100
build:build_buddy --noslim_profile
build:build_buddy --experimental_profile_include_target_label
build:build_buddy --experimental_profile_include_primary_output
build:build_buddy --nolegacy_important_outputs

build:build_buddy_readonly --noremote_upload_local_results

# This config should be used locally. It downloads more than the CI version
build:remote_user --config=build_buddy
build:remote_user --config=build_buddy_readonly
build:remote_user --remote_download_toplevel

build:ci --config=build_buddy
build:ci --remote_download_minimal

build --build_metadata=REPO_URL=https://github.com/wpilibsuite/allwpilib.git
1 change: 1 addition & 0 deletions .bazelversion
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
7.3.1
27 changes: 27 additions & 0 deletions .github/actions/setup-build-buddy/action.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
name: 'Setup BuildBuddy acache'
description: 'Sets up the build buddy cache to be readonly / writing based on the presence of environment variables'

inputs:
token:
description: 'Build Buddy API token'

runs:
using: "composite"
steps:
- name: Setup without key
env:
API_KEY: ${{ inputs.token }}
if: ${{ env.API_KEY == '' }}
shell: bash
run: |
echo "No API key secret detected, will setup readonly cache"
echo "build:ci --config=build_buddy_readonly" > bazel_auth.rc

- name: Set with key
env:
API_KEY: ${{ inputs.token }}
if: ${{ env.API_KEY != '' }}
shell: bash
run: |
echo "API Key detected!"
echo "build:build_buddy --remote_header=x-buildbuddy-api-key=${{ env.API_KEY }}" > bazel_auth.rc
149 changes: 149 additions & 0 deletions .github/workflows/bazel.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,149 @@
name: Bazel

on: [pull_request, push]

concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
cancel-in-progress: true

jobs:
build-windows:
strategy:
fail-fast: false
matrix:
include:
- { name: "Windows (native)", os: windows-2022, action: "test", config: "--config=windows", }
- { name: "Windows (arm)", os: windows-2022, action: "build", config: "--config=windows_arm", }
- { name: "Windows (roborio)", os: windows-2022, action: "build", config: "--config=roborio", }

name: "Build ${{ matrix.name }}"
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v4
with: { fetch-depth: 0 }
- uses: actions/setup-java@v4
with:
distribution: 'zulu'
java-version: 17
architecture: x64

- id: Setup_build_buddy
uses: ./.github/actions/setup-build-buddy
with:
token: ${{ secrets.BUILDBUDDY_API_KEY }}

- name: Build Release
run: bazel --output_user_root=C:\\bazelroot ${{ matrix.action }} -k ... --config=ci -c opt ${{ matrix.config }} --verbose_failures
shell: bash

- name: Build Debug
run: bazel --output_user_root=C:\\bazelroot ${{ matrix.action }} -k ... --config=ci -c dbg ${{ matrix.config }} --verbose_failures
shell: bash

build-mac:
strategy:
fail-fast: false
matrix:
include:
- { name: "Mac (native)", os: macos-14, action: "test", config: "--config=macos", }
- { name: "Mac (roborio)", os: macos-14, action: "build", config: "--config=roborio", }

name: "${{ matrix.name }}"
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v4
with: { fetch-depth: 0 }

- id: Setup_build_buddy
uses: ./.github/actions/setup-build-buddy
with:
token: ${{ secrets.BUILDBUDDY_API_KEY }}

- name: Build Release
run: bazel ${{ matrix.action }} -k ... --config=ci -c opt ${{ matrix.config }} --nojava_header_compilation --verbose_failures
shell: bash

- name: Build Debug
run: bazel ${{ matrix.action }} -k ... --config=ci -c dbg ${{ matrix.config }} --nojava_header_compilation --verbose_failures
shell: bash

build-linux:
strategy:
fail-fast: false
matrix:
include:
- { name: "Linux (native)", os: ubuntu-22.04, action: "test", config: "--config=linux", }
- { name: "Linux (roborio)", os: ubuntu-22.04, action: "build", config: "--config=roborio", }
name: "${{ matrix.name }}"
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v4
with: { fetch-depth: 0 }
- uses: bazelbuild/setup-bazelisk@v3

- id: Setup_build_buddy
uses: ./.github/actions/setup-build-buddy
with:
token: ${{ secrets.BUILDBUDDY_API_KEY }}

- name: Build and Test Release
run: bazel ${{ matrix.action }} ... --config=ci -c opt ${{ matrix.config }} -k --verbose_failures

- name: Build and Test Debug
run: bazel ${{ matrix.action }} ... --config=ci -c dbg ${{ matrix.config }} -k --verbose_failures

build-sanitizers:
strategy:
fail-fast: false
matrix:
include:
- config: "asan"
- config: "ubsan"
runs-on: ubuntu-22.04
name: "Sanitizer ${{ matrix.config }}"
steps:
- uses: actions/checkout@v4
with: { fetch-depth: 0 }
- uses: bazelbuild/setup-bazelisk@v3

- id: Setup_build_buddy
uses: ./.github/actions/setup-build-buddy
with:
token: ${{ secrets.BUILDBUDDY_API_KEY }}

- name: Build and Test
run: bazel test -k --config=ci --config=linux --config=${{ matrix.config }} //...

buildifier:
name: "buildifier"
runs-on: ubuntu-22.04
steps:
- name: Set up Go 1.15.x
uses: actions/setup-go@v5
with:
go-version: 1.15.x
id: go
pjreiniger marked this conversation as resolved.
Show resolved Hide resolved

- name: Install Buildifier
run: |
cd $(mktemp -d)
GO111MODULE=on go get github.com/bazelbuild/buildtools/[email protected]

- uses: actions/checkout@v4
with: { fetch-depth: 0 }

- name: Run buildifier
run: buildifier -warnings all --lint=fix -r .

- name: Check Output
run: git --no-pager diff --exit-code HEAD

- name: Generate diff
run: git diff HEAD > bazel-lint-fixes.patch
if: ${{ failure() }}

- uses: actions/upload-artifact@v4
with:
name: ${{ matrix.platform }}-bazel-lint-fixes
path: bazel-lint-fixes.patch
if: ${{ failure() }}
2 changes: 1 addition & 1 deletion .github/workflows/upstream-utils.yml
Original file line number Diff line number Diff line change
Expand Up @@ -133,4 +133,4 @@ jobs:
- name: Add untracked files to index so they count as changes
run: git add -A
- name: Check output
run: git --no-pager diff --exit-code HEAD
run: git --no-pager diff --exit-code HEAD ':!*.bazel'
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -249,6 +249,7 @@ imgui.ini
/bazel-*
user.bazelrc
coverage_report/
bazel_auth.rc

# ctest
/Testing/
Expand Down
25 changes: 25 additions & 0 deletions README-Bazel.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
# WPILib Bazel Support

WPILib is normally built with Gradle, but [Bazel](https://www.bazel.build/) can also be used to increase development speed due to the superior caching ability and the ability to use remote caching and remote execution (on select platforms)


## Prerequisites
- Install [Bazelisk](https://github.com/bazelbuild/bazelisk/releases) and add it to your path. Bazelisk is a wrapper that will download the correct version of bazel specified in the repository. Note: You can alias/rename the binary to `bazel` if you want to keep the familiar `bazel build` vs `bazelisk build` syntax.

## Building
To build the entire repository, simply run `bazel build //...`. To run all of the unit tests, run `bazel test //...`
Other examples:
- `bazel build //wpimath/...` - Builds every target in the wpimath folder
- `bazel test //wpiutil:wpiutil-cpp-test` - Runs only the cpp test target in the wpiutil folder
- `bazel coverage //wpiutil/...` - (*Nix only) - Runs a code coverage report for both C++ and Java on all the targets under wpiutil

## User settings
When invoking bazel, it will check if `user.bazelrc` exists for additional, user specified flags. You can use these settings to do things like always ignore buildin a specific folder, or limiting the CPU/RAM usage during a build.
Examples:
- `build --build_tag_filters=-wpi-example` - Do not build any targets tagged with `wpi-example` (Currently all of the targets in wpilibcExamples and wpilibjExamples contain this tag)
- `build -c opt` - Always build optimized targets. The default compiler flags were chosen to build as fast as possible, and thus don't contain many optimizations
- `build -k` - `-k` is analogous to the MAKE flag `--keep-going`, so the build will not stop on the first error.
- ```
build --local_ram_resources=HOST_RAM*.5 # Don't use more than half my RAM when building
build --local_cpu_resources=HOST_CPUS-1 # Leave one core alone
```
100 changes: 100 additions & 0 deletions WORKSPACE
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")

# Download Extra java rules
http_archive(
name = "rules_jvm_external",
sha256 = "08ea921df02ffe9924123b0686dc04fd0ff875710bfadb7ad42badb931b0fd50",
strip_prefix = "rules_jvm_external-6.1",
url = "https://github.com/bazelbuild/rules_jvm_external/releases/download/6.1/rules_jvm_external-6.1.tar.gz",
)

load("@rules_jvm_external//:repositories.bzl", "rules_jvm_external_deps")

rules_jvm_external_deps()

load("@rules_jvm_external//:defs.bzl", "maven_install")

maven_artifacts = [
"org.ejml:ejml-simple:0.43.1",
"com.fasterxml.jackson.core:jackson-annotations:2.15.2",
"com.fasterxml.jackson.core:jackson-core:2.15.2",
"com.fasterxml.jackson.core:jackson-databind:2.15.2",
"us.hebi.quickbuf:quickbuf-runtime:1.3.3",
"com.google.code.gson:gson:2.10.1",
]

maven_install(
name = "maven",
artifacts = maven_artifacts,
repositories = [
"https://repo1.maven.org/maven2",
"https://frcmaven.wpi.edu/artifactory/release/",
],
)

# Download toolchains
http_archive(
name = "rules_bzlmodrio_toolchains",
sha256 = "2ef1cafce7f4fd4e909bb5de8b0dc771a934646afd55d5f100ff31f6b500df98",
url = "https://github.com/wpilibsuite/rules_bzlmodRio_toolchains/releases/download/2024-1.bcr1/rules_bzlmodRio_toolchains-2024-1.bcr1.tar.gz",
)

load("@rules_bzlmodrio_toolchains//:maven_deps.bzl", "setup_legacy_setup_toolchains_dependencies")

setup_legacy_setup_toolchains_dependencies()

load("@rules_bzlmodrio_toolchains//toolchains:load_toolchains.bzl", "load_toolchains")

load_toolchains()

#
http_archive(
name = "rules_bzlmodrio_jdk",
sha256 = "a00d5fa971fbcad8a17b1968cdc5350688397035e90b0cb94e040d375ecd97b4",
url = "https://github.com/wpilibsuite/rules_bzlmodRio_jdk/releases/download/17.0.8.1-1/rules_bzlmodRio_jdk-17.0.8.1-1.tar.gz",
)

load("@rules_bzlmodrio_jdk//:maven_deps.bzl", "setup_legacy_setup_jdk_dependencies")

setup_legacy_setup_jdk_dependencies()

register_toolchains(
"@local_roborio//:macos",
"@local_roborio//:linux",
"@local_roborio//:windows",
"@local_raspi_32//:macos",
"@local_raspi_32//:linux",
"@local_raspi_32//:windows",
"@local_bullseye_32//:macos",
"@local_bullseye_32//:linux",
"@local_bullseye_32//:windows",
"@local_bullseye_64//:macos",
"@local_bullseye_64//:linux",
"@local_bullseye_64//:windows",
)

setup_legacy_setup_jdk_dependencies()

http_archive(
name = "bzlmodrio-ni",
sha256 = "197fceac88bf44fb8427d5e000b0083118d3346172dd2ad31eccf83a5e61b3ce",
url = "https://github.com/wpilibsuite/bzlmodRio-ni/releases/download/2025.0.0/bzlmodRio-ni-2025.0.0.tar.gz",
)

load("@bzlmodrio-ni//:maven_cpp_deps.bzl", "setup_legacy_bzlmodrio_ni_cpp_dependencies")

setup_legacy_bzlmodrio_ni_cpp_dependencies()

http_archive(
name = "bzlmodrio-opencv",
sha256 = "5314cce05b49451a46bf3e3140fc401342e53d5f3357612ed4473e59bb616cba",
url = "https://github.com/wpilibsuite/bzlmodRio-opencv/releases/download/2024.4.8.0-4.bcr1/bzlmodRio-opencv-2024.4.8.0-4.bcr1.tar.gz",
)

load("@bzlmodrio-opencv//:maven_cpp_deps.bzl", "setup_legacy_bzlmodrio_opencv_cpp_dependencies")

setup_legacy_bzlmodrio_opencv_cpp_dependencies()

load("@bzlmodrio-opencv//:maven_java_deps.bzl", "setup_legacy_bzlmodrio_opencv_java_dependencies")

setup_legacy_bzlmodrio_opencv_java_dependencies()
Loading
Loading