Windows build: Enable RBE build in Windows CI

.azure-pipelines/pipelines.yml

@@ -168,6 +168,8 @@ jobs:
  - bash: ci/run_envoy_docker_windows.sh ci/windows_ci_steps.sh
  displayName: "Run Windows CI"
  env:
+ ENVOY_RBE: "true"
+ BAZEL_BUILD_EXTRA_OPTIONS: "--config=remote-ci --config=remote-msvc-cl --jobs=$(RbeJobs)"
  BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com
  BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance
  GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey)

.bazelrc

@@ -160,6 +160,11 @@ build:rbe-toolchain-gcc --config=rbe-toolchain
 build:rbe-toolchain-gcc --crosstool_top=@rbe_ubuntu_gcc//cc:toolchain
 build:rbe-toolchain-gcc --extra_toolchains=@rbe_ubuntu_gcc//config:cc-toolchain
 
+build:rbe-toolchain-msvc-cl --host_platform=@rbe_windows_msvc_cl//config:platform
+build:rbe-toolchain-msvc-cl --platforms=@rbe_windows_msvc_cl//config:platform
+build:rbe-toolchain-msvc-cl --crosstool_top=@rbe_windows_msvc_cl//cc:toolchain
+build:rbe-toolchain-msvc-cl --extra_toolchains=@rbe_windows_msvc_cl//config:cc-toolchain
+
 build:remote --spawn_strategy=remote,sandboxed,local
 build:remote --strategy=Javac=remote,sandboxed,local
 build:remote --strategy=Closure=remote,sandboxed,local
@@ -168,6 +173,15 @@ build:remote --remote_timeout=7200
 build:remote --auth_enabled=true
 build:remote --remote_download_toplevel
 
+# Windows bazel does not allow sandboxed as a spawn strategy
+build:remote-windows --spawn_strategy=remote,local
+build:remote-windows --strategy=Javac=remote,local
+build:remote-windows --strategy=Closure=remote,local
+build:remote-windows --strategy=Genrule=remote,local
+build:remote-windows --remote_timeout=7200
+build:remote-windows --auth_enabled=true
+build:remote-windows --remote_download_toplevel
+
 build:remote-clang --config=remote
 build:remote-clang --config=rbe-toolchain-clang
 
@@ -181,9 +195,12 @@ build:remote-msan --config=remote
 build:remote-msan --config=rbe-toolchain-clang-libc++
 build:remote-msan --config=rbe-toolchain-msan
 
+build:remote-msvc-cl --config=remote-windows
+build:remote-msvc-cl --config=rbe-toolchain-msvc-cl
+
 # Docker sandbox
 # NOTE: Update this from https://github.com/envoyproxy/envoy-build-tools/blob/master/toolchains/rbe_toolchains_config.bzl#L8
-build:docker-sandbox --experimental_docker_image=envoyproxy/envoy-build-ubuntu:12b3d2c2ffa582507e5d6dd34632b2b990f1b195
+build:docker-sandbox --experimental_docker_image=envoyproxy/envoy-build-ubuntu:f21773ab398a879f976936f72c78c9dd3718ca1e
 build:docker-sandbox --spawn_strategy=docker
 build:docker-sandbox --strategy=Javac=docker
 build:docker-sandbox --strategy=Closure=docker

.circleci/config.yml

@@ -5,7 +5,7 @@ executors:
  description: "A regular build executor based on ubuntu image"
  docker:
  # NOTE: Update this from https://github.com/envoyproxy/envoy-build-tools/blob/master/toolchains/rbe_toolchains_config.bzl#L8
- - image: envoyproxy/envoy-build-ubuntu:12b3d2c2ffa582507e5d6dd34632b2b990f1b195
+ - image: envoyproxy/envoy-build-ubuntu:f21773ab398a879f976936f72c78c9dd3718ca1e
  resource_class: xlarge
  working_directory: /source
 

.devcontainer/Dockerfile

@@ -1,4 +1,4 @@
-FROM gcr.io/envoy-ci/envoy-build:12b3d2c2ffa582507e5d6dd34632b2b990f1b195
+FROM gcr.io/envoy-ci/envoy-build:f21773ab398a879f976936f72c78c9dd3718ca1e
 
 ARG USERNAME=vscode
 ARG USER_UID=501

bazel/repositories.bzl

@@ -125,7 +125,12 @@ def _go_deps(skip_targets):
  # Keep the skip_targets check around until Istio Proxy has stopped using
  # it to exclude the Go rules.
  if "io_bazel_rules_go" not in skip_targets:
- _repository_impl("io_bazel_rules_go")
+ _repository_impl(
+ name = "io_bazel_rules_go",
+ # TODO(wrowe, sunjayBhatia): remove when Windows RBE supports batch file invocation
+ patch_args = ["-p1"],
+ patches = ["@envoy//bazel:rules_go.patch"],
+ )
  _repository_impl("bazel_gazelle")
 
 def envoy_dependencies(skip_targets = []):

bazel/repository_locations.bzl

@@ -53,11 +53,11 @@ DEPENDENCY_REPOSITORIES = dict(
  use_category = ["build"],
  ),
  bazel_toolchains = dict(
- sha256 = "144290c4166bd67e76a54f96cd504ed86416ca3ca82030282760f0823c10be48",
- strip_prefix = "bazel-toolchains-3.1.1",
+ sha256 = "db48eed61552e25d36fe051a65d2a329cc0fb08442627e8f13960c5ab087a44e",
+ strip_prefix = "bazel-toolchains-3.2.0",
  urls = [
- "https://github.com/bazelbuild/bazel-toolchains/releases/download/3.1.1/bazel-toolchains-3.1.1.tar.gz",
- "https://mirror.bazel.build/github.com/bazelbuild/bazel-toolchains/archive/3.1.1.tar.gz",
+ "https://github.com/bazelbuild/bazel-toolchains/releases/download/3.2.0/bazel-toolchains-3.2.0.tar.gz",
+ "https://mirror.bazel.build/github.com/bazelbuild/bazel-toolchains/archive/3.2.0.tar.gz",
  ],
  use_category = ["build"],
  ),
@@ -67,10 +67,10 @@ DEPENDENCY_REPOSITORIES = dict(
  use_category = ["build"],
  ),
  envoy_build_tools = dict(
- sha256 = "b0efe70a1d122fffb89570771f4ec3b912aa0a8a0ce56218223918d7737d01e2",
- strip_prefix = "envoy-build-tools-3cbc1d66b9e9ead42daf69e01597cacf4fb52151",
- # 2020-05-15
- urls = ["https://github.com/envoyproxy/envoy-build-tools/archive/3cbc1d66b9e9ead42daf69e01597cacf4fb52151.tar.gz"],
+ sha256 = "c4d27c0a5db918e861b7164d69cdffe920daafbe3f597ffdda5a1d10c1abc992",
+ strip_prefix = "envoy-build-tools-557ee9b44a3d08cf38d9ce6f08adb872c385d6a5",
+ # 2020-06-16
+ urls = ["https://github.com/envoyproxy/envoy-build-tools/archive/557ee9b44a3d08cf38d9ce6f08adb872c385d6a5.tar.gz"],
  use_category = ["build"],
  ),
  boringssl = dict(

bazel/rules_go.patch

@@ -0,0 +1,30 @@
+#
+# Bazel RBE on Windows GCP workers currently will not invoke cmd.exe batch files correctly
+#
+# Symptom is program not found 'bazel-out', because of the way that the CreateProcess command
+# is constructed by bazel with actions.run with forward slashes, e.g. the command
+# cmd.exe /c "bazel-out/host/bin/external/go_sdk/builder.exe.bat"
+# where cmd.exe on GCP is treating 'bazel-out' as the target, and /host as a command line switch.
+# This problem was not observed on Azure CI pipelines or locally by the developers. The eventual
+# fix is not specific to rules_go; this patch simply addresses immediate breakage and can be removed
+# once the underlying issue within Bazel/RBE is fixed.
+# See:
+# - https://github.com/bazelbuild/rules_go/pull/2542
+# - https://github.com/envoyproxy/envoy/issues/11657
+#
+diff --git a/go/private/rules/binary.bzl b/go/private/rules/binary.bzl
+index b88dfd96..e68b5ece 100644
+--- a/go/private/rules/binary.bzl
++++ b/go/private/rules/binary.bzl
+@@ -128,8 +128,9 @@ def _go_tool_binary_impl(ctx):
+ content = cmd,
+ )
+ ctx.actions.run(
+- executable = bat,
+- inputs = sdk.libs + sdk.headers + sdk.tools + ctx.files.srcs + [sdk.go],
++ executable = "cmd.exe",
++ arguments = ["/S", "/C", bat.path.replace("/", "\\")],
++ inputs = sdk.libs + sdk.headers + sdk.tools + ctx.files.srcs + [sdk.go, bat],
+ outputs = [cout],
+ env = {"GOROOT": sdk.root_file.dirname}, # NOTE(#2005): avoid realpath in sandbox
+ mnemonic = "GoToolchainBinaryCompile",

ci/run_envoy_docker_windows.sh

@@ -2,9 +2,8 @@
 
 set -e
 
-# TODO(sunjayBhatia, wrowe): update this note once we have an RBE toolchain generated for Windows
-# NOTE: Update this from the latest pushed image here: https://hub.docker.com/r/envoyproxy/envoy-build-windows2019/tags
-ENVOY_BUILD_SHA="9b7dc527351b9888805377a05e5975d6ef8d6ae1"
+# The image tag for the Windows image is the same as the Linux one so we use the same mechanism to find it
+. $(dirname $0)/envoy_build_sha.sh
 
 [[ -z "${IMAGE_NAME}" ]] && IMAGE_NAME="envoyproxy/envoy-build-windows2019"
 # The IMAGE_ID defaults to the CI hash but can be set to an arbitrary image ID (found with 'docker

ci/windows_ci_steps.sh

@@ -26,18 +26,12 @@ fi
 
 BAZEL_STARTUP_OPTIONS="--output_base=c:/_eb"
 BAZEL_BUILD_OPTIONS="-c opt --config=msvc-cl --show_task_finish --verbose_failures \
- --test_output=all ${BAZEL_BUILD_EXTRA_OPTIONS} ${BAZEL_EXTRA_TEST_OPTIONS}"
-
-# With all envoy-static and //test/ tree building, no need to test compile externals
-# bazel ${BAZEL_STARTUP_OPTIONS} build ${BAZEL_BUILD_OPTIONS} //bazel/... --build_tag_filters=-skip_on_windows
+ --test_output=errors ${BAZEL_BUILD_EXTRA_OPTIONS} ${BAZEL_EXTRA_TEST_OPTIONS}"
 
 bazel ${BAZEL_STARTUP_OPTIONS} build ${BAZEL_BUILD_OPTIONS} //source/exe:envoy-static --build_tag_filters=-skip_on_windows
 
-# TODO(sunjayBhatia, wrowe): We are disabling building/running tests for now as the AZP pipelines
-# workers do not provide enough resources for us to produce fast enough or reliable enough builds.
-# Test compilation of known MSVC-compatible test sources
-# bazel ${BAZEL_STARTUP_OPTIONS} build ${BAZEL_BUILD_OPTIONS} //test/... --test_tag_filters=-skip_on_windows --build_tests_only
-
 # Test invocations of known-working tests on Windows
-# bazel ${BAZEL_STARTUP_OPTIONS} test ${BAZEL_BUILD_OPTIONS} //test/... --test_tag_filters=-skip_on_windows,-fails_on_windows --build_tests_only --test_summary=terse --test_output=errors
+bazel ${BAZEL_STARTUP_OPTIONS} test ${BAZEL_BUILD_OPTIONS} //test/... --test_tag_filters=-skip_on_windows,-fails_on_windows --build_tests_only
 
+# Build tests that are failing to ensure no regressions
+bazel ${BAZEL_STARTUP_OPTIONS} build ${BAZEL_BUILD_OPTIONS} //test/... --test_tag_filters=-skip_on_windows,fails_on_windows --build_tests_only

test/common/event/BUILD

@@ -11,6 +11,7 @@ envoy_package()
 envoy_cc_test(
  name = "dispatcher_impl_test",
  srcs = ["dispatcher_impl_test.cc"],
+ tags = ["fails_on_windows"],
  deps = [
  "//source/common/api:api_lib",
  "//source/common/event:deferred_task",

test/common/network/utility_test.cc

@@ -171,7 +171,9 @@ TEST_P(NetworkUtilityGetLocalAddress, GetLocalAddress) {
 
 TEST(NetworkUtility, GetOriginalDst) {
  testing::NiceMock<Network::MockConnectionSocket> socket;
+#ifdef SOL_IP
  EXPECT_CALL(socket, ipVersion()).WillOnce(testing::Return(absl::nullopt));
+#endif
  EXPECT_EQ(nullptr, Utility::getOriginalDst(socket));
 }
 

test/extensions/filters/http/admission_control/BUILD

@@ -60,6 +60,7 @@ envoy_extension_cc_test(
  name = "admission_control_integration_test",
  srcs = ["admission_control_integration_test.cc"],
  extension_name = "envoy.filters.http.admission_control",
+ tags = ["fails_on_windows"],
  deps = [
  "//source/extensions/filters/http/admission_control:config",
  "//test/integration:http_integration_lib",

test/extensions/filters/udp/dns_filter/BUILD

@@ -28,6 +28,7 @@ envoy_extension_cc_test(
  name = "dns_filter_test",
  srcs = ["dns_filter_test.cc"],
  extension_name = "envoy.filters.udp_listener.dns_filter",
+ tags = ["fails_on_windows"],
  deps = [
  ":dns_filter_test_lib",
  "//source/extensions/filters/udp/dns_filter:dns_filter_lib",

test/integration/BUILD

@@ -237,6 +237,7 @@ envoy_cc_test(
  srcs = [
  "drain_close_integration_test.cc",
  ],
+ tags = ["fails_on_windows"],
  deps = [
  ":http_protocol_integration_lib",
  "//source/extensions/filters/http/health_check:config",
@@ -293,6 +294,7 @@ envoy_cc_test(
  data = [
  "//test/config/integration/certs",
  ],
+ tags = ["fails_on_windows"],
  deps = [
  ":http_integration_lib",
  "//test/test_common:utility_lib",