Merge pull request #14542 from adelez/foundry_integration

Add ASAN, UBSAN and MSAN scripts

bazel/grpc_deps.bzl

@@ -120,12 +120,12 @@ def grpc_deps():
     if "com_github_bazelbuild_bazeltoolchains" not in native.existing_rules():
         native.http_archive(
             name = "com_github_bazelbuild_bazeltoolchains",
-            strip_prefix = "bazel-toolchains-44200e0c026d86c53470d107b3697a3e46469c43",
+            strip_prefix = "bazel-toolchains-b850ccdf53fed1ccab7670f52d6b297d74348d1b",
             urls = [
-                "https://mirror.bazel.build/github.com/bazelbuild/bazel-toolchains/archive/44200e0c026d86c53470d107b3697a3e46469c43.tar.gz",
-                "https://github.com/bazelbuild/bazel-toolchains/archive/44200e0c026d86c53470d107b3697a3e46469c43.tar.gz",
+                "https://mirror.bazel.build/github.com/bazelbuild/bazel-toolchains/archive/b850ccdf53fed1ccab7670f52d6b297d74348d1b.tar.gz",
+                "https://github.com/bazelbuild/bazel-toolchains/archive/b850ccdf53fed1ccab7670f52d6b297d74348d1b.tar.gz",
             ],
-            sha256 = "699b55a6916c687f4b7dc092dbbf5f64672cde0dc965f79717735ec4e5416556",
+            sha256 = "d84d6b2fe88ef99963febf91ddce33503eed14c155ace922e2122271b483be64",
         )
 
 # TODO: move some dependencies from "grpc_deps" here?

tools/internal_ci/linux/grpc_asan_on_foundry.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+# Copyright 2017 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+EXTRA_FLAGS="--copt=-gmlt --strip=never --copt=-fsanitize=address --linkopt=-fsanitize=address"
+github/grpc/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh "${EXTRA_FLAGS}"
+

tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh

@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+# Copyright 2017 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+# A temporary solution to give Kokoro credentials. 
+# The file name 4321_grpc-testing-service needs to match auth_credential in 
+# the build config.
+mkdir -p ${KOKORO_KEYSTORE_DIR}
+cp ${KOKORO_GFILE_DIR}/GrpcTesting-d0eeee2db331.json ${KOKORO_KEYSTORE_DIR}/4321_grpc-testing-service
+
+temp_dir=$(mktemp -d)
+ln -f "${KOKORO_GFILE_DIR}/bazel-canary" ${temp_dir}/bazel
+chmod 755 "${KOKORO_GFILE_DIR}/bazel-canary"
+export PATH="${temp_dir}:${PATH}"
+# This should show ${temp_dir}/bazel
+which bazel
+chmod +x "${KOKORO_GFILE_DIR}/bazel_wrapper.py"
+
+# change to grpc repo root
+cd $(dirname $0)/../../..
+
+source tools/internal_ci/helper_scripts/prepare_build_linux_rc
+
+# TODO(adelez): implement size for test targets and change test_timeout back
+"${KOKORO_GFILE_DIR}/bazel_wrapper.py" \
+  --host_jvm_args=-Dbazel.DigestFunction=SHA256 \
+  test --jobs="50" \
+  --test_timeout="3600,3600,3600,3600" \
+  --test_output=errors  \
+  --verbose_failures=true  \
+  --keep_going  \
+  --remote_accept_cached=true  \
+  --spawn_strategy=remote  \
+  --remote_local_fallback=false  \
+  --remote_timeout=3600  \
+  --strategy=Javac=remote  \
+  --strategy=Closure=remote  \
+  --genrule_strategy=remote  \
+  --experimental_strict_action_env=true \
+  --experimental_remote_platform_override='properties:{name:"container-image" value:"docker://gcr.io/cloud-marketplace/google/rbe-debian8@sha256:b2d946c1ddc20af250fe85cf98bd648ac5519131659f7c36e64184b433175a33" }' \
+  --crosstool_top=@com_github_bazelbuild_bazeltoolchains//configs/debian8_clang/0.3.0/bazel_0.10.0:toolchain \
+  --define GRPC_PORT_ISOLATED_RUNTIME=1 \
+  $1 \
+  -- //test/...

tools/internal_ci/linux/grpc_bazel_on_foundry_dbg.sh

@@ -13,45 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-set -ex
+EXTRA_FLAGS="-c dbg"
+github/grpc/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh "${EXTRA_FLAGS}"
 
-# A temporary solution to give Kokoro credentials. 
-# The file name 4321_grpc-testing-service needs to match auth_credential in 
-# the build config.
-mkdir -p ${KOKORO_KEYSTORE_DIR}
-cp ${KOKORO_GFILE_DIR}/GrpcTesting-d0eeee2db331.json ${KOKORO_KEYSTORE_DIR}/4321_grpc-testing-service
-
-mkdir -p /tmpfs/tmp/bazel-canary
-ln -f "${KOKORO_GFILE_DIR}/bazel-canary" /tmpfs/tmp/bazel-canary/bazel
-chmod 755 "${KOKORO_GFILE_DIR}/bazel-canary"
-export PATH="/tmpfs/tmp/bazel-canary:${PATH}"
-# This should show /tmpfs/tmp/bazel-canary/bazel
-which bazel
-chmod +x "${KOKORO_GFILE_DIR}/bazel_wrapper.py"
-
-# change to grpc repo root
-cd $(dirname $0)/../../..
-
-source tools/internal_ci/helper_scripts/prepare_build_linux_rc
-
-# TODO(adelez): implement size for test targets and change test_timeout back
-"${KOKORO_GFILE_DIR}/bazel_wrapper.py" \
-  --host_jvm_args=-Dbazel.DigestFunction=SHA256 \
-  test --jobs="50" \
-  --test_timeout="1200,1200,1200,3600" \
-  --test_output=errors  \
-  --verbose_failures=true  \
-  --keep_going  \
-  --remote_accept_cached=true  \
-  --spawn_strategy=remote  \
-  --remote_local_fallback=false  \
-  --remote_timeout=3600  \
-  --strategy=Javac=remote  \
-  --strategy=Closure=remote  \
-  --genrule_strategy=remote  \
-  --experimental_strict_action_env=true \
-  --experimental_remote_platform_override='properties:{name:"container-image" value:"docker://gcr.io/asci-toolchain/nosla-debian8-clang-fl@sha256:b2d946c1ddc20af250fe85cf98bd648ac5519131659f7c36e64184b433175a33" }' \
-  --crosstool_top=@com_github_bazelbuild_bazeltoolchains//configs/debian8_clang/0.3.0/bazel_0.10.0:toolchain \
-  --define GRPC_PORT_ISOLATED_RUNTIME=1 \
-  -c dbg \
-  -- //test/...

tools/internal_ci/linux/grpc_bazel_on_foundry_opt.sh

@@ -13,45 +13,5 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-set -ex
-
-# A temporary solution to give Kokoro credentials. 
-# The file name 4321_grpc-testing-service needs to match auth_credential in 
-# the build config.
-mkdir -p ${KOKORO_KEYSTORE_DIR}
-cp ${KOKORO_GFILE_DIR}/GrpcTesting-d0eeee2db331.json ${KOKORO_KEYSTORE_DIR}/4321_grpc-testing-service
-
-mkdir -p /tmpfs/tmp/bazel-canary
-ln -f "${KOKORO_GFILE_DIR}/bazel-canary" /tmpfs/tmp/bazel-canary/bazel
-chmod 755 "${KOKORO_GFILE_DIR}/bazel-canary"
-export PATH="/tmpfs/tmp/bazel-canary:${PATH}"
-# This should show /tmpfs/tmp/bazel-canary/bazel
-which bazel
-chmod +x "${KOKORO_GFILE_DIR}/bazel_wrapper.py"
-
-# change to grpc repo root
-cd $(dirname $0)/../../..
-
-source tools/internal_ci/helper_scripts/prepare_build_linux_rc
-
-# TODO(adelez): implement size for test targets and change test_timeout back
-"${KOKORO_GFILE_DIR}/bazel_wrapper.py" \
-  --host_jvm_args=-Dbazel.DigestFunction=SHA256 \
-  test --jobs="50" \
-  --test_timeout="1200,1200,1200,3600" \
-  --test_output=errors  \
-  --verbose_failures=true  \
-  --keep_going  \
-  --remote_accept_cached=true  \
-  --spawn_strategy=remote  \
-  --remote_local_fallback=false  \
-  --remote_timeout=3600  \
-  --strategy=Javac=remote  \
-  --strategy=Closure=remote  \
-  --genrule_strategy=remote  \
-  --experimental_strict_action_env=true \
-  --experimental_remote_platform_override='properties:{name:"container-image" value:"docker://gcr.io/asci-toolchain/nosla-debian8-clang-fl@sha256:b2d946c1ddc20af250fe85cf98bd648ac5519131659f7c36e64184b433175a33" }' \
-  --crosstool_top=@com_github_bazelbuild_bazeltoolchains//configs/debian8_clang/0.3.0/bazel_0.10.0:toolchain \
-  --define GRPC_PORT_ISOLATED_RUNTIME=1 \
-  -c opt \
-  -- //test/...
+EXTRA_FLAGS="-c opt"
+github/grpc/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh "${EXTRA_FLAGS}"

tools/internal_ci/linux/grpc_msan_on_foundry.sh

@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+# Copyright 2017 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+# A temporary solution to give Kokoro credentials. 
+# The file name 4321_grpc-testing-service needs to match auth_credential in 
+# the build config.
+# TODO: Use keystore.
+mkdir -p ${KOKORO_KEYSTORE_DIR}
+cp ${KOKORO_GFILE_DIR}/GrpcTesting-d0eeee2db331.json ${KOKORO_KEYSTORE_DIR}/4321_grpc-testing-service
+
+temp_dir=$(mktemp -d)
+ln -f "${KOKORO_GFILE_DIR}/bazel-canary" ${temp_dir}/bazel
+chmod 755 "${KOKORO_GFILE_DIR}/bazel-canary"
+export PATH="${temp_dir}:${PATH}"
+# This should show ${temp_dir}/bazel
+which bazel
+chmod +x "${KOKORO_GFILE_DIR}/bazel_wrapper.py"
+
+# change to grpc repo root
+cd $(dirname $0)/../../..
+
+source tools/internal_ci/helper_scripts/prepare_build_linux_rc
+
+"${KOKORO_GFILE_DIR}/bazel_wrapper.py" \
+  --host_jvm_args=-Dbazel.DigestFunction=SHA256 \
+  test --jobs="50" \
+  --test_timeout="3600,3600,3600,3600" \
+  --test_output=errors  \
+  --verbose_failures=true  \
+  --keep_going  \
+  --remote_accept_cached=true  \
+  --spawn_strategy=remote  \
+  --remote_local_fallback=false  \
+  --remote_timeout=3600  \
+  --strategy=Javac=remote  \
+  --strategy=Closure=remote  \
+  --genrule_strategy=remote  \
+  --experimental_strict_action_env=true \
+  --experimental_remote_platform_override='properties:{name:"container-image" value:"docker://gcr.io/asci-toolchain/nosla-debian8-clang-msan@sha256:8f381d55c0456fb65821c90ada902c2584977e03a1eaca8fba8ce77e644c775b" }' \
+  --define GRPC_PORT_ISOLATED_RUNTIME=1 \
+  --copt=-gmlt \
+  --strip=never \
+  --cxxopt=--stdlib=libc++ \
+  --copt=-fsanitize=memory \ 
+  --linkopt=-fsanitize=memory \
+  --copt=-fsanitize-memory-track-origins \
+  --action_env=LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH \
+  --host_crosstool_top=@bazel_toolchains//configs/debian8_clang/0.3.0/bazel_0.10.0:toolchain \
+  --crosstool_top=@bazel_toolchains//configs/experimental/debian8_clang/0.3.0/bazel_0.10.0/msan:msan_experimental_toolchain \
+  -- //test/...

tools/internal_ci/linux/grpc_tsan_on_foundry.sh

@@ -13,49 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-set -ex
+EXTRA_FLAGS="--copt=-gmlt --strip=never --copt=-fsanitize=thread --linkopt=-fsanitize=thread"
+github/grpc/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh "${EXTRA_FLAGS}"
 
-# A temporary solution to give Kokoro credentials. 
-# The file name 4321_grpc-testing-service needs to match auth_credential in 
-# the build config.
-# TODO: Use keystore.
-mkdir -p ${KOKORO_KEYSTORE_DIR}
-cp ${KOKORO_GFILE_DIR}/GrpcTesting-d0eeee2db331.json ${KOKORO_KEYSTORE_DIR}/4321_grpc-testing-service
-
-mkdir -p /tmpfs/tmp/bazel-canary
-ln -f "${KOKORO_GFILE_DIR}/bazel-canary" /tmpfs/tmp/bazel-canary/bazel
-chmod 755 "${KOKORO_GFILE_DIR}/bazel-canary"
-export PATH="/tmpfs/tmp/bazel-canary:${PATH}"
-# This should show /tmpfs/tmp/bazel-canary/bazel
-which bazel
-chmod +x "${KOKORO_GFILE_DIR}/bazel_wrapper.py"
-
-# change to grpc repo root
-cd $(dirname $0)/../../..
-
-source tools/internal_ci/helper_scripts/prepare_build_linux_rc
-
-"${KOKORO_GFILE_DIR}/bazel_wrapper.py" \
-  --host_jvm_args=-Dbazel.DigestFunction=SHA256 \
-  test --jobs="50" \
-  --test_timeout="1500,1500,1500,3600" \
-  --test_output=errors  \
-  --verbose_failures=true  \
-  --keep_going  \
-  --remote_accept_cached=true  \
-  --spawn_strategy=remote  \
-  --remote_local_fallback=false  \
-  --remote_timeout=3600  \
-  --strategy=Javac=remote  \
-  --strategy=Closure=remote  \
-  --genrule_strategy=remote  \
-  --experimental_strict_action_env=true \
-  --experimental_remote_platform_override='properties:{name:"container-image" value:"docker://gcr.io/asci-toolchain/nosla-debian8-clang-fl@sha256:b2d946c1ddc20af250fe85cf98bd648ac5519131659f7c36e64184b433175a33" }' \
-  --crosstool_top=@com_github_bazelbuild_bazeltoolchains//configs/debian8_clang/0.3.0/bazel_0.10.0:toolchain \
-  --define GRPC_PORT_ISOLATED_RUNTIME=1 \
-  --copt=-gmlt \
-  --strip=never \
-  --copt=-fsanitize=thread \
-  --linkopt=-fsanitize=thread \
-  --test_verbose_timeout_warnings \
-  -- //test/...

tools/internal_ci/linux/grpc_ubsan_on_foundry.sh

@@ -22,11 +22,11 @@ set -ex
 mkdir -p ${KOKORO_KEYSTORE_DIR}
 cp ${KOKORO_GFILE_DIR}/GrpcTesting-d0eeee2db331.json ${KOKORO_KEYSTORE_DIR}/4321_grpc-testing-service
 
-mkdir -p /tmpfs/tmp/bazel-canary
-ln -f "${KOKORO_GFILE_DIR}/bazel-canary" /tmpfs/tmp/bazel-canary/bazel
+temp_dir=$(mktemp -d)
+ln -f "${KOKORO_GFILE_DIR}/bazel-canary" ${temp_dir}/bazel
 chmod 755 "${KOKORO_GFILE_DIR}/bazel-canary"
-export PATH="/tmpfs/tmp/bazel-canary:${PATH}"
-# This should show /tmpfs/tmp/bazel-canary/bazel
+export PATH="${temp_dir}:${PATH}"
+# This should show ${temp_dir}/bazel
 which bazel
 chmod +x "${KOKORO_GFILE_DIR}/bazel_wrapper.py"
 
@@ -38,7 +38,7 @@ source tools/internal_ci/helper_scripts/prepare_build_linux_rc
 "${KOKORO_GFILE_DIR}/bazel_wrapper.py" \
   --host_jvm_args=-Dbazel.DigestFunction=SHA256 \
   test --jobs="50" \
-  --test_timeout="1500,1500,1500,3600" \
+  --test_timeout="3600,3600,3600,3600" \
   --test_output=errors  \
   --verbose_failures=true  \
   --keep_going  \
@@ -50,12 +50,11 @@ source tools/internal_ci/helper_scripts/prepare_build_linux_rc
   --strategy=Closure=remote  \
   --genrule_strategy=remote  \
   --experimental_strict_action_env=true \
-  --experimental_remote_platform_override='properties:{name:"container-image" value:"docker://gcr.io/asci-toolchain/nosla-debian8-clang-fl@sha256:b2d946c1ddc20af250fe85cf98bd648ac5519131659f7c36e64184b433175a33" }' \
-  --crosstool_top=@com_github_bazelbuild_bazeltoolchains//configs/experimental/debian8_clang/0.3.0/bazel_0.10.0/ubsan:ubsan_experimental_toolchain \
+  --experimental_remote_platform_override='properties:{name:"container-image" value:"docker://gcr.io/cloud-marketplace/google/rbe-debian8@sha256:b2d946c1ddc20af250fe85cf98bd648ac5519131659f7c36e64184b433175a33" }' \
   --define GRPC_PORT_ISOLATED_RUNTIME=1 \
   --copt=-gmlt \
   --strip=never \
   --copt=-fsanitize=undefined \
   --linkopt=-fsanitize=undefined \
-  --test_verbose_timeout_warnings \
+  --crosstool_top=@com_github_bazelbuild_bazeltoolchains//configs/experimental/debian8_clang/0.3.0/bazel_0.10.0/ubsan:ubsan_experimental_toolchain \
   -- //test/...