Merge pull request #22317 from jtattermusch/bazel_flaky_under_run_tests

bazel tests marked as flaky should not run under run_tests.py

Makefile

@@ -2184,12 +2184,8 @@ test_cxx: buildtests_cxx
 	$(Q) $(BINDIR)/$(CONFIG)/bm_error || ( echo test bm_error failed ; exit 1 )
 	$(E) "[RUN]     Testing bm_fullstack_streaming_ping_pong"
 	$(Q) $(BINDIR)/$(CONFIG)/bm_fullstack_streaming_ping_pong || ( echo test bm_fullstack_streaming_ping_pong failed ; exit 1 )
-	$(E) "[RUN]     Testing bm_fullstack_streaming_pump"
-	$(Q) $(BINDIR)/$(CONFIG)/bm_fullstack_streaming_pump || ( echo test bm_fullstack_streaming_pump failed ; exit 1 )
 	$(E) "[RUN]     Testing bm_fullstack_unary_ping_pong"
 	$(Q) $(BINDIR)/$(CONFIG)/bm_fullstack_unary_ping_pong || ( echo test bm_fullstack_unary_ping_pong failed ; exit 1 )
-	$(E) "[RUN]     Testing bm_metadata"
-	$(Q) $(BINDIR)/$(CONFIG)/bm_metadata || ( echo test bm_metadata failed ; exit 1 )
 	$(E) "[RUN]     Testing bm_pollset"
 	$(Q) $(BINDIR)/$(CONFIG)/bm_pollset || ( echo test bm_pollset failed ; exit 1 )
 	$(E) "[RUN]     Testing bm_timer"
@@ -2260,8 +2256,6 @@ test_cxx: buildtests_cxx
 	$(Q) $(BINDIR)/$(CONFIG)/grpc_tool_test || ( echo test grpc_tool_test failed ; exit 1 )
 	$(E) "[RUN]     Testing grpclb_api_test"
 	$(Q) $(BINDIR)/$(CONFIG)/grpclb_api_test || ( echo test grpclb_api_test failed ; exit 1 )
-	$(E) "[RUN]     Testing grpclb_end2end_test"
-	$(Q) $(BINDIR)/$(CONFIG)/grpclb_end2end_test || ( echo test grpclb_end2end_test failed ; exit 1 )
 	$(E) "[RUN]     Testing h2_ssl_session_reuse_test"
 	$(Q) $(BINDIR)/$(CONFIG)/h2_ssl_session_reuse_test || ( echo test h2_ssl_session_reuse_test failed ; exit 1 )
 	$(E) "[RUN]     Testing head_of_line_blocking_bad_client_test"
@@ -2338,8 +2332,6 @@ test_cxx: buildtests_cxx
 	$(Q) $(BINDIR)/$(CONFIG)/service_config_end2end_test || ( echo test service_config_end2end_test failed ; exit 1 )
 	$(E) "[RUN]     Testing service_config_test"
 	$(Q) $(BINDIR)/$(CONFIG)/service_config_test || ( echo test service_config_test failed ; exit 1 )
-	$(E) "[RUN]     Testing settings_timeout_test"
-	$(Q) $(BINDIR)/$(CONFIG)/settings_timeout_test || ( echo test settings_timeout_test failed ; exit 1 )
 	$(E) "[RUN]     Testing shutdown_test"
 	$(Q) $(BINDIR)/$(CONFIG)/shutdown_test || ( echo test shutdown_test failed ; exit 1 )
 	$(E) "[RUN]     Testing simple_request_bad_client_test"
@@ -2386,8 +2378,6 @@ test_cxx: buildtests_cxx
 	$(Q) $(BINDIR)/$(CONFIG)/writes_per_rpc_test || ( echo test writes_per_rpc_test failed ; exit 1 )
 	$(E) "[RUN]     Testing xds_bootstrap_test"
 	$(Q) $(BINDIR)/$(CONFIG)/xds_bootstrap_test || ( echo test xds_bootstrap_test failed ; exit 1 )
-	$(E) "[RUN]     Testing xds_end2end_test"
-	$(Q) $(BINDIR)/$(CONFIG)/xds_end2end_test || ( echo test xds_end2end_test failed ; exit 1 )
 
 
 flaky_test_cxx: buildtests_cxx

build_autogenerated.yaml

@@ -5128,6 +5128,7 @@ targets:
   - posix
 - name: bm_fullstack_streaming_pump
   build: test
+  run: false
   language: c++
   headers:
   - test/cpp/microbenchmarks/fullstack_streaming_pump.h
@@ -5195,6 +5196,7 @@ targets:
   - posix
 - name: bm_metadata
   build: test
+  run: false
   language: c++
   headers: []
   src:
@@ -6054,6 +6056,7 @@ targets:
 - name: grpclb_end2end_test
   gtest: true
   build: test
+  run: false
   language: c++
   headers:
   - test/cpp/end2end/test_service_impl.h
@@ -7058,6 +7061,7 @@ targets:
 - name: settings_timeout_test
   gtest: true
   build: test
+  run: false
   language: c++
   headers: []
   src:
@@ -7538,6 +7542,7 @@ targets:
 - name: xds_end2end_test
   gtest: true
   build: test
+  run: false
   language: c++
   headers:
   - test/cpp/end2end/test_service_impl.h

tools/buildgen/extract_metadata_from_bazel_xml.py

@@ -43,6 +43,7 @@ def _rule_dict_from_xml_node(rule_xml_node):
         'args': [],
         'generator_function': None,
         'size': None,
+        'flaky': False,
     }
     for child in rule_xml_node:
         # all the metadata we want is stored under "list" tags
@@ -54,6 +55,10 @@ def _rule_dict_from_xml_node(rule_xml_node):
             string_name = child.attrib['name']
             if string_name in ['generator_function', 'size']:
                 result[string_name] = child.attrib['value']
+        if child.tag == 'boolean':
+            bool_name = child.attrib['name']
+            if bool_name in ['flaky']:
+                result[bool_name] = child.attrib['value'] == 'true'
     return result
 
 
@@ -478,6 +483,13 @@ def _generate_build_extra_metadata_for_tests(tests, bazel_rules):
             # don't run the tests marked as "manual"
             test_dict['run'] = False
 
+        if bazel_rule['flaky']:
+            # don't run tests that are marked as "flaky" under bazel
+            # because that would only add noise for the run_tests.py tests
+            # and seeing more failures for tests that we already know are flaky
+            # doesn't really help anything
+            test_dict['run'] = False
+
         if 'no_uses_polling' in bazel_tags:
             test_dict['uses_polling'] = False
 

tools/run_tests/generated/tests.json

@@ -3829,26 +3829,6 @@
     ], 
     "uses_polling": true
   }, 
-  {
-    "args": [], 
-    "benchmark": true, 
-    "ci_platforms": [
-      "linux", 
-      "posix"
-    ], 
-    "cpu_cost": 1.0, 
-    "exclude_configs": [], 
-    "exclude_iomgrs": [], 
-    "flaky": false, 
-    "gtest": false, 
-    "language": "c++", 
-    "name": "bm_fullstack_streaming_pump", 
-    "platforms": [
-      "linux", 
-      "posix"
-    ], 
-    "uses_polling": true
-  }, 
   {
     "args": [], 
     "benchmark": true, 
@@ -3869,26 +3849,6 @@
     ], 
     "uses_polling": true
   }, 
-  {
-    "args": [], 
-    "benchmark": true, 
-    "ci_platforms": [
-      "linux", 
-      "posix"
-    ], 
-    "cpu_cost": 1.0, 
-    "exclude_configs": [], 
-    "exclude_iomgrs": [], 
-    "flaky": false, 
-    "gtest": false, 
-    "language": "c++", 
-    "name": "bm_metadata", 
-    "platforms": [
-      "linux", 
-      "posix"
-    ], 
-    "uses_polling": false
-  }, 
   {
     "args": [], 
     "benchmark": true, 
@@ -4713,28 +4673,6 @@
     ], 
     "uses_polling": true
   }, 
-  {
-    "args": [], 
-    "benchmark": false, 
-    "ci_platforms": [
-      "linux", 
-      "mac", 
-      "posix"
-    ], 
-    "cpu_cost": 1.0, 
-    "exclude_configs": [], 
-    "exclude_iomgrs": [], 
-    "flaky": false, 
-    "gtest": true, 
-    "language": "c++", 
-    "name": "grpclb_end2end_test", 
-    "platforms": [
-      "linux", 
-      "mac", 
-      "posix"
-    ], 
-    "uses_polling": true
-  }, 
   {
     "args": [], 
     "benchmark": false, 
@@ -5639,30 +5577,6 @@
     ], 
     "uses_polling": true
   }, 
-  {
-    "args": [], 
-    "benchmark": false, 
-    "ci_platforms": [
-      "linux", 
-      "mac", 
-      "posix", 
-      "windows"
-    ], 
-    "cpu_cost": 1.0, 
-    "exclude_configs": [], 
-    "exclude_iomgrs": [], 
-    "flaky": false, 
-    "gtest": true, 
-    "language": "c++", 
-    "name": "settings_timeout_test", 
-    "platforms": [
-      "linux", 
-      "mac", 
-      "posix", 
-      "windows"
-    ], 
-    "uses_polling": true
-  }, 
   {
     "args": [], 
     "benchmark": false, 
@@ -6207,28 +6121,6 @@
     ], 
     "uses_polling": true
   }, 
-  {
-    "args": [], 
-    "benchmark": false, 
-    "ci_platforms": [
-      "linux", 
-      "mac", 
-      "posix"
-    ], 
-    "cpu_cost": 1.0, 
-    "exclude_configs": [], 
-    "exclude_iomgrs": [], 
-    "flaky": false, 
-    "gtest": true, 
-    "language": "c++", 
-    "name": "xds_end2end_test", 
-    "platforms": [
-      "linux", 
-      "mac", 
-      "posix"
-    ], 
-    "uses_polling": true
-  }, 
   {
     "args": [], 
     "boringssl": true,