Revise c-ares timeouts to use c-ares's internal timeout/retry logic

CMakeLists.txt

@@ -2905,6 +2905,49 @@ target_link_libraries(test_tcp_server
 )
 
 
+endif (gRPC_BUILD_TESTS)
+if (gRPC_BUILD_TESTS)
+
+add_library(dns_test_util
+  test/cpp/naming/dns_test_util.cc
+)
+
+if(WIN32 AND MSVC)
+  set_target_properties(dns_test_util PROPERTIES COMPILE_PDB_NAME "dns_test_util"
+    COMPILE_PDB_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}"
+  )
+  if (gRPC_INSTALL)
+    install(FILES ${CMAKE_CURRENT_BINARY_DIR}/dns_test_util.pdb
+      DESTINATION ${gRPC_INSTALL_LIBDIR} OPTIONAL
+    )
+  endif()
+endif()
+
+
+target_include_directories(dns_test_util
+  PUBLIC $<INSTALL_INTERFACE:${gRPC_INSTALL_INCLUDEDIR}> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
+  PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR}
+  PRIVATE ${_gRPC_NANOPB_INCLUDE_DIR}
+  PRIVATE third_party/googletest/googletest/include
+  PRIVATE third_party/googletest/googletest
+  PRIVATE third_party/googletest/googlemock/include
+  PRIVATE third_party/googletest/googlemock
+  PRIVATE ${_gRPC_PROTO_GENS_DIR}
+)
+target_link_libraries(dns_test_util
+  ${_gRPC_PROTOBUF_LIBRARIES}
+  ${_gRPC_ALLTARGETS_LIBRARIES}
+  ${_gRPC_GFLAGS_LIBRARIES}
+)
+
+
 endif (gRPC_BUILD_TESTS)
 
 add_library(grpc++
@@ -18490,6 +18533,7 @@ target_include_directories(resolver_component_test_unsecure
 target_link_libraries(resolver_component_test_unsecure
   ${_gRPC_PROTOBUF_LIBRARIES}
   ${_gRPC_ALLTARGETS_LIBRARIES}
+  dns_test_util
   grpc++_test_util_unsecure
   grpc_test_util_unsecure
   grpc++_unsecure
@@ -18531,6 +18575,7 @@ target_include_directories(resolver_component_test
 target_link_libraries(resolver_component_test
   ${_gRPC_PROTOBUF_LIBRARIES}
   ${_gRPC_ALLTARGETS_LIBRARIES}
+  dns_test_util
   grpc++_test_util
   grpc_test_util
   grpc++
@@ -18740,6 +18785,7 @@ target_include_directories(cancel_ares_query_test
 target_link_libraries(cancel_ares_query_test
   ${_gRPC_PROTOBUF_LIBRARIES}
   ${_gRPC_ALLTARGETS_LIBRARIES}
+  dns_test_util
   grpc++_test_util
   grpc_test_util
   grpc++

Makefile

@@ -1411,9 +1411,9 @@ pc_cxx: $(LIBDIR)/$(CONFIG)/pkgconfig/grpc++.pc
 pc_cxx_unsecure: $(LIBDIR)/$(CONFIG)/pkgconfig/grpc++_unsecure.pc
 
 ifeq ($(EMBED_OPENSSL),true)
-privatelibs_cxx:  $(LIBDIR)/$(CONFIG)/libgrpc++_core_stats.a $(LIBDIR)/$(CONFIG)/libgrpc++_proto_reflection_desc_db.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc_cli_libs.a $(LIBDIR)/$(CONFIG)/libhttp2_client_main.a $(LIBDIR)/$(CONFIG)/libinterop_client_helper.a $(LIBDIR)/$(CONFIG)/libinterop_client_main.a $(LIBDIR)/$(CONFIG)/libinterop_server_helper.a $(LIBDIR)/$(CONFIG)/libinterop_server_lib.a $(LIBDIR)/$(CONFIG)/libinterop_server_main.a $(LIBDIR)/$(CONFIG)/libqps.a $(LIBDIR)/$(CONFIG)/libboringssl_test_util.a $(LIBDIR)/$(CONFIG)/libbenchmark.a
+privatelibs_cxx:  $(LIBDIR)/$(CONFIG)/libdns_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++_core_stats.a $(LIBDIR)/$(CONFIG)/libgrpc++_proto_reflection_desc_db.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc_cli_libs.a $(LIBDIR)/$(CONFIG)/libhttp2_client_main.a $(LIBDIR)/$(CONFIG)/libinterop_client_helper.a $(LIBDIR)/$(CONFIG)/libinterop_client_main.a $(LIBDIR)/$(CONFIG)/libinterop_server_helper.a $(LIBDIR)/$(CONFIG)/libinterop_server_lib.a $(LIBDIR)/$(CONFIG)/libinterop_server_main.a $(LIBDIR)/$(CONFIG)/libqps.a $(LIBDIR)/$(CONFIG)/libboringssl_test_util.a $(LIBDIR)/$(CONFIG)/libbenchmark.a
 else
-privatelibs_cxx:  $(LIBDIR)/$(CONFIG)/libgrpc++_core_stats.a $(LIBDIR)/$(CONFIG)/libgrpc++_proto_reflection_desc_db.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc_cli_libs.a $(LIBDIR)/$(CONFIG)/libhttp2_client_main.a $(LIBDIR)/$(CONFIG)/libinterop_client_helper.a $(LIBDIR)/$(CONFIG)/libinterop_client_main.a $(LIBDIR)/$(CONFIG)/libinterop_server_helper.a $(LIBDIR)/$(CONFIG)/libinterop_server_lib.a $(LIBDIR)/$(CONFIG)/libinterop_server_main.a $(LIBDIR)/$(CONFIG)/libqps.a $(LIBDIR)/$(CONFIG)/libbenchmark.a
+privatelibs_cxx:  $(LIBDIR)/$(CONFIG)/libdns_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++_core_stats.a $(LIBDIR)/$(CONFIG)/libgrpc++_proto_reflection_desc_db.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc_cli_libs.a $(LIBDIR)/$(CONFIG)/libhttp2_client_main.a $(LIBDIR)/$(CONFIG)/libinterop_client_helper.a $(LIBDIR)/$(CONFIG)/libinterop_client_main.a $(LIBDIR)/$(CONFIG)/libinterop_server_helper.a $(LIBDIR)/$(CONFIG)/libinterop_server_lib.a $(LIBDIR)/$(CONFIG)/libinterop_server_main.a $(LIBDIR)/$(CONFIG)/libqps.a $(LIBDIR)/$(CONFIG)/libbenchmark.a
 endif
 
 
@@ -5290,6 +5290,55 @@ endif
 endif
 
 
+LIBDNS_TEST_UTIL_SRC = \
+    test/cpp/naming/dns_test_util.cc \
+
+PUBLIC_HEADERS_CXX += \
+
+LIBDNS_TEST_UTIL_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(LIBDNS_TEST_UTIL_SRC))))
+
+
+ifeq ($(NO_SECURE),true)
+
+# You can't build secure libraries if you don't have OpenSSL.
+
+$(LIBDIR)/$(CONFIG)/libdns_test_util.a: openssl_dep_error
+
+
+else
+
+ifeq ($(NO_PROTOBUF),true)
+
+# You can't build a C++ library if you don't have protobuf - a bit overreached, but still okay.
+
+$(LIBDIR)/$(CONFIG)/libdns_test_util.a: protobuf_dep_error
+
+
+else
+
+$(LIBDIR)/$(CONFIG)/libdns_test_util.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(CARES_DEP) $(ADDRESS_SORTING_DEP) $(PROTOBUF_DEP) $(LIBDNS_TEST_UTIL_OBJS) 
+	$(E) "[AR]      Creating $@"
+	$(Q) mkdir -p `dirname $@`
+	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libdns_test_util.a
+	$(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libdns_test_util.a $(LIBDNS_TEST_UTIL_OBJS) 
+ifeq ($(SYSTEM),Darwin)
+	$(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libdns_test_util.a
+endif
+
+
+
+
+endif
+
+endif
+
+ifneq ($(NO_SECURE),true)
+ifneq ($(NO_DEPS),true)
+-include $(LIBDNS_TEST_UTIL_OBJS:.o=.dep)
+endif
+endif
+
+
 LIBGRPC++_SRC = \
     src/cpp/client/insecure_credentials.cc \
     src/cpp/client/secure_credentials.cc \
@@ -21283,16 +21332,16 @@ $(BINDIR)/$(CONFIG)/resolver_component_test_unsecure: protobuf_dep_error
 
 else
 
-$(BINDIR)/$(CONFIG)/resolver_component_test_unsecure: $(PROTOBUF_DEP) $(RESOLVER_COMPONENT_TEST_UNSECURE_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_test_util_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc++_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc_unsecure.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a
+$(BINDIR)/$(CONFIG)/resolver_component_test_unsecure: $(PROTOBUF_DEP) $(RESOLVER_COMPONENT_TEST_UNSECURE_OBJS) $(LIBDIR)/$(CONFIG)/libdns_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc++_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc_unsecure.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a
 	$(E) "[LD]      Linking $@"
 	$(Q) mkdir -p `dirname $@`
-	$(Q) $(LDXX) $(LDFLAGS) $(RESOLVER_COMPONENT_TEST_UNSECURE_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_test_util_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc++_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc_unsecure.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/resolver_component_test_unsecure
+	$(Q) $(LDXX) $(LDFLAGS) $(RESOLVER_COMPONENT_TEST_UNSECURE_OBJS) $(LIBDIR)/$(CONFIG)/libdns_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc++_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc_unsecure.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/resolver_component_test_unsecure
 
 endif
 
 endif
 
-$(OBJDIR)/$(CONFIG)/test/cpp/naming/resolver_component_test.o:  $(LIBDIR)/$(CONFIG)/libgrpc++_test_util_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc++_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc_unsecure.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a
+$(OBJDIR)/$(CONFIG)/test/cpp/naming/resolver_component_test.o:  $(LIBDIR)/$(CONFIG)/libdns_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc++_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc_unsecure.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a
 
 deps_resolver_component_test_unsecure: $(RESOLVER_COMPONENT_TEST_UNSECURE_OBJS:.o=.dep)
 
@@ -21326,16 +21375,16 @@ $(BINDIR)/$(CONFIG)/resolver_component_test: protobuf_dep_error
 
 else
 
-$(BINDIR)/$(CONFIG)/resolver_component_test: $(PROTOBUF_DEP) $(RESOLVER_COMPONENT_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a
+$(BINDIR)/$(CONFIG)/resolver_component_test: $(PROTOBUF_DEP) $(RESOLVER_COMPONENT_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libdns_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a
 	$(E) "[LD]      Linking $@"
 	$(Q) mkdir -p `dirname $@`
-	$(Q) $(LDXX) $(LDFLAGS) $(RESOLVER_COMPONENT_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/resolver_component_test
+	$(Q) $(LDXX) $(LDFLAGS) $(RESOLVER_COMPONENT_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libdns_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/resolver_component_test
 
 endif
 
 endif
 
-$(OBJDIR)/$(CONFIG)/test/cpp/naming/resolver_component_test.o:  $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a
+$(OBJDIR)/$(CONFIG)/test/cpp/naming/resolver_component_test.o:  $(LIBDIR)/$(CONFIG)/libdns_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a
 
 deps_resolver_component_test: $(RESOLVER_COMPONENT_TEST_OBJS:.o=.dep)
 
@@ -21541,16 +21590,16 @@ $(BINDIR)/$(CONFIG)/cancel_ares_query_test: protobuf_dep_error
 
 else
 
-$(BINDIR)/$(CONFIG)/cancel_ares_query_test: $(PROTOBUF_DEP) $(CANCEL_ARES_QUERY_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a
+$(BINDIR)/$(CONFIG)/cancel_ares_query_test: $(PROTOBUF_DEP) $(CANCEL_ARES_QUERY_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libdns_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a
 	$(E) "[LD]      Linking $@"
 	$(Q) mkdir -p `dirname $@`
-	$(Q) $(LDXX) $(LDFLAGS) $(CANCEL_ARES_QUERY_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/cancel_ares_query_test
+	$(Q) $(LDXX) $(LDFLAGS) $(CANCEL_ARES_QUERY_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libdns_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/cancel_ares_query_test
 
 endif
 
 endif
 
-$(OBJDIR)/$(CONFIG)/test/cpp/naming/cancel_ares_query_test.o:  $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a
+$(OBJDIR)/$(CONFIG)/test/cpp/naming/cancel_ares_query_test.o:  $(LIBDIR)/$(CONFIG)/libdns_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a
 
 deps_cancel_ares_query_test: $(CANCEL_ARES_QUERY_TEST_OBJS:.o=.dep)
 
@@ -22180,6 +22229,7 @@ test/cpp/interop/interop_server.cc: $(OPENSSL_DEP)
 test/cpp/interop/interop_server_bootstrap.cc: $(OPENSSL_DEP)
 test/cpp/interop/server_helper.cc: $(OPENSSL_DEP)
 test/cpp/microbenchmarks/helpers.cc: $(OPENSSL_DEP)
+test/cpp/naming/dns_test_util.cc: $(OPENSSL_DEP)
 test/cpp/qps/benchmark_config.cc: $(OPENSSL_DEP)
 test/cpp/qps/client_async.cc: $(OPENSSL_DEP)
 test/cpp/qps/client_callback.cc: $(OPENSSL_DEP)

build.yaml

@@ -1677,6 +1677,13 @@ libs:
   - grpc_test_util
   - grpc
   - gpr
+- name: dns_test_util
+  build: private
+  language: c++
+  headers:
+  - test/cpp/naming/dns_test_util.h
+  src:
+  - test/cpp/naming/dns_test_util.cc
 - name: grpc++
   build: all
   language: c++

grpc.gyp

@@ -1405,6 +1405,15 @@
         'test/core/util/test_tcp_server.cc',
       ],
     },
+    {
+      'target_name': 'dns_test_util',
+      'type': 'static_library',
+      'dependencies': [
+      ],
+      'sources': [
+        'test/cpp/naming/dns_test_util.cc',
+      ],
+    },
     {
       'target_name': 'grpc++',
       'type': 'static_library',

include/grpc/impl/codegen/grpc_types.h

@@ -359,10 +359,12 @@ typedef struct {
  * load balancing policy. Note that this only works with the "ares"
  * DNS resolver, and isn't supported by the "native" DNS resolver. */
 #define GRPC_ARG_DNS_ENABLE_SRV_QUERIES "grpc.dns_enable_srv_queries"
-/** If set, determines the number of milliseconds that the c-ares based
- * DNS resolver will wait on queries before cancelling them. The default value
- * is 10000. Setting this to "0" will disable c-ares query timeouts
- * entirely. */
+/** If set, determines an upper bound on the number of milliseconds that the
+ * c-ares based DNS resolver will wait on queries before cancelling them.
+ * The default value is 120,000. Setting this to "0" will disable the
+ * overall timeout entirely. Note that this doesn't include internal c-ares
+ * timeouts/backoff/retry logic, and so the actual DNS resolution may time out
+ * sooner than the value specified here. */
 #define GRPC_ARG_DNS_ARES_QUERY_TIMEOUT_MS "grpc.dns_ares_query_timeout"
 /** If set, uses a local subchannel pool within the channel. Otherwise, uses the
  * global subchannel pool. */

src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.cc

@@ -84,6 +84,10 @@ struct grpc_ares_ev_driver {
   grpc_timer query_timeout;
   /** cancels queries on a timeout */
   grpc_closure on_timeout_locked;
+  /** alarm to poll ares_process on in case fd events don't happen */
+  grpc_timer ares_backup_poll_alarm;
+  /** polls ares_process on a periodic timer */
+  grpc_closure on_ares_backup_poll_alarm_locked;
 };
 
 static void grpc_ares_notify_on_event_locked(grpc_ares_ev_driver* ev_driver);
@@ -130,6 +134,13 @@ static void fd_node_shutdown_locked(fd_node* fdn, const char* reason) {
 
 static void on_timeout_locked(void* arg, grpc_error* error);
 
+static void on_ares_backup_poll_alarm_locked(void* arg, grpc_error* error);
+
+static void noop_inject_channel_config(ares_channel channel) {}
+
+void (*grpc_ares_test_only_inject_config)(ares_channel channel) =
+    noop_inject_channel_config;
+
 grpc_error* grpc_ares_ev_driver_create_locked(grpc_ares_ev_driver** ev_driver,
                                               grpc_pollset_set* pollset_set,
                                               int query_timeout_ms,
@@ -140,6 +151,7 @@ grpc_error* grpc_ares_ev_driver_create_locked(grpc_ares_ev_driver** ev_driver,
   memset(&opts, 0, sizeof(opts));
   opts.flags |= ARES_FLAG_STAYOPEN;
   int status = ares_init_options(&(*ev_driver)->channel, &opts, ARES_OPT_FLAGS);
+  grpc_ares_test_only_inject_config((*ev_driver)->channel);
   GRPC_CARES_TRACE_LOG("request:%p grpc_ares_ev_driver_create_locked", request);
   if (status != ARES_SUCCESS) {
     char* err_msg;
@@ -163,6 +175,9 @@ grpc_error* grpc_ares_ev_driver_create_locked(grpc_ares_ev_driver** ev_driver,
       ->polled_fd_factory->ConfigureAresChannelLocked((*ev_driver)->channel);
   GRPC_CLOSURE_INIT(&(*ev_driver)->on_timeout_locked, on_timeout_locked,
                     *ev_driver, grpc_combiner_scheduler(combiner));
+  GRPC_CLOSURE_INIT(&(*ev_driver)->on_ares_backup_poll_alarm_locked,
+                    on_ares_backup_poll_alarm_locked, *ev_driver,
+                    grpc_combiner_scheduler(combiner));
   (*ev_driver)->query_timeout_ms = query_timeout_ms;
   return GRPC_ERROR_NONE;
 }
@@ -174,6 +189,7 @@ void grpc_ares_ev_driver_on_queries_complete_locked(
   // fds; if it's not working, there are no fds to shut down.
   ev_driver->shutting_down = true;
   grpc_timer_cancel(&ev_driver->query_timeout);
+  grpc_timer_cancel(&ev_driver->ares_backup_poll_alarm);
   grpc_ares_ev_driver_unref(ev_driver);
 }
 
@@ -204,6 +220,21 @@ static fd_node* pop_fd_node_locked(fd_node** head, ares_socket_t as) {
   return nullptr;
 }
 
+static grpc_millis calculate_next_ares_backup_poll_alarm_ms(
+    grpc_ares_ev_driver* driver) {
+  // An alternative here could be to use ares_timeout to try to be more
+  // accurate, but that would require using "struct timeval"'s, which just makes
+  // things a bit more complicated. So just poll every second, as suggested
+  // by the c-ares code comments.
+  grpc_millis ms_until_next_ares_backup_poll_alarm = 1000;
+  GRPC_CARES_TRACE_LOG(
+      "request:%p ev_driver=%p. next ares process poll time in "
+      "%" PRId64 " ms",
+      driver->request, driver, ms_until_next_ares_backup_poll_alarm);
+  return ms_until_next_ares_backup_poll_alarm +
+         grpc_core::ExecCtx::Get()->Now();
+}
+
 static void on_timeout_locked(void* arg, grpc_error* error) {
   grpc_ares_ev_driver* driver = static_cast<grpc_ares_ev_driver*>(arg);
   GRPC_CARES_TRACE_LOG(
@@ -216,6 +247,47 @@ static void on_timeout_locked(void* arg, grpc_error* error) {
   grpc_ares_ev_driver_unref(driver);
 }
 
+/* In case of non-responsive DNS servers, dropped packets, etc., c-ares has
+ * intelligent timeout and retry logic, which we can take advantage of by
+ * polling ares_process_fd on time intervals. Overall, the c-ares library is
+ * meant to be called into and given a chance to proceed name resolution:
+ *   a) when fd events happen
+ *   b) when some time has passed without fd events having happened
+ * For the latter, we use this backup poller. Also see
+ * https://github.com/grpc/grpc/pull/17688 description for more details. */
+static void on_ares_backup_poll_alarm_locked(void* arg, grpc_error* error) {
+  grpc_ares_ev_driver* driver = static_cast<grpc_ares_ev_driver*>(arg);
+  GRPC_CARES_TRACE_LOG(
+      "request:%p ev_driver=%p on_ares_backup_poll_alarm_locked. "
+      "driver->shutting_down=%d. "
+      "err=%s",
+      driver->request, driver, driver->shutting_down, grpc_error_string(error));
+  if (!driver->shutting_down && error == GRPC_ERROR_NONE) {
+    fd_node* fdn = driver->fds;
+    while (fdn != nullptr) {
+      if (!fdn->already_shutdown) {
+        GRPC_CARES_TRACE_LOG(
+            "request:%p ev_driver=%p on_ares_backup_poll_alarm_locked; "
+            "ares_process_fd. fd=%s",
+            driver->request, driver, fdn->grpc_polled_fd->GetName());
+        ares_socket_t as = fdn->grpc_polled_fd->GetWrappedAresSocketLocked();
+        ares_process_fd(driver->channel, as, as);
+      }
+      fdn = fdn->next;
+    }
+    if (!driver->shutting_down) {
+      grpc_millis next_ares_backup_poll_alarm =
+          calculate_next_ares_backup_poll_alarm_ms(driver);
+      grpc_ares_ev_driver_ref(driver);
+      grpc_timer_init(&driver->ares_backup_poll_alarm,
+                      next_ares_backup_poll_alarm,
+                      &driver->on_ares_backup_poll_alarm_locked);
+    }
+    grpc_ares_notify_on_event_locked(driver);
+  }
+  grpc_ares_ev_driver_unref(driver);
+}
+
 static void on_readable_locked(void* arg, grpc_error* error) {
   fd_node* fdn = static_cast<fd_node*>(arg);
   GPR_ASSERT(fdn->readable_registered);
@@ -353,6 +425,7 @@ void grpc_ares_ev_driver_start_locked(grpc_ares_ev_driver* ev_driver) {
   if (!ev_driver->working) {
     ev_driver->working = true;
     grpc_ares_notify_on_event_locked(ev_driver);
+    // Initialize overall DNS resolution timeout alarm
     grpc_millis timeout =
         ev_driver->query_timeout_ms == 0
             ? GRPC_MILLIS_INF_FUTURE
@@ -364,6 +437,13 @@ void grpc_ares_ev_driver_start_locked(grpc_ares_ev_driver* ev_driver) {
     grpc_ares_ev_driver_ref(ev_driver);
     grpc_timer_init(&ev_driver->query_timeout, timeout,
                     &ev_driver->on_timeout_locked);
+    // Initialize the backup poll alarm
+    grpc_millis next_ares_backup_poll_alarm =
+        calculate_next_ares_backup_poll_alarm_ms(ev_driver);
+    grpc_ares_ev_driver_ref(ev_driver);
+    grpc_timer_init(&ev_driver->ares_backup_poll_alarm,
+                    next_ares_backup_poll_alarm,
+                    &ev_driver->on_ares_backup_poll_alarm_locked);
   }
 }
 

src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.h

@@ -54,6 +54,9 @@ void grpc_ares_ev_driver_on_queries_complete_locked(
 /* Shutdown all the grpc_fds used by \a ev_driver */
 void grpc_ares_ev_driver_shutdown_locked(grpc_ares_ev_driver* ev_driver);
 
+/* Exposed in this header for C-core tests only */
+extern void (*grpc_ares_test_only_inject_config)(ares_channel channel);
+
 namespace grpc_core {
 
 /* A wrapped fd that integrates with the grpc iomgr of the current platform.

src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc

@@ -109,6 +109,7 @@ class GrpcPolledFdWindows : public GrpcPolledFd {
     read_closure_ = read_closure;
     GPR_ASSERT(GRPC_SLICE_LENGTH(read_buf_) == 0);
     grpc_slice_unref_internal(read_buf_);
+    GPR_ASSERT(!read_buf_has_data_);
     read_buf_ = GRPC_SLICE_MALLOC(4192);
     WSABUF buffer;
     buffer.buf = (char*)GRPC_SLICE_START_PTR(read_buf_);
@@ -175,7 +176,7 @@ class GrpcPolledFdWindows : public GrpcPolledFd {
     GRPC_CARES_TRACE_LOG(
         "RecvFrom called on fd:|%s|. Current read buf length:|%d|", GetName(),
         GRPC_SLICE_LENGTH(read_buf_));
-    if (GRPC_SLICE_LENGTH(read_buf_) == 0) {
+    if (!read_buf_has_data_) {
       WSASetLastError(WSAEWOULDBLOCK);
       return -1;
     }
@@ -186,6 +187,9 @@ class GrpcPolledFdWindows : public GrpcPolledFd {
     }
     read_buf_ = grpc_slice_sub_no_ref(read_buf_, bytes_read,
                                       GRPC_SLICE_LENGTH(read_buf_));
+    if (GRPC_SLICE_LENGTH(read_buf_) == 0) {
+      read_buf_has_data_ = false;
+    }
     /* c-ares overloads this recv_from virtual socket function to receive
      * data on both UDP and TCP sockets, and from is nullptr for TCP. */
     if (from != nullptr) {
@@ -302,6 +306,11 @@ class GrpcPolledFdWindows : public GrpcPolledFd {
     polled_fd->OnIocpReadableInner(error);
   }
 
+  // TODO(apolcyn): improve this error handling to be less conversative.
+  // An e.g. ECONNRESET error here should result in errors when
+  // c-ares reads from this socket later, but it shouldn't necessarily cancel
+  // the entire resolution attempt. Doing so will allow the "inject broken
+  // nameserver list" test to pass on Windows.
   void OnIocpReadableInner(grpc_error* error) {
     if (error == GRPC_ERROR_NONE) {
       if (winsocket_->read_info.wsa_error != 0) {
@@ -323,6 +332,7 @@ class GrpcPolledFdWindows : public GrpcPolledFd {
     if (error == GRPC_ERROR_NONE) {
       read_buf_ = grpc_slice_sub_no_ref(read_buf_, 0,
                                         winsocket_->read_info.bytes_transfered);
+      read_buf_has_data_ = true;
     } else {
       grpc_slice_unref_internal(read_buf_);
       read_buf_ = grpc_empty_slice();
@@ -370,6 +380,7 @@ class GrpcPolledFdWindows : public GrpcPolledFd {
   char recv_from_source_addr_[200];
   ares_socklen_t recv_from_source_addr_len_;
   grpc_slice read_buf_;
+  bool read_buf_has_data_ = false;
   grpc_slice write_buf_;
   grpc_closure* read_closure_ = nullptr;
   grpc_closure* write_closure_ = nullptr;

src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h

@@ -26,7 +26,7 @@
 #include "src/core/lib/iomgr/polling_entity.h"
 #include "src/core/lib/iomgr/resolve_address.h"
 
-#define GRPC_DNS_ARES_DEFAULT_QUERY_TIMEOUT_MS 10000
+#define GRPC_DNS_ARES_DEFAULT_QUERY_TIMEOUT_MS 120000
 
 extern grpc_core::TraceFlag grpc_trace_cares_address_sorting;
 

test/core/iomgr/resolve_address_test.cc

@@ -83,7 +83,9 @@ static grpc_millis n_sec_deadline(int seconds) {
 
 static void poll_pollset_until_request_done(args_struct* args) {
   grpc_core::ExecCtx exec_ctx;
-  grpc_millis deadline = n_sec_deadline(10);
+  // Try to give enough time for c-ares to run through its retries
+  // a few times if needed.
+  grpc_millis deadline = n_sec_deadline(90);
   while (true) {
     bool done = gpr_atm_acq_load(&args->done_atm) != 0;
     if (done) {
@@ -371,15 +373,10 @@ int main(int argc, char** argv) {
     test_missing_default_port();
     test_ipv6_with_port();
     test_ipv6_without_port();
-    if (gpr_stricmp(resolver_type, "ares") != 0) {
-      // These tests can trigger DNS queries to the nearby nameserver
-      // that need to come back in order for the test to succeed.
-      // c-ares is prone to not using the local system caches that the
-      // native getaddrinfo implementations take advantage of, so running
-      // these unit tests under c-ares risks flakiness.
-      test_invalid_ip_addresses();
-      test_unparseable_hostports();
-    } else {
+    test_invalid_ip_addresses();
+    test_unparseable_hostports();
+    if (gpr_stricmp(resolver_type, "ares") == 0) {
+      // This behavior expectation is specific to c-ares.
       test_localhost_result_has_ipv6_first();
     }
     grpc_core::Executor::ShutdownAll();

test/cpp/naming/BUILD

@@ -22,7 +22,7 @@ package(
 
 licenses(["notice"])  # Apache v2
 
-load("//bazel:grpc_build_system.bzl", "grpc_py_binary", "grpc_cc_test")
+load("//bazel:grpc_build_system.bzl", "grpc_py_binary", "grpc_cc_test", "grpc_cc_library")
 load(":generate_resolver_component_tests.bzl", "generate_resolver_component_tests")
 
 # Meant to be invoked only through the top-level shell script driver.
@@ -39,6 +39,7 @@ grpc_cc_test(
     srcs = ["cancel_ares_query_test.cc"],
     external_deps = ["gmock"],
     deps = [
+        ":dns_test_util",
         "//:gpr",
         "//:grpc",
         "//:grpc++",
@@ -49,4 +50,14 @@ grpc_cc_test(
     ],
 )
 
+grpc_cc_library(
+    name = "dns_test_util",
+    hdrs = ["dns_test_util.h"],
+    srcs = ["dns_test_util.cc"],
+    deps = [
+        "//:gpr",
+        "//:grpc",
+    ],
+)
+
 generate_resolver_component_tests()

test/cpp/naming/cancel_ares_query_test.cc

@@ -44,6 +44,7 @@
 #include "test/core/util/cmdline.h"
 #include "test/core/util/port.h"
 #include "test/core/util/test_config.h"
+#include "test/cpp/naming/dns_test_util.h"
 
 #ifdef GPR_WINDOWS
 #include "src/core/lib/iomgr/sockaddr_windows.h"
@@ -76,36 +77,6 @@ void EndTest(grpc_channel* client, grpc_completion_queue* cq) {
   grpc_completion_queue_destroy(cq);
 }
 
-class FakeNonResponsiveDNSServer {
- public:
-  FakeNonResponsiveDNSServer(int port) {
-    socket_ = socket(AF_INET6, SOCK_DGRAM, 0);
-    if (socket_ == BAD_SOCKET_RETURN_VAL) {
-      gpr_log(GPR_DEBUG, "Failed to create UDP ipv6 socket");
-      abort();
-    }
-    sockaddr_in6 addr;
-    memset(&addr, 0, sizeof(addr));
-    addr.sin6_family = AF_INET6;
-    addr.sin6_port = htons(port);
-    ((char*)&addr.sin6_addr)[15] = 1;
-    if (bind(socket_, (const sockaddr*)&addr, sizeof(addr)) != 0) {
-      gpr_log(GPR_DEBUG, "Failed to bind UDP ipv6 socket to [::1]:%d", port);
-      abort();
-    }
-  }
-  ~FakeNonResponsiveDNSServer() {
-#ifdef GPR_WINDOWS
-    closesocket(socket_);
-#else
-    close(socket_);
-#endif
-  }
-
- private:
-  int socket_;
-};
-
 struct ArgsStruct {
   gpr_atm done_atm;
   gpr_mu* mu;
@@ -184,7 +155,7 @@ class AssertFailureResultHandler : public grpc_core::Resolver::ResultHandler {
 
 void TestCancelActiveDNSQuery(ArgsStruct* args) {
   int fake_dns_port = grpc_pick_unused_port_or_die();
-  FakeNonResponsiveDNSServer fake_dns_server(fake_dns_port);
+  grpc::testing::FakeNonResponsiveDNSServer fake_dns_server(fake_dns_port);
   char* client_target;
   GPR_ASSERT(gpr_asprintf(
       &client_target,
@@ -282,7 +253,7 @@ void TestCancelDuringActiveQuery(
     cancellation_test_query_timeout_setting query_timeout_setting) {
   // Start up fake non responsive DNS server
   int fake_dns_port = grpc_pick_unused_port_or_die();
-  FakeNonResponsiveDNSServer fake_dns_server(fake_dns_port);
+  grpc::testing::FakeNonResponsiveDNSServer fake_dns_server(fake_dns_port);
   // Create a call that will try to use the fake DNS server
   char* client_target = nullptr;
   GPR_ASSERT(gpr_asprintf(

test/cpp/naming/dns_test_util.cc

@@ -0,0 +1,97 @@
+/*
+ *
+ * Copyright 2015 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include <stdio.h>
+#include <string.h>
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+
+#include "test/cpp/naming/dns_test_util.h"
+
+#ifdef GPR_WINDOWS
+#include "src/core/lib/iomgr/sockaddr_windows.h"
+#include "src/core/lib/iomgr/socket_windows.h"
+#define BAD_SOCKET_RETURN_VAL INVALID_SOCKET
+#else
+#include "src/core/lib/iomgr/sockaddr_posix.h"
+#define BAD_SOCKET_RETURN_VAL -1
+#endif
+
+namespace grpc {
+namespace testing {
+
+FakeNonResponsiveDNSServer::FakeNonResponsiveDNSServer(int port) {
+  udp_socket_ = socket(AF_INET6, SOCK_DGRAM, 0);
+  tcp_socket_ = socket(AF_INET6, SOCK_STREAM, 0);
+  if (udp_socket_ == BAD_SOCKET_RETURN_VAL) {
+    gpr_log(GPR_DEBUG, "Failed to create UDP ipv6 socket");
+    abort();
+  }
+  if (tcp_socket_ == BAD_SOCKET_RETURN_VAL) {
+    gpr_log(GPR_DEBUG, "Failed to create TCP ipv6 socket");
+    abort();
+  }
+  sockaddr_in6 addr;
+  memset(&addr, 0, sizeof(addr));
+  addr.sin6_family = AF_INET6;
+  addr.sin6_port = htons(port);
+  ((char*)&addr.sin6_addr)[15] = 1;
+  if (bind(udp_socket_, (const sockaddr*)&addr, sizeof(addr)) != 0) {
+    gpr_log(GPR_DEBUG, "Failed to bind UDP ipv6 socket to [::1]:%d", port);
+    abort();
+  }
+#ifdef GPR_WINDOWS
+  char val = 1;
+  if (setsockopt(tcp_socket_, SOL_SOCKET, SO_REUSEADDR, &val, sizeof(val)) ==
+      SOCKET_ERROR) {
+    gpr_log(GPR_DEBUG,
+            "Failed to set SO_REUSEADDR on TCP ipv6 socket to [::1]:%d", port);
+    abort();
+  }
+#else
+  int val = 1;
+  if (setsockopt(tcp_socket_, SOL_SOCKET, SO_REUSEADDR, &val, sizeof(val)) !=
+      0) {
+    gpr_log(GPR_DEBUG,
+            "Failed to set SO_REUSEADDR on TCP ipv6 socket to [::1]:%d", port);
+    abort();
+  }
+#endif
+  if (bind(tcp_socket_, (const sockaddr*)&addr, sizeof(addr)) != 0) {
+    gpr_log(GPR_DEBUG, "Failed to bind TCP ipv6 socket to [::1]:%d", port);
+    abort();
+  }
+  if (listen(tcp_socket_, 100)) {
+    gpr_log(GPR_DEBUG, "Failed to listen on TCP ipv6 socket to [::1]:%d", port);
+    abort();
+  }
+}
+
+FakeNonResponsiveDNSServer::~FakeNonResponsiveDNSServer() {
+#ifdef GPR_WINDOWS
+  closesocket(udp_socket_);
+  closesocket(tcp_socket_);
+#else
+  close(udp_socket_);
+  close(tcp_socket_);
+#endif
+}
+
+}  // namespace testing
+}  // namespace grpc

test/cpp/naming/dns_test_util.h

@@ -0,0 +1,38 @@
+/*
+ *
+ * Copyright 2015 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#ifndef GRPC_DNS_TEST_UTIL_H
+#define GRPC_DNS_TEST_UTIL_H
+
+namespace grpc {
+namespace testing {
+
+class FakeNonResponsiveDNSServer {
+ public:
+  explicit FakeNonResponsiveDNSServer(int port);
+  virtual ~FakeNonResponsiveDNSServer();
+
+ private:
+  int udp_socket_;
+  int tcp_socket_;
+};
+
+}  // namespace testing
+}  // namespace grpc
+
+#endif /* GRPC_DNS_TEST_UTIL_H */

test/cpp/naming/gen_build_yaml.py

@@ -50,6 +50,7 @@ def _resolver_test_cases(resolver_component_data):
             ('expected_lb_policy', (test_case['expected_lb_policy'] or '')),
             ('enable_srv_queries', test_case['enable_srv_queries']),
             ('enable_txt_queries', test_case['enable_txt_queries']),
+            ('inject_broken_nameserver_list', test_case['inject_broken_nameserver_list']),
         ],
     })
   return out
@@ -72,6 +73,7 @@ def main():
               'src': ['test/cpp/naming/resolver_component_test.cc'],
               'platforms': ['linux', 'posix', 'mac', 'windows'],
               'deps': [
+                  'dns_test_util',
                   'grpc++_test_util' + unsecure_build_config_suffix,
                   'grpc_test_util' + unsecure_build_config_suffix,
                   'grpc++' + unsecure_build_config_suffix,
@@ -130,6 +132,7 @@ def main():
           'src': ['test/cpp/naming/cancel_ares_query_test.cc'],
           'platforms': ['linux', 'posix', 'mac', 'windows'],
           'deps': [
+              'dns_test_util',
               'grpc++_test_util',
               'grpc_test_util',
               'grpc++',

test/cpp/naming/generate_resolver_component_tests.bzl

@@ -46,6 +46,7 @@ def generate_resolver_component_tests():
             "gmock",
         ],
         deps = [
+            ":dns_test_util",
             "//test/cpp/util:test_util%s" % unsecure_build_config_suffix,
             "//test/core/util:grpc_test_util%s" % unsecure_build_config_suffix,
             "//:grpc++%s" % unsecure_build_config_suffix,

test/cpp/naming/resolver_component_test.cc

@@ -39,7 +39,9 @@
 #include "test/cpp/util/test_config.h"
 
 #include "src/core/ext/filters/client_channel/client_channel.h"
+#include "src/core/ext/filters/client_channel/parse_address.h"
 #include "src/core/ext/filters/client_channel/resolver.h"
+#include "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.h"
 #include "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h"
 #include "src/core/ext/filters/client_channel/resolver_registry.h"
 #include "src/core/ext/filters/client_channel/server_address.h"
@@ -53,9 +55,12 @@
 #include "src/core/lib/iomgr/iomgr.h"
 #include "src/core/lib/iomgr/resolve_address.h"
 #include "src/core/lib/iomgr/sockaddr_utils.h"
+#include "src/core/lib/iomgr/socket_utils.h"
 #include "test/core/util/port.h"
 #include "test/core/util/test_config.h"
 
+#include "test/cpp/naming/dns_test_util.h"
+
 // TODO: pull in different headers when enabling this
 // test on windows. Also set BAD_SOCKET_RETURN_VAL
 // to INVALID_SOCKET on windows.
@@ -106,6 +111,17 @@ DEFINE_string(
     "generate "
     "the python script runner doesn't allow us to pass a gflags bool to this "
     "binary.");
+DEFINE_string(
+    inject_broken_nameserver_list, "",
+    "Whether or not to configure c-ares to use a broken nameserver list, in "
+    "which "
+    "the first nameserver in the list is non-responsive, but the second one "
+    "works, i.e "
+    "serves the expected DNS records; using for testing such a real scenario."
+    "It would be better if this arg could be bool, but the way that we "
+    "generate "
+    "the python script runner doesn't allow us to pass a gflags bool to this "
+    "binary.");
 DEFINE_string(expected_lb_policy, "",
               "Expected lb policy name that appears in resolver result channel "
               "arg. Empty for none.");
@@ -217,7 +233,10 @@ gpr_timespec NSecondDeadline(int seconds) {
 }
 
 void PollPollsetUntilRequestDone(ArgsStruct* args) {
-  gpr_timespec deadline = NSecondDeadline(10);
+  // Use a 20-second timeout to give room for the tests that involve
+  // a non-responsive name server (c-ares uses a ~5 second query timeout
+  // for that server before succeeding with the healthy one).
+  gpr_timespec deadline = NSecondDeadline(20);
   while (true) {
     bool done = gpr_atm_acq_load(&args->done_atm) != 0;
     if (done) {
@@ -469,6 +488,50 @@ class CheckingResultHandler : public ResultHandler {
   }
 };
 
+int g_fake_non_responsive_dns_server_port = -1;
+
+/* This function will configure any ares_channel created by the c-ares based
+ * resolver. This is useful to effectively mock /etc/resolv.conf settings
+ * (and equivalent on Windows), which unit tests don't have write permissions.
+ */
+void InjectBrokenNameServerList(ares_channel channel) {
+  struct ares_addr_port_node dns_server_addrs[2];
+  memset(dns_server_addrs, 0, sizeof(dns_server_addrs));
+  char* unused_host;
+  char* local_dns_server_port;
+  GPR_ASSERT(gpr_split_host_port(FLAGS_local_dns_server_address.c_str(),
+                                 &unused_host, &local_dns_server_port));
+  gpr_log(GPR_DEBUG,
+          "Injecting broken nameserver list. Bad server address:|[::1]:%d|. "
+          "Good server address:%s",
+          g_fake_non_responsive_dns_server_port,
+          FLAGS_local_dns_server_address.c_str());
+  // Put the non-responsive DNS server at the front of c-ares's nameserver list.
+  dns_server_addrs[0].family = AF_INET6;
+  ((char*)&dns_server_addrs[0].addr.addr6)[15] = 0x1;
+  dns_server_addrs[0].tcp_port = g_fake_non_responsive_dns_server_port;
+  dns_server_addrs[0].udp_port = g_fake_non_responsive_dns_server_port;
+  dns_server_addrs[0].next = &dns_server_addrs[1];
+  // Put the actual healthy DNS server after the first one. The expectation is
+  // that the resolver will timeout the query to the non-responsive DNS server
+  // and will skip over to this healthy DNS server, without causing any DNS
+  // resolution errors.
+  dns_server_addrs[1].family = AF_INET;
+  ((char*)&dns_server_addrs[1].addr.addr4)[0] = 0x7f;
+  ((char*)&dns_server_addrs[1].addr.addr4)[3] = 0x1;
+  dns_server_addrs[1].tcp_port = atoi(local_dns_server_port);
+  dns_server_addrs[1].udp_port = atoi(local_dns_server_port);
+  dns_server_addrs[1].next = nullptr;
+  GPR_ASSERT(ares_set_servers_ports(channel, dns_server_addrs) == ARES_SUCCESS);
+  gpr_free(local_dns_server_port);
+  gpr_free(unused_host);
+}
+
+void StartResolvingLocked(void* arg, grpc_error* unused) {
+  grpc_core::Resolver* r = static_cast<grpc_core::Resolver*>(arg);
+  r->StartLocked();
+}
+
 void RunResolvesRelevantRecordsTest(
     grpc_core::UniquePtr<grpc_core::Resolver::ResultHandler> (
         *CreateResultHandler)(ArgsStruct* args)) {
@@ -480,9 +543,29 @@ void RunResolvesRelevantRecordsTest(
   args.expected_lb_policy = FLAGS_expected_lb_policy;
   // maybe build the address with an authority
   char* whole_uri = nullptr;
-  GPR_ASSERT(gpr_asprintf(&whole_uri, "dns://%s/%s",
-                          FLAGS_local_dns_server_address.c_str(),
-                          FLAGS_target_name.c_str()));
+  gpr_log(GPR_DEBUG,
+          "resolver_component_test: --inject_broken_nameserver_list: %s",
+          FLAGS_inject_broken_nameserver_list.c_str());
+  grpc_core::UniquePtr<grpc::testing::FakeNonResponsiveDNSServer>
+      fake_non_responsive_dns_server;
+  if (FLAGS_inject_broken_nameserver_list == "True") {
+    g_fake_non_responsive_dns_server_port = grpc_pick_unused_port_or_die();
+    fake_non_responsive_dns_server.reset(
+        grpc_core::New<grpc::testing::FakeNonResponsiveDNSServer>(
+            g_fake_non_responsive_dns_server_port));
+    grpc_ares_test_only_inject_config = InjectBrokenNameServerList;
+    GPR_ASSERT(
+        gpr_asprintf(&whole_uri, "dns:///%s", FLAGS_target_name.c_str()));
+  } else if (FLAGS_inject_broken_nameserver_list == "False") {
+    gpr_log(GPR_INFO, "Specifying authority in uris to: %s",
+            FLAGS_local_dns_server_address.c_str());
+    GPR_ASSERT(gpr_asprintf(&whole_uri, "dns://%s/%s",
+                            FLAGS_local_dns_server_address.c_str(),
+                            FLAGS_target_name.c_str()));
+  } else {
+    gpr_log(GPR_DEBUG, "Invalid value for --inject_broken_nameserver_list.");
+    abort();
+  }
   gpr_log(GPR_DEBUG, "resolver_component_test: --enable_srv_queries: %s",
           FLAGS_enable_srv_queries.c_str());
   grpc_channel_args* resolver_args = nullptr;
@@ -525,7 +608,9 @@ void RunResolvesRelevantRecordsTest(
                                                   CreateResultHandler(&args));
   grpc_channel_args_destroy(resolver_args);
   gpr_free(whole_uri);
-  resolver->StartLocked();
+  GRPC_CLOSURE_SCHED(GRPC_CLOSURE_CREATE(StartResolvingLocked, resolver.get(),
+                                         grpc_combiner_scheduler(args.lock)),
+                     GRPC_ERROR_NONE);
   grpc_core::ExecCtx::Get()->Flush();
   PollPollsetUntilRequestDone(&args);
   ArgsFinish(&args);
@@ -560,10 +645,6 @@ int main(int argc, char** argv) {
     gpr_log(GPR_ERROR, "Missing target_name param.");
     abort();
   }
-  if (FLAGS_local_dns_server_address != "") {
-    gpr_log(GPR_INFO, "Specifying authority in uris to: %s",
-            FLAGS_local_dns_server_address.c_str());
-  }
   auto result = RUN_ALL_TESTS();
   grpc_shutdown();
   return result;

test/cpp/naming/resolver_component_tests_runner.py

@@ -127,6 +127,7 @@ current_test_subprocess = subprocess.Popen([
   '--expected_lb_policy', '',
   '--enable_srv_queries', 'True',
   '--enable_txt_queries', 'True',
+  '--inject_broken_nameserver_list', 'False',
   '--local_dns_server_address', '127.0.0.1:%d' % args.dns_server_port])
 current_test_subprocess.communicate()
 if current_test_subprocess.returncode != 0:
@@ -141,6 +142,7 @@ current_test_subprocess = subprocess.Popen([
   '--expected_lb_policy', '',
   '--enable_srv_queries', 'True',
   '--enable_txt_queries', 'True',
+  '--inject_broken_nameserver_list', 'False',
   '--local_dns_server_address', '127.0.0.1:%d' % args.dns_server_port])
 current_test_subprocess.communicate()
 if current_test_subprocess.returncode != 0:
@@ -155,6 +157,7 @@ current_test_subprocess = subprocess.Popen([
   '--expected_lb_policy', '',
   '--enable_srv_queries', 'True',
   '--enable_txt_queries', 'True',
+  '--inject_broken_nameserver_list', 'False',
   '--local_dns_server_address', '127.0.0.1:%d' % args.dns_server_port])
 current_test_subprocess.communicate()
 if current_test_subprocess.returncode != 0:
@@ -169,6 +172,7 @@ current_test_subprocess = subprocess.Popen([
   '--expected_lb_policy', '',
   '--enable_srv_queries', 'True',
   '--enable_txt_queries', 'True',
+  '--inject_broken_nameserver_list', 'False',
   '--local_dns_server_address', '127.0.0.1:%d' % args.dns_server_port])
 current_test_subprocess.communicate()
 if current_test_subprocess.returncode != 0:
@@ -183,6 +187,7 @@ current_test_subprocess = subprocess.Popen([
   '--expected_lb_policy', '',
   '--enable_srv_queries', 'True',
   '--enable_txt_queries', 'True',
+  '--inject_broken_nameserver_list', 'False',
   '--local_dns_server_address', '127.0.0.1:%d' % args.dns_server_port])
 current_test_subprocess.communicate()
 if current_test_subprocess.returncode != 0:
@@ -197,6 +202,7 @@ current_test_subprocess = subprocess.Popen([
   '--expected_lb_policy', 'round_robin',
   '--enable_srv_queries', 'True',
   '--enable_txt_queries', 'True',
+  '--inject_broken_nameserver_list', 'False',
   '--local_dns_server_address', '127.0.0.1:%d' % args.dns_server_port])
 current_test_subprocess.communicate()
 if current_test_subprocess.returncode != 0:
@@ -211,6 +217,7 @@ current_test_subprocess = subprocess.Popen([
   '--expected_lb_policy', 'round_robin',
   '--enable_srv_queries', 'True',
   '--enable_txt_queries', 'True',
+  '--inject_broken_nameserver_list', 'False',
   '--local_dns_server_address', '127.0.0.1:%d' % args.dns_server_port])
 current_test_subprocess.communicate()
 if current_test_subprocess.returncode != 0:
@@ -225,6 +232,7 @@ current_test_subprocess = subprocess.Popen([
   '--expected_lb_policy', '',
   '--enable_srv_queries', 'True',
   '--enable_txt_queries', 'True',
+  '--inject_broken_nameserver_list', 'False',
   '--local_dns_server_address', '127.0.0.1:%d' % args.dns_server_port])
 current_test_subprocess.communicate()
 if current_test_subprocess.returncode != 0:
@@ -239,6 +247,7 @@ current_test_subprocess = subprocess.Popen([
   '--expected_lb_policy', '',
   '--enable_srv_queries', 'True',
   '--enable_txt_queries', 'True',
+  '--inject_broken_nameserver_list', 'False',
   '--local_dns_server_address', '127.0.0.1:%d' % args.dns_server_port])
 current_test_subprocess.communicate()
 if current_test_subprocess.returncode != 0:
@@ -253,6 +262,7 @@ current_test_subprocess = subprocess.Popen([
   '--expected_lb_policy', 'round_robin',
   '--enable_srv_queries', 'True',
   '--enable_txt_queries', 'True',
+  '--inject_broken_nameserver_list', 'False',
   '--local_dns_server_address', '127.0.0.1:%d' % args.dns_server_port])
 current_test_subprocess.communicate()
 if current_test_subprocess.returncode != 0:
@@ -267,6 +277,7 @@ current_test_subprocess = subprocess.Popen([
   '--expected_lb_policy', 'round_robin',
   '--enable_srv_queries', 'True',
   '--enable_txt_queries', 'True',
+  '--inject_broken_nameserver_list', 'False',
   '--local_dns_server_address', '127.0.0.1:%d' % args.dns_server_port])
 current_test_subprocess.communicate()
 if current_test_subprocess.returncode != 0:
@@ -281,6 +292,7 @@ current_test_subprocess = subprocess.Popen([
   '--expected_lb_policy', '',
   '--enable_srv_queries', 'True',
   '--enable_txt_queries', 'True',
+  '--inject_broken_nameserver_list', 'False',
   '--local_dns_server_address', '127.0.0.1:%d' % args.dns_server_port])
 current_test_subprocess.communicate()
 if current_test_subprocess.returncode != 0:
@@ -295,6 +307,7 @@ current_test_subprocess = subprocess.Popen([
   '--expected_lb_policy', '',
   '--enable_srv_queries', 'True',
   '--enable_txt_queries', 'True',
+  '--inject_broken_nameserver_list', 'False',
   '--local_dns_server_address', '127.0.0.1:%d' % args.dns_server_port])
 current_test_subprocess.communicate()
 if current_test_subprocess.returncode != 0:
@@ -309,6 +322,7 @@ current_test_subprocess = subprocess.Popen([
   '--expected_lb_policy', '',
   '--enable_srv_queries', 'True',
   '--enable_txt_queries', 'True',
+  '--inject_broken_nameserver_list', 'False',
   '--local_dns_server_address', '127.0.0.1:%d' % args.dns_server_port])
 current_test_subprocess.communicate()
 if current_test_subprocess.returncode != 0:
@@ -323,6 +337,7 @@ current_test_subprocess = subprocess.Popen([
   '--expected_lb_policy', '',
   '--enable_srv_queries', 'False',
   '--enable_txt_queries', 'True',
+  '--inject_broken_nameserver_list', 'False',
   '--local_dns_server_address', '127.0.0.1:%d' % args.dns_server_port])
 current_test_subprocess.communicate()
 if current_test_subprocess.returncode != 0:
@@ -337,6 +352,7 @@ current_test_subprocess = subprocess.Popen([
   '--expected_lb_policy', '',
   '--enable_srv_queries', 'False',
   '--enable_txt_queries', 'True',
+  '--inject_broken_nameserver_list', 'False',
   '--local_dns_server_address', '127.0.0.1:%d' % args.dns_server_port])
 current_test_subprocess.communicate()
 if current_test_subprocess.returncode != 0:
@@ -351,6 +367,7 @@ current_test_subprocess = subprocess.Popen([
   '--expected_lb_policy', '',
   '--enable_srv_queries', 'False',
   '--enable_txt_queries', 'True',
+  '--inject_broken_nameserver_list', 'False',
   '--local_dns_server_address', '127.0.0.1:%d' % args.dns_server_port])
 current_test_subprocess.communicate()
 if current_test_subprocess.returncode != 0:
@@ -365,6 +382,7 @@ current_test_subprocess = subprocess.Popen([
   '--expected_lb_policy', '',
   '--enable_srv_queries', 'False',
   '--enable_txt_queries', 'True',
+  '--inject_broken_nameserver_list', 'False',
   '--local_dns_server_address', '127.0.0.1:%d' % args.dns_server_port])
 current_test_subprocess.communicate()
 if current_test_subprocess.returncode != 0:
@@ -379,6 +397,7 @@ current_test_subprocess = subprocess.Popen([
   '--expected_lb_policy', 'round_robin',
   '--enable_srv_queries', 'False',
   '--enable_txt_queries', 'True',
+  '--inject_broken_nameserver_list', 'False',
   '--local_dns_server_address', '127.0.0.1:%d' % args.dns_server_port])
 current_test_subprocess.communicate()
 if current_test_subprocess.returncode != 0:
@@ -393,6 +412,7 @@ current_test_subprocess = subprocess.Popen([
   '--expected_lb_policy', '',
   '--enable_srv_queries', 'True',
   '--enable_txt_queries', 'False',
+  '--inject_broken_nameserver_list', 'False',
   '--local_dns_server_address', '127.0.0.1:%d' % args.dns_server_port])
 current_test_subprocess.communicate()
 if current_test_subprocess.returncode != 0:
@@ -407,6 +427,7 @@ current_test_subprocess = subprocess.Popen([
   '--expected_lb_policy', '',
   '--enable_srv_queries', 'True',
   '--enable_txt_queries', 'False',
+  '--inject_broken_nameserver_list', 'False',
   '--local_dns_server_address', '127.0.0.1:%d' % args.dns_server_port])
 current_test_subprocess.communicate()
 if current_test_subprocess.returncode != 0:
@@ -421,6 +442,37 @@ current_test_subprocess = subprocess.Popen([
   '--expected_lb_policy', '',
   '--enable_srv_queries', 'True',
   '--enable_txt_queries', 'False',
+  '--inject_broken_nameserver_list', 'False',
+  '--local_dns_server_address', '127.0.0.1:%d' % args.dns_server_port])
+current_test_subprocess.communicate()
+if current_test_subprocess.returncode != 0:
+  num_test_failures += 1
+
+test_runner_log('Run test with target: %s' % 'no-srv-ipv4-single-target-inject-broken-nameservers.resolver-tests-version-4.grpctestingexp.')
+current_test_subprocess = subprocess.Popen([
+  args.test_bin_path,
+  '--target_name', 'no-srv-ipv4-single-target-inject-broken-nameservers.resolver-tests-version-4.grpctestingexp.',
+  '--expected_addrs', '5.5.5.5:443,False',
+  '--expected_chosen_service_config', '',
+  '--expected_lb_policy', '',
+  '--enable_srv_queries', 'True',
+  '--enable_txt_queries', 'True',
+  '--inject_broken_nameserver_list', 'True',
+  '--local_dns_server_address', '127.0.0.1:%d' % args.dns_server_port])
+current_test_subprocess.communicate()
+if current_test_subprocess.returncode != 0:
+  num_test_failures += 1
+
+test_runner_log('Run test with target: %s' % 'ipv4-config-causing-fallback-to-tcp-inject-broken-nameservers.resolver-tests-version-4.grpctestingexp.')
+current_test_subprocess = subprocess.Popen([
+  args.test_bin_path,
+  '--target_name', 'ipv4-config-causing-fallback-to-tcp-inject-broken-nameservers.resolver-tests-version-4.grpctestingexp.',
+  '--expected_addrs', '1.2.3.4:443,False',
+  '--expected_chosen_service_config', '{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwo","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooThree","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooFour","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooFive","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooSix","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooSeven","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooEight","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooNine","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTen","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooEleven","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]}]}',
+  '--expected_lb_policy', '',
+  '--enable_srv_queries', 'True',
+  '--enable_txt_queries', 'True',
+  '--inject_broken_nameserver_list', 'True',
   '--local_dns_server_address', '127.0.0.1:%d' % args.dns_server_port])
 current_test_subprocess.communicate()
 if current_test_subprocess.returncode != 0:

test/cpp/naming/resolver_test_record_groups.yaml

@@ -7,6 +7,7 @@ resolver_component_tests:
   expected_lb_policy: null
   enable_srv_queries: true
   enable_txt_queries: true
+  inject_broken_nameserver_list: false
   record_to_resolve: no-srv-ipv4-single-target
   records:
     no-srv-ipv4-single-target:
@@ -17,6 +18,7 @@ resolver_component_tests:
   expected_lb_policy: null
   enable_srv_queries: true
   enable_txt_queries: true
+  inject_broken_nameserver_list: false
   record_to_resolve: srv-ipv4-single-target
   records:
     _grpclb._tcp.srv-ipv4-single-target:
@@ -31,6 +33,7 @@ resolver_component_tests:
   expected_lb_policy: null
   enable_srv_queries: true
   enable_txt_queries: true
+  inject_broken_nameserver_list: false
   record_to_resolve: srv-ipv4-multi-target
   records:
     _grpclb._tcp.srv-ipv4-multi-target:
@@ -45,6 +48,7 @@ resolver_component_tests:
   expected_lb_policy: null
   enable_srv_queries: true
   enable_txt_queries: true
+  inject_broken_nameserver_list: false
   record_to_resolve: srv-ipv6-single-target
   records:
     _grpclb._tcp.srv-ipv6-single-target:
@@ -59,6 +63,7 @@ resolver_component_tests:
   expected_lb_policy: null
   enable_srv_queries: true
   enable_txt_queries: true
+  inject_broken_nameserver_list: false
   record_to_resolve: srv-ipv6-multi-target
   records:
     _grpclb._tcp.srv-ipv6-multi-target:
@@ -73,6 +78,7 @@ resolver_component_tests:
   expected_lb_policy: round_robin
   enable_srv_queries: true
   enable_txt_queries: true
+  inject_broken_nameserver_list: false
   record_to_resolve: srv-ipv4-simple-service-config
   records:
     _grpclb._tcp.srv-ipv4-simple-service-config:
@@ -88,6 +94,7 @@ resolver_component_tests:
   expected_lb_policy: round_robin
   enable_srv_queries: true
   enable_txt_queries: true
+  inject_broken_nameserver_list: false
   record_to_resolve: ipv4-no-srv-simple-service-config
   records:
     ipv4-no-srv-simple-service-config:
@@ -101,6 +108,7 @@ resolver_component_tests:
   expected_lb_policy: null
   enable_srv_queries: true
   enable_txt_queries: true
+  inject_broken_nameserver_list: false
   record_to_resolve: ipv4-no-config-for-cpp
   records:
     ipv4-no-config-for-cpp:
@@ -114,6 +122,7 @@ resolver_component_tests:
   expected_lb_policy: null
   enable_srv_queries: true
   enable_txt_queries: true
+  inject_broken_nameserver_list: false
   record_to_resolve: ipv4-cpp-config-has-zero-percentage
   records:
     ipv4-cpp-config-has-zero-percentage:
@@ -127,6 +136,7 @@ resolver_component_tests:
   expected_lb_policy: round_robin
   enable_srv_queries: true
   enable_txt_queries: true
+  inject_broken_nameserver_list: false
   record_to_resolve: ipv4-second-language-is-cpp
   records:
     ipv4-second-language-is-cpp:
@@ -140,6 +150,7 @@ resolver_component_tests:
   expected_lb_policy: round_robin
   enable_srv_queries: true
   enable_txt_queries: true
+  inject_broken_nameserver_list: false
   record_to_resolve: ipv4-config-with-percentages
   records:
     ipv4-config-with-percentages:
@@ -154,6 +165,7 @@ resolver_component_tests:
   expected_lb_policy: null
   enable_srv_queries: true
   enable_txt_queries: true
+  inject_broken_nameserver_list: false
   record_to_resolve: srv-ipv4-target-has-backend-and-balancer
   records:
     _grpclb._tcp.srv-ipv4-target-has-backend-and-balancer:
@@ -169,6 +181,7 @@ resolver_component_tests:
   expected_lb_policy: null
   enable_srv_queries: true
   enable_txt_queries: true
+  inject_broken_nameserver_list: false
   record_to_resolve: srv-ipv6-target-has-backend-and-balancer
   records:
     _grpclb._tcp.srv-ipv6-target-has-backend-and-balancer:
@@ -183,6 +196,7 @@ resolver_component_tests:
   expected_lb_policy: null
   enable_srv_queries: true
   enable_txt_queries: true
+  inject_broken_nameserver_list: false
   record_to_resolve: ipv4-config-causing-fallback-to-tcp
   records:
     ipv4-config-causing-fallback-to-tcp:
@@ -197,6 +211,7 @@ resolver_component_tests:
   expected_lb_policy: null
   enable_srv_queries: false
   enable_txt_queries: true
+  inject_broken_nameserver_list: false
   record_to_resolve: srv-ipv4-single-target-srv-disabled
   records:
     _grpclb._tcp.srv-ipv4-single-target-srv-disabled:
@@ -213,6 +228,7 @@ resolver_component_tests:
   expected_lb_policy: null
   enable_srv_queries: false
   enable_txt_queries: true
+  inject_broken_nameserver_list: false
   record_to_resolve: srv-ipv4-multi-target-srv-disabled
   records:
     _grpclb._tcp.srv-ipv4-multi-target-srv-disabled:
@@ -231,6 +247,7 @@ resolver_component_tests:
   expected_lb_policy: null
   enable_srv_queries: false
   enable_txt_queries: true
+  inject_broken_nameserver_list: false
   record_to_resolve: srv-ipv6-single-target-srv-disabled
   records:
     _grpclb._tcp.srv-ipv6-single-target-srv-disabled:
@@ -247,6 +264,7 @@ resolver_component_tests:
   expected_lb_policy: null
   enable_srv_queries: false
   enable_txt_queries: true
+  inject_broken_nameserver_list: false
   record_to_resolve: srv-ipv6-multi-target-srv-disabled
   records:
     _grpclb._tcp.srv-ipv6-multi-target-srv-disabled:
@@ -265,6 +283,7 @@ resolver_component_tests:
   expected_lb_policy: round_robin
   enable_srv_queries: false
   enable_txt_queries: true
+  inject_broken_nameserver_list: false
   record_to_resolve: srv-ipv4-simple-service-config-srv-disabled
   records:
     _grpclb._tcp.srv-ipv4-simple-service-config-srv-disabled:
@@ -282,6 +301,7 @@ resolver_component_tests:
   expected_lb_policy: null
   enable_srv_queries: true
   enable_txt_queries: false
+  inject_broken_nameserver_list: false
   record_to_resolve: srv-ipv4-simple-service-config-txt-disabled
   records:
     _grpclb._tcp.srv-ipv4-simple-service-config-txt-disabled:
@@ -297,6 +317,7 @@ resolver_component_tests:
   expected_lb_policy: null
   enable_srv_queries: true
   enable_txt_queries: false
+  inject_broken_nameserver_list: false
   record_to_resolve: ipv4-cpp-config-has-zero-percentage-txt-disabled
   records:
     ipv4-cpp-config-has-zero-percentage-txt-disabled:
@@ -310,6 +331,7 @@ resolver_component_tests:
   expected_lb_policy: null
   enable_srv_queries: true
   enable_txt_queries: false
+  inject_broken_nameserver_list: false
   record_to_resolve: ipv4-second-language-is-cpp-txt-disabled
   records:
     ipv4-second-language-is-cpp-txt-disabled:
@@ -317,3 +339,29 @@ resolver_component_tests:
     _grpc_config.ipv4-second-language-is-cpp-txt-disabled:
     - {TTL: '2100', data: 'grpc_config=[{"clientLanguage":["go"],"serviceConfig":{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"GoService","waitForReady":true}]}]}},{"clientLanguage":["c++"],"serviceConfig":{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"CppService","waitForReady":true}]}]}}]',
       type: TXT}
+# Tests for which we also exercise the resolver's ability to skip past a broken DNS server in its nameserver list
+- expected_addrs:
+  - {address: '5.5.5.5:443', is_balancer: false}
+  expected_chosen_service_config: null
+  expected_lb_policy: null
+  enable_srv_queries: true
+  enable_txt_queries: true
+  inject_broken_nameserver_list: true
+  record_to_resolve: no-srv-ipv4-single-target-inject-broken-nameservers
+  records:
+    no-srv-ipv4-single-target-inject-broken-nameservers:
+    - {TTL: '2100', data: 5.5.5.5, type: A}
+- expected_addrs:
+  - {address: '1.2.3.4:443', is_balancer: false}
+  expected_chosen_service_config: '{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwo","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooThree","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooFour","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooFive","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooSix","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooSeven","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooEight","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooNine","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTen","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooEleven","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]}]}'
+  expected_lb_policy: null
+  enable_srv_queries: true
+  enable_txt_queries: true
+  inject_broken_nameserver_list: true
+  record_to_resolve: ipv4-config-causing-fallback-to-tcp-inject-broken-nameservers
+  records:
+    ipv4-config-causing-fallback-to-tcp-inject-broken-nameservers:
+    - {TTL: '2100', data: 1.2.3.4, type: A}
+    _grpc_config.ipv4-config-causing-fallback-to-tcp-inject-broken-nameservers:
+    - {TTL: '2100', data: 'grpc_config=[{"serviceConfig":{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwo","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooThree","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooFour","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooFive","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooSix","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooSeven","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooEight","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooNine","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTen","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooEleven","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]}]}}]',
+      type: TXT}

tools/run_tests/generated/sources_and_headers.json

@@ -6014,6 +6014,7 @@
   }, 
   {
     "deps": [
+      "dns_test_util", 
       "gpr", 
       "grpc++_test_config", 
       "grpc++_test_util_unsecure", 
@@ -6033,6 +6034,7 @@
   }, 
   {
     "deps": [
+      "dns_test_util", 
       "gpr", 
       "grpc", 
       "grpc++", 
@@ -6128,6 +6130,7 @@
   }, 
   {
     "deps": [
+      "dns_test_util", 
       "gpr", 
       "grpc", 
       "grpc++", 
@@ -6603,6 +6606,21 @@
     "third_party": false, 
     "type": "lib"
   }, 
+  {
+    "deps": [], 
+    "headers": [
+      "test/cpp/naming/dns_test_util.h"
+    ], 
+    "is_filegroup": false, 
+    "language": "c++", 
+    "name": "dns_test_util", 
+    "src": [
+      "test/cpp/naming/dns_test_util.cc", 
+      "test/cpp/naming/dns_test_util.h"
+    ], 
+    "third_party": false, 
+    "type": "lib"
+  }, 
   {
     "deps": [
       "gpr",