Merge branch 'master' into node_perf_tests_again

.clang-tidy

@@ -0,0 +1,6 @@
+---
+Checks: 'modernize-use-nullptr,google-build-namespaces,google-build-explicit-make-pair,readability-function-size,performance-*'
+WarningsAsErrors: 'modernize-use-nullptr,google-build-namespaces,google-build-explicit-make-pair,readability-function-size,performance-*'
+CheckOptions:
+  - key:    readability-function-size.StatementThreshold
+    value:  '450'

.github/ISSUE_TEMPLATE.md

@@ -1,12 +1,12 @@
-Please answer these questions before submitting your issue. 
- 
-### Should this be an issue in the gRPC issue tracker?
- 
-Create new issues for bugs and feature requests. An issue needs to be actionable. General gRPC discussions and usage questions belong to:
-- [grpc.io mailing list](https://groups.google.com/forum/#!forum/grpc-io)
-- [StackOverflow, with `grpc` tag](http://stackoverflow.com/questions/tagged/grpc)
- 
-*Please don't double post your questions in more locations; we are monitoring both channels, and the time spent de-duplicating questions is better spent answering more user questions.*
+<!--
+
+This form is for bug reports and feature requests ONLY!
+For general questions and troubleshooting, please ask/look for answers here:
+- grpc.io mailing list: https://groups.google.com/forum/#!forum/grpc-io
+- StackOverflow, with "grpc" tag: http://stackoverflow.com/questions/tagged/grpc
+
+Issues specific to *grpc-java*, *grpc-go*, *grpc-node*, *grpc-dart*, *grpc-web* should be created in the repository they belong to (e.g. https://github.com/grpc/grpc-LANGUAGE/issues/new)
+-->
  
 ### What version of gRPC and what language are you using?
  

.pylintrc

@@ -72,6 +72,13 @@ disable=
 	# TODO(https://github.com/grpc/grpc/issues/261): Maybe we could have
 	# this one if we extracted just a few more helper functions...
 	too-many-nested-blocks,
-	# NOTE(nathaniel): I have disputed the premise of this inspection from
-	# the beginning and will continue to do so until it goes away for good.
+	# TODO(https://github.com/grpc/grpc/issues/261): Disable unnecessary
+	# super-init requirement for abstract class implementations for now.
+	super-init-not-called,
+	# NOTE(nathaniel): A single statement that always returns program
+	# control is better than two statements the first of which sometimes
+	# returns program control and the second of which always returns
+	# program control. Probably generally, but definitely in the cases of
+	# if:/else: and for:/else:.
 	useless-else-on-loop,
+	no-else-return,

.pylintrc-tests

@@ -103,6 +103,13 @@ disable=
 	# TODO(https://github.com/grpc/grpc/issues/261): Maybe we could have
 	# this one if we extracted just a few more helper functions...
 	too-many-nested-blocks,
-	# NOTE(nathaniel): I have disputed the premise of this inspection from
-	# the beginning and will continue to do so until it goes away for good.
+	# TODO(https://github.com/grpc/grpc/issues/261): Disable unnecessary
+	# super-init requirement for abstract class implementations for now.
+	super-init-not-called,
+	# NOTE(nathaniel): A single statement that always returns program
+	# control is better than two statements the first of which sometimes
+	# returns program control and the second of which always returns
+	# program control. Probably generally, but definitely in the cases of
+	# if:/else: and for:/else:.
 	useless-else-on-loop,
+	no-else-return,

BUILD

@@ -64,11 +64,11 @@ config_setting(
 )
 
 # This should be updated along with build.yaml
-g_stands_for = "glorious"
+g_stands_for = "gladiolus"
 
 core_version = "6.0.0-dev"
 
-version = "1.12.0-dev"
+version = "1.14.0-dev"
 
 GPR_PUBLIC_HDRS = [
     "include/grpc/support/alloc.h",
@@ -485,10 +485,7 @@ grpc_cc_library(
 grpc_cc_library(
     name = "census",
     srcs = [
-        "src/core/ext/census/grpc_context.cc",
-    ],
-    external_deps = [
-        "nanopb",
+        "src/cpp/ext/filters/census/grpc_context.cc",
     ],
     language = "c++",
     public_hdrs = [
@@ -512,7 +509,6 @@ grpc_cc_library(
         "src/core/lib/gpr/env_linux.cc",
         "src/core/lib/gpr/env_posix.cc",
         "src/core/lib/gpr/env_windows.cc",
-        "src/core/lib/gpr/fork.cc",
         "src/core/lib/gpr/host_port.cc",
         "src/core/lib/gpr/log.cc",
         "src/core/lib/gpr/log_android.cc",
@@ -537,6 +533,7 @@ grpc_cc_library(
         "src/core/lib/gpr/tmpfile_posix.cc",
         "src/core/lib/gpr/tmpfile_windows.cc",
         "src/core/lib/gpr/wrap_memcpy.cc",
+        "src/core/lib/gprpp/fork.cc",
         "src/core/lib/gprpp/thd_posix.cc",
         "src/core/lib/gprpp/thd_windows.cc",
         "src/core/lib/profiling/basic_timers.cc",
@@ -545,7 +542,6 @@ grpc_cc_library(
     hdrs = [
         "src/core/lib/gpr/arena.h",
         "src/core/lib/gpr/env.h",
-        "src/core/lib/gpr/fork.h",
         "src/core/lib/gpr/host_port.h",
         "src/core/lib/gpr/mpscq.h",
         "src/core/lib/gpr/murmur_hash.h",
@@ -560,6 +556,7 @@ grpc_cc_library(
         "src/core/lib/gpr/tmpfile.h",
         "src/core/lib/gpr/useful.h",
         "src/core/lib/gprpp/abstract.h",
+        "src/core/lib/gprpp/fork.h",
         "src/core/lib/gprpp/manual_constructor.h",
         "src/core/lib/gprpp/memory.h",
         "src/core/lib/gprpp/thd.h",
@@ -678,7 +675,7 @@ grpc_cc_library(
         "src/core/lib/channel/channel_stack.cc",
         "src/core/lib/channel/channel_stack_builder.cc",
         "src/core/lib/channel/channel_trace.cc",
-        "src/core/lib/channel/channel_trace_registry.cc",
+        "src/core/lib/channel/channelz_registry.cc",
         "src/core/lib/channel/connected_channel.cc",
         "src/core/lib/channel/handshaker.cc",
         "src/core/lib/channel/handshaker_factory.cc",
@@ -825,7 +822,7 @@ grpc_cc_library(
         "src/core/lib/channel/channel_stack.h",
         "src/core/lib/channel/channel_stack_builder.h",
         "src/core/lib/channel/channel_trace.h",
-        "src/core/lib/channel/channel_trace_registry.h",
+        "src/core/lib/channel/channelz_registry.h",
         "src/core/lib/channel/connected_channel.h",
         "src/core/lib/channel/context.h",
         "src/core/lib/channel/handshaker.h",
@@ -1003,6 +1000,25 @@ grpc_cc_library(
     ],
 )
 
+grpc_cc_library(
+    name = "grpc_cfstream",
+    srcs = [
+        "src/core/lib/iomgr/cfstream_handle.cc",
+        "src/core/lib/iomgr/endpoint_cfstream.cc",
+        "src/core/lib/iomgr/error_cfstream.cc",
+        "src/core/lib/iomgr/tcp_client_cfstream.cc",
+    ],
+    hdrs = [
+        "src/core/lib/iomgr/cfstream_handle.h",
+        "src/core/lib/iomgr/endpoint_cfstream.h",
+        "src/core/lib/iomgr/error_cfstream.h",
+    ],
+    deps = [
+        ":gpr_base",
+        ":grpc_base",
+    ],
+)
+
 grpc_cc_library(
     name = "grpc_client_channel",
     srcs = [
@@ -1183,6 +1199,7 @@ grpc_cc_library(
     ],
     hdrs = [
         "src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.h",
+        "src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.h",
         "src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_channel.h",
         "src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.h",
         "src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.h",
@@ -1211,6 +1228,7 @@ grpc_cc_library(
     ],
     hdrs = [
         "src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.h",
+        "src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.h",
         "src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_channel.h",
         "src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.h",
         "src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.h",
@@ -1230,9 +1248,6 @@ grpc_cc_library(
 
 grpc_cc_library(
     name = "grpc_lb_subchannel_list",
-    srcs = [
-        "src/core/ext/filters/client_channel/lb_policy/subchannel_list.cc",
-    ],
     hdrs = [
         "src/core/ext/filters/client_channel/lb_policy/subchannel_list.h",
     ],
@@ -1285,6 +1300,20 @@ grpc_cc_library(
     ],
 )
 
+grpc_cc_library(
+    name = "lb_load_data_store",
+    srcs = [
+        "src/cpp/server/load_reporter/load_data_store.cc",
+    ],
+    hdrs = [
+        "src/cpp/server/load_reporter/load_data_store.h",
+    ],
+    language = "c++",
+    deps = [
+        "grpc++",
+    ],
+)
+
 grpc_cc_library(
     name = "grpc_resolver_dns_native",
     srcs = [
@@ -1301,6 +1330,7 @@ grpc_cc_library(
     name = "grpc_resolver_dns_ares",
     srcs = [
         "src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc",
+        "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.cc",
         "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc",
         "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc",
         "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_fallback.cc",
@@ -1375,6 +1405,7 @@ grpc_cc_library(
         "src/core/lib/surface/init_secure.cc",
     ],
     hdrs = [
+        "src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.h",
         "src/core/lib/security/context/security_context.h",
         "src/core/lib/security/credentials/alts/alts_credentials.h",
         "src/core/lib/security/credentials/composite/composite_credentials.h",
@@ -1611,11 +1642,9 @@ grpc_cc_library(
     name = "tsi_interface",
     srcs = [
         "src/core/tsi/transport_security.cc",
-        "src/core/tsi/transport_security_adapter.cc",
     ],
     hdrs = [
         "src/core/tsi/transport_security.h",
-        "src/core/tsi/transport_security_adapter.h",
         "src/core/tsi/transport_security_interface.h",
     ],
     language = "c++",
@@ -1958,4 +1987,40 @@ grpc_cc_library(
     ],
 )
 
+grpc_cc_library(
+    name = "grpc_opencensus_plugin",
+    srcs = [
+        "src/cpp/ext/filters/census/client_filter.cc",
+        "src/cpp/ext/filters/census/server_filter.cc",
+        "src/cpp/ext/filters/census/channel_filter.cc",
+        "src/cpp/ext/filters/census/context.cc",
+        "src/cpp/ext/filters/census/grpc_context.cc",
+        "src/cpp/ext/filters/census/grpc_plugin.cc",
+        "src/cpp/ext/filters/census/measures.cc",
+        "src/cpp/ext/filters/census/rpc_encoding.cc",
+        "src/cpp/ext/filters/census/views.cc",
+    ],
+    hdrs = [
+        "include/grpcpp/opencensus.h",
+        "src/cpp/ext/filters/census/client_filter.h",
+        "src/cpp/ext/filters/census/server_filter.h",
+        "src/cpp/ext/filters/census/channel_filter.h",
+        "src/cpp/ext/filters/census/context.h",
+        "src/cpp/ext/filters/census/grpc_plugin.h",
+        "src/cpp/ext/filters/census/measures.h",
+        "src/cpp/ext/filters/census/rpc_encoding.h",
+    ],
+    language = "c++",
+    external_deps = [
+        "absl-base",
+        "absl-time",
+        "opencensus-trace",
+        "opencensus-stats",
+    ],
+    deps = [
+        ":census",
+        ":grpc++",
+    ],
+)
+
 grpc_generate_one_off_targets()

INSTALL.md

@@ -39,8 +39,8 @@ and then run the following command from a terminal:
  $ [sudo] xcode-select --install
 ```
 
-To build gRPC from source, you may also need to install the following
-packages, which you can get from [Homebrew](https://brew.sh):
+To build gRPC from source, you may need to install the following
+packages from [Homebrew](https://brew.sh):
 
 ```sh
  $ brew install autoconf automake libtool shtool
@@ -66,11 +66,11 @@ By default gRPC uses [protocol buffers](https://github.com/google/protobuf),
 you will need the `protoc` compiler to generate stub server and client code.
 
 If you compile gRPC from source, as described below, the Makefile will
-automatically try and compile the `protoc` in third_party if you cloned the
-repository recursively and it detects that you don't already have it
+automatically try compiling the `protoc` in third_party if you cloned the
+repository recursively and it detects that you do not already have 'protoc' compiler
 installed.
 
-If it hasn't been installed, you can run the following commands to install it.
+If 'protoc' compiler has not been installed, following commands can be used for installation.
 
 ```sh
 $ cd grpc/third_party/protobuf
@@ -79,7 +79,7 @@ $ sudo make install   # 'make' should have been run by core grpc
 
 # Build from Source
 
-For developers who are interested to contribute, here is how to compile the
+For developers who are interested to contribute, the following commands show how to compile the
 gRPC C Core library.
 
 ```sh
@@ -123,7 +123,7 @@ When using the "Visual Studio" generator,
 cmake will generate a solution (`grpc.sln`) that contains a VS project for 
 every target defined in `CMakeLists.txt` (+ few extra convenience projects
 added automatically by cmake). After opening the solution with Visual Studio 
-you will be able to browse and build the code as usual.
+you will be able to browse and build the code.
 ```
 > @rem Run from grpc directory after cloning the repo with --recursive or updating submodules.
 > md .build
@@ -133,7 +133,7 @@ you will be able to browse and build the code as usual.
 ```
 
 #### cmake: Using Ninja (faster build, supports boringssl's assembly optimizations).
-Please note that when using Ninja, you'll still need Visual C++ (part of Visual Studio)
+Please note that when using Ninja, you will still need Visual C++ (part of Visual Studio)
 installed to be able to compile the C/C++ sources.
 ```
 > @rem Run from grpc directory after cloning the repo with --recursive or updating submodules.
@@ -142,13 +142,12 @@ installed to be able to compile the C/C++ sources.
 > call "%VS140COMNTOOLS%..\..\VC\vcvarsall.bat" x64
 > cmake .. -GNinja -DCMAKE_BUILD_TYPE=Release
 > cmake --build .
-> ninja install
 ```
 
 ### msys2 (with mingw)
 
 The Makefile (and source code) should support msys2's mingw32 and mingw64
-compilers. Building with msys2's native compiler is also possible, but
+compilers. Building with msys2's native compiler is possible, but
 difficult.
 
 This approach requires having [msys2](https://msys2.github.io/) installed.
@@ -165,7 +164,7 @@ MINGW64$ export CPPFLAGS="-D_WIN32_WINNT=0x0600"
 MINGW64$ make
 ```
 
-NOTE: While most of the make targets are buildable under Mingw, some haven't been ported to Windows yet
+NOTE: Though most of the make targets are buildable under Mingw, some haven't been ported to Windows yet
 and may fail to build (mostly trying to include POSIX headers not available on Mingw).
 
 ### Pre-generated Visual Studio solution (DELETED)

README.md

@@ -39,6 +39,7 @@ Libraries in different languages may be in different states of development. We a
 | Java                    | [grpc-java](http://github.com/grpc/grpc-java)        |
 | Go                      | [grpc-go](http://github.com/grpc/grpc-go)            |
 | NodeJS                  | [grpc-node](https://github.com/grpc/grpc-node)       |
+| WebJS                   | [grpc-web](https://github.com/grpc/grpc-web)         |
 | Dart                    | [grpc-dart](https://github.com/grpc/grpc-dart)       |
 
 See [MANIFEST.md](MANIFEST.md) for a listing of top-level items in the

bazel/cc_grpc_library.bzl

@@ -43,12 +43,7 @@ def cc_grpc_library(name, srcs, deps, proto_only, well_known_protos, generate_mo
   )
 
   if not proto_only:
-    if use_external:
-      # when this file is used by non-grpc projects
-      plugin = "//external:grpc_cpp_plugin"
-    else:
-      plugin = "//:grpc_cpp_plugin"
-
+    plugin = "@com_github_grpc_grpc//:grpc_cpp_plugin"
     generate_cc(
         name = codegen_grpc_target,
         srcs = [proto_target],
@@ -57,14 +52,8 @@ def cc_grpc_library(name, srcs, deps, proto_only, well_known_protos, generate_mo
         generate_mocks = generate_mocks,
         **kwargs
     )
-
-    if use_external:
-      # when this file is used by non-grpc projects
-      grpc_deps = ["//external:grpc++_codegen_proto",
-                   "//external:protobuf"]
-    else:
-      grpc_deps = ["//:grpc++_codegen_proto", "//external:protobuf"]
-
+    grpc_deps  = ["@com_github_grpc_grpc//:grpc++_codegen_proto",
+                  "//external:protobuf"]
     native.cc_library(
         name = name,
         srcs = [":" + codegen_grpc_target, ":" + codegen_target],

bazel/grpc_build_system.bzl

@@ -36,9 +36,7 @@ def if_not_windows(a):
 def _get_external_deps(external_deps):
   ret = []
   for dep in external_deps:
-    if dep == "nanopb":
-      ret += ["grpc_nanopb"]
-    elif dep == "address_sorting":
+    if dep == "address_sorting":
       ret += ["//third_party/address_sorting"]
     elif dep == "cares":
       ret += select({"//:grpc_no_ares": [],
@@ -62,7 +60,7 @@ def _maybe_update_cc_library_hdrs(hdrs):
 def grpc_cc_library(name, srcs = [], public_hdrs = [], hdrs = [],
                     external_deps = [], deps = [], standalone = False,
                     language = "C++", testonly = False, visibility = None,
-                    alwayslink = 0):
+                    alwayslink = 0, data = []):
   copts = []
   if language.upper() == "C":
     copts = if_not_windows(["-std=c99"])
@@ -87,6 +85,7 @@ def grpc_cc_library(name, srcs = [], public_hdrs = [], hdrs = [],
         "include"
     ],
     alwayslink = alwayslink,
+    data = data,
   )
 
 def grpc_proto_plugin(name, srcs = [], deps = []):
@@ -110,7 +109,7 @@ def grpc_proto_library(name, srcs = [], deps = [], well_known_protos = False,
     generate_mocks = generate_mocks,
   )
 
-def grpc_cc_test(name, srcs = [], deps = [], external_deps = [], args = [], data = [], uses_polling = True, language = "C++", size = "medium", timeout = "moderate"):
+def grpc_cc_test(name, srcs = [], deps = [], external_deps = [], args = [], data = [], uses_polling = True, language = "C++", size = "medium", timeout = "moderate", tags = []):
   copts = []
   if language.upper() == "C":
     copts = if_not_windows(["-std=c99"])
@@ -140,6 +139,7 @@ def grpc_cc_test(name, srcs = [], deps = [], external_deps = [], args = [], data
           poller,
           '$(location %s)' % name,
         ] + args['args'],
+        tags = tags,
       )
   else:
     native.cc_test(**args)
@@ -160,24 +160,7 @@ def grpc_cc_binary(name, srcs = [], deps = [], external_deps = [], args = [], da
     linkopts = if_not_windows(["-pthread"]) + linkopts,
   )
 
-def grpc_generate_one_off_targets():
-  native.cc_library(
-    name = "grpc_nanopb",
-    hdrs = [
-      "//third_party/nanopb:pb.h",
-      "//third_party/nanopb:pb_common.h",
-      "//third_party/nanopb:pb_decode.h",
-      "//third_party/nanopb:pb_encode.h",
-    ],
-    srcs = [
-      "//third_party/nanopb:pb_common.c",
-      "//third_party/nanopb:pb_decode.c",
-      "//third_party/nanopb:pb_encode.c",
-    ],
-    defines = [
-      "PB_FIELD_16BIT=1",
-    ],
-  )
+def grpc_generate_one_off_targets(): pass
 
 def grpc_sh_test(name, srcs, args = [], data = []):
   native.sh_test(

bazel/grpc_deps.bzl

@@ -2,6 +2,22 @@
 
 def grpc_deps():
     """Loads dependencies need to compile and test the grpc library."""
+
+    native.bind(
+        name = "nanopb",
+        actual = "@com_github_nanopb_nanopb//:nanopb",
+    )
+
+    native.bind(
+        name = "absl-base",
+        actual = "@com_google_absl//absl/base",
+    )
+
+    native.bind(
+        name = "absl-time",
+        actual = "@com_google_absl//absl/time:time",
+    )
+
     native.bind(
         name = "libssl",
         actual = "@boringssl//:ssl",
@@ -59,19 +75,34 @@ def grpc_deps():
 
     native.bind(
         name = "grpc_cpp_plugin",
-        actual = "@com_github_grpc_grpc//:grpc_cpp_plugin"
+        actual = "@com_github_grpc_grpc//:grpc_cpp_plugin",
     )
 
     native.bind(
         name = "grpc++_codegen_proto",
-        actual = "@com_github_grpc_grpc//:grpc++_codegen_proto"
+        actual = "@com_github_grpc_grpc//:grpc++_codegen_proto",
+    )
+
+    native.bind(
+        name = "opencensus-trace",
+        actual = "@io_opencensus_cpp//opencensus/trace:trace"
+    )
+
+    native.bind(
+        name = "opencensus-stats",
+        actual = "@io_opencensus_cpp//opencensus/stats:stats"
+    )
+
+    native.bind(
+        name = "opencensus-stats-test",
+        actual = "@io_opencensus_cpp//opencensus/stats:test_utils"
     )
 
     if "boringssl" not in native.existing_rules():
         native.http_archive(
             name = "boringssl",
-            # on the master-with-bazel branch
-            url = "https://boringssl.googlesource.com/boringssl/+archive/886e7d75368e3f4fab3f4d0d3584e4abfc557755.tar.gz",
+            # on the chromium-stable-with-bazel branch
+            url = "https://boringssl.googlesource.com/boringssl/+archive/dcd3e6e6ecddf059adb48fca45bc7346a108bdd9.tar.gz",
         )
 
     if "com_github_madler_zlib" not in native.existing_rules():
@@ -87,6 +118,14 @@ def grpc_deps():
             name = "com_google_protobuf",
             strip_prefix = "protobuf-b5fbb742af122b565925987e65c08957739976a7",
             url = "https://github.com/google/protobuf/archive/b5fbb742af122b565925987e65c08957739976a7.tar.gz",
+            )
+
+    if "com_github_nanopb_nanopb" not in native.existing_rules():
+        native.new_http_archive(
+            name = "com_github_nanopb_nanopb",
+            build_file = "@com_github_grpc_grpc//third_party:nanopb.BUILD",
+            strip_prefix = "nanopb-f8ac463766281625ad710900479130c7fcb4d63b",
+            url = "https://github.com/nanopb/nanopb/archive/f8ac463766281625ad710900479130c7fcb4d63b.tar.gz",
         )
 
     if "com_github_google_googletest" not in native.existing_rules():
@@ -108,8 +147,8 @@ def grpc_deps():
         native.new_http_archive(
             name = "com_github_google_benchmark",
             build_file = "@com_github_grpc_grpc//third_party:benchmark.BUILD",
-            strip_prefix = "benchmark-5b7683f49e1e9223cf9927b24f6fd3d6bd82e3f8",
-            url = "https://github.com/google/benchmark/archive/5b7683f49e1e9223cf9927b24f6fd3d6bd82e3f8.tar.gz",
+            strip_prefix = "benchmark-9913418d323e64a0111ca0da81388260c2bbe1e9",
+            url = "https://github.com/google/benchmark/archive/9913418d323e64a0111ca0da81388260c2bbe1e9.tar.gz",
         )
 
     if "com_github_cares_cares" not in native.existing_rules():
@@ -123,21 +162,29 @@ def grpc_deps():
     if "com_google_absl" not in native.existing_rules():
         native.http_archive(
             name = "com_google_absl",
-            strip_prefix = "abseil-cpp-cc4bed2d74f7c8717e31f9579214ab52a9c9c610",
-            url = "https://github.com/abseil/abseil-cpp/archive/cc4bed2d74f7c8717e31f9579214ab52a9c9c610.tar.gz",
+            strip_prefix = "abseil-cpp-cd95e71df6eaf8f2a282b1da556c2cf1c9b09207",
+            url = "https://github.com/abseil/abseil-cpp/archive/cd95e71df6eaf8f2a282b1da556c2cf1c9b09207.tar.gz",
         )
 
     if "com_github_bazelbuild_bazeltoolchains" not in native.existing_rules():
         native.http_archive(
             name = "com_github_bazelbuild_bazeltoolchains",
-            strip_prefix = "bazel-toolchains-44200e0c026d86c53470d107b3697a3e46469c43",
+            strip_prefix = "bazel-toolchains-4653c01284d8a4a536f8f9bb47b7d10f94c549e7",
             urls = [
-                "https://mirror.bazel.build/github.com/bazelbuild/bazel-toolchains/archive/44200e0c026d86c53470d107b3697a3e46469c43.tar.gz",
-                "https://github.com/bazelbuild/bazel-toolchains/archive/44200e0c026d86c53470d107b3697a3e46469c43.tar.gz",
+                "https://mirror.bazel.build/github.com/bazelbuild/bazel-toolchains/archive/4653c01284d8a4a536f8f9bb47b7d10f94c549e7.tar.gz",
+                "https://github.com/bazelbuild/bazel-toolchains/archive/4653c01284d8a4a536f8f9bb47b7d10f94c549e7.tar.gz",
             ],
-            sha256 = "699b55a6916c687f4b7dc092dbbf5f64672cde0dc965f79717735ec4e5416556",
+            sha256 = "1c4a532b396c698e6467a1548554571cb85fa091e472b05e398ebc836c315d77",
         )
 
+    if "io_opencensus_cpp" not in native.existing_rules():
+      native.http_archive(
+            name = "io_opencensus_cpp",
+            strip_prefix = "opencensus-cpp-fdf0f308b1631bb4a942e32ba5d22536a6170274",
+            url = "https://github.com/census-instrumentation/opencensus-cpp/archive/fdf0f308b1631bb4a942e32ba5d22536a6170274.tar.gz",
+        )
+
+
 # TODO: move some dependencies from "grpc_deps" here?
 def grpc_test_only_deps():
     """Internal, not intended for use by packages that are consuming grpc.

build.yaml

@@ -13,8 +13,8 @@ settings:
   '#09': Per-language overrides are possible with (eg) ruby_version tag here
   '#10': See the expand_version.py for all the quirks here
   core_version: 6.0.0-dev
-  g_stands_for: glorious
-  version: 1.12.0-dev
+  g_stands_for: gladiolus
+  version: 1.14.0-dev
 filegroups:
 - name: alts_proto
   headers:
@@ -100,10 +100,9 @@ filegroups:
   public_headers:
   - include/grpc/census.h
   src:
-  - src/core/ext/census/grpc_context.cc
+  - src/cpp/ext/filters/census/grpc_context.cc
   uses:
   - grpc_base
-  - nanopb
 - name: cmdline
   headers:
   - test/core/util/cmdline.h
@@ -123,7 +122,6 @@ filegroups:
   - src/core/lib/gpr/env_linux.cc
   - src/core/lib/gpr/env_posix.cc
   - src/core/lib/gpr/env_windows.cc
-  - src/core/lib/gpr/fork.cc
   - src/core/lib/gpr/host_port.cc
   - src/core/lib/gpr/log.cc
   - src/core/lib/gpr/log_android.cc
@@ -148,6 +146,7 @@ filegroups:
   - src/core/lib/gpr/tmpfile_posix.cc
   - src/core/lib/gpr/tmpfile_windows.cc
   - src/core/lib/gpr/wrap_memcpy.cc
+  - src/core/lib/gprpp/fork.cc
   - src/core/lib/gprpp/thd_posix.cc
   - src/core/lib/gprpp/thd_windows.cc
   - src/core/lib/profiling/basic_timers.cc
@@ -176,7 +175,6 @@ filegroups:
   headers:
   - src/core/lib/gpr/arena.h
   - src/core/lib/gpr/env.h
-  - src/core/lib/gpr/fork.h
   - src/core/lib/gpr/host_port.h
   - src/core/lib/gpr/mpscq.h
   - src/core/lib/gpr/murmur_hash.h
@@ -194,6 +192,7 @@ filegroups:
   - src/core/lib/gprpp/atomic.h
   - src/core/lib/gprpp/atomic_with_atm.h
   - src/core/lib/gprpp/atomic_with_std.h
+  - src/core/lib/gprpp/fork.h
   - src/core/lib/gprpp/manual_constructor.h
   - src/core/lib/gprpp/memory.h
   - src/core/lib/gprpp/thd.h
@@ -235,7 +234,7 @@ filegroups:
   - src/core/lib/channel/channel_stack.cc
   - src/core/lib/channel/channel_stack_builder.cc
   - src/core/lib/channel/channel_trace.cc
-  - src/core/lib/channel/channel_trace_registry.cc
+  - src/core/lib/channel/channelz_registry.cc
   - src/core/lib/channel/connected_channel.cc
   - src/core/lib/channel/handshaker.cc
   - src/core/lib/channel/handshaker_factory.cc
@@ -405,7 +404,7 @@ filegroups:
   - src/core/lib/channel/channel_stack.h
   - src/core/lib/channel/channel_stack_builder.h
   - src/core/lib/channel/channel_trace.h
-  - src/core/lib/channel/channel_trace_registry.h
+  - src/core/lib/channel/channelz_registry.h
   - src/core/lib/channel/connected_channel.h
   - src/core/lib/channel/context.h
   - src/core/lib/channel/handshaker.h
@@ -536,6 +535,19 @@ filegroups:
   uses:
   - grpc_codegen
   - grpc_trace_headers
+- name: grpc_cfstream
+  headers:
+  - src/core/lib/iomgr/cfstream_handle.h
+  - src/core/lib/iomgr/endpoint_cfstream.h
+  - src/core/lib/iomgr/error_cfstream.h
+  src:
+  - src/core/lib/iomgr/cfstream_handle.cc
+  - src/core/lib/iomgr/endpoint_cfstream.cc
+  - src/core/lib/iomgr/error_cfstream.cc
+  - src/core/lib/iomgr/tcp_client_cfstream.cc
+  uses:
+  - grpc_base_headers
+  - gpr_base_headers
 - name: grpc_client_authority_filter
   headers:
   - src/core/ext/filters/http/client_authority_filter.h
@@ -628,6 +640,7 @@ filegroups:
 - name: grpc_lb_policy_grpclb
   headers:
   - src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.h
+  - src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.h
   - src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_channel.h
   - src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.h
   - src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.h
@@ -648,6 +661,7 @@ filegroups:
 - name: grpc_lb_policy_grpclb_secure
   headers:
   - src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.h
+  - src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.h
   - src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_channel.h
   - src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.h
   - src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.h
@@ -685,8 +699,6 @@ filegroups:
 - name: grpc_lb_subchannel_list
   headers:
   - src/core/ext/filters/client_channel/lb_policy/subchannel_list.h
-  src:
-  - src/core/ext/filters/client_channel/lb_policy/subchannel_list.cc
   uses:
   - grpc_base
   - grpc_client_channel
@@ -712,6 +724,7 @@ filegroups:
   - src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h
   src:
   - src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc
+  - src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.cc
   - src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc
   - src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc
   - src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_fallback.cc
@@ -746,6 +759,7 @@ filegroups:
   public_headers:
   - include/grpc/grpc_security.h
   headers:
+  - src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.h
   - src/core/lib/security/context/security_context.h
   - src/core/lib/security/credentials/alts/alts_credentials.h
   - src/core/lib/security/credentials/composite/composite_credentials.h
@@ -826,6 +840,7 @@ filegroups:
   - test/core/end2end/fixtures/proxy.h
   - test/core/iomgr/endpoint_tests.h
   - test/core/util/debugger_macros.h
+  - test/core/util/fuzzer_util.h
   - test/core/util/grpc_profiler.h
   - test/core/util/histogram.h
   - test/core/util/memory_counters.h
@@ -845,6 +860,7 @@ filegroups:
   - test/core/end2end/fixtures/proxy.cc
   - test/core/iomgr/endpoint_tests.cc
   - test/core/util/debugger_macros.cc
+  - test/core/util/fuzzer_util.cc
   - test/core/util/grpc_profiler.cc
   - test/core/util/histogram.cc
   - test/core/util/memory_counters.cc
@@ -1077,11 +1093,9 @@ filegroups:
 - name: tsi_interface
   headers:
   - src/core/tsi/transport_security.h
-  - src/core/tsi/transport_security_adapter.h
   - src/core/tsi/transport_security_interface.h
   src:
   - src/core/tsi/transport_security.cc
-  - src/core/tsi/transport_security_adapter.cc
   deps:
   - gpr
   secure: true
@@ -1890,6 +1904,15 @@ libs:
   - test/cpp/interop/interop_server_bootstrap.cc
   deps:
   - interop_server_lib
+- name: lb_load_data_store
+  build: private
+  language: c++
+  headers:
+  - src/cpp/server/load_reporter/load_data_store.h
+  src:
+  - src/cpp/server/load_reporter/load_data_store.cc
+  deps:
+  - grpc++
 - name: qps
   build: private
   language: c++
@@ -1981,6 +2004,19 @@ targets:
   - grpc
   - gpr_test_util
   - gpr
+- name: alts_credentials_fuzzer
+  build: fuzzer
+  language: c
+  src:
+  - test/core/security/alts_credentials_fuzzer.cc
+  deps:
+  - grpc_test_util
+  - grpc
+  - gpr_test_util
+  - gpr
+  corpus_dirs:
+  - test/core/security/corpus/alts_credentials_corpus
+  maxlen: 2048
 - name: api_fuzzer
   build: fuzzer
   language: c
@@ -1996,6 +2032,7 @@ targets:
   dict: test/core/end2end/fuzzers/api_fuzzer.dictionary
   maxlen: 2048
 - name: arena_test
+  cpu_cost: 10
   build: test
   language: c
   src:
@@ -2364,6 +2401,18 @@ targets:
   - mac
   - linux
   - posix
+- name: fork_test
+  build: test
+  language: c
+  src:
+  - test/core/gprpp/fork_test.cc
+  deps:
+  - gpr_test_util
+  - gpr
+  platforms:
+  - mac
+  - linux
+  uses_polling: false
 - name: goaway_server_test
   cpu_cost: 0.1
   build: test
@@ -2743,6 +2792,21 @@ targets:
   platforms:
   - linux
   secure: true
+- name: handshake_verify_peer_options
+  build: test
+  language: c
+  src:
+  - test/core/handshake/verify_peer_options.cc
+  deps:
+  - grpc_test_util
+  - grpc
+  - gpr_test_util
+  - gpr
+  exclude_iomgrs:
+  - uv
+  platforms:
+  - linux
+  secure: true
 - name: histogram_test
   build: test
   language: c
@@ -3812,6 +3876,7 @@ targets:
   - grpc_unsecure
   - gpr_test_util
   - gpr
+  - grpc++_test_config
   benchmark: true
   defaults: benchmark
   platforms:
@@ -3833,6 +3898,7 @@ targets:
   - grpc_unsecure
   - gpr_test_util
   - gpr
+  - grpc++_test_config
   benchmark: true
   defaults: benchmark
   platforms:
@@ -3854,6 +3920,7 @@ targets:
   - grpc_unsecure
   - gpr_test_util
   - gpr
+  - grpc++_test_config
   benchmark: true
   defaults: benchmark
   platforms:
@@ -3875,6 +3942,7 @@ targets:
   - grpc_unsecure
   - gpr_test_util
   - gpr
+  - grpc++_test_config
   benchmark: true
   defaults: benchmark
   platforms:
@@ -3895,6 +3963,7 @@ targets:
   - grpc_unsecure
   - gpr_test_util
   - gpr
+  - grpc++_test_config
   benchmark: true
   defaults: benchmark
   platforms:
@@ -3915,6 +3984,7 @@ targets:
   - grpc_unsecure
   - gpr_test_util
   - gpr
+  - grpc++_test_config
   benchmark: true
   defaults: benchmark
   platforms:
@@ -3935,6 +4005,7 @@ targets:
   - grpc_unsecure
   - gpr_test_util
   - gpr
+  - grpc++_test_config
   benchmark: true
   defaults: benchmark
   platforms:
@@ -3955,6 +4026,7 @@ targets:
   - grpc_unsecure
   - gpr_test_util
   - gpr
+  - grpc++_test_config
   benchmark: true
   defaults: benchmark
   platforms:
@@ -3978,6 +4050,7 @@ targets:
   - grpc_unsecure
   - gpr_test_util
   - gpr
+  - grpc++_test_config
   benchmark: true
   defaults: benchmark
   excluded_poll_engines:
@@ -4004,6 +4077,7 @@ targets:
   - grpc_unsecure
   - gpr_test_util
   - gpr
+  - grpc++_test_config
   benchmark: true
   defaults: benchmark
   excluded_poll_engines:
@@ -4057,6 +4131,7 @@ targets:
   - grpc_unsecure
   - gpr_test_util
   - gpr
+  - grpc++_test_config
   benchmark: true
   defaults: benchmark
   excluded_poll_engines:
@@ -4081,6 +4156,7 @@ targets:
   - grpc_unsecure
   - gpr_test_util
   - gpr
+  - grpc++_test_config
   benchmark: true
   defaults: benchmark
   platforms:
@@ -4102,6 +4178,7 @@ targets:
   - grpc_unsecure
   - gpr_test_util
   - gpr
+  - grpc++_test_config
   benchmark: true
   defaults: benchmark
   platforms:
@@ -4159,6 +4236,21 @@ targets:
   - grpc++_channelz_proto
   uses:
   - grpc++_test
+- name: channelz_registry_test
+  gtest: true
+  build: test
+  language: c++
+  src:
+  - test/core/channel/channelz_registry_test.cc
+  deps:
+  - grpc_test_util
+  - grpc++_test_util
+  - grpc++
+  - grpc
+  - gpr_test_util
+  - gpr
+  uses:
+  - grpc++_test
   uses_polling: false
 - name: check_gcp_environment_linux_test
   build: test
@@ -4753,6 +4845,20 @@ targets:
   - mac
   - linux
   - posix
+- name: lb_load_data_store_test
+  gtest: true
+  build: test
+  language: c++
+  src:
+  - test/cpp/server/load_reporter/load_data_store_test.cc
+  deps:
+  - lb_load_data_store
+  - grpc++_test_util
+  - grpc_test_util
+  - grpc++
+  - grpc
+  - gpr_test_util
+  - gpr
 - name: memory_test
   gtest: true
   build: test
@@ -5072,6 +5178,23 @@ targets:
   - grpc++_unsecure
   - grpc_unsecure
   - gpr
+- name: server_builder_with_socket_mutator_test
+  gtest: true
+  build: test
+  language: c++
+  src:
+  - src/proto/grpc/testing/echo_messages.proto
+  - src/proto/grpc/testing/echo.proto
+  - test/cpp/server/server_builder_with_socket_mutator_test.cc
+  deps:
+  - grpc++_test_util_unsecure
+  - grpc_test_util_unsecure
+  - gpr_test_util
+  - grpc++_unsecure
+  - grpc_unsecure
+  - gpr
+  platforms:
+  - posix
 - name: server_context_test_spouse_test
   gtest: true
   build: test
@@ -5376,7 +5499,7 @@ vspackages:
 configs:
   asan:
     CC: clang
-    CPPFLAGS: -O0 -fsanitize-coverage=edge -fsanitize=address -fno-omit-frame-pointer
+    CPPFLAGS: -O0 -fsanitize-coverage=edge,trace-pc-guard -fsanitize=address -fno-omit-frame-pointer
       -Wno-unused-command-line-argument -DGPR_NO_DIRECT_SYSCALLS
     CXX: clang++
     LD: clang++
@@ -5385,10 +5508,10 @@ configs:
     compile_the_world: true
     test_environ:
       ASAN_OPTIONS: detect_leaks=1:color=always
-      LSAN_OPTIONS: suppressions=tools/lsan_suppressions.txt:report_objects=1
+      LSAN_OPTIONS: suppressions=test/core/util/lsan_suppressions.txt:report_objects=1
   asan-noleaks:
     CC: clang
-    CPPFLAGS: -O0 -fsanitize-coverage=edge -fsanitize=address -fno-omit-frame-pointer
+    CPPFLAGS: -O0 -fsanitize-coverage=edge,trace-pc-guard -fsanitize=address -fno-omit-frame-pointer
       -Wno-unused-command-line-argument -DGPR_NO_DIRECT_SYSCALLS
     CXX: clang++
     LD: clang++
@@ -5399,8 +5522,9 @@ configs:
       ASAN_OPTIONS: detect_leaks=0:color=always
   asan-trace-cmp:
     CC: clang
-    CPPFLAGS: -O0 -fsanitize-coverage=edge -fsanitize-coverage=trace-cmp -fsanitize=address
-      -fno-omit-frame-pointer -Wno-unused-command-line-argument -DGPR_NO_DIRECT_SYSCALLS
+    CPPFLAGS: -O0 -fsanitize-coverage=edge,trace-pc-guard -fsanitize-coverage=trace-cmp
+      -fsanitize=address -fno-omit-frame-pointer -Wno-unused-command-line-argument
+      -DGPR_NO_DIRECT_SYSCALLS
     CXX: clang++
     LD: clang++
     LDFLAGS: -fsanitize=address
@@ -5408,7 +5532,7 @@ configs:
     compile_the_world: true
     test_environ:
       ASAN_OPTIONS: detect_leaks=1:color=always
-      LSAN_OPTIONS: suppressions=tools/lsan_suppressions.txt:report_objects=1
+      LSAN_OPTIONS: suppressions=test/core/util/lsan_suppressions.txt:report_objects=1
   basicprof:
     CPPFLAGS: -O2 -DGRPC_BASIC_PROFILER -DGRPC_TIMERS_RDTSC
     DEFINES: NDEBUG
@@ -5445,7 +5569,7 @@ configs:
     valgrind: --tool=memcheck --leak-check=full
   msan:
     CC: clang
-    CPPFLAGS: -O0 -fsanitize-coverage=edge -fsanitize=memory -fsanitize-memory-track-origins
+    CPPFLAGS: -O0 -fsanitize-coverage=edge,trace-pc-guard -fsanitize=memory -fsanitize-memory-track-origins
       -fsanitize-memory-use-after-dtor -fno-omit-frame-pointer -DGTEST_HAS_TR1_TUPLE=0
       -DGTEST_USE_OWN_TR1_TUPLE=1 -Wno-unused-command-line-argument -fPIE -pie -DGPR_NO_DIRECT_SYSCALLS
     CXX: clang++
@@ -5482,10 +5606,10 @@ configs:
     LDXX: clang++
     compile_the_world: true
     test_environ:
-      TSAN_OPTIONS: suppressions=tools/tsan_suppressions.txt:halt_on_error=1:second_deadlock_stack=1
+      TSAN_OPTIONS: suppressions=test/core/util/tsan_suppressions.txt:halt_on_error=1:second_deadlock_stack=1
   ubsan:
     CC: clang
-    CPPFLAGS: -O0 -fsanitize-coverage=edge -fsanitize=undefined -fno-omit-frame-pointer
+    CPPFLAGS: -O0 -fsanitize-coverage=edge,trace-pc-guard -fsanitize=undefined -fno-omit-frame-pointer
       -Wno-unused-command-line-argument -Wvarargs
     CXX: clang++
     DEFINES: NDEBUG GRPC_UBSAN
@@ -5494,7 +5618,7 @@ configs:
     LDXX: clang++
     compile_the_world: true
     test_environ:
-      UBSAN_OPTIONS: halt_on_error=1:print_stacktrace=1:suppressions=tools/ubsan_suppressions.txt
+      UBSAN_OPTIONS: halt_on_error=1:print_stacktrace=1:suppressions=test/core/util/ubsan_suppressions.txt
 defaults:
   ares:
     CFLAGS: -Wno-sign-conversion $(if $(subst Darwin,,$(SYSTEM)),,-Wno-shorten-64-to-32)
@@ -5516,7 +5640,7 @@ defaults:
   global:
     COREFLAGS: -fno-rtti -fno-exceptions
     CPPFLAGS: -g -Wall -Wextra -Werror -Wno-long-long -Wno-unused-parameter -DOSATOMIC_USE_INLINED=1
-      -Wno-deprecated-declarations
+      -Wno-deprecated-declarations -Ithird_party/nanopb -DPB_FIELD_32BIT
     LDFLAGS: -g
   zlib:
     CFLAGS: -Wno-sign-conversion -Wno-conversion -Wno-unused-value -Wno-implicit-function-declaration

src/python/grpcio_tests/tests/unit/_junkdrawer/__init__.py → cmake/nanopb.cmake

@@ -1,4 +1,4 @@
-# Copyright 2015 gRPC authors.
+# Copyright 2018 gRPC authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -11,3 +11,5 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
+set(_gRPC_NANOPB_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/third_party/nanopb")

config.m4

@@ -53,7 +53,6 @@ if test "$PHP_GRPC" != "no"; then
     src/core/lib/gpr/env_linux.cc \
     src/core/lib/gpr/env_posix.cc \
     src/core/lib/gpr/env_windows.cc \
-    src/core/lib/gpr/fork.cc \
     src/core/lib/gpr/host_port.cc \
     src/core/lib/gpr/log.cc \
     src/core/lib/gpr/log_android.cc \
@@ -78,6 +77,7 @@ if test "$PHP_GRPC" != "no"; then
     src/core/lib/gpr/tmpfile_posix.cc \
     src/core/lib/gpr/tmpfile_windows.cc \
     src/core/lib/gpr/wrap_memcpy.cc \
+    src/core/lib/gprpp/fork.cc \
     src/core/lib/gprpp/thd_posix.cc \
     src/core/lib/gprpp/thd_windows.cc \
     src/core/lib/profiling/basic_timers.cc \
@@ -89,7 +89,7 @@ if test "$PHP_GRPC" != "no"; then
     src/core/lib/channel/channel_stack.cc \
     src/core/lib/channel/channel_stack_builder.cc \
     src/core/lib/channel/channel_trace.cc \
-    src/core/lib/channel/channel_trace_registry.cc \
+    src/core/lib/channel/channelz_registry.cc \
     src/core/lib/channel/connected_channel.cc \
     src/core/lib/channel/handshaker.cc \
     src/core/lib/channel/handshaker_factory.cc \
@@ -321,7 +321,6 @@ if test "$PHP_GRPC" != "no"; then
     third_party/nanopb/pb_decode.c \
     third_party/nanopb/pb_encode.c \
     src/core/tsi/transport_security.cc \
-    src/core/tsi/transport_security_adapter.cc \
     src/core/ext/transport/chttp2/client/insecure/channel_create.cc \
     src/core/ext/transport/chttp2/client/insecure/channel_create_posix.cc \
     src/core/ext/transport/chttp2/client/authority.cc \
@@ -369,9 +368,9 @@ if test "$PHP_GRPC" != "no"; then
     src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c \
     src/core/ext/filters/client_channel/resolver/fake/fake_resolver.cc \
     src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc \
-    src/core/ext/filters/client_channel/lb_policy/subchannel_list.cc \
     src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc \
     src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc \
+    src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.cc \
     src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc \
     src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc \
     src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_fallback.cc \
@@ -379,7 +378,7 @@ if test "$PHP_GRPC" != "no"; then
     src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.cc \
     src/core/ext/filters/load_reporting/server_load_reporting_filter.cc \
     src/core/ext/filters/load_reporting/server_load_reporting_plugin.cc \
-    src/core/ext/census/grpc_context.cc \
+    src/cpp/ext/filters/census/grpc_context.cc \
     src/core/ext/filters/max_age/max_age_filter.cc \
     src/core/ext/filters/message_size/message_size_filter.cc \
     src/core/ext/filters/http/client_authority_filter.cc \
@@ -438,6 +437,7 @@ if test "$PHP_GRPC" != "no"; then
     third_party/boringssl/crypto/chacha/chacha.c \
     third_party/boringssl/crypto/cipher_extra/cipher_extra.c \
     third_party/boringssl/crypto/cipher_extra/derive_key.c \
+    third_party/boringssl/crypto/cipher_extra/e_aesccm.c \
     third_party/boringssl/crypto/cipher_extra/e_aesctrhmac.c \
     third_party/boringssl/crypto/cipher_extra/e_aesgcmsiv.c \
     third_party/boringssl/crypto/cipher_extra/e_chacha20poly1305.c \
@@ -449,6 +449,7 @@ if test "$PHP_GRPC" != "no"; then
     third_party/boringssl/crypto/cipher_extra/tls_cbc.c \
     third_party/boringssl/crypto/cmac/cmac.c \
     third_party/boringssl/crypto/conf/conf.c \
+    third_party/boringssl/crypto/cpu-aarch64-fuchsia.c \
     third_party/boringssl/crypto/cpu-aarch64-linux.c \
     third_party/boringssl/crypto/cpu-arm-linux.c \
     third_party/boringssl/crypto/cpu-arm.c \
@@ -456,7 +457,6 @@ if test "$PHP_GRPC" != "no"; then
     third_party/boringssl/crypto/cpu-ppc64le.c \
     third_party/boringssl/crypto/crypto.c \
     third_party/boringssl/crypto/curve25519/spake25519.c \
-    third_party/boringssl/crypto/curve25519/x25519-x86_64.c \
     third_party/boringssl/crypto/dh/check.c \
     third_party/boringssl/crypto/dh/dh.c \
     third_party/boringssl/crypto/dh/dh_asn1.c \
@@ -612,6 +612,7 @@ if test "$PHP_GRPC" != "no"; then
     third_party/boringssl/ssl/d1_srtp.cc \
     third_party/boringssl/ssl/dtls_method.cc \
     third_party/boringssl/ssl/dtls_record.cc \
+    third_party/boringssl/ssl/handoff.cc \
     third_party/boringssl/ssl/handshake.cc \
     third_party/boringssl/ssl/handshake_client.cc \
     third_party/boringssl/ssl/handshake_server.cc \
@@ -648,9 +649,7 @@ if test "$PHP_GRPC" != "no"; then
   PHP_ADD_BUILD_DIR($ext_builddir/src/php/ext/grpc)
 
   PHP_ADD_BUILD_DIR($ext_builddir/src/boringssl)
-  PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/census)
   PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/filters/client_channel)
-  PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/filters/client_channel/lb_policy)
   PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/filters/client_channel/lb_policy/grpclb)
   PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1)
   PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/filters/client_channel/lb_policy/pick_first)
@@ -712,6 +711,7 @@ if test "$PHP_GRPC" != "no"; then
   PHP_ADD_BUILD_DIR($ext_builddir/src/core/tsi/alts/handshaker)
   PHP_ADD_BUILD_DIR($ext_builddir/src/core/tsi/alts/zero_copy_frame_protector)
   PHP_ADD_BUILD_DIR($ext_builddir/src/core/tsi/ssl/session_cache)
+  PHP_ADD_BUILD_DIR($ext_builddir/src/cpp/ext/filters/census)
   PHP_ADD_BUILD_DIR($ext_builddir/third_party/address_sorting)
   PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto)
   PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/asn1)

config.w32

@@ -29,7 +29,6 @@ if (PHP_GRPC != "no") {
     "src\\core\\lib\\gpr\\env_linux.cc " +
     "src\\core\\lib\\gpr\\env_posix.cc " +
     "src\\core\\lib\\gpr\\env_windows.cc " +
-    "src\\core\\lib\\gpr\\fork.cc " +
     "src\\core\\lib\\gpr\\host_port.cc " +
     "src\\core\\lib\\gpr\\log.cc " +
     "src\\core\\lib\\gpr\\log_android.cc " +
@@ -54,6 +53,7 @@ if (PHP_GRPC != "no") {
     "src\\core\\lib\\gpr\\tmpfile_posix.cc " +
     "src\\core\\lib\\gpr\\tmpfile_windows.cc " +
     "src\\core\\lib\\gpr\\wrap_memcpy.cc " +
+    "src\\core\\lib\\gprpp\\fork.cc " +
     "src\\core\\lib\\gprpp\\thd_posix.cc " +
     "src\\core\\lib\\gprpp\\thd_windows.cc " +
     "src\\core\\lib\\profiling\\basic_timers.cc " +
@@ -65,7 +65,7 @@ if (PHP_GRPC != "no") {
     "src\\core\\lib\\channel\\channel_stack.cc " +
     "src\\core\\lib\\channel\\channel_stack_builder.cc " +
     "src\\core\\lib\\channel\\channel_trace.cc " +
-    "src\\core\\lib\\channel\\channel_trace_registry.cc " +
+    "src\\core\\lib\\channel\\channelz_registry.cc " +
     "src\\core\\lib\\channel\\connected_channel.cc " +
     "src\\core\\lib\\channel\\handshaker.cc " +
     "src\\core\\lib\\channel\\handshaker_factory.cc " +
@@ -297,7 +297,6 @@ if (PHP_GRPC != "no") {
     "third_party\\nanopb\\pb_decode.c " +
     "third_party\\nanopb\\pb_encode.c " +
     "src\\core\\tsi\\transport_security.cc " +
-    "src\\core\\tsi\\transport_security_adapter.cc " +
     "src\\core\\ext\\transport\\chttp2\\client\\insecure\\channel_create.cc " +
     "src\\core\\ext\\transport\\chttp2\\client\\insecure\\channel_create_posix.cc " +
     "src\\core\\ext\\transport\\chttp2\\client\\authority.cc " +
@@ -345,9 +344,9 @@ if (PHP_GRPC != "no") {
     "src\\core\\ext\\filters\\client_channel\\lb_policy\\grpclb\\proto\\grpc\\lb\\v1\\load_balancer.pb.c " +
     "src\\core\\ext\\filters\\client_channel\\resolver\\fake\\fake_resolver.cc " +
     "src\\core\\ext\\filters\\client_channel\\lb_policy\\pick_first\\pick_first.cc " +
-    "src\\core\\ext\\filters\\client_channel\\lb_policy\\subchannel_list.cc " +
     "src\\core\\ext\\filters\\client_channel\\lb_policy\\round_robin\\round_robin.cc " +
     "src\\core\\ext\\filters\\client_channel\\resolver\\dns\\c_ares\\dns_resolver_ares.cc " +
+    "src\\core\\ext\\filters\\client_channel\\resolver\\dns\\c_ares\\grpc_ares_ev_driver.cc " +
     "src\\core\\ext\\filters\\client_channel\\resolver\\dns\\c_ares\\grpc_ares_ev_driver_posix.cc " +
     "src\\core\\ext\\filters\\client_channel\\resolver\\dns\\c_ares\\grpc_ares_wrapper.cc " +
     "src\\core\\ext\\filters\\client_channel\\resolver\\dns\\c_ares\\grpc_ares_wrapper_fallback.cc " +
@@ -355,7 +354,7 @@ if (PHP_GRPC != "no") {
     "src\\core\\ext\\filters\\client_channel\\resolver\\sockaddr\\sockaddr_resolver.cc " +
     "src\\core\\ext\\filters\\load_reporting\\server_load_reporting_filter.cc " +
     "src\\core\\ext\\filters\\load_reporting\\server_load_reporting_plugin.cc " +
-    "src\\core\\ext\\census\\grpc_context.cc " +
+    "src\\cpp\\ext\\filters\\census\\grpc_context.cc " +
     "src\\core\\ext\\filters\\max_age\\max_age_filter.cc " +
     "src\\core\\ext\\filters\\message_size\\message_size_filter.cc " +
     "src\\core\\ext\\filters\\http\\client_authority_filter.cc " +
@@ -414,6 +413,7 @@ if (PHP_GRPC != "no") {
     "third_party\\boringssl\\crypto\\chacha\\chacha.c " +
     "third_party\\boringssl\\crypto\\cipher_extra\\cipher_extra.c " +
     "third_party\\boringssl\\crypto\\cipher_extra\\derive_key.c " +
+    "third_party\\boringssl\\crypto\\cipher_extra\\e_aesccm.c " +
     "third_party\\boringssl\\crypto\\cipher_extra\\e_aesctrhmac.c " +
     "third_party\\boringssl\\crypto\\cipher_extra\\e_aesgcmsiv.c " +
     "third_party\\boringssl\\crypto\\cipher_extra\\e_chacha20poly1305.c " +
@@ -425,6 +425,7 @@ if (PHP_GRPC != "no") {
     "third_party\\boringssl\\crypto\\cipher_extra\\tls_cbc.c " +
     "third_party\\boringssl\\crypto\\cmac\\cmac.c " +
     "third_party\\boringssl\\crypto\\conf\\conf.c " +
+    "third_party\\boringssl\\crypto\\cpu-aarch64-fuchsia.c " +
     "third_party\\boringssl\\crypto\\cpu-aarch64-linux.c " +
     "third_party\\boringssl\\crypto\\cpu-arm-linux.c " +
     "third_party\\boringssl\\crypto\\cpu-arm.c " +
@@ -432,7 +433,6 @@ if (PHP_GRPC != "no") {
     "third_party\\boringssl\\crypto\\cpu-ppc64le.c " +
     "third_party\\boringssl\\crypto\\crypto.c " +
     "third_party\\boringssl\\crypto\\curve25519\\spake25519.c " +
-    "third_party\\boringssl\\crypto\\curve25519\\x25519-x86_64.c " +
     "third_party\\boringssl\\crypto\\dh\\check.c " +
     "third_party\\boringssl\\crypto\\dh\\dh.c " +
     "third_party\\boringssl\\crypto\\dh\\dh_asn1.c " +
@@ -588,6 +588,7 @@ if (PHP_GRPC != "no") {
     "third_party\\boringssl\\ssl\\d1_srtp.cc " +
     "third_party\\boringssl\\ssl\\dtls_method.cc " +
     "third_party\\boringssl\\ssl\\dtls_record.cc " +
+    "third_party\\boringssl\\ssl\\handoff.cc " +
     "third_party\\boringssl\\ssl\\handshake.cc " +
     "third_party\\boringssl\\ssl\\handshake_client.cc " +
     "third_party\\boringssl\\ssl\\handshake_server.cc " +
@@ -652,7 +653,6 @@ if (PHP_GRPC != "no") {
   FSO.CreateFolder(base_dir+"\\ext\\grpc\\src\\boringssl");
   FSO.CreateFolder(base_dir+"\\ext\\grpc\\src\\core");
   FSO.CreateFolder(base_dir+"\\ext\\grpc\\src\\core\\ext");
-  FSO.CreateFolder(base_dir+"\\ext\\grpc\\src\\core\\ext\\census");
   FSO.CreateFolder(base_dir+"\\ext\\grpc\\src\\core\\ext\\filters");
   FSO.CreateFolder(base_dir+"\\ext\\grpc\\src\\core\\ext\\filters\\client_channel");
   FSO.CreateFolder(base_dir+"\\ext\\grpc\\src\\core\\ext\\filters\\client_channel\\lb_policy");
@@ -728,6 +728,10 @@ if (PHP_GRPC != "no") {
   FSO.CreateFolder(base_dir+"\\ext\\grpc\\src\\core\\tsi\\alts\\zero_copy_frame_protector");
   FSO.CreateFolder(base_dir+"\\ext\\grpc\\src\\core\\tsi\\ssl");
   FSO.CreateFolder(base_dir+"\\ext\\grpc\\src\\core\\tsi\\ssl\\session_cache");
+  FSO.CreateFolder(base_dir+"\\ext\\grpc\\src\\cpp");
+  FSO.CreateFolder(base_dir+"\\ext\\grpc\\src\\cpp\\ext");
+  FSO.CreateFolder(base_dir+"\\ext\\grpc\\src\\cpp\\ext\\filters");
+  FSO.CreateFolder(base_dir+"\\ext\\grpc\\src\\cpp\\ext\\filters\\census");
   FSO.CreateFolder(base_dir+"\\ext\\grpc\\src\\php");
   FSO.CreateFolder(base_dir+"\\ext\\grpc\\src\\php\\ext");
   FSO.CreateFolder(base_dir+"\\ext\\grpc\\src\\php\\ext\\grpc");

doc/PROTOCOL-WEB.md

@@ -138,4 +138,4 @@ Versioning
 
 Browser-specific features
 
-* For features that are unique to browser or HTML clients, check the [spec doc](https://github.com/grpc/grpc-web/blob/master/PROTOCOL-WEB.md) published in the grpc/grpc-web repo.
+* For features that are unique to browser or HTML clients, check the [spec doc](https://github.com/grpc/grpc-web/blob/master/BROWSER-FEATURES.md) published in the grpc/grpc-web repo.

doc/environment_variables.md

@@ -44,11 +44,12 @@ some configuration as environment variables that can be set.
   - channel - traces operations on the C core channel stack
   - client_channel - traces client channel activity, including resolver
     and load balancing policy interaction
-  - combiner - traces combiner lock state
   - compression - traces compression operations
   - connectivity_state - traces connectivity state changes to channels
-  - channel_stack_builder - traces information about channel stacks being built
   - executor - traces grpc's internal thread pool ('the executor')
+  - fd_trace - traces fd create(), shutdown() and close() calls for channel fds.
+    Also traces epoll fd create()/close() calls in epollex polling engine
+    traces epoll-fd creation/close calls for epollex polling engine
   - glb - traces the grpclb load balancer
   - handshaker - traces handshaking state
   - http - traces state in the http2 transport engine
@@ -65,7 +66,6 @@ some configuration as environment variables that can be set.
   - resource_quota - trace resource quota objects internals
   - round_robin - traces the round_robin load balancing policy
   - queue_pluck
-  - queue_timeout
   - server_channel - lightweight trace of significant server channel events
   - secure_endpoint - traces bytes flowing through encrypted channels
   - timer - timers (alarms) in the grpc internals
@@ -78,6 +78,7 @@ some configuration as environment variables that can be set.
   accomplished by invoking `CONFIG=dbg make <target>`
   - alarm_refcount - refcounting traces for grpc_alarm structure
   - metadata - tracks creation and mutation of metadata
+  - combiner - traces combiner lock state
   - closure - tracks closure creation, scheduling, and completion
   - pending_tags - traces still-in-progress tags on completion queues
   - polling - traces the selected polling engine

doc/fork_support.md

@@ -0,0 +1,46 @@
+# Background #
+
+In Python, multithreading is ineffective at concurrency for CPU bound tasks
+due to the GIL (global interpreter lock).  Extension modules can release
+the GIL in CPU bound tasks, but that isn't an option in pure Python.
+Users use libraries such as multiprocessing, subprocess, concurrent.futures.ProcessPoolExecutor,
+etc, to work around the GIL. These modules call ```fork()``` underneath the hood. Various issues have
+been reported when using these modules with gRPC Python.  gRPC Python wraps
+gRPC core, which uses multithreading for performance, and hence doesn't support ```fork()```.
+Historically, we didn't support forking in gRPC, but some users seemed
+to be doing fine until their code started to break on version 1.6.  This was
+likely caused by the addition of background c-threads and a background
+Python thread.
+
+# Current Status #
+
+## 1.11 ##
+The background Python thread was removed entirely.  This allows forking
+after creating a channel.  However, the channel must not have issued any
+RPCs prior to the fork.  Attempting to fork with an active channel that
+has been used can result in deadlocks/corrupted wire data.
+
+## 1.9 ##
+A regression was noted in cases where users are doing fork/exec. This
+was due to ```pthread_atfork()``` handler that was added in 1.7 to partially
+support forking in gRPC. A deadlock can happen when pthread_atfork
+handler is running, and an application thread is calling into gRPC.
+We have provided a workaround for this issue by allowing users to turn 
+off the handler using env flag ```GRPC_ENABLE_FORK_SUPPORT=False```.
+This should be set whenever a user expects to always call exec
+immediately following fork.  It will disable the fork handlers.
+
+## 1.7 ##
+A ```pthread_atfork()``` handler was added in 1.7 to automatically shut down
+the background c-threads when fork was called.  This does not shut down the
+background Python thread, so users could not have any open channels when
+forking.
+
+# Future Work #
+
+## 1.13 ##
+The workaround when using fork/exec by setting
+```GRPC_ENABLE_FORK_SUPPORT=False``` should no longer be needed.  Following
+[this PR](https://github.com/grpc/grpc/pull/14647), fork
+handlers will not automatically run when multiple threads are calling
+into gRPC.

doc/g_stands_for.md

@@ -1,18 +1,16 @@
-Each version of gRPC gets a new description of what the 'g' stands for, since
-we've never really been able to figure it out.
+'g' stands for something different every gRPC release:
 
-Below is a list of already-used definitions (that should not be repeated in the
-future), and the corresponding version numbers that used them:
-
-- 1.0 'g' stands for 'gRPC'
-- 1.1 'g' stands for 'good'
-- 1.2 'g' stands for 'green'
-- 1.3 'g' stands for 'gentle'
-- 1.4 'g' stands for 'gregarious'
-- 1.6 'g' stands for 'garcia'
-- 1.7 'g' stands for 'gambit'
-- 1.8 'g' stands for 'generous'
-- 1.9 'g' stands for 'glossy'
-- 1.10 'g' stands for 'glamorous'
-- 1.11 'g' stands for 'gorgeous'
-- 1.12 'g' stands for 'glorious'
+- 1.0 'g' stands for ['gRPC'](https://github.com/grpc/grpc/tree/v1.0.x)
+- 1.1 'g' stands for ['good'](https://github.com/grpc/grpc/tree/v1.1.x)
+- 1.2 'g' stands for ['green'](https://github.com/grpc/grpc/tree/v1.2.x)
+- 1.3 'g' stands for ['gentle'](https://github.com/grpc/grpc/tree/v1.3.x)
+- 1.4 'g' stands for ['gregarious'](https://github.com/grpc/grpc/tree/v1.4.x)
+- 1.6 'g' stands for ['garcia'](https://github.com/grpc/grpc/tree/v1.6.x)
+- 1.7 'g' stands for ['gambit'](https://github.com/grpc/grpc/tree/v1.7.x)
+- 1.8 'g' stands for ['generous'](https://github.com/grpc/grpc/tree/v1.8.x)
+- 1.9 'g' stands for ['glossy'](https://github.com/grpc/grpc/tree/v1.9.x)
+- 1.10 'g' stands for ['glamorous'](https://github.com/grpc/grpc/tree/v1.10.x)
+- 1.11 'g' stands for ['gorgeous'](https://github.com/grpc/grpc/tree/v1.11.x)
+- 1.12 'g' stands for ['glorious'](https://github.com/grpc/grpc/tree/v1.12.x)
+- 1.13 'g' stands for ['gloriosa'](https://github.com/grpc/grpc/tree/v1.13.x)
+- 1.14 'g' stands for ['gladiolus'](https://github.com/grpc/grpc/tree/master)

doc/stress_test_framework.md

@@ -1,92 +0,0 @@
-# Stress Test framework for gRPC
-
-(Sree Kuchibhotla - sreek@)
-
-Status: This is implemented. More details at [README.md](https://github.com/grpc/grpc/blob/master/tools/run_tests/stress_test/README.md)
-
-
-**I. GOALS**
-
-1) Build a stress test suite for gRPC:
-
-* Build a stress test suite that can Identify bugs by testing the system (gRPC server/client) under extreme conditions:
-  * High load
-  * High concurrency
-  * Limited resources 
-  * Intermittent failures
-* Should be integrated with Jenkins CI
-
-2) Make it generic enough (i.e build a generic test framework) that can be used for:
-
-* Executing M instances of a client against N instances of a server with an arbitrarily defined connection matrix
-   * Execute heterogenous test configurations - for example: Java stress test clients against C++ servers or Node clients against Python servers or TSAN C++ clients vs ASAN C++ Servers etc.
-   * Easy and Flexible enough that Devs can use it to recreate complex test scenarios
-
-The implementation effort is divided into two parts:
-
-* Building a "Stress Test Framework" to run the stress test suites- More details in **Section II** (The idea is that the Stress Test framework is generic enough that it would be easier to modify it to run other suites like interop-tests or custom test scenarios)
-* Building a 'Stress test suite' - More details in **section III**
-
-**Terminology:**
-
-GCE - Google compute engine
-GKE - Google Container engine
-Kubernetes - Google's open source service scheduler / orchestrator.
-
-**Note:** The terms GKE and Kubernetes are used interchangeably in this document
-
-# II. STRESS TEST FRAMEWORK
-
-(The details of each step are explained below))
-![image](images/stress_test_framework.png)
-**Figure 1**
-
-### Step 1 Read the test config, generate base docker images
-
-**_Test Config:_** The test configuration contains the following information:
-
-* _GKE info:_ GKE project and cluster info
-* _Docker images:_ Instructions to build docker images
-* _Client templates:_ One or more client templates each containing the following information:
-    * Which docker image to use
-    * Path to the client program to launch (within the docker image)
-    * Parameters to the client program
-* _Server templates:_ Similar to Client templates - except that these are for servers
-* Test matrix containing the following:
-    * _Server groups:_ One or more groups of servers containing the following info for each group
-        * Which server template to use
-        * How many instances to launch
-    * _Client groups:_ One or more groups of clients containing the following (for each group):
-        * Which client template to use
-        * How many instances to launch
-        * Which server group to talk to (all clients in this group will talk to all servers in the server group)
-
-The first step is to read the test config and build the docker images
-
-**_Stress server docker image:_** The following are the main files in the server docker images
-
-* _Interop_server:_ The server program
-* `run_server.py`: This is a python script which is the entry point of the docker image (i.e this is the script that is called when the docker image is run in GKE).  This script launches the interop server and also updates the status in BigQuery.  If the interop_server fails for whatever reason, the script launch_server.py logs that status in BigQuery
-
-**_Stress client docker image:_**
-
-* Stress client: The stress test client.  In addition to talking to the interop_server, the stress client also exports metrics (which can be queried by the metrics_client described below)
-* Metrics client: Metrics client connects to the stress_client to get the current qps metrics.
-* `run_client.py`: This is a python script which is the entry point of the docker image (i.e this is the script that is called when the docker image is run in GKE). This script launches the stress client and also updates the status in BigQuery.  The script then periodically launches metrics client to query the qps from the stress client and then uploads the qps to BigQuery.
-
-### Step 2) Upload the docker images to GKE
-The docker images are uploaded to the GKE registry
-
-### Step 3) Launch the tests in GKE
-The test driver reads the test matrix (described in step 1) and creates the necessary server and client pods in GKE.
-
-### Step 4) Tests are run in GKE
-GKE starts running the tests by calling the entry points in *each* docker image (i.e `run_server.py` or `run_client.py` depending on whcih docker image it is)
-
-### Step 5) Upload the status to GKE and Monitor the status in GKE
-* 5.1 The tests periodically update their status in BigQuery
-* 5.2 The test driver periodically checks the status in Bigquery to see if any tests failed.  If any tests failed, the driver immediately stops the tests.  If not, the driver continues to run the tests for a configurable amount of time.
-
-### Step 6) Create a summary report
-The test driver creates a final summary report containing details about any test failures and information about how to connect the failed pods in GKE for debugging.
-

etc/roots.pem

@@ -3525,39 +3525,6 @@ AAoACxGV2lZFA4gKn2fQ1XmxqI1AbQ3CekD6819kR5LLU7m7Wc5P/dAVUwHY3+vZ
 5nbv0CO7O6l5s9UCKc2Jo5YPSjXnTkLAdc0Hz+Ys63su
 -----END CERTIFICATE-----
 
-# Issuer: CN=TÜRKTRUST Elektronik Sertifika Hizmet Sağlayıcısı H5 O=TÜRKTRUST Bilgi İletişim ve Bilişim Güvenliği Hizmetleri A.Ş.
-# Subject: CN=TÜRKTRUST Elektronik Sertifika Hizmet Sağlayıcısı H5 O=TÜRKTRUST Bilgi İletişim ve Bilişim Güvenliği Hizmetleri A.Ş.
-# Label: "TÜRKTRUST Elektronik Sertifika Hizmet Sağlayıcısı H5"
-# Serial: 156233699172481
-# MD5 Fingerprint: da:70:8e:f0:22:df:93:26:f6:5f:9f:d3:15:06:52:4e
-# SHA1 Fingerprint: c4:18:f6:4d:46:d1:df:00:3d:27:30:13:72:43:a9:12:11:c6:75:fb
-# SHA256 Fingerprint: 49:35:1b:90:34:44:c1:85:cc:dc:5c:69:3d:24:d8:55:5c:b2:08:d6:a8:14:13:07:69:9f:4a:f0:63:19:9d:78
------BEGIN CERTIFICATE-----
-MIIEJzCCAw+gAwIBAgIHAI4X/iQggTANBgkqhkiG9w0BAQsFADCBsTELMAkGA1UE
-BhMCVFIxDzANBgNVBAcMBkFua2FyYTFNMEsGA1UECgxEVMOcUktUUlVTVCBCaWxn
-aSDEsGxldGnFn2ltIHZlIEJpbGnFn2ltIEfDvHZlbmxpxJ9pIEhpem1ldGxlcmkg
-QS7Fni4xQjBABgNVBAMMOVTDnFJLVFJVU1QgRWxla3Ryb25payBTZXJ0aWZpa2Eg
-SGl6bWV0IFNhxJ9sYXnEsWPEsXPEsSBINTAeFw0xMzA0MzAwODA3MDFaFw0yMzA0
-MjgwODA3MDFaMIGxMQswCQYDVQQGEwJUUjEPMA0GA1UEBwwGQW5rYXJhMU0wSwYD
-VQQKDERUw5xSS1RSVVNUIEJpbGdpIMSwbGV0acWfaW0gdmUgQmlsacWfaW0gR8O8
-dmVubGnEn2kgSGl6bWV0bGVyaSBBLsWeLjFCMEAGA1UEAww5VMOcUktUUlVTVCBF
-bGVrdHJvbmlrIFNlcnRpZmlrYSBIaXptZXQgU2HEn2xhecSxY8Sxc8SxIEg1MIIB
-IjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEApCUZ4WWe60ghUEoI5RHwWrom
-/4NZzkQqL/7hzmAD/I0Dpe3/a6i6zDQGn1k19uwsu537jVJp45wnEFPzpALFp/kR
-Gml1bsMdi9GYjZOHp3GXDSHHmflS0yxjXVW86B8BSLlg/kJK9siArs1mep5Fimh3
-4khon6La8eHBEJ/rPCmBp+EyCNSgBbGM+42WAA4+Jd9ThiI7/PS98wl+d+yG6w8z
-5UNP9FR1bSmZLmZaQ9/LXMrI5Tjxfjs1nQ/0xVqhzPMggCTTV+wVunUlm+hkS7M0
-hO8EuPbJbKoCPrZV4jI3X/xml1/N1p7HIL9Nxqw/dV8c7TKcfGkAaZHjIxhT6QID
-AQABo0IwQDAdBgNVHQ4EFgQUVpkHHtOsDGlktAxQR95DLL4gwPswDgYDVR0PAQH/
-BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAJ5FdnsX
-SDLyOIspve6WSk6BGLFRRyDN0GSxDsnZAdkJzsiZ3GglE9Rc8qPoBP5yCccLqh0l
-VX6Wmle3usURehnmp349hQ71+S4pL+f5bFgWV1Al9j4uPqrtd3GqqpmWRgqujuwq
-URawXs3qZwQcWDD1YIq9pr1N5Za0/EKJAWv2cMhQOQwt1WbZyNKzMrcbGW3LM/nf
-peYVhDfwwvJllpKQd/Ct9JDpEXjXk4nAPQu6KfTomZ1yju2dL+6SfaHx/126M2CF
-Yv4HAqGEVka+lgqaE9chTLd8B59OTj+RdPsnnRHM3eaxynFNExc5JsUpISuTKWqW
-+qtB4Uu2NQvAmxU=
------END CERTIFICATE-----
-
 # Issuer: CN=Certinomis - Root CA O=Certinomis OU=0002 433998903
 # Subject: CN=Certinomis - Root CA O=Certinomis OU=0002 433998903
 # Label: "Certinomis - Root CA"

examples/csharp/helloworld-from-cli/global.json

@@ -1,5 +0,0 @@
-{
-  "sdk": {
-    "version": "1.0.0"
-  }
-}

examples/node/dynamic_codegen/route_guide/route_guide_server.js

@@ -122,7 +122,7 @@ function getDistance(start, end) {
   var deltalon = lon2-lon1;
   var a = Math.sin(deltalat/2) * Math.sin(deltalat/2) +
       Math.cos(lat1) * Math.cos(lat2) *
-      Math.sin(dlon/2) * Math.sin(dlon/2);
+      Math.sin(deltalon/2) * Math.sin(deltalon/2);
   var c = 2 * Math.atan2(Math.sqrt(a), Math.sqrt(1-a));
   return R * c;
 }

examples/python/helloworld/greeter_client.py

@@ -22,9 +22,12 @@ import helloworld_pb2_grpc
 
 
 def run():
-    channel = grpc.insecure_channel('localhost:50051')
-    stub = helloworld_pb2_grpc.GreeterStub(channel)
-    response = stub.SayHello(helloworld_pb2.HelloRequest(name='you'))
+    # NOTE(gRPC Python Team): .close() is possible on a channel and should be
+    # used in circumstances in which the with statement does not fit the needs
+    # of the code.
+    with grpc.insecure_channel('localhost:50051') as channel:
+        stub = helloworld_pb2_grpc.GreeterStub(channel)
+        response = stub.SayHello(helloworld_pb2.HelloRequest(name='you'))
     print("Greeter client received: " + response.message)
 
 

examples/python/interceptors/default_value/greeter_client.py

@@ -27,10 +27,14 @@ def run():
         message='Hello from your local interceptor!')
     default_value_interceptor = default_value_client_interceptor.DefaultValueClientInterceptor(
         default_value)
-    channel = grpc.insecure_channel('localhost:50051')
-    channel = grpc.intercept_channel(channel, default_value_interceptor)
-    stub = helloworld_pb2_grpc.GreeterStub(channel)
-    response = stub.SayHello(helloworld_pb2.HelloRequest(name='you'))
+    # NOTE(gRPC Python Team): .close() is possible on a channel and should be
+    # used in circumstances in which the with statement does not fit the needs
+    # of the code.
+    with grpc.insecure_channel('localhost:50051') as channel:
+        intercept_channel = grpc.intercept_channel(channel,
+                                                   default_value_interceptor)
+        stub = helloworld_pb2_grpc.GreeterStub(intercept_channel)
+        response = stub.SayHello(helloworld_pb2.HelloRequest(name='you'))
     print("Greeter client received: " + response.message)
 
 

examples/python/interceptors/headers/greeter_client.py

@@ -25,10 +25,14 @@ import header_manipulator_client_interceptor
 def run():
     header_adder_interceptor = header_manipulator_client_interceptor.header_adder_interceptor(
         'one-time-password', '42')
-    channel = grpc.insecure_channel('localhost:50051')
-    channel = grpc.intercept_channel(channel, header_adder_interceptor)
-    stub = helloworld_pb2_grpc.GreeterStub(channel)
-    response = stub.SayHello(helloworld_pb2.HelloRequest(name='you'))
+    # NOTE(gRPC Python Team): .close() is possible on a channel and should be
+    # used in circumstances in which the with statement does not fit the needs
+    # of the code.
+    with grpc.insecure_channel('localhost:50051') as channel:
+        intercept_channel = grpc.intercept_channel(channel,
+                                                   header_adder_interceptor)
+        stub = helloworld_pb2_grpc.GreeterStub(intercept_channel)
+        response = stub.SayHello(helloworld_pb2.HelloRequest(name='you'))
     print("Greeter client received: " + response.message)
 
 

examples/python/multiplex/multiplex_client.py

@@ -106,20 +106,23 @@ def guide_route_chat(route_guide_stub):
 
 
 def run():
-    channel = grpc.insecure_channel('localhost:50051')
-    greeter_stub = helloworld_pb2_grpc.GreeterStub(channel)
-    route_guide_stub = route_guide_pb2_grpc.RouteGuideStub(channel)
-    greeter_response = greeter_stub.SayHello(
-        helloworld_pb2.HelloRequest(name='you'))
-    print("Greeter client received: " + greeter_response.message)
-    print("-------------- GetFeature --------------")
-    guide_get_feature(route_guide_stub)
-    print("-------------- ListFeatures --------------")
-    guide_list_features(route_guide_stub)
-    print("-------------- RecordRoute --------------")
-    guide_record_route(route_guide_stub)
-    print("-------------- RouteChat --------------")
-    guide_route_chat(route_guide_stub)
+    # NOTE(gRPC Python Team): .close() is possible on a channel and should be
+    # used in circumstances in which the with statement does not fit the needs
+    # of the code.
+    with grpc.insecure_channel('localhost:50051') as channel:
+        greeter_stub = helloworld_pb2_grpc.GreeterStub(channel)
+        route_guide_stub = route_guide_pb2_grpc.RouteGuideStub(channel)
+        greeter_response = greeter_stub.SayHello(
+            helloworld_pb2.HelloRequest(name='you'))
+        print("Greeter client received: " + greeter_response.message)
+        print("-------------- GetFeature --------------")
+        guide_get_feature(route_guide_stub)
+        print("-------------- ListFeatures --------------")
+        guide_list_features(route_guide_stub)
+        print("-------------- RecordRoute --------------")
+        guide_record_route(route_guide_stub)
+        print("-------------- RouteChat --------------")
+        guide_route_chat(route_guide_stub)
 
 
 if __name__ == '__main__':

examples/python/route_guide/route_guide_client.py

@@ -100,16 +100,19 @@ def guide_route_chat(stub):
 
 
 def run():
-    channel = grpc.insecure_channel('localhost:50051')
-    stub = route_guide_pb2_grpc.RouteGuideStub(channel)
-    print("-------------- GetFeature --------------")
-    guide_get_feature(stub)
-    print("-------------- ListFeatures --------------")
-    guide_list_features(stub)
-    print("-------------- RecordRoute --------------")
-    guide_record_route(stub)
-    print("-------------- RouteChat --------------")
-    guide_route_chat(stub)
+    # NOTE(gRPC Python Team): .close() is possible on a channel and should be
+    # used in circumstances in which the with statement does not fit the needs
+    # of the code.
+    with grpc.insecure_channel('localhost:50051') as channel:
+        stub = route_guide_pb2_grpc.RouteGuideStub(channel)
+        print("-------------- GetFeature --------------")
+        guide_get_feature(stub)
+        print("-------------- ListFeatures --------------")
+        guide_list_features(stub)
+        print("-------------- RecordRoute --------------")
+        guide_record_route(stub)
+        print("-------------- RouteChat --------------")
+        guide_route_chat(stub)
 
 
 if __name__ == '__main__':

examples/ruby/pubsub/.gitignore

@@ -0,0 +1,15 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log
+vendor

examples/ruby/pubsub/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org/'
+
+gem 'grpc', '~> 1.0'
+gem 'googleauth', '>= 0.5.1', '< 0.7'

gRPC-C++.podspec

@@ -23,7 +23,7 @@
 Pod::Spec.new do |s|
   s.name     = 'gRPC-C++'
   # TODO (mxyan): use version that match gRPC version when pod is stabilized
-  # version = '1.12.0-dev'
+  # version = '1.14.0-dev'
   version = '0.0.2'
   s.version  = version
   s.summary  = 'gRPC C++ library'
@@ -31,7 +31,7 @@ Pod::Spec.new do |s|
   s.license  = 'Apache License, Version 2.0'
   s.authors  = { 'The gRPC contributors' => 'grpc-packages@google.com' }
 
-  grpc_version = '1.12.0-dev'
+  grpc_version = '1.14.0-dev'
 
   s.source = {
     :git => 'https://github.com/grpc/grpc.git',
@@ -214,7 +214,6 @@ Pod::Spec.new do |s|
                       'src/cpp/codegen/codegen_init.cc',
                       'src/core/lib/gpr/arena.h',
                       'src/core/lib/gpr/env.h',
-                      'src/core/lib/gpr/fork.h',
                       'src/core/lib/gpr/host_port.h',
                       'src/core/lib/gpr/mpscq.h',
                       'src/core/lib/gpr/murmur_hash.h',
@@ -232,6 +231,7 @@ Pod::Spec.new do |s|
                       'src/core/lib/gprpp/atomic.h',
                       'src/core/lib/gprpp/atomic_with_atm.h',
                       'src/core/lib/gprpp/atomic_with_std.h',
+                      'src/core/lib/gprpp/fork.h',
                       'src/core/lib/gprpp/manual_constructor.h',
                       'src/core/lib/gprpp/memory.h',
                       'src/core/lib/gprpp/thd.h',
@@ -260,6 +260,7 @@ Pod::Spec.new do |s|
                       'src/core/ext/filters/http/client/http_client_filter.h',
                       'src/core/ext/filters/http/message_compress/message_compress_filter.h',
                       'src/core/ext/filters/http/server/http_server_filter.h',
+                      'src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.h',
                       'src/core/lib/security/context/security_context.h',
                       'src/core/lib/security/credentials/alts/alts_credentials.h',
                       'src/core/lib/security/credentials/composite/composite_credentials.h',
@@ -307,7 +308,6 @@ Pod::Spec.new do |s|
                       'src/core/tsi/alts/handshaker/handshaker.pb.h',
                       'src/core/tsi/alts/handshaker/transport_security_common.pb.h',
                       'src/core/tsi/transport_security.h',
-                      'src/core/tsi/transport_security_adapter.h',
                       'src/core/tsi/transport_security_interface.h',
                       'src/core/ext/transport/chttp2/client/authority.h',
                       'src/core/ext/transport/chttp2/client/chttp2_connector.h',
@@ -347,7 +347,7 @@ Pod::Spec.new do |s|
                       'src/core/lib/channel/channel_stack.h',
                       'src/core/lib/channel/channel_stack_builder.h',
                       'src/core/lib/channel/channel_trace.h',
-                      'src/core/lib/channel/channel_trace_registry.h',
+                      'src/core/lib/channel/channelz_registry.h',
                       'src/core/lib/channel/connected_channel.h',
                       'src/core/lib/channel/context.h',
                       'src/core/lib/channel/handshaker.h',
@@ -504,7 +504,6 @@ Pod::Spec.new do |s|
                               'src/cpp/thread_manager/thread_manager.h',
                               'src/core/lib/gpr/arena.h',
                               'src/core/lib/gpr/env.h',
-                              'src/core/lib/gpr/fork.h',
                               'src/core/lib/gpr/host_port.h',
                               'src/core/lib/gpr/mpscq.h',
                               'src/core/lib/gpr/murmur_hash.h',
@@ -522,6 +521,7 @@ Pod::Spec.new do |s|
                               'src/core/lib/gprpp/atomic.h',
                               'src/core/lib/gprpp/atomic_with_atm.h',
                               'src/core/lib/gprpp/atomic_with_std.h',
+                              'src/core/lib/gprpp/fork.h',
                               'src/core/lib/gprpp/manual_constructor.h',
                               'src/core/lib/gprpp/memory.h',
                               'src/core/lib/gprpp/thd.h',
@@ -532,7 +532,7 @@ Pod::Spec.new do |s|
                               'src/core/lib/channel/channel_stack.h',
                               'src/core/lib/channel/channel_stack_builder.h',
                               'src/core/lib/channel/channel_trace.h',
-                              'src/core/lib/channel/channel_trace_registry.h',
+                              'src/core/lib/channel/channelz_registry.h',
                               'src/core/lib/channel/connected_channel.h',
                               'src/core/lib/channel/context.h',
                               'src/core/lib/channel/handshaker.h',
@@ -663,9 +663,9 @@ Pod::Spec.new do |s|
   end
 
   s.prepare_command = <<-END_OF_COMMAND
-    find src/cpp/ -type f -exec sed -E -i'.back' 's;#include "third_party/nanopb/(.*)";#include <nanopb/\\1>;g' {} \\\;
-    find src/cpp/ -name "*.back" -type f -delete
-    find src/core/ -regex ".*\.h" -type f -exec sed -E -i'.back' 's;#include "third_party/nanopb/(.*)";#include <nanopb/\\1>;g' {} \\\;
-    find src/core/ -name "*.back" -type f -delete
+    find src/cpp/ -type f ! -path '*.grpc_back' -print0 | xargs -0 -L1 sed -E -i'.grpc_back' 's;#include "(pb(_.*)?\\.h)";#include <nanopb/\\1>;g'
+    find src/cpp/ -type f -path '*.grpc_back' -print0 | xargs -0 rm
+    find src/core/ -type f ! -path '*.grpc_back' -print0 | xargs -0 -L1 sed -E -i'.grpc_back' 's;#include "(pb(_.*)?\\.h)";#include <nanopb/\\1>;g'
+    find src/core/ -type f -path '*.grpc_back' -print0 | xargs -0 rm
   END_OF_COMMAND
 end

gRPC-Core.podspec

@@ -22,7 +22,7 @@
 
 Pod::Spec.new do |s|
   s.name     = 'gRPC-Core'
-  version = '1.12.0-dev'
+  version = '1.14.0-dev'
   s.version  = version
   s.summary  = 'Core cross-platform gRPC library, written in C'
   s.homepage = 'https://grpc.io'
@@ -185,7 +185,6 @@ Pod::Spec.new do |s|
     # To save you from scrolling, this is the last part of the podspec.
     ss.source_files = 'src/core/lib/gpr/arena.h',
                       'src/core/lib/gpr/env.h',
-                      'src/core/lib/gpr/fork.h',
                       'src/core/lib/gpr/host_port.h',
                       'src/core/lib/gpr/mpscq.h',
                       'src/core/lib/gpr/murmur_hash.h',
@@ -203,6 +202,7 @@ Pod::Spec.new do |s|
                       'src/core/lib/gprpp/atomic.h',
                       'src/core/lib/gprpp/atomic_with_atm.h',
                       'src/core/lib/gprpp/atomic_with_std.h',
+                      'src/core/lib/gprpp/fork.h',
                       'src/core/lib/gprpp/manual_constructor.h',
                       'src/core/lib/gprpp/memory.h',
                       'src/core/lib/gprpp/thd.h',
@@ -217,7 +217,6 @@ Pod::Spec.new do |s|
                       'src/core/lib/gpr/env_linux.cc',
                       'src/core/lib/gpr/env_posix.cc',
                       'src/core/lib/gpr/env_windows.cc',
-                      'src/core/lib/gpr/fork.cc',
                       'src/core/lib/gpr/host_port.cc',
                       'src/core/lib/gpr/log.cc',
                       'src/core/lib/gpr/log_android.cc',
@@ -242,6 +241,7 @@ Pod::Spec.new do |s|
                       'src/core/lib/gpr/tmpfile_posix.cc',
                       'src/core/lib/gpr/tmpfile_windows.cc',
                       'src/core/lib/gpr/wrap_memcpy.cc',
+                      'src/core/lib/gprpp/fork.cc',
                       'src/core/lib/gprpp/thd_posix.cc',
                       'src/core/lib/gprpp/thd_windows.cc',
                       'src/core/lib/profiling/basic_timers.cc',
@@ -270,6 +270,7 @@ Pod::Spec.new do |s|
                       'src/core/ext/filters/http/client/http_client_filter.h',
                       'src/core/ext/filters/http/message_compress/message_compress_filter.h',
                       'src/core/ext/filters/http/server/http_server_filter.h',
+                      'src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.h',
                       'src/core/lib/security/context/security_context.h',
                       'src/core/lib/security/credentials/alts/alts_credentials.h',
                       'src/core/lib/security/credentials/composite/composite_credentials.h',
@@ -317,7 +318,6 @@ Pod::Spec.new do |s|
                       'src/core/tsi/alts/handshaker/handshaker.pb.h',
                       'src/core/tsi/alts/handshaker/transport_security_common.pb.h',
                       'src/core/tsi/transport_security.h',
-                      'src/core/tsi/transport_security_adapter.h',
                       'src/core/tsi/transport_security_interface.h',
                       'src/core/ext/transport/chttp2/client/authority.h',
                       'src/core/ext/transport/chttp2/client/chttp2_connector.h',
@@ -357,7 +357,7 @@ Pod::Spec.new do |s|
                       'src/core/lib/channel/channel_stack.h',
                       'src/core/lib/channel/channel_stack_builder.h',
                       'src/core/lib/channel/channel_trace.h',
-                      'src/core/lib/channel/channel_trace_registry.h',
+                      'src/core/lib/channel/channelz_registry.h',
                       'src/core/lib/channel/connected_channel.h',
                       'src/core/lib/channel/context.h',
                       'src/core/lib/channel/handshaker.h',
@@ -507,7 +507,7 @@ Pod::Spec.new do |s|
                       'src/core/lib/channel/channel_stack.cc',
                       'src/core/lib/channel/channel_stack_builder.cc',
                       'src/core/lib/channel/channel_trace.cc',
-                      'src/core/lib/channel/channel_trace_registry.cc',
+                      'src/core/lib/channel/channelz_registry.cc',
                       'src/core/lib/channel/connected_channel.cc',
                       'src/core/lib/channel/handshaker.cc',
                       'src/core/lib/channel/handshaker_factory.cc',
@@ -736,7 +736,6 @@ Pod::Spec.new do |s|
                       'src/core/tsi/alts/handshaker/handshaker.pb.c',
                       'src/core/tsi/alts/handshaker/transport_security_common.pb.c',
                       'src/core/tsi/transport_security.cc',
-                      'src/core/tsi/transport_security_adapter.cc',
                       'src/core/ext/transport/chttp2/client/insecure/channel_create.cc',
                       'src/core/ext/transport/chttp2/client/insecure/channel_create_posix.cc',
                       'src/core/ext/transport/chttp2/client/authority.cc',
@@ -784,9 +783,9 @@ Pod::Spec.new do |s|
                       'src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c',
                       'src/core/ext/filters/client_channel/resolver/fake/fake_resolver.cc',
                       'src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc',
-                      'src/core/ext/filters/client_channel/lb_policy/subchannel_list.cc',
                       'src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc',
                       'src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc',
+                      'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.cc',
                       'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc',
                       'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc',
                       'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_fallback.cc',
@@ -794,7 +793,7 @@ Pod::Spec.new do |s|
                       'src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.cc',
                       'src/core/ext/filters/load_reporting/server_load_reporting_filter.cc',
                       'src/core/ext/filters/load_reporting/server_load_reporting_plugin.cc',
-                      'src/core/ext/census/grpc_context.cc',
+                      'src/cpp/ext/filters/census/grpc_context.cc',
                       'src/core/ext/filters/max_age/max_age_filter.cc',
                       'src/core/ext/filters/message_size/message_size_filter.cc',
                       'src/core/ext/filters/http/client_authority_filter.cc',
@@ -804,7 +803,6 @@ Pod::Spec.new do |s|
 
     ss.private_header_files = 'src/core/lib/gpr/arena.h',
                               'src/core/lib/gpr/env.h',
-                              'src/core/lib/gpr/fork.h',
                               'src/core/lib/gpr/host_port.h',
                               'src/core/lib/gpr/mpscq.h',
                               'src/core/lib/gpr/murmur_hash.h',
@@ -822,6 +820,7 @@ Pod::Spec.new do |s|
                               'src/core/lib/gprpp/atomic.h',
                               'src/core/lib/gprpp/atomic_with_atm.h',
                               'src/core/lib/gprpp/atomic_with_std.h',
+                              'src/core/lib/gprpp/fork.h',
                               'src/core/lib/gprpp/manual_constructor.h',
                               'src/core/lib/gprpp/memory.h',
                               'src/core/lib/gprpp/thd.h',
@@ -850,6 +849,7 @@ Pod::Spec.new do |s|
                               'src/core/ext/filters/http/client/http_client_filter.h',
                               'src/core/ext/filters/http/message_compress/message_compress_filter.h',
                               'src/core/ext/filters/http/server/http_server_filter.h',
+                              'src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.h',
                               'src/core/lib/security/context/security_context.h',
                               'src/core/lib/security/credentials/alts/alts_credentials.h',
                               'src/core/lib/security/credentials/composite/composite_credentials.h',
@@ -897,7 +897,6 @@ Pod::Spec.new do |s|
                               'src/core/tsi/alts/handshaker/handshaker.pb.h',
                               'src/core/tsi/alts/handshaker/transport_security_common.pb.h',
                               'src/core/tsi/transport_security.h',
-                              'src/core/tsi/transport_security_adapter.h',
                               'src/core/tsi/transport_security_interface.h',
                               'src/core/ext/transport/chttp2/client/authority.h',
                               'src/core/ext/transport/chttp2/client/chttp2_connector.h',
@@ -937,7 +936,7 @@ Pod::Spec.new do |s|
                               'src/core/lib/channel/channel_stack.h',
                               'src/core/lib/channel/channel_stack_builder.h',
                               'src/core/lib/channel/channel_trace.h',
-                              'src/core/lib/channel/channel_trace_registry.h',
+                              'src/core/lib/channel/channelz_registry.h',
                               'src/core/lib/channel/connected_channel.h',
                               'src/core/lib/channel/context.h',
                               'src/core/lib/channel/handshaker.h',
@@ -1082,6 +1081,24 @@ Pod::Spec.new do |s|
                               'src/core/ext/filters/workarounds/workaround_utils.h'
   end
 
+  s.subspec 'CFStream-Implementation' do |ss|
+    ss.header_mappings_dir = '.'
+    ss.dependency "#{s.name}/Implementation", version
+    ss.pod_target_xcconfig = {
+      'GCC_PREPROCESSOR_DEFINITIONS' => 'GRPC_CFSTREAM=1'
+    }
+    ss.source_files = 'src/core/lib/iomgr/cfstream_handle.cc',
+                      'src/core/lib/iomgr/endpoint_cfstream.cc',
+                      'src/core/lib/iomgr/error_cfstream.cc',
+                      'src/core/lib/iomgr/tcp_client_cfstream.cc',
+                      'src/core/lib/iomgr/cfstream_handle.h',
+                      'src/core/lib/iomgr/endpoint_cfstream.h',
+                      'src/core/lib/iomgr/error_cfstream.h'
+    ss.private_header_files = 'src/core/lib/iomgr/cfstream_handle.h',
+                              'src/core/lib/iomgr/endpoint_cfstream.h',
+                              'src/core/lib/iomgr/error_cfstream.h'
+  end
+
   s.subspec 'Cronet-Interface' do |ss|
     ss.header_mappings_dir = 'include/grpc'
     ss.source_files = 'include/grpc/grpc_cronet.h'
@@ -1125,6 +1142,7 @@ Pod::Spec.new do |s|
                       'test/core/end2end/fixtures/proxy.cc',
                       'test/core/iomgr/endpoint_tests.cc',
                       'test/core/util/debugger_macros.cc',
+                      'test/core/util/fuzzer_util.cc',
                       'test/core/util/grpc_profiler.cc',
                       'test/core/util/histogram.cc',
                       'test/core/util/memory_counters.cc',
@@ -1147,6 +1165,7 @@ Pod::Spec.new do |s|
                       'test/core/end2end/fixtures/proxy.h',
                       'test/core/iomgr/endpoint_tests.h',
                       'test/core/util/debugger_macros.h',
+                      'test/core/util/fuzzer_util.h',
                       'test/core/util/grpc_profiler.h',
                       'test/core/util/histogram.h',
                       'test/core/util/memory_counters.h',
@@ -1198,6 +1217,7 @@ Pod::Spec.new do |s|
                       'test/core/end2end/tests/max_message_length.cc',
                       'test/core/end2end/tests/negative_deadline.cc',
                       'test/core/end2end/tests/network_status_change.cc',
+                      'test/core/end2end/tests/no_error_on_hotpath.cc',
                       'test/core/end2end/tests/no_logging.cc',
                       'test/core/end2end/tests/no_op.cc',
                       'test/core/end2end/tests/payload.cc',
@@ -1214,6 +1234,7 @@ Pod::Spec.new do |s|
                       'test/core/end2end/tests/retry_exceeds_buffer_size_in_initial_batch.cc',
                       'test/core/end2end/tests/retry_exceeds_buffer_size_in_subsequent_batch.cc',
                       'test/core/end2end/tests/retry_non_retriable_status.cc',
+                      'test/core/end2end/tests/retry_non_retriable_status_before_recv_trailing_metadata_started.cc',
                       'test/core/end2end/tests/retry_recv_initial_metadata.cc',
                       'test/core/end2end/tests/retry_recv_message.cc',
                       'test/core/end2end/tests/retry_server_pushback_delay.cc',
@@ -1244,6 +1265,7 @@ Pod::Spec.new do |s|
 
   # TODO (mxyan): Instead of this hack, add include path "third_party" to C core's include path?
   s.prepare_command = <<-END_OF_COMMAND
-    find src/core/ -type f ! -path '*.back*' -exec sed -E -i'.back' 's;#include "third_party/nanopb/(.*)";#include <nanopb/\\1>;g' {} \\\;
+    find src/core/ -type f ! -path '*.grpc_back' -print0 | xargs -0 -L1 sed -E -i'.grpc_back' 's;#include "(pb(_.*)?\\.h)";#include <nanopb/\\1>;g'
+    find src/core/ -type f -path '*.grpc_back' -print0 | xargs -0 rm
   END_OF_COMMAND
 end

gRPC-ProtoRPC.podspec

@@ -21,7 +21,7 @@
 
 Pod::Spec.new do |s|
   s.name     = 'gRPC-ProtoRPC'
-  version = '1.12.0-dev'
+  version = '1.14.0-dev'
   s.version  = version
   s.summary  = 'RPC library for Protocol Buffers, based on gRPC'
   s.homepage = 'https://grpc.io'
@@ -41,12 +41,25 @@ Pod::Spec.new do |s|
   s.header_dir = name
 
   src_dir = 'src/objective-c/ProtoRPC'
-  s.source_files = "#{src_dir}/*.{h,m}"
-  s.header_mappings_dir = "#{src_dir}"
 
-  s.dependency 'gRPC', version
-  s.dependency 'gRPC-RxLibrary', version
-  s.dependency 'Protobuf', '~> 3.0'
+  s.default_subspec = 'Main'
+
+  s.subspec 'Main' do |ss|
+    ss.header_mappings_dir = "#{src_dir}"
+    ss.dependency 'gRPC', version
+    ss.dependency 'gRPC-RxLibrary', version
+    ss.dependency 'Protobuf', '~> 3.0'
+
+    ss.source_files = "#{src_dir}/*.{h,m}"
+  end
+  s.subspec 'CFStream' do |ss|
+    ss.dependency 'gRPC/CFStream', version
+    ss.dependency "#{s.name}/Main", version
+    ss.pod_target_xcconfig = {
+      'GCC_PREPROCESSOR_DEFINITIONS' => 'GRPC_CFSTREAM=1'
+    }
+  end
+
   s.pod_target_xcconfig = {
     # This is needed by all pods that depend on Protobuf:
     'GCC_PREPROCESSOR_DEFINITIONS' => '$(inherited) GPB_USE_PROTOBUF_FRAMEWORK_IMPORTS=1',

gRPC-RxLibrary.podspec

@@ -21,7 +21,7 @@
 
 Pod::Spec.new do |s|
   s.name     = 'gRPC-RxLibrary'
-  version = '1.12.0-dev'
+  version = '1.14.0-dev'
   s.version  = version
   s.summary  = 'Reactive Extensions library for iOS/OSX.'
   s.homepage = 'https://grpc.io'

gRPC.podspec

@@ -20,7 +20,7 @@
 
 Pod::Spec.new do |s|
   s.name     = 'gRPC'
-  version = '1.12.0-dev'
+  version = '1.14.0-dev'
   s.version  = version
   s.summary  = 'gRPC client library for iOS/OSX'
   s.homepage = 'https://grpc.io'
@@ -63,6 +63,16 @@ Pod::Spec.new do |s|
     ss.dependency 'gRPC-Core', version
   end
 
+  # This subspec is mutually exclusive with the `Main` subspec
+  s.subspec 'CFStream' do |ss|
+    ss.dependency 'gRPC-Core/CFStream-Implementation', version
+    ss.dependency "#{s.name}/Main", version
+
+    ss.pod_target_xcconfig = {
+      'GCC_PREPROCESSOR_DEFINITIONS' => 'GRPC_CFSTREAM=1'
+    }
+  end
+
   s.subspec 'GID' do |ss|
     ss.ios.deployment_target = '7.0'
 

grpc.gemspec

@@ -20,7 +20,9 @@ Gem::Specification.new do |s|
   s.files += Dir.glob('src/ruby/bin/**/*')
   s.files += Dir.glob('src/ruby/ext/**/*')
   s.files += Dir.glob('src/ruby/lib/**/*')
-  s.files += Dir.glob('src/ruby/pb/**/*')
+  s.files += Dir.glob('src/ruby/pb/**/*').reject do |f|
+    f.match(%r{^src/ruby/pb/test})
+  end
   s.files += Dir.glob('include/grpc/**/*')
   s.test_files = Dir.glob('src/ruby/spec/**/*')
   s.bindir = 'src/ruby/bin'
@@ -28,7 +30,6 @@ Gem::Specification.new do |s|
   s.platform      = Gem::Platform::RUBY
 
   s.add_dependency 'google-protobuf', '~> 3.1'
-  s.add_dependency 'googleauth',      '>= 0.5.1', '< 0.7'
   s.add_dependency 'googleapis-common-protos-types', '~> 1.0.0'
 
   s.add_development_dependency 'bundler',            '~> 1.9'
@@ -41,6 +42,7 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'rspec',              '~> 3.6'
   s.add_development_dependency 'rubocop',            '~> 0.49.1'
   s.add_development_dependency 'signet',             '~> 0.7.0'
+  s.add_development_dependency 'googleauth',         '>= 0.5.1', '< 0.7'
 
   s.extensions = %w(src/ruby/ext/grpc/extconf.rb)
 
@@ -81,7 +83,6 @@ Gem::Specification.new do |s|
   s.files += %w( include/grpc/impl/codegen/sync_windows.h )
   s.files += %w( src/core/lib/gpr/arena.h )
   s.files += %w( src/core/lib/gpr/env.h )
-  s.files += %w( src/core/lib/gpr/fork.h )
   s.files += %w( src/core/lib/gpr/host_port.h )
   s.files += %w( src/core/lib/gpr/mpscq.h )
   s.files += %w( src/core/lib/gpr/murmur_hash.h )
@@ -99,6 +100,7 @@ Gem::Specification.new do |s|
   s.files += %w( src/core/lib/gprpp/atomic.h )
   s.files += %w( src/core/lib/gprpp/atomic_with_atm.h )
   s.files += %w( src/core/lib/gprpp/atomic_with_std.h )
+  s.files += %w( src/core/lib/gprpp/fork.h )
   s.files += %w( src/core/lib/gprpp/manual_constructor.h )
   s.files += %w( src/core/lib/gprpp/memory.h )
   s.files += %w( src/core/lib/gprpp/thd.h )
@@ -113,7 +115,6 @@ Gem::Specification.new do |s|
   s.files += %w( src/core/lib/gpr/env_linux.cc )
   s.files += %w( src/core/lib/gpr/env_posix.cc )
   s.files += %w( src/core/lib/gpr/env_windows.cc )
-  s.files += %w( src/core/lib/gpr/fork.cc )
   s.files += %w( src/core/lib/gpr/host_port.cc )
   s.files += %w( src/core/lib/gpr/log.cc )
   s.files += %w( src/core/lib/gpr/log_android.cc )
@@ -138,6 +139,7 @@ Gem::Specification.new do |s|
   s.files += %w( src/core/lib/gpr/tmpfile_posix.cc )
   s.files += %w( src/core/lib/gpr/tmpfile_windows.cc )
   s.files += %w( src/core/lib/gpr/wrap_memcpy.cc )
+  s.files += %w( src/core/lib/gprpp/fork.cc )
   s.files += %w( src/core/lib/gprpp/thd_posix.cc )
   s.files += %w( src/core/lib/gprpp/thd_windows.cc )
   s.files += %w( src/core/lib/profiling/basic_timers.cc )
@@ -201,6 +203,7 @@ Gem::Specification.new do |s|
   s.files += %w( src/core/ext/filters/http/client/http_client_filter.h )
   s.files += %w( src/core/ext/filters/http/message_compress/message_compress_filter.h )
   s.files += %w( src/core/ext/filters/http/server/http_server_filter.h )
+  s.files += %w( src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.h )
   s.files += %w( src/core/lib/security/context/security_context.h )
   s.files += %w( src/core/lib/security/credentials/alts/alts_credentials.h )
   s.files += %w( src/core/lib/security/credentials/composite/composite_credentials.h )
@@ -252,7 +255,6 @@ Gem::Specification.new do |s|
   s.files += %w( third_party/nanopb/pb_decode.h )
   s.files += %w( third_party/nanopb/pb_encode.h )
   s.files += %w( src/core/tsi/transport_security.h )
-  s.files += %w( src/core/tsi/transport_security_adapter.h )
   s.files += %w( src/core/tsi/transport_security_interface.h )
   s.files += %w( src/core/ext/transport/chttp2/client/authority.h )
   s.files += %w( src/core/ext/transport/chttp2/client/chttp2_connector.h )
@@ -292,7 +294,7 @@ Gem::Specification.new do |s|
   s.files += %w( src/core/lib/channel/channel_stack.h )
   s.files += %w( src/core/lib/channel/channel_stack_builder.h )
   s.files += %w( src/core/lib/channel/channel_trace.h )
-  s.files += %w( src/core/lib/channel/channel_trace_registry.h )
+  s.files += %w( src/core/lib/channel/channelz_registry.h )
   s.files += %w( src/core/lib/channel/connected_channel.h )
   s.files += %w( src/core/lib/channel/context.h )
   s.files += %w( src/core/lib/channel/handshaker.h )
@@ -442,7 +444,7 @@ Gem::Specification.new do |s|
   s.files += %w( src/core/lib/channel/channel_stack.cc )
   s.files += %w( src/core/lib/channel/channel_stack_builder.cc )
   s.files += %w( src/core/lib/channel/channel_trace.cc )
-  s.files += %w( src/core/lib/channel/channel_trace_registry.cc )
+  s.files += %w( src/core/lib/channel/channelz_registry.cc )
   s.files += %w( src/core/lib/channel/connected_channel.cc )
   s.files += %w( src/core/lib/channel/handshaker.cc )
   s.files += %w( src/core/lib/channel/handshaker_factory.cc )
@@ -674,7 +676,6 @@ Gem::Specification.new do |s|
   s.files += %w( third_party/nanopb/pb_decode.c )
   s.files += %w( third_party/nanopb/pb_encode.c )
   s.files += %w( src/core/tsi/transport_security.cc )
-  s.files += %w( src/core/tsi/transport_security_adapter.cc )
   s.files += %w( src/core/ext/transport/chttp2/client/insecure/channel_create.cc )
   s.files += %w( src/core/ext/transport/chttp2/client/insecure/channel_create_posix.cc )
   s.files += %w( src/core/ext/transport/chttp2/client/authority.cc )
@@ -722,9 +723,9 @@ Gem::Specification.new do |s|
   s.files += %w( src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c )
   s.files += %w( src/core/ext/filters/client_channel/resolver/fake/fake_resolver.cc )
   s.files += %w( src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc )
-  s.files += %w( src/core/ext/filters/client_channel/lb_policy/subchannel_list.cc )
   s.files += %w( src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc )
   s.files += %w( src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc )
+  s.files += %w( src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.cc )
   s.files += %w( src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc )
   s.files += %w( src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc )
   s.files += %w( src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_fallback.cc )
@@ -732,7 +733,7 @@ Gem::Specification.new do |s|
   s.files += %w( src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.cc )
   s.files += %w( src/core/ext/filters/load_reporting/server_load_reporting_filter.cc )
   s.files += %w( src/core/ext/filters/load_reporting/server_load_reporting_plugin.cc )
-  s.files += %w( src/core/ext/census/grpc_context.cc )
+  s.files += %w( src/cpp/ext/filters/census/grpc_context.cc )
   s.files += %w( src/core/ext/filters/max_age/max_age_filter.cc )
   s.files += %w( src/core/ext/filters/message_size/message_size_filter.cc )
   s.files += %w( src/core/ext/filters/http/client_authority_filter.cc )
@@ -790,18 +791,18 @@ Gem::Specification.new do |s|
   s.files += %w( third_party/boringssl/crypto/fipsmodule/ec/internal.h )
   s.files += %w( third_party/boringssl/crypto/fipsmodule/ec/oct.c )
   s.files += %w( third_party/boringssl/crypto/fipsmodule/ec/p224-64.c )
-  s.files += %w( third_party/boringssl/crypto/fipsmodule/ec/p256-64.c )
   s.files += %w( third_party/boringssl/crypto/fipsmodule/ec/p256-x86_64-table.h )
   s.files += %w( third_party/boringssl/crypto/fipsmodule/ec/p256-x86_64.c )
   s.files += %w( third_party/boringssl/crypto/fipsmodule/ec/p256-x86_64.h )
   s.files += %w( third_party/boringssl/crypto/fipsmodule/ec/simple.c )
-  s.files += %w( third_party/boringssl/crypto/fipsmodule/ec/util-64.c )
+  s.files += %w( third_party/boringssl/crypto/fipsmodule/ec/util.c )
   s.files += %w( third_party/boringssl/crypto/fipsmodule/ec/wnaf.c )
   s.files += %w( third_party/boringssl/crypto/fipsmodule/ecdsa/ecdsa.c )
   s.files += %w( third_party/boringssl/crypto/fipsmodule/hmac/hmac.c )
   s.files += %w( third_party/boringssl/crypto/fipsmodule/md4/md4.c )
   s.files += %w( third_party/boringssl/crypto/fipsmodule/md5/md5.c )
   s.files += %w( third_party/boringssl/crypto/fipsmodule/modes/cbc.c )
+  s.files += %w( third_party/boringssl/crypto/fipsmodule/modes/ccm.c )
   s.files += %w( third_party/boringssl/crypto/fipsmodule/modes/cfb.c )
   s.files += %w( third_party/boringssl/crypto/fipsmodule/modes/ctr.c )
   s.files += %w( third_party/boringssl/crypto/fipsmodule/modes/gcm.c )
@@ -817,10 +818,13 @@ Gem::Specification.new do |s|
   s.files += %w( third_party/boringssl/crypto/fipsmodule/rsa/padding.c )
   s.files += %w( third_party/boringssl/crypto/fipsmodule/rsa/rsa.c )
   s.files += %w( third_party/boringssl/crypto/fipsmodule/rsa/rsa_impl.c )
+  s.files += %w( third_party/boringssl/crypto/fipsmodule/self_check/self_check.c )
   s.files += %w( third_party/boringssl/crypto/fipsmodule/sha/sha1-altivec.c )
   s.files += %w( third_party/boringssl/crypto/fipsmodule/sha/sha1.c )
   s.files += %w( third_party/boringssl/crypto/fipsmodule/sha/sha256.c )
   s.files += %w( third_party/boringssl/crypto/fipsmodule/sha/sha512.c )
+  s.files += %w( third_party/boringssl/crypto/fipsmodule/tls/internal.h )
+  s.files += %w( third_party/boringssl/crypto/fipsmodule/tls/kdf.c )
   s.files += %w( third_party/boringssl/crypto/internal.h )
   s.files += %w( third_party/boringssl/crypto/obj/obj_dat.h )
   s.files += %w( third_party/boringssl/crypto/pkcs7/internal.h )
@@ -906,7 +910,9 @@ Gem::Specification.new do |s|
   s.files += %w( third_party/boringssl/include/openssl/x509_vfy.h )
   s.files += %w( third_party/boringssl/include/openssl/x509v3.h )
   s.files += %w( third_party/boringssl/ssl/internal.h )
+  s.files += %w( third_party/boringssl/third_party/fiat/curve25519_tables.h )
   s.files += %w( third_party/boringssl/third_party/fiat/internal.h )
+  s.files += %w( third_party/boringssl/third_party/fiat/p256.c )
   s.files += %w( src/boringssl/err_data.c )
   s.files += %w( third_party/boringssl/crypto/asn1/a_bitstr.c )
   s.files += %w( third_party/boringssl/crypto/asn1/a_bool.c )
@@ -959,6 +965,7 @@ Gem::Specification.new do |s|
   s.files += %w( third_party/boringssl/crypto/chacha/chacha.c )
   s.files += %w( third_party/boringssl/crypto/cipher_extra/cipher_extra.c )
   s.files += %w( third_party/boringssl/crypto/cipher_extra/derive_key.c )
+  s.files += %w( third_party/boringssl/crypto/cipher_extra/e_aesccm.c )
   s.files += %w( third_party/boringssl/crypto/cipher_extra/e_aesctrhmac.c )
   s.files += %w( third_party/boringssl/crypto/cipher_extra/e_aesgcmsiv.c )
   s.files += %w( third_party/boringssl/crypto/cipher_extra/e_chacha20poly1305.c )
@@ -970,6 +977,7 @@ Gem::Specification.new do |s|
   s.files += %w( third_party/boringssl/crypto/cipher_extra/tls_cbc.c )
   s.files += %w( third_party/boringssl/crypto/cmac/cmac.c )
   s.files += %w( third_party/boringssl/crypto/conf/conf.c )
+  s.files += %w( third_party/boringssl/crypto/cpu-aarch64-fuchsia.c )
   s.files += %w( third_party/boringssl/crypto/cpu-aarch64-linux.c )
   s.files += %w( third_party/boringssl/crypto/cpu-arm-linux.c )
   s.files += %w( third_party/boringssl/crypto/cpu-arm.c )
@@ -977,7 +985,6 @@ Gem::Specification.new do |s|
   s.files += %w( third_party/boringssl/crypto/cpu-ppc64le.c )
   s.files += %w( third_party/boringssl/crypto/crypto.c )
   s.files += %w( third_party/boringssl/crypto/curve25519/spake25519.c )
-  s.files += %w( third_party/boringssl/crypto/curve25519/x25519-x86_64.c )
   s.files += %w( third_party/boringssl/crypto/dh/check.c )
   s.files += %w( third_party/boringssl/crypto/dh/dh.c )
   s.files += %w( third_party/boringssl/crypto/dh/dh_asn1.c )
@@ -1133,6 +1140,7 @@ Gem::Specification.new do |s|
   s.files += %w( third_party/boringssl/ssl/d1_srtp.cc )
   s.files += %w( third_party/boringssl/ssl/dtls_method.cc )
   s.files += %w( third_party/boringssl/ssl/dtls_record.cc )
+  s.files += %w( third_party/boringssl/ssl/handoff.cc )
   s.files += %w( third_party/boringssl/ssl/handshake.cc )
   s.files += %w( third_party/boringssl/ssl/handshake_client.cc )
   s.files += %w( third_party/boringssl/ssl/handshake_server.cc )

grpc.gyp

@@ -59,6 +59,8 @@
       '-Wno-unused-parameter',
       '-DOSATOMIC_USE_INLINED=1',
       '-Wno-deprecated-declarations',
+      '-Ithird_party/nanopb',
+      '-DPB_FIELD_32BIT',
     ],
     'ldflags': [
       '-g',
@@ -75,6 +77,7 @@
       '.',
       '../..',
       'include',
+      '../../third_party/nanopb',
     ],
     'defines': [
       'GRPC_ARES=0',
@@ -137,6 +140,8 @@
             '-Wno-unused-parameter',
             '-DOSATOMIC_USE_INLINED=1',
             '-Wno-deprecated-declarations',
+            '-Ithird_party/nanopb',
+            '-DPB_FIELD_32BIT',
           ],
           'OTHER_CPLUSPLUSFLAGS': [
             '-g',
@@ -147,6 +152,8 @@
             '-Wno-unused-parameter',
             '-DOSATOMIC_USE_INLINED=1',
             '-Wno-deprecated-declarations',
+            '-Ithird_party/nanopb',
+            '-DPB_FIELD_32BIT',
             '-stdlib=libc++',
             '-std=c++11',
             '-Wno-error=deprecated-declarations',
@@ -194,7 +201,6 @@
         'src/core/lib/gpr/env_linux.cc',
         'src/core/lib/gpr/env_posix.cc',
         'src/core/lib/gpr/env_windows.cc',
-        'src/core/lib/gpr/fork.cc',
         'src/core/lib/gpr/host_port.cc',
         'src/core/lib/gpr/log.cc',
         'src/core/lib/gpr/log_android.cc',
@@ -219,6 +225,7 @@
         'src/core/lib/gpr/tmpfile_posix.cc',
         'src/core/lib/gpr/tmpfile_windows.cc',
         'src/core/lib/gpr/wrap_memcpy.cc',
+        'src/core/lib/gprpp/fork.cc',
         'src/core/lib/gprpp/thd_posix.cc',
         'src/core/lib/gprpp/thd_windows.cc',
         'src/core/lib/profiling/basic_timers.cc',
@@ -249,7 +256,7 @@
         'src/core/lib/channel/channel_stack.cc',
         'src/core/lib/channel/channel_stack_builder.cc',
         'src/core/lib/channel/channel_trace.cc',
-        'src/core/lib/channel/channel_trace_registry.cc',
+        'src/core/lib/channel/channelz_registry.cc',
         'src/core/lib/channel/connected_channel.cc',
         'src/core/lib/channel/handshaker.cc',
         'src/core/lib/channel/handshaker_factory.cc',
@@ -481,7 +488,6 @@
         'third_party/nanopb/pb_decode.c',
         'third_party/nanopb/pb_encode.c',
         'src/core/tsi/transport_security.cc',
-        'src/core/tsi/transport_security_adapter.cc',
         'src/core/ext/transport/chttp2/client/insecure/channel_create.cc',
         'src/core/ext/transport/chttp2/client/insecure/channel_create_posix.cc',
         'src/core/ext/transport/chttp2/client/authority.cc',
@@ -529,9 +535,9 @@
         'src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c',
         'src/core/ext/filters/client_channel/resolver/fake/fake_resolver.cc',
         'src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc',
-        'src/core/ext/filters/client_channel/lb_policy/subchannel_list.cc',
         'src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc',
         'src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc',
+        'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.cc',
         'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc',
         'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc',
         'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_fallback.cc',
@@ -539,7 +545,7 @@
         'src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.cc',
         'src/core/ext/filters/load_reporting/server_load_reporting_filter.cc',
         'src/core/ext/filters/load_reporting/server_load_reporting_plugin.cc',
-        'src/core/ext/census/grpc_context.cc',
+        'src/cpp/ext/filters/census/grpc_context.cc',
         'src/core/ext/filters/max_age/max_age_filter.cc',
         'src/core/ext/filters/message_size/message_size_filter.cc',
         'src/core/ext/filters/http/client_authority_filter.cc',
@@ -578,6 +584,7 @@
         'test/core/end2end/fixtures/proxy.cc',
         'test/core/iomgr/endpoint_tests.cc',
         'test/core/util/debugger_macros.cc',
+        'test/core/util/fuzzer_util.cc',
         'test/core/util/grpc_profiler.cc',
         'test/core/util/histogram.cc',
         'test/core/util/memory_counters.cc',
@@ -599,7 +606,7 @@
         'src/core/lib/channel/channel_stack.cc',
         'src/core/lib/channel/channel_stack_builder.cc',
         'src/core/lib/channel/channel_trace.cc',
-        'src/core/lib/channel/channel_trace_registry.cc',
+        'src/core/lib/channel/channelz_registry.cc',
         'src/core/lib/channel/connected_channel.cc',
         'src/core/lib/channel/handshaker.cc',
         'src/core/lib/channel/handshaker_factory.cc',
@@ -808,6 +815,7 @@
         'test/core/end2end/fixtures/proxy.cc',
         'test/core/iomgr/endpoint_tests.cc',
         'test/core/util/debugger_macros.cc',
+        'test/core/util/fuzzer_util.cc',
         'test/core/util/grpc_profiler.cc',
         'test/core/util/histogram.cc',
         'test/core/util/memory_counters.cc',
@@ -829,7 +837,7 @@
         'src/core/lib/channel/channel_stack.cc',
         'src/core/lib/channel/channel_stack_builder.cc',
         'src/core/lib/channel/channel_trace.cc',
-        'src/core/lib/channel/channel_trace_registry.cc',
+        'src/core/lib/channel/channelz_registry.cc',
         'src/core/lib/channel/connected_channel.cc',
         'src/core/lib/channel/handshaker.cc',
         'src/core/lib/channel/handshaker_factory.cc',
@@ -1038,7 +1046,7 @@
         'src/core/lib/channel/channel_stack.cc',
         'src/core/lib/channel/channel_stack_builder.cc',
         'src/core/lib/channel/channel_trace.cc',
-        'src/core/lib/channel/channel_trace_registry.cc',
+        'src/core/lib/channel/channelz_registry.cc',
         'src/core/lib/channel/connected_channel.cc',
         'src/core/lib/channel/handshaker.cc',
         'src/core/lib/channel/handshaker_factory.cc',
@@ -1240,6 +1248,7 @@
         'src/core/ext/transport/inproc/inproc_plugin.cc',
         'src/core/ext/transport/inproc/inproc_transport.cc',
         'src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc',
+        'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.cc',
         'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc',
         'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc',
         'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_fallback.cc',
@@ -1258,9 +1267,8 @@
         'third_party/nanopb/pb_decode.c',
         'third_party/nanopb/pb_encode.c',
         'src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc',
-        'src/core/ext/filters/client_channel/lb_policy/subchannel_list.cc',
         'src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc',
-        'src/core/ext/census/grpc_context.cc',
+        'src/cpp/ext/filters/census/grpc_context.cc',
         'src/core/ext/filters/max_age/max_age_filter.cc',
         'src/core/ext/filters/message_size/message_size_filter.cc',
         'src/core/ext/filters/http/client_authority_filter.cc',
@@ -1637,6 +1645,16 @@
         'test/cpp/interop/interop_server_bootstrap.cc',
       ],
     },
+    {
+      'target_name': 'lb_load_data_store',
+      'type': 'static_library',
+      'dependencies': [
+        'grpc++',
+      ],
+      'sources': [
+        'src/cpp/server/load_reporter/load_data_store.cc',
+      ],
+    },
     {
       'target_name': 'qps',
       'type': 'static_library',
@@ -1736,6 +1754,7 @@
         'third_party/boringssl/crypto/chacha/chacha.c',
         'third_party/boringssl/crypto/cipher_extra/cipher_extra.c',
         'third_party/boringssl/crypto/cipher_extra/derive_key.c',
+        'third_party/boringssl/crypto/cipher_extra/e_aesccm.c',
         'third_party/boringssl/crypto/cipher_extra/e_aesctrhmac.c',
         'third_party/boringssl/crypto/cipher_extra/e_aesgcmsiv.c',
         'third_party/boringssl/crypto/cipher_extra/e_chacha20poly1305.c',
@@ -1747,6 +1766,7 @@
         'third_party/boringssl/crypto/cipher_extra/tls_cbc.c',
         'third_party/boringssl/crypto/cmac/cmac.c',
         'third_party/boringssl/crypto/conf/conf.c',
+        'third_party/boringssl/crypto/cpu-aarch64-fuchsia.c',
         'third_party/boringssl/crypto/cpu-aarch64-linux.c',
         'third_party/boringssl/crypto/cpu-arm-linux.c',
         'third_party/boringssl/crypto/cpu-arm.c',
@@ -1754,7 +1774,6 @@
         'third_party/boringssl/crypto/cpu-ppc64le.c',
         'third_party/boringssl/crypto/crypto.c',
         'third_party/boringssl/crypto/curve25519/spake25519.c',
-        'third_party/boringssl/crypto/curve25519/x25519-x86_64.c',
         'third_party/boringssl/crypto/dh/check.c',
         'third_party/boringssl/crypto/dh/dh.c',
         'third_party/boringssl/crypto/dh/dh_asn1.c',
@@ -1910,6 +1929,7 @@
         'third_party/boringssl/ssl/d1_srtp.cc',
         'third_party/boringssl/ssl/dtls_method.cc',
         'third_party/boringssl/ssl/dtls_record.cc',
+        'third_party/boringssl/ssl/handoff.cc',
         'third_party/boringssl/ssl/handshake.cc',
         'third_party/boringssl/ssl/handshake_client.cc',
         'third_party/boringssl/ssl/handshake_server.cc',
@@ -2414,6 +2434,17 @@
         'third_party/boringssl/crypto/rsa_extra/rsa_test.cc',
       ],
     },
+    {
+      'target_name': 'boringssl_self_test_lib',
+      'type': 'static_library',
+      'dependencies': [
+        'boringssl_test_util',
+        'boringssl',
+      ],
+      'sources': [
+        'third_party/boringssl/crypto/self_test.cc',
+      ],
+    },
     {
       'target_name': 'boringssl_file_test_gtest_lib',
       'type': 'static_library',
@@ -2608,6 +2639,7 @@
         'test/core/end2end/tests/max_message_length.cc',
         'test/core/end2end/tests/negative_deadline.cc',
         'test/core/end2end/tests/network_status_change.cc',
+        'test/core/end2end/tests/no_error_on_hotpath.cc',
         'test/core/end2end/tests/no_logging.cc',
         'test/core/end2end/tests/no_op.cc',
         'test/core/end2end/tests/payload.cc',
@@ -2624,6 +2656,7 @@
         'test/core/end2end/tests/retry_exceeds_buffer_size_in_initial_batch.cc',
         'test/core/end2end/tests/retry_exceeds_buffer_size_in_subsequent_batch.cc',
         'test/core/end2end/tests/retry_non_retriable_status.cc',
+        'test/core/end2end/tests/retry_non_retriable_status_before_recv_trailing_metadata_started.cc',
         'test/core/end2end/tests/retry_recv_initial_metadata.cc',
         'test/core/end2end/tests/retry_recv_message.cc',
         'test/core/end2end/tests/retry_server_pushback_delay.cc',
@@ -2697,6 +2730,7 @@
         'test/core/end2end/tests/max_message_length.cc',
         'test/core/end2end/tests/negative_deadline.cc',
         'test/core/end2end/tests/network_status_change.cc',
+        'test/core/end2end/tests/no_error_on_hotpath.cc',
         'test/core/end2end/tests/no_logging.cc',
         'test/core/end2end/tests/no_op.cc',
         'test/core/end2end/tests/payload.cc',
@@ -2713,6 +2747,7 @@
         'test/core/end2end/tests/retry_exceeds_buffer_size_in_initial_batch.cc',
         'test/core/end2end/tests/retry_exceeds_buffer_size_in_subsequent_batch.cc',
         'test/core/end2end/tests/retry_non_retriable_status.cc',
+        'test/core/end2end/tests/retry_non_retriable_status_before_recv_trailing_metadata_started.cc',
         'test/core/end2end/tests/retry_recv_initial_metadata.cc',
         'test/core/end2end/tests/retry_recv_message.cc',
         'test/core/end2end/tests/retry_server_pushback_delay.cc',

include/grpc/grpc_security.h

@@ -163,6 +163,26 @@ typedef struct {
   const char* cert_chain;
 } grpc_ssl_pem_key_cert_pair;
 
+/** Object that holds additional peer-verification options on a secure
+   channel. */
+typedef struct {
+  /** If non-NULL this callback will be invoked with the expected
+     target_name, the peer's certificate (in PEM format), and whatever
+     userdata pointer is set below. If a non-zero value is returned by this
+     callback then it is treated as a verification failure. Invocation of
+     the callback is blocking, so any implementation should be light-weight.
+     */
+  int (*verify_peer_callback)(const char* target_name, const char* peer_pem,
+                              void* userdata);
+  /** Arbitrary userdata that will be passed as the last argument to
+     verify_peer_callback. */
+  void* verify_peer_callback_userdata;
+  /** A destruct callback that will be invoked when the channel is being
+     cleaned up. The userdata argument will be passed to it. The intent is
+     to perform any cleanup associated with that userdata. */
+  void (*verify_peer_destruct)(void* userdata);
+} verify_peer_options;
+
 /** Creates an SSL credentials object.
    - pem_root_certs is the NULL-terminated string containing the PEM encoding
      of the server root certificates. If this parameter is NULL, the
@@ -173,10 +193,17 @@ typedef struct {
      disk (in the grpc install directory).
    - pem_key_cert_pair is a pointer on the object containing client's private
      key and certificate chain. This parameter can be NULL if the client does
-     not have such a key/cert pair. */
+     not have such a key/cert pair.
+   - verify_options is an optional verify_peer_options object which holds
+     additional options controlling how peer certificates are verified. For
+     example, you can supply a callback which receives the peer's certificate
+     with which you can do additional verification. Can be NULL, in which
+     case verification will retain default behavior. Any settings in
+     verify_options are copied during this call, so the verify_options
+     object can be released afterwards. */
 GRPCAPI grpc_channel_credentials* grpc_ssl_credentials_create(
     const char* pem_root_certs, grpc_ssl_pem_key_cert_pair* pem_key_cert_pair,
-    void* reserved);
+    const verify_peer_options* verify_options, void* reserved);
 
 /** --- grpc_call_credentials object.
 

include/grpc/impl/codegen/fork.h

@@ -37,12 +37,12 @@
  * }
  */
 
-void grpc_prefork();
+void grpc_prefork(void);
 
-void grpc_postfork_parent();
+void grpc_postfork_parent(void);
 
-void grpc_postfork_child();
+void grpc_postfork_child(void);
 
-void grpc_fork_handlers_auto_register();
+void grpc_fork_handlers_auto_register(void);
 
 #endif /* GRPC_IMPL_CODEGEN_FORK_H */

include/grpc/impl/codegen/grpc_types.h

@@ -332,10 +332,12 @@ typedef struct {
 #define GRPC_ARG_PER_RPC_RETRY_BUFFER_SIZE "grpc.per_rpc_retry_buffer_size"
 /** Channel arg that carries the bridged objective c object for custom metrics
  * logging filter. */
-#define GRPC_ARG_MOBILE_LOG_CONFIG "grpc.mobile_log_config"
+#define GRPC_ARG_MOBILE_LOG_CONTEXT "grpc.mobile_log_context"
 /** If non-zero, client authority filter is disabled for the channel */
 #define GRPC_ARG_DISABLE_CLIENT_AUTHORITY_FILTER \
   "grpc.disable_client_authority_filter"
+/** If set to zero, disables use of http proxies. Enabled by default. */
+#define GRPC_ARG_ENABLE_HTTP_PROXY "grpc.enable_http_proxy"
 /** \} */
 
 /** Result of a grpc call. If the caller satisfies the prerequisites of a

include/grpc/impl/codegen/port_platform.h

@@ -227,7 +227,10 @@
 #define GPR_POSIX_SYNC 1
 #define GPR_POSIX_TIME 1
 #define GPR_GETPID_IN_UNISTD_H 1
+/* TODO(mxyan): Remove when CFStream becomes default */
+#ifndef GRPC_CFSTREAM
 #define GPR_SUPPORT_CHANNELS_FROM_FD 1
+#endif
 #ifdef _LP64
 #define GPR_ARCH_64 1
 #else /* _LP64 */

include/grpc/support/log.h

@@ -99,6 +99,12 @@ GPRAPI void gpr_set_log_function(gpr_log_func func);
     }                                                 \
   } while (0)
 
+#ifndef NDEBUG
+#define GPR_DEBUG_ASSERT(x) GPR_ASSERT(x)
+#else
+#define GPR_DEBUG_ASSERT(x)
+#endif
+
 #ifdef __cplusplus
 }
 #endif

include/grpcpp/impl/codegen/call.h

@@ -573,10 +573,13 @@ class CallOpClientRecvStatus {
       binary_error_details =
           grpc::string(iter->second.begin(), iter->second.length());
     }
-    *recv_status_ = Status(static_cast<StatusCode>(status_code_),
-                           grpc::string(GRPC_SLICE_START_PTR(error_message_),
-                                        GRPC_SLICE_END_PTR(error_message_)),
-                           binary_error_details);
+    *recv_status_ =
+        Status(static_cast<StatusCode>(status_code_),
+               GRPC_SLICE_IS_EMPTY(error_message_)
+                   ? grpc::string()
+                   : grpc::string(GRPC_SLICE_START_PTR(error_message_),
+                                  GRPC_SLICE_END_PTR(error_message_)),
+               binary_error_details);
     client_context_->set_debug_error_string(
         debug_error_string_ != nullptr ? debug_error_string_ : "");
     g_core_codegen_interface->grpc_slice_unref(error_message_);

include/grpcpp/impl/codegen/server_interface.h

@@ -49,12 +49,35 @@ class ServerInterface : public internal::CallHook {
  public:
   virtual ~ServerInterface() {}
 
-  /// Shutdown the server, blocking until all rpc processing finishes.
-  /// Forcefully terminate pending calls after \a deadline expires.
+  /// \a Shutdown does the following things:
+  ///
+  /// 1. Shutdown the server: deactivate all listening ports, mark it in
+  ///    "shutdown mode" so that further call Request's or incoming RPC matches
+  ///    are no longer allowed. Also return all Request'ed-but-not-yet-active
+  ///    calls as failed (!ok). This refers to calls that have been requested
+  ///    at the server by the server-side library or application code but that
+  ///    have not yet been matched to incoming RPCs from the client. Note that
+  ///    this would even include default calls added automatically by the gRPC
+  ///    C++ API without the user's input (e.g., "Unimplemented RPC method")
+  ///
+  /// 2. Block until all rpc method handlers invoked automatically by the sync
+  ///    API finish.
+  ///
+  /// 3. If all pending calls complete (and all their operations are
+  ///    retrieved by Next) before \a deadline expires, this finishes
+  ///    gracefully. Otherwise, forcefully cancel all pending calls associated
+  ///    with the server after \a deadline expires. In the case of the sync API,
+  ///    if the RPC function for a streaming call has already been started and
+  ///    takes a week to complete, the RPC function won't be forcefully
+  ///    terminated (since that would leave state corrupt and incomplete) and
+  ///    the method handler will just keep running (which will prevent the
+  ///    server from completing the "join" operation that it needs to do at
+  ///    shutdown time).
   ///
   /// All completion queue associated with the server (for example, for async
   /// serving) must be shutdown *after* this method has returned:
   /// See \a ServerBuilder::AddCompletionQueue for details.
+  /// They must also be drained (by repeated Next) after being shutdown.
   ///
   /// \param deadline How long to wait until pending rpcs are forcefully
   /// terminated.
@@ -63,7 +86,7 @@ class ServerInterface : public internal::CallHook {
     ShutdownInternal(TimePoint<T>(deadline).raw_time());
   }
 
-  /// Shutdown the server, waiting for all rpc processing to finish.
+  /// Shutdown the server without a deadline and forced cancellation.
   ///
   /// All completion queue associated with the server (for example, for async
   /// serving) must be shutdown *after* this method has returned:

include/grpcpp/opencensus.h

@@ -0,0 +1,41 @@
+/*
+ *
+ * Copyright 2018 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#ifndef GRPCPP_OPENCENSUS_H
+#define GRPCPP_OPENCENSUS_H
+
+namespace grpc {
+// These symbols in this file will not be included in the binary unless
+// grpc_opencensus_plugin build target was added as a dependency. At the moment
+// it is only setup to be built with Bazel.
+
+// Registers the OpenCensus plugin with gRPC, so that it will be used for future
+// RPCs. This must be called before any views are created.
+void RegisterOpenCensusPlugin();
+
+// RPC stats definitions, defined by
+// https://github.com/census-instrumentation/opencensus-specs/blob/master/stats/gRPC.md
+
+// Registers the cumulative gRPC views so that they will be exported by any
+// registered stats exporter. For on-task stats, construct a View using the
+// ViewDescriptors below.
+void RegisterOpenCensusViewsForExport();
+
+}  // namespace grpc
+
+#endif  // GRPCPP_OPENCENSUS_H

include/grpcpp/server_builder.h

@@ -86,8 +86,8 @@ class ServerBuilder {
   /// \param creds The credentials associated with the server.
   /// \param selected_port[out] If not `nullptr`, gets populated with the port
   /// number bound to the \a grpc::Server for the corresponding endpoint after
-  /// it is successfully bound, 0 otherwise.
-  ///
+  /// it is successfully bound by BuildAndStart(), 0 otherwise. AddListeningPort
+  /// does not modify this pointer.
   ServerBuilder& AddListeningPort(const grpc::string& addr_uri,
                                   std::shared_ptr<ServerCredentials> creds,
                                   int* selected_port = nullptr);
@@ -144,12 +144,14 @@ class ServerBuilder {
   // Fine control knobs
 
   /// Set max receive message size in bytes.
+  /// The default is GRPC_DEFAULT_MAX_RECV_MESSAGE_LENGTH.
   ServerBuilder& SetMaxReceiveMessageSize(int max_receive_message_size) {
     max_receive_message_size_ = max_receive_message_size;
     return *this;
   }
 
   /// Set max send message size in bytes.
+  /// The default is GRPC_DEFAULT_MAX_SEND_MESSAGE_LENGTH.
   ServerBuilder& SetMaxSendMessageSize(int max_send_message_size) {
     max_send_message_size_ = max_send_message_size;
     return *this;

include/grpcpp/support/channel_arguments.h

@@ -70,7 +70,13 @@ class ChannelArguments {
   /// the resolver.
   void SetGrpclbFallbackTimeout(int fallback_timeout);
 
-  /// Set the socket mutator for the channel.
+  /// For client channel's, the socket mutator operates on
+  /// "channel" sockets. For server's, the socket mutator operates
+  /// only on "listen" sockets.
+  /// TODO(apolcyn): allow socket mutators to also operate
+  /// on server "channel" sockets, and adjust the socket mutator
+  /// object to be more speficic about which type of socket
+  /// it should operate on.
   void SetSocketMutator(grpc_socket_mutator* mutator);
 
   /// Set the string to prepend to the user agent.

package.xml

@@ -13,8 +13,8 @@
  <date>2018-01-19</date>
  <time>16:06:07</time>
  <version>
-  <release>1.12.0dev</release>
-  <api>1.12.0dev</api>
+  <release>1.14.0dev</release>
+  <api>1.14.0dev</api>
  </version>
  <stability>
   <release>beta</release>
@@ -88,7 +88,6 @@
     <file baseinstalldir="/" name="include/grpc/impl/codegen/sync_windows.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/gpr/arena.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/gpr/env.h" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/gpr/fork.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/gpr/host_port.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/gpr/mpscq.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/gpr/murmur_hash.h" role="src" />
@@ -106,6 +105,7 @@
     <file baseinstalldir="/" name="src/core/lib/gprpp/atomic.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/gprpp/atomic_with_atm.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/gprpp/atomic_with_std.h" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gprpp/fork.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/gprpp/manual_constructor.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/gprpp/memory.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/gprpp/thd.h" role="src" />
@@ -120,7 +120,6 @@
     <file baseinstalldir="/" name="src/core/lib/gpr/env_linux.cc" role="src" />
     <file baseinstalldir="/" name="src/core/lib/gpr/env_posix.cc" role="src" />
     <file baseinstalldir="/" name="src/core/lib/gpr/env_windows.cc" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/gpr/fork.cc" role="src" />
     <file baseinstalldir="/" name="src/core/lib/gpr/host_port.cc" role="src" />
     <file baseinstalldir="/" name="src/core/lib/gpr/log.cc" role="src" />
     <file baseinstalldir="/" name="src/core/lib/gpr/log_android.cc" role="src" />
@@ -145,6 +144,7 @@
     <file baseinstalldir="/" name="src/core/lib/gpr/tmpfile_posix.cc" role="src" />
     <file baseinstalldir="/" name="src/core/lib/gpr/tmpfile_windows.cc" role="src" />
     <file baseinstalldir="/" name="src/core/lib/gpr/wrap_memcpy.cc" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gprpp/fork.cc" role="src" />
     <file baseinstalldir="/" name="src/core/lib/gprpp/thd_posix.cc" role="src" />
     <file baseinstalldir="/" name="src/core/lib/gprpp/thd_windows.cc" role="src" />
     <file baseinstalldir="/" name="src/core/lib/profiling/basic_timers.cc" role="src" />
@@ -208,6 +208,7 @@
     <file baseinstalldir="/" name="src/core/ext/filters/http/client/http_client_filter.h" role="src" />
     <file baseinstalldir="/" name="src/core/ext/filters/http/message_compress/message_compress_filter.h" role="src" />
     <file baseinstalldir="/" name="src/core/ext/filters/http/server/http_server_filter.h" role="src" />
+    <file baseinstalldir="/" name="src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/security/context/security_context.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/security/credentials/alts/alts_credentials.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/security/credentials/composite/composite_credentials.h" role="src" />
@@ -259,7 +260,6 @@
     <file baseinstalldir="/" name="third_party/nanopb/pb_decode.h" role="src" />
     <file baseinstalldir="/" name="third_party/nanopb/pb_encode.h" role="src" />
     <file baseinstalldir="/" name="src/core/tsi/transport_security.h" role="src" />
-    <file baseinstalldir="/" name="src/core/tsi/transport_security_adapter.h" role="src" />
     <file baseinstalldir="/" name="src/core/tsi/transport_security_interface.h" role="src" />
     <file baseinstalldir="/" name="src/core/ext/transport/chttp2/client/authority.h" role="src" />
     <file baseinstalldir="/" name="src/core/ext/transport/chttp2/client/chttp2_connector.h" role="src" />
@@ -299,7 +299,7 @@
     <file baseinstalldir="/" name="src/core/lib/channel/channel_stack.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/channel/channel_stack_builder.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/channel/channel_trace.h" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/channel/channel_trace_registry.h" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/channel/channelz_registry.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/channel/connected_channel.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/channel/context.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/channel/handshaker.h" role="src" />
@@ -449,7 +449,7 @@
     <file baseinstalldir="/" name="src/core/lib/channel/channel_stack.cc" role="src" />
     <file baseinstalldir="/" name="src/core/lib/channel/channel_stack_builder.cc" role="src" />
     <file baseinstalldir="/" name="src/core/lib/channel/channel_trace.cc" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/channel/channel_trace_registry.cc" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/channel/channelz_registry.cc" role="src" />
     <file baseinstalldir="/" name="src/core/lib/channel/connected_channel.cc" role="src" />
     <file baseinstalldir="/" name="src/core/lib/channel/handshaker.cc" role="src" />
     <file baseinstalldir="/" name="src/core/lib/channel/handshaker_factory.cc" role="src" />
@@ -681,7 +681,6 @@
     <file baseinstalldir="/" name="third_party/nanopb/pb_decode.c" role="src" />
     <file baseinstalldir="/" name="third_party/nanopb/pb_encode.c" role="src" />
     <file baseinstalldir="/" name="src/core/tsi/transport_security.cc" role="src" />
-    <file baseinstalldir="/" name="src/core/tsi/transport_security_adapter.cc" role="src" />
     <file baseinstalldir="/" name="src/core/ext/transport/chttp2/client/insecure/channel_create.cc" role="src" />
     <file baseinstalldir="/" name="src/core/ext/transport/chttp2/client/insecure/channel_create_posix.cc" role="src" />
     <file baseinstalldir="/" name="src/core/ext/transport/chttp2/client/authority.cc" role="src" />
@@ -729,9 +728,9 @@
     <file baseinstalldir="/" name="src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c" role="src" />
     <file baseinstalldir="/" name="src/core/ext/filters/client_channel/resolver/fake/fake_resolver.cc" role="src" />
     <file baseinstalldir="/" name="src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc" role="src" />
-    <file baseinstalldir="/" name="src/core/ext/filters/client_channel/lb_policy/subchannel_list.cc" role="src" />
     <file baseinstalldir="/" name="src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc" role="src" />
     <file baseinstalldir="/" name="src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc" role="src" />
+    <file baseinstalldir="/" name="src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.cc" role="src" />
     <file baseinstalldir="/" name="src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc" role="src" />
     <file baseinstalldir="/" name="src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc" role="src" />
     <file baseinstalldir="/" name="src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_fallback.cc" role="src" />
@@ -739,7 +738,7 @@
     <file baseinstalldir="/" name="src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.cc" role="src" />
     <file baseinstalldir="/" name="src/core/ext/filters/load_reporting/server_load_reporting_filter.cc" role="src" />
     <file baseinstalldir="/" name="src/core/ext/filters/load_reporting/server_load_reporting_plugin.cc" role="src" />
-    <file baseinstalldir="/" name="src/core/ext/census/grpc_context.cc" role="src" />
+    <file baseinstalldir="/" name="src/cpp/ext/filters/census/grpc_context.cc" role="src" />
     <file baseinstalldir="/" name="src/core/ext/filters/max_age/max_age_filter.cc" role="src" />
     <file baseinstalldir="/" name="src/core/ext/filters/message_size/message_size_filter.cc" role="src" />
     <file baseinstalldir="/" name="src/core/ext/filters/http/client_authority_filter.cc" role="src" />
@@ -797,18 +796,18 @@
     <file baseinstalldir="/" name="third_party/boringssl/crypto/fipsmodule/ec/internal.h" role="src" />
     <file baseinstalldir="/" name="third_party/boringssl/crypto/fipsmodule/ec/oct.c" role="src" />
     <file baseinstalldir="/" name="third_party/boringssl/crypto/fipsmodule/ec/p224-64.c" role="src" />
-    <file baseinstalldir="/" name="third_party/boringssl/crypto/fipsmodule/ec/p256-64.c" role="src" />
     <file baseinstalldir="/" name="third_party/boringssl/crypto/fipsmodule/ec/p256-x86_64-table.h" role="src" />
     <file baseinstalldir="/" name="third_party/boringssl/crypto/fipsmodule/ec/p256-x86_64.c" role="src" />
     <file baseinstalldir="/" name="third_party/boringssl/crypto/fipsmodule/ec/p256-x86_64.h" role="src" />
     <file baseinstalldir="/" name="third_party/boringssl/crypto/fipsmodule/ec/simple.c" role="src" />
-    <file baseinstalldir="/" name="third_party/boringssl/crypto/fipsmodule/ec/util-64.c" role="src" />
+    <file baseinstalldir="/" name="third_party/boringssl/crypto/fipsmodule/ec/util.c" role="src" />
     <file baseinstalldir="/" name="third_party/boringssl/crypto/fipsmodule/ec/wnaf.c" role="src" />
     <file baseinstalldir="/" name="third_party/boringssl/crypto/fipsmodule/ecdsa/ecdsa.c" role="src" />
     <file baseinstalldir="/" name="third_party/boringssl/crypto/fipsmodule/hmac/hmac.c" role="src" />
     <file baseinstalldir="/" name="third_party/boringssl/crypto/fipsmodule/md4/md4.c" role="src" />
     <file baseinstalldir="/" name="third_party/boringssl/crypto/fipsmodule/md5/md5.c" role="src" />
     <file baseinstalldir="/" name="third_party/boringssl/crypto/fipsmodule/modes/cbc.c" role="src" />
+    <file baseinstalldir="/" name="third_party/boringssl/crypto/fipsmodule/modes/ccm.c" role="src" />
     <file baseinstalldir="/" name="third_party/boringssl/crypto/fipsmodule/modes/cfb.c" role="src" />
     <file baseinstalldir="/" name="third_party/boringssl/crypto/fipsmodule/modes/ctr.c" role="src" />
     <file baseinstalldir="/" name="third_party/boringssl/crypto/fipsmodule/modes/gcm.c" role="src" />
@@ -824,10 +823,13 @@
     <file baseinstalldir="/" name="third_party/boringssl/crypto/fipsmodule/rsa/padding.c" role="src" />
     <file baseinstalldir="/" name="third_party/boringssl/crypto/fipsmodule/rsa/rsa.c" role="src" />
     <file baseinstalldir="/" name="third_party/boringssl/crypto/fipsmodule/rsa/rsa_impl.c" role="src" />
+    <file baseinstalldir="/" name="third_party/boringssl/crypto/fipsmodule/self_check/self_check.c" role="src" />
     <file baseinstalldir="/" name="third_party/boringssl/crypto/fipsmodule/sha/sha1-altivec.c" role="src" />
     <file baseinstalldir="/" name="third_party/boringssl/crypto/fipsmodule/sha/sha1.c" role="src" />
     <file baseinstalldir="/" name="third_party/boringssl/crypto/fipsmodule/sha/sha256.c" role="src" />
     <file baseinstalldir="/" name="third_party/boringssl/crypto/fipsmodule/sha/sha512.c" role="src" />
+    <file baseinstalldir="/" name="third_party/boringssl/crypto/fipsmodule/tls/internal.h" role="src" />
+    <file baseinstalldir="/" name="third_party/boringssl/crypto/fipsmodule/tls/kdf.c" role="src" />
     <file baseinstalldir="/" name="third_party/boringssl/crypto/internal.h" role="src" />
     <file baseinstalldir="/" name="third_party/boringssl/crypto/obj/obj_dat.h" role="src" />
     <file baseinstalldir="/" name="third_party/boringssl/crypto/pkcs7/internal.h" role="src" />
@@ -913,7 +915,9 @@
     <file baseinstalldir="/" name="third_party/boringssl/include/openssl/x509_vfy.h" role="src" />
     <file baseinstalldir="/" name="third_party/boringssl/include/openssl/x509v3.h" role="src" />
     <file baseinstalldir="/" name="third_party/boringssl/ssl/internal.h" role="src" />
+    <file baseinstalldir="/" name="third_party/boringssl/third_party/fiat/curve25519_tables.h" role="src" />
     <file baseinstalldir="/" name="third_party/boringssl/third_party/fiat/internal.h" role="src" />
+    <file baseinstalldir="/" name="third_party/boringssl/third_party/fiat/p256.c" role="src" />
     <file baseinstalldir="/" name="src/boringssl/err_data.c" role="src" />
     <file baseinstalldir="/" name="third_party/boringssl/crypto/asn1/a_bitstr.c" role="src" />
     <file baseinstalldir="/" name="third_party/boringssl/crypto/asn1/a_bool.c" role="src" />
@@ -966,6 +970,7 @@
     <file baseinstalldir="/" name="third_party/boringssl/crypto/chacha/chacha.c" role="src" />
     <file baseinstalldir="/" name="third_party/boringssl/crypto/cipher_extra/cipher_extra.c" role="src" />
     <file baseinstalldir="/" name="third_party/boringssl/crypto/cipher_extra/derive_key.c" role="src" />
+    <file baseinstalldir="/" name="third_party/boringssl/crypto/cipher_extra/e_aesccm.c" role="src" />
     <file baseinstalldir="/" name="third_party/boringssl/crypto/cipher_extra/e_aesctrhmac.c" role="src" />
     <file baseinstalldir="/" name="third_party/boringssl/crypto/cipher_extra/e_aesgcmsiv.c" role="src" />
     <file baseinstalldir="/" name="third_party/boringssl/crypto/cipher_extra/e_chacha20poly1305.c" role="src" />
@@ -977,6 +982,7 @@
     <file baseinstalldir="/" name="third_party/boringssl/crypto/cipher_extra/tls_cbc.c" role="src" />
     <file baseinstalldir="/" name="third_party/boringssl/crypto/cmac/cmac.c" role="src" />
     <file baseinstalldir="/" name="third_party/boringssl/crypto/conf/conf.c" role="src" />
+    <file baseinstalldir="/" name="third_party/boringssl/crypto/cpu-aarch64-fuchsia.c" role="src" />
     <file baseinstalldir="/" name="third_party/boringssl/crypto/cpu-aarch64-linux.c" role="src" />
     <file baseinstalldir="/" name="third_party/boringssl/crypto/cpu-arm-linux.c" role="src" />
     <file baseinstalldir="/" name="third_party/boringssl/crypto/cpu-arm.c" role="src" />
@@ -984,7 +990,6 @@
     <file baseinstalldir="/" name="third_party/boringssl/crypto/cpu-ppc64le.c" role="src" />
     <file baseinstalldir="/" name="third_party/boringssl/crypto/crypto.c" role="src" />
     <file baseinstalldir="/" name="third_party/boringssl/crypto/curve25519/spake25519.c" role="src" />
-    <file baseinstalldir="/" name="third_party/boringssl/crypto/curve25519/x25519-x86_64.c" role="src" />
     <file baseinstalldir="/" name="third_party/boringssl/crypto/dh/check.c" role="src" />
     <file baseinstalldir="/" name="third_party/boringssl/crypto/dh/dh.c" role="src" />
     <file baseinstalldir="/" name="third_party/boringssl/crypto/dh/dh_asn1.c" role="src" />
@@ -1140,6 +1145,7 @@
     <file baseinstalldir="/" name="third_party/boringssl/ssl/d1_srtp.cc" role="src" />
     <file baseinstalldir="/" name="third_party/boringssl/ssl/dtls_method.cc" role="src" />
     <file baseinstalldir="/" name="third_party/boringssl/ssl/dtls_record.cc" role="src" />
+    <file baseinstalldir="/" name="third_party/boringssl/ssl/handoff.cc" role="src" />
     <file baseinstalldir="/" name="third_party/boringssl/ssl/handshake.cc" role="src" />
     <file baseinstalldir="/" name="third_party/boringssl/ssl/handshake_client.cc" role="src" />
     <file baseinstalldir="/" name="third_party/boringssl/ssl/handshake_server.cc" role="src" />

requirements.txt

@@ -2,7 +2,6 @@
 coverage>=4.0
 cython>=0.27
 enum34>=1.0.4
-futures>=2.2.0
 protobuf>=3.5.0.post1
 six>=1.10
 wheel>=0.29

setup.py

@@ -35,8 +35,9 @@ egg_info.manifest_maker.template = 'PYTHON-MANIFEST.in'
 PY3 = sys.version_info.major == 3
 PYTHON_STEM = os.path.join('src', 'python', 'grpcio')
 CORE_INCLUDE = ('include', '.',)
-BORINGSSL_INCLUDE = (os.path.join('third_party', 'boringssl', 'include'),)
+SSL_INCLUDE = (os.path.join('third_party', 'boringssl', 'include'),)
 ZLIB_INCLUDE = (os.path.join('third_party', 'zlib'),)
+NANOPB_INCLUDE = (os.path.join('third_party', 'nanopb'),)
 CARES_INCLUDE = (
     os.path.join('third_party', 'cares'),
     os.path.join('third_party', 'cares', 'cares'),)
@@ -84,6 +85,24 @@ CLASSIFIERS = [
 # present, then it will still attempt to use Cython.
 BUILD_WITH_CYTHON = os.environ.get('GRPC_PYTHON_BUILD_WITH_CYTHON', False)
 
+# Export this variable to use the system installation of openssl. You need to
+# have the header files installed (in /usr/include/openssl) and during
+# runtime, the shared libary must be installed
+BUILD_WITH_SYSTEM_OPENSSL = os.environ.get('GRPC_PYTHON_BUILD_SYSTEM_OPENSSL',
+                                           False)
+
+# Export this variable to use the system installation of zlib. You need to
+# have the header files installed (in /usr/include/) and during
+# runtime, the shared libary must be installed
+BUILD_WITH_SYSTEM_ZLIB = os.environ.get('GRPC_PYTHON_BUILD_SYSTEM_ZLIB',
+                                        False)
+
+# Export this variable to use the system installation of cares. You need to
+# have the header files installed (in /usr/include/) and during
+# runtime, the shared libary must be installed
+BUILD_WITH_SYSTEM_CARES = os.environ.get('GRPC_PYTHON_BUILD_SYSTEM_CARES',
+                                         False)
+
 # Environment variable to determine whether or not to enable coverage analysis
 # in Cython modules.
 ENABLE_CYTHON_TRACING = os.environ.get(
@@ -148,9 +167,22 @@ CORE_C_FILES = tuple(grpc_core_dependencies.CORE_SOURCE_FILES)
 if "win32" in sys.platform:
   CORE_C_FILES = filter(lambda x: 'third_party/cares' not in x, CORE_C_FILES)
 
+if BUILD_WITH_SYSTEM_OPENSSL:
+  CORE_C_FILES = filter(lambda x: 'third_party/boringssl' not in x, CORE_C_FILES)
+  CORE_C_FILES = filter(lambda x: 'src/boringssl' not in x, CORE_C_FILES)
+  SSL_INCLUDE = (os.path.join('/usr', 'include', 'openssl'),)
+
+if BUILD_WITH_SYSTEM_ZLIB:
+  CORE_C_FILES = filter(lambda x: 'third_party/zlib' not in x, CORE_C_FILES)
+  ZLIB_INCLUDE = (os.path.join('/usr', 'include'),)
+
+if BUILD_WITH_SYSTEM_CARES:
+  CORE_C_FILES = filter(lambda x: 'third_party/cares' not in x, CORE_C_FILES)
+  CARES_INCLUDE = (os.path.join('/usr', 'include'),)
+
 EXTENSION_INCLUDE_DIRECTORIES = (
-    (PYTHON_STEM,) + CORE_INCLUDE + BORINGSSL_INCLUDE + ZLIB_INCLUDE +
-    CARES_INCLUDE + ADDRESS_SORTING_INCLUDE)
+    (PYTHON_STEM,) + CORE_INCLUDE + SSL_INCLUDE + ZLIB_INCLUDE +
+    NANOPB_INCLUDE + CARES_INCLUDE + ADDRESS_SORTING_INCLUDE)
 
 EXTENSION_LIBRARIES = ()
 if "linux" in sys.platform:
@@ -159,6 +191,12 @@ if not "win32" in sys.platform:
   EXTENSION_LIBRARIES += ('m',)
 if "win32" in sys.platform:
   EXTENSION_LIBRARIES += ('advapi32', 'ws2_32',)
+if BUILD_WITH_SYSTEM_OPENSSL:
+  EXTENSION_LIBRARIES += ('ssl', 'crypto',)
+if BUILD_WITH_SYSTEM_ZLIB:
+  EXTENSION_LIBRARIES += ('z',)
+if BUILD_WITH_SYSTEM_CARES:
+  EXTENSION_LIBRARIES += ('cares',)
 
 DEFINE_MACROS = (
     ('OPENSSL_NO_ASM', 1), ('_WIN32_WINNT', 0x600),

src/boringssl/err_data.c

@@ -185,43 +185,43 @@ const uint32_t kOpenSSLReasonValues[] = {
     0x28340c5e,
     0x283480ac,
     0x283500ea,
-    0x2c322c6c,
+    0x2c322cf7,
     0x2c3292a5,
-    0x2c332c7a,
-    0x2c33ac8c,
-    0x2c342ca0,
-    0x2c34acb2,
-    0x2c352ccd,
-    0x2c35acdf,
-    0x2c362cf2,
+    0x2c332d05,
+    0x2c33ad17,
+    0x2c342d2b,
+    0x2c34ad3d,
+    0x2c352d58,
+    0x2c35ad6a,
+    0x2c362d7d,
     0x2c36832d,
-    0x2c372cff,
-    0x2c37ad11,
-    0x2c382d36,
-    0x2c38ad4d,
-    0x2c392d5b,
-    0x2c39ad6b,
-    0x2c3a2d7d,
-    0x2c3aad91,
-    0x2c3b2da2,
-    0x2c3badc1,
+    0x2c372d8a,
+    0x2c37ad9c,
+    0x2c382dc1,
+    0x2c38add8,
+    0x2c392de6,
+    0x2c39adf6,
+    0x2c3a2e08,
+    0x2c3aae1c,
+    0x2c3b2e2d,
+    0x2c3bae4c,
     0x2c3c12b7,
     0x2c3c92cd,
-    0x2c3d2dd5,
+    0x2c3d2e60,
     0x2c3d92e6,
-    0x2c3e2df2,
-    0x2c3eae00,
-    0x2c3f2e18,
-    0x2c3fae30,
-    0x2c402e3d,
+    0x2c3e2e7d,
+    0x2c3eae8b,
+    0x2c3f2ea3,
+    0x2c3faebb,
+    0x2c402ec8,
     0x2c4091b8,
-    0x2c412e4e,
-    0x2c41ae61,
+    0x2c412ed9,
+    0x2c41aeec,
     0x2c42117e,
-    0x2c42ae72,
+    0x2c42aefd,
     0x2c430720,
-    0x2c43adb3,
-    0x2c442d24,
+    0x2c43ae3e,
+    0x2c442daf,
     0x30320000,
     0x30328015,
     0x3033001f,
@@ -399,177 +399,182 @@ const uint32_t kOpenSSLReasonValues[] = {
     0x40469c71,
     0x40471c7f,
     0x40479ca6,
-    0x40481ce3,
-    0x40489d16,
-    0x40491d2d,
-    0x40499d47,
-    0x404a1d5e,
-    0x404a9d7c,
-    0x404b1d94,
-    0x404b9dab,
-    0x404c1dc1,
-    0x404c9dd3,
-    0x404d1df4,
-    0x404d9e16,
-    0x404e1e2a,
-    0x404e9e37,
-    0x404f1e64,
-    0x404f9e8d,
-    0x40501ec8,
-    0x40509edc,
-    0x40511ef7,
-    0x40521f07,
-    0x40529f2b,
-    0x40531f43,
-    0x40539f56,
-    0x40541f6b,
-    0x40549f8e,
-    0x40551f9c,
-    0x40559fb9,
-    0x40561fc6,
-    0x40569fdf,
-    0x40571ff7,
-    0x4057a00a,
-    0x4058201f,
-    0x4058a046,
-    0x40592075,
-    0x4059a0a2,
-    0x405a20b6,
-    0x405aa0c6,
-    0x405b20de,
-    0x405ba0ef,
-    0x405c2102,
-    0x405ca141,
-    0x405d214e,
-    0x405da165,
-    0x405e21a3,
+    0x40481cf9,
+    0x40489d2c,
+    0x40491d43,
+    0x40499d5d,
+    0x404a1d74,
+    0x404a9d92,
+    0x404b1daa,
+    0x404b9dc1,
+    0x404c1dd7,
+    0x404c9de9,
+    0x404d1e0a,
+    0x404d9e43,
+    0x404e1e57,
+    0x404e9e64,
+    0x404f1e91,
+    0x404f9eba,
+    0x40501ef5,
+    0x40509f09,
+    0x40511f24,
+    0x40521f34,
+    0x40529f58,
+    0x40531f70,
+    0x40539f83,
+    0x40541f98,
+    0x40549fbb,
+    0x40551fc9,
+    0x4055a006,
+    0x40562013,
+    0x4056a02c,
+    0x40572044,
+    0x4057a057,
+    0x4058206c,
+    0x4058a093,
+    0x405920c2,
+    0x4059a0ef,
+    0x405a2103,
+    0x405aa113,
+    0x405b212b,
+    0x405ba13c,
+    0x405c214f,
+    0x405ca18e,
+    0x405d219b,
+    0x405da1b2,
+    0x405e21f0,
     0x405e8ab1,
-    0x405f21c4,
-    0x405fa1d1,
-    0x406021df,
-    0x4060a201,
-    0x40612245,
-    0x4061a27d,
-    0x40622294,
-    0x4062a2a5,
-    0x406322b6,
-    0x4063a2cb,
-    0x406422e2,
-    0x4064a30e,
-    0x40652329,
-    0x4065a340,
-    0x40662358,
-    0x4066a382,
-    0x406723ad,
-    0x4067a3ce,
-    0x406823f5,
-    0x4068a416,
-    0x40692448,
-    0x4069a476,
-    0x406a2497,
-    0x406aa4b7,
-    0x406b263f,
-    0x406ba662,
-    0x406c2678,
-    0x406ca8f3,
-    0x406d2922,
-    0x406da94a,
-    0x406e2978,
-    0x406ea9c5,
-    0x406f29e4,
-    0x406faa1c,
-    0x40702a2f,
-    0x4070aa4c,
+    0x405f2211,
+    0x405fa21e,
+    0x4060222c,
+    0x4060a24e,
+    0x406122af,
+    0x4061a2e7,
+    0x406222fe,
+    0x4062a30f,
+    0x40632320,
+    0x4063a335,
+    0x4064234c,
+    0x4064a378,
+    0x40652393,
+    0x4065a3aa,
+    0x406623c2,
+    0x4066a3ec,
+    0x40672417,
+    0x4067a438,
+    0x40682480,
+    0x4068a4a1,
+    0x406924d3,
+    0x4069a501,
+    0x406a2522,
+    0x406aa542,
+    0x406b26ca,
+    0x406ba6ed,
+    0x406c2703,
+    0x406ca97e,
+    0x406d29ad,
+    0x406da9d5,
+    0x406e2a03,
+    0x406eaa50,
+    0x406f2a6f,
+    0x406faaa7,
+    0x40702aba,
+    0x4070aad7,
     0x40710800,
-    0x4071aa5e,
-    0x40722a71,
-    0x4072aa8a,
-    0x40732aa2,
+    0x4071aae9,
+    0x40722afc,
+    0x4072ab15,
+    0x40732b2d,
     0x407394a4,
-    0x40742ab6,
-    0x4074aad0,
-    0x40752ae1,
-    0x4075aaf5,
-    0x40762b03,
+    0x40742b41,
+    0x4074ab5b,
+    0x40752b6c,
+    0x4075ab80,
+    0x40762b8e,
     0x4076927b,
-    0x40772b28,
-    0x4077ab4a,
-    0x40782b65,
-    0x4078ab9e,
-    0x40792bb5,
-    0x4079abcb,
-    0x407a2bd7,
-    0x407aabea,
-    0x407b2bff,
-    0x407bac11,
-    0x407c2c42,
-    0x407cac4b,
-    0x407d2431,
-    0x407d9e9d,
-    0x407e2b7a,
-    0x407ea056,
+    0x40772bb3,
+    0x4077abd5,
+    0x40782bf0,
+    0x4078ac29,
+    0x40792c40,
+    0x4079ac56,
+    0x407a2c62,
+    0x407aac75,
+    0x407b2c8a,
+    0x407bac9c,
+    0x407c2ccd,
+    0x407cacd6,
+    0x407d24bc,
+    0x407d9eca,
+    0x407e2c05,
+    0x407ea0a3,
     0x407f1c93,
     0x407f9a53,
-    0x40801e74,
+    0x40801ea1,
     0x40809cbb,
-    0x40811f19,
-    0x40819e4e,
-    0x40822963,
+    0x40811f46,
+    0x40819e7b,
+    0x408229ee,
     0x40829a39,
-    0x40832031,
-    0x4083a2f3,
+    0x4083207e,
+    0x4083a35d,
     0x40841ccf,
-    0x4084a08e,
-    0x40852113,
-    0x4085a229,
-    0x40862185,
-    0x40869eb7,
-    0x408729a9,
-    0x4087a25a,
+    0x4084a0db,
+    0x40852160,
+    0x4085a276,
+    0x408621d2,
+    0x40869ee4,
+    0x40872a34,
+    0x4087a2c4,
     0x40881a9a,
-    0x4088a3e1,
+    0x4088a44b,
     0x40891ae9,
     0x40899a76,
-    0x408a2698,
+    0x408a2723,
     0x408a9884,
-    0x408b2c26,
-    0x408ba9f9,
-    0x408c2123,
+    0x408b2cb1,
+    0x408baa84,
+    0x408c2170,
     0x408c98a0,
-    0x408d1cfc,
-    0x41f4256a,
-    0x41f925fc,
-    0x41fe24ef,
-    0x41fea6e4,
-    0x41ff27d5,
-    0x42032583,
-    0x420825a5,
-    0x4208a5e1,
-    0x420924d3,
-    0x4209a61b,
-    0x420a252a,
-    0x420aa50a,
-    0x420b254a,
-    0x420ba5c3,
-    0x420c27f1,
-    0x420ca6b1,
-    0x420d26cb,
-    0x420da702,
-    0x4212271c,
-    0x421727b8,
-    0x4217a75e,
-    0x421c2780,
-    0x421f273b,
-    0x42212808,
-    0x4226279b,
-    0x422b28d7,
-    0x422ba885,
-    0x422c28bf,
-    0x422ca844,
-    0x422d2823,
-    0x422da8a4,
-    0x422e286a,
-    0x422ea990,
+    0x408d1d12,
+    0x408d9ce3,
+    0x408e1e2c,
+    0x408e9fe6,
+    0x408f245f,
+    0x408fa292,
+    0x41f425f5,
+    0x41f92687,
+    0x41fe257a,
+    0x41fea76f,
+    0x41ff2860,
+    0x4203260e,
+    0x42082630,
+    0x4208a66c,
+    0x4209255e,
+    0x4209a6a6,
+    0x420a25b5,
+    0x420aa595,
+    0x420b25d5,
+    0x420ba64e,
+    0x420c287c,
+    0x420ca73c,
+    0x420d2756,
+    0x420da78d,
+    0x421227a7,
+    0x42172843,
+    0x4217a7e9,
+    0x421c280b,
+    0x421f27c6,
+    0x42212893,
+    0x42262826,
+    0x422b2962,
+    0x422ba910,
+    0x422c294a,
+    0x422ca8cf,
+    0x422d28ae,
+    0x422da92f,
+    0x422e28f5,
+    0x422eaa1b,
     0x4432072b,
     0x4432873a,
     0x44330746,
@@ -622,69 +627,69 @@ const uint32_t kOpenSSLReasonValues[] = {
     0x4c4014c9,
     0x4c4092f7,
     0x4c4114ed,
-    0x50322e84,
-    0x5032ae93,
-    0x50332e9e,
-    0x5033aeae,
-    0x50342ec7,
-    0x5034aee1,
-    0x50352eef,
-    0x5035af05,
-    0x50362f17,
-    0x5036af2d,
-    0x50372f46,
-    0x5037af59,
-    0x50382f71,
-    0x5038af82,
-    0x50392f97,
-    0x5039afab,
-    0x503a2fcb,
-    0x503aafe1,
-    0x503b2ff9,
-    0x503bb00b,
-    0x503c3027,
-    0x503cb03e,
-    0x503d3057,
-    0x503db06d,
-    0x503e307a,
-    0x503eb090,
-    0x503f30a2,
+    0x50322f0f,
+    0x5032af1e,
+    0x50332f29,
+    0x5033af39,
+    0x50342f52,
+    0x5034af6c,
+    0x50352f7a,
+    0x5035af90,
+    0x50362fa2,
+    0x5036afb8,
+    0x50372fd1,
+    0x5037afe4,
+    0x50382ffc,
+    0x5038b00d,
+    0x50393022,
+    0x5039b036,
+    0x503a3056,
+    0x503ab06c,
+    0x503b3084,
+    0x503bb096,
+    0x503c30b2,
+    0x503cb0c9,
+    0x503d30e2,
+    0x503db0f8,
+    0x503e3105,
+    0x503eb11b,
+    0x503f312d,
     0x503f8382,
-    0x504030b5,
-    0x5040b0c5,
-    0x504130df,
-    0x5041b0ee,
-    0x50423108,
-    0x5042b125,
-    0x50433135,
-    0x5043b145,
-    0x50443154,
+    0x50403140,
+    0x5040b150,
+    0x5041316a,
+    0x5041b179,
+    0x50423193,
+    0x5042b1b0,
+    0x504331c0,
+    0x5043b1d0,
+    0x504431df,
     0x5044843f,
-    0x50453168,
-    0x5045b186,
-    0x50463199,
-    0x5046b1af,
-    0x504731c1,
-    0x5047b1d6,
-    0x504831fc,
-    0x5048b20a,
-    0x5049321d,
-    0x5049b232,
-    0x504a3248,
-    0x504ab258,
-    0x504b3278,
-    0x504bb28b,
-    0x504c32ae,
-    0x504cb2dc,
-    0x504d32ee,
-    0x504db30b,
-    0x504e3326,
-    0x504eb342,
-    0x504f3354,
-    0x504fb36b,
-    0x5050337a,
+    0x504531f3,
+    0x5045b211,
+    0x50463224,
+    0x5046b23a,
+    0x5047324c,
+    0x5047b261,
+    0x50483287,
+    0x5048b295,
+    0x504932a8,
+    0x5049b2bd,
+    0x504a32d3,
+    0x504ab2e3,
+    0x504b3303,
+    0x504bb316,
+    0x504c3339,
+    0x504cb367,
+    0x504d3379,
+    0x504db396,
+    0x504e33b1,
+    0x504eb3cd,
+    0x504f33df,
+    0x504fb3f6,
+    0x50503405,
     0x505086ef,
-    0x5051338d,
+    0x50513418,
     0x58320f3a,
     0x68320efc,
     0x68328c6a,
@@ -1098,6 +1103,7 @@ const char kOpenSSLReasonStringData[] =
     "DTLS_MESSAGE_TOO_BIG\0"
     "DUPLICATE_EXTENSION\0"
     "DUPLICATE_KEY_SHARE\0"
+    "EARLY_DATA_NOT_IN_USE\0"
     "ECC_CERT_NOT_FOR_SIGNING\0"
     "EMPTY_HELLO_RETRY_REQUEST\0"
     "EMS_STATE_INCONSISTENT\0"
@@ -1110,6 +1116,7 @@ const char kOpenSSLReasonStringData[] =
     "FRAGMENT_MISMATCH\0"
     "GOT_NEXT_PROTO_WITHOUT_EXTENSION\0"
     "HANDSHAKE_FAILURE_ON_CLIENT_HELLO\0"
+    "HANDSHAKE_NOT_COMPLETE\0"
     "HTTPS_PROXY_REQUEST\0"
     "HTTP_REQUEST\0"
     "INAPPROPRIATE_FALLBACK\0"
@@ -1130,6 +1137,7 @@ const char kOpenSSLReasonStringData[] =
     "MIXED_SPECIAL_OPERATOR_WITH_GROUPS\0"
     "MTU_TOO_SMALL\0"
     "NEGOTIATED_BOTH_NPN_AND_ALPN\0"
+    "NEGOTIATED_TB_WITHOUT_EMS_OR_RI\0"
     "NESTED_GROUP\0"
     "NO_CERTIFICATES_RETURNED\0"
     "NO_CERTIFICATE_ASSIGNED\0"
@@ -1159,6 +1167,7 @@ const char kOpenSSLReasonStringData[] =
     "PEER_DID_NOT_RETURN_A_CERTIFICATE\0"
     "PEER_ERROR_UNSUPPORTED_CERTIFICATE_TYPE\0"
     "PRE_SHARED_KEY_MUST_BE_LAST\0"
+    "PRIVATE_KEY_OPERATION_FAILED\0"
     "PROTOCOL_IS_SHUTDOWN\0"
     "PSK_IDENTITY_BINDER_COUNT_MISMATCH\0"
     "PSK_IDENTITY_NOT_FOUND\0"
@@ -1176,6 +1185,7 @@ const char kOpenSSLReasonStringData[] =
     "SCSV_RECEIVED_WHEN_RENEGOTIATING\0"
     "SERVERHELLO_TLSEXT\0"
     "SERVER_CERT_CHANGED\0"
+    "SERVER_ECHOED_INVALID_SESSION_ID\0"
     "SESSION_ID_CONTEXT_UNINITIALIZED\0"
     "SESSION_MAY_NOT_BE_CREATED\0"
     "SHUTDOWN_WHILE_IN_INIT\0"

src/compiler/csharp_generator.cc

@@ -202,7 +202,8 @@ std::string GetCSharpMethodType(MethodType method_type) {
 std::string GetServiceNameFieldName() { return "__ServiceName"; }
 
 std::string GetMarshallerFieldName(const Descriptor* message) {
-  return "__Marshaller_" + message->name();
+  return "__Marshaller_" +
+         grpc_generator::StringReplace(message->full_name(), ".", "_", true);
 }
 
 std::string GetMethodFieldName(const MethodDescriptor* method) {
@@ -590,19 +591,16 @@ void GenerateBindServiceMethod(Printer* out, const ServiceDescriptor* service) {
   out->Print("{\n");
   out->Indent();
 
-  out->Print("return grpc::ServerServiceDefinition.CreateBuilder()\n");
+  out->Print("return grpc::ServerServiceDefinition.CreateBuilder()");
   out->Indent();
   out->Indent();
   for (int i = 0; i < service->method_count(); i++) {
     const MethodDescriptor* method = service->method(i);
-    out->Print(".AddMethod($methodfield$, serviceImpl.$methodname$)",
+    out->Print("\n.AddMethod($methodfield$, serviceImpl.$methodname$)",
                "methodfield", GetMethodFieldName(method), "methodname",
                method->name());
-    if (i == service->method_count() - 1) {
-      out->Print(".Build();");
-    }
-    out->Print("\n");
   }
+  out->Print(".Build();\n");
   out->Outdent();
   out->Outdent();
 
@@ -676,20 +674,26 @@ grpc::string GetServices(const FileDescriptor* file, bool generate_client,
       out.PrintRaw(leading_comments.c_str());
     }
 
-    out.Print("#pragma warning disable 1591\n");
+    out.Print("#pragma warning disable 0414, 1591\n");
+
     out.Print("#region Designer generated code\n");
     out.Print("\n");
     out.Print("using grpc = global::Grpc.Core;\n");
     out.Print("\n");
 
-    out.Print("namespace $namespace$ {\n", "namespace", GetFileNamespace(file));
-    out.Indent();
+    grpc::string file_namespace = GetFileNamespace(file);
+    if (file_namespace != "") {
+      out.Print("namespace $namespace$ {\n", "namespace", file_namespace);
+      out.Indent();
+    }
     for (int i = 0; i < file->service_count(); i++) {
       GenerateService(&out, file->service(i), generate_client, generate_server,
                       internal_access);
     }
-    out.Outdent();
-    out.Print("}\n");
+    if (file_namespace != "") {
+      out.Outdent();
+      out.Print("}\n");
+    }
     out.Print("#endregion\n");
   }
   return output;

src/compiler/node_generator.cc

@@ -20,6 +20,7 @@
 
 #include "src/compiler/config.h"
 #include "src/compiler/generator_helpers.h"
+#include "src/compiler/node_generator.h"
 #include "src/compiler/node_generator_helpers.h"
 
 using grpc::protobuf::Descriptor;
@@ -119,7 +120,8 @@ grpc::string NodeObjectPath(const Descriptor* descriptor) {
 }
 
 // Prints out the message serializer and deserializer functions
-void PrintMessageTransformer(const Descriptor* descriptor, Printer* out) {
+void PrintMessageTransformer(const Descriptor* descriptor, Printer* out,
+                             const Parameters& params) {
   map<grpc::string, grpc::string> template_vars;
   grpc::string full_name = descriptor->full_name();
   template_vars["identifier_name"] = MessageIdentifierName(full_name);
@@ -134,7 +136,12 @@ void PrintMessageTransformer(const Descriptor* descriptor, Printer* out) {
              "throw new Error('Expected argument of type $name$');\n");
   out->Outdent();
   out->Print("}\n");
-  out->Print("return new Buffer(arg.serializeBinary());\n");
+  if (params.minimum_node_version > 5) {
+    // Node version is > 5, we should use Buffer.from
+    out->Print("return Buffer.from(arg.serializeBinary());\n");
+  } else {
+    out->Print("return new Buffer(arg.serializeBinary());\n");
+  }
   out->Outdent();
   out->Print("}\n\n");
 
@@ -219,12 +226,13 @@ void PrintImports(const FileDescriptor* file, Printer* out) {
   out->Print("\n");
 }
 
-void PrintTransformers(const FileDescriptor* file, Printer* out) {
+void PrintTransformers(const FileDescriptor* file, Printer* out,
+                       const Parameters& params) {
   map<grpc::string, const Descriptor*> messages = GetAllMessages(file);
   for (std::map<grpc::string, const Descriptor*>::iterator it =
            messages.begin();
        it != messages.end(); it++) {
-    PrintMessageTransformer(it->second, out);
+    PrintMessageTransformer(it->second, out, params);
   }
   out->Print("\n");
 }
@@ -236,7 +244,8 @@ void PrintServices(const FileDescriptor* file, Printer* out) {
 }
 }  // namespace
 
-grpc::string GenerateFile(const FileDescriptor* file) {
+grpc::string GenerateFile(const FileDescriptor* file,
+                          const Parameters& params) {
   grpc::string output;
   {
     StringOutputStream output_stream(&output);
@@ -257,7 +266,7 @@ grpc::string GenerateFile(const FileDescriptor* file) {
 
     PrintImports(file, &out);
 
-    PrintTransformers(file, &out);
+    PrintTransformers(file, &out, params);
 
     PrintServices(file, &out);
 

src/compiler/node_generator.h

@@ -23,7 +23,14 @@
 
 namespace grpc_node_generator {
 
-grpc::string GenerateFile(const grpc::protobuf::FileDescriptor* file);
+// Contains all the parameters that are parsed from the command line.
+struct Parameters {
+  // Sets the earliest version of nodejs that needs to be supported.
+  int minimum_node_version;
+};
+
+grpc::string GenerateFile(const grpc::protobuf::FileDescriptor* file,
+                          const Parameters& params);
 
 }  // namespace grpc_node_generator
 

src/compiler/node_plugin.cc

@@ -36,7 +36,27 @@ class NodeGrpcGenerator : public grpc::protobuf::compiler::CodeGenerator {
                 const grpc::string& parameter,
                 grpc::protobuf::compiler::GeneratorContext* context,
                 grpc::string* error) const {
-    grpc::string code = GenerateFile(file);
+    grpc_node_generator::Parameters generator_parameters;
+    generator_parameters.minimum_node_version = 4;
+
+    if (!parameter.empty()) {
+      std::vector<grpc::string> parameters_list =
+          grpc_generator::tokenize(parameter, ",");
+      for (auto parameter_string = parameters_list.begin();
+           parameter_string != parameters_list.end(); parameter_string++) {
+        std::vector<grpc::string> param =
+            grpc_generator::tokenize(*parameter_string, "=");
+        if (param[0] == "minimum_node_version") {
+          sscanf(param[1].c_str(), "%d",
+                 &generator_parameters.minimum_node_version);
+        } else {
+          *error = grpc::string("Unknown parameter: ") + *parameter_string;
+          return false;
+        }
+      }
+    }
+
+    grpc::string code = GenerateFile(file, generator_parameters);
     if (code.size() == 0) {
       return true;
     }

src/core/ext/filters/client_channel/channel_connectivity.cc

@@ -40,7 +40,7 @@ grpc_connectivity_state grpc_channel_check_connectivity_state(
   GRPC_API_TRACE(
       "grpc_channel_check_connectivity_state(channel=%p, try_to_connect=%d)", 2,
       (channel, try_to_connect));
-  if (client_channel_elem->filter == &grpc_client_channel_filter) {
+  if (GPR_LIKELY(client_channel_elem->filter == &grpc_client_channel_filter)) {
     state = grpc_client_channel_check_connectivity_state(client_channel_elem,
                                                          try_to_connect);
 

src/core/ext/filters/client_channel/http_proxy.cc

@@ -37,17 +37,21 @@
 #include "src/core/lib/slice/b64.h"
 
 /**
- * Parses the 'http_proxy' env var and returns the proxy hostname to resolve or
- * nullptr on error. Also sets 'user_cred' to user credentials if present in the
- * 'http_proxy' env var, otherwise leaves it unchanged. It is caller's
- * responsibility to gpr_free user_cred.
+ * Parses the 'https_proxy' env var (fallback on 'http_proxy') and returns the
+ * proxy hostname to resolve or nullptr on error. Also sets 'user_cred' to user
+ * credentials if present in the 'http_proxy' env var, otherwise leaves it
+ * unchanged. It is caller's responsibility to gpr_free user_cred.
  */
 static char* get_http_proxy_server(char** user_cred) {
   GPR_ASSERT(user_cred != nullptr);
   char* proxy_name = nullptr;
-  char* uri_str = gpr_getenv("http_proxy");
   char** authority_strs = nullptr;
   size_t authority_nstrs;
+  /* Prefer using 'https_proxy'. Fallback on 'http_proxy' if it is not set. The
+   * fallback behavior can be removed if there's a demand for it.
+   */
+  char* uri_str = gpr_getenv("https_proxy");
+  if (uri_str == nullptr) uri_str = gpr_getenv("http_proxy");
   if (uri_str == nullptr) return nullptr;
   grpc_uri* uri = grpc_uri_parse(uri_str, false /* suppress_errors */);
   if (uri == nullptr || uri->authority == nullptr) {
@@ -83,11 +87,24 @@ done:
   return proxy_name;
 }
 
+/**
+ * Checks the value of GRPC_ARG_ENABLE_HTTP_PROXY to determine if http_proxy
+ * should be used.
+ */
+bool http_proxy_enabled(const grpc_channel_args* args) {
+  const grpc_arg* arg =
+      grpc_channel_args_find(args, GRPC_ARG_ENABLE_HTTP_PROXY);
+  return grpc_channel_arg_get_bool(arg, true);
+}
+
 static bool proxy_mapper_map_name(grpc_proxy_mapper* mapper,
                                   const char* server_uri,
                                   const grpc_channel_args* args,
                                   char** name_to_resolve,
                                   grpc_channel_args** new_args) {
+  if (!http_proxy_enabled(args)) {
+    return false;
+  }
   char* user_cred = nullptr;
   *name_to_resolve = get_http_proxy_server(&user_cred);
   if (*name_to_resolve == nullptr) return false;

src/core/ext/filters/client_channel/lb_policy.cc

@@ -44,13 +44,13 @@ void LoadBalancingPolicy::TryReresolutionLocked(
     GRPC_CLOSURE_SCHED(request_reresolution_, error);
     request_reresolution_ = nullptr;
     if (grpc_lb_trace->enabled()) {
-      gpr_log(GPR_DEBUG,
+      gpr_log(GPR_INFO,
               "%s %p: scheduling re-resolution closure with error=%s.",
               grpc_lb_trace->name(), this, grpc_error_string(error));
     }
   } else {
     if (grpc_lb_trace->enabled()) {
-      gpr_log(GPR_DEBUG, "%s %p: no available re-resolution closure.",
+      gpr_log(GPR_INFO, "%s %p: no available re-resolution closure.",
               grpc_lb_trace->name(), this);
     }
   }

src/core/ext/filters/client_channel/lb_policy.h

@@ -162,6 +162,8 @@ class LoadBalancingPolicy
   GRPC_ABSTRACT_BASE_CLASS
 
  protected:
+  GPRC_ALLOW_CLASS_TO_USE_NON_PUBLIC_DELETE
+
   explicit LoadBalancingPolicy(const Args& args);
   virtual ~LoadBalancingPolicy();
 

src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.cc

@@ -35,9 +35,10 @@ static grpc_error* init_channel_elem(grpc_channel_element* elem,
 static void destroy_channel_elem(grpc_channel_element* elem) {}
 
 namespace {
+
 struct call_data {
   // Stats object to update.
-  grpc_grpclb_client_stats* client_stats;
+  grpc_core::RefCountedPtr<grpc_core::GrpcLbClientStats> client_stats;
   // State for intercepting send_initial_metadata.
   grpc_closure on_complete_for_send;
   grpc_closure* original_on_complete_for_send;
@@ -47,6 +48,7 @@ struct call_data {
   grpc_closure* original_recv_initial_metadata_ready;
   bool recv_initial_metadata_succeeded;
 };
+
 }  // namespace
 
 static void on_complete_for_send(void* arg, grpc_error* error) {
@@ -72,11 +74,11 @@ static grpc_error* init_call_elem(grpc_call_element* elem,
   // Get stats object from context and take a ref.
   GPR_ASSERT(args->context != nullptr);
   if (args->context[GRPC_GRPCLB_CLIENT_STATS].value != nullptr) {
-    calld->client_stats =
-        grpc_grpclb_client_stats_ref(static_cast<grpc_grpclb_client_stats*>(
-            args->context[GRPC_GRPCLB_CLIENT_STATS].value));
+    calld->client_stats = static_cast<grpc_core::GrpcLbClientStats*>(
+                              args->context[GRPC_GRPCLB_CLIENT_STATS].value)
+                              ->Ref();
     // Record call started.
-    grpc_grpclb_client_stats_add_call_started(calld->client_stats);
+    calld->client_stats->AddCallStarted();
   }
   return GRPC_ERROR_NONE;
 }
@@ -88,12 +90,12 @@ static void destroy_call_elem(grpc_call_element* elem,
   if (calld->client_stats != nullptr) {
     // Record call finished, optionally setting client_failed_to_send and
     // received.
-    grpc_grpclb_client_stats_add_call_finished(
+    calld->client_stats->AddCallFinished(
         !calld->send_initial_metadata_succeeded /* client_failed_to_send */,
-        calld->recv_initial_metadata_succeeded /* known_received */,
-        calld->client_stats);
+        calld->recv_initial_metadata_succeeded /* known_received */);
     // All done, so unref the stats object.
-    grpc_grpclb_client_stats_unref(calld->client_stats);
+    // TODO(roth): Eliminate this once filter stack is converted to C++.
+    calld->client_stats.reset();
   }
 }
 

src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc

@@ -76,6 +76,7 @@
 #include "src/core/ext/filters/client_channel/client_channel.h"
 #include "src/core/ext/filters/client_channel/client_channel_factory.h"
 #include "src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.h"
+#include "src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.h"
 #include "src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_channel.h"
 #include "src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.h"
 #include "src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.h"
@@ -158,9 +159,8 @@ class GrpcLb : public LoadBalancingPolicy {
     // The LB token associated with the pick.  This is set via user_data in
     // the pick.
     grpc_mdelem lb_token;
-    // Stats for client-side load reporting. Note that this holds a
-    // reference, which must be either passed on via context or unreffed.
-    grpc_grpclb_client_stats* client_stats = nullptr;
+    // Stats for client-side load reporting.
+    RefCountedPtr<GrpcLbClientStats> client_stats;
     // Next pending pick.
     PendingPick* next = nullptr;
   };
@@ -185,10 +185,15 @@ class GrpcLb : public LoadBalancingPolicy {
 
     void StartQuery();
 
-    grpc_grpclb_client_stats* client_stats() const { return client_stats_; }
+    GrpcLbClientStats* client_stats() const { return client_stats_.get(); }
+
     bool seen_initial_response() const { return seen_initial_response_; }
 
    private:
+    // So Delete() can access our private dtor.
+    template <typename T>
+    friend void grpc_core::Delete(T*);
+
     ~BalancerCallState();
 
     GrpcLb* grpclb_policy() const {
@@ -232,7 +237,7 @@ class GrpcLb : public LoadBalancingPolicy {
 
     // The stats for client-side load reporting associated with this LB call.
     // Created after the first serverlist is received.
-    grpc_grpclb_client_stats* client_stats_ = nullptr;
+    RefCountedPtr<GrpcLbClientStats> client_stats_;
     grpc_millis client_stats_report_interval_ = 0;
     grpc_timer client_load_report_timer_;
     bool client_load_report_timer_callback_pending_ = false;
@@ -394,7 +399,7 @@ grpc_lb_addresses* ExtractBackendAddresses(const grpc_lb_addresses* addresses) {
 bool IsServerValid(const grpc_grpclb_server* server, size_t idx, bool log) {
   if (server->drop) return false;
   const grpc_grpclb_ip_address* ip = &server->ip_address;
-  if (server->port >> 16 != 0) {
+  if (GPR_UNLIKELY(server->port >> 16 != 0)) {
     if (log) {
       gpr_log(GPR_ERROR,
               "Invalid port '%d' at index %lu of serverlist. Ignoring.",
@@ -402,7 +407,7 @@ bool IsServerValid(const grpc_grpclb_server* server, size_t idx, bool log) {
     }
     return false;
   }
-  if (ip->size != 4 && ip->size != 16) {
+  if (GPR_UNLIKELY(ip->size != 4 && ip->size != 16)) {
     if (log) {
       gpr_log(GPR_ERROR,
               "Expected IP to be 4 or 16 bytes, got %d at index %lu of "
@@ -543,9 +548,6 @@ GrpcLb::BalancerCallState::~BalancerCallState() {
   grpc_byte_buffer_destroy(send_message_payload_);
   grpc_byte_buffer_destroy(recv_message_payload_);
   grpc_slice_unref_internal(lb_call_status_details_);
-  if (client_stats_ != nullptr) {
-    grpc_grpclb_client_stats_unref(client_stats_);
-  }
 }
 
 void GrpcLb::BalancerCallState::Orphan() {
@@ -668,22 +670,22 @@ void GrpcLb::BalancerCallState::MaybeSendClientLoadReportLocked(
 
 bool GrpcLb::BalancerCallState::LoadReportCountersAreZero(
     grpc_grpclb_request* request) {
-  grpc_grpclb_dropped_call_counts* drop_entries =
-      static_cast<grpc_grpclb_dropped_call_counts*>(
+  GrpcLbClientStats::DroppedCallCounts* drop_entries =
+      static_cast<GrpcLbClientStats::DroppedCallCounts*>(
           request->client_stats.calls_finished_with_drop.arg);
   return request->client_stats.num_calls_started == 0 &&
          request->client_stats.num_calls_finished == 0 &&
          request->client_stats.num_calls_finished_with_client_failed_to_send ==
              0 &&
          request->client_stats.num_calls_finished_known_received == 0 &&
-         (drop_entries == nullptr || drop_entries->num_entries == 0);
+         (drop_entries == nullptr || drop_entries->size() == 0);
 }
 
 void GrpcLb::BalancerCallState::SendClientLoadReportLocked() {
   // Construct message payload.
   GPR_ASSERT(send_message_payload_ == nullptr);
   grpc_grpclb_request* request =
-      grpc_grpclb_load_report_request_create_locked(client_stats_);
+      grpc_grpclb_load_report_request_create_locked(client_stats_.get());
   // Skip client load report if the counters were all zero in the last
   // report and they are still zero in this one.
   if (LoadReportCountersAreZero(request)) {
@@ -710,7 +712,7 @@ void GrpcLb::BalancerCallState::SendClientLoadReportLocked() {
                     this, grpc_combiner_scheduler(grpclb_policy()->combiner()));
   grpc_call_error call_error = grpc_call_start_batch_and_execute(
       lb_call_, &op, 1, &client_load_report_closure_);
-  if (call_error != GRPC_CALL_OK) {
+  if (GPR_UNLIKELY(call_error != GRPC_CALL_OK)) {
     gpr_log(GPR_ERROR, "[grpclb %p] call_error=%d", grpclb_policy_.get(),
             call_error);
     GPR_ASSERT(GRPC_CALL_OK == call_error);
@@ -774,7 +776,7 @@ void GrpcLb::BalancerCallState::OnBalancerMessageReceivedLocked(
       if (grpc_lb_glb_trace.enabled()) {
         gpr_log(GPR_INFO,
                 "[grpclb %p] Received initial LB response message; "
-                "client load reporting interval = %" PRIdPTR " milliseconds",
+                "client load reporting interval = %" PRId64 " milliseconds",
                 grpclb_policy, lb_calld->client_stats_report_interval_);
       }
     } else if (grpc_lb_glb_trace.enabled()) {
@@ -809,7 +811,7 @@ void GrpcLb::BalancerCallState::OnBalancerMessageReceivedLocked(
       // serverlist returned from the current LB call.
       if (lb_calld->client_stats_report_interval_ > 0 &&
           lb_calld->client_stats_ == nullptr) {
-        lb_calld->client_stats_ = grpc_grpclb_client_stats_create();
+        lb_calld->client_stats_.reset(New<GrpcLbClientStats>());
         // TODO(roth): We currently track this ref manually.  Once the
         // ClosureRef API is ready, we should pass the RefCountedPtr<> along
         // with the callback.
@@ -932,7 +934,7 @@ grpc_lb_addresses* ExtractBalancerAddresses(
   size_t lb_addresses_idx = 0;
   for (size_t i = 0; i < addresses->num_addresses; ++i) {
     if (!addresses->addresses[i].is_balancer) continue;
-    if (addresses->addresses[i].user_data != nullptr) {
+    if (GPR_UNLIKELY(addresses->addresses[i].user_data != nullptr)) {
       gpr_log(GPR_ERROR,
               "This LB policy doesn't support user data. It will be ignored");
     }
@@ -999,6 +1001,9 @@ grpc_channel_args* BuildBalancerChannelArgs(
       // address updates into the LB channel.
       grpc_core::FakeResolverResponseGenerator::MakeChannelArg(
           response_generator),
+      // A channel arg indicating the target is a grpclb load balancer.
+      grpc_channel_arg_integer_create(
+          const_cast<char*>(GRPC_ARG_ADDRESS_IS_GRPCLB_LOAD_BALANCER), 1),
   };
   // Construct channel args.
   grpc_channel_args* new_args = grpc_channel_args_copy_and_add_and_remove(
@@ -1243,7 +1248,7 @@ bool GrpcLb::PickLocked(PickState* pick) {
     }
   } else {  // rr_policy_ == NULL
     if (grpc_lb_glb_trace.enabled()) {
-      gpr_log(GPR_DEBUG,
+      gpr_log(GPR_INFO,
               "[grpclb %p] No RR policy. Adding to grpclb's pending picks",
               this);
     }
@@ -1280,7 +1285,7 @@ void GrpcLb::NotifyOnStateChangeLocked(grpc_connectivity_state* current,
 
 void GrpcLb::ProcessChannelArgsLocked(const grpc_channel_args& args) {
   const grpc_arg* arg = grpc_channel_args_find(&args, GRPC_ARG_LB_ADDRESSES);
-  if (arg == nullptr || arg->type != GRPC_ARG_POINTER) {
+  if (GPR_UNLIKELY(arg == nullptr || arg->type != GRPC_ARG_POINTER)) {
     // Ignore this update.
     gpr_log(
         GPR_ERROR,
@@ -1409,14 +1414,13 @@ void GrpcLb::OnFallbackTimerLocked(void* arg, grpc_error* error) {
 void GrpcLb::StartBalancerCallRetryTimerLocked() {
   grpc_millis next_try = lb_call_backoff_.NextAttemptTime();
   if (grpc_lb_glb_trace.enabled()) {
-    gpr_log(GPR_DEBUG, "[grpclb %p] Connection to LB server lost...", this);
+    gpr_log(GPR_INFO, "[grpclb %p] Connection to LB server lost...", this);
     grpc_millis timeout = next_try - ExecCtx::Get()->Now();
     if (timeout > 0) {
-      gpr_log(GPR_DEBUG,
-              "[grpclb %p] ... retry_timer_active in %" PRIuPTR "ms.", this,
-              timeout);
+      gpr_log(GPR_INFO, "[grpclb %p] ... retry_timer_active in %" PRId64 "ms.",
+              this, timeout);
     } else {
-      gpr_log(GPR_DEBUG, "[grpclb %p] ... retry_timer_active immediately.",
+      gpr_log(GPR_INFO, "[grpclb %p] ... retry_timer_active immediately.",
               this);
     }
   }
@@ -1509,7 +1513,7 @@ grpc_error* AddLbTokenToInitialMetadata(
 
 // Destroy function used when embedding client stats in call context.
 void DestroyClientStats(void* arg) {
-  grpc_grpclb_client_stats_unref(static_cast<grpc_grpclb_client_stats*>(arg));
+  static_cast<GrpcLbClientStats*>(arg)->Unref();
 }
 
 void GrpcLb::PendingPickSetMetadataAndContext(PendingPick* pp) {
@@ -1517,7 +1521,7 @@ void GrpcLb::PendingPickSetMetadataAndContext(PendingPick* pp) {
    * policy (e.g., all addresses failed to connect). There won't be any
    * user_data/token available */
   if (pp->pick->connected_subchannel != nullptr) {
-    if (!GRPC_MDISNULL(pp->lb_token)) {
+    if (GPR_LIKELY(!GRPC_MDISNULL(pp->lb_token))) {
       AddLbTokenToInitialMetadata(GRPC_MDELEM_REF(pp->lb_token),
                                   &pp->pick->lb_token_mdelem_storage,
                                   pp->pick->initial_metadata);
@@ -1530,14 +1534,12 @@ void GrpcLb::PendingPickSetMetadataAndContext(PendingPick* pp) {
     // Pass on client stats via context. Passes ownership of the reference.
     if (pp->client_stats != nullptr) {
       pp->pick->subchannel_call_context[GRPC_GRPCLB_CLIENT_STATS].value =
-          pp->client_stats;
+          pp->client_stats.release();
       pp->pick->subchannel_call_context[GRPC_GRPCLB_CLIENT_STATS].destroy =
           DestroyClientStats;
     }
   } else {
-    if (pp->client_stats != nullptr) {
-      grpc_grpclb_client_stats_unref(pp->client_stats);
-    }
+    pp->client_stats.reset();
   }
 }
 
@@ -1603,8 +1605,8 @@ bool GrpcLb::PickFromRoundRobinPolicyLocked(bool force_async, PendingPick* pp) {
       // subchannel call (and therefore no client_load_reporting filter)
       // for dropped calls.
       if (lb_calld_ != nullptr && lb_calld_->client_stats() != nullptr) {
-        grpc_grpclb_client_stats_add_call_dropped_locked(
-            server->load_balance_token, lb_calld_->client_stats());
+        lb_calld_->client_stats()->AddCallDroppedLocked(
+            server->load_balance_token);
       }
       if (force_async) {
         GRPC_CLOSURE_SCHED(pp->original_on_complete, GRPC_ERROR_NONE);
@@ -1617,7 +1619,7 @@ bool GrpcLb::PickFromRoundRobinPolicyLocked(bool force_async, PendingPick* pp) {
   }
   // Set client_stats and user_data.
   if (lb_calld_ != nullptr && lb_calld_->client_stats() != nullptr) {
-    pp->client_stats = grpc_grpclb_client_stats_ref(lb_calld_->client_stats());
+    pp->client_stats = lb_calld_->client_stats()->Ref();
   }
   GPR_ASSERT(pp->pick->user_data == nullptr);
   pp->pick->user_data = (void**)&pp->lb_token;
@@ -1642,7 +1644,7 @@ void GrpcLb::CreateRoundRobinPolicyLocked(const Args& args) {
   GPR_ASSERT(rr_policy_ == nullptr);
   rr_policy_ = LoadBalancingPolicyRegistry::CreateLoadBalancingPolicy(
       "round_robin", args);
-  if (rr_policy_ == nullptr) {
+  if (GPR_UNLIKELY(rr_policy_ == nullptr)) {
     gpr_log(GPR_ERROR, "[grpclb %p] Failure creating a RoundRobin policy",
             this);
     return;
@@ -1695,9 +1697,11 @@ void GrpcLb::CreateRoundRobinPolicyLocked(const Args& args) {
 
 grpc_channel_args* GrpcLb::CreateRoundRobinPolicyArgsLocked() {
   grpc_lb_addresses* addresses;
+  bool is_backend_from_grpclb_load_balancer = false;
   if (serverlist_ != nullptr) {
     GPR_ASSERT(serverlist_->num_servers > 0);
     addresses = ProcessServerlist(serverlist_);
+    is_backend_from_grpclb_load_balancer = true;
   } else {
     // If CreateOrUpdateRoundRobinPolicyLocked() is invoked when we haven't
     // received any serverlist from the balancer, we use the fallback backends
@@ -1711,9 +1715,18 @@ grpc_channel_args* GrpcLb::CreateRoundRobinPolicyArgsLocked() {
   // Replace the LB addresses in the channel args that we pass down to
   // the subchannel.
   static const char* keys_to_remove[] = {GRPC_ARG_LB_ADDRESSES};
-  const grpc_arg arg = grpc_lb_addresses_create_channel_arg(addresses);
+  const grpc_arg args_to_add[] = {
+      grpc_lb_addresses_create_channel_arg(addresses),
+      // A channel arg indicating if the target is a backend inferred from a
+      // grpclb load balancer.
+      grpc_channel_arg_integer_create(
+          const_cast<char*>(
+              GRPC_ARG_ADDRESS_IS_BACKEND_FROM_GRPCLB_LOAD_BALANCER),
+          is_backend_from_grpclb_load_balancer),
+  };
   grpc_channel_args* args = grpc_channel_args_copy_and_add_and_remove(
-      args_, keys_to_remove, GPR_ARRAY_SIZE(keys_to_remove), &arg, 1);
+      args_, keys_to_remove, GPR_ARRAY_SIZE(keys_to_remove), args_to_add,
+      GPR_ARRAY_SIZE(args_to_add));
   grpc_lb_addresses_destroy(addresses);
   return args;
 }
@@ -1724,7 +1737,7 @@ void GrpcLb::CreateOrUpdateRoundRobinPolicyLocked() {
   GPR_ASSERT(args != nullptr);
   if (rr_policy_ != nullptr) {
     if (grpc_lb_glb_trace.enabled()) {
-      gpr_log(GPR_DEBUG, "[grpclb %p] Updating RR policy %p", this,
+      gpr_log(GPR_INFO, "[grpclb %p] Updating RR policy %p", this,
               rr_policy_.get());
     }
     rr_policy_->UpdateLocked(*args);
@@ -1735,7 +1748,7 @@ void GrpcLb::CreateOrUpdateRoundRobinPolicyLocked() {
     lb_policy_args.args = args;
     CreateRoundRobinPolicyLocked(lb_policy_args);
     if (grpc_lb_glb_trace.enabled()) {
-      gpr_log(GPR_DEBUG, "[grpclb %p] Created new RR policy %p", this,
+      gpr_log(GPR_INFO, "[grpclb %p] Created new RR policy %p", this,
               rr_policy_.get());
     }
   }
@@ -1751,7 +1764,7 @@ void GrpcLb::OnRoundRobinRequestReresolutionLocked(void* arg,
   }
   if (grpc_lb_glb_trace.enabled()) {
     gpr_log(
-        GPR_DEBUG,
+        GPR_INFO,
         "[grpclb %p] Re-resolution requested from the internal RR policy (%p).",
         grpclb_policy, grpclb_policy->rr_policy_.get());
   }

src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.h

@@ -0,0 +1,36 @@
+/*
+ *
+ * Copyright 2018 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#ifndef GRPC_CORE_EXT_FILTERS_CLIENT_CHANNEL_LB_POLICY_GRPCLB_GRPCLB_H
+#define GRPC_CORE_EXT_FILTERS_CLIENT_CHANNEL_LB_POLICY_GRPCLB_GRPCLB_H
+
+#include <grpc/support/port_platform.h>
+
+/** Channel arg indicating if a target corresponding to the address is grpclb
+ * loadbalancer. The type of this arg is an integer and the value is treated as
+ * a bool. */
+#define GRPC_ARG_ADDRESS_IS_GRPCLB_LOAD_BALANCER \
+  "grpc.address_is_grpclb_load_balancer"
+/** Channel arg indicating if a target corresponding to the address is a backend
+ * received from a balancer. The type of this arg is an integer and the value is
+ * treated as a bool. */
+#define GRPC_ARG_ADDRESS_IS_BACKEND_FROM_GRPCLB_LOAD_BALANCER \
+  "grpc.address_is_backend_from_grpclb_load_balancer"
+
+#endif /* GRPC_CORE_EXT_FILTERS_CLIENT_CHANNEL_LB_POLICY_GRPCLB_GRPCLB_H \
+        */

src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.cc

@@ -22,131 +22,65 @@
 
 #include <string.h>
 
-#include <grpc/support/alloc.h>
 #include <grpc/support/atm.h>
 #include <grpc/support/string_util.h>
-#include <grpc/support/sync.h>
 
-#include "src/core/lib/channel/channel_args.h"
+namespace grpc_core {
 
-#define GRPC_ARG_GRPCLB_CLIENT_STATS "grpc.grpclb_client_stats"
-
-struct grpc_grpclb_client_stats {
-  gpr_refcount refs;
-  // This field must only be accessed via *_locked() methods.
-  grpc_grpclb_dropped_call_counts* drop_token_counts;
-  // These fields may be accessed from multiple threads at a time.
-  gpr_atm num_calls_started;
-  gpr_atm num_calls_finished;
-  gpr_atm num_calls_finished_with_client_failed_to_send;
-  gpr_atm num_calls_finished_known_received;
-};
-
-grpc_grpclb_client_stats* grpc_grpclb_client_stats_create() {
-  grpc_grpclb_client_stats* client_stats =
-      static_cast<grpc_grpclb_client_stats*>(gpr_zalloc(sizeof(*client_stats)));
-  gpr_ref_init(&client_stats->refs, 1);
-  return client_stats;
-}
-
-grpc_grpclb_client_stats* grpc_grpclb_client_stats_ref(
-    grpc_grpclb_client_stats* client_stats) {
-  gpr_ref(&client_stats->refs);
-  return client_stats;
-}
-
-void grpc_grpclb_client_stats_unref(grpc_grpclb_client_stats* client_stats) {
-  if (gpr_unref(&client_stats->refs)) {
-    grpc_grpclb_dropped_call_counts_destroy(client_stats->drop_token_counts);
-    gpr_free(client_stats);
-  }
-}
-
-void grpc_grpclb_client_stats_add_call_started(
-    grpc_grpclb_client_stats* client_stats) {
-  gpr_atm_full_fetch_add(&client_stats->num_calls_started, (gpr_atm)1);
+void GrpcLbClientStats::AddCallStarted() {
+  gpr_atm_full_fetch_add(&num_calls_started_, (gpr_atm)1);
 }
 
-void grpc_grpclb_client_stats_add_call_finished(
-    bool finished_with_client_failed_to_send, bool finished_known_received,
-    grpc_grpclb_client_stats* client_stats) {
-  gpr_atm_full_fetch_add(&client_stats->num_calls_finished, (gpr_atm)1);
+void GrpcLbClientStats::AddCallFinished(
+    bool finished_with_client_failed_to_send, bool finished_known_received) {
+  gpr_atm_full_fetch_add(&num_calls_finished_, (gpr_atm)1);
   if (finished_with_client_failed_to_send) {
-    gpr_atm_full_fetch_add(
-        &client_stats->num_calls_finished_with_client_failed_to_send,
-        (gpr_atm)1);
+    gpr_atm_full_fetch_add(&num_calls_finished_with_client_failed_to_send_,
+                           (gpr_atm)1);
   }
   if (finished_known_received) {
-    gpr_atm_full_fetch_add(&client_stats->num_calls_finished_known_received,
-                           (gpr_atm)1);
+    gpr_atm_full_fetch_add(&num_calls_finished_known_received_, (gpr_atm)1);
   }
 }
 
-void grpc_grpclb_client_stats_add_call_dropped_locked(
-    char* token, grpc_grpclb_client_stats* client_stats) {
+void GrpcLbClientStats::AddCallDroppedLocked(char* token) {
   // Increment num_calls_started and num_calls_finished.
-  gpr_atm_full_fetch_add(&client_stats->num_calls_started, (gpr_atm)1);
-  gpr_atm_full_fetch_add(&client_stats->num_calls_finished, (gpr_atm)1);
+  gpr_atm_full_fetch_add(&num_calls_started_, (gpr_atm)1);
+  gpr_atm_full_fetch_add(&num_calls_finished_, (gpr_atm)1);
   // Record the drop.
-  if (client_stats->drop_token_counts == nullptr) {
-    client_stats->drop_token_counts =
-        static_cast<grpc_grpclb_dropped_call_counts*>(
-            gpr_zalloc(sizeof(grpc_grpclb_dropped_call_counts)));
+  if (drop_token_counts_ == nullptr) {
+    drop_token_counts_.reset(New<DroppedCallCounts>());
   }
-  grpc_grpclb_dropped_call_counts* drop_token_counts =
-      client_stats->drop_token_counts;
-  for (size_t i = 0; i < drop_token_counts->num_entries; ++i) {
-    if (strcmp(drop_token_counts->token_counts[i].token, token) == 0) {
-      ++drop_token_counts->token_counts[i].count;
+  for (size_t i = 0; i < drop_token_counts_->size(); ++i) {
+    if (strcmp((*drop_token_counts_)[i].token.get(), token) == 0) {
+      ++(*drop_token_counts_)[i].count;
       return;
     }
   }
-  // Not found, so add a new entry.  We double the size of the array each time.
-  size_t new_num_entries = 2;
-  while (new_num_entries < drop_token_counts->num_entries + 1) {
-    new_num_entries *= 2;
-  }
-  drop_token_counts->token_counts = static_cast<grpc_grpclb_drop_token_count*>(
-      gpr_realloc(drop_token_counts->token_counts,
-                  new_num_entries * sizeof(grpc_grpclb_drop_token_count)));
-  grpc_grpclb_drop_token_count* new_entry =
-      &drop_token_counts->token_counts[drop_token_counts->num_entries++];
-  new_entry->token = gpr_strdup(token);
-  new_entry->count = 1;
+  // Not found, so add a new entry.
+  drop_token_counts_->emplace_back(UniquePtr<char>(gpr_strdup(token)), 1);
 }
 
-static void atomic_get_and_reset_counter(int64_t* value, gpr_atm* counter) {
-  *value = static_cast<int64_t>(gpr_atm_acq_load(counter));
-  gpr_atm_full_fetch_add(counter, (gpr_atm)(-*value));
+namespace {
+
+void AtomicGetAndResetCounter(int64_t* value, gpr_atm* counter) {
+  *value = static_cast<int64_t>(gpr_atm_full_xchg(counter, (gpr_atm)0));
 }
 
-void grpc_grpclb_client_stats_get_locked(
-    grpc_grpclb_client_stats* client_stats, int64_t* num_calls_started,
-    int64_t* num_calls_finished,
+}  // namespace
+
+void GrpcLbClientStats::GetLocked(
+    int64_t* num_calls_started, int64_t* num_calls_finished,
     int64_t* num_calls_finished_with_client_failed_to_send,
     int64_t* num_calls_finished_known_received,
-    grpc_grpclb_dropped_call_counts** drop_token_counts) {
-  atomic_get_and_reset_counter(num_calls_started,
-                               &client_stats->num_calls_started);
-  atomic_get_and_reset_counter(num_calls_finished,
-                               &client_stats->num_calls_finished);
-  atomic_get_and_reset_counter(
-      num_calls_finished_with_client_failed_to_send,
-      &client_stats->num_calls_finished_with_client_failed_to_send);
-  atomic_get_and_reset_counter(
-      num_calls_finished_known_received,
-      &client_stats->num_calls_finished_known_received);
-  *drop_token_counts = client_stats->drop_token_counts;
-  client_stats->drop_token_counts = nullptr;
+    UniquePtr<DroppedCallCounts>* drop_token_counts) {
+  AtomicGetAndResetCounter(num_calls_started, &num_calls_started_);
+  AtomicGetAndResetCounter(num_calls_finished, &num_calls_finished_);
+  AtomicGetAndResetCounter(num_calls_finished_with_client_failed_to_send,
+                           &num_calls_finished_with_client_failed_to_send_);
+  AtomicGetAndResetCounter(num_calls_finished_known_received,
+                           &num_calls_finished_known_received_);
+  *drop_token_counts = std::move(drop_token_counts_);
 }
 
-void grpc_grpclb_dropped_call_counts_destroy(
-    grpc_grpclb_dropped_call_counts* drop_entries) {
-  if (drop_entries != nullptr) {
-    for (size_t i = 0; i < drop_entries->num_entries; ++i) {
-      gpr_free(drop_entries->token_counts[i].token);
-    }
-    gpr_free(drop_entries->token_counts);
-    gpr_free(drop_entries);
-  }
-}
+}  // namespace grpc_core

src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.h

@@ -21,47 +21,52 @@
 
 #include <grpc/support/port_platform.h>
 
-#include <stdbool.h>
+#include <grpc/support/atm.h>
 
-#include <grpc/impl/codegen/grpc_types.h>
+#include "src/core/lib/gprpp/inlined_vector.h"
+#include "src/core/lib/gprpp/memory.h"
+#include "src/core/lib/gprpp/ref_counted.h"
 
-typedef struct grpc_grpclb_client_stats grpc_grpclb_client_stats;
+namespace grpc_core {
 
-typedef struct {
-  char* token;
-  int64_t count;
-} grpc_grpclb_drop_token_count;
+class GrpcLbClientStats : public RefCounted<GrpcLbClientStats> {
+ public:
+  struct DropTokenCount {
+    UniquePtr<char> token;
+    int64_t count;
 
-typedef struct {
-  grpc_grpclb_drop_token_count* token_counts;
-  size_t num_entries;
-} grpc_grpclb_dropped_call_counts;
+    DropTokenCount(UniquePtr<char> token, int64_t count)
+        : token(std::move(token)), count(count) {}
+  };
 
-grpc_grpclb_client_stats* grpc_grpclb_client_stats_create();
-grpc_grpclb_client_stats* grpc_grpclb_client_stats_ref(
-    grpc_grpclb_client_stats* client_stats);
-void grpc_grpclb_client_stats_unref(grpc_grpclb_client_stats* client_stats);
+  typedef InlinedVector<DropTokenCount, 10> DroppedCallCounts;
 
-void grpc_grpclb_client_stats_add_call_started(
-    grpc_grpclb_client_stats* client_stats);
-void grpc_grpclb_client_stats_add_call_finished(
-    bool finished_with_client_failed_to_send, bool finished_known_received,
-    grpc_grpclb_client_stats* client_stats);
+  GrpcLbClientStats() {}
 
-// This method is not thread-safe; caller must synchronize.
-void grpc_grpclb_client_stats_add_call_dropped_locked(
-    char* token, grpc_grpclb_client_stats* client_stats);
+  void AddCallStarted();
+  void AddCallFinished(bool finished_with_client_failed_to_send,
+                       bool finished_known_received);
 
-// This method is not thread-safe; caller must synchronize.
-void grpc_grpclb_client_stats_get_locked(
-    grpc_grpclb_client_stats* client_stats, int64_t* num_calls_started,
-    int64_t* num_calls_finished,
-    int64_t* num_calls_finished_with_client_failed_to_send,
-    int64_t* num_calls_finished_known_received,
-    grpc_grpclb_dropped_call_counts** drop_token_counts);
+  // This method is not thread-safe; caller must synchronize.
+  void AddCallDroppedLocked(char* token);
 
-void grpc_grpclb_dropped_call_counts_destroy(
-    grpc_grpclb_dropped_call_counts* drop_entries);
+  // This method is not thread-safe; caller must synchronize.
+  void GetLocked(int64_t* num_calls_started, int64_t* num_calls_finished,
+                 int64_t* num_calls_finished_with_client_failed_to_send,
+                 int64_t* num_calls_finished_known_received,
+                 UniquePtr<DroppedCallCounts>* drop_token_counts);
+
+ private:
+  // This field must only be accessed via *_locked() methods.
+  UniquePtr<DroppedCallCounts> drop_token_counts_;
+  // These fields may be accessed from multiple threads at a time.
+  gpr_atm num_calls_started_ = 0;
+  gpr_atm num_calls_finished_ = 0;
+  gpr_atm num_calls_finished_with_client_failed_to_send_ = 0;
+  gpr_atm num_calls_finished_known_received_ = 0;
+};
+
+}  // namespace grpc_core
 
 #endif /* GRPC_CORE_EXT_FILTERS_CLIENT_CHANNEL_LB_POLICY_GRPCLB_GRPCLB_CLIENT_STATS_H \
         */

src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.cc

@@ -18,9 +18,9 @@
 
 #include <grpc/support/port_platform.h>
 
+#include "pb_decode.h"
+#include "pb_encode.h"
 #include "src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.h"
-#include "third_party/nanopb/pb_decode.h"
-#include "third_party/nanopb/pb_encode.h"
 
 #include <grpc/support/alloc.h>
 
@@ -29,7 +29,7 @@ static bool count_serverlist(pb_istream_t* stream, const pb_field_t* field,
                              void** arg) {
   grpc_grpclb_serverlist* sl = static_cast<grpc_grpclb_serverlist*>(*arg);
   grpc_grpclb_server server;
-  if (!pb_decode(stream, grpc_lb_v1_Server_fields, &server)) {
+  if (GPR_UNLIKELY(!pb_decode(stream, grpc_lb_v1_Server_fields, &server))) {
     gpr_log(GPR_ERROR, "nanopb error: %s", PB_GET_ERROR(stream));
     return false;
   }
@@ -52,7 +52,7 @@ static bool decode_serverlist(pb_istream_t* stream, const pb_field_t* field,
   GPR_ASSERT(dec_arg->serverlist->num_servers >= dec_arg->decoding_idx);
   grpc_grpclb_server* server =
       static_cast<grpc_grpclb_server*>(gpr_zalloc(sizeof(grpc_grpclb_server)));
-  if (!pb_decode(stream, grpc_lb_v1_Server_fields, server)) {
+  if (GPR_UNLIKELY(!pb_decode(stream, grpc_lb_v1_Server_fields, server))) {
     gpr_free(server);
     gpr_log(GPR_ERROR, "nanopb error: %s", PB_GET_ERROR(stream));
     return false;
@@ -89,16 +89,16 @@ static bool encode_string(pb_ostream_t* stream, const pb_field_t* field,
 
 static bool encode_drops(pb_ostream_t* stream, const pb_field_t* field,
                          void* const* arg) {
-  grpc_grpclb_dropped_call_counts* drop_entries =
-      static_cast<grpc_grpclb_dropped_call_counts*>(*arg);
+  grpc_core::GrpcLbClientStats::DroppedCallCounts* drop_entries =
+      static_cast<grpc_core::GrpcLbClientStats::DroppedCallCounts*>(*arg);
   if (drop_entries == nullptr) return true;
-  for (size_t i = 0; i < drop_entries->num_entries; ++i) {
+  for (size_t i = 0; i < drop_entries->size(); ++i) {
     if (!pb_encode_tag_for_field(stream, field)) return false;
     grpc_lb_v1_ClientStatsPerToken drop_message;
     drop_message.load_balance_token.funcs.encode = encode_string;
-    drop_message.load_balance_token.arg = drop_entries->token_counts[i].token;
+    drop_message.load_balance_token.arg = (*drop_entries)[i].token.get();
     drop_message.has_num_calls = true;
-    drop_message.num_calls = drop_entries->token_counts[i].count;
+    drop_message.num_calls = (*drop_entries)[i].count;
     if (!pb_encode_submessage(stream, grpc_lb_v1_ClientStatsPerToken_fields,
                               &drop_message)) {
       return false;
@@ -108,7 +108,7 @@ static bool encode_drops(pb_ostream_t* stream, const pb_field_t* field,
 }
 
 grpc_grpclb_request* grpc_grpclb_load_report_request_create_locked(
-    grpc_grpclb_client_stats* client_stats) {
+    grpc_core::GrpcLbClientStats* client_stats) {
   grpc_grpclb_request* req = static_cast<grpc_grpclb_request*>(
       gpr_zalloc(sizeof(grpc_grpclb_request)));
   req->has_client_stats = true;
@@ -120,13 +120,15 @@ grpc_grpclb_request* grpc_grpclb_load_report_request_create_locked(
   req->client_stats.has_num_calls_finished_with_client_failed_to_send = true;
   req->client_stats.has_num_calls_finished_known_received = true;
   req->client_stats.calls_finished_with_drop.funcs.encode = encode_drops;
-  grpc_grpclb_client_stats_get_locked(
-      client_stats, &req->client_stats.num_calls_started,
+  grpc_core::UniquePtr<grpc_core::GrpcLbClientStats::DroppedCallCounts>
+      drop_counts;
+  client_stats->GetLocked(
+      &req->client_stats.num_calls_started,
       &req->client_stats.num_calls_finished,
       &req->client_stats.num_calls_finished_with_client_failed_to_send,
-      &req->client_stats.num_calls_finished_known_received,
-      reinterpret_cast<grpc_grpclb_dropped_call_counts**>(
-          &req->client_stats.calls_finished_with_drop.arg));
+      &req->client_stats.num_calls_finished_known_received, &drop_counts);
+  // Will be deleted in grpc_grpclb_request_destroy().
+  req->client_stats.calls_finished_with_drop.arg = drop_counts.release();
   return req;
 }
 
@@ -149,10 +151,10 @@ grpc_slice grpc_grpclb_request_encode(const grpc_grpclb_request* request) {
 
 void grpc_grpclb_request_destroy(grpc_grpclb_request* request) {
   if (request->has_client_stats) {
-    grpc_grpclb_dropped_call_counts* drop_entries =
-        static_cast<grpc_grpclb_dropped_call_counts*>(
+    grpc_core::GrpcLbClientStats::DroppedCallCounts* drop_entries =
+        static_cast<grpc_core::GrpcLbClientStats::DroppedCallCounts*>(
             request->client_stats.calls_finished_with_drop.arg);
-    grpc_grpclb_dropped_call_counts_destroy(drop_entries);
+    grpc_core::Delete(drop_entries);
   }
   gpr_free(request);
 }
@@ -165,7 +167,8 @@ grpc_grpclb_initial_response* grpc_grpclb_initial_response_parse(
                              GRPC_SLICE_LENGTH(encoded_grpc_grpclb_response));
   grpc_grpclb_response res;
   memset(&res, 0, sizeof(grpc_grpclb_response));
-  if (!pb_decode(&stream, grpc_lb_v1_LoadBalanceResponse_fields, &res)) {
+  if (GPR_UNLIKELY(
+          !pb_decode(&stream, grpc_lb_v1_LoadBalanceResponse_fields, &res))) {
     gpr_log(GPR_ERROR, "nanopb error: %s", PB_GET_ERROR(&stream));
     return nullptr;
   }
@@ -195,7 +198,7 @@ grpc_grpclb_serverlist* grpc_grpclb_response_parse_serverlist(
   res.server_list.servers.funcs.decode = count_serverlist;
   res.server_list.servers.arg = sl;
   bool status = pb_decode(&stream, grpc_lb_v1_LoadBalanceResponse_fields, &res);
-  if (!status) {
+  if (GPR_UNLIKELY(!status)) {
     gpr_free(sl);
     gpr_log(GPR_ERROR, "nanopb error: %s", PB_GET_ERROR(&stream));
     return nullptr;
@@ -211,7 +214,7 @@ grpc_grpclb_serverlist* grpc_grpclb_response_parse_serverlist(
     res.server_list.servers.arg = &decode_arg;
     status = pb_decode(&stream_at_start, grpc_lb_v1_LoadBalanceResponse_fields,
                        &res);
-    if (!status) {
+    if (GPR_UNLIKELY(!status)) {
       grpc_grpclb_destroy_serverlist(sl);
       gpr_log(GPR_ERROR, "nanopb error: %s", PB_GET_ERROR(&stream));
       return nullptr;

src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.h

@@ -42,7 +42,7 @@ typedef struct {
 /** Create a request for a gRPC LB service under \a lb_service_name */
 grpc_grpclb_request* grpc_grpclb_request_create(const char* lb_service_name);
 grpc_grpclb_request* grpc_grpclb_load_report_request_create_locked(
-    grpc_grpclb_client_stats* client_stats);
+    grpc_core::GrpcLbClientStats* client_stats);
 
 /** Protocol Buffers v3-encode \a request */
 grpc_slice grpc_grpclb_request_encode(const grpc_grpclb_request* request);

src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h

@@ -3,7 +3,7 @@
 
 #ifndef PB_GRPC_LB_V1_LOAD_BALANCER_PB_H_INCLUDED
 #define PB_GRPC_LB_V1_LOAD_BALANCER_PB_H_INCLUDED
-#include "third_party/nanopb/pb.h"
+#include "pb.h"
 /* @@protoc_insertion_point(includes) */
 #if PB_PROTO_HEADER_VERSION != 30
 #error Regenerate this file with the current version of nanopb generator.

src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc

@@ -62,31 +62,65 @@ class PickFirst : public LoadBalancingPolicy {
  private:
   ~PickFirst();
 
+  class PickFirstSubchannelList;
+
+  class PickFirstSubchannelData
+      : public SubchannelData<PickFirstSubchannelList,
+                              PickFirstSubchannelData> {
+   public:
+    PickFirstSubchannelData(PickFirstSubchannelList* subchannel_list,
+                            const grpc_lb_user_data_vtable* user_data_vtable,
+                            const grpc_lb_address& address,
+                            grpc_subchannel* subchannel,
+                            grpc_combiner* combiner)
+        : SubchannelData(subchannel_list, user_data_vtable, address, subchannel,
+                         combiner) {}
+
+    void ProcessConnectivityChangeLocked(
+        grpc_connectivity_state connectivity_state, grpc_error* error) override;
+  };
+
+  class PickFirstSubchannelList
+      : public SubchannelList<PickFirstSubchannelList,
+                              PickFirstSubchannelData> {
+   public:
+    PickFirstSubchannelList(PickFirst* policy, TraceFlag* tracer,
+                            const grpc_lb_addresses* addresses,
+                            grpc_combiner* combiner,
+                            grpc_client_channel_factory* client_channel_factory,
+                            const grpc_channel_args& args)
+        : SubchannelList(policy, tracer, addresses, combiner,
+                         client_channel_factory, args) {
+      // Need to maintain a ref to the LB policy as long as we maintain
+      // any references to subchannels, since the subchannels'
+      // pollset_sets will include the LB policy's pollset_set.
+      policy->Ref(DEBUG_LOCATION, "subchannel_list").release();
+    }
+
+    ~PickFirstSubchannelList() {
+      PickFirst* p = static_cast<PickFirst*>(policy());
+      p->Unref(DEBUG_LOCATION, "subchannel_list");
+    }
+  };
+
   void ShutdownLocked() override;
 
   void StartPickingLocked();
   void DestroyUnselectedSubchannelsLocked();
 
-  static void OnConnectivityChangedLocked(void* arg, grpc_error* error);
-
-  void SubchannelListRefForConnectivityWatch(
-      grpc_lb_subchannel_list* subchannel_list, const char* reason);
-  void SubchannelListUnrefForConnectivityWatch(
-      grpc_lb_subchannel_list* subchannel_list, const char* reason);
-
-  /** all our subchannels */
-  grpc_lb_subchannel_list* subchannel_list_ = nullptr;
-  /** latest pending subchannel list */
-  grpc_lb_subchannel_list* latest_pending_subchannel_list_ = nullptr;
-  /** selected subchannel in \a subchannel_list */
-  grpc_lb_subchannel_data* selected_ = nullptr;
-  /** have we started picking? */
+  // All our subchannels.
+  OrphanablePtr<PickFirstSubchannelList> subchannel_list_;
+  // Latest pending subchannel list.
+  OrphanablePtr<PickFirstSubchannelList> latest_pending_subchannel_list_;
+  // Selected subchannel in \a subchannel_list_.
+  PickFirstSubchannelData* selected_ = nullptr;
+  // Have we started picking?
   bool started_picking_ = false;
-  /** are we shut down? */
+  // Are we shut down?
   bool shutdown_ = false;
-  /** list of picks that are waiting on connectivity */
+  // List of picks that are waiting on connectivity.
   PickState* pending_picks_ = nullptr;
-  /** our connectivity state tracker */
+  // Our connectivity state tracker.
   grpc_connectivity_state_tracker state_tracker_;
 };
 
@@ -95,7 +129,7 @@ PickFirst::PickFirst(const Args& args) : LoadBalancingPolicy(args) {
   grpc_connectivity_state_init(&state_tracker_, GRPC_CHANNEL_IDLE,
                                "pick_first");
   if (grpc_lb_pick_first_trace.enabled()) {
-    gpr_log(GPR_DEBUG, "Pick First %p created.", this);
+    gpr_log(GPR_INFO, "Pick First %p created.", this);
   }
   UpdateLocked(*args.args);
   grpc_subchannel_index_ref();
@@ -103,7 +137,7 @@ PickFirst::PickFirst(const Args& args) : LoadBalancingPolicy(args) {
 
 PickFirst::~PickFirst() {
   if (grpc_lb_pick_first_trace.enabled()) {
-    gpr_log(GPR_DEBUG, "Destroying Pick First %p", this);
+    gpr_log(GPR_INFO, "Destroying Pick First %p", this);
   }
   GPR_ASSERT(subchannel_list_ == nullptr);
   GPR_ASSERT(latest_pending_subchannel_list_ == nullptr);
@@ -126,7 +160,7 @@ void PickFirst::HandOffPendingPicksLocked(LoadBalancingPolicy* new_policy) {
 void PickFirst::ShutdownLocked() {
   grpc_error* error = GRPC_ERROR_CREATE_FROM_STATIC_STRING("Channel shutdown");
   if (grpc_lb_pick_first_trace.enabled()) {
-    gpr_log(GPR_DEBUG, "Pick First %p Shutting down", this);
+    gpr_log(GPR_INFO, "Pick First %p Shutting down", this);
   }
   shutdown_ = true;
   PickState* pick;
@@ -137,15 +171,8 @@ void PickFirst::ShutdownLocked() {
   }
   grpc_connectivity_state_set(&state_tracker_, GRPC_CHANNEL_SHUTDOWN,
                               GRPC_ERROR_REF(error), "shutdown");
-  if (subchannel_list_ != nullptr) {
-    grpc_lb_subchannel_list_shutdown_and_unref(subchannel_list_, "pf_shutdown");
-    subchannel_list_ = nullptr;
-  }
-  if (latest_pending_subchannel_list_ != nullptr) {
-    grpc_lb_subchannel_list_shutdown_and_unref(latest_pending_subchannel_list_,
-                                               "pf_shutdown");
-    latest_pending_subchannel_list_ = nullptr;
-  }
+  subchannel_list_.reset();
+  latest_pending_subchannel_list_.reset();
   TryReresolutionLocked(&grpc_lb_pick_first_trace, GRPC_ERROR_CANCELLED);
   GRPC_ERROR_UNREF(error);
 }
@@ -192,14 +219,10 @@ void PickFirst::CancelMatchingPicksLocked(uint32_t initial_metadata_flags_mask,
 
 void PickFirst::StartPickingLocked() {
   started_picking_ = true;
-  if (subchannel_list_ != nullptr && subchannel_list_->num_subchannels > 0) {
-    subchannel_list_->checking_subchannel = 0;
-    for (size_t i = 0; i < subchannel_list_->num_subchannels; ++i) {
-      if (subchannel_list_->subchannels[i].subchannel != nullptr) {
-        SubchannelListRefForConnectivityWatch(
-            subchannel_list_, "connectivity_watch+start_picking");
-        grpc_lb_subchannel_data_start_connectivity_watch(
-            &subchannel_list_->subchannels[i]);
+  if (subchannel_list_ != nullptr) {
+    for (size_t i = 0; i < subchannel_list_->num_subchannels(); ++i) {
+      if (subchannel_list_->subchannel(i)->subchannel() != nullptr) {
+        subchannel_list_->subchannel(i)->StartConnectivityWatchLocked();
         break;
       }
     }
@@ -215,7 +238,7 @@ void PickFirst::ExitIdleLocked() {
 bool PickFirst::PickLocked(PickState* pick) {
   // If we have a selected subchannel already, return synchronously.
   if (selected_ != nullptr) {
-    pick->connected_subchannel = selected_->connected_subchannel;
+    pick->connected_subchannel = selected_->connected_subchannel()->Ref();
     return true;
   }
   // No subchannel selected yet, so handle asynchronously.
@@ -228,11 +251,10 @@ bool PickFirst::PickLocked(PickState* pick) {
 }
 
 void PickFirst::DestroyUnselectedSubchannelsLocked() {
-  for (size_t i = 0; i < subchannel_list_->num_subchannels; ++i) {
-    grpc_lb_subchannel_data* sd = &subchannel_list_->subchannels[i];
+  for (size_t i = 0; i < subchannel_list_->num_subchannels(); ++i) {
+    PickFirstSubchannelData* sd = subchannel_list_->subchannel(i);
     if (selected_ != sd) {
-      grpc_lb_subchannel_data_unref_subchannel(sd,
-                                               "selected_different_subchannel");
+      sd->UnrefSubchannelLocked("selected_different_subchannel");
     }
   }
 }
@@ -249,7 +271,7 @@ void PickFirst::NotifyOnStateChangeLocked(grpc_connectivity_state* current,
 
 void PickFirst::PingOneLocked(grpc_closure* on_initiate, grpc_closure* on_ack) {
   if (selected_ != nullptr) {
-    selected_->connected_subchannel->Ping(on_initiate, on_ack);
+    selected_->connected_subchannel()->Ping(on_initiate, on_ack);
   } else {
     GRPC_CLOSURE_SCHED(on_initiate,
                        GRPC_ERROR_CREATE_FROM_STATIC_STRING("Not connected"));
@@ -258,24 +280,6 @@ void PickFirst::PingOneLocked(grpc_closure* on_initiate, grpc_closure* on_ack) {
   }
 }
 
-void PickFirst::SubchannelListRefForConnectivityWatch(
-    grpc_lb_subchannel_list* subchannel_list, const char* reason) {
-  // TODO(roth): We currently track this ref manually.  Once the new
-  // ClosureRef API is ready and the subchannel_list code has been
-  // converted to a C++ API, find a way to hold the RefCountedPtr<>
-  // somewhere (maybe in the subchannel_data object) instead of doing
-  // this manually.
-  auto self = Ref(DEBUG_LOCATION, reason);
-  self.release();
-  grpc_lb_subchannel_list_ref(subchannel_list, reason);
-}
-
-void PickFirst::SubchannelListUnrefForConnectivityWatch(
-    grpc_lb_subchannel_list* subchannel_list, const char* reason) {
-  Unref(DEBUG_LOCATION, reason);
-  grpc_lb_subchannel_list_unref(subchannel_list, reason);
-}
-
 void PickFirst::UpdateLocked(const grpc_channel_args& args) {
   const grpc_arg* arg = grpc_channel_args_find(&args, GRPC_ARG_LB_ADDRESSES);
   if (arg == nullptr || arg->type != GRPC_ARG_POINTER) {
@@ -295,75 +299,67 @@ void PickFirst::UpdateLocked(const grpc_channel_args& args) {
     return;
   }
   const grpc_lb_addresses* addresses =
-      (const grpc_lb_addresses*)arg->value.pointer.p;
+      static_cast<const grpc_lb_addresses*>(arg->value.pointer.p);
   if (grpc_lb_pick_first_trace.enabled()) {
     gpr_log(GPR_INFO,
             "Pick First %p received update with %" PRIuPTR " addresses", this,
             addresses->num_addresses);
   }
-  grpc_lb_subchannel_list* subchannel_list = grpc_lb_subchannel_list_create(
+  auto subchannel_list = MakeOrphanable<PickFirstSubchannelList>(
       this, &grpc_lb_pick_first_trace, addresses, combiner(),
-      client_channel_factory(), args, &PickFirst::OnConnectivityChangedLocked);
-  if (subchannel_list->num_subchannels == 0) {
+      client_channel_factory(), args);
+  if (subchannel_list->num_subchannels() == 0) {
     // Empty update or no valid subchannels. Unsubscribe from all current
     // subchannels and put the channel in TRANSIENT_FAILURE.
     grpc_connectivity_state_set(
         &state_tracker_, GRPC_CHANNEL_TRANSIENT_FAILURE,
         GRPC_ERROR_CREATE_FROM_STATIC_STRING("Empty update"),
         "pf_update_empty");
-    if (subchannel_list_ != nullptr) {
-      grpc_lb_subchannel_list_shutdown_and_unref(subchannel_list_,
-                                                 "sl_shutdown_empty_update");
-    }
-    subchannel_list_ = subchannel_list;  // Empty list.
+    subchannel_list_ = std::move(subchannel_list);  // Empty list.
     selected_ = nullptr;
     return;
   }
   if (selected_ == nullptr) {
     // We don't yet have a selected subchannel, so replace the current
     // subchannel list immediately.
-    if (subchannel_list_ != nullptr) {
-      grpc_lb_subchannel_list_shutdown_and_unref(subchannel_list_,
-                                                 "pf_update_before_selected");
+    subchannel_list_ = std::move(subchannel_list);
+    // If we've started picking, start trying to connect to the first
+    // subchannel in the new list.
+    if (started_picking_) {
+      subchannel_list_->subchannel(0)->StartConnectivityWatchLocked();
     }
-    subchannel_list_ = subchannel_list;
   } else {
     // We do have a selected subchannel.
     // Check if it's present in the new list.  If so, we're done.
-    for (size_t i = 0; i < subchannel_list->num_subchannels; ++i) {
-      grpc_lb_subchannel_data* sd = &subchannel_list->subchannels[i];
-      if (sd->subchannel == selected_->subchannel) {
+    for (size_t i = 0; i < subchannel_list->num_subchannels(); ++i) {
+      PickFirstSubchannelData* sd = subchannel_list->subchannel(i);
+      if (sd->subchannel() == selected_->subchannel()) {
         // The currently selected subchannel is in the update: we are done.
         if (grpc_lb_pick_first_trace.enabled()) {
           gpr_log(GPR_INFO,
                   "Pick First %p found already selected subchannel %p "
                   "at update index %" PRIuPTR " of %" PRIuPTR "; update done",
-                  this, selected_->subchannel, i,
-                  subchannel_list->num_subchannels);
-        }
-        if (selected_->connected_subchannel != nullptr) {
-          sd->connected_subchannel = selected_->connected_subchannel;
+                  this, selected_->subchannel(), i,
+                  subchannel_list->num_subchannels());
         }
-        selected_ = sd;
-        if (subchannel_list_ != nullptr) {
-          grpc_lb_subchannel_list_shutdown_and_unref(
-              subchannel_list_, "pf_update_includes_selected");
+        // Make sure it's in state READY.  It might not be if we grabbed
+        // the combiner while a connectivity state notification
+        // informing us otherwise is pending.
+        // Note that CheckConnectivityStateLocked() also takes a ref to
+        // the connected subchannel.
+        grpc_error* error = GRPC_ERROR_NONE;
+        if (sd->CheckConnectivityStateLocked(&error) == GRPC_CHANNEL_READY) {
+          selected_ = sd;
+          subchannel_list_ = std::move(subchannel_list);
+          DestroyUnselectedSubchannelsLocked();
+          sd->StartConnectivityWatchLocked();
+          // If there was a previously pending update (which may or may
+          // not have contained the currently selected subchannel), drop
+          // it, so that it doesn't override what we've done here.
+          latest_pending_subchannel_list_.reset();
+          return;
         }
-        subchannel_list_ = subchannel_list;
-        DestroyUnselectedSubchannelsLocked();
-        SubchannelListRefForConnectivityWatch(
-            subchannel_list, "connectivity_watch+replace_selected");
-        grpc_lb_subchannel_data_start_connectivity_watch(sd);
-        // If there was a previously pending update (which may or may
-        // not have contained the currently selected subchannel), drop
-        // it, so that it doesn't override what we've done here.
-        if (latest_pending_subchannel_list_ != nullptr) {
-          grpc_lb_subchannel_list_shutdown_and_unref(
-              latest_pending_subchannel_list_,
-              "pf_update_includes_selected+outdated");
-          latest_pending_subchannel_list_ = nullptr;
-        }
-        return;
+        GRPC_ERROR_UNREF(error);
       }
     }
     // Not keeping the previous selected subchannel, so set the latest
@@ -372,88 +368,66 @@ void PickFirst::UpdateLocked(const grpc_channel_args& args) {
     // subchannel list.
     if (latest_pending_subchannel_list_ != nullptr) {
       if (grpc_lb_pick_first_trace.enabled()) {
-        gpr_log(GPR_DEBUG,
+        gpr_log(GPR_INFO,
                 "Pick First %p Shutting down latest pending subchannel list "
                 "%p, about to be replaced by newer latest %p",
-                this, latest_pending_subchannel_list_, subchannel_list);
+                this, latest_pending_subchannel_list_.get(),
+                subchannel_list.get());
       }
-      grpc_lb_subchannel_list_shutdown_and_unref(
-          latest_pending_subchannel_list_, "sl_outdated_dont_smash");
     }
-    latest_pending_subchannel_list_ = subchannel_list;
-  }
-  // If we've started picking, start trying to connect to the first
-  // subchannel in the new list.
-  if (started_picking_) {
-    SubchannelListRefForConnectivityWatch(subchannel_list,
-                                          "connectivity_watch+update");
-    grpc_lb_subchannel_data_start_connectivity_watch(
-        &subchannel_list->subchannels[0]);
+    latest_pending_subchannel_list_ = std::move(subchannel_list);
+    // If we've started picking, start trying to connect to the first
+    // subchannel in the new list.
+    if (started_picking_) {
+      latest_pending_subchannel_list_->subchannel(0)
+          ->StartConnectivityWatchLocked();
+    }
   }
 }
 
-void PickFirst::OnConnectivityChangedLocked(void* arg, grpc_error* error) {
-  grpc_lb_subchannel_data* sd = static_cast<grpc_lb_subchannel_data*>(arg);
-  PickFirst* p = static_cast<PickFirst*>(sd->subchannel_list->policy);
-  if (grpc_lb_pick_first_trace.enabled()) {
-    gpr_log(GPR_DEBUG,
-            "Pick First %p connectivity changed for subchannel %p (%" PRIuPTR
-            " of %" PRIuPTR
-            "), subchannel_list %p: state=%s p->shutdown_=%d "
-            "sd->subchannel_list->shutting_down=%d error=%s",
-            p, sd->subchannel, sd->subchannel_list->checking_subchannel,
-            sd->subchannel_list->num_subchannels, sd->subchannel_list,
-            grpc_connectivity_state_name(sd->pending_connectivity_state_unsafe),
-            p->shutdown_, sd->subchannel_list->shutting_down,
-            grpc_error_string(error));
-  }
-  // If the policy is shutting down, unref and return.
-  if (p->shutdown_) {
-    grpc_lb_subchannel_data_stop_connectivity_watch(sd);
-    grpc_lb_subchannel_data_unref_subchannel(sd, "pf_shutdown");
-    p->SubchannelListUnrefForConnectivityWatch(sd->subchannel_list,
-                                               "pf_shutdown");
-    return;
-  }
-  // If the subchannel list is shutting down, stop watching.
-  if (sd->subchannel_list->shutting_down || error == GRPC_ERROR_CANCELLED) {
-    grpc_lb_subchannel_data_stop_connectivity_watch(sd);
-    grpc_lb_subchannel_data_unref_subchannel(sd, "pf_sl_shutdown");
-    p->SubchannelListUnrefForConnectivityWatch(sd->subchannel_list,
-                                               "pf_sl_shutdown");
-    return;
-  }
-  // If we're still here, the notification must be for a subchannel in
-  // either the current or latest pending subchannel lists.
-  GPR_ASSERT(sd->subchannel_list == p->subchannel_list_ ||
-             sd->subchannel_list == p->latest_pending_subchannel_list_);
-  // Update state.
-  sd->curr_connectivity_state = sd->pending_connectivity_state_unsafe;
+void PickFirst::PickFirstSubchannelData::ProcessConnectivityChangeLocked(
+    grpc_connectivity_state connectivity_state, grpc_error* error) {
+  PickFirst* p = static_cast<PickFirst*>(subchannel_list()->policy());
+  // The notification must be for a subchannel in either the current or
+  // latest pending subchannel lists.
+  GPR_ASSERT(subchannel_list() == p->subchannel_list_.get() ||
+             subchannel_list() == p->latest_pending_subchannel_list_.get());
   // Handle updates for the currently selected subchannel.
-  if (p->selected_ == sd) {
+  if (p->selected_ == this) {
+    if (grpc_lb_pick_first_trace.enabled()) {
+      gpr_log(GPR_INFO,
+              "Pick First %p connectivity changed for selected subchannel", p);
+    }
     // If the new state is anything other than READY and there is a
     // pending update, switch to the pending update.
-    if (sd->curr_connectivity_state != GRPC_CHANNEL_READY &&
+    if (connectivity_state != GRPC_CHANNEL_READY &&
         p->latest_pending_subchannel_list_ != nullptr) {
+      if (grpc_lb_pick_first_trace.enabled()) {
+        gpr_log(GPR_INFO,
+                "Pick First %p promoting pending subchannel list %p to "
+                "replace %p",
+                p, p->latest_pending_subchannel_list_.get(),
+                p->subchannel_list_.get());
+      }
       p->selected_ = nullptr;
-      grpc_lb_subchannel_data_stop_connectivity_watch(sd);
-      p->SubchannelListUnrefForConnectivityWatch(
-          sd->subchannel_list, "selected_not_ready+switch_to_update");
-      grpc_lb_subchannel_list_shutdown_and_unref(
-          p->subchannel_list_, "selected_not_ready+switch_to_update");
-      p->subchannel_list_ = p->latest_pending_subchannel_list_;
-      p->latest_pending_subchannel_list_ = nullptr;
+      StopConnectivityWatchLocked();
+      p->subchannel_list_ = std::move(p->latest_pending_subchannel_list_);
       grpc_connectivity_state_set(
           &p->state_tracker_, GRPC_CHANNEL_TRANSIENT_FAILURE,
-          GRPC_ERROR_REF(error), "selected_not_ready+switch_to_update");
+          error != GRPC_ERROR_NONE
+              ? GRPC_ERROR_REF(error)
+              : GRPC_ERROR_CREATE_FROM_STATIC_STRING(
+                    "selected subchannel not ready; switching to pending "
+                    "update"),
+          "selected_not_ready+switch_to_update");
     } else {
       // TODO(juanlishen): we re-resolve when the selected subchannel goes to
       // TRANSIENT_FAILURE because we used to shut down in this case before
       // re-resolution is introduced. But we need to investigate whether we
       // really want to take any action instead of waiting for the selected
       // subchannel reconnecting.
-      GPR_ASSERT(sd->curr_connectivity_state != GRPC_CHANNEL_SHUTDOWN);
-      if (sd->curr_connectivity_state == GRPC_CHANNEL_TRANSIENT_FAILURE) {
+      GPR_ASSERT(connectivity_state != GRPC_CHANNEL_SHUTDOWN);
+      if (connectivity_state == GRPC_CHANNEL_TRANSIENT_FAILURE) {
         // If the selected channel goes bad, request a re-resolution.
         grpc_connectivity_state_set(&p->state_tracker_, GRPC_CHANNEL_IDLE,
                                     GRPC_ERROR_NONE,
@@ -462,19 +436,16 @@ void PickFirst::OnConnectivityChangedLocked(void* arg, grpc_error* error) {
         p->TryReresolutionLocked(&grpc_lb_pick_first_trace, GRPC_ERROR_NONE);
         // In transient failure. Rely on re-resolution to recover.
         p->selected_ = nullptr;
-        grpc_lb_subchannel_data_stop_connectivity_watch(sd);
-        p->SubchannelListUnrefForConnectivityWatch(sd->subchannel_list,
-                                                   "pf_selected_shutdown");
-        grpc_lb_subchannel_data_unref_subchannel(
-            sd, "pf_selected_shutdown");  // Unrefs connected subchannel
+        UnrefSubchannelLocked("pf_selected_shutdown");
+        StopConnectivityWatchLocked();
       } else {
-        grpc_connectivity_state_set(&p->state_tracker_,
-                                    sd->curr_connectivity_state,
+        grpc_connectivity_state_set(&p->state_tracker_, connectivity_state,
                                     GRPC_ERROR_REF(error), "selected_changed");
         // Renew notification.
-        grpc_lb_subchannel_data_start_connectivity_watch(sd);
+        RenewConnectivityWatchLocked();
       }
     }
+    GRPC_ERROR_UNREF(error);
     return;
   }
   // If we get here, there are two possible cases:
@@ -486,26 +457,27 @@ void PickFirst::OnConnectivityChangedLocked(void* arg, grpc_error* error) {
   //    for a subchannel in p->latest_pending_subchannel_list_.  The
   //    goal here is to find a subchannel from the update that we can
   //    select in place of the current one.
-  switch (sd->curr_connectivity_state) {
+  switch (connectivity_state) {
     case GRPC_CHANNEL_READY: {
       // Case 2.  Promote p->latest_pending_subchannel_list_ to
       // p->subchannel_list_.
-      sd->connected_subchannel =
-          grpc_subchannel_get_connected_subchannel(sd->subchannel);
-      if (sd->subchannel_list == p->latest_pending_subchannel_list_) {
-        GPR_ASSERT(p->subchannel_list_ != nullptr);
-        grpc_lb_subchannel_list_shutdown_and_unref(p->subchannel_list_,
-                                                   "finish_update");
-        p->subchannel_list_ = p->latest_pending_subchannel_list_;
-        p->latest_pending_subchannel_list_ = nullptr;
+      if (subchannel_list() == p->latest_pending_subchannel_list_.get()) {
+        if (grpc_lb_pick_first_trace.enabled()) {
+          gpr_log(GPR_INFO,
+                  "Pick First %p promoting pending subchannel list %p to "
+                  "replace %p",
+                  p, p->latest_pending_subchannel_list_.get(),
+                  p->subchannel_list_.get());
+        }
+        p->subchannel_list_ = std::move(p->latest_pending_subchannel_list_);
       }
       // Cases 1 and 2.
       grpc_connectivity_state_set(&p->state_tracker_, GRPC_CHANNEL_READY,
                                   GRPC_ERROR_NONE, "connecting_ready");
-      p->selected_ = sd;
+      p->selected_ = this;
       if (grpc_lb_pick_first_trace.enabled()) {
         gpr_log(GPR_INFO, "Pick First %p selected subchannel %p", p,
-                sd->subchannel);
+                subchannel());
       }
       // Drop all other subchannels, since we are now connected.
       p->DestroyUnselectedSubchannelsLocked();
@@ -513,54 +485,53 @@ void PickFirst::OnConnectivityChangedLocked(void* arg, grpc_error* error) {
       PickState* pick;
       while ((pick = p->pending_picks_)) {
         p->pending_picks_ = pick->next;
-        pick->connected_subchannel = p->selected_->connected_subchannel;
+        pick->connected_subchannel =
+            p->selected_->connected_subchannel()->Ref();
         if (grpc_lb_pick_first_trace.enabled()) {
           gpr_log(GPR_INFO,
                   "Servicing pending pick with selected subchannel %p",
-                  p->selected_);
+                  p->selected_->subchannel());
         }
         GRPC_CLOSURE_SCHED(pick->on_complete, GRPC_ERROR_NONE);
       }
       // Renew notification.
-      grpc_lb_subchannel_data_start_connectivity_watch(sd);
+      RenewConnectivityWatchLocked();
       break;
     }
     case GRPC_CHANNEL_TRANSIENT_FAILURE: {
-      grpc_lb_subchannel_data_stop_connectivity_watch(sd);
+      StopConnectivityWatchLocked();
+      PickFirstSubchannelData* sd = this;
       do {
-        sd->subchannel_list->checking_subchannel =
-            (sd->subchannel_list->checking_subchannel + 1) %
-            sd->subchannel_list->num_subchannels;
-        sd = &sd->subchannel_list
-                  ->subchannels[sd->subchannel_list->checking_subchannel];
-      } while (sd->subchannel == nullptr);
+        size_t next_index =
+            (sd->Index() + 1) % subchannel_list()->num_subchannels();
+        sd = subchannel_list()->subchannel(next_index);
+      } while (sd->subchannel() == nullptr);
       // Case 1: Only set state to TRANSIENT_FAILURE if we've tried
       // all subchannels.
-      if (sd->subchannel_list->checking_subchannel == 0 &&
-          sd->subchannel_list == p->subchannel_list_) {
+      if (sd->Index() == 0 && subchannel_list() == p->subchannel_list_.get()) {
         grpc_connectivity_state_set(
             &p->state_tracker_, GRPC_CHANNEL_TRANSIENT_FAILURE,
             GRPC_ERROR_REF(error), "connecting_transient_failure");
       }
-      // Reuses the connectivity refs from the previous watch.
-      grpc_lb_subchannel_data_start_connectivity_watch(sd);
+      sd->StartConnectivityWatchLocked();
       break;
     }
     case GRPC_CHANNEL_CONNECTING:
     case GRPC_CHANNEL_IDLE: {
       // Only update connectivity state in case 1.
-      if (sd->subchannel_list == p->subchannel_list_) {
+      if (subchannel_list() == p->subchannel_list_.get()) {
         grpc_connectivity_state_set(&p->state_tracker_, GRPC_CHANNEL_CONNECTING,
                                     GRPC_ERROR_REF(error),
                                     "connecting_changed");
       }
       // Renew notification.
-      grpc_lb_subchannel_data_start_connectivity_watch(sd);
+      RenewConnectivityWatchLocked();
       break;
     }
     case GRPC_CHANNEL_SHUTDOWN:
       GPR_UNREACHABLE_CODE(break);
   }
+  GRPC_ERROR_UNREF(error);
 }
 
 //

src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc

@@ -73,23 +73,127 @@ class RoundRobin : public LoadBalancingPolicy {
  private:
   ~RoundRobin();
 
-  void ShutdownLocked() override;
+  // Forward declaration.
+  class RoundRobinSubchannelList;
+
+  // Data for a particular subchannel in a subchannel list.
+  // This subclass adds the following functionality:
+  // - Tracks user_data associated with each address, which will be
+  //   returned along with picks that select the subchannel.
+  // - Tracks the previous connectivity state of the subchannel, so that
+  //   we know how many subchannels are in each state.
+  class RoundRobinSubchannelData
+      : public SubchannelData<RoundRobinSubchannelList,
+                              RoundRobinSubchannelData> {
+   public:
+    RoundRobinSubchannelData(RoundRobinSubchannelList* subchannel_list,
+                             const grpc_lb_user_data_vtable* user_data_vtable,
+                             const grpc_lb_address& address,
+                             grpc_subchannel* subchannel,
+                             grpc_combiner* combiner)
+        : SubchannelData(subchannel_list, user_data_vtable, address, subchannel,
+                         combiner),
+          user_data_vtable_(user_data_vtable),
+          user_data_(user_data_vtable_ != nullptr
+                         ? user_data_vtable_->copy(address.user_data)
+                         : nullptr) {}
+
+    void UnrefSubchannelLocked(const char* reason) override {
+      SubchannelData::UnrefSubchannelLocked(reason);
+      if (user_data_ != nullptr) {
+        GPR_ASSERT(user_data_vtable_ != nullptr);
+        user_data_vtable_->destroy(user_data_);
+        user_data_ = nullptr;
+      }
+    }
 
-  void StartPickingLocked();
-  size_t GetNextReadySubchannelIndexLocked();
-  void UpdateLastReadySubchannelIndexLocked(size_t last_ready_index);
-  void UpdateConnectivityStatusLocked(grpc_lb_subchannel_data* sd,
-                                      grpc_error* error);
+    void* user_data() const { return user_data_; }
+
+    grpc_connectivity_state connectivity_state() const {
+      return last_connectivity_state_;
+    }
 
-  static void OnConnectivityChangedLocked(void* arg, grpc_error* error);
+    void UpdateConnectivityStateLocked(
+        grpc_connectivity_state connectivity_state, grpc_error* error);
+
+   private:
+    void ProcessConnectivityChangeLocked(
+        grpc_connectivity_state connectivity_state, grpc_error* error) override;
+
+    const grpc_lb_user_data_vtable* user_data_vtable_;
+    void* user_data_ = nullptr;
+    grpc_connectivity_state last_connectivity_state_ = GRPC_CHANNEL_IDLE;
+  };
+
+  // A list of subchannels.
+  class RoundRobinSubchannelList
+      : public SubchannelList<RoundRobinSubchannelList,
+                              RoundRobinSubchannelData> {
+   public:
+    RoundRobinSubchannelList(
+        RoundRobin* policy, TraceFlag* tracer,
+        const grpc_lb_addresses* addresses, grpc_combiner* combiner,
+        grpc_client_channel_factory* client_channel_factory,
+        const grpc_channel_args& args)
+        : SubchannelList(policy, tracer, addresses, combiner,
+                         client_channel_factory, args) {
+      // Need to maintain a ref to the LB policy as long as we maintain
+      // any references to subchannels, since the subchannels'
+      // pollset_sets will include the LB policy's pollset_set.
+      policy->Ref(DEBUG_LOCATION, "subchannel_list").release();
+    }
 
-  void SubchannelListRefForConnectivityWatch(
-      grpc_lb_subchannel_list* subchannel_list, const char* reason);
-  void SubchannelListUnrefForConnectivityWatch(
-      grpc_lb_subchannel_list* subchannel_list, const char* reason);
+    ~RoundRobinSubchannelList() {
+      GRPC_ERROR_UNREF(last_transient_failure_error_);
+      RoundRobin* p = static_cast<RoundRobin*>(policy());
+      p->Unref(DEBUG_LOCATION, "subchannel_list");
+    }
+
+    // Starts watching the subchannels in this list.
+    void StartWatchingLocked();
+
+    // Updates the counters of subchannels in each state when a
+    // subchannel transitions from old_state to new_state.
+    // transient_failure_error is the error that is reported when
+    // new_state is TRANSIENT_FAILURE.
+    void UpdateStateCountersLocked(grpc_connectivity_state old_state,
+                                   grpc_connectivity_state new_state,
+                                   grpc_error* transient_failure_error);
+
+    // If this subchannel list is the RR policy's current subchannel
+    // list, updates the RR policy's connectivity state based on the
+    // subchannel list's state counters.
+    void MaybeUpdateRoundRobinConnectivityStateLocked();
+
+    // Updates the RR policy's overall state based on the counters of
+    // subchannels in each state.
+    void UpdateRoundRobinStateFromSubchannelStateCountsLocked();
+
+    size_t GetNextReadySubchannelIndexLocked();
+    void UpdateLastReadySubchannelIndexLocked(size_t last_ready_index);
+
+   private:
+    size_t num_ready_ = 0;
+    size_t num_connecting_ = 0;
+    size_t num_transient_failure_ = 0;
+    grpc_error* last_transient_failure_error_ = GRPC_ERROR_NONE;
+    size_t last_ready_index_ = -1;  // Index into list of last pick.
+  };
+
+  void ShutdownLocked() override;
+
+  void StartPickingLocked();
+  bool DoPickLocked(PickState* pick);
+  void DrainPendingPicksLocked();
 
   /** list of subchannels */
-  grpc_lb_subchannel_list* subchannel_list_ = nullptr;
+  OrphanablePtr<RoundRobinSubchannelList> subchannel_list_;
+  /** Latest version of the subchannel list.
+   * Subchannel connectivity callbacks will only promote updated subchannel
+   * lists if they equal \a latest_pending_subchannel_list. In other words,
+   * racing callbacks that reference outdated subchannel lists won't perform any
+   * update. */
+  OrphanablePtr<RoundRobinSubchannelList> latest_pending_subchannel_list_;
   /** have we started picking? */
   bool started_picking_ = false;
   /** are we shutting down? */
@@ -98,14 +202,6 @@ class RoundRobin : public LoadBalancingPolicy {
   PickState* pending_picks_ = nullptr;
   /** our connectivity state tracker */
   grpc_connectivity_state_tracker state_tracker_;
-  /** Index into subchannels for last pick. */
-  size_t last_ready_subchannel_index_ = 0;
-  /** Latest version of the subchannel list.
-   * Subchannel connectivity callbacks will only promote updated subchannel
-   * lists if they equal \a latest_pending_subchannel_list. In other words,
-   * racing callbacks that reference outdated subchannel lists won't perform any
-   * update. */
-  grpc_lb_subchannel_list* latest_pending_subchannel_list_ = nullptr;
 };
 
 RoundRobin::RoundRobin(const Args& args) : LoadBalancingPolicy(args) {
@@ -114,15 +210,15 @@ RoundRobin::RoundRobin(const Args& args) : LoadBalancingPolicy(args) {
                                "round_robin");
   UpdateLocked(*args.args);
   if (grpc_lb_round_robin_trace.enabled()) {
-    gpr_log(GPR_DEBUG, "[RR %p] Created with %" PRIuPTR " subchannels", this,
-            subchannel_list_->num_subchannels);
+    gpr_log(GPR_INFO, "[RR %p] Created with %" PRIuPTR " subchannels", this,
+            subchannel_list_->num_subchannels());
   }
   grpc_subchannel_index_ref();
 }
 
 RoundRobin::~RoundRobin() {
   if (grpc_lb_round_robin_trace.enabled()) {
-    gpr_log(GPR_DEBUG, "[RR %p] Destroying Round Robin policy", this);
+    gpr_log(GPR_INFO, "[RR %p] Destroying Round Robin policy", this);
   }
   GPR_ASSERT(subchannel_list_ == nullptr);
   GPR_ASSERT(latest_pending_subchannel_list_ == nullptr);
@@ -131,68 +227,6 @@ RoundRobin::~RoundRobin() {
   grpc_subchannel_index_unref();
 }
 
-/** Returns the index into p->subchannel_list->subchannels of the next
- * subchannel in READY state, or p->subchannel_list->num_subchannels if no
- * subchannel is READY.
- *
- * Note that this function does *not* update p->last_ready_subchannel_index.
- * The caller must do that if it returns a pick. */
-size_t RoundRobin::GetNextReadySubchannelIndexLocked() {
-  GPR_ASSERT(subchannel_list_ != nullptr);
-  if (grpc_lb_round_robin_trace.enabled()) {
-    gpr_log(GPR_INFO,
-            "[RR %p] getting next ready subchannel (out of %" PRIuPTR
-            "), "
-            "last_ready_subchannel_index=%" PRIuPTR,
-            this, subchannel_list_->num_subchannels,
-            last_ready_subchannel_index_);
-  }
-  for (size_t i = 0; i < subchannel_list_->num_subchannels; ++i) {
-    const size_t index = (i + last_ready_subchannel_index_ + 1) %
-                         subchannel_list_->num_subchannels;
-    if (grpc_lb_round_robin_trace.enabled()) {
-      gpr_log(
-          GPR_DEBUG,
-          "[RR %p] checking subchannel %p, subchannel_list %p, index %" PRIuPTR
-          ": state=%s",
-          this, subchannel_list_->subchannels[index].subchannel,
-          subchannel_list_, index,
-          grpc_connectivity_state_name(
-              subchannel_list_->subchannels[index].curr_connectivity_state));
-    }
-    if (subchannel_list_->subchannels[index].curr_connectivity_state ==
-        GRPC_CHANNEL_READY) {
-      if (grpc_lb_round_robin_trace.enabled()) {
-        gpr_log(GPR_DEBUG,
-                "[RR %p] found next ready subchannel (%p) at index %" PRIuPTR
-                " of subchannel_list %p",
-                this, subchannel_list_->subchannels[index].subchannel, index,
-                subchannel_list_);
-      }
-      return index;
-    }
-  }
-  if (grpc_lb_round_robin_trace.enabled()) {
-    gpr_log(GPR_DEBUG, "[RR %p] no subchannels in ready state", this);
-  }
-  return subchannel_list_->num_subchannels;
-}
-
-// Sets last_ready_subchannel_index_ to last_ready_index.
-void RoundRobin::UpdateLastReadySubchannelIndexLocked(size_t last_ready_index) {
-  GPR_ASSERT(last_ready_index < subchannel_list_->num_subchannels);
-  last_ready_subchannel_index_ = last_ready_index;
-  if (grpc_lb_round_robin_trace.enabled()) {
-    gpr_log(GPR_DEBUG,
-            "[RR %p] setting last_ready_subchannel_index=%" PRIuPTR
-            " (SC %p, CSC %p)",
-            this, last_ready_index,
-            subchannel_list_->subchannels[last_ready_index].subchannel,
-            subchannel_list_->subchannels[last_ready_index]
-                .connected_subchannel.get());
-  }
-}
-
 void RoundRobin::HandOffPendingPicksLocked(LoadBalancingPolicy* new_policy) {
   PickState* pick;
   while ((pick = pending_picks_) != nullptr) {
@@ -207,7 +241,7 @@ void RoundRobin::HandOffPendingPicksLocked(LoadBalancingPolicy* new_policy) {
 void RoundRobin::ShutdownLocked() {
   grpc_error* error = GRPC_ERROR_CREATE_FROM_STATIC_STRING("Channel shutdown");
   if (grpc_lb_round_robin_trace.enabled()) {
-    gpr_log(GPR_DEBUG, "[RR %p] Shutting down", this);
+    gpr_log(GPR_INFO, "[RR %p] Shutting down", this);
   }
   shutdown_ = true;
   PickState* pick;
@@ -218,16 +252,8 @@ void RoundRobin::ShutdownLocked() {
   }
   grpc_connectivity_state_set(&state_tracker_, GRPC_CHANNEL_SHUTDOWN,
                               GRPC_ERROR_REF(error), "rr_shutdown");
-  if (subchannel_list_ != nullptr) {
-    grpc_lb_subchannel_list_shutdown_and_unref(subchannel_list_,
-                                               "sl_shutdown_rr_shutdown");
-    subchannel_list_ = nullptr;
-  }
-  if (latest_pending_subchannel_list_ != nullptr) {
-    grpc_lb_subchannel_list_shutdown_and_unref(
-        latest_pending_subchannel_list_, "sl_shutdown_pending_rr_shutdown");
-    latest_pending_subchannel_list_ = nullptr;
-  }
+  subchannel_list_.reset();
+  latest_pending_subchannel_list_.reset();
   TryReresolutionLocked(&grpc_lb_round_robin_trace, GRPC_ERROR_CANCELLED);
   GRPC_ERROR_UNREF(error);
 }
@@ -273,70 +299,59 @@ void RoundRobin::CancelMatchingPicksLocked(uint32_t initial_metadata_flags_mask,
   GRPC_ERROR_UNREF(error);
 }
 
-void RoundRobin::SubchannelListRefForConnectivityWatch(
-    grpc_lb_subchannel_list* subchannel_list, const char* reason) {
-  // TODO(roth): We currently track this ref manually.  Once the new
-  // ClosureRef API is ready and the subchannel_list code has been
-  // converted to a C++ API, find a way to hold the RefCountedPtr<>
-  // somewhere (maybe in the subchannel_data object) instead of doing
-  // this manually.
-  auto self = Ref(DEBUG_LOCATION, reason);
-  self.release();
-  grpc_lb_subchannel_list_ref(subchannel_list, reason);
+void RoundRobin::StartPickingLocked() {
+  started_picking_ = true;
+  subchannel_list_->StartWatchingLocked();
 }
 
-void RoundRobin::SubchannelListUnrefForConnectivityWatch(
-    grpc_lb_subchannel_list* subchannel_list, const char* reason) {
-  Unref(DEBUG_LOCATION, reason);
-  grpc_lb_subchannel_list_unref(subchannel_list, reason);
+void RoundRobin::ExitIdleLocked() {
+  if (!started_picking_) {
+    StartPickingLocked();
+  }
 }
 
-void RoundRobin::StartPickingLocked() {
-  started_picking_ = true;
-  for (size_t i = 0; i < subchannel_list_->num_subchannels; i++) {
-    if (subchannel_list_->subchannels[i].subchannel != nullptr) {
-      SubchannelListRefForConnectivityWatch(subchannel_list_,
-                                            "connectivity_watch");
-      grpc_lb_subchannel_data_start_connectivity_watch(
-          &subchannel_list_->subchannels[i]);
+bool RoundRobin::DoPickLocked(PickState* pick) {
+  const size_t next_ready_index =
+      subchannel_list_->GetNextReadySubchannelIndexLocked();
+  if (next_ready_index < subchannel_list_->num_subchannels()) {
+    /* readily available, report right away */
+    RoundRobinSubchannelData* sd =
+        subchannel_list_->subchannel(next_ready_index);
+    GPR_ASSERT(sd->connected_subchannel() != nullptr);
+    pick->connected_subchannel = sd->connected_subchannel()->Ref();
+    if (pick->user_data != nullptr) {
+      *pick->user_data = sd->user_data();
     }
+    if (grpc_lb_round_robin_trace.enabled()) {
+      gpr_log(GPR_INFO,
+              "[RR %p] Picked target <-- Subchannel %p (connected %p) (sl %p, "
+              "index %" PRIuPTR ")",
+              this, sd->subchannel(), pick->connected_subchannel.get(),
+              sd->subchannel_list(), next_ready_index);
+    }
+    /* only advance the last picked pointer if the selection was used */
+    subchannel_list_->UpdateLastReadySubchannelIndexLocked(next_ready_index);
+    return true;
   }
+  return false;
 }
 
-void RoundRobin::ExitIdleLocked() {
-  if (!started_picking_) {
-    StartPickingLocked();
+void RoundRobin::DrainPendingPicksLocked() {
+  PickState* pick;
+  while ((pick = pending_picks_)) {
+    pending_picks_ = pick->next;
+    GPR_ASSERT(DoPickLocked(pick));
+    GRPC_CLOSURE_SCHED(pick->on_complete, GRPC_ERROR_NONE);
   }
 }
 
 bool RoundRobin::PickLocked(PickState* pick) {
   if (grpc_lb_round_robin_trace.enabled()) {
-    gpr_log(GPR_DEBUG, "[RR %p] Trying to pick (shutdown: %d)", this,
-            shutdown_);
+    gpr_log(GPR_INFO, "[RR %p] Trying to pick (shutdown: %d)", this, shutdown_);
   }
   GPR_ASSERT(!shutdown_);
   if (subchannel_list_ != nullptr) {
-    const size_t next_ready_index = GetNextReadySubchannelIndexLocked();
-    if (next_ready_index < subchannel_list_->num_subchannels) {
-      /* readily available, report right away */
-      grpc_lb_subchannel_data* sd =
-          &subchannel_list_->subchannels[next_ready_index];
-      pick->connected_subchannel = sd->connected_subchannel;
-      if (pick->user_data != nullptr) {
-        *pick->user_data = sd->user_data;
-      }
-      if (grpc_lb_round_robin_trace.enabled()) {
-        gpr_log(
-            GPR_DEBUG,
-            "[RR %p] Picked target <-- Subchannel %p (connected %p) (sl %p, "
-            "index %" PRIuPTR ")",
-            this, sd->subchannel, pick->connected_subchannel.get(),
-            sd->subchannel_list, next_ready_index);
-      }
-      /* only advance the last picked pointer if the selection was used */
-      UpdateLastReadySubchannelIndexLocked(next_ready_index);
-      return true;
-    }
+    if (DoPickLocked(pick)) return true;
   }
   /* no pick currently available. Save for later in list of pending picks */
   if (!started_picking_) {
@@ -347,36 +362,62 @@ bool RoundRobin::PickLocked(PickState* pick) {
   return false;
 }
 
-void UpdateStateCountersLocked(grpc_lb_subchannel_data* sd) {
-  grpc_lb_subchannel_list* subchannel_list = sd->subchannel_list;
-  GPR_ASSERT(sd->prev_connectivity_state != GRPC_CHANNEL_SHUTDOWN);
-  GPR_ASSERT(sd->curr_connectivity_state != GRPC_CHANNEL_SHUTDOWN);
-  if (sd->prev_connectivity_state == GRPC_CHANNEL_READY) {
-    GPR_ASSERT(subchannel_list->num_ready > 0);
-    --subchannel_list->num_ready;
-  } else if (sd->prev_connectivity_state == GRPC_CHANNEL_TRANSIENT_FAILURE) {
-    GPR_ASSERT(subchannel_list->num_transient_failures > 0);
-    --subchannel_list->num_transient_failures;
-  } else if (sd->prev_connectivity_state == GRPC_CHANNEL_IDLE) {
-    GPR_ASSERT(subchannel_list->num_idle > 0);
-    --subchannel_list->num_idle;
+void RoundRobin::RoundRobinSubchannelList::StartWatchingLocked() {
+  if (num_subchannels() == 0) return;
+  // Check current state of each subchannel synchronously, since any
+  // subchannel already used by some other channel may have a non-IDLE
+  // state.
+  for (size_t i = 0; i < num_subchannels(); ++i) {
+    grpc_error* error = GRPC_ERROR_NONE;
+    grpc_connectivity_state state =
+        subchannel(i)->CheckConnectivityStateLocked(&error);
+    if (state != GRPC_CHANNEL_IDLE) {
+      subchannel(i)->UpdateConnectivityStateLocked(state, error);
+    }
   }
-  sd->prev_connectivity_state = sd->curr_connectivity_state;
-  if (sd->curr_connectivity_state == GRPC_CHANNEL_READY) {
-    ++subchannel_list->num_ready;
-  } else if (sd->curr_connectivity_state == GRPC_CHANNEL_TRANSIENT_FAILURE) {
-    ++subchannel_list->num_transient_failures;
-  } else if (sd->curr_connectivity_state == GRPC_CHANNEL_IDLE) {
-    ++subchannel_list->num_idle;
+  // Now set the LB policy's state based on the subchannels' states.
+  UpdateRoundRobinStateFromSubchannelStateCountsLocked();
+  // Start connectivity watch for each subchannel.
+  for (size_t i = 0; i < num_subchannels(); i++) {
+    if (subchannel(i)->subchannel() != nullptr) {
+      subchannel(i)->StartConnectivityWatchLocked();
+    }
   }
 }
 
-/** Sets the policy's connectivity status based on that of the passed-in \a sd
- * (the grpc_lb_subchannel_data associated with the updated subchannel) and the
- * subchannel list \a sd belongs to (sd->subchannel_list). \a error will be used
- * only if the policy transitions to state TRANSIENT_FAILURE. */
-void RoundRobin::UpdateConnectivityStatusLocked(grpc_lb_subchannel_data* sd,
-                                                grpc_error* error) {
+void RoundRobin::RoundRobinSubchannelList::UpdateStateCountersLocked(
+    grpc_connectivity_state old_state, grpc_connectivity_state new_state,
+    grpc_error* transient_failure_error) {
+  GPR_ASSERT(old_state != GRPC_CHANNEL_SHUTDOWN);
+  GPR_ASSERT(new_state != GRPC_CHANNEL_SHUTDOWN);
+  if (old_state == GRPC_CHANNEL_READY) {
+    GPR_ASSERT(num_ready_ > 0);
+    --num_ready_;
+  } else if (old_state == GRPC_CHANNEL_CONNECTING) {
+    GPR_ASSERT(num_connecting_ > 0);
+    --num_connecting_;
+  } else if (old_state == GRPC_CHANNEL_TRANSIENT_FAILURE) {
+    GPR_ASSERT(num_transient_failure_ > 0);
+    --num_transient_failure_;
+  }
+  if (new_state == GRPC_CHANNEL_READY) {
+    ++num_ready_;
+  } else if (new_state == GRPC_CHANNEL_CONNECTING) {
+    ++num_connecting_;
+  } else if (new_state == GRPC_CHANNEL_TRANSIENT_FAILURE) {
+    ++num_transient_failure_;
+  }
+  GRPC_ERROR_UNREF(last_transient_failure_error_);
+  last_transient_failure_error_ = transient_failure_error;
+}
+
+// Sets the RR policy's connectivity state based on the current
+// subchannel list.
+void RoundRobin::RoundRobinSubchannelList::
+    MaybeUpdateRoundRobinConnectivityStateLocked() {
+  RoundRobin* p = static_cast<RoundRobin*>(policy());
+  // Only set connectivity state if this is the current subchannel list.
+  if (p->subchannel_list_.get() != this) return;
   /* In priority order. The first rule to match terminates the search (ie, if we
    * are on rule n, all previous rules were unfulfilled).
    *
@@ -391,155 +432,151 @@ void RoundRobin::UpdateConnectivityStatusLocked(grpc_lb_subchannel_data* sd,
    *    CHECK: subchannel_list->num_transient_failures ==
    *           subchannel_list->num_subchannels.
    */
-  grpc_lb_subchannel_list* subchannel_list = sd->subchannel_list;
-  GPR_ASSERT(sd->curr_connectivity_state != GRPC_CHANNEL_IDLE);
-  if (subchannel_list->num_ready > 0) {
+  if (num_ready_ > 0) {
     /* 1) READY */
-    grpc_connectivity_state_set(&state_tracker_, GRPC_CHANNEL_READY,
+    grpc_connectivity_state_set(&p->state_tracker_, GRPC_CHANNEL_READY,
                                 GRPC_ERROR_NONE, "rr_ready");
-  } else if (sd->curr_connectivity_state == GRPC_CHANNEL_CONNECTING) {
+  } else if (num_connecting_ > 0) {
     /* 2) CONNECTING */
-    grpc_connectivity_state_set(&state_tracker_, GRPC_CHANNEL_CONNECTING,
+    grpc_connectivity_state_set(&p->state_tracker_, GRPC_CHANNEL_CONNECTING,
                                 GRPC_ERROR_NONE, "rr_connecting");
-  } else if (subchannel_list->num_transient_failures ==
-             subchannel_list->num_subchannels) {
+  } else if (num_transient_failure_ == num_subchannels()) {
     /* 3) TRANSIENT_FAILURE */
-    grpc_connectivity_state_set(&state_tracker_, GRPC_CHANNEL_TRANSIENT_FAILURE,
-                                GRPC_ERROR_REF(error),
+    grpc_connectivity_state_set(&p->state_tracker_,
+                                GRPC_CHANNEL_TRANSIENT_FAILURE,
+                                GRPC_ERROR_REF(last_transient_failure_error_),
                                 "rr_exhausted_subchannels");
   }
-  GRPC_ERROR_UNREF(error);
 }
 
-void RoundRobin::OnConnectivityChangedLocked(void* arg, grpc_error* error) {
-  grpc_lb_subchannel_data* sd = static_cast<grpc_lb_subchannel_data*>(arg);
-  RoundRobin* p = static_cast<RoundRobin*>(sd->subchannel_list->policy);
+void RoundRobin::RoundRobinSubchannelList::
+    UpdateRoundRobinStateFromSubchannelStateCountsLocked() {
+  RoundRobin* p = static_cast<RoundRobin*>(policy());
+  if (num_ready_ > 0) {
+    if (p->subchannel_list_.get() != this) {
+      // Promote this list to p->subchannel_list_.
+      // This list must be p->latest_pending_subchannel_list_, because
+      // any previous update would have been shut down already and
+      // therefore we would not be receiving a notification for them.
+      GPR_ASSERT(p->latest_pending_subchannel_list_.get() == this);
+      GPR_ASSERT(!shutting_down());
+      if (grpc_lb_round_robin_trace.enabled()) {
+        const size_t old_num_subchannels =
+            p->subchannel_list_ != nullptr
+                ? p->subchannel_list_->num_subchannels()
+                : 0;
+        gpr_log(GPR_INFO,
+                "[RR %p] phasing out subchannel list %p (size %" PRIuPTR
+                ") in favor of %p (size %" PRIuPTR ")",
+                p, p->subchannel_list_.get(), old_num_subchannels, this,
+                num_subchannels());
+      }
+      p->subchannel_list_ = std::move(p->latest_pending_subchannel_list_);
+    }
+    // Drain pending picks.
+    p->DrainPendingPicksLocked();
+  }
+  // Update the RR policy's connectivity state if needed.
+  MaybeUpdateRoundRobinConnectivityStateLocked();
+}
+
+void RoundRobin::RoundRobinSubchannelData::UpdateConnectivityStateLocked(
+    grpc_connectivity_state connectivity_state, grpc_error* error) {
+  RoundRobin* p = static_cast<RoundRobin*>(subchannel_list()->policy());
   if (grpc_lb_round_robin_trace.enabled()) {
     gpr_log(
-        GPR_DEBUG,
-        "[RR %p] connectivity changed for subchannel %p, subchannel_list %p: "
-        "prev_state=%s new_state=%s p->shutdown=%d "
-        "sd->subchannel_list->shutting_down=%d error=%s",
-        p, sd->subchannel, sd->subchannel_list,
-        grpc_connectivity_state_name(sd->prev_connectivity_state),
-        grpc_connectivity_state_name(sd->pending_connectivity_state_unsafe),
-        p->shutdown_, sd->subchannel_list->shutting_down,
-        grpc_error_string(error));
-  }
-  GPR_ASSERT(sd->subchannel != nullptr);
-  // If the policy is shutting down, unref and return.
-  if (p->shutdown_) {
-    grpc_lb_subchannel_data_stop_connectivity_watch(sd);
-    grpc_lb_subchannel_data_unref_subchannel(sd, "rr_shutdown");
-    p->SubchannelListUnrefForConnectivityWatch(sd->subchannel_list,
-                                               "rr_shutdown");
-    return;
+        GPR_INFO,
+        "[RR %p] connectivity changed for subchannel %p, subchannel_list %p "
+        "(index %" PRIuPTR " of %" PRIuPTR "): prev_state=%s new_state=%s",
+        p, subchannel(), subchannel_list(), Index(),
+        subchannel_list()->num_subchannels(),
+        grpc_connectivity_state_name(last_connectivity_state_),
+        grpc_connectivity_state_name(connectivity_state));
+  }
+  subchannel_list()->UpdateStateCountersLocked(last_connectivity_state_,
+                                               connectivity_state, error);
+  last_connectivity_state_ = connectivity_state;
+}
+
+void RoundRobin::RoundRobinSubchannelData::ProcessConnectivityChangeLocked(
+    grpc_connectivity_state connectivity_state, grpc_error* error) {
+  RoundRobin* p = static_cast<RoundRobin*>(subchannel_list()->policy());
+  GPR_ASSERT(subchannel() != nullptr);
+  // If the new state is TRANSIENT_FAILURE, re-resolve.
+  // Only do this if we've started watching, not at startup time.
+  // Otherwise, if the subchannel was already in state TRANSIENT_FAILURE
+  // when the subchannel list was created, we'd wind up in a constant
+  // loop of re-resolution.
+  if (connectivity_state == GRPC_CHANNEL_TRANSIENT_FAILURE) {
+    if (grpc_lb_round_robin_trace.enabled()) {
+      gpr_log(GPR_INFO,
+              "[RR %p] Subchannel %p has gone into TRANSIENT_FAILURE. "
+              "Requesting re-resolution",
+              p, subchannel());
+    }
+    p->TryReresolutionLocked(&grpc_lb_round_robin_trace, GRPC_ERROR_NONE);
   }
-  // If the subchannel list is shutting down, stop watching.
-  if (sd->subchannel_list->shutting_down || error == GRPC_ERROR_CANCELLED) {
-    grpc_lb_subchannel_data_stop_connectivity_watch(sd);
-    grpc_lb_subchannel_data_unref_subchannel(sd, "rr_sl_shutdown");
-    p->SubchannelListUnrefForConnectivityWatch(sd->subchannel_list,
-                                               "rr_sl_shutdown");
-    return;
+  // Update state counters.
+  UpdateConnectivityStateLocked(connectivity_state, error);
+  // Update overall state and renew notification.
+  subchannel_list()->UpdateRoundRobinStateFromSubchannelStateCountsLocked();
+  RenewConnectivityWatchLocked();
+}
+
+/** Returns the index into p->subchannel_list->subchannels of the next
+ * subchannel in READY state, or p->subchannel_list->num_subchannels if no
+ * subchannel is READY.
+ *
+ * Note that this function does *not* update p->last_ready_subchannel_index.
+ * The caller must do that if it returns a pick. */
+size_t
+RoundRobin::RoundRobinSubchannelList::GetNextReadySubchannelIndexLocked() {
+  if (grpc_lb_round_robin_trace.enabled()) {
+    gpr_log(GPR_INFO,
+            "[RR %p] getting next ready subchannel (out of %" PRIuPTR
+            "), last_ready_index=%" PRIuPTR,
+            policy(), num_subchannels(), last_ready_index_);
   }
-  // If we're still here, the notification must be for a subchannel in
-  // either the current or latest pending subchannel lists.
-  GPR_ASSERT(sd->subchannel_list == p->subchannel_list_ ||
-             sd->subchannel_list == p->latest_pending_subchannel_list_);
-  GPR_ASSERT(sd->pending_connectivity_state_unsafe != GRPC_CHANNEL_SHUTDOWN);
-  // Now that we're inside the combiner, copy the pending connectivity
-  // state (which was set by the connectivity state watcher) to
-  // curr_connectivity_state, which is what we use inside of the combiner.
-  sd->curr_connectivity_state = sd->pending_connectivity_state_unsafe;
-  // If the sd's new state is TRANSIENT_FAILURE, unref the *connected*
-  // subchannel, if any.
-  switch (sd->curr_connectivity_state) {
-    case GRPC_CHANNEL_TRANSIENT_FAILURE: {
-      sd->connected_subchannel.reset();
-      if (grpc_lb_round_robin_trace.enabled()) {
-        gpr_log(GPR_DEBUG,
-                "[RR %p] Subchannel %p has gone into TRANSIENT_FAILURE. "
-                "Requesting re-resolution",
-                p, sd->subchannel);
-      }
-      p->TryReresolutionLocked(&grpc_lb_round_robin_trace, GRPC_ERROR_NONE);
-      break;
+  for (size_t i = 0; i < num_subchannels(); ++i) {
+    const size_t index = (i + last_ready_index_ + 1) % num_subchannels();
+    if (grpc_lb_round_robin_trace.enabled()) {
+      gpr_log(
+          GPR_INFO,
+          "[RR %p] checking subchannel %p, subchannel_list %p, index %" PRIuPTR
+          ": state=%s",
+          policy(), subchannel(index)->subchannel(), this, index,
+          grpc_connectivity_state_name(
+              subchannel(index)->connectivity_state()));
     }
-    case GRPC_CHANNEL_READY: {
-      if (sd->connected_subchannel == nullptr) {
-        sd->connected_subchannel =
-            grpc_subchannel_get_connected_subchannel(sd->subchannel);
-      }
-      if (sd->subchannel_list != p->subchannel_list_) {
-        // promote sd->subchannel_list to p->subchannel_list_.
-        // sd->subchannel_list must be equal to
-        // p->latest_pending_subchannel_list_ because we have already filtered
-        // for sds belonging to outdated subchannel lists.
-        GPR_ASSERT(sd->subchannel_list == p->latest_pending_subchannel_list_);
-        GPR_ASSERT(!sd->subchannel_list->shutting_down);
-        if (grpc_lb_round_robin_trace.enabled()) {
-          const size_t num_subchannels =
-              p->subchannel_list_ != nullptr
-                  ? p->subchannel_list_->num_subchannels
-                  : 0;
-          gpr_log(GPR_DEBUG,
-                  "[RR %p] phasing out subchannel list %p (size %" PRIuPTR
-                  ") in favor of %p (size %" PRIuPTR ")",
-                  p, p->subchannel_list_, num_subchannels, sd->subchannel_list,
-                  num_subchannels);
-        }
-        if (p->subchannel_list_ != nullptr) {
-          // dispose of the current subchannel_list
-          grpc_lb_subchannel_list_shutdown_and_unref(p->subchannel_list_,
-                                                     "sl_phase_out_shutdown");
-        }
-        p->subchannel_list_ = p->latest_pending_subchannel_list_;
-        p->latest_pending_subchannel_list_ = nullptr;
-      }
-      /* at this point we know there's at least one suitable subchannel. Go
-       * ahead and pick one and notify the pending suitors in
-       * p->pending_picks. This preemptively replicates rr_pick()'s actions. */
-      const size_t next_ready_index = p->GetNextReadySubchannelIndexLocked();
-      GPR_ASSERT(next_ready_index < p->subchannel_list_->num_subchannels);
-      grpc_lb_subchannel_data* selected =
-          &p->subchannel_list_->subchannels[next_ready_index];
-      if (p->pending_picks_ != nullptr) {
-        // if the selected subchannel is going to be used for the pending
-        // picks, update the last picked pointer
-        p->UpdateLastReadySubchannelIndexLocked(next_ready_index);
-      }
-      PickState* pick;
-      while ((pick = p->pending_picks_)) {
-        p->pending_picks_ = pick->next;
-        pick->connected_subchannel = selected->connected_subchannel;
-        if (pick->user_data != nullptr) {
-          *pick->user_data = selected->user_data;
-        }
-        if (grpc_lb_round_robin_trace.enabled()) {
-          gpr_log(GPR_DEBUG,
-                  "[RR %p] Fulfilling pending pick. Target <-- subchannel %p "
-                  "(subchannel_list %p, index %" PRIuPTR ")",
-                  p, selected->subchannel, p->subchannel_list_,
-                  next_ready_index);
-        }
-        GRPC_CLOSURE_SCHED(pick->on_complete, GRPC_ERROR_NONE);
+    if (subchannel(index)->connectivity_state() == GRPC_CHANNEL_READY) {
+      if (grpc_lb_round_robin_trace.enabled()) {
+        gpr_log(GPR_INFO,
+                "[RR %p] found next ready subchannel (%p) at index %" PRIuPTR
+                " of subchannel_list %p",
+                policy(), subchannel(index)->subchannel(), index, this);
       }
-      break;
+      return index;
     }
-    case GRPC_CHANNEL_SHUTDOWN:
-      GPR_UNREACHABLE_CODE(return );
-    case GRPC_CHANNEL_CONNECTING:
-    case GRPC_CHANNEL_IDLE:;  // fallthrough
   }
-  // Update state counters.
-  UpdateStateCountersLocked(sd);
-  // Only update connectivity based on the selected subchannel list.
-  if (sd->subchannel_list == p->subchannel_list_) {
-    p->UpdateConnectivityStatusLocked(sd, GRPC_ERROR_REF(error));
+  if (grpc_lb_round_robin_trace.enabled()) {
+    gpr_log(GPR_INFO, "[RR %p] no subchannels in ready state", this);
+  }
+  return num_subchannels();
+}
+
+// Sets last_ready_index_ to last_ready_index.
+void RoundRobin::RoundRobinSubchannelList::UpdateLastReadySubchannelIndexLocked(
+    size_t last_ready_index) {
+  GPR_ASSERT(last_ready_index < num_subchannels());
+  last_ready_index_ = last_ready_index;
+  if (grpc_lb_round_robin_trace.enabled()) {
+    gpr_log(GPR_INFO,
+            "[RR %p] setting last_ready_subchannel_index=%" PRIuPTR
+            " (SC %p, CSC %p)",
+            policy(), last_ready_index,
+            subchannel(last_ready_index)->subchannel(),
+            subchannel(last_ready_index)->connected_subchannel());
   }
-  // Renew notification.
-  grpc_lb_subchannel_data_start_connectivity_watch(sd);
 }
 
 grpc_connectivity_state RoundRobin::CheckConnectivityLocked(
@@ -555,11 +592,12 @@ void RoundRobin::NotifyOnStateChangeLocked(grpc_connectivity_state* current,
 
 void RoundRobin::PingOneLocked(grpc_closure* on_initiate,
                                grpc_closure* on_ack) {
-  const size_t next_ready_index = GetNextReadySubchannelIndexLocked();
-  if (next_ready_index < subchannel_list_->num_subchannels) {
-    grpc_lb_subchannel_data* selected =
-        &subchannel_list_->subchannels[next_ready_index];
-    selected->connected_subchannel->Ping(on_initiate, on_ack);
+  const size_t next_ready_index =
+      subchannel_list_->GetNextReadySubchannelIndexLocked();
+  if (next_ready_index < subchannel_list_->num_subchannels()) {
+    RoundRobinSubchannelData* selected =
+        subchannel_list_->subchannel(next_ready_index);
+    selected->connected_subchannel()->Ping(on_initiate, on_ack);
   } else {
     GRPC_CLOSURE_SCHED(on_initiate, GRPC_ERROR_CREATE_FROM_STATIC_STRING(
                                         "Round Robin not connected"));
@@ -570,7 +608,7 @@ void RoundRobin::PingOneLocked(grpc_closure* on_initiate,
 
 void RoundRobin::UpdateLocked(const grpc_channel_args& args) {
   const grpc_arg* arg = grpc_channel_args_find(&args, GRPC_ARG_LB_ADDRESSES);
-  if (arg == nullptr || arg->type != GRPC_ARG_POINTER) {
+  if (GPR_UNLIKELY(arg == nullptr || arg->type != GRPC_ARG_POINTER)) {
     gpr_log(GPR_ERROR, "[RR %p] update provided no addresses; ignoring", this);
     // If we don't have a current subchannel list, go into TRANSIENT_FAILURE.
     // Otherwise, keep using the current subchannel list (ignore this update).
@@ -582,80 +620,37 @@ void RoundRobin::UpdateLocked(const grpc_channel_args& args) {
     }
     return;
   }
-  grpc_lb_addresses* addresses = (grpc_lb_addresses*)arg->value.pointer.p;
+  grpc_lb_addresses* addresses =
+      static_cast<grpc_lb_addresses*>(arg->value.pointer.p);
   if (grpc_lb_round_robin_trace.enabled()) {
-    gpr_log(GPR_DEBUG, "[RR %p] received update with %" PRIuPTR " addresses",
+    gpr_log(GPR_INFO, "[RR %p] received update with %" PRIuPTR " addresses",
             this, addresses->num_addresses);
   }
-  grpc_lb_subchannel_list* subchannel_list = grpc_lb_subchannel_list_create(
-      this, &grpc_lb_round_robin_trace, addresses, combiner(),
-      client_channel_factory(), args, &RoundRobin::OnConnectivityChangedLocked);
-  if (subchannel_list->num_subchannels == 0) {
-    grpc_connectivity_state_set(
-        &state_tracker_, GRPC_CHANNEL_TRANSIENT_FAILURE,
-        GRPC_ERROR_CREATE_FROM_STATIC_STRING("Empty update"),
-        "rr_update_empty");
-    if (subchannel_list_ != nullptr) {
-      grpc_lb_subchannel_list_shutdown_and_unref(subchannel_list_,
-                                                 "sl_shutdown_empty_update");
+  // Replace latest_pending_subchannel_list_.
+  if (latest_pending_subchannel_list_ != nullptr) {
+    if (grpc_lb_round_robin_trace.enabled()) {
+      gpr_log(GPR_INFO,
+              "[RR %p] Shutting down previous pending subchannel list %p", this,
+              latest_pending_subchannel_list_.get());
     }
-    subchannel_list_ = subchannel_list;  // empty list
-    return;
   }
-  if (started_picking_) {
-    for (size_t i = 0; i < subchannel_list->num_subchannels; ++i) {
-      const grpc_connectivity_state subchannel_state =
-          grpc_subchannel_check_connectivity(
-              subchannel_list->subchannels[i].subchannel, nullptr);
-      // Override the default setting of IDLE for connectivity notification
-      // purposes if the subchannel is already in transient failure. Otherwise
-      // we'd be immediately notified of the IDLE-TRANSIENT_FAILURE
-      // discrepancy, attempt to re-resolve and end up here again.
-      // TODO(roth): As part of C++-ifying the subchannel_list API, design a
-      // better API for notifying the LB policy of subchannel states, which can
-      // be used both for the subchannel's initial state and for subsequent
-      // state changes. This will allow us to handle this more generally instead
-      // of special-casing TRANSIENT_FAILURE (e.g., we can also distribute any
-      // pending picks across all READY subchannels rather than sending them all
-      // to the first one).
-      if (subchannel_state == GRPC_CHANNEL_TRANSIENT_FAILURE) {
-        subchannel_list->subchannels[i].pending_connectivity_state_unsafe =
-            subchannel_list->subchannels[i].curr_connectivity_state =
-                subchannel_list->subchannels[i].prev_connectivity_state =
-                    subchannel_state;
-        --subchannel_list->num_idle;
-        ++subchannel_list->num_transient_failures;
-      }
-    }
-    if (latest_pending_subchannel_list_ != nullptr) {
-      if (grpc_lb_round_robin_trace.enabled()) {
-        gpr_log(GPR_DEBUG,
-                "[RR %p] Shutting down latest pending subchannel list %p, "
-                "about to be replaced by newer latest %p",
-                this, latest_pending_subchannel_list_, subchannel_list);
-      }
-      grpc_lb_subchannel_list_shutdown_and_unref(
-          latest_pending_subchannel_list_, "sl_outdated");
-    }
-    latest_pending_subchannel_list_ = subchannel_list;
-    for (size_t i = 0; i < subchannel_list->num_subchannels; ++i) {
-      /* Watch every new subchannel. A subchannel list becomes active the
-       * moment one of its subchannels is READY. At that moment, we swap
-       * p->subchannel_list for sd->subchannel_list, provided the subchannel
-       * list is still valid (ie, isn't shutting down) */
-      SubchannelListRefForConnectivityWatch(subchannel_list,
-                                            "connectivity_watch");
-      grpc_lb_subchannel_data_start_connectivity_watch(
-          &subchannel_list->subchannels[i]);
+  latest_pending_subchannel_list_ = MakeOrphanable<RoundRobinSubchannelList>(
+      this, &grpc_lb_round_robin_trace, addresses, combiner(),
+      client_channel_factory(), args);
+  // If we haven't started picking yet or the new list is empty,
+  // immediately promote the new list to the current list.
+  if (!started_picking_ ||
+      latest_pending_subchannel_list_->num_subchannels() == 0) {
+    if (latest_pending_subchannel_list_->num_subchannels() == 0) {
+      grpc_connectivity_state_set(
+          &state_tracker_, GRPC_CHANNEL_TRANSIENT_FAILURE,
+          GRPC_ERROR_CREATE_FROM_STATIC_STRING("Empty update"),
+          "rr_update_empty");
     }
+    subchannel_list_ = std::move(latest_pending_subchannel_list_);
   } else {
-    // The policy isn't picking yet. Save the update for later, disposing of
-    // previous version if any.
-    if (subchannel_list_ != nullptr) {
-      grpc_lb_subchannel_list_shutdown_and_unref(
-          subchannel_list_, "rr_update_before_started_picking");
-    }
-    subchannel_list_ = subchannel_list;
+    // If we've started picking, start watching the new list.
+    latest_pending_subchannel_list_->StartWatchingLocked();
   }
 }
 

src/core/ext/filters/client_channel/lb_policy/subchannel_list.cc

@@ -1,253 +0,0 @@
-/*
- *
- * Copyright 2015 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#include <grpc/support/port_platform.h>
-
-#include <string.h>
-
-#include <grpc/support/alloc.h>
-
-#include "src/core/ext/filters/client_channel/lb_policy/subchannel_list.h"
-#include "src/core/lib/channel/channel_args.h"
-#include "src/core/lib/debug/trace.h"
-#include "src/core/lib/iomgr/closure.h"
-#include "src/core/lib/iomgr/combiner.h"
-#include "src/core/lib/iomgr/sockaddr_utils.h"
-#include "src/core/lib/transport/connectivity_state.h"
-
-void grpc_lb_subchannel_data_unref_subchannel(grpc_lb_subchannel_data* sd,
-                                              const char* reason) {
-  if (sd->subchannel != nullptr) {
-    if (sd->subchannel_list->tracer->enabled()) {
-      gpr_log(GPR_DEBUG,
-              "[%s %p] subchannel list %p index %" PRIuPTR " of %" PRIuPTR
-              " (subchannel %p): unreffing subchannel",
-              sd->subchannel_list->tracer->name(), sd->subchannel_list->policy,
-              sd->subchannel_list,
-              static_cast<size_t>(sd - sd->subchannel_list->subchannels),
-              sd->subchannel_list->num_subchannels, sd->subchannel);
-    }
-    GRPC_SUBCHANNEL_UNREF(sd->subchannel, reason);
-    sd->subchannel = nullptr;
-    sd->connected_subchannel.reset();
-    if (sd->user_data != nullptr) {
-      GPR_ASSERT(sd->user_data_vtable != nullptr);
-      sd->user_data_vtable->destroy(sd->user_data);
-      sd->user_data = nullptr;
-    }
-  }
-}
-
-void grpc_lb_subchannel_data_start_connectivity_watch(
-    grpc_lb_subchannel_data* sd) {
-  if (sd->subchannel_list->tracer->enabled()) {
-    gpr_log(
-        GPR_DEBUG,
-        "[%s %p] subchannel list %p index %" PRIuPTR " of %" PRIuPTR
-        " (subchannel %p): requesting connectivity change "
-        "notification (from %s)",
-        sd->subchannel_list->tracer->name(), sd->subchannel_list->policy,
-        sd->subchannel_list,
-        static_cast<size_t>(sd - sd->subchannel_list->subchannels),
-        sd->subchannel_list->num_subchannels, sd->subchannel,
-        grpc_connectivity_state_name(sd->pending_connectivity_state_unsafe));
-  }
-  sd->connectivity_notification_pending = true;
-  grpc_subchannel_notify_on_state_change(
-      sd->subchannel, sd->subchannel_list->policy->interested_parties(),
-      &sd->pending_connectivity_state_unsafe,
-      &sd->connectivity_changed_closure);
-}
-
-void grpc_lb_subchannel_data_stop_connectivity_watch(
-    grpc_lb_subchannel_data* sd) {
-  if (sd->subchannel_list->tracer->enabled()) {
-    gpr_log(GPR_DEBUG,
-            "[%s %p] subchannel list %p index %" PRIuPTR " of %" PRIuPTR
-            " (subchannel %p): stopping connectivity watch",
-            sd->subchannel_list->tracer->name(), sd->subchannel_list->policy,
-            sd->subchannel_list,
-            static_cast<size_t>(sd - sd->subchannel_list->subchannels),
-            sd->subchannel_list->num_subchannels, sd->subchannel);
-  }
-  GPR_ASSERT(sd->connectivity_notification_pending);
-  sd->connectivity_notification_pending = false;
-}
-
-grpc_lb_subchannel_list* grpc_lb_subchannel_list_create(
-    grpc_core::LoadBalancingPolicy* p, grpc_core::TraceFlag* tracer,
-    const grpc_lb_addresses* addresses, grpc_combiner* combiner,
-    grpc_client_channel_factory* client_channel_factory,
-    const grpc_channel_args& args, grpc_iomgr_cb_func connectivity_changed_cb) {
-  grpc_lb_subchannel_list* subchannel_list =
-      static_cast<grpc_lb_subchannel_list*>(
-          gpr_zalloc(sizeof(*subchannel_list)));
-  if (tracer->enabled()) {
-    gpr_log(GPR_DEBUG,
-            "[%s %p] Creating subchannel list %p for %" PRIuPTR " subchannels",
-            tracer->name(), p, subchannel_list, addresses->num_addresses);
-  }
-  subchannel_list->policy = p;
-  subchannel_list->tracer = tracer;
-  gpr_ref_init(&subchannel_list->refcount, 1);
-  subchannel_list->subchannels = static_cast<grpc_lb_subchannel_data*>(
-      gpr_zalloc(sizeof(grpc_lb_subchannel_data) * addresses->num_addresses));
-  // We need to remove the LB addresses in order to be able to compare the
-  // subchannel keys of subchannels from a different batch of addresses.
-  static const char* keys_to_remove[] = {GRPC_ARG_SUBCHANNEL_ADDRESS,
-                                         GRPC_ARG_LB_ADDRESSES};
-  // Create a subchannel for each address.
-  grpc_subchannel_args sc_args;
-  size_t subchannel_index = 0;
-  for (size_t i = 0; i < addresses->num_addresses; i++) {
-    // If there were any balancer, we would have chosen grpclb policy instead.
-    GPR_ASSERT(!addresses->addresses[i].is_balancer);
-    memset(&sc_args, 0, sizeof(grpc_subchannel_args));
-    grpc_arg addr_arg =
-        grpc_create_subchannel_address_arg(&addresses->addresses[i].address);
-    grpc_channel_args* new_args = grpc_channel_args_copy_and_add_and_remove(
-        &args, keys_to_remove, GPR_ARRAY_SIZE(keys_to_remove), &addr_arg, 1);
-    gpr_free(addr_arg.value.string);
-    sc_args.args = new_args;
-    grpc_subchannel* subchannel = grpc_client_channel_factory_create_subchannel(
-        client_channel_factory, &sc_args);
-    grpc_channel_args_destroy(new_args);
-    if (subchannel == nullptr) {
-      // Subchannel could not be created.
-      if (tracer->enabled()) {
-        char* address_uri =
-            grpc_sockaddr_to_uri(&addresses->addresses[i].address);
-        gpr_log(GPR_DEBUG,
-                "[%s %p] could not create subchannel for address uri %s, "
-                "ignoring",
-                tracer->name(), subchannel_list->policy, address_uri);
-        gpr_free(address_uri);
-      }
-      continue;
-    }
-    if (tracer->enabled()) {
-      char* address_uri =
-          grpc_sockaddr_to_uri(&addresses->addresses[i].address);
-      gpr_log(GPR_DEBUG,
-              "[%s %p] subchannel list %p index %" PRIuPTR
-              ": Created subchannel %p for address uri %s",
-              tracer->name(), p, subchannel_list, subchannel_index, subchannel,
-              address_uri);
-      gpr_free(address_uri);
-    }
-    grpc_lb_subchannel_data* sd =
-        &subchannel_list->subchannels[subchannel_index++];
-    sd->subchannel_list = subchannel_list;
-    sd->subchannel = subchannel;
-    GRPC_CLOSURE_INIT(&sd->connectivity_changed_closure,
-                      connectivity_changed_cb, sd,
-                      grpc_combiner_scheduler(combiner));
-    // We assume that the current state is IDLE.  If not, we'll get a
-    // callback telling us that.
-    sd->prev_connectivity_state = GRPC_CHANNEL_IDLE;
-    sd->curr_connectivity_state = GRPC_CHANNEL_IDLE;
-    sd->pending_connectivity_state_unsafe = GRPC_CHANNEL_IDLE;
-    sd->user_data_vtable = addresses->user_data_vtable;
-    if (sd->user_data_vtable != nullptr) {
-      sd->user_data =
-          sd->user_data_vtable->copy(addresses->addresses[i].user_data);
-    }
-  }
-  subchannel_list->num_subchannels = subchannel_index;
-  subchannel_list->num_idle = subchannel_index;
-  return subchannel_list;
-}
-
-static void subchannel_list_destroy(grpc_lb_subchannel_list* subchannel_list) {
-  if (subchannel_list->tracer->enabled()) {
-    gpr_log(GPR_DEBUG, "[%s %p] Destroying subchannel_list %p",
-            subchannel_list->tracer->name(), subchannel_list->policy,
-            subchannel_list);
-  }
-  for (size_t i = 0; i < subchannel_list->num_subchannels; i++) {
-    grpc_lb_subchannel_data* sd = &subchannel_list->subchannels[i];
-    grpc_lb_subchannel_data_unref_subchannel(sd, "subchannel_list_destroy");
-  }
-  gpr_free(subchannel_list->subchannels);
-  gpr_free(subchannel_list);
-}
-
-void grpc_lb_subchannel_list_ref(grpc_lb_subchannel_list* subchannel_list,
-                                 const char* reason) {
-  gpr_ref_non_zero(&subchannel_list->refcount);
-  if (subchannel_list->tracer->enabled()) {
-    const gpr_atm count = gpr_atm_acq_load(&subchannel_list->refcount.count);
-    gpr_log(GPR_DEBUG, "[%s %p] subchannel_list %p REF %lu->%lu (%s)",
-            subchannel_list->tracer->name(), subchannel_list->policy,
-            subchannel_list, static_cast<unsigned long>(count - 1),
-            static_cast<unsigned long>(count), reason);
-  }
-}
-
-void grpc_lb_subchannel_list_unref(grpc_lb_subchannel_list* subchannel_list,
-                                   const char* reason) {
-  const bool done = gpr_unref(&subchannel_list->refcount);
-  if (subchannel_list->tracer->enabled()) {
-    const gpr_atm count = gpr_atm_acq_load(&subchannel_list->refcount.count);
-    gpr_log(GPR_DEBUG, "[%s %p] subchannel_list %p UNREF %lu->%lu (%s)",
-            subchannel_list->tracer->name(), subchannel_list->policy,
-            subchannel_list, static_cast<unsigned long>(count + 1),
-            static_cast<unsigned long>(count), reason);
-  }
-  if (done) {
-    subchannel_list_destroy(subchannel_list);
-  }
-}
-
-static void subchannel_data_cancel_connectivity_watch(
-    grpc_lb_subchannel_data* sd, const char* reason) {
-  if (sd->subchannel_list->tracer->enabled()) {
-    gpr_log(GPR_DEBUG,
-            "[%s %p] subchannel list %p index %" PRIuPTR " of %" PRIuPTR
-            " (subchannel %p): canceling connectivity watch (%s)",
-            sd->subchannel_list->tracer->name(), sd->subchannel_list->policy,
-            sd->subchannel_list,
-            static_cast<size_t>(sd - sd->subchannel_list->subchannels),
-            sd->subchannel_list->num_subchannels, sd->subchannel, reason);
-  }
-  grpc_subchannel_notify_on_state_change(sd->subchannel, nullptr, nullptr,
-                                         &sd->connectivity_changed_closure);
-}
-
-void grpc_lb_subchannel_list_shutdown_and_unref(
-    grpc_lb_subchannel_list* subchannel_list, const char* reason) {
-  if (subchannel_list->tracer->enabled()) {
-    gpr_log(GPR_DEBUG, "[%s %p] Shutting down subchannel_list %p (%s)",
-            subchannel_list->tracer->name(), subchannel_list->policy,
-            subchannel_list, reason);
-  }
-  GPR_ASSERT(!subchannel_list->shutting_down);
-  subchannel_list->shutting_down = true;
-  for (size_t i = 0; i < subchannel_list->num_subchannels; i++) {
-    grpc_lb_subchannel_data* sd = &subchannel_list->subchannels[i];
-    // If there's a pending notification for this subchannel, cancel it;
-    // the callback is responsible for unreffing the subchannel.
-    // Otherwise, unref the subchannel directly.
-    if (sd->connectivity_notification_pending) {
-      subchannel_data_cancel_connectivity_watch(sd, reason);
-    } else if (sd->subchannel != nullptr) {
-      grpc_lb_subchannel_data_unref_subchannel(sd, reason);
-    }
-  }
-  grpc_lb_subchannel_list_unref(subchannel_list, reason);
-}

src/core/ext/filters/client_channel/lb_policy/subchannel_list.h

@@ -21,116 +21,516 @@
 
 #include <grpc/support/port_platform.h>
 
+#include <string.h>
+
+#include <grpc/support/alloc.h>
+
 #include "src/core/ext/filters/client_channel/lb_policy_registry.h"
 #include "src/core/ext/filters/client_channel/subchannel.h"
+#include "src/core/lib/channel/channel_args.h"
 #include "src/core/lib/debug/trace.h"
+#include "src/core/lib/gprpp/abstract.h"
+#include "src/core/lib/gprpp/inlined_vector.h"
+#include "src/core/lib/gprpp/orphanable.h"
+#include "src/core/lib/gprpp/ref_counted.h"
 #include "src/core/lib/gprpp/ref_counted_ptr.h"
+#include "src/core/lib/iomgr/closure.h"
+#include "src/core/lib/iomgr/combiner.h"
+#include "src/core/lib/iomgr/sockaddr_utils.h"
 #include "src/core/lib/transport/connectivity_state.h"
 
-// TODO(roth): This code is intended to be shared between pick_first and
-// round_robin.  However, the interface needs more work to provide clean
-// encapsulation.  For example, the structs here have some fields that are
-// only used in one of the two (e.g., the state counters in
-// grpc_lb_subchannel_list and the prev_connectivity_state field in
-// grpc_lb_subchannel_data are only used in round_robin, and the
-// checking_subchannel field in grpc_lb_subchannel_list is only used by
-// pick_first).  Also, there is probably some code duplication between the
-// connectivity state notification callback code in both pick_first and
-// round_robin that could be refactored and moved here.  In a future PR,
-// need to clean this up.
-
-typedef struct grpc_lb_subchannel_list grpc_lb_subchannel_list;
-
-typedef struct {
-  /** backpointer to owning subchannel list */
-  grpc_lb_subchannel_list* subchannel_list;
-  /** subchannel itself */
-  grpc_subchannel* subchannel;
-  grpc_core::RefCountedPtr<grpc_core::ConnectedSubchannel> connected_subchannel;
-  /** Is a connectivity notification pending? */
-  bool connectivity_notification_pending;
-  /** notification that connectivity has changed on subchannel */
-  grpc_closure connectivity_changed_closure;
-  /** previous and current connectivity states.  Updated by \a
-   * \a connectivity_changed_closure based on
-   * \a pending_connectivity_state_unsafe. */
-  grpc_connectivity_state prev_connectivity_state;
-  grpc_connectivity_state curr_connectivity_state;
-  /** connectivity state to be updated by
-   * grpc_subchannel_notify_on_state_change(), not guarded by
-   * the combiner.  To be copied to \a curr_connectivity_state by
-   * \a connectivity_changed_closure. */
-  grpc_connectivity_state pending_connectivity_state_unsafe;
-  /** the subchannel's target user data */
-  void* user_data;
-  /** vtable to operate over \a user_data */
-  const grpc_lb_user_data_vtable* user_data_vtable;
-} grpc_lb_subchannel_data;
-
-/// Unrefs the subchannel contained in sd.
-void grpc_lb_subchannel_data_unref_subchannel(grpc_lb_subchannel_data* sd,
-                                              const char* reason);
-
-/// Starts watching the connectivity state of the subchannel.
-/// The connectivity_changed_cb callback must invoke either
-/// grpc_lb_subchannel_data_stop_connectivity_watch() or again call
-/// grpc_lb_subchannel_data_start_connectivity_watch().
-void grpc_lb_subchannel_data_start_connectivity_watch(
-    grpc_lb_subchannel_data* sd);
-
-/// Stops watching the connectivity state of the subchannel.
-void grpc_lb_subchannel_data_stop_connectivity_watch(
-    grpc_lb_subchannel_data* sd);
-
-struct grpc_lb_subchannel_list {
-  /** backpointer to owning policy */
-  grpc_core::LoadBalancingPolicy* policy;
-
-  grpc_core::TraceFlag* tracer;
-
-  /** all our subchannels */
-  size_t num_subchannels;
-  grpc_lb_subchannel_data* subchannels;
-
-  /** Index into subchannels of the one we're currently checking.
-   * Used when connecting to subchannels serially instead of in parallel. */
-  // TODO(roth): When we have time, we can probably make this go away
-  // and compute the index dynamically by subtracting
-  // subchannel_list->subchannels from the subchannel_data pointer.
-  size_t checking_subchannel;
-
-  /** how many subchannels are in state READY */
-  size_t num_ready;
-  /** how many subchannels are in state TRANSIENT_FAILURE */
-  size_t num_transient_failures;
-  /** how many subchannels are in state IDLE */
-  size_t num_idle;
-
-  /** There will be one ref for each entry in subchannels for which there is a
-   * pending connectivity state watcher callback. */
-  gpr_refcount refcount;
-
-  /** Is this list shutting down? This may be true due to the shutdown of the
-   * policy itself or because a newer update has arrived while this one hadn't
-   * finished processing. */
-  bool shutting_down;
+// Code for maintaining a list of subchannels within an LB policy.
+//
+// To use this, callers must create their own subclasses, like so:
+/*
+
+class MySubchannelList;  // Forward declaration.
+
+class MySubchannelData
+    : public SubchannelData<MySubchannelList, MySubchannelData> {
+ public:
+  void ProcessConnectivityChangeLocked(
+      grpc_connectivity_state connectivity_state, grpc_error* error) override {
+    // ...code to handle connectivity changes...
+  }
+};
+
+class MySubchannelList
+    : public SubchannelList<MySubchannelList, MySubchannelData> {
 };
 
-grpc_lb_subchannel_list* grpc_lb_subchannel_list_create(
-    grpc_core::LoadBalancingPolicy* p, grpc_core::TraceFlag* tracer,
+*/
+// All methods with a Locked() suffix must be called from within the
+// client_channel combiner.
+
+namespace grpc_core {
+
+// Stores data for a particular subchannel in a subchannel list.
+// Callers must create a subclass that implements the
+// ProcessConnectivityChangeLocked() method.
+template <typename SubchannelListType, typename SubchannelDataType>
+class SubchannelData {
+ public:
+  // Returns a pointer to the subchannel list containing this object.
+  SubchannelListType* subchannel_list() const { return subchannel_list_; }
+
+  // Returns the index into the subchannel list of this object.
+  size_t Index() const {
+    return static_cast<size_t>(static_cast<const SubchannelDataType*>(this) -
+                               subchannel_list_->subchannel(0));
+  }
+
+  // Returns a pointer to the subchannel.
+  grpc_subchannel* subchannel() const { return subchannel_; }
+
+  // Returns the connected subchannel.  Will be null if the subchannel
+  // is not connected.
+  ConnectedSubchannel* connected_subchannel() const {
+    return connected_subchannel_.get();
+  }
+
+  // Synchronously checks the subchannel's connectivity state.
+  // Must not be called while there is a connectivity notification
+  // pending (i.e., between calling StartConnectivityWatchLocked() or
+  // RenewConnectivityWatchLocked() and the resulting invocation of
+  // ProcessConnectivityChangeLocked()).
+  grpc_connectivity_state CheckConnectivityStateLocked(grpc_error** error) {
+    GPR_ASSERT(!connectivity_notification_pending_);
+    pending_connectivity_state_unsafe_ =
+        grpc_subchannel_check_connectivity(subchannel(), error);
+    UpdateConnectedSubchannelLocked();
+    return pending_connectivity_state_unsafe_;
+  }
+
+  // Unrefs the subchannel.  May be used if an individual subchannel is
+  // no longer needed even though the subchannel list as a whole is not
+  // being unreffed.
+  virtual void UnrefSubchannelLocked(const char* reason);
+
+  // Starts watching the connectivity state of the subchannel.
+  // ProcessConnectivityChangeLocked() will be called when the
+  // connectivity state changes.
+  void StartConnectivityWatchLocked();
+
+  // Renews watching the connectivity state of the subchannel.
+  void RenewConnectivityWatchLocked();
+
+  // Stops watching the connectivity state of the subchannel.
+  void StopConnectivityWatchLocked();
+
+  // Cancels watching the connectivity state of the subchannel.
+  // Must be called only while there is a connectivity notification
+  // pending (i.e., between calling StartConnectivityWatchLocked() or
+  // RenewConnectivityWatchLocked() and the resulting invocation of
+  // ProcessConnectivityChangeLocked()).
+  // From within ProcessConnectivityChangeLocked(), use
+  // StopConnectivityWatchLocked() instead.
+  void CancelConnectivityWatchLocked(const char* reason);
+
+  // Cancels any pending connectivity watch and unrefs the subchannel.
+  void ShutdownLocked();
+
+  GRPC_ABSTRACT_BASE_CLASS
+
+ protected:
+  SubchannelData(SubchannelListType* subchannel_list,
+                 const grpc_lb_user_data_vtable* user_data_vtable,
+                 const grpc_lb_address& address, grpc_subchannel* subchannel,
+                 grpc_combiner* combiner);
+
+  virtual ~SubchannelData();
+
+  // After StartConnectivityWatchLocked() or RenewConnectivityWatchLocked()
+  // is called, this method will be invoked when the subchannel's connectivity
+  // state changes.
+  // Implementations must invoke either RenewConnectivityWatchLocked() or
+  // StopConnectivityWatchLocked() before returning.
+  virtual void ProcessConnectivityChangeLocked(
+      grpc_connectivity_state connectivity_state,
+      grpc_error* error) GRPC_ABSTRACT;
+
+ private:
+  // Updates connected_subchannel_ based on pending_connectivity_state_unsafe_.
+  // Returns true if the connectivity state should be reported.
+  bool UpdateConnectedSubchannelLocked();
+
+  static void OnConnectivityChangedLocked(void* arg, grpc_error* error);
+
+  // Backpointer to owning subchannel list.  Not owned.
+  SubchannelListType* subchannel_list_;
+
+  // The subchannel and connected subchannel.
+  grpc_subchannel* subchannel_;
+  RefCountedPtr<ConnectedSubchannel> connected_subchannel_;
+
+  // Notification that connectivity has changed on subchannel.
+  grpc_closure connectivity_changed_closure_;
+  // Is a connectivity notification pending?
+  bool connectivity_notification_pending_ = false;
+  // Connectivity state to be updated by
+  // grpc_subchannel_notify_on_state_change(), not guarded by
+  // the combiner.
+  grpc_connectivity_state pending_connectivity_state_unsafe_;
+};
+
+// A list of subchannels.
+template <typename SubchannelListType, typename SubchannelDataType>
+class SubchannelList
+    : public InternallyRefCountedWithTracing<SubchannelListType> {
+ public:
+  typedef InlinedVector<SubchannelDataType, 10> SubchannelVector;
+
+  // The number of subchannels in the list.
+  size_t num_subchannels() const { return subchannels_.size(); }
+
+  // The data for the subchannel at a particular index.
+  SubchannelDataType* subchannel(size_t index) { return &subchannels_[index]; }
+
+  // Returns true if the subchannel list is shutting down.
+  bool shutting_down() const { return shutting_down_; }
+
+  // Accessors.
+  LoadBalancingPolicy* policy() const { return policy_; }
+  TraceFlag* tracer() const { return tracer_; }
+
+  // Note: Caller must ensure that this is invoked inside of the combiner.
+  void Orphan() override {
+    ShutdownLocked();
+    InternallyRefCountedWithTracing<SubchannelListType>::Unref(DEBUG_LOCATION,
+                                                               "shutdown");
+  }
+
+  GRPC_ABSTRACT_BASE_CLASS
+
+ protected:
+  SubchannelList(LoadBalancingPolicy* policy, TraceFlag* tracer,
+                 const grpc_lb_addresses* addresses, grpc_combiner* combiner,
+                 grpc_client_channel_factory* client_channel_factory,
+                 const grpc_channel_args& args);
+
+  virtual ~SubchannelList();
+
+ private:
+  // So New() can call our private ctor.
+  template <typename T, typename... Args>
+  friend T* New(Args&&... args);
+
+  // For accessing Ref() and Unref().
+  friend class SubchannelData<SubchannelListType, SubchannelDataType>;
+
+  void ShutdownLocked();
+
+  // Backpointer to owning policy.
+  LoadBalancingPolicy* policy_;
+
+  TraceFlag* tracer_;
+
+  grpc_combiner* combiner_;
+
+  // The list of subchannels.
+  SubchannelVector subchannels_;
+
+  // Is this list shutting down? This may be true due to the shutdown of the
+  // policy itself or because a newer update has arrived while this one hadn't
+  // finished processing.
+  bool shutting_down_ = false;
+};
+
+//
+// implementation -- no user-servicable parts below
+//
+
+//
+// SubchannelData
+//
+
+template <typename SubchannelListType, typename SubchannelDataType>
+SubchannelData<SubchannelListType, SubchannelDataType>::SubchannelData(
+    SubchannelListType* subchannel_list,
+    const grpc_lb_user_data_vtable* user_data_vtable,
+    const grpc_lb_address& address, grpc_subchannel* subchannel,
+    grpc_combiner* combiner)
+    : subchannel_list_(subchannel_list),
+      subchannel_(subchannel),
+      // We assume that the current state is IDLE.  If not, we'll get a
+      // callback telling us that.
+      pending_connectivity_state_unsafe_(GRPC_CHANNEL_IDLE) {
+  GRPC_CLOSURE_INIT(
+      &connectivity_changed_closure_,
+      (&SubchannelData<SubchannelListType,
+                       SubchannelDataType>::OnConnectivityChangedLocked),
+      this, grpc_combiner_scheduler(combiner));
+}
+
+template <typename SubchannelListType, typename SubchannelDataType>
+SubchannelData<SubchannelListType, SubchannelDataType>::~SubchannelData() {
+  UnrefSubchannelLocked("subchannel_data_destroy");
+}
+
+template <typename SubchannelListType, typename SubchannelDataType>
+void SubchannelData<SubchannelListType, SubchannelDataType>::
+    UnrefSubchannelLocked(const char* reason) {
+  if (subchannel_ != nullptr) {
+    if (subchannel_list_->tracer()->enabled()) {
+      gpr_log(GPR_INFO,
+              "[%s %p] subchannel list %p index %" PRIuPTR " of %" PRIuPTR
+              " (subchannel %p): unreffing subchannel",
+              subchannel_list_->tracer()->name(), subchannel_list_->policy(),
+              subchannel_list_, Index(), subchannel_list_->num_subchannels(),
+              subchannel_);
+    }
+    GRPC_SUBCHANNEL_UNREF(subchannel_, reason);
+    subchannel_ = nullptr;
+    connected_subchannel_.reset();
+  }
+}
+
+template <typename SubchannelListType, typename SubchannelDataType>
+void SubchannelData<SubchannelListType,
+                    SubchannelDataType>::StartConnectivityWatchLocked() {
+  if (subchannel_list_->tracer()->enabled()) {
+    gpr_log(GPR_INFO,
+            "[%s %p] subchannel list %p index %" PRIuPTR " of %" PRIuPTR
+            " (subchannel %p): starting watch: requesting connectivity change "
+            "notification (from %s)",
+            subchannel_list_->tracer()->name(), subchannel_list_->policy(),
+            subchannel_list_, Index(), subchannel_list_->num_subchannels(),
+            subchannel_,
+            grpc_connectivity_state_name(pending_connectivity_state_unsafe_));
+  }
+  GPR_ASSERT(!connectivity_notification_pending_);
+  connectivity_notification_pending_ = true;
+  subchannel_list()->Ref(DEBUG_LOCATION, "connectivity_watch").release();
+  grpc_subchannel_notify_on_state_change(
+      subchannel_, subchannel_list_->policy()->interested_parties(),
+      &pending_connectivity_state_unsafe_, &connectivity_changed_closure_);
+}
+
+template <typename SubchannelListType, typename SubchannelDataType>
+void SubchannelData<SubchannelListType,
+                    SubchannelDataType>::RenewConnectivityWatchLocked() {
+  if (subchannel_list_->tracer()->enabled()) {
+    gpr_log(GPR_INFO,
+            "[%s %p] subchannel list %p index %" PRIuPTR " of %" PRIuPTR
+            " (subchannel %p): renewing watch: requesting connectivity change "
+            "notification (from %s)",
+            subchannel_list_->tracer()->name(), subchannel_list_->policy(),
+            subchannel_list_, Index(), subchannel_list_->num_subchannels(),
+            subchannel_,
+            grpc_connectivity_state_name(pending_connectivity_state_unsafe_));
+  }
+  GPR_ASSERT(connectivity_notification_pending_);
+  grpc_subchannel_notify_on_state_change(
+      subchannel_, subchannel_list_->policy()->interested_parties(),
+      &pending_connectivity_state_unsafe_, &connectivity_changed_closure_);
+}
+
+template <typename SubchannelListType, typename SubchannelDataType>
+void SubchannelData<SubchannelListType,
+                    SubchannelDataType>::StopConnectivityWatchLocked() {
+  if (subchannel_list_->tracer()->enabled()) {
+    gpr_log(GPR_INFO,
+            "[%s %p] subchannel list %p index %" PRIuPTR " of %" PRIuPTR
+            " (subchannel %p): stopping connectivity watch",
+            subchannel_list_->tracer()->name(), subchannel_list_->policy(),
+            subchannel_list_, Index(), subchannel_list_->num_subchannels(),
+            subchannel_);
+  }
+  GPR_ASSERT(connectivity_notification_pending_);
+  connectivity_notification_pending_ = false;
+  subchannel_list()->Unref(DEBUG_LOCATION, "connectivity_watch");
+}
+
+template <typename SubchannelListType, typename SubchannelDataType>
+void SubchannelData<SubchannelListType, SubchannelDataType>::
+    CancelConnectivityWatchLocked(const char* reason) {
+  if (subchannel_list_->tracer()->enabled()) {
+    gpr_log(GPR_INFO,
+            "[%s %p] subchannel list %p index %" PRIuPTR " of %" PRIuPTR
+            " (subchannel %p): canceling connectivity watch (%s)",
+            subchannel_list_->tracer()->name(), subchannel_list_->policy(),
+            subchannel_list_, Index(), subchannel_list_->num_subchannels(),
+            subchannel_, reason);
+  }
+  GPR_ASSERT(connectivity_notification_pending_);
+  grpc_subchannel_notify_on_state_change(subchannel_, nullptr, nullptr,
+                                         &connectivity_changed_closure_);
+}
+
+template <typename SubchannelListType, typename SubchannelDataType>
+bool SubchannelData<SubchannelListType,
+                    SubchannelDataType>::UpdateConnectedSubchannelLocked() {
+  // If the subchannel is READY, take a ref to the connected subchannel.
+  if (pending_connectivity_state_unsafe_ == GRPC_CHANNEL_READY) {
+    connected_subchannel_ =
+        grpc_subchannel_get_connected_subchannel(subchannel_);
+    // If the subchannel became disconnected between the time that READY
+    // was reported and the time we got here (e.g., between when a
+    // notification callback is scheduled and when it was actually run in
+    // the combiner), then the connected subchannel may have disappeared out
+    // from under us.  In that case, we don't actually want to consider the
+    // subchannel to be in state READY.  Instead, we use IDLE as the
+    // basis for any future connectivity watch; this is the one state that
+    // the subchannel will never transition back into, so this ensures
+    // that we will get a notification for the next state, even if that state
+    // is READY again (e.g., if the subchannel has transitioned back to
+    // READY before the next watch gets requested).
+    if (connected_subchannel_ == nullptr) {
+      if (subchannel_list_->tracer()->enabled()) {
+        gpr_log(GPR_INFO,
+                "[%s %p] subchannel list %p index %" PRIuPTR " of %" PRIuPTR
+                " (subchannel %p): state is READY but connected subchannel is "
+                "null; moving to state IDLE",
+                subchannel_list_->tracer()->name(), subchannel_list_->policy(),
+                subchannel_list_, Index(), subchannel_list_->num_subchannels(),
+                subchannel_);
+      }
+      pending_connectivity_state_unsafe_ = GRPC_CHANNEL_IDLE;
+      return false;
+    }
+  } else {
+    // For any state other than READY, unref the connected subchannel.
+    connected_subchannel_.reset();
+  }
+  return true;
+}
+
+template <typename SubchannelListType, typename SubchannelDataType>
+void SubchannelData<SubchannelListType, SubchannelDataType>::
+    OnConnectivityChangedLocked(void* arg, grpc_error* error) {
+  SubchannelData* sd = static_cast<SubchannelData*>(arg);
+  if (sd->subchannel_list_->tracer()->enabled()) {
+    gpr_log(
+        GPR_INFO,
+        "[%s %p] subchannel list %p index %" PRIuPTR " of %" PRIuPTR
+        " (subchannel %p): connectivity changed: state=%s, error=%s, "
+        "shutting_down=%d",
+        sd->subchannel_list_->tracer()->name(), sd->subchannel_list_->policy(),
+        sd->subchannel_list_, sd->Index(),
+        sd->subchannel_list_->num_subchannels(), sd->subchannel_,
+        grpc_connectivity_state_name(sd->pending_connectivity_state_unsafe_),
+        grpc_error_string(error), sd->subchannel_list_->shutting_down());
+  }
+  // If shutting down, unref subchannel and stop watching.
+  if (sd->subchannel_list_->shutting_down() || error == GRPC_ERROR_CANCELLED) {
+    sd->UnrefSubchannelLocked("connectivity_shutdown");
+    sd->StopConnectivityWatchLocked();
+    return;
+  }
+  // Get or release ref to connected subchannel.
+  if (!sd->UpdateConnectedSubchannelLocked()) {
+    // We don't want to report this connectivity state, so renew the watch.
+    sd->RenewConnectivityWatchLocked();
+    return;
+  }
+  // Call the subclass's ProcessConnectivityChangeLocked() method.
+  sd->ProcessConnectivityChangeLocked(sd->pending_connectivity_state_unsafe_,
+                                      GRPC_ERROR_REF(error));
+}
+
+template <typename SubchannelListType, typename SubchannelDataType>
+void SubchannelData<SubchannelListType, SubchannelDataType>::ShutdownLocked() {
+  // If there's a pending notification for this subchannel, cancel it;
+  // the callback is responsible for unreffing the subchannel.
+  // Otherwise, unref the subchannel directly.
+  if (connectivity_notification_pending_) {
+    CancelConnectivityWatchLocked("shutdown");
+  } else if (subchannel_ != nullptr) {
+    UnrefSubchannelLocked("shutdown");
+  }
+}
+
+//
+// SubchannelList
+//
+
+template <typename SubchannelListType, typename SubchannelDataType>
+SubchannelList<SubchannelListType, SubchannelDataType>::SubchannelList(
+    LoadBalancingPolicy* policy, TraceFlag* tracer,
     const grpc_lb_addresses* addresses, grpc_combiner* combiner,
     grpc_client_channel_factory* client_channel_factory,
-    const grpc_channel_args& args, grpc_iomgr_cb_func connectivity_changed_cb);
+    const grpc_channel_args& args)
+    : InternallyRefCountedWithTracing<SubchannelListType>(tracer),
+      policy_(policy),
+      tracer_(tracer),
+      combiner_(GRPC_COMBINER_REF(combiner, "subchannel_list")) {
+  if (tracer_->enabled()) {
+    gpr_log(GPR_INFO,
+            "[%s %p] Creating subchannel list %p for %" PRIuPTR " subchannels",
+            tracer_->name(), policy, this, addresses->num_addresses);
+  }
+  subchannels_.reserve(addresses->num_addresses);
+  // We need to remove the LB addresses in order to be able to compare the
+  // subchannel keys of subchannels from a different batch of addresses.
+  static const char* keys_to_remove[] = {GRPC_ARG_SUBCHANNEL_ADDRESS,
+                                         GRPC_ARG_LB_ADDRESSES};
+  // Create a subchannel for each address.
+  grpc_subchannel_args sc_args;
+  for (size_t i = 0; i < addresses->num_addresses; i++) {
+    // If there were any balancer, we would have chosen grpclb policy instead.
+    GPR_ASSERT(!addresses->addresses[i].is_balancer);
+    memset(&sc_args, 0, sizeof(grpc_subchannel_args));
+    grpc_arg addr_arg =
+        grpc_create_subchannel_address_arg(&addresses->addresses[i].address);
+    grpc_channel_args* new_args = grpc_channel_args_copy_and_add_and_remove(
+        &args, keys_to_remove, GPR_ARRAY_SIZE(keys_to_remove), &addr_arg, 1);
+    gpr_free(addr_arg.value.string);
+    sc_args.args = new_args;
+    grpc_subchannel* subchannel = grpc_client_channel_factory_create_subchannel(
+        client_channel_factory, &sc_args);
+    grpc_channel_args_destroy(new_args);
+    if (subchannel == nullptr) {
+      // Subchannel could not be created.
+      if (tracer_->enabled()) {
+        char* address_uri =
+            grpc_sockaddr_to_uri(&addresses->addresses[i].address);
+        gpr_log(GPR_INFO,
+                "[%s %p] could not create subchannel for address uri %s, "
+                "ignoring",
+                tracer_->name(), policy_, address_uri);
+        gpr_free(address_uri);
+      }
+      continue;
+    }
+    if (tracer_->enabled()) {
+      char* address_uri =
+          grpc_sockaddr_to_uri(&addresses->addresses[i].address);
+      gpr_log(GPR_INFO,
+              "[%s %p] subchannel list %p index %" PRIuPTR
+              ": Created subchannel %p for address uri %s",
+              tracer_->name(), policy_, this, subchannels_.size(), subchannel,
+              address_uri);
+      gpr_free(address_uri);
+    }
+    subchannels_.emplace_back(static_cast<SubchannelListType*>(this),
+                              addresses->user_data_vtable,
+                              addresses->addresses[i], subchannel, combiner);
+  }
+}
 
-void grpc_lb_subchannel_list_ref(grpc_lb_subchannel_list* subchannel_list,
-                                 const char* reason);
+template <typename SubchannelListType, typename SubchannelDataType>
+SubchannelList<SubchannelListType, SubchannelDataType>::~SubchannelList() {
+  if (tracer_->enabled()) {
+    gpr_log(GPR_INFO, "[%s %p] Destroying subchannel_list %p", tracer_->name(),
+            policy_, this);
+  }
+  GRPC_COMBINER_UNREF(combiner_, "subchannel_list");
+}
 
-void grpc_lb_subchannel_list_unref(grpc_lb_subchannel_list* subchannel_list,
-                                   const char* reason);
+template <typename SubchannelListType, typename SubchannelDataType>
+void SubchannelList<SubchannelListType, SubchannelDataType>::ShutdownLocked() {
+  if (tracer_->enabled()) {
+    gpr_log(GPR_INFO, "[%s %p] Shutting down subchannel_list %p",
+            tracer_->name(), policy_, this);
+  }
+  GPR_ASSERT(!shutting_down_);
+  shutting_down_ = true;
+  for (size_t i = 0; i < subchannels_.size(); i++) {
+    SubchannelDataType* sd = &subchannels_[i];
+    sd->ShutdownLocked();
+  }
+}
 
-/// Mark subchannel_list as discarded. Unsubscribes all its subchannels. The
-/// connectivity state notification callback will ultimately unref it.
-void grpc_lb_subchannel_list_shutdown_and_unref(
-    grpc_lb_subchannel_list* subchannel_list, const char* reason);
+}  // namespace grpc_core
 
 #endif /* GRPC_CORE_EXT_FILTERS_CLIENT_CHANNEL_LB_POLICY_SUBCHANNEL_LIST_H */

src/core/ext/filters/client_channel/method_params.h

@@ -60,6 +60,10 @@ class ClientChannelMethodParams : public RefCounted<ClientChannelMethodParams> {
   template <typename T, typename... Args>
   friend T* grpc_core::New(Args&&... args);
 
+  // So Delete() can call our private dtor.
+  template <typename T>
+  friend void grpc_core::Delete(T*);
+
   ClientChannelMethodParams() {}
   virtual ~ClientChannelMethodParams() {}
 

src/core/ext/filters/client_channel/resolver.h

@@ -105,6 +105,8 @@ class Resolver : public InternallyRefCountedWithTracing<Resolver> {
   GRPC_ABSTRACT_BASE_CLASS
 
  protected:
+  GPRC_ALLOW_CLASS_TO_USE_NON_PUBLIC_DELETE
+
   /// Does NOT take ownership of the reference to \a combiner.
   // TODO(roth): Once we have a C++-like interface for combiners, this
   // API should change to take a RefCountedPtr<>, so that we always take

src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc

@@ -346,7 +346,7 @@ void AresDnsResolver::OnResolvedLocked(void* arg, grpc_error* error) {
     RefCountedPtr<Resolver> self = r->Ref(DEBUG_LOCATION, "retry-timer");
     self.release();
     if (timeout > 0) {
-      gpr_log(GPR_DEBUG, "retrying in %" PRIdPTR " milliseconds", timeout);
+      gpr_log(GPR_DEBUG, "retrying in %" PRId64 " milliseconds", timeout);
     } else {
       gpr_log(GPR_DEBUG, "retrying immediately");
     }
@@ -363,6 +363,15 @@ void AresDnsResolver::OnResolvedLocked(void* arg, grpc_error* error) {
 }
 
 void AresDnsResolver::MaybeStartResolvingLocked() {
+  // If there is an existing timer, the time it fires is the earliest time we
+  // can start the next resolution.
+  if (have_next_resolution_timer_) {
+    // TODO(dgq): remove the following two lines once Pick First stops
+    // discarding subchannels after selecting.
+    ++resolved_version_;
+    MaybeFinishNextLocked();
+    return;
+  }
   if (last_resolution_timestamp_ >= 0) {
     const grpc_millis earliest_next_resolution =
         last_resolution_timestamp_ + min_time_between_resolutions_;
@@ -372,20 +381,18 @@ void AresDnsResolver::MaybeStartResolvingLocked() {
       const grpc_millis last_resolution_ago =
           grpc_core::ExecCtx::Get()->Now() - last_resolution_timestamp_;
       gpr_log(GPR_DEBUG,
-              "In cooldown from last resolution (from %" PRIdPTR
-              " ms ago). Will resolve again in %" PRIdPTR " ms",
+              "In cooldown from last resolution (from %" PRId64
+              " ms ago). Will resolve again in %" PRId64 " ms",
               last_resolution_ago, ms_until_next_resolution);
-      if (!have_next_resolution_timer_) {
-        have_next_resolution_timer_ = true;
-        // TODO(roth): We currently deal with this ref manually.  Once the
-        // new closure API is done, find a way to track this ref with the timer
-        // callback as part of the type system.
-        RefCountedPtr<Resolver> self =
-            Ref(DEBUG_LOCATION, "next_resolution_timer_cooldown");
-        self.release();
-        grpc_timer_init(&next_resolution_timer_, ms_until_next_resolution,
-                        &on_next_resolution_);
-      }
+      have_next_resolution_timer_ = true;
+      // TODO(roth): We currently deal with this ref manually.  Once the
+      // new closure API is done, find a way to track this ref with the timer
+      // callback as part of the type system.
+      RefCountedPtr<Resolver> self =
+          Ref(DEBUG_LOCATION, "next_resolution_timer_cooldown");
+      self.release();
+      grpc_timer_init(&next_resolution_timer_, ms_until_next_resolution,
+                      &on_next_resolution_);
       // TODO(dgq): remove the following two lines once Pick First stops
       // discarding subchannels after selecting.
       ++resolved_version_;
@@ -397,6 +404,7 @@ void AresDnsResolver::MaybeStartResolvingLocked() {
 }
 
 void AresDnsResolver::StartResolvingLocked() {
+  gpr_log(GPR_DEBUG, "Start resolving.");
   // TODO(roth): We currently deal with this ref manually.  Once the
   // new closure API is done, find a way to track this ref with the timer
   // callback as part of the type system.
@@ -406,10 +414,10 @@ void AresDnsResolver::StartResolvingLocked() {
   resolving_ = true;
   lb_addresses_ = nullptr;
   service_config_json_ = nullptr;
-  pending_request_ = grpc_dns_lookup_ares(
+  pending_request_ = grpc_dns_lookup_ares_locked(
       dns_server_, name_to_resolve_, kDefaultPort, interested_parties_,
       &on_resolved_, &lb_addresses_, true /* check_grpclb */,
-      request_service_config_ ? &service_config_json_ : nullptr);
+      request_service_config_ ? &service_config_json_ : nullptr, combiner());
   last_resolution_timestamp_ = grpc_core::ExecCtx::Get()->Now();
 }
 

src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.cc

@@ -0,0 +1,311 @@
+/*
+ *
+ * Copyright 2016 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#include <grpc/support/port_platform.h>
+
+#include "src/core/lib/iomgr/port.h"
+#if GRPC_ARES == 1 && defined(GRPC_POSIX_SOCKET_ARES_EV_DRIVER)
+
+#include <ares.h>
+#include <string.h>
+#include <sys/ioctl.h>
+
+#include "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.h"
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+#include <grpc/support/string_util.h>
+#include <grpc/support/time.h>
+#include "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h"
+#include "src/core/lib/gpr/string.h"
+#include "src/core/lib/iomgr/ev_posix.h"
+#include "src/core/lib/iomgr/iomgr_internal.h"
+#include "src/core/lib/iomgr/sockaddr_utils.h"
+
+typedef struct fd_node {
+  /** the owner of this fd node */
+  grpc_ares_ev_driver* ev_driver;
+  /** a closure wrapping on_readable_locked, which should be
+     invoked when the grpc_fd in this node becomes readable. */
+  grpc_closure read_closure;
+  /** a closure wrapping on_writable_locked, which should be
+     invoked when the grpc_fd in this node becomes writable. */
+  grpc_closure write_closure;
+  /** next fd node in the list */
+  struct fd_node* next;
+
+  /** wrapped fd that's polled by grpc's poller for the current platform */
+  grpc_core::GrpcPolledFd* grpc_polled_fd;
+  /** if the readable closure has been registered */
+  bool readable_registered;
+  /** if the writable closure has been registered */
+  bool writable_registered;
+  /** if the fd has been shutdown yet from grpc iomgr perspective */
+  bool already_shutdown;
+} fd_node;
+
+struct grpc_ares_ev_driver {
+  /** the ares_channel owned by this event driver */
+  ares_channel channel;
+  /** pollset set for driving the IO events of the channel */
+  grpc_pollset_set* pollset_set;
+  /** refcount of the event driver */
+  gpr_refcount refs;
+
+  /** combiner to synchronize c-ares and I/O callbacks on */
+  grpc_combiner* combiner;
+  /** a list of grpc_fd that this event driver is currently using. */
+  fd_node* fds;
+  /** is this event driver currently working? */
+  bool working;
+  /** is this event driver being shut down */
+  bool shutting_down;
+};
+
+static void grpc_ares_notify_on_event_locked(grpc_ares_ev_driver* ev_driver);
+
+static grpc_ares_ev_driver* grpc_ares_ev_driver_ref(
+    grpc_ares_ev_driver* ev_driver) {
+  gpr_log(GPR_DEBUG, "Ref ev_driver %" PRIuPTR, (uintptr_t)ev_driver);
+  gpr_ref(&ev_driver->refs);
+  return ev_driver;
+}
+
+static void grpc_ares_ev_driver_unref(grpc_ares_ev_driver* ev_driver) {
+  gpr_log(GPR_DEBUG, "Unref ev_driver %" PRIuPTR, (uintptr_t)ev_driver);
+  if (gpr_unref(&ev_driver->refs)) {
+    gpr_log(GPR_DEBUG, "destroy ev_driver %" PRIuPTR, (uintptr_t)ev_driver);
+    GPR_ASSERT(ev_driver->fds == nullptr);
+    GRPC_COMBINER_UNREF(ev_driver->combiner, "free ares event driver");
+    ares_destroy(ev_driver->channel);
+    gpr_free(ev_driver);
+  }
+}
+
+static void fd_node_destroy_locked(fd_node* fdn) {
+  gpr_log(GPR_DEBUG, "delete fd: %s", fdn->grpc_polled_fd->GetName());
+  GPR_ASSERT(!fdn->readable_registered);
+  GPR_ASSERT(!fdn->writable_registered);
+  GPR_ASSERT(fdn->already_shutdown);
+  grpc_core::Delete(fdn->grpc_polled_fd);
+  gpr_free(fdn);
+}
+
+static void fd_node_shutdown_locked(fd_node* fdn, const char* reason) {
+  if (!fdn->already_shutdown) {
+    fdn->already_shutdown = true;
+    fdn->grpc_polled_fd->ShutdownLocked(
+        GRPC_ERROR_CREATE_FROM_STATIC_STRING(reason));
+  }
+}
+
+grpc_error* grpc_ares_ev_driver_create_locked(grpc_ares_ev_driver** ev_driver,
+                                              grpc_pollset_set* pollset_set,
+                                              grpc_combiner* combiner) {
+  *ev_driver = static_cast<grpc_ares_ev_driver*>(
+      gpr_malloc(sizeof(grpc_ares_ev_driver)));
+  ares_options opts;
+  memset(&opts, 0, sizeof(opts));
+  opts.flags |= ARES_FLAG_STAYOPEN;
+  int status = ares_init_options(&(*ev_driver)->channel, &opts, ARES_OPT_FLAGS);
+  grpc_core::ConfigureAresChannelLocked(&(*ev_driver)->channel);
+  gpr_log(GPR_DEBUG, "grpc_ares_ev_driver_create_locked");
+  if (status != ARES_SUCCESS) {
+    char* err_msg;
+    gpr_asprintf(&err_msg, "Failed to init ares channel. C-ares error: %s",
+                 ares_strerror(status));
+    grpc_error* err = GRPC_ERROR_CREATE_FROM_COPIED_STRING(err_msg);
+    gpr_free(err_msg);
+    gpr_free(*ev_driver);
+    return err;
+  }
+  (*ev_driver)->combiner = GRPC_COMBINER_REF(combiner, "ares event driver");
+  gpr_ref_init(&(*ev_driver)->refs, 1);
+  (*ev_driver)->pollset_set = pollset_set;
+  (*ev_driver)->fds = nullptr;
+  (*ev_driver)->working = false;
+  (*ev_driver)->shutting_down = false;
+  return GRPC_ERROR_NONE;
+}
+
+void grpc_ares_ev_driver_destroy_locked(grpc_ares_ev_driver* ev_driver) {
+  // We mark the event driver as being shut down. If the event driver
+  // is working, grpc_ares_notify_on_event_locked will shut down the
+  // fds; if it's not working, there are no fds to shut down.
+  ev_driver->shutting_down = true;
+  grpc_ares_ev_driver_unref(ev_driver);
+}
+
+void grpc_ares_ev_driver_shutdown_locked(grpc_ares_ev_driver* ev_driver) {
+  ev_driver->shutting_down = true;
+  fd_node* fn = ev_driver->fds;
+  while (fn != nullptr) {
+    fd_node_shutdown_locked(fn, "grpc_ares_ev_driver_shutdown");
+    fn = fn->next;
+  }
+}
+
+// Search fd in the fd_node list head. This is an O(n) search, the max possible
+// value of n is ARES_GETSOCK_MAXNUM (16). n is typically 1 - 2 in our tests.
+static fd_node* pop_fd_node_locked(fd_node** head, ares_socket_t as) {
+  fd_node dummy_head;
+  dummy_head.next = *head;
+  fd_node* node = &dummy_head;
+  while (node->next != nullptr) {
+    if (node->next->grpc_polled_fd->GetWrappedAresSocketLocked() == as) {
+      fd_node* ret = node->next;
+      node->next = node->next->next;
+      *head = dummy_head.next;
+      return ret;
+    }
+    node = node->next;
+  }
+  return nullptr;
+}
+
+static void on_readable_locked(void* arg, grpc_error* error) {
+  fd_node* fdn = static_cast<fd_node*>(arg);
+  grpc_ares_ev_driver* ev_driver = fdn->ev_driver;
+  const ares_socket_t as = fdn->grpc_polled_fd->GetWrappedAresSocketLocked();
+  fdn->readable_registered = false;
+  gpr_log(GPR_DEBUG, "readable on %s", fdn->grpc_polled_fd->GetName());
+  if (error == GRPC_ERROR_NONE) {
+    do {
+      ares_process_fd(ev_driver->channel, as, ARES_SOCKET_BAD);
+    } while (fdn->grpc_polled_fd->IsFdStillReadableLocked());
+  } else {
+    // If error is not GRPC_ERROR_NONE, it means the fd has been shutdown or
+    // timed out. The pending lookups made on this ev_driver will be cancelled
+    // by the following ares_cancel() and the on_done callbacks will be invoked
+    // with a status of ARES_ECANCELLED. The remaining file descriptors in this
+    // ev_driver will be cleaned up in the follwing
+    // grpc_ares_notify_on_event_locked().
+    ares_cancel(ev_driver->channel);
+  }
+  grpc_ares_notify_on_event_locked(ev_driver);
+  grpc_ares_ev_driver_unref(ev_driver);
+}
+
+static void on_writable_locked(void* arg, grpc_error* error) {
+  fd_node* fdn = static_cast<fd_node*>(arg);
+  grpc_ares_ev_driver* ev_driver = fdn->ev_driver;
+  const ares_socket_t as = fdn->grpc_polled_fd->GetWrappedAresSocketLocked();
+  fdn->writable_registered = false;
+  gpr_log(GPR_DEBUG, "writable on %s", fdn->grpc_polled_fd->GetName());
+  if (error == GRPC_ERROR_NONE) {
+    ares_process_fd(ev_driver->channel, ARES_SOCKET_BAD, as);
+  } else {
+    // If error is not GRPC_ERROR_NONE, it means the fd has been shutdown or
+    // timed out. The pending lookups made on this ev_driver will be cancelled
+    // by the following ares_cancel() and the on_done callbacks will be invoked
+    // with a status of ARES_ECANCELLED. The remaining file descriptors in this
+    // ev_driver will be cleaned up in the follwing
+    // grpc_ares_notify_on_event_locked().
+    ares_cancel(ev_driver->channel);
+  }
+  grpc_ares_notify_on_event_locked(ev_driver);
+  grpc_ares_ev_driver_unref(ev_driver);
+}
+
+ares_channel* grpc_ares_ev_driver_get_channel_locked(
+    grpc_ares_ev_driver* ev_driver) {
+  return &ev_driver->channel;
+}
+
+// Get the file descriptors used by the ev_driver's ares channel, register
+// driver_closure with these filedescriptors.
+static void grpc_ares_notify_on_event_locked(grpc_ares_ev_driver* ev_driver) {
+  fd_node* new_list = nullptr;
+  if (!ev_driver->shutting_down) {
+    ares_socket_t socks[ARES_GETSOCK_MAXNUM];
+    int socks_bitmask =
+        ares_getsock(ev_driver->channel, socks, ARES_GETSOCK_MAXNUM);
+    for (size_t i = 0; i < ARES_GETSOCK_MAXNUM; i++) {
+      if (ARES_GETSOCK_READABLE(socks_bitmask, i) ||
+          ARES_GETSOCK_WRITABLE(socks_bitmask, i)) {
+        fd_node* fdn = pop_fd_node_locked(&ev_driver->fds, socks[i]);
+        // Create a new fd_node if sock[i] is not in the fd_node list.
+        if (fdn == nullptr) {
+          fdn = static_cast<fd_node*>(gpr_malloc(sizeof(fd_node)));
+          fdn->grpc_polled_fd = grpc_core::NewGrpcPolledFdLocked(
+              socks[i], ev_driver->pollset_set);
+          gpr_log(GPR_DEBUG, "new fd: %s", fdn->grpc_polled_fd->GetName());
+          fdn->ev_driver = ev_driver;
+          fdn->readable_registered = false;
+          fdn->writable_registered = false;
+          fdn->already_shutdown = false;
+          GRPC_CLOSURE_INIT(&fdn->read_closure, on_readable_locked, fdn,
+                            grpc_combiner_scheduler(ev_driver->combiner));
+          GRPC_CLOSURE_INIT(&fdn->write_closure, on_writable_locked, fdn,
+                            grpc_combiner_scheduler(ev_driver->combiner));
+        }
+        fdn->next = new_list;
+        new_list = fdn;
+        // Register read_closure if the socket is readable and read_closure has
+        // not been registered with this socket.
+        if (ARES_GETSOCK_READABLE(socks_bitmask, i) &&
+            !fdn->readable_registered) {
+          grpc_ares_ev_driver_ref(ev_driver);
+          gpr_log(GPR_DEBUG, "notify read on: %s",
+                  fdn->grpc_polled_fd->GetName());
+          fdn->grpc_polled_fd->RegisterForOnReadableLocked(&fdn->read_closure);
+          fdn->readable_registered = true;
+        }
+        // Register write_closure if the socket is writable and write_closure
+        // has not been registered with this socket.
+        if (ARES_GETSOCK_WRITABLE(socks_bitmask, i) &&
+            !fdn->writable_registered) {
+          gpr_log(GPR_DEBUG, "notify write on: %s",
+                  fdn->grpc_polled_fd->GetName());
+          grpc_ares_ev_driver_ref(ev_driver);
+          fdn->grpc_polled_fd->RegisterForOnWriteableLocked(
+              &fdn->write_closure);
+          fdn->writable_registered = true;
+        }
+      }
+    }
+  }
+  // Any remaining fds in ev_driver->fds were not returned by ares_getsock() and
+  // are therefore no longer in use, so they can be shut down and removed from
+  // the list.
+  while (ev_driver->fds != nullptr) {
+    fd_node* cur = ev_driver->fds;
+    ev_driver->fds = ev_driver->fds->next;
+    fd_node_shutdown_locked(cur, "c-ares fd shutdown");
+    if (!cur->readable_registered && !cur->writable_registered) {
+      fd_node_destroy_locked(cur);
+    } else {
+      cur->next = new_list;
+      new_list = cur;
+    }
+  }
+  ev_driver->fds = new_list;
+  // If the ev driver has no working fd, all the tasks are done.
+  if (new_list == nullptr) {
+    ev_driver->working = false;
+    gpr_log(GPR_DEBUG, "ev driver stop working");
+  }
+}
+
+void grpc_ares_ev_driver_start_locked(grpc_ares_ev_driver* ev_driver) {
+  if (!ev_driver->working) {
+    ev_driver->working = true;
+    grpc_ares_notify_on_event_locked(ev_driver);
+  }
+}
+
+#endif /* GRPC_ARES == 1 && defined(GRPC_POSIX_SOCKET_ARES_EV_DRIVER) */

src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.h

@@ -22,6 +22,7 @@
 #include <grpc/support/port_platform.h>
 
 #include <ares.h>
+#include "src/core/lib/gprpp/abstract.h"
 #include "src/core/lib/iomgr/pollset_set.h"
 
 typedef struct grpc_ares_ev_driver grpc_ares_ev_driver;
@@ -29,25 +30,62 @@ typedef struct grpc_ares_ev_driver grpc_ares_ev_driver;
 /* Start \a ev_driver. It will keep working until all IO on its ares_channel is
    done, or grpc_ares_ev_driver_destroy() is called. It may notify the callbacks
    bound to its ares_channel when necessary. */
-void grpc_ares_ev_driver_start(grpc_ares_ev_driver* ev_driver);
+void grpc_ares_ev_driver_start_locked(grpc_ares_ev_driver* ev_driver);
 
 /* Returns the ares_channel owned by \a ev_driver. To bind a c-ares query to
    \a ev_driver, use the ares_channel owned by \a ev_driver as the arg of the
    query. */
-ares_channel* grpc_ares_ev_driver_get_channel(grpc_ares_ev_driver* ev_driver);
+ares_channel* grpc_ares_ev_driver_get_channel_locked(
+    grpc_ares_ev_driver* ev_driver);
 
 /* Creates a new grpc_ares_ev_driver. Returns GRPC_ERROR_NONE if \a ev_driver is
    created successfully. */
-grpc_error* grpc_ares_ev_driver_create(grpc_ares_ev_driver** ev_driver,
-                                       grpc_pollset_set* pollset_set);
+grpc_error* grpc_ares_ev_driver_create_locked(grpc_ares_ev_driver** ev_driver,
+                                              grpc_pollset_set* pollset_set,
+                                              grpc_combiner* combiner);
 
 /* Destroys \a ev_driver asynchronously. Pending lookups made on \a ev_driver
    will be cancelled and their on_done callbacks will be invoked with a status
    of ARES_ECANCELLED. */
-void grpc_ares_ev_driver_destroy(grpc_ares_ev_driver* ev_driver);
+void grpc_ares_ev_driver_destroy_locked(grpc_ares_ev_driver* ev_driver);
 
 /* Shutdown all the grpc_fds used by \a ev_driver */
-void grpc_ares_ev_driver_shutdown(grpc_ares_ev_driver* ev_driver);
+void grpc_ares_ev_driver_shutdown_locked(grpc_ares_ev_driver* ev_driver);
+
+namespace grpc_core {
+
+/* A wrapped fd that integrates with the grpc iomgr of the current platform.
+ * A GrpcPolledFd knows how to create grpc platform-specific iomgr endpoints
+ * from "ares_socket_t" sockets, and then sign up for readability/writeability
+ * with that poller, and do shutdown and destruction. */
+class GrpcPolledFd {
+ public:
+  virtual ~GrpcPolledFd() {}
+  /* Called when c-ares library is interested and there's no pending callback */
+  virtual void RegisterForOnReadableLocked(grpc_closure* read_closure)
+      GRPC_ABSTRACT;
+  /* Called when c-ares library is interested and there's no pending callback */
+  virtual void RegisterForOnWriteableLocked(grpc_closure* write_closure)
+      GRPC_ABSTRACT;
+  /* Indicates if there is data left even after just being read from */
+  virtual bool IsFdStillReadableLocked() GRPC_ABSTRACT;
+  /* Called once and only once. Must cause cancellation of any pending
+   * read/write callbacks. */
+  virtual void ShutdownLocked(grpc_error* error) GRPC_ABSTRACT;
+  /* Get the underlying ares_socket_t that this was created from */
+  virtual ares_socket_t GetWrappedAresSocketLocked() GRPC_ABSTRACT;
+  /* A unique name, for logging */
+  virtual const char* GetName() GRPC_ABSTRACT;
+
+  GRPC_ABSTRACT_BASE_CLASS
+};
+
+/* Creates a new wrapped fd for the current platform */
+GrpcPolledFd* NewGrpcPolledFdLocked(ares_socket_t as,
+                                    grpc_pollset_set* driver_pollset_set);
+void ConfigureAresChannelLocked(ares_channel* channel);
+
+}  // namespace grpc_core
 
 #endif /* GRPC_CORE_EXT_FILTERS_CLIENT_CHANNEL_RESOLVER_DNS_C_ARES_GRPC_ARES_EV_DRIVER_H \
         */

src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc

@@ -18,9 +18,10 @@
 #include <grpc/support/port_platform.h>
 
 #include "src/core/lib/iomgr/port.h"
-#if GRPC_ARES == 1 && defined(GRPC_POSIX_SOCKET)
+#if GRPC_ARES == 1 && defined(GRPC_POSIX_SOCKET_ARES_EV_DRIVER)
 
 #include <ares.h>
+#include <string.h>
 #include <sys/ioctl.h>
 
 #include "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.h"
@@ -35,317 +36,60 @@
 #include "src/core/lib/iomgr/iomgr_internal.h"
 #include "src/core/lib/iomgr/sockaddr_utils.h"
 
-typedef struct fd_node {
-  /** the owner of this fd node */
-  grpc_ares_ev_driver* ev_driver;
-  /** a closure wrapping on_readable_cb, which should be invoked when the
-      grpc_fd in this node becomes readable. */
-  grpc_closure read_closure;
-  /** a closure wrapping on_writable_cb, which should be invoked when the
-      grpc_fd in this node becomes writable. */
-  grpc_closure write_closure;
-  /** next fd node in the list */
-  struct fd_node* next;
+namespace grpc_core {
 
-  /** mutex guarding the rest of the state */
-  gpr_mu mu;
-  /** the grpc_fd owned by this fd node */
-  grpc_fd* fd;
-  /** if the readable closure has been registered */
-  bool readable_registered;
-  /** if the writable closure has been registered */
-  bool writable_registered;
-  /** if the fd is being shut down */
-  bool shutting_down;
-} fd_node;
-
-struct grpc_ares_ev_driver {
-  /** the ares_channel owned by this event driver */
-  ares_channel channel;
-  /** pollset set for driving the IO events of the channel */
-  grpc_pollset_set* pollset_set;
-  /** refcount of the event driver */
-  gpr_refcount refs;
-
-  /** mutex guarding the rest of the state */
-  gpr_mu mu;
-  /** a list of grpc_fd that this event driver is currently using. */
-  fd_node* fds;
-  /** is this event driver currently working? */
-  bool working;
-  /** is this event driver being shut down */
-  bool shutting_down;
-};
-
-static void grpc_ares_notify_on_event_locked(grpc_ares_ev_driver* ev_driver);
-
-static grpc_ares_ev_driver* grpc_ares_ev_driver_ref(
-    grpc_ares_ev_driver* ev_driver) {
-  gpr_log(GPR_DEBUG, "Ref ev_driver %" PRIuPTR, (uintptr_t)ev_driver);
-  gpr_ref(&ev_driver->refs);
-  return ev_driver;
-}
-
-static void grpc_ares_ev_driver_unref(grpc_ares_ev_driver* ev_driver) {
-  gpr_log(GPR_DEBUG, "Unref ev_driver %" PRIuPTR, (uintptr_t)ev_driver);
-  if (gpr_unref(&ev_driver->refs)) {
-    gpr_log(GPR_DEBUG, "destroy ev_driver %" PRIuPTR, (uintptr_t)ev_driver);
-    GPR_ASSERT(ev_driver->fds == nullptr);
-    gpr_mu_destroy(&ev_driver->mu);
-    ares_destroy(ev_driver->channel);
-    gpr_free(ev_driver);
+class GrpcPolledFdPosix : public GrpcPolledFd {
+ public:
+  GrpcPolledFdPosix(ares_socket_t as, grpc_pollset_set* driver_pollset_set)
+      : as_(as) {
+    gpr_asprintf(&name_, "c-ares fd: %d", (int)as);
+    fd_ = grpc_fd_create((int)as, name_, false);
+    grpc_pollset_set_add_fd(driver_pollset_set, fd_);
   }
-}
 
-static void fd_node_destroy(fd_node* fdn) {
-  gpr_log(GPR_DEBUG, "delete fd: %d", grpc_fd_wrapped_fd(fdn->fd));
-  GPR_ASSERT(!fdn->readable_registered);
-  GPR_ASSERT(!fdn->writable_registered);
-  gpr_mu_destroy(&fdn->mu);
-  /* c-ares library has closed the fd inside grpc_fd. This fd may be picked up
-     immediately by another thread, and should not be closed by the following
-     grpc_fd_orphan. */
-  grpc_fd_orphan(fdn->fd, nullptr, nullptr, true /* already_closed */,
-                 "c-ares query finished");
-  gpr_free(fdn);
-}
-
-static void fd_node_shutdown(fd_node* fdn) {
-  gpr_mu_lock(&fdn->mu);
-  fdn->shutting_down = true;
-  if (!fdn->readable_registered && !fdn->writable_registered) {
-    gpr_mu_unlock(&fdn->mu);
-    fd_node_destroy(fdn);
-  } else {
-    grpc_fd_shutdown(
-        fdn->fd, GRPC_ERROR_CREATE_FROM_STATIC_STRING("c-ares fd shutdown"));
-    gpr_mu_unlock(&fdn->mu);
+  ~GrpcPolledFdPosix() {
+    gpr_free(name_);
+    /* c-ares library will close the fd inside grpc_fd. This fd may be picked up
+       immediately by another thread, and should not be closed by the following
+       grpc_fd_orphan. */
+    int dummy_release_fd;
+    grpc_fd_orphan(fd_, nullptr, &dummy_release_fd, "c-ares query finished");
   }
-}
 
-grpc_error* grpc_ares_ev_driver_create(grpc_ares_ev_driver** ev_driver,
-                                       grpc_pollset_set* pollset_set) {
-  *ev_driver = static_cast<grpc_ares_ev_driver*>(
-      gpr_malloc(sizeof(grpc_ares_ev_driver)));
-  int status = ares_init(&(*ev_driver)->channel);
-  gpr_log(GPR_DEBUG, "grpc_ares_ev_driver_create");
-  if (status != ARES_SUCCESS) {
-    char* err_msg;
-    gpr_asprintf(&err_msg, "Failed to init ares channel. C-ares error: %s",
-                 ares_strerror(status));
-    grpc_error* err = GRPC_ERROR_CREATE_FROM_COPIED_STRING(err_msg);
-    gpr_free(err_msg);
-    gpr_free(*ev_driver);
-    return err;
+  void RegisterForOnReadableLocked(grpc_closure* read_closure) override {
+    grpc_fd_notify_on_read(fd_, read_closure);
   }
-  gpr_mu_init(&(*ev_driver)->mu);
-  gpr_ref_init(&(*ev_driver)->refs, 1);
-  (*ev_driver)->pollset_set = pollset_set;
-  (*ev_driver)->fds = nullptr;
-  (*ev_driver)->working = false;
-  (*ev_driver)->shutting_down = false;
-  return GRPC_ERROR_NONE;
-}
 
-void grpc_ares_ev_driver_destroy(grpc_ares_ev_driver* ev_driver) {
-  // It's not safe to shut down remaining fds here directly, becauses
-  // ares_host_callback does not provide an exec_ctx. We mark the event driver
-  // as being shut down. If the event driver is working,
-  // grpc_ares_notify_on_event_locked will shut down the fds; if it's not
-  // working, there are no fds to shut down.
-  gpr_mu_lock(&ev_driver->mu);
-  ev_driver->shutting_down = true;
-  gpr_mu_unlock(&ev_driver->mu);
-  grpc_ares_ev_driver_unref(ev_driver);
-}
-
-void grpc_ares_ev_driver_shutdown(grpc_ares_ev_driver* ev_driver) {
-  gpr_mu_lock(&ev_driver->mu);
-  ev_driver->shutting_down = true;
-  fd_node* fn = ev_driver->fds;
-  while (fn != nullptr) {
-    grpc_fd_shutdown(fn->fd, GRPC_ERROR_CREATE_FROM_STATIC_STRING(
-                                 "grpc_ares_ev_driver_shutdown"));
-    fn = fn->next;
+  void RegisterForOnWriteableLocked(grpc_closure* write_closure) override {
+    grpc_fd_notify_on_write(fd_, write_closure);
   }
-  gpr_mu_unlock(&ev_driver->mu);
-}
 
-// Search fd in the fd_node list head. This is an O(n) search, the max possible
-// value of n is ARES_GETSOCK_MAXNUM (16). n is typically 1 - 2 in our tests.
-static fd_node* pop_fd_node(fd_node** head, int fd) {
-  fd_node dummy_head;
-  dummy_head.next = *head;
-  fd_node* node = &dummy_head;
-  while (node->next != nullptr) {
-    if (grpc_fd_wrapped_fd(node->next->fd) == fd) {
-      fd_node* ret = node->next;
-      node->next = node->next->next;
-      *head = dummy_head.next;
-      return ret;
-    }
-    node = node->next;
+  bool IsFdStillReadableLocked() override {
+    size_t bytes_available = 0;
+    return ioctl(grpc_fd_wrapped_fd(fd_), FIONREAD, &bytes_available) == 0 &&
+           bytes_available > 0;
   }
-  return nullptr;
-}
-
-/* Check if \a fd is still readable */
-static bool grpc_ares_is_fd_still_readable(grpc_ares_ev_driver* ev_driver,
-                                           int fd) {
-  size_t bytes_available = 0;
-  return ioctl(fd, FIONREAD, &bytes_available) == 0 && bytes_available > 0;
-}
 
-static void on_readable_cb(void* arg, grpc_error* error) {
-  fd_node* fdn = static_cast<fd_node*>(arg);
-  grpc_ares_ev_driver* ev_driver = fdn->ev_driver;
-  gpr_mu_lock(&fdn->mu);
-  const int fd = grpc_fd_wrapped_fd(fdn->fd);
-  fdn->readable_registered = false;
-  if (fdn->shutting_down && !fdn->writable_registered) {
-    gpr_mu_unlock(&fdn->mu);
-    fd_node_destroy(fdn);
-    grpc_ares_ev_driver_unref(ev_driver);
-    return;
+  void ShutdownLocked(grpc_error* error) override {
+    grpc_fd_shutdown(fd_, error);
   }
-  gpr_mu_unlock(&fdn->mu);
 
-  gpr_log(GPR_DEBUG, "readable on %d", fd);
-  if (error == GRPC_ERROR_NONE) {
-    do {
-      ares_process_fd(ev_driver->channel, fd, ARES_SOCKET_BAD);
-    } while (grpc_ares_is_fd_still_readable(ev_driver, fd));
-  } else {
-    // If error is not GRPC_ERROR_NONE, it means the fd has been shutdown or
-    // timed out. The pending lookups made on this ev_driver will be cancelled
-    // by the following ares_cancel() and the on_done callbacks will be invoked
-    // with a status of ARES_ECANCELLED. The remaining file descriptors in this
-    // ev_driver will be cleaned up in the follwing
-    // grpc_ares_notify_on_event_locked().
-    ares_cancel(ev_driver->channel);
-  }
-  gpr_mu_lock(&ev_driver->mu);
-  grpc_ares_notify_on_event_locked(ev_driver);
-  gpr_mu_unlock(&ev_driver->mu);
-  grpc_ares_ev_driver_unref(ev_driver);
-}
+  ares_socket_t GetWrappedAresSocketLocked() override { return as_; }
 
-static void on_writable_cb(void* arg, grpc_error* error) {
-  fd_node* fdn = static_cast<fd_node*>(arg);
-  grpc_ares_ev_driver* ev_driver = fdn->ev_driver;
-  gpr_mu_lock(&fdn->mu);
-  const int fd = grpc_fd_wrapped_fd(fdn->fd);
-  fdn->writable_registered = false;
-  if (fdn->shutting_down && !fdn->readable_registered) {
-    gpr_mu_unlock(&fdn->mu);
-    fd_node_destroy(fdn);
-    grpc_ares_ev_driver_unref(ev_driver);
-    return;
-  }
-  gpr_mu_unlock(&fdn->mu);
+  const char* GetName() override { return name_; }
 
-  gpr_log(GPR_DEBUG, "writable on %d", fd);
-  if (error == GRPC_ERROR_NONE) {
-    ares_process_fd(ev_driver->channel, ARES_SOCKET_BAD, fd);
-  } else {
-    // If error is not GRPC_ERROR_NONE, it means the fd has been shutdown or
-    // timed out. The pending lookups made on this ev_driver will be cancelled
-    // by the following ares_cancel() and the on_done callbacks will be invoked
-    // with a status of ARES_ECANCELLED. The remaining file descriptors in this
-    // ev_driver will be cleaned up in the follwing
-    // grpc_ares_notify_on_event_locked().
-    ares_cancel(ev_driver->channel);
-  }
-  gpr_mu_lock(&ev_driver->mu);
-  grpc_ares_notify_on_event_locked(ev_driver);
-  gpr_mu_unlock(&ev_driver->mu);
-  grpc_ares_ev_driver_unref(ev_driver);
-}
+  char* name_;
+  ares_socket_t as_;
+  grpc_fd* fd_;
+};
 
-ares_channel* grpc_ares_ev_driver_get_channel(grpc_ares_ev_driver* ev_driver) {
-  return &ev_driver->channel;
+GrpcPolledFd* NewGrpcPolledFdLocked(ares_socket_t as,
+                                    grpc_pollset_set* driver_pollset_set) {
+  return grpc_core::New<GrpcPolledFdPosix>(as, driver_pollset_set);
 }
 
-// Get the file descriptors used by the ev_driver's ares channel, register
-// driver_closure with these filedescriptors.
-static void grpc_ares_notify_on_event_locked(grpc_ares_ev_driver* ev_driver) {
-  fd_node* new_list = nullptr;
-  if (!ev_driver->shutting_down) {
-    ares_socket_t socks[ARES_GETSOCK_MAXNUM];
-    int socks_bitmask =
-        ares_getsock(ev_driver->channel, socks, ARES_GETSOCK_MAXNUM);
-    for (size_t i = 0; i < ARES_GETSOCK_MAXNUM; i++) {
-      if (ARES_GETSOCK_READABLE(socks_bitmask, i) ||
-          ARES_GETSOCK_WRITABLE(socks_bitmask, i)) {
-        fd_node* fdn = pop_fd_node(&ev_driver->fds, socks[i]);
-        // Create a new fd_node if sock[i] is not in the fd_node list.
-        if (fdn == nullptr) {
-          char* fd_name;
-          gpr_asprintf(&fd_name, "ares_ev_driver-%" PRIuPTR, i);
-          fdn = static_cast<fd_node*>(gpr_malloc(sizeof(fd_node)));
-          gpr_log(GPR_DEBUG, "new fd: %d", socks[i]);
-          fdn->fd = grpc_fd_create(socks[i], fd_name);
-          fdn->ev_driver = ev_driver;
-          fdn->readable_registered = false;
-          fdn->writable_registered = false;
-          fdn->shutting_down = false;
-          gpr_mu_init(&fdn->mu);
-          GRPC_CLOSURE_INIT(&fdn->read_closure, on_readable_cb, fdn,
-                            grpc_schedule_on_exec_ctx);
-          GRPC_CLOSURE_INIT(&fdn->write_closure, on_writable_cb, fdn,
-                            grpc_schedule_on_exec_ctx);
-          grpc_pollset_set_add_fd(ev_driver->pollset_set, fdn->fd);
-          gpr_free(fd_name);
-        }
-        fdn->next = new_list;
-        new_list = fdn;
-        gpr_mu_lock(&fdn->mu);
-        // Register read_closure if the socket is readable and read_closure has
-        // not been registered with this socket.
-        if (ARES_GETSOCK_READABLE(socks_bitmask, i) &&
-            !fdn->readable_registered) {
-          grpc_ares_ev_driver_ref(ev_driver);
-          gpr_log(GPR_DEBUG, "notify read on: %d", grpc_fd_wrapped_fd(fdn->fd));
-          grpc_fd_notify_on_read(fdn->fd, &fdn->read_closure);
-          fdn->readable_registered = true;
-        }
-        // Register write_closure if the socket is writable and write_closure
-        // has not been registered with this socket.
-        if (ARES_GETSOCK_WRITABLE(socks_bitmask, i) &&
-            !fdn->writable_registered) {
-          gpr_log(GPR_DEBUG, "notify write on: %d",
-                  grpc_fd_wrapped_fd(fdn->fd));
-          grpc_ares_ev_driver_ref(ev_driver);
-          grpc_fd_notify_on_write(fdn->fd, &fdn->write_closure);
-          fdn->writable_registered = true;
-        }
-        gpr_mu_unlock(&fdn->mu);
-      }
-    }
-  }
-  // Any remaining fds in ev_driver->fds were not returned by ares_getsock() and
-  // are therefore no longer in use, so they can be shut down and removed from
-  // the list.
-  while (ev_driver->fds != nullptr) {
-    fd_node* cur = ev_driver->fds;
-    ev_driver->fds = ev_driver->fds->next;
-    fd_node_shutdown(cur);
-  }
-  ev_driver->fds = new_list;
-  // If the ev driver has no working fd, all the tasks are done.
-  if (new_list == nullptr) {
-    ev_driver->working = false;
-    gpr_log(GPR_DEBUG, "ev driver stop working");
-  }
-}
+void ConfigureAresChannelLocked(ares_channel* channel) {}
 
-void grpc_ares_ev_driver_start(grpc_ares_ev_driver* ev_driver) {
-  gpr_mu_lock(&ev_driver->mu);
-  if (!ev_driver->working) {
-    ev_driver->working = true;
-    grpc_ares_notify_on_event_locked(ev_driver);
-  }
-  gpr_mu_unlock(&ev_driver->mu);
-}
+}  // namespace grpc_core
 
-#endif /* GRPC_ARES == 1 && defined(GRPC_POSIX_SOCKET) */
+#endif /* GRPC_ARES == 1 && defined(GRPC_POSIX_SOCKET_ARES_EV_DRIVER) */

src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc

@@ -65,8 +65,6 @@ struct grpc_ares_request {
   /** number of ongoing queries */
   gpr_refcount pending_queries;
 
-  /** mutex guarding the rest of the state */
-  gpr_mu mu;
   /** is there at least one successful query, set in on_done_cb */
   bool success;
   /** the errors explaining the request failure, set in on_done_cb */
@@ -74,7 +72,8 @@ struct grpc_ares_request {
 };
 
 typedef struct grpc_ares_hostbyname_request {
-  /** following members are set in create_hostbyname_request */
+  /** following members are set in create_hostbyname_request_locked
+   */
   /** the top-level request instance */
   grpc_ares_request* parent_request;
   /** host to resolve, parsed from the name to resolve */
@@ -96,10 +95,6 @@ static uint16_t strhtons(const char* port) {
   return htons(static_cast<unsigned short>(atoi(port)));
 }
 
-static void grpc_ares_request_ref(grpc_ares_request* r) {
-  gpr_ref(&r->pending_queries);
-}
-
 static void log_address_sorting_list(grpc_lb_addresses* lb_addrs,
                                      const char* input_output_str) {
   for (size_t i = 0; i < lb_addrs->num_addresses; i++) {
@@ -149,19 +144,25 @@ void grpc_cares_wrapper_test_only_address_sorting_sort(
   grpc_cares_wrapper_address_sorting_sort(lb_addrs);
 }
 
-static void grpc_ares_request_unref(grpc_ares_request* r) {
+static void grpc_ares_request_ref_locked(grpc_ares_request* r) {
+  gpr_ref(&r->pending_queries);
+}
+
+static void grpc_ares_request_unref_locked(grpc_ares_request* r) {
   /* If there are no pending queries, invoke on_done callback and destroy the
      request */
   if (gpr_unref(&r->pending_queries)) {
-    grpc_cares_wrapper_address_sorting_sort(*(r->lb_addrs_out));
+    grpc_lb_addresses* lb_addrs = *(r->lb_addrs_out);
+    if (lb_addrs != nullptr) {
+      grpc_cares_wrapper_address_sorting_sort(lb_addrs);
+    }
     GRPC_CLOSURE_SCHED(r->on_done, r->error);
-    gpr_mu_destroy(&r->mu);
-    grpc_ares_ev_driver_destroy(r->ev_driver);
+    grpc_ares_ev_driver_destroy_locked(r->ev_driver);
     gpr_free(r);
   }
 }
 
-static grpc_ares_hostbyname_request* create_hostbyname_request(
+static grpc_ares_hostbyname_request* create_hostbyname_request_locked(
     grpc_ares_request* parent_request, char* host, uint16_t port,
     bool is_balancer) {
   grpc_ares_hostbyname_request* hr = static_cast<grpc_ares_hostbyname_request*>(
@@ -170,22 +171,22 @@ static grpc_ares_hostbyname_request* create_hostbyname_request(
   hr->host = gpr_strdup(host);
   hr->port = port;
   hr->is_balancer = is_balancer;
-  grpc_ares_request_ref(parent_request);
+  grpc_ares_request_ref_locked(parent_request);
   return hr;
 }
 
-static void destroy_hostbyname_request(grpc_ares_hostbyname_request* hr) {
-  grpc_ares_request_unref(hr->parent_request);
+static void destroy_hostbyname_request_locked(
+    grpc_ares_hostbyname_request* hr) {
+  grpc_ares_request_unref_locked(hr->parent_request);
   gpr_free(hr->host);
   gpr_free(hr);
 }
 
-static void on_hostbyname_done_cb(void* arg, int status, int timeouts,
-                                  struct hostent* hostent) {
+static void on_hostbyname_done_locked(void* arg, int status, int timeouts,
+                                      struct hostent* hostent) {
   grpc_ares_hostbyname_request* hr =
       static_cast<grpc_ares_hostbyname_request*>(arg);
   grpc_ares_request* r = hr->parent_request;
-  gpr_mu_lock(&r->mu);
   if (status == ARES_SUCCESS) {
     GRPC_ERROR_UNREF(r->error);
     r->error = GRPC_ERROR_NONE;
@@ -260,34 +261,33 @@ static void on_hostbyname_done_cb(void* arg, int status, int timeouts,
       r->error = grpc_error_add_child(error, r->error);
     }
   }
-  gpr_mu_unlock(&r->mu);
-  destroy_hostbyname_request(hr);
+  destroy_hostbyname_request_locked(hr);
 }
 
-static void on_srv_query_done_cb(void* arg, int status, int timeouts,
-                                 unsigned char* abuf, int alen) {
+static void on_srv_query_done_locked(void* arg, int status, int timeouts,
+                                     unsigned char* abuf, int alen) {
   grpc_ares_request* r = static_cast<grpc_ares_request*>(arg);
-  grpc_core::ExecCtx exec_ctx;
-  gpr_log(GPR_DEBUG, "on_query_srv_done_cb");
+  gpr_log(GPR_DEBUG, "on_query_srv_done_locked");
   if (status == ARES_SUCCESS) {
-    gpr_log(GPR_DEBUG, "on_query_srv_done_cb ARES_SUCCESS");
+    gpr_log(GPR_DEBUG, "on_query_srv_done_locked ARES_SUCCESS");
     struct ares_srv_reply* reply;
     const int parse_status = ares_parse_srv_reply(abuf, alen, &reply);
     if (parse_status == ARES_SUCCESS) {
-      ares_channel* channel = grpc_ares_ev_driver_get_channel(r->ev_driver);
+      ares_channel* channel =
+          grpc_ares_ev_driver_get_channel_locked(r->ev_driver);
       for (struct ares_srv_reply* srv_it = reply; srv_it != nullptr;
            srv_it = srv_it->next) {
         if (grpc_ipv6_loopback_available()) {
-          grpc_ares_hostbyname_request* hr = create_hostbyname_request(
+          grpc_ares_hostbyname_request* hr = create_hostbyname_request_locked(
               r, srv_it->host, htons(srv_it->port), true /* is_balancer */);
           ares_gethostbyname(*channel, hr->host, AF_INET6,
-                             on_hostbyname_done_cb, hr);
+                             on_hostbyname_done_locked, hr);
         }
-        grpc_ares_hostbyname_request* hr = create_hostbyname_request(
+        grpc_ares_hostbyname_request* hr = create_hostbyname_request_locked(
             r, srv_it->host, htons(srv_it->port), true /* is_balancer */);
-        ares_gethostbyname(*channel, hr->host, AF_INET, on_hostbyname_done_cb,
-                           hr);
-        grpc_ares_ev_driver_start(r->ev_driver);
+        ares_gethostbyname(*channel, hr->host, AF_INET,
+                           on_hostbyname_done_locked, hr);
+        grpc_ares_ev_driver_start_locked(r->ev_driver);
       }
     }
     if (reply != nullptr) {
@@ -305,21 +305,20 @@ static void on_srv_query_done_cb(void* arg, int status, int timeouts,
       r->error = grpc_error_add_child(error, r->error);
     }
   }
-  grpc_ares_request_unref(r);
+  grpc_ares_request_unref_locked(r);
 }
 
 static const char g_service_config_attribute_prefix[] = "grpc_config=";
 
-static void on_txt_done_cb(void* arg, int status, int timeouts,
-                           unsigned char* buf, int len) {
-  gpr_log(GPR_DEBUG, "on_txt_done_cb");
+static void on_txt_done_locked(void* arg, int status, int timeouts,
+                               unsigned char* buf, int len) {
+  gpr_log(GPR_DEBUG, "on_txt_done_locked");
   char* error_msg;
   grpc_ares_request* r = static_cast<grpc_ares_request*>(arg);
   const size_t prefix_len = sizeof(g_service_config_attribute_prefix) - 1;
   struct ares_txt_ext* result = nullptr;
   struct ares_txt_ext* reply = nullptr;
   grpc_error* error = GRPC_ERROR_NONE;
-  gpr_mu_lock(&r->mu);
   if (status != ARES_SUCCESS) goto fail;
   status = ares_parse_txt_reply_ext(buf, len, &reply);
   if (status != ARES_SUCCESS) goto fail;
@@ -364,14 +363,14 @@ fail:
     r->error = grpc_error_add_child(error, r->error);
   }
 done:
-  gpr_mu_unlock(&r->mu);
-  grpc_ares_request_unref(r);
+  grpc_ares_request_unref_locked(r);
 }
 
-static grpc_ares_request* grpc_dns_lookup_ares_impl(
+static grpc_ares_request* grpc_dns_lookup_ares_locked_impl(
     const char* dns_server, const char* name, const char* default_port,
     grpc_pollset_set* interested_parties, grpc_closure* on_done,
-    grpc_lb_addresses** addrs, bool check_grpclb, char** service_config_json) {
+    grpc_lb_addresses** addrs, bool check_grpclb, char** service_config_json,
+    grpc_combiner* combiner) {
   grpc_error* error = GRPC_ERROR_NONE;
   grpc_ares_hostbyname_request* hr = nullptr;
   grpc_ares_request* r = nullptr;
@@ -400,20 +399,19 @@ static grpc_ares_request* grpc_dns_lookup_ares_impl(
     }
     port = gpr_strdup(default_port);
   }
-
   grpc_ares_ev_driver* ev_driver;
-  error = grpc_ares_ev_driver_create(&ev_driver, interested_parties);
+  error = grpc_ares_ev_driver_create_locked(&ev_driver, interested_parties,
+                                            combiner);
   if (error != GRPC_ERROR_NONE) goto error_cleanup;
 
   r = static_cast<grpc_ares_request*>(gpr_zalloc(sizeof(grpc_ares_request)));
-  gpr_mu_init(&r->mu);
   r->ev_driver = ev_driver;
   r->on_done = on_done;
   r->lb_addrs_out = addrs;
   r->service_config_json_out = service_config_json;
   r->success = false;
   r->error = GRPC_ERROR_NONE;
-  channel = grpc_ares_ev_driver_get_channel(r->ev_driver);
+  channel = grpc_ares_ev_driver_get_channel_locked(r->ev_driver);
 
   // If dns_server is specified, use it.
   if (dns_server != nullptr) {
@@ -455,32 +453,34 @@ static grpc_ares_request* grpc_dns_lookup_ares_impl(
   }
   gpr_ref_init(&r->pending_queries, 1);
   if (grpc_ipv6_loopback_available()) {
-    hr = create_hostbyname_request(r, host, strhtons(port),
-                                   false /* is_balancer */);
-    ares_gethostbyname(*channel, hr->host, AF_INET6, on_hostbyname_done_cb, hr);
+    hr = create_hostbyname_request_locked(r, host, strhtons(port),
+                                          false /* is_balancer */);
+    ares_gethostbyname(*channel, hr->host, AF_INET6, on_hostbyname_done_locked,
+                       hr);
   }
-  hr = create_hostbyname_request(r, host, strhtons(port),
-                                 false /* is_balancer */);
-  ares_gethostbyname(*channel, hr->host, AF_INET, on_hostbyname_done_cb, hr);
+  hr = create_hostbyname_request_locked(r, host, strhtons(port),
+                                        false /* is_balancer */);
+  ares_gethostbyname(*channel, hr->host, AF_INET, on_hostbyname_done_locked,
+                     hr);
   if (check_grpclb) {
     /* Query the SRV record */
-    grpc_ares_request_ref(r);
+    grpc_ares_request_ref_locked(r);
     char* service_name;
     gpr_asprintf(&service_name, "_grpclb._tcp.%s", host);
-    ares_query(*channel, service_name, ns_c_in, ns_t_srv, on_srv_query_done_cb,
-               r);
+    ares_query(*channel, service_name, ns_c_in, ns_t_srv,
+               on_srv_query_done_locked, r);
     gpr_free(service_name);
   }
   if (service_config_json != nullptr) {
-    grpc_ares_request_ref(r);
+    grpc_ares_request_ref_locked(r);
     char* config_name;
     gpr_asprintf(&config_name, "_grpc_config.%s", host);
-    ares_search(*channel, config_name, ns_c_in, ns_t_txt, on_txt_done_cb, r);
+    ares_search(*channel, config_name, ns_c_in, ns_t_txt, on_txt_done_locked,
+                r);
     gpr_free(config_name);
   }
-  /* TODO(zyc): Handle CNAME records here. */
-  grpc_ares_ev_driver_start(r->ev_driver);
-  grpc_ares_request_unref(r);
+  grpc_ares_ev_driver_start_locked(r->ev_driver);
+  grpc_ares_request_unref_locked(r);
   gpr_free(host);
   gpr_free(port);
   return r;
@@ -492,15 +492,15 @@ error_cleanup:
   return nullptr;
 }
 
-grpc_ares_request* (*grpc_dns_lookup_ares)(
+grpc_ares_request* (*grpc_dns_lookup_ares_locked)(
     const char* dns_server, const char* name, const char* default_port,
     grpc_pollset_set* interested_parties, grpc_closure* on_done,
-    grpc_lb_addresses** addrs, bool check_grpclb,
-    char** service_config_json) = grpc_dns_lookup_ares_impl;
+    grpc_lb_addresses** addrs, bool check_grpclb, char** service_config_json,
+    grpc_combiner* combiner) = grpc_dns_lookup_ares_locked_impl;
 
 void grpc_cancel_ares_request(grpc_ares_request* r) {
-  if (grpc_dns_lookup_ares == grpc_dns_lookup_ares_impl) {
-    grpc_ares_ev_driver_shutdown(r->ev_driver);
+  if (grpc_dns_lookup_ares_locked == grpc_dns_lookup_ares_locked_impl) {
+    grpc_ares_ev_driver_shutdown_locked(r->ev_driver);
   }
 }
 
@@ -532,6 +532,8 @@ void grpc_ares_cleanup(void) {
  */
 
 typedef struct grpc_resolve_address_ares_request {
+  /* combiner that queries and related callbacks run under */
+  grpc_combiner* combiner;
   /** the pointer to receive the resolved addresses */
   grpc_resolved_addresses** addrs_out;
   /** currently resolving lb addresses */
@@ -539,8 +541,14 @@ typedef struct grpc_resolve_address_ares_request {
   /** closure to call when the resolve_address_ares request completes */
   grpc_closure* on_resolve_address_done;
   /** a closure wrapping on_dns_lookup_done_cb, which should be invoked when the
-      grpc_dns_lookup_ares operation is done. */
+      grpc_dns_lookup_ares_locked operation is done. */
   grpc_closure on_dns_lookup_done;
+  /* target name */
+  const char* name;
+  /* default port to use if none is specified */
+  const char* default_port;
+  /* pollset_set to be driven by */
+  grpc_pollset_set* interested_parties;
 } grpc_resolve_address_ares_request;
 
 static void on_dns_lookup_done_cb(void* arg, grpc_error* error) {
@@ -564,9 +572,20 @@ static void on_dns_lookup_done_cb(void* arg, grpc_error* error) {
   }
   GRPC_CLOSURE_SCHED(r->on_resolve_address_done, GRPC_ERROR_REF(error));
   if (r->lb_addrs != nullptr) grpc_lb_addresses_destroy(r->lb_addrs);
+  GRPC_COMBINER_UNREF(r->combiner, "on_dns_lookup_done_cb");
   gpr_free(r);
 }
 
+static void grpc_resolve_address_invoke_dns_lookup_ares_locked(
+    void* arg, grpc_error* unused_error) {
+  grpc_resolve_address_ares_request* r =
+      static_cast<grpc_resolve_address_ares_request*>(arg);
+  grpc_dns_lookup_ares_locked(
+      nullptr /* dns_server */, r->name, r->default_port, r->interested_parties,
+      &r->on_dns_lookup_done, &r->lb_addrs, false /* check_grpclb */,
+      nullptr /* service_config_json */, r->combiner);
+}
+
 static void grpc_resolve_address_ares_impl(const char* name,
                                            const char* default_port,
                                            grpc_pollset_set* interested_parties,
@@ -575,14 +594,18 @@ static void grpc_resolve_address_ares_impl(const char* name,
   grpc_resolve_address_ares_request* r =
       static_cast<grpc_resolve_address_ares_request*>(
           gpr_zalloc(sizeof(grpc_resolve_address_ares_request)));
+  r->combiner = grpc_combiner_create();
   r->addrs_out = addrs;
   r->on_resolve_address_done = on_done;
   GRPC_CLOSURE_INIT(&r->on_dns_lookup_done, on_dns_lookup_done_cb, r,
                     grpc_schedule_on_exec_ctx);
-  grpc_dns_lookup_ares(nullptr /* dns_server */, name, default_port,
-                       interested_parties, &r->on_dns_lookup_done, &r->lb_addrs,
-                       false /* check_grpclb */,
-                       nullptr /* service_config_json */);
+  r->name = name;
+  r->default_port = default_port;
+  r->interested_parties = interested_parties;
+  GRPC_CLOSURE_SCHED(
+      GRPC_CLOSURE_CREATE(grpc_resolve_address_invoke_dns_lookup_ares_locked, r,
+                          grpc_combiner_scheduler(r->combiner)),
+      GRPC_ERROR_NONE);
 }
 
 void (*grpc_resolve_address_ares)(

src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h

@@ -48,11 +48,11 @@ extern void (*grpc_resolve_address_ares)(const char* name,
   function. \a on_done may be called directly in this function without being
   scheduled with \a exec_ctx, so it must not try to acquire locks that are
   being held by the caller. */
-extern grpc_ares_request* (*grpc_dns_lookup_ares)(
+extern grpc_ares_request* (*grpc_dns_lookup_ares_locked)(
     const char* dns_server, const char* name, const char* default_port,
     grpc_pollset_set* interested_parties, grpc_closure* on_done,
     grpc_lb_addresses** addresses, bool check_grpclb,
-    char** service_config_json);
+    char** service_config_json, grpc_combiner* combiner);
 
 /* Cancel the pending grpc_ares_request \a request */
 void grpc_cancel_ares_request(grpc_ares_request* request);

src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_fallback.cc

@@ -26,18 +26,19 @@ struct grpc_ares_request {
   char val;
 };
 
-static grpc_ares_request* grpc_dns_lookup_ares_impl(
+static grpc_ares_request* grpc_dns_lookup_ares_locked_impl(
     const char* dns_server, const char* name, const char* default_port,
     grpc_pollset_set* interested_parties, grpc_closure* on_done,
-    grpc_lb_addresses** addrs, bool check_grpclb, char** service_config_json) {
+    grpc_lb_addresses** addrs, bool check_grpclb, char** service_config_json,
+    grpc_combiner* combiner) {
   return NULL;
 }
 
-grpc_ares_request* (*grpc_dns_lookup_ares)(
+grpc_ares_request* (*grpc_dns_lookup_ares_locked)(
     const char* dns_server, const char* name, const char* default_port,
     grpc_pollset_set* interested_parties, grpc_closure* on_done,
-    grpc_lb_addresses** addrs, bool check_grpclb,
-    char** service_config_json) = grpc_dns_lookup_ares_impl;
+    grpc_lb_addresses** addrs, bool check_grpclb, char** service_config_json,
+    grpc_combiner* combiner) = grpc_dns_lookup_ares_locked_impl;
 
 void grpc_cancel_ares_request(grpc_ares_request* r) {}
 

src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc

@@ -218,7 +218,7 @@ void NativeDnsResolver::OnResolvedLocked(void* arg, grpc_error* error) {
         r->Ref(DEBUG_LOCATION, "next_resolution_timer");
     self.release();
     if (timeout > 0) {
-      gpr_log(GPR_DEBUG, "retrying in %" PRIdPTR " milliseconds", timeout);
+      gpr_log(GPR_DEBUG, "retrying in %" PRId64 " milliseconds", timeout);
     } else {
       gpr_log(GPR_DEBUG, "retrying immediately");
     }
@@ -236,6 +236,15 @@ void NativeDnsResolver::OnResolvedLocked(void* arg, grpc_error* error) {
 }
 
 void NativeDnsResolver::MaybeStartResolvingLocked() {
+  // If there is an existing timer, the time it fires is the earliest time we
+  // can start the next resolution.
+  if (have_next_resolution_timer_) {
+    // TODO(dgq): remove the following two lines once Pick First stops
+    // discarding subchannels after selecting.
+    ++resolved_version_;
+    MaybeFinishNextLocked();
+    return;
+  }
   if (last_resolution_timestamp_ >= 0) {
     const grpc_millis earliest_next_resolution =
         last_resolution_timestamp_ + min_time_between_resolutions_;
@@ -245,20 +254,18 @@ void NativeDnsResolver::MaybeStartResolvingLocked() {
       const grpc_millis last_resolution_ago =
           grpc_core::ExecCtx::Get()->Now() - last_resolution_timestamp_;
       gpr_log(GPR_DEBUG,
-              "In cooldown from last resolution (from %" PRIdPTR
-              " ms ago). Will resolve again in %" PRIdPTR " ms",
+              "In cooldown from last resolution (from %" PRId64
+              " ms ago). Will resolve again in %" PRId64 " ms",
               last_resolution_ago, ms_until_next_resolution);
-      if (!have_next_resolution_timer_) {
-        have_next_resolution_timer_ = true;
-        // TODO(roth): We currently deal with this ref manually.  Once the
-        // new closure API is done, find a way to track this ref with the timer
-        // callback as part of the type system.
-        RefCountedPtr<Resolver> self =
-            Ref(DEBUG_LOCATION, "next_resolution_timer_cooldown");
-        self.release();
-        grpc_timer_init(&next_resolution_timer_, ms_until_next_resolution,
-                        &on_next_resolution_);
-      }
+      have_next_resolution_timer_ = true;
+      // TODO(roth): We currently deal with this ref manually.  Once the
+      // new closure API is done, find a way to track this ref with the timer
+      // callback as part of the type system.
+      RefCountedPtr<Resolver> self =
+          Ref(DEBUG_LOCATION, "next_resolution_timer_cooldown");
+      self.release();
+      grpc_timer_init(&next_resolution_timer_, ms_until_next_resolution,
+                      &on_next_resolution_);
       // TODO(dgq): remove the following two lines once Pick First stops
       // discarding subchannels after selecting.
       ++resolved_version_;
@@ -270,6 +277,7 @@ void NativeDnsResolver::MaybeStartResolvingLocked() {
 }
 
 void NativeDnsResolver::StartResolvingLocked() {
+  gpr_log(GPR_DEBUG, "Start resolving.");
   // TODO(roth): We currently deal with this ref manually.  Once the
   // new closure API is done, find a way to track this ref with the timer
   // callback as part of the type system.
@@ -302,7 +310,7 @@ class NativeDnsResolverFactory : public ResolverFactory {
  public:
   OrphanablePtr<Resolver> CreateResolver(
       const ResolverArgs& args) const override {
-    if (0 != strcmp(args.uri->authority, "")) {
+    if (GPR_UNLIKELY(0 != strcmp(args.uri->authority, ""))) {
       gpr_log(GPR_ERROR, "authority based dns uri's not supported");
       return OrphanablePtr<Resolver>(nullptr);
     }

src/core/ext/filters/client_channel/retry_throttle.h

@@ -42,6 +42,10 @@ class ServerRetryThrottleData : public RefCounted<ServerRetryThrottleData> {
   intptr_t milli_token_ratio() const { return milli_token_ratio_; }
 
  private:
+  // So Delete() can call our private dtor.
+  template <typename T>
+  friend void grpc_core::Delete(T*);
+
   ~ServerRetryThrottleData();
 
   void GetReplacementThrottleDataIfNeeded(

src/core/ext/filters/client_channel/subchannel.cc

@@ -408,7 +408,7 @@ static void on_external_state_watcher_done(void* arg, grpc_error* error) {
   gpr_mu_unlock(&w->subchannel->mu);
   GRPC_SUBCHANNEL_WEAK_UNREF(w->subchannel, "external_state_watcher");
   gpr_free(w);
-  GRPC_CLOSURE_RUN(follow_up, GRPC_ERROR_REF(error));
+  GRPC_CLOSURE_SCHED(follow_up, GRPC_ERROR_REF(error));
 }
 
 static void on_alarm(void* arg, grpc_error* error) {
@@ -467,7 +467,7 @@ static void maybe_start_connecting_locked(grpc_subchannel* c) {
     if (time_til_next <= 0) {
       gpr_log(GPR_INFO, "Subchannel %p: Retry immediately", c);
     } else {
-      gpr_log(GPR_INFO, "Subchannel %p: Retry in %" PRIdPTR " milliseconds", c,
+      gpr_log(GPR_INFO, "Subchannel %p: Retry in %" PRId64 " milliseconds", c,
               time_til_next);
     }
     GRPC_CLOSURE_INIT(&c->on_alarm, on_alarm, c, grpc_schedule_on_exec_ctx);
@@ -803,7 +803,7 @@ grpc_error* ConnectedSubchannel::CreateCall(const CallArgs& args,
   };
   grpc_error* error = grpc_call_stack_init(
       channel_stack_, 1, subchannel_call_destroy, *call, &call_args);
-  if (error != GRPC_ERROR_NONE) {
+  if (GPR_UNLIKELY(error != GRPC_ERROR_NONE)) {
     const char* error_string = grpc_error_string(error);
     gpr_log(GPR_ERROR, "error: %s", error_string);
     return error;

src/core/ext/filters/deadline/deadline_filter.cc

@@ -128,21 +128,25 @@ static void cancel_timer_if_needed(grpc_deadline_state* deadline_state) {
   }
 }
 
-// Callback run when the call is complete.
-static void on_complete(void* arg, grpc_error* error) {
+// Callback run when we receive trailing metadata.
+static void recv_trailing_metadata_ready(void* arg, grpc_error* error) {
   grpc_deadline_state* deadline_state = static_cast<grpc_deadline_state*>(arg);
   cancel_timer_if_needed(deadline_state);
-  // Invoke the next callback.
-  GRPC_CLOSURE_RUN(deadline_state->next_on_complete, GRPC_ERROR_REF(error));
+  // Invoke the original callback.
+  GRPC_CLOSURE_RUN(deadline_state->original_recv_trailing_metadata_ready,
+                   GRPC_ERROR_REF(error));
 }
 
-// Inject our own on_complete callback into op.
-static void inject_on_complete_cb(grpc_deadline_state* deadline_state,
-                                  grpc_transport_stream_op_batch* op) {
-  deadline_state->next_on_complete = op->on_complete;
-  GRPC_CLOSURE_INIT(&deadline_state->on_complete, on_complete, deadline_state,
+// Inject our own recv_trailing_metadata_ready callback into op.
+static void inject_recv_trailing_metadata_ready(
+    grpc_deadline_state* deadline_state, grpc_transport_stream_op_batch* op) {
+  deadline_state->original_recv_trailing_metadata_ready =
+      op->payload->recv_trailing_metadata.recv_trailing_metadata_ready;
+  GRPC_CLOSURE_INIT(&deadline_state->recv_trailing_metadata_ready,
+                    recv_trailing_metadata_ready, deadline_state,
                     grpc_schedule_on_exec_ctx);
-  op->on_complete = &deadline_state->on_complete;
+  op->payload->recv_trailing_metadata.recv_trailing_metadata_ready =
+      &deadline_state->recv_trailing_metadata_ready;
 }
 
 // Callback and associated state for starting the timer after call stack
@@ -226,7 +230,7 @@ void grpc_deadline_state_client_start_transport_stream_op_batch(
     // Make sure we know when the call is complete, so that we can cancel
     // the timer.
     if (op->recv_trailing_metadata) {
-      inject_on_complete_cb(deadline_state, op);
+      inject_recv_trailing_metadata_ready(deadline_state, op);
     }
   }
 }
@@ -289,11 +293,10 @@ static void client_start_transport_stream_op_batch(
 static void recv_initial_metadata_ready(void* arg, grpc_error* error) {
   grpc_call_element* elem = static_cast<grpc_call_element*>(arg);
   server_call_data* calld = static_cast<server_call_data*>(elem->call_data);
-  // Get deadline from metadata and start the timer if needed.
   start_timer_if_needed(elem, calld->recv_initial_metadata->deadline);
   // Invoke the next callback.
-  calld->next_recv_initial_metadata_ready->cb(
-      calld->next_recv_initial_metadata_ready->cb_arg, error);
+  GRPC_CLOSURE_RUN(calld->next_recv_initial_metadata_ready,
+                   GRPC_ERROR_REF(error));
 }
 
 // Method for starting a call op for server filter.
@@ -323,7 +326,7 @@ static void server_start_transport_stream_op_batch(
     // the client never sends trailing metadata, because this is the
     // hook that tells us when the call is complete on the server side.
     if (op->recv_trailing_metadata) {
-      inject_on_complete_cb(&calld->base.deadline_state, op);
+      inject_recv_trailing_metadata_ready(&calld->base.deadline_state, op);
     }
   }
   // Chain to next filter.

src/core/ext/filters/deadline/deadline_filter.h

@@ -37,12 +37,12 @@ typedef struct grpc_deadline_state {
   grpc_deadline_timer_state timer_state;
   grpc_timer timer;
   grpc_closure timer_callback;
-  // Closure to invoke when the call is complete.
+  // Closure to invoke when we receive trailing metadata.
   // We use this to cancel the timer.
-  grpc_closure on_complete;
-  // The original on_complete closure, which we chain to after our own
-  // closure is invoked.
-  grpc_closure* next_on_complete;
+  grpc_closure recv_trailing_metadata_ready;
+  // The original recv_trailing_metadata_ready closure, which we chain to
+  // after our own closure is invoked.
+  grpc_closure* original_recv_trailing_metadata_ready;
 } grpc_deadline_state;
 
 //

src/core/ext/filters/http/client/http_client_filter.cc

@@ -55,8 +55,8 @@ struct call_data {
   grpc_closure recv_initial_metadata_ready;
   // State for handling recv_trailing_metadata ops.
   grpc_metadata_batch* recv_trailing_metadata;
-  grpc_closure* original_recv_trailing_metadata_on_complete;
-  grpc_closure recv_trailing_metadata_on_complete;
+  grpc_closure* original_recv_trailing_metadata_ready;
+  grpc_closure recv_trailing_metadata_ready;
   // State for handling send_message ops.
   grpc_transport_stream_op_batch* send_message_batch;
   size_t send_message_bytes_read;
@@ -153,8 +153,7 @@ static void recv_initial_metadata_ready(void* user_data, grpc_error* error) {
   GRPC_CLOSURE_RUN(calld->original_recv_initial_metadata_ready, error);
 }
 
-static void recv_trailing_metadata_on_complete(void* user_data,
-                                               grpc_error* error) {
+static void recv_trailing_metadata_ready(void* user_data, grpc_error* error) {
   grpc_call_element* elem = static_cast<grpc_call_element*>(user_data);
   call_data* calld = static_cast<call_data*>(elem->call_data);
   if (error == GRPC_ERROR_NONE) {
@@ -163,7 +162,7 @@ static void recv_trailing_metadata_on_complete(void* user_data,
   } else {
     GRPC_ERROR_REF(error);
   }
-  GRPC_CLOSURE_RUN(calld->original_recv_trailing_metadata_on_complete, error);
+  GRPC_CLOSURE_RUN(calld->original_recv_trailing_metadata_ready, error);
 }
 
 static void send_message_on_complete(void* arg, grpc_error* error) {
@@ -312,8 +311,10 @@ static void hc_start_transport_stream_op_batch(
     /* substitute our callback for the higher callback */
     calld->recv_trailing_metadata =
         batch->payload->recv_trailing_metadata.recv_trailing_metadata;
-    calld->original_recv_trailing_metadata_on_complete = batch->on_complete;
-    batch->on_complete = &calld->recv_trailing_metadata_on_complete;
+    calld->original_recv_trailing_metadata_ready =
+        batch->payload->recv_trailing_metadata.recv_trailing_metadata_ready;
+    batch->payload->recv_trailing_metadata.recv_trailing_metadata_ready =
+        &calld->recv_trailing_metadata_ready;
   }
 
   grpc_error* error = GRPC_ERROR_NONE;
@@ -420,8 +421,8 @@ static grpc_error* init_call_elem(grpc_call_element* elem,
   GRPC_CLOSURE_INIT(&calld->recv_initial_metadata_ready,
                     recv_initial_metadata_ready, elem,
                     grpc_schedule_on_exec_ctx);
-  GRPC_CLOSURE_INIT(&calld->recv_trailing_metadata_on_complete,
-                    recv_trailing_metadata_on_complete, elem,
+  GRPC_CLOSURE_INIT(&calld->recv_trailing_metadata_ready,
+                    recv_trailing_metadata_ready, elem,
                     grpc_schedule_on_exec_ctx);
   GRPC_CLOSURE_INIT(&calld->send_message_on_complete, send_message_on_complete,
                     elem, grpc_schedule_on_exec_ctx);

src/core/ext/filters/http/client_authority_filter.cc

@@ -59,8 +59,9 @@ void authority_start_transport_stream_op_batch(
       initial_metadata->idx.named.authority == nullptr) {
     grpc_error* error = grpc_metadata_batch_add_head(
         initial_metadata, &calld->authority_storage,
-        grpc_mdelem_from_slices(GRPC_MDSTR_AUTHORITY,
-                                grpc_slice_ref(chand->default_authority)));
+        grpc_mdelem_from_slices(
+            GRPC_MDSTR_AUTHORITY,
+            grpc_slice_ref_internal(chand->default_authority)));
     if (error != GRPC_ERROR_NONE) {
       grpc_transport_stream_op_batch_finish_with_failure(batch, error,
                                                          calld->call_combiner);
@@ -102,7 +103,7 @@ grpc_error* init_channel_elem(grpc_channel_element* elem,
         "GRPC_ARG_DEFAULT_AUTHORITY channel arg. must be a string");
   }
   chand->default_authority =
-      grpc_slice_from_copied_string(default_authority_str);
+      grpc_slice_intern(grpc_slice_from_static_string(default_authority_str));
   GPR_ASSERT(!args->is_last);
   return GRPC_ERROR_NONE;
 }
@@ -110,7 +111,7 @@ grpc_error* init_channel_elem(grpc_channel_element* elem,
 /* Destructor for channel data */
 void destroy_channel_elem(grpc_channel_element* elem) {
   channel_data* chand = static_cast<channel_data*>(elem->channel_data);
-  grpc_slice_unref(chand->default_authority);
+  grpc_slice_unref_internal(chand->default_authority);
 }
 }  // namespace
 

src/core/ext/filters/http/message_compress/message_compress_filter.cc

@@ -116,8 +116,8 @@ static grpc_error* process_send_initial_metadata(
   if (initial_metadata->idx.named.grpc_internal_encoding_request != nullptr) {
     grpc_mdelem md =
         initial_metadata->idx.named.grpc_internal_encoding_request->md;
-    if (!grpc_compression_algorithm_parse(GRPC_MDVALUE(md),
-                                          &compression_algorithm)) {
+    if (GPR_UNLIKELY(!grpc_compression_algorithm_parse(
+            GRPC_MDVALUE(md), &compression_algorithm))) {
       char* val = grpc_slice_to_c_string(GRPC_MDVALUE(md));
       gpr_log(GPR_ERROR,
               "Invalid compression algorithm: '%s' (unknown). Ignoring.", val);
@@ -125,8 +125,8 @@ static grpc_error* process_send_initial_metadata(
       calld->message_compression_algorithm = GRPC_MESSAGE_COMPRESS_NONE;
       stream_compression_algorithm = GRPC_STREAM_COMPRESS_NONE;
     }
-    if (!GPR_BITGET(channeld->enabled_algorithms_bitset,
-                    compression_algorithm)) {
+    if (GPR_UNLIKELY(!GPR_BITGET(channeld->enabled_algorithms_bitset,
+                                 compression_algorithm))) {
       char* val = grpc_slice_to_c_string(GRPC_MDVALUE(md));
       gpr_log(GPR_ERROR,
               "Invalid compression algorithm: '%s' (previously disabled). "
@@ -234,7 +234,7 @@ static void finish_send_message(grpc_call_element* elem) {
                                              static_cast<float>(before_size);
       GPR_ASSERT(grpc_message_compression_algorithm_name(
           calld->message_compression_algorithm, &algo_name));
-      gpr_log(GPR_DEBUG,
+      gpr_log(GPR_INFO,
               "Compressed[%s] %" PRIuPTR " bytes vs. %" PRIuPTR
               " bytes (%.2f%% savings)",
               algo_name, before_size, after_size, 100 * savings_ratio);
@@ -246,7 +246,7 @@ static void finish_send_message(grpc_call_element* elem) {
       const char* algo_name;
       GPR_ASSERT(grpc_message_compression_algorithm_name(
           calld->message_compression_algorithm, &algo_name));
-      gpr_log(GPR_DEBUG,
+      gpr_log(GPR_INFO,
               "Algorithm '%s' enabled but decided not to compress. Input size: "
               "%" PRIuPTR,
               algo_name, calld->slices.length);

src/core/ext/filters/http/server/http_server_filter.cc

@@ -35,43 +35,37 @@
 #define EXPECTED_CONTENT_TYPE_LENGTH sizeof(EXPECTED_CONTENT_TYPE) - 1
 
 namespace {
+
 struct call_data {
   grpc_call_combiner* call_combiner;
 
+  // Outgoing headers to add to send_initial_metadata.
   grpc_linked_mdelem status;
   grpc_linked_mdelem content_type;
 
-  /* did this request come with path query containing request payload */
-  bool seen_path_with_query;
-  /* flag to ensure payload_bin is delivered only once */
-  bool payload_bin_delivered;
+  // If we see the recv_message contents in the GET query string, we
+  // store it here.
+  grpc_core::ManualConstructor<grpc_core::SliceBufferByteStream> read_stream;
+  bool have_read_stream;
 
+  // State for intercepting recv_initial_metadata.
+  grpc_closure recv_initial_metadata_ready;
+  grpc_closure* original_recv_initial_metadata_ready;
   grpc_metadata_batch* recv_initial_metadata;
   uint32_t* recv_initial_metadata_flags;
-  /** Closure to call when finished with the hs_on_recv hook */
-  grpc_closure* on_done_recv;
-  /** Closure to call when we retrieve read message from the path URI
-   */
-  grpc_closure* recv_message_ready;
-  grpc_closure* on_complete;
-  grpc_core::OrphanablePtr<grpc_core::ByteStream>* pp_recv_message;
-  grpc_core::ManualConstructor<grpc_core::SliceBufferByteStream> read_stream;
+  bool seen_recv_initial_metadata_ready;
 
-  /** Receive closures are chained: we inject this closure as the on_done_recv
-      up-call on transport_op, and remember to call our on_done_recv member
-      after handling it. */
-  grpc_closure hs_on_recv;
-  grpc_closure hs_on_complete;
-  grpc_closure hs_recv_message_ready;
+  // State for intercepting recv_message.
+  grpc_closure* original_recv_message_ready;
+  grpc_closure recv_message_ready;
+  grpc_core::OrphanablePtr<grpc_core::ByteStream>* recv_message;
+  bool seen_recv_message_ready;
 };
 
-struct channel_data {
-  uint8_t unused;
-};
 }  // namespace
 
-static grpc_error* server_filter_outgoing_metadata(grpc_call_element* elem,
-                                                   grpc_metadata_batch* b) {
+static grpc_error* hs_filter_outgoing_metadata(grpc_call_element* elem,
+                                               grpc_metadata_batch* b) {
   if (b->idx.named.grpc_message != nullptr) {
     grpc_slice pct_encoded_msg = grpc_percent_encode_slice(
         GRPC_MDVALUE(b->idx.named.grpc_message->md),
@@ -86,8 +80,8 @@ static grpc_error* server_filter_outgoing_metadata(grpc_call_element* elem,
   return GRPC_ERROR_NONE;
 }
 
-static void add_error(const char* error_name, grpc_error** cumulative,
-                      grpc_error* new_err) {
+static void hs_add_error(const char* error_name, grpc_error** cumulative,
+                         grpc_error* new_err) {
   if (new_err == GRPC_ERROR_NONE) return;
   if (*cumulative == GRPC_ERROR_NONE) {
     *cumulative = GRPC_ERROR_CREATE_FROM_COPIED_STRING(error_name);
@@ -95,8 +89,8 @@ static void add_error(const char* error_name, grpc_error** cumulative,
   *cumulative = grpc_error_add_child(*cumulative, new_err);
 }
 
-static grpc_error* server_filter_incoming_metadata(grpc_call_element* elem,
-                                                   grpc_metadata_batch* b) {
+static grpc_error* hs_filter_incoming_metadata(grpc_call_element* elem,
+                                               grpc_metadata_batch* b) {
   call_data* calld = static_cast<call_data*>(elem->call_data);
   grpc_error* error = GRPC_ERROR_NONE;
   static const char* error_name = "Failed processing incoming headers";
@@ -119,14 +113,14 @@ static grpc_error* server_filter_incoming_metadata(grpc_call_element* elem,
       *calld->recv_initial_metadata_flags &=
           ~GRPC_INITIAL_METADATA_IDEMPOTENT_REQUEST;
     } else {
-      add_error(error_name, &error,
-                grpc_attach_md_to_error(
-                    GRPC_ERROR_CREATE_FROM_STATIC_STRING("Bad header"),
-                    b->idx.named.method->md));
+      hs_add_error(error_name, &error,
+                   grpc_attach_md_to_error(
+                       GRPC_ERROR_CREATE_FROM_STATIC_STRING("Bad header"),
+                       b->idx.named.method->md));
     }
     grpc_metadata_batch_remove(b, b->idx.named.method);
   } else {
-    add_error(
+    hs_add_error(
         error_name, &error,
         grpc_error_set_str(
             GRPC_ERROR_CREATE_FROM_STATIC_STRING("Missing header"),
@@ -135,31 +129,31 @@ static grpc_error* server_filter_incoming_metadata(grpc_call_element* elem,
 
   if (b->idx.named.te != nullptr) {
     if (!grpc_mdelem_eq(b->idx.named.te->md, GRPC_MDELEM_TE_TRAILERS)) {
-      add_error(error_name, &error,
-                grpc_attach_md_to_error(
-                    GRPC_ERROR_CREATE_FROM_STATIC_STRING("Bad header"),
-                    b->idx.named.te->md));
+      hs_add_error(error_name, &error,
+                   grpc_attach_md_to_error(
+                       GRPC_ERROR_CREATE_FROM_STATIC_STRING("Bad header"),
+                       b->idx.named.te->md));
     }
     grpc_metadata_batch_remove(b, b->idx.named.te);
   } else {
-    add_error(error_name, &error,
-              grpc_error_set_str(
-                  GRPC_ERROR_CREATE_FROM_STATIC_STRING("Missing header"),
-                  GRPC_ERROR_STR_KEY, grpc_slice_from_static_string("te")));
+    hs_add_error(error_name, &error,
+                 grpc_error_set_str(
+                     GRPC_ERROR_CREATE_FROM_STATIC_STRING("Missing header"),
+                     GRPC_ERROR_STR_KEY, grpc_slice_from_static_string("te")));
   }
 
   if (b->idx.named.scheme != nullptr) {
     if (!grpc_mdelem_eq(b->idx.named.scheme->md, GRPC_MDELEM_SCHEME_HTTP) &&
         !grpc_mdelem_eq(b->idx.named.scheme->md, GRPC_MDELEM_SCHEME_HTTPS) &&
         !grpc_mdelem_eq(b->idx.named.scheme->md, GRPC_MDELEM_SCHEME_GRPC)) {
-      add_error(error_name, &error,
-                grpc_attach_md_to_error(
-                    GRPC_ERROR_CREATE_FROM_STATIC_STRING("Bad header"),
-                    b->idx.named.scheme->md));
+      hs_add_error(error_name, &error,
+                   grpc_attach_md_to_error(
+                       GRPC_ERROR_CREATE_FROM_STATIC_STRING("Bad header"),
+                       b->idx.named.scheme->md));
     }
     grpc_metadata_batch_remove(b, b->idx.named.scheme);
   } else {
-    add_error(
+    hs_add_error(
         error_name, &error,
         grpc_error_set_str(
             GRPC_ERROR_CREATE_FROM_STATIC_STRING("Missing header"),
@@ -196,10 +190,11 @@ static grpc_error* server_filter_incoming_metadata(grpc_call_element* elem,
   }
 
   if (b->idx.named.path == nullptr) {
-    add_error(error_name, &error,
-              grpc_error_set_str(
-                  GRPC_ERROR_CREATE_FROM_STATIC_STRING("Missing header"),
-                  GRPC_ERROR_STR_KEY, grpc_slice_from_static_string(":path")));
+    hs_add_error(
+        error_name, &error,
+        grpc_error_set_str(
+            GRPC_ERROR_CREATE_FROM_STATIC_STRING("Missing header"),
+            GRPC_ERROR_STR_KEY, grpc_slice_from_static_string(":path")));
   } else if (*calld->recv_initial_metadata_flags &
              GRPC_INITIAL_METADATA_CACHEABLE_REQUEST) {
     /* We have a cacheable request made with GET verb. The path contains the
@@ -235,7 +230,7 @@ static grpc_error* server_filter_incoming_metadata(grpc_call_element* elem,
               GRPC_SLICE_LENGTH(query_slice), k_url_safe));
       calld->read_stream.Init(&read_slice_buffer, 0);
       grpc_slice_buffer_destroy_internal(&read_slice_buffer);
-      calld->seen_path_with_query = true;
+      calld->have_read_stream = true;
       grpc_slice_unref_internal(query_slice);
     } else {
       gpr_log(GPR_ERROR, "GET request without QUERY");
@@ -246,17 +241,17 @@ static grpc_error* server_filter_incoming_metadata(grpc_call_element* elem,
     grpc_linked_mdelem* el = b->idx.named.host;
     grpc_mdelem md = GRPC_MDELEM_REF(el->md);
     grpc_metadata_batch_remove(b, el);
-    add_error(error_name, &error,
-              grpc_metadata_batch_add_head(
-                  b, el,
-                  grpc_mdelem_from_slices(
-                      GRPC_MDSTR_AUTHORITY,
-                      grpc_slice_ref_internal(GRPC_MDVALUE(md)))));
+    hs_add_error(error_name, &error,
+                 grpc_metadata_batch_add_head(
+                     b, el,
+                     grpc_mdelem_from_slices(
+                         GRPC_MDSTR_AUTHORITY,
+                         grpc_slice_ref_internal(GRPC_MDVALUE(md)))));
     GRPC_MDELEM_UNREF(md);
   }
 
   if (b->idx.named.authority == nullptr) {
-    add_error(
+    hs_add_error(
         error_name, &error,
         grpc_error_set_str(
             GRPC_ERROR_CREATE_FROM_STATIC_STRING("Missing header"),
@@ -266,49 +261,55 @@ static grpc_error* server_filter_incoming_metadata(grpc_call_element* elem,
   return error;
 }
 
-static void hs_on_recv(void* user_data, grpc_error* err) {
+static void hs_recv_initial_metadata_ready(void* user_data, grpc_error* err) {
   grpc_call_element* elem = static_cast<grpc_call_element*>(user_data);
   call_data* calld = static_cast<call_data*>(elem->call_data);
+  calld->seen_recv_initial_metadata_ready = true;
   if (err == GRPC_ERROR_NONE) {
-    err = server_filter_incoming_metadata(elem, calld->recv_initial_metadata);
+    err = hs_filter_incoming_metadata(elem, calld->recv_initial_metadata);
+    if (calld->seen_recv_message_ready) {
+      // We've already seen the recv_message callback, but we previously
+      // deferred it, so we need to return it here.
+      // Replace the recv_message byte stream if needed.
+      if (calld->have_read_stream) {
+        calld->recv_message->reset(calld->read_stream.get());
+        calld->have_read_stream = false;
+      }
+      // Re-enter call combiner for original_recv_message_ready, since the
+      // surface code will release the call combiner for each callback it
+      // receives.
+      GRPC_CALL_COMBINER_START(
+          calld->call_combiner, calld->original_recv_message_ready,
+          GRPC_ERROR_REF(err),
+          "resuming recv_message_ready from recv_initial_metadata_ready");
+    }
   } else {
     GRPC_ERROR_REF(err);
   }
-  GRPC_CLOSURE_RUN(calld->on_done_recv, err);
-}
-
-static void hs_on_complete(void* user_data, grpc_error* err) {
-  grpc_call_element* elem = static_cast<grpc_call_element*>(user_data);
-  call_data* calld = static_cast<call_data*>(elem->call_data);
-  /* Call recv_message_ready if we got the payload via the path field */
-  if (calld->seen_path_with_query && calld->recv_message_ready != nullptr) {
-    calld->pp_recv_message->reset(
-        calld->payload_bin_delivered ? nullptr
-                                     : reinterpret_cast<grpc_core::ByteStream*>(
-                                           calld->read_stream.get()));
-    // Re-enter call combiner for recv_message_ready, since the surface
-    // code will release the call combiner for each callback it receives.
-    GRPC_CALL_COMBINER_START(calld->call_combiner, calld->recv_message_ready,
-                             GRPC_ERROR_REF(err),
-                             "resuming recv_message_ready from on_complete");
-    calld->recv_message_ready = nullptr;
-    calld->payload_bin_delivered = true;
-  }
-  GRPC_CLOSURE_RUN(calld->on_complete, GRPC_ERROR_REF(err));
+  GRPC_CLOSURE_RUN(calld->original_recv_initial_metadata_ready, err);
 }
 
 static void hs_recv_message_ready(void* user_data, grpc_error* err) {
   grpc_call_element* elem = static_cast<grpc_call_element*>(user_data);
   call_data* calld = static_cast<call_data*>(elem->call_data);
-  if (calld->seen_path_with_query) {
-    // Do nothing. This is probably a GET request, and payload will be
-    // returned in hs_on_complete callback.
+  calld->seen_recv_message_ready = true;
+  if (calld->seen_recv_initial_metadata_ready) {
+    // We've already seen the recv_initial_metadata callback, so
+    // replace the recv_message byte stream if needed and invoke the
+    // original recv_message callback immediately.
+    if (calld->have_read_stream) {
+      calld->recv_message->reset(calld->read_stream.get());
+      calld->have_read_stream = false;
+    }
+    GRPC_CLOSURE_RUN(calld->original_recv_message_ready, GRPC_ERROR_REF(err));
+  } else {
+    // We have not yet seen the recv_initial_metadata callback, so we
+    // need to wait to see if this is a GET request.
     // Note that we release the call combiner here, so that other
     // callbacks can run.
-    GRPC_CALL_COMBINER_STOP(calld->call_combiner,
-                            "pausing recv_message_ready until on_complete");
-  } else {
-    GRPC_CLOSURE_RUN(calld->recv_message_ready, GRPC_ERROR_REF(err));
+    GRPC_CALL_COMBINER_STOP(
+        calld->call_combiner,
+        "pausing recv_message_ready until recv_initial_metadata_ready");
   }
 }
 
@@ -320,18 +321,18 @@ static grpc_error* hs_mutate_op(grpc_call_element* elem,
   if (op->send_initial_metadata) {
     grpc_error* error = GRPC_ERROR_NONE;
     static const char* error_name = "Failed sending initial metadata";
-    add_error(error_name, &error,
-              grpc_metadata_batch_add_head(
-                  op->payload->send_initial_metadata.send_initial_metadata,
-                  &calld->status, GRPC_MDELEM_STATUS_200));
-    add_error(error_name, &error,
-              grpc_metadata_batch_add_tail(
-                  op->payload->send_initial_metadata.send_initial_metadata,
-                  &calld->content_type,
-                  GRPC_MDELEM_CONTENT_TYPE_APPLICATION_SLASH_GRPC));
-    add_error(
+    hs_add_error(error_name, &error,
+                 grpc_metadata_batch_add_head(
+                     op->payload->send_initial_metadata.send_initial_metadata,
+                     &calld->status, GRPC_MDELEM_STATUS_200));
+    hs_add_error(error_name, &error,
+                 grpc_metadata_batch_add_tail(
+                     op->payload->send_initial_metadata.send_initial_metadata,
+                     &calld->content_type,
+                     GRPC_MDELEM_CONTENT_TYPE_APPLICATION_SLASH_GRPC));
+    hs_add_error(
         error_name, &error,
-        server_filter_outgoing_metadata(
+        hs_filter_outgoing_metadata(
             elem, op->payload->send_initial_metadata.send_initial_metadata));
     if (error != GRPC_ERROR_NONE) return error;
   }
@@ -343,27 +344,21 @@ static grpc_error* hs_mutate_op(grpc_call_element* elem,
         op->payload->recv_initial_metadata.recv_initial_metadata;
     calld->recv_initial_metadata_flags =
         op->payload->recv_initial_metadata.recv_flags;
-    calld->on_done_recv =
+    calld->original_recv_initial_metadata_ready =
         op->payload->recv_initial_metadata.recv_initial_metadata_ready;
     op->payload->recv_initial_metadata.recv_initial_metadata_ready =
-        &calld->hs_on_recv;
+        &calld->recv_initial_metadata_ready;
   }
 
   if (op->recv_message) {
-    calld->recv_message_ready = op->payload->recv_message.recv_message_ready;
-    calld->pp_recv_message = op->payload->recv_message.recv_message;
-    if (op->payload->recv_message.recv_message_ready) {
-      op->payload->recv_message.recv_message_ready =
-          &calld->hs_recv_message_ready;
-    }
-    if (op->on_complete) {
-      calld->on_complete = op->on_complete;
-      op->on_complete = &calld->hs_on_complete;
-    }
+    calld->recv_message = op->payload->recv_message.recv_message;
+    calld->original_recv_message_ready =
+        op->payload->recv_message.recv_message_ready;
+    op->payload->recv_message.recv_message_ready = &calld->recv_message_ready;
   }
 
   if (op->send_trailing_metadata) {
-    grpc_error* error = server_filter_outgoing_metadata(
+    grpc_error* error = hs_filter_outgoing_metadata(
         elem, op->payload->send_trailing_metadata.send_trailing_metadata);
     if (error != GRPC_ERROR_NONE) return error;
   }
@@ -385,50 +380,47 @@ static void hs_start_transport_stream_op_batch(
 }
 
 /* Constructor for call_data */
-static grpc_error* init_call_elem(grpc_call_element* elem,
-                                  const grpc_call_element_args* args) {
-  /* grab pointers to our data from the call element */
+static grpc_error* hs_init_call_elem(grpc_call_element* elem,
+                                     const grpc_call_element_args* args) {
   call_data* calld = static_cast<call_data*>(elem->call_data);
-  /* initialize members */
   calld->call_combiner = args->call_combiner;
-  GRPC_CLOSURE_INIT(&calld->hs_on_recv, hs_on_recv, elem,
-                    grpc_schedule_on_exec_ctx);
-  GRPC_CLOSURE_INIT(&calld->hs_on_complete, hs_on_complete, elem,
+  GRPC_CLOSURE_INIT(&calld->recv_initial_metadata_ready,
+                    hs_recv_initial_metadata_ready, elem,
                     grpc_schedule_on_exec_ctx);
-  GRPC_CLOSURE_INIT(&calld->hs_recv_message_ready, hs_recv_message_ready, elem,
+  GRPC_CLOSURE_INIT(&calld->recv_message_ready, hs_recv_message_ready, elem,
                     grpc_schedule_on_exec_ctx);
   return GRPC_ERROR_NONE;
 }
 
 /* Destructor for call_data */
-static void destroy_call_elem(grpc_call_element* elem,
-                              const grpc_call_final_info* final_info,
-                              grpc_closure* ignored) {
+static void hs_destroy_call_elem(grpc_call_element* elem,
+                                 const grpc_call_final_info* final_info,
+                                 grpc_closure* ignored) {
   call_data* calld = static_cast<call_data*>(elem->call_data);
-  if (calld->seen_path_with_query && !calld->payload_bin_delivered) {
+  if (calld->have_read_stream) {
     calld->read_stream->Orphan();
   }
 }
 
 /* Constructor for channel_data */
-static grpc_error* init_channel_elem(grpc_channel_element* elem,
-                                     grpc_channel_element_args* args) {
+static grpc_error* hs_init_channel_elem(grpc_channel_element* elem,
+                                        grpc_channel_element_args* args) {
   GPR_ASSERT(!args->is_last);
   return GRPC_ERROR_NONE;
 }
 
 /* Destructor for channel data */
-static void destroy_channel_elem(grpc_channel_element* elem) {}
+static void hs_destroy_channel_elem(grpc_channel_element* elem) {}
 
 const grpc_channel_filter grpc_http_server_filter = {
     hs_start_transport_stream_op_batch,
     grpc_channel_next_op,
     sizeof(call_data),
-    init_call_elem,
+    hs_init_call_elem,
     grpc_call_stack_ignore_set_pollset_or_pollset_set,
-    destroy_call_elem,
-    sizeof(channel_data),
-    init_channel_elem,
-    destroy_channel_elem,
+    hs_destroy_call_elem,
+    0,
+    hs_init_channel_elem,
+    hs_destroy_channel_elem,
     grpc_channel_next_get_info,
     "http-server"};

src/core/ext/filters/http/server/http_server_filter.h

@@ -23,7 +23,7 @@
 
 #include "src/core/lib/channel/channel_stack.h"
 
-/* Processes metadata on the client side for HTTP2 transports */
+/* Processes metadata on the server side for HTTP2 transports */
 extern const grpc_channel_filter grpc_http_server_filter;
 
 #endif /* GRPC_CORE_EXT_FILTERS_HTTP_SERVER_HTTP_SERVER_FILTER_H */

src/core/ext/filters/load_reporting/server_load_reporting_filter.cc

@@ -82,9 +82,7 @@ static void on_initial_md_ready(void* user_data, grpc_error* err) {
   } else {
     GRPC_ERROR_REF(err);
   }
-  calld->ops_recv_initial_metadata_ready->cb(
-      calld->ops_recv_initial_metadata_ready->cb_arg, err);
-  GRPC_ERROR_UNREF(err);
+  GRPC_CLOSURE_RUN(calld->ops_recv_initial_metadata_ready, err);
 }
 
 /* Constructor for call_data */

src/core/ext/transport/chttp2/client/insecure/channel_create_posix.cc

@@ -50,7 +50,7 @@ grpc_channel* grpc_insecure_channel_create_from_fd(
   GPR_ASSERT(fcntl(fd, F_SETFL, flags | O_NONBLOCK) == 0);
 
   grpc_endpoint* client = grpc_tcp_client_create_from_fd(
-      grpc_fd_create(fd, "client"), args, "fd-client");
+      grpc_fd_create(fd, "client", false), args, "fd-client");
 
   grpc_transport* transport =
       grpc_create_chttp2_transport(final_args, client, true);

src/core/ext/transport/chttp2/server/chttp2_server.cc

@@ -179,8 +179,8 @@ static void on_accept(void* arg, grpc_endpoint* tcp,
   grpc_handshake_manager* handshake_mgr = grpc_handshake_manager_create();
   grpc_handshake_manager_pending_list_add(&state->pending_handshake_mgrs,
                                           handshake_mgr);
-  gpr_mu_unlock(&state->mu);
   grpc_tcp_server_ref(state->tcp_server);
+  gpr_mu_unlock(&state->mu);
   server_connection_state* connection_state =
       static_cast<server_connection_state*>(
           gpr_zalloc(sizeof(*connection_state)));