Merge remote-tracking branch 'upstream/master' into upb_upgrade

.github/ISSUE_TEMPLATE/bug_report.md

@@ -2,7 +2,7 @@
 name: Report a bug
 about: Create a report to help us improve
 labels: kind/bug, priority/P2
-assignees: markdroth 
+assignees: nicolasnoble 
 
 ---
 

.github/ISSUE_TEMPLATE/cleanup_request.md

@@ -2,7 +2,7 @@
 name: Request a cleanup
 about: Suggest a cleanup in our repository
 labels: kind/internal cleanup, priority/P2
-assignees: markdroth 
+assignees: nicolasnoble 
 
 ---
 

.github/ISSUE_TEMPLATE/feature_request.md

@@ -2,7 +2,7 @@
 name: Request a feature
 about: Suggest an idea for this project
 labels: kind/enhancement, priority/P2
-assignees: markdroth 
+assignees: nicolasnoble 
 
 ---
 

.github/pull_request_template.md

@@ -8,4 +8,4 @@ If you know who should review your pull request, please remove the mentioning be
 
 -->
 
-@markdroth
+@nicolasnoble

.gitignore

@@ -115,11 +115,7 @@ Podfile.lock
 .idea/
 
 # Bazel files
-bazel-bin
-bazel-genfiles
-bazel-grpc
-bazel-out
-bazel-testlogs
+bazel-*
 bazel_format_virtual_environment/
 tools/bazel-*
 

.pylintrc

@@ -12,14 +12,14 @@ extension-pkg-whitelist=grpc._cython.cygrpc
 
 # TODO(https://github.com/PyCQA/pylint/issues/1345): How does the inspection
 # not include "unused_" and "ignored_" by default?
-dummy-variables-rgx=^ignored_|^unused_
+dummy-variables-rgx=^ignored_|^unused_|_
 
 [DESIGN]
 
 # NOTE(nathaniel): Not particularly attached to this value; it just seems to
 # be what works for us at the moment (excepting the dead-code-walking Beta
 # API).
-max-args=7
+max-args=14
 max-parents=8
 
 [MISCELLANEOUS]

BUILD

@@ -98,6 +98,7 @@ GPR_PUBLIC_HDRS = [
     "include/grpc/support/port_platform.h",
     "include/grpc/support/string_util.h",
     "include/grpc/support/sync.h",
+    "include/grpc/support/sync_abseil.h",
     "include/grpc/support/sync_custom.h",
     "include/grpc/support/sync_generic.h",
     "include/grpc/support/sync_posix.h",
@@ -199,8 +200,6 @@ GRPCXX_PUBLIC_HDRS = [
     "include/grpc++/impl/server_builder_plugin.h",
     "include/grpc++/impl/server_initializer.h",
     "include/grpc++/impl/service_type.h",
-    "include/grpc++/impl/sync_cxx11.h",
-    "include/grpc++/impl/sync_no_cxx11.h",
     "include/grpc++/security/auth_context.h",
     "include/grpc++/resource_quota.h",
     "include/grpc++/security/auth_metadata_processor.h",
@@ -255,8 +254,6 @@ GRPCXX_PUBLIC_HDRS = [
     "include/grpcpp/impl/server_initializer.h",
     "include/grpcpp/impl/server_initializer_impl.h",
     "include/grpcpp/impl/service_type.h",
-    "include/grpcpp/impl/sync_cxx11.h",
-    "include/grpcpp/impl/sync_no_cxx11.h",
     "include/grpcpp/resource_quota.h",
     "include/grpcpp/resource_quota_impl.h",
     "include/grpcpp/security/auth_context.h",
@@ -509,6 +506,7 @@ grpc_cc_library(
         "src/core/lib/gpr/string_util_windows.cc",
         "src/core/lib/gpr/string_windows.cc",
         "src/core/lib/gpr/sync.cc",
+        "src/core/lib/gpr/sync_abseil.cc",
         "src/core/lib/gpr/sync_posix.cc",
         "src/core/lib/gpr/sync_windows.cc",
         "src/core/lib/gpr/time.cc",
@@ -587,6 +585,7 @@ grpc_cc_library(
         "include/grpc/impl/codegen/log.h",
         "include/grpc/impl/codegen/port_platform.h",
         "include/grpc/impl/codegen/sync.h",
+        "include/grpc/impl/codegen/sync_abseil.h",
         "include/grpc/impl/codegen/sync_custom.h",
         "include/grpc/impl/codegen/sync_generic.h",
         "include/grpc/impl/codegen/sync_posix.h",
@@ -747,6 +746,7 @@ grpc_cc_library(
         "src/core/lib/iomgr/iomgr_internal.cc",
         "src/core/lib/iomgr/iomgr_posix.cc",
         "src/core/lib/iomgr/iomgr_posix_cfstream.cc",
+        "src/core/lib/iomgr/iomgr_uv.cc",
         "src/core/lib/iomgr/iomgr_windows.cc",
         "src/core/lib/iomgr/is_epollexclusive_available.cc",
         "src/core/lib/iomgr/load_file.cc",
@@ -770,6 +770,7 @@ grpc_cc_library(
         "src/core/lib/iomgr/socket_utils_common_posix.cc",
         "src/core/lib/iomgr/socket_utils_linux.cc",
         "src/core/lib/iomgr/socket_utils_posix.cc",
+        "src/core/lib/iomgr/socket_utils_uv.cc",
         "src/core/lib/iomgr/socket_utils_windows.cc",
         "src/core/lib/iomgr/socket_windows.cc",
         "src/core/lib/iomgr/tcp_client.cc",
@@ -989,6 +990,7 @@ grpc_cc_library(
     public_hdrs = GRPC_PUBLIC_HDRS,
     use_cfstream = True,
     deps = [
+        "eventmanager_libuv",
         "gpr_base",
         "grpc_codegen",
         "grpc_trace",

BUILD.gn

@@ -66,6 +66,7 @@ config("grpc_config") {
         "include/grpc/impl/codegen/log.h",
         "include/grpc/impl/codegen/port_platform.h",
         "include/grpc/impl/codegen/sync.h",
+        "include/grpc/impl/codegen/sync_abseil.h",
         "include/grpc/impl/codegen/sync_custom.h",
         "include/grpc/impl/codegen/sync_generic.h",
         "include/grpc/impl/codegen/sync_posix.h",
@@ -81,6 +82,7 @@ config("grpc_config") {
         "include/grpc/support/port_platform.h",
         "include/grpc/support/string_util.h",
         "include/grpc/support/sync.h",
+        "include/grpc/support/sync_abseil.h",
         "include/grpc/support/sync_custom.h",
         "include/grpc/support/sync_generic.h",
         "include/grpc/support/sync_posix.h",
@@ -114,6 +116,7 @@ config("grpc_config") {
         "src/core/lib/gpr/string_windows.cc",
         "src/core/lib/gpr/string_windows.h",
         "src/core/lib/gpr/sync.cc",
+        "src/core/lib/gpr/sync_abseil.cc",
         "src/core/lib/gpr/sync_posix.cc",
         "src/core/lib/gpr/sync_windows.cc",
         "src/core/lib/gpr/time.cc",
@@ -199,6 +202,7 @@ config("grpc_config") {
         "include/grpc/impl/codegen/slice.h",
         "include/grpc/impl/codegen/status.h",
         "include/grpc/impl/codegen/sync.h",
+        "include/grpc/impl/codegen/sync_abseil.h",
         "include/grpc/impl/codegen/sync_custom.h",
         "include/grpc/impl/codegen/sync_generic.h",
         "include/grpc/impl/codegen/sync_posix.h",
@@ -1051,6 +1055,7 @@ config("grpc_config") {
         "include/grpc/impl/codegen/slice.h",
         "include/grpc/impl/codegen/status.h",
         "include/grpc/impl/codegen/sync.h",
+        "include/grpc/impl/codegen/sync_abseil.h",
         "include/grpc/impl/codegen/sync_custom.h",
         "include/grpc/impl/codegen/sync_generic.h",
         "include/grpc/impl/codegen/sync_posix.h",
@@ -1070,6 +1075,7 @@ config("grpc_config") {
         "include/grpc/support/port_platform.h",
         "include/grpc/support/string_util.h",
         "include/grpc/support/sync.h",
+        "include/grpc/support/sync_abseil.h",
         "include/grpc/support/sync_custom.h",
         "include/grpc/support/sync_generic.h",
         "include/grpc/support/sync_posix.h",

BUILDING.md

@@ -72,13 +72,13 @@ To prepare for cmake + Microsoft Visual C++ compiler build
 # Clone the repository (including submodules)
 
 Before building, you need to clone the gRPC github repository and download submodules containing source code
-for gRPC's dependencies (that's done by the `submodule` command or `--recursive` flag). The following commands will clone the gRPC
-repository at the latest stable version.
+for gRPC's dependencies (that's done by the `submodule` command or `--recursive` flag). Use following commands
+to clone the gRPC repository at the [latest stable release tag](https://github.com/grpc/grpc/releases)
 
 ## Unix
 
 ```sh
- $ git clone -b $(curl -L https://grpc.io/release) https://github.com/grpc/grpc
+ $ git clone -b RELEASE_TAG_HERE https://github.com/grpc/grpc
  $ cd grpc
  $ git submodule update --init
  ```
@@ -86,10 +86,9 @@ repository at the latest stable version.
 ## Windows
 
 ```
-> @rem You can also do just "git clone --recursive -b THE_BRANCH_YOU_WANT https://github.com/grpc/grpc"
-> powershell git clone --recursive -b ((New-Object System.Net.WebClient).DownloadString(\"https://grpc.io/release\").Trim()) https://github.com/grpc/grpc
+> git clone -b RELEASE_TAG_HERE https://github.com/grpc/grpc
 > cd grpc
-> @rem To update submodules at later time, run "git submodule update --init"
+> git submodule update --init
 ```
 
 NOTE: The `bazel` build tool uses a different model for dependencies. You only need to worry about downloading submodules if you're building

CMakeLists.txt

@@ -654,7 +654,6 @@ if(gRPC_BUILD_TESTS)
     add_dependencies(buildtests_c h2_uds_nosec_test)
   endif()
   add_dependencies(buildtests_c alts_credentials_fuzzer_one_entry)
-  add_dependencies(buildtests_c api_fuzzer_one_entry)
   add_dependencies(buildtests_c client_fuzzer_one_entry)
   add_dependencies(buildtests_c hpack_parser_fuzzer_test_one_entry)
   add_dependencies(buildtests_c http_request_fuzzer_test_one_entry)
@@ -1287,6 +1286,7 @@ foreach(_hdr
   include/grpc/support/port_platform.h
   include/grpc/support/string_util.h
   include/grpc/support/sync.h
+  include/grpc/support/sync_abseil.h
   include/grpc/support/sync_custom.h
   include/grpc/support/sync_generic.h
   include/grpc/support/sync_posix.h
@@ -1303,6 +1303,7 @@ foreach(_hdr
   include/grpc/impl/codegen/log.h
   include/grpc/impl/codegen/port_platform.h
   include/grpc/impl/codegen/sync.h
+  include/grpc/impl/codegen/sync_abseil.h
   include/grpc/impl/codegen/sync_custom.h
   include/grpc/impl/codegen/sync_generic.h
   include/grpc/impl/codegen/sync_posix.h
@@ -1385,6 +1386,7 @@ add_library(gpr
   src/core/lib/gpr/string_util_windows.cc
   src/core/lib/gpr/string_windows.cc
   src/core/lib/gpr/sync.cc
+  src/core/lib/gpr/sync_abseil.cc
   src/core/lib/gpr/sync_posix.cc
   src/core/lib/gpr/sync_windows.cc
   src/core/lib/gpr/time.cc
@@ -1459,6 +1461,7 @@ foreach(_hdr
   include/grpc/support/port_platform.h
   include/grpc/support/string_util.h
   include/grpc/support/sync.h
+  include/grpc/support/sync_abseil.h
   include/grpc/support/sync_custom.h
   include/grpc/support/sync_generic.h
   include/grpc/support/sync_posix.h
@@ -1475,6 +1478,7 @@ foreach(_hdr
   include/grpc/impl/codegen/log.h
   include/grpc/impl/codegen/port_platform.h
   include/grpc/impl/codegen/sync.h
+  include/grpc/impl/codegen/sync_abseil.h
   include/grpc/impl/codegen/sync_custom.h
   include/grpc/impl/codegen/sync_generic.h
   include/grpc/impl/codegen/sync_posix.h
@@ -1943,6 +1947,7 @@ foreach(_hdr
   include/grpc/impl/codegen/log.h
   include/grpc/impl/codegen/port_platform.h
   include/grpc/impl/codegen/sync.h
+  include/grpc/impl/codegen/sync_abseil.h
   include/grpc/impl/codegen/sync_custom.h
   include/grpc/impl/codegen/sync_generic.h
   include/grpc/impl/codegen/sync_posix.h
@@ -2353,6 +2358,7 @@ foreach(_hdr
   include/grpc/impl/codegen/log.h
   include/grpc/impl/codegen/port_platform.h
   include/grpc/impl/codegen/sync.h
+  include/grpc/impl/codegen/sync_abseil.h
   include/grpc/impl/codegen/sync_custom.h
   include/grpc/impl/codegen/sync_generic.h
   include/grpc/impl/codegen/sync_posix.h
@@ -2687,6 +2693,7 @@ foreach(_hdr
   include/grpc/support/port_platform.h
   include/grpc/support/string_util.h
   include/grpc/support/sync.h
+  include/grpc/support/sync_abseil.h
   include/grpc/support/sync_custom.h
   include/grpc/support/sync_generic.h
   include/grpc/support/sync_posix.h
@@ -2703,6 +2710,7 @@ foreach(_hdr
   include/grpc/impl/codegen/log.h
   include/grpc/impl/codegen/port_platform.h
   include/grpc/impl/codegen/sync.h
+  include/grpc/impl/codegen/sync_abseil.h
   include/grpc/impl/codegen/sync_custom.h
   include/grpc/impl/codegen/sync_generic.h
   include/grpc/impl/codegen/sync_posix.h
@@ -3029,6 +3037,7 @@ foreach(_hdr
   include/grpc/support/port_platform.h
   include/grpc/support/string_util.h
   include/grpc/support/sync.h
+  include/grpc/support/sync_abseil.h
   include/grpc/support/sync_custom.h
   include/grpc/support/sync_generic.h
   include/grpc/support/sync_posix.h
@@ -3045,6 +3054,7 @@ foreach(_hdr
   include/grpc/impl/codegen/log.h
   include/grpc/impl/codegen/port_platform.h
   include/grpc/impl/codegen/sync.h
+  include/grpc/impl/codegen/sync_abseil.h
   include/grpc/impl/codegen/sync_custom.h
   include/grpc/impl/codegen/sync_generic.h
   include/grpc/impl/codegen/sync_posix.h
@@ -3437,6 +3447,7 @@ foreach(_hdr
   include/grpc/impl/codegen/log.h
   include/grpc/impl/codegen/port_platform.h
   include/grpc/impl/codegen/sync.h
+  include/grpc/impl/codegen/sync_abseil.h
   include/grpc/impl/codegen/sync_custom.h
   include/grpc/impl/codegen/sync_generic.h
   include/grpc/impl/codegen/sync_posix.h
@@ -3887,6 +3898,7 @@ foreach(_hdr
   include/grpc/support/port_platform.h
   include/grpc/support/string_util.h
   include/grpc/support/sync.h
+  include/grpc/support/sync_abseil.h
   include/grpc/support/sync_custom.h
   include/grpc/support/sync_generic.h
   include/grpc/support/sync_posix.h
@@ -3903,6 +3915,7 @@ foreach(_hdr
   include/grpc/impl/codegen/log.h
   include/grpc/impl/codegen/port_platform.h
   include/grpc/impl/codegen/sync.h
+  include/grpc/impl/codegen/sync_abseil.h
   include/grpc/impl/codegen/sync_custom.h
   include/grpc/impl/codegen/sync_generic.h
   include/grpc/impl/codegen/sync_posix.h
@@ -4586,6 +4599,7 @@ foreach(_hdr
   include/grpc/impl/codegen/log.h
   include/grpc/impl/codegen/port_platform.h
   include/grpc/impl/codegen/sync.h
+  include/grpc/impl/codegen/sync_abseil.h
   include/grpc/impl/codegen/sync_custom.h
   include/grpc/impl/codegen/sync_generic.h
   include/grpc/impl/codegen/sync_posix.h
@@ -4783,6 +4797,7 @@ foreach(_hdr
   include/grpc/impl/codegen/log.h
   include/grpc/impl/codegen/port_platform.h
   include/grpc/impl/codegen/sync.h
+  include/grpc/impl/codegen/sync_abseil.h
   include/grpc/impl/codegen/sync_custom.h
   include/grpc/impl/codegen/sync_generic.h
   include/grpc/impl/codegen/sync_posix.h
@@ -5019,6 +5034,7 @@ foreach(_hdr
   include/grpc/support/port_platform.h
   include/grpc/support/string_util.h
   include/grpc/support/sync.h
+  include/grpc/support/sync_abseil.h
   include/grpc/support/sync_custom.h
   include/grpc/support/sync_generic.h
   include/grpc/support/sync_posix.h
@@ -5035,6 +5051,7 @@ foreach(_hdr
   include/grpc/impl/codegen/log.h
   include/grpc/impl/codegen/port_platform.h
   include/grpc/impl/codegen/sync.h
+  include/grpc/impl/codegen/sync_abseil.h
   include/grpc/impl/codegen/sync_custom.h
   include/grpc/impl/codegen/sync_generic.h
   include/grpc/impl/codegen/sync_posix.h
@@ -18159,34 +18176,6 @@ target_link_libraries(alts_credentials_fuzzer_one_entry
 )
 
 
-endif()
-if(gRPC_BUILD_TESTS)
-
-add_executable(api_fuzzer_one_entry
-  test/core/end2end/fuzzers/api_fuzzer.cc
-  test/core/util/one_corpus_entry_fuzzer.cc
-)
-
-target_include_directories(api_fuzzer_one_entry
-  PRIVATE
-    ${CMAKE_CURRENT_SOURCE_DIR}
-    ${CMAKE_CURRENT_SOURCE_DIR}/include
-    ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR}
-    ${_gRPC_SSL_INCLUDE_DIR}
-    ${_gRPC_UPB_GENERATED_DIR}
-    ${_gRPC_UPB_GRPC_GENERATED_DIR}
-    ${_gRPC_UPB_INCLUDE_DIR}
-    ${_gRPC_ZLIB_INCLUDE_DIR}
-)
-
-target_link_libraries(api_fuzzer_one_entry
-  ${_gRPC_ALLTARGETS_LIBRARIES}
-  grpc_test_util
-  grpc
-  gpr
-)
-
-
 endif()
 if(gRPC_BUILD_TESTS)
 

Makefile

@@ -410,7 +410,7 @@ LDFLAGS += -pthread
 endif
 
 ifeq ($(SYSTEM),MINGW32)
-LIBS = m pthread ws2_32
+LIBS = m pthread ws2_32 dbghelp
 LDFLAGS += -pthread
 endif
 
@@ -1010,7 +1010,6 @@ algorithm_test: $(BINDIR)/$(CONFIG)/algorithm_test
 alloc_test: $(BINDIR)/$(CONFIG)/alloc_test
 alpn_test: $(BINDIR)/$(CONFIG)/alpn_test
 alts_credentials_fuzzer: $(BINDIR)/$(CONFIG)/alts_credentials_fuzzer
-api_fuzzer: $(BINDIR)/$(CONFIG)/api_fuzzer
 arena_test: $(BINDIR)/$(CONFIG)/arena_test
 avl_test: $(BINDIR)/$(CONFIG)/avl_test
 bad_server_response_test: $(BINDIR)/$(CONFIG)/bad_server_response_test
@@ -1374,7 +1373,6 @@ address_sorting_test_unsecure: $(BINDIR)/$(CONFIG)/address_sorting_test_unsecure
 address_sorting_test: $(BINDIR)/$(CONFIG)/address_sorting_test
 cancel_ares_query_test: $(BINDIR)/$(CONFIG)/cancel_ares_query_test
 alts_credentials_fuzzer_one_entry: $(BINDIR)/$(CONFIG)/alts_credentials_fuzzer_one_entry
-api_fuzzer_one_entry: $(BINDIR)/$(CONFIG)/api_fuzzer_one_entry
 client_fuzzer_one_entry: $(BINDIR)/$(CONFIG)/client_fuzzer_one_entry
 hpack_parser_fuzzer_test_one_entry: $(BINDIR)/$(CONFIG)/hpack_parser_fuzzer_test_one_entry
 http_request_fuzzer_test_one_entry: $(BINDIR)/$(CONFIG)/http_request_fuzzer_test_one_entry
@@ -1614,7 +1612,6 @@ buildtests_c: privatelibs_c \
   $(BINDIR)/$(CONFIG)/h2_sockpair_1byte_nosec_test \
   $(BINDIR)/$(CONFIG)/h2_uds_nosec_test \
   $(BINDIR)/$(CONFIG)/alts_credentials_fuzzer_one_entry \
-  $(BINDIR)/$(CONFIG)/api_fuzzer_one_entry \
   $(BINDIR)/$(CONFIG)/client_fuzzer_one_entry \
   $(BINDIR)/$(CONFIG)/hpack_parser_fuzzer_test_one_entry \
   $(BINDIR)/$(CONFIG)/http_request_fuzzer_test_one_entry \
@@ -3762,6 +3759,7 @@ PUBLIC_HEADERS_C += \
     include/grpc/support/port_platform.h \
     include/grpc/support/string_util.h \
     include/grpc/support/sync.h \
+    include/grpc/support/sync_abseil.h \
     include/grpc/support/sync_custom.h \
     include/grpc/support/sync_generic.h \
     include/grpc/support/sync_posix.h \
@@ -3778,6 +3776,7 @@ PUBLIC_HEADERS_C += \
     include/grpc/impl/codegen/log.h \
     include/grpc/impl/codegen/port_platform.h \
     include/grpc/impl/codegen/sync.h \
+    include/grpc/impl/codegen/sync_abseil.h \
     include/grpc/impl/codegen/sync_custom.h \
     include/grpc/impl/codegen/sync_generic.h \
     include/grpc/impl/codegen/sync_posix.h \
@@ -3885,6 +3884,7 @@ LIBGPR_SRC = \
     src/core/lib/gpr/string_util_windows.cc \
     src/core/lib/gpr/string_windows.cc \
     src/core/lib/gpr/sync.cc \
+    src/core/lib/gpr/sync_abseil.cc \
     src/core/lib/gpr/sync_posix.cc \
     src/core/lib/gpr/sync_windows.cc \
     src/core/lib/gpr/time.cc \
@@ -3918,6 +3918,7 @@ PUBLIC_HEADERS_C += \
     include/grpc/support/port_platform.h \
     include/grpc/support/string_util.h \
     include/grpc/support/sync.h \
+    include/grpc/support/sync_abseil.h \
     include/grpc/support/sync_custom.h \
     include/grpc/support/sync_generic.h \
     include/grpc/support/sync_posix.h \
@@ -3934,6 +3935,7 @@ PUBLIC_HEADERS_C += \
     include/grpc/impl/codegen/log.h \
     include/grpc/impl/codegen/port_platform.h \
     include/grpc/impl/codegen/sync.h \
+    include/grpc/impl/codegen/sync_abseil.h \
     include/grpc/impl/codegen/sync_custom.h \
     include/grpc/impl/codegen/sync_generic.h \
     include/grpc/impl/codegen/sync_posix.h \
@@ -3942,7 +3944,7 @@ PUBLIC_HEADERS_C += \
 LIBGPR_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(LIBGPR_SRC))))
 
 
-$(LIBDIR)/$(CONFIG)/libgpr.a: $(ZLIB_DEP) $(CARES_DEP) $(ADDRESS_SORTING_DEP) $(UPB_DEP) $(LIBGRPC_ABSEIL_DEP)  $(LIBGPR_OBJS) 
+$(LIBDIR)/$(CONFIG)/libgpr.a: $(ZLIB_DEP) $(CARES_DEP) $(ADDRESS_SORTING_DEP) $(UPB_DEP) $(GRPC_ABSEIL_DEP)  $(LIBGPR_OBJS) 
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libgpr.a
@@ -4379,6 +4381,7 @@ PUBLIC_HEADERS_C += \
     include/grpc/impl/codegen/log.h \
     include/grpc/impl/codegen/port_platform.h \
     include/grpc/impl/codegen/sync.h \
+    include/grpc/impl/codegen/sync_abseil.h \
     include/grpc/impl/codegen/sync_custom.h \
     include/grpc/impl/codegen/sync_generic.h \
     include/grpc/impl/codegen/sync_posix.h \
@@ -4781,6 +4784,7 @@ PUBLIC_HEADERS_C += \
     include/grpc/impl/codegen/log.h \
     include/grpc/impl/codegen/port_platform.h \
     include/grpc/impl/codegen/sync.h \
+    include/grpc/impl/codegen/sync_abseil.h \
     include/grpc/impl/codegen/sync_custom.h \
     include/grpc/impl/codegen/sync_generic.h \
     include/grpc/impl/codegen/sync_posix.h \
@@ -5111,6 +5115,7 @@ PUBLIC_HEADERS_C += \
     include/grpc/support/port_platform.h \
     include/grpc/support/string_util.h \
     include/grpc/support/sync.h \
+    include/grpc/support/sync_abseil.h \
     include/grpc/support/sync_custom.h \
     include/grpc/support/sync_generic.h \
     include/grpc/support/sync_posix.h \
@@ -5127,6 +5132,7 @@ PUBLIC_HEADERS_C += \
     include/grpc/impl/codegen/log.h \
     include/grpc/impl/codegen/port_platform.h \
     include/grpc/impl/codegen/sync.h \
+    include/grpc/impl/codegen/sync_abseil.h \
     include/grpc/impl/codegen/sync_custom.h \
     include/grpc/impl/codegen/sync_generic.h \
     include/grpc/impl/codegen/sync_posix.h \
@@ -5439,6 +5445,7 @@ PUBLIC_HEADERS_C += \
     include/grpc/support/port_platform.h \
     include/grpc/support/string_util.h \
     include/grpc/support/sync.h \
+    include/grpc/support/sync_abseil.h \
     include/grpc/support/sync_custom.h \
     include/grpc/support/sync_generic.h \
     include/grpc/support/sync_posix.h \
@@ -5455,6 +5462,7 @@ PUBLIC_HEADERS_C += \
     include/grpc/impl/codegen/log.h \
     include/grpc/impl/codegen/port_platform.h \
     include/grpc/impl/codegen/sync.h \
+    include/grpc/impl/codegen/sync_abseil.h \
     include/grpc/impl/codegen/sync_custom.h \
     include/grpc/impl/codegen/sync_generic.h \
     include/grpc/impl/codegen/sync_posix.h \
@@ -5471,7 +5479,7 @@ PUBLIC_HEADERS_C += \
 LIBGRPC_TEST_UTIL_UNSECURE_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(LIBGRPC_TEST_UTIL_UNSECURE_SRC))))
 
 
-$(LIBDIR)/$(CONFIG)/libgrpc_test_util_unsecure.a: $(ZLIB_DEP) $(CARES_DEP) $(ADDRESS_SORTING_DEP) $(UPB_DEP) $(LIBGRPC_ABSEIL_DEP)  $(LIBGRPC_TEST_UTIL_UNSECURE_OBJS) 
+$(LIBDIR)/$(CONFIG)/libgrpc_test_util_unsecure.a: $(ZLIB_DEP) $(CARES_DEP) $(ADDRESS_SORTING_DEP) $(UPB_DEP) $(GRPC_ABSEIL_DEP)  $(LIBGRPC_TEST_UTIL_UNSECURE_OBJS) 
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libgrpc_test_util_unsecure.a
@@ -5816,6 +5824,7 @@ PUBLIC_HEADERS_C += \
     include/grpc/impl/codegen/log.h \
     include/grpc/impl/codegen/port_platform.h \
     include/grpc/impl/codegen/sync.h \
+    include/grpc/impl/codegen/sync_abseil.h \
     include/grpc/impl/codegen/sync_custom.h \
     include/grpc/impl/codegen/sync_generic.h \
     include/grpc/impl/codegen/sync_posix.h \
@@ -5837,7 +5846,7 @@ PUBLIC_HEADERS_C += \
 LIBGRPC_UNSECURE_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(LIBGRPC_UNSECURE_SRC))))
 
 
-$(LIBDIR)/$(CONFIG)/libgrpc_unsecure.a: $(ZLIB_DEP) $(CARES_DEP) $(ADDRESS_SORTING_DEP) $(UPB_DEP) $(LIBGRPC_ABSEIL_DEP)  $(LIBGRPC_UNSECURE_OBJS)  $(LIBGPR_OBJS)  $(LIBGRPC_ABSEIL_OBJS)  $(ZLIB_MERGE_OBJS)  $(CARES_MERGE_OBJS)  $(ADDRESS_SORTING_MERGE_OBJS)  $(UPB_MERGE_OBJS) 
+$(LIBDIR)/$(CONFIG)/libgrpc_unsecure.a: $(ZLIB_DEP) $(CARES_DEP) $(ADDRESS_SORTING_DEP) $(UPB_DEP) $(GRPC_ABSEIL_DEP)  $(LIBGRPC_UNSECURE_OBJS)  $(LIBGPR_OBJS)  $(LIBGRPC_ABSEIL_OBJS)  $(ZLIB_MERGE_OBJS)  $(CARES_MERGE_OBJS)  $(ADDRESS_SORTING_MERGE_OBJS)  $(UPB_MERGE_OBJS) 
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libgrpc_unsecure.a
@@ -6233,6 +6242,7 @@ PUBLIC_HEADERS_CXX += \
     include/grpc/support/port_platform.h \
     include/grpc/support/string_util.h \
     include/grpc/support/sync.h \
+    include/grpc/support/sync_abseil.h \
     include/grpc/support/sync_custom.h \
     include/grpc/support/sync_generic.h \
     include/grpc/support/sync_posix.h \
@@ -6249,6 +6259,7 @@ PUBLIC_HEADERS_CXX += \
     include/grpc/impl/codegen/log.h \
     include/grpc/impl/codegen/port_platform.h \
     include/grpc/impl/codegen/sync.h \
+    include/grpc/impl/codegen/sync_abseil.h \
     include/grpc/impl/codegen/sync_custom.h \
     include/grpc/impl/codegen/sync_generic.h \
     include/grpc/impl/codegen/sync_posix.h \
@@ -6922,6 +6933,7 @@ PUBLIC_HEADERS_CXX += \
     include/grpc/impl/codegen/log.h \
     include/grpc/impl/codegen/port_platform.h \
     include/grpc/impl/codegen/sync.h \
+    include/grpc/impl/codegen/sync_abseil.h \
     include/grpc/impl/codegen/sync_custom.h \
     include/grpc/impl/codegen/sync_generic.h \
     include/grpc/impl/codegen/sync_posix.h \
@@ -7102,6 +7114,7 @@ PUBLIC_HEADERS_CXX += \
     include/grpc/impl/codegen/log.h \
     include/grpc/impl/codegen/port_platform.h \
     include/grpc/impl/codegen/sync.h \
+    include/grpc/impl/codegen/sync_abseil.h \
     include/grpc/impl/codegen/sync_custom.h \
     include/grpc/impl/codegen/sync_generic.h \
     include/grpc/impl/codegen/sync_posix.h \
@@ -7340,6 +7353,7 @@ PUBLIC_HEADERS_CXX += \
     include/grpc/support/port_platform.h \
     include/grpc/support/string_util.h \
     include/grpc/support/sync.h \
+    include/grpc/support/sync_abseil.h \
     include/grpc/support/sync_custom.h \
     include/grpc/support/sync_generic.h \
     include/grpc/support/sync_posix.h \
@@ -7356,6 +7370,7 @@ PUBLIC_HEADERS_CXX += \
     include/grpc/impl/codegen/log.h \
     include/grpc/impl/codegen/port_platform.h \
     include/grpc/impl/codegen/sync.h \
+    include/grpc/impl/codegen/sync_abseil.h \
     include/grpc/impl/codegen/sync_custom.h \
     include/grpc/impl/codegen/sync_generic.h \
     include/grpc/impl/codegen/sync_posix.h \
@@ -7478,7 +7493,7 @@ $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)grpc++_unsecure$(SHARED_VERSION_CPP).$(SHARE
 
 else
 
-$(LIBDIR)/$(CONFIG)/libgrpc++_unsecure.a: $(ZLIB_DEP) $(CARES_DEP) $(ADDRESS_SORTING_DEP) $(UPB_DEP) $(LIBGRPC_ABSEIL_DEP)  $(PROTOBUF_DEP) $(LIBGRPC++_UNSECURE_OBJS)  $(LIBGPR_OBJS)  $(LIBGRPC_ABSEIL_OBJS)  $(ZLIB_MERGE_OBJS)  $(CARES_MERGE_OBJS)  $(ADDRESS_SORTING_MERGE_OBJS)  $(UPB_MERGE_OBJS) 
+$(LIBDIR)/$(CONFIG)/libgrpc++_unsecure.a: $(ZLIB_DEP) $(CARES_DEP) $(ADDRESS_SORTING_DEP) $(UPB_DEP) $(GRPC_ABSEIL_DEP)  $(PROTOBUF_DEP) $(LIBGRPC++_UNSECURE_OBJS)  $(LIBGPR_OBJS)  $(LIBGRPC_ABSEIL_OBJS)  $(ZLIB_MERGE_OBJS)  $(CARES_MERGE_OBJS)  $(ADDRESS_SORTING_MERGE_OBJS)  $(UPB_MERGE_OBJS) 
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libgrpc++_unsecure.a
@@ -7650,7 +7665,7 @@ $(LIBDIR)/$(CONFIG)/libgrpc_plugin_support.a: protobuf_dep_error
 
 else
 
-$(LIBDIR)/$(CONFIG)/libgrpc_plugin_support.a: $(ZLIB_DEP) $(CARES_DEP) $(ADDRESS_SORTING_DEP) $(UPB_DEP) $(LIBGRPC_ABSEIL_DEP)  $(PROTOBUF_DEP) $(LIBGRPC_PLUGIN_SUPPORT_OBJS) 
+$(LIBDIR)/$(CONFIG)/libgrpc_plugin_support.a: $(ZLIB_DEP) $(CARES_DEP) $(ADDRESS_SORTING_DEP) $(UPB_DEP) $(GRPC_ABSEIL_DEP)  $(PROTOBUF_DEP) $(LIBGRPC_PLUGIN_SUPPORT_OBJS) 
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libgrpc_plugin_support.a
@@ -8465,7 +8480,7 @@ $(LIBBORINGSSL_OBJS): CPPFLAGS += -Ithird_party/boringssl-with-bazel/src/include
 $(LIBBORINGSSL_OBJS): CXXFLAGS += -fno-exceptions
 $(LIBBORINGSSL_OBJS): CFLAGS += -g
 
-$(LIBDIR)/$(CONFIG)/libboringssl.a: $(ZLIB_DEP) $(CARES_DEP) $(ADDRESS_SORTING_DEP) $(UPB_DEP) $(LIBGRPC_ABSEIL_DEP)  $(LIBBORINGSSL_OBJS) 
+$(LIBDIR)/$(CONFIG)/libboringssl.a: $(ZLIB_DEP) $(CARES_DEP) $(ADDRESS_SORTING_DEP) $(UPB_DEP) $(GRPC_ABSEIL_DEP)  $(LIBBORINGSSL_OBJS) 
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libboringssl.a
@@ -8505,7 +8520,7 @@ $(LIBDIR)/$(CONFIG)/libboringssl_test_util.a: protobuf_dep_error
 
 else
 
-$(LIBDIR)/$(CONFIG)/libboringssl_test_util.a: $(ZLIB_DEP) $(CARES_DEP) $(ADDRESS_SORTING_DEP) $(UPB_DEP) $(LIBGRPC_ABSEIL_DEP)  $(PROTOBUF_DEP) $(LIBBORINGSSL_TEST_UTIL_OBJS) 
+$(LIBDIR)/$(CONFIG)/libboringssl_test_util.a: $(ZLIB_DEP) $(CARES_DEP) $(ADDRESS_SORTING_DEP) $(UPB_DEP) $(GRPC_ABSEIL_DEP)  $(PROTOBUF_DEP) $(LIBBORINGSSL_TEST_UTIL_OBJS) 
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libboringssl_test_util.a
@@ -8560,7 +8575,7 @@ $(LIBDIR)/$(CONFIG)/libbenchmark.a: protobuf_dep_error
 
 else
 
-$(LIBDIR)/$(CONFIG)/libbenchmark.a: $(ZLIB_DEP) $(CARES_DEP) $(ADDRESS_SORTING_DEP) $(UPB_DEP) $(LIBGRPC_ABSEIL_DEP)  $(PROTOBUF_DEP) $(LIBBENCHMARK_OBJS) 
+$(LIBDIR)/$(CONFIG)/libbenchmark.a: $(ZLIB_DEP) $(CARES_DEP) $(ADDRESS_SORTING_DEP) $(UPB_DEP) $(GRPC_ABSEIL_DEP)  $(PROTOBUF_DEP) $(LIBBENCHMARK_OBJS) 
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libbenchmark.a
@@ -8592,18 +8607,7 @@ PUBLIC_HEADERS_C += \
 LIBUPB_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(LIBUPB_SRC))))
 
 
-ifeq ($(NO_SECURE),true)
-
-# You can't build secure libraries if you don't have OpenSSL.
-
-$(LIBDIR)/$(CONFIG)/libupb.a: openssl_dep_error
-
-$(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)upb$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE): openssl_dep_error
-
-else
-
-
-$(LIBDIR)/$(CONFIG)/libupb.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(CARES_DEP) $(ADDRESS_SORTING_DEP) $(UPB_DEP) $(GRPC_ABSEIL_DEP)  $(LIBUPB_OBJS) 
+$(LIBDIR)/$(CONFIG)/libupb.a:  $(LIBUPB_OBJS) 
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libupb.a
@@ -8615,12 +8619,12 @@ endif
 
 
 ifeq ($(SYSTEM),MINGW32)
-$(LIBDIR)/$(CONFIG)/upb$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE): $(LIBUPB_OBJS)  $(ZLIB_DEP) $(CARES_DEP) $(ADDRESS_SORTING_DEP) $(UPB_DEP) $(GRPC_ABSEIL_DEP) $(OPENSSL_DEP)
+$(LIBDIR)/$(CONFIG)/upb$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE): $(LIBUPB_OBJS)  $(ZLIB_DEP) $(CARES_DEP) $(ADDRESS_SORTING_DEP) $(UPB_DEP) $(GRPC_ABSEIL_DEP)
 	$(E) "[LD]      Linking $@"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,--output-def=$(LIBDIR)/$(CONFIG)/upb$(SHARED_VERSION_CORE).def -Wl,--out-implib=$(LIBDIR)/$(CONFIG)/libupb$(SHARED_VERSION_CORE)-dll.a -o $(LIBDIR)/$(CONFIG)/upb$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBUPB_OBJS) $(ZLIB_MERGE_LIBS) $(CARES_MERGE_LIBS) $(ADDRESS_SORTING_MERGE_LIBS) $(UPB_MERGE_LIBS) $(GRPC_ABSEIL_MERGE_LIBS) $(LDLIBS)
 else
-$(LIBDIR)/$(CONFIG)/libupb$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE): $(LIBUPB_OBJS)  $(ZLIB_DEP) $(CARES_DEP) $(ADDRESS_SORTING_DEP) $(UPB_DEP) $(GRPC_ABSEIL_DEP) $(OPENSSL_DEP)
+$(LIBDIR)/$(CONFIG)/libupb$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE): $(LIBUPB_OBJS)  $(ZLIB_DEP) $(CARES_DEP) $(ADDRESS_SORTING_DEP) $(UPB_DEP) $(GRPC_ABSEIL_DEP)
 	$(E) "[LD]      Linking $@"
 	$(Q) mkdir -p `dirname $@`
 ifeq ($(SYSTEM),Darwin)
@@ -8632,13 +8636,9 @@ else
 endif
 endif
 
-endif
-
-ifneq ($(NO_SECURE),true)
 ifneq ($(NO_DEPS),true)
 -include $(LIBUPB_OBJS:.o=.dep)
 endif
-endif
 
 
 LIBZ_SRC = \
@@ -9049,7 +9049,7 @@ PUBLIC_HEADERS_C += \
 LIBEND2END_NOSEC_TESTS_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(LIBEND2END_NOSEC_TESTS_SRC))))
 
 
-$(LIBDIR)/$(CONFIG)/libend2end_nosec_tests.a: $(ZLIB_DEP) $(CARES_DEP) $(ADDRESS_SORTING_DEP) $(UPB_DEP) $(LIBGRPC_ABSEIL_DEP)  $(LIBEND2END_NOSEC_TESTS_OBJS) 
+$(LIBDIR)/$(CONFIG)/libend2end_nosec_tests.a: $(ZLIB_DEP) $(CARES_DEP) $(ADDRESS_SORTING_DEP) $(UPB_DEP) $(GRPC_ABSEIL_DEP)  $(LIBEND2END_NOSEC_TESTS_OBJS) 
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libend2end_nosec_tests.a
@@ -9103,17 +9103,7 @@ LIBGRPC_ABSEIL_SRC = \
 LIBGRPC_ABSEIL_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(LIBGRPC_ABSEIL_SRC))))
 
 
-ifeq ($(NO_SECURE),true)
-
-# You can't build secure libraries if you don't have OpenSSL.
-
-$(LIBDIR)/$(CONFIG)/libgrpc_abseil.a: openssl_dep_error
-
-
-else
-
-
-$(LIBDIR)/$(CONFIG)/libgrpc_abseil.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(CARES_DEP) $(ADDRESS_SORTING_DEP) $(UPB_DEP) $(GRPC_ABSEIL_DEP)  $(LIBGRPC_ABSEIL_OBJS) 
+$(LIBDIR)/$(CONFIG)/libgrpc_abseil.a:  $(LIBGRPC_ABSEIL_OBJS) 
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libgrpc_abseil.a
@@ -9125,13 +9115,9 @@ endif
 
 
 
-endif
-
-ifneq ($(NO_SECURE),true)
 ifneq ($(NO_DEPS),true)
 -include $(LIBGRPC_ABSEIL_OBJS:.o=.dep)
 endif
-endif
 
 
 # All of the test targets, and protoc plugins
@@ -9265,38 +9251,6 @@ endif
 endif
 
 
-API_FUZZER_SRC = \
-    test/core/end2end/fuzzers/api_fuzzer.cc \
-
-API_FUZZER_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(API_FUZZER_SRC))))
-ifeq ($(NO_SECURE),true)
-
-# You can't build secure targets if you don't have OpenSSL.
-
-$(BINDIR)/$(CONFIG)/api_fuzzer: openssl_dep_error
-
-else
-
-
-
-$(BINDIR)/$(CONFIG)/api_fuzzer: $(API_FUZZER_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a
-	$(E) "[LD]      Linking $@"
-	$(Q) mkdir -p `dirname $@`
-	$(Q) $(LDXX) $(LDFLAGS) $(API_FUZZER_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBS) $(LDLIBS_SECURE) -lFuzzer -o $(BINDIR)/$(CONFIG)/api_fuzzer
-
-endif
-
-$(OBJDIR)/$(CONFIG)/test/core/end2end/fuzzers/api_fuzzer.o:  $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a
-
-deps_api_fuzzer: $(API_FUZZER_OBJS:.o=.dep)
-
-ifneq ($(NO_SECURE),true)
-ifneq ($(NO_DEPS),true)
--include $(API_FUZZER_OBJS:.o=.dep)
-endif
-endif
-
-
 ARENA_TEST_SRC = \
     test/core/gpr/arena_test.cc \
 
@@ -22985,41 +22939,6 @@ endif
 endif
 
 
-API_FUZZER_ONE_ENTRY_SRC = \
-    test/core/end2end/fuzzers/api_fuzzer.cc \
-    test/core/util/one_corpus_entry_fuzzer.cc \
-
-API_FUZZER_ONE_ENTRY_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(API_FUZZER_ONE_ENTRY_SRC))))
-ifeq ($(NO_SECURE),true)
-
-# You can't build secure targets if you don't have OpenSSL.
-
-$(BINDIR)/$(CONFIG)/api_fuzzer_one_entry: openssl_dep_error
-
-else
-
-
-
-$(BINDIR)/$(CONFIG)/api_fuzzer_one_entry: $(API_FUZZER_ONE_ENTRY_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a
-	$(E) "[LD]      Linking $@"
-	$(Q) mkdir -p `dirname $@`
-	$(Q) $(LDXX) $(LDFLAGS) $(API_FUZZER_ONE_ENTRY_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBS) $(LDLIBS_SECURE) -o $(BINDIR)/$(CONFIG)/api_fuzzer_one_entry
-
-endif
-
-$(OBJDIR)/$(CONFIG)/test/core/end2end/fuzzers/api_fuzzer.o:  $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a
-
-$(OBJDIR)/$(CONFIG)/test/core/util/one_corpus_entry_fuzzer.o:  $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a
-
-deps_api_fuzzer_one_entry: $(API_FUZZER_ONE_ENTRY_OBJS:.o=.dep)
-
-ifneq ($(NO_SECURE),true)
-ifneq ($(NO_DEPS),true)
--include $(API_FUZZER_ONE_ENTRY_OBJS:.o=.dep)
-endif
-endif
-
-
 CLIENT_FUZZER_ONE_ENTRY_SRC = \
     test/core/end2end/fuzzers/client_fuzzer.cc \
     test/core/util/one_corpus_entry_fuzzer.cc \
@@ -23600,40 +23519,6 @@ test/cpp/util/string_ref_helper.cc: $(OPENSSL_DEP)
 test/cpp/util/subprocess.cc: $(OPENSSL_DEP)
 test/cpp/util/test_config_cc.cc: $(OPENSSL_DEP)
 test/cpp/util/test_credentials_provider.cc: $(OPENSSL_DEP)
-third_party/abseil-cpp/absl/base/dynamic_annotations.cc: $(OPENSSL_DEP)
-third_party/abseil-cpp/absl/base/internal/cycleclock.cc: $(OPENSSL_DEP)
-third_party/abseil-cpp/absl/base/internal/raw_logging.cc: $(OPENSSL_DEP)
-third_party/abseil-cpp/absl/base/internal/spinlock.cc: $(OPENSSL_DEP)
-third_party/abseil-cpp/absl/base/internal/spinlock_wait.cc: $(OPENSSL_DEP)
-third_party/abseil-cpp/absl/base/internal/sysinfo.cc: $(OPENSSL_DEP)
-third_party/abseil-cpp/absl/base/internal/thread_identity.cc: $(OPENSSL_DEP)
-third_party/abseil-cpp/absl/base/internal/throw_delegate.cc: $(OPENSSL_DEP)
-third_party/abseil-cpp/absl/base/internal/unscaledcycleclock.cc: $(OPENSSL_DEP)
-third_party/abseil-cpp/absl/base/log_severity.cc: $(OPENSSL_DEP)
-third_party/abseil-cpp/absl/numeric/int128.cc: $(OPENSSL_DEP)
-third_party/abseil-cpp/absl/strings/ascii.cc: $(OPENSSL_DEP)
-third_party/abseil-cpp/absl/strings/charconv.cc: $(OPENSSL_DEP)
-third_party/abseil-cpp/absl/strings/escaping.cc: $(OPENSSL_DEP)
-third_party/abseil-cpp/absl/strings/internal/charconv_bigint.cc: $(OPENSSL_DEP)
-third_party/abseil-cpp/absl/strings/internal/charconv_parse.cc: $(OPENSSL_DEP)
-third_party/abseil-cpp/absl/strings/internal/escaping.cc: $(OPENSSL_DEP)
-third_party/abseil-cpp/absl/strings/internal/memutil.cc: $(OPENSSL_DEP)
-third_party/abseil-cpp/absl/strings/internal/ostringstream.cc: $(OPENSSL_DEP)
-third_party/abseil-cpp/absl/strings/internal/utf8.cc: $(OPENSSL_DEP)
-third_party/abseil-cpp/absl/strings/match.cc: $(OPENSSL_DEP)
-third_party/abseil-cpp/absl/strings/numbers.cc: $(OPENSSL_DEP)
-third_party/abseil-cpp/absl/strings/str_cat.cc: $(OPENSSL_DEP)
-third_party/abseil-cpp/absl/strings/str_replace.cc: $(OPENSSL_DEP)
-third_party/abseil-cpp/absl/strings/str_split.cc: $(OPENSSL_DEP)
-third_party/abseil-cpp/absl/strings/string_view.cc: $(OPENSSL_DEP)
-third_party/abseil-cpp/absl/strings/substitute.cc: $(OPENSSL_DEP)
-third_party/abseil-cpp/absl/types/bad_optional_access.cc: $(OPENSSL_DEP)
-third_party/upb/upb/decode.c: $(OPENSSL_DEP)
-third_party/upb/upb/encode.c: $(OPENSSL_DEP)
-third_party/upb/upb/msg.c: $(OPENSSL_DEP)
-third_party/upb/upb/port.c: $(OPENSSL_DEP)
-third_party/upb/upb/table.c: $(OPENSSL_DEP)
-third_party/upb/upb/upb.c: $(OPENSSL_DEP)
 endif
 
 .PHONY: all strip tools dep_error openssl_dep_error openssl_dep_message git_update stop buildtests buildtests_c buildtests_cxx test test_c test_cxx install install_c install_cxx install-headers install-headers_c install-headers_cxx install-shared install-shared_c install-shared_cxx install-static install-static_c install-static_cxx strip strip-shared strip-static strip_c strip-shared_c strip-static_c strip_cxx strip-shared_cxx strip-static_cxx dep_c dep_cxx bins_dep_c bins_dep_cxx clean

README.md

@@ -53,7 +53,7 @@ Sometimes things go wrong. Please check out the [Troubleshooting guide](TROUBLES
 
 # Performance 
 
-See the [Performance dashboard](http://performance-dot-grpc-testing.appspot.com/explore?dashboard=5636470266134528) for performance numbers of the latest released version.
+See the [Performance dashboard](https://performance-dot-grpc-testing.appspot.com/explore?dashboard=5652536396611584) for performance numbers of master branch daily builds.
 
 # Concepts
 

build.yaml

@@ -248,6 +248,7 @@ filegroups:
   - src/core/lib/gpr/string_util_windows.cc
   - src/core/lib/gpr/string_windows.cc
   - src/core/lib/gpr/sync.cc
+  - src/core/lib/gpr/sync_abseil.cc
   - src/core/lib/gpr/sync_posix.cc
   - src/core/lib/gpr/sync_windows.cc
   - src/core/lib/gpr/time.cc
@@ -287,6 +288,7 @@ filegroups:
   - include/grpc/support/port_platform.h
   - include/grpc/support/string_util.h
   - include/grpc/support/sync.h
+  - include/grpc/support/sync_abseil.h
   - include/grpc/support/sync_custom.h
   - include/grpc/support/sync_generic.h
   - include/grpc/support/sync_posix.h
@@ -337,6 +339,7 @@ filegroups:
   - include/grpc/impl/codegen/log.h
   - include/grpc/impl/codegen/port_platform.h
   - include/grpc/impl/codegen/sync.h
+  - include/grpc/impl/codegen/sync_abseil.h
   - include/grpc/impl/codegen/sync_custom.h
   - include/grpc/impl/codegen/sync_generic.h
   - include/grpc/impl/codegen/sync_posix.h
@@ -2342,19 +2345,6 @@ targets:
   corpus_dirs:
   - test/core/security/corpus/alts_credentials_corpus
   maxlen: 2048
-- name: api_fuzzer
-  build: fuzzer
-  language: c
-  src:
-  - test/core/end2end/fuzzers/api_fuzzer.cc
-  deps:
-  - grpc_test_util
-  - grpc
-  - gpr
-  corpus_dirs:
-  - test/core/end2end/fuzzers/api_fuzzer_corpus
-  dict: test/core/end2end/fuzzers/api_fuzzer.dictionary
-  maxlen: 2048
 - name: arena_test
   cpu_cost: 10
   build: test

config.m4

@@ -230,6 +230,7 @@ if test "$PHP_GRPC" != "no"; then
     src/core/lib/gpr/string_util_windows.cc \
     src/core/lib/gpr/string_windows.cc \
     src/core/lib/gpr/sync.cc \
+    src/core/lib/gpr/sync_abseil.cc \
     src/core/lib/gpr/sync_posix.cc \
     src/core/lib/gpr/sync_windows.cc \
     src/core/lib/gpr/time.cc \

config.w32

@@ -199,6 +199,7 @@ if (PHP_GRPC != "no") {
     "src\\core\\lib\\gpr\\string_util_windows.cc " +
     "src\\core\\lib\\gpr\\string_windows.cc " +
     "src\\core\\lib\\gpr\\sync.cc " +
+    "src\\core\\lib\\gpr\\sync_abseil.cc " +
     "src\\core\\lib\\gpr\\sync_posix.cc " +
     "src\\core\\lib\\gpr\\sync_windows.cc " +
     "src\\core\\lib\\gpr\\time.cc " +

doc/python/sphinx/_static/custom.css

@@ -0,0 +1,3 @@
+dl.field-list > dt {
+    word-break: keep-all !important;
+}

doc/python/sphinx/conf.py

@@ -16,8 +16,8 @@
 
 import os
 import sys
-PYTHON_FOLDER = os.path.join(os.path.dirname(os.path.realpath(__file__)),
-                             '..', '..', '..', 'src', 'python')
+PYTHON_FOLDER = os.path.join(os.path.dirname(os.path.realpath(__file__)), '..',
+                             '..', '..', 'src', 'python')
 sys.path.insert(0, os.path.join(PYTHON_FOLDER, 'grpcio'))
 sys.path.insert(0, os.path.join(PYTHON_FOLDER, 'grpcio_channelz'))
 sys.path.insert(0, os.path.join(PYTHON_FOLDER, 'grpcio_health_checking'))
@@ -53,6 +53,7 @@ extensions = [
     'sphinx.ext.todo',
     'sphinx.ext.napoleon',
     'sphinx.ext.coverage',
+    'sphinx.ext.autodoc.typehints',
 ]
 
 napoleon_google_docstring = True
@@ -63,15 +64,9 @@ autodoc_default_options = {
     'members': None,
 }
 
-autodoc_mock_imports = [
-    'grpc._cython',
-    'grpc_channelz.v1.channelz_pb2',
-    'grpc_channelz.v1.channelz_pb2_grpc',
-    'grpc_health.v1.health_pb2',
-    'grpc_health.v1.health_pb2_grpc',
-    'grpc_reflection.v1alpha.reflection_pb2',
-    'grpc_reflection.v1alpha.reflection_pb2_grpc',
-]
+autodoc_mock_imports = []
+
+autodoc_typehints = 'description'
 
 # -- HTML Configuration -------------------------------------------------
 
@@ -84,6 +79,7 @@ html_theme_options = {
     'description': grpc_version.VERSION,
     'show_powered_by': False,
 }
+html_static_path = ["_static"]
 
 # -- Options for manual page output ------------------------------------------
 

doc/python/sphinx/grpc_asyncio.rst

@@ -0,0 +1,132 @@
+gRPC AsyncIO API
+================
+
+.. module:: grpc.experimental.aio
+
+Overview
+--------
+
+gRPC AsyncIO API is the **new version** of gRPC Python whose architecture is
+tailored to AsyncIO. Underlying, it utilizes the same C-extension, gRPC C-Core,
+as existing stack, and it replaces all gRPC IO operations with methods provided
+by the AsyncIO library.
+
+This stack currently is under active development. Feel free to offer
+suggestions by opening issues on our GitHub repo `grpc/grpc <https://github.com/grpc/grpc>`_.
+
+The design doc can be found here as `gRFC <https://github.com/grpc/proposal/pull/155>`_.
+
+
+Caveats
+-------
+
+gRPC Async API objects may only be used on the thread on which they were
+created. AsyncIO doesn't provide thread safety for most of its APIs.
+
+
+Module Contents
+---------------
+
+Enable AsyncIO in gRPC
+^^^^^^^^^^^^^^^^^^^^^^
+
+.. function:: init_grpc_aio
+
+    Enable AsyncIO for gRPC Python.
+
+    This function is idempotent and it should be invoked before creation of
+    AsyncIO stack objects. Otherwise, the application might deadlock.
+
+    This function configurates the gRPC C-Core to invoke AsyncIO methods for IO
+    operations (e.g., socket read, write). The configuration applies to the
+    entire process.
+
+    After invoking this function, making blocking function calls in coroutines
+    or in the thread running event loop will block the event loop, potentially
+    starving all RPCs in the process. Refer to the Python language
+    documentation on AsyncIO for more details (`running-blocking-code <https://docs.python.org/3/library/asyncio-dev.html#running-blocking-code>`_).
+
+
+Create Channel
+^^^^^^^^^^^^^^
+
+Channels are the abstraction of clients, where most of networking logic
+happens, for example, managing one or more underlying connections, name
+resolution, load balancing, flow control, etc.. If you are using ProtoBuf,
+Channel objects works best when further encapsulate into stub objects, then the
+application can invoke remote functions as if they are local functions.
+
+.. autofunction:: insecure_channel
+.. autofunction:: secure_channel
+
+
+Channel Object
+^^^^^^^^^^^^^^
+
+.. autoclass:: Channel
+
+
+Create Server
+^^^^^^^^^^^^^
+
+.. autofunction:: server
+
+
+Server Object
+^^^^^^^^^^^^^
+
+.. autoclass:: Server
+
+
+gRPC Exceptions
+^^^^^^^^^^^^^^^
+
+.. autoexception:: BaseError
+.. autoexception:: UsageError
+.. autoexception:: AbortError
+.. autoexception:: InternalError
+.. autoexception:: AioRpcError
+
+
+Shared Context
+^^^^^^^^^^^^^^^^^^^^
+
+.. autoclass:: RpcContext
+
+
+Client-Side Context
+^^^^^^^^^^^^^^^^^^^^^^^
+
+.. autoclass:: Call
+.. autoclass:: UnaryUnaryCall
+.. autoclass:: UnaryStreamCall
+.. autoclass:: StreamUnaryCall
+.. autoclass:: StreamStreamCall
+
+
+Server-Side Context
+^^^^^^^^^^^^^^^^^^^^^^^
+
+.. autoclass:: ServicerContext
+
+
+Client-Side Interceptor
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. autoclass:: ClientCallDetails
+.. autoclass:: InterceptedUnaryUnaryCall
+.. autoclass:: UnaryUnaryClientInterceptor
+
+.. Service-Side Context
+.. ^^^^^^^^^^^^^^^^^^^^
+
+.. .. autoclass:: ServicerContext
+
+
+Multi-Callable Interfaces
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. autoclass:: UnaryUnaryMultiCallable
+.. autoclass:: UnaryStreamMultiCallable()
+.. autoclass:: StreamUnaryMultiCallable()
+.. autoclass:: StreamStreamMultiCallable()

doc/python/sphinx/index.rst

@@ -10,6 +10,7 @@ API Reference
    :caption: Contents:
 
    grpc
+   grpc_asyncio
    grpc_channelz
    grpc_health_checking
    grpc_reflection

doc/xds-test-descriptions.md

@@ -0,0 +1,286 @@
+# xDS (Load-Balancing) Interop Test Case Descriptions
+
+Client and server use [test.proto](../src/proto/grpc/testing/test.proto).
+
+## Server
+
+The code for the xDS test server can be found at:
+[Java](https://github.com/grpc/grpc-java/blob/master/interop-testing/src/main/java/io/grpc/testing/integration/XdsTestServer.java) (other language implementations are in progress).
+
+Server should accept these arguments:
+
+*   --port=PORT
+    *   The port the server will run on.
+
+## Client
+
+The base behavior of the xDS test client is to send a constant QPS of unary
+messages and record the remote-peer distribution of the responses. Further, the
+client must expose an implementation of the `LoadBalancerStatsService` gRPC
+service to allow the test driver to validate the load balancing behavior for a
+particular test case (see below for more details).
+
+The code for the xDS test client can be at:
+[Java](https://github.com/grpc/grpc-java/blob/master/interop-testing/src/main/java/io/grpc/testing/integration/XdsTestClient.java) (other language implementations are in progress).
+
+Clients should accept these arguments:
+
+*   --num_channels=CHANNELS
+    *   The number of channels to create to the server.
+*   --qps=QPS
+    *   The QPS per channel.
+*   --server=HOSTNAME:PORT
+    *   The server host to connect to. For example, "localhost:8080"
+*   --stats_port=PORT
+    *   The port for to expose the client's `LoadBalancerStatsService`
+        implementation.
+
+## Test Driver
+
+Note that, unlike our other interop tests, neither the client nor the server has
+any notion of which of the following test scenarios is under test. Instead, a
+separate test driver is responsible for configuring the load balancer and the
+server backends, running the client, and then querying the client's
+`LoadBalancerStatsService` to validate load balancer behavior for each of the
+tests described below.
+
+## LoadBalancerStatsService
+
+The service is defined as:
+
+```
+message LoadBalancerStatsRequest {
+  // Request stats for the next num_rpcs sent by client.
+  int32 num_rpcs = 1;
+  // If num_rpcs have not completed within timeout_sec, return partial results.
+  int32 timeout_sec = 2;
+}
+
+message LoadBalancerStatsResponse {
+  // The number of completed RPCs for each peer.
+  map<string, int32> rpcs_by_peer = 1;
+  // The number of RPCs that failed to record a remote peer.
+  int32 num_failures = 2;
+}
+
+service LoadBalancerStatsService {
+  // Gets the backend distribution for RPCs sent by a test client.
+  rpc GetClientStats(LoadBalancerStatsRequest)
+      returns (LoadBalancerStatsResponse) {}
+}
+```
+
+Note that the `LoadBalancerStatsResponse` contains the remote peer distribution
+of the next `num_rpcs` *sent* by the client after receiving the
+`LoadBalancerStatsRequest`. It is important that the remote peer distribution be
+recorded for a block of consecutive outgoing RPCs, to validate the intended
+distribution from the load balancer, rather than just looking at the next
+`num_rpcs` responses received from backends, as different backends may respond
+at different rates.
+
+## Test Cases
+
+### ping_pong
+
+This test verifies that every backend receives traffic.
+
+Client parameters:
+
+1.  --num_channels=1
+1.  --qps=10
+
+Load balancer configuration:
+
+1.  4 backends are created in a single managed instance group (MIG).
+
+Test driver asserts:
+
+1.  All backends receive at least one RPC
+
+### round_robin
+
+This test verifies that RPCs are evenly routed according to an unweighted round
+robin policy.
+
+Client parameters:
+
+1.  --num_channels=1
+1.  --qps=10
+
+Load balancer configuration:
+
+1.  4 backends are created in a single MIG.
+
+Test driver asserts that:
+
+1.  Once all backends receive at least one RPC, the following 100 RPCs are
+    evenly distributed across the 4 backends.
+
+### backends_restart
+
+This test verifies that the load balancer will resume sending traffic to a set
+of backends that is stopped and then resumed.
+
+Client parameters:
+
+1.  --num_channels=1
+1.  --qps=10
+
+Load balancer configuration:
+
+1.  4 backends are created in a single MIG.
+
+Test driver asserts:
+
+1.  All backends receive at least one RPC.
+
+The test driver records the peer distribution for a subsequent block of 100 RPCs
+then stops the backends.
+
+Test driver asserts:
+
+1.  No RPCs from the client are successful.
+
+The test driver resumes the backends.
+
+Test driver asserts:
+
+1.  Once all backends receive at least one RPC, the distribution for a block of
+    100 RPCs is the same as the distribution recorded prior to restart.
+
+### secondary_locality_gets_requests_on_primary_failure
+
+This test verifies that backends in a secondary locality receive traffic when
+all backends in the primary locality fail.
+
+Client parameters:
+
+1.  --num_channels=1
+1.  --qps=10
+
+Load balancer configuration:
+
+1.  The primary MIG with 2 backends in the same zone as the client
+1.  The secondary MIG with 2 backends in a different zone
+
+Test driver asserts:
+
+1.  All backends in the primary locality receive at least 1 RPC.
+1.  No backends in the secondary locality receive RPCs.
+
+The test driver stops the backends in the primary locality.
+
+Test driver asserts:
+
+1.  All backends in the secondary locality receive at least 1 RPC.
+
+The test driver resumes the backends in the primary locality.
+
+Test driver asserts:
+
+1.  All backends in the primary locality receive at least 1 RPC.
+1.  No backends in the secondary locality receive RPCs.
+
+### secondary_locality_gets_no_requests_on_partial_primary_failure
+
+This test verifies that backends in a failover locality do not receive traffic
+when at least one of the backends in the primary locality remain healthy.
+
+**Note:** Future TD features may change the expected behavior and require
+changes to this test case.
+
+Client parameters:
+
+1.  --num_channels=1
+1.  --qps=10
+
+Load balancer configuration:
+
+1.  The primary MIG with 2 backends in the same zone as the client
+1.  The secondary MIG with 2 backends in a different zone
+
+Test driver asserts:
+
+1.  All backends in the primary locality receive at least 1 RPC.
+1.  No backends in the secondary locality receive RPCs.
+
+The test driver stops one of the backends in the primary locality.
+
+Test driver asserts:
+
+1.  All backends in the primary locality receive at least 1 RPC.
+1.  No backends in the secondary locality receive RPCs.
+
+### new_instance_group_receives_traffic
+
+This test verifies that new instance groups added to a backend service in the
+same zone receive traffic.
+
+Client parameters:
+
+1.  --num_channels=1
+1.  --qps=10
+
+Load balancer configuration:
+
+1.  One MIG with two backends, using rate balancing mode.
+
+Test driver asserts:
+
+1.  All backends receive at least one RPC.
+
+The test driver adds a new MIG with two backends in the same zone.
+
+Test driver asserts:
+
+1.  All backends in each MIG receive at least one RPC.
+
+### remove_instance_group
+
+This test verifies that a remaining instance group can successfully serve RPCs
+after removal of another instance group in the same zone.
+
+Client parameters:
+
+1.  --num_channels=1
+1.  --qps=10
+
+Load balancer configuration:
+
+1.  Two MIGs with two backends each, using rate balancing mode.
+
+Test driver asserts:
+
+1.  All backends receive at least one RPC.
+
+The test driver removes one MIG.
+
+Test driver asserts:
+
+1.  All RPCs are directed to the two remaining backends (no RPC failures).
+
+### change_backend_service
+
+This test verifies that the backend service can be replaced and traffic routed
+to the new backends.
+
+Client parameters:
+
+1.  --num_channels=1
+1.  --qps=10
+
+Load balancer configuration:
+
+1.  One MIG with two backends
+
+Test driver asserts:
+
+1.  All backends receive at least one RPC.
+
+The test driver creates a new backend service containing a MIG with two backends
+and changes the TD URL map to point to this new backend service.
+
+Test driver asserts:
+
+1.  All RPCs are directed to the new backend service.
+

examples/cpp/compression/README.md

@@ -5,11 +5,11 @@ Make sure you have run the [hello world example](../helloworld) or understood th
 
 ### Get the tutorial source code
 
-The example code for this and our other examples lives in the `examples` directory. Clone this repository to your local machine by running the following command:
+The example code for this and our other examples lives in the `examples` directory. Clone this repository at the [latest stable release tag](https://github.com/grpc/grpc/releases) to your local machine  by running the following command:
 
 
 ```sh
-$ git clone -b $(curl -L https://grpc.io/release) https://github.com/grpc/grpc
+$ git clone -b RELEASE_TAG_HERE https://github.com/grpc/grpc
 ```
 
 Change your current directory to examples/cpp/compression

examples/cpp/helloworld/README.md

@@ -7,12 +7,12 @@ Make sure you have installed gRPC on your system. Follow the
 ### Get the tutorial source code
 
 The example code for this and our other examples lives in the `examples`
-directory. Clone this repository to your local machine by running the
-following command:
+directory. Clone this repository at the [latest stable release tag](https://github.com/grpc/grpc/releases)
+to your local machine by running the following command:
 
 
 ```sh
-$ git clone -b $(curl -L https://grpc.io/release) https://github.com/grpc/grpc
+$ git clone -b RELEASE_TAG_HERE https://github.com/grpc/grpc
 ```
 
 Change your current directory to examples/cpp/helloworld

examples/cpp/load_balancing/README.md

@@ -5,11 +5,12 @@ Make sure you have run the [hello world example](../helloworld) or understood th
 
 ### Get the tutorial source code
 
-The example code for this and our other examples lives in the `examples` directory. Clone this repository to your local machine by running the following command:
+The example code for this and our other examples lives in the `examples` directory. Clone this repository 
+at the [latest stable release tag](https://github.com/grpc/grpc/releases) to your local machine by running the following command:
 
 
 ```sh
-$ git clone -b $(curl -L https://grpc.io/release) https://github.com/grpc/grpc
+$ git clone -b RELEASE_TAG_HERE https://github.com/grpc/grpc
 ```
 
 Change your current directory to examples/cpp/load_balancing

examples/cpp/metadata/README.md

@@ -10,9 +10,10 @@ https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md, with the
 exception of binary headers, which don't have to be base64 encoded.
 
 ### Get the tutorial source code
- The example code for this and our other examples lives in the `examples` directory. Clone this repository to your local machine by running the following command:
+ The example code for this and our other examples lives in the `examples` directory. Clone this repository 
+ at the [latest stable release tag](https://github.com/grpc/grpc/releases) to your local machine by running the following command:
  ```sh
-$ git clone -b $(curl -L https://grpc.io/release) https://github.com/grpc/grpc
+$ git clone -b RELEASE_TAG_HERE https://github.com/grpc/grpc
 ```
  Change your current directory to examples/cpp/metadata
  ```sh

examples/node/README.md

@@ -12,7 +12,7 @@ INSTALL
    ```sh
    $ # Get the gRPC repository
    $ export REPO_ROOT=grpc # REPO root can be any directory of your choice
-   $ git clone -b $(curl -L https://grpc.io/release) https://github.com/grpc/grpc $REPO_ROOT
+   $ git clone -b RELEASE_TAG_HERE https://github.com/grpc/grpc $REPO_ROOT
    $ cd $REPO_ROOT
 
    $ cd examples/node

examples/objective-c/helloworld/README.md

@@ -18,11 +18,11 @@ Here's how to build and run the Objective-C implementation of the [Hello World](
 example used in [Getting started](https://github.com/grpc/grpc/tree/master/examples).
 
 The example code for this and our other examples lives in the `examples` directory. Clone
-this repository to your local machine by running the following commands:
+this repository at the [latest stable release tag](https://github.com/grpc/grpc/releases) to your local machine by running the following commands:
 
 
 ```sh
-$ git clone -b $(curl -L https://grpc.io/release) https://github.com/grpc/grpc
+$ git clone -b RELEASE_TAG_HERE https://github.com/grpc/grpc
 $ cd grpc
 $ git submodule update --init
 ```

examples/php/README.md

@@ -16,7 +16,7 @@ This requires `php` >= 5.5, `pecl`, `composer`
  - Install the `protoc` compiler plugin `grpc_php_plugin`
 
    ```sh
-   $ git clone -b $(curl -L https://grpc.io/release) https://github.com/grpc/grpc
+   $ git clone -b RELEASE_TAG_HERE https://github.com/grpc/grpc
    $ cd grpc
    $ make grpc_php_plugin
    ```

gRPC-Core.podspec

@@ -139,6 +139,7 @@ Pod::Spec.new do |s|
                       'include/grpc/impl/codegen/slice.h',
                       'include/grpc/impl/codegen/status.h',
                       'include/grpc/impl/codegen/sync.h',
+                      'include/grpc/impl/codegen/sync_abseil.h',
                       'include/grpc/impl/codegen/sync_custom.h',
                       'include/grpc/impl/codegen/sync_generic.h',
                       'include/grpc/impl/codegen/sync_posix.h',
@@ -158,6 +159,7 @@ Pod::Spec.new do |s|
                       'include/grpc/support/port_platform.h',
                       'include/grpc/support/string_util.h',
                       'include/grpc/support/sync.h',
+                      'include/grpc/support/sync_abseil.h',
                       'include/grpc/support/sync_custom.h',
                       'include/grpc/support/sync_generic.h',
                       'include/grpc/support/sync_posix.h',
@@ -531,6 +533,7 @@ Pod::Spec.new do |s|
                       'src/core/lib/gpr/string_windows.cc',
                       'src/core/lib/gpr/string_windows.h',
                       'src/core/lib/gpr/sync.cc',
+                      'src/core/lib/gpr/sync_abseil.cc',
                       'src/core/lib/gpr/sync_posix.cc',
                       'src/core/lib/gpr/sync_windows.cc',
                       'src/core/lib/gpr/time.cc',

grpc.gemspec

@@ -73,6 +73,7 @@ Gem::Specification.new do |s|
   s.files += %w( include/grpc/impl/codegen/slice.h )
   s.files += %w( include/grpc/impl/codegen/status.h )
   s.files += %w( include/grpc/impl/codegen/sync.h )
+  s.files += %w( include/grpc/impl/codegen/sync_abseil.h )
   s.files += %w( include/grpc/impl/codegen/sync_custom.h )
   s.files += %w( include/grpc/impl/codegen/sync_generic.h )
   s.files += %w( include/grpc/impl/codegen/sync_posix.h )
@@ -92,6 +93,7 @@ Gem::Specification.new do |s|
   s.files += %w( include/grpc/support/port_platform.h )
   s.files += %w( include/grpc/support/string_util.h )
   s.files += %w( include/grpc/support/sync.h )
+  s.files += %w( include/grpc/support/sync_abseil.h )
   s.files += %w( include/grpc/support/sync_custom.h )
   s.files += %w( include/grpc/support/sync_generic.h )
   s.files += %w( include/grpc/support/sync_posix.h )
@@ -453,6 +455,7 @@ Gem::Specification.new do |s|
   s.files += %w( src/core/lib/gpr/string_windows.cc )
   s.files += %w( src/core/lib/gpr/string_windows.h )
   s.files += %w( src/core/lib/gpr/sync.cc )
+  s.files += %w( src/core/lib/gpr/sync_abseil.cc )
   s.files += %w( src/core/lib/gpr/sync_posix.cc )
   s.files += %w( src/core/lib/gpr/sync_windows.cc )
   s.files += %w( src/core/lib/gpr/time.cc )

grpc.gyp

@@ -466,6 +466,7 @@
         'src/core/lib/gpr/string_util_windows.cc',
         'src/core/lib/gpr/string_windows.cc',
         'src/core/lib/gpr/sync.cc',
+        'src/core/lib/gpr/sync_abseil.cc',
         'src/core/lib/gpr/sync_posix.cc',
         'src/core/lib/gpr/sync_windows.cc',
         'src/core/lib/gpr/time.cc',

include/grpc++/impl/sync_no_cxx11.h

@@ -1,28 +0,0 @@
-/*
- *
- * Copyright 2018 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-// DEPRECATED: The headers in include/grpc++ are deprecated. Please include the
-// headers in include/grpcpp instead. This header exists only for backwards
-// compatibility.
-
-#ifndef GRPCXX_IMPL_SYNC_NO_CXX11_H
-#define GRPCXX_IMPL_SYNC_NO_CXX11_H
-
-#include <grpcpp/impl/sync_no_cxx11.h>
-
-#endif  // GRPCXX_IMPL_SYNC_NO_CXX11_H

include/grpc/impl/codegen/port_platform.h

@@ -34,6 +34,13 @@
 #define GRPC_USE_ABSL 1
 #endif
 
+/*
+ * Defines GPR_ABSEIL_SYNC to use synchronization features from Abseil
+ */
+#ifndef GPR_ABSEIL_SYNC
+/* #define GPR_ABSEIL_SYNC 1 */
+#endif
+
 /* Get windows.h included everywhere (we need it) */
 #if defined(_WIN64) || defined(WIN64) || defined(_WIN32) || defined(WIN32)
 #ifndef WIN32_LEAN_AND_MEAN

include/grpc/impl/codegen/sync.h

@@ -46,12 +46,14 @@ extern "C" {
 
 #include <grpc/impl/codegen/sync_generic.h>
 
-#if defined(GPR_POSIX_SYNC)
+#if defined(GPR_CUSTOM_SYNC)
+#include <grpc/impl/codegen/sync_custom.h>
+#elif defined(GPR_ABSEIL_SYNC)
+#include <grpc/impl/codegen/sync_abseil.h>
+#elif defined(GPR_POSIX_SYNC)
 #include <grpc/impl/codegen/sync_posix.h>
 #elif defined(GPR_WINDOWS)
 #include <grpc/impl/codegen/sync_windows.h>
-#elif defined(GPR_CUSTOM_SYNC)
-#include <grpc/impl/codegen/sync_custom.h>
 #else
 #error Unable to determine platform for sync
 #endif

include/grpc++/impl/sync_cxx11.h → include/grpc/impl/codegen/sync_abseil.h

@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2018 gRPC authors.
+ * Copyright 2020 gRPC authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -16,13 +16,21 @@
  *
  */
 
-// DEPRECATED: The headers in include/grpc++ are deprecated. Please include the
-// headers in include/grpcpp instead. This header exists only for backwards
-// compatibility.
+#ifndef GRPC_IMPL_CODEGEN_SYNC_ABSEIL_H
+#define GRPC_IMPL_CODEGEN_SYNC_ABSEIL_H
 
-#ifndef GRPCXX_IMPL_SYNC_CXX11_H
-#define GRPCXX_IMPL_SYNC_CXX11_H
+#include <grpc/impl/codegen/port_platform.h>
 
-#include <grpcpp/impl/sync_cxx11.h>
+#include <grpc/impl/codegen/sync_generic.h>
 
-#endif  // GRPCXX_IMPL_SYNC_CXX11_H
+#ifdef GPR_ABSEIL_SYNC
+
+typedef intptr_t gpr_mu;
+typedef intptr_t gpr_cv;
+typedef int32_t gpr_once;
+
+#define GPR_ONCE_INIT 0
+
+#endif
+
+#endif /* GRPC_IMPL_CODEGEN_SYNC_ABSEIL_H */

include/grpc/module.modulemap

@@ -10,6 +10,7 @@ framework module grpc {
   header "support/port_platform.h"
   header "support/string_util.h"
   header "support/sync.h"
+  header "support/sync_abseil.h"
   header "support/sync_generic.h"
   header "support/thd_id.h"
   header "support/time.h"
@@ -20,6 +21,7 @@ framework module grpc {
   header "impl/codegen/log.h"
   header "impl/codegen/port_platform.h"
   header "impl/codegen/sync.h"
+  header "impl/codegen/sync_abseil.h"
   header "impl/codegen/sync_generic.h"
   header "impl/codegen/byte_buffer.h"
   header "impl/codegen/byte_buffer_reader.h"
@@ -36,6 +38,7 @@ framework module grpc {
   header "impl/codegen/log.h"
   header "impl/codegen/port_platform.h"
   header "impl/codegen/sync.h"
+  header "impl/codegen/sync_abseil.h"
   header "impl/codegen/sync_generic.h"
   header "grpc_security.h"
   header "byte_buffer.h"

include/grpcpp/impl/sync_cxx11.h → include/grpc/support/sync_abseil.h

@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015 gRPC authors.
+ * Copyright 2020 gRPC authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -16,9 +16,11 @@
  *
  */
 
-#ifndef GRPCPP_IMPL_SYNC_CXX11_H
-#define GRPCPP_IMPL_SYNC_CXX11_H
+#ifndef GRPC_SUPPORT_SYNC_ABSEIL_H
+#define GRPC_SUPPORT_SYNC_ABSEIL_H
 
-#include <grpcpp/impl/codegen/sync_cxx11.h>
+#include <grpc/support/port_platform.h>
 
-#endif  // GRPCPP_IMPL_SYNC_CXX11_H
+#include <grpc/impl/codegen/sync_abseil.h>
+
+#endif /* GRPC_SUPPORT_SYNC_ABSEIL_H */

include/grpcpp/impl/codegen/completion_queue_impl.h

@@ -32,11 +32,14 @@
 #ifndef GRPCPP_IMPL_CODEGEN_COMPLETION_QUEUE_IMPL_H
 #define GRPCPP_IMPL_CODEGEN_COMPLETION_QUEUE_IMPL_H
 
+#include <list>
+
 #include <grpc/impl/codegen/atm.h>
 #include <grpcpp/impl/codegen/completion_queue_tag.h>
 #include <grpcpp/impl/codegen/core_codegen_interface.h>
 #include <grpcpp/impl/codegen/grpc_library.h>
 #include <grpcpp/impl/codegen/status.h>
+#include <grpcpp/impl/codegen/sync.h>
 #include <grpcpp/impl/codegen/time.h>
 
 struct grpc_completion_queue;
@@ -250,6 +253,11 @@ class CompletionQueue : private ::grpc::GrpcLibraryCodegen {
   }
 
  private:
+  // Friends for access to server registration lists that enable checking and
+  // logging on shutdown
+  friend class ::grpc_impl::ServerBuilder;
+  friend class ::grpc_impl::Server;
+
   // Friend synchronous wrappers so that they can access Pluck(), which is
   // a semi-private API geared towards the synchronous implementation.
   template <class R>
@@ -274,7 +282,6 @@ class CompletionQueue : private ::grpc::GrpcLibraryCodegen {
   friend class ::grpc_impl::internal::TemplatedBidiStreamingHandler;
   template <::grpc::StatusCode code>
   friend class ::grpc_impl::internal::ErrorMethodHandler;
-  friend class ::grpc_impl::Server;
   friend class ::grpc_impl::ServerContextBase;
   friend class ::grpc::ServerInterface;
   template <class InputMessage, class OutputMessage>
@@ -379,13 +386,41 @@ class CompletionQueue : private ::grpc::GrpcLibraryCodegen {
     }
   }
 
+  void RegisterServer(const Server* server) {
+    (void)server;
+#ifndef NDEBUG
+    grpc::internal::MutexLock l(&server_list_mutex_);
+    server_list_.push_back(server);
+#endif
+  }
+  void UnregisterServer(const Server* server) {
+    (void)server;
+#ifndef NDEBUG
+    grpc::internal::MutexLock l(&server_list_mutex_);
+    server_list_.remove(server);
+#endif
+  }
+  bool ServerListEmpty() const {
+#ifndef NDEBUG
+    grpc::internal::MutexLock l(&server_list_mutex_);
+    return server_list_.empty();
+#endif
+    return true;
+  }
+
   grpc_completion_queue* cq_;  // owned
 
   gpr_atm avalanches_in_flight_;
+
+  // List of servers associated with this CQ. Even though this is only used with
+  // NDEBUG, instantiate it in all cases since otherwise the size will be
+  // inconsistent.
+  mutable grpc::internal::Mutex server_list_mutex_;
+  std::list<const Server*> server_list_ /* GUARDED_BY(server_list_mutex_) */;
 };
 
 /// A specific type of completion queue used by the processing of notifications
-/// by servers. Instantiated by \a ServerBuilder.
+/// by servers. Instantiated by \a ServerBuilder or Server (for health checker).
 class ServerCompletionQueue : public CompletionQueue {
  public:
   bool IsFrequentlyPolled() { return polling_type_ != GRPC_CQ_NON_LISTENING; }

include/grpcpp/impl/codegen/server_callback_handlers.h

@@ -117,9 +117,19 @@ class CallbackUnaryHandler : public ::grpc::internal::MethodHandler {
   class ServerCallbackUnaryImpl : public ServerCallbackUnary {
    public:
     void Finish(::grpc::Status s) override {
+      // A callback that only contains a call to MaybeDone can be run as an
+      // inline callback regardless of whether or not OnDone is inlineable
+      // because if the actual OnDone callback needs to be scheduled, MaybeDone
+      // is responsible for dispatching to an executor thread if needed. Thus,
+      // when setting up the finish_tag_, we can set its own callback to
+      // inlineable.
       finish_tag_.Set(
-          call_.call(), [this](bool) { MaybeDone(); }, &finish_ops_,
-          reactor_.load(std::memory_order_relaxed)->InternalInlineable());
+          call_.call(),
+          [this](bool) {
+            this->MaybeDone(
+                reactor_.load(std::memory_order_relaxed)->InternalInlineable());
+          },
+          &finish_ops_, /*can_inline=*/true);
       finish_ops_.set_core_cq_tag(&finish_tag_);
 
       if (!ctx_->sent_initial_metadata_) {
@@ -144,13 +154,19 @@ class CallbackUnaryHandler : public ::grpc::internal::MethodHandler {
     void SendInitialMetadata() override {
       GPR_CODEGEN_ASSERT(!ctx_->sent_initial_metadata_);
       this->Ref();
+      // The callback for this function should not be marked inline because it
+      // is directly invoking a user-controlled reaction
+      // (OnSendInitialMetadataDone). Thus it must be dispatched to an executor
+      // thread. However, any OnDone needed after that can be inlined because it
+      // is already running on an executor thread.
       meta_tag_.Set(call_.call(),
                     [this](bool ok) {
-                      reactor_.load(std::memory_order_relaxed)
-                          ->OnSendInitialMetadataDone(ok);
-                      MaybeDone();
+                      ServerUnaryReactor* reactor =
+                          reactor_.load(std::memory_order_relaxed);
+                      reactor->OnSendInitialMetadataDone(ok);
+                      this->MaybeDone(/*inlineable_ondone=*/true);
                     },
-                    &meta_ops_, false);
+                    &meta_ops_, /*can_inline=*/false);
       meta_ops_.SendInitialMetadata(&ctx_->initial_metadata_,
                                     ctx_->initial_metadata_flags());
       if (ctx_->compression_level_set()) {
@@ -184,22 +200,20 @@ class CallbackUnaryHandler : public ::grpc::internal::MethodHandler {
       reactor_.store(reactor, std::memory_order_relaxed);
       this->BindReactor(reactor);
       this->MaybeCallOnCancel(reactor);
-      this->MaybeDone();
+      this->MaybeDone(reactor->InternalInlineable());
     }
 
     const RequestType* request() { return allocator_state_->request(); }
     ResponseType* response() { return allocator_state_->response(); }
 
-    void MaybeDone() override {
-      if (GPR_UNLIKELY(this->Unref() == 1)) {
-        reactor_.load(std::memory_order_relaxed)->OnDone();
-        grpc_call* call = call_.call();
-        auto call_requester = std::move(call_requester_);
-        allocator_state_->Release();
-        this->~ServerCallbackUnaryImpl();  // explicitly call destructor
-        ::grpc::g_core_codegen_interface->grpc_call_unref(call);
-        call_requester();
-      }
+    void CallOnDone() override {
+      reactor_.load(std::memory_order_relaxed)->OnDone();
+      grpc_call* call = call_.call();
+      auto call_requester = std::move(call_requester_);
+      allocator_state_->Release();
+      this->~ServerCallbackUnaryImpl();  // explicitly call destructor
+      ::grpc::g_core_codegen_interface->grpc_call_unref(call);
+      call_requester();
     }
 
     ServerReactor* reactor() override {
@@ -255,8 +269,13 @@ class CallbackClientStreamingHandler : public ::grpc::internal::MethodHandler {
             static_cast<::grpc_impl::CallbackServerContext*>(
                 param.server_context),
             param.call, std::move(param.call_requester));
+    // Inlineable OnDone can be false in the CompletionOp callback because there
+    // is no read reactor that has an inlineable OnDone; this only applies to
+    // the DefaultReactor (which is unary).
     param.server_context->BeginCompletionOp(
-        param.call, [reader](bool) { reader->MaybeDone(); }, reader);
+        param.call,
+        [reader](bool) { reader->MaybeDone(/*inlineable_ondone=*/false); },
+        reader);
 
     ServerReadReactor<RequestType>* reactor = nullptr;
     if (param.status.ok()) {
@@ -287,8 +306,17 @@ class CallbackClientStreamingHandler : public ::grpc::internal::MethodHandler {
   class ServerCallbackReaderImpl : public ServerCallbackReader<RequestType> {
    public:
     void Finish(::grpc::Status s) override {
-      finish_tag_.Set(call_.call(), [this](bool) { MaybeDone(); }, &finish_ops_,
-                      false);
+      // A finish tag with only MaybeDone can have its callback inlined
+      // regardless even if OnDone is not inlineable because this callback just
+      // checks a ref and then decides whether or not to dispatch OnDone.
+      finish_tag_.Set(call_.call(),
+                      [this](bool) {
+                        // Inlineable OnDone can be false here because there is
+                        // no read reactor that has an inlineable OnDone; this
+                        // only applies to the DefaultReactor (which is unary).
+                        this->MaybeDone(/*inlineable_ondone=*/false);
+                      },
+                      &finish_ops_, /*can_inline=*/true);
       if (!ctx_->sent_initial_metadata_) {
         finish_ops_.SendInitialMetadata(&ctx_->initial_metadata_,
                                         ctx_->initial_metadata_flags());
@@ -311,13 +339,17 @@ class CallbackClientStreamingHandler : public ::grpc::internal::MethodHandler {
     void SendInitialMetadata() override {
       GPR_CODEGEN_ASSERT(!ctx_->sent_initial_metadata_);
       this->Ref();
+      // The callback for this function should not be inlined because it invokes
+      // a user-controlled reaction, but any resulting OnDone can be inlined in
+      // the executor to which this callback is dispatched.
       meta_tag_.Set(call_.call(),
                     [this](bool ok) {
-                      reactor_.load(std::memory_order_relaxed)
-                          ->OnSendInitialMetadataDone(ok);
-                      MaybeDone();
+                      ServerReadReactor<RequestType>* reactor =
+                          reactor_.load(std::memory_order_relaxed);
+                      reactor->OnSendInitialMetadataDone(ok);
+                      this->MaybeDone(/*inlineable_ondone=*/true);
                     },
-                    &meta_ops_, false);
+                    &meta_ops_, /*can_inline=*/false);
       meta_ops_.SendInitialMetadata(&ctx_->initial_metadata_,
                                     ctx_->initial_metadata_flags());
       if (ctx_->compression_level_set()) {
@@ -344,31 +376,35 @@ class CallbackClientStreamingHandler : public ::grpc::internal::MethodHandler {
 
     void SetupReactor(ServerReadReactor<RequestType>* reactor) {
       reactor_.store(reactor, std::memory_order_relaxed);
+      // The callback for this function should not be inlined because it invokes
+      // a user-controlled reaction, but any resulting OnDone can be inlined in
+      // the executor to which this callback is dispatched.
       read_tag_.Set(call_.call(),
-                    [this](bool ok) {
-                      reactor_.load(std::memory_order_relaxed)->OnReadDone(ok);
-                      MaybeDone();
+                    [this, reactor](bool ok) {
+                      reactor->OnReadDone(ok);
+                      this->MaybeDone(/*inlineable_ondone=*/true);
                     },
-                    &read_ops_, false);
+                    &read_ops_, /*can_inline=*/false);
       read_ops_.set_core_cq_tag(&read_tag_);
       this->BindReactor(reactor);
       this->MaybeCallOnCancel(reactor);
-      this->MaybeDone();
+      // Inlineable OnDone can be false here because there is no read
+      // reactor that has an inlineable OnDone; this only applies to the
+      // DefaultReactor (which is unary).
+      this->MaybeDone(/*inlineable_ondone=*/false);
     }
 
     ~ServerCallbackReaderImpl() {}
 
     ResponseType* response() { return &resp_; }
 
-    void MaybeDone() override {
-      if (GPR_UNLIKELY(this->Unref() == 1)) {
-        reactor_.load(std::memory_order_relaxed)->OnDone();
-        grpc_call* call = call_.call();
-        auto call_requester = std::move(call_requester_);
-        this->~ServerCallbackReaderImpl();  // explicitly call destructor
-        ::grpc::g_core_codegen_interface->grpc_call_unref(call);
-        call_requester();
-      }
+    void CallOnDone() override {
+      reactor_.load(std::memory_order_relaxed)->OnDone();
+      grpc_call* call = call_.call();
+      auto call_requester = std::move(call_requester_);
+      this->~ServerCallbackReaderImpl();  // explicitly call destructor
+      ::grpc::g_core_codegen_interface->grpc_call_unref(call);
+      call_requester();
     }
 
     ServerReactor* reactor() override {
@@ -419,8 +455,13 @@ class CallbackServerStreamingHandler : public ::grpc::internal::MethodHandler {
                 param.server_context),
             param.call, static_cast<RequestType*>(param.request),
             std::move(param.call_requester));
+    // Inlineable OnDone can be false in the CompletionOp callback because there
+    // is no write reactor that has an inlineable OnDone; this only applies to
+    // the DefaultReactor (which is unary).
     param.server_context->BeginCompletionOp(
-        param.call, [writer](bool) { writer->MaybeDone(); }, writer);
+        param.call,
+        [writer](bool) { writer->MaybeDone(/*inlineable_ondone=*/false); },
+        writer);
 
     ServerWriteReactor<ResponseType>* reactor = nullptr;
     if (param.status.ok()) {
@@ -467,8 +508,17 @@ class CallbackServerStreamingHandler : public ::grpc::internal::MethodHandler {
   class ServerCallbackWriterImpl : public ServerCallbackWriter<ResponseType> {
    public:
     void Finish(::grpc::Status s) override {
-      finish_tag_.Set(call_.call(), [this](bool) { MaybeDone(); }, &finish_ops_,
-                      false);
+      // A finish tag with only MaybeDone can have its callback inlined
+      // regardless even if OnDone is not inlineable because this callback just
+      // checks a ref and then decides whether or not to dispatch OnDone.
+      finish_tag_.Set(call_.call(),
+                      [this](bool) {
+                        // Inlineable OnDone can be false here because there is
+                        // no write reactor that has an inlineable OnDone; this
+                        // only applies to the DefaultReactor (which is unary).
+                        this->MaybeDone(/*inlineable_ondone=*/false);
+                      },
+                      &finish_ops_, /*can_inline=*/true);
       finish_ops_.set_core_cq_tag(&finish_tag_);
 
       if (!ctx_->sent_initial_metadata_) {
@@ -486,13 +536,17 @@ class CallbackServerStreamingHandler : public ::grpc::internal::MethodHandler {
     void SendInitialMetadata() override {
       GPR_CODEGEN_ASSERT(!ctx_->sent_initial_metadata_);
       this->Ref();
+      // The callback for this function should not be inlined because it invokes
+      // a user-controlled reaction, but any resulting OnDone can be inlined in
+      // the executor to which this callback is dispatched.
       meta_tag_.Set(call_.call(),
                     [this](bool ok) {
-                      reactor_.load(std::memory_order_relaxed)
-                          ->OnSendInitialMetadataDone(ok);
-                      MaybeDone();
+                      ServerWriteReactor<ResponseType>* reactor =
+                          reactor_.load(std::memory_order_relaxed);
+                      reactor->OnSendInitialMetadataDone(ok);
+                      this->MaybeDone(/*inlineable_ondone=*/true);
                     },
-                    &meta_ops_, false);
+                    &meta_ops_, /*can_inline=*/false);
       meta_ops_.SendInitialMetadata(&ctx_->initial_metadata_,
                                     ctx_->initial_metadata_flags());
       if (ctx_->compression_level_set()) {
@@ -547,31 +601,34 @@ class CallbackServerStreamingHandler : public ::grpc::internal::MethodHandler {
 
     void SetupReactor(ServerWriteReactor<ResponseType>* reactor) {
       reactor_.store(reactor, std::memory_order_relaxed);
-      write_tag_.Set(
-          call_.call(),
-          [this](bool ok) {
-            reactor_.load(std::memory_order_relaxed)->OnWriteDone(ok);
-            MaybeDone();
-          },
-          &write_ops_, false);
+      // The callback for this function should not be inlined because it invokes
+      // a user-controlled reaction, but any resulting OnDone can be inlined in
+      // the executor to which this callback is dispatched.
+      write_tag_.Set(call_.call(),
+                     [this, reactor](bool ok) {
+                       reactor->OnWriteDone(ok);
+                       this->MaybeDone(/*inlineable_ondone=*/true);
+                     },
+                     &write_ops_, /*can_inline=*/false);
       write_ops_.set_core_cq_tag(&write_tag_);
       this->BindReactor(reactor);
       this->MaybeCallOnCancel(reactor);
-      this->MaybeDone();
+      // Inlineable OnDone can be false here because there is no write
+      // reactor that has an inlineable OnDone; this only applies to the
+      // DefaultReactor (which is unary).
+      this->MaybeDone(/*inlineable_ondone=*/false);
     }
     ~ServerCallbackWriterImpl() { req_->~RequestType(); }
 
     const RequestType* request() { return req_; }
 
-    void MaybeDone() override {
-      if (GPR_UNLIKELY(this->Unref() == 1)) {
-        reactor_.load(std::memory_order_relaxed)->OnDone();
-        grpc_call* call = call_.call();
-        auto call_requester = std::move(call_requester_);
-        this->~ServerCallbackWriterImpl();  // explicitly call destructor
-        ::grpc::g_core_codegen_interface->grpc_call_unref(call);
-        call_requester();
-      }
+    void CallOnDone() override {
+      reactor_.load(std::memory_order_relaxed)->OnDone();
+      grpc_call* call = call_.call();
+      auto call_requester = std::move(call_requester_);
+      this->~ServerCallbackWriterImpl();  // explicitly call destructor
+      ::grpc::g_core_codegen_interface->grpc_call_unref(call);
+      call_requester();
     }
 
     ServerReactor* reactor() override {
@@ -620,8 +677,13 @@ class CallbackBidiHandler : public ::grpc::internal::MethodHandler {
             static_cast<::grpc_impl::CallbackServerContext*>(
                 param.server_context),
             param.call, std::move(param.call_requester));
+    // Inlineable OnDone can be false in the CompletionOp callback because there
+    // is no bidi reactor that has an inlineable OnDone; this only applies to
+    // the DefaultReactor (which is unary).
     param.server_context->BeginCompletionOp(
-        param.call, [stream](bool) { stream->MaybeDone(); }, stream);
+        param.call,
+        [stream](bool) { stream->MaybeDone(/*inlineable_ondone=*/false); },
+        stream);
 
     ServerBidiReactor<RequestType, ResponseType>* reactor = nullptr;
     if (param.status.ok()) {
@@ -652,8 +714,17 @@ class CallbackBidiHandler : public ::grpc::internal::MethodHandler {
       : public ServerCallbackReaderWriter<RequestType, ResponseType> {
    public:
     void Finish(::grpc::Status s) override {
-      finish_tag_.Set(call_.call(), [this](bool) { MaybeDone(); }, &finish_ops_,
-                      false);
+      // A finish tag with only MaybeDone can have its callback inlined
+      // regardless even if OnDone is not inlineable because this callback just
+      // checks a ref and then decides whether or not to dispatch OnDone.
+      finish_tag_.Set(call_.call(),
+                      [this](bool) {
+                        // Inlineable OnDone can be false here because there is
+                        // no bidi reactor that has an inlineable OnDone; this
+                        // only applies to the DefaultReactor (which is unary).
+                        this->MaybeDone(/*inlineable_ondone=*/false);
+                      },
+                      &finish_ops_, /*can_inline=*/true);
       finish_ops_.set_core_cq_tag(&finish_tag_);
 
       if (!ctx_->sent_initial_metadata_) {
@@ -671,13 +742,17 @@ class CallbackBidiHandler : public ::grpc::internal::MethodHandler {
     void SendInitialMetadata() override {
       GPR_CODEGEN_ASSERT(!ctx_->sent_initial_metadata_);
       this->Ref();
+      // The callback for this function should not be inlined because it invokes
+      // a user-controlled reaction, but any resulting OnDone can be inlined in
+      // the executor to which this callback is dispatched.
       meta_tag_.Set(call_.call(),
                     [this](bool ok) {
-                      reactor_.load(std::memory_order_relaxed)
-                          ->OnSendInitialMetadataDone(ok);
-                      MaybeDone();
+                      ServerBidiReactor<RequestType, ResponseType>* reactor =
+                          reactor_.load(std::memory_order_relaxed);
+                      reactor->OnSendInitialMetadataDone(ok);
+                      this->MaybeDone(/*inlineable_ondone=*/true);
                     },
-                    &meta_ops_, false);
+                    &meta_ops_, /*can_inline=*/false);
       meta_ops_.SendInitialMetadata(&ctx_->initial_metadata_,
                                     ctx_->initial_metadata_flags());
       if (ctx_->compression_level_set()) {
@@ -733,35 +808,38 @@ class CallbackBidiHandler : public ::grpc::internal::MethodHandler {
 
     void SetupReactor(ServerBidiReactor<RequestType, ResponseType>* reactor) {
       reactor_.store(reactor, std::memory_order_relaxed);
-      write_tag_.Set(
-          call_.call(),
-          [this](bool ok) {
-            reactor_.load(std::memory_order_relaxed)->OnWriteDone(ok);
-            MaybeDone();
-          },
-          &write_ops_, false);
+      // The callbacks for these functions should not be inlined because they
+      // invoke user-controlled reactions, but any resulting OnDones can be
+      // inlined in the executor to which a callback is dispatched.
+      write_tag_.Set(call_.call(),
+                     [this, reactor](bool ok) {
+                       reactor->OnWriteDone(ok);
+                       this->MaybeDone(/*inlineable_ondone=*/true);
+                     },
+                     &write_ops_, /*can_inline=*/false);
       write_ops_.set_core_cq_tag(&write_tag_);
       read_tag_.Set(call_.call(),
-                    [this](bool ok) {
-                      reactor_.load(std::memory_order_relaxed)->OnReadDone(ok);
-                      MaybeDone();
+                    [this, reactor](bool ok) {
+                      reactor->OnReadDone(ok);
+                      this->MaybeDone(/*inlineable_ondone=*/true);
                     },
-                    &read_ops_, false);
+                    &read_ops_, /*can_inline=*/false);
       read_ops_.set_core_cq_tag(&read_tag_);
       this->BindReactor(reactor);
       this->MaybeCallOnCancel(reactor);
-      this->MaybeDone();
-    }
-
-    void MaybeDone() override {
-      if (GPR_UNLIKELY(this->Unref() == 1)) {
-        reactor_.load(std::memory_order_relaxed)->OnDone();
-        grpc_call* call = call_.call();
-        auto call_requester = std::move(call_requester_);
-        this->~ServerCallbackReaderWriterImpl();  // explicitly call destructor
-        ::grpc::g_core_codegen_interface->grpc_call_unref(call);
-        call_requester();
-      }
+      // Inlineable OnDone can be false here because there is no bidi
+      // reactor that has an inlineable OnDone; this only applies to the
+      // DefaultReactor (which is unary).
+      this->MaybeDone(/*inlineable_ondone=*/false);
+    }
+
+    void CallOnDone() override {
+      reactor_.load(std::memory_order_relaxed)->OnDone();
+      grpc_call* call = call_.call();
+      auto call_requester = std::move(call_requester_);
+      this->~ServerCallbackReaderWriterImpl();  // explicitly call destructor
+      ::grpc::g_core_codegen_interface->grpc_call_unref(call);
+      call_requester();
     }
 
     ServerReactor* reactor() override {

include/grpcpp/impl/codegen/server_callback_impl.h

@@ -73,11 +73,33 @@ class ServerCallbackCall {
  public:
   virtual ~ServerCallbackCall() {}
 
-  // This object is responsible for tracking when it is safe to call
-  // OnCancel. This function should not be called until after the method handler
-  // is done and the RPC has completed with a cancellation. This is tracked by
-  // counting how many of these conditions have been met and calling OnCancel
-  // when none remain unmet.
+  // This object is responsible for tracking when it is safe to call OnDone and
+  // OnCancel. OnDone should not be called until the method handler is complete,
+  // Finish has been called, the ServerContext CompletionOp (which tracks
+  // cancellation or successful completion) has completed, and all outstanding
+  // Read/Write actions have seen their reactions. OnCancel should not be called
+  // until after the method handler is done and the RPC has completed with a
+  // cancellation. This is tracked by counting how many of these conditions have
+  // been met and calling OnCancel when none remain unmet.
+
+  // Public versions of MaybeDone: one where we don't know the reactor in
+  // advance (used for the ServerContext CompletionOp), and one for where we
+  // know the inlineability of the OnDone reaction. You should set the inline
+  // flag to true if either the Reactor is InternalInlineable() or if this
+  // callback is already being forced to run dispatched to an executor
+  // (typically because it contains additional work than just the MaybeDone).
+
+  void MaybeDone() {
+    if (GPR_UNLIKELY(Unref() == 1)) {
+      ScheduleOnDone(reactor()->InternalInlineable());
+    }
+  }
+
+  void MaybeDone(bool inline_ondone) {
+    if (GPR_UNLIKELY(Unref() == 1)) {
+      ScheduleOnDone(inline_ondone);
+    }
+  }
 
   // Fast version called with known reactor passed in, used from derived
   // classes, typically in non-cancel case
@@ -101,14 +123,17 @@ class ServerCallbackCall {
   /// Increases the reference count
   void Ref() { callbacks_outstanding_.fetch_add(1, std::memory_order_relaxed); }
 
-  /// Decreases the reference count and returns the previous value
-  int Unref() {
-    return callbacks_outstanding_.fetch_sub(1, std::memory_order_acq_rel);
-  }
-
  private:
   virtual ServerReactor* reactor() = 0;
-  virtual void MaybeDone() = 0;
+
+  // CallOnDone performs the work required at completion of the RPC: invoking
+  // the OnDone function and doing all necessary cleanup. This function is only
+  // ever invoked on a fully-Unref'fed ServerCallbackCall.
+  virtual void CallOnDone() = 0;
+
+  // If the OnDone reaction is inlineable, execute it inline. Otherwise send it
+  // to an executor.
+  void ScheduleOnDone(bool inline_ondone);
 
   // If the OnCancel reaction is inlineable, execute it inline. Otherwise send
   // it to an executor.
@@ -121,6 +146,11 @@ class ServerCallbackCall {
                1, std::memory_order_acq_rel) == 1;
   }
 
+  /// Decreases the reference count and returns the previous value
+  int Unref() {
+    return callbacks_outstanding_.fetch_sub(1, std::memory_order_acq_rel);
+  }
+
   std::atomic_int on_cancel_conditions_remaining_{2};
   std::atomic_int callbacks_outstanding_{
       3};  // reserve for start, Finish, and CompletionOp

include/grpcpp/impl/codegen/server_context_impl.h

@@ -474,7 +474,7 @@ class ServerContextBase {
     ::grpc::Status status() const { return status_; }
 
    private:
-    void MaybeDone() override {}
+    void CallOnDone() override {}
     ::grpc_impl::internal::ServerReactor* reactor() override {
       return reactor_;
     }

include/grpcpp/impl/codegen/time.h

@@ -39,14 +39,12 @@ namespace grpc {
 template <typename T>
 class TimePoint {
  public:
-  TimePoint(const T& /*time*/) { you_need_a_specialization_of_TimePoint(); }
-  gpr_timespec raw_time() {
-    gpr_timespec t;
-    return t;
-  }
-
- private:
-  void you_need_a_specialization_of_TimePoint();
+  // If you see the error with methods below, you may need either
+  // i) using the existing types having a conversion class such as
+  // gpr_timespec and std::chrono::system_clock::time_point or
+  // ii) writing a new TimePoint<YourType> to address your case.
+  TimePoint(const T& /*time*/) = delete;
+  gpr_timespec raw_time() = delete;
 };
 
 template <>

include/grpcpp/impl/sync_no_cxx11.h

@@ -1,24 +0,0 @@
-/*
- *
- * Copyright 2015 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#ifndef GRPCPP_IMPL_SYNC_NO_CXX11_H
-#define GRPCPP_IMPL_SYNC_NO_CXX11_H
-
-#include <grpcpp/impl/codegen/sync_no_cxx11.h>
-
-#endif  // GRPCPP_IMPL_SYNC_NO_CXX11_H

include/grpcpp/security/tls_credentials_options.h

@@ -55,12 +55,13 @@ class TlsKeyMaterialsConfig {
   }
   int version() const { return version_; }
 
-  /** Setter for key materials that will be called by the user. The setter
-   * transfers ownership of the arguments to the config. **/
-  void set_pem_root_certs(grpc::string pem_root_certs);
+  /** Setter for key materials that will be called by the user. Ownership of the
+   * arguments will not be transferred. **/
+  void set_pem_root_certs(const grpc::string& pem_root_certs);
   void add_pem_key_cert_pair(const PemKeyCertPair& pem_key_cert_pair);
-  void set_key_materials(grpc::string pem_root_certs,
-                         std::vector<PemKeyCertPair> pem_key_cert_pair_list);
+  void set_key_materials(
+      const grpc::string& pem_root_certs,
+      const std::vector<PemKeyCertPair>& pem_key_cert_pair_list);
   void set_version(int version) { version_ = version; };
 
  private:
@@ -70,40 +71,36 @@ class TlsKeyMaterialsConfig {
 };
 
 /** TLS credential reload arguments, wraps grpc_tls_credential_reload_arg. It is
- * used for experimental purposes for now and it is subject to change.
+ *  used for experimental purposes for now and it is subject to change.
  *
- * The credential reload arg contains all the info necessary to schedule/cancel
- * a credential reload request. The callback function must be called after
- * finishing the schedule operation. See the description of the
- * grpc_tls_credential_reload_arg struct in grpc_security.h for more details.
+ *  The credential reload arg contains all the info necessary to schedule/cancel
+ *  a credential reload request. The callback function must be called after
+ *  finishing the schedule operation. See the description of the
+ *  grpc_tls_credential_reload_arg struct in grpc_security.h for more details.
  * **/
 class TlsCredentialReloadArg {
  public:
   /** TlsCredentialReloadArg does not take ownership of the C arg that is passed
-   * to the constructor. One must remember to free any memory allocated to the C
-   * arg after using the setter functions below. **/
+   *  to the constructor. One must remember to free any memory allocated to the
+   * C arg after using the setter functions below. **/
   TlsCredentialReloadArg(grpc_tls_credential_reload_arg* arg);
   ~TlsCredentialReloadArg();
 
-  /** Getters for member fields. The callback function is not exposed.
-   * They return the corresponding fields of the underlying C arg. In the case
-   * of the key materials config, it creates a new instance of the C++ key
-   * materials config from the underlying C grpc_tls_key_materials_config. **/
+  /** Getters for member fields. **/
   void* cb_user_data() const;
   bool is_pem_key_cert_pair_list_empty() const;
   grpc_ssl_certificate_config_reload_status status() const;
   grpc::string error_details() const;
 
-  /** Setters for member fields. They modify the fields of the underlying C arg.
-   * The setters for the key_materials_config and the error_details allocate
-   * memory when modifying c_arg_, so one must remember to free c_arg_'s
-   * original key_materials_config or error_details after using the appropriate
-   * setter function.
-   * **/
+  /** Setters for member fields. Ownership of the arguments will not be
+   *  transferred. **/
   void set_cb_user_data(void* cb_user_data);
   void set_pem_root_certs(const grpc::string& pem_root_certs);
   void add_pem_key_cert_pair(
-      TlsKeyMaterialsConfig::PemKeyCertPair pem_key_cert_pair);
+      const TlsKeyMaterialsConfig::PemKeyCertPair& pem_key_cert_pair);
+  void set_key_materials(const grpc::string& pem_root_certs,
+                         std::vector<TlsKeyMaterialsConfig::PemKeyCertPair>
+                             pem_key_cert_pair_list);
   void set_key_materials_config(
       const std::shared_ptr<TlsKeyMaterialsConfig>& key_materials_config);
   void set_status(grpc_ssl_certificate_config_reload_status status);
@@ -187,8 +184,7 @@ class TlsServerAuthorizationCheckArg {
   TlsServerAuthorizationCheckArg(grpc_tls_server_authorization_check_arg* arg);
   ~TlsServerAuthorizationCheckArg();
 
-  /** Getters for member fields. They return the corresponding fields of the
-   * underlying C arg.**/
+  /** Getters for member fields. **/
   void* cb_user_data() const;
   int success() const;
   grpc::string target_name() const;
@@ -197,12 +193,7 @@ class TlsServerAuthorizationCheckArg {
   grpc_status_code status() const;
   grpc::string error_details() const;
 
-  /** Setters for member fields. They modify the fields of the underlying C arg.
-   * The setters for target_name, peer_cert, and error_details allocate memory
-   * when modifying c_arg_, so one must remember to free c_arg_'s original
-   * target_name, peer_cert, or error_details after using the appropriate setter
-   * function.
-   * **/
+  /** Setters for member fields. **/
   void set_cb_user_data(void* cb_user_data);
   void set_success(int success);
   void set_target_name(const grpc::string& target_name);

include/grpcpp/server_impl.h

@@ -385,6 +385,11 @@ class Server : public grpc::ServerInterface, private grpc::GrpcLibraryCodegen {
   // shutdown callback tag (invoked when the CQ is fully shutdown).
   // It is protected by mu_
   CompletionQueue* callback_cq_ = nullptr;
+
+  // List of CQs passed in by user that must be Shutdown only after Server is
+  // Shutdown.  Even though this is only used with NDEBUG, instantiate it in all
+  // cases since otherwise the size will be inconsistent.
+  std::vector<CompletionQueue*> cq_list_;
 };
 
 }  // namespace grpc_impl

package.xml

@@ -56,6 +56,7 @@
     <file baseinstalldir="/" name="include/grpc/impl/codegen/slice.h" role="src" />
     <file baseinstalldir="/" name="include/grpc/impl/codegen/status.h" role="src" />
     <file baseinstalldir="/" name="include/grpc/impl/codegen/sync.h" role="src" />
+    <file baseinstalldir="/" name="include/grpc/impl/codegen/sync_abseil.h" role="src" />
     <file baseinstalldir="/" name="include/grpc/impl/codegen/sync_custom.h" role="src" />
     <file baseinstalldir="/" name="include/grpc/impl/codegen/sync_generic.h" role="src" />
     <file baseinstalldir="/" name="include/grpc/impl/codegen/sync_posix.h" role="src" />
@@ -75,6 +76,7 @@
     <file baseinstalldir="/" name="include/grpc/support/port_platform.h" role="src" />
     <file baseinstalldir="/" name="include/grpc/support/string_util.h" role="src" />
     <file baseinstalldir="/" name="include/grpc/support/sync.h" role="src" />
+    <file baseinstalldir="/" name="include/grpc/support/sync_abseil.h" role="src" />
     <file baseinstalldir="/" name="include/grpc/support/sync_custom.h" role="src" />
     <file baseinstalldir="/" name="include/grpc/support/sync_generic.h" role="src" />
     <file baseinstalldir="/" name="include/grpc/support/sync_posix.h" role="src" />
@@ -436,6 +438,7 @@
     <file baseinstalldir="/" name="src/core/lib/gpr/string_windows.cc" role="src" />
     <file baseinstalldir="/" name="src/core/lib/gpr/string_windows.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/gpr/sync.cc" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gpr/sync_abseil.cc" role="src" />
     <file baseinstalldir="/" name="src/core/lib/gpr/sync_posix.cc" role="src" />
     <file baseinstalldir="/" name="src/core/lib/gpr/sync_windows.cc" role="src" />
     <file baseinstalldir="/" name="src/core/lib/gpr/time.cc" role="src" />

setup.py

@@ -241,7 +241,7 @@ if "linux" in sys.platform:
 if not "win32" in sys.platform:
   EXTENSION_LIBRARIES += ('m',)
 if "win32" in sys.platform:
-  EXTENSION_LIBRARIES += ('advapi32', 'ws2_32',)
+  EXTENSION_LIBRARIES += ('advapi32', 'ws2_32', 'dbghelp',)
 if BUILD_WITH_SYSTEM_OPENSSL:
   EXTENSION_LIBRARIES += ('ssl', 'crypto',)
 if BUILD_WITH_SYSTEM_ZLIB:

src/abseil-cpp/gen_build_yaml.py

@@ -26,4 +26,5 @@ for build in builds:
     build['build'] = 'private'
     build['build_system'] = []
     build['language'] = 'c'
+    build['secure'] = False
 print(yaml.dump({'libs': builds}))

src/core/ext/filters/client_channel/xds/xds_api.cc

@@ -182,7 +182,8 @@ void PopulateMetadataValue(upb_arena* arena, google_protobuf_Value* value_pb,
 }
 
 void PopulateNode(upb_arena* arena, const XdsBootstrap::Node* node,
-                  const char* build_version, envoy_api_v2_core_Node* node_msg) {
+                  const char* build_version, const std::string& server_name,
+                  envoy_api_v2_core_Node* node_msg) {
   if (node != nullptr) {
     if (!node->id.empty()) {
       envoy_api_v2_core_Node_set_id(node_msg,
@@ -197,6 +198,18 @@ void PopulateNode(upb_arena* arena, const XdsBootstrap::Node* node,
           envoy_api_v2_core_Node_mutable_metadata(node_msg, arena);
       PopulateMetadata(arena, metadata, node->metadata.object_value());
     }
+    if (!server_name.empty()) {
+      google_protobuf_Struct* metadata =
+          envoy_api_v2_core_Node_mutable_metadata(node_msg, arena);
+      google_protobuf_Struct_FieldsEntry* field =
+          google_protobuf_Struct_add_fields(metadata, arena);
+      google_protobuf_Struct_FieldsEntry_set_key(
+          field, upb_strview_makez("PROXYLESS_CLIENT_HOSTNAME"));
+      google_protobuf_Value* value =
+          google_protobuf_Struct_FieldsEntry_mutable_value(field, arena);
+      google_protobuf_Value_set_string_value(
+          value, upb_strview_make(server_name.data(), server_name.size()));
+    }
     if (!node->locality_region.empty() || !node->locality_zone.empty() ||
         !node->locality_subzone.empty()) {
       envoy_api_v2_core_Locality* locality =
@@ -257,7 +270,7 @@ envoy_api_v2_DiscoveryRequest* CreateDiscoveryRequest(
   if (build_version != nullptr) {
     envoy_api_v2_core_Node* node_msg =
         envoy_api_v2_DiscoveryRequest_mutable_node(request, arena);
-    PopulateNode(arena, node, build_version, node_msg);
+    PopulateNode(arena, node, build_version, "", node_msg);
   }
   return request;
 }
@@ -957,15 +970,7 @@ grpc_slice XdsApi::CreateLrsInitialRequest(const std::string& server_name) {
   envoy_api_v2_core_Node* node_msg =
       envoy_service_load_stats_v2_LoadStatsRequest_mutable_node(request,
                                                                 arena.ptr());
-  PopulateNode(arena.ptr(), node_, build_version_, node_msg);
-  // Add cluster stats. There is only one because we only use one server name in
-  // one channel.
-  envoy_api_v2_endpoint_ClusterStats* cluster_stats =
-      envoy_service_load_stats_v2_LoadStatsRequest_add_cluster_stats(
-          request, arena.ptr());
-  // Set the cluster name.
-  envoy_api_v2_endpoint_ClusterStats_set_cluster_name(
-      cluster_stats, upb_strview_makez(server_name.c_str()));
+  PopulateNode(arena.ptr(), node_, build_version_, server_name, node_msg);
   return SerializeLrsRequest(request, arena.ptr());
 }
 

src/core/ext/filters/client_channel/xds/xds_client.cc

@@ -1751,8 +1751,16 @@ XdsClient::~XdsClient() { GRPC_COMBINER_UNREF(combiner_, "xds_client"); }
 void XdsClient::Orphan() {
   shutting_down_ = true;
   chand_.reset();
-  cluster_map_.clear();
-  endpoint_map_.clear();
+  // We do not clear cluster_map_ and endpoint_map_ if the xds client was
+  // created by the XdsResolver because the maps contain refs for watchers which
+  // in turn hold refs to the loadbalancing policies. At this point, it is
+  // possible for ADS calls to be in progress. Unreffing the loadbalancing
+  // policies before those calls are done would lead to issues such as
+  // https://github.com/grpc/grpc/issues/20928.
+  if (service_config_watcher_ != nullptr) {
+    cluster_map_.clear();
+    endpoint_map_.clear();
+  }
   Unref(DEBUG_LOCATION, "XdsClient::Orphan()");
 }
 
@@ -1902,13 +1910,13 @@ void XdsClient::NotifyOnError(grpc_error* error) {
 
 void* XdsClient::ChannelArgCopy(void* p) {
   XdsClient* xds_client = static_cast<XdsClient*>(p);
-  xds_client->Ref().release();
+  xds_client->Ref(DEBUG_LOCATION, "channel arg").release();
   return p;
 }
 
 void XdsClient::ChannelArgDestroy(void* p) {
   XdsClient* xds_client = static_cast<XdsClient*>(p);
-  xds_client->Unref();
+  xds_client->Unref(DEBUG_LOCATION, "channel arg");
 }
 
 int XdsClient::ChannelArgCmp(void* p, void* q) { return GPR_ICMP(p, q); }

src/core/ext/transport/inproc/inproc_transport.cc

@@ -51,7 +51,8 @@ grpc_slice g_fake_auth_value;
 
 struct inproc_stream;
 bool cancel_stream_locked(inproc_stream* s, grpc_error* error);
-void op_state_machine(void* arg, grpc_error* error);
+void maybe_process_ops_locked(inproc_stream* s, grpc_error* error);
+void op_state_machine_locked(inproc_stream* s, grpc_error* error);
 void log_metadata(const grpc_metadata_batch* md_batch, bool is_client,
                   bool is_initial);
 grpc_error* fill_in_metadata(inproc_stream* s,
@@ -130,8 +131,6 @@ struct inproc_stream {
 
     grpc_metadata_batch_init(&to_read_initial_md);
     grpc_metadata_batch_init(&to_read_trailing_md);
-    GRPC_CLOSURE_INIT(&op_closure, op_state_machine, this,
-                      grpc_schedule_on_exec_ctx);
     grpc_metadata_batch_init(&write_buffer_initial_md);
     grpc_metadata_batch_init(&write_buffer_trailing_md);
 
@@ -186,6 +185,7 @@ struct inproc_stream {
       if (cs->write_buffer_cancel_error != GRPC_ERROR_NONE) {
         cancel_other_error = cs->write_buffer_cancel_error;
         cs->write_buffer_cancel_error = GRPC_ERROR_NONE;
+        maybe_process_ops_locked(this, cancel_other_error);
       }
 
       gpr_mu_unlock(&t->mu->mu);
@@ -235,8 +235,6 @@ struct inproc_stream {
   grpc_metadata_batch to_read_trailing_md;
   bool to_read_trailing_md_filled = false;
   bool ops_needed = false;
-  bool op_closure_scheduled = false;
-  grpc_closure op_closure;
   // Write buffer used only during gap at init time when client-side
   // stream is set up but server side stream is not yet set up
   grpc_metadata_batch write_buffer_initial_md;
@@ -396,12 +394,10 @@ void complete_if_batch_end_locked(inproc_stream* s, grpc_error* error,
   }
 }
 
-void maybe_schedule_op_closure_locked(inproc_stream* s, grpc_error* error) {
-  if (s && s->ops_needed && !s->op_closure_scheduled) {
-    grpc_core::ExecCtx::Run(DEBUG_LOCATION, &s->op_closure,
-                            GRPC_ERROR_REF(error));
-    s->op_closure_scheduled = true;
+void maybe_process_ops_locked(inproc_stream* s, grpc_error* error) {
+  if (s && (error != GRPC_ERROR_NONE || s->ops_needed)) {
     s->ops_needed = false;
+    op_state_machine_locked(s, error);
   }
 }
 
@@ -429,7 +425,7 @@ void fail_helper_locked(inproc_stream* s, grpc_error* error) {
       if (other->cancel_other_error == GRPC_ERROR_NONE) {
         other->cancel_other_error = GRPC_ERROR_REF(error);
       }
-      maybe_schedule_op_closure_locked(other, error);
+      maybe_process_ops_locked(other, error);
     } else if (s->write_buffer_cancel_error == GRPC_ERROR_NONE) {
       s->write_buffer_cancel_error = GRPC_ERROR_REF(error);
     }
@@ -587,23 +583,17 @@ void message_transfer_locked(inproc_stream* sender, inproc_stream* receiver) {
   sender->send_message_op = nullptr;
 }
 
-void op_state_machine(void* arg, grpc_error* error) {
+void op_state_machine_locked(inproc_stream* s, grpc_error* error) {
   // This function gets called when we have contents in the unprocessed reads
   // Get what we want based on our ops wanted
   // Schedule our appropriate closures
   // and then return to ops_needed state if still needed
 
-  // Since this is a closure directly invoked by the combiner, it should not
-  // unref the error parameter explicitly; the combiner will do that implicitly
   grpc_error* new_err = GRPC_ERROR_NONE;
 
   bool needs_close = false;
 
-  INPROC_LOG(GPR_INFO, "op_state_machine %p", arg);
-  inproc_stream* s = static_cast<inproc_stream*>(arg);
-  gpr_mu* mu = &s->t->mu->mu;  // keep aside in case s gets closed
-  gpr_mu_lock(mu);
-  s->op_closure_scheduled = false;
+  INPROC_LOG(GPR_INFO, "op_state_machine %p", s);
   // cancellation takes precedence
   inproc_stream* other = s->other_side;
 
@@ -621,7 +611,7 @@ void op_state_machine(void* arg, grpc_error* error) {
   if (s->send_message_op && other) {
     if (other->recv_message_op) {
       message_transfer_locked(s, other);
-      maybe_schedule_op_closure_locked(other, GRPC_ERROR_NONE);
+      maybe_process_ops_locked(other, GRPC_ERROR_NONE);
     } else if (!s->t->is_client && s->trailing_md_sent) {
       // A server send will never be matched if the server already sent status
       s->send_message_op->payload->send_message.send_message.reset();
@@ -679,7 +669,7 @@ void op_state_machine(void* arg, grpc_error* error) {
         needs_close = true;
       }
     }
-    maybe_schedule_op_closure_locked(other, GRPC_ERROR_NONE);
+    maybe_process_ops_locked(other, GRPC_ERROR_NONE);
     complete_if_batch_end_locked(
         s, GRPC_ERROR_NONE, s->send_trailing_md_op,
         "op_state_machine scheduling send-trailing-metadata-on-complete");
@@ -741,7 +731,7 @@ void op_state_machine(void* arg, grpc_error* error) {
   if (s->recv_message_op) {
     if (other && other->send_message_op) {
       message_transfer_locked(other, s);
-      maybe_schedule_op_closure_locked(other, GRPC_ERROR_NONE);
+      maybe_process_ops_locked(other, GRPC_ERROR_NONE);
     }
   }
   if (s->to_read_trailing_md_filled) {
@@ -808,7 +798,7 @@ void op_state_machine(void* arg, grpc_error* error) {
                                 s->recv_trailing_md_op->on_complete,
                                 GRPC_ERROR_REF(new_err));
         s->recv_trailing_md_op = nullptr;
-        needs_close = true;
+        needs_close = s->trailing_md_sent;
       } else {
         INPROC_LOG(GPR_INFO,
                    "op_state_machine %p server needs to delay handling "
@@ -860,7 +850,6 @@ done:
     close_other_side_locked(s, "op_state_machine");
     close_stream_locked(s);
   }
-  gpr_mu_unlock(mu);
   GRPC_ERROR_UNREF(new_err);
 }
 
@@ -870,7 +859,9 @@ bool cancel_stream_locked(inproc_stream* s, grpc_error* error) {
   if (s->cancel_self_error == GRPC_ERROR_NONE) {
     ret = true;
     s->cancel_self_error = GRPC_ERROR_REF(error);
-    maybe_schedule_op_closure_locked(s, s->cancel_self_error);
+    // Catch current value of other before it gets closed off
+    inproc_stream* other = s->other_side;
+    maybe_process_ops_locked(s, s->cancel_self_error);
     // Send trailing md to the other side indicating cancellation, even if we
     // already have
     s->trailing_md_sent = true;
@@ -878,7 +869,6 @@ bool cancel_stream_locked(inproc_stream* s, grpc_error* error) {
     grpc_metadata_batch cancel_md;
     grpc_metadata_batch_init(&cancel_md);
 
-    inproc_stream* other = s->other_side;
     grpc_metadata_batch* dest = (other == nullptr)
                                     ? &s->write_buffer_trailing_md
                                     : &other->to_read_trailing_md;
@@ -891,7 +881,7 @@ bool cancel_stream_locked(inproc_stream* s, grpc_error* error) {
       if (other->cancel_other_error == GRPC_ERROR_NONE) {
         other->cancel_other_error = GRPC_ERROR_REF(s->cancel_self_error);
       }
-      maybe_schedule_op_closure_locked(other, other->cancel_other_error);
+      maybe_process_ops_locked(other, other->cancel_other_error);
     } else if (s->write_buffer_cancel_error == GRPC_ERROR_NONE) {
       s->write_buffer_cancel_error = GRPC_ERROR_REF(s->cancel_self_error);
     }
@@ -969,8 +959,6 @@ void perform_stream_op(grpc_transport* gt, grpc_stream* gs,
                op->recv_trailing_metadata ? " recv_trailing_metadata" : "");
   }
 
-  bool needs_close = false;
-
   inproc_stream* other = s->other_side;
   if (error == GRPC_ERROR_NONE &&
       (op->send_initial_metadata || op->send_trailing_metadata)) {
@@ -991,7 +979,7 @@ void perform_stream_op(grpc_transport* gt, grpc_stream* gs,
         INPROC_LOG(GPR_INFO, "Extra initial metadata %p", s);
         error = GRPC_ERROR_CREATE_FROM_STATIC_STRING("Extra initial metadata");
       } else {
-        if (!other || !other->closed) {
+        if (!s->other_side_closed) {
           fill_in_metadata(
               s, op->payload->send_initial_metadata.send_initial_metadata,
               op->payload->send_initial_metadata.send_initial_metadata_flags,
@@ -1005,7 +993,7 @@ void perform_stream_op(grpc_transport* gt, grpc_stream* gs,
           s->initial_md_sent = true;
         }
       }
-      maybe_schedule_op_closure_locked(other, error);
+      maybe_process_ops_locked(other, error);
     }
   }
 
@@ -1013,7 +1001,7 @@ void perform_stream_op(grpc_transport* gt, grpc_stream* gs,
       (op->send_message || op->send_trailing_metadata ||
        op->recv_initial_metadata || op->recv_message ||
        op->recv_trailing_metadata)) {
-    // Mark ops that need to be processed by the closure
+    // Mark ops that need to be processed by the state machine
     if (op->send_message) {
       s->send_message_op = op;
     }
@@ -1030,7 +1018,7 @@ void perform_stream_op(grpc_transport* gt, grpc_stream* gs,
       s->recv_trailing_md_op = op;
     }
 
-    // We want to initiate the closure if:
+    // We want to initiate the state machine if:
     // 1. We want to send a message and the other side wants to receive
     // 2. We want to send trailing metadata and there isn't an unmatched send
     //    or the other side wants trailing metadata
@@ -1044,11 +1032,7 @@ void perform_stream_op(grpc_transport* gt, grpc_stream* gs,
         (op->recv_initial_metadata && s->to_read_initial_md_filled) ||
         (op->recv_message && other && other->send_message_op != nullptr) ||
         (s->to_read_trailing_md_filled || s->trailing_md_recvd)) {
-      if (!s->op_closure_scheduled) {
-        grpc_core::ExecCtx::Run(DEBUG_LOCATION, &s->op_closure,
-                                GRPC_ERROR_NONE);
-        s->op_closure_scheduled = true;
-      }
+      op_state_machine_locked(s, error);
     } else {
       s->ops_needed = true;
     }
@@ -1103,10 +1087,6 @@ void perform_stream_op(grpc_transport* gt, grpc_stream* gs,
                error);
     grpc_core::ExecCtx::Run(DEBUG_LOCATION, on_complete, GRPC_ERROR_REF(error));
   }
-  if (needs_close) {
-    close_other_side_locked(s, "perform_stream_op:other_side");
-    close_stream_locked(s);
-  }
   gpr_mu_unlock(mu);
   GRPC_ERROR_UNREF(error);
 }

src/core/lib/gpr/sync_abseil.cc

@@ -0,0 +1,114 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include <grpc/support/port_platform.h>
+
+#if defined(GPR_ABSEIL_SYNC) && !defined(GPR_CUSTOM_SYNC)
+
+#include <grpc/support/alloc.h>
+
+#include <errno.h>
+#include <grpc/support/log.h>
+#include <grpc/support/sync.h>
+#include <grpc/support/time.h>
+#include <time.h>
+#include "src/core/lib/profiling/timers.h"
+
+#include "absl/base/call_once.h"
+#include "absl/synchronization/mutex.h"
+#include "absl/time/clock.h"
+#include "absl/time/time.h"
+
+#ifdef GPR_LOW_LEVEL_COUNTERS
+gpr_atm gpr_mu_locks = 0;
+gpr_atm gpr_counter_atm_cas = 0;
+gpr_atm gpr_counter_atm_add = 0;
+#endif
+
+void gpr_mu_init(gpr_mu* mu) {
+  static_assert(sizeof(gpr_mu) == sizeof(absl::Mutex),
+                "gpr_mu and Mutex must be the same size");
+  new (mu) absl::Mutex;
+}
+
+void gpr_mu_destroy(gpr_mu* mu) {
+  reinterpret_cast<absl::Mutex*>(mu)->~Mutex();
+}
+
+void gpr_mu_lock(gpr_mu* mu) ABSL_NO_THREAD_SAFETY_ANALYSIS {
+  GPR_TIMER_SCOPE("gpr_mu_lock", 0);
+  reinterpret_cast<absl::Mutex*>(mu)->Lock();
+}
+
+void gpr_mu_unlock(gpr_mu* mu) ABSL_NO_THREAD_SAFETY_ANALYSIS {
+  GPR_TIMER_SCOPE("gpr_mu_unlock", 0);
+  reinterpret_cast<absl::Mutex*>(mu)->Unlock();
+}
+
+int gpr_mu_trylock(gpr_mu* mu) {
+  GPR_TIMER_SCOPE("gpr_mu_trylock", 0);
+  int ret = reinterpret_cast<absl::Mutex*>(mu)->TryLock() == true;
+  return ret;
+}
+
+/*----------------------------------------*/
+
+void gpr_cv_init(gpr_cv* cv) {
+  static_assert(sizeof(gpr_cv) == sizeof(absl::CondVar),
+                "gpr_cv and CondVar must be the same size");
+  new (cv) absl::CondVar;
+}
+
+void gpr_cv_destroy(gpr_cv* cv) {
+  reinterpret_cast<absl::CondVar*>(cv)->~CondVar();
+}
+
+int gpr_cv_wait(gpr_cv* cv, gpr_mu* mu, gpr_timespec abs_deadline) {
+  GPR_TIMER_SCOPE("gpr_cv_wait", 0);
+  if (gpr_time_cmp(abs_deadline, gpr_inf_future(abs_deadline.clock_type)) ==
+      0) {
+    reinterpret_cast<absl::CondVar*>(cv)->Wait(
+        reinterpret_cast<absl::Mutex*>(mu));
+    return 0;
+  }
+  abs_deadline = gpr_convert_clock_type(abs_deadline, GPR_CLOCK_REALTIME);
+  timespec ts = {static_cast<decltype(ts.tv_sec)>(abs_deadline.tv_sec),
+                 static_cast<decltype(ts.tv_nsec)>(abs_deadline.tv_nsec)};
+  int ret = reinterpret_cast<absl::CondVar*>(cv)->WaitWithDeadline(
+                reinterpret_cast<absl::Mutex*>(mu),
+                absl::TimeFromTimespec(ts)) == true;
+  return ret;
+}
+
+void gpr_cv_signal(gpr_cv* cv) {
+  GPR_TIMER_MARK("gpr_cv_signal", 0);
+  reinterpret_cast<absl::CondVar*>(cv)->Signal();
+}
+
+void gpr_cv_broadcast(gpr_cv* cv) {
+  GPR_TIMER_MARK("gpr_cv_broadcast", 0);
+  reinterpret_cast<absl::CondVar*>(cv)->SignalAll();
+}
+
+/*----------------------------------------*/
+
+void gpr_once_init(gpr_once* once, void (*init_function)(void)) {
+  absl::call_once(*reinterpret_cast<absl::once_flag*>(once), init_function);
+}
+
+#endif /* defined(GPR_ABSEIL_SYNC) && !defined(GPR_CUSTOM_SYNC) */

src/core/lib/gpr/sync_posix.cc

@@ -18,15 +18,17 @@
 
 #include <grpc/support/port_platform.h>
 
-#include <grpc/support/alloc.h>
-
-#ifdef GPR_POSIX_SYNC
+#if defined(GPR_POSIX_SYNC) && !defined(GPR_ABSEIL_SYNC) && \
+    !defined(GPR_CUSTOM_SYNC)
 
-#include <errno.h>
+#include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 #include <grpc/support/sync.h>
 #include <grpc/support/time.h>
+
+#include <errno.h>
 #include <time.h>
+
 #include "src/core/lib/profiling/timers.h"
 
 #ifdef GPR_LOW_LEVEL_COUNTERS
@@ -170,4 +172,5 @@ void gpr_once_init(gpr_once* once, void (*init_function)(void)) {
   GPR_ASSERT(pthread_once(once, init_function) == 0);
 }
 
-#endif /* GRP_POSIX_SYNC */
+#endif /* defined(GPR_POSIX_SYNC) && !defined(GPR_ABSEIL_SYNC) && \
+          !defined(GPR_CUSTOM_SYNC) */

src/core/lib/gpr/sync_windows.cc

@@ -20,7 +20,8 @@
 
 #include <grpc/support/port_platform.h>
 
-#ifdef GPR_WINDOWS
+#if defined(GPR_WINDOWS) && !defined(GPR_ABSEIL_SYNC) && \
+    !defined(GPR_CUSTOM_SYNC)
 
 #include <grpc/support/log.h>
 #include <grpc/support/sync.h>
@@ -115,4 +116,5 @@ void gpr_once_init(gpr_once* once, void (*init_function)(void)) {
   InitOnceExecuteOnce(once, run_once_func, &arg, &dummy);
 }
 
-#endif /* GPR_WINDOWS */
+#endif /* defined(GPR_WINDOWS) && !defined(GPR_ABSEIL_SYNC) && \
+          !defined(GPR_CUSTOM_SYNC) */

src/core/lib/iomgr/tcp_posix.cc

@@ -1385,7 +1385,8 @@ static bool do_tcp_flush_zerocopy(grpc_tcp* tcp, TcpZerocopySendRecord* record,
 
 static void UnrefMaybePutZerocopySendRecord(grpc_tcp* tcp,
                                             TcpZerocopySendRecord* record,
-                                            uint32_t seq, const char* tag) {
+                                            uint32_t seq,
+                                            const char* /* tag */) {
   if (record->Unref()) {
     tcp->tcp_zerocopy_send_ctx.PutSendRecord(record);
   }

src/core/lib/json/json_reader.cc

@@ -25,6 +25,9 @@
 
 #include "src/core/lib/json/json.h"
 
+#define GRPC_JSON_MAX_DEPTH 255
+#define GRPC_JSON_MAX_ERRORS 16
+
 namespace grpc_core {
 
 namespace {
@@ -92,7 +95,7 @@ class JsonReader {
   void StringAddUtf32(uint32_t c);
 
   Json* CreateAndLinkValue();
-  void StartContainer(Json::Type type);
+  bool StartContainer(Json::Type type);
   void EndContainer();
   void SetKey();
   void SetString();
@@ -111,6 +114,7 @@ class JsonReader {
   uint16_t unicode_char_ = 0;
   uint16_t unicode_high_surrogate_ = 0;
   std::vector<grpc_error*> errors_;
+  bool truncated_errors_ = false;
 
   Json root_value_;
   std::vector<Json*> stack_;
@@ -169,11 +173,15 @@ Json* JsonReader::CreateAndLinkValue() {
     Json* parent = stack_.back();
     if (parent->type() == Json::Type::OBJECT) {
       if (parent->object_value().find(key_) != parent->object_value().end()) {
-        char* msg;
-        gpr_asprintf(&msg, "duplicate key \"%s\" at index %" PRIuPTR,
-                     key_.c_str(), CurrentIndex());
-        errors_.push_back(GRPC_ERROR_CREATE_FROM_COPIED_STRING(msg));
-        gpr_free(msg);
+        if (errors_.size() == GRPC_JSON_MAX_ERRORS) {
+          truncated_errors_ = true;
+        } else {
+          char* msg;
+          gpr_asprintf(&msg, "duplicate key \"%s\" at index %" PRIuPTR,
+                       key_.c_str(), CurrentIndex());
+          errors_.push_back(GRPC_ERROR_CREATE_FROM_COPIED_STRING(msg));
+          gpr_free(msg);
+        }
       }
       value = &(*parent->mutable_object())[std::move(key_)];
     } else {
@@ -185,7 +193,19 @@ Json* JsonReader::CreateAndLinkValue() {
   return value;
 }
 
-void JsonReader::StartContainer(Json::Type type) {
+bool JsonReader::StartContainer(Json::Type type) {
+  if (stack_.size() == GRPC_JSON_MAX_DEPTH) {
+    if (errors_.size() == GRPC_JSON_MAX_ERRORS) {
+      truncated_errors_ = true;
+    } else {
+      char* msg;
+      gpr_asprintf(&msg, "exceeded max stack depth (%d) at index %" PRIuPTR,
+                   GRPC_JSON_MAX_DEPTH, CurrentIndex());
+      errors_.push_back(GRPC_ERROR_CREATE_FROM_COPIED_STRING(msg));
+      gpr_free(msg);
+    }
+    return false;
+  }
   Json* value = CreateAndLinkValue();
   if (type == Json::Type::OBJECT) {
     *value = Json::Object();
@@ -194,6 +214,7 @@ void JsonReader::StartContainer(Json::Type type) {
     *value = Json::Array();
   }
   stack_.push_back(value);
+  return true;
 }
 
 void JsonReader::EndContainer() {
@@ -483,13 +504,17 @@ JsonReader::Status JsonReader::Run() {
 
               case '{':
                 container_just_begun_ = true;
-                StartContainer(Json::Type::OBJECT);
+                if (!StartContainer(Json::Type::OBJECT)) {
+                  return Status::GRPC_JSON_PARSE_ERROR;
+                }
                 state_ = State::GRPC_JSON_STATE_OBJECT_KEY_BEGIN;
                 break;
 
               case '[':
                 container_just_begun_ = true;
-                StartContainer(Json::Type::ARRAY);
+                if (!StartContainer(Json::Type::ARRAY)) {
+                  return Status::GRPC_JSON_PARSE_ERROR;
+                }
                 break;
               default:
                 return Status::GRPC_JSON_PARSE_ERROR;
@@ -793,6 +818,11 @@ JsonReader::Status JsonReader::Run() {
 grpc_error* JsonReader::Parse(StringView input, Json* output) {
   JsonReader reader(input);
   Status status = reader.Run();
+  if (reader.truncated_errors_) {
+    reader.errors_.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING(
+        "too many errors encountered during JSON parsing -- fix reported "
+        "errors and try again to see additional errors"));
+  }
   if (status == Status::GRPC_JSON_INTERNAL_ERROR) {
     char* msg;
     gpr_asprintf(&msg, "internal error in JSON parser at index %" PRIuPTR,

src/core/lib/security/credentials/alts/check_gcp_environment.cc

@@ -30,7 +30,7 @@
 const size_t kBiosDataBufferSize = 256;
 
 static char* trim(const char* src) {
-  if (src == nullptr) {
+  if (src == nullptr || *src == '\0') {
     return nullptr;
   }
   char* des = nullptr;

src/core/lib/security/security_connector/local/local_security_connector.cc

@@ -66,8 +66,7 @@ grpc_core::RefCountedPtr<grpc_auth_context> local_auth_context_create(
   return ctx;
 }
 
-void local_check_peer(grpc_security_connector* sc, tsi_peer peer,
-                      grpc_endpoint* ep,
+void local_check_peer(tsi_peer peer, grpc_endpoint* ep,
                       grpc_core::RefCountedPtr<grpc_auth_context>* auth_context,
                       grpc_closure* on_peer_checked,
                       grpc_local_connect_type type) {
@@ -178,7 +177,7 @@ class grpc_local_channel_security_connector final
                   grpc_closure* on_peer_checked) override {
     grpc_local_credentials* creds =
         reinterpret_cast<grpc_local_credentials*>(mutable_channel_creds());
-    local_check_peer(this, peer, ep, auth_context, on_peer_checked,
+    local_check_peer(peer, ep, auth_context, on_peer_checked,
                      creds->connect_type());
   }
 
@@ -227,7 +226,7 @@ class grpc_local_server_security_connector final
                   grpc_closure* on_peer_checked) override {
     grpc_local_server_credentials* creds =
         static_cast<grpc_local_server_credentials*>(mutable_server_creds());
-    local_check_peer(this, peer, ep, auth_context, on_peer_checked,
+    local_check_peer(peer, ep, auth_context, on_peer_checked,
                      creds->connect_type());
   }
 

src/core/tsi/alts/handshaker/alts_handshaker_client.cc

@@ -632,7 +632,7 @@ static void handshaker_client_shutdown(alts_handshaker_client* c) {
   }
 }
 
-static void handshaker_call_unref(void* arg, grpc_error* error) {
+static void handshaker_call_unref(void* arg, grpc_error* /* error */) {
   grpc_call* call = static_cast<grpc_call*>(arg);
   grpc_call_unref(call);
 }

src/core/tsi/alts/handshaker/alts_tsi_handshaker.cc

@@ -430,7 +430,7 @@ struct alts_tsi_handshaker_continue_handshaker_next_args {
 };
 
 static void alts_tsi_handshaker_create_channel(void* arg,
-                                               grpc_error* unused_error) {
+                                               grpc_error* /* unused_error */) {
   alts_tsi_handshaker_continue_handshaker_next_args* next_args =
       static_cast<alts_tsi_handshaker_continue_handshaker_next_args*>(arg);
   alts_tsi_handshaker* handshaker = next_args->handshaker;

src/cpp/common/completion_queue_cc.cc

@@ -39,6 +39,12 @@ CompletionQueue::CompletionQueue(grpc_completion_queue* take)
 
 void CompletionQueue::Shutdown() {
   g_gli_initializer.summon();
+#ifndef NDEBUG
+  if (!ServerListEmpty()) {
+    gpr_log(GPR_ERROR,
+            "CompletionQueue shutdown being shutdown before its server.");
+  }
+#endif
   CompleteAvalanching();
 }
 

src/cpp/common/tls_credentials_options.cc

@@ -16,19 +16,18 @@
  *
  */
 
+#include <grpc/support/alloc.h>
 #include <grpcpp/security/tls_credentials_options.h>
 #include "src/core/lib/security/credentials/tls/grpc_tls_credentials_options.h"
-
-#include <grpc/support/alloc.h>
-
 #include "src/cpp/common/tls_credentials_options_util.h"
 
 namespace grpc_impl {
 namespace experimental {
 
 /** TLS key materials config API implementation **/
-void TlsKeyMaterialsConfig::set_pem_root_certs(grpc::string pem_root_certs) {
-  pem_root_certs_ = std::move(pem_root_certs);
+void TlsKeyMaterialsConfig::set_pem_root_certs(
+    const grpc::string& pem_root_certs) {
+  pem_root_certs_ = pem_root_certs;
 }
 
 void TlsKeyMaterialsConfig::add_pem_key_cert_pair(
@@ -37,10 +36,10 @@ void TlsKeyMaterialsConfig::add_pem_key_cert_pair(
 }
 
 void TlsKeyMaterialsConfig::set_key_materials(
-    grpc::string pem_root_certs,
-    std::vector<PemKeyCertPair> pem_key_cert_pair_list) {
-  pem_key_cert_pair_list_ = std::move(pem_key_cert_pair_list);
-  pem_root_certs_ = std::move(pem_root_certs);
+    const grpc::string& pem_root_certs,
+    const std::vector<PemKeyCertPair>& pem_key_cert_pair_list) {
+  pem_key_cert_pair_list_ = pem_key_cert_pair_list;
+  pem_root_certs_ = pem_root_certs;
 }
 
 /** TLS credential reload arg API implementation **/
@@ -59,7 +58,6 @@ TlsCredentialReloadArg::~TlsCredentialReloadArg() {}
 void* TlsCredentialReloadArg::cb_user_data() const {
   return c_arg_->cb_user_data;
 }
-
 bool TlsCredentialReloadArg::is_pem_key_cert_pair_list_empty() const {
   return c_arg_->key_materials_config->pem_key_cert_pair_list().empty();
 }
@@ -85,17 +83,46 @@ void TlsCredentialReloadArg::set_pem_root_certs(
   c_arg_->key_materials_config->set_pem_root_certs(std::move(c_pem_root_certs));
 }
 
-void TlsCredentialReloadArg::add_pem_key_cert_pair(
-    TlsKeyMaterialsConfig::PemKeyCertPair pem_key_cert_pair) {
+namespace {
+
+::grpc_core::PemKeyCertPair ConvertToCorePemKeyCertPair(
+    const TlsKeyMaterialsConfig::PemKeyCertPair& pem_key_cert_pair) {
   grpc_ssl_pem_key_cert_pair* ssl_pair =
       (grpc_ssl_pem_key_cert_pair*)gpr_malloc(
           sizeof(grpc_ssl_pem_key_cert_pair));
   ssl_pair->private_key = gpr_strdup(pem_key_cert_pair.private_key.c_str());
   ssl_pair->cert_chain = gpr_strdup(pem_key_cert_pair.cert_chain.c_str());
-  ::grpc_core::PemKeyCertPair c_pem_key_cert_pair =
-      ::grpc_core::PemKeyCertPair(ssl_pair);
+  return ::grpc_core::PemKeyCertPair(ssl_pair);
+}
+
+}  //  namespace
+
+void TlsCredentialReloadArg::add_pem_key_cert_pair(
+    const TlsKeyMaterialsConfig::PemKeyCertPair& pem_key_cert_pair) {
   c_arg_->key_materials_config->add_pem_key_cert_pair(
-      std::move(c_pem_key_cert_pair));
+      ConvertToCorePemKeyCertPair(pem_key_cert_pair));
+}
+
+void TlsCredentialReloadArg::set_key_materials(
+    const grpc::string& pem_root_certs,
+    std::vector<TlsKeyMaterialsConfig::PemKeyCertPair> pem_key_cert_pair_list) {
+  /** Initialize the |key_materials_config| field of |c_arg_|, if it has not
+   *  already been done. **/
+  if (c_arg_->key_materials_config == nullptr) {
+    c_arg_->key_materials_config = grpc_tls_key_materials_config_create();
+  }
+  /** Convert |pem_key_cert_pair_list| to an inlined vector of ssl pairs. **/
+  ::grpc_core::InlinedVector<::grpc_core::PemKeyCertPair, 1>
+      c_pem_key_cert_pair_list;
+  for (const auto& key_cert_pair : pem_key_cert_pair_list) {
+    c_pem_key_cert_pair_list.emplace_back(
+        ConvertToCorePemKeyCertPair(key_cert_pair));
+  }
+  /** Populate the key materials config field of |c_arg_|. **/
+  ::grpc_core::UniquePtr<char> c_pem_root_certs(
+      gpr_strdup(pem_root_certs.c_str()));
+  c_arg_->key_materials_config->set_key_materials(std::move(c_pem_root_certs),
+                                                  c_pem_key_cert_pair_list);
 }
 
 void TlsCredentialReloadArg::set_key_materials_config(
@@ -288,6 +315,11 @@ TlsCredentialsOptions::TlsCredentialsOptions(
       c_credentials_options_, server_verification_option);
 }
 
+/** Whenever a TlsCredentialsOptions instance is created, the caller takes
+ *  ownership of the c_credentials_options_ pointer (see e.g. the implementation
+ *  of the TlsCredentials API in secure_credentials.cc). For this reason, the
+ *  TlsCredentialsOptions destructor is not responsible for freeing
+ *  c_credentials_options_. **/
 TlsCredentialsOptions::~TlsCredentialsOptions() {}
 
 }  // namespace experimental

src/cpp/server/server_builder.cc

@@ -354,9 +354,8 @@ std::unique_ptr<grpc::Server> ServerBuilder::BuildAndStart() {
   //     server
   //  2. cqs_: Completion queues added via AddCompletionQueue() call
 
-  for (const auto& value : *sync_server_cqs) {
-    grpc_server_register_completion_queue(server->server_, value->cq(),
-                                          nullptr);
+  for (const auto& cq : *sync_server_cqs) {
+    grpc_server_register_completion_queue(server->server_, cq->cq(), nullptr);
     has_frequently_polled_cqs = true;
   }
 
@@ -369,10 +368,12 @@ std::unique_ptr<grpc::Server> ServerBuilder::BuildAndStart() {
   // AddCompletionQueue() API. Some of them may not be frequently polled (i.e by
   // calling Next() or AsyncNext()) and hence are not safe to be used for
   // listening to incoming channels. Such completion queues must be registered
-  // as non-listening queues
-  for (const auto& value : cqs_) {
-    grpc_server_register_completion_queue(server->server_, value->cq(),
-                                          nullptr);
+  // as non-listening queues. In debug mode, these should have their server list
+  // tracked since these are provided the user and must be Shutdown by the user
+  // after the server is shutdown.
+  for (const auto& cq : cqs_) {
+    grpc_server_register_completion_queue(server->server_, cq->cq(), nullptr);
+    cq->RegisterServer(server.get());
   }
 
   if (!has_frequently_polled_cqs) {

src/cpp/server/server_callback.cc

@@ -24,27 +24,59 @@
 namespace grpc_impl {
 namespace internal {
 
+void ServerCallbackCall::ScheduleOnDone(bool inline_ondone) {
+  if (inline_ondone) {
+    CallOnDone();
+  } else {
+    // Unlike other uses of closure, do not Ref or Unref here since at this
+    // point, all the Ref'fing and Unref'fing is done for this call.
+    grpc_core::ExecCtx exec_ctx;
+    struct ClosureWithArg {
+      grpc_closure closure;
+      ServerCallbackCall* call;
+      explicit ClosureWithArg(ServerCallbackCall* call_arg) : call(call_arg) {
+        GRPC_CLOSURE_INIT(&closure,
+                          [](void* void_arg, grpc_error*) {
+                            ClosureWithArg* arg =
+                                static_cast<ClosureWithArg*>(void_arg);
+                            arg->call->CallOnDone();
+                            delete arg;
+                          },
+                          this, grpc_schedule_on_exec_ctx);
+      }
+    };
+    ClosureWithArg* arg = new ClosureWithArg(this);
+    grpc_core::Executor::Run(&arg->closure, GRPC_ERROR_NONE);
+  }
+}
+
 void ServerCallbackCall::CallOnCancel(ServerReactor* reactor) {
   if (reactor->InternalInlineable()) {
     reactor->OnCancel();
   } else {
+    // Ref to make sure that the closure executes before the whole call gets
+    // destructed, and Unref within the closure.
     Ref();
     grpc_core::ExecCtx exec_ctx;
-    struct ClosureArg {
+    struct ClosureWithArg {
+      grpc_closure closure;
       ServerCallbackCall* call;
       ServerReactor* reactor;
+      ClosureWithArg(ServerCallbackCall* call_arg, ServerReactor* reactor_arg)
+          : call(call_arg), reactor(reactor_arg) {
+        GRPC_CLOSURE_INIT(&closure,
+                          [](void* void_arg, grpc_error*) {
+                            ClosureWithArg* arg =
+                                static_cast<ClosureWithArg*>(void_arg);
+                            arg->reactor->OnCancel();
+                            arg->call->MaybeDone();
+                            delete arg;
+                          },
+                          this, grpc_schedule_on_exec_ctx);
+      }
     };
-    ClosureArg* arg = new ClosureArg{this, reactor};
-    grpc_core::Executor::Run(GRPC_CLOSURE_CREATE(
-                                 [](void* void_arg, grpc_error*) {
-                                   ClosureArg* arg =
-                                       static_cast<ClosureArg*>(void_arg);
-                                   arg->reactor->OnCancel();
-                                   arg->call->MaybeDone();
-                                   delete arg;
-                                 },
-                                 arg, nullptr),
-                             GRPC_ERROR_NONE);
+    ClosureWithArg* arg = new ClosureWithArg(this, reactor);
+    grpc_core::Executor::Run(&arg->closure, GRPC_ERROR_NONE);
   }
 }
 

src/cpp/server/server_cc.cc

@@ -1249,6 +1249,9 @@ void Server::Start(grpc::ServerCompletionQueue** cqs, size_t num_cqs) {
     }
 
     for (size_t i = 0; i < num_cqs; i++) {
+#ifndef NDEBUG
+      cq_list_.push_back(cqs[i]);
+#endif
       if (cqs[i]->IsFrequentlyPolled()) {
         new UnimplementedAsyncRequest(this, cqs[i]);
       }
@@ -1360,6 +1363,15 @@ void Server::ShutdownInternal(gpr_timespec deadline) {
 
   shutdown_notified_ = true;
   shutdown_cv_.Broadcast();
+
+#ifndef NDEBUG
+  // Unregister this server with the CQs passed into it by the user so that
+  // those can be checked for properly-ordered shutdown.
+  for (auto* cq : cq_list_) {
+    cq->UnregisterServer(this);
+  }
+  cq_list_.clear();
+#endif
 }
 
 void Server::Wait() {

src/php/README.md

@@ -47,10 +47,10 @@ You can download the pre-compiled `grpc.dll` extension from the PECL
 
 ### Build from source
 
-Clone this repository
+Clone this repository at the [latest stable release tag](https://github.com/grpc/grpc/releases)
 
 ```sh
-$ git clone -b $(curl -L https://grpc.io/release) https://github.com/grpc/grpc
+$ git clone -b RELEASE_TAG_HERE https://github.com/grpc/grpc
 ```
 
 #### Build and install the gRPC C core library
@@ -184,7 +184,7 @@ in the future.
 You can also just build the `grpc_php_plugin` by running:
 
 ```sh
-$ git clone -b $(curl -L https://grpc.io/release) https://github.com/grpc/grpc
+$ git clone -b RELEASE_TAG_HERE https://github.com/grpc/grpc
 $ cd grpc
 $ git submodule update --init
 $ make grpc_php_plugin
@@ -246,7 +246,7 @@ $ protoc -I=. echo.proto --php_out=. --grpc_out=. \
 You will need the source code to run tests
 
 ```sh
-$ git clone -b $(curl -L https://grpc.io/release) https://github.com/grpc/grpc
+$ git clone -b RELEASE_TAG_HERE https://github.com/grpc/grpc
 $ cd grpc
 $ git submodule update --init
 ```

src/proto/grpc/core/BUILD

@@ -15,6 +15,7 @@
 licenses(["notice"])  # Apache v2
 
 load("//bazel:grpc_build_system.bzl", "grpc_package", "grpc_proto_library")
+load("//bazel:python_rules.bzl", "py_proto_library")
 
 grpc_package(
     name = "core",
@@ -25,3 +26,13 @@ grpc_proto_library(
     name = "stats_proto",
     srcs = ["stats.proto"],
 )
+
+proto_library(
+    name = "stats_descriptor",
+    srcs = ["stats.proto"],
+)
+
+py_proto_library(
+    name = "stats_py_pb2",
+    deps = [":stats_descriptor"],
+)

src/proto/grpc/testing/BUILD

@@ -233,3 +233,55 @@ py_grpc_library(
     srcs = [":test_proto_descriptor"],
     deps = [":py_test_proto"],
 )
+
+proto_library(
+    name = "worker_service_descriptor",
+    srcs = ["worker_service.proto"],
+    deps = [":control_descriptor"],
+)
+
+py_proto_library(
+    name = "worker_service_py_pb2",
+    deps = [":worker_service_descriptor"],
+)
+
+py_grpc_library(
+    name = "worker_service_py_pb2_grpc",
+    srcs = [":worker_service_descriptor"],
+    deps = [":worker_service_py_pb2"],
+)
+
+proto_library(
+    name = "stats_descriptor",
+    srcs = ["stats.proto"],
+    deps = ["//src/proto/grpc/core:stats_descriptor"],
+)
+
+py_proto_library(
+    name = "stats_py_pb2",
+    deps = [":stats_descriptor"],
+)
+
+proto_library(
+    name = "payloads_descriptor",
+    srcs = ["payloads.proto"],
+)
+
+py_proto_library(
+    name = "payloads_py_pb2",
+    deps = [":payloads_descriptor"],
+)
+
+proto_library(
+    name = "control_descriptor",
+    srcs = ["control.proto"],
+    deps = [
+        ":payloads_descriptor",
+        ":stats_descriptor",
+    ],
+)
+
+py_proto_library(
+    name = "control_py_pb2",
+    deps = [":control_descriptor"],
+)

src/proto/grpc/testing/control.proto

@@ -117,6 +117,9 @@ message ClientConfig {
   // If 0, disabled. Else, specifies the period between gathering latency
   // medians in milliseconds.
   int32 median_latency_collection_interval_millis = 20;
+
+  // Number of client processes. 0 indicates no restriction.
+  int32 client_processes = 21;
 }
 
 message ClientStatus { ClientStats stats = 1; }
@@ -163,6 +166,9 @@ message ServerConfig {
   // Buffer pool size (no buffer pool specified if unset)
   int32 resource_quota_size = 1001;
   repeated ChannelArg channel_args = 1002;
+
+  // Number of server processes. 0 indicates no restriction.
+  int32 server_processes = 21;
 }
 
 message ServerArgs {

src/python/grpcio/README.rst

@@ -59,7 +59,7 @@ package named :code:`python-dev`).
 ::
 
   $ export REPO_ROOT=grpc  # REPO_ROOT can be any directory of your choice
-  $ git clone -b $(curl -L https://grpc.io/release) https://github.com/grpc/grpc $REPO_ROOT
+  $ git clone -b RELEASE_TAG_HERE https://github.com/grpc/grpc $REPO_ROOT
   $ cd $REPO_ROOT
   $ git submodule update --init
 

src/python/grpcio/grpc/BUILD.bazel

@@ -1,30 +1,5 @@
 package(default_visibility = ["//visibility:public"])
 
-py_library(
-    name = "grpcio",
-    srcs = ["__init__.py"],
-    data = [
-        "//:grpc",
-    ],
-    imports = ["../"],
-    deps = [
-        ":utilities",
-        ":auth",
-        ":plugin_wrapping",
-        ":channel",
-        ":interceptor",
-        ":server",
-        ":compression",
-        "//src/python/grpcio/grpc/_cython:cygrpc",
-        "//src/python/grpcio/grpc/experimental",
-        "//src/python/grpcio/grpc/framework",
-        "@six//:six",
-    ] + select({
-        "//conditions:default": ["@enum34//:enum34"],
-        "//:python3": [],
-    }),
-)
-
 py_library(
     name = "auth",
     srcs = ["_auth.py"],
@@ -85,3 +60,34 @@ py_library(
         ":common",
     ],
 )
+
+py_library(
+    name = "_simple_stubs",
+    srcs = ["_simple_stubs.py"],
+)
+
+py_library(
+    name = "grpcio",
+    srcs = ["__init__.py"],
+    data = [
+        "//:grpc",
+    ],
+    imports = ["../"],
+    deps = [
+        ":utilities",
+        ":auth",
+        ":plugin_wrapping",
+        ":channel",
+        ":interceptor",
+        ":server",
+        ":compression",
+        ":_simple_stubs",
+        "//src/python/grpcio/grpc/_cython:cygrpc",
+        "//src/python/grpcio/grpc/experimental",
+        "//src/python/grpcio/grpc/framework",
+        "@six//:six",
+    ] + select({
+        "//conditions:default": ["@enum34//:enum34"],
+        "//:python3": [],
+    }),
+)

src/python/grpcio/grpc/__init__.py

@@ -1879,6 +1879,11 @@ def secure_channel(target, credentials, options=None, compression=None):
       A Channel.
     """
     from grpc import _channel  # pylint: disable=cyclic-import
+    from grpc.experimental import _insecure_channel_credentials
+    if credentials._credentials is _insecure_channel_credentials:
+        raise ValueError(
+            "secure_channel cannot be called with insecure credentials." +
+            " Call insecure_channel instead.")
     return _channel.Channel(target, () if options is None else options,
                             credentials._credentials, compression)
 

src/python/grpcio/grpc/_cython/_cygrpc/aio/call.pxd.pxi

@@ -15,7 +15,7 @@
 
 cdef class _AioCall(GrpcCallWrapper):
     cdef:
-        AioChannel _channel
+        readonly AioChannel _channel
         list _references
         object _deadline
         list _done_callbacks

src/python/grpcio/grpc/_cython/_cygrpc/aio/common.pyx.pxi

@@ -42,6 +42,8 @@ cdef bytes serialize(object serializer, object message):
 
     Failure to serialize is a fatal error.
     """
+    if isinstance(message, str):
+        message = message.encode('utf-8')
     if serializer:
         return serializer(message)
     else:

src/python/grpcio/grpc/_cython/_cygrpc/aio/server.pyx.pxi

@@ -333,6 +333,9 @@ async def _handle_unary_unary_rpc(object method_handler,
                                   object loop):
     # Receives request message
     cdef bytes request_raw = await _receive_message(rpc_state, loop)
+    if request_raw is None:
+        # The RPC was cancelled immediately after start on client side.
+        return
 
     # Deserializes the request message
     cdef object request_message = deserialize(
@@ -364,6 +367,8 @@ async def _handle_unary_stream_rpc(object method_handler,
                                    object loop):
     # Receives request message
     cdef bytes request_raw = await _receive_message(rpc_state, loop)
+    if request_raw is None:
+        return
 
     # Deserializes the request message
     cdef object request_message = deserialize(
@@ -466,6 +471,8 @@ async def _handle_exceptions(RPCState rpc_state, object rpc_coro, object loop):
                 )
     except (KeyboardInterrupt, SystemExit):
         raise
+    except asyncio.CancelledError:
+        _LOGGER.debug('RPC cancelled for servicer method [%s]', _decode(rpc_state.method()))
     except _ServerStoppedError:
         _LOGGER.info('Aborting RPC due to server stop.')
     except Exception as e:

src/python/grpcio/grpc/_simple_stubs.py

@@ -0,0 +1,450 @@
+# Copyright 2020 The gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Functions that obviate explicit stubs and explicit channels."""
+
+import collections
+import datetime
+import os
+import logging
+import threading
+from typing import (Any, AnyStr, Callable, Dict, Iterator, Optional, Sequence,
+                    Tuple, TypeVar, Union)
+
+import grpc
+from grpc.experimental import experimental_api
+
+RequestType = TypeVar('RequestType')
+ResponseType = TypeVar('ResponseType')
+
+OptionsType = Sequence[Tuple[str, str]]
+CacheKey = Tuple[str, OptionsType, Optional[grpc.ChannelCredentials], Optional[
+    grpc.Compression]]
+
+_LOGGER = logging.getLogger(__name__)
+
+_EVICTION_PERIOD_KEY = "GRPC_PYTHON_MANAGED_CHANNEL_EVICTION_SECONDS"
+if _EVICTION_PERIOD_KEY in os.environ:
+    _EVICTION_PERIOD = datetime.timedelta(
+        seconds=float(os.environ[_EVICTION_PERIOD_KEY]))
+    _LOGGER.debug("Setting managed channel eviction period to %s",
+                  _EVICTION_PERIOD)
+else:
+    _EVICTION_PERIOD = datetime.timedelta(minutes=10)
+
+_MAXIMUM_CHANNELS_KEY = "GRPC_PYTHON_MANAGED_CHANNEL_MAXIMUM"
+if _MAXIMUM_CHANNELS_KEY in os.environ:
+    _MAXIMUM_CHANNELS = int(os.environ[_MAXIMUM_CHANNELS_KEY])
+    _LOGGER.debug("Setting maximum managed channels to %d", _MAXIMUM_CHANNELS)
+else:
+    _MAXIMUM_CHANNELS = 2**8
+
+
+def _create_channel(target: str, options: Sequence[Tuple[str, str]],
+                    channel_credentials: Optional[grpc.ChannelCredentials],
+                    compression: Optional[grpc.Compression]) -> grpc.Channel:
+    channel_credentials = channel_credentials or grpc.local_channel_credentials(
+    )
+    if channel_credentials._credentials is grpc.experimental._insecure_channel_credentials:
+        _LOGGER.debug(f"Creating insecure channel with options '{options}' " +
+                      f"and compression '{compression}'")
+        return grpc.insecure_channel(target,
+                                     options=options,
+                                     compression=compression)
+    else:
+        _LOGGER.debug(
+            f"Creating secure channel with credentials '{channel_credentials}', "
+            + f"options '{options}' and compression '{compression}'")
+        return grpc.secure_channel(target,
+                                   credentials=channel_credentials,
+                                   options=options,
+                                   compression=compression)
+
+
+class ChannelCache:
+    # NOTE(rbellevi): Untyped due to reference cycle.
+    _singleton = None
+    _lock: threading.RLock = threading.RLock()
+    _condition: threading.Condition = threading.Condition(lock=_lock)
+    _eviction_ready: threading.Event = threading.Event()
+
+    _mapping: Dict[CacheKey, Tuple[grpc.Channel, datetime.datetime]]
+    _eviction_thread: threading.Thread
+
+    def __init__(self):
+        self._mapping = collections.OrderedDict()
+        self._eviction_thread = threading.Thread(
+            target=ChannelCache._perform_evictions, daemon=True)
+        self._eviction_thread.start()
+
+    @staticmethod
+    def get():
+        with ChannelCache._lock:
+            if ChannelCache._singleton is None:
+                ChannelCache._singleton = ChannelCache()
+        ChannelCache._eviction_ready.wait()
+        return ChannelCache._singleton
+
+    def _evict_locked(self, key: CacheKey):
+        channel, _ = self._mapping.pop(key)
+        _LOGGER.debug("Evicting channel %s with configuration %s.", channel,
+                      key)
+        channel.close()
+        del channel
+
+    @staticmethod
+    def _perform_evictions():
+        while True:
+            with ChannelCache._lock:
+                ChannelCache._eviction_ready.set()
+                if not ChannelCache._singleton._mapping:
+                    ChannelCache._condition.wait()
+                elif len(ChannelCache._singleton._mapping) > _MAXIMUM_CHANNELS:
+                    key = next(iter(ChannelCache._singleton._mapping.keys()))
+                    ChannelCache._singleton._evict_locked(key)
+                    # And immediately reevaluate.
+                else:
+                    key, (_, eviction_time) = next(
+                        iter(ChannelCache._singleton._mapping.items()))
+                    now = datetime.datetime.now()
+                    if eviction_time <= now:
+                        ChannelCache._singleton._evict_locked(key)
+                        continue
+                    else:
+                        time_to_eviction = (eviction_time - now).total_seconds()
+                        # NOTE: We aim to *eventually* coalesce to a state in
+                        # which no overdue channels are in the cache and the
+                        # length of the cache is longer than _MAXIMUM_CHANNELS.
+                        # We tolerate momentary states in which these two
+                        # criteria are not met.
+                        ChannelCache._condition.wait(timeout=time_to_eviction)
+
+    def get_channel(self, target: str, options: Sequence[Tuple[str, str]],
+                    channel_credentials: Optional[grpc.ChannelCredentials],
+                    compression: Optional[grpc.Compression]) -> grpc.Channel:
+        key = (target, options, channel_credentials, compression)
+        with self._lock:
+            channel_data = self._mapping.get(key, None)
+            if channel_data is not None:
+                channel = channel_data[0]
+                self._mapping.pop(key)
+                self._mapping[key] = (channel, datetime.datetime.now() +
+                                      _EVICTION_PERIOD)
+                return channel
+            else:
+                channel = _create_channel(target, options, channel_credentials,
+                                          compression)
+                self._mapping[key] = (channel, datetime.datetime.now() +
+                                      _EVICTION_PERIOD)
+                if len(self._mapping) == 1 or len(
+                        self._mapping) >= _MAXIMUM_CHANNELS:
+                    self._condition.notify()
+                return channel
+
+    def _test_only_channel_count(self) -> int:
+        with self._lock:
+            return len(self._mapping)
+
+
+# TODO(rbellevi): Consider a credential type that has the
+#   following functionality matrix:
+#
+#   +----------+-------+--------+
+#   |          | local | remote |
+#   |----------+-------+--------+
+#   | secure   | o     | o      |
+#   | insecure | o     | x      |
+#   +----------+-------+--------+
+#
+#  Make this the default option.
+
+
+@experimental_api
+def unary_unary(
+        request: RequestType,
+        target: str,
+        method: str,
+        request_serializer: Optional[Callable[[Any], bytes]] = None,
+        request_deserializer: Optional[Callable[[bytes], Any]] = None,
+        options: Sequence[Tuple[AnyStr, AnyStr]] = (),
+        channel_credentials: Optional[grpc.ChannelCredentials] = None,
+        call_credentials: Optional[grpc.CallCredentials] = None,
+        compression: Optional[grpc.Compression] = None,
+        wait_for_ready: Optional[bool] = None,
+        timeout: Optional[float] = None,
+        metadata: Optional[Sequence[Tuple[str, Union[str, bytes]]]] = None
+) -> ResponseType:
+    """Invokes a unary-unary RPC without an explicitly specified channel.
+
+    THIS IS AN EXPERIMENTAL API.
+
+    This is backed by a per-process cache of channels. Channels are evicted
+    from the cache after a fixed period by a background. Channels will also be
+    evicted if more than a configured maximum accumulate.
+
+    The default eviction period is 10 minutes. One may set the environment
+    variable "GRPC_PYTHON_MANAGED_CHANNEL_EVICTION_SECONDS" to configure this.
+
+    The default maximum number of channels is 256. One may set the
+    environment variable "GRPC_PYTHON_MANAGED_CHANNEL_MAXIMUM" to configure
+    this.
+
+    Args:
+      request: An iterator that yields request values for the RPC.
+      target: The server address.
+      method: The name of the RPC method.
+      request_serializer: Optional behaviour for serializing the request
+        message. Request goes unserialized in case None is passed.
+      response_deserializer: Optional behaviour for deserializing the response
+        message. Response goes undeserialized in case None is passed.
+      options: An optional list of key-value pairs (channel args in gRPC Core
+        runtime) to configure the channel.
+      channel_credentials: A credential applied to the whole channel, e.g. the
+        return value of grpc.ssl_channel_credentials() or
+        grpc.insecure_channel_credentials().
+      call_credentials: A call credential applied to each call individually,
+        e.g. the output of grpc.metadata_call_credentials() or
+        grpc.access_token_call_credentials().
+      compression: An optional value indicating the compression method to be
+        used over the lifetime of the channel, e.g. grpc.Compression.Gzip.
+      wait_for_ready: An optional flag indicating whether the RPC should fail
+        immediately if the connection is not ready at the time the RPC is
+        invoked, or if it should wait until the connection to the server
+        becomes ready. When using this option, the user will likely also want
+        to set a timeout. Defaults to False.
+      timeout: An optional duration of time in seconds to allow for the RPC,
+        after which an exception will be raised.
+      metadata: Optional metadata to send to the server.
+
+    Returns:
+      The response to the RPC.
+    """
+    channel = ChannelCache.get().get_channel(target, options,
+                                             channel_credentials, compression)
+    multicallable = channel.unary_unary(method, request_serializer,
+                                        request_deserializer)
+    return multicallable(request,
+                         metadata=metadata,
+                         wait_for_ready=wait_for_ready,
+                         credentials=call_credentials,
+                         timeout=timeout)
+
+
+@experimental_api
+def unary_stream(
+        request: RequestType,
+        target: str,
+        method: str,
+        request_serializer: Optional[Callable[[Any], bytes]] = None,
+        request_deserializer: Optional[Callable[[bytes], Any]] = None,
+        options: Sequence[Tuple[AnyStr, AnyStr]] = (),
+        channel_credentials: Optional[grpc.ChannelCredentials] = None,
+        call_credentials: Optional[grpc.CallCredentials] = None,
+        compression: Optional[grpc.Compression] = None,
+        wait_for_ready: Optional[bool] = None,
+        timeout: Optional[float] = None,
+        metadata: Optional[Sequence[Tuple[str, Union[str, bytes]]]] = None
+) -> Iterator[ResponseType]:
+    """Invokes a unary-stream RPC without an explicitly specified channel.
+
+    THIS IS AN EXPERIMENTAL API.
+
+    This is backed by a per-process cache of channels. Channels are evicted
+    from the cache after a fixed period by a background. Channels will also be
+    evicted if more than a configured maximum accumulate.
+
+    The default eviction period is 10 minutes. One may set the environment
+    variable "GRPC_PYTHON_MANAGED_CHANNEL_EVICTION_SECONDS" to configure this.
+
+    The default maximum number of channels is 256. One may set the
+    environment variable "GRPC_PYTHON_MANAGED_CHANNEL_MAXIMUM" to configure
+    this.
+
+    Args:
+      request: An iterator that yields request values for the RPC.
+      target: The server address.
+      method: The name of the RPC method.
+      request_serializer: Optional behaviour for serializing the request
+        message. Request goes unserialized in case None is passed.
+      response_deserializer: Optional behaviour for deserializing the response
+        message. Response goes undeserialized in case None is passed.
+      options: An optional list of key-value pairs (channel args in gRPC Core
+        runtime) to configure the channel.
+      channel_credentials: A credential applied to the whole channel, e.g. the
+        return value of grpc.ssl_channel_credentials().
+      call_credentials: A call credential applied to each call individually,
+        e.g. the output of grpc.metadata_call_credentials() or
+        grpc.access_token_call_credentials().
+      compression: An optional value indicating the compression method to be
+        used over the lifetime of the channel, e.g. grpc.Compression.Gzip.
+      wait_for_ready: An optional flag indicating whether the RPC should fail
+        immediately if the connection is not ready at the time the RPC is
+        invoked, or if it should wait until the connection to the server
+        becomes ready. When using this option, the user will likely also want
+        to set a timeout. Defaults to False.
+      timeout: An optional duration of time in seconds to allow for the RPC,
+        after which an exception will be raised.
+      metadata: Optional metadata to send to the server.
+
+    Returns:
+      An iterator of responses.
+    """
+    channel = ChannelCache.get().get_channel(target, options,
+                                             channel_credentials, compression)
+    multicallable = channel.unary_stream(method, request_serializer,
+                                         request_deserializer)
+    return multicallable(request,
+                         metadata=metadata,
+                         wait_for_ready=wait_for_ready,
+                         credentials=call_credentials,
+                         timeout=timeout)
+
+
+@experimental_api
+def stream_unary(
+        request_iterator: Iterator[RequestType],
+        target: str,
+        method: str,
+        request_serializer: Optional[Callable[[Any], bytes]] = None,
+        request_deserializer: Optional[Callable[[bytes], Any]] = None,
+        options: Sequence[Tuple[AnyStr, AnyStr]] = (),
+        channel_credentials: Optional[grpc.ChannelCredentials] = None,
+        call_credentials: Optional[grpc.CallCredentials] = None,
+        compression: Optional[grpc.Compression] = None,
+        wait_for_ready: Optional[bool] = None,
+        timeout: Optional[float] = None,
+        metadata: Optional[Sequence[Tuple[str, Union[str, bytes]]]] = None
+) -> ResponseType:
+    """Invokes a stream-unary RPC without an explicitly specified channel.
+
+    THIS IS AN EXPERIMENTAL API.
+
+    This is backed by a per-process cache of channels. Channels are evicted
+    from the cache after a fixed period by a background. Channels will also be
+    evicted if more than a configured maximum accumulate.
+
+    The default eviction period is 10 minutes. One may set the environment
+    variable "GRPC_PYTHON_MANAGED_CHANNEL_EVICTION_SECONDS" to configure this.
+
+    The default maximum number of channels is 256. One may set the
+    environment variable "GRPC_PYTHON_MANAGED_CHANNEL_MAXIMUM" to configure
+    this.
+
+    Args:
+      request_iterator: An iterator that yields request values for the RPC.
+      target: The server address.
+      method: The name of the RPC method.
+      request_serializer: Optional behaviour for serializing the request
+        message. Request goes unserialized in case None is passed.
+      response_deserializer: Optional behaviour for deserializing the response
+        message. Response goes undeserialized in case None is passed.
+      options: An optional list of key-value pairs (channel args in gRPC Core
+        runtime) to configure the channel.
+      channel_credentials: A credential applied to the whole channel, e.g. the
+        return value of grpc.ssl_channel_credentials().
+      call_credentials: A call credential applied to each call individually,
+        e.g. the output of grpc.metadata_call_credentials() or
+        grpc.access_token_call_credentials().
+      compression: An optional value indicating the compression method to be
+        used over the lifetime of the channel, e.g. grpc.Compression.Gzip.
+      wait_for_ready: An optional flag indicating whether the RPC should fail
+        immediately if the connection is not ready at the time the RPC is
+        invoked, or if it should wait until the connection to the server
+        becomes ready. When using this option, the user will likely also want
+        to set a timeout. Defaults to False.
+      timeout: An optional duration of time in seconds to allow for the RPC,
+        after which an exception will be raised.
+      metadata: Optional metadata to send to the server.
+
+    Returns:
+      The response to the RPC.
+    """
+    channel = ChannelCache.get().get_channel(target, options,
+                                             channel_credentials, compression)
+    multicallable = channel.stream_unary(method, request_serializer,
+                                         request_deserializer)
+    return multicallable(request_iterator,
+                         metadata=metadata,
+                         wait_for_ready=wait_for_ready,
+                         credentials=call_credentials,
+                         timeout=timeout)
+
+
+@experimental_api
+def stream_stream(
+        request_iterator: Iterator[RequestType],
+        target: str,
+        method: str,
+        request_serializer: Optional[Callable[[Any], bytes]] = None,
+        request_deserializer: Optional[Callable[[bytes], Any]] = None,
+        options: Sequence[Tuple[AnyStr, AnyStr]] = (),
+        channel_credentials: Optional[grpc.ChannelCredentials] = None,
+        call_credentials: Optional[grpc.CallCredentials] = None,
+        compression: Optional[grpc.Compression] = None,
+        wait_for_ready: Optional[bool] = None,
+        timeout: Optional[float] = None,
+        metadata: Optional[Sequence[Tuple[str, Union[str, bytes]]]] = None
+) -> Iterator[ResponseType]:
+    """Invokes a stream-stream RPC without an explicitly specified channel.
+
+    THIS IS AN EXPERIMENTAL API.
+
+    This is backed by a per-process cache of channels. Channels are evicted
+    from the cache after a fixed period by a background. Channels will also be
+    evicted if more than a configured maximum accumulate.
+
+    The default eviction period is 10 minutes. One may set the environment
+    variable "GRPC_PYTHON_MANAGED_CHANNEL_EVICTION_SECONDS" to configure this.
+
+    The default maximum number of channels is 256. One may set the
+    environment variable "GRPC_PYTHON_MANAGED_CHANNEL_MAXIMUM" to configure
+    this.
+
+    Args:
+      request_iterator: An iterator that yields request values for the RPC.
+      target: The server address.
+      method: The name of the RPC method.
+      request_serializer: Optional behaviour for serializing the request
+        message. Request goes unserialized in case None is passed.
+      response_deserializer: Optional behaviour for deserializing the response
+        message. Response goes undeserialized in case None is passed.
+      options: An optional list of key-value pairs (channel args in gRPC Core
+        runtime) to configure the channel.
+      channel_credentials: A credential applied to the whole channel, e.g. the
+        return value of grpc.ssl_channel_credentials().
+      call_credentials: A call credential applied to each call individually,
+        e.g. the output of grpc.metadata_call_credentials() or
+        grpc.access_token_call_credentials().
+      compression: An optional value indicating the compression method to be
+        used over the lifetime of the channel, e.g. grpc.Compression.Gzip.
+      wait_for_ready: An optional flag indicating whether the RPC should fail
+        immediately if the connection is not ready at the time the RPC is
+        invoked, or if it should wait until the connection to the server
+        becomes ready. When using this option, the user will likely also want
+        to set a timeout. Defaults to False.
+      timeout: An optional duration of time in seconds to allow for the RPC,
+        after which an exception will be raised.
+      metadata: Optional metadata to send to the server.
+
+    Returns:
+      An iterator of responses.
+    """
+    channel = ChannelCache.get().get_channel(target, options,
+                                             channel_credentials, compression)
+    multicallable = channel.stream_stream(method, request_serializer,
+                                          request_deserializer)
+    return multicallable(request_iterator,
+                         metadata=metadata,
+                         wait_for_ready=wait_for_ready,
+                         credentials=call_credentials,
+                         timeout=timeout)

src/python/grpcio/grpc/experimental/BUILD.bazel

@@ -2,16 +2,7 @@ package(default_visibility = ["//visibility:public"])
 
 py_library(
     name = "aio",
-    srcs = [
-        "aio/__init__.py",
-        "aio/_base_call.py",
-        "aio/_call.py",
-        "aio/_channel.py",
-        "aio/_interceptor.py",
-        "aio/_server.py",
-        "aio/_typing.py",
-        "aio/_utils.py",
-    ],
+    srcs = glob(["aio/**/*.py"]),
     deps = [
         "//src/python/grpcio/grpc/_cython:cygrpc",
     ],

src/python/grpcio/grpc/experimental/__init__.py

@@ -16,6 +16,14 @@
 These APIs are subject to be removed during any minor version release.
 """
 
+import functools
+import sys
+import warnings
+
+import grpc
+
+_EXPERIMENTAL_APIS_USED = set()
+
 
 class ChannelOptions(object):
     """Indicates a channel option unique to gRPC Python.
@@ -30,3 +38,53 @@ class ChannelOptions(object):
 
 class UsageError(Exception):
     """Raised by the gRPC library to indicate usage not allowed by the API."""
+
+
+_insecure_channel_credentials = object()
+
+
+def insecure_channel_credentials():
+    """Creates a ChannelCredentials for use with an insecure channel.
+
+    THIS IS AN EXPERIMENTAL API.
+
+    This is not for use with secure_channel function. Intead, this should be
+    used with grpc.unary_unary, grpc.unary_stream, grpc.stream_unary, or
+    grpc.stream_stream.
+    """
+    return grpc.ChannelCredentials(_insecure_channel_credentials)
+
+
+class ExperimentalApiWarning(Warning):
+    """A warning that an API is experimental."""
+
+
+def _warn_experimental(api_name, stack_offset):
+    if api_name not in _EXPERIMENTAL_APIS_USED:
+        _EXPERIMENTAL_APIS_USED.add(api_name)
+        msg = ("'{}' is an experimental API. It is subject to change or ".
+               format(api_name) +
+               "removal between minor releases. Proceed with caution.")
+        warnings.warn(msg, ExperimentalApiWarning, stacklevel=2 + stack_offset)
+
+
+def experimental_api(f):
+
+    @functools.wraps(f)
+    def _wrapper(*args, **kwargs):
+        _warn_experimental(f.__name__, 1)
+        return f(*args, **kwargs)
+
+    return _wrapper
+
+
+__all__ = (
+    'ChannelOptions',
+    'ExperimentalApiWarning',
+    'UsageError',
+    'insecure_channel_credentials',
+)
+
+if sys.version_info[0] >= 3:
+    from grpc._simple_stubs import unary_unary, unary_stream, stream_unary, stream_stream
+    __all__ = __all__ + (unary_unary, unary_stream, stream_unary, stream_stream)

src/python/grpcio/grpc/experimental/aio/__init__.py

@@ -20,71 +20,49 @@ created. AsyncIO doesn't provide thread safety for most of its APIs.
 from typing import Any, Optional, Sequence, Tuple
 
 import grpc
-from grpc._cython.cygrpc import (EOF, AbortError, BaseError, UsageError,
-                                 init_grpc_aio)
+from grpc._cython.cygrpc import (EOF, AbortError, BaseError, InternalError,
+                                 UsageError, init_grpc_aio)
 
-from ._base_call import Call, RpcContext, UnaryStreamCall, UnaryUnaryCall
+from ._base_call import (Call, RpcContext, StreamStreamCall, StreamUnaryCall,
+                         UnaryStreamCall, UnaryUnaryCall)
+from ._base_channel import (Channel, StreamStreamMultiCallable,
+                            StreamUnaryMultiCallable, UnaryStreamMultiCallable,
+                            UnaryUnaryMultiCallable)
 from ._call import AioRpcError
-from ._channel import Channel, UnaryUnaryMultiCallable
 from ._interceptor import (ClientCallDetails, InterceptedUnaryUnaryCall,
                            UnaryUnaryClientInterceptor)
-from ._server import Server, server
+from ._server import server
+from ._base_server import Server, ServicerContext
 from ._typing import ChannelArgumentType
-
-
-def insecure_channel(
-        target: str,
-        options: Optional[ChannelArgumentType] = None,
-        compression: Optional[grpc.Compression] = None,
-        interceptors: Optional[Sequence[UnaryUnaryClientInterceptor]] = None):
-    """Creates an insecure asynchronous Channel to a server.
-
-    Args:
-      target: The server address
-      options: An optional list of key-value pairs (channel args
-        in gRPC Core runtime) to configure the channel.
-      compression: An optional value indicating the compression method to be
-        used over the lifetime of the channel. This is an EXPERIMENTAL option.
-      interceptors: An optional sequence of interceptors that will be executed for
-        any call executed with this channel.
-
-    Returns:
-      A Channel.
-    """
-    return Channel(target, () if options is None else options, None,
-                   compression, interceptors)
-
-
-def secure_channel(
-        target: str,
-        credentials: grpc.ChannelCredentials,
-        options: Optional[ChannelArgumentType] = None,
-        compression: Optional[grpc.Compression] = None,
-        interceptors: Optional[Sequence[UnaryUnaryClientInterceptor]] = None):
-    """Creates a secure asynchronous Channel to a server.
-
-    Args:
-      target: The server address.
-      credentials: A ChannelCredentials instance.
-      options: An optional list of key-value pairs (channel args
-        in gRPC Core runtime) to configure the channel.
-      compression: An optional value indicating the compression method to be
-        used over the lifetime of the channel. This is an EXPERIMENTAL option.
-      interceptors: An optional sequence of interceptors that will be executed for
-        any call executed with this channel.
-
-    Returns:
-      An aio.Channel.
-    """
-    return Channel(target, () if options is None else options,
-                   credentials._credentials, compression, interceptors)
-
+from ._channel import insecure_channel, secure_channel
 
 ###################################  __all__  #################################
 
-__all__ = ('AioRpcError', 'RpcContext', 'Call', 'UnaryUnaryCall',
-           'UnaryStreamCall', 'init_grpc_aio', 'Channel',
-           'UnaryUnaryMultiCallable', 'ClientCallDetails',
-           'UnaryUnaryClientInterceptor', 'InterceptedUnaryUnaryCall',
-           'insecure_channel', 'server', 'Server', 'EOF', 'secure_channel',
-           'AbortError', 'BaseError', 'UsageError')
+__all__ = (
+    'AioRpcError',
+    'RpcContext',
+    'Call',
+    'UnaryUnaryCall',
+    'UnaryStreamCall',
+    'StreamUnaryCall',
+    'StreamStreamCall',
+    'init_grpc_aio',
+    'Channel',
+    'UnaryUnaryMultiCallable',
+    'UnaryStreamMultiCallable',
+    'StreamUnaryMultiCallable',
+    'StreamStreamMultiCallable',
+    'ClientCallDetails',
+    'UnaryUnaryClientInterceptor',
+    'InterceptedUnaryUnaryCall',
+    'insecure_channel',
+    'server',
+    'Server',
+    'ServicerContext',
+    'EOF',
+    'secure_channel',
+    'AbortError',
+    'BaseError',
+    'UsageError',
+    'InternalError',
+)

src/python/grpcio/grpc/experimental/aio/_base_channel.py

@@ -0,0 +1,345 @@
+# Copyright 2020 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Abstract base classes for Channel objects and Multicallable objects."""
+
+import abc
+from typing import Any, AsyncIterable, Optional
+
+import grpc
+
+from . import _base_call
+from ._typing import DeserializingFunction, MetadataType, SerializingFunction
+
+_IMMUTABLE_EMPTY_TUPLE = tuple()
+
+
+class UnaryUnaryMultiCallable(abc.ABC):
+    """Enables asynchronous invocation of a unary-call RPC."""
+
+    @abc.abstractmethod
+    def __call__(self,
+                 request: Any,
+                 *,
+                 timeout: Optional[float] = None,
+                 metadata: Optional[MetadataType] = _IMMUTABLE_EMPTY_TUPLE,
+                 credentials: Optional[grpc.CallCredentials] = None,
+                 wait_for_ready: Optional[bool] = None,
+                 compression: Optional[grpc.Compression] = None
+                ) -> _base_call.UnaryUnaryCall:
+        """Asynchronously invokes the underlying RPC.
+
+        Args:
+          request: The request value for the RPC.
+          timeout: An optional duration of time in seconds to allow
+            for the RPC.
+          metadata: Optional :term:`metadata` to be transmitted to the
+            service-side of the RPC.
+          credentials: An optional CallCredentials for the RPC. Only valid for
+            secure Channel.
+          wait_for_ready: This is an EXPERIMENTAL argument. An optional
+            flag to enable wait for ready mechanism
+          compression: An element of grpc.compression, e.g.
+            grpc.compression.Gzip. This is an EXPERIMENTAL option.
+
+        Returns:
+          A UnaryUnaryCall object.
+
+        Raises:
+          RpcError: Indicates that the RPC terminated with non-OK status. The
+            raised RpcError will also be a Call for the RPC affording the RPC's
+            metadata, status code, and details.
+        """
+
+
+class UnaryStreamMultiCallable(abc.ABC):
+    """Enables asynchronous invocation of a server-streaming RPC."""
+
+    @abc.abstractmethod
+    def __call__(self,
+                 request: Any,
+                 *,
+                 timeout: Optional[float] = None,
+                 metadata: Optional[MetadataType] = _IMMUTABLE_EMPTY_TUPLE,
+                 credentials: Optional[grpc.CallCredentials] = None,
+                 wait_for_ready: Optional[bool] = None,
+                 compression: Optional[grpc.Compression] = None
+                ) -> _base_call.UnaryStreamCall:
+        """Asynchronously invokes the underlying RPC.
+
+        Args:
+          request: The request value for the RPC.
+          timeout: An optional duration of time in seconds to allow
+            for the RPC.
+          metadata: Optional :term:`metadata` to be transmitted to the
+            service-side of the RPC.
+          credentials: An optional CallCredentials for the RPC. Only valid for
+            secure Channel.
+          wait_for_ready: This is an EXPERIMENTAL argument. An optional
+            flag to enable wait for ready mechanism
+          compression: An element of grpc.compression, e.g.
+            grpc.compression.Gzip. This is an EXPERIMENTAL option.
+
+        Returns:
+          A UnaryStreamCall object.
+
+        Raises:
+          RpcError: Indicates that the RPC terminated with non-OK status. The
+            raised RpcError will also be a Call for the RPC affording the RPC's
+            metadata, status code, and details.
+        """
+
+
+class StreamUnaryMultiCallable(abc.ABC):
+    """Enables asynchronous invocation of a client-streaming RPC."""
+
+    @abc.abstractmethod
+    def __call__(self,
+                 request_async_iterator: Optional[AsyncIterable[Any]] = None,
+                 timeout: Optional[float] = None,
+                 metadata: Optional[MetadataType] = _IMMUTABLE_EMPTY_TUPLE,
+                 credentials: Optional[grpc.CallCredentials] = None,
+                 wait_for_ready: Optional[bool] = None,
+                 compression: Optional[grpc.Compression] = None
+                ) -> _base_call.StreamUnaryCall:
+        """Asynchronously invokes the underlying RPC.
+
+        Args:
+          request: The request value for the RPC.
+          timeout: An optional duration of time in seconds to allow
+            for the RPC.
+          metadata: Optional :term:`metadata` to be transmitted to the
+            service-side of the RPC.
+          credentials: An optional CallCredentials for the RPC. Only valid for
+            secure Channel.
+          wait_for_ready: This is an EXPERIMENTAL argument. An optional
+            flag to enable wait for ready mechanism
+          compression: An element of grpc.compression, e.g.
+            grpc.compression.Gzip. This is an EXPERIMENTAL option.
+
+        Returns:
+          A StreamUnaryCall object.
+
+        Raises:
+          RpcError: Indicates that the RPC terminated with non-OK status. The
+            raised RpcError will also be a Call for the RPC affording the RPC's
+            metadata, status code, and details.
+        """
+
+
+class StreamStreamMultiCallable(abc.ABC):
+    """Enables asynchronous invocation of a bidirectional-streaming RPC."""
+
+    @abc.abstractmethod
+    def __call__(self,
+                 request_async_iterator: Optional[AsyncIterable[Any]] = None,
+                 timeout: Optional[float] = None,
+                 metadata: Optional[MetadataType] = _IMMUTABLE_EMPTY_TUPLE,
+                 credentials: Optional[grpc.CallCredentials] = None,
+                 wait_for_ready: Optional[bool] = None,
+                 compression: Optional[grpc.Compression] = None
+                ) -> _base_call.StreamStreamCall:
+        """Asynchronously invokes the underlying RPC.
+
+        Args:
+          request: The request value for the RPC.
+          timeout: An optional duration of time in seconds to allow
+            for the RPC.
+          metadata: Optional :term:`metadata` to be transmitted to the
+            service-side of the RPC.
+          credentials: An optional CallCredentials for the RPC. Only valid for
+            secure Channel.
+          wait_for_ready: This is an EXPERIMENTAL argument. An optional
+            flag to enable wait for ready mechanism
+          compression: An element of grpc.compression, e.g.
+            grpc.compression.Gzip. This is an EXPERIMENTAL option.
+
+        Returns:
+          A StreamStreamCall object.
+
+        Raises:
+          RpcError: Indicates that the RPC terminated with non-OK status. The
+            raised RpcError will also be a Call for the RPC affording the RPC's
+            metadata, status code, and details.
+        """
+
+
+class Channel(abc.ABC):
+    """Enables asynchronous RPC invocation as a client.
+
+    Channel objects implement the Asynchronous Context Manager (aka. async
+    with) type, although they are not supportted to be entered and exited
+    multiple times.
+    """
+
+    @abc.abstractmethod
+    async def __aenter__(self):
+        """Starts an asynchronous context manager.
+
+        Returns:
+          Channel the channel that was instantiated.
+        """
+
+    @abc.abstractmethod
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        """Finishes the asynchronous context manager by closing the channel.
+
+        Still active RPCs will be cancelled.
+        """
+
+    @abc.abstractmethod
+    async def close(self, grace: Optional[float] = None):
+        """Closes this Channel and releases all resources held by it.
+
+        This method immediately stops the channel from executing new RPCs in
+        all cases.
+
+        If a grace period is specified, this method wait until all active
+        RPCs are finshed, once the grace period is reached the ones that haven't
+        been terminated are cancelled. If a grace period is not specified
+        (by passing None for grace), all existing RPCs are cancelled immediately.
+
+        This method is idempotent.
+        """
+
+    @abc.abstractmethod
+    def get_state(self,
+                  try_to_connect: bool = False) -> grpc.ChannelConnectivity:
+        """Checks the connectivity state of a channel.
+
+        This is an EXPERIMENTAL API.
+
+        If the channel reaches a stable connectivity state, it is guaranteed
+        that the return value of this function will eventually converge to that
+        state.
+
+        Args:
+          try_to_connect: a bool indicate whether the Channel should try to
+            connect to peer or not.
+
+        Returns: A ChannelConnectivity object.
+        """
+
+    @abc.abstractmethod
+    async def wait_for_state_change(
+            self,
+            last_observed_state: grpc.ChannelConnectivity,
+    ) -> None:
+        """Waits for a change in connectivity state.
+
+        This is an EXPERIMENTAL API.
+
+        The function blocks until there is a change in the channel connectivity
+        state from the "last_observed_state". If the state is already
+        different, this function will return immediately.
+
+        There is an inherent race between the invocation of
+        "Channel.wait_for_state_change" and "Channel.get_state". The state can
+        change arbitrary many times during the race, so there is no way to
+        observe every state transition.
+
+        If there is a need to put a timeout for this function, please refer to
+        "asyncio.wait_for".
+
+        Args:
+          last_observed_state: A grpc.ChannelConnectivity object representing
+            the last known state.
+        """
+
+    @abc.abstractmethod
+    async def channel_ready(self) -> None:
+        """Creates a coroutine that blocks until the Channel is READY."""
+
+    @abc.abstractmethod
+    def unary_unary(
+            self,
+            method: str,
+            request_serializer: Optional[SerializingFunction] = None,
+            response_deserializer: Optional[DeserializingFunction] = None
+    ) -> UnaryUnaryMultiCallable:
+        """Creates a UnaryUnaryMultiCallable for a unary-unary method.
+
+        Args:
+          method: The name of the RPC method.
+          request_serializer: Optional behaviour for serializing the request
+            message. Request goes unserialized in case None is passed.
+          response_deserializer: Optional behaviour for deserializing the
+            response message. Response goes undeserialized in case None
+            is passed.
+
+        Returns:
+          A UnaryUnaryMultiCallable value for the named unary-unary method.
+        """
+
+    @abc.abstractmethod
+    def unary_stream(
+            self,
+            method: str,
+            request_serializer: Optional[SerializingFunction] = None,
+            response_deserializer: Optional[DeserializingFunction] = None
+    ) -> UnaryStreamMultiCallable:
+        """Creates a UnaryStreamMultiCallable for a unary-stream method.
+
+        Args:
+          method: The name of the RPC method.
+          request_serializer: Optional behaviour for serializing the request
+            message. Request goes unserialized in case None is passed.
+          response_deserializer: Optional behaviour for deserializing the
+            response message. Response goes undeserialized in case None
+            is passed.
+
+        Returns:
+          A UnarySteramMultiCallable value for the named unary-stream method.
+        """
+
+    @abc.abstractmethod
+    def stream_unary(
+            self,
+            method: str,
+            request_serializer: Optional[SerializingFunction] = None,
+            response_deserializer: Optional[DeserializingFunction] = None
+    ) -> StreamUnaryMultiCallable:
+        """Creates a StreamUnaryMultiCallable for a stream-unary method.
+
+        Args:
+          method: The name of the RPC method.
+          request_serializer: Optional behaviour for serializing the request
+            message. Request goes unserialized in case None is passed.
+          response_deserializer: Optional behaviour for deserializing the
+            response message. Response goes undeserialized in case None
+            is passed.
+
+        Returns:
+          A StreamUnaryMultiCallable value for the named stream-unary method.
+        """
+
+    @abc.abstractmethod
+    def stream_stream(
+            self,
+            method: str,
+            request_serializer: Optional[SerializingFunction] = None,
+            response_deserializer: Optional[DeserializingFunction] = None
+    ) -> StreamStreamMultiCallable:
+        """Creates a StreamStreamMultiCallable for a stream-stream method.
+
+        Args:
+          method: The name of the RPC method.
+          request_serializer: Optional behaviour for serializing the request
+            message. Request goes unserialized in case None is passed.
+          response_deserializer: Optional behaviour for deserializing the
+            response message. Response goes undeserialized in case None
+            is passed.
+
+        Returns:
+          A StreamStreamMultiCallable value for the named stream-stream method.
+        """

src/python/grpcio/grpc/experimental/aio/_base_server.py

@@ -0,0 +1,254 @@
+# Copyright 2020 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Abstract base classes for server-side classes."""
+
+import abc
+from typing import Generic, Optional, Sequence
+
+import grpc
+
+from ._typing import MetadataType, RequestType, ResponseType
+
+
+class Server(abc.ABC):
+    """Serves RPCs."""
+
+    @abc.abstractmethod
+    def add_generic_rpc_handlers(
+            self,
+            generic_rpc_handlers: Sequence[grpc.GenericRpcHandler]) -> None:
+        """Registers GenericRpcHandlers with this Server.
+
+        This method is only safe to call before the server is started.
+
+        Args:
+          generic_rpc_handlers: A sequence of GenericRpcHandlers that will be
+          used to service RPCs.
+        """
+
+    @abc.abstractmethod
+    def add_insecure_port(self, address: str) -> int:
+        """Opens an insecure port for accepting RPCs.
+
+        A port is a communication endpoint that used by networking protocols,
+        like TCP and UDP. To date, we only support TCP.
+
+        This method may only be called before starting the server.
+
+        Args:
+          address: The address for which to open a port. If the port is 0,
+            or not specified in the address, then the gRPC runtime will choose a port.
+
+        Returns:
+          An integer port on which the server will accept RPC requests.
+        """
+
+    @abc.abstractmethod
+    def add_secure_port(self, address: str,
+                        server_credentials: grpc.ServerCredentials) -> int:
+        """Opens a secure port for accepting RPCs.
+
+        A port is a communication endpoint that used by networking protocols,
+        like TCP and UDP. To date, we only support TCP.
+
+        This method may only be called before starting the server.
+
+        Args:
+          address: The address for which to open a port.
+            if the port is 0, or not specified in the address, then the gRPC
+            runtime will choose a port.
+          server_credentials: A ServerCredentials object.
+
+        Returns:
+          An integer port on which the server will accept RPC requests.
+        """
+
+    @abc.abstractmethod
+    async def start(self) -> None:
+        """Starts this Server.
+
+        This method may only be called once. (i.e. it is not idempotent).
+        """
+
+    @abc.abstractmethod
+    async def stop(self, grace: Optional[float]) -> None:
+        """Stops this Server.
+
+        This method immediately stops the server from servicing new RPCs in
+        all cases.
+
+        If a grace period is specified, this method returns immediately and all
+        RPCs active at the end of the grace period are aborted. If a grace
+        period is not specified (by passing None for grace), all existing RPCs
+        are aborted immediately and this method blocks until the last RPC
+        handler terminates.
+
+        This method is idempotent and may be called at any time. Passing a
+        smaller grace value in a subsequent call will have the effect of
+        stopping the Server sooner (passing None will have the effect of
+        stopping the server immediately). Passing a larger grace value in a
+        subsequent call will not have the effect of stopping the server later
+        (i.e. the most restrictive grace value is used).
+
+        Args:
+          grace: A duration of time in seconds or None.
+        """
+
+    @abc.abstractmethod
+    async def wait_for_termination(self,
+                                   timeout: Optional[float] = None) -> bool:
+        """Continues current coroutine once the server stops.
+
+        This is an EXPERIMENTAL API.
+
+        The wait will not consume computational resources during blocking, and
+        it will block until one of the two following conditions are met:
+
+        1) The server is stopped or terminated;
+        2) A timeout occurs if timeout is not `None`.
+
+        The timeout argument works in the same way as `threading.Event.wait()`.
+        https://docs.python.org/3/library/threading.html#threading.Event.wait
+
+        Args:
+          timeout: A floating point number specifying a timeout for the
+            operation in seconds.
+
+        Returns:
+          A bool indicates if the operation times out.
+        """
+
+
+class ServicerContext(Generic[RequestType, ResponseType], abc.ABC):
+    """A context object passed to method implementations."""
+
+    @abc.abstractmethod
+    async def read(self) -> RequestType:
+        """Reads one message from the RPC.
+
+        Only one read operation is allowed simultaneously.
+
+        Returns:
+          A response message of the RPC.
+
+        Raises:
+          An RpcError exception if the read failed.
+        """
+
+    @abc.abstractmethod
+    async def write(self, message: ResponseType) -> None:
+        """Writes one message to the RPC.
+
+        Only one write operation is allowed simultaneously.
+
+        Raises:
+          An RpcError exception if the write failed.
+        """
+
+    @abc.abstractmethod
+    async def send_initial_metadata(self,
+                                    initial_metadata: MetadataType) -> None:
+        """Sends the initial metadata value to the client.
+
+        This method need not be called by implementations if they have no
+        metadata to add to what the gRPC runtime will transmit.
+
+        Args:
+          initial_metadata: The initial :term:`metadata`.
+        """
+
+    @abc.abstractmethod
+    async def abort(self, code: grpc.StatusCode, details: str,
+                    trailing_metadata: MetadataType) -> None:
+        """Raises an exception to terminate the RPC with a non-OK status.
+
+        The code and details passed as arguments will supercede any existing
+        ones.
+
+        Args:
+          code: A StatusCode object to be sent to the client.
+            It must not be StatusCode.OK.
+          details: A UTF-8-encodable string to be sent to the client upon
+            termination of the RPC.
+          trailing_metadata: A sequence of tuple represents the trailing
+            :term:`metadata`.
+
+        Raises:
+          Exception: An exception is always raised to signal the abortion the
+            RPC to the gRPC runtime.
+        """
+
+    @abc.abstractmethod
+    async def set_trailing_metadata(self,
+                                    trailing_metadata: MetadataType) -> None:
+        """Sends the trailing metadata for the RPC.
+
+        This method need not be called by implementations if they have no
+        metadata to add to what the gRPC runtime will transmit.
+
+        Args:
+          trailing_metadata: The trailing :term:`metadata`.
+        """
+
+    @abc.abstractmethod
+    def invocation_metadata(self) -> Optional[MetadataType]:
+        """Accesses the metadata from the sent by the client.
+
+        Returns:
+          The invocation :term:`metadata`.
+        """
+
+    @abc.abstractmethod
+    def set_code(self, code: grpc.StatusCode) -> None:
+        """Sets the value to be used as status code upon RPC completion.
+
+        This method need not be called by method implementations if they wish
+        the gRPC runtime to determine the status code of the RPC.
+
+        Args:
+          code: A StatusCode object to be sent to the client.
+        """
+
+    @abc.abstractmethod
+    def set_details(self, details: str) -> None:
+        """Sets the value to be used the as detail string upon RPC completion.
+
+        This method need not be called by method implementations if they have
+        no details to transmit.
+
+        Args:
+          details: A UTF-8-encodable string to be sent to the client upon
+            termination of the RPC.
+        """
+
+    @abc.abstractmethod
+    def set_compression(self, compression: grpc.Compression) -> None:
+        """Set the compression algorithm to be used for the entire call.
+
+        This is an EXPERIMENTAL method.
+
+        Args:
+          compression: An element of grpc.compression, e.g.
+            grpc.compression.Gzip.
+        """
+
+    @abc.abstractmethod
+    def disable_next_message_compression(self) -> None:
+        """Disables compression for the next response message.
+
+        This is an EXPERIMENTAL method.
+
+        This method will override any compression configuration set during
+        server creation or set on the call.
+        """

src/python/grpcio/grpc/experimental/aio/_channel.py

@@ -12,18 +12,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """Invocation-side implementation of gRPC Asyncio Python."""
+
 import asyncio
-from typing import Any, AsyncIterable, Optional, Sequence, AbstractSet
-from weakref import WeakSet
+import sys
+from typing import Any, AsyncIterable, Iterable, Optional, Sequence
 
-import logging
 import grpc
-from grpc import _common
+from grpc import _common, _compression, _grpcio_metadata
 from grpc._cython import cygrpc
-from grpc import _compression
-from grpc import _grpcio_metadata
 
-from . import _base_call
+from . import _base_call, _base_channel
 from ._call import (StreamStreamCall, StreamUnaryCall, UnaryStreamCall,
                     UnaryUnaryCall)
 from ._interceptor import (InterceptedUnaryUnaryCall,
@@ -35,6 +33,15 @@ from ._utils import _timeout_to_deadline
 _IMMUTABLE_EMPTY_TUPLE = tuple()
 _USER_AGENT = 'grpc-python-asyncio/{}'.format(_grpcio_metadata.__version__)
 
+if sys.version_info[1] < 7:
+
+    def _all_tasks() -> Iterable[asyncio.Task]:
+        return asyncio.Task.all_tasks()
+else:
+
+    def _all_tasks() -> Iterable[asyncio.Task]:
+        return asyncio.all_tasks()
+
 
 def _augment_channel_arguments(base_options: ChannelArgumentType,
                                compression: Optional[grpc.Compression]):
@@ -48,50 +55,12 @@ def _augment_channel_arguments(base_options: ChannelArgumentType,
                 ) + compression_channel_argument + user_agent_channel_argument
 
 
-_LOGGER = logging.getLogger(__name__)
-
-
-class _OngoingCalls:
-    """Internal class used for have visibility of the ongoing calls."""
-
-    _calls: AbstractSet[_base_call.RpcContext]
-
-    def __init__(self):
-        self._calls = WeakSet()
-
-    def _remove_call(self, call: _base_call.RpcContext):
-        try:
-            self._calls.remove(call)
-        except KeyError:
-            pass
-
-    @property
-    def calls(self) -> AbstractSet[_base_call.RpcContext]:
-        """Returns the set of ongoing calls."""
-        return self._calls
-
-    def size(self) -> int:
-        """Returns the number of ongoing calls."""
-        return len(self._calls)
-
-    def trace_call(self, call: _base_call.RpcContext):
-        """Adds and manages a new ongoing call."""
-        self._calls.add(call)
-        call.add_done_callback(self._remove_call)
-
-
 class _BaseMultiCallable:
     """Base class of all multi callable objects.
 
     Handles the initialization logic and stores common attributes.
     """
     _loop: asyncio.AbstractEventLoop
-    _channel: cygrpc.AioChannel
-    _ongoing_calls: _OngoingCalls
-    _method: bytes
-    _request_serializer: SerializingFunction
-    _response_deserializer: DeserializingFunction
-
     _channel: cygrpc.AioChannel
     _method: bytes
     _request_serializer: SerializingFunction
@@ -103,7 +72,6 @@ class _BaseMultiCallable:
     def __init__(
             self,
             channel: cygrpc.AioChannel,
-            ongoing_calls: _OngoingCalls,
             method: bytes,
             request_serializer: SerializingFunction,
             response_deserializer: DeserializingFunction,
@@ -112,15 +80,14 @@ class _BaseMultiCallable:
     ) -> None:
         self._loop = loop
         self._channel = channel
-        self._ongoing_calls = ongoing_calls
         self._method = method
         self._request_serializer = request_serializer
         self._response_deserializer = response_deserializer
         self._interceptors = interceptors
 
 
-class UnaryUnaryMultiCallable(_BaseMultiCallable):
-    """Factory an asynchronous unary-unary RPC stub call from client-side."""
+class UnaryUnaryMultiCallable(_BaseMultiCallable,
+                              _base_channel.UnaryUnaryMultiCallable):
 
     def __call__(self,
                  request: Any,
@@ -131,29 +98,6 @@ class UnaryUnaryMultiCallable(_BaseMultiCallable):
                  wait_for_ready: Optional[bool] = None,
                  compression: Optional[grpc.Compression] = None
                 ) -> _base_call.UnaryUnaryCall:
-        """Asynchronously invokes the underlying RPC.
-
-        Args:
-          request: The request value for the RPC.
-          timeout: An optional duration of time in seconds to allow
-            for the RPC.
-          metadata: Optional :term:`metadata` to be transmitted to the
-            service-side of the RPC.
-          credentials: An optional CallCredentials for the RPC. Only valid for
-            secure Channel.
-          wait_for_ready: This is an EXPERIMENTAL argument. An optional
-            flag to enable wait for ready mechanism
-          compression: An element of grpc.compression, e.g.
-            grpc.compression.Gzip. This is an EXPERIMENTAL option.
-
-        Returns:
-          A Call object instance which is an awaitable object.
-
-        Raises:
-          RpcError: Indicating that the RPC terminated with non-OK status. The
-            raised RpcError will also be a Call for the RPC affording the RPC's
-            metadata, status code, and details.
-        """
         if compression:
             metadata = _compression.augment_metadata(metadata, compression)
 
@@ -170,12 +114,11 @@ class UnaryUnaryMultiCallable(_BaseMultiCallable):
                 self._request_serializer, self._response_deserializer,
                 self._loop)
 
-        self._ongoing_calls.trace_call(call)
         return call
 
 
-class UnaryStreamMultiCallable(_BaseMultiCallable):
-    """Affords invoking a unary-stream RPC from client-side in an asynchronous way."""
+class UnaryStreamMultiCallable(_BaseMultiCallable,
+                               _base_channel.UnaryStreamMultiCallable):
 
     def __call__(self,
                  request: Any,
@@ -186,24 +129,6 @@ class UnaryStreamMultiCallable(_BaseMultiCallable):
                  wait_for_ready: Optional[bool] = None,
                  compression: Optional[grpc.Compression] = None
                 ) -> _base_call.UnaryStreamCall:
-        """Asynchronously invokes the underlying RPC.
-
-        Args:
-          request: The request value for the RPC.
-          timeout: An optional duration of time in seconds to allow
-            for the RPC.
-          metadata: Optional :term:`metadata` to be transmitted to the
-            service-side of the RPC.
-          credentials: An optional CallCredentials for the RPC. Only valid for
-            secure Channel.
-          wait_for_ready: This is an EXPERIMENTAL argument. An optional
-            flag to enable wait for ready mechanism
-          compression: An element of grpc.compression, e.g.
-            grpc.compression.Gzip. This is an EXPERIMENTAL option.
-
-        Returns:
-          A Call object instance which is an awaitable object.
-        """
         if compression:
             metadata = _compression.augment_metadata(metadata, compression)
 
@@ -213,12 +138,12 @@ class UnaryStreamMultiCallable(_BaseMultiCallable):
                                wait_for_ready, self._channel, self._method,
                                self._request_serializer,
                                self._response_deserializer, self._loop)
-        self._ongoing_calls.trace_call(call)
+
         return call
 
 
-class StreamUnaryMultiCallable(_BaseMultiCallable):
-    """Affords invoking a stream-unary RPC from client-side in an asynchronous way."""
+class StreamUnaryMultiCallable(_BaseMultiCallable,
+                               _base_channel.StreamUnaryMultiCallable):
 
     def __call__(self,
                  request_async_iterator: Optional[AsyncIterable[Any]] = None,
@@ -228,29 +153,6 @@ class StreamUnaryMultiCallable(_BaseMultiCallable):
                  wait_for_ready: Optional[bool] = None,
                  compression: Optional[grpc.Compression] = None
                 ) -> _base_call.StreamUnaryCall:
-        """Asynchronously invokes the underlying RPC.
-
-        Args:
-          request: The request value for the RPC.
-          timeout: An optional duration of time in seconds to allow
-            for the RPC.
-          metadata: Optional :term:`metadata` to be transmitted to the
-            service-side of the RPC.
-          credentials: An optional CallCredentials for the RPC. Only valid for
-            secure Channel.
-          wait_for_ready: This is an EXPERIMENTAL argument. An optional
-            flag to enable wait for ready mechanism
-          compression: An element of grpc.compression, e.g.
-            grpc.compression.Gzip. This is an EXPERIMENTAL option.
-
-        Returns:
-          A Call object instance which is an awaitable object.
-
-        Raises:
-          RpcError: Indicating that the RPC terminated with non-OK status. The
-            raised RpcError will also be a Call for the RPC affording the RPC's
-            metadata, status code, and details.
-        """
         if compression:
             metadata = _compression.augment_metadata(metadata, compression)
 
@@ -260,12 +162,12 @@ class StreamUnaryMultiCallable(_BaseMultiCallable):
                                credentials, wait_for_ready, self._channel,
                                self._method, self._request_serializer,
                                self._response_deserializer, self._loop)
-        self._ongoing_calls.trace_call(call)
+
         return call
 
 
-class StreamStreamMultiCallable(_BaseMultiCallable):
-    """Affords invoking a stream-stream RPC from client-side in an asynchronous way."""
+class StreamStreamMultiCallable(_BaseMultiCallable,
+                                _base_channel.StreamStreamMultiCallable):
 
     def __call__(self,
                  request_async_iterator: Optional[AsyncIterable[Any]] = None,
@@ -275,29 +177,6 @@ class StreamStreamMultiCallable(_BaseMultiCallable):
                  wait_for_ready: Optional[bool] = None,
                  compression: Optional[grpc.Compression] = None
                 ) -> _base_call.StreamStreamCall:
-        """Asynchronously invokes the underlying RPC.
-
-        Args:
-          request: The request value for the RPC.
-          timeout: An optional duration of time in seconds to allow
-            for the RPC.
-          metadata: Optional :term:`metadata` to be transmitted to the
-            service-side of the RPC.
-          credentials: An optional CallCredentials for the RPC. Only valid for
-            secure Channel.
-          wait_for_ready: This is an EXPERIMENTAL argument. An optional
-            flag to enable wait for ready mechanism
-          compression: An element of grpc.compression, e.g.
-            grpc.compression.Gzip. This is an EXPERIMENTAL option.
-
-        Returns:
-          A Call object instance which is an awaitable object.
-
-        Raises:
-          RpcError: Indicating that the RPC terminated with non-OK status. The
-            raised RpcError will also be a Call for the RPC affording the RPC's
-            metadata, status code, and details.
-        """
         if compression:
             metadata = _compression.augment_metadata(metadata, compression)
 
@@ -307,19 +186,14 @@ class StreamStreamMultiCallable(_BaseMultiCallable):
                                 credentials, wait_for_ready, self._channel,
                                 self._method, self._request_serializer,
                                 self._response_deserializer, self._loop)
-        self._ongoing_calls.trace_call(call)
-        return call
 
+        return call
 
-class Channel:
-    """Asynchronous Channel implementation.
 
-    A cygrpc.AioChannel-backed implementation.
-    """
+class Channel(_base_channel.Channel):
     _loop: asyncio.AbstractEventLoop
     _channel: cygrpc.AioChannel
     _unary_unary_interceptors: Optional[Sequence[UnaryUnaryClientInterceptor]]
-    _ongoing_calls: _OngoingCalls
 
     def __init__(self, target: str, options: ChannelArgumentType,
                  credentials: Optional[grpc.ChannelCredentials],
@@ -359,21 +233,11 @@ class Channel:
             _common.encode(target),
             _augment_channel_arguments(options, compression), credentials,
             self._loop)
-        self._ongoing_calls = _OngoingCalls()
 
     async def __aenter__(self):
-        """Starts an asynchronous context manager.
-
-        Returns:
-          Channel the channel that was instantiated.
-        """
         return self
 
     async def __aexit__(self, exc_type, exc_val, exc_tb):
-        """Finishes the asynchronous context manager by closing the channel.
-
-        Still active RPCs will be cancelled.
-        """
         await self._close(None)
 
     async def _close(self, grace):
@@ -383,54 +247,55 @@ class Channel:
         # No new calls will be accepted by the Cython channel.
         self._channel.closing()
 
-        if grace:
-            # pylint: disable=unused-variable
-            _, pending = await asyncio.wait(self._ongoing_calls.calls,
-                                            timeout=grace,
-                                            loop=self._loop)
-
-            if not pending:
-                return
-
-        # A new set is created acting as a shallow copy because
-        # when cancellation happens the calls are automatically
-        # removed from the originally set.
-        calls = WeakSet(data=self._ongoing_calls.calls)
+        # Iterate through running tasks
+        tasks = _all_tasks()
+        calls = []
+        call_tasks = []
+        for task in tasks:
+            stack = task.get_stack(limit=1)
+
+            # If the Task is created by a C-extension, the stack will be empty.
+            if not stack:
+                continue
+
+            # Locate ones created by `aio.Call`.
+            frame = stack[0]
+            candidate = frame.f_locals.get('self')
+            if candidate:
+                if isinstance(candidate, _base_call.Call):
+                    if hasattr(candidate, '_channel'):
+                        # For intercepted Call object
+                        if candidate._channel is not self._channel:
+                            continue
+                    elif hasattr(candidate, '_cython_call'):
+                        # For normal Call object
+                        if candidate._cython_call._channel is not self._channel:
+                            continue
+                    else:
+                        # Unidentified Call object
+                        raise cygrpc.InternalError(
+                            f'Unrecognized call object: {candidate}')
+
+                    calls.append(candidate)
+                    call_tasks.append(task)
+
+        # If needed, try to wait for them to finish.
+        # Call objects are not always awaitables.
+        if grace and call_tasks:
+            await asyncio.wait(call_tasks, timeout=grace, loop=self._loop)
+
+        # Time to cancel existing calls.
         for call in calls:
             call.cancel()
 
+        # Destroy the channel
         self._channel.close()
 
     async def close(self, grace: Optional[float] = None):
-        """Closes this Channel and releases all resources held by it.
-
-        This method immediately stops the channel from executing new RPCs in
-        all cases.
-
-        If a grace period is specified, this method wait until all active
-        RPCs are finshed, once the grace period is reached the ones that haven't
-        been terminated are cancelled. If a grace period is not specified
-        (by passing None for grace), all existing RPCs are cancelled immediately.
-
-        This method is idempotent.
-        """
         await self._close(grace)
 
     def get_state(self,
                   try_to_connect: bool = False) -> grpc.ChannelConnectivity:
-        """Check the connectivity state of a channel.
-
-        This is an EXPERIMENTAL API.
-
-        If the channel reaches a stable connectivity state, it is guaranteed
-        that the return value of this function will eventually converge to that
-        state.
-
-        Args: try_to_connect: a bool indicate whether the Channel should try to
-          connect to peer or not.
-
-        Returns: A ChannelConnectivity object.
-        """
         result = self._channel.check_connectivity_state(try_to_connect)
         return _common.CYGRPC_CONNECTIVITY_STATE_TO_CHANNEL_CONNECTIVITY[result]
 
@@ -438,31 +303,10 @@ class Channel:
             self,
             last_observed_state: grpc.ChannelConnectivity,
     ) -> None:
-        """Wait for a change in connectivity state.
-
-        This is an EXPERIMENTAL API.
-
-        The function blocks until there is a change in the channel connectivity
-        state from the "last_observed_state". If the state is already
-        different, this function will return immediately.
-
-        There is an inherent race between the invocation of
-        "Channel.wait_for_state_change" and "Channel.get_state". The state can
-        change arbitrary times during the race, so there is no way to observe
-        every state transition.
-
-        If there is a need to put a timeout for this function, please refer to
-        "asyncio.wait_for".
-
-        Args:
-          last_observed_state: A grpc.ChannelConnectivity object representing
-            the last known state.
-        """
         assert await self._channel.watch_connectivity_state(
             last_observed_state.value[0], None)
 
     async def channel_ready(self) -> None:
-        """Creates a coroutine that ends when a Channel is ready."""
         state = self.get_state(try_to_connect=True)
         while state != grpc.ChannelConnectivity.READY:
             await self.wait_for_state_change(state)
@@ -474,21 +318,7 @@ class Channel:
             request_serializer: Optional[SerializingFunction] = None,
             response_deserializer: Optional[DeserializingFunction] = None
     ) -> UnaryUnaryMultiCallable:
-        """Creates a UnaryUnaryMultiCallable for a unary-unary method.
-
-        Args:
-          method: The name of the RPC method.
-          request_serializer: Optional behaviour for serializing the request
-            message. Request goes unserialized in case None is passed.
-          response_deserializer: Optional behaviour for deserializing the
-            response message. Response goes undeserialized in case None
-            is passed.
-
-        Returns:
-          A UnaryUnaryMultiCallable value for the named unary-unary method.
-        """
-        return UnaryUnaryMultiCallable(self._channel, self._ongoing_calls,
-                                       _common.encode(method),
+        return UnaryUnaryMultiCallable(self._channel, _common.encode(method),
                                        request_serializer,
                                        response_deserializer,
                                        self._unary_unary_interceptors,
@@ -500,8 +330,7 @@ class Channel:
             request_serializer: Optional[SerializingFunction] = None,
             response_deserializer: Optional[DeserializingFunction] = None
     ) -> UnaryStreamMultiCallable:
-        return UnaryStreamMultiCallable(self._channel, self._ongoing_calls,
-                                        _common.encode(method),
+        return UnaryStreamMultiCallable(self._channel, _common.encode(method),
                                         request_serializer,
                                         response_deserializer, None, self._loop)
 
@@ -511,8 +340,7 @@ class Channel:
             request_serializer: Optional[SerializingFunction] = None,
             response_deserializer: Optional[DeserializingFunction] = None
     ) -> StreamUnaryMultiCallable:
-        return StreamUnaryMultiCallable(self._channel, self._ongoing_calls,
-                                        _common.encode(method),
+        return StreamUnaryMultiCallable(self._channel, _common.encode(method),
                                         request_serializer,
                                         response_deserializer, None, self._loop)
 
@@ -522,8 +350,55 @@ class Channel:
             request_serializer: Optional[SerializingFunction] = None,
             response_deserializer: Optional[DeserializingFunction] = None
     ) -> StreamStreamMultiCallable:
-        return StreamStreamMultiCallable(self._channel, self._ongoing_calls,
-                                         _common.encode(method),
+        return StreamStreamMultiCallable(self._channel, _common.encode(method),
                                          request_serializer,
                                          response_deserializer, None,
                                          self._loop)
+
+
+def insecure_channel(
+        target: str,
+        options: Optional[ChannelArgumentType] = None,
+        compression: Optional[grpc.Compression] = None,
+        interceptors: Optional[Sequence[UnaryUnaryClientInterceptor]] = None):
+    """Creates an insecure asynchronous Channel to a server.
+
+    Args:
+      target: The server address
+      options: An optional list of key-value pairs (channel args
+        in gRPC Core runtime) to configure the channel.
+      compression: An optional value indicating the compression method to be
+        used over the lifetime of the channel. This is an EXPERIMENTAL option.
+      interceptors: An optional sequence of interceptors that will be executed for
+        any call executed with this channel.
+
+    Returns:
+      A Channel.
+    """
+    return Channel(target, () if options is None else options, None,
+                   compression, interceptors)
+
+
+def secure_channel(
+        target: str,
+        credentials: grpc.ChannelCredentials,
+        options: Optional[ChannelArgumentType] = None,
+        compression: Optional[grpc.Compression] = None,
+        interceptors: Optional[Sequence[UnaryUnaryClientInterceptor]] = None):
+    """Creates a secure asynchronous Channel to a server.
+
+    Args:
+      target: The server address.
+      credentials: A ChannelCredentials instance.
+      options: An optional list of key-value pairs (channel args
+        in gRPC Core runtime) to configure the channel.
+      compression: An optional value indicating the compression method to be
+        used over the lifetime of the channel. This is an EXPERIMENTAL option.
+      interceptors: An optional sequence of interceptors that will be executed for
+        any call executed with this channel.
+
+    Returns:
+      An aio.Channel.
+    """
+    return Channel(target, () if options is None else options,
+                   credentials._credentials, compression, interceptors)

src/python/grpcio/grpc/experimental/aio/_interceptor.py

@@ -35,6 +35,19 @@ class ClientCallDetails(
             'ClientCallDetails',
             ('method', 'timeout', 'metadata', 'credentials', 'wait_for_ready')),
         grpc.ClientCallDetails):
+    """Describes an RPC to be invoked.
+
+    This is an EXPERIMENTAL API.
+
+    Args:
+        method: The method name of the RPC.
+        timeout: An optional duration of time in seconds to allow for the RPC.
+        metadata: Optional metadata to be transmitted to the service-side of
+          the RPC.
+        credentials: An optional CallCredentials for the RPC.
+        wait_for_ready: This is an EXPERIMENTAL argument. An optional flag to
+          enable wait for ready mechanism.
+    """
 
     method: str
     timeout: Optional[float]
@@ -53,6 +66,7 @@ class UnaryUnaryClientInterceptor(metaclass=ABCMeta):
             client_call_details: ClientCallDetails,
             request: RequestType) -> Union[UnaryUnaryCall, ResponseType]:
         """Intercepts a unary-unary invocation asynchronously.
+
         Args:
           continuation: A coroutine that proceeds with the invocation by
             executing the next interceptor in chain or invoking the
@@ -65,8 +79,10 @@ class UnaryUnaryClientInterceptor(metaclass=ABCMeta):
           client_call_details: A ClientCallDetails object describing the
             outgoing RPC.
           request: The request value for the RPC.
+
         Returns:
-            An object with the RPC response.
+          An object with the RPC response.
+
         Raises:
           AioRpcError: Indicating that the RPC terminated with non-OK status.
           asyncio.CancelledError: Indicating that the RPC was canceled.

src/python/grpcio/grpc/experimental/aio/_server.py

@@ -21,6 +21,7 @@ import grpc
 from grpc import _common, _compression
 from grpc._cython import cygrpc
 
+from . import _base_server
 from ._typing import ChannelArgumentType
 
 
@@ -30,7 +31,7 @@ def _augment_channel_arguments(base_options: ChannelArgumentType,
     return tuple(base_options) + compression_option
 
 
-class Server:
+class Server(_base_server.Server):
     """Serves RPCs."""
 
     def __init__(self, thread_pool: Optional[Executor],

src/python/grpcio/grpc_core_dependencies.py

@@ -208,6 +208,7 @@ CORE_SOURCE_FILES = [
     'src/core/lib/gpr/string_util_windows.cc',
     'src/core/lib/gpr/string_windows.cc',
     'src/core/lib/gpr/sync.cc',
+    'src/core/lib/gpr/sync_abseil.cc',
     'src/core/lib/gpr/sync_posix.cc',
     'src/core/lib/gpr/sync_windows.cc',
     'src/core/lib/gpr/time.cc',

src/python/grpcio_health_checking/grpc_health/v1/BUILD.bazel

@@ -16,7 +16,10 @@ py_grpc_library(
 
 py_library(
     name = "grpc_health",
-    srcs = ["health.py"],
+    srcs = [
+        "_async.py",
+        "health.py",
+    ],
     imports = ["../../"],
     deps = [
         ":health_py_pb2",

src/python/grpcio_health_checking/grpc_health/v1/_async.py

@@ -0,0 +1,113 @@
+# Copyright 2020 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Reference implementation for health checking in gRPC Python."""
+
+import asyncio
+import collections
+from typing import MutableMapping
+import grpc
+
+from grpc_health.v1 import health_pb2 as _health_pb2
+from grpc_health.v1 import health_pb2_grpc as _health_pb2_grpc
+
+
+class HealthServicer(_health_pb2_grpc.HealthServicer):
+    """An AsyncIO implementation of health checking servicer."""
+    _server_status: MutableMapping[
+        str, '_health_pb2.HealthCheckResponse.ServingStatus']
+    _server_watchers: MutableMapping[str, asyncio.Condition]
+    _gracefully_shutting_down: bool
+
+    def __init__(self) -> None:
+        self._server_status = dict()
+        self._server_watchers = collections.defaultdict(asyncio.Condition)
+        self._gracefully_shutting_down = False
+
+    async def Check(self, request: _health_pb2.HealthCheckRequest,
+                    context) -> None:
+        status = self._server_status.get(request.service)
+
+        if status is None:
+            await context.abort(grpc.StatusCode.NOT_FOUND)
+        else:
+            return _health_pb2.HealthCheckResponse(status=status)
+
+    async def Watch(self, request: _health_pb2.HealthCheckRequest,
+                    context) -> None:
+        condition = self._server_watchers[request.service]
+        last_status = None
+        try:
+            async with condition:
+                while True:
+                    status = self._server_status.get(
+                        request.service,
+                        _health_pb2.HealthCheckResponse.SERVICE_UNKNOWN)
+
+                    # NOTE(lidiz) If the observed status is the same, it means
+                    # there are missing intermediate statuses. It's considered
+                    # acceptable since peer only interested in eventual status.
+                    if status != last_status:
+                        # Responds with current health state
+                        await context.write(
+                            _health_pb2.HealthCheckResponse(status=status))
+
+                    # Records the last sent status
+                    last_status = status
+
+                    # Polling on health state changes
+                    await condition.wait()
+        finally:
+            if request.service in self._server_watchers:
+                del self._server_watchers[request.service]
+
+    async def _set(self, service: str,
+                   status: _health_pb2.HealthCheckResponse.ServingStatus
+                  ) -> None:
+        if service in self._server_watchers:
+            condition = self._server_watchers.get(service)
+            async with condition:
+                self._server_status[service] = status
+                condition.notify_all()
+        else:
+            self._server_status[service] = status
+
+    async def set(self, service: str,
+                  status: _health_pb2.HealthCheckResponse.ServingStatus
+                 ) -> None:
+        """Sets the status of a service.
+
+        Args:
+          service: string, the name of the service.
+          status: HealthCheckResponse.status enum value indicating the status of
+            the service
+        """
+        if self._gracefully_shutting_down:
+            return
+        else:
+            await self._set(service, status)
+
+    async def enter_graceful_shutdown(self) -> None:
+        """Permanently sets the status of all services to NOT_SERVING.
+
+        This should be invoked when the server is entering a graceful shutdown
+        period. After this method is invoked, future attempts to set the status
+        of a service will be ignored.
+        """
+        if self._gracefully_shutting_down:
+            return
+        else:
+            self._gracefully_shutting_down = True
+            for service in self._server_status:
+                await self._set(service,
+                                _health_pb2.HealthCheckResponse.NOT_SERVING)

src/python/grpcio_health_checking/grpc_health/v1/health.py

@@ -15,13 +15,20 @@
 
 import collections
 import threading
-
+import sys
 import grpc
 
 from grpc_health.v1 import health_pb2 as _health_pb2
 from grpc_health.v1 import health_pb2_grpc as _health_pb2_grpc
 
+if sys.version_info[0] >= 3 and sys.version_info[1] >= 6:
+    # Exposes AsyncHealthServicer as public API.
+    from . import _async as aio  # pylint: disable=unused-import
+
+# The service name of the health checking servicer.
 SERVICE_NAME = _health_pb2.DESCRIPTOR.services_by_name['Health'].full_name
+# The entry of overall health for the entire server.
+OVERALL_HEALTH = ''
 
 
 class _Watcher():
@@ -131,7 +138,7 @@ class HealthServicer(_health_pb2_grpc.HealthServicer):
         """Sets the status of a service.
 
         Args:
-          service: string, the name of the service. NOTE, '' must be set.
+          service: string, the name of the service.
           status: HealthCheckResponse.status enum value indicating the status of
             the service
         """

src/python/grpcio_tests/commands.py

@@ -106,6 +106,37 @@ class TestLite(setuptools.Command):
         self.distribution.fetch_build_eggs(self.distribution.tests_require)
 
 
+class TestPy3Only(setuptools.Command):
+    """Command to run tests for Python 3+ features.
+
+    This does not include asyncio tests, which are housed in a separate
+    directory.
+    """
+
+    description = 'run tests for py3+ features'
+    user_options = []
+
+    def initialize_options(self):
+        pass
+
+    def finalize_options(self):
+        pass
+
+    def run(self):
+        self._add_eggs_to_path()
+        import tests
+        loader = tests.Loader()
+        loader.loadTestsFromNames(['tests_py3_only'])
+        runner = tests.Runner()
+        result = runner.run(loader.suite)
+        if not result.wasSuccessful():
+            sys.exit('Test failure')
+
+    def _add_eggs_to_path(self):
+        self.distribution.fetch_build_eggs(self.distribution.install_requires)
+        self.distribution.fetch_build_eggs(self.distribution.tests_require)
+
+
 class TestAio(setuptools.Command):
     """Command to run aio tests without fetching or building anything."""
 

src/python/grpcio_tests/setup.py

@@ -59,6 +59,7 @@ COMMAND_CLASS = {
     'test_lite': commands.TestLite,
     'test_gevent': commands.TestGevent,
     'test_aio': commands.TestAio,
+    'test_py3_only': commands.TestPy3Only,
 }
 
 PACKAGE_DATA = {

src/python/grpcio_tests/tests/qps/BUILD.bazel

@@ -0,0 +1,27 @@
+# Copyright 2020 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package(
+    default_testonly = 1,
+    default_visibility = ["//visibility:public"],
+)
+
+py_library(
+    name = "histogram",
+    srcs = ["histogram.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        "//src/proto/grpc/testing:stats_py_pb2",
+    ],
+)

src/python/grpcio_tests/tests/qps/histogram.py

@@ -65,6 +65,16 @@ class Histogram(object):
             data.count = self._count
             return data
 
+    def merge(self, another_data):
+        with self._lock:
+            for i in range(len(self._buckets)):
+                self._buckets[i] += another_data.bucket[i]
+            self._min = min(self._min, another_data.min_seen)
+            self._max = max(self._max, another_data.max_seen)
+            self._sum += another_data.sum
+            self._sum_of_squares += another_data.sum_of_squares
+            self._count += another_data.count
+
     def _bucket_for(self, val):
         val = min(val, self._max_possible)
         return int(math.log(val, self.multiplier))

src/python/grpcio_tests/tests/unit/_invocation_defects_test.py

@@ -12,8 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import itertools
-import threading
 import unittest
 import logging
 
@@ -35,26 +33,6 @@ _STREAM_STREAM = '/test/StreamStream'
 _DEFECTIVE_GENERIC_RPC_HANDLER = '/test/DefectiveGenericRpcHandler'
 
 
-class _Callback(object):
-
-    def __init__(self):
-        self._condition = threading.Condition()
-        self._value = None
-        self._called = False
-
-    def __call__(self, value):
-        with self._condition:
-            self._value = value
-            self._called = True
-            self._condition.notify_all()
-
-    def value(self):
-        with self._condition:
-            while not self._called:
-                self._condition.wait()
-            return self._value
-
-
 class _Handler(object):
 
     def __init__(self, control):
@@ -199,6 +177,7 @@ def _defective_handler_multi_callable(channel):
 
 
 class InvocationDefectsTest(unittest.TestCase):
+    """Tests the handling of exception-raising user code on the client-side."""
 
     def setUp(self):
         self._control = test_control.PauseFailControl()
@@ -216,35 +195,44 @@ class InvocationDefectsTest(unittest.TestCase):
         self._channel.close()
 
     def testIterableStreamRequestBlockingUnaryResponse(self):
-        requests = [b'\x07\x08' for _ in range(test_constants.STREAM_LENGTH)]
+        requests = object()
         multi_callable = _stream_unary_multi_callable(self._channel)
 
-        with self.assertRaises(grpc.RpcError):
-            response = multi_callable(
+        with self.assertRaises(grpc.RpcError) as exception_context:
+            multi_callable(
                 requests,
                 metadata=(('test',
                            'IterableStreamRequestBlockingUnaryResponse'),))
 
+        self.assertIs(grpc.StatusCode.UNKNOWN,
+                      exception_context.exception.code())
+
     def testIterableStreamRequestFutureUnaryResponse(self):
-        requests = [b'\x07\x08' for _ in range(test_constants.STREAM_LENGTH)]
+        requests = object()
         multi_callable = _stream_unary_multi_callable(self._channel)
         response_future = multi_callable.future(
             requests,
             metadata=(('test', 'IterableStreamRequestFutureUnaryResponse'),))
 
-        with self.assertRaises(grpc.RpcError):
-            response = response_future.result()
+        with self.assertRaises(grpc.RpcError) as exception_context:
+            response_future.result()
+
+        self.assertIs(grpc.StatusCode.UNKNOWN,
+                      exception_context.exception.code())
 
     def testIterableStreamRequestStreamResponse(self):
-        requests = [b'\x77\x58' for _ in range(test_constants.STREAM_LENGTH)]
+        requests = object()
         multi_callable = _stream_stream_multi_callable(self._channel)
         response_iterator = multi_callable(
             requests,
             metadata=(('test', 'IterableStreamRequestStreamResponse'),))
 
-        with self.assertRaises(grpc.RpcError):
+        with self.assertRaises(grpc.RpcError) as exception_context:
             next(response_iterator)
 
+        self.assertIs(grpc.StatusCode.UNKNOWN,
+                      exception_context.exception.code())
+
     def testIteratorStreamRequestStreamResponse(self):
         requests_iterator = FailAfterFewIterationsCounter(
             test_constants.STREAM_LENGTH // 2, b'\x07\x08')
@@ -253,18 +241,21 @@ class InvocationDefectsTest(unittest.TestCase):
             requests_iterator,
             metadata=(('test', 'IteratorStreamRequestStreamResponse'),))
 
-        with self.assertRaises(grpc.RpcError):
+        with self.assertRaises(grpc.RpcError) as exception_context:
             for _ in range(test_constants.STREAM_LENGTH // 2 + 1):
                 next(response_iterator)
 
+        self.assertIs(grpc.StatusCode.UNKNOWN,
+                      exception_context.exception.code())
+
     def testDefectiveGenericRpcHandlerUnaryResponse(self):
         request = b'\x07\x08'
         multi_callable = _defective_handler_multi_callable(self._channel)
 
         with self.assertRaises(grpc.RpcError) as exception_context:
-            response = multi_callable(
-                request,
-                metadata=(('test', 'DefectiveGenericRpcHandlerUnary'),))
+            multi_callable(request,
+                           metadata=(('test',
+                                      'DefectiveGenericRpcHandlerUnary'),))
 
         self.assertIs(grpc.StatusCode.UNKNOWN,
                       exception_context.exception.code())

src/python/grpcio_tests/tests/unit/_metadata_code_details_test.py

@@ -250,7 +250,6 @@ class MetadataCodeDetailsTest(unittest.TestCase):
             test_common.metadata_transmitted(_SERVER_TRAILING_METADATA,
                                              call.trailing_metadata()))
         self.assertIs(grpc.StatusCode.OK, call.code())
-        self.assertEqual(_DETAILS, call.details())
 
     def testSuccessfulUnaryStream(self):
         self._servicer.set_details(_DETAILS)
@@ -271,7 +270,6 @@ class MetadataCodeDetailsTest(unittest.TestCase):
                 _SERVER_TRAILING_METADATA,
                 response_iterator_call.trailing_metadata()))
         self.assertIs(grpc.StatusCode.OK, response_iterator_call.code())
-        self.assertEqual(_DETAILS, response_iterator_call.details())
 
     def testSuccessfulStreamUnary(self):
         self._servicer.set_details(_DETAILS)
@@ -290,7 +288,6 @@ class MetadataCodeDetailsTest(unittest.TestCase):
             test_common.metadata_transmitted(_SERVER_TRAILING_METADATA,
                                              call.trailing_metadata()))
         self.assertIs(grpc.StatusCode.OK, call.code())
-        self.assertEqual(_DETAILS, call.details())
 
     def testSuccessfulStreamStream(self):
         self._servicer.set_details(_DETAILS)
@@ -312,7 +309,6 @@ class MetadataCodeDetailsTest(unittest.TestCase):
                 _SERVER_TRAILING_METADATA,
                 response_iterator_call.trailing_metadata()))
         self.assertIs(grpc.StatusCode.OK, response_iterator_call.code())
-        self.assertEqual(_DETAILS, response_iterator_call.details())
 
     def testAbortedUnaryUnary(self):
         test_cases = zip(_ABORT_CODES, _EXPECTED_CLIENT_CODES,

src/python/grpcio_tests/tests_aio/benchmark/BUILD.bazel

@@ -17,16 +17,67 @@ package(
     default_visibility = ["//visibility:public"],
 )
 
-py_binary(
-    name = "server",
-    srcs = ["server.py"],
-    python_version = "PY3",
+py_library(
+    name = "benchmark_client",
+    srcs = ["benchmark_client.py"],
+    srcs_version = "PY3",
     deps = [
-        "//src/proto/grpc/testing:benchmark_service_py_pb2",
         "//src/proto/grpc/testing:benchmark_service_py_pb2_grpc",
         "//src/proto/grpc/testing:py_messages_proto",
         "//src/python/grpcio/grpc:grpcio",
+        "//src/python/grpcio_tests/tests/qps:histogram",
+        "//src/python/grpcio_tests/tests/unit:resources",
+    ],
+)
+
+py_library(
+    name = "benchmark_servicer",
+    srcs = ["benchmark_servicer.py"],
+    srcs_version = "PY3",
+    deps = [
+        "//src/proto/grpc/testing:benchmark_service_py_pb2_grpc",
+        "//src/proto/grpc/testing:py_messages_proto",
+        "//src/python/grpcio/grpc:grpcio",
+    ],
+)
+
+py_library(
+    name = "worker_servicer",
+    srcs = ["worker_servicer.py"],
+    data = [
+        "//src/python/grpcio_tests/tests/unit/credentials",
+    ],
+    srcs_version = "PY3",
+    deps = [
+        ":benchmark_client",
+        ":benchmark_servicer",
+        "//src/proto/grpc/core:stats_py_pb2",
+        "//src/proto/grpc/testing:benchmark_service_py_pb2_grpc",
+        "//src/proto/grpc/testing:control_py_pb2",
+        "//src/proto/grpc/testing:payloads_py_pb2",
+        "//src/proto/grpc/testing:stats_py_pb2",
+        "//src/proto/grpc/testing:worker_service_py_pb2_grpc",
+        "//src/python/grpcio/grpc:grpcio",
+        "//src/python/grpcio_tests/tests/qps:histogram",
+        "//src/python/grpcio_tests/tests/unit:resources",
         "//src/python/grpcio_tests/tests/unit/framework/common",
-        "@six",
+    ],
+)
+
+py_binary(
+    name = "server",
+    srcs = ["server.py"],
+    python_version = "PY3",
+    deps = [":benchmark_servicer"],
+)
+
+py_binary(
+    name = "worker",
+    srcs = ["worker.py"],
+    imports = ["../../"],
+    python_version = "PY3",
+    deps = [
+        ":worker_servicer",
+        "//src/proto/grpc/testing:worker_service_py_pb2_grpc",
     ],
 )

src/python/grpcio_tests/tests_aio/benchmark/benchmark_client.py

@@ -0,0 +1,155 @@
+# Copyright 2020 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""The Python AsyncIO Benchmark Clients."""
+
+import abc
+import asyncio
+import time
+import logging
+import random
+
+import grpc
+from grpc.experimental import aio
+
+from src.proto.grpc.testing import (benchmark_service_pb2_grpc, control_pb2,
+                                    messages_pb2)
+from tests.qps import histogram
+from tests.unit import resources
+
+
+class GenericStub(object):
+
+    def __init__(self, channel: aio.Channel):
+        self.UnaryCall = channel.unary_unary(
+            '/grpc.testing.BenchmarkService/UnaryCall')
+        self.StreamingCall = channel.stream_stream(
+            '/grpc.testing.BenchmarkService/StreamingCall')
+
+
+class BenchmarkClient(abc.ABC):
+    """Benchmark client interface that exposes a non-blocking send_request()."""
+
+    def __init__(self, address: str, config: control_pb2.ClientConfig,
+                 hist: histogram.Histogram):
+        # Disables underlying reuse of subchannels
+        unique_option = (('iv', random.random()),)
+
+        # Parses the channel argument from config
+        channel_args = tuple(
+            (arg.name, arg.str_value) if arg.HasField('str_value') else (
+                arg.name, int(arg.int_value)) for arg in config.channel_args)
+
+        # Creates the channel
+        if config.HasField('security_params'):
+            channel_credentials = grpc.ssl_channel_credentials(
+                resources.test_root_certificates(),)
+            server_host_override_option = ((
+                'grpc.ssl_target_name_override',
+                config.security_params.server_host_override,
+            ),)
+            self._channel = aio.secure_channel(
+                address, channel_credentials,
+                unique_option + channel_args + server_host_override_option)
+        else:
+            self._channel = aio.insecure_channel(address,
+                                                 options=unique_option +
+                                                 channel_args)
+
+        # Creates the stub
+        if config.payload_config.WhichOneof('payload') == 'simple_params':
+            self._generic = False
+            self._stub = benchmark_service_pb2_grpc.BenchmarkServiceStub(
+                self._channel)
+            payload = messages_pb2.Payload(
+                body=b'\0' * config.payload_config.simple_params.req_size)
+            self._request = messages_pb2.SimpleRequest(
+                payload=payload,
+                response_size=config.payload_config.simple_params.resp_size)
+        else:
+            self._generic = True
+            self._stub = GenericStub(self._channel)
+            self._request = b'\0' * config.payload_config.bytebuf_params.req_size
+
+        self._hist = hist
+        self._response_callbacks = []
+        self._concurrency = config.outstanding_rpcs_per_channel
+
+    async def run(self) -> None:
+        await self._channel.channel_ready()
+
+    async def stop(self) -> None:
+        await self._channel.close()
+
+    def _record_query_time(self, query_time: float) -> None:
+        self._hist.add(query_time * 1e9)
+
+
+class UnaryAsyncBenchmarkClient(BenchmarkClient):
+
+    def __init__(self, address: str, config: control_pb2.ClientConfig,
+                 hist: histogram.Histogram):
+        super().__init__(address, config, hist)
+        self._running = None
+        self._stopped = asyncio.Event()
+
+    async def _send_request(self):
+        start_time = time.monotonic()
+        await self._stub.UnaryCall(self._request)
+        self._record_query_time(time.monotonic() - start_time)
+
+    async def _send_indefinitely(self) -> None:
+        while self._running:
+            await self._send_request()
+
+    async def run(self) -> None:
+        await super().run()
+        self._running = True
+        senders = (self._send_indefinitely() for _ in range(self._concurrency))
+        await asyncio.gather(*senders)
+        self._stopped.set()
+
+    async def stop(self) -> None:
+        self._running = False
+        await self._stopped.wait()
+        await super().stop()
+
+
+class StreamingAsyncBenchmarkClient(BenchmarkClient):
+
+    def __init__(self, address: str, config: control_pb2.ClientConfig,
+                 hist: histogram.Histogram):
+        super().__init__(address, config, hist)
+        self._running = None
+        self._stopped = asyncio.Event()
+
+    async def _one_streaming_call(self):
+        call = self._stub.StreamingCall()
+        while self._running:
+            start_time = time.time()
+            await call.write(self._request)
+            await call.read()
+            self._record_query_time(time.time() - start_time)
+        await call.done_writing()
+
+    async def run(self):
+        await super().run()
+        self._running = True
+        senders = (self._one_streaming_call() for _ in range(self._concurrency))
+        await asyncio.gather(*senders)
+        self._stopped.set()
+
+    async def stop(self):
+        self._running = False
+        await self._stopped.wait()
+        await super().stop()

src/python/grpcio_tests/tests_aio/benchmark/benchmark_servicer.py

@@ -0,0 +1,55 @@
+# Copyright 2020 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""The Python AsyncIO Benchmark Servicers."""
+
+import asyncio
+import logging
+import unittest
+
+from grpc.experimental import aio
+
+from src.proto.grpc.testing import benchmark_service_pb2_grpc, messages_pb2
+
+
+class BenchmarkServicer(benchmark_service_pb2_grpc.BenchmarkServiceServicer):
+
+    async def UnaryCall(self, request, unused_context):
+        payload = messages_pb2.Payload(body=b'\0' * request.response_size)
+        return messages_pb2.SimpleResponse(payload=payload)
+
+    async def StreamingFromServer(self, request, unused_context):
+        payload = messages_pb2.Payload(body=b'\0' * request.response_size)
+        # Sends response at full capacity!
+        while True:
+            yield messages_pb2.SimpleResponse(payload=payload)
+
+    async def StreamingCall(self, request_iterator, unused_context):
+        async for request in request_iterator:
+            payload = messages_pb2.Payload(body=b'\0' * request.response_size)
+            yield messages_pb2.SimpleResponse(payload=payload)
+
+
+class GenericBenchmarkServicer(
+        benchmark_service_pb2_grpc.BenchmarkServiceServicer):
+    """Generic (no-codec) Server implementation for the Benchmark service."""
+
+    def __init__(self, resp_size):
+        self._response = '\0' * resp_size
+
+    async def UnaryCall(self, unused_request, unused_context):
+        return self._response
+
+    async def StreamingCall(self, request_iterator, unused_context):
+        async for _ in request_iterator:
+            yield self._response

src/python/grpcio_tests/tests_aio/benchmark/server.py

@@ -17,28 +17,16 @@ import logging
 import unittest
 
 from grpc.experimental import aio
-from src.proto.grpc.testing import messages_pb2
-from src.proto.grpc.testing import benchmark_service_pb2_grpc
-
-
-class BenchmarkServer(benchmark_service_pb2_grpc.BenchmarkServiceServicer):
 
-    async def UnaryCall(self, request, context):
-        payload = messages_pb2.Payload(body=b'\0' * request.response_size)
-        return messages_pb2.SimpleResponse(payload=payload)
-
-    async def StreamingFromServer(self, request, context):
-        payload = messages_pb2.Payload(body=b'\0' * request.response_size)
-        # Sends response at full capacity!
-        while True:
-            yield messages_pb2.SimpleResponse(payload=payload)
+from src.proto.grpc.testing import benchmark_service_pb2_grpc
+from tests_aio.benchmark import benchmark_servicer
 
 
 async def _start_async_server():
     server = aio.server()
 
     port = server.add_insecure_port('localhost:%s' % 50051)
-    servicer = BenchmarkServer()
+    servicer = benchmark_servicer.BenchmarkServicer()
     benchmark_service_pb2_grpc.add_BenchmarkServiceServicer_to_server(
         servicer, server)
 

src/python/grpcio_tests/tests_aio/benchmark/worker.py

@@ -0,0 +1,58 @@
+# Copyright 2020 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import asyncio
+import logging
+
+from grpc.experimental import aio
+
+from src.proto.grpc.testing import worker_service_pb2_grpc
+from tests_aio.benchmark import worker_servicer
+
+
+async def run_worker_server(port: int) -> None:
+    aio.init_grpc_aio()
+    server = aio.server()
+
+    servicer = worker_servicer.WorkerServicer()
+    worker_service_pb2_grpc.add_WorkerServiceServicer_to_server(
+        servicer, server)
+
+    server.add_insecure_port('[::]:{}'.format(port))
+
+    await server.start()
+
+    await servicer.wait_for_quit()
+    await server.stop(None)
+
+
+if __name__ == '__main__':
+    logging.basicConfig(level=logging.DEBUG)
+    parser = argparse.ArgumentParser(
+        description='gRPC Python performance testing worker')
+    parser.add_argument('--driver_port',
+                        type=int,
+                        dest='port',
+                        help='The port the worker should listen on')
+    parser.add_argument('--uvloop',
+                        action='store_true',
+                        help='Use uvloop or not')
+    args = parser.parse_args()
+
+    if args.uvloop:
+        import uvloop
+        uvloop.install()
+
+    asyncio.get_event_loop().run_until_complete(run_worker_server(args.port))

src/python/grpcio_tests/tests_aio/benchmark/worker_servicer.py

@@ -0,0 +1,367 @@
+# Copyright 2020 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import asyncio
+import collections
+import logging
+import multiprocessing
+import os
+import sys
+import time
+from typing import Tuple
+
+import grpc
+from grpc.experimental import aio
+
+from src.proto.grpc.testing import (benchmark_service_pb2_grpc, control_pb2,
+                                    stats_pb2, worker_service_pb2_grpc)
+from tests.qps import histogram
+from tests.unit import resources
+from tests.unit.framework.common import get_socket
+from tests_aio.benchmark import benchmark_client, benchmark_servicer
+
+_NUM_CORES = multiprocessing.cpu_count()
+_WORKER_ENTRY_FILE = os.path.join(
+    os.path.split(os.path.abspath(__file__))[0], 'worker.py')
+
+_LOGGER = logging.getLogger(__name__)
+
+
+class _SubWorker(
+        collections.namedtuple('_SubWorker',
+                               ['process', 'port', 'channel', 'stub'])):
+    """A data class that holds information about a child qps worker."""
+
+    def _repr(self):
+        return f'<_SubWorker pid={self.process.pid} port={self.port}>'
+
+    def __repr__(self):
+        return self._repr()
+
+    def __str__(self):
+        return self._repr()
+
+
+def _get_server_status(start_time: float, end_time: float,
+                       port: int) -> control_pb2.ServerStatus:
+    """Creates ServerStatus proto message."""
+    end_time = time.monotonic()
+    elapsed_time = end_time - start_time
+    # TODO(lidiz) Collect accurate time system to compute QPS/core-second.
+    stats = stats_pb2.ServerStats(time_elapsed=elapsed_time,
+                                  time_user=elapsed_time,
+                                  time_system=elapsed_time)
+    return control_pb2.ServerStatus(stats=stats, port=port, cores=_NUM_CORES)
+
+
+def _create_server(config: control_pb2.ServerConfig) -> Tuple[aio.Server, int]:
+    """Creates a server object according to the ServerConfig."""
+    channel_args = tuple(
+        (arg.name,
+         arg.str_value) if arg.HasField('str_value') else (arg.name,
+                                                           int(arg.int_value))
+        for arg in config.channel_args)
+
+    server = aio.server(options=channel_args + (('grpc.so_reuseport', 1),))
+    if config.server_type == control_pb2.ASYNC_SERVER:
+        servicer = benchmark_servicer.BenchmarkServicer()
+        benchmark_service_pb2_grpc.add_BenchmarkServiceServicer_to_server(
+            servicer, server)
+    elif config.server_type == control_pb2.ASYNC_GENERIC_SERVER:
+        resp_size = config.payload_config.bytebuf_params.resp_size
+        servicer = benchmark_servicer.GenericBenchmarkServicer(resp_size)
+        method_implementations = {
+            'StreamingCall':
+                grpc.stream_stream_rpc_method_handler(servicer.StreamingCall),
+            'UnaryCall':
+                grpc.unary_unary_rpc_method_handler(servicer.UnaryCall),
+        }
+        handler = grpc.method_handlers_generic_handler(
+            'grpc.testing.BenchmarkService', method_implementations)
+        server.add_generic_rpc_handlers((handler,))
+    else:
+        raise NotImplementedError('Unsupported server type {}'.format(
+            config.server_type))
+
+    if config.HasField('security_params'):  # Use SSL
+        server_creds = grpc.ssl_server_credentials(
+            ((resources.private_key(), resources.certificate_chain()),))
+        port = server.add_secure_port('[::]:{}'.format(config.port),
+                                      server_creds)
+    else:
+        port = server.add_insecure_port('[::]:{}'.format(config.port))
+
+    return server, port
+
+
+def _get_client_status(start_time: float, end_time: float,
+                       qps_data: histogram.Histogram
+                      ) -> control_pb2.ClientStatus:
+    """Creates ClientStatus proto message."""
+    latencies = qps_data.get_data()
+    end_time = time.monotonic()
+    elapsed_time = end_time - start_time
+    # TODO(lidiz) Collect accurate time system to compute QPS/core-second.
+    stats = stats_pb2.ClientStats(latencies=latencies,
+                                  time_elapsed=elapsed_time,
+                                  time_user=elapsed_time,
+                                  time_system=elapsed_time)
+    return control_pb2.ClientStatus(stats=stats)
+
+
+def _create_client(server: str, config: control_pb2.ClientConfig,
+                   qps_data: histogram.Histogram
+                  ) -> benchmark_client.BenchmarkClient:
+    """Creates a client object according to the ClientConfig."""
+    if config.load_params.WhichOneof('load') != 'closed_loop':
+        raise NotImplementedError(
+            f'Unsupported load parameter {config.load_params}')
+
+    if config.client_type == control_pb2.ASYNC_CLIENT:
+        if config.rpc_type == control_pb2.UNARY:
+            client_type = benchmark_client.UnaryAsyncBenchmarkClient
+        elif config.rpc_type == control_pb2.STREAMING:
+            client_type = benchmark_client.StreamingAsyncBenchmarkClient
+        else:
+            raise NotImplementedError(
+                f'Unsupported rpc_type [{config.rpc_type}]')
+    else:
+        raise NotImplementedError(
+            f'Unsupported client type {config.client_type}')
+
+    return client_type(server, config, qps_data)
+
+
+def _pick_an_unused_port() -> int:
+    """Picks an unused TCP port."""
+    _, port, sock = get_socket()
+    sock.close()
+    return port
+
+
+async def _create_sub_worker() -> _SubWorker:
+    """Creates a child qps worker as a subprocess."""
+    port = _pick_an_unused_port()
+
+    _LOGGER.info('Creating sub worker at port [%d]...', port)
+    process = await asyncio.create_subprocess_exec(sys.executable,
+                                                   _WORKER_ENTRY_FILE,
+                                                   '--driver_port', str(port))
+    _LOGGER.info('Created sub worker process for port [%d] at pid [%d]', port,
+                 process.pid)
+    channel = aio.insecure_channel(f'localhost:{port}')
+    _LOGGER.info('Waiting for sub worker at port [%d]', port)
+    await channel.channel_ready()
+    stub = worker_service_pb2_grpc.WorkerServiceStub(channel)
+    return _SubWorker(
+        process=process,
+        port=port,
+        channel=channel,
+        stub=stub,
+    )
+
+
+class WorkerServicer(worker_service_pb2_grpc.WorkerServiceServicer):
+    """Python Worker Server implementation."""
+
+    def __init__(self):
+        self._loop = asyncio.get_event_loop()
+        self._quit_event = asyncio.Event()
+
+    async def _run_single_server(self, config, request_iterator, context):
+        server, port = _create_server(config)
+        await server.start()
+        _LOGGER.info('Server started at port [%d]', port)
+
+        start_time = time.monotonic()
+        await context.write(_get_server_status(start_time, start_time, port))
+
+        async for request in request_iterator:
+            end_time = time.monotonic()
+            status = _get_server_status(start_time, end_time, port)
+            if request.mark.reset:
+                start_time = end_time
+            await context.write(status)
+        await server.stop(None)
+
+    async def RunServer(self, request_iterator, context):
+        config_request = await context.read()
+        config = config_request.setup
+        _LOGGER.info('Received ServerConfig: %s', config)
+
+        if config.server_processes <= 0:
+            _LOGGER.info('Using server_processes == [%d]', _NUM_CORES)
+            config.server_processes = _NUM_CORES
+
+        if config.port == 0:
+            config.port = _pick_an_unused_port()
+        _LOGGER.info('Port picked [%d]', config.port)
+
+        if config.server_processes == 1:
+            # If server_processes == 1, start the server in this process.
+            await self._run_single_server(config, request_iterator, context)
+        else:
+            # If server_processes > 1, offload to other processes.
+            sub_workers = await asyncio.gather(*(
+                _create_sub_worker() for _ in range(config.server_processes)))
+
+            calls = [worker.stub.RunServer() for worker in sub_workers]
+
+            config_request.setup.server_processes = 1
+
+            for call in calls:
+                await call.write(config_request)
+                # An empty status indicates the peer is ready
+                await call.read()
+
+            start_time = time.monotonic()
+            await context.write(
+                _get_server_status(
+                    start_time,
+                    start_time,
+                    config.port,
+                ))
+
+            _LOGGER.info('Servers are ready to serve.')
+
+            async for request in request_iterator:
+                end_time = time.monotonic()
+
+                for call in calls:
+                    await call.write(request)
+                    # Reports from sub workers doesn't matter
+                    await call.read()
+
+                status = _get_server_status(
+                    start_time,
+                    end_time,
+                    config.port,
+                )
+                if request.mark.reset:
+                    start_time = end_time
+                await context.write(status)
+
+            for call in calls:
+                await call.done_writing()
+
+            for worker in sub_workers:
+                await worker.stub.QuitWorker(control_pb2.Void())
+                await worker.channel.close()
+                _LOGGER.info('Waiting for [%s] to quit...', worker)
+                await worker.process.wait()
+
+    async def _run_single_client(self, config, request_iterator, context):
+        running_tasks = []
+        qps_data = histogram.Histogram(config.histogram_params.resolution,
+                                       config.histogram_params.max_possible)
+        start_time = time.monotonic()
+
+        # Create a client for each channel as asyncio.Task
+        for i in range(config.client_channels):
+            server = config.server_targets[i % len(config.server_targets)]
+            client = _create_client(server, config, qps_data)
+            _LOGGER.info('Client created against server [%s]', server)
+            running_tasks.append(self._loop.create_task(client.run()))
+
+        end_time = time.monotonic()
+        await context.write(_get_client_status(start_time, end_time, qps_data))
+
+        # Respond to stat requests
+        async for request in request_iterator:
+            end_time = time.monotonic()
+            status = _get_client_status(start_time, end_time, qps_data)
+            if request.mark.reset:
+                qps_data.reset()
+                start_time = time.monotonic()
+            await context.write(status)
+
+        # Cleanup the clients
+        for task in running_tasks:
+            task.cancel()
+
+    async def RunClient(self, request_iterator, context):
+        config_request = await context.read()
+        config = config_request.setup
+        _LOGGER.info('Received ClientConfig: %s', config)
+
+        if config.client_processes <= 0:
+            _LOGGER.info('client_processes can\'t be [%d]',
+                         config.client_processes)
+            _LOGGER.info('Using client_processes == [%d]', _NUM_CORES)
+            config.client_processes = _NUM_CORES
+
+        if config.client_processes == 1:
+            # If client_processes == 1, run the benchmark in this process.
+            await self._run_single_client(config, request_iterator, context)
+        else:
+            # If client_processes > 1, offload the work to other processes.
+            sub_workers = await asyncio.gather(*(
+                _create_sub_worker() for _ in range(config.client_processes)))
+
+            calls = [worker.stub.RunClient() for worker in sub_workers]
+
+            config_request.setup.client_processes = 1
+
+            for call in calls:
+                await call.write(config_request)
+                # An empty status indicates the peer is ready
+                await call.read()
+
+            start_time = time.monotonic()
+            result = histogram.Histogram(config.histogram_params.resolution,
+                                         config.histogram_params.max_possible)
+            end_time = time.monotonic()
+            await context.write(_get_client_status(start_time, end_time,
+                                                   result))
+
+            async for request in request_iterator:
+                end_time = time.monotonic()
+
+                for call in calls:
+                    _LOGGER.debug('Fetching status...')
+                    await call.write(request)
+                    sub_status = await call.read()
+                    result.merge(sub_status.stats.latencies)
+                    _LOGGER.debug('Update from sub worker count=[%d]',
+                                  sub_status.stats.latencies.count)
+
+                status = _get_client_status(start_time, end_time, result)
+                if request.mark.reset:
+                    result.reset()
+                    start_time = time.monotonic()
+                _LOGGER.debug('Reporting count=[%d]',
+                              status.stats.latencies.count)
+                await context.write(status)
+
+            for call in calls:
+                await call.done_writing()
+
+            for worker in sub_workers:
+                await worker.stub.QuitWorker(control_pb2.Void())
+                await worker.channel.close()
+                _LOGGER.info('Waiting for sub worker [%s] to quit...', worker)
+                await worker.process.wait()
+                _LOGGER.info('Sub worker [%s] quit', worker)
+
+    @staticmethod
+    async def CoreCount(unused_request, unused_context):
+        return control_pb2.CoreResponse(cores=_NUM_CORES)
+
+    async def QuitWorker(self, unused_request, unused_context):
+        _LOGGER.info('QuitWorker command received.')
+        self._quit_event.set()
+        return control_pb2.Void()
+
+    async def wait_for_quit(self):
+        await self._quit_event.wait()

src/python/grpcio_tests/tests_aio/health_check/BUILD.bazel

@@ -0,0 +1,29 @@
+# Copyright 2020 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package(default_testonly = 1)
+
+py_test(
+    name = "health_servicer_test",
+    size = "small",
+    srcs = ["health_servicer_test.py"],
+    imports = ["../../"],
+    python_version = "PY3",
+    deps = [
+        "//src/python/grpcio/grpc:grpcio",
+        "//src/python/grpcio_health_checking/grpc_health/v1:grpc_health",
+        "//src/python/grpcio_tests/tests/unit/framework/common",
+        "//src/python/grpcio_tests/tests_aio/unit:_test_base",
+    ],
+)