Merge pull request #2647 from vjpai/deepend

Dynamic-sized thread pool

BUILD

@@ -665,6 +665,7 @@ cc_library(
     "src/cpp/proto/proto_utils.cc",
     "src/cpp/server/async_generic_service.cc",
     "src/cpp/server/create_default_thread_pool.cc",
+    "src/cpp/server/dynamic_thread_pool.cc",
     "src/cpp/server/fixed_size_thread_pool.cc",
     "src/cpp/server/insecure_server_credentials.cc",
     "src/cpp/server/server.cc",
@@ -690,6 +691,7 @@ cc_library(
     "include/grpc++/config_protobuf.h",
     "include/grpc++/create_channel.h",
     "include/grpc++/credentials.h",
+    "include/grpc++/dynamic_thread_pool.h",
     "include/grpc++/fixed_size_thread_pool.h",
     "include/grpc++/generic_stub.h",
     "include/grpc++/impl/call.h",
@@ -750,6 +752,7 @@ cc_library(
     "src/cpp/proto/proto_utils.cc",
     "src/cpp/server/async_generic_service.cc",
     "src/cpp/server/create_default_thread_pool.cc",
+    "src/cpp/server/dynamic_thread_pool.cc",
     "src/cpp/server/fixed_size_thread_pool.cc",
     "src/cpp/server/insecure_server_credentials.cc",
     "src/cpp/server/server.cc",
@@ -775,6 +778,7 @@ cc_library(
     "include/grpc++/config_protobuf.h",
     "include/grpc++/create_channel.h",
     "include/grpc++/credentials.h",
+    "include/grpc++/dynamic_thread_pool.h",
     "include/grpc++/fixed_size_thread_pool.h",
     "include/grpc++/generic_stub.h",
     "include/grpc++/impl/call.h",

build.json

@@ -43,6 +43,7 @@
         "include/grpc++/config_protobuf.h",
         "include/grpc++/create_channel.h",
         "include/grpc++/credentials.h",
+        "include/grpc++/dynamic_thread_pool.h",
         "include/grpc++/fixed_size_thread_pool.h",
         "include/grpc++/generic_stub.h",
         "include/grpc++/impl/call.h",
@@ -90,6 +91,7 @@
         "src/cpp/proto/proto_utils.cc",
         "src/cpp/server/async_generic_service.cc",
         "src/cpp/server/create_default_thread_pool.cc",
+        "src/cpp/server/dynamic_thread_pool.cc",
         "src/cpp/server/fixed_size_thread_pool.cc",
         "src/cpp/server/insecure_server_credentials.cc",
         "src/cpp/server/server.cc",
@@ -2044,6 +2046,21 @@
         "gpr"
       ]
     },
+    {
+      "name": "dynamic_thread_pool_test",
+      "build": "test",
+      "language": "c++",
+      "src": [
+        "test/cpp/server/dynamic_thread_pool_test.cc"
+      ],
+      "deps": [
+        "grpc_test_util",
+        "grpc++",
+        "grpc",
+        "gpr_test_util",
+        "gpr"
+      ]
+    },
     {
       "name": "end2end_test",
       "build": "test",

include/grpc++/dynamic_thread_pool.h

@@ -0,0 +1,81 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPCXX_DYNAMIC_THREAD_POOL_H
+#define GRPCXX_DYNAMIC_THREAD_POOL_H
+
+#include <grpc++/config.h>
+
+#include <grpc++/impl/sync.h>
+#include <grpc++/impl/thd.h>
+#include <grpc++/thread_pool_interface.h>
+
+#include <list>
+#include <queue>
+
+namespace grpc {
+
+class DynamicThreadPool GRPC_FINAL : public ThreadPoolInterface {
+ public:
+  explicit DynamicThreadPool(int reserve_threads);
+  ~DynamicThreadPool();
+
+  void Add(const std::function<void()>& callback) GRPC_OVERRIDE;
+
+ private:
+  class DynamicThread {
+  public:
+    DynamicThread(DynamicThreadPool *pool);
+    ~DynamicThread();
+  private:
+    DynamicThreadPool *pool_;
+    std::unique_ptr<grpc::thread> thd_;
+    void ThreadFunc();
+  };
+  grpc::mutex mu_;
+  grpc::condition_variable cv_;
+  grpc::condition_variable shutdown_cv_;
+  bool shutdown_;
+  std::queue<std::function<void()>> callbacks_;
+  int reserve_threads_;
+  int nthreads_;
+  int threads_waiting_;
+  std::list<DynamicThread*> dead_threads_;
+
+  void ThreadFunc();
+  static void ReapThreads(std::list<DynamicThread*>* tlist);
+};
+
+}  // namespace grpc
+
+#endif  // GRPCXX_DYNAMIC_THREAD_POOL_H

src/cpp/server/create_default_thread_pool.cc

@@ -32,7 +32,7 @@
  */
 
 #include <grpc/support/cpu.h>
-#include <grpc++/fixed_size_thread_pool.h>
+#include <grpc++/dynamic_thread_pool.h>
 
 #ifndef GRPC_CUSTOM_DEFAULT_THREAD_POOL
 
@@ -41,7 +41,7 @@ namespace grpc {
 ThreadPoolInterface* CreateDefaultThreadPool() {
    int cores = gpr_cpu_num_cores();
    if (!cores) cores = 4;
-   return new FixedSizeThreadPool(cores);
+   return new DynamicThreadPool(cores);
 }
 
 }  // namespace grpc

src/cpp/server/dynamic_thread_pool.cc

@@ -0,0 +1,131 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include <grpc++/impl/sync.h>
+#include <grpc++/impl/thd.h>
+#include <grpc++/dynamic_thread_pool.h>
+
+namespace grpc {
+DynamicThreadPool::DynamicThread::DynamicThread(DynamicThreadPool *pool):
+  pool_(pool),
+  thd_(new grpc::thread(&DynamicThreadPool::DynamicThread::ThreadFunc, this)) {
+}
+DynamicThreadPool::DynamicThread::~DynamicThread() {
+  thd_->join();
+  thd_.reset();
+}
+
+void DynamicThreadPool::DynamicThread::ThreadFunc() {
+  pool_->ThreadFunc();
+  // Now that we have killed ourselves, we should reduce the thread count
+  grpc::unique_lock<grpc::mutex> lock(pool_->mu_);
+  pool_->nthreads_--;
+  // Move ourselves to dead list
+  pool_->dead_threads_.push_back(this);
+
+  if ((pool_->shutdown_) && (pool_->nthreads_ == 0)) {
+    pool_->shutdown_cv_.notify_one();
+  }
+}
+  
+void DynamicThreadPool::ThreadFunc() {
+  for (;;) {
+    // Wait until work is available or we are shutting down.
+    grpc::unique_lock<grpc::mutex> lock(mu_);
+    if (!shutdown_ && callbacks_.empty()) {
+      // If there are too many threads waiting, then quit this thread
+      if (threads_waiting_ >= reserve_threads_) {
+	break;
+      }
+      threads_waiting_++;
+      cv_.wait(lock);
+      threads_waiting_--;
+    }
+    // Drain callbacks before considering shutdown to ensure all work
+    // gets completed.
+    if (!callbacks_.empty()) {
+      auto cb = callbacks_.front();
+      callbacks_.pop();
+      lock.unlock();
+      cb();
+    } else if (shutdown_) {
+      break;
+    }
+  }
+}
+
+DynamicThreadPool::DynamicThreadPool(int reserve_threads) :
+  shutdown_(false), reserve_threads_(reserve_threads), nthreads_(0),
+  threads_waiting_(0) {
+  for (int i = 0; i < reserve_threads_; i++) {
+    grpc::lock_guard<grpc::mutex> lock(mu_);
+    nthreads_++;
+    new DynamicThread(this);
+  }
+}
+
+void DynamicThreadPool::ReapThreads(std::list<DynamicThread*>* tlist) {
+  for (auto t = tlist->begin(); t != tlist->end(); t = tlist->erase(t)) {
+    delete *t;    
+  }
+}
+  
+DynamicThreadPool::~DynamicThreadPool() {
+  grpc::unique_lock<grpc::mutex> lock(mu_);
+  shutdown_ = true;
+  cv_.notify_all();
+  while (nthreads_ != 0) {
+    shutdown_cv_.wait(lock);
+  }
+  ReapThreads(&dead_threads_);
+}
+
+void DynamicThreadPool::Add(const std::function<void()>& callback) {
+  grpc::lock_guard<grpc::mutex> lock(mu_);
+  // Add works to the callbacks list
+  callbacks_.push(callback);
+  // Increase pool size or notify as needed
+  if (threads_waiting_ == 0) {
+    // Kick off a new thread
+    nthreads_++;
+    new DynamicThread(this);
+  } else {
+    cv_.notify_one();
+  }
+  // Also use this chance to harvest dead threads
+  if (!dead_threads_.empty()) {
+    ReapThreads(&dead_threads_);
+  }
+}
+
+}  // namespace grpc

test/cpp/end2end/end2end_test.cc

@@ -45,7 +45,7 @@
 #include <grpc++/client_context.h>
 #include <grpc++/create_channel.h>
 #include <grpc++/credentials.h>
-#include <grpc++/fixed_size_thread_pool.h>
+#include <grpc++/dynamic_thread_pool.h>
 #include <grpc++/server.h>
 #include <grpc++/server_builder.h>
 #include <grpc++/server_context.h>
@@ -275,7 +275,7 @@ class End2endTest : public ::testing::Test {
   TestServiceImpl service_;
   TestServiceImpl special_service_;
   TestServiceImplDupPkg dup_pkg_service_;
-  FixedSizeThreadPool thread_pool_;
+  DynamicThreadPool thread_pool_;
 };
 
 static void SendRpc(grpc::cpp::test::util::TestService::Stub* stub,

test/cpp/end2end/mock_test.cc

@@ -42,7 +42,7 @@
 #include <grpc++/client_context.h>
 #include <grpc++/create_channel.h>
 #include <grpc++/credentials.h>
-#include <grpc++/fixed_size_thread_pool.h>
+#include <grpc++/dynamic_thread_pool.h>
 #include <grpc++/server.h>
 #include <grpc++/server_builder.h>
 #include <grpc++/server_context.h>
@@ -260,7 +260,7 @@ class MockTest : public ::testing::Test {
   std::unique_ptr<Server> server_;
   std::ostringstream server_address_;
   TestServiceImpl service_;
-  FixedSizeThreadPool thread_pool_;
+  DynamicThreadPool thread_pool_;
 };
 
 // Do one real rpc and one mocked one

test/cpp/end2end/thread_stress_test.cc

@@ -43,7 +43,7 @@
 #include <grpc++/client_context.h>
 #include <grpc++/create_channel.h>
 #include <grpc++/credentials.h>
-#include <grpc++/fixed_size_thread_pool.h>
+#include <grpc++/dynamic_thread_pool.h>
 #include <grpc++/server.h>
 #include <grpc++/server_builder.h>
 #include <grpc++/server_context.h>
@@ -208,7 +208,7 @@ class End2endTest : public ::testing::Test {
   const int kMaxMessageSize_;
   TestServiceImpl service_;
   TestServiceImplDupPkg dup_pkg_service_;
-  FixedSizeThreadPool thread_pool_;
+  DynamicThreadPool thread_pool_;
 };
 
 static void SendRpc(grpc::cpp::test::util::TestService::Stub* stub,

test/cpp/qps/server_async.cc

@@ -45,7 +45,6 @@
 #include <grpc/support/host_port.h>
 #include <grpc++/async_unary_call.h>
 #include <grpc++/config.h>
-#include <grpc++/fixed_size_thread_pool.h>
 #include <grpc++/server.h>
 #include <grpc++/server_builder.h>
 #include <grpc++/server_context.h>

test/cpp/qps/server_sync.cc

@@ -40,6 +40,7 @@
 #include <grpc/support/alloc.h>
 #include <grpc/support/host_port.h>
 #include <grpc++/config.h>
+#include <grpc++/dynamic_thread_pool.h>
 #include <grpc++/fixed_size_thread_pool.h>
 #include <grpc++/server.h>
 #include <grpc++/server_builder.h>
@@ -92,7 +93,13 @@ class TestServiceImpl GRPC_FINAL : public TestService::Service {
 class SynchronousServer GRPC_FINAL : public grpc::testing::Server {
  public:
   SynchronousServer(const ServerConfig& config, int port)
-      : thread_pool_(config.threads()), impl_(MakeImpl(port)) {}
+      : thread_pool_(), impl_(MakeImpl(port)) {
+    if (config.threads() > 0) {
+      thread_pool_.reset(new FixedSizeThreadPool(config.threads()));
+    } else {
+      thread_pool_.reset(new DynamicThreadPool(-config.threads()));
+    }
+  }
 
  private:
   std::unique_ptr<grpc::Server> MakeImpl(int port) {
@@ -105,13 +112,13 @@ class SynchronousServer GRPC_FINAL : public grpc::testing::Server {
 
     builder.RegisterService(&service_);
 
-    builder.SetThreadPool(&thread_pool_);
+    builder.SetThreadPool(thread_pool_.get());
 
     return builder.BuildAndStart();
   }
 
   TestServiceImpl service_;
-  FixedSizeThreadPool thread_pool_;
+  std::unique_ptr<ThreadPoolInterface> thread_pool_;
   std::unique_ptr<grpc::Server> impl_;
 };
 

test/cpp/server/dynamic_thread_pool_test.cc

@@ -0,0 +1,77 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include <condition_variable>
+#include <functional>
+#include <mutex>
+
+#include <grpc++/dynamic_thread_pool.h>
+#include <gtest/gtest.h>
+
+namespace grpc {
+
+class DynamicThreadPoolTest : public ::testing::Test {
+ public:
+  DynamicThreadPoolTest() : thread_pool_(0) {}
+
+ protected:
+  DynamicThreadPool thread_pool_;
+};
+
+void Callback(std::mutex* mu, std::condition_variable* cv, bool* done) {
+  std::unique_lock<std::mutex> lock(*mu);
+  *done = true;
+  cv->notify_all();
+}
+
+TEST_F(DynamicThreadPoolTest, Add) {
+  std::mutex mu;
+  std::condition_variable cv;
+  bool done = false;
+  std::function<void()> callback = std::bind(Callback, &mu, &cv, &done);
+  thread_pool_.Add(callback);
+
+  // Wait for the callback to finish.
+  std::unique_lock<std::mutex> lock(mu);
+  while (!done) {
+    cv.wait(lock);
+  }
+}
+
+}  // namespace grpc
+
+int main(int argc, char** argv) {
+  ::testing::InitGoogleTest(&argc, argv);
+  int result = RUN_ALL_TESTS();
+  return result;
+}

test/cpp/util/cli_call_test.cc

@@ -39,7 +39,7 @@
 #include <grpc++/client_context.h>
 #include <grpc++/create_channel.h>
 #include <grpc++/credentials.h>
-#include <grpc++/fixed_size_thread_pool.h>
+#include <grpc++/dynamic_thread_pool.h>
 #include <grpc++/server.h>
 #include <grpc++/server_builder.h>
 #include <grpc++/server_context.h>
@@ -102,7 +102,7 @@ class CliCallTest : public ::testing::Test {
   std::unique_ptr<Server> server_;
   std::ostringstream server_address_;
   TestServiceImpl service_;
-  FixedSizeThreadPool thread_pool_;
+  DynamicThreadPool thread_pool_;
 };
 
 // Send a rpc with a normal stub and then a CliCall. Verify they match.

tools/doxygen/Doxyfile.c++

@@ -773,6 +773,7 @@ include/grpc++/config.h \
 include/grpc++/config_protobuf.h \
 include/grpc++/create_channel.h \
 include/grpc++/credentials.h \
+include/grpc++/dynamic_thread_pool.h \
 include/grpc++/fixed_size_thread_pool.h \
 include/grpc++/generic_stub.h \
 include/grpc++/impl/call.h \

tools/doxygen/Doxyfile.c++.internal

@@ -773,6 +773,7 @@ include/grpc++/config.h \
 include/grpc++/config_protobuf.h \
 include/grpc++/create_channel.h \
 include/grpc++/credentials.h \
+include/grpc++/dynamic_thread_pool.h \
 include/grpc++/fixed_size_thread_pool.h \
 include/grpc++/generic_stub.h \
 include/grpc++/impl/call.h \
@@ -825,6 +826,7 @@ src/cpp/common/rpc_method.cc \
 src/cpp/proto/proto_utils.cc \
 src/cpp/server/async_generic_service.cc \
 src/cpp/server/create_default_thread_pool.cc \
+src/cpp/server/dynamic_thread_pool.cc \
 src/cpp/server/fixed_size_thread_pool.cc \
 src/cpp/server/insecure_server_credentials.cc \
 src/cpp/server/server.cc \

tools/run_tests/sources_and_headers.json

@@ -1181,6 +1181,21 @@
       "test/cpp/util/time_test.cc"
     ]
   }, 
+  {
+    "deps": [
+      "gpr", 
+      "gpr_test_util", 
+      "grpc", 
+      "grpc++", 
+      "grpc_test_util"
+    ], 
+    "headers": [], 
+    "language": "c++", 
+    "name": "dynamic_thread_pool_test", 
+    "src": [
+      "test/cpp/server/dynamic_thread_pool_test.cc"
+    ]
+  }, 
   {
     "deps": [
       "gpr", 
@@ -10652,6 +10667,7 @@
       "include/grpc++/config_protobuf.h", 
       "include/grpc++/create_channel.h", 
       "include/grpc++/credentials.h", 
+      "include/grpc++/dynamic_thread_pool.h", 
       "include/grpc++/fixed_size_thread_pool.h", 
       "include/grpc++/generic_stub.h", 
       "include/grpc++/impl/call.h", 
@@ -10701,6 +10717,7 @@
       "include/grpc++/config_protobuf.h", 
       "include/grpc++/create_channel.h", 
       "include/grpc++/credentials.h", 
+      "include/grpc++/dynamic_thread_pool.h", 
       "include/grpc++/fixed_size_thread_pool.h", 
       "include/grpc++/generic_stub.h", 
       "include/grpc++/impl/call.h", 
@@ -10751,6 +10768,7 @@
       "src/cpp/proto/proto_utils.cc", 
       "src/cpp/server/async_generic_service.cc", 
       "src/cpp/server/create_default_thread_pool.cc", 
+      "src/cpp/server/dynamic_thread_pool.cc", 
       "src/cpp/server/fixed_size_thread_pool.cc", 
       "src/cpp/server/insecure_server_credentials.cc", 
       "src/cpp/server/secure_server_credentials.cc", 
@@ -10823,6 +10841,7 @@
       "include/grpc++/config_protobuf.h", 
       "include/grpc++/create_channel.h", 
       "include/grpc++/credentials.h", 
+      "include/grpc++/dynamic_thread_pool.h", 
       "include/grpc++/fixed_size_thread_pool.h", 
       "include/grpc++/generic_stub.h", 
       "include/grpc++/impl/call.h", 
@@ -10869,6 +10888,7 @@
       "include/grpc++/config_protobuf.h", 
       "include/grpc++/create_channel.h", 
       "include/grpc++/credentials.h", 
+      "include/grpc++/dynamic_thread_pool.h", 
       "include/grpc++/fixed_size_thread_pool.h", 
       "include/grpc++/generic_stub.h", 
       "include/grpc++/impl/call.h", 
@@ -10913,6 +10933,7 @@
       "src/cpp/proto/proto_utils.cc", 
       "src/cpp/server/async_generic_service.cc", 
       "src/cpp/server/create_default_thread_pool.cc", 
+      "src/cpp/server/dynamic_thread_pool.cc", 
       "src/cpp/server/fixed_size_thread_pool.cc", 
       "src/cpp/server/insecure_server_credentials.cc", 
       "src/cpp/server/server.cc", 

tools/run_tests/tests.json

@@ -675,6 +675,15 @@
       "posix"
     ]
   }, 
+  {
+    "flaky": false, 
+    "language": "c++", 
+    "name": "dynamic_thread_pool_test", 
+    "platforms": [
+      "windows", 
+      "posix"
+    ]
+  }, 
   {
     "flaky": false, 
     "language": "c++", 

vsprojects/grpc++/grpc++.vcxproj

@@ -159,6 +159,7 @@
     <ClInclude Include="..\..\include\grpc++\config_protobuf.h" />
     <ClInclude Include="..\..\include\grpc++\create_channel.h" />
     <ClInclude Include="..\..\include\grpc++\credentials.h" />
+    <ClInclude Include="..\..\include\grpc++\dynamic_thread_pool.h" />
     <ClInclude Include="..\..\include\grpc++\fixed_size_thread_pool.h" />
     <ClInclude Include="..\..\include\grpc++\generic_stub.h" />
     <ClInclude Include="..\..\include\grpc++\impl\call.h" />
@@ -235,6 +236,8 @@
     </ClCompile>
     <ClCompile Include="..\..\src\cpp\server\create_default_thread_pool.cc">
     </ClCompile>
+    <ClCompile Include="..\..\src\cpp\server\dynamic_thread_pool.cc">
+    </ClCompile>
     <ClCompile Include="..\..\src\cpp\server\fixed_size_thread_pool.cc">
     </ClCompile>
     <ClCompile Include="..\..\src\cpp\server\insecure_server_credentials.cc">

vsprojects/grpc++/grpc++.vcxproj.filters

@@ -61,6 +61,9 @@
     <ClCompile Include="..\..\src\cpp\server\create_default_thread_pool.cc">
       <Filter>src\cpp\server</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\src\cpp\server\dynamic_thread_pool.cc">
+      <Filter>src\cpp\server</Filter>
+    </ClCompile>
     <ClCompile Include="..\..\src\cpp\server\fixed_size_thread_pool.cc">
       <Filter>src\cpp\server</Filter>
     </ClCompile>
@@ -132,6 +135,9 @@
     <ClInclude Include="..\..\include\grpc++\credentials.h">
       <Filter>include\grpc++</Filter>
     </ClInclude>
+    <ClInclude Include="..\..\include\grpc++\dynamic_thread_pool.h">
+      <Filter>include\grpc++</Filter>
+    </ClInclude>
     <ClInclude Include="..\..\include\grpc++\fixed_size_thread_pool.h">
       <Filter>include\grpc++</Filter>
     </ClInclude>

vsprojects/grpc++_unsecure/grpc++_unsecure.vcxproj

@@ -159,6 +159,7 @@
     <ClInclude Include="..\..\include\grpc++\config_protobuf.h" />
     <ClInclude Include="..\..\include\grpc++\create_channel.h" />
     <ClInclude Include="..\..\include\grpc++\credentials.h" />
+    <ClInclude Include="..\..\include\grpc++\dynamic_thread_pool.h" />
     <ClInclude Include="..\..\include\grpc++\fixed_size_thread_pool.h" />
     <ClInclude Include="..\..\include\grpc++\generic_stub.h" />
     <ClInclude Include="..\..\include\grpc++\impl\call.h" />
@@ -222,6 +223,8 @@
     </ClCompile>
     <ClCompile Include="..\..\src\cpp\server\create_default_thread_pool.cc">
     </ClCompile>
+    <ClCompile Include="..\..\src\cpp\server\dynamic_thread_pool.cc">
+    </ClCompile>
     <ClCompile Include="..\..\src\cpp\server\fixed_size_thread_pool.cc">
     </ClCompile>
     <ClCompile Include="..\..\src\cpp\server\insecure_server_credentials.cc">

vsprojects/grpc++_unsecure/grpc++_unsecure.vcxproj.filters

@@ -46,6 +46,9 @@
     <ClCompile Include="..\..\src\cpp\server\create_default_thread_pool.cc">
       <Filter>src\cpp\server</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\src\cpp\server\dynamic_thread_pool.cc">
+      <Filter>src\cpp\server</Filter>
+    </ClCompile>
     <ClCompile Include="..\..\src\cpp\server\fixed_size_thread_pool.cc">
       <Filter>src\cpp\server</Filter>
     </ClCompile>
@@ -117,6 +120,9 @@
     <ClInclude Include="..\..\include\grpc++\credentials.h">
       <Filter>include\grpc++</Filter>
     </ClInclude>
+    <ClInclude Include="..\..\include\grpc++\dynamic_thread_pool.h">
+      <Filter>include\grpc++</Filter>
+    </ClInclude>
     <ClInclude Include="..\..\include\grpc++\fixed_size_thread_pool.h">
       <Filter>include\grpc++</Filter>
     </ClInclude>