Deprecation of qps_driver and use of shell scripts, in progress

Makefile

@@ -1023,7 +1023,6 @@ interop_test: $(BINDIR)/$(CONFIG)/interop_test
 json_run_localhost: $(BINDIR)/$(CONFIG)/json_run_localhost
 metrics_client: $(BINDIR)/$(CONFIG)/metrics_client
 mock_test: $(BINDIR)/$(CONFIG)/mock_test
-qps_driver: $(BINDIR)/$(CONFIG)/qps_driver
 qps_interarrival_test: $(BINDIR)/$(CONFIG)/qps_interarrival_test
 qps_json_driver: $(BINDIR)/$(CONFIG)/qps_json_driver
 qps_openloop_test: $(BINDIR)/$(CONFIG)/qps_openloop_test
@@ -1764,7 +1763,7 @@ tools_c: privatelibs_c $(BINDIR)/$(CONFIG)/gen_hpack_tables $(BINDIR)/$(CONFIG)/
 
 tools_cxx: privatelibs_cxx
 
-buildbenchmarks: privatelibs $(BINDIR)/$(CONFIG)/low_level_ping_pong_benchmark $(BINDIR)/$(CONFIG)/qps_driver
+buildbenchmarks: privatelibs $(BINDIR)/$(CONFIG)/low_level_ping_pong_benchmark
 
 benchmarks: buildbenchmarks
 
@@ -10906,49 +10905,6 @@ endif
 endif
 
 
-QPS_DRIVER_SRC = \
-    test/cpp/qps/qps_driver.cc \
-
-QPS_DRIVER_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(QPS_DRIVER_SRC))))
-ifeq ($(NO_SECURE),true)
-
-# You can't build secure targets if you don't have OpenSSL.
-
-$(BINDIR)/$(CONFIG)/qps_driver: openssl_dep_error
-
-else
-
-
-
-
-ifeq ($(NO_PROTOBUF),true)
-
-# You can't build the protoc plugins or protobuf-enabled targets if you don't have protobuf 3.0.0+.
-
-$(BINDIR)/$(CONFIG)/qps_driver: protobuf_dep_error
-
-else
-
-$(BINDIR)/$(CONFIG)/qps_driver: $(PROTOBUF_DEP) $(QPS_DRIVER_OBJS) $(LIBDIR)/$(CONFIG)/libqps.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a
-	$(E) "[LD]      Linking $@"
-	$(Q) mkdir -p `dirname $@`
-	$(Q) $(LDXX) $(LDFLAGS) $(QPS_DRIVER_OBJS) $(LIBDIR)/$(CONFIG)/libqps.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/qps_driver
-
-endif
-
-endif
-
-$(OBJDIR)/$(CONFIG)/test/cpp/qps/qps_driver.o:  $(LIBDIR)/$(CONFIG)/libqps.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a
-
-deps_qps_driver: $(QPS_DRIVER_OBJS:.o=.dep)
-
-ifneq ($(NO_SECURE),true)
-ifneq ($(NO_DEPS),true)
--include $(QPS_DRIVER_OBJS:.o=.dep)
-endif
-endif
-
-
 QPS_INTERARRIVAL_TEST_SRC = \
     test/cpp/qps/qps_interarrival_test.cc \
 

build.yaml

@@ -2706,20 +2706,6 @@ targets:
   - grpc
   - gpr_test_util
   - gpr
-- name: qps_driver
-  build: benchmark
-  language: c++
-  src:
-  - test/cpp/qps/qps_driver.cc
-  deps:
-  - qps
-  - grpc++_test_util
-  - grpc_test_util
-  - grpc++
-  - grpc
-  - gpr_test_util
-  - gpr
-  - grpc++_test_config
 - name: qps_interarrival_test
   build: test
   run: false

test/cpp/qps/qps-sweep.sh

@@ -1,170 +0,0 @@
-#!/bin/sh
-
-# Copyright 2015, Google Inc.
-# All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are
-# met:
-#
-#     * Redistributions of source code must retain the above copyright
-# notice, this list of conditions and the following disclaimer.
-#     * Redistributions in binary form must reproduce the above
-# copyright notice, this list of conditions and the following disclaimer
-# in the documentation and/or other materials provided with the
-# distribution.
-#     * Neither the name of Google Inc. nor the names of its
-# contributors may be used to endorse or promote products derived from
-# this software without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-if [ x"$QPS_WORKERS" == x ]; then
-  echo Error: Must set QPS_WORKERS variable in form \
-    "host:port,host:port,..." 1>&2
-  exit 1
-fi
-
-bins=`find . .. ../.. ../../.. -name bins | head -1`
-
-# Print out each command that gets executed
-set -x
-
-#
-# Specify parameters used in some of the tests
-#
-
-# big is the size in bytes of large messages (0 is the size otherwise)
-big=65536
-
-# wide is the number of client channels in multi-channel tests (1 otherwise)
-wide=64
-
-# deep is the number of RPCs outstanding on a channel in non-ping-pong tests
-# (the value used is 1 otherwise)
-deep=100
-
-# half is half the count of worker processes, used in the crossbar scenario
-# that uses equal clients and servers. The other scenarios use only 1 server
-# and either 1 client or N-1 clients as appropriate
-half=`echo $QPS_WORKERS | awk -F, '{print int(NF/2)}'`
-
-for secure in true false; do
-  # Scenario 1: generic async streaming ping-pong (contentionless latency)
-  "$bins"/opt/qps_driver --rpc_type=STREAMING --client_type=ASYNC_CLIENT \
-    --server_type=ASYNC_GENERIC_SERVER --outstanding_rpcs_per_channel=1 \
-    --client_channels=1 --bbuf_req_size=0 --bbuf_resp_size=0 \
-    --async_client_threads=1 --async_server_threads=1 --secure_test=$secure \
-    --num_servers=1 --num_clients=1
-
-  # Scenario 2: generic async streaming "unconstrained" (QPS)
-  "$bins"/opt/qps_driver --rpc_type=STREAMING --client_type=ASYNC_CLIENT \
-    --server_type=ASYNC_GENERIC_SERVER --outstanding_rpcs_per_channel=$deep \
-    --client_channels=$wide --bbuf_req_size=0 --bbuf_resp_size=0 \
-    --async_client_threads=0 --async_server_threads=0 --secure_test=$secure \
-    --num_servers=1 --num_clients=0 2>&1 | tee /tmp/qps-test.$$
-
-  # Scenario 2b: QPS with a single server core
-  "$bins"/opt/qps_driver --rpc_type=STREAMING --client_type=ASYNC_CLIENT \
-    --server_type=ASYNC_GENERIC_SERVER --outstanding_rpcs_per_channel=$deep \
-    --client_channels=$wide --bbuf_req_size=0 --bbuf_resp_size=0 \
-    --async_client_threads=0 --async_server_threads=0 --secure_test=$secure \
-    --num_servers=1 --num_clients=0 --server_core_limit=1
-
-  # Scenario 2c: protobuf-based QPS
-  "$bins"/opt/qps_driver --rpc_type=STREAMING --client_type=ASYNC_CLIENT \
-    --server_type=ASYNC_SERVER --outstanding_rpcs_per_channel=$deep \
-    --client_channels=$wide --simple_req_size=0 --simple_resp_size=0 \
-    --async_client_threads=0 --async_server_threads=0 --secure_test=$secure \
-    --num_servers=1 --num_clients=0
-
-  # Scenario 3: Latency at sub-peak load (all clients equally loaded)
-  for loadfactor in 0.2 0.5 0.7; do
-    "$bins"/opt/qps_driver --rpc_type=STREAMING --client_type=ASYNC_CLIENT \
-      --server_type=ASYNC_GENERIC_SERVER --outstanding_rpcs_per_channel=$deep \
-      --client_channels=$wide --bbuf_req_size=0 --bbuf_resp_size=0 \
-      --async_client_threads=0 --async_server_threads=0 --secure_test=$secure \
-      --num_servers=1 --num_clients=0 --poisson_load=`awk -v lf=$loadfactor \
-      '$5 == "QPS:" {print int(lf * $6); exit}' /tmp/qps-test.$$`
-  done
-
-  rm /tmp/qps-test.$$
-
-  # Scenario 4: Single-channel bidirectional throughput test (like TCP_STREAM).
-  "$bins"/opt/qps_driver --rpc_type=STREAMING --client_type=ASYNC_CLIENT \
-    --server_type=ASYNC_GENERIC_SERVER --outstanding_rpcs_per_channel=$deep \
-    --client_channels=1 --bbuf_req_size=$big --bbuf_resp_size=$big \
-    --async_client_threads=1 --async_server_threads=1 --secure_test=$secure \
-    --num_servers=1 --num_clients=1
-
-  # Scenario 5: Sync unary ping-pong with protobufs
-  "$bins"/opt/qps_driver --rpc_type=UNARY --client_type=SYNC_CLIENT \
-    --server_type=SYNC_SERVER --outstanding_rpcs_per_channel=1 \
-    --client_channels=1 --simple_req_size=0 --simple_resp_size=0 \
-    --secure_test=$secure --num_servers=1 --num_clients=1
-
-  # Scenario 6: Sync streaming ping-pong with protobufs
-  "$bins"/opt/qps_driver --rpc_type=STREAMING --client_type=SYNC_CLIENT \
-    --server_type=SYNC_SERVER --outstanding_rpcs_per_channel=1 \
-    --client_channels=1 --simple_req_size=0 --simple_resp_size=0 \
-    --secure_test=$secure --num_servers=1 --num_clients=1
-
-  # Scenario 7: Async unary ping-pong with protobufs
-  "$bins"/opt/qps_driver --rpc_type=UNARY --client_type=ASYNC_CLIENT \
-    --server_type=ASYNC_SERVER --outstanding_rpcs_per_channel=1 \
-    --client_channels=1 --simple_req_size=0 --simple_resp_size=0 \
-    --async_client_threads=1 --async_server_threads=1 --secure_test=$secure \
-    --num_servers=1 --num_clients=1
-
-  # Scenario 8: Async streaming ping-pong with protobufs
-  "$bins"/opt/qps_driver --rpc_type=STREAMING --client_type=ASYNC_CLIENT \
-    --server_type=ASYNC_SERVER --outstanding_rpcs_per_channel=1 \
-    --client_channels=1 --simple_req_size=0 --simple_resp_size=0 \
-    --async_client_threads=1 --async_server_threads=1 --secure_test=$secure \
-    --num_servers=1 --num_clients=1
-
-  # Scenario 9: Crossbar QPS test
-  "$bins"/opt/qps_driver --rpc_type=STREAMING --client_type=ASYNC_CLIENT \
-    --server_type=ASYNC_GENERIC_SERVER --outstanding_rpcs_per_channel=$deep \
-    --client_channels=$wide --bbuf_req_size=0 --bbuf_resp_size=0 \
-    --async_client_threads=0 --async_server_threads=0 --secure_test=$secure \
-    --num_servers=$half --num_clients=0
-
-  # Scenario 10: Multi-channel bidir throughput test
-  "$bins"/opt/qps_driver --rpc_type=STREAMING --client_type=ASYNC_CLIENT \
-    --server_type=ASYNC_GENERIC_SERVER --outstanding_rpcs_per_channel=1 \
-    --client_channels=$wide --bbuf_req_size=$big --bbuf_resp_size=$big \
-    --async_client_threads=0 --async_server_threads=0 --secure_test=$secure \
-    --num_servers=1 --num_clients=1
-
-  # Scenario 11: Single-channel request throughput test
-  "$bins"/opt/qps_driver --rpc_type=STREAMING --client_type=ASYNC_CLIENT \
-    --server_type=ASYNC_GENERIC_SERVER --outstanding_rpcs_per_channel=$deep \
-    --client_channels=1 --bbuf_req_size=$big --bbuf_resp_size=0 \
-    --async_client_threads=1 --async_server_threads=1 --secure_test=$secure \
-    --num_servers=1 --num_clients=1
-
-  # Scenario 12: Single-channel response throughput test
-  "$bins"/opt/qps_driver --rpc_type=STREAMING --client_type=ASYNC_CLIENT \
-    --server_type=ASYNC_GENERIC_SERVER --outstanding_rpcs_per_channel=$deep \
-    --client_channels=1 --bbuf_req_size=0 --bbuf_resp_size=$big \
-    --async_client_threads=1 --async_server_threads=1 --secure_test=$secure \
-    --num_servers=1 --num_clients=1
-
-  # Scenario 13: Single-channel bidirectional protobuf throughput test
-  "$bins"/opt/qps_driver --rpc_type=STREAMING --client_type=ASYNC_CLIENT \
-    --server_type=ASYNC_SERVER --outstanding_rpcs_per_channel=$deep \
-    --client_channels=1 --simple_req_size=$big --simple_resp_size=$big \
-    --async_client_threads=1 --async_server_threads=1 --secure_test=$secure \
-    --num_servers=1 --num_clients=1
-done

test/cpp/qps/qps_json_driver.cc

@@ -48,6 +48,7 @@ DEFINE_string(scenarios_file, "",
               "JSON file containing an array of Scenario objects");
 DEFINE_string(scenarios_json, "",
               "JSON string containing an array of Scenario objects");
+DEFINE_bool(quit, false, "Quit the workers");
 
 namespace grpc {
 namespace testing {
@@ -55,12 +56,17 @@ namespace testing {
 static void QpsDriver() {
   grpc::string json;
 
-  if (FLAGS_scenarios_file != "") {
-    if (FLAGS_scenarios_json != "") {
-      gpr_log(GPR_ERROR,
-              "Only one of --scenarios_file or --scenarios_json must be set");
-      abort();
-    }
+  bool scfile = (FLAGS_scenarios_file != "");
+  bool scjson = (FLAGS_scenarios_json != "");
+  if ((!scfile && !scjson && !FLAGS_quit) ||
+      (scfile && (scjson || FLAGS_quit)) ||
+      (scjson && FLAGS_quit)) {
+    gpr_log(GPR_ERROR, "Exactly one of --scenarios_file, --scenarios_json, "
+	    "or --quit must be set");
+    abort();
+  }
+
+  if (scfile) {
     // Read the json data from disk
     FILE *json_file = fopen(FLAGS_scenarios_file.c_str(), "r");
     GPR_ASSERT(json_file != NULL);
@@ -72,12 +78,11 @@ static void QpsDriver() {
     fclose(json_file);
     json = grpc::string(data, data + len);
     delete[] data;
-  } else if (FLAGS_scenarios_json != "") {
+  } else if (scjson) {
     json = FLAGS_scenarios_json.c_str();
-  } else {
-    gpr_log(GPR_ERROR,
-            "One of --scenarios_file or --scenarios_json must be set");
-    abort();
+  } else if (FLAGS_quit) {
+    RunQuit();
+    return;
   }
 
   // Parse into an array of scenarios

test/cpp/qps/single_run_localhost.sh

@@ -1,56 +0,0 @@
-#!/bin/sh
-# Copyright 2015, Google Inc.
-# All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are
-# met:
-#
-#     * Redistributions of source code must retain the above copyright
-# notice, this list of conditions and the following disclaimer.
-#     * Redistributions in binary form must reproduce the above
-# copyright notice, this list of conditions and the following disclaimer
-# in the documentation and/or other materials provided with the
-# distribution.
-#     * Neither the name of Google Inc. nor the names of its
-# contributors may be used to endorse or promote products derived from
-# this software without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-# performs a single qps run with one client and one server
-
-set -ex
-
-cd $(dirname $0)/../../..
-
-killall qps_worker || true
-
-config=opt
-
-NUMCPUS=`python2.7 -c 'import multiprocessing; print multiprocessing.cpu_count()'`
-
-make CONFIG=$config qps_worker qps_driver -j$NUMCPUS
-
-bins/$config/qps_worker -driver_port 10000 &
-PID1=$!
-bins/$config/qps_worker -driver_port 10010 &
-PID2=$!
-
-export QPS_WORKERS="localhost:10000,localhost:10010"
-
-bins/$config/qps_driver $*
-
-kill -2 $PID1 $PID2
-wait
-

tools/jenkins/run_performance.sh

@@ -40,7 +40,7 @@ cd $(dirname $0)/../..
 
 config=opt
 
-make CONFIG=$config qps_worker qps_driver -j8
+make CONFIG=$config qps_worker qps_json_driver -j8
 
 bins/$config/qps_worker -driver_port 10000 &
 PID1=$!
@@ -66,7 +66,7 @@ deep=100
 
 #
 # Get total core count
-cores=`grep -c ^processor /proc/cpuinfo`
+cores=`grep -c ^processor /proc/cpuinfo || sysctl -n hw.ncpu`
 halfcores=`expr $cores / 2`
 
 for secure in true false; do
@@ -84,7 +84,8 @@ for secure in true false; do
     --client_channels=$wide --bbuf_req_size=0 --bbuf_resp_size=0 \
     --async_client_threads=0 --async_server_threads=0 --secure_test=$secure \
     --num_servers=1 --num_clients=0 \
-    --server_core_limit=$halfcores --client_core_limit=0 |& tee /tmp/qps-test.$$
+    --server_core_limit=$halfcores --client_core_limit=0 2>&1 | \
+      tee /tmp/qps-test.$$
 
   # Scenario 2b: QPS with a single server core
   bins/$config/qps_driver --rpc_type=STREAMING --client_type=ASYNC_CLIENT \
@@ -131,6 +132,6 @@ for secure in true false; do
 
 done
 
-bins/$config/qps_driver --quit=true
+bins/$config/qps_json_driver --quit=true
 
 wait

tools/run_tests/performance/build_performance.sh

@@ -42,7 +42,7 @@ CONFIG=${CONFIG:-opt}
 # TODO(jtattermusch): not embedding OpenSSL breaks the C# build because
 # grpc_csharp_ext needs OpenSSL embedded and some intermediate files from
 # this build will be reused.
-make CONFIG=${CONFIG} EMBED_OPENSSL=true EMBED_ZLIB=true qps_worker qps_driver qps_json_driver -j8
+make CONFIG=${CONFIG} EMBED_OPENSSL=true EMBED_ZLIB=true qps_worker qps_json_driver -j8
 
 for language in $@
 do

tools/run_tests/run_performance_tests.py

@@ -118,14 +118,14 @@ def create_scenario_jobspec(scenario_json, workers, remote_host=None,
 def create_quit_jobspec(workers, remote_host=None):
   """Runs quit using QPS driver."""
   # setting QPS_WORKERS env variable here makes sure it works with SSH too.
-  cmd = 'QPS_WORKERS="%s" bins/opt/qps_driver --quit' % ','.join(workers)
+  cmd = 'QPS_WORKERS="%s" bins/opt/qps_json_driver --quit' % ','.join(workers)
   if remote_host:
     user_at_host = '%s@%s' % (_REMOTE_HOST_USERNAME, remote_host)
     cmd = 'ssh %s "cd ~/performance_workspace/grpc/ && "%s' % (user_at_host, pipes.quote(cmd))
 
   return jobset.JobSpec(
       cmdline=[cmd],
-      shortname='qps_driver.quit',
+      shortname='qps_json_driver.quit',
       timeout_seconds=3*60,
       shell=True,
       verbose_success=True)

tools/run_tests/sources_and_headers.json

@@ -2342,26 +2342,6 @@
     "third_party": false, 
     "type": "target"
   }, 
-  {
-    "deps": [
-      "gpr", 
-      "gpr_test_util", 
-      "grpc", 
-      "grpc++", 
-      "grpc++_test_config", 
-      "grpc++_test_util", 
-      "grpc_test_util", 
-      "qps"
-    ], 
-    "headers": [], 
-    "language": "c++", 
-    "name": "qps_driver", 
-    "src": [
-      "test/cpp/qps/qps_driver.cc"
-    ], 
-    "third_party": false, 
-    "type": "target"
-  }, 
   {
     "deps": [
       "gpr",