Go Stress test: Docker stress test images and config files to run on GKE

templates/tools/dockerfile/go_path.include

@@ -0,0 +1,2 @@
+# Using login shell removes Go from path, so we add it.
+RUN ln -s /usr/src/go/bin/go /usr/local/bin

templates/tools/dockerfile/stress_test/grpc_interop_stress_go/Dockerfile.template

@@ -0,0 +1,37 @@
+%YAML 1.2
+--- |
+  # Copyright 2016, Google Inc.
+  # All rights reserved.
+  #
+  # Redistribution and use in source and binary forms, with or without
+  # modification, are permitted provided that the following conditions are
+  # met:
+  #
+  #     * Redistributions of source code must retain the above copyright
+  # notice, this list of conditions and the following disclaimer.
+  #     * Redistributions in binary form must reproduce the above
+  # copyright notice, this list of conditions and the following disclaimer
+  # in the documentation and/or other materials provided with the
+  # distribution.
+  #     * Neither the name of Google Inc. nor the names of its
+  # contributors may be used to endorse or promote products derived from
+  # this software without specific prior written permission.
+  #
+  # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+  # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+  # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+  # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+  # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+  # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+  # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+  # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+  # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+  # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+  # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+  
+  FROM golang:1.4
+  
+  <%include file="../../gcp_api_libraries.include"/>
+  <%include file="../../go_path.include"/>
+  # Define the default command.
+  CMD ["bash"]

tools/dockerfile/stress_test/grpc_interop_stress_go/Dockerfile

@@ -0,0 +1,41 @@
+# Copyright 2016, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+FROM golang:1.4
+
+# Google Cloud platform API libraries
+RUN apt-get update && apt-get install -y python-pip && apt-get clean
+RUN pip install --upgrade google-api-python-client
+
+
+# Using login shell removes Go from path, so we add it.
+RUN ln -s /usr/src/go/bin/go /usr/local/bin
+
+# Define the default command.
+CMD ["bash"]

tools/dockerfile/stress_test/grpc_interop_stress_go/build_interop_stress.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+# Copyright 2015, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# Builds Go interop server, Stress client and metrics client in a base image.
+set -e
+
+# Clone just the grpc-go source code without any dependencies.
+# We are cloning from a local git repo that contains the right revision
+# to test instead of using "go get" to download from Github directly.
+git clone --recursive /var/local/jenkins/grpc-go src/google.golang.org/grpc
+
+# Clone the 'grpc' repo. We just need this for the wrapper scripts under
+# grpc/tools/gcp/stress_tests
+git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc
+
+# copy service account keys if available
+cp -r /var/local/jenkins/service_account $HOME || true
+
+# Get dependencies from GitHub
+# NOTE: once grpc-go dependencies change, this needs to be updated manually
+# but we don't expect this to happen any time soon.
+go get github.com/golang/protobuf/proto
+go get golang.org/x/net/context
+go get golang.org/x/net/trace
+go get golang.org/x/oauth2
+go get google.golang.org/cloud
+
+# Build the interop server, stress client and stress metrics client
+(cd src/google.golang.org/grpc/interop/server && go install)
+(cd src/google.golang.org/grpc/stress/client && go install)
+(cd src/google.golang.org/grpc/stress/metrics_client && go install)

tools/jenkins/build_interop_stress_image.sh

@@ -48,6 +48,22 @@ cd `dirname $0`/../..
 GRPC_ROOT=`pwd`
 MOUNT_ARGS="-v $GRPC_ROOT:/var/local/jenkins/grpc:ro"
 
+GRPC_JAVA_ROOT=`cd ../grpc-java && pwd`
+if [ "$GRPC_JAVA_ROOT" != "" ]
+then
+  MOUNT_ARGS+=" -v $GRPC_JAVA_ROOT:/var/local/jenkins/grpc-java:ro"
+else
+  echo "WARNING: grpc-java not found, it won't be mounted to the docker container."
+fi
+
+GRPC_GO_ROOT=`cd ../grpc-go && pwd`
+if [ "$GRPC_GO_ROOT" != "" ]
+then
+  MOUNT_ARGS+=" -v $GRPC_GO_ROOT:/var/local/jenkins/grpc-go:ro"
+else
+  echo "WARNING: grpc-go not found, it won't be mounted to the docker container."
+fi
+
 mkdir -p /tmp/ccache
 
 # Mount service account dir if available.

tools/run_tests/stress_test/configs/go.json

@@ -0,0 +1,96 @@
+{
+  "dockerImages": {
+    "grpc_stress_go" : {
+      "buildScript": "tools/jenkins/build_interop_stress_image.sh",
+      "dockerFileDir": "grpc_interop_stress_go"
+    }
+  },
+
+  "clientTemplates": {
+    "baseTemplates": {
+      "default": {
+        "wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_client.py",
+        "pollIntervalSecs": 60,
+        "clientArgs": {
+          "num_channels_per_server":5,
+          "num_stubs_per_channel":10,
+          "test_cases": "empty_unary:1,large_unary:1,client_streaming:1,server_streaming:1,empty_stream:1",
+          "metrics_port": 8081
+        },
+        "metricsPort": 8081,
+        "metricsArgs": {
+          "metrics_server_address": "localhost:8081",
+          "total_only": "true"
+        }
+      }
+    },
+    "templates": {
+      "go_client": {
+        "baseTemplate": "default",
+        "stressClientCmd": [
+          "go",
+          "run",
+          "/go/src/google.golang.org/grpc/stress/client/main.go"
+        ],
+        "metricsClientCmd": [
+          "go",
+          "run",
+          "/go/src/google.golang.org/grpc/stress/metrics_client/main.go"
+        ]
+      }
+    }
+  },
+
+  "serverTemplates": {
+    "baseTemplates":{
+      "default": {
+        "wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_server.py",
+        "serverPort": 8080,
+        "serverArgs": {
+          "port": 8080
+        }
+      }
+    },
+    "templates": {
+      "go_server": {
+        "baseTemplate": "default",
+        "stressServerCmd": [
+          "go",
+          "run",
+          "/go/src/google.golang.org/grpc/interop/server/server.go"
+        ]
+      }
+    }
+  },
+
+  "testMatrix": {
+    "serverPodSpecs": {
+      "go-stress-server": {
+        "serverTemplate": "go_server",
+        "dockerImage": "grpc_stress_go",
+        "numInstances": 1
+      }
+    },
+
+    "clientPodSpecs": {
+      "go-stress-client": {
+        "clientTemplate": "go_client",
+        "dockerImage": "grpc_stress_go",
+        "numInstances": 15,
+        "serverPodSpec": "go-stress-server"
+      }
+    }
+  },
+
+  "globalSettings": {
+    "buildDockerImages": true,
+    "pollIntervalSecs": 60,
+    "testDurationSecs": 7200,
+    "kubernetesProxyPort": 8007,
+    "datasetIdNamePrefix": "stress_test_go",
+    "summaryTableId": "summary",
+    "qpsTableId": "qps",
+    "podWarmupSecs": 60
+  }
+}
+