Merge pull request #10463 from ericgribkoff/cherry_pick_http2_fix

Backport http2 interop test fixes to v1.2.x

templates/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile.template

@@ -33,7 +33,7 @@
   
   <%include file="../../go_path.include"/>
   <%include file="../../python_deps.include"/>
-  RUN pip install twisted h2
+  RUN pip install twisted h2==2.6.1 hyper
 
   # Define the default command.
   CMD ["bash"]

test/http2_test/http2_server_health_check.py

@@ -0,0 +1,49 @@
+# Copyright 2017, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import argparse
+import hyper
+import sys
+
+# Utility to healthcheck the http2 server. Used when starting the server to
+# verify that the server is live before tests begin.
+if __name__ == '__main__':
+  parser = argparse.ArgumentParser()
+  parser.add_argument('--server_host', type=str, default='localhost')
+  parser.add_argument('--server_port', type=int, default=8080)
+  args = parser.parse_args()
+  server_host = args.server_host
+  server_port = args.server_port
+  conn = hyper.HTTP20Connection('%s:%d' % (server_host, server_port))
+  conn.request('POST', '/grpc.testing.TestService/UnaryCall')
+  resp = conn.get_response()
+  if resp.headers.get('grpc-encoding') is None:
+    sys.exit(1)
+  else:
+    sys.exit(0)

test/http2_test/http2_test_server.py

@@ -31,6 +31,7 @@
 
 import argparse
 import logging
+import sys
 import twisted
 import twisted.internet
 import twisted.internet.endpoints
@@ -53,9 +54,11 @@ _TEST_CASE_MAPPING = {
   'max_streams': test_max_streams.TestcaseSettingsMaxStreams,
 }
 
+_exit_code = 0
+
 class H2Factory(twisted.internet.protocol.Factory):
   def __init__(self, testcase):
-    logging.info('Creating H2Factory for new connection.')
+    logging.info('Creating H2Factory for new connection (%s)', testcase)
     self._num_streams = 0
     self._testcase = testcase
 
@@ -83,6 +86,17 @@ def parse_arguments():
     )
   return parser.parse_args()
 
+def listen(endpoint, test_case):
+  deferred = endpoint.listen(H2Factory(test_case))
+  def listen_error(reason):
+    # If listening fails, we stop the reactor and exit the program
+    # with exit code 1.
+    global _exit_code
+    _exit_code = 1
+    logging.error('Listening failed: %s' % reason.value)
+    twisted.internet.reactor.stop()
+  deferred.addErrback(listen_error)
+
 def start_test_servers(base_port):
   """ Start one server per test case on incrementing port numbers
   beginning with base_port """
@@ -92,7 +106,9 @@ def start_test_servers(base_port):
     logging.warning('serving on port %d : %s'%(portnum, test_case))
     endpoint = twisted.internet.endpoints.TCP4ServerEndpoint(
       twisted.internet.reactor, portnum, backlog=128)
-    endpoint.listen(H2Factory(test_case))
+    # Wait until the reactor is running before calling endpoint.listen().
+    twisted.internet.reactor.callWhenRunning(listen, endpoint, test_case)
+
     index += 1
 
 if __name__ == '__main__':
@@ -102,3 +118,4 @@ if __name__ == '__main__':
   args = parse_arguments()
   start_test_servers(args.base_port)
   twisted.internet.reactor.run()
+  sys.exit(_exit_code)

tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile

@@ -47,7 +47,7 @@ RUN pip install pip --upgrade
 RUN pip install virtualenv
 RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0
 
-RUN pip install twisted h2
+RUN pip install twisted h2==2.6.1 hyper
 
 # Define the default command.
 CMD ["bash"]

tools/run_tests/python_utils/dockerjob.py

@@ -70,6 +70,23 @@ def docker_mapped_port(cid, port, timeout_seconds=15):
                   (port, cid))
 
 
+def wait_for_healthy(cid, shortname, timeout_seconds):
+  """Wait timeout_seconds for the container to become healthy"""
+  started = time.time()
+  while time.time() - started < timeout_seconds:
+    try:
+      output = subprocess.check_output(
+          ['docker', 'inspect', '--format="{{.State.Health.Status}}"', cid],
+          stderr=_DEVNULL)
+      if output.strip('\n') == 'healthy':
+        return
+    except subprocess.CalledProcessError as e:
+      pass
+    time.sleep(1)
+  raise Exception('Timed out waiting for %s (%s) to pass health check' %
+                  (shortname, cid))
+
+
 def finish_jobs(jobs):
   """Kills given docker containers and waits for corresponding jobs to finish"""
   for job in jobs:
@@ -113,6 +130,9 @@ class DockerJob:
   def mapped_port(self, port):
     return docker_mapped_port(self._container_name, port)
 
+  def wait_for_healthy(self, timeout_seconds):
+    wait_for_healthy(self._container_name, self._spec.shortname, timeout_seconds)
+
   def kill(self, suppress_failure=False):
     """Sends kill signal to the container."""
     if suppress_failure:

tools/run_tests/run_interop_tests.py

@@ -607,17 +607,13 @@ def cloud_to_cloud_jobspec(language, test_case, server_name, server_host,
   common_options = [
       '--test_case=%s' % test_case,
       '--server_host=%s' % server_host,
+      '--server_port=%s' % server_port,
   ]
   if test_case in _HTTP2_BADSERVER_TEST_CASES:
-    # We are running the http2_badserver_interop test. Adjust command line accordingly.
-    offset = sorted(_HTTP2_BADSERVER_TEST_CASES).index(test_case)
-    client_options = common_options + ['--server_port=%s' %
-                                       (int(server_port)+offset)]
-    cmdline = bash_cmdline(language.client_cmd_http2interop(client_options))
+    cmdline = bash_cmdline(language.client_cmd_http2interop(common_options))
     cwd = language.http2_cwd
   else:
-    client_options = interop_only_options + common_options + ['--server_port=%s' % server_port]
-    cmdline = bash_cmdline(language.client_cmd(client_options))
+    cmdline = bash_cmdline(language.client_cmd(common_options+interop_only_options))
     cwd = language.client_cwd
 
   environ = language.global_env()
@@ -653,30 +649,40 @@ def server_jobspec(language, docker_image, insecure=False):
       language.server_cmd(['--port=%s' % _DEFAULT_SERVER_PORT,
                            '--use_tls=%s' % ('false' if insecure else 'true')]))
   environ = language.global_env()
+  docker_args = ['--name=%s' % container_name]
   if language.safename == 'http2':
     # we are running the http2 interop server. Open next N ports beginning
     # with the server port. These ports are used for http2 interop test
-    # (one test case per port). We also attach the docker container running
-    # the server to local network, so we don't have to mess with port mapping
-    port_args = [
-      '-p', str(_DEFAULT_SERVER_PORT+0),
-      '-p', str(_DEFAULT_SERVER_PORT+1),
-      '-p', str(_DEFAULT_SERVER_PORT+2),
-      '-p', str(_DEFAULT_SERVER_PORT+3),
-      '-p', str(_DEFAULT_SERVER_PORT+4),
-      '-p', str(_DEFAULT_SERVER_PORT+5),
-      '-p', str(_DEFAULT_SERVER_PORT+6),
-      '--net=host',
+    # (one test case per port).
+    docker_args += list(
+        itertools.chain.from_iterable(('-p', str(_DEFAULT_SERVER_PORT + i))
+                                      for i in range(
+                                          len(_HTTP2_BADSERVER_TEST_CASES))))
+    # Enable docker's healthcheck mechanism.
+    # This runs a Python script inside the container every second. The script
+    # pings the http2 server to verify it is ready. The 'health-retries' flag
+    # specifies the number of consecutive failures before docker will report
+    # the container's status as 'unhealthy'. Prior to the first 'health_retries'
+    # failures or the first success, the status will be 'starting'. 'docker ps'
+    # or 'docker inspect' can be used to see the health of the container on the
+    # command line.
+    docker_args += [
+        '--health-cmd=python test/http2_test/http2_server_health_check.py '
+        '--server_host=%s --server_port=%d'
+        % ('localhost', _DEFAULT_SERVER_PORT),
+        '--health-interval=1s',
+        '--health-retries=5',
+        '--health-timeout=10s',
     ]
+
   else:
-    port_args = ['-p', str(_DEFAULT_SERVER_PORT)]
+    docker_args += ['-p', str(_DEFAULT_SERVER_PORT)]
 
   docker_cmdline = docker_run_cmdline(cmdline,
                                       image=docker_image,
                                       cwd=language.server_cwd,
                                       environ=environ,
-                                      docker_args=port_args +
-                                        ['--name', container_name])
+                                      docker_args=docker_args)
   server_job = jobset.JobSpec(
           cmdline=docker_cmdline,
           environ=environ,
@@ -849,7 +855,8 @@ languages = set(_LANGUAGES[l]
 languages_http2_badserver_interop = set()
 if args.http2_badserver_interop:
   languages_http2_badserver_interop = set(
-      _LANGUAGES[l] for l in _LANGUAGES_FOR_HTTP2_BADSERVER_TESTS)
+      _LANGUAGES[l] for l in _LANGUAGES_FOR_HTTP2_BADSERVER_TESTS
+      if 'all' in args.language or l in args.language)
 
 http2Interop = Http2Client() if args.http2_interop else None
 http2InteropServer = Http2Server() if args.http2_badserver_interop else None
@@ -888,8 +895,9 @@ if args.use_docker:
       sys.exit(1)
 
 # Start interop servers.
-server_jobs={}
-server_addresses={}
+server_jobs = {}
+server_addresses = {}
+http2_badserver_ports = ()
 try:
   for s in servers:
     lang = str(s)
@@ -899,12 +907,13 @@ try:
     server_jobs[lang] = job
     server_addresses[lang] = ('localhost', job.mapped_port(_DEFAULT_SERVER_PORT))
 
+  http2_badserver_job = None
   if args.http2_badserver_interop:
     # launch a HTTP2 server emulator that creates edge cases
     lang = str(http2InteropServer)
     spec = server_jobspec(http2InteropServer, docker_images.get(lang))
-    job = dockerjob.DockerJob(spec)
-    server_jobs[lang] = job
+    http2_badserver_job = dockerjob.DockerJob(spec)
+    server_jobs[lang] = http2_badserver_job
 
   jobs = []
   if args.cloud_to_prod:
@@ -981,13 +990,16 @@ try:
         jobs.append(test_job)
 
   if args.http2_badserver_interop:
+    http2_badserver_job.wait_for_healthy(timeout_seconds=600)
     for language in languages_http2_badserver_interop:
       for test_case in _HTTP2_BADSERVER_TEST_CASES:
+        offset = sorted(_HTTP2_BADSERVER_TEST_CASES).index(test_case)
+        server_port = http2_badserver_job.mapped_port(_DEFAULT_SERVER_PORT+offset)
         test_job = cloud_to_cloud_jobspec(language,
                                           test_case,
                                           str(http2InteropServer),
                                           'localhost',
-                                          _DEFAULT_SERVER_PORT,
+                                          server_port,
                                           docker_image=docker_images.get(str(language)))
         jobs.append(test_job)