Merge pull request #17427 from jtattermusch/go_interop_matrix_fixes

Allow skipping runtimes for releases in interop_matrix. Add go v1.17.0.

templates/tools/dockerfile/interoptest/grpc_interop_go1.11/Dockerfile.template

@@ -0,0 +1,23 @@
+%YAML 1.2
+--- |
+  # Copyright 2018 The gRPC Authors
+  #
+  # Licensed under the Apache License, Version 2.0 (the "License");
+  # you may not use this file except in compliance with the License.
+  # You may obtain a copy of the License at
+  #
+  #     http://www.apache.org/licenses/LICENSE-2.0
+  #
+  # Unless required by applicable law or agreed to in writing, software
+  # distributed under the License is distributed on an "AS IS" BASIS,
+  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  # See the License for the specific language governing permissions and
+  # limitations under the License.
+
+  FROM golang:1.11
+
+  <%include file="../../go_path.include"/>
+  <%include file="../../python_deps.include"/>
+  # Define the default command.
+  CMD ["bash"]
+

templates/tools/dockerfile/interoptest/grpc_interop_go1.11/build_interop.sh.template

@@ -0,0 +1,3 @@
+%YAML 1.2
+--- |
+  <%include file="../../go_build_interop.sh.include"/>  

tools/dockerfile/interoptest/grpc_interop_go1.11/Dockerfile

@@ -0,0 +1,36 @@
+# Copyright 2018 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM golang:1.11
+
+# Using login shell removes Go from path, so we add it.
+RUN ln -s /usr/local/go/bin/go /usr/local/bin
+
+#====================
+# Python dependencies
+
+# Install dependencies
+
+RUN apt-get update && apt-get install -y \
+    python-all-dev \
+    python3-all-dev \
+    python-pip
+
+# Install Python packages from PyPI
+RUN pip install --upgrade pip==10.0.1
+RUN pip install virtualenv
+RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.2.post1 six==1.10.0 twisted==17.5.0
+
+# Define the default command.
+CMD ["bash"]

tools/dockerfile/interoptest/grpc_interop_go1.11/build_interop.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+# Copyright 2015 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Builds Go interop server and client in a base image.
+set -e
+
+# Clone just the grpc-go source code without any dependencies.
+# We are cloning from a local git repo that contains the right revision
+# to test instead of using "go get" to download from Github directly.
+git clone --recursive /var/local/jenkins/grpc-go src/google.golang.org/grpc
+
+# Get all gRPC Go dependencies
+(cd src/google.golang.org/grpc && make deps && make testdeps)
+
+# copy service account keys if available
+cp -r /var/local/jenkins/service_account $HOME || true
+
+# Build the interop client and server
+(cd src/google.golang.org/grpc/interop/client && go install)
+(cd src/google.golang.org/grpc/interop/server && go install)
+  

tools/interop_matrix/client_matrix.py

@@ -35,6 +35,21 @@ def get_release_tag_name(release_info):
     return release_info.keys()[0]
 
 
+def get_runtimes_for_lang_release(lang, release):
+    """Get list of valid runtimes for given release of lang."""
+    runtimes_to_skip = []
+    # see if any the lang release has "skip_runtime" annotation.
+    for release_info in LANG_RELEASE_MATRIX[lang]:
+        if get_release_tag_name(release_info) == release:
+            if release_info[release] is not None:
+                runtimes_to_skip = release_info[release].get('skip_runtime', [])
+                break
+    return [
+        runtime for runtime in LANG_RUNTIME_MATRIX[lang]
+        if runtime not in runtimes_to_skip
+    ]
+
+
 def should_build_docker_interop_image_from_release_tag(lang):
     if lang in ['go', 'java', 'node']:
         return False
@@ -44,7 +59,7 @@ def should_build_docker_interop_image_from_release_tag(lang):
 # Dictionary of runtimes per language
 LANG_RUNTIME_MATRIX = {
     'cxx': ['cxx'],  # This is actually debian8.
-    'go': ['go1.7', 'go1.8'],
+    'go': ['go1.7', 'go1.8', 'go1.11'],
     'java': ['java_oracle8'],
     'python': ['python'],
     'node': ['node'],
@@ -110,52 +125,89 @@ LANG_RELEASE_MATRIX = {
     ],
     'go': [
         {
-            'v1.0.5': None
+            'v1.0.5': {
+                'skip_runtime': ['go1.11']
+            }
         },
         {
-            'v1.2.1': None
+            'v1.2.1': {
+                'skip_runtime': ['go1.11']
+            }
         },
         {
-            'v1.3.0': None
+            'v1.3.0': {
+                'skip_runtime': ['go1.11']
+            }
         },
         {
-            'v1.4.2': None
+            'v1.4.2': {
+                'skip_runtime': ['go1.11']
+            }
         },
         {
-            'v1.5.2': None
+            'v1.5.2': {
+                'skip_runtime': ['go1.11']
+            }
         },
         {
-            'v1.6.0': None
+            'v1.6.0': {
+                'skip_runtime': ['go1.11']
+            }
         },
         {
-            'v1.7.4': None
+            'v1.7.4': {
+                'skip_runtime': ['go1.11']
+            }
         },
         {
-            'v1.8.2': None
+            'v1.8.2': {
+                'skip_runtime': ['go1.11']
+            }
         },
         {
-            'v1.9.2': None
+            'v1.9.2': {
+                'skip_runtime': ['go1.11']
+            }
         },
         {
-            'v1.10.1': None
+            'v1.10.1': {
+                'skip_runtime': ['go1.11']
+            }
         },
         {
-            'v1.11.3': None
+            'v1.11.3': {
+                'skip_runtime': ['go1.11']
+            }
         },
         {
-            'v1.12.2': None
+            'v1.12.2': {
+                'skip_runtime': ['go1.11']
+            }
         },
         {
-            'v1.13.0': None
+            'v1.13.0': {
+                'skip_runtime': ['go1.11']
+            }
         },
         {
-            'v1.14.0': None
+            'v1.14.0': {
+                'skip_runtime': ['go1.11']
+            }
         },
         {
-            'v1.15.0': None
+            'v1.15.0': {
+                'skip_runtime': ['go1.11']
+            }
         },
         {
-            'v1.16.0': None
+            'v1.16.0': {
+                'skip_runtime': ['go1.11']
+            }
+        },
+        {
+            'v1.17.0': {
+                'skip_runtime': ['go1.7', 'go1.8']
+            }
         },
     ],
     'java': [

tools/interop_matrix/create_matrix_images.py

@@ -217,7 +217,7 @@ def build_all_images_for_release(lang, release):
         }.get(lang, 'GRPC_ROOT')
         env[var] = stack_base
 
-    for runtime in client_matrix.LANG_RUNTIME_MATRIX[lang]:
+    for runtime in client_matrix.get_runtimes_for_lang_release(lang, release):
         job = build_image_jobspec(runtime, env, release, stack_base)
         docker_images.append(job.tag)
         build_jobs.append(job)

tools/interop_matrix/run_interop_matrix_tests.py

@@ -113,13 +113,17 @@ def _get_test_images_for_lang(lang, release_arg, image_path_prefix):
             return {}
         releases = [release_arg]
 
-    # Images tuples keyed by runtime.
+    # Image tuples keyed by runtime.
     images = {}
-    for runtime in client_matrix.LANG_RUNTIME_MATRIX[lang]:
-        image_path = '%s/grpc_interop_%s' % (image_path_prefix, runtime)
-        images[runtime] = [
-            (tag, '%s:%s' % (image_path, tag)) for tag in releases
-        ]
+    for tag in releases:
+        for runtime in client_matrix.get_runtimes_for_lang_release(lang, tag):
+            image_name = '%s/grpc_interop_%s:%s' % (image_path_prefix, runtime,
+                                                    tag)
+            image_tuple = (tag, image_name)
+
+            if not images.has_key(runtime):
+                images[runtime] = []
+            images[runtime].append(image_tuple)
     return images