Upgrade sanity Docker image to debian:stretch

* Use latest pylint in Python 3.7 (they dropped support for PY2)
* Make latest pylint happy
* Forced to upgrade to shellcheck 0.4.4
* Make shellcheck 0.4.4 happy
* Adopt reviewers' advice to reduce global disabled rules

.pylintrc

@@ -1,3 +1,11 @@
+[MASTER]
+ignore=
+	src/python/grpcio/grpc/beta,
+	src/python/grpcio/grpc/framework,
+	src/python/grpcio/grpc/framework/common,
+	src/python/grpcio/grpc/framework/foundation,
+	src/python/grpcio/grpc/framework/interfaces,
+
 [VARIABLES]
 
 # TODO(https://github.com/PyCQA/pylint/issues/1345): How does the inspection
@@ -82,3 +90,5 @@ disable=
 	# if:/else: and for:/else:.
 	useless-else-on-loop,
 	no-else-return,
+	# NOTE(lidiz): Python 3 make object inheritance default, but not PY2
+	useless-object-inheritance,

.pylintrc-tests

@@ -1,3 +1,7 @@
+[MASTER]
+ignore-patterns=
+	.+?(beta|framework).+?\.py
+
 [VARIABLES]
 
 # TODO(https://github.com/PyCQA/pylint/issues/1345): How does the inspection
@@ -115,3 +119,5 @@ disable=
 	# if:/else: and for:/else:.
 	useless-else-on-loop,
 	no-else-return,
+	# NOTE(lidiz): Python 3 make object inheritance default, but not PY2
+	useless-object-inheritance,

src/python/grpcio/grpc/_auth.py

@@ -46,7 +46,7 @@ class GoogleCallCredentials(grpc.AuthMetadataPlugin):
 
         # Hack to determine if these are JWT creds and we need to pass
         # additional_claims when getting a token
-        self._is_jwt = 'additional_claims' in inspect.getargspec(
+        self._is_jwt = 'additional_claims' in inspect.getargspec(  # pylint: disable=deprecated-method
             credentials.get_access_token).args
 
     def __call__(self, context, callback):

src/python/grpcio/grpc/_channel.py

@@ -525,7 +525,7 @@ class _UnaryUnaryMultiCallable(grpc.UnaryUnaryMultiCallable):
         state, operations, deadline, rendezvous = self._prepare(
             request, timeout, metadata, wait_for_ready)
         if state is None:
-            raise rendezvous
+            raise rendezvous  # pylint: disable-msg=raising-bad-type
         else:
             call = self._channel.segregated_call(
                 0, self._method, None, deadline, metadata, None
@@ -535,7 +535,7 @@ class _UnaryUnaryMultiCallable(grpc.UnaryUnaryMultiCallable):
                 ),))
             event = call.next_event()
             _handle_event(event, state, self._response_deserializer)
-            return state, call,
+            return state, call
 
     def __call__(self,
                  request,
@@ -566,7 +566,7 @@ class _UnaryUnaryMultiCallable(grpc.UnaryUnaryMultiCallable):
         state, operations, deadline, rendezvous = self._prepare(
             request, timeout, metadata, wait_for_ready)
         if state is None:
-            raise rendezvous
+            raise rendezvous  # pylint: disable-msg=raising-bad-type
         else:
             event_handler = _event_handler(state, self._response_deserializer)
             call = self._managed_call(
@@ -599,7 +599,7 @@ class _UnaryStreamMultiCallable(grpc.UnaryStreamMultiCallable):
         initial_metadata_flags = _InitialMetadataFlags().with_wait_for_ready(
             wait_for_ready)
         if serialized_request is None:
-            raise rendezvous
+            raise rendezvous  # pylint: disable-msg=raising-bad-type
         else:
             state = _RPCState(_UNARY_STREAM_INITIAL_DUE, None, None, None, None)
             operationses = (
@@ -653,7 +653,7 @@ class _StreamUnaryMultiCallable(grpc.StreamUnaryMultiCallable):
                 state.condition.notify_all()
                 if not state.due:
                     break
-        return state, call,
+        return state, call
 
     def __call__(self,
                  request_iterator,
@@ -745,10 +745,10 @@ class _InitialMetadataFlags(int):
     def with_wait_for_ready(self, wait_for_ready):
         if wait_for_ready is not None:
             if wait_for_ready:
-                self = self.__class__(self | cygrpc.InitialMetadataFlags.wait_for_ready | \
+                return self.__class__(self | cygrpc.InitialMetadataFlags.wait_for_ready | \
                     cygrpc.InitialMetadataFlags.wait_for_ready_explicitly_set)
             elif not wait_for_ready:
-                self = self.__class__(self & ~cygrpc.InitialMetadataFlags.wait_for_ready | \
+                return self.__class__(self & ~cygrpc.InitialMetadataFlags.wait_for_ready | \
                     cygrpc.InitialMetadataFlags.wait_for_ready_explicitly_set)
         return self
 

src/python/grpcio/grpc/_utilities.py

@@ -132,15 +132,12 @@ class _ChannelReadyFuture(grpc.Future):
 
     def result(self, timeout=None):
         self._block(timeout)
-        return None
 
     def exception(self, timeout=None):
         self._block(timeout)
-        return None
 
     def traceback(self, timeout=None):
         self._block(timeout)
-        return None
 
     def add_done_callback(self, fn):
         with self._condition:

src/python/grpcio_testing/grpc_testing/_server/_handler.py

@@ -185,7 +185,7 @@ class _Handler(Handler):
                 elif self._code is None:
                     self._condition.wait()
                 else:
-                    return self._trailing_metadata, self._code, self._details,
+                    return self._trailing_metadata, self._code, self._details
 
     def expire(self):
         with self._condition:

src/python/grpcio_tests/tests/_runner.py

@@ -203,7 +203,7 @@ class Runner(object):
                             check_kill_self()
                             time.sleep(0)
                         case_thread.join()
-                except:
+                except:  # pylint: disable=try-except-raise
                     # re-raise the exception after forcing the with-block to end
                     raise
                 result.set_output(augmented_case.case, stdout_pipe.output(),

src/python/grpcio_tests/tests/protoc_plugin/_split_definitions_test.py

@@ -144,7 +144,7 @@ class _ProtoBeforeGrpcProtocStyle(object):
                                        absolute_proto_file_names)
         pb2_grpc_protoc_exit_code = _protoc(
             proto_path, None, 'grpc_2_0', python_out, absolute_proto_file_names)
-        return pb2_protoc_exit_code, pb2_grpc_protoc_exit_code,
+        return pb2_protoc_exit_code, pb2_grpc_protoc_exit_code
 
 
 class _GrpcBeforeProtoProtocStyle(object):
@@ -160,7 +160,7 @@ class _GrpcBeforeProtoProtocStyle(object):
             proto_path, None, 'grpc_2_0', python_out, absolute_proto_file_names)
         pb2_protoc_exit_code = _protoc(proto_path, python_out, None, None,
                                        absolute_proto_file_names)
-        return pb2_grpc_protoc_exit_code, pb2_protoc_exit_code,
+        return pb2_grpc_protoc_exit_code, pb2_protoc_exit_code
 
 
 _PROTOC_STYLES = (
@@ -243,9 +243,9 @@ class _Test(six.with_metaclass(abc.ABCMeta, unittest.TestCase)):
 
     def _services_modules(self):
         if self.PROTOC_STYLE.grpc_in_pb2_expected():
-            return self._services_pb2, self._services_pb2_grpc,
+            return self._services_pb2, self._services_pb2_grpc
         else:
-            return self._services_pb2_grpc,
+            return (self._services_pb2_grpc,)
 
     def test_imported_attributes(self):
         self._protoc()

src/python/grpcio_tests/tests/protoc_plugin/beta_python_plugin_test.py

@@ -223,7 +223,7 @@ def _CreateService(payload_pb2, responses_pb2, service_pb2):
     server.start()
     channel = implementations.insecure_channel('localhost', port)
     stub = getattr(service_pb2, STUB_FACTORY_IDENTIFIER)(channel)
-    yield servicer_methods, stub,
+    yield servicer_methods, stub
     server.stop(0)
 
 

src/python/grpcio_tests/tests/qps/benchmark_client.py

@@ -180,7 +180,7 @@ class StreamingSyncBenchmarkClient(BenchmarkClient):
         self._streams = [
             _SyncStream(self._stub, self._generic, self._request,
                         self._handle_response)
-            for _ in xrange(config.outstanding_rpcs_per_channel)
+            for _ in range(config.outstanding_rpcs_per_channel)
         ]
         self._curr_stream = 0
 

src/python/grpcio_tests/tests/qps/client_runner.py

@@ -77,7 +77,7 @@ class ClosedLoopClientRunner(ClientRunner):
     def start(self):
         self._is_running = True
         self._client.start()
-        for _ in xrange(self._request_count):
+        for _ in range(self._request_count):
             self._client.send_request()
 
     def stop(self):

src/python/grpcio_tests/tests/qps/worker_server.py

@@ -109,7 +109,7 @@ class WorkerServer(worker_service_pb2_grpc.WorkerServiceServicer):
         start_time = time.time()
 
         # Create a client for each channel
-        for i in xrange(config.client_channels):
+        for i in range(config.client_channels):
             server = config.server_targets[i % len(config.server_targets)]
             runner = self._create_client_runner(server, config, qps_data)
             client_runners.append(runner)

src/python/grpcio_tests/tests/stress/client.py

@@ -132,9 +132,9 @@ def run_test(args):
     server.start()
 
     for test_server_target in test_server_targets:
-        for _ in xrange(args.num_channels_per_server):
+        for _ in range(args.num_channels_per_server):
             channel = _get_channel(test_server_target, args)
-            for _ in xrange(args.num_stubs_per_channel):
+            for _ in range(args.num_stubs_per_channel):
                 stub = test_pb2_grpc.TestServiceStub(channel)
                 runner = test_runner.TestRunner(stub, test_cases, hist,
                                                 exception_queue, stop_event)

src/python/grpcio_tests/tests/testing/_client_application.py

@@ -130,9 +130,9 @@ def _run_stream_stream(stub):
     request_pipe = _Pipe()
     response_iterator = stub.StreStre(iter(request_pipe))
     request_pipe.add(_application_common.STREAM_STREAM_REQUEST)
-    first_responses = next(response_iterator), next(response_iterator),
+    first_responses = next(response_iterator), next(response_iterator)
     request_pipe.add(_application_common.STREAM_STREAM_REQUEST)
-    second_responses = next(response_iterator), next(response_iterator),
+    second_responses = next(response_iterator), next(response_iterator)
     request_pipe.close()
     try:
         next(response_iterator)

src/python/grpcio_tests/tests/unit/_from_grpc_import_star.py

@@ -14,7 +14,7 @@
 
 _BEFORE_IMPORT = tuple(globals())
 
-from grpc import *  # pylint: disable=wildcard-import
+from grpc import *  # pylint: disable=wildcard-import,unused-wildcard-import
 
 _AFTER_IMPORT = tuple(globals())
 

templates/tools/dockerfile/test/sanity/Dockerfile.template

@@ -14,28 +14,24 @@
   # See the License for the specific language governing permissions and
   # limitations under the License.
   
-  FROM debian:jessie
-  
-  <%include file="../../apt_get_basic.include"/>
-  <%include file="../../gcp_api_libraries.include"/>
-  <%include file="../../python_deps.include"/>
+  <%include file="../../python_stretch.include"/>
   <%include file="../../cxx_deps.include"/>
   #========================
   # Sanity test dependencies
+  RUN apt-get update && apt-get -t testing install -y python3.7 python3-all-dev
+  RUN curl https://bootstrap.pypa.io/get-pip.py | python3.7
+  # Make Python 3.7 the default Python 3 version
+  RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.7 1
   RUN apt-get update && apt-get install -y ${"\\"}
-        python-pip ${"\\"}
         autoconf ${"\\"}
         automake ${"\\"}
         libtool ${"\\"}
         curl ${"\\"}
-        python-virtualenv ${"\\"}
-        python-lxml ${"\\"}
         shellcheck
-  RUN pip install simplejson mako
-  
+  RUN python2 -m pip install simplejson mako virtualenv lxml
+  RUN python3 -m pip install simplejson mako virtualenv lxml
+
   <%include file="../../clang5.include"/>
-  <%include file="../../run_tests_addons.include"/>
-  
+
   # Define the default command.
   CMD ["bash"]
-  

tools/distrib/pylint_code.sh

@@ -31,12 +31,12 @@ TEST_DIRS=(
 )
 
 VIRTUALENV=python_pylint_venv
-python -m virtualenv $VIRTUALENV
+python3 -m virtualenv $VIRTUALENV
 
 PYTHON=$VIRTUALENV/bin/python
 
-$PYTHON -m pip install --upgrade pip==10.0.1
-$PYTHON -m pip install pylint==1.9.2
+$PYTHON -m pip install --upgrade pip==18.1
+$PYTHON -m pip install --upgrade pylint==2.2.2
 
 EXIT=0
 for dir in "${DIRS[@]}"; do

tools/dockerfile/test/sanity/Dockerfile

@@ -12,8 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM debian:jessie
-
+FROM debian:stretch
+  
 # Install Git and basic packages.
 RUN apt-get update && apt-get install -y \
   autoconf \
@@ -53,20 +53,19 @@ RUN apt-get update && apt-get install -y time && apt-get clean
 RUN apt-get update && apt-get install -y python-pip && apt-get clean
 RUN pip install --upgrade google-api-python-client oauth2client
 
-#====================
-# Python dependencies
+# Install Python 2.7
+RUN apt-get update && apt-get install -y python2.7 python-all-dev
+RUN curl https://bootstrap.pypa.io/get-pip.py | python2.7
 
-# Install dependencies
+# Add Debian 'testing' repository
+RUN echo 'deb http://ftp.de.debian.org/debian testing main' >> /etc/apt/sources.list
+RUN echo 'APT::Default-Release "stable";' | tee -a /etc/apt/apt.conf.d/00local
 
-RUN apt-get update && apt-get install -y \
-    python-all-dev \
-    python3-all-dev \
-    python-pip
 
-# Install Python packages from PyPI
-RUN pip install --upgrade pip==10.0.1
-RUN pip install virtualenv
-RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.2.post1 six==1.10.0 twisted==17.5.0
+RUN mkdir /var/local/jenkins
+
+# Define the default command.
+CMD ["bash"]
 
 #=================
 # C++ dependencies
@@ -74,16 +73,18 @@ RUN apt-get update && apt-get -y install libgflags-dev libgtest-dev libc++-dev c
 
 #========================
 # Sanity test dependencies
+RUN apt-get update && apt-get -t testing install -y python3.7 python3-all-dev
+RUN curl https://bootstrap.pypa.io/get-pip.py | python3.7
+# Make Python 3.7 the default Python 3 version
+RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.7 1
 RUN apt-get update && apt-get install -y \
-      python-pip \
       autoconf \
       automake \
       libtool \
       curl \
-      python-virtualenv \
-      python-lxml \
       shellcheck
-RUN pip install simplejson mako
+RUN python2 -m pip install simplejson mako virtualenv lxml
+RUN python3 -m pip install simplejson mako virtualenv lxml
 
 RUN apt-get update && apt-get -y install wget xz-utils
 RUN wget http://releases.llvm.org/5.0.0/clang+llvm-5.0.0-linux-x86_64-ubuntu14.04.tar.xz
@@ -94,8 +95,5 @@ RUN ln -s /clang+llvm-5.0.0-linux-x86_64-ubuntu14.04/bin/clang-tidy /usr/local/b
 ENV CLANG_TIDY=clang-tidy
 
 
-RUN mkdir /var/local/jenkins
-
-
 # Define the default command.
 CMD ["bash"]

tools/run_tests/artifacts/build_artifact_protoc.sh

@@ -14,6 +14,7 @@
 # limitations under the License.
 
 # Use devtoolset environment that has GCC 4.8 before set -ex
+# shellcheck disable=SC1091
 source scl_source enable devtoolset-2
 
 set -ex

tools/run_tests/artifacts/build_artifact_ruby.sh

@@ -18,7 +18,9 @@ SYSTEM=$(uname | cut -f 1 -d_)
 
 cd "$(dirname "$0")/../../.."
 set +ex
+# shellcheck disable=SC1091
 [[ -s /etc/profile.d/rvm.sh ]] && . /etc/profile.d/rvm.sh
+# shellcheck disable=SC1090
 [[ -s "$HOME/.rvm/scripts/rvm" ]] && source "$HOME/.rvm/scripts/rvm"
 set -ex
 

tools/run_tests/artifacts/run_in_workspace.sh

@@ -19,7 +19,8 @@
 set -ex
 
 cd "$(dirname "$0")/../../.."
-export repo_root=$(pwd)
+repo_root=$(pwd)
+export repo_root
 
 # TODO: fix file to pass shellcheck
 

tools/run_tests/dockerize/build_and_run_docker.sh

@@ -20,7 +20,7 @@ set -ex
 
 cd "$(dirname "$0")/../../.."
 git_root=$(pwd)
-cd -
+cd - # shellcheck disable=SC2103
 
 # Inputs
 # DOCKERFILE_DIR - Directory in which Dockerfile file is located.

tools/run_tests/dockerize/build_docker_and_run_tests.sh

@@ -20,7 +20,7 @@ set -ex
 
 cd "$(dirname "$0")/../../.."
 git_root=$(pwd)
-cd -
+cd - # shellcheck disable=SC2103
 
 # Inputs
 # DOCKERFILE_DIR - Directory in which Dockerfile file is located.
@@ -48,7 +48,7 @@ docker_instance_git_root=/var/local/jenkins/grpc
 # Run tests inside docker
 DOCKER_EXIT_CODE=0
 # TODO: silence complaint about $TTY_FLAG expansion in some other way
-# shellcheck disable=SC2086
+# shellcheck disable=SC2086,SC2154
 docker run \
   --cap-add SYS_PTRACE \
   -e "RUN_TESTS_COMMAND=$RUN_TESTS_COMMAND" \

tools/run_tests/dockerize/build_interop_image.sh

@@ -16,7 +16,7 @@
 # This script is invoked by run_interop_tests.py to build the docker image
 # for interop testing. You should never need to call this script on your own.
 
-set -x
+set -ex
 
 # Params:
 #  INTEROP_IMAGE - name of tag of the final interop image

tools/run_tests/dockerize/docker_run_tests.sh

@@ -18,6 +18,7 @@
 
 set -e
 
+# shellcheck disable=SC2154
 export CONFIG=$config
 export ASAN_SYMBOLIZER_PATH=/usr/bin/llvm-symbolizer
 export PATH=$PATH:/usr/bin/llvm-symbolizer

tools/run_tests/helper_scripts/run_grpc-node.sh

@@ -16,6 +16,8 @@
 # This script runs grpc/grpc-node tests with their grpc submodule updated
 # to this reference
 
+set -ex
+
 # cd to gRPC root directory
 cd "$(dirname "$0")/../../.."
 

tools/run_tests/helper_scripts/run_tests_in_workspace.sh

@@ -20,7 +20,8 @@
 set -ex
 
 cd "$(dirname "$0")/../../.."
-export repo_root="$(pwd)"
+repo_root="$(pwd)"
+export repo_root
 
 rm -rf "${WORKSPACE_NAME}"
 git clone . "${WORKSPACE_NAME}"

tools/run_tests/interop/with_nvm.sh

@@ -15,5 +15,6 @@
 # limitations under the License.
 
 # Makes sure NVM is loaded before executing the command passed as an argument
+# shellcheck disable=SC1090
 source ~/.nvm/nvm.sh
 "$@"

tools/run_tests/interop/with_rvm.sh

@@ -15,5 +15,6 @@
 # limitations under the License.
 
 # Makes sure RVM is loaded before executing the command passed as an argument
+# shellcheck disable=SC1091
 source /usr/local/rvm/scripts/rvm
 "$@"

tools/run_tests/performance/build_performance.sh

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# shellcheck disable=SC1090
 source ~/.rvm/scripts/rvm
 set -ex
 

tools/run_tests/performance/build_performance_go.sh

@@ -17,7 +17,8 @@ set -ex
 
 cd "$(dirname "$0")/../../.."
 
-export GOPATH=$(pwd)/../gopath
+GOPATH=$(pwd)/../gopath
+export GOPATH
 
 # Get grpc-go and the dependencies but get rid of the upstream/master version
 go get google.golang.org/grpc

tools/run_tests/performance/build_performance_node.sh

@@ -15,6 +15,7 @@
 
 set +ex
 
+# shellcheck disable=SC1090
 . "$HOME/.nvm/nvm.sh"
 
 nvm install 10

tools/run_tests/performance/run_worker_go.sh

@@ -17,6 +17,7 @@ set -ex
 
 cd "$(dirname "$0")/../../.."
 
-export GOPATH=$(pwd)/../gopath
+GOPATH=$(pwd)/../gopath
+export GOPATH
 
 "${GOPATH}/bin/worker" "$@"

tools/run_tests/performance/run_worker_node.sh

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# shellcheck disable=SC1090
 . "$HOME/.nvm/nvm.sh"
 
 nvm use 10

tools/run_tests/performance/run_worker_php.sh

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# shellcheck disable=SC1090
 source ~/.rvm/scripts/rvm
 set -ex
 

tools/run_tests/performance/run_worker_ruby.sh

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# shellcheck disable=SC1090
 source ~/.rvm/scripts/rvm
 set -ex