Changed python proto build to use shared protos

src/python/.gitignore

@@ -0,0 +1 @@
+gens/

src/python/grpcio/commands.py

@@ -50,6 +50,9 @@ from setuptools.command import test
 import support
 
 PYTHON_STEM = os.path.dirname(os.path.abspath(__file__))
+GRPC_STEM = os.path.abspath(PYTHON_STEM + '../../../../')
+PROTO_STEM = os.path.join(GRPC_STEM, 'src', 'proto')
+PROTO_GEN_STEM = os.path.join(GRPC_STEM, 'src', 'python', 'gens')
 
 CONF_PY_ADDENDUM = """
 extensions.append('sphinx.ext.napoleon')
@@ -157,30 +160,45 @@ class BuildProtoModules(setuptools.Command):
     if not self.grpc_python_plugin_command:
       raise CommandError('could not find grpc_python_plugin '
                          '(protoc plugin for GRPC Python)')
+
+    if not os.path.exists(PROTO_GEN_STEM):
+      os.makedirs(PROTO_GEN_STEM)
+
     include_regex = re.compile(self.include)
     exclude_regex = re.compile(self.exclude) if self.exclude else None
     paths = []
-    root_directory = PYTHON_STEM
-    for walk_root, directories, filenames in os.walk(root_directory):
+    for walk_root, directories, filenames in os.walk(PROTO_STEM):
       for filename in filenames:
         path = os.path.join(walk_root, filename)
         if include_regex.match(path) and not (
             exclude_regex and exclude_regex.match(path)):
           paths.append(path)
-    command = [
-        self.protoc_command,
-        '--plugin=protoc-gen-python-grpc={}'.format(
-            self.grpc_python_plugin_command),
-        '-I {}'.format(root_directory),
-        '--python_out={}'.format(root_directory),
-        '--python-grpc_out={}'.format(root_directory),
-    ] + paths
-    try:
-      subprocess.check_output(' '.join(command), cwd=root_directory, shell=True,
-                              stderr=subprocess.STDOUT)
-    except subprocess.CalledProcessError as e:
-      raise CommandError('Command:\n{}\nMessage:\n{}\nOutput:\n{}'.format(
-          command, e.message, e.output))
+
+    # TODO(kpayson): It would be nice to do this in a batch command,
+    # but we currently have name conflicts in src/proto
+    for path in paths:
+      command = [
+          self.protoc_command,
+          '--plugin=protoc-gen-python-grpc={}'.format(
+              self.grpc_python_plugin_command),
+          '-I {}'.format(GRPC_STEM),
+          '--python_out={}'.format(PROTO_GEN_STEM),
+          '--python-grpc_out={}'.format(PROTO_GEN_STEM),
+      ] + [path]
+      try:
+        subprocess.check_output(' '.join(command), cwd=PYTHON_STEM, shell=True,
+                                stderr=subprocess.STDOUT)
+      except subprocess.CalledProcessError as e:
+        sys.stderr.write(
+            'warning: Command:\n{}\nMessage:\n{}\nOutput:\n{}'.format(
+                command, e.message, e.output))
+
+    # Generated proto directories dont include __init__.py, but
+    # these are needed for python package resolution
+    for walk_root, _, _ in os.walk(PROTO_GEN_STEM):
+      if walk_root != PROTO_GEN_STEM:
+        path = os.path.join(walk_root, '__init__.py')
+        open(path, 'a').close()
 
 
 class BuildProjectMetadata(setuptools.Command):

src/python/grpcio/tests/interop/_insecure_interop_test.py

@@ -32,11 +32,11 @@
 import unittest
 
 from grpc.beta import implementations
+from src.proto.grpc.testing import test_pb2
 
 from tests.interop import _interop_test_case
 from tests.interop import methods
 from tests.interop import server
-from tests.interop import test_pb2
 
 
 class InsecureInteropTest(

src/python/grpcio/tests/interop/_secure_interop_test.py

@@ -32,11 +32,11 @@
 import unittest
 
 from grpc.beta import implementations
+from src.proto.grpc.testing import test_pb2
 
 from tests.interop import _interop_test_case
 from tests.interop import methods
 from tests.interop import resources
-from tests.interop import test_pb2
 
 from tests.unit.beta import test_utilities
 

src/python/grpcio/tests/interop/client.py

@@ -33,10 +33,10 @@ import argparse
 from oauth2client import client as oauth2client_client
 
 from grpc.beta import implementations
+from src.proto.grpc.testing import test_pb2
 
 from tests.interop import methods
 from tests.interop import resources
-from tests.interop import test_pb2
 from tests.unit.beta import test_utilities
 
 _ONE_DAY_IN_SECONDS = 60 * 60 * 24

src/python/grpcio/tests/interop/empty.proto

@@ -1,43 +0,0 @@
-
-// Copyright 2015, Google Inc.
-// All rights reserved.
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are
-// met:
-//
-//     * Redistributions of source code must retain the above copyright
-// notice, this list of conditions and the following disclaimer.
-//     * Redistributions in binary form must reproduce the above
-// copyright notice, this list of conditions and the following disclaimer
-// in the documentation and/or other materials provided with the
-// distribution.
-//     * Neither the name of Google Inc. nor the names of its
-// contributors may be used to endorse or promote products derived from
-// this software without specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-syntax = "proto3";
-
-package grpc.testing;
-
-// An empty message that you can re-use to avoid defining duplicated empty
-// messages in your project. A typical example is to use it as argument or the
-// return value of a service API. For instance:
-//
-//   service Foo {
-//     rpc Bar (grpc.testing.Empty) returns (grpc.testing.Empty) { };
-//   };
-//
-message Empty {}

src/python/grpcio/tests/interop/messages.proto

@@ -1,167 +0,0 @@
-
-// Copyright 2015, Google Inc.
-// All rights reserved.
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are
-// met:
-//
-//     * Redistributions of source code must retain the above copyright
-// notice, this list of conditions and the following disclaimer.
-//     * Redistributions in binary form must reproduce the above
-// copyright notice, this list of conditions and the following disclaimer
-// in the documentation and/or other materials provided with the
-// distribution.
-//     * Neither the name of Google Inc. nor the names of its
-// contributors may be used to endorse or promote products derived from
-// this software without specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-// Message definitions to be used by integration test service definitions.
-
-syntax = "proto3";
-
-package grpc.testing;
-
-// The type of payload that should be returned.
-enum PayloadType {
-  // Compressable text format.
-  COMPRESSABLE = 0;
-
-  // Uncompressable binary format.
-  UNCOMPRESSABLE = 1;
-
-  // Randomly chosen from all other formats defined in this enum.
-  RANDOM = 2;
-}
-
-// Compression algorithms
-enum CompressionType {
-  // No compression
-  NONE = 0;
-  GZIP = 1;
-  DEFLATE = 2;
-}
-
-// A block of data, to simply increase gRPC message size.
-message Payload {
-  // The type of data in body.
-  PayloadType type = 1;
-  // Primary contents of payload.
-  bytes body = 2;
-}
-
-// A protobuf representation for grpc status. This is used by test
-// clients to specify a status that the server should attempt to return.
-message EchoStatus {
-  int32 code = 1;
-  string message = 2;
-}
-
-// Unary request.
-message SimpleRequest {
-  // Desired payload type in the response from the server.
-  // If response_type is RANDOM, server randomly chooses one from other formats.
-  PayloadType response_type = 1;
-
-  // Desired payload size in the response from the server.
-  // If response_type is COMPRESSABLE, this denotes the size before compression.
-  int32 response_size = 2;
-
-  // Optional input payload sent along with the request.
-  Payload payload = 3;
-
-  // Whether SimpleResponse should include username.
-  bool fill_username = 4;
-
-  // Whether SimpleResponse should include OAuth scope.
-  bool fill_oauth_scope = 5;
-
-  // Compression algorithm to be used by the server for the response (stream)
-  CompressionType response_compression = 6;
-
-  // Whether server should return a given status
-  EchoStatus response_status = 7;
-}
-
-// Unary response, as configured by the request.
-message SimpleResponse {
-  // Payload to increase message size.
-  Payload payload = 1;
-  // The user the request came from, for verifying authentication was
-  // successful when the client expected it.
-  string username = 2;
-  // OAuth scope.
-  string oauth_scope = 3;
-}
-
-// Client-streaming request.
-message StreamingInputCallRequest {
-  // Optional input payload sent along with the request.
-  Payload payload = 1;
-
-  // Not expecting any payload from the response.
-}
-
-// Client-streaming response.
-message StreamingInputCallResponse {
-  // Aggregated size of payloads received from the client.
-  int32 aggregated_payload_size = 1;
-}
-
-// Configuration for a particular response.
-message ResponseParameters {
-  // Desired payload sizes in responses from the server.
-  // If response_type is COMPRESSABLE, this denotes the size before compression.
-  int32 size = 1;
-
-  // Desired interval between consecutive responses in the response stream in
-  // microseconds.
-  int32 interval_us = 2;
-}
-
-// Server-streaming request.
-message StreamingOutputCallRequest {
-  // Desired payload type in the response from the server.
-  // If response_type is RANDOM, the payload from each response in the stream
-  // might be of different types. This is to simulate a mixed type of payload
-  // stream.
-  PayloadType response_type = 1;
-
-  // Configuration for each expected response message.
-  repeated ResponseParameters response_parameters = 2;
-
-  // Optional input payload sent along with the request.
-  Payload payload = 3;
-
-  // Compression algorithm to be used by the server for the response (stream)
-  CompressionType response_compression = 6;
-
-  // Whether server should return a given status
-  EchoStatus response_status = 7;
-}
-
-// Server-streaming response, as configured by the request and parameters.
-message StreamingOutputCallResponse {
-  // Payload to increase response size.
-  Payload payload = 1;
-}
-
-// For reconnect interop test only.
-// Server tells client whether its reconnects are following the spec and the
-// reconnect backoffs it saw.
-message ReconnectInfo {
-  bool passed = 1;
-  repeated int32 backoff_ms = 2;
-}

src/python/grpcio/tests/interop/methods.py

@@ -42,9 +42,9 @@ from oauth2client import client as oauth2client_client
 from grpc.framework.common import cardinality
 from grpc.framework.interfaces.face import face
 
-from tests.interop import empty_pb2
-from tests.interop import messages_pb2
-from tests.interop import test_pb2
+from src.proto.grpc.testing import empty_pb2
+from src.proto.grpc.testing import messages_pb2
+from src.proto.grpc.testing import test_pb2
 
 _TIMEOUT = 7
 

src/python/grpcio/tests/interop/server.py

@@ -34,10 +34,10 @@ import logging
 import time
 
 from grpc.beta import implementations
+from src.proto.grpc.testing import test_pb2
 
 from tests.interop import methods
 from tests.interop import resources
-from tests.interop import test_pb2
 
 _ONE_DAY_IN_SECONDS = 60 * 60 * 24
 

src/python/grpcio/tests/interop/test.proto

@@ -1,86 +0,0 @@
-
-// Copyright 2015, Google Inc.
-// All rights reserved.
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are
-// met:
-//
-//     * Redistributions of source code must retain the above copyright
-// notice, this list of conditions and the following disclaimer.
-//     * Redistributions in binary form must reproduce the above
-// copyright notice, this list of conditions and the following disclaimer
-// in the documentation and/or other materials provided with the
-// distribution.
-//     * Neither the name of Google Inc. nor the names of its
-// contributors may be used to endorse or promote products derived from
-// this software without specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-// An integration test service that covers all the method signature permutations
-// of unary/streaming requests/responses.
-
-syntax = "proto3";
-
-import "tests/interop/empty.proto";
-import "tests/interop/messages.proto";
-
-package grpc.testing;
-
-// A simple service to test the various types of RPCs and experiment with
-// performance with various types of payload.
-service TestService {
-  // One empty request followed by one empty response.
-  rpc EmptyCall(grpc.testing.Empty) returns (grpc.testing.Empty);
-
-  // One request followed by one response.
-  rpc UnaryCall(SimpleRequest) returns (SimpleResponse);
-
-  // One request followed by a sequence of responses (streamed download).
-  // The server returns the payload with client desired type and sizes.
-  rpc StreamingOutputCall(StreamingOutputCallRequest)
-      returns (stream StreamingOutputCallResponse);
-
-  // A sequence of requests followed by one response (streamed upload).
-  // The server returns the aggregated size of client payload as the result.
-  rpc StreamingInputCall(stream StreamingInputCallRequest)
-      returns (StreamingInputCallResponse);
-
-  // A sequence of requests with each request served by the server immediately.
-  // As one request could lead to multiple responses, this interface
-  // demonstrates the idea of full duplexing.
-  rpc FullDuplexCall(stream StreamingOutputCallRequest)
-      returns (stream StreamingOutputCallResponse);
-
-  // A sequence of requests followed by a sequence of responses.
-  // The server buffers all the client requests and then serves them in order. A
-  // stream of responses are returned to the client when the server starts with
-  // first request.
-  rpc HalfDuplexCall(stream StreamingOutputCallRequest)
-      returns (stream StreamingOutputCallResponse);
-}
-
-
-// A simple service NOT implemented at servers so clients can test for
-// that case.
-service UnimplementedService {
-  // A call that no server should implement
-  rpc UnimplementedCall(grpc.testing.Empty) returns(grpc.testing.Empty);  
-}
-
-// A service used to control reconnect server.
-service ReconnectService {
-  rpc Start(grpc.testing.Empty) returns (grpc.testing.Empty);
-  rpc Stop(grpc.testing.Empty) returns (grpc.testing.ReconnectInfo);
-}

tools/run_tests/run_interop_tests.py

@@ -314,7 +314,8 @@ class PythonLanguage:
     ]
 
   def global_env(self):
-    return {'LD_LIBRARY_PATH': '{}/libs/opt'.format(DOCKER_WORKDIR_ROOT)}
+    return {'LD_LIBRARY_PATH': '{}/libs/opt'.format(DOCKER_WORKDIR_ROOT),
+            'PYTHONPATH': '{}/src/python/gens'.format(DOCKER_WORKDIR_ROOT)}
 
   def unimplemented_test_cases(self):
     return _SKIP_ADVANCED + _SKIP_COMPRESSION + ['jwt_token_creds',

tools/run_tests/run_tests.py

@@ -371,6 +371,7 @@ class PythonLanguage(object):
       tests_json = json.load(tests_json_file)
     environment = dict(_FORCE_ENVIRON_FOR_WRAPPERS)
     environment['PYVER'] = '2.7'
+    environment['PYTHONPATH'] = os.path.abspath('src/python/gens')
     if self.config.build_config != 'gcov':
       return [self.config.job_spec(
           ['tools/run_tests/run_python.sh'],