Merge pull request #1586 from davidzchen/python

Bazel build: Keep generated sources and Python runtime in the same directory

BUILD

@@ -32,6 +32,7 @@ load(
     "protobuf",
     "cc_proto_library",
     "py_proto_library",
+    "internal_copied_filegroup",
     "internal_gen_well_known_protos_java",
     "internal_protobuf_py_tests",
 )
@@ -560,6 +561,8 @@ py_library(
             "python/google/protobuf/**/*.py",
         ],
         exclude = [
+            "python/google/protobuf/__init__.py",
+            "python/google/protobuf/**/__init__.py",
             "python/google/protobuf/internal/*_test.py",
             "python/google/protobuf/internal/test_util.py",
         ],
@@ -622,10 +625,26 @@ config_setting(
     },
 )
 
+# Copy the builtin proto files from src/google/protobuf to
+# python/google/protobuf. This way, the generated Python sources will be in the
+# same directory as the Python runtime sources. This is necessary for the
+# modules to be imported correctly since they are all part of the same Python
+# package.
+internal_copied_filegroup(
+    name = "protos_python",
+    srcs = WELL_KNOWN_PROTOS,
+    strip_prefix = "src",
+    dest = "python",
+)
+
+# TODO(dzc): Remove this once py_proto_library can have labels in srcs, in
+# which case we can simply add :protos_python in srcs.
+COPIED_WELL_KNOWN_PROTOS = ["python/" + s for s in RELATIVE_WELL_KNOWN_PROTOS]
+
 py_proto_library(
     name = "protobuf_python",
-    srcs = WELL_KNOWN_PROTOS,
-    include = "src",
+    srcs = COPIED_WELL_KNOWN_PROTOS,
+    include = "python",
     data = select({
         "//conditions:default": [],
         ":use_fast_cpp_protos": [
@@ -643,10 +662,27 @@ py_proto_library(
     visibility = ["//visibility:public"],
 )
 
+# Copy the test proto files from src/google/protobuf to
+# python/google/protobuf. This way, the generated Python sources will be in the
+# same directory as the Python runtime sources. This is necessary for the
+# modules to be imported correctly by the tests since they are all part of the
+# same Python package.
+internal_copied_filegroup(
+    name = "protos_python_test",
+    srcs = LITE_TEST_PROTOS + TEST_PROTOS,
+    strip_prefix = "src",
+    dest = "python",
+)
+
+# TODO(dzc): Remove this once py_proto_library can have labels in srcs, in
+# which case we can simply add :protos_python_test in srcs.
+COPIED_LITE_TEST_PROTOS = ["python/" + s for s in RELATIVE_LITE_TEST_PROTOS]
+COPIED_TEST_PROTOS = ["python/" + s for s in RELATIVE_TEST_PROTOS]
+
 py_proto_library(
     name = "python_common_test_protos",
-    srcs = LITE_TEST_PROTOS + TEST_PROTOS,
-    include = "src",
+    srcs = COPIED_LITE_TEST_PROTOS + COPIED_TEST_PROTOS,
+    include = "python",
     default_runtime = "",
     protoc = ":protoc",
     srcs_version = "PY2AND3",
@@ -672,6 +708,7 @@ py_library(
         [
             "python/google/protobuf/internal/*_test.py",
             "python/google/protobuf/internal/test_util.py",
+            "python/google/protobuf/internal/import_test_package/__init__.py",
         ],
     ),
     imports = ["python"],

protobuf.bzl

@@ -26,7 +26,7 @@ def _CcOuts(srcs, use_grpc_plugin=False):
 def _PyOuts(srcs):
   return [s[:-len(".proto")] + "_pb2.py" for s in srcs]
 
-def _RelativeOutputPath(path, include):
+def _RelativeOutputPath(path, include, dest=""):
   if include == None:
     return path
 
@@ -35,16 +35,11 @@ def _RelativeOutputPath(path, include):
 
   if include and include[-1] != '/':
     include = include + '/'
+  if dest and dest[-1] != '/':
+    dest = dest + '/'
 
   path = path[len(include):]
-
-  if not path.startswith(PACKAGE_NAME):
-    fail("The package %s is not within the path %s" % (PACKAGE_NAME, path))
-
-  if not PACKAGE_NAME:
-    return path
-
-  return path[len(PACKAGE_NAME)+1:]
+  return dest + path
 
 def _proto_gen_impl(ctx):
   """General implementation for generating protos"""
@@ -53,7 +48,7 @@ def _proto_gen_impl(ctx):
   deps += ctx.files.srcs
   gen_dir = _GenDir(ctx)
   if gen_dir:
-    import_flags = ["-I" + gen_dir]
+    import_flags = ["-I" + gen_dir, "-I" + ctx.var["GENDIR"] + "/" + gen_dir]
   else:
     import_flags = ["-I."]
 
@@ -224,6 +219,36 @@ def internal_gen_well_known_protos_java(srcs):
   )
 
 
+def internal_copied_filegroup(name, srcs, strip_prefix, dest, **kwargs):
+  """Macro to copy files to a different directory and then create a filegroup.
+
+  This is used by the //:protobuf_python py_proto_library target to work around
+  an issue caused by Python source files that are part of the same Python
+  package being in separate directories.
+
+  Args:
+    srcs: The source files to copy and add to the filegroup.
+    strip_prefix: Path to the root of the files to copy.
+    dest: The directory to copy the source files into.
+    **kwargs: extra arguments that will be passesd to the filegroup.
+  """
+  outs = [_RelativeOutputPath(s, strip_prefix, dest) for s in srcs]
+
+  native.genrule(
+      name = name + "_genrule",
+      srcs = srcs,
+      outs = outs,
+      cmd = " && ".join(
+          ["cp $(location %s) $(location %s)" %
+           (s, _RelativeOutputPath(s, strip_prefix, dest)) for s in srcs]),
+  )
+
+  native.filegroup(
+      name = name,
+      srcs = outs,
+      **kwargs)
+
+
 def py_proto_library(
         name,
         srcs=[],