Merge pull request #18859 from grpc/internal_py_proto_library

Remove dependency on pubref/rules_proto.

.gitignore

@@ -115,6 +115,7 @@ bazel-genfiles
 bazel-grpc
 bazel-out
 bazel-testlogs
+bazel_format_virtual_environment/
 
 # Debug output
 gdb.txt

WORKSPACE

@@ -18,13 +18,6 @@ register_toolchains(
     "//third_party/toolchains/bazel_0.23.2_rbe_windows:cc-toolchain-x64_windows",
 )
 
-# TODO(https://github.com/grpc/grpc/issues/18331): Move off of this dependency.
-git_repository(
-    name = "org_pubref_rules_protobuf",
-    remote = "https://github.com/ghostwriternr/rules_protobuf",
-    tag = "v0.8.2.1-alpha",
-)
-
 git_repository(
     name = "io_bazel_rules_python",
     commit = "8b5d0683a7d878b28fffe464779c8a53659fc645",

bazel/generate_cc.bzl

@@ -4,81 +4,132 @@ This is an internal rule used by cc_grpc_library, and shouldn't be used
 directly.
 """
 
-def generate_cc_impl(ctx):
-  """Implementation of the generate_cc rule."""
-  protos = [f for src in ctx.attr.srcs for f in src.proto.direct_sources]
-  includes = [f for src in ctx.attr.srcs for f in src.proto.transitive_imports]
-  outs = []
-  # label_len is length of the path from WORKSPACE root to the location of this build file
-  label_len = 0
-  # proto_root is the directory relative to which generated include paths should be
-  proto_root = ""
-  if ctx.label.package:
-    # The +1 is for the trailing slash.
-    label_len += len(ctx.label.package) + 1
-  if ctx.label.workspace_root:
-    label_len += len(ctx.label.workspace_root) + 1
-    proto_root = "/" + ctx.label.workspace_root
+load(
+    "//bazel:protobuf.bzl",
+    "get_include_protoc_args",
+    "get_plugin_args",
+    "get_proto_root",
+    "proto_path_to_generated_filename",
+)
+
+_GRPC_PROTO_HEADER_FMT = "{}.grpc.pb.h"
+_GRPC_PROTO_SRC_FMT = "{}.grpc.pb.cc"
+_GRPC_PROTO_MOCK_HEADER_FMT = "{}_mock.grpc.pb.h"
+_PROTO_HEADER_FMT = "{}.pb.h"
+_PROTO_SRC_FMT = "{}.pb.cc"
 
-  if ctx.executable.plugin:
-    outs += [proto.path[label_len:-len(".proto")] + ".grpc.pb.h" for proto in protos]
-    outs += [proto.path[label_len:-len(".proto")] + ".grpc.pb.cc" for proto in protos]
-    if ctx.attr.generate_mocks:
-      outs += [proto.path[label_len:-len(".proto")] + "_mock.grpc.pb.h" for proto in protos]
-  else:
-    outs += [proto.path[label_len:-len(".proto")] + ".pb.h" for proto in protos]
-    outs += [proto.path[label_len:-len(".proto")] + ".pb.cc" for proto in protos]
-  out_files = [ctx.actions.declare_file(out) for out in outs]
-  dir_out = str(ctx.genfiles_dir.path + proto_root)
+def _strip_package_from_path(label_package, path):
+    if len(label_package) == 0:
+        return path
+    if not path.startswith(label_package + "/"):
+        fail("'{}' does not lie within '{}'.".format(path, label_package))
+    return path[len(label_package + "/"):]
 
-  arguments = []
-  if ctx.executable.plugin:
-    arguments += ["--plugin=protoc-gen-PLUGIN=" + ctx.executable.plugin.path]
-    flags = list(ctx.attr.flags)
-    if ctx.attr.generate_mocks:
-      flags.append("generate_mock_code=true")
-    arguments += ["--PLUGIN_out=" + ",".join(flags) + ":" + dir_out]
-    tools = [ctx.executable.plugin]
-  else:
-    arguments += ["--cpp_out=" + ",".join(ctx.attr.flags) + ":" + dir_out]
-    tools = []
+def _join_directories(directories):
+    massaged_directories = [directory for directory in directories if len(directory) != 0]
+    return "/".join(massaged_directories)
 
-  # Import protos relative to their workspace root so that protoc prints the
-  # right include paths.
-  for include in includes:
-    directory = include.path
-    if directory.startswith("external"):
-      external_sep = directory.find("/")
-      repository_sep = directory.find("/", external_sep + 1)
-      arguments += ["--proto_path=" + directory[:repository_sep]]
+def generate_cc_impl(ctx):
+    """Implementation of the generate_cc rule."""
+    protos = [f for src in ctx.attr.srcs for f in src.proto.direct_sources]
+    includes = [
+        f
+        for src in ctx.attr.srcs
+        for f in src.proto.transitive_imports
+    ]
+    outs = []
+    proto_root = get_proto_root(
+        ctx.label.workspace_root,
+    )
+
+    label_package = _join_directories([ctx.label.workspace_root, ctx.label.package])
+    if ctx.executable.plugin:
+        outs += [
+            proto_path_to_generated_filename(
+                _strip_package_from_path(label_package, proto.path),
+                _GRPC_PROTO_HEADER_FMT,
+            )
+            for proto in protos
+        ]
+        outs += [
+            proto_path_to_generated_filename(
+                _strip_package_from_path(label_package, proto.path),
+                _GRPC_PROTO_SRC_FMT,
+            )
+            for proto in protos
+        ]
+        if ctx.attr.generate_mocks:
+            outs += [
+                proto_path_to_generated_filename(
+                    _strip_package_from_path(label_package, proto.path),
+                    _GRPC_PROTO_MOCK_HEADER_FMT,
+                )
+                for proto in protos
+            ]
     else:
-      arguments += ["--proto_path=."]
-  # Include the output directory so that protoc puts the generated code in the
-  # right directory.
-  arguments += ["--proto_path={0}{1}".format(dir_out, proto_root)]
-  arguments += [proto.path for proto in protos]
+        outs += [
+            proto_path_to_generated_filename(
+                _strip_package_from_path(label_package, proto.path),
+                _PROTO_HEADER_FMT,
+            )
+            for proto in protos
+        ]
+        outs += [
+            proto_path_to_generated_filename(
+                _strip_package_from_path(label_package, proto.path),
+                _PROTO_SRC_FMT,
+            )
+            for proto in protos
+        ]
+    out_files = [ctx.actions.declare_file(out) for out in outs]
+    dir_out = str(ctx.genfiles_dir.path + proto_root)
 
-  # create a list of well known proto files if the argument is non-None
-  well_known_proto_files = []
-  if ctx.attr.well_known_protos:
-    f = ctx.attr.well_known_protos.files.to_list()[0].dirname
-    if f != "external/com_google_protobuf/src/google/protobuf":
-      print("Error: Only @com_google_protobuf//:well_known_protos is supported")
+    arguments = []
+    if ctx.executable.plugin:
+        arguments += get_plugin_args(
+            ctx.executable.plugin,
+            ctx.attr.flags,
+            dir_out,
+            ctx.attr.generate_mocks,
+        )
+        tools = [ctx.executable.plugin]
     else:
-      # f points to "external/com_google_protobuf/src/google/protobuf"
-      # add -I argument to protoc so it knows where to look for the proto files.
-      arguments += ["-I{0}".format(f + "/../..")]
-      well_known_proto_files = [f for f in ctx.attr.well_known_protos.files]
+        arguments += ["--cpp_out=" + ",".join(ctx.attr.flags) + ":" + dir_out]
+        tools = []
+
+    arguments += get_include_protoc_args(includes)
 
-  ctx.actions.run(
-      inputs = protos + includes + well_known_proto_files,
-      tools = tools,
-      outputs = out_files,
-      executable = ctx.executable._protoc,
-      arguments = arguments,
-  )
+    # Include the output directory so that protoc puts the generated code in the
+    # right directory.
+    arguments += ["--proto_path={0}{1}".format(dir_out, proto_root)]
+    arguments += [proto.path for proto in protos]
 
-  return struct(files=depset(out_files))
+    # create a list of well known proto files if the argument is non-None
+    well_known_proto_files = []
+    if ctx.attr.well_known_protos:
+        f = ctx.attr.well_known_protos.files.to_list()[0].dirname
+        if f != "external/com_google_protobuf/src/google/protobuf":
+            print(
+                "Error: Only @com_google_protobuf//:well_known_protos is supported",
+            )
+        else:
+            # f points to "external/com_google_protobuf/src/google/protobuf"
+            # add -I argument to protoc so it knows where to look for the proto files.
+            arguments += ["-I{0}".format(f + "/../..")]
+            well_known_proto_files = [
+                f
+                for f in ctx.attr.well_known_protos.files
+            ]
+
+    ctx.actions.run(
+        inputs = protos + includes + well_known_proto_files,
+        tools = tools,
+        outputs = out_files,
+        executable = ctx.executable._protoc,
+        arguments = arguments,
+    )
+
+    return struct(files = depset(out_files))
 
 _generate_cc = rule(
     attrs = {
@@ -96,10 +147,8 @@ _generate_cc = rule(
             mandatory = False,
             allow_empty = True,
         ),
-        "well_known_protos" : attr.label(
-            mandatory = False,
-        ),
-        "generate_mocks" : attr.bool(
+        "well_known_protos": attr.label(mandatory = False),
+        "generate_mocks": attr.bool(
             default = False,
             mandatory = False,
         ),
@@ -115,7 +164,10 @@ _generate_cc = rule(
 )
 
 def generate_cc(well_known_protos, **kwargs):
-  if well_known_protos:
-    _generate_cc(well_known_protos="@com_google_protobuf//:well_known_protos", **kwargs)
-  else:
-    _generate_cc(**kwargs)
+    if well_known_protos:
+        _generate_cc(
+            well_known_protos = "@com_google_protobuf//:well_known_protos",
+            **kwargs
+        )
+    else:
+        _generate_cc(**kwargs)

bazel/grpc_python_deps.bzl

@@ -1,16 +1,8 @@
 load("//third_party/py:python_configure.bzl", "python_configure")
 load("@io_bazel_rules_python//python:pip.bzl", "pip_repositories")
 load("@grpc_python_dependencies//:requirements.bzl", "pip_install")
-load("@org_pubref_rules_protobuf//python:rules.bzl", "py_proto_repositories")
 
 def grpc_python_deps():
-    # TODO(https://github.com/grpc/grpc/issues/18256): Remove conditional.
-    if hasattr(native, "http_archive"):
-        python_configure(name = "local_config_python")
-        pip_repositories()
-        pip_install()
-        py_proto_repositories()
-    else:
-        print("Building Python gRPC with bazel 23.0+ is disabled pending " +
-              "resolution of https://github.com/grpc/grpc/issues/18256.")
-
+    python_configure(name = "local_config_python")
+    pip_repositories()
+    pip_install()

bazel/protobuf.bzl

@@ -0,0 +1,84 @@
+"""Utility functions for generating protobuf code."""
+
+_PROTO_EXTENSION = ".proto"
+
+def get_proto_root(workspace_root):
+    """Gets the root protobuf directory.
+
+    Args:
+      workspace_root: context.label.workspace_root
+
+    Returns:
+      The directory relative to which generated include paths should be.
+    """
+    if workspace_root:
+        return "/{}".format(workspace_root)
+    else:
+        return ""
+
+def _strip_proto_extension(proto_filename):
+    if not proto_filename.endswith(_PROTO_EXTENSION):
+        fail('"{}" does not end with "{}"'.format(
+            proto_filename,
+            _PROTO_EXTENSION,
+        ))
+    return proto_filename[:-len(_PROTO_EXTENSION)]
+
+def proto_path_to_generated_filename(proto_path, fmt_str):
+    """Calculates the name of a generated file for a protobuf path.
+
+    For example, "examples/protos/helloworld.proto" might map to
+      "helloworld.pb.h".
+
+    Args:
+      proto_path: The path to the .proto file.
+      fmt_str: A format string used to calculate the generated filename. For
+        example, "{}.pb.h" might be used to calculate a C++ header filename.
+
+    Returns:
+      The generated filename.
+    """
+    return fmt_str.format(_strip_proto_extension(proto_path))
+
+def _get_include_directory(include):
+    directory = include.path
+    if directory.startswith("external"):
+        external_separator = directory.find("/")
+        repository_separator = directory.find("/", external_separator + 1)
+        return directory[:repository_separator]
+    else:
+        return "."
+
+def get_include_protoc_args(includes):
+    """Returns protoc args that imports protos relative to their import root.
+
+    Args:
+      includes: A list of included proto files.
+
+    Returns:
+      A list of arguments to be passed to protoc. For example, ["--proto_path=."].
+    """
+    return [
+        "--proto_path={}".format(_get_include_directory(include))
+        for include in includes
+    ]
+
+def get_plugin_args(plugin, flags, dir_out, generate_mocks):
+    """Returns arguments configuring protoc to use a plugin for a language.
+
+    Args:
+      plugin: An executable file to run as the protoc plugin.
+      flags: The plugin flags to be passed to protoc.
+      dir_out: The output directory for the plugin.
+      generate_mocks: A bool indicating whether to generate mocks.
+
+    Returns:
+      A list of protoc arguments configuring the plugin.
+    """
+    augmented_flags = list(flags)
+    if generate_mocks:
+        augmented_flags.append("generate_mock_code=true")
+    return [
+        "--plugin=protoc-gen-PLUGIN=" + plugin.path,
+        "--PLUGIN_out=" + ",".join(augmented_flags) + ":" + dir_out,
+    ]

bazel/python_rules.bzl

@@ -0,0 +1,203 @@
+"""Generates and compiles Python gRPC stubs from proto_library rules."""
+
+load("@grpc_python_dependencies//:requirements.bzl", "requirement")
+load(
+    "//bazel:protobuf.bzl",
+    "get_include_protoc_args",
+    "get_plugin_args",
+    "get_proto_root",
+    "proto_path_to_generated_filename",
+)
+
+_GENERATED_PROTO_FORMAT = "{}_pb2.py"
+_GENERATED_GRPC_PROTO_FORMAT = "{}_pb2_grpc.py"
+
+def _get_staged_proto_file(context, source_file):
+    if source_file.dirname == context.label.package:
+        return source_file
+    else:
+        copied_proto = context.actions.declare_file(source_file.basename)
+        context.actions.run_shell(
+            inputs = [source_file],
+            outputs = [copied_proto],
+            command = "cp {} {}".format(source_file.path, copied_proto.path),
+            mnemonic = "CopySourceProto",
+        )
+        return copied_proto
+
+def _generate_py_impl(context):
+    protos = []
+    for src in context.attr.deps:
+        for file in src.proto.direct_sources:
+            protos.append(_get_staged_proto_file(context, file))
+    includes = [
+        file
+        for src in context.attr.deps
+        for file in src.proto.transitive_imports
+    ]
+    proto_root = get_proto_root(context.label.workspace_root)
+    format_str = (_GENERATED_GRPC_PROTO_FORMAT if context.executable.plugin else _GENERATED_PROTO_FORMAT)
+    out_files = [
+        context.actions.declare_file(
+            proto_path_to_generated_filename(
+                proto.basename,
+                format_str,
+            ),
+        )
+        for proto in protos
+    ]
+
+    arguments = []
+    tools = [context.executable._protoc]
+    if context.executable.plugin:
+        arguments += get_plugin_args(
+            context.executable.plugin,
+            context.attr.flags,
+            context.genfiles_dir.path,
+            False,
+        )
+        tools += [context.executable.plugin]
+    else:
+        arguments += [
+            "--python_out={}:{}".format(
+                ",".join(context.attr.flags),
+                context.genfiles_dir.path,
+            ),
+        ]
+
+    arguments += get_include_protoc_args(includes)
+    arguments += [
+        "--proto_path={}".format(context.genfiles_dir.path)
+        for proto in protos
+    ]
+    for proto in protos:
+        massaged_path = proto.path
+        if massaged_path.startswith(context.genfiles_dir.path):
+            massaged_path = proto.path[len(context.genfiles_dir.path) + 1:]
+        arguments.append(massaged_path)
+
+    well_known_proto_files = []
+    if context.attr.well_known_protos:
+        well_known_proto_directory = context.attr.well_known_protos.files.to_list(
+        )[0].dirname
+
+        arguments += ["-I{}".format(well_known_proto_directory + "/../..")]
+        well_known_proto_files = context.attr.well_known_protos.files.to_list()
+
+    context.actions.run(
+        inputs = protos + includes + well_known_proto_files,
+        tools = tools,
+        outputs = out_files,
+        executable = context.executable._protoc,
+        arguments = arguments,
+        mnemonic = "ProtocInvocation",
+    )
+    return struct(files = depset(out_files))
+
+__generate_py = rule(
+    attrs = {
+        "deps": attr.label_list(
+            mandatory = True,
+            allow_empty = False,
+            providers = ["proto"],
+        ),
+        "plugin": attr.label(
+            executable = True,
+            providers = ["files_to_run"],
+            cfg = "host",
+        ),
+        "flags": attr.string_list(
+            mandatory = False,
+            allow_empty = True,
+        ),
+        "well_known_protos": attr.label(mandatory = False),
+        "_protoc": attr.label(
+            default = Label("//external:protocol_compiler"),
+            executable = True,
+            cfg = "host",
+        ),
+    },
+    output_to_genfiles = True,
+    implementation = _generate_py_impl,
+)
+
+def _generate_py(well_known_protos, **kwargs):
+    if well_known_protos:
+        __generate_py(
+            well_known_protos = "@com_google_protobuf//:well_known_protos",
+            **kwargs
+        )
+    else:
+        __generate_py(**kwargs)
+
+_WELL_KNOWN_PROTO_LIBS = [
+    "@com_google_protobuf//:any_proto",
+    "@com_google_protobuf//:api_proto",
+    "@com_google_protobuf//:compiler_plugin_proto",
+    "@com_google_protobuf//:descriptor_proto",
+    "@com_google_protobuf//:duration_proto",
+    "@com_google_protobuf//:empty_proto",
+    "@com_google_protobuf//:field_mask_proto",
+    "@com_google_protobuf//:source_context_proto",
+    "@com_google_protobuf//:struct_proto",
+    "@com_google_protobuf//:timestamp_proto",
+    "@com_google_protobuf//:type_proto",
+    "@com_google_protobuf//:wrappers_proto",
+]
+
+def py_proto_library(
+        name,
+        deps,
+        well_known_protos = True,
+        proto_only = False,
+        **kwargs):
+    """Generate python code for a protobuf.
+
+    Args:
+      name: The name of the target.
+      deps: A list of dependencies. Must contain a single element.
+      well_known_protos: A bool indicating whether or not to include well-known
+        protos.
+      proto_only: A bool indicating whether to generate vanilla protobuf code
+        or to also generate gRPC code.
+    """
+    if len(deps) > 1:
+        fail("The supported length of 'deps' is 1.")
+
+    codegen_target = "_{}_codegen".format(name)
+    codegen_grpc_target = "_{}_grpc_codegen".format(name)
+
+    well_known_proto_rules = _WELL_KNOWN_PROTO_LIBS if well_known_protos else []
+
+    _generate_py(
+        name = codegen_target,
+        deps = deps,
+        well_known_protos = well_known_protos,
+        **kwargs
+    )
+
+    if not proto_only:
+        _generate_py(
+            name = codegen_grpc_target,
+            deps = deps,
+            plugin = "//:grpc_python_plugin",
+            well_known_protos = well_known_protos,
+            **kwargs
+        )
+
+        native.py_library(
+            name = name,
+            srcs = [
+                ":{}".format(codegen_grpc_target),
+                ":{}".format(codegen_target),
+            ],
+            deps = [requirement("protobuf")],
+            **kwargs
+        )
+    else:
+        native.py_library(
+            name = name,
+            srcs = [":{}".format(codegen_target), ":{}".format(codegen_target)],
+            deps = [requirement("protobuf")],
+            **kwargs
+        )

examples/BUILD

@@ -16,9 +16,8 @@ licenses(["notice"])  # 3-clause BSD
 
 package(default_visibility = ["//visibility:public"])
 
-load("@grpc_python_dependencies//:requirements.bzl", "requirement")
 load("//bazel:grpc_build_system.bzl", "grpc_proto_library")
-load("@org_pubref_rules_protobuf//python:rules.bzl", "py_proto_library")
+load("//bazel:python_rules.bzl", "py_proto_library")
 
 grpc_proto_library(
     name = "auth_sample",
@@ -45,11 +44,14 @@ grpc_proto_library(
     srcs = ["protos/keyvaluestore.proto"],
 )
 
+proto_library(
+    name = "helloworld_proto_descriptor",
+    srcs = ["protos/helloworld.proto"],
+)
+
 py_proto_library(
     name = "py_helloworld",
-    protos = ["protos/helloworld.proto"],
-    with_grpc = True,
-    deps = [requirement('protobuf'),],
+    deps = [":helloworld_proto_descriptor"],
 )
 
 cc_binary(

examples/python/errors/client.py

@@ -20,8 +20,8 @@ import grpc
 from grpc_status import rpc_status
 from google.rpc import error_details_pb2
 
-from examples.protos import helloworld_pb2
-from examples.protos import helloworld_pb2_grpc
+from examples import helloworld_pb2
+from examples import helloworld_pb2_grpc
 
 _LOGGER = logging.getLogger(__name__)
 

examples/python/errors/server.py

@@ -24,8 +24,8 @@ from grpc_status import rpc_status
 from google.protobuf import any_pb2
 from google.rpc import code_pb2, status_pb2, error_details_pb2
 
-from examples.protos import helloworld_pb2
-from examples.protos import helloworld_pb2_grpc
+from examples import helloworld_pb2
+from examples import helloworld_pb2_grpc
 
 _ONE_DAY_IN_SECONDS = 60 * 60 * 24
 

examples/python/errors/test/_error_handling_example_test.py

@@ -26,7 +26,7 @@ import logging
 
 import grpc
 
-from examples.protos import helloworld_pb2_grpc
+from examples import helloworld_pb2_grpc
 from examples.python.errors import client as error_handling_client
 from examples.python.errors import server as error_handling_server
 

examples/python/multiprocessing/BUILD

@@ -15,12 +15,17 @@
 # limitations under the License.
 
 load("@grpc_python_dependencies//:requirements.bzl", "requirement")
-load("@org_pubref_rules_protobuf//python:rules.bzl", "py_proto_library")
+load("//bazel:python_rules.bzl", "py_proto_library")
 
-py_proto_library(
+proto_library(
     name = "prime_proto",
-    protos = ["prime.proto",],
-    deps = [requirement("protobuf")],
+    srcs = ["prime.proto"]
+)
+
+py_proto_library(
+    name = "prime_proto_pb2",
+    deps = [":prime_proto"],
+    well_known_protos = False,
 )
 
 py_binary(
@@ -29,7 +34,7 @@ py_binary(
     srcs = ["client.py"],
     deps = [
         "//src/python/grpcio/grpc:grpcio",
-        ":prime_proto",
+        ":prime_proto_pb2",
     ],
     default_python_version = "PY3",
 )
@@ -40,7 +45,7 @@ py_binary(
     srcs = ["server.py"],
     deps = [
         "//src/python/grpcio/grpc:grpcio",
-        ":prime_proto"
+        ":prime_proto_pb2"
     ] + select({
         "//conditions:default": [requirement("futures")],
         "//:python3": [],

examples/python/wait_for_ready/wait_for_ready_example.py

@@ -22,8 +22,8 @@ import threading
 
 import grpc
 
-from examples.protos import helloworld_pb2
-from examples.protos import helloworld_pb2_grpc
+from examples import helloworld_pb2
+from examples import helloworld_pb2_grpc
 
 _LOGGER = logging.getLogger(__name__)
 _LOGGER.setLevel(logging.INFO)
@@ -33,10 +33,13 @@ _ONE_DAY_IN_SECONDS = 60 * 60 * 24
 
 @contextmanager
 def get_free_loopback_tcp_port():
-    tcp_socket = socket.socket(socket.AF_INET6)
+    if socket.has_ipv6:
+        tcp_socket = socket.socket(socket.AF_INET6)
+    else:
+        tcp_socket = socket.socket(socket.AF_INET)
     tcp_socket.bind(('', 0))
     address_tuple = tcp_socket.getsockname()
-    yield "[::1]:%s" % (address_tuple[1])
+    yield "localhost:%s" % (address_tuple[1])
     tcp_socket.close()
 
 

src/proto/grpc/channelz/BUILD

@@ -25,6 +25,11 @@ grpc_proto_library(
     well_known_protos = True,
 )
 
+proto_library(
+    name = "channelz_proto_descriptors",
+    srcs = ["channelz.proto"],
+)
+
 filegroup(
     name = "channelz_proto_file",
     srcs = [

src/proto/grpc/health/v1/BUILD

@@ -23,6 +23,11 @@ grpc_proto_library(
     srcs = ["health.proto"],
 )
 
+proto_library(
+    name = "health_proto_descriptor",
+    srcs = ["health.proto"],
+)
+
 filegroup(
     name = "health_proto_file",
     srcs = [

src/proto/grpc/reflection/v1alpha/BUILD

@@ -23,6 +23,11 @@ grpc_proto_library(
     srcs = ["reflection.proto"],
 )
 
+proto_library(
+    name = "reflection_proto_descriptor",
+    srcs = ["reflection.proto"],
+)
+
 filegroup(
     name = "reflection_proto_file",
     srcs = [

src/proto/grpc/testing/BUILD

@@ -16,7 +16,7 @@ licenses(["notice"])  # Apache v2
 
 load("//bazel:grpc_build_system.bzl", "grpc_proto_library", "grpc_package")
 load("@grpc_python_dependencies//:requirements.bzl", "requirement")
-load("@org_pubref_rules_protobuf//python:rules.bzl", "py_proto_library")
+load("//bazel:python_rules.bzl", "py_proto_library")
 
 grpc_package(name = "testing", visibility = "public")
 
@@ -61,13 +61,14 @@ grpc_proto_library(
     has_services = False,
 )
 
+proto_library(
+    name = "empty_proto_descriptor",
+    srcs = ["empty.proto"],
+)
+
 py_proto_library(
     name = "py_empty_proto",
-    protos = ["empty.proto",],
-    with_grpc = True,
-    deps = [
-        requirement('protobuf'),
-    ],
+    deps = [":empty_proto_descriptor"],
 )
 
 grpc_proto_library(
@@ -76,13 +77,14 @@ grpc_proto_library(
     has_services = False,
 )
 
+proto_library(
+    name = "messages_proto_descriptor",
+    srcs = ["messages.proto"],
+)
+
 py_proto_library(
     name = "py_messages_proto",
-    protos = ["messages.proto",],
-    with_grpc = True,
-    deps = [
-        requirement('protobuf'),
-    ],
+    deps = [":messages_proto_descriptor"],
 )
 
 grpc_proto_library(
@@ -144,16 +146,19 @@ grpc_proto_library(
     ],
 )
 
+proto_library(
+    name = "test_proto_descriptor",
+    srcs = ["test.proto"],
+    deps = [
+        ":empty_proto_descriptor",
+        ":messages_proto_descriptor",
+    ],
+)
+
 py_proto_library(
     name = "py_test_proto",
-    protos = ["test.proto",],
-    with_grpc = True,
     deps = [
-        requirement('protobuf'),
-    ],
-    proto_deps = [
-        ":py_empty_proto",
-        ":py_messages_proto",
+        ":test_proto_descriptor",
     ]
 )
 

src/proto/grpc/testing/proto2/BUILD.bazel

@@ -1,30 +1,32 @@
 load("@grpc_python_dependencies//:requirements.bzl", "requirement")
-load("@org_pubref_rules_protobuf//python:rules.bzl", "py_proto_library")
 
 package(default_visibility = ["//visibility:public"])
+load("//bazel:python_rules.bzl", "py_proto_library")
+
+proto_library(
+    name = "empty2_proto_descriptor",
+    srcs = ["empty2.proto"],
+)
 
 py_proto_library(
     name = "empty2_proto",
-    protos = [
-        "empty2.proto",
-    ],
-    with_grpc = True,
     deps = [
-        requirement('protobuf'),
+        ":empty2_proto_descriptor",
     ],
 )
 
+proto_library(
+    name = "empty2_extensions_proto_descriptor",
+    srcs = ["empty2_extensions.proto"],
+    deps = [
+        ":empty2_proto_descriptor",
+    ]
+)
+
 py_proto_library(
     name = "empty2_extensions_proto",
-    protos = [
-        "empty2_extensions.proto",
-    ],
-    proto_deps = [
-        ":empty2_proto",
-    ],
-    with_grpc = True,
     deps = [
-        requirement('protobuf'),
+        ":empty2_extensions_proto_descriptor",
     ],
 )
 

src/python/grpcio_channelz/grpc_channelz/v1/BUILD.bazel

@@ -1,30 +1,10 @@
-load("@grpc_python_dependencies//:requirements.bzl", "requirement")
-load("@org_pubref_rules_protobuf//python:rules.bzl", "py_proto_library")
-
+load("//bazel:python_rules.bzl", "py_proto_library")
 package(default_visibility = ["//visibility:public"])
 
-genrule(
-    name = "mv_channelz_proto",
-    srcs = [
-        "//src/proto/grpc/channelz:channelz_proto_file",
-    ],
-    outs = ["channelz.proto",],
-    cmd = "cp $< $@",
-)
-
 py_proto_library(
     name = "py_channelz_proto",
-    protos = ["mv_channelz_proto",],
-    imports = [
-        "external/com_google_protobuf/src/",
-    ],
-    inputs = [
-        "@com_google_protobuf//:well_known_protos",
-    ],
-    with_grpc = True,
-    deps = [
-        requirement('protobuf'),
-    ],
+    well_known_protos = True,
+    deps = ["//src/proto/grpc/channelz:channelz_proto_descriptors"],
 )
 
 py_library(

src/python/grpcio_health_checking/grpc_health/v1/BUILD.bazel

@@ -1,24 +1,9 @@
-load("@grpc_python_dependencies//:requirements.bzl", "requirement")
-load("@org_pubref_rules_protobuf//python:rules.bzl", "py_proto_library")
-
+load("//bazel:python_rules.bzl", "py_proto_library")
 package(default_visibility = ["//visibility:public"])
 
-genrule(
-    name = "mv_health_proto",
-    srcs = [
-        "//src/proto/grpc/health/v1:health_proto_file",
-    ],
-    outs = ["health.proto",],
-    cmd = "cp $< $@",
-)
-
 py_proto_library(
     name = "py_health_proto",
-    protos = [":mv_health_proto",],
-    with_grpc = True,
-    deps = [
-        requirement('protobuf'),
-    ],
+    deps = ["//src/proto/grpc/health/v1:health_proto_descriptor",],
 )
 
 py_library(

src/python/grpcio_reflection/grpc_reflection/v1alpha/BUILD.bazel

@@ -1,24 +1,11 @@
+load("//bazel:python_rules.bzl", "py_proto_library")
 load("@grpc_python_dependencies//:requirements.bzl", "requirement")
-load("@org_pubref_rules_protobuf//python:rules.bzl", "py_proto_library")
 
 package(default_visibility = ["//visibility:public"])
 
-genrule(
-    name = "mv_reflection_proto",
-    srcs = [
-        "//src/proto/grpc/reflection/v1alpha:reflection_proto_file",
-    ],
-    outs = ["reflection.proto",],
-    cmd = "cp $< $@",
-)
-
 py_proto_library(
     name = "py_reflection_proto",
-    protos = [":mv_reflection_proto",],
-    with_grpc = True,
-    deps = [
-        requirement('protobuf'),
-    ],
+    deps = ["//src/proto/grpc/reflection/v1alpha:reflection_proto_descriptor",],
 )
 
 py_library(

src/python/grpcio_tests/tests/reflection/BUILD.bazel

@@ -14,6 +14,7 @@ py_test(
         "//src/python/grpcio_tests/tests/unit:test_common",
         "//src/proto/grpc/testing:py_empty_proto",
         "//src/proto/grpc/testing/proto2:empty2_extensions_proto",
+        "//src/proto/grpc/testing/proto2:empty2_proto",
         requirement('protobuf'),
     ],
     imports=["../../",],

tools/distrib/bazel_style.cfg

@@ -0,0 +1,4 @@
+[style]
+based_on_style = google
+allow_split_before_dict_value = False
+spaces_around_default_or_named_assign = True

tools/distrib/format_bazel.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+# Copyright 2019 The gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set=-ex
+
+VIRTUAL_ENV=bazel_format_virtual_environment
+
+CONFIG_PATH="$(dirname ${0})/bazel_style.cfg"
+
+python -m virtualenv ${VIRTUAL_ENV}
+PYTHON=${VIRTUAL_ENV}/bin/python
+"$PYTHON" -m pip install --upgrade pip==10.0.1
+"$PYTHON" -m pip install --upgrade futures
+"$PYTHON" -m pip install yapf==0.20.0
+
+pushd "$(dirname "${0}")/../.."
+FILES=$(find . -path ./third_party -prune -o -name '*.bzl' -print)
+echo "${FILES}" | xargs "$PYTHON" -m yapf -i --style="${CONFIG_PATH}"
+
+if ! which buildifier &>/dev/null; then
+    echo 'buildifer must be installed.' >/dev/stderr
+    exit 1
+fi
+
+echo "${FILES}" | xargs buildifier --type=bzl
+
+popd

tools/internal_ci/linux/grpc_bazel_build_in_docker.sh

@@ -24,5 +24,4 @@ git clone /var/local/jenkins/grpc /var/local/git/grpc
 && git submodule update --init --reference /var/local/jenkins/grpc/${name} \
 ${name}')
 cd /var/local/git/grpc
-#TODO(yfen): add back examples/... to build targets once python rules issues are resolved
-bazel build --spawn_strategy=standalone --genrule_strategy=standalone :all test/...
+bazel build --spawn_strategy=standalone --genrule_strategy=standalone :all test/... examples/...

tools/internal_ci/linux/grpc_python_bazel_test_in_docker.sh

@@ -23,10 +23,9 @@ git clone /var/local/jenkins/grpc /var/local/git/grpc
 (cd /var/local/jenkins/grpc/ && git submodule foreach 'cd /var/local/git/grpc \
 && git submodule update --init --reference /var/local/jenkins/grpc/${name} \
 ${name}')
-#TODO(yfen): temporarily disabled all python bazel tests due to incompatibility with bazel 0.23.2
-#cd /var/local/git/grpc/test
-#bazel test --spawn_strategy=standalone --genrule_strategy=standalone --test_output=errors //src/python/...
-#bazel test --spawn_strategy=standalone --genrule_strategy=standalone --test_output=errors //examples/python/...
-#bazel clean --expunge
-#bazel test --config=python3 --spawn_strategy=standalone --genrule_strategy=standalone --test_output=errors //src/python/...
-#bazel test --config=python3 --spawn_strategy=standalone --genrule_strategy=standalone --test_output=errors //examples/python/...
+cd /var/local/git/grpc/test
+bazel test --spawn_strategy=standalone --genrule_strategy=standalone --test_output=errors //src/python/...
+bazel test --spawn_strategy=standalone --genrule_strategy=standalone --test_output=errors //examples/python/...
+bazel clean --expunge
+bazel test --config=python3 --spawn_strategy=standalone --genrule_strategy=standalone --test_output=errors //src/python/...
+bazel test --config=python3 --spawn_strategy=standalone --genrule_strategy=standalone --test_output=errors //examples/python/...