Merge pull request #23243 from gnossen/setup_py_improvements

Incrementally Improve setup.py

setup.py

@@ -383,6 +383,7 @@ COMMAND_CLASS = {
     'build_py': commands.BuildPy,
     'build_ext': commands.BuildExt,
     'gather': commands.Gather,
+    'clean': commands.Clean,
 }
 
 # Ensure that package data is copied over before any commands have been run:

src/python/grpcio/_parallel_compile_patch.py

@@ -22,7 +22,10 @@ import os
 
 try:
     BUILD_EXT_COMPILER_JOBS = int(
-        os.environ.get('GRPC_PYTHON_BUILD_EXT_COMPILER_JOBS', '1'))
+        os.environ['GRPC_PYTHON_BUILD_EXT_COMPILER_JOBS'])
+except KeyError:
+    import multiprocessing
+    BUILD_EXT_COMPILER_JOBS = multiprocessing.cpu_count()
 except ValueError:
     BUILD_EXT_COMPILER_JOBS = 1
 

src/python/grpcio/commands.py

@@ -13,6 +13,8 @@
 # limitations under the License.
 """Provides distutils command classes for the GRPC Python setup process."""
 
+from __future__ import print_function
+
 import distutils
 import glob
 import os
@@ -290,3 +292,41 @@ class Gather(setuptools.Command):
                 self.distribution.install_requires)
         if self.test and self.distribution.tests_require:
             self.distribution.fetch_build_eggs(self.distribution.tests_require)
+
+
+class Clean(setuptools.Command):
+    """Command to clean build artifacts."""
+
+    description = 'Clean build artifacts.'
+    user_options = []
+
+    _FILE_PATTERNS = (
+        'python_build',
+        'src/python/grpcio/__pycache__/',
+        'src/python/grpcio/grpc/_cython/cygrpc.cpp',
+        'src/python/grpcio/grpc/_cython/*.so',
+        'src/python/grpcio/grpcio.egg-info/',
+    )
+    _CURRENT_DIRECTORY = os.path.normpath(
+        os.path.join(os.path.dirname(os.path.realpath(__file__)), "../../.."))
+
+    def initialize_options(self):
+        pass
+
+    def finalize_options(self):
+        pass
+
+    def run(self):
+        for path_spec in self._FILE_PATTERNS:
+            this_glob = os.path.normpath(
+                os.path.join(Clean._CURRENT_DIRECTORY, path_spec))
+            abs_paths = glob.glob(this_glob)
+            for path in abs_paths:
+                if not str(path).startswith(Clean._CURRENT_DIRECTORY):
+                    raise ValueError(
+                        "Cowardly refusing to delete {}.".format(path))
+                print("Removing {}".format(os.path.relpath(path)))
+                if os.path.isfile(path):
+                    os.remove(str(path))
+                else:
+                    shutil.rmtree(str(path))

tools/distrib/install_all_python_modules.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+# Copyright 2020 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+echo "It's recommended that you run this script from a virtual environment."
+
+set -e
+
+BASEDIR=$(dirname "$0")
+BASEDIR=$(realpath "$BASEDIR")/../..
+
+(cd "$BASEDIR";
+  pip install --upgrade cython;
+  python setup.py install;
+  pushd tools/distrib/python/grpcio_tools;
+    ../make_grpcio_tools.py
+    GRPC_PYTHON_BUILD_WITH_CYTHON=1 pip install .
+  popd;
+  pushd src/python;
+    for PACKAGE in ./grpcio_*; do
+      pushd "${PACKAGE}";
+        python setup.py preprocess;
+        python setup.py install;
+      popd;
+    done
+  popd;
+)