Merge pull request #24732 from lidizheng/fix-new-cygwin

Hopefully fixed 27 build on winserver2016

src/python/grpcio/commands.py

@@ -218,8 +218,6 @@ class BuildExt(build_ext.build_ext):
             when invoked in C mode. GCC is okay with this, while clang is not.
             """
             try:
-                if platform.system() == 'Windows':
-                    return False
                 # TODO(lidiz) Remove the generated a.out for success tests.
                 cc_test = subprocess.Popen(['cc', '-x', 'c', '-std=c++11', '-'],
                                            stdin=subprocess.PIPE,
@@ -228,7 +226,8 @@ class BuildExt(build_ext.build_ext):
                 _, cc_err = cc_test.communicate(input=b'int main(){return 0;}')
                 return not 'invalid argument' in str(cc_err)
             except:
-                sys.stderr.write(traceback.format_exc() + '\n')
+                sys.stderr.write('Non-fatal exception:' +
+                                 traceback.format_exc() + '\n')
                 return False
 
         # This special conditioning is here due to difference of compiler

tools/internal_ci/helper_scripts/install_python_interpreters.ps1

@@ -85,7 +85,7 @@ $Python39x86Config = @{
     PythonInstallPath = "C:\Python39_32bit"
     PythonInstallerHash = "4a2812db8ab9f2e522c96c7728cfcccb"
 }
-# Install-Python @Python39x86Config
+Install-Python @Python39x86Config
 
 $Python39x64Config = @{
     PythonVersion = "3.9.0"
@@ -93,4 +93,4 @@ $Python39x64Config = @{
     PythonInstallPath = "C:\Python39"
     PythonInstallerHash = "b61a33dc28f13b561452f3089c87eb63"
 }
-# Install-Python @Python39x64Config
+Install-Python @Python39x64Config

tools/internal_ci/helper_scripts/prepare_build_windows.bat

@@ -36,16 +36,16 @@ python -m pip install google-api-python-client || goto :error
 powershell -File src\csharp\install_dotnet_sdk.ps1 || goto :error
 set PATH=%LOCALAPPDATA%\Microsoft\dotnet;%PATH%
 
-@rem Install Python interpreters
-@rem NOTE(lidiz): Python installer process may live longer than expected, and
-@rem has other side effects. It needs to be installed last to reduce impact.
-powershell -File tools\internal_ci\helper_scripts\install_python_interpreters.ps1 || goto :error
-
 @rem Disable some unwanted dotnet options
 set NUGET_XMLDOC_MODE=skip
 set DOTNET_SKIP_FIRST_TIME_EXPERIENCE=true
 set DOTNET_CLI_TELEMETRY_OPTOUT=true
 
+@rem Only install Python interpreters if we are running Python tests
+If "%PREPARE_BUILD_INSTALL_DEPS_PYTHON%" == "true" (
+    powershell -File tools\internal_ci\helper_scripts\install_python_interpreters.ps1 || goto :error
+)
+
 git submodule update --init || goto :error
 
 goto :EOF

tools/internal_ci/windows/grpc_build_artifacts.bat

@@ -19,6 +19,7 @@ choco install nasm -y --limit-output
 @rem enter repo root
 cd /d %~dp0\..\..\..
 
+set PREPARE_BUILD_INSTALL_DEPS_PYTHON=true
 call tools/internal_ci/helper_scripts/prepare_build_windows.bat || exit /b 1
 
 python tools/run_tests/task_runner.py -f artifact windows -j 4

tools/internal_ci/windows/grpc_build_artifacts.cfg

@@ -16,7 +16,7 @@
 
 # Location of the continuous shell script in repository.
 build_file: "grpc/tools/internal_ci/windows/grpc_build_artifacts.bat"
-timeout_mins: 120
+timeout_mins: 180
 action {
   define_artifacts {
     regex: "**/*sponge_log.*"

tools/internal_ci/windows/grpc_run_tests_matrix.bat

@@ -15,6 +15,9 @@
 @rem enter repo root
 cd /d %~dp0\..\..\..
 
+If Not "%RUN_TESTS_FLAGS%"=="%RUN_TESTS_FLAGS:python=%" (
+    set PREPARE_BUILD_INSTALL_DEPS_PYTHON=true
+)
 call tools/internal_ci/helper_scripts/prepare_build_windows.bat || exit /b 1
 
 python tools/run_tests/run_tests_matrix.py %RUN_TESTS_FLAGS%

tools/internal_ci/windows/pull_request/grpc_basictests_python.cfg

@@ -16,7 +16,7 @@
 
 # Location of the continuous shell script in repository.
 build_file: "grpc/tools/internal_ci/windows/grpc_run_tests_matrix.bat"
-timeout_mins: 60
+timeout_mins: 90
 action {
   define_artifacts {
     regex: "**/*sponge_log.*"
@@ -26,5 +26,5 @@ action {
 
 env_vars {
   key: "RUN_TESTS_FLAGS"
-  value: "-f basictests windows python -j 1 --inner_jobs 8 --internal_ci --max_time=3600"
+  value: "-f basictests windows python -j 1 --inner_jobs 8 --internal_ci --max_time=5400"
 }

tools/run_tests/artifacts/artifact_targets.py

@@ -400,15 +400,13 @@ def targets():
         PythonArtifact('windows', 'x86', 'Python36_32bit'),
         PythonArtifact('windows', 'x86', 'Python37_32bit'),
         PythonArtifact('windows', 'x86', 'Python38_32bit'),
-        # TODO(lidiz) uncomment if Python39 installs stably.
-        # PythonArtifact('windows', 'x86', 'Python39_32bit'),
+        PythonArtifact('windows', 'x86', 'Python39_32bit'),
         PythonArtifact('windows', 'x64', 'Python27'),
         PythonArtifact('windows', 'x64', 'Python35'),
         PythonArtifact('windows', 'x64', 'Python36'),
         PythonArtifact('windows', 'x64', 'Python37'),
         PythonArtifact('windows', 'x64', 'Python38'),
-        # TODO(lidiz) uncomment if Python39 installs stably.
-        # PythonArtifact('windows', 'x64', 'Python39'),
+        PythonArtifact('windows', 'x64', 'Python39'),
         RubyArtifact('linux', 'x64'),
         RubyArtifact('macos', 'x64'),
         PHPArtifact('linux', 'x64')

tools/run_tests/artifacts/build_artifact_python.bat

@@ -12,8 +12,14 @@
 @rem See the License for the specific language governing permissions and
 @rem limitations under the License.
 
-@rem set path to python & mingw compiler
-set PATH=C:\%1;C:\%1\scripts;C:\msys64\mingw%2\bin;C:\tools\msys64\mingw%2\bin;%PATH%
+@rem set path to python
+set PATH=C:\%1;C:\%1\scripts;%PATH%
+
+if "%1" == "Python27" goto :install_mingw64_with_msvcr90
+if "%1" == "Python27_32bit" goto :install_mingw64_with_msvcr90
+@rem set path to the existed mingw compiler
+set PATH=C:\msys64\mingw%2\bin;C:\tools\msys64\mingw%2\bin;%PATH%
+:end_mingw64_installation
 
 python -m pip install --upgrade six
 @rem some artifacts are broken for setuptools 38.5.0. See https://github.com/grpc/grpc/issues/14317
@@ -21,11 +27,12 @@ python -m pip install --upgrade setuptools==44.1.1
 python -m pip install --upgrade cython
 python -m pip install -rrequirements.txt --user
 
+@rem set GRPC_PYTHON_OVERRIDE_CYGWIN_DETECTION_FOR_27=1
 set GRPC_PYTHON_BUILD_WITH_CYTHON=1
 
 @rem Allow build_ext to build C/C++ files in parallel
 @rem by enabling a monkeypatch. It speeds up the build a lot.
-set GRPC_PYTHON_BUILD_EXT_COMPILER_JOBS=2
+set GRPC_PYTHON_BUILD_EXT_COMPILER_JOBS=4
 
 mkdir -p %ARTIFACTS_OUT%
 set ARTIFACT_DIR=%cd%\%ARTIFACTS_OUT%
@@ -59,3 +66,13 @@ goto :EOF
 :error
 popd
 exit /b 1
+
+:install_mingw64_with_msvcr90
+set MSYS64_DOWNLOAD_URL=https://storage.googleapis.com/grpc-build-helper/msys64.zip
+set MSYS64_PATH=C:\tools\msys64_win7
+set PATH=%MSYS64_PATH%\mingw%2\bin;%PATH%
+@rem Skip the installation if the directory exists
+if exist "%MSYS64_PATH%" goto :end_mingw64_installation
+python -m pip install requests || goto :error
+python tools\run_tests\python_utils\download_and_unzip.py "%MSYS64_DOWNLOAD_URL%" "%MSYS64_PATH%" || goto :error
+goto :end_mingw64_installation

tools/run_tests/python_utils/download_and_unzip.py

@@ -0,0 +1,47 @@
+# Copyright 2020 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Download and unzip the target file to the destination."""
+
+from __future__ import print_function
+
+import os
+import sys
+import zipfile
+import requests
+import tempfile
+
+
+def main():
+    if len(sys.argv) != 3:
+        print("Usage: python download_and_unzip.py [zipfile-url] [destination]")
+        sys.exit(1)
+    download_url = sys.argv[1]
+    destination = sys.argv[2]
+
+    with tempfile.TemporaryFile() as tmp_file:
+        r = requests.get(download_url)
+        if r.status_code != requests.codes.ok:
+            print("Download %s failed with [%d] \"%s\"" %
+                  (download_url, r.status_code, r.text()))
+            sys.exit(1)
+        else:
+            tmp_file.write(r.content)
+            print("Successfully downloaded from %s", download_url)
+        with zipfile.ZipFile(tmp_file, 'r') as target_zip_file:
+            target_zip_file.extractall(destination)
+        print("Successfully unzip to %s" % destination)
+
+
+if __name__ == "__main__":
+    main()