Add several fixes for python toolchain (#5667)

* Add several fixes for python toolchain

* Fix versin regex

* Make script exit on error

* Fix version regex

* Fix version regex

* Fix version regex

* Fix version regex

* Make test run on the current commit

* Fix test

* Fix test

* Use git to retrieve current commit

* Fix tests

* Fix tests

* Also make linux and mac work on the current commit

* Fix test

cmake/CMakeLists.txt

@@ -68,23 +68,28 @@ string(REGEX REPLACE        "${protobuf_AC_INIT_REGEX}" "\\2"
 string(REGEX REPLACE        "${protobuf_AC_INIT_REGEX}" "\\3"
     protobuf_CONTACT        "${protobuf_AC_INIT_LINE}")
 # Parse version tweaks
-set(protobuf_VERSION_REGEX "^([0-9]+)\\.([0-9]+)\\.([0-9]+)-?(.*)$")
+set(protobuf_VERSION_REGEX "^([0-9]+)\\.([0-9]+)\\.([0-9]+)([-]rc[-]|\\.)?([0-9]*)$")
 string(REGEX REPLACE     "${protobuf_VERSION_REGEX}" "\\1"
   protobuf_VERSION_MAJOR "${protobuf_VERSION_STRING}")
 string(REGEX REPLACE     "${protobuf_VERSION_REGEX}" "\\2"
   protobuf_VERSION_MINOR "${protobuf_VERSION_STRING}")
 string(REGEX REPLACE     "${protobuf_VERSION_REGEX}" "\\3"
   protobuf_VERSION_PATCH "${protobuf_VERSION_STRING}")
-string(REGEX REPLACE     "${protobuf_VERSION_REGEX}" "\\4"
+string(REGEX REPLACE     "${protobuf_VERSION_REGEX}" "\\5"
   protobuf_VERSION_PRERELEASE "${protobuf_VERSION_STRING}")
 
+message(STATUS "${protobuf_VERSION_PRERELEASE}")
+
 # Package version
 set(protobuf_VERSION
   "${protobuf_VERSION_MAJOR}.${protobuf_VERSION_MINOR}.${protobuf_VERSION_PATCH}")
 
 if(protobuf_VERSION_PRERELEASE)
-  set(protobuf_VERSION "${protobuf_VERSION}-${protobuf_VERSION_PRERELEASE}")
+  set(protobuf_VERSION "${protobuf_VERSION}.${protobuf_VERSION_PRERELEASE}")
+else()
+  set(protobuf_VERSION "${protobuf_VERSION}.0")
 endif()
+message(STATUS "${protobuf_VERSION}")
 
 if(protobuf_VERBOSE)
   message(STATUS "Configuration script parsing status [")

cmake/version.rc.in

@@ -15,8 +15,8 @@
 
 
 VS_VERSION_INFO VERSIONINFO
-  FILEVERSION    @protobuf_RC_FILEVERSION@,0
-  PRODUCTVERSION @protobuf_RC_FILEVERSION@,0
+  FILEVERSION    @protobuf_RC_FILEVERSION@
+  PRODUCTVERSION @protobuf_RC_FILEVERSION@
   FILEFLAGSMASK  VS_FFI_FILEFLAGSMASK
   FILEFLAGS      VER_DEBUG
   FILEOS         VOS__WINDOWS32

kokoro/release/python/linux/build_artifacts.sh

@@ -5,13 +5,17 @@ set -ex
 # change to repo root
 pushd $(dirname $0)/../../../..
 
+# Create stage dir
+ORIGINAL_DIR=`pwd`
+pushd ..
+cp -R $ORIGINAL_DIR stage
+export STAGE_DIR="`pwd`/stage"
+popd
+
 export REPO_DIR=protobuf
 export BUILD_VERSION=`grep -i "version" python/google/protobuf/__init__.py | grep -o "'.*'" | tr -d "'"`
-if [ -z $KOKORO_JOB_NAME ]; then
-  export BUILD_COMMIT=master
-else
-  export BUILD_COMMIT=`echo "$KOKORO_JOB_NAME" | cut -d '/' -f 3`
-fi
+
+export BUILD_COMMIT=`git rev-parse HEAD`
 export PLAT=x86_64
 export UNICODE_WIDTH=32
 export MACOSX_DEPLOYMENT_TARGET=10.9
@@ -29,8 +33,8 @@ build_artifact_version() {
 
   # Clean up env
   rm -rf venv
-  sudo rm -rf protobuf
-  git clone https://github.com/google/protobuf.git
+  sudo rm -rf $REPO_DIR
+  cp -R $STAGE_DIR $REPO_DIR
 
   source multibuild/common_utils.sh
   source multibuild/travis_steps.sh

kokoro/release/python/macos/build_artifacts.sh

@@ -5,13 +5,16 @@ set -ex
 # change to repo root
 pushd $(dirname $0)/../../../..
 
+# Create stage dir
+ORIGINAL_DIR=`pwd`
+pushd ..
+cp -R $ORIGINAL_DIR stage
+export STAGE_DIR="`pwd`/stage"
+popd
+
 export REPO_DIR=protobuf
 export BUILD_VERSION=`grep -i "version" python/google/protobuf/__init__.py | grep -o "'.*'" | tr -d "'"`
-if [ -z $KOKORO_JOB_NAME ]; then
-  export BUILD_COMMIT=master
-else
-  export BUILD_COMMIT=`echo "$KOKORO_JOB_NAME" | cut -d '/' -f 3`
-fi
+export BUILD_COMMIT=`git rev-parse HEAD`
 export PLAT=x86_64
 export UNICODE_WIDTH=32
 export MACOSX_DEPLOYMENT_TARGET=10.9
@@ -32,8 +35,8 @@ build_artifact_version() {
 
   # Clean up env
   rm -rf venv
-  sudo rm -rf protobuf
-  git clone https://github.com/google/protobuf.git
+  sudo rm -rf $REPO_DIR
+  cp -R $STAGE_DIR $REPO_DIR
   export PATH=$OLD_PATH
 
   source multibuild/common_utils.sh

kokoro/release/python/windows/build_artifacts.bat

@@ -1,17 +1,17 @@
 REM Move scripts to root
+set REPO_DIR_STAGE=%cd%\github\protobuf-stage
+xcopy /S  github\protobuf "%REPO_DIR_STAGE%\"
 cd github\protobuf
 copy kokoro\release\python\windows\build_single_artifact.bat build_single_artifact.bat
 
 REM Set environment variables
-set REPO_DIR=protobuf
 set PACKAGE_NAME=protobuf
+set REPO_DIR=protobuf
 set BUILD_DLL=OFF
 set UNICODE=ON
 set PB_TEST_DEP="six==1.9"
 set OTHER_TEST_DEP="setuptools==38.5.1"
 set OLD_PATH=C:\Program Files (x86)\MSBuild\14.0\bin\;%PATH%
-for /f "tokens=*" %%i in ( 'grep -i "version" python/google/protobuf/__init__.py ^| grep -o "'.*'"' ) do set BUILD_VERSION=%%i
-set BUILD_COMMIT=v%BUILD_VERSION:'=%
 
 REM Fetch multibuild
 git clone https://github.com/matthew-brett/multibuild.git
@@ -35,29 +35,34 @@ REM Build wheel
 SET PYTHON=C:\python35_32bit
 SET PYTHON_VERSION=3.5
 SET PYTHON_ARCH=32
-CALL build_single_artifact.bat
+CALL build_single_artifact.bat || goto :error
 
 SET PYTHON=C:\python35
 SET PYTHON_VERSION=3.5
 SET PYTHON_ARCH=64
-CALL build_single_artifact.bat
+CALL build_single_artifact.bat || goto :error
 
 SET PYTHON=C:\python36_32bit
 SET PYTHON_VERSION=3.6
 SET PYTHON_ARCH=32
-CALL build_single_artifact.bat
+CALL build_single_artifact.bat || goto :error
 
 SET PYTHON=C:\python36
 SET PYTHON_VERSION=3.6
 SET PYTHON_ARCH=64
-CALL build_single_artifact.bat
+CALL build_single_artifact.bat || goto :error
 
 SET PYTHON=C:\python37_32bit
 SET PYTHON_VERSION=3.7
 SET PYTHON_ARCH=32
-CALL build_single_artifact.bat
+CALL build_single_artifact.bat || goto :error
 
 SET PYTHON=C:\python37
 SET PYTHON_VERSION=3.7
 SET PYTHON_ARCH=64
-CALL build_single_artifact.bat
+CALL build_single_artifact.bat || goto :error
+
+goto :EOF
+
+:error
+exit /b %errorlevel%

kokoro/release/python/windows/build_single_artifact.bat

@@ -12,6 +12,12 @@ if %PYTHON%==C:\python36_32bit set vcplatform=Win32
 if %PYTHON%==C:\python36 set generator=Visual Studio 14 Win64
 if %PYTHON%==C:\python36 set vcplatform=x64
 
+if %PYTHON%==C:\python37_32bit set generator=Visual Studio 14
+if %PYTHON%==C:\python37_32bit set vcplatform=Win32
+
+if %PYTHON%==C:\python37 set generator=Visual Studio 14 Win64
+if %PYTHON%==C:\python37 set vcplatform=x64
+
 REM Prepend newly installed Python to the PATH of this build (this cannot be
 REM done from inside the powershell script as it would require to restart
 REM the parent CMD process).
@@ -23,12 +29,11 @@ REM Check that we have the expected version and architecture for Python
 python --version
 python -c "import struct; print(struct.calcsize('P') * 8)"
 
-rmdir /s/q protobuf
-git clone https://github.com/google/protobuf.git
+rmdir /s/q %REPO_DIR%
+xcopy /s  %REPO_DIR_STAGE% "%REPO_DIR%\"
 
 REM Checkout release commit
 cd %REPO_DIR%
-git checkout %BUILD_COMMIT%
 
 REM ======================
 REM Build Protobuf Library
@@ -38,8 +43,8 @@ mkdir src\.libs
 
 mkdir vcprojects
 pushd vcprojects
-cmake -G "%generator%" -Dprotobuf_BUILD_SHARED_LIBS=%BUILD_DLL% -Dprotobuf_UNICODE=%UNICODE% -Dprotobuf_BUILD_TESTS=OFF ../cmake
-msbuild protobuf.sln /p:Platform=%vcplatform% /p:Configuration=Release
+cmake -G "%generator%" -Dprotobuf_BUILD_SHARED_LIBS=%BUILD_DLL% -Dprotobuf_UNICODE=%UNICODE% -Dprotobuf_BUILD_TESTS=OFF ../cmake || goto :error
+msbuild protobuf.sln /p:Platform=%vcplatform% /p:Configuration=Release || goto :error
 dir /s /b
 popd
 copy vcprojects\Release\libprotobuf.lib src\.libs\libprotobuf.a
@@ -60,3 +65,8 @@ dir dist
 copy dist\* %ARTIFACT_DIR%
 dir %ARTIFACT_DIR%
 cd ..\..
+
+goto :EOF
+
+:error
+exit /b %errorlevel%

kokoro/release/python/windows/continuous.cfg

@@ -0,0 +1,8 @@
+# Configuration for Windows protoc release builds
+build_file: "protobuf/kokoro/release/python/windows/build_artifacts.bat"
+
+action {
+  define_artifacts {
+    regex: "github/protobuf/artifacts/**"
+  }
+}

kokoro/release/python/windows/presubmit.cfg

@@ -0,0 +1,8 @@
+# Configuration for Windows protoc release builds
+build_file: "protobuf/kokoro/release/python/windows/build_artifacts.bat"
+
+action {
+  define_artifacts {
+    regex: "github/protobuf/artifacts/**"
+  }
+}

python/release.sh

@@ -84,13 +84,13 @@ python setup.py sdist upload -r https://test.pypi.org/legacy/
 
 # Test locally with different python versions.
 run_install_test ${TESTING_VERSION} python2.7 https://test.pypi.org/simple
-run_install_test ${TESTING_VERSION} python3.4 https://test.pypi.org/simple
+run_install_test ${TESTING_VERSION} python3 https://test.pypi.org/simple
 
 # Deploy egg/wheel packages to testing PyPI and test again.
 python setup.py bdist_egg bdist_wheel upload -r https://test.pypi.org/legacy/
 
 run_install_test ${TESTING_VERSION} python2.7 https://test.pypi.org/simple
-run_install_test ${TESTING_VERSION} python3.4 https://test.pypi.org/simple
+run_install_test ${TESTING_VERSION} python3 https://test.pypi.org/simple
 
 echo "All install tests have passed using testing PyPI."