Merge pull request #4924 from xfxyjwf/3.6.x

Cherry-pick master build fixes and kokoro changes to 3.6.x

.gitignore

@@ -47,7 +47,6 @@ any_test.pb.*
 map*unittest.pb.*
 unittest*.pb.*
 cpp_test*.pb.*
-src/google/protobuf/compiler/js/well_known_types_embed.cc
 src/google/protobuf/util/**/*.pb.cc
 src/google/protobuf/util/**/*.pb.h
 

BUILD

@@ -257,24 +257,6 @@ internal_copied_filegroup(
 # Protocol Buffers Compiler
 ################################################################################
 
-cc_binary(
-    name = "js_embed",
-    srcs = ["src/google/protobuf/compiler/js/embed.cc"],
-    visibility = ["//visibility:public"],
-)
-
-genrule(
-    name = "generate_js_well_known_types_embed",
-    srcs = [
-        "src/google/protobuf/compiler/js/well_known_types/any.js",
-        "src/google/protobuf/compiler/js/well_known_types/struct.js",
-        "src/google/protobuf/compiler/js/well_known_types/timestamp.js",
-    ],
-    outs = ["src/google/protobuf/compiler/js/well_known_types_embed.cc"],
-    cmd = "$(location :js_embed) $(SRCS) > $@",
-    tools = [":js_embed"],
-)
-
 cc_library(
     name = "protoc_lib",
     srcs = [

autogen.sh

@@ -28,6 +28,11 @@ fi
 
 set -ex
 
+# The absence of a m4 directory in googletest causes autoreconf to fail when
+# building under the CentOS docker image. It's a warning in regular build on
+# Ubuntu/gLinux as well.
+mkdir -p third_party/googletest/m4
+
 # TODO(kenton):  Remove the ",no-obsolete" part and fix the resulting warnings.
 autoreconf -f -i -Wall,no-obsolete
 

benchmarks/Makefile.am

@@ -109,6 +109,7 @@ cpp_benchmark_CPPFLAGS = -I$(top_srcdir)/src -I$(srcdir)/cpp -I$(top_srcdir)/thi
 # so a direct "make test_cpp" could fail if parallel enough.
 # See: https://www.gnu.org/software/automake/manual/html_node/Built-Sources-Example.html#Recording-Dependencies-manually
 cpp/cpp_benchmark-cpp_benchmark.$(OBJEXT): $(benchmarks_protoc_outputs) $(benchmarks_protoc_outputs_proto2) $(benchmarks_protoc_outputs_header) $(benchmarks_protoc_outputs_proto2_header) $(top_srcdir)/src/libprotobuf.la $(top_srcdir)/third_party/benchmark/src/libbenchmark.a
+cpp/benchmark-cpp_benchmark.$(OBJEXT): $(benchmarks_protoc_outputs) $(benchmarks_protoc_outputs_proto2) $(benchmarks_protoc_outputs_header) $(benchmarks_protoc_outputs_proto2_header) $(top_srcdir)/src/libprotobuf.la $(top_srcdir)/third_party/benchmark/src/libbenchmark.a
 nodist_cpp_benchmark_SOURCES =                                             \
 	$(benchmarks_protoc_outputs)                                             \
 	$(benchmarks_protoc_outputs_proto2)                                      \

cmake/libprotoc.cmake

@@ -161,17 +161,11 @@ set(libprotoc_headers
   ${protobuf_source_dir}/src/google/protobuf/compiler/zip_writer.h
 )
 
-set(js_well_known_types_sources
-  ${protobuf_source_dir}/src/google/protobuf/compiler/js/well_known_types/any.js
-  ${protobuf_source_dir}/src/google/protobuf/compiler/js/well_known_types/struct.js
-  ${protobuf_source_dir}/src/google/protobuf/compiler/js/well_known_types/timestamp.js
-)
-add_executable(js_embed ${protobuf_source_dir}/src/google/protobuf/compiler/js/embed.cc)
-add_custom_command(
-  OUTPUT ${protobuf_source_dir}/src/google/protobuf/compiler/js/well_known_types_embed.cc
-  DEPENDS js_embed ${js_well_known_types_sources}
-  COMMAND js_embed ${js_well_known_types_sources} > ${protobuf_source_dir}/src/google/protobuf/compiler/js/well_known_types_embed.cc
+if (MSVC)
+set(libprotoc_rc_files
+  ${CMAKE_CURRENT_BINARY_DIR}/version.rc
 )
+endif()
 
 add_library(libprotoc ${protobuf_SHARED_OR_STATIC}
   ${libprotoc_files} ${libprotoc_headers})

conformance/Makefile.am

@@ -279,8 +279,6 @@ $(protoc_outputs): protoc_middleman
 
 $(other_language_protoc_outputs): protoc_middleman
 
-BUILT_SOURCES = $(protoc_outputs) $(other_language_protoc_outputs)
-
 CLEANFILES = $(protoc_outputs) protoc_middleman javac_middleman conformance-java javac_middleman_lite conformance-java-lite conformance-csharp conformance-php conformance-php-c $(other_language_protoc_outputs)
 
 MAINTAINERCLEANFILES =   \

conformance/conformance_php.php

@@ -6,8 +6,8 @@ require_once("Conformance/ConformanceRequest.php");
 require_once("Protobuf_test_messages/Proto3/ForeignMessage.php");
 require_once("Protobuf_test_messages/Proto3/ForeignEnum.php");
 require_once("Protobuf_test_messages/Proto3/TestAllTypesProto3.php");
-require_once("Protobuf_test_messages/Proto3/TestAllTypesProto3_NestedMessage.php");
-require_once("Protobuf_test_messages/Proto3/TestAllTypesProto3_NestedEnum.php");
+require_once("Protobuf_test_messages/Proto3/TestAllTypesProto3/NestedMessage.php");
+require_once("Protobuf_test_messages/Proto3/TestAllTypesProto3/NestedEnum.php");
 
 require_once("GPBMetadata/Conformance.php");
 require_once("GPBMetadata/Google/Protobuf/TestMessagesProto3.php");

examples/list_people_test.go

@@ -34,7 +34,7 @@ func TestWritePersonWritesPerson(t *testing.T) {
 
 func TestListPeopleWritesList(t *testing.T) {
 	buf := new(bytes.Buffer)
-	in := pb.AddressBook{[]*pb.Person{
+	in := pb.AddressBook{People: []*pb.Person {
 		{
 			Name:  "John Doe",
 			Id:    101,

kokoro/linux/build_and_run_docker.sh

@@ -35,6 +35,8 @@ echo $git_root
 docker run \
   "$@" \
   -e CCACHE_DIR=$CCACHE_DIR \
+  -e KOKORO_BUILD_NUMBER=$KOKORO_BUILD_NUMBER \
+  -e KOKORO_BUILD_ID=$KOKORO_BUILD_ID \
   -e EXTERNAL_GIT_ROOT="/var/local/kokoro/protobuf" \
   -e TEST_SET="$TEST_SET" \
   -e THIS_IS_REALLY_NEEDED='see https://github.com/docker/docker/issues/14203 for why docker is awful' \

kokoro/linux/cpp_distcheck/build.sh

@@ -5,7 +5,15 @@
 # Change to repo root
 cd $(dirname $0)/../../..
 
-# Prepare worker environment to run tests
-source kokoro/linux/prepare_build_linux_rc
-
 ./tests.sh cpp_distcheck
+
+# Run tests under release docker image.
+DOCKER_IMAGE_NAME=protobuf/protoc_$(sha1sum protoc-artifacts/Dockerfile | cut -f1 -d " ")
+docker pull $DOCKER_IMAGE_NAME
+
+docker run -v $(pwd):/var/local/protobuf --rm $DOCKER_IMAGE_NAME \
+  bash -l /var/local/protobuf/tests.sh cpp || FAILED="true"
+
+if [ "$FAILED" = "true" ]; then
+  exit 1
+fi

kokoro/linux/pull_request_in_docker.sh

@@ -5,7 +5,7 @@
 
 WORKSPACE_BASE=`pwd`
 MY_DIR="$(dirname "$0")"
-TEST_SCRIPT=$MY_DIR/../../tests.sh
+TEST_SCRIPT=./tests.sh
 BUILD_DIR=/tmp/protobuf
 
 set -e  # exit immediately on error
@@ -53,7 +53,7 @@ $TIME_CMD $TEST_SCRIPT cpp > >(tee $CPP_STDOUT) 2> >(tee $CPP_STDERR >&2)
 
 parallel --results $LOG_OUTPUT_DIR --joblog $OUTPUT_DIR/joblog $TEST_SCRIPT ::: \
   $TEST_SET \
-  || true  # Process test results even if tests fail.
+  || FAILED="true"  # Process test results even if tests fail.
 
 cat $OUTPUT_DIR/joblog
 
@@ -67,3 +67,7 @@ TESTOUTPUT_XML_FILE=$COPY_FROM_DOCKER/sponge_log.xml
 python $MY_DIR/make_test_output.py $OUTPUT_DIR > $TESTOUTPUT_XML_FILE
 
 ls -l $TESTOUTPUT_XML_FILE
+
+if [ "$FAILED" == "true" ]; then
+	exit 1
+fi

kokoro/release/protoc/linux/build.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+set -x
+
+# Change to repo root.
+cd $(dirname $0)/../../../..
+
+# Initialize any submodules.
+git submodule update --init --recursive
+
+# Generate the configure script.
+./autogen.sh
+
+# Use docker image to build linux artifacts.
+DOCKER_IMAGE_NAME=protobuf/protoc_$(sha1sum protoc-artifacts/Dockerfile | cut -f1 -d " ")
+docker pull $DOCKER_IMAGE_NAME
+
+docker run -v $(pwd):/var/local/protobuf --rm $DOCKER_IMAGE_NAME \
+    bash -l /var/local/protobuf/protoc-artifacts/build-protoc.sh \
+    linux x86_64 protoc || {
+  echo "Failed to build protoc for linux + x86_64."
+  exit 1
+}
+
+docker run -v $(pwd):/var/local/protobuf --rm $DOCKER_IMAGE_NAME \
+    bash -l /var/local/protobuf/protoc-artifacts/build-protoc.sh \
+    linux x86_32 protoc || {
+  echo "Failed to build protoc for linux + x86_32."
+  exit 1
+}
+
+# Cross-build for some architectures.
+# TODO(xiaofeng): It currently fails with "machine `aarch64' not recognized"
+# error.
+# sudo apt install -y g++-aarch64-linux-gnu
+# protoc-artifacts/build-protoc.sh linux aarch_64 protoc

kokoro/release/protoc/linux/release.cfg

@@ -0,0 +1,7 @@
+build_file: "protobuf/kokoro/release/protoc/linux/build.sh"
+
+action {
+  define_artifacts {
+    regex: "**/protoc.exe"
+  }
+}

php/ext/google/protobuf/message.c

@@ -283,6 +283,7 @@ void build_class_from_descriptor(
 // -----------------------------------------------------------------------------
 
 void Message_construct(zval* msg, zval* array_wrapper) {
+  TSRMLS_FETCH();
   zend_class_entry* ce = Z_OBJCE_P(msg);
   MessageHeader* intern = NULL;
   if (EXPECTED(class_added(ce))) {

php/src/Google/Protobuf/Internal/MapField.php

@@ -156,15 +156,22 @@ class MapField implements \ArrayAccess, \IteratorAggregate, \Countable
         $this->checkKey($this->key_type, $key);
 
         switch ($this->value_type) {
+            case GPBType::SFIXED32:
+            case GPBType::SINT32:
             case GPBType::INT32:
+            case GPBType::ENUM:
                 GPBUtil::checkInt32($value);
                 break;
+            case GPBType::FIXED32:
             case GPBType::UINT32:
                 GPBUtil::checkUint32($value);
                 break;
+            case GPBType::SFIXED64:
+            case GPBType::SINT64:
             case GPBType::INT64:
                 GPBUtil::checkInt64($value);
                 break;
+            case GPBType::FIXED64:
             case GPBType::UINT64:
                 GPBUtil::checkUint64($value);
                 break;
@@ -249,36 +256,24 @@ class MapField implements \ArrayAccess, \IteratorAggregate, \Countable
     private function checkKey($key_type, &$key)
     {
         switch ($key_type) {
+            case GPBType::SFIXED32:
+            case GPBType::SINT32:
             case GPBType::INT32:
                 GPBUtil::checkInt32($key);
                 break;
+            case GPBType::FIXED32:
             case GPBType::UINT32:
                 GPBUtil::checkUint32($key);
                 break;
+            case GPBType::SFIXED64:
+            case GPBType::SINT64:
             case GPBType::INT64:
                 GPBUtil::checkInt64($key);
                 break;
-            case GPBType::UINT64:
-                GPBUtil::checkUint64($key);
-                break;
             case GPBType::FIXED64:
+            case GPBType::UINT64:
                 GPBUtil::checkUint64($key);
                 break;
-            case GPBType::FIXED32:
-                GPBUtil::checkUint32($key);
-                break;
-            case GPBType::SFIXED64:
-                GPBUtil::checkInt64($key);
-                break;
-            case GPBType::SFIXED32:
-                GPBUtil::checkInt32($key);
-                break;
-            case GPBType::SINT64:
-                GPBUtil::checkInt64($key);
-                break;
-            case GPBType::SINT32:
-                GPBUtil::checkInt32($key);
-                break;
             case GPBType::BOOL:
                 GPBUtil::checkBool($key);
                 break;

php/src/Google/Protobuf/Internal/RepeatedField.php

@@ -141,15 +141,22 @@ class RepeatedField implements \ArrayAccess, \IteratorAggregate, \Countable
     public function offsetSet($offset, $value)
     {
         switch ($this->type) {
+            case GPBType::SFIXED32:
+            case GPBType::SINT32:
             case GPBType::INT32:
+            case GPBType::ENUM:
                 GPBUtil::checkInt32($value);
                 break;
+            case GPBType::FIXED32:
             case GPBType::UINT32:
                 GPBUtil::checkUint32($value);
                 break;
+            case GPBType::SFIXED64:
+            case GPBType::SINT64:
             case GPBType::INT64:
                 GPBUtil::checkInt64($value);
                 break;
+            case GPBType::FIXED64:
             case GPBType::UINT64:
                 GPBUtil::checkUint64($value);
                 break;
@@ -162,6 +169,9 @@ class RepeatedField implements \ArrayAccess, \IteratorAggregate, \Countable
             case GPBType::BOOL:
                 GPBUtil::checkBool($value);
                 break;
+            case GPBType::BYTES:
+                GPBUtil::checkString($value, false);
+                break;
             case GPBType::STRING:
                 GPBUtil::checkString($value, true);
                 break;

php/tests/compatibility_test.sh

@@ -122,6 +122,8 @@ composer install
 tests=( array_test.php encode_decode_test.php generated_class_test.php map_field_test.php well_known_test.php )
 sed -i.bak '/php_implementation_test.php/d' phpunit.xml
 sed -i.bak '/generated_phpdoc_test.php/d' phpunit.xml
+sed -i.bak 's/generated_phpdoc_test.php//g' tests/test.sh
+sed -i.bak '/memory_leak_test.php/d' tests/test.sh
 for t in "${tests[@]}"
 do
   remove_error_test tests/$t

php/tests/test.sh

@@ -14,7 +14,7 @@ set -e
 phpize && ./configure CFLAGS='-g -O0' && make
 popd
 
-tests=( array_test.php encode_decode_test.php generated_class_test.php generated_phpdoc_test.php map_field_test.php well_known_test.php generated_service_test.php descriptors_test.php )
+tests=( array_test.php encode_decode_test.php generated_class_test.php map_field_test.php well_known_test.php descriptors_test.php )
 
 for t in "${tests[@]}"
 do

protoc-artifacts/Dockerfile

@@ -1,4 +1,4 @@
-FROM centos:6.6
+FROM centos:6.9
 
 RUN yum install -y git \
                    tar \
@@ -28,15 +28,14 @@ RUN wget -q http://apache.cs.utah.edu/maven/maven-3/3.3.9/binaries/apache-maven-
     tar xz -C /var/local
 ENV PATH /var/local/apache-maven-3.3.9/bin:$PATH
 
-# Install GCC 4.7 to support -static-libstdc++
-RUN wget http://people.centos.org/tru/devtools-1.1/devtools-1.1.repo -P /etc/yum.repos.d && \
-    bash -c 'echo "enabled=1" >> /etc/yum.repos.d/devtools-1.1.repo' && \
-    bash -c "sed -e 's/\$basearch/i386/g' /etc/yum.repos.d/devtools-1.1.repo > /etc/yum.repos.d/devtools-i386-1.1.repo" && \
-    sed -e 's/testing-/testing-i386-/g' -i /etc/yum.repos.d/devtools-i386-1.1.repo && \
+# Install GCC 4.8 to support -static-libstdc++
+RUN wget http://people.centos.org/tru/devtools-2/devtools-2.repo -P /etc/yum.repos.d && \
+    bash -c 'echo "enabled=1" >> /etc/yum.repos.d/devtools-2.repo' && \
+    bash -c "sed -e 's/\$basearch/i386/g' /etc/yum.repos.d/devtools-2.repo > /etc/yum.repos.d/devtools-i386-2.repo" && \
+    sed -e 's/testing-/testing-i386-/g' -i /etc/yum.repos.d/devtools-i386-2.repo && \
     rpm --rebuilddb && \
-    yum install -y devtoolset-1.1 \
-                   devtoolset-1.1-libstdc++-devel \
-                   devtoolset-1.1-libstdc++-devel.i686 && \
+    yum install -y devtoolset-2-gcc devtoolset-2-gcc-c++ devtoolset-2-binutils devtoolset-2-libstdc++-devel \
+                   devtoolset-2-gcc.i686 devtoolset-2-gcc-c++.i686 devtoolset-2-binutils.i686 devtoolset-2-libstdc++-devel.i686 && \
     yum clean all
 
 COPY scl-enable-devtoolset.sh /var/local/

protoc-artifacts/README.md

@@ -7,13 +7,31 @@ build and publish a ``protoc`` executable (a.k.a. artifact) to Maven
 repositories. The artifact can be used by build automation tools so that users
 would not need to compile and install ``protoc`` for their systems.
 
+If you would like us to publish protoc artifact for a new platform, please send
+us a pull request to add support for the new platform. You would need to change
+the following files:
+
+* [build-protoc.sh](build-protoc.sh): script to cross-build the protoc for your
+  platform.
+* [pom.xml](pom.xml): script to upload artifacts to maven.
+* [build-zip.sh](build-zip.sh): script to package published maven artifacts in
+  our release page.
+
+## Maven Location
+The published protoc artifacts are available on Maven here:
+
+    http://central.maven.org/maven2/com/google/protobuf/protoc/
+
 ## Versioning
 The version of the ``protoc`` artifact must be the same as the version of the
 Protobuf project.
 
 ## Artifact name
 The name of a published ``protoc`` artifact is in the following format:
-``protoc-<version>-<os>-<arch>.exe``, e.g., ``protoc-3.0.0-alpha-3-windows-x86_64.exe``.
+``protoc-<version>-<os>-<arch>.exe``, e.g., ``protoc-3.6.1-linux-x86_64.exe``.
+
+Note that artifacts for linux/macos also have the `.exe` suffix but they are
+not windows binaries.
 
 ## System requirement
 Install [Apache Maven](http://maven.apache.org/) if you don't have it.
@@ -29,98 +47,116 @@ generate the configure script.
 
 Under the protobuf project directory:
 
+
+```
+$ ./autogen.sh
+```
+
+### Build the artifact for each platform
+
+Run the build-protoc.sh script under this protoc-artifacts directory to build the protoc
+artifact for each platform.  For example:
+
 ```
-$ ./autogen.sh && ./configure && make
+$ cd protoc-artifacts
+$ ./build-protoc.sh linux x86_64 protoc
 ```
 
-## To install artifacts locally
-The following command will install the ``protoc`` artifact to your local Maven repository.
+The above command will produce a `target/linux/x86_64/protoc` binary under the
+protoc-artifacts directory.
+
+For a list of supported platforms, see the comments in the build-protoc.sh
+script. We only use this script to build artifacts on Ubuntu and MacOS (both
+with x86_64, and do cross-compilation for other platforms.
+
+### Tips for building for Linux
+We build on Centos 6.9 to provide a good compatibility for not very new
+systems. We have provided a ``Dockerfile`` under this directory to build the
+environment. It has been tested with Docker 1.6.1.
+
+To build a image:
+
 ```
-$ mvn install
+$ docker build -t protoc-artifacts .
 ```
 
-## Cross-compilation
-The Maven script will try to detect the OS and the architecture from Java
-system properties. It's possible to build a protoc binary for an architecture
-that is different from what Java has detected, as long as you have the proper
-compilers installed.
+To run the image:
+
+```
+$ docker run -it --rm=true protoc-artifacts bash
+```
 
-You can override the Maven properties ``os.detected.name`` and
-``os.detected.arch`` to force the script to generate binaries for a specific OS
-and/or architecture. Valid values are defined as the return values of
-``normalizeOs()`` and ``normalizeArch()`` of ``Detector`` from
-[os-maven-plugin](https://github.com/trustin/os-maven-plugin/blob/master/src/main/java/kr/motd/maven/os/Detector.java).
-Frequently used values are:
-- ``os.detected.name``: ``linux``, ``osx``, ``windows``.
-- ``os.detected.arch``: ``x86_32``, ``x86_64``, ``ppcle_64``
+To checkout protobuf (run within the container):
 
-For example, MinGW32 only ships with 32-bit compilers, but you can still build
-32-bit protoc under 64-bit Windows, with the following command:
 ```
-$ mvn install -Dos.detected.arch=x86_32
+$ # Replace v3.5.1 with the version you want
+$ wget -O - https://github.com/google/protobuf/archive/v3.5.1.tar.gz | tar xvzp
 ```
 
+### Windows build
+We no longer use scripts in this directory to build windows artifacts. Instead,
+we use Visual Studio 2015 to build our windows release artifacts. See our
+[kokoro windows build scripts here](../kokoro/release/protoc/windows/build.bat).
+
+To upload windows artifacts, copy the built binaries into this directory and
+put it into the target/windows/(x86_64|x86_32) directory the same way as the
+artifacts for other platforms. That will allow the maven script to find and
+upload the artifacts to maven.
+
 ## To push artifacts to Maven Central
 Before you can upload artifacts to Maven Central repository, make sure you have
 read [this page](http://central.sonatype.org/pages/apache-maven.html) on how to
 configure GPG and Sonatype account.
 
-You need to perform the deployment for every platform that you want to
-support. DO NOT close the staging repository until you have done the
-deployment for all platforms. Currently the following platforms are supported:
-- Linux (x86_32, x86_64, ppcle_64 and cross compiled aarch_64 or ppcle_64)
-- Windows (x86_32 and x86_64) with
-  - Cygwin64 with MinGW compilers (x86_64)
-  - MSYS with MinGW32 (x86_32)
-  - Cross compile in Linux with MinGW-w64 (x86_32, x86_64)
-- MacOSX (x86_32 and x86_64)
-
-As for MSYS2/MinGW64 for Windows: protoc will build, but it insists on
-adding a dependency of `libwinpthread-1.dll`, which isn't shipped with
-Windows.
+Before you do the deployment, make sure you have built the protoc artifacts for
+every supported platform and put them under the target directory. Example
+target directory layout:
+
+    + pom.xml
+    + target
+      + linux
+        + x86_64
+          protoc.exe
+        + x86_32
+          protoc.exe
+      + macos
+        + x86_64
+          protoc.exe
+        + x86_32
+          protoc.exe
+      + windows
+        + x86_64
+          protoc.exe
+        + x86_32
+          protoc.exe
+
+You will need to build the artifacts on multiple machines and gather them
+together into one place.
 
 Use the following command to deploy artifacts for the host platform to a
 staging repository.
+
 ```
 $ mvn clean deploy -P release
 ```
+
 It creates a new staging repository. Go to
 https://oss.sonatype.org/#stagingRepositories and find the repository, usually
-in the name like ``comgoogle-123``.
-
-You will want to run this command on a different platform. Remember, in
-subsequent deployments you will need to provide the repository name that you
-have found in the first deployment so that all artifacts go to the same
-repository:
-```
-$ mvn clean deploy -P release -Dstaging.repository=comgoogle-123
-```
-
-A 32-bit artifact can be deployed from a 64-bit host with
-``-Dos.detected.arch=x86_32``
-
-An arm64 artifact can be deployed from x86 host with
-``-Dos.detected.arch=aarch_64``
-
-A ppcle_64 artifact can be deployed from x86 host with
-``-Dos.detected.arch=ppcle_64``
-
-A windows artifact can be deployed from a linux machine with
-``-Dos.detected.name=windows``
-
-When you have done deployment for all platforms, go to
-https://oss.sonatype.org/#stagingRepositories, verify that the staging
-repository has all the binaries, close and release this repository.
+in the name like ``comgoogle-123``. Verify that the staging repository has all
+the binaries, close and release this repository.
 
 ## Upload zip packages to github release page.
 After uploading protoc artifacts to Maven Central repository, run the
 build-zip.sh script to bulid zip packages for these protoc binaries
 and upload these zip packages to the download section of the github
 release. For example:
+
 ```
 $ ./build-zip.sh 3.6.0
 ```
-The above command will create 6 zip files:
+
+The above command will create 5 zip files:
+
 ```
 dist/protoc-3.6.0-win32.zip
 dist/protoc-3.6.0-osx-x86_32.zip
@@ -129,70 +165,14 @@ dist/protoc-3.6.0-linux-x86_32.zip
 dist/protoc-3.6.0-linux-x86_64.zip
 dist/protoc-3.6.0-linux-ppcle_64.zip
 ```
+
 Before running the script, make sure the artifacts are accessible from:
 http://repo1.maven.org/maven2/com/google/protobuf/protoc/
 
-### Tips for deploying on Linux
-We build on Centos 6.6 to provide a good compatibility for not very new
-systems. We have provided a ``Dockerfile`` under this directory to build the
-environment. It has been tested with Docker 1.6.1.
-
-To build a image:
-```
-$ docker build -t protoc-artifacts .
-```
-
-To run the image:
-```
-$ docker run -it --rm=true protoc-artifacts bash
-```
-
-To checkout protobuf (run within the container):
-```
-$ # Replace v3.5.1 with the version you want
-$ wget -O - https://github.com/google/protobuf/archive/v3.5.1.tar.gz | tar xvzp
-```
-
-### Tips for deploying on Windows
-Under Windows the following error may occur: ``gpg: cannot open tty `no tty':
-No such file or directory``. This can be fixed by configuring gpg through an
-active profile in ``.m2\settings.xml`` where also the Sonatype password is
-stored:
-```xml
-<settings>
-  <servers>
-    <server>
-      <id>sonatype-nexus-staging</id>
-      <username>[username]</username>
-      <password>[password]</password>
-    </server>
-  </servers>
-  <profiles>
-    <profile>
-      <id>gpg</id>
-      <properties>
-        <gpg.executable>gpg</gpg.executable>
-        <gpg.passphrase>[password]</gpg.passphrase>
-      </properties>
-    </profile>
-  </profiles>
-  <activeProfiles>
-    <activeProfile>gpg</activeProfile>
-  </activeProfiles>
-</settings>
-```
-
-### Tested build environments
+## Tested build environments
 We have successfully built artifacts on the following environments:
 - Linux x86_32 and x86_64:
-  - Centos 6.6 (within Docker 1.6.1)
-  - Ubuntu 14.04.2 64-bit
-- Linux ppc64le:
-  - Debian 9.4
-  - Cross compiled with `g++-powerpc64le-linux-gnu` on Debian 9.4 x86_64
-- Linux aarch_64: Cross compiled with `g++-aarch64-linux-gnu` on Ubuntu 14.04.2 64-bit
-- Windows x86_32: MSYS with ``mingw32-gcc-g++ 4.8.1-4`` on Windows 7 64-bit
-- Windows x86_32: Cross compile with ``i686-w64-mingw32-g++ 4.8.2`` on Ubuntu 14.04.2 64-bit
-- Windows x86_64: Cygwin64 with ``mingw64-x86_64-gcc-g++ 4.8.3-1`` on Windows 7 64-bit
-- Windows x86_64: Cross compile with ``x86_64-w64-mingw32-g++ 4.8.2`` on Ubuntu 14.04.2 64-bit
+  - Centos 6.9 (within Docker 1.6.1)
+  - Ubuntu 14.04.5 64-bit
+- Linux aarch_64: Cross compiled with `g++-aarch64-linux-gnu` on Ubuntu 14.04.5 64-bit
 - Mac OS X x86_32 and x86_64: Mac OS X 10.9.5

protoc-artifacts/build-protoc.sh

@@ -1,24 +1,34 @@
 #!/bin/bash
 
-# Builds protoc executable into target/protoc.exe; optionally build protoc
-# plugins into target/protoc-gen-*.exe
-# To be run from Maven.
-# Usage: build-protoc.sh <OS> <ARCH> <TARGET>
-# <OS> and <ARCH> are ${os.detected.name} and ${os.detected.arch} from os-maven-plugin
-# <TARGET> can be "protoc" or "protoc-gen-javalite"
+# Builds protoc executable into target/<OS>/<ARCH>/protoc.exe; optionally builds
+# protoc plugins into target/<OS>/<ARCH>/protoc-gen-*.exe
 #
-# The script now supports cross-compiling windows, linux-arm64, and linux-ppc64le in linux-x86
-# environment. Required packages:
-# - Windows: i686-w64-mingw32-gcc (32bit) and x86_64-w64-mingw32-gcc (64bit)
-# - Arm64: g++-aarch64-linux-gnu
-# - Ppc64le: g++-powerpc64le-linux-gcc
+# Usage: ./build-protoc.sh <OS> <ARCH> <TARGET>
+#
+# <TARGET> can be "protoc" or "protoc-gen-javalite". Supported <OS> <ARCH>
+# combinations:
+#   HOST   <OS>    <ARCH>   <COMMENT>
+#   cygwin windows x86_32   Requires: i686-w64-mingw32-gcc
+#   cygwin windows x86_64   Requires: x86_64-w64-mingw32-gcc
+#   linux  linux   aarch_64 Requires: g++-aarch64-linux-gnu
+#   linux  linux   x86_32
+#   linux  linux   x86_64
+#   linux  windows x86_32   Requires: i686-w64-mingw32-gcc
+#   linux  windows x86_64   Requires: x86_64-w64-mingw32-gcc
+#   macos  osx     x86_32
+#   macos  osx     x86_64
+#   mingw  windows x86_32
+#   mingw  windows x86_64
+#
+# Before running this script, make sure you have generated the configure script
+# in the parent directory (i.e., run ./autogen.sh there).
 
 OS=$1
 ARCH=$2
 MAKE_TARGET=$3
 
 if [[ $# < 3 ]]; then
-  echo "No arguments provided. This script is intended to be run from Maven."
+  echo "Not enough arguments provided."
   exit 1
 fi
 
@@ -28,7 +38,7 @@ case $MAKE_TARGET in
   protoc)
     ;;
   *)
-    echo "Target ""$TARGET"" invalid."
+    echo "Target ""$MAKE_TARGET"" invalid."
     exit 1
 esac
 
@@ -140,7 +150,7 @@ checkDependencies ()
       white_list="linux-vdso64\.so\.1\|libpthread\.so\.0\|libm\.so\.6\|libc\.so\.6\|libz\.so\.1\|ld64\.so\.2"
     elif [[ "$ARCH" == aarch_64 ]]; then
       dump_cmd='objdump -p '"$1"' | grep NEEDED'
-      white_list="libpthread\.so\.0\|libc\.so\.6\|ld-linux-aarch64\.so\.1"
+      white_list="libpthread\.so\.0\|libm\.so\.6\|libc\.so\.6\|ld-linux-aarch64\.so\.1"
     fi
   elif [[ "$OS" == osx ]]; then
     dump_cmd='otool -L '"$1"' | fgrep dylib'
@@ -164,15 +174,10 @@ checkDependencies ()
 }
 ############################################################################
 
-echo "Building protoc, OS=$OS ARCH=$ARCH TARGET=$TARGET"
-
-# Nested double quotes are unintuitive, but it works.
-cd "$(dirname "$0")"
+echo "Building protoc, OS=$OS ARCH=$ARCH TARGET=$MAKE_TARGET"
 
-WORKING_DIR=$(pwd)
 CONFIGURE_ARGS="--disable-shared"
 
-TARGET_FILE=target/$MAKE_TARGET.exe
 if [[ "$OS" == windows ]]; then
   MAKE_TARGET="${MAKE_TARGET}.exe"
 fi
@@ -253,10 +258,18 @@ fi
 
 export CXXFLAGS LDFLAGS
 
-cd "$WORKING_DIR"/.. && ./configure $CONFIGURE_ARGS &&
-  cd src && make clean && make $MAKE_TARGET &&
-  cd "$WORKING_DIR" && mkdir -p target &&
-  cp ../src/$MAKE_TARGET $TARGET_FILE ||
+# Nested double quotes are unintuitive, but it works.
+cd "$(dirname "$0")"
+
+WORKING_DIR="$(pwd)"
+BUILD_DIR="build/$OS/$ARCH"
+TARGET_FILE="target/$OS/$ARCH/$MAKE_TARGET.exe"
+
+mkdir -p "$BUILD_DIR" && cd "$BUILD_DIR" &&
+  ../../../../configure $CONFIGURE_ARGS &&
+  cd src && make $MAKE_TARGET -j8 &&
+  cd "$WORKING_DIR" && mkdir -p $(dirname $TARGET_FILE) &&
+  cp $BUILD_DIR/src/$MAKE_TARGET $TARGET_FILE ||
   exit 1
 
 if [[ "$OS" == osx ]]; then

protoc-artifacts/pom.xml

@@ -33,36 +33,7 @@
     </connection>
   </scm>
   <build>
-    <extensions>
-      <extension>
-        <groupId>kr.motd.maven</groupId>
-        <artifactId>os-maven-plugin</artifactId>
-        <version>1.5.0.Final</version>
-      </extension>
-    </extensions>
     <plugins>
-      <plugin>
-        <groupId>org.codehaus.mojo</groupId>
-        <artifactId>exec-maven-plugin</artifactId>
-        <version>1.1.1</version>
-        <executions>
-          <execution>
-            <phase>compile</phase>
-            <goals>
-              <goal>exec</goal>
-            </goals>
-          </execution>
-        </executions>
-        <configuration>
-          <executable>bash</executable>
-          <arguments>
-            <argument>build-protoc.sh</argument>
-            <argument>${os.detected.name}</argument>
-            <argument>${os.detected.arch}</argument>
-            <argument>protoc</argument>
-          </arguments>
-        </configuration>
-      </plugin>
       <plugin>
         <groupId>org.codehaus.mojo</groupId>
         <artifactId>build-helper-maven-plugin</artifactId>
@@ -77,8 +48,38 @@
             <configuration>
               <artifacts>
                 <artifact>
-                  <file>${basedir}/target/protoc.exe</file>
-                  <classifier>${os.detected.name}-${os.detected.arch}</classifier>
+                  <file>${basedir}/target/linux/x86_64/protoc.exe</file>
+                  <classifier>linux-x86_64</classifier>
+                  <type>exe</type>
+                </artifact>
+                <artifact>
+                  <file>${basedir}/target/linux/x86_32/protoc.exe</file>
+                  <classifier>linux-x86_32</classifier>
+                  <type>exe</type>
+                </artifact>
+                <artifact>
+                  <file>${basedir}/target/windows/x86_64/protoc.exe</file>
+                  <classifier>windows-x86_64</classifier>
+                  <type>exe</type>
+                </artifact>
+                <artifact>
+                  <file>${basedir}/target/windows/x86_32/protoc.exe</file>
+                  <classifier>windows-x86_32</classifier>
+                  <type>exe</type>
+                </artifact>
+                <artifact>
+                  <file>${basedir}/target/osx/x86_64/protoc.exe</file>
+                  <classifier>osx-x86_64</classifier>
+                  <type>exe</type>
+                </artifact>
+                <artifact>
+                  <file>${basedir}/target/osx/x86_32/protoc.exe</file>
+                  <classifier>osx-x86_32</classifier>
+                  <type>exe</type>
+                </artifact>
+                <artifact>
+                  <file>${basedir}/target/linux/aarch_64/protoc.exe</file>
+                  <classifier>linux-aarch_64</classifier>
                   <type>exe</type>
                 </artifact>
               </artifacts>

protoc-artifacts/scl-enable-devtoolset.sh

@@ -10,4 +10,4 @@ quote() {
   done
 }
 
-exec scl enable devtoolset-1.1 "$(quote "$@")"
+exec scl enable devtoolset-2 "$(quote "$@")"

python/tox.ini

@@ -4,7 +4,8 @@ envlist =
 
 [testenv]
 usedevelop=true
-passenv = CC
+passenv = 
+    CC KOKORO_BUILD_ID KOKORO_BUILD_NUMBER
 setenv =
     cpp: LD_LIBRARY_PATH={toxinidir}/../src/.libs
     cpp: DYLD_LIBRARY_PATH={toxinidir}/../src/.libs

ruby/compatibility_tests/v3.0.0/tests/basic.rb

@@ -222,33 +222,55 @@ module BasicTest
 
     def test_type_errors
       m = TestMessage.new
-      assert_raise TypeError do
+
+      # Use rescue to allow subclasses of error
+      success = false
+      begin
         m.optional_int32 = "hello"
+      rescue TypeError
+        success = true
       end
-      assert_raise TypeError do
-        m.optional_string = 42
-      end
-      assert_raise TypeError do
+      assert(success)
+
+      success = false
+      begin
         m.optional_string = nil
+      rescue TypeError
+        success = true
       end
-      assert_raise TypeError do
+      assert(success)
+
+      success = false
+      begin
         m.optional_bool = 42
+      rescue TypeError
+        success = true
       end
-      assert_raise TypeError do
+      assert(success)
+
+      success = false
+      begin
         m.optional_msg = TestMessage.new  # expects TestMessage2
+      rescue TypeError
+        success = true
       end
+      assert(success)
 
-      assert_raise TypeError do
+      success = false
+      begin
         m.repeated_int32 = []  # needs RepeatedField
+      rescue TypeError
+        success = true
       end
+      assert(success)
 
-      assert_raise TypeError do
-        m.repeated_int32.push "hello"
-      end
-
-      assert_raise TypeError do
+      success = false
+      begin
         m.repeated_msg.push TestMessage.new
+      rescue TypeError
+        success = true
       end
+      assert(success)
     end
 
     def test_string_encoding
@@ -275,7 +297,7 @@ module BasicTest
 
       # strings are immutable so we can't do this, but serialize should catch it.
       m.optional_string = "asdf".encode!('UTF-8')
-      assert_raise RuntimeError do
+      assert_raise do
         m.optional_string.encode!('ASCII-8BIT')
       end
     end
@@ -312,10 +334,14 @@ module BasicTest
       assert l.pop == 9
       assert l == [5, 2, 3, 4, 7, 8]
 
-      assert_raise TypeError do
+      success = false
+      begin
         m = TestMessage.new
         l.push m
+      rescue TypeError
+        success = true
       end
+      assert(success)
 
       m = TestMessage.new
       m.repeated_int32 = l
@@ -362,12 +388,22 @@ module BasicTest
       l = Google::Protobuf::RepeatedField.new(:message, TestMessage)
       l.push TestMessage.new
       assert l.count == 1
-      assert_raise TypeError do
+
+      success = false
+      begin
         l.push TestMessage2.new
+      rescue TypeError
+        success = true
       end
-      assert_raise TypeError do
+      assert(success)
+
+      success = false
+      begin
         l.push 42
+      rescue TypeError
+        success = true
       end
+      assert(success)
 
       l2 = l.dup
       assert l2[0] == l[0]
@@ -493,9 +529,14 @@ module BasicTest
       assert m.length == 0
       assert m == {}
 
-      assert_raise TypeError do
+      success = false
+      begin
         m[1] = 1
+      rescue TypeError
+        success = true
       end
+      assert(success)
+
       assert_raise RangeError do
         m["asdf"] = 0x1_0000_0000
       end
@@ -514,18 +555,28 @@ module BasicTest
       assert_raise RangeError do
         m[0x8000_0000] = 1
       end
-      assert_raise TypeError do
+
+      success = false
+      begin
         m["asdf"] = 1
+      rescue TypeError
+        success = true
       end
+      assert(success)
 
       m = Google::Protobuf::Map.new(:int64, :int32)
       m[0x1000_0000_0000_0000] = 1
       assert_raise RangeError do
         m[0x1_0000_0000_0000_0000] = 1
       end
-      assert_raise TypeError do
+
+      success = false
+      begin
         m["asdf"] = 1
+      rescue TypeError
+        success = true
       end
+      assert(success)
 
       m = Google::Protobuf::Map.new(:uint32, :int32)
       m[0x8000_0000] = 1
@@ -548,18 +599,32 @@ module BasicTest
       m = Google::Protobuf::Map.new(:bool, :int32)
       m[true] = 1
       m[false] = 2
-      assert_raise TypeError do
+
+      success = false
+      begin
         m[1] = 1
+      rescue TypeError
+        success = true
       end
-      assert_raise TypeError do
+      assert(success)
+
+      success = false
+      begin
         m["asdf"] = 1
+      rescue TypeError
+        success = true
       end
+      assert(success)
 
       m = Google::Protobuf::Map.new(:string, :int32)
       m["asdf"] = 1
-      assert_raise TypeError do
+      success = false
+      begin
         m[1] = 1
+      rescue TypeError
+        success = true
       end
+      assert(success)
       assert_raise Encoding::UndefinedConversionError do
         bytestring = ["FFFF"].pack("H*")
         m[bytestring] = 1
@@ -570,17 +635,25 @@ module BasicTest
       m[bytestring] = 1
       # Allowed -- we will automatically convert to ASCII-8BIT.
       m["asdf"] = 1
-      assert_raise TypeError do
+      success = false
+      begin
         m[1] = 1
+      rescue TypeError
+        success = true
       end
+      assert(success)
     end
 
     def test_map_msg_enum_valuetypes
       m = Google::Protobuf::Map.new(:string, :message, TestMessage)
       m["asdf"] = TestMessage.new
-      assert_raise TypeError do
+      success = false
+      begin
         m["jkl;"] = TestMessage2.new
+      rescue TypeError
+        success = true
       end
+      assert(success)
 
       m = Google::Protobuf::Map.new(
         :string, :message, TestMessage,
@@ -645,23 +718,39 @@ module BasicTest
       m.map_string_msg.delete("c")
       assert m.map_string_msg == { "a" => TestMessage2.new(:foo => 1) }
 
-      assert_raise TypeError do
+      success = false
+      begin
         m.map_string_msg["e"] = TestMessage.new # wrong value type
+      rescue TypeError
+        success = true
       end
+      assert(success)
       # ensure nothing was added by the above
       assert m.map_string_msg == { "a" => TestMessage2.new(:foo => 1) }
 
       m.map_string_int32 = Google::Protobuf::Map.new(:string, :int32)
-      assert_raise TypeError do
+      success = false
+      begin
         m.map_string_int32 = Google::Protobuf::Map.new(:string, :int64)
+      rescue TypeError
+        success = true
       end
-      assert_raise TypeError do
+      assert(success)
+      success = false
+      begin
         m.map_string_int32 = {}
+      rescue TypeError
+        success = true
       end
+      assert(success)
 
-      assert_raise TypeError do
+      success = false
+      begin
         m = MapMessage.new(:map_string_int32 => { 1 => "I am not a number" })
+      rescue TypeError
+        success = true
       end
+      assert(success)
     end
 
     def test_map_encode_decode
@@ -922,22 +1011,30 @@ module BasicTest
 
     def test_def_errors
       s = Google::Protobuf::DescriptorPool.new
-      assert_raise TypeError do
+      success = false
+      begin
         s.build do
           # enum with no default (integer value 0)
           add_enum "MyEnum" do
             value :A, 1
           end
         end
+      rescue TypeError
+        success = true
       end
-      assert_raise TypeError do
+      assert(success)
+      success = false
+      begin
         s.build do
           # message with required field (unsupported in proto3)
           add_message "MyMessage" do
             required :foo, :int32, 1
           end
         end
+      rescue TypeError
+        success = true
       end
+      assert(success)
     end
 
     def test_corecursive

ruby/compatibility_tests/v3.0.0/tests/repeated_field_test.rb

@@ -18,7 +18,7 @@ class RepeatedFieldTest < Test::Unit::TestCase
     # jRuby additions to the Array class that we can ignore
     arr_methods -= [ :indices, :iter_for_each, :iter_for_each_index,
       :iter_for_each_with_index, :dimensions, :copy_data, :copy_data_simple,
-      :nitems, :iter_for_reverse_each, :indexes]
+      :nitems, :iter_for_reverse_each, :indexes, :append, :prepend]
     arr_methods.each do |method_name|
       assert m.repeated_string.respond_to?(method_name) == true, "does not respond to #{method_name}"
     end

ruby/ext/google/protobuf_c/storage.c

@@ -177,7 +177,7 @@ void native_slot_set_value_and_case(upb_fieldtype_t type, VALUE type_class,
     }
     case UPB_TYPE_STRING:
       if (CLASS_OF(value) == rb_cSymbol) {
-        value = rb_funcall(value, rb_intern("to_s"), 0, NULL);
+        value = rb_funcall(value, rb_intern("to_s"), 0);
       } else if (CLASS_OF(value) != rb_cString) {
         rb_raise(rb_eTypeError, "Invalid argument for string field.");
       }
@@ -207,7 +207,7 @@ void native_slot_set_value_and_case(upb_fieldtype_t type, VALUE type_class,
     case UPB_TYPE_ENUM: {
       int32_t int_val = 0;
       if (TYPE(value) == T_STRING) {
-        value = rb_funcall(value, rb_intern("to_sym"), 0, NULL);
+        value = rb_funcall(value, rb_intern("to_sym"), 0);
       } else if (!is_ruby_num(value) && TYPE(value) != T_SYMBOL) {
         rb_raise(rb_eTypeError,
                  "Expected number or symbol type for enum field.");

ruby/tests/basic.rb

@@ -336,7 +336,9 @@ module BasicTest
 
       # strings are immutable so we can't do this, but serialize should catch it.
       m.optional_string = "asdf".encode!('UTF-8')
-      assert_raise RuntimeError do
+      # Ruby 2.5 changed to raise FrozenError. However, assert_raise don't
+      # accept subclass. Don't specify type here.
+      assert_raise do
         m.optional_string.encode!('ASCII-8BIT')
       end
     end

ruby/tests/repeated_field_test.rb

@@ -18,7 +18,7 @@ class RepeatedFieldTest < Test::Unit::TestCase
     # jRuby additions to the Array class that we can ignore
     arr_methods -= [ :indices, :iter_for_each, :iter_for_each_index,
       :iter_for_each_with_index, :dimensions, :copy_data, :copy_data_simple,
-      :nitems, :iter_for_reverse_each, :indexes]
+      :nitems, :iter_for_reverse_each, :indexes, :append, :prepend]
     arr_methods.each do |method_name|
       assert m.repeated_string.respond_to?(method_name) == true, "does not respond to #{method_name}"
     end

src/Makefile.am

@@ -56,9 +56,7 @@ clean-local:
 
 CLEANFILES = $(protoc_outputs) unittest_proto_middleman \
              testzip.jar testzip.list testzip.proto testzip.zip \
-             no_warning_test.cc \
-             google/protobuf/compiler/js/well_known_types_embed.cc \
-             js_embed$(EXEEXT)
+             no_warning_test.cc
 
 MAINTAINERCLEANFILES =   \
   Makefile.in
@@ -471,22 +469,6 @@ bin_PROGRAMS = protoc
 protoc_LDADD = $(PTHREAD_LIBS) libprotobuf.la libprotoc.la
 protoc_SOURCES = google/protobuf/compiler/main.cc
 
-# The special JS code for the well-known types is linked into the compiler via
-# well_known_types_embed.cc, which is generated from .js source files. We have
-# to build the js_embed binary using $(CXX_FOR_BUILD) so that it is executable
-# on the build machine in a cross-compilation setup.
-js_embed$(EXEEXT): $(srcdir)/google/protobuf/compiler/js/embed.cc
-	$(CXX_FOR_BUILD) -o $@ $<
-js_well_known_types_sources =                                  \
-	google/protobuf/compiler/js/well_known_types/any.js          \
-	google/protobuf/compiler/js/well_known_types/struct.js       \
-	google/protobuf/compiler/js/well_known_types/timestamp.js
-# We have to cd to $(srcdir) so that out-of-tree builds work properly.
-google/protobuf/compiler/js/well_known_types_embed.cc: js_embed$(EXEEXT) $(js_well_known_types_sources)
-	mkdir -p `dirname $@` && \
-	oldpwd=`pwd` && cd $(srcdir) && \
-	$$oldpwd/js_embed$(EXEEXT) $(js_well_known_types_sources) > $$oldpwd/$@
-
 # Tests ==============================================================
 
 protoc_inputs =                                                   \
@@ -565,7 +547,6 @@ EXTRA_DIST =                                                   \
   google/protobuf/package_info.h                               \
   google/protobuf/io/package_info.h                            \
   google/protobuf/util/package_info.h                          \
-  google/protobuf/compiler/js/embed.cc                         \
   google/protobuf/compiler/ruby/ruby_generated_code.proto      \
   google/protobuf/compiler/ruby/ruby_generated_code_pb.rb      \
   google/protobuf/compiler/package_info.h                      \

src/google/protobuf/compiler/js/embed.cc

@@ -1,110 +0,0 @@
-// Protocol Buffers - Google's data interchange format
-// Copyright 2008 Google Inc.  All rights reserved.
-// https://developers.google.com/protocol-buffers/
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are
-// met:
-//
-//     * Redistributions of source code must retain the above copyright
-// notice, this list of conditions and the following disclaimer.
-//     * Redistributions in binary form must reproduce the above
-// copyright notice, this list of conditions and the following disclaimer
-// in the documentation and/or other materials provided with the
-// distribution.
-//     * Neither the name of Google Inc. nor the names of its
-// contributors may be used to endorse or promote products derived from
-// this software without specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-#include <cassert>
-#include <cstdlib>
-#include <fstream>
-#include <iostream>
-#include <string>
-
-static bool AsciiIsPrint(unsigned char c) {
-  return c >= 32 && c < 127;
-}
-
-static char ToDecimalDigit(int num) {
-  assert(num < 10);
-  return '0' + num;
-}
-
-static std::string CEscape(const std::string& str) {
-  std::string dest;
-
-  for (size_t i = 0; i < str.size(); ++i) {
-    unsigned char ch = str[i];
-    switch (ch) {
-      case '\n': dest += "\\n"; break;
-      case '\r': dest += "\\r"; break;
-      case '\t': dest += "\\t"; break;
-      case '\"': dest += "\\\""; break;
-      case '\\': dest += "\\\\"; break;
-      default:
-        if (AsciiIsPrint(ch)) {
-          dest += ch;
-        } else {
-          dest += "\\";
-          dest += ToDecimalDigit(ch / 64);
-          dest += ToDecimalDigit((ch % 64) / 8);
-          dest += ToDecimalDigit(ch % 8);
-        }
-        break;
-    }
-  }
-
-  return dest;
-}
-
-static void AddFile(const char* name, std::basic_ostream<char>* out) {
-  std::ifstream in(name);
-
-  if (!in.is_open()) {
-    std::cerr << "Couldn't open input file: " << name << "\n";
-    std::exit(EXIT_FAILURE);
-  }
-
-  // Make canonical name only include the final element.
-  for (const char *p = name; *p; p++) {
-    if (*p == '/') {
-      name = p + 1;
-    }
-  }
-
-  *out << "{\"" << CEscape(name) << "\",\n";
-
-  for (std::string line; std::getline(in, line); ) {
-    *out << "  \"" << CEscape(line) << "\\n\"\n";
-  }
-
-  *out << "},\n";
-}
-
-int main(int argc, char *argv[]) {
-  std::cout << "#include "
-               "\"google/protobuf/compiler/js/well_known_types_embed.h\"\n";
-  std::cout << "struct FileToc well_known_types_js[] = {\n";
-
-  for (int i = 1; i < argc; i++) {
-    AddFile(argv[i], &std::cout);
-  }
-
-  std::cout << "  {NULL, NULL}  // Terminate the list.\n";
-  std::cout << "};\n";
-
-  return EXIT_SUCCESS;
-}

src/google/protobuf/compiler/js/well_known_types_embed.cc

@@ -0,0 +1,225 @@
+#include <google/protobuf/compiler/js/well_known_types_embed.h>
+
+struct FileToc well_known_types_js[] = {
+    {"any.js",
+     "/* This code will be inserted into generated code for\n"
+     " * google/protobuf/any.proto. */\n"
+     "\n"
+     "/**\n"
+     " * Returns the type name contained in this instance, if any.\n"
+     " * @return {string|undefined}\n"
+     " */\n"
+     "proto.google.protobuf.Any.prototype.getTypeName = function() {\n"
+     "  return this.getTypeUrl().split('/').pop();\n"
+     "};\n"
+     "\n"
+     "\n"
+     "/**\n"
+     " * Packs the given message instance into this Any.\n"
+     " * @param {!Uint8Array} serialized The serialized data to pack.\n"
+     " * @param {string} name The type name of this message object.\n"
+     " * @param {string=} opt_typeUrlPrefix the type URL prefix.\n"
+     " */\n"
+     "proto.google.protobuf.Any.prototype.pack = function(serialized, name,\n"
+     "                                                    opt_typeUrlPrefix) "
+     "{\n"
+     "  if (!opt_typeUrlPrefix) {\n"
+     "    opt_typeUrlPrefix = 'type.googleapis.com/';\n"
+     "  }\n"
+     "\n"
+     "  if (opt_typeUrlPrefix.substr(-1) != '/') {\n"
+     "    this.setTypeUrl(opt_typeUrlPrefix + '/' + name);\n"
+     "  } else {\n"
+     "    this.setTypeUrl(opt_typeUrlPrefix + name);\n"
+     "  }\n"
+     "\n"
+     "  this.setValue(serialized);\n"
+     "};\n"
+     "\n"
+     "\n"
+     "/**\n"
+     " * @template T\n"
+     " * Unpacks this Any into the given message object.\n"
+     " * @param {function(Uint8Array):T} deserialize Function that will "
+     "deserialize\n"
+     " *     the binary data properly.\n"
+     " * @param {string} name The expected type name of this message object.\n"
+     " * @return {?T} If the name matched the expected name, returns the "
+     "deserialized\n"
+     " *     object, otherwise returns null.\n"
+     " */\n"
+     "proto.google.protobuf.Any.prototype.unpack = function(deserialize, name) "
+     "{\n"
+     "  if (this.getTypeName() == name) {\n"
+     "    return deserialize(this.getValue_asU8());\n"
+     "  } else {\n"
+     "    return null;\n"
+     "  }\n"
+     "};\n"},
+    {"timestamp.js",
+     "/* This code will be inserted into generated code for\n"
+     " * google/protobuf/timestamp.proto. */\n"
+     "\n"
+     "/**\n"
+     " * Returns a JavaScript 'Date' object corresponding to this Timestamp.\n"
+     " * @return {!Date}\n"
+     " */\n"
+     "proto.google.protobuf.Timestamp.prototype.toDate = function() {\n"
+     "  var seconds = this.getSeconds();\n"
+     "  var nanos = this.getNanos();\n"
+     "\n"
+     "  return new Date((seconds * 1000) + (nanos / 1000000));\n"
+     "};\n"
+     "\n"
+     "\n"
+     "/**\n"
+     " * Sets the value of this Timestamp object to be the given Date.\n"
+     " * @param {!Date} value The value to set.\n"
+     " */\n"
+     "proto.google.protobuf.Timestamp.prototype.fromDate = function(value) {\n"
+     "  this.setSeconds(Math.floor(value.getTime() / 1000));\n"
+     "  this.setNanos(value.getMilliseconds() * 1000000);\n"
+     "};\n"},
+    {"struct.js",
+     "/* This code will be inserted into generated code for\n"
+     " * google/protobuf/struct.proto. */\n"
+     "\n"
+     "/**\n"
+     " * Typedef representing plain JavaScript values that can go into a\n"
+     " *     Struct.\n"
+     " * @typedef {null|number|string|boolean|Array|Object}\n"
+     " */\n"
+     "proto.google.protobuf.JavaScriptValue;\n"
+     "\n"
+     "\n"
+     "/**\n"
+     " * Converts this Value object to a plain JavaScript value.\n"
+     " * @return {?proto.google.protobuf.JavaScriptValue} a plain JavaScript\n"
+     " *     value representing this Struct.\n"
+     " */\n"
+     "proto.google.protobuf.Value.prototype.toJavaScript = function() {\n"
+     "  var kindCase = proto.google.protobuf.Value.KindCase;\n"
+     "  switch (this.getKindCase()) {\n"
+     "    case kindCase.NULL_VALUE:\n"
+     "      return null;\n"
+     "    case kindCase.NUMBER_VALUE:\n"
+     "      return this.getNumberValue();\n"
+     "    case kindCase.STRING_VALUE:\n"
+     "      return this.getStringValue();\n"
+     "    case kindCase.BOOL_VALUE:\n"
+     "      return this.getBoolValue();\n"
+     "    case kindCase.STRUCT_VALUE:\n"
+     "      return this.getStructValue().toJavaScript();\n"
+     "    case kindCase.LIST_VALUE:\n"
+     "      return this.getListValue().toJavaScript();\n"
+     "    default:\n"
+     "      throw new Error('Unexpected struct type');\n"
+     "  }\n"
+     "};\n"
+     "\n"
+     "\n"
+     "/**\n"
+     " * Converts this JavaScript value to a new Value proto.\n"
+     " * @param {!proto.google.protobuf.JavaScriptValue} value The value to\n"
+     " *     convert.\n"
+     " * @return {!proto.google.protobuf.Value} The newly constructed value.\n"
+     " */\n"
+     "proto.google.protobuf.Value.fromJavaScript = function(value) {\n"
+     "  var ret = new proto.google.protobuf.Value();\n"
+     "  switch (goog.typeOf(value)) {\n"
+     "    case 'string':\n"
+     "      ret.setStringValue(/** @type {string} */ (value));\n"
+     "      break;\n"
+     "    case 'number':\n"
+     "      ret.setNumberValue(/** @type {number} */ (value));\n"
+     "      break;\n"
+     "    case 'boolean':\n"
+     "      ret.setBoolValue(/** @type {boolean} */ (value));\n"
+     "      break;\n"
+     "    case 'null':\n"
+     "      ret.setNullValue(proto.google.protobuf.NullValue.NULL_VALUE);\n"
+     "      break;\n"
+     "    case 'array':\n"
+     "      ret.setListValue(proto.google.protobuf.ListValue.fromJavaScript(\n"
+     "          /** @type{!Array} */ (value)));\n"
+     "      break;\n"
+     "    case 'object':\n"
+     "      ret.setStructValue(proto.google.protobuf.Struct.fromJavaScript(\n"
+     "          /** @type{!Object} */ (value)));\n"
+     "      break;\n"
+     "    default:\n"
+     "      throw new Error('Unexpected struct type.');\n"
+     "  }\n"
+     "\n"
+     "  return ret;\n"
+     "};\n"
+     "\n"
+     "\n"
+     "/**\n"
+     " * Converts this ListValue object to a plain JavaScript array.\n"
+     " * @return {!Array} a plain JavaScript array representing this List.\n"
+     " */\n"
+     "proto.google.protobuf.ListValue.prototype.toJavaScript = function() {\n"
+     "  var ret = [];\n"
+     "  var values = this.getValuesList();\n"
+     "\n"
+     "  for (var i = 0; i < values.length; i++) {\n"
+     "    ret[i] = values[i].toJavaScript();\n"
+     "  }\n"
+     "\n"
+     "  return ret;\n"
+     "};\n"
+     "\n"
+     "\n"
+     "/**\n"
+     " * Constructs a ListValue protobuf from this plain JavaScript array.\n"
+     " * @param {!Array} array a plain JavaScript array\n"
+     " * @return {proto.google.protobuf.ListValue} a new ListValue object\n"
+     " */\n"
+     "proto.google.protobuf.ListValue.fromJavaScript = function(array) {\n"
+     "  var ret = new proto.google.protobuf.ListValue();\n"
+     "\n"
+     "  for (var i = 0; i < array.length; i++) {\n"
+     "    "
+     "ret.addValues(proto.google.protobuf.Value.fromJavaScript(array[i]));\n"
+     "  }\n"
+     "\n"
+     "  return ret;\n"
+     "};\n"
+     "\n"
+     "\n"
+     "/**\n"
+     " * Converts this Struct object to a plain JavaScript object.\n"
+     " * @return {!Object<string, !proto.google.protobuf.JavaScriptValue>} a "
+     "plain\n"
+     " *     JavaScript object representing this Struct.\n"
+     " */\n"
+     "proto.google.protobuf.Struct.prototype.toJavaScript = function() {\n"
+     "  var ret = {};\n"
+     "\n"
+     "  this.getFieldsMap().forEach(function(value, key) {\n"
+     "    ret[key] = value.toJavaScript();\n"
+     "  });\n"
+     "\n"
+     "  return ret;\n"
+     "};\n"
+     "\n"
+     "\n"
+     "/**\n"
+     " * Constructs a Struct protobuf from this plain JavaScript object.\n"
+     " * @param {!Object} obj a plain JavaScript object\n"
+     " * @return {proto.google.protobuf.Struct} a new Struct object\n"
+     " */\n"
+     "proto.google.protobuf.Struct.fromJavaScript = function(obj) {\n"
+     "  var ret = new proto.google.protobuf.Struct();\n"
+     "  var map = ret.getFieldsMap();\n"
+     "\n"
+     "  for (var property in obj) {\n"
+     "    var val = obj[property];\n"
+     "    map.set(property, proto.google.protobuf.Value.fromJavaScript(val));\n"
+     "  }\n"
+     "\n"
+     "  return ret;\n"
+     "};\n"},
+    {NULL, NULL}  // Terminate the list.
+};

tests.sh

@@ -1,14 +1,8 @@
 #!/bin/bash
 #
-# Build and runs tests for the protobuf project.  The tests as written here are
-# used by both Jenkins and Travis, though some specialized logic is required to
-# handle the differences between them.
-
-on_travis() {
-  if [ "$TRAVIS" == "true" ]; then
-    "$@"
-  fi
-}
+# Build and runs tests for the protobuf project. We use this script to run
+# tests on kokoro (Ubuntu and MacOS). It can run locally as well but you
+# will need to make sure the required compilers/tools are available.
 
 # For when some other test needs the C++ main build, including protoc and
 # libprotobuf.
@@ -18,15 +12,6 @@ internal_build_cpp() {
     return
   fi
 
-  if [[ $(uname -s) == "Linux" && "$TRAVIS" == "true" ]]; then
-    # Install GCC 4.8 to replace the default GCC 4.6. We need 4.8 for more
-    # decent C++ 11 support in order to compile conformance tests.
-    sudo add-apt-repository ppa:ubuntu-toolchain-r/test -y
-    sudo apt-get update -qq
-    sudo apt-get install -qq g++-4.8
-    export CXX="g++-4.8" CC="gcc-4.8"
-  fi
-
   # Initialize any submodules.
   git submodule update --init --recursive
 
@@ -43,8 +28,6 @@ build_cpp() {
 
   # The benchmark code depends on cmake, so test if it is installed before
   # trying to do the build.
-  # NOTE: The travis macOS images say they have cmake, but the xcode8.1 image
-  # appears to be missing it: https://github.com/travis-ci/travis-ci/issues/6996
   if [[ $(type cmake 2>/dev/null) ]]; then
     # Verify benchmarking code can build successfully.
     cd benchmarks && make cpp-benchmark && cd ..
@@ -121,10 +104,6 @@ build_golang() {
   # Add protoc to the path so that the examples build finds it.
   export PATH="`pwd`/src:$PATH"
 
-  # Install Go and the Go protobuf compiler plugin.
-  on_travis sudo apt-get update -qq
-  on_travis sudo apt-get install -qq golang
-
   export GOPATH="$HOME/gocode"
   mkdir -p "$GOPATH/src/github.com/google"
   rm -f "$GOPATH/src/github.com/google/protobuf"
@@ -139,27 +118,17 @@ use_java() {
   version=$1
   case "$version" in
     jdk7)
-      on_travis sudo apt-get install openjdk-7-jdk
       export PATH=/usr/lib/jvm/java-7-openjdk-amd64/bin:$PATH
       export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64
       ;;
     oracle7)
-      if [ "$TRAVIS" == "true" ]; then
-        sudo apt-get install python-software-properties # for apt-add-repository
-        echo "oracle-java7-installer shared/accepted-oracle-license-v1-1 select true" | \
-          sudo debconf-set-selections
-        yes | sudo apt-add-repository ppa:webupd8team/java
-        yes | sudo apt-get install oracle-java7-installer
-      fi;
       export PATH=/usr/lib/jvm/java-7-oracle/bin:$PATH
       export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64
       ;;
   esac
 
-  if [ "$TRAVIS" != "true" ]; then
-    MAVEN_LOCAL_REPOSITORY=/var/maven_local_repository
-    MVN="$MVN -e -X --offline -Dmaven.repo.local=$MAVEN_LOCAL_REPOSITORY"
-  fi;
+  MAVEN_LOCAL_REPOSITORY=/var/maven_local_repository
+  MVN="$MVN -e -X --offline -Dmaven.repo.local=$MAVEN_LOCAL_REPOSITORY"
 
   which java
   java -version
@@ -190,12 +159,6 @@ build_java_with_conformance_tests() {
   cd conformance && make test_java && cd ..
 }
 
-build_javanano() {
-  # Java build needs `protoc`.
-  internal_build_cpp
-  cd javanano && $MVN test && cd ..
-}
-
 build_java_jdk7() {
   use_java jdk7
   build_java_with_conformance_tests
@@ -213,38 +176,6 @@ build_java_compatibility() {
   ./test.sh 3.0.0-beta-4
 }
 
-build_javanano_jdk7() {
-  use_java jdk7
-  build_javanano
-}
-build_javanano_oracle7() {
-  use_java oracle7
-  build_javanano
-}
-
-internal_install_python_deps() {
-  if [ "$TRAVIS" != "true" ]; then
-    return;
-  fi
-  # Install tox (OS X doesn't have pip).
-  if [ $(uname -s) == "Darwin" ]; then
-    brew upgrade python
-    python3 -m pip install tox
-  else
-    sudo pip install tox
-  fi
-  # Only install Python2.6/3.x on Linux.
-  if [ $(uname -s) == "Linux" ]; then
-    sudo apt-get install -y python-software-properties # for apt-add-repository
-    sudo apt-add-repository -y ppa:fkrull/deadsnakes
-    sudo apt-get update -qq
-    sudo apt-get install -y python3.3 python3.3-dev
-    sudo apt-get install -y python3.4 python3.4-dev
-    sudo apt-get install -y python3.5 python3.5-dev
-    sudo apt-get install -y python3.6 python3.6-dev
-  fi
-}
-
 build_objectivec_ios() {
   # Reused the build script that takes care of configuring and ensuring things
   # are up to date.  The OS X test runs the objc conformance test, so skip it
@@ -279,9 +210,7 @@ build_objectivec_cocoapods_integration() {
 
 build_python() {
   internal_build_cpp
-  internal_install_python_deps
   cd python
-  # Only test Python 2.6/3.x on Linux
   if [ $(uname -s) == "Linux" ]; then
     envlist=py\{27,33,34,35,36\}-python
   else
@@ -293,11 +222,9 @@ build_python() {
 
 build_python_cpp() {
   internal_build_cpp
-  internal_install_python_deps
   export LD_LIBRARY_PATH=../src/.libs # for Linux
   export DYLD_LIBRARY_PATH=../src/.libs # for OS X
   cd python
-  # Only test Python 3.x on Linux
   if [ $(uname -s) == "Linux" ]; then
     envlist=py\{27,33,34,35,36\}-cpp
   else
@@ -325,18 +252,24 @@ build_ruby22() {
   internal_build_cpp  # For conformance tests.
   cd ruby && bash travis-test.sh ruby-2.2 && cd ..
 }
-build_jruby() {
+build_ruby23() {
+  internal_build_cpp  # For conformance tests.
+  cd ruby && bash travis-test.sh ruby-2.3 && cd ..
+}
+build_ruby24() {
   internal_build_cpp  # For conformance tests.
-  # TODO(xiaofeng): Upgrade to jruby-9.x. There are some broken jests to be
-  # fixed.
-  cd ruby && bash travis-test.sh jruby-1.7 && cd ..
+  cd ruby && bash travis-test.sh ruby-2.4 && cd ..
+}
+build_ruby25() {
+  internal_build_cpp  # For conformance tests.
+  cd ruby && bash travis-test.sh ruby-2.5.0 && cd ..
 }
 build_ruby_all() {
   build_ruby21
   build_ruby22
-  # TODO(teboring): Disable jruby test temperarily for it randomly fails.
-  # https://grpc-testing.appspot.com/job/protobuf_pull_request/735/consoleFull.
-  # build_jruby
+  build_ruby23
+  build_ruby24
+  build_ruby25
 }
 
 build_javascript() {
@@ -617,14 +550,6 @@ build_php_all() {
   build_php_compatibility
 }
 
-# Note: travis currently does not support testing more than one language so the
-# .travis.yml cheats and claims to only be cpp.  If they add multiple language
-# support, this should probably get updated to install steps and/or
-# rvm/gemfile/jdk/etc. entries rather than manually doing the work.
-
-# .travis.yml uses matrix.exclude to block the cases where app-get can't be
-# use to install things.
-
 # -------- main --------
 
 if [ "$#" -ne 1 ]; then
@@ -635,8 +560,6 @@ Usage: $0 { cpp |
             java_jdk7 |
             java_oracle7 |
             java_compatibility |
-            javanano_jdk7 |
-            javanano_oracle7 |
             objectivec_ios |
             objectivec_ios_debug |
             objectivec_ios_release |
@@ -665,4 +588,5 @@ fi
 
 set -e  # exit immediately on error
 set -x  # display all commands
+cd $(dirname $0)
 eval "build_$1"