Merge branch 'master' into fixbug_enum2json2

BUILD

@@ -121,7 +121,6 @@ cc_library(
         "src/google/protobuf/api.pb.cc",
         "src/google/protobuf/compiler/importer.cc",
         "src/google/protobuf/compiler/parser.cc",
-        "src/google/protobuf/compiler/plugin.pb.cc",
         "src/google/protobuf/descriptor.cc",
         "src/google/protobuf/descriptor.pb.cc",
         "src/google/protobuf/descriptor_database.cc",
@@ -254,7 +253,7 @@ internal_copied_filegroup(
     srcs = WELL_KNOWN_PROTOS,
     dest = "",
     strip_prefix = "src",
-    visibility = ["//visibility:hidden"],
+    visibility = ["//visibility:private"],
 )
 
 [proto_library(
@@ -379,6 +378,7 @@ cc_library(
         "src/google/protobuf/compiler/objectivec/objectivec_primitive_field.cc",
         "src/google/protobuf/compiler/php/php_generator.cc",
         "src/google/protobuf/compiler/plugin.cc",
+        "src/google/protobuf/compiler/plugin.pb.cc",
         "src/google/protobuf/compiler/python/python_generator.cc",
         "src/google/protobuf/compiler/ruby/ruby_generator.cc",
         "src/google/protobuf/compiler/subprocess.cc",

Makefile.am

@@ -354,6 +354,7 @@ java_EXTRA_DIST=
   java/core/src/test/proto/com/google/protobuf/map_for_proto2_lite_test.proto      \
   java/core/src/test/proto/com/google/protobuf/map_for_proto2_test.proto           \
   java/core/src/test/proto/com/google/protobuf/map_initialization_order_test.proto \
+  java/core/src/test/proto/com/google/protobuf/map_lite_test.proto                 \
   java/core/src/test/proto/com/google/protobuf/map_test.proto                      \
   java/core/src/test/proto/com/google/protobuf/multiple_files_test.proto           \
   java/core/src/test/proto/com/google/protobuf/nested_builders_test.proto          \
@@ -808,6 +809,7 @@ python_EXTRA_DIST=                                                           \
   python/google/protobuf/internal/more_extensions.proto                      \
   python/google/protobuf/internal/more_extensions_dynamic.proto              \
   python/google/protobuf/internal/more_messages.proto                        \
+  python/google/protobuf/internal/no_package.proto                           \
   python/google/protobuf/internal/packed_field_test.proto                    \
   python/google/protobuf/internal/proto_builder_test.py                      \
   python/google/protobuf/internal/python_message.py                          \
@@ -861,6 +863,7 @@ python_EXTRA_DIST=                                                           \
   python/google/protobuf/pyext/repeated_scalar_container.h                   \
   python/google/protobuf/pyext/safe_numerics.h                               \
   python/google/protobuf/pyext/scoped_pyobject_ptr.h                         \
+  python/google/protobuf/pyext/thread_unsafe_shared_ptr.h                    \
   python/google/protobuf/reflection.py                                       \
   python/google/protobuf/service.py                                          \
   python/google/protobuf/service_reflection.py                               \

appveyor.yml

@@ -10,6 +10,7 @@ configuration:
 environment:
   matrix:
     - language: cpp
+      image: Visual Studio 2015
       BUILD_DLL: ON
       UNICODE: ON
 
@@ -32,8 +33,8 @@ install:
   - move gtest gmock
 
 before_build:
-  - if %platform%==Win32 set generator=Visual Studio 12
-  - if %platform%==Win64 set generator=Visual Studio 12 Win64
+  - if %platform%==Win32 set generator=Visual Studio 14
+  - if %platform%==Win64 set generator=Visual Studio 14 Win64
   - if %platform%==Win32 set vcplatform=Win32
   - if %platform%==Win64 set vcplatform=x64
 

benchmarks/Makefile.am

@@ -1,9 +1,11 @@
+benchmarks_protoc_inputs_benchmark_wrapper =                               \
+	benchmarks.proto
+
 benchmarks_protoc_inputs =                                                 \
-	benchmarks.proto                                                         \
-	datasets/google_message1/benchmark_message1_proto3.proto
+	datasets/google_message1/proto3/benchmark_message1_proto3.proto
 
 benchmarks_protoc_inputs_proto2 =                                          \
-	datasets/google_message1/benchmark_message1_proto2.proto                 \
+	datasets/google_message1/proto2/benchmark_message1_proto2.proto          \
 	datasets/google_message2/benchmark_message2.proto                        \
 	datasets/google_message3/benchmark_message3.proto                        \
 	datasets/google_message3/benchmark_message3_1.proto                      \
@@ -26,7 +28,7 @@ make_tmp_dir:
 if USE_EXTERNAL_PROTOC
 
 protoc_middleman: make_tmp_dir $(benchmarks_protoc_inputs)
-	$(PROTOC) -I$(srcdir) -I$(top_srcdir) --cpp_out=. --java_out=./tmp $(benchmarks_protoc_inputs)
+	$(PROTOC) -I$(srcdir) -I$(top_srcdir) --cpp_out=. --java_out=./tmp $(benchmarks_protoc_inputs) $(benchmarks_protoc_inputs_benchmark_wrapper)
 	touch protoc_middleman
 
 protoc_middleman2: make_tmp_dir $(benchmarks_protoc_inputs_proto2)
@@ -38,8 +40,8 @@ else
 # We have to cd to $(srcdir) before executing protoc because $(protoc_inputs) is
 # relative to srcdir, which may not be the same as the current directory when
 # building out-of-tree.
-protoc_middleman: make_tmp_dir $(top_srcdir)/src/protoc$(EXEEXT) $(benchmarks_protoc_inputs) $(well_known_type_protoc_inputs)
-	oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --cpp_out=$$oldpwd --java_out=$$oldpwd/tmp/java/src/main/java --python_out=$$oldpwd/tmp $(benchmarks_protoc_inputs) )
+protoc_middleman: make_tmp_dir $(top_srcdir)/src/protoc$(EXEEXT) $(benchmarks_protoc_inputs) $(well_known_type_protoc_inputs) $(benchmarks_protoc_inputs_benchmark_wrapper)
+	oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --cpp_out=$$oldpwd --java_out=$$oldpwd/tmp/java/src/main/java --python_out=$$oldpwd/tmp $(benchmarks_protoc_inputs) $(benchmarks_protoc_inputs_benchmark_wrapper) )
 	touch protoc_middleman
 
 protoc_middleman2:  make_tmp_dir $(top_srcdir)/src/protoc$(EXEEXT) $(benchmarks_protoc_inputs_proto2) $(well_known_type_protoc_inputs)
@@ -54,14 +56,14 @@ all_data = `find . -type f -name "dataset.*.pb"`
 
 benchmarks_protoc_outputs =                                                \
 	benchmarks.pb.cc                                                         \
-	datasets/google_message1/benchmark_message1_proto3.pb.cc
+	datasets/google_message1/proto3/benchmark_message1_proto3.pb.cc
   
 benchmarks_protoc_outputs_header =                                         \
 	benchmarks.pb.h                                                          \
-	datasets/google_message1/benchmark_message1_proto3.pb.h
+	datasets/google_message1/proto3/benchmark_message1_proto3.pb.h
 
 benchmarks_protoc_outputs_proto2_header =                                  \
-	datasets/google_message1/benchmark_message1_proto2.pb.h                  \
+	datasets/google_message1/proto2/benchmark_message1_proto2.pb.h           \
 	datasets/google_message2/benchmark_message2.pb.h                         \
 	datasets/google_message3/benchmark_message3.pb.h                         \
 	datasets/google_message3/benchmark_message3_1.pb.h                       \
@@ -78,7 +80,7 @@ benchmarks_protoc_outputs_proto2_header =                                  \
 	datasets/google_message4/benchmark_message4_3.pb.h
 
 benchmarks_protoc_outputs_proto2 =                                         \
-	datasets/google_message1/benchmark_message1_proto2.pb.cc                 \
+	datasets/google_message1/proto2/benchmark_message1_proto2.pb.cc          \
 	datasets/google_message2/benchmark_message2.pb.cc                        \
 	datasets/google_message3/benchmark_message3.pb.cc                        \
 	datasets/google_message3/benchmark_message3_1.pb.cc                      \
@@ -224,6 +226,78 @@ python-cpp-generated-code: python-cpp-generated-code-benchmark
 
 ############# PYTHON RULES END ##############
 
+############# GO RULES BEGIN ##############
+
+benchmarks_protoc_inputs_proto2_message1 =                                 \
+	datasets/google_message1/proto2/benchmark_message1_proto2.proto
+
+benchmarks_protoc_inputs_proto2_message2 =                                 \
+	datasets/google_message2/benchmark_message2.proto
+
+benchmarks_protoc_inputs_proto2_message3 =                                 \
+	datasets/google_message3/benchmark_message3.proto                        \
+	datasets/google_message3/benchmark_message3_1.proto                      \
+	datasets/google_message3/benchmark_message3_2.proto                      \
+	datasets/google_message3/benchmark_message3_3.proto                      \
+	datasets/google_message3/benchmark_message3_4.proto                      \
+	datasets/google_message3/benchmark_message3_5.proto                      \
+	datasets/google_message3/benchmark_message3_6.proto                      \
+	datasets/google_message3/benchmark_message3_7.proto                      \
+	datasets/google_message3/benchmark_message3_8.proto
+
+benchmarks_protoc_inputs_proto2_message4 =                                 \
+	datasets/google_message4/benchmark_message4.proto                        \
+	datasets/google_message4/benchmark_message4_1.proto                      \
+	datasets/google_message4/benchmark_message4_2.proto                      \
+	datasets/google_message4/benchmark_message4_3.proto
+
+if USE_EXTERNAL_PROTOC
+
+go_protoc_middleman: make_tmp_dir $(benchmarks_protoc_inputs)
+	$(PROTOC) -I$(srcdir) -I$(top_srcdir) --go_out=$$oldpwd/tmp $(benchmarks_protoc_inputs) 
+	$(PROTOC) -I$(srcdir) -I$(top_srcdir) --go_out=$$oldpwd/tmp $(benchmarks_protoc_inputs_benchmark_wrapper)
+	touch protoc_middleman
+
+go_protoc_middleman2: make_tmp_dir $(benchmarks_protoc_inputs_proto2_message1) $(benchmarks_protoc_inputs_proto2_message2) $(benchmarks_protoc_inputs_proto2_message3) $(benchmarks_protoc_inputs_proto2_message4)
+	$(PROTOC) -I$(srcdir) -I$(top_srcdir) --go_out=$$oldpwd/tmp $(benchmarks_protoc_inputs_proto2_message1)
+	$(PROTOC) -I$(srcdir) -I$(top_srcdir) --go_out=$$oldpwd/tmp $(benchmarks_protoc_inputs_proto2_message2)
+	$(PROTOC) -I$(srcdir) -I$(top_srcdir) --go_out=$$oldpwd/tmp $(benchmarks_protoc_inputs_proto2_message3)
+	$(PROTOC) -I$(srcdir) -I$(top_srcdir) --go_out=$$oldpwd/tmp $(benchmarks_protoc_inputs_proto2_message4)
+	touch protoc_middleman2
+
+else
+
+# We have to cd to $(srcdir) before executing protoc because $(protoc_inputs) is
+# relative to srcdir, which may not be the same as the current directory when
+# building out-of-tree.
+go_protoc_middleman: make_tmp_dir $(top_srcdir)/src/protoc$(EXEEXT) $(benchmarks_protoc_inputs) $(well_known_type_protoc_inputs)
+	oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --go_out=$$oldpwd/tmp $(benchmarks_protoc_inputs) )
+	oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --go_out=$$oldpwd/tmp $(benchmarks_protoc_inputs_benchmark_wrapper) )
+	touch protoc_middleman
+
+go_protoc_middleman2:  make_tmp_dir $(top_srcdir)/src/protoc$(EXEEXT) $(benchmarks_protoc_inputs_proto2_message1) $(benchmarks_protoc_inputs_proto2_message2) $(benchmarks_protoc_inputs_proto2_message3) $(benchmarks_protoc_inputs_proto2_message4) $(well_known_type_protoc_inputs)
+	oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --go_out=$$oldpwd/tmp $(benchmarks_protoc_inputs_proto2_message1) )
+	oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --go_out=$$oldpwd/tmp $(benchmarks_protoc_inputs_proto2_message2) )
+	oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --go_out=$$oldpwd/tmp $(benchmarks_protoc_inputs_proto2_message3) )
+	oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --go_out=$$oldpwd/tmp $(benchmarks_protoc_inputs_proto2_message4) )
+	touch protoc_middleman2
+
+endif
+
+go-benchmark: go_protoc_middleman go_protoc_middleman2 
+	@echo "Writing shortcut script go-benchmark..."
+	@echo '#! /bin/sh' > go-benchmark
+	@echo 'mkdir tmp_cc && mv *.cc tmp_cc' >> go-benchmark
+	@echo 'go test -bench=. -- $$@' >> go-benchmark
+	@echo 'mv tmp_cc/* . && rm -rf tmp_cc' >> go-benchmark
+	@chmod +x go-benchmark
+
+go: go_protoc_middleman go_protoc_middleman2 go-benchmark
+	./go-benchmark $(all_data)
+
+############# GO RULES END ##############
+
+
 MAINTAINERCLEANFILES =                                                     \
 	Makefile.in
 
@@ -241,7 +315,10 @@ CLEANFILES =                                                               \
 	python_cpp_proto_library                                                 \
 	python-pure-python-benchmark                                             \
 	python-cpp-reflection-benchmark                                          \
-	python-cpp-generated-code-benchmark
+	python-cpp-generated-code-benchmark                                      \
+	go-benchmark                                                             \
+	go_protoc_middleman                                                      \
+	go_protoc_middleman2
 
 clean-local:
 	-rm -rf tmp/*

benchmarks/README.md

@@ -36,6 +36,21 @@ $ sudo apt-get install python3-dev
 ```
 And you also need to make sure `pkg-config` is installed.
 
+### Go
+Go protobufs are maintained at [github.com/golang/protobuf](
+http://github.com/golang/protobuf). If not done already, you need to install the 
+toolchain and the Go protoc-gen-go plugin for protoc. 
+
+To install protoc-gen-go, run:
+
+```
+$ go get -u github.com/golang/protobuf/protoc-gen-go
+$ export PATH=$PATH:$(go env GOPATH)/bin
+```
+
+The first command installs `protoc-gen-go` into the `bin` directory in your local `GOPATH`.
+The second command adds the `bin` directory to your `PATH` so that `protoc` can locate the plugin later.
+
 ### Big data
 
 There's some optional big testing data which is not included in the directory
@@ -87,6 +102,11 @@ $ make python-cpp-reflection
 $ make python-cpp-generated-code
 ```
 
+### Go
+```
+$ make go
+```
+
 To run a specific dataset:
 
 ### Java:
@@ -126,6 +146,13 @@ $ make python-cpp-generated-code-benchmark
 $ ./python-cpp-generated-code-benchmark $(specific generated dataset file name)
 ```
 
+### Go:
+```
+$ make go-benchmark
+$ ./go-benchmark $(specific generated dataset file name)
+```
+
+
 ## Benchmark datasets
 
 Each data set is in the format of benchmarks.proto:

benchmarks/cpp_benchmark.cc

@@ -32,8 +32,8 @@
 #include <iostream>
 #include "benchmark/benchmark_api.h"
 #include "benchmarks.pb.h"
-#include "datasets/google_message1/benchmark_message1_proto2.pb.h"
-#include "datasets/google_message1/benchmark_message1_proto3.pb.h"
+#include "datasets/google_message1/proto2/benchmark_message1_proto2.pb.h"
+#include "datasets/google_message1/proto3/benchmark_message1_proto3.pb.h"
 #include "datasets/google_message2/benchmark_message2.pb.h"
 #include "datasets/google_message3/benchmark_message3.pb.h"
 #include "datasets/google_message4/benchmark_message4.pb.h"

benchmarks/go_benchmark_test.go

@@ -0,0 +1,124 @@
+package main
+
+import (
+	benchmarkWrapper "./tmp"
+	googleMessage1Proto2 "./tmp/datasets/google_message1/proto2"
+	googleMessage1Proto3 "./tmp/datasets/google_message1/proto3"
+	googleMessage2 "./tmp/datasets/google_message2"
+	googleMessage3 "./tmp/datasets/google_message3"
+	googleMessage4 "./tmp/datasets/google_message4"
+	"flag"
+	"github.com/golang/protobuf/proto"
+	"io/ioutil"
+	"testing"
+)
+
+// Data is returned by the Load function.
+type Dataset struct {
+	name        string
+	newMessage  func() proto.Message
+	marshaled   [][]byte
+	unmarshaled []proto.Message
+}
+
+var datasets []Dataset
+
+// This is used to getDefaultInstance for a message type.
+func generateNewMessageFunction(dataset benchmarkWrapper.BenchmarkDataset) func() proto.Message {
+	switch dataset.MessageName {
+	case "benchmarks.proto3.GoogleMessage1":
+		return func() proto.Message { return new(googleMessage1Proto3.GoogleMessage1) }
+	case "benchmarks.proto2.GoogleMessage1":
+		return func() proto.Message { return new(googleMessage1Proto2.GoogleMessage1) }
+	case "benchmarks.proto2.GoogleMessage2":
+		return func() proto.Message { return new(googleMessage2.GoogleMessage2) }
+	case "benchmarks.google_message3.GoogleMessage3":
+		return func() proto.Message { return new(googleMessage3.GoogleMessage3) }
+	case "benchmarks.google_message4.GoogleMessage4":
+		return func() proto.Message { return new(googleMessage4.GoogleMessage4) }
+	default:
+		panic("Unknown message type: " + dataset.MessageName)
+	}
+}
+
+func init() {
+	flag.Parse()
+	for _, f := range flag.Args() {
+		// Load the benchmark.
+		b, err := ioutil.ReadFile(f)
+		if err != nil {
+			panic(err)
+		}
+
+		// Parse the benchmark.
+		var dm benchmarkWrapper.BenchmarkDataset
+		if err := proto.Unmarshal(b, &dm); err != nil {
+			panic(err)
+		}
+
+		// Determine the concrete protobuf message type to use.
+		var ds Dataset
+		ds.newMessage = generateNewMessageFunction(dm)
+
+		// Unmarshal each test message.
+		for _, payload := range dm.Payload {
+			ds.marshaled = append(ds.marshaled, payload)
+			m := ds.newMessage()
+			if err := proto.Unmarshal(payload, m); err != nil {
+				panic(err)
+			}
+			ds.unmarshaled = append(ds.unmarshaled, m)
+		}
+		ds.name = f
+
+		datasets = append(datasets, ds)
+	}
+}
+
+func Benchmark(b *testing.B) {
+	for _, ds := range datasets {
+		b.Run(ds.name, func(b *testing.B) {
+			b.Run("Unmarshal", func(b *testing.B) {
+				for i := 0; i < b.N; i++ {
+					for j, payload := range ds.marshaled {
+						out := ds.newMessage()
+						if err := proto.Unmarshal(payload, out); err != nil {
+							b.Fatalf("can't unmarshal message %d %v", j, err)
+						}
+					}
+				}
+			})
+			b.Run("Marshal", func(b *testing.B) {
+				for i := 0; i < b.N; i++ {
+					for j, m := range ds.unmarshaled {
+						if _, err := proto.Marshal(m); err != nil {
+							b.Fatalf("can't marshal message %d %+v: %v", j, m, err)
+						}
+					}
+				}
+			})
+			b.Run("Size", func(b *testing.B) {
+				for i := 0; i < b.N; i++ {
+					for _, m := range ds.unmarshaled {
+						proto.Size(m)
+					}
+				}
+			})
+			b.Run("Clone", func(b *testing.B) {
+				for i := 0; i < b.N; i++ {
+					for _, m := range ds.unmarshaled {
+						proto.Clone(m)
+					}
+				}
+			})
+			b.Run("Merge", func(b *testing.B) {
+				for i := 0; i < b.N; i++ {
+					for _, m := range ds.unmarshaled {
+						out := ds.newMessage()
+						proto.Merge(out, m)
+					}
+				}
+			})
+		})
+	}
+}

benchmarks/java/src/main/java/com/google/protobuf/ProtoCaliperBenchmark.java

@@ -243,3 +243,4 @@ public class ProtoCaliperBenchmark {
   }
 }
 
+

benchmarks/py_benchmark.py

@@ -17,8 +17,8 @@ elif sys.argv[1] != "false":
   raise IOError("Need string argument \"true\" or \"false\" for whether to use cpp generated code")
 # END CPP GENERATED MESSAGE
 
-import datasets.google_message1.benchmark_message1_proto2_pb2 as benchmark_message1_proto2_pb2
-import datasets.google_message1.benchmark_message1_proto3_pb2 as benchmark_message1_proto3_pb2
+import datasets.google_message1.proto2.benchmark_message1_proto2_pb2 as benchmark_message1_proto2_pb2
+import datasets.google_message1.proto3.benchmark_message1_proto3_pb2 as benchmark_message1_proto3_pb2
 import datasets.google_message2.benchmark_message2_pb2 as benchmark_message2_pb2
 import datasets.google_message3.benchmark_message3_pb2 as benchmark_message3_pb2
 import datasets.google_message4.benchmark_message4_pb2 as benchmark_message4_pb2
@@ -115,3 +115,4 @@ class Benchmark:
 if __name__ == "__main__":
   for i in range(2, len(sys.argv)):
     run_one_test(sys.argv[i])
+

cmake/extract_includes.bat.in

@@ -79,20 +79,6 @@ copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\repeated_field.h" inc
 copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\service.h" include\google\protobuf\service.h
 copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\source_context.pb.h" include\google\protobuf\source_context.pb.h
 copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\struct.pb.h" include\google\protobuf\struct.pb.h
-copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\atomic_sequence_num.h" include\google\protobuf\stubs\atomic_sequence_num.h
-copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\atomicops.h" include\google\protobuf\stubs\atomicops.h
-copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\atomicops_internals_arm64_gcc.h" include\google\protobuf\stubs\atomicops_internals_arm64_gcc.h
-copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\atomicops_internals_arm_gcc.h" include\google\protobuf\stubs\atomicops_internals_arm_gcc.h
-copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\atomicops_internals_arm_qnx.h" include\google\protobuf\stubs\atomicops_internals_arm_qnx.h
-copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\atomicops_internals_generic_c11_atomic.h" include\google\protobuf\stubs\atomicops_internals_generic_c11_atomic.h
-copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\atomicops_internals_generic_gcc.h" include\google\protobuf\stubs\atomicops_internals_generic_gcc.h
-copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\atomicops_internals_mips_gcc.h" include\google\protobuf\stubs\atomicops_internals_mips_gcc.h
-copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\atomicops_internals_power.h" include\google\protobuf\stubs\atomicops_internals_power.h
-copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\atomicops_internals_ppc_gcc.h" include\google\protobuf\stubs\atomicops_internals_ppc_gcc.h
-copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\atomicops_internals_solaris.h" include\google\protobuf\stubs\atomicops_internals_solaris.h
-copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\atomicops_internals_tsan.h" include\google\protobuf\stubs\atomicops_internals_tsan.h
-copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\atomicops_internals_x86_gcc.h" include\google\protobuf\stubs\atomicops_internals_x86_gcc.h
-copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\atomicops_internals_x86_msvc.h" include\google\protobuf\stubs\atomicops_internals_x86_msvc.h
 copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\bytestream.h" include\google\protobuf\stubs\bytestream.h
 copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\callback.h" include\google\protobuf\stubs\callback.h
 copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\casts.h" include\google\protobuf\stubs\casts.h
@@ -105,8 +91,6 @@ copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\mutex.h" includ
 copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\once.h" include\google\protobuf\stubs\once.h
 copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\platform_macros.h" include\google\protobuf\stubs\platform_macros.h
 copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\port.h" include\google\protobuf\stubs\port.h
-copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\scoped_ptr.h" include\google\protobuf\stubs\scoped_ptr.h
-copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\shared_ptr.h" include\google\protobuf\stubs\shared_ptr.h
 copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\singleton.h" include\google\protobuf\stubs\singleton.h
 copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\status.h" include\google\protobuf\stubs\status.h
 copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\stl_util.h" include\google\protobuf\stubs\stl_util.h

cmake/libprotobuf-lite.cmake

@@ -10,13 +10,10 @@ set(libprotobuf_lite_files
   ${protobuf_source_dir}/src/google/protobuf/io/zero_copy_stream_impl_lite.cc
   ${protobuf_source_dir}/src/google/protobuf/message_lite.cc
   ${protobuf_source_dir}/src/google/protobuf/repeated_field.cc
-  ${protobuf_source_dir}/src/google/protobuf/stubs/atomicops_internals_x86_gcc.cc
-  ${protobuf_source_dir}/src/google/protobuf/stubs/atomicops_internals_x86_msvc.cc
   ${protobuf_source_dir}/src/google/protobuf/stubs/bytestream.cc
   ${protobuf_source_dir}/src/google/protobuf/stubs/common.cc
   ${protobuf_source_dir}/src/google/protobuf/stubs/int128.cc
   ${protobuf_source_dir}/src/google/protobuf/stubs/io_win32.cc
-  ${protobuf_source_dir}/src/google/protobuf/stubs/once.cc
   ${protobuf_source_dir}/src/google/protobuf/stubs/status.cc
   ${protobuf_source_dir}/src/google/protobuf/stubs/statusor.cc
   ${protobuf_source_dir}/src/google/protobuf/stubs/stringpiece.cc
@@ -38,7 +35,6 @@ set(libprotobuf_lite_includes
   ${protobuf_source_dir}/src/google/protobuf/io/zero_copy_stream_impl_lite.h
   ${protobuf_source_dir}/src/google/protobuf/message_lite.h
   ${protobuf_source_dir}/src/google/protobuf/repeated_field.h
-  ${protobuf_source_dir}/src/google/protobuf/stubs/atomicops_internals_x86_msvc.h
   ${protobuf_source_dir}/src/google/protobuf/stubs/bytestream.h
   ${protobuf_source_dir}/src/google/protobuf/stubs/common.h
   ${protobuf_source_dir}/src/google/protobuf/stubs/int128.h

cmake/libprotobuf.cmake

@@ -4,7 +4,6 @@ set(libprotobuf_files
   ${protobuf_source_dir}/src/google/protobuf/api.pb.cc
   ${protobuf_source_dir}/src/google/protobuf/compiler/importer.cc
   ${protobuf_source_dir}/src/google/protobuf/compiler/parser.cc
-  ${protobuf_source_dir}/src/google/protobuf/compiler/plugin.pb.cc
   ${protobuf_source_dir}/src/google/protobuf/descriptor.cc
   ${protobuf_source_dir}/src/google/protobuf/descriptor.pb.cc
   ${protobuf_source_dir}/src/google/protobuf/descriptor_database.cc

cmake/libprotoc.cmake

@@ -88,6 +88,7 @@ set(libprotoc_files
   ${protobuf_source_dir}/src/google/protobuf/compiler/objectivec/objectivec_primitive_field.cc
   ${protobuf_source_dir}/src/google/protobuf/compiler/php/php_generator.cc
   ${protobuf_source_dir}/src/google/protobuf/compiler/plugin.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/plugin.pb.cc
   ${protobuf_source_dir}/src/google/protobuf/compiler/python/python_generator.cc
   ${protobuf_source_dir}/src/google/protobuf/compiler/ruby/ruby_generator.cc
   ${protobuf_source_dir}/src/google/protobuf/compiler/subprocess.cc

cmake/tests.cmake

@@ -155,7 +155,8 @@ set(tests_files
   ${protobuf_source_dir}/src/google/protobuf/preserve_unknown_enum_test.cc
   ${protobuf_source_dir}/src/google/protobuf/proto3_arena_lite_unittest.cc
   ${protobuf_source_dir}/src/google/protobuf/proto3_arena_unittest.cc
-  ${protobuf_source_dir}/src/google/protobuf/proto3_lite_unittest.cc
+# TODO(b/74491957) Make this unittest work
+#  ${protobuf_source_dir}/src/google/protobuf/proto3_lite_unittest.cc
   ${protobuf_source_dir}/src/google/protobuf/reflection_ops_unittest.cc
   ${protobuf_source_dir}/src/google/protobuf/repeated_field_reflection_unittest.cc
   ${protobuf_source_dir}/src/google/protobuf/repeated_field_unittest.cc
@@ -163,7 +164,6 @@ set(tests_files
   ${protobuf_source_dir}/src/google/protobuf/stubs/common_unittest.cc
   ${protobuf_source_dir}/src/google/protobuf/stubs/int128_unittest.cc
   ${protobuf_source_dir}/src/google/protobuf/stubs/io_win32_unittest.cc
-  ${protobuf_source_dir}/src/google/protobuf/stubs/once_unittest.cc
   ${protobuf_source_dir}/src/google/protobuf/stubs/status_test.cc
   ${protobuf_source_dir}/src/google/protobuf/stubs/statusor_test.cc
   ${protobuf_source_dir}/src/google/protobuf/stubs/stringpiece_unittest.cc
@@ -172,7 +172,6 @@ set(tests_files
   ${protobuf_source_dir}/src/google/protobuf/stubs/strutil_unittest.cc
   ${protobuf_source_dir}/src/google/protobuf/stubs/template_util_unittest.cc
   ${protobuf_source_dir}/src/google/protobuf/stubs/time_test.cc
-  ${protobuf_source_dir}/src/google/protobuf/stubs/type_traits_unittest.cc
   ${protobuf_source_dir}/src/google/protobuf/text_format_unittest.cc
   ${protobuf_source_dir}/src/google/protobuf/unknown_field_set_unittest.cc
   ${protobuf_source_dir}/src/google/protobuf/util/delimited_message_util_test.cc

configure.ac

@@ -184,9 +184,8 @@ AS_IF([test "$with_protoc" != "no"], [
 ])
 AM_CONDITIONAL([USE_EXTERNAL_PROTOC], [test "$with_protoc" != "no"])
 
-ACX_PTHREAD
-AM_CONDITIONAL([HAVE_PTHREAD], [test "x$acx_pthread_ok" = "xyes"])
-
+AX_PTHREAD
+AM_CONDITIONAL([HAVE_PTHREAD], [test "x$ax_pthread_ok" = "xyes"])
 # We still keep this for improving pbconfig.h for unsupported platforms.
 AC_CXX_STL_HASH
 

conformance/conformance_cpp.cc

@@ -45,7 +45,6 @@ using google::protobuf::Descriptor;
 using google::protobuf::DescriptorPool;
 using google::protobuf::Message;
 using google::protobuf::MessageFactory;
-using google::protobuf::internal::scoped_ptr;
 using google::protobuf::util::BinaryToJsonString;
 using google::protobuf::util::JsonToBinaryString;
 using google::protobuf::util::NewTypeResolverForDescriptorPool;

conformance/conformance_test.cc

@@ -1910,6 +1910,10 @@ bool ConformanceTestSuite::RunSuite(ConformanceTestRunner* runner,
   {
     TestAllTypesProto3 messageProto3;
     TestAllTypesProto2 messageProto2;
+    //TODO(yilunchong): update this behavior when unknown field's behavior
+    // changed in open source. Also delete
+    // Required.Proto3.ProtobufInput.UnknownVarint.ProtobufOutput
+    // from failure list of python_cpp python java
     TestUnknownMessage(messageProto3, true);
     TestUnknownMessage(messageProto2, false);
   }
@@ -2320,6 +2324,24 @@ bool ConformanceTestSuite::RunSuite(ConformanceTestRunner* runner,
           }
         }
       )");
+  RunValidJsonTest(
+      "ValueAcceptListWithNull", REQUIRED,
+      R"({"optionalValue": ["x", null, "y"]})",
+      R"(
+        optional_value: {
+          list_value: {
+            values: {
+              string_value: "x"
+            }
+            values: {
+              null_value: NULL_VALUE
+            }
+            values: {
+              string_value: "y"
+            }
+          }
+        }
+      )");
   RunValidJsonTest(
       "ValueAcceptObject", REQUIRED,
       R"({"optionalValue": {"value": 1}})",

conformance/conformance_test.h

@@ -256,8 +256,7 @@ class ConformanceTestSuite {
   // The set of tests that the testee opted out of;
   std::set<std::string> skipped_;
 
-  google::protobuf::internal::scoped_ptr<google::protobuf::util::TypeResolver>
-      type_resolver_;
+  std::unique_ptr<google::protobuf::util::TypeResolver> type_resolver_;
   std::string type_url_;
 };
 

conformance/conformance_test_runner.cc

@@ -68,7 +68,6 @@
 
 using conformance::ConformanceRequest;
 using conformance::ConformanceResponse;
-using google::protobuf::internal::scoped_array;
 using google::protobuf::StringAppendF;
 using std::string;
 using std::vector;
@@ -183,7 +182,7 @@ class ForkPipeRunner : public google::protobuf::ConformanceTestRunner {
       CHECK_SYSCALL(close(toproc_pipe_fd[1]));
       CHECK_SYSCALL(close(fromproc_pipe_fd[0]));
 
-      scoped_array<char> executable(new char[executable_.size() + 1]);
+      std::unique_ptr<char[]> executable(new char[executable_.size() + 1]);
       memcpy(executable.get(), executable_.c_str(), executable_.size());
       executable[executable_.size()] = '\0';
 

csharp/src/Google.Protobuf.Test/CodedInputStreamTest.cs

@@ -615,5 +615,88 @@ namespace Google.Protobuf
             var stream = new CodedInputStream(new byte[10]);
             stream.Dispose();
         }
+
+        [Test]
+        public void TestParseMessagesCloseTo2G()
+        {
+            byte[] serializedMessage = GenerateBigSerializedMessage();
+            // How many of these big messages do we need to take us near our 2GB limit?
+            int count = Int32.MaxValue / serializedMessage.Length;
+            // Now make a MemoryStream that will fake a near-2GB stream of messages by returning
+            // our big serialized message 'count' times.
+            using (RepeatingMemoryStream stream = new RepeatingMemoryStream(serializedMessage, count))
+            {
+                Assert.DoesNotThrow(()=>TestAllTypes.Parser.ParseFrom(stream));
+            }
+        }
+
+        [Test]
+        public void TestParseMessagesOver2G()
+        {
+            byte[] serializedMessage = GenerateBigSerializedMessage();
+            // How many of these big messages do we need to take us near our 2GB limit?
+            int count = Int32.MaxValue / serializedMessage.Length;
+            // Now add one to take us over the 2GB limit
+            count++;
+            // Now make a MemoryStream that will fake a near-2GB stream of messages by returning
+            // our big serialized message 'count' times.
+            using (RepeatingMemoryStream stream = new RepeatingMemoryStream(serializedMessage, count))
+            {
+                Assert.Throws<InvalidProtocolBufferException>(() => TestAllTypes.Parser.ParseFrom(stream),
+                    "Protocol message was too large.  May be malicious.  " +
+                    "Use CodedInputStream.SetSizeLimit() to increase the size limit.");
+            }
+        }
+
+        /// <returns>A serialized big message</returns>
+        private static byte[] GenerateBigSerializedMessage()
+        {
+            byte[] value = new byte[16 * 1024 * 1024];
+            TestAllTypes message = SampleMessages.CreateFullTestAllTypes();
+            message.SingleBytes = ByteString.CopyFrom(value);
+            return message.ToByteArray();
+        }
+
+        /// <summary>
+        /// A MemoryStream that repeats a byte arrays' content a number of times.
+        /// Simulates really large input without consuming loads of memory. Used above
+        /// to test the parsing behavior when the input size exceeds 2GB or close to it.
+        /// </summary>
+        private class RepeatingMemoryStream: MemoryStream
+        {
+            private readonly byte[] bytes;
+            private readonly int maxIterations;
+            private int index = 0;
+
+            public RepeatingMemoryStream(byte[] bytes, int maxIterations)
+            {
+                this.bytes = bytes;
+                this.maxIterations = maxIterations;
+            }
+
+            public override int Read(byte[] buffer, int offset, int count)
+            {
+                if (bytes.Length == 0)
+                {
+                    return 0;
+                }
+                int numBytesCopiedTotal = 0;
+                while (numBytesCopiedTotal < count && index < maxIterations)
+                {
+                    int numBytesToCopy = Math.Min(bytes.Length - (int)Position, count);
+                    Array.Copy(bytes, (int)Position, buffer, offset, numBytesToCopy);
+                    numBytesCopiedTotal += numBytesToCopy;
+                    offset += numBytesToCopy;
+                    count -= numBytesCopiedTotal;
+                    Position += numBytesToCopy;
+                    if (Position >= bytes.Length)
+                    {
+                        Position = 0;
+                        index++;
+                    }
+                }
+                return numBytesCopiedTotal;
+            }
+        }
     }
 }

csharp/src/Google.Protobuf.Test/JsonParserTest.cs

@@ -695,6 +695,22 @@ namespace Google.Protobuf
             Assert.AreEqual(Value.ForList(Value.ForNumber(1), Value.ForString("x")), Value.Parser.ParseJson("[1, \"x\"]"));
         }
 
+        [Test]
+        public void Value_List_WithNullElement()
+        {
+            var expected = Value.ForList(Value.ForString("x"), Value.ForNull(), Value.ForString("y"));
+            var actual = Value.Parser.ParseJson("[\"x\", null, \"y\"]");
+            Assert.AreEqual(expected, actual);
+        }
+
+        [Test]
+        public void StructValue_NullElement()
+        {
+            var expected = Value.ForStruct(new Struct { Fields = { { "x", Value.ForNull() } } });
+            var actual = Value.Parser.ParseJson("{ \"x\": null }");
+            Assert.AreEqual(expected, actual);
+        }
+
         [Test]
         public void ParseListValue()
         {

csharp/src/Google.Protobuf/CodedInputStream.cs

@@ -94,7 +94,7 @@ namespace Google.Protobuf
         private bool hasNextTag = false;
 
         internal const int DefaultRecursionLimit = 64;
-        internal const int DefaultSizeLimit = 64 << 20; // 64MB
+        internal const int DefaultSizeLimit = Int32.MaxValue;
         internal const int BufferSize = 4096;
 
         /// <summary>
@@ -248,7 +248,7 @@ namespace Google.Protobuf
         /// <remarks>
         /// This limit is applied when reading from the underlying stream, as a sanity check. It is
         /// not applied when reading from a byte array data source without an underlying stream.
-        /// The default value is 64MB.
+        /// The default value is Int32.MaxValue.
         /// </remarks>
         /// <value>
         /// The size limit.
@@ -1058,7 +1058,7 @@ namespace Google.Protobuf
                 RecomputeBufferSizeAfterLimit();
                 int totalBytesRead =
                     totalBytesRetired + bufferSize + bufferSizeAfterLimit;
-                if (totalBytesRead > sizeLimit || totalBytesRead < 0)
+                if (totalBytesRead < 0 || totalBytesRead > sizeLimit)
                 {
                     throw InvalidProtocolBufferException.SizeLimitExceeded();
                 }

csharp/src/Google.Protobuf/JsonParser.cs

@@ -264,11 +264,12 @@ namespace Google.Protobuf
                     return;
                 }
                 tokenizer.PushBack(token);
-                if (token.Type == JsonToken.TokenType.Null)
+                object value = ParseSingleValue(field, tokenizer);
+                if (value == null)
                 {
                     throw new InvalidProtocolBufferException("Repeated field elements cannot be null");
                 }
-                list.Add(ParseSingleValue(field, tokenizer));
+                list.Add(value);
             }
         }
 

csharp/src/Google.Protobuf/WellKnownTypes/FieldMask.cs

@@ -244,8 +244,8 @@ namespace Google.Protobuf.WellKnownTypes {
   ///
   /// ## Field Mask Verification
   ///
-  /// The implementation of the all the API methods, which have any FieldMask type
-  /// field in the request, should verify the included field paths, and return
+  /// The implementation of any API method which has a FieldMask type field in the
+  /// request should verify the included field paths, and return an
   /// `INVALID_ARGUMENT` error if any path is duplicated or unmappable.
   /// </summary>
   public sealed partial class FieldMask : pb::IMessage<FieldMask> {

java/core/generate-test-sources-build.xml

@@ -39,6 +39,7 @@
         <arg value="${test.proto.dir}/com/google/protobuf/field_presence_test.proto"/>
         <arg value="${test.proto.dir}/com/google/protobuf/map_for_proto2_lite_test.proto"/>
         <arg value="${test.proto.dir}/com/google/protobuf/map_for_proto2_test.proto"/>
+        <arg value="${test.proto.dir}/com/google/protobuf/map_lite_test.proto"/>
         <arg value="${test.proto.dir}/com/google/protobuf/map_test.proto"/>
         <arg value="${test.proto.dir}/com/google/protobuf/map_initialization_order_test.proto"/>
     </exec>

java/core/src/main/java/com/google/protobuf/AbstractMessage.java

@@ -124,6 +124,16 @@ public abstract class AbstractMessage
 
   protected int memoizedSize = -1;
 
+  @Override
+  int getMemoizedSerializedSize() {
+    return memoizedSize;
+  }
+
+  @Override
+  void setMemoizedSerializedSize(int size) {
+    memoizedSize = size;
+  }
+
   @Override
   public int getSerializedSize() {
     int size = memoizedSize;

java/core/src/main/java/com/google/protobuf/AbstractMessageLite.java

@@ -99,6 +99,16 @@ public abstract class AbstractMessageLite<
     codedOutput.flush();
   }
 
+  // We'd like these to be abstract but some folks are extending this class directly. They shouldn't
+  // be doing that and they should feel bad.
+  int getMemoizedSerializedSize() {
+    throw new UnsupportedOperationException();
+  }
+
+  void setMemoizedSerializedSize(int size) {
+    throw new UnsupportedOperationException();
+  }
+
 
   /**
    * Package private helper method for AbstractParser to create

java/core/src/main/java/com/google/protobuf/BooleanArrayList.java

@@ -81,6 +81,18 @@ final class BooleanArrayList extends AbstractProtobufList<Boolean>
     this.size = size;
   }
 
+  @Override
+  protected void removeRange(int fromIndex, int toIndex) {
+    ensureIsMutable();
+    if (toIndex < fromIndex) {
+      throw new IndexOutOfBoundsException("toIndex < fromIndex");
+    }
+
+    System.arraycopy(array, toIndex, array, fromIndex, size - toIndex);
+    size -= (toIndex - fromIndex);
+    modCount++;
+  }
+
   @Override
   public boolean equals(Object o) {
     if (this == o) {
@@ -246,7 +258,9 @@ final class BooleanArrayList extends AbstractProtobufList<Boolean>
     ensureIsMutable();
     ensureIndexInRange(index);
     boolean value = array[index];
-    System.arraycopy(array, index + 1, array, index, size - index);
+    if (index < size - 1) {
+      System.arraycopy(array, index + 1, array, index, size - index);
+    }
     size--;
     modCount++;
     return value;

java/core/src/main/java/com/google/protobuf/CodedInputStream.java

@@ -66,11 +66,9 @@ public abstract class CodedInputStream {
 
   /**
    * Whether to enable our custom UTF-8 decode codepath which does not use {@link StringCoding}.
-   * Enabled by default, disable by setting
-   * {@code -Dcom.google.protobuf.enableCustomutf8Decode=false} in JVM args.
+   * Currently disabled.
    */
-  private static final boolean ENABLE_CUSTOM_UTF8_DECODE
-      = !"false".equals(System.getProperty("com.google.protobuf.enableCustomUtf8Decode"));
+  private static final boolean ENABLE_CUSTOM_UTF8_DECODE = false;
 
   /** Visible for subclasses. See setRecursionLimit() */
   int recursionDepth;

java/core/src/main/java/com/google/protobuf/CodedOutputStream.java

@@ -377,6 +377,7 @@ public abstract class CodedOutputStream extends ByteOutput {
   public abstract void writeMessage(final int fieldNumber, final MessageLite value)
       throws IOException;
 
+
   /**
    * Write a MessageSet extension field to the stream.  For historical reasons,
    * the wire format differs from normal fields.
@@ -481,6 +482,7 @@ public abstract class CodedOutputStream extends ByteOutput {
   // Abstract to avoid overhead of additional virtual method calls.
   public abstract void writeMessageNoTag(final MessageLite value) throws IOException;
 
+
   //=================================================================
 
   @ExperimentalApi
@@ -666,6 +668,7 @@ public abstract class CodedOutputStream extends ByteOutput {
     return computeTagSize(fieldNumber) + computeMessageSizeNoTag(value);
   }
 
+
   /**
    * Compute the number of bytes that would be needed to encode a
    * MessageSet extension to the stream.  For historical reasons,
@@ -913,6 +916,7 @@ public abstract class CodedOutputStream extends ByteOutput {
     return computeLengthDelimitedFieldSize(value.getSerializedSize());
   }
 
+
   static int computeLengthDelimitedFieldSize(int fieldLength) {
     return computeUInt32SizeNoTag(fieldLength) + fieldLength;
   }
@@ -1049,6 +1053,7 @@ public abstract class CodedOutputStream extends ByteOutput {
     writeTag(fieldNumber, WireFormat.WIRETYPE_END_GROUP);
   }
 
+
   /**
    * Write a {@code group} field to the stream.
    *
@@ -1059,6 +1064,7 @@ public abstract class CodedOutputStream extends ByteOutput {
     value.writeTo(this);
   }
 
+
   /**
    * Compute the number of bytes that would be needed to encode a
    * {@code group} field, including tag.
@@ -1070,6 +1076,7 @@ public abstract class CodedOutputStream extends ByteOutput {
     return computeTagSize(fieldNumber) * 2 + computeGroupSizeNoTag(value);
   }
 
+
   /**
    * Compute the number of bytes that would be needed to encode a
    * {@code group} field.
@@ -1079,6 +1086,7 @@ public abstract class CodedOutputStream extends ByteOutput {
     return value.getSerializedSize();
   }
 
+
   /**
    * Encode and write a varint.  {@code value} is treated as
    * unsigned, so it won't be sign-extended if negative.
@@ -1273,6 +1281,7 @@ public abstract class CodedOutputStream extends ByteOutput {
       writeMessageNoTag(value);
     }
 
+
     @Override
     public final void writeMessageSetExtension(final int fieldNumber, final MessageLite value)
         throws IOException {
@@ -1297,6 +1306,7 @@ public abstract class CodedOutputStream extends ByteOutput {
       value.writeTo(this);
     }
 
+
     @Override
     public final void write(byte value) throws IOException {
       try {
@@ -1608,6 +1618,7 @@ public abstract class CodedOutputStream extends ByteOutput {
       writeMessageNoTag(value);
     }
 
+
     @Override
     public void writeMessageSetExtension(final int fieldNumber, final MessageLite value)
         throws IOException {
@@ -1632,6 +1643,7 @@ public abstract class CodedOutputStream extends ByteOutput {
       value.writeTo(this);
     }
 
+
     @Override
     public void write(byte value) throws IOException {
       try {
@@ -1928,6 +1940,7 @@ public abstract class CodedOutputStream extends ByteOutput {
       writeMessageNoTag(value);
     }
 
+
     @Override
     public void writeMessageSetExtension(int fieldNumber, MessageLite value) throws IOException {
       writeTag(WireFormat.MESSAGE_SET_ITEM, WireFormat.WIRETYPE_START_GROUP);
@@ -1950,6 +1963,7 @@ public abstract class CodedOutputStream extends ByteOutput {
       value.writeTo(this);
     }
 
+
     @Override
     public void write(byte value) throws IOException {
       if (position >= limit) {
@@ -2456,6 +2470,7 @@ public abstract class CodedOutputStream extends ByteOutput {
       writeMessageNoTag(value);
     }
 
+
     @Override
     public void writeMessageSetExtension(final int fieldNumber, final MessageLite value)
         throws IOException {
@@ -2480,6 +2495,7 @@ public abstract class CodedOutputStream extends ByteOutput {
       value.writeTo(this);
     }
 
+
     @Override
     public void write(byte value) throws IOException {
       if (position == limit) {
@@ -2759,6 +2775,7 @@ public abstract class CodedOutputStream extends ByteOutput {
       writeMessageNoTag(value);
     }
 
+
     @Override
     public void writeMessageSetExtension(final int fieldNumber, final MessageLite value)
         throws IOException {
@@ -2783,6 +2800,7 @@ public abstract class CodedOutputStream extends ByteOutput {
       value.writeTo(this);
     }
 
+
     @Override
     public void write(byte value) throws IOException {
       if (position == limit) {

java/core/src/main/java/com/google/protobuf/DoubleArrayList.java

@@ -81,6 +81,18 @@ final class DoubleArrayList extends AbstractProtobufList<Double>
     this.size = size;
   }
 
+  @Override
+  protected void removeRange(int fromIndex, int toIndex) {
+    ensureIsMutable();
+    if (toIndex < fromIndex) {
+      throw new IndexOutOfBoundsException("toIndex < fromIndex");
+    }
+
+    System.arraycopy(array, toIndex, array, fromIndex, size - toIndex);
+    size -= (toIndex - fromIndex);
+    modCount++;
+  }
+
   @Override
   public boolean equals(Object o) {
     if (this == o) {
@@ -247,7 +259,9 @@ final class DoubleArrayList extends AbstractProtobufList<Double>
     ensureIsMutable();
     ensureIndexInRange(index);
     double value = array[index];
-    System.arraycopy(array, index + 1, array, index, size - index);
+    if (index < size - 1) {
+      System.arraycopy(array, index + 1, array, index, size - index);
+    }
     size--;
     modCount++;
     return value;

java/core/src/main/java/com/google/protobuf/DynamicMessage.java

@@ -339,6 +339,20 @@ public final class DynamicMessage extends AbstractMessage {
       this.fields = FieldSet.newFieldSet();
       this.unknownFields = UnknownFieldSet.getDefaultInstance();
       this.oneofCases = new FieldDescriptor[type.toProto().getOneofDeclCount()];
+      // A MapEntry has all of its fields present at all times.
+      if (type.getOptions().getMapEntry()) {
+        populateMapEntry();
+      }
+    }
+
+    private void populateMapEntry() {
+      for (FieldDescriptor field : type.getFields()) {
+        if (field.getJavaType() == FieldDescriptor.JavaType.MESSAGE) {
+          fields.setField(field, getDefaultInstance(field.getMessageType()));
+        } else {
+          fields.setField(field, field.getDefaultValue());
+        }
+      }
     }
 
     // ---------------------------------------------------------------
@@ -351,6 +365,10 @@ public final class DynamicMessage extends AbstractMessage {
       } else {
         fields.clear();
       }
+      // A MapEntry has all of its fields present at all times.
+      if (type.getOptions().getMapEntry()) {
+        populateMapEntry();
+      }
       unknownFields = UnknownFieldSet.getDefaultInstance();
       return this;
     }

java/core/src/main/java/com/google/protobuf/FloatArrayList.java

@@ -81,6 +81,18 @@ final class FloatArrayList extends AbstractProtobufList<Float>
     this.size = size;
   }
 
+  @Override
+  protected void removeRange(int fromIndex, int toIndex) {
+    ensureIsMutable();
+    if (toIndex < fromIndex) {
+      throw new IndexOutOfBoundsException("toIndex < fromIndex");
+    }
+
+    System.arraycopy(array, toIndex, array, fromIndex, size - toIndex);
+    size -= (toIndex - fromIndex);
+    modCount++;
+  }
+
   @Override
   public boolean equals(Object o) {
     if (this == o) {
@@ -246,7 +258,9 @@ final class FloatArrayList extends AbstractProtobufList<Float>
     ensureIsMutable();
     ensureIndexInRange(index);
     float value = array[index];
-    System.arraycopy(array, index + 1, array, index, size - index);
+    if (index < size - 1) {
+      System.arraycopy(array, index + 1, array, index, size - index);
+    }
     size--;
     modCount++;
     return value;

java/core/src/main/java/com/google/protobuf/GeneratedMessageLite.java

@@ -230,9 +230,13 @@ public abstract class GeneratedMessageLite<
    * Called by subclasses to complete parsing. For use by generated code only.
    */
   protected void makeImmutable() {
+    // BEGIN REGULAR
     dynamicMethod(MethodToInvoke.MAKE_IMMUTABLE);
-
     unknownFields.makeImmutable();
+    // END REGULAR
+    // BEGIN EXPERIMENTAL
+    // Protobuf.getInstance().schemaFor(this).makeImmutable(this);
+    // END EXPERIMENTAL
   }
 
   protected final <
@@ -269,15 +273,15 @@ public abstract class GeneratedMessageLite<
    * For use by generated code only.
    */
   public static enum MethodToInvoke {
-    IS_INITIALIZED,
     // BEGIN REGULAR
+    IS_INITIALIZED,
     VISIT,
+    MERGE_FROM_STREAM,
+    MAKE_IMMUTABLE,
     // END REGULAR
     // Rely on/modify instance state
     GET_MEMOIZED_IS_INITIALIZED,
     SET_MEMOIZED_IS_INITIALIZED,
-    MERGE_FROM_STREAM,
-    MAKE_IMMUTABLE,
 
     // Rely on static state
     NEW_MUTABLE_INSTANCE,
@@ -339,6 +343,16 @@ public abstract class GeneratedMessageLite<
   }
   // END REGULAR
 
+  @Override
+  int getMemoizedSerializedSize() {
+    return memoizedSerializedSize;
+  }
+
+  @Override
+  void setMemoizedSerializedSize(int size) {
+    memoizedSerializedSize = size;
+  }
+
 
 
   /**
@@ -448,6 +462,28 @@ public abstract class GeneratedMessageLite<
       return defaultInstance;
     }
 
+    @Override
+    public BuilderType mergeFrom(byte[] input, int offset, int length)
+        throws InvalidProtocolBufferException {
+      // BEGIN REGULAR
+      return super.mergeFrom(input, offset, length);
+      // END REGULAR
+      // BEGIN EXPERIMENTAL
+      // copyOnWrite();
+      // try {
+      //   Protobuf.getInstance().schemaFor(instance).mergeFrom(
+      //       instance, input, offset, offset + length, new ArrayDecoders.Registers());
+      // } catch (InvalidProtocolBufferException e) {
+      //   throw e;
+      // } catch (IndexOutOfBoundsException e) {
+      //   throw InvalidProtocolBufferException.truncatedMessage();
+      // } catch (IOException e) {
+      //   throw new RuntimeException("Reading from byte array should not throw IOException.", e);
+      // }
+      // return (BuilderType) this;
+      // END EXPERIMENTAL
+    }
+
     @Override
     public BuilderType mergeFrom(
         com.google.protobuf.CodedInputStream input,
@@ -455,7 +491,13 @@ public abstract class GeneratedMessageLite<
         throws IOException {
       copyOnWrite();
       try {
+        // BEGIN REGULAR
         instance.dynamicMethod(MethodToInvoke.MERGE_FROM_STREAM, input, extensionRegistry);
+        // END REGULAR
+        // BEGIN EXPERIMENTAL
+        // Protobuf.getInstance().schemaFor(instance).mergeFrom(
+        //     instance, CodedInputStreamReader.forCodedInput(input), extensionRegistry);
+        // END EXPERIMENTAL
       } catch (RuntimeException e) {
         if (e.getCause() instanceof IOException) {
           throw (IOException) e.getCause();
@@ -576,9 +618,7 @@ public abstract class GeneratedMessageLite<
         return parseUnknownField(tag, input);
       }
 
-      if (extensions.isImmutable()) {
-        extensions = extensions.clone();
-      }
+      ensureExtensionsAreMutable();
 
       if (packed) {
         int length = input.readRawVarint32();
@@ -794,10 +834,18 @@ public abstract class GeneratedMessageLite<
       if (subBuilder == null) {
         subBuilder = extension.getMessageDefaultInstance().newBuilderForType();
       }
-      rawBytes.newCodedInput().readMessage(subBuilder, extensionRegistry);
+      subBuilder.mergeFrom(rawBytes, extensionRegistry);
       MessageLite value = subBuilder.build();
 
-      extensions.setField(extension.descriptor, extension.singularToFieldSetType(value));
+      ensureExtensionsAreMutable().setField(
+          extension.descriptor, extension.singularToFieldSetType(value));
+    }
+
+    private FieldSet<ExtensionDescriptor> ensureExtensionsAreMutable() {
+      if (extensions.isImmutable()) {
+        extensions = extensions.clone();
+      }
+      return extensions;
     }
 
     private void verifyExtensionContainingType(
@@ -869,10 +917,12 @@ public abstract class GeneratedMessageLite<
     @Override
     protected final void makeImmutable() {
       super.makeImmutable();
-
+      // BEGIN REGULAR
       extensions.makeImmutable();
+      // END REGULAR
     }
 
+
     /**
      * Used by subclasses to serialize extensions.  Extension ranges may be
      * interleaved with field numbers, but we must write them in canonical
@@ -1468,8 +1518,13 @@ public abstract class GeneratedMessageLite<
     if (memoizedIsInitialized == 0) {
       return false;
     }
+    // BEGIN EXPERIMENTAL
+    // boolean isInitialized = Protobuf.getInstance().schemaFor(message).isInitialized(message);
+    // END EXPERIMENTAL
+    // BEGIN REGULAR
     boolean isInitialized =
         message.dynamicMethod(MethodToInvoke.IS_INITIALIZED, Boolean.FALSE) != null;
+    // END REGULAR
     if (shouldMemoize) {
       message.dynamicMethod(
           MethodToInvoke.SET_MEMOIZED_IS_INITIALIZED, isInitialized ? message : null);
@@ -1477,10 +1532,6 @@ public abstract class GeneratedMessageLite<
     return isInitialized;
   }
 
-  protected static final <T extends GeneratedMessageLite<T, ?>> void makeImmutable(T message) {
-    message.dynamicMethod(MethodToInvoke.MAKE_IMMUTABLE);
-  }
-
   protected static IntList emptyIntList() {
     return IntArrayList.emptyList();
   }
@@ -1560,6 +1611,11 @@ public abstract class GeneratedMessageLite<
         throws InvalidProtocolBufferException {
       return GeneratedMessageLite.parsePartialFrom(defaultInstance, input, extensionRegistry);
     }
+
+    @Override
+    public T parsePartialFrom(byte[] input) throws InvalidProtocolBufferException {
+      return GeneratedMessageLite.parsePartialFrom(defaultInstance, input);
+    }
   }
 
   /**
@@ -1573,8 +1629,21 @@ public abstract class GeneratedMessageLite<
     @SuppressWarnings("unchecked") // Guaranteed by protoc
     T result = (T) instance.dynamicMethod(MethodToInvoke.NEW_MUTABLE_INSTANCE);
     try {
+      // BEGIN REGULAR
       result.dynamicMethod(MethodToInvoke.MERGE_FROM_STREAM, input, extensionRegistry);
+      // END REGULAR
+      // BEGIN EXPERIMENTAL
+      // Protobuf.getInstance().schemaFor(result).mergeFrom(
+      //     result, CodedInputStreamReader.forCodedInput(input), extensionRegistry);
+      // END EXPERIMENTAL
       result.makeImmutable();
+      // BEGIN EXPERIMENTAL
+      // } catch (IOException e) {
+      //   if (e.getCause() instanceof InvalidProtocolBufferException) {
+      //     throw (InvalidProtocolBufferException) e.getCause();
+      //   }
+      //   throw new InvalidProtocolBufferException(e.getMessage()).setUnfinishedMessage(result);
+      // END EXPERIMENTAL
     } catch (RuntimeException e) {
       if (e.getCause() instanceof InvalidProtocolBufferException) {
         throw (InvalidProtocolBufferException) e.getCause();
@@ -1584,6 +1653,34 @@ public abstract class GeneratedMessageLite<
     return result;
   }
 
+  /** A static helper method for parsing a partial from byte array. */
+  static <T extends GeneratedMessageLite<T, ?>> T parsePartialFrom(T instance, byte[] input)
+      throws InvalidProtocolBufferException {
+    // BEGIN REGULAR
+    return parsePartialFrom(instance, input, ExtensionRegistryLite.getEmptyRegistry());
+    // END REGULAR
+    // BEGIN EXPERIMENTAL
+    // @SuppressWarnings("unchecked") // Guaranteed by protoc
+    // T result = (T) instance.dynamicMethod(MethodToInvoke.NEW_MUTABLE_INSTANCE);
+    // try {
+    //   Protobuf.getInstance().schemaFor(result).mergeFrom(
+    //       result, input, 0, input.length, new ArrayDecoders.Registers());
+    //   result.makeImmutable();
+    //   if (result.memoizedHashCode != 0) {
+    //     throw new RuntimeException();
+    //   }
+    // } catch (IOException e) {
+    //   if (e.getCause() instanceof InvalidProtocolBufferException) {
+    //     throw (InvalidProtocolBufferException) e.getCause();
+    //   }
+    //   throw new InvalidProtocolBufferException(e.getMessage()).setUnfinishedMessage(result);
+    // } catch (IndexOutOfBoundsException e) {
+    //   throw InvalidProtocolBufferException.truncatedMessage().setUnfinishedMessage(result);
+    // }
+    // return result;
+    // END EXPERIMENTAL
+  }
+
   protected static <T extends GeneratedMessageLite<T, ?>> T parsePartialFrom(
       T defaultInstance,
       CodedInputStream input)
@@ -1680,8 +1777,7 @@ public abstract class GeneratedMessageLite<
   protected static <T extends GeneratedMessageLite<T, ?>> T parseFrom(
       T defaultInstance, byte[] data)
       throws InvalidProtocolBufferException {
-    return checkMessageInitialized(
-        parsePartialFrom(defaultInstance, data, ExtensionRegistryLite.getEmptyRegistry()));
+    return checkMessageInitialized(parsePartialFrom(defaultInstance, data));
   }
 
   // Validates last tag.

java/core/src/main/java/com/google/protobuf/IntArrayList.java

@@ -81,6 +81,18 @@ final class IntArrayList extends AbstractProtobufList<Integer>
     this.size = size;
   }
 
+  @Override
+  protected void removeRange(int fromIndex, int toIndex) {
+    ensureIsMutable();
+    if (toIndex < fromIndex) {
+      throw new IndexOutOfBoundsException("toIndex < fromIndex");
+    }
+
+    System.arraycopy(array, toIndex, array, fromIndex, size - toIndex);
+    size -= (toIndex - fromIndex);
+    modCount++;
+  }
+
   @Override
   public boolean equals(Object o) {
     if (this == o) {
@@ -246,7 +258,9 @@ final class IntArrayList extends AbstractProtobufList<Integer>
     ensureIsMutable();
     ensureIndexInRange(index);
     int value = array[index];
-    System.arraycopy(array, index + 1, array, index, size - index);
+    if (index < size - 1) {
+      System.arraycopy(array, index + 1, array, index, size - index);
+    }
     size--;
     modCount++;
     return value;

java/core/src/main/java/com/google/protobuf/LongArrayList.java

@@ -81,6 +81,18 @@ final class LongArrayList extends AbstractProtobufList<Long>
     this.size = size;
   }
 
+  @Override
+  protected void removeRange(int fromIndex, int toIndex) {
+    ensureIsMutable();
+    if (toIndex < fromIndex) {
+      throw new IndexOutOfBoundsException("toIndex < fromIndex");
+    }
+
+    System.arraycopy(array, toIndex, array, fromIndex, size - toIndex);
+    size -= (toIndex - fromIndex);
+    modCount++;
+  }
+
   @Override
   public boolean equals(Object o) {
     if (this == o) {
@@ -246,7 +258,9 @@ final class LongArrayList extends AbstractProtobufList<Long>
     ensureIsMutable();
     ensureIndexInRange(index);
     long value = array[index];
-    System.arraycopy(array, index + 1, array, index, size - index);
+    if (index < size - 1) {
+      System.arraycopy(array, index + 1, array, index, size - index);
+    }
     size--;
     modCount++;
     return value;

java/core/src/main/java/com/google/protobuf/MessageLiteToString.java

@@ -31,6 +31,7 @@
 package com.google.protobuf;
 
 import java.lang.reflect.Method;
+import java.lang.reflect.Modifier;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
@@ -38,20 +39,18 @@ import java.util.Map;
 import java.util.Set;
 import java.util.TreeSet;
 
-/**
- * Helps generate {@link String} representations of {@link MessageLite} protos.
- */
-// TODO(dweis): Fix map fields.
+/** Helps generate {@link String} representations of {@link MessageLite} protos. */
 final class MessageLiteToString {
 
   private static final String LIST_SUFFIX = "List";
   private static final String BUILDER_LIST_SUFFIX = "OrBuilderList";
+  private static final String MAP_SUFFIX = "Map";
   private static final String BYTES_SUFFIX = "Bytes";
-  
+
   /**
-   * Returns a {@link String} representation of the {@link MessageLite} object.  The first line of
+   * Returns a {@link String} representation of the {@link MessageLite} object. The first line of
    * the {@code String} representation representation includes a comment string to uniquely identify
-   * the objcet instance. This acts as an indicator that this should not be relied on for
+   * the object instance. This acts as an indicator that this should not be relied on for
    * comparisons.
    *
    * <p>For use by generated code only.
@@ -71,8 +70,9 @@ final class MessageLiteToString {
    */
   private static void reflectivePrintWithIndent(
       MessageLite messageLite, StringBuilder buffer, int indent) {
-    // Build a map of method name to method. We're looking for methods like getFoo(), hasFoo(), and
-    // getFooList() which might be useful for building an object's string representation.
+    // Build a map of method name to method. We're looking for methods like getFoo(), hasFoo(),
+    // getFooList() and getFooMap() which might be useful for building an object's string
+    // representation.
     Map<String, Method> nameToNoArgMethod = new HashMap<String, Method>();
     Map<String, Method> nameToMethod = new HashMap<String, Method>();
     Set<String> getters = new TreeSet<String>();
@@ -89,12 +89,16 @@ final class MessageLiteToString {
 
     for (String getter : getters) {
       String suffix = getter.replaceFirst("get", "");
-      if (suffix.endsWith(LIST_SUFFIX) && !suffix.endsWith(BUILDER_LIST_SUFFIX)) {
-        String camelCase = suffix.substring(0, 1).toLowerCase()
-            + suffix.substring(1, suffix.length() - LIST_SUFFIX.length());
+      if (suffix.endsWith(LIST_SUFFIX)
+          && !suffix.endsWith(BUILDER_LIST_SUFFIX)
+          // Sometimes people have fields named 'list' that aren't repeated.
+          && !suffix.equals(LIST_SUFFIX)) {
+        String camelCase =
+            suffix.substring(0, 1).toLowerCase()
+                + suffix.substring(1, suffix.length() - LIST_SUFFIX.length());
         // Try to reflectively get the value and toString() the field as if it were repeated. This
-        // only works if the method names have not be proguarded out or renamed.
-        Method listMethod = nameToNoArgMethod.get("get" + suffix);
+        // only works if the method names have not been proguarded out or renamed.
+        Method listMethod = nameToNoArgMethod.get(getter);
         if (listMethod != null && listMethod.getReturnType().equals(List.class)) {
           printField(
               buffer,
@@ -104,6 +108,30 @@ final class MessageLiteToString {
           continue;
         }
       }
+      if (suffix.endsWith(MAP_SUFFIX)
+          // Sometimes people have fields named 'map' that aren't maps.
+          && !suffix.equals(MAP_SUFFIX)) {
+        String camelCase =
+            suffix.substring(0, 1).toLowerCase()
+                + suffix.substring(1, suffix.length() - MAP_SUFFIX.length());
+        // Try to reflectively get the value and toString() the field as if it were a map. This only
+        // works if the method names have not been proguarded out or renamed.
+        Method mapMethod = nameToNoArgMethod.get(getter);
+        if (mapMethod != null
+            && mapMethod.getReturnType().equals(Map.class)
+            // Skip the deprecated getter method with no prefix "Map" when the field name ends with
+            // "map".
+            && !mapMethod.isAnnotationPresent(Deprecated.class)
+            // Skip the internal mutable getter method.
+            && Modifier.isPublic(mapMethod.getModifiers())) {
+          printField(
+              buffer,
+              indent,
+              camelCaseToSnakeCase(camelCase),
+              GeneratedMessageLite.invokeOrDie(mapMethod, messageLite));
+          continue;
+        }
+      }
 
       Method setter = nameToMethod.get("set" + suffix);
       if (setter == null) {
@@ -119,22 +147,19 @@ final class MessageLiteToString {
       String camelCase = suffix.substring(0, 1).toLowerCase() + suffix.substring(1);
 
       // Try to reflectively get the value and toString() the field as if it were optional. This
-      // only works if the method names have not be proguarded out or renamed.
+      // only works if the method names have not been proguarded out or renamed.
       Method getMethod = nameToNoArgMethod.get("get" + suffix);
       Method hasMethod = nameToNoArgMethod.get("has" + suffix);
       // TODO(dweis): Fix proto3 semantics.
       if (getMethod != null) {
         Object value = GeneratedMessageLite.invokeOrDie(getMethod, messageLite);
-        final boolean hasValue = hasMethod == null
-            ? !isDefaultValue(value)
-            : (Boolean) GeneratedMessageLite.invokeOrDie(hasMethod, messageLite);
-         // TODO(dweis): This doesn't stop printing oneof case twice: value and enum style.
+        final boolean hasValue =
+            hasMethod == null
+                ? !isDefaultValue(value)
+                : (Boolean) GeneratedMessageLite.invokeOrDie(hasMethod, messageLite);
+        // TODO(dweis): This doesn't stop printing oneof case twice: value and enum style.
         if (hasValue) {
-          printField(
-              buffer,
-              indent,
-              camelCaseToSnakeCase(camelCase),
-              value);
+          printField(buffer, indent, camelCaseToSnakeCase(camelCase), value);
         }
         continue;
       }
@@ -153,7 +178,7 @@ final class MessageLiteToString {
       ((GeneratedMessageLite<?, ?>) messageLite).unknownFields.printWithIndent(buffer, indent);
     }
   }
-  
+
   private static boolean isDefaultValue(Object o) {
     if (o instanceof Boolean) {
       return !((Boolean) o);
@@ -179,7 +204,7 @@ final class MessageLiteToString {
     if (o instanceof java.lang.Enum<?>) { // Catches oneof enums.
       return ((java.lang.Enum<?>) o).ordinal() == 0;
     }
-    
+
     return false;
   }
 
@@ -201,6 +226,13 @@ final class MessageLiteToString {
       }
       return;
     }
+    if (object instanceof Map<?, ?>) {
+      Map<?, ?> map = (Map<?, ?>) object;
+      for (Map.Entry<?, ?> entry : map.entrySet()) {
+        printField(buffer, indent, name, entry);
+      }
+      return;
+    }
 
     buffer.append('\n');
     for (int i = 0; i < indent; i++) {
@@ -220,11 +252,21 @@ final class MessageLiteToString {
         buffer.append(' ');
       }
       buffer.append("}");
+    } else if (object instanceof Map.Entry<?, ?>) {
+      buffer.append(" {");
+      Map.Entry<?, ?> entry = (Map.Entry<?, ?>) object;
+      printField(buffer, indent + 2, "key", entry.getKey());
+      printField(buffer, indent + 2, "value", entry.getValue());
+      buffer.append("\n");
+      for (int i = 0; i < indent; i++) {
+        buffer.append(' ');
+      }
+      buffer.append("}");
     } else {
       buffer.append(": ").append(object.toString());
     }
   }
-  
+
   private static final String camelCaseToSnakeCase(String camelCase) {
     StringBuilder builder = new StringBuilder();
     for (int i = 0; i < camelCase.length(); i++) {

java/core/src/main/java/com/google/protobuf/TextFormat.java

@@ -987,7 +987,7 @@ public final class TextFormat {
         nextToken();
         return false;
       } else {
-        throw parseException("Expected \"true\" or \"false\".");
+        throw parseException("Expected \"true\" or \"false\". Found \"" + currentToken + "\".");
       }
     }
 
@@ -1311,13 +1311,17 @@ public final class TextFormat {
     }
 
     private final boolean allowUnknownFields;
+    private final boolean allowUnknownEnumValues;
     private final SingularOverwritePolicy singularOverwritePolicy;
     private TextFormatParseInfoTree.Builder parseInfoTreeBuilder;
 
     private Parser(
-        boolean allowUnknownFields, SingularOverwritePolicy singularOverwritePolicy,
+        boolean allowUnknownFields,
+        boolean allowUnknownEnumValues,
+        SingularOverwritePolicy singularOverwritePolicy,
         TextFormatParseInfoTree.Builder parseInfoTreeBuilder) {
       this.allowUnknownFields = allowUnknownFields;
+      this.allowUnknownEnumValues = allowUnknownEnumValues;
       this.singularOverwritePolicy = singularOverwritePolicy;
       this.parseInfoTreeBuilder = parseInfoTreeBuilder;
     }
@@ -1334,6 +1338,7 @@ public final class TextFormat {
      */
     public static class Builder {
       private boolean allowUnknownFields = false;
+      private boolean allowUnknownEnumValues = false;
       private SingularOverwritePolicy singularOverwritePolicy =
           SingularOverwritePolicy.ALLOW_SINGULAR_OVERWRITES;
       private TextFormatParseInfoTree.Builder parseInfoTreeBuilder = null;
@@ -1355,7 +1360,10 @@ public final class TextFormat {
 
       public Parser build() {
         return new Parser(
-            allowUnknownFields, singularOverwritePolicy, parseInfoTreeBuilder);
+            allowUnknownFields,
+            allowUnknownEnumValues,
+            singularOverwritePolicy,
+            parseInfoTreeBuilder);
       }
     }
 
@@ -1419,7 +1427,7 @@ public final class TextFormat {
       return text;
     }
 
-    // Check both unknown fields and unknown extensions and log warming messages
+    // Check both unknown fields and unknown extensions and log warning messages
     // or throw exceptions according to the flag.
     private void checkUnknownFields(final List<String> unknownFields)
         throws ParseException {
@@ -1737,17 +1745,40 @@ public final class TextFormat {
               final int number = tokenizer.consumeInt32();
               value = enumType.findValueByNumber(number);
               if (value == null) {
-                throw tokenizer.parseExceptionPreviousToken(
-                  "Enum type \"" + enumType.getFullName()
-                  + "\" has no value with number " + number + '.');
+                String unknownValueMsg =
+                    "Enum type \""
+                        + enumType.getFullName()
+                        + "\" has no value with number "
+                        + number
+                        + '.';
+                if (allowUnknownEnumValues) {
+                  logger.warning(unknownValueMsg);
+                  return;
+                } else {
+                  throw tokenizer.parseExceptionPreviousToken(
+                      "Enum type \""
+                          + enumType.getFullName()
+                          + "\" has no value with number "
+                          + number
+                          + '.');
+                }
               }
             } else {
               final String id = tokenizer.consumeIdentifier();
               value = enumType.findValueByName(id);
               if (value == null) {
-                throw tokenizer.parseExceptionPreviousToken(
-                  "Enum type \"" + enumType.getFullName()
-                  + "\" has no value named \"" + id + "\".");
+                String unknownValueMsg =
+                    "Enum type \""
+                        + enumType.getFullName()
+                        + "\" has no value named \""
+                        + id
+                        + "\".";
+                if (allowUnknownEnumValues) {
+                  logger.warning(unknownValueMsg);
+                  return;
+                } else {
+                  throw tokenizer.parseExceptionPreviousToken(unknownValueMsg);
+                }
               }
             }
 

java/core/src/main/java/com/google/protobuf/UnknownFieldSetLite.java

@@ -295,14 +295,30 @@ public final class UnknownFieldSetLite {
     return true;
   }
 
+  private static int hashCode(int[] tags, int count) {
+    int hashCode = 17;
+    for (int i = 0; i < count; ++i) {
+      hashCode = 31 * hashCode + tags[i];
+    }
+    return hashCode;
+  }
+
+  private static int hashCode(Object[] objects, int count) {
+    int hashCode = 17;
+    for (int i = 0; i < count; ++i) {
+      hashCode = 31 * hashCode + objects[i].hashCode();
+    }
+    return hashCode;
+  }
+
   @Override
   public int hashCode() {
     int hashCode = 17;
-    
+
     hashCode = 31 * hashCode + count;
-    hashCode = 31 * hashCode + Arrays.hashCode(tags);
-    hashCode = 31 * hashCode + Arrays.deepHashCode(objects);
-    
+    hashCode = 31 * hashCode + hashCode(tags, count);
+    hashCode = 31 * hashCode + hashCode(objects, count);
+
     return hashCode;
   }
 

java/core/src/main/java/com/google/protobuf/UnsafeUtil.java

@@ -33,6 +33,7 @@ package com.google.protobuf;
 import java.lang.reflect.Field;
 import java.nio.Buffer;
 import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
 import java.security.AccessController;
 import java.security.PrivilegedExceptionAction;
 import java.util.logging.Level;
@@ -83,6 +84,7 @@ final class UnsafeUtil {
     return HAS_UNSAFE_BYTEBUFFER_OPERATIONS;
   }
 
+
   static long objectFieldOffset(Field field) {
     return MEMORY_ACCESSOR.objectFieldOffset(field);
   }
@@ -287,7 +289,7 @@ final class UnsafeUtil {
   /**
    * Gets the {@code sun.misc.Unsafe} instance, or {@code null} if not available on this platform.
    */
-  private static sun.misc.Unsafe getUnsafe() {
+  static sun.misc.Unsafe getUnsafe() {
     sun.misc.Unsafe unsafe = null;
     try {
       unsafe =
@@ -367,6 +369,10 @@ final class UnsafeUtil {
       clazz.getMethod("objectFieldOffset", Field.class);
       clazz.getMethod("getLong", Object.class, long.class);
 
+      if (bufferAddressField() == null) {
+        return false;
+      }
+
       clazz.getMethod("getByte", long.class);
       clazz.getMethod("putByte", long.class, byte.class);
       clazz.getMethod("getInt", long.class);
@@ -387,12 +393,14 @@ final class UnsafeUtil {
 
   /** Finds the address field within a direct {@link Buffer}. */
   private static Field bufferAddressField() {
-    return field(Buffer.class, "address", long.class);
+    Field field = field(Buffer.class, "address");
+    return field != null && field.getType() == long.class ? field : null;
   }
 
   /** Finds the value field within a {@link String}. */
   private static Field stringValueField() {
-    return field(String.class, "value", char[].class);
+    Field field = field(String.class, "value");
+    return field != null && field.getType() == char[].class ? field : null;
   }
 
   /**
@@ -407,14 +415,11 @@ final class UnsafeUtil {
    * Gets the field with the given name within the class, or {@code null} if not found. If found,
    * the field is made accessible.
    */
-  private static Field field(Class<?> clazz, String fieldName, Class<?> expectedType) {
+  private static Field field(Class<?> clazz, String fieldName) {
     Field field;
     try {
       field = clazz.getDeclaredField(fieldName);
       field.setAccessible(true);
-      if (!field.getType().equals(expectedType)) {
-        return null;
-      }
     } catch (Throwable t) {
       // Failed to access the fields.
       field = null;

java/core/src/test/java/com/google/protobuf/BooleanArrayListTest.java

@@ -32,6 +32,7 @@ package com.google.protobuf;
 
 import static java.util.Arrays.asList;
 
+import com.google.protobuf.Internal.BooleanList;
 import java.util.Collections;
 import java.util.ConcurrentModificationException;
 import java.util.Iterator;
@@ -297,6 +298,20 @@ public class BooleanArrayListTest extends TestCase {
     }
   }
 
+  public void testRemoveEndOfCapacity() {
+    BooleanList toRemove = BooleanArrayList.emptyList().mutableCopyWithCapacity(1);
+    toRemove.addBoolean(true);
+    toRemove.remove(0);
+    assertEquals(0, toRemove.size());
+  }
+
+  public void testSublistRemoveEndOfCapacity() {
+    BooleanList toRemove = BooleanArrayList.emptyList().mutableCopyWithCapacity(1);
+    toRemove.addBoolean(true);
+    toRemove.subList(0, 1).clear();
+    assertEquals(0, toRemove.size());
+  }
+
   private void assertImmutable(BooleanArrayList list) {
 
     try {

java/core/src/test/java/com/google/protobuf/CheckUtf8Test.java

@@ -34,7 +34,7 @@ import proto2_test_check_utf8.TestCheckUtf8.BytesWrapper;
 import proto2_test_check_utf8.TestCheckUtf8.StringWrapper;
 import proto2_test_check_utf8_size.TestCheckUtf8Size.BytesWrapperSize;
 import proto2_test_check_utf8_size.TestCheckUtf8Size.StringWrapperSize;
-
+import java.io.ByteArrayInputStream;
 import junit.framework.TestCase;
 
 /**
@@ -90,14 +90,9 @@ public class CheckUtf8Test extends TestCase {
   }
 
   public void testParseRequiredStringWithBadUtf8() throws Exception {
-    ByteString serialized =
-        BytesWrapper.newBuilder().setReq(NON_UTF8_BYTE_STRING).build().toByteString();
-    try {
-      StringWrapper.parser().parseFrom(serialized);
-      fail("Expected InvalidProtocolBufferException for non UTF-8 byte string.");
-    } catch (InvalidProtocolBufferException exception) {
-      assertEquals("Protocol message had invalid UTF-8.", exception.getMessage());
-    }
+    byte[] serialized =
+        BytesWrapper.newBuilder().setReq(NON_UTF8_BYTE_STRING).build().toByteArray();
+    assertParseBadUtf8(StringWrapper.getDefaultInstance(), serialized);
   }
 
   public void testBuildRequiredStringWithBadUtf8Size() throws Exception {
@@ -128,14 +123,36 @@ public class CheckUtf8Test extends TestCase {
   }
 
   public void testParseRequiredStringWithBadUtf8Size() throws Exception {
-    ByteString serialized =
-        BytesWrapperSize.newBuilder().setReq(NON_UTF8_BYTE_STRING).build().toByteString();
+    byte[] serialized =
+        BytesWrapperSize.newBuilder().setReq(NON_UTF8_BYTE_STRING).build().toByteArray();
+    assertParseBadUtf8(StringWrapperSize.getDefaultInstance(), serialized);
+  }
+
+  private void assertParseBadUtf8(MessageLite defaultInstance, byte[] data) throws Exception {
+    // Check combinations of (parser vs. builder) x (byte[] vs. InputStream)
+    try {
+      defaultInstance.getParserForType().parseFrom(data);
+      fail("Expected InvalidProtocolBufferException for non UTF-8 byte string.");
+    } catch (InvalidProtocolBufferException exception) {
+      assertEquals("Protocol message had invalid UTF-8.", exception.getMessage());
+    }
+    try {
+      defaultInstance.newBuilderForType().mergeFrom(data);
+      fail("Expected InvalidProtocolBufferException for non UTF-8 byte string.");
+    } catch (InvalidProtocolBufferException exception) {
+      assertEquals("Protocol message had invalid UTF-8.", exception.getMessage());
+    }
     try {
-      StringWrapperSize.parser().parseFrom(serialized);
+      defaultInstance.getParserForType().parseFrom(new ByteArrayInputStream(data));
+      fail("Expected InvalidProtocolBufferException for non UTF-8 byte string.");
+    } catch (InvalidProtocolBufferException exception) {
+      assertEquals("Protocol message had invalid UTF-8.", exception.getMessage());
+    }
+    try {
+      defaultInstance.newBuilderForType().mergeFrom(new ByteArrayInputStream(data));
       fail("Expected InvalidProtocolBufferException for non UTF-8 byte string.");
     } catch (InvalidProtocolBufferException exception) {
       assertEquals("Protocol message had invalid UTF-8.", exception.getMessage());
     }
   }
-
 }

java/core/src/test/java/com/google/protobuf/DoubleArrayListTest.java

@@ -32,6 +32,7 @@ package com.google.protobuf;
 
 import static java.util.Arrays.asList;
 
+import com.google.protobuf.Internal.DoubleList;
 import java.util.Collections;
 import java.util.ConcurrentModificationException;
 import java.util.Iterator;
@@ -297,6 +298,20 @@ public class DoubleArrayListTest extends TestCase {
     }
   }
 
+  public void testRemoveEndOfCapacity() {
+    DoubleList toRemove = DoubleArrayList.emptyList().mutableCopyWithCapacity(1);
+    toRemove.addDouble(3);
+    toRemove.remove(0);
+    assertEquals(0, toRemove.size());
+  }
+
+  public void testSublistRemoveEndOfCapacity() {
+    DoubleList toRemove = DoubleArrayList.emptyList().mutableCopyWithCapacity(1);
+    toRemove.addDouble(3);
+    toRemove.subList(0, 1).clear();
+    assertEquals(0, toRemove.size());
+  }
+
   private void assertImmutable(DoubleArrayList list) {
     if (list.contains(1D)) {
       throw new RuntimeException("Cannot test the immutability of lists that contain 1.");

java/core/src/test/java/com/google/protobuf/FloatArrayListTest.java

@@ -32,6 +32,7 @@ package com.google.protobuf;
 
 import static java.util.Arrays.asList;
 
+import com.google.protobuf.Internal.FloatList;
 import java.util.Collections;
 import java.util.ConcurrentModificationException;
 import java.util.Iterator;
@@ -297,6 +298,20 @@ public class FloatArrayListTest extends TestCase {
     }
   }
 
+  public void testRemoveEndOfCapacity() {
+    FloatList toRemove = FloatArrayList.emptyList().mutableCopyWithCapacity(1);
+    toRemove.addFloat(3);
+    toRemove.remove(0);
+    assertEquals(0, toRemove.size());
+  }
+
+  public void testSublistRemoveEndOfCapacity() {
+    FloatList toRemove = FloatArrayList.emptyList().mutableCopyWithCapacity(1);
+    toRemove.addFloat(3);
+    toRemove.subList(0, 1).clear();
+    assertEquals(0, toRemove.size());
+  }
+
   private void assertImmutable(FloatArrayList list) {
     if (list.contains(1F)) {
       throw new RuntimeException("Cannot test the immutability of lists that contain 1.");

java/core/src/test/java/com/google/protobuf/IntArrayListTest.java

@@ -32,6 +32,7 @@ package com.google.protobuf;
 
 import static java.util.Arrays.asList;
 
+import com.google.protobuf.Internal.IntList;
 import java.util.Collections;
 import java.util.ConcurrentModificationException;
 import java.util.Iterator;
@@ -297,6 +298,20 @@ public class IntArrayListTest extends TestCase {
     }
   }
 
+  public void testRemoveEndOfCapacity() {
+    IntList toRemove = IntArrayList.emptyList().mutableCopyWithCapacity(1);
+    toRemove.addInt(3);
+    toRemove.remove(0);
+    assertEquals(0, toRemove.size());
+  }
+
+  public void testSublistRemoveEndOfCapacity() {
+    IntList toRemove = IntArrayList.emptyList().mutableCopyWithCapacity(1);
+    toRemove.addInt(3);
+    toRemove.subList(0, 1).clear();
+    assertEquals(0, toRemove.size());
+  }
+
   private void assertImmutable(IntArrayList list) {
     if (list.contains(1)) {
       throw new RuntimeException("Cannot test the immutability of lists that contain 1.");

java/core/src/test/java/com/google/protobuf/LongArrayListTest.java

@@ -32,6 +32,7 @@ package com.google.protobuf;
 
 import static java.util.Arrays.asList;
 
+import com.google.protobuf.Internal.LongList;
 import java.util.Collections;
 import java.util.ConcurrentModificationException;
 import java.util.Iterator;
@@ -297,6 +298,20 @@ public class LongArrayListTest extends TestCase {
     }
   }
 
+  public void testRemoveEndOfCapacity() {
+    LongList toRemove = LongArrayList.emptyList().mutableCopyWithCapacity(1);
+    toRemove.addLong(3);
+    toRemove.remove(0);
+    assertEquals(0, toRemove.size());
+  }
+
+  public void testSublistRemoveEndOfCapacity() {
+    LongList toRemove = LongArrayList.emptyList().mutableCopyWithCapacity(1);
+    toRemove.addLong(3);
+    toRemove.subList(0, 1).clear();
+    assertEquals(0, toRemove.size());
+  }
+
   private void assertImmutable(LongArrayList list) {
     if (list.contains(1L)) {
       throw new RuntimeException("Cannot test the immutability of lists that contain 1.");

java/core/src/test/java/com/google/protobuf/MapForProto2Test.java

@@ -788,6 +788,24 @@ public class MapForProto2Test extends TestCase {
     assertEquals(message.hashCode(), dynamicMessage.hashCode());
   }
 
+  // Check that DynamicMessage handles map field serialization the same way as generated code
+  // regarding unset key and value field in a map entry.
+  public void testDynamicMessageUnsetKeyAndValue() throws Exception {
+    FieldDescriptor field = f("int32_to_int32_field");
+
+    Message dynamicDefaultInstance =
+        DynamicMessage.getDefaultInstance(TestMap.getDescriptor());
+    Message.Builder builder = dynamicDefaultInstance.newBuilderForType();
+    // Add an entry without key and value.
+    builder.addRepeatedField(field, builder.newBuilderForField(field).build());
+    Message message = builder.build();
+    ByteString bytes = message.toByteString();
+    // Parse it back to the same generated type.
+    Message generatedMessage = TestMap.parseFrom(bytes);
+    // Assert the serialized bytes are equivalent.
+    assertEquals(generatedMessage.toByteString(), bytes);
+  }
+
   public void testReflectionEqualsAndHashCode() throws Exception {
     // Test that generated equals() and hashCode() will disregard the order
     // of map entries when comparing/hashing map fields.

java/core/src/test/java/com/google/protobuf/MapTest.java

@@ -893,6 +893,24 @@ public class MapTest extends TestCase {
     assertEquals(message.hashCode(), dynamicMessage.hashCode());
   }
 
+  // Check that DynamicMessage handles map field serialization the same way as generated code
+  // regarding unset key and value field in a map entry.
+  public void testDynamicMessageUnsetKeyAndValue() throws Exception {
+    FieldDescriptor field = f("int32_to_int32_field");
+
+    Message dynamicDefaultInstance =
+        DynamicMessage.getDefaultInstance(TestMap.getDescriptor());
+    Message.Builder builder = dynamicDefaultInstance.newBuilderForType();
+    // Add an entry without key and value.
+    builder.addRepeatedField(field, builder.newBuilderForField(field).build());
+    Message message = builder.build();
+    ByteString bytes = message.toByteString();
+    // Parse it back to the same generated type.
+    Message generatedMessage = TestMap.parseFrom(bytes);
+    // Assert the serialized bytes are equivalent.
+    assertEquals(generatedMessage.toByteString(), bytes);
+  }
+
   public void testReflectionEqualsAndHashCode() throws Exception {
     // Test that generated equals() and hashCode() will disregard the order
     // of map entries when comparing/hashing map fields.

java/core/src/test/java/com/google/protobuf/TextFormatTest.java

@@ -61,12 +61,11 @@ import junit.framework.TestCase;
 public class TextFormatTest extends TestCase {
 
   // A basic string with different escapable characters for testing.
-  private final static String kEscapeTestString =
-      "\"A string with ' characters \n and \r newlines and \t tabs and \001 "
-          + "slashes \\";
+  private static final String ESCAPE_TEST_STRING =
+      "\"A string with ' characters \n and \r newlines and \t tabs and \001 " + "slashes \\";
 
   // A representation of the above string with all the characters escaped.
-  private final static String kEscapeTestStringEscaped =
+  private static final String ESCAPE_TEST_STRING_ESCAPED =
       "\\\"A string with \\' characters \\n and \\r newlines "
           + "and \\t tabs and \\001 slashes \\\\";
 
@@ -576,10 +575,10 @@ public class TextFormatTest extends TestCase {
         "integer: 82301481290849012385230157",
       "optional_int32: 82301481290849012385230157");
     assertParseError(
-      "1:16: Expected \"true\" or \"false\".",
+      "1:16: Expected \"true\" or \"false\". Found \"maybe\".",
       "optional_bool: maybe");
     assertParseError(
-      "1:16: Expected \"true\" or \"false\".",
+      "1:16: Expected \"true\" or \"false\". Found \"2\".",
       "optional_bool: 2");
     assertParseError(
       "1:18: Expected string.",
@@ -643,10 +642,8 @@ public class TextFormatTest extends TestCase {
       TextFormat.unescapeBytes("\\000\\001\\a\\b\\f\\n\\r\\t\\v\\\\\\'\\\""));
     assertEquals("\0\001\007\b\f\n\r\t\013\\\'\"",
       TextFormat.unescapeText("\\000\\001\\a\\b\\f\\n\\r\\t\\v\\\\\\'\\\""));
-    assertEquals(kEscapeTestStringEscaped,
-      TextFormat.escapeText(kEscapeTestString));
-    assertEquals(kEscapeTestString,
-      TextFormat.unescapeText(kEscapeTestStringEscaped));
+    assertEquals(ESCAPE_TEST_STRING_ESCAPED, TextFormat.escapeText(ESCAPE_TEST_STRING));
+    assertEquals(ESCAPE_TEST_STRING, TextFormat.unescapeText(ESCAPE_TEST_STRING_ESCAPED));
 
     // Invariant
     assertEquals("hello",

java/core/src/test/java/com/google/protobuf/UnknownFieldSetLiteTest.java

@@ -30,6 +30,8 @@
 
 package com.google.protobuf;
 
+import static junit.framework.TestCase.assertEquals;
+
 import com.google.protobuf.UnittestLite.TestAllExtensionsLite;
 import com.google.protobuf.UnittestLite.TestAllTypesLite;
 import protobuf_unittest.UnittestProto;
@@ -133,6 +135,25 @@ public class UnknownFieldSetLiteTest extends TestCase {
     assertEquals(foo.toByteString().size(), instance.getSerializedSize());
   }
 
+  public void testHashCodeAfterDeserialization() throws IOException {
+    Foo foo = Foo.newBuilder()
+      .setValue(2)
+      .build();
+
+    Foo fooDeserialized = Foo.parseFrom(foo.toByteArray());
+
+    assertEquals(fooDeserialized, foo);
+    assertEquals(foo.hashCode(), fooDeserialized.hashCode());
+  }
+
+  public void testNewInstanceHashCode() {
+    UnknownFieldSetLite emptyFieldSet = UnknownFieldSetLite.getDefaultInstance();
+    UnknownFieldSetLite paddedFieldSet = UnknownFieldSetLite.newInstance();
+
+    assertEquals(emptyFieldSet, paddedFieldSet);
+    assertEquals(emptyFieldSet.hashCode(), paddedFieldSet.hashCode());
+  }
+
   public void testMergeVarintField() throws IOException {
     UnknownFieldSetLite unknownFields = UnknownFieldSetLite.newInstance();
     unknownFields.mergeVarintField(10, 2);

java/core/src/test/java/com/google/protobuf/WireFormatTest.java

@@ -36,6 +36,7 @@ import protobuf_unittest.UnittestMset.TestMessageSetExtension2;
 import protobuf_unittest.UnittestProto;
 import protobuf_unittest.UnittestProto.TestAllExtensions;
 import protobuf_unittest.UnittestProto.TestAllTypes;
+import protobuf_unittest.UnittestProto.TestExtensionInsideTable;
 import protobuf_unittest.UnittestProto.TestFieldOrderings;
 import protobuf_unittest.UnittestProto.TestOneof2;
 import protobuf_unittest.UnittestProto.TestOneofBackwardsCompatible;
@@ -235,6 +236,26 @@ public class WireFormatTest extends TestCase {
                                    getTestFieldOrderingsRegistry());
     assertEquals(source, dest);
   }
+  
+  private static ExtensionRegistry getTestExtensionInsideTableRegistry() {
+    ExtensionRegistry result = ExtensionRegistry.newInstance();
+    result.add(UnittestProto.testExtensionInsideTableExtension);
+    return result;
+  }
+  
+  public void testExtensionInsideTable() throws Exception {
+    // Make sure the extension within the range of table is parsed correctly in experimental
+    // runtime.
+    TestExtensionInsideTable source =
+        TestExtensionInsideTable.newBuilder()
+        .setField1(1)
+        .setExtension(UnittestProto.testExtensionInsideTableExtension, 23)
+        .build();
+    TestExtensionInsideTable dest =
+        TestExtensionInsideTable.parseFrom(source.toByteString(),
+            getTestExtensionInsideTableRegistry());
+    assertEquals(source, dest);
+  }
 
   public void testParseMultipleExtensionRangesDynamic() throws Exception {
     // Same as above except with DynamicMessage.

java/core/src/test/proto/com/google/protobuf/map_lite_test.proto

@@ -0,0 +1,111 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+syntax = "proto3";
+
+package map_lite_test;
+
+option optimize_for = LITE_RUNTIME;
+option java_package = "map_lite_test";
+option java_outer_classname = "MapTestProto";
+
+message TestMap {
+  message MessageValue {
+    int32 value = 1;
+  }
+  enum EnumValue {
+    FOO = 0;
+    BAR = 1;
+    BAZ = 2;
+    QUX = 3;
+  }
+
+  map<int32, int32>        int32_to_int32_field = 1;
+  map<int32, string>       int32_to_string_field = 2;
+  map<int32, bytes>        int32_to_bytes_field = 3;
+  map<int32, EnumValue>    int32_to_enum_field = 4;
+  map<int32, MessageValue> int32_to_message_field = 5;
+  map<string, int32>       string_to_int32_field = 6;
+  map<uint32, int32>       uint32_to_int32_field = 7;
+  map<int64, int32>        int64_to_int32_field = 8;
+}
+
+// Used to test that a nested builder containing map fields will properly
+// propagate the onChange event and mark its parent dirty when a change
+// is made to a map field.
+message TestOnChangeEventPropagation {
+  TestMap optional_message = 1;
+}
+
+// a decoy of TestMap for testing parsing errors
+message BizarroTestMap {
+  map<int32, bytes> int32_to_int32_field = 1; // same key type, different value
+  map<string, int32> int32_to_string_field = 2; // different key and value types
+  map<string, int32> int32_to_bytes_field = 3; // different key types, same value
+  map<string, bytes> int32_to_enum_field = 4; // different key and value types
+  map<string, bytes> int32_to_message_field = 5; // different key and value types
+  map<string, bytes> string_to_int32_field = 6;  // same key type, different value
+}
+
+// Used to test that java reserved words can be used as protobuf field names
+// Not all reserved words are tested (to avoid bloat) but instead an arbitrary
+// subset of them chosen to cover various keyword categories like
+// type, modifier, declaration, etc.
+message ReservedAsMapField {
+  map<string, uint32> if = 1;
+  map<string, uint32> const = 2;
+  map<string, uint32> private = 3;
+  map<string, uint32> class = 4;
+  map<string, uint32> int = 5;
+  map<string, uint32> void = 6;
+  map<string, uint32> string = 7; // These are also proto keywords
+  map<string, uint32> package = 8;
+  map<string, uint32> enum = 9; // Most recent Java reserved word
+  map<string, uint32> null = 10;
+  // null is not a 'reserved word' per se but as a literal needs similar care
+}
+
+message ReservedAsMapFieldWithEnumValue {
+  enum SampleEnum {
+    A = 0;
+    B = 1;
+  }
+  map<string, SampleEnum> if = 1;
+  map<string, SampleEnum> const = 2;
+  map<string, SampleEnum> private = 3;
+  map<string, SampleEnum> class = 4;
+  map<string, SampleEnum> int = 5;
+  map<string, SampleEnum> void = 6;
+  map<string, SampleEnum> string = 7; // These are also proto keywords
+  map<string, SampleEnum> package = 8;
+  map<string, SampleEnum> enum = 9; // Most recent Java reserved word
+  map<string, SampleEnum> null = 10;
+  // null is not a 'reserved word' per se but as a literal needs similar care
+}

js/binary/constants.js

@@ -60,14 +60,25 @@ goog.forwardDeclare('jsproto.BinaryExtension');
 
 
 /**
- * Base interface class for all const messages. Does __not__ define any
- * methods, as doing so on a widely-used interface defeats dead-code
- * elimination.
+ * Base interface class for all const messages.
  * @interface
  */
 jspb.ConstBinaryMessage = function() {};
 
+/**
+ * Generate a debug string for this proto that is in proto2 text format.
+ * @return {string} The debug string.
+ */
+jspb.ConstBinaryMessage.prototype.toDebugString;
 
+/**
+ * Helper to generate a debug string for this proto at some indent level. The
+ * first line is not indented.
+ * @param {number} indentLevel The number of spaces by which to indent lines.
+ * @return {string} The debug string.
+ * @protected
+ */
+jspb.ConstBinaryMessage.prototype.toDebugStringInternal;
 
 /**
  * Base interface class for all messages. Does __not__ define any methods, as
@@ -97,6 +108,7 @@ jspb.ScalarFieldType;
  * A repeated field in jspb is an array of scalars, blobs, or messages.
  * @typedef {!Array<jspb.ScalarFieldType>|
              !Array<!Uint8Array>|
+             !Array<!jspb.ConstBinaryMessage>|
              !Array<!jspb.BinaryMessage>}
  */
 jspb.RepeatedFieldType;
@@ -108,6 +120,7 @@ jspb.RepeatedFieldType;
  * @typedef {jspb.ScalarFieldType|
              jspb.RepeatedFieldType|
              !Uint8Array|
+             !jspb.ConstBinaryMessage|
              !jspb.BinaryMessage|
              !jsproto.BinaryExtension}
  */

js/binary/decoder.js

@@ -986,7 +986,7 @@ jspb.BinaryDecoder.prototype.readString = function(length) {
       codeUnits.push(high, low);
     }
 
-    // Avoid exceeding the maximum stack size when calling {@code apply}.
+    // Avoid exceeding the maximum stack size when calling `apply`.
     if (codeUnits.length >= 8192) {
       result += String.fromCharCode.apply(null, codeUnits);
       codeUnits.length = 0;

js/binary/utils.js

@@ -971,6 +971,10 @@ jspb.utils.byteSourceToUint8Array = function(data) {
     return /** @type {!Uint8Array} */(new Uint8Array(data));
   }
 
+  if (data.constructor === Buffer) {
+    return /** @type {!Uint8Array} */(new Uint8Array(data));
+  }
+
   if (data.constructor === Array) {
     data = /** @type {!Array<number>} */(data);
     return /** @type {!Uint8Array} */(new Uint8Array(data));

js/binary/writer.js

@@ -236,10 +236,12 @@ jspb.BinaryWriter.prototype.getResultBuffer = function() {
 
 /**
  * Converts the encoded data into a base64-encoded string.
+ * @param {boolean=} opt_webSafe True indicates we should use a websafe
+ *     alphabet, which does not require escaping for use in URLs.
  * @return {string}
  */
-jspb.BinaryWriter.prototype.getResultBase64String = function() {
-  return goog.crypt.base64.encodeByteArray(this.getResultBuffer());
+jspb.BinaryWriter.prototype.getResultBase64String = function(opt_webSafe) {
+  return goog.crypt.base64.encodeByteArray(this.getResultBuffer(), opt_webSafe);
 };
 
 

js/binary/writer_test.js

@@ -118,4 +118,16 @@ describe('binaryWriterTest', function() {
     var buffer = writer.getResultBuffer();
     assertEquals(expected, goog.crypt.byteArrayToHex(buffer));
   });
+
+
+  /**
+   * Tests websafe encodings for base64 strings.
+   */
+  it('testWebSafeOption', function() {
+    var writer = new jspb.BinaryWriter();
+    writer.writeBytes(1, new Uint8Array([127]));
+    assertEquals('CgF/', writer.getResultBase64String());
+    assertEquals('CgF/', writer.getResultBase64String(false));
+    assertEquals('CgF_', writer.getResultBase64String(true));
+  });
 });

js/debug.js

@@ -42,7 +42,7 @@ goog.require('jspb.Message');
 
 /**
  * Turns a proto into a human readable object that can i.e. be written to the
- * console: {@code console.log(jspb.debug.dump(myProto))}.
+ * console: `console.log(jspb.debug.dump(myProto))`.
  * This function makes a best effort and may not work in all cases. It will not
  * work in obfuscated and or optimized code.
  * Use this in environments where {@see jspb.Message.prototype.toObject} is

js/map.js

@@ -48,9 +48,9 @@ goog.forwardDeclare('jspb.BinaryWriter');
  *
  * @template K, V
  *
- * @param {!Array<!Array<!Object>>} arr
+ * @param {!Array<!Array<?>>} arr
  *
- * @param {?function(new:V)|function(new:V,?)=} opt_valueCtor
+ * @param {?function(new:V, ?=)=} opt_valueCtor
  *    The constructor for type V, if type V is a message type.
  *
  * @constructor
@@ -118,7 +118,7 @@ jspb.Map.prototype.toArray = function() {
     strKeys.sort();
     for (var i = 0; i < strKeys.length; i++) {
       var entry = this.map_[strKeys[i]];
-      var valueWrapper = /** @type {!Object} */ (entry.valueWrapper);
+      var valueWrapper = /** @type {?jspb.Message} */ (entry.valueWrapper);
       if (valueWrapper) {
         valueWrapper.toArray();
       }
@@ -165,7 +165,7 @@ jspb.Map.prototype.toObject = function(includeInstance, valueToObject) {
  *
  * @template K, V
  * @param {!Array<!Array<!Object>>} entries
- * @param {!function(new:V)|function(new:V,?)} valueCtor
+ * @param {!function(new:V,?=)} valueCtor
  *    The constructor for type V.
  * @param {!function(!Object):V} valueFromObject
  *    The fromObject function for type V.
@@ -432,7 +432,8 @@ jspb.Map.prototype.serializeBinary = function(
       valueWriterFn.call(writer, 2, this.wrapEntry_(entry),
                          opt_valueWriterCallback);
     } else {
-      valueWriterFn.call(writer, 2, entry.value);
+      /** @type {function(this:jspb.BinaryWriter,number,?)} */ (valueWriterFn)
+          .call(writer, 2, entry.value);
     }
     writer.endSubMessage();
   }
@@ -475,10 +476,13 @@ jspb.Map.deserializeBinary = function(map, reader, keyReaderFn, valueReaderFn,
     } else if (field == 2) {
       // Value.
       if (map.valueCtor_) {
+        goog.asserts.assert(opt_valueReaderCallback);
         value = new map.valueCtor_();
         valueReaderFn.call(reader, value, opt_valueReaderCallback);
       } else {
-        value = valueReaderFn.call(reader);
+        value =
+            (/** @type {function(this:jspb.BinaryReader):?} */ (valueReaderFn))
+                .call(reader);
       }
     }
   }

js/message.js

@@ -215,17 +215,6 @@ goog.define('jspb.Message.ASSUME_LOCAL_ARRAYS', false);
 goog.define('jspb.Message.SERIALIZE_EMPTY_TRAILING_FIELDS', true);
 
 
-/**
- * @define {boolean} Turning on this flag does NOT change the behavior of JSPB
- *     and only affects private internal state. It may, however, break some
- *     tests that use naive deeply-equals algorithms, because using a proto
- *     mutates its internal state.
- *     Projects are advised to turn this flag always on.
- */
-goog.define('jspb.Message.MINIMIZE_MEMORY_ALLOCATIONS', true);
-// TODO(b/19419436): Delete this flag.
-
-
 /**
  * Does this JavaScript environment support Uint8Aray typed arrays?
  * @type {boolean}
@@ -369,7 +358,7 @@ jspb.Message.getFieldNumber_ = function(msg, index) {
  */
 jspb.Message.initialize = function(
     msg, data, messageId, suggestedPivot, repeatedFields, opt_oneofFields) {
-  msg.wrappers_ = jspb.Message.MINIMIZE_MEMORY_ALLOCATIONS ? null : {};
+  msg.wrappers_ = null;
   if (!data) {
     data = messageId ? [messageId] : [];
   }
@@ -394,17 +383,12 @@ jspb.Message.initialize = function(
       var fieldNumber = repeatedFields[i];
       if (fieldNumber < msg.pivot_) {
         var index = jspb.Message.getIndex_(msg, fieldNumber);
-        msg.array[index] = msg.array[index] ||
-            (jspb.Message.MINIMIZE_MEMORY_ALLOCATIONS ?
-                jspb.Message.EMPTY_LIST_SENTINEL_ :
-                []);
+        msg.array[index] =
+            msg.array[index] || jspb.Message.EMPTY_LIST_SENTINEL_;
       } else {
         jspb.Message.maybeInitEmptyExtensionObject_(msg);
-        msg.extensionObject_[fieldNumber] =
-            msg.extensionObject_[fieldNumber] ||
-            (jspb.Message.MINIMIZE_MEMORY_ALLOCATIONS ?
-                jspb.Message.EMPTY_LIST_SENTINEL_ :
-                []);
+        msg.extensionObject_[fieldNumber] = msg.extensionObject_[fieldNumber] ||
+            jspb.Message.EMPTY_LIST_SENTINEL_;
       }
     }
   }
@@ -517,8 +501,7 @@ jspb.Message.toObjectList = function(field, toObjectFn, opt_includeInstance) {
   // And not using it here to avoid a function call.
   var result = [];
   for (var i = 0; i < field.length; i++) {
-    result[i] = toObjectFn.call(field[i], opt_includeInstance,
-      /** @type {!jspb.Message} */ (field[i]));
+    result[i] = toObjectFn.call(field[i], opt_includeInstance, field[i]);
   }
   return result;
 };
@@ -551,10 +534,11 @@ jspb.Message.toObjectExtension = function(proto, obj, extensions,
       } else {
         if (fieldInfo.isRepeated) {
           obj[name] = jspb.Message.toObjectList(
-              /** @type {!Array<jspb.Message>} */ (value),
+              /** @type {!Array<!jspb.Message>} */ (value),
               fieldInfo.toObjectFn, opt_includeInstance);
         } else {
-          obj[name] = fieldInfo.toObjectFn(opt_includeInstance, value);
+          obj[name] = fieldInfo.toObjectFn(
+              opt_includeInstance, /** @type {!jspb.Message} */ (value));
         }
       }
     }
@@ -1419,7 +1403,7 @@ jspb.Message.prototype.setExtension = function(fieldInfo, value) {
     if (fieldInfo.isMessageType()) {
       self.wrappers_[fieldNumber] = value;
       self.extensionObject_[fieldNumber] = goog.array.map(
-          /** @type {Array<jspb.Message>} */ (value), function(msg) {
+          /** @type {!Array<!jspb.Message>} */ (value), function(msg) {
         return msg.toArray();
       });
     } else {
@@ -1428,7 +1412,8 @@ jspb.Message.prototype.setExtension = function(fieldInfo, value) {
   } else {
     if (fieldInfo.isMessageType()) {
       self.wrappers_[fieldNumber] = value;
-      self.extensionObject_[fieldNumber] = value ? value.toArray() : value;
+      self.extensionObject_[fieldNumber] =
+          value ? /** @type {!jspb.Message} */ (value).toArray() : value;
     } else {
       self.extensionObject_[fieldNumber] = value;
     }
@@ -1530,9 +1515,15 @@ jspb.Message.compareFields = function(field1, field2) {
   // If the fields are trivially equal, they're equal.
   if (field1 == field2) return true;
 
-  // If the fields aren't trivially equal and one of them isn't an object,
-  // they can't possibly be equal.
   if (!goog.isObject(field1) || !goog.isObject(field2)) {
+    // NaN != NaN so we cover this case.
+    if ((goog.isNumber(field1) && isNaN(field1)) ||
+        (goog.isNumber(field2) && isNaN(field2))) {
+      // One of the fields might be a string 'NaN'.
+      return String(field1) == String(field2);
+    }
+    // If the fields aren't trivially equal and one of them isn't an object,
+    // they can't possibly be equal.
     return false;
   }
 
@@ -1555,24 +1546,26 @@ jspb.Message.compareFields = function(field1, field2) {
   // If they're both Arrays, compare them element by element except for the
   // optional extension objects at the end, which we compare separately.
   if (field1.constructor === Array) {
+    var typedField1 = /** @type {!Array<?>} */ (field1);
+    var typedField2 = /** @type {!Array<?>} */ (field2);
     var extension1 = undefined;
     var extension2 = undefined;
 
-    var length = Math.max(field1.length, field2.length);
+    var length = Math.max(typedField1.length, typedField2.length);
     for (var i = 0; i < length; i++) {
-      var val1 = field1[i];
-      var val2 = field2[i];
+      var val1 = typedField1[i];
+      var val2 = typedField2[i];
 
       if (val1 && (val1.constructor == Object)) {
         goog.asserts.assert(extension1 === undefined);
-        goog.asserts.assert(i === field1.length - 1);
+        goog.asserts.assert(i === typedField1.length - 1);
         extension1 = val1;
         val1 = undefined;
       }
 
       if (val2 && (val2.constructor == Object)) {
         goog.asserts.assert(extension2 === undefined);
-        goog.asserts.assert(i === field2.length - 1);
+        goog.asserts.assert(i === typedField2.length - 1);
         extension2 = val2;
         val2 = undefined;
       }
@@ -1695,8 +1688,13 @@ jspb.Message.clone_ = function(obj) {
     var clonedArray = new Array(obj.length);
     // Use array iteration where possible because it is faster than for-in.
     for (var i = 0; i < obj.length; i++) {
-      if ((o = obj[i]) != null) {
-        clonedArray[i] = typeof o == 'object' ? jspb.Message.clone_(o) : o;
+      o = obj[i];
+      if (o != null) {
+        // NOTE:redundant null check existing for NTI compatibility.
+        // see b/70515949
+        clonedArray[i] = (typeof o == 'object') ?
+            jspb.Message.clone_(goog.asserts.assert(o)) :
+            o;
       }
     }
     return clonedArray;
@@ -1706,8 +1704,13 @@ jspb.Message.clone_ = function(obj) {
   }
   var clone = {};
   for (var key in obj) {
-    if ((o = obj[key]) != null) {
-      clone[key] = typeof o == 'object' ? jspb.Message.clone_(o) : o;
+    o = obj[key];
+    if (o != null) {
+      // NOTE:redundant null check existing for NTI compatibility.
+      // see b/70515949
+      clone[key] = (typeof o == 'object') ?
+          jspb.Message.clone_(goog.asserts.assert(o)) :
+          o;
     }
   }
   return clone;
@@ -1723,6 +1726,9 @@ jspb.Message.registerMessageType = function(id, constructor) {
   jspb.Message.registry_[id] = constructor;
   // This is needed so we can later access messageId directly on the contructor,
   // otherwise it is not available due to 'property collapsing' by the compiler.
+  /**
+   * @suppress {strictMissingProperties} messageId is not defined on Function
+   */
   constructor.messageId = id;
 };
 

js/message_test.js

@@ -418,6 +418,18 @@ describe('Message test suite', function() {
         ['hi',,, {100: [{200: 'a'}]}], ['hi', {100: [{200: 'a'}]}]));
   });
 
+  it('testEqualsNonFinite', function() {
+    assertTrue(jspb.Message.compareFields(NaN, NaN));
+    assertTrue(jspb.Message.compareFields(NaN, 'NaN'));
+    assertTrue(jspb.Message.compareFields('NaN', NaN));
+    assertTrue(jspb.Message.compareFields(Infinity, Infinity));
+    assertTrue(jspb.Message.compareFields(Infinity, 'Infinity'));
+    assertTrue(jspb.Message.compareFields('-Infinity', -Infinity));
+    assertTrue(jspb.Message.compareFields([NaN], ['NaN']));
+    assertFalse(jspb.Message.compareFields(undefined, NaN));
+    assertFalse(jspb.Message.compareFields(NaN, undefined));
+  });
+
   it('testToMap', function() {
     var p1 = new proto.jspb.test.Simple1(['k', ['v']]);
     var p2 = new proto.jspb.test.Simple1(['k1', ['v1', 'v2']]);

m4/acx_pthread.m4

@@ -1,397 +0,0 @@
-# This was retrieved from
-#    http://svn.0pointer.de/viewvc/trunk/common/acx_pthread.m4?revision=1277&root=avahi
-# See also (perhaps for new versions?)
-#    http://svn.0pointer.de/viewvc/trunk/common/acx_pthread.m4?root=avahi
-#
-# We've rewritten the inconsistency check code (from avahi), to work
-# more broadly.  In particular, it no longer assumes ld accepts -zdefs.
-# This caused a restructing of the code, but the functionality has only
-# changed a little.
-
-dnl @synopsis ACX_PTHREAD([ACTION-IF-FOUND[, ACTION-IF-NOT-FOUND]])
-dnl
-dnl @summary figure out how to build C programs using POSIX threads
-dnl
-dnl This macro figures out how to build C programs using POSIX threads.
-dnl It sets the PTHREAD_LIBS output variable to the threads library and
-dnl linker flags, and the PTHREAD_CFLAGS output variable to any special
-dnl C compiler flags that are needed. (The user can also force certain
-dnl compiler flags/libs to be tested by setting these environment
-dnl variables.)
-dnl
-dnl Also sets PTHREAD_CC to any special C compiler that is needed for
-dnl multi-threaded programs (defaults to the value of CC otherwise).
-dnl (This is necessary on AIX to use the special cc_r compiler alias.)
-dnl
-dnl NOTE: You are assumed to not only compile your program with these
-dnl flags, but also link it with them as well. e.g. you should link
-dnl with $PTHREAD_CC $CFLAGS $PTHREAD_CFLAGS $LDFLAGS ... $PTHREAD_LIBS
-dnl $LIBS
-dnl
-dnl If you are only building threads programs, you may wish to use
-dnl these variables in your default LIBS, CFLAGS, and CC:
-dnl
-dnl        LIBS="$PTHREAD_LIBS $LIBS"
-dnl        CFLAGS="$CFLAGS $PTHREAD_CFLAGS"
-dnl        CC="$PTHREAD_CC"
-dnl
-dnl In addition, if the PTHREAD_CREATE_JOINABLE thread-attribute
-dnl constant has a nonstandard name, defines PTHREAD_CREATE_JOINABLE to
-dnl that name (e.g. PTHREAD_CREATE_UNDETACHED on AIX).
-dnl
-dnl ACTION-IF-FOUND is a list of shell commands to run if a threads
-dnl library is found, and ACTION-IF-NOT-FOUND is a list of commands to
-dnl run it if it is not found. If ACTION-IF-FOUND is not specified, the
-dnl default action will define HAVE_PTHREAD.
-dnl
-dnl Please let the authors know if this macro fails on any platform, or
-dnl if you have any other suggestions or comments. This macro was based
-dnl on work by SGJ on autoconf scripts for FFTW (www.fftw.org) (with
-dnl help from M. Frigo), as well as ac_pthread and hb_pthread macros
-dnl posted by Alejandro Forero Cuervo to the autoconf macro repository.
-dnl We are also grateful for the helpful feedback of numerous users.
-dnl
-dnl @category InstalledPackages
-dnl @author Steven G. Johnson <stevenj@alum.mit.edu>
-dnl @version 2006-05-29
-dnl @license GPLWithACException
-dnl 
-dnl Checks for GCC shared/pthread inconsistency based on work by
-dnl Marcin Owsiany <marcin@owsiany.pl>
-
-
-AC_DEFUN([ACX_PTHREAD], [
-AC_REQUIRE([AC_CANONICAL_HOST])
-AC_LANG_SAVE
-AC_LANG_C
-acx_pthread_ok=no
-
-# We used to check for pthread.h first, but this fails if pthread.h
-# requires special compiler flags (e.g. on True64 or Sequent).
-# It gets checked for in the link test anyway.
-
-# First of all, check if the user has set any of the PTHREAD_LIBS,
-# etcetera environment variables, and if threads linking works using
-# them:
-if test x"$PTHREAD_LIBS$PTHREAD_CFLAGS" != x; then
-        save_CFLAGS="$CFLAGS"
-        CFLAGS="$CFLAGS $PTHREAD_CFLAGS"
-        save_LIBS="$LIBS"
-        LIBS="$PTHREAD_LIBS $LIBS"
-        AC_MSG_CHECKING([for pthread_join in LIBS=$PTHREAD_LIBS with CFLAGS=$PTHREAD_CFLAGS])
-        AC_TRY_LINK_FUNC(pthread_join, acx_pthread_ok=yes)
-        AC_MSG_RESULT($acx_pthread_ok)
-        if test x"$acx_pthread_ok" = xno; then
-                PTHREAD_LIBS=""
-                PTHREAD_CFLAGS=""
-        fi
-        LIBS="$save_LIBS"
-        CFLAGS="$save_CFLAGS"
-fi
-
-# We must check for the threads library under a number of different
-# names; the ordering is very important because some systems
-# (e.g. DEC) have both -lpthread and -lpthreads, where one of the
-# libraries is broken (non-POSIX).
-
-# Create a list of thread flags to try.  Items starting with a "-" are
-# C compiler flags, and other items are library names, except for "none"
-# which indicates that we try without any flags at all, and "pthread-config"
-# which is a program returning the flags for the Pth emulation library.
-
-acx_pthread_flags="pthreads none -Kthread -kthread lthread -pthread -pthreads -mthreads pthread --thread-safe -mt pthread-config"
-
-# The ordering *is* (sometimes) important.  Some notes on the
-# individual items follow:
-
-# pthreads: AIX (must check this before -lpthread)
-# none: in case threads are in libc; should be tried before -Kthread and
-#       other compiler flags to prevent continual compiler warnings
-# -Kthread: Sequent (threads in libc, but -Kthread needed for pthread.h)
-# -kthread: FreeBSD kernel threads (preferred to -pthread since SMP-able)
-# lthread: LinuxThreads port on FreeBSD (also preferred to -pthread)
-# -pthread: Linux/gcc (kernel threads), BSD/gcc (userland threads)
-# -pthreads: Solaris/gcc
-# -mthreads: Mingw32/gcc, Lynx/gcc
-# -mt: Sun Workshop C (may only link SunOS threads [-lthread], but it
-#      doesn't hurt to check since this sometimes defines pthreads too;
-#      also defines -D_REENTRANT)
-#      ... -mt is also the pthreads flag for HP/aCC
-# pthread: Linux, etcetera
-# --thread-safe: KAI C++
-# pthread-config: use pthread-config program (for GNU Pth library)
-
-case "${host_cpu}-${host_os}" in
-        *solaris*)
-
-        # On Solaris (at least, for some versions), libc contains stubbed
-        # (non-functional) versions of the pthreads routines, so link-based
-        # tests will erroneously succeed.  (We need to link with -pthreads/-mt/
-        # -lpthread.)  (The stubs are missing pthread_cleanup_push, or rather
-        # a function called by this macro, so we could check for that, but
-        # who knows whether they'll stub that too in a future libc.)  So,
-        # we'll just look for -pthreads and -lpthread first:
-
-        acx_pthread_flags="-pthreads pthread -mt -pthread $acx_pthread_flags"
-        ;;
-esac
-
-if test x"$acx_pthread_ok" = xno; then
-for flag in $acx_pthread_flags; do
-
-        case $flag in
-                none)
-                AC_MSG_CHECKING([whether pthreads work without any flags])
-                ;;
-
-                -*)
-                AC_MSG_CHECKING([whether pthreads work with $flag])
-                PTHREAD_CFLAGS="$flag"
-                ;;
-
-		pthread-config)
-		AC_CHECK_PROG(acx_pthread_config, pthread-config, yes, no)
-		if test x"$acx_pthread_config" = xno; then continue; fi
-		PTHREAD_CFLAGS="`pthread-config --cflags`"
-		PTHREAD_LIBS="`pthread-config --ldflags` `pthread-config --libs`"
-		;;
-
-                *)
-                AC_MSG_CHECKING([for the pthreads library -l$flag])
-                PTHREAD_LIBS="-l$flag"
-                ;;
-        esac
-
-        save_LIBS="$LIBS"
-        save_CFLAGS="$CFLAGS"
-        LIBS="$PTHREAD_LIBS $LIBS"
-        CFLAGS="$CFLAGS $PTHREAD_CFLAGS"
-
-        # Check for various functions.  We must include pthread.h,
-        # since some functions may be macros.  (On the Sequent, we
-        # need a special flag -Kthread to make this header compile.)
-        # We check for pthread_join because it is in -lpthread on IRIX
-        # while pthread_create is in libc.  We check for pthread_attr_init
-        # due to DEC craziness with -lpthreads.  We check for
-        # pthread_cleanup_push because it is one of the few pthread
-        # functions on Solaris that doesn't have a non-functional libc stub.
-        # We try pthread_create on general principles.
-        AC_TRY_LINK([#include <pthread.h>],
-                    [pthread_t th; pthread_join(th, 0);
-                     pthread_attr_init(0); pthread_cleanup_push(0, 0);
-                     pthread_create(0,0,0,0); pthread_cleanup_pop(0); ],
-                    [acx_pthread_ok=yes])
-
-        LIBS="$save_LIBS"
-        CFLAGS="$save_CFLAGS"
-
-        AC_MSG_RESULT($acx_pthread_ok)
-        if test "x$acx_pthread_ok" = xyes; then
-                break;
-        fi
-
-        PTHREAD_LIBS=""
-        PTHREAD_CFLAGS=""
-done
-fi
-
-# Various other checks:
-if test "x$acx_pthread_ok" = xyes; then
-        save_LIBS="$LIBS"
-        LIBS="$PTHREAD_LIBS $LIBS"
-        save_CFLAGS="$CFLAGS"
-        CFLAGS="$CFLAGS $PTHREAD_CFLAGS"
-
-        # Detect AIX lossage: JOINABLE attribute is called UNDETACHED.
-	AC_MSG_CHECKING([for joinable pthread attribute])
-	attr_name=unknown
-	for attr in PTHREAD_CREATE_JOINABLE PTHREAD_CREATE_UNDETACHED; do
-	    AC_TRY_LINK([#include <pthread.h>], [int attr=$attr; return attr;],
-                        [attr_name=$attr; break])
-	done
-        AC_MSG_RESULT($attr_name)
-        if test "$attr_name" != PTHREAD_CREATE_JOINABLE; then
-            AC_DEFINE_UNQUOTED(PTHREAD_CREATE_JOINABLE, $attr_name,
-                               [Define to necessary symbol if this constant
-                                uses a non-standard name on your system.])
-        fi
-
-        AC_MSG_CHECKING([if more special flags are required for pthreads])
-        flag=no
-        case "${host_cpu}-${host_os}" in
-            *-aix* | *-freebsd* | *-darwin*) flag="-D_THREAD_SAFE";;
-            *solaris* | *-osf* | *-hpux*) flag="-D_REENTRANT";;
-        esac
-        AC_MSG_RESULT(${flag})
-        if test "x$flag" != xno; then
-            PTHREAD_CFLAGS="$flag $PTHREAD_CFLAGS"
-        fi
-
-        LIBS="$save_LIBS"
-        CFLAGS="$save_CFLAGS"
-        # More AIX lossage: must compile with xlc_r or cc_r
-	if test x"$GCC" != xyes; then
-          AC_CHECK_PROGS(PTHREAD_CC, xlc_r cc_r, ${CC})
-        else
-          PTHREAD_CC=$CC
-	fi
-
-	# The next part tries to detect GCC inconsistency with -shared on some
-	# architectures and systems. The problem is that in certain
-	# configurations, when -shared is specified, GCC "forgets" to
-	# internally use various flags which are still necessary.
-	
-	#
-	# Prepare the flags
-	#
-	save_CFLAGS="$CFLAGS"
-	save_LIBS="$LIBS"
-	save_CC="$CC"
-	
-	# Try with the flags determined by the earlier checks.
-	#
-	# -Wl,-z,defs forces link-time symbol resolution, so that the
-	# linking checks with -shared actually have any value
-	#
-	# FIXME: -fPIC is required for -shared on many architectures,
-	# so we specify it here, but the right way would probably be to
-	# properly detect whether it is actually required.
-	CFLAGS="-shared -fPIC -Wl,-z,defs $CFLAGS $PTHREAD_CFLAGS"
-	LIBS="$PTHREAD_LIBS $LIBS"
-	CC="$PTHREAD_CC"
-	
-	# In order not to create several levels of indentation, we test
-	# the value of "$done" until we find the cure or run out of ideas.
-	done="no"
-	
-	# First, make sure the CFLAGS we added are actually accepted by our
-	# compiler.  If not (and OS X's ld, for instance, does not accept -z),
-	# then we can't do this test.
-	if test x"$done" = xno; then
-	   AC_MSG_CHECKING([whether to check for GCC pthread/shared inconsistencies])
-	   AC_TRY_LINK(,, , [done=yes])
-	
-	   if test "x$done" = xyes ; then
-	      AC_MSG_RESULT([no])
-	   else
-	      AC_MSG_RESULT([yes])
-	   fi
-	fi
-	
-	if test x"$done" = xno; then
-	   AC_MSG_CHECKING([whether -pthread is sufficient with -shared])
-	   AC_TRY_LINK([#include <pthread.h>],
-	      [pthread_t th; pthread_join(th, 0);
-	      pthread_attr_init(0); pthread_cleanup_push(0, 0);
-	      pthread_create(0,0,0,0); pthread_cleanup_pop(0); ],
-	      [done=yes])
-	   
-	   if test "x$done" = xyes; then
-	      AC_MSG_RESULT([yes])
-	   else
-	      AC_MSG_RESULT([no])
-	   fi
-	fi
-	
-	#
-	# Linux gcc on some architectures such as mips/mipsel forgets
-	# about -lpthread
-	#
-	if test x"$done" = xno; then
-	   AC_MSG_CHECKING([whether -lpthread fixes that])
-	   LIBS="-lpthread $PTHREAD_LIBS $save_LIBS"
-	   AC_TRY_LINK([#include <pthread.h>],
-	      [pthread_t th; pthread_join(th, 0);
-	      pthread_attr_init(0); pthread_cleanup_push(0, 0);
-	      pthread_create(0,0,0,0); pthread_cleanup_pop(0); ],
-	      [done=yes])
-	
-	   if test "x$done" = xyes; then
-	      AC_MSG_RESULT([yes])
-	      PTHREAD_LIBS="-lpthread $PTHREAD_LIBS"
-	   else
-	      AC_MSG_RESULT([no])
-	   fi
-	fi
-	#
-	# FreeBSD 4.10 gcc forgets to use -lc_r instead of -lc
-	#
-	if test x"$done" = xno; then
-	   AC_MSG_CHECKING([whether -lc_r fixes that])
-	   LIBS="-lc_r $PTHREAD_LIBS $save_LIBS"
-	   AC_TRY_LINK([#include <pthread.h>],
-	       [pthread_t th; pthread_join(th, 0);
-	        pthread_attr_init(0); pthread_cleanup_push(0, 0);
-	        pthread_create(0,0,0,0); pthread_cleanup_pop(0); ],
-	       [done=yes])
-	
-	   if test "x$done" = xyes; then
-	      AC_MSG_RESULT([yes])
-	      PTHREAD_LIBS="-lc_r $PTHREAD_LIBS"
-	   else
-	      AC_MSG_RESULT([no])
-	   fi
-	fi
-	if test x"$done" = xno; then
-	   # OK, we have run out of ideas
-	   AC_MSG_WARN([Impossible to determine how to use pthreads with shared libraries])
-	
-	   # so it's not safe to assume that we may use pthreads
-	   acx_pthread_ok=no
-	fi
-	
-	AC_MSG_CHECKING([whether what we have so far is sufficient with -nostdlib])
-	CFLAGS="-nostdlib $CFLAGS"
-	# we need c with nostdlib
-	LIBS="$LIBS -lc"
-	AC_TRY_LINK([#include <pthread.h>],
-	      [pthread_t th; pthread_join(th, 0);
-	       pthread_attr_init(0); pthread_cleanup_push(0, 0);
-	       pthread_create(0,0,0,0); pthread_cleanup_pop(0); ],
-	      [done=yes],[done=no])
-
-	if test "x$done" = xyes; then
-	   AC_MSG_RESULT([yes])
-	else
-	   AC_MSG_RESULT([no])
-	fi
-	
-	if test x"$done" = xno; then
-	   AC_MSG_CHECKING([whether -lpthread saves the day])
-	   LIBS="-lpthread $LIBS"
-	   AC_TRY_LINK([#include <pthread.h>],
-	      [pthread_t th; pthread_join(th, 0);
-	       pthread_attr_init(0); pthread_cleanup_push(0, 0);
-	       pthread_create(0,0,0,0); pthread_cleanup_pop(0); ],
-	      [done=yes],[done=no])
-
-	   if test "x$done" = xyes; then
-	      AC_MSG_RESULT([yes])
-	      PTHREAD_LIBS="$PTHREAD_LIBS -lpthread"
-	   else
-	      AC_MSG_RESULT([no])
-	      AC_MSG_WARN([Impossible to determine how to use pthreads with shared libraries and -nostdlib])
-	   fi
-	fi
-
-	CFLAGS="$save_CFLAGS"
-	LIBS="$save_LIBS"
-	CC="$save_CC"
-else
-        PTHREAD_CC="$CC"
-fi
-
-AC_SUBST(PTHREAD_LIBS)
-AC_SUBST(PTHREAD_CFLAGS)
-AC_SUBST(PTHREAD_CC)
-
-# Finally, execute ACTION-IF-FOUND/ACTION-IF-NOT-FOUND:
-if test x"$acx_pthread_ok" = xyes; then
-        ifelse([$1],,AC_DEFINE(HAVE_PTHREAD,1,[Define if you have POSIX threads libraries and header files.]),[$1])
-        :
-else
-        acx_pthread_ok=no
-        $2
-fi
-AC_LANG_RESTORE
-])dnl ACX_PTHREAD

m4/ax_pthread.m4

@@ -0,0 +1,485 @@
+# ===========================================================================
+#        http://www.gnu.org/software/autoconf-archive/ax_pthread.html
+# ===========================================================================
+#
+# SYNOPSIS
+#
+#   AX_PTHREAD([ACTION-IF-FOUND[, ACTION-IF-NOT-FOUND]])
+#
+# DESCRIPTION
+#
+#   This macro figures out how to build C programs using POSIX threads. It
+#   sets the PTHREAD_LIBS output variable to the threads library and linker
+#   flags, and the PTHREAD_CFLAGS output variable to any special C compiler
+#   flags that are needed. (The user can also force certain compiler
+#   flags/libs to be tested by setting these environment variables.)
+#
+#   Also sets PTHREAD_CC to any special C compiler that is needed for
+#   multi-threaded programs (defaults to the value of CC otherwise). (This
+#   is necessary on AIX to use the special cc_r compiler alias.)
+#
+#   NOTE: You are assumed to not only compile your program with these flags,
+#   but also to link with them as well. For example, you might link with
+#   $PTHREAD_CC $CFLAGS $PTHREAD_CFLAGS $LDFLAGS ... $PTHREAD_LIBS $LIBS
+#
+#   If you are only building threaded programs, you may wish to use these
+#   variables in your default LIBS, CFLAGS, and CC:
+#
+#     LIBS="$PTHREAD_LIBS $LIBS"
+#     CFLAGS="$CFLAGS $PTHREAD_CFLAGS"
+#     CC="$PTHREAD_CC"
+#
+#   In addition, if the PTHREAD_CREATE_JOINABLE thread-attribute constant
+#   has a nonstandard name, this macro defines PTHREAD_CREATE_JOINABLE to
+#   that name (e.g. PTHREAD_CREATE_UNDETACHED on AIX).
+#
+#   Also HAVE_PTHREAD_PRIO_INHERIT is defined if pthread is found and the
+#   PTHREAD_PRIO_INHERIT symbol is defined when compiling with
+#   PTHREAD_CFLAGS.
+#
+#   ACTION-IF-FOUND is a list of shell commands to run if a threads library
+#   is found, and ACTION-IF-NOT-FOUND is a list of commands to run it if it
+#   is not found. If ACTION-IF-FOUND is not specified, the default action
+#   will define HAVE_PTHREAD.
+#
+#   Please let the authors know if this macro fails on any platform, or if
+#   you have any other suggestions or comments. This macro was based on work
+#   by SGJ on autoconf scripts for FFTW (http://www.fftw.org/) (with help
+#   from M. Frigo), as well as ac_pthread and hb_pthread macros posted by
+#   Alejandro Forero Cuervo to the autoconf macro repository. We are also
+#   grateful for the helpful feedback of numerous users.
+#
+#   Updated for Autoconf 2.68 by Daniel Richard G.
+#
+# LICENSE
+#
+#   Copyright (c) 2008 Steven G. Johnson <stevenj@alum.mit.edu>
+#   Copyright (c) 2011 Daniel Richard G. <skunk@iSKUNK.ORG>
+#
+#   This program is free software: you can redistribute it and/or modify it
+#   under the terms of the GNU General Public License as published by the
+#   Free Software Foundation, either version 3 of the License, or (at your
+#   option) any later version.
+#
+#   This program is distributed in the hope that it will be useful, but
+#   WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
+#   Public License for more details.
+#
+#   You should have received a copy of the GNU General Public License along
+#   with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+#   As a special exception, the respective Autoconf Macro's copyright owner
+#   gives unlimited permission to copy, distribute and modify the configure
+#   scripts that are the output of Autoconf when processing the Macro. You
+#   need not follow the terms of the GNU General Public License when using
+#   or distributing such scripts, even though portions of the text of the
+#   Macro appear in them. The GNU General Public License (GPL) does govern
+#   all other use of the material that constitutes the Autoconf Macro.
+#
+#   This special exception to the GPL applies to versions of the Autoconf
+#   Macro released by the Autoconf Archive. When you make and distribute a
+#   modified version of the Autoconf Macro, you may extend this special
+#   exception to the GPL to apply to your modified version as well.
+
+#serial 22
+
+AU_ALIAS([ACX_PTHREAD], [AX_PTHREAD])
+AC_DEFUN([AX_PTHREAD], [
+AC_REQUIRE([AC_CANONICAL_HOST])
+AC_REQUIRE([AC_PROG_CC])
+AC_REQUIRE([AC_PROG_SED])
+AC_LANG_PUSH([C])
+ax_pthread_ok=no
+
+# We used to check for pthread.h first, but this fails if pthread.h
+# requires special compiler flags (e.g. on Tru64 or Sequent).
+# It gets checked for in the link test anyway.
+
+# First of all, check if the user has set any of the PTHREAD_LIBS,
+# etcetera environment variables, and if threads linking works using
+# them:
+if test "x$PTHREAD_CFLAGS$PTHREAD_LIBS" != "x"; then
+	ax_pthread_save_CC="$CC"
+	ax_pthread_save_CFLAGS="$CFLAGS"
+	ax_pthread_save_LIBS="$LIBS"
+	AS_IF([test "x$PTHREAD_CC" != "x"], [CC="$PTHREAD_CC"])
+	CFLAGS="$CFLAGS $PTHREAD_CFLAGS"
+	LIBS="$PTHREAD_LIBS $LIBS"
+	AC_MSG_CHECKING([for pthread_join using $CC $PTHREAD_CFLAGS $PTHREAD_LIBS])
+	AC_LINK_IFELSE([AC_LANG_CALL([], [pthread_join])], [ax_pthread_ok=yes])
+	AC_MSG_RESULT([$ax_pthread_ok])
+	if test "x$ax_pthread_ok" = "xno"; then
+		PTHREAD_LIBS=""
+		PTHREAD_CFLAGS=""
+	fi
+	CC="$ax_pthread_save_CC"
+	CFLAGS="$ax_pthread_save_CFLAGS"
+	LIBS="$ax_pthread_save_LIBS"
+fi
+
+# We must check for the threads library under a number of different
+# names; the ordering is very important because some systems
+# (e.g. DEC) have both -lpthread and -lpthreads, where one of the
+# libraries is broken (non-POSIX).
+
+# Create a list of thread flags to try.  Items starting with a "-" are
+# C compiler flags, and other items are library names, except for "none"
+# which indicates that we try without any flags at all, and "pthread-config"
+# which is a program returning the flags for the Pth emulation library.
+
+ax_pthread_flags="pthreads none -Kthread -pthread -pthreads -mthreads pthread --thread-safe -mt pthread-config"
+
+# The ordering *is* (sometimes) important.  Some notes on the
+# individual items follow:
+
+# pthreads: AIX (must check this before -lpthread)
+# none: in case threads are in libc; should be tried before -Kthread and
+#       other compiler flags to prevent continual compiler warnings
+# -Kthread: Sequent (threads in libc, but -Kthread needed for pthread.h)
+# -pthread: Linux/gcc (kernel threads), BSD/gcc (userland threads), Tru64
+#           (Note: HP C rejects this with "bad form for `-t' option")
+# -pthreads: Solaris/gcc (Note: HP C also rejects)
+# -mt: Sun Workshop C (may only link SunOS threads [-lthread], but it
+#      doesn't hurt to check since this sometimes defines pthreads and
+#      -D_REENTRANT too), HP C (must be checked before -lpthread, which
+#      is present but should not be used directly; and before -mthreads,
+#      because the compiler interprets this as "-mt" + "-hreads")
+# -mthreads: Mingw32/gcc, Lynx/gcc
+# pthread: Linux, etcetera
+# --thread-safe: KAI C++
+# pthread-config: use pthread-config program (for GNU Pth library)
+
+case $host_os in
+
+	freebsd*)
+
+	# -kthread: FreeBSD kernel threads (preferred to -pthread since SMP-able)
+	# lthread: LinuxThreads port on FreeBSD (also preferred to -pthread)
+
+	ax_pthread_flags="-kthread lthread $ax_pthread_flags"
+	;;
+
+	hpux*)
+
+	# From the cc(1) man page: "[-mt] Sets various -D flags to enable
+	# multi-threading and also sets -lpthread."
+
+	ax_pthread_flags="-mt -pthread pthread $ax_pthread_flags"
+	;;
+
+	openedition*)
+
+	# IBM z/OS requires a feature-test macro to be defined in order to
+	# enable POSIX threads at all, so give the user a hint if this is
+	# not set. (We don't define these ourselves, as they can affect
+	# other portions of the system API in unpredictable ways.)
+
+	AC_EGREP_CPP([AX_PTHREAD_ZOS_MISSING],
+	    [
+#	     if !defined(_OPEN_THREADS) && !defined(_UNIX03_THREADS)
+	     AX_PTHREAD_ZOS_MISSING
+#	     endif
+	    ],
+	    [AC_MSG_WARN([IBM z/OS requires -D_OPEN_THREADS or -D_UNIX03_THREADS to enable pthreads support.])])
+	;;
+
+	solaris*)
+
+	# On Solaris (at least, for some versions), libc contains stubbed
+	# (non-functional) versions of the pthreads routines, so link-based
+	# tests will erroneously succeed. (N.B.: The stubs are missing
+	# pthread_cleanup_push, or rather a function called by this macro,
+	# so we could check for that, but who knows whether they'll stub
+	# that too in a future libc.)  So we'll check first for the
+	# standard Solaris way of linking pthreads (-mt -lpthread).
+
+	ax_pthread_flags="-mt,pthread pthread $ax_pthread_flags"
+	;;
+esac
+
+# GCC generally uses -pthread, or -pthreads on some platforms (e.g. SPARC)
+
+AS_IF([test "x$GCC" = "xyes"],
+      [ax_pthread_flags="-pthread -pthreads $ax_pthread_flags"])
+
+# The presence of a feature test macro requesting re-entrant function
+# definitions is, on some systems, a strong hint that pthreads support is
+# correctly enabled
+
+case $host_os in
+	darwin* | hpux* | linux* | osf* | solaris*)
+	ax_pthread_check_macro="_REENTRANT"
+	;;
+
+	aix* | freebsd*)
+	ax_pthread_check_macro="_THREAD_SAFE"
+	;;
+
+	*)
+	ax_pthread_check_macro="--"
+	;;
+esac
+AS_IF([test "x$ax_pthread_check_macro" = "x--"],
+      [ax_pthread_check_cond=0],
+      [ax_pthread_check_cond="!defined($ax_pthread_check_macro)"])
+
+# Are we compiling with Clang?
+
+AC_CACHE_CHECK([whether $CC is Clang],
+    [ax_cv_PTHREAD_CLANG],
+    [ax_cv_PTHREAD_CLANG=no
+     # Note that Autoconf sets GCC=yes for Clang as well as GCC
+     if test "x$GCC" = "xyes"; then
+	AC_EGREP_CPP([AX_PTHREAD_CC_IS_CLANG],
+	    [/* Note: Clang 2.7 lacks __clang_[a-z]+__ */
+#	     if defined(__clang__) && defined(__llvm__)
+	     AX_PTHREAD_CC_IS_CLANG
+#	     endif
+	    ],
+	    [ax_cv_PTHREAD_CLANG=yes])
+     fi
+    ])
+ax_pthread_clang="$ax_cv_PTHREAD_CLANG"
+
+ax_pthread_clang_warning=no
+
+# Clang needs special handling, because older versions handle the -pthread
+# option in a rather... idiosyncratic way
+
+if test "x$ax_pthread_clang" = "xyes"; then
+
+	# Clang takes -pthread; it has never supported any other flag
+
+	# (Note 1: This will need to be revisited if a system that Clang
+	# supports has POSIX threads in a separate library.  This tends not
+	# to be the way of modern systems, but it's conceivable.)
+
+	# (Note 2: On some systems, notably Darwin, -pthread is not needed
+	# to get POSIX threads support; the API is always present and
+	# active.  We could reasonably leave PTHREAD_CFLAGS empty.  But
+	# -pthread does define _REENTRANT, and while the Darwin headers
+	# ignore this macro, third-party headers might not.)
+
+	PTHREAD_CFLAGS="-pthread"
+	PTHREAD_LIBS=
+
+	ax_pthread_ok=yes
+
+	# However, older versions of Clang make a point of warning the user
+	# that, in an invocation where only linking and no compilation is
+	# taking place, the -pthread option has no effect ("argument unused
+	# during compilation").  They expect -pthread to be passed in only
+	# when source code is being compiled.
+	#
+	# Problem is, this is at odds with the way Automake and most other
+	# C build frameworks function, which is that the same flags used in
+	# compilation (CFLAGS) are also used in linking.  Many systems
+	# supported by AX_PTHREAD require exactly this for POSIX threads
+	# support, and in fact it is often not straightforward to specify a
+	# flag that is used only in the compilation phase and not in
+	# linking.  Such a scenario is extremely rare in practice.
+	#
+	# Even though use of the -pthread flag in linking would only print
+	# a warning, this can be a nuisance for well-run software projects
+	# that build with -Werror.  So if the active version of Clang has
+	# this misfeature, we search for an option to squash it.
+
+	AC_CACHE_CHECK([whether Clang needs flag to prevent "argument unused" warning when linking with -pthread],
+	    [ax_cv_PTHREAD_CLANG_NO_WARN_FLAG],
+	    [ax_cv_PTHREAD_CLANG_NO_WARN_FLAG=unknown
+	     # Create an alternate version of $ac_link that compiles and
+	     # links in two steps (.c -> .o, .o -> exe) instead of one
+	     # (.c -> exe), because the warning occurs only in the second
+	     # step
+	     ax_pthread_save_ac_link="$ac_link"
+	     ax_pthread_sed='s/conftest\.\$ac_ext/conftest.$ac_objext/g'
+	     ax_pthread_link_step=`$as_echo "$ac_link" | sed "$ax_pthread_sed"`
+	     ax_pthread_2step_ac_link="($ac_compile) && (echo ==== >&5) && ($ax_pthread_link_step)"
+	     ax_pthread_save_CFLAGS="$CFLAGS"
+	     for ax_pthread_try in '' -Qunused-arguments -Wno-unused-command-line-argument unknown; do
+		AS_IF([test "x$ax_pthread_try" = "xunknown"], [break])
+		CFLAGS="-Werror -Wunknown-warning-option $ax_pthread_try -pthread $ax_pthread_save_CFLAGS"
+		ac_link="$ax_pthread_save_ac_link"
+		AC_LINK_IFELSE([AC_LANG_SOURCE([[int main(void){return 0;}]])],
+		    [ac_link="$ax_pthread_2step_ac_link"
+		     AC_LINK_IFELSE([AC_LANG_SOURCE([[int main(void){return 0;}]])],
+			 [break])
+		    ])
+	     done
+	     ac_link="$ax_pthread_save_ac_link"
+	     CFLAGS="$ax_pthread_save_CFLAGS"
+	     AS_IF([test "x$ax_pthread_try" = "x"], [ax_pthread_try=no])
+	     ax_cv_PTHREAD_CLANG_NO_WARN_FLAG="$ax_pthread_try"
+	    ])
+
+	case "$ax_cv_PTHREAD_CLANG_NO_WARN_FLAG" in
+		no | unknown) ;;
+		*) PTHREAD_CFLAGS="$ax_cv_PTHREAD_CLANG_NO_WARN_FLAG $PTHREAD_CFLAGS" ;;
+	esac
+
+fi # $ax_pthread_clang = yes
+
+if test "x$ax_pthread_ok" = "xno"; then
+for ax_pthread_try_flag in $ax_pthread_flags; do
+
+	case $ax_pthread_try_flag in
+		none)
+		AC_MSG_CHECKING([whether pthreads work without any flags])
+		;;
+
+		-mt,pthread)
+		AC_MSG_CHECKING([whether pthreads work with -mt -lpthread])
+		PTHREAD_CFLAGS="-mt"
+		PTHREAD_LIBS="-lpthread"
+		;;
+
+		-*)
+		AC_MSG_CHECKING([whether pthreads work with $ax_pthread_try_flag])
+		PTHREAD_CFLAGS="$ax_pthread_try_flag"
+		;;
+
+		pthread-config)
+		AC_CHECK_PROG([ax_pthread_config], [pthread-config], [yes], [no])
+		AS_IF([test "x$ax_pthread_config" = "xno"], [continue])
+		PTHREAD_CFLAGS="`pthread-config --cflags`"
+		PTHREAD_LIBS="`pthread-config --ldflags` `pthread-config --libs`"
+		;;
+
+		*)
+		AC_MSG_CHECKING([for the pthreads library -l$ax_pthread_try_flag])
+		PTHREAD_LIBS="-l$ax_pthread_try_flag"
+		;;
+	esac
+
+	ax_pthread_save_CFLAGS="$CFLAGS"
+	ax_pthread_save_LIBS="$LIBS"
+	CFLAGS="$CFLAGS $PTHREAD_CFLAGS"
+	LIBS="$PTHREAD_LIBS $LIBS"
+
+	# Check for various functions.  We must include pthread.h,
+	# since some functions may be macros.  (On the Sequent, we
+	# need a special flag -Kthread to make this header compile.)
+	# We check for pthread_join because it is in -lpthread on IRIX
+	# while pthread_create is in libc.  We check for pthread_attr_init
+	# due to DEC craziness with -lpthreads.  We check for
+	# pthread_cleanup_push because it is one of the few pthread
+	# functions on Solaris that doesn't have a non-functional libc stub.
+	# We try pthread_create on general principles.
+
+	AC_LINK_IFELSE([AC_LANG_PROGRAM([#include <pthread.h>
+#			if $ax_pthread_check_cond
+#			 error "$ax_pthread_check_macro must be defined"
+#			endif
+			static void routine(void *a) { a = 0; }
+			static void *start_routine(void *a) { return a; }],
+		       [pthread_t th; pthread_attr_t attr;
+			pthread_create(&th, 0, start_routine, 0);
+			pthread_join(th, 0);
+			pthread_attr_init(&attr);
+			pthread_cleanup_push(routine, 0);
+			pthread_cleanup_pop(0) /* ; */])],
+	    [ax_pthread_ok=yes],
+	    [])
+
+	CFLAGS="$ax_pthread_save_CFLAGS"
+	LIBS="$ax_pthread_save_LIBS"
+
+	AC_MSG_RESULT([$ax_pthread_ok])
+	AS_IF([test "x$ax_pthread_ok" = "xyes"], [break])
+
+	PTHREAD_LIBS=""
+	PTHREAD_CFLAGS=""
+done
+fi
+
+# Various other checks:
+if test "x$ax_pthread_ok" = "xyes"; then
+	ax_pthread_save_CFLAGS="$CFLAGS"
+	ax_pthread_save_LIBS="$LIBS"
+	CFLAGS="$CFLAGS $PTHREAD_CFLAGS"
+	LIBS="$PTHREAD_LIBS $LIBS"
+
+	# Detect AIX lossage: JOINABLE attribute is called UNDETACHED.
+	AC_CACHE_CHECK([for joinable pthread attribute],
+	    [ax_cv_PTHREAD_JOINABLE_ATTR],
+	    [ax_cv_PTHREAD_JOINABLE_ATTR=unknown
+	     for ax_pthread_attr in PTHREAD_CREATE_JOINABLE PTHREAD_CREATE_UNDETACHED; do
+		 AC_LINK_IFELSE([AC_LANG_PROGRAM([#include <pthread.h>],
+						 [int attr = $ax_pthread_attr; return attr /* ; */])],
+				[ax_cv_PTHREAD_JOINABLE_ATTR=$ax_pthread_attr; break],
+				[])
+	     done
+	    ])
+	AS_IF([test "x$ax_cv_PTHREAD_JOINABLE_ATTR" != "xunknown" && \
+	       test "x$ax_cv_PTHREAD_JOINABLE_ATTR" != "xPTHREAD_CREATE_JOINABLE" && \
+	       test "x$ax_pthread_joinable_attr_defined" != "xyes"],
+	      [AC_DEFINE_UNQUOTED([PTHREAD_CREATE_JOINABLE],
+				  [$ax_cv_PTHREAD_JOINABLE_ATTR],
+				  [Define to necessary symbol if this constant
+				   uses a non-standard name on your system.])
+	       ax_pthread_joinable_attr_defined=yes
+	      ])
+
+	AC_CACHE_CHECK([whether more special flags are required for pthreads],
+	    [ax_cv_PTHREAD_SPECIAL_FLAGS],
+	    [ax_cv_PTHREAD_SPECIAL_FLAGS=no
+	     case $host_os in
+	     solaris*)
+	     ax_cv_PTHREAD_SPECIAL_FLAGS="-D_POSIX_PTHREAD_SEMANTICS"
+	     ;;
+	     esac
+	    ])
+	AS_IF([test "x$ax_cv_PTHREAD_SPECIAL_FLAGS" != "xno" && \
+	       test "x$ax_pthread_special_flags_added" != "xyes"],
+	      [PTHREAD_CFLAGS="$ax_cv_PTHREAD_SPECIAL_FLAGS $PTHREAD_CFLAGS"
+	       ax_pthread_special_flags_added=yes])
+
+	AC_CACHE_CHECK([for PTHREAD_PRIO_INHERIT],
+	    [ax_cv_PTHREAD_PRIO_INHERIT],
+	    [AC_LINK_IFELSE([AC_LANG_PROGRAM([[#include <pthread.h>]],
+					     [[int i = PTHREAD_PRIO_INHERIT;]])],
+			    [ax_cv_PTHREAD_PRIO_INHERIT=yes],
+			    [ax_cv_PTHREAD_PRIO_INHERIT=no])
+	    ])
+	AS_IF([test "x$ax_cv_PTHREAD_PRIO_INHERIT" = "xyes" && \
+	       test "x$ax_pthread_prio_inherit_defined" != "xyes"],
+	      [AC_DEFINE([HAVE_PTHREAD_PRIO_INHERIT], [1], [Have PTHREAD_PRIO_INHERIT.])
+	       ax_pthread_prio_inherit_defined=yes
+	      ])
+
+	CFLAGS="$ax_pthread_save_CFLAGS"
+	LIBS="$ax_pthread_save_LIBS"
+
+	# More AIX lossage: compile with *_r variant
+	if test "x$GCC" != "xyes"; then
+	    case $host_os in
+		aix*)
+		AS_CASE(["x/$CC"],
+		    [x*/c89|x*/c89_128|x*/c99|x*/c99_128|x*/cc|x*/cc128|x*/xlc|x*/xlc_v6|x*/xlc128|x*/xlc128_v6],
+		    [#handle absolute path differently from PATH based program lookup
+		     AS_CASE(["x$CC"],
+			 [x/*],
+			 [AS_IF([AS_EXECUTABLE_P([${CC}_r])],[PTHREAD_CC="${CC}_r"])],
+			 [AC_CHECK_PROGS([PTHREAD_CC],[${CC}_r],[$CC])])])
+		;;
+	    esac
+	fi
+fi
+
+test -n "$PTHREAD_CC" || PTHREAD_CC="$CC"
+
+AC_SUBST([PTHREAD_LIBS])
+AC_SUBST([PTHREAD_CFLAGS])
+AC_SUBST([PTHREAD_CC])
+
+# Finally, execute ACTION-IF-FOUND/ACTION-IF-NOT-FOUND:
+if test "x$ax_pthread_ok" = "xyes"; then
+	ifelse([$1],,[AC_DEFINE([HAVE_PTHREAD],[1],[Define if you have POSIX threads libraries and header files.])],[$1])
+	:
+else
+	ax_pthread_ok=no
+	$2
+fi
+AC_LANG_POP
+])dnl AX_PTHREAD

objectivec/GPBCodedOutputStream.m

@@ -942,7 +942,10 @@ static void GPBWriteRawLittleEndian64(GPBOutputBufferState *state,
       state_.position = length;
     } else {
       // Write is very big.  Let's do it all at once.
-      [state_.output write:((uint8_t *)value) + offset maxLength:length];
+      NSInteger written = [state_.output write:((uint8_t *)value) + offset maxLength:length];
+      if (written != (NSInteger)length) {
+        [NSException raise:GPBCodedOutputStreamException_WriteFailed format:@""];
+      }
     }
   }
 }

objectivec/Tests/GPBCodedOuputStreamTests.m

@@ -423,4 +423,14 @@
   }
 }
 
+- (void)testThatItThrowsWhenWriteRawPtrFails {
+  NSOutputStream *output = [NSOutputStream outputStreamToMemory];
+  GPBCodedOutputStream *codedOutput =
+      [GPBCodedOutputStream streamWithOutputStream:output bufferSize:0];  // Skip buffering.
+  [output close];  // Close the output stream to force failure on write.
+  const char *cString = "raw";
+  XCTAssertThrowsSpecificNamed([codedOutput writeRawPtr:cString offset:0 length:strlen(cString)],
+                               NSException, GPBCodedOutputStreamException_WriteFailed);
+}
+
 @end

objectivec/google/protobuf/FieldMask.pbobjc.h

@@ -255,8 +255,8 @@ typedef GPB_ENUM(GPBFieldMask_FieldNumber) {
  *
  * ## Field Mask Verification
  *
- * The implementation of the all the API methods, which have any FieldMask type
- * field in the request, should verify the included field paths, and return
+ * The implementation of any API method which has a FieldMask type field in the
+ * request should verify the included field paths, and return an
  * `INVALID_ARGUMENT` error if any path is duplicated or unmappable.
  **/
 @interface GPBFieldMask : GPBMessage

php/ext/google/protobuf/upb.h

@@ -1,4 +1,111 @@
 // Amalgamated source file
+
+// php.h intentionally defined NDEBUG. We have to define this macro in order to
+// be used together with php.h
+#ifndef NDEBUG
+#define NDEBUG
+#endif
+
+/*
+** upb_decode: parsing into a upb_msg using a upb_msglayout.
+*/
+
+#ifndef UPB_DECODE_H_
+#define UPB_DECODE_H_
+
+/*
+** upb::Message is a representation for protobuf messages.
+**
+** However it differs from other common representations like
+** google::protobuf::Message in one key way: it does not prescribe any
+** ownership between messages and submessages, and it relies on the
+** client to delete each message/submessage/array/map at the appropriate
+** time.
+**
+** A client can access a upb::Message without knowing anything about
+** ownership semantics, but to create or mutate a message a user needs
+** to implement the memory management themselves.
+**
+** Currently all messages, arrays, and maps store a upb_alloc* internally.
+** Mutating operations use this when they require dynamically-allocated
+** memory.  We could potentially eliminate this size overhead later by
+** letting the user flip a bit on the factory that prevents this from
+** being stored.  The user would then need to use separate functions where
+** the upb_alloc* is passed explicitly.  However for handlers to populate
+** such structures, they would need a place to store this upb_alloc* during
+** parsing; upb_handlers don't currently have a good way to accommodate this.
+**
+** TODO: UTF-8 checking?
+**/
+
+#ifndef UPB_MSG_H_
+#define UPB_MSG_H_
+
+/*
+** Defs are upb's internal representation of the constructs that can appear
+** in a .proto file:
+**
+** - upb::MessageDef (upb_msgdef): describes a "message" construct.
+** - upb::FieldDef (upb_fielddef): describes a message field.
+** - upb::FileDef (upb_filedef): describes a .proto file and its defs.
+** - upb::EnumDef (upb_enumdef): describes an enum.
+** - upb::OneofDef (upb_oneofdef): describes a oneof.
+** - upb::Def (upb_def): base class of all the others.
+**
+** TODO: definitions of services.
+**
+** Like upb_refcounted objects, defs are mutable only until frozen, and are
+** only thread-safe once frozen.
+**
+** This is a mixed C/C++ interface that offers a full API to both languages.
+** See the top-level README for more information.
+*/
+
+#ifndef UPB_DEF_H_
+#define UPB_DEF_H_
+
+/*
+** upb::RefCounted (upb_refcounted)
+**
+** A refcounting scheme that supports circular refs.  It accomplishes this by
+** partitioning the set of objects into groups such that no cycle spans groups;
+** we can then reference-count the group as a whole and ignore refs within the
+** group.  When objects are mutable, these groups are computed very
+** conservatively; we group any objects that have ever had a link between them.
+** When objects are frozen, we compute strongly-connected components which
+** allows us to be precise and only group objects that are actually cyclic.
+**
+** This is a mixed C/C++ interface that offers a full API to both languages.
+** See the top-level README for more information.
+*/
+
+#ifndef UPB_REFCOUNTED_H_
+#define UPB_REFCOUNTED_H_
+
+/*
+** upb_table
+**
+** This header is INTERNAL-ONLY!  Its interfaces are not public or stable!
+** This file defines very fast int->upb_value (inttable) and string->upb_value
+** (strtable) hash tables.
+**
+** The table uses chained scatter with Brent's variation (inspired by the Lua
+** implementation of hash tables).  The hash function for strings is Austin
+** Appleby's "MurmurHash."
+**
+** The inttable uses uintptr_t as its key, which guarantees it can be used to
+** store pointers or integers of at least 32 bits (upb isn't really useful on
+** systems where sizeof(void*) < 4).
+**
+** The table must be homogenous (all values of the same type).  In debug
+** mode, we check this on insert and lookup.
+*/
+
+#ifndef UPB_TABLE_H_
+#define UPB_TABLE_H_
+
+#include <stdint.h>
+#include <string.h>
 /*
 ** This file contains shared definitions that are widely used across upb.
 **
@@ -9,12 +116,6 @@
 #ifndef UPB_H_
 #define UPB_H_
 
-// php.h intentionally defined NDEBUG. We have to define this macro in order to
-// be used together with php.h
-#ifndef NDEBUG
-#define NDEBUG
-#endif
-
 #include <assert.h>
 #include <stdarg.h>
 #include <stdbool.h>
@@ -743,106 +844,6 @@ template <int N> class upb::InlinedEnvironment : public upb::Environment {
 
 
 #endif  /* UPB_H_ */
-/*
-** upb_decode: parsing into a upb_msg using a upb_msglayout.
-*/
-
-#ifndef UPB_DECODE_H_
-#define UPB_DECODE_H_
-
-/*
-** upb::Message is a representation for protobuf messages.
-**
-** However it differs from other common representations like
-** google::protobuf::Message in one key way: it does not prescribe any
-** ownership between messages and submessages, and it relies on the
-** client to delete each message/submessage/array/map at the appropriate
-** time.
-**
-** A client can access a upb::Message without knowing anything about
-** ownership semantics, but to create or mutate a message a user needs
-** to implement the memory management themselves.
-**
-** Currently all messages, arrays, and maps store a upb_alloc* internally.
-** Mutating operations use this when they require dynamically-allocated
-** memory.  We could potentially eliminate this size overhead later by
-** letting the user flip a bit on the factory that prevents this from
-** being stored.  The user would then need to use separate functions where
-** the upb_alloc* is passed explicitly.  However for handlers to populate
-** such structures, they would need a place to store this upb_alloc* during
-** parsing; upb_handlers don't currently have a good way to accommodate this.
-**
-** TODO: UTF-8 checking?
-**/
-
-#ifndef UPB_MSG_H_
-#define UPB_MSG_H_
-
-/*
-** Defs are upb's internal representation of the constructs that can appear
-** in a .proto file:
-**
-** - upb::MessageDef (upb_msgdef): describes a "message" construct.
-** - upb::FieldDef (upb_fielddef): describes a message field.
-** - upb::FileDef (upb_filedef): describes a .proto file and its defs.
-** - upb::EnumDef (upb_enumdef): describes an enum.
-** - upb::OneofDef (upb_oneofdef): describes a oneof.
-** - upb::Def (upb_def): base class of all the others.
-**
-** TODO: definitions of services.
-**
-** Like upb_refcounted objects, defs are mutable only until frozen, and are
-** only thread-safe once frozen.
-**
-** This is a mixed C/C++ interface that offers a full API to both languages.
-** See the top-level README for more information.
-*/
-
-#ifndef UPB_DEF_H_
-#define UPB_DEF_H_
-
-/*
-** upb::RefCounted (upb_refcounted)
-**
-** A refcounting scheme that supports circular refs.  It accomplishes this by
-** partitioning the set of objects into groups such that no cycle spans groups;
-** we can then reference-count the group as a whole and ignore refs within the
-** group.  When objects are mutable, these groups are computed very
-** conservatively; we group any objects that have ever had a link between them.
-** When objects are frozen, we compute strongly-connected components which
-** allows us to be precise and only group objects that are actually cyclic.
-**
-** This is a mixed C/C++ interface that offers a full API to both languages.
-** See the top-level README for more information.
-*/
-
-#ifndef UPB_REFCOUNTED_H_
-#define UPB_REFCOUNTED_H_
-
-/*
-** upb_table
-**
-** This header is INTERNAL-ONLY!  Its interfaces are not public or stable!
-** This file defines very fast int->upb_value (inttable) and string->upb_value
-** (strtable) hash tables.
-**
-** The table uses chained scatter with Brent's variation (inspired by the Lua
-** implementation of hash tables).  The hash function for strings is Austin
-** Appleby's "MurmurHash."
-**
-** The inttable uses uintptr_t as its key, which guarantees it can be used to
-** store pointers or integers of at least 32 bits (upb isn't really useful on
-** systems where sizeof(void*) < 4).
-**
-** The table must be homogenous (all values of the same type).  In debug
-** mode, we check this on insert and lookup.
-*/
-
-#ifndef UPB_TABLE_H_
-#define UPB_TABLE_H_
-
-#include <stdint.h>
-#include <string.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -2001,6 +2002,9 @@ typedef enum {
   UPB_SYNTAX_PROTO3 = 3
 } upb_syntax_t;
 
+/* Maps descriptor type -> upb field type.  */
+extern const uint8_t upb_desctype_to_fieldtype[];
+
 /* Maximum field number allowed for FieldDefs.  This is an inherent limit of the
  * protobuf wire format. */
 #define UPB_MAX_FIELDNUMBER ((1 << 29) - 1)
@@ -6793,6 +6797,714 @@ UPB_END_EXTERN_C
 
 #endif  /* UPB_DECODE_H_ */
 /*
+** upb_encode: parsing into a upb_msg using a upb_msglayout.
+*/
+
+#ifndef UPB_ENCODE_H_
+#define UPB_ENCODE_H_
+
+
+UPB_BEGIN_EXTERN_C
+
+char *upb_encode(const void *msg, const upb_msglayout_msginit_v1 *l,
+                 upb_env *env, size_t *size);
+
+UPB_END_EXTERN_C
+
+#endif  /* UPB_ENCODE_H_ */
+/* This file was generated by upbc (the upb compiler) from the input
+ * file:
+ *
+ *     google/protobuf/descriptor.proto
+ *
+ * Do not edit -- your changes will be discarded when the file is
+ * regenerated. */
+
+#ifndef GOOGLE_PROTOBUF_DESCRIPTOR_PROTO_UPB_H_
+#define GOOGLE_PROTOBUF_DESCRIPTOR_PROTO_UPB_H_
+
+
+UPB_BEGIN_EXTERN_C
+
+struct google_protobuf_FileDescriptorSet;
+typedef struct google_protobuf_FileDescriptorSet google_protobuf_FileDescriptorSet;
+struct google_protobuf_FileDescriptorProto;
+typedef struct google_protobuf_FileDescriptorProto google_protobuf_FileDescriptorProto;
+struct google_protobuf_DescriptorProto;
+typedef struct google_protobuf_DescriptorProto google_protobuf_DescriptorProto;
+struct google_protobuf_DescriptorProto_ExtensionRange;
+typedef struct google_protobuf_DescriptorProto_ExtensionRange google_protobuf_DescriptorProto_ExtensionRange;
+struct google_protobuf_DescriptorProto_ReservedRange;
+typedef struct google_protobuf_DescriptorProto_ReservedRange google_protobuf_DescriptorProto_ReservedRange;
+struct google_protobuf_ExtensionRangeOptions;
+typedef struct google_protobuf_ExtensionRangeOptions google_protobuf_ExtensionRangeOptions;
+struct google_protobuf_FieldDescriptorProto;
+typedef struct google_protobuf_FieldDescriptorProto google_protobuf_FieldDescriptorProto;
+struct google_protobuf_OneofDescriptorProto;
+typedef struct google_protobuf_OneofDescriptorProto google_protobuf_OneofDescriptorProto;
+struct google_protobuf_EnumDescriptorProto;
+typedef struct google_protobuf_EnumDescriptorProto google_protobuf_EnumDescriptorProto;
+struct google_protobuf_EnumDescriptorProto_EnumReservedRange;
+typedef struct google_protobuf_EnumDescriptorProto_EnumReservedRange google_protobuf_EnumDescriptorProto_EnumReservedRange;
+struct google_protobuf_EnumValueDescriptorProto;
+typedef struct google_protobuf_EnumValueDescriptorProto google_protobuf_EnumValueDescriptorProto;
+struct google_protobuf_ServiceDescriptorProto;
+typedef struct google_protobuf_ServiceDescriptorProto google_protobuf_ServiceDescriptorProto;
+struct google_protobuf_MethodDescriptorProto;
+typedef struct google_protobuf_MethodDescriptorProto google_protobuf_MethodDescriptorProto;
+struct google_protobuf_FileOptions;
+typedef struct google_protobuf_FileOptions google_protobuf_FileOptions;
+struct google_protobuf_MessageOptions;
+typedef struct google_protobuf_MessageOptions google_protobuf_MessageOptions;
+struct google_protobuf_FieldOptions;
+typedef struct google_protobuf_FieldOptions google_protobuf_FieldOptions;
+struct google_protobuf_OneofOptions;
+typedef struct google_protobuf_OneofOptions google_protobuf_OneofOptions;
+struct google_protobuf_EnumOptions;
+typedef struct google_protobuf_EnumOptions google_protobuf_EnumOptions;
+struct google_protobuf_EnumValueOptions;
+typedef struct google_protobuf_EnumValueOptions google_protobuf_EnumValueOptions;
+struct google_protobuf_ServiceOptions;
+typedef struct google_protobuf_ServiceOptions google_protobuf_ServiceOptions;
+struct google_protobuf_MethodOptions;
+typedef struct google_protobuf_MethodOptions google_protobuf_MethodOptions;
+struct google_protobuf_UninterpretedOption;
+typedef struct google_protobuf_UninterpretedOption google_protobuf_UninterpretedOption;
+struct google_protobuf_UninterpretedOption_NamePart;
+typedef struct google_protobuf_UninterpretedOption_NamePart google_protobuf_UninterpretedOption_NamePart;
+struct google_protobuf_SourceCodeInfo;
+typedef struct google_protobuf_SourceCodeInfo google_protobuf_SourceCodeInfo;
+struct google_protobuf_SourceCodeInfo_Location;
+typedef struct google_protobuf_SourceCodeInfo_Location google_protobuf_SourceCodeInfo_Location;
+struct google_protobuf_GeneratedCodeInfo;
+typedef struct google_protobuf_GeneratedCodeInfo google_protobuf_GeneratedCodeInfo;
+struct google_protobuf_GeneratedCodeInfo_Annotation;
+typedef struct google_protobuf_GeneratedCodeInfo_Annotation google_protobuf_GeneratedCodeInfo_Annotation;
+/* Enums */
+
+typedef enum {
+  google_protobuf_FieldDescriptorProto_LABEL_OPTIONAL = 1,
+  google_protobuf_FieldDescriptorProto_LABEL_REQUIRED = 2,
+  google_protobuf_FieldDescriptorProto_LABEL_REPEATED = 3
+} google_protobuf_FieldDescriptorProto_Label;
+
+typedef enum {
+  google_protobuf_FieldDescriptorProto_TYPE_DOUBLE = 1,
+  google_protobuf_FieldDescriptorProto_TYPE_FLOAT = 2,
+  google_protobuf_FieldDescriptorProto_TYPE_INT64 = 3,
+  google_protobuf_FieldDescriptorProto_TYPE_UINT64 = 4,
+  google_protobuf_FieldDescriptorProto_TYPE_INT32 = 5,
+  google_protobuf_FieldDescriptorProto_TYPE_FIXED64 = 6,
+  google_protobuf_FieldDescriptorProto_TYPE_FIXED32 = 7,
+  google_protobuf_FieldDescriptorProto_TYPE_BOOL = 8,
+  google_protobuf_FieldDescriptorProto_TYPE_STRING = 9,
+  google_protobuf_FieldDescriptorProto_TYPE_GROUP = 10,
+  google_protobuf_FieldDescriptorProto_TYPE_MESSAGE = 11,
+  google_protobuf_FieldDescriptorProto_TYPE_BYTES = 12,
+  google_protobuf_FieldDescriptorProto_TYPE_UINT32 = 13,
+  google_protobuf_FieldDescriptorProto_TYPE_ENUM = 14,
+  google_protobuf_FieldDescriptorProto_TYPE_SFIXED32 = 15,
+  google_protobuf_FieldDescriptorProto_TYPE_SFIXED64 = 16,
+  google_protobuf_FieldDescriptorProto_TYPE_SINT32 = 17,
+  google_protobuf_FieldDescriptorProto_TYPE_SINT64 = 18
+} google_protobuf_FieldDescriptorProto_Type;
+
+typedef enum {
+  google_protobuf_FieldOptions_STRING = 0,
+  google_protobuf_FieldOptions_CORD = 1,
+  google_protobuf_FieldOptions_STRING_PIECE = 2
+} google_protobuf_FieldOptions_CType;
+
+typedef enum {
+  google_protobuf_FieldOptions_JS_NORMAL = 0,
+  google_protobuf_FieldOptions_JS_STRING = 1,
+  google_protobuf_FieldOptions_JS_NUMBER = 2
+} google_protobuf_FieldOptions_JSType;
+
+typedef enum {
+  google_protobuf_FileOptions_SPEED = 1,
+  google_protobuf_FileOptions_CODE_SIZE = 2,
+  google_protobuf_FileOptions_LITE_RUNTIME = 3
+} google_protobuf_FileOptions_OptimizeMode;
+
+typedef enum {
+  google_protobuf_MethodOptions_IDEMPOTENCY_UNKNOWN = 0,
+  google_protobuf_MethodOptions_NO_SIDE_EFFECTS = 1,
+  google_protobuf_MethodOptions_IDEMPOTENT = 2
+} google_protobuf_MethodOptions_IdempotencyLevel;
+
+/* google_protobuf_FileDescriptorSet */
+extern const upb_msglayout_msginit_v1 google_protobuf_FileDescriptorSet_msginit;
+google_protobuf_FileDescriptorSet *google_protobuf_FileDescriptorSet_new(upb_env *env);
+google_protobuf_FileDescriptorSet *google_protobuf_FileDescriptorSet_parsenew(upb_stringview buf, upb_env *env);
+char *google_protobuf_FileDescriptorSet_serialize(google_protobuf_FileDescriptorSet *msg, upb_env *env, size_t *len);
+void google_protobuf_FileDescriptorSet_free(google_protobuf_FileDescriptorSet *msg, upb_env *env);
+
+/* getters. */
+const upb_array* google_protobuf_FileDescriptorSet_file(const google_protobuf_FileDescriptorSet *msg);
+
+/* setters. */
+void google_protobuf_FileDescriptorSet_set_file(google_protobuf_FileDescriptorSet *msg, upb_array* value);
+
+
+/* google_protobuf_FileDescriptorProto */
+extern const upb_msglayout_msginit_v1 google_protobuf_FileDescriptorProto_msginit;
+google_protobuf_FileDescriptorProto *google_protobuf_FileDescriptorProto_new(upb_env *env);
+google_protobuf_FileDescriptorProto *google_protobuf_FileDescriptorProto_parsenew(upb_stringview buf, upb_env *env);
+char *google_protobuf_FileDescriptorProto_serialize(google_protobuf_FileDescriptorProto *msg, upb_env *env, size_t *len);
+void google_protobuf_FileDescriptorProto_free(google_protobuf_FileDescriptorProto *msg, upb_env *env);
+
+/* getters. */
+upb_stringview google_protobuf_FileDescriptorProto_name(const google_protobuf_FileDescriptorProto *msg);
+upb_stringview google_protobuf_FileDescriptorProto_package(const google_protobuf_FileDescriptorProto *msg);
+const upb_array* google_protobuf_FileDescriptorProto_dependency(const google_protobuf_FileDescriptorProto *msg);
+const upb_array* google_protobuf_FileDescriptorProto_message_type(const google_protobuf_FileDescriptorProto *msg);
+const upb_array* google_protobuf_FileDescriptorProto_enum_type(const google_protobuf_FileDescriptorProto *msg);
+const upb_array* google_protobuf_FileDescriptorProto_service(const google_protobuf_FileDescriptorProto *msg);
+const upb_array* google_protobuf_FileDescriptorProto_extension(const google_protobuf_FileDescriptorProto *msg);
+const google_protobuf_FileOptions* google_protobuf_FileDescriptorProto_options(const google_protobuf_FileDescriptorProto *msg);
+const google_protobuf_SourceCodeInfo* google_protobuf_FileDescriptorProto_source_code_info(const google_protobuf_FileDescriptorProto *msg);
+const upb_array* google_protobuf_FileDescriptorProto_public_dependency(const google_protobuf_FileDescriptorProto *msg);
+const upb_array* google_protobuf_FileDescriptorProto_weak_dependency(const google_protobuf_FileDescriptorProto *msg);
+upb_stringview google_protobuf_FileDescriptorProto_syntax(const google_protobuf_FileDescriptorProto *msg);
+
+/* setters. */
+void google_protobuf_FileDescriptorProto_set_name(google_protobuf_FileDescriptorProto *msg, upb_stringview value);
+void google_protobuf_FileDescriptorProto_set_package(google_protobuf_FileDescriptorProto *msg, upb_stringview value);
+void google_protobuf_FileDescriptorProto_set_dependency(google_protobuf_FileDescriptorProto *msg, upb_array* value);
+void google_protobuf_FileDescriptorProto_set_message_type(google_protobuf_FileDescriptorProto *msg, upb_array* value);
+void google_protobuf_FileDescriptorProto_set_enum_type(google_protobuf_FileDescriptorProto *msg, upb_array* value);
+void google_protobuf_FileDescriptorProto_set_service(google_protobuf_FileDescriptorProto *msg, upb_array* value);
+void google_protobuf_FileDescriptorProto_set_extension(google_protobuf_FileDescriptorProto *msg, upb_array* value);
+void google_protobuf_FileDescriptorProto_set_options(google_protobuf_FileDescriptorProto *msg, google_protobuf_FileOptions* value);
+void google_protobuf_FileDescriptorProto_set_source_code_info(google_protobuf_FileDescriptorProto *msg, google_protobuf_SourceCodeInfo* value);
+void google_protobuf_FileDescriptorProto_set_public_dependency(google_protobuf_FileDescriptorProto *msg, upb_array* value);
+void google_protobuf_FileDescriptorProto_set_weak_dependency(google_protobuf_FileDescriptorProto *msg, upb_array* value);
+void google_protobuf_FileDescriptorProto_set_syntax(google_protobuf_FileDescriptorProto *msg, upb_stringview value);
+
+
+/* google_protobuf_DescriptorProto */
+extern const upb_msglayout_msginit_v1 google_protobuf_DescriptorProto_msginit;
+google_protobuf_DescriptorProto *google_protobuf_DescriptorProto_new(upb_env *env);
+google_protobuf_DescriptorProto *google_protobuf_DescriptorProto_parsenew(upb_stringview buf, upb_env *env);
+char *google_protobuf_DescriptorProto_serialize(google_protobuf_DescriptorProto *msg, upb_env *env, size_t *len);
+void google_protobuf_DescriptorProto_free(google_protobuf_DescriptorProto *msg, upb_env *env);
+
+/* getters. */
+upb_stringview google_protobuf_DescriptorProto_name(const google_protobuf_DescriptorProto *msg);
+const upb_array* google_protobuf_DescriptorProto_field(const google_protobuf_DescriptorProto *msg);
+const upb_array* google_protobuf_DescriptorProto_nested_type(const google_protobuf_DescriptorProto *msg);
+const upb_array* google_protobuf_DescriptorProto_enum_type(const google_protobuf_DescriptorProto *msg);
+const upb_array* google_protobuf_DescriptorProto_extension_range(const google_protobuf_DescriptorProto *msg);
+const upb_array* google_protobuf_DescriptorProto_extension(const google_protobuf_DescriptorProto *msg);
+const google_protobuf_MessageOptions* google_protobuf_DescriptorProto_options(const google_protobuf_DescriptorProto *msg);
+const upb_array* google_protobuf_DescriptorProto_oneof_decl(const google_protobuf_DescriptorProto *msg);
+const upb_array* google_protobuf_DescriptorProto_reserved_range(const google_protobuf_DescriptorProto *msg);
+const upb_array* google_protobuf_DescriptorProto_reserved_name(const google_protobuf_DescriptorProto *msg);
+
+/* setters. */
+void google_protobuf_DescriptorProto_set_name(google_protobuf_DescriptorProto *msg, upb_stringview value);
+void google_protobuf_DescriptorProto_set_field(google_protobuf_DescriptorProto *msg, upb_array* value);
+void google_protobuf_DescriptorProto_set_nested_type(google_protobuf_DescriptorProto *msg, upb_array* value);
+void google_protobuf_DescriptorProto_set_enum_type(google_protobuf_DescriptorProto *msg, upb_array* value);
+void google_protobuf_DescriptorProto_set_extension_range(google_protobuf_DescriptorProto *msg, upb_array* value);
+void google_protobuf_DescriptorProto_set_extension(google_protobuf_DescriptorProto *msg, upb_array* value);
+void google_protobuf_DescriptorProto_set_options(google_protobuf_DescriptorProto *msg, google_protobuf_MessageOptions* value);
+void google_protobuf_DescriptorProto_set_oneof_decl(google_protobuf_DescriptorProto *msg, upb_array* value);
+void google_protobuf_DescriptorProto_set_reserved_range(google_protobuf_DescriptorProto *msg, upb_array* value);
+void google_protobuf_DescriptorProto_set_reserved_name(google_protobuf_DescriptorProto *msg, upb_array* value);
+
+
+/* google_protobuf_DescriptorProto_ExtensionRange */
+extern const upb_msglayout_msginit_v1 google_protobuf_DescriptorProto_ExtensionRange_msginit;
+google_protobuf_DescriptorProto_ExtensionRange *google_protobuf_DescriptorProto_ExtensionRange_new(upb_env *env);
+google_protobuf_DescriptorProto_ExtensionRange *google_protobuf_DescriptorProto_ExtensionRange_parsenew(upb_stringview buf, upb_env *env);
+char *google_protobuf_DescriptorProto_ExtensionRange_serialize(google_protobuf_DescriptorProto_ExtensionRange *msg, upb_env *env, size_t *len);
+void google_protobuf_DescriptorProto_ExtensionRange_free(google_protobuf_DescriptorProto_ExtensionRange *msg, upb_env *env);
+
+/* getters. */
+int32_t google_protobuf_DescriptorProto_ExtensionRange_start(const google_protobuf_DescriptorProto_ExtensionRange *msg);
+int32_t google_protobuf_DescriptorProto_ExtensionRange_end(const google_protobuf_DescriptorProto_ExtensionRange *msg);
+const google_protobuf_ExtensionRangeOptions* google_protobuf_DescriptorProto_ExtensionRange_options(const google_protobuf_DescriptorProto_ExtensionRange *msg);
+
+/* setters. */
+void google_protobuf_DescriptorProto_ExtensionRange_set_start(google_protobuf_DescriptorProto_ExtensionRange *msg, int32_t value);
+void google_protobuf_DescriptorProto_ExtensionRange_set_end(google_protobuf_DescriptorProto_ExtensionRange *msg, int32_t value);
+void google_protobuf_DescriptorProto_ExtensionRange_set_options(google_protobuf_DescriptorProto_ExtensionRange *msg, google_protobuf_ExtensionRangeOptions* value);
+
+
+/* google_protobuf_DescriptorProto_ReservedRange */
+extern const upb_msglayout_msginit_v1 google_protobuf_DescriptorProto_ReservedRange_msginit;
+google_protobuf_DescriptorProto_ReservedRange *google_protobuf_DescriptorProto_ReservedRange_new(upb_env *env);
+google_protobuf_DescriptorProto_ReservedRange *google_protobuf_DescriptorProto_ReservedRange_parsenew(upb_stringview buf, upb_env *env);
+char *google_protobuf_DescriptorProto_ReservedRange_serialize(google_protobuf_DescriptorProto_ReservedRange *msg, upb_env *env, size_t *len);
+void google_protobuf_DescriptorProto_ReservedRange_free(google_protobuf_DescriptorProto_ReservedRange *msg, upb_env *env);
+
+/* getters. */
+int32_t google_protobuf_DescriptorProto_ReservedRange_start(const google_protobuf_DescriptorProto_ReservedRange *msg);
+int32_t google_protobuf_DescriptorProto_ReservedRange_end(const google_protobuf_DescriptorProto_ReservedRange *msg);
+
+/* setters. */
+void google_protobuf_DescriptorProto_ReservedRange_set_start(google_protobuf_DescriptorProto_ReservedRange *msg, int32_t value);
+void google_protobuf_DescriptorProto_ReservedRange_set_end(google_protobuf_DescriptorProto_ReservedRange *msg, int32_t value);
+
+
+/* google_protobuf_ExtensionRangeOptions */
+extern const upb_msglayout_msginit_v1 google_protobuf_ExtensionRangeOptions_msginit;
+google_protobuf_ExtensionRangeOptions *google_protobuf_ExtensionRangeOptions_new(upb_env *env);
+google_protobuf_ExtensionRangeOptions *google_protobuf_ExtensionRangeOptions_parsenew(upb_stringview buf, upb_env *env);
+char *google_protobuf_ExtensionRangeOptions_serialize(google_protobuf_ExtensionRangeOptions *msg, upb_env *env, size_t *len);
+void google_protobuf_ExtensionRangeOptions_free(google_protobuf_ExtensionRangeOptions *msg, upb_env *env);
+
+/* getters. */
+const upb_array* google_protobuf_ExtensionRangeOptions_uninterpreted_option(const google_protobuf_ExtensionRangeOptions *msg);
+
+/* setters. */
+void google_protobuf_ExtensionRangeOptions_set_uninterpreted_option(google_protobuf_ExtensionRangeOptions *msg, upb_array* value);
+
+
+/* google_protobuf_FieldDescriptorProto */
+extern const upb_msglayout_msginit_v1 google_protobuf_FieldDescriptorProto_msginit;
+google_protobuf_FieldDescriptorProto *google_protobuf_FieldDescriptorProto_new(upb_env *env);
+google_protobuf_FieldDescriptorProto *google_protobuf_FieldDescriptorProto_parsenew(upb_stringview buf, upb_env *env);
+char *google_protobuf_FieldDescriptorProto_serialize(google_protobuf_FieldDescriptorProto *msg, upb_env *env, size_t *len);
+void google_protobuf_FieldDescriptorProto_free(google_protobuf_FieldDescriptorProto *msg, upb_env *env);
+
+/* getters. */
+upb_stringview google_protobuf_FieldDescriptorProto_name(const google_protobuf_FieldDescriptorProto *msg);
+upb_stringview google_protobuf_FieldDescriptorProto_extendee(const google_protobuf_FieldDescriptorProto *msg);
+int32_t google_protobuf_FieldDescriptorProto_number(const google_protobuf_FieldDescriptorProto *msg);
+google_protobuf_FieldDescriptorProto_Label google_protobuf_FieldDescriptorProto_label(const google_protobuf_FieldDescriptorProto *msg);
+google_protobuf_FieldDescriptorProto_Type google_protobuf_FieldDescriptorProto_type(const google_protobuf_FieldDescriptorProto *msg);
+upb_stringview google_protobuf_FieldDescriptorProto_type_name(const google_protobuf_FieldDescriptorProto *msg);
+upb_stringview google_protobuf_FieldDescriptorProto_default_value(const google_protobuf_FieldDescriptorProto *msg);
+const google_protobuf_FieldOptions* google_protobuf_FieldDescriptorProto_options(const google_protobuf_FieldDescriptorProto *msg);
+int32_t google_protobuf_FieldDescriptorProto_oneof_index(const google_protobuf_FieldDescriptorProto *msg);
+upb_stringview google_protobuf_FieldDescriptorProto_json_name(const google_protobuf_FieldDescriptorProto *msg);
+
+/* setters. */
+void google_protobuf_FieldDescriptorProto_set_name(google_protobuf_FieldDescriptorProto *msg, upb_stringview value);
+void google_protobuf_FieldDescriptorProto_set_extendee(google_protobuf_FieldDescriptorProto *msg, upb_stringview value);
+void google_protobuf_FieldDescriptorProto_set_number(google_protobuf_FieldDescriptorProto *msg, int32_t value);
+void google_protobuf_FieldDescriptorProto_set_label(google_protobuf_FieldDescriptorProto *msg, google_protobuf_FieldDescriptorProto_Label value);
+void google_protobuf_FieldDescriptorProto_set_type(google_protobuf_FieldDescriptorProto *msg, google_protobuf_FieldDescriptorProto_Type value);
+void google_protobuf_FieldDescriptorProto_set_type_name(google_protobuf_FieldDescriptorProto *msg, upb_stringview value);
+void google_protobuf_FieldDescriptorProto_set_default_value(google_protobuf_FieldDescriptorProto *msg, upb_stringview value);
+void google_protobuf_FieldDescriptorProto_set_options(google_protobuf_FieldDescriptorProto *msg, google_protobuf_FieldOptions* value);
+void google_protobuf_FieldDescriptorProto_set_oneof_index(google_protobuf_FieldDescriptorProto *msg, int32_t value);
+void google_protobuf_FieldDescriptorProto_set_json_name(google_protobuf_FieldDescriptorProto *msg, upb_stringview value);
+
+
+/* google_protobuf_OneofDescriptorProto */
+extern const upb_msglayout_msginit_v1 google_protobuf_OneofDescriptorProto_msginit;
+google_protobuf_OneofDescriptorProto *google_protobuf_OneofDescriptorProto_new(upb_env *env);
+google_protobuf_OneofDescriptorProto *google_protobuf_OneofDescriptorProto_parsenew(upb_stringview buf, upb_env *env);
+char *google_protobuf_OneofDescriptorProto_serialize(google_protobuf_OneofDescriptorProto *msg, upb_env *env, size_t *len);
+void google_protobuf_OneofDescriptorProto_free(google_protobuf_OneofDescriptorProto *msg, upb_env *env);
+
+/* getters. */
+upb_stringview google_protobuf_OneofDescriptorProto_name(const google_protobuf_OneofDescriptorProto *msg);
+const google_protobuf_OneofOptions* google_protobuf_OneofDescriptorProto_options(const google_protobuf_OneofDescriptorProto *msg);
+
+/* setters. */
+void google_protobuf_OneofDescriptorProto_set_name(google_protobuf_OneofDescriptorProto *msg, upb_stringview value);
+void google_protobuf_OneofDescriptorProto_set_options(google_protobuf_OneofDescriptorProto *msg, google_protobuf_OneofOptions* value);
+
+
+/* google_protobuf_EnumDescriptorProto */
+extern const upb_msglayout_msginit_v1 google_protobuf_EnumDescriptorProto_msginit;
+google_protobuf_EnumDescriptorProto *google_protobuf_EnumDescriptorProto_new(upb_env *env);
+google_protobuf_EnumDescriptorProto *google_protobuf_EnumDescriptorProto_parsenew(upb_stringview buf, upb_env *env);
+char *google_protobuf_EnumDescriptorProto_serialize(google_protobuf_EnumDescriptorProto *msg, upb_env *env, size_t *len);
+void google_protobuf_EnumDescriptorProto_free(google_protobuf_EnumDescriptorProto *msg, upb_env *env);
+
+/* getters. */
+upb_stringview google_protobuf_EnumDescriptorProto_name(const google_protobuf_EnumDescriptorProto *msg);
+const upb_array* google_protobuf_EnumDescriptorProto_value(const google_protobuf_EnumDescriptorProto *msg);
+const google_protobuf_EnumOptions* google_protobuf_EnumDescriptorProto_options(const google_protobuf_EnumDescriptorProto *msg);
+const upb_array* google_protobuf_EnumDescriptorProto_reserved_range(const google_protobuf_EnumDescriptorProto *msg);
+const upb_array* google_protobuf_EnumDescriptorProto_reserved_name(const google_protobuf_EnumDescriptorProto *msg);
+
+/* setters. */
+void google_protobuf_EnumDescriptorProto_set_name(google_protobuf_EnumDescriptorProto *msg, upb_stringview value);
+void google_protobuf_EnumDescriptorProto_set_value(google_protobuf_EnumDescriptorProto *msg, upb_array* value);
+void google_protobuf_EnumDescriptorProto_set_options(google_protobuf_EnumDescriptorProto *msg, google_protobuf_EnumOptions* value);
+void google_protobuf_EnumDescriptorProto_set_reserved_range(google_protobuf_EnumDescriptorProto *msg, upb_array* value);
+void google_protobuf_EnumDescriptorProto_set_reserved_name(google_protobuf_EnumDescriptorProto *msg, upb_array* value);
+
+
+/* google_protobuf_EnumDescriptorProto_EnumReservedRange */
+extern const upb_msglayout_msginit_v1 google_protobuf_EnumDescriptorProto_EnumReservedRange_msginit;
+google_protobuf_EnumDescriptorProto_EnumReservedRange *google_protobuf_EnumDescriptorProto_EnumReservedRange_new(upb_env *env);
+google_protobuf_EnumDescriptorProto_EnumReservedRange *google_protobuf_EnumDescriptorProto_EnumReservedRange_parsenew(upb_stringview buf, upb_env *env);
+char *google_protobuf_EnumDescriptorProto_EnumReservedRange_serialize(google_protobuf_EnumDescriptorProto_EnumReservedRange *msg, upb_env *env, size_t *len);
+void google_protobuf_EnumDescriptorProto_EnumReservedRange_free(google_protobuf_EnumDescriptorProto_EnumReservedRange *msg, upb_env *env);
+
+/* getters. */
+int32_t google_protobuf_EnumDescriptorProto_EnumReservedRange_start(const google_protobuf_EnumDescriptorProto_EnumReservedRange *msg);
+int32_t google_protobuf_EnumDescriptorProto_EnumReservedRange_end(const google_protobuf_EnumDescriptorProto_EnumReservedRange *msg);
+
+/* setters. */
+void google_protobuf_EnumDescriptorProto_EnumReservedRange_set_start(google_protobuf_EnumDescriptorProto_EnumReservedRange *msg, int32_t value);
+void google_protobuf_EnumDescriptorProto_EnumReservedRange_set_end(google_protobuf_EnumDescriptorProto_EnumReservedRange *msg, int32_t value);
+
+
+/* google_protobuf_EnumValueDescriptorProto */
+extern const upb_msglayout_msginit_v1 google_protobuf_EnumValueDescriptorProto_msginit;
+google_protobuf_EnumValueDescriptorProto *google_protobuf_EnumValueDescriptorProto_new(upb_env *env);
+google_protobuf_EnumValueDescriptorProto *google_protobuf_EnumValueDescriptorProto_parsenew(upb_stringview buf, upb_env *env);
+char *google_protobuf_EnumValueDescriptorProto_serialize(google_protobuf_EnumValueDescriptorProto *msg, upb_env *env, size_t *len);
+void google_protobuf_EnumValueDescriptorProto_free(google_protobuf_EnumValueDescriptorProto *msg, upb_env *env);
+
+/* getters. */
+upb_stringview google_protobuf_EnumValueDescriptorProto_name(const google_protobuf_EnumValueDescriptorProto *msg);
+int32_t google_protobuf_EnumValueDescriptorProto_number(const google_protobuf_EnumValueDescriptorProto *msg);
+const google_protobuf_EnumValueOptions* google_protobuf_EnumValueDescriptorProto_options(const google_protobuf_EnumValueDescriptorProto *msg);
+
+/* setters. */
+void google_protobuf_EnumValueDescriptorProto_set_name(google_protobuf_EnumValueDescriptorProto *msg, upb_stringview value);
+void google_protobuf_EnumValueDescriptorProto_set_number(google_protobuf_EnumValueDescriptorProto *msg, int32_t value);
+void google_protobuf_EnumValueDescriptorProto_set_options(google_protobuf_EnumValueDescriptorProto *msg, google_protobuf_EnumValueOptions* value);
+
+
+/* google_protobuf_ServiceDescriptorProto */
+extern const upb_msglayout_msginit_v1 google_protobuf_ServiceDescriptorProto_msginit;
+google_protobuf_ServiceDescriptorProto *google_protobuf_ServiceDescriptorProto_new(upb_env *env);
+google_protobuf_ServiceDescriptorProto *google_protobuf_ServiceDescriptorProto_parsenew(upb_stringview buf, upb_env *env);
+char *google_protobuf_ServiceDescriptorProto_serialize(google_protobuf_ServiceDescriptorProto *msg, upb_env *env, size_t *len);
+void google_protobuf_ServiceDescriptorProto_free(google_protobuf_ServiceDescriptorProto *msg, upb_env *env);
+
+/* getters. */
+upb_stringview google_protobuf_ServiceDescriptorProto_name(const google_protobuf_ServiceDescriptorProto *msg);
+const upb_array* google_protobuf_ServiceDescriptorProto_method(const google_protobuf_ServiceDescriptorProto *msg);
+const google_protobuf_ServiceOptions* google_protobuf_ServiceDescriptorProto_options(const google_protobuf_ServiceDescriptorProto *msg);
+
+/* setters. */
+void google_protobuf_ServiceDescriptorProto_set_name(google_protobuf_ServiceDescriptorProto *msg, upb_stringview value);
+void google_protobuf_ServiceDescriptorProto_set_method(google_protobuf_ServiceDescriptorProto *msg, upb_array* value);
+void google_protobuf_ServiceDescriptorProto_set_options(google_protobuf_ServiceDescriptorProto *msg, google_protobuf_ServiceOptions* value);
+
+
+/* google_protobuf_MethodDescriptorProto */
+extern const upb_msglayout_msginit_v1 google_protobuf_MethodDescriptorProto_msginit;
+google_protobuf_MethodDescriptorProto *google_protobuf_MethodDescriptorProto_new(upb_env *env);
+google_protobuf_MethodDescriptorProto *google_protobuf_MethodDescriptorProto_parsenew(upb_stringview buf, upb_env *env);
+char *google_protobuf_MethodDescriptorProto_serialize(google_protobuf_MethodDescriptorProto *msg, upb_env *env, size_t *len);
+void google_protobuf_MethodDescriptorProto_free(google_protobuf_MethodDescriptorProto *msg, upb_env *env);
+
+/* getters. */
+upb_stringview google_protobuf_MethodDescriptorProto_name(const google_protobuf_MethodDescriptorProto *msg);
+upb_stringview google_protobuf_MethodDescriptorProto_input_type(const google_protobuf_MethodDescriptorProto *msg);
+upb_stringview google_protobuf_MethodDescriptorProto_output_type(const google_protobuf_MethodDescriptorProto *msg);
+const google_protobuf_MethodOptions* google_protobuf_MethodDescriptorProto_options(const google_protobuf_MethodDescriptorProto *msg);
+bool google_protobuf_MethodDescriptorProto_client_streaming(const google_protobuf_MethodDescriptorProto *msg);
+bool google_protobuf_MethodDescriptorProto_server_streaming(const google_protobuf_MethodDescriptorProto *msg);
+
+/* setters. */
+void google_protobuf_MethodDescriptorProto_set_name(google_protobuf_MethodDescriptorProto *msg, upb_stringview value);
+void google_protobuf_MethodDescriptorProto_set_input_type(google_protobuf_MethodDescriptorProto *msg, upb_stringview value);
+void google_protobuf_MethodDescriptorProto_set_output_type(google_protobuf_MethodDescriptorProto *msg, upb_stringview value);
+void google_protobuf_MethodDescriptorProto_set_options(google_protobuf_MethodDescriptorProto *msg, google_protobuf_MethodOptions* value);
+void google_protobuf_MethodDescriptorProto_set_client_streaming(google_protobuf_MethodDescriptorProto *msg, bool value);
+void google_protobuf_MethodDescriptorProto_set_server_streaming(google_protobuf_MethodDescriptorProto *msg, bool value);
+
+
+/* google_protobuf_FileOptions */
+extern const upb_msglayout_msginit_v1 google_protobuf_FileOptions_msginit;
+google_protobuf_FileOptions *google_protobuf_FileOptions_new(upb_env *env);
+google_protobuf_FileOptions *google_protobuf_FileOptions_parsenew(upb_stringview buf, upb_env *env);
+char *google_protobuf_FileOptions_serialize(google_protobuf_FileOptions *msg, upb_env *env, size_t *len);
+void google_protobuf_FileOptions_free(google_protobuf_FileOptions *msg, upb_env *env);
+
+/* getters. */
+upb_stringview google_protobuf_FileOptions_java_package(const google_protobuf_FileOptions *msg);
+upb_stringview google_protobuf_FileOptions_java_outer_classname(const google_protobuf_FileOptions *msg);
+google_protobuf_FileOptions_OptimizeMode google_protobuf_FileOptions_optimize_for(const google_protobuf_FileOptions *msg);
+bool google_protobuf_FileOptions_java_multiple_files(const google_protobuf_FileOptions *msg);
+upb_stringview google_protobuf_FileOptions_go_package(const google_protobuf_FileOptions *msg);
+bool google_protobuf_FileOptions_cc_generic_services(const google_protobuf_FileOptions *msg);
+bool google_protobuf_FileOptions_java_generic_services(const google_protobuf_FileOptions *msg);
+bool google_protobuf_FileOptions_py_generic_services(const google_protobuf_FileOptions *msg);
+bool google_protobuf_FileOptions_java_generate_equals_and_hash(const google_protobuf_FileOptions *msg);
+bool google_protobuf_FileOptions_deprecated(const google_protobuf_FileOptions *msg);
+bool google_protobuf_FileOptions_java_string_check_utf8(const google_protobuf_FileOptions *msg);
+bool google_protobuf_FileOptions_cc_enable_arenas(const google_protobuf_FileOptions *msg);
+upb_stringview google_protobuf_FileOptions_objc_class_prefix(const google_protobuf_FileOptions *msg);
+upb_stringview google_protobuf_FileOptions_csharp_namespace(const google_protobuf_FileOptions *msg);
+upb_stringview google_protobuf_FileOptions_swift_prefix(const google_protobuf_FileOptions *msg);
+upb_stringview google_protobuf_FileOptions_php_class_prefix(const google_protobuf_FileOptions *msg);
+upb_stringview google_protobuf_FileOptions_php_namespace(const google_protobuf_FileOptions *msg);
+bool google_protobuf_FileOptions_php_generic_services(const google_protobuf_FileOptions *msg);
+const upb_array* google_protobuf_FileOptions_uninterpreted_option(const google_protobuf_FileOptions *msg);
+
+/* setters. */
+void google_protobuf_FileOptions_set_java_package(google_protobuf_FileOptions *msg, upb_stringview value);
+void google_protobuf_FileOptions_set_java_outer_classname(google_protobuf_FileOptions *msg, upb_stringview value);
+void google_protobuf_FileOptions_set_optimize_for(google_protobuf_FileOptions *msg, google_protobuf_FileOptions_OptimizeMode value);
+void google_protobuf_FileOptions_set_java_multiple_files(google_protobuf_FileOptions *msg, bool value);
+void google_protobuf_FileOptions_set_go_package(google_protobuf_FileOptions *msg, upb_stringview value);
+void google_protobuf_FileOptions_set_cc_generic_services(google_protobuf_FileOptions *msg, bool value);
+void google_protobuf_FileOptions_set_java_generic_services(google_protobuf_FileOptions *msg, bool value);
+void google_protobuf_FileOptions_set_py_generic_services(google_protobuf_FileOptions *msg, bool value);
+void google_protobuf_FileOptions_set_java_generate_equals_and_hash(google_protobuf_FileOptions *msg, bool value);
+void google_protobuf_FileOptions_set_deprecated(google_protobuf_FileOptions *msg, bool value);
+void google_protobuf_FileOptions_set_java_string_check_utf8(google_protobuf_FileOptions *msg, bool value);
+void google_protobuf_FileOptions_set_cc_enable_arenas(google_protobuf_FileOptions *msg, bool value);
+void google_protobuf_FileOptions_set_objc_class_prefix(google_protobuf_FileOptions *msg, upb_stringview value);
+void google_protobuf_FileOptions_set_csharp_namespace(google_protobuf_FileOptions *msg, upb_stringview value);
+void google_protobuf_FileOptions_set_swift_prefix(google_protobuf_FileOptions *msg, upb_stringview value);
+void google_protobuf_FileOptions_set_php_class_prefix(google_protobuf_FileOptions *msg, upb_stringview value);
+void google_protobuf_FileOptions_set_php_namespace(google_protobuf_FileOptions *msg, upb_stringview value);
+void google_protobuf_FileOptions_set_php_generic_services(google_protobuf_FileOptions *msg, bool value);
+void google_protobuf_FileOptions_set_uninterpreted_option(google_protobuf_FileOptions *msg, upb_array* value);
+
+
+/* google_protobuf_MessageOptions */
+extern const upb_msglayout_msginit_v1 google_protobuf_MessageOptions_msginit;
+google_protobuf_MessageOptions *google_protobuf_MessageOptions_new(upb_env *env);
+google_protobuf_MessageOptions *google_protobuf_MessageOptions_parsenew(upb_stringview buf, upb_env *env);
+char *google_protobuf_MessageOptions_serialize(google_protobuf_MessageOptions *msg, upb_env *env, size_t *len);
+void google_protobuf_MessageOptions_free(google_protobuf_MessageOptions *msg, upb_env *env);
+
+/* getters. */
+bool google_protobuf_MessageOptions_message_set_wire_format(const google_protobuf_MessageOptions *msg);
+bool google_protobuf_MessageOptions_no_standard_descriptor_accessor(const google_protobuf_MessageOptions *msg);
+bool google_protobuf_MessageOptions_deprecated(const google_protobuf_MessageOptions *msg);
+bool google_protobuf_MessageOptions_map_entry(const google_protobuf_MessageOptions *msg);
+const upb_array* google_protobuf_MessageOptions_uninterpreted_option(const google_protobuf_MessageOptions *msg);
+
+/* setters. */
+void google_protobuf_MessageOptions_set_message_set_wire_format(google_protobuf_MessageOptions *msg, bool value);
+void google_protobuf_MessageOptions_set_no_standard_descriptor_accessor(google_protobuf_MessageOptions *msg, bool value);
+void google_protobuf_MessageOptions_set_deprecated(google_protobuf_MessageOptions *msg, bool value);
+void google_protobuf_MessageOptions_set_map_entry(google_protobuf_MessageOptions *msg, bool value);
+void google_protobuf_MessageOptions_set_uninterpreted_option(google_protobuf_MessageOptions *msg, upb_array* value);
+
+
+/* google_protobuf_FieldOptions */
+extern const upb_msglayout_msginit_v1 google_protobuf_FieldOptions_msginit;
+google_protobuf_FieldOptions *google_protobuf_FieldOptions_new(upb_env *env);
+google_protobuf_FieldOptions *google_protobuf_FieldOptions_parsenew(upb_stringview buf, upb_env *env);
+char *google_protobuf_FieldOptions_serialize(google_protobuf_FieldOptions *msg, upb_env *env, size_t *len);
+void google_protobuf_FieldOptions_free(google_protobuf_FieldOptions *msg, upb_env *env);
+
+/* getters. */
+google_protobuf_FieldOptions_CType google_protobuf_FieldOptions_ctype(const google_protobuf_FieldOptions *msg);
+bool google_protobuf_FieldOptions_packed(const google_protobuf_FieldOptions *msg);
+bool google_protobuf_FieldOptions_deprecated(const google_protobuf_FieldOptions *msg);
+bool google_protobuf_FieldOptions_lazy(const google_protobuf_FieldOptions *msg);
+google_protobuf_FieldOptions_JSType google_protobuf_FieldOptions_jstype(const google_protobuf_FieldOptions *msg);
+bool google_protobuf_FieldOptions_weak(const google_protobuf_FieldOptions *msg);
+const upb_array* google_protobuf_FieldOptions_uninterpreted_option(const google_protobuf_FieldOptions *msg);
+
+/* setters. */
+void google_protobuf_FieldOptions_set_ctype(google_protobuf_FieldOptions *msg, google_protobuf_FieldOptions_CType value);
+void google_protobuf_FieldOptions_set_packed(google_protobuf_FieldOptions *msg, bool value);
+void google_protobuf_FieldOptions_set_deprecated(google_protobuf_FieldOptions *msg, bool value);
+void google_protobuf_FieldOptions_set_lazy(google_protobuf_FieldOptions *msg, bool value);
+void google_protobuf_FieldOptions_set_jstype(google_protobuf_FieldOptions *msg, google_protobuf_FieldOptions_JSType value);
+void google_protobuf_FieldOptions_set_weak(google_protobuf_FieldOptions *msg, bool value);
+void google_protobuf_FieldOptions_set_uninterpreted_option(google_protobuf_FieldOptions *msg, upb_array* value);
+
+
+/* google_protobuf_OneofOptions */
+extern const upb_msglayout_msginit_v1 google_protobuf_OneofOptions_msginit;
+google_protobuf_OneofOptions *google_protobuf_OneofOptions_new(upb_env *env);
+google_protobuf_OneofOptions *google_protobuf_OneofOptions_parsenew(upb_stringview buf, upb_env *env);
+char *google_protobuf_OneofOptions_serialize(google_protobuf_OneofOptions *msg, upb_env *env, size_t *len);
+void google_protobuf_OneofOptions_free(google_protobuf_OneofOptions *msg, upb_env *env);
+
+/* getters. */
+const upb_array* google_protobuf_OneofOptions_uninterpreted_option(const google_protobuf_OneofOptions *msg);
+
+/* setters. */
+void google_protobuf_OneofOptions_set_uninterpreted_option(google_protobuf_OneofOptions *msg, upb_array* value);
+
+
+/* google_protobuf_EnumOptions */
+extern const upb_msglayout_msginit_v1 google_protobuf_EnumOptions_msginit;
+google_protobuf_EnumOptions *google_protobuf_EnumOptions_new(upb_env *env);
+google_protobuf_EnumOptions *google_protobuf_EnumOptions_parsenew(upb_stringview buf, upb_env *env);
+char *google_protobuf_EnumOptions_serialize(google_protobuf_EnumOptions *msg, upb_env *env, size_t *len);
+void google_protobuf_EnumOptions_free(google_protobuf_EnumOptions *msg, upb_env *env);
+
+/* getters. */
+bool google_protobuf_EnumOptions_allow_alias(const google_protobuf_EnumOptions *msg);
+bool google_protobuf_EnumOptions_deprecated(const google_protobuf_EnumOptions *msg);
+const upb_array* google_protobuf_EnumOptions_uninterpreted_option(const google_protobuf_EnumOptions *msg);
+
+/* setters. */
+void google_protobuf_EnumOptions_set_allow_alias(google_protobuf_EnumOptions *msg, bool value);
+void google_protobuf_EnumOptions_set_deprecated(google_protobuf_EnumOptions *msg, bool value);
+void google_protobuf_EnumOptions_set_uninterpreted_option(google_protobuf_EnumOptions *msg, upb_array* value);
+
+
+/* google_protobuf_EnumValueOptions */
+extern const upb_msglayout_msginit_v1 google_protobuf_EnumValueOptions_msginit;
+google_protobuf_EnumValueOptions *google_protobuf_EnumValueOptions_new(upb_env *env);
+google_protobuf_EnumValueOptions *google_protobuf_EnumValueOptions_parsenew(upb_stringview buf, upb_env *env);
+char *google_protobuf_EnumValueOptions_serialize(google_protobuf_EnumValueOptions *msg, upb_env *env, size_t *len);
+void google_protobuf_EnumValueOptions_free(google_protobuf_EnumValueOptions *msg, upb_env *env);
+
+/* getters. */
+bool google_protobuf_EnumValueOptions_deprecated(const google_protobuf_EnumValueOptions *msg);
+const upb_array* google_protobuf_EnumValueOptions_uninterpreted_option(const google_protobuf_EnumValueOptions *msg);
+
+/* setters. */
+void google_protobuf_EnumValueOptions_set_deprecated(google_protobuf_EnumValueOptions *msg, bool value);
+void google_protobuf_EnumValueOptions_set_uninterpreted_option(google_protobuf_EnumValueOptions *msg, upb_array* value);
+
+
+/* google_protobuf_ServiceOptions */
+extern const upb_msglayout_msginit_v1 google_protobuf_ServiceOptions_msginit;
+google_protobuf_ServiceOptions *google_protobuf_ServiceOptions_new(upb_env *env);
+google_protobuf_ServiceOptions *google_protobuf_ServiceOptions_parsenew(upb_stringview buf, upb_env *env);
+char *google_protobuf_ServiceOptions_serialize(google_protobuf_ServiceOptions *msg, upb_env *env, size_t *len);
+void google_protobuf_ServiceOptions_free(google_protobuf_ServiceOptions *msg, upb_env *env);
+
+/* getters. */
+bool google_protobuf_ServiceOptions_deprecated(const google_protobuf_ServiceOptions *msg);
+const upb_array* google_protobuf_ServiceOptions_uninterpreted_option(const google_protobuf_ServiceOptions *msg);
+
+/* setters. */
+void google_protobuf_ServiceOptions_set_deprecated(google_protobuf_ServiceOptions *msg, bool value);
+void google_protobuf_ServiceOptions_set_uninterpreted_option(google_protobuf_ServiceOptions *msg, upb_array* value);
+
+
+/* google_protobuf_MethodOptions */
+extern const upb_msglayout_msginit_v1 google_protobuf_MethodOptions_msginit;
+google_protobuf_MethodOptions *google_protobuf_MethodOptions_new(upb_env *env);
+google_protobuf_MethodOptions *google_protobuf_MethodOptions_parsenew(upb_stringview buf, upb_env *env);
+char *google_protobuf_MethodOptions_serialize(google_protobuf_MethodOptions *msg, upb_env *env, size_t *len);
+void google_protobuf_MethodOptions_free(google_protobuf_MethodOptions *msg, upb_env *env);
+
+/* getters. */
+bool google_protobuf_MethodOptions_deprecated(const google_protobuf_MethodOptions *msg);
+google_protobuf_MethodOptions_IdempotencyLevel google_protobuf_MethodOptions_idempotency_level(const google_protobuf_MethodOptions *msg);
+const upb_array* google_protobuf_MethodOptions_uninterpreted_option(const google_protobuf_MethodOptions *msg);
+
+/* setters. */
+void google_protobuf_MethodOptions_set_deprecated(google_protobuf_MethodOptions *msg, bool value);
+void google_protobuf_MethodOptions_set_idempotency_level(google_protobuf_MethodOptions *msg, google_protobuf_MethodOptions_IdempotencyLevel value);
+void google_protobuf_MethodOptions_set_uninterpreted_option(google_protobuf_MethodOptions *msg, upb_array* value);
+
+
+/* google_protobuf_UninterpretedOption */
+extern const upb_msglayout_msginit_v1 google_protobuf_UninterpretedOption_msginit;
+google_protobuf_UninterpretedOption *google_protobuf_UninterpretedOption_new(upb_env *env);
+google_protobuf_UninterpretedOption *google_protobuf_UninterpretedOption_parsenew(upb_stringview buf, upb_env *env);
+char *google_protobuf_UninterpretedOption_serialize(google_protobuf_UninterpretedOption *msg, upb_env *env, size_t *len);
+void google_protobuf_UninterpretedOption_free(google_protobuf_UninterpretedOption *msg, upb_env *env);
+
+/* getters. */
+const upb_array* google_protobuf_UninterpretedOption_name(const google_protobuf_UninterpretedOption *msg);
+upb_stringview google_protobuf_UninterpretedOption_identifier_value(const google_protobuf_UninterpretedOption *msg);
+uint64_t google_protobuf_UninterpretedOption_positive_int_value(const google_protobuf_UninterpretedOption *msg);
+int64_t google_protobuf_UninterpretedOption_negative_int_value(const google_protobuf_UninterpretedOption *msg);
+double google_protobuf_UninterpretedOption_double_value(const google_protobuf_UninterpretedOption *msg);
+upb_stringview google_protobuf_UninterpretedOption_string_value(const google_protobuf_UninterpretedOption *msg);
+upb_stringview google_protobuf_UninterpretedOption_aggregate_value(const google_protobuf_UninterpretedOption *msg);
+
+/* setters. */
+void google_protobuf_UninterpretedOption_set_name(google_protobuf_UninterpretedOption *msg, upb_array* value);
+void google_protobuf_UninterpretedOption_set_identifier_value(google_protobuf_UninterpretedOption *msg, upb_stringview value);
+void google_protobuf_UninterpretedOption_set_positive_int_value(google_protobuf_UninterpretedOption *msg, uint64_t value);
+void google_protobuf_UninterpretedOption_set_negative_int_value(google_protobuf_UninterpretedOption *msg, int64_t value);
+void google_protobuf_UninterpretedOption_set_double_value(google_protobuf_UninterpretedOption *msg, double value);
+void google_protobuf_UninterpretedOption_set_string_value(google_protobuf_UninterpretedOption *msg, upb_stringview value);
+void google_protobuf_UninterpretedOption_set_aggregate_value(google_protobuf_UninterpretedOption *msg, upb_stringview value);
+
+
+/* google_protobuf_UninterpretedOption_NamePart */
+extern const upb_msglayout_msginit_v1 google_protobuf_UninterpretedOption_NamePart_msginit;
+google_protobuf_UninterpretedOption_NamePart *google_protobuf_UninterpretedOption_NamePart_new(upb_env *env);
+google_protobuf_UninterpretedOption_NamePart *google_protobuf_UninterpretedOption_NamePart_parsenew(upb_stringview buf, upb_env *env);
+char *google_protobuf_UninterpretedOption_NamePart_serialize(google_protobuf_UninterpretedOption_NamePart *msg, upb_env *env, size_t *len);
+void google_protobuf_UninterpretedOption_NamePart_free(google_protobuf_UninterpretedOption_NamePart *msg, upb_env *env);
+
+/* getters. */
+upb_stringview google_protobuf_UninterpretedOption_NamePart_name_part(const google_protobuf_UninterpretedOption_NamePart *msg);
+bool google_protobuf_UninterpretedOption_NamePart_is_extension(const google_protobuf_UninterpretedOption_NamePart *msg);
+
+/* setters. */
+void google_protobuf_UninterpretedOption_NamePart_set_name_part(google_protobuf_UninterpretedOption_NamePart *msg, upb_stringview value);
+void google_protobuf_UninterpretedOption_NamePart_set_is_extension(google_protobuf_UninterpretedOption_NamePart *msg, bool value);
+
+
+/* google_protobuf_SourceCodeInfo */
+extern const upb_msglayout_msginit_v1 google_protobuf_SourceCodeInfo_msginit;
+google_protobuf_SourceCodeInfo *google_protobuf_SourceCodeInfo_new(upb_env *env);
+google_protobuf_SourceCodeInfo *google_protobuf_SourceCodeInfo_parsenew(upb_stringview buf, upb_env *env);
+char *google_protobuf_SourceCodeInfo_serialize(google_protobuf_SourceCodeInfo *msg, upb_env *env, size_t *len);
+void google_protobuf_SourceCodeInfo_free(google_protobuf_SourceCodeInfo *msg, upb_env *env);
+
+/* getters. */
+const upb_array* google_protobuf_SourceCodeInfo_location(const google_protobuf_SourceCodeInfo *msg);
+
+/* setters. */
+void google_protobuf_SourceCodeInfo_set_location(google_protobuf_SourceCodeInfo *msg, upb_array* value);
+
+
+/* google_protobuf_SourceCodeInfo_Location */
+extern const upb_msglayout_msginit_v1 google_protobuf_SourceCodeInfo_Location_msginit;
+google_protobuf_SourceCodeInfo_Location *google_protobuf_SourceCodeInfo_Location_new(upb_env *env);
+google_protobuf_SourceCodeInfo_Location *google_protobuf_SourceCodeInfo_Location_parsenew(upb_stringview buf, upb_env *env);
+char *google_protobuf_SourceCodeInfo_Location_serialize(google_protobuf_SourceCodeInfo_Location *msg, upb_env *env, size_t *len);
+void google_protobuf_SourceCodeInfo_Location_free(google_protobuf_SourceCodeInfo_Location *msg, upb_env *env);
+
+/* getters. */
+const upb_array* google_protobuf_SourceCodeInfo_Location_path(const google_protobuf_SourceCodeInfo_Location *msg);
+const upb_array* google_protobuf_SourceCodeInfo_Location_span(const google_protobuf_SourceCodeInfo_Location *msg);
+upb_stringview google_protobuf_SourceCodeInfo_Location_leading_comments(const google_protobuf_SourceCodeInfo_Location *msg);
+upb_stringview google_protobuf_SourceCodeInfo_Location_trailing_comments(const google_protobuf_SourceCodeInfo_Location *msg);
+const upb_array* google_protobuf_SourceCodeInfo_Location_leading_detached_comments(const google_protobuf_SourceCodeInfo_Location *msg);
+
+/* setters. */
+void google_protobuf_SourceCodeInfo_Location_set_path(google_protobuf_SourceCodeInfo_Location *msg, upb_array* value);
+void google_protobuf_SourceCodeInfo_Location_set_span(google_protobuf_SourceCodeInfo_Location *msg, upb_array* value);
+void google_protobuf_SourceCodeInfo_Location_set_leading_comments(google_protobuf_SourceCodeInfo_Location *msg, upb_stringview value);
+void google_protobuf_SourceCodeInfo_Location_set_trailing_comments(google_protobuf_SourceCodeInfo_Location *msg, upb_stringview value);
+void google_protobuf_SourceCodeInfo_Location_set_leading_detached_comments(google_protobuf_SourceCodeInfo_Location *msg, upb_array* value);
+
+
+/* google_protobuf_GeneratedCodeInfo */
+extern const upb_msglayout_msginit_v1 google_protobuf_GeneratedCodeInfo_msginit;
+google_protobuf_GeneratedCodeInfo *google_protobuf_GeneratedCodeInfo_new(upb_env *env);
+google_protobuf_GeneratedCodeInfo *google_protobuf_GeneratedCodeInfo_parsenew(upb_stringview buf, upb_env *env);
+char *google_protobuf_GeneratedCodeInfo_serialize(google_protobuf_GeneratedCodeInfo *msg, upb_env *env, size_t *len);
+void google_protobuf_GeneratedCodeInfo_free(google_protobuf_GeneratedCodeInfo *msg, upb_env *env);
+
+/* getters. */
+const upb_array* google_protobuf_GeneratedCodeInfo_annotation(const google_protobuf_GeneratedCodeInfo *msg);
+
+/* setters. */
+void google_protobuf_GeneratedCodeInfo_set_annotation(google_protobuf_GeneratedCodeInfo *msg, upb_array* value);
+
+
+/* google_protobuf_GeneratedCodeInfo_Annotation */
+extern const upb_msglayout_msginit_v1 google_protobuf_GeneratedCodeInfo_Annotation_msginit;
+google_protobuf_GeneratedCodeInfo_Annotation *google_protobuf_GeneratedCodeInfo_Annotation_new(upb_env *env);
+google_protobuf_GeneratedCodeInfo_Annotation *google_protobuf_GeneratedCodeInfo_Annotation_parsenew(upb_stringview buf, upb_env *env);
+char *google_protobuf_GeneratedCodeInfo_Annotation_serialize(google_protobuf_GeneratedCodeInfo_Annotation *msg, upb_env *env, size_t *len);
+void google_protobuf_GeneratedCodeInfo_Annotation_free(google_protobuf_GeneratedCodeInfo_Annotation *msg, upb_env *env);
+
+/* getters. */
+const upb_array* google_protobuf_GeneratedCodeInfo_Annotation_path(const google_protobuf_GeneratedCodeInfo_Annotation *msg);
+upb_stringview google_protobuf_GeneratedCodeInfo_Annotation_source_file(const google_protobuf_GeneratedCodeInfo_Annotation *msg);
+int32_t google_protobuf_GeneratedCodeInfo_Annotation_begin(const google_protobuf_GeneratedCodeInfo_Annotation *msg);
+int32_t google_protobuf_GeneratedCodeInfo_Annotation_end(const google_protobuf_GeneratedCodeInfo_Annotation *msg);
+
+/* setters. */
+void google_protobuf_GeneratedCodeInfo_Annotation_set_path(google_protobuf_GeneratedCodeInfo_Annotation *msg, upb_array* value);
+void google_protobuf_GeneratedCodeInfo_Annotation_set_source_file(google_protobuf_GeneratedCodeInfo_Annotation *msg, upb_stringview value);
+void google_protobuf_GeneratedCodeInfo_Annotation_set_begin(google_protobuf_GeneratedCodeInfo_Annotation *msg, int32_t value);
+void google_protobuf_GeneratedCodeInfo_Annotation_set_end(google_protobuf_GeneratedCodeInfo_Annotation *msg, int32_t value);
+
+
+UPB_END_EXTERN_C
+
+#endif  /* GOOGLE_PROTOBUF_DESCRIPTOR_PROTO_UPB_H_ */
+/*
 ** structs.int.h: structures definitions that are internal to upb.
 */
 
@@ -7001,22 +7713,6 @@ extern const struct upb_refcounted_vtbl upb_filedef_vtbl;
 
 #endif  /* UPB_STATICINIT_H_ */
 /*
-** upb_encode: parsing into a upb_msg using a upb_msglayout.
-*/
-
-#ifndef UPB_ENCODE_H_
-#define UPB_ENCODE_H_
-
-
-UPB_BEGIN_EXTERN_C
-
-char *upb_encode(const void *msg, const upb_msglayout_msginit_v1 *l,
-                 upb_env *env, size_t *size);
-
-UPB_END_EXTERN_C
-
-#endif  /* UPB_ENCODE_H_ */
-/*
 ** upb::descriptor::Reader (upb_descreader)
 **
 ** Provides a way of building upb::Defs from data in descriptor.proto format.
@@ -7118,53 +7814,6 @@ inline FileDef* Reader::file(size_t i) const {
 
 UPB_BEGIN_EXTERN_C
 
-/* Enums */
-
-typedef enum {
-  google_protobuf_FieldDescriptorProto_LABEL_OPTIONAL = 1,
-  google_protobuf_FieldDescriptorProto_LABEL_REQUIRED = 2,
-  google_protobuf_FieldDescriptorProto_LABEL_REPEATED = 3
-} google_protobuf_FieldDescriptorProto_Label;
-
-typedef enum {
-  google_protobuf_FieldDescriptorProto_TYPE_DOUBLE = 1,
-  google_protobuf_FieldDescriptorProto_TYPE_FLOAT = 2,
-  google_protobuf_FieldDescriptorProto_TYPE_INT64 = 3,
-  google_protobuf_FieldDescriptorProto_TYPE_UINT64 = 4,
-  google_protobuf_FieldDescriptorProto_TYPE_INT32 = 5,
-  google_protobuf_FieldDescriptorProto_TYPE_FIXED64 = 6,
-  google_protobuf_FieldDescriptorProto_TYPE_FIXED32 = 7,
-  google_protobuf_FieldDescriptorProto_TYPE_BOOL = 8,
-  google_protobuf_FieldDescriptorProto_TYPE_STRING = 9,
-  google_protobuf_FieldDescriptorProto_TYPE_GROUP = 10,
-  google_protobuf_FieldDescriptorProto_TYPE_MESSAGE = 11,
-  google_protobuf_FieldDescriptorProto_TYPE_BYTES = 12,
-  google_protobuf_FieldDescriptorProto_TYPE_UINT32 = 13,
-  google_protobuf_FieldDescriptorProto_TYPE_ENUM = 14,
-  google_protobuf_FieldDescriptorProto_TYPE_SFIXED32 = 15,
-  google_protobuf_FieldDescriptorProto_TYPE_SFIXED64 = 16,
-  google_protobuf_FieldDescriptorProto_TYPE_SINT32 = 17,
-  google_protobuf_FieldDescriptorProto_TYPE_SINT64 = 18
-} google_protobuf_FieldDescriptorProto_Type;
-
-typedef enum {
-  google_protobuf_FieldOptions_STRING = 0,
-  google_protobuf_FieldOptions_CORD = 1,
-  google_protobuf_FieldOptions_STRING_PIECE = 2
-} google_protobuf_FieldOptions_CType;
-
-typedef enum {
-  google_protobuf_FieldOptions_JS_NORMAL = 0,
-  google_protobuf_FieldOptions_JS_STRING = 1,
-  google_protobuf_FieldOptions_JS_NUMBER = 2
-} google_protobuf_FieldOptions_JSType;
-
-typedef enum {
-  google_protobuf_FileOptions_SPEED = 1,
-  google_protobuf_FileOptions_CODE_SIZE = 2,
-  google_protobuf_FileOptions_LITE_RUNTIME = 3
-} google_protobuf_FileOptions_OptimizeMode;
-
 /* MessageDefs: call these functions to get a ref to a msgdef. */
 const upb_msgdef *upbdefs_google_protobuf_DescriptorProto_get(const void *owner);
 const upb_msgdef *upbdefs_google_protobuf_DescriptorProto_ExtensionRange_get(const void *owner);

protobuf.bzl

@@ -45,8 +45,11 @@ def _CcSrcs(srcs, use_grpc_plugin=False):
 def _CcOuts(srcs, use_grpc_plugin=False):
   return _CcHdrs(srcs, use_grpc_plugin) + _CcSrcs(srcs, use_grpc_plugin)
 
-def _PyOuts(srcs):
-  return [s[:-len(".proto")] + "_pb2.py" for s in srcs]
+def _PyOuts(srcs, use_grpc_plugin=False):
+  ret = [s[:-len(".proto")] + "_pb2.py" for s in srcs]
+  if use_grpc_plugin:
+    ret += [s[:-len(".proto")] + "_pb2_grpc.py" for s in srcs]
+  return ret
 
 def _RelativeOutputPath(path, include, dest=""):
   if include == None:
@@ -171,10 +174,10 @@ def cc_proto_library(
         deps=[],
         cc_libs=[],
         include=None,
-        protoc="//:protoc",
+        protoc="@com_google_protobuf//:protoc",
         internal_bootstrap_hack=False,
         use_grpc_plugin=False,
-        default_runtime="//:protobuf",
+        default_runtime="@com_google_protobuf//:protobuf",
         **kargs):
   """Bazel rule to create a C++ protobuf library from proto source files
 
@@ -317,8 +320,8 @@ def py_proto_library(
         py_libs=[],
         py_extra_srcs=[],
         include=None,
-        default_runtime="//:protobuf_python",
-        protoc="//:protoc",
+        default_runtime="@com_google_protobuf//:protobuf_python",
+        protoc="@com_google_protobuf//:protoc",
         use_grpc_plugin=False,
         **kargs):
   """Bazel rule to create a Python protobuf library from proto source files
@@ -344,7 +347,7 @@ def py_proto_library(
     **kargs: other keyword arguments that are passed to cc_library.
 
   """
-  outs = _PyOuts(srcs)
+  outs = _PyOuts(srcs, use_grpc_plugin)
 
   includes = []
   if include != None:

python/google/protobuf/descriptor.py

@@ -34,6 +34,7 @@ file, in types that make this information accessible in Python.
 
 __author__ = 'robinson@google.com (Will Robinson)'
 
+import threading
 import six
 
 from google.protobuf.internal import api_implementation
@@ -72,6 +73,24 @@ else:
   DescriptorMetaclass = type
 
 
+class _Lock(object):
+  """Wrapper class of threading.Lock(), which is allowed by 'with'."""
+
+  def __new__(cls):
+    self = object.__new__(cls)
+    self._lock = threading.Lock()  # pylint: disable=protected-access
+    return self
+
+  def __enter__(self):
+    self._lock.acquire()
+
+  def __exit__(self, exc_type, exc_value, exc_tb):
+    self._lock.release()
+
+
+_lock = threading.Lock()
+
+
 class DescriptorBase(six.with_metaclass(DescriptorMetaclass)):
 
   """Descriptors base class.
@@ -92,16 +111,17 @@ class DescriptorBase(six.with_metaclass(DescriptorMetaclass)):
     # subclasses" of this descriptor class.
     _C_DESCRIPTOR_CLASS = ()
 
-  def __init__(self, options, options_class_name):
+  def __init__(self, options, serialized_options, options_class_name):
     """Initialize the descriptor given its options message and the name of the
     class of the options message. The name of the class is required in case
     the options message is None and has to be created.
     """
     self._options = options
     self._options_class_name = options_class_name
+    self._serialized_options = serialized_options
 
     # Does this descriptor have non-default options?
-    self.has_options = options is not None
+    self.has_options = (options is not None) or (serialized_options is not None)
 
   def _SetOptions(self, options, options_class_name):
     """Sets the descriptor's options
@@ -123,14 +143,23 @@ class DescriptorBase(six.with_metaclass(DescriptorMetaclass)):
     """
     if self._options:
       return self._options
+
     from google.protobuf import descriptor_pb2
     try:
-      options_class = getattr(descriptor_pb2, self._options_class_name)
+      options_class = getattr(descriptor_pb2,
+                              self._options_class_name)
     except AttributeError:
       raise RuntimeError('Unknown options class name %s!' %
                          (self._options_class_name))
-    self._options = options_class()
-    return self._options
+
+    with _lock:
+      if self._serialized_options is None:
+        self._options = options_class()
+      else:
+        self._options = _ParseOptions(options_class(),
+                                      self._serialized_options)
+
+      return self._options
 
 
 class _NestedDescriptorBase(DescriptorBase):
@@ -138,7 +167,7 @@ class _NestedDescriptorBase(DescriptorBase):
 
   def __init__(self, options, options_class_name, name, full_name,
                file, containing_type, serialized_start=None,
-               serialized_end=None):
+               serialized_end=None, serialized_options=None):
     """Constructor.
 
     Args:
@@ -157,9 +186,10 @@ class _NestedDescriptorBase(DescriptorBase):
         file.serialized_pb that describes this descriptor.
       serialized_end: The end index (exclusive) in block in the
         file.serialized_pb that describes this descriptor.
+      serialized_options: Protocol message serilized options or None.
     """
     super(_NestedDescriptorBase, self).__init__(
-        options, options_class_name)
+        options, serialized_options, options_class_name)
 
     self.name = name
     # TODO(falk): Add function to calculate full_name instead of having it in
@@ -250,6 +280,7 @@ class Descriptor(_NestedDescriptorBase):
 
     def __new__(cls, name, full_name, filename, containing_type, fields,
                 nested_types, enum_types, extensions, options=None,
+                serialized_options=None,
                 is_extendable=True, extension_ranges=None, oneofs=None,
                 file=None, serialized_start=None, serialized_end=None,  # pylint: disable=redefined-builtin
                 syntax=None):
@@ -261,6 +292,7 @@ class Descriptor(_NestedDescriptorBase):
   # name of the argument.
   def __init__(self, name, full_name, filename, containing_type, fields,
                nested_types, enum_types, extensions, options=None,
+               serialized_options=None,
                is_extendable=True, extension_ranges=None, oneofs=None,
                file=None, serialized_start=None, serialized_end=None,  # pylint: disable=redefined-builtin
                syntax=None):
@@ -273,7 +305,7 @@ class Descriptor(_NestedDescriptorBase):
     super(Descriptor, self).__init__(
         options, 'MessageOptions', name, full_name, file,
         containing_type, serialized_start=serialized_start,
-        serialized_end=serialized_end)
+        serialized_end=serialized_end, serialized_options=serialized_options)
 
     # We have fields in addition to fields_by_name and fields_by_number,
     # so that:
@@ -492,8 +524,9 @@ class FieldDescriptor(DescriptorBase):
     def __new__(cls, name, full_name, index, number, type, cpp_type, label,
                 default_value, message_type, enum_type, containing_type,
                 is_extension, extension_scope, options=None,
+                serialized_options=None,
                 has_default_value=True, containing_oneof=None, json_name=None,
-                file=None):
+                file=None):  # pylint: disable=redefined-builtin
       _message.Message._CheckCalledFromGeneratedFile()
       if is_extension:
         return _message.default_pool.FindExtensionByName(full_name)
@@ -503,8 +536,9 @@ class FieldDescriptor(DescriptorBase):
   def __init__(self, name, full_name, index, number, type, cpp_type, label,
                default_value, message_type, enum_type, containing_type,
                is_extension, extension_scope, options=None,
+               serialized_options=None,
                has_default_value=True, containing_oneof=None, json_name=None,
-               file=None):
+               file=None):  # pylint: disable=redefined-builtin
     """The arguments are as described in the description of FieldDescriptor
     attributes above.
 
@@ -512,7 +546,8 @@ class FieldDescriptor(DescriptorBase):
     (to deal with circular references between message types, for example).
     Likewise for extension_scope.
     """
-    super(FieldDescriptor, self).__init__(options, 'FieldOptions')
+    super(FieldDescriptor, self).__init__(
+        options, serialized_options, 'FieldOptions')
     self.name = name
     self.full_name = full_name
     self.file = file
@@ -598,13 +633,15 @@ class EnumDescriptor(_NestedDescriptorBase):
     _C_DESCRIPTOR_CLASS = _message.EnumDescriptor
 
     def __new__(cls, name, full_name, filename, values,
-                containing_type=None, options=None, file=None,
+                containing_type=None, options=None,
+                serialized_options=None, file=None,  # pylint: disable=redefined-builtin
                 serialized_start=None, serialized_end=None):
       _message.Message._CheckCalledFromGeneratedFile()
       return _message.default_pool.FindEnumTypeByName(full_name)
 
   def __init__(self, name, full_name, filename, values,
-               containing_type=None, options=None, file=None,
+               containing_type=None, options=None,
+               serialized_options=None, file=None,  # pylint: disable=redefined-builtin
                serialized_start=None, serialized_end=None):
     """Arguments are as described in the attribute description above.
 
@@ -614,7 +651,7 @@ class EnumDescriptor(_NestedDescriptorBase):
     super(EnumDescriptor, self).__init__(
         options, 'EnumOptions', name, full_name, file,
         containing_type, serialized_start=serialized_start,
-        serialized_end=serialized_end)
+        serialized_end=serialized_end, serialized_options=serialized_options)
 
     self.values = values
     for value in self.values:
@@ -650,7 +687,9 @@ class EnumValueDescriptor(DescriptorBase):
   if _USE_C_DESCRIPTORS:
     _C_DESCRIPTOR_CLASS = _message.EnumValueDescriptor
 
-    def __new__(cls, name, index, number, type=None, options=None):
+    def __new__(cls, name, index, number,
+                type=None,  # pylint: disable=redefined-builtin
+                options=None, serialized_options=None):
       _message.Message._CheckCalledFromGeneratedFile()
       # There is no way we can build a complete EnumValueDescriptor with the
       # given parameters (the name of the Enum is not known, for example).
@@ -658,9 +697,12 @@ class EnumValueDescriptor(DescriptorBase):
       # constructor, which will ignore it, so returning None is good enough.
       return None
 
-  def __init__(self, name, index, number, type=None, options=None):
+  def __init__(self, name, index, number,
+               type=None,  # pylint: disable=redefined-builtin
+               options=None, serialized_options=None):
     """Arguments are as described in the attribute description above."""
-    super(EnumValueDescriptor, self).__init__(options, 'EnumValueOptions')
+    super(EnumValueDescriptor, self).__init__(
+        options, serialized_options, 'EnumValueOptions')
     self.name = name
     self.index = index
     self.number = number
@@ -685,14 +727,17 @@ class OneofDescriptor(DescriptorBase):
     _C_DESCRIPTOR_CLASS = _message.OneofDescriptor
 
     def __new__(
-        cls, name, full_name, index, containing_type, fields, options=None):
+        cls, name, full_name, index, containing_type, fields, options=None,
+        serialized_options=None):
       _message.Message._CheckCalledFromGeneratedFile()
       return _message.default_pool.FindOneofByName(full_name)
 
   def __init__(
-      self, name, full_name, index, containing_type, fields, options=None):
+      self, name, full_name, index, containing_type, fields, options=None,
+      serialized_options=None):
     """Arguments are as described in the attribute description above."""
-    super(OneofDescriptor, self).__init__(options, 'OneofOptions')
+    super(OneofDescriptor, self).__init__(
+        options, serialized_options, 'OneofOptions')
     self.name = name
     self.full_name = full_name
     self.index = index
@@ -721,17 +766,19 @@ class ServiceDescriptor(_NestedDescriptorBase):
   if _USE_C_DESCRIPTORS:
     _C_DESCRIPTOR_CLASS = _message.ServiceDescriptor
 
-    def __new__(cls, name, full_name, index, methods, options=None, file=None,  # pylint: disable=redefined-builtin
+    def __new__(cls, name, full_name, index, methods, options=None,
+                serialized_options=None, file=None,  # pylint: disable=redefined-builtin
                 serialized_start=None, serialized_end=None):
       _message.Message._CheckCalledFromGeneratedFile()  # pylint: disable=protected-access
       return _message.default_pool.FindServiceByName(full_name)
 
-  def __init__(self, name, full_name, index, methods, options=None, file=None,
+  def __init__(self, name, full_name, index, methods, options=None,
+               serialized_options=None, file=None,  # pylint: disable=redefined-builtin
                serialized_start=None, serialized_end=None):
     super(ServiceDescriptor, self).__init__(
         options, 'ServiceOptions', name, full_name, file,
         None, serialized_start=serialized_start,
-        serialized_end=serialized_end)
+        serialized_end=serialized_end, serialized_options=serialized_options)
     self.index = index
     self.methods = methods
     self.methods_by_name = dict((m.name, m) for m in methods)
@@ -772,18 +819,19 @@ class MethodDescriptor(DescriptorBase):
     _C_DESCRIPTOR_CLASS = _message.MethodDescriptor
 
     def __new__(cls, name, full_name, index, containing_service,
-                input_type, output_type, options=None):
+                input_type, output_type, options=None, serialized_options=None):
       _message.Message._CheckCalledFromGeneratedFile()  # pylint: disable=protected-access
       return _message.default_pool.FindMethodByName(full_name)
 
   def __init__(self, name, full_name, index, containing_service,
-               input_type, output_type, options=None):
+               input_type, output_type, options=None, serialized_options=None):
     """The arguments are as described in the description of MethodDescriptor
     attributes above.
 
     Note that containing_service may be None, and may be set later if necessary.
     """
-    super(MethodDescriptor, self).__init__(options, 'MethodOptions')
+    super(MethodDescriptor, self).__init__(
+        options, serialized_options, 'MethodOptions')
     self.name = name
     self.full_name = full_name
     self.index = index
@@ -818,7 +866,8 @@ class FileDescriptor(DescriptorBase):
   if _USE_C_DESCRIPTORS:
     _C_DESCRIPTOR_CLASS = _message.FileDescriptor
 
-    def __new__(cls, name, package, options=None, serialized_pb=None,
+    def __new__(cls, name, package, options=None,
+                serialized_options=None, serialized_pb=None,
                 dependencies=None, public_dependencies=None,
                 syntax=None, pool=None):
       # FileDescriptor() is called from various places, not only from generated
@@ -830,11 +879,13 @@ class FileDescriptor(DescriptorBase):
       else:
         return super(FileDescriptor, cls).__new__(cls)
 
-  def __init__(self, name, package, options=None, serialized_pb=None,
+  def __init__(self, name, package, options=None,
+               serialized_options=None, serialized_pb=None,
                dependencies=None, public_dependencies=None,
                syntax=None, pool=None):
     """Constructor."""
-    super(FileDescriptor, self).__init__(options, 'FileOptions')
+    super(FileDescriptor, self).__init__(
+        options, serialized_options, 'FileOptions')
 
     if pool is None:
       from google.protobuf import descriptor_pool

python/google/protobuf/internal/_parameterized.py

@@ -37,8 +37,8 @@ argument tuples.
 
 A simple example:
 
-  class AdditionExample(parameterized.ParameterizedTestCase):
-    @parameterized.Parameters(
+  class AdditionExample(parameterized.TestCase):
+    @parameterized.parameters(
        (1, 2, 3),
        (4, 5, 9),
        (1, 1, 3))
@@ -54,8 +54,8 @@ fail due to an assertion error (1 + 1 != 3).
 Parameters for invididual test cases can be tuples (with positional parameters)
 or dictionaries (with named parameters):
 
-  class AdditionExample(parameterized.ParameterizedTestCase):
-    @parameterized.Parameters(
+  class AdditionExample(parameterized.TestCase):
+    @parameterized.parameters(
        {'op1': 1, 'op2': 2, 'result': 3},
        {'op1': 4, 'op2': 5, 'result': 9},
     )
@@ -77,13 +77,13 @@ stay the same across several invocations, object representations like
   '<__main__.Foo object at 0x23d8610>'
 
 are turned into '<__main__.Foo>'. For even more descriptive names,
-especially in test logs, you can use the NamedParameters decorator. In
+especially in test logs, you can use the named_parameters decorator. In
 this case, only tuples are supported, and the first parameters has to
 be a string (or an object that returns an apt name when converted via
 str()):
 
-  class NamedExample(parameterized.ParameterizedTestCase):
-    @parameterized.NamedParameters(
+  class NamedExample(parameterized.TestCase):
+    @parameterized.named_parameters(
        ('Normal', 'aa', 'aaa', True),
        ('EmptyPrefix', '', 'abc', True),
        ('BothEmpty', '', '', True))
@@ -103,13 +103,13 @@ from the command line:
 Parameterized Classes
 =====================
 If invocation arguments are shared across test methods in a single
-ParameterizedTestCase class, instead of decorating all test methods
+TestCase class, instead of decorating all test methods
 individually, the class itself can be decorated:
 
-  @parameterized.Parameters(
+  @parameterized.parameters(
     (1, 2, 3)
     (4, 5, 9))
-  class ArithmeticTest(parameterized.ParameterizedTestCase):
+  class ArithmeticTest(parameterized.TestCase):
     def testAdd(self, arg1, arg2, result):
       self.assertEqual(arg1 + arg2, result)
 
@@ -122,8 +122,8 @@ If parameters should be shared across several test cases, or are dynamically
 created from other sources, a single non-tuple iterable can be passed into
 the decorator. This iterable will be used to obtain the test cases:
 
-  class AdditionExample(parameterized.ParameterizedTestCase):
-    @parameterized.Parameters(
+  class AdditionExample(parameterized.TestCase):
+    @parameterized.parameters(
       c.op1, c.op2, c.result for c in testcases
     )
     def testAddition(self, op1, op2, result):
@@ -135,8 +135,8 @@ Single-Argument Test Methods
 If a test method takes only one argument, the single argument does not need to
 be wrapped into a tuple:
 
-  class NegativeNumberExample(parameterized.ParameterizedTestCase):
-    @parameterized.Parameters(
+  class NegativeNumberExample(parameterized.TestCase):
+    @parameterized.parameters(
        -1, -3, -4, -5
     )
     def testIsNegative(self, arg):
@@ -212,7 +212,7 @@ class _ParameterizedTestIter(object):
   def __call__(self, *args, **kwargs):
     raise RuntimeError('You appear to be running a parameterized test case '
                        'without having inherited from parameterized.'
-                       'ParameterizedTestCase. This is bad because none of '
+                       'TestCase. This is bad because none of '
                        'your test cases are actually being run.')
 
   def __iter__(self):
@@ -306,7 +306,7 @@ def _ParameterDecorator(naming_type, testcases):
   return _Apply
 
 
-def Parameters(*testcases):
+def parameters(*testcases):  # pylint: disable=invalid-name
   """A decorator for creating parameterized tests.
 
   See the module docstring for a usage example.
@@ -321,7 +321,7 @@ def Parameters(*testcases):
   return _ParameterDecorator(_ARGUMENT_REPR, testcases)
 
 
-def NamedParameters(*testcases):
+def named_parameters(*testcases):  # pylint: disable=invalid-name
   """A decorator for creating parameterized tests.
 
   See the module docstring for a usage example. The first element of
@@ -348,7 +348,7 @@ class TestGeneratorMetaclass(type):
   up as tests by the unittest framework.
 
   In general, it is supposed to be used in conjunction with the
-  Parameters decorator.
+  parameters decorator.
   """
 
   def __new__(mcs, class_name, bases, dct):
@@ -385,8 +385,8 @@ def _UpdateClassDictForParamTestCase(dct, id_suffix, name, iterator):
     id_suffix[new_name] = getattr(func, '__x_extra_id__', '')
 
 
-class ParameterizedTestCase(unittest.TestCase):
-  """Base class for test cases using the Parameters decorator."""
+class TestCase(unittest.TestCase):
+  """Base class for test cases using the parameters decorator."""
   __metaclass__ = TestGeneratorMetaclass
 
   def _OriginalName(self):
@@ -409,10 +409,10 @@ class ParameterizedTestCase(unittest.TestCase):
                         self._id_suffix.get(self._testMethodName, ''))
 
 
-def CoopParameterizedTestCase(other_base_class):
+def CoopTestCase(other_base_class):
   """Returns a new base class with a cooperative metaclass base.
 
-  This enables the ParameterizedTestCase to be used in combination
+  This enables the TestCase to be used in combination
   with other base classes that have custom metaclasses, such as
   mox.MoxTestBase.
 
@@ -425,7 +425,7 @@ def CoopParameterizedTestCase(other_base_class):
 
     from google3.testing.pybase import parameterized
 
-    class ExampleTest(parameterized.CoopParameterizedTestCase(mox.MoxTestBase)):
+    class ExampleTest(parameterized.CoopTestCase(mox.MoxTestBase)):
       ...
 
   Args:
@@ -439,5 +439,5 @@ def CoopParameterizedTestCase(other_base_class):
       (other_base_class.__metaclass__,
        TestGeneratorMetaclass), {})
   return metaclass(
-      'CoopParameterizedTestCase',
-      (other_base_class, ParameterizedTestCase), {})
+      'CoopTestCase',
+      (other_base_class, TestCase), {})

python/google/protobuf/internal/api_implementation.py

@@ -66,10 +66,13 @@ if _api_version < 0:  # Still unspecified?
       from google.protobuf.internal import use_pure_python
       del use_pure_python  # Avoids a pylint error and namespace pollution.
     except ImportError:
-      if _proto_extension_modules_exist_in_build:
-        if sys.version_info[0] >= 3:  # Python 3 defaults to C++ impl v2.
-          _api_version = 2
-          # TODO(b/17427486): Make Python 2 default to C++ impl v2.
+      # TODO(b/74017912): It's unsafe to enable :use_fast_cpp_protos by default;
+      # it can cause data loss if you have any Python-only extensions to any
+      # message passed back and forth with C++ code.
+      #
+      # TODO(b/17427486): Once that bug is fixed, we want to make both Python 2
+      # and Python 3 default to `_api_version = 2` (C++ implementation V2).
+      pass
 
 _default_implementation_type = (
     'python' if _api_version <= 0 else 'cpp')

python/google/protobuf/internal/encoder.py

@@ -372,7 +372,7 @@ def MapSizer(field_descriptor, is_message_map):
 def _VarintEncoder():
   """Return an encoder for a basic varint value (does not include tag)."""
 
-  def EncodeVarint(write, value, unused_deterministic):
+  def EncodeVarint(write, value, unused_deterministic=None):
     bits = value & 0x7f
     value >>= 7
     while value:
@@ -388,7 +388,7 @@ def _SignedVarintEncoder():
   """Return an encoder for a basic signed varint value (does not include
   tag)."""
 
-  def EncodeSignedVarint(write, value, unused_deterministic):
+  def EncodeSignedVarint(write, value, unused_deterministic=None):
     if value < 0:
       value += (1 << 64)
     bits = value & 0x7f
@@ -524,14 +524,14 @@ def _StructPackEncoder(wire_type, format):
       return EncodePackedField
     elif is_repeated:
       tag_bytes = TagBytes(field_number, wire_type)
-      def EncodeRepeatedField(write, value, unused_deterministic):
+      def EncodeRepeatedField(write, value, unused_deterministic=None):
         for element in value:
           write(tag_bytes)
           write(local_struct_pack(format, element))
       return EncodeRepeatedField
     else:
       tag_bytes = TagBytes(field_number, wire_type)
-      def EncodeField(write, value, unused_deterministic):
+      def EncodeField(write, value, unused_deterministic=None):
         write(tag_bytes)
         return write(local_struct_pack(format, value))
       return EncodeField
@@ -595,7 +595,7 @@ def _FloatingPointEncoder(wire_type, format):
       return EncodePackedField
     elif is_repeated:
       tag_bytes = TagBytes(field_number, wire_type)
-      def EncodeRepeatedField(write, value, unused_deterministic):
+      def EncodeRepeatedField(write, value, unused_deterministic=None):
         for element in value:
           write(tag_bytes)
           try:
@@ -605,7 +605,7 @@ def _FloatingPointEncoder(wire_type, format):
       return EncodeRepeatedField
     else:
       tag_bytes = TagBytes(field_number, wire_type)
-      def EncodeField(write, value, unused_deterministic):
+      def EncodeField(write, value, unused_deterministic=None):
         write(tag_bytes)
         try:
           write(local_struct_pack(format, value))
@@ -662,7 +662,7 @@ def BoolEncoder(field_number, is_repeated, is_packed):
     return EncodePackedField
   elif is_repeated:
     tag_bytes = TagBytes(field_number, wire_format.WIRETYPE_VARINT)
-    def EncodeRepeatedField(write, value, unused_deterministic):
+    def EncodeRepeatedField(write, value, unused_deterministic=None):
       for element in value:
         write(tag_bytes)
         if element:
@@ -672,7 +672,7 @@ def BoolEncoder(field_number, is_repeated, is_packed):
     return EncodeRepeatedField
   else:
     tag_bytes = TagBytes(field_number, wire_format.WIRETYPE_VARINT)
-    def EncodeField(write, value, unused_deterministic):
+    def EncodeField(write, value, unused_deterministic=None):
       write(tag_bytes)
       if value:
         return write(true_byte)

python/google/protobuf/internal/json_format_test.py

@@ -983,6 +983,18 @@ class JsonFormatTest(JsonFormatBase):
     self.assertEqual('{\n"int32Value": 12345\n}',
                      json_format.MessageToJson(message, indent=0))
 
+  def testFormatEnumsAsInts(self):
+    message = json_format_proto3_pb2.TestMessage()
+    message.enum_value = json_format_proto3_pb2.BAR
+    message.repeated_enum_value.append(json_format_proto3_pb2.FOO)
+    message.repeated_enum_value.append(json_format_proto3_pb2.BAR)
+    self.assertEqual(json.loads('{\n'
+                                '  "enumValue": 1,\n'
+                                '  "repeatedEnumValue": [0, 1]\n'
+                                '}\n'),
+                     json.loads(json_format.MessageToJson(
+                         message, use_integers_for_enums=True)))
+
   def testParseDict(self):
     expected = 12345
     js_dict = {'int32Value': expected}

python/google/protobuf/internal/message_test.py

@@ -99,7 +99,7 @@ def IsNegInf(val):
 BaseTestCase = testing_refleaks.BaseTestCase
 
 
-@_parameterized.NamedParameters(
+@_parameterized.named_parameters(
     ('_proto2', unittest_pb2),
     ('_proto3', unittest_proto3_arena_pb2))
 class MessageTest(BaseTestCase):
@@ -1694,6 +1694,33 @@ class Proto3Test(BaseTestCase):
     with self.assertRaises(TypeError):
       del msg2.map_int32_foreign_message['']
 
+  def testMapMergeFrom(self):
+    msg = map_unittest_pb2.TestMap()
+    msg.map_int32_int32[12] = 34
+    msg.map_int32_int32[56] = 78
+    msg.map_int64_int64[22] = 33
+    msg.map_int32_foreign_message[111].c = 5
+    msg.map_int32_foreign_message[222].c = 10
+
+    msg2 = map_unittest_pb2.TestMap()
+    msg2.map_int32_int32[12] = 55
+    msg2.map_int64_int64[88] = 99
+    msg2.map_int32_foreign_message[222].c = 15
+    msg2.map_int32_foreign_message[222].d = 20
+
+    msg2.map_int32_int32.MergeFrom(msg.map_int32_int32)
+    self.assertEqual(34, msg2.map_int32_int32[12])
+    self.assertEqual(78, msg2.map_int32_int32[56])
+
+    msg2.map_int64_int64.MergeFrom(msg.map_int64_int64)
+    self.assertEqual(33, msg2.map_int64_int64[22])
+    self.assertEqual(99, msg2.map_int64_int64[88])
+
+    msg2.map_int32_foreign_message.MergeFrom(msg.map_int32_foreign_message)
+    self.assertEqual(5, msg2.map_int32_foreign_message[111].c)
+    self.assertEqual(10, msg2.map_int32_foreign_message[222].c)
+    self.assertFalse(msg2.map_int32_foreign_message[222].HasField('d'))
+
   def testMergeFromBadType(self):
     msg = map_unittest_pb2.TestMap()
     with self.assertRaisesRegexp(

python/google/protobuf/internal/no_package.proto

@@ -0,0 +1,10 @@
+syntax = "proto2";
+
+enum NoPackageEnum {
+  NO_PACKAGE_VALUE_0 = 0;
+  NO_PACKAGE_VALUE_1 = 1;
+}
+
+message NoPackageMessage {
+  optional NoPackageEnum no_package_enum = 1;
+}

python/google/protobuf/internal/text_format_test.py

@@ -48,6 +48,7 @@ except ImportError:
 
 from google.protobuf.internal import _parameterized
 
+from google.protobuf import any_pb2
 from google.protobuf import any_test_pb2
 from google.protobuf import map_unittest_pb2
 from google.protobuf import unittest_mset_pb2
@@ -99,7 +100,7 @@ class TextFormatBase(unittest.TestCase):
     return text
 
 
-@_parameterized.Parameters((unittest_pb2), (unittest_proto3_arena_pb2))
+@_parameterized.parameters((unittest_pb2), (unittest_proto3_arena_pb2))
 class TextFormatTest(TextFormatBase):
 
   def testPrintExotic(self, message_module):
@@ -369,6 +370,7 @@ class TextFormatTest(TextFormatBase):
   def testParseRepeatedScalarShortFormat(self, message_module):
     message = message_module.TestAllTypes()
     text = ('repeated_int64: [100, 200];\n'
+            'repeated_int64: []\n'
             'repeated_int64: 300,\n'
             'repeated_string: ["one", "two"];\n')
     text_format.Parse(text, message)
@@ -524,20 +526,68 @@ class OnlyWorksWithProto2RightNowTests(TextFormatBase):
 
   def testPrintInIndexOrder(self):
     message = unittest_pb2.TestFieldOrderings()
-    message.my_string = '115'
+    # Fields are listed in index order instead of field number.
+    message.my_string = 'str'
     message.my_int = 101
     message.my_float = 111
     message.optional_nested_message.oo = 0
     message.optional_nested_message.bb = 1
+    message.Extensions[unittest_pb2.my_extension_string] = 'ext_str0'
+    # Extensions are listed based on the order of extension number.
+    # Extension number 12.
+    message.Extensions[unittest_pb2.TestExtensionOrderings2.
+                       test_ext_orderings2].my_string = 'ext_str2'
+    # Extension number 13.
+    message.Extensions[unittest_pb2.TestExtensionOrderings1.
+                       test_ext_orderings1].my_string = 'ext_str1'
+    # Extension number 14.
+    message.Extensions[
+        unittest_pb2.TestExtensionOrderings2.TestExtensionOrderings3.
+        test_ext_orderings3].my_string = 'ext_str3'
+
+    # Print in index order.
     self.CompareToGoldenText(
-        self.RemoveRedundantZeros(text_format.MessageToString(
-            message, use_index_order=True)),
-        'my_string: \"115\"\nmy_int: 101\nmy_float: 111\n'
-        'optional_nested_message {\n  oo: 0\n  bb: 1\n}\n')
+        self.RemoveRedundantZeros(
+            text_format.MessageToString(message, use_index_order=True)),
+        'my_string: "str"\n'
+        'my_int: 101\n'
+        'my_float: 111\n'
+        'optional_nested_message {\n'
+        '  oo: 0\n'
+        '  bb: 1\n'
+        '}\n'
+        '[protobuf_unittest.TestExtensionOrderings2.test_ext_orderings2] {\n'
+        '  my_string: "ext_str2"\n'
+        '}\n'
+        '[protobuf_unittest.TestExtensionOrderings1.test_ext_orderings1] {\n'
+        '  my_string: "ext_str1"\n'
+        '}\n'
+        '[protobuf_unittest.TestExtensionOrderings2.TestExtensionOrderings3'
+        '.test_ext_orderings3] {\n'
+        '  my_string: "ext_str3"\n'
+        '}\n'
+        '[protobuf_unittest.my_extension_string]: "ext_str0"\n')
+    # By default, print in field number order.
     self.CompareToGoldenText(
         self.RemoveRedundantZeros(text_format.MessageToString(message)),
-        'my_int: 101\nmy_string: \"115\"\nmy_float: 111\n'
-        'optional_nested_message {\n  bb: 1\n  oo: 0\n}\n')
+        'my_int: 101\n'
+        'my_string: "str"\n'
+        '[protobuf_unittest.TestExtensionOrderings2.test_ext_orderings2] {\n'
+        '  my_string: "ext_str2"\n'
+        '}\n'
+        '[protobuf_unittest.TestExtensionOrderings1.test_ext_orderings1] {\n'
+        '  my_string: "ext_str1"\n'
+        '}\n'
+        '[protobuf_unittest.TestExtensionOrderings2.TestExtensionOrderings3'
+        '.test_ext_orderings3] {\n'
+        '  my_string: "ext_str3"\n'
+        '}\n'
+        '[protobuf_unittest.my_extension_string]: "ext_str0"\n'
+        'my_float: 111\n'
+        'optional_nested_message {\n'
+        '  bb: 1\n'
+        '  oo: 0\n'
+        '}\n')
 
   def testMergeLinesGolden(self):
     opened = self.ReadGolden('text_format_unittest_data_oneof_implemented.txt')
@@ -970,15 +1020,26 @@ class Proto2Tests(TextFormatBase):
         '"protobuf_unittest.optional_int32_extension" extensions.'),
                           text_format.Parse, text, message)
 
-  def testParseDuplicateNestedMessageScalars(self):
+  def testParseDuplicateMessages(self):
     message = unittest_pb2.TestAllTypes()
     text = ('optional_nested_message { bb: 1 } '
             'optional_nested_message { bb: 2 }')
     six.assertRaisesRegex(self, text_format.ParseError, (
-        '1:65 : Message type "protobuf_unittest.TestAllTypes.NestedMessage" '
-        'should not have multiple "bb" fields.'), text_format.Parse, text,
+        '1:59 : Message type "protobuf_unittest.TestAllTypes" '
+        'should not have multiple "optional_nested_message" fields.'),
+                          text_format.Parse, text,
                           message)
 
+  def testParseDuplicateExtensionMessages(self):
+    message = unittest_pb2.TestAllExtensions()
+    text = ('[protobuf_unittest.optional_nested_message_extension]: {} '
+            '[protobuf_unittest.optional_nested_message_extension]: {}')
+    six.assertRaisesRegex(self, text_format.ParseError, (
+        '1:114 : Message type "protobuf_unittest.TestAllExtensions" '
+        'should not have multiple '
+        '"protobuf_unittest.optional_nested_message_extension" extensions.'),
+                          text_format.Parse, text, message)
+
   def testParseDuplicateScalars(self):
     message = unittest_pb2.TestAllTypes()
     text = ('optional_int32: 42 ' 'optional_int32: 67')
@@ -1065,6 +1126,14 @@ class Proto3Tests(unittest.TestCase):
         '  }\n'
         '}\n')
 
+  def testTopAnyMessage(self):
+    packed_msg = unittest_pb2.OneString()
+    msg = any_pb2.Any()
+    msg.Pack(packed_msg)
+    text = text_format.MessageToString(msg)
+    other_msg = text_format.Parse(text, any_pb2.Any())
+    self.assertEqual(msg, other_msg)
+
   def testPrintMessageExpandAnyRepeated(self):
     packed_message = unittest_pb2.OneString()
     message = any_test_pb2.TestAny()
@@ -1489,7 +1558,7 @@ class TokenizerTest(unittest.TestCase):
 
 
 # Tests for pretty printer functionality.
-@_parameterized.Parameters((unittest_pb2), (unittest_proto3_arena_pb2))
+@_parameterized.parameters((unittest_pb2), (unittest_proto3_arena_pb2))
 class PrettyPrinterTest(TextFormatBase):
 
   def testPrettyPrintNoMatch(self, message_module):

python/google/protobuf/internal/well_known_types.py

@@ -375,6 +375,9 @@ def _CheckDurationValid(seconds, nanos):
     raise Error(
         'Duration is not valid: Nanos {0} must be in range '
         '[-999999999, 999999999].'.format(nanos))
+  if (nanos < 0 and seconds > 0) or (nanos > 0 and seconds < 0):
+    raise Error(
+        'Duration is not valid: Sign mismatch.')
 
 
 def _RoundTowardZero(value, divider):
@@ -649,9 +652,10 @@ def _MergeMessage(
         raise ValueError('Error: Field {0} in message {1} is not a singular '
                          'message field and cannot have sub-fields.'.format(
                              name, source_descriptor.full_name))
-      _MergeMessage(
-          child, getattr(source, name), getattr(destination, name),
-          replace_message, replace_repeated)
+      if source.HasField(name):
+        _MergeMessage(
+            child, getattr(source, name), getattr(destination, name),
+            replace_message, replace_repeated)
       continue
     if field.label == FieldDescriptor.LABEL_REPEATED:
       if replace_repeated:

python/google/protobuf/internal/well_known_types_test.py

@@ -345,6 +345,12 @@ class TimeUtilTest(TimeUtilTestBase):
         r'Duration is not valid\: Nanos 1000000000 must be in range'
         r' \[-999999999\, 999999999\].',
         message.ToJsonString)
+    message.seconds = -1
+    message.nanos = 1
+    self.assertRaisesRegexp(
+        well_known_types.Error,
+        r'Duration is not valid\: Sign mismatch.',
+        message.ToJsonString)
 
 
 class FieldMaskTest(unittest.TestCase):
@@ -599,6 +605,16 @@ class FieldMaskTest(unittest.TestCase):
     self.assertEqual(1, len(nested_dst.payload.repeated_int32))
     self.assertEqual(1234, nested_dst.payload.repeated_int32[0])
 
+    # Test Merge oneof field.
+    new_msg = unittest_pb2.TestOneof2()
+    dst = unittest_pb2.TestOneof2()
+    dst.foo_message.qux_int = 1
+    mask = field_mask_pb2.FieldMask()
+    mask.FromJsonString('fooMessage,fooLazyMessage.quxInt')
+    mask.MergeMessage(new_msg, dst)
+    self.assertTrue(dst.HasField('foo_message'))
+    self.assertFalse(dst.HasField('foo_lazy_message'))
+
   def testMergeErrors(self):
     src = unittest_pb2.TestAllTypes()
     dst = unittest_pb2.TestAllTypes()

python/google/protobuf/json_format.py

@@ -42,21 +42,28 @@ Simple usage example:
 
 __author__ = 'jieluo@google.com (Jie Luo)'
 
+# pylint: disable=g-statement-before-imports,g-import-not-at-top
 try:
   from collections import OrderedDict
 except ImportError:
-  from ordereddict import OrderedDict  #PY26
+  from ordereddict import OrderedDict  # PY26
+# pylint: enable=g-statement-before-imports,g-import-not-at-top
+
 import base64
 import json
 import math
+
+from operator import methodcaller
+
 import re
-import six
 import sys
 
-from operator import methodcaller
+import six
+
 from google.protobuf import descriptor
 from google.protobuf import symbol_database
 
+
 _TIMESTAMPFOMAT = '%Y-%m-%dT%H:%M:%S'
 _INT_TYPES = frozenset([descriptor.FieldDescriptor.CPPTYPE_INT32,
                         descriptor.FieldDescriptor.CPPTYPE_UINT32,
@@ -93,7 +100,8 @@ def MessageToJson(message,
                   including_default_value_fields=False,
                   preserving_proto_field_name=False,
                   indent=2,
-                  sort_keys=False):
+                  sort_keys=False,
+                  use_integers_for_enums=False):
   """Converts protobuf message to JSON format.
 
   Args:
@@ -108,18 +116,21 @@ def MessageToJson(message,
     indent: The JSON object will be pretty-printed with this indent level.
         An indent level of 0 or negative will only insert newlines.
     sort_keys: If True, then the output will be sorted by field names.
+    use_integers_for_enums: If true, print integers instead of enum names.
 
   Returns:
     A string containing the JSON formatted protocol buffer message.
   """
   printer = _Printer(including_default_value_fields,
-                     preserving_proto_field_name)
+                     preserving_proto_field_name,
+                     use_integers_for_enums)
   return printer.ToJsonString(message, indent, sort_keys)
 
 
 def MessageToDict(message,
                   including_default_value_fields=False,
-                  preserving_proto_field_name=False):
+                  preserving_proto_field_name=False,
+                  use_integers_for_enums=False):
   """Converts protobuf message to a dictionary.
 
   When the dictionary is encoded to JSON, it conforms to proto3 JSON spec.
@@ -133,12 +144,14 @@ def MessageToDict(message,
     preserving_proto_field_name: If True, use the original proto field
         names as defined in the .proto file. If False, convert the field
         names to lowerCamelCase.
+    use_integers_for_enums: If true, print integers instead of enum names.
 
   Returns:
     A dict representation of the protocol buffer message.
   """
   printer = _Printer(including_default_value_fields,
-                     preserving_proto_field_name)
+                     preserving_proto_field_name,
+                     use_integers_for_enums)
   # pylint: disable=protected-access
   return printer._MessageToJsonObject(message)
 
@@ -154,9 +167,11 @@ class _Printer(object):
 
   def __init__(self,
                including_default_value_fields=False,
-               preserving_proto_field_name=False):
+               preserving_proto_field_name=False,
+               use_integers_for_enums=False):
     self.including_default_value_fields = including_default_value_fields
     self.preserving_proto_field_name = preserving_proto_field_name
+    self.use_integers_for_enums = use_integers_for_enums
 
   def ToJsonString(self, message, indent, sort_keys):
     js = self._MessageToJsonObject(message)
@@ -247,6 +262,8 @@ class _Printer(object):
     if field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE:
       return self._MessageToJsonObject(value)
     elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_ENUM:
+      if self.use_integers_for_enums:
+        return value
       enum_value = field.enum_type.values_by_number.get(value, None)
       if enum_value is not None:
         return enum_value.name

python/google/protobuf/pyext/descriptor.cc

@@ -193,38 +193,35 @@ const FileDescriptor* GetFileDescriptor(const MethodDescriptor* descriptor) {
 // Always returns a new reference.
 template<class DescriptorClass>
 static PyObject* GetOrBuildOptions(const DescriptorClass *descriptor) {
-  // Options (and their extensions) are completely resolved in the proto file
-  // containing the descriptor.
-  PyDescriptorPool* pool = GetDescriptorPool_FromPool(
+  // Options are cached in the pool that owns the descriptor.
+  // First search in the cache.
+  PyDescriptorPool* caching_pool = GetDescriptorPool_FromPool(
       GetFileDescriptor(descriptor)->pool());
-
   hash_map<const void*, PyObject*>* descriptor_options =
-      pool->descriptor_options;
-  // First search in the cache.
+      caching_pool->descriptor_options;
   if (descriptor_options->find(descriptor) != descriptor_options->end()) {
     PyObject *value = (*descriptor_options)[descriptor];
     Py_INCREF(value);
     return value;
   }
 
+  // Similar to the C++ implementation, we return an Options object from the
+  // default (generated) factory, so that client code know that they can use
+  // extensions from generated files:
+  //    d.GetOptions().Extensions[some_pb2.extension]
+  //
+  // The consequence is that extensions not defined in the default pool won't
+  // be available.  If needed, we could add an optional 'message_factory'
+  // parameter to the GetOptions() function.
+  PyMessageFactory* message_factory =
+      GetDefaultDescriptorPool()->py_message_factory;
+
   // Build the Options object: get its Python class, and make a copy of the C++
   // read-only instance.
   const Message& options(descriptor->options());
   const Descriptor *message_type = options.GetDescriptor();
-  PyMessageFactory* message_factory = pool->py_message_factory;
-  CMessageClass* message_class = message_factory::GetMessageClass(
+  CMessageClass* message_class = message_factory::GetOrCreateMessageClass(
       message_factory, message_type);
-  if (message_class == NULL) {
-    // The Options message was not found in the current DescriptorPool.
-    // This means that the pool cannot contain any extensions to the Options
-    // message either, so falling back to the basic pool we can only increase
-    // the chances of successfully parsing the options.
-    PyErr_Clear();
-    pool = GetDefaultDescriptorPool();
-    message_factory = pool->py_message_factory;
-    message_class = message_factory::GetMessageClass(
-      message_factory, message_type);
-  }
   if (message_class == NULL) {
     PyErr_Format(PyExc_TypeError, "Could not retrieve class for Options: %s",
                  message_type->full_name().c_str());
@@ -253,7 +250,8 @@ static PyObject* GetOrBuildOptions(const DescriptorClass *descriptor) {
     options.SerializeToString(&serialized);
     io::CodedInputStream input(
         reinterpret_cast<const uint8*>(serialized.c_str()), serialized.size());
-    input.SetExtensionRegistry(pool->pool, message_factory->message_factory);
+    input.SetExtensionRegistry(message_factory->pool->pool,
+                               message_factory->message_factory);
     bool success = cmsg->message->MergePartialFromCodedStream(&input);
     if (!success) {
       PyErr_Format(PyExc_ValueError, "Error parsing Options message");
@@ -569,6 +567,11 @@ static int SetOptions(PyBaseDescriptor *self, PyObject *value,
   return CheckCalledFromGeneratedFile("_options");
 }
 
+static int SetSerializedOptions(PyBaseDescriptor *self, PyObject *value,
+                                void *closure) {
+  return CheckCalledFromGeneratedFile("_serialized_options");
+}
+
 static PyObject* CopyToProto(PyBaseDescriptor *self, PyObject *target) {
   return CopyToPythonProto<DescriptorProto>(_GetDescriptor(self), target);
 }
@@ -628,6 +631,8 @@ static PyGetSetDef Getters[] = {
   { "is_extendable", (getter)IsExtendable, (setter)NULL},
   { "has_options", (getter)GetHasOptions, (setter)SetHasOptions, "Has Options"},
   { "_options", (getter)NULL, (setter)SetOptions, "Options"},
+  { "_serialized_options", (getter)NULL, (setter)SetSerializedOptions,
+    "Serialized Options"},
   { "syntax", (getter)GetSyntax, (setter)NULL, "Syntax"},
   {NULL}
 };
@@ -790,7 +795,7 @@ static PyObject* GetDefaultValue(PyBaseDescriptor *self, void *closure) {
       break;
     }
     case FieldDescriptor::CPPTYPE_STRING: {
-      string value = _GetDescriptor(self)->default_value_string();
+      const string& value = _GetDescriptor(self)->default_value_string();
       result = ToStringObject(_GetDescriptor(self), value);
       break;
     }
@@ -902,6 +907,10 @@ static int SetOptions(PyBaseDescriptor *self, PyObject *value,
   return CheckCalledFromGeneratedFile("_options");
 }
 
+static int SetSerializedOptions(PyBaseDescriptor *self, PyObject *value,
+                                void *closure) {
+  return CheckCalledFromGeneratedFile("_serialized_options");
+}
 
 static PyGetSetDef Getters[] = {
   { "full_name", (getter)GetFullName, NULL, "Full name"},
@@ -931,6 +940,8 @@ static PyGetSetDef Getters[] = {
     "Containing oneof"},
   { "has_options", (getter)GetHasOptions, (setter)SetHasOptions, "Has Options"},
   { "_options", (getter)NULL, (setter)SetOptions, "Options"},
+  { "_serialized_options", (getter)NULL, (setter)SetSerializedOptions,
+    "Serialized Options"},
   {NULL}
 };
 
@@ -1060,6 +1071,11 @@ static int SetOptions(PyBaseDescriptor *self, PyObject *value,
   return CheckCalledFromGeneratedFile("_options");
 }
 
+static int SetSerializedOptions(PyBaseDescriptor *self, PyObject *value,
+                                void *closure) {
+  return CheckCalledFromGeneratedFile("_serialized_options");
+}
+
 static PyObject* CopyToProto(PyBaseDescriptor *self, PyObject *target) {
   return CopyToPythonProto<EnumDescriptorProto>(_GetDescriptor(self), target);
 }
@@ -1084,6 +1100,8 @@ static PyGetSetDef Getters[] = {
     "Containing type"},
   { "has_options", (getter)GetHasOptions, (setter)SetHasOptions, "Has Options"},
   { "_options", (getter)NULL, (setter)SetOptions, "Options"},
+  { "_serialized_options", (getter)NULL, (setter)SetSerializedOptions,
+    "Serialized Options"},
   {NULL}
 };
 
@@ -1184,6 +1202,10 @@ static int SetOptions(PyBaseDescriptor *self, PyObject *value,
   return CheckCalledFromGeneratedFile("_options");
 }
 
+static int SetSerializedOptions(PyBaseDescriptor *self, PyObject *value,
+                                void *closure) {
+  return CheckCalledFromGeneratedFile("_serialized_options");
+}
 
 static PyGetSetDef Getters[] = {
   { "name", (getter)GetName, NULL, "name"},
@@ -1193,6 +1215,8 @@ static PyGetSetDef Getters[] = {
 
   { "has_options", (getter)GetHasOptions, (setter)SetHasOptions, "Has Options"},
   { "_options", (getter)NULL, (setter)SetOptions, "Options"},
+  { "_serialized_options", (getter)NULL, (setter)SetSerializedOptions,
+    "Serialized Options"},
   {NULL}
 };
 
@@ -1335,6 +1359,11 @@ static int SetOptions(PyFileDescriptor *self, PyObject *value,
   return CheckCalledFromGeneratedFile("_options");
 }
 
+static int SetSerializedOptions(PyFileDescriptor *self, PyObject *value,
+                                void *closure) {
+  return CheckCalledFromGeneratedFile("_serialized_options");
+}
+
 static PyObject* GetSyntax(PyFileDescriptor *self, void *closure) {
   return PyString_InternFromString(
       FileDescriptor::SyntaxName(_GetDescriptor(self)->syntax()));
@@ -1360,6 +1389,8 @@ static PyGetSetDef Getters[] = {
 
   { "has_options", (getter)GetHasOptions, (setter)SetHasOptions, "Has Options"},
   { "_options", (getter)NULL, (setter)SetOptions, "Options"},
+  { "_serialized_options", (getter)NULL, (setter)SetSerializedOptions,
+    "Serialized Options"},
   { "syntax", (getter)GetSyntax, (setter)NULL, "Syntax"},
   {NULL}
 };
@@ -1505,6 +1536,11 @@ static int SetOptions(PyBaseDescriptor *self, PyObject *value,
   return CheckCalledFromGeneratedFile("_options");
 }
 
+static int SetSerializedOptions(PyBaseDescriptor *self, PyObject *value,
+                                void *closure) {
+  return CheckCalledFromGeneratedFile("_serialized_options");
+}
+
 static PyGetSetDef Getters[] = {
   { "name", (getter)GetName, NULL, "Name"},
   { "full_name", (getter)GetFullName, NULL, "Full name"},
@@ -1513,6 +1549,8 @@ static PyGetSetDef Getters[] = {
   { "containing_type", (getter)GetContainingType, NULL, "Containing type"},
   { "has_options", (getter)GetHasOptions, (setter)SetHasOptions, "Has Options"},
   { "_options", (getter)NULL, (setter)SetOptions, "Options"},
+  { "_serialized_options", (getter)NULL, (setter)SetSerializedOptions,
+    "Serialized Options"},
   { "fields", (getter)GetFields, NULL, "Fields"},
   {NULL}
 };

python/google/protobuf/pyext/descriptor_pool.cc

@@ -149,7 +149,8 @@ static PyObject* New(PyTypeObject* type,
       PyDescriptorPool_NewWithDatabase(database));
 }
 
-static void Dealloc(PyDescriptorPool* self) {
+static void Dealloc(PyObject* pself) {
+  PyDescriptorPool* self = reinterpret_cast<PyDescriptorPool*>(pself);
   descriptor_pool_map.erase(self->pool);
   Py_CLEAR(self->py_message_factory);
   for (hash_map<const void*, PyObject*>::iterator it =
@@ -163,7 +164,7 @@ static void Dealloc(PyDescriptorPool* self) {
   Py_TYPE(self)->tp_free(reinterpret_cast<PyObject*>(self));
 }
 
-PyObject* FindMessageByName(PyDescriptorPool* self, PyObject* arg) {
+static PyObject* FindMessageByName(PyObject* self, PyObject* arg) {
   Py_ssize_t name_size;
   char* name;
   if (PyString_AsStringAndSize(arg, &name, &name_size) < 0) {
@@ -171,7 +172,8 @@ PyObject* FindMessageByName(PyDescriptorPool* self, PyObject* arg) {
   }
 
   const Descriptor* message_descriptor =
-      self->pool->FindMessageTypeByName(string(name, name_size));
+      reinterpret_cast<PyDescriptorPool*>(self)->pool->FindMessageTypeByName(
+          string(name, name_size));
 
   if (message_descriptor == NULL) {
     PyErr_Format(PyExc_KeyError, "Couldn't find message %.200s", name);
@@ -184,7 +186,7 @@ PyObject* FindMessageByName(PyDescriptorPool* self, PyObject* arg) {
 
 
 
-PyObject* FindFileByName(PyDescriptorPool* self, PyObject* arg) {
+static PyObject* FindFileByName(PyObject* self, PyObject* arg) {
   Py_ssize_t name_size;
   char* name;
   if (PyString_AsStringAndSize(arg, &name, &name_size) < 0) {
@@ -192,7 +194,8 @@ PyObject* FindFileByName(PyDescriptorPool* self, PyObject* arg) {
   }
 
   const FileDescriptor* file_descriptor =
-      self->pool->FindFileByName(string(name, name_size));
+      reinterpret_cast<PyDescriptorPool*>(self)->pool->FindFileByName(
+          string(name, name_size));
   if (file_descriptor == NULL) {
     PyErr_Format(PyExc_KeyError, "Couldn't find file %.200s", name);
     return NULL;
@@ -218,6 +221,10 @@ PyObject* FindFieldByName(PyDescriptorPool* self, PyObject* arg) {
   return PyFieldDescriptor_FromDescriptor(field_descriptor);
 }
 
+static PyObject* FindFieldByNameMethod(PyObject* self, PyObject* arg) {
+  return FindFieldByName(reinterpret_cast<PyDescriptorPool*>(self), arg);
+}
+
 PyObject* FindExtensionByName(PyDescriptorPool* self, PyObject* arg) {
   Py_ssize_t name_size;
   char* name;
@@ -235,6 +242,10 @@ PyObject* FindExtensionByName(PyDescriptorPool* self, PyObject* arg) {
   return PyFieldDescriptor_FromDescriptor(field_descriptor);
 }
 
+static PyObject* FindExtensionByNameMethod(PyObject* self, PyObject* arg) {
+  return FindExtensionByName(reinterpret_cast<PyDescriptorPool*>(self), arg);
+}
+
 PyObject* FindEnumTypeByName(PyDescriptorPool* self, PyObject* arg) {
   Py_ssize_t name_size;
   char* name;
@@ -252,6 +263,10 @@ PyObject* FindEnumTypeByName(PyDescriptorPool* self, PyObject* arg) {
   return PyEnumDescriptor_FromDescriptor(enum_descriptor);
 }
 
+static PyObject* FindEnumTypeByNameMethod(PyObject* self, PyObject* arg) {
+  return FindEnumTypeByName(reinterpret_cast<PyDescriptorPool*>(self), arg);
+}
+
 PyObject* FindOneofByName(PyDescriptorPool* self, PyObject* arg) {
   Py_ssize_t name_size;
   char* name;
@@ -269,7 +284,11 @@ PyObject* FindOneofByName(PyDescriptorPool* self, PyObject* arg) {
   return PyOneofDescriptor_FromDescriptor(oneof_descriptor);
 }
 
-PyObject* FindServiceByName(PyDescriptorPool* self, PyObject* arg) {
+static PyObject* FindOneofByNameMethod(PyObject* self, PyObject* arg) {
+  return FindOneofByName(reinterpret_cast<PyDescriptorPool*>(self), arg);
+}
+
+static PyObject* FindServiceByName(PyObject* self, PyObject* arg) {
   Py_ssize_t name_size;
   char* name;
   if (PyString_AsStringAndSize(arg, &name, &name_size) < 0) {
@@ -277,7 +296,8 @@ PyObject* FindServiceByName(PyDescriptorPool* self, PyObject* arg) {
   }
 
   const ServiceDescriptor* service_descriptor =
-      self->pool->FindServiceByName(string(name, name_size));
+      reinterpret_cast<PyDescriptorPool*>(self)->pool->FindServiceByName(
+          string(name, name_size));
   if (service_descriptor == NULL) {
     PyErr_Format(PyExc_KeyError, "Couldn't find service %.200s", name);
     return NULL;
@@ -286,7 +306,7 @@ PyObject* FindServiceByName(PyDescriptorPool* self, PyObject* arg) {
   return PyServiceDescriptor_FromDescriptor(service_descriptor);
 }
 
-PyObject* FindMethodByName(PyDescriptorPool* self, PyObject* arg) {
+static PyObject* FindMethodByName(PyObject* self, PyObject* arg) {
   Py_ssize_t name_size;
   char* name;
   if (PyString_AsStringAndSize(arg, &name, &name_size) < 0) {
@@ -294,7 +314,8 @@ PyObject* FindMethodByName(PyDescriptorPool* self, PyObject* arg) {
   }
 
   const MethodDescriptor* method_descriptor =
-      self->pool->FindMethodByName(string(name, name_size));
+      reinterpret_cast<PyDescriptorPool*>(self)->pool->FindMethodByName(
+          string(name, name_size));
   if (method_descriptor == NULL) {
     PyErr_Format(PyExc_KeyError, "Couldn't find method %.200s", name);
     return NULL;
@@ -303,7 +324,7 @@ PyObject* FindMethodByName(PyDescriptorPool* self, PyObject* arg) {
   return PyMethodDescriptor_FromDescriptor(method_descriptor);
 }
 
-PyObject* FindFileContainingSymbol(PyDescriptorPool* self, PyObject* arg) {
+static PyObject* FindFileContainingSymbol(PyObject* self, PyObject* arg) {
   Py_ssize_t name_size;
   char* name;
   if (PyString_AsStringAndSize(arg, &name, &name_size) < 0) {
@@ -311,7 +332,8 @@ PyObject* FindFileContainingSymbol(PyDescriptorPool* self, PyObject* arg) {
   }
 
   const FileDescriptor* file_descriptor =
-      self->pool->FindFileContainingSymbol(string(name, name_size));
+      reinterpret_cast<PyDescriptorPool*>(self)->pool->FindFileContainingSymbol(
+          string(name, name_size));
   if (file_descriptor == NULL) {
     PyErr_Format(PyExc_KeyError, "Couldn't find symbol %.200s", name);
     return NULL;
@@ -320,7 +342,7 @@ PyObject* FindFileContainingSymbol(PyDescriptorPool* self, PyObject* arg) {
   return PyFileDescriptor_FromDescriptor(file_descriptor);
 }
 
-PyObject* FindExtensionByNumber(PyDescriptorPool* self, PyObject* args) {
+static PyObject* FindExtensionByNumber(PyObject* self, PyObject* args) {
   PyObject* message_descriptor;
   int number;
   if (!PyArg_ParseTuple(args, "Oi", &message_descriptor, &number)) {
@@ -333,7 +355,8 @@ PyObject* FindExtensionByNumber(PyDescriptorPool* self, PyObject* args) {
   }
 
   const FieldDescriptor* extension_descriptor =
-      self->pool->FindExtensionByNumber(descriptor, number);
+      reinterpret_cast<PyDescriptorPool*>(self)->pool->FindExtensionByNumber(
+          descriptor, number);
   if (extension_descriptor == NULL) {
     PyErr_Format(PyExc_KeyError, "Couldn't find extension %d", number);
     return NULL;
@@ -342,14 +365,15 @@ PyObject* FindExtensionByNumber(PyDescriptorPool* self, PyObject* args) {
   return PyFieldDescriptor_FromDescriptor(extension_descriptor);
 }
 
-PyObject* FindAllExtensions(PyDescriptorPool* self, PyObject* arg) {
+static PyObject* FindAllExtensions(PyObject* self, PyObject* arg) {
   const Descriptor* descriptor = PyMessageDescriptor_AsDescriptor(arg);
   if (descriptor == NULL) {
     return NULL;
   }
 
   std::vector<const FieldDescriptor*> extensions;
-  self->pool->FindAllExtensions(descriptor, &extensions);
+  reinterpret_cast<PyDescriptorPool*>(self)->pool->FindAllExtensions(
+      descriptor, &extensions);
 
   ScopedPyObjectPtr result(PyList_New(extensions.size()));
   if (result == NULL) {
@@ -374,14 +398,15 @@ PyObject* FindAllExtensions(PyDescriptorPool* self, PyObject* arg) {
 // call a function that will just be a no-op?
 // TODO(amauryfa): Need to investigate further.
 
-PyObject* AddFileDescriptor(PyDescriptorPool* self, PyObject* descriptor) {
+static PyObject* AddFileDescriptor(PyObject* self, PyObject* descriptor) {
   const FileDescriptor* file_descriptor =
       PyFileDescriptor_AsDescriptor(descriptor);
   if (!file_descriptor) {
     return NULL;
   }
   if (file_descriptor !=
-      self->pool->FindFileByName(file_descriptor->name())) {
+      reinterpret_cast<PyDescriptorPool*>(self)->pool->FindFileByName(
+          file_descriptor->name())) {
     PyErr_Format(PyExc_ValueError,
                  "The file descriptor %s does not belong to this pool",
                  file_descriptor->name().c_str());
@@ -390,14 +415,15 @@ PyObject* AddFileDescriptor(PyDescriptorPool* self, PyObject* descriptor) {
   Py_RETURN_NONE;
 }
 
-PyObject* AddDescriptor(PyDescriptorPool* self, PyObject* descriptor) {
+static PyObject* AddDescriptor(PyObject* self, PyObject* descriptor) {
   const Descriptor* message_descriptor =
       PyMessageDescriptor_AsDescriptor(descriptor);
   if (!message_descriptor) {
     return NULL;
   }
   if (message_descriptor !=
-      self->pool->FindMessageTypeByName(message_descriptor->full_name())) {
+      reinterpret_cast<PyDescriptorPool*>(self)->pool->FindMessageTypeByName(
+          message_descriptor->full_name())) {
     PyErr_Format(PyExc_ValueError,
                  "The message descriptor %s does not belong to this pool",
                  message_descriptor->full_name().c_str());
@@ -406,14 +432,15 @@ PyObject* AddDescriptor(PyDescriptorPool* self, PyObject* descriptor) {
   Py_RETURN_NONE;
 }
 
-PyObject* AddEnumDescriptor(PyDescriptorPool* self, PyObject* descriptor) {
+static PyObject* AddEnumDescriptor(PyObject* self, PyObject* descriptor) {
   const EnumDescriptor* enum_descriptor =
       PyEnumDescriptor_AsDescriptor(descriptor);
   if (!enum_descriptor) {
     return NULL;
   }
   if (enum_descriptor !=
-      self->pool->FindEnumTypeByName(enum_descriptor->full_name())) {
+      reinterpret_cast<PyDescriptorPool*>(self)->pool->FindEnumTypeByName(
+          enum_descriptor->full_name())) {
     PyErr_Format(PyExc_ValueError,
                  "The enum descriptor %s does not belong to this pool",
                  enum_descriptor->full_name().c_str());
@@ -422,14 +449,15 @@ PyObject* AddEnumDescriptor(PyDescriptorPool* self, PyObject* descriptor) {
   Py_RETURN_NONE;
 }
 
-PyObject* AddExtensionDescriptor(PyDescriptorPool* self, PyObject* descriptor) {
+static PyObject* AddExtensionDescriptor(PyObject* self, PyObject* descriptor) {
   const FieldDescriptor* extension_descriptor =
       PyFieldDescriptor_AsDescriptor(descriptor);
   if (!extension_descriptor) {
     return NULL;
   }
   if (extension_descriptor !=
-      self->pool->FindExtensionByName(extension_descriptor->full_name())) {
+      reinterpret_cast<PyDescriptorPool*>(self)->pool->FindExtensionByName(
+          extension_descriptor->full_name())) {
     PyErr_Format(PyExc_ValueError,
                  "The extension descriptor %s does not belong to this pool",
                  extension_descriptor->full_name().c_str());
@@ -438,14 +466,15 @@ PyObject* AddExtensionDescriptor(PyDescriptorPool* self, PyObject* descriptor) {
   Py_RETURN_NONE;
 }
 
-PyObject* AddServiceDescriptor(PyDescriptorPool* self, PyObject* descriptor) {
+static PyObject* AddServiceDescriptor(PyObject* self, PyObject* descriptor) {
   const ServiceDescriptor* service_descriptor =
       PyServiceDescriptor_AsDescriptor(descriptor);
   if (!service_descriptor) {
     return NULL;
   }
   if (service_descriptor !=
-      self->pool->FindServiceByName(service_descriptor->full_name())) {
+      reinterpret_cast<PyDescriptorPool*>(self)->pool->FindServiceByName(
+          service_descriptor->full_name())) {
     PyErr_Format(PyExc_ValueError,
                  "The service descriptor %s does not belong to this pool",
                  service_descriptor->full_name().c_str());
@@ -481,7 +510,8 @@ class BuildFileErrorCollector : public DescriptorPool::ErrorCollector {
   bool had_errors;
 };
 
-PyObject* AddSerializedFile(PyDescriptorPool* self, PyObject* serialized_pb) {
+static PyObject* AddSerializedFile(PyObject* pself, PyObject* serialized_pb) {
+  PyDescriptorPool* self = reinterpret_cast<PyDescriptorPool*>(pself);
   char* message_type;
   Py_ssize_t message_len;
 
@@ -529,7 +559,7 @@ PyObject* AddSerializedFile(PyDescriptorPool* self, PyObject* serialized_pb) {
       descriptor, serialized_pb);
 }
 
-PyObject* Add(PyDescriptorPool* self, PyObject* file_descriptor_proto) {
+static PyObject* Add(PyObject* self, PyObject* file_descriptor_proto) {
   ScopedPyObjectPtr serialized_pb(
       PyObject_CallMethod(file_descriptor_proto, "SerializeToString", NULL));
   if (serialized_pb == NULL) {
@@ -539,46 +569,46 @@ PyObject* Add(PyDescriptorPool* self, PyObject* file_descriptor_proto) {
 }
 
 static PyMethodDef Methods[] = {
-  { "Add", (PyCFunction)Add, METH_O,
+  { "Add", Add, METH_O,
     "Adds the FileDescriptorProto and its types to this pool." },
-  { "AddSerializedFile", (PyCFunction)AddSerializedFile, METH_O,
+  { "AddSerializedFile", AddSerializedFile, METH_O,
     "Adds a serialized FileDescriptorProto to this pool." },
 
   // TODO(amauryfa): Understand why the Python implementation differs from
   // this one, ask users to use another API and deprecate these functions.
-  { "AddFileDescriptor", (PyCFunction)AddFileDescriptor, METH_O,
+  { "AddFileDescriptor", AddFileDescriptor, METH_O,
     "No-op. Add() must have been called before." },
-  { "AddDescriptor", (PyCFunction)AddDescriptor, METH_O,
+  { "AddDescriptor", AddDescriptor, METH_O,
     "No-op. Add() must have been called before." },
-  { "AddEnumDescriptor", (PyCFunction)AddEnumDescriptor, METH_O,
+  { "AddEnumDescriptor", AddEnumDescriptor, METH_O,
     "No-op. Add() must have been called before." },
-  { "AddExtensionDescriptor", (PyCFunction)AddExtensionDescriptor, METH_O,
+  { "AddExtensionDescriptor", AddExtensionDescriptor, METH_O,
     "No-op. Add() must have been called before." },
-  { "AddServiceDescriptor", (PyCFunction)AddServiceDescriptor, METH_O,
+  { "AddServiceDescriptor", AddServiceDescriptor, METH_O,
     "No-op. Add() must have been called before." },
 
-  { "FindFileByName", (PyCFunction)FindFileByName, METH_O,
+  { "FindFileByName", FindFileByName, METH_O,
     "Searches for a file descriptor by its .proto name." },
-  { "FindMessageTypeByName", (PyCFunction)FindMessageByName, METH_O,
+  { "FindMessageTypeByName", FindMessageByName, METH_O,
     "Searches for a message descriptor by full name." },
-  { "FindFieldByName", (PyCFunction)FindFieldByName, METH_O,
+  { "FindFieldByName", FindFieldByNameMethod, METH_O,
     "Searches for a field descriptor by full name." },
-  { "FindExtensionByName", (PyCFunction)FindExtensionByName, METH_O,
+  { "FindExtensionByName", FindExtensionByNameMethod, METH_O,
     "Searches for extension descriptor by full name." },
-  { "FindEnumTypeByName", (PyCFunction)FindEnumTypeByName, METH_O,
+  { "FindEnumTypeByName", FindEnumTypeByNameMethod, METH_O,
     "Searches for enum type descriptor by full name." },
-  { "FindOneofByName", (PyCFunction)FindOneofByName, METH_O,
+  { "FindOneofByName", FindOneofByNameMethod, METH_O,
     "Searches for oneof descriptor by full name." },
-  { "FindServiceByName", (PyCFunction)FindServiceByName, METH_O,
+  { "FindServiceByName", FindServiceByName, METH_O,
     "Searches for service descriptor by full name." },
-  { "FindMethodByName", (PyCFunction)FindMethodByName, METH_O,
+  { "FindMethodByName", FindMethodByName, METH_O,
     "Searches for method descriptor by full name." },
 
-  { "FindFileContainingSymbol", (PyCFunction)FindFileContainingSymbol, METH_O,
+  { "FindFileContainingSymbol", FindFileContainingSymbol, METH_O,
     "Gets the FileDescriptor containing the specified symbol." },
-  { "FindExtensionByNumber", (PyCFunction)FindExtensionByNumber, METH_VARARGS,
+  { "FindExtensionByNumber", FindExtensionByNumber, METH_VARARGS,
     "Gets the extension descriptor for the given number." },
-  { "FindAllExtensions", (PyCFunction)FindAllExtensions, METH_O,
+  { "FindAllExtensions", FindAllExtensions, METH_O,
     "Gets all known extensions of the given message descriptor." },
   {NULL}
 };
@@ -590,7 +620,7 @@ PyTypeObject PyDescriptorPool_Type = {
   FULL_MODULE_NAME ".DescriptorPool",  // tp_name
   sizeof(PyDescriptorPool),            // tp_basicsize
   0,                                   // tp_itemsize
-  (destructor)cdescriptor_pool::Dealloc,  // tp_dealloc
+  cdescriptor_pool::Dealloc,           // tp_dealloc
   0,                                   // tp_print
   0,                                   // tp_getattr
   0,                                   // tp_setattr

python/google/protobuf/pyext/extension_dict.cc

@@ -33,9 +33,6 @@
 
 #include <google/protobuf/pyext/extension_dict.h>
 #include <memory>
-#ifndef _SHARED_PTR_H
-#include <google/protobuf/stubs/shared_ptr.h>
-#endif
 
 #include <google/protobuf/stubs/logging.h>
 #include <google/protobuf/stubs/common.h>

python/google/protobuf/pyext/extension_dict.h

@@ -37,9 +37,8 @@
 #include <Python.h>
 
 #include <memory>
-#ifndef _SHARED_PTR_H
-#include <google/protobuf/stubs/shared_ptr.h>
-#endif
+
+#include <google/protobuf/pyext/message.h>
 
 namespace google {
 namespace protobuf {
@@ -47,16 +46,8 @@ namespace protobuf {
 class Message;
 class FieldDescriptor;
 
-#ifdef _SHARED_PTR_H
-using std::shared_ptr;
-#else
-using internal::shared_ptr;
-#endif
-
 namespace python {
 
-struct CMessage;
-
 typedef struct ExtensionDict {
   PyObject_HEAD;
 
@@ -64,7 +55,7 @@ typedef struct ExtensionDict {
   // proto tree.  Every Python container class holds a
   // reference to it in order to keep it alive as long as there's a
   // Python object that references any part of the tree.
-  shared_ptr<Message> owner;
+  CMessage::OwnerRef owner;
 
   // Weak reference to parent message. Used to make sure
   // the parent is writable when an extension field is modified.

python/google/protobuf/pyext/map_container.cc

@@ -33,9 +33,6 @@
 #include <google/protobuf/pyext/map_container.h>
 
 #include <memory>
-#ifndef _SHARED_PTR_H
-#include <google/protobuf/stubs/shared_ptr.h>
-#endif
 
 #include <google/protobuf/stubs/logging.h>
 #include <google/protobuf/stubs/common.h>
@@ -76,7 +73,7 @@ class MapReflectionFriend {
 struct MapIterator {
   PyObject_HEAD;
 
-  google::protobuf::scoped_ptr< ::google::protobuf::MapIterator> iter;
+  std::unique_ptr<::google::protobuf::MapIterator> iter;
 
   // A pointer back to the container, so we can notice changes to the version.
   // We own a ref on this.
@@ -94,7 +91,7 @@ struct MapIterator {
   // as this iterator does.  This is solely for the benefit of the MapIterator
   // destructor -- we should never actually access the iterator in this state
   // except to delete it.
-  shared_ptr<Message> owner;
+  CMessage::OwnerRef owner;
 
   // The version of the map when we took the iterator to it.
   //
@@ -339,6 +336,24 @@ PyObject* GetEntryClass(PyObject* _self) {
   return reinterpret_cast<PyObject*>(message_class);
 }
 
+PyObject* MergeFrom(PyObject* _self, PyObject* arg) {
+  MapContainer* self = GetMap(_self);
+  MapContainer* other_map = GetMap(arg);
+  Message* message = self->GetMutableMessage();
+  const Message* other_message = other_map->message;
+  const Reflection* reflection = message->GetReflection();
+  const Reflection* other_reflection = other_message->GetReflection();
+  int count = other_reflection->FieldSize(
+      *other_message, other_map->parent_field_descriptor);
+  for (int i = 0 ; i < count; i ++) {
+    reflection->AddMessage(message, self->parent_field_descriptor)->MergeFrom(
+        other_reflection->GetRepeatedMessage(
+            *other_message, other_map->parent_field_descriptor, i));
+  }
+  self->version++;
+  Py_RETURN_NONE;
+}
+
 PyObject* MapReflectionFriend::Contains(PyObject* _self, PyObject* key) {
   MapContainer* self = GetMap(_self);
 
@@ -535,6 +550,8 @@ static PyMethodDef ScalarMapMethods[] = {
     "Gets the value for the given key if present, or otherwise a default" },
   { "GetEntryClass", (PyCFunction)GetEntryClass, METH_NOARGS,
     "Return the class used to build Entries of (key, value) pairs." },
+  { "MergeFrom", (PyCFunction)MergeFrom, METH_O,
+    "Merges a map into the current map." },
   /*
   { "__deepcopy__", (PyCFunction)DeepCopy, METH_VARARGS,
     "Makes a deep copy of the class." },
@@ -810,6 +827,8 @@ static PyMethodDef MessageMapMethods[] = {
     "Alias for getitem, useful to make explicit that the map is mutated." },
   { "GetEntryClass", (PyCFunction)GetEntryClass, METH_NOARGS,
     "Return the class used to build Entries of (key, value) pairs." },
+  { "MergeFrom", (PyCFunction)MergeFrom, METH_O,
+    "Merges a map into the current map." },
   /*
   { "__deepcopy__", (PyCFunction)DeepCopy, METH_VARARGS,
     "Makes a deep copy of the class." },

python/google/protobuf/pyext/map_container.h

@@ -34,27 +34,18 @@
 #include <Python.h>
 
 #include <memory>
-#ifndef _SHARED_PTR_H
-#include <google/protobuf/stubs/shared_ptr.h>
-#endif
 
 #include <google/protobuf/descriptor.h>
 #include <google/protobuf/message.h>
+#include <google/protobuf/pyext/message.h>
 
 namespace google {
 namespace protobuf {
 
 class Message;
 
-#ifdef _SHARED_PTR_H
-using std::shared_ptr;
-#else
-using internal::shared_ptr;
-#endif
-
 namespace python {
 
-struct CMessage;
 struct CMessageClass;
 
 // This struct is used directly for ScalarMap, and is the base class of
@@ -66,7 +57,7 @@ struct MapContainer {
   // proto tree.  Every Python MapContainer holds a
   // reference to it in order to keep it alive as long as there's a
   // Python object that references any part of the tree.
-  shared_ptr<Message> owner;
+  CMessage::OwnerRef owner;
 
   // Pointer to the C++ Message that contains this container.  The
   // MapContainer does not own this pointer.
@@ -99,9 +90,7 @@ struct MapContainer {
   int Release();
 
   // Set the owner field of self and any children of self.
-  void SetOwner(const shared_ptr<Message>& new_owner) {
-    owner = new_owner;
-  }
+  void SetOwner(const CMessage::OwnerRef& new_owner) { owner = new_owner; }
 };
 
 struct MessageMapContainer : public MapContainer {

python/google/protobuf/pyext/message.cc

@@ -35,9 +35,6 @@
 
 #include <map>
 #include <memory>
-#ifndef _SHARED_PTR_H
-#include <google/protobuf/stubs/shared_ptr.h>
-#endif
 #include <string>
 #include <vector>
 #include <structmember.h>  // A Python header file.
@@ -658,7 +655,7 @@ bool CheckAndGetInteger(PyObject* arg, T* value) {
       // Unlike PyLong_AsLongLong, PyLong_AsUnsignedLongLong is very
       // picky about the exact type.
       PyObject* casted = PyNumber_Long(arg);
-      if (GOOGLE_PREDICT_FALSE(casted == NULL)) {
+      if (GOOGLE_PREDICT_FALSE(casted == nullptr)) {
         // Propagate existing error.
         return false;
         }
@@ -683,7 +680,7 @@ bool CheckAndGetInteger(PyObject* arg, T* value) {
       // Valid subclasses of numbers.Integral should have a __long__() method
       // so fall back to that.
       PyObject* casted = PyNumber_Long(arg);
-      if (GOOGLE_PREDICT_FALSE(casted == NULL)) {
+      if (GOOGLE_PREDICT_FALSE(casted == nullptr)) {
         // Propagate existing error.
         return false;
         }
@@ -830,7 +827,8 @@ bool CheckAndSetString(
   return true;
 }
 
-PyObject* ToStringObject(const FieldDescriptor* descriptor, string value) {
+PyObject* ToStringObject(const FieldDescriptor* descriptor,
+                         const string& value) {
   if (descriptor->type() != FieldDescriptor::TYPE_STRING) {
     return PyBytes_FromStringAndSize(value.c_str(), value.length());
   }
@@ -1318,6 +1316,8 @@ CMessage* NewEmptyMessage(CMessageClass* type) {
     return NULL;
   }
 
+  // Use "placement new" syntax to initialize the C++ object.
+  new (&self->owner) CMessage::OwnerRef(NULL);
   self->message = NULL;
   self->parent = NULL;
   self->parent_field_descriptor = NULL;
@@ -1414,7 +1414,7 @@ static void Dealloc(CMessage* self) {
 
   Py_CLEAR(self->extensions);
   Py_CLEAR(self->composite_fields);
-  self->owner.reset();
+  self->owner.~ThreadUnsafeSharedPtr<Message>();
   Py_TYPE(self)->tp_free(reinterpret_cast<PyObject*>(self));
 }
 
@@ -1616,9 +1616,10 @@ PyObject* HasExtension(CMessage* self, PyObject* extension) {
 // * Clear the weak references from the released container to the
 //   parent.
 
-struct SetOwnerVisitor : public ChildVisitor {
+class SetOwnerVisitor : public ChildVisitor {
+ public:
   // new_owner must outlive this object.
-  explicit SetOwnerVisitor(const shared_ptr<Message>& new_owner)
+  explicit SetOwnerVisitor(const CMessage::OwnerRef& new_owner)
       : new_owner_(new_owner) {}
 
   int VisitRepeatedCompositeContainer(RepeatedCompositeContainer* container) {
@@ -1642,11 +1643,11 @@ struct SetOwnerVisitor : public ChildVisitor {
   }
 
  private:
-  const shared_ptr<Message>& new_owner_;
+  const CMessage::OwnerRef& new_owner_;
 };
 
 // Change the owner of this CMessage and all its children, recursively.
-int SetOwner(CMessage* self, const shared_ptr<Message>& new_owner) {
+int SetOwner(CMessage* self, const CMessage::OwnerRef& new_owner) {
   self->owner = new_owner;
   if (ForEachCompositeField(self, SetOwnerVisitor(new_owner)) == -1)
     return -1;
@@ -1679,7 +1680,7 @@ int ReleaseSubMessage(CMessage* self,
                       const FieldDescriptor* field_descriptor,
                       CMessage* child_cmessage) {
   // Release the Message
-  shared_ptr<Message> released_message(ReleaseMessage(
+  CMessage::OwnerRef released_message(ReleaseMessage(
       self, child_cmessage->message->GetDescriptor(), field_descriptor));
   child_cmessage->message = released_message.get();
   child_cmessage->owner.swap(released_message);
@@ -2329,7 +2330,9 @@ PyObject* InternalGetScalar(const Message* message,
       break;
     }
     case FieldDescriptor::CPPTYPE_STRING: {
-      string value = reflection->GetString(*message, field_descriptor);
+      string scratch;
+      const string& value =
+          reflection->GetStringReference(*message, field_descriptor, &scratch);
       result = ToStringObject(field_descriptor, value);
       break;
     }

python/google/protobuf/pyext/message.h

@@ -37,11 +37,11 @@
 #include <Python.h>
 
 #include <memory>
-#ifndef _SHARED_PTR_H
-#include <google/protobuf/stubs/shared_ptr.h>
-#endif
 #include <string>
 
+#include <google/protobuf/stubs/common.h>
+#include <google/protobuf/pyext/thread_unsafe_shared_ptr.h>
+
 namespace google {
 namespace protobuf {
 
@@ -52,13 +52,6 @@ class Descriptor;
 class DescriptorPool;
 class MessageFactory;
 
-#ifdef _SHARED_PTR_H
-using std::shared_ptr;
-using std::string;
-#else
-using internal::shared_ptr;
-#endif
-
 namespace python {
 
 struct ExtensionDict;
@@ -71,7 +64,9 @@ typedef struct CMessage {
   // proto tree.  Every Python CMessage holds a reference to it in
   // order to keep it alive as long as there's a Python object that
   // references any part of the tree.
-  shared_ptr<Message> owner;
+
+  typedef ThreadUnsafeSharedPtr<Message> OwnerRef;
+  OwnerRef owner;
 
   // Weak reference to a parent CMessage object. This is NULL for any top-level
   // message and is set for any child message (i.e. a child submessage or a
@@ -255,7 +250,7 @@ PyObject* FindInitializationErrors(CMessage* self);
 // Set the owner field of self and any children of self, recursively.
 // Used when self is being released and thus has a new owner (the
 // released Message.)
-int SetOwner(CMessage* self, const shared_ptr<Message>& new_owner);
+int SetOwner(CMessage* self, const CMessage::OwnerRef& new_owner);
 
 int AssureWritable(CMessage* self);
 
@@ -336,7 +331,8 @@ bool CheckAndSetString(
     const Reflection* reflection,
     bool append,
     int index);
-PyObject* ToStringObject(const FieldDescriptor* descriptor, string value);
+PyObject* ToStringObject(const FieldDescriptor* descriptor,
+                         const string& value);
 
 // Check if the passed field descriptor belongs to the given message.
 // If not, return false and set a Python exception (a KeyError)
@@ -347,6 +343,15 @@ extern PyObject* PickleError_class;
 
 bool InitProto2MessageModule(PyObject *m);
 
+#if LANG_CXX11
+// These are referenced by repeated_scalar_container, and must
+// be explicitly instantiated.
+extern template bool CheckAndGetInteger<int32>(PyObject*, int32*);
+extern template bool CheckAndGetInteger<int64>(PyObject*, int64*);
+extern template bool CheckAndGetInteger<uint32>(PyObject*, uint32*);
+extern template bool CheckAndGetInteger<uint64>(PyObject*, uint64*);
+#endif
+
 }  // namespace python
 }  // namespace protobuf
 

python/google/protobuf/pyext/message_factory.cc

@@ -100,7 +100,9 @@ PyObject* New(PyTypeObject* type, PyObject* args, PyObject* kwargs) {
       NewMessageFactory(type, reinterpret_cast<PyDescriptorPool*>(pool)));
 }
 
-static void Dealloc(PyMessageFactory* self) {
+static void Dealloc(PyObject* pself) {
+  PyMessageFactory* self = reinterpret_cast<PyMessageFactory*>(pself);
+
   // TODO(amauryfa): When the MessageFactory is not created from the
   // DescriptorPool this reference should be owned, not borrowed.
   // Py_CLEAR(self->pool);
@@ -111,7 +113,7 @@ static void Dealloc(PyMessageFactory* self) {
   }
   delete self->classes_by_descriptor;
   delete self->message_factory;
-  Py_TYPE(self)->tp_free(reinterpret_cast<PyObject*>(self));
+  Py_TYPE(self)->tp_free(pself);
 }
 
 // Add a message class to our database.
@@ -231,7 +233,7 @@ PyTypeObject PyMessageFactory_Type = {
     ".MessageFactory",                        // tp_name
     sizeof(PyMessageFactory),                 // tp_basicsize
     0,                                        // tp_itemsize
-    (destructor)message_factory::Dealloc,     // tp_dealloc
+    message_factory::Dealloc,                 // tp_dealloc
     0,                                        // tp_print
     0,                                        // tp_getattr
     0,                                        // tp_setattr

python/google/protobuf/pyext/repeated_composite_container.cc

@@ -34,9 +34,6 @@
 #include <google/protobuf/pyext/repeated_composite_container.h>
 
 #include <memory>
-#ifndef _SHARED_PTR_H
-#include <google/protobuf/stubs/shared_ptr.h>
-#endif
 
 #include <google/protobuf/stubs/logging.h>
 #include <google/protobuf/stubs/common.h>
@@ -81,7 +78,10 @@ namespace repeated_composite_container {
 // ---------------------------------------------------------------------
 // len()
 
-static Py_ssize_t Length(RepeatedCompositeContainer* self) {
+static Py_ssize_t Length(PyObject* pself) {
+  RepeatedCompositeContainer* self =
+      reinterpret_cast<RepeatedCompositeContainer*>(pself);
+
   Message* message = self->message;
   if (message != NULL) {
     return message->GetReflection()->FieldSize(*message,
@@ -102,7 +102,7 @@ static int UpdateChildMessages(RepeatedCompositeContainer* self) {
   // A MergeFrom on a parent message could have caused extra messages to be
   // added in the underlying protobuf so add them to our list. They can never
   // be removed in such a way so there's no need to worry about that.
-  Py_ssize_t message_length = Length(self);
+  Py_ssize_t message_length = Length(reinterpret_cast<PyObject*>(self));
   Py_ssize_t child_length = PyList_GET_SIZE(self->child_messages);
   Message* message = self->message;
   const Reflection* reflection = message->GetReflection();
@@ -191,6 +191,10 @@ PyObject* Add(RepeatedCompositeContainer* self,
     return AddToAttached(self, args, kwargs);
 }
 
+static PyObject* AddMethod(PyObject* self, PyObject* args, PyObject* kwargs) {
+  return Add(reinterpret_cast<RepeatedCompositeContainer*>(self), args, kwargs);
+}
+
 // ---------------------------------------------------------------------
 // extend()
 
@@ -226,6 +230,10 @@ PyObject* Extend(RepeatedCompositeContainer* self, PyObject* value) {
   Py_RETURN_NONE;
 }
 
+static PyObject* ExtendMethod(PyObject* self, PyObject* value) {
+  return Extend(reinterpret_cast<RepeatedCompositeContainer*>(self), value);
+}
+
 PyObject* MergeFrom(RepeatedCompositeContainer* self, PyObject* other) {
   if (UpdateChildMessages(self) < 0) {
     return NULL;
@@ -233,6 +241,10 @@ PyObject* MergeFrom(RepeatedCompositeContainer* self, PyObject* other) {
   return Extend(self, other);
 }
 
+static PyObject* MergeFromMethod(PyObject* self, PyObject* other) {
+  return MergeFrom(reinterpret_cast<RepeatedCompositeContainer*>(self), other);
+}
+
 PyObject* Subscript(RepeatedCompositeContainer* self, PyObject* slice) {
   if (UpdateChildMessages(self) < 0) {
     return NULL;
@@ -242,6 +254,10 @@ PyObject* Subscript(RepeatedCompositeContainer* self, PyObject* slice) {
   return PyObject_GetItem(self->child_messages, slice);
 }
 
+static PyObject* SubscriptMethod(PyObject* self, PyObject* slice) {
+  return Subscript(reinterpret_cast<RepeatedCompositeContainer*>(self), slice);
+}
+
 int AssignSubscript(RepeatedCompositeContainer* self,
                     PyObject* slice,
                     PyObject* value) {
@@ -265,7 +281,7 @@ int AssignSubscript(RepeatedCompositeContainer* self,
     Py_ssize_t from;
     Py_ssize_t to;
     Py_ssize_t step;
-    Py_ssize_t length = Length(self);
+    Py_ssize_t length = Length(reinterpret_cast<PyObject*>(self));
     Py_ssize_t slicelength;
     if (PySlice_Check(slice)) {
 #if PY_MAJOR_VERSION >= 3
@@ -290,7 +306,16 @@ int AssignSubscript(RepeatedCompositeContainer* self,
   return 0;
 }
 
-static PyObject* Remove(RepeatedCompositeContainer* self, PyObject* value) {
+static int AssignSubscriptMethod(PyObject* self, PyObject* slice,
+                                 PyObject* value) {
+  return AssignSubscript(reinterpret_cast<RepeatedCompositeContainer*>(self),
+                         slice, value);
+}
+
+static PyObject* Remove(PyObject* pself, PyObject* value) {
+  RepeatedCompositeContainer* self =
+      reinterpret_cast<RepeatedCompositeContainer*>(pself);
+
   if (UpdateChildMessages(self) < 0) {
     return NULL;
   }
@@ -305,9 +330,10 @@ static PyObject* Remove(RepeatedCompositeContainer* self, PyObject* value) {
   Py_RETURN_NONE;
 }
 
-static PyObject* RichCompare(RepeatedCompositeContainer* self,
-                             PyObject* other,
-                             int opid) {
+static PyObject* RichCompare(PyObject* pself, PyObject* other, int opid) {
+  RepeatedCompositeContainer* self =
+      reinterpret_cast<RepeatedCompositeContainer*>(pself);
+
   if (UpdateChildMessages(self) < 0) {
     return NULL;
   }
@@ -340,12 +366,13 @@ static PyObject* RichCompare(RepeatedCompositeContainer* self,
   }
 }
 
-static PyObject* ToStr(RepeatedCompositeContainer* self) {
+static PyObject* ToStr(PyObject* pself) {
   ScopedPyObjectPtr full_slice(PySlice_New(NULL, NULL, NULL));
   if (full_slice == NULL) {
     return NULL;
   }
-  ScopedPyObjectPtr list(Subscript(self, full_slice.get()));
+  ScopedPyObjectPtr list(Subscript(
+      reinterpret_cast<RepeatedCompositeContainer*>(pself), full_slice.get()));
   if (list == NULL) {
     return NULL;
   }
@@ -359,7 +386,7 @@ static void ReorderAttached(RepeatedCompositeContainer* self) {
   Message* message = self->message;
   const Reflection* reflection = message->GetReflection();
   const FieldDescriptor* descriptor = self->parent_field_descriptor;
-  const Py_ssize_t length = Length(self);
+  const Py_ssize_t length = Length(reinterpret_cast<PyObject*>(self));
 
   // Since Python protobuf objects are never arena-allocated, adding and
   // removing message pointers to the underlying array is just updating
@@ -390,9 +417,10 @@ static int SortPythonMessages(RepeatedCompositeContainer* self,
   return 0;
 }
 
-static PyObject* Sort(RepeatedCompositeContainer* self,
-                      PyObject* args,
-                      PyObject* kwds) {
+static PyObject* Sort(PyObject* pself, PyObject* args, PyObject* kwds) {
+  RepeatedCompositeContainer* self =
+      reinterpret_cast<RepeatedCompositeContainer*>(pself);
+
   // Support the old sort_function argument for backwards
   // compatibility.
   if (kwds != NULL) {
@@ -416,11 +444,14 @@ static PyObject* Sort(RepeatedCompositeContainer* self,
 
 // ---------------------------------------------------------------------
 
-static PyObject* Item(RepeatedCompositeContainer* self, Py_ssize_t index) {
+static PyObject* Item(PyObject* pself, Py_ssize_t index) {
+  RepeatedCompositeContainer* self =
+      reinterpret_cast<RepeatedCompositeContainer*>(pself);
+
   if (UpdateChildMessages(self) < 0) {
     return NULL;
   }
-  Py_ssize_t length = Length(self);
+  Py_ssize_t length = Length(pself);
   if (index < 0) {
     index = length + index;
   }
@@ -432,17 +463,17 @@ static PyObject* Item(RepeatedCompositeContainer* self, Py_ssize_t index) {
   return item;
 }
 
-static PyObject* Pop(RepeatedCompositeContainer* self,
-                     PyObject* args) {
+static PyObject* Pop(PyObject* pself, PyObject* args) {
+  RepeatedCompositeContainer* self =
+      reinterpret_cast<RepeatedCompositeContainer*>(pself);
+
   Py_ssize_t index = -1;
   if (!PyArg_ParseTuple(args, "|n", &index)) {
     return NULL;
   }
-  PyObject* item = Item(self, index);
+  PyObject* item = Item(pself, index);
   if (item == NULL) {
-    PyErr_Format(PyExc_IndexError,
-                 "list index (%zd) out of range",
-                 index);
+    PyErr_Format(PyExc_IndexError, "list index (%zd) out of range", index);
     return NULL;
   }
   ScopedPyObjectPtr py_index(PyLong_FromSsize_t(index));
@@ -460,7 +491,7 @@ void ReleaseLastTo(CMessage* parent,
   GOOGLE_CHECK_NOTNULL(field);
   GOOGLE_CHECK_NOTNULL(target);
 
-  shared_ptr<Message> released_message(
+  CMessage::OwnerRef released_message(
       parent->message->GetReflection()->ReleaseLast(parent->message, field));
   // TODO(tibell): Deal with proto1.
 
@@ -503,7 +534,10 @@ int Release(RepeatedCompositeContainer* self) {
   return 0;
 }
 
-PyObject* DeepCopy(RepeatedCompositeContainer* self, PyObject* arg) {
+PyObject* DeepCopy(PyObject* pself, PyObject* arg) {
+  RepeatedCompositeContainer* self =
+      reinterpret_cast<RepeatedCompositeContainer*>(pself);
+
   ScopedPyObjectPtr cloneObj(
       PyType_GenericAlloc(&RepeatedCompositeContainer_Type, 0));
   if (cloneObj == NULL) {
@@ -530,7 +564,7 @@ PyObject* DeepCopy(RepeatedCompositeContainer* self, PyObject* arg) {
 }
 
 int SetOwner(RepeatedCompositeContainer* self,
-             const shared_ptr<Message>& new_owner) {
+             const CMessage::OwnerRef& new_owner) {
   GOOGLE_CHECK_ATTACHED(self);
 
   self->owner = new_owner;
@@ -571,43 +605,46 @@ PyObject *NewContainer(
   return reinterpret_cast<PyObject*>(self);
 }
 
-static void Dealloc(RepeatedCompositeContainer* self) {
+static void Dealloc(PyObject* pself) {
+  RepeatedCompositeContainer* self =
+      reinterpret_cast<RepeatedCompositeContainer*>(pself);
+
   Py_CLEAR(self->child_messages);
   Py_CLEAR(self->child_message_class);
   // TODO(tibell): Do we need to call delete on these objects to make
   // sure their destructors are called?
   self->owner.reset();
 
-  Py_TYPE(self)->tp_free(reinterpret_cast<PyObject*>(self));
+  Py_TYPE(self)->tp_free(pself);
 }
 
 static PySequenceMethods SqMethods = {
-  (lenfunc)Length,        /* sq_length */
-  0, /* sq_concat */
-  0, /* sq_repeat */
-  (ssizeargfunc)Item /* sq_item */
+  Length,                 /* sq_length */
+  0,                      /* sq_concat */
+  0,                      /* sq_repeat */
+  Item                    /* sq_item */
 };
 
 static PyMappingMethods MpMethods = {
-  (lenfunc)Length,               /* mp_length */
-  (binaryfunc)Subscript,      /* mp_subscript */
-  (objobjargproc)AssignSubscript,/* mp_ass_subscript */
+  Length,                 /* mp_length */
+  SubscriptMethod,        /* mp_subscript */
+  AssignSubscriptMethod,  /* mp_ass_subscript */
 };
 
 static PyMethodDef Methods[] = {
-  { "__deepcopy__", (PyCFunction)DeepCopy, METH_VARARGS,
+  { "__deepcopy__", DeepCopy, METH_VARARGS,
     "Makes a deep copy of the class." },
-  { "add", (PyCFunction) Add, METH_VARARGS | METH_KEYWORDS,
+  { "add", (PyCFunction)AddMethod, METH_VARARGS | METH_KEYWORDS,
     "Adds an object to the repeated container." },
-  { "extend", (PyCFunction) Extend, METH_O,
+  { "extend", ExtendMethod, METH_O,
     "Adds objects to the repeated container." },
-  { "pop", (PyCFunction)Pop, METH_VARARGS,
+  { "pop", Pop, METH_VARARGS,
     "Removes an object from the repeated container and returns it." },
-  { "remove", (PyCFunction) Remove, METH_O,
+  { "remove", Remove, METH_O,
     "Removes an object from the repeated container." },
-  { "sort", (PyCFunction) Sort, METH_VARARGS | METH_KEYWORDS,
+  { "sort", (PyCFunction)Sort, METH_VARARGS | METH_KEYWORDS,
     "Sorts the repeated container." },
-  { "MergeFrom", (PyCFunction) MergeFrom, METH_O,
+  { "MergeFrom", MergeFromMethod, METH_O,
     "Adds objects to the repeated container." },
   { NULL, NULL }
 };
@@ -619,12 +656,12 @@ PyTypeObject RepeatedCompositeContainer_Type = {
   FULL_MODULE_NAME ".RepeatedCompositeContainer",  // tp_name
   sizeof(RepeatedCompositeContainer),  // tp_basicsize
   0,                                   //  tp_itemsize
-  (destructor)repeated_composite_container::Dealloc,  //  tp_dealloc
+  repeated_composite_container::Dealloc,  //  tp_dealloc
   0,                                   //  tp_print
   0,                                   //  tp_getattr
   0,                                   //  tp_setattr
   0,                                   //  tp_compare
-  (reprfunc)repeated_composite_container::ToStr,      //  tp_repr
+  repeated_composite_container::ToStr,      //  tp_repr
   0,                                   //  tp_as_number
   &repeated_composite_container::SqMethods,   //  tp_as_sequence
   &repeated_composite_container::MpMethods,   //  tp_as_mapping
@@ -638,7 +675,7 @@ PyTypeObject RepeatedCompositeContainer_Type = {
   "A Repeated scalar container",       //  tp_doc
   0,                                   //  tp_traverse
   0,                                   //  tp_clear
-  (richcmpfunc)repeated_composite_container::RichCompare,  //  tp_richcompare
+  repeated_composite_container::RichCompare,  //  tp_richcompare
   0,                                   //  tp_weaklistoffset
   0,                                   //  tp_iter
   0,                                   //  tp_iternext

python/google/protobuf/pyext/repeated_composite_container.h

@@ -37,27 +37,19 @@
 #include <Python.h>
 
 #include <memory>
-#ifndef _SHARED_PTR_H
-#include <google/protobuf/stubs/shared_ptr.h>
-#endif
 #include <string>
 #include <vector>
 
+#include <google/protobuf/pyext/message.h>
+
 namespace google {
 namespace protobuf {
 
 class FieldDescriptor;
 class Message;
 
-#ifdef _SHARED_PTR_H
-using std::shared_ptr;
-#else
-using internal::shared_ptr;
-#endif
-
 namespace python {
 
-struct CMessage;
 struct CMessageClass;
 
 // A RepeatedCompositeContainer can be in one of two states: attached
@@ -77,7 +69,7 @@ typedef struct RepeatedCompositeContainer {
   // proto tree.  Every Python RepeatedCompositeContainer holds a
   // reference to it in order to keep it alive as long as there's a
   // Python object that references any part of the tree.
-  shared_ptr<Message> owner;
+  CMessage::OwnerRef owner;
 
   // Weak reference to parent object. May be NULL. Used to make sure
   // the parent is writable before modifying the
@@ -148,11 +140,6 @@ int AssignSubscript(RepeatedCompositeContainer* self,
                     PyObject* slice,
                     PyObject* value);
 
-// Releases the messages in the container to the given message.
-//
-// Returns 0 on success, -1 on failure.
-int ReleaseToMessage(RepeatedCompositeContainer* self, Message* new_message);
-
 // Releases the messages in the container to a new message.
 //
 // Returns 0 on success, -1 on failure.
@@ -160,7 +147,7 @@ int Release(RepeatedCompositeContainer* self);
 
 // Returns 0 on success, -1 on failure.
 int SetOwner(RepeatedCompositeContainer* self,
-             const shared_ptr<Message>& new_owner);
+             const CMessage::OwnerRef& new_owner);
 
 // Removes the last element of the repeated message field 'field' on
 // the Message 'parent', and transfers the ownership of the released

python/google/protobuf/pyext/repeated_scalar_container.cc

@@ -34,9 +34,6 @@
 #include <google/protobuf/pyext/repeated_scalar_container.h>
 
 #include <memory>
-#ifndef _SHARED_PTR_H
-#include <google/protobuf/stubs/shared_ptr.h>
-#endif
 
 #include <google/protobuf/stubs/common.h>
 #include <google/protobuf/stubs/logging.h>
@@ -77,15 +74,18 @@ static int InternalAssignRepeatedField(
   return 0;
 }
 
-static Py_ssize_t Len(RepeatedScalarContainer* self) {
+static Py_ssize_t Len(PyObject* pself) {
+  RepeatedScalarContainer* self =
+      reinterpret_cast<RepeatedScalarContainer*>(pself);
   Message* message = self->message;
   return message->GetReflection()->FieldSize(*message,
                                              self->parent_field_descriptor);
 }
 
-static int AssignItem(RepeatedScalarContainer* self,
-                      Py_ssize_t index,
-                      PyObject* arg) {
+static int AssignItem(PyObject* pself, Py_ssize_t index, PyObject* arg) {
+  RepeatedScalarContainer* self =
+      reinterpret_cast<RepeatedScalarContainer*>(pself);
+
   cmessage::AssureWritable(self->parent);
   Message* message = self->message;
   const FieldDescriptor* field_descriptor = self->parent_field_descriptor;
@@ -188,7 +188,10 @@ static int AssignItem(RepeatedScalarContainer* self,
   return 0;
 }
 
-static PyObject* Item(RepeatedScalarContainer* self, Py_ssize_t index) {
+static PyObject* Item(PyObject* pself, Py_ssize_t index) {
+  RepeatedScalarContainer* self =
+      reinterpret_cast<RepeatedScalarContainer*>(pself);
+
   Message* message = self->message;
   const FieldDescriptor* field_descriptor = self->parent_field_descriptor;
   const Reflection* reflection = message->GetReflection();
@@ -256,8 +259,9 @@ static PyObject* Item(RepeatedScalarContainer* self, Py_ssize_t index) {
       break;
     }
     case FieldDescriptor::CPPTYPE_STRING: {
-      string value = reflection->GetRepeatedString(
-          *message, field_descriptor, index);
+      string scratch;
+      const string& value = reflection->GetRepeatedStringReference(
+          *message, field_descriptor, index, &scratch);
       result = ToStringObject(field_descriptor, value);
       break;
     }
@@ -271,7 +275,7 @@ static PyObject* Item(RepeatedScalarContainer* self, Py_ssize_t index) {
   return result;
 }
 
-static PyObject* Subscript(RepeatedScalarContainer* self, PyObject* slice) {
+static PyObject* Subscript(PyObject* pself, PyObject* slice) {
   Py_ssize_t from;
   Py_ssize_t to;
   Py_ssize_t step;
@@ -286,14 +290,13 @@ static PyObject* Subscript(RepeatedScalarContainer* self, PyObject* slice) {
   if (PyLong_Check(slice)) {
     from = to = PyLong_AsLong(slice);
   } else if (PySlice_Check(slice)) {
-    length = Len(self);
+    length = Len(pself);
 #if PY_MAJOR_VERSION >= 3
     if (PySlice_GetIndicesEx(slice,
                              length, &from, &to, &step, &slicelength) == -1) {
 #else
     if (PySlice_GetIndicesEx(reinterpret_cast<PySliceObject*>(slice),
                              length, &from, &to, &step, &slicelength) == -1) {
-
 #endif
       return NULL;
     }
@@ -304,7 +307,7 @@ static PyObject* Subscript(RepeatedScalarContainer* self, PyObject* slice) {
   }
 
   if (!return_list) {
-    return Item(self, from);
+    return Item(pself, from);
   }
 
   PyObject* list = PyList_New(0);
@@ -319,7 +322,7 @@ static PyObject* Subscript(RepeatedScalarContainer* self, PyObject* slice) {
       if (index < 0 || index >= length) {
         break;
       }
-      ScopedPyObjectPtr s(Item(self, index));
+      ScopedPyObjectPtr s(Item(pself, index));
       PyList_Append(list, s.get());
     }
   } else {
@@ -330,7 +333,7 @@ static PyObject* Subscript(RepeatedScalarContainer* self, PyObject* slice) {
       if (index < 0 || index >= length) {
         break;
       }
-      ScopedPyObjectPtr s(Item(self, index));
+      ScopedPyObjectPtr s(Item(pself, index));
       PyList_Append(list, s.get());
     }
   }
@@ -417,9 +420,14 @@ PyObject* Append(RepeatedScalarContainer* self, PyObject* item) {
   Py_RETURN_NONE;
 }
 
-static int AssSubscript(RepeatedScalarContainer* self,
-                        PyObject* slice,
-                        PyObject* value) {
+static PyObject* AppendMethod(PyObject* self, PyObject* item) {
+  return Append(reinterpret_cast<RepeatedScalarContainer*>(self), item);
+}
+
+static int AssSubscript(PyObject* pself, PyObject* slice, PyObject* value) {
+  RepeatedScalarContainer* self =
+      reinterpret_cast<RepeatedScalarContainer*>(pself);
+
   Py_ssize_t from;
   Py_ssize_t to;
   Py_ssize_t step;
@@ -435,7 +443,7 @@ static int AssSubscript(RepeatedScalarContainer* self,
 #if PY_MAJOR_VERSION < 3
   if (PyInt_Check(slice)) {
     from = to = PyInt_AsLong(slice);
-  } else
+  } else  // NOLINT
 #endif
   if (PyLong_Check(slice)) {
     from = to = PyLong_AsLong(slice);
@@ -463,14 +471,14 @@ static int AssSubscript(RepeatedScalarContainer* self,
   }
 
   if (!create_list) {
-    return AssignItem(self, from, value);
+    return AssignItem(pself, from, value);
   }
 
   ScopedPyObjectPtr full_slice(PySlice_New(NULL, NULL, NULL));
   if (full_slice == NULL) {
     return -1;
   }
-  ScopedPyObjectPtr new_list(Subscript(self, full_slice.get()));
+  ScopedPyObjectPtr new_list(Subscript(pself, full_slice.get()));
   if (new_list == NULL) {
     return -1;
   }
@@ -509,14 +517,17 @@ PyObject* Extend(RepeatedScalarContainer* self, PyObject* value) {
   Py_RETURN_NONE;
 }
 
-static PyObject* Insert(RepeatedScalarContainer* self, PyObject* args) {
+static PyObject* Insert(PyObject* pself, PyObject* args) {
+  RepeatedScalarContainer* self =
+      reinterpret_cast<RepeatedScalarContainer*>(pself);
+
   Py_ssize_t index;
   PyObject* value;
   if (!PyArg_ParseTuple(args, "lO", &index, &value)) {
     return NULL;
   }
   ScopedPyObjectPtr full_slice(PySlice_New(NULL, NULL, NULL));
-  ScopedPyObjectPtr new_list(Subscript(self, full_slice.get()));
+  ScopedPyObjectPtr new_list(Subscript(pself, full_slice.get()));
   if (PyList_Insert(new_list.get(), index, value) < 0) {
     return NULL;
   }
@@ -527,10 +538,10 @@ static PyObject* Insert(RepeatedScalarContainer* self, PyObject* args) {
   Py_RETURN_NONE;
 }
 
-static PyObject* Remove(RepeatedScalarContainer* self, PyObject* value) {
+static PyObject* Remove(PyObject* pself, PyObject* value) {
   Py_ssize_t match_index = -1;
-  for (Py_ssize_t i = 0; i < Len(self); ++i) {
-    ScopedPyObjectPtr elem(Item(self, i));
+  for (Py_ssize_t i = 0; i < Len(pself); ++i) {
+    ScopedPyObjectPtr elem(Item(pself, i));
     if (PyObject_RichCompareBool(elem.get(), value, Py_EQ)) {
       match_index = i;
       break;
@@ -540,15 +551,17 @@ static PyObject* Remove(RepeatedScalarContainer* self, PyObject* value) {
     PyErr_SetString(PyExc_ValueError, "remove(x): x not in container");
     return NULL;
   }
-  if (AssignItem(self, match_index, NULL) < 0) {
+  if (AssignItem(pself, match_index, NULL) < 0) {
     return NULL;
   }
   Py_RETURN_NONE;
 }
 
-static PyObject* RichCompare(RepeatedScalarContainer* self,
-                             PyObject* other,
-                             int opid) {
+static PyObject* ExtendMethod(PyObject* self, PyObject* value) {
+  return Extend(reinterpret_cast<RepeatedScalarContainer*>(self), value);
+}
+
+static PyObject* RichCompare(PyObject* pself, PyObject* other, int opid) {
   if (opid != Py_EQ && opid != Py_NE) {
     Py_INCREF(Py_NotImplemented);
     return Py_NotImplemented;
@@ -565,28 +578,25 @@ static PyObject* RichCompare(RepeatedScalarContainer* self,
 
   ScopedPyObjectPtr other_list_deleter;
   if (PyObject_TypeCheck(other, &RepeatedScalarContainer_Type)) {
-    other_list_deleter.reset(Subscript(
-        reinterpret_cast<RepeatedScalarContainer*>(other), full_slice.get()));
+    other_list_deleter.reset(Subscript(other, full_slice.get()));
     other = other_list_deleter.get();
   }
 
-  ScopedPyObjectPtr list(Subscript(self, full_slice.get()));
+  ScopedPyObjectPtr list(Subscript(pself, full_slice.get()));
   if (list == NULL) {
     return NULL;
   }
   return PyObject_RichCompare(list.get(), other, opid);
 }
 
-PyObject* Reduce(RepeatedScalarContainer* unused_self) {
+PyObject* Reduce(PyObject* unused_self, PyObject* unused_other) {
   PyErr_Format(
       PickleError_class,
       "can't pickle repeated message fields, convert to list first");
   return NULL;
 }
 
-static PyObject* Sort(RepeatedScalarContainer* self,
-                      PyObject* args,
-                      PyObject* kwds) {
+static PyObject* Sort(PyObject* pself, PyObject* args, PyObject* kwds) {
   // Support the old sort_function argument for backwards
   // compatibility.
   if (kwds != NULL) {
@@ -605,7 +615,7 @@ static PyObject* Sort(RepeatedScalarContainer* self,
   if (full_slice == NULL) {
     return NULL;
   }
-  ScopedPyObjectPtr list(Subscript(self, full_slice.get()));
+  ScopedPyObjectPtr list(Subscript(pself, full_slice.get()));
   if (list == NULL) {
     return NULL;
   }
@@ -617,38 +627,36 @@ static PyObject* Sort(RepeatedScalarContainer* self,
   if (res == NULL) {
     return NULL;
   }
-  int ret = InternalAssignRepeatedField(self, list.get());
+  int ret = InternalAssignRepeatedField(
+      reinterpret_cast<RepeatedScalarContainer*>(pself), list.get());
   if (ret < 0) {
     return NULL;
   }
   Py_RETURN_NONE;
 }
 
-static PyObject* Pop(RepeatedScalarContainer* self,
-                     PyObject* args) {
+static PyObject* Pop(PyObject* pself, PyObject* args) {
   Py_ssize_t index = -1;
   if (!PyArg_ParseTuple(args, "|n", &index)) {
     return NULL;
   }
-  PyObject* item = Item(self, index);
+  PyObject* item = Item(pself, index);
   if (item == NULL) {
-    PyErr_Format(PyExc_IndexError,
-                 "list index (%zd) out of range",
-                 index);
+    PyErr_Format(PyExc_IndexError, "list index (%zd) out of range", index);
     return NULL;
   }
-  if (AssignItem(self, index, NULL) < 0) {
+  if (AssignItem(pself, index, NULL) < 0) {
     return NULL;
   }
   return item;
 }
 
-static PyObject* ToStr(RepeatedScalarContainer* self) {
+static PyObject* ToStr(PyObject* pself) {
   ScopedPyObjectPtr full_slice(PySlice_New(NULL, NULL, NULL));
   if (full_slice == NULL) {
     return NULL;
   }
-  ScopedPyObjectPtr list(Subscript(self, full_slice.get()));
+  ScopedPyObjectPtr list(Subscript(pself, full_slice.get()));
   if (list == NULL) {
     return NULL;
   }
@@ -687,7 +695,8 @@ static int InitializeAndCopyToParentContainer(
   if (full_slice == NULL) {
     return -1;
   }
-  ScopedPyObjectPtr values(Subscript(from, full_slice.get()));
+  ScopedPyObjectPtr values(
+      Subscript(reinterpret_cast<PyObject*>(from), full_slice.get()));
   if (values == NULL) {
     return -1;
   }
@@ -706,7 +715,10 @@ int Release(RepeatedScalarContainer* self) {
   return InitializeAndCopyToParentContainer(self, self);
 }
 
-PyObject* DeepCopy(RepeatedScalarContainer* self, PyObject* arg) {
+PyObject* DeepCopy(PyObject* pself, PyObject* arg) {
+  RepeatedScalarContainer* self =
+      reinterpret_cast<RepeatedScalarContainer*>(pself);
+
   RepeatedScalarContainer* clone = reinterpret_cast<RepeatedScalarContainer*>(
       PyType_GenericAlloc(&RepeatedScalarContainer_Type, 0));
   if (clone == NULL) {
@@ -720,45 +732,47 @@ PyObject* DeepCopy(RepeatedScalarContainer* self, PyObject* arg) {
   return reinterpret_cast<PyObject*>(clone);
 }
 
-static void Dealloc(RepeatedScalarContainer* self) {
+static void Dealloc(PyObject* pself) {
+  RepeatedScalarContainer* self =
+      reinterpret_cast<RepeatedScalarContainer*>(pself);
   self->owner.reset();
-  Py_TYPE(self)->tp_free(reinterpret_cast<PyObject*>(self));
+  Py_TYPE(self)->tp_free(pself);
 }
 
 void SetOwner(RepeatedScalarContainer* self,
-              const shared_ptr<Message>& new_owner) {
+              const CMessage::OwnerRef& new_owner) {
   self->owner = new_owner;
 }
 
 static PySequenceMethods SqMethods = {
-  (lenfunc)Len,           /* sq_length */
-  0, /* sq_concat */
-  0, /* sq_repeat */
-  (ssizeargfunc)Item, /* sq_item */
-  0, /* sq_slice */
-  (ssizeobjargproc)AssignItem /* sq_ass_item */
+  Len,        /* sq_length */
+  0,          /* sq_concat */
+  0,          /* sq_repeat */
+  Item,       /* sq_item */
+  0,          /* sq_slice */
+  AssignItem  /* sq_ass_item */
 };
 
 static PyMappingMethods MpMethods = {
-  (lenfunc)Len,               /* mp_length */
-  (binaryfunc)Subscript,      /* mp_subscript */
-  (objobjargproc)AssSubscript, /* mp_ass_subscript */
+  Len,               /* mp_length */
+  Subscript,      /* mp_subscript */
+  AssSubscript, /* mp_ass_subscript */
 };
 
 static PyMethodDef Methods[] = {
-  { "__deepcopy__", (PyCFunction)DeepCopy, METH_VARARGS,
+  { "__deepcopy__", DeepCopy, METH_VARARGS,
     "Makes a deep copy of the class." },
-  { "__reduce__", (PyCFunction)Reduce, METH_NOARGS,
+  { "__reduce__", Reduce, METH_NOARGS,
     "Outputs picklable representation of the repeated field." },
-  { "append", (PyCFunction)Append, METH_O,
+  { "append", AppendMethod, METH_O,
     "Appends an object to the repeated container." },
-  { "extend", (PyCFunction)Extend, METH_O,
-    "Appends objects to the repeated container." },
-  { "insert", (PyCFunction)Insert, METH_VARARGS,
+  { "extend", ExtendMethod, METH_O,
     "Appends objects to the repeated container." },
-  { "pop", (PyCFunction)Pop, METH_VARARGS,
+  { "insert", Insert, METH_VARARGS,
+    "Inserts an object at the specified position in the container." },
+  { "pop", Pop, METH_VARARGS,
     "Removes an object from the repeated container and returns it." },
-  { "remove", (PyCFunction)Remove, METH_O,
+  { "remove", Remove, METH_O,
     "Removes an object from the repeated container." },
   { "sort", (PyCFunction)Sort, METH_VARARGS | METH_KEYWORDS,
     "Sorts the repeated container."},
@@ -772,12 +786,12 @@ PyTypeObject RepeatedScalarContainer_Type = {
   FULL_MODULE_NAME ".RepeatedScalarContainer",  // tp_name
   sizeof(RepeatedScalarContainer),     // tp_basicsize
   0,                                   //  tp_itemsize
-  (destructor)repeated_scalar_container::Dealloc,  //  tp_dealloc
+  repeated_scalar_container::Dealloc,  //  tp_dealloc
   0,                                   //  tp_print
   0,                                   //  tp_getattr
   0,                                   //  tp_setattr
   0,                                   //  tp_compare
-  (reprfunc)repeated_scalar_container::ToStr,      //  tp_repr
+  repeated_scalar_container::ToStr,    //  tp_repr
   0,                                   //  tp_as_number
   &repeated_scalar_container::SqMethods,   //  tp_as_sequence
   &repeated_scalar_container::MpMethods,   //  tp_as_mapping
@@ -791,7 +805,7 @@ PyTypeObject RepeatedScalarContainer_Type = {
   "A Repeated scalar container",       //  tp_doc
   0,                                   //  tp_traverse
   0,                                   //  tp_clear
-  (richcmpfunc)repeated_scalar_container::RichCompare,  //  tp_richcompare
+  repeated_scalar_container::RichCompare,  //  tp_richcompare
   0,                                   //  tp_weaklistoffset
   0,                                   //  tp_iter
   0,                                   //  tp_iternext

python/google/protobuf/pyext/repeated_scalar_container.h

@@ -37,27 +37,14 @@
 #include <Python.h>
 
 #include <memory>
-#ifndef _SHARED_PTR_H
-#include <google/protobuf/stubs/shared_ptr.h>
-#endif
 
 #include <google/protobuf/descriptor.h>
+#include <google/protobuf/pyext/message.h>
 
 namespace google {
 namespace protobuf {
-
-class Message;
-
-#ifdef _SHARED_PTR_H
-using std::shared_ptr;
-#else
-using internal::shared_ptr;
-#endif
-
 namespace python {
 
-struct CMessage;
-
 typedef struct RepeatedScalarContainer {
   PyObject_HEAD;
 
@@ -65,7 +52,7 @@ typedef struct RepeatedScalarContainer {
   // proto tree.  Every Python RepeatedScalarContainer holds a
   // reference to it in order to keep it alive as long as there's a
   // Python object that references any part of the tree.
-  shared_ptr<Message> owner;
+  CMessage::OwnerRef owner;
 
   // Pointer to the C++ Message that contains this container.  The
   // RepeatedScalarContainer does not own this pointer.
@@ -112,7 +99,7 @@ PyObject* Extend(RepeatedScalarContainer* self, PyObject* value);
 
 // Set the owner field of self and any children of self.
 void SetOwner(RepeatedScalarContainer* self,
-              const shared_ptr<Message>& new_owner);
+              const CMessage::OwnerRef& new_owner);
 
 }  // namespace repeated_scalar_container
 }  // namespace python

python/google/protobuf/pyext/thread_unsafe_shared_ptr.h

@@ -0,0 +1,104 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// ThreadUnsafeSharedPtr<T> is the same as shared_ptr<T> without the locking
+// overhread (and thread-safety).
+#ifndef GOOGLE_PROTOBUF_PYTHON_CPP_THREAD_UNSAFE_SHARED_PTR_H__
+#define GOOGLE_PROTOBUF_PYTHON_CPP_THREAD_UNSAFE_SHARED_PTR_H__
+
+#include <algorithm>
+#include <utility>
+
+#include <google/protobuf/stubs/logging.h>
+#include <google/protobuf/stubs/common.h>
+
+namespace google {
+namespace protobuf {
+namespace python {
+
+template <typename T>
+class ThreadUnsafeSharedPtr {
+ public:
+  // Takes ownership.
+  explicit ThreadUnsafeSharedPtr(T* ptr)
+      : ptr_(ptr), refcount_(ptr ? new RefcountT(1) : nullptr) {
+  }
+
+  ThreadUnsafeSharedPtr(const ThreadUnsafeSharedPtr& other)
+      : ThreadUnsafeSharedPtr(nullptr) {
+    *this = other;
+  }
+
+  ThreadUnsafeSharedPtr& operator=(const ThreadUnsafeSharedPtr& other) {
+    if (other.refcount_ == refcount_) {
+      return *this;
+    }
+    this->~ThreadUnsafeSharedPtr();
+    ptr_ = other.ptr_;
+    refcount_ = other.refcount_;
+    if (refcount_) {
+      ++*refcount_;
+    }
+    return *this;
+  }
+
+  ~ThreadUnsafeSharedPtr() {
+    if (refcount_ == nullptr) {
+      GOOGLE_DCHECK(ptr_ == nullptr);
+      return;
+    }
+    if (--*refcount_ == 0) {
+      delete refcount_;
+      delete ptr_;
+    }
+  }
+
+  void reset(T* ptr = nullptr) { *this = ThreadUnsafeSharedPtr(ptr); }
+
+  T* get() { return ptr_; }
+  const T* get() const { return ptr_; }
+
+  void swap(ThreadUnsafeSharedPtr& other) {
+    using std::swap;
+    swap(ptr_, other.ptr_);
+    swap(refcount_, other.refcount_);
+  }
+
+ private:
+  typedef int RefcountT;
+  T* ptr_;
+  RefcountT* refcount_;
+};
+
+}  // namespace python
+}  // namespace protobuf
+
+}  // namespace google
+#endif  // GOOGLE_PROTOBUF_PYTHON_CPP_THREAD_UNSAFE_SHARED_PTR_H__

python/google/protobuf/text_format.py

@@ -141,9 +141,11 @@ def MessageToString(message,
     as_one_line: Don't introduce newlines between fields.
     pointy_brackets: If True, use angle brackets instead of curly braces for
       nesting.
-    use_index_order: If True, print fields of a proto message using the order
-      defined in source code instead of the field number. By default, use the
-      field number order.
+    use_index_order: If True, fields of a proto message will be printed using
+      the order defined in source code instead of the field number, extensions
+      will be printed at the end of the message and their relative order is
+      determined by the extension number. By default, use the field number
+      order.
     float_format: If set, use this to specify floating point number formatting
       (per the "Format Specification Mini-Language"); otherwise, str() is used.
     use_field_number: If True, print field numbers instead of names.
@@ -336,11 +338,12 @@ class _Printer(object):
       return
     fields = message.ListFields()
     if self.use_index_order:
-      fields.sort(key=lambda x: x[0].index)
+      fields.sort(
+          key=lambda x: x[0].number if x[0].is_extension else x[0].index)
     for field, value in fields:
       if _IsMapEntry(field):
         for key in sorted(value):
-          # This is slow for maps with submessage entires because it copies the
+          # This is slow for maps with submessage entries because it copies the
           # entire tree.  Unfortunately this would take significant refactoring
           # of this file to work around.
           #
@@ -645,6 +648,30 @@ class _Parser(object):
       ParseError: In case of text parsing problems.
     """
     message_descriptor = message.DESCRIPTOR
+    if (message_descriptor.full_name == _ANY_FULL_TYPE_NAME and
+        tokenizer.TryConsume('[')):
+      type_url_prefix, packed_type_name = self._ConsumeAnyTypeUrl(tokenizer)
+      tokenizer.Consume(']')
+      tokenizer.TryConsume(':')
+      if tokenizer.TryConsume('<'):
+        expanded_any_end_token = '>'
+      else:
+        tokenizer.Consume('{')
+        expanded_any_end_token = '}'
+      expanded_any_sub_message = _BuildMessageFromTypeName(packed_type_name,
+                                                           self.descriptor_pool)
+      if not expanded_any_sub_message:
+        raise ParseError('Type %s not found in descriptor pool' %
+                         packed_type_name)
+      while not tokenizer.TryConsume(expanded_any_end_token):
+        if tokenizer.AtEnd():
+          raise tokenizer.ParseErrorPreviousToken('Expected "%s".' %
+                                                  (expanded_any_end_token,))
+        self._MergeField(tokenizer, expanded_any_sub_message)
+      message.Pack(expanded_any_sub_message,
+                   type_url_prefix=type_url_prefix)
+      return
+
     if tokenizer.TryConsume('['):
       name = [tokenizer.ConsumeIdentifier()]
       while tokenizer.TryConsume('.'):
@@ -725,11 +752,12 @@ class _Parser(object):
       if (field.label == descriptor.FieldDescriptor.LABEL_REPEATED and
           tokenizer.TryConsume('[')):
         # Short repeated format, e.g. "foo: [1, 2, 3]"
-        while True:
-          merger(tokenizer, message, field)
-          if tokenizer.TryConsume(']'):
-            break
-          tokenizer.Consume(',')
+        if not tokenizer.TryConsume(']'):
+          while True:
+            merger(tokenizer, message, field)
+            if tokenizer.TryConsume(']'):
+              break
+            tokenizer.Consume(',')
 
       else:
         merger(tokenizer, message, field)
@@ -777,33 +805,7 @@ class _Parser(object):
       tokenizer.Consume('{')
       end_token = '}'
 
-    if (field.message_type.full_name == _ANY_FULL_TYPE_NAME and
-        tokenizer.TryConsume('[')):
-      type_url_prefix, packed_type_name = self._ConsumeAnyTypeUrl(tokenizer)
-      tokenizer.Consume(']')
-      tokenizer.TryConsume(':')
-      if tokenizer.TryConsume('<'):
-        expanded_any_end_token = '>'
-      else:
-        tokenizer.Consume('{')
-        expanded_any_end_token = '}'
-      expanded_any_sub_message = _BuildMessageFromTypeName(packed_type_name,
-                                                           self.descriptor_pool)
-      if not expanded_any_sub_message:
-        raise ParseError('Type %s not found in descriptor pool' %
-                         packed_type_name)
-      while not tokenizer.TryConsume(expanded_any_end_token):
-        if tokenizer.AtEnd():
-          raise tokenizer.ParseErrorPreviousToken('Expected "%s".' %
-                                                  (expanded_any_end_token,))
-        self._MergeField(tokenizer, expanded_any_sub_message)
-      if field.label == descriptor.FieldDescriptor.LABEL_REPEATED:
-        any_message = getattr(message, field.name).add()
-      else:
-        any_message = getattr(message, field.name)
-      any_message.Pack(expanded_any_sub_message,
-                       type_url_prefix=type_url_prefix)
-    elif field.label == descriptor.FieldDescriptor.LABEL_REPEATED:
+    if field.label == descriptor.FieldDescriptor.LABEL_REPEATED:
       if field.is_extension:
         sub_message = message.Extensions[field].add()
       elif is_map_entry:
@@ -812,8 +814,20 @@ class _Parser(object):
         sub_message = getattr(message, field.name).add()
     else:
       if field.is_extension:
+        if (not self._allow_multiple_scalars and
+            message.HasExtension(field)):
+          raise tokenizer.ParseErrorPreviousToken(
+              'Message type "%s" should not have multiple "%s" extensions.' %
+              (message.DESCRIPTOR.full_name, field.full_name))
         sub_message = message.Extensions[field]
       else:
+        # Also apply _allow_multiple_scalars to message field.
+        # TODO(jieluo): Change to _allow_singular_overwrites.
+        if (not self._allow_multiple_scalars and
+            message.HasField(field.name)):
+          raise tokenizer.ParseErrorPreviousToken(
+              'Message type "%s" should not have multiple "%s" fields.' %
+              (message.DESCRIPTOR.full_name, field.name))
         sub_message = getattr(message, field.name)
       sub_message.SetInParent()