Merge branch 'master' of github.com:google/protobuf

BUILD

@@ -377,6 +377,8 @@ RELATIVE_TEST_PROTOS = [
     "google/protobuf/unittest_preserve_unknown_enum.proto",
     "google/protobuf/unittest_preserve_unknown_enum2.proto",
     "google/protobuf/unittest_proto3_arena.proto",
+    "google/protobuf/unittest_proto3_arena_lite.proto",
+    "google/protobuf/unittest_proto3_lite.proto",
     "google/protobuf/unittest_well_known_types.proto",
     "google/protobuf/util/internal/testdata/anys.proto",
     "google/protobuf/util/internal/testdata/books.proto",
@@ -463,6 +465,8 @@ cc_test(
         "src/google/protobuf/no_field_presence_test.cc",
         "src/google/protobuf/preserve_unknown_enum_test.cc",
         "src/google/protobuf/proto3_arena_unittest.cc",
+        "src/google/protobuf/proto3_arena_lite_unittest.cc",
+        "src/google/protobuf/proto3_lite_unittest.cc",
         "src/google/protobuf/reflection_ops_unittest.cc",
         "src/google/protobuf/repeated_field_reflection_unittest.cc",
         "src/google/protobuf/repeated_field_unittest.cc",

csharp/src/Google.Protobuf/WellKnownTypes/Any.cs

@@ -139,6 +139,8 @@ namespace Google.Protobuf.WellKnownTypes {
     ///  * If no schema is provided, `https` is assumed.
     ///  * The last segment of the URL's path must represent the fully
     ///    qualified name of the type (as in `path/google.protobuf.Duration`).
+    ///    The name should be in a canonical form (e.g., leading "." is
+    ///    not accepted).
     ///  * An HTTP GET on the URL must yield a [google.protobuf.Type][]
     ///    value in binary format, or produce an error.
     ///  * Applications are allowed to cache lookup results based on the

csharp/src/Google.Protobuf/WellKnownTypes/FieldMask.cs

@@ -86,7 +86,7 @@ namespace Google.Protobuf.WellKnownTypes {
   ///  operation applies to all fields (as if a FieldMask of all fields
   ///  had been specified).
   ///
-  ///  Note that a field mask does not necessarily applies to the
+  ///  Note that a field mask does not necessarily apply to the
   ///  top-level response message. In case of a REST get operation, the
   ///  field mask applies directly to the response, but in case of a REST
   ///  list operation, the mask instead applies to each individual message

docs/swift/DesignDoc.md

@@ -0,0 +1,674 @@
+# Protocol Buffers in Swift
+
+## Objective
+
+This document describes the user-facing API and internal implementation of
+proto2 and proto3 messages in Apple’s Swift programming language.
+
+One of the key goals of protobufs is to provide idiomatic APIs for each
+language. In that vein, **interoperability with Objective-C is a non-goal of
+this proposal.** Protobuf users who need to pass messages between Objective-C
+and Swift code in the same application should use the existing Objective-C proto
+library. The goal of the effort described here is to provide an API for protobuf
+messages that uses features specific to Swift—optional types, algebraic
+enumerated types, value types, and so forth—in a natural way that will delight,
+rather than surprise, users of the language.
+
+## Naming
+
+*   By convention, both typical protobuf message names and Swift structs/classes
+    are `UpperCamelCase`, so for most messages, the name of a message can be the
+    same as the name of its generated type. (However, see the discussion below
+    about prefixes under [Packages](#packages).)
+
+*   Enum cases in protobufs typically are `UPPERCASE_WITH_UNDERSCORES`, whereas
+    in Swift they are `lowerCamelCase` (as of the Swift 3 API design
+    guidelines). We will transform the names to match Swift convention, using
+    a whitelist similar to the Objective-C compiler plugin to handle commonly
+    used acronyms.
+
+*   Typical fields in proto messages are `lowercase_with_underscores`, while in
+    Swift they are `lowerCamelCase`. We will transform the names to match
+    Swift convention by removing the underscores and uppercasing the subsequent
+    letter.
+
+## Swift reserved words
+
+Swift has a large set of reserved words—some always reserved and some
+contextually reserved (that is, they can be used as identifiers in contexts
+where they would not be confused). As of Swift 2.2, the set of always-reserved
+words is:
+
+```
+_, #available, #column, #else, #elseif, #endif, #file, #function, #if, #line,
+#selector, as, associatedtype, break, case, catch, class, continue, default,
+defer, deinit, do, dynamicType, else, enum, extension, fallthrough, false, for,
+func, guard, if, import, in, init, inout, internal, is, let, nil, operator,
+private, protocol, public, repeat, rethrows, return, self, Self, static,
+struct, subscript, super, switch, throw, throws, true, try, typealias, var,
+where, while
+```
+
+The set of contextually reserved words is:
+
+```
+associativity, convenience, dynamic, didSet, final, get, infix, indirect,
+lazy, left, mutating, none, nonmutating, optional, override, postfix,
+precedence, prefix, Protocol, required, right, set, Type, unowned, weak,
+willSet
+```
+
+It is possible to use any reserved word as an identifier by escaping it with
+backticks (for example, ``let `class` = 5``). Other name-mangling schemes would
+require us to transform the names themselves (for example, by appending an
+underscore), which requires us to then ensure that the new name does not collide
+with something else in the same namespace.
+
+While the backtick feature may not be widely known by all Swift developers, a
+small amount of user education can address this and it seems like the best
+approach. We can unconditionally surround all property names with backticks to
+simplify generation.
+
+Some remapping will still be required, though, to avoid collisions between
+generated properties and the names of methods and properties defined in the base
+protocol/implementation of messages.
+
+# Features of Protocol Buffers
+
+This section describes how the features of the protocol buffer syntaxes (proto2
+and proto3) map to features in Swift—what the code generated from a proto will
+look like, and how it will be implemented in the underlying library.
+
+## Packages
+
+Modules are the main form of namespacing in Swift, but they are not declared
+using syntactic constructs like namespaces in C++ or packages in Java. Instead,
+they are tied to build targets in Xcode (or, in the future with open-source
+Swift, declarations in a Swift Package Manager manifest). They also do not
+easily support nesting submodules (Clang module maps support this, but pure
+Swift does not yet provide a way to define submodules).
+
+We will generate types with fully-qualified underscore-delimited names. For
+example, a message `Baz` in package `foo.bar` would generate a struct named
+`Foo_Bar_Baz`. For each fully-qualified proto message, there will be exactly one
+unique type symbol emitted in the generated binary.
+
+Users are likely to balk at the ugliness of underscore-delimited names for every
+generated type. To improve upon this situation, we will add a new string file
+level option, `swift_package_typealias`, that can be added to `.proto` files.
+When present, this will cause `typealias`es to be added to the generated Swift
+messages that replace the package name prefix with the provided string. For
+example, the following `.proto` file:
+
+```protobuf
+option swift_package_typealias = "FBP";
+package foo.bar;
+
+message Baz {
+  // Message fields
+}
+```
+
+would generate the following Swift source:
+
+```swift
+public struct Foo_Bar_Baz {
+  // Message fields and other methods
+}
+
+typealias FBPBaz = Foo_Bar_Baz
+```
+
+It should be noted that this type alias is recorded in the generated
+`.swiftmodule` so that code importing the module can refer to it, but it does
+not cause a new symbol to be generated in the compiled binary (i.e., we do not
+risk compiled size bloat by adding `typealias`es for every type).
+
+Other strategies to handle packages that were considered and rejected can be
+found in [Appendix A](#appendix-a-rejected-strategies-to-handle-packages).
+
+## Messages
+
+Proto messages are natural value types and we will generate messages as structs
+instead of classes. Users will benefit from Swift’s built-in behavior with
+regard to mutability. We will define a `ProtoMessage` protocol that defines the
+common methods and properties for all messages (such as serialization) and also
+lets users treat messages polymorphically. Any shared method implementations
+that do not differ between individual messages can be implemented in a protocol
+extension.
+
+The backing storage itself for fields of a message will be managed by a
+`ProtoFieldStorage` type that uses an internal dictionary keyed by field number,
+and whose values are the value of the field with that number (up-cast to Swift’s
+`Any` type). This class will provide type-safe getters and setters so that
+generated messages can manipulate this storage, and core serialization logic
+will live here as well. Furthermore, factoring the storage out into a separate
+type, rather than inlining the fields as stored properties in the message
+itself, lets us implement copy-on-write efficiently to support passing around
+large messages. (Furthermore, because the messages themselves are value types,
+inlining fields is not possible if the fields are submessages of the same type,
+or a type that eventually includes a submessage of the same type.)
+
+### Required fields (proto2 only)
+
+Required fields in proto2 messages seem like they could be naturally represented
+by non-optional properties in Swift, but this presents some problems/concerns.
+
+Serialization APIs permit partial serialization, which allows required fields to
+remain unset. Furthermore, other language APIs still provide `has*` and `clear*`
+methods for required fields, and knowing whether a property has a value when the
+message is in memory is still useful.
+
+For example, an e-mail draft message may have the “to” address required on the
+wire, but when the user constructs it in memory, it doesn’t make sense to force
+a value until they provide one. We only want to force a value to be present when
+the message is serialized to the wire. Using non-optional properties prevents
+this use case, and makes client usage awkward because the user would be forced
+to select a sentinel or placeholder value for any required fields at the time
+the message was created.
+
+### Default values
+
+In proto2, fields can have a default value specified that may be a value other
+than the default value for its corresponding language type (for example, a
+default value of 5 instead of 0 for an integer). When reading a field that is
+not explicitly set, the user expects to get that value. This makes Swift
+optionals (i.e., `Foo?`) unsuitable for fields in general. Unfortunately, we
+cannot implement our own “enhanced optional” type without severely complicating
+usage (Swift’s use of type inference and its lack of implicit conversions would
+require manual unwrapping of every property value).
+
+Instead, we can use **implicitly unwrapped optionals.** For example, a property
+generated for a field of type `int32` would have Swift type `Int32!`. These
+properties would behave with the following characteristics, which mirror the
+nil-resettable properties used elsewhere in Apple’s SDKs (for example,
+`UIView.tintColor`):
+
+*   Assigning a non-nil value to a property sets the field to that value.
+*   Assigning nil to a property clears the field (its internal representation is
+    nilled out).
+*   Reading the value of a property returns its value if it is set, or returns
+    its default value if it is not set. Reading a property never returns nil.
+
+The final point in the list above implies that the optional cannot be checked to
+determine if the field is set to a value other than its default: it will never
+be nil. Instead, we must provide `has*` methods for each field to allow the user
+to check this. These methods will be public in proto2. In proto3, these methods
+will be private (if generated at all), since the user can test the returned
+value against the zero value for that type.
+
+### Autocreation of nested messages
+
+For convenience, dotting into an unset field representing a nested message will
+return an instance of that message with default values. As in the Objective-C
+implementation, this does not actually cause the field to be set until the
+returned message is mutated. Fortunately, thanks to the way mutability of value
+types is implemented in Swift, the language automatically handles the
+reassignment-on-mutation for us. A static singleton instance containing default
+values can be associated with each message that can be returned when reading, so
+copies are only made by the Swift runtime when mutation occurs. For example,
+given the following proto:
+
+```protobuf
+message Node {
+  Node child = 1;
+  string value = 2 [default = "foo"];
+}
+```
+
+The following Swift code would act as commented, where setting deeply nested
+properties causes the copies and mutations to occur as the assignment statement
+is unwound:
+
+```swift
+var node = Node()
+
+let s = node.child.child.value
+// 1. node.child returns the "default Node".
+// 2. Reading .child on the result of (1) returns the same default Node.
+// 3. Reading .value on the result of (2) returns the default value "foo".
+
+node.child.child.value = "bar"
+// 4. Setting .value on the default Node causes a copy to be made and sets
+//    the property on that copy. Subsequently, the language updates the
+//    value of "node.child.child" to point to that copy.
+// 5. Updating "node.child.child" in (4) requires another copy, because
+//    "node.child" was also the instance of the default node. The copy is
+//    assigned back to "node.child".
+// 6. Setting "node.child" in (5) is a simple value reassignment, since
+//    "node" is a mutable var.
+```
+
+In other words, the generated messages do not internally have to manage parental
+relationships to backfill the appropriate properties on mutation. Swift provides
+this for free.
+
+## Scalar value fields
+
+Proto scalar value fields will map to Swift types in the following way:
+
+.proto Type | Swift Type
+----------- | -------------------
+`double`    | `Double`
+`float`     | `Float`
+`int32`     | `Int32`
+`int64`     | `Int64`
+`uint32`    | `UInt32`
+`uint64`    | `UInt64`
+`sint32`    | `Int32`
+`sint64`    | `Int64`
+`fixed32`   | `UInt32`
+`fixed64`   | `UInt64`
+`sfixed32`  | `Int32`
+`sfixed64`  | `Int64`
+`bool`      | `Bool`
+`string`    | `String`
+`bytes`     | `Foundation.NSData`
+
+The proto spec defines a number of integral types that map to the same Swift
+type; for example, `intXX`, `sintXX`, and `sfixedXX` are all signed integers,
+and `uintXX` and `fixedXX` are both unsigned integers. No other language
+implementation distinguishes these further, so we do not do so either. The
+rationale is that the various types only serve to distinguish how the value is
+**encoded on the wire**; once loaded in memory, the user is not concerned about
+these variations.
+
+Swift’s lack of implicit conversions among types will make it slightly annoying
+to use these types in a context expecting an `Int`, or vice-versa, but since
+this is a data-interchange format with explicitly-sized fields, we should not
+hide that information from the user. Users will have to explicitly write
+`Int(message.myField)`, for example.
+
+## Embedded message fields
+
+Embedded message fields can be represented using an optional variable of the
+generated message type. Thus, the message
+
+```protobuf
+message Foo {
+  Bar bar = 1;
+}
+```
+
+would be represented in Swift as
+
+```swift
+public struct Foo: ProtoMessage {
+  public var bar: Bar! {
+    get { ... }
+    set { ... }
+  }
+}
+```
+
+If the user explicitly sets `bar` to nil, or if it was never set when read from
+the wire, retrieving the value of `bar` would return a default, statically
+allocated instance of `Bar` containing default values for its fields. This
+achieves the desired behavior for default values in the same way that scalar
+fields are designed, and also allows users to deep-drill into complex object
+graphs to get or set fields without checking for nil at each step.
+
+## Enum fields
+
+The design and implementation of enum fields will differ somewhat drastically
+depending on whether the message being generated is a proto2 or proto3 message.
+
+### proto2 enums
+
+For proto2, we do not need to be concerned about unknown enum values, so we can
+use the simple raw-value enum syntax provided by Swift. So the following enum in
+proto2:
+
+```protobuf
+enum ContentType {
+  TEXT = 0;
+  IMAGE = 1;
+}
+```
+
+would become this Swift enum:
+
+```swift
+public enum ContentType: Int32, NilLiteralConvertible {
+  case text = 0
+  case image = 1
+
+  public init(nilLiteral: ()) {
+    self = .text
+  }
+}
+```
+
+See below for the discussion about `NilLiteralConvertible`.
+
+### proto3 enums
+
+For proto3, we need to be able to preserve unknown enum values that may come
+across the wire so that they can be written back if unmodified. We can
+accomplish this in Swift by using a case with an associated value for unknowns.
+So the following enum in proto3:
+
+```protobuf
+enum ContentType {
+  TEXT = 0;
+  IMAGE = 1;
+}
+```
+
+would become this Swift enum:
+
+```swift
+public enum ContentType: RawRepresentable, NilLiteralConvertible {
+  case text
+  case image
+  case UNKNOWN_VALUE(Int32)
+
+  public typealias RawValue = Int32
+
+  public init(nilLiteral: ()) {
+    self = .text
+  }
+
+  public init(rawValue: RawValue) {
+    switch rawValue {
+      case 0: self = .text
+      case 1: self = .image
+      default: self = .UNKNOWN_VALUE(rawValue)
+  }
+
+  public var rawValue: RawValue {
+    switch self {
+      case .text: return 0
+      case .image: return 1
+      case .UNKNOWN_VALUE(let value): return value
+    }
+  }
+}
+```
+
+Note that the use of a parameterized case prevents us from inheriting from the
+raw `Int32` type; Swift does not allow an enum with a raw type to have cases
+with arguments. Instead, we must implement the raw value initializer and
+computed property manually. The `UNKNOWN_VALUE` case is explicitly chosen to be
+"ugly" so that it stands out and does not conflict with other possible case
+names.
+
+Using this approach, proto3 consumers must always have a default case or handle
+the `.UNKNOWN_VALUE` case to satisfy case exhaustion in a switch statement; the
+Swift compiler considers it an error if switch statements are not exhaustive.
+
+### NilLiteralConvertible conformance
+
+This is required to clean up the usage of enum-typed properties in switch
+statements. Unlike other field types, enum properties cannot be
+implicitly-unwrapped optionals without requiring that uses in switch statements
+be explicitly unwrapped. For example, if we consider a message with the enum
+above, this usage will fail to compile:
+
+```swift
+// Without NilLiteralConvertible conformance on ContentType
+public struct SomeMessage: ProtoMessage {
+  public var contentType: ContentType! { ... }
+}
+
+// ERROR: no case named text or image
+switch someMessage.contentType {
+  case .text: { ... }
+  case .image: { ... }
+}
+```
+
+Even though our implementation guarantees that `contentType` will never be nil,
+if it is an optional type, its cases would be `some` and `none`, not the cases
+of the underlying enum type. In order to use it in this context, the user must
+write `someMessage.contentType!` in their switch statement.
+
+Making the enum itself `NilLiteralConvertible` permits us to make the property
+non-optional, so the user can still set it to nil to clear it (i.e., reset it to
+its default value), while eliminating the need to explicitly unwrap it in a
+switch statement.
+
+```swift
+// With NilLiteralConvertible conformance on ContentType
+public struct SomeMessage: ProtoMessage {
+  // Note that the property type is no longer optional
+  public var contentType: ContentType { ... }
+}
+
+// OK: Compiles and runs as expected
+switch someMessage.contentType {
+  case .text: { ... }
+  case .image: { ... }
+}
+
+// The enum can be reset to its default value this way
+someMessage.contentType = nil
+```
+
+One minor oddity with this approach is that nil will be auto-converted to the
+default value of the enum in any context, not just field assignment. In other
+words, this is valid:
+
+```swift
+func foo(contentType: ContentType) { ... }
+foo(nil) // Inside foo, contentType == .text
+```
+
+That being said, the advantage of being able to simultaneously support
+nil-resettability and switch-without-unwrapping outweighs this side effect,
+especially if appropriately documented. It is our hope that a new form of
+resettable properties will be added to Swift that eliminates this inconsistency.
+Some community members have already drafted or sent proposals for review that
+would benefit our designs:
+
+*   [SE-0030: Property Behaviors]
+    (https://github.com/apple/swift-evolution/blob/master/proposals/0030-property-behavior-decls.md)
+*   [Drafted: Resettable Properties]
+    (https://github.com/patters/swift-evolution/blob/master/proposals/0000-resettable-properties.md)
+
+### Enum aliases
+
+The `allow_alias` option in protobuf slightly complicates the use of Swift enums
+to represent that type, because raw values of cases in an enum must be unique.
+Swift lets us define static variables in an enum that alias actual cases. For
+example, the following protobuf enum:
+
+```protobuf
+enum Foo {
+  option allow_alias = true;
+  BAR = 0;
+  BAZ = 0;
+}
+```
+
+will be represented in Swift as:
+
+```swift
+public enum Foo: Int32, NilLiteralConvertible {
+  case bar = 0
+  static public let baz = bar
+
+  // ... etc.
+}
+
+// Can still use .baz shorthand to reference the alias in contexts
+// where the type is inferred
+```
+
+That is, we use the first name as the actual case and use static variables for
+the other aliases. One drawback to this approach is that the static aliases
+cannot be used as cases in a switch statement (the compiler emits the error
+*“Enum case ‘baz’ not found in type ‘Foo’”*). However, in our own code bases,
+there are only a few places where enum aliases are not mere renamings of an
+older value, but they also don’t appear to be the type of value that one would
+expect to switch on (for example, a group of named constants representing
+metrics rather than a set of options), so this restriction is not significant.
+
+This strategy also implies that changing the name of an enum and adding the old
+name as an alias below the new name will be a breaking change in the generated
+Swift code.
+
+## Oneof types
+
+The `oneof` feature represents a “variant/union” data type that maps nicely to
+Swift enums with associated values (algebraic types). These fields can also be
+accessed independently though, and, specifically in the case of proto2, it’s
+reasonable to expect access to default values when accessing a field that is not
+explicitly set.
+
+Taking all this into account, we can represent a `oneof` in Swift with two sets
+of constructs:
+
+*   Properties in the message that correspond to the `oneof` fields.
+*   A nested enum named after the `oneof` and which provides the corresponding
+    field values as case arguments.
+
+This approach fulfills the needs of proto consumers by providing a
+Swift-idiomatic way of simultaneously checking which field is set and accessing
+its value, providing individual properties to access the default values
+(important for proto2), and safely allows a field to be moved into a `oneof`
+without breaking clients.
+
+Consider the following proto:
+
+```protobuf
+message MyMessage {
+  oneof record {
+    string name = 1 [default = "unnamed"];
+    int32 id_number = 2 [default = 0];
+  }
+}
+```
+
+In Swift, we would generate an enum, a property for that enum, and properties
+for the fields themselves:
+
+```swift
+public struct MyMessage: ProtoMessage {
+  public enum Record: NilLiteralConvertible {
+    case name(String)
+    case idNumber(Int32)
+    case NOT_SET
+
+    public init(nilLiteral: ()) { self = .NOT_SET }
+  }
+
+  // This is the "Swifty" way of accessing the value
+  public var record: Record { ... }
+
+  // Direct access to the underlying fields
+  public var name: String! { ... }
+  public var idNumber: Int32! { ... }
+}
+```
+
+This makes both usage patterns possible:
+
+```swift
+// Usage 1: Case-based dispatch
+switch message.record {
+  case .name(let name):
+    // Do something with name if it was explicitly set
+  case .idNumber(let id):
+    // Do something with id_number if it was explicitly set
+  case .NOT_SET:
+    // Do something if it’s not set
+}
+
+// Usage 2: Direct access for default value fallback
+// Sets the label text to the name if it was explicitly set, or to
+// "unnamed" (the default value for the field) if id_number was set
+// instead
+let myLabel = UILabel()
+myLabel.text = message.name
+```
+
+As with proto enums, the generated `oneof` enum conforms to
+`NilLiteralConvertible` to avoid switch statement issues. Setting the property
+to nil will clear it (i.e., reset it to `NOT_SET`).
+
+## Unknown Fields (proto2 only)
+
+To be written.
+
+## Extensions (proto2 only)
+
+To be written.
+
+## Reflection and Descriptors
+
+We will not include reflection or descriptors in the first version of the Swift
+library. The use cases for reflection on mobile are not as strong and the static
+data to represent the descriptors would add bloat when we wish to keep the code
+size small.
+
+In the future, we will investigate whether they can be included as extensions
+which might be able to be excluded from a build and/or automatically dead
+stripped by the compiler if they are not used.
+
+## Appendix A: Rejected strategies to handle packages
+
+### Each package is its own Swift module
+
+Each proto package could be declared as its own Swift module, replacing dots
+with underscores (e.g., package `foo.bar` becomes module `Foo_Bar`). Then, users
+would simply import modules containing whatever proto modules they want to use
+and refer to the generated types by their short names.
+
+**This solution is simply not possible, however.** Swift modules cannot
+circularly reference each other, but there is no restriction against proto
+packages doing so. Circular imports are forbidden (e.g., `foo.proto` importing
+`bar.proto` importing `foo.proto`), but nothing prevents package `foo` from
+using a type in package `bar` which uses a different type in package `foo`, as
+long as there is no import cycle. If these packages were generated as Swift
+modules, then `Foo` would contain an `import Bar` statement and `Bar` would
+contain an `import Foo` statement, and there is no way to compile this.
+
+### Ad hoc namespacing with structs
+
+We can “fake” namespaces in Swift by declaring empty structs with private
+initializers. Since modules are constructed based on compiler arguments, not by
+syntactic constructs, and because there is no pure Swift way to define
+submodules (even though Clang module maps support this), there is no
+source-drive way to group generated code into namespaces aside from this
+approach.
+
+Types can be added to those intermediate package structs using Swift extensions.
+For example, a message `Baz` in package `foo.bar` could be represented in Swift
+as follows:
+
+```swift
+public struct Foo {
+  private init() {}
+}
+
+public extension Foo {
+  public struct Bar {
+    private init() {}
+  }
+}
+
+public extension Foo.Bar {
+  public struct Baz {
+    // Message fields and other methods
+  }
+}
+
+let baz = Foo.Bar.Baz()
+```
+
+Each of these constructs would actually be defined in a separate file; Swift
+lets us keep them separate and add multiple structs to a single “namespace”
+through extensions.
+
+Unfortunately, these intermediate structs generate symbols of their own
+(metatype information in the data segment). This becomes problematic if multiple
+build targets contain Swift sources generated from different messages in the
+same package. At link time, these symbols would collide, resulting in multiple
+definition errors.
+
+This approach also has the disadvantage that there is no automatic “short” way
+to refer to the generated messages at the deepest nesting levels; since this use
+of structs is a hack around the lack of namespaces, there is no equivalent to
+import (Java) or using (C++) to simplify this. Users would have to declare type
+aliases to make this cleaner, or we would have to generate them for users.

docs/third_party.md

@@ -0,0 +1,147 @@
+# Third-Party Add-ons for Protocol Buffers
+
+This page lists code related to Protocol Buffers which is developed and maintained by third parties.  You may find this code useful, but note that **these projects are not affiliated with or endorsed by Google (unless explicitly marked)**; try them at your own risk.  Also note that many projects here are in the early stages of development and not production-ready.
+
+If you have a project that should be listed here, please [send us a pull request](https://github.com/google/protobuf/pulls) to update this page.
+
+## Programming Languages
+
+These are projects we know about implementing Protocol Buffers for other programming languages:
+* Action Script: http://code.google.com/p/protobuf-actionscript3/
+* Action Script: https://code.google.com/p/protoc-gen-as3/
+* Action Script: https://github.com/matrix3d/JProtoc
+* C: https://github.com/protobuf-c/protobuf-c
+* C: http://koti.kapsi.fi/jpa/nanopb/
+* C: https://github.com/cloudwu/pbc/
+* C: https://github.com/haberman/upb/wiki
+* C: https://github.com/squidfunk/protobluff
+* C++: https://github.com/google/protobuf (Google-official implementation)
+* C/C++: http://spbc.sf.net/
+* C#: http://code.google.com/p/protobuf-csharp-port
+* C#: http://code.google.com/p/protosharp/
+* C#: https://silentorbit.com/protobuf/
+* C#/.NET/WCF/VB: http://code.google.com/p/protobuf-net/
+* Clojure: http://github.com/ninjudd/clojure-protobuf
+* Common Lisp: http://www.prism.gatech.edu/~ndantam3/docs/s-protobuf/
+* Common Lisp: http://github.com/brown/protobuf
+* D: https://github.com/msoucy/dproto
+* D: http://256.makerslocal.org/wiki/index.php/ProtocolBuffer
+* D: https://github.com/opticron/ProtocolBuffer
+* Dart: https://github.com/dart-lang/dart-protobuf (runtime) https://github.com/dart-lang/dart-protoc-plugin (code generator)
+* Delphi: http://sourceforge.net/projects/protobuf-delphi/
+* Delphi: http://fundementals.sourceforge.net/dl.html
+* Elixir: https://github.com/jeremyong/exprotoc
+* Erlang: http://github.com/ngerakines/erlang_protobuffs/tree/master
+* Erlang: http://piqi.org/
+* Erlang: https://code.google.com/p/protoc-gen-erl/
+* Erlang: https://github.com/basho/erlang_protobuffs
+* Go: https://github.com/golang/protobuf (Google-official implementation)
+* Go: http://code.google.com/p/goprotobuf/
+* Go: https://github.com/akunspy/gopbuf
+* Haskell: http://hackage.haskell.org/package/hprotoc
+* Haxe: https://github.com/Atry/protoc-gen-haxe
+* Java: https://github.com/google/protobuf (Google-official implementation)
+* Java/Android: https://github.com/square/wire
+* Java ME: http://code.google.com/p/protobuf-javame/
+* Java ME: http://swingme.sourceforge.net/encode.shtml
+* Java ME: http://github.com/ponderingpanda/protobuf-j2me
+* Java ME: http://code.google.com/p/protobuf-j2me/
+* Javascript: http://code.google.com/p/protobuf-js/
+* Javascript: http://github.com/sirikata/protojs
+* Javascript: https://github.com/dcodeIO/ProtoBuf.js
+* Javascript: http://code.google.com/p/protobuf-for-node/
+* Javascript: http://code.google.com/p/protostuff/
+* Julia: https://github.com/tanmaykm/ProtoBuf.jl
+* Lua: http://code.google.com/p/protoc-gen-lua/
+* Lua: http://github.com/indygreg/lua-protobuf
+* Lua: https://github.com/Neopallium/lua-pb
+* Matlab: http://code.google.com/p/protobuf-matlab/
+* Mercury: http://code.google.com/p/protobuf-mercury/
+* Objective C: http://code.google.com/p/protobuf-objc/
+* Objective C: https://github.com/alexeyxo/protobuf-objc
+* OCaml: http://piqi.org/
+* Perl: http://groups.google.com/group/protobuf-perl
+* Perl: http://search.cpan.org/perldoc?Google::ProtocolBuffers
+* Perl/XS: http://code.google.com/p/protobuf-perlxs/
+* PHP: http://code.google.com/p/pb4php/
+* PHP: https://github.com/allegro/php-protobuf/
+* PHP: https://github.com/chobie/php-protocolbuffers
+* PHP: http://drslump.github.com/Protobuf-PHP
+* Prolog: http://www.swi-prolog.org/pldoc/package/protobufs.html
+* Python: https://github.com/google/protobuf (Google-official implementation)
+* Python: http://eigenein.github.com/protobuf/
+* R: http://cran.r-project.org/package=RProtoBuf
+* Ruby: http://code.google.com/p/ruby-protobuf/
+* Ruby: http://github.com/mozy/ruby-protocol-buffers
+* Ruby: https://github.com/bmizerany/beefcake/tree/master/lib/beefcake
+* Ruby: https://github.com/localshred/protobuf
+* Rust: https://github.com/stepancheg/rust-protobuf/
+* Scala: http://github.com/jeffplaisance/scala-protobuf
+* Scala: http://code.google.com/p/protobuf-scala
+* Scala: https://github.com/SandroGrzicic/ScalaBuff
+* Scala: http://trueaccord.github.io/ScalaPB/
+* Swift: https://github.com/alexeyxo/protobuf-swift
+* Vala: https://launchpad.net/protobuf-vala
+* Visual Basic: http://code.google.com/p/protobuf-net/
+
+## RPC Implementations
+
+GRPC (http://www.grpc.io/) is Google's RPC implementation for Protocol Buffers. There are other third-party RPC implementations as well.  Some of these actually work with Protocol Buffers service definitions (defined using the `service` keyword in `.proto` files) while others just use Protocol Buffers message objects.
+
+* https://github.com/grpc/grpc (C++, Node.js, Python, Ruby, Objective-C, PHP, C#, Google-official implementation)
+* http://zeroc.com/ice.html (Multiple languages)
+* http://code.google.com/p/protobuf-net/ (C#/.NET/WCF/VB)
+* https://launchpad.net/txprotobuf/ (Python)
+* https://github.com/modeswitch/protobuf-rpc (Python)
+* http://code.google.com/p/protobuf-socket-rpc/ (Java, Python)
+* http://code.google.com/p/proto-streamer/ (Java)
+* http://code.google.com/p/server1/ (C++)
+* http://deltavsoft.com/RcfUserGuide/Protobufs (C++)
+* http://code.google.com/p/protobuf-mina-rpc/ (Python client, Java server)
+* http://code.google.com/p/casocklib/ (C++)
+* http://code.google.com/p/cxf-protobuf/ (Java)
+* http://code.google.com/p/protobuf-remote/ (C++/C#)
+* http://code.google.com/p/protobuf-rpc-pro/ (Java)
+* https://code.google.com/p/protorpc/ (Go/C++)
+* https://code.google.com/p/eneter-protobuf-serializer/ (Java/.NET)
+* http://www.deltavsoft.com/RCFProto.html (C++/Java/Python/C#)
+* https://github.com/robbinfan/claire-protorpc (C++)
+* https://github.com/BaiduPS/sofa-pbrpc (C++)
+* https://github.com/ebencheung/arab (C++)
+* http://code.google.com/p/protobuf-csharp-rpc/ (C#)
+* https://github.com/thesamet/rpcz (C++/Python, based on ZeroMQ)
+* https://github.com/w359405949/libmaid (C++, Python)
+* https://github.com/madwyn/libpbrpc (C++)
+
+## Other Utilities
+
+There are miscellaneous other things you may find useful as a Protocol Buffers developer.
+
+* [NetBeans IDE plugin](http://code.google.com/p/protobuf-netbeans-plugin/)
+* [Wireshark/Ethereal packet sniffer plugin](http://code.google.com/p/protobuf-wireshark/)
+* [Alternate encodings (JSON, XML, HTML) for Java protobufs](http://code.google.com/p/protobuf-java-format/)
+* [Another JSON encoder/decoder for Java](https://github.com/sijuv/protobuf-codec)
+* [Editor for serialized protobufs](http://code.google.com/p/protobufeditor/)
+* [Intellij IDEA plugin](http://github.com/nnmatveev/idea-plugin-protobuf)
+* [TextMate syntax highlighting](http://github.com/michaeledgar/protobuf-tmbundle)
+* [Oracle PL SQL plugin](http://code.google.com/p/protocol-buffer-plsql/)
+* [Eclipse editor for protobuf (from Google)](http://code.google.com/p/protobuf-dt/)
+* [C++ Builder compatible protobuf](https://github.com/saadware/protobuf-cppbuilder)
+* Maven Protocol Compiler Plugin
+    * https://github.com/sergei-ivanov/maven-protoc-plugin/
+    * http://igor-petruk.github.com/protobuf-maven-plugin/
+    * http://code.google.com/p/maven-protoc-plugin/
+    * https://github.com/os72/protoc-jar-maven-plugin
+* [Documentation generator plugin (Markdown/HTML/DocBook/...)](https://github.com/estan/protoc-gen-doc)
+* [DocBook generator for .proto files](http://code.google.com/p/protoc-gen-docbook/)
+* [Protobuf for nginx module](https://github.com/dbcode/protobuf-nginx/)
+* [RSpec matchers and Cucumber step defs for testing Protocol Buffers](https://github.com/connamara/protobuf_spec)
+* [Sbt plugin for Protocol Buffers](https://github.com/Atry/sbt-cppp)
+* [Gradle Protobuf Plugin](https://github.com/aantono/gradle-plugin-protobuf)
+* [Multi-platform executable JAR and Java API for protoc](https://github.com/os72/protoc-jar)
+* [Python scripts to convert between Protocol Buffers and JSON](https://github.com/NextTuesday/py-pb-converters)
+* [Visual Studio Language Service support for Protocol Buffers](http://visualstudiogallery.msdn.microsoft.com/4bc0f38c-b058-4e05-ae38-155e053c19c5)
+* [C++ library for serialization/de-serialization between Protocol Buffers and JSON.](https://github.com/yinqiwen/pbjson)
+* [ProtoBuf with Java EE7 Expression Language 3.0; pure Java ProtoBuf Parser and Builder.](https://github.com/protobufel/protobuf-el)
+* [Notepad++ Syntax Highlighting for .proto files](https://github.com/chai2010/notepadplus-protobuf)
+* [Linter for .proto files](https://github.com/ckaznocha/protoc-gen-lint)

java/core/src/main/java/com/google/protobuf/GeneratedMessage.java

@@ -797,6 +797,8 @@ public abstract class GeneratedMessage extends AbstractMessage
       extends GeneratedMessage
       implements ExtendableMessageOrBuilder<MessageType> {
 
+    private static final long serialVersionUID = 1L;
+
     private final FieldSet<FieldDescriptor> extensions;
 
     protected ExtendableMessage() {

objectivec/GPBMessage.m

@@ -2603,9 +2603,13 @@ static void MergeRepeatedNotPackedFieldFromCodedInputStream(
       size_t fieldOffset = field->description_->offset;
       switch (fieldDataType) {
         case GPBDataTypeBool: {
-          BOOL *selfValPtr = (BOOL *)&selfStorage[fieldOffset];
-          BOOL *otherValPtr = (BOOL *)&otherStorage[fieldOffset];
-          if (*selfValPtr != *otherValPtr) {
+          // Bools are stored in has_bits to avoid needing explicit space in
+          // the storage structure.
+          // (the field number passed to the HasIvar helper doesn't really
+          // matter since the offset is never negative)
+          BOOL selfValue = GPBGetHasIvar(self, (int32_t)(fieldOffset), 0);
+          BOOL otherValue = GPBGetHasIvar(other, (int32_t)(fieldOffset), 0);
+          if (selfValue != otherValue) {
             return NO;
           }
           break;
@@ -2714,8 +2718,12 @@ static void MergeRepeatedNotPackedFieldFromCodedInputStream(
       size_t fieldOffset = field->description_->offset;
       switch (fieldDataType) {
         case GPBDataTypeBool: {
-          BOOL *valPtr = (BOOL *)&storage[fieldOffset];
-          result = prime * result + *valPtr;
+          // Bools are stored in has_bits to avoid needing explicit space in
+          // the storage structure.
+          // (the field number passed to the HasIvar helper doesn't really
+          // matter since the offset is never negative)
+          BOOL value = GPBGetHasIvar(self, (int32_t)(fieldOffset), 0);
+          result = prime * result + value;
           break;
         }
         case GPBDataTypeSFixed32:

objectivec/Tests/GPBMessageTests.m

@@ -1947,4 +1947,76 @@
                  EnumTestMsg_MyEnum_NegTwo);
 }
 
+- (void)testOneBasedEnumHolder {
+  // Test case for https://github.com/google/protobuf/issues/1453
+  // Message with no explicit defaults, but a non zero default for an enum.
+  MessageWithOneBasedEnum *enumMsg = [MessageWithOneBasedEnum message];
+  XCTAssertEqual(enumMsg.enumField, MessageWithOneBasedEnum_OneBasedEnum_One);
+}
+
+- (void)testBoolOffsetUsage {
+  // Bools use storage within has_bits; this test ensures that this is honored
+  // in all places where things should crash or fail based on reading out of
+  // field storage instead.
+  BoolOnlyMessage *msg1 = [BoolOnlyMessage message];
+  BoolOnlyMessage *msg2 = [BoolOnlyMessage message];
+
+  msg1.boolField1 = YES;
+  msg2.boolField1 = YES;
+  msg1.boolField3 = YES;
+  msg2.boolField3 = YES;
+  msg1.boolField5 = YES;
+  msg2.boolField5 = YES;
+  msg1.boolField7 = YES;
+  msg2.boolField7 = YES;
+  msg1.boolField9 = YES;
+  msg2.boolField9 = YES;
+  msg1.boolField11 = YES;
+  msg2.boolField11 = YES;
+  msg1.boolField13 = YES;
+  msg2.boolField13 = YES;
+  msg1.boolField15 = YES;
+  msg2.boolField15 = YES;
+  msg1.boolField17 = YES;
+  msg2.boolField17 = YES;
+  msg1.boolField19 = YES;
+  msg2.boolField19 = YES;
+  msg1.boolField21 = YES;
+  msg2.boolField21 = YES;
+  msg1.boolField23 = YES;
+  msg2.boolField23 = YES;
+  msg1.boolField25 = YES;
+  msg2.boolField25 = YES;
+  msg1.boolField27 = YES;
+  msg2.boolField27 = YES;
+  msg1.boolField29 = YES;
+  msg2.boolField29 = YES;
+  msg1.boolField31 = YES;
+  msg2.boolField31 = YES;
+
+  msg1.boolField32 = YES;
+  msg2.boolField32 = YES;
+
+  XCTAssertTrue(msg1 != msg2); // Different pointers.
+  XCTAssertEqual([msg1 hash], [msg2 hash]);
+  XCTAssertEqualObjects(msg1, msg2);
+
+  BoolOnlyMessage *msg1Prime = [[msg1 copy] autorelease];
+  XCTAssertTrue(msg1Prime != msg1); // Different pointers.
+  XCTAssertEqual([msg1 hash], [msg1Prime hash]);
+  XCTAssertEqualObjects(msg1, msg1Prime);
+
+  // Field set in one, but not the other means they don't match (even if
+  // set to default value).
+  msg1Prime.boolField2 = NO;
+  XCTAssertNotEqualObjects(msg1Prime, msg1);
+  // And when set to different values.
+  msg1.boolField2 = YES;
+  XCTAssertNotEqualObjects(msg1Prime, msg1);
+  // And then they match again.
+  msg1.boolField2 = NO;
+  XCTAssertEqualObjects(msg1Prime, msg1);
+  XCTAssertEqual([msg1 hash], [msg1Prime hash]);
+}
+
 @end

objectivec/Tests/unittest_objc.proto

@@ -401,3 +401,49 @@ message EnumTestMsg {
 
   repeated MyEnum mumble = 4;
 }
+
+// Test case for https://github.com/google/protobuf/issues/1453
+// Message with no explicit defaults, but a non zero default for an enum.
+message MessageWithOneBasedEnum {
+  enum OneBasedEnum {
+    ONE = 1;
+    TWO = 2;
+  }
+  optional OneBasedEnum enum_field = 1;
+}
+
+// Message with all bools for testing things related to bool storage.
+message BoolOnlyMessage {
+  optional bool bool_field_1 = 1;
+  optional bool bool_field_2 = 2;
+  optional bool bool_field_3 = 3;
+  optional bool bool_field_4 = 4;
+  optional bool bool_field_5 = 5;
+  optional bool bool_field_6 = 6;
+  optional bool bool_field_7 = 7;
+  optional bool bool_field_8 = 8;
+  optional bool bool_field_9 = 9;
+  optional bool bool_field_10 = 10;
+  optional bool bool_field_11 = 11;
+  optional bool bool_field_12 = 12;
+  optional bool bool_field_13 = 13;
+  optional bool bool_field_14 = 14;
+  optional bool bool_field_15 = 15;
+  optional bool bool_field_16 = 16;
+  optional bool bool_field_17 = 17;
+  optional bool bool_field_18 = 18;
+  optional bool bool_field_19 = 19;
+  optional bool bool_field_20 = 20;
+  optional bool bool_field_21 = 21;
+  optional bool bool_field_22 = 22;
+  optional bool bool_field_23 = 23;
+  optional bool bool_field_24 = 24;
+  optional bool bool_field_25 = 25;
+  optional bool bool_field_26 = 26;
+  optional bool bool_field_27 = 27;
+  optional bool bool_field_28 = 28;
+  optional bool bool_field_29 = 29;
+  optional bool bool_field_30 = 30;
+  optional bool bool_field_31 = 31;
+  optional bool bool_field_32 = 32;
+}

ruby/ext/google/protobuf_c/encode_decode.c

@@ -656,7 +656,6 @@ static const upb_json_parsermethod *msgdef_jsonparsermethod(Descriptor* desc) {
 #define STACK_ENV_STACKBYTES 4096
 typedef struct {
   upb_env env;
-  upb_seededalloc alloc;
   const char* ruby_error_template;
   char allocbuf[STACK_ENV_STACKBYTES];
 } stackenv;
@@ -681,16 +680,12 @@ static bool env_error_func(void* ud, const upb_status* status) {
 
 static void stackenv_init(stackenv* se, const char* errmsg) {
   se->ruby_error_template = errmsg;
-  upb_env_init(&se->env);
-  upb_seededalloc_init(&se->alloc, &se->allocbuf, STACK_ENV_STACKBYTES);
-  upb_env_setallocfunc(
-      &se->env, upb_seededalloc_getallocfunc(&se->alloc), &se->alloc);
+  upb_env_init2(&se->env, se->allocbuf, sizeof(se->allocbuf), NULL);
   upb_env_seterrorfunc(&se->env, env_error_func, se);
 }
 
 static void stackenv_uninit(stackenv* se) {
   upb_env_uninit(&se->env);
-  upb_seededalloc_uninit(&se->alloc);
 }
 
 /*

ruby/ext/google/protobuf_c/message.c

@@ -151,32 +151,30 @@ VALUE Message_method_missing(int argc, VALUE* argv, VALUE _self) {
     name_len--;
   }
 
-  // Check for a oneof name first.
-  o = upb_msgdef_ntoo(self->descriptor->msgdef,
-                                          name, name_len);
+  // See if this name corresponds to either a oneof or field in this message.
+  if (!upb_msgdef_lookupname(self->descriptor->msgdef, name, name_len, &f,
+                             &o)) {
+    return rb_call_super(argc, argv);
+  }
+
   if (o != NULL) {
+    // This is a oneof -- return which field inside the oneof is set.
     if (setter) {
       rb_raise(rb_eRuntimeError, "Oneof accessors are read-only.");
     }
     return which_oneof_field(self, o);
-  }
-
-  // Otherwise, check for a field with that name.
-  f = upb_msgdef_ntof(self->descriptor->msgdef,
-                                          name, name_len);
-
-  if (f == NULL) {
-    return rb_call_super(argc, argv);
-  }
-
-  if (setter) {
-    if (argc < 2) {
-      rb_raise(rb_eArgError, "No value provided to setter.");
-    }
-    layout_set(self->descriptor->layout, Message_data(self), f, argv[1]);
-    return Qnil;
   } else {
-    return layout_get(self->descriptor->layout, Message_data(self), f);
+    // This is a field -- get or set the field's value.
+    assert(f);
+    if (setter) {
+      if (argc < 2) {
+        rb_raise(rb_eArgError, "No value provided to setter.");
+      }
+      layout_set(self->descriptor->layout, Message_data(self), f, argv[1]);
+      return Qnil;
+    } else {
+      return layout_get(self->descriptor->layout, Message_data(self), f);
+    }
   }
 }
 

ruby/lib/google/protobuf/repeated_field.rb

@@ -69,8 +69,8 @@ module Google
       #        relationship explicit instead of implicit
       def_delegators :to_ary,
         :&, :*, :-, :'<=>',
-        :assoc, :bsearch, :combination, :compact, :count, :cycle,
-        :drop, :drop_while, :eql?, :fetch, :find_index, :flatten,
+        :assoc, :bsearch, :bsearch_index, :combination, :compact, :count,
+        :cycle, :dig, :drop, :drop_while, :eql?, :fetch, :find_index, :flatten,
         :include?, :index, :inspect, :join,
         :pack, :permutation, :product, :pretty_print, :pretty_print_cycle,
         :rassoc, :repeated_combination, :repeated_permutation, :reverse,

src/Makefile.am

@@ -507,6 +507,8 @@ protoc_inputs =                                                   \
   google/protobuf/unittest_preserve_unknown_enum.proto            \
   google/protobuf/unittest.proto                                  \
   google/protobuf/unittest_proto3_arena.proto                     \
+  google/protobuf/unittest_proto3_arena_lite.proto                     \
+  google/protobuf/unittest_proto3_lite.proto                     \
   google/protobuf/unittest_well_known_types.proto                 \
   google/protobuf/util/internal/testdata/anys.proto               \
   google/protobuf/util/internal/testdata/books.proto              \
@@ -609,6 +611,10 @@ protoc_outputs =                                                  \
   google/protobuf/unittest_preserve_unknown_enum.pb.h             \
   google/protobuf/unittest_proto3_arena.pb.cc                     \
   google/protobuf/unittest_proto3_arena.pb.h                      \
+  google/protobuf/unittest_proto3_arena_lite.pb.cc                     \
+  google/protobuf/unittest_proto3_arena_lite.pb.h                      \
+  google/protobuf/unittest_proto3_lite.pb.cc                     \
+  google/protobuf/unittest_proto3_lite.pb.h                      \
   google/protobuf/unittest_well_known_types.pb.cc                 \
   google/protobuf/unittest_well_known_types.pb.h                  \
   google/protobuf/util/internal/testdata/anys.pb.cc               \
@@ -710,6 +716,8 @@ protobuf_test_SOURCES =                                        \
   google/protobuf/no_field_presence_test.cc                    \
   google/protobuf/preserve_unknown_enum_test.cc                \
   google/protobuf/proto3_arena_unittest.cc                     \
+  google/protobuf/proto3_arena_lite_unittest.cc                     \
+  google/protobuf/proto3_lite_unittest.cc                     \
   google/protobuf/reflection_ops_unittest.cc                   \
   google/protobuf/repeated_field_reflection_unittest.cc        \
   google/protobuf/repeated_field_unittest.cc                   \

src/google/protobuf/api.pb.cc

@@ -996,8 +996,16 @@ Method* Method::New(::google::protobuf::Arena* arena) const {
 
 void Method::Clear() {
 // @@protoc_insertion_point(message_clear_start:google.protobuf.Method)
+#if defined(__clang__)
+#define ZR_HELPER_(f) \
+  _Pragma("clang diagnostic push") \
+  _Pragma("clang diagnostic ignored \"-Winvalid-offsetof\"") \
+  __builtin_offsetof(Method, f) \
+  _Pragma("clang diagnostic pop")
+#else
 #define ZR_HELPER_(f) reinterpret_cast<char*>(\
   &reinterpret_cast<Method*>(16)->f)
+#endif
 
 #define ZR_(first, last) do {\
   ::memset(&first, 0,\

src/google/protobuf/compiler/objectivec/objectivec_helpers.cc

@@ -757,10 +757,11 @@ bool HasNonZeroDefaultValue(const FieldDescriptor* field) {
     return false;
   }
 
-  if (!field->has_default_value()) {
-    // No custom default set in the proto file.
-    return false;
-  }
+  // As much as checking field->has_default_value() seems useful, it isn't
+  // because of enums. proto2 syntax allows the first item in an enum (the
+  // default) to be non zero. So checking field->has_default_value() would
+  // result in missing this non zero default.  See MessageWithOneBasedEnum in
+  // objectivec/Tests/unittest_objc.proto for a test Message to confirm this.
 
   // Some proto file set the default to the zero value, so make sure the value
   // isn't the zero case.

src/google/protobuf/descriptor.pb.cc

@@ -2491,8 +2491,16 @@ DescriptorProto_ExtensionRange* DescriptorProto_ExtensionRange::New(::google::pr
 
 void DescriptorProto_ExtensionRange::Clear() {
 // @@protoc_insertion_point(message_clear_start:google.protobuf.DescriptorProto.ExtensionRange)
+#if defined(__clang__)
+#define ZR_HELPER_(f) \
+  _Pragma("clang diagnostic push") \
+  _Pragma("clang diagnostic ignored \"-Winvalid-offsetof\"") \
+  __builtin_offsetof(DescriptorProto_ExtensionRange, f) \
+  _Pragma("clang diagnostic pop")
+#else
 #define ZR_HELPER_(f) reinterpret_cast<char*>(\
   &reinterpret_cast<DescriptorProto_ExtensionRange*>(16)->f)
+#endif
 
 #define ZR_(first, last) do {\
   ::memset(&first, 0,\
@@ -2782,8 +2790,16 @@ DescriptorProto_ReservedRange* DescriptorProto_ReservedRange::New(::google::prot
 
 void DescriptorProto_ReservedRange::Clear() {
 // @@protoc_insertion_point(message_clear_start:google.protobuf.DescriptorProto.ReservedRange)
+#if defined(__clang__)
+#define ZR_HELPER_(f) \
+  _Pragma("clang diagnostic push") \
+  _Pragma("clang diagnostic ignored \"-Winvalid-offsetof\"") \
+  __builtin_offsetof(DescriptorProto_ReservedRange, f) \
+  _Pragma("clang diagnostic pop")
+#else
 #define ZR_HELPER_(f) reinterpret_cast<char*>(\
   &reinterpret_cast<DescriptorProto_ReservedRange*>(16)->f)
+#endif
 
 #define ZR_(first, last) do {\
   ::memset(&first, 0,\
@@ -7161,8 +7177,16 @@ MethodDescriptorProto* MethodDescriptorProto::New(::google::protobuf::Arena* are
 
 void MethodDescriptorProto::Clear() {
 // @@protoc_insertion_point(message_clear_start:google.protobuf.MethodDescriptorProto)
+#if defined(__clang__)
+#define ZR_HELPER_(f) \
+  _Pragma("clang diagnostic push") \
+  _Pragma("clang diagnostic ignored \"-Winvalid-offsetof\"") \
+  __builtin_offsetof(MethodDescriptorProto, f) \
+  _Pragma("clang diagnostic pop")
+#else
 #define ZR_HELPER_(f) reinterpret_cast<char*>(\
   &reinterpret_cast<MethodDescriptorProto*>(16)->f)
+#endif
 
 #define ZR_(first, last) do {\
   ::memset(&first, 0,\
@@ -7965,8 +7989,16 @@ FileOptions* FileOptions::New(::google::protobuf::Arena* arena) const {
 void FileOptions::Clear() {
 // @@protoc_insertion_point(message_clear_start:google.protobuf.FileOptions)
   _extensions_.Clear();
+#if defined(__clang__)
+#define ZR_HELPER_(f) \
+  _Pragma("clang diagnostic push") \
+  _Pragma("clang diagnostic ignored \"-Winvalid-offsetof\"") \
+  __builtin_offsetof(FileOptions, f) \
+  _Pragma("clang diagnostic pop")
+#else
 #define ZR_HELPER_(f) reinterpret_cast<char*>(\
   &reinterpret_cast<FileOptions*>(16)->f)
+#endif
 
 #define ZR_(first, last) do {\
   ::memset(&first, 0,\
@@ -9360,8 +9392,16 @@ MessageOptions* MessageOptions::New(::google::protobuf::Arena* arena) const {
 void MessageOptions::Clear() {
 // @@protoc_insertion_point(message_clear_start:google.protobuf.MessageOptions)
   _extensions_.Clear();
+#if defined(__clang__)
+#define ZR_HELPER_(f) \
+  _Pragma("clang diagnostic push") \
+  _Pragma("clang diagnostic ignored \"-Winvalid-offsetof\"") \
+  __builtin_offsetof(MessageOptions, f) \
+  _Pragma("clang diagnostic pop")
+#else
 #define ZR_HELPER_(f) reinterpret_cast<char*>(\
   &reinterpret_cast<MessageOptions*>(16)->f)
+#endif
 
 #define ZR_(first, last) do {\
   ::memset(&first, 0,\
@@ -9961,8 +10001,16 @@ FieldOptions* FieldOptions::New(::google::protobuf::Arena* arena) const {
 void FieldOptions::Clear() {
 // @@protoc_insertion_point(message_clear_start:google.protobuf.FieldOptions)
   _extensions_.Clear();
+#if defined(__clang__)
+#define ZR_HELPER_(f) \
+  _Pragma("clang diagnostic push") \
+  _Pragma("clang diagnostic ignored \"-Winvalid-offsetof\"") \
+  __builtin_offsetof(FieldOptions, f) \
+  _Pragma("clang diagnostic pop")
+#else
 #define ZR_HELPER_(f) reinterpret_cast<char*>(\
   &reinterpret_cast<FieldOptions*>(16)->f)
+#endif
 
 #define ZR_(first, last) do {\
   ::memset(&first, 0,\
@@ -10645,8 +10693,16 @@ EnumOptions* EnumOptions::New(::google::protobuf::Arena* arena) const {
 void EnumOptions::Clear() {
 // @@protoc_insertion_point(message_clear_start:google.protobuf.EnumOptions)
   _extensions_.Clear();
+#if defined(__clang__)
+#define ZR_HELPER_(f) \
+  _Pragma("clang diagnostic push") \
+  _Pragma("clang diagnostic ignored \"-Winvalid-offsetof\"") \
+  __builtin_offsetof(EnumOptions, f) \
+  _Pragma("clang diagnostic pop")
+#else
 #define ZR_HELPER_(f) reinterpret_cast<char*>(\
   &reinterpret_cast<EnumOptions*>(16)->f)
+#endif
 
 #define ZR_(first, last) do {\
   ::memset(&first, 0,\
@@ -12473,8 +12529,16 @@ UninterpretedOption* UninterpretedOption::New(::google::protobuf::Arena* arena)
 
 void UninterpretedOption::Clear() {
 // @@protoc_insertion_point(message_clear_start:google.protobuf.UninterpretedOption)
+#if defined(__clang__)
+#define ZR_HELPER_(f) \
+  _Pragma("clang diagnostic push") \
+  _Pragma("clang diagnostic ignored \"-Winvalid-offsetof\"") \
+  __builtin_offsetof(UninterpretedOption, f) \
+  _Pragma("clang diagnostic pop")
+#else
 #define ZR_HELPER_(f) reinterpret_cast<char*>(\
   &reinterpret_cast<UninterpretedOption*>(16)->f)
+#endif
 
 #define ZR_(first, last) do {\
   ::memset(&first, 0,\
@@ -14338,8 +14402,16 @@ GeneratedCodeInfo_Annotation* GeneratedCodeInfo_Annotation::New(::google::protob
 
 void GeneratedCodeInfo_Annotation::Clear() {
 // @@protoc_insertion_point(message_clear_start:google.protobuf.GeneratedCodeInfo.Annotation)
+#if defined(__clang__)
+#define ZR_HELPER_(f) \
+  _Pragma("clang diagnostic push") \
+  _Pragma("clang diagnostic ignored \"-Winvalid-offsetof\"") \
+  __builtin_offsetof(GeneratedCodeInfo_Annotation, f) \
+  _Pragma("clang diagnostic pop")
+#else
 #define ZR_HELPER_(f) reinterpret_cast<char*>(\
   &reinterpret_cast<GeneratedCodeInfo_Annotation*>(16)->f)
+#endif
 
 #define ZR_(first, last) do {\
   ::memset(&first, 0,\

src/google/protobuf/duration.pb.cc

@@ -180,8 +180,16 @@ Duration* Duration::New(::google::protobuf::Arena* arena) const {
 
 void Duration::Clear() {
 // @@protoc_insertion_point(message_clear_start:google.protobuf.Duration)
+#if defined(__clang__)
+#define ZR_HELPER_(f) \
+  _Pragma("clang diagnostic push") \
+  _Pragma("clang diagnostic ignored \"-Winvalid-offsetof\"") \
+  __builtin_offsetof(Duration, f) \
+  _Pragma("clang diagnostic pop")
+#else
 #define ZR_HELPER_(f) reinterpret_cast<char*>(\
   &reinterpret_cast<Duration*>(16)->f)
+#endif
 
 #define ZR_(first, last) do {\
   ::memset(&first, 0,\

src/google/protobuf/proto3_arena_lite_unittest.cc

@@ -0,0 +1,164 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include <string>
+#include <memory>
+#ifndef _SHARED_PTR_H
+#include <google/protobuf/stubs/shared_ptr.h>
+#endif
+#include <vector>
+
+#include <google/protobuf/test_util.h>
+#include <google/protobuf/unittest.pb.h>
+#include <google/protobuf/unittest_proto3_arena_lite.pb.h>
+#include <google/protobuf/arena.h>
+#include <google/protobuf/testing/googletest.h>
+#include <gtest/gtest.h>
+
+namespace google {
+using proto3_arena_lite_unittest::TestAllTypes;
+
+namespace protobuf {
+namespace {
+// We selectively set/check a few representative fields rather than all fields
+// as this test is only expected to cover the basics of arena support.
+void SetAllFields(TestAllTypes* m) {
+  m->set_optional_int32(100);
+  m->set_optional_string("asdf");
+  m->set_optional_bytes("jkl;");
+  m->mutable_optional_nested_message()->set_bb(42);
+  m->mutable_optional_foreign_message()->set_c(43);
+  m->set_optional_nested_enum(
+      proto3_arena_lite_unittest::TestAllTypes_NestedEnum_BAZ);
+  m->set_optional_foreign_enum(
+      proto3_arena_lite_unittest::FOREIGN_BAZ);
+  m->mutable_optional_lazy_message()->set_bb(45);
+  m->add_repeated_int32(100);
+  m->add_repeated_string("asdf");
+  m->add_repeated_bytes("jkl;");
+  m->add_repeated_nested_message()->set_bb(46);
+  m->add_repeated_foreign_message()->set_c(47);
+  m->add_repeated_nested_enum(
+      proto3_arena_lite_unittest::TestAllTypes_NestedEnum_BAZ);
+  m->add_repeated_foreign_enum(
+      proto3_arena_lite_unittest::FOREIGN_BAZ);
+  m->add_repeated_lazy_message()->set_bb(49);
+
+  m->set_oneof_uint32(1);
+  m->mutable_oneof_nested_message()->set_bb(50);
+  m->set_oneof_string("test");  // only this one remains set
+}
+
+void ExpectAllFieldsSet(const TestAllTypes& m) {
+  EXPECT_EQ(100, m.optional_int32());
+  EXPECT_EQ("asdf", m.optional_string());
+  EXPECT_EQ("jkl;", m.optional_bytes());
+  EXPECT_EQ(true, m.has_optional_nested_message());
+  EXPECT_EQ(42, m.optional_nested_message().bb());
+  EXPECT_EQ(true, m.has_optional_foreign_message());
+  EXPECT_EQ(43, m.optional_foreign_message().c());
+  EXPECT_EQ(proto3_arena_lite_unittest::TestAllTypes_NestedEnum_BAZ,
+            m.optional_nested_enum());
+  EXPECT_EQ(proto3_arena_lite_unittest::FOREIGN_BAZ,
+            m.optional_foreign_enum());
+  EXPECT_EQ(true, m.has_optional_lazy_message());
+  EXPECT_EQ(45, m.optional_lazy_message().bb());
+
+  EXPECT_EQ(1, m.repeated_int32_size());
+  EXPECT_EQ(100, m.repeated_int32(0));
+  EXPECT_EQ(1, m.repeated_string_size());
+  EXPECT_EQ("asdf", m.repeated_string(0));
+  EXPECT_EQ(1, m.repeated_bytes_size());
+  EXPECT_EQ("jkl;", m.repeated_bytes(0));
+  EXPECT_EQ(1, m.repeated_nested_message_size());
+  EXPECT_EQ(46, m.repeated_nested_message(0).bb());
+  EXPECT_EQ(1, m.repeated_foreign_message_size());
+  EXPECT_EQ(47, m.repeated_foreign_message(0).c());
+  EXPECT_EQ(1, m.repeated_nested_enum_size());
+  EXPECT_EQ(proto3_arena_lite_unittest::TestAllTypes_NestedEnum_BAZ,
+            m.repeated_nested_enum(0));
+  EXPECT_EQ(1, m.repeated_foreign_enum_size());
+  EXPECT_EQ(proto3_arena_lite_unittest::FOREIGN_BAZ,
+            m.repeated_foreign_enum(0));
+  EXPECT_EQ(1, m.repeated_lazy_message_size());
+  EXPECT_EQ(49, m.repeated_lazy_message(0).bb());
+
+  EXPECT_EQ(proto3_arena_lite_unittest::TestAllTypes::kOneofString,
+            m.oneof_field_case());
+  EXPECT_EQ("test", m.oneof_string());
+}
+
+// In this file we only test some basic functionalities of arena support in
+// proto3 and expect the arena support to be fully tested in proto2 unittests
+// because proto3 shares most code with proto2.
+
+TEST(Proto3ArenaLiteTest, Parsing) {
+  TestAllTypes original;
+  SetAllFields(&original);
+
+  Arena arena;
+  TestAllTypes* arena_message = Arena::CreateMessage<TestAllTypes>(&arena);
+  arena_message->ParseFromString(original.SerializeAsString());
+  ExpectAllFieldsSet(*arena_message);
+}
+
+TEST(Proto3ArenaLiteTest, Swap) {
+  Arena arena1;
+  Arena arena2;
+
+  // Test Swap().
+  TestAllTypes* arena1_message = Arena::CreateMessage<TestAllTypes>(&arena1);
+  TestAllTypes* arena2_message = Arena::CreateMessage<TestAllTypes>(&arena2);
+  arena1_message->Swap(arena2_message);
+  EXPECT_EQ(&arena1, arena1_message->GetArena());
+  EXPECT_EQ(&arena2, arena2_message->GetArena());
+}
+
+TEST(Proto3ArenaLiteTest, SetAllocatedMessage) {
+  Arena arena;
+  TestAllTypes *arena_message = Arena::CreateMessage<TestAllTypes>(&arena);
+  TestAllTypes::NestedMessage* nested = new TestAllTypes::NestedMessage;
+  nested->set_bb(118);
+  arena_message->set_allocated_optional_nested_message(nested);
+  EXPECT_EQ(118, arena_message->optional_nested_message().bb());
+}
+
+TEST(Proto3ArenaLiteTest, ReleaseMessage) {
+  Arena arena;
+  TestAllTypes* arena_message = Arena::CreateMessage<TestAllTypes>(&arena);
+  arena_message->mutable_optional_nested_message()->set_bb(118);
+  google::protobuf::scoped_ptr<TestAllTypes::NestedMessage> nested(
+      arena_message->release_optional_nested_message());
+  EXPECT_EQ(118, nested->bb());
+}
+
+}  // namespace
+}  // namespace protobuf
+}  // namespace google

src/google/protobuf/proto3_lite_unittest.cc

@@ -0,0 +1,145 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include <string>
+#include <memory>
+#ifndef _SHARED_PTR_H
+#include <google/protobuf/stubs/shared_ptr.h>
+#endif
+#include <vector>
+
+#include <google/protobuf/test_util.h>
+#include <google/protobuf/unittest.pb.h>
+#include <google/protobuf/unittest_proto3_lite.pb.h>
+#include <google/protobuf/arena.h>
+#include <google/protobuf/testing/googletest.h>
+#include <gtest/gtest.h>
+
+namespace google {
+using proto3_lite_unittest::TestAllTypes;
+
+namespace protobuf {
+namespace {
+// We selectively set/check a few representative fields rather than all fields
+// as this test is only expected to cover the basics of lite support.
+void SetAllFields(TestAllTypes* m) {
+  m->set_optional_int32(100);
+  m->set_optional_string("asdf");
+  m->set_optional_bytes("jkl;");
+  m->mutable_optional_nested_message()->set_bb(42);
+  m->mutable_optional_foreign_message()->set_c(43);
+  m->set_optional_nested_enum(
+      proto3_lite_unittest::TestAllTypes_NestedEnum_BAZ);
+  m->set_optional_foreign_enum(
+      proto3_lite_unittest::FOREIGN_BAZ);
+  m->mutable_optional_lazy_message()->set_bb(45);
+  m->add_repeated_int32(100);
+  m->add_repeated_string("asdf");
+  m->add_repeated_bytes("jkl;");
+  m->add_repeated_nested_message()->set_bb(46);
+  m->add_repeated_foreign_message()->set_c(47);
+  m->add_repeated_nested_enum(
+      proto3_lite_unittest::TestAllTypes_NestedEnum_BAZ);
+  m->add_repeated_foreign_enum(
+      proto3_lite_unittest::FOREIGN_BAZ);
+  m->add_repeated_lazy_message()->set_bb(49);
+
+  m->set_oneof_uint32(1);
+  m->mutable_oneof_nested_message()->set_bb(50);
+  m->set_oneof_string("test");  // only this one remains set
+}
+
+void ExpectAllFieldsSet(const TestAllTypes& m) {
+  EXPECT_EQ(100, m.optional_int32());
+  EXPECT_EQ("asdf", m.optional_string());
+  EXPECT_EQ("jkl;", m.optional_bytes());
+  EXPECT_EQ(true, m.has_optional_nested_message());
+  EXPECT_EQ(42, m.optional_nested_message().bb());
+  EXPECT_EQ(true, m.has_optional_foreign_message());
+  EXPECT_EQ(43, m.optional_foreign_message().c());
+  EXPECT_EQ(proto3_lite_unittest::TestAllTypes_NestedEnum_BAZ,
+            m.optional_nested_enum());
+  EXPECT_EQ(proto3_lite_unittest::FOREIGN_BAZ,
+            m.optional_foreign_enum());
+  EXPECT_EQ(true, m.has_optional_lazy_message());
+  EXPECT_EQ(45, m.optional_lazy_message().bb());
+
+  EXPECT_EQ(1, m.repeated_int32_size());
+  EXPECT_EQ(100, m.repeated_int32(0));
+  EXPECT_EQ(1, m.repeated_string_size());
+  EXPECT_EQ("asdf", m.repeated_string(0));
+  EXPECT_EQ(1, m.repeated_bytes_size());
+  EXPECT_EQ("jkl;", m.repeated_bytes(0));
+  EXPECT_EQ(1, m.repeated_nested_message_size());
+  EXPECT_EQ(46, m.repeated_nested_message(0).bb());
+  EXPECT_EQ(1, m.repeated_foreign_message_size());
+  EXPECT_EQ(47, m.repeated_foreign_message(0).c());
+  EXPECT_EQ(1, m.repeated_nested_enum_size());
+  EXPECT_EQ(proto3_lite_unittest::TestAllTypes_NestedEnum_BAZ,
+            m.repeated_nested_enum(0));
+  EXPECT_EQ(1, m.repeated_foreign_enum_size());
+  EXPECT_EQ(proto3_lite_unittest::FOREIGN_BAZ,
+            m.repeated_foreign_enum(0));
+  EXPECT_EQ(1, m.repeated_lazy_message_size());
+  EXPECT_EQ(49, m.repeated_lazy_message(0).bb());
+
+  EXPECT_EQ(proto3_lite_unittest::TestAllTypes::kOneofString,
+            m.oneof_field_case());
+  EXPECT_EQ("test", m.oneof_string());
+}
+
+// In this file we only test some basic functionalities of in proto3 and expect
+// the rest is fully tested in proto2 unittests because proto3 shares most code
+// with proto2.
+
+TEST(Proto3LiteTest, Parsing) {
+  TestAllTypes original;
+  SetAllFields(&original);
+
+  TestAllTypes msg;
+  msg.ParseFromString(original.SerializeAsString());
+  ExpectAllFieldsSet(msg);
+}
+
+TEST(Proto3LiteTest, Swap) {
+  // Test Swap().
+  TestAllTypes msg1;
+  TestAllTypes msg2;
+  msg1.set_optional_string("123");
+  msg2.set_optional_string("3456");
+  msg1.Swap(&msg2);
+  EXPECT_EQ("3456", msg1.optional_string());
+  EXPECT_EQ("123", msg2.optional_string());
+  EXPECT_EQ(msg1.ByteSize(), msg2.ByteSize() + 1);
+}
+
+}  // namespace
+}  // namespace protobuf
+}  // namespace google

src/google/protobuf/timestamp.pb.cc

@@ -194,8 +194,16 @@ Timestamp* Timestamp::New(::google::protobuf::Arena* arena) const {
 
 void Timestamp::Clear() {
 // @@protoc_insertion_point(message_clear_start:google.protobuf.Timestamp)
+#if defined(__clang__)
+#define ZR_HELPER_(f) \
+  _Pragma("clang diagnostic push") \
+  _Pragma("clang diagnostic ignored \"-Winvalid-offsetof\"") \
+  __builtin_offsetof(Timestamp, f) \
+  _Pragma("clang diagnostic pop")
+#else
 #define ZR_HELPER_(f) reinterpret_cast<char*>(\
   &reinterpret_cast<Timestamp*>(16)->f)
+#endif
 
 #define ZR_(first, last) do {\
   ::memset(&first, 0,\

src/google/protobuf/type.pb.cc

@@ -1116,8 +1116,16 @@ Field* Field::New(::google::protobuf::Arena* arena) const {
 
 void Field::Clear() {
 // @@protoc_insertion_point(message_clear_start:google.protobuf.Field)
+#if defined(__clang__)
+#define ZR_HELPER_(f) \
+  _Pragma("clang diagnostic push") \
+  _Pragma("clang diagnostic ignored \"-Winvalid-offsetof\"") \
+  __builtin_offsetof(Field, f) \
+  _Pragma("clang diagnostic pop")
+#else
 #define ZR_HELPER_(f) reinterpret_cast<char*>(\
   &reinterpret_cast<Field*>(16)->f)
+#endif
 
 #define ZR_(first, last) do {\
   ::memset(&first, 0,\

src/google/protobuf/unittest_proto3_arena_lite.proto

@@ -0,0 +1,207 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+syntax = "proto3";
+
+option cc_enable_arenas = true;
+option optimize_for = LITE_RUNTIME;
+
+import "google/protobuf/unittest_import.proto";
+
+package proto3_arena_lite_unittest;
+
+// This proto includes every type of field in both singular and repeated
+// forms.
+message TestAllTypes {
+  message NestedMessage {
+    // The field name "b" fails to compile in proto1 because it conflicts with
+    // a local variable named "b" in one of the generated methods.  Doh.
+    // This file needs to compile in proto1 to test backwards-compatibility.
+    int32 bb = 1;
+  }
+
+  enum NestedEnum {
+    ZERO = 0;
+    FOO = 1;
+    BAR = 2;
+    BAZ = 3;
+    NEG = -1;  // Intentionally negative.
+  }
+
+  // Singular
+     int32 optional_int32    =  1;
+     int64 optional_int64    =  2;
+    uint32 optional_uint32   =  3;
+    uint64 optional_uint64   =  4;
+    sint32 optional_sint32   =  5;
+    sint64 optional_sint64   =  6;
+   fixed32 optional_fixed32  =  7;
+   fixed64 optional_fixed64  =  8;
+  sfixed32 optional_sfixed32 =  9;
+  sfixed64 optional_sfixed64 = 10;
+     float optional_float    = 11;
+    double optional_double   = 12;
+      bool optional_bool     = 13;
+    string optional_string   = 14;
+     bytes optional_bytes    = 15;
+
+  // Groups are not allowed in proto3.
+  // optional group OptionalGroup = 16 {
+  //   optional int32 a = 17;
+  // }
+
+  NestedMessage                        optional_nested_message  = 18;
+  ForeignMessage                       optional_foreign_message = 19;
+  protobuf_unittest_import.ImportMessage optional_import_message  = 20;
+
+  NestedEnum                           optional_nested_enum     = 21;
+  ForeignEnum                          optional_foreign_enum    = 22;
+
+  // Omitted (compared to unittest.proto) because proto2 enums are not allowed
+  // inside proto2 messages.
+  //
+  // optional protobuf_unittest_import.ImportEnum    optional_import_enum  = 23;
+
+  string optional_string_piece = 24 [ctype=STRING_PIECE];
+  string optional_cord = 25 [ctype=CORD];
+
+  // Defined in unittest_import_public.proto
+  protobuf_unittest_import.PublicImportMessage
+      optional_public_import_message = 26;
+
+  NestedMessage optional_lazy_message = 27 [lazy=true];
+
+  // Repeated
+  repeated    int32 repeated_int32    = 31;
+  repeated    int64 repeated_int64    = 32;
+  repeated   uint32 repeated_uint32   = 33;
+  repeated   uint64 repeated_uint64   = 34;
+  repeated   sint32 repeated_sint32   = 35;
+  repeated   sint64 repeated_sint64   = 36;
+  repeated  fixed32 repeated_fixed32  = 37;
+  repeated  fixed64 repeated_fixed64  = 38;
+  repeated sfixed32 repeated_sfixed32 = 39;
+  repeated sfixed64 repeated_sfixed64 = 40;
+  repeated    float repeated_float    = 41;
+  repeated   double repeated_double   = 42;
+  repeated     bool repeated_bool     = 43;
+  repeated   string repeated_string   = 44;
+  repeated    bytes repeated_bytes    = 45;
+
+  // Groups are not allowed in proto3.
+  // repeated group RepeatedGroup = 46 {
+  //   optional int32 a = 47;
+  // }
+
+  repeated NestedMessage                        repeated_nested_message  = 48;
+  repeated ForeignMessage                       repeated_foreign_message = 49;
+  repeated protobuf_unittest_import.ImportMessage repeated_import_message  = 50;
+
+  repeated NestedEnum                           repeated_nested_enum     = 51;
+  repeated ForeignEnum                          repeated_foreign_enum    = 52;
+
+  // Omitted (compared to unittest.proto) because proto2 enums are not allowed
+  // inside proto2 messages.
+  //
+  // repeated protobuf_unittest_import.ImportEnum    repeated_import_enum  = 53;
+
+  repeated string repeated_string_piece = 54 [ctype=STRING_PIECE];
+  repeated string repeated_cord = 55 [ctype=CORD];
+
+  repeated NestedMessage repeated_lazy_message = 57 [lazy=true];
+
+  oneof oneof_field {
+    uint32 oneof_uint32 = 111;
+    NestedMessage oneof_nested_message = 112;
+    string oneof_string = 113;
+    bytes oneof_bytes = 114;
+  }
+}
+
+// Test messages for packed fields
+
+message TestPackedTypes {
+  repeated    int32 packed_int32    =  90 [packed = true];
+  repeated    int64 packed_int64    =  91 [packed = true];
+  repeated   uint32 packed_uint32   =  92 [packed = true];
+  repeated   uint64 packed_uint64   =  93 [packed = true];
+  repeated   sint32 packed_sint32   =  94 [packed = true];
+  repeated   sint64 packed_sint64   =  95 [packed = true];
+  repeated  fixed32 packed_fixed32  =  96 [packed = true];
+  repeated  fixed64 packed_fixed64  =  97 [packed = true];
+  repeated sfixed32 packed_sfixed32 =  98 [packed = true];
+  repeated sfixed64 packed_sfixed64 =  99 [packed = true];
+  repeated    float packed_float    = 100 [packed = true];
+  repeated   double packed_double   = 101 [packed = true];
+  repeated     bool packed_bool     = 102 [packed = true];
+  repeated ForeignEnum packed_enum  = 103 [packed = true];
+}
+
+// Explicitly set packed to false
+message TestUnpackedTypes {
+  repeated    int32 repeated_int32    =  1 [packed = false];
+  repeated    int64 repeated_int64    =  2 [packed = false];
+  repeated   uint32 repeated_uint32   =  3 [packed = false];
+  repeated   uint64 repeated_uint64   =  4 [packed = false];
+  repeated   sint32 repeated_sint32   =  5 [packed = false];
+  repeated   sint64 repeated_sint64   =  6 [packed = false];
+  repeated  fixed32 repeated_fixed32  =  7 [packed = false];
+  repeated  fixed64 repeated_fixed64  =  8 [packed = false];
+  repeated sfixed32 repeated_sfixed32 =  9 [packed = false];
+  repeated sfixed64 repeated_sfixed64 = 10 [packed = false];
+  repeated    float repeated_float    = 11 [packed = false];
+  repeated   double repeated_double   = 12 [packed = false];
+  repeated     bool repeated_bool     = 13 [packed = false];
+  repeated TestAllTypes.NestedEnum repeated_nested_enum = 14 [packed = false];
+}
+
+// This proto includes a recusively nested message.
+message NestedTestAllTypes {
+  NestedTestAllTypes child = 1;
+  TestAllTypes payload = 2;
+}
+
+// Define these after TestAllTypes to make sure the compiler can handle
+// that.
+message ForeignMessage {
+  int32 c = 1;
+}
+
+enum ForeignEnum {
+  FOREIGN_ZERO = 0;
+  FOREIGN_FOO = 4;
+  FOREIGN_BAR = 5;
+  FOREIGN_BAZ = 6;
+}
+
+// TestEmptyMessage is used to test behavior of unknown fields.
+message TestEmptyMessage {
+}
+

src/google/protobuf/unittest_proto3_lite.proto

@@ -0,0 +1,206 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+syntax = "proto3";
+
+option optimize_for = LITE_RUNTIME;
+
+import "google/protobuf/unittest_import.proto";
+
+package proto3_lite_unittest;
+
+// This proto includes every type of field in both singular and repeated
+// forms.
+message TestAllTypes {
+  message NestedMessage {
+    // The field name "b" fails to compile in proto1 because it conflicts with
+    // a local variable named "b" in one of the generated methods.  Doh.
+    // This file needs to compile in proto1 to test backwards-compatibility.
+    int32 bb = 1;
+  }
+
+  enum NestedEnum {
+    ZERO = 0;
+    FOO = 1;
+    BAR = 2;
+    BAZ = 3;
+    NEG = -1;  // Intentionally negative.
+  }
+
+  // Singular
+     int32 optional_int32    =  1;
+     int64 optional_int64    =  2;
+    uint32 optional_uint32   =  3;
+    uint64 optional_uint64   =  4;
+    sint32 optional_sint32   =  5;
+    sint64 optional_sint64   =  6;
+   fixed32 optional_fixed32  =  7;
+   fixed64 optional_fixed64  =  8;
+  sfixed32 optional_sfixed32 =  9;
+  sfixed64 optional_sfixed64 = 10;
+     float optional_float    = 11;
+    double optional_double   = 12;
+      bool optional_bool     = 13;
+    string optional_string   = 14;
+     bytes optional_bytes    = 15;
+
+  // Groups are not allowed in proto3.
+  // optional group OptionalGroup = 16 {
+  //   optional int32 a = 17;
+  // }
+
+  NestedMessage                        optional_nested_message  = 18;
+  ForeignMessage                       optional_foreign_message = 19;
+  protobuf_unittest_import.ImportMessage optional_import_message  = 20;
+
+  NestedEnum                           optional_nested_enum     = 21;
+  ForeignEnum                          optional_foreign_enum    = 22;
+
+  // Omitted (compared to unittest.proto) because proto2 enums are not allowed
+  // inside proto2 messages.
+  //
+  // optional protobuf_unittest_import.ImportEnum    optional_import_enum  = 23;
+
+  string optional_string_piece = 24 [ctype=STRING_PIECE];
+  string optional_cord = 25 [ctype=CORD];
+
+  // Defined in unittest_import_public.proto
+  protobuf_unittest_import.PublicImportMessage
+      optional_public_import_message = 26;
+
+  NestedMessage optional_lazy_message = 27 [lazy=true];
+
+  // Repeated
+  repeated    int32 repeated_int32    = 31;
+  repeated    int64 repeated_int64    = 32;
+  repeated   uint32 repeated_uint32   = 33;
+  repeated   uint64 repeated_uint64   = 34;
+  repeated   sint32 repeated_sint32   = 35;
+  repeated   sint64 repeated_sint64   = 36;
+  repeated  fixed32 repeated_fixed32  = 37;
+  repeated  fixed64 repeated_fixed64  = 38;
+  repeated sfixed32 repeated_sfixed32 = 39;
+  repeated sfixed64 repeated_sfixed64 = 40;
+  repeated    float repeated_float    = 41;
+  repeated   double repeated_double   = 42;
+  repeated     bool repeated_bool     = 43;
+  repeated   string repeated_string   = 44;
+  repeated    bytes repeated_bytes    = 45;
+
+  // Groups are not allowed in proto3.
+  // repeated group RepeatedGroup = 46 {
+  //   optional int32 a = 47;
+  // }
+
+  repeated NestedMessage                        repeated_nested_message  = 48;
+  repeated ForeignMessage                       repeated_foreign_message = 49;
+  repeated protobuf_unittest_import.ImportMessage repeated_import_message  = 50;
+
+  repeated NestedEnum                           repeated_nested_enum     = 51;
+  repeated ForeignEnum                          repeated_foreign_enum    = 52;
+
+  // Omitted (compared to unittest.proto) because proto2 enums are not allowed
+  // inside proto2 messages.
+  //
+  // repeated protobuf_unittest_import.ImportEnum    repeated_import_enum  = 53;
+
+  repeated string repeated_string_piece = 54 [ctype=STRING_PIECE];
+  repeated string repeated_cord = 55 [ctype=CORD];
+
+  repeated NestedMessage repeated_lazy_message = 57 [lazy=true];
+
+  oneof oneof_field {
+    uint32 oneof_uint32 = 111;
+    NestedMessage oneof_nested_message = 112;
+    string oneof_string = 113;
+    bytes oneof_bytes = 114;
+  }
+}
+
+// Test messages for packed fields
+
+message TestPackedTypes {
+  repeated    int32 packed_int32    =  90 [packed = true];
+  repeated    int64 packed_int64    =  91 [packed = true];
+  repeated   uint32 packed_uint32   =  92 [packed = true];
+  repeated   uint64 packed_uint64   =  93 [packed = true];
+  repeated   sint32 packed_sint32   =  94 [packed = true];
+  repeated   sint64 packed_sint64   =  95 [packed = true];
+  repeated  fixed32 packed_fixed32  =  96 [packed = true];
+  repeated  fixed64 packed_fixed64  =  97 [packed = true];
+  repeated sfixed32 packed_sfixed32 =  98 [packed = true];
+  repeated sfixed64 packed_sfixed64 =  99 [packed = true];
+  repeated    float packed_float    = 100 [packed = true];
+  repeated   double packed_double   = 101 [packed = true];
+  repeated     bool packed_bool     = 102 [packed = true];
+  repeated ForeignEnum packed_enum  = 103 [packed = true];
+}
+
+// Explicitly set packed to false
+message TestUnpackedTypes {
+  repeated    int32 repeated_int32    =  1 [packed = false];
+  repeated    int64 repeated_int64    =  2 [packed = false];
+  repeated   uint32 repeated_uint32   =  3 [packed = false];
+  repeated   uint64 repeated_uint64   =  4 [packed = false];
+  repeated   sint32 repeated_sint32   =  5 [packed = false];
+  repeated   sint64 repeated_sint64   =  6 [packed = false];
+  repeated  fixed32 repeated_fixed32  =  7 [packed = false];
+  repeated  fixed64 repeated_fixed64  =  8 [packed = false];
+  repeated sfixed32 repeated_sfixed32 =  9 [packed = false];
+  repeated sfixed64 repeated_sfixed64 = 10 [packed = false];
+  repeated    float repeated_float    = 11 [packed = false];
+  repeated   double repeated_double   = 12 [packed = false];
+  repeated     bool repeated_bool     = 13 [packed = false];
+  repeated TestAllTypes.NestedEnum repeated_nested_enum = 14 [packed = false];
+}
+
+// This proto includes a recusively nested message.
+message NestedTestAllTypes {
+  NestedTestAllTypes child = 1;
+  TestAllTypes payload = 2;
+}
+
+// Define these after TestAllTypes to make sure the compiler can handle
+// that.
+message ForeignMessage {
+  int32 c = 1;
+}
+
+enum ForeignEnum {
+  FOREIGN_ZERO = 0;
+  FOREIGN_FOO = 4;
+  FOREIGN_BAR = 5;
+  FOREIGN_BAZ = 6;
+}
+
+// TestEmptyMessage is used to test behavior of unknown fields.
+message TestEmptyMessage {
+}
+