Add tests for field presence and default values

Adjust APIs for extensions to properly return default values for extensions
Fix issues with IsInitialized and proto2 field reflection

csharp/protos/unittest.proto

@@ -764,6 +764,12 @@ message TestRequiredOneof {
   }
 }
 
+message TestRequiredMap {
+  map<int32, NestedMessage> foo = 1;
+  message NestedMessage {
+    required int32 required_int32 = 1;
+  }
+}
 
 // Test messages for packed fields
 

csharp/src/Google.Protobuf.Test/GeneratedMessageTest.cs

@@ -33,6 +33,7 @@
 using System;
 using System.IO;
 using Google.Protobuf.TestProtos;
+using Proto2 = Google.Protobuf.TestProtos.Proto2;
 using NUnit.Framework;
 using System.Collections;
 using System.Collections.Generic;
@@ -111,6 +112,50 @@ namespace Google.Protobuf
             Assert.AreEqual("", message.OneofString);
             Assert.AreEqual(ByteString.Empty, message.OneofBytes);
             Assert.IsNull(message.OneofNestedMessage);
+
+            // proto2 default values
+            var message2 = new Proto2.TestAllTypes();
+            Assert.AreEqual(true, message2.DefaultBool);
+            Assert.AreEqual(ByteString.CopyFromUtf8("world"), message2.DefaultBytes);
+            Assert.AreEqual("123", message2.DefaultCord);
+            Assert.AreEqual(52e3, message2.DefaultDouble);
+            Assert.AreEqual(47, message2.DefaultFixed32);
+            Assert.AreEqual(48, message2.DefaultFixed64);
+            Assert.AreEqual(51.5, message2.DefaultFloat);
+            Assert.AreEqual(Proto2.ForeignEnum.ForeignBar, message2.DefaultForeignEnum);
+            Assert.AreEqual(Proto2.ImportEnum.ImportBar, message2.DefaultImportEnum);
+            Assert.AreEqual(41, message2.DefaultInt32);
+            Assert.AreEqual(42, message2.DefaultInt64);
+            Assert.AreEqual(Proto2.TestAllTypes.Types.NestedEnum.Bar, message2.DefaultNestedEnum);
+            Assert.AreEqual(49, message2.DefaultSfixed32);
+            Assert.AreEqual(-50, message2.DefaultSfixed64);
+            Assert.AreEqual(-45, message2.DefaultSint32);
+            Assert.AreEqual(46, message2.DefaultSint64);
+            Assert.AreEqual("hello", message2.DefaultString);
+            Assert.AreEqual("abc", message2.DefaultStringPiece);
+            Assert.AreEqual(43, message2.DefaultUint32);
+            Assert.AreEqual(44, message2.DefaultUint64);
+
+            Assert.False(message2.HasDefaultBool);
+            Assert.False(message2.HasDefaultBytes);
+            Assert.False(message2.HasDefaultCord);
+            Assert.False(message2.HasDefaultDouble);
+            Assert.False(message2.HasDefaultFixed32);
+            Assert.False(message2.HasDefaultFixed64);
+            Assert.False(message2.HasDefaultFloat);
+            Assert.False(message2.HasDefaultForeignEnum);
+            Assert.False(message2.HasDefaultImportEnum);
+            Assert.False(message2.HasDefaultInt32);
+            Assert.False(message2.HasDefaultInt64);
+            Assert.False(message2.HasDefaultNestedEnum);
+            Assert.False(message2.HasDefaultSfixed32);
+            Assert.False(message2.HasDefaultSfixed64);
+            Assert.False(message2.HasDefaultSint32);
+            Assert.False(message2.HasDefaultSint64);
+            Assert.False(message2.HasDefaultString);
+            Assert.False(message2.HasDefaultStringPiece);
+            Assert.False(message2.HasDefaultUint32);
+            Assert.False(message2.HasDefaultUint64);
         }
 
         [Test]
@@ -123,6 +168,107 @@ namespace Google.Protobuf
             Assert.Throws<ArgumentNullException>(() => message.OneofBytes = null);
         }
 
+        [Test]
+        public void FieldPresence()
+        {
+            var message = new Proto2.TestAllTypes();
+
+            Assert.False(message.HasOptionalBool);
+            Assert.False(message.OptionalBool);
+
+            message.OptionalBool = true;
+
+            Assert.True(message.HasOptionalBool);
+            Assert.True(message.OptionalBool);
+
+            message.OptionalBool = false;
+
+            Assert.True(message.HasOptionalBool);
+            Assert.False(message.OptionalBool);
+
+            message.ClearOptionalBool();
+
+            Assert.False(message.HasOptionalBool);
+            Assert.False(message.OptionalBool);
+
+            Assert.False(message.HasDefaultBool);
+            Assert.True(message.DefaultBool);
+
+            message.DefaultBool = false;
+
+            Assert.True(message.HasDefaultBool);
+            Assert.False(message.DefaultBool);
+
+            message.DefaultBool = true;
+
+            Assert.True(message.HasDefaultBool);
+            Assert.True(message.DefaultBool);
+
+            message.ClearDefaultBool();
+
+            Assert.False(message.HasDefaultBool);
+            Assert.True(message.DefaultBool);
+        }
+
+        [Test]
+        public void RequiredFields()
+        {
+            var message = new Proto2.TestRequired();
+            Assert.False(message.IsInitialized());
+
+            message.A = 1;
+            message.B = 2;
+            message.C = 3;
+
+            Assert.True(message.IsInitialized());
+        }
+
+        [Test]
+        public void RequiredFieldsInExtensions()
+        {
+            var message = new Proto2.TestAllExtensions();
+            Assert.True(message.IsInitialized());
+
+            message.SetExtension(Proto2.TestRequired.Extensions.Single, new Proto2.TestRequired());
+
+            Assert.False(message.IsInitialized());
+
+            var extensionMessage = message.GetExtension(Proto2.TestRequired.Extensions.Single);
+            extensionMessage.A = 1;
+            extensionMessage.B = 2;
+            extensionMessage.C = 3;
+
+            Assert.True(message.IsInitialized());
+
+            message.RegisterExtension(Proto2.TestRequired.Extensions.Multi);
+
+            Assert.True(message.IsInitialized());
+
+            message.GetExtension(Proto2.TestRequired.Extensions.Multi).Add(new Proto2.TestRequired());
+
+            Assert.False(message.IsInitialized());
+
+            extensionMessage = message.GetExtension(Proto2.TestRequired.Extensions.Multi)[0];
+            extensionMessage.A = 1;
+            extensionMessage.B = 2;
+            extensionMessage.C = 3;
+
+            Assert.True(message.IsInitialized());
+        }
+
+        [Test]
+        public void RequiredFieldInNestedMessageMapValue()
+        {
+            var message = new Proto2.TestRequiredMap();
+            message.Foo.Add(0, new Proto2.TestRequiredMap.Types.NestedMessage());
+
+            Assert.False(message.IsInitialized());
+
+            message.Foo[0].RequiredInt32 = 12;
+
+            Assert.True(message.IsInitialized());
+        }
+
         [Test]
         public void RoundTrip_Empty()
         {

csharp/src/Google.Protobuf.Test/SampleMessages.cs

@@ -32,6 +32,7 @@
 
 using System;
 using Google.Protobuf.TestProtos;
+using Proto2 = Google.Protobuf.TestProtos.Proto2;
 
 namespace Google.Protobuf
 {
@@ -95,5 +96,56 @@ namespace Google.Protobuf
                 OneofString = "Oneof string"
             };
         }
+
+        public static Proto2.TestAllTypes CreateFullTestAllTypesProto2()
+        {
+            return new Proto2.TestAllTypes
+            {
+                OptionalBool = true,
+                OptionalBytes = ByteString.CopyFrom(1, 2, 3, 4),
+                OptionalDouble = 23.5,
+                OptionalFixed32 = 23,
+                OptionalFixed64 = 1234567890123,
+                OptionalFloat = 12.25f,
+                OptionalForeignEnum = Proto2.ForeignEnum.ForeignBar,
+                OptionalForeignMessage = new Proto2.ForeignMessage { C = 10 },
+                OptionalImportEnum = Proto2.ImportEnum.ImportBaz,
+                OptionalImportMessage = new Proto2.ImportMessage { D = 20 },
+                OptionalInt32 = 100,
+                OptionalInt64 = 3210987654321,
+                OptionalNestedEnum = Proto2.TestAllTypes.Types.NestedEnum.Foo,
+                OptionalNestedMessage = new Proto2.TestAllTypes.Types.NestedMessage { Bb = 35 },
+                OptionalPublicImportMessage = new Proto2.PublicImportMessage { E = 54 },
+                OptionalSfixed32 = -123,
+                OptionalSfixed64 = -12345678901234,
+                OptionalSint32 = -456,
+                OptionalSint64 = -12345678901235,
+                OptionalString = "test",
+                OptionalUint32 = UInt32.MaxValue,
+                OptionalUint64 = UInt64.MaxValue,
+                RepeatedBool = { true, false },
+                RepeatedBytes = { ByteString.CopyFrom(1, 2, 3, 4), ByteString.CopyFrom(5, 6), ByteString.CopyFrom(new byte[1000]) },
+                RepeatedDouble = { -12.25, 23.5 },
+                RepeatedFixed32 = { UInt32.MaxValue, 23 },
+                RepeatedFixed64 = { UInt64.MaxValue, 1234567890123 },
+                RepeatedFloat = { 100f, 12.25f },
+                RepeatedForeignEnum = { Proto2.ForeignEnum.ForeignFoo, Proto2.ForeignEnum.ForeignBar },
+                RepeatedForeignMessage = { new Proto2.ForeignMessage(), new Proto2.ForeignMessage { C = 10 } },
+                RepeatedImportEnum = { Proto2.ImportEnum.ImportBaz, Proto2.ImportEnum.ImportFoo },
+                RepeatedImportMessage = { new Proto2.ImportMessage { D = 20 }, new Proto2.ImportMessage { D = 25 } },
+                RepeatedInt32 = { 100, 200 },
+                RepeatedInt64 = { 3210987654321, Int64.MaxValue },
+                RepeatedNestedEnum = { Proto2.TestAllTypes.Types.NestedEnum.Foo, Proto2.TestAllTypes.Types.NestedEnum.Neg },
+                RepeatedNestedMessage = { new Proto2.TestAllTypes.Types.NestedMessage { Bb = 35 }, new Proto2.TestAllTypes.Types.NestedMessage { Bb = 10 } },
+                RepeatedSfixed32 = { -123, 123 },
+                RepeatedSfixed64 = { -12345678901234, 12345678901234 },
+                RepeatedSint32 = { -456, 100 },
+                RepeatedSint64 = { -12345678901235, 123 },
+                RepeatedString = { "foo", "bar" },
+                RepeatedUint32 = { UInt32.MaxValue, UInt32.MinValue },
+                RepeatedUint64 = { UInt64.MaxValue, UInt32.MinValue },
+                OneofString = "Oneof string"
+            };
+        }
     }
 }

csharp/src/Google.Protobuf/Extension.cs

@@ -79,6 +79,8 @@ namespace Google.Protobuf
 
         internal override Type TargetType => typeof(TTarget);
 
+        internal TValue DefaultValue => codec.DefaultValue;
+
         internal override IExtensionValue CreateValue()
         {
             return new ExtensionValue<TValue>(codec);

csharp/src/Google.Protobuf/ExtensionSet.cs

@@ -330,5 +330,10 @@ namespace Google.Protobuf
                 value.WriteTo(stream);
             }
         }
+
+        internal bool IsInitialized()
+        {
+            return ValuesByNumber.Values.All(v => v.IsInitialized());
+        }
     }
 }

csharp/src/Google.Protobuf/ExtensionValue.cs

@@ -32,6 +32,7 @@
 
 using Google.Protobuf.Collections;
 using System;
+using System.Linq;
 
 namespace Google.Protobuf
 {
@@ -41,6 +42,7 @@ namespace Google.Protobuf
         void MergeFrom(IExtensionValue value);
         void WriteTo(CodedOutputStream output);
         int CalculateSize();
+        bool IsInitialized();
     }
 
     internal sealed class ExtensionValue<T> : IExtensionValue
@@ -137,6 +139,11 @@ namespace Google.Protobuf
         }
 
         public bool HasValue => hasValue;
+
+        public bool IsInitialized()
+        {
+            return HasValue && field is IMessage && (field as IMessage).IsInitialized();
+        }
     }
 
     internal sealed class RepeatedExtensionValue<T> : IExtensionValue
@@ -203,5 +210,10 @@ namespace Google.Protobuf
         }
 
         public RepeatedField<T> GetValue() => field;
+
+        public bool IsInitialized()
+        {
+            return field.All(m => m is IMessage && (m as IMessage).IsInitialized());
+        }
     }
 }

csharp/src/Google.Protobuf/MessageExtensions.cs

@@ -148,11 +148,16 @@ namespace Google.Protobuf
         /// </summary>
         public static bool IsInitialized(this IMessage message)
         {
-            if (message.Descriptor.File.Proto.Syntax != "proto2")
+            if (message.Descriptor.File.Proto.Syntax == "proto3")
             {
                 return true;
             }
 
+            if (!message.Descriptor.GetIsExtensionsInitialized(message))
+            {
+                return false;
+            }
+
             return message.Descriptor
                 .Fields
                 .InDeclarationOrder()
@@ -160,8 +165,16 @@ namespace Google.Protobuf
                 {
                     if (f.IsMap)
                     {
-                        var map = (IDictionary)f.Accessor.GetValue(message);
-                        return map.Values.OfType<IMessage>().All(IsInitialized);
+                        var valueField = f.MessageType.Fields[2];
+                        if (valueField.FieldType == FieldType.Message)
+                        {
+                            var map = (IDictionary)f.Accessor.GetValue(message);
+                            return map.Values.Cast<IMessage>().All(IsInitialized);
+                        }
+                        else
+                        {
+                            return true;
+                        }
                     }
                     else if (f.IsRepeated && f.FieldType == FieldType.Message || f.FieldType == FieldType.Group)
                     {

csharp/src/Google.Protobuf/Reflection/MessageDescriptor.cs

@@ -34,6 +34,7 @@ using System;
 using System.Collections.Generic;
 using System.Collections.ObjectModel;
 using System.Linq;
+using System.Reflection;
 #if NET35
 // Needed for ReadOnlyDictionary, which does not exist in .NET 3.5
 using Google.Protobuf.Collections;
@@ -63,6 +64,7 @@ namespace Google.Protobuf.Reflection
         private readonly IList<FieldDescriptor> fieldsInDeclarationOrder;
         private readonly IList<FieldDescriptor> fieldsInNumberOrder;
         private readonly IDictionary<string, FieldDescriptor> jsonFieldMap;
+        private Func<IMessage, bool> extensionSetIsInitialized;
 
         internal MessageDescriptor(DescriptorProto proto, FileDescriptor file, MessageDescriptor parent, int typeIndex, GeneratedClrTypeInfo generatedCodeInfo)
             : base(file, file.ComputeFullName(parent, proto.Name), typeIndex)
@@ -134,6 +136,26 @@ namespace Google.Protobuf.Reflection
 
         internal DescriptorProto Proto { get; }
 
+        internal bool GetIsExtensionsInitialized(IMessage message)
+        {
+            if (!object.ReferenceEquals(message.Descriptor, this))
+            {
+                throw new InvalidOperationException("message's descriptor reference does not match this");
+            }
+
+            if (Proto.ExtensionRange.Count == 0)
+            {
+                return true;
+            }
+
+            if (extensionSetIsInitialized == null)
+            {
+                extensionSetIsInitialized = ReflectionUtil.CreateIsInitializedCaller(ClrType);
+            }
+
+            return extensionSetIsInitialized(message);
+        }
+
         /// <summary>
         /// The CLR type used to represent message instances from this descriptor.
         /// </summary>

csharp/src/Google.Protobuf/Reflection/ReflectionUtil.cs

@@ -71,7 +71,7 @@ namespace Google.Protobuf.Reflection
 
         /// <summary>
         /// Empty Type[] used when calling GetProperty to force property instead of indexer fetching.
-        /// </summary>
+        /// </summary>getFieldFunc
         internal static readonly Type[] EmptyTypes = new Type[0];
 
         /// <summary>
@@ -115,6 +115,9 @@ namespace Google.Protobuf.Reflection
         internal static Func<IMessage, bool> CreateFuncIMessageBool(MethodInfo method) =>
             GetReflectionHelper(method.DeclaringType, method.ReturnType).CreateFuncIMessageBool(method);
 
+        internal static Func<IMessage, bool> CreateIsInitializedCaller(Type msg) =>
+            ((IExtensionSetReflector)Activator.CreateInstance(typeof(ExtensionSetReflector<>).MakeGenericType(msg))).CreateIsInitializedCaller();
+
         /// <summary>
         /// Creates a delegate which will execute the given method after casting the first argument to
         /// the type that declares the method, and the second argument to the first parameter type of the method.
@@ -150,6 +153,11 @@ namespace Google.Protobuf.Reflection
             void ClearExtension(IMessage message);
         }
 
+        private interface IExtensionSetReflector
+        {
+            Func<IMessage, bool> CreateIsInitializedCaller();
+        }
+
         private class ReflectionHelper<T1, T2> : IReflectionHelper
         {
 
@@ -300,6 +308,23 @@ namespace Google.Protobuf.Reflection
             }
         }
 
+        private class ExtensionSetReflector<T1> : IExtensionSetReflector where T1 : IExtendableMessage<T1>
+        {
+            public Func<IMessage, bool> CreateIsInitializedCaller()
+            {
+                var field = typeof(T1).GetTypeInfo().GetDeclaredField("_extensions");
+                var initializedFunc = (Func<ExtensionSet<T1>, bool>)
+                    typeof(ExtensionSet<T1>)
+                        .GetTypeInfo()
+                        .GetDeclaredMethod("IsInitialized")
+                        .CreateDelegate(typeof(Func<ExtensionSet<T1>, bool>));
+                return (m) => {
+                    var set = field.GetValue(m) as ExtensionSet<T1>;
+                    return set == null || initializedFunc(set);
+                };
+            }
+        }
+
         // Runtime compatibility checking code - see ReflectionHelper<T1, T2>.CreateFuncIMessageInt32 for
         // details about why we're doing this.
 

csharp/src/Google.Protobuf/Reflection/SingleFieldAccessor.cs

@@ -57,20 +57,7 @@ namespace Google.Protobuf.Reflection
                 throw new ArgumentException("Not all required properties/methods available");
             }
             setValueDelegate = ReflectionUtil.CreateActionIMessageObject(property.GetSetMethod());
-            if (descriptor.File.Proto.Syntax == "proto2")
-            {
-                MethodInfo hasMethod = property.DeclaringType.GetRuntimeProperty("Has" + property.Name).GetMethod;
-                if (hasMethod == null) {
-                  throw new ArgumentException("Not all required properties/methods are available");
-                }
-                hasDelegate = ReflectionUtil.CreateFuncIMessageBool(hasMethod);
-                MethodInfo clearMethod = property.DeclaringType.GetRuntimeMethod("Clear" + property.Name, ReflectionUtil.EmptyTypes);
-                if (clearMethod == null) {
-                  throw new ArgumentException("Not all required properties/methods are available");
-                }
-                clearDelegate = ReflectionUtil.CreateActionIMessage(clearMethod);
-            }
-            else
+            if (descriptor.File.Proto.Syntax == "proto3")
             {
                 hasDelegate = message => {
                   throw new InvalidOperationException("HasValue is not implemented for proto3 fields");
@@ -85,6 +72,19 @@ namespace Google.Protobuf.Reflection
                     : Activator.CreateInstance(clrType);
                 clearDelegate = message => SetValue(message, defaultValue);
             }
+            else
+            {
+                MethodInfo hasMethod = property.DeclaringType.GetRuntimeProperty("Has" + property.Name).GetMethod;
+                if (hasMethod == null) {
+                  throw new ArgumentException("Not all required properties/methods are available");
+                }
+                hasDelegate = ReflectionUtil.CreateFuncIMessageBool(hasMethod);
+                MethodInfo clearMethod = property.DeclaringType.GetRuntimeMethod("Clear" + property.Name, ReflectionUtil.EmptyTypes);
+                if (clearMethod == null) {
+                  throw new ArgumentException("Not all required properties/methods are available");
+                }
+                clearDelegate = ReflectionUtil.CreateActionIMessage(clearMethod);
+            }
         }
 
         public override void Clear(IMessage message)