More tests for CodedInputStream, and some more WireFormat

committer: Jon Skeet <skeet@pobox.com>

csharp/ProtocolBuffers.Test/CodedInputStreamTest.cs

@@ -2,16 +2,153 @@
 using System.Collections.Generic;
 using System.Text;
 using NUnit.Framework;
+using System.IO;
 
 namespace Google.ProtocolBuffers {
   [TestFixture]
   public class CodedInputStreamTest {
-    
+
+    /// <summary>
+    /// Helper to construct a byte array from a bunch of bytes.  The inputs are
+    /// actually ints so that I can use hex notation and not get stupid errors
+    /// about precision.
+    /// </summary>
+    private static byte[] Bytes(params int[] bytesAsInts) {
+      byte[] bytes = new byte[bytesAsInts.Length];
+      for (int i = 0; i < bytesAsInts.Length; i++) {
+        bytes[i] = (byte)bytesAsInts[i];
+      }
+      return bytes;
+    }
+
+    /// <summary>
+    /// Parses the given bytes using ReadRawVarint32() and ReadRawVarint64() and
+    /// </summary>
+    private static void AssertReadVarint(byte[] data, ulong value) {
+      CodedInputStream input = CodedInputStream.CreateInstance(data);
+      Assert.AreEqual((uint)value, input.ReadRawVarint32());
+
+      input = CodedInputStream.CreateInstance(data);
+      Assert.AreEqual(value, input.ReadRawVarint64());
+
+      // Try different block sizes.
+      for (int bufferSize = 1; bufferSize <= 16; bufferSize *= 2) {
+        input = CodedInputStream.CreateInstance(new SmallBlockInputStream(data, bufferSize));
+        Assert.AreEqual((uint)value, input.ReadRawVarint32());
+
+        input = CodedInputStream.CreateInstance(new SmallBlockInputStream(data, bufferSize));
+        Assert.AreEqual(value, input.ReadRawVarint64());
+      }
+    }
+
+    /// <summary>
+    /// Parses the given bytes using ReadRawVarint32() and ReadRawVarint64() and
+    /// expects them to fail with an InvalidProtocolBufferException whose
+    /// description matches the given one.
+    /// </summary>
+    private void AssertReadVarintFailure(InvalidProtocolBufferException expected, byte[] data) {
+      CodedInputStream input = CodedInputStream.CreateInstance(data);
+      try {
+        input.ReadRawVarint32();
+        Assert.Fail("Should have thrown an exception.");
+      } catch (InvalidProtocolBufferException e) {
+        Assert.AreEqual(expected.Message, e.Message);
+      }
+
+      input = CodedInputStream.CreateInstance(data);
+      try {
+        input.ReadRawVarint64();
+        Assert.Fail("Should have thrown an exception.");
+      } catch (InvalidProtocolBufferException e) {
+        Assert.AreEqual(expected.Message, e.Message);
+      }
+    }
+
+    [Test]
+    public void ReadVarint() {
+      AssertReadVarint(Bytes(0x00), 0);
+      AssertReadVarint(Bytes(0x01), 1);
+      AssertReadVarint(Bytes(0x7f), 127);
+      // 14882
+      AssertReadVarint(Bytes(0xa2, 0x74), (0x22 << 0) | (0x74 << 7));
+      // 2961488830
+      AssertReadVarint(Bytes(0xbe, 0xf7, 0x92, 0x84, 0x0b),
+        (0x3e << 0) | (0x77 << 7) | (0x12 << 14) | (0x04 << 21) |
+        (0x0bL << 28));
+
+      // 64-bit
+      // 7256456126
+      AssertReadVarint(Bytes(0xbe, 0xf7, 0x92, 0x84, 0x1b),
+        (0x3e << 0) | (0x77 << 7) | (0x12 << 14) | (0x04 << 21) |
+        (0x1bL << 28));
+      // 41256202580718336
+      AssertReadVarint(Bytes(0x80, 0xe6, 0xeb, 0x9c, 0xc3, 0xc9, 0xa4, 0x49),
+        (0x00 << 0) | (0x66 << 7) | (0x6b << 14) | (0x1c << 21) |
+        (0x43L << 28) | (0x49L << 35) | (0x24L << 42) | (0x49L << 49));
+      // 11964378330978735131
+      AssertReadVarint(Bytes(0x9b, 0xa8, 0xf9, 0xc2, 0xbb, 0xd6, 0x80, 0x85, 0xa6, 0x01),
+        (0x1b << 0) | (0x28 << 7) | (0x79 << 14) | (0x42 << 21) |
+        (0x3bUL << 28) | (0x56UL << 35) | (0x00UL << 42) |
+        (0x05UL << 49) | (0x26UL << 56) | (0x01UL << 63));
+
+      // Failures
+      AssertReadVarintFailure(
+        InvalidProtocolBufferException.MalformedVarint(),
+        Bytes(0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
+              0x00));
+      AssertReadVarintFailure(
+        InvalidProtocolBufferException.TruncatedMessage(),
+        Bytes(0x80));
+    }
+
+    /// <summary>
+    /// Parses the given bytes using ReadRawLittleEndian32() and checks
+    /// that the result matches the given value.
+    /// </summary>
+    private static void AssertReadLittleEndian32(byte[] data, int value) {
+      CodedInputStream input = CodedInputStream.CreateInstance(data);
+      Assert.AreEqual(value, input.ReadRawLittleEndian32());
+
+      // Try different block sizes.
+      for (int blockSize = 1; blockSize <= 16; blockSize *= 2) {
+        input = CodedInputStream.CreateInstance(
+          new SmallBlockInputStream(data, blockSize));
+        Assert.AreEqual(value, input.ReadRawLittleEndian32());
+      }
+    }
+
+    /// <summary>
+    /// Parses the given bytes using ReadRawLittleEndian64() and checks
+    /// that the result matches the given value.
+    /// </summary>
+    private static void AssertReadLittleEndian64(byte[] data, long value) {
+      CodedInputStream input = CodedInputStream.CreateInstance(data);
+      Assert.AreEqual(value, input.ReadRawLittleEndian64());
+
+      // Try different block sizes.
+      for (int blockSize = 1; blockSize <= 16; blockSize *= 2) {
+        input = CodedInputStream.CreateInstance(
+          new SmallBlockInputStream(data, blockSize));
+        Assert.AreEqual(value, input.ReadRawLittleEndian64());
+      }
+    }
+
+    [Test]
+    public void ReadLittleEndian() {
+      AssertReadLittleEndian32(Bytes(0x78, 0x56, 0x34, 0x12), 0x12345678);
+      AssertReadLittleEndian32(Bytes(0xf0, 0xde, 0xbc, 0x9a), unchecked((int)0x9abcdef0));
+
+      AssertReadLittleEndian64(Bytes(0xf0, 0xde, 0xbc, 0x9a, 0x78, 0x56, 0x34, 0x12),
+        0x123456789abcdef0L);
+      AssertReadLittleEndian64(
+        Bytes(0x78, 0x56, 0x34, 0x12, 0xf0, 0xde, 0xbc, 0x9a), unchecked((long)0x9abcdef012345678L));
+    }
+
     [Test]
     public void DecodeZigZag32() {
-      Assert.AreEqual( 0, CodedInputStream.DecodeZigZag32(0));
+      Assert.AreEqual(0, CodedInputStream.DecodeZigZag32(0));
       Assert.AreEqual(-1, CodedInputStream.DecodeZigZag32(1));
-      Assert.AreEqual( 1, CodedInputStream.DecodeZigZag32(2));
+      Assert.AreEqual(1, CodedInputStream.DecodeZigZag32(2));
       Assert.AreEqual(-2, CodedInputStream.DecodeZigZag32(3));
       Assert.AreEqual(0x3FFFFFFF, CodedInputStream.DecodeZigZag32(0x7FFFFFFE));
       Assert.AreEqual(unchecked((int)0xC0000000), CodedInputStream.DecodeZigZag32(0x7FFFFFFF));
@@ -21,16 +158,208 @@ namespace Google.ProtocolBuffers {
 
     [Test]
     public void DecodeZigZag64() {
-      Assert.AreEqual( 0, CodedInputStream.DecodeZigZag64(0));
+      Assert.AreEqual(0, CodedInputStream.DecodeZigZag64(0));
       Assert.AreEqual(-1, CodedInputStream.DecodeZigZag64(1));
-      Assert.AreEqual( 1, CodedInputStream.DecodeZigZag64(2));
+      Assert.AreEqual(1, CodedInputStream.DecodeZigZag64(2));
       Assert.AreEqual(-2, CodedInputStream.DecodeZigZag64(3));
       Assert.AreEqual(0x000000003FFFFFFFL, CodedInputStream.DecodeZigZag64(0x000000007FFFFFFEL));
       Assert.AreEqual(unchecked((long)0xFFFFFFFFC0000000L), CodedInputStream.DecodeZigZag64(0x000000007FFFFFFFL));
       Assert.AreEqual(0x000000007FFFFFFFL, CodedInputStream.DecodeZigZag64(0x00000000FFFFFFFEL));
       Assert.AreEqual(unchecked((long)0xFFFFFFFF80000000L), CodedInputStream.DecodeZigZag64(0x00000000FFFFFFFFL));
       Assert.AreEqual(0x7FFFFFFFFFFFFFFFL, CodedInputStream.DecodeZigZag64(0xFFFFFFFFFFFFFFFEL));
-      Assert.AreEqual(unchecked((long)0x8000000000000000L),CodedInputStream.DecodeZigZag64(0xFFFFFFFFFFFFFFFFL));
+      Assert.AreEqual(unchecked((long)0x8000000000000000L), CodedInputStream.DecodeZigZag64(0xFFFFFFFFFFFFFFFFL));
+    }
+
+    /* TODO(jonskeet): Reinstate this when protoc is ready
+    public void testReadWholeMessage() throws Exception {
+      TestAllTypes message = TestUtil.getAllSet();
+
+      byte[] rawBytes = message.toByteArray();
+      assertEquals(rawBytes.length, message.getSerializedSize());
+
+      TestAllTypes message2 = TestAllTypes.parseFrom(rawBytes);
+      TestUtil.assertAllFieldsSet(message2);
+
+      // Try different block sizes.
+      for (int blockSize = 1; blockSize < 256; blockSize *= 2) {
+        message2 = TestAllTypes.parseFrom(
+          new SmallBlockInputStream(rawBytes, blockSize));
+        TestUtil.assertAllFieldsSet(message2);
+      }
+    }*/
+
+    /* TODO(jonskeet): Reinstate this when protoc is ready
+    public void testSkipWholeMessage() throws Exception {
+      TestAllTypes message = TestUtil.getAllSet();
+      byte[] rawBytes = message.toByteArray();
+
+      // Create two parallel inputs.  Parse one as unknown fields while using
+      // skipField() to skip each field on the other.  Expect the same tags.
+      CodedInputStream input1 = CodedInputStream.newInstance(rawBytes);
+      CodedInputStream input2 = CodedInputStream.newInstance(rawBytes);
+      UnknownFieldSet.Builder unknownFields = UnknownFieldSet.newBuilder();
+
+      while (true) {
+        int tag = input1.readTag();
+        assertEquals(tag, input2.readTag());
+        if (tag == 0) {
+          break;
+        }
+        unknownFields.mergeFieldFrom(tag, input1);
+        input2.skipField(tag);
+      }
+    }*/
+
+    /* TODO(jonskeet): Reinstate this when protoc is ready
+    public void testReadHugeBlob() throws Exception {
+      // Allocate and initialize a 1MB blob.
+      byte[] blob = new byte[1 << 20];
+      for (int i = 0; i < blob.length; i++) {
+        blob[i] = (byte)i;
+      }
+
+      // Make a message containing it.
+      TestAllTypes.Builder builder = TestAllTypes.newBuilder();
+      TestUtil.setAllFields(builder);
+      builder.setOptionalBytes(ByteString.copyFrom(blob));
+      TestAllTypes message = builder.build();
+
+      // Serialize and parse it.  Make sure to parse from an InputStream, not
+      // directly from a ByteString, so that CodedInputStream uses buffered
+      // reading.
+      TestAllTypes message2 =
+        TestAllTypes.parseFrom(message.toByteString().newInput());
+
+      assertEquals(message.getOptionalBytes(), message2.getOptionalBytes());
+
+      // Make sure all the other fields were parsed correctly.
+      TestAllTypes message3 = TestAllTypes.newBuilder(message2)
+        .setOptionalBytes(TestUtil.getAllSet().getOptionalBytes())
+        .build();
+      TestUtil.assertAllFieldsSet(message3);
+    }*/
+
+    [Test]
+    public void ReadMaliciouslyLargeBlob() {
+      MemoryStream ms = new MemoryStream();
+      CodedOutputStream output = CodedOutputStream.CreateInstance(ms);
+
+      uint tag = WireFormat.MakeTag(1, WireFormat.WireType.LengthDelimited);
+      output.WriteRawVarint32(tag);
+      output.WriteRawVarint32(0x7FFFFFFF);
+      output.WriteRawBytes(new byte[32]);  // Pad with a few random bytes.
+      output.Flush();
+      ms.Position = 0;
+
+      CodedInputStream input = CodedInputStream.CreateInstance(ms);
+      Assert.AreEqual(tag, input.ReadTag());
+
+      try {
+        input.ReadBytes();
+        Assert.Fail("Should have thrown an exception!");
+      } catch (InvalidProtocolBufferException) {
+        // success.
+      }
+    }
+
+    
+    /* TODO(jonskeet): Reinstate this when protoc is ready
+    private TestRecursiveMessage makeRecursiveMessage(int depth) {
+      if (depth == 0) {
+        return TestRecursiveMessage.newBuilder().setI(5).build();
+      } else {
+        return TestRecursiveMessage.newBuilder()
+          .setA(makeRecursiveMessage(depth - 1)).build();
+      }
+    }
+
+    private void assertMessageDepth(TestRecursiveMessage message, int depth) {
+      if (depth == 0) {
+        assertFalse(message.hasA());
+        assertEquals(5, message.getI());
+      } else {
+        assertTrue(message.hasA());
+        assertMessageDepth(message.getA(), depth - 1);
+      }
+    }
+
+    public void testMaliciousRecursion() {
+      ByteString data64 = makeRecursiveMessage(64).toByteString();
+      ByteString data65 = makeRecursiveMessage(65).toByteString();
+
+      assertMessageDepth(TestRecursiveMessage.parseFrom(data64), 64);
+
+      try {
+        TestRecursiveMessage.parseFrom(data65);
+        fail("Should have thrown an exception!");
+      } catch (InvalidProtocolBufferException e) {
+        // success.
+      }
+
+      CodedInputStream input = data64.newCodedInput();
+      input.setRecursionLimit(8);
+      try {
+        TestRecursiveMessage.parseFrom(input);
+        fail("Should have thrown an exception!");
+      } catch (InvalidProtocolBufferException e) {
+        // success.
+      }
+    }
+     */
+
+     /* TODO(jonskeet): Reinstate this when protoc is ready
+    public void testSizeLimit() throws Exception {
+      CodedInputStream input = CodedInputStream.newInstance(
+        TestUtil.getAllSet().toByteString().newInput());
+      input.setSizeLimit(16);
+
+      try {
+        TestAllTypes.parseFrom(input);
+        fail("Should have thrown an exception!");
+      } catch (InvalidProtocolBufferException e) {
+        // success.
+      }
+    }*/
+
+    /// <summary>
+    /// Tests that if we read an string that contains invalid UTF-8, no exception
+    /// is thrown.  Instead, the invalid bytes are replaced with the Unicode
+    /// "replacement character" U+FFFD.
+    /// </summary>
+    [Test]
+    public void ReadInvalidUtf8()  {
+      MemoryStream ms = new MemoryStream();
+      CodedOutputStream output = CodedOutputStream.CreateInstance(ms);
+
+      uint tag = WireFormat.MakeTag(1, WireFormat.WireType.LengthDelimited);
+      output.WriteRawVarint32(tag);
+      output.WriteRawVarint32(1);
+      output.WriteRawBytes(new byte[] { 0x80 });
+      output.Flush();
+      ms.Position = 0;
+
+      CodedInputStream input = CodedInputStream.CreateInstance(ms);
+      Assert.AreEqual(tag, input.ReadTag());
+      string text = input.ReadString();
+      Assert.AreEqual('\ufffd', text[0]);
+    }
+
+    /// <summary>
+    /// A stream which limits the number of bytes it reads at a time.
+    /// We use this to make sure that CodedInputStream doesn't screw up when
+    /// reading in small blocks.
+    /// </summary>
+    private sealed class SmallBlockInputStream : MemoryStream {
+      private readonly int blockSize;
+
+      public SmallBlockInputStream(byte[] data, int blockSize)
+        : base(data) {
+        this.blockSize = blockSize;
+      }
+
+      public override int Read(byte[] buffer, int offset, int count) {
+        return base.Read(buffer, offset, Math.Min(count, blockSize));
+      }
     }
   }
 }

csharp/ProtocolBuffers/CodedInputStream.cs

@@ -37,11 +37,11 @@ namespace Google.ProtocolBuffers {
   /// set at construction time.
   /// </remarks>
   public sealed class CodedInputStream {
-    private byte[] buffer;
+    private readonly byte[] buffer;
     private int bufferSize;
     private int bufferSizeAfterLimit = 0;
     private int bufferPos = 0;
-    private Stream input;
+    private readonly Stream input;
     private uint lastTag = 0;
 
     const int DefaultRecursionLimit = 64;
@@ -102,14 +102,6 @@ namespace Google.ProtocolBuffers {
     #endregion
 
     #region Uncategorised (TODO: Fix this!)
-      /*
-   * Verifies that the last call to readTag() returned the given tag value.
-   * This is used to verify that a nested group ended with the correct
-   * end tag.
-   *
-   * @throws InvalidProtocolBufferException {@code value} does not match the
-   *                                        last tag.
-   */
     /// <summary>
     /// Verifies that the last call to ReadTag() returned the given tag value.
     /// This is used to verify that a nested group ended with the correct
@@ -131,7 +123,7 @@ namespace Google.ProtocolBuffers {
     /// since a protocol message may legally end wherever a tag occurs, and
     /// zero is not a valid tag number.
     /// </summary>
-    public int ReadTag() {
+    public uint ReadTag() {
       if (bufferPos == bufferSize && !RefillBuffer(false)) {
         lastTag = 0;
         return 0;
@@ -142,7 +134,7 @@ namespace Google.ProtocolBuffers {
         // If we actually read zero, that's not a valid tag.
         throw InvalidProtocolBufferException.InvalidTag();
       }
-      return (int) lastTag;
+      return lastTag;
     }
 
     /// <summary>
@@ -324,16 +316,10 @@ namespace Google.ProtocolBuffers {
       return DecodeZigZag64(ReadRawVarint64());
     }
 
-    /**
-     * Read a field of any primitive type.  Enums, groups, and embedded
-     * messages are not handled by this method.
-     *
-     * @param type Declared type of the field.
-     * @return An object representing the field's value, of the exact
-     *         type which would be returned by
-     *         {@link Message#getField(Descriptors.FieldDescriptor)} for
-     *         this field.
-     */
+    /// <summary>
+    /// Reads a field of any primitive type. Enums, groups and embedded
+    /// messages are not handled by this method.
+    /// </summary>
     public object readPrimitiveField(Descriptors.FieldDescriptor.Type fieldType) {
       switch (fieldType) {
         case Descriptors.FieldDescriptor.Type.Double:   return ReadDouble();
@@ -372,35 +358,35 @@ namespace Google.ProtocolBuffers {
     /// </summary>
     /// <returns></returns>
     public uint ReadRawVarint32() {
-      uint tmp = ReadRawByte();
-      if (tmp >= 0) {
-        return tmp;
+      int tmp = ReadRawByte();
+      if (tmp < 128) {
+        return (uint) tmp;
       }
-      uint result = tmp & 0x7f;
-      if ((tmp =ReadRawByte()) >= 0) {
+      int result = tmp & 0x7f;
+      if ((tmp = ReadRawByte()) < 128) {
         result |= tmp << 7;
       } else {
         result |= (tmp & 0x7f) << 7;
-        if ((tmp = ReadRawByte()) >= 0) {
+        if ((tmp = ReadRawByte()) < 128) {
           result |= tmp << 14;
         } else {
           result |= (tmp & 0x7f) << 14;
-          if ((tmp = ReadRawByte()) >= 0) {
+          if ((tmp = ReadRawByte()) < 128) {
             result |= tmp << 21;
           } else {
             result |= (tmp & 0x7f) << 21;
             result |= (tmp = ReadRawByte()) << 28;
-            if (tmp < 0) {
+            if (tmp >= 128) {
               // Discard upper 32 bits.
               for (int i = 0; i < 5; i++) {
-                if (ReadRawByte() >= 0) return result;
+                if (ReadRawByte() < 128) return (uint) result;
               }
               throw InvalidProtocolBufferException.MalformedVarint();
             }
           }
         }
       }
-      return result;
+      return (uint) result;
     }
 
     /// <summary>
@@ -443,7 +429,7 @@ namespace Google.ProtocolBuffers {
       long b6 = ReadRawByte();
       long b7 = ReadRawByte();
       long b8 = ReadRawByte();
-      return b1 | (b2 << 8) | (b3 << 8) | (b4 << 8)
+      return b1 | (b2 << 8) | (b3 << 16) | (b4 << 24)
           | (b5 << 32) | (b6 << 40) | (b7 << 48) | (b8 << 56);
     }
     #endregion
@@ -689,9 +675,8 @@ namespace Google.ProtocolBuffers {
           byte[] chunk = new byte[Math.Min(sizeLeft, BufferSize)];
           int pos = 0;
           while (pos < chunk.Length) {
-            int n = (input == null) ? -1 :
-              input.Read(chunk, pos, chunk.Length - pos);
-            if (n == -1) {
+            int n = (input == null) ? -1 : input.Read(chunk, pos, chunk.Length - pos);
+            if (n <= 0) {
               throw InvalidProtocolBufferException.TruncatedMessage();
             }
             totalBytesRetired += n;

csharp/ProtocolBuffers/IBuilder.cs

@@ -1,12 +1,63 @@
 using System;
 using System.Collections.Generic;
-using System.Text;
 using System.IO;
 
 namespace Google.ProtocolBuffers {
 
+  /// <summary>
+  /// Non-generic interface for all members whose signatures don't require knowledge of
+  /// the type being built. The generic interface extends this one. Some methods return
+  /// either an IBuilder or an IMessage; in these cases the generic interface redeclares
+  /// the same method with a type-specific signature. Implementations are encouraged to
+  /// use explicit interface implemenation for the non-generic form. This mirrors
+  /// how IEnumerable and IEnumerable&lt;T&gt; work.
+  /// </summary>
   public interface IBuilder {
-    void MergeFrom(CodedInputStream codedInputStream, ExtensionRegistry extensionRegistry);
+    IBuilder MergeFrom(CodedInputStream codedInputStream, ExtensionRegistry extensionRegistry);
+    /// <summary>
+    /// Returns true iff all required fields in the message and all
+    /// embedded messages are set.
+    /// </summary>
+    bool Initialized { get; }
+
+    /// <summary>
+    /// Behaves like the equivalent property in IMessage&lt;T&gt;.
+    /// The returned map may or may not reflect future changes to the builder.
+    /// Either way, the returned map is unmodifiable.
+    /// </summary>
+    IDictionary<ProtocolBuffers.Descriptors.FieldDescriptor, object> AllFields { get; }
+
+    /// <summary>
+    /// Allows getting and setting of a field.
+    /// <see cref="IMessage{T}.Item(Descriptors.FieldDescriptor)"/>
+    /// </summary>
+    /// <param name="field"></param>
+    /// <returns></returns>
+    object this[Descriptors.FieldDescriptor field] { get; set; }
+
+    /// <summary>
+    /// Get the message's type's descriptor.
+    /// <see cref="IMessage{T}.DescriptorForType"/>
+    /// </summary>
+    Descriptors.Descriptor DescriptorForType { get; }
+
+    /// <summary>
+    /// <see cref="IMessage{T}.GetRepeatedFieldCount"/>
+    /// </summary>
+    /// <param name="field"></param>
+    /// <returns></returns>
+    int GetRepeatedFieldCount(Descriptors.FieldDescriptor field);
+
+    /// <summary>
+    /// Allows getting and setting of a repeated field value.
+    /// <see cref="IMessage{T}.Item(Descriptors.FieldDescriptor, int)"/>
+    /// </summary>
+    object this[Descriptors.FieldDescriptor field, int index] { get; set; }
+
+    /// <summary>
+    /// <see cref="IMessage{T}.HasField"/>
+    /// </summary>
+    bool HasField(Descriptors.FieldDescriptor field);
   }
 
   /// <summary>
@@ -14,7 +65,7 @@ namespace Google.ProtocolBuffers {
   /// TODO(jonskeet): Consider "SetXXX" methods returning the builder, as well as the properties.
   /// </summary>
   /// <typeparam name="T">Type of message</typeparam>
-  public interface IBuilder<T> where T : IMessage<T> {
+  public interface IBuilder<T> : IBuilder where T : IMessage<T> {
     /// <summary>
     /// Resets all fields to their default values.
     /// </summary>
@@ -59,12 +110,6 @@ namespace Google.ProtocolBuffers {
     /// </summary>
     IBuilder<T> Clone();
 
-    /// <summary>
-    /// Returns true iff all required fields in the message and all
-    /// embedded messages are set.
-    /// </summary>
-    bool Initialized { get; }
-
     /// <summary>
     /// Parses a message of this type from the input and merges it with this
     /// message, as if using MergeFrom(IMessage&lt;T&gt;).
@@ -92,13 +137,7 @@ namespace Google.ProtocolBuffers {
     /// in <paramref name="extensionRegistry"/>. Extensions not in the registry
     /// will be treated as unknown fields.
     /// </summary>
-    IBuilder<T> MergeFrom(CodedInputStream input, ExtensionRegistry extensionRegistry);
-
-    /// <summary>
-    /// Get the message's type's descriptor.
-    /// <see cref="IMessage{T}.DescriptorForType"/>
-    /// </summary>
-    Descriptors.Descriptor DescriptorForType { get; }
+    new IBuilder<T> MergeFrom(CodedInputStream input, ExtensionRegistry extensionRegistry);
 
     /// <summary>
     /// Get's the message's type's default instance.
@@ -106,13 +145,6 @@ namespace Google.ProtocolBuffers {
     /// </summary>
     IMessage<T> DefaultInstanceForType { get; }
 
-    /// <summary>
-    /// Behaves like the equivalent property in IMessage&lt;T&gt;.
-    /// The returned map may or may not reflect future changes to the builder.
-    /// Either way, the returned map is unmodifiable.
-    /// </summary>
-    IDictionary<ProtocolBuffers.Descriptors.FieldDescriptor, object> AllFields { get; }
-
     /// <summary>
     /// Create a builder for messages of the appropriate type for the given field.
     /// Messages built with this can then be passed to the various mutation properties
@@ -121,21 +153,7 @@ namespace Google.ProtocolBuffers {
     /// <typeparam name="TField"></typeparam>
     /// <param name="field"></param>
     /// <returns></returns>
-    IBuilder<TField> NewBuilderForField<TField>(Descriptors.FieldDescriptor field)
-        where TField : IMessage<TField>;
-
-    /// <summary>
-    /// <see cref="IMessage{T}.HasField"/>
-    /// </summary>
-    bool HasField(Descriptors.FieldDescriptor field);
-
-    /// <summary>
-    /// Allows getting and setting of a field.
-    /// <see cref="IMessage{T}.Item(Descriptors.FieldDescriptor)"/>
-    /// </summary>
-    /// <param name="field"></param>
-    /// <returns></returns>
-    object this[Descriptors.FieldDescriptor field] { get; set; }
+    IBuilder<TField> NewBuilderForField<TField>(Descriptors.FieldDescriptor field) where TField : IMessage<TField>;
 
     /// <summary>
     /// Clears the field. This is exactly equivalent to calling the generated
@@ -145,20 +163,6 @@ namespace Google.ProtocolBuffers {
     /// <returns></returns>
     IBuilder<T> ClearField(Descriptors.FieldDescriptor field);
 
-    /// <summary>
-    /// <see cref="IMessage{T}.GetRepeatedFieldCount"/>
-    /// </summary>
-    /// <param name="field"></param>
-    /// <returns></returns>
-    int GetRepeatedFieldCount(Descriptors.FieldDescriptor field);
-
-
-    /// <summary>
-    /// Allows getting and setting of a repeated field value.
-    /// <see cref="IMessage{T}.Item(Descriptors.FieldDescriptor, int)"/>
-    /// </summary>
-    object this[Descriptors.FieldDescriptor field, int index] { get; set; }
-
     /// <summary>
     /// Appends the given value as a new element for the specified repeated field.
     /// </summary>

csharp/ProtocolBuffers/IMessage.cs

@@ -5,18 +5,13 @@ using System.IO;
 namespace Google.ProtocolBuffers {
 
   /// <summary>
-  /// Non-generic interface.
-  /// TODO(jonskeet): Do we want or need this?
+  /// Non-generic interface implemented by all Protocol Buffers messages.
+  /// Some members are repeated in the generic interface but with a
+  /// type-specific signature. Type-safe implementations
+  /// are encouraged to implement these non-generic members explicitly,
+  /// and the generic members implicitly.
   /// </summary>
   public interface IMessage {
-    void WriteTo(CodedOutputStream output);
-    int SerializedSize { get; }
-  }
-
-  /// <summary>
-  /// Interface implemented by all Protocol Buffers messages.
-  /// </summary>
-  public interface IMessage<T> where T : IMessage<T> {
     /// <summary>
     /// Returns the message's type's descriptor. This differs from the
     /// Descriptor property of each generated message class in that this
@@ -24,16 +19,6 @@ namespace Google.ProtocolBuffers {
     /// a static property of a specific class. They return the same thing.
     /// </summary>
     Descriptors.Descriptor DescriptorForType { get; }
-
-    /// <summary>
-    /// Returns an instance of this message type with all fields set to
-    /// their default values. This may or may not be a singleton. This differs
-    /// from the DefaultInstance property of each generated message class in that this
-    /// method is an abstract method of IMessage whereas DefaultInstance is
-    /// a static property of a specific class. They return the same thing.
-    /// </summary>
-    IMessage<T> DefaultInstanceForType { get; }
-
     /// <summary>
     /// Returns a collection of all the fields in this message which are set
     /// and their corresponding values.  A singular ("required" or "optional")
@@ -151,12 +136,42 @@ namespace Google.ProtocolBuffers {
     void WriteTo(Stream output);
     #endregion
 
-    #region Builders
+    #region Weakly typed members
+    /// <summary>
+    /// Returns an instance of this message type with all fields set to
+    /// their default values. This may or may not be a singleton. This differs
+    /// from the DefaultInstance property of each generated message class in that this
+    /// method is an abstract method of IMessage whereas DefaultInstance is
+    /// a static property of a specific class. They return the same thing.
+    /// </summary>
+    IMessage DefaultInstanceForType { get; }
+
     /// <summary>
     /// Constructs a new builder for a message of the same type as this message.
     /// </summary>
-    IBuilder<T> NewBuilderForType();
+    IBuilder NewBuilderForType();
     #endregion
+  }
+
+  /// <summary>
+  /// Type-safe interface for all generated messages to implement.
+  /// </summary>
+  public interface IMessage<T> : IMessage where T : IMessage<T> {
+    /// <summary>
+    /// Returns an instance of this message type with all fields set to
+    /// their default values. This may or may not be a singleton. This differs
+    /// from the DefaultInstance property of each generated message class in that this
+    /// method is an abstract method of IMessage whereas DefaultInstance is
+    /// a static property of a specific class. They return the same thing.
+    /// </summary>
+    new IMessage<T> DefaultInstanceForType { get; }
+
 
+    #region Builders
+    /// <summary>
+    /// Constructs a new builder for a message of the same type as this message.
+    /// </summary>
+    new IBuilder<T> NewBuilderForType();
+    #endregion
   }
 }

csharp/ProtocolBuffers/InvalidProtocolBufferException.cs

@@ -14,7 +14,8 @@ namespace Google.ProtocolBuffers {
       : base(message) {
     }
 
-    internal static InvalidProtocolBufferException TruncatedMessage() {
+    /// TODO(jonskeet): Make this internal again and use InternalVisibleTo?
+    public static InvalidProtocolBufferException TruncatedMessage() {
       return new InvalidProtocolBufferException(
         "While parsing a protocol message, the input ended unexpectedly " +
         "in the middle of a field.  This could mean either than the " +
@@ -22,13 +23,14 @@ namespace Google.ProtocolBuffers {
         "misreported its own length.");
     }
 
+    /// TODO(jonskeet): Make this internal again and use InternalVisibleTo?
     internal static InvalidProtocolBufferException NegativeSize() {
       return new InvalidProtocolBufferException(
         "CodedInputStream encountered an embedded string or message " +
         "which claimed to have negative size.");
     }
 
-    internal static InvalidProtocolBufferException MalformedVarint() {
+    public static InvalidProtocolBufferException MalformedVarint() {
       return new InvalidProtocolBufferException(
         "CodedInputStream encountered a malformed varint.");
     }

csharp/ProtocolBuffers/WireFormat.cs

@@ -4,7 +4,7 @@ using System.Text;
 
 namespace Google.ProtocolBuffers {
   public class WireFormat {
-    public enum WireType {
+    public enum WireType : uint {
       Varint = 0,
       Fixed64 = 1,
       LengthDelimited = 2,
@@ -19,10 +19,28 @@ namespace Google.ProtocolBuffers {
       internal const int Message = 3;
     }
 
-    public static uint MakeTag(int fieldNumber, WireType type) {
+    private const int TagTypeBits = 3;
+    private const uint TagTypeMask = (1 << TagTypeBits) - 1;
 
-      // FIXME
-      return 0;
+    /// <summary>
+    /// Given a tag value, determines the wire type (lower 3 bits).
+    /// </summary>
+    public static WireType GetTagWireType(uint tag) {
+      return (WireType) (tag & TagTypeMask);
+    }
+
+    /// <summary>
+    /// Given a tag value, determines the field number (the upper 29 bits).
+    /// </summary>
+    public static uint GetTagFieldNumber(uint tag) {
+      return tag >> TagTypeBits;
+    }
+
+    /// <summary>
+    /// Makes a tag value given a field number and wire type.
+    /// </summary>
+    public static uint MakeTag(int fieldNumber, WireType wireType) {
+      return (uint) (fieldNumber << TagTypeBits) | (uint) wireType;
     }
   }
 }