Merge branch 'master' into protosplit

Makefile

@@ -10287,6 +10287,7 @@ ifneq ($(NO_DEPS),true)
 -include $(RECONNECT_INTEROP_CLIENT_OBJS:.o=.dep)
 endif
 endif
+$(OBJDIR)/$(CONFIG)/test/cpp/interop/reconnect_interop_client.o: $(GENDIR)/test/proto/empty.pb.cc $(GENDIR)/test/proto/empty.grpc.pb.cc $(GENDIR)/test/proto/messages.pb.cc $(GENDIR)/test/proto/messages.grpc.pb.cc $(GENDIR)/test/proto/test.pb.cc $(GENDIR)/test/proto/test.grpc.pb.cc
 
 
 RECONNECT_INTEROP_SERVER_SRC = \
@@ -10333,6 +10334,7 @@ ifneq ($(NO_DEPS),true)
 -include $(RECONNECT_INTEROP_SERVER_OBJS:.o=.dep)
 endif
 endif
+$(OBJDIR)/$(CONFIG)/test/cpp/interop/reconnect_interop_server.o: $(GENDIR)/test/proto/empty.pb.cc $(GENDIR)/test/proto/empty.grpc.pb.cc $(GENDIR)/test/proto/messages.pb.cc $(GENDIR)/test/proto/messages.grpc.pb.cc $(GENDIR)/test/proto/test.pb.cc $(GENDIR)/test/proto/test.grpc.pb.cc
 
 
 SECURE_AUTH_CONTEXT_TEST_SRC = \
@@ -10663,6 +10665,9 @@ ifneq ($(NO_DEPS),true)
 -include $(STRESS_TEST_OBJS:.o=.dep)
 endif
 endif
+$(OBJDIR)/$(CONFIG)/test/cpp/interop/interop_client.o: $(GENDIR)/test/proto/empty.pb.cc $(GENDIR)/test/proto/empty.grpc.pb.cc $(GENDIR)/test/proto/messages.pb.cc $(GENDIR)/test/proto/messages.grpc.pb.cc $(GENDIR)/test/proto/test.pb.cc $(GENDIR)/test/proto/test.grpc.pb.cc
+$(OBJDIR)/$(CONFIG)/test/cpp/interop/stress_interop_client.o: $(GENDIR)/test/proto/empty.pb.cc $(GENDIR)/test/proto/empty.grpc.pb.cc $(GENDIR)/test/proto/messages.pb.cc $(GENDIR)/test/proto/messages.grpc.pb.cc $(GENDIR)/test/proto/test.pb.cc $(GENDIR)/test/proto/test.grpc.pb.cc
+$(OBJDIR)/$(CONFIG)/test/cpp/interop/stress_test.o: $(GENDIR)/test/proto/empty.pb.cc $(GENDIR)/test/proto/empty.grpc.pb.cc $(GENDIR)/test/proto/messages.pb.cc $(GENDIR)/test/proto/messages.grpc.pb.cc $(GENDIR)/test/proto/test.pb.cc $(GENDIR)/test/proto/test.grpc.pb.cc
 
 
 SYNC_STREAMING_PING_PONG_TEST_SRC = \

include/grpc++/support/string_ref.h

@@ -56,7 +56,7 @@ class string_ref {
   typedef std::reverse_iterator<const_iterator> const_reverse_iterator;
 
   // constants
-  const static size_t npos = size_t(-1);
+  const static size_t npos;
 
   // construct/copy.
   string_ref() : data_(nullptr), length_(0) {}

include/grpc/byte_buffer.h

@@ -106,6 +106,9 @@ void grpc_byte_buffer_reader_destroy(grpc_byte_buffer_reader *reader);
 int grpc_byte_buffer_reader_next(grpc_byte_buffer_reader *reader,
                                  gpr_slice *slice);
 
+/** Merge all data from \a reader into single slice */
+gpr_slice grpc_byte_buffer_reader_readall(grpc_byte_buffer_reader *reader);
+
 /** Returns a RAW byte buffer instance from the output of \a reader. */
 grpc_byte_buffer *grpc_raw_byte_buffer_from_reader(
     grpc_byte_buffer_reader *reader);

src/core/surface/byte_buffer_reader.c

@@ -31,6 +31,7 @@
  *
  */
 
+#include <string.h>
 #include <grpc/byte_buffer_reader.h>
 
 #include <grpc/compression.h>
@@ -103,3 +104,21 @@ int grpc_byte_buffer_reader_next(grpc_byte_buffer_reader *reader,
   }
   return 0;
 }
+
+gpr_slice grpc_byte_buffer_reader_readall(grpc_byte_buffer_reader *reader) {
+  gpr_slice in_slice;
+  size_t bytes_read = 0;
+  const size_t input_size = grpc_byte_buffer_length(reader->buffer_out);
+  gpr_slice out_slice = gpr_slice_malloc(input_size);
+  gpr_uint8 *const outbuf = GPR_SLICE_START_PTR(out_slice); /* just an alias */
+
+  while (grpc_byte_buffer_reader_next(reader, &in_slice) != 0) {
+    const size_t slice_length = GPR_SLICE_LENGTH(in_slice);
+    memcpy(&(outbuf[bytes_read]), GPR_SLICE_START_PTR(in_slice), slice_length);
+    bytes_read += slice_length;
+    gpr_slice_unref(in_slice);
+    GPR_ASSERT(bytes_read <= input_size);
+  }
+  return out_slice;
+}
+

src/cpp/util/string_ref.cc

@@ -40,7 +40,7 @@
 
 namespace grpc {
 
-const size_t string_ref::npos;
+const size_t string_ref::npos = size_t(-1);
 
 string_ref& string_ref::operator=(const string_ref& rhs) {
   data_ = rhs.data_;

src/csharp/Grpc.Core.Tests/ClientServerTest.cs

@@ -38,6 +38,7 @@ using System.Threading;
 using System.Threading.Tasks;
 using Grpc.Core;
 using Grpc.Core.Internal;
+using Grpc.Core.Profiling;
 using Grpc.Core.Utils;
 using NUnit.Framework;
 
@@ -200,19 +201,6 @@ namespace Grpc.Core.Tests
             Assert.AreEqual(headers[1].Key, trailers[1].Key);
             CollectionAssert.AreEqual(headers[1].ValueBytes, trailers[1].ValueBytes);
         }
-
-        [Test]
-        public void UnaryCallPerformance()
-        {
-            helper.UnaryHandler = new UnaryServerMethod<string, string>(async (request, context) =>
-            {
-                return request;
-            });
-
-            var callDetails = helper.CreateUnaryCall();
-            BenchmarkUtil.RunBenchmark(1, 10,
-                                       () => { Calls.BlockingUnaryCall(callDetails, "ABC"); });
-        }
             
         [Test]
         public void UnknownMethodHandler()

src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj

@@ -88,6 +88,7 @@
     <Compile Include="CompressionTest.cs" />
     <Compile Include="ContextPropagationTest.cs" />
     <Compile Include="MetadataTest.cs" />
+    <Compile Include="PerformanceTest.cs" />
   </ItemGroup>
   <Import Project="$(MSBuildBinPath)\Microsoft.CSharp.targets" />
   <ItemGroup>

src/csharp/Grpc.Core.Tests/Internal/TimespecTest.cs

@@ -34,6 +34,7 @@
 using System;
 using System.Runtime.InteropServices;
 using Grpc.Core.Internal;
+using Grpc.Core.Utils;
 using NUnit.Framework;
 
 namespace Grpc.Core.Internal.Tests
@@ -198,5 +199,23 @@ namespace Grpc.Core.Internal.Tests
                 Console.WriteLine("Test cannot be run on this platform, skipping the test.");
             }
         }
+            
+        // Test attribute commented out to prevent running as part of the default test suite.
+        // [Test]
+        // [Category("Performance")]
+        public void NowBenchmark() 
+        {
+            // approx Timespec.Now latency <33ns
+            BenchmarkUtil.RunBenchmark(10000000, 1000000000, () => { var now = Timespec.Now; });
+        }
+
+        // Test attribute commented out to prevent running as part of the default test suite.
+        // [Test]
+        // [Category("Performance")]
+        public void PreciseNowBenchmark()
+        {
+            // approx Timespec.PreciseNow latency <18ns (when compiled with GRPC_TIMERS_RDTSC)
+            BenchmarkUtil.RunBenchmark(10000000, 1000000000, () => { var now = Timespec.PreciseNow; });
+        }
     }
 }

src/csharp/Grpc.Core.Tests/PerformanceTest.cs

@@ -0,0 +1,99 @@
+#region Copyright notice and license
+
+// Copyright 2015, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#endregion
+
+using System;
+using System.Diagnostics;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using Grpc.Core;
+using Grpc.Core.Internal;
+using Grpc.Core.Profiling;
+using Grpc.Core.Utils;
+using NUnit.Framework;
+
+namespace Grpc.Core.Tests
+{
+    public class PerformanceTest
+    {
+        const string Host = "127.0.0.1";
+
+        MockServiceHelper helper;
+        Server server;
+        Channel channel;
+
+        [SetUp]
+        public void Init()
+        {
+            helper = new MockServiceHelper(Host);
+            server = helper.GetServer();
+            server.Start();
+            channel = helper.GetChannel();
+        }
+
+        [TearDown]
+        public void Cleanup()
+        {
+            channel.ShutdownAsync().Wait();
+            server.ShutdownAsync().Wait();
+        }
+
+        // Test attribute commented out to prevent running as part of the default test suite.
+        //[Test]
+        //[Category("Performance")]
+        public void UnaryCallPerformance()
+        {
+            var profiler = new BasicProfiler();
+            Profilers.SetForCurrentThread(profiler);
+
+            helper.UnaryHandler = new UnaryServerMethod<string, string>(async (request, context) =>
+            {
+                return request;
+            });
+
+            var callDetails = helper.CreateUnaryCall();
+            for(int i = 0; i < 3000; i++)
+            {
+                Calls.BlockingUnaryCall(callDetails, "ABC");
+            }
+
+            profiler.Reset();
+
+            for(int i = 0; i < 3000; i++)
+            {
+                Calls.BlockingUnaryCall(callDetails, "ABC");
+            }
+            profiler.Dump("latency_trace_csharp.txt");
+        }
+    }
+}

src/csharp/Grpc.Core/Grpc.Core.csproj

@@ -119,6 +119,10 @@
     <Compile Include="CompressionLevel.cs" />
     <Compile Include="WriteOptions.cs" />
     <Compile Include="ContextPropagationToken.cs" />
+    <Compile Include="Profiling\ProfilerEntry.cs" />
+    <Compile Include="Profiling\ProfilerScope.cs" />
+    <Compile Include="Profiling\IProfiler.cs" />
+    <Compile Include="Profiling\Profilers.cs" />
   </ItemGroup>
   <ItemGroup>
     <None Include="Grpc.Core.nuspec" />
@@ -150,4 +154,7 @@
   <Import Project="..\packages\grpc.dependencies.openssl.redist.1.0.2.2\build\portable-net45\grpc.dependencies.openssl.redist.targets" Condition="Exists('..\packages\grpc.dependencies.openssl.redist.1.0.2.2\build\portable-net45\grpc.dependencies.openssl.redist.targets')" />
   <Import Project="..\packages\grpc.dependencies.zlib.redist.1.2.8.9\build\portable-net45\grpc.dependencies.zlib.redist.targets" Condition="Exists('..\packages\grpc.dependencies.zlib.redist.1.2.8.9\build\portable-net45\grpc.dependencies.zlib.redist.targets')" />
   <ItemGroup />
+  <ItemGroup>
+    <Folder Include="Profiling\" />
+  </ItemGroup>
 </Project>

src/csharp/Grpc.Core/Internal/AsyncCall.cs

@@ -39,6 +39,7 @@ using System.Threading;
 using System.Threading.Tasks;
 using Grpc.Core.Internal;
 using Grpc.Core.Logging;
+using Grpc.Core.Profiling;
 using Grpc.Core.Utils;
 
 namespace Grpc.Core.Internal
@@ -87,6 +88,9 @@ namespace Grpc.Core.Internal
         /// </summary>
         public TResponse UnaryCall(TRequest msg)
         {
+            var profiler = Profilers.ForCurrentThread();
+
+            using (profiler.NewScope("AsyncCall.UnaryCall"))
             using (CompletionQueueSafeHandle cq = CompletionQueueSafeHandle.Create())
             {
                 byte[] payload = UnsafeSerialize(msg);
@@ -104,24 +108,26 @@ namespace Grpc.Core.Internal
                 }
 
                 using (var metadataArray = MetadataArraySafeHandle.Create(details.Options.Headers))
+                using (var ctx = BatchContextSafeHandle.Create())
                 {
-                    using (var ctx = BatchContextSafeHandle.Create())
-                    {
-                        call.StartUnary(ctx, payload, metadataArray, GetWriteFlagsForCall());
-                        var ev = cq.Pluck(ctx.Handle);
+                    call.StartUnary(ctx, payload, metadataArray, GetWriteFlagsForCall());
+
+                    var ev = cq.Pluck(ctx.Handle);
 
-                        bool success = (ev.success != 0);
-                        try
+                    bool success = (ev.success != 0);
+                    try
+                    {
+                        using (profiler.NewScope("AsyncCall.UnaryCall.HandleBatch"))
                         {
                             HandleUnaryResponse(success, ctx.GetReceivedStatusOnClient(), ctx.GetReceivedMessage(), ctx.GetReceivedInitialMetadata());
                         }
-                        catch (Exception e)
-                        {
-                            Logger.Error(e, "Exception occured while invoking completion delegate.");
-                        }
+                    }
+                    catch (Exception e)
+                    {
+                        Logger.Error(e, "Exception occured while invoking completion delegate.");
                     }
                 }
-
+                    
                 // Once the blocking call returns, the result should be available synchronously.
                 // Note that GetAwaiter().GetResult() doesn't wrap exceptions in AggregateException.
                 return unaryResponseTcs.Task.GetAwaiter().GetResult();
@@ -329,27 +335,35 @@ namespace Grpc.Core.Internal
 
         private void Initialize(CompletionQueueSafeHandle cq)
         {
-            var call = CreateNativeCall(cq);
-            details.Channel.AddCallReference(this);
-            InitializeInternal(call);
-            RegisterCancellationCallback();
+            using (Profilers.ForCurrentThread().NewScope("AsyncCall.Initialize"))
+            { 
+                var call = CreateNativeCall(cq);
+
+                details.Channel.AddCallReference(this);
+                InitializeInternal(call);
+                RegisterCancellationCallback();
+            }
         }
 
         private INativeCall CreateNativeCall(CompletionQueueSafeHandle cq)
         {
-            if (injectedNativeCall != null)
-            {
-                return injectedNativeCall;  // allows injecting a mock INativeCall in tests.
-            }
+            using (Profilers.ForCurrentThread().NewScope("AsyncCall.CreateNativeCall"))
+            { 
+                if (injectedNativeCall != null)
+                {
+                    return injectedNativeCall;  // allows injecting a mock INativeCall in tests.
+                }
 
-            var parentCall = details.Options.PropagationToken != null ? details.Options.PropagationToken.ParentCall : CallSafeHandle.NullInstance;
+                var parentCall = details.Options.PropagationToken != null ? details.Options.PropagationToken.ParentCall : CallSafeHandle.NullInstance;
 
-            var credentials = details.Options.Credentials;
-            using (var nativeCredentials = credentials != null ? credentials.ToNativeCredentials() : null)
-            {
-                return details.Channel.Handle.CreateCall(environment.CompletionRegistry,
-                    parentCall, ContextPropagationToken.DefaultMask, cq,
-                    details.Method, details.Host, Timespec.FromDateTime(details.Options.Deadline.Value), nativeCredentials);
+                var credentials = details.Options.Credentials;
+                using (var nativeCredentials = credentials != null ? credentials.ToNativeCredentials() : null)
+                {
+                    var result = details.Channel.Handle.CreateCall(environment.CompletionRegistry,
+                                 parentCall, ContextPropagationToken.DefaultMask, cq,
+                                 details.Method, details.Host, Timespec.FromDateTime(details.Options.Deadline.Value), nativeCredentials);
+                    return result;
+                }
             }
         }
 
@@ -385,33 +399,37 @@ namespace Grpc.Core.Internal
         /// </summary>
         private void HandleUnaryResponse(bool success, ClientSideStatus receivedStatus, byte[] receivedMessage, Metadata responseHeaders)
         {
-            TResponse msg = default(TResponse);
-            var deserializeException = success ? TryDeserialize(receivedMessage, out msg) : null;
-
-            lock (myLock)
+            using (Profilers.ForCurrentThread().NewScope("AsyncCall.HandleUnaryResponse"))
             {
-                finished = true;
+                TResponse msg = default(TResponse);
+                var deserializeException = success ? TryDeserialize(receivedMessage, out msg) : null;
 
-                if (deserializeException != null && receivedStatus.Status.StatusCode == StatusCode.OK)
+                lock (myLock)
                 {
-                    receivedStatus = new ClientSideStatus(DeserializeResponseFailureStatus, receivedStatus.Trailers);
+                    finished = true;
+
+                    if (deserializeException != null && receivedStatus.Status.StatusCode == StatusCode.OK)
+                    {
+                        receivedStatus = new ClientSideStatus(DeserializeResponseFailureStatus, receivedStatus.Trailers);
+                    }
+                    finishedStatus = receivedStatus;
+
+                    ReleaseResourcesIfPossible();
+
                 }
-                finishedStatus = receivedStatus;
 
-                ReleaseResourcesIfPossible();
-            }
+                responseHeadersTcs.SetResult(responseHeaders);
 
-            responseHeadersTcs.SetResult(responseHeaders);
+                var status = receivedStatus.Status;
 
-            var status = receivedStatus.Status;
+                if (!success || status.StatusCode != StatusCode.OK)
+                {
+                    unaryResponseTcs.SetException(new RpcException(status));
+                    return;
+                }
 
-            if (!success || status.StatusCode != StatusCode.OK)
-            {
-                unaryResponseTcs.SetException(new RpcException(status));
-                return;
+                unaryResponseTcs.SetResult(msg);
             }
-
-            unaryResponseTcs.SetResult(msg);
         }
 
         /// <summary>

src/csharp/Grpc.Core/Internal/AsyncCallBase.cs

@@ -41,6 +41,7 @@ using System.Threading.Tasks;
 
 using Grpc.Core.Internal;
 using Grpc.Core.Logging;
+using Grpc.Core.Profiling;
 using Grpc.Core.Utils;
 
 namespace Grpc.Core.Internal
@@ -167,16 +168,19 @@ namespace Grpc.Core.Internal
         /// </summary>
         protected bool ReleaseResourcesIfPossible()
         {
-            if (!disposed && call != null)
+            using (Profilers.ForCurrentThread().NewScope("AsyncCallBase.ReleaseResourcesIfPossible"))
             {
-                bool noMoreSendCompletions = sendCompletionDelegate == null && (halfcloseRequested || cancelRequested || finished);
-                if (noMoreSendCompletions && readingDone && finished)
+                if (!disposed && call != null)
                 {
-                    ReleaseResources();
-                    return true;
+                    bool noMoreSendCompletions = sendCompletionDelegate == null && (halfcloseRequested || cancelRequested || finished);
+                    if (noMoreSendCompletions && readingDone && finished)
+                    {
+                        ReleaseResources();
+                        return true;
+                    }
                 }
+                return false;
             }
-            return false;
         }
 
         protected abstract bool IsClient
@@ -228,7 +232,10 @@ namespace Grpc.Core.Internal
 
         protected byte[] UnsafeSerialize(TWrite msg)
         {
-            return serializer(msg);
+            using (Profilers.ForCurrentThread().NewScope("AsyncCallBase.UnsafeSerialize"))
+            {
+                return serializer(msg);
+            }
         }
 
         protected Exception TrySerialize(TWrite msg, out byte[] payload)
@@ -247,15 +254,20 @@ namespace Grpc.Core.Internal
 
         protected Exception TryDeserialize(byte[] payload, out TRead msg)
         {
-            try
-            {
-                msg = deserializer(payload);
-                return null;
-            } 
-            catch (Exception e)
+            using (Profilers.ForCurrentThread().NewScope("AsyncCallBase.TryDeserialize"))
             {
-                msg = default(TRead);
-                return e;
+                try
+                {
+                
+                    msg = deserializer(payload);
+                    return null;
+             
+                }
+                catch (Exception e)
+                {
+                    msg = default(TRead);
+                    return e;
+                }
             }
         }
 

src/csharp/Grpc.Core/Internal/CallSafeHandle.cs

@@ -34,6 +34,7 @@ using System.Diagnostics;
 using System.Runtime.InteropServices;
 using Grpc.Core;
 using Grpc.Core.Utils;
+using Grpc.Core.Profiling;
 
 namespace Grpc.Core.Internal
 {
@@ -131,8 +132,11 @@ namespace Grpc.Core.Internal
 
         public void StartUnary(BatchContextSafeHandle ctx, byte[] payload, MetadataArraySafeHandle metadataArray, WriteFlags writeFlags)
         {
-            grpcsharp_call_start_unary(this, ctx, payload, new UIntPtr((ulong)payload.Length), metadataArray, writeFlags)
-                .CheckOk();
+            using (Profilers.ForCurrentThread().NewScope("CallSafeHandle.StartUnary"))
+            {
+                grpcsharp_call_start_unary(this, ctx, payload, new UIntPtr((ulong)payload.Length), metadataArray, writeFlags)
+                    .CheckOk();
+            }
         }
 
         public void StartClientStreaming(UnaryResponseClientHandler callback, MetadataArraySafeHandle metadataArray)

src/csharp/Grpc.Core/Internal/ChannelSafeHandle.cs

@@ -32,6 +32,7 @@ using System;
 using System.Runtime.InteropServices;
 using System.Threading;
 using System.Threading.Tasks;
+using Grpc.Core.Profiling;
 
 namespace Grpc.Core.Internal
 {
@@ -84,13 +85,16 @@ namespace Grpc.Core.Internal
 
         public CallSafeHandle CreateCall(CompletionRegistry registry, CallSafeHandle parentCall, ContextPropagationFlags propagationMask, CompletionQueueSafeHandle cq, string method, string host, Timespec deadline, CredentialsSafeHandle credentials)
         {
-            var result = grpcsharp_channel_create_call(this, parentCall, propagationMask, cq, method, host, deadline);
-            if (credentials != null)
+            using (Profilers.ForCurrentThread().NewScope("ChannelSafeHandle.CreateCall"))
             {
-                result.SetCredentials(credentials);
+                var result = grpcsharp_channel_create_call(this, parentCall, propagationMask, cq, method, host, deadline);
+                if (credentials != null)
+                {
+                    result.SetCredentials(credentials);
+                }
+                result.SetCompletionRegistry(registry);
+                return result;
             }
-            result.SetCompletionRegistry(registry);
-            return result;
         }
 
         public ChannelState CheckConnectivityState(bool tryToConnect)

src/csharp/Grpc.Core/Internal/CompletionQueueSafeHandle.cs

@@ -31,6 +31,7 @@
 using System;
 using System.Runtime.InteropServices;
 using System.Threading.Tasks;
+using Grpc.Core.Profiling;
 
 namespace Grpc.Core.Internal
 {
@@ -70,7 +71,10 @@ namespace Grpc.Core.Internal
 
         public CompletionQueueEvent Pluck(IntPtr tag)
         {
-            return grpcsharp_completion_queue_pluck(this, tag);
+            using (Profilers.ForCurrentThread().NewScope("CompletionQueueSafeHandle.Pluck"))
+            {
+                return grpcsharp_completion_queue_pluck(this, tag);
+            }
         }
 
         public void Shutdown()

src/csharp/Grpc.Core/Internal/Enums.cs

@@ -102,6 +102,9 @@ namespace Grpc.Core.Internal
         /* Realtime clock */
         Realtime,
 
+        /* Precise clock good for performance profiling. */
+        Precise,
+
         /* Timespan - the distance between two time points */
         Timespan
     }

src/csharp/Grpc.Core/Internal/MetadataArraySafeHandle.cs

@@ -31,6 +31,7 @@
 using System;
 using System.Runtime.InteropServices;
 using System.Threading.Tasks;
+using Grpc.Core.Profiling;
 
 namespace Grpc.Core.Internal
 {
@@ -66,14 +67,17 @@ namespace Grpc.Core.Internal
             
         public static MetadataArraySafeHandle Create(Metadata metadata)
         {
-            // TODO(jtattermusch): we might wanna check that the metadata is readonly 
-            var metadataArray = grpcsharp_metadata_array_create(new UIntPtr((ulong)metadata.Count));
-            for (int i = 0; i < metadata.Count; i++)
+            using (Profilers.ForCurrentThread().NewScope("MetadataArraySafeHandle.Create"))
             {
-                var valueBytes = metadata[i].GetSerializedValueUnsafe();
-                grpcsharp_metadata_array_add(metadataArray, metadata[i].Key, valueBytes, new UIntPtr((ulong)valueBytes.Length));
+                // TODO(jtattermusch): we might wanna check that the metadata is readonly 
+                var metadataArray = grpcsharp_metadata_array_create(new UIntPtr((ulong)metadata.Count));
+                for (int i = 0; i < metadata.Count; i++)
+                {
+                    var valueBytes = metadata[i].GetSerializedValueUnsafe();
+                    grpcsharp_metadata_array_add(metadataArray, metadata[i].Key, valueBytes, new UIntPtr((ulong)valueBytes.Length));
+                }
+                return metadataArray;
             }
-            return metadataArray;
         }
 
         /// <summary>

src/csharp/Grpc.Core/Internal/Timespec.cs

@@ -239,6 +239,19 @@ namespace Grpc.Core.Internal
             }
         }
 
+        /// <summary>
+        /// Gets current timestamp using <c>GPRClockType.Precise</c>.
+        /// Only available internally because core needs to be compiled with 
+        /// GRPC_TIMERS_RDTSC support for this to use RDTSC.
+        /// </summary>
+        internal static Timespec PreciseNow
+        {
+            get
+            {
+                return gprsharp_now(GPRClockType.Precise);
+            }
+        }
+
         internal static int NativeSize
         {
             get

src/csharp/Grpc.Core/Profiling/IProfiler.cs

@@ -0,0 +1,47 @@
+#region Copyright notice and license
+
+// Copyright 2015, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#endregion
+
+using System;
+using System.IO;
+using System.Threading;
+using Grpc.Core.Internal;
+
+namespace Grpc.Core.Profiling
+{
+    internal interface IProfiler 
+    {
+        void Begin(string tag);
+        void End(string tag);
+        void Mark(string tag);
+    }
+}

src/csharp/Grpc.Core/Profiling/ProfilerEntry.cs

@@ -0,0 +1,87 @@
+#region Copyright notice and license
+
+// Copyright 2015, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#endregion
+
+using System;
+using System.IO;
+using System.Threading;
+using Grpc.Core.Internal;
+
+namespace Grpc.Core.Profiling
+{
+    internal struct ProfilerEntry
+    {
+        public enum Type {
+            BEGIN,
+            END,
+            MARK
+        }
+
+        public ProfilerEntry(Timespec timespec, Type type, string tag)
+        {
+            this.timespec = timespec;
+            this.type = type;
+            this.tag = tag;
+        }
+
+        public Timespec timespec;
+        public Type type;
+        public string tag;
+
+        public override string ToString()
+        {
+            // mimic the output format used by C core.
+            return string.Format(
+                "{{\"t\": {0}.{1}, \"thd\":\"unknown\", \"type\": \"{2}\", \"tag\": \"{3}\", " +
+                "\"file\": \"unknown\", \"line\": 0, \"imp\": 0}}",
+                timespec.TimevalSeconds, timespec.TimevalNanos.ToString("D9"),
+                GetTypeAbbreviation(type), tag);
+        }
+
+        internal static string GetTypeAbbreviation(Type type)
+        {
+            switch (type)
+            {
+                case Type.BEGIN:
+                    return "{";
+
+                case Type.END:
+                    return "}";
+                
+                case Type.MARK:
+                    return ".";
+                default:
+                    throw new ArgumentException("Unknown type");
+            }
+        }
+    }
+}

src/csharp/Grpc.Core/Profiling/ProfilerScope.cs

@@ -0,0 +1,60 @@
+#region Copyright notice and license
+
+// Copyright 2015, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#endregion
+
+using System;
+using System.IO;
+using System.Threading;
+using Grpc.Core.Internal;
+
+namespace Grpc.Core.Profiling
+{
+    // Allows declaring Begin and End of a profiler scope with a using statement.
+    // declared as struct for better performance.
+    internal struct ProfilerScope : IDisposable
+    {
+        readonly IProfiler profiler;
+        readonly string tag;
+
+        public ProfilerScope(IProfiler profiler, string tag)
+        {
+            this.profiler = profiler;
+            this.tag = tag;
+            this.profiler.Begin(this.tag);
+        }
+            
+        public void Dispose()
+        {
+            profiler.End(tag);
+        }
+    }
+}

src/csharp/Grpc.Core/Profiling/Profilers.cs

@@ -0,0 +1,131 @@
+#region Copyright notice and license
+
+// Copyright 2015, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#endregion
+
+using System;
+using System.IO;
+using System.Threading;
+using Grpc.Core.Internal;
+
+namespace Grpc.Core.Profiling
+{
+    internal static class Profilers
+    {
+        static readonly NopProfiler defaultProfiler = new NopProfiler();
+        static readonly ThreadLocal<IProfiler> profilers = new ThreadLocal<IProfiler>();
+
+        public static IProfiler ForCurrentThread()
+        {
+            return profilers.Value ?? defaultProfiler;
+        }
+
+        public static void SetForCurrentThread(IProfiler profiler)
+        {
+            profilers.Value = profiler;
+        }
+
+        public static ProfilerScope NewScope(this IProfiler profiler, string tag)
+        {
+            return new ProfilerScope(profiler, tag);
+        }
+    }
+
+    internal class NopProfiler : IProfiler
+    {
+        public void Begin(string tag)
+        {
+        }
+
+        public void End(string tag)
+        {
+        }
+
+        public void Mark(string tag)
+        {
+        }
+    }
+
+    // Profiler using Timespec.PreciseNow
+    internal class BasicProfiler : IProfiler
+    {
+        ProfilerEntry[] entries;
+        int count;
+
+        public BasicProfiler() : this(1024*1024)
+        {
+        }
+
+        public BasicProfiler(int capacity)
+        {
+            this.entries = new ProfilerEntry[capacity];
+        }
+
+        public void Begin(string tag) {
+            AddEntry(new ProfilerEntry(Timespec.PreciseNow, ProfilerEntry.Type.BEGIN, tag));
+        }
+
+        public void End(string tag) {
+            AddEntry(new ProfilerEntry(Timespec.PreciseNow, ProfilerEntry.Type.END, tag));
+        }
+
+        public void Mark(string tag) {
+            AddEntry(new ProfilerEntry(Timespec.PreciseNow, ProfilerEntry.Type.MARK, tag));
+        }
+
+        public void Reset()
+        {
+            count = 0;
+        }
+
+        public void Dump(string filepath)
+        {
+            using (var stream = new StreamWriter(filepath))
+            {
+                Dump(stream);
+            }
+        }
+
+        public void Dump(TextWriter stream)
+        {
+            for (int i = 0; i < count; i++)
+            {
+                var entry = entries[i];
+                stream.WriteLine(entry.ToString());
+            }
+        }
+
+        // NOT THREADSAFE!
+        void AddEntry(ProfilerEntry entry) {
+            entries[count++] = entry;
+        }
+    }
+}

templates/Makefile.template

@@ -1851,6 +1851,11 @@
   endif
   % endif
   % endif
+  % for src in tgt.src:
+  % if not proto_re.match(src) and any(proto_re.match(src2) for src2 in tgt.src):
+  $(OBJDIR)/$(CONFIG)/${os.path.splitext(src)[0]}.o: ${' '.join(proto_to_cc(src2) for src2 in tgt.src if proto_re.match(src2))}
+  % endif
+  % endfor
   </%def>
 
   ifneq ($(OPENSSL_DEP),)

test/core/surface/byte_buffer_reader_test.c

@@ -184,6 +184,39 @@ static void test_byte_buffer_from_reader(void) {
   grpc_byte_buffer_destroy(buffer_from_reader);
 }
 
+static void test_readall(void) {
+  const char* lotsa_as[512];
+  const char* lotsa_bs[1024];
+  gpr_slice slices[2];
+  grpc_byte_buffer *buffer;
+  grpc_byte_buffer_reader reader;
+  gpr_slice slice_out;
+
+  LOG_TEST("test_readall");
+
+  memset(lotsa_as, 'a', 512);
+  memset(lotsa_bs, 'b', 1024);
+  /* use slices large enough to overflow inlining */
+  slices[0] = gpr_slice_malloc(512);
+  memcpy(GPR_SLICE_START_PTR(slices[0]), lotsa_as, 512);
+  slices[1] = gpr_slice_malloc(1024);
+  memcpy(GPR_SLICE_START_PTR(slices[1]), lotsa_bs, 1024);
+
+  buffer = grpc_raw_byte_buffer_create(slices, 2);
+  gpr_slice_unref(slices[0]);
+  gpr_slice_unref(slices[1]);
+
+  grpc_byte_buffer_reader_init(&reader, buffer);
+  slice_out = grpc_byte_buffer_reader_readall(&reader);
+
+  GPR_ASSERT(GPR_SLICE_LENGTH(slice_out) == 512 + 1024);
+  GPR_ASSERT(memcmp(GPR_SLICE_START_PTR(slice_out), lotsa_as, 512) == 0);
+  GPR_ASSERT(memcmp(&(GPR_SLICE_START_PTR(slice_out)[512]), lotsa_bs, 1024) ==
+             0);
+  gpr_slice_unref(slice_out);
+  grpc_byte_buffer_destroy(buffer);
+}
+
 int main(int argc, char **argv) {
   grpc_test_init(argc, argv);
   test_read_one_slice();
@@ -192,6 +225,6 @@ int main(int argc, char **argv) {
   test_read_gzip_compressed_slice();
   test_read_deflate_compressed_slice();
   test_byte_buffer_from_reader();
-
+  test_readall();
   return 0;
 }

tools/run_tests/dockerjob.py

@@ -101,7 +101,7 @@ class DockerJob:
 
   def __init__(self, spec):
     self._spec = spec
-    self._job = jobset.Job(spec, bin_hash=None, newline_on_success=True, travis=True, add_env={}, xml_report=None)
+    self._job = jobset.Job(spec, bin_hash=None, newline_on_success=True, travis=True, add_env={})
     self._container_name = spec.container_name
 
   def mapped_port(self, port):

tools/run_tests/jobset.py

@@ -34,15 +34,14 @@ import multiprocessing
 import os
 import platform
 import signal
-import string
 import subprocess
 import sys
 import tempfile
 import time
-import xml.etree.cElementTree as ET
 
 
 _DEFAULT_MAX_JOBS = 16 * multiprocessing.cpu_count()
+_MAX_RESULT_SIZE = 8192
 
 
 # setup a signal handler so that signal.pause registers 'something'
@@ -130,14 +129,6 @@ def which(filename):
   raise Exception('%s not found' % filename)
 
 
-def _filter_stdout(stdout):
-  """Filters out nonprintable and XML-illegal characters from stdout."""
-  # keep whitespaces but remove formfeed and vertical tab characters
-  # that make XML report unparseable.
-  return filter(lambda x: x in string.printable and x != '\f' and x != '\v',
-                stdout.decode(errors='ignore'))
-
-
 class JobSpec(object):
   """Specifies what to run for a job."""
 
@@ -190,14 +181,12 @@ class JobResult(object):
 class Job(object):
   """Manages one job."""
 
-  def __init__(self, spec, bin_hash, newline_on_success, travis, add_env, xml_report):
+  def __init__(self, spec, bin_hash, newline_on_success, travis, add_env):
     self._spec = spec
     self._bin_hash = bin_hash
     self._newline_on_success = newline_on_success
     self._travis = travis
     self._add_env = add_env.copy()
-    self._xml_test = ET.SubElement(xml_report, 'testcase',
-                                   name=self._spec.shortname) if xml_report is not None else None
     self._retries = 0
     self._timeout_retries = 0
     self._suppress_failure_message = False
@@ -224,20 +213,12 @@ class Job(object):
 
   def state(self, update_cache):
     """Poll current state of the job. Prints messages at completion."""
+    self._tempfile.seek(0)
+    stdout = self._tempfile.read()
+    self.result.message = stdout[-_MAX_RESULT_SIZE:]
     if self._state == _RUNNING and self._process.poll() is not None:
       elapsed = time.time() - self._start
-      self._tempfile.seek(0)
-      stdout = self._tempfile.read()
-      filtered_stdout = _filter_stdout(stdout)
-      # TODO: looks like jenkins master is slow because parsing the junit results XMLs is not
-      # implemented efficiently. This is an experiment to workaround the issue by making sure
-      # results.xml file is small enough.
-      filtered_stdout = filtered_stdout[-128:]
-      self.result.message = filtered_stdout
       self.result.elapsed_time = elapsed
-      if self._xml_test is not None:
-        self._xml_test.set('time', str(elapsed))
-        ET.SubElement(self._xml_test, 'system-out').text = filtered_stdout
       if self._process.returncode != 0:
         if self._retries < self._spec.flake_retries:
           message('FLAKE', '%s [ret=%d, pid=%d]' % (
@@ -256,8 +237,6 @@ class Job(object):
           self.result.state = 'FAILED'
           self.result.num_failures += 1
           self.result.returncode = self._process.returncode
-          if self._xml_test is not None:
-            ET.SubElement(self._xml_test, 'failure', message='Failure')
       else:
         self._state = _SUCCESS
         message('PASSED', '%s [time=%.1fsec; retries=%d;%d]' % (
@@ -267,10 +246,6 @@ class Job(object):
         if self._bin_hash:
           update_cache.finished(self._spec.identity(), self._bin_hash)
     elif self._state == _RUNNING and time.time() - self._start > self._spec.timeout_seconds:
-      self._tempfile.seek(0)
-      stdout = self._tempfile.read()
-      filtered_stdout = _filter_stdout(stdout)
-      self.result.message = filtered_stdout
       if self._timeout_retries < self._spec.timeout_retries:
         message('TIMEOUT_FLAKE', self._spec.shortname, stdout, do_newline=True)
         self._timeout_retries += 1
@@ -285,9 +260,6 @@ class Job(object):
         self.kill()
         self.result.state = 'TIMEOUT'
         self.result.num_failures += 1
-        if self._xml_test is not None:
-          ET.SubElement(self._xml_test, 'system-out').text = filtered_stdout
-          ET.SubElement(self._xml_test, 'error', message='Timeout')
     return self._state
 
   def kill(self):
@@ -305,7 +277,7 @@ class Jobset(object):
   """Manages one run of jobs."""
 
   def __init__(self, check_cancelled, maxjobs, newline_on_success, travis,
-               stop_on_failure, add_env, cache, xml_report):
+               stop_on_failure, add_env, cache):
     self._running = set()
     self._check_cancelled = check_cancelled
     self._cancelled = False
@@ -317,7 +289,6 @@ class Jobset(object):
     self._cache = cache
     self._stop_on_failure = stop_on_failure
     self._hashes = {}
-    self._xml_report = xml_report
     self._add_env = add_env
     self.resultset = {}
     
@@ -349,8 +320,7 @@ class Jobset(object):
                 bin_hash,
                 self._newline_on_success,
                 self._travis,
-                self._add_env,
-                self._xml_report)
+                self._add_env)
       self._running.add(job)
       self.resultset[job.GetSpec().shortname] = []
     return True
@@ -424,13 +394,11 @@ def run(cmdlines,
         infinite_runs=False,
         stop_on_failure=False,
         cache=None,
-        xml_report=None,
         add_env={}):
   js = Jobset(check_cancelled,
               maxjobs if maxjobs is not None else _DEFAULT_MAX_JOBS,
               newline_on_success, travis, stop_on_failure, add_env,
-              cache if cache is not None else NoCache(),
-              xml_report)
+              cache if cache is not None else NoCache())
   for cmdline in cmdlines:
     if not js.start(cmdline):
       break

tools/run_tests/post_tests_c.sh

@@ -34,8 +34,12 @@ if [ "$CONFIG" != "gcov" ] ; then exit ; fi
 
 root=$(readlink -f $(dirname $0)/../..)
 out=$root/reports/c_cxx_coverage
-tmp=$(mktemp)
+tmp1=$(mktemp)
+tmp2=$(mktemp)
 cd $root
-lcov --capture --directory . --output-file $tmp
-genhtml $tmp --output-directory $out
-rm $tmp
+lcov --capture --directory . --output-file $tmp1
+lcov --extract $tmp1 "$root/src/*" "$root/include/*" --output-file $tmp2
+genhtml $tmp2 --output-directory $out
+rm $tmp2
+rm $tmp1
+

tools/run_tests/report_utils.py

@@ -0,0 +1,189 @@
+# Copyright 2015, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Generate XML and HTML test reports."""
+
+import os
+import string
+import xml.etree.cElementTree as ET
+
+
+def _filter_msg(msg, output_format):
+  """Filters out nonprintable and illegal characters from the message."""
+  if output_format in ['XML', 'HTML']:
+    # keep whitespaces but remove formfeed and vertical tab characters
+    # that make XML report unparseable.
+    filtered_msg = filter(
+        lambda x: x in string.printable and x != '\f' and x != '\v',
+        msg.decode(errors='ignore'))
+    if output_format == 'HTML':
+      filtered_msg = filtered_msg.replace('"', '"')
+    return filtered_msg
+  else:
+    return msg
+
+
+def render_xml_report(resultset, xml_report):
+  """Generate JUnit-like XML report."""
+  root = ET.Element('testsuites')
+  testsuite = ET.SubElement(root, 'testsuite', id='1', package='grpc', 
+                            name='tests')
+  for shortname, results in resultset.iteritems(): 
+    for result in results:
+      xml_test = ET.SubElement(testsuite, 'testcase', name=shortname) 
+      if result.elapsed_time:
+        xml_test.set('time', str(result.elapsed_time))
+      ET.SubElement(xml_test, 'system-out').text = _filter_msg(result.message,
+                                                               'XML')
+      if result.state == 'FAILED':
+        ET.SubElement(xml_test, 'failure', message='Failure')
+      elif result.state == 'TIMEOUT':
+        ET.SubElement(xml_test, 'error', message='Timeout')
+  tree = ET.ElementTree(root)
+  tree.write(xml_report, encoding='UTF-8')
+
+
+# TODO(adelez): Use mako template.
+def fill_one_test_result(shortname, resultset, html_str):
+  if shortname in resultset:
+    # Because interop tests does not have runs_per_test flag, each test is run
+    # once. So there should only be one element for each result.
+    result = resultset[shortname][0] 
+    if result.state == 'PASSED':
+      html_str = '%s<td bgcolor=\"green\">PASS</td>\n' % html_str
+    else:
+      tooltip = ''
+      if result.returncode > 0 or result.message:
+        if result.returncode > 0:
+          tooltip = 'returncode: %d ' % result.returncode
+        if result.message:
+          escaped_msg = _filter_msg(result.message, 'HTML')
+          tooltip = '%smessage: %s' % (tooltip, escaped_msg)       
+      if result.state == 'FAILED':
+        html_str = '%s<td bgcolor=\"red\">' % html_str
+        if tooltip:  
+          html_str = ('%s<a href=\"#\" data-toggle=\"tooltip\" '
+                      'data-placement=\"auto\" title=\"%s\">FAIL</a></td>\n' % 
+                      (html_str, tooltip))
+        else:
+          html_str = '%sFAIL</td>\n' % html_str
+      elif result.state == 'TIMEOUT':
+        html_str = '%s<td bgcolor=\"yellow\">' % html_str
+        if tooltip:
+          html_str = ('%s<a href=\"#\" data-toggle=\"tooltip\" '
+                      'data-placement=\"auto\" title=\"%s\">TIMEOUT</a></td>\n' 
+                      % (html_str, tooltip))
+        else:
+          html_str = '%sTIMEOUT</td>\n' % html_str
+  else:
+    html_str = '%s<td bgcolor=\"magenta\">Not implemented</td>\n' % html_str
+  
+  return html_str
+
+
+def render_html_report(client_langs, server_langs, test_cases, auth_test_cases,
+                       resultset, num_failures, cloud_to_prod):
+  """Generate html report."""
+  sorted_test_cases = sorted(test_cases)
+  sorted_auth_test_cases = sorted(auth_test_cases)
+  sorted_client_langs = sorted(client_langs)
+  sorted_server_langs = sorted(server_langs)
+  html_str = ('<!DOCTYPE html>\n'
+              '<html lang=\"en\">\n'
+              '<head><title>Interop Test Result</title></head>\n'
+              '<body>\n')
+  if num_failures > 1:
+    html_str = (
+        '%s<p><h2><font color=\"red\">%d tests failed!</font></h2></p>\n' % 
+        (html_str, num_failures))
+  elif num_failures:
+    html_str = (
+        '%s<p><h2><font color=\"red\">%d test failed!</font></h2></p>\n' % 
+        (html_str, num_failures))
+  else:
+    html_str = (
+        '%s<p><h2><font color=\"green\">All tests passed!</font></h2></p>\n' % 
+        html_str)
+  if cloud_to_prod:
+    # Each column header is the client language.
+    html_str = ('%s<h2>Cloud to Prod</h2>\n' 
+                '<table style=\"width:100%%\" border=\"1\">\n'
+                '<tr bgcolor=\"#00BFFF\">\n'
+                '<th>Client languages &#9658;</th>\n') % html_str
+    for client_lang in sorted_client_langs:
+      html_str = '%s<th>%s\n' % (html_str, client_lang)
+    html_str = '%s</tr>\n' % html_str
+    for test_case in sorted_test_cases + sorted_auth_test_cases:
+      html_str = '%s<tr><td><b>%s</b></td>\n' % (html_str, test_case)
+      for client_lang in sorted_client_langs:
+        if not test_case in sorted_auth_test_cases:
+          shortname = 'cloud_to_prod:%s:%s' % (client_lang, test_case)
+        else:
+          shortname = 'cloud_to_prod_auth:%s:%s' % (client_lang, test_case)
+        html_str = fill_one_test_result(shortname, resultset, html_str)
+      html_str = '%s</tr>\n' % html_str 
+    html_str = '%s</table>\n' % html_str
+  if server_langs:
+    for test_case in sorted_test_cases:
+      # Each column header is the client language.
+      html_str = ('%s<h2>%s</h2>\n' 
+                  '<table style=\"width:100%%\" border=\"1\">\n'
+                  '<tr bgcolor=\"#00BFFF\">\n'
+                  '<th>Client languages &#9658;<br/>'
+                  'Server languages &#9660;</th>\n') % (html_str, test_case)
+      for client_lang in sorted_client_langs:
+        html_str = '%s<th>%s\n' % (html_str, client_lang)
+      html_str = '%s</tr>\n' % html_str
+      # Each row head is the server language.
+      for server_lang in sorted_server_langs:
+        html_str = '%s<tr><td><b>%s</b></td>\n' % (html_str, server_lang)
+        # Fill up the cells with test result.
+        for client_lang in sorted_client_langs:
+          shortname = 'cloud_to_cloud:%s:%s_server:%s' % (
+              client_lang, server_lang, test_case)
+          html_str = fill_one_test_result(shortname, resultset, html_str)
+        html_str = '%s</tr>\n' % html_str
+      html_str = '%s</table>\n' % html_str
+
+  html_str = ('%s\n'
+              '<script>\n'
+              '$(document).ready(function(){'
+              '$(\'[data-toggle=\"tooltip\"]\').tooltip();\n'   
+              '});\n'
+              '</script>\n'
+              '</body>\n'
+              '</html>') % html_str  
+  
+  # Write to reports/index.html as set up in Jenkins plugin.
+  html_report_dir = 'reports'
+  if not os.path.exists(html_report_dir):
+    os.mkdir(html_report_dir)
+  html_file_path = os.path.join(html_report_dir, 'index.html')
+  with open(html_file_path, 'w') as f:
+    f.write(html_str)

tools/run_tests/run_interop_tests.py

@@ -33,10 +33,10 @@
 import argparse
 import dockerjob
 import itertools
-import xml.etree.cElementTree as ET
 import jobset
 import multiprocessing
 import os
+import report_utils
 import subprocess
 import sys
 import tempfile
@@ -471,126 +471,6 @@ def build_interop_image_jobspec(language, tag=None):
   return build_job
 
 
-# TODO(adelez): Use mako template.
-def fill_one_test_result(shortname, resultset, html_str):
-  if shortname in resultset:
-    # Because interop tests does not have runs_per_test flag, each test is run
-    # once. So there should only be one element for each result.
-    result = resultset[shortname][0] 
-    if result.state == 'PASSED':
-      html_str = '%s<td bgcolor=\"green\">PASS</td>\n' % html_str
-    else:
-      tooltip = ''
-      if result.returncode > 0 or result.message:
-        if result.returncode > 0:
-          tooltip = 'returncode: %d ' % result.returncode
-        if result.message:
-          escaped_msg = result.message.replace('"', '&quot;')
-          tooltip = '%smessage: %s' % (tooltip, escaped_msg)     
-      if result.state == 'FAILED':
-        html_str = '%s<td bgcolor=\"red\">' % html_str
-        if tooltip:  
-          html_str = ('%s<a href=\"#\" data-toggle=\"tooltip\" '
-                      'data-placement=\"auto\" title=\"%s\">FAIL</a></td>\n' % 
-                      (html_str, tooltip))
-        else:
-          html_str = '%sFAIL</td>\n' % html_str
-      elif result.state == 'TIMEOUT':
-        html_str = '%s<td bgcolor=\"yellow\">' % html_str
-        if tooltip:
-          html_str = ('%s<a href=\"#\" data-toggle=\"tooltip\" '
-                      'data-placement=\"auto\" title=\"%s\">TIMEOUT</a></td>\n' 
-                      % (html_str, tooltip))
-        else:
-          html_str = '%sTIMEOUT</td>\n' % html_str
-  else:
-    html_str = '%s<td bgcolor=\"magenta\">Not implemented</td>\n' % html_str
-  
-  return html_str
-
-
-def render_html_report(client_langs, server_langs, resultset,
-                       num_failures):
-  """Generate html report."""
-  sorted_test_cases = sorted(_TEST_CASES)
-  sorted_auth_test_cases = sorted(_AUTH_TEST_CASES)
-  sorted_client_langs = sorted(client_langs)
-  sorted_server_langs = sorted(server_langs)
-  html_str = ('<!DOCTYPE html>\n'
-              '<html lang=\"en\">\n'
-              '<head><title>Interop Test Result</title></head>\n'
-              '<body>\n')
-  if num_failures > 1:
-    html_str = (
-        '%s<p><h2><font color=\"red\">%d tests failed!</font></h2></p>\n' % 
-        (html_str, num_failures))
-  elif num_failures:
-    html_str = (
-        '%s<p><h2><font color=\"red\">%d test failed!</font></h2></p>\n' % 
-        (html_str, num_failures))
-  else:
-    html_str = (
-        '%s<p><h2><font color=\"green\">All tests passed!</font></h2></p>\n' % 
-        html_str)
-  if args.cloud_to_prod_auth or args.cloud_to_prod:
-    # Each column header is the client language.
-    html_str = ('%s<h2>Cloud to Prod</h2>\n' 
-                '<table style=\"width:100%%\" border=\"1\">\n'
-                '<tr bgcolor=\"#00BFFF\">\n'
-                '<th>Client languages &#9658;</th>\n') % html_str
-    for client_lang in sorted_client_langs:
-      html_str = '%s<th>%s\n' % (html_str, client_lang)
-    html_str = '%s</tr>\n' % html_str
-    for test_case in sorted_test_cases + sorted_auth_test_cases:
-      html_str = '%s<tr><td><b>%s</b></td>\n' % (html_str, test_case)
-      for client_lang in sorted_client_langs:
-        if not test_case in sorted_auth_test_cases:
-          shortname = 'cloud_to_prod:%s:%s' % (client_lang, test_case)
-        else:
-          shortname = 'cloud_to_prod_auth:%s:%s' % (client_lang, test_case)
-        html_str = fill_one_test_result(shortname, resultset, html_str)
-      html_str = '%s</tr>\n' % html_str 
-    html_str = '%s</table>\n' % html_str
-  if servers:
-    for test_case in sorted_test_cases:
-      # Each column header is the client language.
-      html_str = ('%s<h2>%s</h2>\n' 
-                  '<table style=\"width:100%%\" border=\"1\">\n'
-                  '<tr bgcolor=\"#00BFFF\">\n'
-                  '<th>Client languages &#9658;<br/>'
-                  'Server languages &#9660;</th>\n') % (html_str, test_case)
-      for client_lang in sorted_client_langs:
-        html_str = '%s<th>%s\n' % (html_str, client_lang)
-      html_str = '%s</tr>\n' % html_str
-      # Each row head is the server language.
-      for server_lang in sorted_server_langs:
-        html_str = '%s<tr><td><b>%s</b></td>\n' % (html_str, server_lang)
-        # Fill up the cells with test result.
-        for client_lang in sorted_client_langs:
-          shortname = 'cloud_to_cloud:%s:%s_server:%s' % (
-              client_lang, server_lang, test_case)
-          html_str = fill_one_test_result(shortname, resultset, html_str)
-        html_str = '%s</tr>\n' % html_str
-      html_str = '%s</table>\n' % html_str
-
-  html_str = ('%s\n'
-              '<script>\n'
-              '$(document).ready(function(){'
-              '$(\'[data-toggle=\"tooltip\"]\').tooltip();\n'   
-              '});\n'
-              '</script>\n'
-              '</body>\n'
-              '</html>') % html_str  
-  
-  # Write to reports/index.html as set up in Jenkins plugin.
-  html_report_dir = 'reports'
-  if not os.path.exists(html_report_dir):
-    os.mkdir(html_report_dir)
-  html_file_path = os.path.join(html_report_dir, 'index.html')
-  with open(html_file_path, 'w') as f:
-    f.write(html_str)
-
-
 argp = argparse.ArgumentParser(description='Run interop tests.')
 argp.add_argument('-l', '--language',
                   choices=['all'] + sorted(_LANGUAGES),
@@ -740,22 +620,18 @@ try:
       dockerjob.remove_image(image, skip_nonexistent=True)
     sys.exit(1)
 
-  root = ET.Element('testsuites')
-  testsuite = ET.SubElement(root, 'testsuite', id='1', package='grpc', name='tests')
-
   num_failures, resultset = jobset.run(jobs, newline_on_success=True, 
-                                       maxjobs=args.jobs, xml_report=testsuite)
+                                       maxjobs=args.jobs)
   if num_failures:
     jobset.message('FAILED', 'Some tests failed', do_newline=True)
   else:
     jobset.message('SUCCESS', 'All tests passed', do_newline=True)
 
-  tree = ET.ElementTree(root)
-  tree.write('report.xml', encoding='UTF-8')
+  report_utils.render_xml_report(resultset, 'report.xml')
   
-  # Generate HTML report.
-  render_html_report(set([str(l) for l in languages]), servers,
-                     resultset, num_failures)
+  report_utils.render_html_report(
+      set([str(l) for l in languages]), servers, _TEST_CASES, _AUTH_TEST_CASES, 
+      resultset, num_failures, args.cloud_to_prod_auth or args.cloud_to_prod)
 
 finally:
   # Check if servers are still running.

tools/run_tests/run_tests.py

@@ -46,10 +46,10 @@ import sys
 import tempfile
 import traceback
 import time
-import xml.etree.cElementTree as ET
 import urllib2
 
 import jobset
+import report_utils
 import watch_dirs
 
 ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../..'))
@@ -844,6 +844,7 @@ def _build_and_run(
                  for _ in range(0, args.antagonists)]
   port_server_port = 32767
   _start_port_server(port_server_port)
+  resultset = None
   try:
     infinite_runs = runs_per_test == 0
     one_run = set(
@@ -867,15 +868,11 @@ def _build_and_run(
                      else itertools.repeat(massaged_one_run, runs_per_test))
     all_runs = itertools.chain.from_iterable(runs_sequence)
 
-    root = ET.Element('testsuites') if xml_report else None
-    testsuite = ET.SubElement(root, 'testsuite', id='1', package='grpc', name='tests') if xml_report else None
-
     number_failures, resultset = jobset.run(
-        all_runs, check_cancelled, newline_on_success=newline_on_success, 
+        all_runs, check_cancelled, newline_on_success=newline_on_success,
         travis=travis, infinite_runs=infinite_runs, maxjobs=args.jobs,
         stop_on_failure=args.stop_on_failure,
         cache=cache if not xml_report else None,
-        xml_report=testsuite,
         add_env={'GRPC_TEST_PORT_SERVER': 'localhost:%d' % port_server_port})
     if resultset:
       for k, v in resultset.iteritems():
@@ -893,9 +890,8 @@ def _build_and_run(
   finally:
     for antagonist in antagonists:
       antagonist.kill()
-    if xml_report:
-      tree = ET.ElementTree(root)
-      tree.write(xml_report, encoding='UTF-8')
+    if xml_report and resultset:
+      report_utils.render_xml_report(resultset, xml_report)
 
   number_failures, _ = jobset.run(
       post_tests_steps, maxjobs=1, stop_on_failure=True,