convert micro-benchmarks to benchmarkdotnet

.gitignore

@@ -146,3 +146,6 @@ bm_*.json
 
 # Clion artifacts
 cmake-build-debug/
+
+# Benchmark outputs
+BenchmarkDotNet.Artifacts/

src/csharp/Grpc.Microbenchmarks/CommonThreadedBase.cs

@@ -0,0 +1,67 @@
+#region Copyright notice and license
+
+// Copyright 2015 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#endregion
+
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+using BenchmarkDotNet.Attributes;
+using Grpc.Core;
+
+namespace Grpc.Microbenchmarks
+{
+
+    // common base-type for tests that need to run with some level of concurrency;
+    // note there's nothing *special* about this type - it is just to save some
+    // boilerplate
+
+    [ClrJob, CoreJob] // test .NET Core and .NET Framework
+    [MemoryDiagnoser] // allocations
+    [ShortRunJob] // don't take too long
+    public abstract class CommonThreadedBase
+    {
+        protected virtual bool NeedsEnvironment => true;
+
+        [Params(1, 1, 2, 4, 8, 12)]
+        public int ThreadCount { get; set; }
+
+        protected GrpcEnvironment Environment { get; private set; }
+
+        [GlobalSetup]
+        public virtual void Setup()
+        {
+            ThreadPool.GetMinThreads(out var workers, out var iocp);
+            if (workers <= ThreadCount) ThreadPool.SetMinThreads(ThreadCount + 1, iocp);
+            if (NeedsEnvironment) Environment = GrpcEnvironment.AddRef();
+        }
+
+        [GlobalCleanup]
+        public virtual void Cleanup()
+        {
+            if (Environment != null)
+            {
+                Environment = null;
+                GrpcEnvironment.ReleaseAsync().Wait();
+            }
+        }
+
+        protected void RunConcurrent(Action operation)
+        {
+            Parallel.For(0, ThreadCount, _ => operation());
+        }
+    }
+}

src/csharp/Grpc.Microbenchmarks/CompletionRegistryBenchmark.cs

@@ -1,4 +1,4 @@
-#region Copyright notice and license
+#region Copyright notice and license
 
 // Copyright 2015 gRPC authors.
 //
@@ -17,62 +17,38 @@
 #endregion
 
 using System;
-using System.Runtime.InteropServices;
-using System.Threading;
-using Grpc.Core;
+using BenchmarkDotNet.Attributes;
 using Grpc.Core.Internal;
-using System.Collections.Generic;
-using System.Diagnostics;
 
 namespace Grpc.Microbenchmarks
 {
-    public class CompletionRegistryBenchmark
+    public class CompletionRegistryBenchmarks : CommonThreadedBase
     {
-        GrpcEnvironment environment;
+        [Params(false, true)]
+        public bool UseSharedRegistry { get; set; }
 
-        public void Init()
+        const int Iterations = 1000;
+        [Benchmark(OperationsPerInvoke = Iterations)]
+        public void Run()
         {
-            environment = GrpcEnvironment.AddRef();
+            RunConcurrent(() => {
+                CompletionRegistry sharedRegistry = UseSharedRegistry ? new CompletionRegistry(Environment, () => BatchContextSafeHandle.Create(), () => RequestCallContextSafeHandle.Create()) : null;
+                RunBody(sharedRegistry);
+            });
         }
 
-        public void Cleanup()
+        private void RunBody(CompletionRegistry optionalSharedRegistry)
         {
-            GrpcEnvironment.ReleaseAsync().Wait();
-        }
-
-        public void Run(int threadCount, int iterations, bool useSharedRegistry)
-        {
-            Console.WriteLine(string.Format("CompletionRegistryBenchmark: threads={0}, iterations={1}, useSharedRegistry={2}", threadCount, iterations, useSharedRegistry));
-            CompletionRegistry sharedRegistry = useSharedRegistry ? new CompletionRegistry(environment, () => BatchContextSafeHandle.Create(), () => RequestCallContextSafeHandle.Create()) : null;
-            var threadedBenchmark = new ThreadedBenchmark(threadCount, () => ThreadBody(iterations, sharedRegistry));
-            threadedBenchmark.Run();
-            // TODO: parametrize by number of pending completions
-        }
-
-        private void ThreadBody(int iterations, CompletionRegistry optionalSharedRegistry)
-        {
-            var completionRegistry = optionalSharedRegistry ?? new CompletionRegistry(environment, () => throw new NotImplementedException(), () => throw new NotImplementedException());
+            var completionRegistry = optionalSharedRegistry ?? new CompletionRegistry(Environment, () => throw new NotImplementedException(), () => throw new NotImplementedException());
             var ctx = BatchContextSafeHandle.Create();
-  
-            var stopwatch = Stopwatch.StartNew();
-            for (int i = 0; i < iterations; i++)
+
+            for (int i = 0; i < Iterations; i++)
             {
                 completionRegistry.Register(ctx.Handle, ctx);
                 var callback = completionRegistry.Extract(ctx.Handle);
                 // NOTE: we are not calling the callback to avoid disposing ctx.
             }
-            stopwatch.Stop();
-            Console.WriteLine("Elapsed millis: " + stopwatch.ElapsedMilliseconds);          
-
             ctx.Recycle();
         }
-
-        private class NopCompletionCallback : IOpCompletionCallback
-        {
-            public void OnComplete(bool success)
-            {
-
-            }
-        }
     }
 }

src/csharp/Grpc.Microbenchmarks/GCStats.cs

@@ -1,69 +0,0 @@
-#region Copyright notice and license
-
-// Copyright 2015 gRPC authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#endregion
-
-using System;
-using Grpc.Core;
-using Grpc.Core.Internal;
-
-namespace Grpc.Microbenchmarks
-{
-    internal class GCStats
-    {
-        readonly object myLock = new object();
-        GCStatsSnapshot lastSnapshot;
-
-        public GCStats()
-        {
-            lastSnapshot = new GCStatsSnapshot(GC.CollectionCount(0), GC.CollectionCount(1), GC.CollectionCount(2));
-        }
-
-        public GCStatsSnapshot GetSnapshot(bool reset = false)
-        {
-            lock (myLock)
-            {
-                var newSnapshot = new GCStatsSnapshot(GC.CollectionCount(0) - lastSnapshot.Gen0,
-                    GC.CollectionCount(1) - lastSnapshot.Gen1,
-                    GC.CollectionCount(2) - lastSnapshot.Gen2);
-                if (reset)
-                {
-                    lastSnapshot = newSnapshot;
-                }
-                return newSnapshot;
-            }
-        }
-    }
-
-    public class GCStatsSnapshot
-    {
-        public GCStatsSnapshot(int gen0, int gen1, int gen2)
-        {
-            this.Gen0 = gen0;
-            this.Gen1 = gen1;
-            this.Gen2 = gen2;
-        }
-
-        public int Gen0 { get; }
-        public int Gen1 { get; }
-        public int Gen2 { get; }
-
-        public override string ToString()
-        {
-            return string.Format("[GCCollectionCount: gen0 {0}, gen1 {1}, gen2 {2}]", Gen0, Gen1, Gen2);
-        }
-    }
-}

src/csharp/Grpc.Microbenchmarks/Grpc.Microbenchmarks.csproj

@@ -3,7 +3,7 @@
   <Import Project="..\Grpc.Core\Common.csproj.include" />
 
   <PropertyGroup>
-    <TargetFrameworks>net45;netcoreapp2.1</TargetFrameworks>
+    <TargetFrameworks>net461;netcoreapp2.1</TargetFrameworks>
     <OutputType>Exe</OutputType>
     <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
   </PropertyGroup>
@@ -13,10 +13,10 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="CommandLineParser" Version="2.3.0" />
+    <PackageReference Include="BenchmarkDotNet" Version="0.11.5" />
   </ItemGroup>
 
-  <ItemGroup Condition=" '$(TargetFramework)' == 'net45' ">
+  <ItemGroup Condition=" '$(TargetFramework)' == 'net461' ">
     <Reference Include="System" />
     <Reference Include="Microsoft.CSharp" />
   </ItemGroup>
@@ -24,5 +24,4 @@
   <ItemGroup>
     <Compile Include="..\Grpc.Core.Api\Version.cs" />
   </ItemGroup>
-
 </Project>

src/csharp/Grpc.Microbenchmarks/PInvokeByteArrayBenchmark.cs

@@ -1,4 +1,4 @@
-#region Copyright notice and license
+#region Copyright notice and license
 
 // Copyright 2015 gRPC authors.
 //
@@ -16,49 +16,39 @@
 
 #endregion
 
-using System;
 using System.Runtime.InteropServices;
-using System.Threading;
-using Grpc.Core;
+using BenchmarkDotNet.Attributes;
 using Grpc.Core.Internal;
-using System.Collections.Generic;
-using System.Diagnostics;
 
 namespace Grpc.Microbenchmarks
 {
-    public class PInvokeByteArrayBenchmark
+    public class PInvokeByteArrayBenchmark : CommonThreadedBase
     {
         static readonly NativeMethods Native = NativeMethods.Get();
 
-        public void Init()
-        {
-        }
+        protected override bool NeedsEnvironment => false;
 
-        public void Cleanup()
-        {
-        }
 
-        public void Run(int threadCount, int iterations, int payloadSize)
+        [Params(0)]
+        public int PayloadSize { get; set; }
+
+        const int Iterations = 1000;
+        [Benchmark(OperationsPerInvoke = Iterations)]
+        public void Run()
         {
-            Console.WriteLine(string.Format("PInvokeByteArrayBenchmark: threads={0}, iterations={1}, payloadSize={2}", threadCount, iterations, payloadSize));
-            var threadedBenchmark = new ThreadedBenchmark(threadCount, () => ThreadBody(iterations, payloadSize));
-            threadedBenchmark.Run();
+            RunConcurrent(RunBody);
         }
 
-        private void ThreadBody(int iterations, int payloadSize)
+        private void RunBody()
         {
-            var payload = new byte[payloadSize];
-         
-            var stopwatch = Stopwatch.StartNew();
-            for (int i = 0; i < iterations; i++)
+            var payload = new byte[PayloadSize];
+            for (int i = 0; i < Iterations; i++)
             {
                 var gcHandle = GCHandle.Alloc(payload, GCHandleType.Pinned);
                 var payloadPtr = gcHandle.AddrOfPinnedObject();
                 Native.grpcsharp_test_nop(payloadPtr);
                 gcHandle.Free();
             }
-            stopwatch.Stop();
-            Console.WriteLine("Elapsed millis: " + stopwatch.ElapsedMilliseconds);
         }
     }
 }

src/csharp/Grpc.Microbenchmarks/Program.cs

@@ -1,4 +1,4 @@
-#region Copyright notice and license
+#region Copyright notice and license
 
 // Copyright 2015 gRPC authors.
 //
@@ -16,95 +16,18 @@
 
 #endregion
 
-using System;
-using Grpc.Core;
-using Grpc.Core.Internal;
-using Grpc.Core.Logging;
-using CommandLine;
-using CommandLine.Text;
+using BenchmarkDotNet.Running;
 
 namespace Grpc.Microbenchmarks
 {
     class Program
     {
-        public enum MicrobenchmarkType
-        {
-            CompletionRegistry,
-            PInvokeByteArray,
-            SendMessage
-        }
-
-        private class BenchmarkOptions
-        {
-            [Option("benchmark", Required = true, HelpText = "Benchmark to run")]
-            public MicrobenchmarkType Benchmark { get; set; }
-        }
-
+        // typical usage: dotnet run -c Release -f netcoreapp2.1
+        // (this will profile both .net core and .net framework; for some reason
+        // if you start from "-f net461", it goes horribly wrong)
         public static void Main(string[] args)
         {
-            GrpcEnvironment.SetLogger(new ConsoleLogger());
-            var parserResult = Parser.Default.ParseArguments<BenchmarkOptions>(args)
-                .WithNotParsed(errors => {
-                    Console.WriteLine("Supported benchmarks:");
-                    foreach (var enumValue in Enum.GetValues(typeof(MicrobenchmarkType)))
-                    {
-                        Console.WriteLine("  " + enumValue);
-                    }
-                    Environment.Exit(1);
-                })
-                .WithParsed(options =>
-                {
-                    switch (options.Benchmark)
-                    {
-                        case MicrobenchmarkType.CompletionRegistry:
-                          RunCompletionRegistryBenchmark();
-                          break;
-                        case MicrobenchmarkType.PInvokeByteArray:
-                          RunPInvokeByteArrayBenchmark();
-                          break;
-                        case MicrobenchmarkType.SendMessage:
-                          RunSendMessageBenchmark();
-                          break;
-                        default:
-                          throw new ArgumentException("Unsupported benchmark.");
-                    }
-                });
-        }
-
-        static void RunCompletionRegistryBenchmark()
-        {
-            var benchmark = new CompletionRegistryBenchmark();
-            benchmark.Init();
-            foreach (int threadCount in new int[] {1, 1, 2, 4, 8, 12})
-            {
-                foreach (bool useSharedRegistry in new bool[] {false, true})
-                {
-                    benchmark.Run(threadCount, 4 * 1000 * 1000, useSharedRegistry);
-                }
-            }
-            benchmark.Cleanup();
-        }
-
-        static void RunPInvokeByteArrayBenchmark()
-        {
-            var benchmark = new PInvokeByteArrayBenchmark();
-            benchmark.Init();
-            foreach (int threadCount in new int[] {1, 1, 2, 4, 8, 12})
-            {
-                benchmark.Run(threadCount, 4 * 1000 * 1000, 0);
-            }
-            benchmark.Cleanup();
-        }
-
-        static void RunSendMessageBenchmark()
-        {
-            var benchmark = new SendMessageBenchmark();
-            benchmark.Init();
-            foreach (int threadCount in new int[] {1, 1, 2, 4, 8, 12})
-            {
-                benchmark.Run(threadCount, 4 * 1000 * 1000, 0);
-            }
-            benchmark.Cleanup();
+            BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args);
         }
     }
 }

src/csharp/Grpc.Microbenchmarks/SendMessageBenchmark.cs

@@ -17,59 +17,48 @@
 #endregion
 
 using System;
-using System.Threading;
+using BenchmarkDotNet.Attributes;
 using Grpc.Core;
 using Grpc.Core.Internal;
-using System.Collections.Generic;
-using System.Diagnostics;
 
 namespace Grpc.Microbenchmarks
 {
-    public class SendMessageBenchmark
+    public class SendMessageBenchmark : CommonThreadedBase
     {
         static readonly NativeMethods Native = NativeMethods.Get();
 
-        GrpcEnvironment environment;
-
-        public void Init()
+        public override void Setup()
         {
             Native.grpcsharp_test_override_method("grpcsharp_call_start_batch", "nop");
-            environment = GrpcEnvironment.AddRef();
+            base.Setup();
         }
 
-        public void Cleanup()
-        {
-            GrpcEnvironment.ReleaseAsync().Wait();
-            // TODO(jtattermusch): track GC stats
-        }
+        [Params(0)]
+        public int PayloadSize { get; set; }
 
-        public void Run(int threadCount, int iterations, int payloadSize)
+        const int Iterations = 1000;
+        [Benchmark(OperationsPerInvoke = Iterations)]
+        public void Run()
         {
-            Console.WriteLine(string.Format("SendMessageBenchmark: threads={0}, iterations={1}, payloadSize={2}", threadCount, iterations, payloadSize));
-            var threadedBenchmark = new ThreadedBenchmark(threadCount, () => ThreadBody(iterations, payloadSize));
-            threadedBenchmark.Run();
+            RunConcurrent(RunBody);
         }
 
-        private void ThreadBody(int iterations, int payloadSize)
+        private void RunBody()
         {
-            var completionRegistry = new CompletionRegistry(environment, () => environment.BatchContextPool.Lease(), () => throw new NotImplementedException());
+            var completionRegistry = new CompletionRegistry(Environment, () => Environment.BatchContextPool.Lease(), () => throw new NotImplementedException());
             var cq = CompletionQueueSafeHandle.CreateAsync(completionRegistry);
             var call = CreateFakeCall(cq);
 
             var sendCompletionCallback = new NopSendCompletionCallback();
-            var payload = new byte[payloadSize];
+            var payload = new byte[PayloadSize];
             var writeFlags = default(WriteFlags);
 
-            var stopwatch = Stopwatch.StartNew();
-            for (int i = 0; i < iterations; i++)
+            for (int i = 0; i < Iterations; i++)
             {
                 call.StartSendMessage(sendCompletionCallback, payload, writeFlags, false);
                 var callback = completionRegistry.Extract(completionRegistry.LastRegisteredKey);
                 callback.OnComplete(true);
             }
-            stopwatch.Stop();
-            Console.WriteLine("Elapsed millis: " + stopwatch.ElapsedMilliseconds);
-
             cq.Dispose();
         }
 

src/csharp/Grpc.Microbenchmarks/ThreadedBenchmark.cs

@@ -1,65 +0,0 @@
-#region Copyright notice and license
-
-// Copyright 2015 gRPC authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#endregion
-
-using System;
-using System.Threading;
-using Grpc.Core;
-using Grpc.Core.Internal;
-using System.Collections.Generic;
-using System.Diagnostics;
-
-namespace Grpc.Microbenchmarks
-{
-    public class ThreadedBenchmark
-    {
-        List<ThreadStart> runners;
-
-        public ThreadedBenchmark(IEnumerable<ThreadStart> runners)
-        {
-            this.runners = new List<ThreadStart>(runners);
-        }
-
-        public ThreadedBenchmark(int threadCount, Action threadBody)
-        {
-            this.runners = new List<ThreadStart>();
-            for (int i = 0; i < threadCount; i++)
-            {
-                this.runners.Add(new ThreadStart(() => threadBody()));
-            }
-        }
-        
-        public void Run()
-        {
-            Console.WriteLine("Running threads.");
-            var gcStats = new GCStats();
-            var threads = new List<Thread>();
-            for (int i = 0; i < runners.Count; i++)
-            {
-                var thread = new Thread(runners[i]);
-                thread.Start();
-                threads.Add(thread);
-            }
-
-            foreach (var thread in threads)
-            {
-                thread.Join();
-            }
-            Console.WriteLine("All threads finished (GC Stats Delta: " + gcStats.GetSnapshot() + ")");
-        }
-    }
-}