Evolve benchmark

Signed-off-by: Tomasz Maruszak <[email protected]>
zarusz · Jan 10, 2025 · aeab47b · aeab47b
1 parent 0abaafa
commit aeab47b
Show file tree

Hide file tree

Showing 11 changed files with 122 additions and 60 deletions.
diff --git a/src/Host.Plugin.Properties.xml b/src/Host.Plugin.Properties.xml
@@ -4,7 +4,7 @@
   <Import Project="Common.NuGet.Properties.xml" />
 
   <PropertyGroup>
-    <Version>3.0.0-rc902</Version>
+    <Version>3.0.0-rc903</Version>
   </PropertyGroup>
 
 </Project>
diff --git a/src/SlimMessageBus.Host.Configuration/SlimMessageBus.Host.Configuration.csproj b/src/SlimMessageBus.Host.Configuration/SlimMessageBus.Host.Configuration.csproj
@@ -6,7 +6,7 @@
     <Description>Core configuration interfaces of SlimMessageBus</Description>
     <PackageTags>SlimMessageBus</PackageTags>
     <RootNamespace>SlimMessageBus.Host</RootNamespace>
-    <Version>3.0.0-rc901</Version>
+    <Version>3.0.0-rc903</Version>
   </PropertyGroup>
 
   <ItemGroup>

diff --git a/src/SlimMessageBus.Host.Interceptor/SlimMessageBus.Host.Interceptor.csproj b/src/SlimMessageBus.Host.Interceptor/SlimMessageBus.Host.Interceptor.csproj
@@ -3,7 +3,7 @@
   <Import Project="../Common.NuGet.Properties.xml" />
 
   <PropertyGroup>
-    <Version>3.0.0-rc901</Version>
+    <Version>3.0.0-rc903</Version>
     <Description>Core interceptor interfaces of SlimMessageBus</Description>
     <PackageTags>SlimMessageBus</PackageTags>
   </PropertyGroup>

diff --git a/src/SlimMessageBus.Host.Memory/Consumers/MessageProcessorQueue.cs b/src/SlimMessageBus.Host.Memory/Consumers/MessageProcessorQueue.cs
@@ -1,6 +1,9 @@
 namespace SlimMessageBus.Host.Memory;
 
-public class MessageProcessorQueue(IMessageProcessor<object> messageProcessor, ILogger<MessageProcessorQueue> logger, CancellationToken cancellationToken) : AbstractMessageProcessorQueue(messageProcessor, logger)
+public class MessageProcessorQueue(IMessageProcessor<object> messageProcessor,
+                                   ILogger<MessageProcessorQueue> logger,
+                                   CancellationToken cancellationToken)
+    : AbstractMessageProcessorQueue(messageProcessor, logger)
 {
     private readonly object _prevTaskLock = new();
     private Task _prevTask = null;

diff --git a/src/SlimMessageBus.Host.Serialization.Avro/AvroMessageSerializer.cs b/src/SlimMessageBus.Host.Serialization.Avro/AvroMessageSerializer.cs
@@ -52,8 +52,8 @@ public AvroMessageSerializer(ILoggerFactory loggerFactory = null)
         var mf = new ReflectionMessageCreationStrategy(loggerFactory.CreateLogger<ReflectionMessageCreationStrategy>());
         var ml = new ReflectionSchemaLookupStrategy(loggerFactory.CreateLogger<ReflectionSchemaLookupStrategy>());
 
-        MessageFactory = (Type type) => mf.Create(type);
-        WriteSchemaLookup = (Type type) => ml.Lookup(type);
+        MessageFactory = mf.Create;
+        WriteSchemaLookup = ml.Lookup;
         ReadSchemaLookup = WriteSchemaLookup;
     }
 
@@ -73,7 +73,7 @@ public AvroMessageSerializer(ILoggerFactory loggerFactory, IMessageCreationStrat
     public object Deserialize(Type t, byte[] payload)
     {
         using var ms = ReadMemoryStreamFactory(payload);
-        
+
         var dec = new BinaryDecoder(ms);
 
         var message = MessageFactory(t);
@@ -84,7 +84,7 @@ public object Deserialize(Type t, byte[] payload)
         var writerSchema = WriteSchemaLookup(t);
         AssertSchemaNotNull(t, writerSchema, true);
 
-        _logger.LogDebug("Type {0} writer schema: {1}, reader schema: {2}", t, writerSchema, readerSchema);
+        _logger.LogDebug("Type {Type} writer schema: {WriterSchema}, reader schema: {ReaderSchema}", t, writerSchema, readerSchema);
 
         var reader = new SpecificDefaultReader(writerSchema, readerSchema);
         reader.Read(message, dec);
@@ -108,7 +108,7 @@ public byte[] Serialize(Type t, object message)
         var writerSchema = WriteSchemaLookup(t);
         AssertSchemaNotNull(t, writerSchema, true);
 
-        _logger.LogDebug("Type {0} writer schema: {1}", t, writerSchema);
+        _logger.LogDebug("Type {Type} writer schema: {WriterSchema}", t, writerSchema);
 
         var writer = new SpecificDefaultWriter(writerSchema); // Schema comes from pre-compiled, code-gen phase
         writer.Write(message, enc);

diff --git a/src/SlimMessageBus.Host.Serialization/SlimMessageBus.Host.Serialization.csproj b/src/SlimMessageBus.Host.Serialization/SlimMessageBus.Host.Serialization.csproj
@@ -3,7 +3,7 @@
   <Import Project="../Common.NuGet.Properties.xml" />
 
   <PropertyGroup>
-    <Version>3.0.0-rc901</Version>
+    <Version>3.0.0-rc903</Version>
     <Description>Core serialization interfaces of SlimMessageBus</Description>
     <PackageTags>SlimMessageBus</PackageTags>
   </PropertyGroup>

diff --git a/src/SlimMessageBus/SlimMessageBus.csproj b/src/SlimMessageBus/SlimMessageBus.csproj
@@ -3,7 +3,7 @@
   <Import Project="../Common.NuGet.Properties.xml" />
 
   <PropertyGroup>
-    <Version>3.0.0-rc902</Version>
+    <Version>3.0.0-rc903</Version>
     <Description>
       This library provides a lightweight, easy-to-use message bus interface for .NET, offering a simplified facade for working with messaging brokers.
       It supports multiple transport providers for popular messaging systems, as well as in-memory (in-process) messaging for efficient local communication.

diff --git a/src/Tests/SlimMessageBus.Host.Memory.Benchmark/AbstractMemoryBenchmark.cs b/src/Tests/SlimMessageBus.Host.Memory.Benchmark/AbstractMemoryBenchmark.cs
@@ -1,29 +1,41 @@
 namespace SlimMessageBus.Host.Memory.Benchmark;
 
+using System.Reflection;
+
 using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Logging.Abstractions;
 
 using SlimMessageBus.Host;
 
-using System.Reflection;
-
 public abstract class AbstractMemoryBenchmark : IDisposable
 {
-    protected ServiceProvider svp;
-    protected readonly IMessageBus bus;
+    private Lazy<ServiceProvider> _serviceProvider;
+
+    protected IServiceProvider ServiceProvider => _serviceProvider.Value;
+
+    protected bool PerMessageScopeEnabled { get; set; }
+
+    protected IMessageBus Bus => ServiceProvider.GetRequiredService<IMessageBus>();
 
     protected AbstractMemoryBenchmark()
     {
-        var services = new ServiceCollection();
-
-        services.AddSlimMessageBus(mbb => mbb.WithProviderMemory().AutoDeclareFrom(Assembly.GetExecutingAssembly()));
+        _serviceProvider = new Lazy<ServiceProvider>(() =>
+        {
+            var services = new ServiceCollection();
 
-        services.AddSingleton<TestResult>();
-        services.AddTransient<SomeRequestHandler>();
-        Setup(services);
+            services.AddSlimMessageBus(mbb => mbb
+                .WithProviderMemory()
+                .AutoDeclareFrom(Assembly.GetExecutingAssembly())
+                .PerMessageScopeEnabled(PerMessageScopeEnabled));
 
-        svp = services.BuildServiceProvider();
+            services.AddSingleton<ILoggerFactory>(NullLoggerFactory.Instance);
+            services.AddSingleton<TestResult>();
+            services.AddTransient<SomeRequestHandler>();
+            Setup(services);
 
-        bus = svp.GetRequiredService<IMessageBus>();
+            return services.BuildServiceProvider();
+        });
     }
 
     protected virtual void Setup(ServiceCollection services)
@@ -32,10 +44,10 @@ protected virtual void Setup(ServiceCollection services)
 
     public void Dispose()
     {
-        if (svp != null)
+        if (_serviceProvider.Value != null)
         {
-            svp.Dispose();
-            svp = null;
+            _serviceProvider.Value.Dispose();
+            _serviceProvider = null;
         }
     }
 }
diff --git a/src/Tests/SlimMessageBus.Host.Memory.Benchmark/PubSubBenchmark.cs b/src/Tests/SlimMessageBus.Host.Memory.Benchmark/PubSubBenchmark.cs
@@ -8,25 +8,21 @@
 
 public abstract class PubSubBaseBenchmark : AbstractMemoryBenchmark
 {
-    private readonly TestResult testResult;
-
-    public PubSubBaseBenchmark()
-    {
-        testResult = svp.GetRequiredService<TestResult>();
-    }
-
     protected override void Setup(ServiceCollection services)
     {
         services.AddSingleton<TestResult>();
         services.AddTransient<SomeEventConsumer>();
     }
 
-    protected async Task RunTest(int messageCount)
+    protected async Task RunTest(int messageCount, bool createMessageScope)
     {
+        PerMessageScopeEnabled = createMessageScope;
+        var bus = Bus;
         var publishTasks = Enumerable.Range(0, messageCount).Select(x => bus.Publish(new SomeEvent(DateTimeOffset.Now, x)));
 
         await Task.WhenAll(publishTasks);
 
+        var testResult = ServiceProvider.GetRequiredService<TestResult>();
         while (testResult.ArrivedCount < messageCount)
         {
             await Task.Yield();
@@ -38,8 +34,9 @@ protected async Task RunTest(int messageCount)
 public class PubSubBenchmark : PubSubBaseBenchmark
 {
     [Benchmark]
-    [Arguments(1000000)]
-    public Task PubSub(int messageCount) => RunTest(messageCount);
+    [Arguments(1000000, true)]
+    [Arguments(1000000, false)]
+    public Task PubSub(int messageCount, bool createMessageScope) => RunTest(messageCount, createMessageScope);
 }
 
 [MemoryDiagnoser]
@@ -53,8 +50,9 @@ protected override void Setup(ServiceCollection services)
     }
 
     [Benchmark]
-    [Arguments(1000000)]
-    public Task PubSubWithProducerInterceptor(int messageCount) => RunTest(messageCount);
+    [Arguments(1000000, true)]
+    [Arguments(1000000, false)]
+    public Task PubSubWithProducerInterceptor(int messageCount, bool createMessageScope) => RunTest(messageCount, createMessageScope);
 }
 
 [MemoryDiagnoser]
@@ -68,8 +66,9 @@ protected override void Setup(ServiceCollection services)
     }
 
     [Benchmark]
-    [Arguments(1000000)]
-    public Task PubSubWithPublishInterceptor(int messageCount) => RunTest(messageCount);
+    [Arguments(1000000, true)]
+    [Arguments(1000000, false)]
+    public Task PubSubWithPublishInterceptor(int messageCount, bool createMessageScope) => RunTest(messageCount, createMessageScope);
 }
 
 [MemoryDiagnoser]
@@ -83,8 +82,9 @@ protected override void Setup(ServiceCollection services)
     }
 
     [Benchmark]
-    [Arguments(1000000)]
-    public Task PubSubWithConsumerInterceptor(int messageCount) => RunTest(messageCount);
+    [Arguments(1000000, true)]
+    [Arguments(1000000, false)]
+    public Task PubSubWithConsumerInterceptor(int messageCount, bool createMessageScope) => RunTest(messageCount, createMessageScope);
 }
 
 public record SomeEvent(DateTimeOffset Timestamp, long Id);

diff --git a/src/Tests/SlimMessageBus.Host.Memory.Benchmark/README.md b/src/Tests/SlimMessageBus.Host.Memory.Benchmark/README.md
@@ -0,0 +1,46 @@
+Sample Benchmark results
+
+```
+// * Summary *
+
+BenchmarkDotNet v0.14.0, Windows 11 (10.0.26100.2454)
+12th Gen Intel Core i7-1260P, 1 CPU, 16 logical and 12 physical cores
+.NET SDK 9.0.100
+  [Host]     : .NET 8.0.11 (8.0.1124.51707), X64 RyuJIT AVX2
+  Job-WFUQPN : .NET 8.0.11 (8.0.1124.51707), X64 RyuJIT AVX2
+
+MaxIterationCount=30  MaxWarmupIterationCount=10
+```
+
+| Type                                   | Method                        | messageCount | createMessageScope |     Mean |    Error |   StdDev |        Gen0 |      Gen1 |      Gen2 | Allocated |
+| -------------------------------------- | ----------------------------- | ------------ | ------------------ | -------: | -------: | -------: | ----------: | --------: | --------: | --------: |
+| PubSubBenchmark                        | PubSub                        | 1000000      | False              | 730.6 ms | 14.58 ms | 17.36 ms | 119000.0000 | 3000.0000 | 3000.0000 |   1.04 GB |
+| PubSubWithConsumerInterceptorBenchmark | PubSubWithConsumerInterceptor | 1000000      | False              | 810.5 ms | 16.16 ms | 16.59 ms | 146000.0000 | 3000.0000 | 3000.0000 |   1.28 GB |
+| PubSubWithProducerInterceptorBenchmark | PubSubWithProducerInterceptor | 1000000      | False              | 823.5 ms |  7.74 ms |  6.86 ms | 156000.0000 | 3000.0000 | 3000.0000 |   1.37 GB |
+| PubSubWithPublishInterceptorBenchmark  | PubSubWithPublishInterceptor  | 1000000      | False              | 831.8 ms |  9.43 ms |  7.87 ms | 156000.0000 | 3000.0000 | 3000.0000 |   1.37 GB |
+| PubSubBenchmark                        | PubSub                        | 1000000      | True               | 794.8 ms |  5.44 ms |  4.54 ms | 137000.0000 | 3000.0000 | 3000.0000 |    1.2 GB |
+| PubSubWithConsumerInterceptorBenchmark | PubSubWithConsumerInterceptor | 1000000      | True               | 900.9 ms | 11.65 ms | 10.32 ms | 164000.0000 | 3000.0000 | 3000.0000 |   1.44 GB |
+| PubSubWithProducerInterceptorBenchmark | PubSubWithProducerInterceptor | 1000000      | True               | 934.5 ms | 14.15 ms | 13.24 ms | 174000.0000 | 3000.0000 | 3000.0000 |   1.53 GB |
+| PubSubWithPublishInterceptorBenchmark  | PubSubWithPublishInterceptor  | 1000000      | True               | 930.6 ms | 14.29 ms | 13.37 ms | 174000.0000 | 3000.0000 | 3000.0000 |   1.53 GB |
+
+```
+// * Hints *
+
+Outliers
+  PubSubWithProducerInterceptorBenchmark.PubSubWithProducerInterceptor: MaxIterationCount=30, MaxWarmupIterationCount=10 -> 1 outlier  was  removed (840.20 ms)
+  PubSubWithPublishInterceptorBenchmark.PubSubWithPublishInterceptor: MaxIterationCount=30, MaxWarmupIterationCount=10   -> 2 outliers were removed (862.16 ms, 863.90 ms)
+  PubSubBenchmark.PubSub: MaxIterationCount=30, MaxWarmupIterationCount=10                                               -> 2 outliers were removed (810.49 ms, 823.65 ms)
+  PubSubWithConsumerInterceptorBenchmark.PubSubWithConsumerInterceptor: MaxIterationCount=30, MaxWarmupIterationCount=10 -> 1 outlier  was  removed (947.16 ms)
+
+// * Legends *
+  messageCount       : Value of the 'messageCount' parameter
+  createMessageScope : Value of the 'createMessageScope' parameter
+  Mean               : Arithmetic mean of all measurements
+  Error              : Half of 99.9% confidence interval
+  StdDev             : Standard deviation of all measurements
+  Gen0               : GC Generation 0 collects per 1000 operations
+  Gen1               : GC Generation 1 collects per 1000 operations
+  Gen2               : GC Generation 2 collects per 1000 operations
+  Allocated          : Allocated memory per single operation (managed only, inclusive, 1KB = 1024B)
+  1 ms               : 1 Millisecond (0.001 sec)
+```
diff --git a/src/Tests/SlimMessageBus.Host.Memory.Benchmark/ReqRespBenchmark.cs b/src/Tests/SlimMessageBus.Host.Memory.Benchmark/ReqRespBenchmark.cs
@@ -8,25 +8,21 @@
 
 public abstract class ReqRespBaseBenchmark : AbstractMemoryBenchmark
 {
-    private readonly TestResult testResult;
-
-    protected ReqRespBaseBenchmark()
-    {
-        testResult = svp.GetRequiredService<TestResult>();
-    }
-
     protected override void Setup(ServiceCollection services)
     {
         services.AddSingleton<TestResult>();
         services.AddTransient<SomeRequestHandler>();
     }
 
-    public async Task RunTest(int messageCount)
+    public async Task RunTest(int messageCount, bool createMessageScope)
     {
+        PerMessageScopeEnabled = createMessageScope;
+        var bus = Bus;
         var sendRequests = Enumerable.Range(0, messageCount).Select(x => bus.Send(new SomeRequest(DateTimeOffset.Now, x)));
 
         await Task.WhenAll(sendRequests);
 
+        var testResult = ServiceProvider.GetRequiredService<TestResult>();
         while (testResult.ArrivedCount < messageCount)
         {
             await Task.Yield();
@@ -38,8 +34,9 @@ public async Task RunTest(int messageCount)
 public class ReqRespBenchmark : ReqRespBaseBenchmark
 {
     [Benchmark]
-    [Arguments(1000000)]
-    public Task RequestResponse(int messageCount) => RunTest(messageCount);
+    [Arguments(1000000, true)]
+    [Arguments(1000000, false)]
+    public Task RequestResponse(int messageCount, bool createMessageScope) => RunTest(messageCount, createMessageScope);
 }
 
 [MemoryDiagnoser]
@@ -53,8 +50,9 @@ protected override void Setup(ServiceCollection services)
     }
 
     [Benchmark]
-    [Arguments(1000000)]
-    public Task ReqRespWithProducerInterceptor(int messageCount) => RunTest(messageCount);
+    [Arguments(1000000, true)]
+    [Arguments(1000000, false)]
+    public Task ReqRespWithProducerInterceptor(int messageCount, bool createMessageScope) => RunTest(messageCount, createMessageScope);
 }
 
 [MemoryDiagnoser]
@@ -68,8 +66,9 @@ protected override void Setup(ServiceCollection services)
     }
 
     [Benchmark]
-    [Arguments(1000000)]
-    public Task ReqRespWithSendInterceptor(int messageCount) => RunTest(messageCount);
+    [Arguments(1000000, true)]
+    [Arguments(1000000, false)]
+    public Task ReqRespWithSendInterceptor(int messageCount, bool createMessageScope) => RunTest(messageCount, createMessageScope);
 }
 
 [MemoryDiagnoser]
@@ -83,8 +82,9 @@ protected override void Setup(ServiceCollection services)
     }
 
     [Benchmark]
-    [Arguments(1000000)]
-    public Task ReqRespWithConsumerInterceptor(int messageCount) => RunTest(messageCount);
+    [Arguments(1000000, true)]
+    [Arguments(1000000, false)]
+    public Task ReqRespWithConsumerInterceptor(int messageCount, bool createMessageScope) => RunTest(messageCount, createMessageScope);
 }
 
 [MemoryDiagnoser]
@@ -98,8 +98,9 @@ protected override void Setup(ServiceCollection services)
     }
 
     [Benchmark]
-    [Arguments(1000000)]
-    public Task ReqRespWithRequestHandlerInterceptor(int messageCount) => RunTest(messageCount);
+    [Arguments(1000000, true)]
+    [Arguments(1000000, false)]
+    public Task ReqRespWithRequestHandlerInterceptor(int messageCount, bool createMessageScope) => RunTest(messageCount, createMessageScope);
 }
 
 public record SomeRequest(DateTimeOffset Timestamp, long Id) : IRequest<SomeResponse>;