diff --git a/src/BuildingBlocks/EventBus/EventBusRabbitMQ/DefaultRabbitMQPersisterConnection.cs b/src/BuildingBlocks/EventBus/EventBusRabbitMQ/DefaultRabbitMQPersisterConnection.cs index 0aafaf90a..2e0555e61 100644 --- a/src/BuildingBlocks/EventBus/EventBusRabbitMQ/DefaultRabbitMQPersisterConnection.cs +++ b/src/BuildingBlocks/EventBus/EventBusRabbitMQ/DefaultRabbitMQPersisterConnection.cs @@ -15,16 +15,17 @@ namespace Microsoft.eShopOnContainers.BuildingBlocks.EventBusRabbitMQ { private readonly IConnectionFactory _connectionFactory; private readonly ILogger _logger; - + private readonly int _retryCount; IConnection _connection; bool _disposed; object sync_root = new object(); - public DefaultRabbitMQPersistentConnection(IConnectionFactory connectionFactory,ILogger logger) + public DefaultRabbitMQPersistentConnection(IConnectionFactory connectionFactory, ILogger logger, int retryCount = 5) { _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _retryCount = retryCount; } public bool IsConnected @@ -69,7 +70,7 @@ namespace Microsoft.eShopOnContainers.BuildingBlocks.EventBusRabbitMQ { var policy = RetryPolicy.Handle() .Or() - .WaitAndRetry(5, retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt)), (ex, time) => + .WaitAndRetry(_retryCount, retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt)), (ex, time) => { _logger.LogWarning(ex.ToString()); } @@ -88,7 +89,7 @@ namespace Microsoft.eShopOnContainers.BuildingBlocks.EventBusRabbitMQ _connection.ConnectionBlocked += OnConnectionBlocked; _logger.LogInformation($"RabbitMQ persistent connection acquired a connection {_connection.Endpoint.HostName} and is subscribed to failure events"); - + return true; } else diff --git a/src/BuildingBlocks/EventBus/EventBusRabbitMQ/EventBusRabbitMQ.cs b/src/BuildingBlocks/EventBus/EventBusRabbitMQ/EventBusRabbitMQ.cs index f438623a6..2416e1d55 100644 --- a/src/BuildingBlocks/EventBus/EventBusRabbitMQ/EventBusRabbitMQ.cs +++ b/src/BuildingBlocks/EventBus/EventBusRabbitMQ/EventBusRabbitMQ.cs @@ -26,19 +26,20 @@ namespace Microsoft.eShopOnContainers.BuildingBlocks.EventBusRabbitMQ private readonly IEventBusSubscriptionsManager _subsManager; private readonly ILifetimeScope _autofac; private readonly string AUTOFAC_SCOPE_NAME = "eshop_event_bus"; + private readonly int _retryCount; private IModel _consumerChannel; private string _queueName; public EventBusRabbitMQ(IRabbitMQPersistentConnection persistentConnection, ILogger logger, - ILifetimeScope autofac, IEventBusSubscriptionsManager subsManager) + ILifetimeScope autofac, IEventBusSubscriptionsManager subsManager, int retryCount = 5) { _persistentConnection = persistentConnection ?? throw new ArgumentNullException(nameof(persistentConnection)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); _subsManager = subsManager ?? new InMemoryEventBusSubscriptionsManager(); _consumerChannel = CreateConsumerChannel(); _autofac = autofac; - + _retryCount = retryCount; _subsManager.OnEventRemoved += SubsManager_OnEventRemoved; } @@ -72,7 +73,7 @@ namespace Microsoft.eShopOnContainers.BuildingBlocks.EventBusRabbitMQ var policy = RetryPolicy.Handle() .Or() - .WaitAndRetry(5, retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt)), (ex, time) => + .WaitAndRetry(_retryCount, retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt)), (ex, time) => { _logger.LogWarning(ex.ToString()); }); diff --git a/src/Services/Basket/Basket.API/Startup.cs b/src/Services/Basket/Basket.API/Startup.cs index ae65a8486..371979419 100644 --- a/src/Services/Basket/Basket.API/Startup.cs +++ b/src/Services/Basket/Basket.API/Startup.cs @@ -120,7 +120,13 @@ namespace Microsoft.eShopOnContainers.Services.Basket.API factory.Password = Configuration["EventBusPassword"]; } - return new DefaultRabbitMQPersistentConnection(factory, logger); + var retryCount = 5; + if (!string.IsNullOrEmpty(Configuration["EventBusRetryCount"])) + { + retryCount = int.Parse(Configuration["EventBusRetryCount"]); + } + + return new DefaultRabbitMQPersistentConnection(factory, logger, retryCount); }); } @@ -247,7 +253,21 @@ namespace Microsoft.eShopOnContainers.Services.Basket.API } else { - services.AddSingleton(); + services.AddSingleton(sp => + { + var rabbitMQPersistentConnection = sp.GetRequiredService(); + var iLifetimeScope = sp.GetRequiredService(); + var logger = sp.GetRequiredService>(); + var eventBusSubcriptionsManager = sp.GetRequiredService(); + + var retryCount = 5; + if (!string.IsNullOrEmpty(Configuration["EventBusRetryCount"])) + { + retryCount = int.Parse(Configuration["EventBusRetryCount"]); + } + + return new EventBusRabbitMQ(rabbitMQPersistentConnection, logger, iLifetimeScope, eventBusSubcriptionsManager, retryCount); + }); } services.AddSingleton(); diff --git a/src/Services/Basket/Basket.API/appsettings.json b/src/Services/Basket/Basket.API/appsettings.json index 027fe9d23..16d56d94d 100644 --- a/src/Services/Basket/Basket.API/appsettings.json +++ b/src/Services/Basket/Basket.API/appsettings.json @@ -13,5 +13,6 @@ "SubscriptionClientName": "Basket", "ApplicationInsights": { "InstrumentationKey": "" - } + }, + "EventBusRetryCount": 5 } \ No newline at end of file diff --git a/src/Services/Catalog/Catalog.API/Startup.cs b/src/Services/Catalog/Catalog.API/Startup.cs index c04039a73..6dd4f89c5 100644 --- a/src/Services/Catalog/Catalog.API/Startup.cs +++ b/src/Services/Catalog/Catalog.API/Startup.cs @@ -35,7 +35,7 @@ Configuration = configuration; } - public IConfiguration Configuration { get; } + public IConfiguration Configuration { get; } public IServiceProvider ConfigureServices(IServiceCollection services) { @@ -161,7 +161,13 @@ factory.Password = Configuration["EventBusPassword"]; } - return new DefaultRabbitMQPersistentConnection(factory, logger); + var retryCount = 5; + if (!string.IsNullOrEmpty(Configuration["EventBusRetryCount"])) + { + retryCount = int.Parse(Configuration["EventBusRetryCount"]); + } + + return new DefaultRabbitMQPersistentConnection(factory, logger, retryCount); }); } @@ -188,7 +194,7 @@ loggerFactory.CreateLogger("init").LogDebug($"Using PATH BASE '{pathBase}'"); app.UsePathBase(pathBase); } - + app.UseCors("CorsPolicy"); app.UseMvcWithDefaultRoute(); @@ -221,7 +227,21 @@ } else { - services.AddSingleton(); + services.AddSingleton(sp => + { + var rabbitMQPersistentConnection = sp.GetRequiredService(); + var iLifetimeScope = sp.GetRequiredService(); + var logger = sp.GetRequiredService>(); + var eventBusSubcriptionsManager = sp.GetRequiredService(); + + var retryCount = 5; + if (!string.IsNullOrEmpty(Configuration["EventBusRetryCount"])) + { + retryCount = int.Parse(Configuration["EventBusRetryCount"]); + } + + return new EventBusRabbitMQ(rabbitMQPersistentConnection, logger, iLifetimeScope, eventBusSubcriptionsManager, retryCount); + }); } services.AddSingleton(); diff --git a/src/Services/Catalog/Catalog.API/appsettings.json b/src/Services/Catalog/Catalog.API/appsettings.json index dc4e0aa5c..abaa34316 100644 --- a/src/Services/Catalog/Catalog.API/appsettings.json +++ b/src/Services/Catalog/Catalog.API/appsettings.json @@ -15,5 +15,6 @@ "SubscriptionClientName": "Catalog", "ApplicationInsights": { "InstrumentationKey": "" - } + }, + "EventBusRetryCount": 5 } diff --git a/src/Services/Location/Locations.API/Startup.cs b/src/Services/Location/Locations.API/Startup.cs index 115fa5f32..db878a155 100644 --- a/src/Services/Location/Locations.API/Startup.cs +++ b/src/Services/Location/Locations.API/Startup.cs @@ -87,7 +87,13 @@ namespace Microsoft.eShopOnContainers.Services.Locations.API factory.Password = Configuration["EventBusPassword"]; } - return new DefaultRabbitMQPersistentConnection(factory, logger); + var retryCount = 5; + if (!string.IsNullOrEmpty(Configuration["EventBusRetryCount"])) + { + retryCount = int.Parse(Configuration["EventBusRetryCount"]); + } + + return new DefaultRabbitMQPersistentConnection(factory, logger, retryCount); }); } @@ -217,7 +223,21 @@ namespace Microsoft.eShopOnContainers.Services.Locations.API } else { - services.AddSingleton(); + services.AddSingleton(sp => + { + var rabbitMQPersistentConnection = sp.GetRequiredService(); + var iLifetimeScope = sp.GetRequiredService(); + var logger = sp.GetRequiredService>(); + var eventBusSubcriptionsManager = sp.GetRequiredService(); + + var retryCount = 5; + if (!string.IsNullOrEmpty(Configuration["EventBusRetryCount"])) + { + retryCount = int.Parse(Configuration["EventBusRetryCount"]); + } + + return new EventBusRabbitMQ(rabbitMQPersistentConnection, logger, iLifetimeScope, eventBusSubcriptionsManager, retryCount); + }); } services.AddSingleton(); diff --git a/src/Services/Location/Locations.API/appsettings.Development.json b/src/Services/Location/Locations.API/appsettings.Development.json deleted file mode 100644 index fa8ce71a9..000000000 --- a/src/Services/Location/Locations.API/appsettings.Development.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "Logging": { - "IncludeScopes": false, - "LogLevel": { - "Default": "Debug", - "System": "Information", - "Microsoft": "Information" - } - } -} diff --git a/src/Services/Location/Locations.API/appsettings.json b/src/Services/Location/Locations.API/appsettings.json index 0064466af..512b55f2a 100644 --- a/src/Services/Location/Locations.API/appsettings.json +++ b/src/Services/Location/Locations.API/appsettings.json @@ -14,5 +14,6 @@ "SubscriptionClientName": "Locations", "ApplicationInsights": { "InstrumentationKey": "" - } + }, + "EventBusRetryCount": 5 } \ No newline at end of file diff --git a/src/Services/Marketing/Marketing.API/Startup.cs b/src/Services/Marketing/Marketing.API/Startup.cs index 45f4974c1..76d43d2e3 100644 --- a/src/Services/Marketing/Marketing.API/Startup.cs +++ b/src/Services/Marketing/Marketing.API/Startup.cs @@ -122,7 +122,14 @@ { factory.Password = Configuration["EventBusPassword"]; } - return new DefaultRabbitMQPersistentConnection(factory, logger); + + var retryCount = 5; + if (!string.IsNullOrEmpty(Configuration["EventBusRetryCount"])) + { + retryCount = int.Parse(Configuration["EventBusRetryCount"]); + } + + return new DefaultRabbitMQPersistentConnection(factory, logger, retryCount); }); } @@ -242,7 +249,21 @@ } else { - services.AddSingleton(); + services.AddSingleton(sp => + { + var rabbitMQPersistentConnection = sp.GetRequiredService(); + var iLifetimeScope = sp.GetRequiredService(); + var logger = sp.GetRequiredService>(); + var eventBusSubcriptionsManager = sp.GetRequiredService(); + + var retryCount = 5; + if (!string.IsNullOrEmpty(Configuration["EventBusRetryCount"])) + { + retryCount = int.Parse(Configuration["EventBusRetryCount"]); + } + + return new EventBusRabbitMQ(rabbitMQPersistentConnection, logger, iLifetimeScope, eventBusSubcriptionsManager, retryCount); + }); } services.AddSingleton(); diff --git a/src/Services/Marketing/Marketing.API/appsettings.Development.json b/src/Services/Marketing/Marketing.API/appsettings.Development.json deleted file mode 100644 index 5fff67bac..000000000 --- a/src/Services/Marketing/Marketing.API/appsettings.Development.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "Logging": { - "IncludeScopes": false, - "LogLevel": { - "Default": "Warning" - } - } -} diff --git a/src/Services/Marketing/Marketing.API/appsettings.json b/src/Services/Marketing/Marketing.API/appsettings.json index cc55a4052..c4d908c83 100644 --- a/src/Services/Marketing/Marketing.API/appsettings.json +++ b/src/Services/Marketing/Marketing.API/appsettings.json @@ -15,5 +15,6 @@ "AzureStorageEnabled": false, "ApplicationInsights": { "InstrumentationKey": "" - } + }, + "EventBusRetryCount": 5 } \ No newline at end of file diff --git a/src/Services/Ordering/Ordering.API/Startup.cs b/src/Services/Ordering/Ordering.API/Startup.cs index 79de89394..b1dea71b2 100644 --- a/src/Services/Ordering/Ordering.API/Startup.cs +++ b/src/Services/Ordering/Ordering.API/Startup.cs @@ -180,7 +180,13 @@ factory.Password = Configuration["EventBusPassword"]; } - return new DefaultRabbitMQPersistentConnection(factory, logger); + var retryCount = 5; + if (!string.IsNullOrEmpty(Configuration["EventBusRetryCount"])) + { + retryCount = int.Parse(Configuration["EventBusRetryCount"]); + } + + return new DefaultRabbitMQPersistentConnection(factory, logger, retryCount); }); } @@ -284,7 +290,21 @@ } else { - services.AddSingleton(); + services.AddSingleton(sp => + { + var rabbitMQPersistentConnection = sp.GetRequiredService(); + var iLifetimeScope = sp.GetRequiredService(); + var logger = sp.GetRequiredService>(); + var eventBusSubcriptionsManager = sp.GetRequiredService(); + + var retryCount = 5; + if (!string.IsNullOrEmpty(Configuration["EventBusRetryCount"])) + { + retryCount = int.Parse(Configuration["EventBusRetryCount"]); + } + + return new EventBusRabbitMQ(rabbitMQPersistentConnection, logger, iLifetimeScope, eventBusSubcriptionsManager, retryCount); + }); } services.AddSingleton(); diff --git a/src/Services/Ordering/Ordering.API/settings.json b/src/Services/Ordering/Ordering.API/settings.json index f20261e31..61ebe5325 100644 --- a/src/Services/Ordering/Ordering.API/settings.json +++ b/src/Services/Ordering/Ordering.API/settings.json @@ -16,5 +16,6 @@ "CheckUpdateTime": "30000", "ApplicationInsights": { "InstrumentationKey": "" - } + }, + "EventBusRetryCount": 5 } diff --git a/src/Services/Payment/Payment.API/Startup.cs b/src/Services/Payment/Payment.API/Startup.cs index e20b2fd76..c0ae5b2d5 100644 --- a/src/Services/Payment/Payment.API/Startup.cs +++ b/src/Services/Payment/Payment.API/Startup.cs @@ -74,7 +74,13 @@ namespace Payment.API factory.Password = Configuration["EventBusPassword"]; } - return new DefaultRabbitMQPersistentConnection(factory, logger); + var retryCount = 5; + if (!string.IsNullOrEmpty(Configuration["EventBusRetryCount"])) + { + retryCount = int.Parse(Configuration["EventBusRetryCount"]); + } + + return new DefaultRabbitMQPersistentConnection(factory, logger, retryCount); }); } @@ -123,7 +129,21 @@ namespace Payment.API } else { - services.AddSingleton(); + services.AddSingleton(sp => + { + var rabbitMQPersistentConnection = sp.GetRequiredService(); + var iLifetimeScope = sp.GetRequiredService(); + var logger = sp.GetRequiredService>(); + var eventBusSubcriptionsManager = sp.GetRequiredService(); + + var retryCount = 5; + if (!string.IsNullOrEmpty(Configuration["EventBusRetryCount"])) + { + retryCount = int.Parse(Configuration["EventBusRetryCount"]); + } + + return new EventBusRabbitMQ(rabbitMQPersistentConnection, logger, iLifetimeScope, eventBusSubcriptionsManager, retryCount); + }); } services.AddTransient(); diff --git a/src/Services/Payment/Payment.API/appsettings.json b/src/Services/Payment/Payment.API/appsettings.json index ad668a388..acc9a2be0 100644 --- a/src/Services/Payment/Payment.API/appsettings.json +++ b/src/Services/Payment/Payment.API/appsettings.json @@ -10,5 +10,6 @@ "SubscriptionClientName": "Payment", "ApplicationInsights": { "InstrumentationKey": "" - } + }, + "EventBusRetryCount": 5 } diff --git a/src/Web/WebMVC/Infrastructure/ResilientHttpClientFactory.cs b/src/Web/WebMVC/Infrastructure/ResilientHttpClientFactory.cs index b05c6e038..8805bc752 100644 --- a/src/Web/WebMVC/Infrastructure/ResilientHttpClientFactory.cs +++ b/src/Web/WebMVC/Infrastructure/ResilientHttpClientFactory.cs @@ -9,11 +9,18 @@ namespace Microsoft.eShopOnContainers.WebMVC.Infrastructure public class ResilientHttpClientFactory : IResilientHttpClientFactory { private readonly ILogger _logger; + private readonly int _retryCount; + private readonly int _exceptionsAllowedBeforeBreaking; - public ResilientHttpClientFactory(ILogger logger) - =>_logger = logger; + public ResilientHttpClientFactory(ILogger logger, int exceptionsAllowedBeforeBreaking = 5, int retryCount = 6) + { + _logger = logger; + _exceptionsAllowedBeforeBreaking = exceptionsAllowedBeforeBreaking; + _retryCount = retryCount; + } - public ResilientHttpClient CreateResilientHttpClient() + + public ResilientHttpClient CreateResilientHttpClient() => new ResilientHttpClient((origin) => CreatePolicies(), _logger); private Policy[] CreatePolicies() @@ -22,7 +29,7 @@ namespace Microsoft.eShopOnContainers.WebMVC.Infrastructure Policy.Handle() .WaitAndRetryAsync( // number of retries - 6, + _retryCount, // exponential backofff retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt)), // on retry @@ -36,9 +43,9 @@ namespace Microsoft.eShopOnContainers.WebMVC.Infrastructure _logger.LogDebug(msg); }), Policy.Handle() - .CircuitBreakerAsync( + .CircuitBreakerAsync( // number of exceptions before breaking circuit - 5, + _exceptionsAllowedBeforeBreaking, // time circuit opened before retry TimeSpan.FromMinutes(1), (exception, duration) => @@ -51,6 +58,6 @@ namespace Microsoft.eShopOnContainers.WebMVC.Infrastructure // on circuit closed _logger.LogTrace("Circuit breaker reset"); }) - }; + }; } } diff --git a/src/Web/WebMVC/Startup.cs b/src/Web/WebMVC/Startup.cs index 509a55947..6049aa229 100644 --- a/src/Web/WebMVC/Startup.cs +++ b/src/Web/WebMVC/Startup.cs @@ -79,7 +79,24 @@ namespace Microsoft.eShopOnContainers.WebMVC if (Configuration.GetValue("UseResilientHttp") == bool.TrueString) { - services.AddSingleton(); + services.AddSingleton(sp => + { + var logger = sp.GetRequiredService>(); + + var retryCount = 6; + if (!string.IsNullOrEmpty(Configuration["HttpClientRetryCount"])) + { + retryCount = int.Parse(Configuration["HttpClientRetryCount"]); + } + + var exceptionsAllowedBeforeBreaking = 5; + if (!string.IsNullOrEmpty(Configuration["HttpClientExceptionsAllowedBeforeBreaking"])) + { + exceptionsAllowedBeforeBreaking = int.Parse(Configuration["HttpClientExceptionsAllowedBeforeBreaking"]); + } + + return new ResilientHttpClientFactory(logger, exceptionsAllowedBeforeBreaking, retryCount); + }); services.AddSingleton(sp => sp.GetService().CreateResilientHttpClient()); } else diff --git a/src/Web/WebMVC/appsettings.json b/src/Web/WebMVC/appsettings.json index b68e719dd..d728eeb6f 100644 --- a/src/Web/WebMVC/appsettings.json +++ b/src/Web/WebMVC/appsettings.json @@ -21,5 +21,7 @@ }, "ApplicationInsights": { "InstrumentationKey": "" - } + }, + "HttpClientRetryCount": 6, + "HttpClientExceptionsAllowedBeforeBreaking": 5 } \ No newline at end of file