Resilience enhancements.

MVC app now waits other containers

eShopOnContainers-ServicesAndWebApps.sln

@@ -1,7 +1,7 @@
 
 Microsoft Visual Studio Solution File, Format Version 12.00
 # Visual Studio 15
-VisualStudioVersion = 15.0.26403.3
+VisualStudioVersion = 15.0.26430.12
 MinimumVisualStudioVersion = 10.0.40219.1
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{932D8224-11F6-4D07-B109-DA28AD288A63}"
 EndProject

src/BuildingBlocks/Resilience/Resilience.Http/ResilientHttpClient.cs

@@ -96,7 +96,7 @@ namespace Microsoft.eShopOnContainers.BuildingBlocks.Resilience.Http
             // as it is disposed after each call
             var origin = GetOriginFromUri(uri);
 
-            return HttpInvoker(origin, () =>
+            return HttpInvoker(origin, async () =>
            {
                var requestMessage = new HttpRequestMessage(method, uri);
 
@@ -112,7 +112,7 @@ namespace Microsoft.eShopOnContainers.BuildingBlocks.Resilience.Http
                    requestMessage.Headers.Add("x-requestid", requestId);
                }
 
-               var response = _client.SendAsync(requestMessage).Result;
+               var response = await _client.SendAsync(requestMessage);
 
                // raise exception if HttpResponseCode 500 
                // needed for circuit breaker to track fails
@@ -122,7 +122,7 @@ namespace Microsoft.eShopOnContainers.BuildingBlocks.Resilience.Http
                    throw new HttpRequestException();
                }
 
-               return Task.FromResult(response);
+               return response;
            });
         }
 
@@ -132,13 +132,13 @@ namespace Microsoft.eShopOnContainers.BuildingBlocks.Resilience.Http
 
             if (!_policyWrappers.TryGetValue(normalizedOrigin, out PolicyWrap policyWrap))
             {
-                policyWrap = Policy.Wrap(_policyCreator(normalizedOrigin).ToArray());
+                policyWrap = Policy.WrapAsync(_policyCreator(normalizedOrigin).ToArray());
                 _policyWrappers.TryAdd(normalizedOrigin, policyWrap);
             }
 
             // Executes the action applying all 
             // the policies defined in the wrapper
-            return await policyWrap.Execute(action, new Context(normalizedOrigin));
+            return await policyWrap.ExecuteAsync(action, new Context(normalizedOrigin));
         }
 
 

src/Services/Catalog/Catalog.API/Catalog.API.csproj

@@ -45,6 +45,7 @@
     <PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="1.1.2" />
     <PackageReference Include="Newtonsoft.Json" Version="10.0.2" />
     <PackageReference Include="Swashbuckle.AspNetCore" Version="1.0.0" />
+    <PackageReference Include="Polly" Version="5.1.0" />
   </ItemGroup>
 
   <ItemGroup>

src/Services/Catalog/Catalog.API/Startup.cs

@@ -17,11 +17,13 @@
     using Microsoft.Extensions.HealthChecks;
     using Microsoft.Extensions.Logging;
     using Microsoft.Extensions.Options;
+    using Polly;
     using RabbitMQ.Client;
     using System;
     using System.Data.Common;
     using System.Data.SqlClient;
     using System.Reflection;
+    using System.Threading.Tasks;
 
     public class Startup
     {
@@ -85,7 +87,7 @@
             services.AddSwaggerGen(options =>
             {
                 options.DescribeAllEnumsAsStrings();
-                options.SwaggerDoc("v1",new Swashbuckle.AspNetCore.Swagger.Info
+                options.SwaggerDoc("v1", new Swashbuckle.AspNetCore.Swagger.Info
                 {
                     Title = "eShopOnContainers - Catalog HTTP API",
                     Version = "v1",
@@ -144,11 +146,7 @@
             var context = (CatalogContext)app
                         .ApplicationServices.GetService(typeof(CatalogContext));
 
-            WaitForSqlAvailability(context, loggerFactory);
-
-            //Seed Data
-            CatalogContextSeed.SeedAsync(app, loggerFactory)
-                .Wait();
+            WaitForSqlAvailabilityAsync(context, loggerFactory, app).Wait();
 
             var integrationEventLogContext = new IntegrationEventLogContext(
                 new DbContextOptionsBuilder<IntegrationEventLogContext>()
@@ -158,28 +156,28 @@
             integrationEventLogContext.Database.Migrate();
         }
 
-        private void WaitForSqlAvailability(CatalogContext ctx, ILoggerFactory loggerFactory, int? retry = 0)
+        private async Task WaitForSqlAvailabilityAsync(CatalogContext ctx, ILoggerFactory loggerFactory, IApplicationBuilder app, int retries = 0)
         {
-            int retryForAvailability = retry.Value;
-
-            try
-            {
-                ctx.Database.OpenConnection();
-            }
-            catch (SqlException ex)
-            {
-                if (retryForAvailability < 10)
-                {
-                    retryForAvailability++;
-                    var log = loggerFactory.CreateLogger(nameof(Startup));
-                    log.LogError(ex.Message);
-                    WaitForSqlAvailability(ctx, loggerFactory, retryForAvailability);
-                }
-            }
-            finally
+            var logger = loggerFactory.CreateLogger(nameof(Startup));
+            var policy = CreatePolicy(retries, logger, nameof (WaitForSqlAvailabilityAsync));
+            await policy.ExecuteAsync(async () =>
             {
-                ctx.Database.CloseConnection();
-            }
+                await CatalogContextSeed.SeedAsync(app, loggerFactory);
+            });
+
+        }
+
+        private Policy CreatePolicy(int retries, ILogger logger, string prefix)
+        {
+            return Policy.Handle<SqlException>().
+                WaitAndRetryAsync(
+                    retryCount: retries,
+                    sleepDurationProvider: retry => TimeSpan.FromSeconds(5),
+                    onRetry: (exception, timeSpan, retry, ctx) =>
+                    {
+                        logger.LogTrace($"[{prefix}] Exception {exception.GetType().Name} with message ${exception.Message} detected on attempt {retry} of {retries}");
+                    }
+                );
         }
     }
 }

src/Services/Ordering/Ordering.API/Ordering.API.csproj

@@ -58,6 +58,7 @@
     <PackageReference Include="IdentityServer4.AccessTokenValidation" Version="1.2.0" />
     <PackageReference Include="Dapper" Version="1.50.2" />
     <PackageReference Include="System.ValueTuple" Version="4.3.1" />
+    <PackageReference Include="Polly" Version="5.1.0" />
   </ItemGroup>
 
   <ItemGroup>

src/Services/Ordering/Ordering.API/Startup.cs

@@ -23,10 +23,13 @@
     using Microsoft.Extensions.HealthChecks;
     using Microsoft.Extensions.Logging;
     using Ordering.Infrastructure;
+    using Polly;
     using RabbitMQ.Client;
     using System;
     using System.Data.Common;
+    using System.Data.SqlClient;
     using System.Reflection;
+    using System.Threading.Tasks;
 
     public class Startup
     {
@@ -156,7 +159,7 @@
                    c.SwaggerEndpoint("/swagger/v1/swagger.json", "My API V1");
                });
 
-            OrderingContextSeed.SeedAsync(app).Wait();
+            WaitForSqlAvailabilityAsync(loggerFactory, app).Wait();
 
             var integrationEventLogContext = new IntegrationEventLogContext(
                 new DbContextOptionsBuilder<IntegrationEventLogContext>()
@@ -175,5 +178,30 @@
                 RequireHttpsMetadata = false
             });
         }
+
+
+        private async Task WaitForSqlAvailabilityAsync(ILoggerFactory loggerFactory, IApplicationBuilder app, int retries = 0)
+        {
+            var logger = loggerFactory.CreateLogger(nameof(Startup));
+            var policy = CreatePolicy(retries, logger, nameof(WaitForSqlAvailabilityAsync));
+            await policy.ExecuteAsync(async () =>
+            {
+                await OrderingContextSeed.SeedAsync(app);
+            });
+
+        }
+
+        private Policy CreatePolicy(int retries, ILogger logger, string prefix)
+        {
+            return Policy.Handle<SqlException>().
+                WaitAndRetryAsync(
+                    retryCount: retries,
+                    sleepDurationProvider: retry => TimeSpan.FromSeconds(5),
+                    onRetry: (exception, timeSpan, retry, ctx) =>
+                    {
+                        logger.LogTrace($"[{prefix}] Exception {exception.GetType().Name} with message ${exception.Message} detected on attempt {retry} of {retries}");
+                    }
+                );
+        }
     }
 }

src/Web/WebMVC/Infrastructure/ResilientHttpClientFactory.cs

@@ -20,7 +20,7 @@ namespace Microsoft.eShopOnContainers.WebMVC.Infrastructure
             => new Policy[]
             {
                 Policy.Handle<HttpRequestException>()
-                .WaitAndRetry(
+                .WaitAndRetryAsync(
                     // number of retries
                     6,
                     // exponential backofff
@@ -36,7 +36,7 @@ namespace Microsoft.eShopOnContainers.WebMVC.Infrastructure
                         _logger.LogDebug(msg);
                     }),
                 Policy.Handle<HttpRequestException>()
-                .CircuitBreaker(
+                .CircuitBreakerAsync(
                    // number of exceptions before breaking circuit
                    5,
                    // time circuit opened before retry

test/Services/FunctionalTests/FunctionalTests.csproj

@@ -38,4 +38,8 @@
     </None>
   </ItemGroup>
 
+  <ItemGroup>
+    <Service Include="{82a7f48d-3b50-4b1e-b82e-3ada8210c358}" />
+  </ItemGroup>
+
 </Project>

test/Services/IntegrationTests/IntegrationTests.csproj

@@ -46,4 +46,8 @@
     </Content>
   </ItemGroup>
 
+  <ItemGroup>
+    <Service Include="{82a7f48d-3b50-4b1e-b82e-3ada8210c358}" />
+  </ItemGroup>
+
 </Project>

test/Services/UnitTest/UnitTest.csproj

@@ -28,4 +28,8 @@
     <PackageReference Include="Microsoft.AspNetCore.TestHost" Version="1.1.2" />
   </ItemGroup>
 
+  <ItemGroup>
+    <Service Include="{82a7f48d-3b50-4b1e-b82e-3ada8210c358}" />
+  </ItemGroup>
+
 </Project>