From aedd0e38e9fc9ffdde6feadb6e7752d4105ecb32 Mon Sep 17 00:00:00 2001 From: ericuss Date: Mon, 5 Aug 2019 10:20:20 +0200 Subject: [PATCH 1/2] partial checkin --- .../WebHostExtensions.cs | 9 ++- .../Infrastructure/MarketingContext.cs | 15 +++- .../Marketing/Marketing.API/Startup.cs | 78 +++++++++++-------- .../Marketing.FunctionalTests.csproj | 3 +- .../MarketingTestStartup.cs | 13 +++- .../appsettings.json | 3 +- 6 files changed, 81 insertions(+), 40 deletions(-) diff --git a/src/BuildingBlocks/WebHostCustomization/WebHost.Customization/WebHostExtensions.cs b/src/BuildingBlocks/WebHostCustomization/WebHost.Customization/WebHostExtensions.cs index 17f030e02..e6e1e22a1 100644 --- a/src/BuildingBlocks/WebHostCustomization/WebHost.Customization/WebHostExtensions.cs +++ b/src/BuildingBlocks/WebHostCustomization/WebHost.Customization/WebHostExtensions.cs @@ -39,8 +39,15 @@ namespace Microsoft.AspNetCore.Hosting } else { + var retries = 10; var retry = Policy.Handle() - .WaitAndRetry(10, retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt))); + .WaitAndRetry( + retryCount: retries, + sleepDurationProvider: retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt)), + onRetry: (exception, timeSpan, retry, ctx) => + { + logger.LogWarning(exception, "[{prefix}] Exception {ExceptionType} with message {Message} detected on attempt {retry} of {retries}", nameof(TContext), exception.GetType().Name, exception.Message, retry, retries); + }); //if the sql server container is not created on run docker compose this //migration can't fail for network related exception. The retry options for DbContext only diff --git a/src/Services/Marketing/Marketing.API/Infrastructure/MarketingContext.cs b/src/Services/Marketing/Marketing.API/Infrastructure/MarketingContext.cs index dae9b5bb4..c4a803212 100644 --- a/src/Services/Marketing/Marketing.API/Infrastructure/MarketingContext.cs +++ b/src/Services/Marketing/Marketing.API/Infrastructure/MarketingContext.cs @@ -1,9 +1,12 @@ namespace Microsoft.eShopOnContainers.Services.Marketing.API.Infrastructure { + using System; + using System.IO; using EntityConfigurations; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Design; using Microsoft.eShopOnContainers.Services.Marketing.API.Model; + using Microsoft.Extensions.Configuration; public class MarketingContext : DbContext { @@ -27,9 +30,17 @@ { public MarketingContext CreateDbContext(string[] args) { + IConfigurationRoot configuration = new ConfigurationBuilder() + .SetBasePath(Directory.GetCurrentDirectory()) + .AddJsonFile("appsettings.json", optional: false, reloadOnChange: true) + .AddEnvironmentVariables() + .Build(); + var connectionString = configuration["ConnectionString"]; + Console.WriteLine(" -- Connection string"); + Console.WriteLine(connectionString); var optionsBuilder = new DbContextOptionsBuilder() - .UseSqlServer("Server=.;Initial Catalog=Microsoft.eShopOnContainers.Services.MarketingDb;Integrated Security=true"); - + .UseSqlServer(connectionString); + // .UseSqlServer("Server=.;Initial Catalog=Microsoft.eShopOnContainers.Services.MarketingDb;Integrated Security=true"); return new MarketingContext(optionsBuilder.Options); } } diff --git a/src/Services/Marketing/Marketing.API/Startup.cs b/src/Services/Marketing/Marketing.API/Startup.cs index c3ae02e23..ff6fdd43a 100644 --- a/src/Services/Marketing/Marketing.API/Startup.cs +++ b/src/Services/Marketing/Marketing.API/Startup.cs @@ -23,7 +23,9 @@ using Marketing.API.IntegrationEvents.Handlers; using Microsoft.AspNetCore.Authentication.JwtBearer; using Microsoft.AspNetCore.Diagnostics.HealthChecks; + using Microsoft.AspNetCore.Mvc; using Microsoft.EntityFrameworkCore.Diagnostics; + using Microsoft.eShopOnContainers.Services.Marketing.API.Controllers; using Microsoft.eShopOnContainers.Services.Marketing.API.Infrastructure.Middlewares; using Microsoft.Extensions.Diagnostics.HealthChecks; using Microsoft.OpenApi.Models; @@ -44,13 +46,18 @@ // This method gets called by the runtime. Use this method to add services to the container. - public IServiceProvider ConfigureServices(IServiceCollection services) + public virtual IServiceProvider ConfigureServices(IServiceCollection services) { RegisterAppInsights(services); // Add framework services. services.AddCustomHealthCheck(Configuration); - services.AddControllers().AddNewtonsoftJson(); + services + .AddControllers() + // Added for functional tests + .AddApplicationPart(typeof(LocationsController).Assembly) + .AddNewtonsoftJson() + .SetCompatibilityVersion(CompatibilityVersion.Version_3_0); services.Configure(Configuration); ConfigureAuthService(services); @@ -117,36 +124,8 @@ } // Add framework services. - services.AddSwaggerGen(options => - { - options.DescribeAllEnumsAsStrings(); - options.SwaggerDoc("v1", new OpenApiInfo - { - Title = "eShopOnContainers - Marketing HTTP API", - Version = "v1", - Description = "The Marketing Service HTTP API" - }); - - options.AddSecurityDefinition("oauth2", new OpenApiSecurityScheme - { - Type = SecuritySchemeType.OAuth2, - Flows = new OpenApiOAuthFlows() - { - Implicit = new OpenApiOAuthFlow() - { - AuthorizationUrl = new Uri($"{Configuration.GetValue("IdentityUrlExternal")}/connect/authorize"), - TokenUrl = new Uri($"{Configuration.GetValue("IdentityUrlExternal")}/connect/token"), - Scopes = new Dictionary() - { - { "marketing", "Marketing API" } - } - } - } - }); - - options.OperationFilter(); - }); + AddCustomSwagger(services); services.AddCors(options => { options.AddPolicy("CorsPolicy", @@ -186,10 +165,10 @@ } app.UseCors("CorsPolicy"); + app.UseRouting(); ConfigureAuth(app); - app.UseRouting(); app.UseEndpoints(endpoints => { endpoints.MapDefaultControllerRoute(); @@ -216,6 +195,39 @@ ConfigureEventBus(app); } + private void AddCustomSwagger(IServiceCollection services) + { + services.AddSwaggerGen(options => + { + options.DescribeAllEnumsAsStrings(); + options.SwaggerDoc("v1", new OpenApiInfo + { + Title = "eShopOnContainers - Marketing HTTP API", + Version = "v1", + Description = "The Marketing Service HTTP API" + }); + + options.AddSecurityDefinition("oauth2", new OpenApiSecurityScheme + { + Type = SecuritySchemeType.OAuth2, + Flows = new OpenApiOAuthFlows() + { + Implicit = new OpenApiOAuthFlow() + { + AuthorizationUrl = new Uri($"{Configuration.GetValue("IdentityUrlExternal")}/connect/authorize"), + TokenUrl = new Uri($"{Configuration.GetValue("IdentityUrlExternal")}/connect/token"), + Scopes = new Dictionary() + { + { "marketing", "Marketing API" } + } + } + } + }); + + options.OperationFilter(); + }); + } + private void RegisterAppInsights(IServiceCollection services) { services.AddApplicationInsightsTelemetry(Configuration); @@ -225,7 +237,7 @@ private void ConfigureAuthService(IServiceCollection services) { // prevent from mapping "sub" claim to nameidentifier. - JwtSecurityTokenHandler.DefaultInboundClaimTypeMap.Clear(); + JwtSecurityTokenHandler.DefaultInboundClaimTypeMap.Remove("sub"); services.AddAuthentication(options => { diff --git a/src/Services/Marketing/Marketing.FunctionalTests/Marketing.FunctionalTests.csproj b/src/Services/Marketing/Marketing.FunctionalTests/Marketing.FunctionalTests.csproj index eddf06086..41deec5d4 100644 --- a/src/Services/Marketing/Marketing.FunctionalTests/Marketing.FunctionalTests.csproj +++ b/src/Services/Marketing/Marketing.FunctionalTests/Marketing.FunctionalTests.csproj @@ -17,8 +17,7 @@ - - + diff --git a/src/Services/Marketing/Marketing.FunctionalTests/MarketingTestStartup.cs b/src/Services/Marketing/Marketing.FunctionalTests/MarketingTestStartup.cs index e227deff6..87d0fb991 100644 --- a/src/Services/Marketing/Marketing.FunctionalTests/MarketingTestStartup.cs +++ b/src/Services/Marketing/Marketing.FunctionalTests/MarketingTestStartup.cs @@ -1,6 +1,9 @@ -using Microsoft.AspNetCore.Builder; +using System; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Routing; using Microsoft.eShopOnContainers.Services.Marketing.API; using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; namespace Marketing.FunctionalTests { @@ -10,6 +13,14 @@ namespace Marketing.FunctionalTests { } + public override IServiceProvider ConfigureServices(IServiceCollection services) + { + // Added to avoid the Authorize data annotation in test environment. + // Property "SuppressCheckForUnhandledSecurityMetadata" in appsettings.json + services.Configure(Configuration); + return base.ConfigureServices(services); + } + protected override void ConfigureAuth(IApplicationBuilder app) { if (Configuration["isTest"] == bool.TrueString.ToLowerInvariant()) diff --git a/src/Services/Marketing/Marketing.FunctionalTests/appsettings.json b/src/Services/Marketing/Marketing.FunctionalTests/appsettings.json index 4a57bb0f5..11a1e090d 100644 --- a/src/Services/Marketing/Marketing.FunctionalTests/appsettings.json +++ b/src/Services/Marketing/Marketing.FunctionalTests/appsettings.json @@ -6,5 +6,6 @@ "isTest": "true", "EventBusConnection": "localhost", "PicBaseUrl": "http://localhost:5110/api/v1/campaigns/[0]/pic/", - "SubscriptionClientName": "Marketing" + "SubscriptionClientName": "Marketing", + "SuppressCheckForUnhandledSecurityMetadata":true } From 44af7aa8d267c9c59d87c278c8a518c44a50d9f3 Mon Sep 17 00:00:00 2001 From: ericuss Date: Mon, 5 Aug 2019 14:21:58 +0200 Subject: [PATCH 2/2] Partial checkin --- .../WebHostExtensions.cs | 62 +++++++++++++++++++ .../Infrastructure/MarketingContext.cs | 1 + .../CampaignScenarios.cs | 4 +- .../MarketingScenarioBase.cs | 34 +++++++++- 4 files changed, 97 insertions(+), 4 deletions(-) diff --git a/src/BuildingBlocks/WebHostCustomization/WebHost.Customization/WebHostExtensions.cs b/src/BuildingBlocks/WebHostCustomization/WebHost.Customization/WebHostExtensions.cs index e6e1e22a1..2b7f661da 100644 --- a/src/BuildingBlocks/WebHostCustomization/WebHost.Customization/WebHostExtensions.cs +++ b/src/BuildingBlocks/WebHostCustomization/WebHost.Customization/WebHostExtensions.cs @@ -60,6 +60,7 @@ namespace Microsoft.AspNetCore.Hosting } catch (Exception ex) { + Console.WriteLine(ex.Message.ToString() + "An error occurred while migrating the database used on context {DbContextName}" + typeof(TContext).Name); logger.LogError(ex, "An error occurred while migrating the database used on context {DbContextName}", typeof(TContext).Name); if (underK8s) { @@ -71,11 +72,72 @@ namespace Microsoft.AspNetCore.Hosting return webHost; } + + public static IWebHost RemoveDbContext(this IWebHost webHost) where TContext : DbContext + { + var underK8s = webHost.IsInKubernetes(); + + using (var scope = webHost.Services.CreateScope()) + { + var services = scope.ServiceProvider; + var logger = services.GetRequiredService>(); + var context = services.GetService(); + + try + { + logger.LogInformation("Deleting the database associated with context {DbContextName}" + typeof(TContext).Name); + + if (underK8s) + { + InvokeRemover(context); + } + else + { + var retries = 10; + var retry = Policy.Handle() + .WaitAndRetry( + retryCount: retries, + sleepDurationProvider: retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt)), + onRetry: (exception, timeSpan, retry, ctx) => + { + Console.WriteLine(" --RETRYING Exception " + exception.Message.ToString()); + logger.LogWarning(exception, "[{prefix}] Exception {ExceptionType} with message {Message} detected on attempt {retry} of {retries}", nameof(TContext), exception.GetType().Name, exception.Message, retry, retries); + }); + + //if the sql server container is not created on run docker compose this + //migration can't fail for network related exception. The retry options for DbContext only + //apply to transient exceptions + // Note that this is NOT applied when running some orchestrators (let the orchestrator to recreate the failing service) + retry.Execute(() => InvokeRemover(context)); + } + + Console.WriteLine("Deleted database associated with context {DbContextName}", typeof(TContext).Name); + logger.LogInformation("Deleted database associated with context {DbContextName}", typeof(TContext).Name); + } + catch (Exception ex) + { + Console.WriteLine(ex.Message.ToString() + "An error occurred while deleting the database used on context {DbContextName}" + typeof(TContext).Name); + logger.LogError(ex, "An error occurred while deleting the database used on context {DbContextName}", typeof(TContext).Name); + if (underK8s) + { + throw; // Rethrow under k8s because we rely on k8s to re-run the pod + } + } + } + + return webHost; + } + private static void InvokeSeeder(Action seeder, TContext context, IServiceProvider services) where TContext : DbContext { context.Database.Migrate(); seeder(context, services); } + private static void InvokeRemover(TContext context) + where TContext : DbContext + { + context.Database.EnsureDeleted(); + } } } diff --git a/src/Services/Marketing/Marketing.API/Infrastructure/MarketingContext.cs b/src/Services/Marketing/Marketing.API/Infrastructure/MarketingContext.cs index c4a803212..064a1c3e5 100644 --- a/src/Services/Marketing/Marketing.API/Infrastructure/MarketingContext.cs +++ b/src/Services/Marketing/Marketing.API/Infrastructure/MarketingContext.cs @@ -38,6 +38,7 @@ var connectionString = configuration["ConnectionString"]; Console.WriteLine(" -- Connection string"); Console.WriteLine(connectionString); + var optionsBuilder = new DbContextOptionsBuilder() .UseSqlServer(connectionString); // .UseSqlServer("Server=.;Initial Catalog=Microsoft.eShopOnContainers.Services.MarketingDb;Integrated Security=true"); diff --git a/src/Services/Marketing/Marketing.FunctionalTests/CampaignScenarios.cs b/src/Services/Marketing/Marketing.FunctionalTests/CampaignScenarios.cs index abeb4a504..f2643e998 100644 --- a/src/Services/Marketing/Marketing.FunctionalTests/CampaignScenarios.cs +++ b/src/Services/Marketing/Marketing.FunctionalTests/CampaignScenarios.cs @@ -75,7 +75,7 @@ namespace Marketing.FunctionalTests var campaignResponse = await server.CreateClient() .PostAsync(Post.AddNewCampaign, content); - if (int.TryParse(campaignResponse.Headers.Location.Segments[4], out int id)) + if (int.TryParse(campaignResponse.Headers.Location.Segments[3], out int id)) { var response = await server.CreateClient() .DeleteAsync(Delete.CampaignBy(id)); @@ -99,7 +99,7 @@ namespace Marketing.FunctionalTests var campaignResponse = await server.CreateClient() .PostAsync(Post.AddNewCampaign, content); - if (int.TryParse(campaignResponse.Headers.Location.Segments[4], out int id)) + if (int.TryParse(campaignResponse.Headers.Location.Segments[3], out int id)) { fakeCampaignDto.Description = "FakeCampaignUpdatedDescription"; content = new StringContent(JsonConvert.SerializeObject(fakeCampaignDto), Encoding.UTF8, "application/json"); diff --git a/src/Services/Marketing/Marketing.FunctionalTests/MarketingScenarioBase.cs b/src/Services/Marketing/Marketing.FunctionalTests/MarketingScenarioBase.cs index 274db5f69..fdee81ce3 100644 --- a/src/Services/Marketing/Marketing.FunctionalTests/MarketingScenarioBase.cs +++ b/src/Services/Marketing/Marketing.FunctionalTests/MarketingScenarioBase.cs @@ -1,11 +1,16 @@ using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.TestHost; +using Microsoft.eShopOnContainers.Services.Marketing.API; using Microsoft.eShopOnContainers.Services.Marketing.API.Infrastructure; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using System; using System.IO; using System.Reflection; +using System.Threading; +using System.Threading.Tasks; namespace Marketing.FunctionalTests { @@ -15,29 +20,54 @@ namespace Marketing.FunctionalTests public TestServer CreateServer() { + + Console.WriteLine(" Creating test server"); var path = Assembly.GetAssembly(typeof(MarketingScenarioBase)) .Location; + Console.WriteLine(" Creating builder"); + var hostBuilder = new WebHostBuilder() .UseContentRoot(Path.GetDirectoryName(path)) .ConfigureAppConfiguration(cb => { - cb.AddJsonFile("appsettings.json", optional: false) + var h = cb.AddJsonFile("appsettings.json", optional: false) .AddEnvironmentVariables(); - }).UseStartup(); + }) + .CaptureStartupErrors(true) + .UseStartup(); + Console.WriteLine(" Created builder"); + var testServer = new TestServer(hostBuilder); + using (var scope = testServer.Services.CreateScope()) + { + var services = scope.ServiceProvider; + + var logger = services.GetRequiredService>(); + var settings = services.GetRequiredService>(); + logger.LogError("connectionString " + settings.Value.ConnectionString); + Console.WriteLine("connectionString " + settings.Value.ConnectionString); + } + testServer.Host + .RemoveDbContext() .MigrateDbContext((context, services) => { var logger = services.GetService>(); + logger.LogError("Migrating MarketingContextSeed"); new MarketingContextSeed() .SeedAsync(context, logger) .Wait(); }); + Console.WriteLine(" Thread to sleep"); + + Thread.Sleep(5000); + Console.WriteLine(" Thread after"); + return testServer; }