From 69cc6cb129460ce523697fc8aeeb77f066860eb3 Mon Sep 17 00:00:00 2001 From: Alan West <3676547+alanwest@users.noreply.github.com> Date: Wed, 31 Mar 2021 07:06:16 -0700 Subject: [PATCH] Instrumenting services with OpenTelemetry (#1638) * WebMVC instrumented with OpenTelemetry * Basket.API instrumented with OpenTelemetry * Add Zipkin export * Add Jaeger export * Remove line for debugging purposes --- src/OpenTelemetry/otel-collector-config.yaml | 18 +++++ .../Basket/Basket.API/Basket.API.csproj | 12 ++++ .../Basket/Basket.API/OpenTelemetry.cs | 66 +++++++++++++++++++ src/Services/Basket/Basket.API/Startup.cs | 3 +- src/Web/WebMVC/OpenTelemetry.cs | 62 +++++++++++++++++ src/Web/WebMVC/Startup.cs | 1 + src/Web/WebMVC/WebMVC.csproj | 11 ++++ src/docker-compose.opentelemetry.console.yml | 15 +++++ src/docker-compose.opentelemetry.jaeger.yml | 29 ++++++++ src/docker-compose.opentelemetry.otlp.yml | 27 ++++++++ src/docker-compose.opentelemetry.zipkin.yml | 22 +++++++ 11 files changed, 265 insertions(+), 1 deletion(-) create mode 100644 src/OpenTelemetry/otel-collector-config.yaml create mode 100644 src/Services/Basket/Basket.API/OpenTelemetry.cs create mode 100644 src/Web/WebMVC/OpenTelemetry.cs create mode 100644 src/docker-compose.opentelemetry.console.yml create mode 100644 src/docker-compose.opentelemetry.jaeger.yml create mode 100644 src/docker-compose.opentelemetry.otlp.yml create mode 100644 src/docker-compose.opentelemetry.zipkin.yml diff --git a/src/OpenTelemetry/otel-collector-config.yaml b/src/OpenTelemetry/otel-collector-config.yaml new file mode 100644 index 000000000..e50fdd6e1 --- /dev/null +++ b/src/OpenTelemetry/otel-collector-config.yaml @@ -0,0 +1,18 @@ +receivers: + otlp: + protocols: + grpc: + +exporters: + logging: + loglevel: debug + +processors: + batch: + +service: + pipelines: + traces: + receivers: [otlp] + processors: [batch] + exporters: [logging] diff --git a/src/Services/Basket/Basket.API/Basket.API.csproj b/src/Services/Basket/Basket.API/Basket.API.csproj index 741407c10..b7a4b3743 100644 --- a/src/Services/Basket/Basket.API/Basket.API.csproj +++ b/src/Services/Basket/Basket.API/Basket.API.csproj @@ -42,6 +42,18 @@ + + + + + + + + + + + + diff --git a/src/Services/Basket/Basket.API/OpenTelemetry.cs b/src/Services/Basket/Basket.API/OpenTelemetry.cs new file mode 100644 index 000000000..fff7a56b5 --- /dev/null +++ b/src/Services/Basket/Basket.API/OpenTelemetry.cs @@ -0,0 +1,66 @@ +using Microsoft.Extensions.DependencyInjection; +using OpenTelemetry; +using OpenTelemetry.Resources; +using OpenTelemetry.Trace; +using StackExchange.Redis; +using System; + +static class OpenTelemetryExtensions +{ + public static void AddOpenTelemetry(ConnectionMultiplexer connectionMultiplexer) + { + var exportType = Environment.GetEnvironmentVariable("OTEL_USE_EXPORTER")?.ToLower(); + if (exportType == null) + { + return; + } + + var tracerProviderBuilder = Sdk.CreateTracerProviderBuilder(); + + // Configure resource + tracerProviderBuilder + .SetResourceBuilder(ResourceBuilder.CreateDefault().AddService("Basket.API")); + + // Configure instrumentation + tracerProviderBuilder + .AddAspNetCoreInstrumentation() + .AddHttpClientInstrumentation() + .AddRedisInstrumentation(connectionMultiplexer); + + // Configure exporter + switch (exportType) + { + case "jaeger": + tracerProviderBuilder.AddJaegerExporter(options => + { + var agentHost = Environment.GetEnvironmentVariable("OTEL_EXPORTER_JAEGER_AGENTHOST"); + options.AgentHost = agentHost; + }); + break; + case "otlp": + tracerProviderBuilder.AddOtlpExporter(options => + { + var endpoint = Environment.GetEnvironmentVariable("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT") + ?? Environment.GetEnvironmentVariable("OTEL_EXPORTER_OTLP_ENDPOINT"); + options.Endpoint = new Uri(endpoint); + + var headers = Environment.GetEnvironmentVariable("OTEL_EXPORTER_OTLP_TRACES_HEADERS") + ?? Environment.GetEnvironmentVariable("OTEL_EXPORTER_OTLP_HEADERS"); + options.Headers = headers; + }); + break; + case "zipkin": + tracerProviderBuilder.AddZipkinExporter(options => + { + var endpoint = Environment.GetEnvironmentVariable("OTEL_EXPORTER_ZIPKIN_ENDPOINT"); + options.Endpoint = new Uri(endpoint); + }); + break; + default: + tracerProviderBuilder.AddConsoleExporter(); + break; + } + + tracerProviderBuilder.Build(); + } +} diff --git a/src/Services/Basket/Basket.API/Startup.cs b/src/Services/Basket/Basket.API/Startup.cs index 2f53cbcb0..ef6c37e83 100644 --- a/src/Services/Basket/Basket.API/Startup.cs +++ b/src/Services/Basket/Basket.API/Startup.cs @@ -185,8 +185,9 @@ namespace Microsoft.eShopOnContainers.Services.Basket.API } // This method gets called by the runtime. Use this method to configure the HTTP request pipeline. - public void Configure(IApplicationBuilder app, IWebHostEnvironment env, ILoggerFactory loggerFactory) + public void Configure(IApplicationBuilder app, IWebHostEnvironment env, ILoggerFactory loggerFactory, ConnectionMultiplexer connectionMultiplexer) { + OpenTelemetryExtensions.AddOpenTelemetry(connectionMultiplexer); //loggerFactory.AddAzureWebAppDiagnostics(); //loggerFactory.AddApplicationInsights(app.ApplicationServices, LogLevel.Trace); diff --git a/src/Web/WebMVC/OpenTelemetry.cs b/src/Web/WebMVC/OpenTelemetry.cs new file mode 100644 index 000000000..be73c62c1 --- /dev/null +++ b/src/Web/WebMVC/OpenTelemetry.cs @@ -0,0 +1,62 @@ +using Microsoft.Extensions.DependencyInjection; +using OpenTelemetry.Resources; +using OpenTelemetry.Trace; +using System; + +static class OpenTelemetryExtensions +{ + public static IServiceCollection AddOpenTelemetry(this IServiceCollection services) + { + var exportType = Environment.GetEnvironmentVariable("OTEL_USE_EXPORTER")?.ToLower(); + if (exportType == null) + { + return services; + } + + return services.AddOpenTelemetryTracing((serviceProvider, tracerProviderBuilder) => + { + // Configure resource + tracerProviderBuilder + .SetResourceBuilder(ResourceBuilder.CreateDefault().AddService("WebMVC")); + + // Configure instrumentation + tracerProviderBuilder + .AddAspNetCoreInstrumentation() + .AddHttpClientInstrumentation(); + + // Configure exporter + switch (exportType) + { + case "jaeger": + tracerProviderBuilder.AddJaegerExporter(options => + { + var agentHost = Environment.GetEnvironmentVariable("OTEL_EXPORTER_JAEGER_AGENTHOST"); + options.AgentHost = agentHost; + }); + break; + case "otlp": + tracerProviderBuilder.AddOtlpExporter(options => + { + var endpoint = Environment.GetEnvironmentVariable("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT") + ?? Environment.GetEnvironmentVariable("OTEL_EXPORTER_OTLP_ENDPOINT"); + options.Endpoint = new Uri(endpoint); + + var headers = Environment.GetEnvironmentVariable("OTEL_EXPORTER_OTLP_TRACES_HEADERS") + ?? Environment.GetEnvironmentVariable("OTEL_EXPORTER_OTLP_HEADERS"); + options.Headers = headers; + }); + break; + case "zipkin": + tracerProviderBuilder.AddZipkinExporter(options => + { + var endpoint = Environment.GetEnvironmentVariable("OTEL_EXPORTER_ZIPKIN_ENDPOINT"); + options.Endpoint = new Uri(endpoint); + }); + break; + default: + tracerProviderBuilder.AddConsoleExporter(); + break; + } + }); + } +} diff --git a/src/Web/WebMVC/Startup.cs b/src/Web/WebMVC/Startup.cs index 427e234c7..671f7d193 100644 --- a/src/Web/WebMVC/Startup.cs +++ b/src/Web/WebMVC/Startup.cs @@ -36,6 +36,7 @@ namespace Microsoft.eShopOnContainers.WebMVC services.AddControllersWithViews() .Services .AddAppInsight(Configuration) + .AddOpenTelemetry() .AddHealthChecks(Configuration) .AddCustomMvc(Configuration) .AddDevspaces() diff --git a/src/Web/WebMVC/WebMVC.csproj b/src/Web/WebMVC/WebMVC.csproj index ec922d36c..13ef68d79 100644 --- a/src/Web/WebMVC/WebMVC.csproj +++ b/src/Web/WebMVC/WebMVC.csproj @@ -44,6 +44,17 @@ + + + + + + + + + + + diff --git a/src/docker-compose.opentelemetry.console.yml b/src/docker-compose.opentelemetry.console.yml new file mode 100644 index 000000000..a894d1fef --- /dev/null +++ b/src/docker-compose.opentelemetry.console.yml @@ -0,0 +1,15 @@ +version: '3.4' + +# The OpenTelemetry docker-compose file is used to configure OpenTelemetry for the services +# +# You need to start it with the following CLI command: +# docker-compose -f docker-compose.yml -f docker-compose.override.yml -f docker-compose.opentelemetry.console.yml up + +services: + basket-api: + environment: + - OTEL_USE_EXPORTER=console + + webmvc: + environment: + - OTEL_USE_EXPORTER=console diff --git a/src/docker-compose.opentelemetry.jaeger.yml b/src/docker-compose.opentelemetry.jaeger.yml new file mode 100644 index 000000000..59625ceb1 --- /dev/null +++ b/src/docker-compose.opentelemetry.jaeger.yml @@ -0,0 +1,29 @@ +version: '3.4' + +# The OpenTelemetry docker-compose file is used to configure OpenTelemetry for the services +# +# You need to start it with the following CLI command: +# docker-compose -f docker-compose.yml -f docker-compose.override.yml -f docker-compose.opentelemetry.zipkin.yml up + +services: + jaeger-all-in-one: + image: jaegertracing/all-in-one:latest + ports: + - "5775:5775/udp" + - "6831:6831/udp" + - "6832:6832/udp" + - "5778:5778" + - "16686:16686" + - "14268:14268" + - "14250:14250" + - "9411:9411" + + basket-api: + environment: + - OTEL_USE_EXPORTER=jaeger + - OTEL_EXPORTER_JAEGER_AGENTHOST=${OTEL_EXPORTER_JAEGER_AGENTHOST:-jaeger-all-in-one} + + webmvc: + environment: + - OTEL_USE_EXPORTER=jaeger + - OTEL_EXPORTER_JAEGER_AGENTHOST=${OTEL_EXPORTER_JAEGER_AGENTHOST:-jaeger-all-in-one} diff --git a/src/docker-compose.opentelemetry.otlp.yml b/src/docker-compose.opentelemetry.otlp.yml new file mode 100644 index 000000000..f9015ac65 --- /dev/null +++ b/src/docker-compose.opentelemetry.otlp.yml @@ -0,0 +1,27 @@ +version: '3.4' + +# The OpenTelemetry docker-compose file is used to configure OpenTelemetry for the services +# +# You need to start it with the following CLI command: +# docker-compose -f docker-compose.yml -f docker-compose.override.yml -f docker-compose.opentelemetry.otlp.yml up + +services: + otel-collector: + image: otel/opentelemetry-collector:latest + command: ["--config=/etc/otel-collector-config.yaml"] + volumes: + - ./OpenTelemetry/otel-collector-config.yaml:/etc/otel-collector-config.yaml + ports: + - "4317" # OTLP gRPC receiver + + basket-api: + environment: + - OTEL_USE_EXPORTER=otlp + - OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=${OTEL_EXPORTER_OTLP_TRACES_ENDPOINT:-http://otel-collector:4317} + - OTEL_EXPORTER_OTLP_HEADERS=${OTEL_EXPORTER_OTLP_HEADERS} + + webmvc: + environment: + - OTEL_USE_EXPORTER=otlp + - OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=${OTEL_EXPORTER_OTLP_TRACES_ENDPOINT:-http://otel-collector:4317} + - OTEL_EXPORTER_OTLP_HEADERS=${OTEL_EXPORTER_OTLP_HEADERS} diff --git a/src/docker-compose.opentelemetry.zipkin.yml b/src/docker-compose.opentelemetry.zipkin.yml new file mode 100644 index 000000000..a682ca555 --- /dev/null +++ b/src/docker-compose.opentelemetry.zipkin.yml @@ -0,0 +1,22 @@ +version: '3.4' + +# The OpenTelemetry docker-compose file is used to configure OpenTelemetry for the services +# +# You need to start it with the following CLI command: +# docker-compose -f docker-compose.yml -f docker-compose.override.yml -f docker-compose.opentelemetry.zipkin.yml up + +services: + zipkin: + image: openzipkin/zipkin:latest + ports: + - "9411:9411" + + basket-api: + environment: + - OTEL_USE_EXPORTER=zipkin + - OTEL_EXPORTER_ZIPKIN_ENDPOINT=${OTEL_EXPORTER_ZIPKIN_ENDPOINT:-http://zipkin:9411/api/v2/spans} + + webmvc: + environment: + - OTEL_USE_EXPORTER=zipkin + - OTEL_EXPORTER_ZIPKIN_ENDPOINT=${OTEL_EXPORTER_ZIPKIN_ENDPOINT:-http://zipkin:9411/api/v2/spans}