Instrumenting services with OpenTelemetry (#1638)

* WebMVC instrumented with OpenTelemetry

* Basket.API instrumented with OpenTelemetry

* Add Zipkin export

* Add Jaeger export

* Remove line for debugging purposes

src/OpenTelemetry/otel-collector-config.yaml

@@ -0,0 +1,18 @@
+receivers:
+  otlp:
+    protocols:
+      grpc:
+
+exporters:
+  logging:
+    loglevel: debug
+
+processors:
+  batch:
+
+service:
+  pipelines:
+    traces:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [logging]

src/Services/Basket/Basket.API/Basket.API.csproj

@@ -42,6 +42,18 @@
     <PackageReference Include="Swashbuckle.AspNetCore" Version="5.6.3" />
   </ItemGroup>
 
+  <ItemGroup>
+    <PackageReference Include="OpenTelemetry" Version="1.0.1" />
+    <PackageReference Include="OpenTelemetry.Exporter.Console" Version="1.0.1" />
+    <PackageReference Include="OpenTelemetry.Exporter.Jaeger" Version="1.0.1" />
+    <PackageReference Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.0.1" />
+    <PackageReference Include="OpenTelemetry.Exporter.Zipkin" Version="1.0.1" />
+    <PackageReference Include="OpenTelemetry.Extensions.Hosting" Version="1.0.0-rc2" />
+    <PackageReference Include="OpenTelemetry.Instrumentation.AspNetCore" Version="1.0.0-rc2" />
+    <PackageReference Include="OpenTelemetry.Instrumentation.Http" Version="1.0.0-rc2" />
+    <PackageReference Include="OpenTelemetry.Instrumentation.StackExchangeRedis" Version="1.0.0-rc2" />
+  </ItemGroup>
+
   <ItemGroup>
     <Protobuf Include="Proto\basket.proto" GrpcServices="Server" Generator="MSBuild:Compile" />
     <Content Include="@(Protobuf)" />

src/Services/Basket/Basket.API/OpenTelemetry.cs

@@ -0,0 +1,66 @@
+using Microsoft.Extensions.DependencyInjection;
+using OpenTelemetry;
+using OpenTelemetry.Resources;
+using OpenTelemetry.Trace;
+using StackExchange.Redis;
+using System;
+
+static class OpenTelemetryExtensions
+{
+    public static void AddOpenTelemetry(ConnectionMultiplexer connectionMultiplexer)
+    {
+        var exportType = Environment.GetEnvironmentVariable("OTEL_USE_EXPORTER")?.ToLower();
+        if (exportType == null)
+        {
+            return;
+        }
+
+        var tracerProviderBuilder = Sdk.CreateTracerProviderBuilder();
+
+        // Configure resource
+        tracerProviderBuilder
+            .SetResourceBuilder(ResourceBuilder.CreateDefault().AddService("Basket.API"));
+
+        // Configure instrumentation
+        tracerProviderBuilder
+            .AddAspNetCoreInstrumentation()
+            .AddHttpClientInstrumentation()
+            .AddRedisInstrumentation(connectionMultiplexer);
+
+        // Configure exporter
+        switch (exportType)
+        {
+            case "jaeger":
+                tracerProviderBuilder.AddJaegerExporter(options =>
+                {
+                    var agentHost = Environment.GetEnvironmentVariable("OTEL_EXPORTER_JAEGER_AGENTHOST");
+                    options.AgentHost = agentHost;
+                });
+                break;
+            case "otlp":
+                tracerProviderBuilder.AddOtlpExporter(options =>
+                {
+                    var endpoint = Environment.GetEnvironmentVariable("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT")
+                        ?? Environment.GetEnvironmentVariable("OTEL_EXPORTER_OTLP_ENDPOINT");
+                    options.Endpoint = new Uri(endpoint);
+
+                    var headers = Environment.GetEnvironmentVariable("OTEL_EXPORTER_OTLP_TRACES_HEADERS")
+                        ?? Environment.GetEnvironmentVariable("OTEL_EXPORTER_OTLP_HEADERS");
+                    options.Headers = headers;
+                });
+                break;
+            case "zipkin":
+                tracerProviderBuilder.AddZipkinExporter(options =>
+                {
+                    var endpoint = Environment.GetEnvironmentVariable("OTEL_EXPORTER_ZIPKIN_ENDPOINT");
+                    options.Endpoint = new Uri(endpoint);
+                });
+                break;
+            default:
+                tracerProviderBuilder.AddConsoleExporter();
+                break;
+        }
+
+        tracerProviderBuilder.Build();
+    }
+}

src/Services/Basket/Basket.API/Startup.cs

@@ -185,8 +185,9 @@ namespace Microsoft.eShopOnContainers.Services.Basket.API
         }
 
         // This method gets called by the runtime. Use this method to configure the HTTP request pipeline.
-        public void Configure(IApplicationBuilder app, IWebHostEnvironment env, ILoggerFactory loggerFactory)
+        public void Configure(IApplicationBuilder app, IWebHostEnvironment env, ILoggerFactory loggerFactory, ConnectionMultiplexer connectionMultiplexer)
         {
+            OpenTelemetryExtensions.AddOpenTelemetry(connectionMultiplexer);
             //loggerFactory.AddAzureWebAppDiagnostics();
             //loggerFactory.AddApplicationInsights(app.ApplicationServices, LogLevel.Trace);
 

src/Web/WebMVC/OpenTelemetry.cs

@@ -0,0 +1,62 @@
+using Microsoft.Extensions.DependencyInjection;
+using OpenTelemetry.Resources;
+using OpenTelemetry.Trace;
+using System;
+
+static class OpenTelemetryExtensions
+{
+    public static IServiceCollection AddOpenTelemetry(this IServiceCollection services)
+    {
+        var exportType = Environment.GetEnvironmentVariable("OTEL_USE_EXPORTER")?.ToLower();
+        if (exportType == null)
+        {
+            return services;
+        }
+
+        return services.AddOpenTelemetryTracing((serviceProvider, tracerProviderBuilder) =>
+        {
+            // Configure resource
+            tracerProviderBuilder
+                .SetResourceBuilder(ResourceBuilder.CreateDefault().AddService("WebMVC"));
+
+            // Configure instrumentation
+            tracerProviderBuilder
+                .AddAspNetCoreInstrumentation()
+                .AddHttpClientInstrumentation();
+
+            // Configure exporter
+            switch (exportType)
+            {
+                case "jaeger":
+                    tracerProviderBuilder.AddJaegerExporter(options =>
+                    {
+                        var agentHost = Environment.GetEnvironmentVariable("OTEL_EXPORTER_JAEGER_AGENTHOST");
+                        options.AgentHost = agentHost;
+                    });
+                    break;
+                case "otlp":
+                    tracerProviderBuilder.AddOtlpExporter(options =>
+                    {
+                        var endpoint = Environment.GetEnvironmentVariable("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT")
+                            ?? Environment.GetEnvironmentVariable("OTEL_EXPORTER_OTLP_ENDPOINT");
+                        options.Endpoint = new Uri(endpoint);
+
+                        var headers = Environment.GetEnvironmentVariable("OTEL_EXPORTER_OTLP_TRACES_HEADERS")
+                            ?? Environment.GetEnvironmentVariable("OTEL_EXPORTER_OTLP_HEADERS");
+                        options.Headers = headers;
+                    });
+                    break;
+                case "zipkin":
+                    tracerProviderBuilder.AddZipkinExporter(options =>
+                    {
+                        var endpoint = Environment.GetEnvironmentVariable("OTEL_EXPORTER_ZIPKIN_ENDPOINT");
+                        options.Endpoint = new Uri(endpoint);
+                    });
+                    break;
+                default:
+                    tracerProviderBuilder.AddConsoleExporter();
+                    break;
+            }
+        });
+    }
+}

src/Web/WebMVC/Startup.cs

@@ -36,6 +36,7 @@ namespace Microsoft.eShopOnContainers.WebMVC
             services.AddControllersWithViews()
                 .Services
                 .AddAppInsight(Configuration)
+                .AddOpenTelemetry()
                 .AddHealthChecks(Configuration)
                 .AddCustomMvc(Configuration)
                 .AddDevspaces()

src/Web/WebMVC/WebMVC.csproj

@@ -44,6 +44,17 @@
     <PackageReference Include="Serilog.Sinks.Seq" Version="4.1.0-dev-00166" />
   </ItemGroup>
 
+  <ItemGroup>
+    <PackageReference Include="OpenTelemetry" Version="1.0.1" />
+    <PackageReference Include="OpenTelemetry.Exporter.Console" Version="1.0.1" />
+    <PackageReference Include="OpenTelemetry.Exporter.Jaeger" Version="1.0.1" />
+    <PackageReference Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.0.1" />
+    <PackageReference Include="OpenTelemetry.Exporter.Zipkin" Version="1.0.1" />
+    <PackageReference Include="OpenTelemetry.Extensions.Hosting" Version="1.0.0-rc2" />
+    <PackageReference Include="OpenTelemetry.Instrumentation.AspNetCore" Version="1.0.0-rc2" />
+    <PackageReference Include="OpenTelemetry.Instrumentation.Http" Version="1.0.0-rc2" />
+  </ItemGroup>
+
   <ItemGroup>
     <None Include="ViewModels\CampaignItem.cs" />
   </ItemGroup>

src/docker-compose.opentelemetry.console.yml

@@ -0,0 +1,15 @@
+version: '3.4'
+
+# The OpenTelemetry docker-compose file is used to configure OpenTelemetry for the services
+#
+# You need to start it with the following CLI command:
+# docker-compose -f docker-compose.yml -f docker-compose.override.yml -f docker-compose.opentelemetry.console.yml up
+
+services:
+  basket-api:
+    environment:
+      - OTEL_USE_EXPORTER=console
+
+  webmvc:
+    environment:
+      - OTEL_USE_EXPORTER=console

src/docker-compose.opentelemetry.jaeger.yml

@@ -0,0 +1,29 @@
+version: '3.4'
+
+# The OpenTelemetry docker-compose file is used to configure OpenTelemetry for the services
+#
+# You need to start it with the following CLI command:
+# docker-compose -f docker-compose.yml -f docker-compose.override.yml -f docker-compose.opentelemetry.zipkin.yml up
+
+services:
+  jaeger-all-in-one:
+    image: jaegertracing/all-in-one:latest
+    ports:
+      - "5775:5775/udp"
+      - "6831:6831/udp"
+      - "6832:6832/udp"
+      - "5778:5778"
+      - "16686:16686"
+      - "14268:14268"
+      - "14250:14250"
+      - "9411:9411"
+
+  basket-api:
+    environment:
+      - OTEL_USE_EXPORTER=jaeger
+      - OTEL_EXPORTER_JAEGER_AGENTHOST=${OTEL_EXPORTER_JAEGER_AGENTHOST:-jaeger-all-in-one}
+
+  webmvc:
+    environment:
+      - OTEL_USE_EXPORTER=jaeger
+      - OTEL_EXPORTER_JAEGER_AGENTHOST=${OTEL_EXPORTER_JAEGER_AGENTHOST:-jaeger-all-in-one}

src/docker-compose.opentelemetry.otlp.yml

@@ -0,0 +1,27 @@
+version: '3.4'
+
+# The OpenTelemetry docker-compose file is used to configure OpenTelemetry for the services
+#
+# You need to start it with the following CLI command:
+# docker-compose -f docker-compose.yml -f docker-compose.override.yml -f docker-compose.opentelemetry.otlp.yml up
+
+services:
+  otel-collector:
+    image: otel/opentelemetry-collector:latest
+    command: ["--config=/etc/otel-collector-config.yaml"]
+    volumes:
+      - ./OpenTelemetry/otel-collector-config.yaml:/etc/otel-collector-config.yaml
+    ports:
+      - "4317"        # OTLP gRPC receiver
+
+  basket-api:
+    environment:
+      - OTEL_USE_EXPORTER=otlp
+      - OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=${OTEL_EXPORTER_OTLP_TRACES_ENDPOINT:-http://otel-collector:4317}
+      - OTEL_EXPORTER_OTLP_HEADERS=${OTEL_EXPORTER_OTLP_HEADERS}
+
+  webmvc:
+    environment:
+      - OTEL_USE_EXPORTER=otlp
+      - OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=${OTEL_EXPORTER_OTLP_TRACES_ENDPOINT:-http://otel-collector:4317}
+      - OTEL_EXPORTER_OTLP_HEADERS=${OTEL_EXPORTER_OTLP_HEADERS}

src/docker-compose.opentelemetry.zipkin.yml

@@ -0,0 +1,22 @@
+version: '3.4'
+
+# The OpenTelemetry docker-compose file is used to configure OpenTelemetry for the services
+#
+# You need to start it with the following CLI command:
+# docker-compose -f docker-compose.yml -f docker-compose.override.yml -f docker-compose.opentelemetry.zipkin.yml up
+
+services:
+  zipkin:
+    image: openzipkin/zipkin:latest
+    ports:
+      - "9411:9411"
+
+  basket-api:
+    environment:
+      - OTEL_USE_EXPORTER=zipkin
+      - OTEL_EXPORTER_ZIPKIN_ENDPOINT=${OTEL_EXPORTER_ZIPKIN_ENDPOINT:-http://zipkin:9411/api/v2/spans}
+
+  webmvc:
+    environment:
+      - OTEL_USE_EXPORTER=zipkin
+      - OTEL_EXPORTER_ZIPKIN_ENDPOINT=${OTEL_EXPORTER_ZIPKIN_ENDPOINT:-http://zipkin:9411/api/v2/spans}