Change configuration to use hosted kafka

src/BuildingBlocks/EventBus/EventBusKafka/KafkaConsumerBackgroundService.cs

@@ -48,7 +48,7 @@ public class KafkaConsumerBackgroundService : BackgroundService
 
                 if (!_subsManager.HasSubscriptionsForEvent(eventName))
                 {
-                    _logger.LogWarning("No subscription for Kafka event: {EventName}", eventName);
+                    // _logger.LogWarning("No subscription for Kafka event: {EventName}", eventName);
                     continue;
                 }
                 

src/Services/Basket/Basket.API/appsettings.Development.json

@@ -13,20 +13,5 @@
   "IdentityUrl": "http://localhost:5105",
   "ConnectionString": "127.0.0.1",
   "AzureServiceBusEnabled": false,
-  "EventBusConnection": "localhost",
-  "KafkaEnabled": true,
-  "Kafka": {
-    "AdminSettings": {
-      "BootstrapServers": "broker:9092"
-    },
-    "ProducerSettings": {
-      "BootstrapServers": "broker:9092",
-      "AllowAutoCreateTopics": true
-    },
-    "ConsumerSettings": {
-      "BootstrapServers": "broker:9092",
-      "GroupId": "basket-group-id",
-      "AllowAutoCreateTopics": true
-    }
-  }
+  "EventBusConnection": "localhost"
 }

src/Services/Basket/Basket.API/appsettings.json

@@ -30,16 +30,34 @@
   "KafkaEnabled": true,
   "Kafka": {
     "AdminSettings": {
-      "BootstrapServers": "broker:9092"
+      "GroupId": "basket-group-id",
+      "AllowAutoCreateTopics": true,
+      "BootstrapServers": "pkc-xmzwx.europe-central2.gcp.confluent.cloud:9092",
+      "SslEndpointIdentificationAlgorithm": "Https",
+      "SecurityProtocol": "SaslSsl",
+      "SaslMechanism": "Plain",
+      "SaslUsername": "username",
+      "SaslPassword": "password"      
     },
     "ProducerSettings": {
-      "BootstrapServers": "broker:9092",
-      "AllowAutoCreateTopics": true
+      "GroupId": "basket-group-id",
+      "AllowAutoCreateTopics": true,
+      "BootstrapServers": "pkc-xmzwx.europe-central2.gcp.confluent.cloud:9092",
+      "SslEndpointIdentificationAlgorithm": "Https",
+      "SecurityProtocol": "SaslSsl",
+      "SaslMechanism": "Plain",
+      "SaslUsername": "username",
+      "SaslPassword": "password"
     },
     "ConsumerSettings": {
-      "BootstrapServers": "broker:9092",
       "GroupId": "basket-group-id",
-      "AllowAutoCreateTopics": true
+      "BootstrapServers": "pkc-xmzwx.europe-central2.gcp.confluent.cloud:9092",
+      "AutoOffsetReset": "Earliest",      
+      "SslEndpointIdentificationAlgorithm": "Https",
+      "SecurityProtocol": "SaslSsl",
+      "SaslMechanism": "Plain",
+      "SaslUsername": "username",
+      "SaslPassword": "password"
     }
   }
 }

src/Services/Catalog/Catalog.API/appsettings.Development.json

@@ -10,18 +10,5 @@
         "System": "Warning"
       }
     }
-  },
-  "EventBusConnection": "localhost",
-  "KafkaEnabled": true,
-  "Kafka": {
-    "ProducerSettings": {
-      "BootstrapServers": "broker:9092",
-      "AllowAutoCreateTopics": true
-    },
-    "ConsumerSettings": {
-      "BootstrapServers": "broker:9092",
-      "GroupId": "catalog-group-id",
-      "AllowAutoCreateTopics": true
-    }
   }
 }

src/Services/Catalog/Catalog.API/appsettings.json

@@ -27,14 +27,35 @@
   },
   "KafkaEnabled": true,
   "Kafka": {
+    "AdminSettings": {
+      "GroupId": "catalog-group-id",
+      "AllowAutoCreateTopics": true,
+      "BootstrapServers": "pkc-xmzwx.europe-central2.gcp.confluent.cloud:9092",
+      "SslEndpointIdentificationAlgorithm": "Https",
+      "SecurityProtocol": "SaslSsl",
+      "SaslMechanism": "Plain",
+      "SaslUsername": "username",
+      "SaslPassword": "password"
+    },
     "ProducerSettings": {
-      "BootstrapServers": "broker:9092",
-      "AllowAutoCreateTopics": true
+      "GroupId": "catalog-group-id",
+      "AllowAutoCreateTopics": true,
+      "BootstrapServers": "pkc-xmzwx.europe-central2.gcp.confluent.cloud:9092",
+      "SslEndpointIdentificationAlgorithm": "Https",
+      "SecurityProtocol": "SaslSsl",
+      "SaslMechanism": "Plain",
+      "SaslUsername": "username",
+      "SaslPassword": "password"
     },
     "ConsumerSettings": {
-      "BootstrapServers": "broker:9092",
       "GroupId": "catalog-group-id",
-      "AllowAutoCreateTopics": true
+      "BootstrapServers": "pkc-xmzwx.europe-central2.gcp.confluent.cloud:9092",
+      "AutoOffsetReset": "Earliest",
+      "SslEndpointIdentificationAlgorithm": "Https",
+      "SecurityProtocol": "SaslSsl",
+      "SaslMechanism": "Plain",
+      "SaslUsername": "username",
+      "SaslPassword": "password"
     }
   }
 }

src/Services/Ordering/Ordering.API/Application/Commands/SetPaidOrderStatusCommandHandler.cs

@@ -19,7 +19,7 @@ public class SetPaidOrderStatusCommandHandler : IRequestHandler<SetPaidOrderStat
     public async Task<bool> Handle(SetPaidOrderStatusCommand command, CancellationToken cancellationToken)
     {
         // Simulate a work time for validating the payment
-        await Task.Delay(10000, cancellationToken);
+        await Task.Delay(2000, cancellationToken);
 
         var orderToUpdate = await _orderRepository.GetAsync(command.OrderNumber);
         if (orderToUpdate == null)

src/Services/Ordering/Ordering.API/appsettings.json

@@ -25,15 +25,36 @@
   "EventBusConnection": "localhost",
   "KafkaEnabled": true,
   "Kafka": {
-    "ProducerSettings": {
-      "BootstrapServers": "broker:9092",
-      "AllowAutoCreateTopics": true
-    },
-    "ConsumerSettings": {
-      "BootstrapServers": "broker:9092",
-      "GroupId": "ordering-group-id",
-      "AllowAutoCreateTopics": true
-    }
+  "AdminSettings": {
+    "GroupId": "ordering-group-id",
+    "AllowAutoCreateTopics": true,
+    "BootstrapServers": "pkc-xmzwx.europe-central2.gcp.confluent.cloud:9092",
+    "SslEndpointIdentificationAlgorithm": "Https",
+    "SecurityProtocol": "SaslSsl",
+    "SaslMechanism": "Plain",
+    "SaslUsername": "username",
+    "SaslPassword": "password"
+  },
+  "ProducerSettings": {
+    "GroupId": "ordering-group-id",
+    "AllowAutoCreateTopics": true,
+    "BootstrapServers": "pkc-xmzwx.europe-central2.gcp.confluent.cloud:9092",
+    "SslEndpointIdentificationAlgorithm": "Https",
+    "SecurityProtocol": "SaslSsl",
+    "SaslMechanism": "Plain",
+    "SaslUsername": "username",
+    "SaslPassword": "password"
+  },
+  "ConsumerSettings": {
+    "GroupId": "ordering-group-id",
+    "BootstrapServers": "pkc-xmzwx.europe-central2.gcp.confluent.cloud:9092",
+    "AutoOffsetReset": "Earliest",
+    "SslEndpointIdentificationAlgorithm": "Https",
+    "SecurityProtocol": "SaslSsl",
+    "SaslMechanism": "Plain",
+    "SaslUsername": "username",
+    "SaslPassword": "password"
+  }
   },
   "UseVault": false,
   "Vault": {

src/Services/Ordering/Ordering.BackgroundTasks/appsettings.Development.json

@@ -5,17 +5,5 @@
       "System": "Information",
       "Microsoft": "Information"
     }
-  },
-  "KafkaEnabled": true,
-  "Kafka": {
-    "ProducerSettings": {
-      "BootstrapServers": "broker:9092",
-      "AllowAutoCreateTopics": true
-    },
-    "ConsumerSettings": {
-      "BootstrapServers": "broker:9092",
-      "GroupId": "ordering-background-group-id",
-      "AllowAutoCreateTopics": true
-    }
   }
 }

src/Services/Ordering/Ordering.BackgroundTasks/appsettings.json

@@ -25,14 +25,35 @@
   "EventBusPassword": "",
   "KafkaEnabled": true,
   "Kafka": {
+    "AdminSettings": {
+      "GroupId": "ordering-background-group-id",
+      "AllowAutoCreateTopics": true,
+      "BootstrapServers": "pkc-xmzwx.europe-central2.gcp.confluent.cloud:9092",
+      "SslEndpointIdentificationAlgorithm": "Https",
+      "SecurityProtocol": "SaslSsl",
+      "SaslMechanism": "Plain",
+      "SaslUsername": "username",
+      "SaslPassword": "MMn/CFO53yA631L8/EW1iXbmVWzN/3DZgJgWfQDBTreEgDhp4ica8SyaDqE9iP7r"
+    },
     "ProducerSettings": {
-      "BootstrapServers": "broker:9092",
-      "AllowAutoCreateTopics": true
+      "GroupId": "ordering-background-group-id",
+      "AllowAutoCreateTopics": true,
+      "BootstrapServers": "pkc-xmzwx.europe-central2.gcp.confluent.cloud:9092",
+      "SslEndpointIdentificationAlgorithm": "Https",
+      "SecurityProtocol": "SaslSsl",
+      "SaslMechanism": "Plain",
+      "SaslUsername": "username",
+      "SaslPassword": "MMn/CFO53yA631L8/EW1iXbmVWzN/3DZgJgWfQDBTreEgDhp4ica8SyaDqE9iP7r"
     },
     "ConsumerSettings": {
-      "BootstrapServers": "broker:9092",
       "GroupId": "ordering-background-group-id",
-      "AllowAutoCreateTopics": true
+      "BootstrapServers": "pkc-xmzwx.europe-central2.gcp.confluent.cloud:9092",
+      "AutoOffsetReset": "Earliest",
+      "SslEndpointIdentificationAlgorithm": "Https",
+      "SecurityProtocol": "SaslSsl",
+      "SaslMechanism": "Plain",
+      "SaslUsername": "username",
+      "SaslPassword": "MMn/CFO53yA631L8/EW1iXbmVWzN/3DZgJgWfQDBTreEgDhp4ica8SyaDqE9iP7r"
     }
   }
-}
+}

src/Services/Payment/Payment.API/appsettings.Development.json

@@ -6,17 +6,5 @@
       "System": "Information",
       "Microsoft": "Information"
     }
-  },
-  "KafkaEnabled": true,
-  "Kafka": {
-    "ProducerSettings": {
-      "BootstrapServers": "broker:9092",
-      "AllowAutoCreateTopics": true
-    },
-    "ConsumerSettings": {
-      "BootstrapServers": "broker:9092",
-      "GroupId": "payment-group-id",
-      "AllowAutoCreateTopics": true
-    }
   }
 }

src/Services/Payment/Payment.API/appsettings.json

@@ -20,14 +20,35 @@
   "EventBusRetryCount": 5,
   "KafkaEnabled": true,
   "Kafka": {
-    "ProducerSettings": {
-      "BootstrapServers": "broker:9092",
-      "AllowAutoCreateTopics": true
-    },
-    "ConsumerSettings": {
-      "BootstrapServers": "broker:9092",
-      "GroupId": "payment-group-id",
-      "AllowAutoCreateTopics": true
-    }
+  "AdminSettings": {
+    "GroupId": "payment-group-id",
+    "AllowAutoCreateTopics": true,
+    "BootstrapServers": "pkc-xmzwx.europe-central2.gcp.confluent.cloud:9092",
+    "SslEndpointIdentificationAlgorithm": "Https",
+    "SecurityProtocol": "SaslSsl",
+    "SaslMechanism": "Plain",
+    "SaslUsername": "username",
+    "SaslPassword": "password"
+  },
+  "ProducerSettings": {
+    "GroupId": "payment-group-id",
+    "AllowAutoCreateTopics": true,
+    "BootstrapServers": "pkc-xmzwx.europe-central2.gcp.confluent.cloud:9092",
+    "SslEndpointIdentificationAlgorithm": "Https",
+    "SecurityProtocol": "SaslSsl",
+    "SaslMechanism": "Plain",
+    "SaslUsername": "username",
+    "SaslPassword": "password"
+  },
+  "ConsumerSettings": {
+    "GroupId": "payment-group-id",
+    "BootstrapServers": "pkc-xmzwx.europe-central2.gcp.confluent.cloud:9092",
+    "AutoOffsetReset": "Earliest",
+    "SslEndpointIdentificationAlgorithm": "Https",
+    "SecurityProtocol": "SaslSsl",
+    "SaslMechanism": "Plain",
+    "SaslUsername": "username",
+    "SaslPassword": "password"
+  }
   }
 }

src/docker-compose.override.yml

@@ -34,30 +34,30 @@ services:
     volumes:
       - eshop-basketdata:/data
     
-  rabbitmq:
-    ports:
-      - "15672:15672"
-      - "5672:5672"
+  # rabbitmq:
+  #   ports:
+  #     - "15672:15672"
+  #     - "5672:5672"
         
-  zookeeper:
-    environment:
-      ZOOKEEPER_CLIENT_PORT: 2181
-      ZOOKEEPER_TICK_TIME: 2000
+  # zookeeper:
+  #   environment:
+  #     ZOOKEEPER_CLIENT_PORT: 2181
+  #     ZOOKEEPER_TICK_TIME: 2000
 
-  broker:
-    environment:
-      KAFKA_BROKER_ID: 1
-      KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
-      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
-      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker:9092,PLAINTEXT_HOST://localhost:29092
-      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
-      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
-      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
-    ports:
-      # To learn about configuring Kafka for access across networks see
-      # https://www.confluent.io/blog/kafka-client-cannot-connect-to-broker-on-aws-on-docker-etc/
-      - "9092:9092"
-      - "29092:29092"
+  # broker:
+  #   environment:
+  #     KAFKA_BROKER_ID: 1
+  #     KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
+  #     KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
+  #     KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker:9092,PLAINTEXT_HOST://localhost:29092
+  #     KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+  #     KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
+  #     KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
+  #   ports:
+  #     # To learn about configuring Kafka for access across networks see
+  #     # https://www.confluent.io/blog/kafka-client-cannot-connect-to-broker-on-aws-on-docker-etc/
+  #     - "9092:9092"
+  #     - "29092:29092"
       
   identity-api:
     environment:

src/docker-compose.yml

@@ -14,16 +14,16 @@ services:
   basketdata:
     image: redis:alpine
 
-  rabbitmq:
-    image: rabbitmq:3-management-alpine
+  # rabbitmq:
+  #   image: rabbitmq:3-management-alpine
     
-  zookeeper:
-    image: confluentinc/cp-zookeeper:7.3.0
+  # zookeeper:
+  #   image: confluentinc/cp-zookeeper:7.3.0
 
-  broker:
-    image: confluentinc/cp-kafka:7.3.0
-    depends_on:
-      - zookeeper  
+  # broker:
+  #   image: confluentinc/cp-kafka:7.3.0
+  #   depends_on:
+  #     - zookeeper  
 
   identity-api:
     image: ${REGISTRY:-eshop}/identity.api:${PLATFORM:-linux}-${TAG:-latest}
@@ -41,8 +41,6 @@ services:
     depends_on:
       - basketdata
       - identity-api
-      - rabbitmq
-      - broker
 
   catalog-api:
     image: ${REGISTRY:-eshop}/catalog.api:${PLATFORM:-linux}-${TAG:-latest}
@@ -51,8 +49,6 @@ services:
       dockerfile: Services/Catalog/Catalog.API/Dockerfile
     depends_on:
       - sqldata
-      - rabbitmq
-      - broker
 
   ordering-api:
     image: ${REGISTRY:-eshop}/ordering.api:${PLATFORM:-linux}-${TAG:-latest}
@@ -61,8 +57,6 @@ services:
       dockerfile: Services/Ordering/Ordering.API/Dockerfile
     depends_on:
       - sqldata
-      - rabbitmq
-      - broker
 
   ordering-backgroundtasks:
     image: ${REGISTRY:-eshop}/ordering.backgroundtasks:${PLATFORM:-linux}-${TAG:-latest}
@@ -71,17 +65,12 @@ services:
       dockerfile: Services/Ordering/Ordering.BackgroundTasks/Dockerfile
     depends_on:
       - sqldata
-      - rabbitmq
-      - broker
 
   payment-api:
     image: ${REGISTRY:-eshop}/payment.api:${PLATFORM:-linux}-${TAG:-latest}
     build:
       context: .
       dockerfile: Services/Payment/Payment.API/Dockerfile
-    depends_on:
-      - rabbitmq
-      - broker
 
 #  webhooks-api:
 #    image: ${REGISTRY:-eshop}/webhooks.api:${PLATFORM:-linux}-${TAG:-latest}