From bb2ca8b9ffae999dbe6d9c80fe7935c71b9e1efe Mon Sep 17 00:00:00 2001 From: Philipp Theyssen Date: Tue, 16 May 2023 19:14:47 +0200 Subject: [PATCH] Change configuration to use hosted kafka --- .../KafkaConsumerBackgroundService.cs | 2 +- .../Basket.API/appsettings.Development.json | 17 +------ .../Basket/Basket.API/appsettings.json | 28 +++++++++--- .../Catalog.API/appsettings.Development.json | 13 ------ .../Catalog/Catalog.API/appsettings.json | 29 ++++++++++-- .../SetPaidOrderStatusCommandHandler.cs | 2 +- .../Ordering/Ordering.API/appsettings.json | 39 ++++++++++++---- .../appsettings.Development.json | 12 ----- .../Ordering.BackgroundTasks/appsettings.json | 31 ++++++++++--- .../Payment.API/appsettings.Development.json | 12 ----- .../Payment/Payment.API/appsettings.json | 39 ++++++++++++---- src/docker-compose.override.yml | 44 +++++++++---------- src/docker-compose.yml | 27 ++++-------- 13 files changed, 167 insertions(+), 128 deletions(-) diff --git a/src/BuildingBlocks/EventBus/EventBusKafka/KafkaConsumerBackgroundService.cs b/src/BuildingBlocks/EventBus/EventBusKafka/KafkaConsumerBackgroundService.cs index 473c3a0a2..2439a5f77 100644 --- a/src/BuildingBlocks/EventBus/EventBusKafka/KafkaConsumerBackgroundService.cs +++ b/src/BuildingBlocks/EventBus/EventBusKafka/KafkaConsumerBackgroundService.cs @@ -48,7 +48,7 @@ public class KafkaConsumerBackgroundService : BackgroundService if (!_subsManager.HasSubscriptionsForEvent(eventName)) { - _logger.LogWarning("No subscription for Kafka event: {EventName}", eventName); + // _logger.LogWarning("No subscription for Kafka event: {EventName}", eventName); continue; } diff --git a/src/Services/Basket/Basket.API/appsettings.Development.json b/src/Services/Basket/Basket.API/appsettings.Development.json index baacbecbe..f4a3b9407 100644 --- a/src/Services/Basket/Basket.API/appsettings.Development.json +++ b/src/Services/Basket/Basket.API/appsettings.Development.json @@ -13,20 +13,5 @@ "IdentityUrl": "http://localhost:5105", "ConnectionString": "127.0.0.1", "AzureServiceBusEnabled": false, - "EventBusConnection": "localhost", - "KafkaEnabled": true, - "Kafka": { - "AdminSettings": { - "BootstrapServers": "broker:9092" - }, - "ProducerSettings": { - "BootstrapServers": "broker:9092", - "AllowAutoCreateTopics": true - }, - "ConsumerSettings": { - "BootstrapServers": "broker:9092", - "GroupId": "basket-group-id", - "AllowAutoCreateTopics": true - } - } + "EventBusConnection": "localhost" } \ No newline at end of file diff --git a/src/Services/Basket/Basket.API/appsettings.json b/src/Services/Basket/Basket.API/appsettings.json index bc2b0bc8c..399eb14d4 100644 --- a/src/Services/Basket/Basket.API/appsettings.json +++ b/src/Services/Basket/Basket.API/appsettings.json @@ -30,16 +30,34 @@ "KafkaEnabled": true, "Kafka": { "AdminSettings": { - "BootstrapServers": "broker:9092" + "GroupId": "basket-group-id", + "AllowAutoCreateTopics": true, + "BootstrapServers": "pkc-xmzwx.europe-central2.gcp.confluent.cloud:9092", + "SslEndpointIdentificationAlgorithm": "Https", + "SecurityProtocol": "SaslSsl", + "SaslMechanism": "Plain", + "SaslUsername": "username", + "SaslPassword": "password" }, "ProducerSettings": { - "BootstrapServers": "broker:9092", - "AllowAutoCreateTopics": true + "GroupId": "basket-group-id", + "AllowAutoCreateTopics": true, + "BootstrapServers": "pkc-xmzwx.europe-central2.gcp.confluent.cloud:9092", + "SslEndpointIdentificationAlgorithm": "Https", + "SecurityProtocol": "SaslSsl", + "SaslMechanism": "Plain", + "SaslUsername": "username", + "SaslPassword": "password" }, "ConsumerSettings": { - "BootstrapServers": "broker:9092", "GroupId": "basket-group-id", - "AllowAutoCreateTopics": true + "BootstrapServers": "pkc-xmzwx.europe-central2.gcp.confluent.cloud:9092", + "AutoOffsetReset": "Earliest", + "SslEndpointIdentificationAlgorithm": "Https", + "SecurityProtocol": "SaslSsl", + "SaslMechanism": "Plain", + "SaslUsername": "username", + "SaslPassword": "password" } } } diff --git a/src/Services/Catalog/Catalog.API/appsettings.Development.json b/src/Services/Catalog/Catalog.API/appsettings.Development.json index eaf1ae0b9..98fff05dc 100644 --- a/src/Services/Catalog/Catalog.API/appsettings.Development.json +++ b/src/Services/Catalog/Catalog.API/appsettings.Development.json @@ -10,18 +10,5 @@ "System": "Warning" } } - }, - "EventBusConnection": "localhost", - "KafkaEnabled": true, - "Kafka": { - "ProducerSettings": { - "BootstrapServers": "broker:9092", - "AllowAutoCreateTopics": true - }, - "ConsumerSettings": { - "BootstrapServers": "broker:9092", - "GroupId": "catalog-group-id", - "AllowAutoCreateTopics": true - } } } \ No newline at end of file diff --git a/src/Services/Catalog/Catalog.API/appsettings.json b/src/Services/Catalog/Catalog.API/appsettings.json index 14288ccc4..ae56e81c6 100644 --- a/src/Services/Catalog/Catalog.API/appsettings.json +++ b/src/Services/Catalog/Catalog.API/appsettings.json @@ -27,14 +27,35 @@ }, "KafkaEnabled": true, "Kafka": { + "AdminSettings": { + "GroupId": "catalog-group-id", + "AllowAutoCreateTopics": true, + "BootstrapServers": "pkc-xmzwx.europe-central2.gcp.confluent.cloud:9092", + "SslEndpointIdentificationAlgorithm": "Https", + "SecurityProtocol": "SaslSsl", + "SaslMechanism": "Plain", + "SaslUsername": "username", + "SaslPassword": "password" + }, "ProducerSettings": { - "BootstrapServers": "broker:9092", - "AllowAutoCreateTopics": true + "GroupId": "catalog-group-id", + "AllowAutoCreateTopics": true, + "BootstrapServers": "pkc-xmzwx.europe-central2.gcp.confluent.cloud:9092", + "SslEndpointIdentificationAlgorithm": "Https", + "SecurityProtocol": "SaslSsl", + "SaslMechanism": "Plain", + "SaslUsername": "username", + "SaslPassword": "password" }, "ConsumerSettings": { - "BootstrapServers": "broker:9092", "GroupId": "catalog-group-id", - "AllowAutoCreateTopics": true + "BootstrapServers": "pkc-xmzwx.europe-central2.gcp.confluent.cloud:9092", + "AutoOffsetReset": "Earliest", + "SslEndpointIdentificationAlgorithm": "Https", + "SecurityProtocol": "SaslSsl", + "SaslMechanism": "Plain", + "SaslUsername": "username", + "SaslPassword": "password" } } } diff --git a/src/Services/Ordering/Ordering.API/Application/Commands/SetPaidOrderStatusCommandHandler.cs b/src/Services/Ordering/Ordering.API/Application/Commands/SetPaidOrderStatusCommandHandler.cs index 09520a65f..9bcb42cd6 100644 --- a/src/Services/Ordering/Ordering.API/Application/Commands/SetPaidOrderStatusCommandHandler.cs +++ b/src/Services/Ordering/Ordering.API/Application/Commands/SetPaidOrderStatusCommandHandler.cs @@ -19,7 +19,7 @@ public class SetPaidOrderStatusCommandHandler : IRequestHandler Handle(SetPaidOrderStatusCommand command, CancellationToken cancellationToken) { // Simulate a work time for validating the payment - await Task.Delay(10000, cancellationToken); + await Task.Delay(2000, cancellationToken); var orderToUpdate = await _orderRepository.GetAsync(command.OrderNumber); if (orderToUpdate == null) diff --git a/src/Services/Ordering/Ordering.API/appsettings.json b/src/Services/Ordering/Ordering.API/appsettings.json index d57f795e6..cfa203696 100644 --- a/src/Services/Ordering/Ordering.API/appsettings.json +++ b/src/Services/Ordering/Ordering.API/appsettings.json @@ -25,15 +25,36 @@ "EventBusConnection": "localhost", "KafkaEnabled": true, "Kafka": { - "ProducerSettings": { - "BootstrapServers": "broker:9092", - "AllowAutoCreateTopics": true - }, - "ConsumerSettings": { - "BootstrapServers": "broker:9092", - "GroupId": "ordering-group-id", - "AllowAutoCreateTopics": true - } + "AdminSettings": { + "GroupId": "ordering-group-id", + "AllowAutoCreateTopics": true, + "BootstrapServers": "pkc-xmzwx.europe-central2.gcp.confluent.cloud:9092", + "SslEndpointIdentificationAlgorithm": "Https", + "SecurityProtocol": "SaslSsl", + "SaslMechanism": "Plain", + "SaslUsername": "username", + "SaslPassword": "password" + }, + "ProducerSettings": { + "GroupId": "ordering-group-id", + "AllowAutoCreateTopics": true, + "BootstrapServers": "pkc-xmzwx.europe-central2.gcp.confluent.cloud:9092", + "SslEndpointIdentificationAlgorithm": "Https", + "SecurityProtocol": "SaslSsl", + "SaslMechanism": "Plain", + "SaslUsername": "username", + "SaslPassword": "password" + }, + "ConsumerSettings": { + "GroupId": "ordering-group-id", + "BootstrapServers": "pkc-xmzwx.europe-central2.gcp.confluent.cloud:9092", + "AutoOffsetReset": "Earliest", + "SslEndpointIdentificationAlgorithm": "Https", + "SecurityProtocol": "SaslSsl", + "SaslMechanism": "Plain", + "SaslUsername": "username", + "SaslPassword": "password" + } }, "UseVault": false, "Vault": { diff --git a/src/Services/Ordering/Ordering.BackgroundTasks/appsettings.Development.json b/src/Services/Ordering/Ordering.BackgroundTasks/appsettings.Development.json index 3085e5372..e203e9407 100644 --- a/src/Services/Ordering/Ordering.BackgroundTasks/appsettings.Development.json +++ b/src/Services/Ordering/Ordering.BackgroundTasks/appsettings.Development.json @@ -5,17 +5,5 @@ "System": "Information", "Microsoft": "Information" } - }, - "KafkaEnabled": true, - "Kafka": { - "ProducerSettings": { - "BootstrapServers": "broker:9092", - "AllowAutoCreateTopics": true - }, - "ConsumerSettings": { - "BootstrapServers": "broker:9092", - "GroupId": "ordering-background-group-id", - "AllowAutoCreateTopics": true - } } } diff --git a/src/Services/Ordering/Ordering.BackgroundTasks/appsettings.json b/src/Services/Ordering/Ordering.BackgroundTasks/appsettings.json index f156b527a..0305471ac 100644 --- a/src/Services/Ordering/Ordering.BackgroundTasks/appsettings.json +++ b/src/Services/Ordering/Ordering.BackgroundTasks/appsettings.json @@ -25,14 +25,35 @@ "EventBusPassword": "", "KafkaEnabled": true, "Kafka": { + "AdminSettings": { + "GroupId": "ordering-background-group-id", + "AllowAutoCreateTopics": true, + "BootstrapServers": "pkc-xmzwx.europe-central2.gcp.confluent.cloud:9092", + "SslEndpointIdentificationAlgorithm": "Https", + "SecurityProtocol": "SaslSsl", + "SaslMechanism": "Plain", + "SaslUsername": "username", + "SaslPassword": "MMn/CFO53yA631L8/EW1iXbmVWzN/3DZgJgWfQDBTreEgDhp4ica8SyaDqE9iP7r" + }, "ProducerSettings": { - "BootstrapServers": "broker:9092", - "AllowAutoCreateTopics": true + "GroupId": "ordering-background-group-id", + "AllowAutoCreateTopics": true, + "BootstrapServers": "pkc-xmzwx.europe-central2.gcp.confluent.cloud:9092", + "SslEndpointIdentificationAlgorithm": "Https", + "SecurityProtocol": "SaslSsl", + "SaslMechanism": "Plain", + "SaslUsername": "username", + "SaslPassword": "MMn/CFO53yA631L8/EW1iXbmVWzN/3DZgJgWfQDBTreEgDhp4ica8SyaDqE9iP7r" }, "ConsumerSettings": { - "BootstrapServers": "broker:9092", "GroupId": "ordering-background-group-id", - "AllowAutoCreateTopics": true + "BootstrapServers": "pkc-xmzwx.europe-central2.gcp.confluent.cloud:9092", + "AutoOffsetReset": "Earliest", + "SslEndpointIdentificationAlgorithm": "Https", + "SecurityProtocol": "SaslSsl", + "SaslMechanism": "Plain", + "SaslUsername": "username", + "SaslPassword": "MMn/CFO53yA631L8/EW1iXbmVWzN/3DZgJgWfQDBTreEgDhp4ica8SyaDqE9iP7r" } } -} \ No newline at end of file +} diff --git a/src/Services/Payment/Payment.API/appsettings.Development.json b/src/Services/Payment/Payment.API/appsettings.Development.json index c05c7149b..fa8ce71a9 100644 --- a/src/Services/Payment/Payment.API/appsettings.Development.json +++ b/src/Services/Payment/Payment.API/appsettings.Development.json @@ -6,17 +6,5 @@ "System": "Information", "Microsoft": "Information" } - }, - "KafkaEnabled": true, - "Kafka": { - "ProducerSettings": { - "BootstrapServers": "broker:9092", - "AllowAutoCreateTopics": true - }, - "ConsumerSettings": { - "BootstrapServers": "broker:9092", - "GroupId": "payment-group-id", - "AllowAutoCreateTopics": true - } } } diff --git a/src/Services/Payment/Payment.API/appsettings.json b/src/Services/Payment/Payment.API/appsettings.json index 6e6b0abd6..1dac8d44c 100644 --- a/src/Services/Payment/Payment.API/appsettings.json +++ b/src/Services/Payment/Payment.API/appsettings.json @@ -20,14 +20,35 @@ "EventBusRetryCount": 5, "KafkaEnabled": true, "Kafka": { - "ProducerSettings": { - "BootstrapServers": "broker:9092", - "AllowAutoCreateTopics": true - }, - "ConsumerSettings": { - "BootstrapServers": "broker:9092", - "GroupId": "payment-group-id", - "AllowAutoCreateTopics": true - } + "AdminSettings": { + "GroupId": "payment-group-id", + "AllowAutoCreateTopics": true, + "BootstrapServers": "pkc-xmzwx.europe-central2.gcp.confluent.cloud:9092", + "SslEndpointIdentificationAlgorithm": "Https", + "SecurityProtocol": "SaslSsl", + "SaslMechanism": "Plain", + "SaslUsername": "username", + "SaslPassword": "password" + }, + "ProducerSettings": { + "GroupId": "payment-group-id", + "AllowAutoCreateTopics": true, + "BootstrapServers": "pkc-xmzwx.europe-central2.gcp.confluent.cloud:9092", + "SslEndpointIdentificationAlgorithm": "Https", + "SecurityProtocol": "SaslSsl", + "SaslMechanism": "Plain", + "SaslUsername": "username", + "SaslPassword": "password" + }, + "ConsumerSettings": { + "GroupId": "payment-group-id", + "BootstrapServers": "pkc-xmzwx.europe-central2.gcp.confluent.cloud:9092", + "AutoOffsetReset": "Earliest", + "SslEndpointIdentificationAlgorithm": "Https", + "SecurityProtocol": "SaslSsl", + "SaslMechanism": "Plain", + "SaslUsername": "username", + "SaslPassword": "password" + } } } diff --git a/src/docker-compose.override.yml b/src/docker-compose.override.yml index c4d53d585..d39da3fe3 100644 --- a/src/docker-compose.override.yml +++ b/src/docker-compose.override.yml @@ -34,30 +34,30 @@ services: volumes: - eshop-basketdata:/data - rabbitmq: - ports: - - "15672:15672" - - "5672:5672" + # rabbitmq: + # ports: + # - "15672:15672" + # - "5672:5672" - zookeeper: - environment: - ZOOKEEPER_CLIENT_PORT: 2181 - ZOOKEEPER_TICK_TIME: 2000 + # zookeeper: + # environment: + # ZOOKEEPER_CLIENT_PORT: 2181 + # ZOOKEEPER_TICK_TIME: 2000 - broker: - environment: - KAFKA_BROKER_ID: 1 - KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181' - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker:9092,PLAINTEXT_HOST://localhost:29092 - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 - KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 - KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 - ports: - # To learn about configuring Kafka for access across networks see - # https://www.confluent.io/blog/kafka-client-cannot-connect-to-broker-on-aws-on-docker-etc/ - - "9092:9092" - - "29092:29092" + # broker: + # environment: + # KAFKA_BROKER_ID: 1 + # KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181' + # KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT + # KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker:9092,PLAINTEXT_HOST://localhost:29092 + # KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + # KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 + # KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 + # ports: + # # To learn about configuring Kafka for access across networks see + # # https://www.confluent.io/blog/kafka-client-cannot-connect-to-broker-on-aws-on-docker-etc/ + # - "9092:9092" + # - "29092:29092" identity-api: environment: diff --git a/src/docker-compose.yml b/src/docker-compose.yml index 8600e4612..a2b629bc4 100644 --- a/src/docker-compose.yml +++ b/src/docker-compose.yml @@ -14,16 +14,16 @@ services: basketdata: image: redis:alpine - rabbitmq: - image: rabbitmq:3-management-alpine + # rabbitmq: + # image: rabbitmq:3-management-alpine - zookeeper: - image: confluentinc/cp-zookeeper:7.3.0 + # zookeeper: + # image: confluentinc/cp-zookeeper:7.3.0 - broker: - image: confluentinc/cp-kafka:7.3.0 - depends_on: - - zookeeper + # broker: + # image: confluentinc/cp-kafka:7.3.0 + # depends_on: + # - zookeeper identity-api: image: ${REGISTRY:-eshop}/identity.api:${PLATFORM:-linux}-${TAG:-latest} @@ -41,8 +41,6 @@ services: depends_on: - basketdata - identity-api - - rabbitmq - - broker catalog-api: image: ${REGISTRY:-eshop}/catalog.api:${PLATFORM:-linux}-${TAG:-latest} @@ -51,8 +49,6 @@ services: dockerfile: Services/Catalog/Catalog.API/Dockerfile depends_on: - sqldata - - rabbitmq - - broker ordering-api: image: ${REGISTRY:-eshop}/ordering.api:${PLATFORM:-linux}-${TAG:-latest} @@ -61,8 +57,6 @@ services: dockerfile: Services/Ordering/Ordering.API/Dockerfile depends_on: - sqldata - - rabbitmq - - broker ordering-backgroundtasks: image: ${REGISTRY:-eshop}/ordering.backgroundtasks:${PLATFORM:-linux}-${TAG:-latest} @@ -71,17 +65,12 @@ services: dockerfile: Services/Ordering/Ordering.BackgroundTasks/Dockerfile depends_on: - sqldata - - rabbitmq - - broker payment-api: image: ${REGISTRY:-eshop}/payment.api:${PLATFORM:-linux}-${TAG:-latest} build: context: . dockerfile: Services/Payment/Payment.API/Dockerfile - depends_on: - - rabbitmq - - broker # webhooks-api: # image: ${REGISTRY:-eshop}/webhooks.api:${PLATFORM:-linux}-${TAG:-latest}