From a1f3a60ef919fa12a5957d0baef80f3cc6845ae3 Mon Sep 17 00:00:00 2001 From: Charles Lowell Date: Thu, 16 Feb 2017 10:49:58 -0800 Subject: [PATCH] Configuration and deploy script for k8s on ACS --- k8s/README.md | 26 +++++ k8s/deploy.ps1 | 80 ++++++++++++++ k8s/deployments.yaml | 242 +++++++++++++++++++++++++++++++++++++++++++ k8s/frontend.yaml | 47 +++++++++ k8s/nginx.conf | 74 +++++++++++++ k8s/services.yaml | 83 +++++++++++++++ k8s/sql-data.yaml | 33 ++++++ 7 files changed, 585 insertions(+) create mode 100644 k8s/README.md create mode 100644 k8s/deploy.ps1 create mode 100644 k8s/deployments.yaml create mode 100644 k8s/frontend.yaml create mode 100644 k8s/nginx.conf create mode 100644 k8s/services.yaml create mode 100644 k8s/sql-data.yaml diff --git a/k8s/README.md b/k8s/README.md new file mode 100644 index 000000000..057542983 --- /dev/null +++ b/k8s/README.md @@ -0,0 +1,26 @@ +# eShopOnContainers on Kubernetes +This directory contains Kubernetes configuration for the eShopOnContainers app and a PowerShell script to deploy it to a cluster. Each microservice has a deployment configuration in `deployments.yaml`, and is exposed to the cluster by a service in `services.yaml`. The microservices are exposed externally on individual routes (`/basket-api`, `/webmvc`, etc.) by an nginx reverse proxy as specified in `frontend.yaml` and `nginx.conf`. + +## Deploying the application +### Prerequisites +* A Docker build host. +* A private Docker registry. Follow Azure Container Registry's [guide](https://docs.microsoft.com/en-us/azure/container-registry/container-registry-get-started-portal) to create one. +* A Kubernetes cluster. Follow Azure Container Service's [walkthrough](https://docs.microsoft.com/en-us/azure/container-service/container-service-kubernetes-walkthrough) to create one. + +### Run the deployment script +1. Open a PowerShell command line at `eShopOnContainers/k8s`. +1. Ensure `docker`, `docker-compose`, and `kubectl` are on the path, and configured for your Docker machine and Kubernetes cluster. +1. Run `deploy.ps1` with your registry information. For example: + ``` + ./deploy.ps1 -registry myregistry.azurecr.io -dockerUser User -dockerPassword SecretPassword + ``` + The Docker username and password are provided by Azure Container Registry, and can be retrieved from the Azure portal. + +The script will build the code and corresponding Docker images, push the latter to your registry, and deploy the application to your Kubernetes cluster. + +TODOs +===== +* Host WebSPA at `/webspa` + * This is blocked on correct relative URLs for images. Presently these are set at build by webpack, which isn't aware of where the app will be sited. An Angular solution might exist. Another option is to encode the images in base64. +* Debug microservice resiliency issues--some microservices can enter failure states requiring their pod to be recreated. +* Respond to `kubectl` failures in `deploy.ps1`. \ No newline at end of file diff --git a/k8s/deploy.ps1 b/k8s/deploy.ps1 new file mode 100644 index 000000000..0c5f7a579 --- /dev/null +++ b/k8s/deploy.ps1 @@ -0,0 +1,80 @@ +Param( + [parameter(Mandatory=$true)][string]$registry, + [parameter(Mandatory=$true)][string]$dockerUser, + [parameter(Mandatory=$true)][string]$dockerPassword +) + +$requiredCommands = ("docker", "docker-compose", "kubectl") +foreach ($command in $requiredCommands) { + if ((Get-Command $command -ErrorAction SilentlyContinue) -eq $null) { + Write-Host "$command must be on path" -ForegroundColor Red + exit + } +} + +Write-Host "Logging in to $registry" -ForegroundColor Yellow +docker login -u $dockerUser -p $dockerPassword $registry +if (-not $LastExitCode -eq 0) { + Write-Host "Login failed" -ForegroundColor Red + exit +} + +# create registry key secret +kubectl create secret docker-registry registry-key ` + --docker-server=$registry ` + --docker-username=$dockerUser ` + --docker-password=$dockerPassword ` + --docker-email=not@used.com + +# start sql and frontend deployments +kubectl create configmap config-files --from-file=nginx-conf=nginx.conf +kubectl label configmap config-files app=eshop +kubectl create -f sql-data.yaml -f services.yaml -f frontend.yaml + +Write-Host "Building solution..." -ForegroundColor Yellow +../cli-windows/build-bits-simple.ps1 + +Write-Host "Building Docker images..." -ForegroundColor Yellow +docker-compose -p .. -f ../docker-compose.yml build + +Write-Host "Pushing images to $registry..." -ForegroundColor Yellow +$services = ("basket.api", "catalog.api", "identity.api", "ordering.api", "webmvc", "webspa") +foreach ($service in $services) { + docker tag eshop/$service $registry/$service + docker push $registry/$service +} + +Write-Host "Waiting for frontend's external ip..." -ForegroundColor Yellow +while ($true) { + $frontendUrl = kubectl get svc frontend -o=jsonpath="{.status.loadBalancer.ingress[0].ip}" 2> $_ + if ([bool]($frontendUrl -as [ipaddress])) { + break + } + Start-Sleep -s 15 +} + +kubectl create configmap urls ` + --from-literal=BasketUrl=http://$($frontendUrl)/basket-api ` + --from-literal=CatalogUrl=http://$($frontendUrl)/catalog-api ` + --from-literal=IdentityUrl=http://$($frontendUrl)/identity ` + --from-literal=OrderingUrl=http://$($frontendUrl)/ordering-api ` + --from-literal=MvcClient=http://$($frontendUrl)/webmvc ` + --from-literal=SpaClient=http://$($frontendUrl) +kubectl label configmap urls app=eshop + +# TODO verify database readiness? +Write-Host "Creating deployments..." +kubectl apply -f deployments.yaml + +# update deployments with the private registry +# (deployment templating, or Helm, would obviate this) +kubectl set image -f deployments.yaml ` + basket=$registry/basket.api ` + catalog=$registry/catalog.api ` + identity=$registry/identity.api ` + ordering=$registry/ordering.api ` + webmvc=$registry/webmvc ` + webspa=$registry/webspa +kubectl rollout resume -f deployments.yaml + +Write-Host "WebSPA is exposed at http://$frontendUrl, WebMVC at http://$frontendUrl/webmvc" -ForegroundColor Yellow diff --git a/k8s/deployments.yaml b/k8s/deployments.yaml new file mode 100644 index 000000000..05c860dec --- /dev/null +++ b/k8s/deployments.yaml @@ -0,0 +1,242 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: basket +spec: + paused: true + template: + metadata: + labels: + app: eshop + component: basket + spec: + containers: + - name: basket + image: eshop/basket.api + imagePullPolicy: Always + env: + - name: ASPNETCORE_ENVIRONMENT + value: Development + - name: ASPNETCORE_URLS + value: http://0.0.0.0:80/basket-api + - name: ConnectionString + value: 127.0.0.1 + - name: IdentityUrl + valueFrom: + configMapKeyRef: + name: urls + key: IdentityUrl + ports: + - containerPort: 80 + - name: basket-data + image: redis + ports: + - containerPort: 6379 + imagePullSecrets: + - name: registry-key +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: catalog +spec: + paused: true + template: + metadata: + labels: + app: eshop + component: catalog + spec: + containers: + - name: catalog + image: eshop/catalog.api + imagePullPolicy: Always + env: + - name: ASPNETCORE_ENVIRONMENT + value: Development + - name: ASPNETCORE_URLS + value: http://0.0.0.0:80/catalog-api + - name: ConnectionString + value: "Server=sql-data;Initial Catalog=Microsoft.eShopOnContainers.Services.CatalogDb;User Id=sa;Password=Pass@word" + - name: ExternalCatalogBaseUrl + valueFrom: + configMapKeyRef: + name: urls + key: CatalogUrl + ports: + - containerPort: 80 + imagePullSecrets: + - name: registry-key +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: identity +spec: + paused: true + template: + metadata: + labels: + app: eshop + component: identity + spec: + containers: + - name: identity + image: eshop/identity.api + imagePullPolicy: Always + env: + - name: ASPNETCORE_ENVIRONMENT + value: Development + - name: ASPNETCORE_URLS + value: http://0.0.0.0:80/identity + - name: ConnectionStrings__DefaultConnection + value: "Server=sql-data;Initial Catalog=Microsoft.eShopOnContainers.Services.CatalogDb;User Id=sa;Password=Pass@word" + - name: MvcClient + valueFrom: + configMapKeyRef: + name: urls + key: MvcClient + - name: SpaClient + valueFrom: + configMapKeyRef: + name: urls + key: SpaClient + ports: + - containerPort: 80 + imagePullSecrets: + - name: registry-key +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: ordering +spec: + paused: true + template: + metadata: + labels: + app: eshop + component: ordering + spec: + containers: + - name: ordering + image: eshop/ordering.api + imagePullPolicy: Always + env: + - name: ASPNETCORE_ENVIRONMENT + value: Development + - name: ASPNETCORE_URLS + value: http://0.0.0.0:80/ordering-api + - name: ConnectionString + value: "Server=sql-data;Database=Microsoft.eShopOnContainers.Services.OrderingDb;User Id=sa;Password=Pass@word;" + - name: IdentityUrl + valueFrom: + configMapKeyRef: + name: urls + key: IdentityUrl + ports: + - containerPort: 80 + imagePullSecrets: + - name: registry-key +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: webmvc +spec: + paused: true + template: + metadata: + labels: + app: eshop + component: webmvc + spec: + containers: + - name: webmvc + image: eshop/webmvc + imagePullPolicy: Always + env: + - name: ASPNETCORE_ENVIRONMENT + value: Development + - name: ASPNETCORE_URLS + value: http://0.0.0.0:80/webmvc + - name: BasketUrl + valueFrom: + configMapKeyRef: + name: urls + key: BasketUrl + - name: CallBackUrl + valueFrom: + configMapKeyRef: + name: urls + key: MvcClient + - name: CatalogUrl + valueFrom: + configMapKeyRef: + name: urls + key: CatalogUrl + - name: IdentityUrl + valueFrom: + configMapKeyRef: + name: urls + key: IdentityUrl + - name: OrderingUrl + valueFrom: + configMapKeyRef: + name: urls + key: OrderingUrl + ports: + - containerPort: 80 + imagePullSecrets: + - name: registry-key +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: webspa +spec: + paused: true + template: + metadata: + labels: + app: eshop + component: webspa + spec: + containers: + - name: webspa + image: eshop/webspa + imagePullPolicy: Always + env: + - name: ASPNETCORE_ENVIRONMENT + value: Development + - name: ASPNETCORE_URLS + value: http://0.0.0.0:80 + - name: BasketUrl + valueFrom: + configMapKeyRef: + name: urls + key: BasketUrl + - name: CallBackUrl + valueFrom: + configMapKeyRef: + name: urls + key: SpaClient + - name: CatalogUrl + valueFrom: + configMapKeyRef: + name: urls + key: CatalogUrl + - name: IdentityUrl + valueFrom: + configMapKeyRef: + name: urls + key: IdentityUrl + - name: OrderingUrl + valueFrom: + configMapKeyRef: + name: urls + key: OrderingUrl + ports: + - containerPort: 80 + imagePullSecrets: + - name: registry-key diff --git a/k8s/frontend.yaml b/k8s/frontend.yaml new file mode 100644 index 000000000..291542404 --- /dev/null +++ b/k8s/frontend.yaml @@ -0,0 +1,47 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app: eshop + component: frontend + name: frontend +spec: + ports: + - port: 80 + targetPort: 8080 + selector: + app: eshop + component: frontend + type: LoadBalancer +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: frontend +spec: + template: + metadata: + labels: + app: eshop + component: frontend + spec: + containers: + - name: nginx + image: nginx:1.11.10-alpine + imagePullPolicy: IfNotPresent + ports: + - containerPort: 8080 + lifecycle: + preStop: + exec: + command: ["/usr/sbin/nginx","-s","quit"] + volumeMounts: + - name: config + mountPath: /etc/nginx + volumes: + - name: config + configMap: + name: config-files + items: + - key: nginx-conf + path: nginx.conf diff --git a/k8s/nginx.conf b/k8s/nginx.conf new file mode 100644 index 000000000..33526d486 --- /dev/null +++ b/k8s/nginx.conf @@ -0,0 +1,74 @@ +pid /tmp/nginx.pid; + +worker_processes 1; + +events { + worker_connections 1024; +} + +http { + server_tokens off; + + add_header X-Frame-Options SAMEORIGIN; + add_header X-Content-Type-Options nosniff; + add_header X-XSS-Protection "1; mode=block"; + + client_body_temp_path /tmp/client_body; + fastcgi_temp_path /tmp/fastcgi_temp; + proxy_temp_path /tmp/proxy_temp; + scgi_temp_path /tmp/scgi_temp; + uwsgi_temp_path /tmp/uwsgi_temp; + + gzip on; + gzip_comp_level 6; + gzip_min_length 1024; + gzip_buffers 4 32k; + gzip_types text/plain application/javascript text/css; + gzip_vary on; + + keepalive_timeout 65; + + proxy_buffer_size 128k; + proxy_buffers 4 256k; + proxy_busy_buffers_size 256k; + + server { + listen 8080; + + location /basket-api { + proxy_pass http://basket; + proxy_redirect off; + proxy_set_header Host $host; + } + + location /catalog-api { + proxy_pass http://catalog; + proxy_redirect off; + proxy_set_header Host $host; + } + + location /identity { + proxy_pass http://identity; + proxy_redirect off; + proxy_set_header Host $host; + } + + location /ordering-api { + proxy_pass http://ordering; + proxy_redirect off; + proxy_set_header Host $host; + } + + location /webmvc { + proxy_pass http://webmvc; + proxy_redirect off; + proxy_set_header Host $host; + } + + location / { + proxy_pass http://webspa; + proxy_redirect off; + proxy_set_header Host $host; + } + } +} \ No newline at end of file diff --git a/k8s/services.yaml b/k8s/services.yaml new file mode 100644 index 000000000..fda3e5ea3 --- /dev/null +++ b/k8s/services.yaml @@ -0,0 +1,83 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app: eshop + component: basket + name: basket +spec: + ports: + - port: 80 + selector: + app: eshop + component: basket +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: eshop + component: catalog + name: catalog +spec: + ports: + - port: 80 + selector: + app: eshop + component: catalog +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: eshop + component: identity + name: identity +spec: + ports: + - port: 80 + selector: + app: eshop + component: identity +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: eshop + component: ordering + name: ordering +spec: + ports: + - port: 80 + selector: + app: eshop + component: ordering +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: eshop + component: webmvc + name: webmvc +spec: + ports: + - port: 80 + selector: + app: eshop + component: webmvc +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: eshop + component: webspa + name: webspa +spec: + ports: + - port: 80 + selector: + app: eshop + component: webspa \ No newline at end of file diff --git a/k8s/sql-data.yaml b/k8s/sql-data.yaml new file mode 100644 index 000000000..6edcd21bc --- /dev/null +++ b/k8s/sql-data.yaml @@ -0,0 +1,33 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app: eshop + component: sql-data + name: sql-data +spec: + ports: + - port: 1433 + selector: + app: eshop + component: sql-data +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: sql-data +spec: + template: + metadata: + labels: + app: eshop + component: sql-data + spec: + containers: + - name: sql-data + image: microsoft/mssql-server-linux:ctp1-3 + env: + - name: ACCEPT_EULA + value: "Y" + - name: SA_PASSWORD + value: Pass@word