From eb742bb66244f411c8558c69278952f1a7dda976 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eduard=20Tom=C3=A0s?= Date: Wed, 10 Jan 2018 12:48:23 +0100 Subject: [PATCH 1/2] Support for ingress controller using nginx-ingress --- k8s/deploy-ingress-azure.ps1 | 3 + k8s/deploy-ingress.ps1 | 12 +++ k8s/deploy.ps1 | 32 +++--- k8s/ingress.yaml | 55 +++++++++++ k8s/nginx-ingress/azure/service.yaml | 19 ++++ k8s/nginx-ingress/configmap.yaml | 11 +++ k8s/nginx-ingress/default-backend.yaml | 52 ++++++++++ k8s/nginx-ingress/namespace.yaml | 4 + .../patch-service-without-rbac.yaml | 40 ++++++++ k8s/nginx-ingress/publish-service-patch.yaml | 7 ++ k8s/nginx-ingress/tcp-services-configmap.yaml | 5 + k8s/nginx-ingress/udp-services-configmap.yaml | 5 + k8s/nginx-ingress/without-rbac.yaml | 61 ++++++++++++ k8s/nginx.conf | 98 ------------------- 14 files changed, 286 insertions(+), 118 deletions(-) create mode 100644 k8s/deploy-ingress-azure.ps1 create mode 100644 k8s/deploy-ingress.ps1 create mode 100644 k8s/ingress.yaml create mode 100644 k8s/nginx-ingress/azure/service.yaml create mode 100644 k8s/nginx-ingress/configmap.yaml create mode 100644 k8s/nginx-ingress/default-backend.yaml create mode 100644 k8s/nginx-ingress/namespace.yaml create mode 100644 k8s/nginx-ingress/patch-service-without-rbac.yaml create mode 100644 k8s/nginx-ingress/publish-service-patch.yaml create mode 100644 k8s/nginx-ingress/tcp-services-configmap.yaml create mode 100644 k8s/nginx-ingress/udp-services-configmap.yaml create mode 100644 k8s/nginx-ingress/without-rbac.yaml delete mode 100644 k8s/nginx.conf diff --git a/k8s/deploy-ingress-azure.ps1 b/k8s/deploy-ingress-azure.ps1 new file mode 100644 index 000000000..f93cf437b --- /dev/null +++ b/k8s/deploy-ingress-azure.ps1 @@ -0,0 +1,3 @@ +kubectl patch deployment -n ingress-nginx nginx-ingress-controller --type=json --patch="$(cat nginx-ingress\publish-service-patch.yaml)" +kubectl apply -f nginx-ingress\azure\service.yaml +kubectl apply -f nginx-ingress\patch-service-without-rbac.yaml \ No newline at end of file diff --git a/k8s/deploy-ingress.ps1 b/k8s/deploy-ingress.ps1 new file mode 100644 index 000000000..694361bfa --- /dev/null +++ b/k8s/deploy-ingress.ps1 @@ -0,0 +1,12 @@ +kubectl apply -f ingress.yaml + +# Deploy nginx-ingress core files +kubectl apply -f nginx-ingress\namespace.yaml +kubectl apply -f nginx-ingress\default-backend.yaml +kubectl apply -f nginx-ingress\configmap.yaml +kubectl apply -f nginx-ingress\tcp-services-configmap.yaml +kubectl apply -f nginx-ingress\udp-services-configmap.yaml +kubectl apply -f nginx-ingress\without-rbac.yaml + + + diff --git a/k8s/deploy.ps1 b/k8s/deploy.ps1 index bcb2d7393..c2b400b4e 100644 --- a/k8s/deploy.ps1 +++ b/k8s/deploy.ps1 @@ -6,7 +6,6 @@ Param( [parameter(Mandatory=$false)][string]$kubeconfigPath, [parameter(Mandatory=$true)][string]$configFile, [parameter(Mandatory=$false)][string]$imageTag, - [parameter(Mandatory=$false)][string]$externalDns, [parameter(Mandatory=$false)][bool]$deployCI=$false, [parameter(Mandatory=$false)][bool]$buildImages=$true, [parameter(Mandatory=$false)][bool]$buildBits=$false, @@ -30,6 +29,16 @@ function ExecKube($cmd) { $debugMode = $PSCmdlet.MyInvocation.BoundParameters["Debug"].IsPresent $useDockerHub = [string]::IsNullOrEmpty($registry) +$externalDns = & ExecKube -cmd 'get svc ingress-nginx -n ingress-nginx -o=jsonpath="{.status.loadBalancer.ingress[0].ip}"' +Write-Host "Ingress ip detected: $externalDns" -ForegroundColor Yellow + +if (-not [bool]($externalDns -as [ipaddress])) { + Write-Host "Must install ingress first" -ForegroundColor Red + Write-Host "Run deploy-ingress.ps1 and deploy-ingress-azure.ps1" -ForegroundColor Red + exit +} + + # Check required commands (only if not in CI environment) if(-not $deployCI) { $requiredCommands = ("docker", "docker-compose", "kubectl") @@ -100,35 +109,18 @@ if (-not [string]::IsNullOrEmpty($dockerUser)) { Write-Host "Removing existing services & deployments.." -ForegroundColor Yellow ExecKube -cmd 'delete deployments --all' ExecKube -cmd 'delete services --all' -ExecKube -cmd 'delete configmap config-files' ExecKube -cmd 'delete configmap urls' ExecKube -cmd 'delete configmap externalcfg' # start sql, rabbitmq, frontend deployments -ExecKube -cmd 'create configmap config-files --from-file=nginx-conf=nginx.conf' -ExecKube -cmd 'label configmap config-files app=eshop' - if ($deployInfrastructure) { Write-Host 'Deploying infrastructure deployments (databases, redis, RabbitMQ...)' -ForegroundColor Yellow ExecKube -cmd 'create -f sql-data.yaml -f basket-data.yaml -f keystore-data.yaml -f rabbitmq.yaml -f nosql-data.yaml' } -Write-Host 'Deploying code deployments (Web APIs, Web apps, ...)' -ForegroundColor Yellow -ExecKube -cmd 'create -f services.yaml -f frontend.yaml' - -if ([string]::IsNullOrEmpty($externalDns)) { - Write-Host "Waiting for frontend's external ip..." -ForegroundColor Yellow - while ($true) { - $frontendUrl = & ExecKube -cmd 'get svc frontend -o=jsonpath="{.status.loadBalancer.ingress[0].ip}"' - if ([bool]($frontendUrl -as [ipaddress])) { - break - } - Start-Sleep -s 15 - } - $externalDns = $frontendUrl -} -Write-Host "Using $externalDns as the external DNS/IP of the k8s cluster" +Write-Host 'Deploying code deployments (Web APIs, Web apps, ...)' -ForegroundColor Yellow +ExecKube -cmd 'create -f services.yaml' ExecKube -cmd 'create configmap urls ` --from-literal=BasketUrl=http://basket ` diff --git a/k8s/ingress.yaml b/k8s/ingress.yaml new file mode 100644 index 000000000..3c2fe8bad --- /dev/null +++ b/k8s/ingress.yaml @@ -0,0 +1,55 @@ +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + labels: + app: eshop + component: frontend + name: eshop-ingress + annotations: + ingress.kubernetes.io/ssl-redirect: "false" +spec: + rules: + - http: + paths: + - path: /basket-api + backend: + serviceName: basket + servicePort: 80 + - path: /catalog-api + backend: + serviceName: catalog + servicePort: 80 + - path: /identity + backend: + serviceName: identity + servicePort: 80 + - path: /ordering-api + backend: + serviceName: ordering + servicePort: 80 + - path: /webmvc + backend: + serviceName: webmvc + servicePort: 80 + - path: /webstatus + backend: + serviceName: webstatus + servicePort: 80 + - path: /marketing-api + backend: + serviceName: marketing + servicePort: 80 + - path: /payment-api + backend: + serviceName: payment + servicePort: 80 + - path: /locations-api + backend: + serviceName: locations + servicePort: 80 + - path: / + backend: + serviceName: webspa + servicePort: 80 + + diff --git a/k8s/nginx-ingress/azure/service.yaml b/k8s/nginx-ingress/azure/service.yaml new file mode 100644 index 000000000..8d2f71505 --- /dev/null +++ b/k8s/nginx-ingress/azure/service.yaml @@ -0,0 +1,19 @@ +kind: Service +apiVersion: v1 +metadata: + name: ingress-nginx + namespace: ingress-nginx + labels: + app: ingress-nginx +spec: + externalTrafficPolicy: Local + type: LoadBalancer + selector: + app: ingress-nginx + ports: + - name: http + port: 80 + targetPort: http + - name: https + port: 443 + targetPort: https diff --git a/k8s/nginx-ingress/configmap.yaml b/k8s/nginx-ingress/configmap.yaml new file mode 100644 index 000000000..6703fc38e --- /dev/null +++ b/k8s/nginx-ingress/configmap.yaml @@ -0,0 +1,11 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: nginx-configuration + namespace: ingress-nginx + labels: + app: ingress-nginx +data: + ssl-redirect: "false" + proxy-buffer-size: "128k" + proxy-buffers: "4 256k" diff --git a/k8s/nginx-ingress/default-backend.yaml b/k8s/nginx-ingress/default-backend.yaml new file mode 100644 index 000000000..64f6f58ad --- /dev/null +++ b/k8s/nginx-ingress/default-backend.yaml @@ -0,0 +1,52 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: default-http-backend + labels: + app: default-http-backend + namespace: ingress-nginx +spec: + replicas: 1 + template: + metadata: + labels: + app: default-http-backend + spec: + terminationGracePeriodSeconds: 60 + containers: + - name: default-http-backend + # Any image is permissable as long as: + # 1. It serves a 404 page at / + # 2. It serves 200 on a /healthz endpoint + image: gcr.io/google_containers/defaultbackend:1.4 + livenessProbe: + httpGet: + path: /healthz + port: 8080 + scheme: HTTP + initialDelaySeconds: 30 + timeoutSeconds: 5 + ports: + - containerPort: 8080 + resources: + limits: + cpu: 10m + memory: 20Mi + requests: + cpu: 10m + memory: 20Mi +--- + +apiVersion: v1 +kind: Service +metadata: + name: default-http-backend + namespace: ingress-nginx + labels: + app: default-http-backend +spec: + ports: + - port: 80 + targetPort: 8080 + selector: + app: default-http-backend diff --git a/k8s/nginx-ingress/namespace.yaml b/k8s/nginx-ingress/namespace.yaml new file mode 100644 index 000000000..6878f0be8 --- /dev/null +++ b/k8s/nginx-ingress/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: ingress-nginx diff --git a/k8s/nginx-ingress/patch-service-without-rbac.yaml b/k8s/nginx-ingress/patch-service-without-rbac.yaml new file mode 100644 index 000000000..919efc389 --- /dev/null +++ b/k8s/nginx-ingress/patch-service-without-rbac.yaml @@ -0,0 +1,40 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: nginx-ingress-controller + namespace: ingress-nginx +spec: + replicas: 1 + selector: + matchLabels: + app: ingress-nginx + template: + metadata: + labels: + app: ingress-nginx + spec: + containers: + - name: nginx-ingress-controller + image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.9.0 + args: + - /nginx-ingress-controller + - --default-backend-service=$(POD_NAMESPACE)/default-http-backend + - --configmap=$(POD_NAMESPACE)/nginx-configuration + - --tcp-services-configmap=$(POD_NAMESPACE)/tcp-services + - --udp-services-configmap=$(POD_NAMESPACE)/udp-services + - --publish-service=$(POD_NAMESPACE)/ingress-nginx + - --annotations-prefix=nginx.ingress.kubernetes.io + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + ports: + - name: http + containerPort: 80 + - name: https + containerPort: 443 diff --git a/k8s/nginx-ingress/publish-service-patch.yaml b/k8s/nginx-ingress/publish-service-patch.yaml new file mode 100644 index 000000000..f8f52f772 --- /dev/null +++ b/k8s/nginx-ingress/publish-service-patch.yaml @@ -0,0 +1,7 @@ +[ + { + 'op': 'add', + 'path': '/spec/template/spec/containers/0/args/-', + 'value': '--publish-service=$(POD_NAMESPACE)/ingress-nginx' + } +] diff --git a/k8s/nginx-ingress/tcp-services-configmap.yaml b/k8s/nginx-ingress/tcp-services-configmap.yaml new file mode 100644 index 000000000..a963085d3 --- /dev/null +++ b/k8s/nginx-ingress/tcp-services-configmap.yaml @@ -0,0 +1,5 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: tcp-services + namespace: ingress-nginx diff --git a/k8s/nginx-ingress/udp-services-configmap.yaml b/k8s/nginx-ingress/udp-services-configmap.yaml new file mode 100644 index 000000000..1870931a2 --- /dev/null +++ b/k8s/nginx-ingress/udp-services-configmap.yaml @@ -0,0 +1,5 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: udp-services + namespace: ingress-nginx diff --git a/k8s/nginx-ingress/without-rbac.yaml b/k8s/nginx-ingress/without-rbac.yaml new file mode 100644 index 000000000..1c46b73eb --- /dev/null +++ b/k8s/nginx-ingress/without-rbac.yaml @@ -0,0 +1,61 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: nginx-ingress-controller + namespace: ingress-nginx +spec: + replicas: 1 + selector: + matchLabels: + app: ingress-nginx + template: + metadata: + labels: + app: ingress-nginx + annotations: + prometheus.io/port: '10254' + prometheus.io/scrape: 'true' + spec: + containers: + - name: nginx-ingress-controller + image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.9.0 + args: + - /nginx-ingress-controller + - --default-backend-service=$(POD_NAMESPACE)/default-http-backend + - --configmap=$(POD_NAMESPACE)/nginx-configuration + - --tcp-services-configmap=$(POD_NAMESPACE)/tcp-services + - --udp-services-configmap=$(POD_NAMESPACE)/udp-services + - --annotations-prefix=nginx.ingress.kubernetes.io + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + ports: + - name: http + containerPort: 80 + - name: https + containerPort: 443 + livenessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + readinessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 diff --git a/k8s/nginx.conf b/k8s/nginx.conf deleted file mode 100644 index 22c3414a1..000000000 --- a/k8s/nginx.conf +++ /dev/null @@ -1,98 +0,0 @@ -pid /tmp/nginx.pid; - -worker_processes 1; - -events { - worker_connections 1024; -} - -http { - server_tokens off; - - add_header X-Frame-Options SAMEORIGIN; - add_header X-Content-Type-Options nosniff; - add_header X-XSS-Protection "1; mode=block"; - - client_body_temp_path /tmp/client_body; - fastcgi_temp_path /tmp/fastcgi_temp; - proxy_temp_path /tmp/proxy_temp; - scgi_temp_path /tmp/scgi_temp; - uwsgi_temp_path /tmp/uwsgi_temp; - - gzip on; - gzip_comp_level 6; - gzip_min_length 1024; - gzip_buffers 4 32k; - gzip_types text/plain application/javascript text/css; - gzip_vary on; - - keepalive_timeout 65; - - proxy_buffer_size 128k; - proxy_buffers 4 256k; - proxy_busy_buffers_size 256k; - - server { - listen 8080; - - location /basket-api { - proxy_pass http://basket; - proxy_redirect off; - proxy_set_header Host $host; - } - - location /catalog-api { - proxy_pass http://catalog; - proxy_redirect off; - proxy_set_header Host $host; - } - - location /identity { - proxy_pass http://identity; - proxy_redirect off; - proxy_set_header Host $host; - } - - location /ordering-api { - proxy_pass http://ordering; - proxy_redirect off; - proxy_set_header Host $host; - } - - location /webmvc { - proxy_pass http://webmvc; - proxy_redirect off; - proxy_set_header Host $host; - } - - location /webstatus { - proxy_pass http://webstatus; - proxy_redirect off; - proxy_set_header Host $host; - } - - location /marketing-api { - proxy_pass http://marketing; - proxy_redirect off; - proxy_set_header Host $host; - } - - location /payment-api { - proxy_pass http://payment; - proxy_redirect off; - proxy_set_header Host $host; - } - - location /locations-api { - proxy_pass http://locations; - proxy_redirect off; - proxy_set_header Host $host; - } - - location / { - proxy_pass http://webspa; - proxy_redirect off; - proxy_set_header Host $host; - } - } -} \ No newline at end of file From 33d066389d82100def69ec84875f2a7efe8f09ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eduard=20Tom=C3=A0s?= Date: Thu, 11 Jan 2018 11:14:25 +0100 Subject: [PATCH 2/2] remove -buildBits paramater (no longer used) --- k8s/deploy.ps1 | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/k8s/deploy.ps1 b/k8s/deploy.ps1 index c2b400b4e..42202f637 100644 --- a/k8s/deploy.ps1 +++ b/k8s/deploy.ps1 @@ -8,7 +8,6 @@ Param( [parameter(Mandatory=$false)][string]$imageTag, [parameter(Mandatory=$false)][bool]$deployCI=$false, [parameter(Mandatory=$false)][bool]$buildImages=$true, - [parameter(Mandatory=$false)][bool]$buildBits=$false, [parameter(Mandatory=$false)][bool]$deployInfrastructure=$true, [parameter(Mandatory=$false)][string]$dockerOrg="eshop" ) @@ -50,7 +49,6 @@ if(-not $deployCI) { } } else { - $buildBits = false; $buildImages = false; # Never build images through CI, as they previously built } @@ -60,11 +58,7 @@ if ([string]::IsNullOrEmpty($imageTag)) { } Write-Host "Docker image Tag: $imageTag" -ForegroundColor Yellow -# building and publishing docker images if needed -if($buildBits) { - Write-Host "Building and publishing eShopOnContainers..." -ForegroundColor Yellow - dotnet publish -c Release -o obj/Docker/publish ../eShopOnContainers-ServicesAndWebApps.sln -} +# building docker images if needed if ($buildImages) { Write-Host "Building Docker images tagged with '$imageTag'" -ForegroundColor Yellow $env:TAG=$imageTag