KANS3 - AWS EKS: VPC CNI - 02/02

ExternalDNS

K8S 서비스/인그레스 생성 시 도메인을 설정하면, AWS(Route 53), Azure(DNS), GCP(Cloud DNS) 에 A 레코드(TXT 레코드)로 자동 생성/삭제

MyDomain=kans3.ysyu.kr
echo "export MyDomain=kans3.ysyu.kr" >> /etc/profile

# 자신의 Route 53 도메인 ID 조회 및 변수 지정
aws route53 list-hosted-zones-by-name --dns-name "${MyDomain}." | jq
aws route53 list-hosted-zones-by-name --dns-name "${MyDomain}." --query "HostedZones[0].Name"
aws route53 list-hosted-zones-by-name --dns-name "${MyDomain}." --query "HostedZones[0].Id" --output text
MyDnzHostedZoneId=`aws route53 list-hosted-zones-by-name --dns-name "${MyDomain}." --query "HostedZones[0].Id" --output text`
echo $MyDnzHostedZoneId

# (옵션) NS 레코드 타입 첫번째 조회
aws route53 list-resource-record-sets --output json --hosted-zone-id "${MyDnzHostedZoneId}" --query "ResourceRecordSets[?Type == 'NS']" | jq -r '.[0].ResourceRecords[].Value'
# (옵션) A 레코드 타입 모두 조회
aws route53 list-resource-record-sets --output json --hosted-zone-id "${MyDnzHostedZoneId}" --query "ResourceRecordSets[?Type == 'A']"

# A 레코드 타입 조회
aws route53 list-resource-record-sets --hosted-zone-id "${MyDnzHostedZoneId}" --query "ResourceRecordSets[?Type == 'A']" | jq
aws route53 list-resource-record-sets --hosted-zone-id "${MyDnzHostedZoneId}" --query "ResourceRecordSets[?Type == 'A'].Name" | jq
aws route53 list-resource-record-sets --hosted-zone-id "${MyDnzHostedZoneId}" --query "ResourceRecordSets[?Type == 'A'].Name" --output text

# A 레코드 값 반복 조회
while true; do aws route53 list-resource-record-sets --hosted-zone-id "${MyDnzHostedZoneId}" --query "ResourceRecordSets[?Type == 'A']" | jq ; date ; echo ; sleep 1; done

--------------------------------------------------------------------------------------------------------------
# ExternalDNS 설치
MyDomain=kans3.ysyu.kr

# 자신의 Route 53 도메인 ID 조회 및 변수 지정
MyDnzHostedZoneId=$(aws route53 list-hosted-zones-by-name --dns-name "${MyDomain}." --query "HostedZones[0].Id" --output text)

# 변수 확인
echo $MyDomain, $MyDnzHostedZoneId

# ExternalDNS 배포
curl -s -O https://raw.githubusercontent.com/gasida/PKOS/main/aews/externaldns.yaml
cat externaldns.yaml
MyDomain=$MyDomain MyDnzHostedZoneId=$MyDnzHostedZoneId envsubst < externaldns.yaml | k apply -f -

# 확인 및 로그 모니터링
k get pod -l app.kubernetes.io/name=external-dns -n kube-system
k logs deploy/external-dns -n kube-system -f

--------------------------------------------------------------------------------------------------------------
# 테스트
# 터미널1 (모니터링)
watch -d 'k get pod,svc'
k logs deploy/external-dns -n kube-system -f

# 테트리스 디플로이먼트 배포
cat <<EOF | k apply -f -
apiVersion: apps/v1
kind: Deployment
metadata:
  name: tetris
  labels:
    app: tetris
spec:
  replicas: 1
  selector:
    matchLabels:
      app: tetris
  template:
    metadata:
      labels:
        app: tetris
    spec:
      containers:
      - name: tetris
        image: bsord/tetris
---
apiVersion: v1
kind: Service
metadata:
  name: tetris
  annotations:
    service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip
    service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing
    service.beta.kubernetes.io/aws-load-balancer-cross-zone-load-balancing-enabled: "true"
    service.beta.kubernetes.io/aws-load-balancer-backend-protocol: "http"
    #service.beta.kubernetes.io/aws-load-balancer-healthcheck-port: "80"
spec:
  selector:
    app: tetris
  ports:
  - port: 80
    protocol: TCP
    targetPort: 80
  type: LoadBalancer
  loadBalancerClass: service.k8s.aws/nlb
EOF

# 배포 확인
k get deploy,svc,ep tetris

# NLB에 ExternanDNS 로 도메인 연결
k annotate service tetris "external-dns.alpha.kubernetes.io/hostname=tetris.$MyDomain"
while true; do aws route53 list-resource-record-sets --hosted-zone-id "${MyDnzHostedZoneId}" --query "ResourceRecordSets[?Type == 'A']" | jq ; date ; echo ; sleep 1; done

# Route53에 A레코드 확인
aws route53 list-resource-record-sets --hosted-zone-id "${MyDnzHostedZoneId}" --query "ResourceRecordSets[?Type == 'A']" | jq
aws route53 list-resource-record-sets --hosted-zone-id "${MyDnzHostedZoneId}" --query "ResourceRecordSets[?Type == 'A'].Name" | jq .[]

# 확인
dig +short tetris.$MyDomain @8.8.8.8
dig +short tetris.$MyDomain

# 도메인 체크
echo -e "My Domain Checker = https://www.whatsmydns.net/#A/tetris.$MyDomain"

# 웹 접속 주소 확인 및 접속
echo -e "Tetris Game URL = http://tetris.$MyDomain"

CoreDNS

# CoreDNS 기본 정보 확인 : volumes - configMap - Corefile - coredns
k get deploy coredns -n kube-system -o yaml | k neat 
k get cm -n kube-system coredns -o yaml | k neat
data: 
  Corefile: |
    .:53 {
        errors
        health {
            lameduck 5s
          }
        ready
        kubernetes cluster.local in-addr.arpa ip6.arpa {
          pods insecure
          fallthrough in-addr.arpa ip6.arpa
        }
        prometheus :9153
        forward . /etc/resolv.conf
        cache 30
        loop
        reload
        loadbalance
    }

# JSON 구성 스키마에 topologySpreadConstraints 매개변수를 추가
aws eks describe-addon-configuration --addon-name coredns --addon-version v1.10.1-eksbuild.2 --query 'configurationSchema' --output text | jq .
aws eks describe-addon-configuration --addon-name coredns --addon-version v1.10.1-eksbuild.2 --query 'configurationSchema' --output text | jq . | grep -A3 topologySpreadConstraints

# check coredns deployment - it returns an empty output because it is not set by default 
k get deploy -n kube-system coredns -o yaml | grep topologySpreadConstraints -A8

# coredns addon 정보 확인
aws eks describe-addon --cluster-name $CLUSTER_NAME --addon-name coredns | jq

# add-on configuration YAML blob
cat << EOT > topologySpreadConstraints.yaml
"topologySpreadConstraints":
  - maxSkew: 1
    topologyKey: topology.kubernetes.io/zone
    whenUnsatisfiable: ScheduleAnyway
    labelSelector:
      matchLabels:
        k8s-app: kube-dns
EOT

# apply change to add-on
aws eks update-addon --cluster-name $CLUSTER_NAME --addon-name coredns --configuration-values 'file://topologySpreadConstraints.yaml'

# check add-on configuration to see if it is in ACTIVE status
aws eks describe-addon --cluster-name $CLUSTER_NAME --addon-name coredns | jq
k get deploy coredns -n kube-system -o yaml | k neat
...
    topologySpreadConstraints: 
    - labelSelector: 
        matchLabels: 
          k8s-app: kube-dns
      maxSkew: 1
      topologyKey: topology.kubernetes.io/zone
      whenUnsatisfiable: ScheduleAnyway
...

--------------------------------------------------------------------------------------------------------------

# K8s 내 DNS 장애를 막기 위한 설정
# CoreDNS 기본 정보 확인
k get cm -n kube-system coredns -o yaml | k neat
data: 
  Corefile: |
    .:53 {
        errors
        health {
            lameduck 5s
          }
        ready
        kubernetes cluster.local in-addr.arpa ip6.arpa {
          pods insecure
          fallthrough in-addr.arpa ip6.arpa
        }
        prometheus :9153
        forward . /etc/resolv.conf
        cache 30
        loop
        reload
        loadbalance
    }

--------------------------------------------------------------------------------------------------------------

# readinessprobe 설정 변경
#
k get deploy coredns -n kube-system -o yaml | k neat 
...
      readinessProbe: 
        failureThreshold: 3
        httpGet: 
          path: /ready
          port: 8181
          scheme: HTTP
        periodSeconds: 10
        successThreshold: 1
        timeoutSeconds: 1
...

# The “ready” plugin is already part of the “coredns” ConfigMap:
k get cm -n kube-system coredns -o yaml | k neat
data: 
  Corefile: |
    .:53 {
        errors
        health {
            lameduck 5s
          }
        ready
        kubernetes cluster.local in-addr.arpa ip6.arpa {
          pods insecure
          fallthrough in-addr.arpa ip6.arpa
        }
        prometheus :9153
        forward . /etc/resolv.conf
        cache 30
        loop
        reload
        loadbalance
    }

--------------------------------------------------------------------------------------------------------------

# 라벨 변경
# 파드 Labels 확인
k get deploy coredns -n kube-system -o yaml | k neat 

# pod labels
aws eks describe-addon-configuration --addon-name coredns --addon-version v1.10.1-eksbuild.3 \
  --query 'configurationSchema' --output text | jq . | grep -A4 '\"podLabels\"'
        "podLabels": {
          "properties": {},
          "title": "The podLabels Schema",
          "type": "object"
        },

# YAML configuration blob
cat << EOT > podLabels.yaml
podLabels:
  foo: bar
EOT
  
# apply changes : 적용 시 coredns 파드 재시작됨
aws eks update-addon --cluster-name $CLUSTER_NAME --addon-name coredns --configuration-values 'file://podLabels.yaml'
watch -d k get pod -n kube-system

# wait a while until the add-on is ACTIVE again
aws eks describe-addon --cluster-name $CLUSTER_NAME --addon-name coredns

# 파드 Labels 확인
k get deploy coredns -n kube-system -o yaml | k neat
k get pod -n kube-system -l foo=bar
k get po -n kube-system -l=k8s-app=kube-dns -o custom-columns="POD-NAME":.metadata.name,"POD-LABELS":.metadata.labels

Topology Aware Routing

# 테스트용 app 배포
cat <<EOF | kubectl apply -f -
apiVersion: apps/v1
kind: Deployment
metadata:
  name: deploy-echo
spec:
  replicas: 3
  selector:
    matchLabels:
      app: deploy-websrv
  template:
    metadata:
      labels:
        app: deploy-websrv
    spec:
      terminationGracePeriodSeconds: 0
      containers:
      - name: websrv
        image: registry.k8s.io/echoserver:1.5
        ports:
        - containerPort: 8080
---
apiVersion: v1
kind: Service
metadata:
  name: svc-clusterip
spec:
  ports:
    - name: svc-webport
      port: 80
      targetPort: 8080
  selector:
    app: deploy-websrv
  type: ClusterIP
EOF

# 확인
kubectl get deploy,svc,ep,endpointslices
kubectl get pod -owide
kubectl get svc,ep svc-clusterip
kubectl get endpointslices -l kubernetes.io/service-name=svc-clusterip
kubectl get endpointslices -l kubernetes.io/service-name=svc-clusterip -o yaml

# 접속 테스트를 수행할 클라이언트 파드 배포
cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: Pod
metadata:
  name: netshoot-pod
spec:
  containers:
  - name: netshoot-pod
    image: nicolaka/netshoot
    command: ["tail"]
    args: ["-f", "/dev/null"]
  terminationGracePeriodSeconds: 0
EOF

# 확인
kubectl get pod -owide

--------------------------------------------------------------------------------------------------------------

# 디플로이먼트 파드가 배포된 AZ(zone) 확인
kubectl get pod -l app=deploy-websrv -owide

# 테스트 파드(netshoot-pod)에서 ClusterIP 접속 시 부하분산 확인
kubectl exec -it netshoot-pod -- curl svc-clusterip | grep Hostname

# 100번 반복 접속 : 3개의 파드로 AZ(zone) 상관없이 랜덤 확률 부하분산 동작
kubectl exec -it netshoot-pod -- zsh -c "for i in {1..100}; do curl -s svc-clusterip | grep Hostname; done | sort | uniq -c | sort -nr"

--------------------------------------------------------------------------------------------------------------

# Topology Aware Routing 설정 : 서비스에 annotate에 아래처럼 추가
kubectl annotate service svc-clusterip "service.kubernetes.io/topology-mode=auto"

# 100번 반복 접속 : 테스트 파드(netshoot-pod)와 같은 AZ(zone)의 목적지 파드로만 접속
kubectl exec -it netshoot-pod -- zsh -c "for i in {1..100}; do curl -s svc-clusterip | grep Hostname; done | sort | uniq -c | sort -nr"

# endpointslices 확인 시, 기존에 없던 hints 가 추가되어 있음 >> 참고로 describe로는 hints 정보가 출력되지 않음
kubectl get endpointslices -l kubernetes.io/service-name=svc-clusterip -o yaml

--------------------------------------------------------------------------------------------------------------

# POD 분배
# topoligySpreadConstraints

# 디플로이먼트 배포
cat <<EOF | kubectl apply -f -
apiVersion: apps/v1
kind: Deployment
metadata:
  name: deploy-echo
spec:
  replicas: 6
  selector:
    matchLabels:
      app: deploy-websrv
  template:
    metadata:
      labels:
        app: deploy-websrv
    spec:
      terminationGracePeriodSeconds: 0
      containers:
      - name: websrv
        image: registry.k8s.io/echoserver:1.5
        ports:
        - containerPort: 8080
      topologySpreadConstraints:
      - maxSkew: 1
        topologyKey: "topology.kubernetes.io/zone"
        whenUnsatisfiable: DoNotSchedule
        labelSelector:
          matchLabels:
            app: deploy-websrv
EOF

# 확인
kubectl get pod -owide

후기

약 9주간의 길고긴 대장정을 마무리했습니다. 참여한 분들도 대단핮디만, 9주간의 스터디를 이끄신 가시다님도 너무 고생 많으셨습니다. 다음에는 어떤 스터디로 함께 하게될지는 모르겠으나, 그때에는 더 많은 경험을 가지고 참석해서 경험을 나눠볼수 있는 사람이 되었으면 합니다.