Blue/Green 배포 (4): 운영 및 최적화

# SLO 예시
slos:
  availability:
    target: 99.9%  # 월 43분 다운타임 허용
    window: 30d
    indicator: |
      sum(rate(http_requests_total{status!~"5.."}[5m])) /
      sum(rate(http_requests_total[5m]))
 
  latency:
    target: 95%    # 95%의 요청이 200ms 이내
    window: 30d
    indicator: |
      histogram_quantile(0.95,
        sum(rate(http_request_duration_seconds_bucket[5m])) by (le)
      )
 
  error_budget:
    monthly: 43.2m  # 99.9% SLO = 43.2분/월
    alert_threshold: 50%  # 예산 50% 소진 시 알림

# prometheus-rules.yaml
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: blue-green-alerts
spec:
  groups:
  - name: deployment
    rules:
    # 5xx 에러율 급증
    - alert: HighErrorRate
      expr: |
        sum(rate(http_requests_total{status=~"5.."}[5m])) /
        sum(rate(http_requests_total[5m])) > 0.01
      for: 2m
      labels:
        severity: critical
        action: rollback
      annotations:
        summary: "High error rate detected after deployment"
        description: "Error rate is {{ $value | humanizePercentage }} (threshold: 1%)"
        runbook_url: "https://wiki.example.com/runbooks/high-error-rate"
 
    # 레이턴시 급증
    - alert: HighLatency
      expr: |
        histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket[5m])) by (le))
        > 0.5
      for: 5m
      labels:
        severity: warning
      annotations:
        summary: "P99 latency exceeded 500ms"
        description: "Current P99: {{ $value | humanizeDuration }}"
 
    # Pod 재시작 감지
    - alert: PodRestartLoop
      expr: |
        increase(kube_pod_container_status_restarts_total{
          pod=~"my-app-.*"
        }[15m]) > 3
      labels:
        severity: critical
        action: rollback
      annotations:
        summary: "Pod restart loop detected"
 
    # 배포 후 롤백 감지
    - alert: DeploymentRollback
      expr: |
        kube_deployment_status_observed_generation{deployment="my-app"}
        < kube_deployment_metadata_generation{deployment="my-app"}
      for: 1m
      labels:
        severity: info
      annotations:
        summary: "Deployment rollback in progress"

{
  "dashboard": {
    "title": "Blue/Green Deployment Dashboard",
    "panels": [
      {
        "title": "Traffic Distribution",
        "type": "piechart",
        "targets": [
          {
            "expr": "sum(rate(http_requests_total{color=\"blue\"}[5m]))",
            "legendFormat": "Blue"
          },
          {
            "expr": "sum(rate(http_requests_total{color=\"green\"}[5m]))",
            "legendFormat": "Green"
          }
        ]
      },
      {
        "title": "Error Rate by Version",
        "type": "timeseries",
        "targets": [
          {
            "expr": "sum(rate(http_requests_total{status=~\"5..\",color=\"blue\"}[5m])) / sum(rate(http_requests_total{color=\"blue\"}[5m]))",
            "legendFormat": "Blue 5xx"
          },
          {
            "expr": "sum(rate(http_requests_total{status=~\"5..\",color=\"green\"}[5m])) / sum(rate(http_requests_total{color=\"green\"}[5m]))",
            "legendFormat": "Green 5xx"
          }
        ]
      },
      {
        "title": "P99 Latency Comparison",
        "type": "timeseries",
        "targets": [
          {
            "expr": "histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket{color=\"blue\"}[5m])) by (le))",
            "legendFormat": "Blue P99"
          },
          {
            "expr": "histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket{color=\"green\"}[5m])) by (le))",
            "legendFormat": "Green P99"
          }
        ]
      },
      {
        "title": "Deployment Events",
        "type": "annotations",
        "datasource": "-- Grafana --",
        "query": "tags=deployment"
      }
    ]
  }
}

# deploy_marker.py - 배포 이벤트를 Grafana에 기록
import requests
from datetime import datetime
 
def mark_deployment(version: str, environment: str, color: str):
    """Grafana에 배포 이벤트 마커 생성"""
    grafana_url = "https://grafana.example.com"
    api_key = os.environ["GRAFANA_API_KEY"]
 
    annotation = {
        "time": int(datetime.now().timestamp() * 1000),
        "tags": ["deployment", environment, color],
        "text": f"Deployed {version} to {color} ({environment})"
    }
 
    response = requests.post(
        f"{grafana_url}/api/annotations",
        json=annotation,
        headers={"Authorization": f"Bearer {api_key}"}
    )
    return response.status_code == 200

# Green 환경에 Spot 인스턴스 사용 (AWS EKS)
apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-app-green
spec:
  template:
    spec:
      nodeSelector:
        node.kubernetes.io/lifecycle: spot
      tolerations:
      - key: "node.kubernetes.io/lifecycle"
        operator: "Equal"
        value: "spot"
        effect: "NoSchedule"
      # Spot 중단 대비 graceful shutdown
      terminationGracePeriodSeconds: 30
      containers:
      - name: app
        lifecycle:
          preStop:
            exec:
              command: ["/bin/sh", "-c", "sleep 10"]

# green-hpa.yaml - Green 환경 최소 유지
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: my-app-green
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: my-app-green
  minReplicas: 1  # 평상시 최소 유지
  maxReplicas: 10 # 배포 시 확장
  metrics:
  - type: Resource
    resource:
      name: cpu
      target:
        type: Utilization
        averageUtilization: 70
  behavior:
    scaleUp:
      stabilizationWindowSeconds: 0  # 즉시 스케일 업
      policies:
      - type: Percent
        value: 100
        periodSeconds: 15
    scaleDown:
      stabilizationWindowSeconds: 300  # 5분 대기 후 스케일 다운

#!/bin/bash
# scale-green.sh - 업무 시간 외 Green 환경 축소
 
HOUR=$(date +%H)
DAY=$(date +%u)
 
# 업무 시간 (월-금 09:00-18:00) 외에는 축소
if [ $DAY -gt 5 ] || [ $HOUR -lt 9 ] || [ $HOUR -gt 18 ]; then
  echo "Off-hours: Scaling down Green environment"
  kubectl scale deployment my-app-green --replicas=0
else
  echo "Business hours: Maintaining Green environment"
  kubectl scale deployment my-app-green --replicas=1
fi

# CronJob으로 자동화
apiVersion: batch/v1
kind: CronJob
metadata:
  name: scale-green-down
spec:
  schedule: "0 19 * * 1-5"  # 평일 19:00
  jobTemplate:
    spec:
      template:
        spec:
          serviceAccountName: deployment-scaler
          containers:
          - name: kubectl
            image: bitnami/kubectl
            command:
            - kubectl
            - scale
            - deployment/my-app-green
            - --replicas=0
          restartPolicy: OnFailure
---
apiVersion: batch/v1
kind: CronJob
metadata:
  name: scale-green-up
spec:
  schedule: "0 8 * * 1-5"  # 평일 08:00
  jobTemplate:
    spec:
      template:
        spec:
          serviceAccountName: deployment-scaler
          containers:
          - name: kubectl
            image: bitnami/kubectl
            command:
            - kubectl
            - scale
            - deployment/my-app-green
            - --replicas=1
          restartPolicy: OnFailure

# Route53 TTL 설정 예시
aws route53 change-resource-record-sets \
  --hosted-zone-id Z123456 \
  --change-batch '{
    "Changes": [{
      "Action": "UPSERT",
      "ResourceRecordSet": {
        "Name": "api.example.com",
        "Type": "A",
        "TTL": 60,
        "ResourceRecords": [{"Value": "1.2.3.4"}]
      }
    }]
  }'

# CloudFront 캐시 무효화
aws cloudfront create-invalidation \
  --distribution-id E123456789 \
  --paths "/*"
 
# Fastly 캐시 퍼지
curl -X POST "https://api.fastly.com/service/{service_id}/purge_all" \
  -H "Fastly-Key: $FASTLY_API_KEY"

# Kubernetes Service - Connection draining 설정
apiVersion: v1
kind: Service
metadata:
  name: my-app
  annotations:
    # AWS ALB
    service.beta.kubernetes.io/aws-load-balancer-connection-draining-enabled: "true"
    service.beta.kubernetes.io/aws-load-balancer-connection-draining-timeout: "60"
spec:
  # ...

# Pod - Graceful shutdown
apiVersion: apps/v1
kind: Deployment
spec:
  template:
    spec:
      terminationGracePeriodSeconds: 60
      containers:
      - name: app
        lifecycle:
          preStop:
            exec:
              # 새 연결 중단, 기존 연결 완료 대기
              command:
              - /bin/sh
              - -c
              - |
                # Health check 실패시키기
                touch /tmp/shutdown
                # 기존 연결 완료 대기
                sleep 30

# Redis를 통한 세션 외부화 (Spring Boot)
# application.yml
spring:
  session:
    store-type: redis
    redis:
      namespace: spring:session
  redis:
    host: redis.example.com
    port: 6379
 
---
# Kubernetes에서 Session Affinity 사용 시
apiVersion: v1
kind: Service
metadata:
  name: my-app
spec:
  sessionAffinity: ClientIP
  sessionAffinityConfig:
    clientIP:
      timeoutSeconds: 3600

# 진단
kubectl get pods -l color=green
kubectl describe pod my-app-green-xxx
 
# 일반적인 원인
# 1. 이미지 Pull 실패
kubectl get events --field-selector reason=Failed
 
# 2. Readiness Probe 실패
kubectl logs my-app-green-xxx
 
# 3. 리소스 부족
kubectl describe node | grep -A5 "Allocated resources"

# 즉시 롤백
kubectl patch svc my-app -p '{"spec":{"selector":{"color":"blue"}}}'
 
# 원인 분석
# 1. 로그 확인
kubectl logs -l color=green --tail=100
 
# 2. 의존성 연결 확인
kubectl exec -it my-app-green-xxx -- curl -v http://db-service:5432
 
# 3. 환경 변수 비교
kubectl get deployment my-app-blue -o json | jq '.spec.template.spec.containers[0].env'
kubectl get deployment my-app-green -o json | jq '.spec.template.spec.containers[0].env'

# Redis 세션 확인
redis-cli KEYS "spring:session:*" | wc -l
 
# 해결책: 세션 마이그레이션 확인
# 1. 세션 스토어 연결 확인
kubectl exec my-app-green-xxx -- nc -zv redis.example.com 6379
 
# 2. 세션 키 포맷 확인 (버전 간 호환성)
redis-cli GET "spring:session:sessions:xxx"