Zero-Downtime Kubernetes Upgrades: Production-Ready Strategies

Phase 1: Prepare New Node Pool

# For EKS (AWS)
eksctl create nodegroup \
  --cluster=production \
  --name=ng-1-28 \
  --version=1.28 \
  --node-type=m5.xlarge \
  --nodes=3 \
  --nodes-min=3 \
  --nodes-max=10 \
  --node-labels="pool=ng-1-28,upgrade-target=true"

# For GKE (Google Cloud)
gcloud container node-pools create ng-1-28 \
  --cluster=production \
  --machine-type=n2-standard-4 \
  --num-nodes=3 \
  --enable-autoscaling \
  --min-nodes=3 \
  --max-nodes=10 \
  --node-version=1.28.5-gke.1000 \
  --node-labels=pool=ng-1-28,upgrade-target=true

Phase 2: Gradual Workload Migration

Use pod affinity and taints/tolerations to control which nodes receive new pods:

# Taint old nodes to prevent new pods
kubectl taint nodes -l pool=ng-1-27 upgrade=in-progress:NoSchedule

# Update deployment to prefer new nodes
apiVersion: apps/v1
kind: Deployment
metadata:
  name: web-app
spec:
  template:
    spec:
      affinity:
        nodeAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
          - weight: 100
            preference:
              matchExpressions:
              - key: pool
                operator: In
                values:
                - ng-1-28

Phase 3: Controlled Drain and Migration

#!/bin/bash
# Drain old nodes one at a time with monitoring

OLD_NODES=$(kubectl get nodes -l pool=ng-1-27 -o name)

for node in $OLD_NODES; do
  echo "Draining $node..."
  
  # Drain with grace period
  kubectl drain $node \
    --ignore-daemonsets \
    --delete-emptydir-data \
    --grace-period=300 \
    --timeout=600s
  
  # Wait for all pods to be running on new nodes
  sleep 60
  
  # Check service health
  curl -f https://healthcheck.example.com/ready || {
    echo "Health check failed! Uncordoning $node"
    kubectl uncordon $node
    exit 1
  }
  
  echo "Successfully migrated $node"
done

# Delete old node pool
eksctl delete nodegroup --cluster=production --name=ng-1-27

┌─────────────────────────────────────────────────────────┐
│              External Load Balancer / DNS               │
│         (weighted routing: 100% blue → 100% green)      │
└────────────────────┬────────────────────────────────────┘
                     │
        ┌────────────┴────────────┐
        │                         │
┌───────▼────────┐        ┌───────▼────────┐
│  Blue Cluster  │        │ Green Cluster  │
│   (v1.27)      │        │   (v1.28)      │
│                │        │                │
│  ✓ Production  │        │  ✓ Tested      │
│  ✓ Stable      │        │  ✓ Validated   │
│                │        │  ✓ Ready       │
└────────────────┘        └────────────────┘
        │                         │
        └─────────┬───────────────┘
                  │
          ┌───────▼────────┐
          │  Shared State  │
          │  (RDS, S3...)  │
          └────────────────┘

Step 1: Provision Green Cluster

# Terraform for identical cluster with new version
module "green_cluster" {
  source = "./modules/eks-cluster"
  
  cluster_name    = "production-green"
  cluster_version = "1.28"
  
  # Mirror blue cluster configuration
  node_groups = var.production_node_groups
  vpc_id      = data.aws_vpc.main.id
  
  # Add label to identify as green
  cluster_labels = {
    environment = "production"
    color       = "green"
    upgrade_id  = "2025-11-upgrade"
  }
}

# Deploy all applications to green cluster
resource "helm_release" "apps_green" {
  for_each = var.applications
  
  name       = each.key
  chart      = each.value.chart
  kubeconfig = module.green_cluster.kubeconfig_path
  
  # Use same values as blue
  values = [each.value.values_file]
}

Step 2: Smoke Test Green Cluster

#!/bin/bash
# Comprehensive pre-cutover validation

GREEN_ENDPOINT=$(kubectl --context=green get svc ingress-nginx \
  -n ingress-nginx -o jsonpath='{.status.loadBalancer.ingress[0].hostname}')

# Test all critical endpoints
ENDPOINTS=(
  "/api/health"
  "/api/v1/users"
  "/api/v1/orders"
)

for endpoint in "${ENDPOINTS[@]}"; do
  STATUS=$(curl -s -o /dev/null -w "%{http_code}" \
    -H "Host: api.example.com" \
    "http://$GREEN_ENDPOINT$endpoint")
  
  if [ "$STATUS" != "200" ]; then
    echo "❌ $endpoint failed with $STATUS"
    exit 1
  fi
  echo "✓ $endpoint: $STATUS"
done

# Run synthetic transactions
kubectl --context=green run smoke-test \
  --image=playwright:latest \
  --restart=Never \
  -- npm run e2e:production

Step 3: Gradual Traffic Cutover

# Using AWS Route53 weighted routing
resource "aws_route53_record" "app_blue" {
  zone_id = data.aws_route53_zone.main.zone_id
  name    = "app.example.com"
  type    = "A"
  
  alias {
    name    = module.blue_cluster.ingress_dns
    zone_id = module.blue_cluster.ingress_zone_id
  }
  
  set_identifier = "blue"
  weighted_routing_policy {
    weight = 90  # Start at 90%
  }
}

resource "aws_route53_record" "app_green" {
  zone_id = data.aws_route53_zone.main.zone_id
  name    = "app.example.com"
  type    = "A"
  
  alias {
    name    = module.green_cluster.ingress_dns
    zone_id = module.green_cluster.ingress_zone_id
  }
  
  set_identifier = "green"
  weighted_routing_policy {
    weight = 10  # Start at 10%
  }
}

# Cutover stages: 10% → 25% → 50% → 100%
# Monitor error rates, latency at each stage

Canary Node Implementation

apiVersion: apps/v1
kind: Deployment
metadata:
  name: web-app
spec:
  replicas: 10
  template:
    spec:
      affinity:
        podAntiAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
          # Spread pods across both node versions
          - weight: 100
            podAffinityTerm:
              topologyKey: kubernetes.io/hostname
      
      # Deploy to both old and new nodes
      tolerations:
      - key: "node.kubernetes.io/upgrade-canary"
        operator: "Exists"
        effect: "NoSchedule"

---
# Canary node configuration
apiVersion: v1
kind: Node
metadata:
  labels:
    upgrade-canary: "true"
    kubernetes.io/version: "1.28"
spec:
  taints:
  - key: node.kubernetes.io/upgrade-canary
    value: "true"
    effect: NoSchedule