Secrets Management: Vault vs SSM Decision Guide

# Install Vault on Kubernetes (production-ready with HA) helm repo add hashicorp https://helm.releases.hashicorp.com helm install vault hashicorp/vault \ --namespace vault --create-namespace \ --set server.ha.enabled=true \ --set server.ha.replicas=3 \ --set ui.enabled=true \ --set server.dataStorage.size=10Gi # Initialize Vault (do this ONCE) kubectl exec -n vault vault-0 -- vault operator init -key-shares=5 -key-threshold=3 # Output: 5 unseal keys + 1 root token # STORE THESE SECURELY (split across team members or use PGP encryption) # Unseal Vault on each pod (requires 3 of 5 keys) kubectl exec -n vault vault-0 -- vault operator unseal <KEY1> kubectl exec -n vault vault-0 -- vault operator unseal <KEY2> kubectl exec -n vault vault-0 -- vault operator unseal <KEY3> # Repeat for vault-1 and vault-2 # Verify status kubectl exec -n vault vault-0 -- vault status

# Enable database secrets engine vault secrets enable database # Configure PostgreSQL connection vault write database/config/prod-postgres \ plugin_name=postgresql-database-plugin \ allowed_roles="readonly","readwrite" \ connection_url="postgresql://{{username}}:{{password}}@postgres.prod.svc:5432/app?sslmode=require" \ username="vault-admin" \ password="vault-admin-password" # Create a role that generates 1-hour credentials vault write database/roles/readonly \ db_name=prod-postgres \ creation_statements="CREATE ROLE \"{{name}}\" WITH LOGIN PASSWORD '{{password}}' VALID UNTIL '{{expiration}}'; \ GRANT SELECT ON ALL TABLES IN SCHEMA public TO \"{{name}}\";" \ default_ttl="1h" \ max_ttl="24h" # Test: Generate ephemeral credentials vault read database/creds/readonly # Output: # Key Value # lease_id database/creds/readonly/abc123 # lease_duration 1h # username v-root-readonly-xyz456 # password A1b2C3d4E5f6... # After 1 hour, these credentials are automatically revoked!

# Store a secret (encrypted with KMS) aws ssm put-parameter \ --name "/production/database/password" \ --value "super-secret-password" \ --type SecureString \ --kms-key-id alias/production-secrets \ --description "Production database password" \ --tags Key=Environment,Value=production Key=Owner,Value=platform-team # Retrieve a secret aws ssm get-parameter \ --name "/production/database/password" \ --with-decryption # Output (JSON): { "Parameter": { "Name": "/production/database/password", "Type": "SecureString", "Value": "super-secret-password", "Version": 1, "LastModifiedDate": "2025-01-02T10:00:00.000Z", "ARN": "arn:aws:ssm:us-east-1:123456789012:parameter/production/database/password" } } # Get multiple secrets by path aws ssm get-parameters-by-path \ --path "/production/database" \ --with-decryption \ --recursive # Update a secret (creates new version) aws ssm put-parameter \ --name "/production/database/password" \ --value "new-super-secret-password" \ --type SecureString \ --overwrite

# Install External Secrets Operator helm repo add external-secrets https://charts.external-secrets.io helm install external-secrets external-secrets/external-secrets \ --namespace external-secrets --create-namespace # Verify installation kubectl get pods -n external-secrets

# Create SecretStore (points to Vault) apiVersion: external-secrets.io/v1beta1 kind: SecretStore metadata: name: vault-backend namespace: production spec: provider: vault: server: "https://vault.vault.svc:8200" path: "secret" version: "v2" auth: kubernetes: mountPath: "kubernetes" role: "production-app" --- # Create ExternalSecret (syncs specific secrets) apiVersion: external-secrets.io/v1beta1 kind: ExternalSecret metadata: name: app-database-secret namespace: production spec: refreshInterval: 1h secretStoreRef: name: vault-backend kind: SecretStore target: name: app-database-secret # Name of resulting K8s Secret creationPolicy: Owner data: - secretKey: DB_PASSWORD # Key in K8s Secret remoteRef: key: production/database # Path in Vault property: password # Field in Vault secret

# Create SecretStore (points to SSM) apiVersion: external-secrets.io/v1beta1 kind: SecretStore metadata: name: aws-parameter-store namespace: production spec: provider: aws: service: ParameterStore region: us-east-1 auth: jwt: # Uses IRSA (IAM Roles for Service Accounts) serviceAccountRef: name: external-secrets-sa --- # Create ExternalSecret apiVersion: external-secrets.io/v1beta1 kind: ExternalSecret metadata: name: app-config namespace: production spec: refreshInterval: 5m secretStoreRef: name: aws-parameter-store kind: SecretStore target: name: app-config dataFrom: - extract: key: /production/app/ # Syncs ALL parameters under this path

#!/bin/bash # migrate-ssm-to-vault.sh set -e VAULT_ADDR="https://vault.example.com" VAULT_TOKEN="your-vault-token" SSM_PATH="/production/" echo "Fetching all SSM parameters under $SSM_PATH..." PARAMETERS=$(aws ssm get-parameters-by-path \ --path "$SSM_PATH" \ --with-decryption \ --recursive \ --output json) echo "$PARAMETERS" | jq -r '.Parameters[] | @json' | while read -r param; do NAME=$(echo "$param" | jq -r '.Name') VALUE=$(echo "$param" | jq -r '.Value') # Convert SSM path to Vault path # /production/database/password -> secret/data/production/database VAULT_PATH=$(echo "$NAME" | sed 's|^/||' | sed 's|/[^/]*$||') KEY=$(basename "$NAME") echo "Migrating $NAME -> secret/data/$VAULT_PATH (key: $KEY)" # Write to Vault (KV v2) vault kv put "secret/$VAULT_PATH" "$KEY=$VALUE" done echo "Migration complete!"

#!/bin/bash # migrate-vault-to-ssm.sh set -e VAULT_ADDR="https://vault.example.com" VAULT_TOKEN="your-vault-token" VAULT_PATH="secret/production" echo "Fetching all secrets from Vault at $VAULT_PATH..." SECRETS=$(vault kv list -format=json "$VAULT_PATH") echo "$SECRETS" | jq -r '.[]' | while read -r secret; do echo "Processing $VAULT_PATH/$secret..." # Get secret data SECRET_DATA=$(vault kv get -format=json "$VAULT_PATH/$secret") # Extract key-value pairs echo "$SECRET_DATA" | jq -r '.data.data | to_entries[] | @json' | while read -r kv; do KEY=$(echo "$kv" | jq -r '.key') VALUE=$(echo "$kv" | jq -r '.value') # Convert to SSM path SSM_NAME="/production/$secret/$KEY" echo "Writing to SSM: $SSM_NAME" aws ssm put-parameter \ --name "$SSM_NAME" \ --value "$VALUE" \ --type SecureString \ --overwrite done done echo "Migration complete!"