Kubernetes Secrets

// ❌ BAD — NEVER DO THIS
const client = new Client({
  host: 'postgres.example.com',
  user: 'admin',
  password: 'SuperSecret123!',   // Hardcoded in source code!
  database: 'myapp'
});

# ❌ Still bad — the password is visible in docker inspect
ENV DB_PASSWORD=SuperSecret123!

# ❌ Still bad — password in plain text in your YAML file (which lives in Git!)
env:
- name: DB_PASSWORD
  value: "SuperSecret123!"

# ✅ GOOD — The actual password lives in a Secret object, not your app YAML
env:
- name: DB_PASSWORD
  valueFrom:
    secretKeyRef:
      name: db-credentials    # name of the Secret object
      key: password           # key inside the Secret

# Encoding
echo -n "MyPassword123" | base64
# Output: TXlQYXNzd29yZDEyMw==

# Decoding
echo "TXlQYXNzd29yZDEyMw==" | base64 -d
# Output: MyPassword123

apiVersion: v1
kind: Secret
metadata:
  name: my-app-secrets
type: Opaque                      # ← default type
data:
  db-password: U3VwZXJTZWNyZXQhMjM=    # base64 encoded
  api-key: c2tfdGVzdF9hYmMxMjM=

# Every namespace has a default service account with this token
kubectl get secrets -n default
# NAME                  TYPE                                  DATA
# default-token-abc12   kubernetes.io/service-account-token   3

kubectl create secret docker-registry my-registry-secret \
  --docker-server=registry.example.com \
  --docker-username=myuser \
  --docker-password=mypassword \
  --docker-email=me@example.com

spec:
  imagePullSecrets:
  - name: my-registry-secret
  containers:
  - name: my-app
    image: registry.example.com/myapp:latest

kubectl create secret tls my-tls-secret \
  --cert=path/to/tls.crt \
  --key=path/to/tls.key

# Used in Ingress
spec:
  tls:
  - hosts:
    - myapp.example.com
    secretName: my-tls-secret

apiVersion: v1
kind: Secret
type: kubernetes.io/basic-auth
data:
  username: YWRtaW4=       # admin
  password: cGFzc3dvcmQ=   # password

apiVersion: v1
kind: Secret
type: kubernetes.io/ssh-auth
data:
  ssh-privatekey: <base64 encoded private key>

# Create a generic (Opaque) secret with literal values
kubectl create secret generic db-credentials \
  --from-literal=username=admin \
  --from-literal=password='Sup3rS3cr3t!'

# Verify it was created
kubectl get secret db-credentials
# NAME             TYPE     DATA   AGE
# db-credentials   Opaque   2      5s

# View the secret (values are base64 encoded)
kubectl get secret db-credentials -o yaml

apiVersion: v1
data:
  password: U3VwM3JTM2NyM3Qh
  username: YWRtaW4=
kind: Secret
metadata:
  name: db-credentials
  namespace: default
type: Opaque

# Create secret from a file
echo -n "Sup3rS3cr3t!" > ./db_password.txt
kubectl create secret generic db-credentials \
  --from-file=password=./db_password.txt

# Create from multiple files
kubectl create secret generic app-certs \
  --from-file=tls.crt=./server.crt \
  --from-file=tls.key=./server.key

apiVersion: v1
kind: Secret
metadata:
  name: db-credentials
  namespace: production
type: Opaque
data:
  username: YWRtaW4=           # echo -n "admin" | base64
  password: U3VwM3JTM2NyM3Qh   # echo -n "Sup3rS3cr3t!" | base64

apiVersion: v1
kind: Secret
metadata:
  name: db-credentials
  namespace: production
type: Opaque
stringData:          # ← Use stringData for human-readable YAML
  username: admin
  password: Sup3rS3cr3t!

# Get a specific secret value and decode it
kubectl get secret db-credentials \
  -o jsonpath='{.data.password}' | base64 -d

# Output: Sup3rS3cr3t!

apiVersion: v1
kind: Pod
metadata:
  name: my-app-pod
spec:
  containers:
  - name: my-app
    image: my-app:1.0
    env:
    # Inject a single key from a Secret
    - name: DB_PASSWORD          # env var name in the container
      valueFrom:
        secretKeyRef:
          name: db-credentials   # Secret object name
          key: password          # key inside the Secret

    - name: DB_USERNAME
      valueFrom:
        secretKeyRef:
          name: db-credentials
          key: username

    # Or inject ALL keys from a Secret as env vars at once
    envFrom:
    - secretRef:
        name: db-credentials     # All keys become env vars

import os
db_password = os.environ.get('DB_PASSWORD')

const dbPassword = process.env.DB_PASSWORD;

apiVersion: v1
kind: Pod
metadata:
  name: my-app-pod
spec:
  volumes:
  # Declare the secret as a volume
  - name: secret-volume
    secret:
      secretName: db-credentials   # Secret object name

  containers:
  - name: my-app
    image: my-app:1.0
    volumeMounts:
    # Mount the volume into the container filesystem
    - name: secret-volume
      mountPath: /etc/secrets      # Directory in the container
      readOnly: true               # Always mount secrets as readOnly

with open('/etc/secrets/password', 'r') as f:
    db_password = f.read().strip()

const fs = require('fs');
const dbPassword = fs.readFileSync('/etc/secrets/password', 'utf8').trim();

volumes:
- name: secret-volume
  secret:
    secretName: db-credentials
    items:
    - key: password
      path: db/password.txt    # mounts as /etc/secrets/db/password.txt
      mode: 0400               # read-only for owner only
    - key: username
      path: db/username.txt

apiVersion: v1
kind: Pod
spec:
  imagePullSecrets:
  - name: my-registry-secret   # Secret of type kubernetes.io/dockerconfigjson
  containers:
  - name: private-app
    image: private-registry.example.com/my-app:latest

# Secret
apiVersion: v1
kind: Secret
metadata:
  name: postgres-secret
stringData:
  DB_HOST: postgres-service
  DB_PORT: "5432"
  DB_NAME: myapp
  DB_USER: appuser
  DB_PASSWORD: Secr3tP@ss!
---
# Deployment using the secret
apiVersion: apps/v1
kind: Deployment
metadata:
  name: backend-api
spec:
  template:
    spec:
      containers:
      - name: api
        image: my-backend:v1.2
        envFrom:
        - secretRef:
            name: postgres-secret

apiVersion: v1
kind: Secret
metadata:
  name: external-api-keys
stringData:
  STRIPE_SECRET_KEY: sk_live_abc123xyz
  SENDGRID_API_KEY: SG.abc123...
  TWILIO_AUTH_TOKEN: abc123...

# Create TLS secret
kubectl create secret tls myapp-tls \
  --cert=certs/tls.crt \
  --key=certs/tls.key

# Use in Ingress
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: myapp-ingress
spec:
  tls:
  - hosts:
    - myapp.example.com
    secretName: myapp-tls     # ← TLS Secret
  rules:
  - host: myapp.example.com
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: myapp-service
            port:
              number: 80

# For AWS ECR
kubectl create secret docker-registry ecr-secret \
  --docker-server=123456789.dkr.ecr.us-east-1.amazonaws.com \
  --docker-username=AWS \
  --docker-password=$(aws ecr get-login-password --region us-east-1)

apiVersion: v1
kind: Secret
metadata:
  name: git-credentials
type: kubernetes.io/basic-auth
stringData:
  username: git-bot-user
  password: ghp_token_here

apiVersion: v1
kind: Secret
metadata:
  name: jwt-secret
stringData:
  JWT_SECRET: "a-very-long-and-random-string-used-for-hmac-signing"
  SESSION_SECRET: "another-random-secret-for-express-sessions"

# ✅ ConfigMap — non-sensitive configuration
apiVersion: v1
kind: ConfigMap
metadata:
  name: app-config
data:
  LOG_LEVEL: "info"
  MAX_CONNECTIONS: "100"
  APP_ENV: "production"
  ALLOWED_ORIGINS: "https://myapp.com,https://api.myapp.com"

---
# ✅ Secret — sensitive credentials
apiVersion: v1
kind: Secret
metadata:
  name: app-secrets
stringData:
  DB_PASSWORD: "Secr3tP@ssw0rd"
  JWT_SECRET: "hmac-signing-key-abc123"
  REDIS_PASSWORD: "redis-pass-xyz"

# /etc/kubernetes/encryption-config.yaml
apiVersion: apiserver.config.k8s.io/v1
kind: EncryptionConfiguration
resources:
- resources:
  - secrets
  providers:
  - aescbc:
      keys:
      - name: key1
        secret: <base64-encoded-32-byte-key>
  - identity: {}   # fallback — allows reading unencrypted (during migration)

# Only allow the backend service account to read specific secrets
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: backend-secret-reader
  namespace: production
rules:
- apiGroups: [""]
  resources: ["secrets"]
  resourceNames: ["db-credentials", "jwt-secret"]  # specific secrets only
  verbs: ["get"]    # read-only — no list, no create
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: backend-reads-secrets
  namespace: production
subjects:
- kind: ServiceAccount
  name: backend-sa
  namespace: production
roleRef:
  kind: Role
  name: backend-secret-reader
  apiGroup: rbac.authorization.k8s.io

# Update a secret value (patch in place)
kubectl patch secret db-credentials \
  --type='json' \
  -p='[{"op": "replace", "path": "/data/password", "value": "'$(echo -n "NewP@ssw0rd!" | base64)'"}]'

# Or recreate it
kubectl delete secret db-credentials
kubectl create secret generic db-credentials \
  --from-literal=password='NewP@ssw0rd!'

volumeMounts:
- name: secret-volume
  mountPath: /etc/secrets
  readOnly: true   # ← Always add this

volumes:
- name: secret-volume
  secret:
    secretName: db-credentials
    defaultMode: 0400    # ← Owner read-only (octal)

# ❌ BAD — logs the actual secret
print(f"Connecting with password: {os.environ.get('DB_PASSWORD')}")

# ✅ GOOD — only confirm it's set
print(f"DB password set: {'yes' if os.environ.get('DB_PASSWORD') else 'no'}")

kubectl get pods
# NAME          READY   STATUS                       RESTARTS   AGE
# my-app-pod    0/1     CreateContainerConfigError   0          30s

kubectl describe pod my-app-pod

# Check which namespace the pod is in
kubectl get pod my-app-pod -o jsonpath='{.metadata.namespace}'
# Output: production

# Check if the secret exists in that namespace
kubectl get secret db-credentials -n production
# Error: secrets "db-credentials" not found

# The secret exists in 'default' but not 'production'!
kubectl get secret db-credentials -n default
# NAME             TYPE     DATA   AGE
# db-credentials   Opaque   2      1d

# Fix: Recreate the secret in the correct namespace
kubectl get secret db-credentials -n default -o yaml \
  | sed 's/namespace: default/namespace: production/' \
  | kubectl apply -f -

kubectl create secret generic db-credentials \
  --from-literal=password='Sup3rS3cr3t!' \
  -n production    # ← always specify namespace

kubectl logs my-app-pod
# Error: environment variable DB_PASSWORD is empty or not set

kubectl describe pod my-app-pod
# Events:
#   Warning  Failed  kubelet  Error: couldn't find key password in Secret default/db-credentials

# Check what keys actually exist in the Secret
kubectl get secret db-credentials -o jsonpath='{.data}' | python3 -m json.tool
# {
#   "db_password": "U3VwM3JTM2NyM3Qh",   ← underscore!
#   "db_username": "YWRtaW4="
# }

secretKeyRef:
  name: db-credentials
  key: password    # ← looking for "password" but Secret has "db_password"

# Get current value
CURRENT_PASS=$(kubectl get secret db-credentials -o jsonpath='{.data.db_password}' | base64 -d)

# Recreate with correct key name
kubectl delete secret db-credentials
kubectl create secret generic db-credentials \
  --from-literal=password="$CURRENT_PASS" \
  --from-literal=username="admin"

secretKeyRef:
  name: db-credentials
  key: db_password   # ← match the actual key name

# Check the raw base64 value
kubectl get secret db-credentials -o jsonpath='{.data.password}'
# U3VwM3JTM2NyM3Qh==

# Decode it
kubectl get secret db-credentials -o jsonpath='{.data.password}' | base64 -d
# Sup3rS3cr3t!%    ← notice the % (trailing newline!) on some terminals

# ❌ BAD — includes a newline character
echo "Sup3rS3cr3t!" | base64
# U3VwM3JTM2NyM3Qh Cg==  ← "Cg==" is the base64 of "\n"

# ✅ GOOD — -n flag suppresses trailing newline
echo -n "Sup3rS3cr3t!" | base64
# U3VwM3JTM2NyM3Qh

kubectl delete secret db-credentials

# Use --from-literal (kubectl handles encoding correctly)
kubectl create secret generic db-credentials \
  --from-literal=password='Sup3rS3cr3t!'

# Strip trailing newline from file
printf '%s' 'Sup3rS3cr3t!' > ./password.txt   # printf has no trailing newline

kubectl create secret generic db-credentials \
  --from-file=password=./password.txt

# You updated the secret
kubectl patch secret db-credentials \
  --type='merge' \
  -p '{"stringData": {"password": "NewP@ssw0rd2026!"}}'

# But the app still uses the old password
kubectl exec my-app-pod -- env | grep DB_PASSWORD
# DB_PASSWORD=OldP@ssw0rd!   ← still old value!

# Restart all Pods in the Deployment to pick up new env var values
kubectl rollout restart deployment/my-app

# Watch the rollout
kubectl rollout status deployment/my-app
# Waiting for deployment "my-app" rollout to finish...
# deployment "my-app" successfully rolled out

# Instead of env vars, use volume mounts for secrets that rotate
volumeMounts:
- name: db-secret
  mountPath: /etc/secrets
  readOnly: true
volumes:
- name: db-secret
  secret:
    secretName: db-credentials

kubectl get pods
# NAME        READY   STATUS             RESTARTS
# my-app      0/1     ImagePullBackOff   0

kubectl describe pod my-app
# Events:
#   Warning  Failed  kubelet  Failed to pull image "registry.example.com/my-app:latest":
#            rpc error: unauthorized: authentication required

# Step 1: Create the registry secret
kubectl create secret docker-registry registry-credentials \
  --docker-server=registry.example.com \
  --docker-username=myuser \
  --docker-password=mypassword \
  -n your-namespace

# Step 2: Verify the secret was created
kubectl get secret registry-credentials -o json | \
  python3 -c "import sys,json,base64; \
  d=json.load(sys.stdin); \
  print(base64.b64decode(d['data']['.dockerconfigjson']).decode())"
# Should show: {"auths":{"registry.example.com":{"username":"myuser",...}}}

# Step 3: Add imagePullSecrets to your Pod/Deployment
kubectl patch deployment my-app --patch \
  '{"spec":{"template":{"spec":{"imagePullSecrets":[{"name":"registry-credentials"}]}}}}'

# Step 4: Watch pods recover
kubectl get pods -w

kubectl patch serviceaccount default \
  -p '{"imagePullSecrets": [{"name": "registry-credentials"}]}'

kubectl get secret db-credentials
# Error from server (Forbidden): secrets "db-credentials" is forbidden:
# User "ci-bot" cannot get resource "secrets" in API group ""

# Check what permissions ci-bot has
kubectl auth can-i get secrets --as=ci-bot -n production
# no

kubectl auth can-i list secrets --as=ci-bot -n production
# no

# Create a Role that allows reading specific secrets
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: ci-secret-reader
  namespace: production
rules:
- apiGroups: [""]
  resources: ["secrets"]
  resourceNames: ["db-credentials"]   # restrict to only this secret
  verbs: ["get"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: ci-reads-db-secret
  namespace: production
subjects:
- kind: User
  name: ci-bot
  apiGroup: rbac.authorization.k8s.io
roleRef:
  kind: Role
  name: ci-secret-reader
  apiGroup: rbac.authorization.k8s.io

kubectl apply -f ci-secret-rbac.yaml

# Verify
kubectl auth can-i get secrets/db-credentials --as=ci-bot -n production
# yes

kubectl logs my-app-pod
# Error: EACCES: permission denied, open '/etc/secrets/password'

kubectl exec my-app-pod -- ls -la /etc/secrets/
# -r--------  1 root root 13 Jan 20 10:00 password    ← only root can read!

volumes:
- name: secret-volume
  secret:
    secretName: db-credentials
    defaultMode: 0444    # ← world-readable (for non-sensitive but restricted info)
    # OR
    defaultMode: 0440    # ← owner + group readable

spec:
  securityContext:
    fsGroup: 1000        # ← files will be group-owned by GID 1000
  containers:
  - name: my-app
    securityContext:
      runAsUser: 1000    # ← container runs as UID 1000
    volumeMounts:
    - name: secret-volume
      mountPath: /etc/secrets
      readOnly: true

# 1. Check if Secret exists in correct namespace
kubectl get secrets -n <namespace>

# 2. Check Secret keys and their (base64) values
kubectl get secret <name> -o yaml -n <namespace>

# 3. Decode a specific key
kubectl get secret <name> -n <namespace> \
  -o jsonpath='{.data.<key>}' | base64 -d

# 4. Check Pod events for Secret-related errors
kubectl describe pod <pod-name> | grep -A 20 "Events:"

# 5. Verify env vars inside a running container
kubectl exec <pod-name> -- env | grep <VAR_NAME>

# 6. Check if Secret is mounted correctly
kubectl exec <pod-name> -- ls -la /etc/secrets/
kubectl exec <pod-name> -- cat /etc/secrets/password

# 7. Check RBAC permissions
kubectl auth can-i get secrets -n <namespace>
kubectl auth can-i get secrets --as=<user> -n <namespace>

# 8. Describe the secret (shows metadata, not values)
kubectl describe secret <name> -n <namespace>

# 9. Force restart deployment after secret rotation
kubectl rollout restart deployment/<name> -n <namespace>

# 10. Check etcd encryption (admin only)
kubectl get secret <name> -n <namespace> -o json | \
  python3 -c "import sys,json,base64; \
  d=json.load(sys.stdin); \
  [print(k,'=', base64.b64decode(v).decode()) \
   for k,v in d['data'].items()]"

# Create
kubectl create secret generic <name> --from-literal=key=value
kubectl create secret generic <name> --from-file=key=./file.txt
kubectl create secret tls <name> --cert=tls.crt --key=tls.key
kubectl create secret docker-registry <name> --docker-server=... --docker-username=... --docker-password=...

# Read
kubectl get secrets
kubectl get secret <name> -o yaml
kubectl get secret <name> -o jsonpath='{.data.<key>}' | base64 -d

# Update
kubectl patch secret <name> --type='merge' -p '{"stringData":{"key":"newvalue"}}'

# Delete
kubectl delete secret <name>

# Restart Pods to pick up env var changes
kubectl rollout restart deployment/<name>