Kubernetes Interview Questions & Answers — Scenarios & Troubleshooting

# Step 1: Check pod status and events
kubectl describe pod <pod-name> -n <namespace>
# Look at: Events section, Last State, Exit Code

# Step 2: Check logs (current and previous container)
kubectl logs <pod-name>
kubectl logs <pod-name> --previous    # Logs from crashed container

# Step 3: Check exit code
# Exit 0 = success (but CMD finished unexpectedly)
# Exit 1 = application error
# Exit 137 = OOMKilled
# Exit 139 = segfault

# Step 4: Get into the container temporarily
kubectl exec -it <pod-name> -- sh

# Step 5: If container exits too fast to exec, override CMD
kubectl run debug --image=myimage --command -- sleep 3600
kubectl exec -it debug -- sh

# Step 6: Check resource limits
kubectl describe pod <pod-name> | grep -A5 Limits

# Step 7: Check liveness probe config
kubectl describe pod <pod-name> | grep -A10 Liveness

# Common causes:
# - Wrong command/entrypoint
# - Missing env vars or config
# - OOMKilled (need more memory)
# - App crashes on startup
# - Liveness probe too aggressive

# Step 1: Verify service exists and has correct port
kubectl get service my-service
kubectl describe service my-service

# Step 2: Check endpoints (are pods registered?)
kubectl get endpoints my-service
# If NO endpoints → selector mismatch or pods not Running

# Step 3: Verify pod labels match service selector
kubectl get pods --show-labels
kubectl describe service my-service | grep Selector
# Both must match!

# Step 4: Test from within the cluster (using a debug pod)
kubectl run test --image=busybox --rm -it -- sh
wget -O- http://my-service:80
nslookup my-service

# Step 5: Test DNS resolution
nslookup my-service.default.svc.cluster.local

# Step 6: Check pod is Running and ready
kubectl get pods -l app=myapp
# STATUS must be Running, READY must be 1/1

# Step 7: Check NetworkPolicies blocking traffic
kubectl get networkpolicies -n <namespace>
kubectl describe networkpolicy <policy-name>

# Step 8: Test directly to pod IP (bypass service)
POD_IP=$(kubectl get pod <pod-name> -o jsonpath='{.status.podIP}')
kubectl run test --image=busybox --rm -it -- wget -O- http://$POD_IP:8080

# Step 1: Immediately roll back (fastest)
kubectl rollout undo deployment/my-deployment -n production

# Step 2: Verify rollback is in progress
kubectl rollout status deployment/my-deployment -n production

# Step 3: Confirm old pods are running
kubectl get pods -n production

# Step 4: Check rollout history
kubectl rollout history deployment/my-deployment

# Step 5: Roll back to a specific revision
kubectl rollout undo deployment/my-deployment --to-revision=3

# Step 6: If rollback is slow, force a faster restart
kubectl rollout restart deployment/my-deployment

# Step 7: Verify application is responding
kubectl get svc my-service
curl http://<service-ip>/health

# Prevention for next time:
# - Set minReadySeconds to delay traffic routing
# - Use readiness probes
# - Use PodDisruptionBudget
# - Add deployment annotation: kubernetes.io/change-cause: "version 2.1 - added feature X"

# Step 1: Check rollout state and history
kubectl rollout status deployment/payments-api -n prod
kubectl rollout history deployment/payments-api -n prod

# Step 2: A Deployment never touches Pods directly - it creates a NEW ReplicaSet
# for the new template. Find both ReplicaSets:
kubectl get rs -n prod -l app=payments-api
# OLD ReplicaSet: still at (near) full replica count, Pods Running
# NEW ReplicaSet: replicas created, but Pods not Ready

# Step 3: Inspect the failing pods from the NEW ReplicaSet
kubectl describe pod <new-pod> -n prod
kubectl logs <new-pod> -n prod --previous
kubectl get events -n prod --sort-by='.lastTimestamp'

# Step 4: Check the readiness/liveness probe config tied to the new image
kubectl describe pod <new-pod> -n prod | grep -A5 Readiness

# Fastest mitigation while you fix the image:
kubectl rollout undo deployment/payments-api -n prod

# Cause 1: Using 'latest' tag (most common)
# K8s only pulls 'latest' if imagePullPolicy: Always
# Fix: Always use specific version tags
image: myapp:v1.2.3   # Never: myapp:latest in production

# Cause 2: imagePullPolicy not set correctly
# Default behavior:
# - :latest tag → Always pull
# - other tags → IfNotPresent (uses cached image!)
spec:
  containers:
  - name: app
    image: myapp:v1.2.3
    imagePullPolicy: Always   # Force pull on every start

# Cause 3: Image not pushed to registry before deploy
# Ensure CI pipeline order:
# 1. docker build
# 2. docker push ← must complete before step 3
# 3. kubectl set image

# Cause 4: Wrong image name/tag in deployment
kubectl describe deployment my-deployment | grep Image

# Cause 5: Old ReplicaSet running (deployment not updated)
kubectl get replicasets
kubectl rollout status deployment/my-deployment

# Cause 6: Cached image on node — delete pod to force fresh pull
kubectl delete pod <pod-name>   # New pod will pull fresh
# Or: kubectl rollout restart deployment/my-deployment

# Step 1: Always start with describe — read Events section
kubectl describe pod <pod-name> -n <namespace>
# Focus on the LAST few lines of the Events section

# ─── COMMON PENDING REASONS AND FIXES ───────────────────────────────

# Error 1: "0/3 nodes are available: 3 Insufficient memory"
# → Node doesn't have enough resources
kubectl describe nodes | grep -A 10 "Allocated resources"
kubectl top nodes
# Fix: Reduce pod requests, scale up node group, or delete unused pods

# Error 2: "0/3 nodes are available: 3 node(s) had taint {key: value}"
# → Pod doesn't tolerate node taints
kubectl get nodes -o custom-columns='NAME:.metadata.name,TAINTS:.spec.taints'
# Fix: Add toleration to pod spec

# Error 3: "0/3 nodes are available: 3 node(s) didn't match Pod's node affinity/selector"
# → Node labels don't match pod's nodeSelector or nodeAffinity
kubectl get nodes --show-labels
# Fix: Correct nodeSelector/affinity or add missing labels to nodes

# Error 4: "pod has unbound immediate PersistentVolumeClaims"
# → PVC not bound to a PV
kubectl get pvc -n <namespace>
kubectl describe pvc <pvc-name> -n <namespace>
# Fix: Create StorageClass, create PV, or wait for dynamic provisioning

# Error 5: "persistentvolumeclaim ... not found"
# → PVC doesn't exist at all
# Fix: kubectl apply -f pvc.yaml

# Error 6: "Maximum number of Pods is already running"
# → Node has hit pod limit (default 110 pods/node)
kubectl describe node <node> | grep "pods:"
# Fix: Add more nodes, reduce pods per node, or increase --max-pods kubelet flag

# Check scheduler events specifically
kubectl get events -n <namespace> \
  --field-selector reason=FailedScheduling \
  --sort-by='.lastTimestamp'

# Check if ResourceQuota is blocking
kubectl get resourcequota -n <namespace>
kubectl describe resourcequota -n <namespace>

# Check LimitRange
kubectl get limitrange -n <namespace>
kubectl describe limitrange -n <namespace>

# Step 1: Verify pod and service exist
kubectl get pods -l app=service-b -n production
kubectl get svc service-b -n production

# Step 2: Check endpoints — are any pods backing the service?
kubectl get endpoints service-b -n production
# NAME        ENDPOINTS           AGE
# service-b   <none>              5m  ← PROBLEM: empty endpoints

# Diagnose empty endpoints:
# a) Get service selector
kubectl get svc service-b -o jsonpath='{.spec.selector}' -n production
# {"app":"service-b","version":"v1"}

# b) Check if any pods match ALL selector labels
kubectl get pods -n production -l app=service-b,version=v1 --show-labels
# If no pods found → label mismatch!

# c) Check pod readiness (unready pods are excluded from endpoints)
kubectl get pods -n production -l app=service-b
# If READY=0/1 → pod failing readiness probe

# Step 3: Test DNS resolution from pod-a
kubectl exec -it pod-a -n production -- nslookup service-b
kubectl exec -it pod-a -n production -- \
  nslookup service-b.production.svc.cluster.local

# Step 4: Test port connectivity
kubectl exec -it pod-a -n production -- \
  nc -zv service-b 8080
# or:
kubectl exec -it pod-a -n production -- \
  curl -v http://service-b:8080/health

# Step 5: Check NetworkPolicies blocking traffic
kubectl get networkpolicy -n production
kubectl describe networkpolicy <policy-name> -n production

# Step 6: Debug with ephemeral container (K8s 1.23+)
kubectl debug -it pod-a -n production \
  --image=nicolaka/netshoot \
  --target=app-container

# Inside netshoot: full network debugging toolkit
curl -v http://service-b:8080
tcpdump -i eth0 host service-b
nmap -p 8080 service-b

# Check node pressure
kubectl get nodes
# If STATUS shows "MemoryPressure" or "DiskPressure" — node is struggling

kubectl describe node <node-name>
# Look at: Conditions, Allocated Resources, Events

# Find the top CPU consumers
kubectl top pods -A --sort-by=cpu | head -20

# Find the top memory consumers
kubectl top pods -A --sort-by=memory | head -20

# Check container-level usage
kubectl top pods -A --containers --sort-by=memory | head -30

# Find pods near their memory limits (risk of OOMKill)
kubectl get pods -A -o json | jq -r '
  .items[] |
  .metadata.namespace + "/" + .metadata.name
' | while read pod; do
  ns=$(echo $pod | cut -d/ -f1)
  name=$(echo $pod | cut -d/ -f2)
  kubectl top pod $name -n $ns --containers 2>/dev/null
done

# PromQL queries for investigation:
# Top memory consumers:
# topk(10, container_memory_working_set_bytes{container!="POD"})

# Memory usage % of limit:
# container_memory_working_set_bytes / container_spec_memory_limit_bytes > 0.8

# OOMKill history:
# increase(kube_pod_container_status_restarts_total[24h]) > 0
# kube_pod_container_status_last_terminated_reason{reason="OOMKilled"} == 1

# Find and clean up resources wasting cluster capacity
# Deployments with 0 replicas
kubectl get deployments -A --field-selector=spec.replicas=0

# Completed/failed pods not cleaned up
kubectl get pods -A --field-selector=status.phase=Succeeded
kubectl get pods -A --field-selector=status.phase=Failed

# Clean up completed pods older than 1 day
kubectl get pods -A --field-selector=status.phase=Succeeded -o json | \
  jq -r '.items[] | select(.status.startTime < "2024-01-14") | .metadata.namespace + "/" + .metadata.name'

# Step 1: See the restart count and last exit reason at a glance
kubectl get pod my-pod
# STATUS: CrashLoopBackOff   RESTARTS: 7

# Step 2: Describe the pod — check Events at the bottom and the exit code
kubectl describe pod my-pod
# Look for: "Last State: Terminated, Reason: Error, Exit Code: 1"

# Step 3: Check logs from the CURRENT crashed attempt
kubectl logs my-pod

# Step 4: Check logs from the PREVIOUS attempt (often more useful —
# the current container may have crashed before logging anything)
kubectl logs my-pod --previous

# Step 5: Check resource limits — was it OOMKilled?
kubectl describe pod my-pod | grep -A5 "Last State"
# Exit Code 137 = OOMKilled → increase memory limits or fix a leak

# Step 6: Check readiness/liveness probes — a misconfigured probe can
# kill an otherwise-healthy container
kubectl get pod my-pod -o yaml | grep -A10 livenessProbe

# Step 7: Verify the image, command, and config are correct
kubectl get pod my-pod -o jsonpath='{.spec.containers[0].image}'
kubectl get configmap,secret -n <namespace>   # confirm referenced ones exist

# Step 8: If logs are empty, shell in via a debug container (image itself won't stay up)
kubectl debug my-pod -it --image=busybox --target=my-container

# Step 1 — describe the pod (most important command)
kubectl describe pod <pod-name> -n <namespace>
# Look at the 'Events' section at the bottom

# Step 2 — check node resources
kubectl describe nodes | grep -A 5 "Allocated resources"
kubectl top nodes

# Step 3 — check if PVC is bound (if pod mounts one)
kubectl get pvc -n <namespace>

# Events will show:
# "0/3 nodes are available: 3 Insufficient cpu"

# Fix: Scale up node group or reduce resource requests
kubectl get nodes
kubectl describe node <node> | grep -A 10 "Allocated resources"

# Check what's using resources
kubectl top pods --all-namespaces --sort-by=cpu

# Events: "0/3 nodes are available: 3 node(s) didn't match Pod's node affinity"

# Check node labels
kubectl get nodes --show-labels

# Add missing label to node
kubectl label node <node-name> disktype=ssd

# Events: "persistentvolumeclaim not found" or PVC stuck in Pending

kubectl describe pvc <pvc-name>
# Check if StorageClass exists
kubectl get storageclass

# Events: "0/3 nodes are available: 3 node(s) had untolerated taint"

kubectl describe nodes | grep -i taint
# Add toleration to pod spec

# Each node has a default limit of 110 pods
kubectl describe node <node> | grep "Non-terminated Pods"

# One command to see all pending pods and their reason
kubectl get pods --all-namespaces --field-selector=status.phase=Pending
kubectl describe pods --all-namespaces | grep -A 10 "Events:"

# Step 1 — describe pod for events and exit codes
kubectl describe pod <pod-name> -n <namespace>

# Step 2 — current logs (may be empty if app crashes immediately)
kubectl logs <pod-name> -n <namespace>

# Step 3 — PREVIOUS container logs (before the crash) — most useful
kubectl logs <pod-name> -n <namespace> --previous

# Step 4 — check exit code
kubectl get pod <pod-name> -o jsonpath='{.status.containerStatuses[0].lastState.terminated}'

# Fix OOMKill (exit 137) — increase memory limit
kubectl patch deployment <name> -p \
  '{"spec":{"template":{"spec":{"containers":[{"name":"app","resources":{"limits":{"memory":"1Gi"}}}]}}}}'

# Fix: App can't connect to database
# Check if DB service is reachable from pod
kubectl exec -it <pod-name> -- nc -zv postgres-svc 5432

# Fix: Wrong image command — override to debug
kubectl run debug-pod \
  --image=<same-image> \
  --restart=Never \
  --rm -it \
  --command -- /bin/sh

# Fix: Liveness probe killing app too early — increase initialDelaySeconds
livenessProbe:
  initialDelaySeconds: 60    # Give app more time to start
  failureThreshold: 5

# Fix: Missing environment variable or secret
kubectl exec -it <pod-name> -- env | grep DB_   # Check env vars
kubectl describe pod <pod-name> | grep -A 5 "Environment"

# Step 1 — check node status and conditions
kubectl get nodes
kubectl describe node <node-name>

# Look for conditions:
# Ready = False/Unknown
# MemoryPressure = True
# DiskPressure = True
# PIDPressure = True
# NetworkUnavailable = True

# Check kubelet status (most common cause)
sudo systemctl status kubelet
sudo journalctl -u kubelet -f --no-pager | tail -50

# Common kubelet errors:
# "failed to get node info" → network issue
# "certificate expired" → renew kubelet certificates
# "PLEG is not healthy" → pod lifecycle event generator issues (often disk pressure)

# Check node resources
df -h              # Disk usage (DiskPressure if >85%)
free -m            # Memory (MemoryPressure)
top                # CPU and process check
ps aux | wc -l     # PID count (PIDPressure if >1000)

# Fix 1: kubelet not running
sudo systemctl restart kubelet

# Fix 2: Disk pressure — clean up
docker system prune -af         # Clean Docker images/containers
crictl rmi --prune              # Clean containerd images
sudo journalctl --vacuum-size=500M    # Clean journal logs

# Fix 3: Certificate expired
sudo kubeadm alpha certs renew all
sudo systemctl restart kubelet

# Fix 4: Network plugin not running
kubectl get pods -n kube-system | grep -E "calico|flannel|cilium"
kubectl delete pod -n kube-system <broken-cni-pod>   # Restart CNI pod

# Fix 5: Node has too many pods — eviction happening
kubectl describe node <node> | grep -i "eviction\|pressure"

# Prevent new pods from scheduling on this node
kubectl cordon <node-name>

# Move existing pods to other nodes
kubectl drain <node-name> --ignore-daemonsets --delete-emptydir-data --force

# After fixing the node, uncordon it
kubectl uncordon <node-name>

# Step 1: Check HPA status
kubectl describe hpa my-hpa
# Look for: current metrics, conditions, events

# Step 2: Check if metrics-server is running
kubectl get pods -n kube-system | grep metrics-server
kubectl top pods    # If this fails → metrics-server issue

# Step 3: Check that pod has resource requests defined
kubectl describe pod <pod-name> | grep -A5 Requests
# HPA REQUIRES resource requests to calculate utilization %
# If requests not set → HPA can't calculate utilization → won't scale

# Step 4: Check if min/max replicas are hit
kubectl get hpa my-hpa
# If REPLICAS = MAXREPLICAS → HPA is at max, can't scale more

# Step 5: Check cooldown periods
# HPA has a stabilization window (default 5 min scale-down, 3 min scale-up)
# Check scaleUp.stabilizationWindowSeconds in HPA spec

# Step 6: Check API server for RBAC issues
kubectl get --raw /apis/metrics.k8s.io/v1beta1/pods

# Fix: Add resource requests to pod spec
resources:
  requests:
    cpu: "100m"
    memory: "128Mi"
  limits:
    cpu: "500m"
    memory: "256Mi"

# Step 1: Describe the pending pod for reasons
kubectl describe pod <pod-name>
# Look in Events section for: Insufficient cpu, Insufficient memory, 
# node(s) had untolerated taint, no matching node selector

# Step 2: Check node resources
kubectl describe nodes
kubectl top nodes   # Requires metrics-server

# Step 3: Check if nodes joined correctly
kubectl get nodes
# STATUS must be Ready

# Step 4: Check node labels (if using nodeSelector)
kubectl get node <node-name> --show-labels
kubectl describe pod <pod-name> | grep "Node-Selectors"

# Step 5: Check taints (new nodes may have taints)
kubectl describe node <node-name> | grep -i taint
# If node has taint: node.kubernetes.io/not-ready → node still initializing

# Step 6: Check for missing tolerations in pod spec
kubectl describe pod <pod-name> | grep -i toleration

# Step 7: EKS-specific: Check node group IAM role permissions
# Node needs: AmazonEKSWorkerNodePolicy, AmazonEC2ContainerRegistryReadOnly

# Step 8: Check if EKS add-ons are healthy (VPC CNI, kube-proxy, CoreDNS)
kubectl get pods -n kube-system

# Step 9: Check if PVC is Pending (if pod requires PVC)
kubectl get pvc
# StorageClass must exist and dynamic provisioning must work

kubectl get ingress -n my-namespace
# If ADDRESS column is empty, the Ingress controller hasn't reconciled it yet

kubectl get pods -n kube-system | grep aws-load-balancer
kubectl logs -n kube-system deployment/aws-load-balancer-controller

kubectl describe ingress my-ingress -n my-namespace

# Does the Service exist?
kubectl get svc my-service -n my-namespace

# Does the Service have any Endpoints (pods selected)?
kubectl get endpoints my-service -n my-namespace
# If ENDPOINTS shows <none>, no pods are matching the Service selector

kubectl get pods --show-labels -n my-namespace
kubectl describe svc my-service -n my-namespace | grep Selector

# Check pod status
kubectl get pods -n production
# Look for pods in CrashLoopBackOff, Error, or OOMKilled state

# Check recent events
kubectl get events -n production --sort-by='.lastTimestamp' | tail -20

# Check pod logs
kubectl logs deployment/my-app -n production --previous

kubectl rollout history deployment/my-app -n production
# REVISION  CHANGE-CAUSE
# 1         Initial deployment
# 2         Updated to v1.1 - added user authentication
# 3         Updated to v1.2 - refactored payment service (CURRENT - broken)

# Rollback to the immediately previous version
kubectl rollout undo deployment/my-app -n production

# OR rollback to a specific revision
kubectl rollout undo deployment/my-app -n production --to-revision=2

# Watch the rollback progress
kubectl rollout status deployment/my-app -n production

kubectl get pods -n production
kubectl describe deployment my-app -n production | grep Image

kubectl annotate deployment/my-app kubernetes.io/change-cause="v1.2 - payment service refactor" -n production

helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
helm install monitoring prometheus-community/kube-prometheus-stack \
  --namespace monitoring \
  --create-namespace \
  --set grafana.adminPassword=your-secure-password

kubectl port-forward svc/monitoring-grafana 3000:80 -n monitoring
# Open http://localhost:3000

apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: pod-crash-alert
  namespace: monitoring
spec:
  groups:
  - name: pod-alerts
    rules:
    - alert: PodCrashLooping
      expr: rate(kube_pod_container_status_restarts_total[5m]) > 0
      for: 5m
      labels:
        severity: critical
      annotations:
        summary: "Pod {{ $labels.pod }} is crash looping"
        description: "Pod {{ $labels.pod }} in namespace {{ $labels.namespace }} has been restarting frequently."

# Check what instance types your nodes are (GPU instances = very expensive if underused)
kubectl get nodes -o custom-columns=NAME:.metadata.name,TYPE:.metadata.labels.node\\.kubernetes\\.io/instance-type

# Find pods with no resource requests (they may be over-provisioned)
kubectl get pods --all-namespaces -o json | jq '.items[] | select(.spec.containers[].resources.requests == null) | .metadata.name'

# Use VPA in recommendation mode to see what pods actually use
kubectl get vpa --all-namespaces

# In Karpenter NodePool
spec:
  template:
    spec:
      requirements:
      - key: "karpenter.sh/capacity-type"
        operator: In
        values: ["spot", "on-demand"]   # prefer spot, fallback to on-demand

# Check available resources on each node
kubectl describe nodes | grep -A 5 "Allocated resources"

kubectl describe nodes | grep Taints

kubectl get nodes --show-labels

kubectl get pvc -n <namespace>
# STATUS should be "Bound" not "Pending"

kubectl describe resourcequota -n <namespace>

kubectl logs -n kube-system deployment/cluster-autoscaler | tail -50

kubectl describe pod <pod-name> -n <namespace>

kubectl logs <pod-name> -n <namespace> --previous

# Check memory limit vs actual usage
kubectl top pod <pod-name> -n <namespace>
# Increase the memory limit in the Deployment spec

kubectl describe pod <pod-name> | grep "Failed to pull"

kubectl describe pod <pod-name> | grep -A 10 "Liveness:"

# Override the entrypoint to get a shell instead of running the crashing app
kubectl run debug-pod \
  --image=my-crashing-app:v1.0 \
  --restart=Never \
  -it \
  --command -- /bin/sh
# Now you can explore the container environment manually

kubectl top pods -n production --sort-by=cpu
kubectl top nodes

kubectl exec -it <pod-name> -n production -- /bin/sh
# Time a request to the database
time nc -zv postgres-service 5432

# Time an external API call
time curl -o /dev/null -s -w "%{time_total}" https://api.external-service.com/health

# Check CoreDNS pods
kubectl get pods -n kube-system | grep coredns

# Check CoreDNS logs for errors
kubectl logs -n kube-system -l k8s-app=kube-dns

# Increase CoreDNS replicas if needed
kubectl scale deployment coredns -n kube-system --replicas=4

# Check if there are packet drops at the node level
kubectl debug node/<node-name> -it --image=nicolaka/netshoot
# Inside the debug container:
netstat -s | grep -i "packet"

kubectl rollout history deployment/my-app -n production
# If a recent rollout correlates with the spike, rollback
kubectl rollout undo deployment/my-app -n production

# Step 1: Get overview
kubectl get pods -n production
# NAME           READY   STATUS             RESTARTS   AGE
# app-xxx-yyy    0/1     CrashLoopBackOff   8          12m

# Step 2: Describe pod — read Events section carefully
kubectl describe pod app-xxx-yyy -n production
# Look for:
#   Exit Code (137=OOM, 1=app error, 139=segfault, 143=SIGTERM)
#   Last State: reason, exitCode, finishedAt
#   Events: Failed to pull image, Failed to mount volume, etc.

# Step 3: Check CURRENT logs
kubectl logs app-xxx-yyy -n production

# Step 4: Check PREVIOUS container logs (before crash — most useful!)
kubectl logs app-xxx-yyy -n production --previous

# Step 5: Check resource pressure
kubectl top pod app-xxx-yyy -n production
kubectl describe pod app-xxx-yyy | grep -A5 "Limits\|Requests"

# Step 6: Override entrypoint to keep container alive for debugging
kubectl run debug \
  --image=your-app-image:tag \
  --restart=Never \
  --command -- sleep 3600
kubectl exec -it debug -- /bin/sh

# Step 7: Check events across namespace
kubectl get events -n production --sort-by='.lastTimestamp' | tail -20

# Ephemeral containers (K8s 1.23+ GA) — inject a debug container into a running pod
# The debug container shares the same network, PID namespace as the target container

# Basic debug with busybox
kubectl debug -it <pod-name> \
  --image=busybox:1.35 \
  --target=<container-name>   # --target shares PID namespace

# Debug with netshoot (full network toolkit)
kubectl debug -it <pod-name> \
  --image=nicolaka/netshoot \
  --target=main-app

# Debug with a copy of the pod (with modifications)
kubectl debug <pod-name> \
  -it \
  --copy-to=debug-pod \
  --image=myapp:debug \    # Override with debug-enabled image
  --share-processes        # Share PID namespace to see original app process

# Debug a node (runs privileged container with host filesystem access)
kubectl debug node/<node-name> \
  -it \
  --image=ubuntu:22.04

# Inside node debug pod:
# chroot /host   → access the full node filesystem
# systemctl status kubelet
# journalctl -u kubelet -f

# Alternative: inject a debug sidecar via strategic merge patch
kubectl patch pod <pod-name> -n production \
  --patch '{"spec":{"ephemeralContainers":[{
    "name":"debug",
    "image":"nicolaka/netshoot",
    "stdin":true,
    "tty":true,
    "targetContainerName":"main-app"
  }]}}'

kubectl attach <pod-name> -c debug -it

# Install Jaeger (distributed tracing)
helm repo add jaegertracing https://jaegertracing.github.io/helm-charts
helm install jaeger jaegertracing/jaeger \
  --namespace tracing \
  --create-namespace \
  --set allInOne.enabled=true \
  --set provisionDataStore.cassandra=false

# OpenTelemetry Collector — collects traces from all services
apiVersion: opentelemetry.io/v1alpha1
kind: OpenTelemetryCollector
metadata:
  name: otel-collector
  namespace: tracing
spec:
  config: |
    receivers:
      otlp:
        protocols:
          grpc:
            endpoint: 0.0.0.0:4317
          http:
            endpoint: 0.0.0.0:4318
    processors:
      batch: {}
      memory_limiter:
        limit_mib: 400
    exporters:
      jaeger:
        endpoint: jaeger-collector:14250
        tls:
          insecure: true
    service:
      pipelines:
        traces:
          receivers: [otlp]
          processors: [memory_limiter, batch]
          exporters: [jaeger]

# Python app — instrument with OpenTelemetry
from opentelemetry import trace
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor

provider = TracerProvider()
provider.add_span_processor(
    BatchSpanProcessor(OTLPSpanExporter(endpoint="otel-collector:4317"))
)
trace.set_tracer_provider(provider)
tracer = trace.get_tracer("my-service")

@app.route("/orders")
def get_orders():
    with tracer.start_as_current_span("get-orders") as span:
        span.set_attribute("user.id", user_id)
        # Call downstream services — trace ID propagated automatically
        result = call_inventory_service()
        return result

# Step 1: Check node status
kubectl get nodes
# NAME        STATUS     ROLES    AGE
# node-1      NotReady   <none>   30d  ← Problem

# Step 2: Describe the node — check Conditions
kubectl describe node node-1
# Conditions:
#   MemoryPressure    False   (if True: node is OOM)
#   DiskPressure      False   (if True: disk almost full)
#   PIDPressure       False   (if True: too many processes)
#   Ready             False   ← PROBLEM
#
# Events:
#   Warning NodeNotReady  kubelet stopped posting node status

# Step 3: SSH to the node and check kubelet
ssh node-1
sudo systemctl status kubelet
sudo journalctl -u kubelet -f --since "1 hour ago"

# Common kubelet issues:
# "Unable to connect to the server" → networking issue
# "certificate has expired" → TLS certs expired
# "failed to get node info" → API server unreachable
# "PLEG is not healthy" → container runtime stuck

# Step 4: Check container runtime
sudo systemctl status containerd
sudo crictl ps   # List running containers via CRI

# Step 5: Check disk space
df -h
du -sh /var/lib/containerd   # Check container storage

# Step 6: Check memory
free -h
sudo dmesg | grep -i "oom\|killed"   # OOM kill events

# Step 7: Check network connectivity to control plane
nc -zv <api-server-ip> 6443

# Step 8: Restart kubelet if needed
sudo systemctl restart kubelet
sudo systemctl restart containerd

# 1. Check Pod status and events
kubectl get pods
kubectl describe pod <pod-name>

# 2. Check current logs
kubectl logs <pod-name>

# 3. Check previous container logs (if container already crashed)
kubectl logs <pod-name> --previous

# 4. Check exit code (tells you why the container exited)
kubectl get pod <pod-name> -o jsonpath='{.status.containerStatuses[0].lastState.terminated.exitCode}'

# 5. Debug with a shell (if container has bash)
kubectl debug -it <pod-name> --image=busybox --target=<container-name>

# Step 1: Confirm it's really isolated to this pod, not intermittent everywhere
kubectl get pods -o wide -l app=myapp
# Note which node this pod is scheduled on vs. the healthy pods

# Step 2: Test DNS resolution from inside the failing pod
kubectl exec -it <failing-pod> -- nslookup mydb.namespace.svc.cluster.local
# If this fails but works from a healthy pod → CoreDNS or node-local DNS cache issue

# Step 3: Test raw TCP connectivity to the DB
kubectl exec -it <failing-pod> -- nc -zv mydb.namespace.svc.cluster.local 5432
# Connection refused vs. timeout tells you a lot:
#  - refused  → reaching the DB, but DB is rejecting (auth, max connections, pg_hba.conf)
#  - timeout  → not reaching the DB at all (network policy, security group, routing)

# Step 4: Compare NetworkPolicy exposure — is this pod missing a required label?
kubectl get pod <failing-pod> --show-labels
kubectl get networkpolicy -n <namespace> -o yaml
# A NetworkPolicy selector matching on labels can silently exclude one pod
# if its labels drifted (e.g., deployed from a different manifest/branch)

# Step 5: Check the NODE the pod is on for node-level issues
kubectl describe node <node-name> | grep -A5 Conditions
# Node-level security group / CNI issues affect every pod on that specific node

# Step 6: Check for connection pool / max_connections exhaustion on the DB side
# If 99 pods share a connection pool limit and this pod is the "one too many"
# request, the DB itself may be rejecting only the newest connections

# Step 7: Check the pod's actual runtime env vars — a stale ConfigMap/Secret
# mount is one of the most common "works everywhere except this pod" causes
kubectl exec -it <failing-pod> -- env | grep -i db
kubectl get pod <failing-pod> -o jsonpath='{.spec.containers[0].envFrom}'

# Step 1: "Running" isn't the same as "Ready" — check READY column, not just STATUS
kubectl get pods -l app=myapp
# NAME          READY   STATUS    RESTARTS
# myapp-abc123  0/1     Running   0        ← Running, but NOT Ready = excluded from Service

# Step 2: If Pods show 0/1 Ready, the readiness probe is failing — check why
kubectl describe pod myapp-abc123 | grep -A5 Readiness
kubectl logs myapp-abc123
# Common cause: app takes longer to warm up than initialDelaySeconds allows

# Step 3: Check the Service actually has endpoints
kubectl get endpoints myapp-svc
# If this is EMPTY, the Service's label selector doesn't match any Pod's labels
# (classic cause: a recent Deployment change altered labels without updating the Service)

# Step 4: Compare the Service selector to the Pod's actual labels
kubectl get svc myapp-svc -o jsonpath='{.spec.selector}'
kubectl get pods --show-labels

# Step 5: If using Ingress/ALB — check the Ingress Controller / Load Balancer
# Controller logs, and confirm target group health (on EKS with ALB Ingress)
kubectl logs -n kube-system deploy/aws-load-balancer-controller

# Step 6: Check for a PodDisruptionBudget or rollout blocking enough Ready replicas
kubectl get pdb
kubectl rollout status deployment/myapp

# For ECR: does the NODE's IAM role actually have ecr:GetAuthorizationToken +
# ecr:BatchGetImage + ecr:GetDownloadUrlForLayer permissions?
aws iam get-role-policy --role-name eks-node-role --policy-name ecr-pull

# For a private registry needing a Secret: is it actually attached to the Pod
# (or the ServiceAccount), and not just created and forgotten?
kubectl get pod <pod> -o jsonpath='{.spec.imagePullSecrets}'
kubectl get secret <regcred-name> -o yaml   # confirm it actually exists, isn't expired

# Describe the pod — the Events section usually names the exact auth failure
kubectl describe pod <pod> | grep -A5 Events
# "unauthorized: authentication required" = credentials issue, not a typo

# Nodes in a private subnet need a route to the registry — for ECR specifically,
# either a NAT Gateway (internet route) OR an ECR VPC Interface Endpoint
aws ec2 describe-vpc-endpoints --filters Name=service-name,Values=com.amazonaws.us-east-1.ecr.api

# Test reachability directly from a node (via SSM, not SSH)
aws ssm start-session --target <node-instance-id>
curl -v https://<account-id>.dkr.ecr.us-east-1.amazonaws.com/v2/

# Does the image tag actually exist in the registry — was it deleted by a
# lifecycle policy, or never successfully pushed by a failed CI job?
aws ecr describe-images --repository-name my-app --image-ids imageTag=v1.2.3

# Rate limiting — public Docker Hub images can hit anonymous pull rate limits
# if many nodes pull simultaneously without authentication
kubectl describe pod <pod> | grep -i "toomanyrequests"

# Confirm it was OOMKilled specifically (not a normal crash)
kubectl describe pod <pod> | grep -A3 "Last State"
# Last State:  Terminated
#   Reason:    OOMKilled
#   Exit Code: 137

# Check what the limit actually was vs. what was likely being used
kubectl get pod <pod> -o jsonpath='{.spec.containers[0].resources}'