Kubernetes Interview Questions & Answers part 01

# Pod DNS format
<pod-ip-dashes>.<namespace>.pod.cluster.local
# Example: 10-0-0-1.default.pod.cluster.local

# Service DNS format
<service-name>.<namespace>.svc.cluster.local
# Example: my-svc.production.svc.cluster.local

# Test DNS from inside a pod
kubectl run dns-test --image=busybox --rm -it --restart=Never -- nslookup my-svc.production

apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: backend-netpol
  namespace: production
spec:
  podSelector:
    matchLabels:
      app: backend         # Apply to backend pods
  policyTypes:
  - Ingress
  - Egress
  ingress:
  - from:
    - podSelector:
        matchLabels:
          app: frontend    # Only allow traffic from frontend pods
    ports:
    - protocol: TCP
      port: 8080
  egress:
  - to:
    - podSelector:
        matchLabels:
          app: postgres    # Backend can only talk to postgres
    ports:
    - protocol: TCP
      port: 5432

# 1. Role — read-only in 'development' namespace
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: developer-readonly
  namespace: development
rules:
- apiGroups: [""]                    # Core API group
  resources: ["pods", "pods/log", "services", "configmaps", "endpoints"]
  verbs: ["get", "list", "watch"]
- apiGroups: ["apps"]
  resources: ["deployments", "replicasets", "statefulsets", "daemonsets"]
  verbs: ["get", "list", "watch"]
- apiGroups: ["batch"]
  resources: ["jobs", "cronjobs"]
  verbs: ["get", "list", "watch"]

---
# 2. RoleBinding — attach role to a user
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: developer-readonly-binding
  namespace: development
subjects:
- kind: User
  name: [email protected]   # IAM user or OIDC user
  apiGroup: rbac.authorization.k8s.io
roleRef:
  kind: Role
  name: developer-readonly
  apiGroup: rbac.authorization.k8s.io

# Check what a user can do
kubectl auth can-i list pods -n development --as=[email protected]
# yes

kubectl auth can-i delete pods -n development --as=[email protected]
# no

# List all permissions for a user
kubectl auth can-i --list -n development --as=[email protected]

apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: api-hpa
  namespace: production
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: api-server
  minReplicas: 2
  maxReplicas: 20
  metrics:
  # Scale on CPU utilization
  - type: Resource
    resource:
      name: cpu
      target:
        type: Utilization
        averageUtilization: 70    # Target 70% CPU across all pods
  # Scale on Memory
  - type: Resource
    resource:
      name: memory
      target:
        type: AverageValue
        averageValue: 512Mi       # Target 512Mi per pod
  # Scale on custom metric (requests per second)
  - type: Pods
    pods:
      metric:
        name: http_requests_per_second
      target:
        type: AverageValue
        averageValue: "1000"
  # Scaling behaviour (prevent flapping)
  behavior:
    scaleUp:
      stabilizationWindowSeconds: 60      # Wait 60s before scaling up again
      policies:
      - type: Pods
        value: 4                          # Max 4 pods per scale-up
        periodSeconds: 60
    scaleDown:
      stabilizationWindowSeconds: 300     # Wait 5 min before scaling down
      policies:
      - type: Percent
        value: 10                         # Scale down max 10% at a time
        periodSeconds: 60

# Install Metrics Server (required for HPA)
kubectl apply -f https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml

# Watch HPA in action
kubectl get hpa -n production -w

# Check current metrics
kubectl top pods -n production

kubectl apply -f deployment.yaml
kubectl apply -f service.yaml
kubectl apply -f ingress.yaml
kubectl apply -f configmap.yaml
kubectl apply -f secret.yaml
# ...and more

helm install my-app ./my-chart --namespace production

# Add a chart repository
helm repo add bitnami https://charts.bitnami.com/bitnami
helm repo update

# Search for charts
helm search repo nginx

# Install a chart
helm install my-nginx bitnami/nginx \
  --namespace web \
  --create-namespace \
  --set replicaCount=3 \
  --set service.type=LoadBalancer

# Install with custom values file
helm install my-nginx bitnami/nginx -f custom-values.yaml

# Upgrade a release
helm upgrade my-nginx bitnami/nginx --set image.tag=1.25

# Rollback to previous version
helm rollback my-nginx 1

# List all releases
helm list --all-namespaces

# Uninstall a release
helm uninstall my-nginx -n web

# Render templates without installing (dry-run)
helm template my-nginx bitnami/nginx
helm install my-nginx bitnami/nginx --dry-run --debug

# 1. Taint the GPU node
kubectl taint nodes gpu-node-1 dedicated=gpu:NoSchedule

# 2. View taints on a node
kubectl describe node gpu-node-1 | grep -i taint

# 3. Remove a taint (add minus at the end)
kubectl taint nodes gpu-node-1 dedicated=gpu:NoSchedule-

# 4. Pod with toleration can schedule on GPU node
apiVersion: v1
kind: Pod
metadata:
  name: ml-training
spec:
  tolerations:
  - key: "dedicated"
    operator: "Equal"
    value: "gpu"
    effect: "NoSchedule"
  nodeSelector:
    dedicated: gpu           # Also use nodeSelector to ATTRACT to GPU node
  containers:
  - name: training
    image: tensorflow/tensorflow:latest-gpu
    resources:
      limits:
        nvidia.com/gpu: 1

# Taint spot instances (avoid running critical workloads)
kubectl taint nodes spot-node-1 spot=true:NoSchedule

# Taint nodes for maintenance
kubectl taint nodes node-1 maintenance=true:NoExecute

# Dedicated nodes for monitoring stack
kubectl taint nodes monitoring-node dedicated=monitoring:NoSchedule

spec:
  containers:
  - name: api
    image: my-api:v1
    ports:
    - containerPort: 8080

    # Startup probe — give app 5 mins to start (30 * 10s)
    # Liveness/Readiness are paused until startup succeeds
    startupProbe:
      httpGet:
        path: /health/startup
        port: 8080
      failureThreshold: 30
      periodSeconds: 10

    # Liveness probe — restart if app is dead
    livenessProbe:
      httpGet:
        path: /health/live
        port: 8080
      initialDelaySeconds: 10   # Wait 10s after container starts
      periodSeconds: 15         # Check every 15s
      failureThreshold: 3       # Restart after 3 failures
      timeoutSeconds: 5         # Fail if no response in 5s

    # Readiness probe — remove from load balancer if not ready
    readinessProbe:
      httpGet:
        path: /health/ready
        port: 8080
      initialDelaySeconds: 5
      periodSeconds: 10
      failureThreshold: 3
      successThreshold: 1       # Need 1 success to be marked ready

# HTTP GET — most common
httpGet:
  path: /healthz
  port: 8080
  httpHeaders:
  - name: X-Health-Check
    value: "true"

# TCP Socket — for non-HTTP services
tcpSocket:
  port: 5432     # Just checks if port is open

# Exec command — run a command inside container
exec:
  command:
  - /bin/sh
  - -c
  - "redis-cli ping | grep PONG"

# 1. Create ServiceAccount
apiVersion: v1
kind: ServiceAccount
metadata:
  name: deployment-manager
  namespace: production

---
# 2. Create Role with needed permissions
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: deployment-role
  namespace: production
rules:
- apiGroups: ["apps"]
  resources: ["deployments"]
  verbs: ["get", "list", "update", "patch"]

---
# 3. Bind the Role to the ServiceAccount
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: deployment-role-binding
  namespace: production
subjects:
- kind: ServiceAccount
  name: deployment-manager
  namespace: production
roleRef:
  kind: Role
  name: deployment-role
  apiGroup: rbac.authorization.k8s.io

---
# 4. Use ServiceAccount in Pod
apiVersion: v1
kind: Pod
metadata:
  name: my-app
spec:
  serviceAccountName: deployment-manager    # Attach the SA here
  automountServiceAccountToken: true
  containers:
  - name: app
    image: my-app:v1

# Associate ServiceAccount with IAM role (no access keys needed in pods!)
eksctl create iamserviceaccount \
  --cluster my-cluster \
  --namespace production \
  --name s3-access-sa \
  --attach-policy-arn arn:aws:iam::aws:policy/AmazonS3ReadOnlyAccess \
  --approve

apiVersion: apps/v1
kind: Deployment
metadata:
  name: web-app
spec:
  replicas: 6
  strategy:
    type: RollingUpdate
    rollingUpdate:
      maxSurge: 2          # Max 2 extra pods above desired count
      maxUnavailable: 0    # Never go below 6 pods (zero downtime)
  template:
    spec:
      containers:
      - name: web
        image: web-app:v2.0

# Update the image
kubectl set image deployment/web-app web=web-app:v2.0 -n production

# Or edit deployment directly
kubectl edit deployment web-app -n production

# Watch rollout progress
kubectl rollout status deployment/web-app -n production
# Waiting for deployment "web-app" rollout to finish: 2 out of 6 new replicas...

# View rollout history
kubectl rollout history deployment/web-app

# View details of a specific revision
kubectl rollout history deployment/web-app --revision=2

# Rollback to previous version immediately
kubectl rollout undo deployment/web-app -n production

# Rollback to a specific revision
kubectl rollout undo deployment/web-app --to-revision=3

# Pause a rollout (freeze mid-rollout)
kubectl rollout pause deployment/web-app

# Resume a paused rollout
kubectl rollout resume deployment/web-app

strategy:
  type: Recreate    # All old pods killed, then all new pods started

apiVersion: apps/v1
kind: Deployment
metadata:
  name: web-app
spec:
  template:
    spec:
      initContainers:

      # Init 1 — wait for database to be ready
      - name: wait-for-db
        image: busybox:1.36
        command:
        - /bin/sh
        - -c
        - |
          until nc -z postgres-svc 5432; do
            echo "Waiting for database..."
            sleep 5
          done
          echo "Database is ready!"

      # Init 2 — run database migrations
      - name: run-migrations
        image: my-app:v2.0
        command: ["python", "manage.py", "migrate"]
        env:
        - name: DATABASE_URL
          valueFrom:
            secretKeyRef:
              name: db-secret
              key: DATABASE_URL

      # Main application container starts ONLY after both inits succeed
      containers:
      - name: web-app
        image: my-app:v2.0
        ports:
        - containerPort: 8000

# Option 1: minAvailable — always keep at least N pods running
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
  name: web-app-pdb
  namespace: production
spec:
  minAvailable: 2           # Always keep at least 2 pods up
  selector:
    matchLabels:
      app: web-app

---
# Option 2: maxUnavailable — allow at most N pods to be down
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
  name: api-pdb
  namespace: production
spec:
  maxUnavailable: 1         # Allow at most 1 pod to be unavailable
  selector:
    matchLabels:
      app: api-server

---
# Option 3: percentage
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
  name: backend-pdb
spec:
  minAvailable: "80%"       # Always keep 80% of pods available
  selector:
    matchLabels:
      app: backend

# View PDBs
kubectl get pdb -n production

# Check PDB status
kubectl describe pdb web-app-pdb -n production
# Shows: Allowed disruptions, Current pods, etc.

# Drain a node (respects PDBs)
kubectl drain node-1 --ignore-daemonsets --delete-emptydir-data

# /etc/kubernetes/encryption-config.yaml (on API server)
apiVersion: apiserver.config.k8s.io/v1
kind: EncryptionConfiguration
resources:
- resources:
  - secrets
  providers:
  - aescbc:
      keys:
      - name: key1
        secret: <base64-encoded-32-byte-key>
  - identity: {}   # Fallback for unencrypted data

# External Secrets Operator — syncs AWS Secrets Manager → K8s Secret
apiVersion: external-secrets.io/v1beta1
kind: ExternalSecret
metadata:
  name: db-credentials
  namespace: production
spec:
  refreshInterval: 1h
  secretStoreRef:
    name: aws-secretsmanager
    kind: ClusterSecretStore
  target:
    name: db-secret              # Creates this K8s Secret
  data:
  - secretKey: DB_PASSWORD
    remoteRef:
      key: prod/myapp/database   # AWS Secrets Manager path
      property: password

# Only allow specific service accounts to read secrets
rules:
- apiGroups: [""]
  resources: ["secrets"]
  resourceNames: ["db-secret", "api-key"]  # Specific secrets only
  verbs: ["get"]

# ❌ Never put secrets in ConfigMaps
# ❌ Never commit secrets to Git
# ❌ Never echo secrets in logs
# ❌ Never use environment variables for very sensitive data
#    (they can appear in process lists)

# ✅ Mount secrets as files instead
volumeMounts:
- name: db-credentials
  mountPath: /etc/secrets
  readOnly: true
volumes:
- name: db-credentials
  secret:
    secretName: db-secret
    defaultMode: 0400    # Read-only for owner only

# Audit who accessed a secret
kubectl get events | grep secret
# Enable audit logs in kube-apiserver for full audit trail

# Step 1 — describe the pod (most important command)
kubectl describe pod <pod-name> -n <namespace>
# Look at the 'Events' section at the bottom

# Step 2 — check node resources
kubectl describe nodes | grep -A 5 "Allocated resources"
kubectl top nodes

# Step 3 — check if PVC is bound (if pod mounts one)
kubectl get pvc -n <namespace>

# Events will show:
# "0/3 nodes are available: 3 Insufficient cpu"

# Fix: Scale up node group or reduce resource requests
kubectl get nodes
kubectl describe node <node> | grep -A 10 "Allocated resources"

# Check what's using resources
kubectl top pods --all-namespaces --sort-by=cpu

# Events: "0/3 nodes are available: 3 node(s) didn't match Pod's node affinity"

# Check node labels
kubectl get nodes --show-labels

# Add missing label to node
kubectl label node <node-name> disktype=ssd

# Events: "persistentvolumeclaim not found" or PVC stuck in Pending

kubectl describe pvc <pvc-name>
# Check if StorageClass exists
kubectl get storageclass

# Events: "0/3 nodes are available: 3 node(s) had untolerated taint"

kubectl describe nodes | grep -i taint
# Add toleration to pod spec

# Each node has a default limit of 110 pods
kubectl describe node <node> | grep "Non-terminated Pods"

# One command to see all pending pods and their reason
kubectl get pods --all-namespaces --field-selector=status.phase=Pending
kubectl describe pods --all-namespaces | grep -A 10 "Events:"

# Step 1 — describe pod for events and exit codes
kubectl describe pod <pod-name> -n <namespace>

# Step 2 — current logs (may be empty if app crashes immediately)
kubectl logs <pod-name> -n <namespace>

# Step 3 — PREVIOUS container logs (before the crash) — most useful
kubectl logs <pod-name> -n <namespace> --previous

# Step 4 — check exit code
kubectl get pod <pod-name> -o jsonpath='{.status.containerStatuses[0].lastState.terminated}'

# Fix OOMKill (exit 137) — increase memory limit
kubectl patch deployment <name> -p \
  '{"spec":{"template":{"spec":{"containers":[{"name":"app","resources":{"limits":{"memory":"1Gi"}}}]}}}}'

# Fix: App can't connect to database
# Check if DB service is reachable from pod
kubectl exec -it <pod-name> -- nc -zv postgres-svc 5432

# Fix: Wrong image command — override to debug
kubectl run debug-pod \
  --image=<same-image> \
  --restart=Never \
  --rm -it \
  --command -- /bin/sh

# Fix: Liveness probe killing app too early — increase initialDelaySeconds
livenessProbe:
  initialDelaySeconds: 60    # Give app more time to start
  failureThreshold: 5

# Fix: Missing environment variable or secret
kubectl exec -it <pod-name> -- env | grep DB_   # Check env vars
kubectl describe pod <pod-name> | grep -A 5 "Environment"

apiVersion: apps/v1
kind: Deployment
metadata:
  name: web-app
  namespace: production
spec:
  replicas: 6
  strategy:
    type: RollingUpdate
    rollingUpdate:
      maxSurge: 2          # Can temporarily have 8 pods (6+2)
      maxUnavailable: 0    # Never drop below 6 healthy pods
  selector:
    matchLabels:
      app: web-app
  template:
    metadata:
      labels:
        app: web-app
    spec:
      # 1. Give pods time to finish in-flight requests before shutdown
      terminationGracePeriodSeconds: 60

      containers:
      - name: web
        image: web-app:v2.0
        ports:
        - containerPort: 8080

        # 2. Readiness probe — pod only gets traffic when truly ready
        readinessProbe:
          httpGet:
            path: /ready
            port: 8080
          initialDelaySeconds: 10
          periodSeconds: 5
          failureThreshold: 3

        # 3. Liveness probe — restart if pod is dead
        livenessProbe:
          httpGet:
            path: /health
            port: 8080
          initialDelaySeconds: 30
          periodSeconds: 15

        # 4. preStop hook — wait for traffic to drain before shutdown
        lifecycle:
          preStop:
            exec:
              command:
              - /bin/sh
              - -c
              - sleep 15    # Wait 15s for load balancer to remove pod from rotation

        resources:
          requests:
            cpu: "250m"
            memory: "256Mi"
          limits:
            cpu: "500m"
            memory: "512Mi"

# Deploy v2 to 10% of traffic first
kubectl scale deployment web-app-v1 --replicas=9    # 9 old pods
kubectl scale deployment web-app-v2 --replicas=1    # 1 new pod (10%)

# If v2 is healthy, gradually increase
kubectl scale deployment web-app-v2 --replicas=5    # 50%
kubectl scale deployment web-app-v2 --replicas=10   # 100%
kubectl scale deployment web-app-v1 --replicas=0    # Remove old

# Switch Service selector from blue to green instantly
kubectl patch service web-svc \
  -p '{"spec":{"selector":{"version":"v2"}}}'

# Rollback instantly by switching back
kubectl patch service web-svc \
  -p '{"spec":{"selector":{"version":"v1"}}}'

# Take an etcd snapshot
ETCDCTL_API=3 etcdctl snapshot save /backup/etcd-snapshot-$(date +%Y%m%d).db \
  --endpoints=https://127.0.0.1:2379 \
  --cacert=/etc/kubernetes/pki/etcd/ca.crt \
  --cert=/etc/kubernetes/pki/etcd/server.crt \
  --key=/etc/kubernetes/pki/etcd/server.key

# Verify the snapshot
ETCDCTL_API=3 etcdctl snapshot status /backup/etcd-snapshot-$(date +%Y%m%d).db

# Restore from snapshot
ETCDCTL_API=3 etcdctl snapshot restore /backup/etcd-snapshot.db \
  --data-dir=/var/lib/etcd-restored

# Automate daily backups via CronJob
kubectl apply -f etcd-backup-cronjob.yaml

# Restrict anonymous access
--anonymous-auth=false

# Enable audit logging
--audit-log-path=/var/log/kubernetes/audit.log
--audit-policy-file=/etc/kubernetes/audit-policy.yaml

# Disable insecure port
--insecure-port=0

# Never use cluster-admin in applications
# Create minimal roles per service

# Audit RBAC permissions
kubectl auth can-i --list --as=system:serviceaccount:production:my-sa

# Deny all traffic by default, then allow explicitly
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: deny-all
  namespace: production
spec:
  podSelector: {}          # Applies to ALL pods in namespace
  policyTypes:
  - Ingress
  - Egress
  # No rules = deny all

spec:
  securityContext:
    runAsNonRoot: true       # Never run as root
    runAsUser: 1000
    fsGroup: 2000
    seccompProfile:
      type: RuntimeDefault   # Enable seccomp filtering

  containers:
  - name: app
    image: my-app:v1
    securityContext:
      allowPrivilegeEscalation: false
      readOnlyRootFilesystem: true    # Container cannot write to filesystem
      capabilities:
        drop:
        - ALL                          # Drop all Linux capabilities
        add:
        - NET_BIND_SERVICE             # Add only what you need

# Scan images before pushing
trivy image my-app:v1
grype my-app:v1

# Use Image Policy Webhook to block vulnerable images
# Use private registry — never use :latest tag in production

# Enable encryption at rest for etcd
# Use External Secrets Operator with AWS Secrets Manager / Vault
# Rotate secrets regularly

# Use Falco for runtime threat detection
helm install falco falcosecurity/falco \
  --namespace falco-system \
  --create-namespace

# Falco detects: shell in containers, privilege escalation,
# unexpected network connections, file system changes

# Label namespace to enforce security standards
apiVersion: v1
kind: Namespace
metadata:
  name: production
  labels:
    pod-security.kubernetes.io/enforce: restricted    # Most strict
    pod-security.kubernetes.io/audit: restricted
    pod-security.kubernetes.io/warn: restricted

# 1. Define the Custom Resource Definition
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
  name: postgresclusters.db.example.com
spec:
  group: db.example.com
  versions:
  - name: v1
    served: true
    storage: true
    schema:
      openAPIV3Schema:
        type: object
        properties:
          spec:
            type: object
            properties:
              replicas:
                type: integer
              version:
                type: string
              backupSchedule:
                type: string
  scope: Namespaced
  names:
    plural: postgresclusters
    singular: postgrescluster
    kind: PostgresCluster

---
# 2. Use the custom resource (like any K8s object now)
apiVersion: db.example.com/v1
kind: PostgresCluster
metadata:
  name: my-database
  namespace: production
spec:
  replicas: 3
  version: "15.2"
  backupSchedule: "0 2 * * *"    # Operator handles backups automatically

# Install cert-manager operator (manages TLS certificates)
helm install cert-manager jetstack/cert-manager --set installCRDs=true

# Install Prometheus operator (manages monitoring stack)
helm install kube-prometheus-stack prometheus-community/kube-prometheus-stack

# Install Strimzi operator (manages Kafka clusters)
helm install strimzi-kafka-operator strimzi/strimzi-kafka-operator

# Use Operator SDK (most popular framework)
operator-sdk init --domain example.com --repo github.com/example/my-operator
operator-sdk create api --group apps --version v1 --kind MyApp --resource --controller

# Or use Kubebuilder
kubebuilder init --domain example.com
kubebuilder create api --group apps --version v1 --kind MyApp

apiVersion: v1
kind: ResourceQuota
metadata:
  name: production-quota
  namespace: production
spec:
  hard:
    # Compute resources
    requests.cpu: "20"           # Max 20 CPU cores requested in namespace
    requests.memory: 40Gi        # Max 40Gi memory requested
    limits.cpu: "40"
    limits.memory: 80Gi

    # Object count limits
    pods: "100"
    services: "20"
    persistentvolumeclaims: "30"
    secrets: "50"
    configmaps: "50"

    # Storage limits
    requests.storage: "500Gi"
    storageclass.storage.k8s.io/fast-ssd.requests.storage: "200Gi"

apiVersion: v1
kind: LimitRange
metadata:
  name: production-limitrange
  namespace: production
spec:
  limits:
  # Container-level defaults and max/min
  - type: Container
    default:               # Default LIMIT if not specified
      cpu: "500m"
      memory: "256Mi"
    defaultRequest:        # Default REQUEST if not specified
      cpu: "100m"
      memory: "128Mi"
    max:                   # Maximum any container can request
      cpu: "4"
      memory: "8Gi"
    min:                   # Minimum any container must request
      cpu: "50m"
      memory: "64Mi"

  # Pod-level maximum
  - type: Pod
    max:
      cpu: "8"
      memory: "16Gi"

  # PVC storage limits
  - type: PersistentVolumeClaim
    max:
      storage: "100Gi"
    min:
      storage: "1Gi"

# High priority — for critical system workloads
apiVersion: scheduling.k8s.io/v1
kind: PriorityClass
metadata:
  name: high-priority
value: 1000000
globalDefault: false
description: "Critical production services"

---
# Low priority — for batch jobs
apiVersion: scheduling.k8s.io/v1
kind: PriorityClass
metadata:
  name: low-priority
value: 100
description: "Batch jobs and non-critical workloads"

---
# Use in Pod
spec:
  priorityClassName: high-priority    # This pod won't be evicted first
  containers:
  - name: critical-app
    image: my-app:v1

kubectl create namespace argocd
kubectl apply -n argocd -f https://raw.githubusercontent.com/argoproj/argo-cd/stable/manifests/install.yaml

# Get initial admin password
kubectl -n argocd get secret argocd-initial-admin-secret -o jsonpath="{.data.password}" | base64 -d

# Port-forward to access UI
kubectl port-forward svc/argocd-server -n argocd 8080:443

apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
  name: web-app-production
  namespace: argocd
spec:
  project: default

  # Source — where your manifests live in Git
  source:
    repoURL: https://github.com/myorg/k8s-configs
    targetRevision: main
    path: apps/web-app/production

  # Destination — where to deploy in the cluster
  destination:
    server: https://kubernetes.default.svc
    namespace: production

  # Sync policy — auto-sync when Git changes
  syncPolicy:
    automated:
      prune: true         # Delete resources removed from Git
      selfHeal: true      # Revert manual changes to cluster
    syncOptions:
    - CreateNamespace=true
    - PrunePropagationPolicy=foreground
    retry:
      limit: 5
      backoff:
        duration: 5s
        maxDuration: 3m
        factor: 2

# Developer makes a change
git checkout -b feature/update-image
# Edit k8s/deployment.yaml — change image tag
git commit -m "deploy: bump web-app to v2.5"
git push origin feature/update-image

# Create PR → review → merge to main
# ArgoCD detects the change within 3 minutes
# ArgoCD applies the change to cluster automatically

# Check sync status
argocd app get web-app-production
argocd app sync web-app-production    # Manual sync if needed
argocd app history web-app-production # Deployment history

# Step 1 — check node status and conditions
kubectl get nodes
kubectl describe node <node-name>

# Look for conditions:
# Ready = False/Unknown
# MemoryPressure = True
# DiskPressure = True
# PIDPressure = True
# NetworkUnavailable = True

# Check kubelet status (most common cause)
sudo systemctl status kubelet
sudo journalctl -u kubelet -f --no-pager | tail -50

# Common kubelet errors:
# "failed to get node info" → network issue
# "certificate expired" → renew kubelet certificates
# "PLEG is not healthy" → pod lifecycle event generator issues (often disk pressure)

# Check node resources
df -h              # Disk usage (DiskPressure if >85%)
free -m            # Memory (MemoryPressure)
top                # CPU and process check
ps aux | wc -l     # PID count (PIDPressure if >1000)

# Fix 1: kubelet not running
sudo systemctl restart kubelet

# Fix 2: Disk pressure — clean up
docker system prune -af         # Clean Docker images/containers
crictl rmi --prune              # Clean containerd images
sudo journalctl --vacuum-size=500M    # Clean journal logs

# Fix 3: Certificate expired
sudo kubeadm alpha certs renew all
sudo systemctl restart kubelet

# Fix 4: Network plugin not running
kubectl get pods -n kube-system | grep -E "calico|flannel|cilium"
kubectl delete pod -n kube-system <broken-cni-pod>   # Restart CNI pod

# Fix 5: Node has too many pods — eviction happening
kubectl describe node <node> | grep -i "eviction\|pressure"

# Prevent new pods from scheduling on this node
kubectl cordon <node-name>

# Move existing pods to other nodes
kubectl drain <node-name> --ignore-daemonsets --delete-emptydir-data --force

# After fixing the node, uncordon it
kubectl uncordon <node-name>

# System node pool — control plane components
nodePool: system
  instanceType: m5.xlarge
  count: 3
  taints: [CriticalAddonsOnly=true:NoSchedule]

# Application node pool — production workloads (on-demand)
nodePool: app-ondemand
  instanceType: m5.2xlarge
  minCount: 6
  maxCount: 50
  availabilityZones: [us-east-1a, us-east-1b, us-east-1c]

# Spot node pool — batch/non-critical workloads (80% cheaper)
nodePool: app-spot
  instanceTypes: [m5.2xlarge, m5.4xlarge, m5a.2xlarge]
  spot: true
  minCount: 0
  maxCount: 100

apiVersion: apps/v1
kind: Deployment
metadata:
  name: web-app
  namespace: production
spec:
  replicas: 9              # 3 per AZ
  strategy:
    type: RollingUpdate
    rollingUpdate:
      maxSurge: 3
      maxUnavailable: 0
  template:
    spec:
      # Spread across zones and nodes
      topologySpreadConstraints:
      - maxSkew: 1
        topologyKey: topology.kubernetes.io/zone
        whenUnsatisfiable: DoNotSchedule
        labelSelector:
          matchLabels:
            app: web-app
      - maxSkew: 1
        topologyKey: kubernetes.io/hostname
        whenUnsatisfiable: ScheduleAnyway
        labelSelector:
          matchLabels:
            app: web-app
      # Don't schedule on spot nodes (critical app)
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
            - matchExpressions:
              - key: node.kubernetes.io/lifecycle
                operator: NotIn
                values: [spot]
      containers:
      - name: web-app
        image: web-app:v3.0
        resources:
          requests:
            cpu: "500m"
            memory: "512Mi"
          limits:
            cpu: "1"
            memory: "1Gi"
        readinessProbe:
          httpGet: {path: /ready, port: 8080}
          periodSeconds: 5
        livenessProbe:
          httpGet: {path: /health, port: 8080}
          periodSeconds: 15
        lifecycle:
          preStop:
            exec:
              command: ["/bin/sh", "-c", "sleep 15"]
      terminationGracePeriodSeconds: 60

# Metrics — Prometheus + Grafana
helm install kube-prometheus-stack prometheus-community/kube-prometheus-stack

# Logs — EFK or Loki stack
helm install loki grafana/loki-stack --set grafana.enabled=false

# Tracing — Jaeger or Tempo
helm install jaeger jaegertracing/jaeger

# Alerts — configure PagerDuty/Slack in Alertmanager

# Error rate < 0.1%
# P99 latency < 200ms
# Availability > 99.9%
# Pod restart rate < 1/hour
# Node CPU < 70%
# Node Memory < 80%