CI/CD & DevOps Interview Questions & Answers (2026)

// Jenkinsfile example
pipeline {
  agent any
  stages {
    stage('Build') { steps { sh 'npm install && npm run build' } }
    stage('Test')  { steps { sh 'npm test' } }
    stage('Docker Build') {
      steps {
        sh "docker build -t myapp:${GIT_COMMIT[0..7]} ."
        sh "docker push myapp:${GIT_COMMIT[0..7]}"
      }
    }
    stage('Deploy Staging') {
      steps { sh "helm upgrade --install myapp ./chart --set image.tag=${GIT_COMMIT[0..7]}" }
    }
    stage('Prod Approval') {
      steps { input message: 'Deploy to production?', ok: 'Deploy' }
    }
    stage('Deploy Prod') {
      steps { sh "helm upgrade myapp ./chart -n prod --set image.tag=${GIT_COMMIT[0..7]}" }
    }
  }
}

# Kubernetes Blue-Green with service selector switch
# Blue deployment: label app: myapp-blue
# Green deployment: label app: myapp-green

# Service points to blue:
kubectl patch service myapp -p '{"spec":{"selector":{"version":"blue"}}}'

# After green is ready and tested:
kubectl patch service myapp -p '{"spec":{"selector":{"version":"green"}}}'

# Rollback (instant):
kubectl patch service myapp -p '{"spec":{"selector":{"version":"blue"}}}'

# Kubernetes Canary with nginx-ingress
# Stable service:
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: myapp-stable
spec:
  rules:
  - host: myapp.com
    http:
      paths:
      - backend:
          service:
            name: myapp-v1
            port:
              number: 80
---
# Canary ingress:
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: myapp-canary
  annotations:
    nginx.ingress.kubernetes.io/canary: "true"
    nginx.ingress.kubernetes.io/canary-weight: "10"  # 10% traffic
spec:
  rules:
  - host: myapp.com
    http:
      paths:
      - backend:
          service:
            name: myapp-v2
            port:
              number: 80

# Step 1: Check the console output
# Jenkins UI → Job → Build Number → Console Output

# Step 2: Look for timeout issues
# Stage hung on: sh script, ssh, API call?
# Add timeouts to stages:
stage('Deploy') {
  options { timeout(time: 10, unit: 'MINUTES') }
  steps { sh './deploy.sh' }
}

# Step 3: Check if it's waiting for input
# Jenkins UI → Pending Input Actions → Check for approval buttons

# Step 4: Check agent/node connectivity
# Jenkins → Manage Jenkins → Nodes → Check if agent is online

# Step 5: Check for hung processes on agent
ssh jenkins-agent
ps aux | grep <process-from-pipeline>
# Kill the hung process if needed

# Step 6: Check Jenkins system log
# Jenkins → Manage Jenkins → System Log → All Jenkins Logs

# Step 7: Check build workspace on agent
ls -la /var/lib/jenkins/workspace/<job-name>/

# Step 8: If waiting on external service (Nexus, K8s, AWS):
# - Check if the service is reachable
curl http://nexus:8081/nexus/
kubectl cluster-info

# Step 9: Kill the build if truly stuck
# Jenkins UI → Build → ⬛ (Stop)
# Or via CLI:
curl -X POST http://jenkins:8080/job/myjob/1/stop \
  --user admin:token

// 1. Run independent stages in parallel
stage('Parallel Tests') {
  parallel {
    stage('Unit Tests')        { steps { sh 'npm test' } }
    stage('Integration Tests') { steps { sh 'npm run test:integration' } }
    stage('Lint')              { steps { sh 'npm run lint' } }
  }
}

// 2. Cache dependencies
// Jenkins with Docker:
stage('Build') {
  steps {
    sh 'docker build --cache-from myapp:cache -t myapp:latest .'
  }
}

// 3. Shallow git clone (faster for large repos)
git url: 'https://github.com/org/repo.git', depth: 1

// 4. Use incremental builds
// Only rebuild if relevant files changed
when {
  changeset "src/**"
}

// 5. Use smaller/purpose-built agent images
// Agent with only needed tools, not a 3GB full-featured image

// 6. Artifact caching (Maven, npm, pip)
// ~/.m2, node_modules — mount as Docker volume or use cache action

// 7. Fail fast — run unit tests before expensive integration tests

// 8. Use build agents close to deployment targets (same AZ)

# Typical time breakdown:
# Slow pipeline: 45 min
# After parallel stages: 20 min
# After caching + shallow clone: 8 min

# 1. Developer pushes new image tag to values.yaml in gitops-repo
# 2. ArgoCD detects Git change (polls every 3 min or webhook)
# 3. ArgoCD compares Git state to cluster state
# 4. ArgoCD applies difference (deploys new version)
# 5. If someone manually changes cluster, ArgoCD reverts it

# Install ArgoCD
kubectl apply -n argocd -f https://raw.githubusercontent.com/argoproj/argo-cd/stable/manifests/install.yaml

# Create application
argocd app create myapp \
  --repo https://github.com/org/gitops-repo \
  --path apps/myapp \
  --dest-server https://kubernetes.default.svc \
  --dest-namespace production

# GitHub Actions OIDC (no AWS access keys needed)
- uses: aws-actions/configure-aws-credentials@v4
  with:
    role-to-assume: arn:aws:iam::123456789:role/GitHubActionsRole
    aws-region: us-east-1

# Default ports:
# Prometheus: 9090
# Grafana: 3000
# Alertmanager: 9093
# Node Exporter: 9100

# Example PromQL query in Grafana:
# CPU usage %:
100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)

# HTTP error rate:
rate(http_requests_total{status=~"5.."}[5m]) / rate(http_requests_total[5m])

# SRE principle: if error budget is burned, stop feature releases
# and focus on reliability until budget recovers

# Error budget calculation:
# SLO: 99.9% → 0.1% budget → 43.8 minutes downtime allowed per month
# If we've used 40 minutes → only 3.8 min left → freeze risky deployments

# Distributed tracing example with OpenTelemetry
# Each service adds trace context to requests
# Jaeger UI shows: service A → service B (200ms) → service C (2.3s) ← bottleneck

# 1. Kubernetes rollback (fastest)
kubectl rollout undo deployment/myapp
kubectl rollout undo deployment/myapp --to-revision=3
kubectl rollout status deployment/myapp   # Monitor

# 2. Helm rollback
helm rollback myapp 3      # Roll back to revision 3
helm history myapp         # Show release history

# 3. Docker Swarm rollback
docker service rollback myapp

# 4. Blue-Green rollback
# Switch load balancer back to blue environment
aws elbv2 modify-listener \
  --listener-arn <arn> \
  --default-actions Type=forward,TargetGroupArn=<blue-tg-arn>

# 5. Database rollback (most complex)
# Run down-migration scripts
flyway repair
flyway migrate -target=V2__baseline
# OR restore from pre-deployment snapshot

# 6. Pipeline-level rollback
# Trigger previous successful deployment pipeline
# Jenkins: Build with parameters → pass old image tag

# 7. GitOps rollback
git revert <commit-hash>
git push   # ArgoCD auto-deploys previous version

# Databases
# RDS automated backups (daily + transaction logs → point-in-time recovery)
aws rds modify-db-instance \
  --db-instance-identifier mydb \
  --backup-retention-period 7 \
  --preferred-backup-window "03:00-04:00"

# Manual snapshot before any major change
aws rds create-db-snapshot \
  --db-instance-identifier mydb \
  --db-snapshot-identifier pre-migration-snapshot

# EBS snapshots (automated with DLM)
aws dlm create-lifecycle-policy ...

# Application files → S3 (with versioning + cross-region replication)
aws s3api put-bucket-versioning --bucket myapp-data --versioning-configuration Status=Enabled

# Test DR regularly:
# 1. Terminate primary DB → confirm failover to read replica
# 2. Restore snapshot to new instance
# 3. Failover Route 53 to DR region
# Document recovery runbooks and drill quarterly

{
  "timestamp": "2024-01-01T10:00:00Z",
  "level": "ERROR",
  "service": "payment-api",
  "trace_id": "abc123",
  "user_id": "u-456",
  "message": "Payment gateway timeout",
  "duration_ms": 5001
}

# Each request gets a unique trace_id
# Passed in HTTP headers: X-Trace-ID
# All services log it → you can search all logs for a single user request

# Jaeger/Zipkin shows the full call chain:
# API Gateway → Auth Service → Payment Service → Database

# Prometheus alert for high error rate
groups:
- name: microservices
  rules:
  - alert: HighErrorRate
    expr: rate(http_requests_total{status=~"5.."}[5m]) > 0.01
    for: 2m
    labels:
      severity: critical
    annotations:
      summary: "{{ $labels.service }} error rate > 1%"

# Reduce Detection Time:
# - Alerting thresholds tight enough to catch issues before users notice
# - Synthetic monitoring (ping endpoints every 30s from multiple regions)
# - Real User Monitoring (RUM) — detect impact immediately
# - On-call rotation with <5 min response SLA

# Reduce Diagnosis Time:
# - Runbooks for every alert (link in alert itself)
# - Pre-built dashboards per service
# - Distributed tracing (find bottleneck in seconds, not hours)
# - Structured logs with correlation IDs (grep single trace_id)

# Reduce Fix Time:
# - One-click rollback scripts ready
# - Feature flags (disable bad feature without redeploy)
# - Circuit breakers (auto-isolate failing service)
# - Pre-approved emergency changes

# Reduce Verification Time:
# - Smoke tests that run in < 2 minutes
# - Synthetic traffic patterns

# Cultural practices:
# - Blameless post-mortems → learn without fear
# - GameDays / Chaos Engineering (simulate failures safely)
# - Runbooks kept up-to-date and reviewed quarterly

# Run up to package phase:
mvn package

# Run up to install:
mvn install

# Skip tests:
mvn package -DskipTests

# Run specific phase:
mvn compile

# Clean previous build before build:
mvn clean package

# 1. Build AMI with Packer (bake everything in)
packer build web-server.json
# Creates: ami-1234567890abcdef0

# 2. Terraform deploys instances from that AMI
resource "aws_launch_template" "web" {
  image_id      = "ami-1234567890abcdef0"
  instance_type = "t3.micro"
}

# 3. New version? Build new AMI, update Terraform, roll out
# Old AMI stays for rollback

# Method 1: AWS CloudTrail (primary audit tool)
# CloudTrail records every API call with: who, what, when, from where

# Search CloudTrail for recent changes
aws cloudtrail lookup-events \
  --lookup-attributes AttributeKey=ResourceName,AttributeValue=i-1234567890abcdef0 \
  --start-time 2024-01-01T00:00:00Z

# Query via CloudWatch Logs Insights (if CloudTrail → CloudWatch enabled):
fields userIdentity.arn, eventName, eventTime, requestParameters
| filter eventSource = "ec2.amazonaws.com"
| filter eventName = "ModifyInstanceAttribute"
| sort eventTime desc
| limit 20

# Method 2: AWS Config
# Tracks resource configuration history
aws configservice get-resource-config-history \
  --resource-type AWS::EC2::Instance \
  --resource-id i-1234567890abcdef0

# Method 3: AWS Config Rules + SNS
# Set up Config rule to detect unauthorized changes
# Alert via SNS → email/Slack

# Method 4: EventBridge (CloudWatch Events) → Lambda → Slack
# Rule: "When EC2 SecurityGroup is modified → trigger Lambda → post to Slack"

# Prevention:
# - Enable CloudTrail across all regions + all accounts (org trail)
# - Store CloudTrail logs in central immutable S3 bucket (MFA Delete enabled)
# - Set up Config conformance packs for compliance monitoring