Setting Up KEDA with Terraform

by Bala

Comprehensive guide for deploying and managing KEDA using Terraform on Kubernetes

Setting Up KEDA with Terraform

This guide provides a comprehensive approach to deploying and managing KEDA (Kubernetes Event-driven Autoscaling) using Terraform, including infrastructure setup, scaler configurations, and best practices.

Prerequisites

  • Terraform (v1.0+)
  • Kubernetes cluster
  • kubectl configured
  • Helm provider
  • Basic understanding of KEDA and Terraform

Project Structure

keda-terraform/
├── main.tf
├── variables.tf
├── outputs.tf
├── providers.tf
├── versions.tf
└── modules/
    ├── keda/
    │   ├── main.tf
    │   ├── variables.tf
    │   └── outputs.tf
    └── scalers/
        ├── main.tf
        ├── variables.tf
        └── outputs.tf

Step 1: Provider Configuration

providers.tf

terraform {
  required_providers {
    kubernetes = {
      source  = "hashicorp/kubernetes"
      version = "~> 2.23.0"
    }
    helm = {
      source  = "hashicorp/helm"
      version = "~> 2.11.0"
    }
  }
}

provider "kubernetes" {
  config_path = var.kubeconfig_path
}

provider "helm" {
  kubernetes {
    config_path = var.kubeconfig_path
  }
}

versions.tf

terraform {
  required_version = ">= 1.0.0"
}

Step 2: KEDA Module

modules/keda/main.tf

resource "kubernetes_namespace" "keda" {
  metadata {
    name = var.namespace
    labels = {
      "app.kubernetes.io/managed-by" = "terraform"
    }
  }
}

resource "helm_release" "keda" {
  name       = "keda"
  repository = "https://kedacore.github.io/charts"
  chart      = "keda"
  version    = var.keda_version
  namespace  = kubernetes_namespace.keda.metadata[0].name

  values = [
    yamlencode({
      operator = {
        replicaCount = var.operator_replicas
        resources = {
          requests = {
            cpu    = var.operator_resources.requests.cpu
            memory = var.operator_resources.requests.memory
          }
          limits = {
            cpu    = var.operator_resources.limits.cpu
            memory = var.operator_resources.limits.memory
          }
        }
      }
      metricsServer = {
        replicaCount = var.metrics_server_replicas
        resources = {
          requests = {
            cpu    = var.metrics_server_resources.requests.cpu
            memory = var.metrics_server_resources.requests.memory
          }
          limits = {
            cpu    = var.metrics_server_resources.limits.cpu
            memory = var.metrics_server_resources.limits.memory
          }
        }
      }
      prometheus = {
        enabled = var.prometheus_enabled
      }
      serviceMonitor = {
        enabled = var.service_monitor_enabled
      }
    })
  ]
}

modules/keda/variables.tf

variable "namespace" {
  description = "Namespace for KEDA installation"
  type        = string
  default     = "keda"
}

variable "keda_version" {
  description = "Version of KEDA Helm chart"
  type        = string
  default     = "2.12.0"
}

variable "operator_replicas" {
  description = "Number of KEDA operator replicas"
  type        = number
  default     = 1
}

variable "operator_resources" {
  description = "Resources for KEDA operator"
  type = object({
    requests = object({
      cpu    = string
      memory = string
    })
    limits = object({
      cpu    = string
      memory = string
    })
  })
  default = {
    requests = {
      cpu    = "100m"
      memory = "128Mi"
    }
    limits = {
      cpu    = "1000m"
      memory = "512Mi"
    }
  }
}

variable "metrics_server_replicas" {
  description = "Number of metrics server replicas"
  type        = number
  default     = 1
}

variable "metrics_server_resources" {
  description = "Resources for metrics server"
  type = object({
    requests = object({
      cpu    = string
      memory = string
    })
    limits = object({
      cpu    = string
      memory = string
    })
  })
  default = {
    requests = {
      cpu    = "100m"
      memory = "128Mi"
    }
    limits = {
      cpu    = "1000m"
      memory = "512Mi"
    }
  }
}

variable "prometheus_enabled" {
  description = "Enable Prometheus metrics"
  type        = bool
  default     = true
}

variable "service_monitor_enabled" {
  description = "Enable ServiceMonitor for Prometheus Operator"
  type        = bool
  default     = true
}

Step 3: Scalers Module

modules/scalers/main.tf

resource "kubernetes_manifest" "scaled_object" {
  for_each = var.scaled_objects

  manifest = {
    apiVersion = "keda.sh/v1alpha1"
    kind       = "ScaledObject"
    metadata = {
      name      = each.key
      namespace = each.value.namespace
    }
    spec = {
      scaleTargetRef = {
        name = each.value.target_name
        kind = each.value.target_kind
      }
      minReplicaCount = each.value.min_replicas
      maxReplicaCount = each.value.max_replicas
      pollingInterval = each.value.polling_interval
      cooldownPeriod  = each.value.cooldown_period
      triggers        = each.value.triggers
    }
  }
}

resource "kubernetes_manifest" "trigger_authentication" {
  for_each = var.trigger_authentications

  manifest = {
    apiVersion = "keda.sh/v1alpha1"
    kind       = "TriggerAuthentication"
    metadata = {
      name      = each.key
      namespace = each.value.namespace
    }
    spec = each.value.spec
  }
}

modules/scalers/variables.tf

variable "scaled_objects" {
  description = "Map of ScaledObjects to create"
  type = map(object({
    namespace        = string
    target_name     = string
    target_kind     = string
    min_replicas    = number
    max_replicas    = number
    polling_interval = number
    cooldown_period = number
    triggers        = list(any)
  }))
}

variable "trigger_authentications" {
  description = "Map of TriggerAuthentications to create"
  type = map(object({
    namespace = string
    spec      = any
  }))
}

Step 4: Root Module Configuration

main.tf

module "keda" {
  source = "./modules/keda"

  namespace               = var.keda_namespace
  keda_version           = var.keda_version
  operator_replicas      = var.operator_replicas
  operator_resources     = var.operator_resources
  prometheus_enabled     = var.prometheus_enabled
  service_monitor_enabled = var.service_monitor_enabled
}

module "scalers" {
  source = "./modules/scalers"
  depends_on = [module.keda]

  scaled_objects          = var.scaled_objects
  trigger_authentications = var.trigger_authentications
}

variables.tf

variable "kubeconfig_path" {
  description = "Path to kubeconfig file"
  type        = string
}

variable "keda_namespace" {
  description = "Namespace for KEDA installation"
  type        = string
  default     = "keda"
}

variable "keda_version" {
  description = "Version of KEDA Helm chart"
  type        = string
  default     = "2.12.0"
}

variable "scaled_objects" {
  description = "Map of ScaledObjects to create"
  type = map(object({
    namespace        = string
    target_name     = string
    target_kind     = string
    min_replicas    = number
    max_replicas    = number
    polling_interval = number
    cooldown_period = number
    triggers        = list(any)
  }))
  default = {}
}

variable "trigger_authentications" {
  description = "Map of TriggerAuthentications to create"
  type = map(object({
    namespace = string
    spec      = any
  }))
  default = {}
}

Usage Examples

1. RabbitMQ Scaler

scaled_objects = {
  "rabbitmq-consumer" = {
    namespace        = "default"
    target_name     = "rabbitmq-consumer"
    target_kind     = "Deployment"
    min_replicas    = 1
    max_replicas    = 10
    polling_interval = 30
    cooldown_period = 300
    triggers = [{
      type = "rabbitmq"
      metadata = {
        protocol    = "amqp"
        queueName   = "orders"
        host        = "amqp://rabbitmq.default.svc:5672"
        queueLength = "50"
      }
    }]
  }
}

2. Prometheus Scaler with Authentication

trigger_authentications = {
  "prometheus-auth" = {
    namespace = "default"
    spec = {
      secretTargetRef = [{
        parameter = "bearerToken"
        name      = "prometheus-secret"
        key       = "token"
      }]
    }
  }
}

scaled_objects = {
  "prometheus-scaler" = {
    namespace        = "default"
    target_name     = "my-app"
    target_kind     = "Deployment"
    min_replicas    = 1
    max_replicas    = 10
    polling_interval = 30
    cooldown_period = 300
    triggers = [{
      type = "prometheus"
      metadata = {
        serverAddress = "http://prometheus.monitoring.svc"
        metricName   = "http_requests_total"
        threshold    = "100"
        query        = "sum(rate(http_requests_total{service=\"my-service\"}[2m]))"
      }
      authenticationRef = {
        name = "prometheus-auth"
      }
    }]
  }
}

Best Practices

  1. Secret Management
resource "kubernetes_secret" "scaler_secrets" {
  metadata {
    name      = "scaler-secrets"
    namespace = var.namespace
  }

  data = {
    connection_string = var.connection_string
  }
}
  1. High Availability
operator_resources = {
  requests = {
    cpu    = "200m"
    memory = "256Mi"
  }
  limits = {
    cpu    = "1000m"
    memory = "1Gi"
  }
}

operator_replicas = 2
  1. Monitoring Configuration
prometheus_enabled = true
service_monitor_enabled = true

Troubleshooting

Common Issues

  1. Installation Issues
# Check Helm release status
helm list -n keda
helm status keda -n keda

# Check KEDA pods
kubectl get pods -n keda
kubectl describe pod -l app=keda-operator -n keda
  1. Scaler Issues
# Check ScaledObject status
kubectl get scaledobject
kubectl describe scaledobject <name>

# Check HPA status
kubectl get hpa
kubectl describe hpa <name>

Additional Resources