Managing Cloud Datastore with Terraform

In this guide, we’ll explore how to manage Google Cloud Datastore using Terraform.

Video Tutorial

Prerequisites

• Google Cloud SDK installed and configured
• Terraform installed (version 1.0.0 or later)
• A GCP project with billing enabled

Project Structure

.
├── main.tf                   # Main Terraform configuration file
├── variables.tf              # Variable definitions
├── outputs.tf               # Output definitions
├── terraform.tfvars         # Variable values
└── modules/
    └── datastore/
        ├── main.tf          # Datastore specific configurations
        ├── variables.tf      # Module variables
        ├── indexes.tf       # Index configurations
        └── outputs.tf       # Module outputs

Provider Configuration

terraform {
  required_providers {
    google = {
      source  = "hashicorp/google"
      version = "~> 4.0"
    }
  }
}

provider "google" {
  project = var.project_id
  region  = var.region
}

Variables

variable "project_id" {
  description = "The ID of the GCP project"
  type        = string
}

variable "region" {
  description = "The region to deploy resources to"
  type        = string
  default     = "us-central1"
}

Datastore Configuration

resource "google_datastore_index" "task_index" {
  kind = "Task"
  
  properties {
    name = "created"
    direction = "ASCENDING"
  }
  
  properties {
    name = "priority"
    direction = "DESCENDING"
  }
}

resource "google_datastore_index" "user_index" {
  kind = "User"
  
  properties {
    name = "lastName"
    direction = "ASCENDING"
  }
  
  properties {
    name = "firstName"
    direction = "ASCENDING"
  }
}

IAM Configuration

resource "google_project_iam_member" "datastore_user" {
  project = var.project_id
  role    = "roles/datastore.user"
  member  = "serviceAccount:${google_service_account.datastore_sa.email}"
}

resource "google_service_account" "datastore_sa" {
  account_id   = "datastore-service-account"
  display_name = "Datastore Service Account"
}

resource "google_project_iam_custom_role" "datastore_viewer" {
  role_id     = "datastoreViewer"
  title       = "Datastore Viewer"
  description = "Custom role for Datastore read-only access"
  permissions = [
    "datastore.entities.get",
    "datastore.entities.list",
    "datastore.indexes.list",
    "datastore.statistics.get",
    "datastore.statistics.list"
  ]
}

Firestore in Datastore Mode Configuration

resource "google_app_engine_application" "app" {
  project     = var.project_id
  location_id = var.region
  database_type = "CLOUD_DATASTORE_COMPATIBILITY"
}

resource "google_project_service" "firestore" {
  project = var.project_id
  service = "firestore.googleapis.com"

  disable_on_destroy = false
}

Backup Configuration

resource "google_storage_bucket" "backup_bucket" {
  name          = "${var.project_id}-datastore-backup"
  location      = var.region
  force_destroy = true

  lifecycle_rule {
    condition {
      age = 30  # days
    }
    action {
      type = "Delete"
    }
  }
}

resource "google_storage_bucket_iam_binding" "backup_bucket_binding" {
  bucket = google_storage_bucket.backup_bucket.name
  role   = "roles/storage.objectViewer"
  members = [
    "serviceAccount:${google_service_account.datastore_sa.email}",
  ]
}

Monitoring Configuration

resource "google_monitoring_alert_policy" "datastore_latency" {
  display_name = "Datastore Latency Alert"
  combiner     = "OR"

  conditions {
    display_name = "High Latency"
    condition_threshold {
      filter     = "metric.type=\"datastore.googleapis.com/api/request_latencies\" AND resource.type=\"datastore_request\""
      duration   = "300s"
      comparison = "COMPARISON_GT"
      threshold_value = 1000  # milliseconds

      trigger {
        count = 1
      }

      aggregations {
        alignment_period   = "60s"
        per_series_aligner = "ALIGN_PERCENTILE_99"
      }
    }
  }

  notification_channels = [google_monitoring_notification_channel.email.name]
}

Export Configuration

resource "google_cloud_scheduler_job" "export_job" {
  name        = "datastore-export"
  description = "Daily Datastore export job"
  schedule    = "0 0 * * *"  # Daily at midnight

  http_target {
    http_method = "POST"
    uri         = "https://datastore.googleapis.com/v1/projects/${var.project_id}:export"
    
    oauth_token {
      service_account_email = google_service_account.datastore_sa.email
    }

    body = base64encode(jsonencode({
      outputUrlPrefix = "gs://${google_storage_bucket.backup_bucket.name}/exports"
      entityFilter = {
        kinds = ["Task", "User"]
      }
    }))
  }
}

Outputs

output "app_id" {
  value       = google_app_engine_application.app.app_id
  description = "The App Engine application ID"
}

output "backup_bucket" {
  value       = google_storage_bucket.backup_bucket.name
  description = "The name of the backup bucket"
}

output "service_account_email" {
  value       = google_service_account.datastore_sa.email
  description = "The email of the Datastore service account"
}

Best Practices

1. Index Management:

   • Plan indexes carefully
   • Monitor usage
   • Clean up unused
   • Regular review

2. Security:

   • Use IAM roles
   • Enable audit logs
   • Regular review
   • Monitor access

3. Performance:

   • Optimize queries
   • Monitor latency
   • Use caching
   • Regular maintenance

4. Cost Optimization:

   • Monitor usage
   • Clean up data
   • Optimize queries
   • Regular review

Common Operations

Creating Resources

terraform init
terraform plan
terraform apply

Data Operations

# Export data
gcloud datastore export \
  gs://${BUCKET_NAME}/exports

# Import data
gcloud datastore import \
  gs://${BUCKET_NAME}/exports/export.overall_export_metadata

Best Practices and Tips

1. Data Management:

   • Plan entity groups
   • Use transactions
   • Regular backups
   • Monitor size

2. Query Optimization:

   • Use indexes
   • Limit results
   • Batch operations
   • Monitor performance

3. Operations:

   • Monitor metrics
   • Track usage
   • Set up alerts
   • Regular maintenance

Conclusion

You’ve learned how to set up and manage Google Cloud Datastore using Terraform. This setup provides:

• Scalable NoSQL database
• Automatic indexing
• Backup management
• Best practices implementation