TerraformPilot

Terraform

GCP Pub/Sub with Terraform - Messaging and Event Streaming

Deploy Google Cloud Pub/Sub with Terraform. Topics, subscriptions, push and pull delivery, dead-letter topics, message filtering, and BigQuery subscriptions.

LLuca Berton1 min read

Quick Answer

# 
resource "google_pubsub_topic" "events" {
  name = "events"
}
 
resource "google_pubsub_subscription" "worker" {
  name  = "worker-sub"
  topic = google_pubsub_topic.events.id
  ack_deadline_seconds = 20
}

Production Topic

# 
resource "google_pubsub_topic" "events" {
  name = "${var.project}-events"
 
  message_retention_duration = "86400s"  # 24 hours
 
  schema_settings {
    schema   = google_pubsub_schema.events.id
    encoding = "JSON"
  }
 
  labels = { environment = var.environment }
}
 
# Optional: Schema validation
resource "google_pubsub_schema" "events" {
  name       = "${var.project}-events-schema"
  type       = "AVRO"
  definition = jsonencode({
    type = "record"
    name = "Event"
    fields = [
      { name = "eventType", type = "string" },
      { name = "timestamp", type = "string" },
      { name = "payload",   type = "string" },
    ]
  })
}

Pull Subscription

# 
resource "google_pubsub_subscription" "worker" {
  name  = "${var.project}-worker"
  topic = google_pubsub_topic.events.id
 
  ack_deadline_seconds       = 60
  message_retention_duration = "604800s"  # 7 days
  retain_acked_messages      = false
  enable_exactly_once_delivery = true
 
  expiration_policy {
    ttl = ""  # Never expires
  }
 
  retry_policy {
    minimum_backoff = "10s"
    maximum_backoff = "600s"
  }
 
  dead_letter_policy {
    dead_letter_topic     = google_pubsub_topic.dlq.id
    max_delivery_attempts = 5
  }
 
  labels = { environment = var.environment }
}

Push Subscription

# 
resource "google_pubsub_subscription" "webhook" {
  name  = "${var.project}-webhook"
  topic = google_pubsub_topic.events.id
 
  push_config {
    push_endpoint = "${google_cloud_run_v2_service.api.uri}/webhook"
 
    oidc_token {
      service_account_email = google_service_account.pubsub.email
      audience              = google_cloud_run_v2_service.api.uri
    }
 
    attributes = {
      x-goog-version = "v1"
    }
  }
 
  ack_deadline_seconds = 30
 
  retry_policy {
    minimum_backoff = "10s"
    maximum_backoff = "600s"
  }
}

Dead-Letter Topic

# 
resource "google_pubsub_topic" "dlq" {
  name = "${var.project}-events-dlq"
  message_retention_duration = "604800s"  # 7 days
}
 
resource "google_pubsub_subscription" "dlq_monitor" {
  name  = "${var.project}-dlq-monitor"
  topic = google_pubsub_topic.dlq.id
  ack_deadline_seconds = 60
}
 
# Grant Pub/Sub permission to publish to DLQ
resource "google_pubsub_topic_iam_member" "dlq_publisher" {
  topic  = google_pubsub_topic.dlq.id
  role   = "roles/pubsub.publisher"
  member = "serviceAccount:service-${data.google_project.current.number}@gcp-sa-pubsub.iam.gserviceaccount.com"
}
 
resource "google_pubsub_subscription_iam_member" "dlq_subscriber" {
  subscription = google_pubsub_subscription.worker.id
  role         = "roles/pubsub.subscriber"
  member       = "serviceAccount:service-${data.google_project.current.number}@gcp-sa-pubsub.iam.gserviceaccount.com"
}

BigQuery Subscription

# 
resource "google_pubsub_subscription" "bigquery" {
  name  = "${var.project}-to-bigquery"
  topic = google_pubsub_topic.events.id
 
  bigquery_config {
    table            = "${google_bigquery_table.events.project}.${google_bigquery_table.events.dataset_id}.${google_bigquery_table.events.table_id}"
    write_metadata   = true
    use_topic_schema = true
  }
}

Message Filtering

# 
resource "google_pubsub_subscription" "order_events" {
  name  = "${var.project}-order-events"
  topic = google_pubsub_topic.events.id
 
  filter = "attributes.eventType = \"OrderCreated\""
 
  ack_deadline_seconds = 30
}

IAM

# 
# Publisher access
resource "google_pubsub_topic_iam_member" "publisher" {
  topic  = google_pubsub_topic.events.id
  role   = "roles/pubsub.publisher"
  member = "serviceAccount:${google_service_account.app.email}"
}
 
# Subscriber access
resource "google_pubsub_subscription_iam_member" "subscriber" {
  subscription = google_pubsub_subscription.worker.id
  role         = "roles/pubsub.subscriber"
  member       = "serviceAccount:${google_service_account.worker.email}"
}
#

Conclusion

#

Use pull subscriptions for worker-based processing and push for HTTP webhook delivery. Always configure dead-letter topics, set appropriate retry policies, and use message filtering to route events efficiently. BigQuery subscriptions give you analytics without writing any code.

#Terraform#GCP#Pub/Sub#Messaging#Infrastructure as Code

Share this article

Related articles