Configure CloudWatch metric streams with Terraform

// Include both AWS and Grafana Terraform providers
terraform {
  required_providers {
    aws = {
      source  = "hashicorp/aws"
      version = "~> 5.0"
    }
    grafana = {
      source  = "grafana/grafana"
      version = ">= 3.24.1"
    }
  }
}

// Configure the AWS provider with your region and credentials
provider "aws" {
  region  = var.aws_region  // FILLME: Your AWS region, e.g., "us-east-1"
  profile = var.aws_profile // FILLME: Your AWS CLI profile name
}

// Configure Grafana provider to manage Cloud Provider resources
provider "grafana" {
  // Token for accessing Grafana Cloud stack information
  cloud_provider_access_token = var.cloud_provider_token
  // Token for managing AWS integration resources
  cloud_access_policy_token   = var.cloud_provider_token
  // Your stack's Cloud Provider API endpoint
  cloud_provider_url          = var.cloud_provider_url
}

// Fetch information about your Grafana Cloud stack
data "grafana_cloud_stack" "main" {
  slug = var.grafana_cloud_stack_slug // FILLME: Your stack slug, e.g., "mycompany"
}

// Get current AWS region information
data "aws_region" "current" {}

// S3 bucket for storing metric batches that failed to deliver
resource "aws_s3_bucket" "fallback" {
  bucket = var.fallback_bucket_name
}

// IAM role that the Data Firehose stream assumes
resource "aws_iam_role" "firehose" {
  name               = format("Firehose-%s", var.metric_stream_name)
  assume_role_policy = data.aws_iam_policy_document.firehose_assume_role.json
}

// Trust policy allowing AWS Data Firehose service to assume this role
data "aws_iam_policy_document" "firehose_assume_role" {
  statement {
    effect = "Allow"
    principals {
      type        = "Service"
      identifiers = ["firehose.amazonaws.com"]
    }
    actions = ["sts:AssumeRole"]
  }
}

// Permissions policy for Data Firehose operations
resource "aws_iam_role_policy" "firehose" {
  name = format("Firehose-%s", var.metric_stream_name)
  role = aws_iam_role.firehose.id

  policy = jsonencode({
    Version = "2012-10-17"
    Statement = [
      // Allow Data Firehose to write error logs to CloudWatch Logs
      {
        Effect   = "Allow"
        Resource = ["*"]
        Action   = ["logs:PutLogEvents"]
      },
      // Allow Data Firehose to back up failed batches to S3
      {
        Sid    = "s3Permissions"
        Effect = "Allow"
        Action = [
          "s3:AbortMultipartUpload",
          "s3:GetBucketLocation",
          "s3:GetObject",
          "s3:ListBucket",
          "s3:ListBucketMultipartUploads",
          "s3:PutObject",
        ]
        Resource = [
          aws_s3_bucket.fallback.arn,
          "${aws_s3_bucket.fallback.arn}/*",
        ]
      },
    ]
  })
}

// IAM role for CloudWatch metric stream
resource "aws_iam_role" "metric_stream_role" {
  name               = format("metric-stream-role-%s", var.metric_stream_name)
  assume_role_policy = data.aws_iam_policy_document.metric_stream_assume_role.json
}

// Trust policy allowing CloudWatch metric stream service to assume this role
data "aws_iam_policy_document" "metric_stream_assume_role" {
  statement {
    effect = "Allow"
    principals {
      type        = "Service"
      identifiers = ["streams.metrics.cloudwatch.amazonaws.com"]
    }
    actions = ["sts:AssumeRole"]
  }
}

// Policy allowing metric stream to write to Data Firehose
resource "aws_iam_role_policy" "metric_stream_role" {
  name = "AWSCloudWatchMetricStreamPolicy"
  role = aws_iam_role.metric_stream_role.id

  policy = jsonencode({
    Version = "2012-10-17"
    Statement = [
      // Grant permission to send metric data to Data Firehose
      {
        Action = [
          "firehose:PutRecord",
          "firehose:PutRecordBatch"
        ]
        Effect   = "Allow"
        Resource = [aws_kinesis_firehose_delivery_stream.stream.arn]
      },
    ]
  })
}

// Variable for Grafana Cloud's AWS account ID (provided by Grafana)
variable "grafana_cloud_sts_aws_account_id" {
  description = "Grafana Cloud AWS account ID for assuming roles"
  type        = string
  default     = "008923505332" // Grafana Cloud's AWS account
}

// Trust policy allowing Grafana Cloud to assume this role
data "aws_iam_policy_document" "trust_grafana" {
  statement {
    effect = "Allow"
    principals {
      type        = "AWS"
      identifiers = ["arn:aws:iam::${var.grafana_cloud_sts_aws_account_id}:root"]
    }
    actions = ["sts:AssumeRole"]
    // External ID condition prevents confused deputy attacks
    condition {
      test     = "StringEquals"
      variable = "sts:ExternalId"
      values   = [data.grafana_cloud_stack.main.prometheus_user_id]
    }
  }
}

// IAM role for Grafana Cloud to scrape resource metadata
resource "aws_iam_role" "grafana_cloud_aws_resource_metadata" {
  name               = "GrafanaAWSResourceMetadataScrapeJobAccess"
  description        = "Role used by Grafana CloudWatch integration"
  assume_role_policy = data.aws_iam_policy_document.trust_grafana.json
}

// Policy granting read-only access to AWS resource information
resource "aws_iam_role_policy" "grafana_cloud_aws_resource_metadata" {
  name = "GrafanaAWSResourceMetadataScrapeJobAccess"
  role = aws_iam_role.grafana_cloud_aws_resource_metadata.id

  policy = jsonencode({
    Version = "2012-10-17"
    Statement = [
      {
        Effect = "Allow"
        // Permissions to discover resources via tags and API calls
        Action = [
          "tag:GetResources",              // Get resources by tags
          "apigateway:GET",                // API Gateway information
          "aps:ListWorkspaces",            // Amazon Managed Prometheus
          "autoscaling:DescribeAutoScalingGroups",
          "dms:DescribeReplicationInstances",
          "dms:DescribeReplicationTasks",
          "ec2:DescribeTransitGatewayAttachments",
          "ec2:DescribeSpotFleetRequests",
          "shield:ListProtections",
          "storagegateway:ListGateways",
          "storagegateway:ListTagsForResource"
        ]
        Resource = "*" // These are read-only operations requiring wildcard
      }
    ]
  })
}

// Wait for IAM changes to propagate globally
resource "time_sleep" "wait_iam_propagation" {
  depends_on = [
    aws_iam_role.grafana_cloud_aws_resource_metadata,
    aws_iam_role_policy.grafana_cloud_aws_resource_metadata
  ]
  create_duration = "10s"
}

locals {
  // Build the Grafana Cloud metrics ingestion endpoint from stack URL
  // Converts: https://prometheus-prod-03-prod-us-central-0.grafana.net
  // To: https://aws-metric-streams-prod-03.grafana.net/aws-metrics/api/v1/push
  target_endpoint = var.target_endpoint != "" ? var.target_endpoint : format(
    "%s/aws-metrics/api/v1/push",
    replace(
      replace(
        data.grafana_cloud_stack.main.prometheus_url,
        "prometheus",
        "aws-metric-streams"
      ),
      "-${data.grafana_cloud_stack.main.cluster_slug}",
      ""
    )
  )
}

resource "aws_kinesis_firehose_delivery_stream" "stream" {
  name        = format("%s-firehose", var.metric_stream_name)
  destination = "http_endpoint"

  http_endpoint_configuration {
    // Grafana Cloud metrics ingestion endpoint
    url  = local.target_endpoint
    name = "Grafana AWS Metric Stream Destination"
    
    // Authentication: Prometheus user ID and metrics write token
    access_key = format(
      "%s:%s",
      data.grafana_cloud_stack.main.prometheus_user_id,
      var.metrics_write_token
    )
    
    // Buffer size in MB before delivery (1 MB minimum)
    buffering_size = 1
    // Buffer time in seconds before delivery (60 seconds for low latency)
    buffering_interval = 60
    
    // IAM role for Firehose to access S3 and CloudWatch Logs
    role_arn = aws_iam_role.firehose.arn
    
    // Only back up failed deliveries to S3
    s3_backup_mode = "FailedDataOnly"

    request_configuration {
      // Compress data to reduce bandwidth and costs
      content_encoding = "GZIP"
    }

    // S3 bucket configuration for failed batches
    s3_configuration {
      role_arn           = aws_iam_role.firehose.arn
      bucket_arn         = aws_s3_bucket.fallback.arn
      buffering_size     = 5    // 5 MB buffer for S3
      buffering_interval = 300  // 5 minutes buffer for S3
      compression_format = "GZIP"
    }

    // Optional: Enable CloudWatch Logs for delivery errors
    dynamic "cloudwatch_logging_options" {
      for_each = var.log_delivery_errors ? [1] : []
      content {
        enabled         = true
        log_group_name  = var.errors_log_group_name
        log_stream_name = var.errors_log_stream_name
      }
    }
  }
}

resource "aws_kinesis_firehose_delivery_stream" "stream" {
    // ... other configuration from the delivery stream configured earlier in the section ...
    http_endpoint_configuration {
      // ... other configuration from the delivery stream configured earlier in the section ...
  
  request_configuration {
    content_encoding = "GZIP"
    
    // Static label for environment (appears as "environment" in Grafana)
    common_attributes {
      name  = "lbl_environment"
      value = "production"
    }
    
    // Static label for team (appears as "team" in Grafana)
    common_attributes {
      name  = "lbl_team"
      value = "platform"
    }
  }
 }
}

resource "aws_kinesis_firehose_delivery_stream" "stream" {
    // ... other configuration from the delivery stream configured earlier in the section ...
    http_endpoint_configuration {
      // ... other configuration from the delivery stream configured earlier in the section ...
  
  request_configuration {
    content_encoding = "GZIP"
    
    // Example tag selection entry adding three tags: Name, Environment, and Owner
    common_attributes {
      tag_selection  = "tag_Name,tag_Environment,tag_Owner"
    }
  }
 }
}

resource "aws_kinesis_firehose_delivery_stream" "stream" {
    // ... other configuration from the delivery stream configured earlier in the section ...
    http_endpoint_configuration {
      // ... other configuration from the delivery stream configured earlier in the section ...
  
  request_configuration {
    content_encoding = "GZIP"
    
    // Include average statistic set to false
    common_attributes {
      include_average_statistic  = false
    }
  }
 }
}

resource "aws_cloudwatch_metric_stream" "metric_stream" {
  name = var.metric_stream_name
  
  // IAM role allowing metric stream to write to Data Firehose
  role_arn = aws_iam_role.metric_stream_role.arn
  
  // Target Data Firehose delivery stream
  firehose_arn = aws_kinesis_firehose_delivery_stream.stream.arn
  
  // Use OpenTelemetry format (required for Grafana Cloud)
  output_format = "opentelemetry1.0"

  // Optional: Include only specific AWS namespaces
  // If omitted or empty, all namespaces are included
  dynamic "include_filter" {
    for_each = var.include_namespaces
    content {
      namespace = include_filter.value
    }
  }
}

// Register your AWS account with Grafana Cloud
resource "grafana_cloud_provider_aws_account" "main" {
  depends_on = [time_sleep.wait_iam_propagation]
  
  // Your Grafana Cloud stack ID
  stack_id = data.grafana_cloud_stack.main.id
  
  // IAM role ARN that Grafana Cloud assumes
  role_arn = aws_iam_role.grafana_cloud_aws_resource_metadata.arn
  
  // AWS regions to scrape (should match your metric stream regions)
  regions = [data.aws_region.current.name]
}

// Create resource metadata scrape job
resource "grafana_cloud_provider_aws_resource_metadata_scrape_job" "main" {
  stack_id = data.grafana_cloud_stack.main.id
  name     = "aws-resource-metadata-scraper"
  
  // Reference the AWS account created above
  aws_account_resource_id = grafana_cloud_provider_aws_account.main.resource_id

  // Scrape metadata for the same services as your metric stream
  dynamic "service" {
    for_each = var.include_namespaces
    content {
      name = service.value
      
      // Optional: Customize scrape settings per service
      // scrape_interval_seconds = 300
      // tag_filters = {
      //   "Environment" = "production"
      // }
    }
  }
}