Terraform：InvalidParameterException：PutSubscriptionFilter操作无法与供应商es的destinationArn一起使用

Question

我正在尝试复制AWS控制台功能，只需单击一个日志组，然后在ES上选择流，然后选择我的托管ES之一。

我以为aws_cloudwatch_log_subscription_filter是我要找的东西。即使文档说它只是Kinesis和Lambda，我还是对自己说：“也许文档是旧的/不完整的”

所以我尝试了

resource "aws_cloudwatch_log_subscription_filter" "whatever" {                                                                                                                                                                      
  name            = "lambda_logs_to_es"                                                                                                                                                                           
  role_arn        = aws_iam_role.my_lamda_role.arn                                                                                                                                                                            
  log_group_name  = aws_cloudwatch_log_group.my_log_group.name                                                                                                                                                                
  filter_pattern  = ""                                                                                                                                                                                                                       
  destination_arn = "arn:aws:es:eu-west-3:585399047133:domain/my-es"                                                                                                                          
}    

毫不奇怪，它告诉我

Terraform  InvalidParameterException: PutSubscriptionFilter operation cannot work with destinationArn for vendor es

因此，是否有使用terraform复制此功能的简单方法，而无需编写我自己的lambda并将我的其他lambda（lambda-ception）的日志转发给ES？（这是我从AWS选择ES而不是安装自己的ES的原因之一，感觉它可以更好地与其他AWS服务集成）？

即是否已经有使用该lambda的lambda和terrafor模块（如果可能的话，由AWS支持）来执行此功能？

Answer 1

我稍后将用更多代码编辑答案，但长话短说：

没有什么神奇的，您在 AWS 控制台中单击，实际上创建了一个将日志转发到 ES 的 lambda，与亚马逊管理的弹性搜索没有神奇的直接链接。

所以我实际上最终做了

#
# Create a lambda that will forward other lambda's logs
# to the VPC's elasticsearch, this whole configuration can be done
# manually though the "stream to elasticsearch" option when you
# choose where to stream a log group of cloudwatch
#

resource "aws_iam_role" "logs_to_es_lambda_role" {
  name = "logs_to_es_lambda_${local.env_type}"

  assume_role_policy = <<EOF
{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Action": "sts:AssumeRole",
      "Principal": {
        "Service": "lambda.amazonaws.com"
      },
      "Effect": "Allow",
      "Sid": ""
    }
  ]
}
EOF
}

resource "aws_iam_role_policy_attachment" "attach_basic_role_to_logs_to_es" {
  role = aws_iam_role.logs_to_es_lambda_role.id
  # this policy permits to log in cloudwatch
  policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole"
}

resource "aws_iam_role_policy_attachment" "access_vpc_role_to_logs_to_es" {
  role = aws_iam_role.logs_to_es_lambda_role.id
  # this policy permits the lambda to have access to the vpc (as the Elastic search
  # is only accessible from there
  policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole"
}

resource "aws_cloudwatch_log_group" "logs_to_es_log_group" {
  name              = "/aws/lambda/logs_to_es_lambda_role_${local.env_type}"
  retention_in_days = 7
}

resource "aws_security_group" "logs_to_es" {
  description = "allow all egress network for the lambda logs_to_es"
  vpc_id      = module.vpc.vpc_id
  name        = "lambda-${local.env_type}-logs-to-es"

  egress {
    protocol    = "-1"
    from_port   = 0
    to_port     = 0
    cidr_blocks = ["0.0.0.0/0"]
  }

  tags = {
    Name               = "lambda-${local.env_type}-logs-to-es"
    ManagedByTerraform = "true"
    EnvironmentType    = "${local.env_type}"
  }
}


data "archive_file" "logs_to_es_zip" {
  type        = "zip"
  source_file = "lambdas/logs_to_elasticsearch/logs_to_elasticsearch.js"
  output_path = "lambdas/logs_to_elasticsearch/logs_to_elasticsearch.zip"
}

resource "aws_lambda_function" "logs_to_es" {

  filename         = data.archive_file.logs_to_es_zip.output_path
  source_code_hash = data.archive_file.logs_to_es_zip.output_base64sha256
  function_name    = "logs_to_es_${local.env_type}"

  role    = aws_iam_role.logs_to_es_lambda_role.arn
  handler = "logs_to_elasticsearch.handler"

  timeout = 100

  runtime = "nodejs10.x"

  description = "${local.env_type} script that forwards cloudwatch logs of that VPC's lambda to ES"

  vpc_config {
    subnet_ids = module.vpc.private_subnets
    security_group_ids = [
      aws_security_group.logs_to_es.id
    ]
  }

  environment {
    variables = {
      ELASTICSEARCH_ENDPOINT = aws_elasticsearch_domain.technical_logs.endpoint
    }
  }

  tags = {
    Environment        = local.env_type
    ManagedByTerraform = "true"
  }

  depends_on = [
    "aws_iam_role_policy_attachment.attach_basic_role_to_logs_to_es",
    "aws_iam_role_policy_attachment.access_vpc_role_to_logs_to_es",
    "aws_cloudwatch_log_group.logs_to_es_log_group",
  ]
}

lambda 的代码是

#
# Create a lambda that will forward other lambda's logs
# to the VPC's elasticsearch, this whole configuration can be done
# manually though the "stream to elasticsearch" option when you
# choose where to stream a log group of cloudwatch
#

resource "aws_iam_role" "logs_to_es_lambda_role" {
  name = "logs_to_es_lambda_${local.env_type}"

  assume_role_policy = <<EOF
{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Action": "sts:AssumeRole",
      "Principal": {
        "Service": "lambda.amazonaws.com"
      },
      "Effect": "Allow",
      "Sid": ""
    }
  ]
}
EOF
}

resource "aws_iam_role_policy_attachment" "attach_basic_role_to_logs_to_es" {
  role = aws_iam_role.logs_to_es_lambda_role.id
  # this policy permits to log in cloudwatch
  policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole"
}

resource "aws_iam_role_policy_attachment" "access_vpc_role_to_logs_to_es" {
  role = aws_iam_role.logs_to_es_lambda_role.id
  # this policy permits the lambda to have access to the vpc (as the Elastic search
  # is only accessible from there
  policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole"
}

resource "aws_cloudwatch_log_group" "logs_to_es_log_group" {
  name              = "/aws/lambda/logs_to_es_lambda_role_${local.env_type}"
  retention_in_days = 7
}

resource "aws_security_group" "logs_to_es" {
  description = "allow all egress network for the lambda logs_to_es"
  vpc_id      = module.vpc.vpc_id
  name        = "lambda-${local.env_type}-logs-to-es"

  egress {
    protocol    = "-1"
    from_port   = 0
    to_port     = 0
    cidr_blocks = ["0.0.0.0/0"]
  }

  tags = {
    Name               = "lambda-${local.env_type}-logs-to-es"
    ManagedByTerraform = "true"
    EnvironmentType    = "${local.env_type}"
  }
}


data "archive_file" "logs_to_es_zip" {
  type        = "zip"
  source_file = "lambdas/logs_to_elasticsearch/logs_to_elasticsearch.js"
  output_path = "lambdas/logs_to_elasticsearch/logs_to_elasticsearch.zip"
}

resource "aws_lambda_function" "logs_to_es" {

  filename         = data.archive_file.logs_to_es_zip.output_path
  source_code_hash = data.archive_file.logs_to_es_zip.output_base64sha256
  function_name    = "logs_to_es_${local.env_type}"

  role    = aws_iam_role.logs_to_es_lambda_role.arn
  handler = "logs_to_elasticsearch.handler"

  timeout = 100

  runtime = "nodejs10.x"

  description = "${local.env_type} script that forwards cloudwatch logs of that VPC's lambda to ES"

  vpc_config {
    subnet_ids = module.vpc.private_subnets
    security_group_ids = [
      aws_security_group.logs_to_es.id
    ]
  }

  environment {
    variables = {
      ELASTICSEARCH_ENDPOINT = aws_elasticsearch_domain.technical_logs.endpoint
    }
  }

  tags = {
    Environment        = local.env_type
    ManagedByTerraform = "true"
  }

  depends_on = [
    "aws_iam_role_policy_attachment.attach_basic_role_to_logs_to_es",
    "aws_iam_role_policy_attachment.access_vpc_role_to_logs_to_es",
    "aws_cloudwatch_log_group.logs_to_es_log_group",
  ]
}

然后要使用它，您只需将该 lambda ARN 放入aws_cloudwatch_log_subscription_filter资源中即可。