配置logstash以从Amazon S3存储桶读取日志

Question

我一直在尝试配置logstash来读取在我的亚马逊S3存储桶中生成的日志,但是没有成功.以下是详细信息:

1. 我在ec2实例上安装了logstash
2. 我的日志是s3存储桶中的所有gz文件
3. conf文件如下所示:

  input {
    s3 {
      access_key_id => "MY_ACCESS_KEY_ID"
      bucket => "MY_BUCKET"
      region => "MY_REGION"
      secret_access_key => "MY_SECRET_ACESS_KEY"
      prefix => "/"
      type => "s3"
      add_field => { source => gzfiles }
    }
  }

  filter {
    if [type] == "s3" {
      csv {
        columns => [ "date", "time", "x-edge-location", "sc-bytes", "c-ip", "cs-method", "Host", "cs-uri-stem", "sc-status", "Referer", "User-Agent", "cs-uri-query", "Cookie", "x-edge-result-type", "x-edge-request-id" ]  
      }
    }

    if([message] =~ /^#/) {
      drop{}
    } 
  }

  output {
    elasticsearch {
      host => "ELASTICSEARCH_URL" protocol => "http"
    } 
  }