在现有S3对象上运行S3-put-triggered Lambda函数？

Question

我在Node.js中有一个Lambda函数来处理添加到我的存储桶的新图像.我想为所有现有对象运行该函数.我怎样才能做到这一点？我认为最简单的方法是"重新放置"每个对象,以触发该功能,但我不知道该怎么做.

要清楚 - 我想在每个现有对象上运行一次.触发器已经在为新对象工作,我只需要在创建lambda函数之前插入的对象上运行它.

Answer 1

以下 Lambda 函数将满足您的要求。

它将遍历目标 S3 存储桶中的每个文件，并针对每个文件执行所需的 lambda 函数以模拟放置操作。

您可能希望为此函数设置很长的执行时间余量

var TARGET_BUCKET="my-bucket-goes-here";
var TARGET_LAMBDA_FUNCTION_NAME="TestFunct";
var S3_PUT_SIMULATION_PARAMS={
  "Records": [
    {
      "eventVersion": "2.0",
      "eventTime": "1970-01-01T00:00:00.000Z",
      "requestParameters": {
        "sourceIPAddress": "127.0.0.1"
      },
      "s3": {
        "configurationId": "testConfigRule",
        "object": {
          "eTag": "0123456789abcdef0123456789abcdef",
          "sequencer": "0A1B2C3D4E5F678901",
          "key": "HappyFace.jpg",
          "size": 1024
        },
        "bucket": {
          "arn": "arn:aws:s3:::mybucket",
          "name": "sourcebucket",
          "ownerIdentity": {
            "principalId": "EXAMPLE"
          }
        },
        "s3SchemaVersion": "1.0"
      },
      "responseElements": {
        "x-amz-id-2": "EXAMPLE123/5678abcdefghijklambdaisawesome/mnopqrstuvwxyzABCDEFGH",
        "x-amz-request-id": "EXAMPLE123456789"
      },
      "awsRegion": "us-east-1",
      "eventName": "ObjectCreated:Put",
      "userIdentity": {
        "principalId": "EXAMPLE"
      },
      "eventSource": "aws:s3"
    }
  ]
};

var aws = require('aws-sdk');
var s3 = new aws.S3();
var lambda = new aws.Lambda();


exports.handler = (event, context, callback) => {
    retrieveS3BucketContents(TARGET_BUCKET, function(s3Objects){
        simulateS3PutOperation(TARGET_BUCKET, s3Objects, simulateS3PutOperation, function(){ 
            console.log("complete."); 
        });
    });
};

function retrieveS3BucketContents(bucket, callback){
    s3.listObjectsV2({
        Bucket: TARGET_BUCKET
    }, function(err, data) {
        callback(data.Contents);
    });
}

function simulateS3PutOperation(bucket, s3ObjectStack, callback, callbackEmpty){
    var params = {
      FunctionName: TARGET_LAMBDA_FUNCTION_NAME, 
      Payload: ""
    };

    if(s3ObjectStack.length > 0){
        var s3Obj = s3ObjectStack.pop();
        var p = S3_PUT_SIMULATION_PARAMS;
        p.Records[0].s3.bucket.name = bucket;
        p.Records[0].s3.object.key = s3Obj.Key;
        params.Payload = JSON.stringify(p, null, 2);
        lambda.invoke(params, function(err, data) {
          if (err) console.log(err, err.stack); // an error occurred
          else{
              callback(bucket, s3ObjectStack, callback, callbackEmpty);
          }
        });
    }
    else{
        callbackEmpty();   
    }
}

以下是执行此方法所需的 lambda 查询的完整策略，它允许 R/W 到 CloudWatch 日志和 ListObject 访问 S3。您需要在看到 MY-BUCKET-GOES-HERE 的地方填写您的存储桶详细信息

{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Sid": "Stmt1477382207000",
            "Effect": "Allow",
            "Action": [
                "s3:ListBucket"
            ],
            "Resource": [
                "arn:aws:s3:::MY-BUCKET-GOES-HERE/*"
            ]
        },
        {
      "Effect": "Allow",
      "Action": [
        "logs:CreateLogGroup",
        "logs:CreateLogStream",
        "logs:PutLogEvents"
      ],
      "Resource": "arn:aws:logs:*:*:*"
    }
    ]
}