获取s3对象元数据然后创建流

Question

我正在从s3下载一个对象并从中创建一个读取流对象来处理视频:

s3.getObject(params).createReadStream()

但是,当我通过访问其'metadata'属性获取对象时,我需要从中获取元数据:

s3.getObject()

我怎么会:

1. 获取对象s3.getObject(),从元数据属性中获取元数据,然后将其转换为读取流？

   var stream = fs.createReadStream(response); 不工作 - 输入必须是一个字符串

- 要么 -

2. 通过流获取流s3.getObject().createReadStream(),并从流中提取元数据？

   据我所知,元数据不会在流中传递.

---

告诉我,如果我的假设是错误的,但我目前仍然坚持这两个需求:

• 获取元数据
• 使它成为一个流

Answer 1

您可以通过请求的httpHeaders事件获取元数据.

let fs = require('fs')
let aws = require('aws-sdk')
let s3 = new aws.S3()

let request = s3.getObject({
    Bucket: 'my-bucket',
    Key: 'my-key'
})

let stream

request.on('httpHeaders', (statusCode, httpHeaders) => {
    // object metadata is represented by any header in httpHeaders starting with 'x-amz-meta-'
    // you can use the stream object that this point
    stream.pipe(fs.createWriteStream('./somepath'))
    stream.on('end', () => { 
        console.log('were done')
    })
})

stream = request.createReadStream()

或者,您也可以调用s3.headObject以获取元数据而无需下载对象,然后使用下载对象s3.getObject

Answer 2

所以我找到了一个解决方案。这适用于大多数 10 MB 以下的文件。如果它们大于该值，则缓冲流在文件写入完成之前结束。我尝试将 bufferStream.end 放在 on finish 函数中，但随后我的回调没有完成......

function download(s3Event, srcKey, cb){

  console.log('Starting download');

  s3.getObject({
      Bucket: s3Event.bucket.name,
      Key: srcKey
  }, cb);
}



function writeToFile(data, cb){

  var dlFile = path.join(tempDir, 'download');

  console.log('data = ', data);
  console.log('data.Body = ', data.Body);
  var stream = bufferStream.pipe(fs.createWriteStream(dlFile)).on('finish', function () {
     console.log('finished writing stream');
     cb(null, data);
  });

  bufferStream.end(data.Body);
}


exports.handler = function(event, context) {

// Read options from the event.

console.log("Reading options from event:\n", util.inspect(event, {depth: 5}));

var s3Event = event.Records[0].s3;
var srcKey = decodeURIComponent(s3Event.object.key);
var keyPrefix = srcKey.replace(/\.[^/.]+$/, '');         
var dstBucket = "jump-lambda";


async.waterfall([

    function (cb){
        download(s3Event, srcKey, cb);
    },

    function (data, cb){
        writeToFile(data, cb);
    },

    function (data, cb){
        fluentffmpegProcess(data, cb);
    },

    function (data, cb){
        transform(data, cb);
    },

    function (data, buffer, cb){
        thumbnailUpload(data, buffer, dstBucket, keyPrefix, cb);
    },

    function (data, cb){
        updateParse(data, srcKey, keyPrefix, cb);
    },

],

    function (err) {
        if (err) {
            console.error(
                'Unable to convert video to scene object, with error code: ' + err.description
            );
        } else {
            console.log(
                'Successfully created scene object, updated venue, and created thumbnail'
            );
        }
    }
);
};