使用NodeJS将文件上载到Amazon S3

Question

我在尝试将文件上传到S3存储桶时遇到了问题.除了我的文件参数看起来不合适之外,一切都有效.我正在使用Amazon S3 sdk从nodejs上传到s3.

这些是我的路线设置:

var multiparty = require('connect-multiparty'),
    multipartyMiddleware = multiparty();
app.route('/api/items/upload').post(multipartyMiddleware, items.upload);

这是items.upload()函数:

exports.upload = function(req, res) {
    var file = req.files.file;
    var s3bucket = new AWS.S3({params: {Bucket: 'mybucketname'}});
    s3bucket.createBucket(function() {
        var params = {
            Key: file.name,
            Body: file
        };
        s3bucket.upload(params, function(err, data) {
            console.log("PRINT FILE:", file);
            if (err) {
                console.log('ERROR MSG: ', err);
            } else {
                console.log('Successfully uploaded data');
            }
        });
    });
};

将Bodyparam 设置为字符串就"hello"可以了.根据doc,Bodyparam必须采用 (Buffer,Typed Array,Blob,String,ReadableStream)对象数据.但是,上载文件对象失败,并显示以下错误消息:

[Error: Unsupported body payload object]

这是文件对象:

{ fieldName: 'file',
  originalFilename: 'second_fnp.png',
  path: '/var/folders/ps/l8lvygws0w93trqz7yj1t5sr0000gn/T/26374-7ttwvc.png',
  headers: 
   { 'content-disposition': 'form-data; name="file"; filename="second_fnp.png"',
     'content-type': 'image/png' },
  ws: 
   { _writableState: 
      { highWaterMark: 16384,
        objectMode: false,
        needDrain: true,
        ending: true,
        ended: true,
        finished: true,
        decodeStrings: true,
        defaultEncoding: 'utf8',
        length: 0,
        writing: false,
        sync: false,
        bufferProcessing: false,
        onwrite: [Function],
        writecb: null,
        writelen: 0,
        buffer: [],
        errorEmitted: false },
     writable: true,
     domain: null,
     _events: { error: [Object], close: [Object] },
     _maxListeners: 10,
     path: '/var/folders/ps/l8lvygws0w93trqz7yj1t5sr0000gn/T/26374-7ttwvc.png',
     fd: null,
     flags: 'w',
     mode: 438,
     start: undefined,
     pos: undefined,
     bytesWritten: 261937,
     closed: true },
  size: 261937,
  name: 'second_fnp.png',
  type: 'image/png' }

任何帮助将不胜感激!

Answer 1

所以看起来这里有一些问题.根据您的帖子,您似乎正在尝试使用connect-multiparty中间件支持文件上传.这个中间件的作用是获取上传的文件,将其写入本地文件系统,然后设置req.files为上传的文件.

您的路线配置看起来很好,问题看起来与您的items.upload()功能有关.特别是这部分:

var params = {
  Key: file.name,
  Body: file
};

正如我在答案开头提到的那样,connect-multiparty将文件写入本地文件系统,因此您需要打开文件并读取文件,然后上传文件,然后在本地文件系统上删除它.

那说你可以将你的方法更新为如下所示:

var fs = require('fs');
exports.upload = function (req, res) {
    var file = req.files.file;
    fs.readFile(file.path, function (err, data) {
        if (err) throw err; // Something went wrong!
        var s3bucket = new AWS.S3({params: {Bucket: 'mybucketname'}});
        s3bucket.createBucket(function () {
            var params = {
                Key: file.originalFilename, //file.name doesn't exist as a property
                Body: data
            };
            s3bucket.upload(params, function (err, data) {
                // Whether there is an error or not, delete the temp file
                fs.unlink(file.path, function (err) {
                    if (err) {
                        console.error(err);
                    }
                    console.log('Temp File Delete');
                });

                console.log("PRINT FILE:", file);
                if (err) {
                    console.log('ERROR MSG: ', err);
                    res.status(500).send(err);
                } else {
                    console.log('Successfully uploaded data');
                    res.status(200).end();
                }
            });
        });
    });
};

这样做是从本地文件系统读取上传的文件,然后将其上传到S3,然后删除临时文件并发送响应.

这种方法存在一些问题.首先,它没有那么高效,因为对于大文件,您将在编写之前加载整个文件.其次,这个过程不支持大文件的多部分上传(我认为在你必须进行多部分上传之前,截止值是5 Mb).

我建议使用的是一个我一直在使用的模块,称为S3FS,它提供与Node.JS中的本机FS类似的接口,但抽象出一些细节,如多部分上传和S3 api (以及添加一些额外的功能,如递归方法).

如果您要引入S3FS库,您的代码将如下所示:

var fs = require('fs'),
    S3FS = require('s3fs'),
    s3fsImpl = new S3FS('mybucketname', {
        accessKeyId: XXXXXXXXXXX,
        secretAccessKey: XXXXXXXXXXXXXXXXX
    });

// Create our bucket if it doesn't exist
s3fsImpl.create();

exports.upload = function (req, res) {
    var file = req.files.file;
    var stream = fs.createReadStream(file.path);
    return s3fsImpl.writeFile(file.originalFilename, stream).then(function () {
        fs.unlink(file.path, function (err) {
            if (err) {
                console.error(err);
            }
        });
        res.status(200).end();
    });
};

这将做的是为提供的存储桶和AWS凭证实例化模块,然后创建存储桶(如果它不存在).然后,当请求通过上传文件时,我们将打开一个流到该文件并使用它将文件写入S3到指定的路径.这将处理幕后的多部分上传部分(如果需要),并且具有通过流完成的好处,因此您无需等待在开始上载之前读取整个文件.

如果您愿意,可以将代码更改为Promises的回调.或者使用带有事件侦听器的pipe()方法来确定结束/错误.

如果您正在寻找其他方法,请查看s3fs的文档,如果您正在寻找其他方法或遇到问题,请随时打开问题.

Answer 2

使用AWS SDK v3

npm install @aws-sdk/client-s3

上传代码

import { S3Client, PutObjectCommand } from "@aws-sdk/client-s3";

/**
* advisable to save your AWS credentials and configurations in an environmet file. Not inside the code
* AWS lib will automatically load the AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY if available in your environment
*/
const s3Client = new S3Client({ region: process.env.AWS_S3_REGION });

/**
* upload a file
* @param file the file object to be uploaded
* @param fileKey the fileKey. could be separated with '/' to nest the file into a folder structure. eg. members/user1/profile.png
*/
export function uploadFile(file, fileKey){
    s3Client.send(new PutObjectCommand({
       Bucket: process.env.MY_AWS_S3_BUCKET,
       Key: fileKey,
       Body: file
    }));
}

如果你想下载

import { GetObjectCommand } from "@aws-sdk/client-s3";
/**
 * download a file from AWS and send to your rest client
 */
app.get('/download', function(req, res, next){
    var fileKey = req.query['fileKey'];

    var bucketParams = {
        Bucket: 'my-bucket-name',
        Key: fileKey,
    };

    res.attachment(fileKey);
    var fileStream = await s3Client.send(new GetObjectCommand(bucketParams));
    // for TS you can add: if (fileStream.Body instanceof Readable)
    fileStream.Body.pipe(res)
});

Answer 3

我发现以下是一个有效的解决方案::

npm install aws-sdk

安装aws-sdk后,请使用以下代码将值替换为您需要的值.

var AWS = require('aws-sdk');
var fs =  require('fs');

var s3 = new AWS.S3();

// Bucket names must be unique across all S3 users

var myBucket = 'njera';

var myKey = 'jpeg';
//for text file
//fs.readFile('demo.txt', function (err, data) {
//for Video file
//fs.readFile('demo.avi', function (err, data) {
//for image file                
fs.readFile('demo.jpg', function (err, data) {
  if (err) { throw err; }



     params = {Bucket: myBucket, Key: myKey, Body: data };

     s3.putObject(params, function(err, data) {

         if (err) {

             console.log(err)

         } else {

             console.log("Successfully uploaded data to myBucket/myKey");

         }

      });

});

如果您正在寻找参考资料,我在这里找到了关于这个主题的完整教程::

如何使用node.js上传亚马逊s3中的文件(文本/图像/视频)

Answer 4

或使用承诺：

const AWS = require('aws-sdk');
AWS.config.update({
    accessKeyId: 'accessKeyId',
    secretAccessKey: 'secretAccessKey',
    region: 'region'
});

let params = {
    Bucket: "yourBucketName",
    Key: 'someUniqueKey',
    Body: 'someFile'
};
try {
    let uploadPromise = await new AWS.S3().putObject(params).promise();
    console.log("Successfully uploaded data to bucket");
} catch (e) {
    console.log("Error uploading data: ", e);
}

Answer 5

将文件上传到 AWS s3 并发送 url 作为响应以访问该文件。

Multer是一个用于处理multipart/form-data的node.js中间件，主要用于上传文件。它写在 Busboy 之上，以实现最高效率。在这里检查这个 npm 模块。

发送请求时，请确保标头的 Content-Type 为 multipart/form-data。我们在响应中发送文件位置，这将给出 url，但如果您想访问该 url，请将存储桶公开，否则您将无法访问它。

上传路由器.js

const express = require('express');
const router = express.Router();
const AWS = require('aws-sdk');
const multer = require('multer');
const storage = multer.memoryStorage()
const upload = multer({storage: storage});

const s3Client = new AWS.S3({
    accessKeyId: 'your_access_key_id',
    secretAccessKey: 'your_secret_access_id',
    region :'ur region'
});

const uploadParams = {
         Bucket: 'ur_bucket_name', 
         Key: '', // pass key
         Body: null, // pass file body
};


router.post('/api/file/upload', upload.single("file"),(req,res) => {
    const params = uploadParams;

    uploadParams.Key = req.file.originalname;
    uploadParams.Body = req.file.buffer;

    s3Client.upload(params, (err, data) => {
        if (err) {
            res.status(500).json({error:"Error -> " + err});
        }
        res.json({message: 'File uploaded successfully','filename': 
        req.file.originalname, 'location': data.Location});
    });
});

module.exports = router;

应用程序.js

const express = require('express');
const app = express();

const router = require('./app/routers/upload.router.js');
app.use('/', router);

// Create a Server
  const server = app.listen(8080, () => {
  console.log("App listening at 8080"); 
})

Answer 6

上传 CSV/Excel

const fs = require('fs');
const AWS = require('aws-sdk');

const s3 = new AWS.S3({
    accessKeyId: XXXXXXXXX,
    secretAccessKey: XXXXXXXXX
});

const absoluteFilePath = "C:\\Project\\test.xlsx";

const uploadFile = () => {
  fs.readFile(absoluteFilePath, (err, data) => {
     if (err) throw err;
     const params = {
         Bucket: 'testBucket', // pass your bucket name
         Key: 'folderName/key.xlsx', // file will be saved in <folderName> folder
         Body: data
     };
      s3.upload(params, function (s3Err, data) {
                    if (s3Err) throw s3Err
                    console.log(`File uploaded successfully at ${data.Location}`);
                    debugger;
                });
  });
};

uploadFile();