使用AWS Lambda从S3上的目录创建Tar归档

Question

我需要提取存储在s3上的一堆zip文件,并将它们添加到tar存档并将该存档存储在s3上.可能是zip文件的总和将大于lambda函数允许的512mb本地存储.我有一个部分的功能,从s3获取对象提取它们并将它们放在一个s3对象中,而不使用lambda本地存储.

提取对象线程

public class ExtractObject implements Runnable{

    private String objectName;
    private String uuid;
    private final byte[] buffer = new byte[1024];

    public ExtractAdvert(String name, String uuid) {
        this.objectName= name;
        this.uuid= uuid;
    }

    @Override
    public void run() {
        final String srcBucket = "my-bucket-name";
        final AmazonS3 s3Client = new AmazonS3Client();

        try {
            S3Object s3Object = s3Client.getObject(new GetObjectRequest(srcBucket, objectName));
            ZipInputStream zis = new ZipInputStream(s3Object.getObjectContent());
            ZipEntry entry = zis.getNextEntry();

            while(entry != null) {
                String fileName = entry.getName();
                String mimeType = FileMimeType.fromExtension(FilenameUtils.getExtension(fileName)).mimeType();
                System.out.println("Extracting " + fileName + ", compressed: " + entry.getCompressedSize() + " bytes, extracted: " + entry.getSize() + " bytes, mimetype: " + mimeType);
                ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
                int len;
                while ((len = zis.read(buffer)) > 0) {
                    outputStream.write(buffer, 0, len);
                }
                InputStream is = new ByteArrayInputStream(outputStream.toByteArray());
                ObjectMetadata meta = new ObjectMetadata();
                meta.setContentLength(outputStream.size());
                meta.setContentType(mimeType);
                System.out.println("##### " + srcBucket + ", " + FilenameUtils.getFullPath(objectName) + "tmp" + File.separator + uuid + File.separator + fileName);

                // Add this to tar archive instead of putting back to s3
                s3Client.putObject(srcBucket, FilenameUtils.getFullPath(objectName) + "tmp" + File.separator + uuid + File.separator + fileName, is, meta);
                is.close();
                outputStream.close();
                entry = zis.getNextEntry();
            }
            zis.closeEntry();
            zis.close();
        } catch (IOException ioe) {
                System.out.println(ioe.getMessage());
        }
    }
}

---

这将针对需要提取的每个对象运行,并将它们保存在tar文件所需的结构中的s3对象中.

我认为我需要的是将对象放回到s3而不是将其保存在内存中并将其添加到tar存档中.并上传,但经过大量的环顾和试验和错误后,我还没有创建一个成功的tar文件.主要问题是我不能在lambda中使用tmp目录.

---

编辑 我应该创建tar文件,而不是将对象放到s3？(请参阅注释// Add this to tar archive instead of putting back to s3)如果是这样我如何创建tar流而不在本地存储它？

---

编辑2:尝试去皮重文件

ListObjectsV2Request req = new ListObjectsV2Request().withBucketName(bucketName);
ListObjectsV2Result result;

ByteArrayOutputStream baos = new ByteArrayOutputStream();
TarArchiveOutputStream tarOut = new TarArchiveOutputStream(baos);

do {
    result = s3Client.listObjectsV2(req);

    for (S3ObjectSummary objectSummary : result.getObjectSummaries()) {

        if(objectSummary.getKey().startsWith("tmp/") )  {
            System.out.printf(" - %s (size: %d)\n", objectSummary.getKey(), objectSummary.getSize());
            S3Object s3Object = s3Client.getObject(new GetObjectRequest(bucketName, objectSummary.getKey()));
            InputStream is = s3Object.getObjectContent(); 
            System.out.println("Pre Create entry");
            TarArchiveEntry archiveEntry = new TarArchiveEntry(IOUtils.toByteArray(is));
            // Getting following exception above
            // IllegalArgumentException: Invalid byte 111 at offset 7 in ' positio' len=8
            System.out.println("Pre put entry");
            tarOut.putArchiveEntry(archiveEntry);
            System.out.println("Post put entry");
        }
    }

    String token = result.getNextContinuationToken();
    System.out.println("Next Continuation Token: " + token);
    req.setContinuationToken(token);
} while (result.isTruncated());

ObjectMetadata metadata = new ObjectMetadata();
InputStream is = new ByteArrayInputStream(baos.toByteArray());
s3Client.putObject(new PutObjectRequest(bucketName, bucketFolder + "tar-file", is, metadata));

Answer 1

我找到了一个解决方案，它与我在上面的编辑 2中的尝试非常相似。

private final String bucketName = "bucket-name";
private final String bucketFolder = "tmp/";
private final String tarKey = "tar-dir/tared-file.tar";

private void createTar() throws IOException, ArchiveException {
    ListObjectsV2Request req = new ListObjectsV2Request().withBucketName(bucketName);
    ListObjectsV2Result result;

    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    TarArchiveOutputStream tarOut = new TarArchiveOutputStream(baos);

    do {
        result = s3Client.listObjectsV2(req);

        for (S3ObjectSummary objectSummary : result.getObjectSummaries()) {
            if (objectSummary.getKey().startsWith(bucketFolder)) {
                S3Object s3Object = s3Client.getObject(new GetObjectRequest(bucketName, objectSummary.getKey()));
                InputStream is = s3Object.getObjectContent();

                String s3Key = objectSummary.getKey();
                String tarPath = s3Key.substring(s3Key.indexOf('/') + 1, s3Key.length());
                s3Key.lastIndexOf('.'));

                byte[] ba = IOUtils.toByteArray(is);

                TarArchiveEntry archiveEntry = new TarArchiveEntry(tarPath);
                archiveEntry.setSize(ba.length);
                tarOut.putArchiveEntry(archiveEntry);
                tarOut.write(ba);
                tarOut.closeArchiveEntry();
            }
        }

        String token = result.getNextContinuationToken();
        System.out.println("Next Continuation Token: " + token);
        req.setContinuationToken(token);
    } while (result.isTruncated());

    ObjectMetadata metadata = new ObjectMetadata();
    InputStream is = baos.toInputStream();
    metadata.setContentLength(baos.size());
    s3Client.putObject(new PutObjectRequest(bucketName, tarKey, is, metadata));
}