使用Java从S3并行下载多个文件到zip文件

Question

我想从某些外部来源（如S3）下载多个文件，创建一个zip包含所有这些文件的文件，并向用户显示下载该zip文件的链接。

显然，我可以按顺序处理文件，读取每个文件的输入流并将其写入ZipOutputStream。

如何并行读取所有输入文件流并写入单个输出流，这样我就可以向用户显示下载链接，而不必等到zip文件完全写入后才可以？

我当前的代码：

String realpath = getServletContext().getRealPath("/");
response.setContentType("application/zip");

response.setHeader("Content-Disposition","attachment; filename="+fi.replace('/', '-')+"_"+ff.replace('/', '-')+".zip");

ServletOutputStream out = null;
ZipOutputStream zipfile = null;

try
{
    List<Object[]> cfdis = /*my hibernate criteria source, your Database?*/
    out = response.getOutputStream();
    zipfile = new ZipOutputStream(out);
    ZipEntry zipentry = null;
    for(Object[] cfdi:cfdis)
    {
        zipentry = new ZipEntry(cfdi[1].toString()+".xml");
        zipfile.putNextEntry(zipentry);
        InputStream in = new FileInputStream(new File(realpath+cfdi[0].toString()));
        byte[] bytes = new byte[FILEBUFFERSIZE];
        int bytesRead;
        while ((bytesRead = in.read(bytes)) != -1)
        {
            zipfile.write(bytes, 0, bytesRead);
        }
    }

Answer 1

读取流中的第一个文件并将其开始写入输出流

protected void doGet(HttpServletRequest request, HttpServletResponse response)
        throws ServletException, IOException {

    ServletOutputStream sos = response.getOutputStream();
    ZipOutputStream zos = new ZipOutputStream(sos);

    try {
        S3ServiceWrapper s3Service = new S3ServiceWrapper();
        ZipEntry zipentry = null;
        byte bytes[] = new byte[4096];
        response.setContentType("application/zip");
        response.setHeader("Content-Disposition", "attachment; filename=java-s3-download.ZIP");

        for (String objectKey : objectKeys) {
            String name = objectKey.substring(objectKey.lastIndexOf("/"), objectKey.length());
            log.info("Start Writing File::" + name);

            zipentry = new ZipEntry(name);
            zos.putNextEntry(zipentry);
            InputStream in = s3Service.downloadFileAsStream(bucketName, objectKey);
            int bytesRead = -1;

            while ((bytesRead = in.read(bytes)) != -1) {
                zos.write(bytes, 0, bytesRead);
            }

            log.info("Finsih Writing File::" + name);
            in.close();
        }
    } catch (Exception e)

    {
        e.printStackTrace();
    } finally {
        zos.flush();
        zos.closeEntry();
        zos.close();
        sos.close();
    }

}

public InputStream downloadFileAsStream(String bucketName, String objectKey) {
    if (s3Service == null) {
        return null;
    }

    try {
        GetObjectRequest s3ObjectReq = new GetObjectRequest(bucketName, objectKey);
        log.info("Downloading file having key = " + objectKey);
        long startTime=System.currentTimeMillis();
        S3Object downlodedObjectMD = s3Service.getObject(s3ObjectReq);
        log.info("Time to load Stream is "+(System.currentTimeMillis()-startTime)+" ms");
        return downlodedObjectMD.getObjectContent();

    } catch (Exception e) {
        log.error("EXCEPTION = " + e.getMessage(), e);
    }
    return null;
}