Download multiple files in parallel to a zip-file from S3 using Java

Question

I'd like to download multiple files from some external source like S3, create a single zip file containing all those files and present the user a link to download that zip file.

Obviously I can process the files sequentially, reading the input stream of each one and writing it to ZipOutputStream.

How can I read all the input file streams in parallel and write to a single output stream so that I can present a download link to the user without making them wait until zip file is fully written?

My current code:

String realpath = getServletContext().getRealPath("/");
response.setContentType("application/zip");

response.setHeader("Content-Disposition","attachment; filename="+fi.replace('/', '-')+"_"+ff.replace('/', '-')+".zip");

ServletOutputStream out = null;
ZipOutputStream zipfile = null;

try
{
    List<Object[]> cfdis = /*my hibernate criteria source, your Database?*/
    out = response.getOutputStream();
    zipfile = new ZipOutputStream(out);
    ZipEntry zipentry = null;
    for(Object[] cfdi:cfdis)
    {
        zipentry = new ZipEntry(cfdi[1].toString()+".xml");
        zipfile.putNextEntry(zipentry);
        InputStream in = new FileInputStream(new File(realpath+cfdi[0].toString()));
        byte[] bytes = new byte[FILEBUFFERSIZE];
        int bytesRead;
        while ((bytesRead = in.read(bytes)) != -1)
        {
            zipfile.write(bytes, 0, bytesRead);
        }
    }

Answer 1

Read stream for very first file & start writing it to outputstream

protected void doGet(HttpServletRequest request, HttpServletResponse response)
        throws ServletException, IOException {

    ServletOutputStream sos = response.getOutputStream();
    ZipOutputStream zos = new ZipOutputStream(sos);

    try {
        S3ServiceWrapper s3Service = new S3ServiceWrapper();
        ZipEntry zipentry = null;
        byte bytes[] = new byte[4096];
        response.setContentType("application/zip");
        response.setHeader("Content-Disposition", "attachment; filename=java-s3-download.ZIP");

        for (String objectKey : objectKeys) {
            String name = objectKey.substring(objectKey.lastIndexOf("/"), objectKey.length());
            log.info("Start Writing File::" + name);

            zipentry = new ZipEntry(name);
            zos.putNextEntry(zipentry);
            InputStream in = s3Service.downloadFileAsStream(bucketName, objectKey);
            int bytesRead = -1;

            while ((bytesRead = in.read(bytes)) != -1) {
                zos.write(bytes, 0, bytesRead);
            }

            log.info("Finsih Writing File::" + name);
            in.close();
        }
    } catch (Exception e)

    {
        e.printStackTrace();
    } finally {
        zos.flush();
        zos.closeEntry();
        zos.close();
        sos.close();
    }

}

public InputStream downloadFileAsStream(String bucketName, String objectKey) {
    if (s3Service == null) {
        return null;
    }

    try {
        GetObjectRequest s3ObjectReq = new GetObjectRequest(bucketName, objectKey);
        log.info("Downloading file having key = " + objectKey);
        long startTime=System.currentTimeMillis();
        S3Object downlodedObjectMD = s3Service.getObject(s3ObjectReq);
        log.info("Time to load Stream is "+(System.currentTimeMillis()-startTime)+" ms");
        return downlodedObjectMD.getObjectContent();

    } catch (Exception e) {
        log.error("EXCEPTION = " + e.getMessage(), e);
    }
    return null;
}