Search code examples
javascriptnode.jsamazon-web-servicesamazon-s3fs

Download file from url and upload it to AWS S3 without saving - node.js


I'm writing an application which downloads images from a url and then uploads it to an S3 bucket using the aws-sdk.

Perviously I was just downloading images and saving them to disk like this.

request.head(url, function(err, res, body){

    request(url).pipe(fs.createWriteStream(image_path));

});

And then uploading the images to AWS S3 like this

fs.readFile(image_path, function(err, data){
    s3.client.putObject({
        Bucket: 'myBucket',
        Key: image_path,
        Body: data
        ACL:'public-read'
    }, function(err, resp) {
        if(err){
            console.log("error in s3 put object cb");
        } else { 
            console.log(resp);
            console.log("successfully added image to s3");
        }
    });
});

But I would like to skip the part where I save the image to disk. Is there some way I can pipe the response from request(url) to a variable and then upload that?


Solution

  • Here's some javascript that does this nicely:

        var options = {
            uri: uri,
            encoding: null
        };
        request(options, function(error, response, body) {
            if (error || response.statusCode !== 200) { 
                console.log("failed to get image");
                console.log(error);
            } else {
                s3.putObject({
                    Body: body,
                    Key: path,
                    Bucket: 'bucket_name'
                }, function(error, data) { 
                    if (error) {
                        console.log("error downloading image to s3");
                    } else {
                        console.log("success uploading to s3");
                    }
                }); 
            }   
        });