Receiving "Uploading file too fast!" error from Imgur API

Question

I created a node.js server that uses busboy to take requests, and pipe the files to Imgur for upload. However, I keep getting an "Uploading file too fast!" response from Imgur, and I'm not sure exactly what the problem is. Here is the code snippet involving busboy:

var express = require('express');
var Busboy = require('busboy');
var fs = require('fs');
var request = require('request-promise');
var router = express.Router();

router.post('/u', function(req, res, next) {
    var busboy = new Busboy({headers: req.headers});
    busboy.on('file', function(fieldname, file, filename, encoding, mimetype) {
        if(fieldname == 'image') {
            var options = {
                uri: 'https://api.imgur.com/3/image',
                method: 'POST',
                headers: {
                    'Authorization': 'Client-ID ' + clientID // put client id here
                },

                form: {
                    image: file,
                    type: 'file'
                }
            };

            request(options)
                .then(function(parsedBody) {
                    console.log(parsedBody);
                })
                .catch(function(err) {
                    console.log(err);
                });
        }
    });
    busboy.on('field', function(fieldname, val, fieldnameTruncated, valTruncated) {
        console.log('field');
    });
    busboy.on('finish', function() {
        res.status(200).end();
    });
    req.pipe(busboy);
});

As you can see I'm piping the request file directly into my request for imgur. Providing a ReadStream by simply saving the file to disc and then using fs.createReadStream() works perfectly, so I'm not really sure why trying to pipe directly from request to request gives me the error. The exact response I'm getting from Imgur is:

StatusCodeError: 400 - {"data":{"error":"Uploading file too fast!","request":"\/3\/image","method":"POST"},"success":false,"status":400} 

If anyone has encountered this before, it would be helpful...

Answer 1

The first issue is that you should be using formData instead of form for file uploads. Otherwise, the request library won't send the correct HTTP request.

The second issue is that the stream object won't have the correct content length until it's fully processed. We can buffer the data ourselves and pass it after the initial file stream from busboy has processed.*

This gives us something that looks like

var express = require('express');
var Busboy = require('busboy');
var fs = require('fs');
var request = require('request-promise');
var router = express.Router();

router.post('/u', function(req, res, next) {
    var busboy = new Busboy({headers: req.headers});
    busboy.on('file', function(fieldname, file, filename, encoding, mimetype) {
        if(fieldname == 'image') {
            // the buffer
            file.fileRead = [];
            file.on('data', function(data) {
                // add to the buffer as data comes in
                this.fileRead.push(data);
            });

            file.on('end', function() {
                // create a new stream with our buffered data
                var finalBuffer = Buffer.concat(this.fileRead);

                var options = {
                    uri: 'https://api.imgur.com/3/image',
                    method: 'POST',
                    headers: {
                        'Authorization': 'Client-ID ' + clientID // put client id here
                    },

                    formData: {
                        image: finalBuffer,
                        type: 'file'
                    }
                };

                request(options)
                    .then(function(parsedBody) {
                        console.log(parsedBody);
                    })
                    .catch(function(err) {
                        console.log(err);
                    });
            });
        }
    });
    busboy.on('field', function(fieldname, val, fieldnameTruncated, valTruncated) {
        console.log('field');
    });
    busboy.on('finish', function() {
        res.status(200).end();
    });
    req.pipe(busboy);
});

• Code for the buffering is from http://thau.me/2014/02/nodejs-streaming-files-to-amazons3/

Lastly, you may want to consider using the request library, as the request-promise library discourages the use of streams. See the github repo for more details: https://github.com/request/request-promise