How to upload file from S3 bucket to server (SFTP) in NodeJS?

Question

I am trying to use ssh2-sftp-client in NodeJS to send a file from an S3 bucket in AWS to my server. I have seen many examples and I have tried them all with no success. I feel that this one is close but still is not working for me:

async function sendFileViaSftp(srcEvent){
    let Client = require('ssh2-sftp-client');
    let Path = '/';

    let sftp = new Client();
    await sftp.connect({
        host: '73.XXX.XX.XXX', 
        port: 22,
        username: 'username',
        password: 'mypassword'
    }).then(() => {
        console.log("Connected...");
        return sftp.list(Path);
    }).then((list) => {
        console.log("It worked");
        console.log("file to transfer: " + srcEvent.file);
        var fs = require('fs');
        var path = require('path');

        var params = {
            Bucket: srcEvent.bucket,
            Key: srcEvent.key
        };

        var tempFileName = path.join('/tmp/', srcEvent.file);
        var tempFile = fs.createWriteStream(tempFileName);

        s3.getObject(params).createReadStream().pipe(tempFile);
        console.log("file is in tmp");

        let data = fs.createReadStream(tempFileName);
        let remote = '/';
        sftp.put(data, remote);
        console.log("the code makes it to here and prints this");
        

        return list;
    }).catch((err) => {
        console.log('Catch Error: ', err);
        throw new Error(err);
    });
}

I am calling this function like this:

if (folder === 'something') {
        await sendFileViaSftp(srcEvent);
        return {
            statusCode: 200,
            body: srcEvent
        };
}

I do not get any error message, it looks like it just times out which I don't understand because I am using async/await. In my example, I am attempting to pull the file from the S3 bucket, and storing it in /tmp/ and then sending /tmp/test.xls. srcEvent.file is test.xls in this case.

Even if I can simply send a blank txt file to my server from this function, that would be helpful. Thanks!

Answer 1

I figured it out. Not sure where exactly the problem was but this will successfully pull a file from S3 and then upload to SFTP server:

async function sendFileViaSftp(srcEvent) {
    let Client = require('ssh2-sftp-client');
    var path = require('path');

    var localFolder = "/tmp";
    var remoteFolder = "/complete";

    var localfile = path.join(localFolder, srcEvent.file);
    var remotePath = path.join(remoteFolder, srcEvent.file);
    getFileFromS3(srcEvent, localFolder);

    let sftp = new Client();
    await sftp.connect(sftpCredentials).then(() => {

            try {
                return sftp.fastPut(localfile, remotePath);
            } catch (err) {
                console.log("Could not upload file: " + err);
            }
        })
        .then(() => {
            console.log("ending connections");
            sftp.end();
        })
        .catch(err => {
            console.error(err.message);
        });
}

async function getFileFromS3(srcEvent, localFolder) {
    var params = {
        Bucket: srcEvent.bucket,
        Key: srcEvent.key
    };

    var tempFileName = path.join(localFolder, srcEvent.file);
    var tempFile = fs.createWriteStream(tempFileName);

    s3.getObject(params).createReadStream().pipe(tempFile);
    console.log("Put file in temp");

}