Search code examples
node.jsamazon-s3gulpaws-sdkaws-cli

Copying AWS S3 Bucket root contents to same bucket within subfolder


I want to be able to copy files within the same bucket from the root directory to a subfolder within a subfolder however excluding that subfolder by using the aws-sdk.

i.e:

I want to use this AWS-CLI command in a gulp file task:

aws s3 cp s3://bucketName s3://bucketName/last_good/YYYYMMDD --recursive --exclude "last_good/*"

I've used the copy examples used from How to copy/move all objects in Amazon S3 from one prefix to other using the AWS SDK for Node.js

I am just not sure how to specify the folder to exclude. In my above example it would be the last_good folder.

var gulp = require('gulp');
var AWS = require('aws-sdk');
var async = require('async');
var bucketName = 'bucketname';
var oldPrefix = '';
var newPrefix = 'last_good/20190817/';
var s3 = new AWS.S3({params: {Bucket: bucketName}, region: 'us-west-2'});

gulp.task('publish', function() {
     CopyToLastGood();
}

function CopyToLastGood() {
    var done = function(err, data) {
      if (err) console.log(err);
      else console.log(data);
    };

s3.listObjects({Prefix: oldPrefix}, function(err, data) {
  if (data.Contents.length) {
    async.each(data.Contents, function(file, cb) {
      var params = {
        CopySource: bucketName + '/' + file.Key,
        Key: file.Key.replace(oldPrefix, newPrefix)
      };
      s3.copyObject(params, function(copyErr, copyData){
        if (copyErr) { // an error occured
          console.log(err);
        }
        else {
          console.log('Copied: ', params.Key); //successful response
          cb();
        }
      });
    }, done);
  }
});

}

I expect contents of root to update last_good/20190817/ however not copying the last_good folder itself.


Solution

  • I've solved my solution using a delimiter option on the s3.listObjects params.

    i.e:

    s3.listObjects({Prefix: oldPrefix, Delimiter:'/'}
    

    this only lists files within the root.