BaconJS Directory Structure Recursion Example

Question

I'm just starting out on FRP and baconjs, so forgive my ignorance on terminology.

I am on a project where I'm trying to create a directory structure within google drive. I need to ensure the parent directories are created prior to the children directories to create the hierarchy structure.

I've written the following code. However, after a few minutes, I'm running into "User rate limit exceeded" thrown by Google which means I'm calling google with too many requests too quickly.

I need to find a way to optimize the create directories in sequence recursively and then hopefully add some batching so I can speed up the sequential calls.

Does anyone have an idea on how to do this?

var _generateFolder = function(dirArray, gDrive, parentDir) {
    // chunk array into size of 2 elements
    var chunked = _.chunk(dirArray, 5)
    return Bacon
        .sequentially(1000 * 5, chunked) // Stream with all the chunks
        .flatMapConcat(function(arrayChunk){ // Stream for all items
            return Bacon.fromArray(arrayChunk)
        })
        .flatMap(_createOrUpdateGFolder)
        .filter(function removeAllItemsWithNoChildren(dir) {
            return dir.children.length > 0;
        })
        .flatMap(function createSubDirectoriesForItem(dir) {
            console.log("Sync children folders:" + dir.title);
            return _generateFolder(dir.children, gDrive, dir);
        })
}

dir = {
    title:"A",
    children:[
        {
            title: "AA"
            children:[
                {
                    title:"AAA",
                    children:[]
                }
            ]
        },
        {
            title: "AB"
            children:[
                {
                    title:"ABA",
                    children:[]
                },
                {
                    title:"ABB",
                    children:[]
                }
            ]
        },
    ]
}
_generateFolder(dir, drive, null)

Is there a way to have a single stream, add directories to that stream, recursively? And then use chunk+sequential on that stream?

Answer 1

Note: This is not a solution but a quick and dirty approach.

I would recursively walk through JSON directory provided. Baconjs provides an API Bacon.fromBinder that allows you to create custom streams. Everytime I'd hit a "title" property, I would "emit" a bacon event. This would generate a stream of events for each title visited.

var dirs = {
  title: "A",
  children: [{
    title: "AA",
    children: [{
      title: "AAA",
      children: []
    }]
  }, {
    title: "AB",
    children: [{
      title: "ABA",
      children: []
    }, {
      title: "ABB",
      children: []
    }]
  }, ]
};

function walk(sink, r) {
  if (Array.isArray(r)) {
    r.forEach(function(child) {
      return walk(sink, child);
    })
  } else if (typeof r === 'object') {
    sink(r.title);
    walk(sink, r.children);
  }
}


Bacon.fromBinder(function(sink) {
  walk(sink, dirs);
  return function() {
    // return a sensible unsubscribe
  };
}).log();

// To insert into DOM - demo purposes
var stream = Bacon.fromBinder(function(sink) {
  walk(sink, dirs);
  return function() {};
});

$("#events").append('<ul></ul>');
var list;

stream.onValue(function(val) {
  list = $("#events").find('ul');
  list.append("<li>" + val + "</li>");
})

<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.1.1/jquery.min.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/bacon.js/0.7.71/Bacon.min.js"></script>
<div id="events"></div>