How to return a large set of records from cloudant by using db.search?

Question

db.find works fine but it's very slow. So I'm trying to use db.search with search index but it limits to 200 records. So then I'm calling same search to go thorough all bookmarks by using nested promise that runs in out of memory. Is anything wrong with the code below?

let qOptions;
qOptions = {'include_docs': true, limit: 200}
qOptions.q = "name: 'BEN'";

let allDocuments = [];
this.getAllDocuments(qOptions, allDocuments, 'app', 'myIndex')
.catch(err => {
  ....
})
.then((result) => {
  ....
});

and then the recurring promise:

getAllDocuments(queryOpqOptionstions, allDocuments, designName, indexName) {
  const promise = new Promise((resolve, reject) => {
    dbConnection.search(designName, indexName, qOptions, (err, result) => {
      if (!err) {
        if (result.rows.length === 0) {
          resolve(allDocuments)
        } else {
          qOptions.bookmark = result.bookmark;
          result.rows.forEach((result) => {
            allDocuments.push(result)
          })

          this.getAllDocuments(qOptions, allDocuments, designName, indexName).then((docs) => {
            resolve(docs);
          }).catch((error) => {
            logger.error({err: error});
            reject(error);
          })
        }
      } else {
        logger.error({err: err});
        reject(err);
      }
    })
  })

  return promise;
}

Answer 1

You should not pass docs throu resolve.

be careful getAllDocuments(queryOpqOptionstions... queryOpqOptionstions sounds strange

You'd better use Array.concat instead of forEach...push

initialize allDocuments as member variable

this.allDocuments=[]
this.getAllDocuments(qOptions, 'app', 'myIndex')
.catch(err => {
  ....
})
.then((result) => {
  ....
});

...

getAllDocuments(qOptions, designName, indexName) {
  const promise = new Promise((resolve, reject) => {
    dbConnection.search(designName, indexName, qOptions, (err, result) => {
      if (!err) {
        if (result.rows.length === 0) {
          resolve()
        } else {
          qOptions.bookmark = result.bookmark;
          this.allDocuments = this.allDocuments.concat(result.rows)
          this.getAllDocuments(qOptions).then((docs) => {
            resolve();
          }).catch((error) => {
            logger.error({err: error});
            reject(error);
          })
        }
      } else {
        logger.error({err: err});
        reject(err);
      }
    })
  })

  return promise;
}