Search code examples
javascriptcsveventsasync-awaites6-promise

Write Blocking Js code to write = csv file (using csv-parser) after scraping data from web


In writeFile fn the records returned by makeCsv function are empty (used await still :/ empty) how to make all the code inside makeCsv blocking such that it i get all entries in records array when i call fn(makeCsv)
Expected Code Flow

  1. makeCsv -> reads a csv from local storage and calls function getPubKey
  2. getPubKey-> fetches a key against each call for the account name passed by makeCsv by making a request to url and returns that key
  3. makeCsv-> appends key property to object pushes it to results array and returns it
  4. writeFile -> calls makeCsv and takes array to write new Csv with keys included

    Issue : As soon as call to getPubkey function is made end event is triggred resolving promise with empty array and thus writeFile receives empty array because it runs before makeCsv has ended creating all requests and then adding keys to data objects

Code

const csv = require('csv-parser')
const createCsvWriter = require('csv-writer').createObjectCsvWriter;
const fetch = require('node-fetch');
const fs = require('fs');
let nkas = []

async function makeCsv(results) {
    const readable = fs.createReadStream('/home/vector/Desktop/test.csv')
    return new Promise((resolve, reject) => {
        readable.pipe(csv(['acc', 'balance']))
            .on('data', async (data) => {
                data.key = await getPubkey(data.acc)
                results.push(data)
            })
            .on('end', () => {
               return resolve(results);
            });
    })
}

async function getPubkey(account) {
    const body = { account_name: account }
    let key;
    await fetch('https://eos.greymass.com/v1/chain/get_account', {
        method: 'post',
        body: JSON.stringify(body),
        headers: { 'Content-Type': 'application/json' },
    })
        .then(res => res.json())
        .then(json => key = json.permissions[1].required_auth.keys[0].key)
        .catch(err => console.log(err))
    if (key && key.length > 0)
        return key
    else
        nkas.push(account);
    console.log(nkas);

}

async function writeFile() {
    const csvWriter = createCsvWriter({
        path: 'out.csv',
        header: [
            { id: 'acc', title: 'Account' },
            { id: 'balance', title: 'Balance' },
            { id: 'key', title: 'PubKey' }
        ]
    });
    let records = []
    records = await makeCsv(records)
    console.log(records)
    csvWriter.writeRecords(records)       // returns a promise
        .then(() => {
            console.log('...Done');
        });
}
writeFile();


Solution

  • You need to wait for the getPubKey() promises to resolve before resolving the makeCsv() promise:

    async function makeCsv() {
        const readable = fs.createReadStream('/home/vector/Desktop/test.csv')
        const pending = [];
        return new Promise((resolve, reject) => {
            readable.pipe(csv(['acc', 'balance']))
                .on('data', (data) => {
                    const getKey = async (data) => {
                        data.key = await getPubkey(data.acc)
                        return data
                    }
                    pending.push(getKey(data))
                })
                .on('end', () => {
                    Promise.all(pending).then(results => resolve(results));
                });
        })
    }