Search code examples
node.jslarge-files

How to read large jsonl file backwards in nodejs


A simple way to read files line by line top to bottom:

const { once } = require('events');
const { createReadStream } = require('fs');
const { createInterface } = require('readline');

let storeProducts = async () => {
    try {
        const filename = path.join(__dirname, "bulk-101638307937.jsonl");
        const rl = createInterface({
          input: createReadStream(filename),
          crlfDelay: Infinity
        });
    
        rl.on('line', (line) => {
          // Process the line.
          console.log({line: JSON.parse(line)});
        });
    
        await once(rl, 'close');
    
        console.log('File processed.');
      } catch (err) {
        console.error(err);
    }
}

this works well. But I need to read a large jsonl file backwards line by line. I've tried this .It reads file backwards but not line by line. Any help would be appreciated.

A sample jsonl

{"id": 1, "product": "A"}
{"variant_id": 1, "product": "A"}
{"id": 2, "product": "B"}
{"variant_id": 2, "product": "B"}
{"id": 3, "product": "C"}
{"variant_id": 3, "product": "C"}
{"id": 4, "product": "D"}
{"variant_id": 4, "product": "D"}
{"variant_id": 5, "product": "D"}

Solution

  • The library fs-reverse should do the trick:

    const fsR = require('fs-reverse');
    const filename = './sample.jsonl';
    
    const readStream = fsR(filename, {})
    
    readStream.on('data', (line) => {
      if(line) { // have this check to make sure empty lines are not parsed
        console.log(JSON.parse(line))
      }
    })