Search code examples
javascriptnode.jsparquet

javascript cant write to parquet file (always 1kb)


I am trying to figure out how to create and write data to a parquet file with nodejs. I created the very simple code below:

var parquet = require('parquetjs');

init();

async function init(){
    console.log('BEGIN DEBUG PARQUET')
    var debugschema = new parquet.ParquetSchema({
        name: { type: 'UTF8' },
        quantity: { type: 'INT64' },
        price: { type: 'DOUBLE' },
        date: { type: 'TIMESTAMP_MILLIS' },
        in_stock: { type: 'BOOLEAN' }
      });
    // create new ParquetWriter that writes to 'fruits.parquet`
    var debugwriter = await parquet.ParquetWriter.openFile(debugschema, `./DEBUGPARQUET.parquet`);
    // append a few rows to the file
    await debugwriter.appendRow({name: 'apples', quantity: 10, price: 2.5, date: new Date(), in_stock: true});
    await debugwriter.appendRow({name: 'apples', quantity: 10, price: 2.5, date: new Date(), in_stock: true});
    await debugwriter.appendRow({name: 'apples', quantity: 10, price: 2.5, date: new Date(), in_stock: true});
    await debugwriter.appendRow({name: 'apples', quantity: 10, price: 2.5, date: new Date(), in_stock: true});
    await debugwriter.appendRow({name: 'apples', quantity: 10, price: 2.5, date: new Date(), in_stock: true});
    await debugwriter.appendRow({name: 'apples', quantity: 10, price: 2.5, date: new Date(), in_stock: true});
    console.log('END DEBUG PARQUET')
}

Which I run with the command node index.js in my windows 10 command prompt terminal, the file runs with no errors and now I have a file DEBUGPARQUET.parquet which is only 1 kb in size, and I can't open this file to view the parquet data at all (ive tried installing the apache parquet view program and using that but it wont work, I cant view contents of the file in vscode with the parquet extension, and online sites to view parquet files wont work either)

Is there something im missing in my code? or is there an issue with writing parquet files on windows 10?


Solution

  • var debugwriter = await parquet.ParquetWriter.openFile(debugschema, `./DEBUGPARQUET.parquet`);
        // append a few rows to the file
        await debugwriter.appendRow({name: 'apples', quantity: 10, price: 2.5, date: new Date(), in_stock: true});
        await debugwriter.appendRow({name: 'apples', quantity: 10, price: 2.5, date: new Date(), in_stock: true});
        await debugwriter.appendRow({name: 'apples', quantity: 10, price: 2.5, date: new Date(), in_stock: true});
        await debugwriter.appendRow({name: 'apples', quantity: 10, price: 2.5, date: new Date(), in_stock: true});
        await debugwriter.appendRow({name: 'apples', quantity: 10, price: 2.5, date: new Date(), in_stock: true});
        await debugwriter.appendRow({name: 'apples', quantity: 10, price: 2.5, date: new Date(), in_stock: true});
    await debugwriter.close();
    console.log('END DEBUG PARQUET')