Search code examples
node.jscsvwebdygraphswebpagetest

How to parse http/https response and export it to a csv file

I have comma separated data from http/https response. I want to parse it and export it to csv so that i can give this csv data to dygraph.js to make charts out of it.

Here is the node.js code i have written, it hits the url and gets data in string format, i am using node.js request library to hit the url.

Here is the response which i am getting after hitting the url, the type of the response is string and is comma separated.

Solution 
  • var str = '"column1","column2","column3","row1col1","row1col2","row1col3","row2col1","row2col2","row2col3",';

str = str.replace(/"/g, '');    // get a rid of quotes

var arr = str.split(',');       // get array

// remove last item since it is empty, there's nothing after last comma   
arr.splice(arr.length-1, 1);    

// you need to know how many columns there is 
// since rows are not terminated by new line -> \n 
// at least on the picture you posted
// let's assume 3 columns
var columns = 3;

var numberOfRows = arr.length / columns; // get number of rows

var newArr = [];

var col2 = 1; //index of the column we want

//now let's create array of objects for each row with only 2nd column
for (var i = 1; i < numberOfRows; i++) { // skip first line i = 1;

    if ( arr.length % columns === 0) {
        newArr[i-1] = {};
        newArr[i-1][arr[col2]] = arr[(i * columns) + 1];

        //if you want column 3 as well
        //newArr[i-1][arr[col2+1]] = arr[(i * columns) + 2];
    }
}

console.log(newArr);

// [ { column2: 'row1col2', column3: 'row1col3' },
//   { column2: 'row2col2', column3: 'row2col3' } ]


// this is from http://stackoverflow.com/questions/22784802/how-does-one-convert-an-object-into-csv-using-javascript
function CSV(array) {
    // Use first element to choose the keys and the order
    var keys = [];
    for (var k in array[0]) keys.push(k);

    // Build header
    var result = keys.join(",") + "\n";

    // Add the rows
    array.forEach(function(obj){
        keys.forEach(function(k, ix){
            if (ix) result += ",";
            result += obj[k];
        });
        result += "\n";
    });

    return result;
}

console.log(CSV(newArr));

/*
column2,column3\n
row1col2,row1col3\n
row2col2,row2col3\n
*/