Search code examples
javascriptarraysjsontyped-arrays

Best way to store/retrieve big array (mixture of string and typed array) into/from blob or file


Lets say I have big array which has different type of data like json string and typed array such as mainData

var strObj = JSON.stringify({'name' : 'suman', 'age' : 29};);
var strObj2 = JSON.stringify({'name' : 'laxmi', 'age' : 29});

var tpArr = new Uint8Array(2);
    tpArr[0]  = 42;
    tpArr[1]  = 52;

var tpArr2 = new Uint8Array(2);
    tpArr[0]  = 32;
    tpArr[1]  = 52;

var mainData = [{pt : 20, recObjs : strObj}, {pt : 30, recObjs : strObj2},
                {pt : 40, recObjs : tpArr}, {pt : 50, recObjs : tpArr2}];

I have created blob file(type json) with array mainData and export it into json file, like

var stringiFyData = JSON.stringify(mainData);   
var myBlob = new Blob([stringiFyData], {'type': 'application/json'});
var link2 = document.getElementById('mydata');
link2.href = window.URL.createObjectURL(myBlob);

Now after file exported as json, I read the file something like,

var fileInput = document.getElementById('fileInput');
var file = fileInput.files[0];
var reader = new FileReader();
reader.onload = function(e) {
     mainData = JSON.parse(reader.result);
}
reader.readAsText(file);    

Now elements of mainData array, tpArr, tpArr2 lost their's original type from Uint8Array into Object.

I need this data tpArr, tpArr2 in it's original form. For this, first I do convert tpArr object into array. After get the length of array, create typed array something like

newTypeArr = new Uint8Array(conNewArr.length);
for(var i=0; i<conNewArr.length; i++ ){
    newTypeArr[i] = conNewArr[i];
}

But length of conNewArr array is about 3000 and which means we have to run loop 6000, first 3000 to convert object into array to know the length, and second 3000 to a create typed array.

And of course the mainData may has that kind of array in thousands.

Other alternative

I tried to export(new Blob(mainData, {'type': 'application/octet-stream'}) it into array buffer and after read the exported file, I got all the data in typed array format but I need these data as it was before exported(mixture of string and typed array).

My questions are

  1. How can we export mixtutre data of string and typed array in blob file and retrieve these data from blob/file without loosing it's original form(type)?
  2. Is the my approach okay ?
  3. How can we reduce the iteration of such big loop in my approach?

window.URL = window.URL || window.webkitURL;
	var myObject = {'name' : 'suman', 'age' : 29};
	var myObject2 = {'name' : 'laxmi', 'age' : 29};
	
	var strObj = JSON.stringify(myObject);
	var strObj2 = JSON.stringify(myObject2);
	
	var tpArr = new Uint8Array(5);
		tpArr[0]  = 42;
		tpArr[1]  = 52;
		tpArr[2]  = 62;
		tpArr[3]  = 22;
		tpArr[4]  = 42;
	    
	var	tpArr2 = new Uint8Array(5);
		tpArr[0]  = 32;
		tpArr[1]  = 52;
		tpArr[2]  = 42;
		tpArr[3]  = 402;
		tpArr[4]  = 142;

	var mainData = [
		{pt : 20, recObjs : strObj},
		{pt : 30, recObjs : strObj2},
		{pt : 40, recObjs : tpArr},
		{pt : 50, recObjs : tpArr2}
		];
	
	
	var stringiFyData = JSON.stringify(mainData);	
	var myBlob = new Blob([stringiFyData], {'type': 'application/json'});
	var link2 = document.getElementById('mydata');
	link2.href = window.URL.createObjectURL(myBlob);
	
	function readFile (){
		var fileInput = document.getElementById('fileInput');
		var file = fileInput.files[0];
		var reader = new FileReader();
		reader.onload = function(e) {
			 data = JSON.parse(reader.result);
		}
		reader.readAsText(file);	
	}
<a href="#"	 id="mydata" >Download Data</a>
	<input type='file' id="fileInput" accept='text/plain' onchange='readFile()'>


Solution

  • Why do you want to export everything is Blob? Maybe you want to export all data as single file and import it again at later stage.

    I suggest following.

    Convert all Typed array into Base64 and JSON at end.

    https://jsfiddle.net/esusedf7/2/

    // Create a array with dummy Values
    var typed_array = new Uint8Array(100000);
    for (var i=0;i<100000;i++) {
        typed_array[i]=i;
    }
    
    // Convert Array into Base64
    var sMyBase64 = base64EncArr(typed_array);
    // Convert Base64 to Array
    var ori1 = base64DecToArr(sMyBase64);
    
    // Convert Array into String (comma seperated)
    var comma = bufferToString(typed_array);
    // Convert String (comma seperated) into Array
    var ori2 = stringToBuffer(comma);
    
    // Compare if converted array is exactly equal to Original Array
    if (typed_array.length == ori2.length) {
        var match=0;
        for (var i=0;i<2000;i++) {
            if (typed_array[i] !== ori2[i]) {
                match = 1;
            }
        }
        if (match == 1) {
            alert('ori2 Fail');
        }
    } else {
        alert('ori2 Fail');
    }
    

    Original Array: 100000
    Base64 String: 136844 Percentage Increase 137%
    Comma String: 356989 Percentage Increase 357%
    Json String: 1145881 Percentage Increase 1146%