I have about 35000 xml files (7Gb total size) I need to convert to json files.
I have written a conversion script with help of the npm package xml2js.
It complains that I have not a callback function.
Also it looks like there are some issues with memory (FATAL ERROR: CALL_AND_RETRY_LAST Allocation failed - JavaScript heap out of memory).
I'm quite new to JS so I would much appreciate if some one could point me in right direction on how to fix this error.
code:
const fs = require('fs');
const path = require('path');
const xml2js = require('xml2js');
const parser = new xml2js.Parser();
const pxmls = path.join(__dirname, 'xml-files');
const pjsons = path.join(__dirname, 'json-files');
function onError(err) {
console.log(err);
}
// Here's the reading part:
function readFiles(pxmls, onError) {
fs.readdir(pxmls, (err, filenames) => {
if (err) {
onError(err);
return;
}
filenames.forEach((filename) => {
fs.readFile(`${pxmls}/${filename}`, 'utf-8', (err, data) => {
if (err) {
console.log(err);
return;
}
// changing file extension from .xml to .json
const x = filename.replace(/\.[^/.]+$/, '.json');
parser.parseString(data, (err, result) => {
fs.writeFile(`${pjsons}/${x}`, JSON.stringify(result, null, 4));
});
});
});
});
}
readFiles(pxmls);
Error:
(node:778) [DEP0013] DeprecationWarning: Calling an asynchronous function without callback is deprecated.
(node:778) [DEP0013] DeprecationWarning: Calling an asynchronous function without callback is deprecated.
(node:778) [DEP0013] DeprecationWarning: Calling an asynchronous function without callback is deprecated.
(node:778) [DEP0013] DeprecationWarning: Calling an asynchronous function without callback is deprecated.
(node:778) [DEP0013] DeprecationWarning: Calling an asynchronous function without callback is deprecated.
(node:778) [DEP0013] DeprecationWarning: Calling an asynchronous function without callback is deprecated.
(node:778) [DEP0013] DeprecationWarning: Calling an asynchronous function without callback is deprecated.
(node:778) [DEP0013] DeprecationWarning: Calling an asynchronous function without callback is deprecated.
(node:778) [DEP0013] DeprecationWarning: Calling an asynchronous function without callback is deprecated.
(node:778) [DEP0013] DeprecationWarning: Calling an asynchronous function without callback is deprecated.
(node:778) [DEP0013] DeprecationWarning: Calling an asynchronous function without callback is deprecated.
(node:778) [DEP0013] DeprecationWarning: Calling an asynchronous function without callback is deprecated.
(node:778) [DEP0013] DeprecationWarning: Calling an asynchronous function without callback is deprecated.
(node:778) [DEP0013] DeprecationWarning: Calling an asynchronous function without callback is deprecated.
(node:778) [DEP0013] DeprecationWarning: Calling an asynchronous function without callback is deprecated.
(node:778) [DEP0013] DeprecationWarning: Calling an asynchronous function without callback is deprecated.
(node:778) [DEP0013] DeprecationWarning: Calling an asynchronous function without callback is deprecated.
(node:778) [DEP0013] DeprecationWarning: Calling an asynchronous function without callback is deprecated.
(node:778) [DEP0013] DeprecationWarning: Calling an asynchronous function without callback is deprecated.
(node:778) [DEP0013] DeprecationWarning: Calling an asynchronous function without callback is deprecated.
(node:778) [DEP0013] DeprecationWarning: Calling an asynchronous function without callback is deprecated.
(node:778) [DEP0013] DeprecationWarning: Calling an asynchronous function without callback is deprecated.
(node:778) [DEP0013] DeprecationWarning: Calling an asynchronous function without callback is deprecated.
(node:778) [DEP0013] DeprecationWarning: Calling an asynchronous function without callback is deprecated.
(node:778) [DEP0013] DeprecationWarning: Calling an asynchronous function without callback is deprecated.
(node:778) [DEP0013] DeprecationWarning: Calling an asynchronous function without callback is deprecated.
(node:778) [DEP0013] DeprecationWarning: Calling an asynchronous function without callback is deprecated.
(node:778) [DEP0013] DeprecationWarning: Calling an asynchronous function without callback is deprecated.
(node:778) [DEP0013] DeprecationWarning: Calling an asynchronous function without callback is deprecated.
(node:778) [DEP0013] DeprecationWarning: Calling an asynchronous function without callback is deprecated.
(node:778) [DEP0013] DeprecationWarning: Calling an asynchronous function without callback is deprecated.
(node:778) [DEP0013] DeprecationWarning: Calling an asynchronous function without callback is deprecated.
(node:778) [DEP0013] DeprecationWarning: Calling an asynchronous function without callback is deprecated.
(node:778) [DEP0013] DeprecationWarning: Calling an asynchronous function without callback is deprecated.
(node:778) [DEP0013] DeprecationWarning: Calling an asynchronous function without callback is deprecated.
(node:778) [DEP0013] DeprecationWarning: Calling an asynchronous function without callback is deprecated.
(node:778) [DEP0013] DeprecationWarning: Calling an asynchronous function without callback is deprecated.
(node:778) [DEP0013] DeprecationWarning: Calling an asynchronous function without callback is deprecated.
(node:778) [DEP0013] DeprecationWarning: Calling an asynchronous function without callback is deprecated.
(node:778) [DEP0013] DeprecationWarning: Calling an asynchronous function without callback is deprecated.
(node:778) [DEP0013] DeprecationWarning: Calling an asynchronous function without callback is deprecated.
(node:778) [DEP0013] DeprecationWarning: Calling an asynchronous function without callback is deprecated.
(node:778) [DEP0013] DeprecationWarning: Calling an asynchronous function without callback is deprecated.
(node:778) [DEP0013] DeprecationWarning: Calling an asynchronous function without callback is deprecated.
(node:778) [DEP0013] DeprecationWarning: Calling an asynchronous function without callback is deprecated.
(node:778) [DEP0013] DeprecationWarning: Calling an asynchronous function without callback is deprecated.
<--- Last few GCs --->
[778:0x103000000] 76652 ms: Mark-sweep 1410.3 (1572.0) -> 1410.1 (1572.0) MB, 84.1 / 0.2 ms allocation failure GC in old space requested
[778:0x103000000] 76737 ms: Mark-sweep 1410.1 (1572.0) -> 1410.0 (1552.5) MB, 83.2 / 0.0 ms last resort
[778:0x103000000] 76827 ms: Mark-sweep 1410.0 (1552.5) -> 1410.0 (1552.5) MB, 87.9 / 0.0 ms last resort
<--- JS stacktrace --->
==== JS stack trace =========================================
Security context: 0x9a5c35a9891 <JS Object>
1: stringSlice(aka stringSlice) [buffer.js:527] [pc=0x36d4804ed77c](this=0x1c76a5982311 <undefined>,buf=0x1b593f84b2a1 <an Uint8Array with map 0x2a3c739b1a61>,encoding=0x9a5c35bd099 <String[5]: utf-8>,start=0,end=524715)
2: toString [buffer.js:~569] [pc=0x36d48048b5b3](this=0x1b593f84b2a1 <an Uint8Array with map 0x2a3c739b1a61>,encoding=0x9a5c35bd099 <String[5]: utf-8>,start=0x1c76a59823...
FATAL ERROR: CALL_AND_RETRY_LAST Allocation failed - JavaScript heap out of memory
1: node::Abort() [/usr/local/bin/node]
2: node::FatalException(v8::Isolate*, v8::Local<v8::Value>, v8::Local<v8::Message>) [/usr/local/bin/node]
3: v8::internal::V8::FatalProcessOutOfMemory(char const*, bool) [/usr/local/bin/node]
4: v8::internal::Factory::NewRawOneByteString(int, v8::internal::PretenureFlag) [/usr/local/bin/node]
5: v8::internal::Factory::NewStringFromUtf8(v8::internal::Vector<char const>, v8::internal::PretenureFlag) [/usr/local/bin/node]
6: v8::String::NewFromUtf8(v8::Isolate*, char const*, v8::NewStringType, int) [/usr/local/bin/node]
7: node::StringBytes::Encode(v8::Isolate*, char const*, unsigned long, node::encoding, v8::Local<v8::Value>*) [/usr/local/bin/node]
8: void node::Buffer::(anonymous namespace)::StringSlice<(node::encoding)1>(v8::FunctionCallbackInfo<v8::Value> const&) [/usr/local/bin/node]
9: v8::internal::FunctionCallbackArguments::Call(void (*)(v8::FunctionCallbackInfo<v8::Value> const&)) [/usr/local/bin/node]
10: v8::internal::MaybeHandle<v8::internal::Object> v8::internal::(anonymous namespace)::HandleApiCallHelper<false>(v8::internal::Isolate*, v8::internal::Handle<v8::internal::HeapObject>, v8::internal::Handle<v8::internal::HeapObject>, v8::internal::Handle<v8::internal::FunctionTemplateInfo>, v8::internal::Handle<v8::internal::Object>, v8::internal::BuiltinArguments) [/usr/local/bin/node]
11: v8::internal::Builtin_Impl_HandleApiCall(v8::internal::BuiltinArguments, v8::internal::Isolate*) [/usr/local/bin/node]
12: 0x36d47ff0437d
[1] 778 abort node batch_xml_convert_to_json.js
The heap space error may indicate a memory leak. Otherwise, to increase your maximum heap,
node --max-old-space-size=8192 server.js
to start server.js with 8GB heap.