Search code examples
javanode.jsorientdb

OrientDB - don't release memory after insert


I use NodeJS for insert 8.000.000 records into my orientdb database, but after about 2.000.000 insert records my app is stopped and show error "Java Heap".

Is there a way for release memory after every record inserted?

Ram usage:
-befor start app: 2.6g
-after insert 2milions records: 7.6g

My app.js (NodeJS):

var dbConn = [];
var dbNext = 0;
var dbMax = 25;

for (var i = 0; i <= dbMax; i++) {
  var db = new ODatabase({
      host: orientdb.host,
      port: 2424,
      username: 'root',
      password: orientdb.password,
      name: 'test',
  });
  dbConn.push(db);
}
//---------------------------------------------------
//Start loop
// record = {name: 'test'}
record["@class"] = "table";
var db = nextDB();
db.open().then(function () {
    return db.record.create(record);
}).then(function (res) {
    db.close().then(function () {
             //----resume loop
    });
  }).error(function (err) {
          //------
  });
// end loop - iteration loop
//---------------------------------------------------
function nextDB() {
  if (++dbNext >= dbMax) {
      dbNext -= dbMax;
  }
  return dbConn[dbNext];
}

Solution

  • OrientJS wasn't efficient for insert massive data from SqlServer to OrientDB. I used ETL module for massive insert, that is fastest way and good idea for transpot massive data without increase memory more than 2GB.
    I could transported 7.000 records per minute.

    My ETL's config.json:

    {
      "config": {
        log : "debug"
      },
      "extractor" : {
        "jdbc": { "driver": "com.microsoft.sqlserver.jdbc.SQLServerDriver",
                  "url": "jdbc:sqlserver://10.10.10.10;databaseName=My_DB;",
                  "userName": "sa",
                  "userPassword": "123",
                  "query": "select * from My_Table" 
                }
      },
    
      "transformers" : [
        { "vertex": { "class": "Company"} }
      ],
       "loader" : {
        "orientdb": {
          "dbURL": "plocal:D:\DB\Orient_DB",
          dbUser: "admin",
          dbPassword: "admin",
          "dbAutoCreate": true,
          "tx": false,
          "batchCommit": 1000,
          "wal" : false,
          "dbType": "graph"
        }
      }
    }