Please see edits at the bottom. This seems to be a Mongo-C and or native node-mongodb issue and not a Mongoose issue.
I have a real time collection of seismic data stored in MongoDB. One key, called data, is an int array of samples. When I query this collection via Mongoose, everything looks fine but the data key comes back as an empty array, when is should come back as Number array of length nsamp(number of samples).
I have defined the Mongoose Schema as:
var waveformSchema = new Schema({
key: String,
nsamp: Number,
starttime: Number,
endtime: Number,
samprate: Number,
datatype: String,
data: [Number]
});
Querying the following record via mongo
db.cwaves.find({"_id" : ObjectId("5733bedeadb31b2b8a0fef2e")})
It returns
{ "_id" : ObjectId("5733bedeadb31b2b8a0fef2e"),
"key" : "BROK.HNZ.UW.--", "nsamp" : 172,
"starttime" : 1463008988.38, "endtime" : 1463008989.2350001,
"samprate" : 200, "datatype" : "i4",
"data" : [ 1385, 1384, 1385, 1384, 1381, 1386, 1390, 1381, 1385,
1387, 1382, 1387, 1384, 1381, 1385, 1386, 1384, 1382,
1383, 1384, 1386, 1384, 1381, 1385, 1386, 1384, 1386,
1386, 1385, 1385, 1387, 1382, 1383, 1386, 1381, 1382,
1384, 1382, 1383, 1386, 1386, 1382, 1383, 1386, 1384,
1385, 1385, 1384, 1387, 1387, 1383, 1383, 1388, 1385,
1384, 1387, 1382, 1383, 1385, 1384, 1385, 1384, 1383,
1385, 1384, 1383, 1387, 1382, 1386, 1387, 1381, 1386,
1386, 1390, 1386, 1383, 1388, 1381, 1384, 1389, 1385,
1384, 1386, 1387, 1384, 1382, 1384, 1382, 1378, 1387,
1390, 1383, 1385, 1383, 1381, 1384, 1385, 1387, 1380,
1379, 1387, 1383, 1384, 1384, 1384, 1385, 1382, 1389,
1389, 1381, 1385, 1388, 1387, 1386, 1383, 1386, 1383,
1382, 1385, 1382, 1384, 1386, 1383, 1382, 1385, 1386,
1386, 1387, 1385, 1382, 1380, 1383, 1387, 1378, 1382,
1388, 1383, 1385, 1387, 1385, 1386, 1388, 1386, 1384,
1382, 1382, 1385, 1385, 1384, 1378, 1384, 1387, 1383,
1383, 1382, 1384, 1384, 1388, 1386, 1380, 1386, 1388,
1386, 1383, 1384, 1384, 1383, 1387, 1385, 1384, 1386,
1382 ]
}
But if I do that same query in Mongoose the data array is empty:
var mongoose = require('mongoose')
, assert = require('assert')
, app = require('express')()
, http = require('http').Server(app)
, io = require('socket.io')(http);
//schema stuff and things
var Schema = mongoose.Schema;
var waveformSchema = new Schema({
key: String,
nsamp: Number,
starttime: Number,
endtime: Number,
samprate: Number,
datatype: String,
data: [Number]
});
var Cwave = mongoose.model('Cwave', waveformSchema);
//get configs
var Conf = require("./config.js"); //config file
var conf = new Conf();
//Connect to Mongo
var url = "mongodb://" + conf.mongo.user + ":" + conf.mongo.passwd + "@"
+ conf.mongo.host + ":" + conf.mongo.port + "/" + conf.mongo.dbname
+ "?authMechanism=" + conf.mongo.authMech + "&authSource=" + conf.mongo.authSource;
mongoose.connect(url);
var db = mongoose.connection;
db.on('error', console.error.bind(console, 'connection error:'));
db.once('open', function() {
Cwave.find({"_id" : "5733bedeadb31b2b8a0fef2e"}, function(err, cwaves){
if (err) return console.error(err);
console.log(cwaves);
});
});
I get the following console output:
listening on port: 8888
[ { data: [],
datatype: 'i4',
samprate: 200,
endtime: 1463008989.2350001,
starttime: 1463008988.38,
nsamp: 172,
key: 'BROK.HNZ.UW.--',
_id: 5733bedeadb31b2b8a0fef2e } ]
Am I defining the schema correctly?
Edit 1:Changing the script to:
Cwave.find({"_id":"5733bedeadb31b2b8a0fef2e"}).lean().exec(
function(err, cwaves){ if (err) return console.error(err);
console.log(cwaves); })
Yields:
[ { _id: 5733bedeadb31b2b8a0fef2e,
key: 'BROK.HNZ.UW.--',
nsamp: 172,
starttime: 1463008988.38,
endtime: 1463008989.2350001,
samprate: 200,
datatype: 'i4',
data: [ data: 1382 ] } ]
Edit 2: After trying the above it appears this may be the native mongodb driver. I changed my script to:
var MongoClient = require('mongodb').MongoClient
, assert = require('assert')
, app = require('express')()
, http = require('http').Server(app)
, io = require('socket.io')(http);
//get configs
var Conf = require("./config.js"); //config file
var conf = new Conf();
//Connect to Mongo
var url = "mongodb://" + conf.mongo.user + ":" + conf.mongo.passwd + "@"
+ conf.mongo.host + ":" + conf.mongo.port + "/" + conf.mongo.dbname
+ "?authMechanism=" + conf.mongo.authMech + "&authSource=" + conf.mongo.authSource;
MongoClient.connect(url, function(err, db) {
if(err) throw err;
var collection = db.collection('cwaves');
collection.find({starttime: 1463008988.38, key: "BROK.HNZ.UW.--"}).toArray(function(err, results) {
console.log(results);
db.close();
});
});
And Got the same result as edit 1.
[ { _id: 5733bedeadb31b2b8a0fef2e,
key: 'BROK.HNZ.UW.--',
nsamp: 172,
starttime: 1463008988.38,
endtime: 1463008989.2350001,
samprate: 200,
datatype: 'i4',
data: [ data: 1382 ] } ]
Edit 3: The MongoDb collection is populated via a C program that uses Mongo-C.I created three similar documents: 1) via mongo-c routine(original), 2) node-mongodb and 3) Mongo Console. They appear similar in the mongo console when I do a find(except id, and key name) so I don't feel the need add the output, unless someone really wants it. I changed my node script to the following:
MongoClient.connect(url, function(err, db) {
if(err) throw err;
var collection = db.collection('cwaves');
collection.find().toArray(function(err, results) {
// console.log(result);
for(var i=0; i< results.length; i++){
console.log("key: " + results[i].key + " type: " + Object.prototype.toString.call(results[i].data) + " length: " + results[i].data.length);
}
db.close();
});
});
Which yields:
key: BROK.HNZ.UW.-- type: [object Array] length: 0
key: mongo_console type: [object Array] length: 172
key: node-mongodb type: [object Array] length: 172
I created a ruby script, using the mongo gem, and was able to read all three arrays.
This turned out to be an error with how I was building the array with libbson. The array building part in my C code was:
bson_append_int32 (m_data, "data", -1, long_data[i]);
The second argument is the key, which I thought should be the array key "data". But it should be the index as a string.
http://api.mongodb.com/libbson/current/bson_append_int32.html
so the array build part becomes:
char index[4];
bson_append_array_begin (m_doc, "data", -1, m_data);
for ( i = 0; i < trh->nsamp; i++ ){
snprintf(index, 4, "%d", i);
if ( (strcmp (trh->datatype, "s2")==0) || (strcmp (trh->datatype, "i2")==0) ){
bson_append_int32 (m_data, index, -1, short_data[i]);
}else{
bson_append_int32 (m_data, index, -1, long_data[i]);
}
}
bson_append_array_end (m_doc, m_data);
I don't understand why the first version worked using the mongo console or the ruby library.