How to Insert records into mongo with Node where records have an $oid

Question

I'm new to mongo and node and i'm trying to create a script that essentially works similar to rails rake db:seed task where I have a set of .json files in a specific folder, and when I run $ node seed it will run my seed.js file which takes all of those .json files and inserts the contents into their appropriate mongo collections based on the filename of the .json file.

Essentially I run $ node seed and it executes /seed.js which looks at /seeds/account.json, /seeds/customer.json, etc.

This way I can commit this code and then next developer can just run $ node seed and his mongo instance will be populated with some data to work with.

The problem i'm having is that the data that I got is from a dump of my collections in mongo so some sample data for account looks like this:

{ _id: { '$oid': '534d832177da4e0d38000001' },
  id: 1,
  name: something,
  dateTime: '2014-04-15T14:06:09-05:00' }

Now when I run $ node seed, I get this error:

[Error: key $oid must not start with '$']

Is there any way to get around this error or am I going to have to remove all the $oid's from all my .json files?

seed.js:

var mongoose = require('mongoose');
mongoose.connect('mongodb://localhost/testdb');
var dir = './seeds';
var db = mongoose.connection;

// Show connection error if there is one
db.on('error', console.error.bind(console, 'Database Connection Error:'));

// If we successfully connected to mongo
db.once('open', function callback() {

    var fs = require('fs'); // Used to get all the files in a directory

    // Read all the files in the folder
    fs.readdir(dir, function(err, list) {

        // Log the error if something went wrong
        if(err) {
            console.log('Error: '+err);
        }

        // For every file in the list
        list.forEach(function(file) {

            // Set the filename without the extension to the variable collection_name
            var collection_name = file.split(".")[0];
            var parsedJSON = require(dir + '/' + file);

            for(var i = 0; i < parsedJSON.length; i++) {

                // Counts the number of records in the collection
                db.collection('cohort').count(function(err, count) {
                    if(err) {
                        console.log(err);
                    }
                });

                db.collection(collection_name).insert(parsedJSON[i], function(err, records) {

                    if(err) {
                        console.log(err);
                    }

                    console.log(records[0]);
                    console.log("Record added as "+records[0]);

                });
            }

        });
    });
});

Or is there any libraries out there that already exist for this kind of task?

Answer 1

http://docs.mongodb.org/manual/core/document/#field-names

The field names cannot start with the dollar sign ($) character.

Part of the mongodb spec as it uses $ to indicate operations like $inc or $unset.

Answer 2

So there is a thing called the extended JSON syntax which is more of a spec and while supported by some driver implementations is yet to be implemented in the node native driver itself.

While there is one implementation I know of to work with converting these types, in the jsontomongo module for node, all of the types are not strictly supported. Yet as can clearly be seen in the code:

'use strict';

module.exports = function (query) {
  var key, val;
  for (key in query) {
    if (!query.hasOwnProperty(key)) continue;
    val = query[key];
    switch (key) {
    case '$date':
      return new Date(val);
    case '$regex':
      return new RegExp(val, query.$options);
    case '$undefined':
      return undefined;
    }
    if (typeof val === 'object')
      query[key] = module.exports(val);
  }
  return query;
};

Implementing the full specification or just the things you need such as ObjectId would be a fairly trivial exercise. You might even want to contribute your work as a node module yourself.

So that is essentially the sort of code you need to parse the object structure that results from parsing the JSON string.

Answer 3

Use this instead of $oid:

{ "_id" : ObjectId("5063114bd386d8fadbd6b004")}

Answer 4

Use mongodb-extended-json npm extension: https://www.npmjs.com/package/mongodb-extended-json

Here is the usage example taken from documentation:

var EJSON = require('mongodb-extended-json');
var BSON = require('bson');

var doc = {
  _id: BSON.ObjectID(),
  last_seen_at: new Date(),
  display_name: undefined
};

console.log('Doc', doc);
// > Doc { _id: 53c2ab5e4291b17b666d742a, last_seen_at: Sun Jul 13 2014 11:53:02 GMT-0400 (EDT), display_name: undefined }

console.log('JSON', JSON.stringify(doc));
// > JSON {"_id":"53c2ab5e4291b17b666d742a","last_seen_at":"2014-07-13T15:53:02.008Z"}

console.log('EJSON', EJSON.stringify(doc));
// > EJSON {"_id":{"$oid":"53c2ab5e4291b17b666d742a"},"last_seen_at":{"$date":1405266782008},"display_name":{"$undefined":true}}

// And likewise, EJSON.parse works just as you would expect.
EJSON.parse('{"_id":{"$oid":"53c2ab5e4291b17b666d742a"},"last_seen_at":{"$date":1405266782008},"display_name":{"$undefined":true}}');
// { _id: 53c2ab5e4291b17b666d742a,
//   last_seen_at: Sun Jul 13 2014 11:53:02 GMT-0400 (EDT),