Maximum call stack size exceeded on insert 10000 documents

Question

This is the code am running which returns the Range Maximum call stack size exceeded error. // to insert 10000 values on to mongodb using node.js

            var MongoClient = require('mongodb').MongoClient;
            var mongoServer = require('mongodb').Server;
            var serverOptions = {
                'auto_reconnect': true,
                'poolSize': 100
            };
            var i=0;
            var async =require('async');
            var mongoClient = new MongoClient(new mongoServer('localhost', 27017, serverOptions));
            var db = mongoClient.db('test');
            var collection = db.collection('new_file_test');

               mongoClient.open(function (err, mongoClient) 
                {

                   if(err){console.log(err)};
                   function start(i,call) 
                {
                    if(i<10000) {
                        call(start);
                    }
                }
                function pass(callback) 
                {
                    Insert(save);
                    i++;
                    callback(i,pass);
                }

                start(i,pass); 
                 });    
                function Insert(callback) {
                     console.log("Inserting" );
                     var doc={
                'trip_paramid':i,
                'tripid':'116', 
                'lattitude':'12.8929183',
                'longitude':'77.63627',
                'speed':'2',
                'heading':'0',
                'altitude':'80469',
                'address':'qwertyasdfgxcvbn',
                'engine_status':'Normal',
                'oil_pressure': '83.12',
                'water_temp': '28',
                'fuel_content':'0',
                'brake':'Normal',
                'creation_time':'2013-08-31 23:22:17',
                'brakelight_status':'Normal',
                'battery_status':'12.68',
                'event_code':'8',
                'dbinsert_time':'2013-08-31 23:24:59',
                'gsm_status':'-51',
                'cell_id':'45',
                'vehicle_id':'123456',
                'distance':'0'}
                        callback(doc);          
                }   

                function save(doc)  
                {
                    collection.insert(doc, function(err) 
                    {
                    if (err) 
                    { 
                        console.log('Error occured'); 
                    }
                    else
                        console.log("Saved");
                    });
                }

If the condition is to insert 1000 rows it works fine and the error throws only when the condition goes beyond 10000.

Answer 1

The problem comes from the recursive loop you made:

function start(i, call) {
    if (i < 10000) {
        call(start);
    }
}

function pass(callback) {
    Insert(save);
    i++;
    callback(i, pass);
}

start(i, pass);

You should change it to something like this:

for (var i = 0; i < 10000; i++) {
   Insert(save);
}

Simplifying your code you have this:

var i = 0;
function pass() {
    if (i < 10000) {
        Insert(save);
        pass(i);
    }
    i++;
}

pass();

The problem comes from the part that you are calling this function recursively, and since javascript doesn't have tail recursion elimination, the callstack keeps growing. V8(nodejs javascript engine) has it's limits, the callstack once reached to the maximum defined size the error will be thrown.

You can also have look at the following questions for more information:

• Maximum call stack size exceeded error
• JavaScript recursion: Maximum call stack size exceeded

This is all about fixing Maximum call stack size exceeded error. But 10000 looks like a huge number. I just ran that and it took about 3 seconds on my machine, to finish the loop using monk. Using mongo shell it took about 1 second. If you are running a server, when the loop is running your application is unresponsive.

I suggest instead, insert in batches, and use node's setImmediate function to schedule the next batch to be run after pending I/O events(like handling new web requests):

function insert10000(i) {
    insert100();
    i++;
    if (i < 100) {
        setImmidiate(insert10000, i);
    }
}

function insert100() {
    for (var i = 0; i < 100; i++) {
        Insert(save);
    }
}

And since we came on the topic of batching insert calls, collection.insert method, supports an array of documents instead of just one to be inserted.

So when we currently have something like following:

collection.insert(doc1);
collection.insert(doc2);

It can be changed to this:

collection.insert([doc1, doc2]);

And that actually is faster. So you can change the code to this:

function insert10000(i) {
    insert100(i);
    i++;
    if (i < 100) {
        setImmediate(insert10000, i);
    }
}

function insert100(i) {
    var docs = [];
    for (var l = i + 1000; i < l; i++) {
        docs.push({
            'trip_paramid':i,
            'tripid':'116', 
            'lattitude':'12.8929183',
            'longitude':'77.63627',
            'speed':'2',
            'heading':'0',
            'altitude':'80469',
            'address':'qwertyasdfgxcvbn',
            'engine_status':'Normal',
            'oil_pressure': '83.12',
            'water_temp': '28',
            'fuel_content':'0',
            'brake':'Normal',
            'creation_time':'2013-08-31 23:22:17',
            'brakelight_status':'Normal',
            'battery_status':'12.68',
            'event_code':'8',
            'dbinsert_time':'2013-08-31 23:24:59',
            'gsm_status':'-51',
            'cell_id':'45',
            'vehicle_id':'123456',
            'distance':'0'
        });
    }
    collection.insert(docs, function(err) {
        if (err) {
            console.log('Error occurred', err); 
        }
    });
}

I measured this, it was faster twice faster than the original case.

Answer 2

Looping over 10000 times and performing insert is really a bad idea. But still you can do with async library which might help you fix the issue. I have came across this situation before and i used async.queue to overcome the issue.

Async.js module.