closing mongodb connection in node.js while inserting lot of data

Question

I am trying write a program to parse and insert iis logs data in to mongodb. The files aren't that huge it's around 600 lines. Trying to convince my management nodejs and mongodb is better for this compared to .net and sql server :).
Have a look at the below code in nodejs. Logic: I parse every line and convert into json and insert the save in db. i am using mongonative driver.
Issue : The db connection gets closed even before all lines are inserted into the Db.
I see the log file has 6000 lines, but num of records in db is only arnd 4000. I understand it's nodejs's async characteristic, in this how can i close the connection in more deterministic way (after checking if all lines got inserted)?

var MongoClient = require('mongodb').MongoClient;
var mongoServer = require('mongodb').Server;
var serverOptions = {
    'auto_reconnect': true,
    'poolSize': 5
};

var fs = require('fs');
var readline = require('readline');

var rd = readline.createInterface({
    input: fs.createReadStream('C:/logs/Advisor_Metrics/UI/P20VMADVSRUI01/u_ex130904.log'),
    output: process.stdout,
    terminal: false
});
var mongoClient = new MongoClient(new mongoServer('localhost', 27017, serverOptions));
var db = mongoClient.db('test');
var collection = db.collection('new_file_test');
var cntr = 0;
mongoClient.open(function (err, mongoClient) {
    console.log(err);
    if (mongoClient)
    {        
        rd.on('line', function (line) {
            if (line.indexOf('#') == -1) {
                var lineSplit = line.split(' ');
                var data =
                {
                    d: lineSplit[0],
                    t: lineSplit[1],
                    sip: lineSplit[2],
                    met: lineSplit[3],
                    uri: lineSplit[4],
                    cip: lineSplit[8],
                    cua: lineSplit[9],
                    stat: lineSplit[10],
                    tt: lineSplit[13]
                };

                collection.insert(data, function (err, docs) {
                    console.log('closing connection');
                    //db.close();
                });
            }
        });
    }
})
rd.on('close', function () {
    db.close();
});

Sol 1 : A solution would be parse the json objects and add into an array and add the array to mongodb. i wouldn't like to do that since that would like parsing the entire huge log file into memory!, Any other solution?

Answer 1

I'm 100% sure but as far as I can see you are inserting data synchronous. I mean once you get a line you try to insert it and don't wait for the result. Try using another approach:

• read all the lines and store them in an array
• start inserting the data line by line asynchronously

Something like that:

var lines = [];
var readAllLines = function(callback) {
    // store every line inside lines array
    // and call the callback at the end
    callback();
}
var storeInDb = function(callback) {
    if(lines.length === 0) {
        callback();
        return;
    }
    var line = lines.shift();
    collection.insert(line, function (err, docs) {
        storeInDb(callback);
    });
}

mongoClient.open(function (err, mongoClient) {
    console.log(err);
    if (mongoClient) {
        readAllLines(function() {
            storeInDb(function() {
                // lines are inserted
                // close the db connection
            })
        });
    }
});