Append more properties to results retrieved from NodeJS (express) + MongoDB - simulate SQL join

Question

Can someone enlighten me with a solution for the following situation?

I have a logging application written in nodeJS express framework and using MongoDB collections. I managed to get the results from a collection, but I want to iterate on the results, query another collection based on the reference ID from the first query and append more values to the initial results, then send the response to the client.

I will paste bellow what I have now coded, but I cannot get it to work. I'm either doing it wrong or my desired logic is not the right one.

Please help me with more than theory and the fact that I cannot use the results outside an asynchronous JS function.

Thanks in advance!

CODE:

exports.findLogs = function(req, res) {
    var params = req.body;
    var query = req.query;
    var coll = params.section;
    if (coll === undefined) {
        coll = 'traces';
    }

    db.collection(coll, function(err, collection) {
        collection.count(function(err, count) {
            var criteria = {
            'limit': limit,
                'skip': skip,
                 'sort': [[options.sortname, options.sortorder]]
             }
             collection.find({}, criteria).toArray(function(err, items) {
                 Object.keys(items).forEach(function(logIndex) {
                     var trace = items[logIndex];
                     var crit = {
                         'find': {LogId: trace.LogId},
                         'projection': {},
                         'limit': 1
                     }

                     // Get entry based on Criteria from `log` table
                     findOneBy(crit, 'log', function(err, log) {
                         if (log.length !== 1) {
                             // no reference found
                         } else {
                              // Appending here to log DOES NOT stick as expected
                             trace.ComputerName = log.ComputerName;
                             trace.ComputerIP = log.ComputerIP;
                             trace.LastSeen = log.LastSeen;
                             trace.TestVar = 'test1234';
                         }
                     });

                     // Appending here to trace works as expected
                         trace.Actions = 'MyAction';
                 });

                 results['total'] = count;
                 results['results'] = items.length;
                 results['rows'] = items;
                 res.send(results);
            });
        });
    });
}

function findOneBy(criteria, coll, callback) {
    var cFind = criteria.find;
    var cProj = criteria.projection;
    db.collection(coll, function(err, collection) {
        if (err) return callback(err);
        else return collection.find(cFind, cProj).toArray(callback);
    });
}

Answer 1

Your function findOneBy is asynchronous. As your code loops through the array of results stored in items, each one is triggering an asynchronous look up.

However, before all of those return, you're sending the results to the client via `res.send(results). So, while the data is returned to the Node.JS application, it's after the results have already been sent.

There are a few ways to handle this, but I'd suggest you consider logic more like this (pseudo-code, as I don't have a DB mirroring your data):

collection.find({}, criteria).toArray(function(err, items) {
    var allLogIds = [];
    var logIdsByUse = {};
    Object.keys(items).forEach(function(logIndex) {
        // here, we'll first bit a list of all LogIds
        var trace = items[logIndex];
        allLogIds.push(trace.LogId);
        // now build a index of logId and a list of dependent objects
        logIdsByUse[trace.LogId] = [] || logIdsByUse[trace.LogId];
        logIdsByUse[trace.LogId].push(trace);
    });
    // now that the LogIds are all gathered, we'll do a search for all of them
    // at once.
    // *** You should make certain that the LogId field is indexed or this will
    // *** never perform adequately in any situation
    db.collection("log", function(err, collection) {
        if (err) { /* do something with the error, maybe 500 error */ }
        else { 
            collection.find({ LogId: { $in : allLogIds}})
                .toArray(function(err, logs) {
                    logs.forEach(function(log) {
                       // grab the LogId,
                       // get the array of traces that use the LogId
                       // from the logIdsByUse[LogId]
                       // loop through that array, and do the fix up that you
                       // want
                    });
                    // now, you may send the results                           
                    results['total'] = count;
                    results['results'] = items.length;
                    results['rows'] = items;
                    res.send(results);               
                });
    });        

});

You could also look at using one of the async libraries for Node.JS, but it won't significantly change what you need to do.

Answer 2

     callMe(function(initialResults) {
             //iterate over the array or object
             //take your new results and combine them with the initial Results,
             //To be done differently depending if you have an array, object, or array of objects.
         res.send(newResults)
     }; 




     function callMe(callback) {
            //query your database and get your first results
            callback(initialResults)
     }

Does that help? If not explain what kind of errors or results you are getting