Node.js using async - How can I properly use async.forEachLimit with fs.readFile?

Question

I am developing a part of a Node.js application that needs to get version information from a specific points in various files. I have coded the functionality for this using the npm package, async.

I know exactly what my problem is. However, because I am so new to Node.js, and even newer to the async package, I am not implementing something right.

It seems that the contents of the version variable is not making it to my response in time. In other words, the response is being sent before the version can make it to the response.

Below is the relevant code:

exports.getAllVersionInformation = function(request, response) {
    if (!utilities.checkLogin(request, response))
        return;

    // Get the array of file information stored in the config.js file
    var fileCollection = config.versionsArray;

    // Declare the array to be used for the response
    var responseObjects = [];

    async.forEachLimit(fileCollection, 1, function(fileInformation, taskDone) {
        // Declare an object to be used within the response array
        var responseObject = new Object();

        // Retrieve all information particular to the given file
        var name = fileInformation[0];
        var fullPath = fileInformation[1];
        var lineNumber = fileInformation[2];
        var startIndex = fileInformation[3];
        var endIndex = fileInformation[4];

        // Get the version number in the file
        var version = getVersionInFile(fullPath, lineNumber, startIndex,
                endIndex, taskDone);

        console.log('Ran getVersionInFile()');

        // Set the name and version into an object
        responseObject.name = name;
        responseObject.version = version;

        // Put the object into the response array
        responseObjects.push(responseObject);

        console.log('Pushed an object onto the array');

    }, function(error) {
        console.log('Entered the final');

        if (error == null)
            // Respond with the JSON representation of the response array
            response.json(responseObjects);
        else
            console.log('There was an error: ' + error);
    });
};

function getVersionInFile(fullPath, lineNumber, startIndex, endIndex, taskDone) {
    console.log('Entered getVersionInFile()');
    var version = fs.readFile(fullPath,
            function(error, file) {
                if (error == null) {
                    console.log('Reading file...');

                    var lineArray = file.toString().split('\n');

                    version = lineArray[lineNumber].substring(startIndex,
                            endIndex + 1);
                    console.log('The file was read and the version was set');
                    taskDone();
                } else {
                    console.log('There was a problem with the file: ' + error);
                    version = null;
                    taskDone();
                }
            });
    console.log('Called taskDone(), returning...');
    return version;
};

I have tried playing with how the getVersionInFile function returns the data. I have moved the taskDone() function around to see if that would make a difference. I've asked Google so many questions about async and the way to use it in my context. I can't seem to get it working.

Some of the more important resources I have used are these: http://www.sebastianseilund.com/nodejs-async-in-practice http://book.mixu.net/node/ch7.html

I added console.log statements to get a trace of the code flow. Here's the picture for that:

Additionally, I have part of the response I was expecting. Here's that as well: ![Browser Output]http://imgur.com/rKFq83y

The issue with this output is that each object within the JSON should also have a version value. So, the JSON should look something like: [{"name":"WebSphere","version":"x.x.x.x"},{"name":"Cognos","version":"x.x.x.x"}]

How can I get my getVersionInFile() function to properly give me the version number in time? Also, how can I make sure I do this asynchronously without doing any blocking (hense the reason of using async for flow control)?

Any insight or suggestions would be greatly appreciated.

Answer 1

One issue is that getVersionInFile() is returning a value before the asynchronous readFile() has finished (also being async, readFile() does not return a meaningful value). Also, using a limit/concurrency of 1 for forEachLimit() is the same as forEachSeries(). Here is an example that uses mapSeries() that should get you the same end result:

exports.getAllVersionInformation = function(request, response) {
  if (!utilities.checkLogin(request, response))
    return;

  // Get the array of file information stored in the config.js file
  var fileCollection = config.versionsArray;

  async.mapSeries(fileCollection, function(fileInformation, callback) {
    // Retrieve all information particular to the given file
    var name = fileInformation[0];
    var fullPath = fileInformation[1];
    var lineNumber = fileInformation[2];
    var startIndex = fileInformation[3];
    var endIndex = fileInformation[4];

    // Get the version number in the file
    getVersionInFile(fullPath,
                     lineNumber,
                     startIndex,
                     endIndex,
                     function(error, version) {
      if (error)
        return callback(error);

      callback(null, { name: name, version: version });
    });
  }, function(error, responseObjects) {
    if (error)
      return console.log('There was an error: ' + error);

    // Respond with the JSON representation of the response array
    response.json(responseObjects);
  });
};

function getVersionInFile(fullPath, lineNumber, startIndex, endIndex, callback) {
  fs.readFile(fullPath,
              { encoding: 'utf8' },
              function(error, file) {
                if (error)
                  return callback(error);

                var lineArray = file.split('\n');

                version = lineArray[lineNumber].substring(startIndex,
                        endIndex + 1);
                callback(null, version);
              });
};