How to asynchronously do multiple REST API requests in node.js?

Question

What I want to do is do multiple remote REST API requests, where I want each request to be done sequentially, one after another. I read that async.js is the way to do this.

As I do not know how many times I have to do the request, I am using async.whilst(). The idea is that I will stop requesting once the API returns zero posts. Here is my code so far (for testing purposes I limited the loop to run for 5 times only).

var request = require('request');
var async   = require('async');

var APIKEY  = 'api-key-here';

var i = 0;
var continueWhilst = true;
async.whilst(
  function test() { return continueWhilst; },
  doThisEveryTime,
  function (err) {
    // Done
  }
);

function doThisEveryTime(next) {

  reqURL = 'http://api.tumblr.com/v2/blog/testsite.com/posts/text?api_key=' + APIKEY + '&offset=' + i*20;  

  request(
    reqURL,
    function (err, resp, body) {
      if(!err && resp.statusCode == 200) {
        var resultAsJSON = JSON.parse(body);      
        console.log(reqURL);        
        console.log("Request #" + i + " done");
    }
  });

  i++;
  if (i===5) {
    continueWhilst = false;
  }
  console.log("Iterating, i= " + i);
  next();
}

The test output is like this:

Iterating, i= 1
Iterating, i= 2
Iterating, i= 3
Iterating, i= 4
Iterating, i= 5
http://api.tumblr.com/v2/blog/testsite.com/posts/text?api_key=api-key-here&offset=80
Request #5 done
http://api.tumblr.com/v2/blog/testsite.com/posts/text?api_key=api-key-here&offset=80
Request #5 done
http://api.tumblr.com/v2/blog/testsite.com/posts/text?api_key=api-key-here&offset=80
Request #5 done
http://api.tumblr.com/v2/blog/testsite.com/posts/text?api_key=api-key-here&offset=80
Request #5 done
http://api.tumblr.com/v2/blog/testsite.com/posts/text?api_key=api-key-here&offset=80
Request #5 done

the request method is somehow only taking into account the final i value of 80. What did I do wrong, and how do I fix this?

Answer 1

The problem is occurring because your reqURL variable has no var, and does not belong to the doThisEveryTime scope. You are also making all your requests at once, since you are not waiting for a request to complete before calling next. These two things are leading the request to happen five times at the same time with the same URL. Try the following:

function doThisEveryTime(next) {
    var reqURL = 'http://api.tumblr.com/v2/blog/testsite.com/posts/text?api_key=' + APIKEY + '&offset=' + i*20;  

    request(reqURL, function (err, resp, body) {
        if (!err && resp.statusCode === 200) {
            var resultAsJSON = JSON.parse(body);      
            console.log(reqURL);        
            console.log("Request #" + i + " done");
        }

        i += 1;

        if (i === 5) {
            continueWhilst = false;
        }

        console.log("Iterating, i= " + i);

        // This is in the callback for request now.
        next();
    });
}