Empty body in a response when using request

Question

I have the following code:

var request = require('request');
var cheerio = require('cheerio');
var URL = require('url')
var fs = require('fs')
fs.readFile("urls.txt", 'utf8', function(err, data) {
    if (err) throw err;
    var urls = data.split('\n');
    urls = urls.filter(function(n){return n});
    for(var i in urls) {
        request(urls[i], function(err, resp, body) {
            if (err)
                throw err;
            $ = cheerio.load(body,{lowerCaseTags: true, xmlMode: true});
            $('item').each(function(){
                console.log("----------");
                console.log($(this).find('title').text());
                console.log($(this).find('link').text());
                console.log($(this).find('pubDate').text());
            });
        }).end();
    }
});

and from the urls.txt file I only have the following url:

http://www.visir.is/section/?Template=rss&mime=xml

When I use wget on that url I get a response which looks like an rss feed but when I do it in the code above the body is empty. Can someone explain to me why and how can I fix this?

Answer 1

Update: Simply removing .end() from your original script works. end() terminates the script on callback. IMO, in 2016, I'd definitely choose Request over Needle.

---

Request is an odd bird, and why it's not working in your case it's giving no information in the response at all.

Try with Needle instead:

var needle = require('needle');
var cheerio = require('cheerio');
var URL = require('url')
var fs = require('fs')
fs.readFile("urls.txt", 'utf8', function(err, data) {
    if (err) throw err;
    var urls = data.split('\n');
    urls = urls.filter(function(n){return n});
    for(var i in urls) {
        needle.get(urls[i], function(err, resp, body) {
            if (err)
                throw err;
            $ = cheerio.load(body,{lowerCaseTags: true, xmlMode: true});
            $('item').each(function(){
                console.log("----------");
                console.log($(this).find('title').text());
                console.log($(this).find('link').text());
                console.log($(this).find('pubDate').text());
            });
        });
    }
});