Downloading files with node.js, streams, and promises

Question

Here is a snippet of my code:

var processListing = function (directoryItems) {
    console.log('foreach');
    var itemsToDownload = [];
    directoryItems.forEach(function (element, index, array) {
        //Ignore directories
        if (element.type === 'd') {
            console.log('directory ' + element.name);
            return;
        }
        //Ignore non zips
        if (path.extname(element.name) !== '.zip') {
            console.log('ignoring ' + element.name);
            return;
        }
        //Download zip
        itemsToDownload.push({
            source: element.name,
            destination: element.name
        });
        //aftpSystem.downloadFile(element.name, element.name);
    });
    console.log('after foreach');
    return itemsToDownload;
};

var getFiles = function () {
    console.log('got files');
    return fs.readdirAsync(process.cwd() + "/zips/").then(function (files) {
        return files.filter(function (filename) {
            return path.extname(filename) === '.zip';
        });
    });
};

    aFtpClient. //this has been promisified
    listAsync(). //so calls methodAsync
    then(processListing).
    map(function (object) {
        return processItem(object).then(function (processResult) {
            return {
                input: object,
                result: processResult
            };
        });
    }).
    map(function (downloadItem) {
        console.log('downloading files');

        downloadItem.result.once('close', function () {
            console.log('closed');
        });

        return downloadItem.result.pipe(fs.createWriteStream(process.cwd() + "/zips/" + downloadItem.input.destination));
    }).
    then(getFiles).

I'm trying to use promises to download items via FTP. At the moment it downloads the first file but then fails on subsequent files. I'm new to node but fairly confident that my second map function needs to return a promise however I have been unable to work out how after numerous attempts. I am using bluebird for promises but can't see how to work with it and streams.

Could you point me in the correct direction?

Thanks

Answer 1

I'm not sure where exactly you're stuck but pointing you in the general direction should suffice:

• You have an interface that works with a pipe and events
• You need to promisify that interface.

So what you need to do is:

1. Find out what's the 'completion' event of the download.
2. Create a promise and resolve it on that event, reject it on the failed event.
3. Return that promise.

Promisifying can be done in several ways:

• By the promise library. Bluebird contains a really clever promisifier using dynamic code generation that relies on the JIT - it is very fast - but it's built for the NodeJS "nodeback" case. (i.e. error passed as first argument of the callback.)

• Using the Deferred object. Generally that way is more error prone.

• Using Promise.method in Bluebird, which is great for promisifying APIs easily but is not really our case here.

• Using the Promise constructor. This is what we'll do here. It's also standards complaint.

Generally, the interface of the promise constructor is:

new Promise(function(resolve,reject){

    resolve(); // this resolves the promise
    reject(); // this rejets the promise
});

Note, promisifying event emitters only works well when they fire on a finish event and do so once. Promises are one time, once they settle they can't change state. Events can fire multiple times. It's perfectly fine to promisify things like "load" events or "finished" events - but don't promisify things that repeat multiple times.

Your second map should be something like:

map(function (downloadItem) {
    console.log('downloading files');

    downloadItem.result.once('close', function () {
        console.log('closed');
    });
    var pipeAction = downloadItem.result.pipe(fs.createWriteStream(process.cwd() + "/zips/" + downloadItem.input.destination));
    return new Promise(function(resolve,reject){
        pipeAction.on("end",function(){ //waits for data to be consumed
             // pipe has ended here, so we resolve the promise
             resolve();
        });
    });
}).

You should generally extract promisifications into dedicated methods. For example, the above could be a promisifyPipe or similar.

Answer 2

Here is a promisified pipe:

//example: promisified_pipe(foo, fs.createWriteStream(somepath))enter code here
function promisified_pipe(response, file) {
let ended = false;

return new Promise(function(resolve, reject) {
    response.pipe(file);

    function nice_ending() {
      if (!ended) {
        ended = true;
        resolve();
      }
    }

    function error_ending() {
      if (!ended) {
        ended = true;
        reject("file error");
      }
    }

    file.on('finish', nice_ending);
    file.on('end', nice_ending);
    file.on('error', error_ending);
    file.on('close', error_ending);
  }).finally(() => file.close())
}