Upload-Progress in compoundjs/express with bodyParser({defer:true}) fails

Question

I try to figure out how to use the defer-property of the bodyParser used in express (and compoundjs) correctly. The goal is to get access to the event-property which should be possible duo passing the defer-property to the bodyParser.

What happens is, that the bodyParser doesnt work at all for enctype=multipart/form-data. In my opinion the bodyParser still should parse the request and place all relevant data in the body-Object. But the body-Object is empty for every Request I use a form with enctype=multipart/form-data. That causes several errors like authetification failure or forgery-Check.

So - whats going on here? Did I unserstand something wrong? The bodyParser just should do its job and I want to have access to the progress-event.

PS: I read about errors caused by the order I use bodyParser, sessionParser and so on. Therefore here is my configuration (compoundjs):

module.exports = function (compound) {

var express = require('express');
var app = compound.app;

app.configure(function(){
    app.use(compound.assetsCompiler.init());
    app.use(express.static(app.root + '/public', { maxAge: 86400000 }));
    app.set('jsDirectory', '/javascripts/');
    app.set('cssDirectory', '/stylesheets/');
    app.set('cssEngine', 'less');
    app.set('view engine', 'ejs');
    // make sure you run `npm install browserify uglify-js`
    // app.enable('clientside');
    app.use(express.methodOverride());
    app.use(express.cookieParser('secret'));
    app.use(express.session({secret: 'secret'}));
    app.use(express.bodyParser({
            //keepExtensions: true,
            limit: 10000000, // 10M limit
            defer: true
    }));
    app.use(app.router);
});

};

Answer 1

Here is the answer to my own question. I hope it will help poeple with the same problem.

First of all: You cant observe the progress-event of the fileupload in your normal app-code (e.g. controller, model). I tried to use the before-filter which seemed to work till I realized that it destroys the method-overiding. Thats why I had to wrote my own very simple middleware which just listens to the progress and end-event of req.form which logs the progress and calls next() when the end-event happens.

module.exports = function (compound) {

    var express = require('express');
    var app = compound.app;

    app.configure(function(){
        app.use(compound.assetsCompiler.init());
        app.use(express.static(app.root + '/public', { maxAge: 86400000 }));
        app.set('jsDirectory', '/javascripts/');
        app.set('cssDirectory', '/stylesheets/');
        app.set('cssEngine', 'less');
        app.set('view engine', 'ejs');

        app.use(express.bodyParser({
                //keepExtensions: true,
                limit: 10000000, // 10M limit
                defer: true
        }));

        // Thats the middleware
        app.use(function(req, res, next){
            // Only use it if form isnt parsed yet through bodyParser
            if(!req.form){next();return;}

            req.form.on('progress', function(bytesReceived, bytesExpected) {
                console.log('progress: '+Math.round(bytesReceived/bytesExpected*100)+'%');

            req.form.on('end',function(){
                console.log('fileupload finished');
                next();
            });
        });

        app.use(express.cookieParser('secret'));
        app.use(express.session({secret: 'secret'}));
        app.use(express.methodOverride());
        app.use(app.router);
    });

};

Its worth to mention that the order of the middleware-calls is very important. First you have to call the bodyParser (with defer:true). If that is done the Parser can parse all incoming requests for you and only delegate the forms of enctype="multipart/form-data" to you. Then your middleware can observe the upload. After that Session and Cookies are loaded. I tried to load the session and cookies before my middleware to know whether the user which is currently uploadind has the right to do so but that caused very wired behavior. I could read the session and all seems great but all my data in the form-object became doubled. {name:'dude'} becamed {name:{0:'dude',1:'dude'}} which destroyed the method-Override, too. This order is my only known working order.

If you have a solution to the mentioned problem with the doubled data any help would be appreciated :)

//EDIT: I got a solution for the problem above. The Probem was - of couse as always - the order of the middleware. Here again the "final" Code which works with uploadprogress and authentification through session:

module.exports = function (compound) {

    var express = require('express');
    var app = compound.app;

    app.configure(function(){
        app.use(compound.assetsCompiler.init());
        app.use(express.static(app.root + '/public', { maxAge: 86400000 }));
        app.set('jsDirectory', '/javascripts/');
        app.set('cssDirectory', '/stylesheets/');
        app.set('cssEngine', 'less');
        app.set('view engine', 'ejs');
        // make sure you run `npm install browserify uglify-js`
        // app.enable('clientside');

        // At the very first load Cookie and Session
        app.use(express.cookieParser('secret'));
        app.use(express.session({secret: 'secret'}));

        // Load the bodyParer then with defer:true
        app.use(express.bodyParser({
                //keepExtensions: true,
                limit: 10000000, // 10M limit
                defer: true
        }));

        // Now comes the own middleware with access to the session _and_ the progress
        app.use(function(req, res, next){

            console.log('searching for files to upload...');
            if(!req.form || req.form.type != 'multipart'){next();return;}

            console.log('check if user is autheticated...');
            if(!req.session.userId)
            {
                console.log('user is not authenticated. Throwing Error to prevent further upload!');
                try { throw new Error("Stopping file upload..."); } 
                catch (e) { res.end(e.toString()); }
                next();return;
            }

            console.log('file found - attaching events...');
            req.form.on('progress', function(bytesReceived, bytesExpected) {
                console.log('progress: '+Math.round(bytesReceived/bytesExpected*100)+'%');
            });

            req.form.on('end',function(){
                console.log('fileupload finished');
                next();
            });
        });

        app.use(express.methodOverride());
        app.use(app.router);
    });

};