I am using latest version of express, mongodb, nodejs etc. I am not using body parsing middleware as I DONT want to store files on disk when I upload them. I just want to stream them directly into GridFs. I am using the gridfs-stream npm module. I know that ExpressJs uses formidable underneath. Remember - I dont want the files to hit the disk.
Here is my route handler where I just post one multipart form - the form contains one file only along with 1-3 optional fields.
Why oh Why is form.on('part',function(part)) - called multiple times for the same part for files say over 100kb and yet for smaller files - we just get one call??
What I am really trying to do is actually stream a file from an upload directly into GridFs and attach any form post fields in the same post to the GridFs metadata.
mymodule.prototype.save = function(req,res,next) {
if(req._body) {
//return next();
}else if(contentType(req) !== 'multipart/form-data') {
return next();
}
var gfs = Grid(req.ctx.app.database.connection.db, mongo);
req.body || (req.body = {});
req.files || (req.files = {});
var form = req.form,
body = req.body,
files = req.files,
cb_called = false;
var parts = {};
var info = '';
var fileId = mongoose.Types.ObjectId();
var gfs_ws;
form.on('part', function(part) {
if (part.filename) { ///// THIS IS CALLED MORE THAN ONCE for each part.filename???? on large files -- but why???
sendPart(part);
}
});
form.on('field', function(name, value) {
if ( Array.isArray(body[name]) ) {
body[name].push(value);
}else if (body[name]) {
body[name] = [body[name], value];
}else{
body[name] = value;
}
});
form.on('error', function(err) {
if (err && err.length > 1){
res.statusCode = 200;
res.send({success: false, err: err});
}
});
form.on('close', function() {
try {
//req.body = qs.parse(body); // if additional fields sent - for now we ignore them
}
catch (err) {
return next(err);
}
res.send({ success: true }); //, id: fileId.toString() });
});
form.parse(req);
function contentType(req) {
return req.headers['content-type'] && req.headers['content-type'].split(';')[0];
}
function sendPart(part) {
if (!gfs_ws) { // THIS IS MY ATTEMPT TO STREAM ALL PARTS OF THE SAME FILE TO THE SAME GRIDFS RECORD - but really we should only ever have called sendPart(part) once for each file
var options = {
_id: fileId, // a MongoDb ObjectId
filename: part.filename, // a filename
mode: 'w', // default value: w+, possible options: w, w+ or r, see [GridStore](http://mongodb.github.com/node-mongodb-native/api-generated/gridstore.html)
//any other options from the GridStore may be passed too, e.g.:
//chunkSize: 1024,
content_type: part.headers['content-type'], //file.type , // For content_type to work properly, set "mode"-option to "w" too!
//root: 'my_collection',
metadata: {
recordId: req.params.recordid,
elementId: req.params.elementid
}
};
gfs_ws = gfs.createWriteStream(options);
}
part.pipe(gfs_ws);
}
}
When I post small files (i.e. <100kb) form.on('part' ... gets called just once for each file part - and once for each field part. I only have only file in the post - so sendPart(part) only gets called one and the file gets put in GridFs and happy days.
However - with larger files i.e > 100kb - form.on('part' - gets called multiple times for the same file - i.e. part.filename is the