reassemble binary after flow.js upload on node/express server

Question

I can't figure out how to use the flow.js library with a node backend, and basing my code off the sample on the flow.js github.

I'm getting the blob files up, but I'm not building the binary afterward the upload completes. The final get isn't getting triggered or my route is wrong:

  app.get('/download/:identifier', function(req, res){
    console.log('we writin')
    flow.write(req.params.identifier, res);
  });

anyone have any experience with this could get like a million stackoverflow pts because this seems to be a common issue when using node.js and flow.js and here are two other unanswered questions:

Flowjs file upload - AngularJS and Node Reassembling file chunks produced in a multi-part upload

Answer 1

I've found a method that works but might not be the ideal approach.

Here I'm calling flow.write in flow.post if status is done and currentTestChunk > numberOfChunks. I do the greater than check because sometimes flow.post sends status done more than once as mentioned here.

Edit: I added a way to clean the chunks after creating the file.

flow.post(req, function(status, filename, original_filename, identifier, currentTestChunk, numberOfChunks) {
        console.log('POST', status, original_filename, identifier);
        res.send(200);
        if (status === 'done' && currentTestChunk > numberOfChunks) {
            var stream = fs.createWriteStream('./tmp/' + filename);
            //EDIT: I removed options {end: true} because it isn't needed
            //and added {onDone: flow.clean} to remove the chunks after writing
            //the file.
            flow.write(identifier, stream, { onDone: flow.clean });            
        }            
    })

I had to modify flow.post's callback to send currentTestChunk and numberOfChunks.

File: flow-node.js

$.post = function(req, callback){

//There's some codez here that we can overlook...

  fs.rename(files[$.fileParameterName].path, chunkFilename, function(){

    // Do we have all the chunks?
    var currentTestChunk = 1;
    var numberOfChunks = Math.max(Math.floor(totalSize/(chunkSize*1.0)), 1);
    var testChunkExists = function(){
          fs.exists(getChunkFilename(currentTestChunk, identifier), function(exists){
            if(exists){
              currentTestChunk++;
              if(currentTestChunk>numberOfChunks) {

                //Add currentTestChunk and numberOfChunks to the callback

                callback('done', filename, original_filename, identifier, currentTestChunk, numberOfChunks);
              } else {
                // Recursion
                testChunkExists();
              }
            } else {

              //Add currentTestChunk and numberOfChunks to the callback

              callback('partly_done', filename, original_filename, identifier, currentTestChunk, numberOfChunks);
            }
          });
        }
    testChunkExists();
  });
} else {
      callback(validation, filename, original_filename, identifier);
}

}

In flow.write call flow.clean with onDone if you want to remove the chunks.

  $.write = function(identifier, writableStream, options) {
      options = options || {};
      options.end = (typeof options['end'] == 'undefined' ? true : options['end']);

      // Iterate over each chunk
      var pipeChunk = function(number) {

          var chunkFilename = getChunkFilename(number, identifier);
          fs.exists(chunkFilename, function(exists) {

              if (exists) {
                  // If the chunk with the current number exists,
                  // then create a ReadStream from the file
                  // and pipe it to the specified writableStream.
                  var sourceStream = fs.createReadStream(chunkFilename);
                  sourceStream.pipe(writableStream, {
                      end: false
                  });
                  sourceStream.on('end', function() {
                      // When the chunk is fully streamed,
                      // jump to the next one
                      pipeChunk(number + 1);
                  });
              } else {
                  // When all the chunks have been piped, end the stream
                  if (options.end) {
                          writableStream.end();
                      }

                  //Options.onDone contains flow.clean so here I'm deleting all the chunked files.

                  if (options.onDone) {
                      options.onDone(identifier);
                  }
              }
          });
      }
      pipeChunk(1);
  }

Answer 2

Ok, so i've been working on this and have come up with this, hopefully it will get someone started...

exports.post = function (req, res, next) {

    flow.post(req, function(status, filename, original_filename, identifier) {

        console.log('status: '+ status, filename, original_filename, identifier);

        if(status==='done'){

            var s = fs.createWriteStream('./uploads/' + filename);
            s.on('finish', function() {

                res.send(200, {
                    // NOTE: Uncomment this funciton to enable cross-domain request.
                    //'Access-Control-Allow-Origin': '*'
                });

            });

            flow.write(identifier, s, {end: true});
        }

    });

};

Answer 3

I've issued a pull request which adds reassembling logic to the Node example of Flow.js. See https://github.com/flowjs/flow.js/pull/71 and https://github.com/flowjs/flow.js/issues/17.