Upload and then download a big file, with windows azure, blob storage and nodejs

Question

I'm trying to do a website on windows azure, which allows the user to submit a file through a webpage with a form, which sends the form with the POST method. I'm using the last version of Mac OS, on a mac book air 2013.

The server receives this form, streams the file to a blob, and redirect the user to a new webpage with a link to the blob (which is public). This way, the user can get his file later.

What I did seems to work fine with little files (a image ~35KB, a dmg file ~ 950MB).

But when it comes to bigger files (~20MB), the server notices me the is an error during the upload (after about 1m20s or 2m40s, don't remember). The client (google chrome) seems unaware of it, and goes on uploading the file. But when it ends, it displays a 503 error. I tried with 2 dmg file about 20MB and got the same results (Problem 1).

Then with a ~57MB dmg file (google chrome installer) and several ~700MB files (using a ~30MB/s upload internet access), the servers doesn't seem to have any problem, nor the client. The upload processes until the end, and I get the link to the file stored in the blob storage. But when I download it, the file is a little bigger than expected (4096 more bytes for a 57MB file, and about 40KB for a ~700MB file). I tried with several types of files : it is good enough to be able to replay a video (despite some weird stuff), but a dmg fails to mount when I try it (I guess because integrity checks does its job well) (Problem 2)

I wanted to know what data had been added to my files. So I created 2 files the same size as my ~57MB dmg file. The first one is full of characters 0, and the seconds alternates characters 0 and 1. When I upload them and download them, they do not have the original size, but they still contain only 000000.... or 010101010.... Though, there really is more 0000... and more 0101010.... than the original version (but only 0's and 01's).

So I went to the blob storage manager page, and I saw the size of every file seems smaller than on my computer (for example : 54.33MB instead of ~57MB). Might be because of a different file system, ... ?

Let's sum up problem 2 : I upload a big file, it seems smaller online than on my laptop, and then I download it, and it is in fact bigger than the original version. Weird, isn't it ?

Any ideas ? Thanks for your help !

Here is my server.js file :

var http = require('http');
var url = require("url");
var express = require('express');
var querystring = require('querystring');
var fs = require('fs');
var path = require('path');
var azure = require('azure');
var multiparty = require('multiparty');
var tools = require('./tools.js');

var app = express();
app.use(express.logger("dev"));

var port = process.env.PORT || 1337;
var containerName = 'publicblobs';

app.use('/public', express.static('./public'));

/*
 * \brief Displays upload form
 */
app.get('(/|/upload/?)', function(req, res) {
    res.redirect('/public/upload.html');
});

/*
 * \brief Handles upload post request.
 */
app.post('/upload/request/?', function (req, res) {
    var blobService = azure.createBlobService();
    var form = new multiparty.Form();
    var currentFileName = 'undefinedCurrentFileName';

    blobService.createContainerIfNotExists(containerName
    , {publicAccessLevel : 'blob'}
    , function(error){
        if(!error){
            // Container exists and is public
        } else {
            res.render('log.ejs', {log: error.message});
        }
    });

    function gotPart(part) {
        if (part.filename) {
            console.log("New file received !");
            currentFileName = part.filename;
            var size = part.byteCount;
            console.log("file size : " + size);

            var onError = function(error) {
                if (error) {
                    console.log("blobService.createBlockBlobFromStream : on error");
                    res.render('log.ejs', {log: error.message});
                } else {
                    console.log("onError() : No error");
                }
            };
            blobService.createBlockBlobFromStream(containerName, currentFileName, part, size, onError);
        } else {
            console.log("New part received (not a file)");
            form.handlePart(part);
        }
    }
    form.on('part', gotPart);


    function reqEnd() {
        console.log("Upload completed!");
        res.redirect('/upload/complete/?'+
            // 'url='+req.get('host')+'/library/download/'+targetFileName+//'+req.get('host')+'
            '&fileName='+currentFileName+
            '&result=success');
    }
    req.on('end', reqEnd);

    function handleErrorCallback(error) { // inspired from http://stackoverflow.com/questions/18367824/how-to-cancel-http-upload-from-data-events
        var errorLog = "There was an error while receiving the request. The client may have put an end to the connection : " + error;
        console.log(errorLog);
    }

    req.on('error', handleErrorCallback);

    form.on('error', handleErrorCallback);

    form.parse(req);
});

/*
 * \brief Returns the file to the user.
 */
app.get('/library/download/:fileName', function(req, res) {
    fs.readFile('./public/uploaded/'+req.params.fileName, function(error,data){
        res.end(data);
    });
});

/*
 * \brief Inform the user about his uploaded file (and about error if need be)
 */
app.get('/upload/complete/?', function(req, res) {
    var params = querystring.parse(url.parse(req.url).query);
    res.setHeader('Content-Type', 'text/html');
    res.charset = 'utf8';
    if (params['result'] === 'success') {
        res.render('uploadComplete.ejs', {log: 'File '+params['fileName']+' uploaded to ', url: url.parse('http://'+process.env['AZURE_STORAGE_ACCOUNT']+'.blob.core.windows.net/'+containerName+'/'+params['fileName']).href});
    } else {
        res.render('uploadComplete.ejs', {log: 'An error occurred while uploading '+params['fileName']+' : '+params['log'], url: ''});
    }

});

/*
 * \brief Handles wrong addresses
 */
app.use(function(req, res, next){
    res.setHeader('Content-Type', 'text/plain');
    res.charset = 'utf8';
    console.log("Redirection due to the fact that the page has not been found.");
    res.redirect('/upload');
});

app.listen(port);

Here is my form using bootstrap and jquery :

<div class="container">

  <div class="starter-template">
    <form class="form-signin" role="form" action="/upload/request" method="post" enctype="multipart/form-data">
        <h2 class="form-signin-heading">Select a file to upload</h2>
        <label>
          <input type="file" name="filename2" id="filename2" class="form-control" required>
      </label>
      <button class="btn btn-lg btn-primary btn-block" type="submit">Upload</button>
    </form>
</div>

And here is my web.config file :

<?xml version="1.0" encoding="utf-8"?>
<configuration>
  <system.webServer>
    <webSocket enabled="false" />
    <handlers>
      <add name="iisnode" path="server.js" verb="*" modules="iisnode"/>
    </handlers>
    <rewrite>
      <rules>
        <rule name="NodeInspector" patternSyntax="ECMAScript" stopProcessing="true">
          <match url="^server.js\/debug[\/]?" />
        </rule>
        <rule name="StaticContent">
          <action type="Rewrite" url="public{REQUEST_URI}"/>
        </rule>
        <rule name="DynamicContent">
          <conditions>
            <add input="{REQUEST_FILENAME}" matchType="IsFile" negate="True"/>
          </conditions>
          <action type="Rewrite" url="server.js"/>
        </rule>
      </rules>
    </rewrite>
    <security>
      <requestFiltering>
        <requestLimits maxAllowedContentLength="1073741824"/>
      </requestFiltering>
    </security>
  </system.webServer>
  <system.web>
    <httpRuntime maxRequestLength="2097152"/>
  </system.web>
</configuration>

Answer 1

Microsoft has a longstanding tradition of describing file sizes in power-of-two units, which can cause some confusion. In Microsoft terminology "54.33 MB" = (54.33 * 1024 * 1024) bytes which is about 57,000,000 bytes. So, looking at the file using Microsoft tooling, the behavior you've described is "by design", though I understand if it is a little confusing.

These days, it might be more correct to say 54.33 Mebibytes (MiB) is about 57 MB, but when Microsoft's use of MB = 1024 * 1024 first started (pre-1998), this the Mebibyte terminology simply wasn't around. Its just something you need to accept using Microsoft system's.