nodejs/express - stream stdout instantly to the client

Question 1

Here's a complete example using SSE (Server sent events). This works in Firefox and probably Chrome too:

var cp = require("child_process"),
         express = require("express"),
         app = express();

app.configure(function(){
    app.use(express.static(__dirname));
});


app.get('/msg', function(req, res){
    res.writeHead(200, { "Content-Type": "text/event-stream",
                         "Cache-control": "no-cache" });

    var spw = cp.spawn('ping', ['-c', '100', '127.0.0.1']),
    str = "";

    spw.stdout.on('data', function (data) {
        str += data.toString();

        // just so we can see the server is doing something
        console.log("data");

        // Flush out line by line.
        var lines = str.split("\n");
        for(var i in lines) {
            if(i == lines.length - 1) {
                str = lines[i];
            } else{
                // Note: The double-newline is *required*
                res.write('data: ' + lines[i] + "\n\n");
            }
        }
    });

    spw.on('close', function (code) {
        res.end(str);
    });

    spw.stderr.on('data', function (data) {
        res.end('stderr: ' + data);
    });
});

app.listen(4000);

And the client HTML:

<!DOCTYPE Html>
<html> 
<body>
   <ul id="eventlist"> </ul>

   <script>              
    var eventList = document.getElementById("eventlist");
    var evtSource = new EventSource("http://localhost:4000/msg");

    var newElement = document.createElement("li");
    newElement.innerHTML = "Messages:";
    eventList.appendChild(newElement);


    evtSource.onmessage = function(e) {
        console.log("received event");
        console.log(e);
        var newElement = document.createElement("li");

        newElement.innerHTML = "message: " + e.data;
        eventList.appendChild(newElement);
    };      

    evtSource.onerror = function(e) {
        console.log("EventSource failed.");
    };

    console.log(evtSource);

    </script>

</body>
</html>

Run node index.js and point your browser at http://localhost:4000/client.html. Note that I had to use the "-c" option rather than "-n" since I'm running OS X.

Question 2

If you are using Google Chrome, changing the content-type to "text/event-stream" does what your looking for.

res.writeHead(200, { "Content-Type": "text/event-stream" });

See my gist for complete example: https://gist.github.com/sfarthin/9139500

Question 3

This cannot be achieved with the standard HTTP request/response cycle. Basically what you are trying to do is make a "push" or "realtime" server. This can only be achieved with xhr-polling or websockets.

Code Example 1:

app.get('/path', function(req, res) {
   ...
   spw.stdout.on('data', function (data) {
      var str = data.toString();
      res.write(str + "\n");
   });
   ...
}

This code never sends an end signal and therefore will never respond. If you were to add a call to res.end() within that event handler, you will only get the first ping – which is the expected behavior because you are ending the response stream after the first chunk of data from stdout.

Code Sample 2:

spw.stdout.pipe(res);

Here stdout is flushing the packets to the browser, but the browser will not render the data chunks until all packets are received. Thus the reason why it waits 10 seconds and then renders the entirety of stdout. The major benefit to this method is not buffering the response in memory before sending — keeping your memory footprint lightweight.