HTML5 Canvas - Anti-aliasing and Paint Bucket / Flood Fill

Question 1

If you simply want to change a color of a line: don't use bucket paint fill at all.

Store all your lines and shapes as objects/arrays and redraw them when needed.

This not only allow you to change canvas size without losing everything on it, but to change a color is simply a matter of changing a color property on your object/array and redraw, as well as scaling everything based on vectors instead of raster.

This will be faster than a bucket fill as redrawing is handled in most part internally and not by pixel-by-pixel in JavaScript as is needed with a bucket fill.

That being said: you cannot, unfortunately, disable anti-alias for shapes and lines, only for images (using the imageSmoothingEnabled property).

An object could look like this:

function myLine(x1, y1, x2, y2, color) {
    this.x1 = x1;
    this.y1 = y1;
    this.x2 = x2;
    this.y2 = y2;
    this.color = color;
    return this;
}

And then allocate it by:

var newLine = new myLine(x1, y1, x2, y2, color);

Then store this to an array:

/// globally:
var myLineStack = [];

/// after x1/x2/y1/y2 and color is achieved in the draw function:
myLineStack.push(new myLine(x1, y1, x2, y2, color));

Then it is just a matter of iterating through the objects when an update is needed:

/// some index to a line you want to change color for:
myLineStack[index].color = newColor;

/// Redraw all (room for optimizations here...)
context.clearRect( ... );

for(var i = 0, currentLine; currentLine = myLineStack[i]; i++) {

    /// new path
    context.beginPath();

    /// set the color for this line
    context.strokeStyle = currentLine.color;

    /// draw the actual line
    context.moveTo(currentLine.x1, currentLine.y1);
    context.lineTo(currentLine.x2, currentLine.y2);

    context.stroke();
}

(For optimizations you can for example clear only the area that needs redraw and draw a single index. You can also group lines/shapes with the same colors and draw then with a single setting of strokeStyle etc.)

Question 2

You can not always redraw the canvas, you may have used filters that can not be reversed, or just use so many fill and stroke calls it would be impractical to redraw.

I have my own flood fill based on a simple fill stack that paints to a tolerances and does its best to lessen anti-aliasing artifacts. Unfortunately if you have anti-aliasing on repeated fills will grow the filled region.

Below is the function, adapt it as suited, it is a direct lift from my code with comments added.

// posX,posY are the fill start position. The pixel at the location is used to test tolerance.
// RGBA      is the fill colour as an array of 4 bytes all ranged 0-255 for R,G,B,A
// diagonal  if true the also fill into pixels that touch at the corners. 
// imgData   canvas pixel data from ctx.getImageData method
// tolerance Fill tolerance range 0 only allow exact same colour to fill to 255
// fill      all but the extreme opposite.
// antiAlias if true fill edges to reduce anti-Aliasing artifacts.


Bitmaps.prototype.floodFill = function (posX, posY, RGBA, diagonal,imgData,tolerance,antiAlias) {
    var data = imgData.data; // image data to fill;
    antiAlias = true;
    var stack = [];          // paint stack to find new pixels to paint
    var lookLeft = false;    // test directions
    var lookRight = false;
    var w = imgData.width;   // width and height
    var h = imgData.height;
    var painted = new Uint8ClampedArray(w*h);  // byte array to mark painted area;
    var dw = w*4; // data width.
    var x = posX;   // just short version of pos because I am lazy
    var y = posY;
    var ind = y * dw + x * 4;  // get the starting pixel index
    var sr = data[ind];        // get the start colour tha we will use tollerance against.
    var sg = data[ind+1];
    var sb = data[ind+2];
    var sa = data[ind+3];     
    var sp = 0;
    var dontPaint = false;  // flag to indicate if checkColour can paint

    // function checks a pixel colour passes tollerance, is painted, or out of bounds.
    // if the pixel is over tollerance and not painted set it do reduce anti alising artifacts
    var checkColour = function(x,y){
        if( x<0 || y < 0 || y >=h || x >= w){  // test bounds
            return false;
        }
        var ind = y * dw + x * 4;  // get index of pixel
        var dif = Math.max(        // get the max channel differance;
            Math.abs(sr-data[ind]),
            Math.abs(sg-data[ind+1]),
            Math.abs(sb-data[ind+2]),                
            Math.abs(sa-data[ind+3])
        );        
        if(dif < tolerance){         // if under tollerance pass it
            dif = 0;
        }        
        var paint = Math.abs(sp-painted[y * w + x]); // is it already painted
        if(antiAlias && !dontPaint){  // mitigate anti aliasing effect
            // if failed tollerance and has not been painted set the pixel to 
            // reduce anti alising artifact
            if(dif !== 0 && paint !== 255){  
                data[ind] = RGBA[0];
                data[ind+1] = RGBA[1];
                data[ind+2] = RGBA[2];
                data[ind+3] = (RGBA[3]+data[ind+3])/2; // blend the alpha channel
                painted[y * w + x] = 255;  // flag pixel as painted
            }
        }
        return (dif+paint)===0?true:false;  // return tollerance status;
    }
    // set a pixel and flag it as painted;
    var setPixel = function(x,y){
        var ind = y * dw + x * 4;  // get index;
        data[ind] = RGBA[0];       // set RGBA
        data[ind+1] = RGBA[1];
        data[ind+2] = RGBA[2];
        data[ind+3] = RGBA[3];
        painted[y * w + x] = 255;   // 255 or any number >0 will do;
    }


    stack.push([x,y]);  // push the first pixel to paint onto the paint stack

    while (stack.length) {   // do while pixels on the stack
        var pos = stack.pop();  // get the pixel
        x = pos[0];
        y = pos[1];
        dontPaint = true;    // turn off anti alising 
        while (checkColour(x,y-1)) {  // find the bottom most pixel within tolerance;
            y -= 1;
        }
        dontPaint = false;    // turn on anti alising if being used
        //checkTop left and right if alowing diagonal painting
        if(diagonal){
            if(!checkColour(x-1,y) && checkColour(x-1,y-1)){
                stack.push([x-1,y-1]);
            }
            if(!checkColour(x+1,y) && checkColour(x+1,y-1)){
                stack.push([x+1,y-1]);
            }
        }
        lookLeft = false;  // set look directions
        lookRight = false; // only look is a pixel left or right was blocked
        while (checkColour(x,y)) { // move up till no more room
            setPixel(x,y);         // set the pixel
            if (checkColour(x - 1,y)) {  // check left is blocked
                if (!lookLeft) {        
                    stack.push([x - 1, y]);  // push a new area to fill if found
                    lookLeft = true;
                }
            } else 
            if (lookLeft) {
                lookLeft = false;
            }
            if (checkColour(x+1,y)) {  // check right is blocked
                if (!lookRight) {
                    stack.push([x + 1, y]); // push a new area to fill if found
                    lookRight = true;
                }
            } else 
            if (lookRight) {
                lookRight = false;
            }
            y += 1;                 // move up one pixel
        }
        // check down left 
        if(diagonal){  // check for diagnal areas and push them to be painted 
            if(checkColour(x-1,y) && !lookLeft){
                stack.push([x-1,y]);
            }
            if(checkColour(x+1,y) && !lookRight){
                stack.push([x+1,y]);
            }
        }
    }
    // all done
}

There is a better way that gives high quality results, the above code can be adapted to do this by using the painted array to mark the paint edges and then after the fill has completed scan the painted array and apply a convolution filter to each edge pixel you have marked. The filter is directional (depending on which sides are painted) and the code too long for this answer. I have pointed you in the right direction and the infrastructure is above.

Another way to improve the image quality is to super sample the image you are drawing to. Hold a second canvas that is double the size of the image being painted. Do all you drawing to that image and display it to the user on another canvas with CTX.imageSmoothingEnabled and ctx.setTransform(0.5,0,0,0.5,0,0) half size, when done and the image is ready half its size manually with the following code (don't rely on canvas imageSmoothingEnabled as it gets it wrong.)

Doing this will greatly improve the quality of your final image and with the above fill almost completely eliminate anti-aliasing artifacts from flood fills.

    // ctxS is the source canvas context
    var w = ctxS.canvas.width;
    var h = ctxS.canvas.height;
    var data = ctxS.getImageData(0,0,w,h);
    var d = data.data;
    var x,y;
    var ww = w*4;
    var ww4 = ww+4;
    for(y = 0; y < h; y+=2){
        for(x = 0; x < w; x+=2){
            var id = y*ww+x*4;
            var id1 = Math.floor(y/2)*ww+Math.floor(x/2)*4;
            d[id1] = Math.sqrt((d[id]*d[id]+d[id+4]*d[id+4]+d[id+ww]*d[id+ww]+d[id+ww4]*d[id+ww4])/4);
            id += 1;
            id1 += 1;
            d[id1] = Math.sqrt((d[id]*d[id]+d[id+4]*d[id+4]+d[id+ww]*d[id+ww]+d[id+ww4]*d[id+ww4])/4);
            id += 1;
            id1 += 1;
            d[id1] = Math.sqrt((d[id]*d[id]+d[id+4]*d[id+4]+d[id+ww]*d[id+ww]+d[id+ww4]*d[id+ww4])/4);
            id += 1;
            id1 += 1;
            d[id1] = Math.sqrt((d[id]*d[id]+d[id+4]*d[id+4]+d[id+ww]*d[id+ww]+d[id+ww4]*d[id+ww4])/4);
        }
    }
    ctxS.putImageData(data,0,0); // save imgData
    // grab it again for new image we don't want to add artifacts from the GPU
    var data = ctxS.getImageData(0,0,Math.floor(w/2),Math.floor(h/2));
    var canvas = document.createElement("canvas");
    canvas.width = Math.floor(w/2);
    canvas.height =Math.floor(h/2);
    var ctxS = canvas.getContext("2d",{ alpha: true });
    ctxS.putImageData(data,0,0);  
    // result canvas with downsampled high quality image.