This is where F#'s active patterns are useful -- it lets you put the matching logic into a function, then you can use the patterns without having to worry about how the matching is happening behind the scenes.
For example, you could define an active pattern for your code and use it like this:
open System.Drawing
let (|Transparent|Grayscale|Color|) (color : Color) =
if color.A = 0uy then Transparent
elif color.R = color.G && color.R = color.B && color.R <> 255uy then
let newAlpha = calculateAlpha (int alpha) (int red)
Grayscale (Color.FromArgb (newAlpha, 0, 0, 0))
else
Color color
let private actionPixel (pixelColour : Color) =
match pixelColour with
| Transparent ->
Color.Transparent
| Grayscale c ->
c
| Color c ->
c