You want to define your own subclass of NSTableView and set up -mouseDown:
like so:
- (void)mouseDown:(NSEvent *)theEvent {
NSPoint globalLocation = [theEvent locationInWindow];
NSPoint localLocation = [self convertPoint:globalLocation fromView:nil];
NSInteger clickedRow = [self rowAtPoint:localLocation];
BOOL wasPreselected = (self.selectedRow == clickedRow);
[super mouseDown:theEvent];
if (wasPreselected)
[self deselectRow:self.selectedRow];
}