You need to call -setRowHeight: on the outline view after changing the font size. E.g. [outlineView setRowHeight:[font pointSize]+2.0].
For getting the best height for the row, here's what I do in my app:
CGFloat KBDefaultLineHeightForFont (NSFont *font, BOOL sizeForTextField)
{
if (font == nil)
font = [NSFont userFontOfSize:0]; // Use default system font if font is nil.
NSLayoutManager *lm = [[NSLayoutManager alloc] init];
if (sizeForTextField)
[lm setTypesetterBehavior:NSTypesetterBehavior_10_2_WithCompatibility];
// On Mountain Lion and above, string drawing is done without using screen fonts, so we must do the same.
if (floor(NSAppKitVersionNumber) > NSAppKitVersionNumber10_7)
[lm setUsesScreenFonts:NO];
CGFloat height = [lm defaultLineHeightForFont:font];
[lm release];
return height;
}
Then:
CGFloat rowHeight = MAX(16.0, KBDefaultLineHeightForFont(ovFont, YES) + 3.0);
[outlineView setRowHeight:rowHeight];
[[[[outlineView tableColumns] objectAtIndex:0] dataCell] setFont:ovFont];