I have tried like this and it worked:
NSOpenPanel* dialog = [NSOpenPanel openPanel];
[dialog setAllowedFileTypes:[NSArray arrayWithObject:@"png"]];
NSButton *openPanelAccessoryView = [[[NSButton alloc] initWithFrame:NSMakeRect(0.0, 0.0, 324.0, 22.0)] autorelease];
[openPanelAccessoryView setButtonType:NSSwitchButton];
[openPanelAccessoryView setBezelStyle:0];
[openPanelAccessoryView setAction:@selector(openUnrecognizedFiles:)];
[openPanelAccessoryView setTarget:self];
[dialog setAccessoryView:openPanelAccessoryView];
openPanel = dialog;
[dialog beginSheetModalForWindow:[[self view] window] completionHandler:^(NSInteger result){
if(result == NSFileHandlingPanelOKButton)
{
}
} ];
This is action same as yours:
- (void)openUnrecognizedFiles:(id)sender
{
if ([sender state])
[openPanel setAllowedFileTypes:nil];
else
[openPanel setAllowedFileTypes:[NSArray arrayWithObject:@"jpeg"]];
}
Now file type is changing as per check box on & off.