Solved this myself by adding OnPreferenceChangeListener.
notification = (CheckBoxPreference) findPreference("prefNotification");
notification.setOnPreferenceChangeListener(new OnPreferenceChangeListener(){
@Override
public boolean onPreferenceChange(Preference preference,
Object newValue) {
if (newValue.toString().equals("true"))
{
notificationsOn();
}
else
{
notificationsOff();
}
return true;
}
});