Create a custom method use it in either in name or group and pass the value of name/group to custom function and and place the check in function:
$this->validate['Name'] = array(
'notEmpty' => array(
'rule' => array('notEmpty'),
'message' => __('err_required', array(__('lbl_Name', true))),
)
);
$this->validate['GroupID'] = array(
'notempty' => array(
'rule' => array('notEmpty'),
'message' => __('err_required', array(__('lbl_GroupID', true))),
),
'duplicate' => array(
'rule' => array('isDuplicate', $this->data['ModelName']['name']),
'message' => __('err_required', array(__('lbl_GroupID', true))),
)
);
public function isDuplicate($data, $name){
// check here
}