You could use #element_validate option for your field and write your own validation function. Maybe you could use is_float() or is_number() in your validation function. So your code
$form['yourField'] = array(
....
'#element_validate'=>'myValidation'
);
....
function myValidation($element, &$form_state, $form){
if (!is_float( $element['#value']) )){
form_error($element, t('This field is not decimal.'));
}
}