If I understand correctly you are trying to parse hex decimal string to value in register. It would be easier to see the problem if you posted actual code.
I would do something like this:
; accepts null terminated string with characters '0'-'9', 'A'-'F'
; return value in eax
xor eax, eax
.loop:
mov cl, byte [ebx]
test cl, cl
jz .end
cmp cl, 65
jl .numeric
sub cl, 7 ; 'A' code is 65, need to subtract 7 more so 'A' = 10
.numeric:
sub cl, 48
shl eax, 4
or al, cl
inc ebx
jmp .loop
.end:
ret