EAX and EDX are integer registers, so DIV is an integer division. You cannot expect a rational number like 0.9. DIV gives you in EDX the remainder of an integer division. You can use the FPU floating point registers or - better - multiply testScore with 100 before DIV:
#include <stdio.h>
int main ( void )
{
int testScore = 36;
int gradeScale = 0;
_asm
{
mov eax, testScore //student's score - let's test 36
mov ebx, 40 //max possible score of 40
imul eax,100
xor edx,edx //prevented an integer overflow error.
div ebx // EAX = EDX:EAX / EBX Remainder: EDX
mov gradeScale, eax //move result to gradeScale
}
printf("%i\n",gradeScale);
return 0;
}