I think the problem might be that when you are defining a recursive function you need to tell OCaml using the rec keyword.
Try changing the code to:
let f a =
if a = 0 then
0
else
a * a
let rec repeathelper n f answer accum =
if n = accum then
answer
else
repeathelper n f (f answer) (accum+1)
let repeat n f x = repeathelper n (f 0) 0 0