Try to filter your solutions.
a=1;
b=2;
% a threshold for imagery part
SMALL=1e-9;
syms x real;
y=inline((x^(a-1))*((1-x)^(b-1)));
beta=quad(y,0,1);
g=(1/beta)*(x^(a-1))*((1-x)^(b-1));
p=int(g,x,0,x);
% return true for physically meaningfull results
filter = @(xc) abs(imag(xc))<SMALL && real(xc)>0 && subs(g, x, xc) > 0 && subs(p, x, xc)>0;
for m=0:50
fxi=rand(1);
f=p-fxi;
xi=solve(f, x);
result=eval(xi);
idx = arrayfun (filter, result);
result = result(idx);
% make sure it is OK
assert(length(result)==1);
disp(result)
end