This is binary search with unknown upper bound. Knowing this, we can easily write/adapt implementations:
upper=42 # an arbitrary number >= 1
lower=0
while [[ -f $upper.txt ]]
do
lower=$upper
(( upper*= 2 ))
done
while (( lower < upper-1 ))
do
middle=$(((lower+upper)/2))
if [[ -f $middle.txt ]]
then
lower=$middle
else
upper=$middle
fi
done
echo "The number is $lower"