The array scenarios in your code are being treated as AND conditions when listed in the $orders->where()
method.
Try this:
$orders = $modx->newQuery('Orders');
$orders->where(array(
'Orders.start_date:<=' => $rentalDate->end_date,
'AND:Orders.start_date:>=' => $rentalDate->start_date
));
$orders->orCondition(array( //scenario #2
'Orders.end_date:<=' => $rentalDate->end_date,
'AND:Order.start_date:>=' => $rentalDate->start_date
));
$orders->orCondition(array( //scenario #3
'Orders.end_date:>=' => $rentalDate->start_date,
'AND:Orders.end_date:<=' => $rentalDate->end_date
));
// uncomment the following lines to see the raw query generated
// $orders->prepare();
// print_r($orders->toSql());
$conflictingOrders = $modx->getCollection('Orders',$orders);