400 Error when checking if website is valid with HttpWebRequest and HttpWebResponse Why?

Question

Using the following code here to determine if a url is valid or not:

public bool UrlIsValid(string url)
{
    if(!url.ToLower().StartsWith("http://") && !url.ToLower().StartsWith("https://"))
    {
        url = "http://" + url;
    }

    try
    {
        HttpWebRequest request = HttpWebRequest.Create(url) as HttpWebRequest;
        request.Timeout = 5000; //set the timeout to 5 seconds to keep the user from waiting too long for the page to load
        request.Method = "HEAD"; //Get only the header information -- no need to download any content

        HttpWebResponse response = request.GetResponse() as HttpWebResponse;

        int statusCode = (int)response.StatusCode;
        if (statusCode >= 100 && statusCode < 400) //Good requests
        {
            return true;
        }
        else if (statusCode >= 500 && statusCode <= 510) //Server Errors
        {
            log.Warn(String.Format("The remote server has thrown an internal error. Url is not valid: {0}", url));
            return false;
        }
    }
    catch (WebException ex)
    {
        if (ex.Status == WebExceptionStatus.ProtocolError) //400 errors
        {
            log.Warn(String.Format("400 Error logged: {0}", url));
            return false;
        }
        else
        {
            log.Warn(String.Format("Unhandled status [{0}] returned for url: {1}", ex.Status, url), ex);
        }
    }
    catch (Exception ex)
    {
        log.Error(String.Format("Could not test url {0}.", url), ex);
    }
    return false;
}

Problem is it seems to return false with the following url: https://www.paypal.com or even with http://www.paypal.com giving me a 400 Error in the logs. Why is that? Is there anyway to get around this?

Answer 1

Your code is fine for other sites.

PayPal must not respond to HEAD requests.

The System.Net.WebException returned says 'Unhandled'

The easiest way around this is just to use a GET.