Error for M2Crypto https get thru a web proxy with cac card authentication

Question

I'm trying to make a https get request from behind a squid proxy with cac card authentication. Loading the opensc engine and grabbing the cert and private key seem to work fine. Below is the traceback and the code.

Any help is greatly appreciated.

Traceback

Traceback (most recent call last):
File "testM2Crypto.py", line 64, in <module>
res = m2urllib2.urlopen(req)
File "c:\Python27\lib\urllib2.py", line 135, in urlopen
return _opener.open(url, data, timeout)
File "c:\Python27\lib\urllib2.py", line 415, in open
response = self._open(req, data)
File "c:\Python27\lib\urllib2.py", line 433, in _open
'_open', req)
File "c:\Python27\lib\urllib2.py", line 387, in _call_chain
result = func(*args)
File "c:\Python27\lib\site-packages\M2Crypto\m2urllib2.py", line 94, in https_open
h.request(req.get_method(), req.get_selector(), req.data, headers)
File "c:\Python27\lib\httplib.py", line 963, in request
self._send_request(method, url, body, headers)
File "c:\Python27\lib\httplib.py", line 994, in _send_request
self.putrequest(method, url, **skips)
File "c:\Python27\lib\site-packages\M2Crypto\httpslib.py", line 140, in putrequest
raise ValueError, "unknown URL type: %s" % url
ValueError: unknown URL type: /index.asp?site=SomeSite

Code

from M2Crypto import httpslib, m2urllib2, m2, SSL, Engine
import urllib2

url = 'https://some.domain.com/index.asp?site=SomeSite'

e = Engine.load_dynamic_engine("pkcs11", "c:/windows/syswow64/engine_pkcs11.dll")
pk = Engine.Engine("pkcs11")
pk.ctrl_cmd_string("MODULE_PATH", "c:/windows/syswow64/opensc-pkcs11.dll")
m2.engine_init(m2.engine_by_id("pkcs11"))

cert = e.load_certificate("slot_01-id_01")
privatekey = e.load_private_key("slot_01-id_01")

ctx = SSL.Context("sslv23")
ctx.set_cipher_list("HIGH:!aNULL:!eNULL:@STRENGTH")
ctx.set_session_id_ctx("foobar")
m2.ssl_ctx_use_x509(ctx.ctx, cert.x509)
m2.ssl_ctx_use_pkey_privkey(ctx.ctx, privatekey.pkey)

proxy_support=urllib2.ProxyHandler({'https':'https://proxy:3128'})
opener = m2urllib2.build_opener(ctx, proxy_support)
m2urllib2.install_opener(opener)
req = m2urllib2.Request(url)
res = m2urllib2.urlopen(req)

Answer 1

I finally was able to solve the problem yesterday. I had to make a few modifications to the code. I also had to patch a bug in the M2Crypto library that was preventing https through a proxy(Thanks to Miloslav Trmač from redhat for the patch). The solution is below for anyone else who might be running into a similar problem. Hope this helps.

Code

from M2Crypto import httpslib, m2urllib2, m2, SSL, Engine
import urllib2

userPin = "123456"
rootCertPath = 'd:/path/to/rootCert.pem'
url = 'https://some.domain.com/index.asp?site=SomeSite'

e = Engine.load_dynamic_engine("pkcs11", "c:/windows/syswow64/engine_pkcs11.dll")
pk = Engine.Engine("pkcs11")
pk.ctrl_cmd_string("MODULE_PATH", "c:/windows/syswow64/opensc-pkcs11.dll")
if len(userPin) > 0: pk.ctrl_cmd_string("PIN", userPin)

m2.engine_init(m2.engine_by_id("pkcs11"))

rootcert = X509.load_cert(rootCertPath)
cert = e.load_certificate("slot_01-id_01")
privatekey = e.load_private_key("slot_01-id_01")

ctx = SSL.Context("sslv23")
ctx.set_cipher_list("HIGH:!aNULL:!eNULL:@STRENGTH")
ctx.set_session_id_ctx("foobar")
ctx.load_verify_locations(cafile=rootcert)
m2.ssl_ctx_use_x509(ctx.ctx, cert.x509)
m2.ssl_ctx_use_pkey_privkey(ctx.ctx, privatekey.pkey)

proxy_support=urllib2.ProxyHandler({'https':'https://proxy:3128'})
opener = m2urllib2.build_opener(ctx, proxy_support)
m2urllib2.install_opener(opener)
req = m2urllib2.Request(url)
try:
    res = m2urllib2.urlopen(req)
    print '\nsuccess'
except urllib2.HTTPError, err:
    print '\nerror'
    print 'err.code: '+str(err.code)
    print 'err.reason: '+str(err.reason)
    print 'err.read(): '+str(err.read())

Thanks to Miloslav Trmač from redhat for the patch. I found this patch at the following url, http://arm.koji.fedoraproject.org/koji/buildinfo?buildID=61225 .

M2Crypto Patch

diff -urN M2Crypto/M2Crypto/httpslib.py M2Crypto-0.21.1/M2Crypto/httpslib.py
--- M2Crypto/M2Crypto/httpslib.py   2012-03-15 03:27:22.181524406 +0100
+++ M2Crypto-0.21.1/M2Crypto/httpslib.py    2012-03-15 03:27:40.467485033 +0100
@@ -182,14 +182,14 @@
         else:
             HTTPSConnection.putheader(self, header, value)

-    def endheaders(self):
+    def endheaders(self, *args, **kwargs):
         # We've recieved all of hte headers. Use the supplied username
         # and password for authorization, possibly overriding the authstring
         # supplied in the headers.
         if not self._proxy_auth:
             self._proxy_auth = self._encode_auth()

-        HTTPSConnection.endheaders(self)
+        HTTPSConnection.endheaders(self, *args, **kwargs)

     def connect(self):
         HTTPConnection.connect(self)
diff -urN M2Crypto/M2Crypto/m2urllib2.py M2Crypto-0.21.1/M2Crypto/m2urllib2.py
--- M2Crypto/M2Crypto/m2urllib2.py  2011-01-15 20:10:05.000000000 +0100
+++ M2Crypto-0.21.1/M2Crypto/m2urllib2.py   2012-03-15 03:27:40.467485033 +0100
@@ -64,8 +64,10 @@
         target_host = urlparse.urlparse(full_url)[1]

         if (target_host != host):
+            request_uri = urlparse.urldefrag(full_url)[0]
             h = httpslib.ProxyHTTPSConnection(host = host, ssl_context = self.ctx)
         else:
+            request_uri = req.get_selector()
             h = httpslib.HTTPSConnection(host = host, ssl_context = self.ctx)
         # End our change
         h.set_debuglevel(self._debuglevel)
@@ -80,7 +82,7 @@
         # request.
         headers["Connection"] = "close"
         try:
-            h.request(req.get_method(), req.get_selector(), req.data, headers)
+            h.request(req.get_method(), request_uri, req.data, headers)
             r = h.getresponse()
         except socket.error, err: # XXX what error?
             raise URLError(err)