Python multiprocessing imaplib throws error

Question

I have the following piece of code in which I am trying to fetch mails from different directories of a mailbox concurrently. However, it is showing the folloing problem. Have attached the shortened stacktrace.

import multiprocessing as mlp
import imaplib as impl

def somefunc(dirc):
    mail.select(dirc)
    result, data = mail.uid("search", None, "All")
    uids = data[0].split()
    print dirc
    for mail_id in uids:
       result, data = mail.uid("fetch", mail_id, "(RFC822)")

if __name__ == '__main__':
    mail = impl.IMAP4_SSL("somedomain")
    mail.login("username","password")

    jobs = []
    p1 = mlp.Process(target = somefunc, args = ("INBOX",))
    jobs.append(p1)
    p1.start()
    p2 = mlp.Process(target = somefunc, args = ("Sent",))
    jobs.append(p2)
    p2.start()
    for i in jobs:
        i.join()

It throws error:

Process Process-2:
Traceback (most recent call last):
File "/usr/lib/python2.7/multiprocessing/process.py", line 258, in _bootstrap
Process Process-1:
Traceback (most recent call last):
File "/usr/lib/python2.7/multiprocessing/process.py", line 258, in _bootstrap
    self.run()
File "/usr/lib/python2.7/multiprocessing/process.py", line 114, in run
    self._target(*self._args, **self._kwargs)
........
File "/usr/lib/python2.7/imaplib.py", line 859, in _command
    raise self.abort('socket error: %s' % val)
abort: socket error: [Errno 32] Broken pipe
    typ, dat = self._simple_command(name, mailbox)
........
error: [Errno 104] Connection reset by peer

Is it not possible to do imap connection concurrently???

Thanks... :)

Answer 1

My guess is that you are having problems because you are attempting to share the mail socket connection object across processes. Instead, try having each process create its connection:

import multiprocessing as mlp
import imaplib as impl

def somefunc(domain, name, password, dirc):
    mail = impl.IMAP4_SSL(domain)
    mail.login(name, password)
    mail.select(dirc)
    result, data = mail.uid("search", None, "All")
    uids = data[0].split()
    print dirc
    for mail_id in uids:
       result, data = mail.uid("fetch", mail_id, "(RFC822)")

if __name__ == '__main__':
    jobs = []

    for box in ("INBOX", "Sent"):
        p = mlp.Process(
                target = somefunc, 
                args = ("somedomain", "username", "password", box)
            )
        jobs.append(p)
        p.start()

    for i in jobs:
        i.join()