dill dump_session with ipython

Question

I'm trying to use the dill module to save my ipython session using dump_session() but I'm getting an error message. I'm using Ipython 1.0.0 and dill 0.2-a-dev 20120503. Does anyone out there have any insight? Thanks in advance.

Niall

Here's the enormous traceback:

--------------------------------------------------------------------------- AssertionError                            Traceback (most recent call last) <ipython-input-15-9934c16a537e> in <module>()
----> 1 dill.dump_session("/data/local/nrobin/tset.sess")

/home/h02/nrobin/.local/lib/python2.7/site-packages/dill-0.2a.dev_20120503-py2.7.egg/dill/dill.pyc in dump_session(filename, main_module)
    104         pickler._main_module = main_module
    105         pickler._session = True # is best indicator of when pickling a session
--> 106         pickler.dump(main_module)
    107         pickler._session = False
    108     finally:

/usr/local/sci/lib/python2.7/pickle.pyc in dump(self, obj)
    222         if self.proto >= 2:
    223             self.write(PROTO + chr(self.proto))
--> 224         self.save(obj)
    225         self.write(STOP)
    226 

/usr/local/sci/lib/python2.7/pickle.pyc in save(self, obj)
    284         f = self.dispatch.get(t)
    285         if f:
--> 286             f(self, obj) # Call unbound method with explicit self
    287             return
    288 

/home/h02/nrobin/.local/lib/python2.7/site-packages/dill-0.2a.dev_20120503-py2.7.egg/dill/dill.pyc in save_module(pickler, obj)
    415         if _DEBUG[0]: print "M1: %s" % obj
    416         pickler.save_reduce(__import__, (obj.__name__,), obj=obj,
--> 417                             state=obj.__dict__.copy())
    418     else:
    419         if _DEBUG[0]: print "M2: %s" % obj

/usr/local/sci/lib/python2.7/pickle.pyc in save_reduce(self, func, args, state, listitems, dictitems, obj)
    417 
    418         if state is not None:
--> 419             save(state)
    420             write(BUILD)
    421 

/usr/local/sci/lib/python2.7/pickle.pyc in save(self, obj)
    284         f = self.dispatch.get(t)
    285         if f:
--> 286             f(self, obj) # Call unbound method with explicit self
    287             return
    288 

/home/h02/nrobin/.local/lib/python2.7/site-packages/dill-0.2a.dev_20120503-py2.7.egg/dill/dill.pyc in save_module_dict(pickler, obj)
    284     else:
    285         if _DEBUG[0]: print "D2: %s" % "<dict ...>" #obj
--> 286         StockPickler.save_dict(pickler, obj)
    287     return
    288 

/usr/local/sci/lib/python2.7/pickle.pyc in save_dict(self, obj)
    647 
    648         self.memoize(obj)
--> 649         self._batch_setitems(obj.iteritems())
    650 
    651     dispatch[DictionaryType] = save_dict

/usr/local/sci/lib/python2.7/pickle.pyc in _batch_setitems(self, items)
    679                 for k, v in tmp:
    680                     save(k)
--> 681                     save(v)
    682                 write(SETITEMS)
    683             elif n:

/usr/local/sci/lib/python2.7/pickle.pyc in save(self, obj)
    329 
    330         # Save the reduce() output and finally memoize the object
--> 331         self.save_reduce(obj=obj, *rv)
    332 
    333     def persistent_id(self, obj):

/usr/local/sci/lib/python2.7/pickle.pyc in save_reduce(self, func, args, state, listitems, dictitems, obj)
    417 
    418         if state is not None:
--> 419             save(state)
    420             write(BUILD)
    421 

/usr/local/sci/lib/python2.7/pickle.pyc in save(self, obj)
    284         f = self.dispatch.get(t)
    285         if f:
--> 286             f(self, obj) # Call unbound method with explicit self
    287             return
    288 

/home/h02/nrobin/.local/lib/python2.7/site-packages/dill-0.2a.dev_20120503-py2.7.egg/dill/dill.pyc in save_module_dict(pickler, obj)
    284     else:
    285         if _DEBUG[0]: print "D2: %s" % "<dict ...>" #obj
--> 286         StockPickler.save_dict(pickler, obj)
    287     return
    288 

/usr/local/sci/lib/python2.7/pickle.pyc in save_dict(self, obj)
    647 
    648         self.memoize(obj)
--> 649         self._batch_setitems(obj.iteritems())
    650 
    651     dispatch[DictionaryType] = save_dict

/usr/local/sci/lib/python2.7/pickle.pyc in _batch_setitems(self, items)
    684                 k, v = tmp[0]
    685                 save(k)
--> 686                 save(v)
    687                 write(SETITEM)
    688             # else tmp is empty, and we're done

/usr/local/sci/lib/python2.7/pickle.pyc in save(self, obj)
    329 
    330         # Save the reduce() output and finally memoize the object
--> 331         self.save_reduce(obj=obj, *rv)
    332 
    333     def persistent_id(self, obj):

/usr/local/sci/lib/python2.7/pickle.pyc in save_reduce(self, func, args, state, listitems, dictitems, obj)
    417 
    418         if state is not None:
--> 419             save(state)
    420             write(BUILD)
    421 

/usr/local/sci/lib/python2.7/pickle.pyc in save(self, obj)
    284         f = self.dispatch.get(t)
    285         if f:
--> 286             f(self, obj) # Call unbound method with explicit self
    287             return
    288 

/home/h02/nrobin/.local/lib/python2.7/site-packages/dill-0.2a.dev_20120503-py2.7.egg/dill/dill.pyc in save_module_dict(pickler, obj)
    284     else:
    285         if _DEBUG[0]: print "D2: %s" % "<dict ...>" #obj
--> 286         StockPickler.save_dict(pickler, obj)
    287     return
    288 

/usr/local/sci/lib/python2.7/pickle.pyc in save_dict(self, obj)
    647 
    648         self.memoize(obj)
--> 649         self._batch_setitems(obj.iteritems())
    650 
    651     dispatch[DictionaryType] = save_dict

/usr/local/sci/lib/python2.7/pickle.pyc in _batch_setitems(self, items)
    679                 for k, v in tmp:
    680                     save(k)
--> 681                     save(v)
    682                 write(SETITEMS)
    683             elif n:

/usr/local/sci/lib/python2.7/pickle.pyc in save(self, obj)
    284         f = self.dispatch.get(t)
    285         if f:
--> 286             f(self, obj) # Call unbound method with explicit self
    287             return
    288 

/home/h02/nrobin/.local/lib/python2.7/site-packages/dill-0.2a.dev_20120503-py2.7.egg/dill/dill.pyc in save_instancemethod(pickler, obj)
    303     if _DEBUG[0]: print "Me: %s" % obj
    304     pickler.save_reduce(MethodType, (obj.im_func, obj.im_self,
--> 305                                      obj.im_class), obj=obj)
    306     return
    307 

/usr/local/sci/lib/python2.7/pickle.pyc in save_reduce(self, func, args, state, listitems, dictitems, obj)
    399         else:
    400             save(func)
--> 401             save(args)
    402             write(REDUCE)
    403 

/usr/local/sci/lib/python2.7/pickle.pyc in save(self, obj)
    284         f = self.dispatch.get(t)
    285         if f:
--> 286             f(self, obj) # Call unbound method with explicit self
    287             return
    288 

/usr/local/sci/lib/python2.7/pickle.pyc in save_tuple(self, obj)
    546         if n <= 3 and proto >= 2:
    547             for element in obj:
--> 548                 save(element)
    549             # Subtle.  Same as in the big comment below.
    550             if id(obj) in memo:

/usr/local/sci/lib/python2.7/pickle.pyc in save(self, obj)
    284         f = self.dispatch.get(t)
    285         if f:
--> 286             f(self, obj) # Call unbound method with explicit self
    287             return
    288 

/home/h02/nrobin/.local/lib/python2.7/site-packages/dill-0.2a.dev_20120503-py2.7.egg/dill/dill.pyc in save_function(pickler, obj)
    268         pickler.save_reduce(FunctionType, (obj.func_code, obj.func_globals,
    269                                            obj.func_name, obj.func_defaults,
--> 270                                            obj.func_closure), obj=obj)
    271     else:
    272         if _DEBUG[0]: print "F2: %s" % obj

/usr/local/sci/lib/python2.7/pickle.pyc in save_reduce(self, func, args, state, listitems, dictitems, obj)
    399         else:
    400             save(func)
--> 401             save(args)
    402             write(REDUCE)
    403 

/usr/local/sci/lib/python2.7/pickle.pyc in save(self, obj)
    284         f = self.dispatch.get(t)
    285         if f:
--> 286             f(self, obj) # Call unbound method with explicit self
    287             return
    288 

/usr/local/sci/lib/python2.7/pickle.pyc in save_tuple(self, obj)
    560         write(MARK)
    561         for element in obj:
--> 562             save(element)
    563 
    564         if id(obj) in memo:

/usr/local/sci/lib/python2.7/pickle.pyc in save(self, obj)
    284         f = self.dispatch.get(t)
    285         if f:
--> 286             f(self, obj) # Call unbound method with explicit self
    287             return
    288 

/home/h02/nrobin/.local/lib/python2.7/site-packages/dill-0.2a.dev_20120503-py2.7.egg/dill/dill.pyc in save_module_dict(pickler, obj)
    284     else:
    285         if _DEBUG[0]: print "D2: %s" % "<dict ...>" #obj
--> 286         StockPickler.save_dict(pickler, obj)
    287     return
    288 

/usr/local/sci/lib/python2.7/pickle.pyc in save_dict(self, obj)
    647 
    648         self.memoize(obj)
--> 649         self._batch_setitems(obj.iteritems())
    650 
    651     dispatch[DictionaryType] = save_dict

/usr/local/sci/lib/python2.7/pickle.pyc in _batch_setitems(self, items)
    679                 for k, v in tmp:
    680                     save(k)
--> 681                     save(v)
    682                 write(SETITEMS)
    683             elif n:

/usr/local/sci/lib/python2.7/pickle.pyc in save(self, obj)
    284         f = self.dispatch.get(t)
    285         if f:
--> 286             f(self, obj) # Call unbound method with explicit self
    287             return
    288 

/home/h02/nrobin/.local/lib/python2.7/site-packages/dill-0.2a.dev_20120503-py2.7.egg/dill/dill.pyc in save_instancemethod(pickler, obj)
    303     if _DEBUG[0]: print "Me: %s" % obj
    304     pickler.save_reduce(MethodType, (obj.im_func, obj.im_self,
--> 305                                      obj.im_class), obj=obj)
    306     return
    307 

/usr/local/sci/lib/python2.7/pickle.pyc in save_reduce(self, func, args, state, listitems, dictitems, obj)
    399         else:
    400             save(func)
--> 401             save(args)
    402             write(REDUCE)
    403 

/usr/local/sci/lib/python2.7/pickle.pyc in save(self, obj)
    284         f = self.dispatch.get(t)
    285         if f:
--> 286             f(self, obj) # Call unbound method with explicit self
    287             return
    288 

/usr/local/sci/lib/python2.7/pickle.pyc in save_tuple(self, obj)
    546         if n <= 3 and proto >= 2:
    547             for element in obj:
--> 548                 save(element)
    549             # Subtle.  Same as in the big comment below.
    550             if id(obj) in memo:

/usr/local/sci/lib/python2.7/pickle.pyc in save(self, obj)
    284         f = self.dispatch.get(t)
    285         if f:
--> 286             f(self, obj) # Call unbound method with explicit self
    287             return
    288 

/home/h02/nrobin/.local/lib/python2.7/site-packages/dill-0.2a.dev_20120503-py2.7.egg/dill/dill.pyc in save_function(pickler, obj)
    268         pickler.save_reduce(FunctionType, (obj.func_code, obj.func_globals,
    269                                            obj.func_name, obj.func_defaults,
--> 270                                            obj.func_closure), obj=obj)
    271     else:
    272         if _DEBUG[0]: print "F2: %s" % obj

/usr/local/sci/lib/python2.7/pickle.pyc in save_reduce(self, func, args, state, listitems, dictitems, obj)
    403 
    404         if obj is not None:
--> 405             self.memoize(obj)
    406 
    407         # More new special cases (that work with older protocols as

/usr/local/sci/lib/python2.7/pickle.pyc in memoize(self, obj)
    242         if self.fast:
    243             return
--> 244         assert id(obj) not in self.memo
    245         memo_len = len(self.memo)
    246         self.write(self.put(memo_len))

AssertionError:

Answer 1

dill.dump_session didn't work in ipython due to ipython's 'exit' type... basically, ipython hijacks the interpreter's exit method and turns it into some new object type. I have a flag that recognizes when "IPYTHON is True"... however dump_session was still seeing ipython's exit types in the globals, which were being saved in the session dump. Now, I assume they are singletons... and don't save them. Looks like it works as of dill revision #511, and you can serialize the ipython session to a file.

Let me know if you find other issues in ipython, as I rely on ipython users to tell me when there's an issue with dill in ipython.

The updates have been posted to github at https://github.com/uqfoundation, and the on the issue tracker http://trac.mystic.cacr.caltech.edu/project/pathos/ticket/131.