I am having trouble using namedtuples in objects that I want to put into multiprocessing. I am receiving pickling error. I tried couple of things from other stackoverflow posts, but I could not succeed. Here is the structure of my code:
package_main, test_module
import myprogram.package_of_classes.data_object_module
import ....obj_calculate
class test(object):
if __name__ == '__main__':
my_obj=create_obj('myobject',['f1','f2'])
input = multiprocessing.Queue()
output = multiprocessing.Queue()
input.put(my_obj)
j=Process(target=obj_calculate, args=(input,output))
j.start()
package_of_classes, data_object_module
import collections
import ....load_flat_file
def get_ntuple_format(obj):
nt_fields=''
for fld in obj.fields:
nt_fields=nt_fields+fld+', '
nt_fields=nt_fields[0:-2]
ntuple=collections.namedtuple('ntuple_format',nt_fields)
return ntuple
Class Data_obj:
def __init__(self, name,fields):
self.name=name
self.fields=fields
self.ntuple_form=get_ntuple_format(self)
def calculate(self):
self.file_read('C:/files','division.txt')
def file_read(self,data_directory,filename):
output=load_flat_file(data_directory,filename,self.ntuple_form)
self.data=output
utils_package,utils_module
def create_dataobj(name,fields):
locals()[name]=Data_Obj(name,fields)
return locals()[name]
def obj_calculate(input,output):
obj=input.get()
obj.calculate()
output.put(obj)
loads_module
def load_flat_file(data_directory,filename,ntuple_form):
csv.register_dialect('csvrd', delimiter='\t', quoting=csv.QUOTE_NONE)
ListofTuples=[]
with open(os.path.join(data_directory,filename), 'rb') as f:
reader = csv.reader(f,'csvrd')
for line in reader:
if line:
ListofTuples.append(ntuple_form._make(line))
return ListofTuples
And the error I am getting is:
PicklingError: PicklingError: Can't pickle class '__main__ . ntuple_format: it's not the same object as __ main __. ntuple_format
P.S. As I extracted this sample code from a large project, please ignore minor inconsistencies.