Error while saving djangotoolbox ListField to GAE datastore

Question

While saving ListField to GAE datastore, I get 'Unsupported type for property tags: '
GAE datastore receives a generator object to save into database:

name: 'tags'
value: <generator object <genexpr> at 0x049C6620>

dbindexer\compiler.py or djangotoolbox\db\basecompiler.py is perhaps generating this object from list object passed to it.

(<djangotoolbox.fields.ListField object at 0x047A4070>, [1L])]

Any hints about possible problems?

from django.db import models
from djangotoolbox.fields import ListField

class Tag(models.Model):
    name = models.CharField(max_length=255)

class Note(models.Model):
    tags = ListField(db.ForeignKey(Tag))

Exception:

Django Version: 1.3.1
Exception Type: DatabaseError
Exception Value: Unsupported type for property tags: <type 'generator'>
Exception Location: google_appengine\google\appengine\api\datastore_types.py in ValidateProperty, line 1529

File "myapp\django\db\models\base.py" in save
  462. self.save_base(using=using, force_insert=force_insert, force_update=force_update)
File "myapp\django\db\models\base.py" in save_base
  573. result = manager._insert(values, return_id=update_pk, using=using)
File "myapp\django\db\models\manager.py" in _insert
  195. return insert_query(self.model, values, **kwargs)
File "myapp\django\db\models\query.py" in insert_query
  1438. return query.get_compiler(using=using).execute_sql(return_id)
File "myapp\dbindexer\compiler.py" in execute_sql
  38.  return super(SQLInsertCompiler, self).execute_sql(return_id=return_id)
File "myapp\djangotoolbox\db\basecompiler.py" in execute_sql
  549. key = self.insert(field_values, return_id=return_id)
File "myapp\djangoappengine\db\compiler.py" in _func
  68.   return func(*args, **kwargs)
File "myapp\djangoappengine\db\compiler.py" in insert
  387.  entity.update(properties)
File "google_appengine\google\appengine\api\datastore.py" in update
  890.  self.__setitem__(name, value)
File "google_appengine\google\appengine\api\datastore.py" in __setitem__
  868.  datastore_types.ValidateProperty(name, value)
File "google_appengine\google\appengine\api\datastore_types.py" in ValidateProperty
  1529. 'Unsupported type for property %s: %s' % (name, v.__class__))

generator object is generated by djangotoolbox but GAE is not accepting the generator type. Code from djangotoolbox/db/base.py

   def _value_for_db_collection(self, value, field, field_kind, db_type,
                                 lookup):
      # Do convert filter parameters.
        if lookup:
            # Special case where we are looking for an empty list
            if lookup == 'exact' and db_type == 'list' and value == u'[]':
                return []
            value = self._value_for_db(value, subfield,
                                       subkind, db_subtype, lookup)

        # Convert list/set items or dict values.
        else:
            if field_kind == 'DictField':

                # Generator yielding pairs with converted values.
                value = (
                    (key, self._value_for_db(subvalue, subfield,
                                             subkind, db_subtype, lookup))
                    for key, subvalue in value.iteritems())

                # Return just a dict, a once-flattened list;
                if db_type == 'dict':
                    return dict(value)
                elif db_type == 'list':
                    return list(item for pair in value for item in pair)

            else:

                # Generator producing converted items.
                value = (
                    self._value_for_db(subvalue, subfield,
                                       subkind, db_subtype, lookup)
                    for subvalue in value)

                # "list" may be used for SetField.
                if db_type in 'list':
                    return list(value)
                elif db_type == 'set':
                    # assert field_kind != 'ListField'
                    return set(value)

            # Pickled formats may be used for all collection fields,
            # the fields "natural" type is serialized (something
            # concrete is needed, pickle can't handle generators :-)
            if db_type == 'bytes':
                return pickle.dumps(field._type(value), protocol=2)
            elif db_type == 'string':
                return pickle.dumps(field._type(value))

        # If nothing matched, pass the generator to the back-end.
        return value

AppEngine code that tries to validate the above generator and throws error:

def ValidateProperty(name, values, read_only=False):
  ValidateString(name, 'property name', datastore_errors.BadPropertyError)
  values_type = type(values)

  if values_type is tuple:
    raise datastore_errors.BadValueError('May not use tuple property value; property %s is %s.' %(name, repr(values)))

  **if values_type is not list:
    values = [values]**

  if not values:
    raise datastore_errors.BadValueError(
        'May not use the empty list as a property value; property %s is %s.' %
        (name, repr(values)))
  try:
    for v in values:
      prop_validator = _VALIDATE_PROPERTY_VALUES.get(v.__class__)
      if prop_validator is None:
        raise datastore_errors.BadValueError(
          'Unsupported type for property %s: %s' % (name, v.__class__))
      prop_validator(name, v)

  except (KeyError, ValueError, TypeError, IndexError, AttributeError), msg:
    raise datastore_errors.BadValueError(
      'Error type checking values for property %s: %s' % (name, msg))

Answer 1

Made change in following code (** new code marked **) and issue seems fixed (maybe I have done something wrong which requires this fix:

def _value_for_db_collection(self, value, field, field_kind, db_type,
                                 lookup):
    subfield, subkind, db_subtype = self._convert_as(field.item_field,
                                                     lookup)

    # Do convert filter parameters.
    if lookup:
        # Special case where we are looking for an empty list
        if lookup == 'exact' and db_type == 'list' and value == u'[]':
            return []
        value = self._value_for_db(value, subfield,
                                   subkind, db_subtype, lookup)

    # Convert list/set items or dict values.
    else:
        if field_kind == 'DictField':

            # Generator yielding pairs with converted values.
            value = (
                (key, self._value_for_db(subvalue, subfield,
                                         subkind, db_subtype, lookup))
                for key, subvalue in value.iteritems())

            # Return just a dict, a once-flattened list;
            if db_type == 'dict':
                return dict(value)
            elif db_type == 'list':
                return list(item for pair in value for item in pair)

        else:

            # Generator producing converted items.
            value = (
                self._value_for_db(subvalue, subfield,
                                   subkind, db_subtype, lookup)
                for subvalue in value)

            # "list" may be used for SetField.
            if db_type in 'list':
                return list(value)
            elif db_type == 'set':
                # assert field_kind != 'ListField'
                return set(value)

        # Pickled formats may be used for all collection fields,
        # the fields "natural" type is serialized (something
        # concrete is needed, pickle can't handle generators :-)
        if db_type == 'bytes':
            return pickle.dumps(field._type(value), protocol=2)
        elif db_type == 'string':
            return pickle.dumps(field._type(value))

    # If nothing matched, pass the generator to the back-end.

    #****************NEW CODE - begin **********************
    import types
    if type(value) is types.GeneratorType:
        value = list(value)
    #****************NEW CODE - end****************************

return value