Question

I am trying to get the output of serverStatus command via pymongo and then insert it into a mongodb collection. Here is the dictionary

{u'metrics': {u'getLastError': {u'wtime': {u'num': 0, u'totalMillis': 0}, u'wtimeouts': 0L}, u'queryExecutor': {u'scanned': 0L}, u'record': {u'moves': 0L}, u'repl': {u'buffer': {u'count': 0L, u'sizeBytes': 0L, u'maxSizeBytes': 268435456}, u'apply': {u'batches': {u'num': 0, u'totalMillis': 0}, u'ops': 0L}, u'oplog': {u'insert': {u'num': 0, u'totalMillis': 0}, u'insertBytes': 0L}, u'network': {u'bytes': 0L, u'readersCreated': 0L, u'getmores': {u'num': 0, u'totalMillis': 0}, u'ops': 0L}, u'preload': {u'docs': {u'num': 0, u'totalMillis': 0}, u'indexes': {u'num': 0, u'totalMillis': 0}}}, u'ttl': {u'passes': 108L, u'deletedDocuments': 0L}, u'operation': {u'fastmod': 0L, u'scanAndOrder': 0L, u'idhack': 0L}, u'document': {u'deleted': 0L, u'updated': 0L, u'inserted': 1L, u'returned': 0L}}, u'process': u'mongod', u'pid': 7073, u'connections': {u'current': 1, u'available': 818, u'totalCreated': 58L}, u'locks': {u'admin': {u'timeAcquiringMicros': {}, u'timeLockedMicros': {}}, u'local': {u'timeAcquiringMicros': {u'r': 1942L, u'w': 0L}, u'timeLockedMicros': {u'r': 72369L, u'w': 0L}}, u'.': {u'timeAcquiringMicros': {u'R': 218733L, u'W': 30803L}, u'timeLockedMicros': {u'R': 311478L, u'W': 145679L}}}, u'cursors': {u'clientCursors_size': 0, u'timedOut': 0, u'totalOpen': 0}, u'globalLock': {u'totalTime': 6517358000L, u'lockTime': 145679L, u'currentQueue': {u'total': 0, u'writers': 0, u'readers': 0}, u'activeClients': {u'total': 0, u'writers': 0, u'readers': 0}}, u'extra_info': {u'note': u'fields vary by platform', u'page_faults': 21, u'heap_usage_bytes': 62271152}, u'uptime': 6518.0, u'network': {u'numRequests': 103, u'bytesOut': 106329, u'bytesIn': 6531}, u'uptimeMillis': 6517358L, u'recordStats': {u'local': {u'pageFaultExceptionsThrown': 0, u'accessesNotInMemory': 0}, u'pageFaultExceptionsThrown': 0, u'accessesNotInMemory': 0}, u'version': u'2.4.8', u'dur': {u'compression': 0.0, u'journaledMB': 0.0, u'commits': 30, u'writeToDataFilesMB': 0.0, u'commitsInWriteLock': 0, u'earlyCommits': 0, u'timeMs': {u'writeToJournal': 0, u'dt': 3077, u'remapPrivateView': 0, u'prepLogBuffer': 0, u'writeToDataFiles': 0}}, u'mem': {u'resident': 36, u'supported': True, u'virtual': 376, u'mappedWithJournal': 160, u'mapped': 80, u'bits': 64}, u'opcountersRepl': {u'getmore': 0, u'insert': 0, u'update': 0, u'command': 0, u'query': 0, u'delete': 0}, u'indexCounters': {u'missRatio': 0.0, u'resets': 0, u'hits': 0, u'misses': 0, u'accesses': 0}, u'uptimeEstimate': 6352.0, u'host': u'kal-el', u'writeBacksQueued': False, u'localTime': datetime.datetime(2014, 1, 18, 8, 1, 30, 22000), u'backgroundFlushing': {u'last_finished': datetime.datetime(2014, 1, 18, 8, 0, 52, 713000), u'last_ms': 0, u'flushes': 108, u'average_ms': 1.1111111111111112, u'total_ms': 120}, u'opcounters': {u'getmore': 0, u'insert': 1, u'update': 0, u'command': 105, u'query': 108, u'delete': 0}, u'ok': 1.0, u'asserts': {u'msg': 0, u'rollovers': 0, u'regular': 0, u'warning': 0, u'user': 0}}

I am getting "Key '.' must not contain ." error. What could be the issue here? I am not seeing any . in the key name.

Here is the traceback:

Traceback (most recent call last):
  File "src/mongodb_status.py", line 37, in <module>
    get_mongodb_status()
  File "src/mongodb_status.py", line 23, in get_mongodb_status
    md_status.insert(status_data)
  File "/home/guruprasad/dev/py/src/dnacraft_monitor_servers/venv/local/lib/python2.7/site-packages/pymongo/collection.py", line 362, in insert
    self.database.connection)
bson.errors.InvalidDocument: key '.' must not contain '.'
Was it helpful?

Solution

In the 14th line:

u'.': {u'timeAcquiringMicros': {u'R': 218733L, u'W': 30803L}, u'timeLockedMicros': {u'R': 311478L, u'W': 145679L}}}   

For future safety, iterate through the keys, replace '.'s with '_' or something, and then perform a write.

OTHER TIPS

Here is a function which will remove '.' from your keys:

def fix_dict(data, ignore_duplicate_key=True):
    """
    Removes dots "." from keys, as mongo doesn't like that.
    If the key is already there without the dot, the dot-value get's lost.
    This modifies the existing dict!

    :param ignore_duplicate_key: True: if the replacement key is already in the dict, now the dot-key value will be ignored.
                                 False: raise ValueError in that case.
    """
    if isinstance(data, (list, tuple)):
        list2 = list()
        for e in data:
            list2.append(fix_dict(e))
        # end if
        return list2
    if isinstance(data, dict):
        # end if
        for key, value in data.items():
            value = fix_dict(value)
            old_key = key
            if "." in key:
                key = old_key.replace(".", "")
                if key not in data:
                    data[key] = value
                else:
                    error_msg = "Dict key {key} containing a \".\" was ignored, as {replacement} already exists".format(
                        key=key_old, replacement=key)
                    if force:
                        import warnings
                        warnings.warn(error_msg, category=RuntimeWarning)
                    else:
                        raise ValueError(error_msg)
                    # end if
                # end if
                del data[old_key]
            # end if
            data[key] = value
        # end for
        return data
    # end if
    return data
# end def

A solution that worked for me was to wrap the dictionary inside a python list:

new_item_to_store = list(dict_to_store.items())

An interesting point is that the keys through the dot are not saved for insert_one And if you save the object, and then update it via find_one_and_update, the keys through the dot will not throw an error

Error:

product = {'name': 'test_product', 'links': {'test.xyz': 'https://...'}}
product_id = products_collection.insert_one(product).inserted_id

Working version:

product = {'name': 'test_product', 'links': {}}
product_id = self.products_collection.insert_one(product).inserted_id
products_collection.find_one_and_update(
    {'_id': product_id},
    {'$set': {'links': {'test.xyz': 'https://...'}}}
)
Licensed under: CC-BY-SA with attribution
Not affiliated with StackOverflow
scroll top