python bson.errors.InvalidDocument: Cannot encode object: datetime.date(2015, 3, 1)

Question

I have the following functions:

#  this is in a module called 'dbw_lib'
def dateTimeOuput(start_days_back, end_days_back):
    start_delta = datetime.timedelta(days=start_days_back)
    end_delta = datetime.timedelta(days=end_days_back)
    start_date = datetime.date.today() - start_delta
    end_date = datetime.date.today() - end_delta
    return start_date, end_date

def dictByDate(start_days, end_days):
    start_date, end_date = dbw_lib.dateTimeOuput(start_days, end_days)

    date_string = { "created_at": {"$gte" : start_date, "$lt": end_date }  }

    user_id_email_dict = dbw_lib.dbwIdToEmailD(user_coll_obj, date_query = date_string)  # dict of all user ids and emails  
    print user_id_email_dict
    quit()

when I run key_dicts = dictByDate(90, 60), I'm getting the following traceback:

Traceback (most recent call last):
File "main.py", line 212, in <module>
program.runProgram()
File "main.py", line 61, in runProgram
report.RcreateReport()
File "filepath/report.py", line 86, in RcreateReport
key_dicts = dictByDate(90, 60)
File "filepath/report.py", line 65, in dictByDate
user_id_email_dict = dbw_lib.dbwIdToEmailD(user_coll_obj, date_query = date_string)  # dict of all user ids and emails  
File "filepath/dbw_lib.py", line 50, in dbwIdToEmailD
for pair in id_email_cursor:
File "/Library/Python/2.7/site-packages/pymongo-3.0-py2.7-macosx-10.9-intel.egg/pymongo/cursor.py", line 968, in __next__
File "/Library/Python/2.7/site-packages/pymongo-3.0-py2.7-macosx-10.9-intel.egg/pymongo/cursor.py", line 905, in _refresh
File "/Library/Python/2.7/site-packages/pymongo-3.0-py2.7-macosx-10.9-intel.egg/pymongo/cursor.py", line 812, in __send_message
File "/Library/Python/2.7/site-packages/pymongo-3.0-py2.7-macosx-10.9-intel.egg/pymongo/mongo_client.py", line 732, in _send_message_with_response
File "/Library/Python/2.7/site-packages/pymongo-3.0-py2.7-macosx-10.9-intel.egg/pymongo/mongo_client.py", line 743, in _reset_on_error
File "/Library/Python/2.7/site-packages/pymongo-3.0-py2.7-macosx-10.9-intel.egg/pymongo/server.py", line 85, in send_message_with_response
File "/Library/Python/2.7/site-packages/pymongo-3.0-py2.7-macosx-10.9-intel.egg/pymongo/message.py", line 107, in get_message
bson.errors.InvalidDocument: Cannot encode object: datetime.date(2015, 3, 1)

Answer 1

just replace

datetime.date.today()

with

datetime.datetime.today()

but note former has the date and the later will return datetime

Answer 2

Turns out that

PyMongo doesn't support saving date instances. The server doesn't have a type for dates without times, so there would have to be some convention used to save dates without times. If you need to save a date your client should convert it to a datetime instance and you can save that.

Quote's source.

This answer states that this can be done with the datetime.datetime.combine method like so:

datetime.datetime.combine(dateWithoutTime, datetime.time.min)

Answer 3

datetime.date is not part of the bson Encoders. (Maybe the authors forgot it or left it out intentionally since its ambiguous to append time information to date just like that.)

But you could write a function to extend custom types in pymongo Just as how you would extend a JSONEncoder in json you could do something similar in pymongo using a SONManipulator:

import datetime
import pymongo

class MigrationTransformer(pymongo.son_manipulator.SONManipulator):

    def _encode_date(self, value):
        return datetime.datetime.combine(
                value,
                datetime.datetime.min.time())

    def transform_incoming(self, son, collection):
        for (key, value) in son.items():
            # datetime.datetime is instance of datetime.date
            # compare type explicitly only
            if type(value) == datetime.date:
                son[key] = self._encode_date(value)
            elif isinstance(value, dict):    # recurse into sub-docs
                son[key] = self.transform_incoming(value, collection)
        return son

and then append it to your DB instance:

db.add_son_manipulator(MigrationTransformer())

(I have not given the transform_outgoing method since that is not relevant to the question, but you could find it here: http://api.mongodb.org/python/current/examples/custom_type.html)

Edit: There is a problem if the value of the key in the dict is a list type. For some reason, pymongo does not pass it to the SONManipulator. So the list is not transformed.

I've updated the class to handle this as well (but I didn't do it for sets and tuples).

class MigrationTransformer(SONManipulator):

    def _encode_date(self, value):
        return datetime.datetime.combine(
                value,
                datetime.datetime.min.time())

    def _handle_list(self, value):
        for index, item in enumerate(value):
            if isinstance(item, dict):
                value[index] = self._handle_dict(item)
            elif isinstance(item, list):
                value[index] = self._handle_list(item)
            elif isinstance(item, datetime.date):
                value[index] = self._encode_date(item)
        return value

    def _handle_dict(self, item):
        for (key, value) in item.items():
            if type(value) == datetime.date:
                item[key] = self._encode_date(value)
            elif isinstance(value, dict):    # recurse into sub-docs
                item[key] = self._handle_dict(value)
            elif isinstance(value, list):    # recurse into sub-docs
                item[key] = self._handle_list(value)
        return item

    def transform_incoming(self, son, collection):
        for (key, value) in son.items():
            # datetime.datetime is instance of datetime.date
            # compare type explicitly only
            if type(value) == datetime.date:
                son[key] = self._encode_date(value)
            elif isinstance(value, dict):    # recurse into sub-docs
                son[key] = self.transform_incoming(value, collection)
            elif isinstance(value, list):    # recurse into sub-docs
                son[key] = self._handle_list(value)
        return son