Django: catch RequestDataTooBig exception

Question

Is there a way to catch RequestDataTooBig in Django, break execution and return custom error? Where should I do it properly?

I have a textarea, and user can type as much text as he wants, then he press send button and django fails with RequestDataTooBig exception (I test 5 MB of text).

django.core.exceptions.RequestDataTooBig: Request body exceeded settings.DATA_UPLOAD_MAX_MEMORY_SIZE.

I do not want to disable this setting, I want to catch this exception and return JSON with error.

When I try to catch it in my view, like this:

@require_http_methods([ "POST"])
def some_view(request):
    try:
        # some logic
        return JsonResponse({'key':'val'})
    except RequestDataTooBig:
        return JsonResponse({'error':'too_big_data'})

I get this error repeatedly - there must be chunks.

django.core.exceptions.RequestDataTooBig: Request body exceeded settings.DATA_UPLOAD_MAX_MEMORY_SIZE.
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
...
django.core.exceptions.RequestDataTooBig: Request body exceeded settings.DATA_UPLOAD_MAX_MEMORY_SIZE.

I suspect that I should catch it somewhere earlier - but have no idea where.

I use python3.4 and django 1.10.5

That's the traceback - my view is not even called, exception seem to be deeper

Starting development server at http://0:80/
Quit the server with CONTROL-C.
middleware init
middleware init
middleware call
Request body exceeded settings.DATA_UPLOAD_MAX_MEMORY_SIZE.
Request body exceeded settings.DATA_UPLOAD_MAX_MEMORY_SIZE.
Request body exceeded settings.DATA_UPLOAD_MAX_MEMORY_SIZE.
Request body exceeded settings.DATA_UPLOAD_MAX_MEMORY_SIZE.
Request body exceeded settings.DATA_UPLOAD_MAX_MEMORY_SIZE.
Request body exceeded settings.DATA_UPLOAD_MAX_MEMORY_SIZE.
Request body exceeded settings.DATA_UPLOAD_MAX_MEMORY_SIZE.
Request body exceeded settings.DATA_UPLOAD_MAX_MEMORY_SIZE.
Request body exceeded settings.DATA_UPLOAD_MAX_MEMORY_SIZE.
Request body exceeded settings.DATA_UPLOAD_MAX_MEMORY_SIZE.
Traceback (most recent call last):
  File "/opt/django/djvenv/lib/python3.4/site-packages/django/core/handlers/exception.py", line 39, in inner
    response = get_response(request)
  File "/opt/django/djvenv/lib/python3.4/site-packages/django/core/handlers/base.py", line 178, in _get_response
    response = middleware_method(request, callback, callback_args, callback_kwargs)
  File "/opt/django/djvenv/lib/python3.4/site-packages/django/middleware/csrf.py", line 260, in process_view
    request_csrf_token = request.POST.get('csrfmiddlewaretoken', '')
  File "/opt/django/djvenv/lib/python3.4/site-packages/django/core/handlers/wsgi.py", line 128, in _get_post
    self._load_post_and_files()
  File "/opt/django/djvenv/lib/python3.4/site-packages/django/http/request.py", line 311, in _load_post_and_files
    self._post, self._files = QueryDict(self.body, encoding=self._encoding), MultiValueDict()
  File "/opt/django/djvenv/lib/python3.4/site-packages/django/http/request.py", line 269, in body
    raise RequestDataTooBig('Request body exceeded settings.DATA_UPLOAD_MAX_MEMORY_SIZE.')
django.core.exceptions.RequestDataTooBig: Request body exceeded settings.DATA_UPLOAD_MAX_MEMORY_SIZE.

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/opt/django/djvenv/lib/python3.4/site-packages/django/core/handlers/exception.py", line 39, in inner
    response = get_response(request)
  File "/opt/django/djvenv/lib/python3.4/site-packages/django/utils/deprecation.py", line 136, in __call__
    response = self.get_response(request)
  File "/opt/django/djvenv/lib/python3.4/site-packages/django/core/handlers/exception.py", line 41, in inner
    response = response_for_exception(request, exc)
  File "/opt/django/djvenv/lib/python3.4/site-packages/django/core/handlers/exception.py", line 76, in response_for_exception
    response = debug.technical_500_response(request, *sys.exc_info(), status_code=400)
  File "/opt/django/djvenv/lib/python3.4/site-packages/django/views/debug.py", line 84, in technical_500_response
    html = reporter.get_traceback_html()
  File "/opt/django/djvenv/lib/python3.4/site-packages/django/views/debug.py", line 328, in get_traceback_html
    c = Context(self.get_traceback_data(), use_l10n=False)
  File "/opt/django/djvenv/lib/python3.4/site-packages/django/views/debug.py", line 304, in get_traceback_data
    'filtered_POST': self.filter.get_post_parameters(self.request),
  File "/opt/django/djvenv/lib/python3.4/site-packages/django/views/debug.py", line 167, in get_post_parameters
    return request.POST
  File "/opt/django/djvenv/lib/python3.4/site-packages/django/core/handlers/wsgi.py", line 128, in _get_post
    self._load_post_and_files()
  File "/opt/django/djvenv/lib/python3.4/site-packages/django/http/request.py", line 311, in _load_post_and_files
    self._post, self._files = QueryDict(self.body, encoding=self._encoding), MultiValueDict()
  File "/opt/django/djvenv/lib/python3.4/site-packages/django/http/request.py", line 269, in body
    raise RequestDataTooBig('Request body exceeded settings.DATA_UPLOAD_MAX_MEMORY_SIZE.')
django.core.exceptions.RequestDataTooBig: Request body exceeded settings.DATA_UPLOAD_MAX_MEMORY_SIZE.

During handling of the above exception, another exception occurred:
...

UPD: middleware is called, but doesn't return data. That makes me a bit mad.

What I did: Created a middleware as was advised by Thameem:

from django.http import JsonResponse, HttpResponse
from django.shortcuts import render
from django.core.exceptions import RequestDataTooBig

class CheckRequest(object):

    def __init__(self, get_response):
        print('middleware init')
        self.get_response = get_response

    def __call__(self, request):
        print('middleware call')

        response = self.get_response(request)

        return response

    def process_exception(self, request, exception):
        print('middleware process exeption', exception)
        if isinstance(exception, RequestDataTooBig):
            print('CALLED')
            return HttpResponse("dummy", content_type="text/plain")
            #return JsonResponse({"error":"file is too big"})

I've read that middleware have to return HttpResponse or None, so I've changed the code a bit. JsonResponse didn't work as well.

I've disabled all the other middlewares, in order to test André Cruz advice:

MIDDLEWARE = [
    'projectname.generator.custom_middleware.CheckRequest',
    #'django.middleware.security.SecurityMiddleware',
    #'django.contrib.sessions.middleware.SessionMiddleware',
    #'django.middleware.common.CommonMiddleware',
    #'django.middleware.csrf.CsrfViewMiddleware',
    #'django.contrib.auth.middleware.AuthenticationMiddleware',
    #'django.contrib.messages.middleware.MessageMiddleware',
    #'django.middleware.clickjacking.XFrameOptionsMiddleware',
    #'social_django.middleware.SocialAuthExceptionMiddleware',
]

I also tried to move middleware up and down, common other middlewares deliberately etc, nothing changed.

Since then I can see that errors are being caught:

Starting development server at http://0:80/
Quit the server with CONTROL-C.
middleware init
middleware init
middleware call
middleware process exeption Request body exceeded settings.DATA_UPLOAD_MAX_MEMORY_SIZE.
CALLED
[20/Apr/2017 19:05:55] "POST /text_generator/new_task/ HTTP/1.1" 200 5

But can not get response from the browser:

Answer 1

Finally, after exploring source code, I've found that request objects doesn't have body in this rare case.

https://github.com/django/django/blob/master/django/http/request.py

self._body has to be present, if you want to have repsonse valid, but it creates only if there were no exceptions

@property
    def body(self):
        if not hasattr(self, '_body'):
            if self._read_started:
                raise RawPostDataException("You cannot access body after reading from request's data stream")

            # Limit the maximum request data size that will be handled in-memory.
            if (settings.DATA_UPLOAD_MAX_MEMORY_SIZE is not None and
                    int(self.META.get('CONTENT_LENGTH') or 0) > settings.DATA_UPLOAD_MAX_MEMORY_SIZE):
                # ---------------- self._body is not exists at the moment
                raise RequestDataTooBig('Request body exceeded settings.DATA_UPLOAD_MAX_MEMORY_SIZE.')

            try:
                self._body = self.read()
            except IOError as e:
                raise UnreadablePostError(*e.args) from e
            self._stream = BytesIO(self._body)
        return self._body

My humble solution was just to add it's creation. After I added it, everyting started work as supposed - middleware returns response and browser sees it ( It was reported to django developers as a bug: https://code.djangoproject.com/ticket/28106 ).

 @property
    def body(self):
        if not hasattr(self, '_body'):
            if self._read_started:
                raise RawPostDataException("You cannot access body after reading from request's data stream")

            # Limit the maximum request data size that will be handled in-memory.
            if (settings.DATA_UPLOAD_MAX_MEMORY_SIZE is not None and
                    int(self.META.get('CONTENT_LENGTH') or 0) > settings.DATA_UPLOAD_MAX_MEMORY_SIZE):
                self._body = self.read(None) # <------------- FAST FIX
                raise RequestDataTooBig('Request body exceeded settings.DATA_UPLOAD_MAX_MEMORY_SIZE.')

            try:
                self._body = self.read()
            except IOError as e:
                six.reraise(UnreadablePostError, UnreadablePostError(*e.args), sys.exc_info()[2])
            self._stream = BytesIO(self._body)
        return self._body

Then you can create custom middleware:

from django.http import HttpResponse
from django.conf import settings
from django.core.exceptions import RequestDataTooBig


class CheckRequest(object):

    def __init__(self, get_response):
        self.get_response = get_response

    def __call__(self, request):
        try:
            body = request.body
        except RequestDataTooBig:
            return HttpResponse("dummy", content_type="text/plain")

        response = self.get_response(request)
        return response

And add it to the end of the list

MIDDLEWARE = [
    'django.middleware.security.SecurityMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
    'social_django.middleware.SocialAuthExceptionMiddleware',
    'projectname.appname.custom_middleware.CheckRequest',
]