uwsgi_response_write_body_do() TIMEOUT - But uwsgi_read_timeout not helping (x-post)

Question

I keep getting this error on interfacing with Nginx:

uwsgi_response_write_body_do() TIMEOUT !!!
IOError: write error

It's generally working fine. This only happens when I test with 50+ concurrent connections hitting a method that sends down 200MB of data.

I've been searching all over but all I'm seeing advise to increase uwsgi_read_timeout :-(

(I asked on the uwsgi github page, but I'm thinking since it's probably not a bug it's better to ask here)

Here's how I'm launching uwsgi:

flask/bin/uwsgi -s 127.0.0.1:9001 --need-app --wsgi-file app.py --processes 1 --callable app --daemonize /opt/logs/KVAutobus-uwsgi_01.log
flask/bin/uwsgi -s 127.0.0.1:9002 --need-app --wsgi-file app.py --processes 1 --callable app --daemonize /opt/logs/KVAutobus-uwsgi_02.log
flask/bin/uwsgi -s 127.0.0.1:9003 --need-app --wsgi-file app.py --processes 1 --callable app --daemonize /opt/logs/KVAutobus-uwsgi_03.log
flask/bin/uwsgi -s 127.0.0.1:9004 --need-app --wsgi-file app.py --processes 1 --callable app --daemonize /opt/logs/KVAutobus-uwsgi_04.log
flask/bin/uwsgi -s 127.0.0.1:9005 --need-app --wsgi-file app.py --processes 1 --callable app --daemonize /opt/logs/KVAutobus-uwsgi_05.log
flask/bin/uwsgi -s 127.0.0.1:9006 --need-app --wsgi-file app.py --processes 1 --callable app --daemonize /opt/logs/KVAutobus-uwsgi_06.log
flask/bin/uwsgi -s 127.0.0.1:9007 --need-app --wsgi-file app.py --processes 1 --callable app --daemonize /opt/logs/KVAutobus-uwsgi_07.log
flask/bin/uwsgi -s 127.0.0.1:9008 --need-app --wsgi-file app.py --processes 1 --callable app --daemonize /opt/logs/KVAutobus-uwsgi_08.log
flask/bin/uwsgi -s 127.0.0.1:9009 --need-app --wsgi-file app.py --processes 1 --callable app --daemonize /opt/logs/KVAutobus-uwsgi_09.log
flask/bin/uwsgi -s 127.0.0.1:9010 --need-app --wsgi-file app.py --processes 1 --callable app --daemonize /opt/logs/KVAutobus-uwsgi_10.log
flask/bin/uwsgi -s 127.0.0.1:9011 --need-app --wsgi-file app.py --processes 1 --callable app --daemonize /opt/logs/KVAutobus-uwsgi_11.log
flask/bin/uwsgi -s 127.0.0.1:9012 --need-app --wsgi-file app.py --processes 1 --callable app --daemonize /opt/logs/KVAutobus-uwsgi_12.log
flask/bin/uwsgi -s 127.0.0.1:9013 --need-app --wsgi-file app.py --processes 1 --callable app --daemonize /opt/logs/KVAutobus-uwsgi_13.log
flask/bin/uwsgi -s 127.0.0.1:9014 --need-app --wsgi-file app.py --processes 1 --callable app --daemonize /opt/logs/KVAutobus-uwsgi_14.log
flask/bin/uwsgi -s 127.0.0.1:9015 --need-app --wsgi-file app.py --processes 1 --callable app --daemonize /opt/logs/KVAutobus-uwsgi_15.log
flask/bin/uwsgi -s 127.0.0.1:9016 --need-app --wsgi-file app.py --processes 1 --callable app --daemonize /opt/logs/KVAutobus-uwsgi_16.log
flask/bin/uwsgi -s 127.0.0.1:9017 --need-app --wsgi-file app.py --processes 1 --callable app --daemonize /opt/logs/KVAutobus-uwsgi_17.log
flask/bin/uwsgi -s 127.0.0.1:9018 --need-app --wsgi-file app.py --processes 1 --callable app --daemonize /opt/logs/KVAutobus-uwsgi_18.log
flask/bin/uwsgi -s 127.0.0.1:9019 --need-app --wsgi-file app.py --processes 1 --callable app --daemonize /opt/logs/KVAutobus-uwsgi_19.log
flask/bin/uwsgi -s 127.0.0.1:9020 --need-app --wsgi-file app.py --processes 1 --callable app --daemonize /opt/logs/KVAutobus-uwsgi_20.log

Here is /etc/nginx/nginx.conf:

user  nginx;
worker_processes  10;

error_log  /var/log/nginx/error.log warn;
pid        /var/run/nginx.pid;

events {
    worker_connections  2024;
}
http {
    include   /etc/nginx/mime.types;
    default_type  application/octet-stream;
    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"';

    access_log  /var/log/nginx/access.log  main;
    sendfile        on;
    #tcp_nopush     on;
    keepalive_timeout  20;
    gzip  on;
    include /etc/nginx/conf.d/*.conf;
}

Here is /etc/nginx/conf.d/default.conf:

upstream internal {
    least_conn;
    server 127.0.0.1:9001;
    server 127.0.0.1:9002;
    server 127.0.0.1:9003;
    server 127.0.0.1:9004;
    server 127.0.0.1:9005;
    server 127.0.0.1:9006;
    server 127.0.0.1:9007;
    server 127.0.0.1:9008;
    server 127.0.0.1:9009;
    server 127.0.0.1:9010;
    server 127.0.0.1:9011;
    server 127.0.0.1:9012;
    server 127.0.0.1:9013;
    server 127.0.0.1:9014;
    server 127.0.0.1:9015;
    server 127.0.0.1:9016;
    server 127.0.0.1:9017;
    server 127.0.0.1:9018;
    server 127.0.0.1:9019;
    server 127.0.0.1:9020;
}

server {

    listen   9000 default_server;
    server_name  _;
  proxy_max_temp_file_size 4024m;
  #access_log  logs/host.access.log  main;
  location / {
    uwsgi_pass         internal;

    uwsgi_param   Host                 $host;
    uwsgi_param   X-Real-IP            $remote_addr;
    uwsgi_param   X-Forwarded-For      $proxy_add_x_forwarded_for;
    uwsgi_param   X-Forwarded-Proto    $http_x_forwarded_proto;

    #not sure which timeout parameter I need
    proxy_read_timeout 600;
    proxy_connect_timeout 1d;
    proxy_max_temp_file_size 5024m;
    proxy_send_timeout 600;
    uwsgi_read_timeout 600;
    uwsgi_send_timeout 600;
    include uwsgi_params;
  }
}

Answer 1

Just use directive:

uwsgi_max_temp_file_size 0;

or

uwsgi_buffering off;

to disable buffer in Nginx.

I think the transmission speed from nginx to client is far slower to the speed from uwsgi to nginx. And nginx's buffer is full, so uwsgi wait much time before there are space for nginx to receive new data and this make uwsgi's write request timeout.

You can read my analysis about this problem in my blog: http://kmiku7.github.io/2018/02/02/Solve-the-problem-of-uwsgi-uwsgi-response-write-body-do-TIMEOUT-error/