Nginx, Uwsgi, Python - 502 Bad Gateway - Prematurely Closed Upstream Connection
I tried to figure out what was going on by trying to find solutions for the related quests, but I had no success. I am trying to export some json to xml and it goes smoothly when using not much data (no problem with 2000 lines sample). But when I try to make the same request for a larger file (say over 25000 lines), nginx always gives me 502 Bad Gateway in exactly one minute. All the timeouts in my nginx config file were set to 60 seconds and I raised the whole thing to 120 seconds to see if something changed, but still got the same 502 after one minute. Can anyone please help?
This is how I run uwsgi:
command = / usr / sbin / uwsgi --buffer-size = 320000 --http 0.0.0.0:9090 --wsgi file /usr/local/cengine/bin/adapter.py --master - processes 50 - max -requests 50 --harakiri 120 - threads 1 --disable-logging --stats 127.0.0.1:9190 --logto / var / log / uwsgi / uwsgi.log
And here is my nginx config:
#################################################
# #
# By Puppet #
# #
#################################################
pid /var/run/nginx.pid;
user nginx nginx;
daemon off;
worker_processes 4;
events {
worker_connections 1024;
}
http {
#set_real_ip_from xx.xxx.xxx.0/24;
#real_ip_header X-Forwarded-For;
## MIME types
include /etc/nginx/mime.types;
default_type application/octet-stream;
## Size Limits
client_body_buffer_size 256K;
client_header_buffer_size 256K;
client_max_body_size 20M;
large_client_header_buffers 8 24k;
## Timeouts
client_body_timeout 120;
client_header_timeout 120;
send_timeout 120;
## General Options
ignore_invalid_headers on;
keepalive_requests 100;
keepalive_timeout 60;
recursive_error_pages on;
sendfile on;
server_name_in_redirect off;
server_tokens off;
## TCP options
tcp_nodelay on;
tcp_nopush on;
## Compression
gzip on;
gzip_buffers 24 16k;
gzip_comp_level 8;
gzip_http_version 1.0;
gzip_min_length 0;
gzip_types application/x-javascript application/json text/xml text/plain text/css image/x-icon application/x-perl application/x-httpd-cgi application/xml;
gzip_vary on;
gzip_proxied any;
## Cache setings
proxy_max_temp_file_size 0;
proxy_connect_timeout 120s;
proxy_send_timeout 120s;
proxy_read_timeout 120s;
proxy_buffer_size 8k;
proxy_buffers 8 64k;
proxy_busy_buffers_size 128k;
proxy_temp_file_write_size 128k;
proxy_intercept_errors on;
proxy_cache_path /var/cache/nginx levels=1:2 keys_zone=webcache:1000m inactive=1y max_size=10000m;
proxy_temp_path /var/cache/nginx/tmp;
proxy_cache_min_uses 1;
log_format main '$remote_addr $host [$time_local] Upstream-cache-status: $upstream_cache_status Cengine-UUID: $upstream_http_x_ceng_uuid "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" "$http_x_forwarded_for" time=$request_time';
log_format cache '$time_local $upstream_cache_status Cache-Control: $upstream_http_cache_control Expires: $upstream_http_expires "$request" ($status) "$http_user_agent"';
upstream uwsgi {
server 127.0.0.1:9090 max_fails=3 fail_timeout=120s;
keepalive 20;
}
server {
server_name xxxxx.tpn.xxxxx.com;
access_log /var/log/nginx/access.log main;
access_log /var/log/nginx/cache.log cache;
error_log /var/log/nginx/error.log error;
listen 0.0.0.0:80;
expires off;
location / {
alias /usr/local/getec-fe/;
index index.html;
}
location /files {
internal;
alias /anexos;
}
location /cengine {
# regras rewrite
rewrite ^/(.*)/$ /$1;
rewrite ^/cengine/(.*) /$1 break;
proxy_redirect off;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
#proxy_hide_header "Set-Cookie";
#proxy_ignore_headers X-Accel-Expires Expires Cache-Control Set-Cookie;
proxy_ignore_headers X-Accel-Expires Expires Cache-Control;
add_header X-Cache-Status $upstream_cache_status;
#proxy_cookie_path /;
proxy_pass_header Set-Cookie;
add_header "Cache-Control" "max-age=0, no-cache, no-store, must-revalidate";
proxy_pass http://uwsgi;
#proxy_cache webcache;
proxy_cache off;
proxy_cache_key $scheme$host$request_uri;
proxy_cache_valid any 15s;
proxy_cache_use_stale error timeout updating http_500 http_502 http_503 http_504;
}
location ~ ^/upload(?<username>/.+)?$ {
if ( $username = '' ) {
set $username /api;
}
client_body_temp_path /tmp;
client_body_in_file_only clean;
client_body_buffer_size 128K;
client_max_body_size 20M;
proxy_pass_request_headers on;
proxy_set_header X-FILE $request_body_file;
proxy_set_body off;
proxy_redirect off;
proxy_pass http://uwsgi/getec$username/file;
proxy_set_header Connection "";
default_type mime-type;
}
location ~ ^/~(.+?)(/.*)?$ {
alias /home/$1/public_html/projetos$2;
index index.html;
autoindex on;
#try_files $uri $uri/ /index.html =404;
}
location /csv {
alias /tmp/csv;
add_header Content-Type application/octet-stream;
add_header Content-Disposition "attachment; filename=\"$arg_filename\"";
}
location /status {
stub_status on;
access_log off;
allow xxx.xx.xxx.0/24;
allow xxx.xxx.xx.0/24;
allow xxx.xxx.xx.0/24;
allow 98.142.228.0/24;
deny all;
}
location /health_check {
return 200;
access_log off;
}
}
}
Here is my log for ngnix:
2015/04/23 13:21:04 [error] 12335#0: *188 upstream prematurely closed connection while reading response header from upstream, client: xx.xxx.x.xx, server: xxx.tpn.xxxx.com, request: "GET /cengine/getec/lucian.thomaz/exporta_csv?tabela=arquivamento&sEcho=1&iColumns=6&sColumns=%2C%2C%2C%2C%2C&iDisplayStart=0&iDisplayLength=25&mDataProp_0=ID_ARQUIVAMENTO&mDataProp_1=ID_CONTRATO&mDataProp_2=TP_DOCUMENTO&mDataProp_3=NM_CLIENTE_FORNECEDOR&mDataProp_4=DT_ASSINATURA&mDataProp_5=DS_AREA_RESPONSAVEL&sFilters={%22F_ID_ARQUIVAMENTO%22:{%22Model%22:%22Arquivamento%22,%22Coluna%22:%22ID_ARQUIVAMENTO%22,%22Operador%22:%22eq%22,%22Valor%22:null},%22F_ID_CONTRATO%22:{%22Model%22:%22Arquivamento%22,%22Coluna%22:%22ID_CONTRATO%22,%22Operador%22:%22eq%22,%22Valor%22:null},%22F_TP_DOCUMENTO%22:{%22Model%22:%22Arquivamento%22,%22Coluna%22:%22TP_DOCUMENTO%22,%22Operador%22:%22like%22,%22Valor%22:null},%22F_NM_CLIENTE_FORNECEDOR%22:{%22Model%22:%22Arquivamento%22,%22Coluna%22:%22NM_CLIENTE_FORNECEDOR%22,%22Operador%22:%22like%22,%22Valor%22:null},%22F_NR_CNPJ%22:{%22Model%22:%22Arquivamento%22,%22Coluna%22:%22NR_CNPJ%22,%22Operador%22:%22like%22,%22Valor%22:null},%22F_NR_CPF%22:{%22Model%22:%22Arquivamento%22,%22Coluna%22:%22NR_CPF%22,%22Operador%22:%22like%22,%22Valor%22:null},%22F_DT_ASSINATURA%22:{%22Model%22:%22Arquivamento%22,%22Coluna%22:%22DT_ASSINATURA%22,%22Operador%22:%22having%22,%22Valor_ini%22:null,%22Valor_fin%22:null},%22F_DS_VALOR%22:{%22Model%22:%22Arquivamento%22,%22Coluna%22:%22DS_VALOR%22,%22Operador%22:%22like%22,%22Valor%22:null},%22F_DS_AREA_RESPONSAVEL%22:{%22Model%22:%22Arquivamento%22,%22Coluna%22:%22DS_AREA_RESPONSAVEL%22,%22Operador%22:%22eq%22,%22Valor%22:null},%22F_BL_ATIVO%22:{%22Model%22:%22Arquivamento%22,%22Coluna%22:%22BL_ATIVO%22,%22Operador%22:%22eq%22,%22Valor%22:%221%22}} HTTP/1.1", upstream: "http://127.0.0.1:9090/getec/lucian.thomaz/exporta_csv?tabela=arquivamento&sEcho=1&iColumns=6&sColumns=%2C%2C%2C%2C%2C&iDisplayStart=0&iDisplayLength=25&mDataProp_0=ID_ARQUIVAMENTO&mDataProp_1=
Here is the log for uwsgi:
HARAKIRI: --- uWSGI worker 50 (pid: 12758) WAS managing request /getec/lucian.thomaz/exporta_csv?tabela=arquivamento&sEcho=1&iColumns=6&sColumns=%2C%2C%2C%2C%2C&iDisplayStart=0&iDisplayLength=25&mDataProp_0=ID_ARQUIVAMENTO&mDataProp_1=ID_CONTRATO&mDataProp_2=TP_DOCUMENTO&mDataProp_3=NM_CLIENTE_FORNECEDOR&mDataProp_4=DT_ASSINATURA&mDataProp_5=DS_AREA_RESPONSAVEL&sFilters={%22F_ID_ARQUIVAMENTO%22:{%22Model%22:%22Arquivamento%22,%22Coluna%22:%22ID_ARQUIVAMENTO%22,%22Operador%22:%22eq%22,%22Valor%22:null},%22F_ID_CONTRATO%22:{%22Model%22:%22Arquivamento%22,%22Coluna%22:%22ID_CONTRATO%22,%22Operador%22:%22eq%22,%22Valor%22:null},%22F_TP_DOCUMENTO%22:{%22Model%22:%22Arquivamento%22,%22Coluna%22:%22TP_DOCUMENTO%22,%22Operador%22:%22like%22,%22Valor%22:null},%22F_NM_CLIENTE_FORNECEDOR%22:{%22Model%22:%22Arquivamento%22,%22Coluna%22:%22NM_CLIENTE_FORNECEDOR%22,%22Operador%22:%22like%22,%22Valor%22:null},%22F_NR_CNPJ%22:{%22Model%22:%22Arquivamento%22,%22Coluna%22:%22NR_CNPJ%22,%22Operador%22:%22like%22,%22Valor%22:null},%22F_NR_CPF%22:{%22Model%22:%22Arquivamento%22,%22Coluna%22:%22NR_CPF%22,%22Operador%22:%22like%22,%22Valor%22:null},%22F_DT_ASSINATURA%22:{%22Model%22:%22Arquivamento%22,%22Coluna%22:%22DT_ASSINATURA%22,%22Operador%22:%22having%22,%22Valor_ini%22:null,%22Valor_fin%22:null},%22F_DS_VALOR%22:{%22Model%22:%22Arquivamento%22,%22Coluna%22:%22DS_VALOR%22,%22Operador%22:%22like%22,%22Valor%22:null},%22F_DS_AREA_RESPONSAVEL%22:{%22Model%22:%22Arquivamento%22,%22Coluna%22:%22DS_AREA_RESPONSAVEL%22,%22Operador%22:%22eq%22,%22Valor%22:null},%22F_BL_ATIVO%22:{%22Model%22:%22Arquivamento%22,%22Coluna%22:%22BL_ATIVO%22,%22Operador%22:%22eq%22,%22Valor%22:%221%22}} since Thu Apr 23 13:20:04 2015 ---
*** HARAKIRI ON WORKER 50 (pid: 12758, try: 2) ***
DAMN ! worker 50 (pid: 12758) died, killed by signal 9 :( trying respawn ...
+3
source to share
No one has answered this question yet
Check out similar questions: