Optimize Nginx + PHP-FPM for 5 million daily pageviews

Question

We run a few high volume websites which together generate around 5 million pageviews per day. We have the most overkill servers as we anticipate growth but we are having reports of a few active users saying the site is sometimes slow on the first pageview. I've seen this myself every once in a while where the first pageview will take 3-5 seconds then it's instant after that for the rest of the day. This has happened to me maybe twice in the last 24 hours so not enough to figure out what's happening. Every page on our site uses PHP but one of the times it happened to me it was on a PHP page that doesn't have any database calls which makes me think the issue is limited to NGINX, PHP-FPM or network settings.

We have 3 NGINX servers running behind a load balancer. Our database is separate on a cluster. I included our configuration files for nginx and php-fpm as well as our current RAM usage and PHP-FPM status. This is based on middle of the day (average traffic for us). Please take a look and let me know if you see any red flags in my setup or have any suggestions to optimize further.

Specs for each NGINX Server:

OS: CentOS 7
RAM: 128GB
CPU: 32 cores (2.4Ghz each)
Drives: 2xSSD on RAID 1

RAM Usage (free -g)

              total        used        free      shared  buff/cache   available
Mem:            125          15          10           3         100         103
Swap:            15           0          15

PHP-FPM status (IE: http://server1_ip/status)

pool:                 www
process manager:      dynamic
start time:           03/Mar/2016:03:42:49 -0800
start since:          1171262
accepted conn:        69827961
listen queue:         0
max listen queue:     0
listen queue len:     0
idle processes:       1670
active processes:     1
total processes:      1671
max active processes: 440
max children reached: 0
slow requests:        0

php-fpm config file:

[www]
user = nginx
group = nginx
listen = /var/opt/remi/php70/run/php-fpm/php-fpm.sock
listen.owner = nginx
listen.group = nginx
listen.mode = 0660
listen.allowed_clients = 127.0.0.1
pm = dynamic
pm.max_children = 6000
pm.start_servers = 1600
pm.min_spare_servers = 1500
pm.max_spare_servers = 2000
pm.max_requests = 1000
pm.status_path = /status
slowlog = /var/opt/remi/php70/log/php-fpm/www-slow.log
php_admin_value[error_log] = /var/opt/remi/php70/log/php-fpm/www-error.log
php_admin_flag[log_errors] = on
php_value[session.save_handler] = files
php_value[session.save_path]    = /var/opt/remi/php70/lib/php/session
php_value[soap.wsdl_cache_dir]  = /var/opt/remi/php70/lib/php/wsdlcache

nginx config file:

user nginx;
worker_processes 32;
error_log /var/log/nginx/error.log;
pid /run/nginx.pid;

events {
    worker_connections 1000;
    multi_accept        on;
    use                 epoll;
}

http {
    log_format  main  '$remote_addr - $remote_user [$time_iso8601] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"';

    access_log  /var/log/nginx/access.log  main;

    sendfile            on;
    tcp_nopush          on;
    tcp_nodelay         on;
    keepalive_timeout   10 10;
    send_timeout    60;
    types_hash_max_size 2048;

    client_max_body_size 50M;
    client_body_buffer_size 5m;
    client_body_timeout 60;
    client_header_timeout 60;

    fastcgi_buffers 256 16k;
    fastcgi_buffer_size 128k;
    fastcgi_connect_timeout 60s;
    fastcgi_send_timeout 60s;
    fastcgi_read_timeout 60s;
    fastcgi_busy_buffers_size 256k;
    fastcgi_temp_file_write_size 256k;
    reset_timedout_connection on;
    server_names_hash_bucket_size 100;


    #compression
    gzip  on;
    gzip_vary on;
    gzip_min_length 10240;
    gzip_proxied expired no-cache no-store private auth;
    gzip_types text/plain text/css text/xml text/javascript application/x-javascript application/javascript application/xml;
    gzip_disable "MSIE [1-6]\.";

    include             /etc/nginx/mime.types;
    default_type        application/octet-stream;

    # Load modular configuration files from the /etc/nginx/conf.d directory.
    # See http://nginx.org/en/docs/ngx_core_module.html#include
    # for more information.
    include /etc/nginx/conf.d/*.conf;

    server {
        listen       80 default_server;
        listen       [::]:80 default_server;
        server_name  domain1.com;
        root         /folderpath;

        location / {
            index index.php;
        }
        location = /favicon.ico { access_log off; log_not_found off; }
        location = /robots.txt  { access_log off; log_not_found off; }


    #server status
        location /server-status {
            stub_status on;
        access_log off;
            auth_basic "Restricted";
            auth_basic_user_file /etc/nginx/.htpasswd;
        }

    location = /status {
        access_log off;
        allow 127.0.0.1;
            auth_basic "Restricted";
            auth_basic_user_file /etc/nginx/.htpasswd;

            fastcgi_pass unix:/var/opt/remi/php70/run/php-fpm/php-fpm.sock;
            fastcgi_index index.php;
        fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
        include fastcgi_params;
    }


        location ~ \.php$ {
            try_files $uri =404;
            fastcgi_pass unix:/var/opt/remi/php70/run/php-fpm/php-fpm.sock;
            fastcgi_index index.php;
            fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
            include fastcgi_params;
        }

UPDATE:

I installed opcache as per the suggestion below. Not sure if it fixes the issue. Here are my settings

opcache.enable=1
opcache.memory_consumption=1024
opcache.interned_strings_buffer=64
opcache.max_accelerated_files=32531
opcache.max_wasted_percentage=10

Answer 1

2 minor tips:

• if you use opcache, monitor it to check if its configuration (especially memory size) is ok, and avoid OOM reset, you can use https://github.com/rlerdorf/opcache-status (a single php page)

• increase pm.max_requests to keep using same processes