๐ Linux ๆ่ฝ่ชฟๅช:้ๅฐ้ซไฝต็ผ Nginx ็ฐๅข็ Sysctl ๆ ธๅฟๅๆธๅชๅๅ จ็ด้
ไธ、้็ฏๅจ่งฃๆฑบไป้บผๅ้ก?(้ซไฝต็ผ Nginx ็ๅ ธๅ็ถ้ ธ)
Nginx ๆฌ่บซๅพ่ฝๆ,ไฝ「ๆๅพไฝ」ไธไปฃ่กจ「ๆๅพๆผไบฎ」。้ซไฝต็ผไธๅธธ่ฆๅก้ปไธๅจ Nginx,่ๅจ Linux ๆ ธๅฟ็่ณๆบไธ้่ TCP/IP ไฝๅ:
- ้ฃ็ทไฝๅๅคชๅฐ:accept queue / SYN backlog ๆไธไฝ,ๅบ็พไธๅ 、้ๅณ、ๆกๆๅปถ้ฒ。
- TIME_WAIT/็ญ้ฃ็ทๅฃๅ:ๅคง้็ญ้ฃ็ท้ ๆ port、socket、่จๆถ้ซ่ๅๆถๅฃๅ。
- socket buffer ไธๅค :ๅๅไธไธๅป、ๅปถ้ฒๆๅ、ๅฐๅณฐๆๆดๆ้กฏ。
- ๆชๆกๆ่ฟฐ็ฌฆไธๅค :็ดๆฅๅด
Too many open filesๆ้ฑๆงๆ้ฃ็ท。 - NIC/backlog ็ทฉ่กไธ่ถณ:ๅฐๅ ไธๅคๅฐฑๆ้、drop。
ไบ、ๅ ๅปบ็ซ「ๅบๆบ」:ไธ่ฆไธไธไพๅฐฑไบๆน
ๅปบ่ญฐๅ ๆ็พๆณๆไธไพ,ๅพ้ข่ชฟๆดๆ็ฅ้ๆนไบไป้บผ、ๆๆๆฒๆ。
# 1) ๆ ธๅฟ็ๆฌ / CPU / ่จๆถ้ซ / NIC
uname -a
lscpu
free -h
ip -s link
# 2) ็ฎๅ sysctl(ๆ้้ปๅณๅฏ)
sysctl net.core.somaxconn
sysctl net.core.netdev_max_backlog
sysctl net.ipv4.tcp_max_syn_backlog
sysctl net.ipv4.ip_local_port_range
sysctl net.ipv4.tcp_fin_timeout
sysctl net.ipv4.tcp_tw_reuse
sysctl net.ipv4.tcp_keepalive_time
sysctl net.ipv4.tcp_congestion_control
sysctl net.ipv4.tcp_fastopen
# 3) ้ฃ็ท่ socket ๆฆๆณ(ๅฐๅณฐๆๆๅฅฝไนๆไธๆฌก)
ss -s
cat /proc/net/sockstat
cat /proc/net/sockstat6
# 4) Nginx process ๆชๆกไธ้(ๅพ้่ฆ)
pidof nginx | tr ' ' '\n' | head -n 1 | xargs -I{} cat /proc/{}/limits | grep -i "Max open files"
ไธ、ๆจ่ฆ Sysctl ้ ็ฝฎ(้ซไฝต็ผ Web / Reverse Proxy ๅธธ็จ็ตๅ)
ไปฅไธๆไพไธไปฝ「ๅไฟๅฎ、ๅฏ็จๆผๅคๆธ้ซไฝต็ผ Nginx」็่ตทๆๅผ。ไธๆฏ่ถๅคง่ถๅฅฝ,่ๆฏ่ฆ่ทไฝ ็ๆต้ๅๆ (้ท้ฃ็ท/็ญ้ฃ็ท)、ไธๆธธๆๅ、ไปฅๅ NIC ่ฝๅๅน้ 。
# ๅปบ่ญฐๅปบ็ซ็จ็ซๆชๆก,ไพฟๆผๆง็่ๅๆปพ
sudo tee /etc/sysctl.d/99-nginx-highconcurrency.conf > /dev/null <<'EOF'
# =========================================================
# Nginx High Concurrency Sysctl Tuning
# ็ฎ็:ๆๅไฝๅๆฟ่ผ、้ไฝๅฐๅณฐ drop、ๆนๅ็ญ้ฃ็ทๅฃๅ
# ๆณจๆ:ๅฅ็จๅ่ซๅ
ๅปบ็ซๅบๆบไธฆๆบๅๅๆปพ
# =========================================================
# --- 1) ้ฃ็ทไฝๅ / ็ถฒๅก backlog ---
net.core.somaxconn = 4096
net.core.netdev_max_backlog = 16384
# --- 2) TCP ๆกๆ / SYN backlog(ๅ้ฃ็ท)---
net.ipv4.tcp_max_syn_backlog = 8192
net.ipv4.tcp_syncookies = 1
# --- 3) ephemeral ports(ๅคง้็ญ้ฃ็ท、ๅๅไปฃ็็นๅฅๅธธ่ฆ)---
net.ipv4.ip_local_port_range = 10240 65535
# --- 4) TIME_WAIT / ้ฃ็ทๅๆถ(ไฟๅฎ่จญๅฎ:ไธๆๅฎ็ถ้ญๆณ)---
# tcp_tw_reuse:ๅๅ client-side,ๅคง้็ญ้ฃ็ทๆๅฏ่ฝๆๅฉ,ไฝไธๆฏ้็จ่งฃๆณ
net.ipv4.tcp_tw_reuse = 0
net.ipv4.tcp_fin_timeout = 30
# --- 5) keepalive(้ฟๅ
ๆญป้ฃ็ทไฝ่ณๆบ;ไพๆๅๅๆ
่ชฟๆด)---
net.ipv4.tcp_keepalive_time = 600
net.ipv4.tcp_keepalive_intvl = 30
net.ipv4.tcp_keepalive_probes = 5
# --- 6) socket buffer(ๅๅ่ๅปถ้ฒ็ๅบ็ค็ค;ไธ่ฆ็ก่
ฆๆ็)---
net.core.rmem_max = 134217728
net.core.wmem_max = 134217728
net.ipv4.tcp_rmem = 4096 1048576 134217728
net.ipv4.tcp_wmem = 4096 1048576 134217728
# --- 7) ๅ
ถไปๅธธ่ฆ็ฉฉๅฎ้
(็ถญๆ้ ่จญๅณๅฏ,ไนๅฏๆ็ขบๅฏซๅบ)---
net.ipv4.tcp_timestamps = 1
net.ipv4.tcp_sack = 1
net.ipv4.tcp_window_scaling = 1
# --- 8) FQ + BBR(่ฅๆ ธๅฟ่็ผ่ก็ๆฏๆด,ๅฏ่ๆ
ฎ;ไธๅผทๅถ)---
# net.core.default_qdisc = fq
# net.ipv4.tcp_congestion_control = bbr
# --- 9) ๆชๆกๆ่ฟฐ็ฌฆ(็ณป็ตฑๅฑคไธ้;ไป้้
ๅ systemd / nginx ่จญๅฎ)---
fs.file-max = 2097152
EOF
# ๅฅ็จ
sudo sysctl --system
# ้ฉ่ญ
sysctl net.core.somaxconn
sysctl net.ipv4.tcp_max_syn_backlog
sysctl net.ipv4.ip_local_port_range
ๅ、ๆฏๅๅๆธๅฐๅบๅจ่ชฟไป้บผ?(็ฝ่ฉฑไฝไธๅซ็ณ)
1) net.core.somaxconn:accept queue ไธ้
็ถ้ฃ็ทๅฎๆไธๅไบคๆกๅพ,ๆ้ฒๅ
ฅ็ญๅพ
accept() ็ไฝๅ。้ๅไฝๅๅคชๅฐ,้ซๅณฐๅฐฑๆๆไธ้ฒๅป。Nginx ็ listen backlog ไธๆ้ซๆผ somaxconn,ๅฆๅไนๆ่ขซๆชๆท。
2) net.ipv4.tcp_max_syn_backlog:SYN backlog(ๅ้ฃ็ทไธ้)
ๆกๆๆชๅฎๆๅ็ๅ้ฃ็ทไฝๅ。ๅฐๅณฐๆ้ๅฐ SYN flood ๆ,้่ฃกๅคชๅฐๅฐฑๅฎนๆๆๆกๆ。tcp_syncookies=1 ๅฏไฝ็บๅฃๅไฟ่ญท,ไฝไธๅปบ่ญฐๆๅฎ็ถไฝ้ทๆไพ่ณด,ไป่ฆๆญ้
ๅ็็ไฝๅ่้ฒ่ญท。
3) net.core.netdev_max_backlog:NIC ๆถๅ
ๅฐ CPU ๅ็็ทฉ่ก
็ถฒๅก้ฒไพ็ๅฐๅ ,CPU ๅฟๆๆๅ ๆ้。้่ฃกๅๅฐๅฐฑๅฎนๆ drop,ๅฐๅณฐๆๅๆ่ฎๅคง。ๅธธ่ฆๅๆณๆฏ้ ๅไธญๆท็ถๆ ธ、RPS/RFS、ไปฅๅ NIC ring buffer ไธ่ตท็,ไฝ้็ฏๅ ่็ฆ sysctl。
4) net.ipv4.ip_local_port_range:ๆฌๆฉ่จๆ port ็ฏๅ
ๅๅไปฃ็/ไธๆธธ้ฃ็ทๅคๆ,Nginx ๆๅฐไธๆธธๅปบ็ซๅคง้ๆฌๆฉ้ฃ็ท,ๆ่ๆ ephemeral ports。็ฏๅๅคช็ชๆๅบ็พ「้ฃ็ท็ช็ถไธไธๅป」ๆๅถ็ผ้ฏ่ชค。ๆ็ฏๅๆๅคง้ๅธธๆฏไฝ้ขจ้ช้ซๆถ็็ไธๆญฅ。
5) tcp_fin_timeout / tcp_tw_reuse:็ญ้ฃ็ท็ๅฐ้ทๅ
ๅพๅคไบบ็ๅฐ TIME_WAIT ๅฐฑๆณ「ๆถๆป
ๅฎ」,ไฝ TIME_WAIT ๆฏ TCP ๆญฃๅธธๆฉๅถ็ไธ้จๅ。
ๆฌ็ฏๆกไฟๅฎ็ญ็ฅ:ๅ
ไธๆ tcp_tw_reuse ็ถๆ้็จ่งฃๆณ,ๅชๅๆ้็ๅๆถ้ๅบฆ่ชฟๆด(ไพๅฆ tcp_fin_timeout)。ๅฆๆไฝ ็ๆ
ๅขๆฏ「ๅคง้ client-side ็ญ้ฃ็ท」ๆ「loopback ๅฐๅ
」็ญ็นๆฎๆกไพ,ๅๅฆๅคๅ A/B ้ฉ่ญๆๆฏ่ผๅฎๅ
จ。
6) socket buffer:ๅๅ็ๅบๅบง,ไฝไนๅฏ่ฝๅๅ ่จๆถ้ซ
ๆ rmem/wmem ๆ้ซ,ๆๅฉๆผ้ซๅๅ、้ท RTT ๆ็ช็ผๆต้,ไฝไนๆๅขๅ ่จๆถ้ซๅฃๅ。ๅปบ่ญฐ็จ็ฃๆง็ socket memory、TCP retrans、ไปฅๅ p99 latency,ๅ้ๆญฅ่ชฟๆด,ไธ่ฆไธๆฌกๆๅฐ่ชๅผต。
ไบ、ไธ่ฆๅช่ชฟ Sysctl:Nginx ่ systemd ๅฟ ้ ๅๆญฅ
Sysctl ๆ「้่ทฏๆๅฏฌ」ไบ,ไฝ Nginx ้ๅฐ่ปๅฆๆ้้(nofile / worker_connections)ๆฒ่งฃ้,ๆๆๆ็ดๆฅ่ขซๅกๆญป。
1) systemd:ๆๅ Nginx ๆๅ็ nofile ไธ้
# ๅปบ็ซ override(ไธๅ็ผ่ก็ nginx.service ไฝ็ฝฎไธๅ,ไฝๅๆณไธ่ด)
sudo systemctl edit nginx.service
# ๅ ไธ:
# [Service]
# LimitNOFILE=1048576
sudo systemctl daemon-reload
sudo systemctl restart nginx
# ้ฉ่ญ
pidof nginx | tr ' ' '\n' | head -n 1 | xargs -I{} cat /proc/{}/limits | grep -i "Max open files"
2) Nginx:worker ้ฃ็ท่ๆชๆกๆ่ฟฐ็ฌฆ
# /etc/nginx/nginx.conf(็คบๆ)
worker_processes auto;
worker_rlimit_nofile 1048576;
events {
worker_connections 65535;
multi_accept on;
use epoll;
}
# ่ฅไฝ ๆๆ็ขบ้ซๅณฐๆ้ๅ้ก,ๅฏ่ๆ
ฎๆ็ขบๆๅฎ backlog(้่ somaxconn ๅฐ้ฝ)
# server {
# listen 443 ssl http2 backlog=4096;
# }
ๅ ญ、้ฉ่ญๆฏๅฆ็็「่ฎๅฟซไบ」:็้ไบๅฐฑๅค ็จ
่ชฟๆ กๆๅ้ๅธธไธๆฏ「CPU ้ๅพๅค」้้บผ็ด่ฆบ,่ๆฏๅฐๅณฐๆๅ ไธ้、ๆกๆๆด็ฉฉ、p95/p99 latency ๆถๆ、้ฃ็ท้ฏ่ชคไธ้。
# 1) ้ฃ็ท็ธฝ่ฆฝ
ss -s
# 2) SYN / listen overflow / drop(ๆไบ่จๆธๅฏๅพๅ
งๆ ธ็ตฑ่จ่งๅฏ)
netstat -s | egrep -i "listen|overflow|drop|retrans|syn"
# 3) socket ่ณๆบ
cat /proc/net/sockstat
cat /proc/net/sockstat6
# 4) Nginx ็ๆ
(่ฅไฝ ๆๅ็จ stub_status)
# curl -s http://127.0.0.1/nginx_status
# 5) ๅฃๆธฌ(็ฏไพ:่ซไพไฝ ็ฐๅขๆๅทฅๅ
ท)
# wrk -t8 -c2000 -d60s https://your-domain/
# hey -z 60s -c 2000 https://your-domain/
ไธ、ๅๆปพ็ญ็ฅ(ไธๅฎ่ฆๅ ๆบๅ)
ไธๆฆ็ผ็พ็ฐๅธธ(ไพๅฆ้ฃ็ท่ก็บๆน่ฎ、ไธๆธธ timeout、p99 ่ฎๅทฎ),ๅๆปพ่ฆ่ฝๅจ 1 ๅ้ๅ งๅฎๆ。
# 1) ็ดๆฅ็งป้ค้
็ฝฎๆชไธฆ้่ผ sysctl
sudo mv /etc/sysctl.d/99-nginx-highconcurrency.conf /root/99-nginx-highconcurrency.conf.bak
sudo sysctl --system
# 2) ๆๅชๅ้ๅฎไธๅๆธ(็ซๅณ็ๆไฝไธๆไน
)
sudo sysctl -w net.core.somaxconn=128
# 3) ็ขบ่ชๅผ
sysctl net.core.somaxconn
ๅ ซ、ๅธธ่ฆ้ท้ฑ่ๅๅฏฆๅปบ่ญฐ
- 「ๅผ่ถๅคง่ถๅฅฝ」ๆฏ้ฏ็:ไฝๅ่ buffer ๆๅ่ณๆบ,ๅฐๅณฐๅฅฝ็、ๅธธๆ ่ฎๆ ขไนไธๆๅค。
- TIME_WAIT ไธ็ญๆผๅ้ก:ๅฎๅธธๆฏ「ไฝ ็็ๅพๅฟ」็ๅฏไฝ็จ;ๅ ็ขบ่ชๆฏๅฆๆฏ็ญ้ฃ็ท้ๅค、Keep-Alive ่จญ่จไธๅ็。
- ๅๅไปฃ็่ฆ็ ephemeral ports:ไธๆธธ้ฃ็ทๅคๆ,
ip_local_port_rangeๅพๅพๆฏไฝ ๆณๅพๆด้่ฆ。 - Gateway、DNS、ไธๆธธ RTT:็ถฒ่ทฏ่ทฏๅพ(Gateway)、DNS ๅๆ่ไธๆธธๅปถ้ฒ,ๅธธๅธธๆๆฏ p99 ็็ๆญฃๅ ๆ,ๅฅๅช็ฏ sysctl。
- ๆๆๆ็ๆนๅผ:ๅ ๅปบ็ซๅบๆบ → ๅฐๆญฅๆนๅ → ็ๆๆจ → ๅๆฑบๅฎ่ฆไธ่ฆๆพๅคง。
- ๆกๆๅฐๅณฐ:SYN backlog / accept queue overflow?
- ็ญ้ฃ็ท:TIME_WAIT ๅคชๅค、ephemeral ports ไธๅค ?
- ๅๅ:socket buffer、retrans、p99 latency ๆๅ?
ไฝ ๅฏไปฅๅจ็่จ่ฒผไธ ss -s、netstat -s ็้้ต็ๆฎต(้ฎ่ฝ็ถฒๅ/IP ๅณๅฏ),ๆๆ็จ「ๅฏๅๆปพ」็ๆนๅผๅนซไฝ ๆ่ชฟๆ ก่ทฏ็ทๆถๆๅฐๆๅฐๆนๅ、ๆๅคงๆถ็。
ๅปถไผธ้ฑ่ฎ
- ⚙️ Linux ็ณป็ตฑๆไฝณๅ:sysctl ่ I/O ่ชฟๆ กๅฏฆๆฐ
- ๐ Linux Nginx ๅๅไปฃ็ๅฏฆๆฐ:HTTPS、่ฒ ่ผๅนณ่กก่ๅพ็ซฏๆๅๆดๅ
- ๐งฉ Nginx ๅๅไปฃ็้ฒ้:ๅค็ซๅฐ、Cache、Rewrite ๅฏฆๆฐๆๅ
- ๐ GoAccess ็ถฒ็ซๆต้็ฃๆงๆๅญธ:Nginx/Apache ๆฅ่ชๅณๆๅๅฝขๅ
- ๐ก ็ฐกๅฎไบ่งฃ TCP/IP:่ฎ้ป่ ฆ่ฝไบ็ธๆบ้็้้ตๆ่ก
ๆฒๆ็่จ:
ๅผต่ฒผ็่จ