Всем привет.
При обращении на адрес 185.XXX порт 80 IPVS прокидывает с помощью тунеля на реальный сервер 10.4.163.27 80 - это ок. Но при попытке загруить на сервер что-либо получаем дикие тормоза и потерю скорости. С сервером всё ок, т.к. если перевести метод балансировки из тунеля на тот же RR и дотянуть до сервера L2, то всё летает.
Может, конечно, это не связано с truncated, но появляется эта проблемма имено при методе балансировки ipvs ipip и только когда существует тунель
Помогите советом, что ещё проверить?
Схема физического подключения:
juniper(EX4550) > lb > juniper(EX4550) > new03 l2 l3 l3
Схема логики:
juniper > lb > new03
Все хосты - физические железки.
Балансировщик: тут ловим мелкие обрезки
lb03 ~ # uname -a Linux avi-lb03 3.16.0-4-amd64 #1 SMP Debian 3.16.39-1 (2016-12-30) x86_64 GNU/Linux
lb config (ipvsadm)
-a -t 185.XXX:http -r 10.4.163.27:http -i -w 2 -a -t 185.XXX:http -r 10.4.163.27:http -i -w 2
lb03 ~ # ifconfig eth0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500 inet 10.9.1.253 netmask 255.255.248.0 broadcast 10.9.7.255 inet6 fe80::5eb9:1ff:fe92:ffcc prefixlen 64 scopeid 0x20<link> ether 5c:b9:01:92:ff:cc txqueuelen 1000 (Ethernet) RX packets 175815337936 bytes 64833453222881 (58.9 TiB) RX errors 0 dropped 84 overruns 0 frame 0 TX packets 1824657003210 bytes 724757706679996 (659.1 TiB) TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
eth0:1: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500 inet 10.9.16.253 netmask 255.255.255.0 broadcast 10.9.16.255 ether 5c:b9:01:92:ff:cc txqueuelen 1000 (Ethernet)
eth1: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500 inet6 fe80::5eb9:1ff:fe92:ffcd prefixlen 64 scopeid 0x20<link> ether 5c:b9:01:92:ff:cd txqueuelen 1000 (Ethernet) RX packets 1712401922668 bytes 692260311029555 (629.6 TiB) RX errors 0 dropped 11400 overruns 0 frame 0 TX packets 47999816625 bytes 23135882842272 (21.0 TiB) TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
lo: flags=73<UP,LOOPBACK,RUNNING> mtu 65536 inet 127.0.0.1 netmask 255.0.0.0 inet6 ::1 prefixlen 128 scopeid 0x10<host> loop txqueuelen 0 (Local Loopback) RX packets 173413753 bytes 49631727710 (46.2 GiB) RX errors 0 dropped 0 overruns 0 frame 0 TX packets 173413753 bytes 49631727710 (46.2 GiB) TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
vlan215: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500 inet 185.XXX netmask 255.255.255.248 broadcast 185.XXX inet6 fe80::5eb9:1ff:fe92:ffcd prefixlen 64 scopeid 0x20<link> ether 5c:b9:01:92:ff:cd txqueuelen 0 (Ethernet) RX packets 1639385770882 bytes 663096397295899 (603.0 TiB) RX errors 0 dropped 0 overruns 0 frame 0 TX packets 39765924073 bytes 22597989815435 (20.5 TiB) TX errors 0 dropped 4159 overruns 0 carrier 0 collisions 0
lb03 ~ # ip a 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever inet 185.XXX/32 scope global lo valid_lft forever preferred_lft forever inet 185.XXX/32 scope global lo valid_lft forever preferred_lft forever inet6 ::1/128 scope host valid_lft forever preferred_lft forever 2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000 link/ether 5c:b9:01:92:ff:cc brd ff:ff:ff:ff:ff:ff inet 10.9.1.253/21 brd 10.9.7.255 scope global eth0 valid_lft forever preferred_lft forever inet 10.9.16.253/24 brd 10.9.16.255 scope global eth0:1 valid_lft forever preferred_lft forever inet6 fe80::5eb9:1ff:fe92:ffcc/64 scope link valid_lft forever preferred_lft forever 3: eth1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000 link/ether 5c:b9:01:92:ff:cd brd ff:ff:ff:ff:ff:ff inet6 fe80::5eb9:1ff:fe92:ffcd/64 scope link valid_lft forever preferred_lft forever 4: eth2: <BROADCAST,MULTICAST> mtu 1500 qdisc noop state DOWN group default qlen 1000 link/ether 94:57:a5:6a:0e:b0 brd ff:ff:ff:ff:ff:ff 5: eth3: <BROADCAST,MULTICAST> mtu 1500 qdisc noop state DOWN group default qlen 1000 link/ether 94:57:a5:6a:0e:b1 brd ff:ff:ff:ff:ff:ff 6: eth4: <BROADCAST,MULTICAST> mtu 1500 qdisc noop state DOWN group default qlen 1000 link/ether 94:57:a5:6a:0e:b2 brd ff:ff:ff:ff:ff:ff 7: eth5: <BROADCAST,MULTICAST> mtu 1500 qdisc noop state DOWN group default qlen 1000 link/ether 94:57:a5:6a:0e:b3 brd ff:ff:ff:ff:ff:ff
11: vlan215@eth1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default link/ether 5c:b9:01:92:ff:cd brd ff:ff:ff:ff:ff:ff inet 185.XXX/29 brd 185.XXX scope global vlan215 valid_lft forever preferred_lft forever inet6 fe80::5eb9:1ff:fe92:ffcd/64 scope link valid_lft forever preferred_lft forever
tcp дамп lb03
lb03 ~ # tcpdump -niany -vvv | grep trun tcpdump: listening on any, link-type LINUX_SLL (Linux cooked), capture size 262144 bytes 10.9.1.253 > 10.4.163.27: IP truncated-ip - 20 bytes missing! (tos 0x0, ttl 122, id 29533, offset 0, flags [none], proto TCP (6), length 1500) 10.9.1.253 > 10.4.163.27: IP truncated-ip - 20 bytes missing! (tos 0x0, ttl 119, id 31772, offset 0, flags [none], proto TCP (6), length 1500) 10.9.1.253 > 10.4.163.27: IP truncated-ip - 20 bytes missing! (tos 0x0, ttl 119, id 31773, offset 0, flags [none], proto TCP (6), length 1500) 10.9.1.253 > 10.4.163.27: IP truncated-ip - 20 bytes missing! (tos 0x0, ttl 120, id 24528, offset 0, flags [none], proto TCP (6), length 1500) 10.9.1.253 > 10.4.163.27: IP truncated-ip - 20 bytes missing! (tos 0x0, ttl 120, id 24529, offset 0, flags [none], proto TCP (6), length 1500) 10.9.1.253 > 10.4.163.27: IP truncated-ip - 12 bytes missing! (tos 0x0, ttl 120, id 14618, offset 0, flags [none], proto TCP (6), length 1492) 10.9.1.253 > 10.4.163.27: IP truncated-ip - 20 bytes missing! (tos 0x0, ttl 115, id 4343, offset 0, flags [none], proto TCP (6), length 1500) 10.9.1.253 > 10.4.163.27: IP truncated-ip - 12 bytes missing! (tos 0x0, ttl 119, id 16721, offset 0, flags [none], proto TCP (6), length 1492) 10.9.1.253 > 10.4.163.27: IP truncated-ip - 12 bytes missing! (tos 0x0, ttl 119, id 16810, offset 0, flags [none], proto TCP (6), length 1492) 10.9.1.253 > 10.4.163.27: IP truncated-ip - 20 bytes missing! (tos 0x0, ttl 53, id 23878, offset 0, flags [none], proto TCP (6), length 1500) 10.9.1.253 > 10.4.163.27: IP truncated-ip - 20 bytes missing! (tos 0x0, ttl 53, id 23890, offset 0, flags [none], proto TCP (6), length 1500) 10.9.1.253 > 10.4.163.27: IP truncated-ip - 20 bytes missing! (tos 0x0, ttl 53, id 23895, offset 0, flags [none], proto TCP (6), length 1500) 10.9.1.253 > 10.4.163.27: IP truncated-ip - 20 bytes missing! (tos 0x0, ttl 53, id 23896, offset 0, flags [none], proto TCP (6), length 1500) 10.9.1.253 > 10.4.163.27: IP truncated-ip - 20 bytes missing! (tos 0x0, ttl 53, id 23898, offset 0, flags [none], proto TCP (6), length 1500) 10.9.1.253 > 10.4.163.27: IP truncated-ip - 12 bytes missing! (tos 0x0, ttl 119, id 17114, offset 0, flags [none], proto TCP (6), length 1492) 10.9.1.253 > 10.4.163.27: IP truncated-ip - 12 bytes missing! (tos 0x0, ttl 119, id 17118, offset 0, flags [none], proto TCP (6), length 1492) ^C1318117 packets captured 1783685 packets received by filter 461240 packets dropped by kernel
lb03 ~ # tcpdump -niany | grep trun tcpdump: verbose output suppressed, use -v or -vv for full protocol decode listening on any, link-type LINUX_SLL (Linux cooked), capture size 262144 bytes 10:47:38.653944 IP 10.9.1.253 > 10.4.163.27: IP truncated-ip - 12 bytes missing! 92.101.72.154.50108 > 185.XXX.443: Flags [.], seq 419:1871, ack 4060, win 16409, length 1452 (ipip-proto-4) 10:47:38.822483 IP 10.9.1.253 > 10.4.163.27: IP truncated-ip - 20 bytes missing! 188.247.35.242.60655 > 185.XXX.443: Flags [.], seq 177:1625, ack 39, win 319, options [nop,nop,TS val 1182665413 ecr 251877125], length 1448 (ipip-proto-4) 10:47:38.825358 IP 10.9.1.253 > 10.4.163.27: IP truncated-ip - 12 bytes missing! 92.101.72.154.50108 > 185.XXX.443: Flags [.], seq 2522:3974, ack 4425, win 16688, length 1452 (ipip-proto-4) 10:47:38.966124 IP 10.9.1.253 > 10.4.163.27: IP truncated-ip - 20 bytes missing! 37.77.132.250.62732 > 185.XXX.443: Flags [.], seq 0:1460, ack 1, win 254, length 1460 (ipip-proto-4) 10:47:39.010502 IP 10.9.1.253 > 10.4.163.27: IP truncated-ip - 20 bytes missing! 178.210.50.3.12031 > 185.XXX.443: Flags [.], seq 0:1460, ack 1, win 256, length 1460 (ipip-proto-4) 10:47:39.320380 IP 10.9.1.253 > 10.4.163.27: IP truncated-ip - 12 bytes missing! 92.101.72.154.50108 > 185.XXX.443: Flags [.], seq 4667:6119, ack 5956, win 16698, length 1452 (ipip-proto-4) 10:47:39.331799 IP 10.9.1.253 > 10.4.163.27: IP truncated-ip - 20 bytes missing! 178.210.50.3.12031 > 185.XXX.443: Flags [.], seq 1460:2920, ack 1, win 256, length 1460 (ipip-proto-4)
lb03 ~ # tcpdump -nieth0:1 -vvv | grep trun tcpdump: listening on eth0:1, link-type EN10MB (Ethernet), capture size 262144 bytes 10.9.1.253 > 10.4.163.27: IP truncated-ip - 20 bytes missing! (tos 0x0, ttl 123, id 6771, offset 0, flags [none], proto TCP (6), length 1500) 10.9.1.253 > 10.4.163.27: IP truncated-ip - 20 bytes missing! (tos 0x0, ttl 121, id 27416, offset 0, flags [none], proto TCP (6), length 1500) 10.9.1.253 > 10.4.163.27: IP truncated-ip - 20 bytes missing! (tos 0x0, ttl 111, id 26992, offset 0, flags [none], proto TCP (6), length 1500) 10.9.1.253 > 10.4.163.27: IP truncated-ip - 20 bytes missing! (tos 0x0, ttl 111, id 26993, offset 0, flags [none], proto TCP (6), length 1500)
lb03 ~ # tcpdump -nieth0:1 | grep trun tcpdump: verbose output suppressed, use -v or -vv for full protocol decode listening on eth0:1, link-type EN10MB (Ethernet), capture size 262144 bytes 10:52:57.174286 IP 10.9.1.253 > 10.4.163.27: IP truncated-ip - 12 bytes missing! 88.147.173.138.1170 > 185.XXX.443: Flags [.], seq 1298085074:1298086514, ack 4043257371, win 388, options [nop,nop,TS val 2178414 ecr 251956636], length 1440 (ipip-proto-4) 10:52:57.360873 IP 10.9.1.253 > 10.4.163.27: IP truncated-ip - 12 bytes missing! 88.147.173.138.1170 > 185.XXX.443: Flags [.], seq 4320:5760, ack 1, win 388, options [nop,nop,TS val 2178434 ecr 251956794], length 1440 (ipip-proto-4) 10:52:57.362475 IP 10.9.1.253 > 10.4.163.27: IP truncated-ip - 12 bytes missing! 88.147.173.138.1170 > 185.XXX.443: Flags [.], seq 5760:7200, ack 1, win 388, options [nop,nop,TS val 2178434 ecr 251956794], length 1440 (ipip-proto-4) 10:52:57.594282 IP 10.9.1.253 > 10.4.163.27: IP truncated-ip - 20 bytes missing! 195.9.195.210.57463 > 185.XXX.443: Flags [.], seq 736:2196, ack 4060, win 252, length 1460 (ipip-proto-4) 10:52:57.601784 IP 10.9.1.253 > 10.4.163.27: IP truncated-ip - 12 bytes missing! 88.147.173.138.1170 > 185.XXX.443: Flags [.], seq 1440:2880, ack 1, win 388, options [nop,nop,TS val 2178458 ecr 251956841], length 1440 (ipip-proto-4)
lb03 ~ # tcpdump -nieth0 | grep trun tcpdump: verbose output suppressed, use -v or -vv for full protocol decode listening on eth0, link-type EN10MB (Ethernet), capture size 262144 bytes 10:54:06.017446 IP 10.9.1.253 > 10.4.163.27: IP truncated-ip - 20 bytes missing! 195.182.156.230.34003 > 185.XXX.443: Flags [.], seq 1192624548:1192626008, ack 1230894276, win 64240, length 1460 (ipip-proto-4) 10:54:06.258568 IP 10.9.1.253 > 10.4.163.27: IP truncated-ip - 12 bytes missing! 88.147.173.138.1170 > 185.XXX.443: Flags [.], seq 1298163126:1298164566, ack 4043258290, win 399, options [nop,nop,TS val 2185322 ecr 251973907], length 1440 (ipip-proto-4) 10:54:06.362252 IP 10.9.1.253 > 10.4.163.27: IP truncated-ip - 20 bytes missing! 5.19.9.88.3654 > 185.XXX.443: Flags [.], seq 1137065797:1137067257, ack 2385666710, win 68, length 1460 (ipip-proto-4) 10:54:06.375853 IP 10.9.1.253 > 10.4.163.27: IP truncated-ip - 20 bytes missing! 5.19.9.88.3654 > 185.XXX.443: Flags [.], seq 1460:2920, ack 1, win 68, length 1460 (ipip-proto-4) 10:54:06.606870 IP 10.9.1.253 > 10.4.163.27: IP truncated-ip - 20 bytes missing! 5.19.9.88.3654 > 185.XXX.443: Flags [.], seq 4272:5732, ack 2449, win 64, length 1460 (ipip-proto-4) 10:54:06.608130 IP 10.9.1.253 > 10.4.163.27: IP truncated-ip - 20 bytes missing! 5.19.9.88.3654 > 185.XXX.443: Flags [.], seq 5732:7192, ack 2449, win 64, length 1460 (ipip-proto-4)
new03 ~ # uname -a Linux avi-http-new03 4.9.0-4-amd64 #1 SMP Debian 4.9.65-3+deb9u1 (2017-12-23) x86_64 GNU/Linux
new03 ~ # modinfo i40e filename: /lib/modules/4.9.0-4-amd64/kernel/drivers/net/ethernet/intel/i40e/i40e.ko version: 1.6.16-k пробовали ставить последний драйвер - не помогло
iptables: -A OUTPUT -s 185.89.12.0/22 -p tcp -m tcp --tcp-flags SYN,RST SYN -j TCPMSS --set-mss 1300
new03 ~ # ifconfig enp129s0f0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500 inet 10.4.63.110 netmask 255.255.255.252 broadcast 10.4.63.111 ether 3c:fd:fe:b6:e7:60 txqueuelen 1000 (Ethernet) RX packets 783880208 bytes 361874163619 (337.0 GiB) RX errors 0 dropped 0 overruns 0 frame 0 TX packets 469095613 bytes 321672887375 (299.5 GiB) TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
enp129s0f1: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500 inet 10.4.67.110 netmask 255.255.255.252 broadcast 10.4.67.111 ether 3c:fd:fe:b6:e7:61 txqueuelen 1000 (Ethernet) RX packets 9461 bytes 855574 (835.5 KiB) RX errors 0 dropped 0 overruns 0 frame 0 TX packets 947154314 bytes 652926610391 (608.0 GiB) TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
lo: flags=73<UP,LOOPBACK,RUNNING> mtu 65536 inet 127.0.0.1 netmask 255.0.0.0 loop txqueuelen 1 (Local Loopback) RX packets 38419141 bytes 7436927667 (6.9 GiB) RX errors 0 dropped 0 overruns 0 frame 0 TX packets 38419141 bytes 7436927667 (6.9 GiB) TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
lo:1: flags=73<UP,LOOPBACK,RUNNING> mtu 65536 inet 10.4.163.27 netmask 255.255.255.255 loop txqueuelen 1 (Local Loopback)
tunl0: flags=193<UP,RUNNING,NOARP> mtu 1480 tunnel txqueuelen 1 (IPIP Tunnel) RX packets 38211932960 bytes 13952549386830 (12.6 TiB) RX errors 0 dropped 0 overruns 0 frame 0 TX packets 0 bytes 0 (0.0 B) TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
Сам сервис: Тут ловим в дампе большие обрезки:
new03 ~ # tcpdump -nienp129s0f0 -vvv | grep trun tcpdump: listening on enp129s0f0, link-type EN10MB (Ethernet), capture size 262144 bytes 10.9.1.252 > 10.4.163.27: IP truncated-ip - 354 bytes missing! (tos 0x0, ttl 122, id 51823, offset 0, flags [DF], proto TCP (6), length 1694) 10.9.1.252 > 10.4.163.27: IP truncated-ip - 1300 bytes missing! (tos 0x0, ttl 122, id 51823, offset 0, flags [DF], proto TCP (6), length 1694) 10.9.1.251 > 10.4.163.27: IP truncated-ip - 10 bytes missing! (tos 0x0, ttl 57, id 29339, offset 0, flags [DF], proto TCP (6), length 1350) 10.9.1.251 > 10.4.163.27: IP truncated-ip - 1300 bytes missing! (tos 0x0, ttl 57, id 29339, offset 0, flags [DF], proto TCP (6), length 1350) 10.9.1.251 > 10.4.163.27: IP truncated-ip - 3067 bytes missing! (tos 0x0, ttl 118, id 18112, offset 0, flags [DF], proto TCP (6), length 4407) 10.9.1.251 > 10.4.163.27: IP truncated-ip - 3067 bytes missing! (tos 0x0, ttl 118, id 18112, offset 0, flags [DF], proto TCP (6), length 4407) 10.9.1.251 > 10.4.163.27: IP truncated-ip - 3067 bytes missing! (tos 0x0, ttl 118, id 18112, offset 0, flags [DF], proto TCP (6), length 4407) 10.9.1.251 > 10.4.163.27: IP truncated-ip - 3900 bytes missing! (tos 0x0, ttl 118, id 18112, offset 0, flags [DF], proto TCP (6), length 4407)
new03 ~ # tcpdump -nienp129s0f1 -vvv | grep trun tcpdump: listening on enp129s0f1, link-type EN10MB (Ethernet), capture size 262144 bytes ^C55835 packets captured 55879 packets received by filter 0 packets dropped by kernel
new03 ~ # tcpdump -nitunl0 -vvv | grep trun tcpdump: listening on tunl0, link-type RAW (Raw IP), capture size 262144 bytes 10:37:57.945072 IP truncated-ip - 1236 bytes missing! (tos 0x0, ttl 122, id 27088, offset 0, flags [DF], proto TCP (6), length 2576) 10:37:57.945091 IP truncated-ip - 1300 bytes missing! (tos 0x0, ttl 122, id 27088, offset 0, flags [DF], proto TCP (6), length 2576) 10:37:57.946122 IP truncated-ip - 141 bytes missing! (tos 0x28, ttl 123, id 57273, offset 0, flags [DF], proto TCP (6), length 1481)
10:37:58.002995 IP truncated-ip - 1326 bytes missing! (tos 0x0, ttl 119, id 7803, offset 0, flags [DF], proto TCP (6), length 2666) 10:37:58.003022 IP truncated-ip - 1326 bytes missing! (tos 0x0, ttl 119, id 7803, offset 0, flags [DF], proto TCP (6), length 2666) 10:37:58.003022 IP truncated-ip - 2576 bytes missing! (tos 0x0, ttl 119, id 7803, offset 0, flags [DF], proto TCP (6), length 2666) 10:37:58.009579 IP truncated-ip - 5200 bytes missing! (tos 0x0, ttl 56, id 11362, offset 0, flags [DF], proto TCP (6), length 6540) 10:37:58.009605 IP truncated-ip - 5200 bytes missing! (tos 0x0, ttl 56, id 11362, offset 0, flags [DF], proto TCP (6), length 6540) 10:37:58.009606 IP truncated-ip - 5200 bytes missing! (tos 0x0, ttl 56, id 11362, offset 0, flags [DF], proto TCP (6), length 6540) 10:37:58.009607 IP truncated-ip - 5200 bytes missing! (tos 0x0, ttl 56, id 11362, offset 0, flags [DF], proto TCP (6), length 6540) 10:37:58.009607 IP truncated-ip - 5200 bytes missing! (tos 0x0, ttl 56, id 11362, offset 0, flags [DF], proto TCP (6), length 6540)
