Set Linux Configuration with Fabric (set.py)
Performance survey of seconds interval on Cron
LANG=C
LOGDIR=/var/log/performance
# 1H = 60m = 60m x 1m = 60m x 60sec = 3600sec
# 3600sec = 1sec x 3600 times
# 3600sec = 5sec x 720 times
# 3600sec = 10sec x 360 times
#-------------------------------------------------------------------
#daily output
#-------------------------------------------------------------------
# Multi
0 * * * * root vmstat 5 720 | awk '{print strftime("\%y/\%m/\%d \%H:\%M:\%S"), $0} { system(":") }' >> ${LOGDIR}/vmstat.`date +\%Y\%m\%d`
#0 * * * * root dstat -taf 5 720 >> ${LOGDIR}/dstat-taf.`date +\%Y\%m\%d`
# CPU Details
0 * * * * root mpstat 5 720 >> ${DIR}/mpstat.`date +\%Y\%m\%d`
# Load Average Details
0 * * * * root sar -q 5 720 >> ${LOGDIR}/sar-q.`date +\%Y\%m\%d`
# Memory
0 * * * * root sar -r 5 720 >> ${LOGDIR}/sar-r.`date +\%Y\%m\%d`
# Disk Details
0 * * * * root iostat -xtm 10 360 >> ${LOGDIR}/iostat.`date +\%Y\%m\%d` > /dev/null 2>&1
# Network Details
0 * * * * root sar -n DEV 5 720 >> ${LOGDIR}/sar-n-DEV.`date +\%Y\%m\%d`
* * * * * root /home/fasthandle/scripts/netstat-s.sh > /dev/null 2>&1
* * * * * root /home/fasthandle/scripts/netstat-i.sh > /dev/null 2>&1
# Process Details
* * * * * root /home/fasthandle/scripts/ps.sh > /dev/null 2>&1
* * * * * root /home/fasthandle/scripts/top.sh > /dev/null 2>&
#-------------------------------------------------------------------
# monthly output
#-------------------------------------------------------------------
# Disk Details
0 0,6,12,18 * * * root /home/fasthandle/scripts/df.sh > /dev/null 2>&1
#-------------------------------------------------------------------
# gzip/rm output file
#-------------------------------------------------------------------
5 0 * * * root find ${LOGDIR} -type f -not -name "df*" -mtime +30 -exec rm -rf {} \;
10 0 * * * root gzip ${LOGDIR}/*.`date -d '1day ago' +\%Y\%m\%d`
10 0 1 * * root gzip ${LOGDIR}/df.`date -d '1month ago' +\%Y\%m`
# man 5 crontab
A “%” character in the command, unless escaped with a backslash (\), will be changed into newline characters, and all data after the first % will be sent to the command as standard input.
memory, swap, io, system, cpu
~]# vmstat 1 3 | awk '{print strftime("%y/%m/%d %H:%M:%S"), $0} { system(":") }'
17/11/29 00:18:29 procs -----------memory---------- ---swap-- -----io---- -system-- ------cpu-----
17/11/29 00:18:29 r b swpd free buff cache si so bi bo in cs us sy id wa st
17/11/29 00:18:29 1 0 0 270248 1492 251820 0 0 85 36 64 117 0 0 99 0 0
17/11/29 00:18:30 0 0 0 270092 1492 251852 0 0 0 4 149 265 0 0 100 0 0
17/11/29 00:18:31 1 0 0 270092 1492 251852 0 0 0 21 139 236 0 0 100 0 0
~]# dstat -taf 2 5
----system---- -------cpu0-usage--------------cpu1-usage------ --dsk/sda-- -net/enp0s3 ---paging-- ---system--
time |usr sys idl wai hiq siq:usr sys idl wai hiq siq| read writ| recv send| in out | int csw
01-12 00:52:35| 0 0 99 0 0 0: 0 0 97 3 0 0| 198k 70k| 0 0 | 0 0 | 133 243
01-12 00:52:37| 0 0 100 0 0 0: 0 1 99 1 0 0| 0 511k| 60B 854B| 0 0 | 120 235
01-12 00:52:39| 0 0 100 0 0 0: 0 0 100 0 0 0| 0 4096B| 60B 494B| 0 0 | 112 203
01-12 00:52:41| 0 0 100 0 0 0: 0 0 95 5 0 0| 0 4096B| 60B 494B| 0 0 | 119 203
01-12 00:52:43| 1 0 100 0 0 0: 0 0 100 0 0 0| 0 0 | 148B 486B| 0 0 | 110 203
01-12 00:52:45| 0 0 100 0 0 0: 0 0 100 0 0 0| 0 2048B| 60B 486B| 0 0 | 102 175
CPU
~]# mpstat -P ALL 1 3 Linux 3.10.0-229.el7.x86_64 (centos7) 11/29/17 _x86_64_ (2 CPU) 00:17:23 CPU %usr %nice %sys %iowait %irq %soft %steal %guest %gnice %idle 00:17:24 all 0.00 0.00 0.00 0.50 0.00 0.00 0.00 0.00 0.00 99.50 00:17:24 0 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 100.00 00:17:24 1 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 100.00 00:17:24 CPU %usr %nice %sys %iowait %irq %soft %steal %guest %gnice %idle 00:17:25 all 0.00 0.00 0.50 0.00 0.00 0.00 0.00 0.00 0.00 99.50 00:17:25 0 0.00 0.00 0.00 0.00 0.00 1.00 0.00 0.00 0.00 99.00 00:17:25 1 0.00 0.00 0.00 1.00 0.00 0.00 0.00 0.00 0.00 99.00 00:17:25 CPU %usr %nice %sys %iowait %irq %soft %steal %guest %gnice %idle 00:17:26 all 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 100.00 00:17:26 0 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 100.00 00:17:26 1 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 100.00 Average: CPU %usr %nice %sys %iowait %irq %soft %steal %guest %gnice %idle Average: all 0.00 0.00 0.17 0.17 0.00 0.00 0.00 0.00 0.00 99.66 Average: 0 0.00 0.00 0.00 0.00 0.00 0.33 0.00 0.00 0.00 99.67 Average: 1 0.00 0.00 0.00 0.34 0.00 0.00 0.00 0.00 0.00 99.66
~]# sar -q 2 3 Linux 3.10.0-229.el7.x86_64 (centos7) 01/15/18 _x86_64_ (2 CPU) 01:43:05 runq-sz plist-sz ldavg-1 ldavg-5 ldavg-15 blocked 01:43:07 0 219 0.00 0.01 0.05 0 01:43:09 1 219 0.00 0.01 0.05 0 01:43:11 0 219 0.00 0.01 0.05 0 Average: 0 219 0.00 0.01 0.05 0
~]# sar -r 2 3 Linux 3.10.0-229.el7.x86_64 (centos7) 01/15/18 _x86_64_ (2 CPU) 01:43:26 kbmemfree kbmemused %memused kbbuffers kbcached kbcommit %commit kbactive kbinact kbdirty 01:43:28 249240 509760 67.16 1492 230356 1465816 91.69 247440 185552 0 01:43:30 249240 509760 67.16 1492 230356 1465816 91.69 247444 185552 0 01:43:32 249240 509760 67.16 1492 230360 1465816 91.69 247444 185556 0 Average: 249240 509760 67.16 1492 230357 1465816 91.69 247443 185553 0
IO
~]# iostat -xtm 1 2
Linux 3.10.0-229.el7.x86_64 (centos7) 11/29/17 _x86_64_ (2 CPU)
11/29/17 00:20:58
avg-cpu: %user %nice %system %iowait %steal %idle
0.11 0.00 0.17 0.31 0.00 99.41
Device: rrqm/s wrqm/s r/s w/s rMB/s wMB/s avgrq-sz avgqu-sz await r_await w_await svctm %util
sda 0.45 0.11 3.16 5.10 0.15 0.07 53.91 0.02 2.36 3.68 1.54 0.93 0.77
dm-0 0.00 0.00 3.12 4.35 0.14 0.07 58.00 0.02 2.61 3.70 1.83 1.02 0.76
dm-1 0.00 0.00 0.15 0.00 0.00 0.00 8.00 0.00 0.62 0.62 0.00 0.35 0.01
dm-2 0.00 0.00 0.02 0.00 0.00 0.00 8.00 0.00 0.25 0.25 0.00 0.25 0.00
dm-3 0.00 0.00 0.02 0.00 0.00 0.00 8.00 0.00 0.00 0.00 0.00 0.00 0.00
11/29/17 00:20:59
avg-cpu: %user %nice %system %iowait %steal %idle
0.00 0.00 0.00 0.00 0.00 100.00
Device: rrqm/s wrqm/s r/s w/s rMB/s wMB/s avgrq-sz avgqu-sz await r_await w_await svctm %util
sda 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
dm-0 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
dm-1 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
dm-2 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
dm-3 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
#!/bin/bash
#
# /etc/cron.d/performance
# 0 0,6,12,18 * * * root /home/fasthandle/scripts/df.sh > /dev/null 2>&1
LANG=C
DIR=/var/log/performance
[ -d "$DIR" ] || mkdir $DIR
df -BM | awk '{print strftime("%y/%m/%d %H:%M:%S"), $0}' >> ${LOGDIR}/df.`date +%Y%m`
echo " " >> ${LOGDIR}/df.`date +%Y%m`
~]# df -BM | awk '{print strftime("%y/%m/%d %H:%M:%S"), $0}'
17/11/30 01:58:50 Filesystem 1M-blocks Used Available Use% Mounted on
17/11/30 01:58:50 /dev/mapper/centos-root 6818M 3020M 3799M 45% /
17/11/30 01:58:50 devtmpfs 362M 0M 362M 0% /dev
17/11/30 01:58:50 tmpfs 371M 0M 371M 0% /dev/shm
17/11/30 01:58:50 tmpfs 371M 5M 366M 2% /run
17/11/30 01:58:50 tmpfs 371M 0M 371M 0% /sys/fs/cgroup
17/11/30 01:58:50 /dev/sda1 497M 119M 379M 24% /boot
17/11/30 01:58:50 tmpfs 75M 0M 75M 0% /run/user/0
~]# sar -n DEV 2 3 Linux 3.10.0-229.el7.x86_64 (centos7) 01/15/18 _x86_64_ (2 CPU) 00:44:43 IFACE rxpck/s txpck/s rxkB/s txkB/s rxcmp/s txcmp/s rxmcst/s 00:44:45 enp0s3 0.50 0.50 0.03 0.10 0.00 0.00 0.00 00:44:45 lo 0.00 0.00 0.00 0.00 0.00 0.00 0.00 00:44:45 IFACE rxpck/s txpck/s rxkB/s txkB/s rxcmp/s txcmp/s rxmcst/s 00:44:47 br-295e2a4e2c26 0.00 0.00 0.00 0.00 0.00 0.00 0.00 00:44:47 lo 0.00 0.00 0.00 0.00 0.00 0.00 0.00 00:44:47 IFACE rxpck/s txpck/s rxkB/s txkB/s rxcmp/s txcmp/s rxmcst/s 00:44:49 enp0s3 0.50 0.50 0.03 0.30 0.00 0.00 0.00 00:44:49 lo 0.00 0.00 0.00 0.00 0.00 0.00 0.00 Average: IFACE rxpck/s txpck/s rxkB/s txkB/s rxcmp/s txcmp/s rxmcst/s Average: enp0s3 0.67 0.67 0.04 0.24 0.00 0.00 0.00 Average: lo 0.00 0.00 0.00 0.00 0.00 0.00 0.00
netstat -i is check error packet with every interfaces.
#!/bin/bash
#
# /etc/cron.d/performance
# * * * * * root /home/fasthandle/scripts/netstat-i.sh > /dev/null 2>&1
LOGDIR=/var/log/performance
[ -d "$DIR" ] || mkdir $DIR
NUM=1
while [ "${NUM}" -le 4 ]
do
echo "##################################################" >> ${LOGDIR}/netstat-i.`date +%Y%m%d`
netstat -i >> ${LOGDIR}/netstat-i.`date +%Y%m%d`
NUM=`expr ${NUM} + 1`
sleep 15
done
netstat -i Kernel Interface table Iface MTU RX-OK RX-ERR RX-DRP RX-OVR TX-OK TX-ERR TX-DRP TX-OVR Flg br-295e2 1500 2570 0 0 0 1508 0 0 0 BMU docker0 1500 0 0 0 0 0 0 0 0 BMU enp0s3 1500 2570 0 0 0 1508 0 0 0 BMRU lo 65536 42862 0 0 0 42862 0 0 0 LRU
#!/bin/bash
#
# /etc/cron.d/performance
# * * * * * root /home/fasthandle/scripts/netstat-s.sh > /dev/null 2>&1
LOGDIR=/var/log/performance
[ -d "$DIR" ] || mkdir $DIR
NUM=1
while [ "${NUM}" -le 4 ]
do
echo "##################################################" >> ${LOGDIR}/netstat-s.`date +%Y%m%d`
netstat -s >> ${LOGDIR}/netstat-s.`date +%Y%m%d`
NUM=`expr ${NUM} + 1`
sleep 15
done
~]# netstat -s
Ip:
38025 total packets received
0 forwarded
0 incoming packets discarded
38024 incoming packets delivered
37440 requests sent out
........
abbr
........
Tcp:
3621 active connections openings
3630 passive connection openings
1 failed connection attempts
0 connection resets received
2 connections established
37708 segments received
37404 segments send out
0 segments retransmited
0 bad segments received.
33 resets sent
Udp:
30 packets received
20 packets to unknown port received.
0 packet receive errors
34 packets sent
0 receive buffer errors
0 send buffer errors
#!/bin/sh
#
# /etc/cron.d/performance
# * * * * * root /home/fasthandle/scripts/top.sh > /dev/null 2>&1
LOGDIR=/var/log/performance
[ -d "$DIR" ] || mkdir $DIR
NUM=1
while [ "${NUM}" -le 4 ]
do
echo "##################################################" >> ${LOGDIR}/top.`date +%Y%m%d`
top -b -n 1 >> ${LOGDIR}/top.`date +%Y%m%d`
NUM=`expr ${NUM} + 1`
sleep 15
done
~]# top -b -n 1
top - 00:54:49 up 31 min, 1 user, load average: 0.05, 0.03, 0.05
Tasks: 154 total, 2 running, 152 sleeping, 0 stopped, 0 zombie
%Cpu(s): 0.1 us, 0.1 sy, 0.0 ni, 99.5 id, 0.3 wa, 0.0 hi, 0.0 si, 0.0 st
KiB Mem : 759000 total, 250200 free, 244004 used, 264796 buff/cache
KiB Swap: 839676 total, 839676 free, 0 used. 389024 avail Mem
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
1 root 20 0 41116 3620 2388 S 0.0 0.5 0:00.76 systemd
2 root 20 0 0 0 0 S 0.0 0.0 0:00.00 kthreadd
3 root 20 0 0 0 0 S 0.0 0.0 0:00.01 ksoftirqd/0
5 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 kworker/0:0H
6 root 20 0 0 0 0 S 0.0 0.0 0:00.05 kworker/u4:0
7 root rt 0 0 0 0 S 0.0 0.0 0:00.05 migration/0
8 root 20 0 0 0 0 S 0.0 0.0 0:00.00 rcu_bh
9 root 20 0 0 0 0 S 0.0 0.0 0:00.00 rcuob/0
10 root 20 0 0 0 0 S 0.0 0.0 0:00.00 rcuob/1
...
#!/bin/bash
#
#/etc/cron.d/performance
#* * * * * root /home/fasthandle/scripts/ps.sh > /dev/null 2>&1
LOGDIR=/var/log/performance
[ -d "$LOGDIR" ] || mkdir $LOGDIR
NUM=1
while [ "${NUM}" -le 4 ]
do
echo "`date` ##################################################" >> ${LOGDIR}/ps.`date +%Y%m%d`
ps auxwf >> ${LOGDIR}/ps.`date +%Y%m%d`
NUM=`expr ${NUM} + 1`
sleep 15
done
~]# ps auxwf USER PID %CPU %MEM VSZ RSS TTY STAT START TIME COMMAND root 2 0.0 0.0 0 0 ? S 00:33 0:00 [kthreadd] root 3 0.0 0.0 0 0 ? S 00:33 0:00 \_ [ksoftirqd/0] root 5 0.0 0.0 0 0 ? S< 00:33 0:00 \_ [kworker/0:0H] root 6 0.0 0.0 0 0 ? S 00:33 0:00 \_ [kworker/u4:0] root 7 0.0 0.0 0 0 ? S 00:33 0:00 \_ [migration/0] root 8 0.0 0.0 0 0 ? S 00:33 0:00 \_ [rcu_bh] root 9 0.0 0.0 0 0 ? S 00:33 0:00 \_ [rcuob/0] root 10 0.0 0.0 0 0 ? S 00:33 0:00 \_ [rcuob/1] ............. abbr ............. root 898 0.0 0.4 83020 3652 ? Ss 00:33 0:00 /usr/sbin/sshd -D root 2532 0.0 0.7 141764 5700 ? Ss 00:34 0:00 \_ sshd: root@pts/0 root 2534 0.0 0.4 116448 3096 pts/0 Ss+ 00:34 0:00 | \_ -bash root 2668 0.0 0.7 141764 5700 ? Ss 00:41 0:00 \_ sshd: root@pts/1 root 2670 0.0 0.4 116448 3096 pts/1 Ss 00:41 0:00 \_ -bash root 2839 0.0 0.2 139628 1764 pts/1 R+ 00:55 0:00 \_ ps auxwf root 910 0.0 1.5 379552 12068 ? Ss 00:33 0:00 php-fpm: master process (/etc/php-fpm.conf) nginx 1338 0.0 0.7 379552 5468 ? S 00:33 0:00 \_ php-fpm: pool www nginx 1339 0.0 0.7 379552 5468 ? S 00:33 0:00 \_ php-fpm: pool www nginx 1340 0.0 0.7 379552 5472 ? S 00:33 0:00 \_ php-fpm: pool www nginx 1341 0.0 0.7 379552 5472 ? S 00:33 0:00 \_ php-fpm: pool www nginx 1342 0.0 0.7 379552 5472 ? S 00:33 0:00 \_ php-fpm: pool www
Set Linux Configuration with Fabric (set.py)