Table of Contents

Set Linux Configuration with Fabric (set.py)

/etc/cron.d/performance

Performance survey of seconds interval on Cron



/etc/cron.d/performance

LANG=C
LOGDIR=/var/log/performance

# 1H = 60m = 60m x 1m = 60m x 60sec = 3600sec
# 3600sec = 1sec x 3600 times
# 3600sec = 5sec x 720 times
# 3600sec = 10sec x 360 times

#-------------------------------------------------------------------
#daily output
#-------------------------------------------------------------------
# Multi
0 * * * * root vmstat 5 720 | awk '{print strftime("\%y/\%m/\%d \%H:\%M:\%S"), $0} { system(":") }' >> ${LOGDIR}/vmstat.`date +\%Y\%m\%d`
#0 * * * * root dstat -taf 5 720 >> ${LOGDIR}/dstat-taf.`date +\%Y\%m\%d`

# CPU Details
0 * * * * root mpstat 5 720 >> ${DIR}/mpstat.`date +\%Y\%m\%d`

# Load Average Details
0 * * * * root sar -q 5 720 >> ${LOGDIR}/sar-q.`date +\%Y\%m\%d`

# Memory
0 * * * * root sar -r 5 720 >> ${LOGDIR}/sar-r.`date +\%Y\%m\%d`

# Disk Details
0 * * * * root iostat -xtm 10 360 >> ${LOGDIR}/iostat.`date +\%Y\%m\%d` > /dev/null 2>&1

# Network Details
0 * * * * root sar -n DEV 5 720 >> ${LOGDIR}/sar-n-DEV.`date +\%Y\%m\%d`
* * * * * root /home/fasthandle/scripts/netstat-s.sh > /dev/null 2>&1
* * * * * root /home/fasthandle/scripts/netstat-i.sh > /dev/null 2>&1

# Process Details
* * * * * root /home/fasthandle/scripts/ps.sh > /dev/null 2>&1
* * * * * root /home/fasthandle/scripts/top.sh > /dev/null 2>&


#-------------------------------------------------------------------
# monthly output
#-------------------------------------------------------------------
# Disk Details
0  0,6,12,18 * * * root /home/fasthandle/scripts/df.sh > /dev/null 2>&1


#-------------------------------------------------------------------
# gzip/rm output file
#-------------------------------------------------------------------
5 0 * * * root find ${LOGDIR} -type f -not -name "df*"  -mtime +30  -exec rm -rf {} \;
10 0 * * * root gzip ${LOGDIR}/*.`date -d '1day ago' +\%Y\%m\%d`
10 0 1 * * root gzip ${LOGDIR}/df.`date -d '1month ago' +\%Y\%m`

# man 5 crontab
A “%” character in the command, unless escaped with a backslash (\), will be changed into newline characters, and all data after the first % will be sent to the command as standard input.




Multi

vmstat

memory, swap, io, system, cpu

Output Example
~]# vmstat 1 3 | awk '{print strftime("%y/%m/%d %H:%M:%S"), $0} { system(":") }'
17/11/29 00:18:29 procs -----------memory---------- ---swap-- -----io---- -system-- ------cpu-----
17/11/29 00:18:29  r  b   swpd   free   buff  cache   si   so    bi    bo   in   cs us sy id wa st
17/11/29 00:18:29  1  0      0 270248   1492 251820    0    0    85    36   64  117  0  0 99  0  0
17/11/29 00:18:30  0  0      0 270092   1492 251852    0    0     0     4  149  265  0  0 100  0  0
17/11/29 00:18:31  1  0      0 270092   1492 251852    0    0     0    21  139  236  0  0 100  0  0


dstat -taf

~]# dstat -taf 2 5
----system---- -------cpu0-usage--------------cpu1-usage------ --dsk/sda-- -net/enp0s3 ---paging-- ---system--
     time     |usr sys idl wai hiq siq:usr sys idl wai hiq siq| read  writ| recv  send|  in   out | int   csw
01-12 00:52:35|  0   0  99   0   0   0:  0   0  97   3   0   0| 198k   70k|   0     0 |   0     0 | 133   243
01-12 00:52:37|  0   0 100   0   0   0:  0   1  99   1   0   0|   0   511k|  60B  854B|   0     0 | 120   235
01-12 00:52:39|  0   0 100   0   0   0:  0   0 100   0   0   0|   0  4096B|  60B  494B|   0     0 | 112   203
01-12 00:52:41|  0   0 100   0   0   0:  0   0  95   5   0   0|   0  4096B|  60B  494B|   0     0 | 119   203
01-12 00:52:43|  1   0 100   0   0   0:  0   0 100   0   0   0|   0     0 | 148B  486B|   0     0 | 110   203
01-12 00:52:45|  0   0 100   0   0   0:  0   0 100   0   0   0|   0  2048B|  60B  486B|   0     0 | 102   175




CPU Details

mpstat

CPU

Output Example
~]# mpstat -P ALL 1 3
Linux 3.10.0-229.el7.x86_64 (centos7)   11/29/17        _x86_64_        (2 CPU)

00:17:23     CPU    %usr   %nice    %sys %iowait    %irq   %soft  %steal  %guest  %gnice   %idle
00:17:24     all    0.00    0.00    0.00    0.50    0.00    0.00    0.00    0.00    0.00   99.50
00:17:24       0    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00  100.00
00:17:24       1    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00  100.00

00:17:24     CPU    %usr   %nice    %sys %iowait    %irq   %soft  %steal  %guest  %gnice   %idle
00:17:25     all    0.00    0.00    0.50    0.00    0.00    0.00    0.00    0.00    0.00   99.50
00:17:25       0    0.00    0.00    0.00    0.00    0.00    1.00    0.00    0.00    0.00   99.00
00:17:25       1    0.00    0.00    0.00    1.00    0.00    0.00    0.00    0.00    0.00   99.00

00:17:25     CPU    %usr   %nice    %sys %iowait    %irq   %soft  %steal  %guest  %gnice   %idle
00:17:26     all    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00  100.00
00:17:26       0    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00  100.00
00:17:26       1    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00  100.00

Average:     CPU    %usr   %nice    %sys %iowait    %irq   %soft  %steal  %guest  %gnice   %idle
Average:     all    0.00    0.00    0.17    0.17    0.00    0.00    0.00    0.00    0.00   99.66
Average:       0    0.00    0.00    0.00    0.00    0.00    0.33    0.00    0.00    0.00   99.67
Average:       1    0.00    0.00    0.00    0.34    0.00    0.00    0.00    0.00    0.00   99.66




Load Average Details

sar -q

~]# sar -q 2 3
Linux 3.10.0-229.el7.x86_64 (centos7)   01/15/18        _x86_64_        (2 CPU)

01:43:05      runq-sz  plist-sz   ldavg-1   ldavg-5  ldavg-15   blocked
01:43:07            0       219      0.00      0.01      0.05         0
01:43:09            1       219      0.00      0.01      0.05         0
01:43:11            0       219      0.00      0.01      0.05         0
Average:            0       219      0.00      0.01      0.05         0


Memory Details

sar -r

~]# sar -r 2 3
Linux 3.10.0-229.el7.x86_64 (centos7)   01/15/18        _x86_64_        (2 CPU)

01:43:26    kbmemfree kbmemused  %memused kbbuffers  kbcached  kbcommit   %commit  kbactive   kbinact   kbdirty
01:43:28       249240    509760     67.16      1492    230356   1465816     91.69    247440    185552         0
01:43:30       249240    509760     67.16      1492    230356   1465816     91.69    247444    185552         0
01:43:32       249240    509760     67.16      1492    230360   1465816     91.69    247444    185556         0
Average:       249240    509760     67.16      1492    230357   1465816     91.69    247443    185553         0




Disk Details

iostat

IO

Output Example
~]# iostat -xtm 1 2
Linux 3.10.0-229.el7.x86_64 (centos7)   11/29/17        _x86_64_        (2 CPU)

11/29/17 00:20:58
avg-cpu:  %user   %nice %system %iowait  %steal   %idle
           0.11    0.00    0.17    0.31    0.00   99.41

Device:         rrqm/s   wrqm/s     r/s     w/s    rMB/s    wMB/s avgrq-sz avgqu-sz   await r_await w_await  svctm  %util
sda               0.45     0.11    3.16    5.10     0.15     0.07    53.91     0.02    2.36    3.68    1.54   0.93   0.77
dm-0              0.00     0.00    3.12    4.35     0.14     0.07    58.00     0.02    2.61    3.70    1.83   1.02   0.76
dm-1              0.00     0.00    0.15    0.00     0.00     0.00     8.00     0.00    0.62    0.62    0.00   0.35   0.01
dm-2              0.00     0.00    0.02    0.00     0.00     0.00     8.00     0.00    0.25    0.25    0.00   0.25   0.00
dm-3              0.00     0.00    0.02    0.00     0.00     0.00     8.00     0.00    0.00    0.00    0.00   0.00   0.00

11/29/17 00:20:59
avg-cpu:  %user   %nice %system %iowait  %steal   %idle
           0.00    0.00    0.00    0.00    0.00  100.00

Device:         rrqm/s   wrqm/s     r/s     w/s    rMB/s    wMB/s avgrq-sz avgqu-sz   await r_await w_await  svctm  %util
sda               0.00     0.00    0.00    0.00     0.00     0.00     0.00     0.00    0.00    0.00    0.00   0.00   0.00
dm-0              0.00     0.00    0.00    0.00     0.00     0.00     0.00     0.00    0.00    0.00    0.00   0.00   0.00
dm-1              0.00     0.00    0.00    0.00     0.00     0.00     0.00     0.00    0.00    0.00    0.00   0.00   0.00
dm-2              0.00     0.00    0.00    0.00     0.00     0.00     0.00     0.00    0.00    0.00    0.00   0.00   0.00
dm-3              0.00     0.00    0.00    0.00     0.00     0.00     0.00     0.00    0.00    0.00    0.00   0.00   0.00


df.sh

#!/bin/bash
#
# /etc/cron.d/performance
# 0  0,6,12,18 * * * root /home/fasthandle/scripts/df.sh > /dev/null 2>&1

LANG=C

DIR=/var/log/performance
[ -d "$DIR" ] || mkdir $DIR

df -BM | awk '{print strftime("%y/%m/%d %H:%M:%S"), $0}' >> ${LOGDIR}/df.`date +%Y%m`
echo " " >> ${LOGDIR}/df.`date +%Y%m`

~]# df -BM | awk '{print strftime("%y/%m/%d %H:%M:%S"), $0}'
17/11/30 01:58:50 Filesystem              1M-blocks  Used Available Use% Mounted on
17/11/30 01:58:50 /dev/mapper/centos-root     6818M 3020M     3799M  45% /
17/11/30 01:58:50 devtmpfs                     362M    0M      362M   0% /dev
17/11/30 01:58:50 tmpfs                        371M    0M      371M   0% /dev/shm
17/11/30 01:58:50 tmpfs                        371M    5M      366M   2% /run
17/11/30 01:58:50 tmpfs                        371M    0M      371M   0% /sys/fs/cgroup
17/11/30 01:58:50 /dev/sda1                    497M  119M      379M  24% /boot
17/11/30 01:58:50 tmpfs                         75M    0M       75M   0% /run/user/0




Network Details

sar -n DEV

~]# sar -n DEV 2 3
Linux 3.10.0-229.el7.x86_64 (centos7)   01/15/18        _x86_64_        (2 CPU)

00:44:43        IFACE   rxpck/s   txpck/s    rxkB/s    txkB/s   rxcmp/s   txcmp/s  rxmcst/s
00:44:45       enp0s3      0.50      0.50      0.03      0.10      0.00      0.00      0.00
00:44:45           lo      0.00      0.00      0.00      0.00      0.00      0.00      0.00

00:44:45        IFACE   rxpck/s   txpck/s    rxkB/s    txkB/s   rxcmp/s   txcmp/s  rxmcst/s
00:44:47    br-295e2a4e2c26      0.00      0.00      0.00      0.00      0.00      0.00      0.00
00:44:47           lo      0.00      0.00      0.00      0.00      0.00      0.00      0.00

00:44:47        IFACE   rxpck/s   txpck/s    rxkB/s    txkB/s   rxcmp/s   txcmp/s  rxmcst/s
00:44:49       enp0s3      0.50      0.50      0.03      0.30      0.00      0.00      0.00
00:44:49           lo      0.00      0.00      0.00      0.00      0.00      0.00      0.00

Average:        IFACE   rxpck/s   txpck/s    rxkB/s    txkB/s   rxcmp/s   txcmp/s  rxmcst/s
Average:       enp0s3      0.67      0.67      0.04      0.24      0.00      0.00      0.00
Average:           lo      0.00      0.00      0.00      0.00      0.00      0.00      0.00

netstat-i.sh

netstat -i is check error packet with every interfaces.

#!/bin/bash
#
# /etc/cron.d/performance
# * * * * * root /home/fasthandle/scripts/netstat-i.sh > /dev/null 2>&1
 
LOGDIR=/var/log/performance
[ -d "$DIR" ] || mkdir $DIR
 
NUM=1
while [ "${NUM}" -le 4 ]
do
    echo "##################################################" >> ${LOGDIR}/netstat-i.`date +%Y%m%d`
    netstat -i >> ${LOGDIR}/netstat-i.`date +%Y%m%d`
    NUM=`expr ${NUM} + 1`
    sleep 15
done

netstat -i
Kernel Interface table
Iface      MTU    RX-OK RX-ERR RX-DRP RX-OVR    TX-OK TX-ERR TX-DRP TX-OVR Flg
br-295e2  1500     2570      0      0 0          1508      0      0      0 BMU
docker0   1500        0      0      0 0             0      0      0      0 BMU
enp0s3    1500     2570      0      0 0          1508      0      0      0 BMRU
lo       65536    42862      0      0 0         42862      0      0      0 LRU


netstat-s.sh

#!/bin/bash
#
# /etc/cron.d/performance
# * * * * * root /home/fasthandle/scripts/netstat-s.sh > /dev/null 2>&1
 
LOGDIR=/var/log/performance
[ -d "$DIR" ] || mkdir $DIR
 
NUM=1
while [ "${NUM}" -le 4 ]
do
    echo "##################################################" >> ${LOGDIR}/netstat-s.`date +%Y%m%d`
    netstat -s >> ${LOGDIR}/netstat-s.`date +%Y%m%d`
    NUM=`expr ${NUM} + 1`
    sleep 15
done

~]# netstat -s
Ip:
    38025 total packets received
    0 forwarded
    0 incoming packets discarded
    38024 incoming packets delivered
    37440 requests sent out
........
abbr
........
Tcp:
    3621 active connections openings
    3630 passive connection openings
    1 failed connection attempts
    0 connection resets received
    2 connections established
    37708 segments received
    37404 segments send out
    0 segments retransmited
    0 bad segments received.
    33 resets sent
Udp:
    30 packets received
    20 packets to unknown port received.
    0 packet receive errors
    34 packets sent
    0 receive buffer errors
    0 send buffer errors




Process Details

top.sh

#!/bin/sh
#
# /etc/cron.d/performance
# * * * * * root /home/fasthandle/scripts/top.sh > /dev/null 2>&1

LOGDIR=/var/log/performance
[ -d "$DIR" ] || mkdir $DIR

NUM=1
while [ "${NUM}" -le 4 ]
do
    echo "##################################################" >> ${LOGDIR}/top.`date +%Y%m%d`
    top -b -n 1 >> ${LOGDIR}/top.`date +%Y%m%d`
    NUM=`expr ${NUM} + 1`
    sleep 15
done

~]# top -b -n 1
top - 00:54:49 up 31 min,  1 user,  load average: 0.05, 0.03, 0.05
Tasks: 154 total,   2 running, 152 sleeping,   0 stopped,   0 zombie
%Cpu(s):  0.1 us,  0.1 sy,  0.0 ni, 99.5 id,  0.3 wa,  0.0 hi,  0.0 si,  0.0 st
KiB Mem :   759000 total,   250200 free,   244004 used,   264796 buff/cache
KiB Swap:   839676 total,   839676 free,        0 used.   389024 avail Mem

  PID USER      PR  NI    VIRT    RES    SHR S  %CPU %MEM     TIME+ COMMAND
    1 root      20   0   41116   3620   2388 S   0.0  0.5   0:00.76 systemd
    2 root      20   0       0      0      0 S   0.0  0.0   0:00.00 kthreadd
    3 root      20   0       0      0      0 S   0.0  0.0   0:00.01 ksoftirqd/0
    5 root       0 -20       0      0      0 S   0.0  0.0   0:00.00 kworker/0:0H
    6 root      20   0       0      0      0 S   0.0  0.0   0:00.05 kworker/u4:0
    7 root      rt   0       0      0      0 S   0.0  0.0   0:00.05 migration/0
    8 root      20   0       0      0      0 S   0.0  0.0   0:00.00 rcu_bh
    9 root      20   0       0      0      0 S   0.0  0.0   0:00.00 rcuob/0
   10 root      20   0       0      0      0 S   0.0  0.0   0:00.00 rcuob/1
...


ps.sh

#!/bin/bash
#
#/etc/cron.d/performance
#* * * * * root /home/fasthandle/scripts/ps.sh > /dev/null 2>&1

LOGDIR=/var/log/performance
[ -d "$LOGDIR" ] || mkdir $LOGDIR

NUM=1
while [ "${NUM}" -le 4 ]
do
    echo "`date` ##################################################" >> ${LOGDIR}/ps.`date +%Y%m%d`
    ps auxwf >> ${LOGDIR}/ps.`date +%Y%m%d`
    NUM=`expr ${NUM} + 1`
    sleep 15
done

~]# ps auxwf
USER       PID %CPU %MEM    VSZ   RSS TTY      STAT START   TIME COMMAND
root         2  0.0  0.0      0     0 ?        S    00:33   0:00 [kthreadd]
root         3  0.0  0.0      0     0 ?        S    00:33   0:00  \_ [ksoftirqd/0]
root         5  0.0  0.0      0     0 ?        S<   00:33   0:00  \_ [kworker/0:0H]
root         6  0.0  0.0      0     0 ?        S    00:33   0:00  \_ [kworker/u4:0]
root         7  0.0  0.0      0     0 ?        S    00:33   0:00  \_ [migration/0]
root         8  0.0  0.0      0     0 ?        S    00:33   0:00  \_ [rcu_bh]
root         9  0.0  0.0      0     0 ?        S    00:33   0:00  \_ [rcuob/0]
root        10  0.0  0.0      0     0 ?        S    00:33   0:00  \_ [rcuob/1]
.............
abbr
.............
root       898  0.0  0.4  83020  3652 ?        Ss   00:33   0:00 /usr/sbin/sshd -D
root      2532  0.0  0.7 141764  5700 ?        Ss   00:34   0:00  \_ sshd: root@pts/0
root      2534  0.0  0.4 116448  3096 pts/0    Ss+  00:34   0:00  |   \_ -bash
root      2668  0.0  0.7 141764  5700 ?        Ss   00:41   0:00  \_ sshd: root@pts/1
root      2670  0.0  0.4 116448  3096 pts/1    Ss   00:41   0:00      \_ -bash
root      2839  0.0  0.2 139628  1764 pts/1    R+   00:55   0:00          \_ ps auxwf
root       910  0.0  1.5 379552 12068 ?        Ss   00:33   0:00 php-fpm: master process (/etc/php-fpm.conf)
nginx     1338  0.0  0.7 379552  5468 ?        S    00:33   0:00  \_ php-fpm: pool www
nginx     1339  0.0  0.7 379552  5468 ?        S    00:33   0:00  \_ php-fpm: pool www
nginx     1340  0.0  0.7 379552  5472 ?        S    00:33   0:00  \_ php-fpm: pool www
nginx     1341  0.0  0.7 379552  5472 ?        S    00:33   0:00  \_ php-fpm: pool www
nginx     1342  0.0  0.7 379552  5472 ?        S    00:33   0:00  \_ php-fpm: pool www





Set Linux Configuration with Fabric (set.py)


Top#OS Management