centos7環境下shell指令碼對nginx日誌過濾實現惡意ip的封堵

1.編寫監控nginx日誌指令碼

[root@web:/usr/local/worksh/monitor_ip]# cat monitor_ip.sh
#!/bin/bash
#
# 日誌日期格式：[17/Jul/2020:06:10:

# 存放過去5分鐘臨時日誌
filter_log="/data/www/logs/last_filve_min_access.log"
current_log="/data/www/logs/nginx_log/access/my.chinasoft.com_access.log"

# 每次都需要清理一次，否則越累積越多
echo
 
 ''> ${filter_log}


# 這是剛開始的寫法，這樣會有一個問題，當到小時、天、月、年的分界點時會漏掉其中的5分鐘日誌
function filter_last_five_min_log
{
    # 將過去5分鐘的日誌單獨寫入一個檔案中
    # # Fri Jul 17 06:17:35 PDT 2020
    day=`date +%d`
    month=`env LANG=en_US.UTF-8 date|awk '{print $2}'`
    year=`env LANG=en_US.UTF-8 date|awk '{print $6}'`
    current_hour
 
=`date +%H`

    # 當前開始，5分鐘內的時間
    current_min=`date +%M`
    last_one_min=`date -d "1 minute ago" +"%M"`
    last_two_min=`date -d "2 minute ago" +"%M"`
    last_three_min=`date -d "3 minute ago" +"%M"`
    last_four_min=`date -d "4 minute ago" +"%M"`
    last_five_min=`date -d "5 minute ago" +"%M
 
"`
    for min_num in $current_min $last_one_min $last_two_min $last_three_min $last_four_min $last_five_min;do
    /bin/cat ${current_log} |grep "/chinasoft/login?ramdom="| grep "\[${day}/${month}/${year}:${current_hour}:${min_num}" >> ${filter_log}
    done
}


function current_minute_log
{
    local day=`date +%d`
    local month=`date +%b`
    local year=`date +%Y`
    local current_hour=`date +%H`
    local min_num=`date +%M`
    /bin/cat ${current_log} |grep "/chinasoft/login?ramdom="| grep "\[${day}/${month}/${year}:${current_hour}:${min_num}" >> ${filter_log}
}

function one_minute_log
{
    local day=`date -d "1 minute ago" +"%d"`
    local month=`date -d "1 minute ago" +"%b"`
    local year=`date -d "1 minute ago" +"%Y"`
    local current_hour=`date -d "1 minute ago" +"%H"`
    local min_num=`date -d "1 minute ago" +"%M"`
    /bin/cat ${current_log} |grep "/chinasoft/login?ramdom="| grep "\[${day}/${month}/${year}:${current_hour}:${min_num}" >> ${filter_log}
}

function two_minute_log
{
    local day=`date -d "2 minute ago" +"%d"`
    local month=`date -d "2 minute ago" +"%b"`
    local year=`date -d "2 minute ago" +"%Y"`
    local current_hour=`date -d "2 minute ago" +"%H"`
    local min_num=`date -d "2 minute ago" +"%M"`
    /bin/cat ${current_log} |grep "/chinasoft/login?ramdom="| grep "\[${day}/${month}/${year}:${current_hour}:${min_num}" >> ${filter_log}
}

function three_minute_log
{
    local day=`date -d "3 minute ago" +"%d"`
    local month=`date -d "3 minute ago" +"%b"`
    local year=`date -d "3 minute ago" +"%Y"`
    local current_hour=`date -d "3 minute ago" +"%H"`
    local min_num=`date -d "3 minute ago" +"%M"`
    /bin/cat ${current_log} |grep "/chinasoft/login?ramdom="| grep "\[${day}/${month}/${year}:${current_hour}:${min_num}" >> ${filter_log}
}

function four_minute_log
{
    local day=`date -d "4 minute ago" +"%d"`
    local month=`date -d "4 minute ago" +"%b"`
    local year=`date -d "4 minute ago" +"%Y"`
    local current_hour=`date -d "4 minute ago" +"%H"`
    local min_num=`date -d "4 minute ago" +"%M"`
    /bin/cat ${current_log} |grep "/chinasoft/login?ramdom="| grep "\[${day}/${month}/${year}:${current_hour}:${min_num}" >> ${filter_log}
}

function five_minute_log
{
    local day=`date -d "5 minute ago" +"%d"`
    local month=`date -d "5 minute ago" +"%b"`
    local year=`date -d "5 minute ago" +"%Y"`
    local current_hour=`date -d "5 minute ago" +"%H"`
    local min_num=`date -d "5 minute ago" +"%M"`
    /bin/cat ${current_log} |grep "/chinasoft/login?ramdom="| grep "\[${day}/${month}/${year}:${current_hour}:${min_num}" >> ${filter_log}
}

function count_last_logs
{
    # 將過去5分鐘撞庫的日誌ip記錄下來，並統計次數
    malice_access_ips="/data/www/logs/malice_access_ips.log"
    /bin/cat ${filter_log} |grep "/chinasoft/login?ramdom="|awk '{print $1}'|sort -r|uniq -c|sort -r -n|head -300 > ${malice_access_ips}
}


function count_filter_ip
{
    # 讀取統計的ip次數，如果一個ip撞庫次數超過100次，則防火牆拉黑
    while read line
    do
    access_num=`echo $line|awk '{print $1}'`
    access_ip=`echo $line|awk '{print $2}'`
    # 
    if [[ $access_num -gt 100 ]];
    then
        drop_ip=`iptables -nvL |grep ${access_ip}|awk '{print $8}'`
        if [[ ${access_ip} = ${drop_ip} ]];then
            # 如果已經被拉黑則跳過這個ip
            continue
        fi
        
        company_ip=`grep ${access_ip} /usr/local/worksh/monitor_ip/company_ips.txt`
        if [[ ${access_ip} = ${company_ip} ]];then
            # 如果是公司IP跳過
            continue
        fi

        iptables -I INPUT -s ${access_ip} -j DROP
        echo "access_nu="${access_num} "access_ip="${access_ip}
    fi
    done < ${malice_access_ips}
}


function main
{

    # 篩選過去5分鐘的日誌
    current_minute_log
    one_minute_log
    two_minute_log
    three_minute_log
    four_minute_log
    five_minute_log

    # 統計ip的次數
    count_last_logs
    # 過濾符合要求的ip
    count_filter_ip
}

main

2.計劃任務
# 每5分鐘統計一次訪問日誌，如果5分鐘內同一個ip撞庫超過100次就拉黑這個ip地址
*/5 * * * * /bin/bash /usr/local/worksh/monitor_ip/monitor_ip.sh >/dev/null 2>&1

# 防火牆的示例
[root@web:/usr/local/worksh/monitor_ip]# iptables -nvL
Chain INPUT (policy DROP 1786 packets, 81061 bytes)
pkts bytes target prot opt in out source destination
20 1124 DROP all -- * * 1.1.1.1 0.0.0.0/0

# 公司IP地址
[root@web:/usr/local/worksh/monitor_ip]# cat company_ips.txt
1.1.1.1
1.1.1.2
1.1.1.3

日誌示例
[root@web:/usr/local/worksh/monitor_ip]# tail -n 5 /data/www/logs/nginx_log/access/my.chinasoft.com_access.log
103.147.118.3 - - [17/Jul/2020:07:10:00 -0700] - - "POST /chinasoft/login?ramdom= HTTP/1.1" 200 65 "http://my.chinasoft.com/user/login" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36" 0.223
36.71.239.228 - - [17/Jul/2020:07:10:00 -0700] - - "GET /static/jslibs/jquery.uid.js HTTP/1.1" 200 497 "https://pdf.chinasoft.com/how-to/convert-pdf-to-word.html?gclid=CjwKCAjwmMX4BRAAEiwA-zM4JmOMdq6wDpFgAOMdKNbJv1tHWCiJmDEHlKbVmUtobEFuHZQF7FP6rBoC360QAvD_BwE" "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36" 0.000
91.74.26.54 - - [17/Jul/2020:07:10:00 -0700] - - "GET /static/jslibs/jquery.uid.js HTTP/1.1" 200 497 "https://web.chinasoft.com/" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36 Edg/83.0.478.64" 0.000
46.8.247.3 - - [17/Jul/2020:07:10:00 -0700] - - "POST /chinasoft/login?ramdom= HTTP/1.1" 200 65 "http://my.chinasoft.com/user/login" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36" 0.320
14.116.204.164 - - [17/Jul/2020:07:10:00 -0700] - - "POST /chinasoft/login?ramdom= HTTP/1.1" 200 65 "http://my.chinasoft.com/user/login" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36" 1.609