收集TCP/UDP日誌

通過logstash的tcp/udp外掛收集日誌，通常用於在向elasticsearch日誌補錄丟失的部分日誌，可以將丟失的日誌通過一個TCP埠直接寫入到elasticsearch伺服器。

1.配置Logstash

#進入Logstash配置檔案目錄
[root@redis01 ~]# cd /etc/logstash/conf.d/
#編輯Logstash配置檔案
[root@redis01 conf.d]# vim tcp.conf
input {
  tcp {
    port => 1234
    type => "tcplog"
    mode => "server"
  }
}

output {
  stdout {
    codec => rubydebug
  }
}
 

2.啟動

#啟動Logstash
[root@redis01 conf.d]# /usr/share/logstash/bin/logstash -f  /etc/logstash/conf.d/tcp.conf
#檢測埠是否啟動成功
[root@redis01 ~]# netstat -lntup
tcp        0      0 :::1234                     :::*                        LISTEN      8656/java

3.使用telnet測試

[root@redis02 ~]# telnet 172.16.1.81 1234
Trying 172.16.1.81...
Connected to 172.16.1.81.
Escape character is '^]'.
13
12335346457thgdfhbd


#檢視
{
          "port" => 58991,
      "@version" => "1",
    "@timestamp" => 2020-12-08T16:58:01.351Z,
          "host" => "172.16.1.82",
       "message" => "13\r",
          "type" => "tcplog"
}
{
          "port" => 58991,
      "@version" => "1",
    "@timestamp" => 2020-12-08T16:58:27.160Z,
          "host" => "172.16.1.82",
       "message" => "12335346457thgdfhbd\r",
          "type" => "tcplog"
}
 

4.使用nc工具

1）安裝nc工具

#使用yum安裝nc
[root@web01 ~]# yum install -y nc

2）使用測試

1.使用nc傳輸資料
[root@web01 ~]# echo "test nc" | nc 10.0.0.81 1234

2.收集檔案日誌
[root@web01 ~]# cat /etc/passwd | nc 10.0.0.81 1234

3.實時收集遠端伺服器的日誌
[root@web01 ~]# tail -f /var/log/nginx/access.log | nc 10.0.0.81 1234

5.收集多個tcp日誌到ES

1）配置

[root@redis01 ~]# cat /etc/logstash/conf.d/tcp_es.conf 
input {
  tcp {
    port => 1234
    type => "nginxlog"
    mode => "server"
  }
  tcp {
    port => "2345"
    type => "tomcatlog"
    mode => "server"
  }
}
output {
  if [type] == "nginxlog" {
    elasticsearch {
      hosts => ["10.0.0.71:9200"]
      index => "tcp_nginxlog_%{+YYYY-MM-dd}"
    }
  }
  if [type] == "tomcatlog" {
    elasticsearch {
      hosts => ["10.0.0.71:9200"]
      index => "tcp_tomcatlog_%{+YYYY-MM-dd}"
    }
  }
}
 

2）啟動

[root@redis01 ~]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/tcp_es.conf

3）測試

[root@web01 ~]# tail -f /var/log/nginx/access.log | nc 10.0.0.81 1234

[root@web01 ~]# tail -f /usr/local/tomcat/logs/tomcat_access_json.$(date +%F).log | nc 10.0.0.81 2345

# 頁面檢視索引