linux系統ElK基礎(2)

阿新 • • 發佈：2020-08-19

一、Logstash收集日誌

1.Logstash的配置檔案

[root@web01 ~]# vim /etc/logstash/logstash.yml
path.config: /etc/logstash/conf.d

2.logstash收集日誌檔案到檔案

[root@web01 ~]# vim /etc/logstash/conf.d/file_file.conf
input {
  file {
    path => "/var/log/messages"
    start_position => "beginning"
  }
}
output {
  file {
    path => "/tmp/messages_%{+YYYY-MM-dd}.log"
  }
}

3.logstash收集日誌檔案到ES

[root@web01 ~]# vim /etc/logstash/conf.d/file_es.conf
input {
  file {
    path => "/var/log/messages"
    start_position => "beginning"
  }
}
output {
  elasticsearch {
    hosts => ["172.16.1.51:9200"]
    index => "messages_%{+YYYY-MM-dd}.log"
  }
}

4.Logstash收集多日誌到檔案

[root@web01 ~]# vim /etc/logstash/conf.d/file_file.conf
input {
  file {
    type => "messages_log"
    path => "/var/log/messages"
    start_position => "beginning"
  }
  file {
    type => "secure_log"
    path => "/var/log/secure"
    start_position => "beginning"
  }       
}        
output {  
  if [type] == "messages_log" { 
    file {
      path => "/tmp/messages_%{+YYYY-MM-dd}"
    }        
  }
  if [type] == "secure_log" {
    file {
      path => "/tmp/secure_%{+YYYY-MM-dd}"
    }
  } 
}

5.Logstash收集多日誌到ES

1）方法一：

[root@web01 ~]# vim /etc/logstash/conf.d/more_es.conf 
input {
  file {
    type => "messages_log"
    path => "/var/log/messages"
    start_position => "beginning"
  }
  file {
    type => "secure_log"
    path => "/var/log/secure"
    start_position => "beginning"
  }
}
output {
  if [type] == "messages_log" {
    elasticsearch {
      hosts => ["10.0.0.51:9200"]
      index => "messages_%{+YYYY-MM-dd}"
    }
  }
  if [type] == "secure_log" {
    elasticsearch {
      hosts => ["10.0.0.51:9200"]
      index => "secure_%{+YYYY-MM-dd}"
    }
  }
}

[root@web01 ~]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/more_es.conf &

#啟動後檢視頁面

2）方法二：

[root@web01 ~]# vim /etc/logstash/conf.d/more_es_2.conf 
input {
  file {
    type => "messages_log"
    path => "/var/log/messages"
    start_position => "beginning"
  }
  file {
    type => "secure_log"
    path => "/var/log/secure"
    start_position => "beginning"
  }
}
output {
  elasticsearch {
    hosts => ["10.0.0.51:9200"]
    index => "%{type}_%{+YYYY-MM-dd}"
  }
}

[root@web01 ~]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/more_es_2.conf --path.data=/data/logstash/more_es_2 &

3）啟動多例項

#建立不同的資料目錄
[root@web01 ~]# mkdir /data/logstash/more_es_2
[root@web01 ~]# mkdir /data/logstash/more_es

#啟動時使用--path.data指定資料目錄
[root@web01 ~]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/more_es.conf --path.data=/data/logstash/more_es &
[root@web01 ~]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/more_es_2.conf --path.data=/data/logstash/more_es_2 &

#如果資源充足，可以使用多例項收集多日誌，如果伺服器資源不足，啟動不了多例項，配置一個檔案收集多日誌啟動

二、Logstash收集Tomcat日誌

1.安裝Tomcat

1.安裝java環境
[root@web01 ~]# rpm -ivh jdk-8u181-linux-x64.rpm

2.上傳包
[root@web01 ~]# rz apache-tomcat-10.0.0-M7.tar.gz

3.解壓
[root@web01 ~]# tar xf apache-tomcat-10.0.0-M7.tar.gz -C /usr/local/

4.做軟連線
[root@web01 ~]# ln -s /usr/local/apache-tomcat-10.0.0-M7 /usr/local/tomcat

5.啟動Tomcat
[root@web01 ~]# /usr/local/tomcat/bin/startup.sh

6.訪問頁面 10.0.0.7:8080

2.配置Logstash收集Tomcat日誌到檔案

[root@web01 ~]# vim /etc/logstash/conf.d/tomcat_file.conf
input {
  file {
    path => "/usr/local/tomcat/logs/localhost_access_log.*.txt"
    start_position => "beginning"
  }
}
output {
  file {
    path => "/tmp/tomcat_%{+YYYY-MM-dd}.log"
  }
}

3.配置Logstash收集Tomcat日誌到ES

[root@web01 ~]# vim /etc/logstash/conf.d/tomcat_es.conf
input {
  file {
    path => "/usr/local/tomcat/logs/localhost_access_log.*.txt"
    start_position => "beginning"
  }
}
output {
  elasticsearch {
    hosts => ["10.0.0.51:9200"]
    index => "tomcat_%{+YYYY-MM-dd}.log"
  }
}

三、收集Tomcat日誌修改格式

#收集tomcat日誌，當遇到報錯時，一條報錯會被分割成很多條資料，不方便檢視

解決方法：
1.修改tomcat日誌格式為json
	1）開發修改輸出日誌為json
	2）修改tomcat配置，日誌格式為json
2.使用logstash的input外掛下的mutiline模組

1.方法一：修改tomcat日誌格式

1）配置tomcat日誌為json格式

[root@web01 ~]# vim /usr/local/tomcat/conf/server.xml
#把原來的日誌格式註釋，新增我們的格式
<Valve className="org.apache.catalina.valves.AccessLogValve" directory="logs"
               prefix="tomcat_access_json" suffix=".log"
               pattern="{&quot;clientip&quot;:&quot;%h&quot;,&quot;ClientUser&quot;:&quot;%l&quot;,&quot;authenticated&quot;:&quot;%u&quot;,&quot;AccessTime&quot;:&quot;%t&quot;,&quot;method&quot;:&quot;%r&quot;,&quot;status&quot;:&quot;%s&quot;,&quot;SendBytes&quot;:&quot;%b&quot;,&quot;Query?string&quot;:&quot;%q&quot;,&quot;partner&quot;:&quot;%{Referer}i&quot;,&quot;AgentVersion&quot;:&quot;%{User-Agent}i&quot;}"/>

2）重啟tomcat

[root@web01 ~]# /usr/local/tomcat/bin/shutdown.sh
[root@web01 ~]# /usr/local/tomcat/bin/startup.sh

3）配置收集新的tomcat日誌

[root@web01 ~]# vim /etc/logstash/conf.d/tomcat_json_es.conf
input {
  file {
    path => "/usr/local/tomcat/logs/tomcat_access_json.*.log"
    start_position => "beginning"
  }
}
output {
  elasticsearch {
    hosts => ["10.0.0.51:9200"]
    index => "tomcat_json_%{+YYYY-MM-dd}.log"
  }
}

2.方法二：使用mutiline模組收集日誌

1）配置收集日誌測試

[root@web01 ~]# vim /etc/logstash/conf.d/test_mutiline.conf
input {
  stdin {
    codec => multiline {
	  #以[開頭
      pattern => "^\["
      #匹配到
      negate => true
      #向上合併，向下合併是next
      what => "previous"
    }
  }
}
output {
  stdout {
    codec => json
  }
}

#測試，輸入內容不會直接輸出，當遇到以 [ 開頭才會收集以上的日誌

2）配置收集tomcat錯誤日誌

[root@web01 ~]# vim /etc/logstash/conf.d/tomcat_mutiline.conf 
input {
  file {
    path => "/usr/local/tomcat/logs/tomcat_access_json.*.log"
    start_position => "beginning"
    codec => multiline {
      pattern => "^\["
      negate => true
      what => "previous"
    }
  }
}

output {
  elasticsearch {
    hosts => ["10.0.0.51:9200"]
    index => "tomcat_json_%{+YYYY-MM-dd}"
    codec => "json"
  }
}

3）將錯誤日誌寫入

[root@web01 ~]# cat 1.txt >> /usr/local/tomcat/logs/tomcat_access_json.2020-08-14.log

4）頁面檢視資料

![img](file:///C:\Users\oldboy\AppData\Roaming\Tencent\Users\1240206455\QQ\WinTemp\RichOle\AM25IS`()[email protected])

四、收集Nginx日誌

1.安裝Nginx

[root@web01 ~]# yum install -y nginx

2.配置Nginx日誌格式

[root@web01 ~]# vim /etc/nginx/nginx.conf
... ...
http {
    log_format  json  '{"@timestamp":"$time_iso8601",'
                      '"host":"$server_addr",'
                      '"clientip":"$remote_addr",'
                      '"size":$body_bytes_sent,'
                      '"responsetime":$request_time,'
                      '"upstreamtime":"$upstream_response_time",'
                      '"upstreamhost":"$upstream_addr",'
                      '"http_host":"$host",'
                      '"url":"$uri",'
                      '"referer":"$http_referer",'
                      '"agent":"$http_user_agent",'
                      '"status":"$status"}';

    access_log  /var/log/nginx/access.log  json;
... ...

3.配置收集Nginx日誌

[root@web01 ~]# vim /etc/logstash/conf.d/nginx_json.conf
input {
  file {
    path => "/var/log/nginx/access.log"
    start_position => "beginning"
  }
}
output {
  elasticsearch {
    hosts => ["10.0.0.51:9200"]
    index => "nginx_json_%{+YYYY-MM-dd}.log"
  }
}

五、獲取的日誌引數分離

1.方法一：

1）修改tomcat日誌收集配置

[root@web01 ~]# vim /etc/logstash/conf.d/tomcat_json_es.conf

input {
  file {
    path => "/usr/local/tomcat/logs/tomcat_access_json.*.log"
    start_position => "beginning"
  }
}

#把收集到的資料進行處理
filter {
  json {
    source => "message"
  }
}

output {
  elasticsearch {
    hosts => ["10.0.0.51:9200"]
    index => "tomcat_json_%{+YYYY-MM-dd}.log"
  }
}

2）去掉多餘資料

#message資料已經拆分，資料還在，去掉message資料
filter {
  json {
    source => "message"
    remove_field => ["message"]
  }
}

2.方法二：

1）修改收集Nginx日誌的配置

#nginx不需要配置修改獲取日誌，只需要收集同時修改格式即可
[root@web01 ~]# vim /etc/logstash/conf.d/nginx_json.conf 
input {
  file {
    path => "/var/log/nginx/access.log"
    start_position => "beginning"
    codec => "json"
  }
}
output {
  elasticsearch {
    hosts => ["10.0.0.51:9200"]
    index => "nginx_json_%{+YYYY-MM-dd}.log"
  }
}

六、Logstash收集日誌寫入redis

1.安裝redis

2.配置將資料寫入redis

[root@web01 ~]# vim /etc/logstash/conf.d/nginx_to_redis.conf
input {
  file {
    path => "/var/log/nginx/access.log"
    start_position => "beginning"
    codec => "json"
  }
}
output {
  redis {
    host => "172.16.1.51"
    port => "6379"
    data_type => "list"
    db => "0"
    key => "nginx_log"
  }
}