日誌收集三:rsyslog + elasticsearch + kibana
阿新 • • 發佈:2018-06-09
rsyslog-v8rsyslog v8
配置文件語法改變較大,添加了很多其他輸入輸出模塊,可以更好的進行擴展。
在v5的基礎上,添加elasticsearch的發送。
v8配置參考:請添加鏈接描述
v8采集端完整配置
module(load="imuxsock") # provides support for local system logging (e.g. via logger command) module(load="imklog") # provides kernel logging support (previously done by rklogd) module(load="omelasticsearch") module(load="imfile" mode="inotify" readtimeout="5" ) #input(type="imfile" file="/var/log/zabbix_*.log" tag="C+eebo.test.test+CD" ruleset="fileinput" facility="local6" PersistStateInterval="100" startmsg.regex="^[[:digit:]]{5}:[[:digit:]]{8}:[[:digit:]]{6}.[[:digit:]]{3}" maxsubmitatonce="1024000" readMode="2") module(load="imudp" threads="2") # needs to be done just once input(type="imudp" port="514" address="127.0.0.1" rcvbufSize="1m" Ruleset="udpinput") ### server model need to load and set #module(load="imtcp" keepalive="on" keepalive.time="30" KeepAlive.Probes="1") # needs to be done just once #input(type="imtcp" port="514" Ruleset="tcpinput") #### GLOBAL DIRECTIVES #### # Use default timestamp format $ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat # Include all config files in /etc/rsyslog.d/ $IncludeConfig /etc/rsyslog.d/*.conf # Spool files #$WorkDirectory /var/spool/rsyslog $WorkDirectory /var/lib/rsyslog # Filter duplicate messages $RepeatedMsgReduction on $EscapeControlCharactersOnReceive off #### RULES #### # Log all kernel messages to the console. # Logging much else clutters up the screen. #kern.* /dev/console # Log anything (except mail) of level info or higher. # Don‘t log private authentication messages! #*.info;mail.none;authpriv.none;cron.none;local0.none;local7.none /var/log/messages auth.info;authpriv.info;daemon.info;kern.info;syslog.info;user.info;lpr.info;mark.info /var/log/messages # The authpriv file has restricted access. authpriv.* /var/log/secure # Log all the mail messages in one place. mail.* /var/log/maillog # Log cron stuff cron.* /var/log/cron # Everybody gets emergency messages *.emerg :omusrmsg:* # Save news errors of level crit and higher in a special file. uucp,news.crit /var/log/spooler # Save boot messages also to boot.log local7.* /var/log/boot.log template(name="msg" type="string" string="%rawmsg%\n") template(name="msg" type="string" string="%rawmsg%\n") template(name="timemsg" type="string" string="%timegenerated:8:15% %msg%\n") template(name="udppfile" type="string" string="/data/rsyslogs/%HOSTNAME%/%syslogtag%/%$year%-%$month%-%$day%.log") template(name="tcppfile" type="string" string="/data/rsyslogs/%HOSTNAME%/%syslogtag%/%$year%-%$month%-%$day%_%fromhost-ip%.log") template(name="tcpnfile" type="string" string="/data/rsyslogs/nginx/%syslogtag:F,66:1%/%$year%-%$month%-%$day%_%fromhost-ip%_%syslogtag:F,66:2%.log") ### get local code log from syslog to elasearchtich template(name="udppname" type="string" string="%HOSTNAME%-%$year%.%$month%.%$day%") template(name="udpelastic" type="list" option.json="on"){ constant(value="{") constant(value="\"PROGRAM_NAME\":\"") property(name="hostname") constant(value="\",\"ENV\":\"") property(name="syslogtag") constant(value="\",\"LOG_TYPE\":\"code_syslog\",") constant(value="\"HOSTNAME\":\"") property(name="$!myhostname") #get host ip constant(value="\",\"TIME\":\"") property(name="timereported" dateFormat="rfc3339") constant(value="\",\"MSG\":\"") property(name="msg") constant(value="\"}") } ### get local nginx log from syslog to elasticsearch #ACCESS:access_log syslog:server=127.0.0.1:514,facility=local1,tag=m_test_2haohr_com,severity=info main; #ERROR:error_log syslog:server=127.0.0.1:514,facility=local1,tag=m_test_2haohr_com,severity=debug; template(name="websitename" type="string" string="%syslogtag:F,66:1%-%$year%.%$month%.%$day%") template(name="nginxelastic" type="list" option.json="on"){ constant(value="{") constant(value="\"PROGRAM_NAME\":\"") property(name="syslogtag") constant(value="\",\"ENV\":\"") property(name="$!env") constant(value="\",\"LOG_TYPE\":\"") property(name="$!logtype") constant(value="\",\"HOSTNAME\":\"") property(name="hostname") #get host ip constant(value="\",\"TIME\":\"") property(name="timereported" dateFormat="rfc3339") constant(value="\",\"MSG\":\"") property(name="msg") constant(value="\"}") } ruleset(name="udpinput") { if prifilt("local0.*") then { action(type="omfile" dynaFile="udppfile" template="timemsg") action(type="omfwd" protocol="tcp" target="10.10.10.10" port="514" zipLevel="5" keepalive="on") set $!myhostname = "10-10-3-215"; action(type="omelasticsearch" server="10.10.166.166" serverport="9200" template="udpelastic" dynSearchIndex="on" searchIndex="udppname" searchType="events") }else if $syslogfacility-text == "local1" then { set $!sitename = $syslogtag; if $syslogtag contains "_dev" then { set $!env = "dev"; } else if $syslogtag contains "_test" then { set $!env = "test"; } else { set $!env = "production"; } if $syslogseverity-text == "info" then { set $!logtype = "nginx_access"; } else if $syslogseverity-text == "debug" then { set $!logtype = "nginx_error"; } action(type="omfwd" protocol="tcp" target="10.10.10.10" port="514" zipLevel="5" keepalive="on") action(type="omelasticsearch" server="10.10.166.166" serverport="9200" template="nginxelastic" dynSearchIndex="on" searchIndex="websitename" bulkmode="on" Action.ResumeRetryCount="-1") } else{ action(type="omfile" file="var/log/test.test.log") stop } } template(name="filepname" type="string" string="%syslogtag:F,43:2%-%$year%.%$month%.%$day%") template(name="fileselasticsearch" type="list" option.json="on"){ constant(value="{") constant(value="\"PROGRAM_NAME\":\"") property(name="$!programname") constant(value="\",\"ENV\":\"") property(name="$!env") constant(value="\",\"LOG_TYPE\":\"") property(name="$!logtype") constant(value="\",\"HOSTNAME\":\"") property(name="hostname") #get host ip constant(value="\",\"TIME\":\"") property(name="timereported" dateFormat="rfc3339") constant(value="\",\"MSG\":\"") property(name="msg") constant(value="\"}") } ruleset(name="fileinput"){ if prifilt("local6.*") then { set $!programname = field($syslogtag,"+",2); if re_match($syslogtag,"G+.*+.*E") then { set $!logtype = "gunicorn_error"; } else if re_match($syslogtag,"G+.*+.*A") then { set $!logtype = "gunicorn_access"; } else if re_match($syslogtag,"S+.*+.*G") then { set $!logtype = "gunicorn_supervisor"; } else if re_match($syslogtag,"C+.*+.*C") then { set $!logtype = "celery_log"; } else if re_match($syslogtag,"S+.*+.*C") then { set $!logtype = "celery_supervisor"; } if $syslogtag contains "D" then { set $!env = "dev"; } else if $syslogtag contains "T" then { set $!env = "test"; } else if $syslogtag contains "P" then { set $!env = "production"; } action(type="omfwd" protocol="tcp" target="10.10.10.10" port="514" zipLevel="5" keepalive="on") action(type="omelasticsearch" server="10.10.166.166" serverport="9200" template="fileselasticsearch" dynSearchIndex="on" searchIndex="filepname" bulkmode="on" Action.ResumeRetryCount="-1") } stop }
使用 ansible-playbook 替換所有機器
--- - hosts: GZtest remote_user: root vars: rsyslog_port: 514 protocol: udp tasks: - name: add rsyslogd repository yum_repository: name: rsyslog_v8 description: Adiscon CentOS-$releasever - local packages for $basearch baseurl: http://rpms.adiscon.com/v8-stable/epel-$releasever/$basearch enabled: yes gpgcheck: no gpgkey: http://rpms.adiscon.com/RPM-GPG-KEY-Adiscon file: rsyslog protect: yes - name: install rsyslog-main yum: name: rsyslog state: latest - name: install rsyslog-elastic module yum: name: rsyslog-elasticsearch state: latest - name: copy rsyslog.conf copy: backup: yes src: /etc/rsyslog.d/rsyslog_v8_client.conf dest: /etc/rsyslog.conf # shell: sed -i "s/222/{{ansible_hostname}}/g" /root/shell/test.txt - name: change localhost replace: dest: /etc/rsyslog.conf regexp: ‘10-10-3-215‘ replace: ‘{{ansible_hostname}}‘ notify: - restart rsyslogd handlers: - name: restart rsyslogd service: name: rsyslog enabled: yes state: restarted
其他:
elasticsearsh安裝配置
單機es存儲日誌量有限,可以考慮保存最近一個禮拜的,其他的刪除,有需要去文件中查看。
考慮添加隊列,提升es實時性能
kibana安裝配置
權限不好控制,暫時無法通過插件限制開發只使用discovery模塊。
安全很重要,至少要通過Basic HTTP authentication驗證。
效果
elasticsearch中保存的index
kibana查看日誌:
日誌收集三:rsyslog + elasticsearch + kibana