flume 配置使用
阿新 • • 發佈：2018-05-09
flume安裝使用flume配置獲取信息傳輸到kafka集群
conf目錄下新建配置文件
[root@localhost flume]# vim conf/file-monitor.conf
# 聲明agent
a1.sources = r1
a1.sinks = k1
a1.channels = c1
# 定義數據源
a1.sources.r1.type = exec
a1.sources.r1.command = tail -F /data/xx.log
a1.sources.r1.channels = c1
# filter過濾器
a1.sources.r1.interceptors=i1
a1.sources.r1.interceptors.i1.type=regex_filter
#a1.sources.r1.interceptors.i1.regex=(Parsing events)(.*)(END)
a1.sources.r1.interceptors.i1.regex=(aaaa)(.*) #只有匹配到才傳輸到channels
#定義event暫存位置，可以使內存，磁盤，數據庫等
a1.channels.c1.type = file
a1.channels.c1.checkpointDir = /data/flume/chk
a1.channels.c1.dataDirs = /data/flume/data
# 定義數據流向kafka
#a1.sinks.k1.type = logger
a1.sinks.k1.type = org.apache.flume.sink.kafka.KafkaSink
a1.sinks.k1.brokerList = 192.168.41.47:9092,192.168.41.127:9092,192.168.41.86:9092
a1.sinks.k1.topic = mytopic
#a1.sinks.k1.requiredAcks = 1
#a1.sinks.k1.batchSize = 20
a1.sinks.k1.serializer.class=kafka.serializer.StringEncoder
a1.sinks.k1.channel = c1
啟動
[root@localhost flume]# nohup bin/flume-ng agent -n a1 -c conf/ -f conf/file-monitor.conf -Dflume.root.logger=INFO,console > nohup.out 2>&1 &
flume 配置使用