在CentOS6.9中 搭建 Flume

之前的基本环境准备我就不再说了,请参照我之前的搭建hadoop的文章

在CentOS 6.9中搭建Hadoop

配置 flume 环境变量

export FLUME_HOME=/opt/apache-flume-1.7.0-bin
export PATH=$PATH:$JAVA_HOME/bin:$HADOOP_HOME/bin:$FLUME_HOME/bin:$HOME/bin

然后 记得 source ~/.bash_profile

根据需求，配置不同的 source／channel／sink，添加配置文件到 conf/中

• flume_exec_hdfs.conf

logAgent.sources = logSource
logAgent.channels = fileChannel
logAgent.sinks = hdfsSink

logAgent.sources.logSource.type = exec
logAgent.sources.logSource.command = tail -F /aura/data/flume-search/logs
logAgent.sources.logSource.channels = fileChannel

logAgent.sinks.hdfsSink.type = hdfs
logAgent.sinks.hdfsSink.hdfs.path = hdfs://bigdata:9000/flume/record/%Y-%m-%d/%H%M
logAgent.sinks.hdfsSink.hdfs.rollCount= 10000
logAgent.sinks.hdfsSink.hdfs.rollSize= 0
logAgent.sinks.hdfsSink.hdfs.batchSize= 1000
logAgent.sinks.hdfsSink.hdfs.filePrefix= transaction_log
logAgent.sinks.hdfsSink.hdfs.rollInterval= 600
logAgent.sinks.hdfsSink.hdfs.roundUnit = minute
logAgent.sinks.hdfsSink.hdfs.fileType = DataStream
logAgent.sinks.hdfsSink.hdfs.useLocalTimeStamp = true
logAgent.sinks.hdfsSink.channel = fileChannel

logAgent.channels.fileChannel.type = memory
logAgent.channels.logSource.capacity=1000
logAgent.channels.logSource.transactionCapacity=100

• flume_avro_hdfs.conf

logAgent.sources = logSource
logAgent.channels = fileChannel
logAgent.sinks = hdfsSink

logAgent.sources.logSource.type = avro
logAgent.sources.logSource.bind = 127.0.0.1
logAgent.sources.logSource.port = 44444
logAgent.sources.logSource.channels = fileChannel

logAgent.sinks.hdfsSink.type = hdfs
logAgent.sinks.hdfsSink.hdfs.path = hdfs://bigdata:9000/flume/record/%Y-%m-%d/%H%M
logAgent.sinks.hdfsSink.hdfs.rollCount= 10000
logAgent.sinks.hdfsSink.hdfs.rollSize= 0
logAgent.sinks.hdfsSink.hdfs.batchSize= 1000
logAgent.sinks.hdfsSink.hdfs.filePrefix= transaction_log
logAgent.sinks.hdfsSink.hdfs.rollInterval= 600
logAgent.sinks.hdfsSink.hdfs.roundUnit = minute
logAgent.sinks.hdfsSink.hdfs.fileType = DataStream
logAgent.sinks.hdfsSink.hdfs.useLocalTimeStamp = true
logAgent.sinks.hdfsSink.channel = fileChannel

logAgent.channels.fileChannel.type = memory
logAgent.channels.logSource.capacity=1000
logAgent.channels.logSource.transactionCapacity=100

• flume_dir_hdfs.conf

logAgent.sources = logSource
logAgent.channels = fileChannel
logAgent.sinks = hdfsSink

logAgent.sources.logSource.type = spooldir
logAgent.sources.logSource.spoolDir =/aura/data/flume-search
logAgent.sources.logSource.channels = fileChannel

logAgent.sinks.hdfsSink.type = hdfs
logAgent.sinks.hdfsSink.hdfs.path = hdfs://bigdata:9000/flume/record/%Y-%m-%d/%H%M
logAgent.sinks.hdfsSink.hdfs.rollCount= 10000
logAgent.sinks.hdfsSink.hdfs.rollSize= 0
logAgent.sinks.hdfsSink.hdfs.batchSize= 1000
logAgent.sinks.hdfsSink.hdfs.filePrefix= transaction_log
logAgent.sinks.hdfsSink.hdfs.rollInterval= 600
logAgent.sinks.hdfsSink.hdfs.roundUnit = minute
logAgent.sinks.hdfsSink.hdfs.fileType = DataStream
logAgent.sinks.hdfsSink.hdfs.useLocalTimeStamp = true
logAgent.sinks.hdfsSink.channel = fileChannel

logAgent.channels.fileChannel.type = memory
logAgent.channels.logSource.capacity=1000
logAgent.channels.logSource.transactionCapacity=100

bin/flume-ng agent -n logAgent -c conf -f conf/flume_exec_hdfs.conf -Dflume.root.logger=INFO,console