http://mirrors.tuna.tsinghua.edu.cn/apache/flume/1.8.0/apache-flume-1.8.0-bin.tar.gz
1.将压缩包放在ubunta下
cd ~
tar -zxvf apache-flume-1.8.0-bin.tar.gz -C ~
ln -s apache-flume-1.8.0-bin/ flume
vi ~/.bashrc
source ~/.bashrc
4.配置环境变量
export FLUME_HOME=/home/ubuntu/flume
export PATH=$FLUME_HOME/bin:$PATH
flume -ng version
5.创建文件并打开
mkdir /home/ubuntu/flume/agent
mkdir ~/flume/agent
vi /home/ubuntu/flume/agent/agent1.conf
vi ~/flume/agent/agent1.conf
6.输入内容
agent1.sources=source1
agent1.channels=channel1
agent1.sinks=sink1
agent1.channels.channel1.type=memory
agent1.sources.source1.channels=channel1
agent1.sources.source1.type=spooldir
#mkdir flumetest1
agent1.sources.source1.spoolDir=/home/ubuntu/flumetest1
agent1.sinks.sink1.channel=channel1
agent1.sinks.sink1.type=logger
7.启动Flume Agent
两种方式都可以
flume-ng agent -n agent1 -c /home/ubuntu/flume/conf -f /home/ubuntu/flume/agent/agent1.conf -Dflume.root.logger=INFO,console
flume-ng agent -n agent1 -c ~/flume/conf -f ~/flume/agent/agent1.conf -Dflume.root.logger=INFO,console
8.模拟生成日志文件
echo “hello Flume.” > text.log
二.flume的配置与案例2
1.vi ~/flume/agent/agent2.conf
2.配制文件
agent2.channels=channel2
agent2.sinks=sink2
agent2.channels.channel2.type=memory
agent2.channels.channel2.capacity=10000
agent2.channels.channel2.transactionCapacity=100
agent2.sources.source2.channels=channel2
agent2.sources.source2.type=spooldir
agent2.sources.source2.spoolDir=/home/ubuntu/flume/jill
agent2.sinks.sink2.channel=channel2
agent2.sinks.sink2.type=hdfs
agent2.sinks.sink2.hdfs.path=hdfs://localhost:8020/flume/%Y%m%d
agent2.sinks.sink2.hdfs.filePrefix=events-
agent2.sinks.sink2.hdfs.fileType=DataStream
agent2.sinks.sink2.hdfs.rollCount=0
agent2.sinks.sink2.hdfs.rollSize=134217728
agent2.sinks.sink2.hdfs.rollInterval=60
agent2.sinks.sink2.hdfs.useLocalTimeStamp=true
flume-ng agent -n agent2 -c /home/ubuntu/flume/conf -f /home/ubuntu/flume/agent/agent2.conf -Dflume.root.logger=INFO,console
echo "hello Flume." > text.log
实验1 Flume的配置与使用1–Avro Source + Memory Channel + Logger Sink 211
a1.sources=r1
a1.sinks=k1
a1.channels=c1
a1.sources.r1.type=avro
a1.sources.r1.bind=0.0.0.0
a1.sources.r1.port=4141
a1.sinks.k1.type=logger
a1.channels.c1.type=memory
a1.channels.c1.capacity=1000
a1.channels.c1.transactionCapacity=100
a1.sources.r1.channels=c1
a1.sinks.k1.channel=c1
flume-ng agent --conf /home/ubuntu/flume/conf/ --conf-file /home/ubuntu/flume/conf/avro.conf --name a1 -Dflume.root.logger=INFO,console
echo "hello word" > /home/ubuntu/flume/log.00
flume-ng avro-client -c /home/ubuntu/flume/conf/ -H 0.0.0.0 -p 4141 -F /home/ubuntu/flume/log.00
实验2 Flume的配置与使用2–Syslogtcp Source + Memory Channel + HDFS Sink 212
a1.sources=r1
a1.sinks=k1
a1.channels=c1
a1.sources.r1.type=syslogtcp
a1.sources.r1.port=5140
a1.sources.r1.host=localhost
a1.sinks.k1.type=hdfs
a1.sinks.k1.hdfs.path=hdfs://192.168.30.128:8020/user/flume/syslogtcp
a1.sinks.k1.hdfs.filePrefix=Syslog
a1.sinks.k1.hdfs.round=true
a1.sinks.k1.roundValue=10
a1.sinks.k1.roundUnit=minute
a1.channels.c1.type=memory
a1.channels.c1.capacity=1000
a1.channels.c1.transactionCapacity=100
a1.sources.r1.channels=c1
a1.sinks.k1.channel=c1
flume-ng agent -c /home/ubuntu/flume/conf/ -f /home/ubuntu/flume/conf/syslogtcp.conf -n a1 -Dflume.root.logger=INFO,console
echo "hello syslogtcp" | nc localhost 5140