#配置flume conf
# Please paste flume.conf here. Example:
# Sources, channels, and sinks are defined per
# agent name, in this case 'tier1'.
tier1.sources = source1
tier1.channels = channel1
tier1.sinks = sink1
# For each source, channel, and sink, set
# standard properties.
tier1.sources.source1.type = exec
tier1.sources.source1.shell = /bin/bash -c
# 要扫描的文件
tier1.sources.source1.command = sh /xxx/flume/locktail_rotate.sh /xxx/buried/weChat/DATE_ROTATE.log 'date +"%Y%m%d"'
# standard channels.
tier1.sources.source1.channels = channel1
tier1.channels.channel1.type = memory
tier1.channels.channel1.keep-alive = 60
tier1.channels.channel1.capacity = 1000000
# standard sinks.
tier1.sinks.sink1.channel = channel1
tier1.sinks.sink1.type = org.apache.flume.sink.kafka.KafkaSink
tier1.sinks.sink1.brokerList=xxx.xxx.xxx.xxx:9092,xxx.xxx.xxx.xxx:9092,xxx.xxx.xxx.xxx:9092
tier1.sinks.sink1.partition.key=0
tier1.sinks.sink1.partitioner.class=org.apache.flume.plugins.SinglePartition
tier1.sinks.sink1.serializer.class=kafka.serializer.StringEncoder
tier1.sinks.sink1.request.required.acks=0
tier1.sinks.sink1.max.message.size=1000000
tier1.sinks.sink1.producer.type=sync
tier1.sinks.sink1.encoding=UTF-8
tier1.sinks.sink1.topic=test002
---------------------------------------监控每天按日期滚动的日志文件---------------------------------------
vi locktail_rotate.sh
#!/bin/bash
#
# DATE: Mar 30 2019
# DESC: lock a rotating file(filename rotated by date) and tail
#
PID=$( mktemp )
while true;
do
CURRENT_PATTERN=$( eval "$2")
CURRENT_PATH="$1"
CURRENT_TARGET=$( echo ${CURRENT_PATH} | sed "s/DATE_ROTATE/$CURRENT_PATTERN/g" )
# echo $CURRENT_TARGET
if [ -e ${CURRENT_TARGET} ]; then
IO=`stat -c %i ${CURRENT_TARGET}`
tail -f ${CURRENT_TARGET} 2> /dev/null &
echo $! > ${PID}
echo $CURRENT_TARGET >> /mydata/program/flume/note.log
fi
# as long as the file exists and the inode number did not change
while [[ -e ${CURRENT_TARGET} ]] && [[ ${IO} = `stat -c %i ${CURRENT_TARGET}` ]]
do
CURRENT_PATTERN=$( eval "$2")
CURRENT_PATH="$1"
CURRENT_TARGET=$( echo ${CURRENT_PATH} | sed "s/DATE_ROTATE/$CURRENT_PATTERN/g" )
#echo $CURRENT_TARGET
sleep 0.5
done
# echo "[ ! -z $PID ] && kill `cat $PID` 2> /dev/null && echo > $PID"
if [ ! -z ${PID} ]; then
kill `cat ${PID}` 2> /dev/null && echo > ${PID}
fi
sleep 0.5
done 2> /dev/null
rm -rf ${PID}