flume_tail_日期轮询

 

#配置flume conf

 

# Please paste flume.conf here. Example:

# Sources, channels, and sinks are defined per
# agent name, in this case 'tier1'.
tier1.sources  = source1
tier1.channels = channel1
tier1.sinks    = sink1

# For each source, channel, and sink, set
# standard properties.
tier1.sources.source1.type = exec
tier1.sources.source1.shell = /bin/bash -c
# 要扫描的文件
tier1.sources.source1.command = sh /xxx/flume/locktail_rotate.sh /xxx/buried/weChat/DATE_ROTATE.log 'date +"%Y%m%d"'

# standard channels.
tier1.sources.source1.channels = channel1
tier1.channels.channel1.type   = memory
tier1.channels.channel1.keep-alive = 60
tier1.channels.channel1.capacity = 1000000

# standard sinks.
tier1.sinks.sink1.channel = channel1
tier1.sinks.sink1.type = org.apache.flume.sink.kafka.KafkaSink
tier1.sinks.sink1.brokerList=xxx.xxx.xxx.xxx:9092,xxx.xxx.xxx.xxx:9092,xxx.xxx.xxx.xxx:9092
tier1.sinks.sink1.partition.key=0
tier1.sinks.sink1.partitioner.class=org.apache.flume.plugins.SinglePartition
tier1.sinks.sink1.serializer.class=kafka.serializer.StringEncoder
tier1.sinks.sink1.request.required.acks=0
tier1.sinks.sink1.max.message.size=1000000
tier1.sinks.sink1.producer.type=sync
tier1.sinks.sink1.encoding=UTF-8
tier1.sinks.sink1.topic=test002
 

 

 

---------------------------------------监控每天按日期滚动的日志文件---------------------------------------

vi locktail_rotate.sh

#!/bin/bash

#
# DATE:   Mar 30 2019
# DESC:   lock a rotating file(filename rotated by date) and tail
#

PID=$( mktemp )
while true;
do
    CURRENT_PATTERN=$( eval "$2")
    CURRENT_PATH="$1"
    CURRENT_TARGET=$( echo ${CURRENT_PATH} | sed "s/DATE_ROTATE/$CURRENT_PATTERN/g" )
    # echo $CURRENT_TARGET
    if [ -e ${CURRENT_TARGET} ]; then
        IO=`stat -c %i ${CURRENT_TARGET}`
        tail -f ${CURRENT_TARGET} 2> /dev/null &
        echo $! > ${PID}
        echo $CURRENT_TARGET >> /mydata/program/flume/note.log
    fi

    # as long as the file exists and the inode number did not change
    while [[ -e ${CURRENT_TARGET} ]] && [[ ${IO} = `stat -c %i ${CURRENT_TARGET}` ]]
    do
        CURRENT_PATTERN=$( eval "$2")
        CURRENT_PATH="$1"
        CURRENT_TARGET=$( echo ${CURRENT_PATH} | sed "s/DATE_ROTATE/$CURRENT_PATTERN/g" )
        #echo $CURRENT_TARGET
        sleep 0.5
    done
    # echo "[ ! -z $PID ] && kill `cat $PID` 2> /dev/null && echo > $PID"
    if [ ! -z ${PID} ]; then
        kill `cat ${PID}` 2> /dev/null && echo > ${PID}
    fi
    sleep 0.5
done 2> /dev/null
rm -rf ${PID}