flume常用操作conf代码

一、数据写入到kafka中

event_attendees.sources = event_attendeesSource
event_attendees.channels = event_attendeesChannel
event_attendees.sinks = event_attendeesSink

event_attendees.sources.event_attendeesSource.type = spooldir
event_attendees.sources.event_attendeesSource.spoolDir = /opt/data/kb07/event_attendees
event_attendees.sources.event_attendeesSource.deserializer = LINE
event_attendees.sources.event_attendeesSource.deserializer.maxLineLength = 60000
event_attendees.sources.event_attendeesSource.includePattern = event_attendees_[0-9]{4}-[0-9]{2}-[0-9]{2}.csv

event_attendees.channels.event_attendeesChannel.type = file
event_attendees.channels.event_attendeesChannel.checkpointDir = /opt/data/kb07/flumeFile/checkpoint/event_attendees
event_attendees.channels.event_attendeesChannel.dataDir = /opt/data/kb07/flumeFile/data/event_attendees

event_attendees.sinks.event_attendeesSink.type = org.apache.flume.sink.kafka.KafkaSink
event_attendees.sinks.event_attendeesSink.batchSize = 640
event_attendees.sinks.event_attendeesSink.brokerList = 192.168.255.11:9092
event_attendees.sinks.event_attendeesSink.topic = event_attendees

event_attendees.sources.event_attendeesSource.channels = event_attendeesChannel
event_attendees.sinks.event_attendeesSink.channel = event_attendeesChannel

二、数据写入到hdfs中

user_friends.sources = userFriendsSources
user_friends.channels = userFriendsChannel
user_friends.sinks = userFriendsSink

user_friends.sources.userFriendsSources.type = spooldir
user_friends.sources.userFriendsSources.spoolDir = /opt/data/kb07/user_friends
user_friends.sources.userFriendsSources.deserializer = LINE
user_friends.sources.userFriendsSources.deserializer.maxLineLength = 600000
user_friends.sources.userFriendsSources.includePattern = user_friends_[0-9]{4}_[0-9]{2}_[0-9]{2}.csv

user_friends.channels.userFriendsChannel.type = file
user_friends.channels.userFriendsChannel.checkpointDir = /opt/data/kb07/flumeFile/checkpoint/userFriends
user_friends.channels.userFriendsChannel.dataDirs = /opt/data/kb07/flumeFile/data/userFriends

user_friends.sinks.userFriendsSink.type = hdfs
user_friends.sinks.userFriendsSink.hdfs.fileType = DataStream
user_friends.sinks.userFriendsSink.hdfs.filePrefix = userFriend
user_friends.sinks.userFriendsSink.hdfs.fileSuffix = .csv
user_friends.sinks.userFriendsSink.hdfs.path = hdfs://192.168.255.11:9000/user/userFriend/%Y-%m-%d
user_friends.sinks.userFriendsSink.hdfs.useLocalTimeStamp = true
user_friends.sinks.userFriendsSink.hdfs.batchSize = 640
user_friends.sinks.userFriendsSink.hdfs.rollCount = 0
user_friends.sinks.userFriendsSink.hdfs.rollSize = 64000000
user_friends.sinks.userFriendsSink.hdfs.rollInterval = 30

user_friends.sources.userFriendsSources.channels = userFriendsChannel
user_friends.sinks.userFriendsSink.channel = userFriendsChannel