flume开发环境搭建

有朋友提问flume开发环境怎么搭建的，给个pom文件放着，只能帮到这了

[html]  view plain  copy

 

1. <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"  
2.   xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">  
3.   <modelVersion>4.0.0</modelVersion>  
4.   
5.   <groupId>com.hupu.dace</groupId>  
6.   <artifactId>flume-plugins</artifactId>  
7.   <version>1.0</version>  
8.   <packaging>jar</packaging>  
9.   
10.   <name>kafka message key interceptor</name>  
11.   <url>http://maven.apache.org</url>  
12.   
13.   <!-- 基于项目配置私服信息 -->  
14.   <repositories>  
15.     <repository>  
16.       <id>cloudera</id>  
17.       <url>https://repository.cloudera.com/artifactory/cloudera-repos/</url>  
18.     </repository>  
19.   </repositories>  
20.   
21.   
22.   
23.   <properties>  
24.     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>  
25.   </properties>  
26.   
27.   <dependencies>  
28.     <dependency>  
29.       <groupId>junit</groupId>  
30.       <artifactId>junit</artifactId>  
31.       <version>3.8.1</version>  
32.       <scope>test</scope>  
33.     </dependency>  
34.     <dependency>  
35.       <groupId>org.apache.hadoop</groupId>  
36.       <artifactId>hadoop-common</artifactId>  
37.       <version>2.5.0-cdh5.3.0</version>  
38.       <!--<scope>provided</scope>-->  
39.     </dependency>  
40.     <!--<dependency>-->  
41.       <!--<groupId>org.apache.hadoop</groupId>-->  
42.       <!--<artifactId>hadoop-hdfs</artifactId>-->  
43.       <!--<version>2.5.0-cdh5.3.0</version>-->  
44.       <!--<scope>provided</scope>-->  
45.     <!--</dependency>-->  
46.     <dependency>  
47.       <groupId>org.apache.flume</groupId>  
48.       <artifactId>flume-ng-core</artifactId>  
49.       <version>1.5.0-cdh5.3.0</version>  
50.       <scope>provided</scope>  
51.     </dependency>  
52.   
53.     <dependency>  
54.       <groupId>org.apache.flume.flume-ng-sources</groupId>  
55.       <artifactId>flume-kafka-source</artifactId>  
56.       <version>1.5.0-cdh5.3.0</version>  
57.       <scope>provided</scope>  
58.     </dependency>  
59.     <dependency>  
60.       <groupId>org.jvnet.hudson</groupId>  
61.       <artifactId>ganymed-ssh2</artifactId>  
62.       <version>build210-hudson-1</version>  
63.     </dependency>  
64.   
65.   </dependencies>  
66.   <build>  
67.     <plugins>  
68.       <plugin>  
69.         <groupId>org.apache.maven.plugins</groupId>  
70.         <artifactId>maven-dependency-plugin</artifactId>  
71.         <executions>  
72.           <execution>  
73.             <id>copy</id>  
74.             <phase>package</phase>  
75.             <goals>  
76.               <goal>copy-dependencies</goal>  
77.             </goals>  
78.             <configuration>  
79.               <outputDirectory>  
80.                 ${project.build.directory}/lib/  
81.               </outputDirectory>  
82.               <includeScope>compile</includeScope>  
83.               <includeArtifactIds>ganymed-ssh2</includeArtifactIds>  
84.             </configuration>  
85.           </execution>  
86.         </executions>  
87.       </plugin>  
88.     </plugins>  
89.     <pluginManagement>  
90.       <plugins>  
91.         <plugin>  
92.           <groupId>org.apache.maven.plugins</groupId>  
93.           <artifactId>maven-compiler-plugin</artifactId>  
94.           <version>2.3.2</version>  
95.           <configuration>  
96.             <source>1.6</source>  
97.             <target>1.6</target>  
98.           </configuration>  
99.         </plugin>  
100.       </plugins>  
101.     </pluginManagement>  
102.   </build>  
103. </project>  

再给个自定义拦截器实现：

[java]  view plain  copy

 

1. import com.google.common.collect.Lists;  
2. import org.apache.flume.Context;  
3. import org.apache.flume.Event;  
4. import org.apache.flume.interceptor.Interceptor;  
5.   
6. import java.util.List;  
7. import java.util.Map;  
8.   
9. /** 
10.  *   该拦截器将kafka message的key按照指定分隔符分隔成一个一个的event header key,在sink中可以通过%{}方式来引用 
11.  */  
12. public class KafkaMessageKeyInterceptor implements Interceptor {  
13.   
14.     private final String splitChar;  
15.   
16.     /** 
17.      * Only {@link com.hupu.dace.flume.interceptors.KafkaMessageKeyInterceptor.Builder} can build me 
18.      */  
19.     private KafkaMessageKeyInterceptor(String splitChar) {  
20.         this.splitChar = splitChar;  
21.     }  
22.   
23.     public void initialize() {  
24.         // no-op  
25.     }  
26.   
27.     /** 
28.      * Modifies events in-place. 
29.      */  
30.     public Event intercept(Event event) {  
31.         Map<String, String> headers = event.getHeaders();  
32.         if (headers.containsKey(Constants.KAFKA_MESSAGE_KEY)) {  
33.             String kafkaMessageKey = headers.get(Constants.KAFKA_MESSAGE_KEY); //从header中获取kafka message key  
34.             String[] values = kafkaMessageKey.split(splitChar); //按照指定的分隔符将key拆分  
35.   
36.             for (int i = 0; i < values.length; i++) {  
37.                 headers.put(Constants.SPLIT_KEY_PREFIX + i, values[i]);//拆分出来的每一个片段，按指定的前缀加上序号，写入event header  
38.             }  
39.             return event;  
40.   
41.         } else {  
42.             return null;  
43.         }  
44.     }  
45.   
46.     /** 
47.      * Delegates to {@link #intercept(Event)} in a loop. 
48.      * 
49.      * @param events 
50.      * @return 
51.      */  
52.     public List<Event> intercept(List<Event> events) {  
53.         List<Event> out = Lists.newArrayList();  
54.         for (Event event : events) {  
55.             Event outEvent = intercept(event);  
56.             if (outEvent != null) {  
57.                 out.add(outEvent);  
58.             }  
59.         }  
60.         return out;  
61.     }  
62.   
63.     public void close() {  
64.         // no-op  
65.     }  
66.   
67.   
68.     public static class Builder implements Interceptor.Builder {  
69.   
70.         private String splitChar = Constants.DEFAULT_SPLIT_CHAR;  
71.   
72.         public Interceptor build() {  
73.             return new KafkaMessageKeyInterceptor(splitChar);  
74.         }  
75.   
76.         public void configure(Context context) {  
77.             splitChar = context.getString(Constants.SPLIT_CHAR, Constants.DEFAULT_SPLIT_CHAR);  
78.         }  
79.     }  
80.   
81.     public static class Constants {  
82.         public static String KAFKA_MESSAGE_KEY = "key";   //kafka source会将message key写进event header中一个叫"key"的header  
83.         public static String SPLIT_CHAR = "split";  
84.         public static String DEFAULT_SPLIT_CHAR = ":";  
85.         public static String SPLIT_KEY_PREFIX = "s";  
86.     }  
87.   

1. }  

http://blog.csdn.net/xiao_jun_0820/article/details/50628349