Storm 本地模式

本地模式，是在eclipse等编译器编写strom运行文件，在于模拟storm在集群运行的结果，便于代码的编写和调试。

一、下载开发环境的zip文件，将storm相关jar包导入编辑器。注意是zip文件，不是gz运行linux文件。
http://storm.apache.org/downloads.html

二、创建数据文件
[b]storm有Tail属性，只适合文本源，会对文本文件内容进行监听。[/b]

public class GetData {

	/**
	 * 生成数据
	 * @param args
	 */
	public static void main(String[] args) {
		File logFile = new File("track.log");
		Random random = new Random();

		String[] hosts = { "www.baidu.com" };
		String[] session_id = { "ABYH6Y4V4SCVXTG6DPB4VH9U123", "XXYH6YCGFJYERTT834R52FDXV9U34", "BBYH61456FGHHJ7JL89RG5VV9UYU7",
				"CYYH6Y2345GHI899OFG4V9U567", "VVVYH6Y4V4SFXZ56JIPDPB4V678" };
		String[] time = { "2014-01-07 08:40:50", "2014-01-07 08:40:51", "2014-01-07 08:40:52", "2014-01-07 08:40:53", 
				"2014-01-07 09:40:49", "2014-01-07 10:40:49", "2014-01-07 11:40:49", "2014-01-07 12:40:49" };

		StringBuffer sbBuffer = new StringBuffer() ;
		for (int i = 0; i < 50; i++) {
			sbBuffer.append(hosts[0]+"\t"+session_id[random.nextInt(5)]+"\t"+time[random.nextInt(8)]+"\n");
		}
		if(! logFile.exists())
		{
			try {
				logFile.createNewFile();
			} catch (IOException e) {
				System.out.println("Create logFile fail !");
			}
		}
		byte[] b = (sbBuffer.toString()).getBytes();

		FileOutputStream fs;
		try {
			fs = new FileOutputStream(logFile);
			fs.write(b);
			fs.close();
		} catch (Exception e) {
			e.printStackTrace();
		}
	}

}

文件格式如下：
[quote]
www.baidu.com XXYH6YCGFJYERTT834R52FDXV9U34 2014-01-07 12:40:49
www.baidu.com ABYH6Y4V4SCVXTG6DPB4VH9U123 2014-01-07 08:40:51
www.baidu.com ABYH6Y4V4SCVXTG6DPB4VH9U123 2014-01-07 08:40:53
www.baidu.com BBYH61456FGHHJ7JL89RG5VV9UYU7 2014-01-07 11:40:49
www.baidu.com XXYH6YCGFJYERTT834R52FDXV9U34 2014-01-07 11:40:49
www.baidu.com XXYH6YCGFJYERTT834R52FDXV9U34 2014-01-07 12:40:49
www.baidu.com ABYH6Y4V4SCVXTG6DPB4VH9U123 2014-01-07 08:40:53
www.baidu.com CYYH6Y2345GHI899OFG4V9U567 2014-01-07 12:40:49
www.baidu.com ABYH6Y4V4SCVXTG6DPB4VH9U123 2014-01-07 10:40:49
www.baidu.com ABYH6Y4V4SCVXTG6DPB4VH9U123 2014-01-07 08:40:52
www.baidu.com BBYH61456FGHHJ7JL89RG5VV9UYU7 2014-01-07 12:40:49
www.baidu.com VVVYH6Y4V4SFXZ56JIPDPB4V678 2014-01-07 12:40:49
www.baidu.com VVVYH6Y4V4SFXZ56JIPDPB4V678 2014-01-07 10:40:49
[/quote]


二、编写读取数据源类MySpout
Open(配置文件，上下文，发射topology类)是初始化方法
nextTuple()循环发射数据
ack() 成功处理tuple回调方法
Fail()处理失败tuple回调方法
activate和deactivate ：spout可以被暂时激活和关闭
close方法在该spout关闭当前执行，但是并不能得到保证其一定被执行。kill -9时不执行，
Storm kill {topoName} 时执行。
superviser、numberis 用Kill -9。Topology一般不用Kill -9。

public class MySpout implements IRichSpout{

	/**
	 * 
	 */
	private static final long serialVersionUID = 1L;

	FileInputStream fis;
	InputStreamReader isr;
	BufferedReader br;			

	SpoutOutputCollector collector = null;


	String str = null;


	public void nextTuple() {
		try {
			while ((str = this.br.readLine()) != null) {
				// 过滤动作

				collector.emit(new Values(str));

//				Thread.sleep(3000);
				//to do 
			}
		} catch (Exception e) {
			// TODO: handle exception
		}


	}


	public void open(Map conf, TopologyContext context,
			SpoutOutputCollector collector) {
		try {
			this.collector = collector;
			this.fis = new FileInputStream("track.log");
			this.isr = new InputStreamReader(fis, "UTF-8");
			this.br = new BufferedReader(isr);
		} catch (Exception e) {
			e.printStackTrace();
		}

		// 打开文件

	}


	public void declareOutputFields(OutputFieldsDeclarer declarer) {
		// 发射数据格式,与bolt接收数据一致
		declarer.declare(new Fields("loog"));
	}


	public Map<String, Object> getComponentConfiguration() {
		// 与ope方法中的map对应
		return null;
	}


	public void ack(Object msgId) {
		// TODO Auto-generated method stub

	}


	public void activate() {
		// TODO Auto-generated method stub

	}


	public void close() {
		// TODO Auto-generated method stub

	}


	public void deactivate() {
		// TODO Auto-generated method stub

	}


	public void fail(Object msgId) {
		// TODO Auto-generated method stub

	}

}

三、编写数据处理类
prepare方法进行初始化，传入当前执行的上下文,与Spout的open方法一样
execute接受一个tuple进行处理，也可emit数据到下一级组件
cleanup 同ISpout的close方法，在关闭前调用，不保证其一定执行。

public class MyBolt implements IRichBolt {


	private static final long serialVersionUID = 1L;
	OutputCollector collector = null;

	public void cleanup() {

	}

	int num = 0;
	String valueString = null;

	public void execute(Tuple input) {
//		input.getValueByField("log");
//		input.getValue(0);
		try {
			valueString = input.getStringByField("loog") ;

			if(valueString != null)
			{
				num ++ ;
				System.err.println("lines  :"+num +"   session_id:"+valueString.split("\t")[1]);
			}
			collector.ack(input);
			Thread.sleep(2000);
		} catch (Exception e) {
			collector.fail(input);
			e.printStackTrace();
		}

	}


	public void prepare(Map stormConf, TopologyContext context,
			OutputCollector collector) {
		this.collector = collector ;
	}


	public void declareOutputFields(OutputFieldsDeclarer declarer) {
		declarer.declare(new Fields("")) ;
	}


	public Map<String, Object> getComponentConfiguration() {
		return null;
	}

}

四、构建topology结构并运行

public class Main {

	public static void main(String[] args) {

		TopologyBuilder builder = new TopologyBuilder();

		builder.setSpout("spout", new MySpout(), 1);
		builder.setBolt("bolt", new MyBolt(), 1).shuffleGrouping("spout");//与builder中的spout名称对应

		Map conf = new HashMap();
		conf.put(Config.TOPOLOGY_WORKERS, 4);

		if (args.length > 0) {
			try {
					StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
			} catch (AlreadyAliveException e) {
				e.printStackTrace();
			} catch (InvalidTopologyException e) {
				e.printStackTrace();
			}catch (AuthorizationException e) {
				e.printStackTrace();
			}
		}else {
			LocalCluster localCluster = new LocalCluster();
			localCluster.submitTopology("mytopology", conf, builder.createTopology());
		}

	}

}

六、如果maven项目的pom.xml文件
[quote]
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.test</groupId>
<artifactId>StormMavenProject</artifactId>
<packaging>jar</packaging>
<version>0.0.1-SNAPSHOT</version>
<name>StormMavenProject</name>
<url>http://maven.apache.org</url>
<dependencies>

<dependency>
<groupId>org.ow2.asm</groupId>
<artifactId>asm</artifactId>
<version>5.0.3</version>
</dependency>
<dependency>
<groupId>org.clojure</groupId>
<artifactId>clojure</artifactId>
<version>1.7.0</version>
</dependency>
<dependency>
<groupId>com.lmax</groupId>
<artifactId>disruptor</artifactId>
<version>3.3.2</version>
</dependency>
<dependency>
<groupId>com.esotericsoftware</groupId>
<artifactId>kryo</artifactId>
<version>3.0.3</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
<version>2.8</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
<version>2.8</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>log4j-over-slf4j</artifactId>
<version>1.6.6</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-slf4j-impl</artifactId>
<version>2.8</version>
</dependency>
<dependency>
<groupId>com.esotericsoftware</groupId>
<artifactId>minlog</artifactId>
<version>1.3.0</version>
</dependency>
<dependency>
<groupId>org.objenesis</groupId>
<artifactId>objenesis</artifactId>
<version>2.1</version>
</dependency>
<dependency>
<groupId>com.esotericsoftware</groupId>
<artifactId>reflectasm</artifactId>
<version>1.10.1</version>
</dependency>

<dependency>
<groupId>javax.servlet</groupId>
<artifactId>servlet-api</artifactId>
<version>2.5</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>1.7.21</version>
</dependency>
<dependency>
<groupId>org.apache.storm</groupId>
<artifactId>storm-core</artifactId>
<version>1.1.0</version>
</dependency>
<dependency>
<groupId>org.apache.storm</groupId>
<artifactId>storm-rename-hack</artifactId>
<version>1.1.0</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>3.8.1</version>
<scope>test</scope>
</dependency>

<dependency>
<groupId>ring-cors</groupId>
<artifactId>ring-cors</artifactId>
<version>0.1.5</version>
</dependency>

</dependencies>
<build>
<finalName>StormMavenProject</finalName>
</build>
</project>
[/quote]
七、运行结果
[quote]
lines :2 session_id:ABYH6Y4V4SCVXTG6DPB4VH9U123
lines :3 session_id:ABYH6Y4V4SCVXTG6DPB4VH9U123
lines :4 session_id:BBYH61456FGHHJ7JL89RG5VV9UYU7
lines :5 session_id:XXYH6YCGFJYERTT834R52FDXV9U34
[/quote]