storm代码练习-transaction

最新推荐文章于 2022-04-19 21:36:07 发布

SunWuKong_Hadoop

最新推荐文章于 2022-04-19 21:36:07 发布

阅读量184

点赞数

分类专栏：大数据 Storm

本文链接：https://blog.csdn.net/SunWuKong_Hadoop/article/details/79713908

版权

大数据同时被 2 个专栏收录

18 篇文章 0 订阅

订阅专栏

Storm

8 篇文章 0 订阅

订阅专栏

package transaction1;

import java.math.BigInteger;
import java.util.HashMap;
import java.util.Map;

import org.apache.storm.coordination.BatchOutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.topology.base.BaseTransactionalBolt;
import org.apache.storm.transactional.ICommitter;
import org.apache.storm.transactional.TransactionAttempt;
import org.apache.storm.tuple.Tuple;

public class MyCommiter extends BaseTransactionalBolt implements ICommitter{
	/**
	 * 
	 */
	private static final long serialVersionUID = 1L;
	public static final String GLOBAL_KEY = "GLOBAL_KEY";
	public static Map<String, DbValue> dbMap = new HashMap<String, DbValue>();
	int sum = 0;
	TransactionAttempt id;
	private BatchOutputCollector collector;

	@Override
	public void execute(Tuple tuple) {
		// TODO Auto-generated method stub
		sum += tuple.getInteger(1);

	}

	@Override
	public void finishBatch() {
		// TODO Auto-generated method stub
		DbValue value = dbMap.get(GLOBAL_KEY);
		DbValue newValue = dbMap.get(GLOBAL_KEY);
		if (value == null || value.txid.equals(id.getTransactionId())){
			//更新数据库
			newValue = new DbValue();
			newValue.txid = id.getTransactionId();
			if (value == null) {
				newValue.count = sum;
			}else {
				
				newValue.count = value.count+ sum;
			}
			dbMap.put(GLOBAL_KEY, newValue);
		}else {
			//把新的值赋给老的值
			newValue = value;
		}
		System.out.println("total=======================:" + dbMap.get(GLOBAL_KEY).count);

	}

	@Override
	public void prepare(Map arg0, TopologyContext arg1, BatchOutputCollector collector, TransactionAttempt id) {
		// TODO Auto-generated method stub
		this.id = id;
		this.collector = collector;

	}

	@Override
	public void declareOutputFields(OutputFieldsDeclarer arg0) {
		// TODO Auto-generated method stub

	}
	public static class DbValue {
		BigInteger txid;
		int count;
		
	}

}

package transaction1;

import java.math.BigInteger;

import org.apache.storm.transactional.ITransactionalSpout.Coordinator;
import org.apache.storm.utils.Utils;

public class MyCondinator implements Coordinator<MyMata>{
	public static int BATCH_NUM = 10;

	@Override
	public void close() {
		// TODO Auto-generated method stub
		
	}

	@Override
	public MyMata initializeTransaction(BigInteger arg0, MyMata prevMetadata) {
		// TODO Auto-generated method stub
		long beginPoint = 0;
		if (prevMetadata == null) {
			beginPoint = 0;
		}else {
			beginPoint = prevMetadata.getbeginPoint() + prevMetadata.getNum();
		} 
		
		MyMata mata = new MyMata();
		mata.setbeginPoint(beginPoint); 
		mata.setNum(BATCH_NUM);
		System.err.println("启动一个事务:"+mata.toString());
		return mata;//返回源数据
	}

	@Override
	public boolean isReady() {
		// TODO Auto-generated method stub
		Utils.sleep(2000);//sleep两秒
		return false;
	}

}

package transaction1;

import java.math.BigInteger;
import java.util.Map;
import java.util.stream.Collector;

import org.apache.storm.coordination.BatchOutputCollector;
import org.apache.storm.shade.com.twitter.chill.config.ConfiguredInstantiator;
import org.apache.storm.transactional.ITransactionalSpout.Emitter;
import org.apache.storm.transactional.TransactionAttempt;
import org.apache.storm.tuple.Values;

public class MyEmitter implements Emitter<MyMata>{
	//记得实现MyEmitter的全部方法
	Map<Long, String> dbMap = null;
	//每次发射的多少量，是根据源数据里面决定的
	public  MyEmitter(Map<Long, String> dbMap) {
		// TODO Auto-generated constructor stub
		this.dbMap = dbMap;
	}

	@Override
	public void cleanupBefore(BigInteger arg0) {
		// TODO Auto-generated method stub
		
	}

	@Override
	public void close() {
		// TODO Auto-generated method stub
		
	}

	@Override
	public void emitBatch(TransactionAttempt tx, MyMata coordinatorMata, BatchOutputCollector collector) {
		//emitBatch就是从数据源中得到数据，源数据MyMata的给它初始位置、数量，然后发出去
		// TODO Auto-generated method stub
		//在此处应用，这里是发射事务的第一列必须是它
		long beginPoint = coordinatorMata.getbeginPoint();
		int num = coordinatorMata.getNum();
		
		for (long i = beginPoint; i < num; i++) {
			collector.emit(new Values(tx,dbMap.get(i)));
			
		}
	}

}

package transaction1;

import java.io.Serializable;

public class MyMata implements Serializable{

	/**
	 * 
	 */
	private static final long serialVersionUID = 1L;
	private long beginPoint;//事务开始位置
	private int num;//batch的touple个数
	
	
	
	@Override
	public String toString() {
		// TODO Auto-generated method stub
		return getbeginPoint()+"----"+getNum();
	}
	public long getbeginPoint() {
		return beginPoint;
	}
	public void setbeginPoint(long beginPoint) {
		this.beginPoint = beginPoint;
	}
	public int getNum() {
		return num;
	}
	public void setNum(int num) {
		this.num = num;
	}

		
	}

package transaction1;

import java.util.Map;

import org.apache.storm.coordination.BatchOutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.topology.base.BaseTransactionalBolt;
import org.apache.storm.transactional.TransactionAttempt;
import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Tuple;
import org.apache.storm.tuple.Values;

public class MyTransactionBolt extends BaseTransactionalBolt {

	/**
	 * 
	 */
	private static final long serialVersionUID = 1L;
	Integer count = 0;

	@Override
	public void execute(Tuple tuple) {
		// TODO Auto-generated method stub
		//execute会从emitter接到每一行进行处理，处理完一批，到finishBatch
		TransactionAttempt tx = (TransactionAttempt)tuple.getValue(0);
		System.err.println("MyTransactionBolt TransactionAttempt " +tx.getTransactionId() + " " + tx.getAttemptId());
		String log = tuple.getString(1);
		if (log != null && log.length()>0) {
			count ++;
		}

	}

	@Override
	public void finishBatch() {
		// TODO Auto-generated method stub

	}

	@Override
	public void prepare(Map arg0, TopologyContext arg1, BatchOutputCollector collector, TransactionAttempt id) {
		// TODO Auto-generated method stub
		System.err.println("MyTransactionBolt TransactionAttempt " +id.getTransactionId() + " " + id.getAttemptId());
		collector.emit(new Values(id,count));


	}

	@Override
	public void declareOutputFields(OutputFieldsDeclarer declarer) {
		// TODO Auto-generated method stub
		//定义一下类型
		declarer.declare(new Fields("tx","count"));

	}

}

package transaction1;

import java.util.HashMap;
import java.util.Map;
import java.util.Random;

import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.transactional.ITransactionalSpout;
import org.apache.storm.tuple.Fields;

public class MyTxSpout implements ITransactionalSpout<MyMata> {

	/**
	 * 数据源
	 */
	//定义数据源，100行数据
	Map<Long, String> dbMap = null;
	public MyTxSpout()
	{
		Random random = new Random();

		dbMap = new HashMap<Long, String>();
		String[] hosts = { "www.taobao.com" };
		String[] session_id = { "ABYH6Y4V4SCVXTG6DPB4VH9U123", "XXYH6YCGFJYERTT834R52FDXV9U34", "BBYH61456FGHHJ7JL89RG5VV9UYU7",
				"CYYH6Y2345GHI899OFG4V9U567", "VVVYH6Y4V4SFXZ56JIPDPB4V678" };
		String[] time = { "2014-01-07 08:40:50", "2014-01-07 08:40:51", "2014-01-07 08:40:52", "2014-01-07 08:40:53", 
				"2014-01-07 09:40:49", "2014-01-07 10:40:49", "2014-01-07 11:40:49", "2014-01-07 12:40:49" };
		
		for (long i = 0; i < 100; i++) {
			dbMap.put(i,hosts[0]+"\t"+session_id[random.nextInt(5)]+"\t"+time[random.nextInt(8)]+"\n");
		}
	}
	
	
	private static final long serialVersionUID = 1L;

	@Override
	public void declareOutputFields(OutputFieldsDeclarer declarer) {
		// TODO Auto-generated method stub
		declarer.declare(new Fields("","log"));
		
	}

	@Override
	public Map<String, Object> getComponentConfiguration() {
		// TODO Auto-generated method stub
		return null;
	}

	@Override
	public Coordinator<MyMata> getCoordinator(Map arg0, TopologyContext arg1) {
		// TODO Auto-generated method stub
		//重要1方法
		return new MyCondinator();
	}

	@Override
	public Emitter<MyMata> getEmitter(Map arg0, TopologyContext arg1) {
		// TODO Auto-generated method stub
		//重要2方法
		return new MyEmitter(dbMap);
	}

}