storm 的一个drpc例子(值得一看)

2012-11-26 18:461465人阅读评论(0)收藏举报

[java] view plain copy print ?

1. public static void initConnectDB(){ 
2.         primaryKey = "id"; 
3.         rdbmsUrl = "jdbc:mysql://hadoop/DB" ; 
4.         rdbmsUserName = "";  
5.         rdbmsPassword = ""; 
6.          
7.         connector = new RDBMSConnector(); 
8.         try { 
9.             con = connector.getConnection(rdbmsUrl, rdbmsUserName, rdbmsPassword); 
10.             communicator = new RDBMSCommunicator2UFN(con); 
11.         } catch (Exception e){ 
12.             System.out.println("connect to db exception in initConnectDB()"); 
13.             e.printStackTrace(); 
14.         } 
15.     } 
16.      
17.     public static class GetUserID extends BaseBasicBolt{ 
18.         //private RDBMSCommunicator communicator = null; 
19.         private ResultSet rs = null; 
20.          
21.         @Override 
22.         public void prepare(Map stormConf, TopologyContext context) { 
23.             System.out.println("in prepare con : "+con); 
24.             //this.communicator = new RDBMSCommunicator(con);  
25.             System.out.println("in pretpare communicator :"+communicator); 
26.         } 
27.  
28.         public void execute(Tuple input, BasicOutputCollector collector) { 
29.             Object id = input.getValue(0); 
30.             String userName = input.getString(1); 
31.              
32.             String sql = String.format("select userID from usersinfo where username='%s'", userName); 
33.             System.out.println("sql in get-user-id: "+sql); 
34.             rs = communicator.selecteExec(sql); 
35.             String userID = null; 
36.              
37.             if (rs != null){ 
38.                 try { 
39.                     rs.next(); 
40.                     userID = rs.getString("userID"); 
41.                 } catch (Exception e){ 
42.                     e.printStackTrace(); 
43.                 } 
44.                 collector.emit(new Values(id, userID)); 
45.             } 
46.              
47.         } 
48.  
49.         public void declareOutputFields(OutputFieldsDeclarer declarer) { 
50.             declarer.declare(new Fields("id", "userID")); 
51.         } 
52.          
53.     } 
54.      
55.     public static class GetUserFunctionsID extends BaseBasicBolt{ 
56.         //private RDBMSCommunicator communicator = null; 
57.         private ResultSet rs = null; 
58.          
59.         @Override 
60.         public void prepare(Map stormConf, TopologyContext context) { 
61.             //communicator = new  RDBMSCommunicator(con);  
62.         } 
63.  
64.         public void execute(Tuple input, BasicOutputCollector collector) { 
65.             Object id = input.getValue(0); 
66.             String userID = input.getString(1); 
67.              
68.             if (userID == null || userID.trim().length() == 0){ 
69.                 return; 
70.             } 
71.              
72.             String sql = String.format("select functionID from userfunctions where userID='%s'", userID); 
73.             System.out.println("sql in get-user-functionid : "+sql); 
74.             rs = communicator.selecteExec(sql); 
75.             String functionID = null; 
76.              
77.             if (rs != null){ 
78.                 try { 
79.                     while(rs.next()){ 
80.                         functionID = rs.getString("functionID"); 
81.                         collector.emit(new Values(id,functionID)); 
82.                     } 
83.                 } catch(Exception e){ 
84.                     e.printStackTrace(); 
85.                 } 
86.             } 
87.         } 
88.  
89.         public void declareOutputFields(OutputFieldsDeclarer declarer) { 
90.             declarer.declare(new Fields("id","functionID")); 
91.         } 
92.     } 
93.      
94.     public static class GetUserFunctionsName extends BaseBatchBolt{ 
95.         //private RDBMSCommunicator communicator = null; 
96.         private ResultSet rs = null; 
97.          
98.         List<String> functionsName = new ArrayList<String>(); 
99.         BatchOutputCollector _collector; 
100.         Object _id; 
101.          
102.         public void execute(Tuple tuple) { 
103.             String functionID = tuple.getString(1); 
104.              
105.             if (functionID == null || functionID.trim().length() == 0){ 
106.                 return ; 
107.             } 
108.              
109.             String sql = String.format("select functionName from functionsinfo where functionID='%s'",functionID); 
110.             System.out.println("sql in get-user-functionname : "+sql ); 
111.             rs = communicator.selecteExec(sql); 
112.             String functionName = null; 
113.              
114.             if(rs != null){ 
115.                 try { 
116.                     rs.next(); 
117.                     functionName = rs.getString("functionName"); 
118.                     functionsName.add(functionName); 
119.                 } catch (Exception e){ 
120.                     e.printStackTrace(); 
121.                 } 
122.             } 
123.         } 
124.  
125.         public void finishBatch() { 
126.             _collector.emit(new Values(_id,functionsName.toString())); 
127.         } 
128.  
129.         public void prepare(Map conf, TopologyContext context, 
130.                 BatchOutputCollector collector, Object id) { 
131.             _collector = collector; 
132.             _id = id; 
133.         } 
134.  
135.         public void declareOutputFields(OutputFieldsDeclarer declarer) { 
136.             declarer.declare(new Fields("id", "user-funcions-name")); 
137.         } 
138.          
139.     } 
140.      
141.     public static LinearDRPCTopologyBuilder construct(){ 
142.          initConnectDB(); 
143.          LinearDRPCTopologyBuilder builder = new LinearDRPCTopologyBuilder("user-functions-name"); 
144.           
145.          builder.addBolt(new GetUserID(), 2); 
146.          builder.addBolt(new GetUserFunctionsID(),2).shuffleGrouping(); 
147.          builder.addBolt(new GetUserFunctionsName(),2).fieldsGrouping(new Fields("id","functionID")); 
148.          return builder; 
149.          
150.     } 
151.     public static void main(String[] args) throws Exception{ 
152.         LinearDRPCTopologyBuilder builder = construct(); 
153.          
154.          
155.         Config conf = new Config(); 
156.          
157.         if(args==null || args.length==0) { 
158.             conf.setMaxTaskParallelism(3); 
159.             LocalDRPC drpc = new LocalDRPC(); 
160.             LocalCluster cluster = new LocalCluster(); 
161.             cluster.submitTopology("user-fn-drpc", conf, builder.createLocalTopology(drpc)); 
162.              
163.             String[] userNames = new String[] { "qingwu.fu"}; 
164.             for(String un: userNames) { 
165.                 System.out.println("Functions name of : " + un + ": " + drpc.execute("user-functions-name", un)); 
166.             } 
167.             cluster.shutdown(); 
168.             drpc.shutdown(); 
169.         } else { 
170.             conf.setNumWorkers(6); 
171.             StormSubmitter.submitTopology(args[0], conf, builder.createRemoteTopology()); 
172.         } 
173.          
174.     } 

public static void initConnectDB(){
		primaryKey = "id";
		rdbmsUrl = "jdbc:mysql://hadoop/DB" ;
		rdbmsUserName = ""; 
		rdbmsPassword = "";
		
		connector = new RDBMSConnector();
		try {
			con = connector.getConnection(rdbmsUrl, rdbmsUserName, rdbmsPassword);
			communicator = new RDBMSCommunicator2UFN(con);
		} catch (Exception e){
			System.out.println("connect to db exception in initConnectDB()");
			e.printStackTrace();
		}
	}
	
	public static class GetUserID extends BaseBasicBolt{
		//private RDBMSCommunicator communicator = null;
		private ResultSet rs = null;
		
		@Override
	    public void prepare(Map stormConf, TopologyContext context) {
			System.out.println("in prepare con : "+con);
			//this.communicator = new RDBMSCommunicator(con); 
			System.out.println("in pretpare communicator :"+communicator);
	    }

		public void execute(Tuple input, BasicOutputCollector collector) {
			Object id = input.getValue(0);
			String userName = input.getString(1);
			
			String sql = String.format("select userID from usersinfo where username='%s'", userName);
			System.out.println("sql in get-user-id: "+sql);
			rs = communicator.selecteExec(sql);
			String userID = null;
			
			if (rs != null){
				try {
					rs.next();
					userID = rs.getString("userID");
				} catch (Exception e){
					e.printStackTrace();
				}
				collector.emit(new Values(id, userID));
			}
			
		}

		public void declareOutputFields(OutputFieldsDeclarer declarer) {
			declarer.declare(new Fields("id", "userID"));
		}
		
	}
	
	public static class GetUserFunctionsID extends BaseBasicBolt{
		//private RDBMSCommunicator communicator = null;
		private ResultSet rs = null;
		
		@Override
	    public void prepare(Map stormConf, TopologyContext context) {
			//communicator = new  RDBMSCommunicator(con); 
	    }

		public void execute(Tuple input, BasicOutputCollector collector) {
			Object id = input.getValue(0);
			String userID = input.getString(1);
			
			if (userID == null || userID.trim().length() == 0){
				return;
			}
			
			String sql = String.format("select functionID from userfunctions where userID='%s'", userID);
			System.out.println("sql in get-user-functionid : "+sql);
			rs = communicator.selecteExec(sql);
			String functionID = null;
			
			if (rs != null){
				try {
					while(rs.next()){
						functionID = rs.getString("functionID");
						collector.emit(new Values(id,functionID));
					}
				} catch(Exception e){
					e.printStackTrace();
				}
			}
		}

		public void declareOutputFields(OutputFieldsDeclarer declarer) {
			declarer.declare(new Fields("id","functionID"));
		}
	}
	
	public static class GetUserFunctionsName extends BaseBatchBolt{
		//private RDBMSCommunicator communicator = null;
		private ResultSet rs = null;
		
		List<String> functionsName = new ArrayList<String>();
		BatchOutputCollector _collector;
	    Object _id;
	    
		public void execute(Tuple tuple) {
			String functionID = tuple.getString(1);
			
			if (functionID == null || functionID.trim().length() == 0){
				return ;
			}
			
			String sql = String.format("select functionName from functionsinfo where functionID='%s'",functionID);
			System.out.println("sql in get-user-functionname : "+sql );
			rs = communicator.selecteExec(sql);
			String functionName = null;
			
			if(rs != null){
				try {
					rs.next();
					functionName = rs.getString("functionName");
					functionsName.add(functionName);
				} catch (Exception e){
					e.printStackTrace();
				}
			}
		}

		public void finishBatch() {
			_collector.emit(new Values(_id,functionsName.toString()));
		}

		public void prepare(Map conf, TopologyContext context,
				BatchOutputCollector collector, Object id) {
			_collector = collector;
            _id = id;
		}

		public void declareOutputFields(OutputFieldsDeclarer declarer) {
			declarer.declare(new Fields("id", "user-funcions-name"));
		}
		
	}
	
	public static LinearDRPCTopologyBuilder construct(){
		 initConnectDB();
		 LinearDRPCTopologyBuilder builder = new LinearDRPCTopologyBuilder("user-functions-name");
		 
		 builder.addBolt(new GetUserID(), 2);
		 builder.addBolt(new GetUserFunctionsID(),2).shuffleGrouping();
		 builder.addBolt(new GetUserFunctionsName(),2).fieldsGrouping(new Fields("id","functionID"));
		 return builder;
		
	}
	public static void main(String[] args) throws Exception{
		LinearDRPCTopologyBuilder builder = construct();
        
        
        Config conf = new Config();
        
        if(args==null || args.length==0) {
            conf.setMaxTaskParallelism(3);
            LocalDRPC drpc = new LocalDRPC();
            LocalCluster cluster = new LocalCluster();
            cluster.submitTopology("user-fn-drpc", conf, builder.createLocalTopology(drpc));
            
            String[] userNames = new String[] { "qingwu.fu"};
            for(String un: userNames) {
                System.out.println("Functions name of : " + un + ": " + drpc.execute("user-functions-name", un));
            }
            cluster.shutdown();
            drpc.shutdown();
        } else {
            conf.setNumWorkers(6);
            StormSubmitter.submitTopology(args[0], conf, builder.createRemoteTopology());
        }
		
	}

 

转载： http://blog.csdn.net/baiyangfu_love/article/details/8227085