【六】storm的DRPC

一、RPC

1.概述

        远程过程调用协议，Remote Procedure Call。

序列化的速度会 影响RPC的性能。常用的序列化：avro、thrift(hive、spark有用到这个)

2.基于hadoop的RPC实现

代码演示

3.stormDRPC（分布式RPC）概述

官网DRPC介绍

充分利用storm的计算能力实现高密度的计算。

storm的topology接收函数参数作为输入，调用函数，输入结果。严格意义上说DRPC并不是storm的核心。它能够从storm中分离出来。DRPC是由一个叫DRPC SERVER的东西来整合（storm中已经实现了）。DRPC SERVER它负责接收RPC的请求，把请求发给storm的topology，接收topology的运行结果，并把结果返回给客户端。从客户端的角度来说，DRPC和RPC看起来是一样的。

使用举例

流程

二、本地模式DRPC编程

package drpc;

import java.util.Map;

import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
import org.apache.storm.LocalDRPC;
import org.apache.storm.drpc.LinearDRPCTopologyBuilder;
import org.apache.storm.task.OutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.topology.base.BaseRichBolt;
import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Tuple;
import org.apache.storm.tuple.Values;

/**
 * @author liyijie
 * @date 2018年6月11日下午4:01:51
 * @email 37024760@qq.com
 * @remark
 * @version 
 * 
 * 本地的DRPC
 */
public class StormLocalDRPC {
	public static class DRPCServerBolt extends BaseRichBolt{
		
		private OutputCollector collector;

		public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
			this.collector = collector;
		}

		public void execute(Tuple input) {
			Object requestId = input.getValue(0);//请求的ID
			String name = input.getString(1);//请求的参数
			System.out.println("DRPC Server Bolt TODO. re requestId: "+requestId+" name: "+name);
			
			//业务逻辑
			String result = "add user: "+name;
			
			this.collector.emit(new Values(requestId,result));
		}

		public void declareOutputFields(OutputFieldsDeclarer declarer) {
			declarer.declare(new Fields("id","result"));
		}
		
	}
	
	public static void main(String[] args){
		//参数是要被调用的远程函数的名称
		LinearDRPCTopologyBuilder builder = new LinearDRPCTopologyBuilder("addUser");
		builder.addBolt(new DRPCServerBolt());
		
		LocalCluster localcluster = new LocalCluster();
		
		LocalDRPC drpc = new LocalDRPC();

		localcluster.submitTopology("local-drpc", new Config(),
				builder.createLocalTopology(drpc));

		//函数名称、传的参数
		//调用远程函数，传参，得到结果
		 String result = drpc.execute("addUser", "sid");
		 
		 System.out.println("From client :"+result);

		localcluster.shutdown();
		drpc.shutdown();
	}
}

三、集群模式DRPC编程

修改storm.yaml文件。

配饰drpc.servers

cd /app/storm/conf

vi storm.yaml

 drpc.servers:
     - "node1"
     - "node2"

启动zookeeper

启动storm集群

启动drpc(这里只在node1上启动了)

cd /app/storm/bin

nohup sh storm drpc &

RPC Server的代码

package drpc;

import java.util.Map;

import org.apache.storm.Config;
import org.apache.storm.StormSubmitter;
import org.apache.storm.drpc.LinearDRPCTopologyBuilder;
import org.apache.storm.generated.AlreadyAliveException;
import org.apache.storm.generated.AuthorizationException;
import org.apache.storm.generated.InvalidTopologyException;
import org.apache.storm.task.OutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.topology.base.BaseRichBolt;
import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Tuple;
import org.apache.storm.tuple.Values;

/**
 * @author liyijie
 * @date 2018年6月11日下午4:01:51
 * @email 37024760@qq.com
 * @remark
 * @version 
 * 
 * 集群DRPC	Server
 */
public class StormRemoteDRPCTopology {
	public static class DRPCServerBolt extends BaseRichBolt{
		
		private OutputCollector collector;

		public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
			this.collector = collector;
		}

		public void execute(Tuple input) {
			Object requestId = input.getValue(0);//请求的ID
			String name = input.getString(1);//请求的参数
			System.out.println("DRPC Server Bolt TODO. re requestId: "+requestId+" name: "+name);
			
			//业务逻辑
			String result = "add user: "+name;
			
			this.collector.emit(new Values(requestId,result));
		}

		public void declareOutputFields(OutputFieldsDeclarer declarer) {
			declarer.declare(new Fields("id","result"));
		}
		
	}
	
	public static void main(String[] args){
		//参数是要被调用的远程函数的名称
		LinearDRPCTopologyBuilder builder = new LinearDRPCTopologyBuilder("addUser");
		builder.addBolt(new DRPCServerBolt());
		
		
		try {
			StormSubmitter.submitTopology("remote-drpc", new Config(), builder.createRemoteTopology());
		} catch (AlreadyAliveException e) {
			e.printStackTrace();
		} catch (InvalidTopologyException e) {
			e.printStackTrace();
		} catch (AuthorizationException e) {
			e.printStackTrace();
		}
	}
}

把RPC Server的代码打成jar包上传到storm的nimbus上运行

cd /app/storm

./bin/storm jar /app/sid_test_data/storm-0.0.1.jar drpc.StormRemoteDRPCTopology

编辑且在eclipse运行RPC Client的代码

package drpc;


import org.apache.storm.Config;
import org.apache.storm.utils.DRPCClient;

/**
 * @author liyijie
 * @date 2018年6月11日下午4:01:51
 * @email 37024760@qq.com
 * @remark
 * @version 
 * 
 * 集群DRPC	Clinet
 */
public class StormRemoteDRPCClient {
	
	public static void main(String[] args)throws Exception{
		Config conf = new Config();
		conf.put("storm.thrift.transport", "org.apache.storm.security.auth.SimpleTransportPlugin");
		conf.put(Config.STORM_NIMBUS_RETRY_TIMES, 3);
		conf.put(Config.STORM_NIMBUS_RETRY_INTERVAL, 10);
		conf.put(Config.STORM_NIMBUS_RETRY_INTERVAL_CEILING, 20);
		conf.put(Config.DRPC_MAX_BUFFER_SIZE, 1048576);
		DRPCClient client = new DRPCClient(conf,"node1", 3772);//默认DRPC的端口是3772
		String result = client.execute("addUser", "Sid");//调用的方法和参数
		System.out.println("Client invoked :"+result);
	}
}