基于zookeeper、连接池实现的Thrift服务负载均衡和服务发现

最新推荐文章于 2024-07-03 09:34:10 发布

zero__007

最新推荐文章于 2024-07-03 09:34:10 发布

阅读量7.3k

点赞数

分类专栏： Java进阶

Java进阶专栏收录该内容

15 篇文章 0 订阅

订阅专栏

转载自：http://blog.csdn.net/zhu_tianwei/article/details/44115667

对于Thrift服务化的改造，主要是客户端，可以从如下几个方面进行：
1.服务端的服务注册，客户端自动发现，无需手工修改配置，这里使用zookeeper，采用curator-recipes工具类进行处理服务的注册与发现。
2.客户端使用连接池对服务调用进行管理，提升性能，使用Apache Commons项目commons-pool2，可以大大减少代码的复杂度。
3.关于Failover/LoadBalance，由于zookeeper的watcher，当服务端不可用是及时通知客户端，并移除不可用的服务节点，而LoadBalance有很多算法，这里采用随机加权方式，也是常有的负载算法。

1.Zookeeper的客户端

import org.apache.commons.lang3.StringUtils;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.CuratorFrameworkFactory;
import org.apache.curator.retry.ExponentialBackoffRetry;

public class ZookeeperFactory {

    private String zkHosts;
    private int sessionTimeout = 30000;// session超时
    private int connectionTimeout = 30000;

    private String namespace;// 全局path前缀,常用来区分不同的应用

    private final static String ROOT = "rpc";

    private CuratorFramework zkClient;

    public void setZkHosts(String zkHosts) {
        this.zkHosts = zkHosts;
    }

    public void setSessionTimeout(int sessionTimeout) {
        this.sessionTimeout = sessionTimeout;
    }

    public void setConnectionTimeout(int connectionTimeout) {
        this.connectionTimeout = connectionTimeout;
    }

    public void setNamespace(String namespace) {
        this.namespace = namespace;
    }

    public void setZkClient(CuratorFramework zkClient) {
        this.zkClient = zkClient;
    }

    public synchronized CuratorFramework getZkClient() throws Exception {
        if (zkClient == null) {
            zkClient = create();
            zkClient.start();
            return zkClient;
        }
        return zkClient;
    }

    private CuratorFramework create() throws Exception {
        if (StringUtils.isEmpty(namespace)) {
            namespace = ROOT;
        } else {
            namespace = ROOT + "/" + namespace;
        }
        return create(zkHosts, sessionTimeout, connectionTimeout, namespace);
    }

    private static CuratorFramework create(String connectString, int sessionTimeout, int connectionTimeout, String namespace) {
        CuratorFrameworkFactory.Builder builder = CuratorFrameworkFactory.builder();
        return builder.connectString(connectString)
                .sessionTimeoutMs(sessionTimeout)
                .connectionTimeoutMs(30000)
                .canBeReadOnly(true)
                .namespace(namespace)
                .retryPolicy(new ExponentialBackoffRetry(1000, Integer.MAX_VALUE))
                .defaultData(null)
                .build();
    }

    public void close() {
        if (zkClient != null) {
            zkClient.close();
        }
    }
}

2. 定义发布服务接口，并实现将服务信息（服务接口、版本号，IP、port、weight）发布到zookeeper中。

import org.apache.commons.lang3.StringUtils;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.imps.CuratorFrameworkState;
import org.apache.zookeeper.CreateMode;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.UnsupportedEncodingException;

/**
 *  注册服务列表到Zookeeper
 */
public class ThriftServerAddressRegister {
    private CuratorFramework zkClient;

    public ThriftServerAddressRegister(){}

    public ThriftServerAddressRegister(CuratorFramework zkClient){
        this.zkClient = zkClient;
    }

    public void setZkClient(CuratorFramework zkClient) {
        this.zkClient = zkClient;
    }

    public void register(String service, String version, String address) {
        if(zkClient.getState() == CuratorFrameworkState.LATENT){
            zkClient.start();
        }
        if(StringUtils.isEmpty(version)){
            version="1.0.0";
        }
        //临时节点
        try {
            zkClient.create()
                    .creatingParentsIfNeeded()
                    .withMode(CreateMode.EPHEMERAL)
                    .forPath("/"+service+"/"+version+"/"+address);
        } catch (UnsupportedEncodingException e) {
            System.out.println("register service address to zookeeper exception.");
        } catch (Exception e) {
            System.out.println("register service address to zookeeper exception.");
        }
    }

    public void close(){
        zkClient.close();
    }
}

3. 客户端发现服务, 定义获取服务地址接口

import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.imps.CuratorFrameworkState;
import org.apache.curator.framework.recipes.cache.ChildData;
import org.apache.curator.framework.recipes.cache.PathChildrenCache;
import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent;
import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener;

import java.net.InetSocketAddress;
import java.util.*;

/**
 * thrift server-service地址提供者,以便构建客户端连接池
 */
public class ThriftServerAddressProvider {
    // 注册服务
    private String service;
    // 服务版本号
    private String version = "1.0.0";

    private PathChildrenCache cachedPath;

    private CuratorFramework zkClient;

    // 用来保存当前provider所接触过的地址记录
    // 当zookeeper集群故障时,可以使用trace中地址,作为"备份"
    private Set<String> trace = new HashSet<>();

    private final List<InetSocketAddress> container = new ArrayList<>();

    private Queue<InetSocketAddress> inner = new LinkedList<>();

    private Object lock = new Object();

    private static final Integer DEFAULT_WEIGHT = 1;// 默认权重

    public void setService(String service) {
        this.service = service;
    }

    public void setVersion(String version) {
        this.version = version;
    }

    public ThriftServerAddressProvider() {
    }

    public ThriftServerAddressProvider(CuratorFramework zkClient) {
        this.zkClient = zkClient;
    }

    public void setZkClient(CuratorFramework zkClient) {
        this.zkClient = zkClient;
    }

    public void init() throws Exception {
        // 如果zk尚未启动,则启动
        if (zkClient.getState() == CuratorFrameworkState.LATENT) {
            zkClient.start();
        }
        buildPathChildrenCache(zkClient, getServicePath(), true);
        cachedPath.start(PathChildrenCache.StartMode.POST_INITIALIZED_EVENT);
    }

    private String getServicePath() {
        return "/" + service + "/" + version;
    }

    private void buildPathChildrenCache(final CuratorFramework client, String path, Boolean cacheData) throws Exception {
        cachedPath = new PathChildrenCache(client, path, cacheData);
        cachedPath.getListenable().addListener(new PathChildrenCacheListener() {
            @Override
            public void childEvent(CuratorFramework client, PathChildrenCacheEvent event) throws Exception {
                PathChildrenCacheEvent.Type eventType = event.getType();
                switch (eventType) {
                    case CONNECTION_RECONNECTED:
                        System.out.println("Connection is reconection.");
                        break;
                    case CONNECTION_SUSPENDED:
                        System.out.println("Connection is suspended.");
                        break;
                    case CONNECTION_LOST:
                        System.out.println("Connection error,waiting...");
                        return;
                    default:
                        //
                }
                // 任何节点的时机数据变动,都会rebuild,此处为一个"简单的"做法.
                cachedPath.rebuild();
                rebuild();
            }

            protected void rebuild() throws Exception {
                List<ChildData> children = cachedPath.getCurrentData();
                if (children == null || children.isEmpty()) {
                    // 有可能所有的thrift server都与zookeeper断开了链接
                    // 但是,有可能,thrift client与thrift server之间的网络是良好的
                    // 因此此处是否需要清空container,是需要多方面考虑的.
                    container.clear();
                    System.out.println("thrift server-cluster error....");
                    return;
                }
                List<InetSocketAddress> current = new ArrayList<InetSocketAddress>();
                String path = null;
                for (ChildData data : children) {
                    path = data.getPath();
                    System.out.println("get path:" + path);
                    path = path.substring(getServicePath().length() + 1);
                    System.out.println("get serviceAddress:" + path);
                    String address = new String(path.getBytes(), "utf-8");
                    current.addAll(transfer(address));
                    trace.add(address);
                }
                Collections.shuffle(current);
                synchronized (lock) {
                    container.clear();
                    container.addAll(current);
                    inner.clear();
                    inner.addAll(current);
                }
            }
        });
    }

    private List<InetSocketAddress> transfer(String address) {
        String[] hostname = address.split(":");
        Integer weight = DEFAULT_WEIGHT;
        if (hostname.length == 3) {
            weight = Integer.valueOf(hostname[2]);
        }
        String ip = hostname[0];
        Integer port = Integer.valueOf(hostname[1]);
        List<InetSocketAddress> result = new ArrayList<InetSocketAddress>();
        // 根据优先级，将ip：port添加多次到地址集中，然后随机取地址实现负载
        for (int i = 0; i < weight; i++) {
            result.add(new InetSocketAddress(ip, port));
        }
        return result;
    }

    public List<InetSocketAddress> findServerAddressList() {
        return Collections.unmodifiableList(container);
    }

    public synchronized InetSocketAddress selector() {
        if (inner.isEmpty()) {
            if (!container.isEmpty()) {
                inner.addAll(container);
            } else if (!trace.isEmpty()) {
                synchronized (lock) {
                    for (String hostname : trace) {
                        container.addAll(transfer(hostname));
                    }
                    Collections.shuffle(container);
                    inner.addAll(container);
                }
            }
        }
        return inner.poll();
    }

    public void close() {
        try {
            cachedPath.close();
            zkClient.close();
        } catch (Exception e) {
        }
    }

    public String getService() {
        return service;
    }
}

这里，zkClient会监听服务注册路径上的节点变化，当有变动时，随时更新服务列表。

4. 客户端获取服务代理及连接池实现

import org.apache.commons.pool2.BasePooledObjectFactory;
import org.apache.commons.pool2.PooledObject;
import org.apache.commons.pool2.PooledObjectFactory;
import org.apache.commons.pool2.impl.DefaultPooledObject;
import org.apache.thrift.TServiceClient;
import org.apache.thrift.TServiceClientFactory;
import org.apache.thrift.protocol.TBinaryProtocol;
import org.apache.thrift.protocol.TProtocol;
import org.apache.thrift.transport.TFramedTransport;
import org.apache.thrift.transport.TSocket;
import org.apache.thrift.transport.TTransport;

import java.net.InetSocketAddress;

/**
 * 连接池,thrift-client for spring
 */
public class ThriftClientPoolFactory implements PooledObjectFactory<TServiceClient> {

    private final ThriftServerAddressProvider serverAddressProvider;
    private final TServiceClientFactory<TServiceClient> clientFactory;
    private PoolOperationCallBack callback;

    protected ThriftClientPoolFactory(ThriftServerAddressProvider addressProvider, TServiceClientFactory<TServiceClient> clientFactory) throws Exception {
        this.serverAddressProvider = addressProvider;
        this.clientFactory = clientFactory;
    }

    protected ThriftClientPoolFactory(ThriftServerAddressProvider addressProvider, TServiceClientFactory<TServiceClient> clientFactory,
            PoolOperationCallBack callback) throws Exception {
        this.serverAddressProvider = addressProvider;
        this.clientFactory = clientFactory;
        this.callback = callback;
    }

    static interface PoolOperationCallBack {
        // 销毁client之前执行
        void destroy(TServiceClient client);

        // 创建成功是执行
        void make(TServiceClient client);
    }

    @Override
    public PooledObject<TServiceClient> makeObject() throws Exception {
        InetSocketAddress address = serverAddressProvider.selector();

        TSocket tsocket = new TSocket(address.getHostName(), address.getPort(), 10000);//10000ms超时
        TTransport transport = new TFramedTransport(tsocket);
        TProtocol protocol = new TBinaryProtocol(transport);
        TServiceClient client = this.clientFactory.getClient(protocol);
        transport.open();
        if (callback != null) {
            try {
                callback.make(client);
            } catch (Exception e) {
            }
        }
        return new DefaultPooledObject<>(client);
    }

    @Override
    public void destroyObject(PooledObject<TServiceClient> p) throws Exception {
        TServiceClient client = p.getObject();

        if (callback != null) {
            try {
                callback.destroy(client);
            } catch (Exception e) {
            }
        }
        TTransport pin = client.getInputProtocol().getTransport();
        pin.close();
        TTransport pout = client.getOutputProtocol().getTransport();
        pout.close();
    }

    @Override
    public boolean validateObject(PooledObject<TServiceClient> p) {
        TServiceClient client = p.getObject();
        TTransport pin = client.getInputProtocol().getTransport();
        TTransport pout = client.getOutputProtocol().getTransport();
        return pin.isOpen() && pout.isOpen();
    }

    @Override
    public void activateObject(PooledObject<TServiceClient> p) throws Exception {
    }

    @Override
    public void passivateObject(PooledObject<TServiceClient> p) throws Exception {
    }
}

import org.apache.commons.pool2.impl.GenericObjectPool;
import org.apache.commons.pool2.impl.GenericObjectPoolConfig;
import org.apache.thrift.TServiceClient;
import org.apache.thrift.TServiceClientFactory;

/**
 * 客户端代理
 */
public class ThriftServiceClientProxyFactory {
    private ThriftServerAddressProvider serverAddressProvider;

    private GenericObjectPool<TServiceClient> pool;

    private ThriftClientPoolFactory.PoolOperationCallBack callback = new ThriftClientPoolFactory.PoolOperationCallBack() {
        @Override
        public void make(TServiceClient client) {
            System.out.println("create");
        }

        @Override
        public void destroy(TServiceClient client) {
            System.out.println("destroy");
        }
    };

    public GenericObjectPool<TServiceClient> getPool() {
        return pool;
    }

    public void setServerAddressProvider(ThriftServerAddressProvider serverAddressProvider) {
        this.serverAddressProvider = serverAddressProvider;
    }

    public void init() throws Exception {
        ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
        // 加载Iface接口       
        Class<?> objectClass = classLoader.loadClass(serverAddressProvider.getService() + "$Iface");

        // 加载Client.Factory类
        Class<TServiceClientFactory<TServiceClient>> fi = (Class<TServiceClientFactory<TServiceClient>>) classLoader.loadClass(serverAddressProvider.getService() + "$Client$Factory");
        TServiceClientFactory<TServiceClient> clientFactory = fi.newInstance();
        ThriftClientPoolFactory clientPool = new ThriftClientPoolFactory(serverAddressProvider, clientFactory, callback);

        GenericObjectPoolConfig poolConfig = new GenericObjectPoolConfig();
        poolConfig.setMaxTotal(100);
        poolConfig.setMaxIdle(100);
        poolConfig.setMinIdle(100);
        poolConfig.setBlockWhenExhausted(true);
        poolConfig.setTestOnBorrow(false);
        poolConfig.setTestOnCreate(false);
        poolConfig.setTestWhileIdle(true);
        poolConfig.setTestOnReturn(false);//其他参数设置自行百度

        pool = new GenericObjectPool<>(clientPool, poolConfig);
    }
    public void close() {
        if (serverAddressProvider != null) {
            serverAddressProvider.close();
        }
    }
}

5. 服务端服务注册

import org.apache.thrift.TProcessor;
import org.apache.thrift.TProcessorFactory;
import org.apache.thrift.protocol.TBinaryProtocol;
import org.apache.thrift.server.TServer;
import org.apache.thrift.server.TThreadedSelectorServer;
import org.apache.thrift.transport.TFramedTransport;
import org.apache.thrift.transport.TNonblockingServerSocket;

import java.lang.instrument.IllegalClassFormatException;
import java.lang.reflect.Constructor;

/**
 * 服务端注册服务工厂
 */
public class ThriftServiceServerFactory {

    private String ip = "127.0.0.1";
    private Integer port = 8299;
    // 优先级
    private Integer weight = 1;// default
    // 服务实现类
    private Object service;// serice实现类
    //服务版本号
    private String version;

    private ServerThread serverThread;

    //服务注册
    private ThriftServerAddressRegister thriftServerAddressRegister;

    public void setPort(Integer port) {
        this.port = port;
    }

    public void setWeight(Integer weight) {
        this.weight = weight;
    }

    public void setService(Object service) {
        this.service = service;
    }

    public void setVersion(String version) {
        this.version = version;
    }

    public void init() throws Exception {
        String hostname = ip + ":" + port + ":" + weight;
        Class<?> serviceClass = service.getClass();
        // 获取实现类接口
        Class<?>[] interfaces = serviceClass.getInterfaces();
        if (interfaces.length == 0) {
            throw new IllegalClassFormatException("service-class should implements Iface");
        }
        // reflect,load "Processor";
        TProcessor processor = null;
        String serviceName = null;
        for (Class<?> clazz : interfaces) {
            String cname = clazz.getSimpleName();
            if (!cname.equals("Iface")) {
                continue;
            }
            serviceName = clazz.getEnclosingClass().getName();
            String pname = serviceName + "$Processor";
            try {
                ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
                Class<?> pclass = classLoader.loadClass(pname);
                if (!TProcessor.class.isAssignableFrom(pclass)) {
                    continue;
                }
                Constructor<?> constructor = pclass.getConstructor(clazz);
                processor = (TProcessor) constructor.newInstance(service);
                break;
            } catch (Exception e) {
                //
            }
        }
        if (processor == null) {
            throw new IllegalClassFormatException("service-class should implements Iface");
        }
        //需要单独的线程,因为serve方法是阻塞的.
        serverThread = new ServerThread(processor, port);
        serverThread.start();
        // 注册服务
        if (thriftServerAddressRegister != null) {
            thriftServerAddressRegister.register(serviceName, version, hostname);
        }
    }

    class ServerThread extends Thread {
        private TServer server;

        ServerThread(TProcessor processor, int port) throws Exception {
            TNonblockingServerSocket serverTransport = new TNonblockingServerSocket(port);
            TThreadedSelectorServer.Args tArgs = new TThreadedSelectorServer.Args(serverTransport);
            TProcessorFactory processorFactory = new TProcessorFactory(processor);
            tArgs.processorFactory(processorFactory);
            tArgs.transportFactory(new TFramedTransport.Factory());
            tArgs.protocolFactory(new TBinaryProtocol.Factory(true, true));
            server = new TThreadedSelectorServer(tArgs);
        }

        @Override
        public void run() {
            try {
                //启动服务
                server.serve();
            } catch (Exception e) {
            }
        }

        public void stopServer() {
            server.stop();
        }
    }

    public void close() {
        serverThread.stopServer();
    }
}

6.客户端Demo（示例，未运行）

        CuratorFramework zkClientServer = new ZookeeperFactory().getZkClient();
        ThriftServerAddressProvider addressProvider = new ThriftServerAddressProvider(zkClientServer);
        addressProvider.setService("");
        // ...属性设置
        addressProvider.init();


        ThriftServiceClientProxyFactory clientProxyFactory = new ThriftServiceClientProxyFactory();
        clientProxyFactory.setServerAddressProvider(addressProvider);
        clientProxyFactory.init();


        GenericObjectPool<TServiceClient> thriftClientPool =clientProxyFactory.getPool();
        TServiceClient thriftClient = thriftClientPool.borrowObject();
        //。。。
        thriftClientPool.returnObject(thriftClient);
        //internalPool.invalidateObject(thriftClient);

可以改进的地方：
thriftClientPool通过随机方式基本上实现了负载均衡，但是当某一台机器某段时间性能变差（每秒能处理的请求量要低于其它），导致出现超时时，通常会在pool中关掉对应的client，但是从代码上看出，thriftClientPool中该机器对应的client可能不止一个，而且GenericObjectPool会定时检查pool中的对象数量是否满足我们的设置（GenericObjectPoolConfig的设置），如果不够会调用ThriftServerAddressProvider.selector()来获取服务IP来新建连接填补pool。
于是我们可以做些改进，对有问题的client对应的服务IP，反馈标记出来，然后在ThriftServerAddressProvider的List<InetSocketAddress>中调整其的权重，比如相较于其他服务IP的占比数小于10%或则更多，这样就间接的降低了有性能问题服务器的QPS，避免了反复删毁、重建连接。同时，还要有恢复机制，当机器性能恢复后，我们需要将其QPS提升上来。
一点想法：可以封装下thriftClientPool，当要删毁连接时获取其IP，在ThriftServerAddressProvider反馈出来，如果GenericObjectPool填补时不再使用该IP，这样应该可以降低该服务器的QPS。与此同时，我们也需要重构pool，一来避免某些机器qps一直很高（不断填补了问题机器的缺），二来可以做到服务恢复机制。于是可以定时重构该pool，到了时间点，每次回到pool的client都销毁掉，这样GenericObjectPool会重建client，这时会从Queue<InetSocketAddress>一个一个poll服务IP新建client，Queue<InetSocketAddress>的构建依赖List<InetSocketAddress>，只需要对List<InetSocketAddress>做些文章即可，比如把上个周期内有问题的IP权重继续下调，或则恢复，这就看具体业务了。
以上只是个人想法，不足之处欢迎一起探讨。