简单的分布式请求分发与故障管理

当部署多台服务器（或多个服务实例）用于负载均衡、避免单点故障（备份）、或主从/读写分离等目的时，避开复杂的场景不谈，分布式服务的客户端通常需要具备如下功能：
1) 请求分发(如轮询 round-robin/polling)

2) 故障检测(failure detection)、故障切换/转移(failover)、故障恢复(recovery)

public class Request {
	private String name;
	private String params;
	
	public String getName() {
		return name;
	}
	public void setName(String name) {
		this.name = name;
	}
	public String getParams() {
		return params;
	}
	public void setParams(String params) {
		this.params = params;
	}
}

public class RequestDispatcher {	
	private ServerRouter roundRobinRouter;
	
	public RequestDispatcher(ServerRouter router) {
		this.roundRobinRouter = router;
	}
	
	public void handleRequest(Request req) {
		String info = "RequestDispatcher - dispatch request " + req.getName();
		ServerConnection srvConn = this.roundRobinRouter.getServerConnection();
		while (srvConn != null) {
			try {
				//once success, return
				Response resp = srvConn.handleRequest(req);
				System.out.println(info + " to server " + srvConn.getName() + ", result: " + resp.getResult());
				break;
			} catch (ServerShutDownException sde) {				
				srvConn.setAvailable(false); // failure detection
				srvConn = this.roundRobinRouter.getServerConnection(); // fail over
				if (srvConn != null) {
					System.out.println(info+" *faile over* on ("+sde.getMessage()+") to "+srvConn.getName());
				}
			}
		}		
		if (srvConn == null) {
			System.out.println(info + " failed! no server is available.");
		}
	}
}

public class Response {
	private String result;

	public String getResult() {
		return result;
	}

	public void setResult(String result) {
		this.result = result;
	}
}

public class ServerConnection {
	public static class ServerShutDownException extends Exception {
		private static final long serialVersionUID = 1L;
		
		public ServerShutDownException(String errorMessage) {
			super(errorMessage);
		}
	}
	
	private String name;
	private String url;
	private boolean isAvailable;
	private int priority;
	
	private boolean isShutDown; //test

	public String getName() {
		return name;
	}

	public void setName(String name) {
		this.name = name;
	}

	public String getUrl() {
		return url;
	}

	public void setUrl(String url) {
		this.url = url;
	}

	public int getPriority() {
		return priority;
	}

	public void setPriority(int priority) {
		this.priority = priority;
	}

	public boolean isAvailable() {
		return isAvailable;
	}

	public void setAvailable(boolean isAvailable) {
		this.isAvailable = isAvailable;
	}

	public boolean isShutDown() {
		return isShutDown;
	}

	public void setShutDown(boolean isShutDown) {
		this.isShutDown = isShutDown;
	}
	
	public Response handleRequest(Request req) throws ServerShutDownException {		
		Response resp = new Response();
		if (this.isShutDown) {
			throw new ServerShutDownException(this.name +" was shutdown");
		}
		else {
			resp.setResult("OK");
		}
		return resp;
	}
}

import java.util.ArrayList;
import java.util.List;
import java.util.Timer;
import java.util.TimerTask;

public class ServerRouter {
	public static class RecoverTask extends TimerTask {
		private ServerRouter router;
		
		public RecoverTask(ServerRouter router) {
			this.router = router;
		}
		
		@Override
		public void run() {
			router.checkRecovery();
		}		
	}
	
	private long  checkDelay = 5*1000;
	private long  checkInterval = 3*1000;
	private Timer checkTimer;
	
	private List<ServerConnection> srvConnPool = new ArrayList<ServerConnection>();
	private int pollCount = 0;
	
	public ServerRouter() {
		checkTimer = new Timer(true);
		checkTimer.schedule(new RecoverTask(this), checkDelay, checkInterval);
	}
	
	public void addServerConnection(ServerConnection serverConnection) {
		srvConnPool.add(serverConnection);
	}
	
	public ServerConnection getServerConnection(int idx) {
		if (idx < srvConnPool.size()) {
			return srvConnPool.get(idx);
		}
		return null;
	}
	
	//round robin routing (weighted, random, broadcast)
	public ServerConnection getServerConnection() {			
		int connNum = srvConnPool.size();
		for (int pollTime = 1; pollTime <= connNum; pollTime++) {
			if (pollCount == connNum) {
				pollCount = 0;
			}
			ServerConnection srvConn = srvConnPool.get(pollCount++);
				
			if (!srvConn.isAvailable()) {
				System.out.println("ServerRouter - pollCount="+(pollCount-1)+", "+srvConn.getName()+", skip unavailable");		
				continue;
			}
			else {
				//got a available connection
				//System.out.println("ServerRouter - pollCount="+(pollCount-1)+", "+srvConn.getName());		
				return srvConn;
			}
		}
		return null;
	}
	
	public void checkRecovery() {
		for (ServerConnection srvConn : srvConnPool) {
			if (!srvConn.isAvailable()) {
				// *recover* if connection available again
				if (checkAvailable(srvConn)) {
					srvConn.setAvailable(true);
					System.out.println("ServerRouter - * recovered: " + srvConn.getName());
				}
				else {
					System.out.println("ServerRouter - * still unavailable: " + srvConn.getName());
				}
			}
		}
	}
	
	private boolean checkAvailable(ServerConnection srvConn) {
		// check if connection is available
		if (srvConn.isShutDown()) {
			return false;
		}
		return true;
	}
}

public class Operator {
	public static void shutdownServer(ServerConnection srvConn) {
		if (srvConn != null) {
			srvConn.setShutDown(true);
		}
	}
	
	public static void startServer(ServerConnection srvConn) {
		if (srvConn != null) {
			srvConn.setShutDown(false);
		}
	}
}

public class Main {	
	public static void main(String args[]) throws InterruptedException {
		ServerRouter router = new ServerRouter();
		for (int i=1; i<=3; i++) {
			ServerConnection srvConn = new ServerConnection();
			srvConn.setName("Server"+i);
			srvConn.setAvailable(true);
			router.addServerConnection(srvConn);
		}
		
		RequestDispatcher dispatcher = new RequestDispatcher(router);
		for (int i=1; i<=12; i++) {
			Thread.sleep(1000);
			Request req = new Request();
			req.setName("Request"+i);
			dispatcher.handleRequest(req);
			
			if (i == 3) {				
				Operator.shutdownServer(router.getServerConnection(1));
				Operator.shutdownServer(router.getServerConnection(2));
			}			
			if (i == 6) {
				Operator.startServer(router.getServerConnection(1));
			}
		}
	}
}