HTTPProxy在自动化测试中的运用及精简实现

最新推荐文章于 2023-05-12 10:14:17 发布

xreztento

最新推荐文章于 2023-05-12 10:14:17 发布

阅读量1.4k

点赞数

本文链接：https://blog.csdn.net/xreztento/article/details/39251511

版权

Java 同时被 3 个专栏收录

27 篇文章 1 订阅

订阅专栏

测试工具

19 篇文章 2 订阅

订阅专栏

软件测试

16 篇文章 0 订阅

订阅专栏

HTTPProxy是一个中间程序，它既可以担当浏览器（客户端）的角色也可以担当WebServer（服务器）的角色。HTTPProxy代表浏览器向WebServer发送请求，浏览器的请求经过代理，会在代理内部得到服务或者经过一定的转换转至WebServer。一个代理必须能同时实现HTTP/1.1协议规范（RFC2616）中对客户端和服务器所作的要求。

HTTPProxy在自动化测试中所占的地位十分重要，并作为非透明代理得到广泛应用，非透明代理（non-transparent proxy）需修改请求或响应，以便为用户代理提供附加服务，这些附加服务主要包括：

（1）对于某些性能测试工具如LoadRunner、JMeter等，通过HTTPProxy实现对HTTP请求和响应进行详细日志记录，并通过脚本分析生成器对日志脚本化，从而实现对测试脚本的录制功能；

（2）对于某些安全性测试工具如SPIKE Proxy，通过HTTPProxy在实现用户访问行为的同时，对这些行为自动应用变异技术，构造Fuzzing数据执行模糊测试。

HTTPProxy非透明代理实现的关键在于：

（1）首先，它要成为一个功能完整的HTTPProxy，就必须依照RFC2616中对于代理的要求加以实现，如必须处理响应头属性中包含Transfer-Encoding:chunked的情况；

（2）加入附加服务，如提供对于脚本的实时录制转换或记录日志之后再分析进行脚本化的服务，都必须依赖对于HTTP请求和响应的详细记录，这不仅仅是单纯的对浏览器与WebServer之间HTTP协议数据往来的僵化记录，而是还需要处理如响应头属性中包含如Content-Encoding:gzip的各类情况；又如提供对于访问行为自动应用变异技术的服务，就需要代理内部实现一个完善的模糊器；

（3）实现一个小型的Web容器，容器对所有接入代理的浏览器提供服务，成为管理控制入口。

一个Java版本的HTTPProxy的精简实现如下：

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.PrintWriter;
import java.lang.Runnable;
import java.net.InetSocketAddress;
import java.net.Socket;
import java.net.UnknownHostException;
import java.nio.ByteBuffer;
import java.nio.channels.ServerSocketChannel;
import java.nio.channels.SocketChannel;
import java.nio.channels.UnresolvedAddressException;

public class SimpleProxy implements Runnable{
	
	private final static int BYTEBUFFER_CAPACITY = 1024 * 1024 * 2;//设定字节缓冲区的内存空间大小，用于保存HTTP各类数据
	
	private final static int DEFAULT_HTTP_PORT = 80;//默认的HTTP协议服务端口
	private final static int DEFAULT_HTTPS_PORT = 443;//默认的HTTPS协议服务端口

	private SocketChannel inChannel = null;//代理与浏览器通信Socket
	private SocketChannel outChannel = null;//代理与Web服务器通信Socket
	
	private Socket socket = null;//用于处理代理与Web服务器间通过HTTPS通信 
	
	private InputStream in = null;//代理从Web服务器读取数据的InputStream对象
	
	private ByteBuffer buffer = ByteBuffer.allocateDirect(BYTEBUFFER_CAPACITY);//分配一个字节缓冲区用于处理inChannel通道和outChannel通道的数据
	
	//构造函数
	public SimpleProxy(SocketChannel inChannel){
		
		this.inChannel = inChannel;
	}
	
	public void run(){
		proxy();
	}
	
	private void proxy(){

		String httpRequestHandlerData = fromBrowserToProxy();//处理从浏览器到代理的逻辑
		
		//如果从浏览器到代理请求完成，会初始化outChannel和in对象用于处理代理与Web服务器之间的通信
		if(in != null){//初始化成功
			
			fromProxyToWebServer(httpRequestHandlerData);//处理代理到Web服务器的逻辑
			
			fromWebServerToProxy();//处理Web服务器到代理的逻辑
				
			fromProxyToBrowser();//处理代理到浏览器的逻辑
		}
		
	
		//执行完该代理连接逻辑后，设置字节缓冲区为空
		buffer = null;
		
		//关闭一系列对象
		if(in != null){
			try {
				in.close();
			} catch (IOException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
		}
		if(socket != null){
			try {
				socket.close();
			} catch (IOException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
		}
		
		if(outChannel != null){
			try {
				outChannel.close();
			} catch (IOException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
		}
		
		if(inChannel != null){
			try {
				inChannel.close();
			} catch (IOException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
		}
		
		//由于使用字节缓冲区ByteBuffer，因此强制执行GC操作，回收非虚拟机堆栈空间的内存
		System.gc();
	}
	
	/**
	 * 处理从浏览器到代理的逻辑
	 * @return 返回一个被处理后的HTTP Request Data
	 */
	private String fromBrowserToProxy(){
		
		String httpRequestRawData = receiveHttpRequest();
		String[] httpRequestInfor = httpRequestHandler(httpRequestRawData);//对从浏览器发起的HTTP Request Data进行处理，获取和处理相关信息
		String hostValue = httpRequestInfor[0];//获取Host值
		boolean isSSL = Boolean.getBoolean(httpRequestInfor[1]);//判断是否为HTTPS
		String httpRequestHandledData = httpRequestInfor[2];//获取处理后的HTTP Request Data
		String host;//用于代理与Web服务器建立Socket连接的host
		int port;//用于代理与Web服务器建立Socket连接的port
		
		if(hostValue.contains(":")) {//如果Host值包含':'，说明使用了非默认端口
			host = hostValue.split(":")[0].trim();
			port = Integer.valueOf(hostValue.split(":")[1].trim());
		} else {//使用了默认端口
			host = hostValue;
			if(!isSSL){//HTTP协议默认端口
				port = DEFAULT_HTTP_PORT;
			} else {//HTTPS协议默认端口
				port = DEFAULT_HTTPS_PORT;
			}
		}
		if(host.length() > 0){//获取了host信息
			
			//初始化代理与Web服务器的Socket连接
			createOutChannel(host, port, isSSL);
		}
		
		return httpRequestHandledData;
		
	}
	
	private void fromProxyToWebServer(String httpRequestHandlerData){
		PrintWriter out = null;//用于发送HTTPS数据
		
		buffer.clear();//清空buffer准备写入
		
		//如果HTTP Request Header中带有Referer，进行处理
		buffer.put(httpRequestHandlerData.getBytes());
			
		buffer.flip();//翻转buffer准备读出
		
		if(outChannel != null){
	
			while(buffer.hasRemaining()){
				try {
					outChannel.write(buffer);
				} catch (IOException e) {
					// TODO Auto-generated catch block
					e.printStackTrace();
					buffer.clear();
				}//向Web服务器发送HTTP Request
			}
		} else if(socket != null) {
			try {
				out = new PrintWriter(socket.getOutputStream());
				out.write(httpRequestHandlerData);
			} catch (IOException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
			finally{
				out.close();
			}
		}
	}
	
	private void fromWebServerToProxy(){
		
		StringBuffer stringBuffer = new StringBuffer();
	
		buffer.clear();
		
		HTTPCommonMethod hcm = new HTTPCommonMethod();
		
		//获取Web服务器HTTP Response Header信息
		int headerLength = hcm.readHTTPResponseHeader(buffer,in);
		
		for(int i = 0;i < headerLength; i++){
			stringBuffer.append((char)buffer.get());
		}
		
		String httpResponseHeaderStr = stringBuffer.toString();
		
		HTTPResponseHeaderHandler hrhh = new HTTPResponseHeaderHandler(httpResponseHeaderStr);
		int contentLength = 0;
		contentLength = hrhh.getContentLengthtValue();
		String contentType = hrhh.getContentTypeValue();
		String acceptEncoding = hrhh.getContentEncodingValue();
		int status = hrhh.getResponseStatusValue();
		
		
		if(status == 200)//HTTP Response Status为200 OK
		{
			if(contentType.contains("text"))//HTTP　Response实体类型为text
			{
				if(contentLength == 0 && httpResponseHeaderStr.contains("chunked"))//当contentLength为0，并且存在chunked值，进行chunked处理
				{
					contentLength = hcm.chunkResponseHandler(buffer,in);//chunked实体处理
					httpResponseHeaderStr = hrhh.chunkedHandle(contentLength);//chunked对HTTP Response Header处理
				}
				else//非chunked
				{
					buffer.clear();
					for(int i = 0; i < contentLength;i++){
						
						try {
							buffer.put((byte)in.read());
						} catch (IOException e) {
							// TODO Auto-generated catch block
							e.printStackTrace();
							buffer.clear();
						}
					}
					buffer.flip();
				}
				
				byte[] byteArray = new byte[contentLength];
				for(int i = 0; i < contentLength; i++){
					byteArray[i] = buffer.get();
				}
				
				if(acceptEncoding.contains("gzip")){
					System.out.println("decoding gzip!");
					ByteArrayInputStream bais = new ByteArrayInputStream(byteArray);  
					ByteArrayOutputStream baos = new ByteArrayOutputStream();  
					try {
						hcm.decompress(bais,baos);
					} catch (IOException e) {
						// TODO Auto-generated catch block
						e.printStackTrace();
						buffer.clear();
					}
					
				    System.out.println(new String(baos.toByteArray()));
				}else {
					
					System.out.println(new String(byteArray));
				}
				
			    
			    buffer.clear();//清空buffer准备写入
			    
			    buffer.put(httpResponseHeaderStr.getBytes()).put(byteArray);//将处理后的Header和实体写入buffer
			  
				buffer.flip();//翻转buffer准备读取
				
			} else { //HTTP Response实体类型为其他类型

				buffer.limit(buffer.capacity());
				for(int i = 0; i < contentLength;i++){
					
					try {
						buffer.put((byte)in.read());
					} catch (IOException e) {
						// TODO Auto-generated catch block
						e.printStackTrace();
						buffer.clear();
					}
				}
				buffer.flip();
				
			}
		} else {
			
			buffer.flip();
			System.out.println("no body:" + buffer.remaining());
		}
		
	}
	

	private void fromProxyToBrowser(){
		
		if(inChannel.isConnected())
		{
			while(buffer.hasRemaining()){
				try {
					inChannel.write(buffer);
				} catch (IOException e) {
					// TODO Auto-generated catch block
					e.printStackTrace();
					buffer.clear();
				}
			}
		}
	}
	
	
	private String receiveHttpRequest(){
		StringBuffer stringBuffer = new StringBuffer(BYTEBUFFER_CAPACITY);
		if(inChannel.isConnected()){
			buffer.clear();
			try {
				if(inChannel.read(buffer) != -1){ //接收浏览器HTTP Request
					
					buffer.flip();
					
					while(buffer.hasRemaining()){
						stringBuffer.append((char)buffer.get());//HTTP Request请求字符化
					}
				}
			} catch (IOException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
				buffer.clear();
			}
		}
		return stringBuffer.toString();
	}
	
	/**
	 * 
	 * @param httpResponseData HTTP Request Data字符串
	 * @return 处理后的HTTP Request Data的片段信息及处理后的HTTP Request Data
	 */
	private String[] httpRequestHandler(String httpRequestData){
		
		HTTPRequestHeaderHandler hqhh = new HTTPRequestHeaderHandler(httpRequestData);
		String[] httpResponseInfor = new String[3];
		httpResponseInfor[0] = hqhh.getHostValue().trim();
		httpResponseInfor[1] = String.valueOf(hqhh.isSSL());
		httpResponseInfor[2] = hqhh.handle(httpResponseInfor[0]);
		return httpResponseInfor;
		
	}
	
	/**
	 * 
	 * @param host
	 * @param port
	 * @param isSSL
	 */
	public void createOutChannel(String host, int port, boolean isSSL){
		
		if(isSSL){
			//处理SSL连接
			
		} else {
			
			try {
				outChannel = SocketChannel.open();
				outChannel.connect(new InetSocketAddress(host, port));
				outChannel.finishConnect();
				
				in = outChannel.socket().getInputStream();
			} 
			catch(UnresolvedAddressException e){
				System.out.println(host + ":" + port);
				e.printStackTrace();
				
			} catch (UnknownHostException e) {
				// TODO Auto-generated catch block
				System.out.println(host + ":" + port);
				e.printStackTrace();
			} catch (IOException e) {
				// TODO Auto-generated catch block
				System.out.println(host + ":" + port);
				e.printStackTrace();
			}
		}
	}
	
	public static void main(String[] args){
		ServerSocketChannel ssChannel = null;
		
		try {
			ssChannel = ServerSocketChannel.open();
			ssChannel.socket().bind(new InetSocketAddress(10009));
			ssChannel.configureBlocking(true);
		
			
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}

		
		
		try
		{   
			while(true){
				SocketChannel inChannel = null;
				inChannel =  ssChannel.accept();
				SimpleProxy sp = new SimpleProxy(inChannel);
				Thread thread = new Thread(sp);
				thread.start();
			}
			
		}
		catch(Exception e){
				e.printStackTrace();
		}
		finally{
			
			if(ssChannel != null){
				try {
					ssChannel.close();
				} catch (IOException e) {
					// TODO Auto-generated catch block
					e.printStackTrace();
				}
			}
		}
	}
}

public class HTTPResponseHeaderHandler {
	private final String CRLF = "\r\n";
	private final String CR = "\r";
	
	private String httpContent = null;
	CommonMethod cm = new CommonMethod();
	
	public HTTPResponseHeaderHandler(String httpContent){
		this.httpContent = httpContent;
	}
	
	/*********************************
	 * 处理带有chunked的HTTP Response Header
	 * @param contentLengthValue
	 * @return
	 ************************************/
	public String chunkedHandle(int contentLengthValue){
		StringBuffer sb = new StringBuffer();
		String[] headerLines = httpContent.split(CRLF);
		for(int i = 0; i < headerLines.length; i++){
			if(!headerLines[i].contains("chunked")){
				sb.append(headerLines[i]);
			} else {
				sb.append("Content-Length:");
				sb.append(contentLengthValue);
			}
			sb.append(CRLF);
		}
		sb.append(CRLF);
		return sb.toString();
	
	}
	
	/************************************
	 * 获取HTTP Response Header中的Content-Length值
	 * @return
	 ************************************/
	public int getContentLengthtValue(){
		if(httpContent.contains("Content-Length:")){
			String value = cm.string_param_save(httpContent, "Content-Length:", CR,0);
			return Integer.valueOf(value.trim());
		}
		else {
			return 0;
		}
	}
	
	/**************************************
	 * 获取HTTP Response Header中的HTTP Status值
	 * @param httpVersion
	 * @return
	 *************************************/
	public int getResponseStatusValue(){
		
		if(httpContent.contains("HTTP/")){
			String value = httpContent.substring(9, 12);
			return Integer.valueOf(value);
		}
		else {
			return 0;
		}
	}
	
	public String getContentTypeValue(){
		String value = cm.string_param_save(httpContent, "Content-Type:", CR,0);
		if(value != null){
			return value.trim();
		}
		return "";
	}
	 
	public String getContentEncodingValue(){
			String value = cm.string_param_save(httpContent, "Content-Encoding:", CR,0);
			if(value != null){
				return value.trim();
			}
			return "";
	}
}

public class HTTPRequestHeaderHandler {
	
	private final String CRLF = "\r\n";
	private final String CR = "\r";
	
	
	private String httpContent = null;
	CommonMethod cm = new CommonMethod();
	
	public HTTPRequestHeaderHandler(String httpContent){
		this.httpContent = httpContent;
	}
	
	/************************
	 * 获取HTTP Request中的Host内容
	 * @return
	 */
	public String getHostValue(){
		
		if(httpContent.contains("Host:")){
			String value = cm.string_param_save(httpContent, "Host:", CR,0);
			return value;
		} else if(httpContent.split(CRLF)[0].contains("http://")) {
			String value = cm.string_param_save(httpContent.split(CRLF)[0], "http://", "/",0);
			return value;
		} else if(httpContent.split(CRLF)[0].contains("https://")) {
			String value = cm.string_param_save(httpContent.split(CRLF)[0], "https://", "/",0);
			return value;
		} else {
			
			return "";
		}
	}
	
	public boolean isSSL(){
		if(httpContent.split("CRLF")[0].contains("https://")){
			return true;
		}
		else {
			return false;
		}
	}
	/************************
	 * 获取HTTP Request中的HTTP版本内容
	 * @return
	 */
	public String getHTTPVersion(){
		if(httpContent.contains("HTTP/")){
			String value = cm.string_param_save(httpContent, "HTTP/", " ",0);
			return value;
		}
		else {
			return "";
		}
	}
	
	
	public String handle(String hostValue){
		
		if(httpContent.contains("http://")){
			return httpContent.replace("http://" + hostValue, "");
		}
		else if(httpContent.contains("https://")){
			return httpContent.replace("https://" + hostValue, "");
		}

		return httpContent;
	}
}

import java.util.ArrayList;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class CommonMethod {


public String string_param_save(String httpContent, String leftedge, String rightedge, int index){
		
		if(httpContent.toLowerCase().contains(leftedge.toLowerCase()))
		{
			ArrayList
   
   
    
     paramList = new ArrayList
    
    
     
     (2);
			Pattern pa = Pattern.compile(Pattern.quote(leftedge) + "(.*?)" + Pattern.quote(rightedge),Pattern.CASE_INSENSITIVE);
			Matcher ma = pa.matcher(httpContent);
			while(ma.find()){
				
				paramList.add(ma.group(1));
			}
			return paramList.get(index);
		} else {
			return null;
		}
	}
	
}

import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.ByteBuffer;
import java.util.zip.GZIPInputStream;

public class HTTPCommonMethod {
	private final int GZIP_DECOMPRESS_BUFFER_SIZE = 10240;
	private final char CR = '\r';
	
	/********************
	 * 用于处理HTTP Response Header中具有"Content-Encoding:gzip"的HTTP Response实体数据的解压缩
	 * @param in gzip压缩数据的InputStream
	 * @param out gzip解压数据的OutputStream
	 * @throws IOException 如果输入不符合gzip压缩格式抛出异常
	 */
	public void decompress(InputStream in, OutputStream out) throws IOException{
		GZIPInputStream gis = new GZIPInputStream(in);
		byte[] buffer = new byte[GZIP_DECOMPRESS_BUFFER_SIZE];
		int count;
		//循环处理知道解压完成
		while((count = gis.read(buffer, 0, GZIP_DECOMPRESS_BUFFER_SIZE)) != -1){
			out.write(buffer,0,count);
		}
		gis.close();
	}
	
	
	/***********************
	 * 主要用于处理对HTTP Response Header数据的读取，也用于某些特殊的不含有Header信息的HTTP Response实体数据的读取
	 * @param buffer
	 * @param in
	 * @return
	 */
	public int readHTTPResponseHeader(ByteBuffer buffer,InputStream in){
		int httpResponseHeaderSize = 0;//记录HTTP Response Header的大小
		buffer.clear();//清空ByteBuffer准备写入
		
		/***************************
		 * 循环读取，直到遇到一个空行的CRLF
		 ***************************/
		while(true){
			
			try {
					char ch = (char)in.read();
					//某些不含有Header信息的HTTP Response数据会以字节65535为结束符
					if((int)ch == 65535){
						break;
					}
					
					buffer.put((byte)ch);
					httpResponseHeaderSize++;
					//直到两个连续的CRLF循环终止
					if(ch == CR){
						buffer.put(((byte)in.read()));//读取\n
						httpResponseHeaderSize++;
						ch = (char)in.read();//读取\r
						httpResponseHeaderSize++;
						buffer.put((byte)ch);
						if(ch == CR){
							buffer.put(((byte)in.read()));//读取\n
							httpResponseHeaderSize++;
							
							buffer.flip();
							break;
						}	
				} 
				
			} catch (IOException e) {
				e.printStackTrace();
				break;
			}
		}
		return httpResponseHeaderSize;
	}
	

	/*********************
	 * 用于处理HTTP Response Header中具有"Transfer-Encoding:chunked"的HTTP Response实体数据的读取
	 * @param chunkEntityBuffer 记录chunkEntity内容的ByteBuffer
	 * @param in 与Web服务器建立连接的输入流
	 * @return 返回Content-Length值
	 */
	public int chunkResponseHandler(ByteBuffer chunkEntityBuffer,InputStream in){
		
		chunkEntityBuffer.clear();//清空ByteBuffer准备写入
		int contentLength = 0;//Content-Length值最后经过累加，遵循HTTP协议的要求，对chunk内容组合，向浏览器返回一个具有Content-Length值的HTTP Response
		StringBuffer chunkSizeBuffer = null;
		String hexChunkSize = "";
		while(true){
			
			chunkSizeBuffer = new StringBuffer();
			
			/*********************
			 * 读取Chunk-Size并记录
			 **********************/
			char chunkSizeChar;
			
			try {
					while((chunkSizeChar = (char)in.read()) != CR){
						chunkSizeBuffer.append(chunkSizeChar);
					}
					in.read();//读取\n,实现读取CRLF
			} catch (IOException e) {
				
				e.printStackTrace();
				break;
			}
			
			hexChunkSize = chunkSizeBuffer.toString();//得到一个Hex数字值的Chunk-Size
			
			//如果Chunk-Size为0，表示到达Chunk块的末尾，将ByteBuffer翻转以便读取，退出循环
			if(Integer.decode("0x" + hexChunkSize) == 0){
				chunkEntityBuffer.flip();
				break;
			}
			
			contentLength += Integer.decode("0x" + hexChunkSize);//累加取得Content-Length大小
			
			/******************************
			 * 对Chunk实体进行接收，长度大小为Chunk-Size
			 ****************************/

			for(int i = 0;i < Integer.decode("0x" + hexChunkSize); i++){
				try {
					chunkEntityBuffer.put((byte)in.read());
				} catch (IOException e) {
					
					e.printStackTrace();
					break;
				}
			}
			//读取CRLF,并抛掉,以便拼接Chunk实体
			try {
				in.read();
				in.read();
			} catch (IOException e) {
				
				e.printStackTrace();
				break;
			}

		}
		
		return contentLength;
	}
}