HttpClient服务器模拟浏览器发送请求

前言：
学习第三方登陆的时候，发现开头的知识就用到了HttpClient，也就是服务器模拟浏览器发起的请求，而我不会，于是就先花时间学习了下。
内容包括：GET、POST请求，以及各种零散的知识点。

maven 依赖

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
    <modelVersion>4.0.0</modelVersion>
    <groupId>login</groupId>
    <artifactId>wechat</artifactId>
    <packaging>war</packaging>
    <version>1.0</version>
    <name>wechat Maven Webapp</name>
    <url>http://maven.apache.org</url>
    <!-- maven默认是1.5版本，不写这段话会有警告，资源已过时 -->
    <profiles>
        <profile>
            <id>jdk-1.8</id>
            <activation>
                <activeByDefault>true</activeByDefault>
                <jdk>1.8</jdk>
            </activation>
            <properties>
                <maven.compiler.source>1.8</maven.compiler.source>
                <maven.compiler.target>1.8</maven.compiler.target>
                <maven.compiler.compilerVersion>1.8</maven.compiler.compilerVersion>
            </properties>
        </profile>
    </profiles>
    <dependencies>
        <！--tomcat中有servlet，因此scope应该声明为provided -->
        <dependency>
            <groupId>javax.servlet</groupId>
            <artifactId>servlet-api</artifactId>
            <scope>provided</scope>
            <version>3.0-alpha-1</version>
        </dependency>
        <dependency>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
            <version>3.8.1</version>
            <scope>test</scope>
        </dependency>
        <!-- https://mvnrepository.com/artifact/net.sf.json-lib/json-lib -->
        <dependency>
            <groupId>net.sf.json-lib</groupId>
            <artifactId>json-lib</artifactId>
            <version>2.4</version>
        </dependency>
        <!-- https://mvnrepository.com/artifact/org.apache.httpcomponents/httpclient -->
        <dependency>
            <groupId>org.apache.httpcomponents</groupId>
            <artifactId>httpclient</artifactId>
            <version>4.5.2</version>
        </dependency>
        <dependency>
            <groupId>org.apache.httpcomponents</groupId>
            <artifactId>httpcore</artifactId>
            <version>4.4.4</version>
        </dependency>
        <!-- https://mvnrepository.com/artifact/org.apache.httpcomponents/httpmime -->
        <dependency>
            <groupId>org.apache.httpcomponents</groupId>
            <artifactId>httpmime</artifactId>
            <version>4.5.2</version>
        </dependency>
        <!-- https://mvnrepository.com/artifact/commons-fileupload/commons-fileupload -->
        <!-- 封装了对http传输中文件相关操作 -->
        <dependency>
            <groupId>commons-fileupload</groupId>
            <artifactId>commons-fileupload</artifactId>
            <version>1.3.1</version>
        </dependency>
        <!-- https://mvnrepository.com/artifact/commons-io/commons-io -->
        <dependency>
            <groupId>commons-io</groupId>
            <artifactId>commons-io</artifactId>
            <version>2.4</version>
        </dependency>

        <!-- https://mvnrepository.com/artifact/org.json/json -->
        <dependency>
            <groupId>org.json</groupId>
            <artifactId>json</artifactId>
            <version>20160810</version>
        </dependency>

    </dependencies>
    <build>
        <finalName>wechat</finalName>
    </build>
</project>

预备知识

• HttpClients

  Factory methods for CloseableHttpClient instances.

  HttpClients API

• HttpEntity

  public interface HttpEntity
  An entity that can be sent or received with an HTTP message. Entities can be found in some requests and in responses, where they are optional.
  There are three distinct types of entities in HttpCore, depending on where their content originates:

  HttpEntity API

 public class Test{
 private RequestConfig requestConfig = RequestConfig.custom()
            .setSocketTimeout(15000)
            .setConnectTimeout(15000)
            .setConnectionRequestTimeout(15000)
            .build();

public void testHttpClientGet(String httpUrl) {
        HttpGet httpGet = new HttpGet(httpUrl);
        CloseableHttpClient httpClient = null;
        CloseableHttpResponse httpResponse = null;
        HttpEntity httpEntity = null;
        String responseContent = null;
        try {
            //创建默认的httpClient实例
            httpClient = HttpClients.createDefault();
            httpGet.setConfig(requestConfig);
            httpResponse = httpClient.execute(httpGet);
            httpEntity = httpResponse.getEntity();
            responseContent = EntityUtils.toString(httpEntity, "utf-8");
            /*
            another encode method could refers to this blog
            "http://blog.sina.com.cn/s/blog_6d002146010130wv.html"
             */
            System.out.println(responseContent);
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            try {
                if (httpResponse != null) {
                    httpResponse.close();
                }
                if (httpClient != null) {
                    httpClient.close();
                }
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    }

/**
     * POST请求1
     * @param httpUrl eg:http://localhost:8080/testPost?username=cwy&password=123
     */
    public void sendHttpPost(String httpUrl) {
        HttpPost httpPost = new HttpPost(httpUrl);
        testHttpClietPost(httpPost);
    }

    /**
     * POST请求2
     * @param httpUrl 请求地址
     * @param params  key1=value1&key2=value2
     */
    public void sendHttpPost(String httpUrl, String params) {
        HttpPost httpPost = new HttpPost(httpUrl);
        try {

            StringEntity stringEntity = new StringEntity(params, "UTF-8");
            /*
            *http://blog.csdn.net/blueheart20/article/details/45174399
             */
            stringEntity.setContentType("application/x-www-form-urlencoded");
            /*
            StringEntity implements httpEntity extends AbstractHttpEntity
            AbstractHttpEntity implements httpEntity
            httpEntity has get-Content/ContentEncoding/ContentLength/ContentType 
            method
            but only AbstractHttpEntity has set-xx method
            more information refers to  API     
            "http://hc.apache.org/httpcomponents-core
            ga/httpcore/apidocs/org/apache/http/HttpEntity.html"
             */
            httpPost.setEntity(stringEntity);
        } catch (Exception e) {
            e.printStackTrace();
        }
        testHttpClietPost(httpPost);
    }

     /**
     * 发送POST请求（带文件）
     * @param httpUrl 请求地址
     * @param maps  请求参数
     * @param fileLists 文件列表
     */
    public void sendHttpPost3(String httpUrl, Map<String,String> maps, List<File> fileLists){
        HttpPost httpPost=new HttpPost(httpUrl);
        MultipartEntityBuilder meBuilder=MultipartEntityBuilder.create();
        for(String key:maps.keySet()){
            meBuilder.addPart(key,new StringBody(maps.get(key),ContentType.TEXT_PLAIN));
        }
        for(File file:fileLists){
            FileBody fileBody=new FileBody(file);
            meBuilder.addPart("files",fileBody);
        }
        HttpEntity httpEntity=meBuilder.build();
        httpPost.setEntity(httpEntity);
        testHttpClietPost(httpPost);
    }
/*
//测试
public void testPost3(){
        Map<String,String> maps=new HashMap<>();
        maps.put("username","cwy");
        maps.put("password","123456");
        List<File> files=new ArrayList<>();
        files.add(new File("E:\\前端系列\\作业--完整网页\\images\\11.png"));
        sendHttpPost("http://localhost:8080/testPostFiles",maps,files);
    }
*/
public void testHttpClietPost(HttpPost httpPost) {
        CloseableHttpClient closeableHttpClient = null;
        CloseableHttpResponse closeableHttpResponse = null;
        HttpEntity httpEntity = null;
        String responseContent = null;
        try {
            closeableHttpClient = HttpClients.createDefault();
            httpPost.setConfig(requestConfig);
            closeableHttpResponse = closeableHttpClient.execute(httpPost);
            httpEntity = closeableHttpResponse.getEntity();
            responseContent = EntityUtils.toString(httpEntity, "UTF-8");
            System.out.println("responseContent=" + responseContent);
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            try {
                if (closeableHttpResponse != null) {
                    closeableHttpResponse.close();
                }
                if (closeableHttpClient != null) {
                    closeableHttpClient.close();
                }
            } catch (Exception e) {
            }

        }
    }
 }

GET请求以及非文件POST请求servlet

 //非通用，仅为测试
 protected void doPost(javax.servlet.http.HttpServletRequest request, javax.servlet.http.HttpServletResponse response) throws javax.servlet.ServletException, IOException {
        System.out.println("QueryString"+request.getQueryString());
        System.out.println("CharacterEncoding"+request.getCharacterEncoding());
        System.out.println("ContentType"+request.getContentType());
        Map<String,String> map=new HashMap();
        String username=request.getParameter("username");
        String password=request.getParameter("password");
        map.put("username",username);
        map.put("password",password);
        map.put("requestContentType",request.getContentType());
        System.out.println("map="+map);
        response.setContentType("application/json; charset=utf-8");
        response.setCharacterEncoding("UTF-8");
        String jsonObject=JSONObject.valueToString(map);
        System.out.println("jsonObject="+jsonObject);
        OutputStream out = response.getOutputStream();
        out.write(jsonObject.getBytes("utf-8"));
        out.flush();
    }

文件请求抓包
写了简易的jsp页面对文件上传进行抓包测试

<form action="${pageContext.request.contextPath}/testPostFiles" enctype="multipart/form-data" method="post">
        上传用户: <input type="text" name="username"><br/>
        上传文件1: <input type="file" name="file1"><br/>
        上传文件2：<input type="file" name="file2"><br/>
        <input type="submit" value="submit">
</form>

• 注意，对于form/data这种形式，无法以key-value的形式获取值，只能以流的方式，
  如果选择手动实现，不使用第三方jar包（如我这儿使用的uploadfile），就要以分隔字符、判断、切割等复杂操作来实现流的转换。
  对于Content-Type，这儿有篇文章 点击链接

携带文件POST请求servlet

/*
这只是最简单的实现，还可以优化
1、文件上传能够设置缓冲区 
2、文件名应该有UUID或其他唯一名，避免文件覆盖
*/
protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
        String savePath=getServletContext().getRealPath("/WEB-INF/upload");
        File file=new File(savePath);
        if(!file.exists()&&!file.isDirectory()){
            System.out.println(savePath+"目录不存在,需要创建");
            file.mkdir();
        }
        String message="";
        /*
        使用Apache文件上传组件处理文件上传步骤
        1、创建一个DiskFileItemFactory工厂
        2、创建一个文件上传解析器
        3、判断提交上来的数据是否是上传表单的数据
        4、使用ServletFileUpload解析器解析上传数据，解析结果返回一个List<FileItem>
         */
        try{
            DiskFileItemFactory factory=new DiskFileItemFactory();
            ServletFileUpload upload=new ServletFileUpload(factory);
            //解决上传文件中的中文乱码
            upload.setHeaderEncoding("UTF-8");
            if(!ServletFileUpload.isMultipartContent(request)){
                //按照传统方式获取数据
                return;
            }
            List<FileItem> list=upload.parseRequest(request);
            for(FileItem item:list){
                //如果fileitem中封装的是普通输入项的数据
                if(item.isFormField()){
                    String name=item.getFieldName();
                    String value=item.getString("utf-8");
                    System.out.println("name="+name+" ;value="+value);
                }else{
                    //若封装的是上传文件
                    String filename=item.getName();
                    System.out.println("filename="+filename);
                    if(filename==null||filename.trim().equals("")){
                        continue;
                    }
                    //注意，不同的浏览器提交的文件名是不一样的，有些带路径，有些不带
                    //处理上传文件的路径，统一只保留文件名部分
                    filename=filename.substring(filename.lastIndexOf("\\")+1);
                    InputStream in=item.getInputStream();
                    FileOutputStream out=new FileOutputStream(savePath+"\\"+filename);
                    //创建一个缓冲区
                    byte buffer[]=new byte[1024];
                    //判断输入流中的数据是否已读完的标志
                    int len=0;
                    while((len=in.read(buffer))>0){
                        out.write(buffer,0,len);
                    }
                    in.close();
                    out.close();
                    //删除处理文件上传时生成的临时文件
                    item.delete();
                }
            }
        }catch (Exception e){
            e.printStackTrace();
        }
    }