【网络爬虫】使用HttpClient4.3.5抓取数据

使用jar——Apache client

下载地址：
http://hc.apache.org/downloads.cgi
结构：

代码结构：

具体代码：

抓取结果封装

/**
 *  抓取结果的封装
 * @author tsj-pc
 *
 */
public class CrawlResultPojo {
    private boolean isSuccess;
    private String pageContent;
    private int httpStatuCode;

    public boolean isSuccess() {
        return isSuccess;
    }
    @Override
    public String toString() {
        return "CrawlResultPojo [httpStatuCode=" + httpStatuCode
                + ", isSuccess=" + isSuccess + ", pageContent=" + pageContent
                + "]";
    }

    public void setSuccess(boolean isSuccess) {
        this.isSuccess = isSuccess;
    }

    public String getPageContent() {
        return pageContent;
    }

    public void setPageContent(String pageContent) {
        this.pageContent = pageContent;
    }

    public int getHttpStatuCode() {
        return httpStatuCode;
    }

    public void setHttpStatuCode(int httpStatuCode) {
        this.httpStatuCode = httpStatuCode;
    }

}

RUL任务的POJO类

/**
 * url任务的pojo类
 * @author tsj-pc
 *
 */
public class UrlPojo {
    private Map<String, Object> parasMap;

    public Map<String, Object> getParasMap() {
        return parasMap;
    }

    public void setParasMap(Map<String, Object> parasMap) {
        this.parasMap = parasMap;
    }

    public UrlPojo(String url) {
        this.url = url;
    }

    public UrlPojo(String url, Map<String, Object> parasMap) {
        this.url = url;
        this.parasMap = parasMap;
    }

    @Override
    public String toString() {
        return "UrlPojo [taskLevel=" + taskLevel + ", url=" + url + "]";
    }

    public UrlPojo(String url, TaskLevel taskLevel) {
        this.url = url;
        this.taskLevel = taskLevel;
    }

    private String url;
    private TaskLevel taskLevel = TaskLevel.MIDDLE;

    public String getUrl() {
        return url;
    }

    public void setUrl(String url) {
        this.url = url;
    }

    public TaskLevel getTaskLevel() {
        return taskLevel;
    }

    public void setTaskLevel(TaskLevel taskLevel) {
        this.taskLevel = taskLevel;
    }

    public HttpURLConnection getConnection() {
        try {
            URL url = new URL(this.url);
            URLConnection connection = url.openConnection();
            if (connection instanceof HttpURLConnection) {
                return (HttpURLConnection) connection;
            } else {
                throw new Exception("connection is errr!");
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
        return null;
    }

    public String getHost() {
        try {
            URL url = new URL(this.url);
            return url.getHost();
        } catch (Exception e) {
            e.printStackTrace();
        }
        return null;
    }
}

使用HttpClient4.3.5抓取数据

public class HttpClientCrawlerImpl implements ICrawler {
    //当CloseableHttpClient不再需要，并且不再连接管理的范围，需要调用CloseableHttpClient.close()方法将其关闭
    public CloseableHttpClient httpclient = HttpClients.custom().build();

    @Override
    public CrawlResultPojo crawl(UrlPojo urlPojo) {
        if (urlPojo == null) {
            return null;
        }
        CrawlResultPojo crawlResultPojo = new CrawlResultPojo();
        CloseableHttpResponse response1 = null;
        BufferedReader br = null;
        try {
            HttpGet httpget = new HttpGet(urlPojo.getUrl());
            response1 = httpclient.execute(httpget);
            HttpEntity entity = response1.getEntity();
            InputStreamReader isr = new InputStreamReader(entity.getContent(),
                    "utf-8");
            br = new BufferedReader(isr);

            String line = null;
            StringBuilder stringBuilder = new StringBuilder();
            while ((line = br.readLine()) != null) {
                stringBuilder.append(line + "\n");
            }
            crawlResultPojo.setSuccess(true);
            crawlResultPojo.setPageContent(stringBuilder.toString());
            return crawlResultPojo;
        } catch (Exception e) {
            e.printStackTrace();
            crawlResultPojo.setSuccess(false);
        } finally {
            if (response1 != null) {
                try {
                    response1.close();
                } catch (IOException e1) {
                    e1.printStackTrace();
                }
            }
            if (br != null) {
                try {
                    br.close();
                } catch (IOException e1) {
                    e1.printStackTrace();
                }
            }
        }

        return crawlResultPojo;
    }

    /**
     * 传入加入参数post参数的url pojo
     */
    public CrawlResultPojo crawl4Post(UrlPojo urlPojo) {
        if (urlPojo == null) {
            return null;
        }
        CrawlResultPojo crawlResultPojo = new CrawlResultPojo();
        CloseableHttpResponse response1 = null;
        BufferedReader br = null;
        try {
            RequestBuilder rb = RequestBuilder.post().setUri(
                    new URI(urlPojo.getUrl()));
            ;
            // .addParameter("IDToken1",
            // "username").addParameter("IDToken2", "password").build();

            Map<String, Object> parasMap = urlPojo.getParasMap();
            if (parasMap != null) {
                for (Entry<String, Object> entry : parasMap.entrySet()) {
                    rb
                            .addParameter(entry.getKey(), entry.getValue()
                                    .toString());
                }
            }
            HttpUriRequest httpRequest = rb.build();
            response1 = httpclient.execute(httpRequest);
            HttpEntity entity = response1.getEntity();
            InputStreamReader isr = new InputStreamReader(entity.getContent(),
                    "utf-8");
            br = new BufferedReader(isr);

            String line = null;
            StringBuilder stringBuilder = new StringBuilder();
            while ((line = br.readLine()) != null) {
                stringBuilder.append(line + "\n");
            }
            crawlResultPojo.setSuccess(true);
            crawlResultPojo.setPageContent(stringBuilder.toString());
            return crawlResultPojo;
        } catch (Exception e) {
            e.printStackTrace();
            crawlResultPojo.setSuccess(false);
        } finally {
            if (response1 != null) {
                try {
                    response1.close();
                } catch (IOException e1) {
                    e1.printStackTrace();
                }
            }
            if (br != null) {
                try {
                    br.close();
                } catch (IOException e1) {
                    e1.printStackTrace();
                }
            }
        }

        return crawlResultPojo;
    }

    public static void main(String[] args) throws Exception {
        HttpClientCrawlerImpl httpClientCrawlerImpl = new HttpClientCrawlerImpl();
        String url = "http://www.wangdaizhijia.com/front_select-plat";
        UrlPojo urlPojo = new UrlPojo(url);
        Map<String, Object> parasMap = new HashMap<String, Object>();

        int max_page_number = 1000;

        parasMap.put("currPage", 30);
        parasMap.put("params", "");
        parasMap.put("sort", 0);
        urlPojo.setParasMap(parasMap);

        CrawlResultPojo resultPojo = httpClientCrawlerImpl.crawl4Post(urlPojo);

        if (resultPojo != null) {
            System.out.println(resultPojo);
        }
    }
}

结果：