httpclient爬取网页

1、设置爬取的url

String url = "http://www.gametoutiao.com/toutiao/index.html";

2、建立爬取的客户端

HttpClient client = new HttpClient();
client.getHttpConnectionManager().getParams().setConnectionTimeout(90000);
client.getHttpConnectionManager().getParams().setSoTimeout(90000);

3、建立爬取的请求头

HashMap<String, String> headerMap = new HashMap<>();
headerMap.put("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
headerMap.put("Accept-Language", "zh-CN,zh;q=0.8");
headerMap.put("Host", "http://www.gametoutiao.com");

4、创建请求的get方法

GetMethod getmethod = new GetMethod(url);
getmethod.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler());
getmethod.getParams().setParameter("http.protocol.cookie-policy", CookiePolicy.BROWSER_COMPATIBILITY);
getmethod.setRequestHeader("Accept-Encoding", "gzip,deflate");

5、将请求头添加到get方法中去

for(Entry<String, String> entry:headerMap.entrySet()){
    getmethod.addRequestHeader(entry.getKey(), entry.getValue());
}

6、执行方法

int statusCode = client.executeMethod(getmethod);
if(statusCode != HttpStatus.SC_OK){
    System.out.println("fail!!!!");
}

7、获得方法的返回结果

byte[] result_body = getmethod.getResponseBody();
String body = new String(result_body);

8、对结果的处理，找到最大的页数

Pattern pattern = Pattern.compile("(\\d+).html\">尾页");
Matcher matcher = pattern.matcher(body);
if(matcher.find()){
    System.out.println(matcher.group(1));
}else{
    System.out.println("not find");
}

9、获取每篇文章的title，url

//<li class="col-sm-4 col-md-4">
Pattern pattern_content = Pattern.compile("<li class=\"col-sm-4 col-md-4\">(.*?)</li>",Pattern.DOTALL);
Matcher matcher_content = pattern_content.matcher(body);
while(matcher_content.find()){
    String lis = matcher_content.group(1);
    // <img src="/d/file/toutiao/1603c85157af7077cc29bfa9256f68e8.jpg" width="220" height="145" alt="对游戏行业而言 垄断或是市场竞争的一种最优状态"/>
    Matcher m_title_url = Pattern.compile("<img src=\"(.*?)\" width=\"220\" height=\"145\" alt=\"(.*?)\"/>").matcher(lis);
    if(m_title_url.find()){
        System.out.println(m_title_url.group(1));
        System.out.println(m_title_url.group(2));
    }
}

10、释放链接

getmethod.releaseConnection();