爬虫基础一

httpClient 第一个爬虫测试

/**
 * 第一个爬虫测试
 * Created by DuFei on 2017/7/27.
 * 来源 https://www.datalearner.com/blog/1051501160659926
 */
public class FirstTest {
    public static void main(String[] args) {
        //建立一个新的请求客户端
        CloseableHttpClient httpClient = HttpClients.createDefault();
        //使用HttpGet方式请求网址
        //请求百度首页会有乱码问题http://www.baidu.com"

        HttpGet httpGet = new HttpGet("http://www.datalearner.com/blog");
        //获取网址的返回结果
        CloseableHttpResponse response = null;
        try {
            response = httpClient.execute(httpGet);
        } catch (IOException e) {
            e.printStackTrace();
        }
        //获取返回结果中的实体
        HttpEntity entity = response.getEntity();
        //将返回的实体输出
        try {
            System.out.println(EntityUtils.toString(entity));
            EntityUtils.consume(entity);
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
}

注意的是，爬虫一般都是模拟浏览器的行为，大都get请求

HttpClient详细使用方法

https://www.datalearner.com/blog/1051505270990649
在实际中，有很多网页的请求需要附带许多参数设置。主要包括请求的Header设置以及路径参数。在HttpClient 4.3及以上的版本中，这个过程主要包含如下步骤：

使用List<NameValuePair>添加路径参数（请求参数）
使用URI对请求路径及其参数进行设置
使用List<Header>设置请求的头部
初始化自定义的HttpClient客户端，并设置头部
使用HttpUriRequest设置请求
使用HttpClient请求上述步骤中的HttpUriRequest对象

我们看一个代码示例

import com.google.common.collect.Lists;
import org.apache.http.Header;
import org.apache.http.HttpHeaders;
import org.apache.http.HttpResponse;
import org.apache.http.NameValuePair;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.methods.RequestBuilder;
import org.apache.http.client.utils.URIBuilder;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.message.BasicHeader;
import org.apache.http.message.BasicNameValuePair;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.List;
/************
 * HttpClient 使用示例
 * ***********/
public class HttpClientTest {
  public static void main(String[] args) throws URISyntaxException, IOException {
    String url = "";    //请求路径
    //构造路径参数
    List<NameValuePair> nameValuePairList = Lists.newArrayList();
    nameValuePairList.add(new BasicNameValuePair("username","test"));
    nameValuePairList.add(new BasicNameValuePair("password","password"));
    //构造请求路径，并添加参数
    URI uri = new URIBuilder(url).addParameters(nameValuePairList).build();
    //构造Headers
    List<Header> headerList = Lists.newArrayList();
    headerList.add(new BasicHeader(HttpHeaders.ACCEPT_ENCODING,"gzip, deflate"));
    headerList.add(new BasicHeader(HttpHeaders.CONNECTION, "keep-alive"));
    //构造HttpClient
    HttpClient httpClient = HttpClients.custom().setDefaultHeaders(headerList).build();
    //构造HttpGet请求
    HttpUriRequest httpUriRequest = RequestBuilder.get().setUri(uri).build();
    //获取结果
    HttpResponse httpResponse = httpClient.execute(httpUriRequest);
    //获取返回结果中的实体
    HttpEntity entity = httpResponse.getEntity();
    //查看页面内容结果
    String rawHTMLContent = EntityUtils.toString(entity);
    System.out.println(rawHTMLContent);
    //关闭HttpEntity流
    EntityUtils.consume(entity);
  }
}