java使用HttpClient爬取网页

添加依赖

        <dependency>
            <groupId>org.apache.httpcomponents</groupId>
            <artifactId>httpclient</artifactId>
        </dependency>

代码

import com.sun.deploy.net.HttpResponse;
import org.apache.http.HttpEntity;
import org.apache.http.HttpStatus;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.util.EntityUtils;
import sun.net.www.http.HttpClient;

import javax.swing.text.html.parser.Entity;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;

public class GetHduHome {
    public static void main(String[] args) throws IOException {
        // 创建客户端，相当于打开浏览器
        CloseableHttpClient httpClient = HttpClientBuilder.create().build();
        // 创建get方法类似输入地址
        HttpGet httpGet = new HttpGet("http://baidu.com/");
        // 发送请求并获取相应对象
        CloseableHttpResponse response = httpClient.execute(httpGet);
        // 获取相应状态码
        int statusCode = response.getStatusLine().getStatusCode();
//        // 如果需要存储到文件中
//        String fileName = "C:\\hduhomepage.html";
//        OutputStream fout = null;
        if(statusCode == HttpStatus.SC_OK){
            // 获取消息实体，内容是http传送的报文
            HttpEntity entity = response.getEntity();
            // 查看网页内容
            System.out.println(EntityUtils.toString(entity,"UTF-8"));
//            // 获取页面数据然后存放到文件
//            byte[] pageContent = EntityUtils.toByteArray(entity);
//            fout = new FileOutputStream(fileName);
//            fout.write(pageContent);
        }
//        // 释放资源
//        if(fout !=null)
//            fout.close();
        httpGet.abort();
    }
}