Java爬虫（一）

最新推荐文章于 2024-07-03 19:13:37 发布

qq_40860185

最新推荐文章于 2024-07-03 19:13:37 发布

阅读量84

点赞数

分类专栏： Java爬虫文章标签： java 爬虫

本文链接：https://blog.csdn.net/qq_40860185/article/details/118461971

版权

Java爬虫专栏收录该内容

3 篇文章 1 订阅

订阅专栏

JAVA爬虫学习笔记

环境准备

JDK1.8
IntelliJ IDEA
IDEA自带的Maven

一.导入pom.xml

<dependencies>
        <!-- https://mvnrepository.com/artifact/org.apache.httpcomponents/httpclient -->
        <!-- HttpClient -->
        <dependency>
            <groupId>org.apache.httpcomponents</groupId>
            <artifactId>httpclient</artifactId>
            <version>4.5.12</version>
        </dependency>
        <!-- https://mvnrepository.com/artifact/org.slf4j/slf4j-log4j12 -->
        <!--日志-->
        <dependency>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-log4j12</artifactId>
            <version>1.7.25</version>
            <scope>test</scope>
        </dependency>

    </dependencies>

二、创建slf4j日志配置文件

log4j.rootLogger=DEBUG,A1
log4j.logger.cn.itcast=DEBUG

log4j.appender.A1=org.apache.log4j.ConsoleAppender
log4j.appender.A1.layout=org.apache.log4j.PatternLayout
log4j.appender.A1.layout.ConversionPattern=%-d{yyyy-MM-dd HH:mm:ss,SSS} [%t] [%c]-[%p] %m%n

三、第一个爬虫

import org.apache.http.HttpEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;

import java.io.IOException;
import java.sql.SQLOutput;
//爬虫：采集，处理，存储
public class CrawlerFirst {
    public static void main(String[] args) throws IOException {
        //1.打开浏览器，创建HttpClient对象
        CloseableHttpClient httpClient= HttpClients.createDefault();

        //2.输入网址
        HttpGet httpGet=new HttpGet("http://www.itcast.cn");

        //3.按回车，发起请求，返回响应,使用httpClient对象发起请求
        CloseableHttpResponse response=httpClient.execute(httpGet);

        //4.解析响应，获取数据
        //1.判断状态码：200
        if(response.getStatusLine().getStatusCode()==200){
            HttpEntity httpEntity=response.getEntity();
            String content= EntityUtils.toString(httpEntity,"utf8");
            System.out.println(content);
        }

    }
}

四、HttpClient (用来抓取网页数据）

GET请求

import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;

import java.io.IOException;

public class HttpGetTest {
    public static void main(String[] args) {
        //1.创建HttpClient对象
        CloseableHttpClient httpClient= HttpClients.createDefault();
        //2.创建HttpGet对象，设置url访问地址
        HttpGet httpGet=new HttpGet("http://www.itcast.cn");

        CloseableHttpResponse response=null;
        try {
            //使用HttpClient发起请求，获取response
           response=httpClient.execute(httpGet);
            //解析响应
            if(response.getStatusLine().getStatusCode()==200){
               String content= EntityUtils.toString(response.getEntity(),"utf8");
                System.out.println(content.length());
            }
        } catch (IOException e) {
            e.printStackTrace();
        }finally {
            try{
                response.close();
            }catch (IOException e){
                e.printStackTrace();
            }
            try{
                httpClient.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }

    }
}

GET请求带参数


import org.apache.http.NameValuePair;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.utils.URIBuilder;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

public class HttpPostParmTest {
    public static void main(String[] args) throws Exception {
        //1.创建HttpClient对象
        CloseableHttpClient httpClient= HttpClients.createDefault();

        //2.创建HttpPost对象，设置url访问地址
        HttpPost httpPost=new HttpPost("http://yun.itheima.com/search");

        //声明list集合，封装表单中的参数
        List<NameValuePair> params=new ArrayList<NameValuePair>();

        params.add(new BasicNameValuePair("keys","Java"));
        //设置请求地址：http://yun.iteima.com/search?keys=Java

        //创建表单的Entity对象
        UrlEncodedFormEntity formEntity=new UrlEncodedFormEntity(params,"utf8");

       //设置表单的Entity对象到Post请求中
        httpPost.setEntity(formEntity);
        CloseableHttpResponse response=null;
        try {
            //使用HttpClient发起请求，获取response
           response=httpClient.execute(httpPost);
            //解析响应
            if(response.getStatusLine().getStatusCode()==200){
               String content= EntityUtils.toString(response.getEntity(),"utf8");
                System.out.println(content.length());
            }
        } catch (IOException e) {
            e.printStackTrace();
        }finally {
            try{
                response.close();
            }catch (IOException e){
                e.printStackTrace();
            }
            try{
                httpClient.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }

    }
}

POST请求


import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;

import java.io.IOException;

public class HttpPostTest {
    public static void main(String[] args) {
        //1.创建HttpClient对象
        CloseableHttpClient httpClient= HttpClients.createDefault();
        //2.创建HttpPost对象，设置url访问地址
        HttpPost httppost=new HttpPost("http://www.itcast.cn");

        CloseableHttpResponse response=null;
        try {
            //使用HttpClient发起请求，获取response
           response=httpClient.execute(httppost);
            //解析响应
            if(response.getStatusLine().getStatusCode()==200){
               String content= EntityUtils.toString(response.getEntity(),"utf8");
                System.out.println(content.length());
            }
        } catch (IOException e) {
            e.printStackTrace();
        }finally {
            try{
                response.close();
            }catch (IOException e){
                e.printStackTrace();
            }
            try{
                httpClient.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }

    }
}

POST请求带参数


import org.apache.http.NameValuePair;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.utils.URIBuilder;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

public class HttpPostParmTest {
    public static void main(String[] args) throws Exception {
        //1.创建HttpClient对象
        CloseableHttpClient httpClient= HttpClients.createDefault();

        //2.创建HttpPost对象，设置url访问地址
        HttpPost httpPost=new HttpPost("http://yun.itheima.com/search");

        //声明list集合，封装表单中的参数
        List<NameValuePair> params=new ArrayList<NameValuePair>();

        params.add(new BasicNameValuePair("keys","Java"));
        //设置请求地址：http://yun.iteima.com/search?keys=Java

        //创建表单的Entity对象
        UrlEncodedFormEntity formEntity=new UrlEncodedFormEntity(params,"utf8");

       //设置表单的Entity对象到Post请求中
        httpPost.setEntity(formEntity);
        CloseableHttpResponse response=null;
        try {
            //使用HttpClient发起请求，获取response
           response=httpClient.execute(httpPost);
            //解析响应
            if(response.getStatusLine().getStatusCode()==200){
               String content= EntityUtils.toString(response.getEntity(),"utf8");
                System.out.println(content.length());
            }
        } catch (IOException e) {
            e.printStackTrace();
        }finally {
            try{
                response.close();
            }catch (IOException e){
                e.printStackTrace();
            }
            try{
                httpClient.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }

    }
}

连接池

import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;

import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;

import java.io.IOException;


public class HttpConfigTest {
    public static void main(String[] args) {
        //1.创建HttpClient对象
        CloseableHttpClient httpClient= HttpClients.createDefault();
        //2.创建HttpGet对象，设置url访问地址
        HttpGet httpGet=new HttpGet("http://www.itcast.cn");

        //配置请求信息
        RequestConfig config=RequestConfig.custom().setConnectTimeout(1000)//连接最长时间,单位为毫秒
                .setConnectionRequestTimeout(500)//设置获取连接最长时间
                .setSocketTimeout(10*1000)       //设置数据传输的最长时间
                .build();

        //给请求设置请求信息
        httpGet.setConfig(config);

        CloseableHttpResponse response=null;
        try {
            //使用HttpClient发起请求，获取response
           response=httpClient.execute(httpGet);
            //解析响应
            if(response.getStatusLine().getStatusCode()==200){
               String content= EntityUtils.toString(response.getEntity(),"utf8");
                System.out.println(content.length());
            }
        } catch (IOException e) {
            e.printStackTrace();
        }finally {
            try{
                response.close();
            }catch (IOException e){
                e.printStackTrace();
            }
            try{
                httpClient.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    }

    

视频来源：https://www.bilibili.com/video/BV1cE411u7RA?p=44&spm_id_from=pageDriver

qq_40860185

关注

0
点赞
踩
0

收藏

觉得还不错? 一键收藏
0
评论
Java爬虫（一）

JAVA爬虫学习笔记环境准备JDK1.8IntelliJ IDEAIDEA自带的Maven一.导入pom.xm<dependencies>   <dependency> <g
复制链接

扫一扫