Java网络爬虫

<dependencies>
        <dependency>
            <groupId>org.jsoup</groupId>
            <artifactId>jsoup</artifactId>
            <version>1.10.3</version>
        </dependency>
        <dependency>
            <groupId>org.apache.httpcomponents</groupId>
            <artifactId>httpcore</artifactId>
            <version>4.4.10</version>
        </dependency>
        <dependency>
            <groupId>org.apache.httpcomponents</groupId>
            <artifactId>httpclient</artifactId>
            <version>4.5.6</version>
        </dependency>
        <dependency>
            <groupId>commons-io</groupId>
            <artifactId>commons-io</artifactId>
            <version>2.6</version>
        </dependency>
        <dependency>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-log4j12</artifactId>
            <version>1.7.25</version>
        </dependency>
    </dependencies>

package com.regex;

import org.apache.commons.io.FileUtils;
import org.apache.http.HttpEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.jsoup.*;
import org.jsoup.nodes.*;
import org.jsoup.select.Elements;

import java.io.File;
import java.io.IOException;
import java.io.InputStream;

public class Demo {
    public static void main(String[] args) throws IOException {
        // 1.创建HttpClient对象
        CloseableHttpClient httpClient = HttpClients.createDefault();
        // 2.创建HttpGet对象
        HttpGet httpGet = new HttpGet("https://ibaotu.com/shipin/");
        // 3.使用HttpClient发起请求，获取response
        CloseableHttpResponse httpResponse = httpClient.execute(httpGet);
        // 4.获取响应实体
        HttpEntity httpEntity = httpResponse.getEntity();
        // 5.获取响应字符串
        String content = EntityUtils.toString(httpEntity, "utf-8");
        // 6.解析HTML
        Document document = Jsoup.parse(content);
        // 7.选择爬取的标签内容
        Elements elements = document.select("div.media-list div.video-play video");
        for(int i = 0; i < elements.size(); i++) {
            Element element = elements.get(i);
            // 8.获取爬取内容的src
            String attr = element.attr("src");
            // 9.拼接HTTPS
            CloseableHttpResponse httpResponse1 = httpClient.execute(new HttpGet("https:" + attr));
            // 10.获取响应实体
            HttpEntity httpEntity1 = httpResponse1.getEntity();
            // 11.读取爬取的内容
            InputStream stream = httpEntity1.getContent();
            // 12.设置文件保存位置
            FileUtils.copyToFile(stream, new File("E:/video/" + i + ".mp4"));
            // 13.释放资源
            stream.close();
        }
        //System.out.println(attr);
    }
}

package com.regex;

import org.apache.http.HttpEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;

import java.io.*;

public class TextDemo {
    public static void main(String[] args) throws IOException {
        // 创建HttpClient对象
        CloseableHttpClient httpClient = HttpClients.createDefault();
        // 创建HttpGet对象
        HttpGet httpGet = new HttpGet("https://www.chinanews.com/gn/2021/06-10/9496792.shtml");
        // 使用HttpClient发起请求，获取response
        CloseableHttpResponse httpResponse = httpClient.execute(httpGet);
        if (httpResponse.getStatusLine().getStatusCode() == 200) {
            // 获取响应实体
            HttpEntity httpEntity = httpResponse.getEntity();
            // 获取响应字符串
            String content = EntityUtils.toString(httpEntity, "utf-8");
            // 解析HTML
            Document document = Jsoup.parse(content);
            // 获取头部文本
            // System.out.println(document.head().text());
            // 获取html
            // System.out.println(document.body().html());

            // 选择爬取的标签内容()
            Elements elements = document.select("div.left_zw p");
            // 获取文本
            String text = elements.text();
            System.out.println(text);
            // 指定文本保存路径
            FileWriter fw = new FileWriter("C:\\Users\\ASUS\\Desktop\\news.txt");
            // 写入文本
            fw.write(text);
            fw.flush();
            System.out.println("下载成功！");
        }
        httpResponse.close();
        httpClient.close();
    }
}