java抓取糗事百科内容

最新推荐文章于 2021-05-03 20:23:53 发布

心有不甘

最新推荐文章于 2021-05-03 20:23:53 发布

阅读量728

点赞数 1

分类专栏： java 文章标签： java url

本文链接：https://blog.csdn.net/sqq920556218/article/details/46675511

版权

java 专栏收录该内容

4 篇文章 0 订阅

订阅专栏

解决思路

通过URLConnection对象获取糗百网的响应html
解析html通过正则取出糗事内容
利用FileWrite对象输出到文件

具体步骤

新建QiuShiBaiKe类，创建一个发送get请求的方法，参数为url和params，返回响应html.

将url和发送参数params拼接完整
String urlName = url + "?" + param;
由完整路径构建URL对象
URL realUrl = new URL(urlName);
通过URL对象的openConnection()方法获得URLConnection对象连接
URLConnection con = realUrl.openConnection();
设置request头部参数，可从浏览器net中看到
con.setRequestProperty("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8"); con.setRequestProperty("Accept-Language", "zh-CN,zh;q=0.8"); con.setRequestProperty("Connection", "keep-alive"); con.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.124 Safari/537.36");
连接发送请求
con.connect();
通过URLConnection对象的getInputStream()方法获取响应网络流，并包装成BufferedReader带有缓冲区的字符流对象
in = new BufferedReader(new InputStreamReader(con.getInputStream()));
循环读取流拼接到result字符串中，返回result
String line; while ((line = in.readLine()) != null) { result += "\n"+ line; } return result;
糗百的地址的规律
http://www.qiushibaike.com/text/page/后拼接页码，我们可以通过循环获取前10页的内容
String url = "http://www.qiushibaike.com/text/page/"; String html = ""; for (int i = 1; i <= 10; i++) { html += qiuShiBaiKe.sendGet(url+i, null); }
糗事内容的规则
通过分析页面发现内容是被包含在<div class="content"></div>中，可通过正则抓取
Matcher jokeMatcher = Pattern.compile("<div class=\"content\">[^/]*</div>?").matcher(html);
使用Matcher对象的find()方法作为循环条件输出捕获到的内容group()
通过FileWrite的write方法写入到文件
FileWriter fw = new FileWriter("hello.txt"); while (jokeMatcher.find()) { String one = jokeMatcher.group(); String joke = one.replace("<div class=\"content\">", "").replace("</div>", ""); fw.write(joke+"\n"); }

效果如下图

这里写图片描述

放马过来

package com.sqq.Internate17.demo;

import java.io.BufferedReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * 获取糗事百科段子内容
 */
public class QiuShiBaiKe {

    public String sendGet(String url, String param) {
        String result = "";
        BufferedReader in = null;
        try {
            String urlName = url + "?" + param;
            URL realUrl = new URL(urlName);
            URLConnection con = realUrl.openConnection();

            con.setRequestProperty("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
            con.setRequestProperty("Accept-Language", "zh-CN,zh;q=0.8");
            con.setRequestProperty("Connection", "keep-alive");
            con.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.124 Safari/537.36");

            con.connect();

            in = new BufferedReader(new InputStreamReader(con.getInputStream()));

            String line;
            while ((line = in.readLine()) != null) {
                result += "\n" + line;
            }
        } catch (MalformedURLException e) {
            e.printStackTrace();
        } catch (IOException e) {
            e.printStackTrace();
        }
        return result;
    }

    public static void main(String[] args) throws IOException {
        QiuShiBaiKe qiuShiBaiKe = new QiuShiBaiKe();
        //糗事百科地址
        String url = "http://www.qiushibaike.com/text/page/";
        String html = "";
        for (int i = 1; i <= 10; i++) {
            html += qiuShiBaiKe.sendGet(url + i, null);
        }
        System.out.println(html);
        FileWriter fw = new FileWriter("hello.txt");
        Matcher jokeMatcher = Pattern.compile("<div class=\"content\">[^/]*</div>?").matcher(html);
        fw.write("************** 把快乐建立在别人的痛苦上：糗事百科 *************\n");
        while (jokeMatcher.find()) {
            String one = jokeMatcher.group();
            String joke = one.replace("<div class=\"content\">", "").replace("</div>", "");
            fw.write(joke + "\n");
            fw.write("========== 华丽的分割线 ==========\n");
        }
    }
}