一个简单的在idea(eclipse)控制台上看起点的小工具
假装在工作,其实在moyu?
一、maven依赖
<!--阿里的fastjson-->
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
</dependency>
<!--java解析html网页工具-->
<dependency>
<groupId>org.jsoup</groupId>
<artifactId>jsoup</artifactId>
<version>1.12.2</version>
</dependency>
二、工具类代码
使用Jsoup解析网页,在控制台进行交互。
import com.inspur.wfmcloud.wrapper.WrapMapper;
import com.inspur.wfmcloud.wrapper.Wrapper;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import java.io.File;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.Proxy;
import java.text.SimpleDateFormat;
import java.util.*;
/**
* <p>魔芋工具类</p>
*
* <p>项目名称:lnwlcsMicroService</p>
*
* <p>注释:摸鱼?</p>
*
* <p>Copyright: Copyright Faker(c) 2023/5/8</p>
*
* <p>公司: Faker</p>
*
* @author 淡梦如烟
* @version 1.0
* @date 2023/5/8 16:15
*/
@Slf4j
public class MoYuUtil {
@ApiModelProperty("网站根网址")
private String ROOT_URL = "http://www.baidu.com";
@ApiModelProperty("网站缓存")
private Map<String, Document> documentCache = new HashMap<String, Document>();
@ApiModelProperty("模拟请求头列表")
public final static String[] USER_AGENT = new String[]{
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)"
, "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.163 Safari/535.1"
, "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:6.0) Gecko/20100101 Firefox/6.0"
, "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50"
, "Opera/9.80 (Windows NT 6.1; U; zh-cn) Presto/2.9.168 Version/11.50"
, "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 2.0.50727; SLCC2; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.3; .NET4.0C; Tablet PC 2.0; .NET4.0E)"
, "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; GTB7.0)"
, "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; InfoPath.3)"
, "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:46.0) Gecko/20100101 Firefox/46.0",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.87 Safari/537.36 OPR/37.0.2178.32",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534.57.2 (KHTML, like Gecko) Version/5.1.7 Safari/534.57.2",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2486.0 Safari/537.36 Edge/13.10586",
"Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko",
"Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)",
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)",
"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0)",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.106 BIDUBrowser/8.3 Safari/537.36",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.80 Safari/537.36 Core/1.47.277.400 QQBrowser/9.4.7658.400",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 UBrowser/5.6.12150.8 Safari/537.36",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122 Safari/537.36 SE 2.X MetaSr 1.0",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 Safari/537.36 TheWorld 7",
"Mozilla/4.0 (compatible; MSIE 5.0; Windows NT; DigExt)",
"Mozilla/5.0 (Windows NT 6.1; W…) Gecko/20100101 Firefox/60.0"
};
@ApiModelProperty("请求协议")
private final String protocol = "https:";
@ApiModelProperty("缓存路径")
private final String cachePath = System.getProperty("java.io.tmpdir") + "cache.xlh";
@ApiModelProperty("输入对象")
private java.util.Scanner sc = new java.util.Scanner(System.in);
@ApiModelProperty("时间格式化")
public static SimpleDateFormat sdf1 = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
public static void main(String[] args) {
MoYuUtil moYuUtil = new MoYuUtil();
log.info(moYuUtil.run().toString());
}
// /**
// * 测试开始
// */
// @Test
// public void test() {
// log.info(this.run().toString());
// }
/**
* 启动
*
* @return
*/
public Wrapper run() {
this.init();
this.readQiDian();
return WrapMapper.ok();
}
/**
* 初始化
*
* @return
*/
private Wrapper init() {
log.debug("缓存路径:" + cachePath);
File cacheFile = new File(cachePath);
return WrapMapper.ok("初始化完成");
}
/**
* 总该有个开始吧
*/
private void readQiDian() {
// Document homwPage = this.httpGet("https://www.qidian.com/");
// if(homwPage != null) {
// log.info(homwPage.toString());
// }
//排行榜
Document rank = this.httpGet("https://www.qidian.com/rank/yuepiao/");
if (rank != null) {
// log.info(rank.toString());
Elements pages = rank.select("#page-container .lbf-pagination .lbf-pagination-item a");
Map<Integer, String> pageMap = new HashMap<>();
for (int i = 0; i < pages.size(); i++) {
Element page = pages.get(i);
String dataPage = page.attr("data-page");
if (null != dataPage && !"".equals(dataPage)) {
try {
int pageNumber = Integer.parseInt(dataPage);
pageMap.put(pageNumber, protocol + page.attr("href"));
} catch (NumberFormatException e) {
e.printStackTrace();
}
}
}
log.info("页码:" + pageMap.toString());
Elements books = rank.select("#rank-view-list ul li");
List<QidianBook> qidianBookList = new ArrayList<QidianBook>();
for (int i = 0; i < books.size(); i++) {
Element book = books.get(i);
QidianBook qidianBook = new QidianBook();
qidianBook.setName(book.select(".book-mid-info h2 a").first().text());
qidianBook.setUrl(protocol + book.select(".book-mid-info h2 a").first().attr("href"));
qidianBook.setImg(protocol + book.select(".book-img-box img").first().attr("src"));
qidianBook.setAuthor(book.select(".book-mid-info .author .name").first().text());
qidianBook.setIntro(book.select(".book-mid-info .intro").first().text());
qidianBookList.add(qidianBook);
}
// log.info("图书列表" + qidianBookList);
this.printQidianBookList(qidianBookList);
this.choseTodo(qidianBookList, pageMap, 1);
}
}
/**
* 选择
*/
private void choseTodo(List<QidianBook> qidianBookList, Map<Integer, String> pageMap, int index) {
log.info("请选择阅读本页(R)或者加载上一页(<)、下一页(>)?");
//接收字符串
String chose1 = this.input();
if ("R".equals(chose1) || "r".equals(chose1)) {
this.choseBookToRead(qidianBookList);
} else if (">".equals(chose1)) {
this.loadNextPage(pageMap, index + 1);
} else if ("<".equals(chose1)) {
if (index - 1 < 1) {
log.warn("已经是首页了!");
this.loadNextPage(pageMap, index);
}
this.loadNextPage(pageMap, index - 1);
} else {
this.loadNextPage(pageMap, index);
}
}
/**
* 加载下一页
*
* @param pageMap
* @param index
*/
private void loadNextPage(Map<Integer, String> pageMap, int index) {
//排行榜
Document rank = this.httpGet(pageMap.get(index));
if (rank != null) {
Elements books = rank.select("#rank-view-list ul li");
List<QidianBook> qidianBookList = new ArrayList<QidianBook>();
for (int i = 0; i < books.size(); i++) {
Element book = books.get(i);
QidianBook qidianBook = new QidianBook();
qidianBook.setName(book.select(".book-mid-info h2 a").first().text());
qidianBook.setUrl(protocol + book.select(".book-mid-info h2 a").first().attr("href"));
qidianBook.setImg(protocol + book.select(".book-img-box img").first().attr("src"));
qidianBook.setAuthor(book.select(".book-mid-info .author .name").first().text());
qidianBook.setIntro(book.select(".book-mid-info .intro").first().text());
qidianBookList.add(qidianBook);
}
this.printQidianBookList(qidianBookList);
this.choseTodo(qidianBookList, pageMap, index);
}
}
/**
* 请输入书名以阅读
*
* @param qidianBookList
*/
private void choseBookToRead(List<QidianBook> qidianBookList) {
log.info("请输入书名或序号");
String chose2 = this.input();
boolean findBook = false;
for (int i = 0; i < qidianBookList.size(); i++) {
QidianBook qidianBook = qidianBookList.get(i);
if (qidianBook.getName().equals(chose2) || String.valueOf((i + 1)).equals(chose2)) {
//todo 进入阅读书籍
this.loadBookInfo(qidianBookList, i);
findBook = true;
}
}
if (!findBook) {
if ("qqq".equals(chose2)) {
return;
}
log.warn("未找到书籍[" + chose2 + "]");
this.choseBookToRead(qidianBookList);
}
}
/**
* 输入值
*
* @return
*/
private String input() {
//接收字符串
String input = sc.nextLine();
// sc.close();
return input;
}
/**
* 格式化打印书籍列表
*
* @param qidianBookList
*/
private void printQidianBookList(List<QidianBook> qidianBookList) {
StringBuffer logs = new StringBuffer();
logs.append("\n").append("|---------------------------------[共" + qidianBookList.size() + "本]--------------------------------|").append("\n");
StringBuffer row = new StringBuffer();
row.append("|").append("书名").append("|").append("作者").append("|").append("简介").append("|").append("地址").append("|").append("\n");
logs.append(row.toString());
for (int i = 0; i < qidianBookList.size(); i++) {
QidianBook qidianBook = qidianBookList.get(i);
row.setLength(0);
row.append("|").append(i + 1).append("|").append(qidianBook.getName()).append("|").append(qidianBook.getAuthor()).append("|").append(qidianBook.getIntro()).append("|").append("\n").append("\n");
logs.append(row);
if (i + 1 == qidianBookList.size()) {
logs.append("|----------------------------------------------------------------------|").append("\n");
}
}
log.info(logs.toString());
}
/**
* 封装的get请求,连续解析3次
*
* @param url 请求地址
* @return
*/
public Document httpGet(String url) {
//缓存
Document document = documentCache.get(url);
if (document != null) {
return document;
}
try {
document = this.getDocument(url, 1000);
} catch (IOException e) {
log.info("请求[" + url + "]失败,正在第一次重试。");
}
if (document == null) {
try {
document = this.getDocument(url, 5000);
} catch (IOException e) {
log.warn("请求[" + url + "]再次失败,正在第二次重试。");
}
}
if (document == null) {
try {
Thread.sleep(1000);
document = this.getDocument(url, 20000);
} catch (Exception e) {
log.error("请求[" + url + "]失败,解析终止!!!!");
e.printStackTrace();
}
}
documentCache.put(url, document);
return document;
}
/**
* 获取网页解析对象
*
* @param url
* @param timeout
* @return
* @throws IOException
*/
private Document getDocument(String url, int timeout) throws IOException {
Map<String, String> headers = new HashMap<String, String>();
headers.put("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9");
headers.put("Accept-Language", "zh-CN,zh;q=0.9");
headers.put("Connection", "keep-alive");
headers.put("Host", ROOT_URL.replace("http://", ""));
headers.put("Referer", ROOT_URL);
headers.put("Upgrade-Insecure-Requests", "1");
Map<String, String> cookies = new HashMap<String, String>();
//代理 防止路由器屏蔽
Proxy proxy = null;
//proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress("192.168.123.223", 33335));
return Jsoup.connect(url).timeout(timeout).userAgent(getUserAgent()).proxy(proxy).cookies(cookies).headers(headers).get();
}
/**
* 随机获取一个请求头
*
* @return 请求头
*/
public static String getUserAgent() {
Random rand = new Random();
int i = rand.nextInt(USER_AGENT.length);
String userAgent = USER_AGENT[i];
return userAgent;
}
@Data
@NoArgsConstructor
public class QidianBook {
@ApiModelProperty("书名")
private String name;
@ApiModelProperty("地址")
private String url;
@ApiModelProperty("图片地址")
private String img;
@ApiModelProperty("作者")
private String author;
@ApiModelProperty("简介")
private String intro;
}
@Data
@NoArgsConstructor
public class QidianBookMulu {
@ApiModelProperty("章节名称")
private String name;
@ApiModelProperty("地址")
private String url;
@ApiModelProperty("序列")
private int index;
@ApiModelProperty("简介")
private String intro;
}
/**
* 进入阅读书籍
*
* @param qidianBookList
* @param index
*/
private void loadBookInfo(List<QidianBook> qidianBookList, int index) {
QidianBook qidianBook = qidianBookList.get(index);
Document bookDocument = this.httpGet(qidianBook.getUrl());
if (bookDocument != null) {
// log.info(bookDocument.toString());
Elements muluElements = bookDocument.select(".volume .cf li");
List<QidianBookMulu> qidianBookMuluList = new ArrayList<>();
for (int i = 0; i < muluElements.size(); i++) {
Element muluElement = muluElements.get(i);
int muluIndex = i + 1;
QidianBookMulu qidianBookMulu = new QidianBookMulu();
qidianBookMulu.setName(muluElement.select(".book_name a").first().text());
qidianBookMulu.setUrl(protocol + muluElement.select(".book_name a").first().attr("href"));
qidianBookMulu.setIndex(muluIndex);
qidianBookMulu.setIntro(null);
qidianBookMuluList.add(qidianBookMulu);
}
this.choseChapterToRead(qidianBookMuluList);
} else {
log.warn("加载书籍[" + qidianBook.getName() + "]信息失败" + qidianBook.getUrl());
this.choseBookToRead(qidianBookList);
}
}
/**
* 选择章节以阅读
*
* @param qidianBookMuluList
*/
private void choseChapterToRead(List<QidianBookMulu> qidianBookMuluList) {
StringBuffer sb = new StringBuffer();
for (int i = 0; i < qidianBookMuluList.size(); i++) {
sb.append(qidianBookMuluList.get(i).getIndex()).append("|").append(qidianBookMuluList.get(i).getName()).append("|\n");
}
log.info(sb.toString());
log.info("请选择要阅读的章节序号或名称");
String chose = this.input();
if ("qqq".equals(chose)) {
return;
}
for (int i = 0; i < qidianBookMuluList.size(); i++) {
if (getString(qidianBookMuluList.get(i).getIndex()).equals(chose)
|| qidianBookMuluList.get(i).getName().equals(chose)
) {
this.readBookChapter(qidianBookMuluList, i);
return;
}
}
this.choseChapterToRead(qidianBookMuluList);
}
/**
* 获取字符串
*
* @param object 对象
* @return
*/
public static String getString(Object object) {
if (null == object) {
return "";
} else {
if (object instanceof Date) {
return sdf1.format((Date) object);
} else {
return String.valueOf(object);
}
}
}
/**
* 阅读书籍
*
* @param qidianBookMuluList
* @param index
*/
private void readBookChapter(List<QidianBookMulu> qidianBookMuluList, int index) {
QidianBookMulu qidianBookMulu = qidianBookMuluList.get(index);
Document chapterDocument = this.httpGet(qidianBookMulu.getUrl());
if (null != chapterDocument) {
// log.info(chapterDocument.toString());
Elements p = chapterDocument.select(".relative main p");
StringBuffer text = new StringBuffer("\n");
for (int i = 0; i < p.size(); i++) {
text.append(p.get(i).text()).append("\n");
}
log.info(text.toString());
} else {
log.warn("章节内容加载失败..." + qidianBookMulu);
}
log.info("请选择要阅读的章节序号或下一页(>)");
String chose = this.input();
if ("qqq".equals(chose)) {
return;
}
if (">".equals(chose)) {
this.readBookChapter(qidianBookMuluList, index + 1);
return;
}
this.readBookChapter(qidianBookMuluList, index);
}
}
后记
启动main方法,然后就是简单快乐的moyu啦。