一、运行环境
pycharm 2020 社区版
python 3.7
bs4
二、实际代码
# http://aixibanya.com.cn/sifang/pc6/
import os
import re
import requests
from bs4 import BeautifulSoup
from access.sprider.SpriderAccess import SpriderAccess
from base.BaseConfig import BaseConfig
from base.BaseFrame import BaseFrame
from business.sprider.DownLoadFile import DownLoadFile
from business.sprider.UserAgent import UserAgent
from object.entity.SpriderEntity import SpriderEntity
from plugin.Tools import Tools
class QuLa_WanBen:
base_url = "https://www.qu.la/"
save_path = BaseConfig().CORPUS_ROOT + os.sep + "QuLa"
dict_column_list = {'tabData_3': '都市言情', 'tabData_1': '玄幻奇幻', 'tabData_2': '武侠仙侠', 'tabData_4': '历史军事',
'tabData_5': '科幻灵异', 'tabData_6': '网游竞技'}
def __init__(self):
Tools.judge_diskpath_exits_create(self.save_path)
pass
def sprider_story(self):
BaseFrame.__log__("开始采集笔趣阁排行榜小说...")
self.story_url = self.base_url + "wanbenxiaoshuo"
try:
response = requests.get(self.story_url, timeout=30, headers=UserAgent().get_random_header(self.story_url))
response.encoding = 'UTF-8'
soup = BeautifulSoup(response.text, "html5lib")
for column in self.dict_column_list:
div_tabData_1 = soup.find('div', attrs={"id": "" + column + ""})
div_list = div_tabData_1.findAll('div', attrs={"class": 'topbooks'})
for div in div_list:
# 网址集合<a href="/book/4140/" title="太古神王" target="_blank">太古神王</a>
a_list = div.find_all('a', attrs={"target": '_blank'})
for a in a_list:
content_url = self.base_url + a.get("href") # 小说的网址
txt_title = a.get("title") # 小说名称
main_id = Tools.get_guid() # 主表使用的ID
try:
response = requests.get(content_url, timeout=30,
headers=UserAgent().get_random_header(content_url))
response.encoding = 'UTF-8'
soup = BeautifulSoup(response.text, "html5lib")
# 小说的图片
image_parent_div = soup.find("div", attrs={"id": "fmimg"})
story_image_url = self.base_url + image_parent_div.find("img").get("src")
second_path = self.dict_column_list[column]
DownLoadFile(self.save_path).__down_load_image__(story_image_url, txt_title, second_path, 0)
# 小说的作者
author_parent_div = soup.find("div", attrs={"id": "info"})
author = str(author_parent_div.find("p").text).split(":")[1]
# 小说的正文
dl_tag = soup.find('dl')
a_list = dl_tag.find_all('a')
txt_index = 0 # 小说章节序号
for a_tag in a_list:
href = a_tag.get("href")
if "book" in href: # 去掉介绍和没用
url = self.base_url + href
title = a_tag.text
detail_id = Tools.get_guid()
txt_index = txt_index + 1
mmEntity = SpriderEntity()
mmEntity.sprider_base_url = self.base_url
mmEntity.create_datetime = Tools.get_current_datetime()
mmEntity.sprider_url = url
mmEntity.sprider_pic_title = title
mmEntity.sprider_pic_index = str(1)
if SpriderAccess().query_sprider_entity_by_urlandtitle(url, title) is None:
SpriderAccess().save_sprider(mmEntity)
image_relative_path = "QuLa" + os.sep + str(
self.dict_column_list[
column]) + os.sep + txt_title + os.sep + txt_title + ".jpg"
self.get_content(url, title, txt_title, self.dict_column_list[column],
image_relative_path,
author, txt_index, main_id, detail_id)
except Exception as e:
BaseFrame.__err__("采集" + content_url + "出现错误" + str(e))
pass
pass
except Exception as e:
BaseFrame.__err__("采集出现错误" + str(e))
pass
def get_content(self, url, chapter, txt_title, txt_coloum, image_path, author, txt_index, main_id, detail_id):
"""
写文件至文本中
:param url:采集的URL
:param chapter: 章节名称
:param txt_title: 小说名称
:return:
"""
txt_path = self.save_path + os.sep + txt_coloum + os.sep + str(txt_title)
Tools.judge_diskpath_exits_create(txt_path)
try:
BaseFrame.__log__("正在采集 " + chapter + " " + url + " 上的小说...")
response = requests.get(url, timeout=60, headers=UserAgent().get_random_header(url))
response.encoding = 'UTF-8'
soup = BeautifulSoup(response.text, "html5lib")
content = soup.find('div', attrs={"id": 'content'})
new_content = chapter + "\n" + str(content). \
replace('<br/>', '\n'). \
replace("<script>chaptererror();</script>", ""). \
replace("<div id=\"content\">", ""). \
replace("</div>", "")
rightName = re.sub('[\/:*?"<>|]', '', str(chapter))
chapter_full_path = txt_path + os.sep + rightName + ".txt"
Tools.write_string_to_txt(chapter_full_path, new_content)
execl_path = self.save_path + os.sep + "StoryTemplate.txt"
chapter_path = "QuLa" + os.sep + txt_coloum + os.sep + txt_title + os.sep + rightName + ".txt"
story_data = [txt_title, author, image_path, chapter, chapter_path, txt_index, txt_coloum, main_id,
detail_id]
self.write_execl(execl_path, story_data)
except Exception as e:
BaseFrame.__err__("采集" + chapter + "出现错误" + str(e) + "尝试重新采集.")
self.get_content(url, chapter, txt_title, txt_coloum, image_path, author, txt_index, main_id, detail_id)
pass
# region 写Execl
def write_execl(self, execl_path, story_data):
split_sign = "@"
content = str(story_data[0]) + split_sign + str(story_data[1]) + split_sign + str(story_data[2]) + split_sign + \
story_data[
3] + split_sign + str(story_data[4]) + split_sign + str(story_data[5]) + split_sign + str(
story_data[6]) + split_sign + story_data[7] + split_sign + story_data[8] + "\n"
Tools.write_string_to_txt(execl_path, content)
# endregion
# if __name__ == '__main__':
# QuLa_WanBen().sprider_story()
# pass