python爬取晋江小说和间接_【python】晋江原耽积分榜前5000文的标签统计

import requests

from lxml import etree, html

from bs4 import BeautifulSoup

import json

# 根据排行榜每页的url规律，构造出排行榜页面的url：

def get_url_1(page):

head = 'http://www.jjwxc.net/bookbase.php?fw0=0&fbsj=0&ycx1=1&xx2=2&mainview0=0&sd0=0&lx0=0&fg0=0&sortType=2&isfinish=0&collectiontypes=ors&searchkeywords=&page='

# tail = '&isfinish=0&collectiontypes=ors&searchkeywords='

var = str(page)

# url = head + var + tail

url = head + var

return url

# 构造每篇文章的url：

def get_url_2(number):

head = 'http://www.jjwxc.net/'

var = str(number)

url = head + var

return url

# 根据排行榜url，获取各文章的链接：

def get_info_1(url_1):

# 伪装成浏览器，防止封ip

headers = {

'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.83 Safari/537.36',

'Host': 'www.jjwxc.net',

'Cookie': '__gads=ID=adb1bd199ca80e42:T=1592032062:S=ALNI_MbfwuQah_VUIJ0eFciwrmcI0YVBcQ; CNZZDATA1275156903=1619785691-1592040877-null%7C1592040877; __cfduid=d5561249470bba6af47bba14f331c644e1592045059; UM_distinctid=172ad47ba1a345-0e33c371452b35-f7d123e-144000-172ad47ba1de; timeOffset_o=-395.800048828125; CNZZDATA30075907=cnzz_eid%3D149420612-1592030978-null%26ntime%3D1594286393; testcookie=yes; Hm_lvt_bc3b748c21fe5cf393d26c12b2c38d99=1592471583,1594093184,1594178729,1594290569; token=Mjk5MDY4Mjd8ZWE4Y2Q1ZTc5OTIzZjNjNDgxMmNmZjU5NDI1MGEyMzl8fHx8MTA4MDB8MXx8fOaZi%2Baxn%2BeUqOaIt3wxfG1vYmlsZQ%3D%3D; JJSESS=%7B%22clicktype%22%3A%22%22%2C%22nicknameAndsign%22%3A%222%257E%2529%2524%25E6%25B4%25BB%25E7%259D%2580%25E4%25B8%25BA%25E4%25BA%2586%25E4%25BB%2580%25E4%25B9%2588%22%7D; JJEVER=%7B%22sms_total%22%3A%220%22%2C%22fenzhan%22%3A%22noyq%22%2C%22ispayuser%22%3A%2229906827-1%22%2C%22foreverreader%22%3A%2229906827%22%2C%22user_signin_days%22%3A%2220200709_29906827_3%22%7D; Hm_lpvt_bc3b748c21fe5cf393d26c12b2c38d99=1594291386',

}

tries = 30

rsp = ""

while tries > 0:

try:

rsp = requests.get(url_1, headers=headers)

break

except Exception as e:

tries -= 1

# 防止中文乱码

rsp.encoding = rsp.apparent_encoding

data = rsp.text

print(data)

selector = etree.HTML(data)

# 获取一页所有文章的超链接

url_2 = selector.xpath('//html//body//table//tbody//tr//td[2]//a/@href')

print(url_2)

return url_2

# 获取文章页面的标签：

def get_info_2(url_2):

# 伪装成浏览器，防止封ip

headers = {

'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.83 Safari/537.36',

'Host': 'www.jjwxc.net',

'Cookie': '__gads=ID=adb1bd199ca80e42:T=1592032062:S=ALNI_MbfwuQah_VUIJ0eFciwrmcI0YVBcQ; CNZZDATA1275156903=1619785691-1592040877-null%7C1594437918; __cfduid=df5911758d582c90c410614cae0204c661595644212; UM_distinctid=17383ce87ff12-0b5ff82dc4d561-b7a1334-144000-17383ce88005aa; CNZZDATA30075907=cnzz_eid%3D149420612-1592030978-null%26ntime%3D1598006596; testcookie=yes; timeOffset_o=3822.39990234375; Hm_lvt_bc3b748c21fe5cf393d26c12b2c38d99=1597378421,1597635899,1597657203,1598007103; token=Mjk5MDY4Mjd8Y2NiYzJlMTE1OTU0NzI3NGRjZjNhNGM4YTNlMTZhNmV8fHx8MTA4MDB8MXx8fOaZi%2Baxn%2BeUqOaIt3wxfGpqcmVhZGVy; JJSESS=%7B%22sidkey%22%3A%22pmZ5hDLY8uXBPKeOAGCRIgfsF1vHcMNUEQyo23xq%22%2C%22nicknameAndsign%22%3A%222%257E%2529%2524%25E6%25B4%25BB%25E7%259D%2580%25E4%25B8%25BA%25E4%25BA%2586%25E4%25BB%2580%25E4%25B9%2588%22%7D; JJEVER=%7B%22ispayuser%22%3A%2229906827-1%22%2C%22foreverreader%22%3A%2229906827%22%2C%22user_signin_days%22%3A%2220200821_29906827_2%22%2C%22sms_total%22%3A0%2C%22fenzhan%22%3A%22noyq%22%7D; Hm_lpvt_bc3b748c21fe5cf393d26c12b2c38d99=1598007308',

}

tries = 30

rsp = ""

while tries > 0:

try:

rsp = requests.get(url_2, headers=headers)

break

except Exception as e:

tries -= 1

soup = BeautifulSoup(rsp.content, 'lxml')

# 获取标签

head_tot = soup.find_all('a', target="_blank", style="text-decoration:none;color: red;")

# 排除文案修改中情况

if head_tot == []:

return []

head_text = []

# 获取标签文本

for i in head_tot:

head_text.append(i.get_text())

return head_text

# 字典保存为txt文件：

def save_dict(dict, which):

name_str = which + '.txt'

js = json.dumps(dict)

file = open(name_str, 'w')

file.write(js)

file.close()

def save_it(label_list, min_page, max_page):

# 统计成字典形式

word_num = {}

for i in range(len(label_list)):

word_num[label_list[i]] = label_list.count(label_list[i])

# 按value值排序

word_num = sorted(word_num.items(), key=lambda x: x[1], reverse=True)

print(word_num)

# 保存为txt文件

its_label = "标签的统计" + " " + str(min_page) + "--" + str(max_page-1)

save_dict(word_num, its_label)

return

def main():

# 获取标签列表：

label = []

min = 50

max = 51

for page in range(min, max): # 每页

print("第", page, "页")

link_1 = get_url_1(page)

link_2_half = get_info_1(link_1)

label_page = []

for num in range(0, len(link_2_half)): # 每篇文章

print("文章位：", page, ".", num)

link_2 = get_url_2(link_2_half[num])

label_sig = get_info_2(link_2)

label_page += label_sig

print(label_page)

if label_page == []:

return

label += label_page

# 去除空元素：

label = [x.strip() for x in label if x.strip() != '']

# 统计成字典，并保存：

save_it(label, min, max)

return

if __name__ == '__main__':

main()