import requests
from lxml import etree
import csv
onePage_contents_list = []
for i in range (1,2):
page = str(i)
url = "http://www.ccwin.cn/focus/jinji/index.php?page=" + page
headers = {
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.190 Safari/537.36",
"Referer":"http: // www.ccwin.cn / focus / jinji / index.php?page = 1",
"cookies":"PHPSESSID=b9o8gat640dipglci1mavoqfc1; axSp_2132_saltkey=f0dcgVWv; axSp_2132_lastvisit=1615288435; axSp_2132_sid=WO9Mf9; axSp_2132_sendmail=1; axSp_2132_lastact=1615292436%09index.php%09list"
}
r = requests.get(url, headers=headers)
if r.status_code == 200:
html = r.text
dom = etree.HTML(html)
titles = dom.xpath('//a[@class="nex_arttitles xi2"]/text()')
titles_href = dom.xpath('//a[@class="nex_arttitles xi2"]/@href')
for i in range(15):
content_url = titles_href[i]
content_r = requests.get(content_url,headers = headers)
if r.status_code == 200:
content_html = content_r.text
content_dom = etree.HTML(content_html)
content_list = content_dom.xpath('//p[@class="p"]//text()')
contents_str = "".join(content_list)
onePage_contents_list.append(contents_str)
authors = dom.xpath('//div[@class="nex_atctl"]/span/text()')
times = dom.xpath('//div[@class="nex_atmoresd"]/span[@class="nex_fabushijian"]/text()')
read_numbers = dom.xpath('//div[@class="nex_atmoresd"]/span[@class="nex_yuedushu"]/text()')
with open("D:\practice\Xinwen.csv", mode="a", encoding="utf-8") as file:
film_list = csv.writer(file)
for i in range(15):
title = titles[i]
# content =contents[i]
content_str = onePage_contents_list[i]
author = authors[i]
time = times[i]
read_number = read_numbers[i]
film_list.writerow([title,author,time,read_number,content_str])
任务:http://www.ccwin.cn/focus/jinji/,获取到标题,内容,作者,浏览量,发布时间,而内容是获取的是点进去的文章内容。所以这时候涉及到子网页的获取。
可以通过第一次requests请求,获得他的文章链接,之后在请求一次,可以获取到文章内容。
注:
和豆瓣一样不是通过ajax来传送数据,所以通过观察网址,寻找出url的规律,改变page参数就可以爬取每一页的数据并保存。