第1关:获取新闻url
任务描述
本关任务:编写一个爬虫,并使用正则表达式获取求是周刊 2019
年第一期的所有文章的 url
。详情请查看 《求是》2019年第1期 。
import requests
import re
def geturls():
# ********** Begin ********** #
url = "http://www.qstheory.cn/dukan/qs/2014/2019-01/01/c_1123924172.htm"
#请求头
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/73.0.3683.75 Safari/537.36",
'accept-language': 'zh-CN,zh;q=0.9',
'cache-control': 'max-age=0',
'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8'
}
response = requests.get(url,headers=headers)
result = response.content.decode("utf8")
urls = re.findall('<a\shref="(.+?)"\starget="_[a-z]+?"><strong',result)
# ********** End ********** #
return urls
if __name__ == "__main__":
urls = geturls()
for x in urls:
print(x)
第2关:获取文章内容
任务描述
本关任务:编写一个爬虫,请求上一关获取的每个 url
,获取每篇文章的标题、作者、正文以及文章中全部图片的完整 url
。
import requests
from lxml import etree
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36"
}
def parsepage(urls):
mainbody = [] # 保存新闻内容
# ********** Begin ********** #
for url in urls:
response = requests.get(url,headers=headers)
result = response.content.decode("utf8")
html = etree.HTML(result)
img = html.xpath('//div[@class="highlight"]//p//img//@scr')
# 获取图片的url
imgurl = []
split_url = url.split("/")
split_url.pop(-1)
if len(img) >= 1:
for x in img:
imgurl.append("/".join(split_url)+"/"+x)
else:
imgurl=""
# 获取标题并去除首尾空格
title = "".join(html.xpath('//div[@class="row"]//div[@class="inner"]//h1/text()')).strip()
# 获取作者并去除首尾空格
author = (html.xpath('//div[@class="row"]//span[@class="appellation"]//text()')[-1].split(":")[-1]).strip()
# 获取正文并去除首尾空格
cont = html.xpath('//div[@class="content"]//div[@class="highlight"]/p//*[not(@color="navy")]/text()|//div[@class="highlight"]/p/text()')
content = "".join(cont).strip()
mainbody.append({"title":title,"author":author,"content":content,"imgurl":imgurl})
# ********** End ********** #
return mainbody