爬虫实战——求是网周刊文章爬取

第1关：获取新闻url

任务描述

本关任务：编写一个爬虫，并使用正则表达式获取求是周刊 2019 年第一期的所有文章的 url 。详情请查看 《求是》2019年第1期 。

import requests
import re

def geturls():
    # ********** Begin ********** #
    url = "http://www.qstheory.cn/dukan/qs/2014/2019-01/01/c_1123924172.htm"
    #请求头
    headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
                  "(KHTML, like Gecko) Chrome/73.0.3683.75 Safari/537.36",
    'accept-language': 'zh-CN,zh;q=0.9',
    'cache-control': 'max-age=0',
    'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8'
    }

    response = requests.get(url,headers=headers)

    result = response.content.decode("utf8")

    urls = re.findall('<a\shref="(.+?)"\starget="_[a-z]+?"><strong',result)

    

    # ********** End ********** #

    return urls


if __name__ == "__main__":
    urls = geturls()
    for x in urls:
        print(x)

第2关：获取文章内容

任务描述

本关任务：编写一个爬虫，请求上一关获取的每个 url ，获取每篇文章的标题、作者、正文以及文章中全部图片的完整 url 。

import requests
from lxml import etree


headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36"
}



def parsepage(urls):
    mainbody = []   # 保存新闻内容
    # ********** Begin ********** #
    for url in urls:
        response = requests.get(url,headers=headers)
        result = response.content.decode("utf8")
        html = etree.HTML(result)
        img = html.xpath('//div[@class="highlight"]//p//img//@scr')
    
        # 获取图片的url
        imgurl = []
        split_url = url.split("/")
        split_url.pop(-1)
        if len(img) >= 1:
            for x in img:
                imgurl.append("/".join(split_url)+"/"+x)
        else:
            imgurl=""

        # 获取标题并去除首尾空格
        title = "".join(html.xpath('//div[@class="row"]//div[@class="inner"]//h1/text()')).strip()


        # 获取作者并去除首尾空格
        author = (html.xpath('//div[@class="row"]//span[@class="appellation"]//text()')[-1].split(":")[-1]).strip()

        # 获取正文并去除首尾空格
        cont = html.xpath('//div[@class="content"]//div[@class="highlight"]/p//*[not(@color="navy")]/text()|//div[@class="highlight"]/p/text()')
        content = "".join(cont).strip()
        mainbody.append({"title":title,"author":author,"content":content,"imgurl":imgurl})
        # ********** End ********** #


    return mainbody