python爬取糗事百科

import requests
from lxml import etree
from time import time,sleep

#3.解析网站返回信息
headers ={
    'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.157 Safari/537.36'
}
def response_list_sy(url_list):
    print('正在请求:{}'.format(url_list))
    response_list = requests.get(url_list,headers=headers)
    return response_list.text
#4.获取信息 解析数据 获取详情页URL 组成列表返回
def parse_list(list_resp):

    resp_html = etree.HTML(list_resp)
    li_list = resp_html.xpath('//div/div/div/div/ul/li/div/a/@href')
    list_url=[]
    for li in li_list:
        detail_url = 'https://www.qiushibaike.com{}'.format(li)
        list_url.append(detail_url)
    return(list_url)

#5.解析详情页URL 并解析 标题,时间,好笑度
def list_url_pin(date_url_list):
    for u in date_url_list:
        url_sp = requests.get(u,headers=headers)
        result={}
        res_html_list = etree.HTML(url_sp.text)
        result['title'] = res_html_list.xpath('//div/div/div/h1/text()')[0].strip()
        result['time1'] = res_html_list.xpath('//div/div/div/div/span/text()')[0].strip()
        result['hxd'] = res_html_list.xpath('//div/div/div/div/span/i/text()')[0]
        print(result)

if __name__ == '__main__':
    s_time = time()
    for i in range(10):
#1.请求进入网址
        url_list = 'https://www.qiushibaike.com/8hr/page/{}/'.format(i+1)
#2.传入类
        list_resp = response_list_sy(url_list)

        date_url_list = parse_list(list_resp)

        lp_resp = list_url_pin(date_url_list)
    print('单任务时,耗时{}'.format(time()-s_time))