豆瓣民谣排行100爬取练习

第三天，学习利用Xpath解析库

import requests
import time
from lxml import etree

def get_one_page(url):
    proxies = {"http":"http://61.145.69.27","http":"http://121.61.0.208"}
    headers = {'User-Agent' : 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36'}
    response = requests.get(url,headers=headers,proxies=proxies)
    if response.status_code == 200:
        return response.text
    return None
def parse_one_page(html):
    r = etree.HTML(html)
    for node in r.xpath('//div[@class="pl2"]'):
        print(node.xpath('.//a/text()')[0])
        print(node.xpath('.//p/text()')[0])
        print(node.xpath('.//div//span[@class="rating_nums"]/text()')[0])
        print(node.xpath('.//div/span[@class="pl"]/text()')[0])

def main(offset):
    url = 'https://music.douban.com/tag/民谣?start='+str(offset)+'&type=S'
    html = get_one_page(url)
    parse_one_page(html)
if __name__ == '__main__':
    for i in range(5):
        main(i * 20)
        time.sleep(1)

结果