爬虫笔记_09

获取豆瓣T250音乐排行榜
储存在csv文件

使用BeautifulSoup和正则表达式分析HTML代码

# 专辑名称、表演者、流派、发行时间、出版者、评分
from bs4 import BeautifulSoup
import requests
import re
import csv
import time
headers={'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:102.0) Gecko/20100101 Firefox/102.0'}
# 抓取指定T250排行榜页面
def get_url_music(url):
    html=requests.get(url,headers=headers)
    soup=BeautifulSoup(html.text,'lxml')
    aTags=soup.find_all('a',attrs={'class':'nbg'})
    for aTag in aTags:
        get_music_info(aTag['href'])

# 将提取的信息保存到filename参数指定的文件中，分析结果有info参数提供，该参数是一个字典类型的值
def save_csv(filename,info):
    with open(filename,'a',encoding='utf-8') as f:
        fieldnames=['name','author','style','time','publisher','score']
        writer=csv.DictWriter(f,fieldnames=fieldnames)
        writer.writerow(info)


# 抓取分析专辑页面的代码
def get_music_info(url):
    html=requests.get(url,headers=headers)
    soup=BeautifulSoup(html.text,'lxml')
    # 获取专辑名称
    name=soup.find(attrs={'id':'wrapper'}).h1.span.text
    # 表演者
    author=soup.find(attrs={'id':'info'}).find('a').text
    # 流派
    styles=re.findall('<span class="pl">流派:</span>&nbsp;(.*?)<br />',html.text,re.S)
    if len(styles) ==0:
        style='未知'
    else:
        style=styles[0].strip()
    # 发行时间
    time=re.findall('<span class="pl">发行时间:</span>&nbsp;(.*?)<br />',html.text,re.S)[0].strip()
    # 出版者
    publishers=re.findall('<span class="pl">出版者:</span>&nbsp;(.*?)<br />',html.text,re.S)
    if len(publishers) ==0:
        publisher='未知'
    else:
        publisher=publishers[0].strip()
    # 评分
    score=soup.find(class_='ll rating_num').text
    info={
        'name':name,
        'author':author,
        'style':style,
        'time':time,
        'publisher':publisher,
        'score':score
    }
    print(info)
    # 保存分析
    save_csv(filename,info)

if __name__ == '__main__':
    urls=['https://music.douban.com/top250?start={}'.format(str(i)) for i in range(0,250,25)]
    filename='music.csv'
    # 创建music.csv文件,并向该文件添加头信息
    with open(filename,'w',encoding='utf-8') as f:
        fieldnames=['name','author','style','time','publisher','score']
        writer = csv.DictWriter(f, fieldnames=fieldnames)
        writer.writeheader()
    for url in urls:
        get_url_music(url)
        time.sleep(1)

结果;