java爬取网易云歌单_爬取网易云音乐所有歌单

网易云音乐歌单首页

歌单第二页链接

歌单第三页链接

可以看出只有最后的数字在变化，所以进行url的构造，然后开始爬取

import csv

from lxml import etree

import requests

from multiprocessing.dummy import Pool

import time

headers = {

'Referer':'http://music.163.com/',

'Host':'music.163.com',

# 'User-Agent':'Mozilla/5.0 (X11: Linux *86_64; rv:38.0) Gecko/20100101 Firefox/38.0 Iceweasel/38.3.0',

'User-Agent': 'Mozilla/5.0 (X11: Linux *86_64; rv:38.0) Gecko/20100101 Firefox/38.0 Iceweasel/38.3.0',

'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',

}

wangyi='http://music.163.com'

base_url='http://music.163.com/discover/playlist/?order=hot&cat=%E5%85%A8%E9%83%A8&limit=35&offset={}'

urlList=[] #保存构造出的链接

def totalPage():

for i in range(0,1300,35):

url=base_url.format(i)

urlList.append(url)

def getData(url):

r=requests.get(url,headers=headers)

html=etree.HTML(r.text)

MFlist=html.xpath("//p[@class='dec']/a/text()")

MFurl=html.xpath("//p[@class='dec']/a/@href")

LisNum=html.xpath("//span[@class='nb']/text()")

for i in range(len(MFlist)):

write.writerow([MFlist[i],wangyi+MFurl[i],LisNum[i]])

time.sleep(3) #延时，防止IP被封

f=open('Allmusicform.csv','a+',newline='',encoding='utf-8')

write=csv.writer(f)

p=Pool(4)

totalPage()

p.map(getData,urlList)

p.close()

分别爬取了歌单名称、url、播放次数