基于python网易云音乐歌手名爬取
import os
import time
import requests
import threading
from lxml import etree
from queue import Queue
class GetMain(threading.Thread):
def __init__(self,url,headers,BASE_URl):
super(GetMain, self).__init__()
self.url = url
self.headers = headers
self.BASE_URl = BASE_URl
def run(self):
resp = requests.get(self.url,headers=self.headers).content.decode()
tree = etree.HTML(resp)
country_list = tree.xpath('//div[@class="g-sd2"]/div//div')
for country in country_list:
country_name = country.xpath('h2/text()')[0]
style_list = country.xpath('ul/li')
for style in style_list:
style_name = style.xpath('a/text()')[0]
style_href = self.BASE_URl+style.xpath('a/@href')[0]
styleList_queue.put(style_href)
class GetName(threading.Thread):
def __init__(self,headers,BASE_URl):
super(GetName, self).__init__()
self.headers = headers
self.BASE_URl = BASE_URl
def run(self):
while True:
if styleList_queue.empty() and flag1:
break
try:
style_url = styleList_queue.get(block=False)
response = requests.get(style_url,headers = headers).content.decode()
tree = etree.HTML(response)
letter_li = tree.xpath('//ul[@class="n-ltlst f-cb"]/li[position()>1]')
for letter in letter_li:
letter_url = BASE_URl+letter.xpath('a/@href')[0]
letter_url_queue.put(letter_url)
except:
pass
class GetStarName(threading.Thread):
def __init__(self,headers):
super(GetStarName, self).__init__()
self.headers = headers
def run(self):
while True:
if letter_url_queue.empty() and flag2:
break
try:
letter_url = letter_url_queue.get(block=False)
response = requests.get(letter_url,headers=self.headers).content.decode()
tree = etree.HTML(response)
name_ul = tree.xpath('//ul[@class="m-cvrlst m-cvrlst-5 f-cb"]')[0]
name1 = name_ul.xpath('li/p/a[1]/text()')
name2 = name_ul.xpath('li/a/text()')
for name in name1:
with lock:
with open('star_name_thread.txt', 'a', encoding='utf-8')as f:
f.write(name + '\n')
for name in name2:
with lock:
with open('star_name_thread.txt', 'a', encoding='utf-8')as f:
f.write(name + '\n')
except:
pass
flag1 = False
flag2 = False
styleList_queue = Queue()
letter_url_queue = Queue()
lock = threading.Lock()
if __name__ == '__main__':
start = time.time()
url_main = 'https://music.163.com/discover/artist'
headers = {
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36',
}
BASE_URl = 'https://music.163.com'
t1 = GetMain(url=url_main,headers=headers,BASE_URl=BASE_URl)
t1.start()
t1_list = []
for i in range(5):
t = GetName(headers=headers,BASE_URl=BASE_URl)
t.start()
t1_list.append(t)
t2_list = []
for i in range(5):
t2 = GetStarName(headers=headers)
t2.start()
t2_list.append(t2)
t1.join()
flag1 = True
for i in t1_list:
i.join()
flag2 = True
for i in t2_list:
i.join()
end = time.time()
print('程序结束耗时%s秒'%(end-start))