个人总结的Spider模板。
import requests
from lxml import etree
import threading
import queue
# 获取网页源数据,返回element object
def get_page(url):
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.183 Safari/537.36"
}
try:
respone = requests.get(url=url,headers=headers).text
element_obj = etree.HTML(respone)
except:
return None
return element_obj
# 进行网页数据解析,返回数据任务队列q
def params_page(ele_obj)
titles = ele_obj.xpath('')
links = ele_obj.xpath('')
...
if links:
# 构造任务队列
q = queue.Queue()
data = zip(titles,links,...)
# 压数据进队
for i in data:
q.put(i)
return q
return None
# 进行数据存储
def store_data(qobj):
while not qobj.empty():
data = qobj.get()
print(f'正在下载:{item[0]}')
with open(path, 'wb+') as obf:
obf.write(requests.get(url=data[1], headers={"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.183Safari/537.36"}).content)
print(f'任务下载完成:{item[0]}')
if __name__ == "__main__":
url = ''
elobj = get_page()
if elobj:
qobj = params_page
if qobj:
threads = []
# 构造多线程 20个
for th in range(20):
t = threading.Thread(target=stroge_video, args=(li,))
t.start()
threads.append(t)
for thread in threads:
thread.join()
else:
print("网页数据解析失败")
else:
print("网站请求错误")