如何用迅雷下载python_使用python爬虫爬取迅雷侠下载，呵呵，你懂得

啥也不说了，直接上代码

#!python3

import urllib.request

from lxml import etree

import threading

from multiprocessing import Pool

from multiprocessing.dummy import Pool as ThreadPool

#龙图腾原创

#请求

#三种浏览器的user-agent

#headers = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.63 Safari/537.36'}

#headers = {'User-Agent':'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11'}

headers = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.221 Safari/537.36 SE 2.X MetaSr 1.0'}

max_try_down = 20

max_timeout_down = 1

allresult = "" #最终要下载的内容

allresult_count = 1; #下载个数

all_page_item = [];

#将下载结果总结到allresult

lock = threading.Lock();#线程锁初始化

def add_inresult(weburl,name,link,size):

global allresult_count,allresult

lock.acquire();#线程锁加锁

if link not in allresult:

allresult_count = allresult_count+1;

allresult += ("%s\r\n"%(weburl));

#allresult += ("len name=%d link=%d %d\r\n"%(len(name),len(link),len(size)));

allresult += ("name=%s\r\n"%(name));

allresult += ("size=%s\r\n"%(size));

allresult += ("link=%s\r\n"%(link));

allresult += ("\r\n");

lock.release();#线程锁解锁

#下载文件

def download_this(weburl):

#print("down %s++\n"%(weburl),end='');#因为使用了threadpool故此打印\n可能会遇到问题，故此使用这种打印方法

#下载循环尝试4次

for i in range(max_try_down):

try:

sub_req = urllib.request.Request(url=weburl,headers=headers)

sub_response = urllib.request.urlopen(sub_req,timeout=max_timeout_down)

sub_data = sub_response.read()

sub_data = sub_data.decode('utf-8')

tree=etree.HTML(sub_data);

file_name = " ".join(tree.xpath(u"//h3/text()"));#找到h3的text

file_size = tree.xpath(u"//b[text()='文件大小']/following-sibling::span")[0].text;#找到文本="文件大小"的节点 following-sibling::span寻找下一个span节点

file_link = tree.xpath(u"//a[@href=*]/@href")[0];#寻找存在href的 a节点，并且提取href

add_inresult(weburl,file_name,file_link,file_size);

break

except IndexError as e:

add_inresult(weburl,file_name,"连接失效:"+weburl,file_size);

except Exception as e:

print("%s [download_this]重试 %d <%s>:%s\n"%(weburl,i,type(e),e),end='');

pass

print("down %s ok--\n"%(weburl),end='');#因为使用了threadpool故此打印\n可能会遇到问题，故此使用这种打印方法

#列出当前页面的下载列表

def list_this(weburl):

global all_page_item

for i in range(max_try_down):

try:

#print("+++search+++%s\n"%weburl,end='');#因为使用了threadpool故此打印\n可能会遇到问题，故此使用这种打印方法

request = urllib.request.Request(url=weburl,headers=headers)

#爬取结果

response = urllib.request.urlopen(request,timeout=max_timeout_down)

data = response.read()

#设置解码方式

data = data.decode('utf-8')

tree=etree.HTML(data);

nodes = tree.xpath(u"//h5[@class='item-title']/a/@href");#找寻h5节点的class属性为item-title下的a节点/提取href属性

if len(nodes) <= 0:

return False

all_page_item += nodes;#统计所有的页面下的下载页面

print("allresult_count=%d"%len(all_page_item));

print("---search ok---%s \n"%weburl,end='');#因为使用了threadpool故此打印\n可能会遇到问题，故此使用这种打印方法

return True;

except Exception as e:

print("%s [list_this]重试 %d <%s>:%s\n"%(weburl,i,type(e),e),end='');#因为使用了threadpool故此打印\n可能会遇到问题，故此使用这种打印方法

pass

def search_this(name):

while True:

try:

url = "https://www.soxunlei.com/search/%s/page/%d.html"%(urllib.request.quote(name),1)

request = urllib.request.Request(url=url,headers=headers)

#爬取结果

response = urllib.request.urlopen(request,timeout=1)

data = response.read()

#设置解码方式

data = data.decode('utf-8')

tree=etree.HTML(data);

count = int(tree.xpath(u"//ol[@class='breadcrumb']/li[4]/font[2]")[0].text);#搜索返回个数，并且算出有多少页

page_count = count / 15 + 1 #一页容纳15个下载页，算出页数

print("%s搜索到%d个内容，分为%d页"%(name,count,page_count))

#讲下载页的url算出放入page_nodes

page_nodes = [];

for n in range(1,int(page_count)+1):

url = "https://www.soxunlei.com/search/%s/page/%d.html"%(urllib.request.quote(name),n)

page_nodes.append(url);

print("url=%s"%url);

break

except Exception as e:

print("%s:%s"%(type(e),e));

#调用python线程池模型，抓取所有下载页中的下载项

pool = ThreadPool(32)

results = pool.map(list_this,page_nodes);

pool.close();

pool.join();

#线程池模型,下载所有内容

pool = ThreadPool(32)

results = pool.map(download_this,all_page_item);

pool.close();

pool.join();

print(allresult);

print("抓取完成 项%d 下载%d"%(count,allresult_count));

if __name__ == "__main__":

search_this("上原空");