#!/bin/bash/env python
import MySQLdb
import requests
import threading
def toWrite(fp,lst,url):
for i in lst:
testurl = url+i[0] //定向url
testurl = testurl.lower()
print testurl
rsp = requests.get(testurl)
print rsp.status_code
fp.write(i[0]+'\t'+str(rsp.status_code)+'\n')
url = 'http://127.0.0.1/u/'
conn = MySQLdb.connect('127.0.0.1','root','123456','coupon_site')
cur = conn.cursor()
res = cur.execute('select user_name_slug from app_coupon_api where couponid>642369')
lst = cur.fetchall()
fp = open('404.txt','w+')
threads = []
for i in range(0,len(lst),150000): //分片,线程个数为: 总数组长度/150000
t = threading.Thread(target=toWrite,args=(fp,lst[i:i+150000],url)) //多线程参数传递
threads.append(t)
for t in threads: //循环开启线程,setDaemon不等待同步
t.setDaemon(True)
t.start()
print(len(threads))
for t in threads:
t.join()
python多线程定向爬虫
最新推荐文章于 2024-07-07 23:56:48 发布