python多进程爬虫卡住_python多进程爬虫解决进程挂掉问题

这几天写了个爬虫，爬取的数据比较多。一直挂在服务器上跑，后面发现启动十个进程总会运行着某几个进程挂掉，导致数据采集工作比较延后。

后面重新改进了一下，从日志中读取当前进程断点，继续爬取。

用了一个笨方法解决进程挂掉的问题。就是每半个小时关闭所有进程，重新从断点开始，当然各位有好的方法可以留言告诉我。

脚本使用方法直接运行 run.py

run.py

#-*- coding:utf-8 -*-

import os,time

import re

#检查sqlpwd.py的进程，并关闭进程

def kill():

lines = os.popen('ps -e|grep sqlpwd.py').readlines()

pa = re.compile(r'(\d+) ')

for line in lines:

result = re.findall(pa,line)

print(result[0])

os.popen('kill -9 '+str(result[0]))

def check():

with open('log.txt','r') as f:

lines = f.readlines()

for line in lines:

if 'FinishAll' in line:

return True

return False

'''

如果已经进行爬取了，则直接从上次扫描的地方开始

设置每半个小时重新扫描，解决之前扫描过程中进程掉的问题

'''

def runpwd():

if os.path.exists('log.txt') == False:

os.system('python sqlpwd.py 0 &')

time.sleep(18000)

kill()

while True:

print('-------------启动进程-------------------')

os.system('python sqlpwd.py 1 &')

time.sleep(60)

print('-------------结束进程-------------------')

kill()

if check() == True:

print('所有信息采集完成')

break

else:

pass

if __name__ == '__main__':

runpwd()

sqlpwd.py

#-*- coding:utf-8 -*-

import requests

import re

import random

from bs4 import BeautifulSoup

from multiprocessing import Process,Pool,Lock

import multiprocessing

import time

import sys

import logging

reload(sys)

sys.setdefaultencoding('utf-8')

def get_pwd(j,start,end):

logger = logging.getLogger(__name__)

logger.setLevel(level = logging.INFO)

handler = logging.FileHandler("log.txt")

handler.setLevel(logging.INFO)

formatter = logging.Formatter('%(asctime)s - %(message)s')

handler.setFormatter(formatter)

console = logging.StreamHandler()

console.setLevel(logging.INFO)

logger.addHandler(handler)

logger.addHandler(console)

user_agent=['Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.87 Safari/537.36',

'Mozilla/5.0 (X11; U; Linux x86_64; zh-CN; rv:1.9.2.10) Gecko/20100922 Ubuntu/10.10 (maverick) Firefox/3.6.10',

'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',

'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36',

'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.71 Safari/537.1 LBBROWSER',

'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; QQBrowser/7.0.3698.400)',

]

headers={

'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',

'Accept-Encoding': 'gzip, deflate, sdch',

'Accept-Language': 'zh-CN,zh;q=0.8',

'User-Agent': user_agent[random.randint(0,5)]

}

#print(start,end)

logger.info('进程'+str(j)+'任务从第'+str(start)+'页采集到'+str(end)+'页')

for i in range(start,end):

url = 'https://www.mysql-password.com/database/'+str(i)

#print(url)

logger.info(url)

try:

r = requests.get(url,headers=headers)

r.encoding = 'utf8'

html = r.text

soup = BeautifulSoup(html,'lxml')

except Exception as e:

print('进程'+str(j)+'异常：原因如下')

print(e)

#log.write('进程'+str(j)+'异常：原因如下'+str(e)+'\n')

logger.info('进程'+str(j)+'异常：原因如下'+str(e))

time.sleep(10)

continue

filename = 'password'+str(j)+'.txt'

try:

for tag in soup.find('tbody').find_all('tr'):

ss = str(tag.find_all('td')[2].string)

#print(ss)

with open(filename,'a') as f:

f.write(ss+'\n')

except Exception as e:

print('进程'+str(j)+'异常：原因如下')

print(e)

#log.write('进程'+str(j)+'异常：原因如下'+str(e)+'\n')

logger.info('进程'+str(j)+'异常：原因如下'+str(e))

time.sleep(10)

continue

#print('进程'+str(j)+'采集完成第'+str(i)+'页')

logger.info('进程'+str(j)+'采集完成第'+str(i)+'页')

def check():

num = []

mul = []

with open('log.txt','r') as f:

lines = f.readlines()

pa = re.compile(r'https://www.mysql-password.com/database/(\d+)')

for line in lines:

result = re.findall(pa,line)

if len(result) != 0 :

#print(result)

num.append(int(result[0]))

m = 10

start = 20000

cha = int(((180000-start)/m))

for j in range(0,m):

numList = []

info = []

end = 20000 + cha * (j+1)

for n in num:

if n > start and n < end:

numList.append(n)

print (str(j)+','+str(numList[-1])+','+str(end)+',')

if numList[-1] + 1 == end:

start = end + 1

continue

else:

info.append(j)

info.append(numList[-1])

info.append(end)

mul.append(info)

start = end + 1

return mul

def run(type):

if type == 0:

m = 10

pool = multiprocessing.Pool(m)

#start是开始爬行的页数， 下面写得是20000 180000 意思是从20000页爬到180000页，根据实际情况修改

start = 20000

cha = int(((180000-start)/m))

for j in range(0,m):

end = 20000 + cha * (j+1)

pool.apply_async(get_pwd,args=(j,start,end,))

#print(start,end)

start = end + 1

pool.close()

pool.join()

else:

mul = check()

#print(mul)

if len(mul) == 0:

print('所有进程都爬取完成')

logger.info('FinishAll')

else:

pool = multiprocessing.Pool(len(mul))

for mu in mul:

pool.apply_async(get_pwd,args=(mu[0],mu[1],mu[2],))

pool.close()

pool.join()

if __name__ == '__main__':

runtype = int(sys.argv[1])

run(runtype)