python写爬虫4-多线程爬虫(采集58出租房信息)_python实现多线程爬虫

前言：

本文利用python语言实现了一个多线程爬虫。

正文：

开发环境：

ubuntu16.04，python3.6,bs4,virtualenv(虚拟环境)

创建虚拟环境：

创建项目文件夹，并为项目创建虚拟环境，利用pip安装相关包

mkdir mutiThreadCrawier

cd mutiThreadCrawier

mkdir content #存爬下来的页面

virtualenv env --python =python3.6 #创建虚拟环境

source env/bin/activate #使虚拟环境生效

导包：

import time

import re

import threading

import urllib

import requests

from bs4 import BeautifulSoup

定义变量

g_mutex = threading.Condition() # 可以加锁/释放锁

print(g_mutex)

print(type(g_mutex))

g_urls = [] # 存放解析出来的url对应的网页源码

g_queue_urls = [] # 待爬取的url

g_exist_urls = [] # 已经爬过的url

g_failed_urls = [] # 失败的链接

g_total_count = 0 # 已经下载的页面的计数器

定义线程类：

创建一个线程类，继承于threading.Thread,并进构造，在run函数中根据url路径请求网络连接，并保存页面html文档保存到本地，如果下载失败则抛出异常。并将下载过页面的路由添加到g_exist_urls

class CrawlerThread(threading.Thread):

def __init__(self,url,filename,tid):

threading.Thread.__init__(self)

self.filename=filename

self.url =url

self.tid=tid

def run(self):

try:

resp=urllib.request.urlopen(self.url)

html=resp.read()

with open('content/'+self.filename,'wb') as f:

f.write(html)

except Exception as e:

g_exist_urls.append(self.url)

g_failed_urls.append(self.url)

print(f'页面{self.url}下载失败！')

g_mutex.acquire()

g_urls.append(html)

g_exist_urls.append(self.url)

g_mutex.release()

定义爬虫类：

对其进行构造，创建日志，download()函数创建线程，update_queque_url对连接的列表进行更新,get_url()根据bs4进行匹配获取连接，download_all()通过调用download()函数实现批量下载。spider作为一个入口函数进行爬取

class Crawler:

def __init__(self,name,domain,thread_number):

self.name=name

self.domain=domain

self.thread_number=thread_number

self.logfile=open('log.txt','w')

self.thread_pool=[]

self.url = 'http://'+domain

def spider(self):# 内容会随着爬虫的进行而更新

global g_queue_urls# 初始，队列中仅有一个url

g_queue_urls.append(self.url)# 爬取的深度

depth =0

print(f'爬虫{self.name}开始启动........')

while g_queue_urls:

depth +=1

print(f'当前爬取深度是{depth}')

self.logfile.write(f'URL:{g_queue_urls[0]}')

self.download_all() # 下载所有

self.update_queque_url() # 更新 url队列

self.logfile.write(f">>>Depth:{depth}")

count = 0

while count

self.logfile.write(f"累计爬取{g_total_count}条，爬取是第{g_queue_urls[count]}个")

count+=1

def download_all(self):

global g_queue_urls

global g_total_count

i=0

while i < len(g_queue_urls):

j=0

while j

g_total_count +=1

print(g_queue_urls[i+j])

thread_result=self.download(g_queue_urls[i+j],f"{g_total_count}.html",j)

if thread_result is not None:

print(f'线程{i+j}启动')

j +=1

i=i+j

for thread in self.thread_pool:

thread.join(25)

g_queue_urls=[]

def download(self,url,filename,tid):

print(url,filename,tid)

creawler_thread= CrawlerThread(url,filename,tid)

self.thread_pool.append(creawler_thread)

creawler_thread.start()

def update_queque_url(self):

global g_queue_urls

global g_exist_urls#已经爬过的url

new_urls=[]#新发现的url

for url_content in g_urls:

new_urls +=self.get_Url(url_content)# 从页面中提取新url

g_queue_urls=list(set(new_urls) -set(g_exist_urls)) # 去除重复的和已经爬过的

def get_Url(self,content):

'''

从网页源代码中提取url

'''

links =[] # 保存提取到的href

try:

soup =BeautifulSoup(content)

for link in soup.findAll('a'):

if link is not None and link.get('href') is not None:

if self.domain in link['href']:

# 如果link是本网站的绝对地址

links.append(link)

elif len(link['href']) >10 and 'http://' not in link['href']:

# 如果link是相对地址

links.append(self.url +link['href'])

except Exception as e:

print("fail to get url",e)

return links

主函数

主函数调用爬虫函数的spider()方法

if __name__=="__main__":

domain ="www.geyanw.com"

thread_number=10

name="geyan"

crawler =Crawler(name,domain,thread_number)

crawler.spider()