python多线程爬虫实例-Python 多线程 爬虫 通信 示例

# _*_ coding=utf-8 _*_

import threading

import requests

from queue import Queue

from bs4 import BeautifulSoup

_sentinel = object()

class GetHtml(threading.Thread):

def __init__(self, queue):

super().__init__()

self.queue = queue

def run(self):

while True:

url = self.queue.get()

if url is _sentinel:

break

con = requests.get(url)

print(BeautifulSoup(con.text, 'html.parser').find('title').text)

class GetUrl(threading.Thread):

def __init__(self, queue):

super().__init__()

self.queue = queue

def run(self):

i = 1

while True:

context = requests.get('http://python.jobbole.com/category/project/page/{}/'.format(i))

if context.status_code == 404:

break

urls = [url_text['href'] for url_text in BeautifulSoup(context.text, 'html.parser').find_all('a', class_='archive-title')]

for url in urls:

self.queue.put(url)

i += 1

self.queue.put(_sentinel)

if __name__ == '__main__':

queue_main = Queue(maxsize=3)

thread1 = GetUrl(queue_main)

thread2 = GetHtml(queue_main)

thread2.start()

thread1.start()