python百度贴吧爬虫总体设计_python3爬虫爬取百度贴吧帖子图片

import requests

import re

from threading import Thread

from bs4 import BeautifulSoup

import os

class Pictures(object):

def __init__(self, url):

self.url = url

def main(self):

req = requests.get(self.url)

req.encoding = 'utf-8'

soup = BeautifulSoup(req.text, 'html.parser')

self.get_img_url(soup)

t1 = Thread(target=self.thread, args={0, 11})

t2 = Thread(target=self.thread, args={11, 22})

t1.start()

t2.start()

def thread(self, a, b):

for i in range(a, b):

url = 'https://tieba.baidu.com/p/5245797575?pn=%d' % i

print('第%d页' % i)

sp = self.url_open(url)

self.get_img_url(sp)

def get_img_url(self, soup):

for img in soup.select('.BDE_Image'):

src = re.search(r'https://.+?jpg', str(img)).group()

file_name = re.search('sign=.+/(.+jpg)', str(src)).group(1)

# print(src, file_name)

self.sava_img(src, file_name)

def sava_img(self, url, file_name):

headers = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36'}

req = requests.get(url, headers=headers).content

with open(file_name, 'wb') as f:

print("正在下载", file_name)

f.write(req)

def url_open(self, url):

req = requests.get(url)

req.encoding = 'utf-8'

soup = BeautifulSoup(req.text, 'html.parser')

return soup

if __name__ == "__main__":

# 爬取图片的网页url

url = 'https://tieba.baidu.com/p/5245797575'

if not os.path.exists('classimg'):

os.mkdir('classimg')

os.chdir('classimg')

pictures = Pictures(url)

pictures.main()