爬取百度贴吧照片

最新推荐文章于 2024-09-26 11:55:47 发布

daqinjun

最新推荐文章于 2024-09-26 11:55:47 发布

阅读量270

点赞数

分类专栏：爬虫机器学习文章标签： python 爬虫

本文链接：https://blog.csdn.net/daqinjun/article/details/72140745

版权

机器学习同时被 2 个专栏收录

3 篇文章 0 订阅

订阅专栏

爬虫

2 篇文章 0 订阅

订阅专栏

基于python的百度贴吧爬虫

# -*- coding: utf-8 -*-
from bs4 import BeautifulSoup
import urllib2
import urllib
import re

class imgTest:

    def __init__(self, baseUrl, seeLZ):
        self.baseUrl = baseUrl
        self.seeLZ = '?see_lz='+str(seeLZ)

    #print to log.txt
    def printToLog(self,mystr):
        f = open('txt/log.txt', 'a')
        f.write(mystr+"\n")
        f.close()
    #get the html source code
    def getPage(self, pageNum):
        try:
            url = self.baseUrl+self.seeLZ +'&pn='+str(pageNum)
            request = urllib2.Request(url)
            response = urllib2.urlopen(request)
            content = response.read()
            return content
        except urllib2.URLError, e:
            if hasattr(e, "reason"):
                print "failed to connect baidutieba.",e.reason
                return None

    def getPageNum(self):
        page = self.getPage(1)
        soup = BeautifulSoup(page,'html.parser')
        pageNum = soup.find_all("span",class_='red')[1].string
        return pageNum

    def getTitle(self):
        page = self.getPage(1)
        soup = BeautifulSoup(page,'html.parser')
        return soup.h3.string

    def getAllImageURLs(self,pageNum):
        page = self.getPage(pageNum)
        soup = BeautifulSoup(page,'html.parser')
        imgTags = soup.find_all("img",class_="BDE_Image")
        imgURLs = []
        for item in imgTags:
            imgURLs.append(item.get('src'))
        print imgURLs
        return imgURLs
    #save a single img
    def saveImg(self,imageURL,filename):
        u = urllib.urlopen(imageURL)
        data = u.read()
        f = open(filename,'wb')
        f.write(data)
        f.close()
    #download images
    def saveImgs(self, images, name, num):
        number = num
        for imageURL in images:
            splitPath = imageURL.split('.')
            fTail = splitPath.pop()
            if len(fTail)>3:
                fTail = "jpg"
            fileName = name+"/"+str(number)+"."+fTail
            self.saveImg(imageURL,fileName)
            number += 1


baseURL = 'http://tieba.baidu.com/p/4909074033'
imgtest = imgTest(baseURL,1)
totalnum = int(imgtest.getPageNum())

imageCount = 0
for i in range(1, totalnum+1):
    imageURLs = imgtest.getAllImageURLs(i)
    imgtest.saveImgs(imageURLs,"C:\Users\123\Desktop\9",imageCount)#设置路径
    imageCount += len(imageURLs)
    print imageCount

![这里写图片描述](https://img-blog.csdn.net/20170515140855809?watermark/2/text/aHR0cDovL2Jsb2cuY3Nkbi5uZXQvZGFxaW5qdW4=/font/5a6L5L2T/fontsize/400/fill/I0JBQkFCMA==/dissolve/70/gravity/SouthEast)