抓取贴吧

import urllib.request
from urllib import parse
from lxml import etree
import re

class Tieba():
def __init__(self):
pass
 self. headers = {"User-Agent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:60.0) Gecko/20100101 Firefox/60.0"}

def sendRequest(self,url,begin,end):
for value in range(begin,end+1):
 index = (value - 1) * 50
 u = "&pn="+str(index)
 fullurl = url + u
# print(fullurl)
 request = urllib.request.Request(fullurl,headers=self.headers)
 response = urllib.request.urlopen(request).read()
#print(response)
 content = re.compile(r'href="/p(.*?)"')
 line_list = content.findall(response.decode())
for line in line_list:
 url = "https://tieba.baidu.com/p" + line
# print(url)
 self.URLlHandle(url)
def URLlHandle(self,urll):
 request = urllib.request.Request(urll, headers=self.headers)
 html = urllib.request.urlopen(request).read()
 content = etree.HTML(html)
 line_list = content.xpath('//img[@class="BDE_Image"]/@src')
for line in line_list:
self.writeImage(line)
def writeImage(self,url):
 request = urllib.request.Request(url,headers=self.headers)
 image = urllib.request.urlopen(request).read()
 filename = url[-10:]
print("下载%s中" % (filename))
with open(filename,'wb') as f:
 f.write(image)
if __name__ == "__main__":
 url = "https://tieba.baidu.com/f?"
 keyword = input("请输入搜寻的关键字 >>")
 beginPage = int(input("请输入起始页 >>"))
 endPage = int(input("请输入结束页 >>"))
 kw = parse.urlencode({"kw":keyword})
 fullurl = url + kw
 pattern = Tieba()
 pattern.sendRequest(fullurl,beginPage,endPage)

转载于:https://www.cnblogs.com/angle90/p/9157065.html