根据输入的类别和页数来爬取糗事百科,基于练习,分别使用xpath匹配段子模块、正则匹配视频模块、bs4 匹配热图模块。
# 连接数据库类
class SQL_connect:
conn = ""
cur = ""
def __init__(self):
self.conn = pymysql.connect(
host='localhost',
port=3306,
user='root',
password='123456',
db='pachong'
)
self.cur = self.conn.cursor()
# 添加段子模块元素到数据库table1表中
def ins_text(self, a, b):
s = 'insert into table1 values(%s,%s)'
self.cur.execute(s, (a, b))
self.conn.commit()
# 添加视频模块元素到数据库vid表中
def ins_vid(self, a):
s = 'insert into video values(%s)'
self.cur.execute(s, a)
self.conn.commit()
# 添加热图模块元素到数据库img表中
def ins_img(self, a):
s = 'insert into image values(%s)'
self.cur.execute(s, a)
self.conn.commit()
# 请求类
class URL_Request():
def req(self,a,b):
for i in range(1,b+1):
url = "https://www.qiushibaike.com/{type}/page/{pge}/".format(type= a, pge= i)
header = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:82.0) Gecko/20100101 Firefox/82.0"}
res = request.Request(url=url, headers=header)
# 获取响应
response = request.urlopen(res)
# 读取响应内容
yya = response.read().decode("utf-8")
return yya
#解析HTML类
class Pipei():
# xpath匹配段子模块
def Text_type(self,a,b):
u = URL_Request()
yya = u.req(a, b)
yyb = etree.HTML(yya)
# 作者
yyc = yyb.xpath('//div[@class="article block untagged mb15 typs_hot"]/div/a/h2/text()')
# 内容
yye = yyb.xpath('//div[@class="article block untagged mb15 typs_hot"]/a/div/span/text()')
list1 = []
list2 = []
for i in yyc:
a = i.strip()
list1.append(a)
for i in yye:
a = i.strip()
list2.append(a)
print("成功抓取到作者信息:",list1)
print("成功抓取到发布内容:",list2)
sql = SQL_connect()
for i in range(len(list1)):
sql.ins_text(list1[i], list2[i])
print("------------------------------数据已成功上传数据库!------------------------------")
# 正则匹配视频模块
def Video_type(self, a, b):
u = URL_Request()
yya = u.req(a, b)
yyb = re.findall(r'<source src="(.*?hd.mp4)', yya)
list1 = []
for i in yyb:
j = "http:"+i
list1.append(j)
for z in list1:
print("成功抓取到视频路径:",z)
sql = SQL_connect()
for i in range(len(list1)):
sql.ins_vid(list1[i])
print("------------------------------数据已成功上传数据库!------------------------------")
# bs4 匹配热图模块
def Imgrank_type(self, a, b):
u = URL_Request()
yya = u.req(a, b)
soup = BeautifulSoup(yya,'lxml')
yyb = soup.findAll('img')
list1 = []
for i in yyb:
j = "http:"+i.get('src')
list1.append(j)
for z in list1:
print("成功抓取到图片路径:",z)
sql = SQL_connect()
for i in range(len(list1)):
sql.ins_img(list1[i])
print("------------------------------数据已成功上传数据库!------------------------------")
if __name__ == '__main__':
type = input("请选择要爬取的分类(text/video/imgrank):")
pge = int(input("请输入爬取页数:"))
if type == 'text':
Pipei().Text_type(type, pge)
elif type == 'video':
Pipei().Video_type(type,pge)
elif type == 'imgrank':
Pipei().Imgrank_type(type,pge)
else:
print("输入错误!!!")