一个练手的小爬虫,这个是写在图片爬取的那个前面
熟悉了下requests,BeautifulSoup,正则表达式的操作
本来不想放上来,因为感觉写的不好,放上来方便自己以后修改下
import requests
from bs4 import BeautifulSoup
import re
def gethtml(url):
try:
r = requests.get(url)
r.raise_for_status()
r.encoding = r.apparent_encoding
return r.text
except:
return 'wrong!!!'
def save_way(url,fpath):
html = gethtml(url)
soup = BeautifulSoup(html,'html.parser')
ID = []
level = []
content = []
haoxiao = []
pinglun = []
Fun_pinglun = []
data = ['ID:','等级:','内容:','好笑:','评论:']
ids = soup.find_all('h2')
for k in ids:
ID.append(k.text)
Levels = soup.find_all('div',attrs={'class':'author clearfix'})
for k in Levels:
l = re.findall(r'<div class="articleGender .+">(\d+)</div>',str(k))[0]
level.append(l)
Contents = soup.find_all('div',attrs={'class':"content"})
for k in Contents:
c = re.sub(r'<br/>','\n',str(k))
cend = re.findall(r'<span>(.+)</span>',c,flags = re.S)[0]
content.append(cend)
num_haoxiao = soup.find_all('span',attrs={'class':"stats-vote"})
for k in num_haoxiao:
haoxiao.append(k.i.text)
share = soup.find_all('a',attrs={'class':'qiushi_comments'})
for k in share:
try:
p = k.find('i',attrs={'class':'number'})
pinglun.append(p.text)
except:
continue
count = 0
with open(fpath,'w',encoding = 'utf-8') as f:
print(len(ID),len(level),len(content),len(haoxiao),len(pinglun))
for i in range(len(ID)):
count = count + 1
print('\r正在爬取第%d个段子......'%count)
f.write(data[0] + ID[i] +'\n')
f.write(data[1] + level[i] +'\n')
f.write(data[2] + content[i]+'\n')
f.write(data[3] + haoxiao[i]+'\n')
f.write(data[4] + pinglun[i]+'\n\n\n')
def main():
url = 'https://www.qiushibaike.com/'
fpath = 'E:qiushibaike.txt'
save_way(url,fpath)
main()