爬取百度
爬取慕客网所有关于python的课程名及描述信息, 并通过词云进行分析展示;
- 网址: https://www.imooc.com/search/course?words=python
import re
import requests
from bs4 import BeautifulSoup
import jieba
import numpy
from PIL import Image
import wordcloud
def get_pagehtml(url):
return requests.get(url).text
'''<a href="//coding.imooc.com/class/114.html" target="_blank" class="course-detail-title">
<span class="highlight">Python</span>操作三大主流数据库
</a>'''
def getname(text):
'''
:param text: 获取当前网页的名字和相关信息
:return: 返回的是一个列表,字典里存储了python每门课程的名字和信息
'''
soup = BeautifulSoup(text, 'html5lib')
#<div class="course-item-detail">
nametag_li = soup.find_all('div', class_="course-item-detail")
info_li = []
#<span class="highlight">Python</span>操作三大主流数据库
for i in nametag_li:
info = {}
#匹配当前元素下的所有汉字
i = re.findall(r'[\u4E00-\u9FA5]+',str(i))
info['name'] = i[0]
info['info'] = i[1:]
info_li.append(info)
return info_li
def creat_wordcloud(text):
imgobj = Image.open('img1.jpg')
cloud_mask = numpy.array(imgobj)
result = jieba.lcut(text)
wc = wordcloud.WordCloud(
width=500,
mask=cloud_mask,
max_font_size=100,
min_font_size=10,
background_color='snow',
font_path = './font/msyh.ttf',
)
wc.generate(','.join(result))
wc.to_file('./python.png')
def main():
li = []
for i in range(2):
url = 'https://www.imooc.com/search/course?words=python&page=%d' %(i+1)
info = getname(get_pagehtml(url))
for j in info:
li.append(j['name'])
li.append(''.join(j['info']))
print(j)
creat_wordcloud(''.join(li))
main()
结果:
获取今日百度热点
from bs4 import BeautifulSoup
from urllib.request import urlopen
def getPagehtml(url):
#获取网页源代码
a = urlopen(url).read()
#注意:此时的编码格式为gb2312
return a.decode('gb2312','ignore')
def deal_text(text):
soup = BeautifulSoup(text,'html5lib')
# <a class="list-title" target="_blank"
#找出满足上一行的tag
info_li = soup.find_all('a',class_='list-title')
#从满足要求的tag中提取标题内容,并存入列表当中
news_li = [info_li[i].string for i in range(10)]
return news_li
def main():
url = 'http://top.baidu.com/buzz?b=341'
new_li = deal_text(getPagehtml(url))
[print(i) for i in new_li]
main()
结果: