前言:
原来数据分析需要很好的基础啊!!
1.使用selenium进行获取页面信息;
用这个原因好像是方便?还是因为简单,,忘记了。
from selenium.webdriver import Chrome
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import selenium.webdriver.common.by as BY
from lxml import etree
import pandas as pd
import requests
import re
# 上海堡垒评论首页
url = 'https://movie.douban.com/subject/26581837/comments?status=P'
driver = Chrome()
wait = WebDriverWait(driver, 10)
driver.get(url)
# 定义dataframe
all_data = pd.DataFrame()
# 获取静态网页
# 使用xml进行解析
# 通过找到指定元素进行转页操作
for i in range(44):
next_page = wait.until(EC.element_to_be_clickable(
(BY.By.CSS_SELECTOR, '#paginator > a.next')
))
text = driver.page_source
dom = etree.HTML(text, etree.HTMLParser())
user_name = dom.xpath('//*[@class="comment-info"]/a/text()') # 用户名
score = dom.xpath('//*[@class="comment-info"]/span[2]/@class') # 评分
comment = dom.xpath('//*[@class="short"]/text()') # 评论正文
comment_time = dom.xpath('//*[@class="comment-time "]/@title')
user_url = dom.xpath('//*[@class="comment-info"]/a/@href')
cities = []
for u in user_url:
rq = requests.get(u)
d = etree.HTML(rq.content, etree.HTMLParser())
cities.append(d.xpath('//*[@id="profile"]/div/div[2]/div[1]/div/a/text()'))
import time
time.sleep(2)
cities = [i[0] if i !=[] else '' for i in cities]
score = [int(re.findall('[0-9]{2}', i)[0]) for i in score]
data = pd.DataFrame({'用户名':user_name, '评分':score, '评论正文': comment, '评论时间':comment_time, '用户常居地':cities})
all_data = pd.concat([all_data, data])
next_page.click() # 点击
# 每翻一页就保存吧,避免封ip
all_data.to_excel('./temp.xlsx')
注意事项:
1.虽然加入了等待操作,但是一般都200多条的时候就会被封ip,这里就不给出换ip的操作拉,因为只是一个简单的小程序。
2.用xml解析网页就是这么直接简单哦。
2.制作词云。
老师说加载词库有个小技巧,就是找词库可以直接去搜狗阿那些输入法里面找词库,然后有现成的转换代码。
import pandas as pd
import re
import matplotlib.pyplot as plt
import jieba
from wordcloud import WordCloud
# 展示心形图
# 读取
df = pd.read_excel('F:\\myPrpject\\douban\\temp.xlsx')
df['city'] = [i if i != ' 'else '' for i in df['用户常居地']]
df['comment'] = df['评论正文']
df['comment'].apply(lambda x: re.sub('\n', '', x))
jieba.load_userdict('D:\\工程实践\\1.2豆瓣评论\\day2\\data\\dict.txt')
data_cut = df['comment'].apply(jieba.cut) # jieba分词返回一个生成器
# 加载停用词
with open('D:\\工程实践\\1.2豆瓣评论\\day2\\data\\stoplist.txt', 'r', encoding='utf-8') as f:
stop = f.read()
stop = stop.split() # 去掉空格那些
stop = stop + ['\n']
data_after = data_cut.apply(lambda x: [i for i in x if i not in stop])
from tkinter import _flatten
_flatten(list(data_after)) # 撕开
frequency = pd.Series(_flatten(list(data_after))).value_counts()
pic = plt.imread('D:\\工程实践\\1.2豆瓣评论\\day2\\data\\aixin.jpg')
wc = WordCloud(
mask=pic,
font_path='C:/Windows/Fonts/simhei.ttf',
background_color='white'
)
wc.fit_words(frequency)
plt.axis('off')
plt.imshow(wc)
plt.show()
3.最后结果如下~~
看看大家的关注点在哪呢?
4.感想
熟悉运用pandas实在是太太太重要了,里面的各种方法实在是好用的不行!!