前言
利用正则表达式进行信息提取和文本分析,完成百度新闻、搜狗新闻、新浪财经的数据挖掘。一、获取多家公司的百度新闻并生成数据报告
#批量获取多家公司的百度新闻
from urllib import request,parse
import random
import time
from fake_useragent import UserAgent
import re
class BaiduNewsSpider(object):
def __init__(self):
self.url = 'https://www.baidu.com/s?tn=news7rtt=1&bsst=1&cl=2&wd={}'
#添加计数
self.i = 0
# 请求
def get_html(self,url):
headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.150 Safari/537.36'}
req = request.Request(url=url, headers=headers)
res = request.urlopen(req)
html = res.read().decode()
# 获取完之后直接调用解析函数
return html
# 解析
def parse_html(self,html):
# re_bds = '<div class="movie-item-info">.*?title="(.*?)".*?class="star">(.*?)</p>.*?releasetime">(.*?)</p>'
# pattern = re.compile(re_bds,re.S)
# r_list = pattern.findall(html)
# self.write_html(r_list)
pass
# 保存
def write_html(self,filename,html):
with open(filename,'w') as f:
f.write(html)
self.i += 1
# 主函数
def run(self):
companys = ['华能信托','阿里巴巴','百度集团','万科集团','腾讯','京东']
for i in companys:
try:
wd = parse.quote(i)
url = self.url.format(wd)
html = self.get_html(url)
filename = '{}"百度新闻".html'.format(i)
self.write_html(filename, html)
# 随机休眠
time.sleep(random.uniform(1,2))
print(i + '百度新闻爬取成功')
except:
print(i + '百度新闻爬取失败')
print('数量:', self.i)
if __name__=='__main__':
start = time.time()
spider = BaiduNewsSpider()
spider.run()
end = time.time()
二、批量获取多家公司多页的百度新闻
1.批量获取数据
代码如下(示例):
#批量获取多家公司多页的百度新闻
from urllib import request,parse
import random
import time
import re
class BaiduNewsSpider(object):
def __init__(self):
self.url = 'https://www.baidu.com/s?tn=news7rtt=4&bsst=1&cl=2&wd={}&pn={}'
# 按时间排序https://www.baidu.com/s?tn=news&rtt=4&bsst=1&cl=2&wd=%E4%B8%87%E7%A7%91%E9%9B%86%E5%9B%A2&medium=0
#添加计数
self.i = 0
# 请求
def get_html(self,url):
headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.150 Safari/537.36'}
req = request.Request(url=url, headers=headers)
res = request.urlopen(req)
html = res.read().decode()
# 获取完之后直接调用解析函数
return html
# 解析
def parse_html(self,html):
# #r_list:[('月光宝盒','周星驰','1994'),(),()
# re_bds = '<div class="movie-item-info">.*?title="(.*?)".*?class="star">(.*?)</p>.*?releasetime">(.*?)</p>'
# pattern = re.compile(re_bds,re.S)
# r_list = pattern.findall(html)
# # 直接调用写入函数
# self.write_html(r_list)
pass
# 保存
def write_html(self,filename,html):
with open(filename,'w') as f:
f.write(html)
self.i += 1
# 主函数
def run(self):
companys = ['华能信托','阿里巴巴','百度集团','万科集团','腾讯','京东']
for company in companys:
wd = parse.quote(company)
for i in range(1, 4):
pn = (i - 1) * 10
url = self.url.format(wd, pn)
html = self.get_html(url)
filename = '{}百度新闻-第{}页.html'.format(company, i)
self.write_html(filename, html)
print('第%d页抓取成功' % i)
# 每爬取1个页面随机休眠1-3秒
time.sleep(random.randint(1, 3))
print(company + '百度新闻爬取成功')
print('数量:', self.i)
if __name__=='__main__':
start = time.time()
spider = BaiduNewsSpider()
spider.run()
end = time.time()
2.数据清洗
代码如下(示例):
import requests
import re
headers = {'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.150 Safari/537.36'}
url = 'https://www.sogou.com/sogou?query=%E9%98%BF%E9%87%8C%E5%B7%B4%E5%B7%B4&interation=1728053249&pid=sogou-wsse-9fc36fa768a74fa9&ie=utf8&w=&sut=6046&sst0=1612509795700&lkt=1%2C1612509795594%2C1612509795594'
# timeout访问超时设置,访问超过指定时间就报出异常,访问结束
res = requests.get(url, headers=headers,timeout=10).text
# 正则表达式获取数据
re_bds = '<h3 class="vr-title">.*?<a id="(.*?)".*?href="(.*?)">(.*?)</a>.*?<div class="text-layout">.*?<span>.*?</span><span>(.*?)</span>'
pattern = re.compile(re_bds, re.S)
r_list = pattern.findall(res)
# 数据清洗
item = {}
for r in r_list:
item['id'] = re.sub('<.*?>', '', r[0].strip())
item['网址'] = re.sub('<.*?>', '', r[1].strip())
item['标题'] = re.sub('<.*?>', '', r[2].strip())
item['发布时间'] = re.sub('<.*?>', '', r[3].strip())
print(item)
print('*' * 50)
3.将1.2.合并
代码如下(示例):
#批量获取多家公司多页的百度新闻并清洗数据
from urllib import request,parse
import requests
import random
import time
import re
headers = {
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.150 Safari/537.36'}
def Sougou(company):
url = 'https://www.sogou.com/sogou?ie=utf8&p=40230447&interation=1728053249&interV=&pid=sogou-wsse-8f646834ef1adefa&query=' + company + 'shid=djt1'
url = url.format(parse.quote(company))
res = requests.get(url, headers=headers, timeout=10).text
#print(res)
# 编写正则表达式提取数据
p_title = '<h3 class="vr-title">.*?<a id=".*?".*?href=".*?">(.*?)</a>'
title = re.compile(p_title, re.S).findall(res)
p_href = '<h3 class="vr-title">.*?<a id=".*?".*?href="(.*?)">'
href = re.compile(p_href, re.S).findall(res)
p_date = '<h3 class="vr-title">.*?<span>.*?</span><span>(.*?)</span>'
date = re.compile(p_date, re.S).findall(res)
print(title,href,date)
# 数据清洗及打印输出
for i in range(len(title)):
title[i] = re.sub('<.*?>','',title[i])
title[i] = re.sub('&.*?;', '', title[i])
date[i] = re.sub('<.*?>', '', date[i])
print(str(i+1) + '.' + title[i] +'-' + date[i])
print(href[i])
companys = ['华能信托','阿里巴巴','百度集团','万科集团','腾讯','京东']
for i in companys:
Sougou(i)
# 每爬取1个页面随机休眠1-3秒
time.sleep(random.randint(1, 3))
print(i + '百度新闻爬取成功')
三、批量获取多家公司的新浪财经新闻
#批量获取多家公司多页的新浪财经新闻并清洗数据---这里公司名称不能直接用中文
from urllib import request,parse
import requests
import random
import time
import re
headers = {
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.150 Safari/537.36'}
def Sina(company):
url = 'https://search.sina.com.cn/?q={}&c=news&from=channel'
url = url.format(parse.quote(company))
res = requests.get(url, headers=headers, timeout=10).text
# print(res)
# 编写正则表达式提取数据
p_title = '<h2><a href=.*?target=".*?">(.*?)</a>'
title = re.compile(p_title, re.S).findall(res)
p_href = '<h2><a href="(.*?)">'
href = re.compile(p_href, re.S).findall(res)
p_date = '<h2><a href=.*?<span class="fgray_time">(.*?)</span></h2>'
date = re.compile(p_date, re.S).findall(res)
#print(title,href,date)
# 数据清洗及打印输出
for i in range(len(title)):
title[i] = re.sub('<.*?>','',title[i])
title[i] = re.sub('&.*?;', '', title[i])
date[i] = re.sub('<.*?>', '', date[i])
print(str(i+1) + '.' + title[i] +'-' + date[i])
print(href[i])
companys = ['华能信托','阿里巴巴','百度集团','万科集团','腾讯','京东']
for i in companys:
Sina(i)
# 每爬取1个页面随机休眠1-3秒
time.sleep(random.randint(1, 3))
print(i + '新浪财经新闻爬取成功')