"""
采用正则表达式:
(该例子改为:爬取百度新闻)
"""
import requests
import re
class NeiHan:
# 定义属性,基础
def __init__(self):
# 设置url地址
# self.url = "https://movie.douban.com/tv/#!type=tv&tag=%E7%BE%8E%E5%89%A7&sort=recommend&page_limit=20&page_start=0"
self.url = "https://news.baidu.com/"
# 设置headers
self.headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)"
" AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/54.0.2840.99 Safari/537.36"
}
# 设置代理ip
self.proxies = {
"http": "http://180.121.133.237:808"
# 也可以多个
}
# 设置post请求
# 发送请求,获取爬取的初始内容
def parse_url_get_context(self,url):
# 发送请求
response = requests.get(url,headers=self.headers,proxies=self.proxies)
print("paser_url(self)运行完毕")
# print("response.content.decode()的类型为:%s" % type(response.content.decode()))
# return response.content.decode('gbk')
return response.content.decode()
def run(self): # 主要执行方法(主要实现逻辑)
# 1.start_url(发送请求,获取初始数据)
html_str = self.parse_url_get_context(self.url)
# print(html_str)
print("*"*20)
# 2.根据获取的数据,完成正则表达式匹配
# 在实际过程中,我们会碰到一些html标签中必须存在的“双引号”。
# 通过“\”转义字符
# ret = re.findall("<div class=\"life_left_ul \">(.*)</div>",html_str,re.S)
# ret = re.search("<div class=\"pl2\">.*<a.*>.*</a>", html_str, re.S)
# ret = re.findall("<span class=\"dot\">(.*)</span><a .*/>(.*)<li>", html_str, re.S)
ret = re.findall("(</span>\n|<li>\n)<a (.*?)>(.*?)</a>", html_str, re.S)
# print(ret.group())
print(type(ret))
print(len(ret))
print(ret)
print("*" * 20)
print()
# 循环遍历
for find in ret:
print(find)
find_tuple = find[2]
print(find_tuple)
print()
print("*" * 20)
# 3.提取并保存数据(提取“专属信息”并保存至“专属文件”)
# 5.构造下一页的url地址,进入循环
if __name__ == '__main__':
nahan = NeiHan()
nahan.run()
print("--运行完成--")