pyhton爬虫（8）——获取网易新闻内容

本文主要目的是获取网易新闻标题和正文内容。实现代码如下所示：

# -*- coding: utf-8 -*-
"""
Created on Mon Jul 17 15:46:30 2017

@author: Administrator
"""
from bs4 import BeautifulSoup
import urllib.request
import http.cookiejar

#url = 'http://news.163.com/17/0717/10/CPHORRIE0001899O.html'
url = 'http://news.163.com/17/0717/16/CPIES9NG000187V9.html'

'''
1.将网易新闻页面以html的形式保存到本地
'''

#以字典的形式设置headers

headers = {"Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
           "Accept-Language": "zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3",
           "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:54.0) Gecko/20100101 Firefox/54.0",
           "Connection": "keep-alive",
           "referer": "http://www.163.com/"}

#设置cookie
cjar = http.cookiejar.CookieJar()
proxy = urllib.request.ProxyHandler({'http':"127.0.0.1:8888"})
opener = urllib.request.build_opener(proxy,urllib.request.HTTPHandler,urllib.request.HTTPCookieProcessor(cjar))

#建立空列表，为了以指定格式存储头信息
headall = []
#通过for循环遍字典，构造出指定格式的Headers信息
for key,value in headers.items():
    item = (key,value)
    headall.append(item)

#将指定格式的headers信息添加好
opener.addheaders = headall

#将opener安装为全局
urllib.request.install_opener(opener)
data = urllib.request.urlopen(url).read()
fhandle = open("D:/python/data/163/1.html","wb")
fhandle.write(data)
fhandle.close()

'''
2.提取网易新闻标题和正文内容信息
'''

html1 = urllib.request.urlopen(url).read().decode('gbk')
html1 = str(html1)

soup1 = BeautifulSoup(html1,'lxml')
#提取新闻标题
result1 = soup1.find_all("h1")
title = result1[0].string
print("新闻标题为:{}".format(title))

soup2 = BeautifulSoup(html1,'lxml')
#提取正文所在区块
result2 = soup1.find_all(attrs={"class":"post_text"})
result2 = str(result2)
#print(result2)
soup3 = BeautifulSoup(html1,'lxml')
#提取正文文本内容
result3= soup1.find_all("p")

content = result3[5:8]
print("新闻正文内容为:")
for i in content:
    print(i.string)

实现结果如下图所示：

本文只实现了网易新闻内容的简单提取，但正文信息提取时还需要手动设置区间范围，不够灵活，还有待进一步完善。