网络学习笔记
commentURL = 'http://comment5.news.sina.com.cn/page/info?version=1&format=js&channel=gn&newsid=comos-{}&group=&compress=0&ie=utf-8&oe=utf-8&page=1&page_size=20'
commentURL.format(newsid)
import re
import json
news = 'http://news.sina.com.cn/c/nd/2016-08-20/doc-ifxvctcc8121090.shtml'
def getCommentCount(newsurl):
m = re.search('doc-i(.*).shtml',newsurl)
newsid = m.group(1)
comments = requests.get(commentURL.format(newsid))
jd = json.loads(comments.text.strip('var data='))
return jd['result']['count']['total']
def getNewsDetail(newsurl):
result = {}
res = requests.get(newsurl)
res.encoding = 'utf-8'
soup = BeautifulSoup(res.text, 'html.parser')
result['title'] = soup.select('#artibodyTitle')[0].text
result['newssource'] = soup.select('.time-source span a')[0].text
timesource = soup.select('.time-source')[0].contents[0].strip()
result['dt'] = datetime.strptime(timesource, '%Y年%m月%d日%H:%M')
result['article'] = ' '.join([p.text.strip() for p in soup.select('#artibody p')[:-1]])
result['editor'] = soup.select('.article-editor')[0].text.lstrip('责任编辑:')
result['comments'] = getCommentCount(newsurl)
return result
getNewsDetail('http://news.sina.com.cn/c/nd/2016-08-20/doc-ifxvctcc8121090.shtml')
{'title': '环保部门不再核发机动车环保合格标志', 'newssource': '新华网', 'dt': datetime.datetime(2016, 8, 20, 22, 21), 'article': '原标题:挡风玻璃“减负”:环保部门不再核发机动车环保合格标志 新华社北京8月20日电 记者日前从环境保护部获悉,环保部、公安部、国家认监委联合发布《关于进一步规范排放检验加强机动车环境监督管理工作的通知》,进一步规范机动车排放检验。通知明确,环保部门不再核发机动车环保检验合格标志,以后汽车挡风玻璃上的年检标志将从三个变成两个。', 'editor': '瞿崑 SN117', 'comments': 916}