python爬虫第二天

时间字符串转换

contents获取内容

strftime转化时间格式

内文的提取 实参位置用空格分隔 加一级的标签

读取评论数和评论内容（json）

import requests

comments = requests.get('评论所在地址（自行寻找）')

import json

jd = json.loads(comments.text.strip('需剔除部分'))

抓取内文信息方法写成函数：

commentURL = '网{}址'   #挖空newsid的网址

 

import re

import json

def getCommentCounts(newsurl):

    m = re.search('doc-i(.+).shtml',newsurl)

    newsid = m.group(1)

    comments = requests.get(commentURL.format(newsid))

    jd = json.loads(comments.text.strip('var data='))

    return jd['result']['count']['total']

 

import requests

from bs4 import BeautifulSoup

def getNewsDetail(newsurl):

    result = {}

    res = resquest.get(newsurl)

    res.encoding = 'utf - 8'

    soup = BeautifulSoup(res.text,'html.parser')

    result['title'] = soup.select('#artibodyTitle')[0].text

    result['newssource'] = soup.select('.time-sourse span a')[0].text

    timesource = soup.select('.time-source')[0]countents[0].strip()

    result['dt'] = datetime.striptime(timesource,'%Y年%m月%d日%H:%M')

    result['article'] = ' '.join([p.text.strip() for p in soup.select('#artibody p')[:-1]])

    result['comments'] = getCommentcount(newsurl)

    return result

就会得到一个新闻的字典信息，设计循环可以实现多条新闻的爬取