爬取新闻列表

获取单条新闻的#标题#链接#时间#来源#内容 #点击次数，并包装成一个函数

获取一个新闻列表页的所有新闻的上述详情，并包装成一个函数

获取所有新闻列表页的网址，调用上述函数

完成所有校园新闻的爬取工作

import requests
from bs4 import BeautifulSoup
from datetime import datetime
import re

net = 'http://news.gzcc.cn/html/xiaoyuanxinwen/'
res = requests.get(net)
res.encoding='utf-8'
doc = BeautifulSoup(res.text,'html.parser',from_encoding='utf-8')

def getclick(newsurl):
    id = re.search('_(.*).html',newsurl).group(1).split('/')[1]
    clickurl = 'http://oa.gzcc.cn/api.php?op=count&id={}&modelid=80'.format(id)
    click = int(requests.get(clickurl).text.split('.')[-1].lstrip("html('").rstrip("');"))
    return click

count = int(doc.select(".a1")[0].text.rstrip("条"))
page = count//10

print('共',page,'页')

for news in doc.select('li'):
    if len(news.select('.news-list-title'))>0:
        title = news.select('.news-list-title')[0].text
        url = news.select('a')[0]['href']
        time = news.select('.news-list-info')[0].contents[0].text
        main = news.select('.news-list-description')[0].text
        source = news.select('.news-list-info')[0].contents[1].text

        resd = requests.get(url)
        resd.encoding='utf-8'
        docd = BeautifulSoup(resd.text,'html.parser',from_encoding='utf-8')
        detail = docd.select('.show-content')[0].text

        click = getclick(url)
        
    
        print('Url：{}'.format(news.select('a')[0]['href']))
        print('标题：{}'.format(news.select('.news-list-title')[0].text))
        print('正文：{}'.format(news.select('.news-list-description')[0].text))
        print('时间：{}'.format(news.select('.news-list-info')[0].contents[0].text))
        print('来源：{}'.format(news.select('.news-list-info')[0].contents[1].text))
        print('点击次数：',click)
        print(detail)
        

 

 

转载于:https://www.cnblogs.com/knight-hui/p/7649094.html