python新闻标题分类_Python收集标题,python,采集,头条,文章

Author:song

from multiprocessing import Pool

from urllib.parse import urlencode

import requests

import json

from requests import RequestException

from bs4 import BeautifulSoup

import re

import pymongo

client = pymongo.MongoClient(‘localhost’,connect=False)

db = client[‘toutiaowenzhang’]

def get_index(offset):

data = {

‘offset’: offset,

‘format’: ‘json’,

‘keyword’: ‘美文’,

‘autoload’: ‘true’,

‘count’: 20,

‘cur_tab’: 1,

‘from’:‘search_tab’

}

url = ‘https://www.toutiao.com/search_content/?’+urlencode(data)

response = requests.get(url)

try:

if response.status_code == 200:

return response.text

else:

return None

except RequestException:

return None

def get_urls(html):

data = json.loads(html)

if data and ‘data’ in data.keys():

for item in data.get(‘data’):

yield item.get(‘article_url’)

def get_index_detail(url):

response = requests.get(url)

try:

if response.status_code == 200:

return response.text

else:

return None

except RequestException:

return None

def parse_detail(html):

try:

soup = BeautifulSoup(html,‘lxml’)

title = soup.select(‘title’)[0].get_text()

compile_allarticle= re.compile(‘content.

?

?)

’,re.S)

allarticle = re.findall(compile_allarticle,html)

# article =re.sub(’(<.*?)’,’’,allarticle[0])#正则匹配上不需要的那部分

article =re.sub(’[a-zA-Z0-9/#;&._]’,’’,str(allarticle)).strip()#直接把字母数字全部替换

data = {

‘title’:title,

‘article’:article

}

return data

except TypeError:#解决出现了404界面

pass

def save_to_mongodb(result):

if db[‘toutiaowenzhang’].insert(result):

print(‘successful’)

else:

print(‘fail’)

def main(offset):

html = get_index(offset)

items = get_urls(html)

for item in items:

if item:

ab = get_index_detail(item)

result = parse_detail(ab)

save_to_mongodb(result)

if

name

==‘

main

’:

groups = [x*20 for x in range(3)]

pool = Pool()

pool.map(main,groups)