Python使用newspaper3k对文章进行标题图片关键词文本等提取

1. 效果图

首图如下：

2. 安装及报错解决

python 3.7.4

pip install newspaper3k==0.2.8

3. 源码

import newspaper
from newspaper import Config

sina_paper = newspaper.build('http://www.sina.com.cn/', language='zh')

for category in sina_paper.category_urls():
    print(category)
# http://health.sina.com.cn
# http://eladies.sina.com.cn
# http://english.sina.com

user_agent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.4 Safari/605.1.15'
user_agent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36'

config = Config()
config.browser_user_agent = user_agent

article = sina_paper.articles[0]
article.config = config
print(len(article.url), article.url)
if (article.url.find("http") > 4): article.url = 'http' + article.url.split("http")[2]

print(len(article.url), article.url)
article.url = article.url.replace(' ', '')
print(len(article.url), article.url)
article.download()
article.parse()

print('text: ', article.text)
print('title: ', article.title)
print('publish_date: ', article.publish_date)
print('tags: ', article.tags)
print('top_image: ', article.top_image)
print('top_img: ', article.top_img)
print('url: ', article.url)
print('meta_keywords: ', article.meta_keywords)
print('meta_img: ', article.meta_img)
print('keywords: ', article.keywords)
print('images:', len(article.images), article.images)
print('imgs', len(article.imgs), article.imgs)


# url = "https://www.newsweek.com/new-mexico-compound-charges-dropped-children-1096830".strip()
#
# content = []
# for i in [article.url]:
#     article = Article(str(i), config=config)
#     try:
#         article.download()
#         article.parse()
#         article.download()
#         print(article.html)
#         article.parse()
#         text = article.text
#         print(i, text)
#         content.append(text)
#     except Exception as e:
#         print(e)
#         content.append('error')
#     continue
# print(content)

参考

• https://pypi.org/project/newspaper3k/
• https://medium.com/free-code-camp/an-a-z-of-useful-python-tricks-b467524ee747