每天定时用爬虫爬取新闻发送给三爷

最新推荐文章于 2022-04-24 22:20:06 发布

少侠好走位

最新推荐文章于 2022-04-24 22:20:06 发布

阅读量858

点赞数 1

分类专栏： python 文章标签： python

python 专栏收录该内容

8 篇文章 0 订阅

订阅专栏

需求：

1、获取前十条科技要闻

2、通过邮件发送给指定收件人

3、定时发送

1、获取要闻，选的是新浪科技网https://tech.sina.com.cn/，爬取前十名要闻

import requests
from requests import exceptions
from urllib.request import urlopen
from bs4 import BeautifulSoup
import re

# 获取前十条科技新闻
def get_tech_news():
    # 获取网页文本
    resp = urlopen('https://tech.sina.com.cn/')
    # 使用爬虫工具BeautifulSoup查找需要的文本内容
    soup = BeautifulSoup(resp, 'html.parser')
    news = soup.find('ul', class_='rank-con')
    news_list = []
    for new in news:
        if len(new.string) > 5:
            news_list.append(new.string.replace(' ', '，'))
    # 组装邮件文本，<br>换行
    news_contents = f'''
    今日科技要闻<br>
    1、{news_list[0]}<br>
    2、{news_list[1]}<br>
    3、{news_list[2]}<br>
    4、{news_list[3]}<br>
    5、{news_list[4]}<br>
    6、{news_list[5]}<br>
    7、{news_list[6]}<br>
    8、{news_list[7]}<br>
    9、{news_list[8]}<br>
    10、{news_list[9]}<br>
    详情，请进入https://tech.sina.com.cn 查看
    '''.replace('\t', '')
    print(news_contents)
    send_email("今日科技要闻", news_contents)

2、定义发送邮件函数，跟上次一样，同级目录需要config.yml配置文件

# config.yml配置文件
email:
    sender: 'mikasama007@163.com'
    receiver: 'mikasama007@163.com'
    smtpserver: 'smtp.163.com'
    username: 'mikasama007'
    password: '邮箱密码'

import time
import yaml
import smtplib
import sys
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from email.mime.base import MIMEBase
from email.header import Header


def get_conf():
    with open ("config.yml", "r", encoding='utf-8') as f:
        cfg = f.read()
        dic = yaml.load(cfg)
        # print(type(dic))
        # print(dic)
        sender = dic['email']['sender']
        receiver = dic['email']['receiver']
        smtpserver = dic['email']['smtpserver']
        username = dic['email']['username']
        password = dic['email']['password']
        # print(sender, receiver, smtpserver, username, password)
        return sender, receiver, smtpserver, username, password

def send_email(title, text):
    today = time.strftime('%Y.%m.%d',time.localtime(time.time()))
    sender, receiver, smtpserver, username, password = get_conf()
    # subject为邮件主题 text为邮件正文
    subject = "{}:{}".format(title, today)
    msg = MIMEText(text, 'html', 'utf-8')
    msg['Subject'] = subject
    msg['From'] = sender
    msg['To'] = "".join(receiver)
    # smtp = smtplib.SMTP()
    # smtp.connect(smtpserver)
    # 变量名不能为stmp，改为server
    # windows上端口为25，linux上使用SMTP_SSL
    # server = smtplib.SMTP(smtpserver, 25)
    # https://blog.csdn.net/zekdot/article/details/81013176
    server = smtplib.SMTP_SSL(smtpserver, 465)
    server.set_debuglevel(0)
    server.login(username, password)
    server.sendmail(sender, receiver, msg.as_string())
    server.quit()

3、引入schedule库，定时发送邮件

import schedule

# 每天早上8点发送邮件通知
schedule.every().day.at("08:00").do(do_all_get_tech_news)

# 确保schedule一直运行
while True:
    schedule.run_pending()
    time.sleep(1)

4、丢服务器上去
运行python send_news.py

居然发不了邮件？换个地方就不行？

将登陆模式改为SSL即可：

server = smtplib.SMTP_SSL(smtpserver, 465)

修改后，可以正常发送邮件，cool！

但是这样可不行，ssh连接关闭，程序肯定会退出的，我们需要后台运行：

nohup python send_news.py &

查看下：

# root @ www in /home/cool_things/send_news [23:06:35] C:1
$ nohup python send_news.py &
[1] 24713

# root @ www in /home/cool_things/send_news [23:06:45] 
$ nohup: ignoring input and appending output to ‘nohup.out’


# root @ www in /home/cool_things/send_news [23:06:50] 
$ ps -ef | grep python
root       724     1  0 Apr01 ?        00:25:52 /usr/bin/python -Es /usr/sbin/tuned -l -P
root     10357     1  0 Apr01 ?        00:03:20 /usr/bin/python -Es /usr/sbin/firewalld
root     23391     1  0 Sep01 ?        00:10:10 python send_msg_to_wx.py
root     24713 21231  1 23:06 pts/1    00:00:00 python send_news.py

# root @ www in /home/cool_things/send_news [23:06:58] 
$ ps -ef | grep send_news
root     24713 21231  0 23:06 pts/1    00:00:00 python send_news.py

搞定，程序已经在后台运行，从明天开始，三爷每天早上8点就能收到每日要闻邮件了。

全部代码：

# !/usr/bin/env python
# -*- coding:utf-8 -*-

import requests
from requests import exceptions
from urllib.request import urlopen
from bs4 import BeautifulSoup
import re
import schedule
import time
import yaml
import smtplib
import sys
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from email.mime.base import MIMEBase
from email.header import Header


def get_conf():
    with open ("config.yml", "r", encoding='utf-8') as f:
        cfg = f.read()
        dic = yaml.load(cfg)
        # print(type(dic))
        # print(dic)
        sender = dic['email']['sender']
        receiver = dic['email']['receiver']
        smtpserver = dic['email']['smtpserver']
        username = dic['email']['username']
        password = dic['email']['password']
        # print(sender, receiver, smtpserver, username, password)
        return sender, receiver, smtpserver, username, password

def send_email(title, text):
    today = time.strftime('%Y.%m.%d',time.localtime(time.time()))
    sender, receiver, smtpserver, username, password = get_conf()
    # subject为邮件主题 text为邮件正文
    subject = "{}:{}".format(title, today)
    msg = MIMEText(text, 'html', 'utf-8')
    msg['Subject'] = subject
    msg['From'] = sender
    msg['To'] = "".join(receiver)
    # smtp = smtplib.SMTP()
    # smtp.connect(smtpserver)
    # 变量名不能为stmp，改为server
    # windows上端口为25，linux上使用SMTP_SSL
    # server = smtplib.SMTP(smtpserver, 25)
    # https://blog.csdn.net/zekdot/article/details/81013176
    server = smtplib.SMTP_SSL(smtpserver, 465)
    server.set_debuglevel(0)
    server.login(username, password)
    server.sendmail(sender, receiver, msg.as_string())
    server.quit()

# 获取前十条科技新闻
def get_tech_news():
    # 获取网页文本
    resp = urlopen('https://tech.sina.com.cn/')
    # 使用爬虫工具BeautifulSoup查找需要的文本内容
    soup = BeautifulSoup(resp, 'html.parser')
    news = soup.find('ul', class_='rank-con')
    news_list = []
    for new in news:
        if len(new.string) > 5:
            news_list.append(new.string.replace(' ', '，'))
    # 组装邮件文本，<br>换行
    news_contents = f'''
    今日科技要闻<br>
    1、{news_list[0]}<br>
    2、{news_list[1]}<br>
    3、{news_list[2]}<br>
    4、{news_list[3]}<br>
    5、{news_list[4]}<br>
    6、{news_list[5]}<br>
    7、{news_list[6]}<br>
    8、{news_list[7]}<br>
    9、{news_list[8]}<br>
    10、{news_list[9]}<br>
    详情，请进入https://tech.sina.com.cn 查看
    '''.replace('\t', '')
    print(news_contents)
    send_email("今日科技要闻", news_contents)

# 调试
news = get_tech_news()

def do_all_get_tech_news():
    try:
        get_tech_news()
    except exceptions as e:
        print(e)
        time.sleep(10)
        get_tech_news()

# 每天早上8点发送邮件通知
schedule.every().day.at("08:16").do(do_all_get_tech_news)

# 确保schedule一直运行
while True:
    schedule.run_pending()
    time.sleep(1)