import
smtplib
from
email.mime.text
import
MIMEText
from
email.utils
import
formataddr
status
=
{}
# 状态码
pipeline_item
=
{
"item"
:
0
,
# 成功item项
"item_error"
:
0
,
# 失败item项
"error_info"
: ""
# 失败原因
}
spider_time
=
{
"start"
: "
", "
end
": "
"}
# 开始结束时间
class
EmailHandler(
object
):
def
__init__(
self
, user, password, type_
=
0
):
"""
:param user:str 发送人邮箱地址(用户名)
:param password:str 发送人申请的授权码
:param type_:int 0 为QQ邮箱 1 为163邮箱
"""
self
.__QQ
=
{
'smtp'
:
'smtp.qq.com'
,
'port'
:
465
}
self
.__163
=
{
'smtp'
:
'smtp.163.com'
,
'port'
:
25
}
self
.user
=
user
self
.password
=
password
if
type_
=
=
0
:
self
.server
=
smtplib.SMTP_SSL(
self
.__QQ[
'smtp'
],
self
.__QQ[
'port'
])
self
.server.login(
self
.user,
self
.password)
elif
type_
=
=
1
:
self
.server
=
smtplib.SMTP_SSL(
self
.__163[
'smtp'
],
self
.__163[
'port'
])
self
.server.login(
self
.user,
self
.password)
def
send_mail(
self
, to, subject, content
=
None
):
"""
:param to:str 接收人邮箱地址
:param subject:str 邮件标题
:param content:str 邮件内容
:return:bool True 成功 False 失败
"""
try
:
if
not
content:
content
=
"\r\n"
for
key
in
status.keys():
content
+
=
"状态码%s:%s次\r\n"
%
(key, status.get(key))
content
+
=
"\r\n"
content
+
=
"存入数据库成功条数:%s 条\r\n存入数据库失败条数:%s 条\r\n"
%
(pipeline_item.get(
"item"
),
pipeline_item.get(
'item_error'
))
if
pipeline_item.get(
'item_error'
) >
0
:
content
+
=
"失败原因:%s"
%
pipeline_item.get(
'error_info'
)
content
+
=
"\r\n"
content
+
=
"爬虫启动时间:%s \r\n爬虫结束时间:%s"
%
(spider_time.get(
"start"
), spider_time.get(
"end"
))
msg
=
MIMEText(content,
'plain'
,
'utf-8'
)
msg[
'From'
]
=
formataddr(['',
self
.user])
msg[
'To'
]
=
formataddr(['', to])
msg[
'Subject'
]
=
subject
self
.server.sendmail(
self
.user, to, msg.as_string())
print
(
"【%s】邮件发送成功"
%
subject)
return
True
except
Exception as f:
print
(
"【%s】邮件发送失败,请检查信息"
%
subject)
return
False
from
xxx.emailHandler
import
*
def
process_response(
self
, request, response, spider):
# Called with the response returned from the downloader.
# Must either;
# - return a Response object
# - return a Request object
# - or raise IgnoreRequest
if
response.status
in
status.keys():
status[response.status]
+
=
1
else
:
status[response.status]
=
0
return
response
from
xxx.middlewares
import
pipeline_item
def
process_item(
self
, item, spider):
client
=
MongoClient(
self
.host,
self
.port)
db_auth
=
client.xiaosidb
db_auth.authenticate(
self
.user,
self
.passwd)
db
=
client[
self
.db]
table
=
db[
self
.table]
# data = dict(item)
try
:
table.insert(
dict
(item))
pipeline_item[
'item'
]
+
=
1
except
Exception as e:
pipeline_item[
'item_error'
]
+
=
0
pipeline_item[
'error_info'
]
=
e
return
item
from
xxx.middlewares
import
*
def
start_requests(
self
):
spider_time[
'start'
]
=
time.strftime(
'%Y-%m-%d %H:%M:%S'
, time.localtime(time.time()))
def
close(spider, reason):
spider_time[
'end'
]
=
time.strftime(
'%Y-%m-%d %H:%M:%S'
, time.localtime(time.time()))
to_sender
=
'xxxx'
# 要发送的邮箱
mail
=
EmailHandler(user
=
'xxxx'
, password
=
'xxx'
)
mail.send_mail(to_sender,
'邮件标题'
)
读书 是最美的修行
往期 精彩回顾最新时讯 | 更多内容萃取时光 | 光影凝固
思想视界 | 心灵驿站