本文改自
［网络爬虫｜smtp协议｜python］东方财富网爬虫，python smtp协议发送爬取数据至QQ邮箱
之前写的爬虫单子，代码已经跑了快3个月了，后续又增加了一些需求，修改了一些小bug
如果需要学习的话，上面的链接文章有分析过程，涉及的一些参数，上面博文也有说明
改动：
- 修复了爬虫过程中，qq邮箱发送邮件频率过快，帐号暂时封禁导致的程序异常退出
- 修复了程序定时显示时间错误问题
- 增加了股票涨幅限制参数
- 取消爬取重复数据发送
- 限制主力资金范围
- 剔除股票代码开头号（如，不需要300，688开头的）
- 剔除股票名称包含关键词（如，不需要包含’ST’的股票）
- 修复了已知问题
所有的参数修改，均 main 函数下面修改
问题解释：
- 发送qq邮箱可添加多个邮箱，在 senders中的列表里添加，第一个参数是授权码，2为邮箱。关于获取授权码及其它一系列问题，均在上述博文中说明过，本文不再重复
- 无
- 股票涨幅限制为范围限制。如果不需要，可设置参数范围较大，或自行修改程序代码。注释掉参数，就会报错
- 获取的每个股票数据，只发送一次
- 主力资金范围同3
- 剔除股票开头，可设置多个。如果不需要，可以删除列表里面的股票代码开头。当输入多个开头时，请保证所有开头的长度一致。如[300, 688]正确，[300, 68]错误。股票代码开头以第一个参数长度为准。长度不同会造成股票无法限制，但不会报错
- 股票名称关键词，如果不需要，可删除内容，但不要注释掉
- 修复已知问题，代码能持续运行，捕获异常。如果您发现程序未知错误导致异常退出，请在评论区指出。
# -*- coding: UTF-8 -*-
import random
import sys
import pyperclip
import requests
import re
import json
import time
import pandas as pd
import smtplib
from email.header import Header
from email.mime.text import MIMEText
from email.mime.application import MIMEApplication
from email.mime.multipart import MIMEMultipart
'''
需求：
http://data.eastmoney.com/zjlx/detail.html
前50，主力占比16.4以上，涨幅4.4以上，超大单占比9.4以上，成交量占前五日平均值的的0.6-1.1，换手率不超8
参数我可以自行修改，出现过的就不用再出现了，邮件或其他方式提醒，实时的。
'''
'''
参数说明：
f2: 最新价 f3: 今日涨跌幅 f12: 股票代码 f14: 股票名称
今日主力净流入：
f62: 净额 f184
今日超大单净流入：
f66: 净额 f69: 净占比
今日大单净流入：
f72: 净额 f75: 净占比
今日中单净流入：
f78: 净额 f81: 净占比
今日小单净流入：
f84: 净额 f87: 净占比
f168: 换手率
f47: 成交量
f124
'''
'''
f47数据不对 113行
'''
# requests-exceptions-sslerror : dh key too small
requests.packages.urllib3.util.ssl_.DEFAULT_CIPHERS += 'HIGH:!DH:!aNULL'
try:
requests.packages.urllib3.contrib.pyopenssl.DEFAULT_SSL_CIPHER_LIST += 'HIGH:!DH:!aNULL'
except AttributeError:
# no pyopenssl support used / needed / available
pass
# disable warnings
requests.packages.urllib3.disable_warnings()
class Variable:
def __init__(self, zhuli, zhangfu, chaodadan, chengjiaoliang:list, huanshoulv, num, codehead, zhuliarea, namehead):
self.zhuli = zhuli
self.zhangfu = zhangfu
self.chaodadan = chaodadan
self.chengjiaoliang = chengjiaoliang
self.huanshoulv = huanshoulv
self.num = num
self.all = []
self.head = codehead
self.namehead = namehead
self.zhuliarea = zhuliarea
# self.now = time.localtime().tm_mon*30 + time.localtime().tm_mday # 取近五日的，不用判断时间
self.ua = {"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.54 Safari/537.36",
# "Host":"push2.eastmoney.com",
"Connection":"keep-aliv",
"Cache-Control":"max-age=0"}
class Spider:
def __init__(self, Vb:Variable):
self.url = "http://data.eastmoney.com/zjlx/detail.html"
self.HomePageUrl = "http://push2.eastmoney.com/api/qt/clist/get?cb=jQuery112304456859063185632_1636024376116&fid=f62&po=1&pz={pz}&pn=1&np=1&fltt=2&invt=2&ut=b2884a393a59ad64002292a3e90d46a5&fs=m%3A0%2Bt%3A6%2Bf%3A!2,m%3A0%2Bt%3A13%2Bf%3A!2,m%3A0%2Bt%3A80%2Bf%3A!2,m%3A1%2Bt%3A2%2Bf%3A!2,m%3A1%2Bt%3A23%2Bf%3A!2,m%3A0%2Bt%3A7%2Bf%3A!2,m%3A1%2Bt%3A3%2Bf%3A!2&fields=f12,f14,f2,f3,f62,f184,f66,f69,f72,f75,f78,f81,f84,f87,f204,f205,f124,f1,f13"
self.klineApi = "http://push2his.eastmoney.com/api/qt/stock/kline/get?fields1=f1,f2,f3,f4,f5&fields2=f56&fqt=0&end=29991010&ut=fa5fd1943c7b386f172d6893dbfba10b&cb=jQuery1123038436182619159487_1636971507184&klt=101&secid={f13}.{f12}&fqt=1&lmt=6"
self.f47Api = "http://push2.eastmoney.com/api/qt/stock/get?fltt=2&invt=2&secid={f13}.{f12}&fields=f47,f168,f43,f46&ut=b2884a393a59ad64002292a3e90d46a5&cb=jQuery11230690409531564167_1636970393200"
self.vb = Vb
self.namelist = []
self.retry = 0
self.copy = False
def HomePageGet(self): # 获取主页数据
while True:
response = self.request(self.HomePageUrl.format(pz=self.vb.num))
if isinstance(response, bool):
time.sleep(10)
else:
break
# 获取 data 列表
return self.dataGet(response)['diff']
def request(self, url):
while True:
try:
response = requests.get(url, headers=self.vb.ua, verify=False, timeout=2)
self.vb.retry = 0
return response
except:
self.vb.retry += 1
if self.vb.retry == 3:
self.vb.retry = 0
return False
# 前50，主力占比16.4以上，涨幅4.4以上，超大单占比9.4以上,换手率不超8%，成交量占前五日平均值的0.6-1.1
# 成交量不能筛选，得通过后续请求
# 初次筛选
def FirstClean(self, diff:list): # 主页数据清洗
nowdata = []
# 空数据
if diff[0]['f184'] == '-':
return nowdata
try:
headlength = len(str(self.vb.head[0]))
except:
headlength = False
for i in diff:
if i['f184']>self.vb.zhuli: # 主力占比
if i['f3']>self.vb.zhangfu[0] and i['f3']<self.vb.zhangfu[1]: # 涨幅
if i['f69']>self.vb.chaodadan: # 超大单
# 股票开头和股票名称
if headlength and self.vb.namehead!=None:
if (int(i['f12'][:headlength]) not in self.vb.head) and self.vb.namehead not in i['f14']:
nowdata.append(i)
elif headlength:
if int(i['f12'][:headlength]) not in self.vb.head:
nowdata.append(i)
elif self.vb.namehead!=None:
if self.vb.namehead not in i['f14']:
nowdata.append(i)
else:
nowdata.append(i)
try:
now = []
for i in nowdata:
if i['f62']>self.vb.zhuliarea[0] and i['f62']<self.vb.zhuliarea[1]:
now.append(i)
return now
except:
return nowdata
# 筛选成交量占前五日平均值的 0.6-1.1，需要请求前五日的
'''
成交量api：https://push2his.eastmoney.com/api/qt/stock/kline/get?fields1=f1&fields2=f55&fqt=0&end=29991010&klt=101&secid={f13}.{f12}&lmt=5
'''
# 筛选占前五日0.6-1.1
# HomePageUrl 请求不到 f47 数据，需要单独请求
def SecondClean(self, data:list):
# 请求 k 线图数据，需要更改 secid,
# secid 由两部份组成, f13.f12
result = []
for line in data:
response = self.request(self.klineApi.format(f13=line['f13'], f12=line['f12']))
if not response:
continue
klines = self.dataGet(response)['klines']
kline = []
# for k in klines:
# kline.append(float(k))
try:
for i in range(5):
kline.append(float(klines[i]))
except:
continue
fiveday = sum(kline)/5
# f47
response = self.request(self.f47Api.format(f13=line['f13'], f12=line['f12']))
if not response:
continue
d = self.dataGet(response)
if d['f168']/100>=self.vb.huanshoulv:
continue
today = d['f47']
if not (d['f43']>d['f46']):
print("跳过")
continue
# 成交量占前 5 日平均值的 0.6-1.1
value = today/fiveday
if (value>=self.vb.chengjiaoliang[0] and value<=self.vb.chengjiaoliang[1]):
# 股票代码，名称，最新价，今日涨跌幅，今日主力净流入，占比，换手率，现价，开盘价
result.append([line['f12'], line['f14'], line['f2'], str(line['f3'])+"%", self.Format(line['f62']), str(line['f184'])+"%", str(line['f69'])+'%', "%.2f" % (d['f168']/100) +"%", '%.2f'%value, d['f43'], d['f46']])
return result
# 数据判断是否需要保存
def JudgeSave(self, data:list):
line1 = [['股票代码', '名称', '最新价', '今日涨跌幅', '今日主力净流入', '今日主力占比', '超大单占比', '换手率', '比值', "现价", "开盘价"]]
result = []
# 筛选不重复的保存
for line in data:
if line[1] not in self.namelist:
self.namelist.append(line[1])
result.append(line)
# self.namelist.append(line[1])
# line.insert(0, count)
# result.append(line)
# count += 1
result.sort(key=self.s, reverse=True)
if len(result)!=0:
if not self.copy:
pyperclip.copy(result[0][0])
self.copy = True
print("_"*88)
for i in result:
print(f"\033[31m{i[0]}\033[0m", end="\t")
for b in range(1, len(i)):
print(i[b], end="\t")
print("|")
print("￣"*55)
# 保存文件
self.vb.all = self.vb.all+result
excelData = pd.DataFrame(line1+self.vb.all)
writer = pd.ExcelWriter("股票.xlsx")
excelData.to_excel(excel_writer=writer, columns=None, index=None, header=False)
writer.close()
return True
else:
return False
# 返回 data 字典
def dataGet(self, response):
d = re.compile('"data":({.*)}\);')
data = re.findall(d, response.text)[0]
data = json.loads(data)
return data
def Format(self, data): # 换成万或者亿
if len(str(data))>8: # 换成亿
return '%.2f' % (data/(10**8)) + '亿'
elif len(str(data))==8:
return '%.2f' % (data / (10 ** 7)) + '千万'
elif len(str(data))>4:
return '%.2f' % (data / (10 ** 4)) + '万'
else:
return str(data)
def s(self, elem):
return elem[-4]
def run(self):
self.data = self.HomePageGet()
self.data = self.FirstClean(self.data)
self.data = self.SecondClean(self.data)
return self.JudgeSave(self.data.copy())
class QQEmail:
def __init__(self, senders, senderName, receiver, receiverName):
self.senders = senders
self.senderName = senderName
self.receiver = receiver
self.receiverName = receiverName
self.MailLogin()
# 创建smtp，登录
def MailLogin(self):
self.accoutGet()
try:
self.server = smtplib.SMTP_SSL(host='smtp.qq.com', port=465)
response = self.server.login(self.sender, self.qqCode)
if response[0] == 235:
print("邮箱登录成功！")
except Exception as e:
print("邮箱登录失败！", e)
def accoutGet(self):
try:
self.qqCode, self.sender = random.choice(self.senders.copy().remove((self.qqCode, self.sender)))
except:
self.qqCode, self.sender = random.choice(self.senders)
def MessageBuild(self):
content = MIMEText(' ') # 邮件正文内容，需要可填写
message = MIMEMultipart()
message.attach(content) # 附上正文内容
message['From'] = self.senderName
message['To'] = self.receiverName
message['Subject'] = Header('股票', 'utf-8') # 标题
xlsx = MIMEApplication(open('股票.xlsx', 'rb').read())
xlsx['Content-Type'] = 'application/octet-stream'
xlsx.add_header('Content-Disposition', 'attachment', filename='股票.xlsx')
message.attach(xlsx)
return message
def SendMail(self):
retry = 0
while True:
try:
self.server.sendmail(from_addr=self.sender, to_addrs=self.receiver, msg=self.MessageBuild().as_string())
print('已发送！ ', end="")
break
except smtplib.SMTPException as e:
print("邮箱登录异常，正在重试！", e)
self.MailLogin()
retry += 1
if retry == 2:
print("邮箱登录异常！五分钟后重试")
time.sleep(300)
retry = 0
class Control(Spider, QQEmail):
def __init__(self, zhuli, zhangfu, chaodadan, chengjiaoliang, huanshoulv, senders, senderName, receiver, recieverName, num, codehead, zhuliarea, namehead):
# 变量类
self.vb = Variable(zhuli, zhangfu, chaodadan, chengjiaoliang, huanshoulv, num, codehead, namehead, zhuliarea)
# 初始化邮箱类
self.qqmail = QQEmail(senders, senderName, receiver, recieverName)
# 爬虫类
self.spider = Spider(self.vb)
def main(self, stop, end):
print("爬虫启动")
count = 1
while True:
send = self.spider.run()
if send:
print("获取最新股票数据！", end="")
self.qqmail.SendMail()
print(count, "\n\n")
count += 1
else:
pass
time.sleep(stop)
if time.localtime().tm_hour == end[0]:
if time.localtime().tm_min == end[1]:
break
def getTime(t:str):
return [int(a) for a in t.split(":")]
if __name__ == '__main__':
# 配置参数
zhuli = 3
zhangfu = [2.5, 9.88]
chaodadan = 2
chengjiaoliang = [0.2, 10]
# chengjiaoliang = [1, 10]
huanshoulv = 0.7
num = 95
stop = 1
剔除股票开头 = [30,68]
剔除名称 = 'ST'
主力净流入范围 = [10000000, 300000000]
# 配置邮箱
senders = [("授权码", "邮箱"),
("授权码2", "邮箱2")]
receiver = '接收邮箱'
senderName = '发送名'
receiverName = '接收者'
# 启动时间
start = '22:06'
end = '15:00'
# 准时启动
start = getTime(start)
end = getTime(end)
# 这段是启动时间，如果注释掉就不限制时间了
while True:
print("距离启动还差：{}分钟".format((-time.localtime().tm_hour+start[0])*60-time.localtime().tm_min+start[1]))
if time.localtime().tm_hour == start[0]:
if time.localtime().tm_min == start[1]:
break
time.sleep(10)
# 启动
control = Control(zhuli, zhangfu, chaodadan, chengjiaoliang, huanshoulv, senders, senderName, receiver, receiverName, num, 剔除股票开头, 剔除名称, 主力净流入范围)
control.main(stop, end)
扫一扫
1833
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
