代码如下。我也放在我的GitHub里面。
已经加了注释，运行时候会有warning 信息。其他可以看那边框架解释文章。
# encoding: UTF-8
import warnings
warnings.filterwarnings("ignore")
from pymongo import MongoClient, ASCENDING
import pandas as pd
import numpy as np
from datetime import datetime
import talib
import matplotlib.pyplot as plt
import scipy.stats as st
from sklearn.model_selection import train_test_split
# LogisticRegression 逻辑回归
from sklearn.linear_model import LogisticRegression
# DecisionTreeClassifier 决策树
from sklearn.tree import DecisionTreeClassifier
# SVC 支持向量分类
from sklearn.svm import SVC
# MLP 神经网络
from sklearn.neural_network import MLPClassifier
from sklearn.model_selection import GridSearchCV
class DataAnalyzerforSklearn(object):
"""
这个类是为了SVM做归纳分析数据，以未来6个bar的斜率线性回归为判断分类是否正确。
不是直接分析HLOC，而且用下列分非线性参数（就是和具体点位无关）
1.Percentage
2.std
4.MACD
5.CCI
6.ATR
7. 该bar之前的均线斜率
8. RSI
"""
def __init__(self, exportpath="C:\\Project\\", datformat=['datetime', 'high', 'low', 'open', 'close','volume']):
self.mongohost = None
self.mongoport = None
self.db = None
self.collection = None
self.df = pd.DataFrame()
self.exportpath = exportpath
self.datformat = datformat
self.startBar = 2
self.endBar = 12
self.step = 2
self.pValue = 0.015
#-----------------------------------------导入数据-------------------------------------------------
def db2df(self, db, collection, start, end, mongohost="localhost", mongoport=27017, export2csv=False):
"""读取MongoDB数据库行情记录，输出到Dataframe中"""
self.mongohost = mongohost
self.mongoport = mongoport
self.db = db
self.collection = collection
dbClient = MongoClient(self.mongohost, self.mongoport, connectTimeoutMS=500)
db = dbClient[self.db]
cursor = db[self.collection].find({'datetime':{'$gte':start, '$lt':end}}).sort("datetime",ASCENDING)
self.df = pd.DataFrame(list(cursor))
self.df = self.df[self.datformat]
self.df = self.df.reset_index(drop=True)
path = self.exportpath + self.collection + ".csv"
if export2csv == True:
self.df.to_csv(path, index=True, header=True)
return self.df
def csv2df(self, csvpath, dataname="csv_data", export2csv=False):
"""读取csv行情数据，输入到Dataframe中"""
csv_df = pd.read_csv(csvpath)
self.df = csv_df[self.datformat]
self.df["datetime"] = pd.to_datetime(self.df['datetime'])
self.df = self.df.reset_index(drop=True)
path = self.exportpath + dataname + ".csv"
if export2csv == True:
self.df.to_csv(path, index=True, header=True)
return self.df
def df2Barmin(self, inputdf, barmins, crossmin=1, export2csv=False):
"""输入分钟k线dataframe数据，合并多多种数据，例如三分钟/5分钟等，如果开始时间是9点1分，crossmin = 0；如果是9点0分，crossmin为1"""
dfbarmin = pd.DataFrame()
highBarMin = 0
lowBarMin = 0
openBarMin = 0
volumeBarmin = 0
datetime = 0
for i in range(0, len(inputdf) - 1):
bar = inputdf.iloc[i, :].to_dict()
if openBarMin == 0:
openBarmin = bar["open"]
if highBarMin == 0:
highBarMin = bar["high"]
else:
highBarMin = max(bar["high"], highBarMin)
if lowBarMin == 0:
lowBarMin = bar["low"]
else:
lowBarMin = min(bar["low"], lowBarMin)
closeBarMin = bar["close"]
datetime = bar["datetime"]
volumeBarmin += int(bar["volume"])
# X分钟已经走完
if not (bar["datetime"].minute + crossmin) % barmins: # 可以用X整除
# 生成上一X分钟K线的时间戳
barMin = {'datetime': datetime, 'high': highBarMin, 'low': lowBarMin, 'open': openBarmin,
'close': closeBarMin, 'volume' : volumeBarmin}
dfbarmin = dfbarmin.append(barMin, ignore_index=True)
highBarMin = 0
lowBarMin = 0
openBarMin = 0
volumeBarmin = 0
if export2csv == True:
dfbarmin.to_csv(self.exportpath + "bar" + str(barmins)+ str(self.collection) + ".csv", index=True, header=True)
return dfbarmin
#-----------------------------------------开始计算指标-------------------------------------------------
def dfcci(self, inputdf, n, export2csv=True):
"""调用talib方法计算CCI指标，写入到df并输出"""
dfcci = inputdf
dfcci["cci"] = None
for i in range(n, len(inputdf)):
df_ne = inputdf.loc[i - n + 1:i, :]
cci = talib.CCI(np.array(df_ne["high"]), np.array(df_ne["low"]), np.array(df_ne["close"]), n)
dfcci.loc[i, "cci"] = cci[-1]
dfcci = dfcci.fillna(0)
dfcci = dfcci.replace(np.inf, 0)
if export2csv == True:
dfcci.to_csv(self.exportpath + "dfcci" + str(self.collection) + ".csv", index=True, header=True)
return dfcci
def dfatr(self, inputdf, n, export2csv=True):
"""调用talib方法计算ATR指标，写入到df并输出"""
dfatr = inputdf
for i in range((n+1), len(inputdf)):
df_ne = inputdf.loc[i - n :i, :]
atr = talib.ATR(np.array(df_ne["high"]), np.array(df_ne["low"]), np.array(df_ne["close"]), n)
dfatr.loc[i, "atr"] = atr[-1]
dfatr = dfatr.fillna(0)
dfatr = dfatr.replace(np.inf, 0)
if export2csv == True:
dfatr.to_csv(self.exportpath + "dfatr" + str(self.collection) + ".csv", index=True, header=True)
return dfatr
def dfrsi(self, inputdf, n, export2csv=True):
"""调用talib方法计算ATR指标，写入到df并输出"""
dfrsi = inputdf
dfrsi["rsi"] = None
for i in range(n+1, len(inputdf)):
df_ne = inputdf.loc[i - n :i, :]
rsi = talib.RSI(np.array(df_ne["close"]), n)
dfrsi.loc[i, "rsi"] = rsi[-1]
dfrsi = dfrsi.fillna(0)
dfrsi = dfrsi.replace(np.inf, 0)
if export2csv == True:
dfrsi.to_csv(self.exportpath + "dfrsi" + str(self.collection) + ".csv", index=True, header=True)
return dfrsi
def Percentage(self, inputdf, export2csv=True):
"""调用talib方法计算CCI指标，写入到df并输出"""
dfPercentage = inputdf
# dfPercentage["Percentage"] = None
for i in range(1, len(inputdf)):
# if dfPercentage.loc[i,"close"]>dfPercentage.loc[i,"open"]:
# percentage = ((dfPercentage.loc[i,"high"] - dfPercentage.loc[i-1,"close"])/ dfPercentage.loc[i-1,"close"])*100
# else:
# percentage = (( dfPercentage.loc[i,"low"] - dfPercentage.loc[i-1,"close"] )/ dfPercentage.loc[i-1,"close"])*100
if dfPercentage.loc[ i - 1, "close"] == 0.0:
percentage = 0
else:
percentage = ((dfPercentage.loc[i, "close"] - dfPercentage.loc[i - 1, "close"]) / dfPercentage.loc[ i - 1, "close"]) * 100.0
dfPercentage.loc[i, "Perentage"] = percentage
dfPercentage = dfPercentage.fillna(0)
dfPercentage = dfPercentage.replace(np.inf, 0)
if export2csv == True:
dfPercentage.to_csv(self.exportpath + "Percentage_" + str(self.collection) + ".csv", index=True, header=True)
return dfPercentage
def dfMACD(self, inputdf, n, export2csv=False):
"""调用talib方法计算MACD指标，写入到df并输出"""
dfMACD = inputdf
for i in range(n, len(inputdf)):
df_ne = inputdf.loc[i - n + 1:i, :]
macd,signal,hist = talib.MACD(np.array(df_ne["close"]),12,26,9)
dfMACD.loc[i, "macd"] = macd[-1]
dfMACD.loc[i, "signal"] = signal[-1]
dfMACD.loc[i, "hist"] = hist[-1]
dfMACD = dfMACD.fillna(0)
dfMACD = dfMACD.replace(np.inf, 0)
if export2csv == True:
dfMACD.to_csv(self.exportpath + "macd" + str(self.collection) + ".csv", index=True, header=True)
return dfMACD
def dfSTD(self, inputdf, n, export2csv=False):
"""调用talib方法计算MACD指标，写入到df并输出"""
dfSTD = inputdf
for i in range(n, len(inputdf)):
df_ne = inputdf.loc[i - n + 1:i, :]
std = talib.STDDEV(np.array(df_ne["close"]),n)
dfSTD.loc[i, "std"] = std[-1]
dfSTD = dfSTD.fillna(0)
dfSTD = dfSTD.replace(np.inf, 0)
if export2csv == True:
dfSTD.to_csv(self.exportpath + "dfSTD" + str(self.collection) + ".csv", index=True, header=True)
return dfSTD
#-----------------------------------------加入趋势分类-------------------------------------------------
def addTrend(self, inputdf, trendsetp=6, export2csv=False):
"""以未来6个bar的斜率线性回归为判断分类是否正确"""
dfTrend = inputdf
for i in range(1, len(dfTrend) - trendsetp-1):
histRe = np.array(dfTrend["close"])[i:i+trendsetp]
xAixs = np.arange(trendsetp) + 1
res = st.linregress(y=histRe, x=xAixs)
if res.pvalue < self.pValue+0.01:
if res.slope > 0.5:
dfTrend.loc[i,"tradeindictor"] = 1
elif res.slope < -0.5:
dfTrend.loc[i, "tradeindictor"] = -1
dfTrend = dfTrend.fillna(0)
dfTrend = dfTrend.replace(np.inf, 0)
if export2csv == True:
dfTrend.to_csv(self.exportpath + "addTrend" + str(self.collection) + ".csv", index=True, header=True)
return dfTrend
def GirdValuate(X_train, y_train):
"""1）LogisticRegression
逻辑回归
2）DecisionTreeClassifier
决策树
3）SVC
支持向量分类
4）MLP
神经网络"""
clf_DT=DecisionTreeClassifier()
param_grid_DT= {'max_depth': [1,2,3,4,5,6]}
clf_Logit=LogisticRegression()
param_grid_logit= {'solver': ['liblinear','lbfgs','newton-cg','sag']}
clf_svc=SVC()
param_grid_svc={'kernel':('linear', 'poly', 'rbf', 'sigmoid'),
'C':[1, 2, 4],
'gamma':[0.125, 0.25, 0.5 ,1, 2, 4]}
clf_mlp = MLPClassifier()
param_grid_mlp= {"hidden_layer_sizes": [(100,), (100, 30)],
"solver": ['adam', 'sgd', 'lbfgs'],
"max_iter": [20],
"verbose": [False]
}
#打包参数集合
clf=[clf_DT,clf_Logit,clf_mlp,clf_svc]
param_grid=[param_grid_DT,param_grid_logit,param_grid_mlp,param_grid_svc]
from sklearn.model_selection import StratifiedKFold # 交叉验证
kflod = StratifiedKFold(n_splits=10, shuffle=True, random_state=7) # 将训练/测试数据集划分10个互斥子集，这样方便多进程测试
#网格测试
for i in range(0,4):
grid=GridSearchCV(clf[i], param_grid[i], scoring='accuracy',n_jobs = -1,cv = kflod)
grid.fit(X_train, y_train)
print (grid.best_params_,': ',grid.best_score_)
if __name__ == '__main__':
# 读取数据
# exportpath = "C:\\Users\shui0\OneDrive\Documents\Project\\"
exportpath = "C:\Project\\"
DA = DataAnalyzerforSklearn(exportpath)
#数据库导入
start = datetime.strptime("20180501", '%Y%m%d')
end = datetime.strptime("20190501", '%Y%m%d')
df = DA.db2df(db="VnTrader_1Min_Db", collection="rb8888", start = start, end = end)
df5min = DA.df2Barmin(df, 5)
df5minAdd = DA.addTrend(df5min, export2csv=True)
df5minAdd = DA.dfMACD(df5minAdd, n=34, export2csv=True)
df5minAdd = DA.dfatr(df5minAdd, n=25, export2csv=True)
df5minAdd = DA.dfrsi(df5minAdd, n=35, export2csv=True)
df5minAdd = DA.dfcci(df5minAdd,n = 30,export2csv=True)
df5minAdd = DA.dfSTD(df5minAdd, n=30, export2csv=True)
df5minAdd = DA.Percentage(df5minAdd,export2csv = True)
#划分测试验证。
df_test = df5minAdd.loc[60:,:] #只从第60个开始分析，因为之前很多是空值
y= np.array(df_test["tradeindictor"]) #只保留结果趋势结果，转化为数组
X = df_test.drop(["tradeindictor","close","datetime","high","low","open","volume"],axis = 1).values #不是直接分析HLOC，只保留特征值，转化为数组
X_train, X_test, y_train, y_test = train_test_split(X,y,test_size=0.3,random_state=0) #三七
print("训练集长度: %s, 测试集长度: %s" %(len(X_train),len(X_test)))
from sklearn.feature_selection import SelectKBest
from sklearn.feature_selection import SelectPercentile
from sklearn.feature_selection import mutual_info_classif
#特征工作，可以按照百分比选出最高分特征类，取最优70%，也可以用SelectKBest，指定要几个特征类。
print(X_train.shape)
selectPer = SelectPercentile(mutual_info_classif, percentile=70)
# selectPer = SelectKBest(mutual_info_classif, k=7)
X_train = selectPer.fit_transform(X_train, y_train)
print(X_train.shape)
X_test = selectPer.transform(X_test)
# 也可以用Fpr选择
# selectFea=SelectFpr(alpha=0.01)
# X_train_new = selectFea.fit_transform(X_train, y_train)
# X_test_new = selectFea.transform(X_test)
# 这里使用下面模式进行分析，然后利用网格调参
GirdValuate(X_train,y_train)
# 使用选取最好的模型，进行测试看看拼接
# • 模型预测：model.predict()
# • Accuracy：metrics.accuracy_score()
# • Presicion：metrics.precision_score()
# • Recall：metrics.recall_score()
from sklearn import metrics
clf_selected=MLPClassifier(hidden_layer_sizes=(100,30), max_iter=20, solver='adam') #此处填入网格回测最优模型和参数,
# {'hidden_layer_sizes': (100, 30), 'max_iter': 20, 'solver': 'adam', 'verbose': False} : 0.9897016507648039
clf_selected.fit(X_train, y_train)
y_pred = clf_selected.predict(X_test)
#accuracy
accuracy=metrics.accuracy_score(y_true=y_test, y_pred=y_pred)
print ('accuracy:',accuracy)
#precision
precision=metrics.precision_score(y_true=y_test, y_pred=y_pred,average="micro")
print ('precision:',precision)
#recall
recall=metrics.recall_score(y_true=y_test, y_pred=y_pred,average="micro")
print ('recall:',recall)
#实际值和预测值
print (y_test)
print (y_pred)
dfresult = pd.DataFrame({'Actual':y_test,'Predict':y_pred})
dfresult.to_csv(exportpath + "result" + ".csv", index=True, header=True)
from sklearn.externals import joblib
#模型保存到本地
joblib.dump(clf_selected,'clf_selected.m')
#模型的恢复
clf_tmp=joblib.load('clf_selected.m')
运行结果：
训练集长度: 11673, 测试集长度: 5003
(11673, 8)
(11673, 5)
('accuracy:', '0.7833300019988008')
('precision:', '0.7833300019988008')
('recall:', '0.7833300019988008')
[ 1. 0. 0. ... 0. 0. -1.]
[0. 0. 0. ... 0. 0. 0.]
在vnpy中使用，简单说下, 在策略init 方法中使用 clf_tmp=joblib.load( 'clf_selected.m' ) 读取模型，然后在onXminBar方法中，
使用ArrayManager计算那些特征值，使用 clf_selected.predict()计算中预测分类，如果1开多单，-1空单，0略过。
来自 “ ITPUB博客 ” ，链接：http://blog.itpub.net/22259926/viewspace-2648828/，如需转载，请注明出处，否则将追究法律责任。
转载于:http://blog.itpub.net/22259926/viewspace-2648828/
8280
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
