干货满满，注意休息。
import pandas as pdimport numpy as npfrom sklearn.metrics import roc_auc_scorefrom datetime import date#读入数据data = pd.read_csv('./Combined_News_DJIA.csv')
#我们可以先把数据给分成Training/Testing datatrain = data[data['Date'] < '2015-01-01']test = data[data['Date'] > '2014-12-31']#columns和.index两个属性返回数据集的列索引和行索引X_train = train[train.columns[2:]]X_train
#flatten转变数据为一维数组就会得到list of sentences。astype强制类型转换#同时我们的X_train和X_test可不能随便flatten，他们需要与y_train和y_test对应corpus = X_train.values.flatten().astype(str)X_train = X_train.values.astype(str)X_train = np.array([' '.join(x) for x in X_train])X_test = test[test.columns[2:]]X_test = X_test.values.astype(str)X_test = np.array([' '.join(x) for x in X_test])y_train = train['Label'].valuesy_test = test['Label'].valuesy_train
#tokenize分割单词from nltk.tokenize import word_tokenize#报错 Resource punkt not found.#运行 nltk.download('punkt')corpus = [word_tokenize(x) for x in corpus]X_train = [word_tokenize(x) for x in X_train]X_test = [word_tokenize(x) for x in X_test]
# 数字import redef hasNumbers(inputString): return bool(re.search(r'\d', inputString))# 特殊符号def isSymbol(inputString): return bool(re.match(r'[^\w]', inputString))# lemmafrom nltk.stem import WordNetLemmatizerwordnet_lemmatizer = WordNetLemmatizer()def check(word): """ 如果需要这个单词，则True 如果应该去除，则False """ word= word.lower() if word in stop: return False elif hasNumbers(word) or isSymbol(word): return False else: return True# 把上面的方法综合起来def preprocessing(sen): res = [] for word in sen: if check(word): # 这一段的用处仅仅是去除python里面byte存str时候留下的标识。。之前数据没处理好，其他case里不会有这个情况 word = word.lower().replace("b'", '').replace('b"', '').replace('"', '').replace("'", '') res.append(wordnet_lemmatizer.lemmatize(word)) return res corpus = [preprocessing(x) for x in corpus]X_train = [preprocessing(x) for x in X_train]X_test = [preprocessing(x) for x in X_test]
训练NLP模型
有了这些干净的数据集，我们可以做我们的NLP模型了。
我们先用最简单的Word2Vec
from gensim.models.word2vec import Word2Vecmodel = Word2Vec(corpus, size=128, window=5, min_count=5, workers=4)
用NLP模型表达我们的X
接着，我们于是就可以用这个坐标，来表示我们的之前干干净净的X。
但是这儿有个问题。我们的vec是基于每个单词的，怎么办呢？
由于我们文本本身的量很小，我们可以把所有的单词的vector拿过来取个平均值：
# 先拿到全部的vocabularyvocab = model.wv.vocab# 得到任意text的vectordef get_vector(word_list): # 建立一个全是0的array res =np.zeros([128]) count = 0 for word in word_list: if word in vocab: res += model[word] count += 1 return res/count #我们得到了一个取得任意word list平均vector值wordlist_train = X_trainwordlist_test = X_testX_train = [get_vector(x) for x in X_train]X_test = [get_vector(x) for x in X_test]print(X_train[10])
建立ML模型
这里，因为我们128维的每一个值都是连续关系的。不是分裂开考虑的。所以，道理上讲，我们是不太适合用RandomForest这类把每个column当做单独的variable来看的方法。
from sklearn.svm import SVRfrom sklearn.model_selection import cross_val_scoreparams = [0.1,0.5,1,3,5,7,10,12,16,20,25,30,35,40]test_scores = []for param in params:a clf = SVR(gamma=param) test_score = cross_val_score(clf, X_train, y_train, cv=3, scoring='roc_auc') test_scores.append(np.mean(test_score))import matplotlib.pyplot as plt%matplotlib inlineplt.plot(params, test_scores)plt.title("Param vs CV AUC Score");
用CNN来提升逼格
用vector表示出一个大matrix，并用CNN做“降维+注意力”
(下面内容我会把整个case搞得简单点。要是想更加复杂准确的话，直接调整参数，往大了调，就行)
首先，我们确定一个padding_size。
就是为了让我们生成的matrix是一样的size啊
# vec_size 指的是我们本身vector的sizedef transform_to_matrix(x, padding_size=256, vec_size=128): res = [] for sen in x: matrix = [] for i in range(padding_size): try: matrix.append(model[sen[i]].tolist()) except: # 这里有两种except情况， # 1. 这个单词找不到 # 2. sen没那么长 # 不管哪种情况，我们直接贴上全是0的vec matrix.append([0] * vec_size) res.append(matrix) return resX_train = transform_to_matrix(wordlist_train)X_test = transform_to_matrix(wordlist_test)print(X_train[123])
可以看到，现在我们得到的就是一个大大的Matrix，它的size是 128 * 256
每一个这样的matrix，就是对应了我们每一个数据点
在进行下一步之前，我们把我们的input要reshape一下。
原因是我们要让每一个matrix外部“包裹”一层维度。来告诉我们的CNN model，我们的每个数据点都是独立的。之间木有前后关系。
# 搞成np的数组，便于处理X_train = np.array(X_train)X_test = np.array(X_test)# 看看数组的大小print(X_train.shape)print(X_test.shape)X_train = X_train.reshape(X_train.shape[0], 1, X_train.shape[1], X_train.shape[2])X_test = X_test.reshape(X_test.shape[0], 1, X_test.shape[1], X_test.shape[2])print(X_train.shape)print(X_test.shape)
from keras.preprocessing import sequencefrom keras.models import Sequentialfrom keras.layers import Convolution2D, MaxPooling2Dfrom keras.layers.core import Dense, Dropout, Activation, Flattenfrom keras import backend as K# K.set_image_dim_ordering("th")K.image_data_format() == 'channels_last'# K.set_image_data_format('channels_last')# set parameters:batch_size = 32n_filter = 16filter_length = 4nb_epoch = 5n_pool = 2# 新建一个sequential的模型model = Sequential()model.add(Convolution2D(n_filter,filter_length,filter_length, input_shape=(1, 256, 128)))model.add(Activation('relu'))model.add(Convolution2D(n_filter,filter_length,filter_length))model.add(Activation('relu'))model.add(MaxPooling2D(pool_size=(n_pool, n_pool)))model.add(Dropout(0.25))model.add(Flatten())# 后面接上一个ANNmodel.add(Dense(128))model.add(Activation('relu'))model.add(Dropout(0.5))model.add(Dense(1))model.add(Activation('softmax'))# compile模型model.compile(loss='mse', optimizer='adadelta', metrics=['accuracy'])model.fit(X_train, y_train, batch_size=batch_size, nb_epoch=nb_epoch, verbose=0)score = model.evaluate(X_test, y_test, verbose=0)print('Test score:', score[0])print('Test accuracy:', score[1])
PS：TO Sparkling
This is the back of my hand.
This is the back of my foot.
And you're at the back of my heart.
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
