Datawhale

比赛内容此次比赛，达观数据提供了一批长文本数据和分类信息，希望选手动用自己的智慧，结合当下最先进的NLP和人工智能技术，深入分析文本内在结构和语义信息，构建文本分类模型，实现精准分类。评分规则评分算法 binary-classification评分标准 采用各个品类F1指标的算术平均值，它是Precision 和 Recall 的调和平均数。 其中，Pi是表示第i个种类对...

#import pickleimport pandas as pdimport numpy as npimport timepath='D:/daguanbei/data/proba'print('1 读取概率+投票')svm_1 = pd.read_csv(path+'/result_proba_svm_0.779.csv')svm_2 = pd.read_csv(path+...

import osfrom collections import Counterdef read_dir_by_filter(root_dir,filter): file_list = [] for root,dirs,files in os.walk(root_dir): for filepath in files: if(filte...

"""将tfidf(word)和tfidf(article)拼接成新的特征"""import pickleimport pandas as pdfrom sklearn.feature_extraction.text import TfidfVectorizer"""=========================================================...

"""将tfidf(word+article)特征降维为lsa特征，并将结果保存至本地，并将结果保存到本地"""from sklearn.decomposition import TruncatedSVDimport pickleimport timet_start = time.time()"""=======================================...

'''特征拼接，拼接文章长度#1.载入特征#2 读文章长度#3 特征缩放，拼接特征'''import pickle# 载入特征with open('tfidf(word+article).pkl','rb') as f: x_train,y_train,y_test = pickle.load(f)# 读取文章长度信息import pandas as pd impo...

"""将linearsvm挑选的tfidf(word)特征和lr挑选的tfidf(article)"""import picklefrom scipy import sparsefrom scipy.sparse import hstackwith open('linearsvm-tfidf(word).pkl', 'rb')as f_1: x_train_1, y_tr...

"""将countvector(word)、hash(word)和doc2vec(word)拼接成新特征"""import picklefrom scipy import sparsefrom scipy.sparse import hstack

"""1.特征：linearsvm-tfidf(word)+lr-tfidf(article) / doc2vec_word2.模型：bayes"""from sklearn.calibration import CalibratedClassifierCVfrom sklearn.model_selection import StratifiedKFoldimport pandas..

countvector(a)+doc(a)+hash(a)"""1.特征：countvector(a)+doc(a)+hash(a)2.模型：lgb"""import numpy as npimport pandas as pdfrom sklearn.model_selection import train_test_splitimport pickleimport light...

特征：tfidf(word+article)"""1.特征：tfidf(word+article)2.模型：lr3.参数：C=120"""import pandas as pdimport picklefrom sklearn.linear_model import LogisticRegressionwith open('tfidf(word+article).pkl', ...

'''将原始数据的word特征数字化为countvector特征，并将结果保存到本地article特征可做类似处理'''import pandas as pdfrom sklearn.feature_extraction.text import countvectororizerimport pickleimport timet_start = time.time()""...

'''将原始数据的word特征数字化为tfidf特征，并将结果保存到本地article特征可做类似处理'''import pandas as pdfrom sklearn.feature_extraction.text import TfidfVectorizerimport pickleimport timet_start = time.time()"""======...

'''将原始数据的word特征数字化为doc2vec特征，并将结果保存到本地article特征可做类似处理'''import pandas as pdimport numpy as npfrom gensim.models.doc2vec import Doc2Vec, TaggedDocumentimport timeimport picklet_start = time...

'''将原始数据的word特征数字化为hash特征，并将结果保存到本地article特征可做类似处理'''import pandas as pdfrom sklearn.feature_extraction.text import HashingVectorizerimport pickleimport timet_start = time.time()"""======...

'''将原始数据的word的长度特征，并将结果保存到本地article特征可做类似处理'''df_train=pd.read_csv('train_set.csv')df_test=pd.read_csv('test_set.csv')def get_word_len(df_series): word_len=[] for row in df_series: wo...

lsa"""将tfidf(word)特征降维为lsa特征，并将结果保存至本地，并将结果保存到本地tfidf(article)可做类似处理"""from sklearn.decomposition import TruncatedSVDimport pickleimport timet_start = time.time()&a

linearsvm"""用linearsvm从tfidf(word)中挑选特征，并将结果保存到本地tfidf(article)可做类似处理"""import timeimport picklefrom sklearn.feature_selection import SelectFromModelfrom sklearn.svm import LinearSVCt_sta...

特征：tfidf(word)+tfidf(article)"""1.特征：tfidf(word)+tfidf(article)2.模型：linearsvm3.参数：C=5"""from sklearn.svm import LinearSVC # 支持向量机from sklearn.calibration import CalibratedClassifierCVimport p...

达观杯比赛简介特征工程1.构建特征countvectororizerTfidfVectorizerDoc2VecHashingVectorizerlength降维lsa 和 ldaSelectFromModel2.组合特征tfidf(word+article)tfidf(word+article)+lsatfidf(word+article+length)lin...