2021-09-12

import warnings
warnings.filterwarnings("ignore")

import os

import logging
logging.basicConfig(level=logging.ERROR)

import pandas as pd
import numpy as np
from tqdm import tqdm
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.decomposition import TruncatedSVD
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import roc_auc_score
from sklearn.model_selection import StratifiedKFold
df_train = pd.read_table("train_data/gaiic_track3_round1_train_20210228.tsv", names=['q1', 'q2', 'label']).fillna("0")
df_test = pd.read_table('test_data/gaiic_track3_round1_testB_20210317.tsv', names=['q1', 'q2']).fillna("0")
label = df_train['label'].values
df = pd.concat([df_train, df_test], ignore_index=True)
df['text'] = df['q1'] + " " + df['q2']
tfidf = TfidfVectorizer(ngram_range=(1, 5))
tfidf_feature = tfidf.fit_transform(df['text'])
tfidf_feature.shape
svd_feature = TruncatedSVD(n_components=100).fit_transform(tfidf_feature)
train_df = tfidf_feature[:-len(df_test)]
test_df = tfidf_feature[-len(df_test):]
train_label=df['label'][:-len(df_test)]
from sklearn.model_selection import cross_val_score
from sklearn import metrics
from sklearn.model_selection import GridSearchCV

penaltys = ['l1','l2']
Cs = [0.1, 1, 10]
tuned_parameters = dict(penalty = penaltys, C = Cs)
 
lr_penalty= LogisticRegression()
grid= GridSearchCV(lr_penalty, tuned_parameters, cv=kf,scoring='roc_auc')
grid.fit(train_df,train_label)
grid.best_score_
grid.best_params_
best_lr=grid.best_estimator_
predictions=best_lr.predict(test_df)
pd.DataFrame(predictions).to_csv("result.csv", index=False, header=False)