python类型的操作

1、list

     list连接

    list = list1 +list2

sen_list = []
for i in tqdm(data.index):
    sen_list += data.coreEmotions_list[i]
    
pd.value_counts(sen_list)

    list交集

2、dict

3、pandas

dataframe apply使用两列操作

def difflib_similarity(str1,str2):
    return difflib.SequenceMatcher(a=str1, b=str2).quick_ratio()

data['prefix_title_sim']=data[['prefix','title']].apply(lambda row: difflib_similarity(row[0],row[1]),raw=True,axis=1)

apply raw = True速度会快很多

def list_intersection(listA,listB):
    retB = list(set(listA).intersection(set(listB)))
    return retB

df['query_title'] = df.apply(lambda x: list_intersection(x['query'],x['title']),axis=1)
df

找出空值的行 

press_train[press_train.isnull().values==True]

4、numpy

返回最大k值的下标

arr = result.argsort()[0][::-1][:top_k]

5、groupby 操作

df = data.groupby('query_id')['title_length'].agg(['min','mean','max','sum','std','median','count'])
data = pd.merge(data,df,how='left',on='query_id')

df = data.groupby(['query_id'])['title_length'].rank().reset_index()
df.rename(columns={'title_length':"rank"},inplace=True)
data = pd.concat([data,df[['rank']]],axis=1)

cols = ['title_length','query_title_length','q_len/t_len','qt_len/q_len','qt_len/t_len','tfidf_cos','doc_cos_qt']

for c in cols:
    df = data.groupby('query_id')[c].agg(['min','mean','max','sum','std','median','count'])
    df.columns = [ c+'_'+col for col in df.columns]
    data = pd.merge(data,df,how='left',on='query_id')
    
    df = data.groupby(['query_id'])[c].rank().reset_index()
    df.rename(columns={c:c+'_'+'rank'},inplace=True)
    data = pd.concat([data,df[[c+'_'+'rank']]],axis=1)

def dict_join(df):
    df = df[['itemID','rating']]
    df.set_index(['itemID'],inplace=True)
    df = df.to_dict(orient="dict")
    return df['rating']

df_train = df_train.groupby('uid').apply(dict_join).reset_index()

d = {'query_id': [1,1,1, 2,2,3,3], 'y_true': [0,1,0,1,0,1,0], 'y_pred': [0.2,0.6,0.6,0.7,0.4,0.8,0.3]}
df = pd.DataFrame(data=d)

from sklearn.metrics import roc_auc_score

def q_auc(x):
    
    x['auc_i'] = roc_auc_score(x['y_true'],x['y_pred'])
    
    return x

df = df.groupby(['query_id']).apply(q_auc)
df = df[['query_id','auc_i']].drop_duplicates()
np.mean(df['auc_i'])

for c in X.columns:
    col_type = X[c].dtype
    if col_type == 'object' or col_type.name == 'category':
        X[c] = X[c].astype('category')


for c in categorical_feats:
    train[categorical_feats] = train[categorical_feats].astype('category')