粗糙的短文本抽取

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import jieba

df = pd.read_csv("/Users/magu/Documents/新闻分析/凤凰网新闻1.csv")
papers = list(df[df.columns[-1]])


def sorted_dict(items_,item_id = 1):
    return sorted(items_.items(),key=lambda x:x[item_id],reverse=True)


def extract_key_text(paper,index=3):
    cut_words = list(jieba.cut(paper))
    token_words = [word for word in cut_words if word not in 
    ## TODO : 过滤词停用词条列表
    [ "\n",' ', '“', '”', '。','了','，','：','的','我','你',"他","她","是"]
    and len(word) > 1
    ]
    unique_words,counter = np.unique(token_words,return_counts=token_words)
    #sort_the_words_ = sorted(dict(zip(unique_words,counter)).items(),key=lambda x:x[1],reverse=True)
    sort_the_words_ = sorted_dict(dict(zip(unique_words,counter)))
    sorted_words = {item[0]:item[-1] for item in sort_the_words_}

    expr = {}
    temp_words = [] 
    for word in cut_words:
        temp_words.append(word)
        if word in ['，','。','？','！']:
            expr.update({sum(sorted_words[token] for token in temp_words if token in sorted_words):"".join(temp_words)})
            temp_words = []
    return sorted_dict(expr,item_id=0)[:index+1],sorted_words,paper

        

            




print(extract_key_text(papers[8]))