提取ngram

import re
def extract_ngram(all_sentences, min_feq=0):
	# all_sentences = [['迈','向','充','满','希','望','的','新','世','纪','-','-','一','九','九','八','年',...],[...],...]
	n_gram_dict = {}
	new_all_sentences = []
	for sen in all_sentences:
		str_sen = ''.join(sen)
		new_sen = re.split(u'[^\u4e00-\u9fa50-9a-zA-Z]', str_sen)
		for s in new_sen:
			if len(s) > 0:
				new_all_sentences.append(s) #二维列表转换为一维列表
	for sentence in new_all_sentences:
		for i in range(len(sentence)):
			for n in range(1, 6):
				if i + n > len(sentence):
					break
				n_gram = ''.join(sentence[i:i+n])
				if n_gram not in n_gram_dict:
					n_gram_dict[n_gram] = 1
				else:
					n_gram_dict[n_gram] += 1
	new_ngram_dict = {gram: c for gram, c in n_gram_dict.items() if c > min_feq}
	return new_ngram_dict