使用sh脚本下载斯坦福工具 stanford.sh
wget http://nlp.stanford.edu/software/stanford-corenlp-full-2018-10-05.zip
unzip stanford-corenlp-full-2018-10-05.zip -d stanford-corenlp-full-2018-10-05
wget -P stanford-corenlp-full-2018-10-05 http://nlp.stanford.edu/software/stanford-english-corenlp-2018-10-05-models.jar
wget -P stanford-corenlp-full-2018-10-05 http://nlp.stanford.edu/software/stanford-chinese-corenlp-2018-10-05-models.jar
运行data_helper.py文件对数据集使用工具
import argparse
import re
from tqdm import tqdm
import os
from os import path
from collections import defaultdict
from math import log
from corenlp import StanfordCoreNLP
from nltk.tree import Tree
import json
from random import randint
FULL_MODEL = './stanford-corenlp-full-2018-10-05'
punctuation = ['。', '，', '、', '：', '？', '！', '（', '）', '“', '”', '【', '】']
chunk_pos = ['NP', 'PP', 'VP', 'ADVP', 'SBAR', 'ADJP', 'PRT', 'INTJ', 'CONJP', 'LST']
class Find_Words:
def __init__(self, min_count=10, max_count=10000000, min_pmi=0):
self.min_count = min_count
self.min_pmi = min_pmi
self.chars, self.pairs = defaultdict(int), defaultdict(int)
self.total = 0.
self.max_count = max_count
def text_filter(self, texts):
for a in tqdm(texts):
for t in re.split(u'[^\u4e00-\u9fa50-9a-zA-Z]+', a):
if t:
yield t
def count(self, texts):
mi_list = []
for text in self.text_filter(texts):
self.chars[text[0]] += 1
for i in range(len(text)-1):
self.chars[text[i+1]] += 1
self.pairs[text[i:i+2]] += 1
self.total += 1
self.chars = {i:j for i,j in self.chars.items() if 100 * self.max_count > j > self.min_count}
self.pairs = {i:j for i,j in self.pairs.items() if self.max_count > j > self.min_count}
self.strong_segments = set()
for i,j in self.pairs.items():
if i[0] in self.chars and i[1] in self.chars:
mi = log(self.total*j/(self.chars[i[0]]*self.chars[i[1]]))
mi_list.append(mi)
if mi >= self.min_pmi:
self.strong_segments.add(i)
print('min mi: %.4f' % min(mi_list))
print('max mi: %.4f' % max(mi_list))
print('remaining: %d / %d (%.4f)' % (len(self.strong_segments), len(mi_list), len(self.strong_segments)/len(mi_list)))
def find_words(self, texts, n):
self.words = defaultdict(int)
for text in self.text_filter(texts):
s = text[0]
for i in range(len(text)-1):
if text[i:i+2] in self.strong_segments:
s += text[i+1]
else:
self.words[s] += 1
s = text[i+1]
self.words = {i:j for i,j in self.words.items() if j > self.min_count and n+1 > len(i) > 1}
def read_txt(file_path):
sentence_list = []
label_list = []
with open(file_path, 'r', encoding='utf8') as f:
lines = f.readlines()
sentence = []
labels = []
for line in lines:
line = line.strip()
if line == '':
if len(sentence) > 0:
sentence_list.append(sentence)
label_list.append(labels)
sentence = []
labels = []
continue
items = line.split(" ")
character = items[0]
label = items[-1]
sentence.append(character)
labels.append(label)
return sentence_list, label_list
def get_word2id(data_dir):
word2id_path = path.join(data_dir, 'word2id.json')
word2count_path = path.join(data_dir, 'word2count.json')
word2id = {'<PAD>': 0}
word2count = {}
word = ''
index = 1
with open(path.join(data_dir, "train.txt"), 'r', encoding='utf8') as f:
for line in tqdm(f.readlines()):
line = line.strip()
if len(line) == 0:
continue
splits = line.split('\t')
character = splits[0]
word += character
if word not in word2id:
word2id[word] = index
word2count[word] = 1
index += 1
else:
word2count[word] += 1
word = ''
with open(word2id_path, 'w', encoding='utf8') as f:
json.dump(word2id, f, ensure_ascii=False)
f.write('\n')
with open(path.join(data_dir, 'word2id'), 'w', encoding='utf8') as f:
for w, v in word2id.items():
f.write('%s\t%d\n' % (w, v))
with open(word2count_path, 'w', encoding='utf8') as f:
json.dump(word2count, f, ensure_ascii=False)
f.write('\n')
with open(path.join(data_dir, 'word2count'), 'w', encoding='utf8') as f:
for w, v in word2count.items():
f.write('%s\t%d\n' % (w, v))
def change(char):
if "(" in char:
char = char.replace("(", "-LRB-")
if ")" in char:
char = char.replace(")", "-RRB-")
return char
def request_features_from_stanford(data_dir, flag):
all_sentences, _ = read_txt(path.join(data_dir, flag + '.txt'))
sentences_str = []
for sentence in all_sentences:
sentence = [change(i) for i in sentence]
# if sentence[-1] == '·':
# sentence[-1] = '.'
sentences_str.append(' '.join(sentence))
all_data = []
with StanfordCoreNLP(FULL_MODEL, lang='en', port=randint(38400, 38596)) as nlp:
for sentence in tqdm(sentences_str):
props = {
'timeout': '5000000',
'annotators': 'pos, parse, depparse',
'tokenize.whitespace': 'true',
'ssplit.eolonly': 'true',
'pipelineLanguage': 'en',
'outputFormat': 'json'}
results = nlp.annotate(sentence, properties=props)
# results = nlp.request(annotators='deparser', data=sentence)
# results = nlp.request(annotators='pos', data=sentence)
# result = results['sentences'][0]
all_data.append(results)
# assert len(all_data) == len(sentences_str)
with open(path.join(data_dir, flag + '.stanford.json'), 'w', encoding='utf8') as f:
for data in all_data:
json.dump(data, f, ensure_ascii=False)
f.write('\n')
def getlabels(data_dir):
_, train_labels = read_txt(path.join(data_dir, 'train.txt'))
_, test_labels = read_txt(path.join(data_dir, 'test.txt'))
all_labels = train_labels + test_labels
label2id = defaultdict(int)
for label_list in all_labels:
for label in label_list:
label2id[label] = 0
with open(path.join(data_dir, 'label2id'), 'w', encoding='utf8') as f:
for key in label2id.keys():
f.write(key)
f.write('\n')
class stanford_feature_processor:
def __init__(self, data_dir):
self.data_dir = data_dir
def read_json(self, data_path):
data = []
with open(data_path, 'r', encoding='utf8') as f:
lines = f.readlines()
for line in lines:
line = line.strip()
if line == '':
continue
data.append(json.loads(line))
return data
def _pre_processing(self):
all_data = self.read_json(path.join(self.data_dir, 'train.stanford.json'))
gram2count = defaultdict(int)
pos_tag2count = defaultdict(int)
chunk_tag2count = defaultdict(int)
dep_tag2count = defaultdict(int)
for data in all_data:
sentences_list = data['sentences']
for sentence_l in sentences_list:
tokens = sentence_l['tokens']
for token in tokens:
gram2count[token['originalText']] += 1
pos_tag2count[token['pos']] += 1
pos_tag2count[token['originalText'] + '_' + token['pos']] += 1
deparse = sentence_l['basicDependencies']
for word in deparse:
dep_tag2count[word['dep']] += 1
dep_tag2count[word['dependentGloss'] + '_' + word['dep']] += 1
coparse = Tree.fromstring(sentence_l['parse'])
for s in coparse.subtrees(lambda t: t.label() in chunk_pos):
leaves = s.leaves()
node = s.label()
chunk_tag2count[node] += 1
for leaf in leaves:
chunk_tag2count[leaf + '_' + node] += 1
chunk_tag2count['ROOT'] = 100
print('feature stat')
print('# of gram: %d' % len(gram2count))
print('# of pos: %d' % len(pos_tag2count))
print('# of chunk_tag: %d' % len(chunk_tag2count))
print('# of dep: %d' % len(dep_tag2count))
feature2id = {'gram2count': gram2count, 'pos_tag2count': pos_tag2count,
'chunk_tag2count': chunk_tag2count, 'dep_tag2count': dep_tag2count}
with open(path.join(self.data_dir, 'feature2count.json'), 'w', encoding='utf8') as f:
json.dump(feature2id, f, ensure_ascii=False)
f.write('\n')
def read_feature2count(self):
with open(path.join(self.data_dir, 'feature2count.json'), 'r', encoding='utf8') as f:
line = f.readline()
return json.loads(line)
def feature_stat(self):
all_feature2count = self.read_feature2count()
feature_num = []
for feature in ['gram2count', 'pos_tag2count', 'chunk_tag2count', 'dep_tag2count']:
feature2count = all_feature2count[feature]
num = 0
for f, n in feature2count.items():
if n > 1:
num += 1
feature_num.append(num)
# feature_num.append(len(all_feature2count['gram2count']))
# feature_num.append(len(all_feature2count['pos_tag2count']))
# feature_num.append(len(all_feature2count['chunk_tag2count']))
# feature_num.append(len(all_feature2count['dep_tag2count']))
print('max # of features: %d' % max(feature_num))
return max(feature_num)
def read_features(self, flag):
all_data = self.read_json(path.join(self.data_dir, flag + '.stanford.json'))
all_feature_data = []
for data in all_data:
sentence_len=0
sentence_feature = []
sentence = ''
words = []
index=[]
sentences=data['sentences']
for sentence in sentences:
tokens = sentence['tokens']
for token in tokens:
feature_dict = {}
feature_dict['word'] = token['originalText']
words.append(token['word'].replace('\xa0',''))
# sentence += token['word']
start_index = token['characterOffsetBegin']
end_index = token['characterOffsetEnd']
feature_dict['char_index'] = [i for i in range(start_index, end_index)]
feature_dict['length']= sentence_len+ len(sentence)
feature_dict['pos'] = token['pos']
sentence_feature.append(feature_dict)
# df = df.append([{'word': ' ', 'pos': ' '}], ignore_index=True)
deparse = sentence['basicDependencies']
for dep in deparse:
dependent_index = dep['dependent'] - 1
sentence_feature[dependent_index]['dep'] = dep['dep']
sentence_feature[dependent_index]['governed_index'] = dep['governor'] - 1
c_parse = Tree.fromstring(sentence['parse'].replace('\xa0',''))
current_index = 0
for s in c_parse.subtrees(lambda t: t.label() in chunk_pos):
leaves = s.leaves()
if len(leaves) == 0:
continue
node = s.label()
index = words[current_index:].index(leaves[0]) + current_index
current_index = index
for i, leaf in enumerate(leaves):
if 'chunk_tags' not in sentence_feature[index + i]:
sentence_feature[index + i]['chunk_tags'] = []
sentence_feature[index + i]['chunk_tags'].append({'chunk_tag': node, 'height': 0,
'range': [index, index + len(leaves)-1]})
for chunk_tag in sentence_feature[index + i]['chunk_tags']:
chunk_tag['height'] += 1
for token in sentence_feature:
if 'chunk_tags' not in token:
token['chunk_tags'] = [{'chunk_tag': 'ROOT', 'height': 1, 'range': [0, len(sentence_feature)-1]}]
all_feature_data.append(sentence_feature)
return all_feature_data
def oov_stat(data_dir):
oov_count = 0
word_count = 0
word = ''
char_count = 0
sentence_num = 0
oov_dict = {}
char_dict = {}
word_dict = {}
with open(path.join(data_dir, 'word2id.json'), 'r', encoding='utf8') as f:
word2id = json.loads(f.readline())
with open(path.join(data_dir, "test.txt"), 'r', encoding='utf8') as f:
insentence = False
sentence_len = []
slen = 0
long_num = 0
for line in tqdm(f.readlines()):
line = line.strip()
if len(line) == 0:
if insentence:
sentence_num += 1
sentence_len.append(slen)
if slen > 150:
long_num += 1
slen = 0
insentence = False
continue
insentence = True
slen += 1
splits = line.split('\t')
character = splits[0]
label = splits[-1][0]
word += character
char_count += 1
char_dict[character] = 0
if label in ['S', 'E']:
word_count += 1
word_dict[word] = 0
if word not in word2id:
oov_dict[word] = 0
oov_count += 1
word = ''
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--dataset",
default=None,
type=str,
required=True,
help="The input data dir. Should contain the .txt files (or other data files) for the task.")
args = parser.parse_args()
base_min_freq = 1
av_threshold = 2
min_freq = base_min_freq
print('min freq: %d' % min_freq)
data_dir =args.dataset
print(data_dir)
getlabels(data_dir)
get_word2id(data_dir)
if os.path.exists(path.join(data_dir, 'train' + '.txt')):
request_features_from_stanford(data_dir, 'train')
if os.path.exists(path.join(data_dir, 'test' + '.txt')):
request_features_from_stanford(data_dir, 'test')
if os.path.exists(path.join(data_dir, 'dev' + '.txt')):
request_features_from_stanford(data_dir, 'dev')
sfp = stanford_feature_processor(data_dir)
sfp._pre_processing()
sfp.feature_stat()
通过函数读取句法知识 get_knowledge.py
import os
import json
from os import path
from nltk.tree import Tree
from collections import defaultdict
punctuation = ['。', '，', '、', '：', '？', '！', '（', '）', '“', '”', '【', '】']
chunk_pos = ['NP', 'PP', 'VP', 'ADVP', 'SBAR', 'ADJP', 'PRT', 'INTJ', 'CONJP', 'LST']
class StanfordFeatureProcessor:
def __init__(self, data_dir):
self.data_dir = data_dir
def read_json(self, data_path):
data = []
with open(data_path, 'r', encoding='utf8') as f:
lines = f.readlines()
for line in lines:
line = line.strip()
if line == '':
continue
data.append(json.loads(line))
return data
def _pre_processing(self):
all_data = self.read_json(path.join(self.data_dir, 'train.stanford.json'))
gram2count = defaultdict(int)
pos_tag2count = defaultdict(int)
chunk_tag2count = defaultdict(int)
dep_tag2count = defaultdict(int)
for data in all_data:
sentences_list = data['sentences']
for sentence_l in sentences_list:
tokens = sentence_l['tokens']
for token in tokens:
gram2count[token['originalText']] += 1
pos_tag2count[token['pos']] += 1
pos_tag2count[token['originalText'] + '_' + token['pos']] += 1
deparse = sentence_l['basicDependencies']
for word in deparse:
dep_tag2count[word['dep']] += 1
dep_tag2count[word['dependentGloss'] + '_' + word['dep']] += 1
coparse = Tree.fromstring(sentence_l['parse'])
for s in coparse.subtrees(lambda t: t.label() in chunk_pos):
leaves = s.leaves()
node = s.label()
chunk_tag2count[node] += 1
for leaf in leaves:
chunk_tag2count[leaf + '_' + node] += 1
chunk_tag2count['ROOT'] = 100
print('feature stat')
print('# of gram: %d' % len(gram2count))
print('# of pos: %d' % len(pos_tag2count))
print('# of chunk_tag: %d' % len(chunk_tag2count))
print('# of dep: %d' % len(dep_tag2count))
feature2id = {'gram2count': gram2count, 'pos_tag2count': pos_tag2count,
'chunk_tag2count': chunk_tag2count, 'dep_tag2count': dep_tag2count}
with open(path.join(self.data_dir, 'feature2count.json'), 'w', encoding='utf8') as f:
json.dump(feature2id, f, ensure_ascii=False)
f.write('\n')
def read_feature2count(self):
with open(path.join(self.data_dir, 'feature2count.json'), 'r', encoding='utf8') as f:
return json.loads(f.read())
def feature_stat(self):
all_feature2count = self.read_feature2count()
feature_num = []
for feature in ['gram2count', 'pos_tag2count', 'chunk_tag2count', 'dep_tag2count']:
feature2count = all_feature2count[feature]
num = 0
for f, n in feature2count.items():
if n > 1:
num += 1
feature_num.append(num)
# feature_num.append(len(all_feature2count['gram2count']))
# feature_num.append(len(all_feature2count['pos_tag2count']))
# feature_num.append(len(all_feature2count['chunk_tag2count']))
# feature_num.append(len(all_feature2count['dep_tag2count']))
print('max # of features: %d' % max(feature_num))
return max(feature_num)
def read_features(self, flag):
all_data = self.read_json(path.join(self.data_dir, flag + '.stanford.json'))
all_feature_data = []
print(len(all_data))
for data in all_data:
sentence_len=0
sentence_feature = []
sentence = ''
words = []
index=[]
sentences=data['sentences']
for sentence in sentences:
tokens = sentence['tokens']
for token in tokens:
feature_dict = {}
feature_dict['word'] = token['originalText']
words.append(token['word'].replace('\xa0',''))
# sentence += token['word']
start_index = token['characterOffsetBegin']
end_index = token['characterOffsetEnd']
feature_dict['char_index'] = [i for i in range(start_index, end_index)]
feature_dict['length']= sentence_len+ len(sentence)
feature_dict['pos'] = token['pos']
sentence_feature.append(feature_dict)
# df = df.append([{'word': ' ', 'pos': ' '}], ignore_index=True)
deparse = sentence['basicDependencies']
for dep in deparse:
dependent_index = dep['dependent'] - 1
sentence_feature[dependent_index]['dep'] = dep['dep']
sentence_feature[dependent_index]['governed_index'] = dep['governor'] - 1
c_parse = Tree.fromstring(sentence['parse'].replace('\xa0',''))
current_index = 0
for s in c_parse.subtrees(lambda t: t.label() in chunk_pos):
leaves = s.leaves()
if len(leaves) == 0:
continue
node = s.label()
index = words[current_index:].index(leaves[0]) + current_index
current_index = index
for i, leaf in enumerate(leaves):
if 'chunk_tags' not in sentence_feature[index + i]:
sentence_feature[index + i]['chunk_tags'] = []
sentence_feature[index + i]['chunk_tags'].append(
{'chunk_tag': node, 'height': 0, 'range': [index, index + len(leaves)-1]}
)
for chunk_tag in sentence_feature[index + i]['chunk_tags']:
chunk_tag['height'] += 1
for token in sentence_feature:
if 'chunk_tags' not in token:
token['chunk_tags'] = [{'chunk_tag': 'ROOT', 'height': 1, 'range': [0, len(sentence_feature)-1]}]
all_feature_data.append(sentence_feature)
return all_feature_data
def get_chunk(chunk_tags):
for chunk_tag in chunk_tags:
if chunk_tag.get("height") == 1:
return chunk_tag.get("chunk_tag"), chunk_tag.get("range")
def get_dep(sentence):
words = [change_word(i["word"]) for i in sentence]
deps = [i["dep"] + "_dep" for i in sentence]
dep_matrix = [[0] * len(words) for _ in range(len(words))]
for i, item in enumerate(sentence):
governor = item["governed_index"]
dep_matrix[i][i] = 1
if governor != -1:
dep_matrix[i][governor] = 1
dep_matrix[governor][i] = 1
ret_list = []
for word, dep, dep_range in zip(words, deps, dep_matrix):
ret_list.append({"word": word, "dep": dep, "range": dep_range})
return ret_list
def change_word(word):
if "-RRB-" in word:
return word.replace("-RRB-", ")")
if "-LRB-" in word:
return word.replace("-LRB-", "(")
return word
def filter_useful_feature(feature_list, feature_type):
ret_list = []
# [pos, dep, chunk]
if feature_type == "all":
ret_list = [[], [], []]
for i, sentence in enumerate(feature_list):
ret0 = []
ret2 = []
ret_list[1].append(get_dep(sentence))
for word in sentence:
ret0.append({"word": change_word(word['word']), "pos": word["pos"] + "_pos"})
chunk_tag, range_chunk = get_chunk(word["chunk_tags"])
ret2.append({"word": change_word(word['word']), "chunk": chunk_tag + "_chunk", "range": range_chunk})
ret_list[0].append(ret0)
ret_list[2].append(ret2)
assert len(ret_list[0][i]) == len(ret_list[1][i]) == len(ret_list[2][i])
print("length: ", len(ret_list[0]), len(ret_list[1]), len(ret_list[2]))
return ret_list
def get_feature2count(train_features, dev_features=None, test_features=None):
train_pos_features, train_dep_features, train_chunk_features = train_features
feature2count = defaultdict(int)
for sent in train_pos_features:
for item in sent:
word = item["word"]
pos = item["pos"]
pos_feature = word + "_" + pos
feature2count[pos] += 1
feature2count[pos_feature] += 1
for sent in train_dep_features:
for item in sent:
word = item["word"]
dep = item["dep"]
dep_feature = word + "_" + dep
feature2count[dep] += 1
feature2count[dep_feature] += 1
for sent in train_chunk_features:
for item in sent:
word = item["word"]
chunk = item["chunk"]
chunk_feature = word + "_" + chunk
feature2count[chunk] += 1
feature2count[chunk_feature] += 1
if dev_features:
dev_pos_features, dev_dep_features, dev_chunk_features = dev_features
for sent in dev_pos_features:
for item in sent:
word = item["word"]
pos = item["pos"]
pos_feature = word + "_" + pos
feature2count[pos] += 1
feature2count[pos_feature] += 1
for sent in dev_dep_features:
for item in sent:
word = item["word"]
dep = item["dep"]
dep_feature = word + "_" + dep
feature2count[dep] += 1
feature2count[dep_feature] += 1
for sent in dev_chunk_features:
for item in sent:
word = item["word"]
chunk = item["chunk"]
chunk_feature = word + "_" + chunk
feature2count[chunk] += 1
feature2count[chunk_feature] += 1
if test_features:
test_pos_features, test_dep_features, test_chunk_features = test_features
for sent in test_pos_features:
for item in sent:
word = item["word"]
pos = item["pos"]
pos_feature = word + "_" + pos
feature2count[pos] += 1
feature2count[pos_feature] += 1
for sent in test_dep_features:
for item in sent:
word = item["word"]
dep = item["dep"]
dep_feature = word + "_" + dep
feature2count[dep] += 1
feature2count[dep_feature] += 1
for sent in test_chunk_features:
for item in sent:
word = item["word"]
chunk = item["chunk"]
chunk_feature = word + "_" + chunk
feature2count[chunk] += 1
feature2count[chunk_feature] += 1
return feature2count
def generate_knowledge_api(data_dir, feature_type="all", level="all"):
sfp = StanfordFeatureProcessor(data_dir)
train_feature_data = sfp.read_features(flag="train")
print("len_train: ", len(train_feature_data))
dev_feature_data = sfp.read_features(flag="dev")
print("len_dev: ", len(dev_feature_data))
test_feature_data = sfp.read_features(flag="test")
print("len_test: ", len(test_feature_data))
train_feature_data = filter_useful_feature(train_feature_data, feature_type="all")
dev_feature_data = filter_useful_feature(dev_feature_data, feature_type="all")
test_feature_data = filter_useful_feature(test_feature_data, feature_type="all")
assert level in ["all", "train"]
if level == "train":
feature2count = get_feature2count(train_feature_data)
elif level == "all":
feature2count = get_feature2count(train_feature_data, dev_feature_data, test_feature_data)
feature2id = {"<PAD>": 0}
id2feature = {0: "<PAD>"}
index = 1
for key in feature2count:
feature2id[key] = index
id2feature[index] = key
index += 1
return train_feature_data, dev_feature_data, test_feature_data, feature2count, feature2id, id2feature
扫一扫
1056
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
