简易英文问答系统（glove.6B词向量）

1.读入json文件转换成dict形式`

# doc = open('train-v2.0.json').readlines()此时格式为list,需要转换格式成dict
import json
f = open('train1-v2.0.json')
doc = json.load(f)
# 利用json.load()将list形式转换成dict的形式

2.将全部问题和答案放入question_list和answe_list

question_list = []
answer_list = []
for data in doc['data']:
    for paragraphs in data['paragraphs']:
        for qas in paragraphs['qas']:
            question_list.append(qas['question'])
            if len(qas['answers']) == 0:
                answer_list.append(' ')
            else:
                for contents in qas['answers']:
                    answer_list.append(contents["text"])
print(question_list, answer_list)
# 断言语句,确定question_list的问题和answer_list中的答案数目相同
assert len(question_list) == len(answer_list)

3.统计question_list中单词总数和单词种类

#此处采用的是nltk库中的word_tokenize函数
from nltk.tokenize import word_tokenize
word_in_train = {
   }
for x in question_list:
    for x1 in word_tokenize(x.rstrip('?')):
        if x1 in word_in_train:
            word_in_train[x1] += 1
        else:
            word_in_train[x1] = 1
print(word_in_train)
word_num = sum(word_in_train.values())
word_different = len(word_in_train.keys())
print(word_num