word2vec tf实战

下载语料库

# 第一步: 在下面这个地址下载语料库
def maybe_download(filename, expected_bytes):
	"""
	这个函数的功能是：
		如果filename不存在，就在上面的地址下载它。
		如果filename存在，就跳过下载。
		最终会检查文字的字节数是否和expected_bytes相同。
	"""
	if not os.path.exists(filename):
		print('start downloading...')
		filename, _ = urllib.request.urlretrieve(url + filename, filename)
	statinfo = os.stat(filename)
	if statinfo.st_size == expected_bytes:
		print('Found and verified', filename)
	else:
		print(statinfo.st_size)
		raise Exception(
			'Failed to verify ' + filename + '. Can you get to it with a browser?')
	return filename

# 下载语料库text8.zip并验证下载
url = 'http://mattmahoney.net/dc/'
filename = maybe_download('text8.zip', 31344016)

# 将语料库解压，并转换成一个word的list
def read_data(filename):
	"""
	这个函数的功能是：
		将下载好的zip文件解压并读取为word的list
	"""
	with zipfile.ZipFile(filename) as f:
		data = tf.compat.as_str(f.read(f.namelist()[0])).split()
	return data

vocabulary = read_data(filename)
print('Data size', len(vocabulary)) # 总长度为1700万左右
print(vocabulary[0:100]) # 输出前100个词。

制作词表

# 第二步: 制作一个词表，将不常见的词变成一个UNK标识符
# 词表的大小为5万（即我们只考虑最常出现的5万个词）
vocabulary_size = 50000
def build_dataset(words, n_words):
	"""
	函数功能：将原始的单词表示变成index
	"""
	count = [['UNK', -1]]#筛选5W的单词
	count.extend(collections.Counter(words).most_common(n_words - 1))
	#collections.Counter()计数器，每个单词出现的次数。次数top截取n_words - 1个。
	dictionary = dict()#5W单词，对每个单词进行编码
	for word, _ in count:
		dictionary[word] = len(dictionary)
	data = list()# words：将不常见的词变成一个UNK标识符，常见的词变成编码。
	unk_count = 0# 记录不长出现的词的个数，也就是0的个数
	for word in words:
		if word in dictionary:
		index = dictionary[word]
		else:
			index = 0  # UNK的index为0
			unk_count += 1
		data.append(index)
	count[0][1] = unk_count
	reversed_dictionary = dict(zip(dictionary.values(), dictionary.keys()))#key values互换位置
	return data, count, dictionary, reversed_dictionary

data, count, dictionary, reverse_dictionary = build_dataset(vocabulary,vocabulary_size)
del vocabulary  # 删除已节省内存
print('Most common words (+UNK)', count[:5]) # 输出最常出现的5个单词
# 输出转换后的数据库data，和原来的单词（前10个）
print('Sample data', data[:10], [reverse_dictionary[i] for i in data[:10]])
# 我们下面就使用data来制作训练集
data_index = 0

生成词对

# 第三步：定义一个函数，用于生成skip-gram模型用的batch
def generate_batch(batch_size, num_skips, skip_window):
	# data_index相当于一个指针，初始为0
	# 每次生成一个batch，data_index就会相应地往后推
	global data_index
	assert batch_size % num_skips == 0
	assert num_skips <= 2 * skip_window
	batch = np.ndarray(shape=(batch_size), dtype=np.int32)
	labels = np.ndarray(shape=(batch_size, 1), dtype=np.int32)
	span = 2 * skip_window + 1  # [ skip_window target skip_window ]
	buffer = collections.deque(maxlen=span)#buffer：deque([])
	# data_index是当前数据开始的位置
	# 产生batch后就往后推1位（产生batch）
	for _ in range(span):
		buffer.append(data[data_index])
		data_index = (data_index + 1) % len(data)
	for i in range(batch_size // num_skips):
		# 利用buffer生成batch
		# buffer是一个长度为 2 * skip_window + 1长度的word list
		# 一个buffer生成num_skips个数的样本
		# print([reverse_dictionary[i] for i in buffer])
		target = skip_window  # target label at the center of the buffer
		# targets_to_avoid保证样本不重复
		targets_to_avoid = [skip_window]
		for j in range(num_skips):
			while target in targets_to_avoid:
				target = random.randint(0, span - 1)
			targets_to_avoid.append(target)
			batch[i * num_skips + j] = buffer[skip_window]
			labels[i * num_skips + j, 0] = buffer[target]
		buffer.append(data[data_index])
		# 每利用buffer生成num_skips个样本，data_index就向后推进一位
		data_index = (data_index + 1) % len(data)
	data_index = (data_index + len(data) - span) % len(data)
	return batch, labels
# 默认情况下skip_window=1, num_skips=2
# 此时就是从连续的3(3 = skip_window*2 + 1)个词中生成2(num_skips)个样本。
# 如连续的三个词['used', 'against', 'early']
# 生成两个样本：against -> used, against -> early
batch, labels = generate_batch(batch_size=8, num_skips=2, skip_window=1)#产生8对batch
for i in range(8):
    print(batch[i], reverse_dictionary[batch[i]],
        '->', labels[i, 0], reverse_dictionary[labels[i, 0]])

建立模型