# coding: utf-8
# Generate intial word embedding for headlines and description
# The embedding is limited to a fixed vocabulary size (`vocab_size`) but
# a vocabulary of all the words that appeared in the data is built.
# In[3]:
FN = 'vocabulary-embedding'
# In[4]:
seed=42
# In[5]:
vocab_size = 40000
# In[6]:
embedding_dim = 100
# In[8]:
lower = False # dont lower case the text
# # read tokenized headlines and descriptions
# In[7]:
import pickle
FN0 = 'tokens' # this is the name of the data file which I assume you already have
with open('data/%s.pkl'%FN0, 'rb') as fp:
heads, desc, keywords = pickle.load(fp) # keywords are not used in this project
# In[9]:
if lower:
heads = [h.lower() for h in heads]
# In[10]:
if lower:
desc = [h.lower() for h in desc]
# In[11]:
i=0
heads[i]
# In[12]:
desc[i]
# In[13]:
keywords[i]
# In[14]:
len(heads),len(set(heads))
# In[15]:
len(desc),len(set(desc))
# # build vocabulary
# In[16]:
from collections import Counter
from itertools import chain
def get_vocab(lst):
vocabcount = Counter(w for txt in lst for w in txt.split())
vocab = map(lambda x: x[0], sorted(vocabcount.items(), key=lambda x: -x[1]))
return vocab, vocabcount
# In[17]:
vocab, vocabcount = get_vocab(heads+desc)
# most popular tokens
# In[18]:
vocab = list(vocab)
print (vocab[:50])
print ('...',len(vocab))
# In[19]:
import matplotlib.pyplot as plt
#get_ipython().magic('matplotlib inline')
plt.plot([vocabcount[w] for w in vocab]);
plt.gca().set_xscale("log", nonposx='clip')
plt.gca().set_yscale("log", nonposy='clip')
plt.title('word distribution in headlines and discription')
plt.xlabel('rank')
plt.ylabel('total appearances');
plt.show()
# always nice to see [Zipf's law](https://en.wikipedia.org/wiki/Zipf%27s_law)
# # Index words
# In[21]:
empty = 0 # RNN mask of no data
eos = 1 # end of sentence
start_idx = eos+1 # first real word
# In[22]:
def get_idx(vocab, vocabcount):
word2idx = dict((word, idx+start_idx) for idx,word in enumerate(vocab))
word2idx['<empty>'] = empty
word2idx['<eos>'] = eos
idx2word = dict((idx,word) for word,idx in word2idx.items())
return word2idx, idx2word
# In[23]:
word2idx, idx2word = get_idx(vocab, vocabcount)
# # Word Embedding
# ## read GloVe
# In[24]:
fname = 'glove.6B.%dd.txt'%embedding_dim
import os
datadir_base = os.path.expanduser(os.path.join('~', '.keras')) #把path中包含的"~"和"~user"转换成用户目录
if not os.access(datadir_base, os.W_OK): #包含在access()的mode参数中 , 测试path是否可写。
datadir_base = os.path.join('/tmp', '.keras')
datadir = os.path.join(datadir_base, 'datasets') #将目录组合成路径
glove_name = os.path.join(datadir, fname)
#print(glove_name)
# if not os.path.exists(glove_name): #查某个路径是否存在,返回True or False
# path = 'glove.6B.zip'
# #path = get_file(path, origin="http://nlp.stanford.edu/data/glove.6B.zip")
# os.system('unzip {datadir}/{path}')
# In[25]:
import subprocess
glove_name = 'tmp\keras\datasets\glove.6B.100d.txt'
# lines = glove_name.readlines()
# length = len(lines)
# print(length)
#glove_n_symbols = subprocess.getoutput("wc -l {glove_name}") #显示行数
glove_n_symbols = subprocess.getoutput("ls {glove_name}") #显示行数
print(glove_n_symbols)
glove_n_symbols = int(glove_n_symbols[0].split()[0])
#print(glove_n_symbols)
round(float("1.0"))
# In[26]:
import numpy as np
glove_index_dict = {}
glove_embedding_weights = np.empty((glove_n_symbols, embedding_dim))
globale_scale=.1
with open(glove_name, 'r') as fp:
i = 0
for l in fp:
l = l.strip().split()
w = l[0]
glove_index_dict[w] = i
glove_embedding_weights[i,:] = map(float,l[1:])
i += 1
glove_embedding_weights *= globale_scale
# # In[27]:
#
#
# glove_embedding_weights.std()
#
#
# # In[28]:
#
#
# for w,i in glove_index_dict.items():
# w = w.lower()
# if w not in glove_index_dict:
# glove_index_dict[w] = i
#
#
# # ## embedding matrix
#
# # use GloVe to initialize embedding matrix
#
# # In[30]:
#
#
# import numpy as np
#
# # generate random embedding with same scale as glove
# np.random.seed(seed)
# shape = (vocab_size, embedding_dim)
# scale = glove_embedding_weights.std()*np.sqrt(12)/2 # uniform and not normal
# embedding = np.random.uniform(low=-scale, high=scale, size=shape)
# print ('random-embedding/glove scale', scale, 'std', embedding.std())
#
# # copy from glove weights of words that appear in our short vocabulary (idx2word)
# c = 0
# for i in range(vocab_size):
# w = idx2word[i]
# g = glove_index_dict.get(w, glove_index_dict.get(w.lower()))
# if g is None and w.startswith('#'): # glove has no hastags (I think...)
# w = w[1:]
# g = glove_index_dict.get(w, glove_index_dict.get(w.lower()))
# if g is not None:
# embedding[i,:] = glove_embedding_weights[g,:]
# c+=1
# print ('number of tokens, in small vocab, found in glove and copied to embedding', c,c/float(vocab_size))
#
#
# # lots of word in the full vocabulary (word2idx) are outside `vocab_size`.
# # Build an alterantive which will map them to their closest match in glove but only if the match
# # is good enough (cos distance above `glove_thr`)
#
# # In[31]:
#
#
# glove_thr = 0.5
#
#
# # In[34]:
#
#
# word2glove = {}
# for w in word2idx:
# if w in glove_index_dict:
# g = w
# elif w.lower() in glove_index_dict:
# g = w.lower()
# elif w.startswith('#') and w[1:] in glove_index_dict:
# g = w[1:]
# elif w.startswith('#') and w[1:].lower() in glove_index_dict:
# g = w[1:].lower()
# else:
# continue
# word2glove[w] = g
#
#
# # for every word outside the embedding matrix find the closest word inside the mebedding matrix.
# # Use cos distance of GloVe vectors.
# #
# # Allow for the last `nb_unknown_words` words inside the embedding matrix to be considered to be outside.
# # Dont accept distances below `glove_thr`
#
# # In[47]:
#
#
# normed_embedding = embedding/np.array([np.sqrt(np.dot(gweight,gweight)) for gweight in embedding])[:,None]
#
# nb_unknown_words = 100
#
# glove_match = []
# for w,idx in word2idx.items():
# if idx >= vocab_size-nb_unknown_words and w.isalpha() and w in word2glove:
# gidx = glove_index_dict[word2glove[w]]
# gweight = glove_embedding_weights[gidx,:].copy()
# # find row in embedding that has the highest cos score with gweight
# gweight /= np.sqrt(np.dot(gweight,gweight))
# score = np.dot(normed_embedding[:vocab_size-nb_unknown_words], gweight)
# while True:
# embedding_idx = score.argmax()
# s = score[embedding_idx]
# if s < glove_thr:
# break
# if idx2word[embedding_idx] in word2glove :
# glove_match.append((w, embedding_idx, s))
# break
# score[embedding_idx] = -1
# glove_match.sort(key = lambda x: -x[2])
# print ('# of glove substitutes found', len(glove_match))
#
#
# # manually check that the worst substitutions we are going to do are good enough
#
# # In[48]:
#
#
# for orig, sub, score in glove_match[-10:]:
# print (score, orig,'=>', idx2word[sub])
#
#
# # build a lookup table of index of outside words to index of inside words
#
# # In[49]:
#
#
# glove_idx2idx = dict((word2idx[w],embedding_idx) for w, embedding_idx, _ in glove_match)
#
#
# # # Data
#
# # In[50]:
#
#
# Y = [[word2idx[token] for token in headline.split()] for headline in heads]
# len(Y)
#
#
# # In[51]:
#
#
# plt.hist(map(len,Y),bins=50);
#
#
# # In[52]:
#
#
# X = [[word2idx[token] for token in d.split()] for d in desc]
# len(X)
#
#
# # In[53]:
#
#
# plt.hist(map(len,X),bins=50);
#
#
# # In[54]:
#
#
# import pickle
# with open('data/%s.pkl'%FN,'wb') as fp:
# pickle.dump((embedding, idx2word, word2idx, glove_idx2idx),fp,-1)
#
#
# # In[4]:
#
#
# import pickle
# with open('data/%s.data.pkl'%FN,'wb') as fp:
# pickle.dump((X,Y),fp,-1)
#
vocabulary_embedding.py
最新推荐文章于 2022-06-02 13:02:14 发布