word2vec

 

# -*- coding: utf-8 -*-
import pandas as pd
from bs4 import BeautifulSoup   
import re #替换标点
#frequent words with little meaning
import nltk
from nltk.corpus import stopwords
import numpy as np
from sklearn.feature_extraction.text import CountVectorizer 
from sklearn.ensemble import RandomForestClassifier
import nltk.data
import logging
from gensim.models import word2vec
#to train Word2Vec it is better not to remove stopwords 
#because the algorithm relies on the broader context of 
#the sentence in order to produce high-quality word vectors.


def review_to_wordlist( review, remove_stopwords=False ):
    review_text = BeautifulSoup(review).get_text()
    review_text = re.sub("[^a-zA-Z]"," ", review_text)
    words = review_text.lower().split()
    if remove_stopwords:
        stops = set(stopwords.words("english"))
        words = [w for w in words if not w in stops]
    return(words)

#Word2Vec does not need labels in order to create meaningful representations.
#Words with similar meanings appear in clusters, 
#and clusters are spaced such that some word relationships,


#Word2Vec expects single sentences, each one as a list of words. 
#In other words, the input format is a list of lists.



def review_to_sentences(review,tokenizer,remove_stopwords=False):
    raw_sentences=tokenizer.tokenize(review.strip())
    sentences=[]
    for raw_sentence in raw_sentences :
        if(len(raw_sentence)>0):
            sentences.append(review_to_wordlist(raw_sentence,remove_stopwords)      )
    return sentences
    #将段落分成raw_sentences
    # 将每个raw_sentence都转为wordlist
            




def loadData():
    train = pd.read_csv( "labeledTrainData.tsv", header=0, delimiter="\t", quoting=3,encoding="utf-8")
    unlabeled_train = pd.read_csv( "unlabeledTrainData.tsv", header=0, delimiter="\t", quoting=3,encoding="utf-8" )
    test = pd.read_csv( "testData.tsv", header=0, delimiter="\t", quoting=3,encoding="utf-8" )
    return train,test

def reviewsToSentences():
    
    train,unlabeled_train,test=loadData()
    tokenizer=nltk.data.load('tokenizers/punkt/english.pickle')
    sentences=[]
    print "Parsing sentences from training set"
    for review in train["review"]:
        sentences += review_to_sentences(review,tokenizer)
        
    print "Parsing sentences from training set"
    for review in unlabeled_train["review"]:
        sentences += review_to_sentences(review,tokenizer)



    
def apply(model):
    model.doesnt_match(["man","woman","child","kitchen"])
    model.most_similar("man")
    
    
    
    
#-----------------------average_vector------------------------------
from gensim.models import Word2Vec
import numpy as np
def makeFeatureVec(words, model, num_features):
    
    featureVec = np.zeros((num_features,),dtype="float32")
    nwords = 0.
    index2word_set = set(model.index2word)
    
    for word in words:
        if word in index2word_set: 
            nwords = nwords + 1.
            featureVec = np.add(featureVec,model[word])
    
    featureVec = np.divide(featureVec,nwords)
    return featureVec

def getReviewsVec(reviews,model,num_features):
    
    reviewSetfeatureVec=[]
    counter=0.0
    for words in reviews:
        reviewSetfeatureVec[counter]=makeFeatureVec(words,model,num_features)
        counter+=1
    return reviewSetfeatureVec
    


def run(sentences,train,test):
    #creating nice output messages

    logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s',\
                        level=logging.INFO)
    num_features=400
    min_word_count=40
    num_workers=4
    context=10
    downsampling=1e-3
    
    print "training model..."
    model=word2vec.Word2Vec(sentences,workers=num_workers,\
          size=num_features, min_count=min_word_count,\
          window=context, sample=downsampling)
    model.init_sims(replace=True)
    model.save("FirstWord2Vec")             

    clean_train_reviews = []
    for review in train["review"]:
        clean_train_reviews.append( review_to_wordlist( review,remove_stopwords=True )) 
    trainDataVecs = getReviewsVec( clean_train_reviews, model, num_features )
    
    print "Creating average feature vecs for test reviews"
    clean_test_reviews = []
    for review in test["review"]:
        clean_test_reviews.append( review_to_wordlist( review, remove_stopwords=True ))    
    testDataVecs = getReviewsVec( clean_test_reviews, model, num_features )
    

    forest = RandomForestClassifier( n_estimators = 100 )    
    print "Fitting a random forest to labeled training data..."
    forest = forest.fit( trainDataVecs, train["sentiment"] )
    
    # Test & extract results 
    result = forest.predict( testDataVecs )
    
    # Write the test results 
    output = pd.DataFrame( data={"id":test["id"], "sentiment":result} )
    output.to_csv( "Word2Vec_AverageVectors.csv", index=False, quoting=3 )
    
#-----------------------word-centroid------------------------------

from sklearn.cluster import KMeans
import time

def kmeans_run(model):
    word_vectors = model.syn0
    num_clusters = word_vectors.shape[0] / 5
    
    # Initalize a k-means object and use it to extract centroids
    kmeans_clustering = KMeans( n_clusters = num_clusters )
    centeroid_id = kmeans_clustering.fit_predict( word_vectors )
    word_centroid_map=dict(zip(word_vectors,centeroid_id ))
    return word_centroid_map
    

def create_review_centroids_histogram(clean_train_reviews,clean_test_reviews,model,word_centroid_map):
    dict_words=set(model.index2word)
    review_centroids_vector=np.zeros(num_centroids,dtype="float32")

    counter=0    
    for review_wordlist in clean_train_reviews:
        for word in review_wordlist:
            if word in dict_words:
                index=word_centroid_map[word]
                review_centroid_vector[index]+=1
                reviews_centroid_vector[counter]=review_centroids_vector
                counter+=1
        
    for review_wordlist in clean_test_reviews:
        for word in review_wordlist:
            if word in dict_words:
                index=word_centroid_map[word]
                test_review_centroid_vector[index]+=1
                test_reviews_centroid_vector[counter]=review_centroids_vector
                counter+=1
    forest = RandomForestClassifier(n_estimators = 100)
    forest=forest.fit(reviews_centroid_vector,train["sentiment"])    
    result=forest.predict(test_reviews_centroid_vector)
    output = pd.DataFrame(data={"id":test["id"], "sentiment":result})
    output.to_csv( "BagOfCentroids.csv", index=False, quoting=3 )