deeplearning4j之GloVe实现实现

最新推荐文章于 2024-06-19 00:43:35 发布

旭旭_哥

最新推荐文章于 2024-06-19 00:43:35 发布

阅读量1.8k

点赞数

分类专栏：机器学习 java

本文链接：https://blog.csdn.net/luoyexuge/article/details/70212605

版权

机器学习同时被 2 个专栏收录

114 篇文章 7 订阅

订阅专栏

java

87 篇文章 1 订阅

订阅专栏

glove类似于word2vec，听说效果还比word2vec更加强大，可以用于做自然语言处理，正好学习deeplearning4j的时候看到了，顺便写在这，

文章用到的数据跟上一篇word2vec一样，看看效果吧，训练时间比word2vec要长太多，代码如下：

package com.meituan.deeplearning4j;

import org.datavec.api.util.ClassPathResource;
import org.deeplearning4j.models.glove.Glove;
import org.deeplearning4j.text.sentenceiterator.BasicLineIterator;
import org.deeplearning4j.text.sentenceiterator.SentenceIterator;
import org.deeplearning4j.text.tokenization.tokenizer.preprocessor.CommonPreprocessor;
import org.deeplearning4j.text.tokenization.tokenizerfactory.DefaultTokenizerFactory;
import org.deeplearning4j.text.tokenization.tokenizerfactory.TokenizerFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.FileNotFoundException;
import java.util.Arrays;
import java.util.Collection;

public class GloVeRaw {
	public static void main(String[] args) throws FileNotFoundException {
		String filePath = "/Users/shuubiasahi/Desktop/bayies/deeplearning/part-00000";
		SentenceIterator iter = new BasicLineIterator(new File(filePath));
		TokenizerFactory t = new DefaultTokenizerFactory();
		t.setTokenPreProcessor(new CommonPreprocessor());
		Glove glove = new Glove.Builder().iterate(iter).tokenizerFactory(t)
		.alpha(0.75).learningRate(0.1)
				.epochs(25)
				.xMax(100)
				.batchSize(1000)
				.shuffle(true)
				.symmetric(true).build();

		glove.fit();

		System.out.println("和微信最接近的10个词汇:" + glove.wordsNearest("微信", 10));
		System.out.println(Arrays.toString(glove.getWordVector("微信")));
		System.out.println("微信和qq的相似度为：" + glove.similarity("微信", "腾讯聊天账号"));
		System.out.println("和美女最接近的10个词汇:" + glove.wordsNearest("腾讯聊天账号", 10));

		System.exit(0);
	}

}