使用Lucene词频统计与d3.cloud展示的中文英文词云系统

用Lucene 4.7对进行词频统计，使用的分词器为IKAnalyzer，对中文的支持较低。
中文词频统计仅包含中文，英文词频统计仅包含英文。可在中文解析器与英文解析器中进行修改。
在获取到排序好的词频后，可使用d3.cloud在web中展示出来。
Github地址：github.com/panzejia/WebGuide

package cn.iflin.project.participle;

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.DocsEnum;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.Version;
import org.wltea.analyzer.lucene.IKAnalyzer;

import cn.iflin.project.model.WordModel;
import cn.iflin.project.participle.englishwords.EnglishParser;
import cn.iflin.project.participle.wordcloud.ChineseParser;
/**
主要进行对中文与英语进行词频计算
*/
public class WordsParticiple {
    /**
     * 判断文件夹是否存在之后调用索引计算词频
     */
    private static ArrayList<WordModel> checkFile(String text, String articleId) {
        ArrayList<WordModel> wordList = new ArrayList<WordModel>();
        return wordList;
    }


    private static void addDoc(IndexWriter w, String text) throws IOException {
        Document doc = new Document();
        FieldType ft = new FieldType();
        ft.setIndexed(true);// 存储
        ft.setStored(true);// 索引
        ft.setStoreTermVectors(true);
        ft.setTokenized(true);
        ft.setStoreTermVectorPositions(true);// 存储位置
        ft.setStoreTermVectorOffsets(true);// 存储偏移量
        doc.add(new Field("text", text, ft));
        w.addDocument(doc);
    }

    /**
     * 删除文件
     * 
     * @param dir
     * @return
     */
    private static boolean deleteDir(File dir) {
        if (dir.isDirectory()) {
            String[] children = dir.list();
            for (int i = 0; i < children.length; i++) {
                boolean success = deleteDir(new File(dir, children[i]));
                if (!success) {
                    return false;
                }
            }
        }
        // 目录此时为空，可以删除
        return dir.delete();
    }

    /**
     * 计算英文词频
     * 
     * @param text
     * @return
     * @throws IOException
     */
     public static ArrayList<WordModel> getTF(String text, String articleId, String tag) throws IOException {
        ArrayList<WordModel> wordList = new ArrayList<WordModel>();
        File file = new File("C:\\Spider\\WordCloud_Lucene\\" + articleId);
        WordsParticiple.deleteDir(file);
        Analyzer analyzer = new IKAnalyzer(true);// 智能分词模式，如果构造函数参数为false，那么使用最细粒度分词。
        IndexWriterConfig configfile = new IndexWriterConfig(Version.LUCENE_47, analyzer);// 创建索引的配置信息
        Directory fileindex;
        fileindex = FSDirectory.open(file);
        IndexWriter filew = new IndexWriter(fileindex, configfile);
        try {
            WordsParticiple.addDoc(filew, text);
        } finally {
            // 统一释放内存
            filew.close();
        }
        try {
            IndexReader reader = DirectoryReader.open(fileindex);
            for (int i = 0; i < reader.numDocs(); i++) {
                int docId = i;
                Terms terms = reader.getTermVector(docId, "text");
                if (terms == null)
                    continue;
                TermsEnum termsEnum = terms.iterator(null);
                BytesRef thisTerm = null;
                while ((thisTerm = termsEnum.next()) != null) {
                    String termText = thisTerm.utf8ToString();
                    DocsEnum docsEnum = termsEnum.docs(null, null);
                    while ((docsEnum.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
                        if(tag.equals("noLevel")){
                            WordModel wm = new WordModel();
                            wm.setWord(termText);
                            wm.setWordFrequency(docsEnum.freq());
                            wordList.add(wm);
                        }
                        else if (EnglishParser.checkEnglishWord(termText, tag)) {
                            WordModel wm = new WordModel();
                            wm.setWord(termText);
                            wm.setWordFrequency(docsEnum.freq());
                            wordList.add(wm);
                        }
                    }
                }
            }
            reader.close();
            fileindex.close();
        } catch (Exception e) {
            e.printStackTrace();
        }
        return wordList;
    }

    /**
     * 计算中文词频
     * 
     * @param text
     * @return
     * @throws IOException
     */
     public static ArrayList<WordModel> getTF(String text, String articleId) throws IOException {
        ArrayList<WordModel> wordList = new ArrayList<WordModel>();
        File file = new File("C:\\Spider\\WordCloud_Lucene\\" + articleId);
        WordsParticiple.deleteDir(file);
        Analyzer analyzer = new IKAnalyzer(true);// 智能分词模式，如果构造函数参数为false，那么使用最细粒度分词。
        IndexWriterConfig configfile = new IndexWriterConfig(Version.LUCENE_47, analyzer);// 创建索引的配置信息
        Directory fileindex;
        fileindex = FSDirectory.open(file);
        IndexWriter filew = new IndexWriter(fileindex, configfile);
        try {
            WordsParticiple.addDoc(filew, text);
        } finally {
            // 统一释放内存
            filew.close();
        }
        try {
            IndexReader reader = DirectoryReader.open(fileindex);
            for (int i = 0; i < reader.numDocs(); i++) {
                int docId = i;
                Terms terms = reader.getTermVector(docId, "text");
                if (terms == null)
                    continue;
                TermsEnum termsEnum = terms.iterator(null);
                BytesRef thisTerm = null;
                while ((thisTerm = termsEnum.next()) != null) {
                    String termText = thisTerm.utf8ToString();
                    DocsEnum docsEnum = termsEnum.docs(null, null);
                    while ((docsEnum.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
                        if (ChineseParser.isChinese(termText) && termText.length() >= 2) {
                            WordModel wm = new WordModel();
                            wm.setWord(termText);
                            wm.setWordFrequency(docsEnum.freq());
                            wordList.add(wm);
                        }
                    }
                }
            }
            reader.close();
            fileindex.close();
        } catch (Exception e) {
            e.printStackTrace();
        }
        return wordList;
    }

}

接下来对英语进行词频排序

package cn.iflin.project.participle.englishwords;

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.DocsEnum;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.Version;
import org.wltea.analyzer.lucene.IKAnalyzer;

import cn.iflin.project.model.WordModel;
import cn.iflin.project.participle.WordsParticiple;
import cn.iflin.project.participle.wordcloud.CalculateChineseFrequency;

public class CalculateEnglishFrequency extends WordsParticiple {
    /**
     * 返回排序后的词频
     * 
     * @param text 需分词内容
     * @param articleId 分词内容 属性 （数据库or用户自定义(temp)）
     * @param tag English：siji、liuji、kaoyan;Chinese：chinese
     * @return
     */
    public static ArrayList<WordModel> getWordFre(String text, String articleId, String tag) {
        ArrayList<WordModel> wordList = new ArrayList<WordModel>();
        try {
            //过滤掉常见标点符号
            text = EnglishParser.delPunctuation(text);
            wordList = getTF(text, articleId, tag);
        } catch (IOException e) {
            e.printStackTrace();
        }
        // 定义排序规则
        class SortByFre implements Comparator {
            public int compare(Object o1, Object o2) {
                WordModel s1 = (WordModel) o1;
                WordModel s2 = (WordModel) o2;
                return s2.getWordFrequency().compareTo(s1.getWordFrequency());
            }
        }
        Collections.sort(wordList, new SortByFre());
        return wordList;
    }
}

最后是对中文词频进行排序

package cn.iflin.project.participle.wordcloud;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.DocsEnum;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.Version;
import org.wltea.analyzer.lucene.IKAnalyzer;

import cn.iflin.project.model.WordModel;
import cn.iflin.project.participle.WordsParticiple;
import cn.iflin.project.participle.englishwords.EnglishParser;

/**
 * 计算词频
 * 
 * @author Jaypan
 *
 */
public class CalculateChineseFrequency  extends WordsParticiple{

    /**
     * 返回排序后的词频
     * 
     * @param text 需分词内容
     * @param articleId 分词内容 属性 （数据库or用户自定义(temp)）
     * @param tag English：siji、liuji、kaoyan;Chinese：chinese
     * @return
     */
    public static ArrayList<WordModel> getWordFre(String text, String articleId) {
        ArrayList<WordModel> wordList = new ArrayList<WordModel>();
        try {
            wordList = getTF(text, articleId);
        } catch (IOException e) {
            e.printStackTrace();
        }
        // 定义排序规则
        class SortByFre implements Comparator {
            public int compare(Object o1, Object o2) {
                WordModel s1 = (WordModel) o1;
                WordModel s2 = (WordModel) o2;
                return s2.getWordFrequency().compareTo(s1.getWordFrequency());
            }
        }
        Collections.sort(wordList, new SortByFre());
        return wordList;
    }
}

英语解析器

package cn.iflin.project.participle.englishwords;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class EnglishParser {
//  public static void main(String[] args) {
//      testReg("B. Closing offices on holidays.");
//  }

    public static String delPunctuation(String text) {
        // 先去掉标点,再合并空格
        Pattern p = Pattern.compile("[(.|,|\"|\\?|!|:;')]");// 这边增加所有的符号,例如要加一个'则变成[(.|,|\"|\\?|!|:|')],如果是特殊符号要加转换
        Matcher m = p.matcher(text);// 这为要整理的字符串
        String first = m.replaceAll("");
        p = Pattern.compile("   {2,}");
        m = p.matcher(first);
        String second = m.replaceAll("");
        return second;
    }
    //判断单词是否属于某一分类
    public static boolean checkEnglishWord(String checkWord, String englishClass) {
        ArrayList<String> words = getEnglishWords(englishClass);
        for (String word : words) {
            if (word.equals(checkWord)) {
                return true;
            }
        }
        return false;
    }

    /**
     * 获取单词列表
     * 
     * @param sourceName
     *            单词级别
     * @return 单词列表
     */
    public static ArrayList<String> getEnglishWords(String sourceName) {
        ArrayList<String> words = new ArrayList<String>();
        String filePath = "C:\\Spider\\EnglishWords\\" + sourceName + ".txt";
        try {
            String encoding = "GBK";
            File file = new File(filePath);
            if (file.isFile() && file.exists()) {
                InputStreamReader read = new InputStreamReader(new FileInputStream(file), encoding);
                BufferedReader br = new BufferedReader(read);
                String lineText = null;
                while ((lineText = br.readLine()) != null) {
                    words.add(lineText);
                }
                read.close();
            } else {
                System.out.println("找不到指定文件");
            }
        } catch (Exception e) {
            System.out.println("读取文件出错");
            e.printStackTrace();
        }
        return words;

    }
}

中文解析器

package cn.iflin.project.participle.wordcloud;

import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class ChineseParser {
    // 判断一个字符是否是中文
    private static boolean isChinese(char c) {
        return c >= 0x4E00 && c <= 0x9FA5;// 根据字节码判断
    }

    // 判断一个字符串是否含有中文
    public static boolean isChinese(String str) {
        if (str == null)
            return false;
        for (char c : str.toCharArray()) {
            if (isChinese(c))
                return true;// 有一个中文字符就返回
        }
        return false;
    }
    //将一句话转换成数组
    public static String[] changeList(String s) {
        String[] data =new String[20];
        Pattern pattern =  Pattern.compile("[\u4e00-\u9fa5]*");
        Matcher matcher = pattern.matcher(s);
        int i =0;
        while (matcher.find()){
            if(matcher.group().equals("")){
                continue;
            }
            data[i]=matcher.group();
            i++;
        }
        return data;
    }
}