OpenNLP入门实验

OpenNLP

也是自然语言分析处理的工具是apache的开源项目，感觉这玩意儿更容易入手，使用的人更多。

句子探测的例子

注意，首先，添加opennlp的pom依赖

<!-- OpenNLP -->
    <dependency>
        <groupId>org.apache.opennlp</groupId>
        <artifactId>opennlp-tools</artifactId>
        <version>1.6.0</version>
    </dependency>

接着还得下载训练集

地址是http://opennlp.sourceforge.net/models-1.5/，例如句子探测，分词都有探测模型使用的是en-sent.bin。

在使用的时候得添加训练模型的文件路径

String path="E:/EclipseWorkSpace/hibernettest/src/main/resources/"
            + "com/hainan/cs/OpenNLPModels/en-sent.bin";

    // always start with a model, a model is learned from training data
    InputStream is = new FileInputStream(path);
    SentenceModel model = new SentenceModel(is);
    SentenceDetectorME sdetector = new SentenceDetectorME(model);

句子探测代码：

package com.hainan.cs.opennlp;

import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;

import opennlp.tools.sentdetect.SentenceDetectorME;
import opennlp.tools.sentdetect.SentenceModel;

public class SentenceDetector {
public static void main(String args[]) throws IOException {
    String paragraph = "Hi. How are you? This is Mike.";
    String path="E:/EclipseWorkSpace/hibernettest/src/main/resources/"
            + "com/hainan/cs/OpenNLPModels/en-sent.bin";

    // always start with a model, a model is learned from training data
    InputStream is = new FileInputStream(path);
    SentenceModel model = new SentenceModel(is);
    SentenceDetectorME sdetector = new SentenceDetectorME(model);

    String sentences[] = sdetector.sentDetect(paragraph);

    System.out.println(sentences[0]);
    System.out.println(sentences[1]);
    is.close();
}
}

句子探测结果：

分词的例子

直接上代码，使用的模型在代码里

package com.hainan.cs.opennlp;

import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;

import opennlp.tools.tokenize.TokenizerME;
import opennlp.tools.tokenize.TokenizerModel;
import opennlp.tools.util.InvalidFormatException;

public class Tokenizer {
    public static void main(String args[]) throws InvalidFormatException, IOException {
        String paragraph = "Hi. How are you? This is Mike.";
        String path = "E:/EclipseWorkSpace/hibernettest/src/main/resources/"
            + "com/hainan/cs/OpenNLPModels/en-token.bin";
        InputStream is = new FileInputStream(path);

        TokenizerModel model = new TokenizerModel(is);

        TokenizerME tokenizer = new TokenizerME(model);

        String tokens[] = tokenizer.tokenize("Hi. How are you? This is Mike.");

        for (String a : tokens)
        System.out.println(a);

        is.close();
    }
}

结果：

名字查找，比如说要识别哪些是人命

package com.hainan.cs.opennlp;

import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;

import opennlp.tools.namefind.NameFinderME;
import opennlp.tools.namefind.TokenNameFinderModel;
import opennlp.tools.util.Span;

public class NameFinder {
    public static void main(String args[]) throws IOException {

        String paragraph = "Hi. How are you? This is Mike.";
        String path = "E:/EclipseWorkSpace/hibernettest/src/main/resources/"
            + "com/hainan/cs/OpenNLPModels/en-ner-person.bin";
        InputStream is = new FileInputStream(path);

        TokenNameFinderModel model = new TokenNameFinderModel(is);
        is.close();

        NameFinderME nameFinder = new NameFinderME(model);

        String[] sentence = new String[] { "Mike", "Smith", "is", "a", "good", "person" };

        Span nameSpans[] = nameFinder.find(sentence);

        for (Span s : nameSpans)
        System.out.println(s.toString());
    }
}

词性标注

package com.hainan.cs.opennlp;

import java.io.File;
import java.io.IOException;
import java.io.StringReader;

import opennlp.tools.cmdline.PerformanceMonitor;
import opennlp.tools.cmdline.postag.POSModelLoader;
import opennlp.tools.postag.POSModel;
import opennlp.tools.postag.POSSample;
import opennlp.tools.postag.POSTaggerME;
import opennlp.tools.tokenize.WhitespaceTokenizer;
import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.PlainTextByLineStream;

public class POS {
    public static void main(String args[]) throws IOException {

        String paragraph = "Hi. How are you? This is Mike.";
        String path = "E:/EclipseWorkSpace/hibernettest/src/main/resources/"
            + "com/hainan/cs/OpenNLPModels/en-pos-maxent.bin";
        POSModel model = new POSModelLoader().load(new File(path));
        PerformanceMonitor perfMon = new PerformanceMonitor(System.err, "sent");
        POSTaggerME tagger = new POSTaggerME(model);

        String input = "Hi. How are you? This is Mike.";
        ObjectStream<String> lineStream = new PlainTextByLineStream(new StringReader(input));

        perfMon.start();
        String line;
        while ((line = lineStream.read()) != null) {

            String whitespaceTokenizerLine[] = WhitespaceTokenizer.INSTANCE.tokenize(line);
            String[] tags = tagger.tag(whitespaceTokenizerLine);

            POSSample sample = new POSSample(whitespaceTokenizerLine, tags);
            System.out.println(sample.toString());

            perfMon.incrementCounter();
        }
        perfMon.stopAndPrintFinalResult();
    }
}

结果：

Chunker

package com.hainan.cs.opennlp;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.StringReader;

import opennlp.tools.chunker.ChunkerME;
import opennlp.tools.chunker.ChunkerModel;
import opennlp.tools.cmdline.PerformanceMonitor;
import opennlp.tools.cmdline.postag.POSModelLoader;
import opennlp.tools.postag.POSModel;
import opennlp.tools.postag.POSSample;
import opennlp.tools.postag.POSTaggerME;
import opennlp.tools.tokenize.WhitespaceTokenizer;
import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.PlainTextByLineStream;
import opennlp.tools.util.Span;

public class Chunker {
    public static void main(String args[]) throws IOException {

        String path = "E:/EclipseWorkSpace/hibernettest/src/main/resources/"
            + "com/hainan/cs/OpenNLPModels/en-pos-maxent.bin";
        POSModel model = new POSModelLoader().load(new File(path));
        PerformanceMonitor perfMon = new PerformanceMonitor(System.err, "sent");
        POSTaggerME tagger = new POSTaggerME(model);

        String input = "Hi. How are you? This is Mike.";
        ObjectStream<String> lineStream = new PlainTextByLineStream(new StringReader(input));

        perfMon.start();
        String line;
        String whitespaceTokenizerLine[] = null;

        String[] tags = null;
        while ((line = lineStream.read()) != null) {
            whitespaceTokenizerLine = WhitespaceTokenizer.INSTANCE.tokenize(line);
            tags = tagger.tag(whitespaceTokenizerLine);

            POSSample sample = new POSSample(whitespaceTokenizerLine, tags);
            System.out.println(sample.toString());
            perfMon.incrementCounter();
        }
        perfMon.stopAndPrintFinalResult();

        // chunker
        String path1 = "E:/EclipseWorkSpace/hibernettest/src/main/resources/"
            + "com/hainan/cs/OpenNLPModels/en-chunker.bin";

        InputStream is = new FileInputStream(path1);
        ChunkerModel cModel = new ChunkerModel(is);

        ChunkerME chunkerME = new ChunkerME(cModel);
        String result[] = chunkerME.chunk(whitespaceTokenizerLine, tags);

        for (String s : result)
            System.out.println(s);

        Span[] span = chunkerME.chunkAsSpans(whitespaceTokenizerLine, tags);
        for (Span s : span)
            System.out.println(s.toString());
    }
}

结果： 看不懂这是个啥求，先放着吧

分析器parser

package com.hainan.cs.opennlp;

import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;

import opennlp.tools.cmdline.parser.ParserTool;
import opennlp.tools.parser.Parse;
import opennlp.tools.parser.ParserFactory;
import opennlp.tools.parser.ParserModel;
import opennlp.tools.util.InvalidFormatException;

public class Parser {

    public static void main(String args[]) throws InvalidFormatException,   IOException {
        // http://sourceforge.net/apps/mediawiki/opennlp/index.php?title=Parser#Training_Tool

        String path = "E:/EclipseWorkSpace/hibernettest/src/main/resources/"
            + "com/hainan/cs/OpenNLPModels/en-parser-chunking.bin";
        InputStream is = new FileInputStream(path);

        ParserModel model = new ParserModel(is);

        Parser parser = ParserFactory.create(model);

        String sentence = "Programcreek is a very huge and useful website.";
        Parse topParses[] = ParserTool.parseLine(sentence, (opennlp.tools.parser.Parser) parser, 1);

        for (Parse p : topParses)
            p.show();

        is.close();

        /*
         * (TOP (S (NP (NN Programcreek) ) (VP (VBZ is) (NP (DT a) (ADJP (RB
         * very) (JJ huge) (CC and) (JJ useful) ) ) ) (. website.) ) )
         */
    }
}

结果：

自然语言处理两个最重要的方面就是词性标注和句法分析。例子都有了。