A program that uses several tools of opennlp

最新推荐文章于 2023-09-26 22:37:43 发布

lovelytigerphd

最新推荐文章于 2023-09-26 22:37:43 发布

阅读量987

点赞数

分类专栏：自然语言处理 NLP 文章标签： tools string null path tts tags

本文链接：https://blog.csdn.net/lovelytigerphd/article/details/5809860

版权

自然语言处理 NLP 专栏收录该内容

5 篇文章 0 订阅

订阅专栏

A program that uses several tools of opennlp:

Original URL: http://www.di.ubi.pt/~jpaulo/ensino/PLN/praticas/OpenNLP.html

package hultig.sumo; import opennlp.tools.sentdetect.SentenceDetectorME; import opennlp.tools.tokenize.TokenizerME; import opennlp.tools.postag.DefaultPOSContextGenerator; import opennlp.tools.postag.POSDictionary; import opennlp.tools.postag.POSTaggerME; import opennlp.tools.chunker.ChunkerME; import opennlp.maxent.io.SuffixSensitiveGISModelReader; import opennlp.maxent.MaxentModel; import java.io.IOException; import java.io.File; import tools.Input; /** * Title: OpenNLP Toolkit * * Description: Works as a simple interface to the OpenNLP library. * * Copyright: Copyright (c) 2007 * * Company: UBI/HULTIG * * @author JPC * @version 1.0 */ public class OpenNLPKit { private String models_path; private SentenceDetectorME sdetector= null; private TokenizerME tokenizer= null; private POSTaggerME tagger= null; private ChunkerME chunker= null; /** * Default constructor. */ public OpenNLPKit() { models_path= ""; } /** * Create this object * * @param path String */ public OpenNLPKit(String path) { setLanguageModels(path); } private boolean fileOk(String path) { File f= new File(path); if ( f.isFile() ) return true; else return false; } /** * Set the path for Open NLP language models. * Example: "C:/a/tools/opennlp-tools-1.3.0/models/english" * @param path String * @return boolean */ public boolean setLanguageModels(String path) { models_path= path; String name = models_path + "/sentdetect/EnglishSD.bin.gz"; if ( fileOk(name) ) sdetector = new SentenceDetectorME(getModel(name)); name = models_path + "/tokenize/EnglishTok.bin.gz"; if ( fileOk(name) ) { tokenizer = new TokenizerME(getModel(name)); tokenizer.setAlphaNumericOptimization(true); } name = models_path + "/parser/tag.bin.gz"; String tagdict = models_path + "/parser/tagdict"; if ( fileOk(name) && fileOk(tagdict) ) { try { tagger = new POSTaggerME( getModel(name), new DefaultPOSContextGenerator(null), new POSDictionary(tagdict, true) ); } catch (IOException ex) { ex.printStackTrace(); this.models_path= ""; return false; } } name = models_path + "/chunker/EnglishChunk.bin.gz"; if ( fileOk(name) ) chunker = new ChunkerME(getModel(name)); return true; } public boolean wellDefine() { return wellDefined(true); } public boolean wellDefined(boolean silent) { if ( sdetector == null ) { if ( !silent ) System.err.print(" Sentence detector fail!/n"); return false; } if ( tokenizer == null ) { if ( !silent ) System.err.print(" Tokenizer fail!/n"); return false; } if ( tagger == null ) { if ( !silent ) System.err.print(" Tagger fail!/n"); return false; } if ( chunker == null ) { if ( !silent ) System.err.print(" Chunker fail!/n"); return false; } return true; } /** * Extracted from file * opennlp-tools.models.english.parser.POSTagger.java * @param name - the Open NLP model file path * @return MaxentModel */ private static MaxentModel getModel(String name) { try { return new SuffixSensitiveGISModelReader(new File(name)).getModel(); } catch (IOException e) { e.printStackTrace(); return null; } } /** * Make the Part of Speech Tagging for a given * String. * @param s String * @return String */ public String postag(String s) { if ( tokenizer == null ) return null; String[] tokens = tokenizer.tokenize(s); StringBuffer tline = (tokens.length > 0) ? new StringBuffer(tokens[0]) : new StringBuffer(""); for (int ti = 1, tn = tokens.length; ti < tn; ti++) { tline.append(' ').append(tokens[ti]); } return tagger.tag(tline.toString()); } /** * Make the Part of Speech Tagging for a given * Sentence. This method will reconstruct the * Sentence object, returning a new tagged * one, where every word is POS tagged. * @param stc Sentence * @return Sentence */ public Sentence postag(Sentence stc) { if ( tagger == null ) return null; String[] vs= postag(stc.toString()).split(" "); //System.out.printf("vs.size:............ [%d]/n", vs.length); stc.clear(); for (int k=0; k<vs.length; k++) { int r= vs[k].lastIndexOf('/'); if ( r == -1 ) { stc.addWord(new Word(vs[k], "UNK")); continue; } //String[] wpos= vs[k].split("/"); //if ( wpos == null || wpos.length < 2 ) continue; String token= vs[k].substring(0,r); String tag= vs[k].substring(r+1); //System.out.printf("/t/t/tvs[%d] ---> [%s] / [%s]/n", k, token, tag); stc.addWord(new Word(token, tag)); } //System.out.printf("stc.stx:............ [%s]/n", stc.toStringPOS()); return stc; } /** * Extracted and adapted from file * opennlp-tools.models.english.chunker.TreebankChunker.java * @param tagedline String * @return String */ public String chunk(String tagedline) { if ( chunker == null ) return null; //System.out.printf("OpenNLPModel::chunk(String tagedline)tagedline:........... [%s]/n", tagedline); String[] tts = tagedline.split(" *"); //tools.print(tts); String[] tokens = new String[tts.length]; String[] tags = new String[tts.length]; for (int ti = 0, tn = tts.length; ti < tn; ti++) { try { int r= tts[ti].lastIndexOf('/'); tokens[ti] = tts[ti].substring(0, r); tags[ti] = tts[ti].substring(r+1); //System.out.printf(" [%3d] [%7s] [%s]/n", ti, tags[ti], tokens[ti]); } catch (Exception exc) { System.out.printf("---> [%s] <---/n", tts[ti]); exc.printStackTrace(); } } String[] chunks = chunker.chunk(tokens, tags); //tools.print(chunks); String finalpunct= ".?!"; //System.out.print("/n/n"); StringBuffer outline = new StringBuffer(""); for (int i = 0, n = chunks.length; i < n; i++) { //System.out.printf(" [%4s] [%7s] [%s]/n", chunks[i], tags[i], tokens[i]); if (i > 0 && !chunks[i].startsWith("I-") && !chunks[i - 1].equals("O")) { outline.append(" ]"); } if (chunks[i].startsWith("B-")) { outline.append(" [").append(chunks[i].substring(2)); } outline.append(" " + tokens[i] + "/" + tags[i]); if ( finalpunct.indexOf(tags[i]) >= 0 && i<n-1 && finalpunct.indexOf(tags[i+1]) < 0 ) outline.append("/n/n"); } if (!chunks[chunks.length - 1].equals("O")) { outline.append(" ]"); } //System.out.print("/n/n"); //System.out.println(outline.toString()); return outline.toString(); } /** * Produces a chunked sentence string * @param stc Sentence * @return String */ public String chunk(Sentence stc) { if ( tagger == null || chunker == null ) return null; if ( stc == null ) return null; if ( stc.get(0).getTag().length() == 0 ) stc= postag(stc); return chunk(stc.toStringPOS()); } public static void help() { System.out.print("/nHELP:/n"); System.out.print(" pos <sentence> - mark every word with its POS tag./n"); System.out.print(" chk <sentence> - makes the sentence shallow parsing./n"); } /** * Demonstarte this class main features. * @param args String[] */ public static void main(String[] args) { System.out.print("/nLOAD ENGLISH MODELS ... "); OpenNLPKit model = new OpenNLPKit("/a/tools/opennlp-tools-1.3.0/models/english"); System.out.print("OK/n"); System.out.print("/nEXAMPLE:/n"); String s= "Guatemala accepts the Pet passport and paper, as proof of vaccination?"; System.out.printf("SENTENCE s:... %s/n", s); String spos= model.postag(s); System.out.printf("POS(s):....... %s/n", spos); String schk= model.chunk(spos); System.out.printf("CHUNK(s):.....%s/n", schk); Input in= new Input(); for (;;) { System.out.print("/ninput> "); String command= in.readLn(); if ( command == null ) break; command= command.trim(); if ( command.length() == 0 ) break; if ( command.equals("?") || command.equals("help") ) { help(); continue; } int p= command.indexOf(' '); if ( p < 0 ) continue; String func= command.substring(0,p).trim(); String sent= command.substring(p+1).trim(); if ( func.equals("pos") ) { System.out.printf("%s/n", model.postag(sent)); } else if ( func.equals("chk") || func.equals("chunk") ) { String pos= model.postag(sent); System.out.printf("%s/n", model.chunk(pos)); } } } }