Lucene学习4 Tika内容提取

Tika提取富文本文档简单例子

首先编译Tika，基本步骤：

1. 下载Maven3.0，解压，设置bin环境变量，“mvn -version” 命令会显示maven相关信息；

2. 下载Tika0.7，解压，进入当前目录，“mvn install" 命令会编译Tika，生成相关的jar文件等；

添加tika-app-0.7.jar到java项目编译目录。

package com.l99.lucene.index;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Set;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.Version;
import org.apache.tika.config.TikaConfig;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.parser.AutoDetectParser;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.Parser;
import org.apache.tika.sax.BodyContentHandler;
import org.xml.sax.ContentHandler;

public class TikaIndexer {
private static Set<String> textMetadataFields = new HashSet<String>();

static {
textMetadataFields.add(Metadata.TITLE);
textMetadataFields.add(Metadata.AUTHOR);
textMetadataFields.add(Metadata.COMMENTS);
textMetadataFields.add(Metadata.KEYWORDS);
textMetadataFields.add(Metadata.DESCRIPTION);
textMetadataFields.add(Metadata.SUBJECT);
}

public static void main(String[] args) throws Exception {
if (args.length != 2) {
throw new IllegalArgumentException("Usage: java" +
TikaIndexer.class.getName() + " <index dir> <data dir>");
}

TikaConfig config = TikaConfig.getDefaultConfig();
List<String> parsers = new ArrayList<String>(config.getParsers().keySet());

Collections.sort(parsers);
Iterator<String> itor = parsers.iterator();
System.out.println("Mime type parsers:");

while(itor.hasNext()) {
System.out.println(" " + itor.next());
}
System.out.println();

String indexDir = args[0];
String dataDir = args[1];

long start = System.currentTimeMillis();
int numIndexed = index(indexDir, dataDir);
long end = System.currentTimeMillis();

System.out.println("Indexing " + numIndexed + " files took " +
(end - start) + " milliseconds");
}

public static int index(String indexDir, String dataDir)
throws Exception {
File data = new File(dataDir);
if (!data.exists() || !data.isDirectory()) {
throw new IOException(data + " does not exist or isn't a dir");
}

Directory index = FSDirectory.open(new File(indexDir));
Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_30);
IndexWriter writer = new IndexWriter(index, analyzer, true,
IndexWriter.MaxFieldLength.UNLIMITED);

indexDirectory(writer, data);
writer.optimize();
writer.close();

return writer.numDocs();
}

private static void indexDirectory(IndexWriter writer, File data)
throws IOException, Exception {
File[] datas = data.listFiles();

for (File f: datas) {
if (f.isDirectory()) {
indexDirectory(writer, f);
} else {
indexFile(writer, f);
}
}
}

private static void indexFile(IndexWriter writer, File f)
throws IOException, Exception {
if (f.isHidden() || !f.exists() || !f.canRead()) {
return;
}

System.out.println("Indexing " + f.getCanonicalPath());
Document doc = getDocument(f);
writer.addDocument(doc);
}

private static Document getDocument(File f) throws Exception {
Metadata metadata = new Metadata();
metadata.set(Metadata.RESOURCE_NAME_KEY, f.getName());

InputStream stream = new FileInputStream(f);
Parser parser = new AutoDetectParser();

ContentHandler handler = new BodyContentHandler();
ParseContext context = new ParseContext();
context.set(Parser.class, parser);

try {
parser.parse(stream, handler, metadata, new ParseContext());
} finally {
stream.close();
}

Document doc = new Document();
doc.add(new Field("contents", handler.toString(),
Field.Store.YES, Field.Index.ANALYZED));

for (String name: metadata.names()) {
String value = metadata.get(name);
if (textMetadataFields.contains(name)) {
doc.add(new Field("contents", value,
Field.Store.NO, Field.Index.ANALYZED));
}
doc.add(new Field(name, value,
Field.Store.YES, Field.Index.NO));
}

doc.add(new Field("pathname", f.getCanonicalPath(),
Field.Store.YES, Field.Index.NOT_ANALYZED));
doc.add(new Field("filename", f.getName(),
Field.Store.YES, Field.Index.NOT_ANALYZED));

return doc;
}
}

运行参数：index/richtext data/richtext

运行结果：

Mime type parsers:

application/epub+zip

application/java-vm

application/mbox

application/msword

application/pdf

application/rtf

application/vnd.ms-excel

application/vnd.ms-excel.addin.macroenabled.12

application/vnd.ms-excel.sheet.binary.macroenabled.12

application/vnd.ms-excel.sheet.macroenabled.12

application/vnd.ms-excel.template.macroenabled.12

application/vnd.ms-outlook

application/vnd.ms-powerpoint

application/vnd.ms-powerpoint.addin.macroenabled.12

application/vnd.ms-powerpoint.presentation.macroenabled.12

application/vnd.ms-powerpoint.slideshow.macroenabled.12

application/vnd.ms-word.document.macroenabled.12

application/vnd.ms-word.template.macroenabled.12

application/vnd.oasis.opendocument.chart

application/vnd.oasis.opendocument.chart-template

application/vnd.oasis.opendocument.formula

application/vnd.oasis.opendocument.formula-template

application/vnd.oasis.opendocument.graphics

application/vnd.oasis.opendocument.graphics-template

application/vnd.oasis.opendocument.image

application/vnd.oasis.opendocument.image-template

application/vnd.oasis.opendocument.presentation

application/vnd.oasis.opendocument.presentation-template

application/vnd.oasis.opendocument.spreadsheet

application/vnd.oasis.opendocument.spreadsheet-template

application/vnd.oasis.opendocument.text

application/vnd.oasis.opendocument.text-master

application/vnd.oasis.opendocument.text-template

application/vnd.oasis.opendocument.text-web

application/vnd.openxmlformats-officedocument.presentationml.presentation

application/vnd.openxmlformats-officedocument.presentationml.slideshow

application/vnd.openxmlformats-officedocument.presentationml.template

application/vnd.openxmlformats-officedocument.spreadsheetml.sheet

application/vnd.openxmlformats-officedocument.spreadsheetml.template

application/vnd.openxmlformats-officedocument.wordprocessingml.document

application/vnd.openxmlformats-officedocument.wordprocessingml.template

application/vnd.sun.xml.writer

application/vnd.visio

application/vnd.wap.xhtml+xml

application/x-archive

application/x-asp

application/x-bzip

application/x-bzip2

application/x-cpio

application/x-gtar

application/x-gzip

application/x-midi

application/x-tar

application/x-tika-msoffice

application/x-tika-ooxml

application/x-vnd.oasis.opendocument.chart

application/x-vnd.oasis.opendocument.chart-template

application/x-vnd.oasis.opendocument.formula

application/x-vnd.oasis.opendocument.formula-template

application/x-vnd.oasis.opendocument.graphics

application/x-vnd.oasis.opendocument.graphics-template

application/x-vnd.oasis.opendocument.image

application/x-vnd.oasis.opendocument.image-template

application/x-vnd.oasis.opendocument.presentation

application/x-vnd.oasis.opendocument.presentation-template

application/x-vnd.oasis.opendocument.spreadsheet

application/x-vnd.oasis.opendocument.spreadsheet-template

application/x-vnd.oasis.opendocument.text

application/x-vnd.oasis.opendocument.text-master

application/x-vnd.oasis.opendocument.text-template

application/x-vnd.oasis.opendocument.text-web

application/xhtml+xml

application/xml

application/zip

audio/basic

audio/midi

audio/mpeg

audio/x-aiff

audio/x-wav

image/bmp

image/gif

image/jpeg

image/png

image/svg+xml

image/tiff

image/vnd.wap.wbmp

image/x-icon

image/x-psd

image/x-xcf

text/html

text/plain

video/x-flv

Indexing D:\eclipse\eclipse\workspace\Lucene\data\richtext\addressbook-entry.xml

Indexing D:\eclipse\eclipse\workspace\Lucene\data\richtext\addressbook.xml

Indexing D:\eclipse\eclipse\workspace\Lucene\data\richtext\HTML.html

Indexing D:\eclipse\eclipse\workspace\Lucene\data\richtext\MSWord.doc

Indexing D:\eclipse\eclipse\workspace\Lucene\data\richtext\MSWordXML.docx

Indexing D:\eclipse\eclipse\workspace\Lucene\data\richtext\PDF.pdf

log4j:WARN No appenders could be found for logger (org.apache.pdfbox.util.PDFStreamEngine).

log4j:WARN Please initialize the log4j system properly.

Indexing D:\eclipse\eclipse\workspace\Lucene\data\richtext\PlainText.txt

Indexing D:\eclipse\eclipse\workspace\Lucene\data\richtext\RTF.rtf

Indexing 8 files took 4811 milliseconds