读取nutch内容有如下两种方法:
1 通过Nutch api SegmentReader读取。
public Content readSegment(String segPath,String url){
Text key= new Text(url);
Path path= new Path(segPath);
Content content = null;
ArrayList<Writable> parsedLst = null;
Map<String,List<Writable>> results= new HashMap<String, List<Writable>>();
SegmentReader reader= new SegmentReader(configuration, true, true, true, true, true, true);
try {
reader.get(path, key, new StringWriter(), results);
parsedLst=(ArrayList<Writable>) results.get("co");
Iterator<Writable> parseIter=parsedLst.iterator();
while(parseIter.hasNext()){
content=(Content) parseIter.next();
}
} catch (Exception e) {
e.printStackTrace();
}
return content;
}
2 通过SequenceFile 读取
public static void main(String[] args) throws IOException {
Configuration conf = NutchConfiguration.create();
Options opts = new Options();
GenericOptionsParser parser = new GenericOptionsParser(conf, opts, args);
String[] remainingArgs = parser.getRemainingArgs();
FileSystem fs = FileSystem.get(conf);
String segment = remainingArgs[0];
Path file = new Path(segment, Content.DIR_NAME + "/part-00000/data");
SequenceFile.Reader reader = new SequenceFile.Reader(fs, file, conf);
Text key = new Text();
Content content = new Content();
// Loop through sequence files
while (reader.next(key, content)) {
try {
System.out.write(content.getContent(), 0,
content.getContent().length);
} catch (Exception e) {
}
}
}