2021SC@SDUSC
nutch源码分析—invertlinks
命令“bin/nutch invertlinks crawl/linkdb -dir crawl/segments”最终会调用org.apache.nutch.crawl.LinkDb的main函数。
LinkDb::main
public static void main(String[] args) throws Exception {
int res = ToolRunner.run(NutchConfiguration.create(), new LinkDb(), args);
System.exit(res);
}
ToolRunner的run函数最终调用LinkDb的run函数。
LinkDb::run
public int run(String[] args) throws Exception {
invert(db, segs.toArray(new Path[segs.size()]), normalize, filter, force);
return 0;
}
public void invert(Path linkDb, Path[] segments, boolean normalize,
boolean filter, boolean force) throws IOException {
JobConf job = LinkDb.createJob(getConf(), linkDb, normalize, filter);
Path lock = new Path(linkDb, LOCK_NAME);
FileSystem fs = FileSystem.get(getConf());
LockUtil.createLockFile(fs, lock, force);
Path currentLinkDb = new Path(linkDb, CURRENT_NAME);
for (int i = 0; i < segments.length; i++) {
FileInputFormat.addInputPath(job, new Path(segments[i],
ParseData.DIR_NAME));
}
JobClient.runJob(job);
if (fs.exists(currentLinkDb)) {
Path newLinkDb = FileOutputFormat.getOutputPath(job);
job = LinkDbMerger.createMergeJob(getConf(), linkDb, normalize, filter);
FileInputFormat.addInputPath(job, currentLinkDb);
FileInputFormat.addInputPath(job, newLinkDb);
JobClient.runJob(job);
fs.delete(newLinkDb, true);
}
LinkDb.install(job, linkDb);
}
private static JobConf createJob(Configuration config, Path linkDb,
boolean normalize, boolean filter) {
Path newLinkDb = new Path(“linkdb-”
+ Integer.toString(new Random().nextInt(Integer.MAX_VALUE)));
JobConf job = new NutchJob(config);
job.setJobName("linkdb " + linkDb);
job.setInputFormat(SequenceFileInputFormat.class);
job.setMapperClass(LinkDb.class);
job.setCombinerClass(LinkDbMerger.class);
if (normalize || filter) {
FileSystem fs = FileSystem.get(config);
if (!fs.exists(linkDb)) {
job.setBoolean(LinkDbFilter.URL_FILTERING, filter);
job.setBoolean(LinkDbFilter.URL_NORMALIZING, normalize);
}
}
job.setReducerClass(LinkDbMerger.class);
FileOutputFormat.setOutputPath(job, newLinkDb);
job.setOutputFormat(MapFileOutputFormat.class);
job.setBoolean("mapred.output.compress", true);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Inlinks.class);
return job;
}
public static JobConf createMergeJob(Configuration config, Path linkDb,
boolean normalize, boolean filter) {
Path newLinkDb = new Path(“linkdb-merge-”
+ Integer.toString(new Random().nextInt(Integer.MAX_VALUE)));
JobConf job = new NutchJob(config);
job.setJobName("linkdb merge " + linkDb);
job.setInputFormat(SequenceFileInputFormat.class);
job.setMapperClass(LinkDbFilter.class);
job.setBoolean(LinkDbFilter.URL_NORMALIZING, normalize);
job.setBoolean(LinkDbFilter.URL_FILTERING, filter);
job.setReducerClass(LinkDbMerger.class);
FileOutputFormat.setOutputPath(job, newLinkDb);
job.setOutputFormat(MapFileOutputFormat.class);
job.setBoolean("mapred.output.compress", true);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Inlinks.class);
return job;
}