【无标题】

2021SC@SDUSC

nutch源码分析—invertlinks
命令“bin/nutch invertlinks crawl/linkdb -dir crawl/segments”最终会调用org.apache.nutch.crawl.LinkDb的main函数。

LinkDb::main

public static void main(String[] args) throws Exception {
int res = ToolRunner.run(NutchConfiguration.create(), new LinkDb(), args);
System.exit(res);
}

ToolRunner的run函数最终调用LinkDb的run函数。

LinkDb::run

public int run(String[] args) throws Exception {
invert(db, segs.toArray(new Path[segs.size()]), normalize, filter, force);
return 0;
}

public void invert(Path linkDb, Path[] segments, boolean normalize,
boolean filter, boolean force) throws IOException {
JobConf job = LinkDb.createJob(getConf(), linkDb, normalize, filter);
Path lock = new Path(linkDb, LOCK_NAME);
FileSystem fs = FileSystem.get(getConf());
LockUtil.createLockFile(fs, lock, force);
Path currentLinkDb = new Path(linkDb, CURRENT_NAME);

for (int i = 0; i < segments.length; i++) {
  FileInputFormat.addInputPath(job, new Path(segments[i],
      ParseData.DIR_NAME));
}
JobClient.runJob(job);
if (fs.exists(currentLinkDb)) {
  Path newLinkDb = FileOutputFormat.getOutputPath(job);
  job = LinkDbMerger.createMergeJob(getConf(), linkDb, normalize, filter);
  FileInputFormat.addInputPath(job, currentLinkDb);
  FileInputFormat.addInputPath(job, newLinkDb);
  JobClient.runJob(job);
  fs.delete(newLinkDb, true);
}
LinkDb.install(job, linkDb);

}

private static JobConf createJob(Configuration config, Path linkDb,
boolean normalize, boolean filter) {
Path newLinkDb = new Path(“linkdb-”
+ Integer.toString(new Random().nextInt(Integer.MAX_VALUE)));

JobConf job = new NutchJob(config);
job.setJobName("linkdb " + linkDb);
job.setInputFormat(SequenceFileInputFormat.class);
job.setMapperClass(LinkDb.class);
job.setCombinerClass(LinkDbMerger.class);

if (normalize || filter) {
  FileSystem fs = FileSystem.get(config);
  if (!fs.exists(linkDb)) {
    job.setBoolean(LinkDbFilter.URL_FILTERING, filter);
    job.setBoolean(LinkDbFilter.URL_NORMALIZING, normalize);
  }
}
job.setReducerClass(LinkDbMerger.class);

FileOutputFormat.setOutputPath(job, newLinkDb);
job.setOutputFormat(MapFileOutputFormat.class);
job.setBoolean("mapred.output.compress", true);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Inlinks.class);

return job;

}

public static JobConf createMergeJob(Configuration config, Path linkDb,
boolean normalize, boolean filter) {
Path newLinkDb = new Path(“linkdb-merge-”
+ Integer.toString(new Random().nextInt(Integer.MAX_VALUE)));

JobConf job = new NutchJob(config);
job.setJobName("linkdb merge " + linkDb);

job.setInputFormat(SequenceFileInputFormat.class);

job.setMapperClass(LinkDbFilter.class);
job.setBoolean(LinkDbFilter.URL_NORMALIZING, normalize);
job.setBoolean(LinkDbFilter.URL_FILTERING, filter);
job.setReducerClass(LinkDbMerger.class);

FileOutputFormat.setOutputPath(job, newLinkDb);
job.setOutputFormat(MapFileOutputFormat.class);
job.setBoolean("mapred.output.compress", true);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Inlinks.class);

return job;

}