java sort set_Java Job.setSortComparatorClass方法代码示例

import org.apache.hadoop.mapreduce.Job; //导入方法依赖的package包/类

private void testComparator(String keySpec, int expect)

throws Exception {

String root = System.getProperty("test.build.data", "/tmp");

Path inDir = new Path(root, "test_cmp/in");

Path outDir = new Path(root, "test_cmp/out");

conf.set("mapreduce.partition.keycomparator.options", keySpec);

conf.set("mapreduce.partition.keypartitioner.options", "-k1.1,1.1");

conf.set(MRJobConfig.MAP_OUTPUT_KEY_FIELD_SEPERATOR, " ");

Job job = MapReduceTestUtil.createJob(conf, inDir, outDir, 1, 1,

line1 +"\n" + line2 + "\n");

job.setMapperClass(InverseMapper.class);

job.setReducerClass(Reducer.class);

job.setOutputKeyClass(Text.class);

job.setOutputValueClass(LongWritable.class);

job.setSortComparatorClass(KeyFieldBasedComparator.class);

job.setPartitionerClass(KeyFieldBasedPartitioner.class);

job.waitForCompletion(true);

assertTrue(job.isSuccessful());

// validate output

Path[] outputFiles = FileUtil.stat2Paths(getFileSystem().listStatus(outDir,

new Utils.OutputFileUtils.OutputFilesFilter()));

if (outputFiles.length > 0) {

InputStream is = getFileSystem().open(outputFiles[0]);

BufferedReader reader = new BufferedReader(new InputStreamReader(is));

String line = reader.readLine();

//make sure we get what we expect as the first line, and also

//that we have two lines (both the lines must end up in the same

//reducer since the partitioner takes the same key spec for all

//lines

if (expect == 1) {

assertTrue(line.startsWith(line1));

} else if (expect == 2) {

assertTrue(line.startsWith(line2));

}

line = reader.readLine();

if (expect == 1) {

assertTrue(line.startsWith(line2));

} else if (expect == 2) {

assertTrue(line.startsWith(line1));

}

reader.close();

}

}