job用法 Java_Java Job.addCacheFile方法代码示例

import org.apache.hadoop.mapreduce.Job; //导入方法依赖的package包/类

public void _testDistributedCache(String jobJarPath) throws Exception {

if (!(new File(MiniMRYarnCluster.APPJAR)).exists()) {

LOG.info("MRAppJar " + MiniMRYarnCluster.APPJAR

+ " not found. Not running test.");

return;

}

// Create a temporary file of length 1.

Path first = createTempFile("distributed.first", "x");

// Create two jars with a single file inside them.

Path second =

makeJar(new Path(TEST_ROOT_DIR, "distributed.second.jar"), 2);

Path third =

makeJar(new Path(TEST_ROOT_DIR, "distributed.third.jar"), 3);

Path fourth =

makeJar(new Path(TEST_ROOT_DIR, "distributed.fourth.jar"), 4);

Job job = Job.getInstance(mrCluster.getConfig());

// Set the job jar to a new "dummy" jar so we can check that its extracted

// properly

job.setJar(jobJarPath);

// Because the job jar is a "dummy" jar, we need to include the jar with

// DistributedCacheChecker or it won't be able to find it

Path distributedCacheCheckerJar = new Path(

JarFinder.getJar(DistributedCacheChecker.class));

job.addFileToClassPath(distributedCacheCheckerJar.makeQualified(

localFs.getUri(), distributedCacheCheckerJar.getParent()));

job.setMapperClass(DistributedCacheChecker.class);

job.setOutputFormatClass(NullOutputFormat.class);

FileInputFormat.setInputPaths(job, first);

// Creates the Job Configuration

job.addCacheFile(

new URI(first.toUri().toString() + "#distributed.first.symlink"));

job.addFileToClassPath(second);

// The AppMaster jar itself

job.addFileToClassPath(

APP_JAR.makeQualified(localFs.getUri(), APP_JAR.getParent()));

job.addArchiveToClassPath(third);

job.addCacheArchive(fourth.toUri());

job.setMaxMapAttempts(1); // speed up failures

job.submit();

String trackingUrl = job.getTrackingURL();

String jobId = job.getJobID().toString();

Assert.assertTrue(job.waitForCompletion(false));

Assert.assertTrue("Tracking URL was " + trackingUrl +

" but didn't Match Job ID " + jobId ,

trackingUrl.endsWith(jobId.substring(jobId.lastIndexOf("_")) + "/"));

}