public class ProcedureJob {
public static class MapClass extends Mapper<Object, Text, Text, Text>{
private Path[] localFiles;
private URI[] uris;
@Override
protected void setup(Context context) throws IOException,
InterruptedException {
localFiles = DistributedCache.getLocalCacheFiles(context.getConfiguration());
uris = DistributedCache.getCacheFiles(context.getConfiguration());
System.out.println(uris[0].toString()+"+++++++++++++++"+localFiles+"----------------");
if(uris==null || uris.length==0){
throw new IOException("Error reading file from distributed cache. No URIs found.");
}
FileSystem fs = FileSystem.get(URI.create("hdfs://namenode:9000"),context.getConfiguration());
Path localPath = new Path("~/abc.txt");
fs.copyToLocalFile(new Path(uris[0].toString()),localPath );
BufferedReader br = new BufferedReader(new FileReader(localPath.toString()));
String str="";
while((str=br.readLine())!=null){
System.out.println(str+"---------------------------------------------");
}
br.close();
}
protected void map(Object key, Text value, Context context)
throws IOException, InterruptedException {
context.write(new Text(key.toString()), new Text("1"));
}
}
public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
DistributedCache.addCacheFile(new Path(args[0]).toUri(), conf);
Job job = new Job(conf,"Test");
job.setJarByClass(ProcedureJob.class);
FileInputFormat.setInputPaths(job,new Path(args[1]));
FileOutputFormat.setOutputPath(job, new Path(args[2]));
job.setJobName("Test");
job.setMapperClass(MapClass.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);
job.setNumReduceTasks(0);
job.waitForCompletion(true);
}
}
说明:事先将本地文件上传到HDFS中,/data/pubic/test.txt。
注意点:
1.Job对象必须在使用configuration对象把缓存文件添加之后才能new,否则会无法添加,出现java.lang.NullPointerException。
2.在setup中获取Path[] localFiles = DistributedCache.getLocalCacheFiles(context.getConfiguration());得到的对象是null,可能版本问题。
3.在setup方法中获取文件系统,FileSystem fs = FileSystem.get(URI.create("hdfs://namenode:9000"),context.getConfiguration());
如果直接是FileSystem fs = FileSystem.get(context.getConfiguration());会出现java.lang.IllegalArgumentException: Wrong FS: hdfs://localhost:54310/user/Hadoop/b, expected: file:/// 异常。