这问题好蠢啊,当时好菜鸟,还百度了好久,给自己纪念下,
Driver中:
public void run(String outputFolder,String userData[]) throws IOException, ClassNotFoundException, InterruptedException, URISyntaxException{
Configuration conf = new Configuration();
Job job = new Job(conf,"ComDistanceDriver");
job.setJarByClass(ComDistanceDriver.class);
job.setMapperClass(ComDistanceMapper.class);
job.setReducerClass(ComDistanceReducer.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(DoubleWritable.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);
job.setInputFormatClass(KeyValueTextInputFormat.class);
//添加多个分发文件,保存在HDFS中
DistributedCache.addCacheFile(new URI("hdfs://localhost:9000/user/month/month-r-00000"), job.getConfiguration());
DistributedCache.addCacheFile(new URI("hdfs://localhost:9000/user/monthdkk/monthdeviration-r-00000"), job.getConfiguration());
//添加多个文件
for(String inputData:userData ){
FileInputFormat.addInputPath(job, new Path(inputData));
}
FileOutputFormat.setOutputPath(job, new Path(outputFolder));
job.waitForCompletion(true);
}
mapper或reduce中调用分发文件(setup函数中)
/**
* 从model文件中读取mcc
*/
@Override
protected void setup(Context context) throws IOException,
InterruptedException {
Configuration conf = context.getConfiguration();
try {
Path[] paths = DistributedCache.getLocalCacheFiles(conf);
for (int i = 0; i < paths.length; i++) {
System.out.println(paths[i].toString());
}
FileSystem fsopen = FileSystem.getLocal(conf);
for (int i = 0; i < paths.length; i++) {
FSDataInputStream fin = fsopen.open(paths[i]);
BufferedReader bufin = new BufferedReader(new InputStreamReader(fin, "UTF-8"));
/* FSDataInputStream in = fsopen.open(paths[i]);*/
String line=null;
if (paths[i].getName().contains("xx")) {
//要在mapper中定义private monthCentroids
monthCentroids = new HashMap<String, Set<Double[]>>();
while ((line = bufin.readLine()) != null) {
String[] b = line.split("\t");
for (int j = 0; j < b.length; j++) {
/*System.out.println("month:"+b[j]);*/
}
String MCC = b[0];
String[] a = b[1].split(",");
Double[] centroid = new Double[a.length];
for (int j = 0; j < a.length; j++) {
centroid[j] = Double.parseDouble(a[j]);
System.out.println("cent:"+centroid[j]);
}
Set<Double[]> centroidSet = monthCentroids.get(MCC);
if (centroidSet == null) {
centroidSet = new HashSet<Double[]>();
centroidSet.add(centroid);
monthCentroids.put(MCC, centroidSet);
} else {
centroidSet.add(centroid);
}
}
}else if (paths[i].getName().contains("bbb)) {
monthDevirationCentroids = new HashMap<String, Set<Double[]>>();
while ((line = bufin.readLine()) != null) {
String[] b = line.split("\t");
for (int j = 0; j < b.length; j++) {
System.out.println("monthde:"+b[j]);
}
String MCC = b[0];
String[] a = b[1].split(",");
Double[] centroid = new Double[a.length];
for (int j = 0; j < a.length; j++) {
centroid[j] = Double.parseDouble(a[j]);
System.out.println("cent:"+centroid[j]);
}
Set<Double[]> centroidSet = monthDevirationCentroids.get(MCC);
if (centroidSet == null) {
centroidSet = new HashSet<Double[]>();
centroidSet.add(centroid);
monthDevirationCentroids.put(MCC, centroidSet);
} else {
centroidSet.add(centroid);
}
}
}
}
bufin.close();
}
} catch (Exception e) {
e.printStackTrace();
}
// TODO
}