public class a extends Mapper<LongWritable , Text,NullWritable,Text>{
@Override
protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
String s = value.toString();
if (s.isEmpty()){
return;
}
String[] split = s.split(":");
if (split.length>10){
String s4 = "";
String s1 = s.substring(s.indexOf(split[9])).replace(":", "&");
String s2 = s.substring(0, s.indexOf(split[9]));
String s3 = s2.replace(" & ", ",");
s4 = s3+s1;
context.write(NullWritable.get(),new Text(s4));
}
}
}
public class DSReduce extends Reducer<NullWritable,Text,Text,Text> {
@Override
protected void reduce(NullWritable key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
FileSystem fs = null;
try {
fs = FileSystem.get(new URI("hdfs://192.168.100.191:8020"), new Configuration());
} catch (URISyntaxException e) {
e.printStackTrace();
}
FSDataOutputStream fsDataOutputStream = fs.create(new Path("/video.txt"));
for (Text value : values) {
fsDataOutputStream.write((value.toString()+"\r\n" +
"").getBytes());
}
}
}
public class DSDriver extends Configured implements Tool { @Override public int run(String[] args) throws Exception { Configuration _conf = new Configuration(); Job job = new Job(_conf, "DataSplit"); job.setJarByClass(DSDriver.class); //设置map输出类型 job.setMapperClass(DSMapper.class); job.setMapOutputKeyClass(NullWritable.class); job.setMapOutputValueClass(Text.class); //设置reduce输出类型 job.setReducerClass(DSReduce.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(Text.class); //设置输入 job.setInputFormatClass(TextInputFormat.class); // TextInputFormat.setInputPaths(job,new Path(args[0])); TextInputFormat.setInputPaths(job,new Path("F:\\wordcount\\video.txt")); //设置输出 job.setOutputFormatClass(TextOutputFormat.class); TextOutputFormat.setOutputPath(job,new Path("hdfs://192.168.100.191:8020//DS")); return job.waitForCompletion(true)?0:1; } public static void main(String[] args) throws Exception { Configuration configuration = new Configuration(); ToolRunner.run(configuration,new DSDriver(),args); } }