数据:
Input:
name hwl chinese 100 math 100 english 55
name lh chinese 99 math 45 english 54
name zs chinese 52 math 95 english 51
name bb chinese 96 math 89 english 57
name lk chinese 71 math 60 english 50
name ll chinese 32 math 12 english 72
name dd chinese 63 math 74 english 13
Output:
chinese zs:52;ll:32;dd:63;hwl:100;bb:96;lk:71;lh:99;
english hwl:55;zs:51;bb:57;ll:72;dd:13;lh:54;lk:50;
math hwl:100;lh:45;zs:95;lk:60;ll:12;dd:74;bb:89;
代码:
package MapReducer05;
import com. google. inject. internal. util. $StackTraceElements;
import org. apache. hadoop. conf. Configuration;
import org. apache. hadoop. fs. Path;
import org. apache. hadoop. io. IntWritable;
import org. apache. hadoop. io. LongWritable;
import org. apache. hadoop. io. Text;
import org. apache. hadoop. mapreduce. *;
import org. apache. hadoop. mapreduce. lib. input. FileInputFormat;
import org. apache. hadoop. mapreduce. lib. input. FileSplit;
import org. apache. hadoop. mapreduce. lib. jobcontrol. ControlledJob;
import org. apache. hadoop. mapreduce. lib. jobcontrol. JobControl;
import org. apache. hadoop. mapreduce. lib. output. FileOutputFormat;
import org. apache. log4j. BasicConfigurator;
import java. io. IOException;
public class ScoreMR {
public static class MyMapper01 extends Mapper < LongWritable, Text, Text, Text> {
@Override
protected void map ( LongWritable key, Text value, Context context) throws IOException, InterruptedException {
String[ ] str = value. toString ( ) . split ( " " ) ;
context. write ( new Text ( str[ 2 ] ) , new Text ( str[ 1 ] + ":" + str[ 3 ] ) ) ;
context. write ( new Text ( str[ 4 ] ) , new Text ( str[ 1 ] + ":" + str[ 5 ] ) ) ;
context. write ( new Text ( str[ 6 ] ) , new Text ( str[ 1 ] + ":" + str[ 7 ] ) ) ;
}
}
public static class MyMapper02 extends Mapper < LongWritable, Text, Text, Text> {
@Override
protected void map ( LongWritable key, Text value, Context context) throws IOException, InterruptedException {
String[ ] str = value. toString ( ) . split ( "\t" ) ;
context. write ( new Text ( str[ 0 ] ) , new Text ( str[ 1 ] ) ) ;
}
}
public static class MyReducer02 extends Reducer < Text, Text, Text, Text> {
@Override
protected void reduce ( Text key, Iterable< Text> values, Context context) throws IOException, InterruptedException {
String res = "" ;
for ( Text t : values) {
res += t. toString ( ) + ";" ;
}
context. write ( key, new Text ( res) ) ;
}
}
public static void main ( String[ ] args) throws IOException, ClassNotFoundException, InterruptedException {
System. setProperty ( "hadoop.home.dir" , "F:\\hadoop-2.6.4" ) ;
BasicConfigurator. configure ( ) ;
Configuration conf1 = new Configuration ( ) ;
Configuration conf2 = new Configuration ( ) ;
Job job1 = Job. getInstance ( conf1, "mr1" ) ;
job1. setJarByClass ( ScoreMR. class ) ;
job1. setMapperClass ( MyMapper01. class ) ;
job1. setMapOutputKeyClass ( Text. class ) ;
job1. setMapOutputValueClass ( Text. class ) ;
job1. setPartitionerClass ( MyPartitioner. class ) ;
job1. setNumReduceTasks ( 3 ) ;
FileInputFormat. addInputPath ( job1, new Path ( "C:\\Users\\Chen\\Desktop\\input\\00" ) ) ;
FileOutputFormat. setOutputPath ( job1, new Path ( "C:\\Users\\Chen\\Desktop\\30" ) ) ;
Job job2 = Job. getInstance ( conf2, "mr2" ) ;
job2. setJarByClass ( ScoreMR. class ) ;
job2. setMapperClass ( MyMapper02. class ) ;
job2. setMapOutputKeyClass ( Text. class ) ;
job2. setMapOutputValueClass ( Text. class ) ;
job2. setReducerClass ( MyReducer02. class ) ;
job2. setOutputKeyClass ( Text. class ) ;
job2. setOutputValueClass ( Text. class ) ;
FileInputFormat. setInputPaths ( job2, new Path ( "C:\\Users\\Chen\\Desktop\\30" ) ) ;
FileOutputFormat. setOutputPath ( job2, new Path ( "C:\\Users\\Chen\\Desktop\\31" ) ) ;
JobControl jobControl = new JobControl ( "jobControl" ) ;
ControlledJob cj1 = new ControlledJob ( job1. getConfiguration ( ) ) ;
ControlledJob cj2 = new ControlledJob ( job2. getConfiguration ( ) ) ;
cj2. addDependingJob ( cj1) ;
jobControl. addJob ( cj1) ;
jobControl. addJob ( cj2) ;
Thread thread = new Thread ( jobControl) ;
thread. start ( ) ;
while ( ! jobControl. allFinished ( ) ) {
thread. sleep ( 1000 ) ;
}
jobControl. stop ( ) ;
System. exit ( 0 ) ;
}
}
class MyPartitioner extends Partitioner < Text, Text> {
@Override
public int getPartition ( Text text, Text text2, int numPartitions) {
String str = text. toString ( ) ;
if ( str. startsWith ( "chinese" ) ) {
return 0 % numPartitions;
} else if ( str. startsWith ( "math" ) ) {
return 1 % numPartitions;
} else {
return 2 % numPartitions;
}
}
}
中间文件:
part-r-00000:
chinese lh:99
chinese lk:71
chinese bb:96
chinese hwl:100
chinese dd:63
chinese ll:32
chinese zs:52
part-r-00001:
math bb:89
math dd:74
math ll:12
math lk:60
math zs:95
math lh:45
math hwl:100
part-r-00002:
english lk:50
english lh:54
english dd:13
english ll:72
english bb:57
english zs:51
english hwl:55