微博粉丝互粉统计
mapper阶段
import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Mapper; import java.io.IOException; public class microMapper extends Mapper<Object, Text, Text, NullWritable>{ Text text = new Text(); @Override protected void map(Object key, Text value, Context context) throws IOException, InterruptedException { String[] lines=value.toString().trim().split( " \t " ); String str1=lines[ 0 ]; String str2=lines[ 1 ]; if (str1.compareTo(str2)> 0 ){ text .set(str2+ "_" +str1); } else { text .set(str1+ "_" +str2); } context.write( text ,NullWritable. get ()); } }
reduce阶段
import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Reducer; import java.io.IOException; public class microReduce extends Reducer<Text,NullWritable,Text,LongWritable> { public static long row = 1 ; @Override protected void reduce(Text key, Iterable<NullWritable> values, Context context) throws IOException, InterruptedException { int count= 0 ; for (NullWritable dd:values){ count++; } if (count== 2 ){ context.write(key, new LongWritable( row )); row ++; } } }
runner阶段
import mapper.microMapper; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; import reduce.microReduce; import java.io.IOException; public class microRunner { public static void main(String[] args) { try { // 系统设置 Configuration con= new Configuration(); Job job=Job. getInstance (con, "wordcount" ); job.setJarByClass(microRunner. class ); //map 阶段进行设置 job.setMapperClass(microMapper. class ); job.setMapOutputKeyClass(Text. class ); job.setMapOutputValueClass(NullWritable. class ); FileInputFormat. addInputPath (job, new Path(args[ 0 ])); //reduce 端进行设置 job.setReducerClass(microReduce. class ); job.setOutputKeyClass(Text. class ); job.setOutputValueClass(LongWritable. class ); FileOutputFormat. setOutputPath (job, new Path(args[ 1 ])); // 返回结果 int isok= 0 ; isok = job.waitForCompletion( true )? 0 : 1 ; System. exit (isok); } catch (IOException e) { e.printStackTrace(); } catch (InterruptedException e) { e.printStackTrace(); } catch (ClassNotFoundException e) { e.printStackTrace(); } } }
create table if not exists com_fanslist( mid string, id string ) row format delimited fields terminated by '\t' stored as textfile location '/project/sport_micro_blog/ods_user_fanslist_out/out';