1:随机生成10万个手机号,并输入到文件
package Phone_count;
import java.io.FileNotFoundException;
import java.io.PrintWriter;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.Random;
public class MobileNum {
public static void main(String[] args) throws FileNotFoundException, UnsupportedEncodingException {
PrintWriter writer = new PrintWriter("E:\\bigdata_code\\phone\\phone.txt", "UTF-8");
for (int x =1; x <= 100000; x++) {
if ( x%7 == 0 ) {
writer.println(startMobileNo() + endMobileNo());
}else {
writer.print(startMobileNo() + endMobileNo() + " ");
}
}
writer.close();
}
public static int startMobileNo(){
int[] mobileStart = {134,135,136,137,138,139,150,151,152,157,158,159,188,187,182,183,184,178,147,172,198,
133,149,153,173,177,180,181,189,199,
130,131,132,145,155,156,166,171,175,176,185,186,166};
Random r = new Random();
ArrayList<Integer> mobileList = new ArrayList();
for(int i = 0; i < mobileStart.length; i++){
mobileList.add(mobileStart[i]);
}
return mobileList.get(r.nextInt(mobileList.size()));
}
public static String endMobileNo(){
Random r = new Random();
String temp = "";
for(int i=0; i < 8; i++){
temp += r.nextInt(10);
}
return temp;
}
}
2:创建PhonePartition类
package Phone_count;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Partitioner;
import java.util.Arrays;
public class PhonePartition extends Partitioner<Text, NullWritable> {
//定义移动的规则
public static final String[] YD = {"134","135","136","137","138","139","150","151","152",
"157","158","159","188","187","182","183","184","178","147","172","198"};
//定义电信的规则
public static final String[] DX = {"133","149","153","173","177","180","181","189","199"};
//定义联通的规则
public static final String[] LT = {"130","131","132","145","155","156","166","171","175",
"176","185","186","166"};
@Override
public int getPartition(Text text, NullWritable nullWritable, int i) {
//获取数据
String fields = text.toString();
//截取电话号码的前三位
String substring = fields.substring(0, 3);
//判断是哪个分区的
if (Arrays.asList(YD).contains(substring)) {
return 0;
}else if (Arrays.asList(DX).contains(substring)) {
return 1;
}else if (Arrays.asList(LT).contains(substring)) {
return 2
}
return 3;
}
}
3:创建PhoneMap类
package Phone_count;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
import java.io.IOException;
public class PhoneMap extends Mapper<LongWritable, Text,Text, NullWritable> {
@Override
protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
//输出
String s = value.toString();
//切分
String[] ss = s.split(" ");
//循环输出
for (String fileds:ss) {
context.write(new Text(fileds), NullWritable.get());
}
}
}
4:创建PhoneReduce类
package Phone_count;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;
import java.io.IOException;
public class PhoneReduce extends Reducer<Text, NullWritable, Text, NullWritable> {
@Override
protected void reduce(Text key, Iterable<NullWritable> values, Context context) throws IOException, InterruptedException {
//输出
context.write(key,NullWritable.get());
}
}
5:创建PhoneMain
package Phone_count;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import java.io.IOException;
public class PhoneMain {
public static void main(String[] args) throws ClassNotFoundException, InterruptedException, IOException {
args=new String[]{"E:\\bigdata_code\\phone\\phone.txt","E:\\bigdata_code\\phone\\out"};
//获取job信息
Configuration conf = new Configuration();
Job job = Job.getInstance(conf);
//加载jar包
job.setJarByClass(PhoneMain.class);
//关联map的输出输出数据类型
job.setMapperClass(PhoneMap.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(NullWritable.class);
//关联reduce输入输出数据类型
job.setReducerClass(PhoneReduce.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(NullWritable.class);
//定义分区和设置分区数
job.setPartitionerClass(PhonePartition.class);
job.setNumReduceTasks(4);
//定义输入和输出路径
FileInputFormat.setInputPaths(job,new Path(args[0]));
FileOutputFormat.setOutputPath(job,new Path(args[1]));
//提交
job.waitForCompletion(true);
}
}