import java.io.BufferedReader;
public class ScoreSort {
public static class ScoreWritable implements WritableComparable<ScoreWritable> {
private int totalScore;
private int chineseScore;
public ScoreWritable() {
}
public ScoreWritable(int totalScore, int chineseScore) {
this.totalScore = totalScore;
this.chineseScore = chineseScore;
}
@Override
public void write(DataOutput out) throws IOException {
out.writeInt(totalScore);
out.writeInt(chineseScore);
}
@Override
public void readFields(DataInput in) throws IOException {
totalScore = in.readInt();
chineseScore = in.readInt();
}
@Override
public int compareTo(ScoreWritable other) {
if (this.totalScore > other.totalScore) {
return -1; // 降序排列
} else if (this.totalScore < other.totalScore) {
return 1;
} else {
// 总分相同,按语文升序排列
return Integer.compare(this.chineseScore, other.chineseScore);
}
}
@Override
public String toString() {
return totalScore + "\t" + chineseScore;
}
}
public static class ScoreSortMapper extends Mapper<Object, Text, ScoreWritable, Text> {
private ScoreWritable scoreKey = new ScoreWritable();
private Text studentInfo = new Text();
public void map(Object key, Text value, Context context) throws IOException, InterruptedException {
String[] tokens = value.toString().split("\t");
if (tokens.length == 4) {
String studentId = tokens[0];
int chineseScore = Integer.parseInt(tokens[1]);
int mathScore = Integer.parseInt(tokens[2]);
int englishScore = Integer.parseInt(tokens[3]);
int totalScore = chineseScore + mathScore + englishScore;
scoreKey.totalScore = totalScore;
scoreKey.chineseScore = chineseScore;
studentInfo.set(studentId + "\t" + totalScore); // 修改此行
context.write(scoreKey, studentInfo);
}
}
}
public static class ScoreSortReducer extends Reducer<ScoreWritable, Text, Text, ScoreWritable> {
public void reduce(ScoreWritable key, Iterable<Text> values, Context context)
throws IOException, InterruptedException {
// for (Text value : values) {
// context.write(value, key); // 交换输出的键值对
// }
int count = 0;
for (Text value : values) {
if (count < 2) {
String[] tokens = value.toString().split("\t");
if (tokens.length >= 2) {
context.write(new Text(tokens[0]), new ScoreWritable(Integer.parseInt(tokens[1]), key.chineseScore));
}
count++;
} else {
break; // 仅输出前两个值,即前两列信息
}
}
}
}
public static void main(String[] args) throws Exception {
String namenode_ip = "localhost";
String hdfs = "hdfs://" + namenode_ip + ":9000";
Configuration conf = new Configuration();
conf.set("fs.defaultFS", hdfs);
conf.set("mapreduce.app-submission.cross-platform", "true");
// 设置MapReduce作业配置信息
String jobName = "ScoreSort"; // 定义作业名称
Job job = Job.getInstance(conf, jobName);
job.setJarByClass(ScoreSort.class); // 指定作业类
job.setMapperClass(ScoreSortMapper.class);
job.setReducerClass(ScoreSortReducer.class);
job.setMapOutputKeyClass(ScoreWritable.class);
job.setMapOutputValueClass(Text.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(ScoreWritable.class);
// 设置作业输入和输出路径
String dataDir = "/user/hadoop/input/"; // 实验数据目录
String outputDir = "/user/hadoop/output/"; // 实验输出目录
Path inPath = new Path(hdfs + dataDir);
Path outPath = new Path(hdfs + outputDir);
FileInputFormat.addInputPath(job, inPath);
FileOutputFormat.setOutputPath(job, outPath);
FileSystem fs = FileSystem.get(conf);
if (fs.exists(outPath)) {
fs.delete(outPath, true);
}
// 运行作业
System.out.println("Job: " + jobName + " is running...");
if(job.waitForCompletion(true)) {
System.out.println("success!");
Path resultPath = new Path(hdfs + outputDir + "/part-r-00000"); // 输出结果文件路径
BufferedReader br = new BufferedReader(new InputStreamReader(fs.open(resultPath)));
String line;
while ((line = br.readLine()) != null) {
System.out.println(line);
}
br.close();
System.exit(0);
} else {
System.out.println("failed!");
System.exit(1);
}
}
}
临时保存代码2
最新推荐文章于 2024-09-04 17:44:08 发布