三、去重
四、总和计算
下面是四个经典案例,分别是
-
一、词频统计
-
二、最大值
-
三、去重
-
四、总和计算
=====================================================================
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
import java.io.IOException;
public class WordCountMapper extends Mapper<LongWritable, Text,Text, IntWritable> {
@Override
protected void map(LongWritable key,Text value,Context context) throws IOException,InterruptedException{
String datas = value.toString();
String[] split = datas.split(" ");
for (String data : split){
context.write(new Text(data),new IntWritable(1));
}
}
}
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;
import java.io.IOException;
public class WordCountReducer extends Reducer<Text, IntWritable,Text,IntWritable> {
@Override
protected void reduce(Text key,Iterable values,Context context) throws IOException,InterruptedException{
int sum=0;
for(IntWritable val:values){
sum=sum+1;
}
context.write(key,new IntWritable(sum));
}
}
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import java.io.IOException;
public class WordCountDriver {
public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException {
//向yarn申请一个job任务用于执行mapreduce程序
Job job = Job.getInstance(new Configuration());
//设置入口类
job.setJarByClass(WordCountDriver.class);
//设置mapper类
job.setMapperClass(WordCountMapper.class);
//设置reduce类
job.setReducerClass(WordCountReducer.class);
//设置Mapper类的输出
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(IntWritable.class);
//设置reduce类的输出
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(IntWritable.class);
//设置要处理的文件
FileInputFormat.addInputPath(job,new Path(“hdfs://192.168.21.128:9000/txt/words.txt”));
//设置输出路径
FileOutputFormat.setOutputPath(job,new Path(“hdfs://192.168.21.128:9000/result/wordcount”));
//启动
job.waitForCompletion(true);
}
}
运行时,选中main函数,右键
====================================================================
package cn.top;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
import java.io.IOException;
public class CharTopMapper extends Mapper<LongWritable,Text,Text,IntWritable> {
@Override
protected void map(LongWritable key,Text value,Context context) throws IOException,InterruptedException{
//拆分这一行中的每一个字符
String datas = value.toString();
String[] split = datas.split(" ");
int x = Integer.parseInt(split[1]);
//将每一个字符进行遍历
context.write(new Text(split[0]),new IntWritable(x));
}
}
package cn.top;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;
import java.io.IOException;
public class CharTopReducer extends Reducer<Text,IntWritable,Text,IntWritable> {
@Override
protected void reduce(Text key,Iterable values,Context context) throws IOException,InterruptedException{
int sum=0;
int sum1=0;
for(IntWritable val:values){
sum = Integer.parseInt(String.valueOf(val));
if(sum1 < sum){
sum1=sum;
}
}
context.write(key,new IntWritable(sum1));
}
}
package cn.top;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import java.io.IOException;
public class CharTop {
public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException {
//向yarn申请一个job任务用于执行mapreduce程序
Job job = Job.getInstance(new Configuration());
//设置入口类
job.setJarByClass(CharTop.class);
//设置mapper类
job.setMapperClass(CharTopMapper.class);
//设置reduce类
job.setReducerClass(CharTopReducer.class);
//设置Mapper类的输出
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(IntWritable.class);
//设置reduce类的输出
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(IntWritable.class);
//设置要处理的文件
FileInputFormat.addInputPath(job,new Path(“hdfs://192.168.21.128:9000/txt/score2.txt”));
//设置输出路径
FileOutputFormat.setOutputPath(job,new Path(“hdfs://192.168.21.128:9000/result/charcount”));
//启动
自我介绍一下,小编13年上海交大毕业,曾经在小公司待过,也去过华为、OPPO等大厂,18年进入阿里一直到现在。
深知大多数Java工程师,想要提升技能,往往是自己摸索成长或者是报班学习,但对于培训机构动则几千的学费,着实压力不小。自己不成体系的自学效果低效又漫长,而且极易碰到天花板技术停滞不前!
因此收集整理了一份《2024年Java开发全套学习资料》,初衷也很简单,就是希望能够帮助到想自学提升又不知道该从何学起的朋友,同时减轻大家的负担。
既有适合小白学习的零基础资料,也有适合3年以上经验的小伙伴深入学习提升的进阶课程,基本涵盖了95%以上Java开发知识点,真正体系化!
由于文件比较大,这里只是将部分目录截图出来,每个节点里面都包含大厂面经、学习笔记、源码讲义、实战项目、讲解视频,并且会持续更新!
如果你觉得这些内容对你有帮助,可以扫码获取!!(备注Java获取)
分享
这次面试我也做了一些总结,确实还有很多要学的东西。相关面试题也做了整理,可以分享给大家,了解一下面试真题,想进大厂的或者想跳槽的小伙伴不妨好好利用时间来学习。学习的脚步一定不能停止!
Spring Cloud实战
Spring Boot实战
面试题整理(性能优化+微服务+并发编程+开源框架+分布式)
《互联网大厂面试真题解析、进阶开发核心学习笔记、全套讲解视频、实战项目源码讲义》点击传送门即可获取!
分享
这次面试我也做了一些总结,确实还有很多要学的东西。相关面试题也做了整理,可以分享给大家,了解一下面试真题,想进大厂的或者想跳槽的小伙伴不妨好好利用时间来学习。学习的脚步一定不能停止!
[外链图片转存中…(img-BJsAjWrS-1712648025691)]
Spring Cloud实战
[外链图片转存中…(img-TNsgiLHE-1712648025691)]
Spring Boot实战
[外链图片转存中…(img-mB8wIU5f-1712648025692)]
面试题整理(性能优化+微服务+并发编程+开源框架+分布式)
《互联网大厂面试真题解析、进阶开发核心学习笔记、全套讲解视频、实战项目源码讲义》点击传送门即可获取!