说明:
Hadoop3在windows下能用的版本是hadoop-3.0.3。
其他版本都会报如下错误:
java.lang.UnsupportedOperationException
at java.nio.file.Files.setPosixFilePermissions(Files.java:2044)
建议在windows下使用hadoop-3.0.3版本
下载
hadoop-3.0.3.tar.gz
winutils-master.zip(https://github.com/cdarlint/winutils)
解压
D:\servers\hadoop-3.0.3
环境配置
HADOOP_HOME=D:\servers\hadoop-3.0.3
PATH增加D:\servers\hadoop-3.0.3\bin
Hadoop配置
hadoop-env.cmd(java路径不能有空格,否则报错)
core-site.xml
<property> <name>fs.default.name</name> <value>hdfs://localhost:9000</value> </property> <property> <name>hadoop.tmp.dir</name> <value>/D:/servers/hadoop-3.0.3/data/tmp</value> </property> |
mapred-site.xml
<property> <name>mapreduce.framework.name</name> <value>yarn</value> </property> |
创建目录
D:\servers\hadoop-3.0.3\data\namenode
D:\servers\hadoop-3.0.3\data\datanode
hdfs-site.xml
<property> <name>dfs.replication</name> <value>1</value> </property> <property> <name>dfs.permissions</name> <value>false</value> </property> <property> <name>dfs.namenode.name.dir</name> <value>/D:/servers/hadoop-3.0.3/data/namenode</value> </property> <property> <name>fs.checkpoint.dir</name> <value>/D:/servers/hadoop-3.0.3/data/snn</value> </property> <property> <name>fs.checkpoint.edits.dir</name> <value>/D:/servers/hadoop-3.0.3/data/snn</value> </property> <property> <name>dfs.datanode.data.dir</name> <value>/D:/servers/hadoop-3.0.3/data/datanode</value> </property> |
yarn-site.xml
<property> <name>yarn.nodemanager.aux-services</name> <value>mapreduce_shuffle</value> </property> <property> <name>yarn.nodemanager.auxservices.mapreduce.shuffle.class</name> <value>org.apache.hadoop.mapred.ShuffleHandler</value> </property> <property> <name>yarn.log-aggregation-enable</name> <value>true</value> </property> |
替换
解压winutils-master.zip
用hadoop-3.0.0\bin下的文件替换D:\servers\hadoop-3.0.3\bin下的文件
格式化
进入bin目录
hdfs namenode -format
启动
进入sbin目录
start-all
开发环境建议启动:
mapred historyserver
验证
jps
看到namenode、datanode、resourcemanager、nodemanager,OK
访问:http://localhost:9870(以前是50070)
如果碰到job一直卡到资源申请那里可以使用,查看application id
yarn application -list
然后kill掉
yarn application -kill application_1582478379420_0006
备注:
报错:(hadoop3.0.3)
java.lang.NoClassDefFoundError: org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager
将share\hadoop\yarn\timelineservice\hadoop-yarn-server-timelineservice-3.0.3.jar 放到share\hadoop\yarn\lib 下
Hadoop3 intellij idea 插件安装
下载intellij idea 2018版(2020版不能用于编译该插件,因为该插件太老了)
下载
https://github.com/fangyuzhong2016/HadoopIntellijPlugin
HadoopIntellijPlugin-master.zip
解压
D:\hadoop spark\HadoopIntellijPlugin-master
Intellij idea导入
File -> New -> Project from Existing Sources…
Import project from external model
修改pom
<!--设置hadoop版本-->
|
编译
进入 terminal
mvn clean
mvn assembly:assembly
生成插件包 HadoopIntellijPlugin-1.0.zip
导入插件
Setting -> Plugins -> install plugin from disk… -> restart
左侧边栏看到Hadoop小象
使用
点击左侧边栏小象,点击+,点击+,新建hadoop连接
OK!!!
mapred开发
新建mvn项目
定义pom文件
<?xml version="1.0" encoding="UTF-8"?>
|
配置log4j
新建log4j.properties
log4j.rootLogger=INFO, A1 |
编写代码
package com.xzrj.demo.hd;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.mapreduce.Reducer; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; import org.apache.log4j.LogManager; import org.apache.log4j.Logger;
import java.io.IOException; import java.net.URI;
public class WordCount { private static final Logger lg = LogManager.getLogger(WordCount.class);
// 设置参数 private static String defaultfs = "hdfs://localhost:9000"; private static String hadoopuser = "admin"; private static String inpath = "/in/wordcount.txt"; private static String outpath = "/out/"; private static String jobname = "wordcount"; private static Class<? extends Mapper> mapperclass = WcMapper.class; private static Class<? extends Reducer> reducerclass = WcReducer.class; private static Class<?> jarclass = WordCount.class;
public static void main(String[] args) throws Exception{ Configuration conf = new Configuration(); conf.set("dfs.permissions","false"); conf.set("fs.defaultFS",defaultfs); conf.set("dfs.client.use.datanode.hostname", "true"); System.setProperty("HADOOP_USER_NAME",hadoopuser);
Job job = Job.getInstance(conf, jobname); job.setMapperClass(mapperclass); job.setReducerClass(reducerclass); job.setJarByClass(jarclass);
job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(IntWritable.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(IntWritable.class);
FileSystem fs = FileSystem.get(new URI(defaultfs), conf, hadoopuser); Path op = new Path(outpath); if(fs.exists(op)) fs.delete(op,true);
FileInputFormat.setInputPaths(job, new Path(inpath)); FileOutputFormat.setOutputPath(job, new Path(outpath)); boolean res = job.waitForCompletion(true); if(!res){ lg.error(jobname + " 失败了!"); } lg.info(jobname + " 成功了!"); }
private static class WcMapper extends Mapper<LongWritable, Text, Text, IntWritable> { @Override protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException { String line = value.toString(); String[] words = line.split(" "); for(String word: words){ context.write(new Text(word), new IntWritable(1)); } } }
private static class WcReducer extends Reducer<Text, IntWritable, Text, IntWritable> { @Override protected void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException { Integer count = 0; for(IntWritable value: values){ count += value.get(); } context.write(key, new IntWritable(count)); } }
} |