Macbook Intellij idea与Eclipse远程调试Hadoop应用程序

Intellij idea

准备工作

1. 在虚拟机安装hadoop集群
2. 开发机配置
   （1）idean版本15.0.4
   （2）jdk版本1.7.0_71
   （3）Mac OS X 10.11.6
   （4）hadoop安装（hadoop-2.5.2.tar.gz解压）
   /Users/zhangws/opt/hadoop-2.5.2
3. 配置环境变量

HADOOP_HOME=/Users/zhangws/opt/hadoop-2.5.2
HADOOP_BIN_PATH=%HADOOP_HOME%\bin
HADOOP_PREFIX=/Users/zhangws/opt/hadoop-2.5.2

另外，PATH变量在最后追加;%HADOOP_HOME%\bin

创建工程

pom.xml文件内容

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  <modelVersion>4.0.0</modelVersion>

  <groupId>com.zw</groupId>
  <artifactId>hadoop-demo</artifactId>
  <version>1.0-SNAPSHOT</version>
  <packaging>jar</packaging>

  <name>hadoop-demo</name>
  <url>http://maven.apache.org</url>

  <properties>
      <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
      <hadoop.version>2.5.2</hadoop.version>
  </properties>

  <dependencies>
    <dependency>
      <groupId>org.apache.hadoop</groupId>
      <artifactId>hadoop-common</artifactId>
      <version>${hadoop.version}</version>
    </dependency>
      <dependency>
          <groupId>org.apache.hadoop</groupId>
          <artifactId>hadoop-hdfs</artifactId>
          <version>${hadoop.version}</version>
      </dependency>

      <dependency>
          <groupId>org.apache.hadoop</groupId>
          <artifactId>hadoop-client</artifactId>
          <version>${hadoop.version}</version>
      </dependency>

    <dependency>
      <groupId>junit</groupId>
      <artifactId>junit</artifactId>
      <version>3.8.1</version>
      <scope>test</scope>
    </dependency>
  </dependencies>
</project>

core-site.xml配置（resources目录）

<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
    <property>
        <name>fs.defaultFS</name>
        <value>hdfs://master:9000</value>
    </property>
</configuration>

注：value指定hadoop的地址（虚拟机）

设置运行参数

Working directory是本地hadoop的home路径；
Program arguments的内容如下：

hdfs://master:9000/weekone/words/input/test.txt
hdfs://master:9000/weekone/words/output
分别为输入参数和输出参数。

注：
如果input/test.txt文件没有，请先手动上传；
/output/ 必须是不存在的，否则程序运行到最后，发现目标目录存在，也会报错；

按照上面步骤就可以在适当的位置打断点，调试了。

示例

package com.zw.mr.demo;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.GenericOptionsParser;

import java.io.IOException;

/**
 * 这是统计单词个数的例子
 *
 * Created by zhangws on 16/7/31.
 */
public class WordsCount {
    public static class MyMapper extends Mapper<LongWritable, Text, Text, IntWritable> {
        public void map(LongWritable key, Text value, Context context)
                throws IOException, InterruptedException {
            String[] strings = value.toString().split(" ");
            for (String s : strings) {
                //将文本行放入key
                context.write(new Text(s), new IntWritable(1));
            }
        }
    }

    public static class MyReducer extends Reducer<Text, IntWritable, Text, IntWritable> {
        public void reduce(Text key, Iterable<IntWritable> values, Context context)
                throws IOException, InterruptedException {

            int count = 0;
            for (IntWritable v : values) {
                count += v.get();
            }
            //输出key
            context.write(key, new IntWritable(count));
        }
    }

    public static void main(String[] args) throws Exception {
        Configuration conf = new Configuration();
        String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
        if (otherArgs.length < 2) {
            System.err.println("Usage: wordcount <in> [<in>...] <out>");
            System.exit(2);
        }

        //先删除output目录
        rmr(conf, otherArgs[otherArgs.length - 1]);

        Job job = Job.getInstance(conf, "WordsCount");
        job.setJarByClass(WordsCount.class);

        job.setMapperClass(MyMapper.class);
        job.setCombinerClass(MyReducer.class);
        job.setReducerClass(MyReducer.class);

        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(IntWritable.class);

        FileInputFormat.addInputPath(job, new Path(otherArgs[0]));
        FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));

        if (job.waitForCompletion(true)) {
            cat(conf, otherArgs[1] + "/part-r-00000");
            System.out.println("success");
        } else {
            System.out.println("fail");
        }
    }

    /**
     * 删除指定目录
     *
     * @param conf
     * @param dirPath
     *
     * @throws IOException
     */
    private static void rmr(Configuration conf, String dirPath) throws IOException {
        boolean delResult = false;
//        FileSystem fs = FileSystem.get(conf);
        Path targetPath = new Path(dirPath);
        FileSystem fs = targetPath.getFileSystem(conf);
        if (fs.exists(targetPath)) {
            delResult = fs.delete(targetPath, true);
            if (delResult) {
                System.out.println(targetPath + " has been deleted sucessfullly.");
            } else {
                System.out.println(targetPath + " deletion failed.");
            }
        }
        return delResult;
    }

    /**
     * 输出指定文件内容
     *
     * @param conf     HDFS配置
     * @param filePath 文件路径
     *
     * @return 文件内容
     *
     * @throws IOException
     */
    public static void cat(Configuration conf, String filePath) throws IOException {

//        FileSystem fileSystem = FileSystem.get(conf);
        InputStream in = null;
        Path file = new Path(filePath);
        FileSystem fileSystem = file.getFileSystem(conf);
        try {
            in = fileSystem.open(file);
            IOUtils.copyBytes(in, System.out, 4096, true);
        } finally {
            if (in != null) {
                IOUtils.closeStream(in);
            }
        }
    }
}

日志文件

log4j.rootLogger=INFO, stdout

#log4j.logger.org.springframework=INFO
#log4j.logger.org.apache.activemq=INFO
#log4j.logger.org.apache.activemq.spring=WARN
#log4j.logger.org.apache.activemq.store.journal=INFO
#log4j.logger.org.activeio.journal=INFO

log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%d{ABSOLUTE} | %-5.5p | %-16.16t | %-32.32c{1} | %-32.32C %4L | %m%n

权限设置

由于客户端与服务器的权限问题，对输入目录等需要赋予授权
hdfs dfs -chomod 777 test/

或者hdfs-site.xml里添加

<property>
    <name>dfs.permissions</name>
    <value>false</value>
</property>

运行结果

可能会遇到下面问题（本人是用自编译的hadoop-2.6.4遇到的）
java.io.IOException: No FileSystem for scheme: hdfs

Eclipse

准备工作

1. 在虚拟机安装hadoop集群
2. 开发机配置
   （1）Eclipse Version: Mars.2 Release (4.5.2)
   （2）jdk版本1.7.0_71
   （3）Mac OS X 10.11.6
   （4）hadoop安装（hadoop-2.5.2.tar.gz解压）
   /Users/zhangws/opt/hadoop-2.5.2
3. 环境变量同上
4. 安装插件
   https://github.com/winghc/hadoop2x-eclipse-plugin
   下载hadoop-eclipse-plugin-2.6.0.jar，放入eclipse的plugins目录，启动eclipse

配置eclipse环境

preferences->hadoop map/reduce 指定mac上的hadoop根目录（即：$HADOOP_HOME）

配置hadoop-eclipse-plugin插件

Location name 这里就是起个名字，随便起

Map/Reduce(V2) Master Host 这里就是虚拟机里hadoop master对应的IP地址，下面的端口对应 hdfs-site.xml里dfs.datanode.ipc.address属性所指定的端口，默认端口50020

DFS Master Port： 这里的端口，对应core-site.xml里fs.defaultFS所指定的端口

最后的user name要跟虚拟机里运行hadoop的用户名一致，我是用zkpk身份安装运行hadoop 2.5.2的，所以这里填写zkpk，如果你是用root安装的，相应的改成root

新建工程

运行参数

参考：
http://www.cnblogs.com/yjmyzz/p/how-to-remote-debug-hadoop-with-eclipse-and-intellij-idea.html