hadoop2.7.1 Intellj idea 远程提交job到linux集群

本文章主要说明windows环境下使用idea远程提交job到linux的hadoop集群。

hadoop集群的部署网上有很多，我自己在虚拟机上搭建了伪集群用于个人学习使用。

下面以wordcount例子为例：

先附上目录结构：

ConfigUtil为配置工具类，FileSystemUtil为文件工具类。

ConfigUtil.java内容如下：

 static Configuration getConfigured(){
        Configuration conf = new Configuration();
        conf.set("fs.defaultFS", "hdfs://10.60.20.111:9000");
        conf.set("mapreduce.framework.name", "yarn");
        conf.set("yarn.resourcemanager.address", "http://10.60.20.111:8032");
        conf.set("mapreduce.app-submission.cross-platform", "true");
        //conf.set("mapreduce.job.jar", "E:\\workspace\\idea\\hadoop_wordcount_demo\\classes\\wordcount.jar");
        return conf;
    }

FileSystemUtil.java文件内容如下：

static void deleteDir(Configuration conf, String dirPath) throws IOException {
        //FileSystem fs = FileSystem.get(conf);
        Path targetPath = new Path(dirPath);
        FileSystem fs = targetPath.getFileSystem(conf);
        if (fs.exists(targetPath)) {
            boolean delResult = fs.delete(targetPath, true);
            if (delResult) {
                System.out.println(targetPath + " has been deleted sucessfullly.");
            } else {
                System.out.println(targetPath + " deletion failed.");
            }
        }
    }

    static File createJar(Class<?> clazz) throws Exception {
        String fqn = clazz.getName();
        String base = fqn.substring(0, fqn.lastIndexOf("."));
        base = base.replaceAll("\\.", Matcher.quoteReplacement("/"));
        URL root = clazz.getResource("");

        JarOutputStream out = null;
        final File jar = File.createTempFile("HadoopRunningJar-", ".jar", new File(System.getProperty("java.io.tmpdir")));
        System.out.println(jar.getAbsolutePath());
        Runtime.getRuntime().addShutdownHook(new Thread() {
            public void run() {
                jar.delete();
            }
        });
        try {
            File path = new File(root.toURI());
            Manifest manifest = new Manifest();
            manifest.getMainAttributes().putValue("Manifest-Version", "1.0");
            manifest.getMainAttributes().putValue("Created-By", "RemoteHadoopUtil");
            manifest.getMainAttributes().putValue("Main-Class", fqn);
            out = new JarOutputStream(new FileOutputStream(jar), manifest);
            writeBaseFile(out, path, base);
        } finally {
            if (out != null) {
                out.flush();
                out.close();
            }
        }
        return jar;
    }

    /**
     * 递归添加.class文件
     */
    private static void writeBaseFile(JarOutputStream out, File file, String base) throws IOException {
        if (file.isDirectory()) {
            File[] fl = file.listFiles();
            if (base.length() > 0) {
                base = base + "/";
            }
            for (File aFl : fl) {
                writeBaseFile(out, aFl, base + aFl.getName());
            }
        } else {
            out.putNextEntry(new JarEntry(base));
            try (FileInputStream in = new FileInputStream(file)) {
                byte[] buffer = new byte[1024];
                int n = in.read(buffer);
                while (n != -1) {
                    out.write(buffer, 0, n);
                    n = in.read(buffer);
                }
            }
        }
    }

WordCount.java文件内容如下：

private static class TokenizerMapper extends Mapper<LongWritable, Text, Text, IntWritable> {
        private final static IntWritable one = new IntWritable(1);
        private Text word = new Text();

        public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
            StringTokenizer itr = new StringTokenizer(value.toString());
            while (itr.hasMoreTokens()) {
                word.set(itr.nextToken());
                context.write(word, one);
            }
        }
    }

    private static class IntSumReducer extends Reducer<Text, IntWritable, Text, IntWritable> {
        private IntWritable result = new IntWritable();

        public void reduce(Text key, Iterable<IntWritable> values, Context context)
                throws IOException, InterruptedException {
            int sum = 0;
            for (IntWritable val : values) {
                sum += val.get();
            }
            result.set(sum);
            context.write(key, result);
        }
    }

    public static void main(String[] args) throws Exception {
        System.setProperty("hadoop.home.dir", "D:\\Program Files\\hadoop-2.7.1");
        Configuration conf = ConfigUtil.getConfigured();
        String hdfs = conf.get("fs.defaultFS");
        args = new String[] { hdfs + "/tzwang/input/wordcount/", hdfs + "/tzwang/output/wordcount"};
        //delete output dir
        deleteDir(conf, args[args.length - 1]);
        Job job = Job.getInstance(conf, "word count");
        File jarFile = createJar(WordCount.class);
        ((JobConf) job.getConfiguration()).setJar(jarFile.toString());
        job.setJarByClass(WordCount.class);
        job.setMapperClass(TokenizerMapper.class);
        job.setReducerClass(IntSumReducer.class);
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(IntWritable.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(IntWritable.class);
        FileInputFormat.addInputPath(job, new Path(args[0]));
        FileOutputFormat.setOutputPath(job, new Path(args[1]));
        System.exit(job.waitForCompletion(true) ? 0 : 1);
    }

ConfigUtil.java中的属性可以配置在xml文件中，具体可以参照linux环境下的配置文件，再使用conf.addResource()导入。该类中的所有属性必须设置，其余的可以不加。

因为我使用的是yarn，所以该属性；不适用yarn的使用mapreduce.jobtracker.address属性。

这个属性可以允许windows提交应用到linux。

如果遇到权限禁止访问的问题，mapred-site.xml中添加

<property>
<name>dfs.permissions.enabled</name>
<value>false</value>
</property>

System.setProperty("hadoop.home.dir", “");这个用来设置本地hadoop的路径。

FileSystemUtil.java中的createJar方法用来自动生成jar包提交，若不适用该方法，可以手动指定jar的绝对路径，conf.set("mapreduce.job.jar",”“)；

对于输入和输出路径可以在程序中指定，或者在运行配置中的Program arguments中配置。