hadoop应用1

新建maven项目

在pom.xml里面引入相关依赖

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>cn.edu.swpu.scs.WordCount</groupId>
    <artifactId>WordCount</artifactId>
    <version>1.0-SNAPSHOT</version>
    <packaging>jar</packaging>
    <name>WordCount</name>

    <properties>
        <maven.compiler.source>8</maven.compiler.source>
        <maven.compiler.target>8</maven.compiler.target>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
    </properties>
    <dependencies>
    <!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-hdfs -->
    <dependency>
        <groupId>org.apache.hadoop</groupId>
        <artifactId>hadoop-hdfs</artifactId>
        <version>3.1.2</version>
        <scope>test</scope>
    </dependency>
    <!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-client -->
    <dependency>
        <groupId>org.apache.hadoop</groupId>
        <artifactId>hadoop-client</artifactId>
        <version>3.1.2</version>
    </dependency>
    <!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-common -->
    <dependency>
        <groupId>org.apache.hadoop</groupId>
        <artifactId>hadoop-common</artifactId>
        <version>3.1.2</version>
    </dependency>
    <dependency>
        <groupId>org.apache.commons</groupId>
        <artifactId>commons-configuration2</artifactId>
        <version>2.7</version>
    </dependency>
        <dependency>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-api</artifactId>
            <version>1.7.30</version>
        </dependency>
    <dependency>
        <groupId>org.apache.hadoop</groupId>
        <artifactId>hadoop-common</artifactId>
        <version>3.1.2</version>
    </dependency>
    </dependencies>
</project>

按目录结构建包

将代码复制到WordCountApp中

import java.io.IOException;
import java.util.Iterator;
import java.util.StringTokenizer;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;

public class WordCountApp extends Configured implements Tool {

    public static void main( String[] args ) throws Exception {
        int res = ToolRunner.run(new WordCountApp(), args);
        System.exit(res);
    }

    // 执行
    public int run(String[] args) throws Exception {
        Configuration conf = new Configuration();

        System.out.println("---------创建和配置Job---------");
        System.out.println("------本代码在Client上执行------");

        // 创建作业
        Job job = Job.getInstance(conf, "WordCount");

        // 指定作业的主类
        job.setJarByClass(WordCountApp.class);

        // 指定Map和Reduce类
        job.setMapperClass(Map.class);
        job.setReducerClass(Reduce.class);
        job.setCombinerClass(Reduce.class);

        // 指定输入格式为：文本格式文件
        job.setInputFormatClass(TextInputFormat.class);
        TextInputFormat.addInputPath(job, new Path(args[0]));

        // 指定输出格式为：文本格式文件，键为文本、值为整型数
        job.setOutputFormatClass(TextOutputFormat.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(IntWritable.class);
        TextOutputFormat.setOutputPath(job, new Path(args[1]));

        // 执行MapReduce
        boolean res = job.waitForCompletion(true);
        if(res)
            return 0;
        else
            return -1;
    }

    // Map过程
    public static class Map extends Mapper<LongWritable, Text, Text, IntWritable> {
        private final static IntWritable one = new IntWritable(1);
        private Text word = new Text();
        @Override
        public void map(LongWritable key, Text value, Context context)
                throws IOException, InterruptedException {
            String line = value.toString();
  			/*
  			String[] words = line.split(" ");
  			for(String word : words){
  				context.write(new Text(word), one);
  			}
  			*/
            System.out.println("---------" + key.toString() + "---" + line + "---------");
            StringTokenizer tokenizer = new StringTokenizer(line);	//split line to words by space
            while (tokenizer.hasMoreTokens()) {						//operate all word by loop
                word.set(tokenizer.nextToken());
                context.write(word, one);							//write KV to context, word is key, word number is value
            }
        }
    }

    // Reduce过程
    public static class Reduce extends Reducer<Text, IntWritable, Text, IntWritable> {
        @Override
        public void reduce(Text key, Iterable<IntWritable> val, Context context)	//mothod for each key，input format key(value1,value2,......)
                throws IOException, InterruptedException {
            int sum = 0;
            Iterator<IntWritable> values = val.iterator();
            while (values.hasNext()) {
                sum += values.next().get();					//sum value(one word count)
            }
            context.write(key, new IntWritable(sum));		//write one key and its count
        }
    }
}

在文件中打开项目结构

点击工件

点击新建空的jar包

更改jar包名称

创建清单，选择到src那一层

选择主类和路径

在jar包下面建目录（与自己的项目结构对应），然后再添加class文件（要先编译执行一次，target目录才有class文件）

创建一个META-INF目录，添加下面的文件（如果没有这个文件，说明没有创建清单）

构建工件（即打包）

然后在out目录下就有相应的文件了

把jar包上传至曙光平台

改成ask every time

在自己电脑上找到该目录

选择一个文件夹

然后运行jar包（输入路径是给定的路径，输出路径必须在/user/登录用户名 的下面，其他的大部分没有权限）

如果运行时报找不到类的错误，说明是jar包有问题，检查jar包里面的路径和清单文件里面的路径是否一致。