hive版本wordcount

1. wordcount程序相当于hadoop MapReduce的一个helloworld程序吧，主要是将文件中的单词内容一行一行得读入，在map端进行拆分，拆成key-value的形式, key是具体的单词，value是数字1，map到reduce的过程会进行一次归并，将key一样的进行合并组成key-values的形式，其中key是具体的单词，values是很多个1，在reduce端将这个values循环相加就是这个单词的个数。

2. 纯的MR代码如下：

    

/**
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 */

package com.jthink.bg.hellowrold;

import java.io.File;
import java.io.IOException;
import java.util.StringTokenizer;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

public class WordCount {

    public static class TokenizerMapper extends Mapper<Object, Text, Text, IntWritable> {

        private final static IntWritable one = new IntWritable(1);
        private Text word = new Text();

        public void map(Object key, Text value, Context context) throws IOException, InterruptedException {
            StringTokenizer itr = new StringTokenizer(value.toString());
            while (itr.hasMoreTokens()) {
                word.set(itr.nextToken());
                context.write(word, one);
            }
        }
    }

    public static class IntSumReducer extends Reducer<Text, IntWritable, Text, IntWritable> {
        private IntWritable result = new IntWritable();

        public void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException,
                InterruptedException {
            int sum = 0;
            for (IntWritable val : values) {
                sum += val.get();
            }
            result.set(sum);
            context.write(key, result);
        }
    }

    public static void main(String[] args) throws Exception {
        Configuration conf = new Configuration();
        Job job = new Job(conf, "word count");

//        File jarFile = EJob.createTempJar("bin");
//        System.out.println("jarFile==" + jarFile);
//        ((JobConf) job.getConfiguration()).setJar(jarFile.toString());

        job.setJarByClass(WordCount.class);
        job.setMapperClass(TokenizerMapper.class);
        job.setCombinerClass(IntSumReducer.class);
        job.setReducerClass(IntSumReducer.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(IntWritable.class);
        FileInputFormat.addInputPath(job, new Path("hdfs://bg01:9000/bg/wordcount/input"));
        FileOutputFormat.setOutputPath(job, new Path("hdfs://bg01:9000/bg/wordcount/output"));
        System.exit(job.waitForCompletion(true) ? 0 : 1);
    }
}

3. 这样做需要写很多java代码，但是如果放到hive中就比较简单（关于hive是什么就不细说了），具体做法如下：

    a. 创建一个数据库，如levi

        create database levi;

    b. 建表

create external table src_data(line string) row format delimited fields terminated by '\n' stored as textfile location '/levi/wordcount/src_data';

这里假设我们的数据存放在hadoop下，路径为：/levi/wordcount/src_data，里面主要是一些单词文件，内容大概为：

hi man
what is your name
my name is levi
you
kevin

执行了上述hql就会创建一张表src_data，内容是这些文件的每行数据，每行数据存在字段line中，select * from src_data; 就可以看到这些数据

    c. 根据MapReduce的规则，我们需要进行拆分，把每行数据拆分成单词，这里需要用到一个hive的内置表生成函数（UDTF）：explode(array)，参数是array，其实就是行变多列：

create table words(word string);

insert into table words select explode(split(line, " ")) as word from src_data;

split是拆分函数，跟java的split功能一样，这里是按照空格拆分，所以执行完hql语句，words表里面就全部保存的单个单词

    d. 这样基本实现了，因为hql可以group by，所以最后统计语句为：

select word, count(*) from levi.words group by word;

4. 对比写MR和写hive，还是hive比较简便，对于比较复杂的统计操作可以建一些中间表，或者一些视图之类的，之后博客会持续更新hive的一些操作。