简单wordcount编写及放到Hadoop集群中执行

最新推荐文章于 2024-07-17 17:53:42 发布

weixin_34024034

最新推荐文章于 2024-07-17 17:53:42 发布

阅读量83

点赞数

文章标签：大数据 java python

原文链接：https://my.oschina.net/u/3197158/blog/906222

版权

2019独角兽企业重金招聘Python工程师标准>>>

1.在编写mapreduce程序之前要先找齐maven依赖或所需要的jar包

maven依赖pom.xml：

<?xml version="1.0"?>
<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>lct</groupId>
<artifactId>cleaner</artifactId>
<version>0.0.1-SNAPSHOT</version>
</parent>
<groupId>lct</groupId>
<artifactId>hadoop</artifactId>
<version>0.0.1-SNAPSHOT</version>
<name>hadoop</name>
<url>http://maven.apache.org</url>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>

<dependencies>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.12</version>
<scope>test</scope>
</dependency>
<dependency>
   <groupId>org.apache.hadoop</groupId>
   <artifactId>hadoop-common</artifactId>
   <version>2.6.0</version>
</dependency>

<dependency>
   <groupId>org.apache.hadoop</groupId>
   <artifactId>hadoop-hdfs</artifactId>
   <version>2.6.0</version>
</dependency>

<dependency>
   <groupId>org.apache.hadoop</groupId>
   <artifactId>hadoop-client</artifactId>
   <version>2.6.0</version>
</dependency>
</dependencies>
<build>
<sourceDirectory>src/main/java</sourceDirectory>
<testSourceDirectory>src/test/java</testSourceDirectory>
<resources>
<resource>
<directory>resources</directory>
</resource>
</resources>
<plugins>
   <plugin>
   <artifactId>maven-assembly-plugin</artifactId>
   <configuration>
   <descriptorRefs>
   <descriptorRef>jar-with-dependencies</descriptorRef>
   </descriptorRefs>
   </configuration>
   <executions>

<execution>
<id>make-assembly</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
   </plugin>
   <plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<source>1.7</source> 
<target>1.7</target> 

</configuration>
</plugin>
</plugins>
<defaultGoal>compile</defaultGoal>
</build>
</project>

源代码：

Map:

package lct_mapreduce;

import java.io.IOException;

import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

public class MapTest extends Mapper<LongWritable, Text, Text, LongWritable>{

//map端从文件中读取数据时，一次读一行，行号作为key(k1)，该行的内容作为value(v1)
   @Override
   protected void map(LongWritable key, Text value,
           Mapper<LongWritable, Text, Text, LongWritable>.Context context)
           throws IOException, InterruptedException {
       // TODO Auto-generated method stub
       Text k2 = new Text();
       LongWritable v2 = new LongWritable();
       String line = value.toString();
       String[] splits = line.split("\t");
       for (String str : splits) {

//被切分后，一行中可能包含多个值，每个值都要写到HDFS上。要根据具体情况调整程序的逻辑结构
           k2.set(str);
           v2.set(1L);
           context.write(k2, v2);
       }

   }

}

Reduce:

package lct_mapreduce;

import java.io.IOException;

import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

public class ReduceTest extends Reducer<Text, LongWritable,Text, LongWritable>{
   @Override
   protected void reduce(Text k2, Iterable<LongWritable> v2,
           Reducer<Text, LongWritable, Text, LongWritable>.Context context)
           throws IOException, InterruptedException {
       // TODO Auto-generated method stub

//分组gropups、排序sort、分区partitions都是在reduce端完成的。
       LongWritable v3 = new LongWritable();
       long sum=0;
       for (LongWritable lg : v2) {
           sum+=lg.get();
       }
       v3.set(sum);

//把reduce端处理好的结果写到我们指定的目录下面
context.write(k2, v3);
}

}

App:

package lct_mapreduce;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

public class TestApp {
   public static void main(String[] args) throws Exception {
       //获取输入输出路径
       String inputPath=args[0];
       Path outputDir=new Path(args[1]);
       //加载配置文件
       Configuration configs = new Configuration();
       //如果输出路径已经存在，则删除已经存在的路径，因为输出路径已经存在，会报错
       outputDir.getFileSystem(configs).delete(outputDir, true);
       //获取一个作业对象
       Job job = Job.getInstance(configs, TestApp.class.getSimpleName());
       //设置打jar包必须的语句
       job.setJarByClass(TestApp.class);
       //设置待处理文件的输入路径
       FileInputFormat.setInputPaths(job, inputPath);
       //设置处理后的结果输出路径
       FileOutputFormat.setOutputPath(job, outputDir);

       //设置自定义的map
       job.setMapperClass(MapTest.class);
       job.setMapOutputKeyClass(Text.class);
       job.setMapOutputValueClass(LongWritable.class);

       //设置自定义的reduce
       job.setReducerClass(ReduceTest.class);
       job.setOutputKeyClass(Text.class);
       job.setOutputValueClass(LongWritable.class);

       //提交作业运行
       job.waitForCompletion(true);
   }