windows提交mr到linux报错,2020-09-26-Hadoop-7(yarn调度原理初步重写分区/分组器)

最新推荐文章于 2023-09-05 16:52:20 发布

weixin_39964869

最新推荐文章于 2023-09-05 16:52:20 发布

阅读量108

点赞数

文章标签： windows提交mr到linux报错

1.windows上提交MR程序

代码同下需要变化的如下

// 1 配置对象

System.setProperty("HADOOP_USER_NAME", "root");

Configuration configuration = new Configuration();

//处理HDFS中的数据

configuration.set("fs.default.name", "hdfs://linux03:8020");

// 设置MR程序运行模式

configuration.set("mapreduce.framework.name", "yarn");

// 程序yarn的位置

configuration.set("yarn.resourcemanager.hostname", "linux03");

//设置跨平台参数

configuration.set("mapreduce.app-submission.cross-platform", "true");

Job job = Job.getInstance(configuration, "max3");

//获取对象设置包的位置

job.setJar("C:\\Users\\hp\\Desktop\\考试题及课堂笔记\\demo.jar");

2.linux提交MR程序

hadoop伪分布式wordcount报错Container exited with a non-zero exit code 1. Error file: prelaunch.err

错误的解决mapred-site.xml文件中添加mapreduce所需要用到的classpath

mapreduce.application.classpath

$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*, $HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*

mapreduce.application.classpath

/opt/hadoop/hadoop-3.1.1/share/hadoop/mapreduce/*, /opt/hadoop/hadoop-3.1.1/share/hadoop/mapreduce/lib/*

代码:

import com.google.gson.Gson;

import com.google.gson.JsonSyntaxException;

import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.fs.Path;

import org.apache.hadoop.io.DoubleWritable;

import org.apache.hadoop.io.LongWritable;

import org.apache.hadoop.io.NullWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.Job;

import org.apache.hadoop.mapreduce.Mapper;

import org.apache.hadoop.mapreduce.Reducer;

import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;

import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import java.io.IOException;

//电影评分和

public class MapReduce_Map_Reduce1 {

public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {

Configuration configuration = new Configuration();

configuration.set("fs.default.name", "hdfs://linux03:8020");

configuration.set("mapreduce.framework.name", "yarn");

configuration.set("yarn.resourcemanager.hostname", "linux03");

configuration.set("mapreduce.app-submission.cross-platform", "true");

Job job = Job.getInstance(configuration, "max3");

job.setJarByClass(MapReduce_Map_Reduce1.class);

job.setMapperClass(Map_Test1.class);

job.setReducerClass(Reduce_Text1.class);

//设置输入输出类型

job.setMapOutputKeyClass(Text.class);

job.setMapOutputValueClass(DoubleWritable.class);

job.setOutputKeyClass(Text.class);

job.setOutputValueClass(DoubleWritable.class);

job.setNumReduceTasks(2);

//设置来源和写入路径

//可变参数可传输多个路径

FileInputFormat.setInputPaths(job, new Path("/tmp/data/test.json"));

FileOutputFormat.setOutputPath(job, new Path("/tmp/consequence/consequencedata"));

job.waitForCompletion(true);

}

public static class Map_Test1 extends Mapper {

Gson gson = new Gson();

Text text = new Text();

DoubleWritable doubleWritable = new DoubleWritable();

@Override

protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {

try {

String string = value.toString();

MoviePoint moviePoint = gson.fromJson(string, MoviePoint.class);

text.set(moviePoint.getMovie());

doubleWritable.set(moviePoint.getRate());

context.write(text, doubleWritable);

} catch (Exception e) {

}

public static class Reduce_Text1 extends Reducer {

@Override

protected void reduce(Text key, Iterable values, Context context) throws IOException, InterruptedException {

DoubleWritable doubleWritable = new DoubleWritable();

double sum = 0;

for (DoubleWritable value : values) {

sum += value.get();

}

doubleWritable.set(sum);

context.write(key, doubleWritable);

}

mapred-site.xml(添加配置文件)

yarn.app.mapreduce.am.env