idea本地和远程测试hadoop和spark

两份代码的位置：

链接：https://pan.baidu.com/s/1cZLKhuYCnCJunsbs89CVnQ 密码：qwwy

wordcount_s====>spark代码

wordcount====>hadoop代码

hadoop的代码：

1.本地运行：

代码：

package com.itstar;

import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.fs.FileSystem;

import org.apache.hadoop.fs.Path;

import org.apache.hadoop.io.IntWritable;

import org.apache.hadoop.io.LongWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.Job;

import org.apache.hadoop.mapreduce.Mapper;

import org.apache.hadoop.mapreduce.Reducer;

import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;

import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import java.io.IOException;

 

public class wordCount_h {

public static class mymap extends Mapper<LongWritable,Text,Text,IntWritable>{

final static IntWritable one=new IntWritable(1);

@Override

protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {

String [] words=value.toString().split(" ");

for (String word:words){

context.write(new Text(word),one);

}

}

}

public static class myreduce extends Reducer<Text,IntWritable,Text,IntWritable>{

@Override

protected void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException {

int sum =0;

for(IntWritable val:values){

sum+= val.get();

}

context.write(key,new IntWritable(sum));

}

}

public static void main(String[] args) throws Exception{

System.setProperty("HADOOP_USER_NAME","hadoop");

Configuration conf= new Configuration();

Job job=Job.getInstance(conf,"wordcount_hadoop");

job.setJarByClass(wordCount_h.class);

if(args.length<2){

System.err.println("two path!");

return;

}

FileSystem file=FileSystem.get(conf);

Path p=new Path(args[1]);

if(file.exists(p))

file.delete(p,true);

FileInputFormat.addInputPath(job,new Path(args[0]));

job.setMapperClass(mymap.class);

job.setMapOutputKeyClass(Text.class);

job.setMapOutputValueClass(IntWritable.class);

job.setReducerClass(myreduce.class);

job.setOutputKeyClass(Text.class);

job.setOutputValueClass(IntWritable.class);

FileOutputFormat.setOutputPath(job,new Path(args[1]));

jo