最近在入门spark+hadoop,伪分布式安装,部署推荐这几个地址,不错。这边顺手记录一下自己用到的两个小程序。
推荐教程
http://www.powerxing.com/install-hadoop/
http://blog.csdn.net/yeruby/article/details/41042713
http://blog.csdn.net/tongxinzhazha/article/details/54346311
maven配置
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.landi</groupId>
<artifactId>testspark</artifactId>
<version>0.0.1-SNAPSHOT</version>
<name>testspark</name>
<description>testspark</description>
<properties>
<jdk.version>1.8</jdk.version>
<spark.version>2.1.0</spark.version>
<hadoop.version>2.6.5</hadoop.version>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.10</artifactId>
<version>${spark.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>${hadoop.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>${hadoop.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<version>${hadoop.version}</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.1</version>
<configuration>
<source>${jdk.version}</source>
<target>${jdk.version}</target>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<version>2.4</version>
<configuration>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
<archive>
<manifest>
<addClasspath>true</addClasspath>
<mainClass>com.landi.testspark.TestSpark</mainClass>
</manifest>
</archive>
</configuration>
</plugin>
</plugins>
</build>
</project>
spark统计单词数demo
package com.landi.testspark;
import java.util.Arrays;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.Function;
import scala.Tuple2;
public class TestSpark {
public static void main(String[] args) {
//设置spark master节点地址,app名以及部署方式
SparkConf conf = new SparkConf()
.setMaster(args[1])
.setAppName(args[3]);
JavaSparkContext sc = new JavaSparkContext(conf);
//连接hdfs地址,读取输入内容,并分割处理
JavaRDD<String> textFile = sc.textFile("hdfs://localhost:9070/user/root/input/*").flatMap(s -> Arrays.asList(s.split("\\s")).iterator());
JavaPairRDD<String, Integer> result = textFile
.filter((String s) -> {
Boolean flag = false;
if(s.indexOf("<name>")!=-1){
flag = true;
}else if(s.indexOf("<property>")!=-1){
flag = true;
}else if(s.indexOf("<IF>")!=-1){
flag = true;
}
return flag;})
.mapToPair(word -> new Tuple2<>(word, 1))
.reduceByKey((a, b) -> a + b);
result.saveAsTextFile("hdfs://localhost:9070/user/root/output2");
//
}
}
打包,上传到服务器。
执行前清空一下hdfs
执行
bin/spark-submit testspark-0.0.1-SNAPSHOT-jar-with-dependencies.jar
--master spark://localhost:7077
--name reduce
--num-executors 10
读取hdfs并打印
package com.landi.testspark;
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.net.URISyntaxException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
public class LoadFromHdfs {
public static void main(String[] argvs) {
String uri = "hdfs://localhost:9070/user/root/output2/";
Configuration conf = new Configuration();
//出现无法识别hdsf的问题可以加上以下两句
conf.set("fs.hdfs.impl", "org.apache.hadoop.hdfs.DistributedFileSystem");
conf.set("fs.file.impl", "org.apache.hadoop.fs.LocalFileSystem");
try {
FileSystem fs = FileSystem.get(new URI(uri), conf);
Path path = new Path(uri);
FileStatus status[] = fs.listStatus(path);
for (int i = 0; i < status.length; i++) {
String pathname =status[i].getPath().toString();
System.out.println(pathname);
System.out.println("==========");
FileStatus tmp = status[i];
if(tmp.isFile()&&tmp.getLen()>0){//判断文件大小
InputStream in = fs.open(tmp.getPath());
StringBuffer sb = new StringBuffer("");
byte[] b = new byte[1];
while (in.read(b) != -1) {
// 字符串拼接
sb.append(new String(b));
}
System.out.println(sb.toString());
System.out.println("==========");
in.close();
}
}
} catch (IOException e) {
e.printStackTrace();
} catch (URISyntaxException e) {
e.printStackTrace();
}
}
}
打包上传服务器,执行java -jar LoadFromHdfs 。