batch与spark spring_java – 与Spark一起使用Spring

从问题提出：添加：直接干扰反序列化部分,而不修改自己的类使用parapluplu使用以下

spring-spark project.这个项目在春天被反序列化时自动打开你的bean.

编辑：

为了使用Spark,您需要以下设置(也可在this repository中看到)：

>春季启动Spark：

.

org.springframework.boot

spring-boot-starter-parent

1.5.2.RELEASE

…

org.springframework.boot

spring-boot-starter-web

ch.qos.logback

logback-classic

org.springframework.boot

spring-boot-starter-test

test

org.apache.spark

spark-core_2.11

2.1.0

org.slf4j

slf4j-log4j12

log4j

log4j

org.apache.spark

spark-sql_2.11

2.1.0

org.codehaus.janino

commons-compiler

2.7.8

org.slf4j

log4j-over-slf4j

1.7.25

org.slf4j

slf4j-api

1.7.5

org.slf4j

slf4j-simple

1.6.4

然后你需要应用程序类,像往常一样使用Spring Boot：

@SpringBootApplication

public class SparkExperimentApplication {

public static void main(String[] args) {

SpringApplication.run(SparkExperimentApplication.class, args);

}

}

然后一个将它绑定在一起的配置

@Configuration

@PropertySource("classpath:application.properties")

public class ApplicationConfig {

@Autowired

private Environment env;

@Value("${app.name:jigsaw}")

private String appName;

@Value("${spark.home}")

private String sparkHome;

@Value("${master.uri:local}")

private String masterUri;

@Bean

public SparkConf sparkConf() {

SparkConf sparkConf = new SparkConf()

.setAppName(appName)

.setSparkHome(sparkHome)

.setMaster(masterUri);

return sparkConf;

}

@Bean

public JavaSparkContext javaSparkContext() {

return new JavaSparkContext(sparkConf());

}

@Bean

public SparkSession sparkSession() {

return SparkSession

.builder()

.sparkContext(javaSparkContext().sc())

.appName("Java Spark SQL basic example")

.getOrCreate();

}

@Bean

public static PropertySourcesPlaceholderConfigurer propertySourcesPlaceholderConfigurer() {

return new PropertySourcesPlaceholderConfigurer();

}

}

然后可以使用SparkSession类与Spark SQL通信：

/**

* Created by achat1 on 9/23/15.

* Just an example to see if it works.

*/

@Component

public class WordCount {

@Autowired

private SparkSession sparkSession;

public List count() {

String input = "hello world hello hello hello";

String[] _words = input.split(" ");

List words = Arrays.stream(_words).map(Word::new).collect(Collectors.toList());

Dataset dataFrame = sparkSession.createDataFrame(words, Word.class);

dataFrame.show();

//StructType structType = dataFrame.schema();

RelationalGroupedDataset groupedDataset = dataFrame.groupBy(col("word"));

groupedDataset.count().show();

List rows = groupedDataset.count().collectAsList();//JavaConversions.asScalaBuffer(words)).count();

return rows.stream().map(new Function() {

@Override

public Count apply(Row row) {

return new Count(row.getString(0), row.getLong(1));

}

}).collect(Collectors.toList());

}

}

参考这两个类：

public class Word {

private String word;

public Word() {

}

public Word(String word) {

this.word = word;

}

public void setWord(String word) {

this.word = word;

}

public String getWord() {

return word;

}

}

public class Count {

private String word;

private long count;

public Count() {

}

public Count(String word, long count) {

this.word = word;

this.count = count;

}

public String getWord() {

return word;

}

public void setWord(String word) {

this.word = word;

}

public long getCount() {

return count;

}

public void setCount(long count) {

this.count = count;

}

}

然后你可以运行看到它返回正确的数据：

@RequestMapping("api")

@Controller

public class ApiController {

@Autowired

WordCount wordCount;

@RequestMapping("wordcount")

public ResponseEntity> words() {

return new ResponseEntity<>(wordCount.count(), HttpStatus.OK);

}

}

说

[{"word":"hello","count":4},{"word":"world","count":1}]