0.引用
https://blog.csdn.net/qq_28652401/article/details/83506767
1.livy使用
livy-conf
export SPARK_HOME=/root/new_dev/worsk/my_spark/spark-2.4.0-bin-hadoop2.6
export HADOOP_CONF_DIR=/root/new_dev/worsk/my_hadoop/hadoop-3.2.0/etc/hadoop/conf
livy.server.session.timeout = 1h
livy.impersonation.enabled = true
livy.repl.enableHiveContext = true
livy.spark.master = local
livy.file.local-dir-whitelist = /root/new_dev/worsk/my_livy/mylocal_data_path
使用接口上传文件的时候,这里 livy.file.local-dir-whitelist =/root/.livy-sessions/
2.简单resutful接口
=======step1======
curl -X POST --data '{"files":["file:/root/new_dev/worsk/my_livy/mylocal_data_path/livy.code-1.0-SNAPSHOT.jar"],"proxyUser":"wangly","conf":{"numExecutors":1,"driverMemory":"100m","executorMemory":"100m","driverCores":1,"executorCores":1,"spark.driver.extraJavaOptions":"-XX:MetaspaceSize=10m -XX:MaxMetaspaceSize=256m"}}' -H "Content-Type: application/json" localhost:8998/sessions
=======step2======
curl localhost:8998/sessions/0/statements -X POST -H 'Content-Type: application/json' -d '{"kind":"scala","code":"var a = 1;var b=a+1;println(\"=======>\"+b)"}'
======2=result
{"id":1,"code":"var a = 1;var b=a+1;println(\"=======>\"+b)","state":"waiting","output":null,"progress":0.0}
=======step3======
curl localhost:8998/sessions/0/statements/1
======3=result
{"id":1,"code":"var a = 1;var b=a+1;println(\"=======>\"+b)","state":"available","output":{"status":"ok","execution_count":1,"data":{"text/plain":"=======>2\na: Int = 1\nb: Int = 2\n"}},"progress":1.0}
=======step4======
curl -X DELETE localhost:8998/sessions/0