关于所在节点核数怎么看?
======================================================================
# 总核数 = 物理CPU个数 X 每颗物理CPU的核数
# 总逻辑CPU数 = 物理CPU个数 X 每颗物理CPU的核数 X 超线程数
# 查看物理CPU个数
cat /proc/cpuinfo| grep "physical id"| sort| uniq| wc -l
# 查看每个物理CPU中core的个数(即核数)
cat /proc/cpuinfo| grep "cpu cores"| uniq
# 查看逻辑CPU的个数
cat /proc/cpuinfo| grep "processor"| wc -l
======================================================================
spark资源主要就是core和memery。
spark主题功能分三部分:spark RDD,sparkSQL,spark shell,如果每个部分的功能都要用,那么每块都要占用资源。
其中,spark RDD和spark shell 是动态分配占用资源的,sparkSQL是静态分配资源的(启动后即一直占着分配的资源)
spark分配的总体资源在哪里看?
cat /home/mr/spark/conf/spark-env.sh
JAVA_HOME=/usr/java/jdk
SPARK_HOME=/home/mr/spark
SPARK_PID_DIR=/home/mr/spark/pids
SPARK_LOCAL_DIRS=/data2/zdh/spark/tmp,/data3/zdh/spark/tmp,/data4/zdh/spark/tmp
SPARK_WORKER_DIR=/data2/zdh/spark/work
SPARK_LOG_DIR=/data1/zdh/spark/logs
SPARK_HISTORY_OPTS="-Dspark.history.ui.port=18088-Dspark.history.retainedApplications=500"
SPARK_MASTER_WEBUI_PORT=18080
SPARK_WORKER_WEBUI_PORT=18081
SPARK_WORKER_CORES=25
SPARK_WORKER_MEMORY=150g
SPARK_DAEMON_MEMORY=2g
SPARK_LOCAL_HOSTNAME=`hostname`
YARN_CONF_DIR=/home/mr/yarn/etc/hadoop
SparkSQL的总体资源在哪看?
cat /home/mr/spark/conf/sparksql-default.conf
JAVA_HOME=/usr/java/jdk
SPARK_HOME=/home/mr/spark
SPARK_PID_DIR=/home/mr/spark/pids
SPARK_LOCAL_DIRS=/data2/zdh/spark/tmp,/data3/zdh/spark/tmp,/data4/zdh/spark/tmp
SPARK_WORKER_DIR=/data2/zdh/spark/work
SPARK_LOG_DIR=/data1/zdh/spark/logs
SPARK_HISTORY_OPTS="-Dspark.history.ui.port=18088-Dspark.history.retainedApplications=500"
SPARK_MASTER_WEBUI_PORT=18080
SPARK_WORKER_WEBUI_PORT=18081
SPARK_WORKER_CORES=25
SPARK_WORKER_MEMORY=150g
SPARK_DAEMON_MEMORY=2g
SPARK_LOCAL_HOSTNAME=`hostname`
YARN_CONF_DIR=/home/mr/yarn/etc/hadoop
[root@vmax47 conf]# catsparksql-defaults.conf
spark.serializer=org.apache.spark.serializer.KryoSerializer
spark.driver.extraJavaOptions=-Xss32m-XX:PermSize=128M -XX:MaxPermSize=512m
spark.driver.extraClassPath=/home/mr/spark/libext/*
spark.executor.extraClassPath=/home/mr/spark/libext/*
spark.executor.memory=10g
spark.eventLog.enabled=true
spark.eventLog.dir=/data1/zdh/spark/logs/eventLog
spark.history.fs.logDirectory=/data1/zdh/spark/logs/eventLog
spark.worker.cleanup.enabled=true
spark.shuffle.consolidateFiles=true
spark.ui.retainedJobs=200
spark.ui.retainedStages=200
spark.deploy.retainedApplications=100
spark.deploy.retainedDrivers=100
spark.speculation=true
spark.speculation.interval=1000
spark.speculation.multiplier=4
spark.speculation.quantile=0.85
spark.shuffle.service.enabled=false
spark.dynamicAllocation.enabled=false
spark.dynamicAllocation.minExecutors=0
spark.dynamicAllocation.maxExecutors=2147483647
spark.sql.broadcastTimeout=600
spark.yarn.queue=mr
spark.master=spark://vmax47:7077,SPARK49:7077
spark.deploy.recoveryMode=ZOOKEEPER
spark.deploy.zookeeper.url=SPARK49:2181,HADOOP50:2181,vmax47:2181
spark.ui.port=4100
spark.driver.memory=40G
spark.cores.max=30
查看Spark资源可从18080端口查看: