启动
jerry用户:
mongo:
mongod --dbpath=opt/mongodb-3.4.0/data/db/ --logpath=opt/mongodb-3.4.0/log/mongod.log
mongod --dbpath=opt/mongodb-3.4.0/data/db/ --logpath=opt/mongodb-3.4.0/log/mongod.log --auth
hadoop用户:
zookeeper-3.4.11/bin/zkCli.sh -server 10.26.51.89:2181 ---存在内网,外网的情况下,指定内网IP进行访问
存在内外网的情况下: zookeeper.connect 如果无法指定所有的ip:port/chroot(会出现(/kafka10,10.26.51.89:2181),ip,chroot粘连的情况),可以只指定自身的ip:port/chroot
redis:
redis-server /usr/redis/redis.conf &
Zookeeper:
zkServer.sh start
HDFS:
start-dfs.sh
kafka:
kafka-server-start.sh /usr/kafka/config/server.properties &
kafka-server-start.sh /usr/kafka/config/server1.properties &
flume:
flume-ng agent -c conf -f $FLUME_HOME/conf/hdfsAndKafka.conf -n a1 -Dflume.root.logger=INFO,console
/home/hadoop/properties/hdfs.properties
spark:
/usr/spark2.2/sbin/start-all.sh
启动应用
-Dspark.deploy.defaultCores coreNums(设置启动cpu核数)
--conf spark.cores.max=N (spark application 分配的最大核数
spark-submit --class com.sznongfu.cn.center.StartCenter --name sznongfuzhanggui --master spark://hadoop:7077 --executor-memory 1G --total-executor-cores 10 --conf "spark.executor.extraJavaOptions=-XX:+PrintGCDetails -XX:+PrintGCTimeStamps" --conf "spark.executor.extraJavaOptions= -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+ParallelRefProcEnabled -XX:+CMSClassUnloadingEnabled -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintGCTimeStamps -XX:+PrintHeapAtGC -XX:+HeapDumpOnOutOfMemoryError -verbose:gc -XX:MaxTenuringThreshold=10 -XX:SurvivorRatio=8" --driver-java-options "-XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+ParallelRefProcEnabled -XX:+CMSClassUnloadingEnabled -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintGCTimeStamps -XX:+PrintHeapAtGC -XX:+HeapDumpOnOutOfMemoryError -verbose:gc -XX:MaxTenuringThreshold=10 -XX:SurvivorRatio=8" /home/jerry/racoon/nongfu.ngx.log.analyser/modules/LogProcess/out/libs/LogProcess-1.0-SNAPSHOT-all.jar /home/hadoop/properties/hdfs.properties com.sznongfu.cn.online.statistic.NginxLogStatistic
spark-submit --class com.sznongfu.cn.center.StartCenter --name sznongfuzhanggui --master spark://hadoop:7077 --executor-memory 1G --total-executor-cores 10 --conf "spark.executor.extraJavaOptions=-XX:+PrintGCDetails -XX:+PrintGCTimeStamps" --conf "spark.executor.extraJavaOptions= -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+ParallelRefProcEnabled -XX:+CMSClassUnloadingEnabled -XX:+HeapDumpOnOutOfMemoryError -XX:MaxTenuringThreshold=10 -XX:SurvivorRatio=8" --driver-java-options "-XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+ParallelRefProcEnabled -XX:+CMSClassUnloadingEnabled -XX:+HeapDumpOnOutOfMemoryError -XX:MaxTenuringThreshold=10 -XX:SurvivorRatio=8" /home/jerry/racoon/nongfu.ngx.log.analyser/modules/LogProcess/out/libs/LogProcess-1.0-SNAPSHOT-all.jar /home/hadoop/properties/hdfs.properties com.sznongfu.cn.online.statistic.NginxLogStatistic
--conf "spark.executor.extraJavaOptions=-XX:+PrintGCDetails -XX:+PrintGCTimeStamps" --conf "spark.executor.extraJavaOptions= -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+ParallelRefProcEnabled -XX:+CMSClassUnloadingEnabled -XX:+HeapDumpOnOutOfMemoryError -XX:MaxTenuringThreshold=10 -XX:SurvivorRatio=8"
--driver-java-options "-XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+ParallelRefProcEnabled -XX:+CMSClassUnloadingEnabled -XX:+HeapDumpOnOutOfMemoryError -XX:MaxTenuringThreshold=10 -XX:SurvivorRatio=8"
/* 备用 不可用
spark-submit --class com.sznongfu.cn.center.StartCenter --name sznongfuzhanggui --master spark://hadoop:7077 --executor-memory 1G --total-executor-cores 5 --conf "spark.executor.extraJavaOptions=-XX:+UseG1GC -XX:+PrintFlagsFinal -XX:+PrintReferenceGC -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintAdaptiveSizePolicy -XX:+UnlockDiagnosticVMOptions -XX:+G1SummarizeConcMark -XX:InitiatingHeapOccupancyPercent=35 -XX:ConcGCThread=20" /home/jerry/racoon/nongfu.ngx.log.analyser/modules/LogProcess/out/libs/LogProcess-1.0-SNAPSHOT-all.jar /home/hadoop/properties/hdfs.properties com.sznongfu.cn.online.statistic.NginxLogStatistic
备用 不可用*/
/* 可用 */
spark-submit --class com.sznongfu.cn.center.StartCenter --name sznongfuzhanggui --master spark://hadoop:7077 --executor-memory 1G --total-executor-cores 5 --conf "spark.executor.extraJavaOptions=-XX:+UseG1GC -XX:+PrintFlagsFinal -XX:+PrintReferenceGC -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintAdaptiveSizePolicy -XX:+UnlockDiagnosticVMOptions -XX:+G1SummarizeConcMark " /home/jerry/racoon/nongfu.ngx.log.analyser/modules/LogProcess/out/libs/LogProcess-1.0-SNAPSHOT-all.jar /home/hadoop/properties/hdfs.properties com.sznongfu.cn.online.statistic.NginxLogStatistic
/* 可用 */
英特尔最佳JVM实践
-XX:+UseG1GC -XX:+PrintFlagsFinal -XX:+PrintReferenceGC -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintAdaptiveSizePolicy -XX:+UnlockDiagnosticVMOptions -XX:+G1SummarizeConcMark -Xms88g -Xmx88g -XX:InitiatingHeapOccupancyPercent=35 -XX:ConcGCThread=20
kafka GC方式:
00:05:54 /home/sznongfu/opt/jdk-1.8.0/bin/java -Xmx512M -server -XX:+UseG1GC -XX:MaxGCPauseMillis=20 -XX:InitiatingHeapOccupancyPercent=35 -XX:+DisableExplicitGC -Djava.awt.headless=true -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dkafka.logs.dir=/home/sznongfu/opt/kafka_2.11-0.10.0.0/bin/../logs -Dlog4j.configuration=file:/home/sznongfu/opt/kafka_2.11-0.10.0.0/bin/../config/tools-log4j.properties -cp
spark-submit --class com.sznongfu.cn.center.StartCenter --name sznongfuzhanggui --master spark://hadoop:7077 --executor-memory 2G --total-executor-cores 2 --conf "spark.executor.extraJavaOptions=-XX:+PrintGCDetails -XX:+PrintGCTimeStamps" /home/jerry/racoon/nongfu.ngx.log.analyser/modules/LogProcess/out/libs/LogProcess-1.0-SNAPSHOT-all.jar /home/hadoop/properties/hdfs.properties com.sznongfu.cn.offline.statistic.UserTrajectoryStatistic
spark-submit --class com.sznongfu.cn.center.StartCenter --name sznongfuzhanggui --master spark://hadoop:7077 --driver-cores 5 --driver-memory 1G --total-executor-cores 8 --executor-memory 1G --executor-cores 4 --conf "spark.executor.extraJavaOptions=-XX:+PrintFlagsFinal -XX:+PrintReferenceGC -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintAdaptiveSizePolicy -XX:+UnlockDiagnosticVMOptions" /home/jerry/racoon/nongfu.ngx.log.analyser/modules/LogProcess/out/libs/LogProcess-1.0-SNAPSHOT-all.jar /home/hadoop/properties/hdfs.properties com.sznongfu.cn.offline.statistic.UserTrajectoryStatistic >> trace01.log 2>&1 &
parllel GC 1010 883 902 差不多 763 759 761 405 压测:1131 785 794 二次: 843 703 784
spark-submit --class com.sznongfu.cn.center.StartCenter --name sznongfuzhanggui --master spark://hadoop:7077 --driver-cores 5 --driver-memory 1G --total-executor-cores 4 --executor-memory 1G --conf "spark.executor.extraJavaOptions=-XX:+PrintGCDetails -XX:+PrintGCTimeStamps" /home/jerry/racoon/nongfu.ngx.log.analyser/modules/LogProcess/out/libs/LogProcess-1.0-SNAPSHOT-all.jar /home/hadoop/properties/hdfs.properties com.sznongfu.cn.offline.statistic.UserTrajectoryStatistic >> trace01Tune.log 2>&1 &
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+
21257 hadoop 20 0 4805760 1.193g 31676 S 98.7 15.5 5:00.10
G1 GC 所用时间: 925 939 946 差不多 831 831 1159 压测 810 810 821 二次: 752 783 805
测试环境启动
~/opt/spark-2.2.0-bin-hadoop2.7/bin/spark-submit --class com.sznongfu.cn.center.StartCenter --name sznongfuzhanggui --master spark://IZ2zehlhwjhnr93dioubfzZ:7077 --driver-cores 1 --driver-memory 1G --total-executor-cores 4 --executor-cores 4 --executor-memory 1G --conf "spark.executor.extraJavaOptions=-XX:+UseG1GC -XX:+PrintGCDetails -XX:+PrintGCTimeStamps" ./libs/LogProcess-1.0-SNAPSHOT-all.jar ./conf/la.benchmark.properties com.sznongfu.cn.offline.statistic.UserTrajectoryStatistic >> ./logs/offline.log 2>&1 &
测试环境启动
~/opt/spark-2.2.0-bin-hadoop2.7/bin/spark-submit --class com.sznongfu.cn.center.StartCenter --name sznongfuzhanggui --master spark://iZ2zehlhwjhnr93dioubfzZ:7077 --driver-cores 1 --driver-memory 1G --total-executor-cores 4 --executor-cores 4 --executor-memory 1G --conf "spark.executor.extraJavaOptions=-XX:+UseG1GC -XX:+PrintGCDetails -XX:+PrintGCTimeStamps" /tmp/offlinejar/LogProcess-1.0-SNAPSHOT-all.jar ./conf/la.benchmark.properties com.sznongfu.cn.offline.statistic.UserTrajectoryStatistic
/tmp/offlinejar/LogProcess-1.0-SNAPSHOT-all.jar
spark-submit --class com.sznongfu.cn.center.StartCenter --name sznongfuzhanggui --master spark://hadoop:7077 --driver-cores 2 --driver-memory 1G --total-executor-cores 2 --executor-memory 1G --conf "spark.executor.extraJavaOptions=-XX:+UseG1GC -XX:+PrintFlagsFinal -XX:+PrintReferenceGC -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintAdaptiveSizePolicy -XX:+UnlockDiagnosticVMOptions -XX:+G1SummarizeConcMark -XX:InitiatingHeapOccupancyPercent=35 -XX:ConcGCThread=20" /home/jerry/racoon/nongfu.ngx.log.analyser/modules/LogProcess/out/libs/LogProcess-1.0-SNAPSHOT-all.jar /home/hadoop/properties/hdfs.properties com.sznongfu.cn.offline.statistic.UserTrajectoryStatistic
--conf "spark.executor.extraJavaOptions=-XX:+UseG1GC -XX:MaxGCPauseMillis=20 -XX:InitiatingHeapOccupancyPercent=35 -XX:+DisableExplicitGC -XX:ConcGCThread=20 -XX:MaxGCPauseMillis=20"
--driver-java-options "-XX:+UseG1GC -XX:MaxGCPauseMillis=20 -XX:InitiatingHeapOccupancyPercent=35 -XX:+DisableExplicitGC -XX:ConcGCThread=20 -XX:MaxGCPauseMillis=20"
spark-submit --class com.sznongfu.cn.center.StartCenter --name sznongfuzhanggui --master spark://hadoop:7077 --driver-cores 1 --driver-memory 1G --total-executor-cores 4 --executor-cores 4 --executor-memory 1G --conf "spark.executor.extraJavaOptions=-XX:+UseG1GC -XX:+PrintGCDetails -XX:+PrintGCTimeStamps" /home/jerry/racoon/nongfu.ngx.log.analyser/modules/LogProcess/out/libs/LogProcess-1.0-SNAPSHOT-all.jar /home/hadoop/properties/hdfs.properties com.sznongfu.cn.offline.statistic.UserTrajectoryStatistic
最优实践:
spark-submit --class com.sznongfu.cn.center.StartCenter --name sznongfuzhanggui --master spark://hadoop:7077 --executor-memory 1G --driver-cores 2 --driver-memory 1G --total-executor-cores 2 --conf "spark.executor.extraJavaOptions=-XX:+UseG1GC -XX:+PrintGCDetails -XX:+PrintGCTimeStamps" /home/jerry/racoon/nongfu.ngx.log.analyser/modules/LogProcess/out/libs/LogProcess-1.0-SNAPSHOT-all.jar /home/hadoop/properties/hdfs.properties com.sznongfu.cn.online.statistic.NginxLogStatistic
---可行
spark-submit --class com.sznongfu.cn.center.StartCenter --name sznongfuzhanggui --master spark://hadoop:7077 --executor-memory 1G --driver-cores 2 --driver-memory 1G --total-executor-cores 2 /home/jerry/racoon/nongfu.ngx.log.analyser/modules/LogProcess/out/libs/LogProcess-1.0-SNAPSHOT-all.jar /home/hadoop/properties/hdfs.properties com.sznongfu.cn.offline.statistic.UserTrajectoryStatistic
关闭应用 ctrl+c
/usr/spark2.2/sbin/stop-all.sh
1.kafka:
kafka-server-stop.sh /usr/kafka/config/server.properties
kafka-server-stop.sh /usr/kafka/config/server1.properties
2.reids:
redis-cli shutdown
3.HDFS:
stop-dfs.sh
4.Zookeeper:
zkServer.sh stop
5.mongo:
ctrl+c
zkServer.sh stop
exec $SPARK_HOME/bin/spark-submit \
--class com.bigeyedata.mort.Main \
--driver-class-path $MORT_HOME/libs/*.jar \
--master yarn-client \
--deploy-mode cluster \
--jars /appcom/mort/thirdparty_jars/customer_provided.jar \
--queue queue_0100_01 \
$MORT_HOME/mort.jar > $MORT_HOME/mort.log 2>&1
#! /bin/bash
while true
do
echo "a"
sleep 10s
done
~
回退版本:
git reset --hard 5997af6ace0769da74d5b7659f1e3c62e1c9afa0
2018-05-21 Browser 5aa91001220f130432ba6279
2018-05-21 App 9QZrh9wogSSdIoCS7gdQjMHA
-Xmx1G -Xms1G -server -XX:+UseG1GC -XX:MaxGCPauseMillis=20 -XX:InitiatingHeapOccupancyPercent=35 -XX:+DisableExplicitGC -Djava.awt.headless=true
-Xloggc:/home/sznongfu/opt/kafka_2.11-0.10.0.0/bin/../logs