需求：时序数据下载到本地（2）

#! /bin/bash
dayno=$1
exec > logs/download_${dayno}.log 2>&1
export PATH=/usr/local/share/hadoop/bin:$PATH
export SPARK_HOME=/usr/local/share/spark
export HADOOP_USER_NAME=dataintel   # 数据授权，是按应用组，dataintel 

# configurations
PYTHON_PATH=py37/py37/bin/python
ALG_HDFS=hdfs://alg-hdfs/user/${HADOOP_USER_NAME}

# pyspark code & package
CODE_PATH=$(dirname $(readlink -f "$0"))

# data hdfs & local
HDFS_PATH=hdfs://alg-hdfs/user/dataintel/PEEM00_data
DATA_PATH=$(dirname $(readlink -f "$0"))

${SPARK_HOME}/bin/spark-submit \
--queue root.default \
--name "${GRAPH_NAME}_${dayno}" \
--master yarn \
--deploy-mode cluster \
--driver-memory 20g \
--executor-memory 20g \
--num-executors 80 \
--executor-cores 4 \
--conf spark.port.maxRetries=1 \
--conf spark.yarn.maxAppAttempts=1 \
--conf spark.executor.memoryOverhead=4096 \
--conf spark.default.parallelism=512 \
--conf spark.yarn.appMasterEnv.DGLBACKEND=pytorch \
--conf spark.executorEnv.DGLBACKEND=pytorch \
--conf spark.driverEnv.DGLBACKEND=pytorch \
--conf spark.hadoop.validateOutputSpecs=false \
--conf spark.pyspark.python=$PYTHON_PATH \
--conf spark.pyspark.driver.python=$PYTHON_PATH \
--conf spark.yarn.appMasterEnv.PYSPARK_PYTHON=$PYTHON_PATH \
--conf spark.driver.maxResultSize=2g \
--conf spark.hadoop.fs.defaultFS=$ALG_HDFS \
--conf spark.sql.execution.arrow.pyspark.enabled=true \
--archives hdfs://alg-hdfs/user/dataintel/tools/py37.zip#py37 \
${CODE_PATH}/download.py \
--dayno=${dayno} \
--export_hdfs_path ${HDFS_PATH}

hadoop fs -get ${HDFS_PATH}/data_${dayno}  ./data
awk 'FNR>1 || NR==1' data/data_${dayno}/part* > data/data_${dayno}.csv
echo data/data_${dayno} | xargs rm -r