CCF存储系统测试工具安装脚本解读

setup.sh:

#!/bin/bash
# Install SES with runtime environment and core dependencies
# See usage below

# $0是该脚本的路径，readlink命令结合-f参数可以解析符号链接并获得其绝对路径
SCRIPT=$(readlink -f "$0")
# 提取脚本所在目录
SCRIPT_DIR=$(dirname "$SCRIPT")
logPath="$SCRIPT_DIR/setup.log"
_VERSION="1.9.3"
# Dependent tools ##############################################################
SSH_SCRIPT="sshauth.sh"
SSH_PRE_SCRIPT="sshpre.exp"
SSH_COPY_SCRIPT="sshcopy.exp"
OPENSSL_PACKAGE="openssl-1.1.1w.tar.gz"
PY_PACKAGE="Python-3.8.12.tgz"
JDK_PACKAGE="jdk-8u202-linux-i586.tar.gz"
# whl是python的包格式，称为Wheel，是一个用于分发和安装python软件包的标准格式
SES_PACKAGE="storage_evaluation_system-*.whl"
VDBENCH_PACKAGE="io_v.zip"
# 定义Vdbench测试模板文件
VDBENCH_TEST_TPLT="file_io_test_template.txt"
# 禁用防火墙脚本
DFW_SCRIPT="disable_firewall.sh"
AARCH_SO_FILE="aarch64.so" # For ARM run vdbench

REQUIRED_PY_VERSION=3.7
# python虚拟环境的默认路径
VENV_HOME="$HOME/.virtualenvs"
# 定义ssh授权文件
SLAVE_AUTH_CONF=".ssh_auth"
VDBENCH_TEST_CONF="vdb_test.txt"
VDBENCH_DIR=/opt/ses/vdb
# python安装缓存的路径
PY_CACHE="/tmp/installed_py"
# Package manager command
pm=
localhostIP=
# slaveIP will be set only when installing on slave host
slaveIP=
slaveConfContent=
slaveNum=
# 定义一个数组变量，初始化为空数组
errorSlaves=()

# Parameters ##############################################################
# Requires GNU
# getopt是一个用于解析命令行选项的标准工具。它支持短选项和长选项的解析
# -o: 定义短选项。多个短选项可以合并，例如：-abc 等价于 -a -b -c
# --long: 定义长选项。后跟长选项参数列表。sshAuthConf后面的冒号表示该选项需要参数
# --: 用于结束选项解析。后面的是脚本传入的所有参数
# $@: 脚本传入的所有参数
TEMP=$(getopt -o hs:i: --long help,sshAuthConf:,installDir:,pythonExecutable:,pipIndexUrl:,pipTrustedHost:,pipProxy:,proxy:,slave:,skipSshAuth,onlySlave,skipPmLoading -- "$@")

# 如果刚才执行的命令getopt执行失败，则退出该脚本，终止安装
if [ $? != 0 ] ; then echo "Terminating..." >&2 ; exit 1 ; fi
eval set -- "$TEMP"

# Save args so we can reuse it when installing on slave host
# "$*" 表示将当前脚本接收到的所有位置参数合并为一个单一的字符串。所有参数将用空格分隔。这里的 argStr 用来存储这些参数的拷贝。
argStr="$*"
# 从 argStr 中去除末尾的 -- 字符串
argStr=${argStr%"--"}

user=
password=
# 安装目录和安装脚本setup.sh在一个路径下
installDir=$(dirname "$SCRIPT")
sshAuthConf="ssh_auth_conf.txt" # Config file for defining all client-hosts
pythonExecutable=""
pipIndexUrl="https://pypi.tuna.tsinghua.edu.cn/simple"
pipTrustedHost="pypi.tuna.tsinghua.edu.cn"
pipTimeout=300
pipProxy=
proxy=
skipSshAuth=false
onlySlave=false
skipPmLoading=false

__sshAuthConfHelp="
SSH_AUTH_CONF format:
<LOCALHOST_IP>,user=<USER>,password=<PASSWORD>
<SLAVE_IP>[,port=<PORT>][,password=<PASSWORD>]
<SLAVE_IP>
* All hosts must have the same username as LOCALHOST
* SSH port/password can be specified for each SLAVE_IP.
  Otherwise use port:22, password:localhost password by default

Example:
192.168.1.111,user=root,password=passW0Rd
192.168.1.112,port=2222
192.168.1.113,password=anotherPassW0Rd
"
__usage="Install SES with runtime environment and core dependencies
Usage:
  ./setup.sh [options]

General Options:
  -h, --help                  Show help.
  -s, --sshAuthConf <SSH_AUTH_CONF>
                              File path to ssh auth conf (default: $sshAuthConf under the directory containing
                              setup.sh).
                              If <SSH_AUTH_CONF> is a filename, consider it to be located within the directory
                              containing setup.sh. Otherwise treat it as an absolute path. Check file form as below.
  -i, --installDir <INSTALL_DIR>
                              Install all dependencies to <INSTALL_DIR> (default: the directory containing setup.sh).
                              No special character is allowed except - or _.

Python/pip options:
  --pythonExecutable <PATH>   Path to python executable. If not provided, program will use python3/python, or the one
                              that was installed from previous installation. e.g. /home/my/python/bin/python3.
  --pipIndexUrl <URL>         Base URL of Python Package Index. If you wish to use pip.conf from localhost, provide an
                              empty value (default: $pipIndexUrl)
  --pipTrustedHost <HOSTNAME> Mark this host as trusted, even though it does not have valid or any HTTPS. If you wish
                              to use pip.conf from localhost, provide an empty value.
                              (default: $pipTrustedHost)
  --pipTimeout <SEC>          Set the socket timeout (default: $pipTimeout seconds).
  --pipProxy <PIP_PROXY>      Specify proxy for pip install in the form [user:passwd@]proxy.server:port. This overwrites
                              --proxy if provided, otherwise use --proxy (if --proxy provided).

Other options:
  --proxy <PROXY>             Specify proxy for installation in the form http://[user:passwd@]proxy.server:port.
                              Effective on master host and all slave hosts.
  --skipSshAuth               Skip ssh authorization. May only be used when the authorization have been done for all
                              slave hosts defined in <SSH_AUTH_CONF>
  --onlySlave                 Run installation only for slave hosts. Including ssh authorization (if --skipSshAuth not
                              provided), installing java and file-io tool.
  --skipPmLoading             Skip updating package-management repos. May only be used when the repository is up-to-date
                              and has been loaded successfully. Effective on master host and all slave hosts.

$__sshAuthConfHelp"

while true; do
  case "$1" in
    # 之所以解析到-h选项后只移动一位参数，是因为该选项后面没有值
    -h | --help )         echo "$__usage"; exit 0; shift ;;
    # 而解析到-s选项后需要移动两位参数，是因为-s后面还有一个指定给该选项的参数值，参数选项本身和它的值需要一起移动
    -s | --sshAuthConf )  sshAuthConf="$2"; shift 2 ;;
    -i | --installDir )   installDir="$2"; shift 2 ;;
    --pythonExecutable )  pythonExecutable="$2"; shift 2 ;;
    --pipIndexUrl )       pipIndexUrl="$2"; shift 2 ;;
    --pipTrustedHost )    pipTrustedHost="$2"; shift 2 ;;
    --pipTimeout )        pipTimeout="$2"; shift 2 ;;
    --pipProxy )          pipProxy="$2"; shift 2 ;;
    --slave )             slaveIP="$2"; shift ;;
    # 以下三个参数是没有允许接收指定值的（当初在getopt命令中，这三个参数后面没有冒号，表示该选项不需要参数值）
    --skipSshAuth )       skipSshAuth=true; shift ;;
    --onlySlave )         onlySlave=true; shift ;;
    --skipPmLoading )     skipPmLoading=true; shift ;;
    --proxy)
      proxy="$2"
      export http_proxy="$2"
      export https_proxy="$2"
      shift 2 ;;
    -- ) shift; break ;;
    * ) break ;;
  esac
done


# 对于这行代码，我看着不是很顺眼，我觉得逻辑连接符两边的命令如果用括号括起来的话逻辑会更加清晰，一目了然
# 于是我自作主张把它改成了这样： (test -e "$logPath") && (rm -f "$logPath")
# 可千万不要这样！一条命令用括号括起来会让它去子shell执行
# 如果日志文件已经存在，则删除它
test -e "$logPath" && rm -f "$logPath"

# 检查安装目录是否包含空格
if [[ $installDir = *" "* ]]; then
  echo "Invalid installDir='$installDir': Containing one or more spaces"
  exit 1
fi

# 如果系统不存在installDir这个目录，则创建该目录
test -d $installDir || mkdir -p $installDir

# 检测当前运行环境是否为 ARM 架构，具体来说是检查是否为基于 aarch64 的架构
isARM=false
# 以下几种写法都可以：
# if [[ $(uname -m) = *"aarch64"* ]]; then
# if [[ $(uname -m) == *"aarch64"* ]]; then
# if [[ `uname -m` == *"aarch64"* ]]; then
if [[ `uname -m` = *"aarch64"* ]]; then
  isARM=true
fi


# Log message
# @param 1: message
# @param 2: (Optional)
#   -e: log error
#   -s: log success
#   -w: log warning
log(){
    str=$1  # 将传递给该函数的第一个参数（日志信息）赋值给变量str
    prefix="[$(date +"%Y-%m-%d %H:%M:%S,%3N")]"
    if [[ ! "$slaveIP" == "" ]]; then
      str="[$slaveIP] $str"
    fi

    if [[ $2 == "-e" ]] ; then
      cmsg="$prefix \e[31m[ERROR]\033[0m $str"
      msg="$prefix [ERROR] $str"
    elif [[ $2 == "-s" ]] ; then
      cmsg="$prefix \e[1;32m[SUCCESS]\033[0m $str"
      msg="$prefix [SUCCESS] $str"
    elif [[ $2 == "-w" ]] ; then
      cmsg="$prefix \e[93m[WARNING]\033[0m $str"
      msg="$prefix [WARNING] $str"
    else
      msg="$prefix [INFO] $str"
      cmsg=$msg
    fi;

    echo -e "$cmsg"
    echo "$msg" >> $logPath
}
# logError "message": Log error and exit with code 1
# logError "message" -nq : Log error and keep running
logError(){
  log "$1" -e
  if [[ "$2" != "-nq" ]]; then
    log "Installation failed" -e
    exit 1
  fi
}
logSuccess(){
  log "$1" -s
}
logWarning(){
  log "$1" -w
}
# 该函数将字符串str添加到文件中，如果文件中已经存在该字符串，则不再添加
addToSource(){
  str="$1"
  files=("/etc/profile" "$HOME/.bashrc" "$HOME/.bash_profile" )
  # files是一个数组，${files[@]}表示数组所有元素
  for f in "${files[@]}";
  do
    # add only when file not contains str
    # 使用grep命令，查看文件$f是否包含字符串$str，-q参数表示静默模式，即不显示任何信息，仅返回退出状态码
    if ! grep -q "$str" "$f"; then
      echo "$str" >> $f
    fi
    # 重新加载配置文件，使修改立即生效
    # 特别是在 .bashrc 和 .bash_profile 等文件中，这个操作是为了确保新的环境变量或配置会被当前 Shell 环境识别。
    source $f
  done
}
# 根据系统的包管理工具（如 apt-get 或 yum）安装指定的软件包
# Install package by name
# If there's only one parameter, consider it suits both package manager:
#   - apt-get install <PACKAGE>
#   - yum install <PACKAGE>
# If there are multiple parameters, function works as follows:
#   - apt-get install <PARAMETER_1>
#   - yum install <PARAMETER_2>
installPackage(){
  # 在函数最开头定义一个变量，初始为空
  # 定义在函数中的变量，它的作用域只局限于函数内部，而不会影响到全局变量
  # pm是定义在函数外部的，是一个全局变量，它的作用域是全局的
  packName=
  # “#” 是shell一个特殊的变量，当在一个函数中查看该变量时，它表示传递给该函数的参数个数
  if [ $# -eq 1 ];then
    packName="$1"
  elif [[ $pm == "apt-get" ]]; then
    packName="$1"
  elif [[ $pm == "yum" ]]; then
    packName="$2"
  else
    logError "Unsupported package management tool: $pm"
  fi

  log "Installing [$packName] using $pm"
  $pm -y -q install "$packName" > /dev/null

  if [ ! $? -eq 0 ];then
    logWarning "Installing $packName failed" -nq
  fi
}

# Install pacakge
############################# Main steps ###############################

# Check if all scripts and tools exist
_checkTool(){
  if [ ! -e "$1" ]; then
    logError "Dependent setup tool not found: $1"
  fi
}
checkTools(){
  cd $SCRIPT_DIR
  if ! $onlySlave; then
    # _checkTool $OPENSSL_PACKAGE
    # If provided python meets the requirement, python install package is unnecessary
    if [[ $pyExecutable == "" ]]; then
      _checkTool $PY_PACKAGE
    fi

  fi

  _checkTool $SSH_SCRIPT
  _checkTool $SSH_PRE_SCRIPT
  _checkTool $SSH_COPY_SCRIPT
  _checkTool $JDK_PACKAGE
  _checkTool $VDBENCH_PACKAGE
  _checkTool $VDBENCH_TEST_TPLT
  # ssh connection is called AFTER checkTools, so we need to check in advance
  _checkTool $AARCH_SO_FILE
}

# Check and update package management tool. apt-get/yum
# 该函数用于检查并更新包管理工具，如apt-get或yum。
checkPackageManagementTool(){
  log "Check package management tool"
  # 定义一个局部变量，用于存储更新命令，初始为空
  updateCmd=
  # 重定向符号“&>”将标准输出和标准错误重定向到同一个文件，此处它们都输入到一个特殊文件/dev/null
  # 使用command命令检查系统中是否存在apt-get或yum命令，如果存在，则将pm变量设置为apt-get或yum
  if command -v apt-get &> /dev/null; then
    pm="apt-get"
    if ! $skipPmLoading; then
      updateCmd="apt-get update"
    fi
  fi

  # 在我的系统上存在apt-get包管理器，但是没有yum包管理器，所以下面这段代码不会执行
  if command -v yum &> /dev/null; then
    pm="yum"
    if ! $skipPmLoading; then
      yum clean all &> /dev/null
      updateCmd="yum makecache"
    fi
  fi

  # skipPmLoading：默认为false，用于指示是否跳过更新包管理工具的操作
  if $skipPmLoading ; then
    log "Skip updating package-management repository"
  elif ! eval "$updateCmd" &> /dev/null;then
    logError "Updating $pm repo failed! Please make sure updating command works: $updateCmd"
  fi

  # Install common packages for all hosts
  # 安装常见软件包
  if ! command -V bc >/dev/null 2>&1; then
	  installPackage bc
  fi
  if ! command -V dmidecode >/dev/null 2>&1; then
	  installPackage dmidecode
  fi
  if ! command -V sysstat >/dev/null 2>&1; then
	  installPackage sysstat
  fi

  if [[  $pm == "" ]]; then
    logError "Package management tool not found: Requires apt-get or yum"
  else
    log "Using $pm for package management"
  fi
}


# Install openssl
opensslVer=
isOpensslValid=false
checkOpenssl(){
  # return code might not be 0
  # 执行命令openssl version 2>/dev/null，并捕获命令的标准输出
  opensslVer=$(openssl version 2>/dev/null)
  # 对变量opensslVer进行正则匹配，匹配到的结果存入BASH_REMATCH数组
  if [[ "$opensslVer" =~ ([0-9]+\.[0-9]+\.[0-9]+) ]]; then
    verNo=${BASH_REMATCH[1]}
    # ${verNo:0:1}是一种参数扩展，用于提取verNo的子字符串，从第0个字符开始，取1个字符
    if [[ "${verNo:0:1}" -gt 1 || "$verNo" == "1.1.1"  ]]; then
      isOpensslValid=true
    fi
  fi
}

installOpenssl(){
  log "Installing dependency: openssl"
  cd $SCRIPT_DIR
  checkOpenssl
  if ! $isOpensslValid; then
    # 安装OpenSSL所需要的gcc编译器和zlib开发库
    installPackage gcc
    installPackage "zlib1g-dev" "zlib-devel"

    # 检查并安装perl
    if ! command -V perl >/dev/null 2>&1; then
      installPackage "perl" "perl-core"
    else
      log "Requirement already satisfied: perl"
    fi

    log "Making openssl, this might take a while"
    tar -xzf $OPENSSL_PACKAGE -C $installDir
    cd "$installDir/$(basename $OPENSSL_PACKAGE .tar.gz)"
    ./config --prefix=/usr --openssldir=/etc/ssl --libdir=lib zlib-dynamic > /dev/null
    make > /dev/null
    make test > /dev/null
    make install > /dev/null
    # Add environment variable
    envStr="export LD_LIBRARY_PATH=/usr/local/lib:/usr/local/lib64:/usr/lib:/usr/lib64"
    echo $envStr >> /etc/profile.d/openssl.sh
    source /etc/profile.d/openssl.sh
    addToSource $envStr
  else
    log "Requirement already satisfied: $opensslVer"
    return
  fi

  checkOpenssl
  if ! $isOpensslValid; then
    logError "openssl installation failed"
  else
    logSuccess "openssl installed successfully"
  fi
}

# Configure ssh auth for all clients
configSSHAuth(){
  log "Configuring ssh authorized keys for all hosts"
  if command -v expect &> /dev/null; then
    log "Requirement already satisfied: expect"
  else
    installPackage expect
  fi

  # Install dos2unix to ensure all configs are properly formatted
  if ! command -v dos2unix &> /dev/null; then
    installPackage dos2unix
  fi
  dos2unix "$sshAuthConf" 2> /dev/null

  # 执行另外一个脚本文件，将配置文件等作为该脚本的输入参数
  $SCRIPT_DIR/$SSH_SCRIPT "$SLAVE_AUTH_CONF" "$localhostIP" "$user" "$password" "$logPath"
  if [ ! $? -eq 0 ];then
    logError "ssh authorized keys configuration failed"
  else
    logSuccess "ssh authorized keys configured successfully"
  fi
}

# Install Java
JAVA_HOME=''
jdkDir="$installDir/jdk"
checkAndInstallJava(){
  log "Installing dependency: java"

  javaVersion=$(java -version 2>&1)
  if [ $? -eq 0 ];then
    log "Requirement already satisfied: $javaVersion"
    return
  fi

  cd $installDir
  if [ "$pm" == yum ]; then
    if ! rpm -q glibc.i686 &> /dev/null; then
      installPackage "glibc.i686"
    fi
  elif [ "$pm" == apt-get ]; then
    dpkg -s libc6-i386 >/dev/null 2>&1
    if [ ! $? -eq 0 ]; then
      installPackage "libc6-i386"
    fi
  else
    logError "Unsupported package management tools: $pm"
  fi

  mkdir -p $jdkDir
  tar -xzf $JDK_PACKAGE -C $jdkDir --strip-components 1 &
  while pgrep -f 'tar -xzf' > /dev/null; do
    sleep 1
  done
  JAVA_HOME=$jdkDir

  # Add environment variables
  addToSource "export JAVA_HOME=$JAVA_HOME"
  addToSource "export PATH=\$PATH:\$JAVA_HOME/bin"
  addToSource "export CLASSPATH=.:\$JAVA_HOME/lib/dt.jar:\$JAVA_HOME/lib/tools.jar"

  # check
  javaVersion=$(java -version 2>&1)
  result=$(echo $javaVersion | grep "1.8.")
  if [[ "$result" != "" ]]; then
    logSuccess "Java installed successfully to: $JAVA_HOME"
  else
    logError "Java installation failed"
  fi
}

# Install Vdbench
InstallVdbench(){
  log "Installing dependency: file-io tool"
  cd $SCRIPT_DIR

  if command -v unzip >/dev/null 2>&1; then
    log "Requirement already satisfied: unzip"
  else
    installPackage unzip
  fi

  mkdir -p $VDBENCH_DIR
  unzip -o -q -d $VDBENCH_DIR $VDBENCH_PACKAGE
  if [ ! $? -eq 0 ]; then
    logError "Unzip file-io tool package failed"
  fi

  # make vdbench50403 works
  test -e /bin/csh && mv -f /bin/csh /bin/csh.bak
  cp /bin/bash /bin/csh
  logWarning "Replaced /bin/csh with /bin/bash"

  chmod 777 -R $VDBENCH_DIR
  addToSource "export VDB_DIR=$VDBENCH_DIR"

  # Modify java location for all hosts
  javaExec=$(which java)
  sed -i -e "0,/java=java/ s|java=java|java=$javaExec|" $VDBENCH_DIR/vdbench

  # Replace .so file for ARM os
  if $isARM; then
     cp ./$AARCH_SO_FILE $VDBENCH_DIR/linux/
     mv $VDBENCH_DIR/linux/linux64.so $VDBENCH_DIR/linux/linux64.so.bak
     cp ./$AARCH_SO_FILE $VDBENCH_DIR/linux/linux64.so
     log "ARM: $AARCH_SO_FILE file copied"
  fi
}

_loopCurrentHost=
_loopCurrentPort=
# Invoke this in a for-loop parsing each line of slave info
# _loopCurrentHost and _loopCurrentPort will be assigned real-time
_loopUpdateSlaveInfo(){
  line=$1
  _loopCurrentPort="22"
  hostInfo=(${line//,/ })
  for entry in "${hostInfo[@]}"
  do
     # Parse ip,port
     if [[ $entry != *"="* ]]; then
        _loopCurrentHost=$entry
     else
        keyVal=(${entry//=/ })
        key=${keyVal[0]}
        value=${keyVal[1]}
        if [[ "$key" == "port" ]]; then
          _loopCurrentPort=$value
          break
        fi
     fi
  done
}

_stopFireWallOnSalveHost(){
  index=1
  # Reuse slaves assigned from installOnSlaves
  for line in `echo $slaveConfContent`
  do
    _loopUpdateSlaveInfo "$line"
    host=$_loopCurrentHost
    port=$_loopCurrentPort
    log "Disable firewall on \e[34m$host\033[0m ($index/$slaveNum)"
    scp -P $port -q "$DFW_SCRIPT" $host:$SCRIPT_DIR
    ssh -P $port -q $host "cd $SCRIPT_DIR; bash $DFW_SCRIPT >/dev/null"  | tee -a $logPath
    if [ ! $? -eq 0 ]; then
      logWarning "Disabling firewall for $host may have failed"
    fi

    index=$((index+1))
  done
}

vdbTestPass=false
_runVdbenchCmd(){
  cmd=$1
  echo "vdbench command: $cmd" >> $logPath
  eval $cmd >/dev/null 2>&1 &

  vdbenchPid=$!
  log "Running File-io tool test, this will take about 1 minute"
  while kill -0 $vdbenchPid 2>/dev/null; do
    sleep 10
  done

  if wait $vdbenchPid; then
    vdbTestPass=true
  fi
}

vdbTestAnchor="$installDir/vdb_test_anchor"
vdbenchTestFailed=false
# Generate vdbench param file and run vdbench test (Works for single/multi hosts)
runVdbenchTest(){
  hdDef=$1
  fsdDef=$2
  fwdDef=$3

  cd $SCRIPT_DIR

  dos2unix $VDBENCH_TEST_TPLT 2> /dev/null
  rm -f $VDBENCH_TEST_CONF && cp $VDBENCH_TEST_TPLT $VDBENCH_TEST_CONF

  # Replace def
  perl -i -p0e "s|{hd_def}|$hdDef|igs" $VDBENCH_TEST_CONF
  perl -i -p0e "s|{fsd_def}|$fsdDef|igs" $VDBENCH_TEST_CONF
  perl -i -p0e "s|{fwd_def}|$fwdDef|igs" $VDBENCH_TEST_CONF
  # Run test
  anchor="$vdbTestAnchor/host0"
  outputDir="$SCRIPT_DIR/file_io_output"
  cmd="$VDBENCH_DIR/vdbench -f $VDBENCH_TEST_CONF -o $outputDir vdbenchDir=$VDBENCH_DIR user=$user anchor=$anchor"
  _runVdbenchCmd "$cmd"

  # Check log
  if ! $vdbTestPass && grep -q 'Host unreachable' $outputDir/logfile.html; then
    logWarning "File-io tool requires firewall to be disabled on all slave hosts"
    read -p "Continue? This will disable firewall, enter y to confirm (y/n):" isDisableFw
    if [[ "${isDisableFw,,}" == "y" ]]; then
      _stopFireWallOnSalveHost
      log "Rerun file-io tool test"
       _runVdbenchCmd "$cmd"
    fi
  fi

  if $vdbTestPass; then
    logSuccess "File-io tool installed successfully"
  else
    logWarning "File-io tool test failed, please check your ssh connection and rerun the installation later"
    vdbenchTestFailed=true
  fi
}

# Install Python
# "libdb-dev" "db4-devel"
PY_APT_REQ_LIB=("libbz2-dev" "libssl-dev" "libncurses5-dev" "libsqlite3-dev" "libreadline-dev" "tk-dev" "libgdbm-dev" "libpcap-dev" "liblzma-dev" "libffi-dev")
PY_YUM_REQ_LIB=("bzip2-devel" "openssl-devel" "ncurses-devel" "sqlite-devel" "readline-devel" "tk-devel" "gdbm-devel" "libpcap-devel" "xz-devel" "libffi-devel")
pythonDir=
pyExecutable=
pyVer=
_pipInstall(){
  py=$1
  packName=$2
  args="$3"

  pipInstallCmd="$py -m pip install $packName"
  test "$pipIndexUrl" && pipInstallCmd="$pipInstallCmd -i $pipIndexUrl"
  test "$pipTrustedHost" && pipInstallCmd="$pipInstallCmd --trusted-host $pipTrustedHost"
  test "$pipTimeout" && pipInstallCmd="$pipInstallCmd --default-time $pipTimeout"
  test "$pipProxy" && pipInstallCmd="$pipInstallCmd --proxy $pipProxy"
  pipInstallCmd="$pipInstallCmd --no-warn-script-location --disable-pip-version-check"

  test "$args" && pipInstallCmd="$pipInstallCmd $args"

  eval "$pipInstallCmd"
}

# Check if provided python meets the requirement
# If so, update variable: pyExecutable
_checkPyVer(){
  pye=$1
  isCheckVer=false
  if [[ $pye != *"/"* ]]; then
    if command -v $pye >/dev/null 2>&1; then
      isCheckVer=true
    fi
  else
    isCheckVer=true
    log "Verifying custom Python: $pye"
  fi

  if $isCheckVer; then
    pyVer=$($pye --version 2>&1 | awk '{print $2}' | awk -F. '{print $1"."$2}')
    if [[ $pyVer =~ ^[0-9]+([.][0-9]+)?$ ]] ; then
      if (( $(echo "$pyVer >= $REQUIRED_PY_VERSION" | bc -l) )); then
        log "Requirement already satisfied: Python$pyVer(>=$REQUIRED_PY_VERSION)"
        pyExecutable="$pye"
      else
        log "Found unsatisfied Python version=$pyVer(<$REQUIRED_PY_VERSION)"
      fi
    else
      log "Unrecognized Python: $pye"
    fi
  fi
}


checkPython(){
  # Try user specified one
  # 如果用户指定了一个python路径，那么就检查一下系统是否确实存在这个路径
  if [[ "$pythonExecutable" != "" ]]; then
    _checkPyVer "$pythonExecutable"
  fi

  # Find python from previous installation
  if [[ $pyExecutable == "" ]] && [ -e $PY_CACHE ]; then
    _pyeCache=`head -1 $PY_CACHE`
    log "Found python executable from previous installation"
    _checkPyVer "$_pyeCache"
    if [[ "$pyExecutable" != "" ]]; then
      log "Use Python3 installed from previous installation"
      log ">>> pythonExecutable=$pyExecutable"
      return
    fi
  fi

  if [[ $pyExecutable == "" ]]; then
    _checkPyVer python3
  fi

  if [[ $pyExecutable == "" ]]; then
    _checkPyVer python
  fi

}

installPython(){
  # Already satisfied  
  if [[ $pyExecutable != "" ]]; then
    pipVer=$($pyExecutable -m pip --version 2>/dev/null)
    if [[ ! "$pipVer" =~ ([0-9]+\.[0-9]+.[0-9]+) ]]; then
      log "Installing dependency: $(basename $PY_PACKAGE .tgz)"
      installPackage "python3-pip"
    fi
    return
  fi
  
  log "Installing dependency: $(basename $PY_PACKAGE .tgz)"
  cd $SCRIPT_DIR

  # install gcc
  if [ "$pm" == yum ]; then
    yum groupinstall -y "Development Tools"
  elif [ "$pm" == apt-get ]; then
    installPackage "build-essential"
  else
    logError "Other package management tools are not supported: $pm"
  fi

  for ((i=0;i<${#PY_YUM_REQ_LIB[@]};i++))
  do
    yum_lib="${PY_YUM_REQ_LIB[$i]}"
    apt_lib="${PY_APT_REQ_LIB[$i]}"
    installPackage $apt_lib $yum_lib
  done

  log "Configure & making Python3, this might take a while"
  tar -xzf $PY_PACKAGE -C $installDir

  cd "$installDir/$(basename $PY_PACKAGE .tgz)"
  pythonDir="$installDir/python3"
  if ! ./configure --prefix=$pythonDir &> install.log; then
      logError "Failed to compile Python. Check the log file for details"
  fi

  make &>/dev/null && make install &>/dev/null

  pyExecutable="$pythonDir/bin/python3"
  pyVer=$($pyExecutable --version 2>&1 | awk '{print $2}' | awk -F. '{print $1"."$2}')
  if (( $(echo "$pyVer < $REQUIRED_PY_VERSION" | bc -l) )); then
    logError "Python installation failed"
  else
    logSuccess "Python installed successfully to: $pythonDir"
    # cache to file so we can reuse it if user want reinstall
    echo "$pyExecutable" > $PY_CACHE
  fi

}


# Install SES
installSes(){
  log "Installing dependency: python package: virtualenv"
  _pipInstall "$pyExecutable" virtualenv

  # Create python virtual environment
  mkdir -p $VENV_HOME
  venvDir=$VENV_HOME/ses
  $pyExecutable -m venv --clear $venvDir
  log "Python-virtualenv created: $venvDir"

  log "Installing SES, this might take a while"
  cd $installDir
  _pipInstall "$venvDir/bin/python" pip "--upgrade"
  _pipInstall "$venvDir/bin/python" $SES_PACKAGE "--find-links=."

  ln -sf "$venvDir/bin/ses" /usr/bin/ses
  ses -h
  if [ $? -eq 0 ];then
    sesVer=$(ses -v)
    logSuccess "SES ($sesVer) installed successfully"
  else
    logError "SES installation failed"
  fi
}


#
installOnSlaves(){
  if [[ ! "$slaveIP" == "" ]]; then
    logError "Function installOnSlaves should only run on master host"
  fi

  cd $SCRIPT_DIR
  jdkPackMd5=$(md5sum $JDK_PACKAGE 2>/dev/null)
  vdbPackMd5=$(md5sum $VDBENCH_PACKAGE 2>/dev/null)
  if [ ! -e $SLAVE_AUTH_CONF ]; then
    logError "Parsed ssh-auth-config file not found, please ensure validateSSHAuthConfig has been run successfully"
  fi

  slaveConfContent=`cat $SLAVE_AUTH_CONF`
  slaveNum=`awk 'NF' $SLAVE_AUTH_CONF | wc -l`
  vdbHDDef=""
  vdbFSDDef=""
  vdbFWDDef=""
  # Make sure index starts from 1, for 0 is master host
  index=1
  for line in `echo $slaveConfContent`
  do
    _loopUpdateSlaveInfo "$line"
    host=$_loopCurrentHost
    port=$_loopCurrentPort
    log "Installing dependencies on salve host \e[34m$host\033[0m ($index/$slaveNum)"
    ssh -p $port -q $host "mkdir -p $SCRIPT_DIR"
    scp -P $port -q "setup.sh" $host:$SCRIPT_DIR
    ssh -p $port -q $host "chmod 777 $SCRIPT_DIR/setup.sh"

    # Prepare jdk
    hJdkPackMd5=$(ssh -p $port $host "cd $SCRIPT_DIR; md5sum $JDK_PACKAGE 2>/dev/null")
    if [[ $jdkPackMd5 != "" && "$jdkPackMd5" == "$hJdkPackMd5" ]];then
      log "$JDK_PACKAGE already exists, skip scp upload"
    else
      log "Upload $JDK_PACKAGE"
      scp -P $port -q "$JDK_PACKAGE" $host:$SCRIPT_DIR
    fi

    # Prepare vdbench
    hVdbPackMd5=$(ssh -p $port $host "cd $SCRIPT_DIR; md5sum $VDBENCH_PACKAGE 2>/dev/null")
    if [[ $vdbPackMd5 != "" && "$vdbPackMd5" == "$hVdbPackMd5" ]];then
      log "$VDBENCH_PACKAGE already exists, skip scp upload"
    else
      log "Upload $VDBENCH_PACKAGE"
      scp -P $port -q "$VDBENCH_PACKAGE" $host:$SCRIPT_DIR
    fi

    # tee makes return code 0 by default
    set -o pipefail
    cmd="bash $SCRIPT_DIR/setup.sh"
    # somehow slave must be the last parameter
    if $skipPmLoading; then
      cmd="$cmd --skipPmLoading"
    fi
    if [[ $proxy != "" ]] ; then
      cmd="$cmd --proxy $proxy"
    fi
    cmd="$cmd --slave $host"

    echo "Run slave command: $cmd" >>  $logPath
    ssh -p $port -q $host $cmd | tee -a $logPath

    if [ ! $? -eq 0 ];then
      logError "Failed to setup salve host:$host" -nq
      errorSlaves+=("$host")
    fi
    set +o pipefail

    anchor="$vdbTestAnchor/host$index"

    vdbHDDef="$vdbHDDef\nhd=hd$index,system=$host"
    vdbFSDDef="$vdbFSDDef\nfsd=fsd$index,anchor=$anchor"
    vdbFWDDef="$vdbFWDDef\nfwd=fwd$index,hd=hd$index,fsd=fsd$index"

    index=$((index+1))
  done

  if [[ ${#errorSlaves[@]} -eq 0 ]]; then
    runVdbenchTest "$vdbHDDef" "$vdbFSDDef" "$vdbFWDDef"
  else
    logWarning "Found failed salve host, skip file-io tool test"
  fi
}

# Parse all ip to a file, and validate the definition of localhost
# 验证SSH认证配置文件，并提取localhost的信息
validateSSHAuthConfig(){
  cd $SCRIPT_DIR
  log "Validating ssh auth config file: $sshAuthConf"
  # 使用 grep 命令去掉以 # 开头的注释行，并将结果输出到一个临时文件 $SLAVE_AUTH_CONF 中
  grep -v '^\s*#' $sshAuthConf  > $SLAVE_AUTH_CONF
  # Get the first line then do the trim
  # 使用 head -1 $SLAVE_AUTH_CONF 获取临时文件的第一行，并将其存入变量 localhostInfo
  # 第一行会是master的信息
  localhostInfo=`head -1 $SLAVE_AUTH_CONF`
  # 使用 sed -i '1d' $SLAVE_AUTH_CONF 删除文件中的第一行
  sed -i '1d' $SLAVE_AUTH_CONF

  # 定义一个正则表达式 regex，用于匹配期望的格式。格式应为 IP地址,user=用户名,password=密码。
  regex='^([^[:space:]]+),user=([^[:space:]]+),password=(.*)$'
  if [[ $localhostInfo =~ $regex ]]; then
    localhostIP="${BASH_REMATCH[1]}";
    user="${BASH_REMATCH[2]}";
    password="${BASH_REMATCH[3]}";
    if [[ $localhostIP == "" ]] || [[ $user == "" ]] || [[ $password == "" ]] ; then
      logError "Invalid sshAuthConf file format" -nq
      echo "$__sshAuthConfHelp"
      exit 1
    fi
  else
    logError "Invalid sshAuthConf file format" -nq
    echo "$__sshAuthConfHelp"
    exit 1
  fi
}

# print summary
summary(){
  if [[ ! ${#errorSlaves[@]} -eq 0 ]]; then
    printf -v es '%s,' "${errorSlaves[@]}"
    logError "Installation for these salve hosts failed (most likely caused by java installation failure): ${es%,}" -nq
  fi

  if $vdbenchTestFailed; then
    logError  "File-io tool test failed, please check your ssh connection, disable firewall and rerun the installation later" -nq
  fi

}

main(){
  # Check parameters
  # absolute path or filename
  if [[  $sshAuthConf != *"/"* ]]; then
      sshAuthConf="$SCRIPT_DIR/$sshAuthConf"
  fi;

  test -e $sshAuthConf ||  logError "Config file for defining client-hosts not exist: $sshAuthConf"
  if [[ "$pythonExecutable" != "" ]]; then
    test -e "$pythonExecutable" || logError "--pythonExecutable: file not exist: $pythonExecutable"
  fi;

  log "Installing SES with runtime environment and core dependencies (v$_VERSION)"
  log "sshAuthConf=$sshAuthConf"
  log "installDir=$installDir"
  log "pythonExecutable=$pythonExecutable"
  log "pipIndexUrl=$pipIndexUrl"
  log "pipTrustedHost=$pipTrustedHost"
  log "pipProxy=$pipProxy"
  log "proxy=$http_proxy"
  log "skipSshAuth=$skipSshAuth"
  log "onlySlave=$onlySlave"
  log "skipPmLoading=$skipPmLoading"
  log "logPath=$logPath"

  # Main steps on master host
  if ! $onlySlave; then
    # Check python first, so python install pack may become unnecessary
    checkPython
  else
    logWarning "Install for only slave hosts"
  fi

  checkTools

  if ! $onlySlave; then
    checkPackageManagementTool
    installOpenssl
  fi

  validateSSHAuthConfig
  if ! $skipSshAuth; then
    configSSHAuth
  else
    log "Skip ssh authorization due to user input parameter"
  fi

  if ! $onlySlave; then
    checkAndInstallJava
    InstallVdbench
  fi

  installOnSlaves

  if ! $onlySlave; then
    installPython
    installSes
    summary
  fi

  log "Complete"
}

# Ensure history config take effect
# 确保历史配置生效
# source命令用于在当前shell环境中执行文件中的命令，加载配置文件
source /etc/profile # 系统级配置文件，通常用于所有用户的环境转准备
source ~/.bashrc  # 用户级配置文件，通常用于当前用户的环境准备
source ~/.bash_profile # 用户级配置文件，通常用于当前用户的环境准备

# 如果slaveIP为空，则为主节点，执行主流程
if [[ "$slaveIP" == "" ]]; then
  main
else
  checkPackageManagementTool
  checkAndInstallJava
  InstallVdbench
fi

sshauth.sh:

#!/bin/bash
# Add local ssh key to all slave hosts
# Config file format example:
# 1.1.1.2,port=2222
# 1.1.1.3

if [ $# -eq 0 ];then
    echo "Usage : ./sshauth.sh <CONF_FILE> <LOCALHOST> <USER> <PASSWORD> <LOG_PATH>"
    exit 1
fi

SCRIPT=$(readlink -f "$0")
SCRIPT_DIR=$(dirname "$SCRIPT")

cd $SCRIPT_DIR

ipConf=$1
localhostIP=$2
user=$3
pass=$4
logPath=$5

currentHost=

# 计算传入的配置文件 $ipConf 中非空行的数量
NUM=`awk 'NF' $ipConf | wc -l`
if [ $NUM -lt 1 ];then
    echo "No slave hosts, skip ssh authorization"
    exit 0
else
    echo "Found $NUM slave-host(s) to authorize. Localhost=$localhostIP, user=$user"
fi

function checkCode(){
    if [ ! $1 -eq 0 ];then
      echo "Failed to authorize host=$currentHost"
      exit 1
    fi
}

print(){
  printf "* %.60s" "$1 ..................................................................."
}

# 创建 SSH 密钥并将其分发到指定的远程主机上，以实现无密码登录

# 创建本地 SSH 密钥
echo "Creating local ssh-key"
test -e ~/.ssh/id_rsa && rm -f ~/.ssh/id_rsa ||  mkdir -p ~/.ssh
# 生成一个新的 RSA 密钥对，使用空密码
ssh-keygen -t rsa -N '' -f ~/.ssh/id_rsa  > /dev/null

index=1
slaves=`cat $ipConf`
for line in `echo $slaves`
do
    port="22"
    password=$pass
    # Info should be in form key=value expect ip
    # line中的字符串原本由多个逗号分隔，${line//,/ }将其改为用空格分隔
    # 一对括号是在创建一个数组，line用空格分隔后，将其变为数组
    hostInfo=(${line//,/ })
    for entry in "${hostInfo[@]}"
    do
       if [[ $entry != *"="* ]]; then
          host=$entry
       else
          # 将等号替换为空格，空格分隔开的几个字符串变为数组元素
          keyVal=(${entry//=/ })
          key=${keyVal[0]}
          value=${keyVal[1]}
          if [[ "$key" == "port" ]]; then
            port=$value
          elif [[ "$key" == "password" ]]; then
            password=$value
          else
            echo "\e[31m[ERROR]\033[0m Invalid slave ssh info: $key=$value"
            exit 1
          fi
       fi
    done

    currentHost=$host
    echo -e "Configuring host \e[34m$host:$port\033[0m ($index/$NUM)"

    print "Prepare ssh-key"
    expect sshpre.exp "$host" "$port" "$user" "$password" >> $logPath
    checkCode $?
    # 行奇怪代码的作用是控制输出文本的颜色和样式
    printf " \033[92mok\033[0m\n"

    #Copy local pub key to other
    print "Copy local id_rsa.pub -> remote"
    expect sshcopy.exp "$host" "$port" "$password" >> $logPath
    checkCode $?
    printf " \033[92mok\033[0m\n"

    #Copy remote pub key to local
    print "Copy remote id_rsa.pub -> local"
    ssh -p $port -q ${host} "cat ~/.ssh/id_rsa.pub " >> ~/.ssh/authorized_keys
    checkCode $?
    printf " \033[92mok\033[0m\n"

    #Copy known_hosts to remote
    print "Add local known_hosts -> remote"
    keyStr=`tail -1 ~/.ssh/known_hosts | awk {'print $2 " " $3'}`
    keyStr="$localhostIP $keyStr"
    ssh -p $port -q ${host} "echo $keyStr >> ~/.ssh/known_hosts"
    checkCode $?
    printf " \033[92mok\033[0m\n"

    print "Validate $host"
    ssh -p $port -q $host "date" > /dev/null
    checkCode $?
    printf " \033[92mok\033[0m\n"

    index=$((index+1))
done