目录
shell中的break continue exit return
shell中的break continue exit return
Linux脚本中的break continue exit return
break: 结束并退出循环
continue: 在循环中不执行continue下面的代码,转而进入下一轮循环
exit: 退出脚本,常带一个整数给系统,如 exit 0
return: 在函数中将数据返回,或返回一个结果给调用函数的脚本
scp
scp -r /home/hadoop/apache-phoenix-4.8.1-HBase-1.1-bin/ hadoop@x.x.x.x:/home/hadoop/apache-phoenix-4.8.1-HBase-1.1-bin/
循环语句
案例一
#!/bin/bash
# 显示一系列数字
count=1
while [ $count -le 6 ]; do
echo $count
count=$((count + 1))
done
echo "finished"
案例二
指定次数的for循环
for ((i=1;i<=30;i++));
do
source_date=`date -d "-$i day $statistics_date" +%F`
add_retention_step $cluster_id $source_date $statistics_date
if [ "$source_date" = "2017-08-01" ];then
break
fi
done
案例三
path_format(){
path=$1
arr=(/c /d /e /f /g /h /i)
input=${path:0:2}
for format in ${arr[@]}
do
echo ${format}
if [ "${input}" = "${format}" ];then
return 2
fi
done
return 3
}
案例四
#!/bin/bash
cd /root/crontab/data/
./SaveSMSLog.groovy
paths="sms_log/ smsCode_log/"
yesterday=`date -d "yesterday" "+%Y-%m-%d"`
DATE_PATH=`date -d "yesterday" "+%Y/%m/%d/"`
s3_path="s3://logs-datacenter/mongodata/"
for path in ${paths};
do
if [ -f "${path}$yesterday" ]; then
aws s3 cp ${path}$yesterday ${s3_path}${path}${DATE_PATH}
if [ $? -eq 0 ];then
echo "info##$(date +"%Y-%m-%d %H:%M:%S")##${path}下的文件已经成功上传到s3"
rm -rf ${path}$yesterday
else
echo "error##$(date +"%Y-%m-%d %H:%M:%S")##${path}下的文件上传到s3失败"
#发送邮件通知
fi
fi
done
案例五_带数组下标的for循环遍历
#!/bin/bash
today=`date -d "today" "+%Y-%m-%d"`
#//与/之间与分割的字符 ,另外/后有一个空格不可省略
#别忘了加括号!!!
str=(${today//-/ });
#遍历数组
#${!str[@]}表示将数组下标组成一个数组
for i in "${!str[@]}"; do
echo "$i" "${str[$i]}"
done
案例六_带数组下标while循环遍历
#!/bin/bash
today=`date -d "today" "+%Y-%m-%d"`
#//与/之间与分割的字符 ,另外/后有一个空格不可省略
str=(${today//-/ });
i=0
while [ $i -lt ${#str[@]} ]
#当变量(下标)小于数组长度时进入循环体
do
echo ${str[$i]} "${i}"
#按下标打印数组元素
let i++
done
时间域
%H小时(00..23)
%I小时(01..12)
%k小时(0..23)
%l小时(1..12)
%M分(00..59)
%p显示出AM或PM
%r时间(hh:mm:ssAM或PM),12小时
%s从1970年1月1日00:00:00到目前经历的秒数
%S秒(00..59)
%T时间(24小时制)(hh:mm:ss)
%X显示时间的格式(%H:%M:%S)
%Z时区日期域
%a星期几的简称(Sun..Sat)
%A星期几的全称(Sunday..Saturday)
%b月的简称(Jan..Dec)
%B月的全称(January..December)
%c日期和时间(MonNov814:12:46CST1999)
%d一个月的第几天(01..31)
%D日期(mm/dd/yy)
%h和%b选项相同
%j一年的第几天(001..366)
%m月(01..12)
%w一个星期的第几天(0代表星期天)
%W一年的第几个星期(00..53,星期一为第一天)
%x显示日期的格式(mm/dd/yy)
%y年的最后两个数字(1999则是99)
%Y年(例如:1970,1996等)
%F:等同于%Y-%m-%d
时间语句
#!/bin/bash
#今天
today=`date -d "today" "+%Y-%m-%d"`
today=`date -d "" "+%Y-%m-%d"`
today=`date "+%Y-%m-%d"`
#星期
week=`date -d "-1 day" "+%w"`
#昨天
yesterday=`date -d "yesterday" "+%Y-%m-%d"`
#2天前
Xday=`date -d "2 day ago" "+%Y-%m-%d"`
echo "yesterday:$yesterday"
echo "Xday:$Xday"
#2天后
afterDay=`date -d "2 day" "+%Y-%m-%d"`
echo "afterDay:$afterDay"
# 获取上个月
lastmonth=`date -d "last month" "+%Y-%m-%d"`
#lastmonth=`date -d "1 month ago" "+%Y-%m-%d"`
echo "一个月前:$lastmonth"
# 获取下个月
nextmonth=`date -d "next month" "+%Y-%m-%d"`
#nextmonth=`date -d "1 month" "+%Y-%m-%d"`
echo "一个月后:$nextmonth"
# 获取上一年
lastyear=`date -d "last year" "+%Y-%m-%d"`
#lastyear=`date -d "1 year ago" "+%Y-%m-%d"`
echo "一年前:$lastyear"
# 获取下一年
nextyear=`date -d "next year" "+%Y-%m-%d"`
#nextyear=`date -d "1 year" "+%Y-%m-%d"`
echo "一年后:$nextyear"
#年
year=$(date +%Y)
#year=`date -d "today" "+%Y"`
echo "year:$year"
#月
month=$(date +%m)
#month=`date -d "today" "+%m"`
echo "month:$month"
#日
day=$(date +%d)
#day=`date -d "today" "+%d"`
echo "day:$day"
#时分秒
hms=`date -d "today" "+%H:%M:%S"`
echo "时分秒:$hms"
#年月日的目录
dateDir=`date -d "today" "+%Y/%m/%d/"`
echo "年月日的目录:$dateDir"
#详细日期及时间
time_log=$(date +"%Y-%m-%d %H:%M:%S")
echo "详细日期及时间:$time_log"
#获取指定日期的秒数:注意此处是秒,不是毫秒
time="2019-11-13 17:00:00"
now=$(date -d "$time" +%s)
echo "date_mil:$now"
#获取指定日期的毫秒数:注意此处是秒,不是毫秒
sec=${date_mil}000
echo "毫秒数:$sec"
截取字符串
#!/bin/bash
filename='2018-01-01'
year=${filename:0:4}
month=${filename:5:2}
day=${filename:8:2}
echo "year:${year}"
echo "month:${month}"
echo "day:${day}"
获取字符串长度
d='2018/07/26'
${#d}
字符串分割为数组
#!/bin/bash
str="1,2,3,4";
#//与/之间与分割的字符 ,另外/后有一个空格不可省略
str=${str//,/ };
#遍历数组
#此处的*表示索引
for each in ${str[*]}
do
echo $each
done
提取当前文件所在目录
filepath=$(cd "$(dirname "$0")"; pwd)
判断文件是否存在及是否为空
# 判断文件是否存在
if ls $log_file/"$day".log; then
# 判断文件是否为空
if [[ "$(wc -l $log_file/"$day".log | grep -E -o '^[0-9]+')" == "0" ]] || [[ "$(cat $log_file/"$day".log)" == "" ]]; then
rm -rf $log_file/"$day".log
echo "文件为空,删除本地文件"
echo "$(date +"%Y-%m-%d %H:%M:%S") data_flow_monitor delta task exec success."
else
local2hive
fi
fi
获取目录或文件名
#!/bin/bash
path='/root/logs/soft/'
files=$(ls $path)
for filename in $files
do
if [ ".ok" != ${filename} ]; then
echo ${filename}
fi
done
数值是否相等
testFormat(){
num=$1
if [ ${num} -eq 3 ];then
echo "格式:日期 路径"
echo "C盘根路径用/c表示,其他类似:/d/s3/file/,不需要精确到文件名"
echo "时间格式:20180727"
exit
else
echo "格式正确"
fi
}
字符串截取,循环,相等功能组合
path='/d/s3/file/'
input=${path:0:2}
formats=(/c /d /e /f /g /h /i)
for format in $formats
do
if [ "$input" = "$format" ];then
return 2
fi
done
判断是否包含字符串
案例一
#!/bin/bash
type="track_signup"
result=$(echo $type | grep "track")
if [ "$result" != "" ]; then
echo "包含track"
else
echo "不包含track"
fi
案例二
#!/usr/bin/env bash
s3_path="s3://data-sourcedata/mongodata/"
#s3_path="s3://data-datacenter/bd_test/metadata/"
filepath=$(cd "$(dirname "$0")"; pwd)
cd ${filepath}
fileName='2019-10-10'
if [[ ${fileName} =~ "-" ]]; then
echo "包含"
else
echo "不包含"
fi
判断程序是否执行
ps -ef|grep originaldata|grep -v grep
# 判断originaldata解析程序是否执行
if [ $? -eq 0 ];then
echo "not run"
else
echo "is run"
fi
优化后代码:
# 判断originaldata解析程序是否执行
if ps -ef|grep originaldata|grep -v grep;then
echo "not run"
else
echo "is run"
fi
if ! ps -ef|grep originaldata|grep -v grep;then
echo "is run"
else
echo "not run"
fi
case语句
案例一
#!/bin/sh
SYSTEM=`uname -s`
case $SYSTEM in
Linux)
echo "My system is Linux"
echo "Do Linux stuff here..."
;;
FreeBSD)
echo "My system is FreeBSD"
echo "Do FreeBSD stuff here..."
;;
*)
echo "Unknown system : $SYSTEM"
echo "I don't what to do..."
;;
esac
案例二
for i in $list;
do
case "$i" in
"pageview/")
${aws_path}aws s3 sync ${DATA_PATH}${i}data/${datepath} ${s3_path}page_view/${datepath}${HOSTNAME}
;;
"AppViewScreen/")
${aws_path}aws s3 sync ${DATA_PATH}${i}data/${datepath} ${s3_path}app_view/${datepath}${HOSTNAME}
;;
"MPViewScreen/")
${aws_path}aws s3 sync ${DATA_PATH}${i}data/${datepath} ${s3_path}mp_view/${datepath}${HOSTNAME}
;;
"startWatching/")
${aws_path}aws s3 sync ${DATA_PATH}${i}data/${datepath} ${s3_path}enter_room/${datepath}${HOSTNAME}
;;
"leaveRoom/")
${aws_path}aws s3 sync ${DATA_PATH}${i}data/${datepath} ${s3_path}leave_room/${datepath}${HOSTNAME}
;;
"datagrandShow/")
${aws_path}aws s3 sync ${DATA_PATH}${i}data/${datepath} ${s3_path}datagrand/${datepath}${HOSTNAME}
;;
"livingEnd/")
${aws_path}aws s3 sync ${DATA_PATH}${i}data/${datepath} ${s3_path}living_end/${datepath}${HOSTNAME}
;;
"AppEnd/")
${aws_path}aws s3 sync ${DATA_PATH}${i}data/${datepath} ${s3_path}app_end/${datepath}${HOSTNAME}
;;
"livingStart/")
${aws_path}aws s3 sync ${DATA_PATH}${i}data/${datepath} ${s3_path}living_start/${datepath}${HOSTNAME}
;;
"tag/")
${aws_path}aws s3 sync ${DATA_PATH}${i}data/${datepath} ${s3_path}tags/${datepath}${HOSTNAME}
;;
"AppInstall/")
${aws_path}aws s3 sync ${DATA_PATH}${i}data/${datepath} ${s3_path}app_install/${datepath}${HOSTNAME}
;;
"AppStart/")
${aws_path}aws s3 sync ${DATA_PATH}${i}data/${datepath} ${s3_path}app_start/${datepath}${HOSTNAME}
;;
"memeRec/")
${aws_path}aws s3 sync ${DATA_PATH}${i}data/${datepath} ${s3_path}meme_rec/${datepath}${HOSTNAME}
;;
"click-APP/")
${aws_path}aws s3 sync ${DATA_PATH}${i}data/${datepath} ${s3_path}click_app/${datepath}${HOSTNAME}
;;
"click-link/")
${aws_path}aws s3 sync ${DATA_PATH}${i}data/${datepath} ${s3_path}click_link/${datepath}${HOSTNAME}
;;
"AppClick/")
${aws_path}aws s3 sync ${DATA_PATH}${i}data/${datepath} ${s3_path}app_click/${datepath}${HOSTNAME}
;;
"MPClick/")
${aws_path}aws s3 sync ${DATA_PATH}${i}data/${datepath} ${s3_path}mp_click/${datepath}${HOSTNAME}
;;
"follow/")
;;
*)
${aws_path}aws s3 sync ${DATA_PATH}${i}data/${datepath} ${s3_path}${i}${datepath}${HOSTNAME}
;;
esac
done
删除字符串所有空格
echo " 123 567 " | tr -d '[ \t]'
e=`echo ${end}| tr -d '[ \t]'`
echo ${e}
每小时/天/周执行命令
#!/bin/bash
today=`date "+%Y-%m-%d"`
day_of_month=$(date -d "0 day $today" +%d)
day_of_week=$(date -d "0 day $today" +%w)
WEEK=$(date +%w)
HOUR=$(date +%H)
if [ $# -eq 0 ]; then
hourTime=`date -d "-1 hour" "+%Y%m%d%H"`
todayTime=`date -d "" "+%Y-%m-%d"`
yesTime=`date -d "-1 day" "+%Y-%m-%d"`
beforeYesTime=`date -d "-2 day" "+%Y-%m-%d"`
echo "小时"
# hourTask # 每小时执行
if [ ${HOUR} -eq 7 ];then
echo "天"
fi
if [ ${WEEK} -eq 1 ] && [ ${HOUR} -eq 2 ];then
echo "周"
fi
fi
shell接收外部参数getopts
func() {
echo "Usage:"
echo "test.sh [-a action] [-j jdbcUrl ] [-u username ] [-p password ] [-s "sql" ] [-d targetDir ] [-t phoenixTable]"
echo "Description:"
echo "action,1:sql==>hdfs;2:hdfs==>phoenix;3:包含步骤1和2."
exit -1
}
while getopts 'a:j:u:p:s:d:t:h' OPT; do
case $OPT in
a) action="$OPTARG";;
j) jdbcUrl="$OPTARG";;
u) username="$OPTARG";;
p) password="$OPTARG";;
s) sql="$OPTARG";;
d) targetDir="$OPTARG";;
t) phoenixTable="$OPTARG";;
h) func;;
?) func;;
esac
done
内部调用其他shell,判断语句执行结果
judgeSuccessOrFailAndInvoker.sh
#!/bin/bash
echo "1.sh开始执行";
./2.sh;
if [ $? -eq 0 ];then
echo "2.sh脚本执行成功"
else
echo "2.sh脚本执行失败"
fi
echo "1.sh执行结束";
shell调用mysql
user.sql内容
use test;
select * from user;
#!/bin/bash
#execute all script in specified directory
MYDATE=`date +%F'-'%T'-'%w`
#sql文件存放目录
MYSQL_PATH=/root/logs/test/
#sql文件名称
FILE_NAME=user
#log文件,包含路径
today=`date -d "today" "+%Y%m%d"`
LOG_FILE=/root/logs/test/${FILE_NAME}${today}.log
#数据库用户名
USER=root
#数据库密码
PWD=123456
#执行sql文件,将结果写到log文件中
mysql -u${USER} -p${PWD} -e "source ${MYSQL_PATH}${FILE_NAME}.sql" > ${LOG_FILE}
if [ $? -eq 0 ];then
echo "${MYSQL_PATH}${FILE_NAME}.sql脚本执行成功"
else
echo "${MYSQL_PATH}${FILE_NAME}.sql脚本执行失败"
fi
shell异步调用
案例一
#!/bin/bash
#
# 异步执行(wait)使用样例-父脚本
#
echo "父脚本:启动子脚本.."
./test1.py &
# 通过将shell参数 $! 赋给pid变量,以记录子进程的进程ID
pid1=$!
echo "父脚本:子脚本(PID=${pid1})已启动"
./test2.py &
# 通过将shell参数 $! 赋给pid变量,以记录子进程的进程ID
pid2=$!
echo "父脚本:子脚本(PID=${pid2})已启动"
echo "所有任务开始异步执行"
wait
echo "父脚本:子脚本已结束,父脚本继续.."
echo "父脚本:父脚本执行结束。脚本退出!"
案例二
#!/bin/bash
fun1(){
# ./test1.py &
sleep 5 &
pid=$!
echo "父脚本:子脚本(PID=${pid})已启动"
wait
echo "父脚本:子脚本(PID=${pid})已结束"
}
fun2(){
# ./test2.py &
sleep 5 &
pid=$!
echo "父脚本:子脚本(PID=${pid})已启动"
wait
echo "父脚本:子脚本(PID=${pid})已结束"
}
fun1 &
fun2 &
echo "所有脚本开始异步执行"
wait # 如果fun里面没有wait,则整个脚本立刻退出,不会等待fun里面的sleep
echo "任务完成"
执行shell,写入log
使用2>&1,可以将mr的相关信息写入log,否则不会写入
00 15 * * * bash /home/hadoop/zhouyong/sqoop/sqoop-tbDunRecord.sh 1 > /home/hadoop/zhouyong/sqoop/sqoop-tbDunRecord.log 2>&1
判断输入参数是否为日期格式
#!/bin/bash
###
# @Author: yong.zhou
# @Time: 2020/12/14 14:02:38
# @Description: 工程描述
###
# shellcheck disable=SC1091
source /etc/profile
now=$(date +"%Y-%m-%d %H:%M:%S")
# yesterday=$(date -d "-1 day ${now}" "+%Y-%m-%d")
filepath=$(cd "$(dirname "$0")" || exit; pwd)
cd "${filepath}" || exit
echo "当前时间: ${now} *** 进入 ${filepath} 目录"
# here put the import lib
# 判断输入参数是否符合日期格式
date_regx() {
date_pattern="^[0-9]{4}-[0-9]{2}-[0-9]{2}"
if echo "$1" | grep -Eq "${date_pattern}" && date -d "$1" +%Y%m%d > /dev/null 2>&1; then
echo "0"
else
echo "日期不合法"
fi
}
# 执行函数,不接收返回值
eval date_regx "$1"
# 执行函数,并接收返回值
code=$(eval date_regx "$1")
echo "接收返回值:${code}"
if [ "$(eval date_regx "$1")" == "0" ]; then
echo "正常:$1"
else
echo "异常:$1"
fi