#!/bin/ksh
########################################################
#
# argument 1: Buniness Unit
# argument 2: File Type
# argument 3: File extension
#
########################################################
scriptDIR=`dirname $0`
JOBNAME=`basename $0`
CurDate=`date +%Y%m%d`
JobStartDate=`date +%m/%d/%y`
JobStartTime=`date +%H:%M:%S`
curr_datetime=`date +%Y%m%d_%H%M%S`
HOST=`hostname`
TMPFILE=/tmp/`basename $0`.tmp.$$
TMP_OKAY_FLIST=/tmp/`basename $0`_okay_flist.${curr_datetime}.$$
OK_EXT=OK
CURR_DIR=`pwd`
### 20120912
FTP_COUNT=1
FTP_FAILED="N"
### 20120912
###
if [ ! -f ${scriptDIR}/set_env.sh ];
then
echo "Cannot found ${scriptDIR}/set_env.sh, please check"
exit 1
else
echo "Sourcing ${scriptDIR}/set_env.sh environment file ..."
echo
. ${scriptDIR}/set_env.sh
fi
if [ $# -ne 3 ];
then
echo "Usage: $0 <BU> <File Type> <File extension>"
exit 1
else
BU=$1
F_TYPE=$2
F_EXT=$3
echo "BU : $BU"
echo "File Type : $F_TYPE"
echo "File Extension: $F_EXT"
echo
fi
get_env_var () {
SEARCH_STR="${BU}_FTP_${F_TYPE}"
echo
echo "Search string for ${scriptDIR}/set_env.sh: $SEARCH_STR"
echo
cat ${scriptDIR}/set_env.sh | grep -v '\#' | grep $SEARCH_STR > ${TMPFILE}
. ${TMPFILE}
rm $TMPFILE
nameref ftp_srv=${SEARCH_STR}_SRV
nameref ftp_user=${SEARCH_STR}_USER
nameref ftp_passwd=${SEARCH_STR}_PASSWD
nameref ftp_src_dir=${SEARCH_STR}_SRC_DIR
nameref ftp_dest_dir=${SEARCH_STR}_DEST_DIR
nameref ftp_log_dir=${SEARCH_STR}_LOG_DIR
nameref ftp_processed_dir=${SEARCH_STR}_PROCESSED_DIR
ftp_logfile="${ftp_log_dir}/`basename $0`_${curr_datetime}.log.$$"
echo
echo "${SEARCH_STR}_SRV : $ftp_srv"
echo "${SEARCH_STR}_USER : $ftp_user"
echo "${SEARCH_STR}_PASSWD : $ftp_passwd"
echo "${SEARCH_STR}_SRC_DIR : $ftp_src_dir"
echo "${SEARCH_STR}_DEST_DIR : $ftp_dest_dir"
echo "${SEARCH_STR}_LOG_DIR : $ftp_log_dir"
echo "${SEARCH_STR}_PROCESSED_DIR: $ftp_processed_dir"
# echo "ftp_logfile: $ftp_logfile"
echo
do_checking
}
do_checking () {
if [[ -z ${ftp_srv} ]]; then
echo "ftp server value is null "
exit 1
fi
if [[ -z ${ftp_user} ]]; then
echo "ftp user value is null "
exit 1
fi
if [[ -z ${ftp_passwd} ]]; then
echo "ftp user password value is null "
exit 1
fi
if [ ! -x $ftp_src_dir -o -z $ftp_src_dir ]; then
echo "Cannot access directory ftp source directory "
exit 1
fi
if [ ! -x $ftp_log_dir -o -z $ftp_log_dir ]; then
echo "Cannot access directory ftp log directory "
exit 1
fi
if [ ! -x $ftp_processed_dir -o -z $ftp_processed_dir ]; then
echo "Cannot access directory ftp processed_dir "
exit 1
fi
}
do_ftp () {
ftpserver=$1
ftpuser=$2
ftppasswd=$3
change_dir=$4
TMP_CMDS="/tmp/`basename $0`_${BU}_${F_TYPE}_${curr_datetime}.ftp.$$"
if [ -f $TMP_CMDS ];then
rm $TMP_CMDS
fi
echo "ftp -inv $ftp_srv 2>&1 1> $ftp_logfile << EOF" >> ${TMP_CMDS}
echo "user $ftp_user $ftp_passwd" >> ${TMP_CMDS}
echo "cd $change_dir" >> ${TMP_CMDS}
echo "binary" >> ${TMP_CMDS}
echo "lcd $ftp_src_dir" >> ${TMP_CMDS}
cat $TMP_OKAY_FLIST | sed -e 's/^/put /g' >> ${TMP_CMDS}
echo "bye" >> ${TMP_CMDS}
echo "EOF" >> ${TMP_CMDS}
echo
echo "ftp command file location: $TMP_CMDS"
### Added 20120912 Ben Lu
while [ ${FTP_COUNT} -lt 4 ]; do
### Added 20120912 Ben Lu
sh ${TMP_CMDS}
### Added 20120912 Ben Lu
if [ ${FTP_COUNT} -lt 3 ]; then
echo "TRY FTP ($FTP_COUNT) time"
FTP_COUNT=`expr ${FTP_COUNT} + 1`
grep -i "^Not connected." $ftp_logfile > /dev/null 2>&1
if [ $? -eq 0 ]; then
echo "$ftp_srv ERROR FTP CONNECT"
FTP_FAILED="Y"
fi
grep -i "No such file" $ftp_logfile > /dev/null 2>&1
if [ $? -eq 0 ]; then
echo "$ftp_srv ERROR FTP NO SUCH FILE"
FTP_FAILED="Y"
fi
grep -i "access denied" $ftp_logfile > /dev/null 2>&1
if [ $? -eq 0 ]; then
echo "$ftp_srv ERROR FTP ACCESS DENIED"
FTP_FAILED="Y"
fi
grep -i "^Please login" $ftp_logfile > /dev/null 2>&1
if [ $? -eq 0 ]; then
echo "$ftp_srv ERROR FTP LOGIN"
FTP_FAILED="Y"
fi
grep -i "Login failed" $ftp_logfile > /dev/null 2>&1
if [ $? -eq 0 ]; then
echo "$ftp_srv ERROR FTP LOGIN"
FTP_FAILED="Y"
fi
if [ ${FTP_FAILED} = "Y" ]; then
FTP_FAILED="N"
if [ -f $ftp_logfile ]; then
rm $ftp_logfile
fi
else
FTP_COUNT=4
fi
else
echo "TRY FTP ($FTP_COUNT) time"
### Added 20120912 Ben Lu
FTP_COUNT=`expr ${FTP_COUNT} + 1`
grep -i "^Not connected." $ftp_logfile > /dev/null 2>&1
if [ $? -eq 0 ]; then
echo "$ftp_srv ERROR FTP CONNECT"
exit 2
fi
grep -i "No such file" $ftp_logfile > /dev/null 2>&1
if [ $? -eq 0 ]; then
echo "$ftp_srv ERROR FTP NO SUCH FILE"
exit 2
fi
grep -i "access denied" $ftp_logfile > /dev/null 2>&1
if [ $? -eq 0 ]; then
echo "$ftp_srv ERROR FTP ACCESS DENIED"
exit 2
fi
grep -i "^Please login" $ftp_logfile > /dev/null 2>&1
if [ $? -eq 0 ]; then
echo "$ftp_srv ERROR FTP LOGIN"
exit 2
fi
grep -i "Login failed" $ftp_logfile > /dev/null 2>&1
if [ $? -eq 0 ]; then
echo "$ftp_srv ERROR FTP LOGIN"
exit 2
fi
if [ -f $TMP_CMDS ]; then
rm $TMP_CMDS
fi
### Added 20120912 Ben Lu
fi
done
### Added 20120912 Ben Lu
}
move_ftpfile_to_process_dir () {
echo "OKAY file list: $TMP_OKAY_FLIST"
cd $ftp_src_dir
echo "Current directory: `pwd`"
if [ -f $TMP_OKAY_FLIST ]; then
size=`stat -c %s $TMP_OKAY_FLIST`
if [ $size -gt 0 ]; then
if [ "${F_EXT}" != "${OK_EXT}" ]; then
cat $TMP_OKAY_FLIST | grep -v "${OK_EXT}" | while read have_okay_fname
do
#f_suffix=`basename $have_okay_fname .zip`
f_suffix=`basename $have_okay_fname .${F_EXT}`
f_filename1=${f_suffix}.TMP
sed -e '1s/^\xEF\xBB\xBF//' $have_okay_fname > $f_filename1
mv $f_filename1 $have_okay_fname
f_suffix=`basename $have_okay_fname .${F_EXT}`
echo "mv ${f_suffix}* $ftp_processed_dir"
mv ${f_suffix}* $ftp_processed_dir
done
else
cat $TMP_OKAY_FLIST | grep "${OK_EXT}" | while read have_okay_fname
do
mv_fname=`basename $have_okay_fname`
echo "mv ${mv_fname} $ftp_processed_dir"
mv ${mv_fname}* $ftp_processed_dir
done
fi
fi
fi
}
print_footer () {
echo "
===============================================================
=
= FTP transfer log: $ftp_logfile
= `date`
=
===============================================================
"
}
do_cleanup () {
if [ -f $TMPFILE ]; then
rm $TMPFILE
fi
if [ -f $TMP_OKAY_FLIST ]; then
rm $TMP_OKAY_FLIST
fi
}
#############
# Main
#############
## get ftp related variables
get_env_var
cd $ftp_src_dir
chk_acxiom=`echo ${BU} | awk -F'_' '{print $2}'`
## check for acxiom file type
if [ "${chk_acxiom}" = "ACXIOM" ]; then
ftp_file_pattern="${F_TYPE}*.${F_EXT}"
acxiom_ftype="yes"
else
ftp_file_pattern="${F_TYPE}_${BU}*.${F_EXT}"
acxiom_ftype="no"
fi
flist=`ls -1 $ftp_file_pattern`
echo
echo "Current Directory : $ftp_src_dir"
echo "Search pattern : $ftp_file_pattern"
echo
echo
if [[ -z ${flist} ]]
then
echo
echo "No files found for pattern $ftp_file_pattern"
echo
exit 0
else
flst_size=`stat -c %s $flist`
for fname in $flist
do
file_suffix=`basename $fname .${F_EXT}`
okay_filename="${file_suffix}.${OK_EXT}"
if [ -f $okay_filename ];then
if [[ ${3} != "${OK_EXT}" || $flst_size -gt 1 ]];then
#zip_fname="${file_suffix}.zip"
zip_fname=${fname}
echo "Zipping file $fname to ${zip_fname}"
echo ""
#zip ${zip_fname} ${fname}
if [ $? -eq 0 ]; then
echo "$zip_fname" >> $TMP_OKAY_FLIST
echo "$okay_filename" >> $TMP_OKAY_FLIST
echo "$fname, $okay_filename OK found"
else
echo "Problem in zipping file $fname"
exit 2
fi
else
echo "$okay_filename" >> $TMP_OKAY_FLIST
echo "$fname, $okay_filename OK found"
fi
else
echo "$fname, $okay_filename OK file NOT found"
fi
done
if [ -f $TMP_OKAY_FLIST ]; then
size=`stat -c %s $TMP_OKAY_FLIST`
if [ $size -gt 0 ]; then
do_ftp $ftp_srv $ftp_user $ftp_passwd $ftp_dest_dir
fi
fi
fi
######
#move uploaded files to processed directory
#######
if [ -f $TMP_OKAY_FLIST ]; then
total_okay_file=`wc -l $TMP_OKAY_FLIST | awk '{print $1}'`
else
total_okay_file=0
fi
if [ -f $ftp_logfile ]; then
if [ ${acxiom_ftype} = "yes" ];then
total_file_ftp=`cat $ftp_logfile | grep $F_TYPE | grep -v '^150' | wc -l |awk '{print $1}'`
else
total_file_ftp=`cat $ftp_logfile | grep $BU| grep $F_TYPE | grep -v '^150' | wc -l |awk '{print $1}'`
fi
else
total_file_ftp=0
fi
echo
echo "Total number of files need to upload: $total_okay_file"
echo "Total number of files ftp to $ftp_srv: $total_file_ftp"
echo
if [ $total_okay_file -eq $total_file_ftp ]; then
if [ $total_okay_file -gt 0 ]; then
move_ftpfile_to_process_dir
print_footer
fi
exit 0
else
echo
echo "Number of files upload does not match !!"
echo
print_footer
exit 1
fi
do_cleanup
cd $CURR_DIR
########################################################
#
# argument 1: Buniness Unit
# argument 2: File Type
# argument 3: File extension
#
########################################################
scriptDIR=`dirname $0`
JOBNAME=`basename $0`
CurDate=`date +%Y%m%d`
JobStartDate=`date +%m/%d/%y`
JobStartTime=`date +%H:%M:%S`
curr_datetime=`date +%Y%m%d_%H%M%S`
HOST=`hostname`
TMPFILE=/tmp/`basename $0`.tmp.$$
TMP_OKAY_FLIST=/tmp/`basename $0`_okay_flist.${curr_datetime}.$$
OK_EXT=OK
CURR_DIR=`pwd`
### 20120912
FTP_COUNT=1
FTP_FAILED="N"
### 20120912
###
if [ ! -f ${scriptDIR}/set_env.sh ];
then
echo "Cannot found ${scriptDIR}/set_env.sh, please check"
exit 1
else
echo "Sourcing ${scriptDIR}/set_env.sh environment file ..."
echo
. ${scriptDIR}/set_env.sh
fi
if [ $# -ne 3 ];
then
echo "Usage: $0 <BU> <File Type> <File extension>"
exit 1
else
BU=$1
F_TYPE=$2
F_EXT=$3
echo "BU : $BU"
echo "File Type : $F_TYPE"
echo "File Extension: $F_EXT"
echo
fi
get_env_var () {
SEARCH_STR="${BU}_FTP_${F_TYPE}"
echo
echo "Search string for ${scriptDIR}/set_env.sh: $SEARCH_STR"
echo
cat ${scriptDIR}/set_env.sh | grep -v '\#' | grep $SEARCH_STR > ${TMPFILE}
. ${TMPFILE}
rm $TMPFILE
nameref ftp_srv=${SEARCH_STR}_SRV
nameref ftp_user=${SEARCH_STR}_USER
nameref ftp_passwd=${SEARCH_STR}_PASSWD
nameref ftp_src_dir=${SEARCH_STR}_SRC_DIR
nameref ftp_dest_dir=${SEARCH_STR}_DEST_DIR
nameref ftp_log_dir=${SEARCH_STR}_LOG_DIR
nameref ftp_processed_dir=${SEARCH_STR}_PROCESSED_DIR
ftp_logfile="${ftp_log_dir}/`basename $0`_${curr_datetime}.log.$$"
echo
echo "${SEARCH_STR}_SRV : $ftp_srv"
echo "${SEARCH_STR}_USER : $ftp_user"
echo "${SEARCH_STR}_PASSWD : $ftp_passwd"
echo "${SEARCH_STR}_SRC_DIR : $ftp_src_dir"
echo "${SEARCH_STR}_DEST_DIR : $ftp_dest_dir"
echo "${SEARCH_STR}_LOG_DIR : $ftp_log_dir"
echo "${SEARCH_STR}_PROCESSED_DIR: $ftp_processed_dir"
# echo "ftp_logfile: $ftp_logfile"
echo
do_checking
}
do_checking () {
if [[ -z ${ftp_srv} ]]; then
echo "ftp server value is null "
exit 1
fi
if [[ -z ${ftp_user} ]]; then
echo "ftp user value is null "
exit 1
fi
if [[ -z ${ftp_passwd} ]]; then
echo "ftp user password value is null "
exit 1
fi
if [ ! -x $ftp_src_dir -o -z $ftp_src_dir ]; then
echo "Cannot access directory ftp source directory "
exit 1
fi
if [ ! -x $ftp_log_dir -o -z $ftp_log_dir ]; then
echo "Cannot access directory ftp log directory "
exit 1
fi
if [ ! -x $ftp_processed_dir -o -z $ftp_processed_dir ]; then
echo "Cannot access directory ftp processed_dir "
exit 1
fi
}
do_ftp () {
ftpserver=$1
ftpuser=$2
ftppasswd=$3
change_dir=$4
TMP_CMDS="/tmp/`basename $0`_${BU}_${F_TYPE}_${curr_datetime}.ftp.$$"
if [ -f $TMP_CMDS ];then
rm $TMP_CMDS
fi
echo "ftp -inv $ftp_srv 2>&1 1> $ftp_logfile << EOF" >> ${TMP_CMDS}
echo "user $ftp_user $ftp_passwd" >> ${TMP_CMDS}
echo "cd $change_dir" >> ${TMP_CMDS}
echo "binary" >> ${TMP_CMDS}
echo "lcd $ftp_src_dir" >> ${TMP_CMDS}
cat $TMP_OKAY_FLIST | sed -e 's/^/put /g' >> ${TMP_CMDS}
echo "bye" >> ${TMP_CMDS}
echo "EOF" >> ${TMP_CMDS}
echo
echo "ftp command file location: $TMP_CMDS"
### Added 20120912 Ben Lu
while [ ${FTP_COUNT} -lt 4 ]; do
### Added 20120912 Ben Lu
sh ${TMP_CMDS}
### Added 20120912 Ben Lu
if [ ${FTP_COUNT} -lt 3 ]; then
echo "TRY FTP ($FTP_COUNT) time"
FTP_COUNT=`expr ${FTP_COUNT} + 1`
grep -i "^Not connected." $ftp_logfile > /dev/null 2>&1
if [ $? -eq 0 ]; then
echo "$ftp_srv ERROR FTP CONNECT"
FTP_FAILED="Y"
fi
grep -i "No such file" $ftp_logfile > /dev/null 2>&1
if [ $? -eq 0 ]; then
echo "$ftp_srv ERROR FTP NO SUCH FILE"
FTP_FAILED="Y"
fi
grep -i "access denied" $ftp_logfile > /dev/null 2>&1
if [ $? -eq 0 ]; then
echo "$ftp_srv ERROR FTP ACCESS DENIED"
FTP_FAILED="Y"
fi
grep -i "^Please login" $ftp_logfile > /dev/null 2>&1
if [ $? -eq 0 ]; then
echo "$ftp_srv ERROR FTP LOGIN"
FTP_FAILED="Y"
fi
grep -i "Login failed" $ftp_logfile > /dev/null 2>&1
if [ $? -eq 0 ]; then
echo "$ftp_srv ERROR FTP LOGIN"
FTP_FAILED="Y"
fi
if [ ${FTP_FAILED} = "Y" ]; then
FTP_FAILED="N"
if [ -f $ftp_logfile ]; then
rm $ftp_logfile
fi
else
FTP_COUNT=4
fi
else
echo "TRY FTP ($FTP_COUNT) time"
### Added 20120912 Ben Lu
FTP_COUNT=`expr ${FTP_COUNT} + 1`
grep -i "^Not connected." $ftp_logfile > /dev/null 2>&1
if [ $? -eq 0 ]; then
echo "$ftp_srv ERROR FTP CONNECT"
exit 2
fi
grep -i "No such file" $ftp_logfile > /dev/null 2>&1
if [ $? -eq 0 ]; then
echo "$ftp_srv ERROR FTP NO SUCH FILE"
exit 2
fi
grep -i "access denied" $ftp_logfile > /dev/null 2>&1
if [ $? -eq 0 ]; then
echo "$ftp_srv ERROR FTP ACCESS DENIED"
exit 2
fi
grep -i "^Please login" $ftp_logfile > /dev/null 2>&1
if [ $? -eq 0 ]; then
echo "$ftp_srv ERROR FTP LOGIN"
exit 2
fi
grep -i "Login failed" $ftp_logfile > /dev/null 2>&1
if [ $? -eq 0 ]; then
echo "$ftp_srv ERROR FTP LOGIN"
exit 2
fi
if [ -f $TMP_CMDS ]; then
rm $TMP_CMDS
fi
### Added 20120912 Ben Lu
fi
done
### Added 20120912 Ben Lu
}
move_ftpfile_to_process_dir () {
echo "OKAY file list: $TMP_OKAY_FLIST"
cd $ftp_src_dir
echo "Current directory: `pwd`"
if [ -f $TMP_OKAY_FLIST ]; then
size=`stat -c %s $TMP_OKAY_FLIST`
if [ $size -gt 0 ]; then
if [ "${F_EXT}" != "${OK_EXT}" ]; then
cat $TMP_OKAY_FLIST | grep -v "${OK_EXT}" | while read have_okay_fname
do
#f_suffix=`basename $have_okay_fname .zip`
f_suffix=`basename $have_okay_fname .${F_EXT}`
f_filename1=${f_suffix}.TMP
sed -e '1s/^\xEF\xBB\xBF//' $have_okay_fname > $f_filename1
mv $f_filename1 $have_okay_fname
f_suffix=`basename $have_okay_fname .${F_EXT}`
echo "mv ${f_suffix}* $ftp_processed_dir"
mv ${f_suffix}* $ftp_processed_dir
done
else
cat $TMP_OKAY_FLIST | grep "${OK_EXT}" | while read have_okay_fname
do
mv_fname=`basename $have_okay_fname`
echo "mv ${mv_fname} $ftp_processed_dir"
mv ${mv_fname}* $ftp_processed_dir
done
fi
fi
fi
}
print_footer () {
echo "
===============================================================
=
= FTP transfer log: $ftp_logfile
= `date`
=
===============================================================
"
}
do_cleanup () {
if [ -f $TMPFILE ]; then
rm $TMPFILE
fi
if [ -f $TMP_OKAY_FLIST ]; then
rm $TMP_OKAY_FLIST
fi
}
#############
# Main
#############
## get ftp related variables
get_env_var
cd $ftp_src_dir
chk_acxiom=`echo ${BU} | awk -F'_' '{print $2}'`
## check for acxiom file type
if [ "${chk_acxiom}" = "ACXIOM" ]; then
ftp_file_pattern="${F_TYPE}*.${F_EXT}"
acxiom_ftype="yes"
else
ftp_file_pattern="${F_TYPE}_${BU}*.${F_EXT}"
acxiom_ftype="no"
fi
flist=`ls -1 $ftp_file_pattern`
echo
echo "Current Directory : $ftp_src_dir"
echo "Search pattern : $ftp_file_pattern"
echo
echo
if [[ -z ${flist} ]]
then
echo
echo "No files found for pattern $ftp_file_pattern"
echo
exit 0
else
flst_size=`stat -c %s $flist`
for fname in $flist
do
file_suffix=`basename $fname .${F_EXT}`
okay_filename="${file_suffix}.${OK_EXT}"
if [ -f $okay_filename ];then
if [[ ${3} != "${OK_EXT}" || $flst_size -gt 1 ]];then
#zip_fname="${file_suffix}.zip"
zip_fname=${fname}
echo "Zipping file $fname to ${zip_fname}"
echo ""
#zip ${zip_fname} ${fname}
if [ $? -eq 0 ]; then
echo "$zip_fname" >> $TMP_OKAY_FLIST
echo "$okay_filename" >> $TMP_OKAY_FLIST
echo "$fname, $okay_filename OK found"
else
echo "Problem in zipping file $fname"
exit 2
fi
else
echo "$okay_filename" >> $TMP_OKAY_FLIST
echo "$fname, $okay_filename OK found"
fi
else
echo "$fname, $okay_filename OK file NOT found"
fi
done
if [ -f $TMP_OKAY_FLIST ]; then
size=`stat -c %s $TMP_OKAY_FLIST`
if [ $size -gt 0 ]; then
do_ftp $ftp_srv $ftp_user $ftp_passwd $ftp_dest_dir
fi
fi
fi
######
#move uploaded files to processed directory
#######
if [ -f $TMP_OKAY_FLIST ]; then
total_okay_file=`wc -l $TMP_OKAY_FLIST | awk '{print $1}'`
else
total_okay_file=0
fi
if [ -f $ftp_logfile ]; then
if [ ${acxiom_ftype} = "yes" ];then
total_file_ftp=`cat $ftp_logfile | grep $F_TYPE | grep -v '^150' | wc -l |awk '{print $1}'`
else
total_file_ftp=`cat $ftp_logfile | grep $BU| grep $F_TYPE | grep -v '^150' | wc -l |awk '{print $1}'`
fi
else
total_file_ftp=0
fi
echo
echo "Total number of files need to upload: $total_okay_file"
echo "Total number of files ftp to $ftp_srv: $total_file_ftp"
echo
if [ $total_okay_file -eq $total_file_ftp ]; then
if [ $total_okay_file -gt 0 ]; then
move_ftpfile_to_process_dir
print_footer
fi
exit 0
else
echo
echo "Number of files upload does not match !!"
echo
print_footer
exit 1
fi
do_cleanup
cd $CURR_DIR