shell脚本定时采集日志数据到hdfs_模板

最新推荐文章于 2023-04-10 15:15:35 发布

不由不该-

最新推荐文章于 2023-04-10 15:15:35 发布

阅读量481

点赞数

本文链接：https://blog.csdn.net/weixin_42295141/article/details/89521621

版权

shell的单使用语法---待续

#!/bin/bash

#set java env
export JAVA_HOME=/HOME/hadoop/app/jdk1.8.0
export JRE_HOME=${JAVA_HOME}/jre
export CLASSPATH=.:${JAVA_HOME}/lib:${JRE_HOME}/lib
export PATH=${JAVA_HOME}/bin:$PATH

#set hadoop env
export HADOOP_HOME=/HOME/hadoop/app/hadoop-2.6.4
export PATH=${HADOOP_HOME}/bin:${HADOOP_HOME}/abin:$PATH

#日志文件存放目录
log_src_dir=/home/hadoop/logs/log

#待上传文件存放的目录
log_toupload_dir=/home/hadoop/logs/toupload

#日志文件上传到hdfs的根目录	
#可以用data来获取当前日期到小时
hdfs_root_dir=/data/clicklog/20190425

#获取日志文件目录，判断是否有需要上传的目录
echo "log_src_dir:"$log_src_dir
ls $log_src_dir | while read filename
do 
	if [["$filename" == acess.log.* ]]; then
	date=`date +%Y_%m_%d_%H_%M_%S`
	#将文件移动到待上传目录并重命名
	echo "moving $log_src_dir$filename to $log_toupload_dir"xxx_click_log_$filename"date"
	mv $log_src_dir$filename to $log_toupload_dir"xxx_click_log_$filename"$date
	echo $log_toupload_dir"xxx_click_log_$filename"$date >> $log_toupload_dir"willdoing."$date
	fi
done

ls $log_toupload_dir | grep will | grep -v "_copy_" | grep -v "_done_"; while read line
do
	#打印信息
	echo "upload is in file:"$line
	mv $log_toupload_dir$line $log_toupload_dir$line"_copy_"
	#一个一个读取待上传的文件名
	cat $log_toupload_dir$line"_copy_" | while read line
	do 
		#打印信息
		echo "puting...$line to hdfs path...$hdfs_root_dir"
		hadoop fs -put $line $hdfs_root_dir
	done
	mv 	$log_toupload_dir$line"_copy_" $log_toupload_dir$line"_done_"
done