ETL开发常用shell文件操作

#!/bin/bash

file_path=$1
file_path_tmp='/data/2023_file_temp'
file_result='/data/result_data/file_result.txt'

#:<<!
rm -rf $file_path_tmp
mkdir $file_path_tmp
for file7z in $file_path/*
do
  7za x $file7z -o/$file_path_tmp
  echo $file7z
done

#2、补充日期和股票代码
echo "文件处理：逗号分隔，增加日期、股票字段"
for dirfile_date in /data/2023_file_temp/*
do
  echo $dirfile_date
  for dirfile in $dirfile_date/*
  do
    echo $dirfile
    dos2unix $dirfile
	
	#删除文件
	cd $dirfile_date
	rm -rf `ls |grep *QQ842992799*`
    
	for file in $dirfile
    do
      echo $file
      echo $dirfile_date
	  echo $dirfile
      stockdate=${dirfile_date:21:8}
      stockfile=${dirfile:30:8}
	  echo $stockdate
	  echo $stockfile
      sed -i 's/$/	'$stockfile'	'$stockdate'/' $file

    done
		
  done
    
done
#!

# 合并文件
rm -rf $file_result
for dirfile_date in /data/2023_file_temp/*
do
  echo $dirfile_date
  for dirfile in $dirfile_date/*
  do
    echo $dirfile    
	for file in $dirfile
    do
      echo "file" $file
      echo $dirfile_date
	  echo $dirfile
 
	  echo $stockdate
	  echo $stockfile
	  
	  cat $file >> $file_result

    done
		
  done
    
done

psql -h 192.168.3.34 -p5432 -U postgres -d stock -c "insert into dw_stock_base_d select * from dw_stock_base_d_temp; commit;"