解决方案:需要将此2张表的列字段顺序进行调换
首先自己创建一个deal_data.rb文件,将下面的代码直接复制粘贴,注意修改相关的路径目录:
lineitem_str = "$11,$1,$4,$2,$3,$5,$6,$7,$8,$9,$10,$12,$13,$14,$15,$16"
orders_str = "$1,$5,$2,$3,$4,$6,$7,$8,$9"
# "awk -F'|' -v OFS='|' '{print #{str}}' test.csv"
prod_mode = ARGV[0] == "prod"
out_dir = "/home/hdfs/test/" #【修改】存放数据的目录
%x(mkdir #{out_dir}/lineitem) unless Dir.exist?("#{out_dir}/lineitem")
%x(mkdir #{out_dir}/orders) unless Dir.exist?("#{out_dir}/orders")
lineitem_files = Dir["lineitem/lineitem_tbl*"] #【修改】存放要修改的文件目录
orders_files = Dir["orders/orders_tbl*"] #【修改】存放要修改的文件目录
total_count = lineitem_files.size + orders_files.size
cur_count = 0
lineitem_files.each do |filepath|
# lineitem/lineitem_tbl25
file = filepath.match(/\w+\/(\w+)/)[1]
cmd = "awk -F'|' -v OFS='|' '{print #{lineitem_str}}' #{filepath} > #{out_dir}/lineitem/#{file}"
if prod_mode
%x(#{cmd})
else
puts cmd
end
cur_count += 1
puts "#{cur_count}/#{total_count} finished"
end
orders_files.each do |filepath|
# orders/orders_tbl00
file = filepath.match(/\w+\/(\w+)/)[1]
cmd = "awk -F'|' -v OFS='|' '{print #{orders_str}}' #{filepath} > #{out_dir}/orders/#{file}"
if prod_mode
%x(#{cmd})
else
puts cmd
end
cur_count += 1
puts "#{cur_count}/#{total_count} finished"
end