[nlp] bash脚本划分训练集/验证集/测试集

#!/bin/bash

# 计算总行数
filename="seq.in"
number=$(wc -l <"${filename}")
echo $number
# 计算训练集、验证集和测试集的行数
train_lines=$((${number}*8/10))
valid_lines=$((${number}/10))
test_lines=$((${number}/10))
echo $train_lines, $valid_lines, $test_lines

file_list="seq.in seq.out label"
for file in $file_list;do
# 使用sed命令提取训练集、验证集和测试集的数据
echo "1,${train_lines}p"
sed -n "1,${train_lines}p" $file > ./train/${file}
sed -n "$((${train_lines}+1)),$((${train_lines}+${valid_lines}))p" $file > ./dev/${file}
sed -n "$((${train_lines}+${valid_lines}+1)),$((${train_lines}+${valid_lines}+${test_lines}))p" $file > ./test/${file}
done

echo "Train set saved in ./train/seq.in"
echo "Validation set saved in ./valid/seq.in"
echo "Test set saved in ./test/seq.in"