【LINUX】shell中模拟爬虫

url_home="http://localhost:8080"
curl ${url_home} -o 1.html
cat 1.html | grep 'href="' | awk -F"\"" '{ print $2 }' >urls.txt
cat 1.html | grep 'href="' | awk -F"\"" '{ print $3 }'  | awk -F"<" '{ print $1 }' | sed "s:>::g"|sed "s: :_:g" > titles.txt
  
# 打开文件1
exec 3< urls.txt
 
# 打开文件2
exec 4< titles.txt
 
# 读取文件1和文件2的内容
while read -u 3 url && read -u 4 title
do
    echo "url: $url"
    echo "title: $title"
    curl "${url_home}/$url" -o "${title}.html"
done
 
# 关闭文件
exec 3<&-
exec 4<&-