用Shell脚本实现自动从NewSmth.net的MyPhoto版下载照片

写了一个脚本，用于自动从水木社区（Newsmth.net）的MyPhoto版自动下载图片到本地。

运行该脚本后，将自动从水木社区的MyPhoto版自动下载最近N（默认为3，通过参数1指定）天内的所有图片到本地的photo目录下。

用法：

1、把以下代码保存为autoPicSmth.sh

2、为脚本增加可执行权限，并运行脚本。

CHEYO:~/auto # chmod +x autoPicSmth.sh
CHEYO:~/auto # ./autoPicSmth.sh

脚本写得比较粗糙，欢迎优化改进。

源码：

# !/bin/bash

#####################################################################
#       Script:  autoPicSmth.sh
#       Author:  cheyo
#        Email:  icheyo at Gmail dot com
#         From:  www.icheyo.net
#         Date:  2008-02-22
#
#  Description:
#      This script is used for downloading pictures from the MyPhoto 
#      board in the newsmth.net automatically.
#
#####################################################################

# Usage: autoPicSmth.sh [days] 
# days:  download all pictures of recent /days/ days
# For Example:   ./autoPicSmth.sh 3

WORKING_DIR = working
PIC_OUT_DIR = photo
DAYS_TO_DOWN = 3
QUERY_FILE = QueryResult . tmp
THREAD_FILE = ThreadUrl . tmp
FORMAT_FILE = ThreadInfo . tmp
CURR_THREAD_FILE = CurrThread . tmp 
PIC_URL_FILE = PicUrl . tmp
PIC_DOWN_LOG = PicDown . log
PIC_INFO_FILE1 = PicInfo1 . tmp
PIC_INFO_FILE2 = PicInfo2 . tmp
PIC_INFO_FILE3 = PicInfoFinal . tmp

#  ------------------------------------------------------------------ #
# ShowUsage()
# Show the usage of this script
# ------------------------------------------------------------------ #
ShowUsage()
{
    echo  " This script is used for automatic downloading pictures from MyPhoto board in the newsmth.net "
    echo  "   Usage: autoPicSmth.sh [days] "
    echo  "   days:  download all pictures of recent /days/ days. 3 for default. "
    echo  "   Example:   ./autoPicSmth.sh 3 "
}

#  check arguments
if  [ $ #  -gt 1 ]; then
    ShowUsage
     exit   1 ;
elif [ $ #  -eq 1 ]; then
    DAYS_TO_DOWN = $ 1
fi

mkdir   - p  $WORKING_DIR  
cd  $WORKING_DIR

#  Get the thread search result HTML page to local
SearchUrl = " http://bbs4.newsmth.net/bbsbfind.php?q=1&board=MyPhoto&dt=${DAYS_TO_DOWN}&ag=1 "
curl  " ${SearchUrl} "   - o ${QUERY_FILE}

#  Create a file to store all Thread URLs
egrep  " <a href="bbscon.php?bid= "   $QUERY_FILE   |  awk  - F[ <> " ] '{print  " http :// bbs4 . newsmth . net / " $9}' > $THREAD_FILE
ThreadCount=`cat $THREAD_FILE | wc -l`
echo  " Total ${ThreadCount} threads are found . "

# Create a file to store all BoardId and ThreadId
awk -F[=&] '{print $2,$4}' $THREAD_FILE > $FORMAT_FILE 

# Create a file to sotre all pictures infomation
# Format: BoardId ArticleId FileName FileSize FileId
echo  " #  BoardId ArticleId FileName FileSize FileId" > $PIC_INFO_FILE1

cat  $FORMAT_FILE   |   while   read  BoardId ArticleId
do
    ThreadUrl = `echo  " http://bbs4.newsmth.net/bbscon.php?bid=$BoardId&id=$ArticleId " `
    curl  " $ThreadUrl "   - o  $CURR_THREAD_FILE
     grep   " attach "   $CURR_THREAD_FILE   |  tr  " ); "   " )  "   |   grep   " attach "   |  awk  - F[ '  ,)] -v BoardId=$BoardId -v ArticleId=$ArticleId  ' { print  BoardId , ArticleId , $ 2 , $ 5 , $ 7 } '  >> $PIC_INFO_FILE1
done

# Create a file to store all pictures info with file extention name 
# but not full file name.
# Format:  BoardId ArticleId FileExt FileSize FileId
# echo "# BoardId ArticleId FileExt FileSize FileId" > $PIC_INFO_FILE2
awk -F[. ]   ' $ 0 ~/^ [ ^ # ]/ {print $1,$2,$4,$5,$6}' $PIC_INFO_FILE1 >> $PIC_INFO_FILE2
# Remove the records which don't contain enough info.
# in normal case, it should be 5 columns in the file.
awk  ' $5~/^[^$]/ {print $0} '   $PIC_INFO_FILE2   >   $PIC_INFO_FILE3  

#  Create a file to store all picture url
grep   ^ [ ^ # ] $PIC_INFO_FILE3 | while read BoardId ArticleId FileExt FileSize FileId
do
     if  [  $FileSize   - gt  51200  ]; then
        FileType = " p "
     else
        FileType = " s "
    fi

    PicUrl = `echo  " http://att.newsmth.net/att.php?$FileType.$BoardId.$ArticleId.$FileId.$FileExt " `
    echo  " $PicUrl "   >>   $PIC_URL_FILE  
done

#  Remove all duplicted URL from the file
mv ${PIC_URL_FILE} ${PIC_URL_FILE} . tmp
sort   - dfu ${PIC_URL_FILE} . tmp  >  ${PIC_URL_FILE}
rm ${PIC_URL_FILE} . tmp

#  Remove the URLs which have been downed before
if  [  - f  " ../${PIC_OUT_DIR}/${PIC_DOWN_LOG} "  ]; then
    cp  ../ $PIC_OUT_DIR / ${PIC_DOWN_LOG}  .
    awk  ' {print $3} '  ${PIC_DOWN_LOG}  >  ${PIC_URL_FILE} . history
     sort   - dfu ${PIC_URL_FILE} . history  >  ${PIC_URL_FILE} . tmp
    mv ${PIC_URL_FILE} . tmp ${PIC_URL_FILE} . history
    comm  - 1   - 3  ${PIC_URL_FILE} . history ${PIC_URL_FILE}  >  ${PIC_URL_FILE} . tmp
    mv ${PIC_URL_FILE} . tmp ${PIC_URL_FILE}
    rm ${PIC_URL_FILE} . history
fi

#  Download all pictures from server to local 
PicCount = `wc  - l  $PIC_URL_FILE   |  awk  ' {print $1} ' `
PicIndex = 1
mkdir   - p  ../ $PIC_OUT_DIR  
echo  " Total number of pictures to be downloaded: $PicCount "

cat  $PIC_URL_FILE   |   while   read  CurrUrl
do
    FileName = `echo  " $CurrUrl "   |  awk  - F[ ? ]  ' {print $2} ' `
    echo  " [$PicIndex/$PicCount] Start to download $CurrUrl "
    curl  " $CurrUrl "   - o  ../ $PIC_OUT_DIR / $FileName
     #  Write download log to log file
    CurrTime = `date  + " %Y-%m-%d %H:%M:%S " `
    echo  " $CurrTime $CurrUrl "   >>   " ../$PIC_OUT_DIR/$PIC_DOWN_LOG "
    echo  " [$PicIndex/$PicCount] Download finished. "
    echo  ""
    PicIndex = `expr  $PicIndex   +   1 `
done

# mv $PIC_URL_FILE ../$PIC_OUT_DIR/PicUrl.list
#mv $PIC_INFO_FILE3 ../$PIC_OUT_DIR/PicInfo.list
cd  ..
rm  - r  $WORKING_DIR

echo  " All Pictures Downloading finished. "

http://write.blog.csdn.net/postedit