爬豆瓣1000个用户电影信息

获取1000个用户看的电影信息

---

#/bin/bash

# name:         get_see_movies.sh
# version:      1.0
# ceateTime:    2018-08-12
# description:  输入豆瓣用户id,获取所有看过的电影以及评分，标签等，并存到数据库中
# author:       mengyanhuangchao
# email:        406993906@qq.com

# description：  判断输入参数是否合

#INFO打印
info_log(){
    echo -e "[INFO]$1"
}

#SUCCESS打印

success_log(){
    echo -e "\033[32m[SUCCESS]\033[0m$1"
}

#ERROR打印
error_log(){
    echo -e "\033[31m[ERROR]\033[0m$1"
}

if [ $# -eq 1 ];then
    if [ -n "$(echo $1| sed -n "/^[0-9]\+$/p")" ];then
        info_log "The user id you searched for is $1"
    else
        error_log "The user id must number"
        exit 1
    fi
else
    error_log "Usage: bash $0 162545416";
    exit 1
fi
user_id=$1

num=1
while [ $num  -lt 20 ];do
    movie_number=`curl -s https://movie.douban.com/people/$user_id/collect|egrep "看过的电影"|awk -F '(' '{print $2}' |awk -F ')' '{print $1}'|uniq`
    user_name=`curl -s https://movie.douban.com/people/$user_id/collect|egrep "看过的电影"|awk -F '看过的电影' '{print $1}'|awk -F '>' '{print $2}'|tail -n1`
    if [  -n "$user_name" ];then
        if [  -n "$movie_number" ];then
            if [ $movie_number -gt 2 ];then
                info_log "$user_name see $movie_number movies"
                info_log 'Please wait a moment....'
                echo "============" >>movietable
                echo "$user_name" >>movietable
                echo "============" >>movietable
                for i in `seq 0 15 $movie_number`;do
                    curl -s https://movie.douban.com/people/$user_id/collect?start=$i > html
                    cat html |egrep "<em>"| egrep -v '= title'|awk -F '>' '{print $2}' |awk -F '<' '{print $1}'|awk -F '/ ' '{print $1}'  > moviename
                    #for moviename in `cat moviename`;do
                     while read line;do
                        moviename=$line
                        cat html| egrep -A13 "<em>$moviename" >test1
                        rating=`cat test1|egrep 'rating' |awk -F 'rating' '{print $2}' |awk -F '-' '{print $1}'`
                        date=`cat test1|egrep 'date' |awk -F '>' '{print $2}' |awk -F '<' '{print $1}'`
                        comment=`cat test1|egrep 'comment' |awk -F '>' '{print $2}' |awk -F '<' '{print $1}'`
                        tags=`cat test1|egrep 'tags' |awk -F '>' '{print $2}' |awk -F '<' '{print $1}'`
                        echo -e "|$moviename |$date |$comment |$tags |$rating" >>movietable
                    done < moviename
                done
                name="$user_name"_"$user_id"
                mv movietable $name
                rm html  moviename  test1
                success_log "all info save $PWD/$name"
                num=$[ $num + 1 ]
                user_id=$[ $user_id + 1 ]
                sleep 3
                echo $user_id
            else
                user_id=$[ $user_id + 1 ]
                sleep 3
                echo $user_id
            fi
        else
            user_id=$[ $user_id + 1 ]
            sleep 3
            echo $user_id
        fi
    else
        user_id=$[ $user_id + 1 ]
        sleep 3
        echo $user_id
    fi
done