爬取猫眼评分前100的电影

这是简单的聚焦爬虫，爬取猫眼前100的电影，刚学爬虫时写的，没有什么困难的东西，但是如果以后再有新人看到的话，希望能对你产生一点点的帮助，具体代码和注释如下：

import requests
from bs4 import BeautifulSoup
from requests.exceptions import RequestException
import pymongo
from config import *
import json

client=pymongo.MongoClient(MONGO_URL,connect=False)
db=client[MONGO_DB]

# url='http://maoyan.com/board/4'
headers={
    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36',
    'Cookie':'__mta=20387463.1534995524246.1534998228834.1535008834516.15"; uuid_n_v=v1; uuid=09407D90A68611E8836EF5378CA03BDF6D7D6F64A44A44D79AB89AC4FFEF8C16; _csrf=35078fdcb3f945837528736f009a0e621a84ee74b5c85a39cf91ab7c385a3949; _lx_utm=utm_source%3Dwww.sogou%26utm_medium%3Dorganic; _lxsdk_cuid=16564dce62bc8-09fea28dc57f5f-5e442e19-144000-16564dce62bc8; _lxsdk=09407D90A68611E8836EF5378CA03BDF6D7D6F64A44A44D79AB89AC4FFEF8C16; __mta=20387463.1534995524246.1534995949923.1534996074539.5"; _lxsdk_s=%7C%7C0'
}

#分析排行榜的url，产生url列表
urls=['http://maoyan.com/board/4?offset={}'.format(str(i)) for i in range(0,100,10)]


#对排行页发送get请求，获取排行榜内电影的链接，拼接成url内容
def get_one_page_url(url):
    try:
        response=requests.get(url,headers=headers)
        #判断是否成功获得相应
        if response.status_code==200:
            soup = BeautifulSoup(response.text, 'lxml')
            links=soup.select('#app > div > div > div.main > dl > dd > a')
            for lin in links:
                url = 'http://maoyan.com' + lin.get('href')
                parse_one_page(url)

        return '无法连接'
    except RequestException:
        return '出现错误'





# 解析每一页的网页
def parse_one_page(url):
    try:
        response=requests.get(url,headers=headers)
        if response.status_code==200:
            soup=BeautifulSoup(response.text,'lxml')
            name=soup.select('body > div.banner > div > div.celeInfo-right.clearfix > div.movie-brief-container > h3')
            classification=soup.select('body > div.banner > div > div.celeInfo-right.clearfix > div.movie-brief-container > ul > li:nth-of-type(1)')
            time=soup.select('body > div.banner > div > div.celeInfo-right.clearfix > div.movie-brief-container > ul > li:nth-of-type(3)')
            introduce=soup.select('#app > div > div.main-content > div > div.tab-content-container > div.tab-desc.tab-content.active > div:nth-of-type(1) > div.mod-content > span')
            for 电影名称, 电影类别, 上映时间, 电影介绍 in zip(name, classification, time, introduce):
                data = {
                '电影名称': 电影名称.get_text(),
                '电影类别': 电影类别.get_text(),
                '上映时间': 上映时间.get_text(),
                '电影介绍': 电影介绍.get_text()
        }
                print(data)
                # save_to_mongo(data)
                write_to_file(data)
        return '无法连接'
    except RequestException:
        return '出现错误'

#存入数据库
def save_to_mongo(result):
    try:
        if db[MONGO_DB].insert(result):
            print('成功存入')
    except Exception:
        print("失败")

#保存成text格式
def write_to_file(content):
    with open('data.txt', 'w+') as f:
        f.write('data.txt')
        f.close()






def main():
    for url in urls:
        get_one_page_url(url)


if __name__=='__main__':
    main()