利用python的BeautifulSoup4库爬取豆瓣top250基本电影信息

利用python的BeautifulSoup4库爬取豆瓣top250基本电影信息
存在.txt文件中

```python
import requests
import bs4
import re
def open_url(url):
    #那么User-Agent到底是什么呢？User-Agent会告诉网站服务器，访问者是通过什么工具来请求的，如果是爬虫请求，一般会拒绝，如果是用户浏览器，就会应答
    headers={'user-agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.122 Safari/537.36'}
    res=requests.get(url,headers=headers)
    return res
def find_movies(res):
    soup=bs4.BeautifulSoup(res.text,'html.parser')
    # 序号
    number = []
    targets = soup.find_all("div", class_='pic')
    for each in targets:
        number.append(each.em.text+' ')
    # 电影名
    movies = []
    targets = soup.find_all("div", class_='hd')
    for each in targets:
         movies.append(each.a.span.text+' ')
    # 地址
    address = []
    targets = soup.find_all("div", class_='hd')
    for each in targets:
        address.append(each.a.get('href')+' ')  # 这里得到属性值里的地址
    # 图片
    picture = []
    targets = soup.find_all("div", class_='pic')
    for each in targets:
        picture.append(each.a.img.get('src')+' ')  # 这里得到属性值里的地址

#评分
ranks=[]
targets = soup.find_all("span", class_='rating_num')
for each in targets:
    ranks.append(each.text)

#类型名

kinds = []
targets = soup.find_all("div", class_='bd')
for each in targets:
    kinds.append(each.p.text.split('/')[-1])  # 将p标记下的最后一块切出来  [n]:表示选取第n个分片
    '''
    results=[]
    length=len(movies)-5
    for i in range(length):
        results.append(number[i]+movies[i]+address[i]+picture[i]+ranks[i]+kinds[i+1]+'\n')
        #results.append(movies[i] + ranks[i]+'\n')
    return results

#找出有多少页面

def find_depth(res):
    soup=bs4.BeautifulSoup(res.text,'html.parser')
    depth=soup.find('span',class_='next').previous_sibling.previous_sibling.text
    return int(depth)
def main():
    host='https://movie.douban.com/top250'
    res=open_url(host)  #这端代码的作用就是打开https://movie.douban.com/top250这个网站，返回一个response对象
    depth=find_depth(res)-2

    info=[]
    for i in range(depth):
        url=host+'?start='+str(25*i)+'&filter='
        res=open_url(url)
        info.extend(find_movies(res))

        with open('E:\\pythonimg\\电影.txt','w',encoding='utf-8') as f:
            for each in info:
                f.write(each)

if __name__=="__main__":
    main()