python爬虫爬取豆瓣第一页

   嗨，又到爬虫学习的日子了宝子们有没有好好练习接下来我将带领大家学习爬虫爬取豆瓣第一页

老样子分三步一、请求   二、解析  三、存储

一、请求网页链接

1.网页链接

url = 'https://movie.douban.com/top250'

2.请求

def main(url):
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36'
    }
    con = requests.get(url=url, headers=headers)
    con.encoding = 'utf-8'
    print(con)

    return con.text

二、数据解析

1.获取HTML前端网页后找到我们要的代码

2.进行解析

我们这里用的是re正侧表达式和Beatifulsoup进行解析找到我们需要的代码

def gatdata(a):
    dataList = []
    data = []
    html = str(a)
    sup = BeautifulSoup(html, "html.parser")
    for item in sup.find_all("div", "item"):
        item = str(item)
        time.sleep(0.1)
        title = re.findall(r'<span class="title">(.*)</span>', item)[0]
        data.append(title)
        src = re.findall(r'<a href=(.*?)>', item)[0]
        data.append(src)
    dataList.append(data)
    print(dataList)
    return data

三、进行存储

我们用xlwt进行存储到execl中

def datasave(datalist, savepath):
    workbook = xlwt.Workbook(encoding="UTF-8", style_compression=0)  ##style_compression=0不压缩
    worksheet = workbook.add_sheet("豆瓣电影top25", cell_overwrite_ok=True)  # cell_overwrite_ok=True再次写入数据覆盖
    column = ("电影详情链接", "图片链接", "影片中文名", "影片外国名", "评分", "评价数", "概况", "相关信息")  ##execl项目栏
    p = 0
    for i in range(0, 8):
        worksheet.write(0, i, column[i])  # 将column[i]的内容保存在第0行，第i列
    for i in range(0, 25):
        datalist[i]
        for j in range(0, 2):
            worksheet.write(i + 1, j, datalist[p])
            p = p + 1

    workbook.save(savepath)

四、完整代码

import re
import xlwt
from bs4 import BeautifulSoup
import time
import requests

url = 'https://movie.douban.com/top250'


def main(url):
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36'
    }
    con = requests.get(url=url, headers=headers)
    con.encoding = 'utf-8'
    print(con)

    return con.text


def gatdata(a):
    dataList = []
    data = []
    html = str(a)
    sup = BeautifulSoup(html, "html.parser")
    for item in sup.find_all("div", "item"):
        item = str(item)
        time.sleep(0.1)
        title = re.findall(r'<span class="title">(.*)</span>', item)[0]
        data.append(title)
        src = re.findall(r'<a href=(.*?)>', item)[0]
        data.append(src)
    dataList.append(data)
    print(dataList)
    return data


def datasave(datalist, savepath):
    workbook = xlwt.Workbook(encoding="UTF-8", style_compression=0)  ##style_compression=0不压缩
    worksheet = workbook.add_sheet("豆瓣电影top25", cell_overwrite_ok=True)  # cell_overwrite_ok=True再次写入数据覆盖
    column = ("电影详情链接", "图片链接", "影片中文名", "影片外国名", "评分", "评价数", "概况", "相关信息")  ##execl项目栏
    p = 0
    for i in range(0, 8):
        worksheet.write(0, i, column[i])  # 将column[i]的内容保存在第0行，第i列
    for i in range(0, 25):
        datalist[i]
        for j in range(0, 2):
            worksheet.write(i + 1, j, datalist[p])
            p = p + 1

    workbook.save(savepath)


if __name__ == '__main__':
    savepath = ".\\豆瓣电影top25.xls"
    a = main(url)
    datalist = gatdata(a)
    datasave(datalist, savepath)

效果如下：

接下来请宝子们将剩下的信息从豆瓣网上爬取下来，进行练习，下一期我们将爬豆瓣多页