爬虫笔记-豆瓣

一、根据url获取到网页源码：def asrURL(url)

import urllib.error
import urllib.parse   #解析文件
import urllib.request

def askURL(url):  
    head = {
        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36',
    }
    reque = urllib.request.Request(url,headers=head)
    html_data = ''
    try:
        response = urllib.request.urlopen(reque)
        html_data = response.read().decode('utf-8')
    except urllib.error.URLError as e:
        if hasattr(e,'code'):
            print(e.code)
        if hasattr(e,'reason'):
            print(e.reason)
    return html_data

二、获取到影视信息：def getData(url)

#链接
findLink = re.compile(r'<a href="(.*?)">')   #compile生成或创建正则表达式
#图片
findImgSrc = re.compile(r'<img.*src="(.*?)"',re.S)   #re.S让换行符包含在内
#片名
findTitle = re.compile(r'<span class="title">(.*)</span>')
#评分
findRating = re.compile(r'<span class="rating_num" property="v:average">(.*)</span>')
#评价人数
findJudge = re.compile(r'<span>(\d*)人评价</span>')
#概况
findInq = re.compile(r'<span class="inq">(.*)</span>')
#影片相关内容
findBd = re.compile(r'<p class="">(.*?)</p>',re.S)

def getData(url):
    datalist = []
    for i in range(0, 1):
        base_url = url + str(i*25)    #总数据250条，
        get_html = askURL(base_url)   #保存获取到的网页源码
        bs = BS(get_html,'html.parser')  #type(bs) <class 'bs4.BeautifulSoup'>
        # print(bs.body.contents)
        '''
        print(bs.title)#bs.title的type:element.Tag   标签及其内容
        print(bs.title.string,type(bs.title.string))   #bs.title.string的type:element.NavigableString
        print(bs.a.attrs)   #{'href': 'https://accounts.douban.com/passport/login?source=movie', 'class': ['nav-login'], 'rel': ['nofollow']}
        element.Comment   带注释的数据
        t_list = bs.find_all('a')  #find_all查找所有的'a'标签
        t_list = bs.find_all(re.compile('a'))   #re.compile正则表达式，返回有'a'字母的标签及其内容
        
        kwargs 参数
        '''
        for item in bs.find_all('div',class_='item'):
            print(item)
            data_1 = []
            item = str(item)
            link = re.findall(findLink,item)[0]   #电影链接
            data_1.append(link)

            imgSrc = re.findall(findImgSrc,item)[0]
            data_1.append(imgSrc)

            titles = re.findall(findTitle, item)
            if len(titles) == 2:
                ctitle = titles[0]
                data_1.append(ctitle)
                otitle = titles[1].replace('^/','')
                data_1.append(otitle)   #添加外国名
            else:
                data_1.append(titles[0])
                data_1.append(' ')   #留空

            rating = re.findall(findRating,item)[0]   #评分
            data_1.append(rating)

            judgeNum = re.findall(findJudge,item)[0]   #评价人数
            data_1.append(judgeNum)

            inq = re.findall(findInq,item)  #添加概述
            if len(inq) != 0:
                inq = inq[0].replace("。",'')
                data_1.append(inq)
            else:
                data_1.append(' ')  # 留空

            bd = re.findall(findBd,item)[0]
            bd = re.sub('<br(\s+)?/>(\s+)?',' ',bd)  #去掉br
            bd = re.sub('/','',bd)
            data_1.append(bd.strip())   #去掉前后空格

            datalist.append(data_1)

    return datalist

三、将获取到的信息保存到xlsx文件中

3.1、测试-保存乘法表到xlsx文件中

import xlwt
def save_video_data():
    video_list = []
    book = xlwt.Workbook(encoding='utf-8')   #创建workbook对象
    sheet = book.add_sheet('乘法表',cell_overwrite_ok=True)      #创建工作表      cell_overwrite_ok=True单元格覆盖之前的内容
    # worksheet.write(0,0,'hello')    #写入数据，第一个参数'行'，第二个参数'列'，第三个参数是写入的内容
    for i in range(0,9):
        for j in range(0,i+1):
            # worksheet.write(i, j, "%d * %d = %d "%(i+1,j+1,(i+1)*(j+1)))
            sheet.write(i, j, '{} * {} = {}'.format(j+1,i+1,(i+1)*(j+1)))
    

3.2、保存影视信息

def save_video_data(video_list,write_path):
    book = xlwt.Workbook(encoding='utf-8')   
    sheet = book.add_sheet('电影top250',cell_overwrite_ok=True)
    col = ('电影详情连接','图片链接','影片中文名','影片外国名','评分','评价数','概况','相关信息')   #表头

    for i in range(0,8):
        sheet.write(0,i,col[i])    #将表头信息写入表格里面

    for i in range(0,250):
        data = video_list[i]
        for j in range(0,8):
            sheet.write(i+1,j,data[j])


    book.save(write_path)   #保存数据表
    print('数据保存成功！！！！')

四、保存数据到SQLIT

def init_db(write_db):
    sql = '''create table movie250
          (id integer primary key autoincrement,
          info_link text,
          pic_link text,
          cname varchar ,
          ename varchar ,
          score numeric ,
          rated numeric ,
          instroduction text,
          info text
          
          )'''   #创建数据表
    conn = sqlite3.connect(write_db)
    cursor = conn.cursor()
    cursor.execute(sql)
    conn.commit()     #提交数据
    conn.close()


def save_video_db(data_list,write_db):
    print('开始保存数据！！！')
    init_db(write_db)
    conn = sqlite3.connect(write_db)
    cur = conn.cursor()

    for data in data_list:
        for index in range(len(data)):
            data[index] = '"' + data[index] + '"'
        sql = '''
        insert into movie250(
        info_link,pic_link,cname,ename,score,rated,instroduction,info)
        values(%s)
        ''' % ','.join(data)
        cur.execute(sql)
        conn.commit()
    cur.close()
    conn.close()

    print('数据保存数据库完成！！！')

五、总执行函数

def douban():
    write_path = './data/test_video.xlsx'
    url = 'https://movie.douban.com/top250?start='
    write_db = './test-data/movie250.db'

    data_list = getData(url)
    save_video_data(data_list,write_path)
    save_video_db(data_list,write_db)