“21天好习惯”第一期- 13 爬虫综合

写了一段时间的爬虫，做了一个综合训练，目标是爬取微博热搜前六位，哔哩哔哩热门前六条视频，将其存入sqlite,爬取3G壁纸的六张图存入img文件夹，这个爬虫都是为了和前端项目对接起来，所以都是保存数据之类的。

下面是代码：

from sqlite3.dbapi2 import Cursor
import requests,sqlite3
from lxml import etree


def init_db(dbpath):
    sql1 = '''
    create table if not exists homepage
    (
        num integer primary key autoincrement,
        info text,
        href text,
        spot text
    )
    ''' 

    sql2 = '''
    create table if not exists bilibili
    (
        num integer primary key autoincrement,
        href text,
        title text,
        UP text
    )    
    '''

    if dbpath == 'homepage.db':
        sql = sql1
    elif dbpath == 'bilibili.db':
        sql = sql2
    conn = sqlite3.connect(dbpath)
    cursor = conn.cursor()
    cursor.execute(sql)
    conn.commit()
    conn.close()

def saveDB(all,dbpath):
    init_db(dbpath)
    conn = sqlite3.connect(dbpath)
    cur = conn.cursor()
    for i in all:
        for index in range(len(i)):
            i[index] = '"'+i[index]+'"'
        sql1 = '''
            replace into homepage (num,info,href,spot)
            values(%s)'''%",".join(i)
        sql2 = '''
            replace into bilibili (num,href,title,UP)
            values(%s)'''%",".join(i)
        if dbpath == 'homepage.db':
            sql = sql1
        elif dbpath == 'bilibili.db':
            sql = sql2
        cur.execute(sql)
        conn.commit()
    cur.close()
    conn.close()

def weibo(weiboUrl):
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.81 Safari/537.36 Edg/94.0.992.50',
        'Host':'s.weibo.com',
        'Cookie':'SINAGLOBAL=4435446268469.208.1609664272951; SUBP=0033WrSXqPxfM725Ws9jqgMF55529P9D9WF4UWi41iI4GDPBa6GgZFZx5JpX5KMhUgL.FoMXSK2ReKqRSKB2dJLoIE-LxKqL122LBo2LxKqL122LBo2LxKML12zLB.eLxK-LBoBLB.xy; UOR=,,www.baidu.com; ALF=1667560015; SSOLoginState=1636024016; SCF=Ao0Qf2t3Yfn9GK1VfNg5-2r9kERJPIc-daOhF5J4LexM6jfQzt5Kcj7JAYugXGB7JDI4W3sb-sUYZJY5yn--ERI.; SUB=_2A25Mh86ADeRhGeFK7lMZ8SjEzjiIHXVv9KdIrDV8PUNbmtAKLVDzkW9NQ1qYCpQk7EGD3LFHOZuWS4ZLhxg5CJsB; _s_tentry=login.sina.com.cn; Apache=1400543527553.1685.1636024017310; ULV=1636024017316:25:3:4:1400543527553.1685.1636024017310:1635922688090'
        }
    res = requests.get(weiboUrl, headers=headers).text
    html = etree.HTML(res)
    items = html.xpath('//*[@id="pl_top_realtimehot"]/table/tbody/tr')
    all = []        
    i=0
    for item in items[1:7]:
        if(item.xpath('.//td[1]/text()') != ['•']):
            nums = item.xpath('.//td[1]/text()')
            infos = item.xpath('.//td[2]/a/text()')
            hrefs = item.xpath('.//td[2]/a/@href')
            spots = item.xpath('.//td[2]/span/text()')
            for i in range(len(nums)):
                num = nums[i]
                info = infos[i]
                href = hrefs[i]
                spot = spots[i]
            all.append([num,info,href,spot])
            i+=1
    return all

def bilibili(bilibiliUrl):
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.54 Safari/537.36 Edg/95.0.1020.30',
        'Cookie':"_uuid=FB826D72-0DD4-6529-66C0-B41DA61A7E5159617infoc; buvid3=C44CFAFD-DABB-4176-B4A4-72C3089AF23C143085infoc; sid=blhheuhy; rpdid=|(JYYJYY|)JR0J'uY|Rmmlmm~; LIVE_BUVID=AUTO7316053400914985; fingerprint3=06575671fef0d32400ca8fc17ec0c1a1; buivd_fp=C44CFAFD-DABB-4176-B4A4-72C3089AF23C143085infoc; buvid_fp=C44CFAFD-DABB-4176-B4A4-72C3089AF23C143085infoc; blackside_state=1; fingerprint_s=4e211682af48f357f684155aad630a52; DedeUserID=174976202; DedeUserID__ckMd5=e0601685fccfe74f; CURRENT_BLACKGAP=1; CURRENT_QUALITY=64; fingerprint=9276324110ba055e6f9e7391e8e845b4; buvid_fp_plain=87CE8CFB-FFAF-40F5-B7EC-8E8A8D73C6A0167631infoc; SESSDATA=122bc1da,1647848602,f8e50*91; bili_jct=cda1b5d8bed4824c7bdb2f00518d5531; CURRENT_FNVAL=976; PVID=1; bp_video_offset_174976202=584236026094520654; bp_t_offset_174976202=584640998555479923; bsource=search_baidu; innersign=0",
        'referer':'https://www.bilibili.com/'
    }
    res = requests.get(bilibiliUrl, headers=headers).text
    html = etree.HTML(res)
    items = html.xpath('//*[@id="app"]/div/div[2]/div[1]/div[2]/div[1]/div[position()>0 and position()<7]')
    all = []
    j=1      
    for item in items[0:6]:             
        hrefs = item.xpath('.//div/a/@href')
        titles = item.xpath('.//div/a/img/@alt')
        UPs = item.xpath('.//div/a/div/p[2]/text()')
        for i in range(len(hrefs)):
            num = str(j)
            j+=1
            href = hrefs[i]
            title = titles[i]
            UP = UPs[i]        
        all.append([num,href,title,UP])    
    return all



def picture():
    headers = {
        'Cookie': 'Hm_lvt_c8263f264e5db13b29b03baeb1840f60=1632291839,1632373348; Hm_lpvt_c8263f264e5db13b29b03baeb1840f60=1632373697',
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.82 Safari/537.36'}
    url = f'https://www.3gbizhi.com/tupian/fjtp.html'
    response = requests.get(url, headers=headers)
    html = etree.HTML(response.text)
    href_list = html.xpath('//div[@class="contlistw mtw"]//ul[@class="cl"]/li/a/@href')
    for href in href_list:
        res = requests.get(href, headers=headers)
        html_data = etree.HTML(res.text)
        img_url_list = html_data.xpath('//div[@class="picimglist pos"]/ul/li/a/img/@src')
        num = 0
        for img_url in img_url_list:
            img_url = ''.join(img_url.split('thumb_200_0_'))
            result = requests.get(img_url, headers=headers).content
            with open('static/img/' + str(num) + '.jpg', 'wb')as f:
                f.write(result)
            num += 1
            if num==6:
                break
            print(f'正在下载第{num}张！！！！')
        break



if __name__ == "__main__":
    weiboUrl = 'https://s.weibo.com/top/summary'
    dbpath1 = 'homepage.db'
    all1 = weibo(weiboUrl)
    saveDB(all1,dbpath1)

    bilibiliUrl = 'https://www.bilibili.com/'
    dbpath2 = 'bilibili.db'
    all2 = bilibili(bilibiliUrl)
    saveDB(all2,dbpath2)

    picture()