python爬虫7：完整的一个爬虫小demo

最新推荐文章于 2022-11-03 17:49:53 发布

goxingman

最新推荐文章于 2022-11-03 17:49:53 发布

阅读量1.4k

点赞数

分类专栏： python 文章标签： python 爬虫开发语言

本文链接：https://blog.csdn.net/goxingman/article/details/121967376

版权

python 专栏收录该内容

8 篇文章 0 订阅

订阅专栏

from bs4 import BeautifulSoup
import re
import urllib.request
import xlwt
import sqlite3
import ssl

ssl._create_default_https_context = ssl._create_unverified_context
def main():
#网址就不说了，避免侵权
    baseUrl = "xxxxxx"
    data = getData(baseUrl)
    # saveDataToExcel(data)
    saveDataToDB(data)

def getData(baseUrl):
    headers = {"User-Agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 13_2_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.3 Mobile/15E148 Safari/604.1",
                "cookie": "__51uvsct__JUTBIqXUbBvBL3hd=1; __51vcke__JUTBIqXUbBvBL3hd=f89fdbc5-1dbf-531a-8d48-608ad53241e3; __51vuft__JUTBIqXUbBvBL3hd=1639549481152; XSRF-TOKEN=eyJpdiI6ImFIK2VFbDg2NXQzbE9lTUZmNGFWMFE9PSIsInZhbHVlIjoiQ1NYU2dZVTV4NlpSM1FNOUVVdHFoZVB1N1daS2NoXC9hZUhOUThhcVZBbnpoYlBNTlwvZWtKUHNoNUdBdWxueTUwIiwibWFjIjoiMzMyMzE1MjRhMDU2MTIzMDBhM2UzZTQwMzJmMTk5YmM3NjgwMTJhMzM5MzZhZjcxM2I0YmRiZTYwNzBjNzM4ZCJ9; 91btlife_session=eyJpdiI6InEybEZiNUFUc3FwT1VMaGE1V3F4Qnc9PSIsInZhbHVlIjoiQ2M0UUY0ZFRDZGFQK3VEU3NyXC9VV1lsZ2dUZWQ2cHd2dVlMWVU4TktMYVZrN3RhTldPUXg4bmpub0ZSeHBmZkMwNHNweEd1Z09yMGQ1OTNld1dEYUdJY3FJOXZPTmpObUFcLzZxQjQ1MHl4Tlp6YW9SYXFBRzhGOGpLK1B6NnRBayIsIm1hYyI6ImVjYjM3OTFmNWI1NTMzMzEzM2Y0MzVmYzM1ZjNkNTNiMjM3ZmEyOTVlYmY5NjhiZDA5ODM5ZDkwNTkwM2RkMzgifQ%3D%3D; __vtins__JUTBIqXUbBvBL3hd=%7B%22sid%22%3A%20%222761075f-9fcf-5878-a827-2fecd1e51f8d%22%2C%20%22vd%22%3A%202%2C%20%22stt%22%3A%2083220%2C%20%22dr%22%3A%2083220%2C%20%22expires%22%3A%201639551364368%2C%20%22ct%22%3A%201639549564368%7D"
               }
    request = urllib.request.Request(url=baseUrl, headers=headers)
    rsp = urllib.request.urlopen(request)

    soup = BeautifulSoup(rsp.read().decode('utf-8'), "html.parser")
    find_all = soup.find_all(class_="main-content")
    res={}
    titleList = {}
    detailList = []
    i=0
    #一级分类
    titles = re.findall(r'style="margin-right: 7px;"></i>(.*?)</h4>', str(find_all[0]))
    for title in titles:
        title_temp={i:title}
        titleList.update(title_temp)

        detail_htmls = re.findall('</i>'+title+'</h4>[\s\S]+?<h4 class="text-gray">', str(find_all[0]))
        if len(detail_htmls)>0:
            dts=re.findall('<strong>(.*?)</strong>',detail_htmls[0])
            urls=re.findall(r'data-original-title="(.*?)"',detail_htmls[0])
            for j in range(0, len(dts)):
                dlt = []
                dlt.append(i)
                dlt.append(dts[j])
                dlt.append(urls[j])
                detailList.append(dlt)
        i+=1
    res.update({"titleList":titleList})
    res.update({"detailList":detailList})
    return res

def saveDataToExcel(data):
    workbook = xlwt.Workbook(encoding="utf-8")
    sheet = workbook.add_sheet("sheet1")
    i=0
    for title in data:
        sheet.write(i,0,title)
        i+=1
    workbook.save("te.xls")

def saveDataToDB(data):
    connect = sqlite3.connect("test.db")
    c = connect.cursor()  #获取游标
    # i=1
    # for title in data:
    #     sql = r"insert into t1 (id,title) values (%d,'%s');"%(i,title)
    #     i+=1
    #     print(sql)
    #     c.execute(sql)
    titleList = data['titleList']
    detailList = data['detailList']
    for key,value in titleList.items():
        sql = r"insert into title (id,title) values (%d,'%s');" %(key, value)
        c.execute(sql)
    for val in detailList:
        sql = r"insert into detail (`name`,titleId,url) values ('%s',%d,'%s');" %(val[1], val[0],val[2])
        c.execute(sql)
    connect.commit()
    c.close()


if __name__ == "__main__":
    main()