封装 python爬虫 爬取妹子图到mysql数据库

import requests
from lxml import etree
from mysql import MYSQL

1. 声明一个lianjia函数

def lianjia(url,headers):

    # 2.给于url路径

    # url = 'http://bj.lianjia.com/ershoufang/pg2/'

    for i in range(1,3,30):
        url = url.format(i)
        # 3.发送get请求
        req = requests.get(url,headers=headers)

        # 4.获取页面
        html = req.text
        # print(html)

        # 5.转换页面类型
        html = etree.HTML(html)

        url=html.xpath('//ul[@class="sellListContent"]//div[@class="title"]/a/@href')
        # print(url)
        for detaurl in url:
            xiangqing(detaurl)

def xiangqing(url):
    # print(url)
    # 9.发起详情页url访问请求
    req = requests.get(url)
    # print(req)

    # 10.获取页面
    html = req.text
    # print(html)

    # # 11.转换页面类型
    html = etree.HTML(html)

    # 发起详情页url访问请求
    req = requests.get(url)
    # print(req)

    xiaoqu=html.xpath('//div[@class="aroundInfo"]//a[@target="_blank"]/text()')[0]
    # print(url)
    # print(xiaoqu)

    # dizhi = html.xpath('//div[@class="aroundInfo"]//a[@target="_blank"]/text()')[2]
    # # print(url)
    # print(dizhi)

    pingmi = html.xpath('//div[@class="houseInfo"]//div[@class="mainInfo"]/text()')[2]
    # print(pingmi)
    jiage = html.xpath('//div[@class="overview"]//span[@class="total"]/text()')
    # print(jiage)

    # sql 语句             表名  （字段名）相应字段传入的数据
    sql = 'insert into lianjia(xiaoqu,pingmi,jiage) values (%s,%s,%s)'

    # 根据导入的sql包，给拼接完的url赋值给deta
    deta = (xiaoqu,pingmi,jiage)
    lanj.execute_modify_mysql(sql, deta)

if __name__ == "__main__":
    lanj = MYSQL()
    url = "https://bj.lianjia.com/ershoufang/huilongguan2/pg2/"


    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36'
    }
    lianjia(url,headers)