Python3用requests实现京东手机信息的爬取

用requests实现京东手机（手机名称，手机价格，手机图片）信息的爬取

import requests
from lxml import etree
import time
import os
import pymysql

#选择页码爬取商品数据
def jdphone_spider(url,beginPage,endPage):
    # 建立一个MySQL连接
    database = pymysql.connect(host="127.0.0.1", user="root", passwd="960505", db="jdPhone", charset='utf8')
    # 创建游标
    cursor = database.cursor()

    for page in range(beginPage,endPage):
        pn = page*2 - 1 #奇数递增抓取数据
        print('正在抓取第{}页'.format(page))
        #带页码的url
        full_url = url+'&page='+str(pn)

        time.sleep(2)
        #读取页面
        load_page(database,cursor,full_url)

     # 关闭游标
    cursor.close()
    # 关闭数据库连接
    database.close()


def load_page(database,cursor,url):
    #定义请求头
    headers = {
        'user-agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36 Edg/86.0.622.691'
    }
    #发送请求获取响应数据
    response = requests.get(url=url,headers=headers)
    page_text = response.text
     #数据解析：src的属性值  alt属性
    tree = etree.HTML(page_text)
    li_list = tree.xpath('//*[@id="J_goodsList"]/ul/li')

    #创建一个文件夹
    if not os.path.exists('./jdPicLibs'):
        os.mkdir('./jdPicLibs')

    for li in li_list:
        #解析获取手机价格
        phone_price = li.xpath('./div/div[3]/strong//text()')
        phone_price = phone_price[1]+phone_price[2]
        #解析详情页，获取手机图片和名称
        detail_url = 'https:'+li.xpath('./div/div[1]/a/@href')[0]
        detail_text = requests.get(url=detail_url,headers=headers).text

        #解析详情页
        tree = etree.HTML(detail_text)
        #解析商品名称
        phone_name = tree.xpath('/html/body/div[6]/div/div[2]/div[1]/text()')
        phone_name = ''.join(phone_name).strip()
        print(phone_name)
        #解析下载图片
        img_url = 'https:'+tree.xpath('//div[@id="spec-n1"]/img/@data-origin')[0]
        #img_name = img_url.split('/')[-1]
        #通用处理中文乱码的解决方案
        #img_name = img_name.encode('iso-8859-1').decode('gbk')
        img_data = requests.get(url=img_url,headers=headers).content
        # img_path = 'jdPicLibs/'+img_name
        # with open(img_path,'wb') as fp:
        #     fp.write(img_data)
        #     print(img_name,'下载成功！！！')
        save_phone_info(database,cursor,phone_name,phone_price,img_data)


def save_phone_info(database,cursor,phone_name,phone_price,phone_img):
    try:
        # 注意使用Binary()函数来指定存储的是二进制
        sql = "INSERT INTO phone VALUES  (null,%s, %s, %s);"
        args = (phone_name, phone_price, phone_img)
        cursor.execute(sql, args)
        database.commit()
    except Exception as e:
        print(e)
        database.rollback()
   
    

#抓取手机商品信息  分页抓取
if __name__ == '__main__':
    beginPage = int(input('请输入起始页：'))
    endPage = int(input('请输入结束页:'))
    url = 'https://search.jd.com/Search?keyword=手机&enc=utf-8'
    print('开始爬取......')
    start_time = time.time()
    jdphone_spider(url,beginPage,endPage)
    end_time = time.time()
    print('爬取完毕！！！\n'+(end_time-start_time))