Python爬取网页数据

爬取商品数据

import requests
from lxml import etree
import urllib.parse
import xlwt

def get_html(url):
    html = requests.get(url[0])
    return html


def get_html_link(link):
    link_seq = etree.HTML(link.content)
    title_list = link_seq.xpath('//*[@class="prod-name"]/text()')
    length = len(title_list)
    product_material_list = link_seq.xpath('//*[@class="desc"]/text()')
    # price_list = link_seq.xpath('//*[@class="price"]//div[1]/text()')
    # for i in price_list:
    #     print(i)

    kind_list = link_seq.xpath('//*[@class="prod-name"]/text()')
    kind_list_1 = []
    for o in kind_list:
        o1 = o[::-1].split('，', 1)[-1][::-1][-2:]
        kind_list_1.append(o1)
    # for j in range(0, length):
    #     print(title[j])
    #     print(product[j])

    wb = xlwt.Workbook()
    ws = wb.add_sheet("商品详情")
    ws.write(0, 0, '商品名称')
    ws.write(0, 1, '商品材质')
    ws.write(0, 2, '商品类别')
    ws.write(0, 3, '商品价格')
    ws.write(['商品名称', '商品材质', '商品类别', '商品价格', ''])

    for i in range(0, 255):
        # f.write(title_list[i] + ""*20 + product_material_list[i] + '\n')
        ws.write(i+1, 0, title_list[i])
        ws.write(i+1, 1, product_material_list[i])
        ws.write(i+1, 2, kind_list_1[i])
        # ws.write(3, i+1, price_list[i])
        # ws.write([title_list[i], product_material_list[i], kind_list_1[i], price_list[i]])

    wb.save('商品数据.xls')

def funcall(url):

    link = get_html(url)
    get_html_link(link)

def main():
    word1 = '系列'
    word2 = '珠宝系列'
    word3 = '系列'
    word1 = urllib.parse.quote(word1)
    word2 = urllib.parse.quote(word2)
    word3 = urllib.parse.quote(word3)
    url = [
            'https://www.cartier.cn/zh-cn/' + word1 + '/' + word2 + '/' + word3 + '.viewall.html'
            ]
    funcall(url)


if __name__ == "__main__":
    main()