python爬虫——利用requests库re正则表达式获取购物网站指定商品信息

学习和练习使用，简单爬取信息内容。

经过对taobao url链接的分析，比如：

我们搜索“耳机”就会看到这样的url：

https://s.taobao.com/search?q=%E8%80%B3%E6%9C%BA&imgfile=&commend=all&ssid=s5-e&search_type=item&sourceId=tb.index&spm=a21bo.2017.201856-taobao-item.1&ie=utf8&initiative_id=tbindexz_20170306

斜体加粗部分是我们检索的商品名字。

再看看第2页，会发现url最后面以‘&s=44’结束，第三页则是‘&s=88’......依次类推，每一页显示44个商品。

下来给出总体代码：

代码依旧是3大块，老步骤。

import requests
import re

def getHTMLText(url):
    try:
        r = requests.get(url, timeout = 30)
        r.raise_for_status()
        r.encoding = r.apparent_encoding
        return r.text
    except:
        return ""

def parsePage(ulist, html):
    try:
        pricelists = re.findall(r'\"view_price\"\:\"[\d\.]*\"', html)
        titlelists = re.findall(r'\"raw_title\"\:\".*?\"', html)
        locallists = re.findall(r'\"item_loc\"\:\".*?\"', html)
        salelists = re.findall(r'\"view_sales\"\:\".*?\"', html) 
        for i in range(len(pricelists)):
            price = eval(pricelists[i].split(':')[1])
            title = eval(titlelists[i].split(':')[1])
            local = eval(locallists[i].split(':')[1])
            sale = eval(salelists[i].split(':')[1])
            ulist.append([price, sale, local, title])
    except:
        return ""

def printGoodsList(ulist):
    tplt = "{0:^4}\t{1:^4}\t{2:^10}\t{3:^8}\t{4:<16}"
    with open ('taobaoresult.txt', 'a', encoding = 'utf-8') as f:
        f.write(tplt.format("序号", "价格", "销售量", "发货地", "商品名称")+'\n')
        f.close()
    print(tplt.format("序号", "价格", "销售量", "发货地", "商品名称"))
    count = 0
    for g in ulist:
        count = count + 1
        with open ('taobaoresult.txt', 'a', encoding = 'utf-8') as f:
            f.write(tplt.format(count, g[0], g[1], g[2], g[3]) + '\n')
            f.close()
        print(tplt.format(count, g[0], g[1], g[2], g[3]))

def main():
    goods = input("请输入需要检索的商品：")
    depth = 3
    start_url = 'http://s.taobao.com/search?q=' + goods
    infoList = []
    for i in range(depth):
        try:
            url = start_url + '&s=' + str(44*i)
            html = getHTMLText(url)
            parsePage(infoList, html)
        except:
            continue
    printGoodsList(infoList)
    
    while True:
        comd = input("是否继续检索（是/否）：")
        if comd in ["是"]:
            main()
        else:
            break
main()

运行实例：

打开保存成.exe应用程序

内容存入指定目录txt文件中。