网络爬虫实例

"""淘宝商品信息定向爬取"""
import requests
import re




def get_html_text(url):
    try:
        r = requests.get(url, timeout=30)
        r.raise_for_status()
        r.encoding = r.apparent_encoding
        return r.text
    except:
        return ''




def parse_page(ilt, html):
    try:
        plt = re.findall(r'\"view_price\"\:\"[\d\.]*\"', html)
        tlt = re.findall(r'\"raw_title\":\".*?\"', html)
        for i in range(len(plt)):
            price = eval(plt[i].split(':')[1])  # eval函数将最外面""去掉。
            title = eval(tlt[i].split(':')[1])  # 使用:分割键值对，获得后面的部分。
            ilt.append([price, title])
    except:
        print("parse_page none")




def print_goods_list(ilt):
    tplt = "{:4}\t{:8}\t{:16}"  # 槽函数，定义长度分别为4, 8, 16
    print(tplt.format("序号", "价格", "商品名称"))  # 表头序号价格商品名称
    count = 0
    for g in ilt:
        count += 1
        print(tplt.format(count, g[0], g[1]))




def main():
    goods = '书包'
    url = 'http://s.taobao.com/search?q=' + goods
    info_list = []
    try:
        html = get_html_text(url)
        parse_page(info_list, html)
    except:
        print("NNNO")
    print_goods_list(info_list)




if __name__ == "__main__":
    main()












"""股票数据定向爬虫


新浪股票  http://finance.sina.com.cn/stock/
百度股票  https://gupiao.baidu.com/stock/ 
东方财富  http://quote.eastmoney.com/stocklist.html """
import requests
from bs4 import BeautifulSoup
import re
import traceback




def get_html_text(url):
    try:
        r = requests.get(url, timeout=30)
        r.raise_for_status()
        r.encoding = r.apparent_encoding
        return r.text
    except:
        return ""




def get_stock_list(lst, stock_url):
    html = get_html_text(stock_url)
    soup = BeautifulSoup(html, 'html.parser')  # BeautifulSoup解析，找到每个标签。
    a = soup.find_all('a')
    for i in a:
        try:
            href = i.attrs['href']
            lst.append(re.findall(r'[s][hz]\d{6}', href))  # 得到6位数股票代码。
        except:
            continue




def get_stock_info(lst, stock_url, f_path):
    for stock in lst:
        url = stock_url + str(stock) + '.html'
        html = get_html_text(url)
        try:
            if html == '':
                continue
            info_dict = []
            soup = BeautifulSoup(html, 'html.parser')
            stock_info = soup.find('div', attrs={'class': 'stock-bets'})


            name = stock_info.find_all(attrs={'class': 'bets-name'})[0]
            info_dict.update({'股票名称': name.text.split()[0]})


            key_list = stock_info.find_all('dt')
            value_list = stock_info.find_all('dd')
            for i in range(len(key_list)):
                key = key_list[i].text
                val = value_list[i].text
                info_dict[key] = val


            with open(f_path, 'a', encoding='utf-8') as f:
                f.write(str(info_dict) + '\n')
        except:
            traceback.print_exc()
            continue




def main():
    stock_list_url = 'http://quote.eastmoney.com/stocklist.html'
    stock_info_url = 'http://gupiao.baidu.com/stock/'
    output_file = r'C:\Users\poseidon\Desktop\text'
    s_list = []
    get_stock_list(s_list, stock_list_url)
    get_stock_info(s_list, stock_info_url, output_file)




if __name__ == "__main__":
    main()