Python爬虫的学习和实现-双福菜价爬取

简单数据的爬取和保存

通过在网上的学习，运用requests库和bs4的Beautifulsoup库实现了对URL的简单爬取。
爬取链接：双福菜价数据
实现代码：

import requests
from bs4 import BeautifulSoup
import bs4
import xlwt

def getHTMLText(url):
    try:
        r = requests.get(url, timeout = 30)
        r.raise_for_status()
        r.encoding = r.apparent_encoding
        return r.text
    except:
        return ""

def fillUnivlist(ulist, html):
    soup = BeautifulSoup(html,"html.parser")
    num = 0
    for p in soup.find(attrs={'class':'pri_k'}).children:
        if isinstance(p, bs4.element.Tag):
            spans = p('span')
            ulist.append([spans[1].string, spans[5].string, spans[6].string])

def printUnivList(ulist):
    for i in range(len(ulist)):
        u = ulist[i]
        print("{:^10}\t{:^10}\t{:^10}".format(u[0],u[1],u[2]))

def write_excel(ulist):
    book = xlwt.Workbook(encoding = 'utf-8', style_compression=0)
    sheet = book.add_sheet('dede',cell_overwrite_ok=True)
    for i in range(len(ulist)):
        u = ulist[i]
        for j in range(len(u)):
            print(u[j])
            sheet.write(i,j,u[j])
    book.save('g:\\test.xls')



def main():
    uinfo = []
    url2 = 'http://www.vegnet.com.cn/Price/List?marketID=497&year=2017&month=9&day=1'
    html = getHTMLText(url2)
    num = fillUnivlist(uinfo, html)
    write_excel(uinfo)
    #printUnivList(uinfo)

main()

补充：xlwt库的使用方法