P2P小爬虫范例

# -*- coding: UTF-8 -*-
import urllib
import requests
import re
import sys
import chardet
import pandas as pd
from bs4 import BeautifulSoup
import sys
reload(sys)
sys.setdefaultencoding('utf-8')

user_agent = 'User-Agent: Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Maxthon 2.0)'
headers = {'User-Agent':user_agent}
#定义函数,得到每日报的链接,并以列表形式返回
def GetHtml( url):
    page = urllib.urlopen(url)
    contex = page.read()
    return contex

def get_newsurl():
    newsurl=[]
    url1='http://www.p2p001.com/licai/index/id/147/p/'
    num=1
    url2='.html'
    while num<=1:
        url=url1+str(num)+url2
        try:
            r1=requests.get(url,headers=headers)
        except:
            print ('wrong %s' % url)
        else:
            r1.encoding = r1.apparent_encoding
            #print r1.text
            s1=BeautifulSoup(r1.text,'lxml')
            for x in s1.find_all(href=re.compile('licai/shownews')):
                newsurl.append(x['href'])
            num=num+1
    return newsurl
#定义函数,得到的数据,以字典形式返回
def get_info():
    url='http://www.p2p001.com'
    date=[]
    zonghe=[]
    one=[]
    one_three=[]
    three_six=[]
    six_twelve=[]
    twelve_most=[]
    backurl=[]
    backurl = get_newsurl()
    print len(backurl)
    for y in range(0, 1):
        try:
            main_url=url+backurl[y]
            r2=requests.get(main_url,headers=headers)
        except:
                print ('wrong %s' % main_url)
        else:
            s2=BeautifulSoup(r2.text,'lxml')
            #datetemp = s2.find(text=re.compile('统计日期'))[5:]
            date.append(s2.find(text=re.compile(u'统计日期'))[5:])
            print "---date---"
            #date = repr(date).decode("unicode–escape")
            print date
            print "---date---"
            rate=s2.find_all('td')
            print rate
            print "---begin to print rate---"
            for i in rate:
                print i.string
            print "---begin to print rate---"
            #print repr(rate[2].string).decode("unicode–escape")
            print "-----rate[2]---------"
            #zonghe.append(repr(rate[2].string).decode("unicode–escape"))
            zonghe.append(rate[10].string)
            one.append(rate[11].string)
            one_three.append(rate[12].string)
            three_six.append(rate[13].string)
            six_twelve.append(rate[14].string)
            twelve_most.append(rate[15].string)
    p={'Date':date,
       '综合':zonghe,
       '1个月':one,
       '1-3个月':one_three,
       '3-6个月':three_six,
       '6-12个月':six_twelve,
       '12个月及以上':twelve_most}
    return p
#pandas存储数据
#print sys.getfilesystemencoding()
#print 'Html is encoding by : %',chardet.detect(GetHtml('http://www.p2p001.com'))
p=pd.DataFrame(get_info())
p.to_csv('e://rate1.csv',index=False,encoding='gbk', #'utf_8 with BOM',
           columns=['Date','综合','1个月','1-3个月','3-6个月','6-12个月','12个月及以上'],
           header=['Date','综合','1个月','1-3个月','3-6个月','6-12个月','12个月及以上'])