数据保存！！！Python 爬取网页数据后，三种保存格式---保存为txt文件、CSV文件和mysql数据库

    Python爬取网站数据后，数据的保存方式是大家比较关心的意一件事情，也是为接下来是否能够更简便的处理数据的关键步骤。下面，就Python爬取网页数据后的保存格式进行简单介绍。三种保存格式为txt格式、CSV格式和数据库格式。

    首先，保存为txt格式。话不多说，直接上代码！    

# -*- coding: utf-8 -*-
import requests
import json
import html
import urllib
import sys
import re
import random
import time
from threading import Timer
from bs4 import BeautifulSoup


reload(sys)
sys.setdefaultencoding('utf-8')
headers ={'user-agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.106 BIDUBrowser/8.7 Safari/537.36'}


def get_html1(i):
    url = 'https://www.ppmoney.com/StepUp/List/-1/{}/fixedterm/true/false?_={}'
    html = requests.get(url.format(i,random.randint(1501050773102,1501051774102)),headers=headers)
    return html.content
def get_data1(html):
    data1 = json.loads(html)
    data = data1['PackageList']['Data']
    for i in data:
        #产品名称，利率，金额
         print i['name'],'\t',i['profit'],'\t',i['investedMoney']
       with open('d:PPmonenyshengxinbao9.6.txt','a') as f:
           f.write(i['name']+'\t'+str(i['profit'])+'\t'+str(i['investedMoney'])+'\n'
for i in range(1,10):
    get_data1(get_html1(i))   

执行代码后，生成文件打开后显示如下：

    2.保存为CSV格式。

# -*- coding: utf-8 -*-
import requests
import pandas as pd
import numpy as np
import json
import html
import urllib
import sys
import re
import random
import time
from threading import Timer
from bs4 import BeautifulSoup


reload(sys)
sys.setdefaultencoding('utf8')
headers ={'user-agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.106 BIDUBrowser/8.7 Safari/537.36'}

def get_html1(i):
    url = 'https://www.ppmoney.com/StepUp/List/-1/{}/fixedterm/true/false?_={}'
    html = requests.get(url.format(i,random.randint(1501050773102,1501051774102)),headers=headers)
    ceshi1=html.content
    data = json.loads(ceshi1)
    return(data['PackageList']['Data'])


data_ceshi=pd.DataFrame([])
html_list=[]
for i in range(100):
    html_list.append(get_html1(i))
for i,heml_avg in enumerate(html_list):
    tmp=pd.DataFrame(heml_avg)
    tmp["page_id"]=i
    data_ceshi=data_ceshi.append(tmp)


print data_ceshi
data_ceshi.to_csv('e:/data.csv',encoding='gbk')

保存后，结果如下：

3.保存到数据库。

# -*- coding: utf-8 -*-
import requests
import pandas as pd
import numpy as np
import json
import html
import urllib
import sys
import re
import random
import MySQLdb
import time
from threading import Timer
from bs4 import BeautifulSoup


reload(sys)
sys.setdefaultencoding('utf8')
headers ={'user-agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.106 BIDUBrowser/8.7 Safari/537.36'}

db = MySQLdb.connect(host = 'localhost',port = 3306,user = 'zhouliye',passwd = '123456zz',db = 'abbc',charset='utf8')
print '连接上数据库了！'
cursor = db.cursor()


cursor.execute("DROP TABLE IF EXISTS shengxb")
sql = """CREAtE TABLE SHENGXB(
          beginTime DATETIME,
          endTime DATETIME,
          investedMoney float,
          name CHAR(50))"""


cursor.execute(sql)


def get_html1(i):
    url = 'https://www.ppmoney.com/StepUp/List/-1/{}/fixedterm/true/false?_={}'
    html = requests.get(url.format(i,random.randint(1501050773102,1501051774102)),headers=headers)
    ceshi1=html.content
    data = json.loads(ceshi1)
    return(data['PackageList']['Data'])


data_ceshi=pd.DataFrame([])#建立一个空数据框
html_list =[]#建立一个空列表


for i in range(10):
    html_list.append(get_html1(i))


for i in html_list:
    for j in i:
        a = j['beginTime']
        b = j['endTime']
        c = j['investedMoney']
        d = j['name']
        print u'开始时间: ' + str(a) + u'结束时间: '+ str(b) + u'投资金额: ' + str(c) + u'项目名称' + str(d)
        insert_SHENGXB = ("INSERT INTO SHENGXB (beginTime,endTime,investedMoney,name) VALUES(%s,%s,%s,%s)")
        data_data= (a, b, c, d)
        cursor.execute(insert_SHENGXB,data_data)
        db.commit()
        print '******完成此条插入!'

print '爬取数据并插入mysql数据库完成...'

保存后显示如下：