雪球数据抓取和数据库存储

import requests
import json,time

headers = {
'Accept':'*/*',
'Accept-Encoding':'gzip, deflate, br',
'Accept-Language':'zh-CN,zh;q=0.9',
'Connection':'keep-alive',
'Cookie':'_ga=GA1.2.385314115.1530877538; device_id=056c84c5656cedd9fbb112dece5e7727; s=f013ab6sm2; __utma=1.385314115.1530877538.1530877556.1530877556.1; __utmz=1.1530877556.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); aliyungf_tc=AQAAANeNQk+2GAsA1JKePSYqlY/QR7ul; xq_a_token=584d0cf8d5a5a9809761f2244d8d272bac729ed4; xq_a_token.sig=x0gT9jm6qnwd-ddLu66T3A8KiVA; xq_r_token=98f278457fc4e1e5eb0846e36a7296e642b8138a; xq_r_token.sig=2Uxv_DgYTcCjz7qx4j570JpNHIs; u=771534332808893; Hm_lvt_1db88642e346389874251b5a1eded6e3=1534332809; Hm_lpvt_1db88642e346389874251b5a1eded6e3=1534332809; _gid=GA1.2.245410259.1534332809; _gat_gtag_UA_16079156_4=1',
'Host':'xueqiu.com',
'Referer':'https://xueqiu.com/today',
'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.62 Safari/537.36',
'X-Requested-With':'XMLHttpRequest',
}

url = 'https://xueqiu.com/v4/statuses/public_timeline_by_category.json?since_id=-1&max_id=-1&count=10&category=-1'
url2 = 'https://xueqiu.com/v4/statuses/public_timeline_by_category.json?since_id=-1&max_id=20304184&count=15&category=-1'
url3 = 'https://xueqiu.com/v4/statuses/public_timeline_by_category.json?since_id=-1&max_id=20304169&count=15&category=-1'
response = requests.get(url3,headers=headers)

res = json.loads(response.text)
data_mysql = res['list']
from pymysql_conn import Mysql_connect
data_xq = Mysql_connect()
for i in data_mysql:
    if i['column'] == '':
        i['column'] = '--'
    sql = 'insert into xueqiu (id,category,detal,columns)value({},{},{},{})'.format(i['id'],i['category'],"'"+i['data']+"'","'"+i['column']+"'")
    data_xq.execute_sql(sql)
    # print(i['id'],i['category'],len(i['data']),len(i['column']))
    time.sleep(1)

pymysql_conn

import pymysql

class Mysql_connect(object):
    def __init__(self):
        self.db = pymysql.connect(host='127.0.0.1',user='root',password='123456',port=3306,database='db815',charset='utf8')
        self.cursor = self.db.cursor()

    def execute_sql(self,sql):
        self.cursor.execute(sql)
        self.db.commit()

    def __del__(self):
        self.cursor.close()
        self.db.close()


if __name__ == '__main__':
    con_test = Mysql_connect()
    sql = 'insert into xueqiu (id)value(3)'
    con_test.execute_sql(sql)