雪球----爬取

import requests,json
from day3.xueqiu_mysql import mysql_conn

# url = 'https://xueqiu.com/v4/statuses/public_timeline_by_category.json?since_id=-1&max_id=-1&count=10&category=111'
url = 'https://xueqiu.com/v4/statuses/public_timeline_by_category.json?since_id=-1&max_id=184275&count=15&category=111'

headers = {
    'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
    # 'Accept-Encoding':'gzip, deflate, br',
    'Accept-Language':'zh-CN,zh;q=0.9',
    'Cache-Control':'max-age=0',
    'Connection':'keep-alive',
    'Cookie':'aliyungf_tc=AQAAAIN3zSGWOAMA9uQvaqnknOTGm14G; xq_a_token=584d0cf8d5a5a9809761f2244d8d272bac729ed4; xq_a_token.sig=x0gT9jm6qnwd-ddLu66T3A8KiVA; xq_r_token=98f278457fc4e1e5eb0846e36a7296e642b8138a; xq_r_token.sig=2Uxv_DgYTcCjz7qx4j570JpNHIs; _ga=GA1.2.1985976485.1534341208; _gid=GA1.2.26871005.1534341208; _gat_gtag_UA_16079156_4=1; u=931534341210320; Hm_lvt_1db88642e346389874251b5a1eded6e3=1534341211; Hm_lpvt_1db88642e346389874251b5a1eded6e3=1534341211; device_id=e2d92df89e85e4d0e8c6c9721c7e3382',
    'Host':'xueqiu.com',
    'Referer':'https://www.sogou.com/link?url=MKQkp13LDfVNhOpu93WLVxvx9Yiptv-_',
    'Upgrade-Insecure-Requests':'1',
    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36',
}

response = requests.get(url,headers = headers)

# print(type(response.text))
str_dict = json.loads(response.text)
# print(str_dict)
for ma in str_dict['list']:
    ma_str = ma['data']
    ma_dict = json.loads(ma_str)
    # print(ma_dict)
    a = ma_dict['id']
    b = ma_dict['title']
    c = ma_dict['target']
    d = ma_dict['description']
    sql = "insert into xueqiu(xid,title,target,description) values(%d,'%s','%s','%s')"%(a,b,c,d)
    print(sql)
    mm = mysql_conn()
    mm.execute_modify_mysql(sql)