抓取一个网页数据的python 程序

 

import urllib.request as request
import json
import csv
import xlwt
import os
import sys
def readjson(fr):
    s = []
    data = json.load(fr)  # 用json中的load方法，将json串转换成字典
    s.append(data)  # 保存所有字典到列表中
    return s


def writeExcel(J,id):
    workbook = xlwt.Workbook(encoding='utf-8')
    booksheet = workbook.add_sheet('Sheet', cell_overwrite_ok=True)
    rvalue = J
    # print rvalue
    title = []
    for k, v in enumerate(rvalue[0]):
        title.append(v)
        booksheet.write(0, k, v)
    print(title)

    for a in range(len(rvalue)):
        for b in range(len(title)):
            try:
                d = title[b]
                c = str(rvalue[a][d])
                # if c:
                booksheet.write(a + 1, b, c)
                # else:
                #     if c == '':
                #         booksheet.write(a + 1, b, '{}')
                #     else:
                #         booksheet.write(a + 1, b, '{}')
            except:
                booksheet.write(a + 1, b, '')
    workbook.save(u'Merge'+str(id) + '.xls')

def main():

    total=0
    id=sys.argv[0]
    print("stock:",id)
    url='https://hero-api.jin10.com/accounts/'+str(id)+'/ticket?status=closed&'
    page=1

    res = [];
    while True:

        li=request.urlopen(url+'page='+str(page))
        page=page+1
        re=li.read().decode()
        dict=json.loads(re);

        t=dict['meta']['total']
        print(dict['meta'])
        if t<total:
            break
        total = total + 100
        for item in dict['data']:
            re={};
            re['开仓时间']=item['open_time']
            re['平仓时间'] = item['close_time']
            re['品种'] = item['item']
            re['方向'] = item['type']
            re['手数'] = item['size']
            re['止损价'] = item['sl']
            re['止盈价'] = item['tp']
            re['开仓价'] = item['open_price']
            re['平仓价'] = item['close_price']
            re['点数'] = item['point']
            re['净利润'] = item['profit']
            res.append(re)

    print(len(res))
    writeExcel(res,id)

if __name__ == '__main__':
    main()

 这段程序，可以抓取jin10网站的某个人的交易记录， 保存为Excel