获取动态加载ajax的数据

get_info

import re
import urllib2
import json
import pandas as pd


class Qyinyue(object):
    def __init__(self, url):
        self.url = url
        global datas
        datas = []
        self.start()

    def start(self):
        my_page = self.get_page()
        self.get_info(my_page)

    def get_page(self):
        url = self.url
        try:
            req = urllib2.Request(url, headers={'User-agent': 'Magic Browser'})
            my_page = urllib2.urlopen(req).read().decode('utf-8')
        except urllib2.HTTPError as e:
            if hasattr(e, 'code'):
                print "e.code:%s" % e.code
            elif hasattr(e, 'reason'):
                print "e.reason:%s" % e.reason
        return my_page

    def get_info(self, my_page):
        find_start = my_page.find('[')
        end_find = my_page.find(']')
        my_page = my_page[find_start: end_find + 1]
        pd_json = json.loads(my_page)
        datas.extend(pd_json)

mythread

import time

import pandas as pd

import mythread as mt
import get_info as gi

if __name__ == '__main__':
    urlList = []
    for i in range(1, 100):
        urlList.append(
            'https://c.y.qq.com/v8/fcg-bin/v8.fcg?channel=singer&page=list&key=all_all_all&pagesize=100&g_tk=5381&jsonpCallback=GetSingerListCallback&loginUin=0&hostUin=0&format=jsonp&inCharset=utf8&outCharset=utf-8&notice=0&platform=yqq&needNewCode=0&pagenum=' + str(
                i))
    threadList = [mt.Mythread(gi.Qyinyue, (url,)) for url in urlList]
    start_time = time.time()
    for t in threadList:
        t.setDaemon(True)
        t.start()
    for t in threadList:
        t.join()
    end_time = time.time()
    df = pd.DataFrame(gi.datas, columns=['Fother_name', 'Fsinger_id', 'Fsinger_name'])
    print df
    print "use the time is %s s" % (end_time - start_time)
''

testmain

import time

import pandas as pd

import mythread as mt
import get_info as gi

if __name__ == '__main__':
    urlList = []
    for i in range(1, 100):
        urlList.append(
            'https://c.y.qq.com/v8/fcg-bin/v8.fcg?channel=singer&page=list&key=all_all_all&pagesize=100&g_tk=5381&jsonpCallback=GetSingerListCallback&loginUin=0&hostUin=0&format=jsonp&inCharset=utf8&outCharset=utf-8&notice=0&platform=yqq&needNewCode=0&pagenum=' + str(
                i))
    threadList = [mt.Mythread(gi.Qyinyue, (url,)) for url in urlList]
    start_time = time.time()
    for t in threadList:
        t.setDaemon(True)
        t.start()
    for t in threadList:
        t.join()
    end_time = time.time()
    df = pd.DataFrame(gi.datas, columns=['Fother_name', 'Fsinger_id', 'Fsinger_name'])
    print df
    print "use the time is %s s" % (end_time - start_time)