python动态网页爬取_Python 动态页面内容爬取

实在没啥技术含量。用python3写的。现在已经不用python2了。直接终端下随手ipython调试

╰─$ ipython3

Python 3.4.3 (default, Oct 14 2015, 20:28:29)

Type "copyright", "credits" or "license" for more information.

IPython 1.2.1 -- An enhanced Interactive Python.

? -> Introduction and overview of IPython's features.

%quickref -> Quick reference.

help -> Python's own help system.

object? -> Details about 'object', use 'object??' for extra details.

In [1]: import urllib.re

urllib.request urllib.response

In [1]: import urllib.request

In [2]: import json

In [3]: response = urllib.request.urlopen('http://score.5211game.com/arena/request/handler.ashx', b'action=UserRankDatas').read()

In [4]: json.l

json.load json.loads

In [4]: json.loads

json.loads

In [4]: json.loads(res)

%reset %reset_selective response

In [4]: json.loads(response)

---------------------------------------------------------------------------

TypeError Traceback (most recent call last)

in ()

----> 1 json.loads(response)

/usr/lib/python3.4/json/__init__.py in loads(s, encoding, cls, object_hook, parse_float, parse_int, parse_constant, object_pairs_hook, **kw)

310 if not isinstance(s, str):

311 raise TypeError('the JSON object must be str, not {!r}'.format(

--> 312 s.__class__.__name__))

313 if s.startswith(u'\ufeff'):

314 raise ValueError("Unexpected UTF-8 BOM (decode using utf-8-sig)")

TypeError: the JSON object must be str, not 'bytes'

In [5]: json.loads(response.decode('utf8'))

Out[5]:

{'Code': 100,

'DataModel': [{'ExtendProperties': '{"win":205,"lost":65,"win_rate":"75.09%","float":0,"common":[{"code":"H00S","name":"\\u98CE\\u66B4\\u4E4B\\u7075","win":1844},{"code":"H071","name":"\\u9C7C\\u4EBA\\u591C\\u884C\\u8005","win":1781},{"code":"Usyl","name":"\\u77EE\\u4EBA\\u72D9\\u51FB\\u624B","win":1779}]}',

'HeroHashCode': '',

'HeroId': 0,

'HeroName': '',

'HeroType': 0,

'Rank': 1,

'Ranking': 1928,

'UserId': 1000983122,

'UserName': '丨苏九儿女'},

{'ExtendProperties': '{"win":146,"lost":32,"win_rate":"82.02%","float":0,"common":[{"code":"H004","name":"\\u79C0\\u9017\\u9B54\\u5BFC\\u58EB","win":1218},{"code":"Ekee","name":"\\u53D7\\u6298\\u78E8\\u7684\\u7075\\u9B42","win":853},{"code":"UC76","name":"\\u6B7B\\u4EA1\\u5148\\u77E5","win":787}]}',

'HeroHashCode': '',

'HeroId': 0,

'HeroName': '',

'HeroType': 0,

'Rank': 2,

'Ranking': 1902,

'UserId': 1000262319,

'UserName': '熊猫Tv子阳'},

{'ExtendProperties': '{"win":136,"lost":23,"win_rate":"85%","float":0,"common":[{"code":"H00S","name":"\\u98CE\\u66B4\\u4E4B\\u7075","win":2238},{"code":"Ekee","name":"\\u53D7\\u6298\\u78E8\\u7684\\u7075\\u9B42","win":1953},{"code":"Nfir","name":"\\u5F71\\u9B54","win":1186}]}',

'HeroHashCode': '',

'HeroId': 0,

'HeroName': '',

'HeroType': 0,

'Rank': 3,

'Ranking': 1890,

'UserId': 300673572,

'UserName': 'Mr586'},

{'ExtendProperties': '{"win":165,"lost":57,"win_rate":"73.99%","float":"-1","common":[{"code":"U00C","name":"\\u566C\\u9B42\\u9B3C","win":667},{"code":"N01I","name":"\\u70BC\\u91D1\\u672F\\u58EB","win":540},{"code":"E032","name":"\\u5730\\u7CBE\\u6495\\u88C2\\u8005","win":535}]}',

'HeroHashCode': '',

'HeroId': 0,

'HeroName': '',

'HeroType': 0,

'Rank': 4,

'Ranking': 1843,

'UserId': 456970372,

'UserName': 'Rose赵云'},

{'ExtendProperties': '{"win":154,"lost":60,"win_rate":"70.97%","float":1,"common":[{"code":"EC77","name":"\\u51A5\\u754C\\u4E9A\\u9F99","win":1498},{"code":"Ekee","name":"\\u53D7\\u6298\\u78E8\\u7684\\u7075\\u9B42","win":1312},{"code":"U00A","name":"\\u6DF7\\u6C8C\\u9A91\\u58EB","win":1175}]}',

'HeroHashCode': '',

'HeroId': 0,

'HeroName': '',

'HeroType': 0,

'Rank': 5,

'Ranking': 1842,

'UserId': 410665135,

... SNIP ...