爬虫521错误

请求网页返回521错误很大概率是一种js反爬.

 

这种网站正常网页打开通过抓包工具可以观察到室友两次请求的,第一次返回一段js代码,第二次才是正常的结果,

 

所以我们需要把第一次返回的js执行出来,

比如网站:http://www.cbrc.gov.cn/chinese/newListDoc/111005/1.html就是一个返回521的典型网站

通过观察js可以用python进行更改js然后执行

网页返回的js

var x="String@rOm9XFMtA3QKV7nYsPGT4lifyWwkq5vcjH2IdxUoCbhERLaz81DNB6@firstChild@@JgSe0upZ@17@mzM@document@attachEvent@parseInt@href@@0xEDB88320@split@cookie@catch@@captcha@@length@0@function@1500@Fri@eval@@@Yt7@@39@match@__jsl_clearance@@0xFF@@@window@DOMContentLoaded@@@10@@innerHTML@36@false@@GMT@@join@setTimeout@@3D@@@charCodeAt@Expires@@@onreadystatechange@https@RegExp@@charAt@BN@aM@d@@@var@2@addEventListener@@challenge@Path@@chars@19@e@search@May@2F@@@@HqDzM@@createElement@toString@g@767@@f@@@new@if@@while@1558086399@else@@location@@reverse@8@@@@@try@Array@1@pathname@@@for@substr@@46@replace@fromCharCode@@return@@w@toLowerCase@div@a".replace(/@*$/,"").split("@"),y="76 60=24(){55('123.12=123.135+123.87.143(/[\\?|&]20-81/,\\'\\')',25);8.16='35=120.110|23|'+(24(){76 46=[24(60){146 60},24(46){146 46},24(60){138(76 46=23;46<60.22;46++){60[46]=11(60[46]).107(48)};146 60.54('')}],60=[(((+!-{})<<(+!-{}))+[]+[]),((+!-{})+(-~[]+[~~{}])/[-~{}-~{}]+[[]][23]),[(+!-{})]+[(+!-{})],[(+!-{})]+((-~[]|((+!-{})<<(+!-{})))+[]+[]),[(-~-~{}^-~{})+[-~{}-~{}]*((-~[]|((+!-{})<<(+!-{}))))],[(+!-{})]+((+[])+[[]][23]),[-~((-~[]|((+!-{})<<(+!-{}))))-~((-~[]|((+!-{})<<(+!-{}))))],(-~((-~[]|((+!-{})<<(+!-{}))))+[]+[]),[(+!-{})]+(((+!-{})<<(+!-{}))+[]+[]),((-~[]|((+!-{})<<(+!-{})))+[]+[]),[(+!-{})],(-~!{}-~[-~~~!{}+((+!-{})<<(+!-{}))+((+!-{})<<(+!-{}))]+[]),((+[])+[[]][23]),[-~((-~-~{}<<(+!-{})))]];138(76 131=23;131<60.22;131++){60[131]=46[[134,77,134,77,134,77,23,77,23,134,77,134,77,134][131]]([[[(+!-{})]+[-~((-~-~{}<<(+!-{})))]],[(((+!-{})<<(+!-{}))+[]+[])+(-~!{}-~[-~~~!{}+((+!-{})<<(+!-{}))+((+!-{})<<(+!-{}))]+[])],'71','7',[[(+!-{})]+[(+!-{})]],'104%57',[(((+!-{})<<(+!-{}))+[]+[])+(((+!-{})<<(+!-{}))+[]+[])],'31%100',[(-~[-~{}-~{}])/~~{}+[]+[[]][23]][23].70(~~!{}),'148',[[(+!-{})]+((+!-{})+(-~[]+[~~{}])/[-~{}-~{}]+[[]][23]),(((+!-{})<<(+!-{}))+[]+[])+((+!-{})+(-~[]+[~~{}])/[-~{}-~{}]+[[]][23])],'72',[{}+[[]][23]][23].70((-~{}-~{}<<-~{}-~{})),[[(+!-{})]+(-~((-~[]|((+!-{})<<(+!-{}))))+[]+[]),[(+!-{})]+[-~((-~-~{}<<(+!-{})))]]][60[131]])};146 60.54('')})()+';62=26, 6-88-85 45:142:33 52;82=/;'};116((24(){132{146 !!41.78;}17(86){146 50;}})()){8.78('42',60,50)}121{8.10('65',60)}",f=function(x,y){var a=0,b=0,c=0;x=x.split("");y=y||99;while((a=x.shift())&&(b=a.charCodeAt(0)-77.5))c=(Math.abs(b)<13?(b+48.5):parseInt(a,36))+y*c;return c},z=f(y.match(/\w/g).sort(function(x,y){return f(x)-f(y)}).pop());while(z++)try{eval(y.replace(/\b\w+\b/g, function(y){return x[f(y,z)-1]||("_"+y)}));break}catch(_){}

如下所示便是对网页返回的js通过python修改然后通过execjs库执行染回Cookie

def exec_content_js(response):
    content_js_str = response.doc('script').html()
    print(content_js_str)
    evel_js_str = 'function f1(){%s}' % content_js_str.replace('eval(', 'return (')
    print(evel_js_str)
    real_js_str = execjs.compile(evel_js_str).call('f1')
    print(real_js_str)
    parse_js_str = 'function f1(){return %s' % real_js_str.rsplit('if', 1)[0].split('document.cookie=', 1)[-1]
    parse_js_str = re.sub('document\.createElement.*?toLowerCase\(\)', '"www.cbrc.gov.cn/"', parse_js_str)
    print(parse_js_str)
    cookie_res = execjs.compile(parse_js_str).call('f1')
    cookies  = dict(response.cookies)
    cookies['__jsl_clearance'] = cookie_res.split(';', 1)[0].split('=')[-1]
    return cookies

将返回的Cookie带入header请求便能获取到网页内容