声明:本文章所有内容仅供学习使用,无其它任何目的,严禁用于商业用途和非法用途,否则产生一切后果均与作者无关!
目标链接:aHR0cHM6Ly93d3cubXBzLmdvdi5jbi9uMjI1NDA5OC9uNDkwNDM1Mi8=
1.抓包分析
打开抓包软件(Fiddler),输入目标链接,会出现以下几个状态码;分别是521、521、200,我们的目标数据就在状态码为200的里面:
查看第一个状态码为521返回的数据,有服务器返回的cookie和一段类似表情包(AAEncode)的js代码:
2.控制台调试
我们把这段js代码拿到浏览器控制台调试下,发现打印出来了cookie:
再把上面的set-cookie和js代码生成的cookie去和另外两个请求头对比,发现和第二个状态码为521的请求头带的cookie一模一样,而状态码为200的请求头带的cookie只有‘__jsluid_s’是一样的,所以这里可以猜测下网站执行顺序:第一次请求服务器,服务器返回cookie和一段(AAEncode)js代码,再用第一个请求返回的代码生成的cookie和服务器返回的cookie去请求服务器,服务器再一次返回一段混淆代码,最后再用混淆代码生成cookie和服务器第一次返回的cookie去请求目标网站,最终才返回我们要的数据。第二次响应数据是一段混淆代码,这段代码比较长拿到source面板进行调试:
调试后发现并没有我们想到的结果,这里我们可以hook一下cookie。hook方式很多种,我这里是采用Fiddler插件(编程喵)进行hook:
debugger断住以后,追栈进行调试:
发现这里documen[‘cookie’]就是我们最终要的cookie,我们可以采用全局导出的方式来获取这段cookie,也可以一点点跟栈进行分析扣出js代码然后生成cookie,
本地调试后发现检测了浏览器环境,我对应补了一些,补完后发现还检测了定时器(setTimeout),定时器补的话需要借助外部语言,补起来比较费时,后面我还是回到浏览器进行调试;
我把刚刚那段生成的cookie的代码进行拆开打印,可以看到 _0xdf22ce[0x0] 就是cookie,全局搜索 _0xdf22ce = 立马可以定位到709行生成的位置:
我们来分析下 _0xdf22ce 后面代码的意义:第一个是go里面的ct对应的值,第二个参数是bts对应的值,那继续跟 _0x3fcd2a 函数;
进入到 _0x3fcd2a 函数后进行逐语句运行会进去到else对应的代码里面,我们在 return 返回值的地方下一个断点,可以看到最终返回结果就是 _0x334824
而判断条件是 hash(_0x334824) == _0x170c2d ,_0x170c2d是go传进来ct的值,ct的值刚好是32位而go里面ha对应的是md5,我们把 _0x334824的值打印出来并进行md5加密发现和ct的值一模一样,所以for循环结束的条件就是当hash(_0x334824) == _0x170c2d返回最后结果,hash就是go里面ha对应的加密类型函数:
后面刷次几次网页发现有md5、sha1、sha256三种加密类型:
3.本地生成
JavaScript加密库可以用crypto-js来进行实现,完整代码如下:
var CryptoJS = require('crypto-js');
window = this;
window.navigator = {
userAgent: 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36'
};
var _0x1f08 = ['wrnDnSrDvQ==', 'VMKua8Oa', 'XR8zwpc=', 'YMK5wo4A', 'w7PDvFYT', 'D8OxcMOv', 'WQsow4o=', 'w65aZi8=', 'wpMzMcKg', 'woAgKcKo', 'B8ONHMOi', 'JcOgHsKO', 'FMOvw4By', 'NWgvIg==', 'dyfCoC8=', 'PgLCkcOk', 'JGInMg==', 'AX8rKw==', 'w7fDiFg6', 'OsOgC8KT', 'GcKTw5PDuA==', 'HmjDssKa', 'A0Yaw74=', 'PRMWwrU=', 'Bm3Dk8Kh', 'AMKYw6xl', 'dMKGwqY2', 'wo0Qw7TDtw==', 'woJvw4Nb', 'w6vCqcK5woo=', 'wrfDihfDpg==', 'w5PCsMOvwo0=', '6K6t5rOL6aqy6K6d', 'e3xsw5Y=', 'IsODw58d', 'w6fClMKqwok=', 'M1o7w58=', 'F30kMw==', 'flDDssOT', 'wpBxw7lR', 'w6nCqMOewqY=', 'XmlMw48=', 'AVHDoFA=', 'LMONNw==', 'wqTDmArDkw==', 'w4FVwp9/', 'M30wOw==', 'M8OlO8O5', 'wo7DrB0u', 'DSDCg0M=', 'w7Qdw4VL', 'CyHCpXg=', 'UMOuYMOc', 'wq90w4l/', 'X8Ova8Od', 'wqd7wojDgA==', 'W15yw7c=', 'w419YyA=', 'wrsBwrjCtg==', 'NcOyT8OX', 'fBUm', 'w4fCosO4wq0=', 'w54dw4BX', 'w7Qvw7pp', 'FsOSGcOF', 'w53CtsOUXw==', 'OQDCpsOQ', 'Az/CtcOP', 'GsO7LcKx', 'BGw+w7M=', 'w47CosO6wqs=', 'MGsjw7g=', 'N8OrVgk=', 'wpd8w4Z3', 'wrsrw5jDiQ==', 'XcKkwpo9', 'RATCvCo=', 'eMOcw6tZ', 'QEnDssOs', 'FcOlNMOf', 'KcK0w7/DmQ==', 'w5PDskQH', 'wq1Sw7pQ', 'G8Kfd8Oq', 'CsOuesOy', 'I8K2w7XDgg==', 'wqLDtyHDsg==', 'woBpw4lV', 'E2I3w5A=', 'WknCkUg=', 'w6zCh8Oawoo=', 'PjgSTg==', 'fsKtWMOM', 'HsKLw5jDsw==', 'wosLwrjCjQ==', 'J8O+TsOc', 'EF/DrsKM', 'TX12Gg==', 'HnEwPw==', 'wpEbw43Dkw==', 'w49Uw5JM', 'XyJwwpg=', 'wrrCvcOZw5U=', 'wopow4lh', 'wrDCtMOtw6k=', 'MmEkLQ==', 'J8OyPcOY', 'HksrIA==', 'wrNqwrbDiw==', 'KRohXw==', 'w5PCmcK6wo0=', 'OSHChcOY', 'KcKzw7HDgA==', 'ZB8lw5k=', 'w5DDqkI7', 'OHoHw7E=', 'Iikywrc=', 'LsKUw7HDvw==', 'WQIxw6Q=', 'UsKuLcKx', 'dsOLw5ZC', 'TsOhw7Rz', 'ImYjDA==', 'w7nCqcOVwps=', 'wofCvMOvw6c=', 'WArCuQo=', 'TUh2AQ==', 'w63CpcOmwrs=', 'wqTDuSnDmw==', 'E8OPw45U', 'w4tOaTA=', 'a3h6w50=', 'woxDw7Vl', 'GsOxw4hz', 'CkQiw4k=', 'LyEjwpc=', 'YsK9wpYO', 'F8OkfBs=', 'K2bDgTo=', 'a8ObbMOY', 'VMOIw55u', 'w7FrahM=', 'B3TDlC0=', 'H1vCpcO1', 'wpdEw4p4', 'KCERZA==', 'w7HCn8K1', 'wpESwojCvQ==', 'SMOMw5ZF', 'w6tZbTI=', 'GkM4w4A=', 'NsOsdMOi', 'NHAoFw==', 'AcOAIsKm', 'wqF2w7h8', 'wpdxw5JG', 'McKfw6pm', 'L8OoSjk=', 'Vi95wrU=', 'wpQVwqXClw==', 'RsO5YMOI', 'w53CvMOEZw==', 'VMKdTcOW', 'C8O4XSg=', 'BnvDi8Kq', 'w4rCi8Odwoc=', 'woEPwovCvA==', 'THV5w5c=', 'BjINwpw=', 'f8OOU8Oy', 'wo1ZwpTDqw==', 'XwHCpAo=', 'H0Y1w7w=', 'O8O6BcOo', 'B1M0Eg==', 'D1wABg==', 'K8KEw7Fm', 'dcOzw6Na', 'ABEMbg==', 'DMOJCsKT', 'C3vDkcKv', 'YDnDiMOX', 'w6LCt8OYwrQ=', 'RcOvd8Ol', 'YErCr20=', 'Txgjw5E=', 'FsOlw5Qh', 'UsOnLsO6', 'C8OPBMOc', 'GSbChcO/', 'cgtMwpI=', 'KRgGwoo=', 'QkHCtWU=', 'EMO5CsKE', 'woAPwpzCqQ==', 'YVfDmcOP', 'EAPCgUk=', 'esOAdMOW', 'FyvCo8OT', 'w57CvsOFdg==', 'w6bCmMK5wpI=', 'J2DDrgA=', 'M8OsY8OX', 'CsO3LcOP', 'ZCfDuMOc', 'Cm7Djg=='];
(function(_0x32288f, _0x1f0807) {
var _0x1e8e72 = function(_0x517011) {
while (--_0x517011) {
_0x32288f['push'](_0x32288f['shift']());
}
};
_0x1e8e72(++_0x1f0807);
}(_0x1f08, 0x136));
var _0x1e8e = function(_0x32288f, _0x1f0807) {
_0x32288f = _0x32288f - 0x0;
var _0x1e8e72 = _0x1f08[_0x32288f];
if (_0x1e8e['oatwKj'] === undefined) {
(function() {
var _0x1c255c;
try {
var _0x244985 = Function('return\x20(function()\x20' + '{}.constructor(\x22return\x20this\x22)(\x20)' + ');');
_0x1c255c = _0x244985();
} catch (_0x2e351f) {
_0x1c255c = window;
}
var _0x4c97f9 = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=';
_0x1c255c['atob'] || (_0x1c255c['atob'] = function(_0x1d489e) {
var _0x3c0b03 = String(_0x1d489e)['replace'](/=+$/, '');
var _0x5b4d33 = '';
for (var _0x574747 = 0x0, _0x427c06, _0x1535a0, _0x144e58 = 0x0; _0x1535a0 = _0x3c0b03['charAt'](_0x144e58++); ~_0x1535a0 && (_0x427c06 = _0x574747 % 0x4 ? _0x427c06 * 0x40 + _0x1535a0 : _0x1535a0,
_0x574747++ % 0x4) ? _0x5b4d33 += String['fromCharCode'](0xff & _0x427c06 >> (-0x2 * _0x574747 & 0x6)) : 0x0) {
_0x1535a0 = _0x4c97f9['indexOf'](_0x1535a0);
}
return _0x5b4d33;
}
);
}());
var _0x304446 = function(_0x4c143a, _0x512cc7) {
var _0x5c7801 = [], _0x1ff892 = 0x0, _0x4e18b8, _0x744958 = '', _0x10a3ee = '';
_0x4c143a = atob(_0x4c143a);
for (var _0x5ab0e3 = 0x0, _0x305d73 = _0x4c143a['length']; _0x5ab0e3 < _0x305d73; _0x5ab0e3++) {
_0x10a3ee += '%' + ('00' + _0x4c143a['charCodeAt'](_0x5ab0e3)['toString'](0x10))['slice'](-0x2);
}
_0x4c143a = decodeURIComponent(_0x10a3ee);
var _0x55644d;
for (_0x55644d = 0x0; _0x55644d < 0x100; _0x55644d++) {
_0x5c7801[_0x55644d] = _0x55644d;
}
for (_0x55644d = 0x0; _0x55644d < 0x100; _0x55644d++) {
_0x1ff892 = (_0x1ff892 + _0x5c7801[_0x55644d] + _0x512cc7['charCodeAt'](_0x55644d % _0x512cc7['length'])) % 0x100;
_0x4e18b8 = _0x5c7801[_0x55644d];
_0x5c7801[_0x55644d] = _0x5c7801[_0x1ff892];
_0x5c7801[_0x1ff892] = _0x4e18b8;
}
_0x55644d = 0x0;
_0x1ff892 = 0x0;
for (var _0x15bb2a = 0x0; _0x15bb2a < _0x4c143a['length']; _0x15bb2a++) {
_0x55644d = (_0x55644d + 0x1) % 0x100;
_0x1ff892 = (_0x1ff892 + _0x5c7801[_0x55644d]) % 0x100;
_0x4e18b8 = _0x5c7801[_0x55644d];
_0x5c7801[_0x55644d] = _0x5c7801[_0x1ff892];
_0x5c7801[_0x1ff892] = _0x4e18b8;
_0x744958 += String['fromCharCode'](_0x4c143a['charCodeAt'](_0x15bb2a) ^ _0x5c7801[(_0x5c7801[_0x55644d] + _0x5c7801[_0x1ff892]) % 0x100]);
}
return _0x744958;
};
_0x1e8e['VTYzuC'] = _0x304446;
_0x1e8e['uqLLuz'] = {};
_0x1e8e['oatwKj'] = !![];
}
var _0x517011 = _0x1e8e['uqLLuz'][_0x32288f];
if (_0x517011 === undefined) {
if (_0x1e8e['qiRzWV'] === undefined) {
_0x1e8e['qiRzWV'] = !![];
}
_0x1e8e72 = _0x1e8e['VTYzuC'](_0x1e8e72, _0x1f0807);
_0x1e8e['uqLLuz'][_0x32288f] = _0x1e8e72;
} else {
_0x1e8e72 = _0x517011;
}
return _0x1e8e72;
};
function hash(type, value) {
if (type == 'md5') {
return CryptoJS.MD5(value).toString();
}
if (type == 'sha1') {
return CryptoJS.SHA1(value).toString();
}
if (type == 'sha256') {
return CryptoJS.SHA256(value).toString();
}
}
var get_cookie = function (bts, chars, ct, ha) {
var new_cookie = "";
go({
"bts": bts,
"chars": chars,
"ct": ct,
"ha": ha,
"tn": "__jsl_clearance_s",
"vt": "3600",
"wt": "1500"
});
function go(_0x3402c2) {
var _0x2631a4 = {};
_0x2631a4[_0x1e8e('0x5c', ']*4V') + 'A'] = _0x1e8e('0x7c', 'uOjp') + _0x1e8e('0x2a', '!6$E');
_0x2631a4[_0x1e8e('0x83', '!6$E') + 'S'] = function (_0x46b5dd, _0x30862d) {
return _0x46b5dd != _0x30862d;
}
;
_0x2631a4[_0x1e8e('0x1c', 'Xvnj') + 'l'] = _0x1e8e('0x27', 'x0gG') + _0x1e8e('0x96', 'A@y@') + _0x1e8e('0x53', '*L6%') + '2';
_0x2631a4[_0x1e8e('0x9b', 'Pv(u') + 'q'] = function (_0x4b872e, _0x4b34ef, _0x40405f) {
return _0x4b872e(_0x4b34ef, _0x40405f);
}
;
_0x2631a4[_0x1e8e('0x39', 'gJ]d') + 'O'] = function (_0x354835, _0x16da81, _0xec84d6, _0x21babe, _0x4cb85e) {
return _0x354835(_0x16da81, _0xec84d6, _0x21babe, _0x4cb85e);
}
;
_0x2631a4[_0x1e8e('0xb8', 'Pv(u') + 'U'] = function (_0x25b619, _0x19b14c, _0x4de1e0) {
return _0x25b619(_0x19b14c, _0x4de1e0);
}
;
_0x2631a4[_0x1e8e('0x5', 'Qpcm') + 'e'] = function (_0x45553d, _0x1307ca) {
return _0x45553d(_0x1307ca);
}
;
_0x2631a4[_0x1e8e('0xa7', 'gJ]d') + 'S'] = function (_0x18e234, _0x4b43d8) {
return _0x18e234 < _0x4b43d8;
}
;
_0x2631a4[_0x1e8e('0xab', '(&d(') + 'C'] = function (_0x4a6dfb, _0x26522c) {
return _0x4a6dfb + _0x26522c;
}
;
_0x2631a4[_0x1e8e('0xc1', 'Qpcm') + 'V'] = function (_0x43454d, _0x3cc1fa) {
return _0x43454d ^ _0x3cc1fa;
}
;
_0x2631a4[_0x1e8e('0xcb', 'O&^h') + 'P'] = function (_0x30feac, _0x24e04c) {
return _0x30feac ^ _0x24e04c;
}
;
_0x2631a4[_0x1e8e('0xb1', '%!hX') + 'i'] = function (_0x568cb2, _0x43d939) {
return _0x568cb2 - _0x43d939;
}
;
_0x2631a4[_0x1e8e('0x3b', 'y(zJ') + 'v'] = function (_0x1413e0, _0x5725f8) {
return _0x1413e0 - _0x5725f8;
}
;
_0x2631a4[_0x1e8e('0x97', ']*4V') + 'G'] = function (_0x3425a8, _0x3e6859) {
return _0x3425a8 - _0x3e6859;
}
;
_0x2631a4[_0x1e8e('0x4', 'NMDD') + 'V'] = _0x1e8e('0xb7', 'A@y@') + 'S';
_0x2631a4[_0x1e8e('0x46', 'RGrC') + 'V'] = function (_0x5d13bb, _0x69d547) {
return _0x5d13bb + _0x69d547;
}
;
_0x2631a4[_0x1e8e('0x85', 'P!aD') + 'P'] = function (_0x386b4b, _0x18eeac) {
return _0x386b4b == _0x18eeac;
}
;
_0x2631a4[_0x1e8e('0x12', 'Tprk') + 'V'] = function (_0x17e294, _0x4b3166) {
return _0x17e294 !== _0x4b3166;
}
;
_0x2631a4[_0x1e8e('0x88', 'rsNB') + 's'] = _0x1e8e('0x99', ']*4V') + 'q';
_0x2631a4[_0x1e8e('0x41', 'wiDo') + 'j'] = _0x1e8e('0x51', 'U8[d') + 'Q';
_0x2631a4[_0x1e8e('0x15', 'h$b8') + 'm'] = function (_0x2c33d3, _0x596828) {
return _0x2c33d3 + _0x596828;
}
;
_0x2631a4[_0x1e8e('0x23', 'wiDo') + 'H'] = _0x1e8e('0x1d', '^zTO') + 'd';
_0x2631a4[_0x1e8e('0xb2', 'h$b8') + 'q'] = _0x1e8e('0xca', 'Xvnj') + _0x1e8e('0x68', 'rl&S') + '=';
_0x2631a4[_0x1e8e('0x70', '9nz1') + 'P'] = _0x1e8e('0x8a', 'U8[d') + _0x1e8e('0xc5', 's5#R') + '\x20/';
_0x2631a4[_0x1e8e('0x93', 'Q^^n') + 'h'] = function (_0x309466, _0x5ae3db) {
return _0x309466 + _0x5ae3db;
}
;
_0x2631a4[_0x1e8e('0x26', '4G]j') + 'b'] = function (_0x241da1, _0x244bb2) {
return _0x241da1 > _0x244bb2;
}
;
_0x2631a4[_0x1e8e('0xbe', '(&d(') + 'F'] = function (_0x2c769d, _0x4fd6af) {
return _0x2c769d(_0x4fd6af);
}
;
_0x2631a4[_0x1e8e('0x21', 'y(zJ') + 'e'] = function (_0x3d34ec, _0x173c72) {
return _0x3d34ec(_0x173c72);
}
;
_0x2631a4[_0x1e8e('0xbd', 'Pv(u') + 'P'] = function (_0x1fcd3e, _0x4fe4cf, _0x4cc1d8) {
return _0x1fcd3e(_0x4fe4cf, _0x4cc1d8);
}
;
_0x2631a4[_0x1e8e('0x40', 'rl&S') + 'n'] = _0x1e8e('0x86', 'y(zJ') + '失败';
var _0x281d5b = _0x2631a4;
function _0x1530ce() {
var _0x249919 = window[_0x1e8e('0x20', '%!hX') + _0x1e8e('0x5f', 'CUMC') + 'r'][_0x1e8e('0x38', 'wiDo') + _0x1e8e('0xb5', '9nz1') + 't']
, _0x540838 = [_0x281d5b[_0x1e8e('0x5b', 'x0gG') + 'A']];
for (var _0x5f5282 = 0x0; _0x5f5282 < _0x540838[_0x1e8e('0x95', '6qtH') + 'th']; _0x5f5282++) {
if (_0x281d5b[_0x1e8e('0x28', 'Pv(u') + 'S'](_0x249919[_0x1e8e('0xb4', 'x0gG') + _0x1e8e('0x91', '9nz1')](_0x540838[_0x5f5282]), -0x1)) {
return !![];
}
}
if (window[_0x1e8e('0x4c', '0GUB') + _0x1e8e('0x3f', 'DPJ[') + _0x1e8e('0xa2', 'U8[d')] || window[_0x1e8e('0x7f', '(umq') + _0x1e8e('0x16', 't[x(')] || window[_0x1e8e('0x2d', 'qWHL') + _0x1e8e('0x11', 'sCTl')] || window[_0x1e8e('0xac', 'P!aD') + _0x1e8e('0x87', 'DPJ[') + 'r'][_0x1e8e('0x1b', 'DPJ[') + _0x1e8e('0x94', 'RGrC') + 'r'] || window[_0x1e8e('0x3c', '0GUB') + _0x1e8e('0x5e', 'MbjT') + 'r'][_0x1e8e('0x4b', 's5#R') + _0x1e8e('0x79', 's5#R') + _0x1e8e('0xc8', 'RGrC') + _0x1e8e('0xba', '*L6%') + 'e'] || window[_0x1e8e('0x43', 'h$b8') + _0x1e8e('0xa3', 'P!aD') + 'r'][_0x1e8e('0x1f', 'rl&S') + _0x1e8e('0x89', '!6$E') + _0x1e8e('0x4f', 'wiDo') + _0x1e8e('0x76', 'RGrC') + _0x1e8e('0x35', 'y(zJ')]) {
return !![];
}
}
;
if (_0x1530ce()) {
return;
}
function _0x3f0719(type, _0x32f11f, _0x23f819) {
var _0x2f61ec = _0x3402c2[_0x1e8e('0x33', 'Xvnj') + 's'][_0x1e8e('0x69', '%!hX') + 'th'];
for (var _0x1c2b16 = 0x0; _0x1c2b16 < _0x2f61ec; _0x1c2b16++) {
for (var _0x1861db = 0x0; _0x281d5b[_0x1e8e('0x6d', 'qWHL') + 'S'](_0x1861db, _0x2f61ec); _0x1861db++) {
var _0x29b990 = _0x281d5b[_0x1e8e('0x9d', 'NMDD') + 'V'](_0x281d5b[_0x1e8e('0x74', 'h$b8') + 'V'](_0x23f819[0x0] + _0x3402c2[_0x1e8e('0xb0', '3cYK') + 's'][_0x1e8e('0x1', 'RGrC') + 'tr'](_0x1c2b16, 0x1), _0x3402c2[_0x1e8e('0xb6', 'L*zs') + 's'][_0x1e8e('0xcd', 'Pv(u') + 'tr'](_0x1861db, 0x1)), _0x23f819[0x1]);
if (hash(type, _0x29b990) == _0x32f11f) {
return _0x29b990;
}
}
}
}
;var _0x11c29b = _0x3f0719(type = ha, _0x3402c2['ct'], _0x3402c2[_0x1e8e('0x65', '0GUB')]);
new_cookie += _0x11c29b;
}
return new_cookie
};
console.log(get_cookie(bts = ["1670688969.658|0|hE5", "vgK6Yg82BfZEspVKp1Fk%2FQ%3D"],
chars = "iStvfhxijtnADp1fPymqAw", ct = "f2bd82f26a701a4d0814aa8ac01982ed4f271b3c", ha = "sha1"));
最后在python里面导入execjs库获取 _jsl_clearance_s:
# -*- coding: utf-8 -*-
import requests
import execjs
import re
import json
from loguru import logger
from scrapy import Selector
import urllib3
urllib3.disable_warnings()
def get_jsluid_s(url, headers):
"""获取__jsluid_s"""
resp = requests.request(method="GET", url=url, headers=headers, allow_redirects=False, verify=False)
logger.info(f"status_code1: {resp.status_code}")
set_cookies = resp.headers.get("Set-Cookie").split(";")[0]
document_cookie_str = re.search("cookie=(.*?);location", resp.text)
if document_cookie_str:
document_cookie_str = document_cookie_str.group(1)
document_cookie = execjs.eval(document_cookie_str).split(';')[0]
headers.update({"Cookie": f"{set_cookies}; {document_cookie}"})
headers.update({"Referer": "https://www.mps.gov.cn/n2254098/n4904352/"})
resp2 = requests.request("GET", url=url, headers=headers, verify=False)
logger.info(f"status_code2: {resp2.status_code}")
if resp2.status_code == 521:
datas = re.search(";go\((.*?)\)", resp2.text, re.S)
return set_cookies, datas.group(1)
def get_jsl_clearance_s(datas):
"""获取__jsl_clearance_s"""
data = json.loads(datas)
bts = data.get("bts")
chars = data.get("chars")
ct = data.get("ct")
ha = data.get("ha")
with open("./get_cookie.js", 'r') as fp:
js_code = fp.read()
comp = execjs.compile(js_code)
jsl_clearance_s = comp.call("get_cookie", bts, chars, ct, ha)
if jsl_clearance_s:
return "__jsl_clearance_s=" + jsl_clearance_s
else:
print(data)
def get_result(url, headers, set_cookies, jsl_clearance_s):
headers.update({"Cookie": f"{set_cookies}; {jsl_clearance_s}"})
resp3 = requests.request(method="GET", url=url, headers=headers, verify=False)
logger.info(f"status_code3: {resp3.status_code}")
if resp3.status_code == 200:
html = Selector(text=resp3.content.decode("utf-8"))
infos = html.xpath("//*[@class='list']/li")
for info in infos:
data = info.xpath("./span/text()").get()
title = info.xpath("./a/text()").get()
href = info.xpath("./a/@href").get()
logger.info(f"{data}:{title}:{href}")
if __name__ == '__main__':
url = "https://www.mps.gov.cn/n2254098/n4904352/"
headers = {
# "Host": "www.mps.gov.cn",
"Connection": "keep-alive",
"Pragma": "no-cache",
"Cache-Control": "no-cache",
"Upgrade-Insecure-Requests": "1",
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36",
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
"Sec-Fetch-Site": "none",
"Sec-Fetch-Mode": "navigate",
"Sec-Fetch-Dest": "document",
"Accept-Encoding": "gzip, deflate",
"Accept-Language": "zh-CN,zh;q=0.9"
}
set_cookies, datas = get_jsluid_s(url=url, headers=headers)
jsl_clearance_s = get_jsl_clearance_s(datas)
get_result(url=url, headers=headers, set_cookies=set_cookies, jsl_clearance_s=jsl_clearance_s)