1.分析请求过程
在控制台中获取相应的参数
2.分析js并修改参数
function stringToHex(str) {
var val = "";
for (var i = 0; i < str.length; i++) {
if (val == "") val = str.charCodeAt(i).toString(16); else val += str.charCodeAt(i).toString(16);
}
return val;
}
function YunSuoAutoJump(text) {
// var width = screen.width;
var width = "1920";
// var height = screen.height;
var height = "1080";
if (text == null){
var screendate = width + "," + height;
} else {
var screendate = text;
}
// var curlocation = window.location.href;
// if (-1 == curlocation.indexOf("security_verify_")) {
// document.cookie = "srcurl=" + stringToHex(window.location.href) + ";path=/;";
// }
var location = "/default.aspx?tabid=226&security_verify_data=" + stringToHex(screendate);
return location;
}
// 接收传入的参数
var _ = process.argv.splice(2)
console.log(YunSuoAutoJump(_[0]));
构建相应的爬虫代码
# -*- coding: utf-8 -*-
# @Time : 2019/11/1 9:45
# @Author :
import os
import re
import requests
from lxml import etree
def generate_signature(value):
"""
generate _signature parameter
:param value:share_url id
:return:signature string
"""
cwd = os.path.dirname(__file__)
p = os.popen('cd %s && node landchina.js %s' % (cwd, value))
return p.readlines()[0]
def landchina():
# 构建session会话
s = requests.Session()
headers = {
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3',
'Accept-Language': 'zh-CN,zh;q=0.9',
'Cache-Control': 'no-cache',
'Connection': 'keep-alive',
'Host': 'www.landchina.com',
'Pragma': 'no-cache',
'Sec-Fetch-Mode': 'navigate',
'Sec-Fetch-Site': 'none',
'Sec-Fetch-User': '?1',
'Upgrade-Insecure-Requests': '1',
'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.120 Safari/537.36',
}
s.headers = headers
# 第一次请求
url = "https://www.landchina.com/default.aspx?tabid=226"
resp1 = s.get(url=url)
cookie1 = resp1.headers["Set-Cookie"]
cookie_re = "(security_session_verify=\w+;)"
security_session_verify = "".join(re.findall(cookie_re, cookie1))
security_session_verify_url = generate_signature("https://www.landchina.com/default.aspx?tabid=226").strip()
s.headers = {
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3',
'Accept-Language': 'zh-CN,zh;q=0.9',
'Cache-Control': 'no-cache',
'Connection': 'keep-alive',
'Cookie': security_session_verify + " srcurl=" + security_session_verify_url[
security_session_verify_url.rfind("=") + 1:],
'Host': 'www.landchina.com',
'Pragma': 'no-cache',
'Referer': 'https://www.landchina.com/default.aspx?tabid=226',
'Sec-Fetch-Mode': 'navigate',
'Sec-Fetch-Site': 'same-origin',
'Upgrade-Insecure-Requests': '1',
'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.120 Safari/537.36',
}
var_re = "(security_verify_data.*)"
var = "".join(re.findall(var_re, generate_signature("")))
# 第二次请求
url2 = "https://www.landchina.com/default.aspx?tabid=226&" + var
resp2 = s.get(url2)
cookie2 = resp2.headers["Set-Cookie"]
cookie_re = "(security_session_mid_verify=\w+;)"
security_session_mid_verify = "".join(re.findall(cookie_re, cookie2))
s.headers = {
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3',
'Accept-Encoding': 'gzip, deflate, br',
'Accept-Language': 'zh-CN,zh;q=0.9',
'Cache-Control': 'no-cache',
'Connection': 'keep-alive',
'Cookie': '%s; %s' % (security_session_verify, security_session_mid_verify),
'Host': 'www.landchina.com',
'Pragma': 'no-cache',
'Referer': url2,
'Sec-Fetch-Mode': 'navigate',
'Sec-Fetch-Site': 'same-origin',
'Upgrade-Insecure-Requests': '1',
'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.120 Safari/537.36',
}
# 第三次请求
rep = s.get(url=url)
mytree = etree.HTML(rep.text)
urls = mytree.xpath('//*[@id="TAB_contentTable"]//tr/td[@class="queryCellBordy"]/a')
for ur in urls:
title = "".join(ur.xpath(".//text()"))
c_url = "https://www.landchina.com/"+"".join(ur.xpath("./@href"))
data = {
"title" : title,
"c_url": c_url,
}
print(data)
if __name__ == '__main__':
landchina()