python异步+参数逆向爬取＜福建省公共资源交易电子公共服务平台＞招标文件

m0_72472535

于 2024-05-09 10:59:08 发布

阅读量303

点赞数 3

文章标签： python javascript

本文链接：https://blog.csdn.net/m0_72472535/article/details/138604992

版权

const crypto = require('crypto');


var t = {
    "url": "/Trade/TradeInfo",
    "method": "post",
    "data": {
        "pageNo": null,
        "pageSize": 20,
        "total": 3855,
        "AREACODE": "",
        "M_PROJECT_TYPE": "",
        "KIND": "ZFCG",
        "GGTYPE": "1",
        "PROTYPE": "D01",
        "timeType": "6",
        "BeginTime": "2023-11-07 00:00:00",
        "EndTime": "2024-05-07 23:59:59",
        "createTime": []
    },
    "headers": {
        "common": {
            "Accept": "application/json, text/plain, */*"
        },
        "delete": {},
        "get": {},
        "head": {},
        "post": {
            "Content-Type": "application/x-www-form-urlencoded"
        },
        "put": {
            "Content-Type": "application/x-www-form-urlencoded"
        },
        "patch": {
            "Content-Type": "application/x-www-form-urlencoded"
        },
        "content-type": "application/json;charset=UTF-8"
    },
    "baseURL": "/FwPortalApi",
    "transformRequest": [
        null
    ],
    "transformResponse": [
        null
    ],
    "timeout": 0,
    "xsrfCookieName": "XSRF-TOKEN",
    "xsrfHeaderName": "X-XSRF-TOKEN",
    "maxContentLength": -1
}

var t1 = {
    "url": "/Trade/TradeInfoContent",
    "method": "post",
    "data": {
        "m_id": null,
        "type": "PURCHASE_QUALI_INQUERY_ANN"
    },
    "headers": {
        "common": {
            "Accept": "application/json, text/plain, */*"
        },
        "delete": {},
        "get": {},
        "head": {},
        "post": {
            "Content-Type": "application/x-www-form-urlencoded"
        },
        "put": {
            "Content-Type": "application/x-www-form-urlencoded"
        },
        "patch": {
            "Content-Type": "application/x-www-form-urlencoded"
        },
        "content-type": "application/json;charset=UTF-8"
    },
    "baseURL": "/FwPortalApi",
    "transformRequest": [
        null
    ],
    "transformResponse": [
        null
    ],
    "timeout": 0,
    "xsrfCookieName": "XSRF-TOKEN",
    "xsrfHeaderName": "X-XSRF-TOKEN",
    "maxContentLength": -1
}


var r = {
    "a": "B3978D054A72A7002063637CCDF6B2E5",
    'e': 'EB444973714E4A40876CE66BE45D5930',
    'i' : 'B5A8904209931867',
}


function v(t, e) {
    var n = Object.keys(t);
    if (Object.getOwnPropertySymbols) {
        var a = Object.getOwnPropertySymbols(t);
        e && (a = a.filter((function(e) {
            return Object.getOwnPropertyDescriptor(t, e).enumerable
        }
        ))),
        n.push.apply(n, a)
    }
    return n
}

function a(t, e, r) {
    return e in t ? Object.defineProperty(t, e, {
        value: r,
        enumerable: !0,
        configurable: !0,
        writable: !0
    }) : t[e] = r,
    t
}


function g(t) {
    for (var e = 1; e < arguments.length; e++) {
        var n = null != arguments[e] ? arguments[e] : {};
        e % 2 ? v(Object(n), !0).forEach((function(e) {
            Object(a)(t, e, n[e])
        }
        )) : Object.getOwnPropertyDescriptors ? Object.defineProperties(t, Object.getOwnPropertyDescriptors(n)) : v(Object(n)).forEach((function(e) {
            Object.defineProperty(t, e, Object.getOwnPropertyDescriptor(n, e))
        }
        ))
    }
    return t
}

function s(t, e) {
    return t.toString().toUpperCase() > e.toString().toUpperCase() ? 1 : t.toString().toUpperCase() == e.toString().toUpperCase() ? 0 : -1
}

function l(t) {
    for (var e = Object.keys(t).sort(s), n = "", a = 0; a < e.length; a++)
        if (void 0 !== t[e[a]])
            if (t[e[a]] && t[e[a]]instanceof Object || t[e[a]]instanceof Array) {
                var i = JSON.stringify(t[e[a]]);
                n += e[a] + i
            } else
                n += e[a] + t[e[a]];
    return n
}

function md5(text) {
    return crypto.createHash("md5").update(text).digest("hex")
}

function d(t) {
    for (var e in t)
        "" !== t[e] && void 0 !== t[e] || delete t[e];
    var n = r["a"] + l(t);
    return md5(n).toLocaleLowerCase()
}


function get_portal_sign(pageNo) {
    t['data']['pageNo'] = pageNo;
    t.headers.baseURL && (t.baseURL = t.headers.baseURL);
    var e = Object.assign({}, t.params);
    return e["ts"] = (new Date).getTime(),
        "string" === typeof t.data && (t.data = JSON.parse(t.data)),
        "post" === t.method && t.data && Object.assign(e, t.data),
        t.headers["portal-sign"] = d(e),
        "post" == t.method ? t.data = g(g({}, t.data), {}, {
            ts: e["ts"]
        }) : "get" == t.method && (t.params = g(g({}, t.params), {}, {
            ts: e["ts"]
        })),
        t
}

function get_portal_sign1(mid) {
    t1['data']['m_id'] = mid;
    t1.headers.baseURL && (t1.baseURL = t1.headers.baseURL);
    var e = Object.assign({}, t1.params);
    return e["ts"] = (new Date).getTime(),
        "string" === typeof t1.data && (t1.data = JSON.parse(t1.data)),
        "post" === t1.method && t1.data && Object.assign(e, t1.data),
        t1.headers["portal-sign"] = d(e),
        "post" == t1.method ? t1.data = g(g({}, t1.data), {}, {
            ts: e["ts"]
        }) : "get" == t1.method && (t1.params = g(g({}, t1.params), {}, {
            ts: e["ts"]
        })),
        t1
}

const crypto1 = require("crypto-js");
// 数据解密
function b(t) {
    var e = crypto1.enc.Utf8.parse(r["e"])
      , n = crypto1.enc.Utf8.parse(r["i"])
      , a = crypto1.AES.decrypt(t, e, {
        iv: n,
        mode: crypto1.mode.CBC,
        padding: crypto1.pad.Pkcs7
    });
    return a.toString(crypto1.enc.Utf8)
  }


function payload_main(page){
    return get_portal_sign(page)
}


function res_main(response){
    return b(response.Data)
}

以上为js部分：js文件中的t变量的EndTime和Total需修改

import asyncio
import aiohttp
import aiofiles
import execjs
import json
from tqdm.asyncio import tqdm


async def fetch_post(session, url, headers, cookies, json_data):
    async with session.post(url, headers=headers, cookies=cookies, json=json_data) as response:
        return await response.json()


async def main():
    pay_js = open('./a.js', 'r', encoding='utf-8').read()

    async with aiohttp.ClientSession() as session:
        for a in tqdm(range(3)):
            ctx = execjs.compile(pay_js).call('payload_main', a)

            cookies = {
                'ASP.NET_SessionId': 'kfmfmj5i4d04wfoehkixuzie',
            }

            headers = {
                'Accept': 'application/json, text/plain, */*',
                'Accept-Language': 'zh,zh-CN;q=0.9',
                'Connection': 'keep-alive',
                'Content-Type': 'application/json;charset=UTF-8',
                'Origin': 'https://ggzyfw.fj.gov.cn',
                'Referer': 'https://ggzyfw.fj.gov.cn/business/list/',
                'Sec-Fetch-Dest': 'empty',
                'Sec-Fetch-Mode': 'cors',
                'Sec-Fetch-Site': 'same-origin',
                'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
                'portal-sign': ctx['headers']['portal-sign'],
                'sec-ch-ua': '"Chromium";v="124", "Google Chrome";v="124", "Not-A.Brand";v="99"',
                'sec-ch-ua-mobile': '?0',
                'sec-ch-ua-platform': '"macOS"',
            }

            json_data = ctx['data']
            response = await fetch_post(session, 'https://ggzyfw.fj.gov.cn/FwPortalApi/Trade/TradeInfo', headers, cookies, json_data)
            data_res = execjs.compile(pay_js).call('res_main', response)
            data = json.loads(data_res)

            for i in data['Table']:
                name = i['NAME']
                mid = i['M_ID']
                ts_pay = open('./a.js', 'r', encoding='utf-8').read()
                content_res = execjs.compile(ts_pay).call('get_portal_sign1', mid)

                data1 = content_res['data']
                headers1 = headers.copy()  # Copying the main headers
                headers1['portal-sign'] = content_res['headers']['portal-sign']
                headers1['Referer'] = f'https://ggzyfw.fj.gov.cn/business/detail?name={name}&cid=D01-12350200426605541R-20240428-023872-7&type=ZFCG'
                zhaobiao_response = await fetch_post(session, 'https://ggzyfw.fj.gov.cn/FwPortalApi/Trade/TradeInfoContent', headers1, cookies, data1)

                res_html = json.loads(execjs.compile(ts_pay).call('res_main', zhaobiao_response))
                async with aiofiles.open(f'./res_html/{name}.html', 'w', encoding='utf-8') as f:
                    await f.write('<meta charset="UTF-8">' + res_html['Contents'])
                print(name, '下载成功')

# 运行异步主函数
asyncio.run(main())

以上为python部分