爬虫实战_中国招标投标

仅作练习,侵权联系删除

爬取中国招标投标公共服务平台

python文件

注:会封ip !!!!!!!!!

注:会封ip !!!!!!!!!

注:会封ip !!!!!!!!!

文件中使用代理IP,注意更换

import json
import os
import time
import requests
import execjs
import re

cookies = {
    'Hm_lvt_ef2114bed21175425a21bb8a1e40ebdf': str(int(time.time())),
    'HMACCOUNT': '53EA9B4384671029',
    'Hm_lpvt_ef2114bed21175425a21bb8a1e40ebdf': str(int(time.time())),
    'JSESSIONID': '9F16445BE772EFB9577165A69F19659B',
    'cmsurl': '/../../index.shtml',
    'acw_tc': '1a0c639b17237125133932521e0031ef83c222186be8845ca6fd9ca6190af6',
}

headers = {
    'Accept': 'application/json, text/javascript, */*; q=0.01',
    'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
    'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
    # Requests sorts cookies= alphabetically
    # 'Cookie': 'Hm_lvt_ef2114bed21175425a21bb8a1e40ebdf=1723698178; HMACCOUNT=53EA9B4384671029; Hm_lpvt_ef2114bed21175425a21bb8a1e40ebdf=1723712517; JSESSIONID=9F16445BE772EFB9577165A69F19659B; cmsurl=/../../index.shtml; acw_tc=1a0c639b17237125133932521e0031ef83c222186be8845ca6fd9ca6190af6',
    'Origin': 'http://www.cebpubservice.com',
    'Proxy-Connection': 'keep-alive',
    'Referer': 'http://www.cebpubservice.com/ctpsp_iiss/searchbusinesstypebeforedooraction/getSearch.do',
    'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64; Trident/7.0; rv:11.0) like Gecko',
    'X-Requested-With': 'XMLHttpRequest',
}
for i in range(3):
    data = {
        'searchName': '',
        'searchArea': '130000',
        'searchIndustry': '',
        'centerPlat': '',
        'businessType': '招标公告',
        'searchTimeStart': '',
        'searchTimeStop': '',
        'timeTypeParam': '',
        'bulletinIssnTime': '',
        'bulletinIssnTimeStart': '',
        'bulletinIssnTimeStop': '',
        'pageNo': i,
        'row': '15',
    }
    #===========找个代理IP=================
    proxies = {
        'http':'',
    }
    response = requests.post(
        'http://www.cebpubservice.com/ctpsp_iiss/searchbusinesstypebeforedooraction/getStringMethod.do',
        cookies=cookies, headers=headers, data=data, verify=False,proxies=proxies,timeout=300)
    print(response.text)
    data = response.json()['object']['returnlist']
    for a in data:
        # 招标名称
        businessObjectName = a['businessObjectName']
        print(businessObjectName)
        # businessId加密1
        businessId = a['businessId']
        # 读取 2.js 文件内容
        with open('2.js', 'r', encoding='utf-8') as file:
            js_code = file.read()

        # 创建 ExecJS 上下文
        ctx = execjs.compile(js_code)

        # 调用 strEncode 函数                    时间xxxx-xx-xx xx,最后是几点
        result = ctx.call('strEncode', businessId, '2024-08-15 18', 'cebpubservice', 'iiss')
        print(1, result)
        # tenderProjectCode加密2
        tenderProjectCode = a['tenderProjectCode']
        # 调用 strEncode 函数                                时间xxxx-xx-xx xx,最后是几点
        result1 = ctx.call('strEncode', tenderProjectCode, '2024-08-15 18', 'cebpubservice', 'iiss')
        print(2, result1)
        schemaVersion = a['schemaVersion']
        print(schemaVersion)
        # 所属专业
        industriesType = a['industriesType']
        # 所属地区
        regionName = a['regionName']
        # 所属平台
        transactionPlatfName = a['transactionPlatfName']
        # 开始时间
        receiveTime = a['receiveTime']
        # 结束时间
        bulletinEndTime = a['bulletinEndTime']
        data = {
            'schemaVersion': schemaVersion,
            'businessKeyWord': 'tenderBulletin',
            'tenderProjectCode': result1,
            'businessObjectName': businessObjectName,
            'businessId': result,
        }

        response = requests.post('http://www.cebpubservice.com/ctpsp_iiss/SecondaryAction/findDetails.do',
                                 cookies=cookies,
                                 headers=headers, data=data, verify=False,proxies=proxies,timeout=300)
        # 提取所有文本
        text_pattern = re.compile(r'>\s*(.*?)\s*<', re.S)
        texts = text_pattern.findall(response.text)

        # 过滤掉空文本并连接成一个字符串
        all_text = ' '.join(texts).strip()

        # 去掉所有空格
        all_text_no_spaces = ''.join(all_text.split())
        print("招标内容", all_text_no_spaces)
        
        
        # 创建一个字典来存储所有信息
        data_info = {
            '招标公告名称': businessObjectName,
            '所属专业': industriesType,
            '所属地区': regionName,
            '所属平台': transactionPlatfName,
            '公告': all_text_no_spaces,
            '公告发布时间': receiveTime,
            '公告结束时间': bulletinEndTime,
        }

        # 读取已有的 JSON 文件内容
        file_path = '河北招标公告.json'
        if os.path.exists(file_path):
            with open(file_path, 'r', encoding='utf-8') as f:
                existing_data = json.load(f)
        else:
            existing_data = []

        # 添加新的电影信息到现有数据中
        existing_data.append(data_info)

        # 将更新后的数据写回到 JSON 文件
        with open(file_path, 'w', encoding='utf-8') as f:
            json.dump(existing_data, f, ensure_ascii=False, indent=4)

        print(f"{businessObjectName}已追加到 data_info.json 文件中。")

js文件 

文件名“2.js”,这个文件是加密文件
/*
*(1).加密:
*  第一步:strEncode(data,firstKey,secondKey,thirdKey);
*(2).解密:
*  第一步:strDecode(data,firstKey,secondKey,thirdKey);
*/
/*
* encrypt the string to string made up of hex
* return the encrypted string
*/
/*加密的函数,三个密钥*/
function strEncode(data,firstKey,secondKey,thirdKey){
 var leng = data.length;
 var encData = "";
 var firstKeyBt,secondKeyBt,thirdKeyBt,firstLength,secondLength,thirdLength;
 if(firstKey != null && firstKey != ""){
   firstKeyBt = getKeyBytes(firstKey);
   firstLength = firstKeyBt.length;
 }
 if(secondKey != null && secondKey != ""){
   secondKeyBt = getKeyBytes(secondKey);
   secondLength = secondKeyBt.length;
 }
 if(thirdKey != null && thirdKey != ""){
   thirdKeyBt = getKeyBytes(thirdKey);
   thirdLength = thirdKeyBt.length;
 }

 if(leng > 0){
   if(leng < 4){
     var bt = strToBt(data);
     var encByte ;
     if(firstKey != null && firstKey !="" && secondKey != null && secondKey != "" && thirdKey != null && thirdKey != ""){
       var tempBt;
       var x,y,z;
       tempBt = bt;
       for(x = 0;x < firstLength ;x ++){
         tempBt = enc(tempBt,firstKeyBt[x]);
       }
       for(y = 0;y < secondLength ;y ++){
         tempBt = enc(tempBt,secondKeyBt[y]);
       }
       for(z = 0;z < thirdLength ;z ++){
         tempBt = enc(tempBt,thirdKeyBt[z]);
       }
       encByte = tempBt;
     }else{
       if(firstKey != null && firstKey !="" && secondKey != null && secondKey != ""){
         var tempBt;
         var x,y;
         tempBt = bt;
         for(x = 0;x < firstLength ;x ++){
           tempBt = enc(tempBt,firstKeyBt[x]);
         }
         for(y = 0;y < secondLength ;y ++){
           tempBt = enc(tempBt,secondKeyBt[y]);
         }
         encByte = tempBt;
       }else{
         if(firstKey != null && firstKey !=""){
           var tempBt;
           var x = 0;
           tempBt = bt;
           for(x = 0;x < firstLength ;x ++){
             tempBt = enc(tempBt,firstKeyBt[x]);
           }
           encByte = tempBt;
         }
       }
     }
     encData = bt64ToHex(encByte);
   }else{
     var iterator = parseInt(leng/4);
     var remainder = leng%4;
     var i=0;
     for(i = 0;i < iterator;i++){
       var tempData = data.substring(i*4+0,i*4+4);
       var tempByte = strToBt(tempData);
       var encByte ;
       if(firstKey != null && firstKey !="" && secondKey != null && secondKey != "" && thirdKey != null && thirdKey != ""){
         var tempBt;
         var x,y,z;
         tempBt = tempByte;
         for(x = 0;x < firstLength ;x ++){
           tempBt = enc(tempBt,firstKeyBt[x]);
         }
         for(y = 0;y < secondLength ;y ++){
           tempBt = enc(tempBt,secondKeyBt[y]);
         }
         for(z = 0;z < thirdLength ;z ++){
           tempBt = enc(tempBt,thirdKeyBt[z]);
         }
         encByte = tempBt;
       }else{
         if(firstKey != null && firstKey !="" && secondKey != null && secondKey != ""){
           var tempBt;
           var x,y;
           tempBt = tempByte;
           for(x = 0;x < firstLength ;x ++){
             tempBt = enc(tempBt,firstKeyBt[x]);
           }
           for(y = 0;y < secondLength ;y ++){
             tempBt = enc(tempBt,secondKeyBt[y]);
           }
           encByte = tempBt;
         }else{
           if(firstKey != null && firstKey !=""){
             var tempBt;
             var x;
             tempBt = tempByte;
             for(x = 0;x < firstLength ;x ++){
               tempBt = enc(tempBt,firstKeyBt[x]);
             }
             encByte = tempBt;
           }
         }
       }
       encData += bt64ToHex(encByte);
     }
     if(remainder > 0){
       var remainderData = data.substring(iterator*4+0,leng);
       var tempByte = strToBt(remainderData);
       var encByte ;
       if(firstKey != null && firstKey !="" && secondKey != null && secondKey != "" && thirdKey != null && thirdKey != ""){
         var tempBt;
         var x,y,z;
         tempBt = tempByte;
         for(x = 0;x < firstLength ;x ++){
           tempBt = enc(tempBt,firstKeyBt[x]);
         }
         for(y = 0;y < secondLength ;y ++){
           tempBt = enc(tempBt,secondKeyBt[y]);
         }
         for(z = 0;z < thirdLength ;z ++){
           tempBt = enc(tempBt,thirdKeyBt[z]);
         }
         encByte = tempBt;
       }else{
         if(firstKey != null && firstKey !="" && secondKey != null && secondKey != ""){
           var tempBt;
           var x,y;
           tempBt = tempByte;
           for(x = 0;x < firstLength ;x ++){
             tempBt = enc(tempBt,firstKeyBt[x]);
           }
           for(y = 0;y < secondLength ;y ++){
             tempBt = enc(tempBt,secondKeyBt[y]);
           }
           encByte = tempBt;
         }else{
           if(firstKey != null && firstKey !=""){
             var tempBt;
             var x;
             tempBt = tempByte;
             for(x = 0;x < firstLength ;x ++){
               tempBt = enc(tempBt,firstKeyBt[x]);
             }
             encByte = tempBt;
           }
         }
       }
       encData += bt64ToHex(encByte);
     }
   }
 }
 return encData;
}

/*
* decrypt the encrypted string to the original string
*
* return  the original string
*/
/*解密的函数,三个密钥(解密密钥同加密密钥)*/
function strDecode(data,firstKey,secondKey,thirdKey){
 var leng = data.length;
 var decStr = "";
 var firstKeyBt,secondKeyBt,thirdKeyBt,firstLength,secondLength,thirdLength;
 if(firstKey != null && firstKey != ""){
   firstKeyBt = getKeyBytes(firstKey);
   firstLength = firstKeyBt.length;
 }
 if(secondKey != null && secondKey != ""){
   secondKeyBt = getKeyBytes(secondKey);
   secondLength = secondKeyBt.length;
 }
 if(thirdKey != null && thirdKey != ""){
   thirdKeyBt = getKeyBytes(thirdKey);
   thirdLength = thirdKeyBt.length;
 }
 var iterator = parseInt(leng/16);
 var i=0;
 for(i = 0;i < iterator;i++){
   var tempData = data.substring(i*16+0,i*16+16);
   var strByte = hexToBt64(tempData);
   var intByte = new Array(64);
   var j = 0;
   for(j = 0;j < 64; j++){
     intByte[j] = parseInt(strByte.substring(j,j+1));
   }
   var decByte;
   if(firstKey != null && firstKey !="" && secondKey != null && secondKey != "" && thirdKey != null && thirdKey != ""){
     var tempBt;
     var x,y,z;
     tempBt = intByte;
     for(x = thirdLength - 1;x >= 0;x --){
       tempBt = dec(tempBt,thirdKeyBt[x]);
     }
     for(y = secondLength - 1;y >= 0;y --){
       tempBt = dec(tempBt,secondKeyBt[y]);
     }
     for(z = firstLength - 1;z >= 0 ;z --){
       tempBt = dec(tempBt,firstKeyBt[z]);
     }
     decByte = tempBt;
   }else{
     if(firstKey != null && firstKey !="" && secondKey != null && secondKey != ""){
       var tempBt;
       var x,y,z;
       tempBt = intByte;
       for(x = secondLength - 1;x >= 0 ;x --){
         tempBt = dec(tempBt,secondKeyBt[x]);
       }
       for(y = firstLength - 1;y >= 0 ;y --){
         tempBt = dec(tempBt,firstKeyBt[y]);
       }
       decByte = tempBt;
     }else{
       if(firstKey != null && firstKey !=""){
         var tempBt;
         var x,y,z;
         tempBt = intByte;
         for(x = firstLength - 1;x >= 0 ;x --){
           tempBt = dec(tempBt,firstKeyBt[x]);
         }
         decByte = tempBt;
       }
     }
   }
   decStr += byteToString(decByte);
 }
 return decStr;
}
/*
* chang the string into the bit array
*
* return bit array(it's length % 64 = 0)
*/
function getKeyBytes(key){
 var keyBytes = new Array();
 var leng = key.length;
 var iterator = parseInt(leng/4);
 var remainder = leng%4;
 var i = 0;
 for(i = 0;i < iterator; i ++){
   keyBytes[i] = strToBt(key.substring(i*4+0,i*4+4));
 }
 if(remainder > 0){
   keyBytes[i] = strToBt(key.substring(i*4+0,leng));
 }
 return keyBytes;
}

/*
* chang the string(it's length <= 4) into the bit array
*
* return bit array(it's length = 64)
*/
function strToBt(str){
 var leng = str.length;
 var bt = new Array(64);
 if(leng < 4){
   var i=0,j=0,p=0,q=0;
   for(i = 0;i<leng;i++){
     var k = str.charCodeAt(i);
     for(j=0;j<16;j++){
       var pow=1,m=0;
       for(m=15;m>j;m--){
         pow *= 2;
       }
       bt[16*i+j]=parseInt(k/pow)%2;
     }
   }
   for(p = leng;p<4;p++){
     var k = 0;
     for(q=0;q<16;q++){
       var pow=1,m=0;
       for(m=15;m>q;m--){
         pow *= 2;
       }
       bt[16*p+q]=parseInt(k/pow)%2;
     }
   }
 }else{
   for(i = 0;i<4;i++){
     var k = str.charCodeAt(i);
     for(j=0;j<16;j++){
       var pow=1;
       for(m=15;m>j;m--){
         pow *= 2;
       }
       bt[16*i+j]=parseInt(k/pow)%2;
     }
   }
 }
 return bt;
}

/*
* chang the bit(it's length = 4) into the hex
*
* return hex
*/
function bt4ToHex(binary) {
 var hex;
 switch (binary) {
   case "0000" : hex = "0"; break;
   case "0001" : hex = "1"; break;
   case "0010" : hex = "2"; break;
   case "0011" : hex = "3"; break;
   case "0100" : hex = "4"; break;
   case "0101" : hex = "5"; break;
   case "0110" : hex = "6"; break;
   case "0111" : hex = "7"; break;
   case "1000" : hex = "8"; break;
   case "1001" : hex = "9"; break;
   case "1010" : hex = "A"; break;
   case "1011" : hex = "B"; break;
   case "1100" : hex = "C"; break;
   case "1101" : hex = "D"; break;
   case "1110" : hex = "E"; break;
   case "1111" : hex = "F"; break;
 }
 return hex;
}

/*
* chang the hex into the bit(it's length = 4)
*
* return the bit(it's length = 4)
*/
function hexToBt4(hex) {
 var binary;
 switch (hex) {
   case "0" : binary = "0000"; break;
   case "1" : binary = "0001"; break;
   case "2" : binary = "0010"; break;
   case "3" : binary = "0011"; break;
   case "4" : binary = "0100"; break;
   case "5" : binary = "0101"; break;
   case "6" : binary = "0110"; break;
   case "7" : binary = "0111"; break;
   case "8" : binary = "1000"; break;
   case "9" : binary = "1001"; break;
   case "A" : binary = "1010"; break;
   case "B" : binary = "1011"; break;
   case "C" : binary = "1100"; break;
   case "D" : binary = "1101"; break;
   case "E" : binary = "1110"; break;
   case "F" : binary = "1111"; break;
 }
 return binary;
}

/*
* chang the bit(it's length = 64) into the string
*
* return string
*/
function byteToString(byteData){
 var str="";
 for(i = 0;i<4;i++){
   var count=0;
   for(j=0;j<16;j++){
     var pow=1;
     for(m=15;m>j;m--){
       pow*=2;
     }
     count+=byteData[16*i+j]*pow;
   }
   if(count != 0){
     str+=String.fromCharCode(count);
   }
 }
 return str;
}

function bt64ToHex(byteData){
 var hex = "";
 for(i = 0;i<16;i++){
   var bt = "";
   for(j=0;j<4;j++){
     bt += byteData[i*4+j];
   }
   hex+=bt4ToHex(bt);
 }
 return hex;
}

function hexToBt64(hex){
 var binary = "";
 for(i = 0;i<16;i++){
   binary+=hexToBt4(hex.substring(i,i+1));
 }
 return binary;
}

/*
* the 64 bit des core arithmetic
*/

function enc(dataByte,keyByte){
 var keys = generateKeys(keyByte);
 var ipByte   = initPermute(dataByte);
 var ipLeft   = new Array(32);
 var ipRight  = new Array(32);
 var tempLeft = new Array(32);
 var i = 0,j = 0,k = 0,m = 0, n = 0;
 for(k = 0;k < 32;k ++){
   ipLeft[k] = ipByte[k];
   ipRight[k] = ipByte[32+k];
 }
 for(i = 0;i < 16;i ++){
   for(j = 0;j < 32;j ++){
     tempLeft[j] = ipLeft[j];
     ipLeft[j] = ipRight[j];
   }
   var key = new Array(48);
   for(m = 0;m < 48;m ++){
     key[m] = keys[i][m];
   }
   var  tempRight = xor(pPermute(sBoxPermute(xor(expandPermute(ipRight),key))), tempLeft);
   for(n = 0;n < 32;n ++){
     ipRight[n] = tempRight[n];
   }
 }


 var finalData =new Array(64);
 for(i = 0;i < 32;i ++){
   finalData[i] = ipRight[i];
   finalData[32+i] = ipLeft[i];
 }
 return finallyPermute(finalData);
}

function dec(dataByte,keyByte){
 var keys = generateKeys(keyByte);
 var ipByte   = initPermute(dataByte);
 var ipLeft   = new Array(32);
 var ipRight  = new Array(32);
 var tempLeft = new Array(32);
 var i = 0,j = 0,k = 0,m = 0, n = 0;
 for(k = 0;k < 32;k ++){
   ipLeft[k] = ipByte[k];
   ipRight[k] = ipByte[32+k];
 }
 for(i = 15;i >= 0;i --){
   for(j = 0;j < 32;j ++){
     tempLeft[j] = ipLeft[j];
     ipLeft[j] = ipRight[j];
   }
   var key = new Array(48);
   for(m = 0;m < 48;m ++){
     key[m] = keys[i][m];
   }

   var  tempRight = xor(pPermute(sBoxPermute(xor(expandPermute(ipRight),key))), tempLeft);
   for(n = 0;n < 32;n ++){
     ipRight[n] = tempRight[n];
   }
 }

 var finalData =new Array(64);
 for(i = 0;i < 32;i ++){
   finalData[i] = ipRight[i];
   finalData[32+i] = ipLeft[i];
 }
 return finallyPermute(finalData);
}

function initPermute(originalData){
 var ipByte = new Array(64);
 for (i = 0, m = 1, n = 0; i < 4; i++, m += 2, n += 2) {
   for (j = 7, k = 0; j >= 0; j--, k++) {
     ipByte[i * 8 + k] = originalData[j * 8 + m];
     ipByte[i * 8 + k + 32] = originalData[j * 8 + n];
   }
 }
 return ipByte;
}

function expandPermute(rightData){
 var epByte = new Array(48);
 for (i = 0; i < 8; i++) {
   if (i == 0) {
     epByte[i * 6 + 0] = rightData[31];
   } else {
     epByte[i * 6 + 0] = rightData[i * 4 - 1];
   }
   epByte[i * 6 + 1] = rightData[i * 4 + 0];
   epByte[i * 6 + 2] = rightData[i * 4 + 1];
   epByte[i * 6 + 3] = rightData[i * 4 + 2];
   epByte[i * 6 + 4] = rightData[i * 4 + 3];
   if (i == 7) {
     epByte[i * 6 + 5] = rightData[0];
   } else {
     epByte[i * 6 + 5] = rightData[i * 4 + 4];
   }
 }
 return epByte;
}

function xor(byteOne,byteTwo){
 var xorByte = new Array(byteOne.length);
 for(i = 0;i < byteOne.length; i ++){
   xorByte[i] = byteOne[i] ^ byteTwo[i];
 }
 return xorByte;
}

function sBoxPermute(expandByte){

   var sBoxByte = new Array(32);
   var binary = "";
   var s1 = [
       [14, 4, 13, 1, 2, 15, 11, 8, 3, 10, 6, 12, 5, 9, 0, 7],
       [0, 15, 7, 4, 14, 2, 13, 1, 10, 6, 12, 11, 9, 5, 3, 8],
       [4, 1, 14, 8, 13, 6, 2, 11, 15, 12, 9, 7, 3, 10, 5, 0],
       [15, 12, 8, 2, 4, 9, 1, 7, 5, 11, 3, 14, 10, 0, 6, 13 ]];
       /* Table - s2 */
   var s2 = [
       [15, 1, 8, 14, 6, 11, 3, 4, 9, 7, 2, 13, 12, 0, 5, 10],
       [3, 13, 4, 7, 15, 2, 8, 14, 12, 0, 1, 10, 6, 9, 11, 5],
       [0, 14, 7, 11, 10, 4, 13, 1, 5, 8, 12, 6, 9, 3, 2, 15],
       [13, 8, 10, 1, 3, 15, 4, 2, 11, 6, 7, 12, 0, 5, 14, 9 ]];
       /* Table - s3 */
   var s3= [
       [10, 0, 9, 14, 6, 3, 15, 5, 1, 13, 12, 7, 11, 4, 2, 8],
       [13, 7, 0, 9, 3, 4, 6, 10, 2, 8, 5, 14, 12, 11, 15, 1],
       [13, 6, 4, 9, 8, 15, 3, 0, 11, 1, 2, 12, 5, 10, 14, 7],
       [1, 10, 13, 0, 6, 9, 8, 7, 4, 15, 14, 3, 11, 5, 2, 12 ]];
       /* Table - s4 */
   var s4 = [
       [7, 13, 14, 3, 0, 6, 9, 10, 1, 2, 8, 5, 11, 12, 4, 15],
       [13, 8, 11, 5, 6, 15, 0, 3, 4, 7, 2, 12, 1, 10, 14, 9],
       [10, 6, 9, 0, 12, 11, 7, 13, 15, 1, 3, 14, 5, 2, 8, 4],
       [3, 15, 0, 6, 10, 1, 13, 8, 9, 4, 5, 11, 12, 7, 2, 14 ]];
       /* Table - s5 */
   var s5 = [
       [2, 12, 4, 1, 7, 10, 11, 6, 8, 5, 3, 15, 13, 0, 14, 9],
       [14, 11, 2, 12, 4, 7, 13, 1, 5, 0, 15, 10, 3, 9, 8, 6],
       [4, 2, 1, 11, 10, 13, 7, 8, 15, 9, 12, 5, 6, 3, 0, 14],
       [11, 8, 12, 7, 1, 14, 2, 13, 6, 15, 0, 9, 10, 4, 5, 3 ]];
       /* Table - s6 */
   var s6 = [
       [12, 1, 10, 15, 9, 2, 6, 8, 0, 13, 3, 4, 14, 7, 5, 11],
       [10, 15, 4, 2, 7, 12, 9, 5, 6, 1, 13, 14, 0, 11, 3, 8],
       [9, 14, 15, 5, 2, 8, 12, 3, 7, 0, 4, 10, 1, 13, 11, 6],
       [4, 3, 2, 12, 9, 5, 15, 10, 11, 14, 1, 7, 6, 0, 8, 13 ]];
       /* Table - s7 */
   var s7 = [
       [4, 11, 2, 14, 15, 0, 8, 13, 3, 12, 9, 7, 5, 10, 6, 1],
       [13, 0, 11, 7, 4, 9, 1, 10, 14, 3, 5, 12, 2, 15, 8, 6],
       [1, 4, 11, 13, 12, 3, 7, 14, 10, 15, 6, 8, 0, 5, 9, 2],
       [6, 11, 13, 8, 1, 4, 10, 7, 9, 5, 0, 15, 14, 2, 3, 12]];
       /* Table - s8 */
   var s8 = [
       [13, 2, 8, 4, 6, 15, 11, 1, 10, 9, 3, 14, 5, 0, 12, 7],
       [1, 15, 13, 8, 10, 3, 7, 4, 12, 5, 6, 11, 0, 14, 9, 2],
       [7, 11, 4, 1, 9, 12, 14, 2, 0, 6, 10, 13, 15, 3, 5, 8],
       [2, 1, 14, 7, 4, 10, 8, 13, 15, 12, 9, 0, 3, 5, 6, 11]];

   for(m=0;m<8;m++){
   var i=0,j=0;
   i = expandByte[m*6+0]*2+expandByte[m*6+5];
   j = expandByte[m * 6 + 1] * 2 * 2 * 2
     + expandByte[m * 6 + 2] * 2* 2
     + expandByte[m * 6 + 3] * 2
     + expandByte[m * 6 + 4];
   switch (m) {
     case 0 :
       binary = getBoxBinary(s1[i][j]);
       break;
     case 1 :
       binary = getBoxBinary(s2[i][j]);
       break;
     case 2 :
       binary = getBoxBinary(s3[i][j]);
       break;
     case 3 :
       binary = getBoxBinary(s4[i][j]);
       break;
     case 4 :
       binary = getBoxBinary(s5[i][j]);
       break;
     case 5 :
       binary = getBoxBinary(s6[i][j]);
       break;
     case 6 :
       binary = getBoxBinary(s7[i][j]);
       break;
     case 7 :
       binary = getBoxBinary(s8[i][j]);
       break;
   }
   sBoxByte[m*4+0] = parseInt(binary.substring(0,1));
   sBoxByte[m*4+1] = parseInt(binary.substring(1,2));
   sBoxByte[m*4+2] = parseInt(binary.substring(2,3));
   sBoxByte[m*4+3] = parseInt(binary.substring(3,4));
 }
 return sBoxByte;
}

function pPermute(sBoxByte){
 var pBoxPermute = new Array(32);
 pBoxPermute[ 0] = sBoxByte[15];
 pBoxPermute[ 1] = sBoxByte[ 6];
 pBoxPermute[ 2] = sBoxByte[19];
 pBoxPermute[ 3] = sBoxByte[20];
 pBoxPermute[ 4] = sBoxByte[28];
 pBoxPermute[ 5] = sBoxByte[11];
 pBoxPermute[ 6] = sBoxByte[27];
 pBoxPermute[ 7] = sBoxByte[16];
 pBoxPermute[ 8] = sBoxByte[ 0];
 pBoxPermute[ 9] = sBoxByte[14];
 pBoxPermute[10] = sBoxByte[22];
 pBoxPermute[11] = sBoxByte[25];
 pBoxPermute[12] = sBoxByte[ 4];
 pBoxPermute[13] = sBoxByte[17];
 pBoxPermute[14] = sBoxByte[30];
 pBoxPermute[15] = sBoxByte[ 9];
 pBoxPermute[16] = sBoxByte[ 1];
 pBoxPermute[17] = sBoxByte[ 7];
 pBoxPermute[18] = sBoxByte[23];
 pBoxPermute[19] = sBoxByte[13];
 pBoxPermute[20] = sBoxByte[31];
 pBoxPermute[21] = sBoxByte[26];
 pBoxPermute[22] = sBoxByte[ 2];
 pBoxPermute[23] = sBoxByte[ 8];
 pBoxPermute[24] = sBoxByte[18];
 pBoxPermute[25] = sBoxByte[12];
 pBoxPermute[26] = sBoxByte[29];
 pBoxPermute[27] = sBoxByte[ 5];
 pBoxPermute[28] = sBoxByte[21];
 pBoxPermute[29] = sBoxByte[10];
 pBoxPermute[30] = sBoxByte[ 3];
 pBoxPermute[31] = sBoxByte[24];
 return pBoxPermute;
}

function finallyPermute(endByte){
 var fpByte = new Array(64);
 fpByte[ 0] = endByte[39];
 fpByte[ 1] = endByte[ 7];
 fpByte[ 2] = endByte[47];
 fpByte[ 3] = endByte[15];
 fpByte[ 4] = endByte[55];
 fpByte[ 5] = endByte[23];
 fpByte[ 6] = endByte[63];
 fpByte[ 7] = endByte[31];
 fpByte[ 8] = endByte[38];
 fpByte[ 9] = endByte[ 6];
 fpByte[10] = endByte[46];
 fpByte[11] = endByte[14];
 fpByte[12] = endByte[54];
 fpByte[13] = endByte[22];
 fpByte[14] = endByte[62];
 fpByte[15] = endByte[30];
 fpByte[16] = endByte[37];
 fpByte[17] = endByte[ 5];
 fpByte[18] = endByte[45];
 fpByte[19] = endByte[13];
 fpByte[20] = endByte[53];
 fpByte[21] = endByte[21];
 fpByte[22] = endByte[61];
 fpByte[23] = endByte[29];
 fpByte[24] = endByte[36];
 fpByte[25] = endByte[ 4];
 fpByte[26] = endByte[44];
 fpByte[27] = endByte[12];
 fpByte[28] = endByte[52];
 fpByte[29] = endByte[20];
 fpByte[30] = endByte[60];
 fpByte[31] = endByte[28];
 fpByte[32] = endByte[35];
 fpByte[33] = endByte[ 3];
 fpByte[34] = endByte[43];
 fpByte[35] = endByte[11];
 fpByte[36] = endByte[51];
 fpByte[37] = endByte[19];
 fpByte[38] = endByte[59];
 fpByte[39] = endByte[27];
 fpByte[40] = endByte[34];
 fpByte[41] = endByte[ 2];
 fpByte[42] = endByte[42];
 fpByte[43] = endByte[10];
 fpByte[44] = endByte[50];
 fpByte[45] = endByte[18];
 fpByte[46] = endByte[58];
 fpByte[47] = endByte[26];
 fpByte[48] = endByte[33];
 fpByte[49] = endByte[ 1];
 fpByte[50] = endByte[41];
 fpByte[51] = endByte[ 9];
 fpByte[52] = endByte[49];
 fpByte[53] = endByte[17];
 fpByte[54] = endByte[57];
 fpByte[55] = endByte[25];
 fpByte[56] = endByte[32];
 fpByte[57] = endByte[ 0];
 fpByte[58] = endByte[40];
 fpByte[59] = endByte[ 8];
 fpByte[60] = endByte[48];
 fpByte[61] = endByte[16];
 fpByte[62] = endByte[56];
 fpByte[63] = endByte[24];
 return fpByte;
}

function getBoxBinary(i) {
 var binary = "";
 switch (i) {
   case  0 :binary = "0000";break;
   case  1 :binary = "0001";break;
   case  2 :binary = "0010";break;
   case  3 :binary = "0011";break;
   case  4 :binary = "0100";break;
   case  5 :binary = "0101";break;
   case  6 :binary = "0110";break;
   case  7 :binary = "0111";break;
   case  8 :binary = "1000";break;
   case  9 :binary = "1001";break;
   case 10 :binary = "1010";break;
   case 11 :binary = "1011";break;
   case 12 :binary = "1100";break;
   case 13 :binary = "1101";break;
   case 14 :binary = "1110";break;
   case 15 :binary = "1111";break;
 }
 return binary;
}
/*
* generate 16 keys for xor
*
*/

function generateKeys(keyByte){
 var key   = new Array(56);
 var keys = new Array();

 keys[ 0] = new Array();
 keys[ 1] = new Array();
 keys[ 2] = new Array();
 keys[ 3] = new Array();
 keys[ 4] = new Array();
 keys[ 5] = new Array();
 keys[ 6] = new Array();
 keys[ 7] = new Array();
 keys[ 8] = new Array();
 keys[ 9] = new Array();
 keys[10] = new Array();
 keys[11] = new Array();
 keys[12] = new Array();
 keys[13] = new Array();
 keys[14] = new Array();
 keys[15] = new Array();
 var loop = [1,1,2,2,2,2,2,2,1,2,2,2,2,2,2,1];

 for(i=0;i<7;i++){
   for(j=0,k=7;j<8;j++,k--){
     key[i*8+j]=keyByte[8*k+i];
   }
 }

 var i = 0;
 for(i = 0;i < 16;i ++){
   var tempLeft=0;
   var tempRight=0;
   for(j = 0; j < loop[i];j ++){
     tempLeft = key[0];
     tempRight = key[28];
     for(k = 0;k < 27 ;k ++){
       key[k] = key[k+1];
       key[28+k] = key[29+k];
     }
     key[27]=tempLeft;
     key[55]=tempRight;
   }
   var tempKey = new Array(48);
   tempKey[ 0] = key[13];
   tempKey[ 1] = key[16];
   tempKey[ 2] = key[10];
   tempKey[ 3] = key[23];
   tempKey[ 4] = key[ 0];
   tempKey[ 5] = key[ 4];
   tempKey[ 6] = key[ 2];
   tempKey[ 7] = key[27];
   tempKey[ 8] = key[14];
   tempKey[ 9] = key[ 5];
   tempKey[10] = key[20];
   tempKey[11] = key[ 9];
   tempKey[12] = key[22];
   tempKey[13] = key[18];
   tempKey[14] = key[11];
   tempKey[15] = key[ 3];
   tempKey[16] = key[25];
   tempKey[17] = key[ 7];
   tempKey[18] = key[15];
   tempKey[19] = key[ 6];
   tempKey[20] = key[26];
   tempKey[21] = key[19];
   tempKey[22] = key[12];
   tempKey[23] = key[ 1];
   tempKey[24] = key[40];
   tempKey[25] = key[51];
   tempKey[26] = key[30];
   tempKey[27] = key[36];
   tempKey[28] = key[46];
   tempKey[29] = key[54];
   tempKey[30] = key[29];
   tempKey[31] = key[39];
   tempKey[32] = key[50];
   tempKey[33] = key[44];
   tempKey[34] = key[32];
   tempKey[35] = key[47];
   tempKey[36] = key[43];
   tempKey[37] = key[48];
   tempKey[38] = key[38];
   tempKey[39] = key[55];
   tempKey[40] = key[33];
   tempKey[41] = key[52];
   tempKey[42] = key[45];
   tempKey[43] = key[41];
   tempKey[44] = key[49];
   tempKey[45] = key[35];
   tempKey[46] = key[28];
   tempKey[47] = key[31];
   switch(i){
     case 0: for(m=0;m < 48 ;m++){ keys[ 0][m] = tempKey[m]; } break;
     case 1: for(m=0;m < 48 ;m++){ keys[ 1][m] = tempKey[m]; } break;
     case 2: for(m=0;m < 48 ;m++){ keys[ 2][m] = tempKey[m]; } break;
     case 3: for(m=0;m < 48 ;m++){ keys[ 3][m] = tempKey[m]; } break;
     case 4: for(m=0;m < 48 ;m++){ keys[ 4][m] = tempKey[m]; } break;
     case 5: for(m=0;m < 48 ;m++){ keys[ 5][m] = tempKey[m]; } break;
     case 6: for(m=0;m < 48 ;m++){ keys[ 6][m] = tempKey[m]; } break;
     case 7: for(m=0;m < 48 ;m++){ keys[ 7][m] = tempKey[m]; } break;
     case 8: for(m=0;m < 48 ;m++){ keys[ 8][m] = tempKey[m]; } break;
     case 9: for(m=0;m < 48 ;m++){ keys[ 9][m] = tempKey[m]; } break;

     case 10: for(m=0;m < 48 ;m++){ keys[10][m] = tempKey[m]; } break;
     case 11: for(m=0;m < 48 ;m++){ keys[11][m] = tempKey[m]; } break;
     case 12: for(m=0;m < 48 ;m++){ keys[12][m] = tempKey[m]; } break;
     case 13: for(m=0;m < 48 ;m++){ keys[13][m] = tempKey[m]; } break;
     case 14: for(m=0;m < 48 ;m++){ keys[14][m] = tempKey[m]; } break;
     case 15: for(m=0;m < 48 ;m++){ keys[15][m] = tempKey[m]; } break;
   }
 }
 return keys;
}
function strDecode(data,firstKey,secondKey,thirdKey){
 var leng = data.length;
 var decStr = "";
 var firstKeyBt,secondKeyBt,thirdKeyBt,firstLength,secondLength,thirdLength;
 if(firstKey != null && firstKey != ""){
   firstKeyBt = getKeyBytes(firstKey);
   firstLength = firstKeyBt.length;
 }
 if(secondKey != null && secondKey != ""){
   secondKeyBt = getKeyBytes(secondKey);
   secondLength = secondKeyBt.length;
 }
 if(thirdKey != null && thirdKey != ""){
   thirdKeyBt = getKeyBytes(thirdKey);
   thirdLength = thirdKeyBt.length;
 }
 var iterator = parseInt(leng/16);
 var i=0;
 for(i = 0;i < iterator;i++){
   var tempData = data.substring(i*16+0,i*16+16);
   var strByte = hexToBt64(tempData);
   var intByte = new Array(64);
   var j = 0;
   for(j = 0;j < 64; j++){
     intByte[j] = parseInt(strByte.substring(j,j+1));
   }
   var decByte;
   if(firstKey != null && firstKey !="" && secondKey != null && secondKey != "" && thirdKey != null && thirdKey != ""){
     var tempBt;
     var x,y,z;
     tempBt = intByte;
     for(x = thirdLength - 1;x >= 0;x --){
       tempBt = dec(tempBt,thirdKeyBt[x]);
     }
     for(y = secondLength - 1;y >= 0;y --){
       tempBt = dec(tempBt,secondKeyBt[y]);
     }
     for(z = firstLength - 1;z >= 0 ;z --){
       tempBt = dec(tempBt,firstKeyBt[z]);
     }
     decByte = tempBt;
   }else{
     if(firstKey != null && firstKey !="" && secondKey != null && secondKey != ""){
       var tempBt;
       var x,y,z;
       tempBt = intByte;
       for(x = secondLength - 1;x >= 0 ;x --){
         tempBt = dec(tempBt,secondKeyBt[x]);
       }
       for(y = firstLength - 1;y >= 0 ;y --){
         tempBt = dec(tempBt,firstKeyBt[y]);
       }
       decByte = tempBt;
     }else{
       if(firstKey != null && firstKey !=""){
         var tempBt;
         var x,y,z;
         tempBt = intByte;
         for(x = firstLength - 1;x >= 0 ;x --){
           tempBt = dec(tempBt,firstKeyBt[x]);
         }
         decByte = tempBt;
       }
     }
   }
   decStr += byteToString(decByte);
 }
 return decStr;
}
/*end*/


/*测试调用,源文件中没有*/
en = strEncode('34a6338e68604237892c09b521fa97c5', '2024-08-15 17', 'cebpubservice', 'iiss')
console.log(en)
console.log(en.length)

 即下面这个Encrypt.js文件的全部内容,IP被封了,调用位置可以搜strEncode

 注:使用代理ip,会封IP

爬虫(Web Crawler)是一种自动化程序,用于从互联网上收集信息。其主要功能是访问网页、提取数据并存储,以便后续分析或展示。爬虫通常由搜索引擎、数据挖掘工具、监测系统等应用于网络数据抓取的场景。 爬虫的工作流程包括以下几个关键步骤: URL收集: 爬虫从一个或多个初始URL开始,递归或迭代地发现新的URL,构建一个URL队列。这些URL可以通过链接分析、站点地图、搜索引擎等方式获取。 请求网页: 爬虫使用HTTP或其他协议向目标URL发起请求,获取网页的HTML内容。这通常通过HTTP请求库实现,如Python中的Requests库。 解析内容: 爬虫对获取的HTML进行解析,提取有用的信息。常用的解析工具有正则表达式、XPath、Beautiful Soup等。这些工具帮助爬虫定位和提取目标数据,如文本、图片、链接等。 数据存储: 爬虫将提取的数据存储到数据库、文件或其他存储介质中,以备后续分析或展示。常用的存储形式包括关系型数据库、NoSQL数据库、JSON文件等。 遵守规则: 为避免对网站造成过大负担或触发反爬虫机制,爬虫需要遵守网站的robots.txt协议,限制访问频率和深度,并模拟人类访问行为,如设置User-Agent。 反爬虫应对: 由于爬虫的存在,一些网站采取了反爬虫措施,如验证码、IP封锁等。爬虫工程师需要设计相应的策略来应对这些挑战。 爬虫在各个领域都有广泛的应用,包括搜索引擎索引、数据挖掘、价格监测、新闻聚合等。然而,使用爬虫需要遵守法律和伦理规范,尊重网站的使用政策,并确保对被访问网站的服务器负责。
ERR_INSUFFICIENT_RESOURCES是一个常见的网络错误,它表示请求的资源超过了服务器的处理能力或者网络连接的限制。这个错误通常与爬虫相关,因为爬虫在访问网页时会发送大量的请求,如果服务器无法处理这些请求或者网络连接不稳定,就会导致出现ERR_INSUFFICIENT_RESOURCES错误。 造成ERR_INSUFFICIENT_RESOURCES错误的原因可能有以下几种: 1. 服务器负载过高:当服务器同时处理大量请求时,可能会超出其处理能力,导致资源不足。这可能是因为网站流量激增或者服务器配置不足所致。 2. 网络连接问题:如果网络连接不稳定或者带宽有限,服务器可能无法及时响应请求,从而导致资源不足的错误。 3. IP被封禁:有些网站会对频繁访问或者异常访问行为进行监控,并可能封禁对应的IP地址。如果你的爬虫被封禁了,就会导致资源不足的错误。 解决ERR_INSUFFICIENT_RESOURCES错误的方法包括: 1. 降低请求频率:可以通过减少请求的频率来减轻服务器的负载。可以增加请求之间的时间间隔,或者使用并发控制机制来限制同时发送的请求数量。 2. 增加延迟时间:可以在爬虫代码中增加延迟时间,让爬虫在发送请求之间等待一段时间,以减轻服务器的负载。 3. 使用代理IP:如果你的IP被封禁了,可以考虑使用代理IP来隐藏真实的IP地址,以避免被封禁。 4. 增加网络带宽:如果你有权限控制网络连接的配置,可以尝试增加网络带宽来提高网络连接的稳定性和响应速度。 5. 联系网站管理员:如果以上方法都无效,可以尝试联系网站管理员,了解是否有其他限制或者解决方案。
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值